130e7468fSPeter Dunlap /* 230e7468fSPeter Dunlap * CDDL HEADER START 330e7468fSPeter Dunlap * 430e7468fSPeter Dunlap * The contents of this file are subject to the terms of the 530e7468fSPeter Dunlap * Common Development and Distribution License (the "License"). 630e7468fSPeter Dunlap * You may not use this file except in compliance with the License. 730e7468fSPeter Dunlap * 830e7468fSPeter Dunlap * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 930e7468fSPeter Dunlap * or http://www.opensolaris.org/os/licensing. 1030e7468fSPeter Dunlap * See the License for the specific language governing permissions 1130e7468fSPeter Dunlap * and limitations under the License. 1230e7468fSPeter Dunlap * 1330e7468fSPeter Dunlap * When distributing Covered Code, include this CDDL HEADER in each 1430e7468fSPeter Dunlap * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 1530e7468fSPeter Dunlap * If applicable, add the following below this CDDL HEADER, with the 1630e7468fSPeter Dunlap * fields enclosed by brackets "[]" replaced with your own identifying 1730e7468fSPeter Dunlap * information: Portions Copyright [yyyy] [name of copyright owner] 1830e7468fSPeter Dunlap * 1930e7468fSPeter Dunlap * CDDL HEADER END 2030e7468fSPeter Dunlap */ 2130e7468fSPeter Dunlap /* 2230e7468fSPeter Dunlap * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 2330e7468fSPeter Dunlap * Use is subject to license terms. 2430e7468fSPeter Dunlap */ 2530e7468fSPeter Dunlap 2630e7468fSPeter Dunlap #include <sys/types.h> 2730e7468fSPeter Dunlap #include <sys/ddi.h> 2830e7468fSPeter Dunlap #include <sys/types.h> 2930e7468fSPeter Dunlap #include <sys/socket.h> 3030e7468fSPeter Dunlap #include <netinet/in.h> 3130e7468fSPeter Dunlap #include <sys/sunddi.h> 3230e7468fSPeter Dunlap #include <sys/sdt.h> 3330e7468fSPeter Dunlap #include <sys/ib/ibtl/ibti.h> 3430e7468fSPeter Dunlap #include <sys/ib/ibtl/ibtl_types.h> 3530e7468fSPeter Dunlap 3630e7468fSPeter Dunlap #include <sys/ib/clients/iser/iser.h> 3730e7468fSPeter Dunlap 3830e7468fSPeter Dunlap /* 3930e7468fSPeter Dunlap * iser_cq.c 4030e7468fSPeter Dunlap * Routines for completion queue handlers for iSER. 4130e7468fSPeter Dunlap */ 4230e7468fSPeter Dunlap static void iser_msg_handle(iser_chan_t *chan, iser_msg_t *msg); 4330e7468fSPeter Dunlap int iser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg); 4430e7468fSPeter Dunlap static int iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl, 4530e7468fSPeter Dunlap iser_chan_t *iser_chan); 4630e7468fSPeter Dunlap static int iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, 4730e7468fSPeter Dunlap iser_chan_t *iser_chan); 4830e7468fSPeter Dunlap 4930e7468fSPeter Dunlap void 5030e7468fSPeter Dunlap iser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl, void *arg) 5130e7468fSPeter Dunlap { 5230e7468fSPeter Dunlap iser_chan_t *iser_chan; 5330e7468fSPeter Dunlap ibt_status_t status; 5430e7468fSPeter Dunlap 5530e7468fSPeter Dunlap iser_chan = (iser_chan_t *)arg; 5630e7468fSPeter Dunlap 57*a82ec3cfSPriya Krishnan /* 58*a82ec3cfSPriya Krishnan * Poll for work request completion while successful. If the 59*a82ec3cfSPriya Krishnan * queue empties or otherwise becomes invalid, stop polling. 60*a82ec3cfSPriya Krishnan */ 6130e7468fSPeter Dunlap do { 6230e7468fSPeter Dunlap status = iser_ib_poll_send_completions(cq_hdl, iser_chan); 63*a82ec3cfSPriya Krishnan } while (status == IBT_SUCCESS); 6430e7468fSPeter Dunlap 65*a82ec3cfSPriya Krishnan if (status == IBT_CQ_EMPTY) { 66*a82ec3cfSPriya Krishnan /* We've emptied the CQ, rearm it before we're done here */ 67*a82ec3cfSPriya Krishnan status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION); 68*a82ec3cfSPriya Krishnan if (status != IBT_SUCCESS) { 69*a82ec3cfSPriya Krishnan /* Unexpected error */ 70*a82ec3cfSPriya Krishnan ISER_LOG(CE_NOTE, "iser_ib_sendcq_handler: " 71*a82ec3cfSPriya Krishnan "ibt_enable_cq_notify error (%d)", status); 72*a82ec3cfSPriya Krishnan return; 73*a82ec3cfSPriya Krishnan } 7430e7468fSPeter Dunlap 75*a82ec3cfSPriya Krishnan /* Now, check for more completions after the rearm */ 76*a82ec3cfSPriya Krishnan do { 77*a82ec3cfSPriya Krishnan status = iser_ib_poll_send_completions( 78*a82ec3cfSPriya Krishnan cq_hdl, iser_chan); 79*a82ec3cfSPriya Krishnan } while (status == IBT_SUCCESS); 80*a82ec3cfSPriya Krishnan } 8130e7468fSPeter Dunlap } 8230e7468fSPeter Dunlap 8330e7468fSPeter Dunlap static int 8430e7468fSPeter Dunlap iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan) 8530e7468fSPeter Dunlap { 8630e7468fSPeter Dunlap ibt_wc_t wc[ISER_IB_SCQ_POLL_MAX]; 8730e7468fSPeter Dunlap ibt_wrid_t wrid; 8830e7468fSPeter Dunlap idm_buf_t *idb = NULL; 8930e7468fSPeter Dunlap idm_task_t *idt = NULL; 9030e7468fSPeter Dunlap iser_wr_t *wr = NULL; 9130e7468fSPeter Dunlap int i; 9230e7468fSPeter Dunlap uint_t npoll = 0; 9330e7468fSPeter Dunlap ibt_status_t status; 9430e7468fSPeter Dunlap iser_conn_t *iser_conn; 9530e7468fSPeter Dunlap idm_status_t idm_status; 96a668b114SPriya Krishnan iser_mr_t *mr; 9730e7468fSPeter Dunlap 9830e7468fSPeter Dunlap iser_conn = iser_chan->ic_conn; 9930e7468fSPeter Dunlap 100*a82ec3cfSPriya Krishnan /* Poll ISER_IB_SCQ_POLL_MAX completions from the CQ */ 10130e7468fSPeter Dunlap status = ibt_poll_cq(cq_hdl, wc, ISER_IB_SCQ_POLL_MAX, &npoll); 10230e7468fSPeter Dunlap 10330e7468fSPeter Dunlap if (status != IBT_SUCCESS) { 10430e7468fSPeter Dunlap if (status != IBT_CQ_EMPTY) { 10530e7468fSPeter Dunlap /* Unexpected error */ 10630e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_sendcq_handler: ibt_poll_cq " 107*a82ec3cfSPriya Krishnan "unexpected error (%d)", status); 10830e7468fSPeter Dunlap } 10930e7468fSPeter Dunlap /* CQ is empty. Either way, move along... */ 11030e7468fSPeter Dunlap return (status); 11130e7468fSPeter Dunlap } 11230e7468fSPeter Dunlap 11330e7468fSPeter Dunlap /* 11430e7468fSPeter Dunlap * Handle each of the completions we've polled 11530e7468fSPeter Dunlap */ 11630e7468fSPeter Dunlap for (i = 0; i < npoll; i++) { 11730e7468fSPeter Dunlap 11830e7468fSPeter Dunlap DTRACE_PROBE3(iser__send__cqe, iser_chan_t *, iser_chan, 11930e7468fSPeter Dunlap ibt_wc_t *, &wc[i], ibt_wc_status_t, wc[i].wc_status); 12030e7468fSPeter Dunlap 12130e7468fSPeter Dunlap /* Grab the wrid of the completion */ 12230e7468fSPeter Dunlap wrid = wc[i].wc_id; 12330e7468fSPeter Dunlap 12430e7468fSPeter Dunlap /* Decrement this channel's SQ posted count */ 12530e7468fSPeter Dunlap mutex_enter(&iser_chan->ic_sq_post_lock); 12630e7468fSPeter Dunlap iser_chan->ic_sq_post_count--; 12730e7468fSPeter Dunlap mutex_exit(&iser_chan->ic_sq_post_lock); 12830e7468fSPeter Dunlap 12930e7468fSPeter Dunlap /* Pull in the wr handle */ 13030e7468fSPeter Dunlap wr = (iser_wr_t *)(uintptr_t)wrid; 13130e7468fSPeter Dunlap ASSERT(wr != NULL); 13230e7468fSPeter Dunlap 13330e7468fSPeter Dunlap /* Set an idm_status for return to IDM */ 13430e7468fSPeter Dunlap idm_status = (wc[i].wc_status == IBT_WC_SUCCESS) ? 13530e7468fSPeter Dunlap IDM_STATUS_SUCCESS : IDM_STATUS_FAIL; 13630e7468fSPeter Dunlap 13730e7468fSPeter Dunlap /* 13830e7468fSPeter Dunlap * A non-success status here indicates the QP went 13930e7468fSPeter Dunlap * into an error state while this WR was being 14030e7468fSPeter Dunlap * processed. This can also happen when the 14130e7468fSPeter Dunlap * channel is closed on the remote end. Clean up 14230e7468fSPeter Dunlap * the resources, then push CE_TRANSPORT_FAIL 14330e7468fSPeter Dunlap * into IDM. 14430e7468fSPeter Dunlap */ 14530e7468fSPeter Dunlap if (wc[i].wc_status != IBT_WC_SUCCESS) { 14630e7468fSPeter Dunlap /* 14730e7468fSPeter Dunlap * Free the resources attached to this 14830e7468fSPeter Dunlap * completion. 14930e7468fSPeter Dunlap */ 15030e7468fSPeter Dunlap if (wr->iw_msg != NULL) { 15130e7468fSPeter Dunlap /* Free iser_msg handle */ 15230e7468fSPeter Dunlap iser_msg_free(wr->iw_msg); 15330e7468fSPeter Dunlap } 15430e7468fSPeter Dunlap 15530e7468fSPeter Dunlap if (wr->iw_pdu != NULL) { 15630e7468fSPeter Dunlap /* Complete the PDU */ 15730e7468fSPeter Dunlap idm_pdu_complete(wr->iw_pdu, idm_status); 15830e7468fSPeter Dunlap } 15930e7468fSPeter Dunlap 16030e7468fSPeter Dunlap if (wr->iw_buf != NULL) { 16130e7468fSPeter Dunlap /* Invoke buffer callback */ 16230e7468fSPeter Dunlap idb = wr->iw_buf; 163a668b114SPriya Krishnan mr = ((iser_buf_t *) 164a668b114SPriya Krishnan idb->idb_buf_private)->iser_mr; 16530e7468fSPeter Dunlap #ifdef DEBUG 16630e7468fSPeter Dunlap bcopy(&wc[i], 16730e7468fSPeter Dunlap &((iser_buf_t *)idb->idb_buf_private)-> 16830e7468fSPeter Dunlap buf_wc, sizeof (ibt_wc_t)); 16930e7468fSPeter Dunlap #endif 17030e7468fSPeter Dunlap idt = idb->idb_task_binding; 17130e7468fSPeter Dunlap mutex_enter(&idt->idt_mutex); 17230e7468fSPeter Dunlap if (wr->iw_type == ISER_WR_RDMAW) { 173a668b114SPriya Krishnan DTRACE_ISCSI_8(xfer__done, 174a668b114SPriya Krishnan idm_conn_t *, idt->idt_ic, 175a668b114SPriya Krishnan uintptr_t, idb->idb_buf, 176a668b114SPriya Krishnan uint32_t, idb->idb_bufoffset, 177a668b114SPriya Krishnan uint64_t, mr->is_mrva, uint32_t, 0, 178a668b114SPriya Krishnan uint32_t, mr->is_mrrkey, 179a668b114SPriya Krishnan uint32_t, idb->idb_xfer_len, 180a668b114SPriya Krishnan int, XFER_BUF_TX_TO_INI); 18130e7468fSPeter Dunlap idm_buf_tx_to_ini_done(idt, idb, 18230e7468fSPeter Dunlap IDM_STATUS_FAIL); 18330e7468fSPeter Dunlap } else { /* ISER_WR_RDMAR */ 184a668b114SPriya Krishnan DTRACE_ISCSI_8(xfer__done, 185a668b114SPriya Krishnan idm_conn_t *, idt->idt_ic, 186a668b114SPriya Krishnan uintptr_t, idb->idb_buf, 187a668b114SPriya Krishnan uint32_t, idb->idb_bufoffset, 188a668b114SPriya Krishnan uint64_t, mr->is_mrva, uint32_t, 0, 189a668b114SPriya Krishnan uint32_t, mr->is_mrrkey, 190a668b114SPriya Krishnan uint32_t, idb->idb_xfer_len, 191a668b114SPriya Krishnan int, XFER_BUF_RX_FROM_INI); 19230e7468fSPeter Dunlap idm_buf_rx_from_ini_done(idt, idb, 19330e7468fSPeter Dunlap IDM_STATUS_FAIL); 19430e7468fSPeter Dunlap } 19530e7468fSPeter Dunlap } 19630e7468fSPeter Dunlap 19730e7468fSPeter Dunlap /* Free the iser wr handle */ 19830e7468fSPeter Dunlap iser_wr_free(wr); 19930e7468fSPeter Dunlap 20030e7468fSPeter Dunlap /* 20130e7468fSPeter Dunlap * Tell IDM that the channel has gone down, 20230e7468fSPeter Dunlap * unless he already knows. 20330e7468fSPeter Dunlap */ 20430e7468fSPeter Dunlap mutex_enter(&iser_conn->ic_lock); 20530e7468fSPeter Dunlap switch (iser_conn->ic_stage) { 20630e7468fSPeter Dunlap case ISER_CONN_STAGE_IC_DISCONNECTED: 20730e7468fSPeter Dunlap case ISER_CONN_STAGE_IC_FREED: 20830e7468fSPeter Dunlap case ISER_CONN_STAGE_CLOSING: 20930e7468fSPeter Dunlap case ISER_CONN_STAGE_CLOSED: 21030e7468fSPeter Dunlap break; 21130e7468fSPeter Dunlap 21230e7468fSPeter Dunlap default: 21330e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc, 21430e7468fSPeter Dunlap CE_TRANSPORT_FAIL, idm_status); 21530e7468fSPeter Dunlap iser_conn->ic_stage = ISER_CONN_STAGE_CLOSING; 21630e7468fSPeter Dunlap } 21730e7468fSPeter Dunlap mutex_exit(&iser_conn->ic_lock); 21830e7468fSPeter Dunlap 21930e7468fSPeter Dunlap /* Move onto the next completion */ 22030e7468fSPeter Dunlap continue; 22130e7468fSPeter Dunlap } 22230e7468fSPeter Dunlap 22330e7468fSPeter Dunlap /* 22430e7468fSPeter Dunlap * For a success status, just invoke the PDU or 22530e7468fSPeter Dunlap * buffer completion. We use our WR handle's 22630e7468fSPeter Dunlap * "iw_type" here so that we can properly process 22730e7468fSPeter Dunlap * because the CQE's opcode is invalid if the status 22830e7468fSPeter Dunlap * is failed. 22930e7468fSPeter Dunlap */ 23030e7468fSPeter Dunlap switch (wr->iw_type) { 23130e7468fSPeter Dunlap case ISER_WR_SEND: 23230e7468fSPeter Dunlap /* Free the msg handle */ 23330e7468fSPeter Dunlap ASSERT(wr->iw_msg != NULL); 23430e7468fSPeter Dunlap iser_msg_free(wr->iw_msg); 23530e7468fSPeter Dunlap 23630e7468fSPeter Dunlap if (wr->iw_pdu == NULL) { 23730e7468fSPeter Dunlap /* This is a hello exchange message */ 23830e7468fSPeter Dunlap mutex_enter(&iser_conn->ic_lock); 23930e7468fSPeter Dunlap if (iser_conn->ic_stage == 24030e7468fSPeter Dunlap ISER_CONN_STAGE_HELLOREPLY_SENT) { 24130e7468fSPeter Dunlap /* 24230e7468fSPeter Dunlap * We're on the target side, 24330e7468fSPeter Dunlap * and have just successfully 24430e7468fSPeter Dunlap * sent the HelloReply msg. 24530e7468fSPeter Dunlap */ 24630e7468fSPeter Dunlap iser_conn->ic_stage = 24730e7468fSPeter Dunlap ISER_CONN_STAGE_LOGGED_IN; 24830e7468fSPeter Dunlap } 24930e7468fSPeter Dunlap mutex_exit(&iser_conn->ic_lock); 25030e7468fSPeter Dunlap } else { 25130e7468fSPeter Dunlap /* This is a normal control message */ 25230e7468fSPeter Dunlap idm_pdu_complete(wr->iw_pdu, idm_status); 25330e7468fSPeter Dunlap } 25430e7468fSPeter Dunlap 25530e7468fSPeter Dunlap /* Free the wr handle */ 25630e7468fSPeter Dunlap iser_wr_free(wr); 25730e7468fSPeter Dunlap 25830e7468fSPeter Dunlap break; 25930e7468fSPeter Dunlap 26030e7468fSPeter Dunlap case ISER_WR_RDMAW: 26130e7468fSPeter Dunlap case ISER_WR_RDMAR: 26230e7468fSPeter Dunlap /* 26330e7468fSPeter Dunlap * Invoke the appropriate callback; 26430e7468fSPeter Dunlap * the buffer will be freed there. 26530e7468fSPeter Dunlap */ 26630e7468fSPeter Dunlap idb = wr->iw_buf; 267a668b114SPriya Krishnan mr = ((iser_buf_t *)idb->idb_buf_private)->iser_mr; 26830e7468fSPeter Dunlap #ifdef DEBUG 26930e7468fSPeter Dunlap bcopy(&wc[i], 27030e7468fSPeter Dunlap &((iser_buf_t *)idb->idb_buf_private)->buf_wc, 27130e7468fSPeter Dunlap sizeof (ibt_wc_t)); 27230e7468fSPeter Dunlap #endif 27330e7468fSPeter Dunlap idt = idb->idb_task_binding; 27430e7468fSPeter Dunlap 27530e7468fSPeter Dunlap mutex_enter(&idt->idt_mutex); 27630e7468fSPeter Dunlap if (wr->iw_type == ISER_WR_RDMAW) { 277a668b114SPriya Krishnan DTRACE_ISCSI_8(xfer__done, 278a668b114SPriya Krishnan idm_conn_t *, idt->idt_ic, 279a668b114SPriya Krishnan uintptr_t, idb->idb_buf, 280a668b114SPriya Krishnan uint32_t, idb->idb_bufoffset, 281a668b114SPriya Krishnan uint64_t, mr->is_mrva, uint32_t, 0, 282a668b114SPriya Krishnan uint32_t, mr->is_mrrkey, 283a668b114SPriya Krishnan uint32_t, idb->idb_xfer_len, 284a668b114SPriya Krishnan int, XFER_BUF_TX_TO_INI); 28530e7468fSPeter Dunlap idm_buf_tx_to_ini_done(idt, idb, idm_status); 28630e7468fSPeter Dunlap } else { 287a668b114SPriya Krishnan DTRACE_ISCSI_8(xfer__done, 288a668b114SPriya Krishnan idm_conn_t *, idt->idt_ic, 289a668b114SPriya Krishnan uintptr_t, idb->idb_buf, 290a668b114SPriya Krishnan uint32_t, idb->idb_bufoffset, 291a668b114SPriya Krishnan uint64_t, mr->is_mrva, uint32_t, 0, 292a668b114SPriya Krishnan uint32_t, mr->is_mrrkey, 293a668b114SPriya Krishnan uint32_t, idb->idb_xfer_len, 294a668b114SPriya Krishnan int, XFER_BUF_RX_FROM_INI); 29530e7468fSPeter Dunlap idm_buf_rx_from_ini_done(idt, idb, idm_status); 29630e7468fSPeter Dunlap } 29730e7468fSPeter Dunlap 29830e7468fSPeter Dunlap /* Free the wr handle */ 29930e7468fSPeter Dunlap iser_wr_free(wr); 30030e7468fSPeter Dunlap 30130e7468fSPeter Dunlap break; 30230e7468fSPeter Dunlap 30330e7468fSPeter Dunlap default: 30430e7468fSPeter Dunlap ASSERT(0); 30530e7468fSPeter Dunlap break; 30630e7468fSPeter Dunlap } 30730e7468fSPeter Dunlap } 30830e7468fSPeter Dunlap 30930e7468fSPeter Dunlap return (status); 31030e7468fSPeter Dunlap } 31130e7468fSPeter Dunlap 31230e7468fSPeter Dunlap void 31330e7468fSPeter Dunlap iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl, void *arg) 31430e7468fSPeter Dunlap { 31530e7468fSPeter Dunlap iser_chan_t *iser_chan; 31630e7468fSPeter Dunlap ibt_status_t status; 31730e7468fSPeter Dunlap 31830e7468fSPeter Dunlap iser_chan = (iser_chan_t *)arg; 31930e7468fSPeter Dunlap 320*a82ec3cfSPriya Krishnan /* 321*a82ec3cfSPriya Krishnan * Poll for work request completion while successful. If the 322*a82ec3cfSPriya Krishnan * queue empties or otherwise becomes invalid, stop polling. 323*a82ec3cfSPriya Krishnan */ 32430e7468fSPeter Dunlap do { 32530e7468fSPeter Dunlap status = iser_ib_poll_recv_completions(cq_hdl, iser_chan); 326*a82ec3cfSPriya Krishnan } while (status == IBT_SUCCESS); 32730e7468fSPeter Dunlap 328*a82ec3cfSPriya Krishnan if (status == IBT_CQ_EMPTY) { 329*a82ec3cfSPriya Krishnan /* We've emptied the CQ, rearm it before we're done here */ 330*a82ec3cfSPriya Krishnan status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION); 331*a82ec3cfSPriya Krishnan if (status != IBT_SUCCESS) { 332*a82ec3cfSPriya Krishnan /* Unexpected error */ 333*a82ec3cfSPriya Krishnan ISER_LOG(CE_NOTE, "iser_ib_recvcq_handler: " 334*a82ec3cfSPriya Krishnan "ibt_enable_cq_notify error (%d)", status); 335*a82ec3cfSPriya Krishnan return; 336*a82ec3cfSPriya Krishnan } 33730e7468fSPeter Dunlap 338*a82ec3cfSPriya Krishnan /* Now, check for more completions after the rearm */ 339*a82ec3cfSPriya Krishnan do { 340*a82ec3cfSPriya Krishnan status = iser_ib_poll_recv_completions( 341*a82ec3cfSPriya Krishnan cq_hdl, iser_chan); 342*a82ec3cfSPriya Krishnan } while (status == IBT_SUCCESS); 343*a82ec3cfSPriya Krishnan } 34430e7468fSPeter Dunlap } 34530e7468fSPeter Dunlap 34630e7468fSPeter Dunlap static int 34730e7468fSPeter Dunlap iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan) 34830e7468fSPeter Dunlap { 34930e7468fSPeter Dunlap ibt_wc_t wc; 35030e7468fSPeter Dunlap iser_msg_t *msg; 35130e7468fSPeter Dunlap iser_qp_t *iser_qp; 35230e7468fSPeter Dunlap int status; 35330e7468fSPeter Dunlap 35430e7468fSPeter Dunlap iser_qp = &(iser_chan->ic_qp); 35530e7468fSPeter Dunlap 35630e7468fSPeter Dunlap bzero(&wc, sizeof (ibt_wc_t)); 35730e7468fSPeter Dunlap status = ibt_poll_cq(cq_hdl, &wc, 1, NULL); 35830e7468fSPeter Dunlap if (status == IBT_CQ_EMPTY) { 35930e7468fSPeter Dunlap /* CQ is empty, return */ 36030e7468fSPeter Dunlap return (status); 36130e7468fSPeter Dunlap } 36230e7468fSPeter Dunlap 36330e7468fSPeter Dunlap if (status != IBT_SUCCESS) { 36430e7468fSPeter Dunlap /* Unexpected error */ 36530e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: " 36630e7468fSPeter Dunlap "ibt_poll_cq error (%d)", status); 36730e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock); 36830e7468fSPeter Dunlap iser_qp->rq_level--; 36930e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock); 37030e7468fSPeter Dunlap /* Free the msg handle (if we got it back) */ 37130e7468fSPeter Dunlap if ((msg = (iser_msg_t *)(uintptr_t)wc.wc_id) != NULL) { 37230e7468fSPeter Dunlap iser_msg_free(msg); 37330e7468fSPeter Dunlap } 37430e7468fSPeter Dunlap return (status); 37530e7468fSPeter Dunlap } 37630e7468fSPeter Dunlap 37730e7468fSPeter Dunlap /* Retrieve the iSER msg handle */ 37830e7468fSPeter Dunlap msg = (iser_msg_t *)(uintptr_t)wc.wc_id; 37930e7468fSPeter Dunlap ASSERT(msg != NULL); 38030e7468fSPeter Dunlap 38130e7468fSPeter Dunlap /* 38230e7468fSPeter Dunlap * Decrement the posted level in the RQ, then check 38330e7468fSPeter Dunlap * to see if we need to fill the RQ back up (or if 38430e7468fSPeter Dunlap * we are already on the taskq). 38530e7468fSPeter Dunlap */ 386cf8c0ebaSPeter Dunlap mutex_enter(&iser_chan->ic_conn->ic_lock); 38730e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock); 38830e7468fSPeter Dunlap iser_qp->rq_level--; 38930e7468fSPeter Dunlap 39030e7468fSPeter Dunlap if ((iser_qp->rq_taskqpending == B_FALSE) && 39172cf3143Speter dunlap (iser_qp->rq_level <= iser_qp->rq_lwm) && 39272cf3143Speter dunlap (iser_chan->ic_conn->ic_stage >= ISER_CONN_STAGE_IC_CONNECTED) && 39372cf3143Speter dunlap (iser_chan->ic_conn->ic_stage <= ISER_CONN_STAGE_LOGGED_IN)) { 39430e7468fSPeter Dunlap /* Set the pending flag and fire off a post_recv */ 39530e7468fSPeter Dunlap iser_qp->rq_taskqpending = B_TRUE; 39630e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock); 39730e7468fSPeter Dunlap 398cf8c0ebaSPeter Dunlap status = iser_ib_post_recv_async(iser_chan->ic_chanhdl); 39930e7468fSPeter Dunlap 40030e7468fSPeter Dunlap if (status != DDI_SUCCESS) { 40130e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: " 40230e7468fSPeter Dunlap "task dispatch failed"); 40330e7468fSPeter Dunlap /* Failure to launch, unset the pending flag */ 40430e7468fSPeter Dunlap mutex_enter(&iser_qp->qp_lock); 40530e7468fSPeter Dunlap iser_qp->rq_taskqpending = B_FALSE; 40630e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock); 40730e7468fSPeter Dunlap } 40830e7468fSPeter Dunlap } else { 40930e7468fSPeter Dunlap mutex_exit(&iser_qp->qp_lock); 41030e7468fSPeter Dunlap } 41130e7468fSPeter Dunlap 41230e7468fSPeter Dunlap DTRACE_PROBE3(iser__recv__cqe, iser_chan_t *, iser_chan, 41330e7468fSPeter Dunlap ibt_wc_t *, &wc, ibt_wc_status_t, wc.wc_status); 41430e7468fSPeter Dunlap if (wc.wc_status != IBT_WC_SUCCESS) { 41530e7468fSPeter Dunlap /* 41630e7468fSPeter Dunlap * Tell IDM that the channel has gone down, 41730e7468fSPeter Dunlap * unless he already knows. 41830e7468fSPeter Dunlap */ 41930e7468fSPeter Dunlap switch (iser_chan->ic_conn->ic_stage) { 42030e7468fSPeter Dunlap case ISER_CONN_STAGE_IC_DISCONNECTED: 42130e7468fSPeter Dunlap case ISER_CONN_STAGE_IC_FREED: 42230e7468fSPeter Dunlap case ISER_CONN_STAGE_CLOSING: 42330e7468fSPeter Dunlap case ISER_CONN_STAGE_CLOSED: 42430e7468fSPeter Dunlap break; 42530e7468fSPeter Dunlap 42630e7468fSPeter Dunlap default: 42730e7468fSPeter Dunlap idm_conn_event(iser_chan->ic_conn->ic_idmc, 42830e7468fSPeter Dunlap CE_TRANSPORT_FAIL, IDM_STATUS_FAIL); 42930e7468fSPeter Dunlap iser_chan->ic_conn->ic_stage = 43030e7468fSPeter Dunlap ISER_CONN_STAGE_CLOSING; 43130e7468fSPeter Dunlap } 43230e7468fSPeter Dunlap mutex_exit(&iser_chan->ic_conn->ic_lock); 43330e7468fSPeter Dunlap 43430e7468fSPeter Dunlap iser_msg_free(msg); 43530e7468fSPeter Dunlap return (DDI_SUCCESS); 43630e7468fSPeter Dunlap } else { 43772cf3143Speter dunlap mutex_exit(&iser_chan->ic_conn->ic_lock); 43872cf3143Speter dunlap 43930e7468fSPeter Dunlap /* 44030e7468fSPeter Dunlap * We have an iSER message in, let's handle it. 44130e7468fSPeter Dunlap * We will free the iser_msg_t later in this path, 44230e7468fSPeter Dunlap * depending upon the action required. 44330e7468fSPeter Dunlap */ 44430e7468fSPeter Dunlap iser_msg_handle(iser_chan, msg); 44530e7468fSPeter Dunlap return (DDI_SUCCESS); 44630e7468fSPeter Dunlap } 44730e7468fSPeter Dunlap } 44830e7468fSPeter Dunlap 44930e7468fSPeter Dunlap static void 45030e7468fSPeter Dunlap iser_msg_handle(iser_chan_t *chan, iser_msg_t *msg) 45130e7468fSPeter Dunlap { 45230e7468fSPeter Dunlap int opcode; 45330e7468fSPeter Dunlap iser_ctrl_hdr_t *hdr = NULL; 45430e7468fSPeter Dunlap iser_conn_t *iser_conn = chan->ic_conn; 45530e7468fSPeter Dunlap int status; 45630e7468fSPeter Dunlap 45730e7468fSPeter Dunlap hdr = (iser_ctrl_hdr_t *)(uintptr_t)msg->msg_ds.ds_va; 45830e7468fSPeter Dunlap ASSERT(hdr != NULL); 45930e7468fSPeter Dunlap 46030e7468fSPeter Dunlap opcode = hdr->opcode; 46130e7468fSPeter Dunlap if (opcode == ISER_OPCODE_CTRL_TYPE_PDU) { 46230e7468fSPeter Dunlap /* 46330e7468fSPeter Dunlap * Handle an iSCSI Control PDU iSER message. 46430e7468fSPeter Dunlap * Note we'll free the msg handle in the PDU callback. 46530e7468fSPeter Dunlap */ 46630e7468fSPeter Dunlap status = iser_iscsihdr_handle(chan, msg); 46730e7468fSPeter Dunlap if (status != DDI_SUCCESS) { 46830e7468fSPeter Dunlap /* 46930e7468fSPeter Dunlap * We are unable to handle this message, and 47030e7468fSPeter Dunlap * have no way to recover from this. Fail the 47130e7468fSPeter Dunlap * transport. 47230e7468fSPeter Dunlap */ 47330e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_msg_handle: failed " 47430e7468fSPeter Dunlap "iser_iscsihdr_handle"); 47530e7468fSPeter Dunlap iser_msg_free(msg); 47630e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc, 47730e7468fSPeter Dunlap CE_TRANSPORT_FAIL, IDM_STATUS_FAIL); 47830e7468fSPeter Dunlap } 47930e7468fSPeter Dunlap } else if (opcode == ISER_OPCODE_HELLO_MSG) { /* at the target */ 48030e7468fSPeter Dunlap /* 48130e7468fSPeter Dunlap * We are currently not supporting Hello Exchange, 48230e7468fSPeter Dunlap * since OFED iSER does not. May be revisited. 48330e7468fSPeter Dunlap */ 48430e7468fSPeter Dunlap ASSERT(opcode != ISER_OPCODE_HELLO_MSG); 48530e7468fSPeter Dunlap 48630e7468fSPeter Dunlap if (iser_conn->ic_type != ISER_CONN_TYPE_TGT) { 48730e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc, 48830e7468fSPeter Dunlap CE_TRANSPORT_FAIL, IDM_STATUS_FAIL); 48930e7468fSPeter Dunlap } 49030e7468fSPeter Dunlap 49130e7468fSPeter Dunlap iser_hello_hdr_t *hello_hdr = (iser_hello_hdr_t *)hdr; 49230e7468fSPeter Dunlap 49330e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "received Hello message: opcode[%d], " 49430e7468fSPeter Dunlap "maxver[%d], minver[%d], iser_ird[%d], msg (0x%p)", 49530e7468fSPeter Dunlap hello_hdr->opcode, hello_hdr->maxver, hello_hdr->minver, 49630e7468fSPeter Dunlap ntohs(hello_hdr->iser_ird), (void *)msg); 49730e7468fSPeter Dunlap 49830e7468fSPeter Dunlap mutex_enter(&iser_conn->ic_lock); 49930e7468fSPeter Dunlap 50030e7468fSPeter Dunlap if (iser_conn->ic_stage != ISER_CONN_STAGE_HELLO_WAIT) { 50130e7468fSPeter Dunlap /* target is not expected to receive a Hello */ 50230e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc, 50330e7468fSPeter Dunlap CE_TRANSPORT_FAIL, IDM_STATUS_FAIL); 50430e7468fSPeter Dunlap } 50530e7468fSPeter Dunlap 50630e7468fSPeter Dunlap iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_SENT; 50730e7468fSPeter Dunlap mutex_exit(&iser_conn->ic_lock); 50830e7468fSPeter Dunlap 50930e7468fSPeter Dunlap /* Prepare and send a HelloReply message */ 51030e7468fSPeter Dunlap status = iser_xfer_helloreply_msg(chan); 51130e7468fSPeter Dunlap if (status != ISER_STATUS_SUCCESS) { 51230e7468fSPeter Dunlap 51330e7468fSPeter Dunlap mutex_enter(&iser_conn->ic_lock); 51430e7468fSPeter Dunlap iser_conn->ic_stage = 51530e7468fSPeter Dunlap ISER_CONN_STAGE_HELLOREPLY_SENT_FAIL; 51630e7468fSPeter Dunlap mutex_exit(&iser_conn->ic_lock); 51730e7468fSPeter Dunlap 51830e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc, 51930e7468fSPeter Dunlap CE_TRANSPORT_FAIL, status); 52030e7468fSPeter Dunlap } 52130e7468fSPeter Dunlap 52230e7468fSPeter Dunlap /* Free this msg handle */ 52330e7468fSPeter Dunlap iser_msg_free(msg); 52430e7468fSPeter Dunlap 52530e7468fSPeter Dunlap } else if (opcode == ISER_OPCODE_HELLOREPLY_MSG) { /* at initiator */ 52630e7468fSPeter Dunlap 52730e7468fSPeter Dunlap /* 52830e7468fSPeter Dunlap * We are currently not supporting Hello Exchange, 52930e7468fSPeter Dunlap * since OFED iSER does not. May be revisited. 53030e7468fSPeter Dunlap */ 53130e7468fSPeter Dunlap ASSERT(opcode != ISER_OPCODE_HELLOREPLY_MSG); 53230e7468fSPeter Dunlap 53330e7468fSPeter Dunlap if (iser_conn->ic_type != ISER_CONN_TYPE_INI) { 53430e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc, 53530e7468fSPeter Dunlap CE_TRANSPORT_FAIL, status); 53630e7468fSPeter Dunlap } 53730e7468fSPeter Dunlap 53830e7468fSPeter Dunlap iser_helloreply_hdr_t *hello_hdr = (iser_helloreply_hdr_t *)hdr; 53930e7468fSPeter Dunlap 54030e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "received Hello Reply message: opcode[%d], " 54130e7468fSPeter Dunlap "maxver[%d], curver[%d], iser_ord[%d], msg (0x%p)", 54230e7468fSPeter Dunlap hello_hdr->opcode, hello_hdr->maxver, hello_hdr->curver, 54330e7468fSPeter Dunlap ntohs(hello_hdr->iser_ord), (void *)msg); 54430e7468fSPeter Dunlap 54530e7468fSPeter Dunlap /* Free this msg handle */ 54630e7468fSPeter Dunlap iser_msg_free(msg); 54730e7468fSPeter Dunlap 54830e7468fSPeter Dunlap /* 54930e7468fSPeter Dunlap * Signal the receipt of HelloReply to the waiting thread 55030e7468fSPeter Dunlap * so that the initiator can proceed to the Full Feature 55130e7468fSPeter Dunlap * Phase. 55230e7468fSPeter Dunlap */ 55330e7468fSPeter Dunlap mutex_enter(&iser_conn->ic_lock); 55430e7468fSPeter Dunlap iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_RCV; 55530e7468fSPeter Dunlap cv_signal(&iser_conn->ic_stage_cv); 55630e7468fSPeter Dunlap mutex_exit(&iser_conn->ic_lock); 55730e7468fSPeter Dunlap } else { 55830e7468fSPeter Dunlap /* Protocol error: free the msg handle and fail the session */ 55930e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_msg_handle: unsupported opcode (0x%x): " 56030e7468fSPeter Dunlap "terminating session on IDM handle (0x%p)", opcode, 56130e7468fSPeter Dunlap (void *) iser_conn->ic_idmc); 56230e7468fSPeter Dunlap 56330e7468fSPeter Dunlap iser_msg_free(msg); 56430e7468fSPeter Dunlap idm_conn_event(iser_conn->ic_idmc, CE_TRANSPORT_FAIL, 56530e7468fSPeter Dunlap IDM_STATUS_FAIL); 56630e7468fSPeter Dunlap } 56730e7468fSPeter Dunlap } 56830e7468fSPeter Dunlap 56930e7468fSPeter Dunlap #define IDM_PDU_OPCODE(PDU) \ 57030e7468fSPeter Dunlap ((PDU)->isp_hdr->opcode & ISCSI_OPCODE_MASK) 57130e7468fSPeter Dunlap 57230e7468fSPeter Dunlap /* network to host translation for 24b integers */ 57330e7468fSPeter Dunlap static uint32_t 57430e7468fSPeter Dunlap n2h24(uchar_t *ptr) 57530e7468fSPeter Dunlap { 57630e7468fSPeter Dunlap return ((ptr[0] << 16) | (ptr[1] << 8) | ptr[2]); 57730e7468fSPeter Dunlap } 57830e7468fSPeter Dunlap 57930e7468fSPeter Dunlap /* ARGSUSED */ 58030e7468fSPeter Dunlap static void 58130e7468fSPeter Dunlap iser_rx_pdu_cb(idm_pdu_t *pdu, idm_status_t status) 58230e7468fSPeter Dunlap { 58330e7468fSPeter Dunlap /* Free the iser msg handle and the PDU handle */ 58430e7468fSPeter Dunlap iser_msg_free((iser_msg_t *)pdu->isp_transport_private); 58530e7468fSPeter Dunlap idm_pdu_free(pdu); 58630e7468fSPeter Dunlap } 58730e7468fSPeter Dunlap 58830e7468fSPeter Dunlap int 58930e7468fSPeter Dunlap iser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg) 59030e7468fSPeter Dunlap { 59130e7468fSPeter Dunlap idm_pdu_t *pdu; 59230e7468fSPeter Dunlap uint8_t *iser_hdrp; 59330e7468fSPeter Dunlap uint8_t *iscsi_hdrp; 59430e7468fSPeter Dunlap iscsi_hdr_t *bhs; 59530e7468fSPeter Dunlap 59630e7468fSPeter Dunlap pdu = idm_pdu_alloc_nosleep(sizeof (iscsi_hdr_t), 0); 59730e7468fSPeter Dunlap pdu->isp_ic = chan->ic_conn->ic_idmc; 59830e7468fSPeter Dunlap ASSERT(pdu->isp_ic != NULL); 59930e7468fSPeter Dunlap 60030e7468fSPeter Dunlap /* Set the iser_msg handle into the transport-private field */ 60130e7468fSPeter Dunlap pdu->isp_transport_private = (void *)msg; 60230e7468fSPeter Dunlap 60330e7468fSPeter Dunlap /* Set up a pointer in the pdu handle to the iSER header */ 60430e7468fSPeter Dunlap iser_hdrp = (uint8_t *)(uintptr_t)msg->msg_ds.ds_va; 60530e7468fSPeter Dunlap if (iser_hdrp == NULL) { 60630e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iser_hdrp is NULL"); 60730e7468fSPeter Dunlap idm_pdu_free(pdu); 60830e7468fSPeter Dunlap return (ISER_STATUS_FAIL); 60930e7468fSPeter Dunlap } 61030e7468fSPeter Dunlap pdu->isp_transport_hdr = (void *)iser_hdrp; 61130e7468fSPeter Dunlap pdu->isp_transport_hdrlen = ISER_HEADER_LENGTH; 61230e7468fSPeter Dunlap 61330e7468fSPeter Dunlap /* 61430e7468fSPeter Dunlap * Set up a pointer to the iSCSI header, which is directly 61530e7468fSPeter Dunlap * after the iSER header in the message. 61630e7468fSPeter Dunlap */ 61730e7468fSPeter Dunlap iscsi_hdrp = ((uint8_t *)(uintptr_t)msg->msg_ds.ds_va) + 61830e7468fSPeter Dunlap ISER_HEADER_LENGTH; 61930e7468fSPeter Dunlap if (iscsi_hdrp == NULL) { 62030e7468fSPeter Dunlap ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iscsi_hdrp is NULL"); 62130e7468fSPeter Dunlap idm_pdu_free(pdu); 62230e7468fSPeter Dunlap return (ISER_STATUS_FAIL); 62330e7468fSPeter Dunlap } 62430e7468fSPeter Dunlap pdu->isp_hdr = (iscsi_hdr_t *)(uintptr_t)iscsi_hdrp; 62530e7468fSPeter Dunlap 62630e7468fSPeter Dunlap /* Fill in the BHS */ 62730e7468fSPeter Dunlap bhs = pdu->isp_hdr; 62830e7468fSPeter Dunlap pdu->isp_hdrlen = sizeof (iscsi_hdr_t) + 62930e7468fSPeter Dunlap (bhs->hlength * sizeof (uint32_t)); 63030e7468fSPeter Dunlap pdu->isp_datalen = n2h24(bhs->dlength); 63130e7468fSPeter Dunlap pdu->isp_callback = iser_rx_pdu_cb; 63230e7468fSPeter Dunlap 63330e7468fSPeter Dunlap /* 63430e7468fSPeter Dunlap * If datalen > 0, then non-scsi data may be present. Allocate 63530e7468fSPeter Dunlap * space in the PDU handle and set a pointer to the data. 63630e7468fSPeter Dunlap */ 63730e7468fSPeter Dunlap if (pdu->isp_datalen) { 63830e7468fSPeter Dunlap pdu->isp_data = ((uint8_t *)(uintptr_t)pdu->isp_hdr) + 63930e7468fSPeter Dunlap pdu->isp_hdrlen; 64030e7468fSPeter Dunlap } 64130e7468fSPeter Dunlap 64230e7468fSPeter Dunlap /* Process RX PDU */ 64330e7468fSPeter Dunlap idm_pdu_rx(pdu->isp_ic, pdu); 64430e7468fSPeter Dunlap 64530e7468fSPeter Dunlap return (DDI_SUCCESS); 64630e7468fSPeter Dunlap } 647