1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/stat.h> 28 #include <sys/conf.h> 29 #include <sys/ddi.h> 30 #include <sys/sunddi.h> 31 #include <sys/modctl.h> 32 #include <sys/byteorder.h> 33 #include <sys/sdt.h> 34 35 #include <sys/ib/clients/iser/iser.h> 36 37 /* 38 * iser_xfer.c 39 */ 40 41 int 42 iser_xfer_hello_msg(iser_chan_t *chan) 43 { 44 iser_hca_t *hca; 45 iser_wr_t *iser_wr; 46 iser_msg_t *msg; 47 ibt_send_wr_t wr; 48 iser_hello_hdr_t *hdr; 49 int status; 50 51 ASSERT(chan != NULL); 52 53 hca = (iser_hca_t *)chan->ic_hca; 54 if (hca == NULL) { 55 ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: no hca handle found"); 56 return (ISER_STATUS_FAIL); 57 } 58 59 msg = iser_msg_get(hca, 1, NULL); 60 61 if (msg == NULL) { 62 ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: iser message cache " 63 "alloc failed"); 64 return (ISER_STATUS_FAIL); 65 } 66 67 /* Send iSER Hello Message to declare iSER parameters to the target */ 68 hdr = (iser_hello_hdr_t *)(uintptr_t)msg->msg_ds.ds_va; 69 70 hdr->opcode = ISER_OPCODE_HELLO_MSG; 71 hdr->rsvd1 = 0; 72 hdr->maxver = 1; 73 hdr->minver = 1; 74 hdr->iser_ird = htons(ISER_IB_DEFAULT_IRD); 75 hdr->rsvd2[0] = 0; 76 hdr->rsvd2[1] = 0; 77 78 /* Allocate an iSER WR handle and tuck this msg into it */ 79 iser_wr = iser_wr_get(); 80 if (iser_wr == NULL) { 81 ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: unable to allocate " 82 "iser wr handle"); 83 iser_msg_free(msg); 84 return (ISER_STATUS_FAIL); 85 } 86 iser_wr->iw_msg = msg; 87 iser_wr->iw_type = ISER_WR_SEND; 88 89 /* Use the address of our generic iser_wr handle as our WRID */ 90 wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr; 91 92 /* Populate the rest of the work request */ 93 wr.wr_trans = IBT_RC_SRV; 94 wr.wr_opcode = IBT_WRC_SEND; 95 wr.wr_nds = 1; 96 wr.wr_sgl = &msg->msg_ds; 97 98 status = ibt_post_send(chan->ic_chanhdl, &wr, 1, NULL); 99 if (status != IBT_SUCCESS) { 100 ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: ibt_post_send " 101 "failure (%d)", status); 102 iser_msg_free(msg); 103 iser_wr_free(iser_wr); 104 return (ISER_STATUS_FAIL); 105 } 106 /* Increment this channel's SQ posted count */ 107 mutex_enter(&chan->ic_sq_post_lock); 108 chan->ic_sq_post_count++; 109 if (chan->ic_sq_post_count > chan->ic_sq_max_post_count) 110 chan->ic_sq_max_post_count = chan->ic_sq_post_count; 111 mutex_exit(&chan->ic_sq_post_lock); 112 113 ISER_LOG(CE_NOTE, "Posting iSER Hello message: chan (0x%p): " 114 "IP [%x to %x]", (void *)chan, chan->ic_localip.un.ip4addr, 115 chan->ic_remoteip.un.ip4addr); 116 117 return (ISER_STATUS_SUCCESS); 118 } 119 120 int 121 iser_xfer_helloreply_msg(iser_chan_t *chan) 122 { 123 iser_hca_t *hca; 124 iser_wr_t *iser_wr; 125 ibt_send_wr_t wr; 126 iser_msg_t *msg; 127 iser_helloreply_hdr_t *hdr; 128 int status; 129 130 ASSERT(chan != NULL); 131 132 hca = (iser_hca_t *)chan->ic_hca; 133 if (hca == NULL) { 134 ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: no hca handle " 135 "found"); 136 return (ISER_STATUS_FAIL); 137 } 138 139 msg = iser_msg_get(hca, 1, NULL); 140 141 if (msg == NULL) { 142 ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: iser message " 143 "cache alloc failed"); 144 return (ISER_STATUS_FAIL); 145 } 146 147 /* Use the iSER Hello Reply Message */ 148 hdr = (iser_helloreply_hdr_t *)(uintptr_t)msg->msg_ds.ds_va; 149 150 hdr->opcode = ISER_OPCODE_HELLOREPLY_MSG; 151 hdr->rsvd1 = 0; 152 hdr->flag = 0; 153 hdr->maxver = 1; 154 hdr->curver = 1; 155 hdr->iser_ord = htons(ISER_IB_DEFAULT_ORD); 156 hdr->rsvd2[0] = 0; 157 hdr->rsvd2[1] = 0; 158 159 /* Allocate an iSER WR handle and tuck this msg into it */ 160 iser_wr = iser_wr_get(); 161 if (iser_wr == NULL) { 162 ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: unable to " 163 "allocate iser wr handle"); 164 iser_msg_free(msg); 165 return (ISER_STATUS_FAIL); 166 } 167 iser_wr->iw_msg = msg; 168 iser_wr->iw_type = ISER_WR_SEND; 169 170 /* Use the address of our generic iser_wr handle as our WRID */ 171 wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr; 172 173 /* Populate the rest of the work request */ 174 wr.wr_trans = IBT_RC_SRV; 175 wr.wr_opcode = IBT_WRC_SEND; 176 wr.wr_nds = 1; 177 wr.wr_sgl = &msg->msg_ds; 178 179 status = ibt_post_send(chan->ic_chanhdl, &wr, 1, NULL); 180 if (status != IBT_SUCCESS) { 181 ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: ibt_post_send " 182 "failure (%d)", status); 183 iser_msg_free(msg); 184 iser_wr_free(iser_wr); 185 return (ISER_STATUS_FAIL); 186 } 187 /* Increment this channel's SQ posted count */ 188 mutex_enter(&chan->ic_sq_post_lock); 189 chan->ic_sq_post_count++; 190 if (chan->ic_sq_post_count > chan->ic_sq_max_post_count) 191 chan->ic_sq_max_post_count = chan->ic_sq_post_count; 192 mutex_exit(&chan->ic_sq_post_lock); 193 194 ISER_LOG(CE_NOTE, "Posting iSER HelloReply message: chan (0x%p): " 195 "IP [%x to %x]", (void *)chan, chan->ic_localip.un.ip4addr, 196 chan->ic_remoteip.un.ip4addr); 197 198 return (ISER_STATUS_SUCCESS); 199 } 200 201 /* 202 * iser_xfer_ctrlpdu 203 * 204 * This is iSER's implementation of the 'Send_control' operational primitive. 205 * This iSER layer uses the Send Message type of RCaP to transfer the iSCSI 206 * Control-type PDU. A special case is that the transfer of SCSI Data-out PDUs 207 * carrying unsolicited data are also treated as iSCSI Control-Type PDUs. The 208 * message payload contains an iSER header followed by the iSCSI Control-type 209 * the iSCSI Control-type PDU. 210 * This function is invoked by an initiator iSCSI layer requesting the transfer 211 * of a iSCSI command PDU or a target iSCSI layer requesting the transfer of a 212 * iSCSI response PDU. 213 */ 214 int 215 iser_xfer_ctrlpdu(iser_chan_t *chan, idm_pdu_t *pdu) 216 { 217 iser_hca_t *hca; 218 iser_ctrl_hdr_t *hdr; 219 iser_msg_t *msg; 220 iser_wr_t *iser_wr; 221 ibt_send_wr_t wr; 222 int status; 223 iser_mr_t *mr; 224 iscsi_data_hdr_t *bhs; 225 idm_conn_t *ic; 226 idm_task_t *idt = NULL; 227 idm_buf_t *buf; 228 229 ASSERT(chan != NULL); 230 231 /* 232 * All SCSI command PDU (except SCSI Read and SCSI Write) and the SCSI 233 * Response PDU are sent to the remote end using the SendSE Message. 234 * 235 * Setup a Send Message for carrying the iSCSI control-type PDU 236 * preceeded by an iSER header. 237 */ 238 hca = (iser_hca_t *)chan->ic_hca; 239 if (hca == NULL) { 240 ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: no hca handle found"); 241 return (ISER_STATUS_FAIL); 242 } 243 244 msg = iser_msg_get(hca, 1, NULL); 245 if (msg == NULL) { 246 ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: iser message cache " 247 "alloc failed"); 248 return (ISER_STATUS_FAIL); 249 } 250 251 /* Pull the BHS out of the PDU handle */ 252 bhs = (iscsi_data_hdr_t *)pdu->isp_hdr; 253 254 ASSERT(chan->ic_conn != NULL && chan->ic_conn->ic_idmc != NULL); 255 ic = chan->ic_conn->ic_idmc; 256 ASSERT(ic != NULL); 257 258 hdr = (iser_ctrl_hdr_t *)(uintptr_t)msg->msg_ds.ds_va; 259 260 /* 261 * Initialize header assuming no transfers 262 */ 263 bzero(hdr, sizeof (*hdr)); 264 hdr->opcode = ISER_OPCODE_CTRL_TYPE_PDU; 265 266 /* 267 * On the initiator side, the task buffers will be used to identify 268 * if there are any buffers to be advertised 269 */ 270 if ((ic->ic_conn_type == CONN_TYPE_INI) && 271 ((bhs->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_SCSI_CMD) && 272 ((idt = idm_task_find(ic, bhs->itt, bhs->ttt)) != NULL)) { 273 274 if (!list_is_empty(&idt->idt_inbufv)) { 275 buf = idm_buf_find(&idt->idt_inbufv, 0); 276 ASSERT(buf != NULL); 277 278 mr = (iser_mr_t *)buf->idb_reg_private; 279 ASSERT(mr != NULL); 280 281 hdr->rsv_flag = 1; 282 hdr->rstag = htonl(mr->is_mrrkey); 283 BE_OUT64(&hdr->rva, mr->is_mrva); 284 } 285 286 if (!list_is_empty(&idt->idt_outbufv)) { 287 buf = idm_buf_find(&idt->idt_outbufv, 0); 288 ASSERT(buf != NULL); 289 290 mr = (iser_mr_t *)buf->idb_reg_private; 291 ASSERT(mr != NULL); 292 293 hdr->wsv_flag = 1; 294 hdr->wstag = htonl(mr->is_mrrkey); 295 BE_OUT64(&hdr->wva, mr->is_mrva); 296 } 297 298 /* Release our reference on the task */ 299 idm_task_rele(idt); 300 } 301 302 /* Copy the BHS after the iSER header */ 303 bcopy(pdu->isp_hdr, 304 (uint8_t *)(uintptr_t)msg->msg_ds.ds_va + ISER_HEADER_LENGTH, 305 pdu->isp_hdrlen); 306 307 if (pdu->isp_datalen > 0) { 308 /* Copy the isp_data after the PDU header */ 309 bcopy(pdu->isp_data, 310 (uint8_t *)(uintptr_t)msg->msg_ds.ds_va + 311 ISER_HEADER_LENGTH + pdu->isp_hdrlen, 312 pdu->isp_datalen); 313 314 /* Set the SGE's ds_len */ 315 msg->msg_ds.ds_len = ISER_HEADER_LENGTH + pdu->isp_hdrlen + 316 pdu->isp_datalen; 317 } else { 318 /* No data, so set the SGE's ds_len to the headers length */ 319 msg->msg_ds.ds_len = ISER_HEADER_LENGTH + pdu->isp_hdrlen; 320 } 321 322 /* 323 * Build Work Request to be posted on the Send Queue. 324 */ 325 bzero(&wr, sizeof (wr)); 326 327 /* Allocate an iSER WR handle and tuck the msg and pdu into it */ 328 iser_wr = iser_wr_get(); 329 if (iser_wr == NULL) { 330 ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: unable to allocate " 331 "iser wr handle"); 332 iser_msg_free(msg); 333 return (ISER_STATUS_FAIL); 334 } 335 iser_wr->iw_pdu = pdu; 336 iser_wr->iw_msg = msg; 337 iser_wr->iw_type = ISER_WR_SEND; 338 339 /* 340 * Use the address of our generic iser_wr handle as our WRID 341 * and populate the rest of the work request 342 */ 343 wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr; 344 wr.wr_trans = IBT_RC_SRV; 345 wr.wr_opcode = IBT_WRC_SEND; 346 wr.wr_nds = 1; 347 wr.wr_sgl = &msg->msg_ds; 348 349 /* Post Send Work Request on the specified channel */ 350 status = ibt_post_send(chan->ic_chanhdl, &wr, 1, NULL); 351 if (status != IBT_SUCCESS) { 352 ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: ibt_post_send " 353 "failure (%d)", status); 354 iser_msg_free(msg); 355 iser_wr_free(iser_wr); 356 return (ISER_STATUS_FAIL); 357 } 358 /* Increment this channel's SQ posted count */ 359 mutex_enter(&chan->ic_sq_post_lock); 360 chan->ic_sq_post_count++; 361 if (chan->ic_sq_post_count > chan->ic_sq_max_post_count) 362 chan->ic_sq_max_post_count = chan->ic_sq_post_count; 363 mutex_exit(&chan->ic_sq_post_lock); 364 365 return (ISER_STATUS_SUCCESS); 366 } 367 368 /* 369 * iser_xfer_buf_to_ini 370 * This is iSER's implementation of the 'Put_Data' operational primitive. 371 * The iSCSI layer at the target invokes this function when it is ready to 372 * return the SCSI Read Data to the initiator. This function generates and 373 * sends an RDMA Write Message containing the read data to the initiator. 374 */ 375 int 376 iser_xfer_buf_to_ini(idm_task_t *idt, idm_buf_t *buf) 377 { 378 iser_conn_t *iser_conn; 379 iser_chan_t *iser_chan; 380 iser_buf_t *iser_buf; 381 iser_wr_t *iser_wr; 382 iser_ctrl_hdr_t *iser_hdr; 383 ibt_send_wr_t wr; 384 uint64_t reg_raddr; 385 uint32_t reg_rkey; 386 int status; 387 388 /* Grab the iSER resources from the task and buf handles */ 389 iser_conn = (iser_conn_t *)idt->idt_ic->ic_transport_private; 390 iser_chan = iser_conn->ic_chan; 391 iser_buf = (iser_buf_t *)buf->idb_buf_private; 392 iser_hdr = (iser_ctrl_hdr_t *)idt->idt_transport_hdr; 393 394 /* Pull the Read STag data out of the iSER header in the task hdl */ 395 reg_raddr = BE_IN64(&iser_hdr->rva); 396 reg_rkey = (ntohl(iser_hdr->rstag)); 397 398 /* Set up the WR raddr and rkey based upon the Read iSER STag */ 399 bzero(&wr, sizeof (ibt_send_wr_t)); 400 wr.wr.rc.rcwr.rdma.rdma_raddr = reg_raddr + buf->idb_bufoffset; 401 wr.wr.rc.rcwr.rdma.rdma_rkey = reg_rkey; 402 403 /* Set the transfer length from the IDM buf handle */ 404 iser_buf->buf_ds.ds_len = buf->idb_xfer_len; 405 406 /* Allocate an iSER WR handle and tuck the IDM buf handle into it */ 407 iser_wr = iser_wr_get(); 408 if (iser_wr == NULL) { 409 ISER_LOG(CE_NOTE, "iser_xfer_buf_to_ini: unable to allocate " 410 "iser wr handle"); 411 return (ISER_STATUS_FAIL); 412 } 413 iser_wr->iw_buf = buf; 414 iser_wr->iw_type = ISER_WR_RDMAW; 415 416 /* Use the address of our generic iser_wr handle as our WRID */ 417 wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr; 418 419 /* Populate the rest of the work request */ 420 wr.wr_flags = IBT_WR_SEND_SIGNAL; 421 wr.wr_trans = IBT_RC_SRV; 422 wr.wr_opcode = IBT_WRC_RDMAW; 423 wr.wr_nds = 1; 424 wr.wr_sgl = &iser_buf->buf_ds; 425 426 #ifdef DEBUG 427 bcopy(&wr, &iser_buf->buf_wr, sizeof (ibt_send_wr_t)); 428 #endif 429 430 DTRACE_ISCSI_8(xfer__start, idm_conn_t *, idt->idt_ic, 431 uintptr_t, buf->idb_buf, uint32_t, buf->idb_bufoffset, 432 uint64_t, reg_raddr, uint32_t, buf->idb_bufoffset, 433 uint32_t, reg_rkey, 434 uint32_t, buf->idb_xfer_len, int, XFER_BUF_TX_TO_INI); 435 436 status = ibt_post_send(iser_chan->ic_chanhdl, &wr, 1, NULL); 437 if (status != IBT_SUCCESS) { 438 ISER_LOG(CE_NOTE, "iser_xfer_buf_to_ini: ibt_post_send " 439 "failure (%d)", status); 440 iser_wr_free(iser_wr); 441 return (ISER_STATUS_FAIL); 442 } 443 /* Increment this channel's SQ posted count */ 444 mutex_enter(&iser_chan->ic_sq_post_lock); 445 iser_chan->ic_sq_post_count++; 446 if (iser_chan->ic_sq_post_count > iser_chan->ic_sq_max_post_count) 447 iser_chan->ic_sq_max_post_count = iser_chan->ic_sq_post_count; 448 mutex_exit(&iser_chan->ic_sq_post_lock); 449 450 return (ISER_STATUS_SUCCESS); 451 } 452 453 /* 454 * iser_xfer_buf_from_ini 455 * This is iSER's implementation of the 'Get_Data' operational primitive. 456 * The iSCSI layer at the target invokes this function when it is ready to 457 * receive the SCSI Write Data from the initiator. This function generates and 458 * sends an RDMA Read Message to get the data from the initiator. No R2T PDUs 459 * are generated. 460 */ 461 int 462 iser_xfer_buf_from_ini(idm_task_t *idt, idm_buf_t *buf) 463 { 464 iser_conn_t *iser_conn; 465 iser_chan_t *iser_chan; 466 iser_buf_t *iser_buf; 467 iser_wr_t *iser_wr; 468 iser_ctrl_hdr_t *iser_hdr; 469 ibt_send_wr_t wr; 470 uint64_t reg_raddr; 471 uint32_t reg_rkey; 472 int status; 473 474 /* Grab the iSER resources from the task and buf handles */ 475 iser_conn = (iser_conn_t *)idt->idt_ic->ic_transport_private; 476 iser_chan = iser_conn->ic_chan; 477 iser_buf = (iser_buf_t *)buf->idb_buf_private; 478 iser_hdr = (iser_ctrl_hdr_t *)idt->idt_transport_hdr; 479 480 /* Pull the Write STag data out of the iSER header in the task hdl */ 481 reg_raddr = BE_IN64(&iser_hdr->wva); 482 reg_rkey = (ntohl(iser_hdr->wstag)); 483 484 /* Set up the WR raddr and rkey based upon the iSER Write STag */ 485 bzero(&wr, sizeof (ibt_send_wr_t)); 486 wr.wr.rc.rcwr.rdma.rdma_raddr = reg_raddr + buf->idb_bufoffset; 487 wr.wr.rc.rcwr.rdma.rdma_rkey = reg_rkey; 488 489 /* Set the transfer length from the IDM buf handle */ 490 iser_buf->buf_ds.ds_len = buf->idb_xfer_len; 491 492 /* Allocate an iSER WR handle and tuck the IDM buf handle into it */ 493 iser_wr = iser_wr_get(); 494 if (iser_wr == NULL) { 495 ISER_LOG(CE_NOTE, "iser_xfer_buf_from_ini: unable to allocate " 496 "iser wr handle"); 497 return (ISER_STATUS_FAIL); 498 } 499 iser_wr->iw_buf = buf; 500 iser_wr->iw_type = ISER_WR_RDMAR; 501 502 /* Use the address of our generic iser_wr handle as our WRID */ 503 wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr; 504 505 /* Populate the rest of the work request */ 506 wr.wr_flags = IBT_WR_SEND_SIGNAL; 507 wr.wr_trans = IBT_RC_SRV; 508 wr.wr_opcode = IBT_WRC_RDMAR; 509 wr.wr_nds = 1; 510 wr.wr_sgl = &iser_buf->buf_ds; 511 512 #ifdef DEBUG 513 bcopy(&wr, &iser_buf->buf_wr, sizeof (ibt_send_wr_t)); 514 #endif 515 516 DTRACE_ISCSI_8(xfer__start, idm_conn_t *, idt->idt_ic, 517 uintptr_t, buf->idb_buf, uint32_t, buf->idb_bufoffset, 518 uint64_t, reg_raddr, uint32_t, buf->idb_bufoffset, 519 uint32_t, reg_rkey, 520 uint32_t, buf->idb_xfer_len, int, XFER_BUF_RX_FROM_INI); 521 522 status = ibt_post_send(iser_chan->ic_chanhdl, &wr, 1, NULL); 523 if (status != IBT_SUCCESS) { 524 ISER_LOG(CE_NOTE, "iser_xfer_buf_from_ini: ibt_post_send " 525 "failure (%d)", status); 526 iser_wr_free(iser_wr); 527 return (ISER_STATUS_FAIL); 528 } 529 /* Increment this channel's SQ posted count */ 530 mutex_enter(&iser_chan->ic_sq_post_lock); 531 iser_chan->ic_sq_post_count++; 532 if (iser_chan->ic_sq_post_count > iser_chan->ic_sq_max_post_count) 533 iser_chan->ic_sq_max_post_count = iser_chan->ic_sq_post_count; 534 mutex_exit(&iser_chan->ic_sq_post_lock); 535 536 return (ISER_STATUS_SUCCESS); 537 } 538