1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/stat.h> 28 #include <sys/conf.h> 29 #include <sys/ddi.h> 30 #include <sys/sunddi.h> 31 #include <sys/modctl.h> 32 #include <sys/byteorder.h> 33 34 #include <sys/ib/clients/iser/iser.h> 35 36 /* 37 * iser_xfer.c 38 */ 39 40 int 41 iser_xfer_hello_msg(iser_chan_t *chan) 42 { 43 iser_hca_t *hca; 44 iser_wr_t *iser_wr; 45 iser_msg_t *msg; 46 ibt_send_wr_t wr; 47 iser_hello_hdr_t *hdr; 48 int status; 49 50 ASSERT(chan != NULL); 51 52 hca = (iser_hca_t *)chan->ic_hca; 53 if (hca == NULL) { 54 ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: no hca handle found"); 55 return (ISER_STATUS_FAIL); 56 } 57 58 msg = iser_msg_get(hca, 1, NULL); 59 60 if (msg == NULL) { 61 ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: iser message cache " 62 "alloc failed"); 63 return (ISER_STATUS_FAIL); 64 } 65 66 /* Send iSER Hello Message to declare iSER parameters to the target */ 67 hdr = (iser_hello_hdr_t *)(uintptr_t)msg->msg_ds.ds_va; 68 69 hdr->opcode = ISER_OPCODE_HELLO_MSG; 70 hdr->rsvd1 = 0; 71 hdr->maxver = 1; 72 hdr->minver = 1; 73 hdr->iser_ird = htons(ISER_IB_DEFAULT_IRD); 74 hdr->rsvd2[0] = 0; 75 hdr->rsvd2[1] = 0; 76 77 /* Allocate an iSER WR handle and tuck this msg into it */ 78 iser_wr = iser_wr_get(); 79 if (iser_wr == NULL) { 80 ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: unable to allocate " 81 "iser wr handle"); 82 iser_msg_free(msg); 83 return (ISER_STATUS_FAIL); 84 } 85 iser_wr->iw_msg = msg; 86 iser_wr->iw_type = ISER_WR_SEND; 87 88 /* Use the address of our generic iser_wr handle as our WRID */ 89 wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr; 90 91 /* Populate the rest of the work request */ 92 wr.wr_trans = IBT_RC_SRV; 93 wr.wr_opcode = IBT_WRC_SEND; 94 wr.wr_nds = 1; 95 wr.wr_sgl = &msg->msg_ds; 96 97 status = ibt_post_send(chan->ic_chanhdl, &wr, 1, NULL); 98 if (status != IBT_SUCCESS) { 99 ISER_LOG(CE_NOTE, "iser_xfer_hello_msg: ibt_post_send " 100 "failure (%d)", status); 101 iser_msg_free(msg); 102 iser_wr_free(iser_wr); 103 return (ISER_STATUS_FAIL); 104 } 105 /* Increment this channel's SQ posted count */ 106 mutex_enter(&chan->ic_sq_post_lock); 107 chan->ic_sq_post_count++; 108 if (chan->ic_sq_post_count > chan->ic_sq_max_post_count) 109 chan->ic_sq_max_post_count = chan->ic_sq_post_count; 110 mutex_exit(&chan->ic_sq_post_lock); 111 112 ISER_LOG(CE_NOTE, "Posting iSER Hello message: chan (0x%p): " 113 "IP [%x to %x]", (void *)chan, chan->ic_localip.un.ip4addr, 114 chan->ic_remoteip.un.ip4addr); 115 116 return (ISER_STATUS_SUCCESS); 117 } 118 119 int 120 iser_xfer_helloreply_msg(iser_chan_t *chan) 121 { 122 iser_hca_t *hca; 123 iser_wr_t *iser_wr; 124 ibt_send_wr_t wr; 125 iser_msg_t *msg; 126 iser_helloreply_hdr_t *hdr; 127 int status; 128 129 ASSERT(chan != NULL); 130 131 hca = (iser_hca_t *)chan->ic_hca; 132 if (hca == NULL) { 133 ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: no hca handle " 134 "found"); 135 return (ISER_STATUS_FAIL); 136 } 137 138 msg = iser_msg_get(hca, 1, NULL); 139 140 if (msg == NULL) { 141 ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: iser message " 142 "cache alloc failed"); 143 return (ISER_STATUS_FAIL); 144 } 145 146 /* Use the iSER Hello Reply Message */ 147 hdr = (iser_helloreply_hdr_t *)(uintptr_t)msg->msg_ds.ds_va; 148 149 hdr->opcode = ISER_OPCODE_HELLOREPLY_MSG; 150 hdr->rsvd1 = 0; 151 hdr->flag = 0; 152 hdr->maxver = 1; 153 hdr->curver = 1; 154 hdr->iser_ord = htons(ISER_IB_DEFAULT_ORD); 155 hdr->rsvd2[0] = 0; 156 hdr->rsvd2[1] = 0; 157 158 /* Allocate an iSER WR handle and tuck this msg into it */ 159 iser_wr = iser_wr_get(); 160 if (iser_wr == NULL) { 161 ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: unable to " 162 "allocate iser wr handle"); 163 iser_msg_free(msg); 164 return (ISER_STATUS_FAIL); 165 } 166 iser_wr->iw_msg = msg; 167 iser_wr->iw_type = ISER_WR_SEND; 168 169 /* Use the address of our generic iser_wr handle as our WRID */ 170 wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr; 171 172 /* Populate the rest of the work request */ 173 wr.wr_trans = IBT_RC_SRV; 174 wr.wr_opcode = IBT_WRC_SEND; 175 wr.wr_nds = 1; 176 wr.wr_sgl = &msg->msg_ds; 177 178 status = ibt_post_send(chan->ic_chanhdl, &wr, 1, NULL); 179 if (status != IBT_SUCCESS) { 180 ISER_LOG(CE_NOTE, "iser_xfer_helloreply_msg: ibt_post_send " 181 "failure (%d)", status); 182 iser_msg_free(msg); 183 iser_wr_free(iser_wr); 184 return (ISER_STATUS_FAIL); 185 } 186 /* Increment this channel's SQ posted count */ 187 mutex_enter(&chan->ic_sq_post_lock); 188 chan->ic_sq_post_count++; 189 if (chan->ic_sq_post_count > chan->ic_sq_max_post_count) 190 chan->ic_sq_max_post_count = chan->ic_sq_post_count; 191 mutex_exit(&chan->ic_sq_post_lock); 192 193 ISER_LOG(CE_NOTE, "Posting iSER HelloReply message: chan (0x%p): " 194 "IP [%x to %x]", (void *)chan, chan->ic_localip.un.ip4addr, 195 chan->ic_remoteip.un.ip4addr); 196 197 return (ISER_STATUS_SUCCESS); 198 } 199 200 /* 201 * iser_xfer_ctrlpdu 202 * 203 * This is iSER's implementation of the 'Send_control' operational primitive. 204 * This iSER layer uses the Send Message type of RCaP to transfer the iSCSI 205 * Control-type PDU. A special case is that the transfer of SCSI Data-out PDUs 206 * carrying unsolicited data are also treated as iSCSI Control-Type PDUs. The 207 * message payload contains an iSER header followed by the iSCSI Control-type 208 * the iSCSI Control-type PDU. 209 * This function is invoked by an initiator iSCSI layer requesting the transfer 210 * of a iSCSI command PDU or a target iSCSI layer requesting the transfer of a 211 * iSCSI response PDU. 212 */ 213 int 214 iser_xfer_ctrlpdu(iser_chan_t *chan, idm_pdu_t *pdu) 215 { 216 iser_hca_t *hca; 217 iser_ctrl_hdr_t *hdr; 218 iser_msg_t *msg; 219 iser_wr_t *iser_wr; 220 ibt_send_wr_t wr; 221 int status; 222 iser_mr_t *mr; 223 iscsi_data_hdr_t *bhs; 224 idm_conn_t *ic; 225 idm_task_t *idt = NULL; 226 idm_buf_t *buf; 227 228 ASSERT(chan != NULL); 229 230 /* 231 * All SCSI command PDU (except SCSI Read and SCSI Write) and the SCSI 232 * Response PDU are sent to the remote end using the SendSE Message. 233 * 234 * Setup a Send Message for carrying the iSCSI control-type PDU 235 * preceeded by an iSER header. 236 */ 237 hca = (iser_hca_t *)chan->ic_hca; 238 if (hca == NULL) { 239 ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: no hca handle found"); 240 return (ISER_STATUS_FAIL); 241 } 242 243 msg = iser_msg_get(hca, 1, NULL); 244 if (msg == NULL) { 245 ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: iser message cache " 246 "alloc failed"); 247 return (ISER_STATUS_FAIL); 248 } 249 250 /* Pull the BHS out of the PDU handle */ 251 bhs = (iscsi_data_hdr_t *)pdu->isp_hdr; 252 253 ASSERT(chan->ic_conn != NULL && chan->ic_conn->ic_idmc != NULL); 254 ic = chan->ic_conn->ic_idmc; 255 ASSERT(ic != NULL); 256 257 hdr = (iser_ctrl_hdr_t *)(uintptr_t)msg->msg_ds.ds_va; 258 259 /* 260 * Initialize header assuming no transfers 261 */ 262 bzero(hdr, sizeof (*hdr)); 263 hdr->opcode = ISER_OPCODE_CTRL_TYPE_PDU; 264 265 /* 266 * On the initiator side, the task buffers will be used to identify 267 * if there are any buffers to be advertised 268 */ 269 if ((ic->ic_conn_type == CONN_TYPE_INI) && 270 ((bhs->opcode & ISCSI_OPCODE_MASK) == ISCSI_OP_SCSI_CMD) && 271 ((idt = idm_task_find(ic, bhs->itt, bhs->ttt)) != NULL)) { 272 273 if (!list_is_empty(&idt->idt_inbufv)) { 274 buf = idm_buf_find(&idt->idt_inbufv, 0); 275 ASSERT(buf != NULL); 276 277 mr = (iser_mr_t *)buf->idb_reg_private; 278 ASSERT(mr != NULL); 279 280 hdr->rsv_flag = 1; 281 hdr->rstag = htonl(mr->is_mrrkey); 282 BE_OUT64(&hdr->rva, mr->is_mrva); 283 } 284 285 if (!list_is_empty(&idt->idt_outbufv)) { 286 buf = idm_buf_find(&idt->idt_outbufv, 0); 287 ASSERT(buf != NULL); 288 289 mr = (iser_mr_t *)buf->idb_reg_private; 290 ASSERT(mr != NULL); 291 292 hdr->wsv_flag = 1; 293 hdr->wstag = htonl(mr->is_mrrkey); 294 BE_OUT64(&hdr->wva, mr->is_mrva); 295 } 296 297 /* Release our reference on the task */ 298 idm_task_rele(idt); 299 } 300 301 /* Copy the BHS after the iSER header */ 302 bcopy(pdu->isp_hdr, 303 (uint8_t *)(uintptr_t)msg->msg_ds.ds_va + ISER_HEADER_LENGTH, 304 pdu->isp_hdrlen); 305 306 if (pdu->isp_datalen > 0) { 307 /* Copy the isp_data after the PDU header */ 308 bcopy(pdu->isp_data, 309 (uint8_t *)(uintptr_t)msg->msg_ds.ds_va + 310 ISER_HEADER_LENGTH + pdu->isp_hdrlen, 311 pdu->isp_datalen); 312 313 /* Set the SGE's ds_len */ 314 msg->msg_ds.ds_len = ISER_HEADER_LENGTH + pdu->isp_hdrlen + 315 pdu->isp_datalen; 316 } else { 317 /* No data, so set the SGE's ds_len to the headers length */ 318 msg->msg_ds.ds_len = ISER_HEADER_LENGTH + pdu->isp_hdrlen; 319 } 320 321 /* 322 * Build Work Request to be posted on the Send Queue. 323 */ 324 bzero(&wr, sizeof (wr)); 325 326 /* Allocate an iSER WR handle and tuck the msg and pdu into it */ 327 iser_wr = iser_wr_get(); 328 if (iser_wr == NULL) { 329 ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: unable to allocate " 330 "iser wr handle"); 331 iser_msg_free(msg); 332 return (ISER_STATUS_FAIL); 333 } 334 iser_wr->iw_pdu = pdu; 335 iser_wr->iw_msg = msg; 336 iser_wr->iw_type = ISER_WR_SEND; 337 338 /* 339 * Use the address of our generic iser_wr handle as our WRID 340 * and populate the rest of the work request 341 */ 342 wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr; 343 wr.wr_trans = IBT_RC_SRV; 344 wr.wr_opcode = IBT_WRC_SEND; 345 wr.wr_nds = 1; 346 wr.wr_sgl = &msg->msg_ds; 347 348 /* Post Send Work Request on the specified channel */ 349 status = ibt_post_send(chan->ic_chanhdl, &wr, 1, NULL); 350 if (status != IBT_SUCCESS) { 351 ISER_LOG(CE_NOTE, "iser_xfer_ctrlpdu: ibt_post_send " 352 "failure (%d)", status); 353 iser_msg_free(msg); 354 iser_wr_free(iser_wr); 355 return (ISER_STATUS_FAIL); 356 } 357 /* Increment this channel's SQ posted count */ 358 mutex_enter(&chan->ic_sq_post_lock); 359 chan->ic_sq_post_count++; 360 if (chan->ic_sq_post_count > chan->ic_sq_max_post_count) 361 chan->ic_sq_max_post_count = chan->ic_sq_post_count; 362 mutex_exit(&chan->ic_sq_post_lock); 363 364 return (ISER_STATUS_SUCCESS); 365 } 366 367 /* 368 * iser_xfer_buf_to_ini 369 * This is iSER's implementation of the 'Put_Data' operational primitive. 370 * The iSCSI layer at the target invokes this function when it is ready to 371 * return the SCSI Read Data to the initiator. This function generates and 372 * sends an RDMA Write Message containing the read data to the initiator. 373 */ 374 int 375 iser_xfer_buf_to_ini(idm_task_t *idt, idm_buf_t *buf) 376 { 377 iser_conn_t *iser_conn; 378 iser_chan_t *iser_chan; 379 iser_buf_t *iser_buf; 380 iser_wr_t *iser_wr; 381 iser_ctrl_hdr_t *iser_hdr; 382 ibt_send_wr_t wr; 383 uint64_t reg_raddr; 384 uint32_t reg_rkey; 385 int status; 386 387 /* Grab the iSER resources from the task and buf handles */ 388 iser_conn = (iser_conn_t *)idt->idt_ic->ic_transport_private; 389 iser_chan = iser_conn->ic_chan; 390 iser_buf = (iser_buf_t *)buf->idb_buf_private; 391 iser_hdr = (iser_ctrl_hdr_t *)idt->idt_transport_hdr; 392 393 /* Pull the Read STag data out of the iSER header in the task hdl */ 394 reg_raddr = BE_IN64(&iser_hdr->rva); 395 reg_rkey = (ntohl(iser_hdr->rstag)); 396 397 /* Set up the WR raddr and rkey based upon the Read iSER STag */ 398 bzero(&wr, sizeof (ibt_send_wr_t)); 399 wr.wr.rc.rcwr.rdma.rdma_raddr = reg_raddr + buf->idb_bufoffset; 400 wr.wr.rc.rcwr.rdma.rdma_rkey = reg_rkey; 401 402 /* Set the transfer length from the IDM buf handle */ 403 iser_buf->buf_ds.ds_len = buf->idb_xfer_len; 404 405 /* Allocate an iSER WR handle and tuck the IDM buf handle into it */ 406 iser_wr = iser_wr_get(); 407 if (iser_wr == NULL) { 408 ISER_LOG(CE_NOTE, "iser_xfer_buf_to_ini: unable to allocate " 409 "iser wr handle"); 410 return (ISER_STATUS_FAIL); 411 } 412 iser_wr->iw_buf = buf; 413 iser_wr->iw_type = ISER_WR_RDMAW; 414 415 /* Use the address of our generic iser_wr handle as our WRID */ 416 wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr; 417 418 /* Populate the rest of the work request */ 419 wr.wr_flags = IBT_WR_SEND_SIGNAL; 420 wr.wr_trans = IBT_RC_SRV; 421 wr.wr_opcode = IBT_WRC_RDMAW; 422 wr.wr_nds = 1; 423 wr.wr_sgl = &iser_buf->buf_ds; 424 425 #ifdef DEBUG 426 bcopy(&wr, &iser_buf->buf_wr, sizeof (ibt_send_wr_t)); 427 #endif 428 429 status = ibt_post_send(iser_chan->ic_chanhdl, &wr, 1, NULL); 430 if (status != IBT_SUCCESS) { 431 ISER_LOG(CE_NOTE, "iser_xfer_buf_to_ini: ibt_post_send " 432 "failure (%d)", status); 433 iser_wr_free(iser_wr); 434 return (ISER_STATUS_FAIL); 435 } 436 /* Increment this channel's SQ posted count */ 437 mutex_enter(&iser_chan->ic_sq_post_lock); 438 iser_chan->ic_sq_post_count++; 439 if (iser_chan->ic_sq_post_count > iser_chan->ic_sq_max_post_count) 440 iser_chan->ic_sq_max_post_count = iser_chan->ic_sq_post_count; 441 mutex_exit(&iser_chan->ic_sq_post_lock); 442 443 return (ISER_STATUS_SUCCESS); 444 } 445 446 /* 447 * iser_xfer_buf_from_ini 448 * This is iSER's implementation of the 'Get_Data' operational primitive. 449 * The iSCSI layer at the target invokes this function when it is ready to 450 * receive the SCSI Write Data from the initiator. This function generates and 451 * sends an RDMA Read Message to get the data from the initiator. No R2T PDUs 452 * are generated. 453 */ 454 int 455 iser_xfer_buf_from_ini(idm_task_t *idt, idm_buf_t *buf) 456 { 457 iser_conn_t *iser_conn; 458 iser_chan_t *iser_chan; 459 iser_buf_t *iser_buf; 460 iser_wr_t *iser_wr; 461 iser_ctrl_hdr_t *iser_hdr; 462 ibt_send_wr_t wr; 463 uint64_t reg_raddr; 464 uint32_t reg_rkey; 465 int status; 466 467 /* Grab the iSER resources from the task and buf handles */ 468 iser_conn = (iser_conn_t *)idt->idt_ic->ic_transport_private; 469 iser_chan = iser_conn->ic_chan; 470 iser_buf = (iser_buf_t *)buf->idb_buf_private; 471 iser_hdr = (iser_ctrl_hdr_t *)idt->idt_transport_hdr; 472 473 /* Pull the Write STag data out of the iSER header in the task hdl */ 474 reg_raddr = BE_IN64(&iser_hdr->wva); 475 reg_rkey = (ntohl(iser_hdr->wstag)); 476 477 /* Set up the WR raddr and rkey based upon the iSER Write STag */ 478 bzero(&wr, sizeof (ibt_send_wr_t)); 479 wr.wr.rc.rcwr.rdma.rdma_raddr = reg_raddr + buf->idb_bufoffset; 480 wr.wr.rc.rcwr.rdma.rdma_rkey = reg_rkey; 481 482 /* Set the transfer length from the IDM buf handle */ 483 iser_buf->buf_ds.ds_len = buf->idb_xfer_len; 484 485 /* Allocate an iSER WR handle and tuck the IDM buf handle into it */ 486 iser_wr = iser_wr_get(); 487 if (iser_wr == NULL) { 488 ISER_LOG(CE_NOTE, "iser_xfer_buf_from_ini: unable to allocate " 489 "iser wr handle"); 490 return (ISER_STATUS_FAIL); 491 } 492 iser_wr->iw_buf = buf; 493 iser_wr->iw_type = ISER_WR_RDMAR; 494 495 /* Use the address of our generic iser_wr handle as our WRID */ 496 wr.wr_id = (ibt_wrid_t)(uintptr_t)iser_wr; 497 498 /* Populate the rest of the work request */ 499 wr.wr_flags = IBT_WR_SEND_SIGNAL; 500 wr.wr_trans = IBT_RC_SRV; 501 wr.wr_opcode = IBT_WRC_RDMAR; 502 wr.wr_nds = 1; 503 wr.wr_sgl = &iser_buf->buf_ds; 504 505 #ifdef DEBUG 506 bcopy(&wr, &iser_buf->buf_wr, sizeof (ibt_send_wr_t)); 507 #endif 508 509 status = ibt_post_send(iser_chan->ic_chanhdl, &wr, 1, NULL); 510 if (status != IBT_SUCCESS) { 511 ISER_LOG(CE_NOTE, "iser_xfer_buf_from_ini: ibt_post_send " 512 "failure (%d)", status); 513 iser_wr_free(iser_wr); 514 return (ISER_STATUS_FAIL); 515 } 516 /* Increment this channel's SQ posted count */ 517 mutex_enter(&iser_chan->ic_sq_post_lock); 518 iser_chan->ic_sq_post_count++; 519 if (iser_chan->ic_sq_post_count > iser_chan->ic_sq_max_post_count) 520 iser_chan->ic_sq_max_post_count = iser_chan->ic_sq_post_count; 521 mutex_exit(&iser_chan->ic_sq_post_lock); 522 523 return (ISER_STATUS_SUCCESS); 524 } 525