1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * tavor_ioctl.c 29 * Tavor IOCTL Routines 30 * 31 * Implements all ioctl access into the driver. This includes all routines 32 * necessary for updating firmware, accessing the tavor flash device, and 33 * providing interfaces for VTS. 34 */ 35 36 #include <sys/types.h> 37 #include <sys/conf.h> 38 #include <sys/ddi.h> 39 #include <sys/sunddi.h> 40 #include <sys/modctl.h> 41 #include <sys/file.h> 42 43 #include <sys/ib/adapters/tavor/tavor.h> 44 45 /* Tavor HCA state pointer (extern) */ 46 extern void *tavor_statep; 47 48 /* 49 * The ioctl declarations (for firmware flash burning, register read/write 50 * (DEBUG-only), and VTS interfaces) 51 */ 52 static int tavor_ioctl_flash_read(tavor_state_t *state, dev_t dev, 53 intptr_t arg, int mode); 54 static int tavor_ioctl_flash_write(tavor_state_t *state, dev_t dev, 55 intptr_t arg, int mode); 56 static int tavor_ioctl_flash_erase(tavor_state_t *state, dev_t dev, 57 intptr_t arg, int mode); 58 static int tavor_ioctl_flash_init(tavor_state_t *state, dev_t dev, 59 intptr_t arg, int mode); 60 static int tavor_ioctl_flash_fini(tavor_state_t *state, dev_t dev); 61 static void tavor_ioctl_flash_cleanup(tavor_state_t *state); 62 static void tavor_ioctl_flash_cleanup_nolock(tavor_state_t *state); 63 #ifdef DEBUG 64 static int tavor_ioctl_reg_write(tavor_state_t *state, intptr_t arg, 65 int mode); 66 static int tavor_ioctl_reg_read(tavor_state_t *state, intptr_t arg, 67 int mode); 68 #endif /* DEBUG */ 69 static int tavor_ioctl_info(tavor_state_t *state, dev_t dev, 70 intptr_t arg, int mode); 71 static int tavor_ioctl_ports(tavor_state_t *state, intptr_t arg, 72 int mode); 73 static int tavor_ioctl_loopback(tavor_state_t *state, intptr_t arg, 74 int mode); 75 static int tavor_ioctl_ddr_read(tavor_state_t *state, intptr_t arg, 76 int mode); 77 78 /* Tavor Flash Functions */ 79 static void tavor_flash_read_sector(tavor_state_t *state, uint32_t sector_num); 80 static void tavor_flash_read_quadlet(tavor_state_t *state, uint32_t *data, 81 uint32_t addr); 82 static int tavor_flash_write_sector(tavor_state_t *state, uint32_t sector_num); 83 static int tavor_flash_write_byte(tavor_state_t *state, uint32_t addr, 84 uchar_t data); 85 static int tavor_flash_erase_sector(tavor_state_t *state, uint32_t sector_num); 86 static int tavor_flash_erase_chip(tavor_state_t *state); 87 static void tavor_flash_bank(tavor_state_t *state, uint32_t addr); 88 static uint32_t tavor_flash_read(tavor_state_t *state, uint32_t addr); 89 static void tavor_flash_write(tavor_state_t *state, uint32_t addr, 90 uchar_t data); 91 static void tavor_flash_init(tavor_state_t *state); 92 static void tavor_flash_cfi_init(tavor_state_t *state, uint32_t *cfi_info, 93 int *intel_xcmd); 94 static void tavor_flash_fini(tavor_state_t *state); 95 static void tavor_flash_reset(tavor_state_t *state); 96 static uint32_t tavor_flash_read_cfg(ddi_acc_handle_t pci_config_hdl, 97 uint32_t addr); 98 static void tavor_flash_write_cfg(ddi_acc_handle_t pci_config_hdl, 99 uint32_t addr, uint32_t data); 100 static void tavor_flash_cfi_byte(uint8_t *ch, uint32_t dword, int i); 101 static void tavor_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i); 102 103 /* Tavor loopback test functions */ 104 static void tavor_loopback_free_qps(tavor_loopback_state_t *lstate); 105 static void tavor_loopback_free_state(tavor_loopback_state_t *lstate); 106 static int tavor_loopback_init(tavor_state_t *state, 107 tavor_loopback_state_t *lstate); 108 static void tavor_loopback_init_qp_info(tavor_loopback_state_t *lstate, 109 tavor_loopback_comm_t *comm); 110 static int tavor_loopback_alloc_mem(tavor_loopback_state_t *lstate, 111 tavor_loopback_comm_t *comm, int sz); 112 static int tavor_loopback_alloc_qps(tavor_loopback_state_t *lstate, 113 tavor_loopback_comm_t *comm); 114 static int tavor_loopback_modify_qp(tavor_loopback_state_t *lstate, 115 tavor_loopback_comm_t *comm, uint_t qp_num); 116 static int tavor_loopback_copyout(tavor_loopback_ioctl_t *lb, 117 intptr_t arg, int mode); 118 static int tavor_loopback_post_send(tavor_loopback_state_t *lstate, 119 tavor_loopback_comm_t *tx, tavor_loopback_comm_t *rx); 120 static int tavor_loopback_poll_cq(tavor_loopback_state_t *lstate, 121 tavor_loopback_comm_t *comm); 122 123 /* Patchable timeout values for flash operations */ 124 int tavor_hw_flash_timeout_gpio_sema = TAVOR_HW_FLASH_TIMEOUT_GPIO_SEMA; 125 int tavor_hw_flash_timeout_config = TAVOR_HW_FLASH_TIMEOUT_CONFIG; 126 int tavor_hw_flash_timeout_write = TAVOR_HW_FLASH_TIMEOUT_WRITE; 127 int tavor_hw_flash_timeout_erase = TAVOR_HW_FLASH_TIMEOUT_ERASE; 128 129 /* 130 * tavor_ioctl() 131 */ 132 /* ARGSUSED */ 133 int 134 tavor_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp, 135 int *rvalp) 136 { 137 tavor_state_t *state; 138 minor_t instance; 139 int status; 140 141 TAVOR_TNF_ENTER(tavor_ioctl); 142 143 if (drv_priv(credp) != 0) { 144 TNF_PROBE_0(tavor_ioctl_priv_fail, TAVOR_TNF_ERROR, ""); 145 TAVOR_TNF_EXIT(tavor_ioctl); 146 return (EPERM); 147 } 148 149 instance = TAVOR_DEV_INSTANCE(dev); 150 if (instance == -1) { 151 TNF_PROBE_0(tavor_ioctl_inst_fail, TAVOR_TNF_ERROR, ""); 152 TAVOR_TNF_EXIT(tavor_ioctl); 153 return (EBADF); 154 } 155 156 state = ddi_get_soft_state(tavor_statep, instance); 157 if (state == NULL) { 158 TNF_PROBE_0(tavor_ioctl_gss_fail, TAVOR_TNF_ERROR, ""); 159 TAVOR_TNF_EXIT(tavor_ioctl); 160 return (EBADF); 161 } 162 163 status = 0; 164 165 switch (cmd) { 166 case TAVOR_IOCTL_FLASH_READ: 167 status = tavor_ioctl_flash_read(state, dev, arg, mode); 168 break; 169 170 case TAVOR_IOCTL_FLASH_WRITE: 171 status = tavor_ioctl_flash_write(state, dev, arg, mode); 172 break; 173 174 case TAVOR_IOCTL_FLASH_ERASE: 175 status = tavor_ioctl_flash_erase(state, dev, arg, mode); 176 break; 177 178 case TAVOR_IOCTL_FLASH_INIT: 179 status = tavor_ioctl_flash_init(state, dev, arg, mode); 180 break; 181 182 case TAVOR_IOCTL_FLASH_FINI: 183 status = tavor_ioctl_flash_fini(state, dev); 184 break; 185 186 case TAVOR_IOCTL_INFO: 187 status = tavor_ioctl_info(state, dev, arg, mode); 188 break; 189 190 case TAVOR_IOCTL_PORTS: 191 status = tavor_ioctl_ports(state, arg, mode); 192 break; 193 194 case TAVOR_IOCTL_DDR_READ: 195 status = tavor_ioctl_ddr_read(state, arg, mode); 196 break; 197 198 case TAVOR_IOCTL_LOOPBACK: 199 status = tavor_ioctl_loopback(state, arg, mode); 200 break; 201 202 #ifdef DEBUG 203 case TAVOR_IOCTL_REG_WRITE: 204 status = tavor_ioctl_reg_write(state, arg, mode); 205 break; 206 207 case TAVOR_IOCTL_REG_READ: 208 status = tavor_ioctl_reg_read(state, arg, mode); 209 break; 210 #endif /* DEBUG */ 211 212 default: 213 status = ENOTTY; 214 TNF_PROBE_0(tavor_ioctl_default_fail, TAVOR_TNF_ERROR, ""); 215 break; 216 } 217 *rvalp = status; 218 219 TAVOR_TNF_EXIT(tavor_ioctl); 220 return (status); 221 } 222 223 /* 224 * tavor_ioctl_flash_read() 225 */ 226 static int 227 tavor_ioctl_flash_read(tavor_state_t *state, dev_t dev, intptr_t arg, int mode) 228 { 229 tavor_flash_ioctl_t ioctl_info; 230 int status = 0; 231 232 TAVOR_TNF_ENTER(tavor_ioctl_flash_read); 233 234 /* 235 * Check that flash init ioctl has been called first. And check 236 * that the same dev_t that called init is the one calling read now. 237 */ 238 mutex_enter(&state->ts_fw_flashlock); 239 if ((state->ts_fw_flashdev != dev) || 240 (state->ts_fw_flashstarted == 0)) { 241 mutex_exit(&state->ts_fw_flashlock); 242 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, ""); 243 TAVOR_TNF_EXIT(tavor_ioctl_flash_read); 244 return (EIO); 245 } 246 247 /* copy user struct to kernel */ 248 #ifdef _MULTI_DATAMODEL 249 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) { 250 tavor_flash_ioctl32_t info32; 251 252 if (ddi_copyin((void *)arg, &info32, 253 sizeof (tavor_flash_ioctl32_t), mode) != 0) { 254 mutex_exit(&state->ts_fw_flashlock); 255 TNF_PROBE_0(tavor_ioctl_flash_read_copyin_fail, 256 TAVOR_TNF_ERROR, ""); 257 TAVOR_TNF_EXIT(tavor_ioctl_flash_read); 258 return (EFAULT); 259 } 260 ioctl_info.tf_type = info32.tf_type; 261 ioctl_info.tf_sector = (caddr_t)(uintptr_t)info32.tf_sector; 262 ioctl_info.tf_sector_num = info32.tf_sector_num; 263 ioctl_info.tf_addr = info32.tf_addr; 264 } else 265 #endif /* _MULTI_DATAMODEL */ 266 if (ddi_copyin((void *)arg, &ioctl_info, sizeof (tavor_flash_ioctl_t), 267 mode) != 0) { 268 mutex_exit(&state->ts_fw_flashlock); 269 TNF_PROBE_0(tavor_ioctl_flash_read_copyin_fail, 270 TAVOR_TNF_ERROR, ""); 271 TAVOR_TNF_EXIT(tavor_ioctl_flash_read); 272 return (EFAULT); 273 } 274 275 /* 276 * Determine type of READ ioctl 277 */ 278 switch (ioctl_info.tf_type) { 279 case TAVOR_FLASH_READ_SECTOR: 280 /* Check if sector num is too large for flash device */ 281 if (ioctl_info.tf_sector_num >= 282 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) { 283 mutex_exit(&state->ts_fw_flashlock); 284 TNF_PROBE_0(tavor_flash_read_sector_num_too_large, 285 TAVOR_TNF_ERROR, ""); 286 TAVOR_TNF_EXIT(tavor_ioctl_flash_read); 287 return (EFAULT); 288 } 289 290 /* Perform the Sector Read */ 291 tavor_flash_reset(state); 292 tavor_flash_read_sector(state, ioctl_info.tf_sector_num); 293 294 /* copyout the firmware sector image data */ 295 if (ddi_copyout(&state->ts_fw_sector[0], 296 &ioctl_info.tf_sector[0], 1 << state->ts_fw_log_sector_sz, 297 mode) != 0) { 298 mutex_exit(&state->ts_fw_flashlock); 299 TNF_PROBE_0(tavor_flash_read_copyout_fail, 300 TAVOR_TNF_ERROR, ""); 301 TAVOR_TNF_EXIT(tavor_ioctl_flash_read); 302 return (EFAULT); 303 } 304 break; 305 306 case TAVOR_FLASH_READ_QUADLET: 307 /* Check if addr is too large for flash device */ 308 if (ioctl_info.tf_addr >= state->ts_fw_device_sz) { 309 mutex_exit(&state->ts_fw_flashlock); 310 TNF_PROBE_0(tavor_flash_read_quad_addr_too_large, 311 TAVOR_TNF_ERROR, ""); 312 TAVOR_TNF_EXIT(tavor_ioctl_flash_read); 313 return (EFAULT); 314 } 315 316 /* Perform the Quadlet Read */ 317 tavor_flash_reset(state); 318 tavor_flash_read_quadlet(state, &ioctl_info.tf_quadlet, 319 ioctl_info.tf_addr); 320 break; 321 322 default: 323 TNF_PROBE_0(tavor_ioctl_flash_read_invalid_type, 324 TAVOR_TNF_ERROR, ""); 325 status = EIO; 326 break; 327 } 328 329 /* copy results back to userland */ 330 #ifdef _MULTI_DATAMODEL 331 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) { 332 tavor_flash_ioctl32_t info32; 333 334 info32.tf_quadlet = ioctl_info.tf_quadlet; 335 info32.tf_type = ioctl_info.tf_type; 336 info32.tf_sector_num = ioctl_info.tf_sector_num; 337 info32.tf_sector = (caddr32_t)(uintptr_t)ioctl_info.tf_sector; 338 info32.tf_addr = ioctl_info.tf_addr; 339 340 if (ddi_copyout(&info32, (void *)arg, 341 sizeof (tavor_flash_ioctl32_t), mode) != 0) { 342 mutex_exit(&state->ts_fw_flashlock); 343 TNF_PROBE_0(tavor_flash_read_copyout_fail, 344 TAVOR_TNF_ERROR, ""); 345 TAVOR_TNF_EXIT(tavor_ioctl_flash_read); 346 return (EFAULT); 347 } 348 } else 349 #endif /* _MULTI_DATAMODEL */ 350 if (ddi_copyout(&ioctl_info, (void *)arg, 351 sizeof (tavor_flash_ioctl_t), mode) != 0) { 352 mutex_exit(&state->ts_fw_flashlock); 353 TNF_PROBE_0(tavor_flash_read_copyout_fail, 354 TAVOR_TNF_ERROR, ""); 355 TAVOR_TNF_EXIT(tavor_ioctl_flash_read); 356 return (EFAULT); 357 } 358 359 mutex_exit(&state->ts_fw_flashlock); 360 TAVOR_TNF_EXIT(tavor_ioctl_flash_read); 361 return (status); 362 } 363 364 /* 365 * tavor_ioctl_flash_write() 366 */ 367 static int 368 tavor_ioctl_flash_write(tavor_state_t *state, dev_t dev, intptr_t arg, int mode) 369 { 370 tavor_flash_ioctl_t ioctl_info; 371 int status = 0; 372 373 TAVOR_TNF_ENTER(tavor_ioctl_flash_write); 374 375 /* 376 * Check that flash init ioctl has been called first. And check 377 * that the same dev_t that called init is the one calling write now. 378 */ 379 mutex_enter(&state->ts_fw_flashlock); 380 if ((state->ts_fw_flashdev != dev) || 381 (state->ts_fw_flashstarted == 0)) { 382 mutex_exit(&state->ts_fw_flashlock); 383 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, ""); 384 TAVOR_TNF_EXIT(tavor_ioctl_flash_write); 385 return (EIO); 386 } 387 388 /* copy user struct to kernel */ 389 #ifdef _MULTI_DATAMODEL 390 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) { 391 tavor_flash_ioctl32_t info32; 392 393 if (ddi_copyin((void *)arg, &info32, 394 sizeof (tavor_flash_ioctl32_t), mode) != 0) { 395 mutex_exit(&state->ts_fw_flashlock); 396 TNF_PROBE_0(tavor_ioctl_flash_write_copyin_fail, 397 TAVOR_TNF_ERROR, ""); 398 TAVOR_TNF_EXIT(tavor_ioctl_flash_write); 399 return (EFAULT); 400 } 401 ioctl_info.tf_type = info32.tf_type; 402 ioctl_info.tf_sector = (caddr_t)(uintptr_t)info32.tf_sector; 403 ioctl_info.tf_sector_num = info32.tf_sector_num; 404 ioctl_info.tf_addr = info32.tf_addr; 405 ioctl_info.tf_byte = info32.tf_byte; 406 } else 407 #endif /* _MULTI_DATAMODEL */ 408 if (ddi_copyin((void *)arg, &ioctl_info, 409 sizeof (tavor_flash_ioctl_t), mode) != 0) { 410 mutex_exit(&state->ts_fw_flashlock); 411 TNF_PROBE_0(tavor_ioctl_flash_write_ci_fail, 412 TAVOR_TNF_ERROR, ""); 413 TAVOR_TNF_EXIT(tavor_ioctl_flash_write); 414 return (EFAULT); 415 } 416 417 /* 418 * Determine type of WRITE ioctl 419 */ 420 switch (ioctl_info.tf_type) { 421 case TAVOR_FLASH_WRITE_SECTOR: 422 /* Check if sector num is too large for flash device */ 423 if (ioctl_info.tf_sector_num >= 424 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) { 425 mutex_exit(&state->ts_fw_flashlock); 426 TNF_PROBE_0(tavor_flash_write_sector_num_too_large, 427 TAVOR_TNF_ERROR, ""); 428 TAVOR_TNF_EXIT(tavor_ioctl_flash_write); 429 return (EFAULT); 430 } 431 432 /* copy in fw sector image data */ 433 if (ddi_copyin(&ioctl_info.tf_sector[0], 434 &state->ts_fw_sector[0], 1 << state->ts_fw_log_sector_sz, 435 mode) != 0) { 436 mutex_exit(&state->ts_fw_flashlock); 437 TNF_PROBE_0(tavor_ioctl_flash_write_fw_sector_ci_fail, 438 TAVOR_TNF_ERROR, ""); 439 TAVOR_TNF_EXIT(tavor_ioctl_flash_write); 440 return (EFAULT); 441 } 442 443 /* Perform Write Sector */ 444 status = tavor_flash_write_sector(state, 445 ioctl_info.tf_sector_num); 446 break; 447 448 case TAVOR_FLASH_WRITE_BYTE: 449 /* Check if addr is too large for flash device */ 450 if (ioctl_info.tf_addr >= state->ts_fw_device_sz) { 451 mutex_exit(&state->ts_fw_flashlock); 452 TNF_PROBE_0(tavor_flash_write_byte_addr_too_large, 453 TAVOR_TNF_ERROR, ""); 454 TAVOR_TNF_EXIT(tavor_ioctl_flash_write); 455 return (EFAULT); 456 } 457 458 /* Perform Write Byte */ 459 tavor_flash_bank(state, ioctl_info.tf_addr); 460 tavor_flash_reset(state); 461 status = tavor_flash_write_byte(state, ioctl_info.tf_addr, 462 ioctl_info.tf_byte); 463 tavor_flash_reset(state); 464 break; 465 466 default: 467 TNF_PROBE_0(tavor_ioctl_flash_write_invalid_type, 468 TAVOR_TNF_ERROR, ""); 469 status = EIO; 470 break; 471 } 472 473 mutex_exit(&state->ts_fw_flashlock); 474 TAVOR_TNF_EXIT(tavor_ioctl_flash_write); 475 return (status); 476 } 477 478 /* 479 * tavor_ioctl_flash_erase() 480 */ 481 static int 482 tavor_ioctl_flash_erase(tavor_state_t *state, dev_t dev, intptr_t arg, int mode) 483 { 484 tavor_flash_ioctl_t ioctl_info; 485 int status = 0; 486 487 TAVOR_TNF_ENTER(tavor_ioctl_flash_erase); 488 489 /* 490 * Check that flash init ioctl has been called first. And check 491 * that the same dev_t that called init is the one calling erase now. 492 */ 493 mutex_enter(&state->ts_fw_flashlock); 494 if ((state->ts_fw_flashdev != dev) || 495 (state->ts_fw_flashstarted == 0)) { 496 mutex_exit(&state->ts_fw_flashlock); 497 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, ""); 498 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase); 499 return (EIO); 500 } 501 502 /* copy user struct to kernel */ 503 #ifdef _MULTI_DATAMODEL 504 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) { 505 tavor_flash_ioctl32_t info32; 506 507 if (ddi_copyin((void *)arg, &info32, 508 sizeof (tavor_flash_ioctl32_t), mode) != 0) { 509 mutex_exit(&state->ts_fw_flashlock); 510 TNF_PROBE_0(tavor_ioctl_flash_read_copyin_fail, 511 TAVOR_TNF_ERROR, ""); 512 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase); 513 return (EFAULT); 514 } 515 ioctl_info.tf_type = info32.tf_type; 516 ioctl_info.tf_sector_num = info32.tf_sector_num; 517 } else 518 #endif /* _MULTI_DATAMODEL */ 519 if (ddi_copyin((void *)arg, &ioctl_info, sizeof (tavor_flash_ioctl_t), 520 mode) != 0) { 521 mutex_exit(&state->ts_fw_flashlock); 522 TNF_PROBE_0(tavor_ioctl_flash_erase_ci_fail, 523 TAVOR_TNF_ERROR, ""); 524 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase); 525 return (EFAULT); 526 } 527 528 /* 529 * Determine type of ERASE ioctl 530 */ 531 switch (ioctl_info.tf_type) { 532 case TAVOR_FLASH_ERASE_SECTOR: 533 /* Check if sector num is too large for flash device */ 534 if (ioctl_info.tf_sector_num >= 535 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) { 536 mutex_exit(&state->ts_fw_flashlock); 537 TNF_PROBE_0(tavor_flash_erase_sector_num_too_large, 538 TAVOR_TNF_ERROR, ""); 539 TAVOR_TNF_EXIT(tavor_ioctl_flash_write); 540 return (EFAULT); 541 } 542 543 /* Perform Sector Erase */ 544 status = tavor_flash_erase_sector(state, 545 ioctl_info.tf_sector_num); 546 break; 547 548 case TAVOR_FLASH_ERASE_CHIP: 549 /* Perform Chip Erase */ 550 status = tavor_flash_erase_chip(state); 551 break; 552 553 default: 554 TNF_PROBE_0(tavor_ioctl_flash_erase_invalid_type, 555 TAVOR_TNF_ERROR, ""); 556 status = EIO; 557 break; 558 } 559 560 mutex_exit(&state->ts_fw_flashlock); 561 TAVOR_TNF_EXIT(tavor_ioctl_flash_erase); 562 return (status); 563 } 564 565 /* 566 * tavor_ioctl_flash_init() 567 */ 568 static int 569 tavor_ioctl_flash_init(tavor_state_t *state, dev_t dev, intptr_t arg, int mode) 570 { 571 tavor_flash_init_ioctl_t init_info; 572 int ret; 573 int intel_xcmd = 0; 574 575 TAVOR_TNF_ENTER(tavor_ioctl_flash_init); 576 577 /* 578 * init cannot be called more than once. If we have already init'd the 579 * flash, return directly. 580 */ 581 mutex_enter(&state->ts_fw_flashlock); 582 if (state->ts_fw_flashstarted == 1) { 583 mutex_exit(&state->ts_fw_flashlock); 584 TNF_PROBE_0(tavor_ioctl_flash_init_already_started, 585 TAVOR_TNF_ERROR, ""); 586 TAVOR_TNF_EXIT(tavor_ioctl_flash_init); 587 return (EIO); 588 } 589 590 /* copyin the user struct to kernel */ 591 if (ddi_copyin((void *)arg, &init_info, 592 sizeof (tavor_flash_init_ioctl_t), mode) != 0) { 593 mutex_exit(&state->ts_fw_flashlock); 594 TNF_PROBE_0(tavor_flash_init_ioctl_copyin_fail, 595 TAVOR_TNF_ERROR, ""); 596 TAVOR_TNF_EXIT(tavor_ioctl_flash_init); 597 return (EFAULT); 598 } 599 600 /* Init Flash */ 601 tavor_flash_init(state); 602 603 /* Read CFI info */ 604 tavor_flash_cfi_init(state, &init_info.tf_cfi_info[0], &intel_xcmd); 605 606 /* 607 * Return error if the command set is unknown. 608 */ 609 if (state->ts_fw_cmdset == TAVOR_FLASH_UNKNOWN_CMDSET) { 610 mutex_exit(&state->ts_fw_flashlock); 611 TNF_PROBE_1(tavor_ioctl_flash_init_cmdset_fail, 612 TAVOR_TNF_ERROR, "", tnf_string, errmsg, 613 "UNKNOWN flash command set"); 614 TAVOR_TNF_EXIT(tavor_ioctl_flash_init); 615 return (EFAULT); 616 } 617 618 /* Read HWREV - least significant 8 bits is revision ID */ 619 init_info.tf_hwrev = pci_config_get32(state->ts_pci_cfghdl, 620 TAVOR_HW_FLASH_CFG_HWREV) & 0xFF; 621 622 /* Fill in the firmwate revision numbers */ 623 init_info.tf_fwrev.tfi_maj = state->ts_fw.fw_rev_major; 624 init_info.tf_fwrev.tfi_min = state->ts_fw.fw_rev_minor; 625 init_info.tf_fwrev.tfi_sub = state->ts_fw.fw_rev_subminor; 626 627 /* Alloc flash mem for one sector size */ 628 state->ts_fw_sector = (uint32_t *)kmem_zalloc(1 << 629 state->ts_fw_log_sector_sz, KM_SLEEP); 630 631 /* Set HW part number and length */ 632 init_info.tf_pn_len = state->ts_hca_pn_len; 633 if (state->ts_hca_pn_len != 0) { 634 (void) memcpy(init_info.tf_hwpn, state->ts_hca_pn, 635 state->ts_hca_pn_len); 636 } 637 638 /* Copy ioctl results back to userland */ 639 if (ddi_copyout(&init_info, (void *)arg, 640 sizeof (tavor_flash_init_ioctl_t), mode) != 0) { 641 642 tavor_ioctl_flash_cleanup_nolock(state); 643 644 mutex_exit(&state->ts_fw_flashlock); 645 TNF_PROBE_0(tavor_ioctl_flash_init_copyout_fail, 646 TAVOR_TNF_ERROR, ""); 647 TAVOR_TNF_EXIT(tavor_ioctl_flash_init); 648 return (EFAULT); 649 } 650 651 /* Set flash state to started */ 652 state->ts_fw_flashstarted = 1; 653 state->ts_fw_flashdev = dev; 654 655 mutex_exit(&state->ts_fw_flashlock); 656 657 /* 658 * If "flash init" is successful, add an "on close" callback to the 659 * current dev node to ensure that "flash fini" gets called later 660 * even if the userland process prematurely exits. 661 */ 662 ret = tavor_umap_db_set_onclose_cb(dev, 663 TAVOR_ONCLOSE_FLASH_INPROGRESS, 664 (void (*)(void *))tavor_ioctl_flash_cleanup, state); 665 if (ret != DDI_SUCCESS) { 666 (void) tavor_ioctl_flash_fini(state, dev); 667 668 TNF_PROBE_0(tavor_ioctl_flash_init_set_cb_fail, 669 TAVOR_TNF_ERROR, ""); 670 TAVOR_TNF_EXIT(tavor_ioctl_flash_init); 671 return (EFAULT); 672 } 673 674 TAVOR_TNF_EXIT(tavor_ioctl_flash_init); 675 return (0); 676 } 677 678 /* 679 * tavor_ioctl_flash_fini() 680 */ 681 static int 682 tavor_ioctl_flash_fini(tavor_state_t *state, dev_t dev) 683 { 684 int ret; 685 686 TAVOR_TNF_ENTER(tavor_ioctl_flash_fini); 687 688 /* 689 * Check that flash init ioctl has been called first. And check 690 * that the same dev_t that called init is the one calling fini now. 691 */ 692 mutex_enter(&state->ts_fw_flashlock); 693 if ((state->ts_fw_flashdev != dev) || 694 (state->ts_fw_flashstarted == 0)) { 695 mutex_exit(&state->ts_fw_flashlock); 696 TNF_PROBE_0(tavor_flash_bad_state, TAVOR_TNF_ERROR, ""); 697 TAVOR_TNF_EXIT(tavor_ioctl_flash_fini); 698 return (EIO); 699 } 700 701 tavor_ioctl_flash_cleanup_nolock(state); 702 703 mutex_exit(&state->ts_fw_flashlock); 704 705 /* 706 * If "flash fini" is successful, remove the "on close" callback 707 * that was setup during "flash init". 708 */ 709 ret = tavor_umap_db_clear_onclose_cb(dev, 710 TAVOR_ONCLOSE_FLASH_INPROGRESS); 711 if (ret != DDI_SUCCESS) { 712 TNF_PROBE_0(tavor_flash_fini_clear_cb_fail, TAVOR_TNF_ERROR, 713 ""); 714 TAVOR_TNF_EXIT(tavor_ioctl_flash_fini); 715 return (EFAULT); 716 } 717 718 TAVOR_TNF_EXIT(tavor_ioctl_flash_fini); 719 return (0); 720 } 721 722 723 /* 724 * tavor_ioctl_flash_cleanup() 725 */ 726 static void 727 tavor_ioctl_flash_cleanup(tavor_state_t *state) 728 { 729 TAVOR_TNF_ENTER(tavor_ioctl_flash_cleanup); 730 731 mutex_enter(&state->ts_fw_flashlock); 732 tavor_ioctl_flash_cleanup_nolock(state); 733 mutex_exit(&state->ts_fw_flashlock); 734 735 TAVOR_TNF_EXIT(tavor_ioctl_flash_cleanup); 736 } 737 738 739 /* 740 * tavor_ioctl_flash_cleanup_nolock() 741 */ 742 static void 743 tavor_ioctl_flash_cleanup_nolock(tavor_state_t *state) 744 { 745 TAVOR_TNF_ENTER(tavor_ioctl_flash_cleanup_nolock); 746 747 ASSERT(MUTEX_HELD(&state->ts_fw_flashlock)); 748 749 /* free flash mem */ 750 kmem_free(state->ts_fw_sector, 1 << state->ts_fw_log_sector_sz); 751 752 /* Fini the Flash */ 753 tavor_flash_fini(state); 754 755 /* Set flash state to fini */ 756 state->ts_fw_flashstarted = 0; 757 state->ts_fw_flashdev = 0; 758 759 TAVOR_TNF_EXIT(tavor_ioctl_flash_cleanup_nolock); 760 } 761 762 763 /* 764 * tavor_ioctl_info() 765 */ 766 static int 767 tavor_ioctl_info(tavor_state_t *state, dev_t dev, intptr_t arg, int mode) 768 { 769 tavor_info_ioctl_t info; 770 tavor_flash_init_ioctl_t init_info; 771 772 TAVOR_TNF_ENTER(tavor_ioctl_info); 773 774 /* 775 * Access to Tavor VTS ioctls is not allowed in "maintenance mode". 776 */ 777 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) { 778 TNF_PROBE_0(tavor_ioctl_info_maintenance_mode_fail, 779 TAVOR_TNF_ERROR, ""); 780 TAVOR_TNF_EXIT(tavor_ioctl_info); 781 return (EFAULT); 782 } 783 784 /* copyin the user struct to kernel */ 785 if (ddi_copyin((void *)arg, &info, sizeof (tavor_info_ioctl_t), 786 mode) != 0) { 787 TNF_PROBE_0(tavor_ioctl_info_copyin_fail, TAVOR_TNF_ERROR, ""); 788 TAVOR_TNF_EXIT(tavor_ioctl_info); 789 return (EFAULT); 790 } 791 792 /* 793 * Check ioctl revision 794 */ 795 if (info.ti_revision != TAVOR_VTS_IOCTL_REVISION) { 796 TNF_PROBE_0(tavor_ioctl_info_bad_rev, TAVOR_TNF_ERROR, ""); 797 TAVOR_TNF_EXIT(tavor_ioctl_info); 798 return (EINVAL); 799 } 800 801 /* 802 * If the 'fw_device_sz' has not been initialized yet, we initialize it 803 * here. This is done by leveraging the 804 * tavor_ioctl_flash_init()/fini() calls. We also hold our own mutex 805 * around this operation in case we have multiple VTS threads in 806 * process at the same time. 807 */ 808 mutex_enter(&state->ts_info_lock); 809 if (state->ts_fw_device_sz == 0) { 810 if (tavor_ioctl_flash_init(state, dev, (intptr_t)&init_info, 811 (FKIOCTL | mode)) != 0) { 812 mutex_exit(&state->ts_info_lock); 813 TNF_PROBE_0(tavor_ioctl_info_flash_init_fail, 814 TAVOR_TNF_ERROR, ""); 815 TAVOR_TNF_EXIT(tavor_ioctl_info); 816 return (EFAULT); 817 } 818 (void) tavor_ioctl_flash_fini(state, dev); 819 } 820 mutex_exit(&state->ts_info_lock); 821 822 info.ti_hw_rev = state->ts_adapter.rev_id; 823 info.ti_flash_sz = state->ts_fw_device_sz; 824 info.ti_fw_rev.tfi_maj = state->ts_fw.fw_rev_major; 825 info.ti_fw_rev.tfi_min = state->ts_fw.fw_rev_minor; 826 info.ti_fw_rev.tfi_sub = state->ts_fw.fw_rev_subminor; 827 info.ti_mem_start_offset = 0; 828 info.ti_mem_end_offset = state->ts_ddr.ddr_endaddr - 829 state->ts_ddr.ddr_baseaddr; 830 831 /* Copy ioctl results back to user struct */ 832 if (ddi_copyout(&info, (void *)arg, sizeof (tavor_info_ioctl_t), 833 mode) != 0) { 834 TNF_PROBE_0(tavor_ioctl_info_copyout_fail, TAVOR_TNF_ERROR, ""); 835 TAVOR_TNF_EXIT(tavor_ioctl_info); 836 return (EFAULT); 837 } 838 839 TAVOR_TNF_EXIT(tavor_ioctl_info); 840 return (0); 841 } 842 843 /* 844 * tavor_ioctl_ports() 845 */ 846 static int 847 tavor_ioctl_ports(tavor_state_t *state, intptr_t arg, int mode) 848 { 849 tavor_ports_ioctl_t info; 850 tavor_stat_port_ioctl_t portstat; 851 ibt_hca_portinfo_t pi; 852 uint_t tbl_size; 853 ib_gid_t *sgid_tbl; 854 ib_pkey_t *pkey_tbl; 855 int i; 856 857 TAVOR_TNF_ENTER(tavor_ioctl_ports); 858 859 /* 860 * Access to Tavor VTS ioctls is not allowed in "maintenance mode". 861 */ 862 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) { 863 TNF_PROBE_0(tavor_ioctl_ports_maintenance_mode_fail, 864 TAVOR_TNF_ERROR, ""); 865 TAVOR_TNF_EXIT(tavor_ioctl_ports); 866 return (EFAULT); 867 } 868 869 /* copyin the user struct to kernel */ 870 #ifdef _MULTI_DATAMODEL 871 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) { 872 tavor_ports_ioctl32_t info32; 873 874 if (ddi_copyin((void *)arg, &info32, 875 sizeof (tavor_ports_ioctl32_t), mode) != 0) { 876 TNF_PROBE_0(tavor_ioctl_ports_copyin_fail, 877 TAVOR_TNF_ERROR, ""); 878 TAVOR_TNF_EXIT(tavor_ioctl_ports); 879 return (EFAULT); 880 } 881 info.tp_revision = info32.tp_revision; 882 info.tp_ports = 883 (tavor_stat_port_ioctl_t *)(uintptr_t)info32.tp_ports; 884 info.tp_num_ports = info32.tp_num_ports; 885 886 } else 887 #endif /* _MULTI_DATAMODEL */ 888 if (ddi_copyin((void *)arg, &info, sizeof (tavor_ports_ioctl_t), 889 mode) != 0) { 890 TNF_PROBE_0(tavor_ioctl_ports_copyin_fail, TAVOR_TNF_ERROR, ""); 891 TAVOR_TNF_EXIT(tavor_ioctl_ports); 892 return (EFAULT); 893 } 894 895 /* 896 * Check ioctl revision 897 */ 898 if (info.tp_revision != TAVOR_VTS_IOCTL_REVISION) { 899 TNF_PROBE_0(tavor_ioctl_ports_bad_rev, TAVOR_TNF_ERROR, ""); 900 TAVOR_TNF_EXIT(tavor_ioctl_ports); 901 return (EINVAL); 902 } 903 904 /* Allocate space for temporary GID table/PKey table */ 905 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl); 906 sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t), 907 KM_SLEEP); 908 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl); 909 pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t), 910 KM_SLEEP); 911 912 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgid_tbl, *pkey_tbl)) 913 914 /* 915 * Setup the number of ports, then loop through all ports and 916 * query properties of each. 917 */ 918 info.tp_num_ports = (uint8_t)state->ts_cfg_profile->cp_num_ports; 919 for (i = 0; i < info.tp_num_ports; i++) { 920 /* 921 * Get portstate information from the device. If 922 * tavor_port_query() fails, leave zeroes in user 923 * struct port entry and continue. 924 */ 925 bzero(&pi, sizeof (ibt_hca_portinfo_t)); 926 pi.p_sgid_tbl = sgid_tbl; 927 pi.p_pkey_tbl = pkey_tbl; 928 if (tavor_port_query(state, i + 1, &pi) != 0) { 929 TNF_PROBE_0(tavor_ioctl_ports_query_failed, 930 TAVOR_TNF_ERROR, ""); 931 } 932 933 portstat.tsp_port_num = pi.p_port_num; 934 portstat.tsp_state = pi.p_linkstate; 935 portstat.tsp_guid = pi.p_sgid_tbl[0].gid_guid; 936 937 /* 938 * Copy queried port results back to user struct. If 939 * this fails, then break out of loop, attempt to copy 940 * out remaining info to user struct, and return (without 941 * error). 942 */ 943 if (ddi_copyout(&portstat, 944 &(((tavor_stat_port_ioctl_t *)info.tp_ports)[i]), 945 sizeof (tavor_stat_port_ioctl_t), mode) != 0) { 946 break; 947 } 948 } 949 950 /* Free the temporary space used for GID table/PKey table */ 951 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl); 952 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t)); 953 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl); 954 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t)); 955 956 /* Copy ioctl results back to user struct */ 957 #ifdef _MULTI_DATAMODEL 958 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) { 959 tavor_ports_ioctl32_t info32; 960 961 info32.tp_revision = info.tp_revision; 962 info32.tp_ports = (caddr32_t)(uintptr_t)info.tp_ports; 963 info32.tp_num_ports = info.tp_num_ports; 964 965 if (ddi_copyout(&info32, (void *)arg, 966 sizeof (tavor_ports_ioctl32_t), mode) != 0) { 967 TNF_PROBE_0(tavor_ioctl_ports_copyout_fail, 968 TAVOR_TNF_ERROR, ""); 969 TAVOR_TNF_EXIT(tavor_ioctl_ports); 970 return (EFAULT); 971 } 972 } else 973 #endif /* _MULTI_DATAMODEL */ 974 if (ddi_copyout(&info, (void *)arg, sizeof (tavor_ports_ioctl_t), 975 mode) != 0) { 976 TNF_PROBE_0(tavor_ioctl_ports_copyout_fail, 977 TAVOR_TNF_ERROR, ""); 978 TAVOR_TNF_EXIT(tavor_ioctl_ports); 979 return (EFAULT); 980 } 981 982 TAVOR_TNF_EXIT(tavor_ioctl_ports); 983 return (0); 984 } 985 986 /* 987 * tavor_ioctl_loopback() 988 */ 989 static int 990 tavor_ioctl_loopback(tavor_state_t *state, intptr_t arg, int mode) 991 { 992 tavor_loopback_ioctl_t lb; 993 tavor_loopback_state_t lstate; 994 ibt_hca_portinfo_t pi; 995 uint_t tbl_size, loopmax, max_usec; 996 ib_gid_t *sgid_tbl; 997 ib_pkey_t *pkey_tbl; 998 int j, iter, ret; 999 1000 TAVOR_TNF_ENTER(tavor_ioctl_loopback); 1001 1002 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(lstate)) 1003 1004 /* 1005 * Access to Tavor VTS ioctls is not allowed in "maintenance mode". 1006 */ 1007 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) { 1008 TNF_PROBE_0(tavor_ioctl_loopback_maintenance_mode_fail, 1009 TAVOR_TNF_ERROR, ""); 1010 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1011 return (EFAULT); 1012 } 1013 1014 /* copyin the user struct to kernel */ 1015 #ifdef _MULTI_DATAMODEL 1016 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) { 1017 tavor_loopback_ioctl32_t lb32; 1018 1019 if (ddi_copyin((void *)arg, &lb32, 1020 sizeof (tavor_loopback_ioctl32_t), mode) != 0) { 1021 TNF_PROBE_0(tavor_ioctl_loopback_copyin_fail, 1022 TAVOR_TNF_ERROR, ""); 1023 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1024 return (EFAULT); 1025 } 1026 lb.tlb_revision = lb32.tlb_revision; 1027 lb.tlb_send_buf = (caddr_t)(uintptr_t)lb32.tlb_send_buf; 1028 lb.tlb_fail_buf = (caddr_t)(uintptr_t)lb32.tlb_fail_buf; 1029 lb.tlb_buf_sz = lb32.tlb_buf_sz; 1030 lb.tlb_num_iter = lb32.tlb_num_iter; 1031 lb.tlb_pass_done = lb32.tlb_pass_done; 1032 lb.tlb_timeout = lb32.tlb_timeout; 1033 lb.tlb_error_type = lb32.tlb_error_type; 1034 lb.tlb_port_num = lb32.tlb_port_num; 1035 lb.tlb_num_retry = lb32.tlb_num_retry; 1036 } else 1037 #endif /* _MULTI_DATAMODEL */ 1038 if (ddi_copyin((void *)arg, &lb, sizeof (tavor_loopback_ioctl_t), 1039 mode) != 0) { 1040 TNF_PROBE_0(tavor_ioctl_loopback_copyin_fail, 1041 TAVOR_TNF_ERROR, ""); 1042 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1043 return (EFAULT); 1044 } 1045 1046 /* Initialize the internal loopback test state structure */ 1047 bzero(&lstate, sizeof (tavor_loopback_state_t)); 1048 1049 /* 1050 * Check ioctl revision 1051 */ 1052 if (lb.tlb_revision != TAVOR_VTS_IOCTL_REVISION) { 1053 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_REVISION; 1054 (void) tavor_loopback_copyout(&lb, arg, mode); 1055 TNF_PROBE_0(tavor_ioctl_loopback_bad_rev, 1056 TAVOR_TNF_ERROR, ""); 1057 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1058 return (EINVAL); 1059 } 1060 1061 /* Validate that specified port number is legal */ 1062 if (!tavor_portnum_is_valid(state, lb.tlb_port_num)) { 1063 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_PORT; 1064 (void) tavor_loopback_copyout(&lb, arg, mode); 1065 TNF_PROBE_0(tavor_ioctl_loopback_inv_port, 1066 TAVOR_TNF_ERROR, ""); 1067 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1068 return (EINVAL); 1069 } 1070 1071 /* Allocate space for temporary GID table/PKey table */ 1072 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl); 1073 sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t), 1074 KM_SLEEP); 1075 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl); 1076 pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t), 1077 KM_SLEEP); 1078 1079 /* 1080 * Get portstate information from specific port on device 1081 */ 1082 bzero(&pi, sizeof (ibt_hca_portinfo_t)); 1083 pi.p_sgid_tbl = sgid_tbl; 1084 pi.p_pkey_tbl = pkey_tbl; 1085 if (tavor_port_query(state, lb.tlb_port_num, &pi) != 0) { 1086 /* Free the temporary space used for GID table/PKey table */ 1087 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl); 1088 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t)); 1089 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl); 1090 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t)); 1091 1092 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_PORT; 1093 (void) tavor_loopback_copyout(&lb, arg, mode); 1094 tavor_loopback_free_state(&lstate); 1095 TNF_PROBE_0(tavor_ioctl_loopback_bad_port, 1096 TAVOR_TNF_ERROR, ""); 1097 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1098 return (EINVAL); 1099 } 1100 1101 lstate.tls_port = pi.p_port_num; 1102 lstate.tls_lid = pi.p_base_lid; 1103 lstate.tls_pkey_ix = (pi.p_linkstate == TAVOR_PORT_LINK_ACTIVE) ? 1 : 0; 1104 lstate.tls_state = state; 1105 lstate.tls_retry = lb.tlb_num_retry; 1106 1107 /* Free the temporary space used for GID table/PKey table */ 1108 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl); 1109 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t)); 1110 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl); 1111 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t)); 1112 1113 /* 1114 * Compute the timeout duration in usec per the formula: 1115 * to_usec_per_retry = 4.096us * (2 ^ supplied_timeout) 1116 * (plus we add a little fudge-factor here too) 1117 */ 1118 lstate.tls_timeout = lb.tlb_timeout; 1119 max_usec = (4096 * (1 << lstate.tls_timeout)) / 1000; 1120 max_usec = max_usec * (lstate.tls_retry + 1); 1121 max_usec = max_usec + 10000; 1122 1123 /* 1124 * Determine how many times we should loop before declaring a 1125 * timeout failure. 1126 */ 1127 loopmax = max_usec/TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR; 1128 if ((max_usec % TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR) != 0) { 1129 loopmax++; 1130 } 1131 1132 if (lb.tlb_send_buf == NULL || lb.tlb_buf_sz == 0) { 1133 lb.tlb_error_type = TAVOR_LOOPBACK_SEND_BUF_INVALID; 1134 (void) tavor_loopback_copyout(&lb, arg, mode); 1135 tavor_loopback_free_state(&lstate); 1136 TNF_PROBE_0(tavor_ioctl_loopback_buf_null, 1137 TAVOR_TNF_ERROR, ""); 1138 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1139 return (EINVAL); 1140 } 1141 1142 /* Allocate protection domain (PD) */ 1143 if (tavor_loopback_init(state, &lstate) != 0) { 1144 lb.tlb_error_type = lstate.tls_err; 1145 (void) tavor_loopback_copyout(&lb, arg, mode); 1146 tavor_loopback_free_state(&lstate); 1147 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1148 return (EFAULT); 1149 } 1150 1151 /* Allocate and register a TX buffer */ 1152 if (tavor_loopback_alloc_mem(&lstate, &lstate.tls_tx, 1153 lb.tlb_buf_sz) != 0) { 1154 lb.tlb_error_type = 1155 TAVOR_LOOPBACK_SEND_BUF_MEM_REGION_ALLOC_FAIL; 1156 (void) tavor_loopback_copyout(&lb, arg, mode); 1157 tavor_loopback_free_state(&lstate); 1158 TNF_PROBE_0(tavor_ioctl_loopback_txbuf_alloc_fail, 1159 TAVOR_TNF_ERROR, ""); 1160 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1161 return (EFAULT); 1162 } 1163 1164 /* Allocate and register an RX buffer */ 1165 if (tavor_loopback_alloc_mem(&lstate, &lstate.tls_rx, 1166 lb.tlb_buf_sz) != 0) { 1167 lb.tlb_error_type = 1168 TAVOR_LOOPBACK_RECV_BUF_MEM_REGION_ALLOC_FAIL; 1169 (void) tavor_loopback_copyout(&lb, arg, mode); 1170 tavor_loopback_free_state(&lstate); 1171 TNF_PROBE_0(tavor_ioctl_loopback_rxbuf_alloc_fail, 1172 TAVOR_TNF_ERROR, ""); 1173 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1174 return (EFAULT); 1175 } 1176 1177 /* Copy in the transmit buffer data */ 1178 if (ddi_copyin((void *)lb.tlb_send_buf, lstate.tls_tx.tlc_buf, 1179 lb.tlb_buf_sz, mode) != 0) { 1180 lb.tlb_error_type = TAVOR_LOOPBACK_SEND_BUF_COPY_FAIL; 1181 (void) tavor_loopback_copyout(&lb, arg, mode); 1182 tavor_loopback_free_state(&lstate); 1183 TNF_PROBE_0(tavor_ioctl_loopback_tx_copyin_fail, 1184 TAVOR_TNF_ERROR, ""); 1185 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1186 return (EFAULT); 1187 } 1188 1189 /* Allocate the transmit QP and CQs */ 1190 lstate.tls_err = TAVOR_LOOPBACK_XMIT_SEND_CQ_ALLOC_FAIL; 1191 if (tavor_loopback_alloc_qps(&lstate, &lstate.tls_tx) != 0) { 1192 lb.tlb_error_type = lstate.tls_err; 1193 (void) tavor_loopback_copyout(&lb, arg, mode); 1194 tavor_loopback_free_state(&lstate); 1195 TNF_PROBE_0(tavor_ioctl_loopback_txqp_alloc_fail, 1196 TAVOR_TNF_ERROR, ""); 1197 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1198 return (EFAULT); 1199 } 1200 1201 /* Allocate the receive QP and CQs */ 1202 lstate.tls_err = TAVOR_LOOPBACK_RECV_SEND_CQ_ALLOC_FAIL; 1203 if (tavor_loopback_alloc_qps(&lstate, &lstate.tls_rx) != 0) { 1204 lb.tlb_error_type = lstate.tls_err; 1205 (void) tavor_loopback_copyout(&lb, arg, mode); 1206 tavor_loopback_free_state(&lstate); 1207 TNF_PROBE_0(tavor_ioctl_loopback_rxqp_alloc_fail, 1208 TAVOR_TNF_ERROR, ""); 1209 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1210 return (EFAULT); 1211 } 1212 1213 /* Activate the TX QP (connect to RX QP) */ 1214 lstate.tls_err = TAVOR_LOOPBACK_XMIT_QP_INIT_FAIL; 1215 if (tavor_loopback_modify_qp(&lstate, &lstate.tls_tx, 1216 lstate.tls_rx.tlc_qp_num) != 0) { 1217 lb.tlb_error_type = lstate.tls_err; 1218 (void) tavor_loopback_copyout(&lb, arg, mode); 1219 tavor_loopback_free_state(&lstate); 1220 TNF_PROBE_0(tavor_ioctl_loopback_txqp_init_fail, 1221 TAVOR_TNF_ERROR, ""); 1222 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1223 return (EFAULT); 1224 } 1225 1226 /* Activate the RX QP (connect to TX QP) */ 1227 lstate.tls_err = TAVOR_LOOPBACK_RECV_QP_INIT_FAIL; 1228 if (tavor_loopback_modify_qp(&lstate, &lstate.tls_rx, 1229 lstate.tls_tx.tlc_qp_num) != 0) { 1230 lb.tlb_error_type = lstate.tls_err; 1231 (void) tavor_loopback_copyout(&lb, arg, mode); 1232 tavor_loopback_free_state(&lstate); 1233 TNF_PROBE_0(tavor_ioctl_loopback_rxqp_init_fail, 1234 TAVOR_TNF_ERROR, ""); 1235 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1236 return (EFAULT); 1237 } 1238 1239 /* Run the loopback test (for specified number of iterations) */ 1240 lb.tlb_pass_done = 0; 1241 for (iter = 0; iter < lb.tlb_num_iter; iter++) { 1242 lstate.tls_err = 0; 1243 bzero(lstate.tls_rx.tlc_buf, lb.tlb_buf_sz); 1244 1245 /* Post RDMA Write work request */ 1246 if (tavor_loopback_post_send(&lstate, &lstate.tls_tx, 1247 &lstate.tls_rx) != IBT_SUCCESS) { 1248 lb.tlb_error_type = TAVOR_LOOPBACK_WQE_POST_FAIL; 1249 (void) tavor_loopback_copyout(&lb, arg, mode); 1250 tavor_loopback_free_state(&lstate); 1251 TNF_PROBE_0(tavor_ioctl_loopback_wqe_post_fail, 1252 TAVOR_TNF_ERROR, ""); 1253 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1254 return (EFAULT); 1255 } 1256 1257 /* Poll the TX CQ for a completion every few ticks */ 1258 for (j = 0; j < loopmax; j++) { 1259 delay(drv_usectohz(TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR)); 1260 1261 ret = tavor_loopback_poll_cq(&lstate, &lstate.tls_tx); 1262 if (((ret != IBT_SUCCESS) && (ret != IBT_CQ_EMPTY)) || 1263 ((ret == IBT_CQ_EMPTY) && (j == loopmax - 1))) { 1264 lb.tlb_error_type = TAVOR_LOOPBACK_CQ_POLL_FAIL; 1265 if (ddi_copyout(lstate.tls_rx.tlc_buf, 1266 lb.tlb_fail_buf, lstate.tls_tx.tlc_buf_sz, 1267 mode) != 0) { 1268 TNF_PROBE_0( 1269 tavor_ioctl_loopback_xfer_co_fail, 1270 TAVOR_TNF_ERROR, ""); 1271 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1272 return (EFAULT); 1273 } 1274 (void) tavor_loopback_copyout(&lb, arg, mode); 1275 tavor_loopback_free_state(&lstate); 1276 TNF_PROBE_0(tavor_ioctl_loopback_xfer_fail, 1277 TAVOR_TNF_ERROR, ""); 1278 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1279 return (EFAULT); 1280 } else if (ret == IBT_CQ_EMPTY) { 1281 continue; 1282 } 1283 1284 /* Compare the data buffers */ 1285 if (bcmp(lstate.tls_tx.tlc_buf, lstate.tls_rx.tlc_buf, 1286 lb.tlb_buf_sz) == 0) { 1287 break; 1288 } else { 1289 lb.tlb_error_type = 1290 TAVOR_LOOPBACK_SEND_RECV_COMPARE_FAIL; 1291 if (ddi_copyout(lstate.tls_rx.tlc_buf, 1292 lb.tlb_fail_buf, lstate.tls_tx.tlc_buf_sz, 1293 mode) != 0) { 1294 TNF_PROBE_0( 1295 tavor_ioctl_loopback_bcmp_co_fail, 1296 TAVOR_TNF_ERROR, ""); 1297 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1298 return (EFAULT); 1299 } 1300 (void) tavor_loopback_copyout(&lb, arg, mode); 1301 tavor_loopback_free_state(&lstate); 1302 TNF_PROBE_0(tavor_ioctl_loopback_bcmp_fail, 1303 TAVOR_TNF_ERROR, ""); 1304 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1305 return (EFAULT); 1306 } 1307 } 1308 1309 lstate.tls_err = TAVOR_LOOPBACK_SUCCESS; 1310 lb.tlb_pass_done = iter + 1; 1311 } 1312 1313 lb.tlb_error_type = TAVOR_LOOPBACK_SUCCESS; 1314 1315 /* Copy ioctl results back to user struct */ 1316 ret = tavor_loopback_copyout(&lb, arg, mode); 1317 1318 /* Free up everything and release all consumed resources */ 1319 tavor_loopback_free_state(&lstate); 1320 1321 TAVOR_TNF_EXIT(tavor_ioctl_loopback); 1322 return (ret); 1323 } 1324 1325 /* 1326 * tavor_ioctl_ddr_read() 1327 */ 1328 static int 1329 tavor_ioctl_ddr_read(tavor_state_t *state, intptr_t arg, int mode) 1330 { 1331 tavor_ddr_read_ioctl_t rdreg; 1332 uint32_t *addr; 1333 uintptr_t baseaddr; 1334 uint64_t ddr_size; 1335 1336 TAVOR_TNF_ENTER(tavor_ioctl_ddr_read); 1337 1338 /* 1339 * Access to Tavor VTS ioctls is not allowed in "maintenance mode". 1340 */ 1341 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) { 1342 TNF_PROBE_0(tavor_ioctl_ddr_read_maintenance_mode_fail, 1343 TAVOR_TNF_ERROR, ""); 1344 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read); 1345 return (EFAULT); 1346 } 1347 1348 /* copyin the user struct to kernel */ 1349 if (ddi_copyin((void *)arg, &rdreg, sizeof (tavor_ddr_read_ioctl_t), 1350 mode) != 0) { 1351 TNF_PROBE_0(tavor_ioctl_ddr_read_copyin_fail, 1352 TAVOR_TNF_ERROR, ""); 1353 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read); 1354 return (EFAULT); 1355 } 1356 1357 /* 1358 * Check ioctl revision 1359 */ 1360 if (rdreg.tdr_revision != TAVOR_VTS_IOCTL_REVISION) { 1361 TNF_PROBE_0(tavor_ioctl_ddr_read_bad_rev, TAVOR_TNF_ERROR, ""); 1362 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read); 1363 return (EINVAL); 1364 } 1365 1366 /* 1367 * Check for valid offset 1368 */ 1369 ddr_size = (state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1); 1370 if ((uint64_t)rdreg.tdr_offset >= ddr_size) { 1371 TNF_PROBE_0(tavor_ioctl_ddr_read_bad_offset, 1372 TAVOR_TNF_ERROR, ""); 1373 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read); 1374 return (EINVAL); 1375 } 1376 1377 /* Determine base address for requested register read */ 1378 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr; 1379 1380 /* Ensure that address is properly-aligned */ 1381 addr = (uint32_t *)((baseaddr + rdreg.tdr_offset) & ~0x3); 1382 1383 /* Read the register pointed to by addr */ 1384 rdreg.tdr_data = ddi_get32(state->ts_reg_cmdhdl, addr); 1385 1386 /* Copy ioctl results back to user struct */ 1387 if (ddi_copyout(&rdreg, (void *)arg, sizeof (tavor_ddr_read_ioctl_t), 1388 mode) != 0) { 1389 TNF_PROBE_0(tavor_ioctl_ddr_read_copyout_fail, 1390 TAVOR_TNF_ERROR, ""); 1391 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read); 1392 return (EFAULT); 1393 } 1394 1395 TAVOR_TNF_EXIT(tavor_ioctl_ddr_read); 1396 return (0); 1397 } 1398 1399 1400 #ifdef DEBUG 1401 /* 1402 * tavor_ioctl_reg_read() 1403 */ 1404 static int 1405 tavor_ioctl_reg_read(tavor_state_t *state, intptr_t arg, int mode) 1406 { 1407 tavor_reg_ioctl_t rdreg; 1408 uint32_t *addr; 1409 uintptr_t baseaddr; 1410 int status; 1411 1412 TAVOR_TNF_ENTER(tavor_ioctl_reg_read); 1413 1414 /* 1415 * Access to Tavor registers is not allowed in "maintenance mode". 1416 * This is primarily because the device may not have BARs to access 1417 */ 1418 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) { 1419 TNF_PROBE_0(tavor_ioctl_reg_read_maintence_mode_fail, 1420 TAVOR_TNF_ERROR, ""); 1421 TAVOR_TNF_EXIT(tavor_ioctl_reg_read); 1422 return (EFAULT); 1423 } 1424 1425 /* Copy in the tavor_reg_ioctl_t structure */ 1426 status = ddi_copyin((void *)arg, &rdreg, sizeof (tavor_reg_ioctl_t), 1427 mode); 1428 if (status != 0) { 1429 TNF_PROBE_0(tavor_ioctl_reg_read_copyin_fail, 1430 TAVOR_TNF_ERROR, ""); 1431 TAVOR_TNF_EXIT(tavor_ioctl_reg_read); 1432 return (EFAULT); 1433 } 1434 1435 /* Determine base address for requested register set */ 1436 switch (rdreg.trg_reg_set) { 1437 case TAVOR_CMD_BAR: 1438 baseaddr = (uintptr_t)state->ts_reg_cmd_baseaddr; 1439 break; 1440 1441 case TAVOR_UAR_BAR: 1442 baseaddr = (uintptr_t)state->ts_reg_uar_baseaddr; 1443 break; 1444 1445 case TAVOR_DDR_BAR: 1446 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr; 1447 break; 1448 1449 default: 1450 TNF_PROBE_0(tavor_ioctl_reg_read_invregset_fail, 1451 TAVOR_TNF_ERROR, ""); 1452 TAVOR_TNF_EXIT(tavor_ioctl_reg_read); 1453 return (EFAULT); 1454 } 1455 1456 /* Ensure that address is properly-aligned */ 1457 addr = (uint32_t *)((baseaddr + rdreg.trg_offset) & ~0x3); 1458 1459 /* Read the register pointed to by addr */ 1460 rdreg.trg_data = ddi_get32(state->ts_reg_cmdhdl, addr); 1461 1462 /* Copy in the result into the tavor_reg_ioctl_t structure */ 1463 status = ddi_copyout(&rdreg, (void *)arg, sizeof (tavor_reg_ioctl_t), 1464 mode); 1465 if (status != 0) { 1466 TNF_PROBE_0(tavor_ioctl_reg_read_copyout_fail, 1467 TAVOR_TNF_ERROR, ""); 1468 TAVOR_TNF_EXIT(tavor_ioctl_reg_read); 1469 return (EFAULT); 1470 } 1471 1472 TAVOR_TNF_EXIT(tavor_ioctl_reg_read); 1473 return (0); 1474 } 1475 1476 1477 /* 1478 * tavor_ioctl_reg_write() 1479 */ 1480 static int 1481 tavor_ioctl_reg_write(tavor_state_t *state, intptr_t arg, int mode) 1482 { 1483 tavor_reg_ioctl_t wrreg; 1484 uint32_t *addr; 1485 uintptr_t baseaddr; 1486 int status; 1487 1488 TAVOR_TNF_ENTER(tavor_ioctl_reg_write); 1489 1490 /* 1491 * Access to Tavor registers is not allowed in "maintenance mode". 1492 * This is primarily because the device may not have BARs to access 1493 */ 1494 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) { 1495 TNF_PROBE_0(tavor_ioctl_reg_write_maintence_mode_fail, 1496 TAVOR_TNF_ERROR, ""); 1497 TAVOR_TNF_EXIT(tavor_ioctl_reg_write); 1498 return (EFAULT); 1499 } 1500 1501 /* Copy in the tavor_reg_ioctl_t structure */ 1502 status = ddi_copyin((void *)arg, &wrreg, sizeof (tavor_reg_ioctl_t), 1503 mode); 1504 if (status != 0) { 1505 TNF_PROBE_0(tavor_ioctl_reg_write_copyin_fail, 1506 TAVOR_TNF_ERROR, ""); 1507 TAVOR_TNF_EXIT(tavor_ioctl_reg_write); 1508 return (EFAULT); 1509 } 1510 1511 /* Determine base address for requested register set */ 1512 switch (wrreg.trg_reg_set) { 1513 case TAVOR_CMD_BAR: 1514 baseaddr = (uintptr_t)state->ts_reg_cmd_baseaddr; 1515 break; 1516 1517 case TAVOR_UAR_BAR: 1518 baseaddr = (uintptr_t)state->ts_reg_uar_baseaddr; 1519 break; 1520 1521 case TAVOR_DDR_BAR: 1522 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr; 1523 break; 1524 1525 default: 1526 TNF_PROBE_0(tavor_ioctl_reg_write_invregset_fail, 1527 TAVOR_TNF_ERROR, ""); 1528 TAVOR_TNF_EXIT(tavor_ioctl_reg_write); 1529 return (EFAULT); 1530 } 1531 1532 /* Ensure that address is properly-aligned */ 1533 addr = (uint32_t *)((baseaddr + wrreg.trg_offset) & ~0x3); 1534 1535 /* Write the data to the register pointed to by addr */ 1536 ddi_put32(state->ts_reg_cmdhdl, addr, wrreg.trg_data); 1537 1538 TAVOR_TNF_EXIT(tavor_ioctl_reg_write); 1539 return (0); 1540 } 1541 #endif /* DEBUG */ 1542 1543 /* 1544 * tavor_flash_reset() 1545 */ 1546 static void 1547 tavor_flash_reset(tavor_state_t *state) 1548 { 1549 TAVOR_TNF_ENTER(tavor_flash_reset); 1550 1551 /* 1552 * Performs a reset to the flash device. After a reset the flash will 1553 * be operating in normal mode (capable of read/write, etc.). 1554 */ 1555 switch (state->ts_fw_cmdset) { 1556 case TAVOR_FLASH_AMD_CMDSET: 1557 tavor_flash_write(state, 0x555, TAVOR_HW_FLASH_RESET_AMD); 1558 break; 1559 1560 case TAVOR_FLASH_INTEL_CMDSET: 1561 tavor_flash_write(state, 0x555, TAVOR_HW_FLASH_RESET_INTEL); 1562 break; 1563 1564 default: 1565 break; 1566 } 1567 1568 TAVOR_TNF_EXIT(tavor_flash_reset); 1569 } 1570 1571 /* 1572 * tavor_flash_read_sector() 1573 */ 1574 static void 1575 tavor_flash_read_sector(tavor_state_t *state, uint32_t sector_num) 1576 { 1577 uint32_t addr; 1578 uint32_t end_addr; 1579 uint32_t *image; 1580 int i; 1581 1582 TAVOR_TNF_ENTER(tavor_flash_read_sector); 1583 1584 image = (uint32_t *)&state->ts_fw_sector[0]; 1585 1586 /* 1587 * Calculate the start and end address of the sector, based on the 1588 * sector number passed in. 1589 */ 1590 addr = sector_num << state->ts_fw_log_sector_sz; 1591 end_addr = addr + (1 << state->ts_fw_log_sector_sz); 1592 1593 /* Set the flash bank correctly for the given address */ 1594 tavor_flash_bank(state, addr); 1595 1596 /* Read the entire sector, one quadlet at a time */ 1597 for (i = 0; addr < end_addr; i++, addr += 4) { 1598 image[i] = tavor_flash_read(state, addr); 1599 } 1600 1601 TAVOR_TNF_EXIT(tavor_flash_read_sector); 1602 } 1603 1604 /* 1605 * tavor_flash_read_quadlet() 1606 */ 1607 static void 1608 tavor_flash_read_quadlet(tavor_state_t *state, uint32_t *data, 1609 uint32_t addr) 1610 { 1611 TAVOR_TNF_ENTER(tavor_flash_read_quadlet); 1612 1613 /* Set the flash bank correctly for the given address */ 1614 tavor_flash_bank(state, addr); 1615 1616 /* Read one quadlet of data */ 1617 *data = tavor_flash_read(state, addr); 1618 1619 TAVOR_TNF_EXIT(tavor_flash_read_quadlet); 1620 } 1621 1622 /* 1623 * tavor_flash_write_sector() 1624 */ 1625 static int 1626 tavor_flash_write_sector(tavor_state_t *state, uint32_t sector_num) 1627 { 1628 uint32_t addr; 1629 uint32_t end_addr; 1630 uchar_t *sector; 1631 int status = 0; 1632 int i; 1633 1634 TAVOR_TNF_ENTER(tavor_flash_write_sector); 1635 1636 sector = (uchar_t *)&state->ts_fw_sector[0]; 1637 1638 /* 1639 * Calculate the start and end address of the sector, based on the 1640 * sector number passed in. 1641 */ 1642 addr = sector_num << state->ts_fw_log_sector_sz; 1643 end_addr = addr + (1 << state->ts_fw_log_sector_sz); 1644 1645 /* Set the flash bank correctly for the given address */ 1646 tavor_flash_bank(state, addr); 1647 1648 /* Erase the sector before writing */ 1649 tavor_flash_reset(state); 1650 status = tavor_flash_erase_sector(state, sector_num); 1651 if (status != 0) { 1652 TAVOR_TNF_EXIT(tavor_flash_write_sector); 1653 return (status); 1654 } 1655 1656 /* Write the entire sector, one byte at a time */ 1657 for (i = 0; addr < end_addr; i++, addr++) { 1658 status = tavor_flash_write_byte(state, addr, sector[i]); 1659 if (status != 0) { 1660 break; 1661 } 1662 } 1663 1664 tavor_flash_reset(state); 1665 TAVOR_TNF_EXIT(tavor_flash_write_sector); 1666 return (status); 1667 } 1668 1669 /* 1670 * tavor_flash_write_byte() 1671 */ 1672 static int 1673 tavor_flash_write_byte(tavor_state_t *state, uint32_t addr, uchar_t data) 1674 { 1675 uint32_t stat; 1676 int status = 0; 1677 int i; 1678 1679 TAVOR_TNF_ENTER(tavor_flash_write_byte); 1680 1681 switch (state->ts_fw_cmdset) { 1682 case TAVOR_FLASH_AMD_CMDSET: 1683 /* Issue Flash Byte program command */ 1684 tavor_flash_write(state, addr, 0xAA); 1685 tavor_flash_write(state, addr, 0x55); 1686 tavor_flash_write(state, addr, 0xA0); 1687 tavor_flash_write(state, addr, data); 1688 1689 /* 1690 * Wait for Write Byte to Complete: 1691 * 1) Wait 1usec 1692 * 2) Read status of the write operation 1693 * 3) Determine if we have timed out the write operation 1694 * 4) Compare correct data value to the status value that 1695 * was read from the same address. 1696 */ 1697 i = 0; 1698 do { 1699 drv_usecwait(1); 1700 stat = tavor_flash_read(state, addr & ~3); 1701 1702 if (i == tavor_hw_flash_timeout_write) { 1703 cmn_err(CE_WARN, 1704 "tavor_flash_write_byte: ACS write " 1705 "timeout: addr: 0x%x, data: 0x%x\n", 1706 addr, data); 1707 status = EIO; 1708 break; 1709 } 1710 1711 i++; 1712 } while (data != ((stat >> ((3 - (addr & 3)) << 3)) & 0xFF)); 1713 break; 1714 1715 case TAVOR_FLASH_INTEL_CMDSET: 1716 /* Issue Flash Byte program command */ 1717 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_WRITE); 1718 tavor_flash_write(state, addr, data); 1719 1720 /* wait for completion */ 1721 i = 0; 1722 do { 1723 drv_usecwait(1); 1724 stat = tavor_flash_read(state, addr & ~3); 1725 1726 if (i == tavor_hw_flash_timeout_write) { 1727 cmn_err(CE_WARN, 1728 "tavor_flash_write_byte: ICS write " 1729 "timeout: addr: %x, data: %x\n", 1730 addr, data); 1731 status = EIO; 1732 break; 1733 } 1734 1735 i++; 1736 } while ((stat & TAVOR_HW_FLASH_ICS_READY) == 0); 1737 1738 if (stat & TAVOR_HW_FLASH_ICS_ERROR) { 1739 cmn_err(CE_WARN, 1740 "tavor_flash_write_byte: ICS write cmd error: " 1741 "addr: %x, data: %x\n", 1742 addr, data); 1743 status = EIO; 1744 } 1745 break; 1746 1747 default: 1748 cmn_err(CE_WARN, 1749 "tavor_flash_write_byte: unknown cmd set: 0x%x\n", 1750 state->ts_fw_cmdset); 1751 status = EIO; 1752 break; 1753 } 1754 1755 TAVOR_TNF_EXIT(tavor_flash_write_byte); 1756 return (status); 1757 } 1758 1759 /* 1760 * tavor_flash_erase_sector() 1761 */ 1762 static int 1763 tavor_flash_erase_sector(tavor_state_t *state, uint32_t sector_num) 1764 { 1765 uint32_t addr; 1766 uint32_t stat; 1767 int status = 0; 1768 int i; 1769 1770 TAVOR_TNF_ENTER(tavor_flash_erase_sector); 1771 1772 /* Get address from sector num */ 1773 addr = sector_num << state->ts_fw_log_sector_sz; 1774 1775 switch (state->ts_fw_cmdset) { 1776 case TAVOR_FLASH_AMD_CMDSET: 1777 /* Issue Flash Sector Erase Command */ 1778 tavor_flash_write(state, addr, 0xAA); 1779 tavor_flash_write(state, addr, 0x55); 1780 tavor_flash_write(state, addr, 0x80); 1781 tavor_flash_write(state, addr, 0xAA); 1782 tavor_flash_write(state, addr, 0x55); 1783 tavor_flash_write(state, addr, 0x30); 1784 1785 /* 1786 * Wait for Sector Erase to Complete 1787 * 1) Wait 1usec 1788 * 2) read the status at the base addr of the sector 1789 * 3) Determine if we have timed out 1790 * 4) Compare status of address with the value of a fully 1791 * erased quadlet. If these are equal, the sector 1792 * has been erased. 1793 */ 1794 i = 0; 1795 do { 1796 /* wait 1usec */ 1797 drv_usecwait(1); 1798 stat = tavor_flash_read(state, addr); 1799 1800 if (i == tavor_hw_flash_timeout_erase) { 1801 cmn_err(CE_WARN, 1802 "tavor_flash_erase_sector: " 1803 "ACS erase timeout\n"); 1804 status = EIO; 1805 break; 1806 } 1807 1808 i++; 1809 } while (stat != 0xFFFFFFFF); 1810 break; 1811 1812 case TAVOR_FLASH_INTEL_CMDSET: 1813 /* Issue Erase Command */ 1814 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_ERASE); 1815 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_CONFIRM); 1816 1817 /* wait for completion */ 1818 i = 0; 1819 do { 1820 drv_usecwait(1); 1821 stat = tavor_flash_read(state, addr & ~3); 1822 1823 if (i == tavor_hw_flash_timeout_erase) { 1824 cmn_err(CE_WARN, 1825 "tavor_flash_erase_sector: " 1826 "ICS erase timeout\n"); 1827 status = EIO; 1828 break; 1829 } 1830 1831 i++; 1832 } while ((stat & TAVOR_HW_FLASH_ICS_READY) == 0); 1833 1834 if (stat & TAVOR_HW_FLASH_ICS_ERROR) { 1835 cmn_err(CE_WARN, 1836 "tavor_flash_erase_sector: " 1837 "ICS erase cmd error\n"); 1838 status = EIO; 1839 } 1840 break; 1841 1842 default: 1843 cmn_err(CE_WARN, 1844 "tavor_flash_erase_sector: unknown cmd set: 0x%x\n", 1845 state->ts_fw_cmdset); 1846 status = EIO; 1847 break; 1848 } 1849 1850 tavor_flash_reset(state); 1851 1852 TAVOR_TNF_EXIT(tavor_flash_erase_sector); 1853 return (status); 1854 } 1855 1856 /* 1857 * tavor_flash_erase_chip() 1858 */ 1859 static int 1860 tavor_flash_erase_chip(tavor_state_t *state) 1861 { 1862 uint_t size; 1863 uint32_t stat; 1864 int status = 0; 1865 int num_sect; 1866 int i; 1867 1868 TAVOR_TNF_ENTER(tavor_flash_erase_chip); 1869 1870 switch (state->ts_fw_cmdset) { 1871 case TAVOR_FLASH_AMD_CMDSET: 1872 /* Issue Flash Chip Erase Command */ 1873 tavor_flash_write(state, 0, 0xAA); 1874 tavor_flash_write(state, 0, 0x55); 1875 tavor_flash_write(state, 0, 0x80); 1876 tavor_flash_write(state, 0, 0xAA); 1877 tavor_flash_write(state, 0, 0x55); 1878 tavor_flash_write(state, 0, 0x10); 1879 1880 /* 1881 * Wait for Chip Erase to Complete 1882 * 1) Wait 1usec 1883 * 2) read the status at the base addr of the sector 1884 * 3) Determine if we have timed out 1885 * 4) Compare status of address with the value of a 1886 * fully erased quadlet. If these are equal, the 1887 * chip has been erased. 1888 */ 1889 i = 0; 1890 do { 1891 /* wait 1usec */ 1892 drv_usecwait(1); 1893 stat = tavor_flash_read(state, 0); 1894 1895 if (i == tavor_hw_flash_timeout_erase) { 1896 cmn_err(CE_WARN, 1897 "tavor_flash_erase_chip: erase timeout\n"); 1898 status = EIO; 1899 break; 1900 } 1901 1902 i++; 1903 } while (stat != 0xFFFFFFFF); 1904 break; 1905 1906 case TAVOR_FLASH_INTEL_CMDSET: 1907 /* 1908 * The Intel chip doesn't have a chip erase command, so erase 1909 * all blocks one at a time. 1910 */ 1911 size = (0x1 << state->ts_fw_log_sector_sz); 1912 num_sect = state->ts_fw_device_sz / size; 1913 1914 for (i = 0; i < num_sect; i++) { 1915 status = tavor_flash_erase_sector(state, i); 1916 if (status != 0) { 1917 cmn_err(CE_WARN, 1918 "tavor_flash_erase_chip: " 1919 "ICS sector %d erase error\n", i); 1920 status = EIO; 1921 break; 1922 } 1923 } 1924 break; 1925 1926 default: 1927 cmn_err(CE_WARN, "tavor_flash_erase_chip: " 1928 "unknown cmd set: 0x%x\n", state->ts_fw_cmdset); 1929 status = EIO; 1930 break; 1931 } 1932 1933 TAVOR_TNF_EXIT(tavor_flash_erase_chip); 1934 return (status); 1935 } 1936 1937 /* 1938 * tavor_flash_bank() 1939 */ 1940 static void 1941 tavor_flash_bank(tavor_state_t *state, uint32_t addr) 1942 { 1943 ddi_acc_handle_t hdl; 1944 uint32_t bank; 1945 1946 TAVOR_TNF_ENTER(tavor_flash_bank); 1947 1948 /* Set handle */ 1949 hdl = state->ts_pci_cfghdl; 1950 1951 /* Determine the bank setting from the address */ 1952 bank = addr & TAVOR_HW_FLASH_BANK_MASK; 1953 1954 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->ts_fw_flashbank)) 1955 1956 /* 1957 * If the bank is different from the currently set bank, we need to 1958 * change it. Also, if an 'addr' of 0 is given, this allows the 1959 * capability to force the flash bank to 0. This is useful at init 1960 * time to initially set the bank value 1961 */ 1962 if (state->ts_fw_flashbank != bank || addr == 0) { 1963 /* Set bank using the GPIO settings */ 1964 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATACLEAR, 0x70); 1965 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATASET, 1966 (bank >> 15) & 0x70); 1967 1968 /* Save the bank state */ 1969 state->ts_fw_flashbank = bank; 1970 } 1971 1972 TAVOR_TNF_EXIT(tavor_flash_bank); 1973 } 1974 1975 /* 1976 * tavor_flash_read() 1977 */ 1978 static uint32_t 1979 tavor_flash_read(tavor_state_t *state, uint32_t addr) 1980 { 1981 ddi_acc_handle_t hdl; 1982 uint32_t data; 1983 int timeout; 1984 1985 TAVOR_TNF_ENTER(tavor_flash_read); 1986 1987 /* Set handle */ 1988 hdl = state->ts_pci_cfghdl; 1989 1990 /* 1991 * The Read operation does the following: 1992 * 1) Write the masked address to the TAVOR_FLASH_ADDR register. 1993 * Only the least significant 19 bits are valid. 1994 * 2) Read back the register until the command has completed. 1995 * 3) Read the data retrieved from the address at the TAVOR_FLASH_DATA 1996 * register. 1997 */ 1998 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_ADDR, 1999 (addr & TAVOR_HW_FLASH_ADDR_MASK) | (1 << 29)); 2000 2001 timeout = 0; 2002 do { 2003 data = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_ADDR); 2004 timeout++; 2005 } while ((data & TAVOR_HW_FLASH_CMD_MASK) && 2006 (timeout < tavor_hw_flash_timeout_config)); 2007 2008 if (timeout == tavor_hw_flash_timeout_config) { 2009 cmn_err(CE_WARN, "tavor_flash_read: config command timeout.\n"); 2010 } 2011 2012 data = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_DATA); 2013 2014 TAVOR_TNF_EXIT(tavor_flash_read); 2015 return (data); 2016 } 2017 2018 /* 2019 * tavor_flash_write() 2020 */ 2021 static void 2022 tavor_flash_write(tavor_state_t *state, uint32_t addr, uchar_t data) 2023 { 2024 ddi_acc_handle_t hdl; 2025 int cmd; 2026 int timeout; 2027 2028 TAVOR_TNF_ENTER(tavor_flash_write); 2029 2030 /* Set handle */ 2031 hdl = state->ts_pci_cfghdl; 2032 2033 /* 2034 * The Write operation does the following: 2035 * 1) Write the data to be written to the TAVOR_FLASH_DATA offset. 2036 * 2) Write the address to write the data to to the TAVOR_FLASH_ADDR 2037 * offset. 2038 * 3) Wait until the write completes. 2039 */ 2040 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_DATA, data << 24); 2041 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_ADDR, 2042 (addr & 0x7FFFF) | (2 << 29)); 2043 2044 timeout = 0; 2045 do { 2046 cmd = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_ADDR); 2047 timeout++; 2048 } while ((cmd & TAVOR_HW_FLASH_CMD_MASK) && 2049 (timeout < tavor_hw_flash_timeout_config)); 2050 2051 if (timeout == tavor_hw_flash_timeout_config) { 2052 cmn_err(CE_WARN, "tavor_flash_write: config cmd timeout.\n"); 2053 } 2054 2055 TAVOR_TNF_EXIT(tavor_flash_write); 2056 } 2057 2058 /* 2059 * tavor_flash_init() 2060 */ 2061 static void 2062 tavor_flash_init(tavor_state_t *state) 2063 { 2064 uint32_t word; 2065 ddi_acc_handle_t hdl; 2066 int sema_cnt; 2067 int gpio; 2068 2069 TAVOR_TNF_ENTER(tavor_flash_init); 2070 2071 /* Set handle */ 2072 hdl = state->ts_pci_cfghdl; 2073 2074 /* Init the flash */ 2075 2076 /* 2077 * Grab the GPIO semaphore. This allows us exclusive access to the 2078 * GPIO settings on the Tavor for the duration of the flash burning 2079 * procedure. 2080 */ 2081 sema_cnt = 0; 2082 do { 2083 word = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_GPIO_SEMA); 2084 if (word == 0) { 2085 break; 2086 } 2087 2088 sema_cnt++; 2089 drv_usecwait(1); 2090 } while (sema_cnt < tavor_hw_flash_timeout_gpio_sema); 2091 2092 /* 2093 * Determine if we timed out trying to grab the GPIO semaphore 2094 */ 2095 if (sema_cnt == tavor_hw_flash_timeout_gpio_sema) { 2096 cmn_err(CE_WARN, "tavor_flash_init: GPIO SEMA timeout\n"); 2097 } 2098 2099 /* Save away original GPIO Values */ 2100 state->ts_fw_gpio[0] = tavor_flash_read_cfg(hdl, 2101 TAVOR_HW_FLASH_GPIO_DIR); 2102 state->ts_fw_gpio[1] = tavor_flash_read_cfg(hdl, 2103 TAVOR_HW_FLASH_GPIO_POL); 2104 state->ts_fw_gpio[2] = tavor_flash_read_cfg(hdl, 2105 TAVOR_HW_FLASH_GPIO_MOD); 2106 state->ts_fw_gpio[3] = tavor_flash_read_cfg(hdl, 2107 TAVOR_HW_FLASH_GPIO_DAT); 2108 2109 /* Set New GPIO Values */ 2110 gpio = state->ts_fw_gpio[0] | 0x70; 2111 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DIR, gpio); 2112 2113 gpio = state->ts_fw_gpio[1] & ~0x70; 2114 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_POL, gpio); 2115 2116 gpio = state->ts_fw_gpio[2] & ~0x70; 2117 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_MOD, gpio); 2118 2119 /* Set CPUMODE to enable tavor to access the flash device */ 2120 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_CPUMODE, 2121 1 << TAVOR_HW_FLASH_CPU_SHIFT); 2122 2123 /* Initialize to bank 0 */ 2124 tavor_flash_bank(state, 0); 2125 2126 TAVOR_TNF_EXIT(tavor_flash_init); 2127 } 2128 2129 /* 2130 * tavor_flash_cfi_init 2131 * Implements access to the CFI (Common Flash Interface) data 2132 */ 2133 static void 2134 tavor_flash_cfi_init(tavor_state_t *state, uint32_t *cfi_info, int *intel_xcmd) 2135 { 2136 uint32_t data; 2137 uint32_t sector_sz_bytes; 2138 uint32_t bit_count; 2139 uint8_t cfi_ch_info[TAVOR_CFI_INFO_SIZE]; 2140 int i; 2141 2142 TAVOR_TNF_ENTER(tavor_flash_cfi_init); 2143 2144 /* 2145 * Determine if the user command supports the Intel Extended 2146 * Command Set. The query string is contained in the fourth 2147 * quad word. 2148 */ 2149 tavor_flash_cfi_byte(cfi_ch_info, cfi_info[0x04], 0x10); 2150 if (cfi_ch_info[0x10] == 'M' && 2151 cfi_ch_info[0x11] == 'X' && 2152 cfi_ch_info[0x12] == '2') { 2153 *intel_xcmd = 1; /* support is there */ 2154 } 2155 2156 /* CFI QUERY */ 2157 tavor_flash_write(state, 0x55, TAVOR_FLASH_CFI_INIT); 2158 2159 /* Read in CFI data */ 2160 for (i = 0; i < TAVOR_CFI_INFO_SIZE; i += 4) { 2161 data = tavor_flash_read(state, i); 2162 tavor_flash_cfi_byte(cfi_ch_info, data, i); 2163 } 2164 2165 /* Determine chip set */ 2166 state->ts_fw_cmdset = TAVOR_FLASH_UNKNOWN_CMDSET; 2167 if (cfi_ch_info[0x20] == 'Q' && 2168 cfi_ch_info[0x22] == 'R' && 2169 cfi_ch_info[0x24] == 'Y') { 2170 /* 2171 * Mode: x16 working in x8 mode (Intel). 2172 * Pack data - skip spacing bytes. 2173 */ 2174 for (i = 0; i < TAVOR_CFI_INFO_SIZE; i += 2) { 2175 cfi_ch_info[i/2] = cfi_ch_info[i]; 2176 } 2177 } 2178 state->ts_fw_cmdset = cfi_ch_info[0x13]; 2179 if (state->ts_fw_cmdset != TAVOR_FLASH_INTEL_CMDSET && 2180 state->ts_fw_cmdset != TAVOR_FLASH_AMD_CMDSET) { 2181 cmn_err(CE_WARN, 2182 "tavor_flash_cfi_init: UNKNOWN chip cmd set\n"); 2183 state->ts_fw_cmdset = TAVOR_FLASH_UNKNOWN_CMDSET; 2184 goto out; 2185 } 2186 2187 /* Determine total bytes in one sector size */ 2188 sector_sz_bytes = ((cfi_ch_info[0x30] << 8) | cfi_ch_info[0x2F]) << 8; 2189 2190 /* Calculate equivalent of log2 (n) */ 2191 for (bit_count = 0; sector_sz_bytes > 1; bit_count++) { 2192 sector_sz_bytes >>= 1; 2193 } 2194 2195 /* Set sector size */ 2196 state->ts_fw_log_sector_sz = bit_count; 2197 2198 /* Set flash size */ 2199 state->ts_fw_device_sz = 0x1 << cfi_ch_info[0x27]; 2200 2201 /* Reset to turn off CFI mode */ 2202 tavor_flash_reset(state); 2203 2204 /* 2205 * Pass CFI data back to user command. 2206 */ 2207 for (i = 0; i < TAVOR_FLASH_CFI_SIZE_QUADLET; i++) { 2208 tavor_flash_cfi_dword(&cfi_info[i], cfi_ch_info, i << 2); 2209 } 2210 2211 if (*intel_xcmd == 1) { 2212 /* 2213 * Inform the user cmd that this driver does support the 2214 * Intel Extended Command Set. 2215 */ 2216 cfi_ch_info[0x10] = 'M'; 2217 cfi_ch_info[0x11] = 'X'; 2218 cfi_ch_info[0x12] = '2'; 2219 } else { 2220 cfi_ch_info[0x10] = 'Q'; 2221 cfi_ch_info[0x11] = 'R'; 2222 cfi_ch_info[0x12] = 'Y'; 2223 } 2224 cfi_ch_info[0x13] = state->ts_fw_cmdset; 2225 tavor_flash_cfi_dword(&cfi_info[0x4], cfi_ch_info, 0x10); 2226 out: 2227 TAVOR_TNF_EXIT(tavor_flash_cfi_init); 2228 } 2229 2230 /* 2231 * tavor_flash_fini() 2232 */ 2233 static void 2234 tavor_flash_fini(tavor_state_t *state) 2235 { 2236 ddi_acc_handle_t hdl; 2237 2238 TAVOR_TNF_ENTER(tavor_flash_fini); 2239 2240 /* Set handle */ 2241 hdl = state->ts_pci_cfghdl; 2242 2243 /* Restore original GPIO Values */ 2244 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DIR, 2245 state->ts_fw_gpio[0]); 2246 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_POL, 2247 state->ts_fw_gpio[1]); 2248 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_MOD, 2249 state->ts_fw_gpio[2]); 2250 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DAT, 2251 state->ts_fw_gpio[3]); 2252 2253 /* Give up semaphore */ 2254 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_SEMA, 0); 2255 2256 TAVOR_TNF_EXIT(tavor_flash_fini); 2257 } 2258 2259 /* 2260 * tavor_flash_read_cfg 2261 */ 2262 static uint32_t 2263 tavor_flash_read_cfg(ddi_acc_handle_t pci_config_hdl, uint32_t addr) 2264 { 2265 uint32_t read; 2266 2267 TAVOR_TNF_ENTER(tavor_flash_read_cfg); 2268 2269 /* 2270 * Perform flash read operation: 2271 * 1) Place addr to read from on the TAVOR_HW_FLASH_CFG_ADDR register 2272 * 2) Read data at that addr from the TAVOR_HW_FLASH_CFG_DATA register 2273 */ 2274 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_ADDR, addr); 2275 read = pci_config_get32(pci_config_hdl, TAVOR_HW_FLASH_CFG_DATA); 2276 2277 TAVOR_TNF_EXIT(tavor_flash_read_cfg); 2278 2279 return (read); 2280 } 2281 2282 /* 2283 * tavor_flash_write_cfg 2284 */ 2285 static void 2286 tavor_flash_write_cfg(ddi_acc_handle_t pci_config_hdl, uint32_t addr, 2287 uint32_t data) 2288 { 2289 TAVOR_TNF_ENTER(tavor_flash_write_cfg); 2290 2291 /* 2292 * Perform flash write operation: 2293 * 1) Place addr to write to on the TAVOR_HW_FLASH_CFG_ADDR register 2294 * 2) Place data to write on to the TAVOR_HW_FLASH_CFG_DATA register 2295 */ 2296 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_ADDR, addr); 2297 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_DATA, data); 2298 2299 TAVOR_TNF_EXIT(tavor_flash_write_cfg); 2300 } 2301 2302 /* 2303 * Support routines to convert Common Flash Interface (CFI) data 2304 * from a 32 bit word to a char array, and from a char array to 2305 * a 32 bit word. 2306 */ 2307 static void 2308 tavor_flash_cfi_byte(uint8_t *ch, uint32_t dword, int i) 2309 { 2310 ch[i] = (uint8_t)((dword & 0xFF000000) >> 24); 2311 ch[i+1] = (uint8_t)((dword & 0x00FF0000) >> 16); 2312 ch[i+2] = (uint8_t)((dword & 0x0000FF00) >> 8); 2313 ch[i+3] = (uint8_t)((dword & 0x000000FF)); 2314 } 2315 2316 static void 2317 tavor_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i) 2318 { 2319 *dword = (uint32_t) 2320 ((uint32_t)ch[i] << 24 | 2321 (uint32_t)ch[i+1] << 16 | 2322 (uint32_t)ch[i+2] << 8 | 2323 (uint32_t)ch[i+3]); 2324 } 2325 2326 /* 2327 * tavor_loopback_free_qps 2328 */ 2329 static void 2330 tavor_loopback_free_qps(tavor_loopback_state_t *lstate) 2331 { 2332 int i; 2333 2334 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate)) 2335 2336 if (lstate->tls_tx.tlc_qp_hdl != NULL) { 2337 (void) tavor_qp_free(lstate->tls_state, 2338 &lstate->tls_tx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL, 2339 TAVOR_NOSLEEP); 2340 } 2341 if (lstate->tls_rx.tlc_qp_hdl != NULL) { 2342 (void) tavor_qp_free(lstate->tls_state, 2343 &lstate->tls_rx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL, 2344 TAVOR_NOSLEEP); 2345 } 2346 lstate->tls_tx.tlc_qp_hdl = NULL; 2347 lstate->tls_rx.tlc_qp_hdl = NULL; 2348 for (i = 0; i < 2; i++) { 2349 if (lstate->tls_tx.tlc_cqhdl[i] != NULL) { 2350 (void) tavor_cq_free(lstate->tls_state, 2351 &lstate->tls_tx.tlc_cqhdl[i], TAVOR_NOSLEEP); 2352 } 2353 if (lstate->tls_rx.tlc_cqhdl[i] != NULL) { 2354 (void) tavor_cq_free(lstate->tls_state, 2355 &lstate->tls_rx.tlc_cqhdl[i], TAVOR_NOSLEEP); 2356 } 2357 lstate->tls_tx.tlc_cqhdl[i] = NULL; 2358 lstate->tls_rx.tlc_cqhdl[i] = NULL; 2359 } 2360 } 2361 2362 /* 2363 * tavor_loopback_free_state 2364 */ 2365 static void 2366 tavor_loopback_free_state(tavor_loopback_state_t *lstate) 2367 { 2368 tavor_loopback_free_qps(lstate); 2369 if (lstate->tls_tx.tlc_mrhdl != NULL) { 2370 (void) tavor_mr_deregister(lstate->tls_state, 2371 &lstate->tls_tx.tlc_mrhdl, TAVOR_MR_DEREG_ALL, 2372 TAVOR_NOSLEEP); 2373 } 2374 if (lstate->tls_rx.tlc_mrhdl != NULL) { 2375 (void) tavor_mr_deregister(lstate->tls_state, 2376 &lstate->tls_rx.tlc_mrhdl, TAVOR_MR_DEREG_ALL, 2377 TAVOR_NOSLEEP); 2378 } 2379 if (lstate->tls_pd_hdl != NULL) { 2380 (void) tavor_pd_free(lstate->tls_state, &lstate->tls_pd_hdl); 2381 } 2382 if (lstate->tls_tx.tlc_buf != NULL) { 2383 kmem_free(lstate->tls_tx.tlc_buf, lstate->tls_tx.tlc_buf_sz); 2384 } 2385 if (lstate->tls_rx.tlc_buf != NULL) { 2386 kmem_free(lstate->tls_rx.tlc_buf, lstate->tls_rx.tlc_buf_sz); 2387 } 2388 bzero(lstate, sizeof (tavor_loopback_state_t)); 2389 } 2390 2391 /* 2392 * tavor_loopback_init 2393 */ 2394 static int 2395 tavor_loopback_init(tavor_state_t *state, tavor_loopback_state_t *lstate) 2396 { 2397 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate)) 2398 2399 lstate->tls_hca_hdl = (ibc_hca_hdl_t)state; 2400 lstate->tls_status = tavor_pd_alloc(lstate->tls_state, 2401 &lstate->tls_pd_hdl, TAVOR_NOSLEEP); 2402 if (lstate->tls_status != IBT_SUCCESS) { 2403 lstate->tls_err = TAVOR_LOOPBACK_PROT_DOMAIN_ALLOC_FAIL; 2404 TNF_PROBE_0(tavor_ioctl_loopback_alloc_pd_fail, 2405 TAVOR_TNF_ERROR, ""); 2406 return (EFAULT); 2407 } 2408 2409 return (0); 2410 } 2411 2412 /* 2413 * tavor_loopback_init_qp_info 2414 */ 2415 static void 2416 tavor_loopback_init_qp_info(tavor_loopback_state_t *lstate, 2417 tavor_loopback_comm_t *comm) 2418 { 2419 bzero(&comm->tlc_cq_attr, sizeof (ibt_cq_attr_t)); 2420 bzero(&comm->tlc_qp_attr, sizeof (ibt_qp_alloc_attr_t)); 2421 bzero(&comm->tlc_qp_info, sizeof (ibt_qp_info_t)); 2422 2423 comm->tlc_wrid = 1; 2424 comm->tlc_cq_attr.cq_size = 128; 2425 comm->tlc_qp_attr.qp_sizes.cs_sq_sgl = 3; 2426 comm->tlc_qp_attr.qp_sizes.cs_rq_sgl = 3; 2427 comm->tlc_qp_attr.qp_sizes.cs_sq = 16; 2428 comm->tlc_qp_attr.qp_sizes.cs_rq = 16; 2429 comm->tlc_qp_attr.qp_flags = IBT_WR_SIGNALED; 2430 2431 comm->tlc_qp_info.qp_state = IBT_STATE_RESET; 2432 comm->tlc_qp_info.qp_trans = IBT_RC_SRV; 2433 comm->tlc_qp_info.qp_flags = IBT_CEP_RDMA_RD | IBT_CEP_RDMA_WR; 2434 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_hca_port_num = 2435 lstate->tls_port; 2436 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_pkey_ix = 2437 lstate->tls_pkey_ix; 2438 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_timeout = 2439 lstate->tls_timeout; 2440 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_srvl = 0; 2441 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_srate = 2442 IBT_SRATE_4X; 2443 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_send_grh = 0; 2444 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid = 2445 lstate->tls_lid; 2446 comm->tlc_qp_info.qp_transport.rc.rc_retry_cnt = lstate->tls_retry; 2447 comm->tlc_qp_info.qp_transport.rc.rc_sq_psn = 0; 2448 comm->tlc_qp_info.qp_transport.rc.rc_rq_psn = 0; 2449 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_in = 4; 2450 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_out = 4; 2451 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = 0; 2452 comm->tlc_qp_info.qp_transport.rc.rc_min_rnr_nak = IBT_RNR_NAK_655ms; 2453 comm->tlc_qp_info.qp_transport.rc.rc_path_mtu = IB_MTU_1K; 2454 } 2455 2456 /* 2457 * tavor_loopback_alloc_mem 2458 */ 2459 static int 2460 tavor_loopback_alloc_mem(tavor_loopback_state_t *lstate, 2461 tavor_loopback_comm_t *comm, int sz) 2462 { 2463 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm)) 2464 2465 /* Allocate buffer of specified size */ 2466 comm->tlc_buf_sz = sz; 2467 comm->tlc_buf = kmem_zalloc(sz, KM_NOSLEEP); 2468 if (comm->tlc_buf == NULL) { 2469 return (EFAULT); 2470 } 2471 2472 /* Register the buffer as a memory region */ 2473 comm->tlc_memattr.mr_vaddr = (uint64_t)(uintptr_t)comm->tlc_buf; 2474 comm->tlc_memattr.mr_len = (ib_msglen_t)sz; 2475 comm->tlc_memattr.mr_as = NULL; 2476 comm->tlc_memattr.mr_flags = IBT_MR_NOSLEEP | 2477 IBT_MR_ENABLE_REMOTE_WRITE | IBT_MR_ENABLE_LOCAL_WRITE; 2478 2479 comm->tlc_status = tavor_mr_register(lstate->tls_state, 2480 lstate->tls_pd_hdl, &comm->tlc_memattr, &comm->tlc_mrhdl, NULL); 2481 2482 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm->tlc_mrhdl)) 2483 2484 comm->tlc_mrdesc.md_vaddr = comm->tlc_mrhdl->mr_bindinfo.bi_addr; 2485 comm->tlc_mrdesc.md_lkey = comm->tlc_mrhdl->mr_lkey; 2486 comm->tlc_mrdesc.md_rkey = comm->tlc_mrhdl->mr_rkey; 2487 if (comm->tlc_status != IBT_SUCCESS) { 2488 return (EFAULT); 2489 } 2490 return (0); 2491 } 2492 2493 /* 2494 * tavor_loopback_alloc_qps 2495 */ 2496 static int 2497 tavor_loopback_alloc_qps(tavor_loopback_state_t *lstate, 2498 tavor_loopback_comm_t *comm) 2499 { 2500 uint32_t i, real_size; 2501 tavor_qp_info_t qpinfo; 2502 2503 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm)) 2504 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate)) 2505 2506 /* Allocate send and recv CQs */ 2507 for (i = 0; i < 2; i++) { 2508 bzero(&comm->tlc_cq_attr, sizeof (ibt_cq_attr_t)); 2509 comm->tlc_cq_attr.cq_size = 128; 2510 comm->tlc_status = tavor_cq_alloc(lstate->tls_state, 2511 (ibt_cq_hdl_t)NULL, &comm->tlc_cq_attr, &real_size, 2512 &comm->tlc_cqhdl[i], TAVOR_NOSLEEP); 2513 if (comm->tlc_status != IBT_SUCCESS) { 2514 lstate->tls_err += i; 2515 return (EFAULT); 2516 } 2517 } 2518 2519 /* Allocate the QP */ 2520 tavor_loopback_init_qp_info(lstate, comm); 2521 comm->tlc_qp_attr.qp_pd_hdl = (ibt_pd_hdl_t)lstate->tls_pd_hdl; 2522 comm->tlc_qp_attr.qp_scq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[0]; 2523 comm->tlc_qp_attr.qp_rcq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[1]; 2524 comm->tlc_qp_attr.qp_ibc_scq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[0]; 2525 comm->tlc_qp_attr.qp_ibc_rcq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[1]; 2526 qpinfo.qpi_attrp = &comm->tlc_qp_attr; 2527 qpinfo.qpi_type = IBT_RC_RQP; 2528 qpinfo.qpi_ibt_qphdl = NULL; 2529 qpinfo.qpi_queueszp = &comm->tlc_chan_sizes; 2530 qpinfo.qpi_qpn = &comm->tlc_qp_num; 2531 comm->tlc_status = tavor_qp_alloc(lstate->tls_state, &qpinfo, 2532 TAVOR_NOSLEEP, NULL); 2533 if (comm->tlc_status == DDI_SUCCESS) { 2534 comm->tlc_qp_hdl = qpinfo.qpi_qphdl; 2535 } 2536 2537 if (comm->tlc_status != IBT_SUCCESS) { 2538 lstate->tls_err += 2; 2539 return (EFAULT); 2540 } 2541 return (0); 2542 } 2543 2544 /* 2545 * tavor_loopback_modify_qp 2546 */ 2547 static int 2548 tavor_loopback_modify_qp(tavor_loopback_state_t *lstate, 2549 tavor_loopback_comm_t *comm, uint_t qp_num) 2550 { 2551 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm)) 2552 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate)) 2553 2554 /* Modify QP to INIT */ 2555 tavor_loopback_init_qp_info(lstate, comm); 2556 comm->tlc_qp_info.qp_state = IBT_STATE_INIT; 2557 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl, 2558 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes); 2559 if (comm->tlc_status != IBT_SUCCESS) { 2560 return (EFAULT); 2561 } 2562 2563 /* 2564 * Modify QP to RTR (set destination LID and QP number to local 2565 * LID and QP number) 2566 */ 2567 comm->tlc_qp_info.qp_state = IBT_STATE_RTR; 2568 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid 2569 = lstate->tls_lid; 2570 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = qp_num; 2571 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl, 2572 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes); 2573 if (comm->tlc_status != IBT_SUCCESS) { 2574 lstate->tls_err += 1; 2575 return (EFAULT); 2576 } 2577 2578 /* Modify QP to RTS */ 2579 comm->tlc_qp_info.qp_current_state = IBT_STATE_RTR; 2580 comm->tlc_qp_info.qp_state = IBT_STATE_RTS; 2581 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl, 2582 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes); 2583 if (comm->tlc_status != IBT_SUCCESS) { 2584 lstate->tls_err += 2; 2585 return (EFAULT); 2586 } 2587 return (0); 2588 } 2589 2590 /* 2591 * tavor_loopback_copyout 2592 */ 2593 static int 2594 tavor_loopback_copyout(tavor_loopback_ioctl_t *lb, intptr_t arg, int mode) 2595 { 2596 #ifdef _MULTI_DATAMODEL 2597 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) { 2598 tavor_loopback_ioctl32_t lb32; 2599 2600 lb32.tlb_revision = lb->tlb_revision; 2601 lb32.tlb_send_buf = 2602 (caddr32_t)(uintptr_t)lb->tlb_send_buf; 2603 lb32.tlb_fail_buf = 2604 (caddr32_t)(uintptr_t)lb->tlb_fail_buf; 2605 lb32.tlb_buf_sz = lb->tlb_buf_sz; 2606 lb32.tlb_num_iter = lb->tlb_num_iter; 2607 lb32.tlb_pass_done = lb->tlb_pass_done; 2608 lb32.tlb_timeout = lb->tlb_timeout; 2609 lb32.tlb_error_type = lb->tlb_error_type; 2610 lb32.tlb_port_num = lb->tlb_port_num; 2611 lb32.tlb_num_retry = lb->tlb_num_retry; 2612 2613 if (ddi_copyout(&lb32, (void *)arg, 2614 sizeof (tavor_loopback_ioctl32_t), mode) != 0) { 2615 TNF_PROBE_0(tavor_ioctl_loopback_copyout_fail, 2616 TAVOR_TNF_ERROR, ""); 2617 return (EFAULT); 2618 } 2619 } else 2620 #endif /* _MULTI_DATAMODEL */ 2621 if (ddi_copyout(lb, (void *)arg, sizeof (tavor_loopback_ioctl_t), 2622 mode) != 0) { 2623 TNF_PROBE_0(tavor_ioctl_loopback_copyout_fail, 2624 TAVOR_TNF_ERROR, ""); 2625 return (EFAULT); 2626 } 2627 return (0); 2628 } 2629 2630 /* 2631 * tavor_loopback_post_send 2632 */ 2633 static int 2634 tavor_loopback_post_send(tavor_loopback_state_t *lstate, 2635 tavor_loopback_comm_t *tx, tavor_loopback_comm_t *rx) 2636 { 2637 int ret; 2638 2639 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tx)) 2640 2641 bzero(&tx->tlc_sgl, sizeof (ibt_wr_ds_t)); 2642 bzero(&tx->tlc_wr, sizeof (ibt_send_wr_t)); 2643 2644 /* Initialize local address for TX buffer */ 2645 tx->tlc_sgl.ds_va = tx->tlc_mrdesc.md_vaddr; 2646 tx->tlc_sgl.ds_key = tx->tlc_mrdesc.md_lkey; 2647 tx->tlc_sgl.ds_len = tx->tlc_buf_sz; 2648 2649 /* Initialize the remaining details of the work request */ 2650 tx->tlc_wr.wr_id = tx->tlc_wrid++; 2651 tx->tlc_wr.wr_flags = IBT_WR_SEND_SIGNAL; 2652 tx->tlc_wr.wr_nds = 1; 2653 tx->tlc_wr.wr_sgl = &tx->tlc_sgl; 2654 tx->tlc_wr.wr_opcode = IBT_WRC_RDMAW; 2655 tx->tlc_wr.wr_trans = IBT_RC_SRV; 2656 2657 /* Initialize the remote address for RX buffer */ 2658 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_raddr = rx->tlc_mrdesc.md_vaddr; 2659 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_rkey = rx->tlc_mrdesc.md_rkey; 2660 tx->tlc_complete = 0; 2661 ret = tavor_post_send(lstate->tls_state, tx->tlc_qp_hdl, &tx->tlc_wr, 2662 1, NULL); 2663 if (ret != IBT_SUCCESS) { 2664 return (EFAULT); 2665 } 2666 return (0); 2667 } 2668 2669 /* 2670 * tavor_loopback_poll_cq 2671 */ 2672 static int 2673 tavor_loopback_poll_cq(tavor_loopback_state_t *lstate, 2674 tavor_loopback_comm_t *comm) 2675 { 2676 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm)) 2677 2678 comm->tlc_wc.wc_status = 0; 2679 comm->tlc_num_polled = 0; 2680 comm->tlc_status = tavor_cq_poll(lstate->tls_state, 2681 comm->tlc_cqhdl[0], &comm->tlc_wc, 1, &comm->tlc_num_polled); 2682 if ((comm->tlc_status == IBT_SUCCESS) && 2683 (comm->tlc_wc.wc_status != IBT_WC_SUCCESS)) { 2684 comm->tlc_status = ibc_get_ci_failure(0); 2685 } 2686 return (comm->tlc_status); 2687 } 2688