1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * tavor_ioctl.c
29 * Tavor IOCTL Routines
30 *
31 * Implements all ioctl access into the driver. This includes all routines
32 * necessary for updating firmware, accessing the tavor flash device, and
33 * providing interfaces for VTS.
34 */
35
36 #include <sys/types.h>
37 #include <sys/conf.h>
38 #include <sys/ddi.h>
39 #include <sys/sunddi.h>
40 #include <sys/modctl.h>
41 #include <sys/file.h>
42
43 #include <sys/ib/adapters/tavor/tavor.h>
44
45 /* Tavor HCA state pointer (extern) */
46 extern void *tavor_statep;
47
48 /*
49 * The ioctl declarations (for firmware flash burning, register read/write
50 * (DEBUG-only), and VTS interfaces)
51 */
52 static int tavor_ioctl_flash_read(tavor_state_t *state, dev_t dev,
53 intptr_t arg, int mode);
54 static int tavor_ioctl_flash_write(tavor_state_t *state, dev_t dev,
55 intptr_t arg, int mode);
56 static int tavor_ioctl_flash_erase(tavor_state_t *state, dev_t dev,
57 intptr_t arg, int mode);
58 static int tavor_ioctl_flash_init(tavor_state_t *state, dev_t dev,
59 intptr_t arg, int mode);
60 static int tavor_ioctl_flash_fini(tavor_state_t *state, dev_t dev);
61 static void tavor_ioctl_flash_cleanup(tavor_state_t *state);
62 static void tavor_ioctl_flash_cleanup_nolock(tavor_state_t *state);
63 #ifdef DEBUG
64 static int tavor_ioctl_reg_write(tavor_state_t *state, intptr_t arg,
65 int mode);
66 static int tavor_ioctl_reg_read(tavor_state_t *state, intptr_t arg,
67 int mode);
68 #endif /* DEBUG */
69 static int tavor_ioctl_info(tavor_state_t *state, dev_t dev,
70 intptr_t arg, int mode);
71 static int tavor_ioctl_ports(tavor_state_t *state, intptr_t arg,
72 int mode);
73 static int tavor_ioctl_loopback(tavor_state_t *state, intptr_t arg,
74 int mode);
75 static int tavor_ioctl_ddr_read(tavor_state_t *state, intptr_t arg,
76 int mode);
77
78 /* Tavor Flash Functions */
79 static void tavor_flash_read_sector(tavor_state_t *state, uint32_t sector_num);
80 static void tavor_flash_read_quadlet(tavor_state_t *state, uint32_t *data,
81 uint32_t addr);
82 static int tavor_flash_write_sector(tavor_state_t *state, uint32_t sector_num);
83 static int tavor_flash_write_byte(tavor_state_t *state, uint32_t addr,
84 uchar_t data);
85 static int tavor_flash_erase_sector(tavor_state_t *state, uint32_t sector_num);
86 static int tavor_flash_erase_chip(tavor_state_t *state);
87 static void tavor_flash_bank(tavor_state_t *state, uint32_t addr);
88 static uint32_t tavor_flash_read(tavor_state_t *state, uint32_t addr);
89 static void tavor_flash_write(tavor_state_t *state, uint32_t addr,
90 uchar_t data);
91 static void tavor_flash_init(tavor_state_t *state);
92 static void tavor_flash_cfi_init(tavor_state_t *state, uint32_t *cfi_info,
93 int *intel_xcmd);
94 static void tavor_flash_fini(tavor_state_t *state);
95 static void tavor_flash_reset(tavor_state_t *state);
96 static uint32_t tavor_flash_read_cfg(ddi_acc_handle_t pci_config_hdl,
97 uint32_t addr);
98 static void tavor_flash_write_cfg(ddi_acc_handle_t pci_config_hdl,
99 uint32_t addr, uint32_t data);
100 static void tavor_flash_cfi_byte(uint8_t *ch, uint32_t dword, int i);
101 static void tavor_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i);
102
103 /* Tavor loopback test functions */
104 static void tavor_loopback_free_qps(tavor_loopback_state_t *lstate);
105 static void tavor_loopback_free_state(tavor_loopback_state_t *lstate);
106 static int tavor_loopback_init(tavor_state_t *state,
107 tavor_loopback_state_t *lstate);
108 static void tavor_loopback_init_qp_info(tavor_loopback_state_t *lstate,
109 tavor_loopback_comm_t *comm);
110 static int tavor_loopback_alloc_mem(tavor_loopback_state_t *lstate,
111 tavor_loopback_comm_t *comm, int sz);
112 static int tavor_loopback_alloc_qps(tavor_loopback_state_t *lstate,
113 tavor_loopback_comm_t *comm);
114 static int tavor_loopback_modify_qp(tavor_loopback_state_t *lstate,
115 tavor_loopback_comm_t *comm, uint_t qp_num);
116 static int tavor_loopback_copyout(tavor_loopback_ioctl_t *lb,
117 intptr_t arg, int mode);
118 static int tavor_loopback_post_send(tavor_loopback_state_t *lstate,
119 tavor_loopback_comm_t *tx, tavor_loopback_comm_t *rx);
120 static int tavor_loopback_poll_cq(tavor_loopback_state_t *lstate,
121 tavor_loopback_comm_t *comm);
122
123 /* Patchable timeout values for flash operations */
124 int tavor_hw_flash_timeout_gpio_sema = TAVOR_HW_FLASH_TIMEOUT_GPIO_SEMA;
125 int tavor_hw_flash_timeout_config = TAVOR_HW_FLASH_TIMEOUT_CONFIG;
126 int tavor_hw_flash_timeout_write = TAVOR_HW_FLASH_TIMEOUT_WRITE;
127 int tavor_hw_flash_timeout_erase = TAVOR_HW_FLASH_TIMEOUT_ERASE;
128
129 /*
130 * tavor_ioctl()
131 */
132 /* ARGSUSED */
133 int
tavor_ioctl(dev_t dev,int cmd,intptr_t arg,int mode,cred_t * credp,int * rvalp)134 tavor_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
135 int *rvalp)
136 {
137 tavor_state_t *state;
138 minor_t instance;
139 int status;
140
141 if (drv_priv(credp) != 0) {
142 return (EPERM);
143 }
144
145 instance = TAVOR_DEV_INSTANCE(dev);
146 if (instance == -1) {
147 return (EBADF);
148 }
149
150 state = ddi_get_soft_state(tavor_statep, instance);
151 if (state == NULL) {
152 return (EBADF);
153 }
154
155 status = 0;
156
157 switch (cmd) {
158 case TAVOR_IOCTL_FLASH_READ:
159 status = tavor_ioctl_flash_read(state, dev, arg, mode);
160 break;
161
162 case TAVOR_IOCTL_FLASH_WRITE:
163 status = tavor_ioctl_flash_write(state, dev, arg, mode);
164 break;
165
166 case TAVOR_IOCTL_FLASH_ERASE:
167 status = tavor_ioctl_flash_erase(state, dev, arg, mode);
168 break;
169
170 case TAVOR_IOCTL_FLASH_INIT:
171 status = tavor_ioctl_flash_init(state, dev, arg, mode);
172 break;
173
174 case TAVOR_IOCTL_FLASH_FINI:
175 status = tavor_ioctl_flash_fini(state, dev);
176 break;
177
178 case TAVOR_IOCTL_INFO:
179 status = tavor_ioctl_info(state, dev, arg, mode);
180 break;
181
182 case TAVOR_IOCTL_PORTS:
183 status = tavor_ioctl_ports(state, arg, mode);
184 break;
185
186 case TAVOR_IOCTL_DDR_READ:
187 status = tavor_ioctl_ddr_read(state, arg, mode);
188 break;
189
190 case TAVOR_IOCTL_LOOPBACK:
191 status = tavor_ioctl_loopback(state, arg, mode);
192 break;
193
194 #ifdef DEBUG
195 case TAVOR_IOCTL_REG_WRITE:
196 status = tavor_ioctl_reg_write(state, arg, mode);
197 break;
198
199 case TAVOR_IOCTL_REG_READ:
200 status = tavor_ioctl_reg_read(state, arg, mode);
201 break;
202 #endif /* DEBUG */
203
204 default:
205 status = ENOTTY;
206 break;
207 }
208 *rvalp = status;
209
210 return (status);
211 }
212
213 /*
214 * tavor_ioctl_flash_read()
215 */
216 static int
tavor_ioctl_flash_read(tavor_state_t * state,dev_t dev,intptr_t arg,int mode)217 tavor_ioctl_flash_read(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
218 {
219 tavor_flash_ioctl_t ioctl_info;
220 int status = 0;
221
222 /*
223 * Check that flash init ioctl has been called first. And check
224 * that the same dev_t that called init is the one calling read now.
225 */
226 mutex_enter(&state->ts_fw_flashlock);
227 if ((state->ts_fw_flashdev != dev) ||
228 (state->ts_fw_flashstarted == 0)) {
229 mutex_exit(&state->ts_fw_flashlock);
230 return (EIO);
231 }
232
233 /* copy user struct to kernel */
234 #ifdef _MULTI_DATAMODEL
235 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
236 tavor_flash_ioctl32_t info32;
237
238 if (ddi_copyin((void *)arg, &info32,
239 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
240 mutex_exit(&state->ts_fw_flashlock);
241 return (EFAULT);
242 }
243 ioctl_info.tf_type = info32.tf_type;
244 ioctl_info.tf_sector = (caddr_t)(uintptr_t)info32.tf_sector;
245 ioctl_info.tf_sector_num = info32.tf_sector_num;
246 ioctl_info.tf_addr = info32.tf_addr;
247 } else
248 #endif /* _MULTI_DATAMODEL */
249 if (ddi_copyin((void *)arg, &ioctl_info, sizeof (tavor_flash_ioctl_t),
250 mode) != 0) {
251 mutex_exit(&state->ts_fw_flashlock);
252 return (EFAULT);
253 }
254
255 /*
256 * Determine type of READ ioctl
257 */
258 switch (ioctl_info.tf_type) {
259 case TAVOR_FLASH_READ_SECTOR:
260 /* Check if sector num is too large for flash device */
261 if (ioctl_info.tf_sector_num >=
262 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) {
263 mutex_exit(&state->ts_fw_flashlock);
264 return (EFAULT);
265 }
266
267 /* Perform the Sector Read */
268 tavor_flash_reset(state);
269 tavor_flash_read_sector(state, ioctl_info.tf_sector_num);
270
271 /* copyout the firmware sector image data */
272 if (ddi_copyout(&state->ts_fw_sector[0],
273 &ioctl_info.tf_sector[0], 1 << state->ts_fw_log_sector_sz,
274 mode) != 0) {
275 mutex_exit(&state->ts_fw_flashlock);
276 return (EFAULT);
277 }
278 break;
279
280 case TAVOR_FLASH_READ_QUADLET:
281 /* Check if addr is too large for flash device */
282 if (ioctl_info.tf_addr >= state->ts_fw_device_sz) {
283 mutex_exit(&state->ts_fw_flashlock);
284 return (EFAULT);
285 }
286
287 /* Perform the Quadlet Read */
288 tavor_flash_reset(state);
289 tavor_flash_read_quadlet(state, &ioctl_info.tf_quadlet,
290 ioctl_info.tf_addr);
291 break;
292
293 default:
294 status = EIO;
295 break;
296 }
297
298 /* copy results back to userland */
299 #ifdef _MULTI_DATAMODEL
300 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
301 tavor_flash_ioctl32_t info32;
302
303 info32.tf_quadlet = ioctl_info.tf_quadlet;
304 info32.tf_type = ioctl_info.tf_type;
305 info32.tf_sector_num = ioctl_info.tf_sector_num;
306 info32.tf_sector = (caddr32_t)(uintptr_t)ioctl_info.tf_sector;
307 info32.tf_addr = ioctl_info.tf_addr;
308
309 if (ddi_copyout(&info32, (void *)arg,
310 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
311 mutex_exit(&state->ts_fw_flashlock);
312 return (EFAULT);
313 }
314 } else
315 #endif /* _MULTI_DATAMODEL */
316 if (ddi_copyout(&ioctl_info, (void *)arg,
317 sizeof (tavor_flash_ioctl_t), mode) != 0) {
318 mutex_exit(&state->ts_fw_flashlock);
319 return (EFAULT);
320 }
321
322 mutex_exit(&state->ts_fw_flashlock);
323 return (status);
324 }
325
326 /*
327 * tavor_ioctl_flash_write()
328 */
329 static int
tavor_ioctl_flash_write(tavor_state_t * state,dev_t dev,intptr_t arg,int mode)330 tavor_ioctl_flash_write(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
331 {
332 tavor_flash_ioctl_t ioctl_info;
333 int status = 0;
334
335 /*
336 * Check that flash init ioctl has been called first. And check
337 * that the same dev_t that called init is the one calling write now.
338 */
339 mutex_enter(&state->ts_fw_flashlock);
340 if ((state->ts_fw_flashdev != dev) ||
341 (state->ts_fw_flashstarted == 0)) {
342 mutex_exit(&state->ts_fw_flashlock);
343 return (EIO);
344 }
345
346 /* copy user struct to kernel */
347 #ifdef _MULTI_DATAMODEL
348 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
349 tavor_flash_ioctl32_t info32;
350
351 if (ddi_copyin((void *)arg, &info32,
352 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
353 mutex_exit(&state->ts_fw_flashlock);
354 return (EFAULT);
355 }
356 ioctl_info.tf_type = info32.tf_type;
357 ioctl_info.tf_sector = (caddr_t)(uintptr_t)info32.tf_sector;
358 ioctl_info.tf_sector_num = info32.tf_sector_num;
359 ioctl_info.tf_addr = info32.tf_addr;
360 ioctl_info.tf_byte = info32.tf_byte;
361 } else
362 #endif /* _MULTI_DATAMODEL */
363 if (ddi_copyin((void *)arg, &ioctl_info,
364 sizeof (tavor_flash_ioctl_t), mode) != 0) {
365 mutex_exit(&state->ts_fw_flashlock);
366 return (EFAULT);
367 }
368
369 /*
370 * Determine type of WRITE ioctl
371 */
372 switch (ioctl_info.tf_type) {
373 case TAVOR_FLASH_WRITE_SECTOR:
374 /* Check if sector num is too large for flash device */
375 if (ioctl_info.tf_sector_num >=
376 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) {
377 mutex_exit(&state->ts_fw_flashlock);
378 return (EFAULT);
379 }
380
381 /* copy in fw sector image data */
382 if (ddi_copyin(&ioctl_info.tf_sector[0],
383 &state->ts_fw_sector[0], 1 << state->ts_fw_log_sector_sz,
384 mode) != 0) {
385 mutex_exit(&state->ts_fw_flashlock);
386 return (EFAULT);
387 }
388
389 /* Perform Write Sector */
390 status = tavor_flash_write_sector(state,
391 ioctl_info.tf_sector_num);
392 break;
393
394 case TAVOR_FLASH_WRITE_BYTE:
395 /* Check if addr is too large for flash device */
396 if (ioctl_info.tf_addr >= state->ts_fw_device_sz) {
397 mutex_exit(&state->ts_fw_flashlock);
398 return (EFAULT);
399 }
400
401 /* Perform Write Byte */
402 tavor_flash_bank(state, ioctl_info.tf_addr);
403 tavor_flash_reset(state);
404 status = tavor_flash_write_byte(state, ioctl_info.tf_addr,
405 ioctl_info.tf_byte);
406 tavor_flash_reset(state);
407 break;
408
409 default:
410 status = EIO;
411 break;
412 }
413
414 mutex_exit(&state->ts_fw_flashlock);
415 return (status);
416 }
417
418 /*
419 * tavor_ioctl_flash_erase()
420 */
421 static int
tavor_ioctl_flash_erase(tavor_state_t * state,dev_t dev,intptr_t arg,int mode)422 tavor_ioctl_flash_erase(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
423 {
424 tavor_flash_ioctl_t ioctl_info;
425 int status = 0;
426
427 /*
428 * Check that flash init ioctl has been called first. And check
429 * that the same dev_t that called init is the one calling erase now.
430 */
431 mutex_enter(&state->ts_fw_flashlock);
432 if ((state->ts_fw_flashdev != dev) ||
433 (state->ts_fw_flashstarted == 0)) {
434 mutex_exit(&state->ts_fw_flashlock);
435 return (EIO);
436 }
437
438 /* copy user struct to kernel */
439 #ifdef _MULTI_DATAMODEL
440 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
441 tavor_flash_ioctl32_t info32;
442
443 if (ddi_copyin((void *)arg, &info32,
444 sizeof (tavor_flash_ioctl32_t), mode) != 0) {
445 mutex_exit(&state->ts_fw_flashlock);
446 return (EFAULT);
447 }
448 ioctl_info.tf_type = info32.tf_type;
449 ioctl_info.tf_sector_num = info32.tf_sector_num;
450 } else
451 #endif /* _MULTI_DATAMODEL */
452 if (ddi_copyin((void *)arg, &ioctl_info, sizeof (tavor_flash_ioctl_t),
453 mode) != 0) {
454 mutex_exit(&state->ts_fw_flashlock);
455 return (EFAULT);
456 }
457
458 /*
459 * Determine type of ERASE ioctl
460 */
461 switch (ioctl_info.tf_type) {
462 case TAVOR_FLASH_ERASE_SECTOR:
463 /* Check if sector num is too large for flash device */
464 if (ioctl_info.tf_sector_num >=
465 (state->ts_fw_device_sz >> state->ts_fw_log_sector_sz)) {
466 mutex_exit(&state->ts_fw_flashlock);
467 return (EFAULT);
468 }
469
470 /* Perform Sector Erase */
471 status = tavor_flash_erase_sector(state,
472 ioctl_info.tf_sector_num);
473 break;
474
475 case TAVOR_FLASH_ERASE_CHIP:
476 /* Perform Chip Erase */
477 status = tavor_flash_erase_chip(state);
478 break;
479
480 default:
481 status = EIO;
482 break;
483 }
484
485 mutex_exit(&state->ts_fw_flashlock);
486 return (status);
487 }
488
489 /*
490 * tavor_ioctl_flash_init()
491 */
492 static int
tavor_ioctl_flash_init(tavor_state_t * state,dev_t dev,intptr_t arg,int mode)493 tavor_ioctl_flash_init(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
494 {
495 tavor_flash_init_ioctl_t init_info;
496 int ret;
497 int intel_xcmd = 0;
498
499 /*
500 * init cannot be called more than once. If we have already init'd the
501 * flash, return directly.
502 */
503 mutex_enter(&state->ts_fw_flashlock);
504 if (state->ts_fw_flashstarted == 1) {
505 mutex_exit(&state->ts_fw_flashlock);
506 return (EIO);
507 }
508
509 /* copyin the user struct to kernel */
510 if (ddi_copyin((void *)arg, &init_info,
511 sizeof (tavor_flash_init_ioctl_t), mode) != 0) {
512 mutex_exit(&state->ts_fw_flashlock);
513 return (EFAULT);
514 }
515
516 /* Init Flash */
517 tavor_flash_init(state);
518
519 /* Read CFI info */
520 tavor_flash_cfi_init(state, &init_info.tf_cfi_info[0], &intel_xcmd);
521
522 /*
523 * Return error if the command set is unknown.
524 */
525 if (state->ts_fw_cmdset == TAVOR_FLASH_UNKNOWN_CMDSET) {
526 mutex_exit(&state->ts_fw_flashlock);
527 return (EFAULT);
528 }
529
530 /* Read HWREV - least significant 8 bits is revision ID */
531 init_info.tf_hwrev = pci_config_get32(state->ts_pci_cfghdl,
532 TAVOR_HW_FLASH_CFG_HWREV) & 0xFF;
533
534 /* Fill in the firmwate revision numbers */
535 init_info.tf_fwrev.tfi_maj = state->ts_fw.fw_rev_major;
536 init_info.tf_fwrev.tfi_min = state->ts_fw.fw_rev_minor;
537 init_info.tf_fwrev.tfi_sub = state->ts_fw.fw_rev_subminor;
538
539 /* Alloc flash mem for one sector size */
540 state->ts_fw_sector = (uint32_t *)kmem_zalloc(1 <<
541 state->ts_fw_log_sector_sz, KM_SLEEP);
542
543 /* Set HW part number and length */
544 init_info.tf_pn_len = state->ts_hca_pn_len;
545 if (state->ts_hca_pn_len != 0) {
546 (void) memcpy(init_info.tf_hwpn, state->ts_hca_pn,
547 state->ts_hca_pn_len);
548 }
549
550 /* Copy ioctl results back to userland */
551 if (ddi_copyout(&init_info, (void *)arg,
552 sizeof (tavor_flash_init_ioctl_t), mode) != 0) {
553
554 tavor_ioctl_flash_cleanup_nolock(state);
555
556 mutex_exit(&state->ts_fw_flashlock);
557 return (EFAULT);
558 }
559
560 /* Set flash state to started */
561 state->ts_fw_flashstarted = 1;
562 state->ts_fw_flashdev = dev;
563
564 mutex_exit(&state->ts_fw_flashlock);
565
566 /*
567 * If "flash init" is successful, add an "on close" callback to the
568 * current dev node to ensure that "flash fini" gets called later
569 * even if the userland process prematurely exits.
570 */
571 ret = tavor_umap_db_set_onclose_cb(dev,
572 TAVOR_ONCLOSE_FLASH_INPROGRESS,
573 (void (*)(void *))tavor_ioctl_flash_cleanup, state);
574 if (ret != DDI_SUCCESS) {
575 (void) tavor_ioctl_flash_fini(state, dev);
576
577 return (EFAULT);
578 }
579
580 return (0);
581 }
582
583 /*
584 * tavor_ioctl_flash_fini()
585 */
586 static int
tavor_ioctl_flash_fini(tavor_state_t * state,dev_t dev)587 tavor_ioctl_flash_fini(tavor_state_t *state, dev_t dev)
588 {
589 int ret;
590
591 /*
592 * Check that flash init ioctl has been called first. And check
593 * that the same dev_t that called init is the one calling fini now.
594 */
595 mutex_enter(&state->ts_fw_flashlock);
596 if ((state->ts_fw_flashdev != dev) ||
597 (state->ts_fw_flashstarted == 0)) {
598 mutex_exit(&state->ts_fw_flashlock);
599 return (EIO);
600 }
601
602 tavor_ioctl_flash_cleanup_nolock(state);
603
604 mutex_exit(&state->ts_fw_flashlock);
605
606 /*
607 * If "flash fini" is successful, remove the "on close" callback
608 * that was setup during "flash init".
609 */
610 ret = tavor_umap_db_clear_onclose_cb(dev,
611 TAVOR_ONCLOSE_FLASH_INPROGRESS);
612 if (ret != DDI_SUCCESS) {
613 return (EFAULT);
614 }
615
616 return (0);
617 }
618
619
620 /*
621 * tavor_ioctl_flash_cleanup()
622 */
623 static void
tavor_ioctl_flash_cleanup(tavor_state_t * state)624 tavor_ioctl_flash_cleanup(tavor_state_t *state)
625 {
626 mutex_enter(&state->ts_fw_flashlock);
627 tavor_ioctl_flash_cleanup_nolock(state);
628 mutex_exit(&state->ts_fw_flashlock);
629 }
630
631
632 /*
633 * tavor_ioctl_flash_cleanup_nolock()
634 */
635 static void
tavor_ioctl_flash_cleanup_nolock(tavor_state_t * state)636 tavor_ioctl_flash_cleanup_nolock(tavor_state_t *state)
637 {
638 ASSERT(MUTEX_HELD(&state->ts_fw_flashlock));
639
640 /* free flash mem */
641 kmem_free(state->ts_fw_sector, 1 << state->ts_fw_log_sector_sz);
642
643 /* Fini the Flash */
644 tavor_flash_fini(state);
645
646 /* Set flash state to fini */
647 state->ts_fw_flashstarted = 0;
648 state->ts_fw_flashdev = 0;
649 }
650
651
652 /*
653 * tavor_ioctl_info()
654 */
655 static int
tavor_ioctl_info(tavor_state_t * state,dev_t dev,intptr_t arg,int mode)656 tavor_ioctl_info(tavor_state_t *state, dev_t dev, intptr_t arg, int mode)
657 {
658 tavor_info_ioctl_t info;
659 tavor_flash_init_ioctl_t init_info;
660
661 /*
662 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
663 */
664 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
665 return (EFAULT);
666 }
667
668 /* copyin the user struct to kernel */
669 if (ddi_copyin((void *)arg, &info, sizeof (tavor_info_ioctl_t),
670 mode) != 0) {
671 return (EFAULT);
672 }
673
674 /*
675 * Check ioctl revision
676 */
677 if (info.ti_revision != TAVOR_VTS_IOCTL_REVISION) {
678 return (EINVAL);
679 }
680
681 /*
682 * If the 'fw_device_sz' has not been initialized yet, we initialize it
683 * here. This is done by leveraging the
684 * tavor_ioctl_flash_init()/fini() calls. We also hold our own mutex
685 * around this operation in case we have multiple VTS threads in
686 * process at the same time.
687 */
688 mutex_enter(&state->ts_info_lock);
689 if (state->ts_fw_device_sz == 0) {
690 if (tavor_ioctl_flash_init(state, dev, (intptr_t)&init_info,
691 (FKIOCTL | mode)) != 0) {
692 mutex_exit(&state->ts_info_lock);
693 return (EFAULT);
694 }
695 (void) tavor_ioctl_flash_fini(state, dev);
696 }
697 mutex_exit(&state->ts_info_lock);
698
699 info.ti_hw_rev = state->ts_adapter.rev_id;
700 info.ti_flash_sz = state->ts_fw_device_sz;
701 info.ti_fw_rev.tfi_maj = state->ts_fw.fw_rev_major;
702 info.ti_fw_rev.tfi_min = state->ts_fw.fw_rev_minor;
703 info.ti_fw_rev.tfi_sub = state->ts_fw.fw_rev_subminor;
704 info.ti_mem_start_offset = 0;
705 info.ti_mem_end_offset = state->ts_ddr.ddr_endaddr -
706 state->ts_ddr.ddr_baseaddr;
707
708 /* Copy ioctl results back to user struct */
709 if (ddi_copyout(&info, (void *)arg, sizeof (tavor_info_ioctl_t),
710 mode) != 0) {
711 return (EFAULT);
712 }
713
714 return (0);
715 }
716
717 /*
718 * tavor_ioctl_ports()
719 */
720 static int
tavor_ioctl_ports(tavor_state_t * state,intptr_t arg,int mode)721 tavor_ioctl_ports(tavor_state_t *state, intptr_t arg, int mode)
722 {
723 tavor_ports_ioctl_t info;
724 tavor_stat_port_ioctl_t portstat;
725 ibt_hca_portinfo_t pi;
726 uint_t tbl_size;
727 ib_gid_t *sgid_tbl;
728 ib_pkey_t *pkey_tbl;
729 int i;
730
731 /*
732 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
733 */
734 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
735 return (EFAULT);
736 }
737
738 /* copyin the user struct to kernel */
739 #ifdef _MULTI_DATAMODEL
740 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
741 tavor_ports_ioctl32_t info32;
742
743 if (ddi_copyin((void *)arg, &info32,
744 sizeof (tavor_ports_ioctl32_t), mode) != 0) {
745 return (EFAULT);
746 }
747 info.tp_revision = info32.tp_revision;
748 info.tp_ports =
749 (tavor_stat_port_ioctl_t *)(uintptr_t)info32.tp_ports;
750 info.tp_num_ports = info32.tp_num_ports;
751
752 } else
753 #endif /* _MULTI_DATAMODEL */
754 if (ddi_copyin((void *)arg, &info, sizeof (tavor_ports_ioctl_t),
755 mode) != 0) {
756 return (EFAULT);
757 }
758
759 /*
760 * Check ioctl revision
761 */
762 if (info.tp_revision != TAVOR_VTS_IOCTL_REVISION) {
763 return (EINVAL);
764 }
765
766 /* Allocate space for temporary GID table/PKey table */
767 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
768 sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t),
769 KM_SLEEP);
770 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
771 pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t),
772 KM_SLEEP);
773
774 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgid_tbl, *pkey_tbl))
775
776 /*
777 * Setup the number of ports, then loop through all ports and
778 * query properties of each.
779 */
780 info.tp_num_ports = (uint8_t)state->ts_cfg_profile->cp_num_ports;
781 for (i = 0; i < info.tp_num_ports; i++) {
782 /*
783 * Get portstate information from the device. If
784 * tavor_port_query() fails, leave zeroes in user
785 * struct port entry and continue.
786 */
787 bzero(&pi, sizeof (ibt_hca_portinfo_t));
788 pi.p_sgid_tbl = sgid_tbl;
789 pi.p_pkey_tbl = pkey_tbl;
790 (void) tavor_port_query(state, i + 1, &pi);
791
792 portstat.tsp_port_num = pi.p_port_num;
793 portstat.tsp_state = pi.p_linkstate;
794 portstat.tsp_guid = pi.p_sgid_tbl[0].gid_guid;
795
796 /*
797 * Copy queried port results back to user struct. If
798 * this fails, then break out of loop, attempt to copy
799 * out remaining info to user struct, and return (without
800 * error).
801 */
802 if (ddi_copyout(&portstat,
803 &(((tavor_stat_port_ioctl_t *)info.tp_ports)[i]),
804 sizeof (tavor_stat_port_ioctl_t), mode) != 0) {
805 break;
806 }
807 }
808
809 /* Free the temporary space used for GID table/PKey table */
810 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
811 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
812 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
813 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
814
815 /* Copy ioctl results back to user struct */
816 #ifdef _MULTI_DATAMODEL
817 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
818 tavor_ports_ioctl32_t info32;
819
820 info32.tp_revision = info.tp_revision;
821 info32.tp_ports = (caddr32_t)(uintptr_t)info.tp_ports;
822 info32.tp_num_ports = info.tp_num_ports;
823
824 if (ddi_copyout(&info32, (void *)arg,
825 sizeof (tavor_ports_ioctl32_t), mode) != 0) {
826 return (EFAULT);
827 }
828 } else
829 #endif /* _MULTI_DATAMODEL */
830 if (ddi_copyout(&info, (void *)arg, sizeof (tavor_ports_ioctl_t),
831 mode) != 0) {
832 return (EFAULT);
833 }
834
835 return (0);
836 }
837
838 /*
839 * tavor_ioctl_loopback()
840 */
841 static int
tavor_ioctl_loopback(tavor_state_t * state,intptr_t arg,int mode)842 tavor_ioctl_loopback(tavor_state_t *state, intptr_t arg, int mode)
843 {
844 tavor_loopback_ioctl_t lb;
845 tavor_loopback_state_t lstate;
846 ibt_hca_portinfo_t pi;
847 uint_t tbl_size, loopmax, max_usec;
848 ib_gid_t *sgid_tbl;
849 ib_pkey_t *pkey_tbl;
850 int j, iter, ret;
851
852 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(lstate))
853
854 /*
855 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
856 */
857 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
858 return (EFAULT);
859 }
860
861 /* copyin the user struct to kernel */
862 #ifdef _MULTI_DATAMODEL
863 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
864 tavor_loopback_ioctl32_t lb32;
865
866 if (ddi_copyin((void *)arg, &lb32,
867 sizeof (tavor_loopback_ioctl32_t), mode) != 0) {
868 return (EFAULT);
869 }
870 lb.tlb_revision = lb32.tlb_revision;
871 lb.tlb_send_buf = (caddr_t)(uintptr_t)lb32.tlb_send_buf;
872 lb.tlb_fail_buf = (caddr_t)(uintptr_t)lb32.tlb_fail_buf;
873 lb.tlb_buf_sz = lb32.tlb_buf_sz;
874 lb.tlb_num_iter = lb32.tlb_num_iter;
875 lb.tlb_pass_done = lb32.tlb_pass_done;
876 lb.tlb_timeout = lb32.tlb_timeout;
877 lb.tlb_error_type = lb32.tlb_error_type;
878 lb.tlb_port_num = lb32.tlb_port_num;
879 lb.tlb_num_retry = lb32.tlb_num_retry;
880 } else
881 #endif /* _MULTI_DATAMODEL */
882 if (ddi_copyin((void *)arg, &lb, sizeof (tavor_loopback_ioctl_t),
883 mode) != 0) {
884 return (EFAULT);
885 }
886
887 /* Initialize the internal loopback test state structure */
888 bzero(&lstate, sizeof (tavor_loopback_state_t));
889
890 /*
891 * Check ioctl revision
892 */
893 if (lb.tlb_revision != TAVOR_VTS_IOCTL_REVISION) {
894 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_REVISION;
895 (void) tavor_loopback_copyout(&lb, arg, mode);
896 return (EINVAL);
897 }
898
899 /* Validate that specified port number is legal */
900 if (!tavor_portnum_is_valid(state, lb.tlb_port_num)) {
901 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_PORT;
902 (void) tavor_loopback_copyout(&lb, arg, mode);
903 return (EINVAL);
904 }
905
906 /* Allocate space for temporary GID table/PKey table */
907 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
908 sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t),
909 KM_SLEEP);
910 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
911 pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t),
912 KM_SLEEP);
913
914 /*
915 * Get portstate information from specific port on device
916 */
917 bzero(&pi, sizeof (ibt_hca_portinfo_t));
918 pi.p_sgid_tbl = sgid_tbl;
919 pi.p_pkey_tbl = pkey_tbl;
920 if (tavor_port_query(state, lb.tlb_port_num, &pi) != 0) {
921 /* Free the temporary space used for GID table/PKey table */
922 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
923 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
924 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
925 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
926
927 lb.tlb_error_type = TAVOR_LOOPBACK_INVALID_PORT;
928 (void) tavor_loopback_copyout(&lb, arg, mode);
929 tavor_loopback_free_state(&lstate);
930 return (EINVAL);
931 }
932
933 lstate.tls_port = pi.p_port_num;
934 lstate.tls_lid = pi.p_base_lid;
935 lstate.tls_pkey_ix = (pi.p_linkstate == TAVOR_PORT_LINK_ACTIVE) ? 1 : 0;
936 lstate.tls_state = state;
937 lstate.tls_retry = lb.tlb_num_retry;
938
939 /* Free the temporary space used for GID table/PKey table */
940 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_gidtbl);
941 kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
942 tbl_size = (1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
943 kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
944
945 /*
946 * Compute the timeout duration in usec per the formula:
947 * to_usec_per_retry = 4.096us * (2 ^ supplied_timeout)
948 * (plus we add a little fudge-factor here too)
949 */
950 lstate.tls_timeout = lb.tlb_timeout;
951 max_usec = (4096 * (1 << lstate.tls_timeout)) / 1000;
952 max_usec = max_usec * (lstate.tls_retry + 1);
953 max_usec = max_usec + 10000;
954
955 /*
956 * Determine how many times we should loop before declaring a
957 * timeout failure.
958 */
959 loopmax = max_usec/TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR;
960 if ((max_usec % TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR) != 0) {
961 loopmax++;
962 }
963
964 if (lb.tlb_send_buf == NULL || lb.tlb_buf_sz == 0) {
965 lb.tlb_error_type = TAVOR_LOOPBACK_SEND_BUF_INVALID;
966 (void) tavor_loopback_copyout(&lb, arg, mode);
967 tavor_loopback_free_state(&lstate);
968 return (EINVAL);
969 }
970
971 /* Allocate protection domain (PD) */
972 if (tavor_loopback_init(state, &lstate) != 0) {
973 lb.tlb_error_type = lstate.tls_err;
974 (void) tavor_loopback_copyout(&lb, arg, mode);
975 tavor_loopback_free_state(&lstate);
976 return (EFAULT);
977 }
978
979 /* Allocate and register a TX buffer */
980 if (tavor_loopback_alloc_mem(&lstate, &lstate.tls_tx,
981 lb.tlb_buf_sz) != 0) {
982 lb.tlb_error_type =
983 TAVOR_LOOPBACK_SEND_BUF_MEM_REGION_ALLOC_FAIL;
984 (void) tavor_loopback_copyout(&lb, arg, mode);
985 tavor_loopback_free_state(&lstate);
986 return (EFAULT);
987 }
988
989 /* Allocate and register an RX buffer */
990 if (tavor_loopback_alloc_mem(&lstate, &lstate.tls_rx,
991 lb.tlb_buf_sz) != 0) {
992 lb.tlb_error_type =
993 TAVOR_LOOPBACK_RECV_BUF_MEM_REGION_ALLOC_FAIL;
994 (void) tavor_loopback_copyout(&lb, arg, mode);
995 tavor_loopback_free_state(&lstate);
996 return (EFAULT);
997 }
998
999 /* Copy in the transmit buffer data */
1000 if (ddi_copyin((void *)lb.tlb_send_buf, lstate.tls_tx.tlc_buf,
1001 lb.tlb_buf_sz, mode) != 0) {
1002 lb.tlb_error_type = TAVOR_LOOPBACK_SEND_BUF_COPY_FAIL;
1003 (void) tavor_loopback_copyout(&lb, arg, mode);
1004 tavor_loopback_free_state(&lstate);
1005 return (EFAULT);
1006 }
1007
1008 /* Allocate the transmit QP and CQs */
1009 lstate.tls_err = TAVOR_LOOPBACK_XMIT_SEND_CQ_ALLOC_FAIL;
1010 if (tavor_loopback_alloc_qps(&lstate, &lstate.tls_tx) != 0) {
1011 lb.tlb_error_type = lstate.tls_err;
1012 (void) tavor_loopback_copyout(&lb, arg, mode);
1013 tavor_loopback_free_state(&lstate);
1014 return (EFAULT);
1015 }
1016
1017 /* Allocate the receive QP and CQs */
1018 lstate.tls_err = TAVOR_LOOPBACK_RECV_SEND_CQ_ALLOC_FAIL;
1019 if (tavor_loopback_alloc_qps(&lstate, &lstate.tls_rx) != 0) {
1020 lb.tlb_error_type = lstate.tls_err;
1021 (void) tavor_loopback_copyout(&lb, arg, mode);
1022 tavor_loopback_free_state(&lstate);
1023 return (EFAULT);
1024 }
1025
1026 /* Activate the TX QP (connect to RX QP) */
1027 lstate.tls_err = TAVOR_LOOPBACK_XMIT_QP_INIT_FAIL;
1028 if (tavor_loopback_modify_qp(&lstate, &lstate.tls_tx,
1029 lstate.tls_rx.tlc_qp_num) != 0) {
1030 lb.tlb_error_type = lstate.tls_err;
1031 (void) tavor_loopback_copyout(&lb, arg, mode);
1032 tavor_loopback_free_state(&lstate);
1033 return (EFAULT);
1034 }
1035
1036 /* Activate the RX QP (connect to TX QP) */
1037 lstate.tls_err = TAVOR_LOOPBACK_RECV_QP_INIT_FAIL;
1038 if (tavor_loopback_modify_qp(&lstate, &lstate.tls_rx,
1039 lstate.tls_tx.tlc_qp_num) != 0) {
1040 lb.tlb_error_type = lstate.tls_err;
1041 (void) tavor_loopback_copyout(&lb, arg, mode);
1042 tavor_loopback_free_state(&lstate);
1043 return (EFAULT);
1044 }
1045
1046 /* Run the loopback test (for specified number of iterations) */
1047 lb.tlb_pass_done = 0;
1048 for (iter = 0; iter < lb.tlb_num_iter; iter++) {
1049 lstate.tls_err = 0;
1050 bzero(lstate.tls_rx.tlc_buf, lb.tlb_buf_sz);
1051
1052 /* Post RDMA Write work request */
1053 if (tavor_loopback_post_send(&lstate, &lstate.tls_tx,
1054 &lstate.tls_rx) != IBT_SUCCESS) {
1055 lb.tlb_error_type = TAVOR_LOOPBACK_WQE_POST_FAIL;
1056 (void) tavor_loopback_copyout(&lb, arg, mode);
1057 tavor_loopback_free_state(&lstate);
1058 return (EFAULT);
1059 }
1060
1061 /* Poll the TX CQ for a completion every few ticks */
1062 for (j = 0; j < loopmax; j++) {
1063 delay(drv_usectohz(TAVOR_VTS_LOOPBACK_MIN_WAIT_DUR));
1064
1065 ret = tavor_loopback_poll_cq(&lstate, &lstate.tls_tx);
1066 if (((ret != IBT_SUCCESS) && (ret != IBT_CQ_EMPTY)) ||
1067 ((ret == IBT_CQ_EMPTY) && (j == loopmax - 1))) {
1068 lb.tlb_error_type = TAVOR_LOOPBACK_CQ_POLL_FAIL;
1069 if (ddi_copyout(lstate.tls_rx.tlc_buf,
1070 lb.tlb_fail_buf, lstate.tls_tx.tlc_buf_sz,
1071 mode) != 0) {
1072 return (EFAULT);
1073 }
1074 (void) tavor_loopback_copyout(&lb, arg, mode);
1075 tavor_loopback_free_state(&lstate);
1076 return (EFAULT);
1077 } else if (ret == IBT_CQ_EMPTY) {
1078 continue;
1079 }
1080
1081 /* Compare the data buffers */
1082 if (bcmp(lstate.tls_tx.tlc_buf, lstate.tls_rx.tlc_buf,
1083 lb.tlb_buf_sz) == 0) {
1084 break;
1085 } else {
1086 lb.tlb_error_type =
1087 TAVOR_LOOPBACK_SEND_RECV_COMPARE_FAIL;
1088 if (ddi_copyout(lstate.tls_rx.tlc_buf,
1089 lb.tlb_fail_buf, lstate.tls_tx.tlc_buf_sz,
1090 mode) != 0) {
1091 return (EFAULT);
1092 }
1093 (void) tavor_loopback_copyout(&lb, arg, mode);
1094 tavor_loopback_free_state(&lstate);
1095 return (EFAULT);
1096 }
1097 }
1098
1099 lstate.tls_err = TAVOR_LOOPBACK_SUCCESS;
1100 lb.tlb_pass_done = iter + 1;
1101 }
1102
1103 lb.tlb_error_type = TAVOR_LOOPBACK_SUCCESS;
1104
1105 /* Copy ioctl results back to user struct */
1106 ret = tavor_loopback_copyout(&lb, arg, mode);
1107
1108 /* Free up everything and release all consumed resources */
1109 tavor_loopback_free_state(&lstate);
1110
1111 return (ret);
1112 }
1113
1114 /*
1115 * tavor_ioctl_ddr_read()
1116 */
1117 static int
tavor_ioctl_ddr_read(tavor_state_t * state,intptr_t arg,int mode)1118 tavor_ioctl_ddr_read(tavor_state_t *state, intptr_t arg, int mode)
1119 {
1120 tavor_ddr_read_ioctl_t rdreg;
1121 uint32_t *addr;
1122 uintptr_t baseaddr;
1123 uint64_t ddr_size;
1124
1125 /*
1126 * Access to Tavor VTS ioctls is not allowed in "maintenance mode".
1127 */
1128 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1129 return (EFAULT);
1130 }
1131
1132 /* copyin the user struct to kernel */
1133 if (ddi_copyin((void *)arg, &rdreg, sizeof (tavor_ddr_read_ioctl_t),
1134 mode) != 0) {
1135 return (EFAULT);
1136 }
1137
1138 /*
1139 * Check ioctl revision
1140 */
1141 if (rdreg.tdr_revision != TAVOR_VTS_IOCTL_REVISION) {
1142 return (EINVAL);
1143 }
1144
1145 /*
1146 * Check for valid offset
1147 */
1148 ddr_size = (state->ts_ddr.ddr_endaddr - state->ts_ddr.ddr_baseaddr + 1);
1149 if ((uint64_t)rdreg.tdr_offset >= ddr_size) {
1150 return (EINVAL);
1151 }
1152
1153 /* Determine base address for requested register read */
1154 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr;
1155
1156 /* Ensure that address is properly-aligned */
1157 addr = (uint32_t *)((baseaddr + rdreg.tdr_offset) & ~0x3);
1158
1159 /* Read the register pointed to by addr */
1160 rdreg.tdr_data = ddi_get32(state->ts_reg_cmdhdl, addr);
1161
1162 /* Copy ioctl results back to user struct */
1163 if (ddi_copyout(&rdreg, (void *)arg, sizeof (tavor_ddr_read_ioctl_t),
1164 mode) != 0) {
1165 return (EFAULT);
1166 }
1167
1168 return (0);
1169 }
1170
1171
1172 #ifdef DEBUG
1173 /*
1174 * tavor_ioctl_reg_read()
1175 */
1176 static int
tavor_ioctl_reg_read(tavor_state_t * state,intptr_t arg,int mode)1177 tavor_ioctl_reg_read(tavor_state_t *state, intptr_t arg, int mode)
1178 {
1179 tavor_reg_ioctl_t rdreg;
1180 uint32_t *addr;
1181 uintptr_t baseaddr;
1182 int status;
1183
1184 /*
1185 * Access to Tavor registers is not allowed in "maintenance mode".
1186 * This is primarily because the device may not have BARs to access
1187 */
1188 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1189 return (EFAULT);
1190 }
1191
1192 /* Copy in the tavor_reg_ioctl_t structure */
1193 status = ddi_copyin((void *)arg, &rdreg, sizeof (tavor_reg_ioctl_t),
1194 mode);
1195 if (status != 0) {
1196 return (EFAULT);
1197 }
1198
1199 /* Determine base address for requested register set */
1200 switch (rdreg.trg_reg_set) {
1201 case TAVOR_CMD_BAR:
1202 baseaddr = (uintptr_t)state->ts_reg_cmd_baseaddr;
1203 break;
1204
1205 case TAVOR_UAR_BAR:
1206 baseaddr = (uintptr_t)state->ts_reg_uar_baseaddr;
1207 break;
1208
1209 case TAVOR_DDR_BAR:
1210 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr;
1211 break;
1212
1213 default:
1214 return (EFAULT);
1215 }
1216
1217 /* Ensure that address is properly-aligned */
1218 addr = (uint32_t *)((baseaddr + rdreg.trg_offset) & ~0x3);
1219
1220 /* Read the register pointed to by addr */
1221 rdreg.trg_data = ddi_get32(state->ts_reg_cmdhdl, addr);
1222
1223 /* Copy in the result into the tavor_reg_ioctl_t structure */
1224 status = ddi_copyout(&rdreg, (void *)arg, sizeof (tavor_reg_ioctl_t),
1225 mode);
1226 if (status != 0) {
1227 return (EFAULT);
1228 }
1229
1230 return (0);
1231 }
1232
1233
1234 /*
1235 * tavor_ioctl_reg_write()
1236 */
1237 static int
tavor_ioctl_reg_write(tavor_state_t * state,intptr_t arg,int mode)1238 tavor_ioctl_reg_write(tavor_state_t *state, intptr_t arg, int mode)
1239 {
1240 tavor_reg_ioctl_t wrreg;
1241 uint32_t *addr;
1242 uintptr_t baseaddr;
1243 int status;
1244
1245 /*
1246 * Access to Tavor registers is not allowed in "maintenance mode".
1247 * This is primarily because the device may not have BARs to access
1248 */
1249 if (state->ts_operational_mode == TAVOR_MAINTENANCE_MODE) {
1250 return (EFAULT);
1251 }
1252
1253 /* Copy in the tavor_reg_ioctl_t structure */
1254 status = ddi_copyin((void *)arg, &wrreg, sizeof (tavor_reg_ioctl_t),
1255 mode);
1256 if (status != 0) {
1257 return (EFAULT);
1258 }
1259
1260 /* Determine base address for requested register set */
1261 switch (wrreg.trg_reg_set) {
1262 case TAVOR_CMD_BAR:
1263 baseaddr = (uintptr_t)state->ts_reg_cmd_baseaddr;
1264 break;
1265
1266 case TAVOR_UAR_BAR:
1267 baseaddr = (uintptr_t)state->ts_reg_uar_baseaddr;
1268 break;
1269
1270 case TAVOR_DDR_BAR:
1271 baseaddr = (uintptr_t)state->ts_reg_ddr_baseaddr;
1272 break;
1273
1274 default:
1275 return (EFAULT);
1276 }
1277
1278 /* Ensure that address is properly-aligned */
1279 addr = (uint32_t *)((baseaddr + wrreg.trg_offset) & ~0x3);
1280
1281 /* Write the data to the register pointed to by addr */
1282 ddi_put32(state->ts_reg_cmdhdl, addr, wrreg.trg_data);
1283
1284 return (0);
1285 }
1286 #endif /* DEBUG */
1287
1288 /*
1289 * tavor_flash_reset()
1290 */
1291 static void
tavor_flash_reset(tavor_state_t * state)1292 tavor_flash_reset(tavor_state_t *state)
1293 {
1294 /*
1295 * Performs a reset to the flash device. After a reset the flash will
1296 * be operating in normal mode (capable of read/write, etc.).
1297 */
1298 switch (state->ts_fw_cmdset) {
1299 case TAVOR_FLASH_AMD_CMDSET:
1300 tavor_flash_write(state, 0x555, TAVOR_HW_FLASH_RESET_AMD);
1301 break;
1302
1303 case TAVOR_FLASH_INTEL_CMDSET:
1304 tavor_flash_write(state, 0x555, TAVOR_HW_FLASH_RESET_INTEL);
1305 break;
1306
1307 default:
1308 break;
1309 }
1310 }
1311
1312 /*
1313 * tavor_flash_read_sector()
1314 */
1315 static void
tavor_flash_read_sector(tavor_state_t * state,uint32_t sector_num)1316 tavor_flash_read_sector(tavor_state_t *state, uint32_t sector_num)
1317 {
1318 uint32_t addr;
1319 uint32_t end_addr;
1320 uint32_t *image;
1321 int i;
1322
1323 image = (uint32_t *)&state->ts_fw_sector[0];
1324
1325 /*
1326 * Calculate the start and end address of the sector, based on the
1327 * sector number passed in.
1328 */
1329 addr = sector_num << state->ts_fw_log_sector_sz;
1330 end_addr = addr + (1 << state->ts_fw_log_sector_sz);
1331
1332 /* Set the flash bank correctly for the given address */
1333 tavor_flash_bank(state, addr);
1334
1335 /* Read the entire sector, one quadlet at a time */
1336 for (i = 0; addr < end_addr; i++, addr += 4) {
1337 image[i] = tavor_flash_read(state, addr);
1338 }
1339 }
1340
1341 /*
1342 * tavor_flash_read_quadlet()
1343 */
1344 static void
tavor_flash_read_quadlet(tavor_state_t * state,uint32_t * data,uint32_t addr)1345 tavor_flash_read_quadlet(tavor_state_t *state, uint32_t *data,
1346 uint32_t addr)
1347 {
1348 /* Set the flash bank correctly for the given address */
1349 tavor_flash_bank(state, addr);
1350
1351 /* Read one quadlet of data */
1352 *data = tavor_flash_read(state, addr);
1353 }
1354
1355 /*
1356 * tavor_flash_write_sector()
1357 */
1358 static int
tavor_flash_write_sector(tavor_state_t * state,uint32_t sector_num)1359 tavor_flash_write_sector(tavor_state_t *state, uint32_t sector_num)
1360 {
1361 uint32_t addr;
1362 uint32_t end_addr;
1363 uchar_t *sector;
1364 int status = 0;
1365 int i;
1366
1367 sector = (uchar_t *)&state->ts_fw_sector[0];
1368
1369 /*
1370 * Calculate the start and end address of the sector, based on the
1371 * sector number passed in.
1372 */
1373 addr = sector_num << state->ts_fw_log_sector_sz;
1374 end_addr = addr + (1 << state->ts_fw_log_sector_sz);
1375
1376 /* Set the flash bank correctly for the given address */
1377 tavor_flash_bank(state, addr);
1378
1379 /* Erase the sector before writing */
1380 tavor_flash_reset(state);
1381 status = tavor_flash_erase_sector(state, sector_num);
1382 if (status != 0) {
1383 return (status);
1384 }
1385
1386 /* Write the entire sector, one byte at a time */
1387 for (i = 0; addr < end_addr; i++, addr++) {
1388 status = tavor_flash_write_byte(state, addr, sector[i]);
1389 if (status != 0) {
1390 break;
1391 }
1392 }
1393
1394 tavor_flash_reset(state);
1395 return (status);
1396 }
1397
1398 /*
1399 * tavor_flash_write_byte()
1400 */
1401 static int
tavor_flash_write_byte(tavor_state_t * state,uint32_t addr,uchar_t data)1402 tavor_flash_write_byte(tavor_state_t *state, uint32_t addr, uchar_t data)
1403 {
1404 uint32_t stat;
1405 int status = 0;
1406 int i;
1407
1408 switch (state->ts_fw_cmdset) {
1409 case TAVOR_FLASH_AMD_CMDSET:
1410 /* Issue Flash Byte program command */
1411 tavor_flash_write(state, addr, 0xAA);
1412 tavor_flash_write(state, addr, 0x55);
1413 tavor_flash_write(state, addr, 0xA0);
1414 tavor_flash_write(state, addr, data);
1415
1416 /*
1417 * Wait for Write Byte to Complete:
1418 * 1) Wait 1usec
1419 * 2) Read status of the write operation
1420 * 3) Determine if we have timed out the write operation
1421 * 4) Compare correct data value to the status value that
1422 * was read from the same address.
1423 */
1424 i = 0;
1425 do {
1426 drv_usecwait(1);
1427 stat = tavor_flash_read(state, addr & ~3);
1428
1429 if (i == tavor_hw_flash_timeout_write) {
1430 cmn_err(CE_WARN,
1431 "tavor_flash_write_byte: ACS write "
1432 "timeout: addr: 0x%x, data: 0x%x\n",
1433 addr, data);
1434 status = EIO;
1435 break;
1436 }
1437
1438 i++;
1439 } while (data != ((stat >> ((3 - (addr & 3)) << 3)) & 0xFF));
1440 break;
1441
1442 case TAVOR_FLASH_INTEL_CMDSET:
1443 /* Issue Flash Byte program command */
1444 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_WRITE);
1445 tavor_flash_write(state, addr, data);
1446
1447 /* wait for completion */
1448 i = 0;
1449 do {
1450 drv_usecwait(1);
1451 stat = tavor_flash_read(state, addr & ~3);
1452
1453 if (i == tavor_hw_flash_timeout_write) {
1454 cmn_err(CE_WARN,
1455 "tavor_flash_write_byte: ICS write "
1456 "timeout: addr: %x, data: %x\n",
1457 addr, data);
1458 status = EIO;
1459 break;
1460 }
1461
1462 i++;
1463 } while ((stat & TAVOR_HW_FLASH_ICS_READY) == 0);
1464
1465 if (stat & TAVOR_HW_FLASH_ICS_ERROR) {
1466 cmn_err(CE_WARN,
1467 "tavor_flash_write_byte: ICS write cmd error: "
1468 "addr: %x, data: %x\n",
1469 addr, data);
1470 status = EIO;
1471 }
1472 break;
1473
1474 default:
1475 cmn_err(CE_WARN,
1476 "tavor_flash_write_byte: unknown cmd set: 0x%x\n",
1477 state->ts_fw_cmdset);
1478 status = EIO;
1479 break;
1480 }
1481
1482 return (status);
1483 }
1484
1485 /*
1486 * tavor_flash_erase_sector()
1487 */
1488 static int
tavor_flash_erase_sector(tavor_state_t * state,uint32_t sector_num)1489 tavor_flash_erase_sector(tavor_state_t *state, uint32_t sector_num)
1490 {
1491 uint32_t addr;
1492 uint32_t stat;
1493 int status = 0;
1494 int i;
1495
1496 /* Get address from sector num */
1497 addr = sector_num << state->ts_fw_log_sector_sz;
1498
1499 switch (state->ts_fw_cmdset) {
1500 case TAVOR_FLASH_AMD_CMDSET:
1501 /* Issue Flash Sector Erase Command */
1502 tavor_flash_write(state, addr, 0xAA);
1503 tavor_flash_write(state, addr, 0x55);
1504 tavor_flash_write(state, addr, 0x80);
1505 tavor_flash_write(state, addr, 0xAA);
1506 tavor_flash_write(state, addr, 0x55);
1507 tavor_flash_write(state, addr, 0x30);
1508
1509 /*
1510 * Wait for Sector Erase to Complete
1511 * 1) Wait 1usec
1512 * 2) read the status at the base addr of the sector
1513 * 3) Determine if we have timed out
1514 * 4) Compare status of address with the value of a fully
1515 * erased quadlet. If these are equal, the sector
1516 * has been erased.
1517 */
1518 i = 0;
1519 do {
1520 /* wait 1usec */
1521 drv_usecwait(1);
1522 stat = tavor_flash_read(state, addr);
1523
1524 if (i == tavor_hw_flash_timeout_erase) {
1525 cmn_err(CE_WARN,
1526 "tavor_flash_erase_sector: "
1527 "ACS erase timeout\n");
1528 status = EIO;
1529 break;
1530 }
1531
1532 i++;
1533 } while (stat != 0xFFFFFFFF);
1534 break;
1535
1536 case TAVOR_FLASH_INTEL_CMDSET:
1537 /* Issue Erase Command */
1538 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_ERASE);
1539 tavor_flash_write(state, addr, TAVOR_HW_FLASH_ICS_CONFIRM);
1540
1541 /* wait for completion */
1542 i = 0;
1543 do {
1544 drv_usecwait(1);
1545 stat = tavor_flash_read(state, addr & ~3);
1546
1547 if (i == tavor_hw_flash_timeout_erase) {
1548 cmn_err(CE_WARN,
1549 "tavor_flash_erase_sector: "
1550 "ICS erase timeout\n");
1551 status = EIO;
1552 break;
1553 }
1554
1555 i++;
1556 } while ((stat & TAVOR_HW_FLASH_ICS_READY) == 0);
1557
1558 if (stat & TAVOR_HW_FLASH_ICS_ERROR) {
1559 cmn_err(CE_WARN,
1560 "tavor_flash_erase_sector: "
1561 "ICS erase cmd error\n");
1562 status = EIO;
1563 }
1564 break;
1565
1566 default:
1567 cmn_err(CE_WARN,
1568 "tavor_flash_erase_sector: unknown cmd set: 0x%x\n",
1569 state->ts_fw_cmdset);
1570 status = EIO;
1571 break;
1572 }
1573
1574 tavor_flash_reset(state);
1575
1576 return (status);
1577 }
1578
1579 /*
1580 * tavor_flash_erase_chip()
1581 */
1582 static int
tavor_flash_erase_chip(tavor_state_t * state)1583 tavor_flash_erase_chip(tavor_state_t *state)
1584 {
1585 uint_t size;
1586 uint32_t stat;
1587 int status = 0;
1588 int num_sect;
1589 int i;
1590
1591 switch (state->ts_fw_cmdset) {
1592 case TAVOR_FLASH_AMD_CMDSET:
1593 /* Issue Flash Chip Erase Command */
1594 tavor_flash_write(state, 0, 0xAA);
1595 tavor_flash_write(state, 0, 0x55);
1596 tavor_flash_write(state, 0, 0x80);
1597 tavor_flash_write(state, 0, 0xAA);
1598 tavor_flash_write(state, 0, 0x55);
1599 tavor_flash_write(state, 0, 0x10);
1600
1601 /*
1602 * Wait for Chip Erase to Complete
1603 * 1) Wait 1usec
1604 * 2) read the status at the base addr of the sector
1605 * 3) Determine if we have timed out
1606 * 4) Compare status of address with the value of a
1607 * fully erased quadlet. If these are equal, the
1608 * chip has been erased.
1609 */
1610 i = 0;
1611 do {
1612 /* wait 1usec */
1613 drv_usecwait(1);
1614 stat = tavor_flash_read(state, 0);
1615
1616 if (i == tavor_hw_flash_timeout_erase) {
1617 cmn_err(CE_WARN,
1618 "tavor_flash_erase_chip: erase timeout\n");
1619 status = EIO;
1620 break;
1621 }
1622
1623 i++;
1624 } while (stat != 0xFFFFFFFF);
1625 break;
1626
1627 case TAVOR_FLASH_INTEL_CMDSET:
1628 /*
1629 * The Intel chip doesn't have a chip erase command, so erase
1630 * all blocks one at a time.
1631 */
1632 size = (0x1 << state->ts_fw_log_sector_sz);
1633 num_sect = state->ts_fw_device_sz / size;
1634
1635 for (i = 0; i < num_sect; i++) {
1636 status = tavor_flash_erase_sector(state, i);
1637 if (status != 0) {
1638 cmn_err(CE_WARN,
1639 "tavor_flash_erase_chip: "
1640 "ICS sector %d erase error\n", i);
1641 status = EIO;
1642 break;
1643 }
1644 }
1645 break;
1646
1647 default:
1648 cmn_err(CE_WARN, "tavor_flash_erase_chip: "
1649 "unknown cmd set: 0x%x\n", state->ts_fw_cmdset);
1650 status = EIO;
1651 break;
1652 }
1653
1654 return (status);
1655 }
1656
1657 /*
1658 * tavor_flash_bank()
1659 */
1660 static void
tavor_flash_bank(tavor_state_t * state,uint32_t addr)1661 tavor_flash_bank(tavor_state_t *state, uint32_t addr)
1662 {
1663 ddi_acc_handle_t hdl;
1664 uint32_t bank;
1665
1666 /* Set handle */
1667 hdl = state->ts_pci_cfghdl;
1668
1669 /* Determine the bank setting from the address */
1670 bank = addr & TAVOR_HW_FLASH_BANK_MASK;
1671
1672 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->ts_fw_flashbank))
1673
1674 /*
1675 * If the bank is different from the currently set bank, we need to
1676 * change it. Also, if an 'addr' of 0 is given, this allows the
1677 * capability to force the flash bank to 0. This is useful at init
1678 * time to initially set the bank value
1679 */
1680 if (state->ts_fw_flashbank != bank || addr == 0) {
1681 /* Set bank using the GPIO settings */
1682 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATACLEAR, 0x70);
1683 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DATASET,
1684 (bank >> 15) & 0x70);
1685
1686 /* Save the bank state */
1687 state->ts_fw_flashbank = bank;
1688 }
1689 }
1690
1691 /*
1692 * tavor_flash_read()
1693 */
1694 static uint32_t
tavor_flash_read(tavor_state_t * state,uint32_t addr)1695 tavor_flash_read(tavor_state_t *state, uint32_t addr)
1696 {
1697 ddi_acc_handle_t hdl;
1698 uint32_t data;
1699 int timeout;
1700
1701 /* Set handle */
1702 hdl = state->ts_pci_cfghdl;
1703
1704 /*
1705 * The Read operation does the following:
1706 * 1) Write the masked address to the TAVOR_FLASH_ADDR register.
1707 * Only the least significant 19 bits are valid.
1708 * 2) Read back the register until the command has completed.
1709 * 3) Read the data retrieved from the address at the TAVOR_FLASH_DATA
1710 * register.
1711 */
1712 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_ADDR,
1713 (addr & TAVOR_HW_FLASH_ADDR_MASK) | (1 << 29));
1714
1715 timeout = 0;
1716 do {
1717 data = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_ADDR);
1718 timeout++;
1719 } while ((data & TAVOR_HW_FLASH_CMD_MASK) &&
1720 (timeout < tavor_hw_flash_timeout_config));
1721
1722 if (timeout == tavor_hw_flash_timeout_config) {
1723 cmn_err(CE_WARN, "tavor_flash_read: config command timeout.\n");
1724 }
1725
1726 data = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_DATA);
1727
1728 return (data);
1729 }
1730
1731 /*
1732 * tavor_flash_write()
1733 */
1734 static void
tavor_flash_write(tavor_state_t * state,uint32_t addr,uchar_t data)1735 tavor_flash_write(tavor_state_t *state, uint32_t addr, uchar_t data)
1736 {
1737 ddi_acc_handle_t hdl;
1738 int cmd;
1739 int timeout;
1740
1741 /* Set handle */
1742 hdl = state->ts_pci_cfghdl;
1743
1744 /*
1745 * The Write operation does the following:
1746 * 1) Write the data to be written to the TAVOR_FLASH_DATA offset.
1747 * 2) Write the address to write the data to to the TAVOR_FLASH_ADDR
1748 * offset.
1749 * 3) Wait until the write completes.
1750 */
1751 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_DATA, data << 24);
1752 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_ADDR,
1753 (addr & 0x7FFFF) | (2 << 29));
1754
1755 timeout = 0;
1756 do {
1757 cmd = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_ADDR);
1758 timeout++;
1759 } while ((cmd & TAVOR_HW_FLASH_CMD_MASK) &&
1760 (timeout < tavor_hw_flash_timeout_config));
1761
1762 if (timeout == tavor_hw_flash_timeout_config) {
1763 cmn_err(CE_WARN, "tavor_flash_write: config cmd timeout.\n");
1764 }
1765 }
1766
1767 /*
1768 * tavor_flash_init()
1769 */
1770 static void
tavor_flash_init(tavor_state_t * state)1771 tavor_flash_init(tavor_state_t *state)
1772 {
1773 uint32_t word;
1774 ddi_acc_handle_t hdl;
1775 int sema_cnt;
1776 int gpio;
1777
1778 /* Set handle */
1779 hdl = state->ts_pci_cfghdl;
1780
1781 /* Init the flash */
1782
1783 /*
1784 * Grab the GPIO semaphore. This allows us exclusive access to the
1785 * GPIO settings on the Tavor for the duration of the flash burning
1786 * procedure.
1787 */
1788 sema_cnt = 0;
1789 do {
1790 word = tavor_flash_read_cfg(hdl, TAVOR_HW_FLASH_GPIO_SEMA);
1791 if (word == 0) {
1792 break;
1793 }
1794
1795 sema_cnt++;
1796 drv_usecwait(1);
1797 } while (sema_cnt < tavor_hw_flash_timeout_gpio_sema);
1798
1799 /*
1800 * Determine if we timed out trying to grab the GPIO semaphore
1801 */
1802 if (sema_cnt == tavor_hw_flash_timeout_gpio_sema) {
1803 cmn_err(CE_WARN, "tavor_flash_init: GPIO SEMA timeout\n");
1804 }
1805
1806 /* Save away original GPIO Values */
1807 state->ts_fw_gpio[0] = tavor_flash_read_cfg(hdl,
1808 TAVOR_HW_FLASH_GPIO_DIR);
1809 state->ts_fw_gpio[1] = tavor_flash_read_cfg(hdl,
1810 TAVOR_HW_FLASH_GPIO_POL);
1811 state->ts_fw_gpio[2] = tavor_flash_read_cfg(hdl,
1812 TAVOR_HW_FLASH_GPIO_MOD);
1813 state->ts_fw_gpio[3] = tavor_flash_read_cfg(hdl,
1814 TAVOR_HW_FLASH_GPIO_DAT);
1815
1816 /* Set New GPIO Values */
1817 gpio = state->ts_fw_gpio[0] | 0x70;
1818 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DIR, gpio);
1819
1820 gpio = state->ts_fw_gpio[1] & ~0x70;
1821 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_POL, gpio);
1822
1823 gpio = state->ts_fw_gpio[2] & ~0x70;
1824 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_MOD, gpio);
1825
1826 /* Set CPUMODE to enable tavor to access the flash device */
1827 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_CPUMODE,
1828 1 << TAVOR_HW_FLASH_CPU_SHIFT);
1829
1830 /* Initialize to bank 0 */
1831 tavor_flash_bank(state, 0);
1832 }
1833
1834 /*
1835 * tavor_flash_cfi_init
1836 * Implements access to the CFI (Common Flash Interface) data
1837 */
1838 static void
tavor_flash_cfi_init(tavor_state_t * state,uint32_t * cfi_info,int * intel_xcmd)1839 tavor_flash_cfi_init(tavor_state_t *state, uint32_t *cfi_info, int *intel_xcmd)
1840 {
1841 uint32_t data;
1842 uint32_t sector_sz_bytes;
1843 uint32_t bit_count;
1844 uint8_t cfi_ch_info[TAVOR_CFI_INFO_SIZE];
1845 int i;
1846
1847 /*
1848 * Determine if the user command supports the Intel Extended
1849 * Command Set. The query string is contained in the fourth
1850 * quad word.
1851 */
1852 tavor_flash_cfi_byte(cfi_ch_info, cfi_info[0x04], 0x10);
1853 if (cfi_ch_info[0x10] == 'M' &&
1854 cfi_ch_info[0x11] == 'X' &&
1855 cfi_ch_info[0x12] == '2') {
1856 *intel_xcmd = 1; /* support is there */
1857 }
1858
1859 /* CFI QUERY */
1860 tavor_flash_write(state, 0x55, TAVOR_FLASH_CFI_INIT);
1861
1862 /* Read in CFI data */
1863 for (i = 0; i < TAVOR_CFI_INFO_SIZE; i += 4) {
1864 data = tavor_flash_read(state, i);
1865 tavor_flash_cfi_byte(cfi_ch_info, data, i);
1866 }
1867
1868 /* Determine chip set */
1869 state->ts_fw_cmdset = TAVOR_FLASH_UNKNOWN_CMDSET;
1870 if (cfi_ch_info[0x20] == 'Q' &&
1871 cfi_ch_info[0x22] == 'R' &&
1872 cfi_ch_info[0x24] == 'Y') {
1873 /*
1874 * Mode: x16 working in x8 mode (Intel).
1875 * Pack data - skip spacing bytes.
1876 */
1877 for (i = 0; i < TAVOR_CFI_INFO_SIZE; i += 2) {
1878 cfi_ch_info[i/2] = cfi_ch_info[i];
1879 }
1880 }
1881 state->ts_fw_cmdset = cfi_ch_info[0x13];
1882 if (state->ts_fw_cmdset != TAVOR_FLASH_INTEL_CMDSET &&
1883 state->ts_fw_cmdset != TAVOR_FLASH_AMD_CMDSET) {
1884 cmn_err(CE_WARN,
1885 "tavor_flash_cfi_init: UNKNOWN chip cmd set\n");
1886 state->ts_fw_cmdset = TAVOR_FLASH_UNKNOWN_CMDSET;
1887 return;
1888 }
1889
1890 /* Determine total bytes in one sector size */
1891 sector_sz_bytes = ((cfi_ch_info[0x30] << 8) | cfi_ch_info[0x2F]) << 8;
1892
1893 /* Calculate equivalent of log2 (n) */
1894 for (bit_count = 0; sector_sz_bytes > 1; bit_count++) {
1895 sector_sz_bytes >>= 1;
1896 }
1897
1898 /* Set sector size */
1899 state->ts_fw_log_sector_sz = bit_count;
1900
1901 /* Set flash size */
1902 state->ts_fw_device_sz = 0x1 << cfi_ch_info[0x27];
1903
1904 /* Reset to turn off CFI mode */
1905 tavor_flash_reset(state);
1906
1907 /*
1908 * Pass CFI data back to user command.
1909 */
1910 for (i = 0; i < TAVOR_FLASH_CFI_SIZE_QUADLET; i++) {
1911 tavor_flash_cfi_dword(&cfi_info[i], cfi_ch_info, i << 2);
1912 }
1913
1914 if (*intel_xcmd == 1) {
1915 /*
1916 * Inform the user cmd that this driver does support the
1917 * Intel Extended Command Set.
1918 */
1919 cfi_ch_info[0x10] = 'M';
1920 cfi_ch_info[0x11] = 'X';
1921 cfi_ch_info[0x12] = '2';
1922 } else {
1923 cfi_ch_info[0x10] = 'Q';
1924 cfi_ch_info[0x11] = 'R';
1925 cfi_ch_info[0x12] = 'Y';
1926 }
1927 cfi_ch_info[0x13] = state->ts_fw_cmdset;
1928 tavor_flash_cfi_dword(&cfi_info[0x4], cfi_ch_info, 0x10);
1929 }
1930
1931 /*
1932 * tavor_flash_fini()
1933 */
1934 static void
tavor_flash_fini(tavor_state_t * state)1935 tavor_flash_fini(tavor_state_t *state)
1936 {
1937 ddi_acc_handle_t hdl;
1938
1939 /* Set handle */
1940 hdl = state->ts_pci_cfghdl;
1941
1942 /* Restore original GPIO Values */
1943 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DIR,
1944 state->ts_fw_gpio[0]);
1945 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_POL,
1946 state->ts_fw_gpio[1]);
1947 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_MOD,
1948 state->ts_fw_gpio[2]);
1949 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_DAT,
1950 state->ts_fw_gpio[3]);
1951
1952 /* Give up semaphore */
1953 tavor_flash_write_cfg(hdl, TAVOR_HW_FLASH_GPIO_SEMA, 0);
1954 }
1955
1956 /*
1957 * tavor_flash_read_cfg
1958 */
1959 static uint32_t
tavor_flash_read_cfg(ddi_acc_handle_t pci_config_hdl,uint32_t addr)1960 tavor_flash_read_cfg(ddi_acc_handle_t pci_config_hdl, uint32_t addr)
1961 {
1962 uint32_t read;
1963
1964 /*
1965 * Perform flash read operation:
1966 * 1) Place addr to read from on the TAVOR_HW_FLASH_CFG_ADDR register
1967 * 2) Read data at that addr from the TAVOR_HW_FLASH_CFG_DATA register
1968 */
1969 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_ADDR, addr);
1970 read = pci_config_get32(pci_config_hdl, TAVOR_HW_FLASH_CFG_DATA);
1971
1972 return (read);
1973 }
1974
1975 /*
1976 * tavor_flash_write_cfg
1977 */
1978 static void
tavor_flash_write_cfg(ddi_acc_handle_t pci_config_hdl,uint32_t addr,uint32_t data)1979 tavor_flash_write_cfg(ddi_acc_handle_t pci_config_hdl, uint32_t addr,
1980 uint32_t data)
1981 {
1982 /*
1983 * Perform flash write operation:
1984 * 1) Place addr to write to on the TAVOR_HW_FLASH_CFG_ADDR register
1985 * 2) Place data to write on to the TAVOR_HW_FLASH_CFG_DATA register
1986 */
1987 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_ADDR, addr);
1988 pci_config_put32(pci_config_hdl, TAVOR_HW_FLASH_CFG_DATA, data);
1989 }
1990
1991 /*
1992 * Support routines to convert Common Flash Interface (CFI) data
1993 * from a 32 bit word to a char array, and from a char array to
1994 * a 32 bit word.
1995 */
1996 static void
tavor_flash_cfi_byte(uint8_t * ch,uint32_t dword,int i)1997 tavor_flash_cfi_byte(uint8_t *ch, uint32_t dword, int i)
1998 {
1999 ch[i] = (uint8_t)((dword & 0xFF000000) >> 24);
2000 ch[i+1] = (uint8_t)((dword & 0x00FF0000) >> 16);
2001 ch[i+2] = (uint8_t)((dword & 0x0000FF00) >> 8);
2002 ch[i+3] = (uint8_t)((dword & 0x000000FF));
2003 }
2004
2005 static void
tavor_flash_cfi_dword(uint32_t * dword,uint8_t * ch,int i)2006 tavor_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i)
2007 {
2008 *dword = (uint32_t)
2009 ((uint32_t)ch[i] << 24 |
2010 (uint32_t)ch[i+1] << 16 |
2011 (uint32_t)ch[i+2] << 8 |
2012 (uint32_t)ch[i+3]);
2013 }
2014
2015 /*
2016 * tavor_loopback_free_qps
2017 */
2018 static void
tavor_loopback_free_qps(tavor_loopback_state_t * lstate)2019 tavor_loopback_free_qps(tavor_loopback_state_t *lstate)
2020 {
2021 int i;
2022
2023 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2024
2025 if (lstate->tls_tx.tlc_qp_hdl != NULL) {
2026 (void) tavor_qp_free(lstate->tls_state,
2027 &lstate->tls_tx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2028 TAVOR_NOSLEEP);
2029 }
2030 if (lstate->tls_rx.tlc_qp_hdl != NULL) {
2031 (void) tavor_qp_free(lstate->tls_state,
2032 &lstate->tls_rx.tlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2033 TAVOR_NOSLEEP);
2034 }
2035 lstate->tls_tx.tlc_qp_hdl = NULL;
2036 lstate->tls_rx.tlc_qp_hdl = NULL;
2037 for (i = 0; i < 2; i++) {
2038 if (lstate->tls_tx.tlc_cqhdl[i] != NULL) {
2039 (void) tavor_cq_free(lstate->tls_state,
2040 &lstate->tls_tx.tlc_cqhdl[i], TAVOR_NOSLEEP);
2041 }
2042 if (lstate->tls_rx.tlc_cqhdl[i] != NULL) {
2043 (void) tavor_cq_free(lstate->tls_state,
2044 &lstate->tls_rx.tlc_cqhdl[i], TAVOR_NOSLEEP);
2045 }
2046 lstate->tls_tx.tlc_cqhdl[i] = NULL;
2047 lstate->tls_rx.tlc_cqhdl[i] = NULL;
2048 }
2049 }
2050
2051 /*
2052 * tavor_loopback_free_state
2053 */
2054 static void
tavor_loopback_free_state(tavor_loopback_state_t * lstate)2055 tavor_loopback_free_state(tavor_loopback_state_t *lstate)
2056 {
2057 tavor_loopback_free_qps(lstate);
2058 if (lstate->tls_tx.tlc_mrhdl != NULL) {
2059 (void) tavor_mr_deregister(lstate->tls_state,
2060 &lstate->tls_tx.tlc_mrhdl, TAVOR_MR_DEREG_ALL,
2061 TAVOR_NOSLEEP);
2062 }
2063 if (lstate->tls_rx.tlc_mrhdl != NULL) {
2064 (void) tavor_mr_deregister(lstate->tls_state,
2065 &lstate->tls_rx.tlc_mrhdl, TAVOR_MR_DEREG_ALL,
2066 TAVOR_NOSLEEP);
2067 }
2068 if (lstate->tls_pd_hdl != NULL) {
2069 (void) tavor_pd_free(lstate->tls_state, &lstate->tls_pd_hdl);
2070 }
2071 if (lstate->tls_tx.tlc_buf != NULL) {
2072 kmem_free(lstate->tls_tx.tlc_buf, lstate->tls_tx.tlc_buf_sz);
2073 }
2074 if (lstate->tls_rx.tlc_buf != NULL) {
2075 kmem_free(lstate->tls_rx.tlc_buf, lstate->tls_rx.tlc_buf_sz);
2076 }
2077 bzero(lstate, sizeof (tavor_loopback_state_t));
2078 }
2079
2080 /*
2081 * tavor_loopback_init
2082 */
2083 static int
tavor_loopback_init(tavor_state_t * state,tavor_loopback_state_t * lstate)2084 tavor_loopback_init(tavor_state_t *state, tavor_loopback_state_t *lstate)
2085 {
2086 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2087
2088 lstate->tls_hca_hdl = (ibc_hca_hdl_t)state;
2089 lstate->tls_status = tavor_pd_alloc(lstate->tls_state,
2090 &lstate->tls_pd_hdl, TAVOR_NOSLEEP);
2091 if (lstate->tls_status != IBT_SUCCESS) {
2092 lstate->tls_err = TAVOR_LOOPBACK_PROT_DOMAIN_ALLOC_FAIL;
2093 return (EFAULT);
2094 }
2095
2096 return (0);
2097 }
2098
2099 /*
2100 * tavor_loopback_init_qp_info
2101 */
2102 static void
tavor_loopback_init_qp_info(tavor_loopback_state_t * lstate,tavor_loopback_comm_t * comm)2103 tavor_loopback_init_qp_info(tavor_loopback_state_t *lstate,
2104 tavor_loopback_comm_t *comm)
2105 {
2106 bzero(&comm->tlc_cq_attr, sizeof (ibt_cq_attr_t));
2107 bzero(&comm->tlc_qp_attr, sizeof (ibt_qp_alloc_attr_t));
2108 bzero(&comm->tlc_qp_info, sizeof (ibt_qp_info_t));
2109
2110 comm->tlc_wrid = 1;
2111 comm->tlc_cq_attr.cq_size = 128;
2112 comm->tlc_qp_attr.qp_sizes.cs_sq_sgl = 3;
2113 comm->tlc_qp_attr.qp_sizes.cs_rq_sgl = 3;
2114 comm->tlc_qp_attr.qp_sizes.cs_sq = 16;
2115 comm->tlc_qp_attr.qp_sizes.cs_rq = 16;
2116 comm->tlc_qp_attr.qp_flags = IBT_WR_SIGNALED;
2117
2118 comm->tlc_qp_info.qp_state = IBT_STATE_RESET;
2119 comm->tlc_qp_info.qp_trans = IBT_RC_SRV;
2120 comm->tlc_qp_info.qp_flags = IBT_CEP_RDMA_RD | IBT_CEP_RDMA_WR;
2121 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_hca_port_num =
2122 lstate->tls_port;
2123 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_pkey_ix =
2124 lstate->tls_pkey_ix;
2125 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_timeout =
2126 lstate->tls_timeout;
2127 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_srvl = 0;
2128 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_srate =
2129 IBT_SRATE_4X;
2130 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_send_grh = 0;
2131 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid =
2132 lstate->tls_lid;
2133 comm->tlc_qp_info.qp_transport.rc.rc_retry_cnt = lstate->tls_retry;
2134 comm->tlc_qp_info.qp_transport.rc.rc_sq_psn = 0;
2135 comm->tlc_qp_info.qp_transport.rc.rc_rq_psn = 0;
2136 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_in = 4;
2137 comm->tlc_qp_info.qp_transport.rc.rc_rdma_ra_out = 4;
2138 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = 0;
2139 comm->tlc_qp_info.qp_transport.rc.rc_min_rnr_nak = IBT_RNR_NAK_655ms;
2140 comm->tlc_qp_info.qp_transport.rc.rc_path_mtu = IB_MTU_1K;
2141 }
2142
2143 /*
2144 * tavor_loopback_alloc_mem
2145 */
2146 static int
tavor_loopback_alloc_mem(tavor_loopback_state_t * lstate,tavor_loopback_comm_t * comm,int sz)2147 tavor_loopback_alloc_mem(tavor_loopback_state_t *lstate,
2148 tavor_loopback_comm_t *comm, int sz)
2149 {
2150 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2151
2152 /* Allocate buffer of specified size */
2153 comm->tlc_buf_sz = sz;
2154 comm->tlc_buf = kmem_zalloc(sz, KM_NOSLEEP);
2155 if (comm->tlc_buf == NULL) {
2156 return (EFAULT);
2157 }
2158
2159 /* Register the buffer as a memory region */
2160 comm->tlc_memattr.mr_vaddr = (uint64_t)(uintptr_t)comm->tlc_buf;
2161 comm->tlc_memattr.mr_len = (ib_msglen_t)sz;
2162 comm->tlc_memattr.mr_as = NULL;
2163 comm->tlc_memattr.mr_flags = IBT_MR_NOSLEEP |
2164 IBT_MR_ENABLE_REMOTE_WRITE | IBT_MR_ENABLE_LOCAL_WRITE;
2165
2166 comm->tlc_status = tavor_mr_register(lstate->tls_state,
2167 lstate->tls_pd_hdl, &comm->tlc_memattr, &comm->tlc_mrhdl, NULL);
2168
2169 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm->tlc_mrhdl))
2170
2171 comm->tlc_mrdesc.md_vaddr = comm->tlc_mrhdl->mr_bindinfo.bi_addr;
2172 comm->tlc_mrdesc.md_lkey = comm->tlc_mrhdl->mr_lkey;
2173 comm->tlc_mrdesc.md_rkey = comm->tlc_mrhdl->mr_rkey;
2174 if (comm->tlc_status != IBT_SUCCESS) {
2175 return (EFAULT);
2176 }
2177 return (0);
2178 }
2179
2180 /*
2181 * tavor_loopback_alloc_qps
2182 */
2183 static int
tavor_loopback_alloc_qps(tavor_loopback_state_t * lstate,tavor_loopback_comm_t * comm)2184 tavor_loopback_alloc_qps(tavor_loopback_state_t *lstate,
2185 tavor_loopback_comm_t *comm)
2186 {
2187 uint32_t i, real_size;
2188 tavor_qp_info_t qpinfo;
2189
2190 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2191 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2192
2193 /* Allocate send and recv CQs */
2194 for (i = 0; i < 2; i++) {
2195 bzero(&comm->tlc_cq_attr, sizeof (ibt_cq_attr_t));
2196 comm->tlc_cq_attr.cq_size = 128;
2197 comm->tlc_status = tavor_cq_alloc(lstate->tls_state,
2198 (ibt_cq_hdl_t)NULL, &comm->tlc_cq_attr, &real_size,
2199 &comm->tlc_cqhdl[i], TAVOR_NOSLEEP);
2200 if (comm->tlc_status != IBT_SUCCESS) {
2201 lstate->tls_err += i;
2202 return (EFAULT);
2203 }
2204 }
2205
2206 /* Allocate the QP */
2207 tavor_loopback_init_qp_info(lstate, comm);
2208 comm->tlc_qp_attr.qp_pd_hdl = (ibt_pd_hdl_t)lstate->tls_pd_hdl;
2209 comm->tlc_qp_attr.qp_scq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[0];
2210 comm->tlc_qp_attr.qp_rcq_hdl = (ibt_cq_hdl_t)comm->tlc_cqhdl[1];
2211 comm->tlc_qp_attr.qp_ibc_scq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[0];
2212 comm->tlc_qp_attr.qp_ibc_rcq_hdl = (ibt_opaque1_t)comm->tlc_cqhdl[1];
2213 qpinfo.qpi_attrp = &comm->tlc_qp_attr;
2214 qpinfo.qpi_type = IBT_RC_RQP;
2215 qpinfo.qpi_ibt_qphdl = NULL;
2216 qpinfo.qpi_queueszp = &comm->tlc_chan_sizes;
2217 qpinfo.qpi_qpn = &comm->tlc_qp_num;
2218 comm->tlc_status = tavor_qp_alloc(lstate->tls_state, &qpinfo,
2219 TAVOR_NOSLEEP, NULL);
2220 if (comm->tlc_status == DDI_SUCCESS) {
2221 comm->tlc_qp_hdl = qpinfo.qpi_qphdl;
2222 }
2223
2224 if (comm->tlc_status != IBT_SUCCESS) {
2225 lstate->tls_err += 2;
2226 return (EFAULT);
2227 }
2228 return (0);
2229 }
2230
2231 /*
2232 * tavor_loopback_modify_qp
2233 */
2234 static int
tavor_loopback_modify_qp(tavor_loopback_state_t * lstate,tavor_loopback_comm_t * comm,uint_t qp_num)2235 tavor_loopback_modify_qp(tavor_loopback_state_t *lstate,
2236 tavor_loopback_comm_t *comm, uint_t qp_num)
2237 {
2238 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2239 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2240
2241 /* Modify QP to INIT */
2242 tavor_loopback_init_qp_info(lstate, comm);
2243 comm->tlc_qp_info.qp_state = IBT_STATE_INIT;
2244 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2245 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2246 if (comm->tlc_status != IBT_SUCCESS) {
2247 return (EFAULT);
2248 }
2249
2250 /*
2251 * Modify QP to RTR (set destination LID and QP number to local
2252 * LID and QP number)
2253 */
2254 comm->tlc_qp_info.qp_state = IBT_STATE_RTR;
2255 comm->tlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid
2256 = lstate->tls_lid;
2257 comm->tlc_qp_info.qp_transport.rc.rc_dst_qpn = qp_num;
2258 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2259 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2260 if (comm->tlc_status != IBT_SUCCESS) {
2261 lstate->tls_err += 1;
2262 return (EFAULT);
2263 }
2264
2265 /* Modify QP to RTS */
2266 comm->tlc_qp_info.qp_current_state = IBT_STATE_RTR;
2267 comm->tlc_qp_info.qp_state = IBT_STATE_RTS;
2268 comm->tlc_status = tavor_qp_modify(lstate->tls_state, comm->tlc_qp_hdl,
2269 IBT_CEP_SET_STATE, &comm->tlc_qp_info, &comm->tlc_queue_sizes);
2270 if (comm->tlc_status != IBT_SUCCESS) {
2271 lstate->tls_err += 2;
2272 return (EFAULT);
2273 }
2274 return (0);
2275 }
2276
2277 /*
2278 * tavor_loopback_copyout
2279 */
2280 static int
tavor_loopback_copyout(tavor_loopback_ioctl_t * lb,intptr_t arg,int mode)2281 tavor_loopback_copyout(tavor_loopback_ioctl_t *lb, intptr_t arg, int mode)
2282 {
2283 #ifdef _MULTI_DATAMODEL
2284 if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
2285 tavor_loopback_ioctl32_t lb32;
2286
2287 lb32.tlb_revision = lb->tlb_revision;
2288 lb32.tlb_send_buf =
2289 (caddr32_t)(uintptr_t)lb->tlb_send_buf;
2290 lb32.tlb_fail_buf =
2291 (caddr32_t)(uintptr_t)lb->tlb_fail_buf;
2292 lb32.tlb_buf_sz = lb->tlb_buf_sz;
2293 lb32.tlb_num_iter = lb->tlb_num_iter;
2294 lb32.tlb_pass_done = lb->tlb_pass_done;
2295 lb32.tlb_timeout = lb->tlb_timeout;
2296 lb32.tlb_error_type = lb->tlb_error_type;
2297 lb32.tlb_port_num = lb->tlb_port_num;
2298 lb32.tlb_num_retry = lb->tlb_num_retry;
2299
2300 if (ddi_copyout(&lb32, (void *)arg,
2301 sizeof (tavor_loopback_ioctl32_t), mode) != 0) {
2302 return (EFAULT);
2303 }
2304 } else
2305 #endif /* _MULTI_DATAMODEL */
2306 if (ddi_copyout(lb, (void *)arg, sizeof (tavor_loopback_ioctl_t),
2307 mode) != 0) {
2308 return (EFAULT);
2309 }
2310 return (0);
2311 }
2312
2313 /*
2314 * tavor_loopback_post_send
2315 */
2316 static int
tavor_loopback_post_send(tavor_loopback_state_t * lstate,tavor_loopback_comm_t * tx,tavor_loopback_comm_t * rx)2317 tavor_loopback_post_send(tavor_loopback_state_t *lstate,
2318 tavor_loopback_comm_t *tx, tavor_loopback_comm_t *rx)
2319 {
2320 int ret;
2321
2322 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tx))
2323
2324 bzero(&tx->tlc_sgl, sizeof (ibt_wr_ds_t));
2325 bzero(&tx->tlc_wr, sizeof (ibt_send_wr_t));
2326
2327 /* Initialize local address for TX buffer */
2328 tx->tlc_sgl.ds_va = tx->tlc_mrdesc.md_vaddr;
2329 tx->tlc_sgl.ds_key = tx->tlc_mrdesc.md_lkey;
2330 tx->tlc_sgl.ds_len = tx->tlc_buf_sz;
2331
2332 /* Initialize the remaining details of the work request */
2333 tx->tlc_wr.wr_id = tx->tlc_wrid++;
2334 tx->tlc_wr.wr_flags = IBT_WR_SEND_SIGNAL;
2335 tx->tlc_wr.wr_nds = 1;
2336 tx->tlc_wr.wr_sgl = &tx->tlc_sgl;
2337 tx->tlc_wr.wr_opcode = IBT_WRC_RDMAW;
2338 tx->tlc_wr.wr_trans = IBT_RC_SRV;
2339
2340 /* Initialize the remote address for RX buffer */
2341 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_raddr = rx->tlc_mrdesc.md_vaddr;
2342 tx->tlc_wr.wr.rc.rcwr.rdma.rdma_rkey = rx->tlc_mrdesc.md_rkey;
2343 tx->tlc_complete = 0;
2344 ret = tavor_post_send(lstate->tls_state, tx->tlc_qp_hdl, &tx->tlc_wr,
2345 1, NULL);
2346 if (ret != IBT_SUCCESS) {
2347 return (EFAULT);
2348 }
2349 return (0);
2350 }
2351
2352 /*
2353 * tavor_loopback_poll_cq
2354 */
2355 static int
tavor_loopback_poll_cq(tavor_loopback_state_t * lstate,tavor_loopback_comm_t * comm)2356 tavor_loopback_poll_cq(tavor_loopback_state_t *lstate,
2357 tavor_loopback_comm_t *comm)
2358 {
2359 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2360
2361 comm->tlc_wc.wc_status = 0;
2362 comm->tlc_num_polled = 0;
2363 comm->tlc_status = tavor_cq_poll(lstate->tls_state,
2364 comm->tlc_cqhdl[0], &comm->tlc_wc, 1, &comm->tlc_num_polled);
2365 if ((comm->tlc_status == IBT_SUCCESS) &&
2366 (comm->tlc_wc.wc_status != IBT_WC_SUCCESS)) {
2367 comm->tlc_status = ibc_get_ci_failure(0);
2368 }
2369 return (comm->tlc_status);
2370 }
2371