1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * hermon_ioctl.c
29  *    Hemron IOCTL Routines
30  *
31  *    Implements all ioctl access into the driver.  This includes all routines
32  *    necessary for updating firmware, accessing the hermon flash device, and
33  *    providing interfaces for VTS.
34  */
35 
36 #include <sys/types.h>
37 #include <sys/conf.h>
38 #include <sys/ddi.h>
39 #include <sys/sunddi.h>
40 #include <sys/modctl.h>
41 #include <sys/file.h>
42 
43 #include <sys/ib/adapters/hermon/hermon.h>
44 
45 /* Hemron HCA state pointer (extern) */
46 extern void	*hermon_statep;
47 extern int	hermon_verbose;
48 
49 #define	DO_WRCONF	1
50 static int do_bar0 = 1;
51 
52 /*
53  * The ioctl declarations (for firmware flash burning, register read/write
54  * (DEBUG-only), and VTS interfaces)
55  */
56 static int hermon_ioctl_flash_read(hermon_state_t *state, dev_t dev,
57     intptr_t arg, int mode);
58 static int hermon_ioctl_flash_write(hermon_state_t *state, dev_t dev,
59     intptr_t arg, int mode);
60 static int hermon_ioctl_flash_erase(hermon_state_t *state, dev_t dev,
61     intptr_t arg, int mode);
62 static int hermon_ioctl_flash_init(hermon_state_t *state, dev_t dev,
63     intptr_t arg, int mode);
64 static int hermon_ioctl_flash_fini(hermon_state_t *state, dev_t dev);
65 static int hermon_ioctl_flash_cleanup(hermon_state_t *state);
66 static int hermon_ioctl_flash_cleanup_nolock(hermon_state_t *state);
67 #ifdef	DEBUG
68 static int hermon_ioctl_reg_write(hermon_state_t *state, intptr_t arg,
69     int mode);
70 static int hermon_ioctl_reg_read(hermon_state_t *state, intptr_t arg,
71     int mode);
72 #endif	/* DEBUG */
73 static int hermon_ioctl_write_boot_addr(hermon_state_t *state, dev_t dev,
74     intptr_t arg, int mode);
75 static int hermon_ioctl_info(hermon_state_t *state, dev_t dev,
76     intptr_t arg, int mode);
77 static int hermon_ioctl_ports(hermon_state_t *state, intptr_t arg,
78     int mode);
79 static int hermon_ioctl_loopback(hermon_state_t *state, intptr_t arg,
80     int mode);
81 
82 /* Hemron Flash Functions */
83 static void hermon_flash_spi_exec_command(hermon_state_t *state,
84     ddi_acc_handle_t hdl, uint32_t cmd);
85 static int hermon_flash_read_sector(hermon_state_t *state,
86     uint32_t sector_num);
87 static int hermon_flash_read_quadlet(hermon_state_t *state, uint32_t *data,
88     uint32_t addr);
89 static int hermon_flash_write_sector(hermon_state_t *state,
90     uint32_t sector_num);
91 static int hermon_flash_spi_write_dword(hermon_state_t *state,
92     uint32_t addr, uint32_t data);
93 static int hermon_flash_write_byte(hermon_state_t *state, uint32_t addr,
94     uchar_t data);
95 static int hermon_flash_erase_sector(hermon_state_t *state,
96     uint32_t sector_num);
97 static int hermon_flash_erase_chip(hermon_state_t *state);
98 static int hermon_flash_bank(hermon_state_t *state, uint32_t addr);
99 static uint32_t hermon_flash_read(hermon_state_t *state, uint32_t addr,
100     int *err);
101 static void hermon_flash_write(hermon_state_t *state, uint32_t addr,
102     uchar_t data, int *err);
103 static int hermon_flash_spi_wait_wip(hermon_state_t *state);
104 static void hermon_flash_spi_write_enable(hermon_state_t *state);
105 static int hermon_flash_init(hermon_state_t *state);
106 static int hermon_flash_cfi_init(hermon_state_t *state, uint32_t *cfi_info,
107     int *intel_xcmd);
108 static int hermon_flash_fini(hermon_state_t *state);
109 static int hermon_flash_reset(hermon_state_t *state);
110 static uint32_t hermon_flash_read_cfg(hermon_state_t *state,
111     ddi_acc_handle_t pci_config_hdl, uint32_t addr);
112 #ifdef DO_WRCONF
113 static void hermon_flash_write_cfg(hermon_state_t *state,
114     ddi_acc_handle_t pci_config_hdl, uint32_t addr, uint32_t data);
115 static void hermon_flash_write_cfg_helper(hermon_state_t *state,
116     ddi_acc_handle_t pci_config_hdl, uint32_t addr, uint32_t data);
117 static void hermon_flash_write_confirm(hermon_state_t *state,
118     ddi_acc_handle_t pci_config_hdl);
119 #endif
120 static void hermon_flash_cfi_byte(uint8_t *ch, uint32_t dword, int i);
121 static void hermon_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i);
122 
123 /* Hemron loopback test functions */
124 static void hermon_loopback_free_qps(hermon_loopback_state_t *lstate);
125 static void hermon_loopback_free_state(hermon_loopback_state_t *lstate);
126 static int hermon_loopback_init(hermon_state_t *state,
127     hermon_loopback_state_t *lstate);
128 static void hermon_loopback_init_qp_info(hermon_loopback_state_t *lstate,
129     hermon_loopback_comm_t *comm);
130 static int hermon_loopback_alloc_mem(hermon_loopback_state_t *lstate,
131     hermon_loopback_comm_t *comm, int sz);
132 static int hermon_loopback_alloc_qps(hermon_loopback_state_t *lstate,
133     hermon_loopback_comm_t *comm);
134 static int hermon_loopback_modify_qp(hermon_loopback_state_t *lstate,
135     hermon_loopback_comm_t *comm, uint_t qp_num);
136 static int hermon_loopback_copyout(hermon_loopback_ioctl_t *lb,
137     intptr_t arg, int mode);
138 static int hermon_loopback_post_send(hermon_loopback_state_t *lstate,
139     hermon_loopback_comm_t *tx, hermon_loopback_comm_t *rx);
140 static int hermon_loopback_poll_cq(hermon_loopback_state_t *lstate,
141     hermon_loopback_comm_t *comm);
142 
143 /* Patchable timeout values for flash operations */
144 int hermon_hw_flash_timeout_gpio_sema = HERMON_HW_FLASH_TIMEOUT_GPIO_SEMA;
145 int hermon_hw_flash_timeout_config = HERMON_HW_FLASH_TIMEOUT_CONFIG;
146 int hermon_hw_flash_timeout_write = HERMON_HW_FLASH_TIMEOUT_WRITE;
147 int hermon_hw_flash_timeout_erase = HERMON_HW_FLASH_TIMEOUT_ERASE;
148 
149 /*
150  * hermon_ioctl()
151  */
152 /* ARGSUSED */
153 int
154 hermon_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *credp,
155     int *rvalp)
156 {
157 	hermon_state_t	*state;
158 	minor_t		instance;
159 	int		status;
160 
161 	if (drv_priv(credp) != 0) {
162 		return (EPERM);
163 	}
164 
165 	instance = HERMON_DEV_INSTANCE(dev);
166 	if (instance == (minor_t)-1) {
167 		return (EBADF);
168 	}
169 
170 	state = ddi_get_soft_state(hermon_statep, instance);
171 	if (state == NULL) {
172 		return (EBADF);
173 	}
174 
175 	status = 0;
176 
177 	switch (cmd) {
178 	case HERMON_IOCTL_FLASH_READ:
179 		status = hermon_ioctl_flash_read(state, dev, arg, mode);
180 		break;
181 
182 	case HERMON_IOCTL_FLASH_WRITE:
183 		status = hermon_ioctl_flash_write(state, dev, arg, mode);
184 		break;
185 
186 	case HERMON_IOCTL_FLASH_ERASE:
187 		status = hermon_ioctl_flash_erase(state, dev, arg, mode);
188 		break;
189 
190 	case HERMON_IOCTL_FLASH_INIT:
191 		status = hermon_ioctl_flash_init(state, dev, arg, mode);
192 		break;
193 
194 	case HERMON_IOCTL_FLASH_FINI:
195 		status = hermon_ioctl_flash_fini(state, dev);
196 		break;
197 
198 	case HERMON_IOCTL_INFO:
199 		status = hermon_ioctl_info(state, dev, arg, mode);
200 		break;
201 
202 	case HERMON_IOCTL_PORTS:
203 		status = hermon_ioctl_ports(state, arg, mode);
204 		break;
205 
206 	case HERMON_IOCTL_LOOPBACK:
207 		status = hermon_ioctl_loopback(state, arg, mode);
208 		break;
209 
210 #ifdef	DEBUG
211 	case HERMON_IOCTL_REG_WRITE:
212 		status = hermon_ioctl_reg_write(state, arg, mode);
213 		break;
214 
215 	case HERMON_IOCTL_REG_READ:
216 		status = hermon_ioctl_reg_read(state, arg, mode);
217 		break;
218 #endif	/* DEBUG */
219 
220 	case HERMON_IOCTL_DDR_READ:
221 		/* XXX guard until the ioctl header is cleaned up */
222 		status = ENODEV;
223 		break;
224 
225 	case HERMON_IOCTL_WRITE_BOOT_ADDR:
226 		status = hermon_ioctl_write_boot_addr(state, dev, arg, mode);
227 		break;
228 
229 	default:
230 		status = ENOTTY;
231 		break;
232 	}
233 	*rvalp = status;
234 
235 	return (status);
236 }
237 
238 /*
239  * hermon_ioctl_flash_read()
240  */
241 static int
242 hermon_ioctl_flash_read(hermon_state_t *state, dev_t dev, intptr_t arg,
243     int mode)
244 {
245 	hermon_flash_ioctl_t ioctl_info;
246 	int status = 0;
247 
248 	/*
249 	 * Check that flash init ioctl has been called first.  And check
250 	 * that the same dev_t that called init is the one calling read now.
251 	 */
252 	mutex_enter(&state->hs_fw_flashlock);
253 	if ((state->hs_fw_flashdev != dev) ||
254 	    (state->hs_fw_flashstarted == 0)) {
255 		mutex_exit(&state->hs_fw_flashlock);
256 		return (EIO);
257 	}
258 
259 	/* copy user struct to kernel */
260 #ifdef _MULTI_DATAMODEL
261 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
262 		hermon_flash_ioctl32_t info32;
263 
264 		if (ddi_copyin((void *)arg, &info32,
265 		    sizeof (hermon_flash_ioctl32_t), mode) != 0) {
266 			mutex_exit(&state->hs_fw_flashlock);
267 			return (EFAULT);
268 		}
269 		ioctl_info.af_type = info32.af_type;
270 		ioctl_info.af_sector = (caddr_t)(uintptr_t)info32.af_sector;
271 		ioctl_info.af_sector_num = info32.af_sector_num;
272 		ioctl_info.af_addr = info32.af_addr;
273 	} else
274 #endif /* _MULTI_DATAMODEL */
275 	if (ddi_copyin((void *)arg, &ioctl_info, sizeof (hermon_flash_ioctl_t),
276 	    mode) != 0) {
277 		mutex_exit(&state->hs_fw_flashlock);
278 		return (EFAULT);
279 	}
280 
281 	/*
282 	 * Determine type of READ ioctl
283 	 */
284 	switch (ioctl_info.af_type) {
285 	case HERMON_FLASH_READ_SECTOR:
286 		/* Check if sector num is too large for flash device */
287 		if (ioctl_info.af_sector_num >=
288 		    (state->hs_fw_device_sz >> state->hs_fw_log_sector_sz)) {
289 			mutex_exit(&state->hs_fw_flashlock);
290 			return (EFAULT);
291 		}
292 
293 		/* Perform the Sector Read */
294 		if ((status = hermon_flash_reset(state)) != 0 ||
295 		    (status = hermon_flash_read_sector(state,
296 		    ioctl_info.af_sector_num)) != 0) {
297 			mutex_exit(&state->hs_fw_flashlock);
298 			return (status);
299 		}
300 
301 		/* copyout the firmware sector image data */
302 		if (ddi_copyout(&state->hs_fw_sector[0],
303 		    &ioctl_info.af_sector[0], 1 << state->hs_fw_log_sector_sz,
304 		    mode) != 0) {
305 			mutex_exit(&state->hs_fw_flashlock);
306 			return (EFAULT);
307 		}
308 		break;
309 
310 	case HERMON_FLASH_READ_QUADLET:
311 		/* Check if addr is too large for flash device */
312 		if (ioctl_info.af_addr >= state->hs_fw_device_sz) {
313 			mutex_exit(&state->hs_fw_flashlock);
314 			return (EFAULT);
315 		}
316 
317 		/* Perform the Quadlet Read */
318 		if ((status = hermon_flash_reset(state)) != 0 ||
319 		    (status = hermon_flash_read_quadlet(state,
320 		    &ioctl_info.af_quadlet, ioctl_info.af_addr)) != 0) {
321 			mutex_exit(&state->hs_fw_flashlock);
322 			return (status);
323 		}
324 		break;
325 
326 	default:
327 		mutex_exit(&state->hs_fw_flashlock);
328 		return (EINVAL);
329 	}
330 
331 	/* copy results back to userland */
332 #ifdef _MULTI_DATAMODEL
333 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
334 		hermon_flash_ioctl32_t info32;
335 
336 		info32.af_quadlet = ioctl_info.af_quadlet;
337 		info32.af_type = ioctl_info.af_type;
338 		info32.af_sector_num = ioctl_info.af_sector_num;
339 		info32.af_sector = (caddr32_t)(uintptr_t)ioctl_info.af_sector;
340 		info32.af_addr = ioctl_info.af_addr;
341 
342 		if (ddi_copyout(&info32, (void *)arg,
343 		    sizeof (hermon_flash_ioctl32_t), mode) != 0) {
344 			mutex_exit(&state->hs_fw_flashlock);
345 			return (EFAULT);
346 		}
347 	} else
348 #endif /* _MULTI_DATAMODEL */
349 	if (ddi_copyout(&ioctl_info, (void *)arg,
350 	    sizeof (hermon_flash_ioctl_t), mode) != 0) {
351 		mutex_exit(&state->hs_fw_flashlock);
352 		return (EFAULT);
353 	}
354 
355 	mutex_exit(&state->hs_fw_flashlock);
356 	return (status);
357 }
358 
359 /*
360  * hermon_ioctl_flash_write()
361  */
362 static int
363 hermon_ioctl_flash_write(hermon_state_t *state, dev_t dev, intptr_t arg,
364     int mode)
365 {
366 	hermon_flash_ioctl_t	ioctl_info;
367 	int status = 0;
368 
369 	/*
370 	 * Check that flash init ioctl has been called first.  And check
371 	 * that the same dev_t that called init is the one calling write now.
372 	 */
373 	mutex_enter(&state->hs_fw_flashlock);
374 	if ((state->hs_fw_flashdev != dev) ||
375 	    (state->hs_fw_flashstarted == 0)) {
376 		mutex_exit(&state->hs_fw_flashlock);
377 		return (EIO);
378 	}
379 
380 	/* copy user struct to kernel */
381 #ifdef _MULTI_DATAMODEL
382 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
383 		hermon_flash_ioctl32_t info32;
384 
385 		if (ddi_copyin((void *)arg, &info32,
386 		    sizeof (hermon_flash_ioctl32_t), mode) != 0) {
387 			mutex_exit(&state->hs_fw_flashlock);
388 			return (EFAULT);
389 		}
390 		ioctl_info.af_type = info32.af_type;
391 		ioctl_info.af_sector = (caddr_t)(uintptr_t)info32.af_sector;
392 		ioctl_info.af_sector_num = info32.af_sector_num;
393 		ioctl_info.af_addr = info32.af_addr;
394 		ioctl_info.af_byte = info32.af_byte;
395 	} else
396 #endif /* _MULTI_DATAMODEL */
397 	if (ddi_copyin((void *)arg, &ioctl_info,
398 	    sizeof (hermon_flash_ioctl_t), mode) != 0) {
399 		mutex_exit(&state->hs_fw_flashlock);
400 		return (EFAULT);
401 	}
402 
403 	/*
404 	 * Determine type of WRITE ioctl
405 	 */
406 	switch (ioctl_info.af_type) {
407 	case HERMON_FLASH_WRITE_SECTOR:
408 		/* Check if sector num is too large for flash device */
409 		if (ioctl_info.af_sector_num >=
410 		    (state->hs_fw_device_sz >> state->hs_fw_log_sector_sz)) {
411 			mutex_exit(&state->hs_fw_flashlock);
412 			return (EFAULT);
413 		}
414 
415 		/* copy in fw sector image data */
416 		if (ddi_copyin(&ioctl_info.af_sector[0],
417 		    &state->hs_fw_sector[0], 1 << state->hs_fw_log_sector_sz,
418 		    mode) != 0) {
419 			mutex_exit(&state->hs_fw_flashlock);
420 			return (EFAULT);
421 		}
422 
423 		/* Perform Write Sector */
424 		status = hermon_flash_write_sector(state,
425 		    ioctl_info.af_sector_num);
426 		break;
427 
428 	case HERMON_FLASH_WRITE_BYTE:
429 		/* Check if addr is too large for flash device */
430 		if (ioctl_info.af_addr >= state->hs_fw_device_sz) {
431 			mutex_exit(&state->hs_fw_flashlock);
432 			return (EFAULT);
433 		}
434 
435 		/* Perform Write Byte */
436 		/*
437 		 * CMJ -- is a reset really needed before and after writing
438 		 * each byte?  This code came from arbel, but we should look
439 		 * into this.  Also, for SPI, no reset is actually performed.
440 		 */
441 		if ((status = hermon_flash_bank(state,
442 		    ioctl_info.af_addr)) != 0 ||
443 		    (status = hermon_flash_reset(state)) != 0 ||
444 		    (status = hermon_flash_write_byte(state,
445 		    ioctl_info.af_addr, ioctl_info.af_byte)) != 0 ||
446 		    (status = hermon_flash_reset(state)) != 0) {
447 			mutex_exit(&state->hs_fw_flashlock);
448 			return (status);
449 		}
450 		break;
451 
452 	default:
453 		status = EINVAL;
454 		break;
455 	}
456 
457 	mutex_exit(&state->hs_fw_flashlock);
458 	return (status);
459 }
460 
461 /*
462  * hermon_ioctl_flash_erase()
463  */
464 static int
465 hermon_ioctl_flash_erase(hermon_state_t *state, dev_t dev, intptr_t arg,
466     int mode)
467 {
468 	hermon_flash_ioctl_t	ioctl_info;
469 	int status = 0;
470 
471 	/*
472 	 * Check that flash init ioctl has been called first.  And check
473 	 * that the same dev_t that called init is the one calling erase now.
474 	 */
475 	mutex_enter(&state->hs_fw_flashlock);
476 	if ((state->hs_fw_flashdev != dev) ||
477 	    (state->hs_fw_flashstarted == 0)) {
478 		mutex_exit(&state->hs_fw_flashlock);
479 		return (EIO);
480 	}
481 
482 	/* copy user struct to kernel */
483 #ifdef _MULTI_DATAMODEL
484 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
485 		hermon_flash_ioctl32_t info32;
486 
487 		if (ddi_copyin((void *)arg, &info32,
488 		    sizeof (hermon_flash_ioctl32_t), mode) != 0) {
489 			mutex_exit(&state->hs_fw_flashlock);
490 			return (EFAULT);
491 		}
492 		ioctl_info.af_type = info32.af_type;
493 		ioctl_info.af_sector_num = info32.af_sector_num;
494 	} else
495 #endif /* _MULTI_DATAMODEL */
496 	if (ddi_copyin((void *)arg, &ioctl_info, sizeof (hermon_flash_ioctl_t),
497 	    mode) != 0) {
498 		mutex_exit(&state->hs_fw_flashlock);
499 		return (EFAULT);
500 	}
501 
502 	/*
503 	 * Determine type of ERASE ioctl
504 	 */
505 	switch (ioctl_info.af_type) {
506 	case HERMON_FLASH_ERASE_SECTOR:
507 		/* Check if sector num is too large for flash device */
508 		if (ioctl_info.af_sector_num >=
509 		    (state->hs_fw_device_sz >> state->hs_fw_log_sector_sz)) {
510 			mutex_exit(&state->hs_fw_flashlock);
511 			return (EFAULT);
512 		}
513 
514 		/* Perform Sector Erase */
515 		status = hermon_flash_erase_sector(state,
516 		    ioctl_info.af_sector_num);
517 		break;
518 
519 	case HERMON_FLASH_ERASE_CHIP:
520 		/* Perform Chip Erase */
521 		status = hermon_flash_erase_chip(state);
522 		break;
523 
524 	default:
525 		status = EINVAL;
526 		break;
527 	}
528 
529 	mutex_exit(&state->hs_fw_flashlock);
530 	return (status);
531 }
532 
533 /*
534  * hermon_ioctl_flash_init()
535  */
536 static int
537 hermon_ioctl_flash_init(hermon_state_t *state, dev_t dev, intptr_t arg,
538     int mode)
539 {
540 	hermon_flash_init_ioctl_t init_info;
541 	int ret;
542 	int intel_xcmd = 0;
543 	ddi_acc_handle_t pci_hdl = hermon_get_pcihdl(state);
544 
545 	/* initialize the FMA retry loop */
546 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
547 
548 	state->hs_fw_sector = NULL;
549 
550 	/*
551 	 * init cannot be called more than once.  If we have already init'd the
552 	 * flash, return directly.
553 	 */
554 	mutex_enter(&state->hs_fw_flashlock);
555 	if (state->hs_fw_flashstarted == 1) {
556 		mutex_exit(&state->hs_fw_flashlock);
557 		return (EINVAL);
558 	}
559 
560 	/* copyin the user struct to kernel */
561 	if (ddi_copyin((void *)arg, &init_info,
562 	    sizeof (hermon_flash_init_ioctl_t), mode) != 0) {
563 		mutex_exit(&state->hs_fw_flashlock);
564 		return (EFAULT);
565 	}
566 
567 	/* Init Flash */
568 	if ((ret = hermon_flash_init(state)) != 0) {
569 		if (ret == EIO) {
570 			goto pio_error;
571 		}
572 		mutex_exit(&state->hs_fw_flashlock);
573 		return (ret);
574 	}
575 
576 	/* Read CFI info */
577 	if ((ret = hermon_flash_cfi_init(state, &init_info.af_cfi_info[0],
578 	    &intel_xcmd)) != 0) {
579 		if (ret == EIO) {
580 			goto pio_error;
581 		}
582 		mutex_exit(&state->hs_fw_flashlock);
583 		return (ret);
584 	}
585 
586 	/*
587 	 * Return error if the command set is unknown.
588 	 */
589 	if (state->hs_fw_cmdset == HERMON_FLASH_UNKNOWN_CMDSET) {
590 		if ((ret = hermon_ioctl_flash_cleanup_nolock(state)) != 0) {
591 			if (ret == EIO) {
592 				goto pio_error;
593 			}
594 			mutex_exit(&state->hs_fw_flashlock);
595 			return (ret);
596 		}
597 		mutex_exit(&state->hs_fw_flashlock);
598 		return (EFAULT);
599 	}
600 
601 	/* the FMA retry loop starts. */
602 	hermon_pio_start(state, pci_hdl, pio_error,
603 	    fm_loop_cnt, fm_status, fm_test);
604 
605 	/* Read HWREV - least significant 8 bits is revision ID */
606 	init_info.af_hwrev = pci_config_get32(pci_hdl,
607 	    HERMON_HW_FLASH_CFG_HWREV) & 0xFF;
608 
609 	/* the FMA retry loop ends. */
610 	hermon_pio_end(state, pci_hdl, pio_error, fm_loop_cnt,
611 	    fm_status, fm_test);
612 
613 	/* Fill in the firmwate revision numbers */
614 	init_info.af_fwrev.afi_maj	= state->hs_fw.fw_rev_major;
615 	init_info.af_fwrev.afi_min	= state->hs_fw.fw_rev_minor;
616 	init_info.af_fwrev.afi_sub	= state->hs_fw.fw_rev_subminor;
617 
618 	/* Alloc flash mem for one sector size */
619 	state->hs_fw_sector = (uint32_t *)kmem_zalloc(1 <<
620 	    state->hs_fw_log_sector_sz, KM_SLEEP);
621 
622 	/* Set HW part number and length */
623 	init_info.af_pn_len = state->hs_hca_pn_len;
624 	if (state->hs_hca_pn_len != 0) {
625 		(void) memcpy(init_info.af_hwpn, state->hs_hca_pn,
626 		    state->hs_hca_pn_len);
627 	}
628 
629 	/* Copy ioctl results back to userland */
630 	if (ddi_copyout(&init_info, (void *)arg,
631 	    sizeof (hermon_flash_init_ioctl_t), mode) != 0) {
632 		if ((ret = hermon_ioctl_flash_cleanup_nolock(state)) != 0) {
633 			if (ret == EIO) {
634 				goto pio_error;
635 			}
636 			mutex_exit(&state->hs_fw_flashlock);
637 			return (ret);
638 		}
639 		mutex_exit(&state->hs_fw_flashlock);
640 		return (EFAULT);
641 	}
642 
643 	/* Set flash state to started */
644 	state->hs_fw_flashstarted = 1;
645 	state->hs_fw_flashdev	  = dev;
646 
647 	mutex_exit(&state->hs_fw_flashlock);
648 
649 	/*
650 	 * If "flash init" is successful, add an "on close" callback to the
651 	 * current dev node to ensure that "flash fini" gets called later
652 	 * even if the userland process prematurely exits.
653 	 */
654 	ret = hermon_umap_db_set_onclose_cb(dev,
655 	    HERMON_ONCLOSE_FLASH_INPROGRESS,
656 	    (int (*)(void *))hermon_ioctl_flash_cleanup, state);
657 	if (ret != DDI_SUCCESS) {
658 		int status = hermon_ioctl_flash_fini(state, dev);
659 		if (status != 0) {
660 			if (status == EIO) {
661 				hermon_fm_ereport(state, HCA_SYS_ERR,
662 				    HCA_ERR_IOCTL);
663 				return (EIO);
664 			}
665 			return (status);
666 		}
667 	}
668 	return (0);
669 
670 pio_error:
671 	mutex_exit(&state->hs_fw_flashlock);
672 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
673 	return (EIO);
674 }
675 
676 /*
677  * hermon_ioctl_flash_fini()
678  */
679 static int
680 hermon_ioctl_flash_fini(hermon_state_t *state, dev_t dev)
681 {
682 	int ret;
683 
684 	/*
685 	 * Check that flash init ioctl has been called first.  And check
686 	 * that the same dev_t that called init is the one calling fini now.
687 	 */
688 	mutex_enter(&state->hs_fw_flashlock);
689 	if ((state->hs_fw_flashdev != dev) ||
690 	    (state->hs_fw_flashstarted == 0)) {
691 		mutex_exit(&state->hs_fw_flashlock);
692 		return (EINVAL);
693 	}
694 
695 	if ((ret = hermon_ioctl_flash_cleanup_nolock(state)) != 0) {
696 		mutex_exit(&state->hs_fw_flashlock);
697 		if (ret == EIO) {
698 			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
699 		}
700 		return (ret);
701 	}
702 	mutex_exit(&state->hs_fw_flashlock);
703 
704 	/*
705 	 * If "flash fini" is successful, remove the "on close" callback
706 	 * that was setup during "flash init".
707 	 */
708 	ret = hermon_umap_db_clear_onclose_cb(dev,
709 	    HERMON_ONCLOSE_FLASH_INPROGRESS);
710 	if (ret != DDI_SUCCESS) {
711 		return (EFAULT);
712 	}
713 	return (0);
714 }
715 
716 
717 /*
718  * hermon_ioctl_flash_cleanup()
719  */
720 static int
721 hermon_ioctl_flash_cleanup(hermon_state_t *state)
722 {
723 	int status;
724 
725 	mutex_enter(&state->hs_fw_flashlock);
726 	status = hermon_ioctl_flash_cleanup_nolock(state);
727 	mutex_exit(&state->hs_fw_flashlock);
728 
729 	return (status);
730 }
731 
732 
733 /*
734  * hermon_ioctl_flash_cleanup_nolock()
735  */
736 static int
737 hermon_ioctl_flash_cleanup_nolock(hermon_state_t *state)
738 {
739 	int status;
740 	ASSERT(MUTEX_HELD(&state->hs_fw_flashlock));
741 
742 	/* free flash mem */
743 	if (state->hs_fw_sector) {
744 		kmem_free(state->hs_fw_sector, 1 << state->hs_fw_log_sector_sz);
745 	}
746 
747 	/* Fini the Flash */
748 	if ((status = hermon_flash_fini(state)) != 0)
749 		return (status);
750 
751 	/* Set flash state to fini */
752 	state->hs_fw_flashstarted = 0;
753 	state->hs_fw_flashdev	  = 0;
754 	return (0);
755 }
756 
757 
758 /*
759  * hermon_ioctl_info()
760  */
761 static int
762 hermon_ioctl_info(hermon_state_t *state, dev_t dev, intptr_t arg, int mode)
763 {
764 	hermon_info_ioctl_t	 info;
765 	hermon_flash_init_ioctl_t init_info;
766 
767 	/*
768 	 * Access to Hemron VTS ioctls is not allowed in "maintenance mode".
769 	 */
770 	if (state->hs_operational_mode == HERMON_MAINTENANCE_MODE) {
771 		return (EFAULT);
772 	}
773 
774 	/* copyin the user struct to kernel */
775 	if (ddi_copyin((void *)arg, &info, sizeof (hermon_info_ioctl_t),
776 	    mode) != 0) {
777 		return (EFAULT);
778 	}
779 
780 	/*
781 	 * Check ioctl revision
782 	 */
783 	if (info.ai_revision != HERMON_VTS_IOCTL_REVISION) {
784 		return (EINVAL);
785 	}
786 
787 	/*
788 	 * If the 'fw_device_sz' has not been initialized yet, we initialize it
789 	 * here.  This is done by leveraging the
790 	 * hermon_ioctl_flash_init()/fini() calls.  We also hold our own mutex
791 	 * around this operation in case we have multiple VTS threads in
792 	 * process at the same time.
793 	 */
794 	mutex_enter(&state->hs_info_lock);
795 	if (state->hs_fw_device_sz == 0) {
796 		if (hermon_ioctl_flash_init(state, dev, (intptr_t)&init_info,
797 		    (FKIOCTL | mode)) != 0) {
798 			mutex_exit(&state->hs_info_lock);
799 			return (EFAULT);
800 		}
801 		(void) hermon_ioctl_flash_fini(state, dev);
802 	}
803 	mutex_exit(&state->hs_info_lock);
804 
805 	info.ai_hw_rev		 = state->hs_revision_id;
806 	info.ai_flash_sz	 = state->hs_fw_device_sz;
807 	info.ai_fw_rev.afi_maj	 = state->hs_fw.fw_rev_major;
808 	info.ai_fw_rev.afi_min	 = state->hs_fw.fw_rev_minor;
809 	info.ai_fw_rev.afi_sub	 = state->hs_fw.fw_rev_subminor;
810 
811 	/* Copy ioctl results back to user struct */
812 	if (ddi_copyout(&info, (void *)arg, sizeof (hermon_info_ioctl_t),
813 	    mode) != 0) {
814 		return (EFAULT);
815 	}
816 
817 	return (0);
818 }
819 
820 /*
821  * hermon_ioctl_ports()
822  */
823 static int
824 hermon_ioctl_ports(hermon_state_t *state, intptr_t arg, int mode)
825 {
826 	hermon_ports_ioctl_t	info;
827 	hermon_stat_port_ioctl_t	portstat;
828 	ibt_hca_portinfo_t	pi;
829 	uint_t			tbl_size;
830 	ib_gid_t		*sgid_tbl;
831 	ib_pkey_t		*pkey_tbl;
832 	int			i;
833 
834 	/*
835 	 * Access to Hemron VTS ioctls is not allowed in "maintenance mode".
836 	 */
837 	if (state->hs_operational_mode == HERMON_MAINTENANCE_MODE) {
838 		return (EFAULT);
839 	}
840 
841 	/* copyin the user struct to kernel */
842 #ifdef _MULTI_DATAMODEL
843 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
844 		hermon_ports_ioctl32_t info32;
845 
846 		if (ddi_copyin((void *)arg, &info32,
847 		    sizeof (hermon_ports_ioctl32_t), mode) != 0) {
848 			return (EFAULT);
849 		}
850 		info.ap_revision  = info32.ap_revision;
851 		info.ap_ports	  =
852 		    (hermon_stat_port_ioctl_t *)(uintptr_t)info32.ap_ports;
853 		info.ap_num_ports = info32.ap_num_ports;
854 
855 	} else
856 #endif /* _MULTI_DATAMODEL */
857 	if (ddi_copyin((void *)arg, &info, sizeof (hermon_ports_ioctl_t),
858 	    mode) != 0) {
859 		return (EFAULT);
860 	}
861 
862 	/*
863 	 * Check ioctl revision
864 	 */
865 	if (info.ap_revision != HERMON_VTS_IOCTL_REVISION) {
866 		return (EINVAL);
867 	}
868 
869 	/* Allocate space for temporary GID table/PKey table */
870 	tbl_size = (1 << state->hs_cfg_profile->cp_log_max_gidtbl);
871 	sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t),
872 	    KM_SLEEP);
873 	tbl_size = (1 << state->hs_cfg_profile->cp_log_max_pkeytbl);
874 	pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t),
875 	    KM_SLEEP);
876 
877 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgid_tbl, *pkey_tbl))
878 
879 	/*
880 	 * Setup the number of ports, then loop through all ports and
881 	 * query properties of each.
882 	 */
883 	info.ap_num_ports = (uint8_t)state->hs_cfg_profile->cp_num_ports;
884 	for (i = 0; i < info.ap_num_ports; i++) {
885 		/*
886 		 * Get portstate information from the device.  If
887 		 * hermon_port_query() fails, leave zeroes in user
888 		 * struct port entry and continue.
889 		 */
890 		bzero(&pi, sizeof (ibt_hca_portinfo_t));
891 		pi.p_sgid_tbl = sgid_tbl;
892 		pi.p_pkey_tbl = pkey_tbl;
893 		(void) hermon_port_query(state, i + 1, &pi);
894 
895 		portstat.asp_port_num	= pi.p_port_num;
896 		portstat.asp_state	= pi.p_linkstate;
897 		portstat.asp_guid	= pi.p_sgid_tbl[0].gid_guid;
898 
899 		/*
900 		 * Copy queried port results back to user struct.  If
901 		 * this fails, then break out of loop, attempt to copy
902 		 * out remaining info to user struct, and return (without
903 		 * error).
904 		 */
905 		if (ddi_copyout(&portstat,
906 		    &(((hermon_stat_port_ioctl_t *)info.ap_ports)[i]),
907 		    sizeof (hermon_stat_port_ioctl_t), mode) != 0) {
908 			break;
909 		}
910 	}
911 
912 	/* Free the temporary space used for GID table/PKey table */
913 	tbl_size = (1 << state->hs_cfg_profile->cp_log_max_gidtbl);
914 	kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
915 	tbl_size = (1 << state->hs_cfg_profile->cp_log_max_pkeytbl);
916 	kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
917 
918 	/* Copy ioctl results back to user struct */
919 #ifdef _MULTI_DATAMODEL
920 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
921 		hermon_ports_ioctl32_t info32;
922 
923 		info32.ap_revision  = info.ap_revision;
924 		info32.ap_ports	    = (caddr32_t)(uintptr_t)info.ap_ports;
925 		info32.ap_num_ports = info.ap_num_ports;
926 
927 		if (ddi_copyout(&info32, (void *)arg,
928 		    sizeof (hermon_ports_ioctl32_t), mode) != 0) {
929 			return (EFAULT);
930 		}
931 	} else
932 #endif /* _MULTI_DATAMODEL */
933 	if (ddi_copyout(&info, (void *)arg, sizeof (hermon_ports_ioctl_t),
934 	    mode) != 0) {
935 		return (EFAULT);
936 	}
937 
938 	return (0);
939 }
940 
941 /*
942  * hermon_ioctl_loopback()
943  */
944 static int
945 hermon_ioctl_loopback(hermon_state_t *state, intptr_t arg, int mode)
946 {
947 	hermon_loopback_ioctl_t	lb;
948 	hermon_loopback_state_t	lstate;
949 	ibt_hca_portinfo_t 	pi;
950 	uint_t			tbl_size, loopmax, max_usec;
951 	ib_gid_t		*sgid_tbl;
952 	ib_pkey_t		*pkey_tbl;
953 	int			j, iter, ret;
954 
955 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(lstate))
956 
957 	/*
958 	 * Access to Hemron VTS ioctls is not allowed in "maintenance mode".
959 	 */
960 	if (state->hs_operational_mode == HERMON_MAINTENANCE_MODE) {
961 		return (EFAULT);
962 	}
963 
964 	/* copyin the user struct to kernel */
965 #ifdef _MULTI_DATAMODEL
966 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
967 		hermon_loopback_ioctl32_t lb32;
968 
969 		if (ddi_copyin((void *)arg, &lb32,
970 		    sizeof (hermon_loopback_ioctl32_t), mode) != 0) {
971 			return (EFAULT);
972 		}
973 		lb.alb_revision	    = lb32.alb_revision;
974 		lb.alb_send_buf	    = (caddr_t)(uintptr_t)lb32.alb_send_buf;
975 		lb.alb_fail_buf	    = (caddr_t)(uintptr_t)lb32.alb_fail_buf;
976 		lb.alb_buf_sz	    = lb32.alb_buf_sz;
977 		lb.alb_num_iter	    = lb32.alb_num_iter;
978 		lb.alb_pass_done    = lb32.alb_pass_done;
979 		lb.alb_timeout	    = lb32.alb_timeout;
980 		lb.alb_error_type   = lb32.alb_error_type;
981 		lb.alb_port_num	    = lb32.alb_port_num;
982 		lb.alb_num_retry    = lb32.alb_num_retry;
983 	} else
984 #endif /* _MULTI_DATAMODEL */
985 	if (ddi_copyin((void *)arg, &lb, sizeof (hermon_loopback_ioctl_t),
986 	    mode) != 0) {
987 		return (EFAULT);
988 	}
989 
990 	/* Initialize the internal loopback test state structure */
991 	bzero(&lstate, sizeof (hermon_loopback_state_t));
992 
993 	/*
994 	 * Check ioctl revision
995 	 */
996 	if (lb.alb_revision != HERMON_VTS_IOCTL_REVISION) {
997 		lb.alb_error_type = HERMON_LOOPBACK_INVALID_REVISION;
998 		(void) hermon_loopback_copyout(&lb, arg, mode);
999 		return (EINVAL);
1000 	}
1001 
1002 	/* Validate that specified port number is legal */
1003 	if (!hermon_portnum_is_valid(state, lb.alb_port_num)) {
1004 		lb.alb_error_type = HERMON_LOOPBACK_INVALID_PORT;
1005 		(void) hermon_loopback_copyout(&lb, arg, mode);
1006 		return (EINVAL);
1007 	}
1008 
1009 	/* Allocate space for temporary GID table/PKey table */
1010 	tbl_size = (1 << state->hs_cfg_profile->cp_log_max_gidtbl);
1011 	sgid_tbl = (ib_gid_t *)kmem_zalloc(tbl_size * sizeof (ib_gid_t),
1012 	    KM_SLEEP);
1013 	tbl_size = (1 << state->hs_cfg_profile->cp_log_max_pkeytbl);
1014 	pkey_tbl = (ib_pkey_t *)kmem_zalloc(tbl_size * sizeof (ib_pkey_t),
1015 	    KM_SLEEP);
1016 
1017 	/*
1018 	 * Get portstate information from specific port on device
1019 	 */
1020 	bzero(&pi, sizeof (ibt_hca_portinfo_t));
1021 	pi.p_sgid_tbl = sgid_tbl;
1022 	pi.p_pkey_tbl = pkey_tbl;
1023 	if (hermon_port_query(state, lb.alb_port_num, &pi) != 0) {
1024 		/* Free the temporary space used for GID table/PKey table */
1025 		tbl_size = (1 << state->hs_cfg_profile->cp_log_max_gidtbl);
1026 		kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
1027 		tbl_size = (1 << state->hs_cfg_profile->cp_log_max_pkeytbl);
1028 		kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
1029 
1030 		lb.alb_error_type = HERMON_LOOPBACK_INVALID_PORT;
1031 		(void) hermon_loopback_copyout(&lb, arg, mode);
1032 		hermon_loopback_free_state(&lstate);
1033 		return (EINVAL);
1034 	}
1035 
1036 	lstate.hls_port	   = pi.p_port_num;
1037 	lstate.hls_lid	   = pi.p_base_lid;
1038 	lstate.hls_pkey_ix = (pi.p_linkstate == HERMON_PORT_LINK_ACTIVE) ?
1039 	    1 : 0;	/* XXX bogus assumption of a SUN subnet manager */
1040 	lstate.hls_state   = state;
1041 	lstate.hls_retry   = lb.alb_num_retry;
1042 
1043 	/* Free the temporary space used for GID table/PKey table */
1044 	tbl_size = (1 << state->hs_cfg_profile->cp_log_max_gidtbl);
1045 	kmem_free(sgid_tbl, tbl_size * sizeof (ib_gid_t));
1046 	tbl_size = (1 << state->hs_cfg_profile->cp_log_max_pkeytbl);
1047 	kmem_free(pkey_tbl, tbl_size * sizeof (ib_pkey_t));
1048 
1049 	/*
1050 	 * Compute the timeout duration in usec per the formula:
1051 	 *    to_usec_per_retry = 4.096us * (2 ^ supplied_timeout)
1052 	 * (plus we add a little fudge-factor here too)
1053 	 */
1054 	lstate.hls_timeout = lb.alb_timeout;
1055 	max_usec = (4096 * (1 << lstate.hls_timeout)) / 1000;
1056 	max_usec = max_usec * (lstate.hls_retry + 1);
1057 	max_usec = max_usec + 10000;
1058 
1059 	/*
1060 	 * Determine how many times we should loop before declaring a
1061 	 * timeout failure.
1062 	 */
1063 	loopmax	 = max_usec/HERMON_VTS_LOOPBACK_MIN_WAIT_DUR;
1064 	if ((max_usec % HERMON_VTS_LOOPBACK_MIN_WAIT_DUR) != 0) {
1065 		loopmax++;
1066 	}
1067 
1068 	if (lb.alb_send_buf == NULL || lb.alb_buf_sz == 0) {
1069 		lb.alb_error_type = HERMON_LOOPBACK_SEND_BUF_INVALID;
1070 		(void) hermon_loopback_copyout(&lb, arg, mode);
1071 		hermon_loopback_free_state(&lstate);
1072 		return (EINVAL);
1073 	}
1074 
1075 	/* Allocate protection domain (PD) */
1076 	if (hermon_loopback_init(state, &lstate) != 0) {
1077 		lb.alb_error_type = lstate.hls_err;
1078 		(void) hermon_loopback_copyout(&lb, arg, mode);
1079 		hermon_loopback_free_state(&lstate);
1080 		return (EFAULT);
1081 	}
1082 
1083 	/* Allocate and register a TX buffer */
1084 	if (hermon_loopback_alloc_mem(&lstate, &lstate.hls_tx,
1085 	    lb.alb_buf_sz) != 0) {
1086 		lb.alb_error_type =
1087 		    HERMON_LOOPBACK_SEND_BUF_MEM_REGION_ALLOC_FAIL;
1088 		(void) hermon_loopback_copyout(&lb, arg, mode);
1089 		hermon_loopback_free_state(&lstate);
1090 		return (EFAULT);
1091 	}
1092 
1093 	/* Allocate and register an RX buffer */
1094 	if (hermon_loopback_alloc_mem(&lstate, &lstate.hls_rx,
1095 	    lb.alb_buf_sz) != 0) {
1096 		lb.alb_error_type =
1097 		    HERMON_LOOPBACK_RECV_BUF_MEM_REGION_ALLOC_FAIL;
1098 		(void) hermon_loopback_copyout(&lb, arg, mode);
1099 		hermon_loopback_free_state(&lstate);
1100 		return (EFAULT);
1101 	}
1102 
1103 	/* Copy in the transmit buffer data */
1104 	if (ddi_copyin((void *)lb.alb_send_buf, lstate.hls_tx.hlc_buf,
1105 	    lb.alb_buf_sz, mode) != 0) {
1106 		lb.alb_error_type = HERMON_LOOPBACK_SEND_BUF_COPY_FAIL;
1107 		(void) hermon_loopback_copyout(&lb, arg, mode);
1108 		hermon_loopback_free_state(&lstate);
1109 		return (EFAULT);
1110 	}
1111 
1112 	/* Allocate the transmit QP and CQs */
1113 	lstate.hls_err = HERMON_LOOPBACK_XMIT_SEND_CQ_ALLOC_FAIL;
1114 	if (hermon_loopback_alloc_qps(&lstate, &lstate.hls_tx) != 0) {
1115 		lb.alb_error_type = lstate.hls_err;
1116 		(void) hermon_loopback_copyout(&lb, arg, mode);
1117 		hermon_loopback_free_state(&lstate);
1118 		return (EFAULT);
1119 	}
1120 
1121 	/* Allocate the receive QP and CQs */
1122 	lstate.hls_err = HERMON_LOOPBACK_RECV_SEND_CQ_ALLOC_FAIL;
1123 	if (hermon_loopback_alloc_qps(&lstate, &lstate.hls_rx) != 0) {
1124 		lb.alb_error_type = lstate.hls_err;
1125 		(void) hermon_loopback_copyout(&lb, arg, mode);
1126 		hermon_loopback_free_state(&lstate);
1127 		return (EFAULT);
1128 	}
1129 
1130 	/* Activate the TX QP (connect to RX QP) */
1131 	lstate.hls_err = HERMON_LOOPBACK_XMIT_QP_INIT_FAIL;
1132 	if (hermon_loopback_modify_qp(&lstate, &lstate.hls_tx,
1133 	    lstate.hls_rx.hlc_qp_num) != 0) {
1134 		lb.alb_error_type = lstate.hls_err;
1135 		(void) hermon_loopback_copyout(&lb, arg, mode);
1136 		hermon_loopback_free_state(&lstate);
1137 		return (EFAULT);
1138 	}
1139 
1140 	/* Activate the RX QP (connect to TX QP) */
1141 	lstate.hls_err = HERMON_LOOPBACK_RECV_QP_INIT_FAIL;
1142 	if (hermon_loopback_modify_qp(&lstate, &lstate.hls_rx,
1143 	    lstate.hls_tx.hlc_qp_num) != 0) {
1144 		lb.alb_error_type = lstate.hls_err;
1145 		(void) hermon_loopback_copyout(&lb, arg, mode);
1146 		hermon_loopback_free_state(&lstate);
1147 		return (EFAULT);
1148 	}
1149 
1150 	/* Run the loopback test (for specified number of iterations) */
1151 	lb.alb_pass_done = 0;
1152 	for (iter = 0; iter < lb.alb_num_iter; iter++) {
1153 		lstate.hls_err = 0;
1154 		bzero(lstate.hls_rx.hlc_buf, lb.alb_buf_sz);
1155 
1156 		/* Post RDMA Write work request */
1157 		if (hermon_loopback_post_send(&lstate, &lstate.hls_tx,
1158 		    &lstate.hls_rx) != IBT_SUCCESS) {
1159 			lb.alb_error_type = HERMON_LOOPBACK_WQE_POST_FAIL;
1160 			(void) hermon_loopback_copyout(&lb, arg, mode);
1161 			hermon_loopback_free_state(&lstate);
1162 			return (EFAULT);
1163 		}
1164 
1165 		/* Poll the TX CQ for a completion every few ticks */
1166 		for (j = 0; j < loopmax; j++) {
1167 			delay(drv_usectohz(HERMON_VTS_LOOPBACK_MIN_WAIT_DUR));
1168 
1169 			ret = hermon_loopback_poll_cq(&lstate, &lstate.hls_tx);
1170 			if (((ret != IBT_SUCCESS) && (ret != IBT_CQ_EMPTY)) ||
1171 			    ((ret == IBT_CQ_EMPTY) && (j == loopmax - 1))) {
1172 				lb.alb_error_type =
1173 				    HERMON_LOOPBACK_CQ_POLL_FAIL;
1174 				if (ddi_copyout(lstate.hls_rx.hlc_buf,
1175 				    lb.alb_fail_buf, lstate.hls_tx.hlc_buf_sz,
1176 				    mode) != 0) {
1177 					return (EFAULT);
1178 				}
1179 				(void) hermon_loopback_copyout(&lb, arg, mode);
1180 				hermon_loopback_free_state(&lstate);
1181 				return (EFAULT);
1182 			} else if (ret == IBT_CQ_EMPTY) {
1183 				continue;
1184 			}
1185 
1186 			/* Compare the data buffers */
1187 			if (bcmp(lstate.hls_tx.hlc_buf, lstate.hls_rx.hlc_buf,
1188 			    lb.alb_buf_sz) == 0) {
1189 				break;
1190 			} else {
1191 				lb.alb_error_type =
1192 				    HERMON_LOOPBACK_SEND_RECV_COMPARE_FAIL;
1193 				if (ddi_copyout(lstate.hls_rx.hlc_buf,
1194 				    lb.alb_fail_buf, lstate.hls_tx.hlc_buf_sz,
1195 				    mode) != 0) {
1196 					return (EFAULT);
1197 				}
1198 				(void) hermon_loopback_copyout(&lb, arg, mode);
1199 				hermon_loopback_free_state(&lstate);
1200 				return (EFAULT);
1201 			}
1202 		}
1203 
1204 		lstate.hls_err	 = HERMON_LOOPBACK_SUCCESS;
1205 		lb.alb_pass_done = iter + 1;
1206 	}
1207 
1208 	lb.alb_error_type = HERMON_LOOPBACK_SUCCESS;
1209 
1210 	/* Copy ioctl results back to user struct */
1211 	ret = hermon_loopback_copyout(&lb, arg, mode);
1212 
1213 	/* Free up everything and release all consumed resources */
1214 	hermon_loopback_free_state(&lstate);
1215 
1216 	return (ret);
1217 }
1218 
1219 #ifdef	DEBUG
1220 /*
1221  * hermon_ioctl_reg_read()
1222  */
1223 static int
1224 hermon_ioctl_reg_read(hermon_state_t *state, intptr_t arg, int mode)
1225 {
1226 	hermon_reg_ioctl_t	rdreg;
1227 	uint32_t		*addr;
1228 	uintptr_t		baseaddr;
1229 	int			status;
1230 	ddi_acc_handle_t	handle;
1231 
1232 	/* initialize the FMA retry loop */
1233 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
1234 
1235 	/*
1236 	 * Access to Hemron registers is not allowed in "maintenance mode".
1237 	 * This is primarily because the device may not have BARs to access
1238 	 */
1239 	if (state->hs_operational_mode == HERMON_MAINTENANCE_MODE) {
1240 		return (EFAULT);
1241 	}
1242 
1243 	/* Copy in the hermon_reg_ioctl_t structure */
1244 	status = ddi_copyin((void *)arg, &rdreg, sizeof (hermon_reg_ioctl_t),
1245 	    mode);
1246 	if (status != 0) {
1247 		return (EFAULT);
1248 	}
1249 
1250 	/* Determine base address for requested register set */
1251 	switch (rdreg.arg_reg_set) {
1252 	case HERMON_CMD_BAR:
1253 		baseaddr = (uintptr_t)state->hs_reg_cmd_baseaddr;
1254 		handle = hermon_get_cmdhdl(state);
1255 		break;
1256 
1257 	case HERMON_UAR_BAR:
1258 		baseaddr = (uintptr_t)state->hs_reg_uar_baseaddr;
1259 		handle = hermon_get_uarhdl(state);
1260 		break;
1261 
1262 
1263 	default:
1264 		return (EINVAL);
1265 	}
1266 
1267 	/* Ensure that address is properly-aligned */
1268 	addr = (uint32_t *)((baseaddr + rdreg.arg_offset) & ~0x3);
1269 
1270 	/* the FMA retry loop starts. */
1271 	hermon_pio_start(state, handle, pio_error, fm_loop_cnt,
1272 	    fm_status, fm_test);
1273 
1274 	/* Read the register pointed to by addr */
1275 	rdreg.arg_data = ddi_get32(handle, addr);
1276 
1277 	/* the FMA retry loop ends. */
1278 	hermon_pio_end(state, handle, pio_error, fm_loop_cnt, fm_status,
1279 	    fm_test);
1280 
1281 	/* Copy in the result into the hermon_reg_ioctl_t structure */
1282 	status = ddi_copyout(&rdreg, (void *)arg, sizeof (hermon_reg_ioctl_t),
1283 	    mode);
1284 	if (status != 0) {
1285 		return (EFAULT);
1286 	}
1287 	return (0);
1288 
1289 pio_error:
1290 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
1291 	return (EIO);
1292 }
1293 
1294 
1295 /*
1296  * hermon_ioctl_reg_write()
1297  */
1298 static int
1299 hermon_ioctl_reg_write(hermon_state_t *state, intptr_t arg, int mode)
1300 {
1301 	hermon_reg_ioctl_t	wrreg;
1302 	uint32_t		*addr;
1303 	uintptr_t		baseaddr;
1304 	int			status;
1305 	ddi_acc_handle_t	handle;
1306 
1307 	/* initialize the FMA retry loop */
1308 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
1309 
1310 	/*
1311 	 * Access to Hermon registers is not allowed in "maintenance mode".
1312 	 * This is primarily because the device may not have BARs to access
1313 	 */
1314 	if (state->hs_operational_mode == HERMON_MAINTENANCE_MODE) {
1315 		return (EFAULT);
1316 	}
1317 
1318 	/* Copy in the hermon_reg_ioctl_t structure */
1319 	status = ddi_copyin((void *)arg, &wrreg, sizeof (hermon_reg_ioctl_t),
1320 	    mode);
1321 	if (status != 0) {
1322 		return (EFAULT);
1323 	}
1324 
1325 	/* Determine base address for requested register set */
1326 	switch (wrreg.arg_reg_set) {
1327 	case HERMON_CMD_BAR:
1328 		baseaddr = (uintptr_t)state->hs_reg_cmd_baseaddr;
1329 		handle = hermon_get_cmdhdl(state);
1330 		break;
1331 
1332 	case HERMON_UAR_BAR:
1333 		baseaddr = (uintptr_t)state->hs_reg_uar_baseaddr;
1334 		handle = hermon_get_uarhdl(state);
1335 		break;
1336 
1337 	default:
1338 		return (EINVAL);
1339 	}
1340 
1341 	/* Ensure that address is properly-aligned */
1342 	addr = (uint32_t *)((baseaddr + wrreg.arg_offset) & ~0x3);
1343 
1344 	/* the FMA retry loop starts. */
1345 	hermon_pio_start(state, handle, pio_error, fm_loop_cnt,
1346 	    fm_status, fm_test);
1347 
1348 	/* Write the data to the register pointed to by addr */
1349 	ddi_put32(handle, addr, wrreg.arg_data);
1350 
1351 	/* the FMA retry loop ends. */
1352 	hermon_pio_end(state, handle, pio_error, fm_loop_cnt, fm_status,
1353 	    fm_test);
1354 	return (0);
1355 
1356 pio_error:
1357 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
1358 	return (EIO);
1359 }
1360 #endif	/* DEBUG */
1361 
1362 static int
1363 hermon_ioctl_write_boot_addr(hermon_state_t *state, dev_t dev, intptr_t arg,
1364     int mode)
1365 {
1366 	hermon_flash_ioctl_t	ioctl_info;
1367 
1368 	/* initialize the FMA retry loop */
1369 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
1370 
1371 	/*
1372 	 * Check that flash init ioctl has been called first.  And check
1373 	 * that the same dev_t that called init is the one calling write now.
1374 	 */
1375 	mutex_enter(&state->hs_fw_flashlock);
1376 	if ((state->hs_fw_flashdev != dev) ||
1377 	    (state->hs_fw_flashstarted == 0)) {
1378 		mutex_exit(&state->hs_fw_flashlock);
1379 		return (EIO);
1380 	}
1381 
1382 	/* copy user struct to kernel */
1383 #ifdef _MULTI_DATAMODEL
1384 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
1385 		hermon_flash_ioctl32_t info32;
1386 
1387 		if (ddi_copyin((void *)arg, &info32,
1388 		    sizeof (hermon_flash_ioctl32_t), mode) != 0) {
1389 			mutex_exit(&state->hs_fw_flashlock);
1390 			return (EFAULT);
1391 		}
1392 		ioctl_info.af_type = info32.af_type;
1393 		ioctl_info.af_sector = (caddr_t)(uintptr_t)info32.af_sector;
1394 		ioctl_info.af_sector_num = info32.af_sector_num;
1395 		ioctl_info.af_addr = info32.af_addr;
1396 		ioctl_info.af_byte = info32.af_byte;
1397 	} else
1398 #endif /* _MULTI_DATAMODEL */
1399 	if (ddi_copyin((void *)arg, &ioctl_info,
1400 	    sizeof (hermon_flash_ioctl_t), mode) != 0) {
1401 		mutex_exit(&state->hs_fw_flashlock);
1402 		return (EFAULT);
1403 	}
1404 
1405 	switch (state->hs_fw_cmdset) {
1406 	case HERMON_FLASH_AMD_CMDSET:
1407 	case HERMON_FLASH_INTEL_CMDSET:
1408 		break;
1409 
1410 	case HERMON_FLASH_SPI_CMDSET:
1411 	{
1412 		ddi_acc_handle_t pci_hdl = hermon_get_pcihdl(state);
1413 
1414 		/* the FMA retry loop starts. */
1415 		hermon_pio_start(state, pci_hdl, pio_error,
1416 		    fm_loop_cnt, fm_status, fm_test);
1417 
1418 		hermon_flash_write_cfg(state, pci_hdl,
1419 		    HERMON_HW_FLASH_SPI_BOOT_ADDR_REG,
1420 		    (ioctl_info.af_addr << 8) | 0x06);
1421 
1422 		/* the FMA retry loop ends. */
1423 		hermon_pio_end(state, pci_hdl, pio_error,
1424 		    fm_loop_cnt, fm_status, fm_test);
1425 		break;
1426 	}
1427 
1428 	case HERMON_FLASH_UNKNOWN_CMDSET:
1429 	default:
1430 		mutex_exit(&state->hs_fw_flashlock);
1431 		return (EINVAL);
1432 	}
1433 	mutex_exit(&state->hs_fw_flashlock);
1434 	return (0);
1435 
1436 pio_error:
1437 	mutex_exit(&state->hs_fw_flashlock);
1438 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
1439 	return (EIO);
1440 }
1441 
1442 /*
1443  * hermon_flash_reset()
1444  */
1445 static int
1446 hermon_flash_reset(hermon_state_t *state)
1447 {
1448 	int status;
1449 
1450 	/*
1451 	 * Performs a reset to the flash device.  After a reset the flash will
1452 	 * be operating in normal mode (capable of read/write, etc.).
1453 	 */
1454 	switch (state->hs_fw_cmdset) {
1455 	case HERMON_FLASH_AMD_CMDSET:
1456 		hermon_flash_write(state, 0x555, HERMON_HW_FLASH_RESET_AMD,
1457 		    &status);
1458 		if (status != 0) {
1459 			return (status);
1460 		}
1461 		break;
1462 
1463 	case HERMON_FLASH_INTEL_CMDSET:
1464 		hermon_flash_write(state, 0x555, HERMON_HW_FLASH_RESET_INTEL,
1465 		    &status);
1466 		if (status != 0) {
1467 			return (status);
1468 		}
1469 		break;
1470 
1471 	/* It appears no reset is needed for SPI */
1472 	case HERMON_FLASH_SPI_CMDSET:
1473 		status = 0;
1474 		break;
1475 
1476 	case HERMON_FLASH_UNKNOWN_CMDSET:
1477 	default:
1478 		status = EINVAL;
1479 		break;
1480 	}
1481 	return (status);
1482 }
1483 
1484 /*
1485  * hermon_flash_read_sector()
1486  */
1487 static int
1488 hermon_flash_read_sector(hermon_state_t *state, uint32_t sector_num)
1489 {
1490 	uint32_t addr;
1491 	uint32_t end_addr;
1492 	uint32_t *image;
1493 	int i, status;
1494 
1495 	image = (uint32_t *)&state->hs_fw_sector[0];
1496 
1497 	/*
1498 	 * Calculate the start and end address of the sector, based on the
1499 	 * sector number passed in.
1500 	 */
1501 	addr = sector_num << state->hs_fw_log_sector_sz;
1502 	end_addr = addr + (1 << state->hs_fw_log_sector_sz);
1503 
1504 	/* Set the flash bank correctly for the given address */
1505 	if ((status = hermon_flash_bank(state, addr)) != 0)
1506 		return (status);
1507 
1508 	/* Read the entire sector, one quadlet at a time */
1509 	for (i = 0; addr < end_addr; i++, addr += 4) {
1510 		image[i] = hermon_flash_read(state, addr, &status);
1511 		if (status != 0) {
1512 			return (status);
1513 		}
1514 	}
1515 	return (0);
1516 }
1517 
1518 /*
1519  * hermon_flash_read_quadlet()
1520  */
1521 static int
1522 hermon_flash_read_quadlet(hermon_state_t *state, uint32_t *data,
1523     uint32_t addr)
1524 {
1525 	int status;
1526 
1527 	/* Set the flash bank correctly for the given address */
1528 	if ((status = hermon_flash_bank(state, addr)) != 0) {
1529 		return (status);
1530 	}
1531 
1532 	/* Read one quadlet of data */
1533 	*data = hermon_flash_read(state, addr, &status);
1534 	if (status != 0) {
1535 		return (EIO);
1536 	}
1537 
1538 	return (0);
1539 }
1540 
1541 /*
1542  * hermon_flash_write_sector()
1543  */
1544 static int
1545 hermon_flash_write_sector(hermon_state_t *state, uint32_t sector_num)
1546 {
1547 	uint32_t	addr;
1548 	uint32_t	end_addr;
1549 	uint32_t	*databuf;
1550 	uchar_t		*sector;
1551 	int		status = 0;
1552 	int		i;
1553 
1554 	sector = (uchar_t *)&state->hs_fw_sector[0];
1555 
1556 	/*
1557 	 * Calculate the start and end address of the sector, based on the
1558 	 * sector number passed in.
1559 	 */
1560 	addr = sector_num << state->hs_fw_log_sector_sz;
1561 	end_addr = addr + (1 << state->hs_fw_log_sector_sz);
1562 
1563 	/* Set the flash bank correctly for the given address */
1564 	if ((status = hermon_flash_bank(state, addr)) != 0 ||
1565 	    (status = hermon_flash_reset(state)) != 0) {
1566 		return (status);
1567 	}
1568 
1569 	/* Erase the sector before writing */
1570 	status = hermon_flash_erase_sector(state, sector_num);
1571 	if (status != 0) {
1572 		return (status);
1573 	}
1574 
1575 	switch (state->hs_fw_cmdset) {
1576 	case HERMON_FLASH_SPI_CMDSET:
1577 		databuf = (uint32_t *)(void *)sector;
1578 		/* Write the sector, one dword at a time */
1579 		for (i = 0; addr < end_addr; i++, addr += 4) {
1580 			if ((status = hermon_flash_spi_write_dword(state, addr,
1581 			    htonl(databuf[i]))) != 0) {
1582 				return (status);
1583 			}
1584 		}
1585 		status = hermon_flash_reset(state);
1586 		break;
1587 
1588 	case HERMON_FLASH_INTEL_CMDSET:
1589 	case HERMON_FLASH_AMD_CMDSET:
1590 		/* Write the sector, one byte at a time */
1591 		for (i = 0; addr < end_addr; i++, addr++) {
1592 			status = hermon_flash_write_byte(state, addr,
1593 			    sector[i]);
1594 			if (status != 0) {
1595 				break;
1596 			}
1597 		}
1598 		status = hermon_flash_reset(state);
1599 		break;
1600 
1601 	case HERMON_FLASH_UNKNOWN_CMDSET:
1602 	default:
1603 		status = EINVAL;
1604 		break;
1605 	}
1606 
1607 	return (status);
1608 }
1609 
1610 /*
1611  * hermon_flash_spi_write_dword()
1612  *
1613  * NOTE: This function assumes that "data" is in network byte order.
1614  *
1615  */
1616 static int
1617 hermon_flash_spi_write_dword(hermon_state_t *state, uint32_t addr,
1618     uint32_t data)
1619 {
1620 	int status;
1621 	ddi_acc_handle_t	hdl;
1622 
1623 	/* initialize the FMA retry loop */
1624 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
1625 
1626 	hdl = hermon_get_pcihdl(state);
1627 
1628 	/* the FMA retry loop starts. */
1629 	hermon_pio_start(state, hdl, pio_error, fm_loop_cnt, fm_status,
1630 	    fm_test);
1631 
1632 	/* Issue Write Enable */
1633 	hermon_flash_spi_write_enable(state);
1634 
1635 	/* Set the Address */
1636 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_SPI_ADDR,
1637 	    addr & HERMON_HW_FLASH_SPI_ADDR_MASK);
1638 
1639 	/* Set the Data */
1640 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_SPI_DATA, data);
1641 
1642 	/* Set the Page Program and execute */
1643 	hermon_flash_spi_exec_command(state, hdl,
1644 	    HERMON_HW_FLASH_SPI_INSTR_PHASE_OFF |
1645 	    HERMON_HW_FLASH_SPI_ADDR_PHASE_OFF |
1646 	    HERMON_HW_FLASH_SPI_DATA_PHASE_OFF |
1647 	    HERMON_HW_FLASH_SPI_TRANS_SZ_4B |
1648 	    (HERMON_HW_FLASH_SPI_PAGE_PROGRAM <<
1649 	    HERMON_HW_FLASH_SPI_INSTR_SHIFT));
1650 
1651 	/* Wait for write to complete */
1652 	if ((status = hermon_flash_spi_wait_wip(state)) != 0) {
1653 		return (status);
1654 	}
1655 
1656 	/* the FMA retry loop ends. */
1657 	hermon_pio_end(state, hdl, pio_error, fm_loop_cnt, fm_status, fm_test);
1658 	return (0);
1659 
1660 pio_error:
1661 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
1662 	return (EIO);
1663 }
1664 
1665 /*
1666  * hermon_flash_write_byte()
1667  */
1668 static int
1669 hermon_flash_write_byte(hermon_state_t *state, uint32_t addr, uchar_t data)
1670 {
1671 	uint32_t stat;
1672 	int status = 0;
1673 	int dword_addr;
1674 	int byte_offset;
1675 	int i;
1676 	union {
1677 		uint8_t		bytes[4];
1678 		uint32_t	dword;
1679 	} dword;
1680 
1681 	switch (state->hs_fw_cmdset) {
1682 	case HERMON_FLASH_AMD_CMDSET:
1683 		/* Issue Flash Byte program command */
1684 		hermon_flash_write(state, addr, 0xAA, &status);
1685 		if (status != 0) {
1686 			return (status);
1687 		}
1688 
1689 		hermon_flash_write(state, addr, 0x55, &status);
1690 		if (status != 0) {
1691 			return (status);
1692 		}
1693 
1694 		hermon_flash_write(state, addr, 0xA0, &status);
1695 		if (status != 0) {
1696 			return (status);
1697 		}
1698 
1699 		hermon_flash_write(state, addr, data, &status);
1700 		if (status != 0) {
1701 			return (status);
1702 		}
1703 
1704 		/* Wait for Write Byte to Complete */
1705 		i = 0;
1706 		do {
1707 			drv_usecwait(1);
1708 			stat = hermon_flash_read(state, addr & ~3, &status);
1709 			if (status != 0) {
1710 				return (status);
1711 			}
1712 
1713 			if (i == hermon_hw_flash_timeout_write) {
1714 				cmn_err(CE_WARN,
1715 				    "hermon_flash_write_byte: ACS write "
1716 				    "timeout: addr: 0x%x, data: 0x%x\n",
1717 				    addr, data);
1718 				hermon_fm_ereport(state, HCA_SYS_ERR,
1719 				    HCA_ERR_IOCTL);
1720 				return (EIO);
1721 			}
1722 			i++;
1723 		} while (data != ((stat >> ((3 - (addr & 3)) << 3)) & 0xFF));
1724 
1725 		break;
1726 
1727 	case HERMON_FLASH_INTEL_CMDSET:
1728 		/* Issue Flash Byte program command */
1729 		hermon_flash_write(state, addr, HERMON_HW_FLASH_ICS_WRITE,
1730 		    &status);
1731 		if (status != 0) {
1732 			return (status);
1733 		}
1734 		hermon_flash_write(state, addr, data, &status);
1735 		if (status != 0) {
1736 			return (status);
1737 		}
1738 
1739 		/* Wait for Write Byte to Complete */
1740 		i = 0;
1741 		do {
1742 			drv_usecwait(1);
1743 			stat = hermon_flash_read(state, addr & ~3, &status);
1744 			if (status != 0) {
1745 				return (status);
1746 			}
1747 
1748 			if (i == hermon_hw_flash_timeout_write) {
1749 				cmn_err(CE_WARN,
1750 				    "hermon_flash_write_byte: ICS write "
1751 				    "timeout: addr: %x, data: %x\n",
1752 				    addr, data);
1753 				hermon_fm_ereport(state, HCA_SYS_ERR,
1754 				    HCA_ERR_IOCTL);
1755 				return (EIO);
1756 			}
1757 			i++;
1758 		} while ((stat & HERMON_HW_FLASH_ICS_READY) == 0);
1759 
1760 		if (stat & HERMON_HW_FLASH_ICS_ERROR) {
1761 			cmn_err(CE_WARN,
1762 			    "hermon_flash_write_byte: ICS write cmd error: "
1763 			    "addr: %x, data: %x\n",
1764 			    addr, data);
1765 			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
1766 			return (EIO);
1767 		}
1768 		break;
1769 
1770 	case HERMON_FLASH_SPI_CMDSET:
1771 		/*
1772 		 * Our lowest write granularity on SPI is a dword.
1773 		 * To support this ioctl option, we can read in the
1774 		 * dword that contains this byte, modify this byte,
1775 		 * and write the dword back out.
1776 		 */
1777 
1778 		/* Determine dword offset and byte offset within the dword */
1779 		byte_offset = addr & 3;
1780 		dword_addr = addr - byte_offset;
1781 #ifdef _LITTLE_ENDIAN
1782 		byte_offset = 3 - byte_offset;
1783 #endif
1784 
1785 		/* Read in dword */
1786 		if ((status = hermon_flash_read_quadlet(state, &dword.dword,
1787 		    dword_addr)) != 0)
1788 			break;
1789 
1790 		/* Set "data" to the appopriate byte */
1791 		dword.bytes[byte_offset] = data;
1792 
1793 		/* Write modified dword back out */
1794 		status = hermon_flash_spi_write_dword(state, dword_addr,
1795 		    dword.dword);
1796 
1797 		break;
1798 
1799 	case HERMON_FLASH_UNKNOWN_CMDSET:
1800 	default:
1801 		cmn_err(CE_WARN,
1802 		    "hermon_flash_write_byte: unknown cmd set: 0x%x\n",
1803 		    state->hs_fw_cmdset);
1804 		status = EINVAL;
1805 		break;
1806 	}
1807 
1808 	return (status);
1809 }
1810 
1811 /*
1812  * hermon_flash_erase_sector()
1813  */
1814 static int
1815 hermon_flash_erase_sector(hermon_state_t *state, uint32_t sector_num)
1816 {
1817 	ddi_acc_handle_t	hdl;
1818 	uint32_t addr;
1819 	uint32_t stat;
1820 	int status = 0;
1821 	int i;
1822 
1823 	/* initialize the FMA retry loop */
1824 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
1825 
1826 	/* Get address from sector num */
1827 	addr = sector_num << state->hs_fw_log_sector_sz;
1828 
1829 	switch (state->hs_fw_cmdset) {
1830 	case HERMON_FLASH_AMD_CMDSET:
1831 		/* Issue Flash Sector Erase Command */
1832 		hermon_flash_write(state, addr, 0xAA, &status);
1833 		if (status != 0) {
1834 			return (status);
1835 		}
1836 
1837 		hermon_flash_write(state, addr, 0x55, &status);
1838 		if (status != 0) {
1839 			return (status);
1840 		}
1841 
1842 		hermon_flash_write(state, addr, 0x80, &status);
1843 		if (status != 0) {
1844 			return (status);
1845 		}
1846 
1847 		hermon_flash_write(state, addr, 0xAA, &status);
1848 		if (status != 0) {
1849 			return (status);
1850 		}
1851 
1852 		hermon_flash_write(state, addr, 0x55, &status);
1853 		if (status != 0) {
1854 			return (status);
1855 		}
1856 
1857 		hermon_flash_write(state, addr, 0x30, &status);
1858 		if (status != 0) {
1859 			return (status);
1860 		}
1861 
1862 		/* Wait for Sector Erase to complete */
1863 		i = 0;
1864 		do {
1865 			drv_usecwait(1);
1866 			stat = hermon_flash_read(state, addr, &status);
1867 			if (status != 0) {
1868 				return (status);
1869 			}
1870 
1871 			if (i == hermon_hw_flash_timeout_erase) {
1872 				cmn_err(CE_WARN,
1873 				    "hermon_flash_erase_sector: "
1874 				    "ACS erase timeout\n");
1875 				hermon_fm_ereport(state, HCA_SYS_ERR,
1876 				    HCA_ERR_IOCTL);
1877 				return (EIO);
1878 			}
1879 			i++;
1880 		} while (stat != 0xFFFFFFFF);
1881 		break;
1882 
1883 	case HERMON_FLASH_INTEL_CMDSET:
1884 		/* Issue Flash Sector Erase Command */
1885 		hermon_flash_write(state, addr, HERMON_HW_FLASH_ICS_ERASE,
1886 		    &status);
1887 		if (status != 0) {
1888 			return (status);
1889 		}
1890 
1891 		hermon_flash_write(state, addr, HERMON_HW_FLASH_ICS_CONFIRM,
1892 		    &status);
1893 		if (status != 0) {
1894 			return (status);
1895 		}
1896 
1897 		/* Wait for Sector Erase to complete */
1898 		i = 0;
1899 		do {
1900 			drv_usecwait(1);
1901 			stat = hermon_flash_read(state, addr & ~3, &status);
1902 			if (status != 0) {
1903 				return (status);
1904 			}
1905 
1906 			if (i == hermon_hw_flash_timeout_erase) {
1907 				cmn_err(CE_WARN,
1908 				    "hermon_flash_erase_sector: "
1909 				    "ICS erase timeout\n");
1910 				hermon_fm_ereport(state, HCA_SYS_ERR,
1911 				    HCA_ERR_IOCTL);
1912 				return (EIO);
1913 			}
1914 			i++;
1915 		} while ((stat & HERMON_HW_FLASH_ICS_READY) == 0);
1916 
1917 		if (stat & HERMON_HW_FLASH_ICS_ERROR) {
1918 			cmn_err(CE_WARN,
1919 			    "hermon_flash_erase_sector: "
1920 			    "ICS erase cmd error\n");
1921 			hermon_fm_ereport(state, HCA_SYS_ERR,
1922 			    HCA_ERR_IOCTL);
1923 			return (EIO);
1924 		}
1925 		break;
1926 
1927 	case HERMON_FLASH_SPI_CMDSET:
1928 		hdl = hermon_get_pcihdl(state);
1929 
1930 		/* the FMA retry loop starts. */
1931 		hermon_pio_start(state, hdl, pio_error, fm_loop_cnt, fm_status,
1932 		    fm_test);
1933 
1934 		/* Issue Write Enable */
1935 		hermon_flash_spi_write_enable(state);
1936 
1937 		/* Set the Address */
1938 		hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_SPI_ADDR,
1939 		    addr & HERMON_HW_FLASH_SPI_ADDR_MASK);
1940 
1941 		/* Issue Flash Sector Erase */
1942 		hermon_flash_spi_exec_command(state, hdl,
1943 		    HERMON_HW_FLASH_SPI_INSTR_PHASE_OFF |
1944 		    HERMON_HW_FLASH_SPI_ADDR_PHASE_OFF |
1945 		    ((uint32_t)(HERMON_HW_FLASH_SPI_SECTOR_ERASE) <<
1946 		    HERMON_HW_FLASH_SPI_INSTR_SHIFT));
1947 
1948 		/* the FMA retry loop ends. */
1949 		hermon_pio_end(state, hdl, pio_error, fm_loop_cnt, fm_status,
1950 		    fm_test);
1951 
1952 		/* Wait for Sector Erase to complete */
1953 		status = hermon_flash_spi_wait_wip(state);
1954 		break;
1955 
1956 	case HERMON_FLASH_UNKNOWN_CMDSET:
1957 	default:
1958 		cmn_err(CE_WARN,
1959 		    "hermon_flash_erase_sector: unknown cmd set: 0x%x\n",
1960 		    state->hs_fw_cmdset);
1961 		status = EINVAL;
1962 		break;
1963 	}
1964 
1965 	/* Reset the flash device */
1966 	if (status == 0) {
1967 		status = hermon_flash_reset(state);
1968 	}
1969 	return (status);
1970 
1971 pio_error:
1972 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
1973 	return (EIO);
1974 }
1975 
1976 /*
1977  * hermon_flash_erase_chip()
1978  */
1979 static int
1980 hermon_flash_erase_chip(hermon_state_t *state)
1981 {
1982 	uint32_t stat;
1983 	uint_t size;
1984 	int status = 0;
1985 	int i;
1986 	int num_sect;
1987 
1988 	switch (state->hs_fw_cmdset) {
1989 	case HERMON_FLASH_AMD_CMDSET:
1990 		/* Issue Flash Chip Erase Command */
1991 		hermon_flash_write(state, 0, 0xAA, &status);
1992 		if (status != 0) {
1993 			return (status);
1994 		}
1995 
1996 		hermon_flash_write(state, 0, 0x55, &status);
1997 		if (status != 0) {
1998 			return (status);
1999 		}
2000 
2001 		hermon_flash_write(state, 0, 0x80, &status);
2002 		if (status != 0) {
2003 			return (status);
2004 		}
2005 
2006 		hermon_flash_write(state, 0, 0xAA, &status);
2007 		if (status != 0) {
2008 			return (status);
2009 		}
2010 
2011 		hermon_flash_write(state, 0, 0x55, &status);
2012 		if (status != 0) {
2013 			return (status);
2014 		}
2015 
2016 		hermon_flash_write(state, 0, 0x10, &status);
2017 		if (status != 0) {
2018 			return (status);
2019 		}
2020 
2021 		/* Wait for Chip Erase to Complete */
2022 		i = 0;
2023 		do {
2024 			drv_usecwait(1);
2025 			stat = hermon_flash_read(state, 0, &status);
2026 			if (status != 0) {
2027 				return (status);
2028 			}
2029 
2030 			if (i == hermon_hw_flash_timeout_erase) {
2031 				cmn_err(CE_WARN,
2032 				    "hermon_flash_erase_chip: erase timeout\n");
2033 				hermon_fm_ereport(state, HCA_SYS_ERR,
2034 				    HCA_ERR_IOCTL);
2035 				return (EIO);
2036 			}
2037 			i++;
2038 		} while (stat != 0xFFFFFFFF);
2039 		break;
2040 
2041 	case HERMON_FLASH_INTEL_CMDSET:
2042 	case HERMON_FLASH_SPI_CMDSET:
2043 		/*
2044 		 * These chips don't have a chip erase command, so erase
2045 		 * all blocks one at a time.
2046 		 */
2047 		size = (0x1 << state->hs_fw_log_sector_sz);
2048 		num_sect = state->hs_fw_device_sz / size;
2049 
2050 		for (i = 0; i < num_sect; i++) {
2051 			status = hermon_flash_erase_sector(state, i);
2052 			if (status != 0) {
2053 				cmn_err(CE_WARN,
2054 				    "hermon_flash_erase_chip: "
2055 				    "sector %d erase error\n", i);
2056 				return (status);
2057 			}
2058 		}
2059 		break;
2060 
2061 	case HERMON_FLASH_UNKNOWN_CMDSET:
2062 	default:
2063 		cmn_err(CE_WARN, "hermon_flash_erase_chip: "
2064 		    "unknown cmd set: 0x%x\n", state->hs_fw_cmdset);
2065 		status = EINVAL;
2066 		break;
2067 	}
2068 
2069 	return (status);
2070 }
2071 
2072 /*
2073  * hermon_flash_spi_write_enable()
2074  */
2075 static void
2076 hermon_flash_spi_write_enable(hermon_state_t *state)
2077 {
2078 	ddi_acc_handle_t	hdl;
2079 
2080 	hdl = hermon_get_pcihdl(state);
2081 
2082 	hermon_flash_spi_exec_command(state, hdl,
2083 	    HERMON_HW_FLASH_SPI_INSTR_PHASE_OFF |
2084 	    (HERMON_HW_FLASH_SPI_WRITE_ENABLE <<
2085 	    HERMON_HW_FLASH_SPI_INSTR_SHIFT));
2086 }
2087 
2088 /*
2089  * hermon_flash_spi_wait_wip()
2090  */
2091 static int
2092 hermon_flash_spi_wait_wip(hermon_state_t *state)
2093 {
2094 	ddi_acc_handle_t	hdl;
2095 	uint32_t		status;
2096 
2097 	/* initialize the FMA retry loop */
2098 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
2099 
2100 	hdl = hermon_get_pcihdl(state);
2101 
2102 	/* the FMA retry loop starts. */
2103 	hermon_pio_start(state, hdl, pio_error, fm_loop_cnt, fm_status,
2104 	    fm_test);
2105 
2106 	/* wait on the gateway to clear busy */
2107 	do {
2108 		status = hermon_flash_read_cfg(state, hdl,
2109 		    HERMON_HW_FLASH_SPI_GW);
2110 	} while (status & HERMON_HW_FLASH_SPI_BUSY);
2111 
2112 	/* now, get the status and check for WIP to clear */
2113 	do {
2114 		hermon_flash_spi_exec_command(state, hdl,
2115 		    HERMON_HW_FLASH_SPI_READ_OP |
2116 		    HERMON_HW_FLASH_SPI_INSTR_PHASE_OFF |
2117 		    HERMON_HW_FLASH_SPI_DATA_PHASE_OFF |
2118 		    HERMON_HW_FLASH_SPI_TRANS_SZ_4B |
2119 		    (HERMON_HW_FLASH_SPI_READ_STATUS_REG <<
2120 		    HERMON_HW_FLASH_SPI_INSTR_SHIFT));
2121 
2122 		status = hermon_flash_read_cfg(state, hdl,
2123 		    HERMON_HW_FLASH_SPI_DATA);
2124 	} while (status & HERMON_HW_FLASH_SPI_WIP);
2125 
2126 	/* the FMA retry loop ends. */
2127 	hermon_pio_end(state, hdl, pio_error, fm_loop_cnt, fm_status, fm_test);
2128 	return (0);
2129 
2130 pio_error:
2131 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
2132 	return (EIO);
2133 }
2134 
2135 /*
2136  * hermon_flash_bank()
2137  */
2138 static int
2139 hermon_flash_bank(hermon_state_t *state, uint32_t addr)
2140 {
2141 	ddi_acc_handle_t	hdl;
2142 	uint32_t		bank;
2143 
2144 	/* initialize the FMA retry loop */
2145 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
2146 
2147 	/* Set handle */
2148 	hdl = hermon_get_pcihdl(state);
2149 
2150 	/* Determine the bank setting from the address */
2151 	bank = addr & HERMON_HW_FLASH_BANK_MASK;
2152 
2153 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(state->hs_fw_flashbank))
2154 
2155 	/*
2156 	 * If the bank is different from the currently set bank, we need to
2157 	 * change it.  Also, if an 'addr' of 0 is given, this allows the
2158 	 * capability to force the flash bank to 0.  This is useful at init
2159 	 * time to initially set the bank value
2160 	 */
2161 	if (state->hs_fw_flashbank != bank || addr == 0) {
2162 		switch (state->hs_fw_cmdset) {
2163 		case HERMON_FLASH_SPI_CMDSET:
2164 			/* CMJ: not needed for hermon */
2165 			break;
2166 
2167 		case HERMON_FLASH_INTEL_CMDSET:
2168 		case HERMON_FLASH_AMD_CMDSET:
2169 			/* the FMA retry loop starts. */
2170 			hermon_pio_start(state, hdl, pio_error, fm_loop_cnt,
2171 			    fm_status, fm_test);
2172 
2173 			hermon_flash_write_cfg(state, hdl,
2174 			    HERMON_HW_FLASH_GPIO_DATACLEAR, 0x70);
2175 			hermon_flash_write_cfg(state, hdl,
2176 			    HERMON_HW_FLASH_GPIO_DATASET, (bank >> 15) & 0x70);
2177 
2178 			/* the FMA retry loop ends. */
2179 			hermon_pio_end(state, hdl, pio_error, fm_loop_cnt,
2180 			    fm_status, fm_test);
2181 			break;
2182 
2183 		case HERMON_FLASH_UNKNOWN_CMDSET:
2184 		default:
2185 			return (EINVAL);
2186 		}
2187 
2188 		state->hs_fw_flashbank = bank;
2189 	}
2190 	return (0);
2191 
2192 pio_error:
2193 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
2194 	return (EIO);
2195 }
2196 
2197 /*
2198  * hermon_flash_spi_exec_command()
2199  */
2200 static void
2201 hermon_flash_spi_exec_command(hermon_state_t *state, ddi_acc_handle_t hdl,
2202     uint32_t cmd)
2203 {
2204 	uint32_t data;
2205 	int timeout = 0;
2206 
2207 	cmd |= HERMON_HW_FLASH_SPI_BUSY | HERMON_HW_FLASH_SPI_ENABLE_OFF;
2208 
2209 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_SPI_GW, cmd);
2210 
2211 	do {
2212 		data = hermon_flash_read_cfg(state, hdl,
2213 		    HERMON_HW_FLASH_SPI_GW);
2214 		timeout++;
2215 	} while ((data & HERMON_HW_FLASH_SPI_BUSY) &&
2216 	    (timeout < hermon_hw_flash_timeout_config));
2217 }
2218 
2219 /*
2220  * hermon_flash_read()
2221  */
2222 static uint32_t
2223 hermon_flash_read(hermon_state_t *state, uint32_t addr, int *err)
2224 {
2225 	ddi_acc_handle_t	hdl;
2226 	uint32_t		data = 0;
2227 	int			timeout, status = 0;
2228 
2229 	/* initialize the FMA retry loop */
2230 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
2231 
2232 	hdl = hermon_get_pcihdl(state);
2233 
2234 	/* the FMA retry loop starts. */
2235 	hermon_pio_start(state, hdl, pio_error, fm_loop_cnt, fm_status,
2236 	    fm_test);
2237 
2238 	switch (state->hs_fw_cmdset) {
2239 	case HERMON_FLASH_SPI_CMDSET:
2240 		/* Set the transaction address */
2241 		hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_SPI_ADDR,
2242 		    (addr & HERMON_HW_FLASH_SPI_ADDR_MASK));
2243 
2244 		hermon_flash_spi_exec_command(state, hdl,
2245 		    HERMON_HW_FLASH_SPI_READ_OP |
2246 		    HERMON_HW_FLASH_SPI_INSTR_PHASE_OFF |
2247 		    HERMON_HW_FLASH_SPI_ADDR_PHASE_OFF |
2248 		    HERMON_HW_FLASH_SPI_DATA_PHASE_OFF |
2249 		    HERMON_HW_FLASH_SPI_TRANS_SZ_4B |
2250 		    (HERMON_HW_FLASH_SPI_READ <<
2251 		    HERMON_HW_FLASH_SPI_INSTR_SHIFT));
2252 
2253 		data = hermon_flash_read_cfg(state, hdl,
2254 		    HERMON_HW_FLASH_SPI_DATA);
2255 		break;
2256 
2257 	case HERMON_FLASH_INTEL_CMDSET:
2258 	case HERMON_FLASH_AMD_CMDSET:
2259 		/*
2260 		 * The Read operation does the following:
2261 		 *   1) Write the masked address to the HERMON_FLASH_ADDR
2262 		 *	register. Only the least significant 19 bits are valid.
2263 		 *   2) Read back the register until the command has completed.
2264 		 *   3) Read the data retrieved from the address at the
2265 		 *	HERMON_FLASH_DATA register.
2266 		 */
2267 		hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_ADDR,
2268 		    (addr & HERMON_HW_FLASH_ADDR_MASK) | (1 << 29));
2269 
2270 		timeout = 0;
2271 		do {
2272 			data = hermon_flash_read_cfg(state, hdl,
2273 			    HERMON_HW_FLASH_ADDR);
2274 			timeout++;
2275 		} while ((data & HERMON_HW_FLASH_CMD_MASK) &&
2276 		    (timeout < hermon_hw_flash_timeout_config));
2277 
2278 		if (timeout == hermon_hw_flash_timeout_config) {
2279 			cmn_err(CE_WARN, "hermon_flash_read: command timed "
2280 			    "out.\n");
2281 			*err = EIO;
2282 			hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
2283 			return (data);
2284 		}
2285 
2286 		data = hermon_flash_read_cfg(state, hdl, HERMON_HW_FLASH_DATA);
2287 		break;
2288 
2289 	case HERMON_FLASH_UNKNOWN_CMDSET:
2290 	default:
2291 		cmn_err(CE_CONT, "hermon_flash_read: unknown cmdset: 0x%x\n",
2292 		    state->hs_fw_cmdset);
2293 		status = EINVAL;
2294 		break;
2295 	}
2296 
2297 
2298 	/* the FMA retry loop ends. */
2299 	hermon_pio_end(state, hdl, pio_error, fm_loop_cnt, fm_status, fm_test);
2300 	*err = status;
2301 	return (data);
2302 
2303 pio_error:
2304 	*err = EIO;
2305 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
2306 	return (data);
2307 }
2308 
2309 /*
2310  * hermon_flash_write()
2311  */
2312 static void
2313 hermon_flash_write(hermon_state_t *state, uint32_t addr, uchar_t data, int *err)
2314 {
2315 	ddi_acc_handle_t	hdl;
2316 	int			cmd;
2317 	int			timeout;
2318 
2319 	/* initialize the FMA retry loop */
2320 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
2321 
2322 	hdl = hermon_get_pcihdl(state);
2323 
2324 	/* the FMA retry loop starts. */
2325 	hermon_pio_start(state, hdl, pio_error, fm_loop_cnt, fm_status,
2326 	    fm_test);
2327 
2328 	/*
2329 	 * The Write operation does the following:
2330 	 *   1) Write the data to be written to the HERMON_FLASH_DATA offset.
2331 	 *   2) Write the address to write the data to to the HERMON_FLASH_ADDR
2332 	 *	offset.
2333 	 *   3) Wait until the write completes.
2334 	 */
2335 
2336 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_DATA, data << 24);
2337 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_ADDR,
2338 	    (addr & 0x7FFFF) | (2 << 29));
2339 
2340 	timeout = 0;
2341 	do {
2342 		cmd = hermon_flash_read_cfg(state, hdl, HERMON_HW_FLASH_ADDR);
2343 		timeout++;
2344 	} while ((cmd & HERMON_HW_FLASH_CMD_MASK) &&
2345 	    (timeout < hermon_hw_flash_timeout_config));
2346 
2347 	if (timeout == hermon_hw_flash_timeout_config) {
2348 		cmn_err(CE_WARN, "hermon_flash_write: config cmd timeout.\n");
2349 		*err = EIO;
2350 		hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
2351 		return;
2352 	}
2353 
2354 	/* the FMA retry loop ends. */
2355 	hermon_pio_end(state, hdl, pio_error, fm_loop_cnt, fm_status, fm_test);
2356 	*err = 0;
2357 	return;
2358 
2359 pio_error:
2360 	*err = EIO;
2361 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
2362 }
2363 
2364 /*
2365  * hermon_flash_init()
2366  */
2367 static int
2368 hermon_flash_init(hermon_state_t *state)
2369 {
2370 	uint32_t		word;
2371 	ddi_acc_handle_t	hdl;
2372 	int			sema_cnt;
2373 	int			gpio;
2374 
2375 	/* initialize the FMA retry loop */
2376 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
2377 
2378 	/* Set handle */
2379 	hdl = hermon_get_pcihdl(state);
2380 
2381 	/* the FMA retry loop starts. */
2382 	hermon_pio_start(state, hdl, pio_error, fm_loop_cnt, fm_status,
2383 	    fm_test);
2384 
2385 	/* Init the flash */
2386 
2387 #ifdef DO_WRCONF
2388 	/*
2389 	 * Grab the WRCONF semaphore.
2390 	 */
2391 	word = hermon_flash_read_cfg(state, hdl, HERMON_HW_FLASH_WRCONF_SEMA);
2392 #endif
2393 
2394 	/*
2395 	 * Grab the GPIO semaphore.  This allows us exclusive access to the
2396 	 * GPIO settings on the Hermon for the duration of the flash burning
2397 	 * procedure.
2398 	 */
2399 	sema_cnt = 0;
2400 	do {
2401 		word = hermon_flash_read_cfg(state, hdl,
2402 		    HERMON_HW_FLASH_GPIO_SEMA);
2403 		if (word == 0) {
2404 			break;
2405 		}
2406 
2407 		sema_cnt++;
2408 		drv_usecwait(1);
2409 
2410 	} while (sema_cnt < hermon_hw_flash_timeout_gpio_sema);
2411 
2412 	/*
2413 	 * Determine if we timed out trying to grab the GPIO semaphore
2414 	 */
2415 	if (sema_cnt == hermon_hw_flash_timeout_gpio_sema) {
2416 		cmn_err(CE_WARN, "hermon_flash_init: GPIO SEMA timeout\n");
2417 		cmn_err(CE_WARN, "GPIO_SEMA value: 0x%x\n", word);
2418 		hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
2419 		return (EIO);
2420 	}
2421 
2422 	/* Save away original GPIO Values */
2423 	state->hs_fw_gpio[0] = hermon_flash_read_cfg(state, hdl,
2424 	    HERMON_HW_FLASH_GPIO_DATA);
2425 
2426 	/* Set new GPIO value */
2427 	gpio = state->hs_fw_gpio[0] | HERMON_HW_FLASH_GPIO_PIN_ENABLE;
2428 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_GPIO_DATA, gpio);
2429 
2430 	/* Save away original GPIO Values */
2431 	state->hs_fw_gpio[1] = hermon_flash_read_cfg(state, hdl,
2432 	    HERMON_HW_FLASH_GPIO_MOD0);
2433 	state->hs_fw_gpio[2] = hermon_flash_read_cfg(state, hdl,
2434 	    HERMON_HW_FLASH_GPIO_MOD1);
2435 
2436 	/* unlock GPIO */
2437 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_GPIO_LOCK,
2438 	    HERMON_HW_FLASH_GPIO_UNLOCK_VAL);
2439 
2440 	/*
2441 	 * Set new GPIO values
2442 	 */
2443 	gpio = state->hs_fw_gpio[1] | HERMON_HW_FLASH_GPIO_PIN_ENABLE;
2444 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_GPIO_MOD0, gpio);
2445 
2446 	gpio = state->hs_fw_gpio[2] & ~HERMON_HW_FLASH_GPIO_PIN_ENABLE;
2447 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_GPIO_MOD1, gpio);
2448 
2449 	/* re-lock GPIO */
2450 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_GPIO_LOCK, 0);
2451 
2452 	/* Set CPUMODE to enable hermon to access the flash device */
2453 	/* CMJ This code came from arbel.  Hermon doesn't seem to need it. */
2454 	/*
2455 	 *	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_CPUMODE,
2456 	 *	    1 << HERMON_HW_FLASH_CPU_SHIFT);
2457 	 */
2458 
2459 	/* the FMA retry loop ends. */
2460 	hermon_pio_end(state, hdl, pio_error, fm_loop_cnt, fm_status, fm_test);
2461 	return (0);
2462 
2463 pio_error:
2464 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
2465 	return (EIO);
2466 }
2467 
2468 /*
2469  * hermon_flash_cfi_init
2470  *   Implements access to the CFI (Common Flash Interface) data
2471  */
2472 static int
2473 hermon_flash_cfi_init(hermon_state_t *state, uint32_t *cfi_info,
2474     int *intel_xcmd)
2475 {
2476 	uint32_t	data;
2477 	uint32_t	sector_sz_bytes;
2478 	uint32_t	bit_count;
2479 	uint8_t		cfi_ch_info[HERMON_CFI_INFO_SIZE];
2480 	uint32_t	cfi_dw_info[HERMON_CFI_INFO_QSIZE];
2481 	int		i;
2482 	int		status;
2483 
2484 	/* Right now, all hermon cards use SPI. */
2485 	if (HERMON_IS_MAINTENANCE_MODE(state->hs_dip) ||
2486 	    HERMON_IS_HCA_MODE(state->hs_dip)) {
2487 		/*
2488 		 * Don't use CFI for SPI part. Just fill in what we need
2489 		 * and return.
2490 		 */
2491 		state->hs_fw_cmdset = HERMON_FLASH_SPI_CMDSET;
2492 		state->hs_fw_log_sector_sz = HERMON_FLASH_SPI_LOG_SECTOR_SIZE;
2493 		state->hs_fw_device_sz = HERMON_FLASH_SPI_DEVICE_SIZE;
2494 
2495 		/*
2496 		 * set this to inform caller of cmdset type.
2497 		 */
2498 		cfi_ch_info[0x13] = HERMON_FLASH_SPI_CMDSET;
2499 		hermon_flash_cfi_dword(&cfi_info[4], cfi_ch_info, 0x10);
2500 		return (0);
2501 	}
2502 
2503 	/*
2504 	 * Determine if the user command supports the Intel Extended
2505 	 * Command Set. The query string is contained in the fourth
2506 	 * quad word.
2507 	 */
2508 	hermon_flash_cfi_byte(cfi_ch_info, cfi_info[0x04], 0x10);
2509 	if (cfi_ch_info[0x10] == 'M' &&
2510 	    cfi_ch_info[0x11] == 'X' &&
2511 	    cfi_ch_info[0x12] == '2') {
2512 		*intel_xcmd = 1; /* support is there */
2513 		if (hermon_verbose) {
2514 			IBTF_DPRINTF_L2("hermon",
2515 			    "Support for Intel X is present\n");
2516 		}
2517 	}
2518 
2519 	/* CFI QUERY */
2520 	hermon_flash_write(state, 0x55, HERMON_FLASH_CFI_INIT, &status);
2521 	if (status != 0) {
2522 		return (status);
2523 	}
2524 
2525 	/* temporarily set the cmdset in order to do the initial read */
2526 	state->hs_fw_cmdset = HERMON_FLASH_INTEL_CMDSET;
2527 
2528 	/* Read in CFI data */
2529 	for (i = 0; i < HERMON_CFI_INFO_SIZE; i += 4) {
2530 		data = hermon_flash_read(state, i, &status);
2531 		if (status != 0) {
2532 			return (status);
2533 		}
2534 		cfi_dw_info[i >> 2] = data;
2535 		hermon_flash_cfi_byte(cfi_ch_info, data, i);
2536 	}
2537 
2538 	/* Determine chip set */
2539 	state->hs_fw_cmdset = HERMON_FLASH_UNKNOWN_CMDSET;
2540 	if (cfi_ch_info[0x20] == 'Q' &&
2541 	    cfi_ch_info[0x22] == 'R' &&
2542 	    cfi_ch_info[0x24] == 'Y') {
2543 		/*
2544 		 * Mode: x16 working in x8 mode (Intel).
2545 		 * Pack data - skip spacing bytes.
2546 		 */
2547 		if (hermon_verbose) {
2548 			IBTF_DPRINTF_L2("hermon",
2549 			    "x16 working in x8 mode (Intel)\n");
2550 		}
2551 		for (i = 0; i < HERMON_CFI_INFO_SIZE; i += 2) {
2552 			cfi_ch_info[i/2] = cfi_ch_info[i];
2553 		}
2554 	}
2555 	state->hs_fw_cmdset = cfi_ch_info[0x13];
2556 
2557 	if (state->hs_fw_cmdset != HERMON_FLASH_INTEL_CMDSET &&
2558 	    state->hs_fw_cmdset != HERMON_FLASH_AMD_CMDSET) {
2559 		cmn_err(CE_WARN,
2560 		    "hermon_flash_cfi_init: UNKNOWN chip cmd set 0x%04x\n",
2561 		    state->hs_fw_cmdset);
2562 		state->hs_fw_cmdset = HERMON_FLASH_UNKNOWN_CMDSET;
2563 		return (0);
2564 	}
2565 
2566 	/* Determine total bytes in one sector size */
2567 	sector_sz_bytes = ((cfi_ch_info[0x30] << 8) | cfi_ch_info[0x2F]) << 8;
2568 
2569 	/* Calculate equivalent of log2 (n) */
2570 	for (bit_count = 0; sector_sz_bytes > 1; bit_count++) {
2571 		sector_sz_bytes >>= 1;
2572 	}
2573 
2574 	/* Set sector size */
2575 	state->hs_fw_log_sector_sz = bit_count;
2576 
2577 	/* Set flash size */
2578 	state->hs_fw_device_sz = 0x1 << cfi_ch_info[0x27];
2579 
2580 	/* Reset to turn off CFI mode */
2581 	if ((status = hermon_flash_reset(state)) != 0)
2582 		goto out;
2583 
2584 	/* Pass CFI data back to user command. */
2585 	for (i = 0; i < HERMON_FLASH_CFI_SIZE_QUADLET; i++) {
2586 		hermon_flash_cfi_dword(&cfi_info[i], cfi_ch_info, i << 2);
2587 	}
2588 
2589 	if (*intel_xcmd == 1) {
2590 		/*
2591 		 * Inform the user cmd that this driver does support the
2592 		 * Intel Extended Command Set.
2593 		 */
2594 		cfi_ch_info[0x10] = 'M';
2595 		cfi_ch_info[0x11] = 'X';
2596 		cfi_ch_info[0x12] = '2';
2597 	} else {
2598 		cfi_ch_info[0x10] = 'Q';
2599 		cfi_ch_info[0x11] = 'R';
2600 		cfi_ch_info[0x12] = 'Y';
2601 	}
2602 	cfi_ch_info[0x13] = state->hs_fw_cmdset;
2603 	hermon_flash_cfi_dword(&cfi_info[0x4], cfi_ch_info, 0x10);
2604 out:
2605 	return (status);
2606 }
2607 
2608 /*
2609  * hermon_flash_fini()
2610  */
2611 static int
2612 hermon_flash_fini(hermon_state_t *state)
2613 {
2614 	int status;
2615 	ddi_acc_handle_t hdl;
2616 
2617 	/* initialize the FMA retry loop */
2618 	hermon_pio_init(fm_loop_cnt, fm_status, fm_test);
2619 
2620 	/* Set handle */
2621 	hdl = hermon_get_pcihdl(state);
2622 
2623 	if ((status = hermon_flash_bank(state, 0)) != 0)
2624 		return (status);
2625 
2626 	/* the FMA retry loop starts. */
2627 	hermon_pio_start(state, hdl, pio_error, fm_loop_cnt, fm_status,
2628 	    fm_test);
2629 
2630 	/*
2631 	 * Restore original GPIO Values
2632 	 */
2633 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_GPIO_DATA,
2634 	    state->hs_fw_gpio[0]);
2635 
2636 	/* unlock GPIOs */
2637 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_GPIO_LOCK,
2638 	    HERMON_HW_FLASH_GPIO_UNLOCK_VAL);
2639 
2640 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_GPIO_MOD0,
2641 	    state->hs_fw_gpio[1]);
2642 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_GPIO_MOD1,
2643 	    state->hs_fw_gpio[2]);
2644 
2645 	/* re-lock GPIOs */
2646 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_GPIO_LOCK, 0);
2647 
2648 	/* Give up gpio semaphore */
2649 	hermon_flash_write_cfg(state, hdl, HERMON_HW_FLASH_GPIO_SEMA, 0);
2650 
2651 	/* the FMA retry loop ends. */
2652 	hermon_pio_end(state, hdl, pio_error, fm_loop_cnt, fm_status, fm_test);
2653 	return (0);
2654 
2655 pio_error:
2656 	hermon_fm_ereport(state, HCA_SYS_ERR, HCA_ERR_IOCTL);
2657 	return (EIO);
2658 }
2659 
2660 /*
2661  * hermon_flash_read_cfg
2662  */
2663 static uint32_t
2664 hermon_flash_read_cfg(hermon_state_t *state, ddi_acc_handle_t pci_config_hdl,
2665     uint32_t addr)
2666 {
2667 	uint32_t	read;
2668 
2669 	if (do_bar0) {
2670 		read = ddi_get32(hermon_get_cmdhdl(state), (uint32_t *)(void *)
2671 		    (state->hs_reg_cmd_baseaddr + addr));
2672 	} else {
2673 		/*
2674 		 * Perform flash read operation:
2675 		 *   1) Place addr to read from on the HERMON_HW_FLASH_CFG_ADDR
2676 		 *	register
2677 		 *   2) Read data at that addr from the HERMON_HW_FLASH_CFG_DATA
2678 		 *	 register
2679 		 */
2680 		pci_config_put32(pci_config_hdl, HERMON_HW_FLASH_CFG_ADDR,
2681 		    addr);
2682 		read = pci_config_get32(pci_config_hdl,
2683 		    HERMON_HW_FLASH_CFG_DATA);
2684 	}
2685 
2686 	return (read);
2687 }
2688 
2689 #ifdef DO_WRCONF
2690 static void
2691 hermon_flash_write_cfg(hermon_state_t *state,
2692     ddi_acc_handle_t pci_config_hdl, uint32_t addr, uint32_t data)
2693 {
2694 	hermon_flash_write_cfg_helper(state, pci_config_hdl, addr, data);
2695 	hermon_flash_write_confirm(state, pci_config_hdl);
2696 }
2697 
2698 static void
2699 hermon_flash_write_confirm(hermon_state_t *state,
2700     ddi_acc_handle_t pci_config_hdl)
2701 {
2702 	uint32_t	sem_value = 1;
2703 
2704 	hermon_flash_write_cfg_helper(state, pci_config_hdl,
2705 	    HERMON_HW_FLASH_WRCONF_SEMA, 0);
2706 	while (sem_value) {
2707 		sem_value = hermon_flash_read_cfg(state, pci_config_hdl,
2708 		    HERMON_HW_FLASH_WRCONF_SEMA);
2709 	}
2710 }
2711 #endif
2712 
2713 /*
2714  * hermon_flash_write_cfg
2715  */
2716 static void
2717 #ifdef DO_WRCONF
2718 hermon_flash_write_cfg_helper(hermon_state_t *state,
2719     ddi_acc_handle_t pci_config_hdl, uint32_t addr, uint32_t data)
2720 #else
2721 hermon_flash_write_cfg(hermon_state_t *state,
2722     ddi_acc_handle_t pci_config_hdl, uint32_t addr, uint32_t data)
2723 #endif
2724 {
2725 	if (do_bar0) {
2726 		ddi_put32(hermon_get_cmdhdl(state), (uint32_t *)(void *)
2727 		    (state->hs_reg_cmd_baseaddr + addr), data);
2728 
2729 	} else {
2730 
2731 		/*
2732 		 * Perform flash write operation:
2733 		 *   1) Place addr to write to on the HERMON_HW_FLASH_CFG_ADDR
2734 		 *	register
2735 		 *   2) Place data to write on to the HERMON_HW_FLASH_CFG_DATA
2736 		 *	register
2737 		 */
2738 		pci_config_put32(pci_config_hdl, HERMON_HW_FLASH_CFG_ADDR,
2739 		    addr);
2740 		pci_config_put32(pci_config_hdl, HERMON_HW_FLASH_CFG_DATA,
2741 		    data);
2742 	}
2743 }
2744 
2745 /*
2746  * Support routines to convert Common Flash Interface (CFI) data
2747  * from a 32  bit word to a char array, and from a char array to
2748  * a 32 bit word.
2749  */
2750 static void
2751 hermon_flash_cfi_byte(uint8_t *ch, uint32_t dword, int i)
2752 {
2753 	ch[i] = (uint8_t)((dword & 0xFF000000) >> 24);
2754 	ch[i+1] = (uint8_t)((dword & 0x00FF0000) >> 16);
2755 	ch[i+2] = (uint8_t)((dword & 0x0000FF00) >> 8);
2756 	ch[i+3] = (uint8_t)((dword & 0x000000FF));
2757 }
2758 
2759 static void
2760 hermon_flash_cfi_dword(uint32_t *dword, uint8_t *ch, int i)
2761 {
2762 	*dword = (uint32_t)
2763 	    ((uint32_t)ch[i] << 24 |
2764 	    (uint32_t)ch[i+1] << 16 |
2765 	    (uint32_t)ch[i+2] << 8 |
2766 	    (uint32_t)ch[i+3]);
2767 }
2768 
2769 /*
2770  * hermon_loopback_free_qps
2771  */
2772 static void
2773 hermon_loopback_free_qps(hermon_loopback_state_t *lstate)
2774 {
2775 	int i;
2776 
2777 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2778 
2779 	if (lstate->hls_tx.hlc_qp_hdl != NULL) {
2780 		(void) hermon_qp_free(lstate->hls_state,
2781 		    &lstate->hls_tx.hlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2782 		    HERMON_NOSLEEP);
2783 	}
2784 	if (lstate->hls_rx.hlc_qp_hdl != NULL) {
2785 		(void) hermon_qp_free(lstate->hls_state,
2786 		    &lstate->hls_rx.hlc_qp_hdl, IBC_FREE_QP_AND_QPN, NULL,
2787 		    HERMON_NOSLEEP);
2788 	}
2789 	lstate->hls_tx.hlc_qp_hdl = NULL;
2790 	lstate->hls_rx.hlc_qp_hdl = NULL;
2791 	for (i = 0; i < 2; i++) {
2792 		if (lstate->hls_tx.hlc_cqhdl[i] != NULL) {
2793 			(void) hermon_cq_free(lstate->hls_state,
2794 			    &lstate->hls_tx.hlc_cqhdl[i], HERMON_NOSLEEP);
2795 		}
2796 		if (lstate->hls_rx.hlc_cqhdl[i] != NULL) {
2797 			(void) hermon_cq_free(lstate->hls_state,
2798 			    &lstate->hls_rx.hlc_cqhdl[i], HERMON_NOSLEEP);
2799 		}
2800 		lstate->hls_tx.hlc_cqhdl[i] = NULL;
2801 		lstate->hls_rx.hlc_cqhdl[i] = NULL;
2802 	}
2803 }
2804 
2805 /*
2806  * hermon_loopback_free_state
2807  */
2808 static void
2809 hermon_loopback_free_state(hermon_loopback_state_t *lstate)
2810 {
2811 	hermon_loopback_free_qps(lstate);
2812 	if (lstate->hls_tx.hlc_mrhdl != NULL) {
2813 		(void) hermon_mr_deregister(lstate->hls_state,
2814 		    &lstate->hls_tx.hlc_mrhdl, HERMON_MR_DEREG_ALL,
2815 		    HERMON_NOSLEEP);
2816 	}
2817 	if (lstate->hls_rx.hlc_mrhdl !=  NULL) {
2818 		(void) hermon_mr_deregister(lstate->hls_state,
2819 		    &lstate->hls_rx.hlc_mrhdl, HERMON_MR_DEREG_ALL,
2820 		    HERMON_NOSLEEP);
2821 	}
2822 	if (lstate->hls_pd_hdl != NULL) {
2823 		(void) hermon_pd_free(lstate->hls_state, &lstate->hls_pd_hdl);
2824 	}
2825 	if (lstate->hls_tx.hlc_buf != NULL) {
2826 		kmem_free(lstate->hls_tx.hlc_buf, lstate->hls_tx.hlc_buf_sz);
2827 	}
2828 	if (lstate->hls_rx.hlc_buf != NULL) {
2829 		kmem_free(lstate->hls_rx.hlc_buf, lstate->hls_rx.hlc_buf_sz);
2830 	}
2831 	bzero(lstate, sizeof (hermon_loopback_state_t));
2832 }
2833 
2834 /*
2835  * hermon_loopback_init
2836  */
2837 static int
2838 hermon_loopback_init(hermon_state_t *state, hermon_loopback_state_t *lstate)
2839 {
2840 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2841 
2842 	lstate->hls_hca_hdl = (ibc_hca_hdl_t)state;
2843 	lstate->hls_status  = hermon_pd_alloc(lstate->hls_state,
2844 	    &lstate->hls_pd_hdl, HERMON_NOSLEEP);
2845 	if (lstate->hls_status != IBT_SUCCESS) {
2846 		lstate->hls_err = HERMON_LOOPBACK_PROT_DOMAIN_ALLOC_FAIL;
2847 		return (EFAULT);
2848 	}
2849 
2850 	return (0);
2851 }
2852 
2853 /*
2854  * hermon_loopback_init_qp_info
2855  */
2856 static void
2857 hermon_loopback_init_qp_info(hermon_loopback_state_t *lstate,
2858     hermon_loopback_comm_t *comm)
2859 {
2860 	bzero(&comm->hlc_cq_attr, sizeof (ibt_cq_attr_t));
2861 	bzero(&comm->hlc_qp_attr, sizeof (ibt_qp_alloc_attr_t));
2862 	bzero(&comm->hlc_qp_info, sizeof (ibt_qp_info_t));
2863 
2864 	comm->hlc_wrid = 1;
2865 	comm->hlc_cq_attr.cq_size = 128;
2866 	comm->hlc_qp_attr.qp_sizes.cs_sq_sgl = 3;
2867 	comm->hlc_qp_attr.qp_sizes.cs_rq_sgl = 3;
2868 	comm->hlc_qp_attr.qp_sizes.cs_sq = 16;
2869 	comm->hlc_qp_attr.qp_sizes.cs_rq = 16;
2870 	comm->hlc_qp_attr.qp_flags = IBT_WR_SIGNALED;
2871 
2872 	comm->hlc_qp_info.qp_state = IBT_STATE_RESET;
2873 	comm->hlc_qp_info.qp_trans = IBT_RC_SRV;
2874 	comm->hlc_qp_info.qp_flags = IBT_CEP_RDMA_RD | IBT_CEP_RDMA_WR;
2875 	comm->hlc_qp_info.qp_transport.rc.rc_path.cep_hca_port_num =
2876 	    lstate->hls_port;
2877 	comm->hlc_qp_info.qp_transport.rc.rc_path.cep_pkey_ix =
2878 	    lstate->hls_pkey_ix;
2879 	comm->hlc_qp_info.qp_transport.rc.rc_path.cep_timeout =
2880 	    lstate->hls_timeout;
2881 	comm->hlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_srvl = 0;
2882 	comm->hlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_srate =
2883 	    IBT_SRATE_4X;
2884 	comm->hlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_send_grh = 0;
2885 	comm->hlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid =
2886 	    lstate->hls_lid;
2887 	comm->hlc_qp_info.qp_transport.rc.rc_retry_cnt = lstate->hls_retry;
2888 	comm->hlc_qp_info.qp_transport.rc.rc_sq_psn = 0;
2889 	comm->hlc_qp_info.qp_transport.rc.rc_rq_psn = 0;
2890 	comm->hlc_qp_info.qp_transport.rc.rc_rdma_ra_in	 = 4;
2891 	comm->hlc_qp_info.qp_transport.rc.rc_rdma_ra_out = 4;
2892 	comm->hlc_qp_info.qp_transport.rc.rc_dst_qpn = 0;
2893 	comm->hlc_qp_info.qp_transport.rc.rc_min_rnr_nak = IBT_RNR_NAK_655ms;
2894 	comm->hlc_qp_info.qp_transport.rc.rc_path_mtu = IB_MTU_1K;
2895 }
2896 
2897 /*
2898  * hermon_loopback_alloc_mem
2899  */
2900 static int
2901 hermon_loopback_alloc_mem(hermon_loopback_state_t *lstate,
2902     hermon_loopback_comm_t *comm, int sz)
2903 {
2904 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2905 
2906 	/* Allocate buffer of specified size */
2907 	comm->hlc_buf_sz = sz;
2908 	comm->hlc_buf	 = kmem_zalloc(sz, KM_NOSLEEP);
2909 	if (comm->hlc_buf == NULL) {
2910 		return (EFAULT);
2911 	}
2912 
2913 	/* Register the buffer as a memory region */
2914 	comm->hlc_memattr.mr_vaddr = (uint64_t)(uintptr_t)comm->hlc_buf;
2915 	comm->hlc_memattr.mr_len   = (ib_msglen_t)sz;
2916 	comm->hlc_memattr.mr_as	   = NULL;
2917 	comm->hlc_memattr.mr_flags = IBT_MR_NOSLEEP |
2918 	    IBT_MR_ENABLE_REMOTE_WRITE | IBT_MR_ENABLE_LOCAL_WRITE;
2919 
2920 	comm->hlc_status = hermon_mr_register(lstate->hls_state,
2921 	    lstate->hls_pd_hdl, &comm->hlc_memattr, &comm->hlc_mrhdl,
2922 	    NULL, HERMON_MPT_DMPT);
2923 
2924 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm->hlc_mrhdl))
2925 
2926 	comm->hlc_mrdesc.md_vaddr  = comm->hlc_mrhdl->mr_bindinfo.bi_addr;
2927 	comm->hlc_mrdesc.md_lkey   = comm->hlc_mrhdl->mr_lkey;
2928 	comm->hlc_mrdesc.md_rkey   = comm->hlc_mrhdl->mr_rkey;
2929 	if (comm->hlc_status != IBT_SUCCESS) {
2930 		return (EFAULT);
2931 	}
2932 	return (0);
2933 }
2934 
2935 /*
2936  * hermon_loopback_alloc_qps
2937  */
2938 static int
2939 hermon_loopback_alloc_qps(hermon_loopback_state_t *lstate,
2940     hermon_loopback_comm_t *comm)
2941 {
2942 	uint32_t		i, real_size;
2943 	hermon_qp_info_t		qpinfo;
2944 
2945 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2946 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2947 
2948 	/* Allocate send and recv CQs */
2949 	for (i = 0; i < 2; i++) {
2950 		bzero(&comm->hlc_cq_attr, sizeof (ibt_cq_attr_t));
2951 		comm->hlc_cq_attr.cq_size = 128;
2952 		comm->hlc_status = hermon_cq_alloc(lstate->hls_state,
2953 		    (ibt_cq_hdl_t)NULL, &comm->hlc_cq_attr, &real_size,
2954 		    &comm->hlc_cqhdl[i], HERMON_NOSLEEP);
2955 		if (comm->hlc_status != IBT_SUCCESS) {
2956 			lstate->hls_err += i;
2957 			return (EFAULT);
2958 		}
2959 	}
2960 
2961 	/* Allocate the QP */
2962 	hermon_loopback_init_qp_info(lstate, comm);
2963 	comm->hlc_qp_attr.qp_pd_hdl	 = (ibt_pd_hdl_t)lstate->hls_pd_hdl;
2964 	comm->hlc_qp_attr.qp_scq_hdl	 = (ibt_cq_hdl_t)comm->hlc_cqhdl[0];
2965 	comm->hlc_qp_attr.qp_rcq_hdl	 = (ibt_cq_hdl_t)comm->hlc_cqhdl[1];
2966 	comm->hlc_qp_attr.qp_ibc_scq_hdl = (ibt_opaque1_t)comm->hlc_cqhdl[0];
2967 	comm->hlc_qp_attr.qp_ibc_rcq_hdl = (ibt_opaque1_t)comm->hlc_cqhdl[1];
2968 	qpinfo.qpi_attrp	= &comm->hlc_qp_attr;
2969 	qpinfo.qpi_type		= IBT_RC_RQP;
2970 	qpinfo.qpi_ibt_qphdl	= NULL;
2971 	qpinfo.qpi_queueszp	= &comm->hlc_chan_sizes;
2972 	qpinfo.qpi_qpn		= &comm->hlc_qp_num;
2973 	comm->hlc_status = hermon_qp_alloc(lstate->hls_state, &qpinfo,
2974 	    HERMON_NOSLEEP);
2975 	if (comm->hlc_status == DDI_SUCCESS) {
2976 		comm->hlc_qp_hdl = qpinfo.qpi_qphdl;
2977 	}
2978 
2979 	if (comm->hlc_status != IBT_SUCCESS) {
2980 		lstate->hls_err += 2;
2981 		return (EFAULT);
2982 	}
2983 	return (0);
2984 }
2985 
2986 /*
2987  * hermon_loopback_modify_qp
2988  */
2989 static int
2990 hermon_loopback_modify_qp(hermon_loopback_state_t *lstate,
2991     hermon_loopback_comm_t *comm, uint_t qp_num)
2992 {
2993 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
2994 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*lstate))
2995 
2996 	/* Modify QP to INIT */
2997 	hermon_loopback_init_qp_info(lstate, comm);
2998 	comm->hlc_qp_info.qp_state = IBT_STATE_INIT;
2999 	comm->hlc_status = hermon_qp_modify(lstate->hls_state, comm->hlc_qp_hdl,
3000 	    IBT_CEP_SET_STATE, &comm->hlc_qp_info, &comm->hlc_queue_sizes);
3001 	if (comm->hlc_status != IBT_SUCCESS) {
3002 		return (EFAULT);
3003 	}
3004 
3005 	/*
3006 	 * Modify QP to RTR (set destination LID and QP number to local
3007 	 * LID and QP number)
3008 	 */
3009 	comm->hlc_qp_info.qp_state = IBT_STATE_RTR;
3010 	comm->hlc_qp_info.qp_transport.rc.rc_path.cep_adds_vect.av_dlid
3011 	    = lstate->hls_lid;
3012 	comm->hlc_qp_info.qp_transport.rc.rc_dst_qpn = qp_num;
3013 	comm->hlc_status = hermon_qp_modify(lstate->hls_state, comm->hlc_qp_hdl,
3014 	    IBT_CEP_SET_STATE, &comm->hlc_qp_info, &comm->hlc_queue_sizes);
3015 	if (comm->hlc_status != IBT_SUCCESS) {
3016 		lstate->hls_err += 1;
3017 		return (EFAULT);
3018 	}
3019 
3020 	/* Modify QP to RTS */
3021 	comm->hlc_qp_info.qp_current_state = IBT_STATE_RTR;
3022 	comm->hlc_qp_info.qp_state = IBT_STATE_RTS;
3023 	comm->hlc_status = hermon_qp_modify(lstate->hls_state, comm->hlc_qp_hdl,
3024 	    IBT_CEP_SET_STATE, &comm->hlc_qp_info, &comm->hlc_queue_sizes);
3025 	if (comm->hlc_status != IBT_SUCCESS) {
3026 		lstate->hls_err += 2;
3027 		return (EFAULT);
3028 	}
3029 	return (0);
3030 }
3031 
3032 /*
3033  * hermon_loopback_copyout
3034  */
3035 static int
3036 hermon_loopback_copyout(hermon_loopback_ioctl_t *lb, intptr_t arg, int mode)
3037 {
3038 #ifdef _MULTI_DATAMODEL
3039 	if (ddi_model_convert_from(mode & FMODELS) == DDI_MODEL_ILP32) {
3040 		hermon_loopback_ioctl32_t lb32;
3041 
3042 		lb32.alb_revision	= lb->alb_revision;
3043 		lb32.alb_send_buf	=
3044 		    (caddr32_t)(uintptr_t)lb->alb_send_buf;
3045 		lb32.alb_fail_buf	=
3046 		    (caddr32_t)(uintptr_t)lb->alb_fail_buf;
3047 		lb32.alb_buf_sz		= lb->alb_buf_sz;
3048 		lb32.alb_num_iter	= lb->alb_num_iter;
3049 		lb32.alb_pass_done	= lb->alb_pass_done;
3050 		lb32.alb_timeout	= lb->alb_timeout;
3051 		lb32.alb_error_type	= lb->alb_error_type;
3052 		lb32.alb_port_num	= lb->alb_port_num;
3053 		lb32.alb_num_retry	= lb->alb_num_retry;
3054 
3055 		if (ddi_copyout(&lb32, (void *)arg,
3056 		    sizeof (hermon_loopback_ioctl32_t), mode) != 0) {
3057 			return (EFAULT);
3058 		}
3059 	} else
3060 #endif /* _MULTI_DATAMODEL */
3061 	if (ddi_copyout(lb, (void *)arg, sizeof (hermon_loopback_ioctl_t),
3062 	    mode) != 0) {
3063 		return (EFAULT);
3064 	}
3065 	return (0);
3066 }
3067 
3068 /*
3069  * hermon_loopback_post_send
3070  */
3071 static int
3072 hermon_loopback_post_send(hermon_loopback_state_t *lstate,
3073     hermon_loopback_comm_t *tx, hermon_loopback_comm_t *rx)
3074 {
3075 	int	 ret;
3076 
3077 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*tx))
3078 
3079 	bzero(&tx->hlc_sgl, sizeof (ibt_wr_ds_t));
3080 	bzero(&tx->hlc_wr, sizeof (ibt_send_wr_t));
3081 
3082 	/* Initialize local address for TX buffer */
3083 	tx->hlc_sgl.ds_va   = tx->hlc_mrdesc.md_vaddr;
3084 	tx->hlc_sgl.ds_key  = tx->hlc_mrdesc.md_lkey;
3085 	tx->hlc_sgl.ds_len  = tx->hlc_buf_sz;
3086 
3087 	/* Initialize the remaining details of the work request */
3088 	tx->hlc_wr.wr_id = tx->hlc_wrid++;
3089 	tx->hlc_wr.wr_flags  = IBT_WR_SEND_SIGNAL;
3090 	tx->hlc_wr.wr_nds    = 1;
3091 	tx->hlc_wr.wr_sgl    = &tx->hlc_sgl;
3092 	tx->hlc_wr.wr_opcode = IBT_WRC_RDMAW;
3093 	tx->hlc_wr.wr_trans  = IBT_RC_SRV;
3094 
3095 	/* Initialize the remote address for RX buffer */
3096 	tx->hlc_wr.wr.rc.rcwr.rdma.rdma_raddr = rx->hlc_mrdesc.md_vaddr;
3097 	tx->hlc_wr.wr.rc.rcwr.rdma.rdma_rkey  = rx->hlc_mrdesc.md_rkey;
3098 	tx->hlc_complete = 0;
3099 	ret = hermon_post_send(lstate->hls_state, tx->hlc_qp_hdl, &tx->hlc_wr,
3100 	    1, NULL);
3101 	if (ret != IBT_SUCCESS) {
3102 		return (EFAULT);
3103 	}
3104 	return (0);
3105 }
3106 
3107 /*
3108  * hermon_loopback_poll_cq
3109  */
3110 static int
3111 hermon_loopback_poll_cq(hermon_loopback_state_t *lstate,
3112     hermon_loopback_comm_t *comm)
3113 {
3114 	_NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*comm))
3115 
3116 	comm->hlc_wc.wc_status	= 0;
3117 	comm->hlc_num_polled	= 0;
3118 	comm->hlc_status = hermon_cq_poll(lstate->hls_state,
3119 	    comm->hlc_cqhdl[0], &comm->hlc_wc, 1, &comm->hlc_num_polled);
3120 	if ((comm->hlc_status == IBT_SUCCESS) &&
3121 	    (comm->hlc_wc.wc_status != IBT_WC_SUCCESS)) {
3122 		comm->hlc_status = ibc_get_ci_failure(0);
3123 	}
3124 	return (comm->hlc_status);
3125 }
3126