1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * tavor.c
28  *    Tavor (InfiniBand) HCA Driver attach/detach Routines
29  *
30  *    Implements all the routines necessary for the attach, setup,
31  *    initialization (and subsequent possible teardown and detach) of the
32  *    Tavor InfiniBand HCA driver.
33  */
34 
35 #include <sys/types.h>
36 #include <sys/file.h>
37 #include <sys/open.h>
38 #include <sys/conf.h>
39 #include <sys/ddi.h>
40 #include <sys/sunddi.h>
41 #include <sys/modctl.h>
42 #include <sys/stat.h>
43 #include <sys/pci.h>
44 #include <sys/pci_cap.h>
45 #include <sys/bitmap.h>
46 #include <sys/policy.h>
47 
48 #include <sys/ib/adapters/tavor/tavor.h>
49 #include <sys/pci.h>
50 
51 /* Tavor HCA State Pointer */
52 void *tavor_statep;
53 
54 /*
55  * The Tavor "userland resource database" is common to instances of the
56  * Tavor HCA driver.  This structure "tavor_userland_rsrc_db" contains all
57  * the necessary information to maintain it.
58  */
59 tavor_umap_db_t tavor_userland_rsrc_db;
60 
61 static int tavor_attach(dev_info_t *, ddi_attach_cmd_t);
62 static int tavor_detach(dev_info_t *, ddi_detach_cmd_t);
63 static int tavor_open(dev_t *, int, int, cred_t *);
64 static int tavor_close(dev_t, int, int, cred_t *);
65 static int tavor_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
66 static int tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance);
67 static void tavor_drv_fini(tavor_state_t *state);
68 static void tavor_drv_fini2(tavor_state_t *state);
69 static int tavor_isr_init(tavor_state_t *state);
70 static void tavor_isr_fini(tavor_state_t *state);
71 static int tavor_hw_init(tavor_state_t *state);
72 static void tavor_hw_fini(tavor_state_t *state,
73     tavor_drv_cleanup_level_t cleanup);
74 static int tavor_soft_state_init(tavor_state_t *state);
75 static void tavor_soft_state_fini(tavor_state_t *state);
76 static int tavor_hca_port_init(tavor_state_t *state);
77 static int tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init);
78 static void tavor_hca_config_setup(tavor_state_t *state,
79     tavor_hw_initqueryhca_t *inithca);
80 static int tavor_internal_uarpgs_init(tavor_state_t *state);
81 static void tavor_internal_uarpgs_fini(tavor_state_t *state);
82 static int tavor_special_qp_contexts_reserve(tavor_state_t *state);
83 static void tavor_special_qp_contexts_unreserve(tavor_state_t *state);
84 static int tavor_sw_reset(tavor_state_t *state);
85 static int tavor_mcg_init(tavor_state_t *state);
86 static void tavor_mcg_fini(tavor_state_t *state);
87 static int tavor_fw_version_check(tavor_state_t *state);
88 static void tavor_device_info_report(tavor_state_t *state);
89 static void tavor_pci_capability_list(tavor_state_t *state,
90     ddi_acc_handle_t hdl);
91 static void tavor_pci_capability_vpd(tavor_state_t *state,
92     ddi_acc_handle_t hdl, uint_t offset);
93 static int tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset,
94     uint32_t addr, uint32_t *data);
95 static void tavor_pci_capability_pcix(tavor_state_t *state,
96     ddi_acc_handle_t hdl, uint_t offset);
97 static int tavor_intr_or_msi_init(tavor_state_t *state);
98 static int tavor_add_intrs(tavor_state_t *state, int intr_type);
99 static int tavor_intr_or_msi_fini(tavor_state_t *state);
100 
101 /* X86 fastreboot support */
102 static int tavor_intr_disable(tavor_state_t *);
103 static int tavor_quiesce(dev_info_t *);
104 
105 /* Character/Block Operations */
106 static struct cb_ops tavor_cb_ops = {
107 	tavor_open,		/* open */
108 	tavor_close,		/* close */
109 	nodev,			/* strategy (block) */
110 	nodev,			/* print (block) */
111 	nodev,			/* dump (block) */
112 	nodev,			/* read */
113 	nodev,			/* write */
114 	tavor_ioctl,		/* ioctl */
115 	tavor_devmap,		/* devmap */
116 	NULL,			/* mmap */
117 	nodev,			/* segmap */
118 	nochpoll,		/* chpoll */
119 	ddi_prop_op,		/* prop_op */
120 	NULL,			/* streams */
121 	D_NEW | D_MP |
122 	D_64BIT | D_HOTPLUG |
123 	D_DEVMAP,		/* flags */
124 	CB_REV			/* rev */
125 };
126 
127 /* Driver Operations */
128 static struct dev_ops tavor_ops = {
129 	DEVO_REV,		/* struct rev */
130 	0,			/* refcnt */
131 	tavor_getinfo,		/* getinfo */
132 	nulldev,		/* identify */
133 	nulldev,		/* probe */
134 	tavor_attach,		/* attach */
135 	tavor_detach,		/* detach */
136 	nodev,			/* reset */
137 	&tavor_cb_ops,		/* cb_ops */
138 	NULL,			/* bus_ops */
139 	nodev,			/* power */
140 	tavor_quiesce,		/* devo_quiesce */
141 };
142 
143 /* Module Driver Info */
144 static struct modldrv tavor_modldrv = {
145 	&mod_driverops,
146 	"Tavor InfiniBand HCA Driver",
147 	&tavor_ops
148 };
149 
150 /* Module Linkage */
151 static struct modlinkage tavor_modlinkage = {
152 	MODREV_1,
153 	&tavor_modldrv,
154 	NULL
155 };
156 
157 /*
158  * This extern refers to the ibc_operations_t function vector that is defined
159  * in the tavor_ci.c file.
160  */
161 extern ibc_operations_t	tavor_ibc_ops;
162 
163 #ifndef NPROBE
164 extern int tnf_mod_load(void);
165 extern int tnf_mod_unload(struct modlinkage *mlp);
166 #endif
167 
168 
169 /*
170  * _init()
171  */
172 int
_init()173 _init()
174 {
175 	int	status;
176 
177 #ifndef NPROBE
178 	(void) tnf_mod_load();
179 #endif
180 	TAVOR_TNF_ENTER(tavor_init);
181 
182 	status = ddi_soft_state_init(&tavor_statep, sizeof (tavor_state_t),
183 	    (size_t)TAVOR_INITIAL_STATES);
184 	if (status != 0) {
185 		TNF_PROBE_0(tavor_init_ssi_fail, TAVOR_TNF_ERROR, "");
186 		TAVOR_TNF_EXIT(tavor_init);
187 #ifndef NPROBE
188 		(void) tnf_mod_unload(&tavor_modlinkage);
189 #endif
190 		return (status);
191 	}
192 
193 	status = ibc_init(&tavor_modlinkage);
194 	if (status != 0) {
195 		TNF_PROBE_0(tavor_init_ibc_init_fail, TAVOR_TNF_ERROR, "");
196 		ddi_soft_state_fini(&tavor_statep);
197 		TAVOR_TNF_EXIT(tavor_init);
198 #ifndef NPROBE
199 		(void) tnf_mod_unload(&tavor_modlinkage);
200 #endif
201 		return (status);
202 	}
203 	status = mod_install(&tavor_modlinkage);
204 	if (status != 0) {
205 		TNF_PROBE_0(tavor_init_modi_fail, TAVOR_TNF_ERROR, "");
206 		ibc_fini(&tavor_modlinkage);
207 		ddi_soft_state_fini(&tavor_statep);
208 		TAVOR_TNF_EXIT(tavor_init);
209 #ifndef NPROBE
210 		(void) tnf_mod_unload(&tavor_modlinkage);
211 #endif
212 		return (status);
213 	}
214 
215 	/* Initialize the Tavor "userland resources database" */
216 	tavor_umap_db_init();
217 
218 	TAVOR_TNF_EXIT(tavor_init);
219 	return (status);
220 }
221 
222 
223 /*
224  * _info()
225  */
226 int
_info(struct modinfo * modinfop)227 _info(struct modinfo *modinfop)
228 {
229 	int	status;
230 
231 	TAVOR_TNF_ENTER(tavor_info);
232 	status = mod_info(&tavor_modlinkage, modinfop);
233 	TAVOR_TNF_EXIT(tavor_info);
234 	return (status);
235 }
236 
237 
238 /*
239  * _fini()
240  */
241 int
_fini()242 _fini()
243 {
244 	int	status;
245 
246 	TAVOR_TNF_ENTER(tavor_fini);
247 
248 	status = mod_remove(&tavor_modlinkage);
249 	if (status != 0) {
250 		TNF_PROBE_0(tavor_fini_modr_fail, TAVOR_TNF_ERROR, "");
251 		TAVOR_TNF_EXIT(tavor_fini);
252 		return (status);
253 	}
254 
255 	/* Destroy the Tavor "userland resources database" */
256 	tavor_umap_db_fini();
257 
258 	ibc_fini(&tavor_modlinkage);
259 	ddi_soft_state_fini(&tavor_statep);
260 #ifndef NPROBE
261 	(void) tnf_mod_unload(&tavor_modlinkage);
262 #endif
263 	TAVOR_TNF_EXIT(tavor_fini);
264 	return (status);
265 }
266 
267 
268 /*
269  * tavor_getinfo()
270  */
271 /* ARGSUSED */
272 static int
tavor_getinfo(dev_info_t * dip,ddi_info_cmd_t cmd,void * arg,void ** result)273 tavor_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result)
274 {
275 	dev_t		dev;
276 	tavor_state_t	*state;
277 	minor_t		instance;
278 
279 	TAVOR_TNF_ENTER(tavor_getinfo);
280 
281 	switch (cmd) {
282 	case DDI_INFO_DEVT2DEVINFO:
283 		dev = (dev_t)arg;
284 		instance = TAVOR_DEV_INSTANCE(dev);
285 		state = ddi_get_soft_state(tavor_statep, instance);
286 		if (state == NULL) {
287 			TNF_PROBE_0(tavor_getinfo_gss_fail,
288 			    TAVOR_TNF_ERROR, "");
289 			TAVOR_TNF_EXIT(tavor_getinfo);
290 			return (DDI_FAILURE);
291 		}
292 		*result = (void *)state->ts_dip;
293 		return (DDI_SUCCESS);
294 
295 	case DDI_INFO_DEVT2INSTANCE:
296 		dev = (dev_t)arg;
297 		instance = TAVOR_DEV_INSTANCE(dev);
298 		*result = (void *)(uintptr_t)instance;
299 		return (DDI_SUCCESS);
300 
301 	default:
302 		TNF_PROBE_0(tavor_getinfo_default_fail, TAVOR_TNF_ERROR, "");
303 		break;
304 	}
305 
306 	TAVOR_TNF_EXIT(tavor_getinfo);
307 	return (DDI_FAILURE);
308 }
309 
310 
311 /*
312  * tavor_open()
313  */
314 /* ARGSUSED */
315 static int
tavor_open(dev_t * devp,int flag,int otyp,cred_t * credp)316 tavor_open(dev_t *devp, int flag, int otyp, cred_t *credp)
317 {
318 	tavor_state_t		*state;
319 	tavor_rsrc_t		*rsrcp;
320 	tavor_umap_db_entry_t	*umapdb, *umapdb2;
321 	minor_t			instance;
322 	uint64_t		key, value;
323 	uint_t			tr_indx;
324 	dev_t			dev;
325 	int			status;
326 
327 	TAVOR_TNF_ENTER(tavor_open);
328 
329 	instance = TAVOR_DEV_INSTANCE(*devp);
330 	state = ddi_get_soft_state(tavor_statep, instance);
331 	if (state == NULL) {
332 		TNF_PROBE_0(tavor_open_gss_fail, TAVOR_TNF_ERROR, "");
333 		TAVOR_TNF_EXIT(tavor_open);
334 		return (ENXIO);
335 	}
336 
337 	/*
338 	 * Only allow driver to be opened for character access, and verify
339 	 * whether exclusive access is allowed.
340 	 */
341 	if ((otyp != OTYP_CHR) || ((flag & FEXCL) &&
342 	    secpolicy_excl_open(credp) != 0)) {
343 		TNF_PROBE_0(tavor_open_invflags_fail, TAVOR_TNF_ERROR, "");
344 		TAVOR_TNF_EXIT(tavor_open);
345 		return (EINVAL);
346 	}
347 
348 	/*
349 	 * Search for the current process PID in the "userland resources
350 	 * database".  If it is not found, then attempt to allocate a UAR
351 	 * page and add the ("key", "value") pair to the database.
352 	 * Note:  As a last step we always return a devp appropriate for
353 	 * the open.  Either we return a new minor number (based on the
354 	 * instance and the UAR page index) or we return the current minor
355 	 * number for the given client process.
356 	 *
357 	 * We also add an entry to the database to allow for lookup from
358 	 * "dev_t" to the current process PID.  This is necessary because,
359 	 * under certain circumstance, the process PID that calls the Tavor
360 	 * close() entry point may not be the same as the one who called
361 	 * open().  Specifically, this can happen if a child process calls
362 	 * the Tavor's open() entry point, gets a UAR page, maps it out (using
363 	 * mmap()), and then exits without calling munmap().  Because mmap()
364 	 * adds a reference to the file descriptor, at the exit of the child
365 	 * process the file descriptor is "inherited" by the parent (and will
366 	 * be close()'d by the parent's PID only when it exits).
367 	 *
368 	 * Note: We use the tavor_umap_db_find_nolock() and
369 	 * tavor_umap_db_add_nolock() database access routines below (with
370 	 * an explicit mutex_enter of the database lock - "tdl_umapdb_lock")
371 	 * to ensure that the multiple accesses (in this case searching for,
372 	 * and then adding _two_ database entries) can be done atomically.
373 	 */
374 	key = ddi_get_pid();
375 	mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
376 	status = tavor_umap_db_find_nolock(instance, key,
377 	    MLNX_UMAP_UARPG_RSRC, &value, 0, NULL);
378 	if (status != DDI_SUCCESS) {
379 		/*
380 		 * If we are in 'maintenance mode', we cannot alloc a UAR page.
381 		 * But we still need some rsrcp value, and a mostly unique
382 		 * tr_indx value.  So we set rsrcp to NULL for maintenance
383 		 * mode, and use a rolling count for tr_indx.  The field
384 		 * 'ts_open_tr_indx' is used only in this maintenance mode
385 		 * condition.
386 		 *
387 		 * Otherwise, if we are in operational mode then we allocate
388 		 * the UAR page as normal, and use the rsrcp value and tr_indx
389 		 * value from that allocation.
390 		 */
391 		if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
392 			rsrcp = NULL;
393 			tr_indx = state->ts_open_tr_indx++;
394 		} else {
395 			/* Allocate a new UAR page for this process */
396 			status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1,
397 			    TAVOR_NOSLEEP, &rsrcp);
398 			if (status != DDI_SUCCESS) {
399 				mutex_exit(
400 				    &tavor_userland_rsrc_db.tdl_umapdb_lock);
401 				TNF_PROBE_0(tavor_open_rsrcalloc_uarpg_fail,
402 				    TAVOR_TNF_ERROR, "");
403 				TAVOR_TNF_EXIT(tavor_open);
404 				return (EAGAIN);
405 			}
406 
407 			tr_indx = rsrcp->tr_indx;
408 		}
409 
410 		/*
411 		 * Allocate an entry to track the UAR page resource in the
412 		 * "userland resources database".
413 		 */
414 		umapdb = tavor_umap_db_alloc(instance, key,
415 		    MLNX_UMAP_UARPG_RSRC, (uint64_t)(uintptr_t)rsrcp);
416 		if (umapdb == NULL) {
417 			mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
418 			/* If in "maintenance mode", don't free the rsrc */
419 			if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
420 				tavor_rsrc_free(state, &rsrcp);
421 			}
422 			TNF_PROBE_0(tavor_open_umap_db_alloc_fail,
423 			    TAVOR_TNF_ERROR, "");
424 			TAVOR_TNF_EXIT(tavor_open);
425 			return (EAGAIN);
426 		}
427 
428 		/*
429 		 * Create a new device number.  Minor number is a function of
430 		 * the UAR page index (15 bits) and the device instance number
431 		 * (3 bits).
432 		 */
433 		dev = makedevice(getmajor(*devp), (tr_indx <<
434 		    TAVOR_MINORNUM_SHIFT) | instance);
435 
436 		/*
437 		 * Allocate another entry in the "userland resources database"
438 		 * to track the association of the device number (above) to
439 		 * the current process ID (in "key").
440 		 */
441 		umapdb2 = tavor_umap_db_alloc(instance, dev,
442 		    MLNX_UMAP_PID_RSRC, (uint64_t)key);
443 		if (umapdb2 == NULL) {
444 			mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
445 			tavor_umap_db_free(umapdb);
446 			/* If in "maintenance mode", don't free the rsrc */
447 			if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
448 				tavor_rsrc_free(state, &rsrcp);
449 			}
450 			TNF_PROBE_0(tavor_open_umap_db_alloc_fail,
451 			    TAVOR_TNF_ERROR, "");
452 			TAVOR_TNF_EXIT(tavor_open);
453 			return (EAGAIN);
454 		}
455 
456 		/* Add the entries to the database */
457 		tavor_umap_db_add_nolock(umapdb);
458 		tavor_umap_db_add_nolock(umapdb2);
459 
460 	} else {
461 		/*
462 		 * Return the same device number as on the original open()
463 		 * call.  This was calculated as a function of the UAR page
464 		 * index (top 16 bits) and the device instance number
465 		 */
466 		rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
467 		dev = makedevice(getmajor(*devp), (rsrcp->tr_indx <<
468 		    TAVOR_MINORNUM_SHIFT) | instance);
469 	}
470 	mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
471 
472 	*devp = dev;
473 
474 	TAVOR_TNF_EXIT(tavor_open);
475 	return (0);
476 }
477 
478 
479 /*
480  * tavor_close()
481  */
482 /* ARGSUSED */
483 static int
tavor_close(dev_t dev,int flag,int otyp,cred_t * credp)484 tavor_close(dev_t dev, int flag, int otyp, cred_t *credp)
485 {
486 	tavor_state_t		*state;
487 	tavor_rsrc_t		*rsrcp;
488 	tavor_umap_db_entry_t	*umapdb;
489 	tavor_umap_db_priv_t	*priv;
490 	minor_t			instance;
491 	uint64_t		key, value;
492 	int			status;
493 
494 	TAVOR_TNF_ENTER(tavor_close);
495 
496 	instance = TAVOR_DEV_INSTANCE(dev);
497 	state = ddi_get_soft_state(tavor_statep, instance);
498 	if (state == NULL) {
499 		TNF_PROBE_0(tavor_close_gss_fail, TAVOR_TNF_ERROR, "");
500 		TAVOR_TNF_EXIT(tavor_close);
501 		return (ENXIO);
502 	}
503 
504 	/*
505 	 * Search for "dev_t" in the "userland resources database".  As
506 	 * explained above in tavor_open(), we can't depend on using the
507 	 * current process ID here to do the lookup because the process
508 	 * that ultimately closes may not be the same one who opened
509 	 * (because of inheritance).
510 	 * So we lookup the "dev_t" (which points to the PID of the process
511 	 * that opened), and we remove the entry from the database (and free
512 	 * it up).  Then we do another query based on the PID value.  And when
513 	 * we find that database entry, we free it up too and then free the
514 	 * Tavor UAR page resource.
515 	 *
516 	 * Note: We use the tavor_umap_db_find_nolock() database access
517 	 * routine below (with an explicit mutex_enter of the database lock)
518 	 * to ensure that the multiple accesses (which attempt to remove the
519 	 * two database entries) can be done atomically.
520 	 *
521 	 * This works the same in both maintenance mode and HCA mode, except
522 	 * for the call to tavor_rsrc_free().  In the case of maintenance mode,
523 	 * this call is not needed, as it was not allocated in tavor_open()
524 	 * above.
525 	 */
526 	key = dev;
527 	mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock);
528 	status = tavor_umap_db_find_nolock(instance, key, MLNX_UMAP_PID_RSRC,
529 	    &value, TAVOR_UMAP_DB_REMOVE, &umapdb);
530 	if (status == DDI_SUCCESS) {
531 		/*
532 		 * If the "tdb_priv" field is non-NULL, it indicates that
533 		 * some "on close" handling is still necessary.  Call
534 		 * tavor_umap_db_handle_onclose_cb() to do the handling (i.e.
535 		 * to invoke all the registered callbacks).  Then free up
536 		 * the resources associated with "tdb_priv" and continue
537 		 * closing.
538 		 */
539 		priv = (tavor_umap_db_priv_t *)umapdb->tdbe_common.tdb_priv;
540 		if (priv != NULL) {
541 			tavor_umap_db_handle_onclose_cb(priv);
542 			kmem_free(priv, sizeof (tavor_umap_db_priv_t));
543 			umapdb->tdbe_common.tdb_priv = (void *)NULL;
544 		}
545 
546 		tavor_umap_db_free(umapdb);
547 
548 		/*
549 		 * Now do another lookup using PID as the key (copy it from
550 		 * "value").  When this lookup is complete, the "value" field
551 		 * will contain the tavor_rsrc_t pointer for the UAR page
552 		 * resource.
553 		 */
554 		key = value;
555 		status = tavor_umap_db_find_nolock(instance, key,
556 		    MLNX_UMAP_UARPG_RSRC, &value, TAVOR_UMAP_DB_REMOVE,
557 		    &umapdb);
558 		if (status == DDI_SUCCESS) {
559 			tavor_umap_db_free(umapdb);
560 			/* If in "maintenance mode", don't free the rsrc */
561 			if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
562 				rsrcp = (tavor_rsrc_t *)(uintptr_t)value;
563 				tavor_rsrc_free(state, &rsrcp);
564 			}
565 		}
566 	}
567 	mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock);
568 
569 	TAVOR_TNF_EXIT(tavor_close);
570 	return (0);
571 }
572 
573 
574 /*
575  * tavor_attach()
576  *    Context: Only called from attach() path context
577  */
578 static int
tavor_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)579 tavor_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
580 {
581 	tavor_state_t	*state;
582 	ibc_clnt_hdl_t	tmp_ibtfpriv;
583 	ibc_status_t	ibc_status;
584 	int		instance;
585 	int		status;
586 
587 	TAVOR_TNF_ENTER(tavor_attach);
588 
589 #ifdef __lock_lint
590 	(void) tavor_quiesce(dip);
591 #endif
592 
593 	switch (cmd) {
594 	case DDI_ATTACH:
595 		instance = ddi_get_instance(dip);
596 		status = ddi_soft_state_zalloc(tavor_statep, instance);
597 		if (status != DDI_SUCCESS) {
598 			TNF_PROBE_0(tavor_attach_ssz_fail, TAVOR_TNF_ERROR, "");
599 			cmn_err(CE_NOTE, "tavor%d: driver failed to attach: "
600 			    "attach_ssz_fail", instance);
601 			goto fail_attach_nomsg;
602 
603 		}
604 		state = ddi_get_soft_state(tavor_statep, instance);
605 		if (state == NULL) {
606 			ddi_soft_state_free(tavor_statep, instance);
607 			TNF_PROBE_0(tavor_attach_gss_fail, TAVOR_TNF_ERROR, "");
608 			cmn_err(CE_NOTE, "tavor%d: driver failed to attach: "
609 			    "attach_gss_fail", instance);
610 			goto fail_attach_nomsg;
611 		}
612 
613 		/* clear the attach error buffer */
614 		TAVOR_ATTACH_MSG_INIT(state->ts_attach_buf);
615 
616 		/*
617 		 * Initialize Tavor driver and hardware.
618 		 *
619 		 * Note: If this initialization fails we may still wish to
620 		 * create a device node and remain operational so that Tavor
621 		 * firmware can be updated/flashed (i.e. "maintenance mode").
622 		 * If this is the case, then "ts_operational_mode" will be
623 		 * equal to TAVOR_MAINTENANCE_MODE.  We will not attempt to
624 		 * attach to the IBTF or register with the IBMF (i.e. no
625 		 * InfiniBand interfaces will be enabled).
626 		 */
627 		status = tavor_drv_init(state, dip, instance);
628 		if ((status != DDI_SUCCESS) &&
629 		    (TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) {
630 			TNF_PROBE_0(tavor_attach_drvinit_fail,
631 			    TAVOR_TNF_ERROR, "");
632 			goto fail_attach;
633 		}
634 
635 		/* Create the minor node for device */
636 		status = ddi_create_minor_node(dip, "devctl", S_IFCHR, instance,
637 		    DDI_PSEUDO, 0);
638 		if (status != DDI_SUCCESS) {
639 			tavor_drv_fini(state);
640 			TAVOR_ATTACH_MSG(state->ts_attach_buf,
641 			    "attach_create_mn_fail");
642 			TNF_PROBE_0(tavor_attach_create_mn_fail,
643 			    TAVOR_TNF_ERROR, "");
644 			goto fail_attach;
645 		}
646 
647 		/*
648 		 * If we are in "maintenance mode", then we don't want to
649 		 * register with the IBTF.  All InfiniBand interfaces are
650 		 * uninitialized, and the device is only capable of handling
651 		 * requests to update/flash firmware (or test/debug requests).
652 		 */
653 		if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
654 
655 			/* Attach to InfiniBand Transport Framework (IBTF) */
656 			ibc_status = ibc_attach(&tmp_ibtfpriv,
657 			    &state->ts_ibtfinfo);
658 			if (ibc_status != IBC_SUCCESS) {
659 				ddi_remove_minor_node(dip, "devctl");
660 				tavor_drv_fini(state);
661 				TNF_PROBE_0(tavor_attach_ibcattach_fail,
662 				    TAVOR_TNF_ERROR, "");
663 				TAVOR_ATTACH_MSG(state->ts_attach_buf,
664 				    "attach_ibcattach_fail");
665 				goto fail_attach;
666 			}
667 
668 			/*
669 			 * Now that we've successfully attached to the IBTF,
670 			 * we enable all appropriate asynch and CQ events to
671 			 * be forwarded to the IBTF.
672 			 */
673 			TAVOR_ENABLE_IBTF_CALLB(state, tmp_ibtfpriv);
674 
675 			ibc_post_attach(state->ts_ibtfpriv);
676 
677 			/* Register agents with IB Mgmt Framework (IBMF) */
678 			status = tavor_agent_handlers_init(state);
679 			if (status != DDI_SUCCESS) {
680 				(void) ibc_pre_detach(tmp_ibtfpriv, DDI_DETACH);
681 				TAVOR_QUIESCE_IBTF_CALLB(state);
682 				if (state->ts_in_evcallb != 0) {
683 					TAVOR_WARNING(state, "unable to "
684 					    "quiesce Tavor IBTF callbacks");
685 				}
686 				ibc_detach(tmp_ibtfpriv);
687 				ddi_remove_minor_node(dip, "devctl");
688 				tavor_drv_fini(state);
689 				TNF_PROBE_0(tavor_attach_agentinit_fail,
690 				    TAVOR_TNF_ERROR, "");
691 				TAVOR_ATTACH_MSG(state->ts_attach_buf,
692 				    "attach_agentinit_fail");
693 				goto fail_attach;
694 			}
695 		}
696 
697 		/* Report that driver was loaded */
698 		ddi_report_dev(dip);
699 
700 		/* Send device information to log file */
701 		tavor_device_info_report(state);
702 
703 		/* Report attach in maintenance mode, if appropriate */
704 		if (!(TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) {
705 			cmn_err(CE_NOTE, "tavor%d: driver attached "
706 			    "(for maintenance mode only)", state->ts_instance);
707 		}
708 
709 		TAVOR_TNF_EXIT(tavor_attach);
710 		return (DDI_SUCCESS);
711 
712 	case DDI_RESUME:
713 		/* Add code here for DDI_RESUME XXX */
714 		TAVOR_TNF_EXIT(tavor_attach);
715 		return (DDI_FAILURE);
716 
717 	default:
718 		TNF_PROBE_0(tavor_attach_default_fail, TAVOR_TNF_ERROR, "");
719 		break;
720 	}
721 
722 fail_attach:
723 	cmn_err(CE_NOTE, "tavor%d: driver failed to attach: %s", instance,
724 	    state->ts_attach_buf);
725 	tavor_drv_fini2(state);
726 	ddi_soft_state_free(tavor_statep, instance);
727 fail_attach_nomsg:
728 	TAVOR_TNF_EXIT(tavor_attach);
729 	return (DDI_FAILURE);
730 }
731 
732 
733 /*
734  * tavor_detach()
735  *    Context: Only called from detach() path context
736  */
737 static int
tavor_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)738 tavor_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
739 {
740 	tavor_state_t	*state;
741 	ibc_clnt_hdl_t	tmp_ibtfpriv;
742 	ibc_status_t	ibc_status;
743 	int		instance, status;
744 
745 	TAVOR_TNF_ENTER(tavor_detach);
746 
747 	instance = ddi_get_instance(dip);
748 	state = ddi_get_soft_state(tavor_statep, instance);
749 	if (state == NULL) {
750 		TNF_PROBE_0(tavor_detach_gss_fail, TAVOR_TNF_ERROR, "");
751 		TAVOR_TNF_EXIT(tavor_detach);
752 		return (DDI_FAILURE);
753 	}
754 
755 	switch (cmd) {
756 	case DDI_DETACH:
757 		/*
758 		 * If we are in "maintenance mode", then we do not want to
759 		 * do teardown for any of the InfiniBand interfaces.
760 		 * Specifically, this means not detaching from IBTF (we never
761 		 * attached to begin with) and not deregistering from IBMF.
762 		 */
763 		if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
764 			/* Unregister agents from IB Mgmt Framework (IBMF) */
765 			status = tavor_agent_handlers_fini(state);
766 			if (status != DDI_SUCCESS) {
767 				TNF_PROBE_0(tavor_detach_agentfini_fail,
768 				    TAVOR_TNF_ERROR, "");
769 				TAVOR_TNF_EXIT(tavor_detach);
770 				return (DDI_FAILURE);
771 			}
772 
773 			/*
774 			 * Attempt the "pre-detach" from InfiniBand Transport
775 			 * Framework (IBTF).  At this point the IBTF is still
776 			 * capable of handling incoming asynch and completion
777 			 * events.  This "pre-detach" is primarily a mechanism
778 			 * to notify the appropriate IBTF clients that the
779 			 * HCA is being removed/offlined.
780 			 */
781 			ibc_status = ibc_pre_detach(state->ts_ibtfpriv, cmd);
782 			if (ibc_status != IBC_SUCCESS) {
783 				status = tavor_agent_handlers_init(state);
784 				if (status != DDI_SUCCESS) {
785 					TAVOR_WARNING(state, "failed to "
786 					    "restart Tavor agents");
787 				}
788 				TNF_PROBE_0(tavor_detach_ibcpredetach_fail,
789 				    TAVOR_TNF_ERROR, "");
790 				TAVOR_TNF_EXIT(tavor_detach);
791 				return (DDI_FAILURE);
792 			}
793 
794 			/*
795 			 * Before we can fully detach from the IBTF we need to
796 			 * ensure that we have handled all outstanding event
797 			 * callbacks.  This is accomplished by quiescing the
798 			 * event callback mechanism.  Note: if we are unable
799 			 * to successfully quiesce the callbacks, then this is
800 			 * an indication that something has probably gone
801 			 * seriously wrong.  We print out a warning, but
802 			 * continue.
803 			 */
804 			tmp_ibtfpriv = state->ts_ibtfpriv;
805 			TAVOR_QUIESCE_IBTF_CALLB(state);
806 			if (state->ts_in_evcallb != 0) {
807 				TAVOR_WARNING(state, "unable to quiesce Tavor "
808 				    "IBTF callbacks");
809 			}
810 
811 			/* Complete the detach from the IBTF */
812 			ibc_detach(tmp_ibtfpriv);
813 		}
814 
815 		/* Remove the minor node for device */
816 		ddi_remove_minor_node(dip, "devctl");
817 
818 		/*
819 		 * Only call tavor_drv_fini() if we are in Tavor HCA mode.
820 		 * (Because if we are in "maintenance mode", then we never
821 		 * successfully finished init.)  Only report successful
822 		 * detach for normal HCA mode.
823 		 */
824 		if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
825 			/* Cleanup driver resources and shutdown hardware */
826 			tavor_drv_fini(state);
827 			cmn_err(CE_CONT, "Tavor driver successfully "
828 			    "detached\n");
829 		}
830 
831 		tavor_drv_fini2(state);
832 		ddi_soft_state_free(tavor_statep, instance);
833 
834 		TAVOR_TNF_EXIT(tavor_detach);
835 		return (DDI_SUCCESS);
836 
837 	case DDI_SUSPEND:
838 		/* Add code here for DDI_SUSPEND XXX */
839 		TAVOR_TNF_EXIT(tavor_detach);
840 		return (DDI_FAILURE);
841 
842 	default:
843 		TNF_PROBE_0(tavor_detach_default_fail, TAVOR_TNF_ERROR, "");
844 		break;
845 	}
846 
847 	TAVOR_TNF_EXIT(tavor_detach);
848 	return (DDI_FAILURE);
849 }
850 
851 
852 /*
853  * tavor_drv_init()
854  *    Context: Only called from attach() path context
855  */
856 static int
tavor_drv_init(tavor_state_t * state,dev_info_t * dip,int instance)857 tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance)
858 {
859 	int			status;
860 
861 	TAVOR_TNF_ENTER(tavor_drv_init);
862 
863 	/* Save away devinfo and instance */
864 	state->ts_dip = dip;
865 	state->ts_instance = instance;
866 
867 	/*
868 	 * Check and set the operational mode of the device. If the driver is
869 	 * bound to the Tavor device in "maintenance mode", then this generally
870 	 * means that either the device has been specifically jumpered to
871 	 * start in this mode or the firmware boot process has failed to
872 	 * successfully load either the primary or the secondary firmware
873 	 * image.
874 	 */
875 	if (TAVOR_IS_HCA_MODE(state->ts_dip)) {
876 		state->ts_operational_mode = TAVOR_HCA_MODE;
877 
878 	} else if (TAVOR_IS_COMPAT_MODE(state->ts_dip)) {
879 		state->ts_operational_mode = TAVOR_COMPAT_MODE;
880 
881 	} else if (TAVOR_IS_MAINTENANCE_MODE(state->ts_dip)) {
882 		state->ts_operational_mode = TAVOR_MAINTENANCE_MODE;
883 		return (DDI_FAILURE);
884 
885 	} else {
886 		state->ts_operational_mode = 0;	/* invalid operational mode */
887 		TAVOR_WARNING(state, "unexpected device type detected");
888 		TNF_PROBE_0(tavor_hw_init_unexpected_dev_fail,
889 		    TAVOR_TNF_ERROR, "");
890 		TAVOR_TNF_EXIT(tavor_hw_init);
891 		return (DDI_FAILURE);
892 	}
893 
894 	/*
895 	 * Initialize the Tavor hardware.
896 	 * Note:  If this routine returns an error, it is often an reasonably
897 	 * good indication that something Tavor firmware-related has caused
898 	 * the failure.  In order to give the user an opportunity (if desired)
899 	 * to update or reflash the Tavor firmware image, we set
900 	 * "ts_operational_mode" flag (described above) to indicate that we
901 	 * wish to enter maintenance mode.
902 	 */
903 	status = tavor_hw_init(state);
904 	if (status != DDI_SUCCESS) {
905 		state->ts_operational_mode = TAVOR_MAINTENANCE_MODE;
906 		cmn_err(CE_NOTE, "tavor%d: error during attach: %s", instance,
907 		    state->ts_attach_buf);
908 		TNF_PROBE_0(tavor_drv_init_hwinit_fail, TAVOR_TNF_ERROR, "");
909 		TAVOR_TNF_EXIT(tavor_drv_init);
910 		return (DDI_FAILURE);
911 	}
912 
913 	/* Setup Tavor interrupt handler */
914 	status = tavor_isr_init(state);
915 	if (status != DDI_SUCCESS) {
916 		tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
917 		TNF_PROBE_0(tavor_drv_init_isrinit_fail, TAVOR_TNF_ERROR, "");
918 		TAVOR_TNF_EXIT(tavor_drv_init);
919 		return (DDI_FAILURE);
920 	}
921 
922 	/* Initialize Tavor softstate */
923 	status = tavor_soft_state_init(state);
924 	if (status != DDI_SUCCESS) {
925 		tavor_isr_fini(state);
926 		tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
927 		TNF_PROBE_0(tavor_drv_init_ssiinit_fail, TAVOR_TNF_ERROR, "");
928 		TAVOR_TNF_EXIT(tavor_drv_init);
929 		return (DDI_FAILURE);
930 	}
931 
932 	TAVOR_TNF_EXIT(tavor_drv_init);
933 	return (DDI_SUCCESS);
934 }
935 
936 
937 /*
938  * tavor_drv_fini()
939  *    Context: Only called from attach() and/or detach() path contexts
940  */
941 static void
tavor_drv_fini(tavor_state_t * state)942 tavor_drv_fini(tavor_state_t *state)
943 {
944 	TAVOR_TNF_ENTER(tavor_drv_fini);
945 
946 	/* Cleanup Tavor softstate */
947 	tavor_soft_state_fini(state);
948 
949 	/* Teardown Tavor interrupts */
950 	tavor_isr_fini(state);
951 
952 	/* Cleanup Tavor resources and shutdown hardware */
953 	tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL);
954 
955 	TAVOR_TNF_EXIT(tavor_drv_fini);
956 }
957 
958 /*
959  * tavor_drv_fini2()
960  *    Context: Only called from attach() and/or detach() path contexts
961  */
962 static void
tavor_drv_fini2(tavor_state_t * state)963 tavor_drv_fini2(tavor_state_t *state)
964 {
965 	TAVOR_TNF_ENTER(tavor_drv_fini2);
966 
967 	/* TAVOR_DRV_CLEANUP_LEVEL1 */
968 	if (state->ts_reg_cmdhdl) {
969 		ddi_regs_map_free(&state->ts_reg_cmdhdl);
970 		state->ts_reg_cmdhdl = NULL;
971 	}
972 
973 	/* TAVOR_DRV_CLEANUP_LEVEL0 */
974 	if (state->ts_pci_cfghdl) {
975 		pci_config_teardown(&state->ts_pci_cfghdl);
976 		state->ts_pci_cfghdl = NULL;
977 	}
978 
979 	TAVOR_TNF_EXIT(tavor_drv_fini2);
980 }
981 
982 /*
983  * tavor_isr_init()
984  *    Context: Only called from attach() path context
985  */
986 static int
tavor_isr_init(tavor_state_t * state)987 tavor_isr_init(tavor_state_t *state)
988 {
989 	int	status;
990 
991 	TAVOR_TNF_ENTER(tavor_isr_init);
992 
993 	/*
994 	 * Add a handler for the interrupt or MSI
995 	 */
996 	status = ddi_intr_add_handler(state->ts_intrmsi_hdl, tavor_isr,
997 	    (caddr_t)state, NULL);
998 	if (status  != DDI_SUCCESS) {
999 		TNF_PROBE_0(tavor_isr_init_addhndlr_fail, TAVOR_TNF_ERROR, "");
1000 		TAVOR_TNF_EXIT(tavor_isr_init);
1001 		return (DDI_FAILURE);
1002 	}
1003 
1004 	/*
1005 	 * Enable the software interrupt.  Note: Even though we are only
1006 	 * using one (1) interrupt/MSI, depending on the value returned in
1007 	 * the capability flag, we have to call either ddi_intr_block_enable()
1008 	 * or ddi_intr_enable().
1009 	 */
1010 	if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
1011 		status = ddi_intr_block_enable(&state->ts_intrmsi_hdl, 1);
1012 		if (status  != DDI_SUCCESS) {
1013 			TNF_PROBE_0(tavor_isr_init_blockenable_fail,
1014 			    TAVOR_TNF_ERROR, "");
1015 			TAVOR_TNF_EXIT(tavor_isr_init);
1016 			return (DDI_FAILURE);
1017 		}
1018 	} else {
1019 		status = ddi_intr_enable(state->ts_intrmsi_hdl);
1020 		if (status  != DDI_SUCCESS) {
1021 			TNF_PROBE_0(tavor_isr_init_intrenable_fail,
1022 			    TAVOR_TNF_ERROR, "");
1023 			TAVOR_TNF_EXIT(tavor_isr_init);
1024 			return (DDI_FAILURE);
1025 		}
1026 	}
1027 
1028 	/*
1029 	 * Now that the ISR has been setup, arm all the EQs for event
1030 	 * generation.
1031 	 */
1032 	tavor_eq_arm_all(state);
1033 
1034 	TAVOR_TNF_EXIT(tavor_isr_init);
1035 	return (DDI_SUCCESS);
1036 }
1037 
1038 
1039 /*
1040  * tavor_isr_fini()
1041  *    Context: Only called from attach() and/or detach() path contexts
1042  */
1043 static void
tavor_isr_fini(tavor_state_t * state)1044 tavor_isr_fini(tavor_state_t *state)
1045 {
1046 	TAVOR_TNF_ENTER(tavor_isr_fini);
1047 
1048 	/* Disable the software interrupt */
1049 	if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) {
1050 		(void) ddi_intr_block_disable(&state->ts_intrmsi_hdl, 1);
1051 	} else {
1052 		(void) ddi_intr_disable(state->ts_intrmsi_hdl);
1053 	}
1054 
1055 	/*
1056 	 * Remove the software handler for the interrupt or MSI
1057 	 */
1058 	(void) ddi_intr_remove_handler(state->ts_intrmsi_hdl);
1059 
1060 	TAVOR_TNF_EXIT(tavor_isr_fini);
1061 }
1062 
1063 
1064 /*
1065  * tavor_fix_error_buf()
1066  *	Context: Only called from attach().
1067  *
1068  * The error_buf_addr returned from QUERY_FW is a PCI address.
1069  * We need to convert it to an offset from the base address,
1070  * which is stored in the assigned-addresses property.
1071  */
1072 static int
tavor_fix_error_buf(tavor_state_t * state)1073 tavor_fix_error_buf(tavor_state_t *state)
1074 {
1075 	int		assigned_addr_len;
1076 	pci_regspec_t	*assigned_addr;
1077 
1078 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->ts_dip,
1079 	    DDI_PROP_DONTPASS, "assigned-addresses", (int **)&assigned_addr,
1080 	    (uint_t *)&assigned_addr_len) != DDI_PROP_SUCCESS)
1081 		return (DDI_FAILURE);
1082 
1083 	state->ts_fw.error_buf_addr -= assigned_addr[0].pci_phys_low +
1084 	    ((uint64_t)(assigned_addr[0].pci_phys_mid) << 32);
1085 	ddi_prop_free(assigned_addr);
1086 	return (DDI_SUCCESS);
1087 }
1088 
1089 /*
1090  * tavor_hw_init()
1091  *    Context: Only called from attach() path context
1092  */
1093 static int
tavor_hw_init(tavor_state_t * state)1094 tavor_hw_init(tavor_state_t *state)
1095 {
1096 	tavor_drv_cleanup_level_t	cleanup;
1097 	sm_nodeinfo_t			nodeinfo;
1098 	uint64_t			errorcode;
1099 	off_t				ddr_size;
1100 	int				status;
1101 	int				retries;
1102 
1103 	TAVOR_TNF_ENTER(tavor_hw_init);
1104 
1105 	/* This is where driver initialization begins */
1106 	cleanup = TAVOR_DRV_CLEANUP_LEVEL0;
1107 
1108 	/* Setup device access attributes */
1109 	state->ts_reg_accattr.devacc_attr_version = DDI_DEVICE_ATTR_V0;
1110 	state->ts_reg_accattr.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC;
1111 	state->ts_reg_accattr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
1112 
1113 	/* Setup for PCI config read/write of HCA device  */
1114 	status = pci_config_setup(state->ts_dip, &state->ts_pci_cfghdl);
1115 	if (status != DDI_SUCCESS) {
1116 		tavor_hw_fini(state, cleanup);
1117 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1118 		    "hw_init_PCI_config_space_regmap_fail");
1119 		/* This case is not the degraded one */
1120 		return (DDI_FAILURE);
1121 	}
1122 
1123 	/* Map in Tavor registers (CMD, UAR, DDR) and setup offsets */
1124 	status = ddi_regs_map_setup(state->ts_dip, TAVOR_CMD_BAR,
1125 	    &state->ts_reg_cmd_baseaddr, 0, 0, &state->ts_reg_accattr,
1126 	    &state->ts_reg_cmdhdl);
1127 	if (status != DDI_SUCCESS) {
1128 		tavor_hw_fini(state, cleanup);
1129 		TNF_PROBE_0(tavor_hw_init_CMD_ddirms_fail, TAVOR_TNF_ERROR, "");
1130 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1131 		    "hw_init_CMD_ddirms_fail");
1132 		TAVOR_TNF_EXIT(tavor_hw_init);
1133 		return (DDI_FAILURE);
1134 	}
1135 	cleanup = TAVOR_DRV_CLEANUP_LEVEL1;
1136 
1137 	status = ddi_regs_map_setup(state->ts_dip, TAVOR_UAR_BAR,
1138 	    &state->ts_reg_uar_baseaddr, 0, 0, &state->ts_reg_accattr,
1139 	    &state->ts_reg_uarhdl);
1140 	if (status != DDI_SUCCESS) {
1141 		tavor_hw_fini(state, cleanup);
1142 		TNF_PROBE_0(tavor_hw_init_UAR_ddirms_fail, TAVOR_TNF_ERROR, "");
1143 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1144 		    "hw_init_UAR_ddirms_fail");
1145 		TAVOR_TNF_EXIT(tavor_hw_init);
1146 		return (DDI_FAILURE);
1147 	}
1148 	cleanup = TAVOR_DRV_CLEANUP_LEVEL2;
1149 
1150 	status = ddi_dev_regsize(state->ts_dip, TAVOR_DDR_BAR, &ddr_size);
1151 	if (status != DDI_SUCCESS) {
1152 		cmn_err(CE_CONT, "Tavor: ddi_dev_regsize() failed "
1153 		    "(check HCA-attached DIMM memory?)\n");
1154 		tavor_hw_fini(state, cleanup);
1155 		TNF_PROBE_0(tavor_hw_init_DDR_ddi_regsize_fail,
1156 		    TAVOR_TNF_ERROR, "");
1157 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1158 		    "hw_init_DDR_ddi_regsize_fail");
1159 		TAVOR_TNF_EXIT(tavor_hw_init);
1160 		return (DDI_FAILURE);
1161 	}
1162 
1163 #if !defined(_ELF64) && !defined(__sparc)
1164 	/*
1165 	 * For 32 bit x86/x64 kernels, where there is limited kernel virtual
1166 	 * memory available, define a minimal memory footprint. This is
1167 	 * specified in order to not take up too much resources, thus starving
1168 	 * out others. Only specified if the HCA DIMM is equal to or greater
1169 	 * than 256MB.
1170 	 *
1171 	 * Note: x86/x64 install and safemode boot are both 32bit.
1172 	 */
1173 	ddr_size = TAVOR_DDR_SIZE_MIN;
1174 #endif	/* !(_ELF64) && !(__sparc) */
1175 
1176 	state->ts_cfg_profile_setting = ddr_size;
1177 
1178 	status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR,
1179 	    &state->ts_reg_ddr_baseaddr, 0, ddr_size, &state->ts_reg_accattr,
1180 	    &state->ts_reg_ddrhdl);
1181 
1182 	/*
1183 	 * On 32-bit platform testing (primarily x86), it was seen that the
1184 	 * ddi_regs_map_setup() call would fail because there wasn't enough
1185 	 * kernel virtual address space available to map in the entire 256MB
1186 	 * DDR.  So we add this check in here, so that if the 256 (or other
1187 	 * larger value of DDR) map in fails, that we fallback to try the lower
1188 	 * size of 128MB.
1189 	 *
1190 	 * Note: If we only have 128MB of DDR in the system in the first place,
1191 	 * we don't try another ddi_regs_map_setup(), and just skip over this
1192 	 * check and return failures.
1193 	 */
1194 	if (status == DDI_ME_NORESOURCES && ddr_size > TAVOR_DDR_SIZE_128) {
1195 		/* Try falling back to 128MB DDR mapping */
1196 		status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR,
1197 		    &state->ts_reg_ddr_baseaddr, 0, TAVOR_DDR_SIZE_128,
1198 		    &state->ts_reg_accattr, &state->ts_reg_ddrhdl);
1199 
1200 		/*
1201 		 * 128MB DDR mapping worked.
1202 		 * Set the updated config profile setting here.
1203 		 */
1204 		if (status == DDI_SUCCESS) {
1205 			TNF_PROBE_0(tavor_hw_init_DDR_128mb_fallback_success,
1206 			    TAVOR_TNF_TRACE, "");
1207 			state->ts_cfg_profile_setting = TAVOR_DDR_SIZE_128;
1208 		}
1209 	}
1210 
1211 	if (status != DDI_SUCCESS) {
1212 		if (status == DDI_ME_RNUMBER_RANGE) {
1213 			cmn_err(CE_CONT, "Tavor: ddi_regs_map_setup() failed "
1214 			    "(check HCA-attached DIMM memory?)\n");
1215 		}
1216 		tavor_hw_fini(state, cleanup);
1217 		TNF_PROBE_0(tavor_hw_init_DDR_ddirms_fail, TAVOR_TNF_ERROR, "");
1218 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1219 		    "hw_init_DDR_ddirms_fail");
1220 		TAVOR_TNF_EXIT(tavor_hw_init);
1221 		return (DDI_FAILURE);
1222 	}
1223 	cleanup = TAVOR_DRV_CLEANUP_LEVEL3;
1224 
1225 	/* Setup Tavor Host Command Register (HCR) */
1226 	state->ts_cmd_regs.hcr = (tavor_hw_hcr_t *)
1227 	    ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_HCR_OFFSET);
1228 
1229 	/* Setup Tavor Event Cause Register (ecr and clr_ecr) */
1230 	state->ts_cmd_regs.ecr = (uint64_t *)
1231 	    ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_ECR_OFFSET);
1232 	state->ts_cmd_regs.clr_ecr = (uint64_t *)
1233 	    ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_ECR_OFFSET);
1234 
1235 	/* Setup Tavor Software Reset register (sw_reset) */
1236 	state->ts_cmd_regs.sw_reset = (uint32_t *)
1237 	    ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_SW_RESET_OFFSET);
1238 
1239 	/* Setup Tavor Clear Interrupt register (clr_int) */
1240 	state->ts_cmd_regs.clr_int = (uint64_t *)
1241 	    ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_INT_OFFSET);
1242 
1243 	/* Initialize the Phase1 Tavor configuration profile */
1244 	status = tavor_cfg_profile_init_phase1(state);
1245 	if (status != DDI_SUCCESS) {
1246 		tavor_hw_fini(state, cleanup);
1247 		TNF_PROBE_0(tavor_hw_init_cfginit_fail, TAVOR_TNF_ERROR, "");
1248 		TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit_fail");
1249 		TAVOR_TNF_EXIT(tavor_hw_init);
1250 		return (DDI_FAILURE);
1251 	}
1252 	cleanup = TAVOR_DRV_CLEANUP_LEVEL4;
1253 
1254 	/* Do a software reset of the Tavor HW to ensure proper state */
1255 	status = tavor_sw_reset(state);
1256 	if (status != TAVOR_CMD_SUCCESS) {
1257 		tavor_hw_fini(state, cleanup);
1258 		TNF_PROBE_0(tavor_hw_init_sw_reset_fail, TAVOR_TNF_ERROR, "");
1259 		TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_sw_reset_fail");
1260 		TAVOR_TNF_EXIT(tavor_hw_init);
1261 		return (DDI_FAILURE);
1262 	}
1263 
1264 	/* Post the SYS_EN command to start the hardware */
1265 	status = tavor_sys_en_cmd_post(state, TAVOR_CMD_SYS_EN_NORMAL,
1266 	    &errorcode, TAVOR_CMD_NOSLEEP_SPIN);
1267 	if (status != TAVOR_CMD_SUCCESS) {
1268 		if ((status == TAVOR_CMD_BAD_NVMEM) ||
1269 		    (status == TAVOR_CMD_DDR_MEM_ERR)) {
1270 			cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x "
1271 			    "0x%" PRIx64 " (invalid firmware image?)\n",
1272 			    status, errorcode);
1273 		} else {
1274 			cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x "
1275 			    "0x%" PRIx64 "\n", status, errorcode);
1276 		}
1277 		tavor_hw_fini(state, cleanup);
1278 		TNF_PROBE_0(tavor_hw_init_sys_en_cmd_fail,
1279 		    TAVOR_TNF_ERROR, "");
1280 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1281 		    "hw_init_sys_en_cmd_fail");
1282 		TAVOR_TNF_EXIT(tavor_hw_init);
1283 		return (DDI_FAILURE);
1284 	}
1285 	cleanup = TAVOR_DRV_CLEANUP_LEVEL5;
1286 
1287 	/* First phase of init for Tavor configuration/resources */
1288 	status = tavor_rsrc_init_phase1(state);
1289 	if (status != DDI_SUCCESS) {
1290 		tavor_hw_fini(state, cleanup);
1291 		TNF_PROBE_0(tavor_hw_init_rsrcinit1_fail, TAVOR_TNF_ERROR, "");
1292 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1293 		    "hw_init_rsrcinit1_fail");
1294 		TAVOR_TNF_EXIT(tavor_hw_init);
1295 		return (DDI_FAILURE);
1296 	}
1297 	cleanup = TAVOR_DRV_CLEANUP_LEVEL6;
1298 
1299 	/* Query the DDR properties (e.g. total DDR size) */
1300 	status = tavor_cmn_query_cmd_post(state, QUERY_DDR, 0,
1301 	    &state->ts_ddr, sizeof (tavor_hw_queryddr_t),
1302 	    TAVOR_CMD_NOSLEEP_SPIN);
1303 	if (status != TAVOR_CMD_SUCCESS) {
1304 		cmn_err(CE_CONT, "Tavor: QUERY_DDR command failed: %08x\n",
1305 		    status);
1306 		tavor_hw_fini(state, cleanup);
1307 		TNF_PROBE_0(tavor_hw_init_query_ddr_cmd_fail,
1308 		    TAVOR_TNF_ERROR, "");
1309 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1310 		    "hw_init_query_ddr_cmd_fail");
1311 		TAVOR_TNF_EXIT(tavor_hw_init);
1312 		return (DDI_FAILURE);
1313 	}
1314 
1315 	/* Figure out how big the firmware image (in DDR) is */
1316 	status = tavor_cmn_query_cmd_post(state, QUERY_FW, 0, &state->ts_fw,
1317 	    sizeof (tavor_hw_queryfw_t), TAVOR_CMD_NOSLEEP_SPIN);
1318 	if (status != TAVOR_CMD_SUCCESS) {
1319 		cmn_err(CE_CONT, "Tavor: QUERY_FW command failed: %08x\n",
1320 		    status);
1321 		tavor_hw_fini(state, cleanup);
1322 		TNF_PROBE_0(tavor_hw_init_query_fw_cmd_fail,
1323 		    TAVOR_TNF_ERROR, "");
1324 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1325 		    "hw_init_query_fw_cmd_fail");
1326 		TAVOR_TNF_EXIT(tavor_hw_init);
1327 		return (DDI_FAILURE);
1328 	}
1329 
1330 	if (tavor_fix_error_buf(state) != DDI_SUCCESS) {
1331 		tavor_hw_fini(state, cleanup);
1332 		TNF_PROBE_0(tavor_hw_init_fixerrorbuf_fail,
1333 		    TAVOR_TNF_ERROR, "");
1334 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1335 		    "hw_init_fixerrorbuf_fail");
1336 		TAVOR_TNF_EXIT(tavor_hw_init);
1337 		return (DDI_FAILURE);
1338 	}
1339 
1340 	/* Validate that the FW version is appropriate */
1341 	status = tavor_fw_version_check(state);
1342 	if (status != DDI_SUCCESS) {
1343 		if (state->ts_operational_mode == TAVOR_HCA_MODE) {
1344 			cmn_err(CE_CONT, "Unsupported Tavor FW version: "
1345 			    "expected: %04d.%04d.%04d, "
1346 			    "actual: %04d.%04d.%04d\n",
1347 			    TAVOR_FW_VER_MAJOR,
1348 			    TAVOR_FW_VER_MINOR,
1349 			    TAVOR_FW_VER_SUBMINOR,
1350 			    state->ts_fw.fw_rev_major,
1351 			    state->ts_fw.fw_rev_minor,
1352 			    state->ts_fw.fw_rev_subminor);
1353 		} else if (state->ts_operational_mode == TAVOR_COMPAT_MODE) {
1354 			cmn_err(CE_CONT, "Unsupported Tavor Compat FW version: "
1355 			    "expected: %04d.%04d.%04d, "
1356 			    "actual: %04d.%04d.%04d\n",
1357 			    TAVOR_COMPAT_FW_VER_MAJOR,
1358 			    TAVOR_COMPAT_FW_VER_MINOR,
1359 			    TAVOR_COMPAT_FW_VER_SUBMINOR,
1360 			    state->ts_fw.fw_rev_major,
1361 			    state->ts_fw.fw_rev_minor,
1362 			    state->ts_fw.fw_rev_subminor);
1363 		} else {
1364 			cmn_err(CE_CONT, "Unsupported FW version: "
1365 			    "%04d.%04d.%04d\n",
1366 			    state->ts_fw.fw_rev_major,
1367 			    state->ts_fw.fw_rev_minor,
1368 			    state->ts_fw.fw_rev_subminor);
1369 		}
1370 		tavor_hw_fini(state, cleanup);
1371 		TNF_PROBE_0(tavor_hw_init_checkfwver_fail,
1372 		    TAVOR_TNF_ERROR, "");
1373 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1374 		    "hw_init_checkfwver_fail");
1375 		TAVOR_TNF_EXIT(tavor_hw_init);
1376 		return (DDI_FAILURE);
1377 	}
1378 
1379 	drv_usecwait(10);
1380 	retries = 1000;		/* retry up to 1 second before giving up */
1381 retry:
1382 	/* Call MOD_STAT_CFG to setup SRQ support (or disable) */
1383 	status = tavor_mod_stat_cfg_cmd_post(state);
1384 	if (status != DDI_SUCCESS) {
1385 		if (retries > 0) {
1386 			drv_usecwait(1000);
1387 			retries--;
1388 			goto retry;
1389 		}
1390 		cmn_err(CE_CONT, "Tavor: MOD_STAT_CFG command failed: %08x\n",
1391 		    status);
1392 		tavor_hw_fini(state, cleanup);
1393 		TNF_PROBE_0(tavor_hw_init_mod_stat_cfg_cmd_fail,
1394 		    TAVOR_TNF_ERROR, "");
1395 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1396 		    "hw_init_mod_stat_cfg_cmd_fail");
1397 		TAVOR_TNF_EXIT(tavor_hw_init);
1398 		return (DDI_FAILURE);
1399 	}
1400 
1401 	/* Figure out Tavor device limits */
1402 	status = tavor_cmn_query_cmd_post(state, QUERY_DEV_LIM, 0,
1403 	    &state->ts_devlim, sizeof (tavor_hw_querydevlim_t),
1404 	    TAVOR_CMD_NOSLEEP_SPIN);
1405 	if (status != TAVOR_CMD_SUCCESS) {
1406 		cmn_err(CE_CONT, "Tavor: QUERY_DEV_LIM command failed: %08x\n",
1407 		    status);
1408 		tavor_hw_fini(state, cleanup);
1409 		TNF_PROBE_0(tavor_hw_init_query_devlim_cmd_fail,
1410 		    TAVOR_TNF_ERROR, "");
1411 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1412 		    "hw_init_query_devlim_cmd_fail");
1413 		TAVOR_TNF_EXIT(tavor_hw_init);
1414 		return (DDI_FAILURE);
1415 	}
1416 
1417 	/* Initialize the Phase2 Tavor configuration profile */
1418 	status = tavor_cfg_profile_init_phase2(state);
1419 	if (status != DDI_SUCCESS) {
1420 		tavor_hw_fini(state, cleanup);
1421 		TNF_PROBE_0(tavor_hw_init_cfginit2_fail, TAVOR_TNF_ERROR, "");
1422 		TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit2_fail");
1423 		TAVOR_TNF_EXIT(tavor_hw_init);
1424 		return (DDI_FAILURE);
1425 	}
1426 
1427 	/* Second phase of init for Tavor configuration/resources */
1428 	status = tavor_rsrc_init_phase2(state);
1429 	if (status != DDI_SUCCESS) {
1430 		tavor_hw_fini(state, cleanup);
1431 		TNF_PROBE_0(tavor_hw_init_rsrcinit2_fail, TAVOR_TNF_ERROR, "");
1432 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1433 		    "hw_init_rsrcinit2_fail");
1434 		TAVOR_TNF_EXIT(tavor_hw_init);
1435 		return (DDI_FAILURE);
1436 	}
1437 	cleanup = TAVOR_DRV_CLEANUP_LEVEL7;
1438 
1439 	/* Miscellaneous query information */
1440 	status = tavor_cmn_query_cmd_post(state, QUERY_ADAPTER, 0,
1441 	    &state->ts_adapter, sizeof (tavor_hw_queryadapter_t),
1442 	    TAVOR_CMD_NOSLEEP_SPIN);
1443 	if (status != TAVOR_CMD_SUCCESS) {
1444 		cmn_err(CE_CONT, "Tavor: QUERY_ADAPTER command failed: %08x\n",
1445 		    status);
1446 		tavor_hw_fini(state, cleanup);
1447 		TNF_PROBE_0(tavor_hw_init_query_adapter_cmd_fail,
1448 		    TAVOR_TNF_ERROR, "");
1449 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1450 		    "hw_init_query_adapter_cmd_fail");
1451 		TAVOR_TNF_EXIT(tavor_hw_init);
1452 		return (DDI_FAILURE);
1453 	}
1454 
1455 	/* Prepare configuration for Tavor INIT_HCA command */
1456 	tavor_hca_config_setup(state, &state->ts_hcaparams);
1457 
1458 	/* Post command to init Tavor HCA */
1459 	status = tavor_init_hca_cmd_post(state, &state->ts_hcaparams,
1460 	    TAVOR_CMD_NOSLEEP_SPIN);
1461 	if (status != TAVOR_CMD_SUCCESS) {
1462 		cmn_err(CE_CONT, "Tavor: INIT_HCA command failed: %08x\n",
1463 		    status);
1464 		tavor_hw_fini(state, cleanup);
1465 		TNF_PROBE_0(tavor_hw_init_init_hca_cmd_fail,
1466 		    TAVOR_TNF_ERROR, "");
1467 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1468 		    "hw_init_init_hca_cmd_fail");
1469 		TAVOR_TNF_EXIT(tavor_hw_init);
1470 		return (DDI_FAILURE);
1471 	}
1472 	cleanup = TAVOR_DRV_CLEANUP_LEVEL8;
1473 
1474 	/* Allocate protection domain (PD) for Tavor internal use */
1475 	status = tavor_pd_alloc(state, &state->ts_pdhdl_internal, TAVOR_SLEEP);
1476 	if (status != DDI_SUCCESS) {
1477 		tavor_hw_fini(state, cleanup);
1478 		TNF_PROBE_0(tavor_hw_init_internal_pd_alloc_fail,
1479 		    TAVOR_TNF_ERROR, "");
1480 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1481 		    "hw_init_internal_pd_alloc_fail");
1482 		TAVOR_TNF_EXIT(tavor_hw_init);
1483 		return (DDI_FAILURE);
1484 	}
1485 	cleanup = TAVOR_DRV_CLEANUP_LEVEL9;
1486 
1487 	/* Setup Tavor internal UAR pages (0 and 1) */
1488 	status = tavor_internal_uarpgs_init(state);
1489 	if (status != DDI_SUCCESS) {
1490 		tavor_hw_fini(state, cleanup);
1491 		TNF_PROBE_0(tavor_hw_init_internal_uarpgs_alloc_fail,
1492 		    TAVOR_TNF_ERROR, "");
1493 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1494 		    "hw_init_internal_uarpgs_alloc_fail");
1495 		TAVOR_TNF_EXIT(tavor_hw_init);
1496 		return (DDI_FAILURE);
1497 	}
1498 	cleanup = TAVOR_DRV_CLEANUP_LEVEL10;
1499 
1500 	/* Query and initialize the Tavor interrupt/MSI information */
1501 	status = tavor_intr_or_msi_init(state);
1502 	if (status != DDI_SUCCESS) {
1503 		tavor_hw_fini(state, cleanup);
1504 		TNF_PROBE_0(tavor_intr_or_msi_init_fail,
1505 		    TAVOR_TNF_ERROR, "");
1506 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1507 		    "intr_or_msi_init_fail");
1508 		TAVOR_TNF_EXIT(tavor_hw_init);
1509 		return (DDI_FAILURE);
1510 	}
1511 	cleanup = TAVOR_DRV_CLEANUP_LEVEL11;
1512 
1513 	/* Setup all of the Tavor EQs */
1514 	status = tavor_eq_init_all(state);
1515 	if (status != DDI_SUCCESS) {
1516 		tavor_hw_fini(state, cleanup);
1517 		TNF_PROBE_0(tavor_hw_init_eqinitall_fail, TAVOR_TNF_ERROR, "");
1518 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1519 		    "hw_init_eqinitall_fail");
1520 		TAVOR_TNF_EXIT(tavor_hw_init);
1521 		return (DDI_FAILURE);
1522 	}
1523 	cleanup = TAVOR_DRV_CLEANUP_LEVEL12;
1524 
1525 	/* Set aside contexts for QP0 and QP1 */
1526 	status = tavor_special_qp_contexts_reserve(state);
1527 	if (status != DDI_SUCCESS) {
1528 		tavor_hw_fini(state, cleanup);
1529 		TNF_PROBE_0(tavor_hw_init_reserve_special_qp_fail,
1530 		    TAVOR_TNF_ERROR, "");
1531 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1532 		    "hw_init_reserve_special_qp_fail");
1533 		TAVOR_TNF_EXIT(tavor_hw_init);
1534 		return (DDI_FAILURE);
1535 	}
1536 	cleanup = TAVOR_DRV_CLEANUP_LEVEL13;
1537 
1538 	/* Initialize for multicast group handling */
1539 	status = tavor_mcg_init(state);
1540 	if (status != DDI_SUCCESS) {
1541 		tavor_hw_fini(state, cleanup);
1542 		TNF_PROBE_0(tavor_hw_init_mcg_init_fail, TAVOR_TNF_ERROR, "");
1543 		TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_mcg_init_fail");
1544 		TAVOR_TNF_EXIT(tavor_hw_init);
1545 		return (DDI_FAILURE);
1546 	}
1547 	cleanup = TAVOR_DRV_CLEANUP_LEVEL14;
1548 
1549 	/* Initialize the Tavor IB port(s) */
1550 	status = tavor_hca_port_init(state);
1551 	if (status != DDI_SUCCESS) {
1552 		tavor_hw_fini(state, cleanup);
1553 		TNF_PROBE_0(tavor_hw_init_hca_port_init_fail,
1554 		    TAVOR_TNF_ERROR, "");
1555 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1556 		    "hw_init_hca_port_init_fail");
1557 		TAVOR_TNF_EXIT(tavor_hw_init);
1558 		return (DDI_FAILURE);
1559 	}
1560 	cleanup = TAVOR_DRV_CLEANUP_ALL;
1561 
1562 	/* Determine NodeGUID and SystemImageGUID */
1563 	status = tavor_getnodeinfo_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN,
1564 	    &nodeinfo);
1565 	if (status != TAVOR_CMD_SUCCESS) {
1566 		cmn_err(CE_CONT, "Tavor: GetNodeInfo command failed: %08x\n",
1567 		    status);
1568 		tavor_hw_fini(state, cleanup);
1569 		TNF_PROBE_0(tavor_hw_init_getnodeinfo_cmd_fail,
1570 		    TAVOR_TNF_ERROR, "");
1571 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1572 		    "hw_init_getnodeinfo_cmd_fail");
1573 		TAVOR_TNF_EXIT(tavor_hw_init);
1574 		return (DDI_FAILURE);
1575 	}
1576 
1577 	/*
1578 	 * If the NodeGUID value was set in OBP properties, then we use that
1579 	 * value.  But we still print a message if the value we queried from
1580 	 * firmware does not match this value.
1581 	 *
1582 	 * Otherwise if OBP value is not set then we use the value from
1583 	 * firmware unconditionally.
1584 	 */
1585 	if (state->ts_cfg_profile->cp_nodeguid) {
1586 		state->ts_nodeguid   = state->ts_cfg_profile->cp_nodeguid;
1587 	} else {
1588 		state->ts_nodeguid = nodeinfo.NodeGUID;
1589 	}
1590 
1591 	if (state->ts_nodeguid != nodeinfo.NodeGUID) {
1592 		cmn_err(CE_NOTE, "!NodeGUID value queried from firmware "
1593 		    "does not match value set by device property");
1594 	}
1595 
1596 	/*
1597 	 * If the SystemImageGUID value was set in OBP properties, then we use
1598 	 * that value.  But we still print a message if the value we queried
1599 	 * from firmware does not match this value.
1600 	 *
1601 	 * Otherwise if OBP value is not set then we use the value from
1602 	 * firmware unconditionally.
1603 	 */
1604 	if (state->ts_cfg_profile->cp_sysimgguid) {
1605 		state->ts_sysimgguid = state->ts_cfg_profile->cp_sysimgguid;
1606 	} else {
1607 		state->ts_sysimgguid = nodeinfo.SystemImageGUID;
1608 	}
1609 
1610 	if (state->ts_sysimgguid != nodeinfo.SystemImageGUID) {
1611 		cmn_err(CE_NOTE, "!SystemImageGUID value queried from firmware "
1612 		    "does not match value set by device property");
1613 	}
1614 
1615 	/* Get NodeDescription */
1616 	status = tavor_getnodedesc_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN,
1617 	    (sm_nodedesc_t *)&state->ts_nodedesc);
1618 	if (status != TAVOR_CMD_SUCCESS) {
1619 		cmn_err(CE_CONT, "Tavor: GetNodeDesc command failed: %08x\n",
1620 		    status);
1621 		tavor_hw_fini(state, cleanup);
1622 		TNF_PROBE_0(tavor_hw_init_getnodedesc_cmd_fail,
1623 		    TAVOR_TNF_ERROR, "");
1624 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1625 		    "hw_init_getnodedesc_cmd_fail");
1626 		TAVOR_TNF_EXIT(tavor_hw_init);
1627 		return (DDI_FAILURE);
1628 	}
1629 
1630 	TAVOR_TNF_EXIT(tavor_hw_init);
1631 	return (DDI_SUCCESS);
1632 }
1633 
1634 
1635 /*
1636  * tavor_hw_fini()
1637  *    Context: Only called from attach() and/or detach() path contexts
1638  */
1639 static void
tavor_hw_fini(tavor_state_t * state,tavor_drv_cleanup_level_t cleanup)1640 tavor_hw_fini(tavor_state_t *state, tavor_drv_cleanup_level_t cleanup)
1641 {
1642 	uint_t		num_ports;
1643 	int		status;
1644 
1645 	TAVOR_TNF_ENTER(tavor_hw_fini);
1646 
1647 	switch (cleanup) {
1648 	/*
1649 	 * If we add more driver initialization steps that should be cleaned
1650 	 * up here, we need to ensure that TAVOR_DRV_CLEANUP_ALL is still the
1651 	 * first entry (i.e. corresponds to the last init step).
1652 	 */
1653 	case TAVOR_DRV_CLEANUP_ALL:
1654 		/* Shutdown the Tavor IB port(s) */
1655 		num_ports = state->ts_cfg_profile->cp_num_ports;
1656 		(void) tavor_hca_ports_shutdown(state, num_ports);
1657 		/* FALLTHROUGH */
1658 
1659 	case TAVOR_DRV_CLEANUP_LEVEL14:
1660 		/* Teardown resources used for multicast group handling */
1661 		tavor_mcg_fini(state);
1662 		/* FALLTHROUGH */
1663 
1664 	case TAVOR_DRV_CLEANUP_LEVEL13:
1665 		/* Unreserve the special QP contexts */
1666 		tavor_special_qp_contexts_unreserve(state);
1667 		/* FALLTHROUGH */
1668 
1669 	case TAVOR_DRV_CLEANUP_LEVEL12:
1670 		/*
1671 		 * Attempt to teardown all event queues (EQ).  If we fail
1672 		 * here then print a warning message and return.  Something
1673 		 * (either in HW or SW) has gone seriously wrong.
1674 		 */
1675 		status = tavor_eq_fini_all(state);
1676 		if (status != DDI_SUCCESS) {
1677 			TAVOR_WARNING(state, "failed to teardown EQs");
1678 			TNF_PROBE_0(tavor_hw_fini_eqfiniall_fail,
1679 			    TAVOR_TNF_ERROR, "");
1680 			TAVOR_TNF_EXIT(tavor_hw_fini);
1681 			return;
1682 		}
1683 		/* FALLTHROUGH */
1684 
1685 	case TAVOR_DRV_CLEANUP_LEVEL11:
1686 		status = tavor_intr_or_msi_fini(state);
1687 		if (status != DDI_SUCCESS) {
1688 			TAVOR_WARNING(state, "failed to free intr/MSI");
1689 			TNF_PROBE_0(tavor_hw_fini_intrmsifini_fail,
1690 			    TAVOR_TNF_ERROR, "");
1691 			TAVOR_TNF_EXIT(tavor_hw_fini);
1692 			return;
1693 		}
1694 		/* FALLTHROUGH */
1695 
1696 	case TAVOR_DRV_CLEANUP_LEVEL10:
1697 		/* Free the resources for the Tavor internal UAR pages */
1698 		tavor_internal_uarpgs_fini(state);
1699 		/* FALLTHROUGH */
1700 
1701 	case TAVOR_DRV_CLEANUP_LEVEL9:
1702 		/*
1703 		 * Free the PD that was used internally by Tavor software.  If
1704 		 * we fail here then print a warning and return.  Something
1705 		 * (probably software-related, but perhaps HW) has gone wrong.
1706 		 */
1707 		status = tavor_pd_free(state, &state->ts_pdhdl_internal);
1708 		if (status != DDI_SUCCESS) {
1709 			TAVOR_WARNING(state, "failed to free internal PD");
1710 			TNF_PROBE_0(tavor_hw_fini_internal_pd_free_fail,
1711 			    TAVOR_TNF_ERROR, "");
1712 			TAVOR_TNF_EXIT(tavor_hw_fini);
1713 			return;
1714 		}
1715 		/* FALLTHROUGH */
1716 
1717 	case TAVOR_DRV_CLEANUP_LEVEL8:
1718 		/*
1719 		 * Post the CLOSE_HCA command to Tavor firmware.  If we fail
1720 		 * here then print a warning and return.  Something (either in
1721 		 * HW or SW) has gone seriously wrong.
1722 		 */
1723 		status = tavor_close_hca_cmd_post(state,
1724 		    TAVOR_CMD_NOSLEEP_SPIN);
1725 		if (status != TAVOR_CMD_SUCCESS) {
1726 			TAVOR_WARNING(state, "failed to shutdown HCA");
1727 			TNF_PROBE_0(tavor_hw_fini_closehcacmd_fail,
1728 			    TAVOR_TNF_ERROR, "");
1729 			TAVOR_TNF_EXIT(tavor_hw_fini);
1730 			return;
1731 		}
1732 		/* FALLTHROUGH */
1733 
1734 	case TAVOR_DRV_CLEANUP_LEVEL7:
1735 		/* Cleanup all the phase2 resources first */
1736 		tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_ALL);
1737 		/* FALLTHROUGH */
1738 
1739 	case TAVOR_DRV_CLEANUP_LEVEL6:
1740 		/* Then cleanup the phase1 resources */
1741 		tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE);
1742 		/* FALLTHROUGH */
1743 
1744 	case TAVOR_DRV_CLEANUP_LEVEL5:
1745 		/*
1746 		 * Post the SYS_DIS command to Tavor firmware to shut
1747 		 * everything down again.  If we fail here then print a
1748 		 * warning and return.  Something (probably in HW, but maybe
1749 		 * in SW) has gone seriously wrong.
1750 		 */
1751 		status = tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN);
1752 		if (status != TAVOR_CMD_SUCCESS) {
1753 			TAVOR_WARNING(state, "failed to shutdown hardware");
1754 			TNF_PROBE_0(tavor_hw_fini_sys_dis_fail,
1755 			    TAVOR_TNF_ERROR, "");
1756 			TAVOR_TNF_EXIT(tavor_hw_fini);
1757 			return;
1758 		}
1759 		/* FALLTHROUGH */
1760 
1761 	case TAVOR_DRV_CLEANUP_LEVEL4:
1762 		/* Teardown any resources allocated for the config profile */
1763 		tavor_cfg_profile_fini(state);
1764 		/* FALLTHROUGH */
1765 
1766 	case TAVOR_DRV_CLEANUP_LEVEL3:
1767 		ddi_regs_map_free(&state->ts_reg_ddrhdl);
1768 		/* FALLTHROUGH */
1769 
1770 	case TAVOR_DRV_CLEANUP_LEVEL2:
1771 		ddi_regs_map_free(&state->ts_reg_uarhdl);
1772 		/* FALLTHROUGH */
1773 
1774 	case TAVOR_DRV_CLEANUP_LEVEL1:
1775 	case TAVOR_DRV_CLEANUP_LEVEL0:
1776 		/*
1777 		 * LEVEL1 and LEVEL0 resources are freed in
1778 		 * tavor_drv_fini2().
1779 		 */
1780 		break;
1781 
1782 	default:
1783 		TAVOR_WARNING(state, "unexpected driver cleanup level");
1784 		TNF_PROBE_0(tavor_hw_fini_default_fail, TAVOR_TNF_ERROR, "");
1785 		TAVOR_TNF_EXIT(tavor_hw_fini);
1786 		return;
1787 	}
1788 
1789 	TAVOR_TNF_EXIT(tavor_hw_fini);
1790 }
1791 
1792 
1793 /*
1794  * tavor_soft_state_init()
1795  *    Context: Only called from attach() path context
1796  */
1797 static int
tavor_soft_state_init(tavor_state_t * state)1798 tavor_soft_state_init(tavor_state_t *state)
1799 {
1800 	ibt_hca_attr_t		*hca_attr;
1801 	uint64_t		maxval, val;
1802 	ibt_hca_flags_t		caps = IBT_HCA_NO_FLAGS;
1803 	int			status;
1804 
1805 	TAVOR_TNF_ENTER(tavor_soft_state_init);
1806 
1807 	/*
1808 	 * The ibc_hca_info_t struct is passed to the IBTF.  This is the
1809 	 * routine where we initialize it.  Many of the init values come from
1810 	 * either configuration variables or successful queries of the Tavor
1811 	 * hardware abilities
1812 	 */
1813 	state->ts_ibtfinfo.hca_ci_vers	= IBCI_V4;
1814 	state->ts_ibtfinfo.hca_handle	= (ibc_hca_hdl_t)state;
1815 	state->ts_ibtfinfo.hca_ops	= &tavor_ibc_ops;
1816 
1817 	hca_attr = kmem_zalloc(sizeof (ibt_hca_attr_t), KM_SLEEP);
1818 	state->ts_ibtfinfo.hca_attr = hca_attr;
1819 
1820 	hca_attr->hca_dip = state->ts_dip;
1821 	hca_attr->hca_fw_major_version = state->ts_fw.fw_rev_major;
1822 	hca_attr->hca_fw_minor_version = state->ts_fw.fw_rev_minor;
1823 	hca_attr->hca_fw_micro_version = state->ts_fw.fw_rev_subminor;
1824 
1825 	/*
1826 	 * Determine HCA capabilities:
1827 	 * No default support for IBT_HCA_RD, IBT_HCA_RAW_MULTICAST,
1828 	 *    IBT_HCA_ATOMICS_GLOBAL, IBT_HCA_RESIZE_CHAN, IBT_HCA_INIT_TYPE,
1829 	 *    or IBT_HCA_SHUTDOWN_PORT
1830 	 * But IBT_HCA_AH_PORT_CHECK, IBT_HCA_SQD_RTS_PORT, IBT_HCA_SI_GUID,
1831 	 *    IBT_HCA_RNR_NAK, and IBT_HCA_CURRENT_QP_STATE are always
1832 	 *    supported
1833 	 * All other features are conditionally supported, depending on the
1834 	 *    status return by the Tavor HCA (in QUERY_DEV_LIM)
1835 	 */
1836 	if (state->ts_devlim.ud_multi) {
1837 		caps |= IBT_HCA_UD_MULTICAST;
1838 	}
1839 	if (state->ts_devlim.atomic) {
1840 		caps |= IBT_HCA_ATOMICS_HCA;
1841 	}
1842 	if (state->ts_devlim.apm) {
1843 		caps |= IBT_HCA_AUTO_PATH_MIG;
1844 	}
1845 	if (state->ts_devlim.pkey_v) {
1846 		caps |= IBT_HCA_PKEY_CNTR;
1847 	}
1848 	if (state->ts_devlim.qkey_v) {
1849 		caps |= IBT_HCA_QKEY_CNTR;
1850 	}
1851 	if (state->ts_cfg_profile->cp_srq_enable) {
1852 		caps |= IBT_HCA_SRQ | IBT_HCA_RESIZE_SRQ;
1853 	}
1854 	caps |= (IBT_HCA_AH_PORT_CHECK | IBT_HCA_SQD_SQD_PORT |
1855 	    IBT_HCA_SI_GUID | IBT_HCA_RNR_NAK | IBT_HCA_CURRENT_QP_STATE |
1856 	    IBT_HCA_PORT_UP | IBT_HCA_SQD_STATE);
1857 	hca_attr->hca_flags = caps;
1858 	hca_attr->hca_flags2 = IBT_HCA2_DMA_MR;
1859 
1860 	/* Determine VendorID, DeviceID, and revision ID */
1861 	hca_attr->hca_vendor_id	 = state->ts_adapter.vendor_id;
1862 	hca_attr->hca_device_id	 = state->ts_adapter.device_id;
1863 	hca_attr->hca_version_id = state->ts_adapter.rev_id;
1864 
1865 	/*
1866 	 * Determine number of available QPs and max QP size.  Number of
1867 	 * available QPs is determined by subtracting the number of
1868 	 * "reserved QPs" (i.e. reserved for firmware use) from the
1869 	 * total number configured.
1870 	 */
1871 	val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp);
1872 	hca_attr->hca_max_qp = val - ((uint64_t)1 <<
1873 	    state->ts_devlim.log_rsvd_qp);
1874 	maxval	= ((uint64_t)1 << state->ts_devlim.log_max_qp_sz);
1875 	val	= ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_qp_sz);
1876 	if (val > maxval) {
1877 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1878 		TNF_PROBE_2(tavor_soft_state_init_maxqpsz_toobig_fail,
1879 		    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max QP size "
1880 		    "exceeds device maximum", tnf_uint, maxsz, maxval);
1881 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1882 		    "soft_state_init_maxqpsz_toobig_fail");
1883 		TAVOR_TNF_EXIT(tavor_soft_state_init);
1884 		return (DDI_FAILURE);
1885 	}
1886 	hca_attr->hca_max_qp_sz = val;
1887 
1888 	/* Determine max scatter-gather size in WQEs */
1889 	maxval	= state->ts_devlim.max_sg;
1890 	val	= state->ts_cfg_profile->cp_wqe_max_sgl;
1891 	if (val > maxval) {
1892 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1893 		TNF_PROBE_2(tavor_soft_state_init_toomanysgl_fail,
1894 		    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of sgl "
1895 		    "exceeds device maximum", tnf_uint, maxsgl, maxval);
1896 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1897 		    "soft_state_init_toomanysgl_fail");
1898 		TAVOR_TNF_EXIT(tavor_soft_state_init);
1899 		return (DDI_FAILURE);
1900 	}
1901 	/* If the rounded value for max SGL is too large, cap it */
1902 	if (state->ts_cfg_profile->cp_wqe_real_max_sgl > maxval) {
1903 		state->ts_cfg_profile->cp_wqe_real_max_sgl = maxval;
1904 		val = maxval;
1905 	} else {
1906 		val = state->ts_cfg_profile->cp_wqe_real_max_sgl;
1907 	}
1908 
1909 	hca_attr->hca_max_sgl	 = val;
1910 	hca_attr->hca_max_rd_sgl = 0;	/* zero because RD is unsupported */
1911 
1912 	/*
1913 	 * Determine number of available CQs and max CQ size. Number of
1914 	 * available CQs is determined by subtracting the number of
1915 	 * "reserved CQs" (i.e. reserved for firmware use) from the
1916 	 * total number configured.
1917 	 */
1918 	val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_cq);
1919 	hca_attr->hca_max_cq = val - ((uint64_t)1 <<
1920 	    state->ts_devlim.log_rsvd_cq);
1921 	maxval	= ((uint64_t)1 << state->ts_devlim.log_max_cq_sz);
1922 	val	= ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_cq_sz) - 1;
1923 	if (val > maxval) {
1924 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1925 		TNF_PROBE_2(tavor_soft_state_init_maxcqsz_toobig_fail,
1926 		    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max CQ size "
1927 		    "exceeds device maximum", tnf_uint, maxsz, maxval);
1928 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1929 		    "soft_state_init_maxcqsz_toobig_fail");
1930 		TAVOR_TNF_EXIT(tavor_soft_state_init);
1931 		return (DDI_FAILURE);
1932 	}
1933 	hca_attr->hca_max_cq_sz = val;
1934 
1935 	/*
1936 	 * Determine number of available SRQs and max SRQ size. Number of
1937 	 * available SRQs is determined by subtracting the number of
1938 	 * "reserved SRQs" (i.e. reserved for firmware use) from the
1939 	 * total number configured.
1940 	 */
1941 	val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_srq);
1942 	hca_attr->hca_max_srqs = val - ((uint64_t)1 <<
1943 	    state->ts_devlim.log_rsvd_srq);
1944 	maxval  = ((uint64_t)1 << state->ts_devlim.log_max_srq_sz);
1945 	val	= ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_srq_sz);
1946 
1947 	if (val > maxval) {
1948 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1949 		TNF_PROBE_2(tavor_soft_state_init_maxsrqsz_toobig_fail,
1950 		    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max SRQ size "
1951 		    "exceeds device maximum", tnf_uint, maxsz, maxval);
1952 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1953 		    "soft_state_init_maxsrqsz_toobig_fail");
1954 		TAVOR_TNF_EXIT(tavor_soft_state_init);
1955 		return (DDI_FAILURE);
1956 	}
1957 	hca_attr->hca_max_srqs_sz = val;
1958 
1959 	val    = state->ts_cfg_profile->cp_srq_max_sgl;
1960 	maxval	= state->ts_devlim.max_sg;
1961 	if (val > maxval) {
1962 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1963 		TNF_PROBE_2(tavor_soft_state_init_toomanysrqsgl_fail,
1964 		    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of srq "
1965 		    "sgl exceeds device maximum", tnf_uint, maxsgl, maxval);
1966 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
1967 		    "soft_state_init_toomanysrqsgl_fail");
1968 		TAVOR_TNF_EXIT(tavor_soft_state_init);
1969 		return (DDI_FAILURE);
1970 	}
1971 	hca_attr->hca_max_srq_sgl = val;
1972 
1973 	/*
1974 	 * Determine supported HCA page sizes
1975 	 * XXX
1976 	 * For now we simply return the system pagesize as the only supported
1977 	 * pagesize
1978 	 */
1979 	hca_attr->hca_page_sz = ((PAGESIZE == (1 << 13)) ? IBT_PAGE_8K :
1980 	    IBT_PAGE_4K);
1981 
1982 	/*
1983 	 * Determine number of available MemReg, MemWin, and their max size.
1984 	 * Number of available MRs and MWs is determined by subtracting
1985 	 * the number of "reserved MPTs" (i.e. reserved for firmware use)
1986 	 * from the total number configured for each.
1987 	 */
1988 	val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mpt);
1989 	hca_attr->hca_max_memr	  = val - ((uint64_t)1 <<
1990 	    state->ts_devlim.log_rsvd_mpt);
1991 	hca_attr->hca_max_mem_win = val - ((uint64_t)1 <<
1992 	    state->ts_devlim.log_rsvd_mpt);
1993 	maxval	= state->ts_devlim.log_max_mrw_sz;
1994 	val	= state->ts_cfg_profile->cp_log_max_mrw_sz;
1995 	if (val > maxval) {
1996 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
1997 		TNF_PROBE_2(tavor_soft_state_init_maxmrwsz_toobig_fail,
1998 		    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max mrw size "
1999 		    "exceeds device maximum", tnf_uint, maxsz, maxval);
2000 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
2001 		    "soft_state_init_maxmrwsz_toobig_fail");
2002 		TAVOR_TNF_EXIT(tavor_soft_state_init);
2003 		return (DDI_FAILURE);
2004 	}
2005 	hca_attr->hca_max_memr_len = ((uint64_t)1 << val);
2006 
2007 	/* Determine RDMA/Atomic properties */
2008 	val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_rdb);
2009 	hca_attr->hca_max_rsc = val;
2010 	val = state->ts_cfg_profile->cp_hca_max_rdma_in_qp;
2011 	hca_attr->hca_max_rdma_in_qp  = val;
2012 	val = state->ts_cfg_profile->cp_hca_max_rdma_out_qp;
2013 	hca_attr->hca_max_rdma_out_qp = val;
2014 	hca_attr->hca_max_rdma_in_ee  = 0;
2015 	hca_attr->hca_max_rdma_out_ee = 0;
2016 
2017 	/*
2018 	 * Determine maximum number of raw IPv6 and Ether QPs.  Set to 0
2019 	 * because neither type of raw QP is supported
2020 	 */
2021 	hca_attr->hca_max_ipv6_qp  = 0;
2022 	hca_attr->hca_max_ether_qp = 0;
2023 
2024 	/* Determine max number of MCGs and max QP-per-MCG */
2025 	val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp);
2026 	hca_attr->hca_max_mcg_qps   = val;
2027 	val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mcg);
2028 	hca_attr->hca_max_mcg	    = val;
2029 	val = state->ts_cfg_profile->cp_num_qp_per_mcg;
2030 	hca_attr->hca_max_qp_per_mcg = val;
2031 
2032 	/* Determine max number partitions (i.e. PKeys) */
2033 	maxval	= ((uint64_t)1 << state->ts_devlim.log_max_pkey);
2034 	val	= ((uint64_t)state->ts_cfg_profile->cp_num_ports <<
2035 	    state->ts_cfg_profile->cp_log_max_pkeytbl);
2036 
2037 	if (val > maxval) {
2038 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2039 		TNF_PROBE_2(tavor_soft_state_init_toomanypkey_fail,
2040 		    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PKeys "
2041 		    "exceeds device maximum", tnf_uint, maxpkey, maxval);
2042 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
2043 		    "soft_state_init_toomanypkey_fail");
2044 		TAVOR_TNF_EXIT(tavor_soft_state_init);
2045 		return (DDI_FAILURE);
2046 	}
2047 	hca_attr->hca_max_partitions = val;
2048 
2049 	/* Determine number of ports */
2050 	maxval = state->ts_devlim.num_ports;
2051 	val = state->ts_cfg_profile->cp_num_ports;
2052 	if ((val > maxval) || (val == 0)) {
2053 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2054 		TNF_PROBE_2(tavor_soft_state_init_toomanyports_fail,
2055 		    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of ports "
2056 		    "exceeds device maximum", tnf_uint, maxports, maxval);
2057 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
2058 		    "soft_state_init_toomanyports_fail");
2059 		TAVOR_TNF_EXIT(tavor_soft_state_init);
2060 		return (DDI_FAILURE);
2061 	}
2062 	hca_attr->hca_nports = val;
2063 
2064 	/* Copy NodeGUID and SystemImageGUID from softstate */
2065 	hca_attr->hca_node_guid = state->ts_nodeguid;
2066 	hca_attr->hca_si_guid	= state->ts_sysimgguid;
2067 
2068 	/*
2069 	 * Determine local ACK delay.  Use the value suggested by the Tavor
2070 	 * hardware (from the QUERY_DEV_LIM command)
2071 	 */
2072 	hca_attr->hca_local_ack_delay = state->ts_devlim.ca_ack_delay;
2073 
2074 	/* Determine max SGID table and PKey table sizes */
2075 	val	= ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_gidtbl);
2076 	hca_attr->hca_max_port_sgid_tbl_sz = val;
2077 	val	= ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_pkeytbl);
2078 	hca_attr->hca_max_port_pkey_tbl_sz = val;
2079 
2080 	/* Determine max number of PDs */
2081 	maxval	= ((uint64_t)1 << state->ts_devlim.log_max_pd);
2082 	val	= ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_pd);
2083 	if (val > maxval) {
2084 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2085 		TNF_PROBE_2(tavor_soft_state_init_toomanypd_fail,
2086 		    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PD "
2087 		    "exceeds device maximum", tnf_uint, maxpd, maxval);
2088 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
2089 		    "soft_state_init_toomanypd_fail");
2090 		TAVOR_TNF_EXIT(tavor_soft_state_init);
2091 		return (DDI_FAILURE);
2092 	}
2093 	hca_attr->hca_max_pd = val;
2094 
2095 	/* Determine max number of Address Handles */
2096 	maxval	= ((uint64_t)1 << state->ts_devlim.log_max_av);
2097 	val	= ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_ah);
2098 	if (val > maxval) {
2099 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2100 		TNF_PROBE_2(tavor_soft_state_init_toomanyah_fail,
2101 		    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of AH "
2102 		    "exceeds device maximum", tnf_uint, maxah, maxval);
2103 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
2104 		    "soft_state_init_toomanyah_fail");
2105 		TAVOR_TNF_EXIT(tavor_soft_state_init);
2106 		return (DDI_FAILURE);
2107 	}
2108 	hca_attr->hca_max_ah = val;
2109 
2110 	/* No RDDs or EECs (since Reliable Datagram is not supported) */
2111 	hca_attr->hca_max_rdd = 0;
2112 	hca_attr->hca_max_eec = 0;
2113 
2114 	/* Initialize lock for reserved UAR page access */
2115 	mutex_init(&state->ts_uar_lock, NULL, MUTEX_DRIVER,
2116 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2117 
2118 	/* Initialize the flash fields */
2119 	state->ts_fw_flashstarted = 0;
2120 	mutex_init(&state->ts_fw_flashlock, NULL, MUTEX_DRIVER,
2121 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2122 
2123 	/* Initialize the lock for the info ioctl */
2124 	mutex_init(&state->ts_info_lock, NULL, MUTEX_DRIVER,
2125 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2126 
2127 	/* Initialize the AVL tree for QP number support */
2128 	tavor_qpn_avl_init(state);
2129 
2130 	/* Initialize the kstat info structure */
2131 	status = tavor_kstat_init(state);
2132 	if (status != DDI_SUCCESS) {
2133 		tavor_qpn_avl_fini(state);
2134 		mutex_destroy(&state->ts_info_lock);
2135 		mutex_destroy(&state->ts_fw_flashlock);
2136 		mutex_destroy(&state->ts_uar_lock);
2137 		kmem_free(hca_attr, sizeof (ibt_hca_attr_t));
2138 		TNF_PROBE_0(tavor_soft_state_init_kstatinit_fail,
2139 		    TAVOR_TNF_ERROR, "");
2140 		TAVOR_ATTACH_MSG(state->ts_attach_buf,
2141 		    "soft_state_init_kstatinit_fail");
2142 		TAVOR_TNF_EXIT(tavor_soft_state_init);
2143 		return (DDI_FAILURE);
2144 	}
2145 
2146 	TAVOR_TNF_EXIT(tavor_soft_state_init);
2147 	return (DDI_SUCCESS);
2148 }
2149 
2150 
2151 /*
2152  * tavor_soft_state_fini()
2153  *    Context: Called only from detach() path context
2154  */
2155 static void
tavor_soft_state_fini(tavor_state_t * state)2156 tavor_soft_state_fini(tavor_state_t *state)
2157 {
2158 	TAVOR_TNF_ENTER(tavor_soft_state_fini);
2159 
2160 	/* Teardown the kstat info */
2161 	tavor_kstat_fini(state);
2162 
2163 	/* Teardown the AVL tree for QP number support */
2164 	tavor_qpn_avl_fini(state);
2165 
2166 	/* Free up info ioctl mutex */
2167 	mutex_destroy(&state->ts_info_lock);
2168 
2169 	/* Free up flash mutex */
2170 	mutex_destroy(&state->ts_fw_flashlock);
2171 
2172 	/* Free up the UAR page access mutex */
2173 	mutex_destroy(&state->ts_uar_lock);
2174 
2175 	/* Free up the hca_attr struct */
2176 	kmem_free(state->ts_ibtfinfo.hca_attr, sizeof (ibt_hca_attr_t));
2177 
2178 	TAVOR_TNF_EXIT(tavor_soft_state_fini);
2179 }
2180 
2181 
2182 /*
2183  * tavor_hca_config_setup()
2184  *    Context: Only called from attach() path context
2185  */
2186 static void
tavor_hca_config_setup(tavor_state_t * state,tavor_hw_initqueryhca_t * inithca)2187 tavor_hca_config_setup(tavor_state_t *state,
2188     tavor_hw_initqueryhca_t *inithca)
2189 {
2190 	tavor_rsrc_pool_info_t	*rsrc_pool;
2191 	uint64_t		ddr_baseaddr, ddr_base_map_addr;
2192 	uint64_t		offset, addr;
2193 	uint_t			mcg_size;
2194 
2195 	TAVOR_TNF_ENTER(tavor_hca_config_setup);
2196 
2197 	/* Set "host endianness".  Default is big endian */
2198 #ifdef	_LITTLE_ENDIAN
2199 	inithca->big_endian	= 0;
2200 #else
2201 	inithca->big_endian	= 1;
2202 #endif
2203 	/* No Address Vector Protection, but Port Checking on by default */
2204 	inithca->udav_chk	= TAVOR_UDAV_PROTECT_DISABLED;
2205 	inithca->udav_port_chk	= TAVOR_UDAV_PORTCHK_ENABLED;
2206 
2207 	ddr_baseaddr	  = (uint64_t)(uintptr_t)state->ts_reg_ddr_baseaddr;
2208 	ddr_base_map_addr = (uint64_t)state->ts_ddr.ddr_baseaddr;
2209 
2210 	/* Setup QPC table */
2211 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC];
2212 	offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2213 	addr = ddr_base_map_addr + offset;
2214 	inithca->context.qpc_baseaddr_h = (addr >> 32);
2215 	inithca->context.qpc_baseaddr_l = (addr & 0xFFFFFFFF) >> 7;
2216 	inithca->context.log_num_qp	= state->ts_cfg_profile->cp_log_num_qp;
2217 
2218 	/* Setup EEC table (initialize to zero - RD unsupported) */
2219 	inithca->context.eec_baseaddr_h	= 0;
2220 	inithca->context.eec_baseaddr_l	= 0;
2221 	inithca->context.log_num_ee	= 0;
2222 
2223 	/* Setup CQC table */
2224 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC];
2225 	offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2226 	addr = ddr_base_map_addr + offset;
2227 	inithca->context.cqc_baseaddr_h = (addr >> 32);
2228 	inithca->context.cqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2229 	inithca->context.log_num_cq	= state->ts_cfg_profile->cp_log_num_cq;
2230 
2231 	/* Setup SRQC table */
2232 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC];
2233 	offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2234 	addr = ddr_base_map_addr + offset;
2235 	inithca->context.srqc_baseaddr_h = (addr >> 32);
2236 	inithca->context.srqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2237 	inithca->context.log_num_srq	 =
2238 	    state->ts_cfg_profile->cp_log_num_srq;
2239 
2240 	/* Setup EQPC table */
2241 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC];
2242 	offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2243 	addr = ddr_base_map_addr + offset;
2244 	inithca->context.eqpc_baseaddr	= addr;
2245 
2246 	/* Setup EEEC table (initialize to zero - RD unsupported) */
2247 	inithca->context.eeec_baseaddr	= 0;
2248 
2249 	/* Setup EQC table */
2250 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC];
2251 	offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2252 	addr = ddr_base_map_addr + offset;
2253 	inithca->context.eqc_baseaddr_h = (addr >> 32);
2254 	inithca->context.eqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6;
2255 	inithca->context.log_num_eq	= TAVOR_NUM_EQ_SHIFT;
2256 
2257 	/* Setup RDB table */
2258 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB];
2259 	offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2260 	addr = ddr_base_map_addr + offset;
2261 	inithca->context.rdb_baseaddr_h	= (addr >> 32);
2262 	inithca->context.rdb_baseaddr_l = 0;
2263 
2264 	/* Setup Multicast */
2265 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG];
2266 	offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2267 	addr = ddr_base_map_addr + offset;
2268 	inithca->multi.mc_baseaddr	= addr;
2269 	mcg_size = TAVOR_MCGMEM_SZ(state);
2270 	inithca->multi.log_mc_tbl_ent	= highbit(mcg_size) - 1;
2271 	inithca->multi.mc_tbl_hash_sz	=
2272 	    (1 << state->ts_cfg_profile->cp_log_num_mcg_hash);
2273 	inithca->multi.mc_hash_fn	= TAVOR_MCG_DEFAULT_HASH_FN;
2274 	inithca->multi.log_mc_tbl_sz	= state->ts_cfg_profile->cp_log_num_mcg;
2275 
2276 
2277 	/* Setup TPT */
2278 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT];
2279 	offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2280 	addr = ddr_base_map_addr + offset;
2281 	inithca->tpt.mpt_baseaddr	= addr;
2282 	inithca->tpt.mttseg_sz		= TAVOR_MTTSEG_SIZE_SHIFT;
2283 	inithca->tpt.log_mpt_sz		= state->ts_cfg_profile->cp_log_num_mpt;
2284 	inithca->tpt.mtt_version	= TAVOR_MTT_PG_WALK_VER;
2285 
2286 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT];
2287 	offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2288 	addr = ddr_base_map_addr + offset;
2289 	inithca->tpt.mtt_baseaddr	= addr;
2290 
2291 	/* Setup UAR */
2292 	rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR];
2293 	offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr;
2294 	addr = ddr_base_map_addr + offset;
2295 	inithca->uar.uarscr_baseaddr	= addr;
2296 
2297 	inithca->uar.uar_pg_sz = PAGESHIFT - 0xC;
2298 
2299 	TAVOR_TNF_EXIT(tavor_hca_config_setup);
2300 }
2301 
2302 
2303 /*
2304  * tavor_hca_port_init()
2305  *    Context: Only called from attach() path context
2306  */
2307 static int
tavor_hca_port_init(tavor_state_t * state)2308 tavor_hca_port_init(tavor_state_t *state)
2309 {
2310 	tavor_hw_initib_t	*portinits, *initib;
2311 	tavor_cfg_profile_t	*cfgprof;
2312 	uint_t			num_ports;
2313 	int			i, status;
2314 	uint64_t		maxval, val;
2315 	uint64_t		sysimgguid, nodeguid, portguid;
2316 
2317 	TAVOR_TNF_ENTER(tavor_hca_port_init);
2318 
2319 	cfgprof = state->ts_cfg_profile;
2320 
2321 	/* Get number of HCA ports */
2322 	num_ports = cfgprof->cp_num_ports;
2323 
2324 	/* Allocate space for Tavor port init struct(s) */
2325 	portinits = (tavor_hw_initib_t *)kmem_zalloc(num_ports *
2326 	    sizeof (tavor_hw_initib_t), KM_SLEEP);
2327 
2328 	/* Post command to initialize Tavor HCA port */
2329 	for (i = 0; i < num_ports; i++) {
2330 		initib = &portinits[i];
2331 
2332 		/*
2333 		 * Determine whether we need to override the firmware's
2334 		 * default SystemImageGUID setting.
2335 		 */
2336 		sysimgguid = cfgprof->cp_sysimgguid;
2337 		if (sysimgguid != 0) {
2338 			initib->set_sysimg_guid	= 1;
2339 			initib->sysimg_guid	= sysimgguid;
2340 		}
2341 
2342 		/*
2343 		 * Determine whether we need to override the firmware's
2344 		 * default NodeGUID setting.
2345 		 */
2346 		nodeguid = cfgprof->cp_nodeguid;
2347 		if (nodeguid != 0) {
2348 			initib->set_node_guid	= 1;
2349 			initib->node_guid	= nodeguid;
2350 		}
2351 
2352 		/*
2353 		 * Determine whether we need to override the firmware's
2354 		 * default PortGUID setting.
2355 		 */
2356 		portguid = cfgprof->cp_portguid[i];
2357 		if (portguid != 0) {
2358 			initib->set_port_guid0	= 1;
2359 			initib->guid0		= portguid;
2360 		}
2361 
2362 		/* Validate max MTU size */
2363 		maxval  = state->ts_devlim.max_mtu;
2364 		val	= cfgprof->cp_max_mtu;
2365 		if (val > maxval) {
2366 			TNF_PROBE_2(tavor_hca_port_init_maxmtu_fail,
2367 			    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2368 			    "MTU size exceeds device maximum", tnf_uint,
2369 			    maxmtu, maxval);
2370 			TAVOR_TNF_EXIT(tavor_hca_port_init);
2371 			goto init_ports_fail;
2372 		}
2373 		initib->mtu_cap = val;
2374 
2375 		/* Validate the max port width */
2376 		maxval  = state->ts_devlim.max_port_width;
2377 		val	= cfgprof->cp_max_port_width;
2378 		if (val > maxval) {
2379 			TNF_PROBE_2(tavor_hca_port_init_maxportwidth_fail,
2380 			    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2381 			    "port width exceeds device maximum", tnf_uint,
2382 			    maxportwidth, maxval);
2383 			TAVOR_TNF_EXIT(tavor_hca_port_init);
2384 			goto init_ports_fail;
2385 		}
2386 		initib->port_width_cap = val;
2387 
2388 		/* Validate max VL cap size */
2389 		maxval  = state->ts_devlim.max_vl;
2390 		val	= cfgprof->cp_max_vlcap;
2391 		if (val > maxval) {
2392 			TNF_PROBE_2(tavor_hca_port_init_maxvlcap_fail,
2393 			    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2394 			    "VLcap size exceeds device maximum", tnf_uint,
2395 			    maxvlcap, maxval);
2396 			TAVOR_TNF_EXIT(tavor_hca_port_init);
2397 			goto init_ports_fail;
2398 		}
2399 		initib->vl_cap = val;
2400 
2401 		/* Validate max GID table size */
2402 		maxval  = ((uint64_t)1 << state->ts_devlim.log_max_gid);
2403 		val	= ((uint64_t)1 << cfgprof->cp_log_max_gidtbl);
2404 		if (val > maxval) {
2405 			TNF_PROBE_2(tavor_hca_port_init_gidtable_fail,
2406 			    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2407 			    "GID table size exceeds device maximum", tnf_uint,
2408 			    maxgidtbl, maxval);
2409 			TAVOR_TNF_EXIT(tavor_hca_port_init);
2410 			goto init_ports_fail;
2411 		}
2412 		initib->max_gid = val;
2413 
2414 		/* Validate max PKey table size */
2415 		maxval	= ((uint64_t)1 << state->ts_devlim.log_max_pkey);
2416 		val	= ((uint64_t)1 << cfgprof->cp_log_max_pkeytbl);
2417 		if (val > maxval) {
2418 			TNF_PROBE_2(tavor_hca_port_init_pkeytable_fail,
2419 			    TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max "
2420 			    "PKey table size exceeds device maximum", tnf_uint,
2421 			    maxpkeytbl, maxval);
2422 			TAVOR_TNF_EXIT(tavor_hca_port_init);
2423 			goto init_ports_fail;
2424 		}
2425 		initib->max_pkey = val;
2426 
2427 		/*
2428 		 * Post the INIT_IB command to Tavor firmware.  When this
2429 		 * command completes, the corresponding Tavor port will be
2430 		 * physically "Up" and initialized.
2431 		 */
2432 		status = tavor_init_ib_cmd_post(state, initib, i + 1,
2433 		    TAVOR_CMD_NOSLEEP_SPIN);
2434 		if (status != TAVOR_CMD_SUCCESS) {
2435 			cmn_err(CE_CONT, "Tavor: INIT_IB (port %02d) command "
2436 			    "failed: %08x\n", i + 1, status);
2437 			TNF_PROBE_2(tavor_hca_port_init_init_ib_cmd_fail,
2438 			    TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status,
2439 			    tnf_uint, port, i + 1);
2440 			TAVOR_TNF_EXIT(tavor_hca_port_init);
2441 			goto init_ports_fail;
2442 		}
2443 	}
2444 
2445 	/* Free up the memory for Tavor port init struct(s), return success */
2446 	kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t));
2447 	TAVOR_TNF_EXIT(tavor_hca_port_init);
2448 	return (DDI_SUCCESS);
2449 
2450 init_ports_fail:
2451 	/*
2452 	 * Free up the memory for Tavor port init struct(s), shutdown any
2453 	 * successfully initialized ports, and return failure
2454 	 */
2455 	kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t));
2456 	(void) tavor_hca_ports_shutdown(state, i);
2457 
2458 	TAVOR_TNF_EXIT(tavor_hca_port_init);
2459 	return (DDI_FAILURE);
2460 }
2461 
2462 
2463 /*
2464  * tavor_hca_ports_shutdown()
2465  *    Context: Only called from attach() and/or detach() path contexts
2466  */
2467 static int
tavor_hca_ports_shutdown(tavor_state_t * state,uint_t num_init)2468 tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init)
2469 {
2470 	int	i, status;
2471 
2472 	TAVOR_TNF_ENTER(tavor_hca_ports_shutdown);
2473 
2474 	/*
2475 	 * Post commands to shutdown all init'd Tavor HCA ports.  Note: if
2476 	 * any of these commands fail for any reason, it would be entirely
2477 	 * unexpected and probably indicative a serious problem (HW or SW).
2478 	 * Although we do return void from this function, this type of failure
2479 	 * should not go unreported.  That is why we have the warning message
2480 	 * and the detailed TNF information.
2481 	 */
2482 	for (i = 0; i < num_init; i++) {
2483 		status = tavor_close_ib_cmd_post(state, i + 1,
2484 		    TAVOR_CMD_NOSLEEP_SPIN);
2485 		if (status != TAVOR_CMD_SUCCESS) {
2486 			TAVOR_WARNING(state, "failed to shutdown HCA port");
2487 			TNF_PROBE_2(tavor_hca_ports_shutdown_close_ib_cmd_fail,
2488 			    TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status,
2489 			    tnf_uint, port, i + 1);
2490 			TAVOR_TNF_EXIT(tavor_hca_ports_shutdown);
2491 			return (status);
2492 		}
2493 	}
2494 
2495 	TAVOR_TNF_EXIT(tavor_hca_ports_shutdown);
2496 
2497 	return (TAVOR_CMD_SUCCESS);
2498 }
2499 
2500 
2501 /*
2502  * tavor_internal_uarpgs_init
2503  *    Context: Only called from attach() path context
2504  */
2505 static int
tavor_internal_uarpgs_init(tavor_state_t * state)2506 tavor_internal_uarpgs_init(tavor_state_t *state)
2507 {
2508 	int	status;
2509 
2510 	TAVOR_TNF_ENTER(tavor_internal_uarpgs_init);
2511 
2512 	/*
2513 	 * Save away reserved Tavor UAR page #0.  This UAR page is not to
2514 	 * be used by software.
2515 	 */
2516 	status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP,
2517 	    &state->ts_uarpg0_rsrc_rsrvd);
2518 	if (status != DDI_SUCCESS) {
2519 		TNF_PROBE_0(tavor_uarpg0_rsrcalloc_fail, TAVOR_TNF_ERROR, "");
2520 		TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2521 		return (DDI_FAILURE);
2522 	}
2523 
2524 	/*
2525 	 * Save away Tavor UAR page #1 (for internal use).  This UAR page is
2526 	 * the privileged UAR page through which all kernel generated
2527 	 * doorbells will be rung.
2528 	 */
2529 	status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP,
2530 	    &state->ts_uarpg1_rsrc);
2531 	if (status != DDI_SUCCESS) {
2532 		tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd);
2533 		TNF_PROBE_0(tavor_uarpg1_rsrcalloc_fail, TAVOR_TNF_ERROR, "");
2534 		TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2535 		return (DDI_FAILURE);
2536 	}
2537 
2538 	/* Setup pointer to UAR page #1 doorbells */
2539 	state->ts_uar = (tavor_hw_uar_t *)state->ts_uarpg1_rsrc->tr_addr;
2540 
2541 	TAVOR_TNF_EXIT(tavor_internal_uarpgs_init);
2542 	return (DDI_SUCCESS);
2543 }
2544 
2545 
2546 /*
2547  * tavor_internal_uarpgs_fini
2548  *    Context: Only called from attach() and/or detach() path contexts
2549  */
2550 static void
tavor_internal_uarpgs_fini(tavor_state_t * state)2551 tavor_internal_uarpgs_fini(tavor_state_t *state)
2552 {
2553 	TAVOR_TNF_ENTER(tavor_internal_uarpgs_fini);
2554 
2555 	/* Free up Tavor UAR page #1 (kernel driver doorbells) */
2556 	tavor_rsrc_free(state, &state->ts_uarpg1_rsrc);
2557 
2558 	/* Free up Tavor UAR page #0 (reserved) */
2559 	tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd);
2560 
2561 	TAVOR_TNF_EXIT(tavor_internal_uarpgs_fini);
2562 }
2563 
2564 
2565 /*
2566  * tavor_special_qp_contexts_reserve()
2567  *    Context: Only called from attach() path context
2568  */
2569 static int
tavor_special_qp_contexts_reserve(tavor_state_t * state)2570 tavor_special_qp_contexts_reserve(tavor_state_t *state)
2571 {
2572 	tavor_rsrc_t	*qp0_rsrc, *qp1_rsrc;
2573 	int		status;
2574 
2575 	TAVOR_TNF_ENTER(tavor_special_qp_contexts_reserve);
2576 
2577 	/* Initialize the lock used for special QP rsrc management */
2578 	mutex_init(&state->ts_spec_qplock, NULL, MUTEX_DRIVER,
2579 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2580 
2581 	/*
2582 	 * Reserve contexts for QP0.  These QP contexts will be setup to
2583 	 * act as aliases for the real QP0.  Note: We are required to grab
2584 	 * two QPs (one per port) even if we are operating in single-port
2585 	 * mode.
2586 	 */
2587 	status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp0_rsrc);
2588 	if (status != DDI_SUCCESS) {
2589 		mutex_destroy(&state->ts_spec_qplock);
2590 		TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp0_fail,
2591 		    TAVOR_TNF_ERROR, "");
2592 		TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2593 		return (DDI_FAILURE);
2594 	}
2595 	state->ts_spec_qp0 = qp0_rsrc;
2596 
2597 	/*
2598 	 * Reserve contexts for QP1.  These QP contexts will be setup to
2599 	 * act as aliases for the real QP1.  Note: We are required to grab
2600 	 * two QPs (one per port) even if we are operating in single-port
2601 	 * mode.
2602 	 */
2603 	status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp1_rsrc);
2604 	if (status != DDI_SUCCESS) {
2605 		tavor_rsrc_free(state, &qp0_rsrc);
2606 		mutex_destroy(&state->ts_spec_qplock);
2607 		TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp1_fail,
2608 		    TAVOR_TNF_ERROR, "");
2609 		TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2610 		return (DDI_FAILURE);
2611 	}
2612 	state->ts_spec_qp1 = qp1_rsrc;
2613 
2614 	TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve);
2615 	return (DDI_SUCCESS);
2616 }
2617 
2618 
2619 /*
2620  * tavor_special_qp_contexts_unreserve()
2621  *    Context: Only called from attach() and/or detach() path contexts
2622  */
2623 static void
tavor_special_qp_contexts_unreserve(tavor_state_t * state)2624 tavor_special_qp_contexts_unreserve(tavor_state_t *state)
2625 {
2626 	TAVOR_TNF_ENTER(tavor_special_qp_contexts_unreserve);
2627 
2628 	/* Unreserve contexts for QP1 */
2629 	tavor_rsrc_free(state, &state->ts_spec_qp1);
2630 
2631 	/* Unreserve contexts for QP0 */
2632 	tavor_rsrc_free(state, &state->ts_spec_qp0);
2633 
2634 	/* Destroy the lock used for special QP rsrc management */
2635 	mutex_destroy(&state->ts_spec_qplock);
2636 
2637 	TAVOR_TNF_EXIT(tavor_special_qp_contexts_unreserve);
2638 }
2639 
2640 
2641 /*
2642  * tavor_sw_reset()
2643  *    Context: Currently called only from attach() path context
2644  */
2645 static int
tavor_sw_reset(tavor_state_t * state)2646 tavor_sw_reset(tavor_state_t *state)
2647 {
2648 	dev_info_t		*dip, *pdip;
2649 	ddi_acc_handle_t	hdl = state->ts_pci_cfghdl, phdl;
2650 	uint32_t		reset_delay;
2651 	int			status, i;
2652 
2653 	TAVOR_TNF_ENTER(tavor_sw_reset);
2654 
2655 	/*
2656 	 * If the configured software reset delay is set to zero, then we
2657 	 * will not attempt a software reset of the Tavor device.
2658 	 */
2659 	reset_delay = state->ts_cfg_profile->cp_sw_reset_delay;
2660 	if (reset_delay == 0) {
2661 		TAVOR_TNF_EXIT(tavor_sw_reset);
2662 		return (DDI_SUCCESS);
2663 	}
2664 
2665 	/*
2666 	 * Get dip for HCA device _and_ parent device as well.  Parent access
2667 	 * is necessary here because software reset of the Tavor hardware
2668 	 * will reinitialize both the config registers of the PCI bridge
2669 	 * (parent, if it exists) and the IB HCA (self)
2670 	 */
2671 	dip  = state->ts_dip;
2672 	pdip = ddi_get_parent(dip);
2673 
2674 	/* Query the PCI capabilities of the HCA device */
2675 	tavor_pci_capability_list(state, hdl);
2676 
2677 	/*
2678 	 * Read all PCI config info (reg0...reg63).  Note: According to the
2679 	 * Tavor software reset application note, we should not read or
2680 	 * restore the values in reg22 and reg23.
2681 	 */
2682 	for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2683 		if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2684 		    (i != TAVOR_SW_RESET_REG23_RSVD)) {
2685 			state->ts_cfg_data[i]  = pci_config_get32(hdl, i << 2);
2686 		}
2687 	}
2688 
2689 	if (TAVOR_PARENT_IS_BRIDGE(pdip)) {
2690 		/*
2691 		 * Setup for PCI config read/write of bridge device
2692 		 */
2693 		status = pci_config_setup(pdip, &phdl);
2694 		if (status != DDI_SUCCESS) {
2695 			TNF_PROBE_0(tavor_sw_reset_pcicfg_p_fail,
2696 			    TAVOR_TNF_ERROR, "");
2697 			TAVOR_TNF_EXIT(tavor_sw_reset);
2698 			return (DDI_FAILURE);
2699 		}
2700 
2701 		/*
2702 		 * Read all PCI config info (reg0...reg63).  Note: According to
2703 		 * the Tavor software reset application note, we should not
2704 		 * read or restore the values in reg22 and reg23.
2705 		 */
2706 		for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2707 			if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2708 			    (i != TAVOR_SW_RESET_REG23_RSVD)) {
2709 				state->ts_cfg_pdata[i] =
2710 				    pci_config_get32(phdl, i << 2);
2711 			}
2712 		}
2713 	}
2714 
2715 	/*
2716 	 * Perform the software reset (by writing 1 at offset 0xF0010)
2717 	 */
2718 	ddi_put32(state->ts_reg_cmdhdl, state->ts_cmd_regs.sw_reset,
2719 	    TAVOR_SW_RESET_START);
2720 
2721 	drv_usecwait(reset_delay);
2722 
2723 	if (TAVOR_PARENT_IS_BRIDGE(pdip)) {
2724 		/*
2725 		 * Bridge exists, so wait for the bridge to become ready.
2726 		 *
2727 		 * The above delay is necessary to avoid system panic from
2728 		 * Master Abort.  If the device is accessed before this delay,
2729 		 * device will not respond to config cycles and they will be
2730 		 * terminate with a Master Abort which will panic the system.
2731 		 * Below is the loop we use to poll status from the device to
2732 		 * determine if it is OK to proceed.
2733 		 */
2734 		i = 0;
2735 		while (pci_config_get32(phdl, 0) == TAVOR_SW_RESET_NOTDONE) {
2736 			drv_usecwait(TAVOR_SW_RESET_POLL_DELAY);
2737 		}
2738 
2739 		/*
2740 		 * Write all the PCI config registers back into each device
2741 		 * (except for reg22 and reg23 - see above)
2742 		 */
2743 		for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2744 			if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2745 			    (i != TAVOR_SW_RESET_REG23_RSVD)) {
2746 				pci_config_put32(phdl, i << 2,
2747 				    state->ts_cfg_pdata[i]);
2748 			}
2749 		}
2750 
2751 		/*
2752 		 * Tear down the config setup (for bridge device)
2753 		 */
2754 		pci_config_teardown(&phdl);
2755 
2756 	/* No Bridge Device */
2757 	} else {
2758 		/*
2759 		 * Bridge does not exist, so instead wait for the device itself
2760 		 * to become ready.
2761 		 *
2762 		 * The above delay is necessary to avoid system panic from
2763 		 * Master Abort.  If the device is accessed before this delay,
2764 		 * device will not respond to config cycles and they will be
2765 		 * terminate with a Master Abort which will panic the system.
2766 		 * Below is the loop we use to poll status from the device to
2767 		 * determine if it is OK to proceed.
2768 		 */
2769 		i = 0;
2770 		while (pci_config_get32(hdl, 0) == TAVOR_SW_RESET_NOTDONE) {
2771 			drv_usecwait(TAVOR_SW_RESET_POLL_DELAY);
2772 		}
2773 	}
2774 
2775 	for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) {
2776 		if ((i != TAVOR_SW_RESET_REG22_RSVD) &&
2777 		    (i != TAVOR_SW_RESET_REG23_RSVD)) {
2778 			pci_config_put32(hdl, i << 2, state->ts_cfg_data[i]);
2779 		}
2780 	}
2781 
2782 	TAVOR_TNF_EXIT(tavor_sw_reset);
2783 	return (DDI_SUCCESS);
2784 }
2785 
2786 
2787 /*
2788  * tavor_mcg_init()
2789  *    Context: Only called from attach() path context
2790  */
2791 static int
tavor_mcg_init(tavor_state_t * state)2792 tavor_mcg_init(tavor_state_t *state)
2793 {
2794 	uint_t		mcg_tmp_sz;
2795 
2796 	TAVOR_TNF_ENTER(tavor_mcg_init);
2797 
2798 	/*
2799 	 * Allocate space for the MCG temporary copy buffer.  This is
2800 	 * used by the Attach/Detach Multicast Group code
2801 	 */
2802 	mcg_tmp_sz = TAVOR_MCGMEM_SZ(state);
2803 	state->ts_mcgtmp = kmem_zalloc(mcg_tmp_sz, KM_SLEEP);
2804 
2805 	/*
2806 	 * Initialize the multicast group mutex.  This ensures atomic
2807 	 * access to add, modify, and remove entries in the multicast
2808 	 * group hash lists.
2809 	 */
2810 	mutex_init(&state->ts_mcglock, NULL, MUTEX_DRIVER,
2811 	    DDI_INTR_PRI(state->ts_intrmsi_pri));
2812 
2813 	TAVOR_TNF_EXIT(tavor_mcg_init);
2814 	return (DDI_SUCCESS);
2815 }
2816 
2817 
2818 /*
2819  * tavor_mcg_fini()
2820  *    Context: Only called from attach() and/or detach() path contexts
2821  */
2822 static void
tavor_mcg_fini(tavor_state_t * state)2823 tavor_mcg_fini(tavor_state_t *state)
2824 {
2825 	uint_t		mcg_tmp_sz;
2826 
2827 	TAVOR_TNF_ENTER(tavor_mcg_fini);
2828 
2829 	/* Free up the space used for the MCG temporary copy buffer */
2830 	mcg_tmp_sz = TAVOR_MCGMEM_SZ(state);
2831 	kmem_free(state->ts_mcgtmp, mcg_tmp_sz);
2832 
2833 	/* Destroy the multicast group mutex */
2834 	mutex_destroy(&state->ts_mcglock);
2835 
2836 	TAVOR_TNF_EXIT(tavor_mcg_fini);
2837 }
2838 
2839 
2840 /*
2841  * tavor_fw_version_check()
2842  *    Context: Only called from attach() path context
2843  */
2844 static int
tavor_fw_version_check(tavor_state_t * state)2845 tavor_fw_version_check(tavor_state_t *state)
2846 {
2847 	uint_t	tavor_fw_ver_major;
2848 	uint_t	tavor_fw_ver_minor;
2849 	uint_t	tavor_fw_ver_subminor;
2850 
2851 	/*
2852 	 * Depending on which version of driver we have attached, the firmware
2853 	 * version checks will be different.  We set up the comparison values
2854 	 * for both HCA Mode (Tavor hardware) or COMPAT Mode (Arbel hardware
2855 	 * running in tavor mode).
2856 	 */
2857 	switch (state->ts_operational_mode) {
2858 	case TAVOR_HCA_MODE:
2859 		tavor_fw_ver_major = TAVOR_FW_VER_MAJOR;
2860 		tavor_fw_ver_minor = TAVOR_FW_VER_MINOR;
2861 		tavor_fw_ver_subminor = TAVOR_FW_VER_SUBMINOR;
2862 		break;
2863 
2864 	case TAVOR_COMPAT_MODE:
2865 		tavor_fw_ver_major = TAVOR_COMPAT_FW_VER_MAJOR;
2866 		tavor_fw_ver_minor = TAVOR_COMPAT_FW_VER_MINOR;
2867 		tavor_fw_ver_subminor = TAVOR_COMPAT_FW_VER_SUBMINOR;
2868 		break;
2869 
2870 	default:
2871 		return (DDI_FAILURE);
2872 	}
2873 
2874 	/*
2875 	 * If FW revision major number is less than acceptable,
2876 	 * return failure, else if greater return success.  If
2877 	 * the major numbers are equal than check the minor number
2878 	 */
2879 	if (state->ts_fw.fw_rev_major < tavor_fw_ver_major) {
2880 		return (DDI_FAILURE);
2881 	} else if (state->ts_fw.fw_rev_major > tavor_fw_ver_major) {
2882 		return (DDI_SUCCESS);
2883 	}
2884 	/*
2885 	 * Do the same check as above, except for minor revision numbers
2886 	 * If the minor numbers are equal than check the subminor number
2887 	 */
2888 	if (state->ts_fw.fw_rev_minor < tavor_fw_ver_minor) {
2889 		return (DDI_FAILURE);
2890 	} else if (state->ts_fw.fw_rev_minor > tavor_fw_ver_minor) {
2891 		return (DDI_SUCCESS);
2892 	}
2893 
2894 	/*
2895 	 * Once again we do the same check as above, except for the subminor
2896 	 * revision number.  If the subminor numbers are equal here, then
2897 	 * these are the same firmware version, return success
2898 	 */
2899 	if (state->ts_fw.fw_rev_subminor < tavor_fw_ver_subminor) {
2900 		return (DDI_FAILURE);
2901 	} else if (state->ts_fw.fw_rev_subminor > tavor_fw_ver_subminor) {
2902 		return (DDI_SUCCESS);
2903 	}
2904 
2905 	return (DDI_SUCCESS);
2906 }
2907 
2908 
2909 /*
2910  * tavor_device_info_report()
2911  *    Context: Only called from attach() path context
2912  */
2913 static void
tavor_device_info_report(tavor_state_t * state)2914 tavor_device_info_report(tavor_state_t *state)
2915 {
2916 	cmn_err(CE_CONT, "?tavor%d: FW ver: %04d.%04d.%04d, "
2917 	    "HW rev: %02x\n", state->ts_instance, state->ts_fw.fw_rev_major,
2918 	    state->ts_fw.fw_rev_minor, state->ts_fw.fw_rev_subminor,
2919 	    state->ts_adapter.rev_id);
2920 	cmn_err(CE_CONT, "?tavor%d: %64s (0x%016" PRIx64 ")\n",
2921 	    state->ts_instance, state->ts_nodedesc, state->ts_nodeguid);
2922 }
2923 
2924 
2925 /*
2926  * tavor_pci_capability_list()
2927  *    Context: Only called from attach() path context
2928  */
2929 static void
tavor_pci_capability_list(tavor_state_t * state,ddi_acc_handle_t hdl)2930 tavor_pci_capability_list(tavor_state_t *state, ddi_acc_handle_t hdl)
2931 {
2932 	uint_t	offset, data;
2933 
2934 	TAVOR_TNF_ENTER(tavor_pci_capability_list);
2935 
2936 	/*
2937 	 * Check for the "PCI Capabilities" bit in the "Status Register".
2938 	 * Bit 4 in this register indicates the presence of a "PCI
2939 	 * Capabilities" list.
2940 	 */
2941 	data = pci_config_get16(hdl, 0x6);
2942 	if ((data & 0x10) == 0) {
2943 		TNF_PROBE_0(tavor_pci_capab_list_fail, TAVOR_TNF_ERROR, "");
2944 		TAVOR_TNF_EXIT(tavor_pci_capability_list);
2945 		return;
2946 	}
2947 
2948 	/*
2949 	 * Starting from offset 0x34 in PCI config space, find the
2950 	 * head of "PCI capabilities" list, and walk the list.  If
2951 	 * capabilities of a known type are encountered (e.g.
2952 	 * "PCI-X Capability"), then call the appropriate handler
2953 	 * function.
2954 	 */
2955 	offset = pci_config_get8(hdl, 0x34);
2956 	while (offset != 0x0) {
2957 		data = pci_config_get8(hdl, offset);
2958 
2959 		/*
2960 		 * Check for known capability types.  Tavor has the
2961 		 * following:
2962 		 *    o VPD Capability   (0x03)
2963 		 *    o PCI-X Capability (0x07)
2964 		 *    o MSI Capability   (0x05)
2965 		 *    o MSIX Capability  (0x11)
2966 		 */
2967 		switch (data) {
2968 		case 0x03:
2969 			tavor_pci_capability_vpd(state, hdl, offset);
2970 			break;
2971 		case 0x07:
2972 			tavor_pci_capability_pcix(state, hdl, offset);
2973 			break;
2974 		case 0x05:
2975 			break;
2976 		default:
2977 			break;
2978 		}
2979 
2980 		/* Get offset of next entry in list */
2981 		offset = pci_config_get8(hdl, offset + 1);
2982 	}
2983 
2984 	TAVOR_TNF_EXIT(tavor_pci_capability_list);
2985 }
2986 
2987 /*
2988  * tavor_pci_read_vpd()
2989  *    Context: Only called from attach() path context
2990  *    utility routine for tavor_pci_capability_vpd()
2991  */
2992 static int
tavor_pci_read_vpd(ddi_acc_handle_t hdl,uint_t offset,uint32_t addr,uint32_t * data)2993 tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, uint32_t addr,
2994     uint32_t *data)
2995 {
2996 	int		retry = 4;  /* retry counter for EEPROM poll */
2997 	uint32_t	val;
2998 	int		vpd_addr = offset + 2;
2999 	int		vpd_data = offset + 4;
3000 
3001 	TAVOR_TNF_ENTER(tavor_pci_read_vpd);
3002 
3003 	/*
3004 	 * In order to read a 32-bit value from VPD, we are to write down
3005 	 * the address (offset in the VPD itself) to the address register.
3006 	 * To signal the read, we also clear bit 31.  We then poll on bit 31
3007 	 * and when it is set, we can then read our 4 bytes from the data
3008 	 * register.
3009 	 */
3010 	(void) pci_config_put32(hdl, offset, addr << 16);
3011 	do {
3012 		drv_usecwait(1000);
3013 		val = pci_config_get16(hdl, vpd_addr);
3014 		if ((val >> 15) & 0x01) {
3015 			*data = pci_config_get32(hdl, vpd_data);
3016 			TAVOR_TNF_EXIT(tavor_pci_read_vpd);
3017 			return (DDI_SUCCESS);
3018 		}
3019 	} while (--retry);
3020 
3021 	TNF_PROBE_0(tavor_pci_read_vpd_fail, TAVOR_TNF_ERROR, "");
3022 	TAVOR_TNF_EXIT(tavor_pci_read_vpd);
3023 	return (DDI_FAILURE);
3024 }
3025 
3026 
3027 /*
3028  * tavor_pci_capability_vpd()
3029  *    Context: Only called from attach() path context
3030  */
3031 static void
tavor_pci_capability_vpd(tavor_state_t * state,ddi_acc_handle_t hdl,uint_t offset)3032 tavor_pci_capability_vpd(tavor_state_t *state, ddi_acc_handle_t hdl,
3033     uint_t offset)
3034 {
3035 	uint8_t			name_length;
3036 	uint8_t			pn_length;
3037 	int			i, err = 0;
3038 	int			vpd_str_id = 0;
3039 	int			vpd_ro_desc;
3040 	int			vpd_ro_pn_desc;
3041 #ifndef _LITTLE_ENDIAN
3042 	uint32_t		data32;
3043 #endif /* _LITTLE_ENDIAN */
3044 	union {
3045 		uint32_t	vpd_int[TAVOR_VPD_HDR_DWSIZE];
3046 		uchar_t		vpd_char[TAVOR_VPD_HDR_BSIZE];
3047 	} vpd;
3048 
3049 	TAVOR_TNF_ENTER(tavor_pci_capability_vpd);
3050 
3051 	/*
3052 	 * Read Vital Product Data (VPD) from PCI-X capability.
3053 	 */
3054 	for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) {
3055 		err = tavor_pci_read_vpd(hdl, offset, i << 2, &vpd.vpd_int[i]);
3056 		if (err != DDI_SUCCESS) {
3057 			cmn_err(CE_NOTE, "!VPD read failed\n");
3058 			goto out;
3059 		}
3060 	}
3061 
3062 #ifndef _LITTLE_ENDIAN
3063 	/*
3064 	 * Need to swap bytes for big endian.
3065 	 */
3066 	for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) {
3067 		data32 = vpd.vpd_int[i];
3068 		vpd.vpd_char[(i << 2) + 3] =
3069 		    (uchar_t)((data32 & 0xFF000000) >> 24);
3070 		vpd.vpd_char[(i << 2) + 2] =
3071 		    (uchar_t)((data32 & 0x00FF0000) >> 16);
3072 		vpd.vpd_char[(i << 2) + 1] =
3073 		    (uchar_t)((data32 & 0x0000FF00) >> 8);
3074 		vpd.vpd_char[i << 2] = (uchar_t)(data32 & 0x000000FF);
3075 	}
3076 #endif	/* _LITTLE_ENDIAN */
3077 
3078 	/* Check for VPD String ID Tag */
3079 	if (vpd.vpd_char[vpd_str_id] == 0x82) {
3080 		/* get the product name */
3081 		name_length = (uint8_t)vpd.vpd_char[vpd_str_id + 1];
3082 		if (name_length > sizeof (state->ts_hca_name)) {
3083 			cmn_err(CE_NOTE, "!VPD name too large (0x%x)\n",
3084 			    name_length);
3085 			goto out;
3086 		}
3087 		(void) memcpy(state->ts_hca_name, &vpd.vpd_char[vpd_str_id + 3],
3088 		    name_length);
3089 		state->ts_hca_name[name_length] = 0;
3090 
3091 		/* get the part number */
3092 		vpd_ro_desc = name_length + 3; /* read-only tag location */
3093 		vpd_ro_pn_desc = vpd_ro_desc + 3; /* P/N keyword location */
3094 		/*
3095 		 * Verify read-only tag and Part Number keyword.
3096 		 */
3097 		if (vpd.vpd_char[vpd_ro_desc] != 0x90 ||
3098 		    (vpd.vpd_char[vpd_ro_pn_desc] != 'P' &&
3099 		    vpd.vpd_char[vpd_ro_pn_desc + 1] != 'N')) {
3100 			cmn_err(CE_NOTE, "!VPD Part Number not found\n");
3101 			goto out;
3102 		}
3103 
3104 		pn_length = (uint8_t)vpd.vpd_char[vpd_ro_pn_desc + 2];
3105 		if (pn_length > sizeof (state->ts_hca_pn)) {
3106 			cmn_err(CE_NOTE, "!VPD part number too large (0x%x)\n",
3107 			    name_length);
3108 			goto out;
3109 		}
3110 		(void) memcpy(state->ts_hca_pn,
3111 		    &vpd.vpd_char[vpd_ro_pn_desc + 3],
3112 		    pn_length);
3113 		state->ts_hca_pn[pn_length] = 0;
3114 		state->ts_hca_pn_len = pn_length;
3115 	} else {
3116 		/* Wrong VPD String ID Tag */
3117 		cmn_err(CE_NOTE, "!VPD String ID Tag not found, tag: %02x\n",
3118 		    vpd.vpd_char[0]);
3119 		goto out;
3120 	}
3121 	TAVOR_TNF_EXIT(tavor_pci_capability_vpd);
3122 	return;
3123 out:
3124 	state->ts_hca_pn_len = 0;
3125 	TNF_PROBE_0(tavor_pci_capability_vpd_fail, TAVOR_TNF_ERROR, "");
3126 	TAVOR_TNF_EXIT(tavor_pci_capability_vpd);
3127 }
3128 
3129 /*
3130  * tavor_pci_capability_pcix()
3131  *    Context: Only called from attach() path context
3132  */
3133 static void
tavor_pci_capability_pcix(tavor_state_t * state,ddi_acc_handle_t hdl,uint_t offset)3134 tavor_pci_capability_pcix(tavor_state_t *state, ddi_acc_handle_t hdl,
3135     uint_t offset)
3136 {
3137 	uint_t	command, status;
3138 	int	max_out_splt_trans, max_mem_rd_byte_cnt;
3139 	int	designed_max_out_splt_trans, designed_max_mem_rd_byte_cnt;
3140 
3141 	TAVOR_TNF_ENTER(tavor_pci_capability_pcix);
3142 
3143 	/*
3144 	 * Query the current values for the PCI-X Command Register and
3145 	 * the PCI-X Status Register.
3146 	 */
3147 	command = pci_config_get16(hdl, offset + 2);
3148 	status  = pci_config_get32(hdl, offset + 4);
3149 
3150 	/*
3151 	 * Check for config property specifying "maximum outstanding
3152 	 * split transactions".  If the property is defined and valid
3153 	 * (i.e. no larger than the so-called "designed maximum"),
3154 	 * then use the specified value to update the PCI-X Command Register.
3155 	 * Otherwise, extract the value from the Tavor config profile.
3156 	 */
3157 	designed_max_out_splt_trans = ((status >> 23) & 7);
3158 	max_out_splt_trans = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip,
3159 	    DDI_PROP_DONTPASS, "pcix-max-outstanding-split-trans", -1);
3160 	if ((max_out_splt_trans != -1) &&
3161 	    ((max_out_splt_trans < 0) ||
3162 	    (max_out_splt_trans > designed_max_out_splt_trans))) {
3163 		cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-outstanding-"
3164 		    "split-trans\" (%d) invalid or exceeds device maximum"
3165 		    " (%d), using default value (%d)\n", state->ts_instance,
3166 		    max_out_splt_trans, designed_max_out_splt_trans,
3167 		    state->ts_cfg_profile->cp_max_out_splt_trans);
3168 		max_out_splt_trans =
3169 		    state->ts_cfg_profile->cp_max_out_splt_trans;
3170 	} else if (max_out_splt_trans == -1) {
3171 		max_out_splt_trans =
3172 		    state->ts_cfg_profile->cp_max_out_splt_trans;
3173 	}
3174 
3175 	/*
3176 	 * The config profile setting for max_out_splt_trans is determined
3177 	 * based on arch.  Check tavor_cfg.c for more information.  A value of
3178 	 * '-1' in the patchable variable means "do not change".  A value of
3179 	 * '0' means 1 outstanding splt trans and other values as defined by
3180 	 * PCI.  So we do one more check here, that if 'max_out_splt_trans' is
3181 	 * -1 (ie: < 0) we do not set the PCI command and leave it at the
3182 	 * default.
3183 	 */
3184 	if (max_out_splt_trans >= 0) {
3185 		command = ((command & 0xFF8F) | max_out_splt_trans << 4);
3186 	}
3187 
3188 	/*
3189 	 * Check for config property specifying "maximum memory read
3190 	 * byte count.  If the property is defined and valid
3191 	 * (i.e. no larger than the so-called "designed maximum"),
3192 	 * then use the specified value to update the PCI-X Command Register.
3193 	 * Otherwise, extract the value from the Tavor config profile.
3194 	 */
3195 	designed_max_mem_rd_byte_cnt = ((status >> 21) & 3);
3196 	max_mem_rd_byte_cnt = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip,
3197 	    DDI_PROP_DONTPASS, "pcix-max-read-byte-count", -1);
3198 	if ((max_mem_rd_byte_cnt != -1) &&
3199 	    ((max_mem_rd_byte_cnt < 0) ||
3200 	    (max_mem_rd_byte_cnt > designed_max_mem_rd_byte_cnt))) {
3201 		cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-read-byte-"
3202 		    "count\" (%d) invalid or exceeds device maximum"
3203 		    " (%d), using default value (%d)\n", state->ts_instance,
3204 		    max_mem_rd_byte_cnt, designed_max_mem_rd_byte_cnt,
3205 		    state->ts_cfg_profile->cp_max_mem_rd_byte_cnt);
3206 		max_mem_rd_byte_cnt =
3207 		    state->ts_cfg_profile->cp_max_mem_rd_byte_cnt;
3208 	} else if (max_mem_rd_byte_cnt == -1) {
3209 		max_mem_rd_byte_cnt =
3210 		    state->ts_cfg_profile->cp_max_mem_rd_byte_cnt;
3211 	}
3212 
3213 	/*
3214 	 * The config profile setting for max_mem_rd_byte_cnt is determined
3215 	 * based on arch.  Check tavor_cfg.c for more information.  A value of
3216 	 * '-1' in the patchable variable means "do not change".  A value of
3217 	 * '0' means minimum (512B) read, and other values as defined by
3218 	 * PCI.  So we do one more check here, that if 'max_mem_rd_byte_cnt' is
3219 	 * -1 (ie: < 0) we do not set the PCI command and leave it at the
3220 	 * default.
3221 	 */
3222 	if (max_mem_rd_byte_cnt >= 0) {
3223 		command = ((command & 0xFFF3) | max_mem_rd_byte_cnt << 2);
3224 	}
3225 
3226 	/*
3227 	 * Update the PCI-X Command Register with the newly configured
3228 	 * values.
3229 	 */
3230 	pci_config_put16(hdl, offset + 2, command);
3231 
3232 	TAVOR_TNF_EXIT(tavor_pci_capability_pcix);
3233 }
3234 
3235 
3236 /*
3237  * tavor_intr_or_msi_init()
3238  *    Context: Only called from attach() path context
3239  */
3240 static int
tavor_intr_or_msi_init(tavor_state_t * state)3241 tavor_intr_or_msi_init(tavor_state_t *state)
3242 {
3243 	int	status;
3244 
3245 	TAVOR_TNF_ENTER(tavor_intr_or_msi_init);
3246 
3247 	/* Query for the list of supported interrupt event types */
3248 	status = ddi_intr_get_supported_types(state->ts_dip,
3249 	    &state->ts_intr_types_avail);
3250 	if (status != DDI_SUCCESS) {
3251 		TNF_PROBE_0(tavor_intr_or_msi_init_gettypes_fail,
3252 		    TAVOR_TNF_ERROR, "");
3253 		TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3254 		return (DDI_FAILURE);
3255 	}
3256 
3257 	/*
3258 	 * If Tavor/Arbel supports MSI in this system (and, if it
3259 	 * hasn't been overridden by a configuration variable), then
3260 	 * the default behavior is to use a single MSI.  Otherwise,
3261 	 * fallback to using legacy interrupts.  Also, if MSI allocatis chosen,
3262 	 * but fails for whatever reasons, then fallback to using legacy
3263 	 * interrupts.
3264 	 */
3265 	if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) &&
3266 	    (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) {
3267 		status = tavor_add_intrs(state, DDI_INTR_TYPE_MSI);
3268 		if (status == DDI_SUCCESS) {
3269 			state->ts_intr_type_chosen = DDI_INTR_TYPE_MSI;
3270 			TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3271 			return (DDI_SUCCESS);
3272 		}
3273 	}
3274 
3275 	/*
3276 	 * MSI interrupt allocation failed, or was not available.  Fallback to
3277 	 * legacy interrupt support.
3278 	 */
3279 	if (state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED) {
3280 		status = tavor_add_intrs(state, DDI_INTR_TYPE_FIXED);
3281 		if (status == DDI_SUCCESS) {
3282 			state->ts_intr_type_chosen = DDI_INTR_TYPE_FIXED;
3283 			TAVOR_TNF_EXIT(tavor_intr_or_msi_init);
3284 			return (DDI_SUCCESS);
3285 		}
3286 	}
3287 
3288 	/*
3289 	 * Neither MSI or legacy interrupts were successful.  return failure.
3290 	 */
3291 	TAVOR_TNF_EXIT(tavor_intr_or_msi_setup);
3292 	return (DDI_FAILURE);
3293 }
3294 
3295 /*
3296  * tavor_add_intrs()
3297  *    Context: Only called from attach() patch context
3298  */
3299 static int
tavor_add_intrs(tavor_state_t * state,int intr_type)3300 tavor_add_intrs(tavor_state_t *state, int intr_type)
3301 {
3302 	int status;
3303 
3304 	TAVOR_TNF_ENTER(tavor_add_intrs);
3305 
3306 	/* Get number of interrupts/MSI supported */
3307 	status = ddi_intr_get_nintrs(state->ts_dip, intr_type,
3308 	    &state->ts_intrmsi_count);
3309 	if (status != DDI_SUCCESS) {
3310 		TNF_PROBE_0(tavor_add_intrs_getnintrs_fail,
3311 		    TAVOR_TNF_ERROR, "");
3312 		TAVOR_TNF_EXIT(tavor_add_intrs);
3313 		return (DDI_FAILURE);
3314 	}
3315 
3316 	/* Get number of available interrupts/MSI */
3317 	status = ddi_intr_get_navail(state->ts_dip, intr_type,
3318 	    &state->ts_intrmsi_avail);
3319 	if (status != DDI_SUCCESS) {
3320 		TNF_PROBE_0(tavor_add_intrs_getnavail_fail,
3321 		    TAVOR_TNF_ERROR, "");
3322 		TAVOR_TNF_EXIT(tavor_add_intrs);
3323 		return (DDI_FAILURE);
3324 	}
3325 
3326 	/* Ensure that we have at least one (1) usable MSI or interrupt */
3327 	if ((state->ts_intrmsi_avail < 1) || (state->ts_intrmsi_count < 1)) {
3328 		TNF_PROBE_0(tavor_add_intrs_notenoughts_intrmsi_fail,
3329 		    TAVOR_TNF_ERROR, "");
3330 		TAVOR_TNF_EXIT(tavor_add_intrs);
3331 		return (DDI_FAILURE);
3332 	}
3333 
3334 	/* Attempt to allocate a single interrupt/MSI handle */
3335 	status = ddi_intr_alloc(state->ts_dip, &state->ts_intrmsi_hdl,
3336 	    intr_type, 0, 1, &state->ts_intrmsi_allocd,
3337 	    DDI_INTR_ALLOC_STRICT);
3338 	if (status != DDI_SUCCESS) {
3339 		TNF_PROBE_0(tavor_add_intrs_intralloc_fail,
3340 		    TAVOR_TNF_ERROR, "");
3341 		TAVOR_TNF_EXIT(tavor_add_intrs);
3342 		return (DDI_FAILURE);
3343 	}
3344 
3345 	/* Ensure that we have allocated at least one (1) MSI or interrupt */
3346 	if (state->ts_intrmsi_allocd < 1) {
3347 		TNF_PROBE_0(tavor_add_intrs_noallocts_intrmsi_fail,
3348 		    TAVOR_TNF_ERROR, "");
3349 		TAVOR_TNF_EXIT(tavor_add_intrs);
3350 		return (DDI_FAILURE);
3351 	}
3352 
3353 	/*
3354 	 * Extract the priority for the allocated interrupt/MSI.  This
3355 	 * will be used later when initializing certain mutexes.
3356 	 */
3357 	status = ddi_intr_get_pri(state->ts_intrmsi_hdl,
3358 	    &state->ts_intrmsi_pri);
3359 	if (status != DDI_SUCCESS) {
3360 		/* Free the allocated interrupt/MSI handle */
3361 		(void) ddi_intr_free(state->ts_intrmsi_hdl);
3362 
3363 		TNF_PROBE_0(tavor_add_intrs_getpri_fail,
3364 		    TAVOR_TNF_ERROR, "");
3365 		TAVOR_TNF_EXIT(tavor_add_intrs);
3366 		return (DDI_FAILURE);
3367 	}
3368 
3369 	/* Make sure the interrupt/MSI priority is below 'high level' */
3370 	if (state->ts_intrmsi_pri >= ddi_intr_get_hilevel_pri()) {
3371 		/* Free the allocated interrupt/MSI handle */
3372 		(void) ddi_intr_free(state->ts_intrmsi_hdl);
3373 
3374 		TNF_PROBE_0(tavor_add_intrs_hilevelpri_fail,
3375 		    TAVOR_TNF_ERROR, "");
3376 		TAVOR_TNF_EXIT(tavor_add_intrs);
3377 		return (DDI_FAILURE);
3378 	}
3379 
3380 	/* Get add'l capability information regarding interrupt/MSI */
3381 	status = ddi_intr_get_cap(state->ts_intrmsi_hdl,
3382 	    &state->ts_intrmsi_cap);
3383 	if (status != DDI_SUCCESS) {
3384 		/* Free the allocated interrupt/MSI handle */
3385 		(void) ddi_intr_free(state->ts_intrmsi_hdl);
3386 
3387 		TNF_PROBE_0(tavor_add_intrs_getcap_fail,
3388 		    TAVOR_TNF_ERROR, "");
3389 		TAVOR_TNF_EXIT(tavor_add_intrs);
3390 		return (DDI_FAILURE);
3391 	}
3392 
3393 	TAVOR_TNF_EXIT(tavor_add_intrs);
3394 	return (DDI_SUCCESS);
3395 }
3396 
3397 
3398 /*
3399  * tavor_intr_or_msi_fini()
3400  *    Context: Only called from attach() and/or detach() path contexts
3401  */
3402 static int
tavor_intr_or_msi_fini(tavor_state_t * state)3403 tavor_intr_or_msi_fini(tavor_state_t *state)
3404 {
3405 	int	status;
3406 
3407 	TAVOR_TNF_ENTER(tavor_intr_or_msi_fini);
3408 
3409 	/* Free the allocated interrupt/MSI handle */
3410 	status = ddi_intr_free(state->ts_intrmsi_hdl);
3411 	if (status != DDI_SUCCESS) {
3412 		TNF_PROBE_0(tavor_intr_or_msi_fini_freehdl_fail,
3413 		    TAVOR_TNF_ERROR, "");
3414 		TAVOR_TNF_EXIT(tavor_intr_or_msi_fini);
3415 		return (DDI_FAILURE);
3416 	}
3417 
3418 	TAVOR_TNF_EXIT(tavor_intr_or_msi_fini);
3419 	return (DDI_SUCCESS);
3420 }
3421 
3422 
3423 /* Disable Tavor interrupts */
3424 static int
tavor_intr_disable(tavor_state_t * state)3425 tavor_intr_disable(tavor_state_t *state)
3426 {
3427 	ushort_t msi_ctrl = 0, caps_ctrl = 0;
3428 	ddi_acc_handle_t pci_cfg_hdl = state->ts_pci_cfghdl;
3429 	ASSERT(pci_cfg_hdl != NULL);
3430 	ASSERT(state->ts_intr_types_avail &
3431 	    (DDI_INTR_TYPE_FIXED | DDI_INTR_TYPE_MSI));
3432 
3433 	/*
3434 	 * Check if MSI interrupts are used. If so, disable MSI interupts.
3435 	 * If not, since Tavor doesn't support MSI-X interrupts, assuming the
3436 	 * legacy interrupt is used instead, disable the legacy interrupt.
3437 	 */
3438 	if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) &&
3439 	    (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) {
3440 
3441 		if ((PCI_CAP_LOCATE(pci_cfg_hdl, PCI_CAP_ID_MSI,
3442 		    &caps_ctrl) == DDI_SUCCESS)) {
3443 			if ((msi_ctrl = PCI_CAP_GET16(pci_cfg_hdl, 0,
3444 			    caps_ctrl, PCI_MSI_CTRL)) == PCI_CAP_EINVAL16)
3445 				return (DDI_FAILURE);
3446 		}
3447 		ASSERT(msi_ctrl != 0);
3448 
3449 		if (!(msi_ctrl & PCI_MSI_ENABLE_BIT))
3450 			return (DDI_SUCCESS);
3451 
3452 		if (msi_ctrl &  PCI_MSI_PVM_MASK) {
3453 			int offset = (msi_ctrl &  PCI_MSI_64BIT_MASK) ?
3454 			    PCI_MSI_64BIT_MASKBITS : PCI_MSI_32BIT_MASK;
3455 
3456 			/* Clear all inums in MSI */
3457 			PCI_CAP_PUT32(pci_cfg_hdl, 0, caps_ctrl, offset, 0);
3458 		}
3459 
3460 		/* Disable MSI interrupts */
3461 		msi_ctrl &= ~PCI_MSI_ENABLE_BIT;
3462 		PCI_CAP_PUT16(pci_cfg_hdl, 0, caps_ctrl, PCI_MSI_CTRL,
3463 		    msi_ctrl);
3464 
3465 	} else {
3466 		uint16_t cmdreg = pci_config_get16(pci_cfg_hdl, PCI_CONF_COMM);
3467 		ASSERT(state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED);
3468 
3469 		/* Disable the legacy interrupts */
3470 		cmdreg |= PCI_COMM_INTX_DISABLE;
3471 		pci_config_put16(pci_cfg_hdl, PCI_CONF_COMM, cmdreg);
3472 	}
3473 
3474 	return (DDI_SUCCESS);
3475 }
3476 
3477 /* Tavor quiesce(9F) entry */
3478 static int
tavor_quiesce(dev_info_t * dip)3479 tavor_quiesce(dev_info_t *dip)
3480 {
3481 	tavor_state_t *state = ddi_get_soft_state(tavor_statep,
3482 	    DEVI(dip)->devi_instance);
3483 	ASSERT(state != NULL);
3484 
3485 	/* start fastreboot */
3486 	state->ts_quiescing = B_TRUE;
3487 
3488 	/* If it's in maintenance mode, do nothing but return with SUCCESS */
3489 	if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) {
3490 		return (DDI_SUCCESS);
3491 	}
3492 
3493 	/* Shutdown HCA ports */
3494 	if (tavor_hca_ports_shutdown(state,
3495 	    state->ts_cfg_profile->cp_num_ports) != TAVOR_CMD_SUCCESS) {
3496 		state->ts_quiescing = B_FALSE;
3497 		return (DDI_FAILURE);
3498 	}
3499 
3500 	/* Close HCA */
3501 	if (tavor_close_hca_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) !=
3502 	    TAVOR_CMD_SUCCESS) {
3503 		state->ts_quiescing = B_FALSE;
3504 		return (DDI_FAILURE);
3505 	}
3506 
3507 	/* Shutdown FW */
3508 	if (tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) !=
3509 	    TAVOR_CMD_SUCCESS) {
3510 		state->ts_quiescing = B_FALSE;
3511 		return (DDI_FAILURE);
3512 	}
3513 
3514 	/* Disable interrupts */
3515 	if (tavor_intr_disable(state) != DDI_SUCCESS) {
3516 		state->ts_quiescing = B_FALSE;
3517 		return (DDI_FAILURE);
3518 	}
3519 
3520 	/* SW-reset */
3521 	if (tavor_sw_reset(state) != DDI_SUCCESS) {
3522 		state->ts_quiescing = B_FALSE;
3523 		return (DDI_FAILURE);
3524 	}
3525 
3526 	return (DDI_SUCCESS);
3527 }
3528