1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2016 Nexenta Systems, Inc.
14  */
15 
16 #include <sys/atomic.h>
17 #include <sys/cmn_err.h>
18 #include <sys/conf.h>
19 #include <sys/cpuvar.h>
20 #include <sys/ddi.h>
21 #include <sys/errno.h>
22 #include <sys/fs/dv_node.h>
23 #include <sys/kmem.h>
24 #include <sys/kmem_impl.h>
25 #include <sys/list.h>
26 #include <sys/modctl.h>
27 #include <sys/pci.h>
28 #include <sys/scsi/scsi.h>
29 #include <sys/sunddi.h>
30 #include <sys/sysmacros.h>
31 #include <sys/time.h>
32 #include <sys/types.h>
33 
34 #include "pvscsi.h"
35 #include "pvscsi_var.h"
36 
37 int pvscsi_enable_msi = 1;
38 int pvscsi_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_PER_RING;
39 int pvscsi_msg_ring_pages = PVSCSI_DEFAULT_NUM_PAGES_MSG_RING;
40 
41 static int pvscsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt);
42 
43 static void *pvscsi_sstate;
44 
45 /* HBA DMA attributes */
46 static ddi_dma_attr_t pvscsi_hba_dma_attr = {
47 	.dma_attr_version =	DMA_ATTR_V0,
48 	.dma_attr_addr_lo =	0x0000000000000000ull,
49 	.dma_attr_addr_hi =	0xFFFFFFFFFFFFFFFFull,
50 	.dma_attr_count_max =	0x000000007FFFFFFFull,
51 	.dma_attr_align =	0x0000000000000001ull,
52 	.dma_attr_burstsizes =	0x7ff,
53 	.dma_attr_minxfer =	0x00000001u,
54 	.dma_attr_maxxfer =	0x00000000FFFFFFFFull,
55 	.dma_attr_seg =		0x00000000FFFFFFFFull,
56 	.dma_attr_sgllen =	1,
57 	.dma_attr_granular =	0x00000200u,
58 	.dma_attr_flags =	0
59 };
60 
61 /* DMA attributes for req/comp rings */
62 static ddi_dma_attr_t pvscsi_ring_dma_attr = {
63 	.dma_attr_version =	DMA_ATTR_V0,
64 	.dma_attr_addr_lo =	0x0000000000000000ull,
65 	.dma_attr_addr_hi =	0xFFFFFFFFFFFFFFFFull,
66 	.dma_attr_count_max =	0x000000007FFFFFFFull,
67 	.dma_attr_align =	0x0000000000000001ull,
68 	.dma_attr_burstsizes =	0x7ff,
69 	.dma_attr_minxfer =	0x00000001u,
70 	.dma_attr_maxxfer =	0x00000000FFFFFFFFull,
71 	.dma_attr_seg =		0x00000000FFFFFFFFull,
72 	.dma_attr_sgllen =	1,
73 	.dma_attr_granular =	0x00000001u,
74 	.dma_attr_flags =	0
75 };
76 
77 /* DMA attributes for buffer I/O */
78 static ddi_dma_attr_t pvscsi_io_dma_attr = {
79 	.dma_attr_version =	DMA_ATTR_V0,
80 	.dma_attr_addr_lo =	0x0000000000000000ull,
81 	.dma_attr_addr_hi =	0xFFFFFFFFFFFFFFFFull,
82 	.dma_attr_count_max =	0x000000007FFFFFFFull,
83 	.dma_attr_align =	0x0000000000000001ull,
84 	.dma_attr_burstsizes =	0x7ff,
85 	.dma_attr_minxfer =	0x00000001u,
86 	.dma_attr_maxxfer =	0x00000000FFFFFFFFull,
87 	.dma_attr_seg =		0x00000000FFFFFFFFull,
88 	.dma_attr_sgllen =	PVSCSI_MAX_SG_SIZE,
89 	.dma_attr_granular =	0x00000200u,
90 	.dma_attr_flags =	0
91 };
92 
93 static ddi_device_acc_attr_t pvscsi_mmio_attr = {
94 	DDI_DEVICE_ATTR_V1,
95 	DDI_STRUCTURE_LE_ACC,
96 	DDI_STRICTORDER_ACC,
97 	DDI_DEFAULT_ACC
98 };
99 
100 static ddi_device_acc_attr_t pvscsi_dma_attrs = {
101 	DDI_DEVICE_ATTR_V0,
102 	DDI_STRUCTURE_LE_ACC,
103 	DDI_STRICTORDER_ACC,
104 	DDI_DEFAULT_ACC,
105 };
106 
107 static void
pvscsi_add_to_queue(pvscsi_cmd_t * cmd)108 pvscsi_add_to_queue(pvscsi_cmd_t *cmd)
109 {
110 	pvscsi_softc_t	*pvs = cmd->cmd_pvs;
111 
112 	ASSERT(pvs != NULL);
113 	ASSERT(mutex_owned(&pvs->mutex));
114 	ASSERT(!list_link_active(&(cmd)->cmd_queue_node));
115 
116 	list_insert_tail(&pvs->cmd_queue, cmd);
117 	pvs->cmd_queue_len++;
118 }
119 
120 static void
pvscsi_remove_from_queue(pvscsi_cmd_t * cmd)121 pvscsi_remove_from_queue(pvscsi_cmd_t *cmd)
122 {
123 	pvscsi_softc_t	*pvs = cmd->cmd_pvs;
124 
125 	ASSERT(pvs != NULL);
126 	ASSERT(mutex_owned(&pvs->mutex));
127 	ASSERT(list_link_active(&cmd->cmd_queue_node));
128 	ASSERT(pvs->cmd_queue_len > 0);
129 
130 	if (list_link_active(&cmd->cmd_queue_node)) {
131 		list_remove(&pvs->cmd_queue, cmd);
132 		pvs->cmd_queue_len--;
133 	}
134 }
135 
136 static uint64_t
pvscsi_map_ctx(pvscsi_softc_t * pvs,pvscsi_cmd_ctx_t * io_ctx)137 pvscsi_map_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_ctx_t *io_ctx)
138 {
139 	return (io_ctx - pvs->cmd_ctx + 1);
140 }
141 
142 static pvscsi_cmd_ctx_t *
pvscsi_lookup_ctx(pvscsi_softc_t * pvs,pvscsi_cmd_t * cmd)143 pvscsi_lookup_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
144 {
145 	pvscsi_cmd_ctx_t *ctx, *end;
146 
147 	end = &pvs->cmd_ctx[pvs->req_depth];
148 	for (ctx = pvs->cmd_ctx; ctx < end; ctx++) {
149 		if (ctx->cmd == cmd)
150 			return (ctx);
151 	}
152 
153 	return (NULL);
154 }
155 
156 static pvscsi_cmd_ctx_t *
pvscsi_resolve_ctx(pvscsi_softc_t * pvs,uint64_t ctx)157 pvscsi_resolve_ctx(pvscsi_softc_t *pvs, uint64_t ctx)
158 {
159 	if (ctx > 0 && ctx <= pvs->req_depth)
160 		return (&pvs->cmd_ctx[ctx - 1]);
161 	else
162 		return (NULL);
163 }
164 
165 static boolean_t
pvscsi_acquire_ctx(pvscsi_softc_t * pvs,pvscsi_cmd_t * cmd)166 pvscsi_acquire_ctx(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
167 {
168 	pvscsi_cmd_ctx_t *ctx;
169 
170 	if (list_is_empty(&pvs->cmd_ctx_pool))
171 		return (B_FALSE);
172 
173 	ctx = (pvscsi_cmd_ctx_t *)list_remove_head(&pvs->cmd_ctx_pool);
174 	ASSERT(ctx != NULL);
175 
176 	ctx->cmd = cmd;
177 	cmd->ctx = ctx;
178 
179 	return (B_TRUE);
180 }
181 
182 static void
pvscsi_release_ctx(pvscsi_cmd_t * cmd)183 pvscsi_release_ctx(pvscsi_cmd_t *cmd)
184 {
185 	pvscsi_softc_t	*pvs = cmd->cmd_pvs;
186 
187 	ASSERT(mutex_owned(&pvs->mutex));
188 
189 	cmd->ctx->cmd = NULL;
190 	list_insert_tail(&pvs->cmd_ctx_pool, cmd->ctx);
191 	cmd->ctx = NULL;
192 }
193 
194 static uint32_t
pvscsi_reg_read(pvscsi_softc_t * pvs,uint32_t offset)195 pvscsi_reg_read(pvscsi_softc_t *pvs, uint32_t offset)
196 {
197 	uint32_t	ret;
198 
199 	ASSERT((offset & (sizeof (uint32_t) - 1)) == 0);
200 
201 	ret = ddi_get32(pvs->mmio_handle,
202 	    (uint32_t *)(pvs->mmio_base + offset));
203 
204 	return (ret);
205 }
206 
207 static void
pvscsi_reg_write(pvscsi_softc_t * pvs,uint32_t offset,uint32_t value)208 pvscsi_reg_write(pvscsi_softc_t *pvs, uint32_t offset, uint32_t value)
209 {
210 	ASSERT((offset & (sizeof (uint32_t) - 1)) == 0);
211 
212 	ddi_put32(pvs->mmio_handle, (uint32_t *)(pvs->mmio_base + offset),
213 	    value);
214 }
215 
216 static void
pvscsi_write_cmd_desc(pvscsi_softc_t * pvs,uint32_t cmd,void * desc,size_t len)217 pvscsi_write_cmd_desc(pvscsi_softc_t *pvs, uint32_t cmd, void *desc, size_t len)
218 {
219 	len /= sizeof (uint32_t);
220 	pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_COMMAND, cmd);
221 	ddi_rep_put32(pvs->mmio_handle, (uint32_t *)desc,
222 	    (uint32_t *)(pvs->mmio_base + PVSCSI_REG_OFFSET_COMMAND_DATA),
223 	    len, DDI_DEV_NO_AUTOINCR);
224 }
225 
226 static uint32_t
pvscsi_read_intr_status(pvscsi_softc_t * pvs)227 pvscsi_read_intr_status(pvscsi_softc_t *pvs)
228 {
229 	return (pvscsi_reg_read(pvs, PVSCSI_REG_OFFSET_INTR_STATUS));
230 }
231 
232 static void
pvscsi_write_intr_status(pvscsi_softc_t * pvs,uint32_t val)233 pvscsi_write_intr_status(pvscsi_softc_t *pvs, uint32_t val)
234 {
235 	pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_STATUS, val);
236 }
237 
238 static void
pvscsi_mask_intr(pvscsi_softc_t * pvs)239 pvscsi_mask_intr(pvscsi_softc_t *pvs)
240 {
241 	mutex_enter(&pvs->intr_mutex);
242 
243 	VERIFY(pvs->intr_lock_counter >= 0);
244 
245 	if (++pvs->intr_lock_counter == 1)
246 		pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK, 0);
247 
248 	mutex_exit(&pvs->intr_mutex);
249 }
250 
251 static void
pvscsi_unmask_intr(pvscsi_softc_t * pvs)252 pvscsi_unmask_intr(pvscsi_softc_t *pvs)
253 {
254 	mutex_enter(&pvs->intr_mutex);
255 
256 	VERIFY(pvs->intr_lock_counter > 0);
257 
258 	if (--pvs->intr_lock_counter == 0) {
259 		pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK,
260 		    PVSCSI_INTR_CMPL_MASK | PVSCSI_INTR_MSG_MASK);
261 	}
262 
263 	mutex_exit(&pvs->intr_mutex);
264 }
265 
266 static void
pvscsi_reset_hba(pvscsi_softc_t * pvs)267 pvscsi_reset_hba(pvscsi_softc_t *pvs)
268 {
269 	pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_ADAPTER_RESET, NULL, 0);
270 }
271 
272 static void
pvscsi_reset_bus(pvscsi_softc_t * pvs)273 pvscsi_reset_bus(pvscsi_softc_t *pvs)
274 {
275 	pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_RESET_BUS, NULL, 0);
276 }
277 
278 static void
pvscsi_submit_nonrw_io(pvscsi_softc_t * pvs)279 pvscsi_submit_nonrw_io(pvscsi_softc_t *pvs)
280 {
281 	pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_KICK_NON_RW_IO, 0);
282 }
283 
284 static void
pvscsi_submit_rw_io(pvscsi_softc_t * pvs)285 pvscsi_submit_rw_io(pvscsi_softc_t *pvs)
286 {
287 	pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_KICK_RW_IO, 0);
288 }
289 
290 
291 static int
pvscsi_inquiry_target(pvscsi_softc_t * pvs,int target,struct scsi_inquiry * inq)292 pvscsi_inquiry_target(pvscsi_softc_t *pvs, int target, struct scsi_inquiry *inq)
293 {
294 	int		len = sizeof (struct scsi_inquiry);
295 	int		ret = -1;
296 	struct buf	*b;
297 	struct scsi_address ap;
298 	struct scsi_pkt	*pkt;
299 	uint8_t		cdb[CDB_GROUP0];
300 
301 	ap.a_hba_tran = pvs->tran;
302 	ap.a_target = (ushort_t)target;
303 	ap.a_lun = (uchar_t)0;
304 
305 	if ((b = scsi_alloc_consistent_buf(&ap, (struct buf *)NULL, len,
306 	    B_READ, NULL_FUNC, NULL)) == NULL)
307 		return (-1);
308 
309 	if ((pkt = scsi_init_pkt(&ap, (struct scsi_pkt *)NULL, b,
310 	    CDB_GROUP0, sizeof (struct scsi_arq_status), 0, 0,
311 	    NULL_FUNC, NULL)) == NULL)
312 		goto free_buf;
313 
314 	cdb[0] = SCMD_INQUIRY;
315 	cdb[1] = 0;
316 	cdb[2] = 0;
317 	cdb[3] = (len & 0xff00) >> 8;
318 	cdb[4] = (len & 0x00ff);
319 	cdb[5] = 0;
320 
321 	if (inq != NULL)
322 		bzero(inq, sizeof (*inq));
323 	bcopy(cdb, pkt->pkt_cdbp, CDB_GROUP0);
324 	bzero((struct scsi_inquiry *)b->b_un.b_addr, sizeof (*inq));
325 
326 	if ((ret = scsi_poll(pkt)) == 0 && inq != NULL)
327 		bcopy(b->b_un.b_addr, inq, sizeof (*inq));
328 
329 	scsi_destroy_pkt(pkt);
330 
331 free_buf:
332 	scsi_free_consistent_buf(b);
333 
334 	return (ret);
335 }
336 
337 static int
pvscsi_config_one(dev_info_t * pdip,pvscsi_softc_t * pvs,int target,dev_info_t ** childp)338 pvscsi_config_one(dev_info_t *pdip, pvscsi_softc_t *pvs, int target,
339     dev_info_t **childp)
340 {
341 	char		**compatible = NULL;
342 	char		*nodename = NULL;
343 	dev_info_t	*dip;
344 	int		inqrc;
345 	int		ncompatible = 0;
346 	pvscsi_device_t	*devnode;
347 	struct scsi_inquiry inq;
348 
349 	ASSERT(DEVI_BUSY_OWNED(pdip));
350 
351 	/* Inquiry target */
352 	inqrc = pvscsi_inquiry_target(pvs, target, &inq);
353 
354 	/* Find devnode */
355 	for (devnode = list_head(&pvs->devnodes); devnode != NULL;
356 	    devnode = list_next(&pvs->devnodes, devnode)) {
357 		if (devnode->target == target)
358 			break;
359 	}
360 
361 	if (devnode != NULL) {
362 		if (inqrc != 0) {
363 			/* Target disappeared, drop devnode */
364 			if (i_ddi_devi_attached(devnode->pdip)) {
365 				char    *devname;
366 				/* Get full devname */
367 				devname = kmem_alloc(MAXPATHLEN, KM_SLEEP);
368 				(void) ddi_deviname(devnode->pdip, devname);
369 				/* Clean cache and name */
370 				(void) devfs_clean(devnode->parent, devname + 1,
371 				    DV_CLEAN_FORCE);
372 				kmem_free(devname, MAXPATHLEN);
373 			}
374 
375 			(void) ndi_devi_offline(devnode->pdip, NDI_DEVI_REMOVE);
376 
377 			list_remove(&pvs->devnodes, devnode);
378 			kmem_free(devnode, sizeof (*devnode));
379 		} else if (childp != NULL) {
380 			/* Target exists */
381 			*childp = devnode->pdip;
382 		}
383 		return (NDI_SUCCESS);
384 	} else if (inqrc != 0) {
385 		/* Target doesn't exist */
386 		return (NDI_FAILURE);
387 	}
388 
389 	scsi_hba_nodename_compatible_get(&inq, NULL, inq.inq_dtype, NULL,
390 	    &nodename, &compatible, &ncompatible);
391 	if (nodename == NULL)
392 		goto free_nodename;
393 
394 	if (ndi_devi_alloc(pdip, nodename, DEVI_SID_NODEID,
395 	    &dip) != NDI_SUCCESS) {
396 		dev_err(pvs->dip, CE_WARN, "!failed to alloc device instance");
397 		goto free_nodename;
398 	}
399 
400 	if (ndi_prop_update_string(DDI_DEV_T_NONE, dip,
401 	    "device-type", "scsi") != DDI_PROP_SUCCESS ||
402 	    ndi_prop_update_int(DDI_DEV_T_NONE, dip,
403 	    "target", target) != DDI_PROP_SUCCESS ||
404 	    ndi_prop_update_int(DDI_DEV_T_NONE, dip,
405 	    "lun", 0) != DDI_PROP_SUCCESS ||
406 	    ndi_prop_update_int(DDI_DEV_T_NONE, dip,
407 	    "pm-capable", 1) != DDI_PROP_SUCCESS ||
408 	    ndi_prop_update_string_array(DDI_DEV_T_NONE, dip,
409 	    "compatible", compatible, ncompatible) != DDI_PROP_SUCCESS) {
410 		dev_err(pvs->dip, CE_WARN,
411 		    "!failed to update props for target %d", target);
412 		goto free_devi;
413 	}
414 
415 	if ((devnode = kmem_zalloc(sizeof (*devnode), KM_NOSLEEP)) == NULL)
416 		goto free_devi;
417 
418 	if (ndi_devi_online(dip, NDI_ONLINE_ATTACH) != NDI_SUCCESS) {
419 		dev_err(pvs->dip, CE_WARN, "!failed to online target %d",
420 		    target);
421 		kmem_free(devnode, sizeof (*devnode));
422 		goto free_devi;
423 	}
424 
425 	devnode->target = target;
426 	devnode->pdip = dip;
427 	devnode->parent = pdip;
428 	list_insert_tail(&pvs->devnodes, devnode);
429 
430 	if (childp != NULL)
431 		*childp = dip;
432 
433 	scsi_hba_nodename_compatible_free(nodename, compatible);
434 
435 	return (NDI_SUCCESS);
436 
437 free_devi:
438 	ndi_prop_remove_all(dip);
439 	(void) ndi_devi_free(dip);
440 free_nodename:
441 	scsi_hba_nodename_compatible_free(nodename, compatible);
442 
443 	return (NDI_FAILURE);
444 }
445 
446 static int
pvscsi_config_all(dev_info_t * pdip,pvscsi_softc_t * pvs)447 pvscsi_config_all(dev_info_t *pdip, pvscsi_softc_t *pvs)
448 {
449 	int		target;
450 
451 	for (target = 0; target < PVSCSI_MAXTGTS; target++) {
452 		/* ndi_devi_enter is done in pvscsi_bus_config */
453 		(void) pvscsi_config_one(pdip, pvs, target, NULL);
454 	}
455 
456 	return (NDI_SUCCESS);
457 }
458 
459 static pvscsi_cmd_t *
pvscsi_process_comp_ring(pvscsi_softc_t * pvs)460 pvscsi_process_comp_ring(pvscsi_softc_t *pvs)
461 {
462 	pvscsi_cmd_t	**pnext_cmd = NULL;
463 	pvscsi_cmd_t	*cmd;
464 	pvscsi_cmd_t	*head = NULL;
465 	struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
466 	uint32_t	cmp_ne = sdesc->cmpNumEntriesLog2;
467 
468 	ASSERT(mutex_owned(&pvs->rx_mutex));
469 
470 	while (sdesc->cmpConsIdx != sdesc->cmpProdIdx) {
471 		pvscsi_cmd_ctx_t *ctx;
472 		struct PVSCSIRingCmpDesc *cdesc;
473 
474 		cdesc = CMP_RING(pvs) + (sdesc->cmpConsIdx & MASK(cmp_ne));
475 		membar_consumer();
476 
477 		ctx = pvscsi_resolve_ctx(pvs, cdesc->context);
478 		ASSERT(ctx != NULL);
479 
480 		if ((cmd = ctx->cmd) != NULL) {
481 			cmd->next_cmd = NULL;
482 
483 			/* Save command status for further processing */
484 			cmd->cmp_stat.host_status = cdesc->hostStatus;
485 			cmd->cmp_stat.scsi_status = cdesc->scsiStatus;
486 			cmd->cmp_stat.data_len = cdesc->dataLen;
487 
488 			/* Mark this command as arrived from hardware */
489 			cmd->flags |= PVSCSI_FLAG_HW_STATUS;
490 
491 			if (head == NULL) {
492 				head = cmd;
493 				head->tail_cmd = cmd;
494 			} else {
495 				head->tail_cmd = cmd;
496 			}
497 
498 			if (pnext_cmd == NULL) {
499 				pnext_cmd = &cmd->next_cmd;
500 			} else {
501 				*pnext_cmd = cmd;
502 				pnext_cmd = &cmd->next_cmd;
503 			}
504 		}
505 
506 		membar_consumer();
507 		sdesc->cmpConsIdx++;
508 	}
509 
510 	return (head);
511 }
512 
513 static pvscsi_msg_t *
pvscsi_process_msg_ring(pvscsi_softc_t * pvs)514 pvscsi_process_msg_ring(pvscsi_softc_t *pvs)
515 {
516 	pvscsi_msg_t	*msg;
517 	struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
518 	struct PVSCSIRingMsgDesc *mdesc;
519 	struct PVSCSIMsgDescDevStatusChanged *desc;
520 	uint32_t	msg_ne = sdesc->msgNumEntriesLog2;
521 
522 	ASSERT(mutex_owned(&pvs->rx_mutex));
523 
524 	if (sdesc->msgProdIdx == sdesc->msgConsIdx)
525 		return (NULL);
526 
527 	mdesc = MSG_RING(pvs) + (sdesc->msgConsIdx & MASK(msg_ne));
528 	membar_consumer();
529 
530 	switch (mdesc->type) {
531 	case PVSCSI_MSG_DEV_ADDED:
532 	case PVSCSI_MSG_DEV_REMOVED:
533 		desc = (struct PVSCSIMsgDescDevStatusChanged *)mdesc;
534 		msg = kmem_alloc(sizeof (pvscsi_msg_t), KM_NOSLEEP);
535 		if (msg == NULL)
536 			return (NULL);
537 		msg->msg_pvs = pvs;
538 		msg->type = mdesc->type;
539 		msg->target = desc->target;
540 		break;
541 	default:
542 		dev_err(pvs->dip, CE_WARN, "!unknown msg type: %d",
543 		    mdesc->type);
544 		return (NULL);
545 	}
546 
547 	membar_consumer();
548 	sdesc->msgConsIdx++;
549 
550 	return (msg);
551 }
552 
553 static void
pvscsi_handle_msg(void * arg)554 pvscsi_handle_msg(void *arg)
555 {
556 	pvscsi_msg_t	*msg = (pvscsi_msg_t *)arg;
557 	dev_info_t	*dip = msg->msg_pvs->dip;
558 	int		circ;
559 
560 	ndi_devi_enter(dip, &circ);
561 	(void) pvscsi_config_one(dip, msg->msg_pvs, msg->target, NULL);
562 	ndi_devi_exit(dip, circ);
563 
564 	kmem_free(msg, sizeof (pvscsi_msg_t));
565 }
566 
567 static int
pvscsi_abort_cmd(pvscsi_cmd_t * cmd,pvscsi_cmd_t ** pending)568 pvscsi_abort_cmd(pvscsi_cmd_t *cmd, pvscsi_cmd_t **pending)
569 {
570 	pvscsi_softc_t	*pvs = cmd->cmd_pvs;
571 	pvscsi_cmd_t	*c;
572 	pvscsi_cmd_t	*done;
573 	struct PVSCSICmdDescAbortCmd acmd;
574 
575 	dev_err(pvs->dip, CE_WARN, "!aborting command %p", (void *)cmd);
576 
577 	ASSERT(mutex_owned(&pvs->rx_mutex));
578 	ASSERT(mutex_owned(&pvs->tx_mutex));
579 
580 	/* Check if the cmd was already completed by the HBA */
581 	*pending = done = pvscsi_process_comp_ring(pvs);
582 	for (c = done; c != NULL; c = c->next_cmd) {
583 		if (c == cmd)
584 			return (CMD_CMPLT);
585 	}
586 
587 	/* Check if cmd was really scheduled by the HBA */
588 	if (pvscsi_lookup_ctx(pvs, cmd) == NULL)
589 		return (CMD_CMPLT);
590 
591 	/* Abort cmd in the HBA */
592 	bzero(&acmd, sizeof (acmd));
593 	acmd.target = cmd->cmd_target;
594 	acmd.context = pvscsi_map_ctx(pvs, cmd->ctx);
595 	pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_ABORT_CMD, &acmd, sizeof (acmd));
596 
597 	/* Check if cmd was completed by the HBA before it could be aborted */
598 	if ((done = pvscsi_process_comp_ring(pvs)) != NULL) {
599 		done->tail_cmd->next_cmd = *pending;
600 		*pending = done;
601 		for (c = done; c != NULL; c = c->next_cmd) {
602 			if (c == cmd)
603 				return (CMD_CMPLT);
604 		}
605 	}
606 
607 	/* Release I/O ctx */
608 	mutex_enter(&pvs->mutex);
609 	if (cmd->ctx != NULL)
610 		pvscsi_release_ctx(cmd);
611 	/* Remove cmd from the queue */
612 	pvscsi_remove_from_queue(cmd);
613 	mutex_exit(&pvs->mutex);
614 
615 	/* Insert cmd at the beginning of the list */
616 	cmd->next_cmd = *pending;
617 	*pending = cmd;
618 
619 	dev_err(pvs->dip, CE_WARN, "!command %p aborted", (void *)cmd);
620 
621 	return (CMD_ABORTED);
622 }
623 
624 static void
pvscsi_map_buffers(pvscsi_cmd_t * cmd,struct PVSCSIRingReqDesc * rdesc)625 pvscsi_map_buffers(pvscsi_cmd_t *cmd, struct PVSCSIRingReqDesc *rdesc)
626 {
627 	int	i;
628 
629 	ASSERT(cmd->ctx);
630 	ASSERT(cmd->cmd_dmaccount > 0 && cmd->cmd_dmaccount <=
631 	    PVSCSI_MAX_SG_SIZE);
632 
633 	rdesc->dataLen = cmd->cmd_dma_count;
634 	rdesc->dataAddr = 0;
635 
636 	if (cmd->cmd_dma_count == 0)
637 		return;
638 
639 	if (cmd->cmd_dmaccount > 1) {
640 		struct PVSCSISGElement *sgl = CMD_CTX_SGLIST_VA(cmd->ctx);
641 
642 		for (i = 0; i < cmd->cmd_dmaccount; i++) {
643 			sgl[i].addr = cmd->cached_cookies[i].dmac_laddress;
644 			sgl[i].length = cmd->cached_cookies[i].dmac_size;
645 			sgl[i].flags = 0;
646 		}
647 		rdesc->flags |= PVSCSI_FLAG_CMD_WITH_SG_LIST;
648 		rdesc->dataAddr = (uint64_t)CMD_CTX_SGLIST_PA(cmd->ctx);
649 	} else {
650 		rdesc->dataAddr = cmd->cached_cookies[0].dmac_laddress;
651 	}
652 }
653 
654 static void
pvscsi_comp_cmd(pvscsi_cmd_t * cmd,uint8_t status)655 pvscsi_comp_cmd(pvscsi_cmd_t *cmd, uint8_t status)
656 {
657 	struct scsi_pkt	*pkt = CMD2PKT(cmd);
658 
659 	pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD |
660 	    STATE_GOT_STATUS);
661 	if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0)
662 		pkt->pkt_state |= STATE_XFERRED_DATA;
663 	pkt->pkt_reason = CMD_CMPLT;
664 	pkt->pkt_resid = 0;
665 	*(pkt->pkt_scbp) = status;
666 }
667 
668 static void
pvscsi_set_status(pvscsi_cmd_t * cmd)669 pvscsi_set_status(pvscsi_cmd_t *cmd)
670 {
671 	pvscsi_softc_t	*pvs = cmd->cmd_pvs;
672 	struct scsi_pkt	*pkt = CMD2PKT(cmd);
673 	uchar_t		scsi_status = cmd->cmp_stat.scsi_status;
674 	uint32_t	host_status = cmd->cmp_stat.host_status;
675 
676 	if (scsi_status != STATUS_GOOD &&
677 	    (host_status == BTSTAT_SUCCESS ||
678 	    (host_status == BTSTAT_LINKED_COMMAND_COMPLETED) ||
679 	    (host_status == BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG))) {
680 		if (scsi_status == STATUS_CHECK) {
681 			struct scsi_arq_status *astat = (void*)(pkt->pkt_scbp);
682 			uint8_t		*sensedata;
683 			int		arq_size;
684 
685 			*pkt->pkt_scbp = scsi_status;
686 			pkt->pkt_state |= STATE_ARQ_DONE;
687 
688 			if ((cmd->flags & PVSCSI_FLAG_XARQ) != 0) {
689 				arq_size = (cmd->cmd_rqslen >=
690 				    SENSE_BUFFER_SIZE) ? SENSE_BUFFER_SIZE :
691 				    cmd->cmd_rqslen;
692 
693 				astat->sts_rqpkt_resid = SENSE_BUFFER_SIZE -
694 				    arq_size;
695 				sensedata = (uint8_t *)&astat->sts_sensedata;
696 				bcopy(cmd->arqbuf->b_un.b_addr, sensedata,
697 				    arq_size);
698 
699 				pkt->pkt_state |= STATE_XARQ_DONE;
700 			} else {
701 				astat->sts_rqpkt_resid = 0;
702 			}
703 
704 			astat->sts_rqpkt_statistics = 0;
705 			astat->sts_rqpkt_reason = CMD_CMPLT;
706 			(*(uint8_t *)&astat->sts_rqpkt_status) = STATUS_GOOD;
707 			astat->sts_rqpkt_state = STATE_GOT_BUS |
708 			    STATE_GOT_TARGET | STATE_SENT_CMD |
709 			    STATE_XFERRED_DATA | STATE_GOT_STATUS;
710 		}
711 		pvscsi_comp_cmd(cmd, scsi_status);
712 
713 		return;
714 	}
715 
716 	switch (host_status) {
717 	case BTSTAT_SUCCESS:
718 	case BTSTAT_LINKED_COMMAND_COMPLETED:
719 	case BTSTAT_LINKED_COMMAND_COMPLETED_WITH_FLAG:
720 		pvscsi_comp_cmd(cmd, STATUS_GOOD);
721 		break;
722 	case BTSTAT_DATARUN:
723 		pkt->pkt_reason = CMD_DATA_OVR;
724 		pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
725 		    STATE_SENT_CMD | STATE_GOT_STATUS |
726 		    STATE_XFERRED_DATA);
727 		pkt->pkt_resid = 0;
728 		break;
729 	case BTSTAT_DATA_UNDERRUN:
730 		pkt->pkt_reason = pkt->pkt_state |= (STATE_GOT_BUS |
731 		    STATE_GOT_TARGET | STATE_SENT_CMD | STATE_GOT_STATUS);
732 		pkt->pkt_resid = cmd->dma_count - cmd->cmp_stat.data_len;
733 		if (pkt->pkt_resid != cmd->dma_count)
734 			pkt->pkt_state |= STATE_XFERRED_DATA;
735 		break;
736 	case BTSTAT_SELTIMEO:
737 		pkt->pkt_reason = CMD_DEV_GONE;
738 		pkt->pkt_state |= STATE_GOT_BUS;
739 		break;
740 	case BTSTAT_TAGREJECT:
741 		pkt->pkt_reason = CMD_TAG_REJECT;
742 		pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
743 		    STATE_SENT_CMD | STATE_GOT_STATUS);
744 		break;
745 	case BTSTAT_BADMSG:
746 		pkt->pkt_reason = CMD_BADMSG;
747 		pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
748 		    STATE_SENT_CMD | STATE_GOT_STATUS);
749 		break;
750 	case BTSTAT_SENTRST:
751 	case BTSTAT_RECVRST:
752 	case BTSTAT_BUSRESET:
753 		pkt->pkt_reason = CMD_RESET;
754 		pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
755 		    STATE_SENT_CMD | STATE_GOT_STATUS);
756 		break;
757 	case BTSTAT_ABORTQUEUE:
758 		pkt->pkt_reason = CMD_ABORTED;
759 		pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
760 		    STATE_SENT_CMD | STATE_GOT_STATUS);
761 		break;
762 	case BTSTAT_HAHARDWARE:
763 	case BTSTAT_INVPHASE:
764 	case BTSTAT_HATIMEOUT:
765 	case BTSTAT_NORESPONSE:
766 	case BTSTAT_DISCONNECT:
767 	case BTSTAT_HASOFTWARE:
768 	case BTSTAT_BUSFREE:
769 	case BTSTAT_SENSFAILED:
770 		pkt->pkt_reason = CMD_TRAN_ERR;
771 		pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
772 		    STATE_SENT_CMD | STATE_GOT_STATUS);
773 		break;
774 	default:
775 		dev_err(pvs->dip, CE_WARN,
776 		    "!unknown host status code: %d", host_status);
777 		pkt->pkt_reason = CMD_TRAN_ERR;
778 		pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET |
779 		    STATE_SENT_CMD | STATE_GOT_STATUS);
780 		break;
781 	}
782 }
783 
784 static void
pvscsi_complete_chained(void * arg)785 pvscsi_complete_chained(void *arg)
786 {
787 	pvscsi_cmd_t	*cmd = (pvscsi_cmd_t *)arg;
788 	pvscsi_cmd_t	*c;
789 	struct scsi_pkt	*pkt;
790 
791 	while (cmd != NULL) {
792 		pvscsi_softc_t	*pvs = cmd->cmd_pvs;
793 
794 		c = cmd->next_cmd;
795 		cmd->next_cmd = NULL;
796 
797 		pkt = CMD2PKT(cmd);
798 		if (pkt == NULL)
799 			return;
800 
801 		if ((cmd->flags & PVSCSI_FLAG_IO_IOPB) != 0 &&
802 		    (cmd->flags & PVSCSI_FLAG_IO_READ) != 0) {
803 			(void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
804 			    DDI_DMA_SYNC_FORCPU);
805 		}
806 
807 		mutex_enter(&pvs->mutex);
808 		/* Release I/O ctx */
809 		if (cmd->ctx != NULL)
810 			pvscsi_release_ctx(cmd);
811 		/* Remove command from queue */
812 		pvscsi_remove_from_queue(cmd);
813 		mutex_exit(&pvs->mutex);
814 
815 		if ((cmd->flags & PVSCSI_FLAG_HW_STATUS) != 0) {
816 			pvscsi_set_status(cmd);
817 		} else {
818 			ASSERT((cmd->flags & PVSCSI_FLAGS_NON_HW_COMPLETION) !=
819 			    0);
820 
821 			if ((cmd->flags & PVSCSI_FLAG_TIMED_OUT) != 0) {
822 				cmd->pkt->pkt_reason = CMD_TIMEOUT;
823 				cmd->pkt->pkt_statistics |=
824 				    (STAT_TIMEOUT | STAT_ABORTED);
825 			} else if ((cmd->flags & PVSCSI_FLAG_ABORTED) != 0) {
826 				cmd->pkt->pkt_reason = CMD_ABORTED;
827 				cmd->pkt->pkt_statistics |=
828 				    (STAT_TIMEOUT | STAT_ABORTED);
829 			} else if ((cmd->flags & PVSCSI_FLAGS_RESET) != 0) {
830 				cmd->pkt->pkt_reason = CMD_RESET;
831 				if ((cmd->flags & PVSCSI_FLAG_RESET_BUS) != 0) {
832 					cmd->pkt->pkt_statistics |=
833 					    STAT_BUS_RESET;
834 				} else {
835 					cmd->pkt->pkt_statistics |=
836 					    STAT_DEV_RESET;
837 				}
838 			}
839 		}
840 
841 		cmd->flags |= PVSCSI_FLAG_DONE;
842 		cmd->flags &= ~PVSCSI_FLAG_TRANSPORT;
843 
844 		if ((pkt->pkt_flags & FLAG_NOINTR) == 0 &&
845 		    pkt->pkt_comp != NULL)
846 			(*pkt->pkt_comp)(pkt);
847 
848 		cmd = c;
849 	}
850 }
851 
852 static void
pvscsi_dev_reset(pvscsi_softc_t * pvs,int target)853 pvscsi_dev_reset(pvscsi_softc_t *pvs, int target)
854 {
855 	struct PVSCSICmdDescResetDevice cmd = { 0 };
856 
857 	cmd.target = target;
858 	pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_RESET_DEVICE, &cmd, sizeof (cmd));
859 }
860 
861 static int
pvscsi_poll_cmd(pvscsi_softc_t * pvs,pvscsi_cmd_t * cmd)862 pvscsi_poll_cmd(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
863 {
864 	boolean_t	seen_intr;
865 	int		cycles = (cmd->pkt->pkt_time * 1000000) / USECS_TO_WAIT;
866 	int		i;
867 	pvscsi_cmd_t	*dcmd;
868 	struct scsi_pkt *pkt = CMD2PKT(cmd);
869 
870 	/*
871 	 * Make sure we're not missing any commands completed
872 	 * concurrently before we have actually disabled interrupts.
873 	 */
874 	mutex_enter(&pvs->rx_mutex);
875 	dcmd = pvscsi_process_comp_ring(pvs);
876 	mutex_exit(&pvs->rx_mutex);
877 
878 	pvscsi_complete_chained(dcmd);
879 
880 	while ((cmd->flags & PVSCSI_FLAG_DONE) == 0) {
881 		seen_intr = B_FALSE;
882 
883 		/* Disable interrupts from H/W */
884 		pvscsi_mask_intr(pvs);
885 
886 		/* Wait for interrupt to arrive */
887 		for (i = 0; i < cycles; i++) {
888 			uint32_t status;
889 
890 			mutex_enter(&pvs->rx_mutex);
891 			mutex_enter(&pvs->intr_mutex);
892 			status = pvscsi_read_intr_status(pvs);
893 			if ((status & PVSCSI_INTR_ALL_SUPPORTED) != 0) {
894 				/* Check completion ring */
895 				mutex_exit(&pvs->intr_mutex);
896 				dcmd = pvscsi_process_comp_ring(pvs);
897 				mutex_exit(&pvs->rx_mutex);
898 				seen_intr = B_TRUE;
899 				break;
900 			} else {
901 				mutex_exit(&pvs->intr_mutex);
902 				mutex_exit(&pvs->rx_mutex);
903 				drv_usecwait(USECS_TO_WAIT);
904 			}
905 		}
906 
907 		/* Enable interrupts from H/W */
908 		pvscsi_unmask_intr(pvs);
909 
910 		if (!seen_intr) {
911 			/* No interrupts seen from device during the timeout */
912 			mutex_enter(&pvs->tx_mutex);
913 			mutex_enter(&pvs->rx_mutex);
914 			if ((cmd->flags & PVSCSI_FLAGS_COMPLETION) != 0) {
915 				/* Command was cancelled asynchronously */
916 				dcmd = NULL;
917 			} else if ((pvscsi_abort_cmd(cmd,
918 			    &dcmd)) == CMD_ABORTED) {
919 				/* Command was cancelled in hardware */
920 				pkt->pkt_state |= (STAT_TIMEOUT | STAT_ABORTED);
921 				pkt->pkt_statistics |= (STAT_TIMEOUT |
922 				    STAT_ABORTED);
923 				pkt->pkt_reason = CMD_TIMEOUT;
924 			}
925 			mutex_exit(&pvs->rx_mutex);
926 			mutex_exit(&pvs->tx_mutex);
927 
928 			/*
929 			 * Complete commands that might be on completion list.
930 			 * Target command can also be on the list in case it was
931 			 * completed before it could be actually cancelled.
932 			 */
933 			break;
934 		}
935 
936 		pvscsi_complete_chained(dcmd);
937 
938 		if (!seen_intr)
939 			break;
940 	}
941 
942 	return (TRAN_ACCEPT);
943 }
944 
945 static void
pvscsi_abort_all(struct scsi_address * ap,pvscsi_softc_t * pvs,pvscsi_cmd_t ** pending,int marker_flag)946 pvscsi_abort_all(struct scsi_address *ap, pvscsi_softc_t *pvs,
947     pvscsi_cmd_t **pending, int marker_flag)
948 {
949 	int		qlen = pvs->cmd_queue_len;
950 	pvscsi_cmd_t	*cmd, *pcmd, *phead = NULL;
951 
952 	ASSERT(mutex_owned(&pvs->rx_mutex));
953 	ASSERT(mutex_owned(&pvs->tx_mutex));
954 
955 	/*
956 	 * Try to abort all queued commands, merging commands waiting
957 	 * for completion into a single list to complete them at one
958 	 * time when mutex is released.
959 	 */
960 	while (qlen > 0) {
961 		mutex_enter(&pvs->mutex);
962 		cmd = list_remove_head(&pvs->cmd_queue);
963 		ASSERT(cmd != NULL);
964 
965 		qlen--;
966 
967 		if (ap == NULL || ap->a_target == cmd->cmd_target) {
968 			int c = --pvs->cmd_queue_len;
969 
970 			mutex_exit(&pvs->mutex);
971 
972 			if (pvscsi_abort_cmd(cmd, &pcmd) == CMD_ABORTED) {
973 				/*
974 				 * Assume command is completely cancelled now,
975 				 * so mark it as requested.
976 				 */
977 				cmd->flags |= marker_flag;
978 			}
979 
980 			qlen -= (c - pvs->cmd_queue_len);
981 
982 			/*
983 			 * Now merge current pending commands with
984 			 * previous ones.
985 			 */
986 			if (phead == NULL) {
987 				phead = pcmd;
988 			} else if (pcmd != NULL) {
989 				phead->tail_cmd->next_cmd = pcmd;
990 				phead->tail_cmd = pcmd->tail_cmd;
991 			}
992 		} else {
993 			list_insert_tail(&pvs->cmd_queue, cmd);
994 			mutex_exit(&pvs->mutex);
995 		}
996 	}
997 
998 	*pending = phead;
999 }
1000 
1001 static void
pvscsi_quiesce_notify(pvscsi_softc_t * pvs)1002 pvscsi_quiesce_notify(pvscsi_softc_t *pvs)
1003 {
1004 	mutex_enter(&pvs->mutex);
1005 	if (pvs->cmd_queue_len == 0 &&
1006 	    (pvs->flags & PVSCSI_HBA_QUIESCE_PENDING) != 0) {
1007 		pvs->flags &= ~PVSCSI_HBA_QUIESCE_PENDING;
1008 		cv_broadcast(&pvs->quiescevar);
1009 	}
1010 	mutex_exit(&pvs->mutex);
1011 }
1012 
1013 static int
pvscsi_transport_command(pvscsi_softc_t * pvs,pvscsi_cmd_t * cmd)1014 pvscsi_transport_command(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd)
1015 {
1016 	struct PVSCSIRingReqDesc *rdesc;
1017 	struct PVSCSIRingsState *sdesc = RINGS_STATE(pvs);
1018 	struct scsi_pkt *pkt = CMD2PKT(cmd);
1019 	uint32_t	req_ne = sdesc->reqNumEntriesLog2;
1020 
1021 	mutex_enter(&pvs->tx_mutex);
1022 	mutex_enter(&pvs->mutex);
1023 	if (!pvscsi_acquire_ctx(pvs, cmd)) {
1024 		mutex_exit(&pvs->mutex);
1025 		mutex_exit(&pvs->tx_mutex);
1026 		dev_err(pvs->dip, CE_WARN, "!no free ctx available");
1027 		return (TRAN_BUSY);
1028 	}
1029 
1030 	if ((sdesc->reqProdIdx - sdesc->cmpConsIdx) >= (1 << req_ne)) {
1031 		pvscsi_release_ctx(cmd);
1032 		mutex_exit(&pvs->mutex);
1033 		mutex_exit(&pvs->tx_mutex);
1034 		dev_err(pvs->dip, CE_WARN, "!no free I/O slots available");
1035 		return (TRAN_BUSY);
1036 	}
1037 	mutex_exit(&pvs->mutex);
1038 
1039 	cmd->flags |= PVSCSI_FLAG_TRANSPORT;
1040 
1041 	rdesc = REQ_RING(pvs) + (sdesc->reqProdIdx & MASK(req_ne));
1042 
1043 	bzero(&rdesc->lun, sizeof (rdesc->lun));
1044 
1045 	rdesc->bus = 0;
1046 	rdesc->target = cmd->cmd_target;
1047 
1048 	if ((cmd->flags & PVSCSI_FLAG_XARQ) != 0) {
1049 		bzero((void*)cmd->arqbuf->b_un.b_addr, SENSE_BUFFER_SIZE);
1050 		rdesc->senseLen = SENSE_BUFFER_SIZE;
1051 		rdesc->senseAddr = cmd->arqc.dmac_laddress;
1052 	} else {
1053 		rdesc->senseLen = 0;
1054 		rdesc->senseAddr = 0;
1055 	}
1056 
1057 	rdesc->vcpuHint = CPU->cpu_id;
1058 	rdesc->cdbLen = cmd->cmdlen;
1059 	bcopy(cmd->cmd_cdb, rdesc->cdb, cmd->cmdlen);
1060 
1061 	/* Setup tag info */
1062 	if ((cmd->flags & PVSCSI_FLAG_TAG) != 0)
1063 		rdesc->tag = cmd->tag;
1064 	else
1065 		rdesc->tag = MSG_SIMPLE_QTAG;
1066 
1067 	/* Setup I/O direction and map data buffers */
1068 	if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1069 		if ((cmd->flags & PVSCSI_FLAG_IO_READ) != 0)
1070 			rdesc->flags = PVSCSI_FLAG_CMD_DIR_TOHOST;
1071 		else
1072 			rdesc->flags = PVSCSI_FLAG_CMD_DIR_TODEVICE;
1073 		pvscsi_map_buffers(cmd, rdesc);
1074 	} else {
1075 		rdesc->flags = 0;
1076 	}
1077 
1078 	rdesc->context = pvscsi_map_ctx(pvs, cmd->ctx);
1079 	membar_producer();
1080 
1081 	sdesc->reqProdIdx++;
1082 	membar_producer();
1083 
1084 	mutex_enter(&pvs->mutex);
1085 	cmd->timeout_lbolt = ddi_get_lbolt() + SEC_TO_TICK(pkt->pkt_time);
1086 	pkt->pkt_state |= (STATE_GOT_BUS | STATE_GOT_TARGET | STATE_SENT_CMD);
1087 	pvscsi_add_to_queue(cmd);
1088 
1089 	switch (cmd->pkt->pkt_cdbp[0]) {
1090 	case SCMD_READ:
1091 	case SCMD_WRITE:
1092 	case SCMD_READ_G1:
1093 	case SCMD_WRITE_G1:
1094 	case SCMD_READ_G4:
1095 	case SCMD_WRITE_G4:
1096 	case SCMD_READ_G5:
1097 	case SCMD_WRITE_G5:
1098 		ASSERT((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0);
1099 		pvscsi_submit_rw_io(pvs);
1100 		break;
1101 	default:
1102 		pvscsi_submit_nonrw_io(pvs);
1103 		break;
1104 	}
1105 	mutex_exit(&pvs->mutex);
1106 	mutex_exit(&pvs->tx_mutex);
1107 
1108 	return (TRAN_ACCEPT);
1109 }
1110 
1111 static int
pvscsi_reset_generic(pvscsi_softc_t * pvs,struct scsi_address * ap)1112 pvscsi_reset_generic(pvscsi_softc_t *pvs, struct scsi_address *ap)
1113 {
1114 	boolean_t	bus_reset = (ap == NULL);
1115 	int		flags;
1116 	pvscsi_cmd_t	*done, *aborted;
1117 
1118 	flags = bus_reset ? PVSCSI_FLAG_RESET_BUS : PVSCSI_FLAG_RESET_DEV;
1119 
1120 	mutex_enter(&pvs->tx_mutex);
1121 	mutex_enter(&pvs->rx_mutex);
1122 	/* Try to process pending requests */
1123 	done = pvscsi_process_comp_ring(pvs);
1124 
1125 	/* Abort all pending requests */
1126 	pvscsi_abort_all(ap, pvs, &aborted, flags);
1127 
1128 	/* Reset at hardware level */
1129 	if (bus_reset) {
1130 		pvscsi_reset_bus(pvs);
1131 		/* Should never happen after bus reset */
1132 		ASSERT(pvscsi_process_comp_ring(pvs) == NULL);
1133 	} else {
1134 		pvscsi_dev_reset(pvs, ap->a_target);
1135 	}
1136 	mutex_exit(&pvs->rx_mutex);
1137 	mutex_exit(&pvs->tx_mutex);
1138 
1139 	pvscsi_complete_chained(done);
1140 	pvscsi_complete_chained(aborted);
1141 
1142 	return (1);
1143 }
1144 
1145 static void
pvscsi_cmd_ext_free(pvscsi_cmd_t * cmd)1146 pvscsi_cmd_ext_free(pvscsi_cmd_t *cmd)
1147 {
1148 	struct scsi_pkt	*pkt = CMD2PKT(cmd);
1149 
1150 	if ((cmd->flags & PVSCSI_FLAG_CDB_EXT) != 0) {
1151 		kmem_free(pkt->pkt_cdbp, cmd->cmdlen);
1152 		cmd->flags &= ~PVSCSI_FLAG_CDB_EXT;
1153 	}
1154 	if ((cmd->flags & PVSCSI_FLAG_SCB_EXT) != 0) {
1155 		kmem_free(pkt->pkt_scbp, cmd->statuslen);
1156 		cmd->flags &= ~PVSCSI_FLAG_SCB_EXT;
1157 	}
1158 	if ((cmd->flags & PVSCSI_FLAG_PRIV_EXT) != 0) {
1159 		kmem_free(pkt->pkt_private, cmd->tgtlen);
1160 		cmd->flags &= ~PVSCSI_FLAG_PRIV_EXT;
1161 	}
1162 }
1163 
1164 /* ARGSUSED pvs */
1165 static int
pvscsi_cmd_ext_alloc(pvscsi_softc_t * pvs,pvscsi_cmd_t * cmd,int kf)1166 pvscsi_cmd_ext_alloc(pvscsi_softc_t *pvs, pvscsi_cmd_t *cmd, int kf)
1167 {
1168 	struct scsi_pkt *pkt = CMD2PKT(cmd);
1169 	void		*buf;
1170 
1171 	if (cmd->cmdlen > sizeof (cmd->cmd_cdb)) {
1172 		if ((buf = kmem_zalloc(cmd->cmdlen, kf)) == NULL)
1173 			return (DDI_FAILURE);
1174 		pkt->pkt_cdbp = buf;
1175 		cmd->flags |= PVSCSI_FLAG_CDB_EXT;
1176 	}
1177 
1178 	if (cmd->statuslen > sizeof (cmd->cmd_scb)) {
1179 		if ((buf = kmem_zalloc(cmd->statuslen, kf)) == NULL)
1180 			goto out;
1181 		pkt->pkt_scbp = buf;
1182 		cmd->flags |= PVSCSI_FLAG_SCB_EXT;
1183 		cmd->cmd_rqslen = (cmd->statuslen - sizeof (cmd->cmd_scb));
1184 	}
1185 
1186 	if (cmd->tgtlen > sizeof (cmd->tgt_priv)) {
1187 		if ((buf = kmem_zalloc(cmd->tgtlen, kf)) == NULL)
1188 			goto out;
1189 		pkt->pkt_private = buf;
1190 		cmd->flags |= PVSCSI_FLAG_PRIV_EXT;
1191 	}
1192 
1193 	return (DDI_SUCCESS);
1194 
1195 out:
1196 	pvscsi_cmd_ext_free(cmd);
1197 
1198 	return (DDI_FAILURE);
1199 }
1200 
1201 static int
pvscsi_setup_dma_buffer(pvscsi_softc_t * pvs,size_t length,pvscsi_dma_buf_t * buf)1202 pvscsi_setup_dma_buffer(pvscsi_softc_t *pvs, size_t length,
1203     pvscsi_dma_buf_t *buf)
1204 {
1205 	ddi_dma_cookie_t cookie;
1206 	uint_t		ccount;
1207 
1208 	if ((ddi_dma_alloc_handle(pvs->dip, &pvscsi_ring_dma_attr,
1209 	    DDI_DMA_SLEEP, NULL, &buf->dma_handle)) != DDI_SUCCESS) {
1210 		dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA handle");
1211 		return (DDI_FAILURE);
1212 	}
1213 
1214 	if ((ddi_dma_mem_alloc(buf->dma_handle, length, &pvscsi_dma_attrs,
1215 	    DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &buf->addr,
1216 	    &buf->real_length, &buf->acc_handle)) != DDI_SUCCESS) {
1217 		dev_err(pvs->dip, CE_WARN,
1218 		    "!failed to allocate %ld bytes for DMA buffer", length);
1219 		ddi_dma_free_handle(&buf->dma_handle);
1220 		return (DDI_FAILURE);
1221 	}
1222 
1223 	if ((ddi_dma_addr_bind_handle(buf->dma_handle, NULL, buf->addr,
1224 	    buf->real_length, DDI_DMA_CONSISTENT | DDI_DMA_RDWR, DDI_DMA_SLEEP,
1225 	    NULL, &cookie, &ccount)) != DDI_SUCCESS) {
1226 		dev_err(pvs->dip, CE_WARN, "!failed to bind DMA buffer");
1227 		ddi_dma_free_handle(&buf->dma_handle);
1228 		ddi_dma_mem_free(&buf->acc_handle);
1229 		return (DDI_FAILURE);
1230 	}
1231 
1232 	/* TODO Support multipart SG regions */
1233 	ASSERT(ccount == 1);
1234 
1235 	buf->pa = cookie.dmac_laddress;
1236 
1237 	return (DDI_SUCCESS);
1238 }
1239 
1240 static void
pvscsi_free_dma_buffer(pvscsi_dma_buf_t * buf)1241 pvscsi_free_dma_buffer(pvscsi_dma_buf_t *buf)
1242 {
1243 	ddi_dma_free_handle(&buf->dma_handle);
1244 	ddi_dma_mem_free(&buf->acc_handle);
1245 }
1246 
1247 static int
pvscsi_setup_sg(pvscsi_softc_t * pvs)1248 pvscsi_setup_sg(pvscsi_softc_t *pvs)
1249 {
1250 	int		i;
1251 	pvscsi_cmd_ctx_t *ctx;
1252 	size_t		size = pvs->req_depth * sizeof (pvscsi_cmd_ctx_t);
1253 
1254 	ctx = pvs->cmd_ctx = kmem_zalloc(size, KM_SLEEP);
1255 
1256 	for (i = 0; i < pvs->req_depth; ++i, ++ctx) {
1257 		list_insert_tail(&pvs->cmd_ctx_pool, ctx);
1258 		if (pvscsi_setup_dma_buffer(pvs, PAGE_SIZE,
1259 		    &ctx->dma_buf) != DDI_SUCCESS)
1260 			goto cleanup;
1261 	}
1262 
1263 	return (DDI_SUCCESS);
1264 
1265 cleanup:
1266 	for (; i >= 0; --i, --ctx) {
1267 		list_remove(&pvs->cmd_ctx_pool, ctx);
1268 		pvscsi_free_dma_buffer(&ctx->dma_buf);
1269 	}
1270 	kmem_free(pvs->cmd_ctx, size);
1271 
1272 	return (DDI_FAILURE);
1273 }
1274 
1275 static void
pvscsi_free_sg(pvscsi_softc_t * pvs)1276 pvscsi_free_sg(pvscsi_softc_t *pvs)
1277 {
1278 	int		i;
1279 	pvscsi_cmd_ctx_t *ctx = pvs->cmd_ctx;
1280 
1281 	for (i = 0; i < pvs->req_depth; ++i, ++ctx) {
1282 		list_remove(&pvs->cmd_ctx_pool, ctx);
1283 		pvscsi_free_dma_buffer(&ctx->dma_buf);
1284 	}
1285 
1286 	kmem_free(pvs->cmd_ctx, pvs->req_pages << PAGE_SHIFT);
1287 }
1288 
1289 static int
pvscsi_allocate_rings(pvscsi_softc_t * pvs)1290 pvscsi_allocate_rings(pvscsi_softc_t *pvs)
1291 {
1292 	/* Allocate DMA buffer for rings state */
1293 	if (pvscsi_setup_dma_buffer(pvs, PAGE_SIZE,
1294 	    &pvs->rings_state_buf) != DDI_SUCCESS)
1295 		return (DDI_FAILURE);
1296 
1297 	/* Allocate DMA buffer for request ring */
1298 	pvs->req_pages = MIN(pvscsi_ring_pages, PVSCSI_MAX_NUM_PAGES_REQ_RING);
1299 	pvs->req_depth = pvs->req_pages * PVSCSI_MAX_NUM_REQ_ENTRIES_PER_PAGE;
1300 	if (pvscsi_setup_dma_buffer(pvs, pvs->req_pages * PAGE_SIZE,
1301 	    &pvs->req_ring_buf) != DDI_SUCCESS)
1302 		goto free_rings_state;
1303 
1304 	/* Allocate completion ring */
1305 	pvs->cmp_pages = MIN(pvscsi_ring_pages, PVSCSI_MAX_NUM_PAGES_CMP_RING);
1306 	if (pvscsi_setup_dma_buffer(pvs, pvs->cmp_pages * PAGE_SIZE,
1307 	    &pvs->cmp_ring_buf) != DDI_SUCCESS)
1308 		goto free_req_buf;
1309 
1310 	/* Allocate message ring */
1311 	pvs->msg_pages = MIN(pvscsi_msg_ring_pages,
1312 	    PVSCSI_MAX_NUM_PAGES_MSG_RING);
1313 	if (pvscsi_setup_dma_buffer(pvs, pvs->msg_pages * PAGE_SIZE,
1314 	    &pvs->msg_ring_buf) != DDI_SUCCESS)
1315 		goto free_cmp_buf;
1316 
1317 	return (DDI_SUCCESS);
1318 
1319 free_cmp_buf:
1320 	pvscsi_free_dma_buffer(&pvs->cmp_ring_buf);
1321 free_req_buf:
1322 	pvscsi_free_dma_buffer(&pvs->req_ring_buf);
1323 free_rings_state:
1324 	pvscsi_free_dma_buffer(&pvs->rings_state_buf);
1325 
1326 	return (DDI_FAILURE);
1327 }
1328 
1329 static void
pvscsi_free_rings(pvscsi_softc_t * pvs)1330 pvscsi_free_rings(pvscsi_softc_t *pvs)
1331 {
1332 	pvscsi_free_dma_buffer(&pvs->msg_ring_buf);
1333 	pvscsi_free_dma_buffer(&pvs->cmp_ring_buf);
1334 	pvscsi_free_dma_buffer(&pvs->req_ring_buf);
1335 	pvscsi_free_dma_buffer(&pvs->rings_state_buf);
1336 }
1337 
1338 static void
pvscsi_setup_rings(pvscsi_softc_t * pvs)1339 pvscsi_setup_rings(pvscsi_softc_t *pvs)
1340 {
1341 	int		i;
1342 	struct PVSCSICmdDescSetupMsgRing cmd_msg = { 0 };
1343 	struct PVSCSICmdDescSetupRings cmd = { 0 };
1344 	uint64_t	base;
1345 
1346 	cmd.ringsStatePPN = pvs->rings_state_buf.pa >> PAGE_SHIFT;
1347 	cmd.reqRingNumPages = pvs->req_pages;
1348 	cmd.cmpRingNumPages = pvs->cmp_pages;
1349 
1350 	/* Setup request ring */
1351 	base = pvs->req_ring_buf.pa;
1352 	for (i = 0; i < pvs->req_pages; i++) {
1353 		cmd.reqRingPPNs[i] = base >> PAGE_SHIFT;
1354 		base += PAGE_SIZE;
1355 	}
1356 
1357 	/* Setup completion ring */
1358 	base = pvs->cmp_ring_buf.pa;
1359 	for (i = 0; i < pvs->cmp_pages; i++) {
1360 		cmd.cmpRingPPNs[i] = base >> PAGE_SHIFT;
1361 		base += PAGE_SIZE;
1362 	}
1363 
1364 	bzero(RINGS_STATE(pvs), PAGE_SIZE);
1365 	bzero(REQ_RING(pvs), pvs->req_pages * PAGE_SIZE);
1366 	bzero(CMP_RING(pvs), pvs->cmp_pages * PAGE_SIZE);
1367 
1368 	/* Issue SETUP command */
1369 	pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_SETUP_RINGS, &cmd, sizeof (cmd));
1370 
1371 	/* Setup message ring */
1372 	cmd_msg.numPages = pvs->msg_pages;
1373 	base = pvs->msg_ring_buf.pa;
1374 
1375 	for (i = 0; i < pvs->msg_pages; i++) {
1376 		cmd_msg.ringPPNs[i] = base >> PAGE_SHIFT;
1377 		base += PAGE_SIZE;
1378 	}
1379 	bzero(MSG_RING(pvs), pvs->msg_pages * PAGE_SIZE);
1380 
1381 	pvscsi_write_cmd_desc(pvs, PVSCSI_CMD_SETUP_MSG_RING, &cmd_msg,
1382 	    sizeof (cmd_msg));
1383 }
1384 
1385 static int
pvscsi_setup_io(pvscsi_softc_t * pvs)1386 pvscsi_setup_io(pvscsi_softc_t *pvs)
1387 {
1388 	int		offset, rcount, rn, type;
1389 	int		ret = DDI_FAILURE;
1390 	off_t		regsize;
1391 	pci_regspec_t	*regs;
1392 	uint_t		regs_length;
1393 
1394 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, pvs->dip,
1395 	    DDI_PROP_DONTPASS, "reg", (int **)&regs,
1396 	    &regs_length) != DDI_PROP_SUCCESS) {
1397 		dev_err(pvs->dip, CE_WARN, "!failed to lookup 'reg' property");
1398 		return (DDI_FAILURE);
1399 	}
1400 
1401 	rcount = regs_length * sizeof (int) / sizeof (pci_regspec_t);
1402 
1403 	for (offset = PCI_CONF_BASE0; offset <= PCI_CONF_BASE5; offset += 4) {
1404 		for (rn = 0; rn < rcount; ++rn) {
1405 			if (PCI_REG_REG_G(regs[rn].pci_phys_hi) == offset) {
1406 				type = regs[rn].pci_phys_hi & PCI_ADDR_MASK;
1407 				break;
1408 			}
1409 		}
1410 
1411 		if (rn >= rcount)
1412 			continue;
1413 
1414 		if (type != PCI_ADDR_IO) {
1415 			if (ddi_dev_regsize(pvs->dip, rn,
1416 			    &regsize) != DDI_SUCCESS) {
1417 				dev_err(pvs->dip, CE_WARN,
1418 				    "!failed to get size of reg %d", rn);
1419 				goto out;
1420 			}
1421 			if (regsize == PVSCSI_MEM_SPACE_SIZE) {
1422 				if (ddi_regs_map_setup(pvs->dip, rn,
1423 				    &pvs->mmio_base, 0, 0,
1424 				    &pvscsi_mmio_attr,
1425 				    &pvs->mmio_handle) != DDI_SUCCESS) {
1426 					dev_err(pvs->dip, CE_WARN,
1427 					    "!failed to map MMIO BAR");
1428 					goto out;
1429 				}
1430 				ret = DDI_SUCCESS;
1431 				break;
1432 			}
1433 		}
1434 	}
1435 
1436 out:
1437 	ddi_prop_free(regs);
1438 
1439 	return (ret);
1440 }
1441 
1442 static void
pvscsi_free_io(pvscsi_softc_t * pvs)1443 pvscsi_free_io(pvscsi_softc_t *pvs)
1444 {
1445 	ddi_regs_map_free(&pvs->mmio_handle);
1446 }
1447 
1448 static int
pvscsi_enable_intrs(pvscsi_softc_t * pvs)1449 pvscsi_enable_intrs(pvscsi_softc_t *pvs)
1450 {
1451 	int	i, rc, intr_caps;
1452 
1453 	if ((rc = ddi_intr_get_cap(pvs->intr_htable[0], &intr_caps)) !=
1454 	    DDI_SUCCESS) {
1455 		dev_err(pvs->dip, CE_WARN, "!failed to get interrupt caps");
1456 		return (DDI_FAILURE);
1457 	}
1458 
1459 	if ((intr_caps & DDI_INTR_FLAG_BLOCK) != 0) {
1460 		if ((rc = ddi_intr_block_enable(pvs->intr_htable,
1461 		    pvs->intr_cnt)) != DDI_SUCCESS) {
1462 			dev_err(pvs->dip, CE_WARN,
1463 			    "!failed to enable interrupt block");
1464 		}
1465 	} else {
1466 		for (i = 0; i < pvs->intr_cnt; i++) {
1467 			if ((rc = ddi_intr_enable(pvs->intr_htable[i])) ==
1468 			    DDI_SUCCESS)
1469 				continue;
1470 			dev_err(pvs->dip, CE_WARN,
1471 			    "!failed to enable interrupt");
1472 			while (--i >= 0)
1473 				(void) ddi_intr_disable(pvs->intr_htable[i]);
1474 			break;
1475 		}
1476 	}
1477 
1478 	/* Unmask interrupts */
1479 	if (rc == DDI_SUCCESS) {
1480 		pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK,
1481 		    PVSCSI_INTR_CMPL_MASK | PVSCSI_INTR_MSG_MASK);
1482 	}
1483 
1484 	return (rc);
1485 }
1486 
1487 /* ARGSUSED arg2 */
1488 static uint32_t
pvscsi_intr_handler(caddr_t arg1,caddr_t arg2)1489 pvscsi_intr_handler(caddr_t arg1, caddr_t arg2)
1490 {
1491 	boolean_t	handled;
1492 	pvscsi_softc_t	*pvs = (pvscsi_softc_t *)arg1;
1493 	uint32_t	status;
1494 
1495 	mutex_enter(&pvs->intr_mutex);
1496 	if (pvs->num_pollers > 0) {
1497 		mutex_exit(&pvs->intr_mutex);
1498 		return (DDI_INTR_CLAIMED);
1499 	}
1500 
1501 	if (pvscsi_enable_msi) {
1502 		handled = B_TRUE;
1503 	} else {
1504 		status = pvscsi_read_intr_status(pvs);
1505 		handled = (status & PVSCSI_INTR_ALL_SUPPORTED) != 0;
1506 		if (handled)
1507 			pvscsi_write_intr_status(pvs, status);
1508 	}
1509 	mutex_exit(&pvs->intr_mutex);
1510 
1511 	if (handled) {
1512 		boolean_t	qnotify;
1513 		pvscsi_cmd_t	*pending;
1514 		pvscsi_msg_t	*msg;
1515 
1516 		mutex_enter(&pvs->rx_mutex);
1517 		pending = pvscsi_process_comp_ring(pvs);
1518 		msg = pvscsi_process_msg_ring(pvs);
1519 		mutex_exit(&pvs->rx_mutex);
1520 
1521 		mutex_enter(&pvs->mutex);
1522 		qnotify = HBA_QUIESCE_PENDING(pvs);
1523 		mutex_exit(&pvs->mutex);
1524 
1525 		if (pending != NULL && ddi_taskq_dispatch(pvs->comp_tq,
1526 		    pvscsi_complete_chained, pending,
1527 		    DDI_NOSLEEP) == DDI_FAILURE)
1528 			pvscsi_complete_chained(pending);
1529 
1530 		if (msg != NULL && ddi_taskq_dispatch(pvs->msg_tq,
1531 		    pvscsi_handle_msg, msg, DDI_NOSLEEP) == DDI_FAILURE) {
1532 			dev_err(pvs->dip, CE_WARN,
1533 			    "!failed to process msg type %d for target %d",
1534 			    msg->type, msg->target);
1535 			kmem_free(msg, sizeof (pvscsi_msg_t));
1536 		}
1537 
1538 		if (qnotify)
1539 			pvscsi_quiesce_notify(pvs);
1540 	}
1541 
1542 	return (handled ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED);
1543 }
1544 
1545 static int
pvscsi_register_isr(pvscsi_softc_t * pvs,int type)1546 pvscsi_register_isr(pvscsi_softc_t *pvs, int type)
1547 {
1548 	int	navail, nactual;
1549 	int	i;
1550 
1551 	if (ddi_intr_get_navail(pvs->dip, type, &navail) != DDI_SUCCESS ||
1552 	    navail == 0) {
1553 		dev_err(pvs->dip, CE_WARN,
1554 		    "!failed to get number of available interrupts of type %d",
1555 		    type);
1556 		return (DDI_FAILURE);
1557 	}
1558 	navail = MIN(navail, PVSCSI_MAX_INTRS);
1559 
1560 	pvs->intr_size = navail * sizeof (ddi_intr_handle_t);
1561 	if ((pvs->intr_htable = kmem_alloc(pvs->intr_size, KM_SLEEP)) == NULL) {
1562 		dev_err(pvs->dip, CE_WARN,
1563 		    "!failed to allocate %d bytes for interrupt hashtable",
1564 		    pvs->intr_size);
1565 		return (DDI_FAILURE);
1566 	}
1567 
1568 	if (ddi_intr_alloc(pvs->dip, pvs->intr_htable, type, 0, navail,
1569 	    &nactual, DDI_INTR_ALLOC_NORMAL) != DDI_SUCCESS || nactual == 0) {
1570 		dev_err(pvs->dip, CE_WARN, "!failed to allocate %d interrupts",
1571 		    navail);
1572 		goto free_htable;
1573 	}
1574 
1575 	pvs->intr_cnt = nactual;
1576 
1577 	if (ddi_intr_get_pri(pvs->intr_htable[0],
1578 	    (uint_t *)&pvs->intr_pri) != DDI_SUCCESS) {
1579 		dev_err(pvs->dip, CE_WARN, "!failed to get interrupt priority");
1580 		goto free_intrs;
1581 	}
1582 
1583 	for (i = 0; i < nactual; i++) {
1584 		if (ddi_intr_add_handler(pvs->intr_htable[i],
1585 		    pvscsi_intr_handler, (caddr_t)pvs, NULL) != DDI_SUCCESS) {
1586 			dev_err(pvs->dip, CE_WARN,
1587 			    "!failed to add interrupt handler");
1588 			goto free_intrs;
1589 		}
1590 	}
1591 
1592 	return (DDI_SUCCESS);
1593 
1594 free_intrs:
1595 	for (i = 0; i < nactual; i++)
1596 		(void) ddi_intr_free(pvs->intr_htable[i]);
1597 free_htable:
1598 	kmem_free(pvs->intr_htable, pvs->intr_size);
1599 
1600 	return (DDI_FAILURE);
1601 }
1602 
1603 static void
pvscsi_free_intr_resources(pvscsi_softc_t * pvs)1604 pvscsi_free_intr_resources(pvscsi_softc_t *pvs)
1605 {
1606 	int	i;
1607 
1608 	for (i = 0; i < pvs->intr_cnt; i++) {
1609 		(void) ddi_intr_disable(pvs->intr_htable[i]);
1610 		(void) ddi_intr_remove_handler(pvs->intr_htable[i]);
1611 		(void) ddi_intr_free(pvs->intr_htable[i]);
1612 	}
1613 	kmem_free(pvs->intr_htable, pvs->intr_size);
1614 }
1615 
1616 static int
pvscsi_setup_isr(pvscsi_softc_t * pvs)1617 pvscsi_setup_isr(pvscsi_softc_t *pvs)
1618 {
1619 	int	intr_types;
1620 
1621 	if (ddi_intr_get_supported_types(pvs->dip,
1622 	    &intr_types) != DDI_SUCCESS) {
1623 		dev_err(pvs->dip, CE_WARN,
1624 		    "!failed to get supported interrupt types");
1625 		return (DDI_FAILURE);
1626 	}
1627 
1628 	if ((intr_types & DDI_INTR_TYPE_MSIX) != 0 && pvscsi_enable_msi) {
1629 		if (pvscsi_register_isr(pvs,
1630 		    DDI_INTR_TYPE_MSIX) == DDI_SUCCESS) {
1631 			pvs->intr_type = DDI_INTR_TYPE_MSIX;
1632 		} else {
1633 			dev_err(pvs->dip, CE_WARN,
1634 			    "!failed to install MSI-X interrupt handler");
1635 		}
1636 	} else if ((intr_types & DDI_INTR_TYPE_MSI) != 0 && pvscsi_enable_msi) {
1637 		if (pvscsi_register_isr(pvs,
1638 		    DDI_INTR_TYPE_MSI) == DDI_SUCCESS) {
1639 			pvs->intr_type = DDI_INTR_TYPE_MSI;
1640 		} else {
1641 			dev_err(pvs->dip, CE_WARN,
1642 			    "!failed to install MSI interrupt handler");
1643 		}
1644 	} else if ((intr_types & DDI_INTR_TYPE_FIXED) != 0) {
1645 		if (pvscsi_register_isr(pvs,
1646 		    DDI_INTR_TYPE_FIXED) == DDI_SUCCESS) {
1647 			pvs->intr_type = DDI_INTR_TYPE_FIXED;
1648 		} else {
1649 			dev_err(pvs->dip, CE_WARN,
1650 			    "!failed to install FIXED interrupt handler");
1651 		}
1652 	}
1653 
1654 	return (pvs->intr_type == 0 ? DDI_FAILURE : DDI_SUCCESS);
1655 }
1656 
1657 static void
pvscsi_wd_thread(pvscsi_softc_t * pvs)1658 pvscsi_wd_thread(pvscsi_softc_t *pvs)
1659 {
1660 	clock_t		now;
1661 	pvscsi_cmd_t	*expired, *c, *cn, **pnext;
1662 
1663 	mutex_enter(&pvs->mutex);
1664 	for (;;) {
1665 		expired = NULL;
1666 		pnext = NULL;
1667 		now = ddi_get_lbolt();
1668 
1669 		for (c = list_head(&pvs->cmd_queue); c != NULL; ) {
1670 			cn = list_next(&pvs->cmd_queue, c);
1671 
1672 			/*
1673 			 * Commands with 'FLAG_NOINTR' are watched using their
1674 			 * own timeouts, so we should not touch them.
1675 			 */
1676 			if ((c->pkt->pkt_flags & FLAG_NOINTR) == 0 &&
1677 			    now > c->timeout_lbolt) {
1678 				dev_err(pvs->dip, CE_WARN,
1679 				    "!expired command: %p (%ld > %ld)",
1680 				    (void *)c, now, c->timeout_lbolt);
1681 				pvscsi_remove_from_queue(c);
1682 				if (expired == NULL)
1683 					expired = c;
1684 				if (pnext == NULL) {
1685 					pnext = &c->next_cmd;
1686 				} else {
1687 					*pnext = c;
1688 					pnext = &c->next_cmd;
1689 				}
1690 			}
1691 			c = cn;
1692 		}
1693 		mutex_exit(&pvs->mutex);
1694 
1695 		/* Now cancel all expired commands */
1696 		if (expired != NULL) {
1697 			struct scsi_address sa = {0};
1698 			/* Build a fake SCSI address */
1699 			sa.a_hba_tran = pvs->tran;
1700 			while (expired != NULL) {
1701 				c = expired->next_cmd;
1702 				sa.a_target = expired->cmd_target;
1703 				sa.a_lun = 0;
1704 				(void) pvscsi_abort(&sa, CMD2PKT(expired));
1705 				expired = c;
1706 			}
1707 		}
1708 
1709 		mutex_enter(&pvs->mutex);
1710 		if ((pvs->flags & PVSCSI_DRIVER_SHUTDOWN) != 0) {
1711 			/* Finish job */
1712 			break;
1713 		}
1714 		if (cv_reltimedwait(&pvs->wd_condvar, &pvs->mutex,
1715 		    SEC_TO_TICK(1), TR_CLOCK_TICK) > 0) {
1716 			/* Explicitly woken up, finish job */
1717 			break;
1718 		}
1719 	}
1720 
1721 	/* Confirm thread termination */
1722 	cv_signal(&pvs->syncvar);
1723 	mutex_exit(&pvs->mutex);
1724 }
1725 
1726 static int
pvscsi_ccache_constructor(void * buf,void * cdrarg,int kmflags)1727 pvscsi_ccache_constructor(void *buf, void *cdrarg, int kmflags)
1728 {
1729 	int		(*callback)(caddr_t);
1730 	uint_t		cookiec;
1731 	pvscsi_cmd_t	*cmd = (pvscsi_cmd_t *)buf;
1732 	pvscsi_softc_t	*pvs = cdrarg;
1733 	struct scsi_address ap;
1734 
1735 	callback = (kmflags == KM_SLEEP) ? DDI_DMA_SLEEP : DDI_DMA_DONTWAIT;
1736 	ap.a_hba_tran = pvs->tran;
1737 	ap.a_target = 0;
1738 	ap.a_lun = 0;
1739 
1740 	/* Allocate a DMA handle for data transfers */
1741 	if ((ddi_dma_alloc_handle(pvs->dip, &pvs->io_dma_attr, callback,
1742 	    NULL, &cmd->cmd_dmahdl)) != DDI_SUCCESS) {
1743 		dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA handle");
1744 		return (-1);
1745 	}
1746 
1747 	/* Setup ARQ buffer */
1748 	if ((cmd->arqbuf = scsi_alloc_consistent_buf(&ap, (struct buf *)NULL,
1749 	    SENSE_BUFFER_SIZE, B_READ, callback, NULL)) == NULL) {
1750 		dev_err(pvs->dip, CE_WARN, "!failed to allocate ARQ buffer");
1751 		goto free_handle;
1752 	}
1753 
1754 	if (ddi_dma_alloc_handle(pvs->dip, &pvs->hba_dma_attr,
1755 	    callback, NULL, &cmd->arqhdl) != DDI_SUCCESS) {
1756 		dev_err(pvs->dip, CE_WARN,
1757 		    "!failed to allocate DMA handle for ARQ buffer");
1758 		goto free_arqbuf;
1759 	}
1760 
1761 	if (ddi_dma_buf_bind_handle(cmd->arqhdl, cmd->arqbuf,
1762 	    (DDI_DMA_READ | DDI_DMA_CONSISTENT), callback, NULL,
1763 	    &cmd->arqc, &cookiec) != DDI_SUCCESS) {
1764 		dev_err(pvs->dip, CE_WARN, "!failed to bind ARQ buffer");
1765 		goto free_arqhdl;
1766 	}
1767 
1768 	return (0);
1769 
1770 free_arqhdl:
1771 	ddi_dma_free_handle(&cmd->arqhdl);
1772 free_arqbuf:
1773 	scsi_free_consistent_buf(cmd->arqbuf);
1774 free_handle:
1775 	ddi_dma_free_handle(&cmd->cmd_dmahdl);
1776 
1777 	return (-1);
1778 }
1779 
1780 /* ARGSUSED cdrarg */
1781 static void
pvscsi_ccache_destructor(void * buf,void * cdrarg)1782 pvscsi_ccache_destructor(void *buf, void *cdrarg)
1783 {
1784 	pvscsi_cmd_t	*cmd = (pvscsi_cmd_t *)buf;
1785 
1786 	if (cmd->cmd_dmahdl != NULL) {
1787 		(void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
1788 		ddi_dma_free_handle(&cmd->cmd_dmahdl);
1789 		cmd->cmd_dmahdl = NULL;
1790 	}
1791 
1792 	if (cmd->arqhdl != NULL) {
1793 		(void) ddi_dma_unbind_handle(cmd->arqhdl);
1794 		ddi_dma_free_handle(&cmd->arqhdl);
1795 		cmd->arqhdl = NULL;
1796 	}
1797 
1798 	if (cmd->arqbuf != NULL) {
1799 		scsi_free_consistent_buf(cmd->arqbuf);
1800 		cmd->arqbuf = NULL;
1801 	}
1802 }
1803 
1804 /* tran_* entry points and setup */
1805 /* ARGSUSED hba_dip tgt_dip hba_tran */
1806 static int
pvscsi_tgt_init(dev_info_t * hba_dip,dev_info_t * tgt_dip,scsi_hba_tran_t * hba_tran,struct scsi_device * sd)1807 pvscsi_tgt_init(dev_info_t *hba_dip, dev_info_t *tgt_dip,
1808     scsi_hba_tran_t *hba_tran, struct scsi_device *sd)
1809 {
1810 	pvscsi_softc_t	*pvs = SDEV2PRIV(sd);
1811 
1812 	ASSERT(pvs != NULL);
1813 
1814 	if (sd->sd_address.a_target >= PVSCSI_MAXTGTS)
1815 		return (DDI_FAILURE);
1816 
1817 	return (DDI_SUCCESS);
1818 }
1819 
1820 static int
pvscsi_start(struct scsi_address * ap,struct scsi_pkt * pkt)1821 pvscsi_start(struct scsi_address *ap, struct scsi_pkt *pkt)
1822 {
1823 	boolean_t	poll = ((pkt->pkt_flags & FLAG_NOINTR) != 0);
1824 	int		rc;
1825 	pvscsi_cmd_t	*cmd = PKT2CMD(pkt);
1826 	pvscsi_softc_t	*pvs = ap->a_hba_tran->tran_hba_private;
1827 
1828 	ASSERT(cmd->pkt == pkt);
1829 	ASSERT(cmd->cmd_pvs == pvs);
1830 
1831 	/*
1832 	 * Reinitialize some fields because the packet may
1833 	 * have been resubmitted.
1834 	 */
1835 	pkt->pkt_reason = CMD_CMPLT;
1836 	pkt->pkt_state = 0;
1837 	pkt->pkt_statistics = 0;
1838 
1839 	/* Zero status byte */
1840 	*(pkt->pkt_scbp) = 0;
1841 
1842 	if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1843 		ASSERT(cmd->cmd_dma_count != 0);
1844 		pkt->pkt_resid = cmd->cmd_dma_count;
1845 
1846 		/*
1847 		 * Consistent packets need to be synced first
1848 		 * (only for data going out).
1849 		 */
1850 		if ((cmd->flags & PVSCSI_FLAG_IO_IOPB) != 0) {
1851 			(void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
1852 			    DDI_DMA_SYNC_FORDEV);
1853 		}
1854 	}
1855 
1856 	cmd->cmd_target = ap->a_target;
1857 
1858 	mutex_enter(&pvs->mutex);
1859 	if (HBA_IS_QUIESCED(pvs) && !poll) {
1860 		mutex_exit(&pvs->mutex);
1861 		return (TRAN_BUSY);
1862 	}
1863 	mutex_exit(&pvs->mutex);
1864 
1865 	rc = pvscsi_transport_command(pvs, cmd);
1866 
1867 	if (poll) {
1868 		pvscsi_cmd_t *dcmd;
1869 		boolean_t qnotify;
1870 
1871 		if (rc == TRAN_ACCEPT)
1872 			rc = pvscsi_poll_cmd(pvs, cmd);
1873 
1874 		mutex_enter(&pvs->rx_mutex);
1875 		dcmd = pvscsi_process_comp_ring(pvs);
1876 		mutex_exit(&pvs->rx_mutex);
1877 
1878 		mutex_enter(&pvs->mutex);
1879 		qnotify = HBA_QUIESCE_PENDING(pvs);
1880 		mutex_exit(&pvs->mutex);
1881 
1882 		pvscsi_complete_chained(dcmd);
1883 
1884 		if (qnotify)
1885 			pvscsi_quiesce_notify(pvs);
1886 	}
1887 
1888 	return (rc);
1889 }
1890 
1891 static int
pvscsi_reset(struct scsi_address * ap,int level)1892 pvscsi_reset(struct scsi_address *ap, int level)
1893 {
1894 	pvscsi_softc_t	*pvs = AP2PRIV(ap);
1895 
1896 	switch (level) {
1897 	case RESET_ALL:
1898 		return (pvscsi_reset_generic(pvs, NULL));
1899 	case RESET_TARGET:
1900 		ASSERT(ap != NULL);
1901 		return (pvscsi_reset_generic(pvs, ap));
1902 	default:
1903 		return (0);
1904 	}
1905 }
1906 
1907 static int
pvscsi_abort(struct scsi_address * ap,struct scsi_pkt * pkt)1908 pvscsi_abort(struct scsi_address *ap, struct scsi_pkt *pkt)
1909 {
1910 	boolean_t	qnotify = B_FALSE;
1911 	pvscsi_cmd_t	*pending;
1912 	pvscsi_softc_t	*pvs = ap->a_hba_tran->tran_hba_private;
1913 
1914 	mutex_enter(&pvs->tx_mutex);
1915 	mutex_enter(&pvs->rx_mutex);
1916 	if (pkt != NULL) {
1917 		/* Abort single command */
1918 		pvscsi_cmd_t *cmd = PKT2CMD(pkt);
1919 
1920 		if (pvscsi_abort_cmd(cmd, &pending) == CMD_ABORTED) {
1921 			/* Assume command is completely cancelled now */
1922 			cmd->flags |= PVSCSI_FLAG_ABORTED;
1923 		}
1924 	} else {
1925 		/* Abort all commands on the bus */
1926 		pvscsi_abort_all(ap, pvs, &pending, PVSCSI_FLAG_ABORTED);
1927 	}
1928 	qnotify = HBA_QUIESCE_PENDING(pvs);
1929 	mutex_exit(&pvs->rx_mutex);
1930 	mutex_exit(&pvs->tx_mutex);
1931 
1932 	pvscsi_complete_chained(pending);
1933 
1934 	if (qnotify)
1935 		pvscsi_quiesce_notify(pvs);
1936 
1937 	return (1);
1938 }
1939 
1940 /* ARGSUSED tgtonly */
1941 static int
pvscsi_getcap(struct scsi_address * ap,char * cap,int tgtonly)1942 pvscsi_getcap(struct scsi_address *ap, char *cap, int tgtonly)
1943 {
1944 	pvscsi_softc_t	*pvs = ap->a_hba_tran->tran_hba_private;
1945 
1946 	if (cap == NULL)
1947 		return (-1);
1948 
1949 	switch (scsi_hba_lookup_capstr(cap)) {
1950 	case SCSI_CAP_ARQ:
1951 		return ((pvs->flags & PVSCSI_HBA_AUTO_REQUEST_SENSE) != 0);
1952 	case SCSI_CAP_UNTAGGED_QING:
1953 		return (1);
1954 	default:
1955 		return (-1);
1956 	}
1957 }
1958 
1959 /* ARGSUSED tgtonly */
1960 static int
pvscsi_setcap(struct scsi_address * ap,char * cap,int value,int tgtonly)1961 pvscsi_setcap(struct scsi_address *ap, char *cap, int value, int tgtonly)
1962 {
1963 	pvscsi_softc_t	*pvs = ap->a_hba_tran->tran_hba_private;
1964 
1965 	if (cap == NULL)
1966 		return (-1);
1967 
1968 	switch (scsi_hba_lookup_capstr(cap)) {
1969 	case SCSI_CAP_ARQ:
1970 		mutex_enter(&pvs->mutex);
1971 		if (value == 0)
1972 			pvs->flags &= ~PVSCSI_HBA_AUTO_REQUEST_SENSE;
1973 		else
1974 			pvs->flags |= PVSCSI_HBA_AUTO_REQUEST_SENSE;
1975 		mutex_exit(&pvs->mutex);
1976 		return (1);
1977 	default:
1978 		return (0);
1979 	}
1980 }
1981 
1982 static void
pvscsi_destroy_pkt(struct scsi_address * ap,struct scsi_pkt * pkt)1983 pvscsi_destroy_pkt(struct scsi_address *ap, struct scsi_pkt *pkt)
1984 {
1985 	pvscsi_cmd_t	*cmd = PKT2CMD(pkt);
1986 	pvscsi_softc_t	*pvs = ap->a_hba_tran->tran_hba_private;
1987 
1988 	ASSERT(cmd->cmd_pvs == pvs);
1989 
1990 	if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
1991 		cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
1992 		(void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
1993 	}
1994 
1995 	if (cmd->ctx != NULL) {
1996 		mutex_enter(&pvs->mutex);
1997 		pvscsi_release_ctx(cmd);
1998 		mutex_exit(&pvs->mutex);
1999 	}
2000 
2001 	if ((cmd->flags & PVSCSI_FLAGS_EXT) != 0)
2002 		pvscsi_cmd_ext_free(cmd);
2003 
2004 	kmem_cache_free(pvs->cmd_cache, cmd);
2005 }
2006 
2007 static struct scsi_pkt *
pvscsi_init_pkt(struct scsi_address * ap,struct scsi_pkt * pkt,struct buf * bp,int cmdlen,int statuslen,int tgtlen,int flags,int (* callback)(),caddr_t arg)2008 pvscsi_init_pkt(struct scsi_address *ap, struct scsi_pkt *pkt, struct buf *bp,
2009     int cmdlen, int statuslen, int tgtlen, int flags, int (*callback)(),
2010     caddr_t arg)
2011 {
2012 	boolean_t	is_new;
2013 	int		kf = (callback == SLEEP_FUNC) ? KM_SLEEP: KM_NOSLEEP;
2014 	int		rc, i;
2015 	pvscsi_cmd_t	*cmd;
2016 	pvscsi_softc_t	*pvs;
2017 
2018 	pvs = ap->a_hba_tran->tran_hba_private;
2019 	ASSERT(pvs != NULL);
2020 
2021 	/* Allocate a new SCSI packet */
2022 	if (pkt == NULL) {
2023 		ddi_dma_handle_t saved_dmahdl, saved_arqhdl;
2024 		struct buf	*saved_arqbuf;
2025 		ddi_dma_cookie_t saved_arqc;
2026 
2027 		is_new = B_TRUE;
2028 
2029 		if ((cmd = kmem_cache_alloc(pvs->cmd_cache, kf)) == NULL)
2030 			return (NULL);
2031 
2032 		saved_dmahdl = cmd->cmd_dmahdl;
2033 		saved_arqhdl = cmd->arqhdl;
2034 		saved_arqbuf = cmd->arqbuf;
2035 		saved_arqc = cmd->arqc;
2036 
2037 		bzero(cmd, sizeof (pvscsi_cmd_t) -
2038 		    sizeof (cmd->cached_cookies));
2039 
2040 		cmd->cmd_pvs = pvs;
2041 		cmd->cmd_dmahdl = saved_dmahdl;
2042 		cmd->arqhdl = saved_arqhdl;
2043 		cmd->arqbuf = saved_arqbuf;
2044 		cmd->arqc = saved_arqc;
2045 
2046 		pkt = &cmd->cached_pkt;
2047 		pkt->pkt_ha_private = (opaque_t)cmd;
2048 		pkt->pkt_address = *ap;
2049 		pkt->pkt_scbp = (uint8_t *)&cmd->cmd_scb;
2050 		pkt->pkt_cdbp = (uint8_t *)&cmd->cmd_cdb;
2051 		pkt->pkt_private = (opaque_t)&cmd->tgt_priv;
2052 
2053 		cmd->tgtlen = tgtlen;
2054 		cmd->statuslen = statuslen;
2055 		cmd->cmdlen = cmdlen;
2056 		cmd->pkt = pkt;
2057 		cmd->ctx = NULL;
2058 
2059 		/* Allocate extended buffers */
2060 		if ((cmdlen > sizeof (cmd->cmd_cdb)) ||
2061 		    (statuslen > sizeof (cmd->cmd_scb)) ||
2062 		    (tgtlen > sizeof (cmd->tgt_priv))) {
2063 			if (pvscsi_cmd_ext_alloc(pvs, cmd, kf) != DDI_SUCCESS) {
2064 				dev_err(pvs->dip, CE_WARN,
2065 				    "!extent allocation failed");
2066 				goto out;
2067 			}
2068 		}
2069 	} else {
2070 		is_new = B_FALSE;
2071 
2072 		cmd = PKT2CMD(pkt);
2073 		cmd->flags &= PVSCSI_FLAGS_PERSISTENT;
2074 	}
2075 
2076 	ASSERT((cmd->flags & PVSCSI_FLAG_TRANSPORT) == 0);
2077 
2078 	if ((flags & PKT_XARQ) != 0)
2079 		cmd->flags |= PVSCSI_FLAG_XARQ;
2080 
2081 	/* Handle partial DMA transfers */
2082 	if (cmd->cmd_nwin > 0) {
2083 		if (++cmd->cmd_winindex >= cmd->cmd_nwin)
2084 			return (NULL);
2085 		if (ddi_dma_getwin(cmd->cmd_dmahdl, cmd->cmd_winindex,
2086 		    &cmd->cmd_dma_offset, &cmd->cmd_dma_len,
2087 		    &cmd->cmd_dmac, &cmd->cmd_dmaccount) == DDI_FAILURE)
2088 			return (NULL);
2089 		goto handle_dma_cookies;
2090 	}
2091 
2092 	/* Setup data buffer */
2093 	if (bp != NULL && bp->b_bcount > 0 &&
2094 	    (cmd->flags & PVSCSI_FLAG_DMA_VALID) == 0) {
2095 		int dma_flags;
2096 
2097 		ASSERT(cmd->cmd_dmahdl != NULL);
2098 
2099 		if ((bp->b_flags & B_READ) != 0) {
2100 			cmd->flags |= PVSCSI_FLAG_IO_READ;
2101 			dma_flags = DDI_DMA_READ;
2102 		} else {
2103 			cmd->flags &= ~PVSCSI_FLAG_IO_READ;
2104 			dma_flags = DDI_DMA_WRITE;
2105 		}
2106 		if ((flags & PKT_CONSISTENT) != 0) {
2107 			cmd->flags |= PVSCSI_FLAG_IO_IOPB;
2108 			dma_flags |= DDI_DMA_CONSISTENT;
2109 		}
2110 		if ((flags & PKT_DMA_PARTIAL) != 0)
2111 			dma_flags |= DDI_DMA_PARTIAL;
2112 
2113 		rc = ddi_dma_buf_bind_handle(cmd->cmd_dmahdl, bp,
2114 		    dma_flags, callback, arg, &cmd->cmd_dmac,
2115 		    &cmd->cmd_dmaccount);
2116 		if (rc == DDI_DMA_PARTIAL_MAP) {
2117 			(void) ddi_dma_numwin(cmd->cmd_dmahdl,
2118 			    &cmd->cmd_nwin);
2119 			cmd->cmd_winindex = 0;
2120 			(void) ddi_dma_getwin(cmd->cmd_dmahdl,
2121 			    cmd->cmd_winindex, &cmd->cmd_dma_offset,
2122 			    &cmd->cmd_dma_len, &cmd->cmd_dmac,
2123 			    &cmd->cmd_dmaccount);
2124 		} else if (rc != 0 && rc != DDI_DMA_MAPPED) {
2125 			switch (rc) {
2126 			case DDI_DMA_NORESOURCES:
2127 				bioerror(bp, 0);
2128 				break;
2129 			case DDI_DMA_BADATTR:
2130 			case DDI_DMA_NOMAPPING:
2131 				bioerror(bp, EFAULT);
2132 				break;
2133 			case DDI_DMA_TOOBIG:
2134 			default:
2135 				bioerror(bp, EINVAL);
2136 				break;
2137 			}
2138 			cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
2139 			goto out;
2140 		}
2141 
2142 handle_dma_cookies:
2143 		ASSERT(cmd->cmd_dmaccount > 0);
2144 		if (cmd->cmd_dmaccount > PVSCSI_MAX_SG_SIZE) {
2145 			dev_err(pvs->dip, CE_WARN,
2146 			    "!invalid cookie count: %d (max %d)",
2147 			    cmd->cmd_dmaccount, PVSCSI_MAX_SG_SIZE);
2148 			bioerror(bp, EINVAL);
2149 			goto out;
2150 		}
2151 
2152 		cmd->flags |= PVSCSI_FLAG_DMA_VALID;
2153 		cmd->cmd_dma_count = cmd->cmd_dmac.dmac_size;
2154 		cmd->cmd_total_dma_count += cmd->cmd_dmac.dmac_size;
2155 
2156 		cmd->cached_cookies[0] = cmd->cmd_dmac;
2157 
2158 		/*
2159 		 * Calculate total amount of bytes for this I/O and
2160 		 * store cookies for further processing.
2161 		 */
2162 		for (i = 1; i < cmd->cmd_dmaccount; i++) {
2163 			ddi_dma_nextcookie(cmd->cmd_dmahdl, &cmd->cmd_dmac);
2164 			cmd->cached_cookies[i] = cmd->cmd_dmac;
2165 			cmd->cmd_dma_count += cmd->cmd_dmac.dmac_size;
2166 			cmd->cmd_total_dma_count += cmd->cmd_dmac.dmac_size;
2167 		}
2168 
2169 		pkt->pkt_resid = (bp->b_bcount - cmd->cmd_total_dma_count);
2170 	}
2171 
2172 	return (pkt);
2173 
2174 out:
2175 	if (is_new)
2176 		pvscsi_destroy_pkt(ap, pkt);
2177 
2178 	return (NULL);
2179 }
2180 
2181 /* ARGSUSED ap */
2182 static void
pvscsi_dmafree(struct scsi_address * ap,struct scsi_pkt * pkt)2183 pvscsi_dmafree(struct scsi_address *ap, struct scsi_pkt *pkt)
2184 {
2185 	pvscsi_cmd_t	*cmd = PKT2CMD(pkt);
2186 
2187 	if ((cmd->flags & PVSCSI_FLAG_DMA_VALID) != 0) {
2188 		(void) ddi_dma_unbind_handle(cmd->cmd_dmahdl);
2189 		cmd->flags &= ~PVSCSI_FLAG_DMA_VALID;
2190 	}
2191 }
2192 
2193 /* ARGSUSED ap */
2194 static void
pvscsi_sync_pkt(struct scsi_address * ap,struct scsi_pkt * pkt)2195 pvscsi_sync_pkt(struct scsi_address *ap, struct scsi_pkt *pkt)
2196 {
2197 	pvscsi_cmd_t	*cmd = PKT2CMD(pkt);
2198 
2199 	if (cmd->cmd_dmahdl != NULL) {
2200 		(void) ddi_dma_sync(cmd->cmd_dmahdl, 0, 0,
2201 		    (cmd->flags & PVSCSI_FLAG_IO_READ) ?
2202 		    DDI_DMA_SYNC_FORCPU : DDI_DMA_SYNC_FORDEV);
2203 	}
2204 
2205 }
2206 
2207 /* ARGSUSED ap flag callback arg */
2208 static int
pvscsi_reset_notify(struct scsi_address * ap,int flag,void (* callback)(caddr_t),caddr_t arg)2209 pvscsi_reset_notify(struct scsi_address *ap, int flag,
2210     void (*callback)(caddr_t), caddr_t arg)
2211 {
2212 	return (DDI_FAILURE);
2213 }
2214 
2215 static int
pvscsi_quiesce_hba(dev_info_t * dip)2216 pvscsi_quiesce_hba(dev_info_t *dip)
2217 {
2218 	pvscsi_softc_t	*pvs;
2219 	scsi_hba_tran_t *tran;
2220 
2221 	if ((tran = ddi_get_driver_private(dip)) == NULL ||
2222 	    (pvs = TRAN2PRIV(tran)) == NULL)
2223 		return (-1);
2224 
2225 	mutex_enter(&pvs->mutex);
2226 	if (!HBA_IS_QUIESCED(pvs))
2227 		pvs->flags |= PVSCSI_HBA_QUIESCED;
2228 
2229 	if (pvs->cmd_queue_len != 0) {
2230 		/* Outstanding commands present, wait */
2231 		pvs->flags |= PVSCSI_HBA_QUIESCE_PENDING;
2232 		cv_wait(&pvs->quiescevar, &pvs->mutex);
2233 		ASSERT(pvs->cmd_queue_len == 0);
2234 	}
2235 	mutex_exit(&pvs->mutex);
2236 
2237 	/* Suspend taskq delivery and complete all scheduled tasks */
2238 	ddi_taskq_suspend(pvs->msg_tq);
2239 	ddi_taskq_wait(pvs->msg_tq);
2240 	ddi_taskq_suspend(pvs->comp_tq);
2241 	ddi_taskq_wait(pvs->comp_tq);
2242 
2243 	return (0);
2244 }
2245 
2246 static int
pvscsi_unquiesce_hba(dev_info_t * dip)2247 pvscsi_unquiesce_hba(dev_info_t *dip)
2248 {
2249 	pvscsi_softc_t	*pvs;
2250 	scsi_hba_tran_t	*tran;
2251 
2252 	if ((tran = ddi_get_driver_private(dip)) == NULL ||
2253 	    (pvs = TRAN2PRIV(tran)) == NULL)
2254 		return (-1);
2255 
2256 	mutex_enter(&pvs->mutex);
2257 	if (!HBA_IS_QUIESCED(pvs)) {
2258 		mutex_exit(&pvs->mutex);
2259 		return (0);
2260 	}
2261 	ASSERT(pvs->cmd_queue_len == 0);
2262 	pvs->flags &= ~PVSCSI_HBA_QUIESCED;
2263 	mutex_exit(&pvs->mutex);
2264 
2265 	/* Resume taskq delivery */
2266 	ddi_taskq_resume(pvs->msg_tq);
2267 	ddi_taskq_resume(pvs->comp_tq);
2268 
2269 	return (0);
2270 }
2271 
2272 static int
pvscsi_bus_config(dev_info_t * pdip,uint_t flags,ddi_bus_config_op_t op,void * arg,dev_info_t ** childp)2273 pvscsi_bus_config(dev_info_t *pdip, uint_t flags, ddi_bus_config_op_t op,
2274     void *arg, dev_info_t **childp)
2275 {
2276 	char		*p;
2277 	int		circ;
2278 	int		ret = NDI_FAILURE;
2279 	long		target = 0;
2280 	pvscsi_softc_t	*pvs;
2281 	scsi_hba_tran_t	*tran;
2282 
2283 	tran = ddi_get_driver_private(pdip);
2284 	pvs = tran->tran_hba_private;
2285 
2286 	ndi_devi_enter(pdip, &circ);
2287 	switch (op) {
2288 	case BUS_CONFIG_ONE:
2289 		if ((p = strrchr((char *)arg, '@')) != NULL &&
2290 		    ddi_strtol(p + 1, NULL, 16, &target) == 0)
2291 			ret = pvscsi_config_one(pdip, pvs, (int)target, childp);
2292 		break;
2293 	case BUS_CONFIG_DRIVER:
2294 	case BUS_CONFIG_ALL:
2295 		ret = pvscsi_config_all(pdip, pvs);
2296 		break;
2297 	default:
2298 		break;
2299 	}
2300 
2301 	if (ret == NDI_SUCCESS)
2302 		ret = ndi_busop_bus_config(pdip, flags, op, arg, childp, 0);
2303 	ndi_devi_exit(pdip, circ);
2304 
2305 	return (ret);
2306 }
2307 
2308 static int
pvscsi_hba_setup(pvscsi_softc_t * pvs)2309 pvscsi_hba_setup(pvscsi_softc_t *pvs)
2310 {
2311 	scsi_hba_tran_t	*hba_tran;
2312 
2313 	hba_tran = pvs->tran = scsi_hba_tran_alloc(pvs->dip,
2314 	    SCSI_HBA_CANSLEEP);
2315 	ASSERT(pvs->tran != NULL);
2316 
2317 	hba_tran->tran_hba_private = pvs;
2318 	hba_tran->tran_tgt_private = NULL;
2319 
2320 	hba_tran->tran_tgt_init	= pvscsi_tgt_init;
2321 	hba_tran->tran_tgt_free	= NULL;
2322 	hba_tran->tran_tgt_probe = scsi_hba_probe;
2323 
2324 	hba_tran->tran_start = pvscsi_start;
2325 	hba_tran->tran_reset = pvscsi_reset;
2326 	hba_tran->tran_abort = pvscsi_abort;
2327 	hba_tran->tran_getcap = pvscsi_getcap;
2328 	hba_tran->tran_setcap = pvscsi_setcap;
2329 	hba_tran->tran_init_pkt = pvscsi_init_pkt;
2330 	hba_tran->tran_destroy_pkt = pvscsi_destroy_pkt;
2331 
2332 	hba_tran->tran_dmafree = pvscsi_dmafree;
2333 	hba_tran->tran_sync_pkt = pvscsi_sync_pkt;
2334 	hba_tran->tran_reset_notify = pvscsi_reset_notify;
2335 
2336 	hba_tran->tran_quiesce = pvscsi_quiesce_hba;
2337 	hba_tran->tran_unquiesce = pvscsi_unquiesce_hba;
2338 	hba_tran->tran_bus_reset = NULL;
2339 
2340 	hba_tran->tran_add_eventcall = NULL;
2341 	hba_tran->tran_get_eventcookie = NULL;
2342 	hba_tran->tran_post_event = NULL;
2343 	hba_tran->tran_remove_eventcall = NULL;
2344 
2345 	hba_tran->tran_bus_config = pvscsi_bus_config;
2346 
2347 	hba_tran->tran_interconnect_type = INTERCONNECT_SAS;
2348 
2349 	if (scsi_hba_attach_setup(pvs->dip, &pvs->hba_dma_attr, hba_tran,
2350 	    SCSI_HBA_TRAN_CDB | SCSI_HBA_TRAN_SCB | SCSI_HBA_TRAN_CLONE) !=
2351 	    DDI_SUCCESS) {
2352 		dev_err(pvs->dip, CE_WARN, "!failed to attach HBA");
2353 		scsi_hba_tran_free(hba_tran);
2354 		pvs->tran = NULL;
2355 		return (-1);
2356 	}
2357 
2358 	return (0);
2359 }
2360 
2361 static int
pvscsi_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)2362 pvscsi_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
2363 {
2364 	int		instance;
2365 	pvscsi_softc_t	*pvs;
2366 	char		buf[32];
2367 
2368 	ASSERT(scsi_hba_iport_unit_address(dip) == NULL);
2369 
2370 	switch (cmd) {
2371 	case DDI_ATTACH:
2372 	case DDI_RESUME:
2373 		break;
2374 	default:
2375 		return (DDI_FAILURE);
2376 	}
2377 
2378 	instance = ddi_get_instance(dip);
2379 
2380 	/* Allocate softstate information */
2381 	if (ddi_soft_state_zalloc(pvscsi_sstate, instance) != DDI_SUCCESS) {
2382 		cmn_err(CE_WARN,
2383 		    "!ddi_soft_state_zalloc() failed for instance %d",
2384 		    instance);
2385 		return (DDI_FAILURE);
2386 	}
2387 
2388 	if ((pvs = ddi_get_soft_state(pvscsi_sstate, instance)) == NULL) {
2389 		cmn_err(CE_WARN, "!failed to get soft state for instance %d",
2390 		    instance);
2391 		goto fail;
2392 	}
2393 
2394 	/*
2395 	 * Indicate that we are 'sizeof (scsi_*(9S))' clean, we use
2396 	 * scsi_pkt_size() instead.
2397 	 */
2398 	scsi_size_clean(dip);
2399 
2400 	/* Setup HBA instance */
2401 	pvs->instance = instance;
2402 	pvs->dip = dip;
2403 	pvs->hba_dma_attr = pvscsi_hba_dma_attr;
2404 	pvs->ring_dma_attr = pvscsi_ring_dma_attr;
2405 	pvs->io_dma_attr = pvscsi_io_dma_attr;
2406 	mutex_init(&pvs->mutex, "pvscsi instance mutex", MUTEX_DRIVER, NULL);
2407 	mutex_init(&pvs->intr_mutex, "pvscsi instance interrupt mutex",
2408 	    MUTEX_DRIVER, NULL);
2409 	mutex_init(&pvs->rx_mutex, "pvscsi rx ring mutex", MUTEX_DRIVER, NULL);
2410 	mutex_init(&pvs->tx_mutex, "pvscsi tx ring mutex", MUTEX_DRIVER, NULL);
2411 	list_create(&pvs->cmd_ctx_pool, sizeof (pvscsi_cmd_ctx_t),
2412 	    offsetof(pvscsi_cmd_ctx_t, list));
2413 	list_create(&pvs->devnodes, sizeof (pvscsi_device_t),
2414 	    offsetof(pvscsi_device_t, list));
2415 	list_create(&pvs->cmd_queue, sizeof (pvscsi_cmd_t),
2416 	    offsetof(pvscsi_cmd_t, cmd_queue_node));
2417 	cv_init(&pvs->syncvar, "pvscsi synchronization cv", CV_DRIVER, NULL);
2418 	cv_init(&pvs->wd_condvar, "pvscsi watchdog cv", CV_DRIVER, NULL);
2419 	cv_init(&pvs->quiescevar, "pvscsi quiesce cv", CV_DRIVER, NULL);
2420 
2421 	(void) sprintf(buf, "pvscsi%d_cache", instance);
2422 	pvs->cmd_cache = kmem_cache_create(buf, sizeof (pvscsi_cmd_t), 0,
2423 	    pvscsi_ccache_constructor, pvscsi_ccache_destructor, NULL,
2424 	    (void *)pvs, NULL, 0);
2425 	if (pvs->cmd_cache == NULL) {
2426 		dev_err(pvs->dip, CE_WARN,
2427 		    "!failed to create a cache for SCSI commands");
2428 		goto fail;
2429 	}
2430 
2431 	if ((pvscsi_setup_io(pvs)) != DDI_SUCCESS) {
2432 		dev_err(pvs->dip, CE_WARN, "!failed to setup I/O region");
2433 		goto free_cache;
2434 	}
2435 
2436 	pvscsi_reset_hba(pvs);
2437 
2438 	if ((pvscsi_allocate_rings(pvs)) != DDI_SUCCESS) {
2439 		dev_err(pvs->dip, CE_WARN, "!failed to allocate DMA rings");
2440 		goto free_io;
2441 	}
2442 
2443 	pvscsi_setup_rings(pvs);
2444 
2445 	if (pvscsi_setup_isr(pvs) != DDI_SUCCESS) {
2446 		dev_err(pvs->dip, CE_WARN, "!failed to setup ISR");
2447 		goto free_rings;
2448 	}
2449 
2450 	if (pvscsi_setup_sg(pvs) != DDI_SUCCESS) {
2451 		dev_err(pvs->dip, CE_WARN, "!failed to setup S/G");
2452 		goto free_intr;
2453 	}
2454 
2455 	if (pvscsi_hba_setup(pvs) != 0) {
2456 		dev_err(pvs->dip, CE_WARN, "!failed to setup HBA");
2457 		goto free_sg;
2458 	}
2459 
2460 	if ((pvs->comp_tq = ddi_taskq_create(pvs->dip, "comp_tq",
2461 	    MIN(UINT16_MAX, ncpus), TASKQ_DEFAULTPRI, 0)) == NULL) {
2462 		dev_err(pvs->dip, CE_WARN,
2463 		    "!failed to create completion taskq");
2464 		goto free_sg;
2465 	}
2466 
2467 	if ((pvs->msg_tq = ddi_taskq_create(pvs->dip, "msg_tq",
2468 	    1, TASKQ_DEFAULTPRI, 0)) == NULL) {
2469 		dev_err(pvs->dip, CE_WARN,
2470 		    "!failed to create message taskq");
2471 		goto free_comp_tq;
2472 	}
2473 
2474 	if (pvscsi_enable_intrs(pvs) != DDI_SUCCESS) {
2475 		dev_err(pvs->dip, CE_WARN, "!failed to enable interrupts");
2476 		goto free_msg_tq;
2477 	}
2478 
2479 	/* Launch watchdog thread */
2480 	pvs->wd_thread = thread_create(NULL, 0, pvscsi_wd_thread, pvs, 0, &p0,
2481 	    TS_RUN, minclsyspri);
2482 
2483 	return (DDI_SUCCESS);
2484 
2485 free_msg_tq:
2486 	ddi_taskq_destroy(pvs->msg_tq);
2487 free_comp_tq:
2488 	ddi_taskq_destroy(pvs->comp_tq);
2489 free_sg:
2490 	pvscsi_free_sg(pvs);
2491 free_intr:
2492 	pvscsi_free_intr_resources(pvs);
2493 free_rings:
2494 	pvscsi_reset_hba(pvs);
2495 	pvscsi_free_rings(pvs);
2496 free_io:
2497 	pvscsi_free_io(pvs);
2498 free_cache:
2499 	kmem_cache_destroy(pvs->cmd_cache);
2500 fail:
2501 	ddi_soft_state_free(pvscsi_sstate, instance);
2502 
2503 	return (DDI_FAILURE);
2504 }
2505 
2506 static int
pvscsi_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)2507 pvscsi_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
2508 {
2509 	int		instance;
2510 	pvscsi_softc_t	*pvs;
2511 
2512 	switch (cmd) {
2513 	case DDI_DETACH:
2514 		break;
2515 	default:
2516 		return (DDI_FAILURE);
2517 	}
2518 
2519 	instance = ddi_get_instance(dip);
2520 	if ((pvs = ddi_get_soft_state(pvscsi_sstate, instance)) == NULL) {
2521 		cmn_err(CE_WARN, "!failed to get soft state for instance %d",
2522 		    instance);
2523 		return (DDI_FAILURE);
2524 	}
2525 
2526 	pvscsi_reset_hba(pvs);
2527 	pvscsi_free_intr_resources(pvs);
2528 
2529 	/* Shutdown message taskq */
2530 	ddi_taskq_wait(pvs->msg_tq);
2531 	ddi_taskq_destroy(pvs->msg_tq);
2532 
2533 	/* Shutdown completion taskq */
2534 	ddi_taskq_wait(pvs->comp_tq);
2535 	ddi_taskq_destroy(pvs->comp_tq);
2536 
2537 	/* Shutdown watchdog thread */
2538 	mutex_enter(&pvs->mutex);
2539 	pvs->flags |= PVSCSI_DRIVER_SHUTDOWN;
2540 	cv_signal(&pvs->wd_condvar);
2541 	cv_wait(&pvs->syncvar, &pvs->mutex);
2542 	mutex_exit(&pvs->mutex);
2543 
2544 	pvscsi_free_sg(pvs);
2545 	pvscsi_free_rings(pvs);
2546 	pvscsi_free_io(pvs);
2547 
2548 	kmem_cache_destroy(pvs->cmd_cache);
2549 
2550 	mutex_destroy(&pvs->mutex);
2551 	mutex_destroy(&pvs->intr_mutex);
2552 	mutex_destroy(&pvs->rx_mutex);
2553 
2554 	cv_destroy(&pvs->syncvar);
2555 	cv_destroy(&pvs->wd_condvar);
2556 	cv_destroy(&pvs->quiescevar);
2557 
2558 	ddi_soft_state_free(pvscsi_sstate, instance);
2559 	ddi_prop_remove_all(dip);
2560 
2561 	return (DDI_SUCCESS);
2562 }
2563 
2564 static int
pvscsi_ioctl(dev_t dev,int cmd,intptr_t data,int mode,cred_t * credp,int * rval)2565 pvscsi_ioctl(dev_t dev, int cmd, intptr_t data, int mode, cred_t *credp,
2566     int *rval)
2567 {
2568 	int	ret;
2569 
2570 	if (ddi_get_soft_state(pvscsi_sstate, getminor(dev)) == NULL) {
2571 		cmn_err(CE_WARN, "!invalid device instance: %d", getminor(dev));
2572 		return (ENXIO);
2573 	}
2574 
2575 	/* Try to handle command in a common way */
2576 	if ((ret = scsi_hba_ioctl(dev, cmd, data, mode, credp, rval)) != ENOTTY)
2577 		return (ret);
2578 
2579 	cmn_err(CE_WARN, "!unsupported IOCTL command: 0x%X", cmd);
2580 
2581 	return (ENXIO);
2582 }
2583 
2584 static int
pvscsi_quiesce(dev_info_t * devi)2585 pvscsi_quiesce(dev_info_t *devi)
2586 {
2587 	scsi_hba_tran_t	*tran;
2588 	pvscsi_softc_t	*pvs;
2589 
2590 	if ((tran = ddi_get_driver_private(devi)) == NULL)
2591 		return (DDI_SUCCESS);
2592 
2593 	if ((pvs = tran->tran_hba_private) == NULL)
2594 		return (DDI_SUCCESS);
2595 
2596 	/* Mask all interrupts from device */
2597 	pvscsi_reg_write(pvs, PVSCSI_REG_OFFSET_INTR_MASK, 0);
2598 
2599 	/* Reset the HBA */
2600 	pvscsi_reset_hba(pvs);
2601 
2602 	return (DDI_SUCCESS);
2603 }
2604 
2605 /* module */
2606 
2607 static struct cb_ops pvscsi_cb_ops = {
2608 	.cb_open =	scsi_hba_open,
2609 	.cb_close =	scsi_hba_close,
2610 	.cb_strategy =	nodev,
2611 	.cb_print =	nodev,
2612 	.cb_dump =	nodev,
2613 	.cb_read =	nodev,
2614 	.cb_write =	nodev,
2615 	.cb_ioctl =	pvscsi_ioctl,
2616 	.cb_devmap =	nodev,
2617 	.cb_mmap =	nodev,
2618 	.cb_segmap =	nodev,
2619 	.cb_chpoll =	nochpoll,
2620 	.cb_prop_op =	ddi_prop_op,
2621 	.cb_str =	NULL,
2622 	.cb_flag =	D_MP,
2623 	.cb_rev =	CB_REV,
2624 	.cb_aread =	nodev,
2625 	.cb_awrite =	nodev
2626 };
2627 
2628 static struct dev_ops pvscsi_ops = {
2629 	.devo_rev =	DEVO_REV,
2630 	.devo_refcnt =	0,
2631 	.devo_getinfo =	ddi_no_info,
2632 	.devo_identify = nulldev,
2633 	.devo_probe =	nulldev,
2634 	.devo_attach =	pvscsi_attach,
2635 	.devo_detach =	pvscsi_detach,
2636 	.devo_reset =	nodev,
2637 	.devo_cb_ops =	&pvscsi_cb_ops,
2638 	.devo_bus_ops =	NULL,
2639 	.devo_power =	NULL,
2640 	.devo_quiesce =	pvscsi_quiesce
2641 };
2642 
2643 #define	PVSCSI_IDENT "VMware PVSCSI"
2644 
2645 static struct modldrv modldrv = {
2646 	&mod_driverops,
2647 	PVSCSI_IDENT,
2648 	&pvscsi_ops,
2649 };
2650 
2651 static struct modlinkage modlinkage = {
2652 	MODREV_1,
2653 	&modldrv,
2654 	NULL
2655 };
2656 
2657 int
_init(void)2658 _init(void)
2659 {
2660 	int	ret;
2661 
2662 	if ((ret = ddi_soft_state_init(&pvscsi_sstate,
2663 	    sizeof (struct pvscsi_softc), PVSCSI_INITIAL_SSTATE_ITEMS)) != 0) {
2664 		cmn_err(CE_WARN, "!ddi_soft_state_init() failed");
2665 		return (ret);
2666 	}
2667 
2668 	if ((ret = scsi_hba_init(&modlinkage)) != 0) {
2669 		cmn_err(CE_WARN, "!scsi_hba_init() failed");
2670 		ddi_soft_state_fini(&pvscsi_sstate);
2671 		return (ret);
2672 	}
2673 
2674 	if ((ret = mod_install(&modlinkage)) != 0) {
2675 		cmn_err(CE_WARN, "!mod_install() failed");
2676 		ddi_soft_state_fini(&pvscsi_sstate);
2677 		scsi_hba_fini(&modlinkage);
2678 	}
2679 
2680 	return (ret);
2681 }
2682 
2683 int
_info(struct modinfo * modinfop)2684 _info(struct modinfo *modinfop)
2685 {
2686 	return (mod_info(&modlinkage, modinfop));
2687 }
2688 
2689 int
_fini(void)2690 _fini(void)
2691 {
2692 	int	ret;
2693 
2694 	if ((ret = mod_remove(&modlinkage)) == 0) {
2695 		ddi_soft_state_fini(&pvscsi_sstate);
2696 		scsi_hba_fini(&modlinkage);
2697 	}
2698 
2699 	return (ret);
2700 }
2701