mps_sas.c revision b215fc814bd3faacf5b6c806c1bb4419093e980b
1/*-
2 * Copyright (c) 2009 Yahoo! Inc.
3 * Copyright (c) 2011-2015 LSI Corp.
4 * Copyright (c) 2013-2015 Avago Technologies
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
29 *
30 * $FreeBSD$
31 */
32
33#include <sys/cdefs.h>
34__FBSDID("$FreeBSD$");
35
36/* Communications core for Avago Technologies (LSI) MPT2 */
37
38/* TODO Move headers to mpsvar */
39#include <sys/types.h>
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/kernel.h>
43#include <sys/selinfo.h>
44#include <sys/module.h>
45#include <sys/bus.h>
46#include <sys/conf.h>
47#include <sys/bio.h>
48#include <sys/malloc.h>
49#include <sys/uio.h>
50#include <sys/sysctl.h>
51#include <sys/endian.h>
52#include <sys/queue.h>
53#include <sys/kthread.h>
54#include <sys/taskqueue.h>
55#include <sys/sbuf.h>
56
57#include <machine/bus.h>
58#include <machine/resource.h>
59#include <sys/rman.h>
60
61#include <machine/stdarg.h>
62
63#include <cam/cam.h>
64#include <cam/cam_ccb.h>
65#include <cam/cam_xpt.h>
66#include <cam/cam_debug.h>
67#include <cam/cam_sim.h>
68#include <cam/cam_xpt_sim.h>
69#include <cam/cam_xpt_periph.h>
70#include <cam/cam_periph.h>
71#include <cam/scsi/scsi_all.h>
72#include <cam/scsi/scsi_message.h>
73#if __FreeBSD_version >= 900026
74#include <cam/scsi/smp_all.h>
75#endif
76
77#include <dev/mps/mpi/mpi2_type.h>
78#include <dev/mps/mpi/mpi2.h>
79#include <dev/mps/mpi/mpi2_ioc.h>
80#include <dev/mps/mpi/mpi2_sas.h>
81#include <dev/mps/mpi/mpi2_cnfg.h>
82#include <dev/mps/mpi/mpi2_init.h>
83#include <dev/mps/mpi/mpi2_tool.h>
84#include <dev/mps/mps_ioctl.h>
85#include <dev/mps/mpsvar.h>
86#include <dev/mps/mps_table.h>
87#include <dev/mps/mps_sas.h>
88
89#define MPSSAS_DISCOVERY_TIMEOUT	20
90#define MPSSAS_MAX_DISCOVERY_TIMEOUTS	10 /* 200 seconds */
91
92/*
93 * static array to check SCSI OpCode for EEDP protection bits
94 */
95#define	PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP
96#define	PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
97#define	PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
98static uint8_t op_code_prot[256] = {
99	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
100	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
101	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
102	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
103	0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
104	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
105	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
106	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
107	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
108	0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
109	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
110	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
111	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
112	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
113	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
114	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
115};
116
117MALLOC_DEFINE(M_MPSSAS, "MPSSAS", "MPS SAS memory");
118
119static void mpssas_remove_device(struct mps_softc *, struct mps_command *);
120static void mpssas_remove_complete(struct mps_softc *, struct mps_command *);
121static void mpssas_action(struct cam_sim *sim, union ccb *ccb);
122static void mpssas_poll(struct cam_sim *sim);
123static int mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm,
124    struct mps_command *cm);
125static void mpssas_scsiio_timeout(void *data);
126static void mpssas_abort_complete(struct mps_softc *sc, struct mps_command *cm);
127static void mpssas_direct_drive_io(struct mpssas_softc *sassc,
128    struct mps_command *cm, union ccb *ccb);
129static void mpssas_action_scsiio(struct mpssas_softc *, union ccb *);
130static void mpssas_scsiio_complete(struct mps_softc *, struct mps_command *);
131static void mpssas_action_resetdev(struct mpssas_softc *, union ccb *);
132#if __FreeBSD_version >= 900026
133static void mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm);
134static void mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb,
135			       uint64_t sasaddr);
136static void mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb);
137#endif //FreeBSD_version >= 900026
138static void mpssas_resetdev_complete(struct mps_softc *, struct mps_command *);
139static void mpssas_async(void *callback_arg, uint32_t code,
140			 struct cam_path *path, void *arg);
141#if (__FreeBSD_version < 901503) || \
142    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
143static void mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
144			      struct ccb_getdev *cgd);
145static void mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb);
146#endif
147static int mpssas_send_portenable(struct mps_softc *sc);
148static void mpssas_portenable_complete(struct mps_softc *sc,
149    struct mps_command *cm);
150
151struct mpssas_target *
152mpssas_find_target_by_handle(struct mpssas_softc *sassc, int start, uint16_t handle)
153{
154	struct mpssas_target *target;
155	int i;
156
157	for (i = start; i < sassc->maxtargets; i++) {
158		target = &sassc->targets[i];
159		if (target->handle == handle)
160			return (target);
161	}
162
163	return (NULL);
164}
165
166/* we need to freeze the simq during attach and diag reset, to avoid failing
167 * commands before device handles have been found by discovery.  Since
168 * discovery involves reading config pages and possibly sending commands,
169 * discovery actions may continue even after we receive the end of discovery
170 * event, so refcount discovery actions instead of assuming we can unfreeze
171 * the simq when we get the event.
172 */
173void
174mpssas_startup_increment(struct mpssas_softc *sassc)
175{
176	MPS_FUNCTRACE(sassc->sc);
177
178	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
179		if (sassc->startup_refcount++ == 0) {
180			/* just starting, freeze the simq */
181			mps_dprint(sassc->sc, MPS_INIT,
182			    "%s freezing simq\n", __func__);
183#if __FreeBSD_version >= 1000039
184			xpt_hold_boot();
185#endif
186			xpt_freeze_simq(sassc->sim, 1);
187		}
188		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
189		    sassc->startup_refcount);
190	}
191}
192
193void
194mpssas_release_simq_reinit(struct mpssas_softc *sassc)
195{
196	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
197		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
198		xpt_release_simq(sassc->sim, 1);
199		mps_dprint(sassc->sc, MPS_INFO, "Unfreezing SIM queue\n");
200	}
201}
202
203void
204mpssas_startup_decrement(struct mpssas_softc *sassc)
205{
206	MPS_FUNCTRACE(sassc->sc);
207
208	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
209		if (--sassc->startup_refcount == 0) {
210			/* finished all discovery-related actions, release
211			 * the simq and rescan for the latest topology.
212			 */
213			mps_dprint(sassc->sc, MPS_INIT,
214			    "%s releasing simq\n", __func__);
215			sassc->flags &= ~MPSSAS_IN_STARTUP;
216			xpt_release_simq(sassc->sim, 1);
217#if __FreeBSD_version >= 1000039
218			xpt_release_boot();
219#else
220			mpssas_rescan_target(sassc->sc, NULL);
221#endif
222		}
223		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
224		    sassc->startup_refcount);
225	}
226}
227
228/* The firmware requires us to stop sending commands when we're doing task
229 * management, so refcount the TMs and keep the simq frozen when any are in
230 * use.
231 */
232struct mps_command *
233mpssas_alloc_tm(struct mps_softc *sc)
234{
235	struct mps_command *tm;
236
237	tm = mps_alloc_high_priority_command(sc);
238	return tm;
239}
240
241void
242mpssas_free_tm(struct mps_softc *sc, struct mps_command *tm)
243{
244	int target_id = 0xFFFFFFFF;
245
246	if (tm == NULL)
247		return;
248
249	/*
250	 * For TM's the devq is frozen for the device.  Unfreeze it here and
251	 * free the resources used for freezing the devq.  Must clear the
252	 * INRESET flag as well or scsi I/O will not work.
253	 */
254	if (tm->cm_targ != NULL) {
255		tm->cm_targ->flags &= ~MPSSAS_TARGET_INRESET;
256		target_id = tm->cm_targ->tid;
257	}
258	if (tm->cm_ccb) {
259		mps_dprint(sc, MPS_INFO, "Unfreezing devq for target ID %d\n",
260		    target_id);
261		xpt_release_devq(tm->cm_ccb->ccb_h.path, 1, TRUE);
262		xpt_free_path(tm->cm_ccb->ccb_h.path);
263		xpt_free_ccb(tm->cm_ccb);
264	}
265
266	mps_free_high_priority_command(sc, tm);
267}
268
269void
270mpssas_rescan_target(struct mps_softc *sc, struct mpssas_target *targ)
271{
272	struct mpssas_softc *sassc = sc->sassc;
273	path_id_t pathid;
274	target_id_t targetid;
275	union ccb *ccb;
276
277	MPS_FUNCTRACE(sc);
278	pathid = cam_sim_path(sassc->sim);
279	if (targ == NULL)
280		targetid = CAM_TARGET_WILDCARD;
281	else
282		targetid = targ - sassc->targets;
283
284	/*
285	 * Allocate a CCB and schedule a rescan.
286	 */
287	ccb = xpt_alloc_ccb_nowait();
288	if (ccb == NULL) {
289		mps_dprint(sc, MPS_ERROR, "unable to alloc CCB for rescan\n");
290		return;
291	}
292
293	if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
294	    targetid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
295		mps_dprint(sc, MPS_ERROR, "unable to create path for rescan\n");
296		xpt_free_ccb(ccb);
297		return;
298	}
299
300	if (targetid == CAM_TARGET_WILDCARD)
301		ccb->ccb_h.func_code = XPT_SCAN_BUS;
302	else
303		ccb->ccb_h.func_code = XPT_SCAN_TGT;
304
305	mps_dprint(sc, MPS_TRACE, "%s targetid %u\n", __func__, targetid);
306	xpt_rescan(ccb);
307}
308
309static void
310mpssas_log_command(struct mps_command *cm, u_int level, const char *fmt, ...)
311{
312	struct sbuf sb;
313	va_list ap;
314	char str[192];
315	char path_str[64];
316
317	if (cm == NULL)
318		return;
319
320	/* No need to be in here if debugging isn't enabled */
321	if ((cm->cm_sc->mps_debug & level) == 0)
322		return;
323
324	sbuf_new(&sb, str, sizeof(str), 0);
325
326	va_start(ap, fmt);
327
328	if (cm->cm_ccb != NULL) {
329		xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str,
330				sizeof(path_str));
331		sbuf_cat(&sb, path_str);
332		if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) {
333			scsi_command_string(&cm->cm_ccb->csio, &sb);
334			sbuf_printf(&sb, "length %d ",
335				    cm->cm_ccb->csio.dxfer_len);
336		}
337	}
338	else {
339		sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ",
340		    cam_sim_name(cm->cm_sc->sassc->sim),
341		    cam_sim_unit(cm->cm_sc->sassc->sim),
342		    cam_sim_bus(cm->cm_sc->sassc->sim),
343		    cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF,
344		    cm->cm_lun);
345	}
346
347	sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID);
348	sbuf_vprintf(&sb, fmt, ap);
349	sbuf_finish(&sb);
350	mps_print_field(cm->cm_sc, "%s", sbuf_data(&sb));
351
352	va_end(ap);
353}
354
355
356static void
357mpssas_remove_volume(struct mps_softc *sc, struct mps_command *tm)
358{
359	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
360	struct mpssas_target *targ;
361	uint16_t handle;
362
363	MPS_FUNCTRACE(sc);
364
365	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
366	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
367	targ = tm->cm_targ;
368
369	if (reply == NULL) {
370		/* XXX retry the remove after the diag reset completes? */
371		mps_dprint(sc, MPS_FAULT,
372		    "%s NULL reply resetting device 0x%04x\n", __func__,
373		    handle);
374		mpssas_free_tm(sc, tm);
375		return;
376	}
377
378	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
379	    MPI2_IOCSTATUS_SUCCESS) {
380		mps_dprint(sc, MPS_ERROR,
381		   "IOCStatus = 0x%x while resetting device 0x%x\n",
382		   le16toh(reply->IOCStatus), handle);
383	}
384
385	mps_dprint(sc, MPS_XINFO,
386	    "Reset aborted %u commands\n", reply->TerminationCount);
387	mps_free_reply(sc, tm->cm_reply_data);
388	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
389
390	mps_dprint(sc, MPS_XINFO,
391	    "clearing target %u handle 0x%04x\n", targ->tid, handle);
392
393	/*
394	 * Don't clear target if remove fails because things will get confusing.
395	 * Leave the devname and sasaddr intact so that we know to avoid reusing
396	 * this target id if possible, and so we can assign the same target id
397	 * to this device if it comes back in the future.
398	 */
399	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
400	    MPI2_IOCSTATUS_SUCCESS) {
401		targ = tm->cm_targ;
402		targ->handle = 0x0;
403		targ->encl_handle = 0x0;
404		targ->encl_slot = 0x0;
405		targ->exp_dev_handle = 0x0;
406		targ->phy_num = 0x0;
407		targ->linkrate = 0x0;
408		targ->devinfo = 0x0;
409		targ->flags = 0x0;
410	}
411
412	mpssas_free_tm(sc, tm);
413}
414
415
416/*
417 * No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal.
418 * Otherwise Volume Delete is same as Bare Drive Removal.
419 */
420void
421mpssas_prepare_volume_remove(struct mpssas_softc *sassc, uint16_t handle)
422{
423	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
424	struct mps_softc *sc;
425	struct mps_command *cm;
426	struct mpssas_target *targ = NULL;
427
428	MPS_FUNCTRACE(sassc->sc);
429	sc = sassc->sc;
430
431#ifdef WD_SUPPORT
432	/*
433	 * If this is a WD controller, determine if the disk should be exposed
434	 * to the OS or not.  If disk should be exposed, return from this
435	 * function without doing anything.
436	 */
437	if (sc->WD_available && (sc->WD_hide_expose ==
438	    MPS_WD_EXPOSE_ALWAYS)) {
439		return;
440	}
441#endif //WD_SUPPORT
442
443	targ = mpssas_find_target_by_handle(sassc, 0, handle);
444	if (targ == NULL) {
445		/* FIXME: what is the action? */
446		/* We don't know about this device? */
447		mps_dprint(sc, MPS_ERROR,
448		   "%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
449		return;
450	}
451
452	targ->flags |= MPSSAS_TARGET_INREMOVAL;
453
454	cm = mpssas_alloc_tm(sc);
455	if (cm == NULL) {
456		mps_dprint(sc, MPS_ERROR,
457		    "%s: command alloc failure\n", __func__);
458		return;
459	}
460
461	mpssas_rescan_target(sc, targ);
462
463	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
464	req->DevHandle = targ->handle;
465	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
466	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
467
468	/* SAS Hard Link Reset / SATA Link Reset */
469	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
470
471	cm->cm_targ = targ;
472	cm->cm_data = NULL;
473	cm->cm_desc.HighPriority.RequestFlags =
474	    MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
475	cm->cm_complete = mpssas_remove_volume;
476	cm->cm_complete_data = (void *)(uintptr_t)handle;
477
478	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
479	    __func__, targ->tid);
480	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
481
482	mps_map_command(sc, cm);
483}
484
485/*
486 * The MPT2 firmware performs debounce on the link to avoid transient link
487 * errors and false removals.  When it does decide that link has been lost
488 * and a device need to go away, it expects that the host will perform a
489 * target reset and then an op remove.  The reset has the side-effect of
490 * aborting any outstanding requests for the device, which is required for
491 * the op-remove to succeed.  It's not clear if the host should check for
492 * the device coming back alive after the reset.
493 */
494void
495mpssas_prepare_remove(struct mpssas_softc *sassc, uint16_t handle)
496{
497	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
498	struct mps_softc *sc;
499	struct mps_command *cm;
500	struct mpssas_target *targ = NULL;
501
502	MPS_FUNCTRACE(sassc->sc);
503
504	sc = sassc->sc;
505
506	targ = mpssas_find_target_by_handle(sassc, 0, handle);
507	if (targ == NULL) {
508		/* FIXME: what is the action? */
509		/* We don't know about this device? */
510		mps_dprint(sc, MPS_ERROR,
511		    "%s : invalid handle 0x%x \n", __func__, handle);
512		return;
513	}
514
515	targ->flags |= MPSSAS_TARGET_INREMOVAL;
516
517	cm = mpssas_alloc_tm(sc);
518	if (cm == NULL) {
519		mps_dprint(sc, MPS_ERROR,
520		    "%s: command alloc failure\n", __func__);
521		return;
522	}
523
524	mpssas_rescan_target(sc, targ);
525
526	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
527	memset(req, 0, sizeof(*req));
528	req->DevHandle = htole16(targ->handle);
529	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
530	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
531
532	/* SAS Hard Link Reset / SATA Link Reset */
533	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
534
535	cm->cm_targ = targ;
536	cm->cm_data = NULL;
537	cm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
538	cm->cm_complete = mpssas_remove_device;
539	cm->cm_complete_data = (void *)(uintptr_t)handle;
540
541	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
542	    __func__, targ->tid);
543	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
544
545	mps_map_command(sc, cm);
546}
547
548static void
549mpssas_remove_device(struct mps_softc *sc, struct mps_command *tm)
550{
551	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
552	MPI2_SAS_IOUNIT_CONTROL_REQUEST *req;
553	struct mpssas_target *targ;
554	struct mps_command *next_cm;
555	uint16_t handle;
556
557	MPS_FUNCTRACE(sc);
558
559	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
560	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
561	targ = tm->cm_targ;
562
563	/*
564	 * Currently there should be no way we can hit this case.  It only
565	 * happens when we have a failure to allocate chain frames, and
566	 * task management commands don't have S/G lists.
567	 */
568	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
569		mps_dprint(sc, MPS_ERROR,
570		    "%s: cm_flags = %#x for remove of handle %#04x! "
571		    "This should not happen!\n", __func__, tm->cm_flags,
572		    handle);
573	}
574
575	if (reply == NULL) {
576		/* XXX retry the remove after the diag reset completes? */
577		mps_dprint(sc, MPS_FAULT,
578		    "%s NULL reply resetting device 0x%04x\n", __func__,
579		    handle);
580		mpssas_free_tm(sc, tm);
581		return;
582	}
583
584	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
585	    MPI2_IOCSTATUS_SUCCESS) {
586		mps_dprint(sc, MPS_ERROR,
587		   "IOCStatus = 0x%x while resetting device 0x%x\n",
588		   le16toh(reply->IOCStatus), handle);
589	}
590
591	mps_dprint(sc, MPS_XINFO, "Reset aborted %u commands\n",
592	    le32toh(reply->TerminationCount));
593	mps_free_reply(sc, tm->cm_reply_data);
594	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
595
596	/* Reuse the existing command */
597	req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req;
598	memset(req, 0, sizeof(*req));
599	req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
600	req->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
601	req->DevHandle = htole16(handle);
602	tm->cm_data = NULL;
603	tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
604	tm->cm_complete = mpssas_remove_complete;
605	tm->cm_complete_data = (void *)(uintptr_t)handle;
606
607	mps_map_command(sc, tm);
608
609	mps_dprint(sc, MPS_XINFO, "clearing target %u handle 0x%04x\n",
610		   targ->tid, handle);
611	TAILQ_FOREACH_SAFE(tm, &targ->commands, cm_link, next_cm) {
612		union ccb *ccb;
613
614		mps_dprint(sc, MPS_XINFO, "Completing missed command %p\n", tm);
615		ccb = tm->cm_complete_data;
616		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
617		mpssas_scsiio_complete(sc, tm);
618	}
619}
620
621static void
622mpssas_remove_complete(struct mps_softc *sc, struct mps_command *tm)
623{
624	MPI2_SAS_IOUNIT_CONTROL_REPLY *reply;
625	uint16_t handle;
626	struct mpssas_target *targ;
627	struct mpssas_lun *lun;
628
629	MPS_FUNCTRACE(sc);
630
631	reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply;
632	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
633
634	/*
635	 * Currently there should be no way we can hit this case.  It only
636	 * happens when we have a failure to allocate chain frames, and
637	 * task management commands don't have S/G lists.
638	 */
639	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
640		mps_dprint(sc, MPS_XINFO,
641			   "%s: cm_flags = %#x for remove of handle %#04x! "
642			   "This should not happen!\n", __func__, tm->cm_flags,
643			   handle);
644		mpssas_free_tm(sc, tm);
645		return;
646	}
647
648	if (reply == NULL) {
649		/* most likely a chip reset */
650		mps_dprint(sc, MPS_FAULT,
651		    "%s NULL reply removing device 0x%04x\n", __func__, handle);
652		mpssas_free_tm(sc, tm);
653		return;
654	}
655
656	mps_dprint(sc, MPS_XINFO,
657	    "%s on handle 0x%04x, IOCStatus= 0x%x\n", __func__,
658	    handle, le16toh(reply->IOCStatus));
659
660	/*
661	 * Don't clear target if remove fails because things will get confusing.
662	 * Leave the devname and sasaddr intact so that we know to avoid reusing
663	 * this target id if possible, and so we can assign the same target id
664	 * to this device if it comes back in the future.
665	 */
666	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
667	    MPI2_IOCSTATUS_SUCCESS) {
668		targ = tm->cm_targ;
669		targ->handle = 0x0;
670		targ->encl_handle = 0x0;
671		targ->encl_slot = 0x0;
672		targ->exp_dev_handle = 0x0;
673		targ->phy_num = 0x0;
674		targ->linkrate = 0x0;
675		targ->devinfo = 0x0;
676		targ->flags = 0x0;
677
678		while(!SLIST_EMPTY(&targ->luns)) {
679			lun = SLIST_FIRST(&targ->luns);
680			SLIST_REMOVE_HEAD(&targ->luns, lun_link);
681			free(lun, M_MPT2);
682		}
683	}
684
685
686	mpssas_free_tm(sc, tm);
687}
688
689static int
690mpssas_register_events(struct mps_softc *sc)
691{
692	u32 events[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];
693
694	bzero(events, 16);
695	setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
696	setbit(events, MPI2_EVENT_SAS_DISCOVERY);
697	setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
698	setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE);
699	setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW);
700	setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
701	setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
702	setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
703	setbit(events, MPI2_EVENT_IR_VOLUME);
704	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
705	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
706	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);
707
708	mps_register_events(sc, events, mpssas_evt_handler, NULL,
709	    &sc->sassc->mpssas_eh);
710
711	return (0);
712}
713
714int
715mps_attach_sas(struct mps_softc *sc)
716{
717	struct mpssas_softc *sassc;
718	cam_status status;
719	int unit, error = 0;
720
721	MPS_FUNCTRACE(sc);
722	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);
723
724	sassc = malloc(sizeof(struct mpssas_softc), M_MPT2, M_WAITOK|M_ZERO);
725	if(!sassc) {
726		mps_dprint(sc, MPS_INIT|MPS_ERROR,
727		    "Cannot allocate SAS controller memory\n");
728		return (ENOMEM);
729	}
730
731	/*
732	 * XXX MaxTargets could change during a reinit.  Since we don't
733	 * resize the targets[] array during such an event, cache the value
734	 * of MaxTargets here so that we don't get into trouble later.  This
735	 * should move into the reinit logic.
736	 */
737	sassc->maxtargets = sc->facts->MaxTargets + sc->facts->MaxVolumes;
738	sassc->targets = malloc(sizeof(struct mpssas_target) *
739	    sassc->maxtargets, M_MPT2, M_WAITOK|M_ZERO);
740	if(!sassc->targets) {
741		mps_dprint(sc, MPS_INIT|MPS_ERROR,
742		    "Cannot allocate SAS target memory\n");
743		free(sassc, M_MPT2);
744		return (ENOMEM);
745	}
746	sc->sassc = sassc;
747	sassc->sc = sc;
748
749	if ((sassc->devq = cam_simq_alloc(sc->num_reqs)) == NULL) {
750		mps_dprint(sc, MPS_ERROR, "Cannot allocate SIMQ\n");
751		error = ENOMEM;
752		goto out;
753	}
754
755	unit = device_get_unit(sc->mps_dev);
756	sassc->sim = cam_sim_alloc(mpssas_action, mpssas_poll, "mps", sassc,
757	    unit, &sc->mps_mtx, sc->num_reqs, sc->num_reqs, sassc->devq);
758	if (sassc->sim == NULL) {
759		mps_dprint(sc, MPS_INIT|MPS_ERROR, "Cannot allocate SIM\n");
760		error = EINVAL;
761		goto out;
762	}
763
764	TAILQ_INIT(&sassc->ev_queue);
765
766	/* Initialize taskqueue for Event Handling */
767	TASK_INIT(&sassc->ev_task, 0, mpssas_firmware_event_work, sc);
768	sassc->ev_tq = taskqueue_create("mps_taskq", M_NOWAIT | M_ZERO,
769	    taskqueue_thread_enqueue, &sassc->ev_tq);
770	taskqueue_start_threads(&sassc->ev_tq, 1, PRIBIO, "%s taskq",
771	    device_get_nameunit(sc->mps_dev));
772
773	mps_lock(sc);
774
775	/*
776	 * XXX There should be a bus for every port on the adapter, but since
777	 * we're just going to fake the topology for now, we'll pretend that
778	 * everything is just a target on a single bus.
779	 */
780	if ((error = xpt_bus_register(sassc->sim, sc->mps_dev, 0)) != 0) {
781		mps_dprint(sc, MPS_INIT|MPS_ERROR,
782		    "Error %d registering SCSI bus\n", error);
783		mps_unlock(sc);
784		goto out;
785	}
786
787	/*
788	 * Assume that discovery events will start right away.
789	 *
790	 * Hold off boot until discovery is complete.
791	 */
792	sassc->flags |= MPSSAS_IN_STARTUP | MPSSAS_IN_DISCOVERY;
793	sc->sassc->startup_refcount = 0;
794	mpssas_startup_increment(sassc);
795
796	callout_init(&sassc->discovery_callout, 1 /*mpsafe*/);
797
798	/*
799	 * Register for async events so we can determine the EEDP
800	 * capabilities of devices.
801	 */
802	status = xpt_create_path(&sassc->path, /*periph*/NULL,
803	    cam_sim_path(sc->sassc->sim), CAM_TARGET_WILDCARD,
804	    CAM_LUN_WILDCARD);
805	if (status != CAM_REQ_CMP) {
806		mps_dprint(sc, MPS_ERROR|MPS_INIT,
807		    "Error %#x creating sim path\n", status);
808		sassc->path = NULL;
809	} else {
810		int event;
811
812#if (__FreeBSD_version >= 1000006) || \
813    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
814		event = AC_ADVINFO_CHANGED;
815#else
816		event = AC_FOUND_DEVICE;
817#endif
818		status = xpt_register_async(event, mpssas_async, sc,
819					    sassc->path);
820		if (status != CAM_REQ_CMP) {
821			mps_dprint(sc, MPS_ERROR,
822			    "Error %#x registering async handler for "
823			    "AC_ADVINFO_CHANGED events\n", status);
824			xpt_free_path(sassc->path);
825			sassc->path = NULL;
826		}
827	}
828	if (status != CAM_REQ_CMP) {
829		/*
830		 * EEDP use is the exception, not the rule.
831		 * Warn the user, but do not fail to attach.
832		 */
833		mps_printf(sc, "EEDP capabilities disabled.\n");
834	}
835
836	mps_unlock(sc);
837
838	mpssas_register_events(sc);
839out:
840	if (error)
841		mps_detach_sas(sc);
842
843	mps_dprint(sc, MPS_INIT, "%s exit error= %d\n", __func__, error);
844	return (error);
845}
846
847int
848mps_detach_sas(struct mps_softc *sc)
849{
850	struct mpssas_softc *sassc;
851	struct mpssas_lun *lun, *lun_tmp;
852	struct mpssas_target *targ;
853	int i;
854
855	MPS_FUNCTRACE(sc);
856
857	if (sc->sassc == NULL)
858		return (0);
859
860	sassc = sc->sassc;
861	mps_deregister_events(sc, sassc->mpssas_eh);
862
863	/*
864	 * Drain and free the event handling taskqueue with the lock
865	 * unheld so that any parallel processing tasks drain properly
866	 * without deadlocking.
867	 */
868	if (sassc->ev_tq != NULL)
869		taskqueue_free(sassc->ev_tq);
870
871	/* Make sure CAM doesn't wedge if we had to bail out early. */
872	mps_lock(sc);
873
874	/* Deregister our async handler */
875	if (sassc->path != NULL) {
876		xpt_register_async(0, mpssas_async, sc, sassc->path);
877		xpt_free_path(sassc->path);
878		sassc->path = NULL;
879	}
880
881	if (sassc->flags & MPSSAS_IN_STARTUP)
882		xpt_release_simq(sassc->sim, 1);
883
884	if (sassc->sim != NULL) {
885		xpt_bus_deregister(cam_sim_path(sassc->sim));
886		cam_sim_free(sassc->sim, FALSE);
887	}
888
889	mps_unlock(sc);
890
891	if (sassc->devq != NULL)
892		cam_simq_free(sassc->devq);
893
894	for(i=0; i< sassc->maxtargets ;i++) {
895		targ = &sassc->targets[i];
896		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
897			free(lun, M_MPT2);
898		}
899	}
900	free(sassc->targets, M_MPT2);
901	free(sassc, M_MPT2);
902	sc->sassc = NULL;
903
904	return (0);
905}
906
907void
908mpssas_discovery_end(struct mpssas_softc *sassc)
909{
910	struct mps_softc *sc = sassc->sc;
911
912	MPS_FUNCTRACE(sc);
913
914	if (sassc->flags & MPSSAS_DISCOVERY_TIMEOUT_PENDING)
915		callout_stop(&sassc->discovery_callout);
916
917	/*
918	 * After discovery has completed, check the mapping table for any
919	 * missing devices and update their missing counts. Only do this once
920	 * whenever the driver is initialized so that missing counts aren't
921	 * updated unnecessarily. Note that just because discovery has
922	 * completed doesn't mean that events have been processed yet. The
923	 * check_devices function is a callout timer that checks if ALL devices
924	 * are missing. If so, it will wait a little longer for events to
925	 * complete and keep resetting itself until some device in the mapping
926	 * table is not missing, meaning that event processing has started.
927	 */
928	if (sc->track_mapping_events) {
929		mps_dprint(sc, MPS_XINFO | MPS_MAPPING, "Discovery has "
930		    "completed. Check for missing devices in the mapping "
931		    "table.\n");
932		callout_reset(&sc->device_check_callout,
933		    MPS_MISSING_CHECK_DELAY * hz, mps_mapping_check_devices,
934		    sc);
935	}
936}
937
938static void
939mpssas_action(struct cam_sim *sim, union ccb *ccb)
940{
941	struct mpssas_softc *sassc;
942
943	sassc = cam_sim_softc(sim);
944
945	MPS_FUNCTRACE(sassc->sc);
946	mps_dprint(sassc->sc, MPS_TRACE, "ccb func_code 0x%x\n",
947	    ccb->ccb_h.func_code);
948	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
949
950	switch (ccb->ccb_h.func_code) {
951	case XPT_PATH_INQ:
952	{
953		struct ccb_pathinq *cpi = &ccb->cpi;
954		struct mps_softc *sc = sassc->sc;
955		uint8_t sges_per_frame;
956
957		cpi->version_num = 1;
958		cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
959		cpi->target_sprt = 0;
960#if __FreeBSD_version >= 1000039
961		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN;
962#else
963		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED;
964#endif
965		cpi->hba_eng_cnt = 0;
966		cpi->max_target = sassc->maxtargets - 1;
967		cpi->max_lun = 255;
968
969		/*
970		 * initiator_id is set here to an ID outside the set of valid
971		 * target IDs (including volumes).
972		 */
973		cpi->initiator_id = sassc->maxtargets;
974		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
975		strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN);
976		strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
977		cpi->unit_number = cam_sim_unit(sim);
978		cpi->bus_id = cam_sim_bus(sim);
979		cpi->base_transfer_speed = 150000;
980		cpi->transport = XPORT_SAS;
981		cpi->transport_version = 0;
982		cpi->protocol = PROTO_SCSI;
983		cpi->protocol_version = SCSI_REV_SPC;
984
985		/*
986		 * Max IO Size is Page Size * the following:
987		 * ((SGEs per frame - 1 for chain element) *
988		 * Max Chain Depth) + 1 for no chain needed in last frame
989		 *
990		 * If user suggests a Max IO size to use, use the smaller of the
991		 * user's value and the calculated value as long as the user's
992		 * value is larger than 0. The user's value is in pages.
993		 */
994		sges_per_frame = ((sc->facts->IOCRequestFrameSize * 4) /
995		    sizeof(MPI2_SGE_SIMPLE64)) - 1;
996		cpi->maxio = (sges_per_frame * sc->facts->MaxChainDepth) + 1;
997		cpi->maxio *= PAGE_SIZE;
998		if ((sc->max_io_pages > 0) && (sc->max_io_pages * PAGE_SIZE <
999		    cpi->maxio))
1000			cpi->maxio = sc->max_io_pages * PAGE_SIZE;
1001		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1002		break;
1003	}
1004	case XPT_GET_TRAN_SETTINGS:
1005	{
1006		struct ccb_trans_settings	*cts;
1007		struct ccb_trans_settings_sas	*sas;
1008		struct ccb_trans_settings_scsi	*scsi;
1009		struct mpssas_target *targ;
1010
1011		cts = &ccb->cts;
1012		sas = &cts->xport_specific.sas;
1013		scsi = &cts->proto_specific.scsi;
1014
1015		KASSERT(cts->ccb_h.target_id < sassc->maxtargets,
1016		    ("Target %d out of bounds in XPT_GET_TRANS_SETTINGS\n",
1017		    cts->ccb_h.target_id));
1018		targ = &sassc->targets[cts->ccb_h.target_id];
1019		if (targ->handle == 0x0) {
1020			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1021			break;
1022		}
1023
1024		cts->protocol_version = SCSI_REV_SPC2;
1025		cts->transport = XPORT_SAS;
1026		cts->transport_version = 0;
1027
1028		sas->valid = CTS_SAS_VALID_SPEED;
1029		switch (targ->linkrate) {
1030		case 0x08:
1031			sas->bitrate = 150000;
1032			break;
1033		case 0x09:
1034			sas->bitrate = 300000;
1035			break;
1036		case 0x0a:
1037			sas->bitrate = 600000;
1038			break;
1039		default:
1040			sas->valid = 0;
1041		}
1042
1043		cts->protocol = PROTO_SCSI;
1044		scsi->valid = CTS_SCSI_VALID_TQ;
1045		scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
1046
1047		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1048		break;
1049	}
1050	case XPT_CALC_GEOMETRY:
1051		cam_calc_geometry(&ccb->ccg, /*extended*/1);
1052		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1053		break;
1054	case XPT_RESET_DEV:
1055		mps_dprint(sassc->sc, MPS_XINFO, "mpssas_action XPT_RESET_DEV\n");
1056		mpssas_action_resetdev(sassc, ccb);
1057		return;
1058	case XPT_RESET_BUS:
1059	case XPT_ABORT:
1060	case XPT_TERM_IO:
1061		mps_dprint(sassc->sc, MPS_XINFO,
1062		    "mpssas_action faking success for abort or reset\n");
1063		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1064		break;
1065	case XPT_SCSI_IO:
1066		mpssas_action_scsiio(sassc, ccb);
1067		return;
1068#if __FreeBSD_version >= 900026
1069	case XPT_SMP_IO:
1070		mpssas_action_smpio(sassc, ccb);
1071		return;
1072#endif
1073	default:
1074		mpssas_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL);
1075		break;
1076	}
1077	xpt_done(ccb);
1078
1079}
1080
1081static void
1082mpssas_announce_reset(struct mps_softc *sc, uint32_t ac_code,
1083    target_id_t target_id, lun_id_t lun_id)
1084{
1085	path_id_t path_id = cam_sim_path(sc->sassc->sim);
1086	struct cam_path *path;
1087
1088	mps_dprint(sc, MPS_XINFO, "%s code %x target %d lun %jx\n", __func__,
1089	    ac_code, target_id, (uintmax_t)lun_id);
1090
1091	if (xpt_create_path(&path, NULL,
1092		path_id, target_id, lun_id) != CAM_REQ_CMP) {
1093		mps_dprint(sc, MPS_ERROR, "unable to create path for reset "
1094			   "notification\n");
1095		return;
1096	}
1097
1098	xpt_async(ac_code, path, NULL);
1099	xpt_free_path(path);
1100}
1101
1102static void
1103mpssas_complete_all_commands(struct mps_softc *sc)
1104{
1105	struct mps_command *cm;
1106	int i;
1107	int completed;
1108
1109	MPS_FUNCTRACE(sc);
1110	mtx_assert(&sc->mps_mtx, MA_OWNED);
1111
1112	/* complete all commands with a NULL reply */
1113	for (i = 1; i < sc->num_reqs; i++) {
1114		cm = &sc->commands[i];
1115		cm->cm_reply = NULL;
1116		completed = 0;
1117
1118		if (cm->cm_flags & MPS_CM_FLAGS_POLLED)
1119			cm->cm_flags |= MPS_CM_FLAGS_COMPLETE;
1120
1121		if (cm->cm_complete != NULL) {
1122			mpssas_log_command(cm, MPS_RECOVERY,
1123			    "completing cm %p state %x ccb %p for diag reset\n",
1124			    cm, cm->cm_state, cm->cm_ccb);
1125
1126			cm->cm_complete(sc, cm);
1127			completed = 1;
1128		}
1129
1130		if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {
1131			mpssas_log_command(cm, MPS_RECOVERY,
1132			    "waking up cm %p state %x ccb %p for diag reset\n",
1133			    cm, cm->cm_state, cm->cm_ccb);
1134			wakeup(cm);
1135			completed = 1;
1136		}
1137
1138		if (cm->cm_sc->io_cmds_active != 0)
1139			cm->cm_sc->io_cmds_active--;
1140
1141		if ((completed == 0) && (cm->cm_state != MPS_CM_STATE_FREE)) {
1142			/* this should never happen, but if it does, log */
1143			mpssas_log_command(cm, MPS_RECOVERY,
1144			    "cm %p state %x flags 0x%x ccb %p during diag "
1145			    "reset\n", cm, cm->cm_state, cm->cm_flags,
1146			    cm->cm_ccb);
1147		}
1148	}
1149}
1150
1151void
1152mpssas_handle_reinit(struct mps_softc *sc)
1153{
1154	int i;
1155
1156	/* Go back into startup mode and freeze the simq, so that CAM
1157	 * doesn't send any commands until after we've rediscovered all
1158	 * targets and found the proper device handles for them.
1159	 *
1160	 * After the reset, portenable will trigger discovery, and after all
1161	 * discovery-related activities have finished, the simq will be
1162	 * released.
1163	 */
1164	mps_dprint(sc, MPS_INIT, "%s startup\n", __func__);
1165	sc->sassc->flags |= MPSSAS_IN_STARTUP;
1166	sc->sassc->flags |= MPSSAS_IN_DISCOVERY;
1167	mpssas_startup_increment(sc->sassc);
1168
1169	/* notify CAM of a bus reset */
1170	mpssas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
1171	    CAM_LUN_WILDCARD);
1172
1173	/* complete and cleanup after all outstanding commands */
1174	mpssas_complete_all_commands(sc);
1175
1176	mps_dprint(sc, MPS_INIT,
1177	    "%s startup %u after command completion\n", __func__,
1178	    sc->sassc->startup_refcount);
1179
1180	/* zero all the target handles, since they may change after the
1181	 * reset, and we have to rediscover all the targets and use the new
1182	 * handles.
1183	 */
1184	for (i = 0; i < sc->sassc->maxtargets; i++) {
1185		if (sc->sassc->targets[i].outstanding != 0)
1186			mps_dprint(sc, MPS_INIT, "target %u outstanding %u\n",
1187			    i, sc->sassc->targets[i].outstanding);
1188		sc->sassc->targets[i].handle = 0x0;
1189		sc->sassc->targets[i].exp_dev_handle = 0x0;
1190		sc->sassc->targets[i].outstanding = 0;
1191		sc->sassc->targets[i].flags = MPSSAS_TARGET_INDIAGRESET;
1192	}
1193}
1194
1195static void
1196mpssas_tm_timeout(void *data)
1197{
1198	struct mps_command *tm = data;
1199	struct mps_softc *sc = tm->cm_sc;
1200
1201	mtx_assert(&sc->mps_mtx, MA_OWNED);
1202
1203	mpssas_log_command(tm, MPS_INFO|MPS_RECOVERY,
1204	    "task mgmt %p timed out\n", tm);
1205	mps_reinit(sc);
1206}
1207
1208static void
1209mpssas_logical_unit_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1210{
1211	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1212	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1213	unsigned int cm_count = 0;
1214	struct mps_command *cm;
1215	struct mpssas_target *targ;
1216
1217	callout_stop(&tm->cm_callout);
1218
1219	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1220	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1221	targ = tm->cm_targ;
1222
1223	/*
1224	 * Currently there should be no way we can hit this case.  It only
1225	 * happens when we have a failure to allocate chain frames, and
1226	 * task management commands don't have S/G lists.
1227	 * XXXSL So should it be an assertion?
1228	 */
1229	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1230		mps_dprint(sc, MPS_RECOVERY|MPS_ERROR,
1231		    "%s: cm_flags = %#x for LUN reset! "
1232		   "This should not happen!\n", __func__, tm->cm_flags);
1233		mpssas_free_tm(sc, tm);
1234		return;
1235	}
1236
1237	if (reply == NULL) {
1238		mps_dprint(sc, MPS_RECOVERY, "NULL reset reply for tm %p\n",
1239		    tm);
1240		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1241			/* this completion was due to a reset, just cleanup */
1242			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1243			    "reset, ignoring NULL LUN reset reply\n");
1244			targ->tm = NULL;
1245			mpssas_free_tm(sc, tm);
1246		}
1247		else {
1248			/* we should have gotten a reply. */
1249			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1250			    "LUN reset attempt, resetting controller\n");
1251			mps_reinit(sc);
1252		}
1253		return;
1254	}
1255
1256	mps_dprint(sc, MPS_RECOVERY,
1257	    "logical unit reset status 0x%x code 0x%x count %u\n",
1258	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1259	    le32toh(reply->TerminationCount));
1260
1261	/*
1262	 * See if there are any outstanding commands for this LUN.
1263	 * This could be made more efficient by using a per-LU data
1264	 * structure of some sort.
1265	 */
1266	TAILQ_FOREACH(cm, &targ->commands, cm_link) {
1267		if (cm->cm_lun == tm->cm_lun)
1268			cm_count++;
1269	}
1270
1271	if (cm_count == 0) {
1272		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1273		    "Finished recovery after LUN reset for target %u\n",
1274		    targ->tid);
1275
1276		mpssas_announce_reset(sc, AC_SENT_BDR, targ->tid, tm->cm_lun);
1277
1278		/*
1279		 * We've finished recovery for this logical unit.  check and
1280		 * see if some other logical unit has a timedout command
1281		 * that needs to be processed.
1282		 */
1283		cm = TAILQ_FIRST(&targ->timedout_commands);
1284		if (cm) {
1285			mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1286			    "More commands to abort for target %u\n",
1287			    targ->tid);
1288			mpssas_send_abort(sc, tm, cm);
1289		} else {
1290			targ->tm = NULL;
1291			mpssas_free_tm(sc, tm);
1292		}
1293	} else {
1294		/*
1295		 * If we still have commands for this LUN, the reset
1296		 * effectively failed, regardless of the status reported.
1297		 * Escalate to a target reset.
1298		 */
1299		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1300		    "logical unit reset complete for target %u, but still "
1301		    "have %u command(s), sending target reset\n", targ->tid,
1302		    cm_count);
1303		mpssas_send_reset(sc, tm,
1304		    MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
1305	}
1306}
1307
1308static void
1309mpssas_target_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1310{
1311	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1312	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1313	struct mpssas_target *targ;
1314
1315	callout_stop(&tm->cm_callout);
1316
1317	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1318	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1319	targ = tm->cm_targ;
1320
1321	/*
1322	 * Currently there should be no way we can hit this case.  It only
1323	 * happens when we have a failure to allocate chain frames, and
1324	 * task management commands don't have S/G lists.
1325	 */
1326	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1327		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x for target reset! "
1328			   "This should not happen!\n", __func__, tm->cm_flags);
1329		mpssas_free_tm(sc, tm);
1330		return;
1331	}
1332
1333	if (reply == NULL) {
1334		mps_dprint(sc, MPS_RECOVERY,
1335		    "NULL target reset reply for tm %pi TaskMID %u\n",
1336		    tm, le16toh(req->TaskMID));
1337		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1338			/* this completion was due to a reset, just cleanup */
1339			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1340			    "reset, ignoring NULL target reset reply\n");
1341			targ->tm = NULL;
1342			mpssas_free_tm(sc, tm);
1343		} else {
1344			/* we should have gotten a reply. */
1345			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1346			    "target reset attempt, resetting controller\n");
1347			mps_reinit(sc);
1348		}
1349		return;
1350	}
1351
1352	mps_dprint(sc, MPS_RECOVERY,
1353	    "target reset status 0x%x code 0x%x count %u\n",
1354	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1355	    le32toh(reply->TerminationCount));
1356
1357	if (targ->outstanding == 0) {
1358		/* we've finished recovery for this target and all
1359		 * of its logical units.
1360		 */
1361		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1362		    "Finished reset recovery for target %u\n", targ->tid);
1363
1364		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1365		    CAM_LUN_WILDCARD);
1366
1367		targ->tm = NULL;
1368		mpssas_free_tm(sc, tm);
1369	} else {
1370		/*
1371		 * After a target reset, if this target still has
1372		 * outstanding commands, the reset effectively failed,
1373		 * regardless of the status reported.  escalate.
1374		 */
1375		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1376		    "Target reset complete for target %u, but still have %u "
1377		    "command(s), resetting controller\n", targ->tid,
1378		    targ->outstanding);
1379		mps_reinit(sc);
1380	}
1381}
1382
1383#define MPS_RESET_TIMEOUT 30
1384
1385int
1386mpssas_send_reset(struct mps_softc *sc, struct mps_command *tm, uint8_t type)
1387{
1388	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1389	struct mpssas_target *target;
1390	int err;
1391
1392	target = tm->cm_targ;
1393	if (target->handle == 0) {
1394		mps_dprint(sc, MPS_ERROR,"%s null devhandle for target_id %d\n",
1395		    __func__, target->tid);
1396		return -1;
1397	}
1398
1399	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1400	req->DevHandle = htole16(target->handle);
1401	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1402	req->TaskType = type;
1403
1404	if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) {
1405		/* XXX Need to handle invalid LUNs */
1406		MPS_SET_LUN(req->LUN, tm->cm_lun);
1407		tm->cm_targ->logical_unit_resets++;
1408		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1409		    "Sending logical unit reset to target %u lun %d\n",
1410		    target->tid, tm->cm_lun);
1411		tm->cm_complete = mpssas_logical_unit_reset_complete;
1412		mpssas_prepare_for_tm(sc, tm, target, tm->cm_lun);
1413	} else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) {
1414		/*
1415		 * Target reset method =
1416		 * 	SAS Hard Link Reset / SATA Link Reset
1417		 */
1418		req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
1419		tm->cm_targ->target_resets++;
1420		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1421		    "Sending target reset to target %u\n", target->tid);
1422		tm->cm_complete = mpssas_target_reset_complete;
1423		mpssas_prepare_for_tm(sc, tm, target, CAM_LUN_WILDCARD);
1424	} else {
1425		mps_dprint(sc, MPS_ERROR, "unexpected reset type 0x%x\n", type);
1426		return -1;
1427	}
1428
1429	tm->cm_data = NULL;
1430	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1431	tm->cm_complete_data = (void *)tm;
1432
1433	callout_reset(&tm->cm_callout, MPS_RESET_TIMEOUT * hz,
1434	    mpssas_tm_timeout, tm);
1435
1436	err = mps_map_command(sc, tm);
1437	if (err)
1438		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1439		    "error %d sending reset type %u\n",
1440		    err, type);
1441
1442	return err;
1443}
1444
1445
1446static void
1447mpssas_abort_complete(struct mps_softc *sc, struct mps_command *tm)
1448{
1449	struct mps_command *cm;
1450	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1451	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1452	struct mpssas_target *targ;
1453
1454	callout_stop(&tm->cm_callout);
1455
1456	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1457	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1458	targ = tm->cm_targ;
1459
1460	/*
1461	 * Currently there should be no way we can hit this case.  It only
1462	 * happens when we have a failure to allocate chain frames, and
1463	 * task management commands don't have S/G lists.
1464	 */
1465	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1466		mps_dprint(sc, MPS_RECOVERY,
1467		    "cm_flags = %#x for abort %p TaskMID %u!\n",
1468		    tm->cm_flags, tm, le16toh(req->TaskMID));
1469		mpssas_free_tm(sc, tm);
1470		return;
1471	}
1472
1473	if (reply == NULL) {
1474		mps_dprint(sc, MPS_RECOVERY,
1475		    "NULL abort reply for tm %p TaskMID %u\n",
1476		    tm, le16toh(req->TaskMID));
1477		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1478			/* this completion was due to a reset, just cleanup */
1479			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1480			    "reset, ignoring NULL abort reply\n");
1481			targ->tm = NULL;
1482			mpssas_free_tm(sc, tm);
1483		} else {
1484			/* we should have gotten a reply. */
1485			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1486			    "abort attempt, resetting controller\n");
1487			mps_reinit(sc);
1488		}
1489		return;
1490	}
1491
1492	mps_dprint(sc, MPS_RECOVERY,
1493	    "abort TaskMID %u status 0x%x code 0x%x count %u\n",
1494	    le16toh(req->TaskMID),
1495	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1496	    le32toh(reply->TerminationCount));
1497
1498	cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands);
1499	if (cm == NULL) {
1500		/*
1501		 * If there are no more timedout commands, we're done with
1502		 * error recovery for this target.
1503		 */
1504		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1505		    "Finished abort recovery for target %u\n", targ->tid);
1506
1507		targ->tm = NULL;
1508		mpssas_free_tm(sc, tm);
1509	} else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) {
1510		/* abort success, but we have more timedout commands to abort */
1511		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1512		    "Continuing abort recovery for target %u\n", targ->tid);
1513
1514		mpssas_send_abort(sc, tm, cm);
1515	} else {
1516		/* we didn't get a command completion, so the abort
1517		 * failed as far as we're concerned.  escalate.
1518		 */
1519		mps_dprint(sc, MPS_RECOVERY,
1520		    "Abort failed for target %u, sending logical unit reset\n",
1521		    targ->tid);
1522
1523		mpssas_send_reset(sc, tm,
1524		    MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET);
1525	}
1526}
1527
1528#define MPS_ABORT_TIMEOUT 5
1529
1530static int
1531mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm, struct mps_command *cm)
1532{
1533	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1534	struct mpssas_target *targ;
1535	int err;
1536
1537	targ = cm->cm_targ;
1538	if (targ->handle == 0) {
1539		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1540		    "%s null devhandle for target_id %d\n",
1541		    __func__, cm->cm_ccb->ccb_h.target_id);
1542		return -1;
1543	}
1544
1545	mpssas_log_command(cm, MPS_RECOVERY|MPS_INFO,
1546	    "Aborting command %p\n", cm);
1547
1548	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1549	req->DevHandle = htole16(targ->handle);
1550	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1551	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK;
1552
1553	/* XXX Need to handle invalid LUNs */
1554	MPS_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun);
1555
1556	req->TaskMID = htole16(cm->cm_desc.Default.SMID);
1557
1558	tm->cm_data = NULL;
1559	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1560	tm->cm_complete = mpssas_abort_complete;
1561	tm->cm_complete_data = (void *)tm;
1562	tm->cm_targ = cm->cm_targ;
1563	tm->cm_lun = cm->cm_lun;
1564
1565	callout_reset(&tm->cm_callout, MPS_ABORT_TIMEOUT * hz,
1566	    mpssas_tm_timeout, tm);
1567
1568	targ->aborts++;
1569
1570	mpssas_prepare_for_tm(sc, tm, targ, tm->cm_lun);
1571
1572	err = mps_map_command(sc, tm);
1573	if (err)
1574		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1575		    "error %d sending abort for cm %p SMID %u\n",
1576		    err, cm, req->TaskMID);
1577	return err;
1578}
1579
1580static void
1581mpssas_scsiio_timeout(void *data)
1582{
1583	sbintime_t elapsed, now;
1584	union ccb *ccb;
1585	struct mps_softc *sc;
1586	struct mps_command *cm;
1587	struct mpssas_target *targ;
1588
1589	cm = (struct mps_command *)data;
1590	sc = cm->cm_sc;
1591	ccb = cm->cm_ccb;
1592	now = sbinuptime();
1593
1594	MPS_FUNCTRACE(sc);
1595	mtx_assert(&sc->mps_mtx, MA_OWNED);
1596
1597	mps_dprint(sc, MPS_XINFO|MPS_RECOVERY, "Timeout checking cm %p\n", sc);
1598
1599	/*
1600	 * Run the interrupt handler to make sure it's not pending.  This
1601	 * isn't perfect because the command could have already completed
1602	 * and been re-used, though this is unlikely.
1603	 */
1604	mps_intr_locked(sc);
1605	if (cm->cm_state == MPS_CM_STATE_FREE) {
1606		mpssas_log_command(cm, MPS_XINFO,
1607		    "SCSI command %p almost timed out\n", cm);
1608		return;
1609	}
1610
1611	if (cm->cm_ccb == NULL) {
1612		mps_dprint(sc, MPS_ERROR, "command timeout with NULL ccb\n");
1613		return;
1614	}
1615
1616	targ = cm->cm_targ;
1617	targ->timeouts++;
1618
1619	elapsed = now - ccb->ccb_h.qos.sim_data;
1620	mpssas_log_command(cm, MPS_INFO|MPS_RECOVERY,
1621	    "Command timeout on target %u(0x%04x) %d set, %d.%d elapsed\n",
1622	    targ->tid, targ->handle, ccb->ccb_h.timeout,
1623	    sbintime_getsec(elapsed), elapsed & 0xffffffff);
1624
1625	/* XXX first, check the firmware state, to see if it's still
1626	 * operational.  if not, do a diag reset.
1627	 */
1628	mpssas_set_ccbstatus(cm->cm_ccb, CAM_CMD_TIMEOUT);
1629	cm->cm_state = MPS_CM_STATE_TIMEDOUT;
1630	TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery);
1631
1632	if (targ->tm != NULL) {
1633		/* target already in recovery, just queue up another
1634		 * timedout command to be processed later.
1635		 */
1636		mps_dprint(sc, MPS_RECOVERY,
1637		    "queued timedout cm %p for processing by tm %p\n",
1638		    cm, targ->tm);
1639	} else if ((targ->tm = mpssas_alloc_tm(sc)) != NULL) {
1640		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1641		    "Sending abort to target %u for SMID %d\n", targ->tid,
1642		    cm->cm_desc.Default.SMID);
1643		mps_dprint(sc, MPS_RECOVERY, "timedout cm %p allocated tm %p\n",
1644		    cm, targ->tm);
1645
1646		/* start recovery by aborting the first timedout command */
1647		mpssas_send_abort(sc, targ->tm, cm);
1648	} else {
1649		/* XXX queue this target up for recovery once a TM becomes
1650		 * available.  The firmware only has a limited number of
1651		 * HighPriority credits for the high priority requests used
1652		 * for task management, and we ran out.
1653		 *
1654		 * Isilon: don't worry about this for now, since we have
1655		 * more credits than disks in an enclosure, and limit
1656		 * ourselves to one TM per target for recovery.
1657		 */
1658		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1659		    "timedout cm %p failed to allocate a tm\n", cm);
1660	}
1661
1662}
1663
1664static void
1665mpssas_action_scsiio(struct mpssas_softc *sassc, union ccb *ccb)
1666{
1667	MPI2_SCSI_IO_REQUEST *req;
1668	struct ccb_scsiio *csio;
1669	struct mps_softc *sc;
1670	struct mpssas_target *targ;
1671	struct mpssas_lun *lun;
1672	struct mps_command *cm;
1673	uint8_t i, lba_byte, *ref_tag_addr;
1674	uint16_t eedp_flags;
1675	uint32_t mpi_control;
1676
1677	sc = sassc->sc;
1678	MPS_FUNCTRACE(sc);
1679	mtx_assert(&sc->mps_mtx, MA_OWNED);
1680
1681	csio = &ccb->csio;
1682	KASSERT(csio->ccb_h.target_id < sassc->maxtargets,
1683	    ("Target %d out of bounds in XPT_SCSI_IO\n",
1684	     csio->ccb_h.target_id));
1685	targ = &sassc->targets[csio->ccb_h.target_id];
1686	mps_dprint(sc, MPS_TRACE, "ccb %p target flag %x\n", ccb, targ->flags);
1687	if (targ->handle == 0x0) {
1688		mps_dprint(sc, MPS_ERROR, "%s NULL handle for target %u\n",
1689		    __func__, csio->ccb_h.target_id);
1690		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1691		xpt_done(ccb);
1692		return;
1693	}
1694	if (targ->flags & MPS_TARGET_FLAGS_RAID_COMPONENT) {
1695		mps_dprint(sc, MPS_ERROR, "%s Raid component no SCSI IO "
1696		    "supported %u\n", __func__, csio->ccb_h.target_id);
1697		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1698		xpt_done(ccb);
1699		return;
1700	}
1701	/*
1702	 * Sometimes, it is possible to get a command that is not "In
1703	 * Progress" and was actually aborted by the upper layer.  Check for
1704	 * this here and complete the command without error.
1705	 */
1706	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_INPROG) {
1707		mps_dprint(sc, MPS_TRACE, "%s Command is not in progress for "
1708		    "target %u\n", __func__, csio->ccb_h.target_id);
1709		xpt_done(ccb);
1710		return;
1711	}
1712	/*
1713	 * If devinfo is 0 this will be a volume.  In that case don't tell CAM
1714	 * that the volume has timed out.  We want volumes to be enumerated
1715	 * until they are deleted/removed, not just failed.
1716	 */
1717	if (targ->flags & MPSSAS_TARGET_INREMOVAL) {
1718		if (targ->devinfo == 0)
1719			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1720		else
1721			mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
1722		xpt_done(ccb);
1723		return;
1724	}
1725
1726	if ((sc->mps_flags & MPS_FLAGS_SHUTDOWN) != 0) {
1727		mps_dprint(sc, MPS_INFO, "%s shutting down\n", __func__);
1728		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1729		xpt_done(ccb);
1730		return;
1731	}
1732
1733	/*
1734	 * If target has a reset in progress, freeze the devq and return.  The
1735	 * devq will be released when the TM reset is finished.
1736	 */
1737	if (targ->flags & MPSSAS_TARGET_INRESET) {
1738		ccb->ccb_h.status = CAM_BUSY | CAM_DEV_QFRZN;
1739		mps_dprint(sc, MPS_INFO, "%s: Freezing devq for target ID %d\n",
1740		    __func__, targ->tid);
1741		xpt_freeze_devq(ccb->ccb_h.path, 1);
1742		xpt_done(ccb);
1743		return;
1744	}
1745
1746	cm = mps_alloc_command(sc);
1747	if (cm == NULL || (sc->mps_flags & MPS_FLAGS_DIAGRESET)) {
1748		if (cm != NULL) {
1749			mps_free_command(sc, cm);
1750		}
1751		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
1752			xpt_freeze_simq(sassc->sim, 1);
1753			sassc->flags |= MPSSAS_QUEUE_FROZEN;
1754		}
1755		ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1756		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1757		xpt_done(ccb);
1758		return;
1759	}
1760
1761	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
1762	bzero(req, sizeof(*req));
1763	req->DevHandle = htole16(targ->handle);
1764	req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
1765	req->MsgFlags = 0;
1766	req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);
1767	req->SenseBufferLength = MPS_SENSE_LEN;
1768	req->SGLFlags = 0;
1769	req->ChainOffset = 0;
1770	req->SGLOffset0 = 24;	/* 32bit word offset to the SGL */
1771	req->SGLOffset1= 0;
1772	req->SGLOffset2= 0;
1773	req->SGLOffset3= 0;
1774	req->SkipCount = 0;
1775	req->DataLength = htole32(csio->dxfer_len);
1776	req->BidirectionalDataLength = 0;
1777	req->IoFlags = htole16(csio->cdb_len);
1778	req->EEDPFlags = 0;
1779
1780	/* Note: BiDirectional transfers are not supported */
1781	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1782	case CAM_DIR_IN:
1783		mpi_control = MPI2_SCSIIO_CONTROL_READ;
1784		cm->cm_flags |= MPS_CM_FLAGS_DATAIN;
1785		break;
1786	case CAM_DIR_OUT:
1787		mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
1788		cm->cm_flags |= MPS_CM_FLAGS_DATAOUT;
1789		break;
1790	case CAM_DIR_NONE:
1791	default:
1792		mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
1793		break;
1794	}
1795
1796	if (csio->cdb_len == 32)
1797                mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
1798	/*
1799	 * It looks like the hardware doesn't require an explicit tag
1800	 * number for each transaction.  SAM Task Management not supported
1801	 * at the moment.
1802	 */
1803	switch (csio->tag_action) {
1804	case MSG_HEAD_OF_Q_TAG:
1805		mpi_control |= MPI2_SCSIIO_CONTROL_HEADOFQ;
1806		break;
1807	case MSG_ORDERED_Q_TAG:
1808		mpi_control |= MPI2_SCSIIO_CONTROL_ORDEREDQ;
1809		break;
1810	case MSG_ACA_TASK:
1811		mpi_control |= MPI2_SCSIIO_CONTROL_ACAQ;
1812		break;
1813	case CAM_TAG_ACTION_NONE:
1814	case MSG_SIMPLE_Q_TAG:
1815	default:
1816		mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
1817		break;
1818	}
1819	mpi_control |= sc->mapping_table[csio->ccb_h.target_id].TLR_bits;
1820	req->Control = htole32(mpi_control);
1821	if (MPS_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) {
1822		mps_free_command(sc, cm);
1823		mpssas_set_ccbstatus(ccb, CAM_LUN_INVALID);
1824		xpt_done(ccb);
1825		return;
1826	}
1827
1828	if (csio->ccb_h.flags & CAM_CDB_POINTER)
1829		bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
1830	else
1831		bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
1832	req->IoFlags = htole16(csio->cdb_len);
1833
1834	/*
1835	 * Check if EEDP is supported and enabled.  If it is then check if the
1836	 * SCSI opcode could be using EEDP.  If so, make sure the LUN exists and
1837	 * is formatted for EEDP support.  If all of this is true, set CDB up
1838	 * for EEDP transfer.
1839	 */
1840	eedp_flags = op_code_prot[req->CDB.CDB32[0]];
1841	if (sc->eedp_enabled && eedp_flags) {
1842		SLIST_FOREACH(lun, &targ->luns, lun_link) {
1843			if (lun->lun_id == csio->ccb_h.target_lun) {
1844				break;
1845			}
1846		}
1847
1848		if ((lun != NULL) && (lun->eedp_formatted)) {
1849			req->EEDPBlockSize = htole16(lun->eedp_block_size);
1850			eedp_flags |= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
1851			    MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
1852			    MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
1853			req->EEDPFlags = htole16(eedp_flags);
1854
1855			/*
1856			 * If CDB less than 32, fill in Primary Ref Tag with
1857			 * low 4 bytes of LBA.  If CDB is 32, tag stuff is
1858			 * already there.  Also, set protection bit.  FreeBSD
1859			 * currently does not support CDBs bigger than 16, but
1860			 * the code doesn't hurt, and will be here for the
1861			 * future.
1862			 */
1863			if (csio->cdb_len != 32) {
1864				lba_byte = (csio->cdb_len == 16) ? 6 : 2;
1865				ref_tag_addr = (uint8_t *)&req->CDB.EEDP32.
1866				    PrimaryReferenceTag;
1867				for (i = 0; i < 4; i++) {
1868					*ref_tag_addr =
1869					    req->CDB.CDB32[lba_byte + i];
1870					ref_tag_addr++;
1871				}
1872				req->CDB.EEDP32.PrimaryReferenceTag =
1873					htole32(req->CDB.EEDP32.PrimaryReferenceTag);
1874				req->CDB.EEDP32.PrimaryApplicationTagMask =
1875				    0xFFFF;
1876				req->CDB.CDB32[1] = (req->CDB.CDB32[1] & 0x1F) |
1877				    0x20;
1878			} else {
1879				eedp_flags |=
1880				    MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG;
1881				req->EEDPFlags = htole16(eedp_flags);
1882				req->CDB.CDB32[10] = (req->CDB.CDB32[10] &
1883				    0x1F) | 0x20;
1884			}
1885		}
1886	}
1887
1888	cm->cm_length = csio->dxfer_len;
1889	if (cm->cm_length != 0) {
1890		cm->cm_data = ccb;
1891		cm->cm_flags |= MPS_CM_FLAGS_USE_CCB;
1892	} else {
1893		cm->cm_data = NULL;
1894	}
1895	cm->cm_sge = &req->SGL;
1896	cm->cm_sglsize = (32 - 24) * 4;
1897	cm->cm_desc.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
1898	cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
1899	cm->cm_complete = mpssas_scsiio_complete;
1900	cm->cm_complete_data = ccb;
1901	cm->cm_targ = targ;
1902	cm->cm_lun = csio->ccb_h.target_lun;
1903	cm->cm_ccb = ccb;
1904
1905	/*
1906	 * If HBA is a WD and the command is not for a retry, try to build a
1907	 * direct I/O message. If failed, or the command is for a retry, send
1908	 * the I/O to the IR volume itself.
1909	 */
1910	if (sc->WD_valid_config) {
1911		if (ccb->ccb_h.sim_priv.entries[0].field == MPS_WD_RETRY) {
1912			mpssas_direct_drive_io(sassc, cm, ccb);
1913		} else {
1914			mpssas_set_ccbstatus(ccb, CAM_REQ_INPROG);
1915		}
1916	}
1917
1918#if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
1919	if (csio->bio != NULL)
1920		biotrack(csio->bio, __func__);
1921#endif
1922	csio->ccb_h.qos.sim_data = sbinuptime();
1923	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
1924	    mpssas_scsiio_timeout, cm, 0);
1925
1926	targ->issued++;
1927	targ->outstanding++;
1928	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
1929	ccb->ccb_h.status |= CAM_SIM_QUEUED;
1930
1931	mpssas_log_command(cm, MPS_XINFO, "%s cm %p ccb %p outstanding %u\n",
1932	    __func__, cm, ccb, targ->outstanding);
1933
1934	mps_map_command(sc, cm);
1935	return;
1936}
1937
1938static void
1939mps_response_code(struct mps_softc *sc, u8 response_code)
1940{
1941        char *desc;
1942
1943        switch (response_code) {
1944        case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
1945                desc = "task management request completed";
1946                break;
1947        case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
1948                desc = "invalid frame";
1949                break;
1950        case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
1951                desc = "task management request not supported";
1952                break;
1953        case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
1954                desc = "task management request failed";
1955                break;
1956        case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
1957                desc = "task management request succeeded";
1958                break;
1959        case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
1960                desc = "invalid lun";
1961                break;
1962        case 0xA:
1963                desc = "overlapped tag attempted";
1964                break;
1965        case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
1966                desc = "task queued, however not sent to target";
1967                break;
1968        default:
1969                desc = "unknown";
1970                break;
1971        }
1972		mps_dprint(sc, MPS_XINFO, "response_code(0x%01x): %s\n",
1973                response_code, desc);
1974}
1975/**
1976 * mps_sc_failed_io_info - translated non-succesfull SCSI_IO request
1977 */
1978static void
1979mps_sc_failed_io_info(struct mps_softc *sc, struct ccb_scsiio *csio,
1980    Mpi2SCSIIOReply_t *mpi_reply)
1981{
1982	u32 response_info;
1983	u8 *response_bytes;
1984	u16 ioc_status = le16toh(mpi_reply->IOCStatus) &
1985	    MPI2_IOCSTATUS_MASK;
1986	u8 scsi_state = mpi_reply->SCSIState;
1987	u8 scsi_status = mpi_reply->SCSIStatus;
1988	char *desc_ioc_state = NULL;
1989	char *desc_scsi_status = NULL;
1990	char *desc_scsi_state = sc->tmp_string;
1991	u32 log_info = le32toh(mpi_reply->IOCLogInfo);
1992
1993	if (log_info == 0x31170000)
1994		return;
1995
1996	switch (ioc_status) {
1997	case MPI2_IOCSTATUS_SUCCESS:
1998		desc_ioc_state = "success";
1999		break;
2000	case MPI2_IOCSTATUS_INVALID_FUNCTION:
2001		desc_ioc_state = "invalid function";
2002		break;
2003	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
2004		desc_ioc_state = "scsi recovered error";
2005		break;
2006	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
2007		desc_ioc_state = "scsi invalid dev handle";
2008		break;
2009	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
2010		desc_ioc_state = "scsi device not there";
2011		break;
2012	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
2013		desc_ioc_state = "scsi data overrun";
2014		break;
2015	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
2016		desc_ioc_state = "scsi data underrun";
2017		break;
2018	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
2019		desc_ioc_state = "scsi io data error";
2020		break;
2021	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
2022		desc_ioc_state = "scsi protocol error";
2023		break;
2024	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
2025		desc_ioc_state = "scsi task terminated";
2026		break;
2027	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
2028		desc_ioc_state = "scsi residual mismatch";
2029		break;
2030	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
2031		desc_ioc_state = "scsi task mgmt failed";
2032		break;
2033	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
2034		desc_ioc_state = "scsi ioc terminated";
2035		break;
2036	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
2037		desc_ioc_state = "scsi ext terminated";
2038		break;
2039	case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
2040		desc_ioc_state = "eedp guard error";
2041		break;
2042	case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
2043		desc_ioc_state = "eedp ref tag error";
2044		break;
2045	case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
2046		desc_ioc_state = "eedp app tag error";
2047		break;
2048	default:
2049		desc_ioc_state = "unknown";
2050		break;
2051	}
2052
2053	switch (scsi_status) {
2054	case MPI2_SCSI_STATUS_GOOD:
2055		desc_scsi_status = "good";
2056		break;
2057	case MPI2_SCSI_STATUS_CHECK_CONDITION:
2058		desc_scsi_status = "check condition";
2059		break;
2060	case MPI2_SCSI_STATUS_CONDITION_MET:
2061		desc_scsi_status = "condition met";
2062		break;
2063	case MPI2_SCSI_STATUS_BUSY:
2064		desc_scsi_status = "busy";
2065		break;
2066	case MPI2_SCSI_STATUS_INTERMEDIATE:
2067		desc_scsi_status = "intermediate";
2068		break;
2069	case MPI2_SCSI_STATUS_INTERMEDIATE_CONDMET:
2070		desc_scsi_status = "intermediate condmet";
2071		break;
2072	case MPI2_SCSI_STATUS_RESERVATION_CONFLICT:
2073		desc_scsi_status = "reservation conflict";
2074		break;
2075	case MPI2_SCSI_STATUS_COMMAND_TERMINATED:
2076		desc_scsi_status = "command terminated";
2077		break;
2078	case MPI2_SCSI_STATUS_TASK_SET_FULL:
2079		desc_scsi_status = "task set full";
2080		break;
2081	case MPI2_SCSI_STATUS_ACA_ACTIVE:
2082		desc_scsi_status = "aca active";
2083		break;
2084	case MPI2_SCSI_STATUS_TASK_ABORTED:
2085		desc_scsi_status = "task aborted";
2086		break;
2087	default:
2088		desc_scsi_status = "unknown";
2089		break;
2090	}
2091
2092	desc_scsi_state[0] = '\0';
2093	if (!scsi_state)
2094		desc_scsi_state = " ";
2095	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID)
2096		strcat(desc_scsi_state, "response info ");
2097	if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
2098		strcat(desc_scsi_state, "state terminated ");
2099	if (scsi_state & MPI2_SCSI_STATE_NO_SCSI_STATUS)
2100		strcat(desc_scsi_state, "no status ");
2101	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_FAILED)
2102		strcat(desc_scsi_state, "autosense failed ");
2103	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID)
2104		strcat(desc_scsi_state, "autosense valid ");
2105
2106	mps_dprint(sc, MPS_XINFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x)\n",
2107	    le16toh(mpi_reply->DevHandle), desc_ioc_state, ioc_status);
2108	/* We can add more detail about underflow data here
2109	 * TO-DO
2110	 * */
2111	mps_dprint(sc, MPS_XINFO, "\tscsi_status(%s)(0x%02x), "
2112	    "scsi_state(%s)(0x%02x)\n", desc_scsi_status, scsi_status,
2113	    desc_scsi_state, scsi_state);
2114
2115	if (sc->mps_debug & MPS_XINFO &&
2116		scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2117		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : Start :\n");
2118		scsi_sense_print(csio);
2119		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : End :\n");
2120	}
2121
2122	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
2123		response_info = le32toh(mpi_reply->ResponseInfo);
2124		response_bytes = (u8 *)&response_info;
2125		mps_response_code(sc,response_bytes[0]);
2126	}
2127}
2128
2129static void
2130mpssas_scsiio_complete(struct mps_softc *sc, struct mps_command *cm)
2131{
2132	MPI2_SCSI_IO_REPLY *rep;
2133	union ccb *ccb;
2134	struct ccb_scsiio *csio;
2135	struct mpssas_softc *sassc;
2136	struct scsi_vpd_supported_page_list *vpd_list = NULL;
2137	u8 *TLR_bits, TLR_on;
2138	int dir = 0, i;
2139	u16 alloc_len;
2140	struct mpssas_target *target;
2141	target_id_t target_id;
2142
2143	MPS_FUNCTRACE(sc);
2144	mps_dprint(sc, MPS_TRACE,
2145	    "cm %p SMID %u ccb %p reply %p outstanding %u\n", cm,
2146	    cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply,
2147	    cm->cm_targ->outstanding);
2148
2149	callout_stop(&cm->cm_callout);
2150	mtx_assert(&sc->mps_mtx, MA_OWNED);
2151
2152	sassc = sc->sassc;
2153	ccb = cm->cm_complete_data;
2154	csio = &ccb->csio;
2155	target_id = csio->ccb_h.target_id;
2156	rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply;
2157	/*
2158	 * XXX KDM if the chain allocation fails, does it matter if we do
2159	 * the sync and unload here?  It is simpler to do it in every case,
2160	 * assuming it doesn't cause problems.
2161	 */
2162	if (cm->cm_data != NULL) {
2163		if (cm->cm_flags & MPS_CM_FLAGS_DATAIN)
2164			dir = BUS_DMASYNC_POSTREAD;
2165		else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT)
2166			dir = BUS_DMASYNC_POSTWRITE;
2167		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
2168		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2169	}
2170
2171	cm->cm_targ->completed++;
2172	cm->cm_targ->outstanding--;
2173	TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link);
2174	ccb->ccb_h.status &= ~(CAM_STATUS_MASK | CAM_SIM_QUEUED);
2175
2176#if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
2177	if (ccb->csio.bio != NULL)
2178		biotrack(ccb->csio.bio, __func__);
2179#endif
2180
2181	if (cm->cm_state == MPS_CM_STATE_TIMEDOUT) {
2182		TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery);
2183		if (cm->cm_reply != NULL)
2184			mpssas_log_command(cm, MPS_RECOVERY,
2185			    "completed timedout cm %p ccb %p during recovery "
2186			    "ioc %x scsi %x state %x xfer %u\n",
2187			    cm, cm->cm_ccb,
2188			    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2189			    le32toh(rep->TransferCount));
2190		else
2191			mpssas_log_command(cm, MPS_RECOVERY,
2192			    "completed timedout cm %p ccb %p during recovery\n",
2193			    cm, cm->cm_ccb);
2194	} else if (cm->cm_targ->tm != NULL) {
2195		if (cm->cm_reply != NULL)
2196			mpssas_log_command(cm, MPS_RECOVERY,
2197			    "completed cm %p ccb %p during recovery "
2198			    "ioc %x scsi %x state %x xfer %u\n",
2199			    cm, cm->cm_ccb,
2200			    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2201			    le32toh(rep->TransferCount));
2202		else
2203			mpssas_log_command(cm, MPS_RECOVERY,
2204			    "completed cm %p ccb %p during recovery\n",
2205			    cm, cm->cm_ccb);
2206	} else if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
2207		mpssas_log_command(cm, MPS_RECOVERY,
2208		    "reset completed cm %p ccb %p\n",
2209		    cm, cm->cm_ccb);
2210	}
2211
2212	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2213		/*
2214		 * We ran into an error after we tried to map the command,
2215		 * so we're getting a callback without queueing the command
2216		 * to the hardware.  So we set the status here, and it will
2217		 * be retained below.  We'll go through the "fast path",
2218		 * because there can be no reply when we haven't actually
2219		 * gone out to the hardware.
2220		 */
2221		mpssas_set_ccbstatus(ccb, CAM_REQUEUE_REQ);
2222
2223		/*
2224		 * Currently the only error included in the mask is
2225		 * MPS_CM_FLAGS_CHAIN_FAILED, which means we're out of
2226		 * chain frames.  We need to freeze the queue until we get
2227		 * a command that completed without this error, which will
2228		 * hopefully have some chain frames attached that we can
2229		 * use.  If we wanted to get smarter about it, we would
2230		 * only unfreeze the queue in this condition when we're
2231		 * sure that we're getting some chain frames back.  That's
2232		 * probably unnecessary.
2233		 */
2234		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
2235			xpt_freeze_simq(sassc->sim, 1);
2236			sassc->flags |= MPSSAS_QUEUE_FROZEN;
2237			mps_dprint(sc, MPS_XINFO, "Error sending command, "
2238				   "freezing SIM queue\n");
2239		}
2240	}
2241
2242	/*
2243	 * If this is a Start Stop Unit command and it was issued by the driver
2244	 * during shutdown, decrement the refcount to account for all of the
2245	 * commands that were sent.  All SSU commands should be completed before
2246	 * shutdown completes, meaning SSU_refcount will be 0 after SSU_started
2247	 * is TRUE.
2248	 */
2249	if (sc->SSU_started && (csio->cdb_io.cdb_bytes[0] == START_STOP_UNIT)) {
2250		mps_dprint(sc, MPS_INFO, "Decrementing SSU count.\n");
2251		sc->SSU_refcount--;
2252	}
2253
2254	/* Take the fast path to completion */
2255	if (cm->cm_reply == NULL) {
2256		if (mpssas_get_ccbstatus(ccb) == CAM_REQ_INPROG) {
2257			if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0)
2258				mpssas_set_ccbstatus(ccb, CAM_SCSI_BUS_RESET);
2259			else {
2260				mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2261				ccb->csio.scsi_status = SCSI_STATUS_OK;
2262			}
2263			if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2264				ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2265				sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2266				mps_dprint(sc, MPS_XINFO,
2267				    "Unfreezing SIM queue\n");
2268			}
2269		}
2270
2271		/*
2272		 * There are two scenarios where the status won't be
2273		 * CAM_REQ_CMP.  The first is if MPS_CM_FLAGS_ERROR_MASK is
2274		 * set, the second is in the MPS_FLAGS_DIAGRESET above.
2275		 */
2276		if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2277			/*
2278			 * Freeze the dev queue so that commands are
2279			 * executed in the correct order after error
2280			 * recovery.
2281			 */
2282			ccb->ccb_h.status |= CAM_DEV_QFRZN;
2283			xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2284		}
2285		mps_free_command(sc, cm);
2286		xpt_done(ccb);
2287		return;
2288	}
2289
2290	mpssas_log_command(cm, MPS_XINFO,
2291	    "ioc %x scsi %x state %x xfer %u\n",
2292	    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2293	    le32toh(rep->TransferCount));
2294
2295	/*
2296	 * If this is a Direct Drive I/O, reissue the I/O to the original IR
2297	 * Volume if an error occurred (normal I/O retry).  Use the original
2298	 * CCB, but set a flag that this will be a retry so that it's sent to
2299	 * the original volume.  Free the command but reuse the CCB.
2300	 */
2301	if (cm->cm_flags & MPS_CM_FLAGS_DD_IO) {
2302		mps_free_command(sc, cm);
2303		ccb->ccb_h.sim_priv.entries[0].field = MPS_WD_RETRY;
2304		mpssas_action_scsiio(sassc, ccb);
2305		return;
2306	} else
2307		ccb->ccb_h.sim_priv.entries[0].field = 0;
2308
2309	switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) {
2310	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
2311		csio->resid = cm->cm_length - le32toh(rep->TransferCount);
2312		/* FALLTHROUGH */
2313	case MPI2_IOCSTATUS_SUCCESS:
2314	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
2315
2316		if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
2317		    MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
2318			mpssas_log_command(cm, MPS_XINFO, "recovered error\n");
2319
2320		/* Completion failed at the transport level. */
2321		if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS |
2322		    MPI2_SCSI_STATE_TERMINATED)) {
2323			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2324			break;
2325		}
2326
2327		/* In a modern packetized environment, an autosense failure
2328		 * implies that there's not much else that can be done to
2329		 * recover the command.
2330		 */
2331		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) {
2332			mpssas_set_ccbstatus(ccb, CAM_AUTOSENSE_FAIL);
2333			break;
2334		}
2335
2336		/*
2337		 * CAM doesn't care about SAS Response Info data, but if this is
2338		 * the state check if TLR should be done.  If not, clear the
2339		 * TLR_bits for the target.
2340		 */
2341		if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) &&
2342		    ((le32toh(rep->ResponseInfo) &
2343		    MPI2_SCSI_RI_MASK_REASONCODE) ==
2344		    MPS_SCSI_RI_INVALID_FRAME)) {
2345			sc->mapping_table[target_id].TLR_bits =
2346			    (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2347		}
2348
2349		/*
2350		 * Intentionally override the normal SCSI status reporting
2351		 * for these two cases.  These are likely to happen in a
2352		 * multi-initiator environment, and we want to make sure that
2353		 * CAM retries these commands rather than fail them.
2354		 */
2355		if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) ||
2356		    (rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) {
2357			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2358			break;
2359		}
2360
2361		/* Handle normal status and sense */
2362		csio->scsi_status = rep->SCSIStatus;
2363		if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD)
2364			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2365		else
2366			mpssas_set_ccbstatus(ccb, CAM_SCSI_STATUS_ERROR);
2367
2368		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2369			int sense_len, returned_sense_len;
2370
2371			returned_sense_len = min(le32toh(rep->SenseCount),
2372			    sizeof(struct scsi_sense_data));
2373			if (returned_sense_len < ccb->csio.sense_len)
2374				ccb->csio.sense_resid = ccb->csio.sense_len -
2375					returned_sense_len;
2376			else
2377				ccb->csio.sense_resid = 0;
2378
2379			sense_len = min(returned_sense_len,
2380			    ccb->csio.sense_len - ccb->csio.sense_resid);
2381			bzero(&ccb->csio.sense_data,
2382			      sizeof(ccb->csio.sense_data));
2383			bcopy(cm->cm_sense, &ccb->csio.sense_data, sense_len);
2384			ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2385		}
2386
2387		/*
2388		 * Check if this is an INQUIRY command.  If it's a VPD inquiry,
2389		 * and it's page code 0 (Supported Page List), and there is
2390		 * inquiry data, and this is for a sequential access device, and
2391		 * the device is an SSP target, and TLR is supported by the
2392		 * controller, turn the TLR_bits value ON if page 0x90 is
2393		 * supported.
2394		 */
2395		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2396		    (csio->cdb_io.cdb_bytes[1] & SI_EVPD) &&
2397		    (csio->cdb_io.cdb_bytes[2] == SVPD_SUPPORTED_PAGE_LIST) &&
2398		    ((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) &&
2399		    (csio->data_ptr != NULL) &&
2400		    ((csio->data_ptr[0] & 0x1f) == T_SEQUENTIAL) &&
2401		    (sc->control_TLR) &&
2402		    (sc->mapping_table[target_id].device_info &
2403		    MPI2_SAS_DEVICE_INFO_SSP_TARGET)) {
2404			vpd_list = (struct scsi_vpd_supported_page_list *)
2405			    csio->data_ptr;
2406			TLR_bits = &sc->mapping_table[target_id].TLR_bits;
2407			*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2408			TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
2409			alloc_len = ((u16)csio->cdb_io.cdb_bytes[3] << 8) +
2410			    csio->cdb_io.cdb_bytes[4];
2411			alloc_len -= csio->resid;
2412			for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
2413				if (vpd_list->list[i] == 0x90) {
2414					*TLR_bits = TLR_on;
2415					break;
2416				}
2417			}
2418		}
2419
2420		/*
2421		 * If this is a SATA direct-access end device, mark it so that
2422		 * a SCSI StartStopUnit command will be sent to it when the
2423		 * driver is being shutdown.
2424		 */
2425		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2426		    ((csio->data_ptr[0] & 0x1f) == T_DIRECT) &&
2427		    (sc->mapping_table[target_id].device_info &
2428		    MPI2_SAS_DEVICE_INFO_SATA_DEVICE) &&
2429		    ((sc->mapping_table[target_id].device_info &
2430		    MPI2_SAS_DEVICE_INFO_MASK_DEVICE_TYPE) ==
2431		    MPI2_SAS_DEVICE_INFO_END_DEVICE)) {
2432			target = &sassc->targets[target_id];
2433			target->supports_SSU = TRUE;
2434			mps_dprint(sc, MPS_XINFO, "Target %d supports SSU\n",
2435			    target_id);
2436		}
2437		break;
2438	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
2439	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
2440		/*
2441		 * If devinfo is 0 this will be a volume.  In that case don't
2442		 * tell CAM that the volume is not there.  We want volumes to
2443		 * be enumerated until they are deleted/removed, not just
2444		 * failed.
2445		 */
2446		if (cm->cm_targ->devinfo == 0)
2447			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2448		else
2449			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2450		break;
2451	case MPI2_IOCSTATUS_INVALID_SGL:
2452		mps_print_scsiio_cmd(sc, cm);
2453		mpssas_set_ccbstatus(ccb, CAM_UNREC_HBA_ERROR);
2454		break;
2455	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
2456		/*
2457		 * This is one of the responses that comes back when an I/O
2458		 * has been aborted.  If it is because of a timeout that we
2459		 * initiated, just set the status to CAM_CMD_TIMEOUT.
2460		 * Otherwise set it to CAM_REQ_ABORTED.  The effect on the
2461		 * command is the same (it gets retried, subject to the
2462		 * retry counter), the only difference is what gets printed
2463		 * on the console.
2464		 */
2465		if (cm->cm_state == MPS_CM_STATE_TIMEDOUT)
2466			mpssas_set_ccbstatus(ccb, CAM_CMD_TIMEOUT);
2467		else
2468			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2469		break;
2470	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
2471		/* resid is ignored for this condition */
2472		csio->resid = 0;
2473		mpssas_set_ccbstatus(ccb, CAM_DATA_RUN_ERR);
2474		break;
2475	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
2476	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
2477		/*
2478		 * These can sometimes be transient transport-related
2479		 * errors, and sometimes persistent drive-related errors.
2480		 * We used to retry these without decrementing the retry
2481		 * count by returning CAM_REQUEUE_REQ.  Unfortunately, if
2482		 * we hit a persistent drive problem that returns one of
2483		 * these error codes, we would retry indefinitely.  So,
2484		 * return CAM_REQ_CMP_ERROR so that we decrement the retry
2485		 * count and avoid infinite retries.  We're taking the
2486		 * potential risk of flagging false failures in the event
2487		 * of a topology-related error (e.g. a SAS expander problem
2488		 * causes a command addressed to a drive to fail), but
2489		 * avoiding getting into an infinite retry loop.
2490		 */
2491		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2492		mps_dprint(sc, MPS_INFO,
2493		    "Controller reported %s status for target %u SMID %u, "
2494		    "loginfo %x\n", ((rep->IOCStatus & MPI2_IOCSTATUS_MASK) ==
2495		    MPI2_IOCSTATUS_SCSI_IOC_TERMINATED) ? "IOC_TERMINATED" :
2496		    "EXT_TERMINATED", target_id, cm->cm_desc.Default.SMID,
2497		    le32toh(rep->IOCLogInfo));
2498		mps_dprint(sc, MPS_XINFO,
2499		    "SCSIStatus %x SCSIState %x xfercount %u\n",
2500		    rep->SCSIStatus, rep->SCSIState,
2501		    le32toh(rep->TransferCount));
2502		break;
2503	case MPI2_IOCSTATUS_INVALID_FUNCTION:
2504	case MPI2_IOCSTATUS_INTERNAL_ERROR:
2505	case MPI2_IOCSTATUS_INVALID_VPID:
2506	case MPI2_IOCSTATUS_INVALID_FIELD:
2507	case MPI2_IOCSTATUS_INVALID_STATE:
2508	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
2509	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
2510	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
2511	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
2512	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
2513	default:
2514		mpssas_log_command(cm, MPS_XINFO,
2515		    "completed ioc %x loginfo %x scsi %x state %x xfer %u\n",
2516		    le16toh(rep->IOCStatus), le32toh(rep->IOCLogInfo),
2517		    rep->SCSIStatus, rep->SCSIState,
2518		    le32toh(rep->TransferCount));
2519		csio->resid = cm->cm_length;
2520		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2521		break;
2522	}
2523
2524	mps_sc_failed_io_info(sc,csio,rep);
2525
2526	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2527		ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2528		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2529		mps_dprint(sc, MPS_XINFO, "Command completed, "
2530		    "unfreezing SIM queue\n");
2531	}
2532
2533	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2534		ccb->ccb_h.status |= CAM_DEV_QFRZN;
2535		xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2536	}
2537
2538	mps_free_command(sc, cm);
2539	xpt_done(ccb);
2540}
2541
2542/* All Request reached here are Endian safe */
2543static void
2544mpssas_direct_drive_io(struct mpssas_softc *sassc, struct mps_command *cm,
2545    union ccb *ccb) {
2546	pMpi2SCSIIORequest_t	pIO_req;
2547	struct mps_softc	*sc = sassc->sc;
2548	uint64_t		virtLBA;
2549	uint32_t		physLBA, stripe_offset, stripe_unit;
2550	uint32_t		io_size, column;
2551	uint8_t			*ptrLBA, lba_idx, physLBA_byte, *CDB;
2552
2553	/*
2554	 * If this is a valid SCSI command (Read6, Read10, Read16, Write6,
2555	 * Write10, or Write16), build a direct I/O message.  Otherwise, the I/O
2556	 * will be sent to the IR volume itself.  Since Read6 and Write6 are a
2557	 * bit different than the 10/16 CDBs, handle them separately.
2558	 */
2559	pIO_req = (pMpi2SCSIIORequest_t)cm->cm_req;
2560	CDB = pIO_req->CDB.CDB32;
2561
2562	/*
2563	 * Handle 6 byte CDBs.
2564	 */
2565	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_6) ||
2566	    (CDB[0] == WRITE_6))) {
2567		/*
2568		 * Get the transfer size in blocks.
2569		 */
2570		io_size = (cm->cm_length >> sc->DD_block_exponent);
2571
2572		/*
2573		 * Get virtual LBA given in the CDB.
2574		 */
2575		virtLBA = ((uint64_t)(CDB[1] & 0x1F) << 16) |
2576		    ((uint64_t)CDB[2] << 8) | (uint64_t)CDB[3];
2577
2578		/*
2579		 * Check that LBA range for I/O does not exceed volume's
2580		 * MaxLBA.
2581		 */
2582		if ((virtLBA + (uint64_t)io_size - 1) <=
2583		    sc->DD_max_lba) {
2584			/*
2585			 * Check if the I/O crosses a stripe boundary.  If not,
2586			 * translate the virtual LBA to a physical LBA and set
2587			 * the DevHandle for the PhysDisk to be used.  If it
2588			 * does cross a boundary, do normal I/O.  To get the
2589			 * right DevHandle to use, get the map number for the
2590			 * column, then use that map number to look up the
2591			 * DevHandle of the PhysDisk.
2592			 */
2593			stripe_offset = (uint32_t)virtLBA &
2594			    (sc->DD_stripe_size - 1);
2595			if ((stripe_offset + io_size) <= sc->DD_stripe_size) {
2596				physLBA = (uint32_t)virtLBA >>
2597				    sc->DD_stripe_exponent;
2598				stripe_unit = physLBA / sc->DD_num_phys_disks;
2599				column = physLBA % sc->DD_num_phys_disks;
2600				pIO_req->DevHandle =
2601				    htole16(sc->DD_column_map[column].dev_handle);
2602				/* ???? Is this endian safe*/
2603				cm->cm_desc.SCSIIO.DevHandle =
2604				    pIO_req->DevHandle;
2605
2606				physLBA = (stripe_unit <<
2607				    sc->DD_stripe_exponent) + stripe_offset;
2608				ptrLBA = &pIO_req->CDB.CDB32[1];
2609				physLBA_byte = (uint8_t)(physLBA >> 16);
2610				*ptrLBA = physLBA_byte;
2611				ptrLBA = &pIO_req->CDB.CDB32[2];
2612				physLBA_byte = (uint8_t)(physLBA >> 8);
2613				*ptrLBA = physLBA_byte;
2614				ptrLBA = &pIO_req->CDB.CDB32[3];
2615				physLBA_byte = (uint8_t)physLBA;
2616				*ptrLBA = physLBA_byte;
2617
2618				/*
2619				 * Set flag that Direct Drive I/O is
2620				 * being done.
2621				 */
2622				cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2623			}
2624		}
2625		return;
2626	}
2627
2628	/*
2629	 * Handle 10, 12 or 16 byte CDBs.
2630	 */
2631	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_10) ||
2632	    (CDB[0] == WRITE_10) || (CDB[0] == READ_16) ||
2633	    (CDB[0] == WRITE_16) || (CDB[0] == READ_12) ||
2634	    (CDB[0] == WRITE_12))) {
2635		/*
2636		 * For 16-byte CDB's, verify that the upper 4 bytes of the CDB
2637		 * are 0.  If not, this is accessing beyond 2TB so handle it in
2638		 * the else section.  10-byte and 12-byte CDB's are OK.
2639		 * FreeBSD sends very rare 12 byte READ/WRITE, but driver is
2640		 * ready to accept 12byte CDB for Direct IOs.
2641		 */
2642		if ((CDB[0] == READ_10 || CDB[0] == WRITE_10) ||
2643		    (CDB[0] == READ_12 || CDB[0] == WRITE_12) ||
2644		    !(CDB[2] | CDB[3] | CDB[4] | CDB[5])) {
2645			/*
2646			 * Get the transfer size in blocks.
2647			 */
2648			io_size = (cm->cm_length >> sc->DD_block_exponent);
2649
2650			/*
2651			 * Get virtual LBA.  Point to correct lower 4 bytes of
2652			 * LBA in the CDB depending on command.
2653			 */
2654			lba_idx = ((CDB[0] == READ_12) ||
2655				(CDB[0] == WRITE_12) ||
2656				(CDB[0] == READ_10) ||
2657				(CDB[0] == WRITE_10))? 2 : 6;
2658			virtLBA = ((uint64_t)CDB[lba_idx] << 24) |
2659			    ((uint64_t)CDB[lba_idx + 1] << 16) |
2660			    ((uint64_t)CDB[lba_idx + 2] << 8) |
2661			    (uint64_t)CDB[lba_idx + 3];
2662
2663			/*
2664			 * Check that LBA range for I/O does not exceed volume's
2665			 * MaxLBA.
2666			 */
2667			if ((virtLBA + (uint64_t)io_size - 1) <=
2668			    sc->DD_max_lba) {
2669				/*
2670				 * Check if the I/O crosses a stripe boundary.
2671				 * If not, translate the virtual LBA to a
2672				 * physical LBA and set the DevHandle for the
2673				 * PhysDisk to be used.  If it does cross a
2674				 * boundary, do normal I/O.  To get the right
2675				 * DevHandle to use, get the map number for the
2676				 * column, then use that map number to look up
2677				 * the DevHandle of the PhysDisk.
2678				 */
2679				stripe_offset = (uint32_t)virtLBA &
2680				    (sc->DD_stripe_size - 1);
2681				if ((stripe_offset + io_size) <=
2682				    sc->DD_stripe_size) {
2683					physLBA = (uint32_t)virtLBA >>
2684					    sc->DD_stripe_exponent;
2685					stripe_unit = physLBA /
2686					    sc->DD_num_phys_disks;
2687					column = physLBA %
2688					    sc->DD_num_phys_disks;
2689					pIO_req->DevHandle =
2690					    htole16(sc->DD_column_map[column].
2691					    dev_handle);
2692					cm->cm_desc.SCSIIO.DevHandle =
2693					    pIO_req->DevHandle;
2694
2695					physLBA = (stripe_unit <<
2696					    sc->DD_stripe_exponent) +
2697					    stripe_offset;
2698					ptrLBA =
2699					    &pIO_req->CDB.CDB32[lba_idx];
2700					physLBA_byte = (uint8_t)(physLBA >> 24);
2701					*ptrLBA = physLBA_byte;
2702					ptrLBA =
2703					    &pIO_req->CDB.CDB32[lba_idx + 1];
2704					physLBA_byte = (uint8_t)(physLBA >> 16);
2705					*ptrLBA = physLBA_byte;
2706					ptrLBA =
2707					    &pIO_req->CDB.CDB32[lba_idx + 2];
2708					physLBA_byte = (uint8_t)(physLBA >> 8);
2709					*ptrLBA = physLBA_byte;
2710					ptrLBA =
2711					    &pIO_req->CDB.CDB32[lba_idx + 3];
2712					physLBA_byte = (uint8_t)physLBA;
2713					*ptrLBA = physLBA_byte;
2714
2715					/*
2716					 * Set flag that Direct Drive I/O is
2717					 * being done.
2718					 */
2719					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2720				}
2721			}
2722		} else {
2723			/*
2724			 * 16-byte CDB and the upper 4 bytes of the CDB are not
2725			 * 0.  Get the transfer size in blocks.
2726			 */
2727			io_size = (cm->cm_length >> sc->DD_block_exponent);
2728
2729			/*
2730			 * Get virtual LBA.
2731			 */
2732			virtLBA = ((uint64_t)CDB[2] << 54) |
2733			    ((uint64_t)CDB[3] << 48) |
2734			    ((uint64_t)CDB[4] << 40) |
2735			    ((uint64_t)CDB[5] << 32) |
2736			    ((uint64_t)CDB[6] << 24) |
2737			    ((uint64_t)CDB[7] << 16) |
2738			    ((uint64_t)CDB[8] << 8) |
2739			    (uint64_t)CDB[9];
2740
2741			/*
2742			 * Check that LBA range for I/O does not exceed volume's
2743			 * MaxLBA.
2744			 */
2745			if ((virtLBA + (uint64_t)io_size - 1) <=
2746			    sc->DD_max_lba) {
2747				/*
2748				 * Check if the I/O crosses a stripe boundary.
2749				 * If not, translate the virtual LBA to a
2750				 * physical LBA and set the DevHandle for the
2751				 * PhysDisk to be used.  If it does cross a
2752				 * boundary, do normal I/O.  To get the right
2753				 * DevHandle to use, get the map number for the
2754				 * column, then use that map number to look up
2755				 * the DevHandle of the PhysDisk.
2756				 */
2757				stripe_offset = (uint32_t)virtLBA &
2758				    (sc->DD_stripe_size - 1);
2759				if ((stripe_offset + io_size) <=
2760				    sc->DD_stripe_size) {
2761					physLBA = (uint32_t)(virtLBA >>
2762					    sc->DD_stripe_exponent);
2763					stripe_unit = physLBA /
2764					    sc->DD_num_phys_disks;
2765					column = physLBA %
2766					    sc->DD_num_phys_disks;
2767					pIO_req->DevHandle =
2768					    htole16(sc->DD_column_map[column].
2769					    dev_handle);
2770					cm->cm_desc.SCSIIO.DevHandle =
2771					    pIO_req->DevHandle;
2772
2773					physLBA = (stripe_unit <<
2774					    sc->DD_stripe_exponent) +
2775					    stripe_offset;
2776
2777					/*
2778					 * Set upper 4 bytes of LBA to 0.  We
2779					 * assume that the phys disks are less
2780					 * than 2 TB's in size.  Then, set the
2781					 * lower 4 bytes.
2782					 */
2783					pIO_req->CDB.CDB32[2] = 0;
2784					pIO_req->CDB.CDB32[3] = 0;
2785					pIO_req->CDB.CDB32[4] = 0;
2786					pIO_req->CDB.CDB32[5] = 0;
2787					ptrLBA = &pIO_req->CDB.CDB32[6];
2788					physLBA_byte = (uint8_t)(physLBA >> 24);
2789					*ptrLBA = physLBA_byte;
2790					ptrLBA = &pIO_req->CDB.CDB32[7];
2791					physLBA_byte = (uint8_t)(physLBA >> 16);
2792					*ptrLBA = physLBA_byte;
2793					ptrLBA = &pIO_req->CDB.CDB32[8];
2794					physLBA_byte = (uint8_t)(physLBA >> 8);
2795					*ptrLBA = physLBA_byte;
2796					ptrLBA = &pIO_req->CDB.CDB32[9];
2797					physLBA_byte = (uint8_t)physLBA;
2798					*ptrLBA = physLBA_byte;
2799
2800					/*
2801					 * Set flag that Direct Drive I/O is
2802					 * being done.
2803					 */
2804					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2805				}
2806			}
2807		}
2808	}
2809}
2810
2811#if __FreeBSD_version >= 900026
2812static void
2813mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm)
2814{
2815	MPI2_SMP_PASSTHROUGH_REPLY *rpl;
2816	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2817	uint64_t sasaddr;
2818	union ccb *ccb;
2819
2820	ccb = cm->cm_complete_data;
2821
2822	/*
2823	 * Currently there should be no way we can hit this case.  It only
2824	 * happens when we have a failure to allocate chain frames, and SMP
2825	 * commands require two S/G elements only.  That should be handled
2826	 * in the standard request size.
2827	 */
2828	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2829		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x on SMP request!\n",
2830			   __func__, cm->cm_flags);
2831		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2832		goto bailout;
2833        }
2834
2835	rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply;
2836	if (rpl == NULL) {
2837		mps_dprint(sc, MPS_ERROR, "%s: NULL cm_reply!\n", __func__);
2838		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2839		goto bailout;
2840	}
2841
2842	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2843	sasaddr = le32toh(req->SASAddress.Low);
2844	sasaddr |= ((uint64_t)(le32toh(req->SASAddress.High))) << 32;
2845
2846	if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
2847	    MPI2_IOCSTATUS_SUCCESS ||
2848	    rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) {
2849		mps_dprint(sc, MPS_XINFO, "%s: IOCStatus %04x SASStatus %02x\n",
2850		    __func__, le16toh(rpl->IOCStatus), rpl->SASStatus);
2851		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2852		goto bailout;
2853	}
2854
2855	mps_dprint(sc, MPS_XINFO, "%s: SMP request to SAS address "
2856		   "%#jx completed successfully\n", __func__,
2857		   (uintmax_t)sasaddr);
2858
2859	if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED)
2860		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2861	else
2862		mpssas_set_ccbstatus(ccb, CAM_SMP_STATUS_ERROR);
2863
2864bailout:
2865	/*
2866	 * We sync in both directions because we had DMAs in the S/G list
2867	 * in both directions.
2868	 */
2869	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
2870			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2871	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2872	mps_free_command(sc, cm);
2873	xpt_done(ccb);
2874}
2875
2876static void
2877mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb, uint64_t sasaddr)
2878{
2879	struct mps_command *cm;
2880	uint8_t *request, *response;
2881	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2882	struct mps_softc *sc;
2883	int error;
2884
2885	sc = sassc->sc;
2886	error = 0;
2887
2888	/*
2889	 * XXX We don't yet support physical addresses here.
2890	 */
2891	switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) {
2892	case CAM_DATA_PADDR:
2893	case CAM_DATA_SG_PADDR:
2894		mps_dprint(sc, MPS_ERROR,
2895			   "%s: physical addresses not supported\n", __func__);
2896		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2897		xpt_done(ccb);
2898		return;
2899	case CAM_DATA_SG:
2900		/*
2901		 * The chip does not support more than one buffer for the
2902		 * request or response.
2903		 */
2904	 	if ((ccb->smpio.smp_request_sglist_cnt > 1)
2905		  || (ccb->smpio.smp_response_sglist_cnt > 1)) {
2906			mps_dprint(sc, MPS_ERROR,
2907				   "%s: multiple request or response "
2908				   "buffer segments not supported for SMP\n",
2909				   __func__);
2910			mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2911			xpt_done(ccb);
2912			return;
2913		}
2914
2915		/*
2916		 * The CAM_SCATTER_VALID flag was originally implemented
2917		 * for the XPT_SCSI_IO CCB, which only has one data pointer.
2918		 * We have two.  So, just take that flag to mean that we
2919		 * might have S/G lists, and look at the S/G segment count
2920		 * to figure out whether that is the case for each individual
2921		 * buffer.
2922		 */
2923		if (ccb->smpio.smp_request_sglist_cnt != 0) {
2924			bus_dma_segment_t *req_sg;
2925
2926			req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request;
2927			request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr;
2928		} else
2929			request = ccb->smpio.smp_request;
2930
2931		if (ccb->smpio.smp_response_sglist_cnt != 0) {
2932			bus_dma_segment_t *rsp_sg;
2933
2934			rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response;
2935			response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr;
2936		} else
2937			response = ccb->smpio.smp_response;
2938		break;
2939	case CAM_DATA_VADDR:
2940		request = ccb->smpio.smp_request;
2941		response = ccb->smpio.smp_response;
2942		break;
2943	default:
2944		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2945		xpt_done(ccb);
2946		return;
2947	}
2948
2949	cm = mps_alloc_command(sc);
2950	if (cm == NULL) {
2951		mps_dprint(sc, MPS_ERROR,
2952		    "%s: cannot allocate command\n", __func__);
2953		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2954		xpt_done(ccb);
2955		return;
2956	}
2957
2958	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2959	bzero(req, sizeof(*req));
2960	req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH;
2961
2962	/* Allow the chip to use any route to this SAS address. */
2963	req->PhysicalPort = 0xff;
2964
2965	req->RequestDataLength = htole16(ccb->smpio.smp_request_len);
2966	req->SGLFlags =
2967	    MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE | MPI2_SGLFLAGS_SGL_TYPE_MPI;
2968
2969	mps_dprint(sc, MPS_XINFO, "%s: sending SMP request to SAS "
2970	    "address %#jx\n", __func__, (uintmax_t)sasaddr);
2971
2972	mpi_init_sge(cm, req, &req->SGL);
2973
2974	/*
2975	 * Set up a uio to pass into mps_map_command().  This allows us to
2976	 * do one map command, and one busdma call in there.
2977	 */
2978	cm->cm_uio.uio_iov = cm->cm_iovec;
2979	cm->cm_uio.uio_iovcnt = 2;
2980	cm->cm_uio.uio_segflg = UIO_SYSSPACE;
2981
2982	/*
2983	 * The read/write flag isn't used by busdma, but set it just in
2984	 * case.  This isn't exactly accurate, either, since we're going in
2985	 * both directions.
2986	 */
2987	cm->cm_uio.uio_rw = UIO_WRITE;
2988
2989	cm->cm_iovec[0].iov_base = request;
2990	cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength);
2991	cm->cm_iovec[1].iov_base = response;
2992	cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len;
2993
2994	cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len +
2995			       cm->cm_iovec[1].iov_len;
2996
2997	/*
2998	 * Trigger a warning message in mps_data_cb() for the user if we
2999	 * wind up exceeding two S/G segments.  The chip expects one
3000	 * segment for the request and another for the response.
3001	 */
3002	cm->cm_max_segs = 2;
3003
3004	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3005	cm->cm_complete = mpssas_smpio_complete;
3006	cm->cm_complete_data = ccb;
3007
3008	/*
3009	 * Tell the mapping code that we're using a uio, and that this is
3010	 * an SMP passthrough request.  There is a little special-case
3011	 * logic there (in mps_data_cb()) to handle the bidirectional
3012	 * transfer.
3013	 */
3014	cm->cm_flags |= MPS_CM_FLAGS_USE_UIO | MPS_CM_FLAGS_SMP_PASS |
3015			MPS_CM_FLAGS_DATAIN | MPS_CM_FLAGS_DATAOUT;
3016
3017	/* The chip data format is little endian. */
3018	req->SASAddress.High = htole32(sasaddr >> 32);
3019	req->SASAddress.Low = htole32(sasaddr);
3020
3021	/*
3022	 * XXX Note that we don't have a timeout/abort mechanism here.
3023	 * From the manual, it looks like task management requests only
3024	 * work for SCSI IO and SATA passthrough requests.  We may need to
3025	 * have a mechanism to retry requests in the event of a chip reset
3026	 * at least.  Hopefully the chip will insure that any errors short
3027	 * of that are relayed back to the driver.
3028	 */
3029	error = mps_map_command(sc, cm);
3030	if ((error != 0) && (error != EINPROGRESS)) {
3031		mps_dprint(sc, MPS_ERROR,
3032			   "%s: error %d returned from mps_map_command()\n",
3033			   __func__, error);
3034		goto bailout_error;
3035	}
3036
3037	return;
3038
3039bailout_error:
3040	mps_free_command(sc, cm);
3041	mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
3042	xpt_done(ccb);
3043	return;
3044
3045}
3046
3047static void
3048mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb)
3049{
3050	struct mps_softc *sc;
3051	struct mpssas_target *targ;
3052	uint64_t sasaddr = 0;
3053
3054	sc = sassc->sc;
3055
3056	/*
3057	 * Make sure the target exists.
3058	 */
3059	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
3060	    ("Target %d out of bounds in XPT_SMP_IO\n", ccb->ccb_h.target_id));
3061	targ = &sassc->targets[ccb->ccb_h.target_id];
3062	if (targ->handle == 0x0) {
3063		mps_dprint(sc, MPS_ERROR,
3064			   "%s: target %d does not exist!\n", __func__,
3065			   ccb->ccb_h.target_id);
3066		mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
3067		xpt_done(ccb);
3068		return;
3069	}
3070
3071	/*
3072	 * If this device has an embedded SMP target, we'll talk to it
3073	 * directly.
3074	 * figure out what the expander's address is.
3075	 */
3076	if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0)
3077		sasaddr = targ->sasaddr;
3078
3079	/*
3080	 * If we don't have a SAS address for the expander yet, try
3081	 * grabbing it from the page 0x83 information cached in the
3082	 * transport layer for this target.  LSI expanders report the
3083	 * expander SAS address as the port-associated SAS address in
3084	 * Inquiry VPD page 0x83.  Maxim expanders don't report it in page
3085	 * 0x83.
3086	 *
3087	 * XXX KDM disable this for now, but leave it commented out so that
3088	 * it is obvious that this is another possible way to get the SAS
3089	 * address.
3090	 *
3091	 * The parent handle method below is a little more reliable, and
3092	 * the other benefit is that it works for devices other than SES
3093	 * devices.  So you can send a SMP request to a da(4) device and it
3094	 * will get routed to the expander that device is attached to.
3095	 * (Assuming the da(4) device doesn't contain an SMP target...)
3096	 */
3097#if 0
3098	if (sasaddr == 0)
3099		sasaddr = xpt_path_sas_addr(ccb->ccb_h.path);
3100#endif
3101
3102	/*
3103	 * If we still don't have a SAS address for the expander, look for
3104	 * the parent device of this device, which is probably the expander.
3105	 */
3106	if (sasaddr == 0) {
3107#ifdef OLD_MPS_PROBE
3108		struct mpssas_target *parent_target;
3109#endif
3110
3111		if (targ->parent_handle == 0x0) {
3112			mps_dprint(sc, MPS_ERROR,
3113				   "%s: handle %d does not have a valid "
3114				   "parent handle!\n", __func__, targ->handle);
3115			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3116			goto bailout;
3117		}
3118#ifdef OLD_MPS_PROBE
3119		parent_target = mpssas_find_target_by_handle(sassc, 0,
3120			targ->parent_handle);
3121
3122		if (parent_target == NULL) {
3123			mps_dprint(sc, MPS_ERROR,
3124				   "%s: handle %d does not have a valid "
3125				   "parent target!\n", __func__, targ->handle);
3126			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3127			goto bailout;
3128		}
3129
3130		if ((parent_target->devinfo &
3131		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3132			mps_dprint(sc, MPS_ERROR,
3133				   "%s: handle %d parent %d does not "
3134				   "have an SMP target!\n", __func__,
3135				   targ->handle, parent_target->handle);
3136			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3137			goto bailout;
3138
3139		}
3140
3141		sasaddr = parent_target->sasaddr;
3142#else /* OLD_MPS_PROBE */
3143		if ((targ->parent_devinfo &
3144		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3145			mps_dprint(sc, MPS_ERROR,
3146				   "%s: handle %d parent %d does not "
3147				   "have an SMP target!\n", __func__,
3148				   targ->handle, targ->parent_handle);
3149			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3150			goto bailout;
3151
3152		}
3153		if (targ->parent_sasaddr == 0x0) {
3154			mps_dprint(sc, MPS_ERROR,
3155				   "%s: handle %d parent handle %d does "
3156				   "not have a valid SAS address!\n",
3157				   __func__, targ->handle, targ->parent_handle);
3158			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3159			goto bailout;
3160		}
3161
3162		sasaddr = targ->parent_sasaddr;
3163#endif /* OLD_MPS_PROBE */
3164
3165	}
3166
3167	if (sasaddr == 0) {
3168		mps_dprint(sc, MPS_INFO,
3169			   "%s: unable to find SAS address for handle %d\n",
3170			   __func__, targ->handle);
3171		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3172		goto bailout;
3173	}
3174	mpssas_send_smpcmd(sassc, ccb, sasaddr);
3175
3176	return;
3177
3178bailout:
3179	xpt_done(ccb);
3180
3181}
3182#endif //__FreeBSD_version >= 900026
3183
3184static void
3185mpssas_action_resetdev(struct mpssas_softc *sassc, union ccb *ccb)
3186{
3187	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3188	struct mps_softc *sc;
3189	struct mps_command *tm;
3190	struct mpssas_target *targ;
3191
3192	MPS_FUNCTRACE(sassc->sc);
3193	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
3194
3195	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
3196	    ("Target %d out of bounds in XPT_RESET_DEV\n",
3197	     ccb->ccb_h.target_id));
3198	sc = sassc->sc;
3199	tm = mps_alloc_command(sc);
3200	if (tm == NULL) {
3201		mps_dprint(sc, MPS_ERROR,
3202		    "command alloc failure in mpssas_action_resetdev\n");
3203		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
3204		xpt_done(ccb);
3205		return;
3206	}
3207
3208	targ = &sassc->targets[ccb->ccb_h.target_id];
3209	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3210	req->DevHandle = htole16(targ->handle);
3211	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
3212	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
3213
3214	/* SAS Hard Link Reset / SATA Link Reset */
3215	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3216
3217	tm->cm_data = NULL;
3218	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
3219	tm->cm_complete = mpssas_resetdev_complete;
3220	tm->cm_complete_data = ccb;
3221	tm->cm_targ = targ;
3222	targ->flags |= MPSSAS_TARGET_INRESET;
3223
3224	mps_map_command(sc, tm);
3225}
3226
3227static void
3228mpssas_resetdev_complete(struct mps_softc *sc, struct mps_command *tm)
3229{
3230	MPI2_SCSI_TASK_MANAGE_REPLY *resp;
3231	union ccb *ccb;
3232
3233	MPS_FUNCTRACE(sc);
3234	mtx_assert(&sc->mps_mtx, MA_OWNED);
3235
3236	resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
3237	ccb = tm->cm_complete_data;
3238
3239	/*
3240	 * Currently there should be no way we can hit this case.  It only
3241	 * happens when we have a failure to allocate chain frames, and
3242	 * task management commands don't have S/G lists.
3243	 */
3244	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3245		MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3246
3247		req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3248
3249		mps_dprint(sc, MPS_ERROR,
3250			   "%s: cm_flags = %#x for reset of handle %#04x! "
3251			   "This should not happen!\n", __func__, tm->cm_flags,
3252			   req->DevHandle);
3253		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3254		goto bailout;
3255	}
3256
3257	mps_dprint(sc, MPS_XINFO,
3258	    "%s: IOCStatus = 0x%x ResponseCode = 0x%x\n", __func__,
3259	    le16toh(resp->IOCStatus), le32toh(resp->ResponseCode));
3260
3261	if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) {
3262		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
3263		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
3264		    CAM_LUN_WILDCARD);
3265	}
3266	else
3267		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3268
3269bailout:
3270
3271	mpssas_free_tm(sc, tm);
3272	xpt_done(ccb);
3273}
3274
3275static void
3276mpssas_poll(struct cam_sim *sim)
3277{
3278	struct mpssas_softc *sassc;
3279
3280	sassc = cam_sim_softc(sim);
3281
3282	if (sassc->sc->mps_debug & MPS_TRACE) {
3283		/* frequent debug messages during a panic just slow
3284		 * everything down too much.
3285		 */
3286		mps_printf(sassc->sc, "%s clearing MPS_TRACE\n", __func__);
3287		sassc->sc->mps_debug &= ~MPS_TRACE;
3288	}
3289
3290	mps_intr_locked(sassc->sc);
3291}
3292
3293static void
3294mpssas_async(void *callback_arg, uint32_t code, struct cam_path *path,
3295	     void *arg)
3296{
3297	struct mps_softc *sc;
3298
3299	sc = (struct mps_softc *)callback_arg;
3300
3301	switch (code) {
3302#if (__FreeBSD_version >= 1000006) || \
3303    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
3304	case AC_ADVINFO_CHANGED: {
3305		struct mpssas_target *target;
3306		struct mpssas_softc *sassc;
3307		struct scsi_read_capacity_data_long rcap_buf;
3308		struct ccb_dev_advinfo cdai;
3309		struct mpssas_lun *lun;
3310		lun_id_t lunid;
3311		int found_lun;
3312		uintptr_t buftype;
3313
3314		buftype = (uintptr_t)arg;
3315
3316		found_lun = 0;
3317		sassc = sc->sassc;
3318
3319		/*
3320		 * We're only interested in read capacity data changes.
3321		 */
3322		if (buftype != CDAI_TYPE_RCAPLONG)
3323			break;
3324
3325		/*
3326		 * We should have a handle for this, but check to make sure.
3327		 */
3328		KASSERT(xpt_path_target_id(path) < sassc->maxtargets,
3329		    ("Target %d out of bounds in mpssas_async\n",
3330		    xpt_path_target_id(path)));
3331		target = &sassc->targets[xpt_path_target_id(path)];
3332		if (target->handle == 0)
3333			break;
3334
3335		lunid = xpt_path_lun_id(path);
3336
3337		SLIST_FOREACH(lun, &target->luns, lun_link) {
3338			if (lun->lun_id == lunid) {
3339				found_lun = 1;
3340				break;
3341			}
3342		}
3343
3344		if (found_lun == 0) {
3345			lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3346				     M_NOWAIT | M_ZERO);
3347			if (lun == NULL) {
3348				mps_dprint(sc, MPS_ERROR, "Unable to alloc "
3349					   "LUN for EEDP support.\n");
3350				break;
3351			}
3352			lun->lun_id = lunid;
3353			SLIST_INSERT_HEAD(&target->luns, lun, lun_link);
3354		}
3355
3356		bzero(&rcap_buf, sizeof(rcap_buf));
3357		xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
3358		cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
3359		cdai.ccb_h.flags = CAM_DIR_IN;
3360		cdai.buftype = CDAI_TYPE_RCAPLONG;
3361#if (__FreeBSD_version >= 1100061) || \
3362    ((__FreeBSD_version >= 1001510) && (__FreeBSD_version < 1100000))
3363		cdai.flags = CDAI_FLAG_NONE;
3364#else
3365		cdai.flags = 0;
3366#endif
3367		cdai.bufsiz = sizeof(rcap_buf);
3368		cdai.buf = (uint8_t *)&rcap_buf;
3369		xpt_action((union ccb *)&cdai);
3370		if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
3371			cam_release_devq(cdai.ccb_h.path,
3372					 0, 0, 0, FALSE);
3373
3374		if ((mpssas_get_ccbstatus((union ccb *)&cdai) == CAM_REQ_CMP)
3375		 && (rcap_buf.prot & SRC16_PROT_EN)) {
3376			lun->eedp_formatted = TRUE;
3377			lun->eedp_block_size = scsi_4btoul(rcap_buf.length);
3378		} else {
3379			lun->eedp_formatted = FALSE;
3380			lun->eedp_block_size = 0;
3381		}
3382		break;
3383	}
3384#else
3385	case AC_FOUND_DEVICE: {
3386		struct ccb_getdev *cgd;
3387
3388		cgd = arg;
3389		mpssas_check_eedp(sc, path, cgd);
3390		break;
3391	}
3392#endif
3393	default:
3394		break;
3395	}
3396}
3397
3398#if (__FreeBSD_version < 901503) || \
3399    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
3400static void
3401mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
3402		  struct ccb_getdev *cgd)
3403{
3404	struct mpssas_softc *sassc = sc->sassc;
3405	struct ccb_scsiio *csio;
3406	struct scsi_read_capacity_16 *scsi_cmd;
3407	struct scsi_read_capacity_eedp *rcap_buf;
3408	path_id_t pathid;
3409	target_id_t targetid;
3410	lun_id_t lunid;
3411	union ccb *ccb;
3412	struct cam_path *local_path;
3413	struct mpssas_target *target;
3414	struct mpssas_lun *lun;
3415	uint8_t	found_lun;
3416	char path_str[64];
3417
3418	sassc = sc->sassc;
3419	pathid = cam_sim_path(sassc->sim);
3420	targetid = xpt_path_target_id(path);
3421	lunid = xpt_path_lun_id(path);
3422
3423	KASSERT(targetid < sassc->maxtargets,
3424	    ("Target %d out of bounds in mpssas_check_eedp\n",
3425	     targetid));
3426	target = &sassc->targets[targetid];
3427	if (target->handle == 0x0)
3428		return;
3429
3430	/*
3431	 * Determine if the device is EEDP capable.
3432	 *
3433	 * If this flag is set in the inquiry data,
3434	 * the device supports protection information,
3435	 * and must support the 16 byte read
3436	 * capacity command, otherwise continue without
3437	 * sending read cap 16
3438	 */
3439	if ((cgd->inq_data.spc3_flags & SPC3_SID_PROTECT) == 0)
3440		return;
3441
3442	/*
3443	 * Issue a READ CAPACITY 16 command.  This info
3444	 * is used to determine if the LUN is formatted
3445	 * for EEDP support.
3446	 */
3447	ccb = xpt_alloc_ccb_nowait();
3448	if (ccb == NULL) {
3449		mps_dprint(sc, MPS_ERROR, "Unable to alloc CCB "
3450		    "for EEDP support.\n");
3451		return;
3452	}
3453
3454	if (xpt_create_path(&local_path, xpt_periph,
3455	    pathid, targetid, lunid) != CAM_REQ_CMP) {
3456		mps_dprint(sc, MPS_ERROR, "Unable to create "
3457		    "path for EEDP support\n");
3458		xpt_free_ccb(ccb);
3459		return;
3460	}
3461
3462	/*
3463	 * If LUN is already in list, don't create a new
3464	 * one.
3465	 */
3466	found_lun = FALSE;
3467	SLIST_FOREACH(lun, &target->luns, lun_link) {
3468		if (lun->lun_id == lunid) {
3469			found_lun = TRUE;
3470			break;
3471		}
3472	}
3473	if (!found_lun) {
3474		lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3475		    M_NOWAIT | M_ZERO);
3476		if (lun == NULL) {
3477			mps_dprint(sc, MPS_ERROR,
3478			    "Unable to alloc LUN for EEDP support.\n");
3479			xpt_free_path(local_path);
3480			xpt_free_ccb(ccb);
3481			return;
3482		}
3483		lun->lun_id = lunid;
3484		SLIST_INSERT_HEAD(&target->luns, lun,
3485		    lun_link);
3486	}
3487
3488	xpt_path_string(local_path, path_str, sizeof(path_str));
3489
3490	mps_dprint(sc, MPS_INFO, "Sending read cap: path %s handle %d\n",
3491	    path_str, target->handle);
3492
3493	/*
3494	 * Issue a READ CAPACITY 16 command for the LUN.
3495	 * The mpssas_read_cap_done function will load
3496	 * the read cap info into the LUN struct.
3497	 */
3498	rcap_buf = malloc(sizeof(struct scsi_read_capacity_eedp),
3499	    M_MPT2, M_NOWAIT | M_ZERO);
3500	if (rcap_buf == NULL) {
3501		mps_dprint(sc, MPS_FAULT,
3502		    "Unable to alloc read capacity buffer for EEDP support.\n");
3503		xpt_free_path(ccb->ccb_h.path);
3504		xpt_free_ccb(ccb);
3505		return;
3506	}
3507	xpt_setup_ccb(&ccb->ccb_h, local_path, CAM_PRIORITY_XPT);
3508	csio = &ccb->csio;
3509	csio->ccb_h.func_code = XPT_SCSI_IO;
3510	csio->ccb_h.flags = CAM_DIR_IN;
3511	csio->ccb_h.retry_count = 4;
3512	csio->ccb_h.cbfcnp = mpssas_read_cap_done;
3513	csio->ccb_h.timeout = 60000;
3514	csio->data_ptr = (uint8_t *)rcap_buf;
3515	csio->dxfer_len = sizeof(struct scsi_read_capacity_eedp);
3516	csio->sense_len = MPS_SENSE_LEN;
3517	csio->cdb_len = sizeof(*scsi_cmd);
3518	csio->tag_action = MSG_SIMPLE_Q_TAG;
3519
3520	scsi_cmd = (struct scsi_read_capacity_16 *)&csio->cdb_io.cdb_bytes;
3521	bzero(scsi_cmd, sizeof(*scsi_cmd));
3522	scsi_cmd->opcode = 0x9E;
3523	scsi_cmd->service_action = SRC16_SERVICE_ACTION;
3524	((uint8_t *)scsi_cmd)[13] = sizeof(struct scsi_read_capacity_eedp);
3525
3526	ccb->ccb_h.ppriv_ptr1 = sassc;
3527	xpt_action(ccb);
3528}
3529
3530static void
3531mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb)
3532{
3533	struct mpssas_softc *sassc;
3534	struct mpssas_target *target;
3535	struct mpssas_lun *lun;
3536	struct scsi_read_capacity_eedp *rcap_buf;
3537
3538	if (done_ccb == NULL)
3539		return;
3540
3541	/* Driver need to release devq, it Scsi command is
3542	 * generated by driver internally.
3543	 * Currently there is a single place where driver
3544	 * calls scsi command internally. In future if driver
3545	 * calls more scsi command internally, it needs to release
3546	 * devq internally, since those command will not go back to
3547	 * cam_periph.
3548	 */
3549	if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) ) {
3550        	done_ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
3551		xpt_release_devq(done_ccb->ccb_h.path,
3552			       	/*count*/ 1, /*run_queue*/TRUE);
3553	}
3554
3555	rcap_buf = (struct scsi_read_capacity_eedp *)done_ccb->csio.data_ptr;
3556
3557	/*
3558	 * Get the LUN ID for the path and look it up in the LUN list for the
3559	 * target.
3560	 */
3561	sassc = (struct mpssas_softc *)done_ccb->ccb_h.ppriv_ptr1;
3562	KASSERT(done_ccb->ccb_h.target_id < sassc->maxtargets,
3563	    ("Target %d out of bounds in mpssas_read_cap_done\n",
3564	     done_ccb->ccb_h.target_id));
3565	target = &sassc->targets[done_ccb->ccb_h.target_id];
3566	SLIST_FOREACH(lun, &target->luns, lun_link) {
3567		if (lun->lun_id != done_ccb->ccb_h.target_lun)
3568			continue;
3569
3570		/*
3571		 * Got the LUN in the target's LUN list.  Fill it in
3572		 * with EEDP info.  If the READ CAP 16 command had some
3573		 * SCSI error (common if command is not supported), mark
3574		 * the lun as not supporting EEDP and set the block size
3575		 * to 0.
3576		 */
3577		if ((mpssas_get_ccbstatus(done_ccb) != CAM_REQ_CMP)
3578		 || (done_ccb->csio.scsi_status != SCSI_STATUS_OK)) {
3579			lun->eedp_formatted = FALSE;
3580			lun->eedp_block_size = 0;
3581			break;
3582		}
3583
3584		if (rcap_buf->protect & 0x01) {
3585			mps_dprint(sassc->sc, MPS_INFO, "LUN %d for "
3586 			    "target ID %d is formatted for EEDP "
3587 			    "support.\n", done_ccb->ccb_h.target_lun,
3588 			    done_ccb->ccb_h.target_id);
3589			lun->eedp_formatted = TRUE;
3590			lun->eedp_block_size = scsi_4btoul(rcap_buf->length);
3591		}
3592		break;
3593	}
3594
3595	// Finished with this CCB and path.
3596	free(rcap_buf, M_MPT2);
3597	xpt_free_path(done_ccb->ccb_h.path);
3598	xpt_free_ccb(done_ccb);
3599}
3600#endif /* (__FreeBSD_version < 901503) || \
3601          ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006)) */
3602
3603void
3604mpssas_prepare_for_tm(struct mps_softc *sc, struct mps_command *tm,
3605    struct mpssas_target *target, lun_id_t lun_id)
3606{
3607	union ccb *ccb;
3608	path_id_t path_id;
3609
3610	/*
3611	 * Set the INRESET flag for this target so that no I/O will be sent to
3612	 * the target until the reset has completed.  If an I/O request does
3613	 * happen, the devq will be frozen.  The CCB holds the path which is
3614	 * used to release the devq.  The devq is released and the CCB is freed
3615	 * when the TM completes.
3616	 */
3617	ccb = xpt_alloc_ccb_nowait();
3618	if (ccb) {
3619		path_id = cam_sim_path(sc->sassc->sim);
3620		if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, path_id,
3621		    target->tid, lun_id) != CAM_REQ_CMP) {
3622			xpt_free_ccb(ccb);
3623		} else {
3624			tm->cm_ccb = ccb;
3625			tm->cm_targ = target;
3626			target->flags |= MPSSAS_TARGET_INRESET;
3627		}
3628	}
3629}
3630
3631int
3632mpssas_startup(struct mps_softc *sc)
3633{
3634
3635	/*
3636	 * Send the port enable message and set the wait_for_port_enable flag.
3637	 * This flag helps to keep the simq frozen until all discovery events
3638	 * are processed.
3639	 */
3640	sc->wait_for_port_enable = 1;
3641	mpssas_send_portenable(sc);
3642	return (0);
3643}
3644
3645static int
3646mpssas_send_portenable(struct mps_softc *sc)
3647{
3648	MPI2_PORT_ENABLE_REQUEST *request;
3649	struct mps_command *cm;
3650
3651	MPS_FUNCTRACE(sc);
3652
3653	if ((cm = mps_alloc_command(sc)) == NULL)
3654		return (EBUSY);
3655	request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req;
3656	request->Function = MPI2_FUNCTION_PORT_ENABLE;
3657	request->MsgFlags = 0;
3658	request->VP_ID = 0;
3659	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3660	cm->cm_complete = mpssas_portenable_complete;
3661	cm->cm_data = NULL;
3662	cm->cm_sge = NULL;
3663
3664	mps_map_command(sc, cm);
3665	mps_dprint(sc, MPS_XINFO,
3666	    "mps_send_portenable finished cm %p req %p complete %p\n",
3667	    cm, cm->cm_req, cm->cm_complete);
3668	return (0);
3669}
3670
3671static void
3672mpssas_portenable_complete(struct mps_softc *sc, struct mps_command *cm)
3673{
3674	MPI2_PORT_ENABLE_REPLY *reply;
3675	struct mpssas_softc *sassc;
3676
3677	MPS_FUNCTRACE(sc);
3678	sassc = sc->sassc;
3679
3680	/*
3681	 * Currently there should be no way we can hit this case.  It only
3682	 * happens when we have a failure to allocate chain frames, and
3683	 * port enable commands don't have S/G lists.
3684	 */
3685	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3686		mps_dprint(sc, MPS_ERROR, "%s: cm_flags = %#x for port enable! "
3687			   "This should not happen!\n", __func__, cm->cm_flags);
3688	}
3689
3690	reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply;
3691	if (reply == NULL)
3692		mps_dprint(sc, MPS_FAULT, "Portenable NULL reply\n");
3693	else if (le16toh(reply->IOCStatus & MPI2_IOCSTATUS_MASK) !=
3694	    MPI2_IOCSTATUS_SUCCESS)
3695		mps_dprint(sc, MPS_FAULT, "Portenable failed\n");
3696
3697	mps_free_command(sc, cm);
3698	if (sc->mps_ich.ich_arg != NULL) {
3699		mps_dprint(sc, MPS_XINFO, "disestablish config intrhook\n");
3700		config_intrhook_disestablish(&sc->mps_ich);
3701		sc->mps_ich.ich_arg = NULL;
3702	}
3703
3704	/*
3705	 * Get WarpDrive info after discovery is complete but before the scan
3706	 * starts.  At this point, all devices are ready to be exposed to the
3707	 * OS.  If devices should be hidden instead, take them out of the
3708	 * 'targets' array before the scan.  The devinfo for a disk will have
3709	 * some info and a volume's will be 0.  Use that to remove disks.
3710	 */
3711	mps_wd_config_pages(sc);
3712
3713	/*
3714	 * Done waiting for port enable to complete.  Decrement the refcount.
3715	 * If refcount is 0, discovery is complete and a rescan of the bus can
3716	 * take place.  Since the simq was explicitly frozen before port
3717	 * enable, it must be explicitly released here to keep the
3718	 * freeze/release count in sync.
3719	 */
3720	sc->wait_for_port_enable = 0;
3721	sc->port_enable_complete = 1;
3722	wakeup(&sc->port_enable_complete);
3723	mpssas_startup_decrement(sassc);
3724}
3725
3726int
3727mpssas_check_id(struct mpssas_softc *sassc, int id)
3728{
3729	struct mps_softc *sc = sassc->sc;
3730	char *ids;
3731	char *name;
3732
3733	ids = &sc->exclude_ids[0];
3734	while((name = strsep(&ids, ",")) != NULL) {
3735		if (name[0] == '\0')
3736			continue;
3737		if (strtol(name, NULL, 0) == (long)id)
3738			return (1);
3739	}
3740
3741	return (0);
3742}
3743
3744void
3745mpssas_realloc_targets(struct mps_softc *sc, int maxtargets)
3746{
3747	struct mpssas_softc *sassc;
3748	struct mpssas_lun *lun, *lun_tmp;
3749	struct mpssas_target *targ;
3750	int i;
3751
3752	sassc = sc->sassc;
3753	/*
3754	 * The number of targets is based on IOC Facts, so free all of
3755	 * the allocated LUNs for each target and then the target buffer
3756	 * itself.
3757	 */
3758	for (i=0; i< maxtargets; i++) {
3759		targ = &sassc->targets[i];
3760		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
3761			free(lun, M_MPT2);
3762		}
3763	}
3764	free(sassc->targets, M_MPT2);
3765
3766	sassc->targets = malloc(sizeof(struct mpssas_target) * maxtargets,
3767	    M_MPT2, M_WAITOK|M_ZERO);
3768	if (!sassc->targets) {
3769		panic("%s failed to alloc targets with error %d\n",
3770		    __func__, ENOMEM);
3771	}
3772}
3773