mps_sas.c revision bac013c4c02c1a9babf2c79cf0b5473fe65eca3a
1/*-
2 * Copyright (c) 2009 Yahoo! Inc.
3 * Copyright (c) 2011-2015 LSI Corp.
4 * Copyright (c) 2013-2015 Avago Technologies
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
29 *
30 * $FreeBSD$
31 */
32
33#include <sys/cdefs.h>
34__FBSDID("$FreeBSD$");
35
36/* Communications core for Avago Technologies (LSI) MPT2 */
37
38/* TODO Move headers to mpsvar */
39#include <sys/types.h>
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/kernel.h>
43#include <sys/selinfo.h>
44#include <sys/module.h>
45#include <sys/bus.h>
46#include <sys/conf.h>
47#include <sys/bio.h>
48#include <sys/malloc.h>
49#include <sys/uio.h>
50#include <sys/sysctl.h>
51#include <sys/endian.h>
52#include <sys/queue.h>
53#include <sys/kthread.h>
54#include <sys/taskqueue.h>
55#include <sys/sbuf.h>
56
57#include <machine/bus.h>
58#include <machine/resource.h>
59#include <sys/rman.h>
60
61#include <machine/stdarg.h>
62
63#include <cam/cam.h>
64#include <cam/cam_ccb.h>
65#include <cam/cam_xpt.h>
66#include <cam/cam_debug.h>
67#include <cam/cam_sim.h>
68#include <cam/cam_xpt_sim.h>
69#include <cam/cam_xpt_periph.h>
70#include <cam/cam_periph.h>
71#include <cam/scsi/scsi_all.h>
72#include <cam/scsi/scsi_message.h>
73#if __FreeBSD_version >= 900026
74#include <cam/scsi/smp_all.h>
75#endif
76
77#include <dev/mps/mpi/mpi2_type.h>
78#include <dev/mps/mpi/mpi2.h>
79#include <dev/mps/mpi/mpi2_ioc.h>
80#include <dev/mps/mpi/mpi2_sas.h>
81#include <dev/mps/mpi/mpi2_cnfg.h>
82#include <dev/mps/mpi/mpi2_init.h>
83#include <dev/mps/mpi/mpi2_tool.h>
84#include <dev/mps/mps_ioctl.h>
85#include <dev/mps/mpsvar.h>
86#include <dev/mps/mps_table.h>
87#include <dev/mps/mps_sas.h>
88
89#define MPSSAS_DISCOVERY_TIMEOUT	20
90#define MPSSAS_MAX_DISCOVERY_TIMEOUTS	10 /* 200 seconds */
91
92/*
93 * static array to check SCSI OpCode for EEDP protection bits
94 */
95#define	PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP
96#define	PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
97#define	PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
98static uint8_t op_code_prot[256] = {
99	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
100	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
101	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
102	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
103	0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
104	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
105	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
106	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
107	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
108	0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
109	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
110	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
111	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
112	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
113	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
114	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
115};
116
117MALLOC_DEFINE(M_MPSSAS, "MPSSAS", "MPS SAS memory");
118
119static void mpssas_remove_device(struct mps_softc *, struct mps_command *);
120static void mpssas_remove_complete(struct mps_softc *, struct mps_command *);
121static void mpssas_action(struct cam_sim *sim, union ccb *ccb);
122static void mpssas_poll(struct cam_sim *sim);
123static int mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm,
124    struct mps_command *cm);
125static void mpssas_scsiio_timeout(void *data);
126static void mpssas_abort_complete(struct mps_softc *sc, struct mps_command *cm);
127static void mpssas_direct_drive_io(struct mpssas_softc *sassc,
128    struct mps_command *cm, union ccb *ccb);
129static void mpssas_action_scsiio(struct mpssas_softc *, union ccb *);
130static void mpssas_scsiio_complete(struct mps_softc *, struct mps_command *);
131static void mpssas_action_resetdev(struct mpssas_softc *, union ccb *);
132#if __FreeBSD_version >= 900026
133static void mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm);
134static void mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb,
135			       uint64_t sasaddr);
136static void mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb);
137#endif //FreeBSD_version >= 900026
138static void mpssas_resetdev_complete(struct mps_softc *, struct mps_command *);
139static void mpssas_async(void *callback_arg, uint32_t code,
140			 struct cam_path *path, void *arg);
141#if (__FreeBSD_version < 901503) || \
142    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
143static void mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
144			      struct ccb_getdev *cgd);
145static void mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb);
146#endif
147static int mpssas_send_portenable(struct mps_softc *sc);
148static void mpssas_portenable_complete(struct mps_softc *sc,
149    struct mps_command *cm);
150
151struct mpssas_target *
152mpssas_find_target_by_handle(struct mpssas_softc *sassc, int start, uint16_t handle)
153{
154	struct mpssas_target *target;
155	int i;
156
157	for (i = start; i < sassc->maxtargets; i++) {
158		target = &sassc->targets[i];
159		if (target->handle == handle)
160			return (target);
161	}
162
163	return (NULL);
164}
165
166/* we need to freeze the simq during attach and diag reset, to avoid failing
167 * commands before device handles have been found by discovery.  Since
168 * discovery involves reading config pages and possibly sending commands,
169 * discovery actions may continue even after we receive the end of discovery
170 * event, so refcount discovery actions instead of assuming we can unfreeze
171 * the simq when we get the event.
172 */
173void
174mpssas_startup_increment(struct mpssas_softc *sassc)
175{
176	MPS_FUNCTRACE(sassc->sc);
177
178	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
179		if (sassc->startup_refcount++ == 0) {
180			/* just starting, freeze the simq */
181			mps_dprint(sassc->sc, MPS_INIT,
182			    "%s freezing simq\n", __func__);
183#if __FreeBSD_version >= 1000039
184			xpt_hold_boot();
185#endif
186			xpt_freeze_simq(sassc->sim, 1);
187		}
188		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
189		    sassc->startup_refcount);
190	}
191}
192
193void
194mpssas_release_simq_reinit(struct mpssas_softc *sassc)
195{
196	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
197		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
198		xpt_release_simq(sassc->sim, 1);
199		mps_dprint(sassc->sc, MPS_INFO, "Unfreezing SIM queue\n");
200	}
201}
202
203void
204mpssas_startup_decrement(struct mpssas_softc *sassc)
205{
206	MPS_FUNCTRACE(sassc->sc);
207
208	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
209		if (--sassc->startup_refcount == 0) {
210			/* finished all discovery-related actions, release
211			 * the simq and rescan for the latest topology.
212			 */
213			mps_dprint(sassc->sc, MPS_INIT,
214			    "%s releasing simq\n", __func__);
215			sassc->flags &= ~MPSSAS_IN_STARTUP;
216			xpt_release_simq(sassc->sim, 1);
217#if __FreeBSD_version >= 1000039
218			xpt_release_boot();
219#else
220			mpssas_rescan_target(sassc->sc, NULL);
221#endif
222		}
223		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
224		    sassc->startup_refcount);
225	}
226}
227
228/* The firmware requires us to stop sending commands when we're doing task
229 * management, so refcount the TMs and keep the simq frozen when any are in
230 * use.
231 */
232struct mps_command *
233mpssas_alloc_tm(struct mps_softc *sc)
234{
235	struct mps_command *tm;
236
237	tm = mps_alloc_high_priority_command(sc);
238	return tm;
239}
240
241void
242mpssas_free_tm(struct mps_softc *sc, struct mps_command *tm)
243{
244	int target_id = 0xFFFFFFFF;
245
246	if (tm == NULL)
247		return;
248
249	/*
250	 * For TM's the devq is frozen for the device.  Unfreeze it here and
251	 * free the resources used for freezing the devq.  Must clear the
252	 * INRESET flag as well or scsi I/O will not work.
253	 */
254	if (tm->cm_targ != NULL) {
255		tm->cm_targ->flags &= ~MPSSAS_TARGET_INRESET;
256		target_id = tm->cm_targ->tid;
257	}
258	if (tm->cm_ccb) {
259		mps_dprint(sc, MPS_INFO, "Unfreezing devq for target ID %d\n",
260		    target_id);
261		xpt_release_devq(tm->cm_ccb->ccb_h.path, 1, TRUE);
262		xpt_free_path(tm->cm_ccb->ccb_h.path);
263		xpt_free_ccb(tm->cm_ccb);
264	}
265
266	mps_free_high_priority_command(sc, tm);
267}
268
269void
270mpssas_rescan_target(struct mps_softc *sc, struct mpssas_target *targ)
271{
272	struct mpssas_softc *sassc = sc->sassc;
273	path_id_t pathid;
274	target_id_t targetid;
275	union ccb *ccb;
276
277	MPS_FUNCTRACE(sc);
278	pathid = cam_sim_path(sassc->sim);
279	if (targ == NULL)
280		targetid = CAM_TARGET_WILDCARD;
281	else
282		targetid = targ - sassc->targets;
283
284	/*
285	 * Allocate a CCB and schedule a rescan.
286	 */
287	ccb = xpt_alloc_ccb_nowait();
288	if (ccb == NULL) {
289		mps_dprint(sc, MPS_ERROR, "unable to alloc CCB for rescan\n");
290		return;
291	}
292
293	if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
294	    targetid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
295		mps_dprint(sc, MPS_ERROR, "unable to create path for rescan\n");
296		xpt_free_ccb(ccb);
297		return;
298	}
299
300	if (targetid == CAM_TARGET_WILDCARD)
301		ccb->ccb_h.func_code = XPT_SCAN_BUS;
302	else
303		ccb->ccb_h.func_code = XPT_SCAN_TGT;
304
305	mps_dprint(sc, MPS_TRACE, "%s targetid %u\n", __func__, targetid);
306	xpt_rescan(ccb);
307}
308
309static void
310mpssas_log_command(struct mps_command *cm, u_int level, const char *fmt, ...)
311{
312	struct sbuf sb;
313	va_list ap;
314	char str[192];
315	char path_str[64];
316
317	if (cm == NULL)
318		return;
319
320	/* No need to be in here if debugging isn't enabled */
321	if ((cm->cm_sc->mps_debug & level) == 0)
322		return;
323
324	sbuf_new(&sb, str, sizeof(str), 0);
325
326	va_start(ap, fmt);
327
328	if (cm->cm_ccb != NULL) {
329		xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str,
330				sizeof(path_str));
331		sbuf_cat(&sb, path_str);
332		if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) {
333			scsi_command_string(&cm->cm_ccb->csio, &sb);
334			sbuf_printf(&sb, "length %d ",
335				    cm->cm_ccb->csio.dxfer_len);
336		}
337	}
338	else {
339		sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ",
340		    cam_sim_name(cm->cm_sc->sassc->sim),
341		    cam_sim_unit(cm->cm_sc->sassc->sim),
342		    cam_sim_bus(cm->cm_sc->sassc->sim),
343		    cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF,
344		    cm->cm_lun);
345	}
346
347	sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID);
348	sbuf_vprintf(&sb, fmt, ap);
349	sbuf_finish(&sb);
350	mps_print_field(cm->cm_sc, "%s", sbuf_data(&sb));
351
352	va_end(ap);
353}
354
355
356static void
357mpssas_remove_volume(struct mps_softc *sc, struct mps_command *tm)
358{
359	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
360	struct mpssas_target *targ;
361	uint16_t handle;
362
363	MPS_FUNCTRACE(sc);
364
365	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
366	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
367	targ = tm->cm_targ;
368
369	if (reply == NULL) {
370		/* XXX retry the remove after the diag reset completes? */
371		mps_dprint(sc, MPS_FAULT,
372		    "%s NULL reply resetting device 0x%04x\n", __func__,
373		    handle);
374		mpssas_free_tm(sc, tm);
375		return;
376	}
377
378	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
379	    MPI2_IOCSTATUS_SUCCESS) {
380		mps_dprint(sc, MPS_ERROR,
381		   "IOCStatus = 0x%x while resetting device 0x%x\n",
382		   le16toh(reply->IOCStatus), handle);
383	}
384
385	mps_dprint(sc, MPS_XINFO,
386	    "Reset aborted %u commands\n", reply->TerminationCount);
387	mps_free_reply(sc, tm->cm_reply_data);
388	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
389
390	mps_dprint(sc, MPS_XINFO,
391	    "clearing target %u handle 0x%04x\n", targ->tid, handle);
392
393	/*
394	 * Don't clear target if remove fails because things will get confusing.
395	 * Leave the devname and sasaddr intact so that we know to avoid reusing
396	 * this target id if possible, and so we can assign the same target id
397	 * to this device if it comes back in the future.
398	 */
399	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
400	    MPI2_IOCSTATUS_SUCCESS) {
401		targ = tm->cm_targ;
402		targ->handle = 0x0;
403		targ->encl_handle = 0x0;
404		targ->encl_slot = 0x0;
405		targ->exp_dev_handle = 0x0;
406		targ->phy_num = 0x0;
407		targ->linkrate = 0x0;
408		targ->devinfo = 0x0;
409		targ->flags = 0x0;
410	}
411
412	mpssas_free_tm(sc, tm);
413}
414
415
416/*
417 * No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal.
418 * Otherwise Volume Delete is same as Bare Drive Removal.
419 */
420void
421mpssas_prepare_volume_remove(struct mpssas_softc *sassc, uint16_t handle)
422{
423	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
424	struct mps_softc *sc;
425	struct mps_command *cm;
426	struct mpssas_target *targ = NULL;
427
428	MPS_FUNCTRACE(sassc->sc);
429	sc = sassc->sc;
430
431#ifdef WD_SUPPORT
432	/*
433	 * If this is a WD controller, determine if the disk should be exposed
434	 * to the OS or not.  If disk should be exposed, return from this
435	 * function without doing anything.
436	 */
437	if (sc->WD_available && (sc->WD_hide_expose ==
438	    MPS_WD_EXPOSE_ALWAYS)) {
439		return;
440	}
441#endif //WD_SUPPORT
442
443	targ = mpssas_find_target_by_handle(sassc, 0, handle);
444	if (targ == NULL) {
445		/* FIXME: what is the action? */
446		/* We don't know about this device? */
447		mps_dprint(sc, MPS_ERROR,
448		   "%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
449		return;
450	}
451
452	targ->flags |= MPSSAS_TARGET_INREMOVAL;
453
454	cm = mpssas_alloc_tm(sc);
455	if (cm == NULL) {
456		mps_dprint(sc, MPS_ERROR,
457		    "%s: command alloc failure\n", __func__);
458		return;
459	}
460
461	mpssas_rescan_target(sc, targ);
462
463	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
464	req->DevHandle = targ->handle;
465	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
466	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
467
468	/* SAS Hard Link Reset / SATA Link Reset */
469	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
470
471	cm->cm_targ = targ;
472	cm->cm_data = NULL;
473	cm->cm_desc.HighPriority.RequestFlags =
474	    MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
475	cm->cm_complete = mpssas_remove_volume;
476	cm->cm_complete_data = (void *)(uintptr_t)handle;
477
478	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
479	    __func__, targ->tid);
480	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
481
482	mps_map_command(sc, cm);
483}
484
485/*
486 * The MPT2 firmware performs debounce on the link to avoid transient link
487 * errors and false removals.  When it does decide that link has been lost
488 * and a device need to go away, it expects that the host will perform a
489 * target reset and then an op remove.  The reset has the side-effect of
490 * aborting any outstanding requests for the device, which is required for
491 * the op-remove to succeed.  It's not clear if the host should check for
492 * the device coming back alive after the reset.
493 */
494void
495mpssas_prepare_remove(struct mpssas_softc *sassc, uint16_t handle)
496{
497	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
498	struct mps_softc *sc;
499	struct mps_command *cm;
500	struct mpssas_target *targ = NULL;
501
502	MPS_FUNCTRACE(sassc->sc);
503
504	sc = sassc->sc;
505
506	targ = mpssas_find_target_by_handle(sassc, 0, handle);
507	if (targ == NULL) {
508		/* FIXME: what is the action? */
509		/* We don't know about this device? */
510		mps_dprint(sc, MPS_ERROR,
511		    "%s : invalid handle 0x%x \n", __func__, handle);
512		return;
513	}
514
515	targ->flags |= MPSSAS_TARGET_INREMOVAL;
516
517	cm = mpssas_alloc_tm(sc);
518	if (cm == NULL) {
519		mps_dprint(sc, MPS_ERROR,
520		    "%s: command alloc failure\n", __func__);
521		return;
522	}
523
524	mpssas_rescan_target(sc, targ);
525
526	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
527	memset(req, 0, sizeof(*req));
528	req->DevHandle = htole16(targ->handle);
529	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
530	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
531
532	/* SAS Hard Link Reset / SATA Link Reset */
533	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
534
535	cm->cm_targ = targ;
536	cm->cm_data = NULL;
537	cm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
538	cm->cm_complete = mpssas_remove_device;
539	cm->cm_complete_data = (void *)(uintptr_t)handle;
540
541	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
542	    __func__, targ->tid);
543	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
544
545	mps_map_command(sc, cm);
546}
547
548static void
549mpssas_remove_device(struct mps_softc *sc, struct mps_command *tm)
550{
551	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
552	MPI2_SAS_IOUNIT_CONTROL_REQUEST *req;
553	struct mpssas_target *targ;
554	struct mps_command *next_cm;
555	uint16_t handle;
556
557	MPS_FUNCTRACE(sc);
558
559	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
560	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
561	targ = tm->cm_targ;
562
563	/*
564	 * Currently there should be no way we can hit this case.  It only
565	 * happens when we have a failure to allocate chain frames, and
566	 * task management commands don't have S/G lists.
567	 */
568	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
569		mps_dprint(sc, MPS_ERROR,
570		    "%s: cm_flags = %#x for remove of handle %#04x! "
571		    "This should not happen!\n", __func__, tm->cm_flags,
572		    handle);
573	}
574
575	if (reply == NULL) {
576		/* XXX retry the remove after the diag reset completes? */
577		mps_dprint(sc, MPS_FAULT,
578		    "%s NULL reply resetting device 0x%04x\n", __func__,
579		    handle);
580		mpssas_free_tm(sc, tm);
581		return;
582	}
583
584	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
585	    MPI2_IOCSTATUS_SUCCESS) {
586		mps_dprint(sc, MPS_ERROR,
587		   "IOCStatus = 0x%x while resetting device 0x%x\n",
588		   le16toh(reply->IOCStatus), handle);
589	}
590
591	mps_dprint(sc, MPS_XINFO, "Reset aborted %u commands\n",
592	    le32toh(reply->TerminationCount));
593	mps_free_reply(sc, tm->cm_reply_data);
594	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
595
596	/* Reuse the existing command */
597	req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req;
598	memset(req, 0, sizeof(*req));
599	req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
600	req->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
601	req->DevHandle = htole16(handle);
602	tm->cm_data = NULL;
603	tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
604	tm->cm_complete = mpssas_remove_complete;
605	tm->cm_complete_data = (void *)(uintptr_t)handle;
606
607	mps_map_command(sc, tm);
608
609	mps_dprint(sc, MPS_XINFO, "clearing target %u handle 0x%04x\n",
610		   targ->tid, handle);
611	TAILQ_FOREACH_SAFE(tm, &targ->commands, cm_link, next_cm) {
612		union ccb *ccb;
613
614		mps_dprint(sc, MPS_XINFO, "Completing missed command %p\n", tm);
615		ccb = tm->cm_complete_data;
616		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
617		mpssas_scsiio_complete(sc, tm);
618	}
619}
620
621static void
622mpssas_remove_complete(struct mps_softc *sc, struct mps_command *tm)
623{
624	MPI2_SAS_IOUNIT_CONTROL_REPLY *reply;
625	uint16_t handle;
626	struct mpssas_target *targ;
627	struct mpssas_lun *lun;
628
629	MPS_FUNCTRACE(sc);
630
631	reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply;
632	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
633
634	/*
635	 * Currently there should be no way we can hit this case.  It only
636	 * happens when we have a failure to allocate chain frames, and
637	 * task management commands don't have S/G lists.
638	 */
639	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
640		mps_dprint(sc, MPS_XINFO,
641			   "%s: cm_flags = %#x for remove of handle %#04x! "
642			   "This should not happen!\n", __func__, tm->cm_flags,
643			   handle);
644		mpssas_free_tm(sc, tm);
645		return;
646	}
647
648	if (reply == NULL) {
649		/* most likely a chip reset */
650		mps_dprint(sc, MPS_FAULT,
651		    "%s NULL reply removing device 0x%04x\n", __func__, handle);
652		mpssas_free_tm(sc, tm);
653		return;
654	}
655
656	mps_dprint(sc, MPS_XINFO,
657	    "%s on handle 0x%04x, IOCStatus= 0x%x\n", __func__,
658	    handle, le16toh(reply->IOCStatus));
659
660	/*
661	 * Don't clear target if remove fails because things will get confusing.
662	 * Leave the devname and sasaddr intact so that we know to avoid reusing
663	 * this target id if possible, and so we can assign the same target id
664	 * to this device if it comes back in the future.
665	 */
666	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
667	    MPI2_IOCSTATUS_SUCCESS) {
668		targ = tm->cm_targ;
669		targ->handle = 0x0;
670		targ->encl_handle = 0x0;
671		targ->encl_slot = 0x0;
672		targ->exp_dev_handle = 0x0;
673		targ->phy_num = 0x0;
674		targ->linkrate = 0x0;
675		targ->devinfo = 0x0;
676		targ->flags = 0x0;
677
678		while(!SLIST_EMPTY(&targ->luns)) {
679			lun = SLIST_FIRST(&targ->luns);
680			SLIST_REMOVE_HEAD(&targ->luns, lun_link);
681			free(lun, M_MPT2);
682		}
683	}
684
685
686	mpssas_free_tm(sc, tm);
687}
688
689static int
690mpssas_register_events(struct mps_softc *sc)
691{
692	u32 events[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];
693
694	bzero(events, 16);
695	setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
696	setbit(events, MPI2_EVENT_SAS_DISCOVERY);
697	setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
698	setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE);
699	setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW);
700	setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
701	setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
702	setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
703	setbit(events, MPI2_EVENT_IR_VOLUME);
704	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
705	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
706	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);
707
708	mps_register_events(sc, events, mpssas_evt_handler, NULL,
709	    &sc->sassc->mpssas_eh);
710
711	return (0);
712}
713
714int
715mps_attach_sas(struct mps_softc *sc)
716{
717	struct mpssas_softc *sassc;
718	cam_status status;
719	int unit, error = 0;
720
721	MPS_FUNCTRACE(sc);
722
723	sassc = malloc(sizeof(struct mpssas_softc), M_MPT2, M_WAITOK|M_ZERO);
724	if(!sassc) {
725		device_printf(sc->mps_dev, "Cannot allocate memory %s %d\n",
726		__func__, __LINE__);
727		return (ENOMEM);
728	}
729
730	/*
731	 * XXX MaxTargets could change during a reinit.  Since we don't
732	 * resize the targets[] array during such an event, cache the value
733	 * of MaxTargets here so that we don't get into trouble later.  This
734	 * should move into the reinit logic.
735	 */
736	sassc->maxtargets = sc->facts->MaxTargets;
737	sassc->targets = malloc(sizeof(struct mpssas_target) *
738	    sassc->maxtargets, M_MPT2, M_WAITOK|M_ZERO);
739	if(!sassc->targets) {
740		device_printf(sc->mps_dev, "Cannot allocate memory %s %d\n",
741		__func__, __LINE__);
742		free(sassc, M_MPT2);
743		return (ENOMEM);
744	}
745	sc->sassc = sassc;
746	sassc->sc = sc;
747
748	if ((sassc->devq = cam_simq_alloc(sc->num_reqs)) == NULL) {
749		mps_dprint(sc, MPS_ERROR, "Cannot allocate SIMQ\n");
750		error = ENOMEM;
751		goto out;
752	}
753
754	unit = device_get_unit(sc->mps_dev);
755	sassc->sim = cam_sim_alloc(mpssas_action, mpssas_poll, "mps", sassc,
756	    unit, &sc->mps_mtx, sc->num_reqs, sc->num_reqs, sassc->devq);
757	if (sassc->sim == NULL) {
758		mps_dprint(sc, MPS_ERROR, "Cannot allocate SIM\n");
759		error = EINVAL;
760		goto out;
761	}
762
763	TAILQ_INIT(&sassc->ev_queue);
764
765	/* Initialize taskqueue for Event Handling */
766	TASK_INIT(&sassc->ev_task, 0, mpssas_firmware_event_work, sc);
767	sassc->ev_tq = taskqueue_create("mps_taskq", M_NOWAIT | M_ZERO,
768	    taskqueue_thread_enqueue, &sassc->ev_tq);
769	taskqueue_start_threads(&sassc->ev_tq, 1, PRIBIO, "%s taskq",
770	    device_get_nameunit(sc->mps_dev));
771
772	mps_lock(sc);
773
774	/*
775	 * XXX There should be a bus for every port on the adapter, but since
776	 * we're just going to fake the topology for now, we'll pretend that
777	 * everything is just a target on a single bus.
778	 */
779	if ((error = xpt_bus_register(sassc->sim, sc->mps_dev, 0)) != 0) {
780		mps_dprint(sc, MPS_ERROR, "Error %d registering SCSI bus\n",
781		    error);
782		mps_unlock(sc);
783		goto out;
784	}
785
786	/*
787	 * Assume that discovery events will start right away.
788	 *
789	 * Hold off boot until discovery is complete.
790	 */
791	sassc->flags |= MPSSAS_IN_STARTUP | MPSSAS_IN_DISCOVERY;
792	sc->sassc->startup_refcount = 0;
793	mpssas_startup_increment(sassc);
794
795	callout_init(&sassc->discovery_callout, 1 /*mpsafe*/);
796
797	/*
798	 * Register for async events so we can determine the EEDP
799	 * capabilities of devices.
800	 */
801	status = xpt_create_path(&sassc->path, /*periph*/NULL,
802	    cam_sim_path(sc->sassc->sim), CAM_TARGET_WILDCARD,
803	    CAM_LUN_WILDCARD);
804	if (status != CAM_REQ_CMP) {
805		mps_printf(sc, "Error %#x creating sim path\n", status);
806		sassc->path = NULL;
807	} else {
808		int event;
809
810#if (__FreeBSD_version >= 1000006) || \
811    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
812		event = AC_ADVINFO_CHANGED;
813#else
814		event = AC_FOUND_DEVICE;
815#endif
816		status = xpt_register_async(event, mpssas_async, sc,
817					    sassc->path);
818		if (status != CAM_REQ_CMP) {
819			mps_dprint(sc, MPS_ERROR,
820			    "Error %#x registering async handler for "
821			    "AC_ADVINFO_CHANGED events\n", status);
822			xpt_free_path(sassc->path);
823			sassc->path = NULL;
824		}
825	}
826	if (status != CAM_REQ_CMP) {
827		/*
828		 * EEDP use is the exception, not the rule.
829		 * Warn the user, but do not fail to attach.
830		 */
831		mps_printf(sc, "EEDP capabilities disabled.\n");
832	}
833
834	mps_unlock(sc);
835
836	mpssas_register_events(sc);
837out:
838	if (error)
839		mps_detach_sas(sc);
840	return (error);
841}
842
843int
844mps_detach_sas(struct mps_softc *sc)
845{
846	struct mpssas_softc *sassc;
847	struct mpssas_lun *lun, *lun_tmp;
848	struct mpssas_target *targ;
849	int i;
850
851	MPS_FUNCTRACE(sc);
852
853	if (sc->sassc == NULL)
854		return (0);
855
856	sassc = sc->sassc;
857	mps_deregister_events(sc, sassc->mpssas_eh);
858
859	/*
860	 * Drain and free the event handling taskqueue with the lock
861	 * unheld so that any parallel processing tasks drain properly
862	 * without deadlocking.
863	 */
864	if (sassc->ev_tq != NULL)
865		taskqueue_free(sassc->ev_tq);
866
867	/* Make sure CAM doesn't wedge if we had to bail out early. */
868	mps_lock(sc);
869
870	/* Deregister our async handler */
871	if (sassc->path != NULL) {
872		xpt_register_async(0, mpssas_async, sc, sassc->path);
873		xpt_free_path(sassc->path);
874		sassc->path = NULL;
875	}
876
877	if (sassc->flags & MPSSAS_IN_STARTUP)
878		xpt_release_simq(sassc->sim, 1);
879
880	if (sassc->sim != NULL) {
881		xpt_bus_deregister(cam_sim_path(sassc->sim));
882		cam_sim_free(sassc->sim, FALSE);
883	}
884
885	mps_unlock(sc);
886
887	if (sassc->devq != NULL)
888		cam_simq_free(sassc->devq);
889
890	for(i=0; i< sassc->maxtargets ;i++) {
891		targ = &sassc->targets[i];
892		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
893			free(lun, M_MPT2);
894		}
895	}
896	free(sassc->targets, M_MPT2);
897	free(sassc, M_MPT2);
898	sc->sassc = NULL;
899
900	return (0);
901}
902
903void
904mpssas_discovery_end(struct mpssas_softc *sassc)
905{
906	struct mps_softc *sc = sassc->sc;
907
908	MPS_FUNCTRACE(sc);
909
910	if (sassc->flags & MPSSAS_DISCOVERY_TIMEOUT_PENDING)
911		callout_stop(&sassc->discovery_callout);
912
913}
914
915static void
916mpssas_action(struct cam_sim *sim, union ccb *ccb)
917{
918	struct mpssas_softc *sassc;
919
920	sassc = cam_sim_softc(sim);
921
922	MPS_FUNCTRACE(sassc->sc);
923	mps_dprint(sassc->sc, MPS_TRACE, "ccb func_code 0x%x\n",
924	    ccb->ccb_h.func_code);
925	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
926
927	switch (ccb->ccb_h.func_code) {
928	case XPT_PATH_INQ:
929	{
930		struct ccb_pathinq *cpi = &ccb->cpi;
931		struct mps_softc *sc = sassc->sc;
932		uint8_t sges_per_frame;
933
934		cpi->version_num = 1;
935		cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
936		cpi->target_sprt = 0;
937#if __FreeBSD_version >= 1000039
938		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN;
939#else
940		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED;
941#endif
942		cpi->hba_eng_cnt = 0;
943		cpi->max_target = sassc->maxtargets - 1;
944		cpi->max_lun = 255;
945		cpi->initiator_id = sassc->maxtargets - 1;
946		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
947		strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN);
948		strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
949		cpi->unit_number = cam_sim_unit(sim);
950		cpi->bus_id = cam_sim_bus(sim);
951		cpi->base_transfer_speed = 150000;
952		cpi->transport = XPORT_SAS;
953		cpi->transport_version = 0;
954		cpi->protocol = PROTO_SCSI;
955		cpi->protocol_version = SCSI_REV_SPC;
956
957		/*
958		 * Max IO Size is Page Size * the following:
959		 * ((SGEs per frame - 1 for chain element) *
960		 * Max Chain Depth) + 1 for no chain needed in last frame
961		 *
962		 * If user suggests a Max IO size to use, use the smaller of the
963		 * user's value and the calculated value as long as the user's
964		 * value is larger than 0. The user's value is in pages.
965		 */
966		sges_per_frame = ((sc->facts->IOCRequestFrameSize * 4) /
967		    sizeof(MPI2_SGE_SIMPLE64)) - 1;
968		cpi->maxio = (sges_per_frame * sc->facts->MaxChainDepth) + 1;
969		cpi->maxio *= PAGE_SIZE;
970		if ((sc->max_io_pages > 0) && (sc->max_io_pages * PAGE_SIZE <
971		    cpi->maxio))
972			cpi->maxio = sc->max_io_pages * PAGE_SIZE;
973		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
974		break;
975	}
976	case XPT_GET_TRAN_SETTINGS:
977	{
978		struct ccb_trans_settings	*cts;
979		struct ccb_trans_settings_sas	*sas;
980		struct ccb_trans_settings_scsi	*scsi;
981		struct mpssas_target *targ;
982
983		cts = &ccb->cts;
984		sas = &cts->xport_specific.sas;
985		scsi = &cts->proto_specific.scsi;
986
987		KASSERT(cts->ccb_h.target_id < sassc->maxtargets,
988		    ("Target %d out of bounds in XPT_GET_TRANS_SETTINGS\n",
989		    cts->ccb_h.target_id));
990		targ = &sassc->targets[cts->ccb_h.target_id];
991		if (targ->handle == 0x0) {
992			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
993			break;
994		}
995
996		cts->protocol_version = SCSI_REV_SPC2;
997		cts->transport = XPORT_SAS;
998		cts->transport_version = 0;
999
1000		sas->valid = CTS_SAS_VALID_SPEED;
1001		switch (targ->linkrate) {
1002		case 0x08:
1003			sas->bitrate = 150000;
1004			break;
1005		case 0x09:
1006			sas->bitrate = 300000;
1007			break;
1008		case 0x0a:
1009			sas->bitrate = 600000;
1010			break;
1011		default:
1012			sas->valid = 0;
1013		}
1014
1015		cts->protocol = PROTO_SCSI;
1016		scsi->valid = CTS_SCSI_VALID_TQ;
1017		scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
1018
1019		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1020		break;
1021	}
1022	case XPT_CALC_GEOMETRY:
1023		cam_calc_geometry(&ccb->ccg, /*extended*/1);
1024		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1025		break;
1026	case XPT_RESET_DEV:
1027		mps_dprint(sassc->sc, MPS_XINFO, "mpssas_action XPT_RESET_DEV\n");
1028		mpssas_action_resetdev(sassc, ccb);
1029		return;
1030	case XPT_RESET_BUS:
1031	case XPT_ABORT:
1032	case XPT_TERM_IO:
1033		mps_dprint(sassc->sc, MPS_XINFO,
1034		    "mpssas_action faking success for abort or reset\n");
1035		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1036		break;
1037	case XPT_SCSI_IO:
1038		mpssas_action_scsiio(sassc, ccb);
1039		return;
1040#if __FreeBSD_version >= 900026
1041	case XPT_SMP_IO:
1042		mpssas_action_smpio(sassc, ccb);
1043		return;
1044#endif
1045	default:
1046		mpssas_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL);
1047		break;
1048	}
1049	xpt_done(ccb);
1050
1051}
1052
1053static void
1054mpssas_announce_reset(struct mps_softc *sc, uint32_t ac_code,
1055    target_id_t target_id, lun_id_t lun_id)
1056{
1057	path_id_t path_id = cam_sim_path(sc->sassc->sim);
1058	struct cam_path *path;
1059
1060	mps_dprint(sc, MPS_XINFO, "%s code %x target %d lun %jx\n", __func__,
1061	    ac_code, target_id, (uintmax_t)lun_id);
1062
1063	if (xpt_create_path(&path, NULL,
1064		path_id, target_id, lun_id) != CAM_REQ_CMP) {
1065		mps_dprint(sc, MPS_ERROR, "unable to create path for reset "
1066			   "notification\n");
1067		return;
1068	}
1069
1070	xpt_async(ac_code, path, NULL);
1071	xpt_free_path(path);
1072}
1073
1074static void
1075mpssas_complete_all_commands(struct mps_softc *sc)
1076{
1077	struct mps_command *cm;
1078	int i;
1079	int completed;
1080
1081	MPS_FUNCTRACE(sc);
1082	mtx_assert(&sc->mps_mtx, MA_OWNED);
1083
1084	/* complete all commands with a NULL reply */
1085	for (i = 1; i < sc->num_reqs; i++) {
1086		cm = &sc->commands[i];
1087		cm->cm_reply = NULL;
1088		completed = 0;
1089
1090		if (cm->cm_flags & MPS_CM_FLAGS_POLLED)
1091			cm->cm_flags |= MPS_CM_FLAGS_COMPLETE;
1092
1093		if (cm->cm_complete != NULL) {
1094			mpssas_log_command(cm, MPS_RECOVERY,
1095			    "completing cm %p state %x ccb %p for diag reset\n",
1096			    cm, cm->cm_state, cm->cm_ccb);
1097
1098			cm->cm_complete(sc, cm);
1099			completed = 1;
1100		}
1101
1102		if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {
1103			mpssas_log_command(cm, MPS_RECOVERY,
1104			    "waking up cm %p state %x ccb %p for diag reset\n",
1105			    cm, cm->cm_state, cm->cm_ccb);
1106			wakeup(cm);
1107			completed = 1;
1108		}
1109
1110		if (cm->cm_sc->io_cmds_active != 0) {
1111			cm->cm_sc->io_cmds_active--;
1112		} else {
1113			mps_dprint(cm->cm_sc, MPS_INFO, "Warning: "
1114			    "io_cmds_active is out of sync - resynching to "
1115			    "0\n");
1116		}
1117
1118		if ((completed == 0) && (cm->cm_state != MPS_CM_STATE_FREE)) {
1119			/* this should never happen, but if it does, log */
1120			mpssas_log_command(cm, MPS_RECOVERY,
1121			    "cm %p state %x flags 0x%x ccb %p during diag "
1122			    "reset\n", cm, cm->cm_state, cm->cm_flags,
1123			    cm->cm_ccb);
1124		}
1125	}
1126}
1127
1128void
1129mpssas_handle_reinit(struct mps_softc *sc)
1130{
1131	int i;
1132
1133	/* Go back into startup mode and freeze the simq, so that CAM
1134	 * doesn't send any commands until after we've rediscovered all
1135	 * targets and found the proper device handles for them.
1136	 *
1137	 * After the reset, portenable will trigger discovery, and after all
1138	 * discovery-related activities have finished, the simq will be
1139	 * released.
1140	 */
1141	mps_dprint(sc, MPS_INIT, "%s startup\n", __func__);
1142	sc->sassc->flags |= MPSSAS_IN_STARTUP;
1143	sc->sassc->flags |= MPSSAS_IN_DISCOVERY;
1144	mpssas_startup_increment(sc->sassc);
1145
1146	/* notify CAM of a bus reset */
1147	mpssas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
1148	    CAM_LUN_WILDCARD);
1149
1150	/* complete and cleanup after all outstanding commands */
1151	mpssas_complete_all_commands(sc);
1152
1153	mps_dprint(sc, MPS_INIT,
1154	    "%s startup %u after command completion\n", __func__,
1155	    sc->sassc->startup_refcount);
1156
1157	/* zero all the target handles, since they may change after the
1158	 * reset, and we have to rediscover all the targets and use the new
1159	 * handles.
1160	 */
1161	for (i = 0; i < sc->sassc->maxtargets; i++) {
1162		if (sc->sassc->targets[i].outstanding != 0)
1163			mps_dprint(sc, MPS_INIT, "target %u outstanding %u\n",
1164			    i, sc->sassc->targets[i].outstanding);
1165		sc->sassc->targets[i].handle = 0x0;
1166		sc->sassc->targets[i].exp_dev_handle = 0x0;
1167		sc->sassc->targets[i].outstanding = 0;
1168		sc->sassc->targets[i].flags = MPSSAS_TARGET_INDIAGRESET;
1169	}
1170}
1171
1172static void
1173mpssas_tm_timeout(void *data)
1174{
1175	struct mps_command *tm = data;
1176	struct mps_softc *sc = tm->cm_sc;
1177
1178	mtx_assert(&sc->mps_mtx, MA_OWNED);
1179
1180	mpssas_log_command(tm, MPS_INFO|MPS_RECOVERY,
1181	    "task mgmt %p timed out\n", tm);
1182	mps_reinit(sc);
1183}
1184
1185static void
1186mpssas_logical_unit_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1187{
1188	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1189	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1190	unsigned int cm_count = 0;
1191	struct mps_command *cm;
1192	struct mpssas_target *targ;
1193
1194	callout_stop(&tm->cm_callout);
1195
1196	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1197	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1198	targ = tm->cm_targ;
1199
1200	/*
1201	 * Currently there should be no way we can hit this case.  It only
1202	 * happens when we have a failure to allocate chain frames, and
1203	 * task management commands don't have S/G lists.
1204	 * XXXSL So should it be an assertion?
1205	 */
1206	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1207		mps_dprint(sc, MPS_ERROR, "%s: cm_flags = %#x for LUN reset! "
1208			   "This should not happen!\n", __func__, tm->cm_flags);
1209		mpssas_free_tm(sc, tm);
1210		return;
1211	}
1212
1213	if (reply == NULL) {
1214		mpssas_log_command(tm, MPS_RECOVERY,
1215		    "NULL reset reply for tm %p\n", tm);
1216		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1217			/* this completion was due to a reset, just cleanup */
1218			targ->tm = NULL;
1219			mpssas_free_tm(sc, tm);
1220		}
1221		else {
1222			/* we should have gotten a reply. */
1223			mps_reinit(sc);
1224		}
1225		return;
1226	}
1227
1228	mpssas_log_command(tm, MPS_RECOVERY,
1229	    "logical unit reset status 0x%x code 0x%x count %u\n",
1230	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1231	    le32toh(reply->TerminationCount));
1232
1233	/* See if there are any outstanding commands for this LUN.
1234	 * This could be made more efficient by using a per-LU data
1235	 * structure of some sort.
1236	 */
1237	TAILQ_FOREACH(cm, &targ->commands, cm_link) {
1238		if (cm->cm_lun == tm->cm_lun)
1239			cm_count++;
1240	}
1241
1242	if (cm_count == 0) {
1243		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1244		    "logical unit %u finished recovery after reset\n",
1245		    tm->cm_lun, tm);
1246
1247		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1248		    tm->cm_lun);
1249
1250		/* we've finished recovery for this logical unit.  check and
1251		 * see if some other logical unit has a timedout command
1252		 * that needs to be processed.
1253		 */
1254		cm = TAILQ_FIRST(&targ->timedout_commands);
1255		if (cm) {
1256			mpssas_send_abort(sc, tm, cm);
1257		}
1258		else {
1259			targ->tm = NULL;
1260			mpssas_free_tm(sc, tm);
1261		}
1262	}
1263	else {
1264		/* if we still have commands for this LUN, the reset
1265		 * effectively failed, regardless of the status reported.
1266		 * Escalate to a target reset.
1267		 */
1268		mpssas_log_command(tm, MPS_RECOVERY,
1269		    "logical unit reset complete for tm %p, but still have %u command(s)\n",
1270		    tm, cm_count);
1271		mpssas_send_reset(sc, tm,
1272		    MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
1273	}
1274}
1275
1276static void
1277mpssas_target_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1278{
1279	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1280	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1281	struct mpssas_target *targ;
1282
1283	callout_stop(&tm->cm_callout);
1284
1285	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1286	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1287	targ = tm->cm_targ;
1288
1289	/*
1290	 * Currently there should be no way we can hit this case.  It only
1291	 * happens when we have a failure to allocate chain frames, and
1292	 * task management commands don't have S/G lists.
1293	 */
1294	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1295		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x for target reset! "
1296			   "This should not happen!\n", __func__, tm->cm_flags);
1297		mpssas_free_tm(sc, tm);
1298		return;
1299	}
1300
1301	if (reply == NULL) {
1302		mpssas_log_command(tm, MPS_RECOVERY,
1303		    "NULL reset reply for tm %p\n", tm);
1304		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1305			/* this completion was due to a reset, just cleanup */
1306			targ->tm = NULL;
1307			mpssas_free_tm(sc, tm);
1308		}
1309		else {
1310			/* we should have gotten a reply. */
1311			mps_reinit(sc);
1312		}
1313		return;
1314	}
1315
1316	mpssas_log_command(tm, MPS_RECOVERY,
1317	    "target reset status 0x%x code 0x%x count %u\n",
1318	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1319	    le32toh(reply->TerminationCount));
1320
1321	if (targ->outstanding == 0) {
1322		/* we've finished recovery for this target and all
1323		 * of its logical units.
1324		 */
1325		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1326		    "recovery finished after target reset\n");
1327
1328		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1329		    CAM_LUN_WILDCARD);
1330
1331		targ->tm = NULL;
1332		mpssas_free_tm(sc, tm);
1333	}
1334	else {
1335		/* after a target reset, if this target still has
1336		 * outstanding commands, the reset effectively failed,
1337		 * regardless of the status reported.  escalate.
1338		 */
1339		mpssas_log_command(tm, MPS_RECOVERY,
1340		    "target reset complete for tm %p, but still have %u command(s)\n",
1341		    tm, targ->outstanding);
1342		mps_reinit(sc);
1343	}
1344}
1345
1346#define MPS_RESET_TIMEOUT 30
1347
1348int
1349mpssas_send_reset(struct mps_softc *sc, struct mps_command *tm, uint8_t type)
1350{
1351	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1352	struct mpssas_target *target;
1353	int err;
1354
1355	target = tm->cm_targ;
1356	if (target->handle == 0) {
1357		mps_dprint(sc, MPS_ERROR,"%s null devhandle for target_id %d\n",
1358		    __func__, target->tid);
1359		return -1;
1360	}
1361
1362	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1363	req->DevHandle = htole16(target->handle);
1364	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1365	req->TaskType = type;
1366
1367	if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) {
1368		/* XXX Need to handle invalid LUNs */
1369		MPS_SET_LUN(req->LUN, tm->cm_lun);
1370		tm->cm_targ->logical_unit_resets++;
1371		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1372		    "sending logical unit reset\n");
1373		tm->cm_complete = mpssas_logical_unit_reset_complete;
1374		mpssas_prepare_for_tm(sc, tm, target, tm->cm_lun);
1375	}
1376	else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) {
1377		/*
1378		 * Target reset method =
1379		 * 	SAS Hard Link Reset / SATA Link Reset
1380		 */
1381		req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
1382		tm->cm_targ->target_resets++;
1383		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1384		    "sending target reset\n");
1385		tm->cm_complete = mpssas_target_reset_complete;
1386		mpssas_prepare_for_tm(sc, tm, target, CAM_LUN_WILDCARD);
1387	}
1388	else {
1389		mps_dprint(sc, MPS_ERROR, "unexpected reset type 0x%x\n", type);
1390		return -1;
1391	}
1392
1393	tm->cm_data = NULL;
1394	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1395	tm->cm_complete_data = (void *)tm;
1396
1397	callout_reset(&tm->cm_callout, MPS_RESET_TIMEOUT * hz,
1398	    mpssas_tm_timeout, tm);
1399
1400	err = mps_map_command(sc, tm);
1401	if (err)
1402		mpssas_log_command(tm, MPS_RECOVERY,
1403		    "error %d sending reset type %u\n",
1404		    err, type);
1405
1406	return err;
1407}
1408
1409
1410static void
1411mpssas_abort_complete(struct mps_softc *sc, struct mps_command *tm)
1412{
1413	struct mps_command *cm;
1414	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1415	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1416	struct mpssas_target *targ;
1417
1418	callout_stop(&tm->cm_callout);
1419
1420	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1421	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1422	targ = tm->cm_targ;
1423
1424	/*
1425	 * Currently there should be no way we can hit this case.  It only
1426	 * happens when we have a failure to allocate chain frames, and
1427	 * task management commands don't have S/G lists.
1428	 */
1429	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1430		mpssas_log_command(tm, MPS_RECOVERY,
1431		    "cm_flags = %#x for abort %p TaskMID %u!\n",
1432		    tm->cm_flags, tm, le16toh(req->TaskMID));
1433		mpssas_free_tm(sc, tm);
1434		return;
1435	}
1436
1437	if (reply == NULL) {
1438		mpssas_log_command(tm, MPS_RECOVERY,
1439		    "NULL abort reply for tm %p TaskMID %u\n",
1440		    tm, le16toh(req->TaskMID));
1441		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1442			/* this completion was due to a reset, just cleanup */
1443			targ->tm = NULL;
1444			mpssas_free_tm(sc, tm);
1445		}
1446		else {
1447			/* we should have gotten a reply. */
1448			mps_reinit(sc);
1449		}
1450		return;
1451	}
1452
1453	mpssas_log_command(tm, MPS_RECOVERY,
1454	    "abort TaskMID %u status 0x%x code 0x%x count %u\n",
1455	    le16toh(req->TaskMID),
1456	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1457	    le32toh(reply->TerminationCount));
1458
1459	cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands);
1460	if (cm == NULL) {
1461		/* if there are no more timedout commands, we're done with
1462		 * error recovery for this target.
1463		 */
1464		mpssas_log_command(tm, MPS_RECOVERY,
1465		    "finished recovery after aborting TaskMID %u\n",
1466		    le16toh(req->TaskMID));
1467
1468		targ->tm = NULL;
1469		mpssas_free_tm(sc, tm);
1470	}
1471	else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) {
1472		/* abort success, but we have more timedout commands to abort */
1473		mpssas_log_command(tm, MPS_RECOVERY,
1474		    "continuing recovery after aborting TaskMID %u\n",
1475		    le16toh(req->TaskMID));
1476
1477		mpssas_send_abort(sc, tm, cm);
1478	}
1479	else {
1480		/* we didn't get a command completion, so the abort
1481		 * failed as far as we're concerned.  escalate.
1482		 */
1483		mpssas_log_command(tm, MPS_RECOVERY,
1484		    "abort failed for TaskMID %u tm %p\n",
1485		    le16toh(req->TaskMID), tm);
1486
1487		mpssas_send_reset(sc, tm,
1488		    MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET);
1489	}
1490}
1491
1492#define MPS_ABORT_TIMEOUT 5
1493
1494static int
1495mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm, struct mps_command *cm)
1496{
1497	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1498	struct mpssas_target *targ;
1499	int err;
1500
1501	targ = cm->cm_targ;
1502	if (targ->handle == 0) {
1503		mps_dprint(sc, MPS_ERROR,"%s null devhandle for target_id %d\n",
1504		    __func__, cm->cm_ccb->ccb_h.target_id);
1505		return -1;
1506	}
1507
1508	mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1509	    "Aborting command %p\n", cm);
1510
1511	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1512	req->DevHandle = htole16(targ->handle);
1513	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1514	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK;
1515
1516	/* XXX Need to handle invalid LUNs */
1517	MPS_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun);
1518
1519	req->TaskMID = htole16(cm->cm_desc.Default.SMID);
1520
1521	tm->cm_data = NULL;
1522	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1523	tm->cm_complete = mpssas_abort_complete;
1524	tm->cm_complete_data = (void *)tm;
1525	tm->cm_targ = cm->cm_targ;
1526	tm->cm_lun = cm->cm_lun;
1527
1528	callout_reset(&tm->cm_callout, MPS_ABORT_TIMEOUT * hz,
1529	    mpssas_tm_timeout, tm);
1530
1531	targ->aborts++;
1532
1533	mps_dprint(sc, MPS_INFO, "Sending reset from %s for target ID %d\n",
1534	    __func__, targ->tid);
1535	mpssas_prepare_for_tm(sc, tm, targ, tm->cm_lun);
1536
1537	err = mps_map_command(sc, tm);
1538	if (err)
1539		mpssas_log_command(tm, MPS_RECOVERY,
1540		    "error %d sending abort for cm %p SMID %u\n",
1541		    err, cm, req->TaskMID);
1542	return err;
1543}
1544
1545static void
1546mpssas_scsiio_timeout(void *data)
1547{
1548	struct mps_softc *sc;
1549	struct mps_command *cm;
1550	struct mpssas_target *targ;
1551
1552	cm = (struct mps_command *)data;
1553	sc = cm->cm_sc;
1554
1555	MPS_FUNCTRACE(sc);
1556	mtx_assert(&sc->mps_mtx, MA_OWNED);
1557
1558	mps_dprint(sc, MPS_XINFO, "Timeout checking cm %p\n", sc);
1559
1560	/*
1561	 * Run the interrupt handler to make sure it's not pending.  This
1562	 * isn't perfect because the command could have already completed
1563	 * and been re-used, though this is unlikely.
1564	 */
1565	mps_intr_locked(sc);
1566	if (cm->cm_state == MPS_CM_STATE_FREE) {
1567		mpssas_log_command(cm, MPS_XINFO,
1568		    "SCSI command %p almost timed out\n", cm);
1569		return;
1570	}
1571
1572	if (cm->cm_ccb == NULL) {
1573		mps_dprint(sc, MPS_ERROR, "command timeout with NULL ccb\n");
1574		return;
1575	}
1576
1577	mpssas_log_command(cm, MPS_INFO, "command timeout cm %p ccb %p\n",
1578	    cm, cm->cm_ccb);
1579
1580	targ = cm->cm_targ;
1581	targ->timeouts++;
1582
1583	/* XXX first, check the firmware state, to see if it's still
1584	 * operational.  if not, do a diag reset.
1585	 */
1586	mpssas_set_ccbstatus(cm->cm_ccb, CAM_CMD_TIMEOUT);
1587	cm->cm_state = MPS_CM_STATE_TIMEDOUT;
1588	TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery);
1589
1590	if (targ->tm != NULL) {
1591		/* target already in recovery, just queue up another
1592		 * timedout command to be processed later.
1593		 */
1594		mps_dprint(sc, MPS_RECOVERY,
1595		    "queued timedout cm %p for processing by tm %p\n",
1596		    cm, targ->tm);
1597	}
1598	else if ((targ->tm = mpssas_alloc_tm(sc)) != NULL) {
1599		mps_dprint(sc, MPS_RECOVERY, "timedout cm %p allocated tm %p\n",
1600		    cm, targ->tm);
1601
1602		/* start recovery by aborting the first timedout command */
1603		mpssas_send_abort(sc, targ->tm, cm);
1604	}
1605	else {
1606		/* XXX queue this target up for recovery once a TM becomes
1607		 * available.  The firmware only has a limited number of
1608		 * HighPriority credits for the high priority requests used
1609		 * for task management, and we ran out.
1610		 *
1611		 * Isilon: don't worry about this for now, since we have
1612		 * more credits than disks in an enclosure, and limit
1613		 * ourselves to one TM per target for recovery.
1614		 */
1615		mps_dprint(sc, MPS_RECOVERY,
1616		    "timedout cm %p failed to allocate a tm\n", cm);
1617	}
1618
1619}
1620
1621static void
1622mpssas_action_scsiio(struct mpssas_softc *sassc, union ccb *ccb)
1623{
1624	MPI2_SCSI_IO_REQUEST *req;
1625	struct ccb_scsiio *csio;
1626	struct mps_softc *sc;
1627	struct mpssas_target *targ;
1628	struct mpssas_lun *lun;
1629	struct mps_command *cm;
1630	uint8_t i, lba_byte, *ref_tag_addr;
1631	uint16_t eedp_flags;
1632	uint32_t mpi_control;
1633
1634	sc = sassc->sc;
1635	MPS_FUNCTRACE(sc);
1636	mtx_assert(&sc->mps_mtx, MA_OWNED);
1637
1638	csio = &ccb->csio;
1639	KASSERT(csio->ccb_h.target_id < sassc->maxtargets,
1640	    ("Target %d out of bounds in XPT_SCSI_IO\n",
1641	     csio->ccb_h.target_id));
1642	targ = &sassc->targets[csio->ccb_h.target_id];
1643	mps_dprint(sc, MPS_TRACE, "ccb %p target flag %x\n", ccb, targ->flags);
1644	if (targ->handle == 0x0) {
1645		mps_dprint(sc, MPS_ERROR, "%s NULL handle for target %u\n",
1646		    __func__, csio->ccb_h.target_id);
1647		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1648		xpt_done(ccb);
1649		return;
1650	}
1651	if (targ->flags & MPS_TARGET_FLAGS_RAID_COMPONENT) {
1652		mps_dprint(sc, MPS_ERROR, "%s Raid component no SCSI IO "
1653		    "supported %u\n", __func__, csio->ccb_h.target_id);
1654		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1655		xpt_done(ccb);
1656		return;
1657	}
1658	/*
1659	 * Sometimes, it is possible to get a command that is not "In
1660	 * Progress" and was actually aborted by the upper layer.  Check for
1661	 * this here and complete the command without error.
1662	 */
1663	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_INPROG) {
1664		mps_dprint(sc, MPS_TRACE, "%s Command is not in progress for "
1665		    "target %u\n", __func__, csio->ccb_h.target_id);
1666		xpt_done(ccb);
1667		return;
1668	}
1669	/*
1670	 * If devinfo is 0 this will be a volume.  In that case don't tell CAM
1671	 * that the volume has timed out.  We want volumes to be enumerated
1672	 * until they are deleted/removed, not just failed.
1673	 */
1674	if (targ->flags & MPSSAS_TARGET_INREMOVAL) {
1675		if (targ->devinfo == 0)
1676			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1677		else
1678			mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
1679		xpt_done(ccb);
1680		return;
1681	}
1682
1683	if ((sc->mps_flags & MPS_FLAGS_SHUTDOWN) != 0) {
1684		mps_dprint(sc, MPS_INFO, "%s shutting down\n", __func__);
1685		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1686		xpt_done(ccb);
1687		return;
1688	}
1689
1690	/*
1691	 * If target has a reset in progress, freeze the devq and return.  The
1692	 * devq will be released when the TM reset is finished.
1693	 */
1694	if (targ->flags & MPSSAS_TARGET_INRESET) {
1695		ccb->ccb_h.status = CAM_BUSY | CAM_DEV_QFRZN;
1696		mps_dprint(sc, MPS_INFO, "%s: Freezing devq for target ID %d\n",
1697		    __func__, targ->tid);
1698		xpt_freeze_devq(ccb->ccb_h.path, 1);
1699		xpt_done(ccb);
1700		return;
1701	}
1702
1703	cm = mps_alloc_command(sc);
1704	if (cm == NULL || (sc->mps_flags & MPS_FLAGS_DIAGRESET)) {
1705		if (cm != NULL) {
1706			mps_free_command(sc, cm);
1707		}
1708		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
1709			xpt_freeze_simq(sassc->sim, 1);
1710			sassc->flags |= MPSSAS_QUEUE_FROZEN;
1711		}
1712		ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1713		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1714		xpt_done(ccb);
1715		return;
1716	}
1717
1718	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
1719	bzero(req, sizeof(*req));
1720	req->DevHandle = htole16(targ->handle);
1721	req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
1722	req->MsgFlags = 0;
1723	req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);
1724	req->SenseBufferLength = MPS_SENSE_LEN;
1725	req->SGLFlags = 0;
1726	req->ChainOffset = 0;
1727	req->SGLOffset0 = 24;	/* 32bit word offset to the SGL */
1728	req->SGLOffset1= 0;
1729	req->SGLOffset2= 0;
1730	req->SGLOffset3= 0;
1731	req->SkipCount = 0;
1732	req->DataLength = htole32(csio->dxfer_len);
1733	req->BidirectionalDataLength = 0;
1734	req->IoFlags = htole16(csio->cdb_len);
1735	req->EEDPFlags = 0;
1736
1737	/* Note: BiDirectional transfers are not supported */
1738	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1739	case CAM_DIR_IN:
1740		mpi_control = MPI2_SCSIIO_CONTROL_READ;
1741		cm->cm_flags |= MPS_CM_FLAGS_DATAIN;
1742		break;
1743	case CAM_DIR_OUT:
1744		mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
1745		cm->cm_flags |= MPS_CM_FLAGS_DATAOUT;
1746		break;
1747	case CAM_DIR_NONE:
1748	default:
1749		mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
1750		break;
1751	}
1752
1753	if (csio->cdb_len == 32)
1754                mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
1755	/*
1756	 * It looks like the hardware doesn't require an explicit tag
1757	 * number for each transaction.  SAM Task Management not supported
1758	 * at the moment.
1759	 */
1760	switch (csio->tag_action) {
1761	case MSG_HEAD_OF_Q_TAG:
1762		mpi_control |= MPI2_SCSIIO_CONTROL_HEADOFQ;
1763		break;
1764	case MSG_ORDERED_Q_TAG:
1765		mpi_control |= MPI2_SCSIIO_CONTROL_ORDEREDQ;
1766		break;
1767	case MSG_ACA_TASK:
1768		mpi_control |= MPI2_SCSIIO_CONTROL_ACAQ;
1769		break;
1770	case CAM_TAG_ACTION_NONE:
1771	case MSG_SIMPLE_Q_TAG:
1772	default:
1773		mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
1774		break;
1775	}
1776	mpi_control |= sc->mapping_table[csio->ccb_h.target_id].TLR_bits;
1777	req->Control = htole32(mpi_control);
1778	if (MPS_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) {
1779		mps_free_command(sc, cm);
1780		mpssas_set_ccbstatus(ccb, CAM_LUN_INVALID);
1781		xpt_done(ccb);
1782		return;
1783	}
1784
1785	if (csio->ccb_h.flags & CAM_CDB_POINTER)
1786		bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
1787	else
1788		bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
1789	req->IoFlags = htole16(csio->cdb_len);
1790
1791	/*
1792	 * Check if EEDP is supported and enabled.  If it is then check if the
1793	 * SCSI opcode could be using EEDP.  If so, make sure the LUN exists and
1794	 * is formatted for EEDP support.  If all of this is true, set CDB up
1795	 * for EEDP transfer.
1796	 */
1797	eedp_flags = op_code_prot[req->CDB.CDB32[0]];
1798	if (sc->eedp_enabled && eedp_flags) {
1799		SLIST_FOREACH(lun, &targ->luns, lun_link) {
1800			if (lun->lun_id == csio->ccb_h.target_lun) {
1801				break;
1802			}
1803		}
1804
1805		if ((lun != NULL) && (lun->eedp_formatted)) {
1806			req->EEDPBlockSize = htole16(lun->eedp_block_size);
1807			eedp_flags |= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
1808			    MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
1809			    MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
1810			req->EEDPFlags = htole16(eedp_flags);
1811
1812			/*
1813			 * If CDB less than 32, fill in Primary Ref Tag with
1814			 * low 4 bytes of LBA.  If CDB is 32, tag stuff is
1815			 * already there.  Also, set protection bit.  FreeBSD
1816			 * currently does not support CDBs bigger than 16, but
1817			 * the code doesn't hurt, and will be here for the
1818			 * future.
1819			 */
1820			if (csio->cdb_len != 32) {
1821				lba_byte = (csio->cdb_len == 16) ? 6 : 2;
1822				ref_tag_addr = (uint8_t *)&req->CDB.EEDP32.
1823				    PrimaryReferenceTag;
1824				for (i = 0; i < 4; i++) {
1825					*ref_tag_addr =
1826					    req->CDB.CDB32[lba_byte + i];
1827					ref_tag_addr++;
1828				}
1829				req->CDB.EEDP32.PrimaryReferenceTag =
1830					htole32(req->CDB.EEDP32.PrimaryReferenceTag);
1831				req->CDB.EEDP32.PrimaryApplicationTagMask =
1832				    0xFFFF;
1833				req->CDB.CDB32[1] = (req->CDB.CDB32[1] & 0x1F) |
1834				    0x20;
1835			} else {
1836				eedp_flags |=
1837				    MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG;
1838				req->EEDPFlags = htole16(eedp_flags);
1839				req->CDB.CDB32[10] = (req->CDB.CDB32[10] &
1840				    0x1F) | 0x20;
1841			}
1842		}
1843	}
1844
1845	cm->cm_length = csio->dxfer_len;
1846	if (cm->cm_length != 0) {
1847		cm->cm_data = ccb;
1848		cm->cm_flags |= MPS_CM_FLAGS_USE_CCB;
1849	} else {
1850		cm->cm_data = NULL;
1851	}
1852	cm->cm_sge = &req->SGL;
1853	cm->cm_sglsize = (32 - 24) * 4;
1854	cm->cm_desc.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
1855	cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
1856	cm->cm_complete = mpssas_scsiio_complete;
1857	cm->cm_complete_data = ccb;
1858	cm->cm_targ = targ;
1859	cm->cm_lun = csio->ccb_h.target_lun;
1860	cm->cm_ccb = ccb;
1861
1862	/*
1863	 * If HBA is a WD and the command is not for a retry, try to build a
1864	 * direct I/O message. If failed, or the command is for a retry, send
1865	 * the I/O to the IR volume itself.
1866	 */
1867	if (sc->WD_valid_config) {
1868		if (ccb->ccb_h.sim_priv.entries[0].field == MPS_WD_RETRY) {
1869			mpssas_direct_drive_io(sassc, cm, ccb);
1870		} else {
1871			mpssas_set_ccbstatus(ccb, CAM_REQ_INPROG);
1872		}
1873	}
1874
1875#if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
1876	if (csio->bio != NULL)
1877		biotrack(csio->bio, __func__);
1878#endif
1879	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
1880	    mpssas_scsiio_timeout, cm, 0);
1881
1882	targ->issued++;
1883	targ->outstanding++;
1884	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
1885	ccb->ccb_h.status |= CAM_SIM_QUEUED;
1886
1887	mpssas_log_command(cm, MPS_XINFO, "%s cm %p ccb %p outstanding %u\n",
1888	    __func__, cm, ccb, targ->outstanding);
1889
1890	mps_map_command(sc, cm);
1891	return;
1892}
1893
1894static void
1895mps_response_code(struct mps_softc *sc, u8 response_code)
1896{
1897        char *desc;
1898
1899        switch (response_code) {
1900        case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
1901                desc = "task management request completed";
1902                break;
1903        case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
1904                desc = "invalid frame";
1905                break;
1906        case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
1907                desc = "task management request not supported";
1908                break;
1909        case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
1910                desc = "task management request failed";
1911                break;
1912        case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
1913                desc = "task management request succeeded";
1914                break;
1915        case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
1916                desc = "invalid lun";
1917                break;
1918        case 0xA:
1919                desc = "overlapped tag attempted";
1920                break;
1921        case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
1922                desc = "task queued, however not sent to target";
1923                break;
1924        default:
1925                desc = "unknown";
1926                break;
1927        }
1928		mps_dprint(sc, MPS_XINFO, "response_code(0x%01x): %s\n",
1929                response_code, desc);
1930}
1931/**
1932 * mps_sc_failed_io_info - translated non-succesfull SCSI_IO request
1933 */
1934static void
1935mps_sc_failed_io_info(struct mps_softc *sc, struct ccb_scsiio *csio,
1936    Mpi2SCSIIOReply_t *mpi_reply)
1937{
1938	u32 response_info;
1939	u8 *response_bytes;
1940	u16 ioc_status = le16toh(mpi_reply->IOCStatus) &
1941	    MPI2_IOCSTATUS_MASK;
1942	u8 scsi_state = mpi_reply->SCSIState;
1943	u8 scsi_status = mpi_reply->SCSIStatus;
1944	char *desc_ioc_state = NULL;
1945	char *desc_scsi_status = NULL;
1946	char *desc_scsi_state = sc->tmp_string;
1947	u32 log_info = le32toh(mpi_reply->IOCLogInfo);
1948
1949	if (log_info == 0x31170000)
1950		return;
1951
1952	switch (ioc_status) {
1953	case MPI2_IOCSTATUS_SUCCESS:
1954		desc_ioc_state = "success";
1955		break;
1956	case MPI2_IOCSTATUS_INVALID_FUNCTION:
1957		desc_ioc_state = "invalid function";
1958		break;
1959	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
1960		desc_ioc_state = "scsi recovered error";
1961		break;
1962	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
1963		desc_ioc_state = "scsi invalid dev handle";
1964		break;
1965	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
1966		desc_ioc_state = "scsi device not there";
1967		break;
1968	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
1969		desc_ioc_state = "scsi data overrun";
1970		break;
1971	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
1972		desc_ioc_state = "scsi data underrun";
1973		break;
1974	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
1975		desc_ioc_state = "scsi io data error";
1976		break;
1977	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
1978		desc_ioc_state = "scsi protocol error";
1979		break;
1980	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
1981		desc_ioc_state = "scsi task terminated";
1982		break;
1983	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
1984		desc_ioc_state = "scsi residual mismatch";
1985		break;
1986	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
1987		desc_ioc_state = "scsi task mgmt failed";
1988		break;
1989	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
1990		desc_ioc_state = "scsi ioc terminated";
1991		break;
1992	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
1993		desc_ioc_state = "scsi ext terminated";
1994		break;
1995	case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
1996		desc_ioc_state = "eedp guard error";
1997		break;
1998	case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
1999		desc_ioc_state = "eedp ref tag error";
2000		break;
2001	case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
2002		desc_ioc_state = "eedp app tag error";
2003		break;
2004	default:
2005		desc_ioc_state = "unknown";
2006		break;
2007	}
2008
2009	switch (scsi_status) {
2010	case MPI2_SCSI_STATUS_GOOD:
2011		desc_scsi_status = "good";
2012		break;
2013	case MPI2_SCSI_STATUS_CHECK_CONDITION:
2014		desc_scsi_status = "check condition";
2015		break;
2016	case MPI2_SCSI_STATUS_CONDITION_MET:
2017		desc_scsi_status = "condition met";
2018		break;
2019	case MPI2_SCSI_STATUS_BUSY:
2020		desc_scsi_status = "busy";
2021		break;
2022	case MPI2_SCSI_STATUS_INTERMEDIATE:
2023		desc_scsi_status = "intermediate";
2024		break;
2025	case MPI2_SCSI_STATUS_INTERMEDIATE_CONDMET:
2026		desc_scsi_status = "intermediate condmet";
2027		break;
2028	case MPI2_SCSI_STATUS_RESERVATION_CONFLICT:
2029		desc_scsi_status = "reservation conflict";
2030		break;
2031	case MPI2_SCSI_STATUS_COMMAND_TERMINATED:
2032		desc_scsi_status = "command terminated";
2033		break;
2034	case MPI2_SCSI_STATUS_TASK_SET_FULL:
2035		desc_scsi_status = "task set full";
2036		break;
2037	case MPI2_SCSI_STATUS_ACA_ACTIVE:
2038		desc_scsi_status = "aca active";
2039		break;
2040	case MPI2_SCSI_STATUS_TASK_ABORTED:
2041		desc_scsi_status = "task aborted";
2042		break;
2043	default:
2044		desc_scsi_status = "unknown";
2045		break;
2046	}
2047
2048	desc_scsi_state[0] = '\0';
2049	if (!scsi_state)
2050		desc_scsi_state = " ";
2051	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID)
2052		strcat(desc_scsi_state, "response info ");
2053	if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
2054		strcat(desc_scsi_state, "state terminated ");
2055	if (scsi_state & MPI2_SCSI_STATE_NO_SCSI_STATUS)
2056		strcat(desc_scsi_state, "no status ");
2057	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_FAILED)
2058		strcat(desc_scsi_state, "autosense failed ");
2059	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID)
2060		strcat(desc_scsi_state, "autosense valid ");
2061
2062	mps_dprint(sc, MPS_XINFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x)\n",
2063	    le16toh(mpi_reply->DevHandle), desc_ioc_state, ioc_status);
2064	/* We can add more detail about underflow data here
2065	 * TO-DO
2066	 * */
2067	mps_dprint(sc, MPS_XINFO, "\tscsi_status(%s)(0x%02x), "
2068	    "scsi_state(%s)(0x%02x)\n", desc_scsi_status, scsi_status,
2069	    desc_scsi_state, scsi_state);
2070
2071	if (sc->mps_debug & MPS_XINFO &&
2072		scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2073		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : Start :\n");
2074		scsi_sense_print(csio);
2075		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : End :\n");
2076	}
2077
2078	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
2079		response_info = le32toh(mpi_reply->ResponseInfo);
2080		response_bytes = (u8 *)&response_info;
2081		mps_response_code(sc,response_bytes[0]);
2082	}
2083}
2084
2085static void
2086mpssas_scsiio_complete(struct mps_softc *sc, struct mps_command *cm)
2087{
2088	MPI2_SCSI_IO_REPLY *rep;
2089	union ccb *ccb;
2090	struct ccb_scsiio *csio;
2091	struct mpssas_softc *sassc;
2092	struct scsi_vpd_supported_page_list *vpd_list = NULL;
2093	u8 *TLR_bits, TLR_on;
2094	int dir = 0, i;
2095	u16 alloc_len;
2096	struct mpssas_target *target;
2097	target_id_t target_id;
2098
2099	MPS_FUNCTRACE(sc);
2100	mps_dprint(sc, MPS_TRACE,
2101	    "cm %p SMID %u ccb %p reply %p outstanding %u\n", cm,
2102	    cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply,
2103	    cm->cm_targ->outstanding);
2104
2105	callout_stop(&cm->cm_callout);
2106	mtx_assert(&sc->mps_mtx, MA_OWNED);
2107
2108	sassc = sc->sassc;
2109	ccb = cm->cm_complete_data;
2110	csio = &ccb->csio;
2111	target_id = csio->ccb_h.target_id;
2112	rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply;
2113	/*
2114	 * XXX KDM if the chain allocation fails, does it matter if we do
2115	 * the sync and unload here?  It is simpler to do it in every case,
2116	 * assuming it doesn't cause problems.
2117	 */
2118	if (cm->cm_data != NULL) {
2119		if (cm->cm_flags & MPS_CM_FLAGS_DATAIN)
2120			dir = BUS_DMASYNC_POSTREAD;
2121		else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT)
2122			dir = BUS_DMASYNC_POSTWRITE;
2123		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
2124		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2125	}
2126
2127	cm->cm_targ->completed++;
2128	cm->cm_targ->outstanding--;
2129	TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link);
2130	ccb->ccb_h.status &= ~(CAM_STATUS_MASK | CAM_SIM_QUEUED);
2131
2132#if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
2133	if (ccb->csio.bio != NULL)
2134		biotrack(ccb->csio.bio, __func__);
2135#endif
2136
2137	if (cm->cm_state == MPS_CM_STATE_TIMEDOUT) {
2138		TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery);
2139		if (cm->cm_reply != NULL)
2140			mpssas_log_command(cm, MPS_RECOVERY,
2141			    "completed timedout cm %p ccb %p during recovery "
2142			    "ioc %x scsi %x state %x xfer %u\n",
2143			    cm, cm->cm_ccb,
2144			    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2145			    le32toh(rep->TransferCount));
2146		else
2147			mpssas_log_command(cm, MPS_RECOVERY,
2148			    "completed timedout cm %p ccb %p during recovery\n",
2149			    cm, cm->cm_ccb);
2150	} else if (cm->cm_targ->tm != NULL) {
2151		if (cm->cm_reply != NULL)
2152			mpssas_log_command(cm, MPS_RECOVERY,
2153			    "completed cm %p ccb %p during recovery "
2154			    "ioc %x scsi %x state %x xfer %u\n",
2155			    cm, cm->cm_ccb,
2156			    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2157			    le32toh(rep->TransferCount));
2158		else
2159			mpssas_log_command(cm, MPS_RECOVERY,
2160			    "completed cm %p ccb %p during recovery\n",
2161			    cm, cm->cm_ccb);
2162	} else if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
2163		mpssas_log_command(cm, MPS_RECOVERY,
2164		    "reset completed cm %p ccb %p\n",
2165		    cm, cm->cm_ccb);
2166	}
2167
2168	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2169		/*
2170		 * We ran into an error after we tried to map the command,
2171		 * so we're getting a callback without queueing the command
2172		 * to the hardware.  So we set the status here, and it will
2173		 * be retained below.  We'll go through the "fast path",
2174		 * because there can be no reply when we haven't actually
2175		 * gone out to the hardware.
2176		 */
2177		mpssas_set_ccbstatus(ccb, CAM_REQUEUE_REQ);
2178
2179		/*
2180		 * Currently the only error included in the mask is
2181		 * MPS_CM_FLAGS_CHAIN_FAILED, which means we're out of
2182		 * chain frames.  We need to freeze the queue until we get
2183		 * a command that completed without this error, which will
2184		 * hopefully have some chain frames attached that we can
2185		 * use.  If we wanted to get smarter about it, we would
2186		 * only unfreeze the queue in this condition when we're
2187		 * sure that we're getting some chain frames back.  That's
2188		 * probably unnecessary.
2189		 */
2190		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
2191			xpt_freeze_simq(sassc->sim, 1);
2192			sassc->flags |= MPSSAS_QUEUE_FROZEN;
2193			mps_dprint(sc, MPS_XINFO, "Error sending command, "
2194				   "freezing SIM queue\n");
2195		}
2196	}
2197
2198	/*
2199	 * If this is a Start Stop Unit command and it was issued by the driver
2200	 * during shutdown, decrement the refcount to account for all of the
2201	 * commands that were sent.  All SSU commands should be completed before
2202	 * shutdown completes, meaning SSU_refcount will be 0 after SSU_started
2203	 * is TRUE.
2204	 */
2205	if (sc->SSU_started && (csio->cdb_io.cdb_bytes[0] == START_STOP_UNIT)) {
2206		mps_dprint(sc, MPS_INFO, "Decrementing SSU count.\n");
2207		sc->SSU_refcount--;
2208	}
2209
2210	/* Take the fast path to completion */
2211	if (cm->cm_reply == NULL) {
2212		if (mpssas_get_ccbstatus(ccb) == CAM_REQ_INPROG) {
2213			if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0)
2214				mpssas_set_ccbstatus(ccb, CAM_SCSI_BUS_RESET);
2215			else {
2216				mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2217				ccb->csio.scsi_status = SCSI_STATUS_OK;
2218			}
2219			if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2220				ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2221				sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2222				mps_dprint(sc, MPS_XINFO,
2223				    "Unfreezing SIM queue\n");
2224			}
2225		}
2226
2227		/*
2228		 * There are two scenarios where the status won't be
2229		 * CAM_REQ_CMP.  The first is if MPS_CM_FLAGS_ERROR_MASK is
2230		 * set, the second is in the MPS_FLAGS_DIAGRESET above.
2231		 */
2232		if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2233			/*
2234			 * Freeze the dev queue so that commands are
2235			 * executed in the correct order after error
2236			 * recovery.
2237			 */
2238			ccb->ccb_h.status |= CAM_DEV_QFRZN;
2239			xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2240		}
2241		mps_free_command(sc, cm);
2242		xpt_done(ccb);
2243		return;
2244	}
2245
2246	mpssas_log_command(cm, MPS_XINFO,
2247	    "ioc %x scsi %x state %x xfer %u\n",
2248	    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2249	    le32toh(rep->TransferCount));
2250
2251	/*
2252	 * If this is a Direct Drive I/O, reissue the I/O to the original IR
2253	 * Volume if an error occurred (normal I/O retry).  Use the original
2254	 * CCB, but set a flag that this will be a retry so that it's sent to
2255	 * the original volume.  Free the command but reuse the CCB.
2256	 */
2257	if (cm->cm_flags & MPS_CM_FLAGS_DD_IO) {
2258		mps_free_command(sc, cm);
2259		ccb->ccb_h.sim_priv.entries[0].field = MPS_WD_RETRY;
2260		mpssas_action_scsiio(sassc, ccb);
2261		return;
2262	} else
2263		ccb->ccb_h.sim_priv.entries[0].field = 0;
2264
2265	switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) {
2266	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
2267		csio->resid = cm->cm_length - le32toh(rep->TransferCount);
2268		/* FALLTHROUGH */
2269	case MPI2_IOCSTATUS_SUCCESS:
2270	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
2271
2272		if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
2273		    MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
2274			mpssas_log_command(cm, MPS_XINFO, "recovered error\n");
2275
2276		/* Completion failed at the transport level. */
2277		if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS |
2278		    MPI2_SCSI_STATE_TERMINATED)) {
2279			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2280			break;
2281		}
2282
2283		/* In a modern packetized environment, an autosense failure
2284		 * implies that there's not much else that can be done to
2285		 * recover the command.
2286		 */
2287		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) {
2288			mpssas_set_ccbstatus(ccb, CAM_AUTOSENSE_FAIL);
2289			break;
2290		}
2291
2292		/*
2293		 * CAM doesn't care about SAS Response Info data, but if this is
2294		 * the state check if TLR should be done.  If not, clear the
2295		 * TLR_bits for the target.
2296		 */
2297		if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) &&
2298		    ((le32toh(rep->ResponseInfo) &
2299		    MPI2_SCSI_RI_MASK_REASONCODE) ==
2300		    MPS_SCSI_RI_INVALID_FRAME)) {
2301			sc->mapping_table[target_id].TLR_bits =
2302			    (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2303		}
2304
2305		/*
2306		 * Intentionally override the normal SCSI status reporting
2307		 * for these two cases.  These are likely to happen in a
2308		 * multi-initiator environment, and we want to make sure that
2309		 * CAM retries these commands rather than fail them.
2310		 */
2311		if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) ||
2312		    (rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) {
2313			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2314			break;
2315		}
2316
2317		/* Handle normal status and sense */
2318		csio->scsi_status = rep->SCSIStatus;
2319		if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD)
2320			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2321		else
2322			mpssas_set_ccbstatus(ccb, CAM_SCSI_STATUS_ERROR);
2323
2324		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2325			int sense_len, returned_sense_len;
2326
2327			returned_sense_len = min(le32toh(rep->SenseCount),
2328			    sizeof(struct scsi_sense_data));
2329			if (returned_sense_len < ccb->csio.sense_len)
2330				ccb->csio.sense_resid = ccb->csio.sense_len -
2331					returned_sense_len;
2332			else
2333				ccb->csio.sense_resid = 0;
2334
2335			sense_len = min(returned_sense_len,
2336			    ccb->csio.sense_len - ccb->csio.sense_resid);
2337			bzero(&ccb->csio.sense_data,
2338			      sizeof(ccb->csio.sense_data));
2339			bcopy(cm->cm_sense, &ccb->csio.sense_data, sense_len);
2340			ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2341		}
2342
2343		/*
2344		 * Check if this is an INQUIRY command.  If it's a VPD inquiry,
2345		 * and it's page code 0 (Supported Page List), and there is
2346		 * inquiry data, and this is for a sequential access device, and
2347		 * the device is an SSP target, and TLR is supported by the
2348		 * controller, turn the TLR_bits value ON if page 0x90 is
2349		 * supported.
2350		 */
2351		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2352		    (csio->cdb_io.cdb_bytes[1] & SI_EVPD) &&
2353		    (csio->cdb_io.cdb_bytes[2] == SVPD_SUPPORTED_PAGE_LIST) &&
2354		    ((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) &&
2355		    (csio->data_ptr != NULL) &&
2356		    ((csio->data_ptr[0] & 0x1f) == T_SEQUENTIAL) &&
2357		    (sc->control_TLR) &&
2358		    (sc->mapping_table[target_id].device_info &
2359		    MPI2_SAS_DEVICE_INFO_SSP_TARGET)) {
2360			vpd_list = (struct scsi_vpd_supported_page_list *)
2361			    csio->data_ptr;
2362			TLR_bits = &sc->mapping_table[target_id].TLR_bits;
2363			*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2364			TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
2365			alloc_len = ((u16)csio->cdb_io.cdb_bytes[3] << 8) +
2366			    csio->cdb_io.cdb_bytes[4];
2367			alloc_len -= csio->resid;
2368			for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
2369				if (vpd_list->list[i] == 0x90) {
2370					*TLR_bits = TLR_on;
2371					break;
2372				}
2373			}
2374		}
2375
2376		/*
2377		 * If this is a SATA direct-access end device, mark it so that
2378		 * a SCSI StartStopUnit command will be sent to it when the
2379		 * driver is being shutdown.
2380		 */
2381		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2382		    ((csio->data_ptr[0] & 0x1f) == T_DIRECT) &&
2383		    (sc->mapping_table[target_id].device_info &
2384		    MPI2_SAS_DEVICE_INFO_SATA_DEVICE) &&
2385		    ((sc->mapping_table[target_id].device_info &
2386		    MPI2_SAS_DEVICE_INFO_MASK_DEVICE_TYPE) ==
2387		    MPI2_SAS_DEVICE_INFO_END_DEVICE)) {
2388			target = &sassc->targets[target_id];
2389			target->supports_SSU = TRUE;
2390			mps_dprint(sc, MPS_XINFO, "Target %d supports SSU\n",
2391			    target_id);
2392		}
2393		break;
2394	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
2395	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
2396		/*
2397		 * If devinfo is 0 this will be a volume.  In that case don't
2398		 * tell CAM that the volume is not there.  We want volumes to
2399		 * be enumerated until they are deleted/removed, not just
2400		 * failed.
2401		 */
2402		if (cm->cm_targ->devinfo == 0)
2403			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2404		else
2405			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2406		break;
2407	case MPI2_IOCSTATUS_INVALID_SGL:
2408		mps_print_scsiio_cmd(sc, cm);
2409		mpssas_set_ccbstatus(ccb, CAM_UNREC_HBA_ERROR);
2410		break;
2411	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
2412		/*
2413		 * This is one of the responses that comes back when an I/O
2414		 * has been aborted.  If it is because of a timeout that we
2415		 * initiated, just set the status to CAM_CMD_TIMEOUT.
2416		 * Otherwise set it to CAM_REQ_ABORTED.  The effect on the
2417		 * command is the same (it gets retried, subject to the
2418		 * retry counter), the only difference is what gets printed
2419		 * on the console.
2420		 */
2421		if (cm->cm_state == MPS_CM_STATE_TIMEDOUT)
2422			mpssas_set_ccbstatus(ccb, CAM_CMD_TIMEOUT);
2423		else
2424			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2425		break;
2426	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
2427		/* resid is ignored for this condition */
2428		csio->resid = 0;
2429		mpssas_set_ccbstatus(ccb, CAM_DATA_RUN_ERR);
2430		break;
2431	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
2432	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
2433		/*
2434		 * These can sometimes be transient transport-related
2435		 * errors, and sometimes persistent drive-related errors.
2436		 * We used to retry these without decrementing the retry
2437		 * count by returning CAM_REQUEUE_REQ.  Unfortunately, if
2438		 * we hit a persistent drive problem that returns one of
2439		 * these error codes, we would retry indefinitely.  So,
2440		 * return CAM_REQ_CMP_ERROR so that we decrement the retry
2441		 * count and avoid infinite retries.  We're taking the
2442		 * potential risk of flagging false failures in the event
2443		 * of a topology-related error (e.g. a SAS expander problem
2444		 * causes a command addressed to a drive to fail), but
2445		 * avoiding getting into an infinite retry loop.
2446		 */
2447		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2448		mpssas_log_command(cm, MPS_INFO,
2449		    "terminated ioc %x loginfo %x scsi %x state %x xfer %u\n",
2450		    le16toh(rep->IOCStatus), le32toh(rep->IOCLogInfo),
2451		    rep->SCSIStatus, rep->SCSIState,
2452		    le32toh(rep->TransferCount));
2453		break;
2454	case MPI2_IOCSTATUS_INVALID_FUNCTION:
2455	case MPI2_IOCSTATUS_INTERNAL_ERROR:
2456	case MPI2_IOCSTATUS_INVALID_VPID:
2457	case MPI2_IOCSTATUS_INVALID_FIELD:
2458	case MPI2_IOCSTATUS_INVALID_STATE:
2459	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
2460	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
2461	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
2462	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
2463	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
2464	default:
2465		mpssas_log_command(cm, MPS_XINFO,
2466		    "completed ioc %x loginfo %x scsi %x state %x xfer %u\n",
2467		    le16toh(rep->IOCStatus), le32toh(rep->IOCLogInfo),
2468		    rep->SCSIStatus, rep->SCSIState,
2469		    le32toh(rep->TransferCount));
2470		csio->resid = cm->cm_length;
2471		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2472		break;
2473	}
2474
2475	mps_sc_failed_io_info(sc,csio,rep);
2476
2477	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2478		ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2479		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2480		mps_dprint(sc, MPS_XINFO, "Command completed, "
2481		    "unfreezing SIM queue\n");
2482	}
2483
2484	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2485		ccb->ccb_h.status |= CAM_DEV_QFRZN;
2486		xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2487	}
2488
2489	mps_free_command(sc, cm);
2490	xpt_done(ccb);
2491}
2492
2493/* All Request reached here are Endian safe */
2494static void
2495mpssas_direct_drive_io(struct mpssas_softc *sassc, struct mps_command *cm,
2496    union ccb *ccb) {
2497	pMpi2SCSIIORequest_t	pIO_req;
2498	struct mps_softc	*sc = sassc->sc;
2499	uint64_t		virtLBA;
2500	uint32_t		physLBA, stripe_offset, stripe_unit;
2501	uint32_t		io_size, column;
2502	uint8_t			*ptrLBA, lba_idx, physLBA_byte, *CDB;
2503
2504	/*
2505	 * If this is a valid SCSI command (Read6, Read10, Read16, Write6,
2506	 * Write10, or Write16), build a direct I/O message.  Otherwise, the I/O
2507	 * will be sent to the IR volume itself.  Since Read6 and Write6 are a
2508	 * bit different than the 10/16 CDBs, handle them separately.
2509	 */
2510	pIO_req = (pMpi2SCSIIORequest_t)cm->cm_req;
2511	CDB = pIO_req->CDB.CDB32;
2512
2513	/*
2514	 * Handle 6 byte CDBs.
2515	 */
2516	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_6) ||
2517	    (CDB[0] == WRITE_6))) {
2518		/*
2519		 * Get the transfer size in blocks.
2520		 */
2521		io_size = (cm->cm_length >> sc->DD_block_exponent);
2522
2523		/*
2524		 * Get virtual LBA given in the CDB.
2525		 */
2526		virtLBA = ((uint64_t)(CDB[1] & 0x1F) << 16) |
2527		    ((uint64_t)CDB[2] << 8) | (uint64_t)CDB[3];
2528
2529		/*
2530		 * Check that LBA range for I/O does not exceed volume's
2531		 * MaxLBA.
2532		 */
2533		if ((virtLBA + (uint64_t)io_size - 1) <=
2534		    sc->DD_max_lba) {
2535			/*
2536			 * Check if the I/O crosses a stripe boundary.  If not,
2537			 * translate the virtual LBA to a physical LBA and set
2538			 * the DevHandle for the PhysDisk to be used.  If it
2539			 * does cross a boundary, do normal I/O.  To get the
2540			 * right DevHandle to use, get the map number for the
2541			 * column, then use that map number to look up the
2542			 * DevHandle of the PhysDisk.
2543			 */
2544			stripe_offset = (uint32_t)virtLBA &
2545			    (sc->DD_stripe_size - 1);
2546			if ((stripe_offset + io_size) <= sc->DD_stripe_size) {
2547				physLBA = (uint32_t)virtLBA >>
2548				    sc->DD_stripe_exponent;
2549				stripe_unit = physLBA / sc->DD_num_phys_disks;
2550				column = physLBA % sc->DD_num_phys_disks;
2551				pIO_req->DevHandle =
2552				    htole16(sc->DD_column_map[column].dev_handle);
2553				/* ???? Is this endian safe*/
2554				cm->cm_desc.SCSIIO.DevHandle =
2555				    pIO_req->DevHandle;
2556
2557				physLBA = (stripe_unit <<
2558				    sc->DD_stripe_exponent) + stripe_offset;
2559				ptrLBA = &pIO_req->CDB.CDB32[1];
2560				physLBA_byte = (uint8_t)(physLBA >> 16);
2561				*ptrLBA = physLBA_byte;
2562				ptrLBA = &pIO_req->CDB.CDB32[2];
2563				physLBA_byte = (uint8_t)(physLBA >> 8);
2564				*ptrLBA = physLBA_byte;
2565				ptrLBA = &pIO_req->CDB.CDB32[3];
2566				physLBA_byte = (uint8_t)physLBA;
2567				*ptrLBA = physLBA_byte;
2568
2569				/*
2570				 * Set flag that Direct Drive I/O is
2571				 * being done.
2572				 */
2573				cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2574			}
2575		}
2576		return;
2577	}
2578
2579	/*
2580	 * Handle 10, 12 or 16 byte CDBs.
2581	 */
2582	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_10) ||
2583	    (CDB[0] == WRITE_10) || (CDB[0] == READ_16) ||
2584	    (CDB[0] == WRITE_16) || (CDB[0] == READ_12) ||
2585	    (CDB[0] == WRITE_12))) {
2586		/*
2587		 * For 16-byte CDB's, verify that the upper 4 bytes of the CDB
2588		 * are 0.  If not, this is accessing beyond 2TB so handle it in
2589		 * the else section.  10-byte and 12-byte CDB's are OK.
2590		 * FreeBSD sends very rare 12 byte READ/WRITE, but driver is
2591		 * ready to accept 12byte CDB for Direct IOs.
2592		 */
2593		if ((CDB[0] == READ_10 || CDB[0] == WRITE_10) ||
2594		    (CDB[0] == READ_12 || CDB[0] == WRITE_12) ||
2595		    !(CDB[2] | CDB[3] | CDB[4] | CDB[5])) {
2596			/*
2597			 * Get the transfer size in blocks.
2598			 */
2599			io_size = (cm->cm_length >> sc->DD_block_exponent);
2600
2601			/*
2602			 * Get virtual LBA.  Point to correct lower 4 bytes of
2603			 * LBA in the CDB depending on command.
2604			 */
2605			lba_idx = ((CDB[0] == READ_12) ||
2606				(CDB[0] == WRITE_12) ||
2607				(CDB[0] == READ_10) ||
2608				(CDB[0] == WRITE_10))? 2 : 6;
2609			virtLBA = ((uint64_t)CDB[lba_idx] << 24) |
2610			    ((uint64_t)CDB[lba_idx + 1] << 16) |
2611			    ((uint64_t)CDB[lba_idx + 2] << 8) |
2612			    (uint64_t)CDB[lba_idx + 3];
2613
2614			/*
2615			 * Check that LBA range for I/O does not exceed volume's
2616			 * MaxLBA.
2617			 */
2618			if ((virtLBA + (uint64_t)io_size - 1) <=
2619			    sc->DD_max_lba) {
2620				/*
2621				 * Check if the I/O crosses a stripe boundary.
2622				 * If not, translate the virtual LBA to a
2623				 * physical LBA and set the DevHandle for the
2624				 * PhysDisk to be used.  If it does cross a
2625				 * boundary, do normal I/O.  To get the right
2626				 * DevHandle to use, get the map number for the
2627				 * column, then use that map number to look up
2628				 * the DevHandle of the PhysDisk.
2629				 */
2630				stripe_offset = (uint32_t)virtLBA &
2631				    (sc->DD_stripe_size - 1);
2632				if ((stripe_offset + io_size) <=
2633				    sc->DD_stripe_size) {
2634					physLBA = (uint32_t)virtLBA >>
2635					    sc->DD_stripe_exponent;
2636					stripe_unit = physLBA /
2637					    sc->DD_num_phys_disks;
2638					column = physLBA %
2639					    sc->DD_num_phys_disks;
2640					pIO_req->DevHandle =
2641					    htole16(sc->DD_column_map[column].
2642					    dev_handle);
2643					cm->cm_desc.SCSIIO.DevHandle =
2644					    pIO_req->DevHandle;
2645
2646					physLBA = (stripe_unit <<
2647					    sc->DD_stripe_exponent) +
2648					    stripe_offset;
2649					ptrLBA =
2650					    &pIO_req->CDB.CDB32[lba_idx];
2651					physLBA_byte = (uint8_t)(physLBA >> 24);
2652					*ptrLBA = physLBA_byte;
2653					ptrLBA =
2654					    &pIO_req->CDB.CDB32[lba_idx + 1];
2655					physLBA_byte = (uint8_t)(physLBA >> 16);
2656					*ptrLBA = physLBA_byte;
2657					ptrLBA =
2658					    &pIO_req->CDB.CDB32[lba_idx + 2];
2659					physLBA_byte = (uint8_t)(physLBA >> 8);
2660					*ptrLBA = physLBA_byte;
2661					ptrLBA =
2662					    &pIO_req->CDB.CDB32[lba_idx + 3];
2663					physLBA_byte = (uint8_t)physLBA;
2664					*ptrLBA = physLBA_byte;
2665
2666					/*
2667					 * Set flag that Direct Drive I/O is
2668					 * being done.
2669					 */
2670					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2671				}
2672			}
2673		} else {
2674			/*
2675			 * 16-byte CDB and the upper 4 bytes of the CDB are not
2676			 * 0.  Get the transfer size in blocks.
2677			 */
2678			io_size = (cm->cm_length >> sc->DD_block_exponent);
2679
2680			/*
2681			 * Get virtual LBA.
2682			 */
2683			virtLBA = ((uint64_t)CDB[2] << 54) |
2684			    ((uint64_t)CDB[3] << 48) |
2685			    ((uint64_t)CDB[4] << 40) |
2686			    ((uint64_t)CDB[5] << 32) |
2687			    ((uint64_t)CDB[6] << 24) |
2688			    ((uint64_t)CDB[7] << 16) |
2689			    ((uint64_t)CDB[8] << 8) |
2690			    (uint64_t)CDB[9];
2691
2692			/*
2693			 * Check that LBA range for I/O does not exceed volume's
2694			 * MaxLBA.
2695			 */
2696			if ((virtLBA + (uint64_t)io_size - 1) <=
2697			    sc->DD_max_lba) {
2698				/*
2699				 * Check if the I/O crosses a stripe boundary.
2700				 * If not, translate the virtual LBA to a
2701				 * physical LBA and set the DevHandle for the
2702				 * PhysDisk to be used.  If it does cross a
2703				 * boundary, do normal I/O.  To get the right
2704				 * DevHandle to use, get the map number for the
2705				 * column, then use that map number to look up
2706				 * the DevHandle of the PhysDisk.
2707				 */
2708				stripe_offset = (uint32_t)virtLBA &
2709				    (sc->DD_stripe_size - 1);
2710				if ((stripe_offset + io_size) <=
2711				    sc->DD_stripe_size) {
2712					physLBA = (uint32_t)(virtLBA >>
2713					    sc->DD_stripe_exponent);
2714					stripe_unit = physLBA /
2715					    sc->DD_num_phys_disks;
2716					column = physLBA %
2717					    sc->DD_num_phys_disks;
2718					pIO_req->DevHandle =
2719					    htole16(sc->DD_column_map[column].
2720					    dev_handle);
2721					cm->cm_desc.SCSIIO.DevHandle =
2722					    pIO_req->DevHandle;
2723
2724					physLBA = (stripe_unit <<
2725					    sc->DD_stripe_exponent) +
2726					    stripe_offset;
2727
2728					/*
2729					 * Set upper 4 bytes of LBA to 0.  We
2730					 * assume that the phys disks are less
2731					 * than 2 TB's in size.  Then, set the
2732					 * lower 4 bytes.
2733					 */
2734					pIO_req->CDB.CDB32[2] = 0;
2735					pIO_req->CDB.CDB32[3] = 0;
2736					pIO_req->CDB.CDB32[4] = 0;
2737					pIO_req->CDB.CDB32[5] = 0;
2738					ptrLBA = &pIO_req->CDB.CDB32[6];
2739					physLBA_byte = (uint8_t)(physLBA >> 24);
2740					*ptrLBA = physLBA_byte;
2741					ptrLBA = &pIO_req->CDB.CDB32[7];
2742					physLBA_byte = (uint8_t)(physLBA >> 16);
2743					*ptrLBA = physLBA_byte;
2744					ptrLBA = &pIO_req->CDB.CDB32[8];
2745					physLBA_byte = (uint8_t)(physLBA >> 8);
2746					*ptrLBA = physLBA_byte;
2747					ptrLBA = &pIO_req->CDB.CDB32[9];
2748					physLBA_byte = (uint8_t)physLBA;
2749					*ptrLBA = physLBA_byte;
2750
2751					/*
2752					 * Set flag that Direct Drive I/O is
2753					 * being done.
2754					 */
2755					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2756				}
2757			}
2758		}
2759	}
2760}
2761
2762#if __FreeBSD_version >= 900026
2763static void
2764mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm)
2765{
2766	MPI2_SMP_PASSTHROUGH_REPLY *rpl;
2767	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2768	uint64_t sasaddr;
2769	union ccb *ccb;
2770
2771	ccb = cm->cm_complete_data;
2772
2773	/*
2774	 * Currently there should be no way we can hit this case.  It only
2775	 * happens when we have a failure to allocate chain frames, and SMP
2776	 * commands require two S/G elements only.  That should be handled
2777	 * in the standard request size.
2778	 */
2779	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2780		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x on SMP request!\n",
2781			   __func__, cm->cm_flags);
2782		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2783		goto bailout;
2784        }
2785
2786	rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply;
2787	if (rpl == NULL) {
2788		mps_dprint(sc, MPS_ERROR, "%s: NULL cm_reply!\n", __func__);
2789		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2790		goto bailout;
2791	}
2792
2793	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2794	sasaddr = le32toh(req->SASAddress.Low);
2795	sasaddr |= ((uint64_t)(le32toh(req->SASAddress.High))) << 32;
2796
2797	if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
2798	    MPI2_IOCSTATUS_SUCCESS ||
2799	    rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) {
2800		mps_dprint(sc, MPS_XINFO, "%s: IOCStatus %04x SASStatus %02x\n",
2801		    __func__, le16toh(rpl->IOCStatus), rpl->SASStatus);
2802		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2803		goto bailout;
2804	}
2805
2806	mps_dprint(sc, MPS_XINFO, "%s: SMP request to SAS address "
2807		   "%#jx completed successfully\n", __func__,
2808		   (uintmax_t)sasaddr);
2809
2810	if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED)
2811		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2812	else
2813		mpssas_set_ccbstatus(ccb, CAM_SMP_STATUS_ERROR);
2814
2815bailout:
2816	/*
2817	 * We sync in both directions because we had DMAs in the S/G list
2818	 * in both directions.
2819	 */
2820	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
2821			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2822	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2823	mps_free_command(sc, cm);
2824	xpt_done(ccb);
2825}
2826
2827static void
2828mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb, uint64_t sasaddr)
2829{
2830	struct mps_command *cm;
2831	uint8_t *request, *response;
2832	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2833	struct mps_softc *sc;
2834	int error;
2835
2836	sc = sassc->sc;
2837	error = 0;
2838
2839	/*
2840	 * XXX We don't yet support physical addresses here.
2841	 */
2842	switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) {
2843	case CAM_DATA_PADDR:
2844	case CAM_DATA_SG_PADDR:
2845		mps_dprint(sc, MPS_ERROR,
2846			   "%s: physical addresses not supported\n", __func__);
2847		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2848		xpt_done(ccb);
2849		return;
2850	case CAM_DATA_SG:
2851		/*
2852		 * The chip does not support more than one buffer for the
2853		 * request or response.
2854		 */
2855	 	if ((ccb->smpio.smp_request_sglist_cnt > 1)
2856		  || (ccb->smpio.smp_response_sglist_cnt > 1)) {
2857			mps_dprint(sc, MPS_ERROR,
2858				   "%s: multiple request or response "
2859				   "buffer segments not supported for SMP\n",
2860				   __func__);
2861			mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2862			xpt_done(ccb);
2863			return;
2864		}
2865
2866		/*
2867		 * The CAM_SCATTER_VALID flag was originally implemented
2868		 * for the XPT_SCSI_IO CCB, which only has one data pointer.
2869		 * We have two.  So, just take that flag to mean that we
2870		 * might have S/G lists, and look at the S/G segment count
2871		 * to figure out whether that is the case for each individual
2872		 * buffer.
2873		 */
2874		if (ccb->smpio.smp_request_sglist_cnt != 0) {
2875			bus_dma_segment_t *req_sg;
2876
2877			req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request;
2878			request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr;
2879		} else
2880			request = ccb->smpio.smp_request;
2881
2882		if (ccb->smpio.smp_response_sglist_cnt != 0) {
2883			bus_dma_segment_t *rsp_sg;
2884
2885			rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response;
2886			response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr;
2887		} else
2888			response = ccb->smpio.smp_response;
2889		break;
2890	case CAM_DATA_VADDR:
2891		request = ccb->smpio.smp_request;
2892		response = ccb->smpio.smp_response;
2893		break;
2894	default:
2895		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2896		xpt_done(ccb);
2897		return;
2898	}
2899
2900	cm = mps_alloc_command(sc);
2901	if (cm == NULL) {
2902		mps_dprint(sc, MPS_ERROR,
2903		    "%s: cannot allocate command\n", __func__);
2904		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2905		xpt_done(ccb);
2906		return;
2907	}
2908
2909	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2910	bzero(req, sizeof(*req));
2911	req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH;
2912
2913	/* Allow the chip to use any route to this SAS address. */
2914	req->PhysicalPort = 0xff;
2915
2916	req->RequestDataLength = htole16(ccb->smpio.smp_request_len);
2917	req->SGLFlags =
2918	    MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE | MPI2_SGLFLAGS_SGL_TYPE_MPI;
2919
2920	mps_dprint(sc, MPS_XINFO, "%s: sending SMP request to SAS "
2921	    "address %#jx\n", __func__, (uintmax_t)sasaddr);
2922
2923	mpi_init_sge(cm, req, &req->SGL);
2924
2925	/*
2926	 * Set up a uio to pass into mps_map_command().  This allows us to
2927	 * do one map command, and one busdma call in there.
2928	 */
2929	cm->cm_uio.uio_iov = cm->cm_iovec;
2930	cm->cm_uio.uio_iovcnt = 2;
2931	cm->cm_uio.uio_segflg = UIO_SYSSPACE;
2932
2933	/*
2934	 * The read/write flag isn't used by busdma, but set it just in
2935	 * case.  This isn't exactly accurate, either, since we're going in
2936	 * both directions.
2937	 */
2938	cm->cm_uio.uio_rw = UIO_WRITE;
2939
2940	cm->cm_iovec[0].iov_base = request;
2941	cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength);
2942	cm->cm_iovec[1].iov_base = response;
2943	cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len;
2944
2945	cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len +
2946			       cm->cm_iovec[1].iov_len;
2947
2948	/*
2949	 * Trigger a warning message in mps_data_cb() for the user if we
2950	 * wind up exceeding two S/G segments.  The chip expects one
2951	 * segment for the request and another for the response.
2952	 */
2953	cm->cm_max_segs = 2;
2954
2955	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
2956	cm->cm_complete = mpssas_smpio_complete;
2957	cm->cm_complete_data = ccb;
2958
2959	/*
2960	 * Tell the mapping code that we're using a uio, and that this is
2961	 * an SMP passthrough request.  There is a little special-case
2962	 * logic there (in mps_data_cb()) to handle the bidirectional
2963	 * transfer.
2964	 */
2965	cm->cm_flags |= MPS_CM_FLAGS_USE_UIO | MPS_CM_FLAGS_SMP_PASS |
2966			MPS_CM_FLAGS_DATAIN | MPS_CM_FLAGS_DATAOUT;
2967
2968	/* The chip data format is little endian. */
2969	req->SASAddress.High = htole32(sasaddr >> 32);
2970	req->SASAddress.Low = htole32(sasaddr);
2971
2972	/*
2973	 * XXX Note that we don't have a timeout/abort mechanism here.
2974	 * From the manual, it looks like task management requests only
2975	 * work for SCSI IO and SATA passthrough requests.  We may need to
2976	 * have a mechanism to retry requests in the event of a chip reset
2977	 * at least.  Hopefully the chip will insure that any errors short
2978	 * of that are relayed back to the driver.
2979	 */
2980	error = mps_map_command(sc, cm);
2981	if ((error != 0) && (error != EINPROGRESS)) {
2982		mps_dprint(sc, MPS_ERROR,
2983			   "%s: error %d returned from mps_map_command()\n",
2984			   __func__, error);
2985		goto bailout_error;
2986	}
2987
2988	return;
2989
2990bailout_error:
2991	mps_free_command(sc, cm);
2992	mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2993	xpt_done(ccb);
2994	return;
2995
2996}
2997
2998static void
2999mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb)
3000{
3001	struct mps_softc *sc;
3002	struct mpssas_target *targ;
3003	uint64_t sasaddr = 0;
3004
3005	sc = sassc->sc;
3006
3007	/*
3008	 * Make sure the target exists.
3009	 */
3010	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
3011	    ("Target %d out of bounds in XPT_SMP_IO\n", ccb->ccb_h.target_id));
3012	targ = &sassc->targets[ccb->ccb_h.target_id];
3013	if (targ->handle == 0x0) {
3014		mps_dprint(sc, MPS_ERROR,
3015			   "%s: target %d does not exist!\n", __func__,
3016			   ccb->ccb_h.target_id);
3017		mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
3018		xpt_done(ccb);
3019		return;
3020	}
3021
3022	/*
3023	 * If this device has an embedded SMP target, we'll talk to it
3024	 * directly.
3025	 * figure out what the expander's address is.
3026	 */
3027	if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0)
3028		sasaddr = targ->sasaddr;
3029
3030	/*
3031	 * If we don't have a SAS address for the expander yet, try
3032	 * grabbing it from the page 0x83 information cached in the
3033	 * transport layer for this target.  LSI expanders report the
3034	 * expander SAS address as the port-associated SAS address in
3035	 * Inquiry VPD page 0x83.  Maxim expanders don't report it in page
3036	 * 0x83.
3037	 *
3038	 * XXX KDM disable this for now, but leave it commented out so that
3039	 * it is obvious that this is another possible way to get the SAS
3040	 * address.
3041	 *
3042	 * The parent handle method below is a little more reliable, and
3043	 * the other benefit is that it works for devices other than SES
3044	 * devices.  So you can send a SMP request to a da(4) device and it
3045	 * will get routed to the expander that device is attached to.
3046	 * (Assuming the da(4) device doesn't contain an SMP target...)
3047	 */
3048#if 0
3049	if (sasaddr == 0)
3050		sasaddr = xpt_path_sas_addr(ccb->ccb_h.path);
3051#endif
3052
3053	/*
3054	 * If we still don't have a SAS address for the expander, look for
3055	 * the parent device of this device, which is probably the expander.
3056	 */
3057	if (sasaddr == 0) {
3058#ifdef OLD_MPS_PROBE
3059		struct mpssas_target *parent_target;
3060#endif
3061
3062		if (targ->parent_handle == 0x0) {
3063			mps_dprint(sc, MPS_ERROR,
3064				   "%s: handle %d does not have a valid "
3065				   "parent handle!\n", __func__, targ->handle);
3066			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3067			goto bailout;
3068		}
3069#ifdef OLD_MPS_PROBE
3070		parent_target = mpssas_find_target_by_handle(sassc, 0,
3071			targ->parent_handle);
3072
3073		if (parent_target == NULL) {
3074			mps_dprint(sc, MPS_ERROR,
3075				   "%s: handle %d does not have a valid "
3076				   "parent target!\n", __func__, targ->handle);
3077			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3078			goto bailout;
3079		}
3080
3081		if ((parent_target->devinfo &
3082		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3083			mps_dprint(sc, MPS_ERROR,
3084				   "%s: handle %d parent %d does not "
3085				   "have an SMP target!\n", __func__,
3086				   targ->handle, parent_target->handle);
3087			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3088			goto bailout;
3089
3090		}
3091
3092		sasaddr = parent_target->sasaddr;
3093#else /* OLD_MPS_PROBE */
3094		if ((targ->parent_devinfo &
3095		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3096			mps_dprint(sc, MPS_ERROR,
3097				   "%s: handle %d parent %d does not "
3098				   "have an SMP target!\n", __func__,
3099				   targ->handle, targ->parent_handle);
3100			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3101			goto bailout;
3102
3103		}
3104		if (targ->parent_sasaddr == 0x0) {
3105			mps_dprint(sc, MPS_ERROR,
3106				   "%s: handle %d parent handle %d does "
3107				   "not have a valid SAS address!\n",
3108				   __func__, targ->handle, targ->parent_handle);
3109			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3110			goto bailout;
3111		}
3112
3113		sasaddr = targ->parent_sasaddr;
3114#endif /* OLD_MPS_PROBE */
3115
3116	}
3117
3118	if (sasaddr == 0) {
3119		mps_dprint(sc, MPS_INFO,
3120			   "%s: unable to find SAS address for handle %d\n",
3121			   __func__, targ->handle);
3122		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3123		goto bailout;
3124	}
3125	mpssas_send_smpcmd(sassc, ccb, sasaddr);
3126
3127	return;
3128
3129bailout:
3130	xpt_done(ccb);
3131
3132}
3133#endif //__FreeBSD_version >= 900026
3134
3135static void
3136mpssas_action_resetdev(struct mpssas_softc *sassc, union ccb *ccb)
3137{
3138	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3139	struct mps_softc *sc;
3140	struct mps_command *tm;
3141	struct mpssas_target *targ;
3142
3143	MPS_FUNCTRACE(sassc->sc);
3144	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
3145
3146	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
3147	    ("Target %d out of bounds in XPT_RESET_DEV\n",
3148	     ccb->ccb_h.target_id));
3149	sc = sassc->sc;
3150	tm = mps_alloc_command(sc);
3151	if (tm == NULL) {
3152		mps_dprint(sc, MPS_ERROR,
3153		    "command alloc failure in mpssas_action_resetdev\n");
3154		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
3155		xpt_done(ccb);
3156		return;
3157	}
3158
3159	targ = &sassc->targets[ccb->ccb_h.target_id];
3160	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3161	req->DevHandle = htole16(targ->handle);
3162	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
3163	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
3164
3165	/* SAS Hard Link Reset / SATA Link Reset */
3166	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3167
3168	tm->cm_data = NULL;
3169	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
3170	tm->cm_complete = mpssas_resetdev_complete;
3171	tm->cm_complete_data = ccb;
3172	tm->cm_targ = targ;
3173	targ->flags |= MPSSAS_TARGET_INRESET;
3174
3175	mps_map_command(sc, tm);
3176}
3177
3178static void
3179mpssas_resetdev_complete(struct mps_softc *sc, struct mps_command *tm)
3180{
3181	MPI2_SCSI_TASK_MANAGE_REPLY *resp;
3182	union ccb *ccb;
3183
3184	MPS_FUNCTRACE(sc);
3185	mtx_assert(&sc->mps_mtx, MA_OWNED);
3186
3187	resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
3188	ccb = tm->cm_complete_data;
3189
3190	/*
3191	 * Currently there should be no way we can hit this case.  It only
3192	 * happens when we have a failure to allocate chain frames, and
3193	 * task management commands don't have S/G lists.
3194	 */
3195	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3196		MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3197
3198		req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3199
3200		mps_dprint(sc, MPS_ERROR,
3201			   "%s: cm_flags = %#x for reset of handle %#04x! "
3202			   "This should not happen!\n", __func__, tm->cm_flags,
3203			   req->DevHandle);
3204		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3205		goto bailout;
3206	}
3207
3208	mps_dprint(sc, MPS_XINFO,
3209	    "%s: IOCStatus = 0x%x ResponseCode = 0x%x\n", __func__,
3210	    le16toh(resp->IOCStatus), le32toh(resp->ResponseCode));
3211
3212	if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) {
3213		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
3214		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
3215		    CAM_LUN_WILDCARD);
3216	}
3217	else
3218		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3219
3220bailout:
3221
3222	mpssas_free_tm(sc, tm);
3223	xpt_done(ccb);
3224}
3225
3226static void
3227mpssas_poll(struct cam_sim *sim)
3228{
3229	struct mpssas_softc *sassc;
3230
3231	sassc = cam_sim_softc(sim);
3232
3233	if (sassc->sc->mps_debug & MPS_TRACE) {
3234		/* frequent debug messages during a panic just slow
3235		 * everything down too much.
3236		 */
3237		mps_printf(sassc->sc, "%s clearing MPS_TRACE\n", __func__);
3238		sassc->sc->mps_debug &= ~MPS_TRACE;
3239	}
3240
3241	mps_intr_locked(sassc->sc);
3242}
3243
3244static void
3245mpssas_async(void *callback_arg, uint32_t code, struct cam_path *path,
3246	     void *arg)
3247{
3248	struct mps_softc *sc;
3249
3250	sc = (struct mps_softc *)callback_arg;
3251
3252	switch (code) {
3253#if (__FreeBSD_version >= 1000006) || \
3254    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
3255	case AC_ADVINFO_CHANGED: {
3256		struct mpssas_target *target;
3257		struct mpssas_softc *sassc;
3258		struct scsi_read_capacity_data_long rcap_buf;
3259		struct ccb_dev_advinfo cdai;
3260		struct mpssas_lun *lun;
3261		lun_id_t lunid;
3262		int found_lun;
3263		uintptr_t buftype;
3264
3265		buftype = (uintptr_t)arg;
3266
3267		found_lun = 0;
3268		sassc = sc->sassc;
3269
3270		/*
3271		 * We're only interested in read capacity data changes.
3272		 */
3273		if (buftype != CDAI_TYPE_RCAPLONG)
3274			break;
3275
3276		/*
3277		 * We should have a handle for this, but check to make sure.
3278		 */
3279		KASSERT(xpt_path_target_id(path) < sassc->maxtargets,
3280		    ("Target %d out of bounds in mpssas_async\n",
3281		    xpt_path_target_id(path)));
3282		target = &sassc->targets[xpt_path_target_id(path)];
3283		if (target->handle == 0)
3284			break;
3285
3286		lunid = xpt_path_lun_id(path);
3287
3288		SLIST_FOREACH(lun, &target->luns, lun_link) {
3289			if (lun->lun_id == lunid) {
3290				found_lun = 1;
3291				break;
3292			}
3293		}
3294
3295		if (found_lun == 0) {
3296			lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3297				     M_NOWAIT | M_ZERO);
3298			if (lun == NULL) {
3299				mps_dprint(sc, MPS_ERROR, "Unable to alloc "
3300					   "LUN for EEDP support.\n");
3301				break;
3302			}
3303			lun->lun_id = lunid;
3304			SLIST_INSERT_HEAD(&target->luns, lun, lun_link);
3305		}
3306
3307		bzero(&rcap_buf, sizeof(rcap_buf));
3308		xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
3309		cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
3310		cdai.ccb_h.flags = CAM_DIR_IN;
3311		cdai.buftype = CDAI_TYPE_RCAPLONG;
3312#if (__FreeBSD_version >= 1100061) || \
3313    ((__FreeBSD_version >= 1001510) && (__FreeBSD_version < 1100000))
3314		cdai.flags = CDAI_FLAG_NONE;
3315#else
3316		cdai.flags = 0;
3317#endif
3318		cdai.bufsiz = sizeof(rcap_buf);
3319		cdai.buf = (uint8_t *)&rcap_buf;
3320		xpt_action((union ccb *)&cdai);
3321		if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
3322			cam_release_devq(cdai.ccb_h.path,
3323					 0, 0, 0, FALSE);
3324
3325		if ((mpssas_get_ccbstatus((union ccb *)&cdai) == CAM_REQ_CMP)
3326		 && (rcap_buf.prot & SRC16_PROT_EN)) {
3327			lun->eedp_formatted = TRUE;
3328			lun->eedp_block_size = scsi_4btoul(rcap_buf.length);
3329		} else {
3330			lun->eedp_formatted = FALSE;
3331			lun->eedp_block_size = 0;
3332		}
3333		break;
3334	}
3335#else
3336	case AC_FOUND_DEVICE: {
3337		struct ccb_getdev *cgd;
3338
3339		cgd = arg;
3340		mpssas_check_eedp(sc, path, cgd);
3341		break;
3342	}
3343#endif
3344	default:
3345		break;
3346	}
3347}
3348
3349#if (__FreeBSD_version < 901503) || \
3350    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
3351static void
3352mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
3353		  struct ccb_getdev *cgd)
3354{
3355	struct mpssas_softc *sassc = sc->sassc;
3356	struct ccb_scsiio *csio;
3357	struct scsi_read_capacity_16 *scsi_cmd;
3358	struct scsi_read_capacity_eedp *rcap_buf;
3359	path_id_t pathid;
3360	target_id_t targetid;
3361	lun_id_t lunid;
3362	union ccb *ccb;
3363	struct cam_path *local_path;
3364	struct mpssas_target *target;
3365	struct mpssas_lun *lun;
3366	uint8_t	found_lun;
3367	char path_str[64];
3368
3369	sassc = sc->sassc;
3370	pathid = cam_sim_path(sassc->sim);
3371	targetid = xpt_path_target_id(path);
3372	lunid = xpt_path_lun_id(path);
3373
3374	KASSERT(targetid < sassc->maxtargets,
3375	    ("Target %d out of bounds in mpssas_check_eedp\n",
3376	     targetid));
3377	target = &sassc->targets[targetid];
3378	if (target->handle == 0x0)
3379		return;
3380
3381	/*
3382	 * Determine if the device is EEDP capable.
3383	 *
3384	 * If this flag is set in the inquiry data,
3385	 * the device supports protection information,
3386	 * and must support the 16 byte read
3387	 * capacity command, otherwise continue without
3388	 * sending read cap 16
3389	 */
3390	if ((cgd->inq_data.spc3_flags & SPC3_SID_PROTECT) == 0)
3391		return;
3392
3393	/*
3394	 * Issue a READ CAPACITY 16 command.  This info
3395	 * is used to determine if the LUN is formatted
3396	 * for EEDP support.
3397	 */
3398	ccb = xpt_alloc_ccb_nowait();
3399	if (ccb == NULL) {
3400		mps_dprint(sc, MPS_ERROR, "Unable to alloc CCB "
3401		    "for EEDP support.\n");
3402		return;
3403	}
3404
3405	if (xpt_create_path(&local_path, xpt_periph,
3406	    pathid, targetid, lunid) != CAM_REQ_CMP) {
3407		mps_dprint(sc, MPS_ERROR, "Unable to create "
3408		    "path for EEDP support\n");
3409		xpt_free_ccb(ccb);
3410		return;
3411	}
3412
3413	/*
3414	 * If LUN is already in list, don't create a new
3415	 * one.
3416	 */
3417	found_lun = FALSE;
3418	SLIST_FOREACH(lun, &target->luns, lun_link) {
3419		if (lun->lun_id == lunid) {
3420			found_lun = TRUE;
3421			break;
3422		}
3423	}
3424	if (!found_lun) {
3425		lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3426		    M_NOWAIT | M_ZERO);
3427		if (lun == NULL) {
3428			mps_dprint(sc, MPS_ERROR,
3429			    "Unable to alloc LUN for EEDP support.\n");
3430			xpt_free_path(local_path);
3431			xpt_free_ccb(ccb);
3432			return;
3433		}
3434		lun->lun_id = lunid;
3435		SLIST_INSERT_HEAD(&target->luns, lun,
3436		    lun_link);
3437	}
3438
3439	xpt_path_string(local_path, path_str, sizeof(path_str));
3440
3441	mps_dprint(sc, MPS_INFO, "Sending read cap: path %s handle %d\n",
3442	    path_str, target->handle);
3443
3444	/*
3445	 * Issue a READ CAPACITY 16 command for the LUN.
3446	 * The mpssas_read_cap_done function will load
3447	 * the read cap info into the LUN struct.
3448	 */
3449	rcap_buf = malloc(sizeof(struct scsi_read_capacity_eedp),
3450	    M_MPT2, M_NOWAIT | M_ZERO);
3451	if (rcap_buf == NULL) {
3452		mps_dprint(sc, MPS_FAULT,
3453		    "Unable to alloc read capacity buffer for EEDP support.\n");
3454		xpt_free_path(ccb->ccb_h.path);
3455		xpt_free_ccb(ccb);
3456		return;
3457	}
3458	xpt_setup_ccb(&ccb->ccb_h, local_path, CAM_PRIORITY_XPT);
3459	csio = &ccb->csio;
3460	csio->ccb_h.func_code = XPT_SCSI_IO;
3461	csio->ccb_h.flags = CAM_DIR_IN;
3462	csio->ccb_h.retry_count = 4;
3463	csio->ccb_h.cbfcnp = mpssas_read_cap_done;
3464	csio->ccb_h.timeout = 60000;
3465	csio->data_ptr = (uint8_t *)rcap_buf;
3466	csio->dxfer_len = sizeof(struct scsi_read_capacity_eedp);
3467	csio->sense_len = MPS_SENSE_LEN;
3468	csio->cdb_len = sizeof(*scsi_cmd);
3469	csio->tag_action = MSG_SIMPLE_Q_TAG;
3470
3471	scsi_cmd = (struct scsi_read_capacity_16 *)&csio->cdb_io.cdb_bytes;
3472	bzero(scsi_cmd, sizeof(*scsi_cmd));
3473	scsi_cmd->opcode = 0x9E;
3474	scsi_cmd->service_action = SRC16_SERVICE_ACTION;
3475	((uint8_t *)scsi_cmd)[13] = sizeof(struct scsi_read_capacity_eedp);
3476
3477	ccb->ccb_h.ppriv_ptr1 = sassc;
3478	xpt_action(ccb);
3479}
3480
3481static void
3482mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb)
3483{
3484	struct mpssas_softc *sassc;
3485	struct mpssas_target *target;
3486	struct mpssas_lun *lun;
3487	struct scsi_read_capacity_eedp *rcap_buf;
3488
3489	if (done_ccb == NULL)
3490		return;
3491
3492	/* Driver need to release devq, it Scsi command is
3493	 * generated by driver internally.
3494	 * Currently there is a single place where driver
3495	 * calls scsi command internally. In future if driver
3496	 * calls more scsi command internally, it needs to release
3497	 * devq internally, since those command will not go back to
3498	 * cam_periph.
3499	 */
3500	if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) ) {
3501        	done_ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
3502		xpt_release_devq(done_ccb->ccb_h.path,
3503			       	/*count*/ 1, /*run_queue*/TRUE);
3504	}
3505
3506	rcap_buf = (struct scsi_read_capacity_eedp *)done_ccb->csio.data_ptr;
3507
3508	/*
3509	 * Get the LUN ID for the path and look it up in the LUN list for the
3510	 * target.
3511	 */
3512	sassc = (struct mpssas_softc *)done_ccb->ccb_h.ppriv_ptr1;
3513	KASSERT(done_ccb->ccb_h.target_id < sassc->maxtargets,
3514	    ("Target %d out of bounds in mpssas_read_cap_done\n",
3515	     done_ccb->ccb_h.target_id));
3516	target = &sassc->targets[done_ccb->ccb_h.target_id];
3517	SLIST_FOREACH(lun, &target->luns, lun_link) {
3518		if (lun->lun_id != done_ccb->ccb_h.target_lun)
3519			continue;
3520
3521		/*
3522		 * Got the LUN in the target's LUN list.  Fill it in
3523		 * with EEDP info.  If the READ CAP 16 command had some
3524		 * SCSI error (common if command is not supported), mark
3525		 * the lun as not supporting EEDP and set the block size
3526		 * to 0.
3527		 */
3528		if ((mpssas_get_ccbstatus(done_ccb) != CAM_REQ_CMP)
3529		 || (done_ccb->csio.scsi_status != SCSI_STATUS_OK)) {
3530			lun->eedp_formatted = FALSE;
3531			lun->eedp_block_size = 0;
3532			break;
3533		}
3534
3535		if (rcap_buf->protect & 0x01) {
3536			mps_dprint(sassc->sc, MPS_INFO, "LUN %d for "
3537 			    "target ID %d is formatted for EEDP "
3538 			    "support.\n", done_ccb->ccb_h.target_lun,
3539 			    done_ccb->ccb_h.target_id);
3540			lun->eedp_formatted = TRUE;
3541			lun->eedp_block_size = scsi_4btoul(rcap_buf->length);
3542		}
3543		break;
3544	}
3545
3546	// Finished with this CCB and path.
3547	free(rcap_buf, M_MPT2);
3548	xpt_free_path(done_ccb->ccb_h.path);
3549	xpt_free_ccb(done_ccb);
3550}
3551#endif /* (__FreeBSD_version < 901503) || \
3552          ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006)) */
3553
3554void
3555mpssas_prepare_for_tm(struct mps_softc *sc, struct mps_command *tm,
3556    struct mpssas_target *target, lun_id_t lun_id)
3557{
3558	union ccb *ccb;
3559	path_id_t path_id;
3560
3561	/*
3562	 * Set the INRESET flag for this target so that no I/O will be sent to
3563	 * the target until the reset has completed.  If an I/O request does
3564	 * happen, the devq will be frozen.  The CCB holds the path which is
3565	 * used to release the devq.  The devq is released and the CCB is freed
3566	 * when the TM completes.
3567	 */
3568	ccb = xpt_alloc_ccb_nowait();
3569	if (ccb) {
3570		path_id = cam_sim_path(sc->sassc->sim);
3571		if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, path_id,
3572		    target->tid, lun_id) != CAM_REQ_CMP) {
3573			xpt_free_ccb(ccb);
3574		} else {
3575			tm->cm_ccb = ccb;
3576			tm->cm_targ = target;
3577			target->flags |= MPSSAS_TARGET_INRESET;
3578		}
3579	}
3580}
3581
3582int
3583mpssas_startup(struct mps_softc *sc)
3584{
3585
3586	/*
3587	 * Send the port enable message and set the wait_for_port_enable flag.
3588	 * This flag helps to keep the simq frozen until all discovery events
3589	 * are processed.
3590	 */
3591	sc->wait_for_port_enable = 1;
3592	mpssas_send_portenable(sc);
3593	return (0);
3594}
3595
3596static int
3597mpssas_send_portenable(struct mps_softc *sc)
3598{
3599	MPI2_PORT_ENABLE_REQUEST *request;
3600	struct mps_command *cm;
3601
3602	MPS_FUNCTRACE(sc);
3603
3604	if ((cm = mps_alloc_command(sc)) == NULL)
3605		return (EBUSY);
3606	request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req;
3607	request->Function = MPI2_FUNCTION_PORT_ENABLE;
3608	request->MsgFlags = 0;
3609	request->VP_ID = 0;
3610	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3611	cm->cm_complete = mpssas_portenable_complete;
3612	cm->cm_data = NULL;
3613	cm->cm_sge = NULL;
3614
3615	mps_map_command(sc, cm);
3616	mps_dprint(sc, MPS_XINFO,
3617	    "mps_send_portenable finished cm %p req %p complete %p\n",
3618	    cm, cm->cm_req, cm->cm_complete);
3619	return (0);
3620}
3621
3622static void
3623mpssas_portenable_complete(struct mps_softc *sc, struct mps_command *cm)
3624{
3625	MPI2_PORT_ENABLE_REPLY *reply;
3626	struct mpssas_softc *sassc;
3627
3628	MPS_FUNCTRACE(sc);
3629	sassc = sc->sassc;
3630
3631	/*
3632	 * Currently there should be no way we can hit this case.  It only
3633	 * happens when we have a failure to allocate chain frames, and
3634	 * port enable commands don't have S/G lists.
3635	 */
3636	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3637		mps_dprint(sc, MPS_ERROR, "%s: cm_flags = %#x for port enable! "
3638			   "This should not happen!\n", __func__, cm->cm_flags);
3639	}
3640
3641	reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply;
3642	if (reply == NULL)
3643		mps_dprint(sc, MPS_FAULT, "Portenable NULL reply\n");
3644	else if (le16toh(reply->IOCStatus & MPI2_IOCSTATUS_MASK) !=
3645	    MPI2_IOCSTATUS_SUCCESS)
3646		mps_dprint(sc, MPS_FAULT, "Portenable failed\n");
3647
3648	mps_free_command(sc, cm);
3649	if (sc->mps_ich.ich_arg != NULL) {
3650		mps_dprint(sc, MPS_XINFO, "disestablish config intrhook\n");
3651		config_intrhook_disestablish(&sc->mps_ich);
3652		sc->mps_ich.ich_arg = NULL;
3653	}
3654
3655	/*
3656	 * Get WarpDrive info after discovery is complete but before the scan
3657	 * starts.  At this point, all devices are ready to be exposed to the
3658	 * OS.  If devices should be hidden instead, take them out of the
3659	 * 'targets' array before the scan.  The devinfo for a disk will have
3660	 * some info and a volume's will be 0.  Use that to remove disks.
3661	 */
3662	mps_wd_config_pages(sc);
3663
3664	/*
3665	 * Done waiting for port enable to complete.  Decrement the refcount.
3666	 * If refcount is 0, discovery is complete and a rescan of the bus can
3667	 * take place.  Since the simq was explicitly frozen before port
3668	 * enable, it must be explicitly released here to keep the
3669	 * freeze/release count in sync.
3670	 */
3671	sc->wait_for_port_enable = 0;
3672	sc->port_enable_complete = 1;
3673	wakeup(&sc->port_enable_complete);
3674	mpssas_startup_decrement(sassc);
3675}
3676
3677int
3678mpssas_check_id(struct mpssas_softc *sassc, int id)
3679{
3680	struct mps_softc *sc = sassc->sc;
3681	char *ids;
3682	char *name;
3683
3684	ids = &sc->exclude_ids[0];
3685	while((name = strsep(&ids, ",")) != NULL) {
3686		if (name[0] == '\0')
3687			continue;
3688		if (strtol(name, NULL, 0) == (long)id)
3689			return (1);
3690	}
3691
3692	return (0);
3693}
3694
3695void
3696mpssas_realloc_targets(struct mps_softc *sc, int maxtargets)
3697{
3698	struct mpssas_softc *sassc;
3699	struct mpssas_lun *lun, *lun_tmp;
3700	struct mpssas_target *targ;
3701	int i;
3702
3703	sassc = sc->sassc;
3704	/*
3705	 * The number of targets is based on IOC Facts, so free all of
3706	 * the allocated LUNs for each target and then the target buffer
3707	 * itself.
3708	 */
3709	for (i=0; i< maxtargets; i++) {
3710		targ = &sassc->targets[i];
3711		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
3712			free(lun, M_MPT2);
3713		}
3714	}
3715	free(sassc->targets, M_MPT2);
3716
3717	sassc->targets = malloc(sizeof(struct mpssas_target) * maxtargets,
3718	    M_MPT2, M_WAITOK|M_ZERO);
3719	if (!sassc->targets) {
3720		panic("%s failed to alloc targets with error %d\n",
3721		    __func__, ENOMEM);
3722	}
3723}
3724