mps_sas.c revision a6307f58e22c5b8a10cbbc520dbdc2061cb2dd42
1/*-
2 * Copyright (c) 2009 Yahoo! Inc.
3 * Copyright (c) 2011, 2012 LSI Corp.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 * LSI MPT-Fusion Host Adapter FreeBSD
28 *
29 * $FreeBSD$
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD$");
34
35/* Communications core for LSI MPT2 */
36
37/* TODO Move headers to mpsvar */
38#include <sys/types.h>
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/kernel.h>
42#include <sys/selinfo.h>
43#include <sys/module.h>
44#include <sys/bus.h>
45#include <sys/conf.h>
46#include <sys/bio.h>
47#include <sys/malloc.h>
48#include <sys/uio.h>
49#include <sys/sysctl.h>
50#include <sys/endian.h>
51#include <sys/queue.h>
52#include <sys/kthread.h>
53#include <sys/taskqueue.h>
54#include <sys/sbuf.h>
55
56#include <machine/bus.h>
57#include <machine/resource.h>
58#include <sys/rman.h>
59
60#include <machine/stdarg.h>
61
62#include <cam/cam.h>
63#include <cam/cam_ccb.h>
64#include <cam/cam_xpt.h>
65#include <cam/cam_debug.h>
66#include <cam/cam_sim.h>
67#include <cam/cam_xpt_sim.h>
68#include <cam/cam_xpt_periph.h>
69#include <cam/cam_periph.h>
70#include <cam/scsi/scsi_all.h>
71#include <cam/scsi/scsi_message.h>
72#if __FreeBSD_version >= 900026
73#include <cam/scsi/smp_all.h>
74#endif
75
76#include <dev/mps/mpi/mpi2_type.h>
77#include <dev/mps/mpi/mpi2.h>
78#include <dev/mps/mpi/mpi2_ioc.h>
79#include <dev/mps/mpi/mpi2_sas.h>
80#include <dev/mps/mpi/mpi2_cnfg.h>
81#include <dev/mps/mpi/mpi2_init.h>
82#include <dev/mps/mpi/mpi2_tool.h>
83#include <dev/mps/mps_ioctl.h>
84#include <dev/mps/mpsvar.h>
85#include <dev/mps/mps_table.h>
86#include <dev/mps/mps_sas.h>
87
88#define MPSSAS_DISCOVERY_TIMEOUT	20
89#define MPSSAS_MAX_DISCOVERY_TIMEOUTS	10 /* 200 seconds */
90
91/*
92 * static array to check SCSI OpCode for EEDP protection bits
93 */
94#define	PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP
95#define	PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
96#define	PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
97static uint8_t op_code_prot[256] = {
98	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
99	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
100	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
101	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
102	0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
103	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
104	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
105	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
106	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
107	0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
108	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
109	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
110	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
111	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
112	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
113	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
114};
115
116MALLOC_DEFINE(M_MPSSAS, "MPSSAS", "MPS SAS memory");
117
118static void mpssas_discovery_timeout(void *data);
119static void mpssas_remove_device(struct mps_softc *, struct mps_command *);
120static void mpssas_remove_complete(struct mps_softc *, struct mps_command *);
121static void mpssas_action(struct cam_sim *sim, union ccb *ccb);
122static void mpssas_poll(struct cam_sim *sim);
123static void mpssas_scsiio_timeout(void *data);
124static void mpssas_abort_complete(struct mps_softc *sc, struct mps_command *cm);
125static void mpssas_direct_drive_io(struct mpssas_softc *sassc,
126    struct mps_command *cm, union ccb *ccb);
127static void mpssas_action_scsiio(struct mpssas_softc *, union ccb *);
128static void mpssas_scsiio_complete(struct mps_softc *, struct mps_command *);
129static void mpssas_action_resetdev(struct mpssas_softc *, union ccb *);
130#if __FreeBSD_version >= 900026
131static void mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm);
132static void mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb,
133			       uint64_t sasaddr);
134static void mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb);
135#endif //FreeBSD_version >= 900026
136static void mpssas_resetdev_complete(struct mps_softc *, struct mps_command *);
137static int  mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm, struct mps_command *cm);
138static int  mpssas_send_reset(struct mps_softc *sc, struct mps_command *tm, uint8_t type);
139static void mpssas_async(void *callback_arg, uint32_t code,
140			 struct cam_path *path, void *arg);
141#if (__FreeBSD_version < 901503) || \
142    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
143static void mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
144			      struct ccb_getdev *cgd);
145static void mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb);
146#endif
147static int mpssas_send_portenable(struct mps_softc *sc);
148static void mpssas_portenable_complete(struct mps_softc *sc,
149    struct mps_command *cm);
150
151struct mpssas_target *
152mpssas_find_target_by_handle(struct mpssas_softc *sassc, int start, uint16_t handle)
153{
154	struct mpssas_target *target;
155	int i;
156
157	for (i = start; i < sassc->sc->facts->MaxTargets; i++) {
158		target = &sassc->targets[i];
159		if (target->handle == handle)
160			return (target);
161	}
162
163	return (NULL);
164}
165
166/* we need to freeze the simq during attach and diag reset, to avoid failing
167 * commands before device handles have been found by discovery.  Since
168 * discovery involves reading config pages and possibly sending commands,
169 * discovery actions may continue even after we receive the end of discovery
170 * event, so refcount discovery actions instead of assuming we can unfreeze
171 * the simq when we get the event.
172 */
173void
174mpssas_startup_increment(struct mpssas_softc *sassc)
175{
176	MPS_FUNCTRACE(sassc->sc);
177
178	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
179		if (sassc->startup_refcount++ == 0) {
180			/* just starting, freeze the simq */
181			mps_dprint(sassc->sc, MPS_INIT,
182			    "%s freezing simq\n", __func__);
183			xpt_freeze_simq(sassc->sim, 1);
184		}
185		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
186		    sassc->startup_refcount);
187	}
188}
189
190void
191mpssas_startup_decrement(struct mpssas_softc *sassc)
192{
193	MPS_FUNCTRACE(sassc->sc);
194
195	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
196		if (--sassc->startup_refcount == 0) {
197			/* finished all discovery-related actions, release
198			 * the simq and rescan for the latest topology.
199			 */
200			mps_dprint(sassc->sc, MPS_INIT,
201			    "%s releasing simq\n", __func__);
202			sassc->flags &= ~MPSSAS_IN_STARTUP;
203#if __FreeBSD_version >= 1000039
204			xpt_release_boot();
205#else
206			xpt_release_simq(sassc->sim, 1);
207			mpssas_rescan_target(sassc->sc, NULL);
208#endif
209		}
210		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
211		    sassc->startup_refcount);
212	}
213}
214
215/* LSI's firmware requires us to stop sending commands when we're doing task
216 * management, so refcount the TMs and keep the simq frozen when any are in
217 * use.
218 */
219struct mps_command *
220mpssas_alloc_tm(struct mps_softc *sc)
221{
222	struct mps_command *tm;
223
224	MPS_FUNCTRACE(sc);
225	tm = mps_alloc_high_priority_command(sc);
226	if (tm != NULL) {
227		if (sc->sassc->tm_count++ == 0) {
228			mps_dprint(sc, MPS_RECOVERY,
229			    "%s freezing simq\n", __func__);
230			xpt_freeze_simq(sc->sassc->sim, 1);
231		}
232		mps_dprint(sc, MPS_RECOVERY, "%s tm_count %u\n", __func__,
233		    sc->sassc->tm_count);
234	}
235	return tm;
236}
237
238void
239mpssas_free_tm(struct mps_softc *sc, struct mps_command *tm)
240{
241	mps_dprint(sc, MPS_TRACE, "%s", __func__);
242	if (tm == NULL)
243		return;
244
245	/* if there are no TMs in use, we can release the simq.  We use our
246	 * own refcount so that it's easier for a diag reset to cleanup and
247	 * release the simq.
248	 */
249	if (--sc->sassc->tm_count == 0) {
250		mps_dprint(sc, MPS_RECOVERY, "%s releasing simq\n", __func__);
251		xpt_release_simq(sc->sassc->sim, 1);
252	}
253	mps_dprint(sc, MPS_RECOVERY, "%s tm_count %u\n", __func__,
254	    sc->sassc->tm_count);
255
256	mps_free_high_priority_command(sc, tm);
257}
258
259void
260mpssas_rescan_target(struct mps_softc *sc, struct mpssas_target *targ)
261{
262	struct mpssas_softc *sassc = sc->sassc;
263	path_id_t pathid;
264	target_id_t targetid;
265	union ccb *ccb;
266
267	MPS_FUNCTRACE(sc);
268	pathid = cam_sim_path(sassc->sim);
269	if (targ == NULL)
270		targetid = CAM_TARGET_WILDCARD;
271	else
272		targetid = targ - sassc->targets;
273
274	/*
275	 * Allocate a CCB and schedule a rescan.
276	 */
277	ccb = xpt_alloc_ccb_nowait();
278	if (ccb == NULL) {
279		mps_dprint(sc, MPS_ERROR, "unable to alloc CCB for rescan\n");
280		return;
281	}
282
283	if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
284	    targetid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
285		mps_dprint(sc, MPS_ERROR, "unable to create path for rescan\n");
286		xpt_free_ccb(ccb);
287		return;
288	}
289
290	if (targetid == CAM_TARGET_WILDCARD)
291		ccb->ccb_h.func_code = XPT_SCAN_BUS;
292	else
293		ccb->ccb_h.func_code = XPT_SCAN_TGT;
294
295	mps_dprint(sc, MPS_TRACE, "%s targetid %u\n", __func__, targetid);
296	xpt_rescan(ccb);
297}
298
299static void
300mpssas_log_command(struct mps_command *cm, u_int level, const char *fmt, ...)
301{
302	struct sbuf sb;
303	va_list ap;
304	char str[192];
305	char path_str[64];
306
307	if (cm == NULL)
308		return;
309
310	/* No need to be in here if debugging isn't enabled */
311	if ((cm->cm_sc->mps_debug & level) == 0)
312		return;
313
314	sbuf_new(&sb, str, sizeof(str), 0);
315
316	va_start(ap, fmt);
317
318	if (cm->cm_ccb != NULL) {
319		xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str,
320				sizeof(path_str));
321		sbuf_cat(&sb, path_str);
322		if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) {
323			scsi_command_string(&cm->cm_ccb->csio, &sb);
324			sbuf_printf(&sb, "length %d ",
325				    cm->cm_ccb->csio.dxfer_len);
326		}
327	}
328	else {
329		sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ",
330		    cam_sim_name(cm->cm_sc->sassc->sim),
331		    cam_sim_unit(cm->cm_sc->sassc->sim),
332		    cam_sim_bus(cm->cm_sc->sassc->sim),
333		    cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF,
334		    cm->cm_lun);
335	}
336
337	sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID);
338	sbuf_vprintf(&sb, fmt, ap);
339	sbuf_finish(&sb);
340	mps_dprint_field(cm->cm_sc, level, "%s", sbuf_data(&sb));
341
342	va_end(ap);
343}
344
345
346static void
347mpssas_remove_volume(struct mps_softc *sc, struct mps_command *tm)
348{
349	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
350	struct mpssas_target *targ;
351	uint16_t handle;
352
353	MPS_FUNCTRACE(sc);
354
355	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
356	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
357	targ = tm->cm_targ;
358
359	if (reply == NULL) {
360		/* XXX retry the remove after the diag reset completes? */
361		mps_dprint(sc, MPS_FAULT,
362		    "%s NULL reply reseting device 0x%04x\n", __func__, handle);
363		mpssas_free_tm(sc, tm);
364		return;
365	}
366
367	if (reply->IOCStatus != MPI2_IOCSTATUS_SUCCESS) {
368		mps_dprint(sc, MPS_FAULT,
369		   "IOCStatus = 0x%x while resetting device 0x%x\n",
370		   reply->IOCStatus, handle);
371		mpssas_free_tm(sc, tm);
372		return;
373	}
374
375	mps_dprint(sc, MPS_XINFO,
376	    "Reset aborted %u commands\n", reply->TerminationCount);
377	mps_free_reply(sc, tm->cm_reply_data);
378	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
379
380	mps_dprint(sc, MPS_XINFO,
381	    "clearing target %u handle 0x%04x\n", targ->tid, handle);
382
383	/*
384	 * Don't clear target if remove fails because things will get confusing.
385	 * Leave the devname and sasaddr intact so that we know to avoid reusing
386	 * this target id if possible, and so we can assign the same target id
387	 * to this device if it comes back in the future.
388	 */
389	if (reply->IOCStatus == MPI2_IOCSTATUS_SUCCESS) {
390		targ = tm->cm_targ;
391		targ->handle = 0x0;
392		targ->encl_handle = 0x0;
393		targ->encl_slot = 0x0;
394		targ->exp_dev_handle = 0x0;
395		targ->phy_num = 0x0;
396		targ->linkrate = 0x0;
397		targ->devinfo = 0x0;
398		targ->flags = 0x0;
399	}
400
401	mpssas_free_tm(sc, tm);
402}
403
404
405/*
406 * No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal.
407 * Otherwise Volume Delete is same as Bare Drive Removal.
408 */
409void
410mpssas_prepare_volume_remove(struct mpssas_softc *sassc, uint16_t handle)
411{
412	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
413	struct mps_softc *sc;
414	struct mps_command *cm;
415	struct mpssas_target *targ = NULL;
416
417	MPS_FUNCTRACE(sassc->sc);
418	sc = sassc->sc;
419
420#ifdef WD_SUPPORT
421	/*
422	 * If this is a WD controller, determine if the disk should be exposed
423	 * to the OS or not.  If disk should be exposed, return from this
424	 * function without doing anything.
425	 */
426	if (sc->WD_available && (sc->WD_hide_expose ==
427	    MPS_WD_EXPOSE_ALWAYS)) {
428		return;
429	}
430#endif //WD_SUPPORT
431
432	targ = mpssas_find_target_by_handle(sassc, 0, handle);
433	if (targ == NULL) {
434		/* FIXME: what is the action? */
435		/* We don't know about this device? */
436		mps_dprint(sc, MPS_ERROR,
437		   "%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
438		return;
439	}
440
441	targ->flags |= MPSSAS_TARGET_INREMOVAL;
442
443	cm = mpssas_alloc_tm(sc);
444	if (cm == NULL) {
445		mps_dprint(sc, MPS_ERROR,
446		    "%s: command alloc failure\n", __func__);
447		return;
448	}
449
450	mpssas_rescan_target(sc, targ);
451
452	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
453	req->DevHandle = targ->handle;
454	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
455	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
456
457	/* SAS Hard Link Reset / SATA Link Reset */
458	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
459
460	cm->cm_targ = targ;
461	cm->cm_data = NULL;
462	cm->cm_desc.HighPriority.RequestFlags =
463	    MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
464	cm->cm_complete = mpssas_remove_volume;
465	cm->cm_complete_data = (void *)(uintptr_t)handle;
466	mps_map_command(sc, cm);
467}
468
469/*
470 * The MPT2 firmware performs debounce on the link to avoid transient link
471 * errors and false removals.  When it does decide that link has been lost
472 * and a device need to go away, it expects that the host will perform a
473 * target reset and then an op remove.  The reset has the side-effect of
474 * aborting any outstanding requests for the device, which is required for
475 * the op-remove to succeed.  It's not clear if the host should check for
476 * the device coming back alive after the reset.
477 */
478void
479mpssas_prepare_remove(struct mpssas_softc *sassc, uint16_t handle)
480{
481	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
482	struct mps_softc *sc;
483	struct mps_command *cm;
484	struct mpssas_target *targ = NULL;
485
486	MPS_FUNCTRACE(sassc->sc);
487
488	sc = sassc->sc;
489
490	targ = mpssas_find_target_by_handle(sassc, 0, handle);
491	if (targ == NULL) {
492		/* FIXME: what is the action? */
493		/* We don't know about this device? */
494		mps_dprint(sc, MPS_ERROR,
495		    "%s : invalid handle 0x%x \n", __func__, handle);
496		return;
497	}
498
499	targ->flags |= MPSSAS_TARGET_INREMOVAL;
500
501	cm = mpssas_alloc_tm(sc);
502	if (cm == NULL) {
503		mps_dprint(sc, MPS_ERROR,
504		    "%s: command alloc failure\n", __func__);
505		return;
506	}
507
508	mpssas_rescan_target(sc, targ);
509
510	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
511	memset(req, 0, sizeof(*req));
512	req->DevHandle = htole16(targ->handle);
513	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
514	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
515
516	/* SAS Hard Link Reset / SATA Link Reset */
517	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
518
519	cm->cm_targ = targ;
520	cm->cm_data = NULL;
521	cm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
522	cm->cm_complete = mpssas_remove_device;
523	cm->cm_complete_data = (void *)(uintptr_t)handle;
524	mps_map_command(sc, cm);
525}
526
527static void
528mpssas_remove_device(struct mps_softc *sc, struct mps_command *tm)
529{
530	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
531	MPI2_SAS_IOUNIT_CONTROL_REQUEST *req;
532	struct mpssas_target *targ;
533	struct mps_command *next_cm;
534	uint16_t handle;
535
536	MPS_FUNCTRACE(sc);
537
538	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
539	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
540	targ = tm->cm_targ;
541
542	/*
543	 * Currently there should be no way we can hit this case.  It only
544	 * happens when we have a failure to allocate chain frames, and
545	 * task management commands don't have S/G lists.
546	 */
547	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
548		mps_dprint(sc, MPS_ERROR,
549		    "%s: cm_flags = %#x for remove of handle %#04x! "
550		    "This should not happen!\n", __func__, tm->cm_flags,
551		    handle);
552		mpssas_free_tm(sc, tm);
553		return;
554	}
555
556	if (reply == NULL) {
557		/* XXX retry the remove after the diag reset completes? */
558		mps_dprint(sc, MPS_FAULT,
559		    "%s NULL reply reseting device 0x%04x\n", __func__, handle);
560		mpssas_free_tm(sc, tm);
561		return;
562	}
563
564	if (le16toh(reply->IOCStatus) != MPI2_IOCSTATUS_SUCCESS) {
565		mps_dprint(sc, MPS_FAULT,
566		   "IOCStatus = 0x%x while resetting device 0x%x\n",
567		   le16toh(reply->IOCStatus), handle);
568		mpssas_free_tm(sc, tm);
569		return;
570	}
571
572	mps_dprint(sc, MPS_XINFO, "Reset aborted %u commands\n",
573	    le32toh(reply->TerminationCount));
574	mps_free_reply(sc, tm->cm_reply_data);
575	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
576
577	/* Reuse the existing command */
578	req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req;
579	memset(req, 0, sizeof(*req));
580	req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
581	req->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
582	req->DevHandle = htole16(handle);
583	tm->cm_data = NULL;
584	tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
585	tm->cm_complete = mpssas_remove_complete;
586	tm->cm_complete_data = (void *)(uintptr_t)handle;
587
588	mps_map_command(sc, tm);
589
590	mps_dprint(sc, MPS_XINFO, "clearing target %u handle 0x%04x\n",
591		   targ->tid, handle);
592	TAILQ_FOREACH_SAFE(tm, &targ->commands, cm_link, next_cm) {
593		union ccb *ccb;
594
595		mps_dprint(sc, MPS_XINFO, "Completing missed command %p\n", tm);
596		ccb = tm->cm_complete_data;
597		ccb->ccb_h.status = CAM_DEV_NOT_THERE;
598		mpssas_scsiio_complete(sc, tm);
599	}
600}
601
602static void
603mpssas_remove_complete(struct mps_softc *sc, struct mps_command *tm)
604{
605	MPI2_SAS_IOUNIT_CONTROL_REPLY *reply;
606	uint16_t handle;
607	struct mpssas_target *targ;
608	struct mpssas_lun *lun;
609
610	MPS_FUNCTRACE(sc);
611
612	reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply;
613	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
614
615	/*
616	 * Currently there should be no way we can hit this case.  It only
617	 * happens when we have a failure to allocate chain frames, and
618	 * task management commands don't have S/G lists.
619	 */
620	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
621		mps_dprint(sc, MPS_XINFO,
622			   "%s: cm_flags = %#x for remove of handle %#04x! "
623			   "This should not happen!\n", __func__, tm->cm_flags,
624			   handle);
625		mpssas_free_tm(sc, tm);
626		return;
627	}
628
629	if (reply == NULL) {
630		/* most likely a chip reset */
631		mps_dprint(sc, MPS_FAULT,
632		    "%s NULL reply removing device 0x%04x\n", __func__, handle);
633		mpssas_free_tm(sc, tm);
634		return;
635	}
636
637	mps_dprint(sc, MPS_XINFO,
638	    "%s on handle 0x%04x, IOCStatus= 0x%x\n", __func__,
639	    handle, le16toh(reply->IOCStatus));
640
641	/*
642	 * Don't clear target if remove fails because things will get confusing.
643	 * Leave the devname and sasaddr intact so that we know to avoid reusing
644	 * this target id if possible, and so we can assign the same target id
645	 * to this device if it comes back in the future.
646	 */
647	if (le16toh(reply->IOCStatus) == MPI2_IOCSTATUS_SUCCESS) {
648		targ = tm->cm_targ;
649		targ->handle = 0x0;
650		targ->encl_handle = 0x0;
651		targ->encl_slot = 0x0;
652		targ->exp_dev_handle = 0x0;
653		targ->phy_num = 0x0;
654		targ->linkrate = 0x0;
655		targ->devinfo = 0x0;
656		targ->flags = 0x0;
657
658		while(!SLIST_EMPTY(&targ->luns)) {
659			lun = SLIST_FIRST(&targ->luns);
660			SLIST_REMOVE_HEAD(&targ->luns, lun_link);
661			free(lun, M_MPT2);
662		}
663	}
664
665
666	mpssas_free_tm(sc, tm);
667}
668
669static int
670mpssas_register_events(struct mps_softc *sc)
671{
672	u32 events[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];
673
674	bzero(events, 16);
675	setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
676	setbit(events, MPI2_EVENT_SAS_DISCOVERY);
677	setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
678	setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE);
679	setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW);
680	setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
681	setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
682	setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
683	setbit(events, MPI2_EVENT_IR_VOLUME);
684	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
685	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
686	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);
687
688	mps_register_events(sc, events, mpssas_evt_handler, NULL,
689	    &sc->sassc->mpssas_eh);
690
691	return (0);
692}
693
694int
695mps_attach_sas(struct mps_softc *sc)
696{
697	struct mpssas_softc *sassc;
698	cam_status status;
699	int unit, error = 0;
700
701	MPS_FUNCTRACE(sc);
702
703	sassc = malloc(sizeof(struct mpssas_softc), M_MPT2, M_WAITOK|M_ZERO);
704	if(!sassc) {
705		device_printf(sc->mps_dev, "Cannot allocate memory %s %d\n",
706		__func__, __LINE__);
707		return (ENOMEM);
708	}
709	sassc->targets = malloc(sizeof(struct mpssas_target) *
710	    sc->facts->MaxTargets, M_MPT2, M_WAITOK|M_ZERO);
711	if(!sassc->targets) {
712		device_printf(sc->mps_dev, "Cannot allocate memory %s %d\n",
713		__func__, __LINE__);
714		free(sassc, M_MPT2);
715		return (ENOMEM);
716	}
717	sc->sassc = sassc;
718	sassc->sc = sc;
719
720	if ((sassc->devq = cam_simq_alloc(sc->num_reqs)) == NULL) {
721		mps_dprint(sc, MPS_ERROR, "Cannot allocate SIMQ\n");
722		error = ENOMEM;
723		goto out;
724	}
725
726	unit = device_get_unit(sc->mps_dev);
727	sassc->sim = cam_sim_alloc(mpssas_action, mpssas_poll, "mps", sassc,
728	    unit, &sc->mps_mtx, sc->num_reqs, sc->num_reqs, sassc->devq);
729	if (sassc->sim == NULL) {
730		mps_dprint(sc, MPS_ERROR, "Cannot allocate SIM\n");
731		error = EINVAL;
732		goto out;
733	}
734
735	TAILQ_INIT(&sassc->ev_queue);
736
737	/* Initialize taskqueue for Event Handling */
738	TASK_INIT(&sassc->ev_task, 0, mpssas_firmware_event_work, sc);
739	sassc->ev_tq = taskqueue_create("mps_taskq", M_NOWAIT | M_ZERO,
740	    taskqueue_thread_enqueue, &sassc->ev_tq);
741
742	/* Run the task queue with lowest priority */
743	taskqueue_start_threads(&sassc->ev_tq, 1, 255, "%s taskq",
744	    device_get_nameunit(sc->mps_dev));
745
746	mps_lock(sc);
747
748	/*
749	 * XXX There should be a bus for every port on the adapter, but since
750	 * we're just going to fake the topology for now, we'll pretend that
751	 * everything is just a target on a single bus.
752	 */
753	if ((error = xpt_bus_register(sassc->sim, sc->mps_dev, 0)) != 0) {
754		mps_dprint(sc, MPS_ERROR, "Error %d registering SCSI bus\n",
755		    error);
756		mps_unlock(sc);
757		goto out;
758	}
759
760	/*
761	 * Assume that discovery events will start right away.
762	 *
763	 * Hold off boot until discovery is complete.
764	 */
765	sassc->flags |= MPSSAS_IN_STARTUP | MPSSAS_IN_DISCOVERY;
766#if __FreeBSD_version >= 1000039
767	xpt_hold_boot();
768#else
769	xpt_freeze_simq(sassc->sim, 1);
770#endif
771	sc->sassc->startup_refcount = 0;
772
773	callout_init(&sassc->discovery_callout, 1 /*mpsafe*/);
774	sassc->discovery_timeouts = 0;
775
776	sassc->tm_count = 0;
777
778	/*
779	 * Register for async events so we can determine the EEDP
780	 * capabilities of devices.
781	 */
782	status = xpt_create_path(&sassc->path, /*periph*/NULL,
783	    cam_sim_path(sc->sassc->sim), CAM_TARGET_WILDCARD,
784	    CAM_LUN_WILDCARD);
785	if (status != CAM_REQ_CMP) {
786		mps_printf(sc, "Error %#x creating sim path\n", status);
787		sassc->path = NULL;
788	} else {
789		int event;
790
791#if (__FreeBSD_version >= 1000006) || \
792    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
793		event = AC_ADVINFO_CHANGED;
794#else
795		event = AC_FOUND_DEVICE;
796#endif
797		status = xpt_register_async(event, mpssas_async, sc,
798					    sassc->path);
799		if (status != CAM_REQ_CMP) {
800			mps_dprint(sc, MPS_ERROR,
801			    "Error %#x registering async handler for "
802			    "AC_ADVINFO_CHANGED events\n", status);
803			xpt_free_path(sassc->path);
804			sassc->path = NULL;
805		}
806	}
807	if (status != CAM_REQ_CMP) {
808		/*
809		 * EEDP use is the exception, not the rule.
810		 * Warn the user, but do not fail to attach.
811		 */
812		mps_printf(sc, "EEDP capabilities disabled.\n");
813	}
814
815	mps_unlock(sc);
816
817	mpssas_register_events(sc);
818out:
819	if (error)
820		mps_detach_sas(sc);
821	return (error);
822}
823
824int
825mps_detach_sas(struct mps_softc *sc)
826{
827	struct mpssas_softc *sassc;
828	struct mpssas_lun *lun, *lun_tmp;
829	struct mpssas_target *targ;
830	int i;
831
832	MPS_FUNCTRACE(sc);
833
834	if (sc->sassc == NULL)
835		return (0);
836
837	sassc = sc->sassc;
838	mps_deregister_events(sc, sassc->mpssas_eh);
839
840	/*
841	 * Drain and free the event handling taskqueue with the lock
842	 * unheld so that any parallel processing tasks drain properly
843	 * without deadlocking.
844	 */
845	if (sassc->ev_tq != NULL)
846		taskqueue_free(sassc->ev_tq);
847
848	/* Make sure CAM doesn't wedge if we had to bail out early. */
849	mps_lock(sc);
850
851	/* Deregister our async handler */
852	if (sassc->path != NULL) {
853		xpt_register_async(0, mpssas_async, sc, sassc->path);
854		xpt_free_path(sassc->path);
855		sassc->path = NULL;
856	}
857
858	if (sassc->flags & MPSSAS_IN_STARTUP)
859		xpt_release_simq(sassc->sim, 1);
860
861	if (sassc->sim != NULL) {
862		xpt_bus_deregister(cam_sim_path(sassc->sim));
863		cam_sim_free(sassc->sim, FALSE);
864	}
865
866	sassc->flags |= MPSSAS_SHUTDOWN;
867	mps_unlock(sc);
868
869	if (sassc->devq != NULL)
870		cam_simq_free(sassc->devq);
871
872	for(i=0; i< sc->facts->MaxTargets ;i++) {
873		targ = &sassc->targets[i];
874		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
875			free(lun, M_MPT2);
876		}
877	}
878	free(sassc->targets, M_MPT2);
879	free(sassc, M_MPT2);
880	sc->sassc = NULL;
881
882	return (0);
883}
884
885void
886mpssas_discovery_end(struct mpssas_softc *sassc)
887{
888	struct mps_softc *sc = sassc->sc;
889
890	MPS_FUNCTRACE(sc);
891
892	if (sassc->flags & MPSSAS_DISCOVERY_TIMEOUT_PENDING)
893		callout_stop(&sassc->discovery_callout);
894
895}
896
897static void
898mpssas_discovery_timeout(void *data)
899{
900	struct mpssas_softc *sassc = data;
901	struct mps_softc *sc;
902
903	sc = sassc->sc;
904	MPS_FUNCTRACE(sc);
905
906	mps_lock(sc);
907	mps_dprint(sc, MPS_INFO,
908	    "Timeout waiting for discovery, interrupts may not be working!\n");
909	sassc->flags &= ~MPSSAS_DISCOVERY_TIMEOUT_PENDING;
910
911	/* Poll the hardware for events in case interrupts aren't working */
912	mps_intr_locked(sc);
913
914	mps_dprint(sassc->sc, MPS_INFO,
915	    "Finished polling after discovery timeout at %d\n", ticks);
916
917	if ((sassc->flags & MPSSAS_IN_DISCOVERY) == 0) {
918		mpssas_discovery_end(sassc);
919	} else {
920		if (sassc->discovery_timeouts < MPSSAS_MAX_DISCOVERY_TIMEOUTS) {
921			sassc->flags |= MPSSAS_DISCOVERY_TIMEOUT_PENDING;
922			callout_reset(&sassc->discovery_callout,
923			    MPSSAS_DISCOVERY_TIMEOUT * hz,
924			    mpssas_discovery_timeout, sassc);
925			sassc->discovery_timeouts++;
926		} else {
927			mps_dprint(sassc->sc, MPS_FAULT,
928			    "Discovery timed out, continuing.\n");
929			sassc->flags &= ~MPSSAS_IN_DISCOVERY;
930			mpssas_discovery_end(sassc);
931		}
932	}
933
934	mps_unlock(sc);
935}
936
937static void
938mpssas_action(struct cam_sim *sim, union ccb *ccb)
939{
940	struct mpssas_softc *sassc;
941
942	sassc = cam_sim_softc(sim);
943
944	MPS_FUNCTRACE(sassc->sc);
945	mps_dprint(sassc->sc, MPS_TRACE, "ccb func_code 0x%x\n",
946	    ccb->ccb_h.func_code);
947	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
948
949	switch (ccb->ccb_h.func_code) {
950	case XPT_PATH_INQ:
951	{
952		struct ccb_pathinq *cpi = &ccb->cpi;
953
954		cpi->version_num = 1;
955		cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
956		cpi->target_sprt = 0;
957#if __FreeBSD_version >= 1000039
958		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN;
959#else
960		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED;
961#endif
962		cpi->hba_eng_cnt = 0;
963		cpi->max_target = sassc->sc->facts->MaxTargets - 1;
964		cpi->max_lun = 255;
965		cpi->initiator_id = sassc->sc->facts->MaxTargets - 1;
966		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
967		strncpy(cpi->hba_vid, "LSILogic", HBA_IDLEN);
968		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
969		cpi->unit_number = cam_sim_unit(sim);
970		cpi->bus_id = cam_sim_bus(sim);
971		cpi->base_transfer_speed = 150000;
972		cpi->transport = XPORT_SAS;
973		cpi->transport_version = 0;
974		cpi->protocol = PROTO_SCSI;
975		cpi->protocol_version = SCSI_REV_SPC;
976#if __FreeBSD_version >= 800001
977		/*
978		 * XXX KDM where does this number come from?
979		 */
980		cpi->maxio = 256 * 1024;
981#endif
982		cpi->ccb_h.status = CAM_REQ_CMP;
983		break;
984	}
985	case XPT_GET_TRAN_SETTINGS:
986	{
987		struct ccb_trans_settings	*cts;
988		struct ccb_trans_settings_sas	*sas;
989		struct ccb_trans_settings_scsi	*scsi;
990		struct mpssas_target *targ;
991
992		cts = &ccb->cts;
993		sas = &cts->xport_specific.sas;
994		scsi = &cts->proto_specific.scsi;
995
996		targ = &sassc->targets[cts->ccb_h.target_id];
997		if (targ->handle == 0x0) {
998			cts->ccb_h.status = CAM_SEL_TIMEOUT;
999			break;
1000		}
1001
1002		cts->protocol_version = SCSI_REV_SPC2;
1003		cts->transport = XPORT_SAS;
1004		cts->transport_version = 0;
1005
1006		sas->valid = CTS_SAS_VALID_SPEED;
1007		switch (targ->linkrate) {
1008		case 0x08:
1009			sas->bitrate = 150000;
1010			break;
1011		case 0x09:
1012			sas->bitrate = 300000;
1013			break;
1014		case 0x0a:
1015			sas->bitrate = 600000;
1016			break;
1017		default:
1018			sas->valid = 0;
1019		}
1020
1021		cts->protocol = PROTO_SCSI;
1022		scsi->valid = CTS_SCSI_VALID_TQ;
1023		scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
1024
1025		cts->ccb_h.status = CAM_REQ_CMP;
1026		break;
1027	}
1028	case XPT_CALC_GEOMETRY:
1029		cam_calc_geometry(&ccb->ccg, /*extended*/1);
1030		ccb->ccb_h.status = CAM_REQ_CMP;
1031		break;
1032	case XPT_RESET_DEV:
1033		mps_dprint(sassc->sc, MPS_XINFO, "mpssas_action XPT_RESET_DEV\n");
1034		mpssas_action_resetdev(sassc, ccb);
1035		return;
1036	case XPT_RESET_BUS:
1037	case XPT_ABORT:
1038	case XPT_TERM_IO:
1039		mps_dprint(sassc->sc, MPS_XINFO,
1040		    "mpssas_action faking success for abort or reset\n");
1041		ccb->ccb_h.status = CAM_REQ_CMP;
1042		break;
1043	case XPT_SCSI_IO:
1044		mpssas_action_scsiio(sassc, ccb);
1045		return;
1046#if __FreeBSD_version >= 900026
1047	case XPT_SMP_IO:
1048		mpssas_action_smpio(sassc, ccb);
1049		return;
1050#endif
1051	default:
1052		ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
1053		break;
1054	}
1055	xpt_done(ccb);
1056
1057}
1058
1059static void
1060mpssas_announce_reset(struct mps_softc *sc, uint32_t ac_code,
1061    target_id_t target_id, lun_id_t lun_id)
1062{
1063	path_id_t path_id = cam_sim_path(sc->sassc->sim);
1064	struct cam_path *path;
1065
1066	mps_dprint(sc, MPS_XINFO, "%s code %x target %d lun %d\n", __func__,
1067	    ac_code, target_id, lun_id);
1068
1069	if (xpt_create_path(&path, NULL,
1070		path_id, target_id, lun_id) != CAM_REQ_CMP) {
1071		mps_dprint(sc, MPS_ERROR, "unable to create path for reset "
1072			   "notification\n");
1073		return;
1074	}
1075
1076	xpt_async(ac_code, path, NULL);
1077	xpt_free_path(path);
1078}
1079
1080static void
1081mpssas_complete_all_commands(struct mps_softc *sc)
1082{
1083	struct mps_command *cm;
1084	int i;
1085	int completed;
1086
1087	MPS_FUNCTRACE(sc);
1088	mtx_assert(&sc->mps_mtx, MA_OWNED);
1089
1090	/* complete all commands with a NULL reply */
1091	for (i = 1; i < sc->num_reqs; i++) {
1092		cm = &sc->commands[i];
1093		cm->cm_reply = NULL;
1094		completed = 0;
1095
1096		if (cm->cm_flags & MPS_CM_FLAGS_POLLED)
1097			cm->cm_flags |= MPS_CM_FLAGS_COMPLETE;
1098
1099		if (cm->cm_complete != NULL) {
1100			mpssas_log_command(cm, MPS_RECOVERY,
1101			    "completing cm %p state %x ccb %p for diag reset\n",
1102			    cm, cm->cm_state, cm->cm_ccb);
1103
1104			cm->cm_complete(sc, cm);
1105			completed = 1;
1106		}
1107
1108		if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {
1109			mpssas_log_command(cm, MPS_RECOVERY,
1110			    "waking up cm %p state %x ccb %p for diag reset\n",
1111			    cm, cm->cm_state, cm->cm_ccb);
1112			wakeup(cm);
1113			completed = 1;
1114		}
1115
1116		if ((completed == 0) && (cm->cm_state != MPS_CM_STATE_FREE)) {
1117			/* this should never happen, but if it does, log */
1118			mpssas_log_command(cm, MPS_RECOVERY,
1119			    "cm %p state %x flags 0x%x ccb %p during diag "
1120			    "reset\n", cm, cm->cm_state, cm->cm_flags,
1121			    cm->cm_ccb);
1122		}
1123	}
1124}
1125
1126void
1127mpssas_handle_reinit(struct mps_softc *sc)
1128{
1129	int i;
1130
1131	/* Go back into startup mode and freeze the simq, so that CAM
1132	 * doesn't send any commands until after we've rediscovered all
1133	 * targets and found the proper device handles for them.
1134	 *
1135	 * After the reset, portenable will trigger discovery, and after all
1136	 * discovery-related activities have finished, the simq will be
1137	 * released.
1138	 */
1139	mps_dprint(sc, MPS_INIT, "%s startup\n", __func__);
1140	sc->sassc->flags |= MPSSAS_IN_STARTUP;
1141	sc->sassc->flags |= MPSSAS_IN_DISCOVERY;
1142	xpt_freeze_simq(sc->sassc->sim, 1);
1143
1144	/* notify CAM of a bus reset */
1145	mpssas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
1146	    CAM_LUN_WILDCARD);
1147
1148	/* complete and cleanup after all outstanding commands */
1149	mpssas_complete_all_commands(sc);
1150
1151	mps_dprint(sc, MPS_INIT,
1152	    "%s startup %u tm %u after command completion\n",
1153	    __func__, sc->sassc->startup_refcount, sc->sassc->tm_count);
1154
1155	/*
1156	 * The simq was explicitly frozen above, so set the refcount to 0.
1157	 * The simq will be explicitly released after port enable completes.
1158	 */
1159	sc->sassc->startup_refcount = 0;
1160
1161	/* zero all the target handles, since they may change after the
1162	 * reset, and we have to rediscover all the targets and use the new
1163	 * handles.
1164	 */
1165	for (i = 0; i < sc->facts->MaxTargets; i++) {
1166		if (sc->sassc->targets[i].outstanding != 0)
1167			mps_dprint(sc, MPS_INIT, "target %u outstanding %u\n",
1168			    i, sc->sassc->targets[i].outstanding);
1169		sc->sassc->targets[i].handle = 0x0;
1170		sc->sassc->targets[i].exp_dev_handle = 0x0;
1171		sc->sassc->targets[i].outstanding = 0;
1172		sc->sassc->targets[i].flags = MPSSAS_TARGET_INDIAGRESET;
1173	}
1174}
1175
1176static void
1177mpssas_tm_timeout(void *data)
1178{
1179	struct mps_command *tm = data;
1180	struct mps_softc *sc = tm->cm_sc;
1181
1182	mtx_assert(&sc->mps_mtx, MA_OWNED);
1183
1184	mpssas_log_command(tm, MPS_INFO|MPS_RECOVERY,
1185	    "task mgmt %p timed out\n", tm);
1186	mps_reinit(sc);
1187}
1188
1189static void
1190mpssas_logical_unit_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1191{
1192	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1193	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1194	unsigned int cm_count = 0;
1195	struct mps_command *cm;
1196	struct mpssas_target *targ;
1197
1198	callout_stop(&tm->cm_callout);
1199
1200	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1201	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1202	targ = tm->cm_targ;
1203
1204	/*
1205	 * Currently there should be no way we can hit this case.  It only
1206	 * happens when we have a failure to allocate chain frames, and
1207	 * task management commands don't have S/G lists.
1208	 * XXXSL So should it be an assertion?
1209	 */
1210	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1211		mps_dprint(sc, MPS_ERROR, "%s: cm_flags = %#x for LUN reset! "
1212			   "This should not happen!\n", __func__, tm->cm_flags);
1213		mpssas_free_tm(sc, tm);
1214		return;
1215	}
1216
1217	if (reply == NULL) {
1218		mpssas_log_command(tm, MPS_RECOVERY,
1219		    "NULL reset reply for tm %p\n", tm);
1220		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1221			/* this completion was due to a reset, just cleanup */
1222			targ->flags &= ~MPSSAS_TARGET_INRESET;
1223			targ->tm = NULL;
1224			mpssas_free_tm(sc, tm);
1225		}
1226		else {
1227			/* we should have gotten a reply. */
1228			mps_reinit(sc);
1229		}
1230		return;
1231	}
1232
1233	mpssas_log_command(tm, MPS_RECOVERY,
1234	    "logical unit reset status 0x%x code 0x%x count %u\n",
1235	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1236	    le32toh(reply->TerminationCount));
1237
1238	/* See if there are any outstanding commands for this LUN.
1239	 * This could be made more efficient by using a per-LU data
1240	 * structure of some sort.
1241	 */
1242	TAILQ_FOREACH(cm, &targ->commands, cm_link) {
1243		if (cm->cm_lun == tm->cm_lun)
1244			cm_count++;
1245	}
1246
1247	if (cm_count == 0) {
1248		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1249		    "logical unit %u finished recovery after reset\n",
1250		    tm->cm_lun, tm);
1251
1252		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1253		    tm->cm_lun);
1254
1255		/* we've finished recovery for this logical unit.  check and
1256		 * see if some other logical unit has a timedout command
1257		 * that needs to be processed.
1258		 */
1259		cm = TAILQ_FIRST(&targ->timedout_commands);
1260		if (cm) {
1261			mpssas_send_abort(sc, tm, cm);
1262		}
1263		else {
1264			targ->tm = NULL;
1265			mpssas_free_tm(sc, tm);
1266		}
1267	}
1268	else {
1269		/* if we still have commands for this LUN, the reset
1270		 * effectively failed, regardless of the status reported.
1271		 * Escalate to a target reset.
1272		 */
1273		mpssas_log_command(tm, MPS_RECOVERY,
1274		    "logical unit reset complete for tm %p, but still have %u command(s)\n",
1275		    tm, cm_count);
1276		mpssas_send_reset(sc, tm,
1277		    MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
1278	}
1279}
1280
1281static void
1282mpssas_target_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1283{
1284	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1285	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1286	struct mpssas_target *targ;
1287
1288	callout_stop(&tm->cm_callout);
1289
1290	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1291	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1292	targ = tm->cm_targ;
1293
1294	/*
1295	 * Currently there should be no way we can hit this case.  It only
1296	 * happens when we have a failure to allocate chain frames, and
1297	 * task management commands don't have S/G lists.
1298	 */
1299	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1300		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x for target reset! "
1301			   "This should not happen!\n", __func__, tm->cm_flags);
1302		mpssas_free_tm(sc, tm);
1303		return;
1304	}
1305
1306	if (reply == NULL) {
1307		mpssas_log_command(tm, MPS_RECOVERY,
1308		    "NULL reset reply for tm %p\n", tm);
1309		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1310			/* this completion was due to a reset, just cleanup */
1311			targ->flags &= ~MPSSAS_TARGET_INRESET;
1312			targ->tm = NULL;
1313			mpssas_free_tm(sc, tm);
1314		}
1315		else {
1316			/* we should have gotten a reply. */
1317			mps_reinit(sc);
1318		}
1319		return;
1320	}
1321
1322	mpssas_log_command(tm, MPS_RECOVERY,
1323	    "target reset status 0x%x code 0x%x count %u\n",
1324	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1325	    le32toh(reply->TerminationCount));
1326
1327	targ->flags &= ~MPSSAS_TARGET_INRESET;
1328
1329	if (targ->outstanding == 0) {
1330		/* we've finished recovery for this target and all
1331		 * of its logical units.
1332		 */
1333		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1334		    "recovery finished after target reset\n");
1335
1336		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1337		    CAM_LUN_WILDCARD);
1338
1339		targ->tm = NULL;
1340		mpssas_free_tm(sc, tm);
1341	}
1342	else {
1343		/* after a target reset, if this target still has
1344		 * outstanding commands, the reset effectively failed,
1345		 * regardless of the status reported.  escalate.
1346		 */
1347		mpssas_log_command(tm, MPS_RECOVERY,
1348		    "target reset complete for tm %p, but still have %u command(s)\n",
1349		    tm, targ->outstanding);
1350		mps_reinit(sc);
1351	}
1352}
1353
1354#define MPS_RESET_TIMEOUT 30
1355
1356static int
1357mpssas_send_reset(struct mps_softc *sc, struct mps_command *tm, uint8_t type)
1358{
1359	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1360	struct mpssas_target *target;
1361	int err;
1362
1363	target = tm->cm_targ;
1364	if (target->handle == 0) {
1365		mps_dprint(sc, MPS_ERROR,"%s null devhandle for target_id %d\n",
1366		    __func__, target->tid);
1367		return -1;
1368	}
1369
1370	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1371	req->DevHandle = htole16(target->handle);
1372	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1373	req->TaskType = type;
1374
1375	if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) {
1376		/* XXX Need to handle invalid LUNs */
1377		MPS_SET_LUN(req->LUN, tm->cm_lun);
1378		tm->cm_targ->logical_unit_resets++;
1379		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1380		    "sending logical unit reset\n");
1381		tm->cm_complete = mpssas_logical_unit_reset_complete;
1382	}
1383	else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) {
1384		/* Target reset method =  SAS Hard Link Reset / SATA Link Reset */
1385		req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
1386		tm->cm_targ->target_resets++;
1387		tm->cm_targ->flags |= MPSSAS_TARGET_INRESET;
1388		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1389		    "sending target reset\n");
1390		tm->cm_complete = mpssas_target_reset_complete;
1391	}
1392	else {
1393		mps_dprint(sc, MPS_ERROR, "unexpected reset type 0x%x\n", type);
1394		return -1;
1395	}
1396
1397	tm->cm_data = NULL;
1398	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1399	tm->cm_complete_data = (void *)tm;
1400
1401	callout_reset(&tm->cm_callout, MPS_RESET_TIMEOUT * hz,
1402	    mpssas_tm_timeout, tm);
1403
1404	err = mps_map_command(sc, tm);
1405	if (err)
1406		mpssas_log_command(tm, MPS_RECOVERY,
1407		    "error %d sending reset type %u\n",
1408		    err, type);
1409
1410	return err;
1411}
1412
1413
1414static void
1415mpssas_abort_complete(struct mps_softc *sc, struct mps_command *tm)
1416{
1417	struct mps_command *cm;
1418	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1419	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1420	struct mpssas_target *targ;
1421
1422	callout_stop(&tm->cm_callout);
1423
1424	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1425	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1426	targ = tm->cm_targ;
1427
1428	/*
1429	 * Currently there should be no way we can hit this case.  It only
1430	 * happens when we have a failure to allocate chain frames, and
1431	 * task management commands don't have S/G lists.
1432	 */
1433	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1434		mpssas_log_command(tm, MPS_RECOVERY,
1435		    "cm_flags = %#x for abort %p TaskMID %u!\n",
1436		    tm->cm_flags, tm, le16toh(req->TaskMID));
1437		mpssas_free_tm(sc, tm);
1438		return;
1439	}
1440
1441	if (reply == NULL) {
1442		mpssas_log_command(tm, MPS_RECOVERY,
1443		    "NULL abort reply for tm %p TaskMID %u\n",
1444		    tm, le16toh(req->TaskMID));
1445		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1446			/* this completion was due to a reset, just cleanup */
1447			targ->tm = NULL;
1448			mpssas_free_tm(sc, tm);
1449		}
1450		else {
1451			/* we should have gotten a reply. */
1452			mps_reinit(sc);
1453		}
1454		return;
1455	}
1456
1457	mpssas_log_command(tm, MPS_RECOVERY,
1458	    "abort TaskMID %u status 0x%x code 0x%x count %u\n",
1459	    le16toh(req->TaskMID),
1460	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1461	    le32toh(reply->TerminationCount));
1462
1463	cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands);
1464	if (cm == NULL) {
1465		/* if there are no more timedout commands, we're done with
1466		 * error recovery for this target.
1467		 */
1468		mpssas_log_command(tm, MPS_RECOVERY,
1469		    "finished recovery after aborting TaskMID %u\n",
1470		    le16toh(req->TaskMID));
1471
1472		targ->tm = NULL;
1473		mpssas_free_tm(sc, tm);
1474	}
1475	else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) {
1476		/* abort success, but we have more timedout commands to abort */
1477		mpssas_log_command(tm, MPS_RECOVERY,
1478		    "continuing recovery after aborting TaskMID %u\n",
1479		    le16toh(req->TaskMID));
1480
1481		mpssas_send_abort(sc, tm, cm);
1482	}
1483	else {
1484		/* we didn't get a command completion, so the abort
1485		 * failed as far as we're concerned.  escalate.
1486		 */
1487		mpssas_log_command(tm, MPS_RECOVERY,
1488		    "abort failed for TaskMID %u tm %p\n",
1489		    le16toh(req->TaskMID), tm);
1490
1491		mpssas_send_reset(sc, tm,
1492		    MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET);
1493	}
1494}
1495
1496#define MPS_ABORT_TIMEOUT 5
1497
1498static int
1499mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm, struct mps_command *cm)
1500{
1501	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1502	struct mpssas_target *targ;
1503	int err;
1504
1505	targ = cm->cm_targ;
1506	if (targ->handle == 0) {
1507		mps_dprint(sc, MPS_ERROR,"%s null devhandle for target_id %d\n",
1508		    __func__, cm->cm_ccb->ccb_h.target_id);
1509		return -1;
1510	}
1511
1512	mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1513	    "Aborting command %p\n", cm);
1514
1515	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1516	req->DevHandle = htole16(targ->handle);
1517	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1518	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK;
1519
1520	/* XXX Need to handle invalid LUNs */
1521	MPS_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun);
1522
1523	req->TaskMID = htole16(cm->cm_desc.Default.SMID);
1524
1525	tm->cm_data = NULL;
1526	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1527	tm->cm_complete = mpssas_abort_complete;
1528	tm->cm_complete_data = (void *)tm;
1529	tm->cm_targ = cm->cm_targ;
1530	tm->cm_lun = cm->cm_lun;
1531
1532	callout_reset(&tm->cm_callout, MPS_ABORT_TIMEOUT * hz,
1533	    mpssas_tm_timeout, tm);
1534
1535	targ->aborts++;
1536
1537	err = mps_map_command(sc, tm);
1538	if (err)
1539		mpssas_log_command(tm, MPS_RECOVERY,
1540		    "error %d sending abort for cm %p SMID %u\n",
1541		    err, cm, req->TaskMID);
1542	return err;
1543}
1544
1545
1546static void
1547mpssas_scsiio_timeout(void *data)
1548{
1549	struct mps_softc *sc;
1550	struct mps_command *cm;
1551	struct mpssas_target *targ;
1552
1553	cm = (struct mps_command *)data;
1554	sc = cm->cm_sc;
1555
1556	MPS_FUNCTRACE(sc);
1557	mtx_assert(&sc->mps_mtx, MA_OWNED);
1558
1559	mps_dprint(sc, MPS_XINFO, "Timeout checking cm %p\n", sc);
1560
1561	/*
1562	 * Run the interrupt handler to make sure it's not pending.  This
1563	 * isn't perfect because the command could have already completed
1564	 * and been re-used, though this is unlikely.
1565	 */
1566	mps_intr_locked(sc);
1567	if (cm->cm_state == MPS_CM_STATE_FREE) {
1568		mpssas_log_command(cm, MPS_XINFO,
1569		    "SCSI command %p almost timed out\n", cm);
1570		return;
1571	}
1572
1573	if (cm->cm_ccb == NULL) {
1574		mps_dprint(sc, MPS_ERROR, "command timeout with NULL ccb\n");
1575		return;
1576	}
1577
1578	mpssas_log_command(cm, MPS_INFO, "command timeout cm %p ccb %p\n",
1579	    cm, cm->cm_ccb);
1580
1581	targ = cm->cm_targ;
1582	targ->timeouts++;
1583
1584	/* XXX first, check the firmware state, to see if it's still
1585	 * operational.  if not, do a diag reset.
1586	 */
1587
1588	cm->cm_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
1589	cm->cm_state = MPS_CM_STATE_TIMEDOUT;
1590	TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery);
1591
1592	if (targ->tm != NULL) {
1593		/* target already in recovery, just queue up another
1594		 * timedout command to be processed later.
1595		 */
1596		mps_dprint(sc, MPS_RECOVERY,
1597		    "queued timedout cm %p for processing by tm %p\n",
1598		    cm, targ->tm);
1599	}
1600	else if ((targ->tm = mpssas_alloc_tm(sc)) != NULL) {
1601		mps_dprint(sc, MPS_RECOVERY, "timedout cm %p allocated tm %p\n",
1602		    cm, targ->tm);
1603
1604		/* start recovery by aborting the first timedout command */
1605		mpssas_send_abort(sc, targ->tm, cm);
1606	}
1607	else {
1608		/* XXX queue this target up for recovery once a TM becomes
1609		 * available.  The firmware only has a limited number of
1610		 * HighPriority credits for the high priority requests used
1611		 * for task management, and we ran out.
1612		 *
1613		 * Isilon: don't worry about this for now, since we have
1614		 * more credits than disks in an enclosure, and limit
1615		 * ourselves to one TM per target for recovery.
1616		 */
1617		mps_dprint(sc, MPS_RECOVERY,
1618		    "timedout cm %p failed to allocate a tm\n", cm);
1619	}
1620
1621}
1622
1623static void
1624mpssas_action_scsiio(struct mpssas_softc *sassc, union ccb *ccb)
1625{
1626	MPI2_SCSI_IO_REQUEST *req;
1627	struct ccb_scsiio *csio;
1628	struct mps_softc *sc;
1629	struct mpssas_target *targ;
1630	struct mpssas_lun *lun;
1631	struct mps_command *cm;
1632	uint8_t i, lba_byte, *ref_tag_addr;
1633	uint16_t eedp_flags;
1634	uint32_t mpi_control;
1635
1636	sc = sassc->sc;
1637	MPS_FUNCTRACE(sc);
1638	mtx_assert(&sc->mps_mtx, MA_OWNED);
1639
1640	csio = &ccb->csio;
1641	targ = &sassc->targets[csio->ccb_h.target_id];
1642	mps_dprint(sc, MPS_TRACE, "ccb %p target flag %x\n", ccb, targ->flags);
1643	if (targ->handle == 0x0) {
1644		mps_dprint(sc, MPS_ERROR, "%s NULL handle for target %u\n",
1645		    __func__, csio->ccb_h.target_id);
1646		csio->ccb_h.status = CAM_SEL_TIMEOUT;
1647		xpt_done(ccb);
1648		return;
1649	}
1650	if (targ->flags & MPS_TARGET_FLAGS_RAID_COMPONENT) {
1651		mps_dprint(sc, MPS_ERROR, "%s Raid component no SCSI IO "
1652		    "supported %u\n", __func__, csio->ccb_h.target_id);
1653		csio->ccb_h.status = CAM_TID_INVALID;
1654		xpt_done(ccb);
1655		return;
1656	}
1657	/*
1658	 * Sometimes, it is possible to get a command that is not "In
1659	 * Progress" and was actually aborted by the upper layer.  Check for
1660	 * this here and complete the command without error.
1661	 */
1662	if (ccb->ccb_h.status != CAM_REQ_INPROG) {
1663		mps_dprint(sc, MPS_TRACE, "%s Command is not in progress for "
1664		    "target %u\n", __func__, csio->ccb_h.target_id);
1665		xpt_done(ccb);
1666		return;
1667	}
1668	/*
1669	 * If devinfo is 0 this will be a volume.  In that case don't tell CAM
1670	 * that the volume has timed out.  We want volumes to be enumerated
1671	 * until they are deleted/removed, not just failed.
1672	 */
1673	if (targ->flags & MPSSAS_TARGET_INREMOVAL) {
1674		if (targ->devinfo == 0)
1675			csio->ccb_h.status = CAM_REQ_CMP;
1676		else
1677			csio->ccb_h.status = CAM_SEL_TIMEOUT;
1678		xpt_done(ccb);
1679		return;
1680	}
1681
1682	if ((sc->mps_flags & MPS_FLAGS_SHUTDOWN) != 0) {
1683		mps_dprint(sc, MPS_INFO, "%s shutting down\n", __func__);
1684		csio->ccb_h.status = CAM_TID_INVALID;
1685		xpt_done(ccb);
1686		return;
1687	}
1688
1689	cm = mps_alloc_command(sc);
1690	if (cm == NULL) {
1691		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
1692			xpt_freeze_simq(sassc->sim, 1);
1693			sassc->flags |= MPSSAS_QUEUE_FROZEN;
1694		}
1695		ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1696		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1697		xpt_done(ccb);
1698		return;
1699	}
1700
1701	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
1702	bzero(req, sizeof(*req));
1703	req->DevHandle = htole16(targ->handle);
1704	req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
1705	req->MsgFlags = 0;
1706	req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);
1707	req->SenseBufferLength = MPS_SENSE_LEN;
1708	req->SGLFlags = 0;
1709	req->ChainOffset = 0;
1710	req->SGLOffset0 = 24;	/* 32bit word offset to the SGL */
1711	req->SGLOffset1= 0;
1712	req->SGLOffset2= 0;
1713	req->SGLOffset3= 0;
1714	req->SkipCount = 0;
1715	req->DataLength = htole32(csio->dxfer_len);
1716	req->BidirectionalDataLength = 0;
1717	req->IoFlags = htole16(csio->cdb_len);
1718	req->EEDPFlags = 0;
1719
1720	/* Note: BiDirectional transfers are not supported */
1721	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1722	case CAM_DIR_IN:
1723		mpi_control = MPI2_SCSIIO_CONTROL_READ;
1724		cm->cm_flags |= MPS_CM_FLAGS_DATAIN;
1725		break;
1726	case CAM_DIR_OUT:
1727		mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
1728		cm->cm_flags |= MPS_CM_FLAGS_DATAOUT;
1729		break;
1730	case CAM_DIR_NONE:
1731	default:
1732		mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
1733		break;
1734	}
1735
1736	if (csio->cdb_len == 32)
1737                mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
1738	/*
1739	 * It looks like the hardware doesn't require an explicit tag
1740	 * number for each transaction.  SAM Task Management not supported
1741	 * at the moment.
1742	 */
1743	switch (csio->tag_action) {
1744	case MSG_HEAD_OF_Q_TAG:
1745		mpi_control |= MPI2_SCSIIO_CONTROL_HEADOFQ;
1746		break;
1747	case MSG_ORDERED_Q_TAG:
1748		mpi_control |= MPI2_SCSIIO_CONTROL_ORDEREDQ;
1749		break;
1750	case MSG_ACA_TASK:
1751		mpi_control |= MPI2_SCSIIO_CONTROL_ACAQ;
1752		break;
1753	case CAM_TAG_ACTION_NONE:
1754	case MSG_SIMPLE_Q_TAG:
1755	default:
1756		mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
1757		break;
1758	}
1759	mpi_control |= sc->mapping_table[csio->ccb_h.target_id].TLR_bits;
1760	req->Control = htole32(mpi_control);
1761	if (MPS_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) {
1762		mps_free_command(sc, cm);
1763		ccb->ccb_h.status = CAM_LUN_INVALID;
1764		xpt_done(ccb);
1765		return;
1766	}
1767
1768	if (csio->ccb_h.flags & CAM_CDB_POINTER)
1769		bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
1770	else
1771		bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
1772	req->IoFlags = htole16(csio->cdb_len);
1773
1774	/*
1775	 * Check if EEDP is supported and enabled.  If it is then check if the
1776	 * SCSI opcode could be using EEDP.  If so, make sure the LUN exists and
1777	 * is formatted for EEDP support.  If all of this is true, set CDB up
1778	 * for EEDP transfer.
1779	 */
1780	eedp_flags = op_code_prot[req->CDB.CDB32[0]];
1781	if (sc->eedp_enabled && eedp_flags) {
1782		SLIST_FOREACH(lun, &targ->luns, lun_link) {
1783			if (lun->lun_id == csio->ccb_h.target_lun) {
1784				break;
1785			}
1786		}
1787
1788		if ((lun != NULL) && (lun->eedp_formatted)) {
1789			req->EEDPBlockSize = htole16(lun->eedp_block_size);
1790			eedp_flags |= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
1791			    MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
1792			    MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
1793			req->EEDPFlags = htole16(eedp_flags);
1794
1795			/*
1796			 * If CDB less than 32, fill in Primary Ref Tag with
1797			 * low 4 bytes of LBA.  If CDB is 32, tag stuff is
1798			 * already there.  Also, set protection bit.  FreeBSD
1799			 * currently does not support CDBs bigger than 16, but
1800			 * the code doesn't hurt, and will be here for the
1801			 * future.
1802			 */
1803			if (csio->cdb_len != 32) {
1804				lba_byte = (csio->cdb_len == 16) ? 6 : 2;
1805				ref_tag_addr = (uint8_t *)&req->CDB.EEDP32.
1806				    PrimaryReferenceTag;
1807				for (i = 0; i < 4; i++) {
1808					*ref_tag_addr =
1809					    req->CDB.CDB32[lba_byte + i];
1810					ref_tag_addr++;
1811				}
1812				req->CDB.EEDP32.PrimaryReferenceTag =
1813					htole32(req->CDB.EEDP32.PrimaryReferenceTag);
1814				req->CDB.EEDP32.PrimaryApplicationTagMask =
1815				    0xFFFF;
1816				req->CDB.CDB32[1] = (req->CDB.CDB32[1] & 0x1F) |
1817				    0x20;
1818			} else {
1819				eedp_flags |=
1820				    MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG;
1821				req->EEDPFlags = htole16(eedp_flags);
1822				req->CDB.CDB32[10] = (req->CDB.CDB32[10] &
1823				    0x1F) | 0x20;
1824			}
1825		}
1826	}
1827
1828	cm->cm_length = csio->dxfer_len;
1829	if (cm->cm_length != 0) {
1830		cm->cm_data = ccb;
1831		cm->cm_flags |= MPS_CM_FLAGS_USE_CCB;
1832	} else {
1833		cm->cm_data = NULL;
1834	}
1835	cm->cm_sge = &req->SGL;
1836	cm->cm_sglsize = (32 - 24) * 4;
1837	cm->cm_desc.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
1838	cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
1839	cm->cm_complete = mpssas_scsiio_complete;
1840	cm->cm_complete_data = ccb;
1841	cm->cm_targ = targ;
1842	cm->cm_lun = csio->ccb_h.target_lun;
1843	cm->cm_ccb = ccb;
1844
1845	/*
1846	 * If HBA is a WD and the command is not for a retry, try to build a
1847	 * direct I/O message. If failed, or the command is for a retry, send
1848	 * the I/O to the IR volume itself.
1849	 */
1850	if (sc->WD_valid_config) {
1851		if (ccb->ccb_h.status != MPS_WD_RETRY) {
1852			mpssas_direct_drive_io(sassc, cm, ccb);
1853		} else {
1854			ccb->ccb_h.status = CAM_REQ_INPROG;
1855		}
1856	}
1857
1858	callout_reset(&cm->cm_callout, (ccb->ccb_h.timeout * hz) / 1000,
1859	   mpssas_scsiio_timeout, cm);
1860
1861	targ->issued++;
1862	targ->outstanding++;
1863	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
1864	ccb->ccb_h.status |= CAM_SIM_QUEUED;
1865
1866	mpssas_log_command(cm, MPS_XINFO, "%s cm %p ccb %p outstanding %u\n",
1867	    __func__, cm, ccb, targ->outstanding);
1868
1869	mps_map_command(sc, cm);
1870	return;
1871}
1872
1873static void
1874mps_response_code(struct mps_softc *sc, u8 response_code)
1875{
1876        char *desc;
1877
1878        switch (response_code) {
1879        case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
1880                desc = "task management request completed";
1881                break;
1882        case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
1883                desc = "invalid frame";
1884                break;
1885        case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
1886                desc = "task management request not supported";
1887                break;
1888        case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
1889                desc = "task management request failed";
1890                break;
1891        case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
1892                desc = "task management request succeeded";
1893                break;
1894        case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
1895                desc = "invalid lun";
1896                break;
1897        case 0xA:
1898                desc = "overlapped tag attempted";
1899                break;
1900        case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
1901                desc = "task queued, however not sent to target";
1902                break;
1903        default:
1904                desc = "unknown";
1905                break;
1906        }
1907		mps_dprint(sc, MPS_XINFO, "response_code(0x%01x): %s\n",
1908                response_code, desc);
1909}
1910/**
1911 * mps_sc_failed_io_info - translated non-succesfull SCSI_IO request
1912 */
1913static void
1914mps_sc_failed_io_info(struct mps_softc *sc, struct ccb_scsiio *csio,
1915    Mpi2SCSIIOReply_t *mpi_reply)
1916{
1917	u32 response_info;
1918	u8 *response_bytes;
1919	u16 ioc_status = le16toh(mpi_reply->IOCStatus) &
1920	    MPI2_IOCSTATUS_MASK;
1921	u8 scsi_state = mpi_reply->SCSIState;
1922	u8 scsi_status = mpi_reply->SCSIStatus;
1923	char *desc_ioc_state = NULL;
1924	char *desc_scsi_status = NULL;
1925	char *desc_scsi_state = sc->tmp_string;
1926	u32 log_info = le32toh(mpi_reply->IOCLogInfo);
1927
1928	if (log_info == 0x31170000)
1929		return;
1930
1931	switch (ioc_status) {
1932	case MPI2_IOCSTATUS_SUCCESS:
1933		desc_ioc_state = "success";
1934		break;
1935	case MPI2_IOCSTATUS_INVALID_FUNCTION:
1936		desc_ioc_state = "invalid function";
1937		break;
1938	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
1939		desc_ioc_state = "scsi recovered error";
1940		break;
1941	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
1942		desc_ioc_state = "scsi invalid dev handle";
1943		break;
1944	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
1945		desc_ioc_state = "scsi device not there";
1946		break;
1947	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
1948		desc_ioc_state = "scsi data overrun";
1949		break;
1950	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
1951		desc_ioc_state = "scsi data underrun";
1952		break;
1953	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
1954		desc_ioc_state = "scsi io data error";
1955		break;
1956	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
1957		desc_ioc_state = "scsi protocol error";
1958		break;
1959	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
1960		desc_ioc_state = "scsi task terminated";
1961		break;
1962	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
1963		desc_ioc_state = "scsi residual mismatch";
1964		break;
1965	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
1966		desc_ioc_state = "scsi task mgmt failed";
1967		break;
1968	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
1969		desc_ioc_state = "scsi ioc terminated";
1970		break;
1971	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
1972		desc_ioc_state = "scsi ext terminated";
1973		break;
1974	case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
1975		desc_ioc_state = "eedp guard error";
1976		break;
1977	case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
1978		desc_ioc_state = "eedp ref tag error";
1979		break;
1980	case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
1981		desc_ioc_state = "eedp app tag error";
1982		break;
1983	default:
1984		desc_ioc_state = "unknown";
1985		break;
1986	}
1987
1988	switch (scsi_status) {
1989	case MPI2_SCSI_STATUS_GOOD:
1990		desc_scsi_status = "good";
1991		break;
1992	case MPI2_SCSI_STATUS_CHECK_CONDITION:
1993		desc_scsi_status = "check condition";
1994		break;
1995	case MPI2_SCSI_STATUS_CONDITION_MET:
1996		desc_scsi_status = "condition met";
1997		break;
1998	case MPI2_SCSI_STATUS_BUSY:
1999		desc_scsi_status = "busy";
2000		break;
2001	case MPI2_SCSI_STATUS_INTERMEDIATE:
2002		desc_scsi_status = "intermediate";
2003		break;
2004	case MPI2_SCSI_STATUS_INTERMEDIATE_CONDMET:
2005		desc_scsi_status = "intermediate condmet";
2006		break;
2007	case MPI2_SCSI_STATUS_RESERVATION_CONFLICT:
2008		desc_scsi_status = "reservation conflict";
2009		break;
2010	case MPI2_SCSI_STATUS_COMMAND_TERMINATED:
2011		desc_scsi_status = "command terminated";
2012		break;
2013	case MPI2_SCSI_STATUS_TASK_SET_FULL:
2014		desc_scsi_status = "task set full";
2015		break;
2016	case MPI2_SCSI_STATUS_ACA_ACTIVE:
2017		desc_scsi_status = "aca active";
2018		break;
2019	case MPI2_SCSI_STATUS_TASK_ABORTED:
2020		desc_scsi_status = "task aborted";
2021		break;
2022	default:
2023		desc_scsi_status = "unknown";
2024		break;
2025	}
2026
2027	desc_scsi_state[0] = '\0';
2028	if (!scsi_state)
2029		desc_scsi_state = " ";
2030	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID)
2031		strcat(desc_scsi_state, "response info ");
2032	if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
2033		strcat(desc_scsi_state, "state terminated ");
2034	if (scsi_state & MPI2_SCSI_STATE_NO_SCSI_STATUS)
2035		strcat(desc_scsi_state, "no status ");
2036	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_FAILED)
2037		strcat(desc_scsi_state, "autosense failed ");
2038	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID)
2039		strcat(desc_scsi_state, "autosense valid ");
2040
2041	mps_dprint(sc, MPS_XINFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x)\n",
2042	    le16toh(mpi_reply->DevHandle), desc_ioc_state, ioc_status);
2043	/* We can add more detail about underflow data here
2044	 * TO-DO
2045	 * */
2046	mps_dprint(sc, MPS_XINFO, "\tscsi_status(%s)(0x%02x), "
2047	    "scsi_state(%s)(0x%02x)\n", desc_scsi_status, scsi_status,
2048	    desc_scsi_state, scsi_state);
2049
2050	if (sc->mps_debug & MPS_XINFO &&
2051		scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2052		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : Start :\n");
2053		scsi_sense_print(csio);
2054		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : End :\n");
2055	}
2056
2057	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
2058		response_info = le32toh(mpi_reply->ResponseInfo);
2059		response_bytes = (u8 *)&response_info;
2060		mps_response_code(sc,response_bytes[0]);
2061	}
2062}
2063
2064static void
2065mpssas_scsiio_complete(struct mps_softc *sc, struct mps_command *cm)
2066{
2067	MPI2_SCSI_IO_REPLY *rep;
2068	union ccb *ccb;
2069	struct ccb_scsiio *csio;
2070	struct mpssas_softc *sassc;
2071	struct scsi_vpd_supported_page_list *vpd_list = NULL;
2072	u8 *TLR_bits, TLR_on;
2073	int dir = 0, i;
2074	u16 alloc_len;
2075
2076	MPS_FUNCTRACE(sc);
2077	mps_dprint(sc, MPS_TRACE,
2078	    "cm %p SMID %u ccb %p reply %p outstanding %u\n", cm,
2079	    cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply,
2080	    cm->cm_targ->outstanding);
2081
2082	callout_stop(&cm->cm_callout);
2083	mtx_assert(&sc->mps_mtx, MA_OWNED);
2084
2085	sassc = sc->sassc;
2086	ccb = cm->cm_complete_data;
2087	csio = &ccb->csio;
2088	rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply;
2089	/*
2090	 * XXX KDM if the chain allocation fails, does it matter if we do
2091	 * the sync and unload here?  It is simpler to do it in every case,
2092	 * assuming it doesn't cause problems.
2093	 */
2094	if (cm->cm_data != NULL) {
2095		if (cm->cm_flags & MPS_CM_FLAGS_DATAIN)
2096			dir = BUS_DMASYNC_POSTREAD;
2097		else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT)
2098			dir = BUS_DMASYNC_POSTWRITE;
2099		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
2100		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2101	}
2102
2103	cm->cm_targ->completed++;
2104	cm->cm_targ->outstanding--;
2105	TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link);
2106	ccb->ccb_h.status |= ~(CAM_STATUS_MASK | CAM_SIM_QUEUED);
2107
2108	if (cm->cm_state == MPS_CM_STATE_TIMEDOUT) {
2109		TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery);
2110		if (cm->cm_reply != NULL)
2111			mpssas_log_command(cm, MPS_RECOVERY,
2112			    "completed timedout cm %p ccb %p during recovery "
2113			    "ioc %x scsi %x state %x xfer %u\n",
2114			    cm, cm->cm_ccb,
2115			    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2116			    le32toh(rep->TransferCount));
2117		else
2118			mpssas_log_command(cm, MPS_RECOVERY,
2119			    "completed timedout cm %p ccb %p during recovery\n",
2120			    cm, cm->cm_ccb);
2121	} else if (cm->cm_targ->tm != NULL) {
2122		if (cm->cm_reply != NULL)
2123			mpssas_log_command(cm, MPS_RECOVERY,
2124			    "completed cm %p ccb %p during recovery "
2125			    "ioc %x scsi %x state %x xfer %u\n",
2126			    cm, cm->cm_ccb,
2127			    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2128			    le32toh(rep->TransferCount));
2129		else
2130			mpssas_log_command(cm, MPS_RECOVERY,
2131			    "completed cm %p ccb %p during recovery\n",
2132			    cm, cm->cm_ccb);
2133	} else if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
2134		mpssas_log_command(cm, MPS_RECOVERY,
2135		    "reset completed cm %p ccb %p\n",
2136		    cm, cm->cm_ccb);
2137	}
2138
2139	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2140		/*
2141		 * We ran into an error after we tried to map the command,
2142		 * so we're getting a callback without queueing the command
2143		 * to the hardware.  So we set the status here, and it will
2144		 * be retained below.  We'll go through the "fast path",
2145		 * because there can be no reply when we haven't actually
2146		 * gone out to the hardware.
2147		 */
2148		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
2149
2150		/*
2151		 * Currently the only error included in the mask is
2152		 * MPS_CM_FLAGS_CHAIN_FAILED, which means we're out of
2153		 * chain frames.  We need to freeze the queue until we get
2154		 * a command that completed without this error, which will
2155		 * hopefully have some chain frames attached that we can
2156		 * use.  If we wanted to get smarter about it, we would
2157		 * only unfreeze the queue in this condition when we're
2158		 * sure that we're getting some chain frames back.  That's
2159		 * probably unnecessary.
2160		 */
2161		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
2162			xpt_freeze_simq(sassc->sim, 1);
2163			sassc->flags |= MPSSAS_QUEUE_FROZEN;
2164			mps_dprint(sc, MPS_XINFO, "Error sending command, "
2165				   "freezing SIM queue\n");
2166		}
2167	}
2168
2169	/* Take the fast path to completion */
2170	if (cm->cm_reply == NULL) {
2171		if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INPROG) {
2172			if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0)
2173				ccb->ccb_h.status = CAM_SCSI_BUS_RESET;
2174			else {
2175				ccb->ccb_h.status = CAM_REQ_CMP;
2176				ccb->csio.scsi_status = SCSI_STATUS_OK;
2177			}
2178			if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2179				ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2180				sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2181				mps_dprint(sc, MPS_XINFO,
2182				    "Unfreezing SIM queue\n");
2183			}
2184		}
2185
2186		/*
2187		 * There are two scenarios where the status won't be
2188		 * CAM_REQ_CMP.  The first is if MPS_CM_FLAGS_ERROR_MASK is
2189		 * set, the second is in the MPS_FLAGS_DIAGRESET above.
2190		 */
2191		if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2192			/*
2193			 * Freeze the dev queue so that commands are
2194			 * executed in the correct order with after error
2195			 * recovery.
2196			 */
2197			ccb->ccb_h.status |= CAM_DEV_QFRZN;
2198			xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2199		}
2200		mps_free_command(sc, cm);
2201		xpt_done(ccb);
2202		return;
2203	}
2204
2205	mpssas_log_command(cm, MPS_XINFO,
2206	    "ioc %x scsi %x state %x xfer %u\n",
2207	    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2208	    le32toh(rep->TransferCount));
2209
2210	/*
2211	 * If this is a Direct Drive I/O, reissue the I/O to the original IR
2212	 * Volume if an error occurred (normal I/O retry).  Use the original
2213	 * CCB, but set a flag that this will be a retry so that it's sent to
2214	 * the original volume.  Free the command but reuse the CCB.
2215	 */
2216	if (cm->cm_flags & MPS_CM_FLAGS_DD_IO) {
2217		mps_free_command(sc, cm);
2218		ccb->ccb_h.status = MPS_WD_RETRY;
2219		mpssas_action_scsiio(sassc, ccb);
2220		return;
2221	}
2222
2223	switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) {
2224	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
2225		csio->resid = cm->cm_length - le32toh(rep->TransferCount);
2226		/* FALLTHROUGH */
2227	case MPI2_IOCSTATUS_SUCCESS:
2228	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
2229
2230		if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
2231		    MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
2232			mpssas_log_command(cm, MPS_XINFO, "recovered error\n");
2233
2234		/* Completion failed at the transport level. */
2235		if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS |
2236		    MPI2_SCSI_STATE_TERMINATED)) {
2237			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2238			break;
2239		}
2240
2241		/* In a modern packetized environment, an autosense failure
2242		 * implies that there's not much else that can be done to
2243		 * recover the command.
2244		 */
2245		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) {
2246			ccb->ccb_h.status = CAM_AUTOSENSE_FAIL;
2247			break;
2248		}
2249
2250		/*
2251		 * CAM doesn't care about SAS Response Info data, but if this is
2252		 * the state check if TLR should be done.  If not, clear the
2253		 * TLR_bits for the target.
2254		 */
2255		if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) &&
2256		    ((le32toh(rep->ResponseInfo) & MPI2_SCSI_RI_MASK_REASONCODE) ==
2257		    MPS_SCSI_RI_INVALID_FRAME)) {
2258			sc->mapping_table[csio->ccb_h.target_id].TLR_bits =
2259			    (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2260		}
2261
2262		/*
2263		 * Intentionally override the normal SCSI status reporting
2264		 * for these two cases.  These are likely to happen in a
2265		 * multi-initiator environment, and we want to make sure that
2266		 * CAM retries these commands rather than fail them.
2267		 */
2268		if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) ||
2269		    (rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) {
2270			ccb->ccb_h.status = CAM_REQ_ABORTED;
2271			break;
2272		}
2273
2274		/* Handle normal status and sense */
2275		csio->scsi_status = rep->SCSIStatus;
2276		if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD)
2277			ccb->ccb_h.status = CAM_REQ_CMP;
2278		else
2279			ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
2280
2281		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2282			int sense_len, returned_sense_len;
2283
2284			returned_sense_len = min(le32toh(rep->SenseCount),
2285			    sizeof(struct scsi_sense_data));
2286			if (returned_sense_len < ccb->csio.sense_len)
2287				ccb->csio.sense_resid = ccb->csio.sense_len -
2288					returned_sense_len;
2289			else
2290				ccb->csio.sense_resid = 0;
2291
2292			sense_len = min(returned_sense_len,
2293			    ccb->csio.sense_len - ccb->csio.sense_resid);
2294			bzero(&ccb->csio.sense_data,
2295			      sizeof(ccb->csio.sense_data));
2296			bcopy(cm->cm_sense, &ccb->csio.sense_data, sense_len);
2297			ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2298		}
2299
2300		/*
2301		 * Check if this is an INQUIRY command.  If it's a VPD inquiry,
2302		 * and it's page code 0 (Supported Page List), and there is
2303		 * inquiry data, and this is for a sequential access device, and
2304		 * the device is an SSP target, and TLR is supported by the
2305		 * controller, turn the TLR_bits value ON if page 0x90 is
2306		 * supported.
2307		 */
2308		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2309		    (csio->cdb_io.cdb_bytes[1] & SI_EVPD) &&
2310		    (csio->cdb_io.cdb_bytes[2] == SVPD_SUPPORTED_PAGE_LIST) &&
2311		    ((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) &&
2312		    (csio->data_ptr != NULL) && (((uint8_t *)cm->cm_data)[0] ==
2313		    T_SEQUENTIAL) && (sc->control_TLR) &&
2314		    (sc->mapping_table[csio->ccb_h.target_id].device_info &
2315		    MPI2_SAS_DEVICE_INFO_SSP_TARGET)) {
2316			vpd_list = (struct scsi_vpd_supported_page_list *)
2317			    csio->data_ptr;
2318			TLR_bits = &sc->mapping_table[csio->ccb_h.target_id].
2319			    TLR_bits;
2320			*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2321			TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
2322			alloc_len = ((u16)csio->cdb_io.cdb_bytes[3] << 8) +
2323			    csio->cdb_io.cdb_bytes[4];
2324			for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
2325				if (vpd_list->list[i] == 0x90) {
2326					*TLR_bits = TLR_on;
2327					break;
2328				}
2329			}
2330		}
2331		break;
2332	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
2333	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
2334		/*
2335		 * If devinfo is 0 this will be a volume.  In that case don't
2336		 * tell CAM that the volume is not there.  We want volumes to
2337		 * be enumerated until they are deleted/removed, not just
2338		 * failed.
2339		 */
2340		if (cm->cm_targ->devinfo == 0)
2341			ccb->ccb_h.status = CAM_REQ_CMP;
2342		else
2343			ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2344		break;
2345	case MPI2_IOCSTATUS_INVALID_SGL:
2346		mps_print_scsiio_cmd(sc, cm);
2347		ccb->ccb_h.status = CAM_UNREC_HBA_ERROR;
2348		break;
2349	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
2350		/*
2351		 * This is one of the responses that comes back when an I/O
2352		 * has been aborted.  If it is because of a timeout that we
2353		 * initiated, just set the status to CAM_CMD_TIMEOUT.
2354		 * Otherwise set it to CAM_REQ_ABORTED.  The effect on the
2355		 * command is the same (it gets retried, subject to the
2356		 * retry counter), the only difference is what gets printed
2357		 * on the console.
2358		 */
2359		if (cm->cm_state == MPS_CM_STATE_TIMEDOUT)
2360			ccb->ccb_h.status = CAM_CMD_TIMEOUT;
2361		else
2362			ccb->ccb_h.status = CAM_REQ_ABORTED;
2363		break;
2364	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
2365		/* resid is ignored for this condition */
2366		csio->resid = 0;
2367		ccb->ccb_h.status = CAM_DATA_RUN_ERR;
2368		break;
2369	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
2370	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
2371		/*
2372		 * Since these are generally external (i.e. hopefully
2373		 * transient transport-related) errors, retry these without
2374		 * decrementing the retry count.
2375		 */
2376		ccb->ccb_h.status = CAM_REQUEUE_REQ;
2377		mpssas_log_command(cm, MPS_INFO,
2378		    "terminated ioc %x scsi %x state %x xfer %u\n",
2379		    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2380		    le32toh(rep->TransferCount));
2381		break;
2382	case MPI2_IOCSTATUS_INVALID_FUNCTION:
2383	case MPI2_IOCSTATUS_INTERNAL_ERROR:
2384	case MPI2_IOCSTATUS_INVALID_VPID:
2385	case MPI2_IOCSTATUS_INVALID_FIELD:
2386	case MPI2_IOCSTATUS_INVALID_STATE:
2387	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
2388	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
2389	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
2390	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
2391	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
2392	default:
2393		mpssas_log_command(cm, MPS_XINFO,
2394		    "completed ioc %x scsi %x state %x xfer %u\n",
2395		    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2396		    le32toh(rep->TransferCount));
2397		csio->resid = cm->cm_length;
2398		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2399		break;
2400	}
2401
2402	mps_sc_failed_io_info(sc,csio,rep);
2403
2404	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2405		ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2406		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2407		mps_dprint(sc, MPS_XINFO, "Command completed, "
2408		    "unfreezing SIM queue\n");
2409	}
2410
2411	if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2412		ccb->ccb_h.status |= CAM_DEV_QFRZN;
2413		xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2414	}
2415
2416	mps_free_command(sc, cm);
2417	xpt_done(ccb);
2418}
2419
2420/* All Request reached here are Endian safe */
2421static void
2422mpssas_direct_drive_io(struct mpssas_softc *sassc, struct mps_command *cm,
2423    union ccb *ccb) {
2424	pMpi2SCSIIORequest_t	pIO_req;
2425	struct mps_softc	*sc = sassc->sc;
2426	uint64_t		virtLBA;
2427	uint32_t		physLBA, stripe_offset, stripe_unit;
2428	uint32_t		io_size, column;
2429	uint8_t			*ptrLBA, lba_idx, physLBA_byte, *CDB;
2430
2431	/*
2432	 * If this is a valid SCSI command (Read6, Read10, Read16, Write6,
2433	 * Write10, or Write16), build a direct I/O message.  Otherwise, the I/O
2434	 * will be sent to the IR volume itself.  Since Read6 and Write6 are a
2435	 * bit different than the 10/16 CDBs, handle them separately.
2436	 */
2437	pIO_req = (pMpi2SCSIIORequest_t)cm->cm_req;
2438	CDB = pIO_req->CDB.CDB32;
2439
2440	/*
2441	 * Handle 6 byte CDBs.
2442	 */
2443	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_6) ||
2444	    (CDB[0] == WRITE_6))) {
2445		/*
2446		 * Get the transfer size in blocks.
2447		 */
2448		io_size = (cm->cm_length >> sc->DD_block_exponent);
2449
2450		/*
2451		 * Get virtual LBA given in the CDB.
2452		 */
2453		virtLBA = ((uint64_t)(CDB[1] & 0x1F) << 16) |
2454		    ((uint64_t)CDB[2] << 8) | (uint64_t)CDB[3];
2455
2456		/*
2457		 * Check that LBA range for I/O does not exceed volume's
2458		 * MaxLBA.
2459		 */
2460		if ((virtLBA + (uint64_t)io_size - 1) <=
2461		    sc->DD_max_lba) {
2462			/*
2463			 * Check if the I/O crosses a stripe boundary.  If not,
2464			 * translate the virtual LBA to a physical LBA and set
2465			 * the DevHandle for the PhysDisk to be used.  If it
2466			 * does cross a boundry, do normal I/O.  To get the
2467			 * right DevHandle to use, get the map number for the
2468			 * column, then use that map number to look up the
2469			 * DevHandle of the PhysDisk.
2470			 */
2471			stripe_offset = (uint32_t)virtLBA &
2472			    (sc->DD_stripe_size - 1);
2473			if ((stripe_offset + io_size) <= sc->DD_stripe_size) {
2474				physLBA = (uint32_t)virtLBA >>
2475				    sc->DD_stripe_exponent;
2476				stripe_unit = physLBA / sc->DD_num_phys_disks;
2477				column = physLBA % sc->DD_num_phys_disks;
2478				pIO_req->DevHandle =
2479				    htole16(sc->DD_column_map[column].dev_handle);
2480				/* ???? Is this endian safe*/
2481				cm->cm_desc.SCSIIO.DevHandle =
2482				    pIO_req->DevHandle;
2483
2484				physLBA = (stripe_unit <<
2485				    sc->DD_stripe_exponent) + stripe_offset;
2486				ptrLBA = &pIO_req->CDB.CDB32[1];
2487				physLBA_byte = (uint8_t)(physLBA >> 16);
2488				*ptrLBA = physLBA_byte;
2489				ptrLBA = &pIO_req->CDB.CDB32[2];
2490				physLBA_byte = (uint8_t)(physLBA >> 8);
2491				*ptrLBA = physLBA_byte;
2492				ptrLBA = &pIO_req->CDB.CDB32[3];
2493				physLBA_byte = (uint8_t)physLBA;
2494				*ptrLBA = physLBA_byte;
2495
2496				/*
2497				 * Set flag that Direct Drive I/O is
2498				 * being done.
2499				 */
2500				cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2501			}
2502		}
2503		return;
2504	}
2505
2506	/*
2507	 * Handle 10, 12 or 16 byte CDBs.
2508	 */
2509	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_10) ||
2510	    (CDB[0] == WRITE_10) || (CDB[0] == READ_16) ||
2511	    (CDB[0] == WRITE_16) || (CDB[0] == READ_12) ||
2512	    (CDB[0] == WRITE_12))) {
2513		/*
2514		 * For 16-byte CDB's, verify that the upper 4 bytes of the CDB
2515		 * are 0.  If not, this is accessing beyond 2TB so handle it in
2516		 * the else section.  10-byte and 12-byte CDB's are OK.
2517		 * FreeBSD sends very rare 12 byte READ/WRITE, but driver is
2518		 * ready to accept 12byte CDB for Direct IOs.
2519		 */
2520		if ((CDB[0] == READ_10 || CDB[0] == WRITE_10) ||
2521		    (CDB[0] == READ_12 || CDB[0] == WRITE_12) ||
2522		    !(CDB[2] | CDB[3] | CDB[4] | CDB[5])) {
2523			/*
2524			 * Get the transfer size in blocks.
2525			 */
2526			io_size = (cm->cm_length >> sc->DD_block_exponent);
2527
2528			/*
2529			 * Get virtual LBA.  Point to correct lower 4 bytes of
2530			 * LBA in the CDB depending on command.
2531			 */
2532			lba_idx = ((CDB[0] == READ_12) ||
2533				(CDB[0] == WRITE_12) ||
2534				(CDB[0] == READ_10) ||
2535				(CDB[0] == WRITE_10))? 2 : 6;
2536			virtLBA = ((uint64_t)CDB[lba_idx] << 24) |
2537			    ((uint64_t)CDB[lba_idx + 1] << 16) |
2538			    ((uint64_t)CDB[lba_idx + 2] << 8) |
2539			    (uint64_t)CDB[lba_idx + 3];
2540
2541			/*
2542			 * Check that LBA range for I/O does not exceed volume's
2543			 * MaxLBA.
2544			 */
2545			if ((virtLBA + (uint64_t)io_size - 1) <=
2546			    sc->DD_max_lba) {
2547				/*
2548				 * Check if the I/O crosses a stripe boundary.
2549				 * If not, translate the virtual LBA to a
2550				 * physical LBA and set the DevHandle for the
2551				 * PhysDisk to be used.  If it does cross a
2552				 * boundry, do normal I/O.  To get the right
2553				 * DevHandle to use, get the map number for the
2554				 * column, then use that map number to look up
2555				 * the DevHandle of the PhysDisk.
2556				 */
2557				stripe_offset = (uint32_t)virtLBA &
2558				    (sc->DD_stripe_size - 1);
2559				if ((stripe_offset + io_size) <=
2560				    sc->DD_stripe_size) {
2561					physLBA = (uint32_t)virtLBA >>
2562					    sc->DD_stripe_exponent;
2563					stripe_unit = physLBA /
2564					    sc->DD_num_phys_disks;
2565					column = physLBA %
2566					    sc->DD_num_phys_disks;
2567					pIO_req->DevHandle =
2568					    htole16(sc->DD_column_map[column].
2569					    dev_handle);
2570					cm->cm_desc.SCSIIO.DevHandle =
2571					    pIO_req->DevHandle;
2572
2573					physLBA = (stripe_unit <<
2574					    sc->DD_stripe_exponent) +
2575					    stripe_offset;
2576					ptrLBA =
2577					    &pIO_req->CDB.CDB32[lba_idx];
2578					physLBA_byte = (uint8_t)(physLBA >> 24);
2579					*ptrLBA = physLBA_byte;
2580					ptrLBA =
2581					    &pIO_req->CDB.CDB32[lba_idx + 1];
2582					physLBA_byte = (uint8_t)(physLBA >> 16);
2583					*ptrLBA = physLBA_byte;
2584					ptrLBA =
2585					    &pIO_req->CDB.CDB32[lba_idx + 2];
2586					physLBA_byte = (uint8_t)(physLBA >> 8);
2587					*ptrLBA = physLBA_byte;
2588					ptrLBA =
2589					    &pIO_req->CDB.CDB32[lba_idx + 3];
2590					physLBA_byte = (uint8_t)physLBA;
2591					*ptrLBA = physLBA_byte;
2592
2593					/*
2594					 * Set flag that Direct Drive I/O is
2595					 * being done.
2596					 */
2597					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2598				}
2599			}
2600		} else {
2601			/*
2602			 * 16-byte CDB and the upper 4 bytes of the CDB are not
2603			 * 0.  Get the transfer size in blocks.
2604			 */
2605			io_size = (cm->cm_length >> sc->DD_block_exponent);
2606
2607			/*
2608			 * Get virtual LBA.
2609			 */
2610			virtLBA = ((uint64_t)CDB[2] << 54) |
2611			    ((uint64_t)CDB[3] << 48) |
2612			    ((uint64_t)CDB[4] << 40) |
2613			    ((uint64_t)CDB[5] << 32) |
2614			    ((uint64_t)CDB[6] << 24) |
2615			    ((uint64_t)CDB[7] << 16) |
2616			    ((uint64_t)CDB[8] << 8) |
2617			    (uint64_t)CDB[9];
2618
2619			/*
2620			 * Check that LBA range for I/O does not exceed volume's
2621			 * MaxLBA.
2622			 */
2623			if ((virtLBA + (uint64_t)io_size - 1) <=
2624			    sc->DD_max_lba) {
2625				/*
2626				 * Check if the I/O crosses a stripe boundary.
2627				 * If not, translate the virtual LBA to a
2628				 * physical LBA and set the DevHandle for the
2629				 * PhysDisk to be used.  If it does cross a
2630				 * boundry, do normal I/O.  To get the right
2631				 * DevHandle to use, get the map number for the
2632				 * column, then use that map number to look up
2633				 * the DevHandle of the PhysDisk.
2634				 */
2635				stripe_offset = (uint32_t)virtLBA &
2636				    (sc->DD_stripe_size - 1);
2637				if ((stripe_offset + io_size) <=
2638				    sc->DD_stripe_size) {
2639					physLBA = (uint32_t)(virtLBA >>
2640					    sc->DD_stripe_exponent);
2641					stripe_unit = physLBA /
2642					    sc->DD_num_phys_disks;
2643					column = physLBA %
2644					    sc->DD_num_phys_disks;
2645					pIO_req->DevHandle =
2646					    htole16(sc->DD_column_map[column].
2647					    dev_handle);
2648					cm->cm_desc.SCSIIO.DevHandle =
2649					    pIO_req->DevHandle;
2650
2651					physLBA = (stripe_unit <<
2652					    sc->DD_stripe_exponent) +
2653					    stripe_offset;
2654
2655					/*
2656					 * Set upper 4 bytes of LBA to 0.  We
2657					 * assume that the phys disks are less
2658					 * than 2 TB's in size.  Then, set the
2659					 * lower 4 bytes.
2660					 */
2661					pIO_req->CDB.CDB32[2] = 0;
2662					pIO_req->CDB.CDB32[3] = 0;
2663					pIO_req->CDB.CDB32[4] = 0;
2664					pIO_req->CDB.CDB32[5] = 0;
2665					ptrLBA = &pIO_req->CDB.CDB32[6];
2666					physLBA_byte = (uint8_t)(physLBA >> 24);
2667					*ptrLBA = physLBA_byte;
2668					ptrLBA = &pIO_req->CDB.CDB32[7];
2669					physLBA_byte = (uint8_t)(physLBA >> 16);
2670					*ptrLBA = physLBA_byte;
2671					ptrLBA = &pIO_req->CDB.CDB32[8];
2672					physLBA_byte = (uint8_t)(physLBA >> 8);
2673					*ptrLBA = physLBA_byte;
2674					ptrLBA = &pIO_req->CDB.CDB32[9];
2675					physLBA_byte = (uint8_t)physLBA;
2676					*ptrLBA = physLBA_byte;
2677
2678					/*
2679					 * Set flag that Direct Drive I/O is
2680					 * being done.
2681					 */
2682					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2683				}
2684			}
2685		}
2686	}
2687}
2688
2689#if __FreeBSD_version >= 900026
2690static void
2691mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm)
2692{
2693	MPI2_SMP_PASSTHROUGH_REPLY *rpl;
2694	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2695	uint64_t sasaddr;
2696	union ccb *ccb;
2697
2698	ccb = cm->cm_complete_data;
2699
2700	/*
2701	 * Currently there should be no way we can hit this case.  It only
2702	 * happens when we have a failure to allocate chain frames, and SMP
2703	 * commands require two S/G elements only.  That should be handled
2704	 * in the standard request size.
2705	 */
2706	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2707		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x on SMP request!\n",
2708			   __func__, cm->cm_flags);
2709		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2710		goto bailout;
2711        }
2712
2713	rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply;
2714	if (rpl == NULL) {
2715		mps_dprint(sc, MPS_ERROR, "%s: NULL cm_reply!\n", __func__);
2716		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2717		goto bailout;
2718	}
2719
2720	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2721	sasaddr = le32toh(req->SASAddress.Low);
2722	sasaddr |= ((uint64_t)(le32toh(req->SASAddress.High))) << 32;
2723
2724	if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
2725	    MPI2_IOCSTATUS_SUCCESS ||
2726	    rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) {
2727		mps_dprint(sc, MPS_XINFO, "%s: IOCStatus %04x SASStatus %02x\n",
2728		    __func__, le16toh(rpl->IOCStatus), rpl->SASStatus);
2729		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2730		goto bailout;
2731	}
2732
2733	mps_dprint(sc, MPS_XINFO, "%s: SMP request to SAS address "
2734		   "%#jx completed successfully\n", __func__,
2735		   (uintmax_t)sasaddr);
2736
2737	if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED)
2738		ccb->ccb_h.status = CAM_REQ_CMP;
2739	else
2740		ccb->ccb_h.status = CAM_SMP_STATUS_ERROR;
2741
2742bailout:
2743	/*
2744	 * We sync in both directions because we had DMAs in the S/G list
2745	 * in both directions.
2746	 */
2747	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
2748			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2749	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2750	mps_free_command(sc, cm);
2751	xpt_done(ccb);
2752}
2753
2754static void
2755mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb, uint64_t sasaddr)
2756{
2757	struct mps_command *cm;
2758	uint8_t *request, *response;
2759	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2760	struct mps_softc *sc;
2761	struct sglist *sg;
2762	int error;
2763
2764	sc = sassc->sc;
2765	sg = NULL;
2766	error = 0;
2767
2768	/*
2769	 * XXX We don't yet support physical addresses here.
2770	 */
2771	switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) {
2772	case CAM_DATA_PADDR:
2773	case CAM_DATA_SG_PADDR:
2774		mps_dprint(sc, MPS_ERROR,
2775			   "%s: physical addresses not supported\n", __func__);
2776		ccb->ccb_h.status = CAM_REQ_INVALID;
2777		xpt_done(ccb);
2778		return;
2779	case CAM_DATA_SG:
2780		/*
2781		 * The chip does not support more than one buffer for the
2782		 * request or response.
2783		 */
2784	 	if ((ccb->smpio.smp_request_sglist_cnt > 1)
2785		  || (ccb->smpio.smp_response_sglist_cnt > 1)) {
2786			mps_dprint(sc, MPS_ERROR,
2787				   "%s: multiple request or response "
2788				   "buffer segments not supported for SMP\n",
2789				   __func__);
2790			ccb->ccb_h.status = CAM_REQ_INVALID;
2791			xpt_done(ccb);
2792			return;
2793		}
2794
2795		/*
2796		 * The CAM_SCATTER_VALID flag was originally implemented
2797		 * for the XPT_SCSI_IO CCB, which only has one data pointer.
2798		 * We have two.  So, just take that flag to mean that we
2799		 * might have S/G lists, and look at the S/G segment count
2800		 * to figure out whether that is the case for each individual
2801		 * buffer.
2802		 */
2803		if (ccb->smpio.smp_request_sglist_cnt != 0) {
2804			bus_dma_segment_t *req_sg;
2805
2806			req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request;
2807			request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr;
2808		} else
2809			request = ccb->smpio.smp_request;
2810
2811		if (ccb->smpio.smp_response_sglist_cnt != 0) {
2812			bus_dma_segment_t *rsp_sg;
2813
2814			rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response;
2815			response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr;
2816		} else
2817			response = ccb->smpio.smp_response;
2818		break;
2819	case CAM_DATA_VADDR:
2820		request = ccb->smpio.smp_request;
2821		response = ccb->smpio.smp_response;
2822		break;
2823	default:
2824		ccb->ccb_h.status = CAM_REQ_INVALID;
2825		xpt_done(ccb);
2826		return;
2827	}
2828
2829	cm = mps_alloc_command(sc);
2830	if (cm == NULL) {
2831		mps_dprint(sc, MPS_ERROR,
2832		    "%s: cannot allocate command\n", __func__);
2833		ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2834		xpt_done(ccb);
2835		return;
2836	}
2837
2838	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2839	bzero(req, sizeof(*req));
2840	req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH;
2841
2842	/* Allow the chip to use any route to this SAS address. */
2843	req->PhysicalPort = 0xff;
2844
2845	req->RequestDataLength = htole16(ccb->smpio.smp_request_len);
2846	req->SGLFlags =
2847	    MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE | MPI2_SGLFLAGS_SGL_TYPE_MPI;
2848
2849	mps_dprint(sc, MPS_XINFO, "%s: sending SMP request to SAS "
2850	    "address %#jx\n", __func__, (uintmax_t)sasaddr);
2851
2852	mpi_init_sge(cm, req, &req->SGL);
2853
2854	/*
2855	 * Set up a uio to pass into mps_map_command().  This allows us to
2856	 * do one map command, and one busdma call in there.
2857	 */
2858	cm->cm_uio.uio_iov = cm->cm_iovec;
2859	cm->cm_uio.uio_iovcnt = 2;
2860	cm->cm_uio.uio_segflg = UIO_SYSSPACE;
2861
2862	/*
2863	 * The read/write flag isn't used by busdma, but set it just in
2864	 * case.  This isn't exactly accurate, either, since we're going in
2865	 * both directions.
2866	 */
2867	cm->cm_uio.uio_rw = UIO_WRITE;
2868
2869	cm->cm_iovec[0].iov_base = request;
2870	cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength);
2871	cm->cm_iovec[1].iov_base = response;
2872	cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len;
2873
2874	cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len +
2875			       cm->cm_iovec[1].iov_len;
2876
2877	/*
2878	 * Trigger a warning message in mps_data_cb() for the user if we
2879	 * wind up exceeding two S/G segments.  The chip expects one
2880	 * segment for the request and another for the response.
2881	 */
2882	cm->cm_max_segs = 2;
2883
2884	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
2885	cm->cm_complete = mpssas_smpio_complete;
2886	cm->cm_complete_data = ccb;
2887
2888	/*
2889	 * Tell the mapping code that we're using a uio, and that this is
2890	 * an SMP passthrough request.  There is a little special-case
2891	 * logic there (in mps_data_cb()) to handle the bidirectional
2892	 * transfer.
2893	 */
2894	cm->cm_flags |= MPS_CM_FLAGS_USE_UIO | MPS_CM_FLAGS_SMP_PASS |
2895			MPS_CM_FLAGS_DATAIN | MPS_CM_FLAGS_DATAOUT;
2896
2897	/* The chip data format is little endian. */
2898	req->SASAddress.High = htole32(sasaddr >> 32);
2899	req->SASAddress.Low = htole32(sasaddr);
2900
2901	/*
2902	 * XXX Note that we don't have a timeout/abort mechanism here.
2903	 * From the manual, it looks like task management requests only
2904	 * work for SCSI IO and SATA passthrough requests.  We may need to
2905	 * have a mechanism to retry requests in the event of a chip reset
2906	 * at least.  Hopefully the chip will insure that any errors short
2907	 * of that are relayed back to the driver.
2908	 */
2909	error = mps_map_command(sc, cm);
2910	if ((error != 0) && (error != EINPROGRESS)) {
2911		mps_dprint(sc, MPS_ERROR,
2912			   "%s: error %d returned from mps_map_command()\n",
2913			   __func__, error);
2914		goto bailout_error;
2915	}
2916
2917	return;
2918
2919bailout_error:
2920	mps_free_command(sc, cm);
2921	ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2922	xpt_done(ccb);
2923	return;
2924
2925}
2926
2927static void
2928mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb)
2929{
2930	struct mps_softc *sc;
2931	struct mpssas_target *targ;
2932	uint64_t sasaddr = 0;
2933
2934	sc = sassc->sc;
2935
2936	/*
2937	 * Make sure the target exists.
2938	 */
2939	targ = &sassc->targets[ccb->ccb_h.target_id];
2940	if (targ->handle == 0x0) {
2941		mps_dprint(sc, MPS_ERROR,
2942			   "%s: target %d does not exist!\n", __func__,
2943			   ccb->ccb_h.target_id);
2944		ccb->ccb_h.status = CAM_SEL_TIMEOUT;
2945		xpt_done(ccb);
2946		return;
2947	}
2948
2949	/*
2950	 * If this device has an embedded SMP target, we'll talk to it
2951	 * directly.
2952	 * figure out what the expander's address is.
2953	 */
2954	if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0)
2955		sasaddr = targ->sasaddr;
2956
2957	/*
2958	 * If we don't have a SAS address for the expander yet, try
2959	 * grabbing it from the page 0x83 information cached in the
2960	 * transport layer for this target.  LSI expanders report the
2961	 * expander SAS address as the port-associated SAS address in
2962	 * Inquiry VPD page 0x83.  Maxim expanders don't report it in page
2963	 * 0x83.
2964	 *
2965	 * XXX KDM disable this for now, but leave it commented out so that
2966	 * it is obvious that this is another possible way to get the SAS
2967	 * address.
2968	 *
2969	 * The parent handle method below is a little more reliable, and
2970	 * the other benefit is that it works for devices other than SES
2971	 * devices.  So you can send a SMP request to a da(4) device and it
2972	 * will get routed to the expander that device is attached to.
2973	 * (Assuming the da(4) device doesn't contain an SMP target...)
2974	 */
2975#if 0
2976	if (sasaddr == 0)
2977		sasaddr = xpt_path_sas_addr(ccb->ccb_h.path);
2978#endif
2979
2980	/*
2981	 * If we still don't have a SAS address for the expander, look for
2982	 * the parent device of this device, which is probably the expander.
2983	 */
2984	if (sasaddr == 0) {
2985#ifdef OLD_MPS_PROBE
2986		struct mpssas_target *parent_target;
2987#endif
2988
2989		if (targ->parent_handle == 0x0) {
2990			mps_dprint(sc, MPS_ERROR,
2991				   "%s: handle %d does not have a valid "
2992				   "parent handle!\n", __func__, targ->handle);
2993			ccb->ccb_h.status = CAM_REQ_INVALID;
2994			goto bailout;
2995		}
2996#ifdef OLD_MPS_PROBE
2997		parent_target = mpssas_find_target_by_handle(sassc, 0,
2998			targ->parent_handle);
2999
3000		if (parent_target == NULL) {
3001			mps_dprint(sc, MPS_ERROR,
3002				   "%s: handle %d does not have a valid "
3003				   "parent target!\n", __func__, targ->handle);
3004			ccb->ccb_h.status = CAM_REQ_INVALID;
3005			goto bailout;
3006		}
3007
3008		if ((parent_target->devinfo &
3009		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3010			mps_dprint(sc, MPS_ERROR,
3011				   "%s: handle %d parent %d does not "
3012				   "have an SMP target!\n", __func__,
3013				   targ->handle, parent_target->handle);
3014			ccb->ccb_h.status = CAM_REQ_INVALID;
3015			goto bailout;
3016
3017		}
3018
3019		sasaddr = parent_target->sasaddr;
3020#else /* OLD_MPS_PROBE */
3021		if ((targ->parent_devinfo &
3022		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3023			mps_dprint(sc, MPS_ERROR,
3024				   "%s: handle %d parent %d does not "
3025				   "have an SMP target!\n", __func__,
3026				   targ->handle, targ->parent_handle);
3027			ccb->ccb_h.status = CAM_REQ_INVALID;
3028			goto bailout;
3029
3030		}
3031		if (targ->parent_sasaddr == 0x0) {
3032			mps_dprint(sc, MPS_ERROR,
3033				   "%s: handle %d parent handle %d does "
3034				   "not have a valid SAS address!\n",
3035				   __func__, targ->handle, targ->parent_handle);
3036			ccb->ccb_h.status = CAM_REQ_INVALID;
3037			goto bailout;
3038		}
3039
3040		sasaddr = targ->parent_sasaddr;
3041#endif /* OLD_MPS_PROBE */
3042
3043	}
3044
3045	if (sasaddr == 0) {
3046		mps_dprint(sc, MPS_INFO,
3047			   "%s: unable to find SAS address for handle %d\n",
3048			   __func__, targ->handle);
3049		ccb->ccb_h.status = CAM_REQ_INVALID;
3050		goto bailout;
3051	}
3052	mpssas_send_smpcmd(sassc, ccb, sasaddr);
3053
3054	return;
3055
3056bailout:
3057	xpt_done(ccb);
3058
3059}
3060#endif //__FreeBSD_version >= 900026
3061
3062static void
3063mpssas_action_resetdev(struct mpssas_softc *sassc, union ccb *ccb)
3064{
3065	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3066	struct mps_softc *sc;
3067	struct mps_command *tm;
3068	struct mpssas_target *targ;
3069
3070	MPS_FUNCTRACE(sassc->sc);
3071	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
3072
3073	sc = sassc->sc;
3074	tm = mps_alloc_command(sc);
3075	if (tm == NULL) {
3076		mps_dprint(sc, MPS_ERROR,
3077		    "command alloc failure in mpssas_action_resetdev\n");
3078		ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
3079		xpt_done(ccb);
3080		return;
3081	}
3082
3083	targ = &sassc->targets[ccb->ccb_h.target_id];
3084	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3085	req->DevHandle = htole16(targ->handle);
3086	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
3087	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
3088
3089	/* SAS Hard Link Reset / SATA Link Reset */
3090	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3091
3092	tm->cm_data = NULL;
3093	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
3094	tm->cm_complete = mpssas_resetdev_complete;
3095	tm->cm_complete_data = ccb;
3096	tm->cm_targ = targ;
3097	mps_map_command(sc, tm);
3098}
3099
3100static void
3101mpssas_resetdev_complete(struct mps_softc *sc, struct mps_command *tm)
3102{
3103	MPI2_SCSI_TASK_MANAGE_REPLY *resp;
3104	union ccb *ccb;
3105
3106	MPS_FUNCTRACE(sc);
3107	mtx_assert(&sc->mps_mtx, MA_OWNED);
3108
3109	resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
3110	ccb = tm->cm_complete_data;
3111
3112	/*
3113	 * Currently there should be no way we can hit this case.  It only
3114	 * happens when we have a failure to allocate chain frames, and
3115	 * task management commands don't have S/G lists.
3116	 */
3117	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3118		MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3119
3120		req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3121
3122		mps_dprint(sc, MPS_ERROR,
3123			   "%s: cm_flags = %#x for reset of handle %#04x! "
3124			   "This should not happen!\n", __func__, tm->cm_flags,
3125			   req->DevHandle);
3126		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
3127		goto bailout;
3128	}
3129
3130	mps_dprint(sc, MPS_XINFO,
3131	    "%s: IOCStatus = 0x%x ResponseCode = 0x%x\n", __func__,
3132	    le16toh(resp->IOCStatus), le32toh(resp->ResponseCode));
3133
3134	if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) {
3135		ccb->ccb_h.status = CAM_REQ_CMP;
3136		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
3137		    CAM_LUN_WILDCARD);
3138	}
3139	else
3140		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
3141
3142bailout:
3143
3144	mpssas_free_tm(sc, tm);
3145	xpt_done(ccb);
3146}
3147
3148static void
3149mpssas_poll(struct cam_sim *sim)
3150{
3151	struct mpssas_softc *sassc;
3152
3153	sassc = cam_sim_softc(sim);
3154
3155	if (sassc->sc->mps_debug & MPS_TRACE) {
3156		/* frequent debug messages during a panic just slow
3157		 * everything down too much.
3158		 */
3159		mps_printf(sassc->sc, "%s clearing MPS_TRACE\n", __func__);
3160		sassc->sc->mps_debug &= ~MPS_TRACE;
3161	}
3162
3163	mps_intr_locked(sassc->sc);
3164}
3165
3166static void
3167mpssas_async(void *callback_arg, uint32_t code, struct cam_path *path,
3168	     void *arg)
3169{
3170	struct mps_softc *sc;
3171
3172	sc = (struct mps_softc *)callback_arg;
3173
3174	switch (code) {
3175#if (__FreeBSD_version >= 1000006) || \
3176    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
3177	case AC_ADVINFO_CHANGED: {
3178		struct mpssas_target *target;
3179		struct mpssas_softc *sassc;
3180		struct scsi_read_capacity_data_long rcap_buf;
3181		struct ccb_dev_advinfo cdai;
3182		struct mpssas_lun *lun;
3183		lun_id_t lunid;
3184		int found_lun;
3185		uintptr_t buftype;
3186
3187		buftype = (uintptr_t)arg;
3188
3189		found_lun = 0;
3190		sassc = sc->sassc;
3191
3192		/*
3193		 * We're only interested in read capacity data changes.
3194		 */
3195		if (buftype != CDAI_TYPE_RCAPLONG)
3196			break;
3197
3198		/*
3199		 * We should have a handle for this, but check to make sure.
3200		 */
3201		target = &sassc->targets[xpt_path_target_id(path)];
3202		if (target->handle == 0)
3203			break;
3204
3205		lunid = xpt_path_lun_id(path);
3206
3207		SLIST_FOREACH(lun, &target->luns, lun_link) {
3208			if (lun->lun_id == lunid) {
3209				found_lun = 1;
3210				break;
3211			}
3212		}
3213
3214		if (found_lun == 0) {
3215			lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3216				     M_NOWAIT | M_ZERO);
3217			if (lun == NULL) {
3218				mps_dprint(sc, MPS_ERROR, "Unable to alloc "
3219					   "LUN for EEDP support.\n");
3220				break;
3221			}
3222			lun->lun_id = lunid;
3223			SLIST_INSERT_HEAD(&target->luns, lun, lun_link);
3224		}
3225
3226		bzero(&rcap_buf, sizeof(rcap_buf));
3227		xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
3228		cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
3229		cdai.ccb_h.flags = CAM_DIR_IN;
3230		cdai.buftype = CDAI_TYPE_RCAPLONG;
3231		cdai.flags = 0;
3232		cdai.bufsiz = sizeof(rcap_buf);
3233		cdai.buf = (uint8_t *)&rcap_buf;
3234		xpt_action((union ccb *)&cdai);
3235		if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
3236			cam_release_devq(cdai.ccb_h.path,
3237					 0, 0, 0, FALSE);
3238
3239		if (((cdai.ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP)
3240		 && (rcap_buf.prot & SRC16_PROT_EN)) {
3241			lun->eedp_formatted = TRUE;
3242			lun->eedp_block_size = scsi_4btoul(rcap_buf.length);
3243		} else {
3244			lun->eedp_formatted = FALSE;
3245			lun->eedp_block_size = 0;
3246		}
3247		break;
3248	}
3249#else
3250	case AC_FOUND_DEVICE: {
3251		struct ccb_getdev *cgd;
3252
3253		cgd = arg;
3254		mpssas_check_eedp(sc, path, cgd);
3255		break;
3256	}
3257#endif
3258	default:
3259		break;
3260	}
3261}
3262
3263#if (__FreeBSD_version < 901503) || \
3264    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
3265static void
3266mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
3267		  struct ccb_getdev *cgd)
3268{
3269	struct mpssas_softc *sassc = sc->sassc;
3270	struct ccb_scsiio *csio;
3271	struct scsi_read_capacity_16 *scsi_cmd;
3272	struct scsi_read_capacity_eedp *rcap_buf;
3273	path_id_t pathid;
3274	target_id_t targetid;
3275	lun_id_t lunid;
3276	union ccb *ccb;
3277	struct cam_path *local_path;
3278	struct mpssas_target *target;
3279	struct mpssas_lun *lun;
3280	uint8_t	found_lun;
3281	char path_str[64];
3282
3283	sassc = sc->sassc;
3284	pathid = cam_sim_path(sassc->sim);
3285	targetid = xpt_path_target_id(path);
3286	lunid = xpt_path_lun_id(path);
3287
3288	target = &sassc->targets[targetid];
3289	if (target->handle == 0x0)
3290		return;
3291
3292	/*
3293	 * Determine if the device is EEDP capable.
3294	 *
3295	 * If this flag is set in the inquiry data,
3296	 * the device supports protection information,
3297	 * and must support the 16 byte read
3298	 * capacity command, otherwise continue without
3299	 * sending read cap 16
3300	 */
3301	if ((cgd->inq_data.spc3_flags & SPC3_SID_PROTECT) == 0)
3302		return;
3303
3304	/*
3305	 * Issue a READ CAPACITY 16 command.  This info
3306	 * is used to determine if the LUN is formatted
3307	 * for EEDP support.
3308	 */
3309	ccb = xpt_alloc_ccb_nowait();
3310	if (ccb == NULL) {
3311		mps_dprint(sc, MPS_ERROR, "Unable to alloc CCB "
3312		    "for EEDP support.\n");
3313		return;
3314	}
3315
3316	if (xpt_create_path(&local_path, xpt_periph,
3317	    pathid, targetid, lunid) != CAM_REQ_CMP) {
3318		mps_dprint(sc, MPS_ERROR, "Unable to create "
3319		    "path for EEDP support\n");
3320		xpt_free_ccb(ccb);
3321		return;
3322	}
3323
3324	/*
3325	 * If LUN is already in list, don't create a new
3326	 * one.
3327	 */
3328	found_lun = FALSE;
3329	SLIST_FOREACH(lun, &target->luns, lun_link) {
3330		if (lun->lun_id == lunid) {
3331			found_lun = TRUE;
3332			break;
3333		}
3334	}
3335	if (!found_lun) {
3336		lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3337		    M_NOWAIT | M_ZERO);
3338		if (lun == NULL) {
3339			mps_dprint(sc, MPS_ERROR,
3340			    "Unable to alloc LUN for EEDP support.\n");
3341			xpt_free_path(local_path);
3342			xpt_free_ccb(ccb);
3343			return;
3344		}
3345		lun->lun_id = lunid;
3346		SLIST_INSERT_HEAD(&target->luns, lun,
3347		    lun_link);
3348	}
3349
3350	xpt_path_string(local_path, path_str, sizeof(path_str));
3351	mps_dprint(sc, MPS_INFO, "Sending read cap: path %s handle %d\n",
3352	    path_str, target->handle);
3353
3354	/*
3355	 * Issue a READ CAPACITY 16 command for the LUN.
3356	 * The mpssas_read_cap_done function will load
3357	 * the read cap info into the LUN struct.
3358	 */
3359	rcap_buf = malloc(sizeof(struct scsi_read_capacity_eedp),
3360	    M_MPT2, M_NOWAIT | M_ZERO);
3361	if (rcap_buf == NULL) {
3362		mps_dprint(sc, MPS_FAULT,
3363		    "Unable to alloc read capacity buffer for EEDP support.\n");
3364		xpt_free_path(ccb->ccb_h.path);
3365		xpt_free_ccb(ccb);
3366		return;
3367	}
3368	xpt_setup_ccb(&ccb->ccb_h, local_path, CAM_PRIORITY_XPT);
3369	csio = &ccb->csio;
3370	csio->ccb_h.func_code = XPT_SCSI_IO;
3371	csio->ccb_h.flags = CAM_DIR_IN;
3372	csio->ccb_h.retry_count = 4;
3373	csio->ccb_h.cbfcnp = mpssas_read_cap_done;
3374	csio->ccb_h.timeout = 60000;
3375	csio->data_ptr = (uint8_t *)rcap_buf;
3376	csio->dxfer_len = sizeof(struct scsi_read_capacity_eedp);
3377	csio->sense_len = MPS_SENSE_LEN;
3378	csio->cdb_len = sizeof(*scsi_cmd);
3379	csio->tag_action = MSG_SIMPLE_Q_TAG;
3380
3381	scsi_cmd = (struct scsi_read_capacity_16 *)&csio->cdb_io.cdb_bytes;
3382	bzero(scsi_cmd, sizeof(*scsi_cmd));
3383	scsi_cmd->opcode = 0x9E;
3384	scsi_cmd->service_action = SRC16_SERVICE_ACTION;
3385	((uint8_t *)scsi_cmd)[13] = sizeof(struct scsi_read_capacity_eedp);
3386
3387	ccb->ccb_h.ppriv_ptr1 = sassc;
3388	xpt_action(ccb);
3389}
3390
3391static void
3392mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb)
3393{
3394	struct mpssas_softc *sassc;
3395	struct mpssas_target *target;
3396	struct mpssas_lun *lun;
3397	struct scsi_read_capacity_eedp *rcap_buf;
3398
3399	if (done_ccb == NULL)
3400		return;
3401
3402	/* Driver need to release devq, it Scsi command is
3403	 * generated by driver internally.
3404	 * Currently there is a single place where driver
3405	 * calls scsi command internally. In future if driver
3406	 * calls more scsi command internally, it needs to release
3407	 * devq internally, since those command will not go back to
3408	 * cam_periph.
3409	 */
3410	if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) ) {
3411        	done_ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
3412		xpt_release_devq(done_ccb->ccb_h.path,
3413			       	/*count*/ 1, /*run_queue*/TRUE);
3414	}
3415
3416	rcap_buf = (struct scsi_read_capacity_eedp *)done_ccb->csio.data_ptr;
3417
3418	/*
3419	 * Get the LUN ID for the path and look it up in the LUN list for the
3420	 * target.
3421	 */
3422	sassc = (struct mpssas_softc *)done_ccb->ccb_h.ppriv_ptr1;
3423	target = &sassc->targets[done_ccb->ccb_h.target_id];
3424	SLIST_FOREACH(lun, &target->luns, lun_link) {
3425		if (lun->lun_id != done_ccb->ccb_h.target_lun)
3426			continue;
3427
3428		/*
3429		 * Got the LUN in the target's LUN list.  Fill it in
3430		 * with EEDP info.  If the READ CAP 16 command had some
3431		 * SCSI error (common if command is not supported), mark
3432		 * the lun as not supporting EEDP and set the block size
3433		 * to 0.
3434		 */
3435		if (((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
3436		 || (done_ccb->csio.scsi_status != SCSI_STATUS_OK)) {
3437			lun->eedp_formatted = FALSE;
3438			lun->eedp_block_size = 0;
3439			break;
3440		}
3441
3442		if (rcap_buf->protect & 0x01) {
3443			mps_dprint(sassc->sc, MPS_INFO, "LUN %d for "
3444 			    "target ID %d is formatted for EEDP "
3445 			    "support.\n", done_ccb->ccb_h.target_lun,
3446 			    done_ccb->ccb_h.target_id);
3447			lun->eedp_formatted = TRUE;
3448			lun->eedp_block_size = scsi_4btoul(rcap_buf->length);
3449		}
3450		break;
3451	}
3452
3453	// Finished with this CCB and path.
3454	free(rcap_buf, M_MPT2);
3455	xpt_free_path(done_ccb->ccb_h.path);
3456	xpt_free_ccb(done_ccb);
3457}
3458#endif /* (__FreeBSD_version < 901503) || \
3459          ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006)) */
3460
3461int
3462mpssas_startup(struct mps_softc *sc)
3463{
3464	struct mpssas_softc *sassc;
3465
3466	/*
3467	 * Send the port enable message and set the wait_for_port_enable flag.
3468	 * This flag helps to keep the simq frozen until all discovery events
3469	 * are processed.
3470	 */
3471	sassc = sc->sassc;
3472	mpssas_startup_increment(sassc);
3473	sc->wait_for_port_enable = 1;
3474	mpssas_send_portenable(sc);
3475	return (0);
3476}
3477
3478static int
3479mpssas_send_portenable(struct mps_softc *sc)
3480{
3481	MPI2_PORT_ENABLE_REQUEST *request;
3482	struct mps_command *cm;
3483
3484	MPS_FUNCTRACE(sc);
3485
3486	if ((cm = mps_alloc_command(sc)) == NULL)
3487		return (EBUSY);
3488	request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req;
3489	request->Function = MPI2_FUNCTION_PORT_ENABLE;
3490	request->MsgFlags = 0;
3491	request->VP_ID = 0;
3492	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3493	cm->cm_complete = mpssas_portenable_complete;
3494	cm->cm_data = NULL;
3495	cm->cm_sge = NULL;
3496
3497	mps_map_command(sc, cm);
3498	mps_dprint(sc, MPS_XINFO,
3499	    "mps_send_portenable finished cm %p req %p complete %p\n",
3500	    cm, cm->cm_req, cm->cm_complete);
3501	return (0);
3502}
3503
3504static void
3505mpssas_portenable_complete(struct mps_softc *sc, struct mps_command *cm)
3506{
3507	MPI2_PORT_ENABLE_REPLY *reply;
3508	struct mpssas_softc *sassc;
3509
3510	MPS_FUNCTRACE(sc);
3511	sassc = sc->sassc;
3512
3513	/*
3514	 * Currently there should be no way we can hit this case.  It only
3515	 * happens when we have a failure to allocate chain frames, and
3516	 * port enable commands don't have S/G lists.
3517	 */
3518	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3519		mps_dprint(sc, MPS_ERROR, "%s: cm_flags = %#x for port enable! "
3520			   "This should not happen!\n", __func__, cm->cm_flags);
3521	}
3522
3523	reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply;
3524	if (reply == NULL)
3525		mps_dprint(sc, MPS_FAULT, "Portenable NULL reply\n");
3526	else if (le16toh(reply->IOCStatus & MPI2_IOCSTATUS_MASK) !=
3527	    MPI2_IOCSTATUS_SUCCESS)
3528		mps_dprint(sc, MPS_FAULT, "Portenable failed\n");
3529
3530	mps_free_command(sc, cm);
3531	if (sc->mps_ich.ich_arg != NULL) {
3532		mps_dprint(sc, MPS_XINFO, "disestablish config intrhook\n");
3533		config_intrhook_disestablish(&sc->mps_ich);
3534		sc->mps_ich.ich_arg = NULL;
3535	}
3536
3537	/*
3538	 * Get WarpDrive info after discovery is complete but before the scan
3539	 * starts.  At this point, all devices are ready to be exposed to the
3540	 * OS.  If devices should be hidden instead, take them out of the
3541	 * 'targets' array before the scan.  The devinfo for a disk will have
3542	 * some info and a volume's will be 0.  Use that to remove disks.
3543	 */
3544	mps_wd_config_pages(sc);
3545
3546	/*
3547	 * Done waiting for port enable to complete.  Decrement the refcount.
3548	 * If refcount is 0, discovery is complete and a rescan of the bus can
3549	 * take place.  Since the simq was explicitly frozen before port
3550	 * enable, it must be explicitly released here to keep the
3551	 * freeze/release count in sync.
3552	 */
3553	sc->wait_for_port_enable = 0;
3554	sc->port_enable_complete = 1;
3555	wakeup(&sc->port_enable_complete);
3556	mpssas_startup_decrement(sassc);
3557	xpt_release_simq(sassc->sim, 1);
3558}
3559
3560int
3561mpssas_check_id(struct mpssas_softc *sassc, int id)
3562{
3563	struct mps_softc *sc = sassc->sc;
3564	char *ids;
3565	char *name;
3566
3567	ids = &sc->exclude_ids[0];
3568	while((name = strsep(&ids, ",")) != NULL) {
3569		if (name[0] == '\0')
3570			continue;
3571		if (strtol(name, NULL, 0) == (long)id)
3572			return (1);
3573	}
3574
3575	return (0);
3576}
3577