mps_sas.c revision 0069682926e482627d3f64225ad0ad6018023ce6
1/*-
2 * Copyright (c) 2009 Yahoo! Inc.
3 * Copyright (c) 2011, 2012 LSI Corp.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 * LSI MPT-Fusion Host Adapter FreeBSD
28 *
29 * $FreeBSD$
30 */
31
32#include <sys/cdefs.h>
33__FBSDID("$FreeBSD$");
34
35/* Communications core for LSI MPT2 */
36
37/* TODO Move headers to mpsvar */
38#include <sys/types.h>
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/kernel.h>
42#include <sys/selinfo.h>
43#include <sys/module.h>
44#include <sys/bus.h>
45#include <sys/conf.h>
46#include <sys/bio.h>
47#include <sys/malloc.h>
48#include <sys/uio.h>
49#include <sys/sysctl.h>
50#include <sys/endian.h>
51#include <sys/queue.h>
52#include <sys/kthread.h>
53#include <sys/taskqueue.h>
54#include <sys/sbuf.h>
55
56#include <machine/bus.h>
57#include <machine/resource.h>
58#include <sys/rman.h>
59
60#include <machine/stdarg.h>
61
62#include <cam/cam.h>
63#include <cam/cam_ccb.h>
64#include <cam/cam_xpt.h>
65#include <cam/cam_debug.h>
66#include <cam/cam_sim.h>
67#include <cam/cam_xpt_sim.h>
68#include <cam/cam_xpt_periph.h>
69#include <cam/cam_periph.h>
70#include <cam/scsi/scsi_all.h>
71#include <cam/scsi/scsi_message.h>
72#if __FreeBSD_version >= 900026
73#include <cam/scsi/smp_all.h>
74#endif
75
76#include <dev/mps/mpi/mpi2_type.h>
77#include <dev/mps/mpi/mpi2.h>
78#include <dev/mps/mpi/mpi2_ioc.h>
79#include <dev/mps/mpi/mpi2_sas.h>
80#include <dev/mps/mpi/mpi2_cnfg.h>
81#include <dev/mps/mpi/mpi2_init.h>
82#include <dev/mps/mpi/mpi2_tool.h>
83#include <dev/mps/mps_ioctl.h>
84#include <dev/mps/mpsvar.h>
85#include <dev/mps/mps_table.h>
86#include <dev/mps/mps_sas.h>
87
88#define MPSSAS_DISCOVERY_TIMEOUT	20
89#define MPSSAS_MAX_DISCOVERY_TIMEOUTS	10 /* 200 seconds */
90
91/*
92 * static array to check SCSI OpCode for EEDP protection bits
93 */
94#define	PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP
95#define	PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
96#define	PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
97static uint8_t op_code_prot[256] = {
98	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
99	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
100	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
101	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
102	0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
103	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
104	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
105	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
106	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
107	0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
108	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
109	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
110	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
111	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
112	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
113	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
114};
115
116MALLOC_DEFINE(M_MPSSAS, "MPSSAS", "MPS SAS memory");
117
118static void mpssas_discovery_timeout(void *data);
119static void mpssas_remove_device(struct mps_softc *, struct mps_command *);
120static void mpssas_remove_complete(struct mps_softc *, struct mps_command *);
121static void mpssas_action(struct cam_sim *sim, union ccb *ccb);
122static void mpssas_poll(struct cam_sim *sim);
123static void mpssas_scsiio_timeout(void *data);
124static void mpssas_abort_complete(struct mps_softc *sc, struct mps_command *cm);
125static void mpssas_direct_drive_io(struct mpssas_softc *sassc,
126    struct mps_command *cm, union ccb *ccb);
127static void mpssas_action_scsiio(struct mpssas_softc *, union ccb *);
128static void mpssas_scsiio_complete(struct mps_softc *, struct mps_command *);
129static void mpssas_action_resetdev(struct mpssas_softc *, union ccb *);
130#if __FreeBSD_version >= 900026
131static void mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm);
132static void mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb,
133			       uint64_t sasaddr);
134static void mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb);
135#endif //FreeBSD_version >= 900026
136static void mpssas_resetdev_complete(struct mps_softc *, struct mps_command *);
137static int  mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm, struct mps_command *cm);
138static int  mpssas_send_reset(struct mps_softc *sc, struct mps_command *tm, uint8_t type);
139static void mpssas_rescan(struct mpssas_softc *sassc, union ccb *ccb);
140static void mpssas_rescan_done(struct cam_periph *periph, union ccb *done_ccb);
141static void mpssas_scanner_thread(void *arg);
142#if __FreeBSD_version >= 1000006
143static void mpssas_async(void *callback_arg, uint32_t code,
144			 struct cam_path *path, void *arg);
145#else
146static void mpssas_check_eedp(struct mpssas_softc *sassc);
147static void mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb);
148#endif
149static int mpssas_send_portenable(struct mps_softc *sc);
150static void mpssas_portenable_complete(struct mps_softc *sc,
151    struct mps_command *cm);
152
153struct mpssas_target *
154mpssas_find_target_by_handle(struct mpssas_softc *sassc, int start, uint16_t handle)
155{
156	struct mpssas_target *target;
157	int i;
158
159	for (i = start; i < sassc->sc->facts->MaxTargets; i++) {
160		target = &sassc->targets[i];
161		if (target->handle == handle)
162			return (target);
163	}
164
165	return (NULL);
166}
167
168/* we need to freeze the simq during attach and diag reset, to avoid failing
169 * commands before device handles have been found by discovery.  Since
170 * discovery involves reading config pages and possibly sending commands,
171 * discovery actions may continue even after we receive the end of discovery
172 * event, so refcount discovery actions instead of assuming we can unfreeze
173 * the simq when we get the event.
174 */
175void
176mpssas_startup_increment(struct mpssas_softc *sassc)
177{
178	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
179		if (sassc->startup_refcount++ == 0) {
180			/* just starting, freeze the simq */
181			mps_dprint(sassc->sc, MPS_INFO,
182			    "%s freezing simq\n", __func__);
183			xpt_freeze_simq(sassc->sim, 1);
184		}
185		mps_dprint(sassc->sc, MPS_TRACE, "%s refcount %u\n", __func__,
186		    sassc->startup_refcount);
187	}
188}
189
190void
191mpssas_startup_decrement(struct mpssas_softc *sassc)
192{
193	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
194		if (--sassc->startup_refcount == 0) {
195			/* finished all discovery-related actions, release
196			 * the simq and rescan for the latest topology.
197			 */
198			mps_dprint(sassc->sc, MPS_INFO,
199			    "%s releasing simq\n", __func__);
200			sassc->flags &= ~MPSSAS_IN_STARTUP;
201			xpt_release_simq(sassc->sim, 1);
202			mpssas_rescan_target(sassc->sc, NULL);
203		}
204		mps_dprint(sassc->sc, MPS_TRACE, "%s refcount %u\n", __func__,
205		    sassc->startup_refcount);
206	}
207}
208
209/* LSI's firmware requires us to stop sending commands when we're doing task
210 * management, so refcount the TMs and keep the simq frozen when any are in
211 * use.
212 */
213struct mps_command *
214mpssas_alloc_tm(struct mps_softc *sc)
215{
216	struct mps_command *tm;
217
218	tm = mps_alloc_high_priority_command(sc);
219	if (tm != NULL) {
220		if (sc->sassc->tm_count++ == 0) {
221			mps_printf(sc, "%s freezing simq\n", __func__);
222			xpt_freeze_simq(sc->sassc->sim, 1);
223		}
224		mps_dprint(sc, MPS_TRACE, "%s tm_count %u\n", __func__,
225		    sc->sassc->tm_count);
226	}
227	return tm;
228}
229
230void
231mpssas_free_tm(struct mps_softc *sc, struct mps_command *tm)
232{
233	if (tm == NULL)
234		return;
235
236	/* if there are no TMs in use, we can release the simq.  We use our
237	 * own refcount so that it's easier for a diag reset to cleanup and
238	 * release the simq.
239	 */
240	if (--sc->sassc->tm_count == 0) {
241		mps_printf(sc, "%s releasing simq\n", __func__);
242		xpt_release_simq(sc->sassc->sim, 1);
243	}
244	mps_dprint(sc, MPS_TRACE, "%s tm_count %u\n", __func__,
245	    sc->sassc->tm_count);
246
247	mps_free_high_priority_command(sc, tm);
248}
249
250
251void
252mpssas_rescan_target(struct mps_softc *sc, struct mpssas_target *targ)
253{
254	struct mpssas_softc *sassc = sc->sassc;
255	path_id_t pathid;
256	target_id_t targetid;
257	union ccb *ccb;
258
259	pathid = cam_sim_path(sassc->sim);
260	if (targ == NULL)
261		targetid = CAM_TARGET_WILDCARD;
262	else
263		targetid = targ - sassc->targets;
264
265	/*
266	 * Allocate a CCB and schedule a rescan.
267	 */
268	ccb = xpt_alloc_ccb_nowait();
269	if (ccb == NULL) {
270		mps_dprint(sc, MPS_FAULT, "unable to alloc CCB for rescan\n");
271		return;
272	}
273
274	if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, pathid,
275		            targetid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
276		mps_dprint(sc, MPS_FAULT, "unable to create path for rescan\n");
277		xpt_free_ccb(ccb);
278		return;
279	}
280
281	if (targetid == CAM_TARGET_WILDCARD)
282		ccb->ccb_h.func_code = XPT_SCAN_BUS;
283	else
284		ccb->ccb_h.func_code = XPT_SCAN_TGT;
285
286	mps_dprint(sc, MPS_TRACE, "%s targetid %u\n", __func__, targetid);
287	mpssas_rescan(sassc, ccb);
288}
289
290static void
291mpssas_log_command(struct mps_command *cm, const char *fmt, ...)
292{
293	struct sbuf sb;
294	va_list ap;
295	char str[192];
296	char path_str[64];
297
298	if (cm == NULL)
299		return;
300
301	sbuf_new(&sb, str, sizeof(str), 0);
302
303	va_start(ap, fmt);
304
305	if (cm->cm_ccb != NULL) {
306		xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str,
307				sizeof(path_str));
308		sbuf_cat(&sb, path_str);
309		if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) {
310			scsi_command_string(&cm->cm_ccb->csio, &sb);
311			sbuf_printf(&sb, "length %d ",
312				    cm->cm_ccb->csio.dxfer_len);
313		}
314	}
315	else {
316		sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ",
317		    cam_sim_name(cm->cm_sc->sassc->sim),
318		    cam_sim_unit(cm->cm_sc->sassc->sim),
319		    cam_sim_bus(cm->cm_sc->sassc->sim),
320		    cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF,
321		    cm->cm_lun);
322	}
323
324	sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID);
325	sbuf_vprintf(&sb, fmt, ap);
326	sbuf_finish(&sb);
327	printf("%s", sbuf_data(&sb));
328
329	va_end(ap);
330}
331
332
333static void
334mpssas_remove_volume(struct mps_softc *sc, struct mps_command *tm)
335{
336	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
337	struct mpssas_target *targ;
338	uint16_t handle;
339
340	mps_dprint(sc, MPS_INFO, "%s\n", __func__);
341
342	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
343	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
344	targ = tm->cm_targ;
345
346	if (reply == NULL) {
347		/* XXX retry the remove after the diag reset completes? */
348		mps_printf(sc, "%s NULL reply reseting device 0x%04x\n",
349		    __func__, handle);
350		mpssas_free_tm(sc, tm);
351		return;
352	}
353
354	if (reply->IOCStatus != MPI2_IOCSTATUS_SUCCESS) {
355		mps_printf(sc, "IOCStatus = 0x%x while resetting device 0x%x\n",
356		   reply->IOCStatus, handle);
357		mpssas_free_tm(sc, tm);
358		return;
359	}
360
361	mps_printf(sc, "Reset aborted %u commands\n", reply->TerminationCount);
362	mps_free_reply(sc, tm->cm_reply_data);
363	tm->cm_reply = NULL;	/* Ensures the the reply won't get re-freed */
364
365	mps_printf(sc, "clearing target %u handle 0x%04x\n", targ->tid, handle);
366
367	/*
368	 * Don't clear target if remove fails because things will get confusing.
369	 * Leave the devname and sasaddr intact so that we know to avoid reusing
370	 * this target id if possible, and so we can assign the same target id
371	 * to this device if it comes back in the future.
372	 */
373	if (reply->IOCStatus == MPI2_IOCSTATUS_SUCCESS) {
374		targ = tm->cm_targ;
375		targ->handle = 0x0;
376		targ->encl_handle = 0x0;
377		targ->encl_slot = 0x0;
378		targ->exp_dev_handle = 0x0;
379		targ->phy_num = 0x0;
380		targ->linkrate = 0x0;
381		targ->devinfo = 0x0;
382		targ->flags = 0x0;
383	}
384
385	mpssas_free_tm(sc, tm);
386}
387
388
389/*
390 * No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal.
391 * Otherwise Volume Delete is same as Bare Drive Removal.
392 */
393void
394mpssas_prepare_volume_remove(struct mpssas_softc *sassc, uint16_t handle)
395{
396	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
397	struct mps_softc *sc;
398	struct mps_command *cm;
399	struct mpssas_target *targ = NULL;
400
401	mps_dprint(sassc->sc, MPS_INFO, "%s\n", __func__);
402	sc = sassc->sc;
403
404#ifdef WD_SUPPORT
405	/*
406	 * If this is a WD controller, determine if the disk should be exposed
407	 * to the OS or not.  If disk should be exposed, return from this
408	 * function without doing anything.
409	 */
410	if (sc->WD_available && (sc->WD_hide_expose ==
411	    MPS_WD_EXPOSE_ALWAYS)) {
412		return;
413	}
414#endif //WD_SUPPORT
415
416	targ = mpssas_find_target_by_handle(sassc, 0, handle);
417	if (targ == NULL) {
418		/* FIXME: what is the action? */
419		/* We don't know about this device? */
420		printf("%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
421		return;
422	}
423
424	targ->flags |= MPSSAS_TARGET_INREMOVAL;
425
426	cm = mpssas_alloc_tm(sc);
427	if (cm == NULL) {
428		mps_printf(sc, "%s: command alloc failure\n", __func__);
429		return;
430	}
431
432	mpssas_rescan_target(sc, targ);
433
434	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
435	req->DevHandle = targ->handle;
436	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
437	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
438
439	/* SAS Hard Link Reset / SATA Link Reset */
440	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
441
442	cm->cm_targ = targ;
443	cm->cm_data = NULL;
444	cm->cm_desc.HighPriority.RequestFlags =
445	    MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
446	cm->cm_complete = mpssas_remove_volume;
447	cm->cm_complete_data = (void *)(uintptr_t)handle;
448	mps_map_command(sc, cm);
449}
450
451/*
452 * The MPT2 firmware performs debounce on the link to avoid transient link
453 * errors and false removals.  When it does decide that link has been lost
454 * and a device need to go away, it expects that the host will perform a
455 * target reset and then an op remove.  The reset has the side-effect of
456 * aborting any outstanding requests for the device, which is required for
457 * the op-remove to succeed.  It's not clear if the host should check for
458 * the device coming back alive after the reset.
459 */
460void
461mpssas_prepare_remove(struct mpssas_softc *sassc, uint16_t handle)
462{
463	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
464	struct mps_softc *sc;
465	struct mps_command *cm;
466	struct mpssas_target *targ = NULL;
467
468	mps_dprint(sassc->sc, MPS_INFO, "%s\n", __func__);
469
470	sc = sassc->sc;
471
472	targ = mpssas_find_target_by_handle(sassc, 0, handle);
473	if (targ == NULL) {
474		/* FIXME: what is the action? */
475		/* We don't know about this device? */
476		printf("%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
477		return;
478	}
479
480	targ->flags |= MPSSAS_TARGET_INREMOVAL;
481
482	cm = mpssas_alloc_tm(sc);
483	if (cm == NULL) {
484		mps_printf(sc, "%s: command alloc failure\n", __func__);
485		return;
486	}
487
488	mpssas_rescan_target(sc, targ);
489
490	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
491	memset(req, 0, sizeof(*req));
492	req->DevHandle = htole16(targ->handle);
493	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
494	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
495
496	/* SAS Hard Link Reset / SATA Link Reset */
497	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
498
499	cm->cm_targ = targ;
500	cm->cm_data = NULL;
501	cm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
502	cm->cm_complete = mpssas_remove_device;
503	cm->cm_complete_data = (void *)(uintptr_t)handle;
504	mps_map_command(sc, cm);
505}
506
507static void
508mpssas_remove_device(struct mps_softc *sc, struct mps_command *tm)
509{
510	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
511	MPI2_SAS_IOUNIT_CONTROL_REQUEST *req;
512	struct mpssas_target *targ;
513	struct mps_command *next_cm;
514	uint16_t handle;
515
516	mps_dprint(sc, MPS_INFO, "%s\n", __func__);
517
518	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
519	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
520	targ = tm->cm_targ;
521
522	/*
523	 * Currently there should be no way we can hit this case.  It only
524	 * happens when we have a failure to allocate chain frames, and
525	 * task management commands don't have S/G lists.
526	 */
527	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
528		mps_printf(sc, "%s: cm_flags = %#x for remove of handle %#04x! "
529			   "This should not happen!\n", __func__, tm->cm_flags,
530			   handle);
531		mpssas_free_tm(sc, tm);
532		return;
533	}
534
535	if (reply == NULL) {
536		/* XXX retry the remove after the diag reset completes? */
537		mps_printf(sc, "%s NULL reply reseting device 0x%04x\n",
538		    __func__, handle);
539		mpssas_free_tm(sc, tm);
540		return;
541	}
542
543	if (le16toh(reply->IOCStatus) != MPI2_IOCSTATUS_SUCCESS) {
544		mps_printf(sc, "IOCStatus = 0x%x while resetting device 0x%x\n",
545		   le16toh(reply->IOCStatus), handle);
546		mpssas_free_tm(sc, tm);
547		return;
548	}
549
550	mps_dprint(sc, MPS_INFO, "Reset aborted %u commands\n",
551	    le32toh(reply->TerminationCount));
552	mps_free_reply(sc, tm->cm_reply_data);
553	tm->cm_reply = NULL;	/* Ensures the the reply won't get re-freed */
554
555	/* Reuse the existing command */
556	req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req;
557	memset(req, 0, sizeof(*req));
558	req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
559	req->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
560	req->DevHandle = htole16(handle);
561	tm->cm_data = NULL;
562	tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
563	tm->cm_complete = mpssas_remove_complete;
564	tm->cm_complete_data = (void *)(uintptr_t)handle;
565
566	mps_map_command(sc, tm);
567
568	mps_dprint(sc, MPS_INFO, "clearing target %u handle 0x%04x\n",
569		   targ->tid, handle);
570	TAILQ_FOREACH_SAFE(tm, &targ->commands, cm_link, next_cm) {
571		union ccb *ccb;
572
573		mps_dprint(sc, MPS_INFO, "Completing missed command %p\n", tm);
574		ccb = tm->cm_complete_data;
575		ccb->ccb_h.status = CAM_DEV_NOT_THERE;
576		mpssas_scsiio_complete(sc, tm);
577	}
578}
579
580static void
581mpssas_remove_complete(struct mps_softc *sc, struct mps_command *tm)
582{
583	MPI2_SAS_IOUNIT_CONTROL_REPLY *reply;
584	uint16_t handle;
585	struct mpssas_target *targ;
586	struct mpssas_lun *lun;
587
588	mps_dprint(sc, MPS_INFO, "%s\n", __func__);
589
590	reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply;
591	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
592
593	/*
594	 * Currently there should be no way we can hit this case.  It only
595	 * happens when we have a failure to allocate chain frames, and
596	 * task management commands don't have S/G lists.
597	 */
598	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
599		mps_printf(sc, "%s: cm_flags = %#x for remove of handle %#04x! "
600			   "This should not happen!\n", __func__, tm->cm_flags,
601			   handle);
602		mpssas_free_tm(sc, tm);
603		return;
604	}
605
606	if (reply == NULL) {
607		/* most likely a chip reset */
608		mps_printf(sc, "%s NULL reply removing device 0x%04x\n",
609		    __func__, handle);
610		mpssas_free_tm(sc, tm);
611		return;
612	}
613
614	mps_printf(sc, "%s on handle 0x%04x, IOCStatus= 0x%x\n", __func__,
615	    handle, le16toh(reply->IOCStatus));
616
617	/*
618	 * Don't clear target if remove fails because things will get confusing.
619	 * Leave the devname and sasaddr intact so that we know to avoid reusing
620	 * this target id if possible, and so we can assign the same target id
621	 * to this device if it comes back in the future.
622	 */
623	if (le16toh(reply->IOCStatus) == MPI2_IOCSTATUS_SUCCESS) {
624		targ = tm->cm_targ;
625		targ->handle = 0x0;
626		targ->encl_handle = 0x0;
627		targ->encl_slot = 0x0;
628		targ->exp_dev_handle = 0x0;
629		targ->phy_num = 0x0;
630		targ->linkrate = 0x0;
631		targ->devinfo = 0x0;
632		targ->flags = 0x0;
633
634		while(!SLIST_EMPTY(&targ->luns)) {
635			lun = SLIST_FIRST(&targ->luns);
636			SLIST_REMOVE_HEAD(&targ->luns, lun_link);
637			free(lun, M_MPT2);
638		}
639	}
640
641
642	mpssas_free_tm(sc, tm);
643}
644
645static int
646mpssas_register_events(struct mps_softc *sc)
647{
648	u32 events[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];
649
650	bzero(events, 16);
651	setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
652	setbit(events, MPI2_EVENT_SAS_DISCOVERY);
653	setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
654	setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE);
655	setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW);
656	setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
657	setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
658	setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
659	setbit(events, MPI2_EVENT_IR_VOLUME);
660	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
661	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
662	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);
663
664	mps_register_events(sc, events, mpssas_evt_handler, NULL,
665	    &sc->sassc->mpssas_eh);
666
667	return (0);
668}
669
670int
671mps_attach_sas(struct mps_softc *sc)
672{
673	struct mpssas_softc *sassc;
674#if __FreeBSD_version >= 1000006
675	cam_status status;
676#endif
677	int unit, error = 0;
678
679	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
680
681	sassc = malloc(sizeof(struct mpssas_softc), M_MPT2, M_WAITOK|M_ZERO);
682	if(!sassc) {
683		device_printf(sc->mps_dev, "Cannot allocate memory %s %d\n",
684		__func__, __LINE__);
685		return (ENOMEM);
686	}
687	sassc->targets = malloc(sizeof(struct mpssas_target) *
688	    sc->facts->MaxTargets, M_MPT2, M_WAITOK|M_ZERO);
689	if(!sassc->targets) {
690		device_printf(sc->mps_dev, "Cannot allocate memory %s %d\n",
691		__func__, __LINE__);
692		free(sassc, M_MPT2);
693		return (ENOMEM);
694	}
695	sc->sassc = sassc;
696	sassc->sc = sc;
697
698	if ((sassc->devq = cam_simq_alloc(sc->num_reqs)) == NULL) {
699		mps_dprint(sc, MPS_FAULT, "Cannot allocate SIMQ\n");
700		error = ENOMEM;
701		goto out;
702	}
703
704	unit = device_get_unit(sc->mps_dev);
705	sassc->sim = cam_sim_alloc(mpssas_action, mpssas_poll, "mps", sassc,
706	    unit, &sc->mps_mtx, sc->num_reqs, sc->num_reqs, sassc->devq);
707	if (sassc->sim == NULL) {
708		mps_dprint(sc, MPS_FAULT, "Cannot allocate SIM\n");
709		error = EINVAL;
710		goto out;
711	}
712
713	TAILQ_INIT(&sassc->ev_queue);
714
715	/* Initialize taskqueue for Event Handling */
716	TASK_INIT(&sassc->ev_task, 0, mpssas_firmware_event_work, sc);
717	sassc->ev_tq = taskqueue_create("mps_taskq", M_NOWAIT | M_ZERO,
718	    taskqueue_thread_enqueue, &sassc->ev_tq);
719
720	/* Run the task queue with lowest priority */
721	taskqueue_start_threads(&sassc->ev_tq, 1, 255, "%s taskq",
722	    device_get_nameunit(sc->mps_dev));
723
724	TAILQ_INIT(&sassc->ccb_scanq);
725	error = mps_kproc_create(mpssas_scanner_thread, sassc,
726	    &sassc->rescan_thread, 0, 0, "mps_scan%d", unit);
727	if (error) {
728		mps_printf(sc, "Error %d starting rescan thread\n", error);
729		goto out;
730	}
731
732	mps_lock(sc);
733	sassc->flags |= MPSSAS_SCANTHREAD;
734
735	/*
736	 * XXX There should be a bus for every port on the adapter, but since
737	 * we're just going to fake the topology for now, we'll pretend that
738	 * everything is just a target on a single bus.
739	 */
740	if ((error = xpt_bus_register(sassc->sim, sc->mps_dev, 0)) != 0) {
741		mps_dprint(sc, MPS_FAULT, "Error %d registering SCSI bus\n",
742		    error);
743		mps_unlock(sc);
744		goto out;
745	}
746
747	/*
748	 * Assume that discovery events will start right away.  Freezing
749	 * the simq will prevent the CAM boottime scanner from running
750	 * before discovery is complete.
751	 */
752	sassc->flags |= MPSSAS_IN_STARTUP | MPSSAS_IN_DISCOVERY;
753	xpt_freeze_simq(sassc->sim, 1);
754	sc->sassc->startup_refcount = 0;
755
756	callout_init(&sassc->discovery_callout, 1 /*mpsafe*/);
757	sassc->discovery_timeouts = 0;
758
759	sassc->tm_count = 0;
760
761#if __FreeBSD_version >= 1000006
762	status = xpt_register_async(AC_ADVINFO_CHANGED, mpssas_async, sc, NULL);
763	if (status != CAM_REQ_CMP) {
764		mps_printf(sc, "Error %#x registering async handler for "
765			   "AC_ADVINFO_CHANGED events\n", status);
766	}
767#endif
768
769	mps_unlock(sc);
770
771	mpssas_register_events(sc);
772out:
773	if (error)
774		mps_detach_sas(sc);
775	return (error);
776}
777
778int
779mps_detach_sas(struct mps_softc *sc)
780{
781	struct mpssas_softc *sassc;
782	struct mpssas_lun *lun, *lun_tmp;
783	struct mpssas_target *targ;
784	int i;
785
786	mps_dprint(sc, MPS_INFO, "%s\n", __func__);
787
788	if (sc->sassc == NULL)
789		return (0);
790
791	sassc = sc->sassc;
792	mps_deregister_events(sc, sassc->mpssas_eh);
793
794	/*
795	 * Drain and free the event handling taskqueue with the lock
796	 * unheld so that any parallel processing tasks drain properly
797	 * without deadlocking.
798	 */
799	if (sassc->ev_tq != NULL)
800		taskqueue_free(sassc->ev_tq);
801
802	/* Make sure CAM doesn't wedge if we had to bail out early. */
803	mps_lock(sc);
804
805	/* Deregister our async handler */
806#if __FreeBSD_version >= 1000006
807	xpt_register_async(0, mpssas_async, sc, NULL);
808#endif
809
810	if (sassc->flags & MPSSAS_IN_STARTUP)
811		xpt_release_simq(sassc->sim, 1);
812
813	if (sassc->sim != NULL) {
814		xpt_bus_deregister(cam_sim_path(sassc->sim));
815		cam_sim_free(sassc->sim, FALSE);
816	}
817
818	if (sassc->flags & MPSSAS_SCANTHREAD) {
819		sassc->flags |= MPSSAS_SHUTDOWN;
820		wakeup(&sassc->ccb_scanq);
821
822		if (sassc->flags & MPSSAS_SCANTHREAD) {
823			msleep(&sassc->flags, &sc->mps_mtx, PRIBIO,
824			       "mps_shutdown", 30 * hz);
825		}
826	}
827	mps_unlock(sc);
828
829	mps_dprint(sc, MPS_INFO, "%s:%d\n", __func__,__LINE__);
830	if (sassc->devq != NULL)
831		cam_simq_free(sassc->devq);
832
833	for(i=0; i< sc->facts->MaxTargets ;i++) {
834		targ = &sassc->targets[i];
835		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
836			free(lun, M_MPT2);
837		}
838	}
839	free(sassc->targets, M_MPT2);
840	free(sassc, M_MPT2);
841	sc->sassc = NULL;
842
843	return (0);
844}
845
846void
847mpssas_discovery_end(struct mpssas_softc *sassc)
848{
849	struct mps_softc *sc = sassc->sc;
850
851	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
852
853	if (sassc->flags & MPSSAS_DISCOVERY_TIMEOUT_PENDING)
854		callout_stop(&sassc->discovery_callout);
855
856}
857
858static void
859mpssas_discovery_timeout(void *data)
860{
861	struct mpssas_softc *sassc = data;
862	struct mps_softc *sc;
863
864	sc = sassc->sc;
865	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
866
867	mps_lock(sc);
868	mps_printf(sc,
869	    "Timeout waiting for discovery, interrupts may not be working!\n");
870	sassc->flags &= ~MPSSAS_DISCOVERY_TIMEOUT_PENDING;
871
872	/* Poll the hardware for events in case interrupts aren't working */
873	mps_intr_locked(sc);
874
875	mps_printf(sassc->sc,
876	    "Finished polling after discovery timeout at %d\n", ticks);
877
878	if ((sassc->flags & MPSSAS_IN_DISCOVERY) == 0) {
879		mpssas_discovery_end(sassc);
880	} else {
881		if (sassc->discovery_timeouts < MPSSAS_MAX_DISCOVERY_TIMEOUTS) {
882			sassc->flags |= MPSSAS_DISCOVERY_TIMEOUT_PENDING;
883			callout_reset(&sassc->discovery_callout,
884			    MPSSAS_DISCOVERY_TIMEOUT * hz,
885			    mpssas_discovery_timeout, sassc);
886			sassc->discovery_timeouts++;
887		} else {
888			mps_dprint(sassc->sc, MPS_FAULT,
889			    "Discovery timed out, continuing.\n");
890			sassc->flags &= ~MPSSAS_IN_DISCOVERY;
891			mpssas_discovery_end(sassc);
892		}
893	}
894
895	mps_unlock(sc);
896}
897
898static void
899mpssas_action(struct cam_sim *sim, union ccb *ccb)
900{
901	struct mpssas_softc *sassc;
902
903	sassc = cam_sim_softc(sim);
904
905	mps_dprint(sassc->sc, MPS_TRACE, "%s func 0x%x\n", __func__,
906	    ccb->ccb_h.func_code);
907	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
908
909	switch (ccb->ccb_h.func_code) {
910	case XPT_PATH_INQ:
911	{
912		struct ccb_pathinq *cpi = &ccb->cpi;
913
914		cpi->version_num = 1;
915		cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
916		cpi->target_sprt = 0;
917		cpi->hba_misc = PIM_NOBUSRESET;
918		cpi->hba_eng_cnt = 0;
919		cpi->max_target = sassc->sc->facts->MaxTargets - 1;
920		cpi->max_lun = 255;
921		cpi->initiator_id = 255;
922		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
923		strncpy(cpi->hba_vid, "LSILogic", HBA_IDLEN);
924		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
925		cpi->unit_number = cam_sim_unit(sim);
926		cpi->bus_id = cam_sim_bus(sim);
927		cpi->base_transfer_speed = 150000;
928		cpi->transport = XPORT_SAS;
929		cpi->transport_version = 0;
930		cpi->protocol = PROTO_SCSI;
931		cpi->protocol_version = SCSI_REV_SPC;
932#if __FreeBSD_version >= 800001
933		/*
934		 * XXX KDM where does this number come from?
935		 */
936		cpi->maxio = 256 * 1024;
937#endif
938		cpi->ccb_h.status = CAM_REQ_CMP;
939		break;
940	}
941	case XPT_GET_TRAN_SETTINGS:
942	{
943		struct ccb_trans_settings	*cts;
944		struct ccb_trans_settings_sas	*sas;
945		struct ccb_trans_settings_scsi	*scsi;
946		struct mpssas_target *targ;
947
948		cts = &ccb->cts;
949		sas = &cts->xport_specific.sas;
950		scsi = &cts->proto_specific.scsi;
951
952		targ = &sassc->targets[cts->ccb_h.target_id];
953		if (targ->handle == 0x0) {
954			cts->ccb_h.status = CAM_SEL_TIMEOUT;
955			break;
956		}
957
958		cts->protocol_version = SCSI_REV_SPC2;
959		cts->transport = XPORT_SAS;
960		cts->transport_version = 0;
961
962		sas->valid = CTS_SAS_VALID_SPEED;
963		switch (targ->linkrate) {
964		case 0x08:
965			sas->bitrate = 150000;
966			break;
967		case 0x09:
968			sas->bitrate = 300000;
969			break;
970		case 0x0a:
971			sas->bitrate = 600000;
972			break;
973		default:
974			sas->valid = 0;
975		}
976
977		cts->protocol = PROTO_SCSI;
978		scsi->valid = CTS_SCSI_VALID_TQ;
979		scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
980
981		cts->ccb_h.status = CAM_REQ_CMP;
982		break;
983	}
984	case XPT_CALC_GEOMETRY:
985		cam_calc_geometry(&ccb->ccg, /*extended*/1);
986		ccb->ccb_h.status = CAM_REQ_CMP;
987		break;
988	case XPT_RESET_DEV:
989		mps_printf(sassc->sc, "mpssas_action XPT_RESET_DEV\n");
990		mpssas_action_resetdev(sassc, ccb);
991		return;
992	case XPT_RESET_BUS:
993	case XPT_ABORT:
994	case XPT_TERM_IO:
995		mps_printf(sassc->sc, "mpssas_action faking success for "
996			   "abort or reset\n");
997		ccb->ccb_h.status = CAM_REQ_CMP;
998		break;
999	case XPT_SCSI_IO:
1000		mpssas_action_scsiio(sassc, ccb);
1001		return;
1002#if __FreeBSD_version >= 900026
1003	case XPT_SMP_IO:
1004		mpssas_action_smpio(sassc, ccb);
1005		return;
1006#endif
1007	default:
1008		ccb->ccb_h.status = CAM_FUNC_NOTAVAIL;
1009		break;
1010	}
1011	xpt_done(ccb);
1012
1013}
1014
1015static void
1016mpssas_announce_reset(struct mps_softc *sc, uint32_t ac_code,
1017    target_id_t target_id, lun_id_t lun_id)
1018{
1019	path_id_t path_id = cam_sim_path(sc->sassc->sim);
1020	struct cam_path *path;
1021
1022	mps_printf(sc, "%s code %x target %d lun %d\n", __func__,
1023	    ac_code, target_id, lun_id);
1024
1025	if (xpt_create_path(&path, NULL,
1026		path_id, target_id, lun_id) != CAM_REQ_CMP) {
1027		mps_printf(sc, "unable to create path for reset "
1028			   "notification\n");
1029		return;
1030	}
1031
1032	xpt_async(ac_code, path, NULL);
1033	xpt_free_path(path);
1034}
1035
1036static void
1037mpssas_complete_all_commands(struct mps_softc *sc)
1038{
1039	struct mps_command *cm;
1040	int i;
1041	int completed;
1042
1043	mps_printf(sc, "%s\n", __func__);
1044	mtx_assert(&sc->mps_mtx, MA_OWNED);
1045
1046	/* complete all commands with a NULL reply */
1047	for (i = 1; i < sc->num_reqs; i++) {
1048		cm = &sc->commands[i];
1049		cm->cm_reply = NULL;
1050		completed = 0;
1051
1052		if (cm->cm_flags & MPS_CM_FLAGS_POLLED)
1053			cm->cm_flags |= MPS_CM_FLAGS_COMPLETE;
1054
1055		if (cm->cm_complete != NULL) {
1056			mpssas_log_command(cm,
1057			    "completing cm %p state %x ccb %p for diag reset\n",
1058			    cm, cm->cm_state, cm->cm_ccb);
1059
1060			cm->cm_complete(sc, cm);
1061			completed = 1;
1062		}
1063
1064		if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {
1065			mpssas_log_command(cm,
1066			    "waking up cm %p state %x ccb %p for diag reset\n",
1067			    cm, cm->cm_state, cm->cm_ccb);
1068			wakeup(cm);
1069			completed = 1;
1070		}
1071
1072		if ((completed == 0) && (cm->cm_state != MPS_CM_STATE_FREE)) {
1073			/* this should never happen, but if it does, log */
1074			mpssas_log_command(cm,
1075			    "cm %p state %x flags 0x%x ccb %p during diag "
1076			    "reset\n", cm, cm->cm_state, cm->cm_flags,
1077			    cm->cm_ccb);
1078		}
1079	}
1080}
1081
1082void
1083mpssas_handle_reinit(struct mps_softc *sc)
1084{
1085	int i;
1086
1087	/* Go back into startup mode and freeze the simq, so that CAM
1088	 * doesn't send any commands until after we've rediscovered all
1089	 * targets and found the proper device handles for them.
1090	 *
1091	 * After the reset, portenable will trigger discovery, and after all
1092	 * discovery-related activities have finished, the simq will be
1093	 * released.
1094	 */
1095	mps_printf(sc, "%s startup\n", __func__);
1096	sc->sassc->flags |= MPSSAS_IN_STARTUP;
1097	sc->sassc->flags |= MPSSAS_IN_DISCOVERY;
1098	xpt_freeze_simq(sc->sassc->sim, 1);
1099
1100	/* notify CAM of a bus reset */
1101	mpssas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
1102	    CAM_LUN_WILDCARD);
1103
1104	/* complete and cleanup after all outstanding commands */
1105	mpssas_complete_all_commands(sc);
1106
1107	mps_printf(sc, "%s startup %u tm %u after command completion\n",
1108	    __func__, sc->sassc->startup_refcount, sc->sassc->tm_count);
1109
1110	/*
1111	 * The simq was explicitly frozen above, so set the refcount to 0.
1112	 * The simq will be explicitly released after port enable completes.
1113	 */
1114	sc->sassc->startup_refcount = 0;
1115
1116	/* zero all the target handles, since they may change after the
1117	 * reset, and we have to rediscover all the targets and use the new
1118	 * handles.
1119	 */
1120	for (i = 0; i < sc->facts->MaxTargets; i++) {
1121		if (sc->sassc->targets[i].outstanding != 0)
1122			mps_printf(sc, "target %u outstanding %u\n",
1123			    i, sc->sassc->targets[i].outstanding);
1124		sc->sassc->targets[i].handle = 0x0;
1125		sc->sassc->targets[i].exp_dev_handle = 0x0;
1126		sc->sassc->targets[i].outstanding = 0;
1127		sc->sassc->targets[i].flags = MPSSAS_TARGET_INDIAGRESET;
1128	}
1129}
1130static void
1131mpssas_tm_timeout(void *data)
1132{
1133	struct mps_command *tm = data;
1134	struct mps_softc *sc = tm->cm_sc;
1135
1136	mtx_assert(&sc->mps_mtx, MA_OWNED);
1137
1138	mpssas_log_command(tm, "task mgmt %p timed out\n", tm);
1139	mps_reinit(sc);
1140}
1141
1142static void
1143mpssas_logical_unit_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1144{
1145	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1146	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1147	unsigned int cm_count = 0;
1148	struct mps_command *cm;
1149	struct mpssas_target *targ;
1150
1151	callout_stop(&tm->cm_callout);
1152
1153	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1154	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1155	targ = tm->cm_targ;
1156
1157	/*
1158	 * Currently there should be no way we can hit this case.  It only
1159	 * happens when we have a failure to allocate chain frames, and
1160	 * task management commands don't have S/G lists.
1161	 */
1162	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1163		mps_printf(sc, "%s: cm_flags = %#x for LUN reset! "
1164			   "This should not happen!\n", __func__, tm->cm_flags);
1165		mpssas_free_tm(sc, tm);
1166		return;
1167	}
1168
1169	if (reply == NULL) {
1170		mpssas_log_command(tm, "NULL reset reply for tm %p\n", tm);
1171		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1172			/* this completion was due to a reset, just cleanup */
1173			targ->flags &= ~MPSSAS_TARGET_INRESET;
1174			targ->tm = NULL;
1175			mpssas_free_tm(sc, tm);
1176		}
1177		else {
1178			/* we should have gotten a reply. */
1179			mps_reinit(sc);
1180		}
1181		return;
1182	}
1183
1184	mpssas_log_command(tm,
1185	    "logical unit reset status 0x%x code 0x%x count %u\n",
1186	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1187	    le32toh(reply->TerminationCount));
1188
1189	/* See if there are any outstanding commands for this LUN.
1190	 * This could be made more efficient by using a per-LU data
1191	 * structure of some sort.
1192	 */
1193	TAILQ_FOREACH(cm, &targ->commands, cm_link) {
1194		if (cm->cm_lun == tm->cm_lun)
1195			cm_count++;
1196	}
1197
1198	if (cm_count == 0) {
1199		mpssas_log_command(tm,
1200		    "logical unit %u finished recovery after reset\n",
1201		    tm->cm_lun, tm);
1202
1203		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1204		    tm->cm_lun);
1205
1206		/* we've finished recovery for this logical unit.  check and
1207		 * see if some other logical unit has a timedout command
1208		 * that needs to be processed.
1209		 */
1210		cm = TAILQ_FIRST(&targ->timedout_commands);
1211		if (cm) {
1212			mpssas_send_abort(sc, tm, cm);
1213		}
1214		else {
1215			targ->tm = NULL;
1216			mpssas_free_tm(sc, tm);
1217		}
1218	}
1219	else {
1220		/* if we still have commands for this LUN, the reset
1221		 * effectively failed, regardless of the status reported.
1222		 * Escalate to a target reset.
1223		 */
1224		mpssas_log_command(tm,
1225		    "logical unit reset complete for tm %p, but still have %u command(s)\n",
1226		    tm, cm_count);
1227		mpssas_send_reset(sc, tm,
1228		    MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
1229	}
1230}
1231
1232static void
1233mpssas_target_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1234{
1235	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1236	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1237	struct mpssas_target *targ;
1238
1239	callout_stop(&tm->cm_callout);
1240
1241	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1242	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1243	targ = tm->cm_targ;
1244
1245	/*
1246	 * Currently there should be no way we can hit this case.  It only
1247	 * happens when we have a failure to allocate chain frames, and
1248	 * task management commands don't have S/G lists.
1249	 */
1250	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1251		mps_printf(sc, "%s: cm_flags = %#x for target reset! "
1252			   "This should not happen!\n", __func__, tm->cm_flags);
1253		mpssas_free_tm(sc, tm);
1254		return;
1255	}
1256
1257	if (reply == NULL) {
1258		mpssas_log_command(tm, "NULL reset reply for tm %p\n", tm);
1259		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1260			/* this completion was due to a reset, just cleanup */
1261			targ->flags &= ~MPSSAS_TARGET_INRESET;
1262			targ->tm = NULL;
1263			mpssas_free_tm(sc, tm);
1264		}
1265		else {
1266			/* we should have gotten a reply. */
1267			mps_reinit(sc);
1268		}
1269		return;
1270	}
1271
1272	mpssas_log_command(tm,
1273	    "target reset status 0x%x code 0x%x count %u\n",
1274	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1275	    le32toh(reply->TerminationCount));
1276
1277	targ->flags &= ~MPSSAS_TARGET_INRESET;
1278
1279	if (targ->outstanding == 0) {
1280		/* we've finished recovery for this target and all
1281		 * of its logical units.
1282		 */
1283		mpssas_log_command(tm,
1284		    "recovery finished after target reset\n");
1285
1286		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1287		    CAM_LUN_WILDCARD);
1288
1289		targ->tm = NULL;
1290		mpssas_free_tm(sc, tm);
1291	}
1292	else {
1293		/* after a target reset, if this target still has
1294		 * outstanding commands, the reset effectively failed,
1295		 * regardless of the status reported.  escalate.
1296		 */
1297		mpssas_log_command(tm,
1298		    "target reset complete for tm %p, but still have %u command(s)\n",
1299		    tm, targ->outstanding);
1300		mps_reinit(sc);
1301	}
1302}
1303
1304#define MPS_RESET_TIMEOUT 30
1305
1306static int
1307mpssas_send_reset(struct mps_softc *sc, struct mps_command *tm, uint8_t type)
1308{
1309	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1310	struct mpssas_target *target;
1311	int err;
1312
1313	target = tm->cm_targ;
1314	if (target->handle == 0) {
1315		mps_printf(sc, "%s null devhandle for target_id %d\n",
1316		    __func__, target->tid);
1317		return -1;
1318	}
1319
1320	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1321	req->DevHandle = htole16(target->handle);
1322	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1323	req->TaskType = type;
1324
1325	if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) {
1326		/* XXX Need to handle invalid LUNs */
1327		MPS_SET_LUN(req->LUN, tm->cm_lun);
1328		tm->cm_targ->logical_unit_resets++;
1329		mpssas_log_command(tm, "sending logical unit reset\n");
1330		tm->cm_complete = mpssas_logical_unit_reset_complete;
1331	}
1332	else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) {
1333		/* Target reset method =  SAS Hard Link Reset / SATA Link Reset */
1334		req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
1335		tm->cm_targ->target_resets++;
1336		tm->cm_targ->flags |= MPSSAS_TARGET_INRESET;
1337		mpssas_log_command(tm, "sending target reset\n");
1338		tm->cm_complete = mpssas_target_reset_complete;
1339	}
1340	else {
1341		mps_printf(sc, "unexpected reset type 0x%x\n", type);
1342		return -1;
1343	}
1344
1345	tm->cm_data = NULL;
1346	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1347	tm->cm_complete_data = (void *)tm;
1348
1349	callout_reset(&tm->cm_callout, MPS_RESET_TIMEOUT * hz,
1350	    mpssas_tm_timeout, tm);
1351
1352	err = mps_map_command(sc, tm);
1353	if (err)
1354		mpssas_log_command(tm,
1355		    "error %d sending reset type %u\n",
1356		    err, type);
1357
1358	return err;
1359}
1360
1361
1362static void
1363mpssas_abort_complete(struct mps_softc *sc, struct mps_command *tm)
1364{
1365	struct mps_command *cm;
1366	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1367	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1368	struct mpssas_target *targ;
1369
1370	callout_stop(&tm->cm_callout);
1371
1372	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1373	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1374	targ = tm->cm_targ;
1375
1376	/*
1377	 * Currently there should be no way we can hit this case.  It only
1378	 * happens when we have a failure to allocate chain frames, and
1379	 * task management commands don't have S/G lists.
1380	 */
1381	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1382		mpssas_log_command(tm,
1383		    "cm_flags = %#x for abort %p TaskMID %u!\n",
1384		    tm->cm_flags, tm, le16toh(req->TaskMID));
1385		mpssas_free_tm(sc, tm);
1386		return;
1387	}
1388
1389	if (reply == NULL) {
1390		mpssas_log_command(tm,
1391		    "NULL abort reply for tm %p TaskMID %u\n",
1392		    tm, le16toh(req->TaskMID));
1393		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1394			/* this completion was due to a reset, just cleanup */
1395			targ->tm = NULL;
1396			mpssas_free_tm(sc, tm);
1397		}
1398		else {
1399			/* we should have gotten a reply. */
1400			mps_reinit(sc);
1401		}
1402		return;
1403	}
1404
1405	mpssas_log_command(tm,
1406	    "abort TaskMID %u status 0x%x code 0x%x count %u\n",
1407	    le16toh(req->TaskMID),
1408	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1409	    le32toh(reply->TerminationCount));
1410
1411	cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands);
1412	if (cm == NULL) {
1413		/* if there are no more timedout commands, we're done with
1414		 * error recovery for this target.
1415		 */
1416		mpssas_log_command(tm,
1417		    "finished recovery after aborting TaskMID %u\n",
1418		    le16toh(req->TaskMID));
1419
1420		targ->tm = NULL;
1421		mpssas_free_tm(sc, tm);
1422	}
1423	else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) {
1424		/* abort success, but we have more timedout commands to abort */
1425		mpssas_log_command(tm,
1426		    "continuing recovery after aborting TaskMID %u\n",
1427		    le16toh(req->TaskMID));
1428
1429		mpssas_send_abort(sc, tm, cm);
1430	}
1431	else {
1432		/* we didn't get a command completion, so the abort
1433		 * failed as far as we're concerned.  escalate.
1434		 */
1435		mpssas_log_command(tm,
1436		    "abort failed for TaskMID %u tm %p\n",
1437		    le16toh(req->TaskMID), tm);
1438
1439		mpssas_send_reset(sc, tm,
1440		    MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET);
1441	}
1442}
1443
1444#define MPS_ABORT_TIMEOUT 5
1445
1446static int
1447mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm, struct mps_command *cm)
1448{
1449	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1450	struct mpssas_target *targ;
1451	int err;
1452
1453	targ = cm->cm_targ;
1454	if (targ->handle == 0) {
1455		mps_printf(sc, "%s null devhandle for target_id %d\n",
1456		    __func__, cm->cm_ccb->ccb_h.target_id);
1457		return -1;
1458	}
1459
1460	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1461	req->DevHandle = htole16(targ->handle);
1462	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1463	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK;
1464
1465	/* XXX Need to handle invalid LUNs */
1466	MPS_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun);
1467
1468	req->TaskMID = htole16(cm->cm_desc.Default.SMID);
1469
1470	tm->cm_data = NULL;
1471	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1472	tm->cm_complete = mpssas_abort_complete;
1473	tm->cm_complete_data = (void *)tm;
1474	tm->cm_targ = cm->cm_targ;
1475	tm->cm_lun = cm->cm_lun;
1476
1477	callout_reset(&tm->cm_callout, MPS_ABORT_TIMEOUT * hz,
1478	    mpssas_tm_timeout, tm);
1479
1480	targ->aborts++;
1481
1482	err = mps_map_command(sc, tm);
1483	if (err)
1484		mpssas_log_command(tm,
1485		    "error %d sending abort for cm %p SMID %u\n",
1486		    err, cm, req->TaskMID);
1487	return err;
1488}
1489
1490
1491static void
1492mpssas_scsiio_timeout(void *data)
1493{
1494	struct mps_softc *sc;
1495	struct mps_command *cm;
1496	struct mpssas_target *targ;
1497
1498	cm = (struct mps_command *)data;
1499	sc = cm->cm_sc;
1500
1501	mtx_assert(&sc->mps_mtx, MA_OWNED);
1502
1503	mps_printf(sc, "%s checking sc %p cm %p\n", __func__, sc, cm);
1504
1505	/*
1506	 * Run the interrupt handler to make sure it's not pending.  This
1507	 * isn't perfect because the command could have already completed
1508	 * and been re-used, though this is unlikely.
1509	 */
1510	mps_intr_locked(sc);
1511	if (cm->cm_state == MPS_CM_STATE_FREE) {
1512		mps_printf(sc, "SCSI command %p sc %p almost timed out\n", cm, sc);
1513		return;
1514	}
1515
1516	if (cm->cm_ccb == NULL) {
1517		mps_printf(sc, "command timeout with NULL ccb\n");
1518		return;
1519	}
1520
1521	mpssas_log_command(cm, "command timeout cm %p ccb %p\n",
1522	    cm, cm->cm_ccb);
1523
1524	targ = cm->cm_targ;
1525	targ->timeouts++;
1526
1527	/* XXX first, check the firmware state, to see if it's still
1528	 * operational.  if not, do a diag reset.
1529	 */
1530
1531	cm->cm_ccb->ccb_h.status = CAM_CMD_TIMEOUT;
1532	cm->cm_state = MPS_CM_STATE_TIMEDOUT;
1533	TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery);
1534
1535	if (targ->tm != NULL) {
1536		/* target already in recovery, just queue up another
1537		 * timedout command to be processed later.
1538		 */
1539		mps_printf(sc, "queued timedout cm %p for processing by tm %p\n",
1540		    cm, targ->tm);
1541	}
1542	else if ((targ->tm = mpssas_alloc_tm(sc)) != NULL) {
1543		mps_printf(sc, "timedout cm %p allocated tm %p\n",
1544		    cm, targ->tm);
1545
1546		/* start recovery by aborting the first timedout command */
1547		mpssas_send_abort(sc, targ->tm, cm);
1548	}
1549	else {
1550		/* XXX queue this target up for recovery once a TM becomes
1551		 * available.  The firmware only has a limited number of
1552		 * HighPriority credits for the high priority requests used
1553		 * for task management, and we ran out.
1554		 *
1555		 * Isilon: don't worry about this for now, since we have
1556		 * more credits than disks in an enclosure, and limit
1557		 * ourselves to one TM per target for recovery.
1558		 */
1559		mps_printf(sc, "timedout cm %p failed to allocate a tm\n",
1560		    cm);
1561	}
1562
1563}
1564
1565static void
1566mpssas_action_scsiio(struct mpssas_softc *sassc, union ccb *ccb)
1567{
1568	MPI2_SCSI_IO_REQUEST *req;
1569	struct ccb_scsiio *csio;
1570	struct mps_softc *sc;
1571	struct mpssas_target *targ;
1572	struct mpssas_lun *lun;
1573	struct mps_command *cm;
1574	uint8_t i, lba_byte, *ref_tag_addr;
1575	uint16_t eedp_flags;
1576	uint32_t mpi_control;
1577
1578	sc = sassc->sc;
1579	mtx_assert(&sc->mps_mtx, MA_OWNED);
1580
1581	csio = &ccb->csio;
1582	targ = &sassc->targets[csio->ccb_h.target_id];
1583	mps_dprint(sc, MPS_TRACE, "%s ccb %p target flag %x\n", __func__, ccb, targ->flags);
1584	if (targ->handle == 0x0) {
1585		mps_dprint(sc, MPS_TRACE, "%s NULL handle for target %u\n",
1586		    __func__, csio->ccb_h.target_id);
1587		csio->ccb_h.status = CAM_SEL_TIMEOUT;
1588		xpt_done(ccb);
1589		return;
1590	}
1591	if (targ->flags & MPS_TARGET_FLAGS_RAID_COMPONENT) {
1592		mps_dprint(sc, MPS_TRACE, "%s Raid component no SCSI IO supported %u\n",
1593		    __func__, csio->ccb_h.target_id);
1594		csio->ccb_h.status = CAM_TID_INVALID;
1595		xpt_done(ccb);
1596		return;
1597	}
1598	/*
1599	 * If devinfo is 0 this will be a volume.  In that case don't tell CAM
1600	 * that the volume has timed out.  We want volumes to be enumerated
1601	 * until they are deleted/removed, not just failed.
1602	 */
1603	if (targ->flags & MPSSAS_TARGET_INREMOVAL) {
1604		if (targ->devinfo == 0)
1605			csio->ccb_h.status = CAM_REQ_CMP;
1606		else
1607			csio->ccb_h.status = CAM_SEL_TIMEOUT;
1608		xpt_done(ccb);
1609		return;
1610	}
1611
1612	if ((sc->mps_flags & MPS_FLAGS_SHUTDOWN) != 0) {
1613		mps_dprint(sc, MPS_TRACE, "%s shutting down\n", __func__);
1614		csio->ccb_h.status = CAM_TID_INVALID;
1615		xpt_done(ccb);
1616		return;
1617	}
1618
1619	cm = mps_alloc_command(sc);
1620	if (cm == NULL) {
1621		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
1622			xpt_freeze_simq(sassc->sim, 1);
1623			sassc->flags |= MPSSAS_QUEUE_FROZEN;
1624		}
1625		ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1626		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1627		xpt_done(ccb);
1628		return;
1629	}
1630
1631	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
1632	bzero(req, sizeof(*req));
1633	req->DevHandle = htole16(targ->handle);
1634	req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
1635	req->MsgFlags = 0;
1636	req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);
1637	req->SenseBufferLength = MPS_SENSE_LEN;
1638	req->SGLFlags = 0;
1639	req->ChainOffset = 0;
1640	req->SGLOffset0 = 24;	/* 32bit word offset to the SGL */
1641	req->SGLOffset1= 0;
1642	req->SGLOffset2= 0;
1643	req->SGLOffset3= 0;
1644	req->SkipCount = 0;
1645	req->DataLength = htole32(csio->dxfer_len);
1646	req->BidirectionalDataLength = 0;
1647	req->IoFlags = htole16(csio->cdb_len);
1648	req->EEDPFlags = 0;
1649
1650	/* Note: BiDirectional transfers are not supported */
1651	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1652	case CAM_DIR_IN:
1653		mpi_control = MPI2_SCSIIO_CONTROL_READ;
1654		cm->cm_flags |= MPS_CM_FLAGS_DATAIN;
1655		break;
1656	case CAM_DIR_OUT:
1657		mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
1658		cm->cm_flags |= MPS_CM_FLAGS_DATAOUT;
1659		break;
1660	case CAM_DIR_NONE:
1661	default:
1662		mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
1663		break;
1664	}
1665
1666  if (csio->cdb_len == 32)
1667                mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
1668	/*
1669	 * It looks like the hardware doesn't require an explicit tag
1670	 * number for each transaction.  SAM Task Management not supported
1671	 * at the moment.
1672	 */
1673	switch (csio->tag_action) {
1674	case MSG_HEAD_OF_Q_TAG:
1675		mpi_control |= MPI2_SCSIIO_CONTROL_HEADOFQ;
1676		break;
1677	case MSG_ORDERED_Q_TAG:
1678		mpi_control |= MPI2_SCSIIO_CONTROL_ORDEREDQ;
1679		break;
1680	case MSG_ACA_TASK:
1681		mpi_control |= MPI2_SCSIIO_CONTROL_ACAQ;
1682		break;
1683	case CAM_TAG_ACTION_NONE:
1684	case MSG_SIMPLE_Q_TAG:
1685	default:
1686		mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
1687		break;
1688	}
1689	mpi_control |= sc->mapping_table[csio->ccb_h.target_id].TLR_bits;
1690	req->Control = htole32(mpi_control);
1691	if (MPS_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) {
1692		mps_free_command(sc, cm);
1693		ccb->ccb_h.status = CAM_LUN_INVALID;
1694		xpt_done(ccb);
1695		return;
1696	}
1697
1698	if (csio->ccb_h.flags & CAM_CDB_POINTER)
1699		bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
1700	else
1701		bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
1702	req->IoFlags = htole16(csio->cdb_len);
1703
1704	/*
1705	 * Check if EEDP is supported and enabled.  If it is then check if the
1706	 * SCSI opcode could be using EEDP.  If so, make sure the LUN exists and
1707	 * is formatted for EEDP support.  If all of this is true, set CDB up
1708	 * for EEDP transfer.
1709	 */
1710	eedp_flags = op_code_prot[req->CDB.CDB32[0]];
1711	if (sc->eedp_enabled && eedp_flags) {
1712		SLIST_FOREACH(lun, &targ->luns, lun_link) {
1713			if (lun->lun_id == csio->ccb_h.target_lun) {
1714				break;
1715			}
1716		}
1717
1718		if ((lun != NULL) && (lun->eedp_formatted)) {
1719			req->EEDPBlockSize = htole16(lun->eedp_block_size);
1720			eedp_flags |= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
1721			    MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
1722			    MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
1723			req->EEDPFlags = htole16(eedp_flags);
1724
1725			/*
1726			 * If CDB less than 32, fill in Primary Ref Tag with
1727			 * low 4 bytes of LBA.  If CDB is 32, tag stuff is
1728			 * already there.  Also, set protection bit.  FreeBSD
1729			 * currently does not support CDBs bigger than 16, but
1730			 * the code doesn't hurt, and will be here for the
1731			 * future.
1732			 */
1733			if (csio->cdb_len != 32) {
1734				lba_byte = (csio->cdb_len == 16) ? 6 : 2;
1735				ref_tag_addr = (uint8_t *)&req->CDB.EEDP32.
1736				    PrimaryReferenceTag;
1737				for (i = 0; i < 4; i++) {
1738					*ref_tag_addr =
1739					    req->CDB.CDB32[lba_byte + i];
1740					ref_tag_addr++;
1741				}
1742				req->CDB.EEDP32.PrimaryReferenceTag =
1743					htole32(req->CDB.EEDP32.PrimaryReferenceTag);
1744				req->CDB.EEDP32.PrimaryApplicationTagMask =
1745				    0xFFFF;
1746				req->CDB.CDB32[1] = (req->CDB.CDB32[1] & 0x1F) |
1747				    0x20;
1748			} else {
1749				eedp_flags |=
1750				    MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG;
1751				req->EEDPFlags = htole16(eedp_flags);
1752				req->CDB.CDB32[10] = (req->CDB.CDB32[10] &
1753				    0x1F) | 0x20;
1754			}
1755		}
1756	}
1757
1758	cm->cm_data = csio->data_ptr;
1759	cm->cm_length = csio->dxfer_len;
1760	cm->cm_sge = &req->SGL;
1761	cm->cm_sglsize = (32 - 24) * 4;
1762	cm->cm_desc.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
1763	cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
1764	cm->cm_complete = mpssas_scsiio_complete;
1765	cm->cm_complete_data = ccb;
1766	cm->cm_targ = targ;
1767	cm->cm_lun = csio->ccb_h.target_lun;
1768	cm->cm_ccb = ccb;
1769
1770	/*
1771	 * If HBA is a WD and the command is not for a retry, try to build a
1772	 * direct I/O message. If failed, or the command is for a retry, send
1773	 * the I/O to the IR volume itself.
1774	 */
1775	if (sc->WD_valid_config) {
1776		if (ccb->ccb_h.status != MPS_WD_RETRY) {
1777			mpssas_direct_drive_io(sassc, cm, ccb);
1778		} else {
1779			ccb->ccb_h.status = CAM_REQ_INPROG;
1780		}
1781	}
1782
1783	callout_reset(&cm->cm_callout, (ccb->ccb_h.timeout * hz) / 1000,
1784	   mpssas_scsiio_timeout, cm);
1785
1786	targ->issued++;
1787	targ->outstanding++;
1788	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
1789
1790	if ((sc->mps_debug & MPS_TRACE) != 0)
1791		mpssas_log_command(cm, "%s cm %p ccb %p outstanding %u\n",
1792		    __func__, cm, ccb, targ->outstanding);
1793
1794	mps_map_command(sc, cm);
1795	return;
1796}
1797
1798static void
1799mps_response_code(struct mps_softc *sc, u8 response_code)
1800{
1801        char *desc;
1802
1803        switch (response_code) {
1804        case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
1805                desc = "task management request completed";
1806                break;
1807        case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
1808                desc = "invalid frame";
1809                break;
1810        case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
1811                desc = "task management request not supported";
1812                break;
1813        case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
1814                desc = "task management request failed";
1815                break;
1816        case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
1817                desc = "task management request succeeded";
1818                break;
1819        case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
1820                desc = "invalid lun";
1821                break;
1822        case 0xA:
1823                desc = "overlapped tag attempted";
1824                break;
1825        case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
1826                desc = "task queued, however not sent to target";
1827                break;
1828        default:
1829                desc = "unknown";
1830                break;
1831        }
1832		mps_dprint(sc, MPS_INFO, "response_code(0x%01x): %s\n",
1833                response_code, desc);
1834}
1835/**
1836 * mps_sc_failed_io_info - translated non-succesfull SCSI_IO request
1837 */
1838static void
1839mps_sc_failed_io_info(struct mps_softc *sc, struct ccb_scsiio *csio,
1840    Mpi2SCSIIOReply_t *mpi_reply)
1841{
1842	u32 response_info;
1843	u8 *response_bytes;
1844	u16 ioc_status = le16toh(mpi_reply->IOCStatus) &
1845	    MPI2_IOCSTATUS_MASK;
1846	u8 scsi_state = mpi_reply->SCSIState;
1847	u8 scsi_status = mpi_reply->SCSIStatus;
1848	char *desc_ioc_state = NULL;
1849	char *desc_scsi_status = NULL;
1850	char *desc_scsi_state = sc->tmp_string;
1851	u32 log_info = le32toh(mpi_reply->IOCLogInfo);
1852
1853	if (log_info == 0x31170000)
1854		return;
1855
1856	switch (ioc_status) {
1857	case MPI2_IOCSTATUS_SUCCESS:
1858		desc_ioc_state = "success";
1859		break;
1860	case MPI2_IOCSTATUS_INVALID_FUNCTION:
1861		desc_ioc_state = "invalid function";
1862		break;
1863	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
1864		desc_ioc_state = "scsi recovered error";
1865		break;
1866	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
1867		desc_ioc_state = "scsi invalid dev handle";
1868		break;
1869	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
1870		desc_ioc_state = "scsi device not there";
1871		break;
1872	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
1873		desc_ioc_state = "scsi data overrun";
1874		break;
1875	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
1876		desc_ioc_state = "scsi data underrun";
1877		break;
1878	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
1879		desc_ioc_state = "scsi io data error";
1880		break;
1881	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
1882		desc_ioc_state = "scsi protocol error";
1883		break;
1884	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
1885		desc_ioc_state = "scsi task terminated";
1886		break;
1887	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
1888		desc_ioc_state = "scsi residual mismatch";
1889		break;
1890	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
1891		desc_ioc_state = "scsi task mgmt failed";
1892		break;
1893	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
1894		desc_ioc_state = "scsi ioc terminated";
1895		break;
1896	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
1897		desc_ioc_state = "scsi ext terminated";
1898		break;
1899	case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
1900		desc_ioc_state = "eedp guard error";
1901		break;
1902	case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
1903		desc_ioc_state = "eedp ref tag error";
1904		break;
1905	case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
1906		desc_ioc_state = "eedp app tag error";
1907		break;
1908	default:
1909		desc_ioc_state = "unknown";
1910		break;
1911	}
1912
1913	switch (scsi_status) {
1914	case MPI2_SCSI_STATUS_GOOD:
1915		desc_scsi_status = "good";
1916		break;
1917	case MPI2_SCSI_STATUS_CHECK_CONDITION:
1918		desc_scsi_status = "check condition";
1919		break;
1920	case MPI2_SCSI_STATUS_CONDITION_MET:
1921		desc_scsi_status = "condition met";
1922		break;
1923	case MPI2_SCSI_STATUS_BUSY:
1924		desc_scsi_status = "busy";
1925		break;
1926	case MPI2_SCSI_STATUS_INTERMEDIATE:
1927		desc_scsi_status = "intermediate";
1928		break;
1929	case MPI2_SCSI_STATUS_INTERMEDIATE_CONDMET:
1930		desc_scsi_status = "intermediate condmet";
1931		break;
1932	case MPI2_SCSI_STATUS_RESERVATION_CONFLICT:
1933		desc_scsi_status = "reservation conflict";
1934		break;
1935	case MPI2_SCSI_STATUS_COMMAND_TERMINATED:
1936		desc_scsi_status = "command terminated";
1937		break;
1938	case MPI2_SCSI_STATUS_TASK_SET_FULL:
1939		desc_scsi_status = "task set full";
1940		break;
1941	case MPI2_SCSI_STATUS_ACA_ACTIVE:
1942		desc_scsi_status = "aca active";
1943		break;
1944	case MPI2_SCSI_STATUS_TASK_ABORTED:
1945		desc_scsi_status = "task aborted";
1946		break;
1947	default:
1948		desc_scsi_status = "unknown";
1949		break;
1950	}
1951
1952	desc_scsi_state[0] = '\0';
1953	if (!scsi_state)
1954		desc_scsi_state = " ";
1955	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID)
1956		strcat(desc_scsi_state, "response info ");
1957	if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
1958		strcat(desc_scsi_state, "state terminated ");
1959	if (scsi_state & MPI2_SCSI_STATE_NO_SCSI_STATUS)
1960		strcat(desc_scsi_state, "no status ");
1961	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_FAILED)
1962		strcat(desc_scsi_state, "autosense failed ");
1963	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID)
1964		strcat(desc_scsi_state, "autosense valid ");
1965
1966	mps_dprint(sc, MPS_INFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x), \n",
1967		le16toh(mpi_reply->DevHandle),
1968	    desc_ioc_state, ioc_status);
1969	/* We can add more detail about underflow data here
1970	 * TO-DO
1971	 * */
1972	mps_dprint(sc, MPS_INFO, "\tscsi_status(%s)(0x%02x), "
1973	    "scsi_state(%s)(0x%02x)\n", desc_scsi_status,
1974	    scsi_status, desc_scsi_state, scsi_state);
1975
1976	if (sc->mps_debug & MPS_INFO &&
1977		scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
1978		mps_dprint(sc, MPS_INFO, "-> Sense Buffer Data : Start :\n");
1979		scsi_sense_print(csio);
1980		mps_dprint(sc, MPS_INFO, "-> Sense Buffer Data : End :\n");
1981	}
1982
1983	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
1984		response_info = le32toh(mpi_reply->ResponseInfo);
1985		response_bytes = (u8 *)&response_info;
1986		mps_response_code(sc,response_bytes[0]);
1987	}
1988}
1989
1990static void
1991mpssas_scsiio_complete(struct mps_softc *sc, struct mps_command *cm)
1992{
1993	MPI2_SCSI_IO_REPLY *rep;
1994	union ccb *ccb;
1995	struct ccb_scsiio *csio;
1996	struct mpssas_softc *sassc;
1997	struct scsi_vpd_supported_page_list *vpd_list = NULL;
1998	u8 *TLR_bits, TLR_on;
1999	int dir = 0, i;
2000	u16 alloc_len;
2001
2002	mps_dprint(sc, MPS_TRACE,
2003	    "%s cm %p SMID %u ccb %p reply %p outstanding %u\n",
2004	    __func__, cm, cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply,
2005	    cm->cm_targ->outstanding);
2006
2007	callout_stop(&cm->cm_callout);
2008	mtx_assert(&sc->mps_mtx, MA_OWNED);
2009
2010	sassc = sc->sassc;
2011	ccb = cm->cm_complete_data;
2012	csio = &ccb->csio;
2013	rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply;
2014	/*
2015	 * XXX KDM if the chain allocation fails, does it matter if we do
2016	 * the sync and unload here?  It is simpler to do it in every case,
2017	 * assuming it doesn't cause problems.
2018	 */
2019	if (cm->cm_data != NULL) {
2020		if (cm->cm_flags & MPS_CM_FLAGS_DATAIN)
2021			dir = BUS_DMASYNC_POSTREAD;
2022		else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT)
2023			dir = BUS_DMASYNC_POSTWRITE;;
2024		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
2025		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2026	}
2027
2028	cm->cm_targ->completed++;
2029	cm->cm_targ->outstanding--;
2030	TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link);
2031
2032	if (cm->cm_state == MPS_CM_STATE_TIMEDOUT) {
2033		TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery);
2034		if (cm->cm_reply != NULL)
2035			mpssas_log_command(cm,
2036			    "completed timedout cm %p ccb %p during recovery "
2037			    "ioc %x scsi %x state %x xfer %u\n",
2038			    cm, cm->cm_ccb,
2039			    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2040			    le32toh(rep->TransferCount));
2041		else
2042			mpssas_log_command(cm,
2043			    "completed timedout cm %p ccb %p during recovery\n",
2044			    cm, cm->cm_ccb);
2045	} else if (cm->cm_targ->tm != NULL) {
2046		if (cm->cm_reply != NULL)
2047			mpssas_log_command(cm,
2048			    "completed cm %p ccb %p during recovery "
2049			    "ioc %x scsi %x state %x xfer %u\n",
2050			    cm, cm->cm_ccb,
2051			    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2052			    le32toh(rep->TransferCount));
2053		else
2054			mpssas_log_command(cm,
2055			    "completed cm %p ccb %p during recovery\n",
2056			    cm, cm->cm_ccb);
2057	} else if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
2058		mpssas_log_command(cm,
2059		    "reset completed cm %p ccb %p\n",
2060		    cm, cm->cm_ccb);
2061	}
2062
2063	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2064		/*
2065		 * We ran into an error after we tried to map the command,
2066		 * so we're getting a callback without queueing the command
2067		 * to the hardware.  So we set the status here, and it will
2068		 * be retained below.  We'll go through the "fast path",
2069		 * because there can be no reply when we haven't actually
2070		 * gone out to the hardware.
2071		 */
2072		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
2073
2074		/*
2075		 * Currently the only error included in the mask is
2076		 * MPS_CM_FLAGS_CHAIN_FAILED, which means we're out of
2077		 * chain frames.  We need to freeze the queue until we get
2078		 * a command that completed without this error, which will
2079		 * hopefully have some chain frames attached that we can
2080		 * use.  If we wanted to get smarter about it, we would
2081		 * only unfreeze the queue in this condition when we're
2082		 * sure that we're getting some chain frames back.  That's
2083		 * probably unnecessary.
2084		 */
2085		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
2086			xpt_freeze_simq(sassc->sim, 1);
2087			sassc->flags |= MPSSAS_QUEUE_FROZEN;
2088			mps_dprint(sc, MPS_INFO, "Error sending command, "
2089				   "freezing SIM queue\n");
2090		}
2091	}
2092
2093	/* Take the fast path to completion */
2094	if (cm->cm_reply == NULL) {
2095		if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INPROG) {
2096			if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0)
2097				ccb->ccb_h.status = CAM_SCSI_BUS_RESET;
2098			else {
2099				ccb->ccb_h.status = CAM_REQ_CMP;
2100				ccb->csio.scsi_status = SCSI_STATUS_OK;
2101			}
2102			if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2103				ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2104				sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2105				mps_dprint(sc, MPS_INFO,
2106					   "Unfreezing SIM queue\n");
2107			}
2108		}
2109
2110		/*
2111		 * There are two scenarios where the status won't be
2112		 * CAM_REQ_CMP.  The first is if MPS_CM_FLAGS_ERROR_MASK is
2113		 * set, the second is in the MPS_FLAGS_DIAGRESET above.
2114		 */
2115		if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2116			/*
2117			 * Freeze the dev queue so that commands are
2118			 * executed in the correct order with after error
2119			 * recovery.
2120			 */
2121			ccb->ccb_h.status |= CAM_DEV_QFRZN;
2122			xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2123		}
2124		mps_free_command(sc, cm);
2125		xpt_done(ccb);
2126		return;
2127	}
2128
2129	if (sc->mps_debug & MPS_TRACE)
2130		mpssas_log_command(cm,
2131		    "ioc %x scsi %x state %x xfer %u\n",
2132			le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2133			le32toh(rep->TransferCount));
2134
2135	/*
2136	 * If this is a Direct Drive I/O, reissue the I/O to the original IR
2137	 * Volume if an error occurred (normal I/O retry).  Use the original
2138	 * CCB, but set a flag that this will be a retry so that it's sent to
2139	 * the original volume.  Free the command but reuse the CCB.
2140	 */
2141	if (cm->cm_flags & MPS_CM_FLAGS_DD_IO) {
2142		mps_free_command(sc, cm);
2143		ccb->ccb_h.status = MPS_WD_RETRY;
2144		mpssas_action_scsiio(sassc, ccb);
2145		return;
2146	}
2147
2148	switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) {
2149	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
2150		csio->resid = cm->cm_length - le32toh(rep->TransferCount);
2151		/* FALLTHROUGH */
2152	case MPI2_IOCSTATUS_SUCCESS:
2153	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
2154
2155		if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
2156		    MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
2157			mpssas_log_command(cm, "recovered error\n");
2158
2159		/* Completion failed at the transport level. */
2160		if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS |
2161		    MPI2_SCSI_STATE_TERMINATED)) {
2162			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2163			break;
2164		}
2165
2166		/* In a modern packetized environment, an autosense failure
2167		 * implies that there's not much else that can be done to
2168		 * recover the command.
2169		 */
2170		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) {
2171			ccb->ccb_h.status = CAM_AUTOSENSE_FAIL;
2172			break;
2173		}
2174
2175		/*
2176		 * CAM doesn't care about SAS Response Info data, but if this is
2177		 * the state check if TLR should be done.  If not, clear the
2178		 * TLR_bits for the target.
2179		 */
2180		if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) &&
2181		    ((le32toh(rep->ResponseInfo) & MPI2_SCSI_RI_MASK_REASONCODE) ==
2182		    MPS_SCSI_RI_INVALID_FRAME)) {
2183			sc->mapping_table[csio->ccb_h.target_id].TLR_bits =
2184			    (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2185		}
2186
2187		/*
2188		 * Intentionally override the normal SCSI status reporting
2189		 * for these two cases.  These are likely to happen in a
2190		 * multi-initiator environment, and we want to make sure that
2191		 * CAM retries these commands rather than fail them.
2192		 */
2193		if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) ||
2194		    (rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) {
2195			ccb->ccb_h.status = CAM_REQ_ABORTED;
2196			break;
2197		}
2198
2199		/* Handle normal status and sense */
2200		csio->scsi_status = rep->SCSIStatus;
2201		if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD)
2202			ccb->ccb_h.status = CAM_REQ_CMP;
2203		else
2204			ccb->ccb_h.status = CAM_SCSI_STATUS_ERROR;
2205
2206		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2207			int sense_len, returned_sense_len;
2208
2209			returned_sense_len = min(le32toh(rep->SenseCount),
2210			    sizeof(struct scsi_sense_data));
2211			if (returned_sense_len < ccb->csio.sense_len)
2212				ccb->csio.sense_resid = ccb->csio.sense_len -
2213					returned_sense_len;
2214			else
2215				ccb->csio.sense_resid = 0;
2216
2217			sense_len = min(returned_sense_len,
2218			    ccb->csio.sense_len - ccb->csio.sense_resid);
2219			bzero(&ccb->csio.sense_data,
2220			      sizeof(ccb->csio.sense_data));
2221			bcopy(cm->cm_sense, &ccb->csio.sense_data, sense_len);
2222			ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2223		}
2224
2225		/*
2226		 * Check if this is an INQUIRY command.  If it's a VPD inquiry,
2227		 * and it's page code 0 (Supported Page List), and there is
2228		 * inquiry data, and this is for a sequential access device, and
2229		 * the device is an SSP target, and TLR is supported by the
2230		 * controller, turn the TLR_bits value ON if page 0x90 is
2231		 * supported.
2232		 */
2233		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2234		    (csio->cdb_io.cdb_bytes[1] & SI_EVPD) &&
2235		    (csio->cdb_io.cdb_bytes[2] == SVPD_SUPPORTED_PAGE_LIST) &&
2236		    (csio->data_ptr != NULL) && (((uint8_t *)cm->cm_data)[0] ==
2237		    T_SEQUENTIAL) && (sc->control_TLR) &&
2238		    (sc->mapping_table[csio->ccb_h.target_id].device_info &
2239		    MPI2_SAS_DEVICE_INFO_SSP_TARGET)) {
2240			vpd_list = (struct scsi_vpd_supported_page_list *)
2241			    csio->data_ptr;
2242			TLR_bits = &sc->mapping_table[csio->ccb_h.target_id].
2243			    TLR_bits;
2244			*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2245			TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
2246			alloc_len = ((u16)csio->cdb_io.cdb_bytes[3] << 8) +
2247			    csio->cdb_io.cdb_bytes[4];
2248			for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
2249				if (vpd_list->list[i] == 0x90) {
2250					*TLR_bits = TLR_on;
2251					break;
2252				}
2253			}
2254		}
2255		break;
2256	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
2257	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
2258		/*
2259		 * If devinfo is 0 this will be a volume.  In that case don't
2260		 * tell CAM that the volume is not there.  We want volumes to
2261		 * be enumerated until they are deleted/removed, not just
2262		 * failed.
2263		 */
2264		if (cm->cm_targ->devinfo == 0)
2265			ccb->ccb_h.status = CAM_REQ_CMP;
2266		else
2267			ccb->ccb_h.status = CAM_DEV_NOT_THERE;
2268		break;
2269	case MPI2_IOCSTATUS_INVALID_SGL:
2270		mps_print_scsiio_cmd(sc, cm);
2271		ccb->ccb_h.status = CAM_UNREC_HBA_ERROR;
2272		break;
2273	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
2274		/*
2275		 * This is one of the responses that comes back when an I/O
2276		 * has been aborted.  If it is because of a timeout that we
2277		 * initiated, just set the status to CAM_CMD_TIMEOUT.
2278		 * Otherwise set it to CAM_REQ_ABORTED.  The effect on the
2279		 * command is the same (it gets retried, subject to the
2280		 * retry counter), the only difference is what gets printed
2281		 * on the console.
2282		 */
2283		if (cm->cm_state == MPS_CM_STATE_TIMEDOUT)
2284			ccb->ccb_h.status = CAM_CMD_TIMEOUT;
2285		else
2286			ccb->ccb_h.status = CAM_REQ_ABORTED;
2287		break;
2288	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
2289		/* resid is ignored for this condition */
2290		csio->resid = 0;
2291		ccb->ccb_h.status = CAM_DATA_RUN_ERR;
2292		break;
2293	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
2294	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
2295		/*
2296		 * Since these are generally external (i.e. hopefully
2297		 * transient transport-related) errors, retry these without
2298		 * decrementing the retry count.
2299		 */
2300		ccb->ccb_h.status = CAM_REQUEUE_REQ;
2301		mpssas_log_command(cm,
2302		    "terminated ioc %x scsi %x state %x xfer %u\n",
2303			le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2304			le32toh(rep->TransferCount));
2305		break;
2306	case MPI2_IOCSTATUS_INVALID_FUNCTION:
2307	case MPI2_IOCSTATUS_INTERNAL_ERROR:
2308	case MPI2_IOCSTATUS_INVALID_VPID:
2309	case MPI2_IOCSTATUS_INVALID_FIELD:
2310	case MPI2_IOCSTATUS_INVALID_STATE:
2311	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
2312	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
2313	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
2314	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
2315	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
2316	default:
2317		mpssas_log_command(cm,
2318		    "completed ioc %x scsi %x state %x xfer %u\n",
2319			le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2320			le32toh(rep->TransferCount));
2321		csio->resid = cm->cm_length;
2322		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2323		break;
2324	}
2325
2326	mps_sc_failed_io_info(sc,csio,rep);
2327
2328	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2329		ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2330		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2331		mps_dprint(sc, MPS_INFO, "Command completed, "
2332			   "unfreezing SIM queue\n");
2333	}
2334
2335	if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
2336		ccb->ccb_h.status |= CAM_DEV_QFRZN;
2337		xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2338	}
2339
2340	mps_free_command(sc, cm);
2341	xpt_done(ccb);
2342}
2343
2344/* All Request reached here are Endian safe */
2345static void
2346mpssas_direct_drive_io(struct mpssas_softc *sassc, struct mps_command *cm,
2347    union ccb *ccb) {
2348	pMpi2SCSIIORequest_t	pIO_req;
2349	struct mps_softc	*sc = sassc->sc;
2350	uint64_t		virtLBA;
2351	uint32_t		physLBA, stripe_offset, stripe_unit;
2352	uint32_t		io_size, column;
2353	uint8_t			*ptrLBA, lba_idx, physLBA_byte, *CDB;
2354
2355	/*
2356	 * If this is a valid SCSI command (Read6, Read10, Read16, Write6,
2357	 * Write10, or Write16), build a direct I/O message.  Otherwise, the I/O
2358	 * will be sent to the IR volume itself.  Since Read6 and Write6 are a
2359	 * bit different than the 10/16 CDBs, handle them separately.
2360	 */
2361	pIO_req = (pMpi2SCSIIORequest_t)cm->cm_req;
2362	CDB = pIO_req->CDB.CDB32;
2363
2364	/*
2365	 * Handle 6 byte CDBs.
2366	 */
2367	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_6) ||
2368	    (CDB[0] == WRITE_6))) {
2369		/*
2370		 * Get the transfer size in blocks.
2371		 */
2372		io_size = (cm->cm_length >> sc->DD_block_exponent);
2373
2374		/*
2375		 * Get virtual LBA given in the CDB.
2376		 */
2377		virtLBA = ((uint64_t)(CDB[1] & 0x1F) << 16) |
2378		    ((uint64_t)CDB[2] << 8) | (uint64_t)CDB[3];
2379
2380		/*
2381		 * Check that LBA range for I/O does not exceed volume's
2382		 * MaxLBA.
2383		 */
2384		if ((virtLBA + (uint64_t)io_size - 1) <=
2385		    sc->DD_max_lba) {
2386			/*
2387			 * Check if the I/O crosses a stripe boundary.  If not,
2388			 * translate the virtual LBA to a physical LBA and set
2389			 * the DevHandle for the PhysDisk to be used.  If it
2390			 * does cross a boundry, do normal I/O.  To get the
2391			 * right DevHandle to use, get the map number for the
2392			 * column, then use that map number to look up the
2393			 * DevHandle of the PhysDisk.
2394			 */
2395			stripe_offset = (uint32_t)virtLBA &
2396			    (sc->DD_stripe_size - 1);
2397			if ((stripe_offset + io_size) <= sc->DD_stripe_size) {
2398				physLBA = (uint32_t)virtLBA >>
2399				    sc->DD_stripe_exponent;
2400				stripe_unit = physLBA / sc->DD_num_phys_disks;
2401				column = physLBA % sc->DD_num_phys_disks;
2402				pIO_req->DevHandle =
2403				    htole16(sc->DD_column_map[column].dev_handle);
2404				/* ???? Is this endian safe*/
2405				cm->cm_desc.SCSIIO.DevHandle =
2406				    pIO_req->DevHandle;
2407
2408				physLBA = (stripe_unit <<
2409				    sc->DD_stripe_exponent) + stripe_offset;
2410				ptrLBA = &pIO_req->CDB.CDB32[1];
2411				physLBA_byte = (uint8_t)(physLBA >> 16);
2412				*ptrLBA = physLBA_byte;
2413				ptrLBA = &pIO_req->CDB.CDB32[2];
2414				physLBA_byte = (uint8_t)(physLBA >> 8);
2415				*ptrLBA = physLBA_byte;
2416				ptrLBA = &pIO_req->CDB.CDB32[3];
2417				physLBA_byte = (uint8_t)physLBA;
2418				*ptrLBA = physLBA_byte;
2419
2420				/*
2421				 * Set flag that Direct Drive I/O is
2422				 * being done.
2423				 */
2424				cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2425			}
2426		}
2427		return;
2428	}
2429
2430	/*
2431	 * Handle 10, 12 or 16 byte CDBs.
2432	 */
2433	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_10) ||
2434	    (CDB[0] == WRITE_10) || (CDB[0] == READ_16) ||
2435	    (CDB[0] == WRITE_16) || (CDB[0] == READ_12) ||
2436	    (CDB[0] == WRITE_12))) {
2437		/*
2438		 * For 16-byte CDB's, verify that the upper 4 bytes of the CDB
2439		 * are 0.  If not, this is accessing beyond 2TB so handle it in
2440		 * the else section.  10-byte and 12-byte CDB's are OK.
2441		 * FreeBSD sends very rare 12 byte READ/WRITE, but driver is
2442		 * ready to accept 12byte CDB for Direct IOs.
2443		 */
2444		if ((CDB[0] == READ_10 || CDB[0] == WRITE_10) ||
2445		    (CDB[0] == READ_12 || CDB[0] == WRITE_12) ||
2446		    !(CDB[2] | CDB[3] | CDB[4] | CDB[5])) {
2447			/*
2448			 * Get the transfer size in blocks.
2449			 */
2450			io_size = (cm->cm_length >> sc->DD_block_exponent);
2451
2452			/*
2453			 * Get virtual LBA.  Point to correct lower 4 bytes of
2454			 * LBA in the CDB depending on command.
2455			 */
2456			lba_idx = ((CDB[0] == READ_12) ||
2457				(CDB[0] == WRITE_12) ||
2458				(CDB[0] == READ_10) ||
2459				(CDB[0] == WRITE_10))? 2 : 6;
2460			virtLBA = ((uint64_t)CDB[lba_idx] << 24) |
2461			    ((uint64_t)CDB[lba_idx + 1] << 16) |
2462			    ((uint64_t)CDB[lba_idx + 2] << 8) |
2463			    (uint64_t)CDB[lba_idx + 3];
2464
2465			/*
2466			 * Check that LBA range for I/O does not exceed volume's
2467			 * MaxLBA.
2468			 */
2469			if ((virtLBA + (uint64_t)io_size - 1) <=
2470			    sc->DD_max_lba) {
2471				/*
2472				 * Check if the I/O crosses a stripe boundary.
2473				 * If not, translate the virtual LBA to a
2474				 * physical LBA and set the DevHandle for the
2475				 * PhysDisk to be used.  If it does cross a
2476				 * boundry, do normal I/O.  To get the right
2477				 * DevHandle to use, get the map number for the
2478				 * column, then use that map number to look up
2479				 * the DevHandle of the PhysDisk.
2480				 */
2481				stripe_offset = (uint32_t)virtLBA &
2482				    (sc->DD_stripe_size - 1);
2483				if ((stripe_offset + io_size) <=
2484				    sc->DD_stripe_size) {
2485					physLBA = (uint32_t)virtLBA >>
2486					    sc->DD_stripe_exponent;
2487					stripe_unit = physLBA /
2488					    sc->DD_num_phys_disks;
2489					column = physLBA %
2490					    sc->DD_num_phys_disks;
2491					pIO_req->DevHandle =
2492					    htole16(sc->DD_column_map[column].
2493					    dev_handle);
2494					cm->cm_desc.SCSIIO.DevHandle =
2495					    pIO_req->DevHandle;
2496
2497					physLBA = (stripe_unit <<
2498					    sc->DD_stripe_exponent) +
2499					    stripe_offset;
2500					ptrLBA =
2501					    &pIO_req->CDB.CDB32[lba_idx];
2502					physLBA_byte = (uint8_t)(physLBA >> 24);
2503					*ptrLBA = physLBA_byte;
2504					ptrLBA =
2505					    &pIO_req->CDB.CDB32[lba_idx + 1];
2506					physLBA_byte = (uint8_t)(physLBA >> 16);
2507					*ptrLBA = physLBA_byte;
2508					ptrLBA =
2509					    &pIO_req->CDB.CDB32[lba_idx + 2];
2510					physLBA_byte = (uint8_t)(physLBA >> 8);
2511					*ptrLBA = physLBA_byte;
2512					ptrLBA =
2513					    &pIO_req->CDB.CDB32[lba_idx + 3];
2514					physLBA_byte = (uint8_t)physLBA;
2515					*ptrLBA = physLBA_byte;
2516
2517					/*
2518					 * Set flag that Direct Drive I/O is
2519					 * being done.
2520					 */
2521					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2522				}
2523			}
2524		} else {
2525			/*
2526			 * 16-byte CDB and the upper 4 bytes of the CDB are not
2527			 * 0.  Get the transfer size in blocks.
2528			 */
2529			io_size = (cm->cm_length >> sc->DD_block_exponent);
2530
2531			/*
2532			 * Get virtual LBA.
2533			 */
2534			virtLBA = ((uint64_t)CDB[2] << 54) |
2535			    ((uint64_t)CDB[3] << 48) |
2536			    ((uint64_t)CDB[4] << 40) |
2537			    ((uint64_t)CDB[5] << 32) |
2538			    ((uint64_t)CDB[6] << 24) |
2539			    ((uint64_t)CDB[7] << 16) |
2540			    ((uint64_t)CDB[8] << 8) |
2541			    (uint64_t)CDB[9];
2542
2543			/*
2544			 * Check that LBA range for I/O does not exceed volume's
2545			 * MaxLBA.
2546			 */
2547			if ((virtLBA + (uint64_t)io_size - 1) <=
2548			    sc->DD_max_lba) {
2549				/*
2550				 * Check if the I/O crosses a stripe boundary.
2551				 * If not, translate the virtual LBA to a
2552				 * physical LBA and set the DevHandle for the
2553				 * PhysDisk to be used.  If it does cross a
2554				 * boundry, do normal I/O.  To get the right
2555				 * DevHandle to use, get the map number for the
2556				 * column, then use that map number to look up
2557				 * the DevHandle of the PhysDisk.
2558				 */
2559				stripe_offset = (uint32_t)virtLBA &
2560				    (sc->DD_stripe_size - 1);
2561				if ((stripe_offset + io_size) <=
2562				    sc->DD_stripe_size) {
2563					physLBA = (uint32_t)(virtLBA >>
2564					    sc->DD_stripe_exponent);
2565					stripe_unit = physLBA /
2566					    sc->DD_num_phys_disks;
2567					column = physLBA %
2568					    sc->DD_num_phys_disks;
2569					pIO_req->DevHandle =
2570					    htole16(sc->DD_column_map[column].
2571					    dev_handle);
2572					cm->cm_desc.SCSIIO.DevHandle =
2573					    pIO_req->DevHandle;
2574
2575					physLBA = (stripe_unit <<
2576					    sc->DD_stripe_exponent) +
2577					    stripe_offset;
2578
2579					/*
2580					 * Set upper 4 bytes of LBA to 0.  We
2581					 * assume that the phys disks are less
2582					 * than 2 TB's in size.  Then, set the
2583					 * lower 4 bytes.
2584					 */
2585					pIO_req->CDB.CDB32[2] = 0;
2586					pIO_req->CDB.CDB32[3] = 0;
2587					pIO_req->CDB.CDB32[4] = 0;
2588					pIO_req->CDB.CDB32[5] = 0;
2589					ptrLBA = &pIO_req->CDB.CDB32[6];
2590					physLBA_byte = (uint8_t)(physLBA >> 24);
2591					*ptrLBA = physLBA_byte;
2592					ptrLBA = &pIO_req->CDB.CDB32[7];
2593					physLBA_byte = (uint8_t)(physLBA >> 16);
2594					*ptrLBA = physLBA_byte;
2595					ptrLBA = &pIO_req->CDB.CDB32[8];
2596					physLBA_byte = (uint8_t)(physLBA >> 8);
2597					*ptrLBA = physLBA_byte;
2598					ptrLBA = &pIO_req->CDB.CDB32[9];
2599					physLBA_byte = (uint8_t)physLBA;
2600					*ptrLBA = physLBA_byte;
2601
2602					/*
2603					 * Set flag that Direct Drive I/O is
2604					 * being done.
2605					 */
2606					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2607				}
2608			}
2609		}
2610	}
2611}
2612
2613#if __FreeBSD_version >= 900026
2614static void
2615mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm)
2616{
2617	MPI2_SMP_PASSTHROUGH_REPLY *rpl;
2618	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2619	uint64_t sasaddr;
2620	union ccb *ccb;
2621
2622	ccb = cm->cm_complete_data;
2623
2624	/*
2625	 * Currently there should be no way we can hit this case.  It only
2626	 * happens when we have a failure to allocate chain frames, and SMP
2627	 * commands require two S/G elements only.  That should be handled
2628	 * in the standard request size.
2629	 */
2630	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2631		mps_printf(sc, "%s: cm_flags = %#x on SMP request!\n",
2632			   __func__, cm->cm_flags);
2633		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2634		goto bailout;
2635        }
2636
2637	rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply;
2638	if (rpl == NULL) {
2639		mps_dprint(sc, MPS_INFO, "%s: NULL cm_reply!\n", __func__);
2640		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2641		goto bailout;
2642	}
2643
2644	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2645	sasaddr = le32toh(req->SASAddress.Low);
2646	sasaddr |= ((uint64_t)(le32toh(req->SASAddress.High))) << 32;
2647
2648	if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) != MPI2_IOCSTATUS_SUCCESS ||
2649	    rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) {
2650		mps_dprint(sc, MPS_INFO, "%s: IOCStatus %04x SASStatus %02x\n",
2651		    __func__, le16toh(rpl->IOCStatus), rpl->SASStatus);
2652		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
2653		goto bailout;
2654	}
2655
2656	mps_dprint(sc, MPS_INFO, "%s: SMP request to SAS address "
2657		   "%#jx completed successfully\n", __func__,
2658		   (uintmax_t)sasaddr);
2659
2660	if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED)
2661		ccb->ccb_h.status = CAM_REQ_CMP;
2662	else
2663		ccb->ccb_h.status = CAM_SMP_STATUS_ERROR;
2664
2665bailout:
2666	/*
2667	 * We sync in both directions because we had DMAs in the S/G list
2668	 * in both directions.
2669	 */
2670	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
2671			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2672	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2673	mps_free_command(sc, cm);
2674	xpt_done(ccb);
2675}
2676
2677static void
2678mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb, uint64_t sasaddr)
2679{
2680	struct mps_command *cm;
2681	uint8_t *request, *response;
2682	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2683	struct mps_softc *sc;
2684	struct sglist *sg;
2685	int error;
2686
2687	sc = sassc->sc;
2688	sg = NULL;
2689	error = 0;
2690
2691	/*
2692	 * XXX We don't yet support physical addresses here.
2693	 */
2694	if (ccb->ccb_h.flags & (CAM_DATA_PHYS|CAM_SG_LIST_PHYS)) {
2695		mps_printf(sc, "%s: physical addresses not supported\n",
2696			   __func__);
2697		ccb->ccb_h.status = CAM_REQ_INVALID;
2698		xpt_done(ccb);
2699		return;
2700	}
2701
2702	/*
2703	 * If the user wants to send an S/G list, check to make sure they
2704	 * have single buffers.
2705	 */
2706	if (ccb->ccb_h.flags & CAM_SCATTER_VALID) {
2707		/*
2708		 * The chip does not support more than one buffer for the
2709		 * request or response.
2710		 */
2711	 	if ((ccb->smpio.smp_request_sglist_cnt > 1)
2712		  || (ccb->smpio.smp_response_sglist_cnt > 1)) {
2713			mps_printf(sc, "%s: multiple request or response "
2714				   "buffer segments not supported for SMP\n",
2715				   __func__);
2716			ccb->ccb_h.status = CAM_REQ_INVALID;
2717			xpt_done(ccb);
2718			return;
2719		}
2720
2721		/*
2722		 * The CAM_SCATTER_VALID flag was originally implemented
2723		 * for the XPT_SCSI_IO CCB, which only has one data pointer.
2724		 * We have two.  So, just take that flag to mean that we
2725		 * might have S/G lists, and look at the S/G segment count
2726		 * to figure out whether that is the case for each individual
2727		 * buffer.
2728		 */
2729		if (ccb->smpio.smp_request_sglist_cnt != 0) {
2730			bus_dma_segment_t *req_sg;
2731
2732			req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request;
2733			request = (uint8_t *)req_sg[0].ds_addr;
2734		} else
2735			request = ccb->smpio.smp_request;
2736
2737		if (ccb->smpio.smp_response_sglist_cnt != 0) {
2738			bus_dma_segment_t *rsp_sg;
2739
2740			rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response;
2741			response = (uint8_t *)rsp_sg[0].ds_addr;
2742		} else
2743			response = ccb->smpio.smp_response;
2744	} else {
2745		request = ccb->smpio.smp_request;
2746		response = ccb->smpio.smp_response;
2747	}
2748
2749	cm = mps_alloc_command(sc);
2750	if (cm == NULL) {
2751		mps_printf(sc, "%s: cannot allocate command\n", __func__);
2752		ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2753		xpt_done(ccb);
2754		return;
2755	}
2756
2757	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2758	bzero(req, sizeof(*req));
2759	req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH;
2760
2761	/* Allow the chip to use any route to this SAS address. */
2762	req->PhysicalPort = 0xff;
2763
2764	req->RequestDataLength = htole16(ccb->smpio.smp_request_len);
2765	req->SGLFlags =
2766	    MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE | MPI2_SGLFLAGS_SGL_TYPE_MPI;
2767
2768	mps_dprint(sc, MPS_INFO, "%s: sending SMP request to SAS "
2769		   "address %#jx\n", __func__, (uintmax_t)sasaddr);
2770
2771	mpi_init_sge(cm, req, &req->SGL);
2772
2773	/*
2774	 * Set up a uio to pass into mps_map_command().  This allows us to
2775	 * do one map command, and one busdma call in there.
2776	 */
2777	cm->cm_uio.uio_iov = cm->cm_iovec;
2778	cm->cm_uio.uio_iovcnt = 2;
2779	cm->cm_uio.uio_segflg = UIO_SYSSPACE;
2780
2781	/*
2782	 * The read/write flag isn't used by busdma, but set it just in
2783	 * case.  This isn't exactly accurate, either, since we're going in
2784	 * both directions.
2785	 */
2786	cm->cm_uio.uio_rw = UIO_WRITE;
2787
2788	cm->cm_iovec[0].iov_base = request;
2789	cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength);
2790	cm->cm_iovec[1].iov_base = response;
2791	cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len;
2792
2793	cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len +
2794			       cm->cm_iovec[1].iov_len;
2795
2796	/*
2797	 * Trigger a warning message in mps_data_cb() for the user if we
2798	 * wind up exceeding two S/G segments.  The chip expects one
2799	 * segment for the request and another for the response.
2800	 */
2801	cm->cm_max_segs = 2;
2802
2803	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
2804	cm->cm_complete = mpssas_smpio_complete;
2805	cm->cm_complete_data = ccb;
2806
2807	/*
2808	 * Tell the mapping code that we're using a uio, and that this is
2809	 * an SMP passthrough request.  There is a little special-case
2810	 * logic there (in mps_data_cb()) to handle the bidirectional
2811	 * transfer.
2812	 */
2813	cm->cm_flags |= MPS_CM_FLAGS_USE_UIO | MPS_CM_FLAGS_SMP_PASS |
2814			MPS_CM_FLAGS_DATAIN | MPS_CM_FLAGS_DATAOUT;
2815
2816	/* The chip data format is little endian. */
2817	req->SASAddress.High = htole32(sasaddr >> 32);
2818	req->SASAddress.Low = htole32(sasaddr);
2819
2820	/*
2821	 * XXX Note that we don't have a timeout/abort mechanism here.
2822	 * From the manual, it looks like task management requests only
2823	 * work for SCSI IO and SATA passthrough requests.  We may need to
2824	 * have a mechanism to retry requests in the event of a chip reset
2825	 * at least.  Hopefully the chip will insure that any errors short
2826	 * of that are relayed back to the driver.
2827	 */
2828	error = mps_map_command(sc, cm);
2829	if ((error != 0) && (error != EINPROGRESS)) {
2830		mps_printf(sc, "%s: error %d returned from mps_map_command()\n",
2831			   __func__, error);
2832		goto bailout_error;
2833	}
2834
2835	return;
2836
2837bailout_error:
2838	mps_free_command(sc, cm);
2839	ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2840	xpt_done(ccb);
2841	return;
2842
2843}
2844
2845static void
2846mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb)
2847{
2848	struct mps_softc *sc;
2849	struct mpssas_target *targ;
2850	uint64_t sasaddr = 0;
2851
2852	sc = sassc->sc;
2853
2854	/*
2855	 * Make sure the target exists.
2856	 */
2857	targ = &sassc->targets[ccb->ccb_h.target_id];
2858	if (targ->handle == 0x0) {
2859		mps_printf(sc, "%s: target %d does not exist!\n", __func__,
2860			   ccb->ccb_h.target_id);
2861		ccb->ccb_h.status = CAM_SEL_TIMEOUT;
2862		xpt_done(ccb);
2863		return;
2864	}
2865
2866	/*
2867	 * If this device has an embedded SMP target, we'll talk to it
2868	 * directly.
2869	 * figure out what the expander's address is.
2870	 */
2871	if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0)
2872		sasaddr = targ->sasaddr;
2873
2874	/*
2875	 * If we don't have a SAS address for the expander yet, try
2876	 * grabbing it from the page 0x83 information cached in the
2877	 * transport layer for this target.  LSI expanders report the
2878	 * expander SAS address as the port-associated SAS address in
2879	 * Inquiry VPD page 0x83.  Maxim expanders don't report it in page
2880	 * 0x83.
2881	 *
2882	 * XXX KDM disable this for now, but leave it commented out so that
2883	 * it is obvious that this is another possible way to get the SAS
2884	 * address.
2885	 *
2886	 * The parent handle method below is a little more reliable, and
2887	 * the other benefit is that it works for devices other than SES
2888	 * devices.  So you can send a SMP request to a da(4) device and it
2889	 * will get routed to the expander that device is attached to.
2890	 * (Assuming the da(4) device doesn't contain an SMP target...)
2891	 */
2892#if 0
2893	if (sasaddr == 0)
2894		sasaddr = xpt_path_sas_addr(ccb->ccb_h.path);
2895#endif
2896
2897	/*
2898	 * If we still don't have a SAS address for the expander, look for
2899	 * the parent device of this device, which is probably the expander.
2900	 */
2901	if (sasaddr == 0) {
2902#ifdef OLD_MPS_PROBE
2903		struct mpssas_target *parent_target;
2904#endif
2905
2906		if (targ->parent_handle == 0x0) {
2907			mps_printf(sc, "%s: handle %d does not have a valid "
2908				   "parent handle!\n", __func__, targ->handle);
2909			ccb->ccb_h.status = CAM_REQ_INVALID;
2910			goto bailout;
2911		}
2912#ifdef OLD_MPS_PROBE
2913		parent_target = mpssas_find_target_by_handle(sassc, 0,
2914			targ->parent_handle);
2915
2916		if (parent_target == NULL) {
2917			mps_printf(sc, "%s: handle %d does not have a valid "
2918				   "parent target!\n", __func__, targ->handle);
2919			ccb->ccb_h.status = CAM_REQ_INVALID;
2920			goto bailout;
2921		}
2922
2923		if ((parent_target->devinfo &
2924		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
2925			mps_printf(sc, "%s: handle %d parent %d does not "
2926				   "have an SMP target!\n", __func__,
2927				   targ->handle, parent_target->handle);
2928			ccb->ccb_h.status = CAM_REQ_INVALID;
2929			goto bailout;
2930
2931		}
2932
2933		sasaddr = parent_target->sasaddr;
2934#else /* OLD_MPS_PROBE */
2935		if ((targ->parent_devinfo &
2936		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
2937			mps_printf(sc, "%s: handle %d parent %d does not "
2938				   "have an SMP target!\n", __func__,
2939				   targ->handle, targ->parent_handle);
2940			ccb->ccb_h.status = CAM_REQ_INVALID;
2941			goto bailout;
2942
2943		}
2944		if (targ->parent_sasaddr == 0x0) {
2945			mps_printf(sc, "%s: handle %d parent handle %d does "
2946				   "not have a valid SAS address!\n",
2947				   __func__, targ->handle, targ->parent_handle);
2948			ccb->ccb_h.status = CAM_REQ_INVALID;
2949			goto bailout;
2950		}
2951
2952		sasaddr = targ->parent_sasaddr;
2953#endif /* OLD_MPS_PROBE */
2954
2955	}
2956
2957	if (sasaddr == 0) {
2958		mps_printf(sc, "%s: unable to find SAS address for handle %d\n",
2959			   __func__, targ->handle);
2960		ccb->ccb_h.status = CAM_REQ_INVALID;
2961		goto bailout;
2962	}
2963	mpssas_send_smpcmd(sassc, ccb, sasaddr);
2964
2965	return;
2966
2967bailout:
2968	xpt_done(ccb);
2969
2970}
2971#endif //__FreeBSD_version >= 900026
2972
2973static void
2974mpssas_action_resetdev(struct mpssas_softc *sassc, union ccb *ccb)
2975{
2976	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
2977	struct mps_softc *sc;
2978	struct mps_command *tm;
2979	struct mpssas_target *targ;
2980
2981	mps_dprint(sassc->sc, MPS_TRACE, __func__);
2982	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
2983
2984	sc = sassc->sc;
2985	tm = mps_alloc_command(sc);
2986	if (tm == NULL) {
2987		mps_printf(sc, "comand alloc failure in mpssas_action_resetdev\n");
2988		ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
2989		xpt_done(ccb);
2990		return;
2991	}
2992
2993	targ = &sassc->targets[ccb->ccb_h.target_id];
2994	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
2995	req->DevHandle = htole16(targ->handle);
2996	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
2997	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
2998
2999	/* SAS Hard Link Reset / SATA Link Reset */
3000	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3001
3002	tm->cm_data = NULL;
3003	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
3004	tm->cm_complete = mpssas_resetdev_complete;
3005	tm->cm_complete_data = ccb;
3006	mps_map_command(sc, tm);
3007}
3008
3009static void
3010mpssas_resetdev_complete(struct mps_softc *sc, struct mps_command *tm)
3011{
3012	MPI2_SCSI_TASK_MANAGE_REPLY *resp;
3013	union ccb *ccb;
3014
3015	mps_dprint(sc, MPS_TRACE, __func__);
3016	mtx_assert(&sc->mps_mtx, MA_OWNED);
3017
3018	resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
3019	ccb = tm->cm_complete_data;
3020
3021	/*
3022	 * Currently there should be no way we can hit this case.  It only
3023	 * happens when we have a failure to allocate chain frames, and
3024	 * task management commands don't have S/G lists.
3025	 */
3026	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3027		MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3028
3029		req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3030
3031		mps_printf(sc, "%s: cm_flags = %#x for reset of handle %#04x! "
3032			   "This should not happen!\n", __func__, tm->cm_flags,
3033			   req->DevHandle);
3034		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
3035		goto bailout;
3036	}
3037
3038	printf("%s: IOCStatus = 0x%x ResponseCode = 0x%x\n", __func__,
3039	    le16toh(resp->IOCStatus), le32toh(resp->ResponseCode));
3040
3041	if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) {
3042		ccb->ccb_h.status = CAM_REQ_CMP;
3043		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
3044		    CAM_LUN_WILDCARD);
3045	}
3046	else
3047		ccb->ccb_h.status = CAM_REQ_CMP_ERR;
3048
3049bailout:
3050
3051	mpssas_free_tm(sc, tm);
3052	xpt_done(ccb);
3053}
3054
3055static void
3056mpssas_poll(struct cam_sim *sim)
3057{
3058	struct mpssas_softc *sassc;
3059
3060	sassc = cam_sim_softc(sim);
3061
3062	if (sassc->sc->mps_debug & MPS_TRACE) {
3063		/* frequent debug messages during a panic just slow
3064		 * everything down too much.
3065		 */
3066		mps_printf(sassc->sc, "%s clearing MPS_TRACE\n", __func__);
3067		sassc->sc->mps_debug &= ~MPS_TRACE;
3068	}
3069
3070	mps_intr_locked(sassc->sc);
3071}
3072
3073static void
3074mpssas_rescan_done(struct cam_periph *periph, union ccb *done_ccb)
3075{
3076	struct mpssas_softc *sassc;
3077	char path_str[64];
3078
3079	if (done_ccb == NULL)
3080		return;
3081
3082	sassc = (struct mpssas_softc *)done_ccb->ccb_h.ppriv_ptr1;
3083
3084	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
3085
3086	xpt_path_string(done_ccb->ccb_h.path, path_str, sizeof(path_str));
3087	mps_dprint(sassc->sc, MPS_INFO, "Completing rescan for %s\n", path_str);
3088
3089	xpt_free_path(done_ccb->ccb_h.path);
3090	xpt_free_ccb(done_ccb);
3091
3092#if __FreeBSD_version < 1000006
3093	/*
3094	 * Before completing scan, get EEDP stuff for all of the existing
3095	 * targets.
3096	 */
3097	mpssas_check_eedp(sassc);
3098#endif
3099
3100}
3101
3102/* thread to handle bus rescans */
3103static void
3104mpssas_scanner_thread(void *arg)
3105{
3106	struct mpssas_softc *sassc;
3107	struct mps_softc *sc;
3108	union ccb	*ccb;
3109
3110	sassc = (struct mpssas_softc *)arg;
3111	sc = sassc->sc;
3112
3113	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
3114
3115	mps_lock(sc);
3116	for (;;) {
3117		/* Sleep for 1 second and check the queue status*/
3118		msleep(&sassc->ccb_scanq, &sc->mps_mtx, PRIBIO,
3119		       "mps_scanq", 1 * hz);
3120		if (sassc->flags & MPSSAS_SHUTDOWN) {
3121			mps_dprint(sc, MPS_TRACE, "Scanner shutting down\n");
3122			break;
3123		}
3124next_work:
3125		// Get first work.
3126		ccb = (union ccb *)TAILQ_FIRST(&sassc->ccb_scanq);
3127		if (ccb == NULL)
3128			continue;
3129		// Got first work.
3130		TAILQ_REMOVE(&sassc->ccb_scanq, &ccb->ccb_h, sim_links.tqe);
3131		xpt_action(ccb);
3132		if (sassc->flags & MPSSAS_SHUTDOWN) {
3133			mps_dprint(sc, MPS_TRACE, "Scanner shutting down\n");
3134			break;
3135		}
3136		goto next_work;
3137	}
3138
3139	sassc->flags &= ~MPSSAS_SCANTHREAD;
3140	wakeup(&sassc->flags);
3141	mps_unlock(sc);
3142	mps_dprint(sc, MPS_TRACE, "Scanner exiting\n");
3143	mps_kproc_exit(0);
3144}
3145
3146/*
3147 * This function will send READ_CAP_16 to find out EEDP protection mode.
3148 * It will check inquiry data before sending READ_CAP_16.
3149 * Callback for READ_CAP_16 is "mpssas_read_cap_done".
3150 * This is insternal scsi command and we need to take care release of devq, if
3151 * CAM_DEV_QFRZN is set. Driver needs to release devq if it has frozen any.
3152 * xpt_release_devq is called from mpssas_read_cap_done.
3153 *
3154 * All other commands will be handled by periph layer and there it will
3155 * check for "CAM_DEV_QFRZN" and release of devq will be done.
3156 */
3157static void
3158mpssas_rescan(struct mpssas_softc *sassc, union ccb *ccb)
3159{
3160	char path_str[64];
3161
3162	mps_dprint(sassc->sc, MPS_TRACE, "%s\n", __func__);
3163
3164	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
3165
3166	if (ccb == NULL)
3167		return;
3168
3169	xpt_path_string(ccb->ccb_h.path, path_str, sizeof(path_str));
3170	mps_dprint(sassc->sc, MPS_INFO, "Queueing rescan for %s\n", path_str);
3171
3172	/* Prepare request */
3173	ccb->ccb_h.ppriv_ptr1 = sassc;
3174	ccb->ccb_h.cbfcnp = mpssas_rescan_done;
3175	xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, MPS_PRIORITY_XPT);
3176	TAILQ_INSERT_TAIL(&sassc->ccb_scanq, &ccb->ccb_h, sim_links.tqe);
3177	wakeup(&sassc->ccb_scanq);
3178}
3179
3180#if __FreeBSD_version >= 1000006
3181static void
3182mpssas_async(void *callback_arg, uint32_t code, struct cam_path *path,
3183	     void *arg)
3184{
3185	struct mps_softc *sc;
3186
3187	sc = (struct mps_softc *)callback_arg;
3188
3189	switch (code) {
3190	case AC_ADVINFO_CHANGED: {
3191		struct mpssas_target *target;
3192		struct mpssas_softc *sassc;
3193		struct scsi_read_capacity_data_long rcap_buf;
3194		struct ccb_dev_advinfo cdai;
3195		struct mpssas_lun *lun;
3196		lun_id_t lunid;
3197		int found_lun;
3198		uintptr_t buftype;
3199
3200		buftype = (uintptr_t)arg;
3201
3202		found_lun = 0;
3203		sassc = sc->sassc;
3204
3205		/*
3206		 * We're only interested in read capacity data changes.
3207		 */
3208		if (buftype != CDAI_TYPE_RCAPLONG)
3209			break;
3210
3211		/*
3212		 * We're only interested in devices that are attached to
3213		 * this controller.
3214		 */
3215		if (xpt_path_path_id(path) != sassc->sim->path_id)
3216			break;
3217
3218		/*
3219		 * We should have a handle for this, but check to make sure.
3220		 */
3221		target = &sassc->targets[xpt_path_target_id(path)];
3222		if (target->handle == 0)
3223			break;
3224
3225		lunid = xpt_path_lun_id(path);
3226
3227		SLIST_FOREACH(lun, &target->luns, lun_link) {
3228			if (lun->lun_id == lunid) {
3229				found_lun = 1;
3230				break;
3231			}
3232		}
3233
3234		if (found_lun == 0) {
3235			lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3236				     M_NOWAIT | M_ZERO);
3237			if (lun == NULL) {
3238				mps_dprint(sc, MPS_FAULT, "Unable to alloc "
3239					   "LUN for EEDP support.\n");
3240				break;
3241			}
3242			lun->lun_id = lunid;
3243			SLIST_INSERT_HEAD(&target->luns, lun, lun_link);
3244		}
3245
3246		bzero(&rcap_buf, sizeof(rcap_buf));
3247		xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
3248		cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
3249		cdai.ccb_h.flags = CAM_DIR_IN;
3250		cdai.buftype = CDAI_TYPE_RCAPLONG;
3251		cdai.flags = 0;
3252		cdai.bufsiz = sizeof(rcap_buf);
3253		cdai.buf = (uint8_t *)&rcap_buf;
3254		xpt_action((union ccb *)&cdai);
3255		if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
3256			cam_release_devq(cdai.ccb_h.path,
3257					 0, 0, 0, FALSE);
3258
3259		if (((cdai.ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP)
3260		 && (rcap_buf.prot & SRC16_PROT_EN)) {
3261			lun->eedp_formatted = TRUE;
3262			lun->eedp_block_size = scsi_4btoul(rcap_buf.length);
3263		} else {
3264			lun->eedp_formatted = FALSE;
3265			lun->eedp_block_size = 0;
3266		}
3267		break;
3268	}
3269	default:
3270		break;
3271	}
3272}
3273#else /* __FreeBSD_version >= 1000006 */
3274
3275static void
3276mpssas_check_eedp(struct mpssas_softc *sassc)
3277{
3278	struct mps_softc *sc = sassc->sc;
3279	struct ccb_scsiio *csio;
3280	struct scsi_read_capacity_16 *scsi_cmd;
3281	struct scsi_read_capacity_eedp *rcap_buf;
3282	union ccb *ccb;
3283	path_id_t pathid = cam_sim_path(sassc->sim);
3284	target_id_t targetid;
3285	lun_id_t lunid;
3286	struct cam_periph *found_periph;
3287	struct mpssas_target *target;
3288	struct mpssas_lun *lun;
3289	uint8_t	found_lun;
3290	struct ccb_getdev cgd;
3291	char path_str[64];
3292
3293	/*
3294	 * Issue a READ CAPACITY 16 command to each LUN of each target.  This
3295	 * info is used to determine if the LUN is formatted for EEDP support.
3296	 */
3297	for (targetid = 0; targetid < sc->facts->MaxTargets; targetid++) {
3298		target = &sassc->targets[targetid];
3299		if (target->handle == 0x0) {
3300			continue;
3301		}
3302
3303		lunid = 0;
3304		do {
3305			ccb = xpt_alloc_ccb_nowait();
3306			if (ccb == NULL) {
3307				mps_dprint(sc, MPS_FAULT, "Unable to alloc CCB "
3308				    "for EEDP support.\n");
3309				return;
3310			}
3311
3312			if (xpt_create_path(&ccb->ccb_h.path, xpt_periph,
3313			    pathid, targetid, lunid) != CAM_REQ_CMP) {
3314				mps_dprint(sc, MPS_FAULT, "Unable to create "
3315				    "path for EEDP support\n");
3316				xpt_free_ccb(ccb);
3317				return;
3318			}
3319
3320			/*
3321			 * If a periph is returned, the LUN exists.  Create an
3322			 * entry in the target's LUN list.
3323			 */
3324			if ((found_periph = cam_periph_find(ccb->ccb_h.path,
3325			    NULL)) != NULL) {
3326				/*
3327				 * If LUN is already in list, don't create a new
3328				 * one.
3329				 */
3330				found_lun = FALSE;
3331				SLIST_FOREACH(lun, &target->luns, lun_link) {
3332					if (lun->lun_id == lunid) {
3333						found_lun = TRUE;
3334						break;
3335					}
3336				}
3337				if (!found_lun) {
3338					lun = malloc(sizeof(struct mpssas_lun),
3339					    M_MPT2, M_NOWAIT | M_ZERO);
3340					if (lun == NULL) {
3341						mps_dprint(sc, MPS_FAULT,
3342						    "Unable to alloc LUN for "
3343						    "EEDP support.\n");
3344						xpt_free_path(ccb->ccb_h.path);
3345						xpt_free_ccb(ccb);
3346						return;
3347					}
3348					lun->lun_id = lunid;
3349					SLIST_INSERT_HEAD(&target->luns, lun,
3350					    lun_link);
3351				}
3352				lunid++;
3353				/* Before Issuing READ CAPACITY 16,
3354				 * check Device type.
3355 				 */
3356				xpt_setup_ccb(&cgd.ccb_h, ccb->ccb_h.path,
3357					CAM_PRIORITY_NORMAL);
3358				cgd.ccb_h.func_code = XPT_GDEV_TYPE;
3359				xpt_action((union ccb *)&cgd);
3360
3361				/*
3362				 * If this flag is set in the inquiry data,
3363				 * the device supports protection information,
3364				 * and must support the 16 byte read
3365				 * capacity command, otherwise continue without
3366				 * sending read cap 16
3367				 */
3368
3369				xpt_path_string(ccb->ccb_h.path, path_str,
3370					sizeof(path_str));
3371
3372				if ((cgd.inq_data.spc3_flags &
3373				SPC3_SID_PROTECT) == 0) {
3374					xpt_free_path(ccb->ccb_h.path);
3375					xpt_free_ccb(ccb);
3376					continue;
3377				}
3378
3379				mps_dprint(sc, MPS_INFO,
3380				"Sending read cap: path %s"
3381				" handle %d\n", path_str, target->handle );
3382
3383				/*
3384				 * Issue a READ CAPACITY 16 command for the LUN.
3385				 * The mpssas_read_cap_done function will load
3386				 * the read cap info into the LUN struct.
3387				 */
3388				rcap_buf =
3389					malloc(sizeof(struct scsi_read_capacity_eedp),
3390					M_MPT2, M_NOWAIT| M_ZERO);
3391				if (rcap_buf == NULL) {
3392					mps_dprint(sc, MPS_FAULT, "Unable to alloc read "
3393						"capacity buffer for EEDP support.\n");
3394					xpt_free_path(ccb->ccb_h.path);
3395					xpt_free_ccb(ccb);
3396					return;
3397				}
3398				csio = &ccb->csio;
3399				csio->ccb_h.func_code = XPT_SCSI_IO;
3400				csio->ccb_h.flags = CAM_DIR_IN;
3401				csio->ccb_h.retry_count = 4;
3402				csio->ccb_h.cbfcnp = mpssas_read_cap_done;
3403				csio->ccb_h.timeout = 60000;
3404				csio->data_ptr = (uint8_t *)rcap_buf;
3405				csio->dxfer_len = sizeof(struct
3406				    scsi_read_capacity_eedp);
3407				csio->sense_len = MPS_SENSE_LEN;
3408				csio->cdb_len = sizeof(*scsi_cmd);
3409				csio->tag_action = MSG_SIMPLE_Q_TAG;
3410
3411				scsi_cmd = (struct scsi_read_capacity_16 *)
3412				    &csio->cdb_io.cdb_bytes;
3413				bzero(scsi_cmd, sizeof(*scsi_cmd));
3414				scsi_cmd->opcode = 0x9E;
3415				scsi_cmd->service_action = SRC16_SERVICE_ACTION;
3416				((uint8_t *)scsi_cmd)[13] = sizeof(struct
3417				    scsi_read_capacity_eedp);
3418
3419				/*
3420				 * Set the path, target and lun IDs for the READ
3421				 * CAPACITY request.
3422				 */
3423				ccb->ccb_h.path_id =
3424				    xpt_path_path_id(ccb->ccb_h.path);
3425				ccb->ccb_h.target_id =
3426				    xpt_path_target_id(ccb->ccb_h.path);
3427				ccb->ccb_h.target_lun =
3428				    xpt_path_lun_id(ccb->ccb_h.path);
3429
3430				ccb->ccb_h.ppriv_ptr1 = sassc;
3431				xpt_action(ccb);
3432			} else {
3433				xpt_free_path(ccb->ccb_h.path);
3434				xpt_free_ccb(ccb);
3435			}
3436		} while (found_periph);
3437	}
3438}
3439
3440
3441static void
3442mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb)
3443{
3444	struct mpssas_softc *sassc;
3445	struct mpssas_target *target;
3446	struct mpssas_lun *lun;
3447	struct scsi_read_capacity_eedp *rcap_buf;
3448
3449	if (done_ccb == NULL)
3450		return;
3451
3452	/* Driver need to release devq, it Scsi command is
3453	 * generated by driver internally.
3454	 * Currently there is a single place where driver
3455	 * calls scsi command internally. In future if driver
3456	 * calls more scsi command internally, it needs to release
3457	 * devq internally, since those command will not go back to
3458	 * cam_periph.
3459	 */
3460	if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) ) {
3461        	done_ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
3462		xpt_release_devq(done_ccb->ccb_h.path,
3463			       	/*count*/ 1, /*run_queue*/TRUE);
3464	}
3465
3466	rcap_buf = (struct scsi_read_capacity_eedp *)done_ccb->csio.data_ptr;
3467
3468	/*
3469	 * Get the LUN ID for the path and look it up in the LUN list for the
3470	 * target.
3471	 */
3472	sassc = (struct mpssas_softc *)done_ccb->ccb_h.ppriv_ptr1;
3473	target = &sassc->targets[done_ccb->ccb_h.target_id];
3474	SLIST_FOREACH(lun, &target->luns, lun_link) {
3475		if (lun->lun_id != done_ccb->ccb_h.target_lun)
3476			continue;
3477
3478		/*
3479		 * Got the LUN in the target's LUN list.  Fill it in
3480		 * with EEDP info.  If the READ CAP 16 command had some
3481		 * SCSI error (common if command is not supported), mark
3482		 * the lun as not supporting EEDP and set the block size
3483		 * to 0.
3484		 */
3485		if (((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
3486		 || (done_ccb->csio.scsi_status != SCSI_STATUS_OK)) {
3487			lun->eedp_formatted = FALSE;
3488			lun->eedp_block_size = 0;
3489			break;
3490		}
3491
3492		if (rcap_buf->protect & 0x01) {
3493			lun->eedp_formatted = TRUE;
3494			lun->eedp_block_size = scsi_4btoul(rcap_buf->length);
3495		}
3496		break;
3497	}
3498
3499	// Finished with this CCB and path.
3500	free(rcap_buf, M_MPT2);
3501	xpt_free_path(done_ccb->ccb_h.path);
3502	xpt_free_ccb(done_ccb);
3503}
3504#endif /* __FreeBSD_version >= 1000006 */
3505
3506int
3507mpssas_startup(struct mps_softc *sc)
3508{
3509	struct mpssas_softc *sassc;
3510
3511	/*
3512	 * Send the port enable message and set the wait_for_port_enable flag.
3513	 * This flag helps to keep the simq frozen until all discovery events
3514	 * are processed.
3515	 */
3516	sassc = sc->sassc;
3517	mpssas_startup_increment(sassc);
3518	sc->wait_for_port_enable = 1;
3519	mpssas_send_portenable(sc);
3520	return (0);
3521}
3522
3523static int
3524mpssas_send_portenable(struct mps_softc *sc)
3525{
3526	MPI2_PORT_ENABLE_REQUEST *request;
3527	struct mps_command *cm;
3528
3529	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
3530
3531	if ((cm = mps_alloc_command(sc)) == NULL)
3532		return (EBUSY);
3533	request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req;
3534	request->Function = MPI2_FUNCTION_PORT_ENABLE;
3535	request->MsgFlags = 0;
3536	request->VP_ID = 0;
3537	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3538	cm->cm_complete = mpssas_portenable_complete;
3539	cm->cm_data = NULL;
3540	cm->cm_sge = NULL;
3541
3542	mps_map_command(sc, cm);
3543	mps_dprint(sc, MPS_TRACE,
3544	    "mps_send_portenable finished cm %p req %p complete %p\n",
3545	    cm, cm->cm_req, cm->cm_complete);
3546	return (0);
3547}
3548
3549static void
3550mpssas_portenable_complete(struct mps_softc *sc, struct mps_command *cm)
3551{
3552	MPI2_PORT_ENABLE_REPLY *reply;
3553	struct mpssas_softc *sassc;
3554
3555	mps_dprint(sc, MPS_TRACE, "%s\n", __func__);
3556	sassc = sc->sassc;
3557
3558	/*
3559	 * Currently there should be no way we can hit this case.  It only
3560	 * happens when we have a failure to allocate chain frames, and
3561	 * port enable commands don't have S/G lists.
3562	 */
3563	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3564		mps_printf(sc, "%s: cm_flags = %#x for port enable! "
3565			   "This should not happen!\n", __func__, cm->cm_flags);
3566	}
3567
3568	reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply;
3569	if (reply == NULL)
3570		mps_dprint(sc, MPS_FAULT, "Portenable NULL reply\n");
3571	else if (le16toh(reply->IOCStatus & MPI2_IOCSTATUS_MASK) !=
3572	    MPI2_IOCSTATUS_SUCCESS)
3573		mps_dprint(sc, MPS_FAULT, "Portenable failed\n");
3574
3575	mps_free_command(sc, cm);
3576	if (sc->mps_ich.ich_arg != NULL) {
3577		mps_dprint(sc, MPS_INFO, "disestablish config intrhook\n");
3578		config_intrhook_disestablish(&sc->mps_ich);
3579		sc->mps_ich.ich_arg = NULL;
3580	}
3581
3582	/*
3583	 * Get WarpDrive info after discovery is complete but before the scan
3584	 * starts.  At this point, all devices are ready to be exposed to the
3585	 * OS.  If devices should be hidden instead, take them out of the
3586	 * 'targets' array before the scan.  The devinfo for a disk will have
3587	 * some info and a volume's will be 0.  Use that to remove disks.
3588	 */
3589	mps_wd_config_pages(sc);
3590
3591	/*
3592	 * Done waiting for port enable to complete.  Decrement the refcount.
3593	 * If refcount is 0, discovery is complete and a rescan of the bus can
3594	 * take place.  Since the simq was explicitly frozen before port
3595	 * enable, it must be explicitly released here to keep the
3596	 * freeze/release count in sync.
3597	 */
3598	sc->wait_for_port_enable = 0;
3599	sc->port_enable_complete = 1;
3600	wakeup(&sc->port_enable_complete);
3601	mpssas_startup_decrement(sassc);
3602	xpt_release_simq(sassc->sim, 1);
3603}
3604
3605