mps_sas.c revision 4a3cd0cbac7415173acbfd6bce4080777648a167
1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2009 Yahoo! Inc.
5 * Copyright (c) 2011-2015 LSI Corp.
6 * Copyright (c) 2013-2015 Avago Technologies
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
31 *
32 * $FreeBSD$
33 */
34
35#include <sys/cdefs.h>
36__FBSDID("$FreeBSD$");
37
38/* Communications core for Avago Technologies (LSI) MPT2 */
39
40/* TODO Move headers to mpsvar */
41#include <sys/types.h>
42#include <sys/param.h>
43#include <sys/systm.h>
44#include <sys/kernel.h>
45#include <sys/selinfo.h>
46#include <sys/module.h>
47#include <sys/bus.h>
48#include <sys/conf.h>
49#include <sys/bio.h>
50#include <sys/malloc.h>
51#include <sys/uio.h>
52#include <sys/sysctl.h>
53#include <sys/endian.h>
54#include <sys/queue.h>
55#include <sys/kthread.h>
56#include <sys/taskqueue.h>
57#include <sys/sbuf.h>
58
59#include <machine/bus.h>
60#include <machine/resource.h>
61#include <sys/rman.h>
62
63#include <machine/stdarg.h>
64
65#include <cam/cam.h>
66#include <cam/cam_ccb.h>
67#include <cam/cam_xpt.h>
68#include <cam/cam_debug.h>
69#include <cam/cam_sim.h>
70#include <cam/cam_xpt_sim.h>
71#include <cam/cam_xpt_periph.h>
72#include <cam/cam_periph.h>
73#include <cam/scsi/scsi_all.h>
74#include <cam/scsi/scsi_message.h>
75#if __FreeBSD_version >= 900026
76#include <cam/scsi/smp_all.h>
77#endif
78
79#include <dev/mps/mpi/mpi2_type.h>
80#include <dev/mps/mpi/mpi2.h>
81#include <dev/mps/mpi/mpi2_ioc.h>
82#include <dev/mps/mpi/mpi2_sas.h>
83#include <dev/mps/mpi/mpi2_cnfg.h>
84#include <dev/mps/mpi/mpi2_init.h>
85#include <dev/mps/mpi/mpi2_tool.h>
86#include <dev/mps/mps_ioctl.h>
87#include <dev/mps/mpsvar.h>
88#include <dev/mps/mps_table.h>
89#include <dev/mps/mps_sas.h>
90
91#define MPSSAS_DISCOVERY_TIMEOUT	20
92#define MPSSAS_MAX_DISCOVERY_TIMEOUTS	10 /* 200 seconds */
93
94/*
95 * static array to check SCSI OpCode for EEDP protection bits
96 */
97#define	PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP
98#define	PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
99#define	PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
100static uint8_t op_code_prot[256] = {
101	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
102	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
103	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
104	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
105	0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
106	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
107	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
108	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
109	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
110	0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
111	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
112	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
113	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
114	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
115	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
116	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
117};
118
119MALLOC_DEFINE(M_MPSSAS, "MPSSAS", "MPS SAS memory");
120
121static void mpssas_remove_device(struct mps_softc *, struct mps_command *);
122static void mpssas_remove_complete(struct mps_softc *, struct mps_command *);
123static void mpssas_action(struct cam_sim *sim, union ccb *ccb);
124static void mpssas_poll(struct cam_sim *sim);
125static int mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm,
126    struct mps_command *cm);
127static void mpssas_scsiio_timeout(void *data);
128static void mpssas_abort_complete(struct mps_softc *sc, struct mps_command *cm);
129static void mpssas_direct_drive_io(struct mpssas_softc *sassc,
130    struct mps_command *cm, union ccb *ccb);
131static void mpssas_action_scsiio(struct mpssas_softc *, union ccb *);
132static void mpssas_scsiio_complete(struct mps_softc *, struct mps_command *);
133static void mpssas_action_resetdev(struct mpssas_softc *, union ccb *);
134#if __FreeBSD_version >= 900026
135static void mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm);
136static void mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb,
137			       uint64_t sasaddr);
138static void mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb);
139#endif //FreeBSD_version >= 900026
140static void mpssas_resetdev_complete(struct mps_softc *, struct mps_command *);
141static void mpssas_async(void *callback_arg, uint32_t code,
142			 struct cam_path *path, void *arg);
143#if (__FreeBSD_version < 901503) || \
144    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
145static void mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
146			      struct ccb_getdev *cgd);
147static void mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb);
148#endif
149static int mpssas_send_portenable(struct mps_softc *sc);
150static void mpssas_portenable_complete(struct mps_softc *sc,
151    struct mps_command *cm);
152
153struct mpssas_target *
154mpssas_find_target_by_handle(struct mpssas_softc *sassc, int start, uint16_t handle)
155{
156	struct mpssas_target *target;
157	int i;
158
159	for (i = start; i < sassc->maxtargets; i++) {
160		target = &sassc->targets[i];
161		if (target->handle == handle)
162			return (target);
163	}
164
165	return (NULL);
166}
167
168/* we need to freeze the simq during attach and diag reset, to avoid failing
169 * commands before device handles have been found by discovery.  Since
170 * discovery involves reading config pages and possibly sending commands,
171 * discovery actions may continue even after we receive the end of discovery
172 * event, so refcount discovery actions instead of assuming we can unfreeze
173 * the simq when we get the event.
174 */
175void
176mpssas_startup_increment(struct mpssas_softc *sassc)
177{
178	MPS_FUNCTRACE(sassc->sc);
179
180	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
181		if (sassc->startup_refcount++ == 0) {
182			/* just starting, freeze the simq */
183			mps_dprint(sassc->sc, MPS_INIT,
184			    "%s freezing simq\n", __func__);
185#if __FreeBSD_version >= 1000039
186			xpt_hold_boot();
187#endif
188			xpt_freeze_simq(sassc->sim, 1);
189		}
190		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
191		    sassc->startup_refcount);
192	}
193}
194
195void
196mpssas_release_simq_reinit(struct mpssas_softc *sassc)
197{
198	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
199		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
200		xpt_release_simq(sassc->sim, 1);
201		mps_dprint(sassc->sc, MPS_INFO, "Unfreezing SIM queue\n");
202	}
203}
204
205void
206mpssas_startup_decrement(struct mpssas_softc *sassc)
207{
208	MPS_FUNCTRACE(sassc->sc);
209
210	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
211		if (--sassc->startup_refcount == 0) {
212			/* finished all discovery-related actions, release
213			 * the simq and rescan for the latest topology.
214			 */
215			mps_dprint(sassc->sc, MPS_INIT,
216			    "%s releasing simq\n", __func__);
217			sassc->flags &= ~MPSSAS_IN_STARTUP;
218			xpt_release_simq(sassc->sim, 1);
219#if __FreeBSD_version >= 1000039
220			xpt_release_boot();
221#else
222			mpssas_rescan_target(sassc->sc, NULL);
223#endif
224		}
225		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
226		    sassc->startup_refcount);
227	}
228}
229
230/* The firmware requires us to stop sending commands when we're doing task
231 * management, so refcount the TMs and keep the simq frozen when any are in
232 * use.
233 */
234struct mps_command *
235mpssas_alloc_tm(struct mps_softc *sc)
236{
237	struct mps_command *tm;
238
239	tm = mps_alloc_high_priority_command(sc);
240	return tm;
241}
242
243void
244mpssas_free_tm(struct mps_softc *sc, struct mps_command *tm)
245{
246	int target_id = 0xFFFFFFFF;
247
248	if (tm == NULL)
249		return;
250
251	/*
252	 * For TM's the devq is frozen for the device.  Unfreeze it here and
253	 * free the resources used for freezing the devq.  Must clear the
254	 * INRESET flag as well or scsi I/O will not work.
255	 */
256	if (tm->cm_targ != NULL) {
257		tm->cm_targ->flags &= ~MPSSAS_TARGET_INRESET;
258		target_id = tm->cm_targ->tid;
259	}
260	if (tm->cm_ccb) {
261		mps_dprint(sc, MPS_INFO, "Unfreezing devq for target ID %d\n",
262		    target_id);
263		xpt_release_devq(tm->cm_ccb->ccb_h.path, 1, TRUE);
264		xpt_free_path(tm->cm_ccb->ccb_h.path);
265		xpt_free_ccb(tm->cm_ccb);
266	}
267
268	mps_free_high_priority_command(sc, tm);
269}
270
271void
272mpssas_rescan_target(struct mps_softc *sc, struct mpssas_target *targ)
273{
274	struct mpssas_softc *sassc = sc->sassc;
275	path_id_t pathid;
276	target_id_t targetid;
277	union ccb *ccb;
278
279	MPS_FUNCTRACE(sc);
280	pathid = cam_sim_path(sassc->sim);
281	if (targ == NULL)
282		targetid = CAM_TARGET_WILDCARD;
283	else
284		targetid = targ - sassc->targets;
285
286	/*
287	 * Allocate a CCB and schedule a rescan.
288	 */
289	ccb = xpt_alloc_ccb_nowait();
290	if (ccb == NULL) {
291		mps_dprint(sc, MPS_ERROR, "unable to alloc CCB for rescan\n");
292		return;
293	}
294
295	if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
296	    targetid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
297		mps_dprint(sc, MPS_ERROR, "unable to create path for rescan\n");
298		xpt_free_ccb(ccb);
299		return;
300	}
301
302	if (targetid == CAM_TARGET_WILDCARD)
303		ccb->ccb_h.func_code = XPT_SCAN_BUS;
304	else
305		ccb->ccb_h.func_code = XPT_SCAN_TGT;
306
307	mps_dprint(sc, MPS_TRACE, "%s targetid %u\n", __func__, targetid);
308	xpt_rescan(ccb);
309}
310
311static void
312mpssas_log_command(struct mps_command *cm, u_int level, const char *fmt, ...)
313{
314	struct sbuf sb;
315	va_list ap;
316	char str[192];
317	char path_str[64];
318
319	if (cm == NULL)
320		return;
321
322	/* No need to be in here if debugging isn't enabled */
323	if ((cm->cm_sc->mps_debug & level) == 0)
324		return;
325
326	sbuf_new(&sb, str, sizeof(str), 0);
327
328	va_start(ap, fmt);
329
330	if (cm->cm_ccb != NULL) {
331		xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str,
332				sizeof(path_str));
333		sbuf_cat(&sb, path_str);
334		if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) {
335			scsi_command_string(&cm->cm_ccb->csio, &sb);
336			sbuf_printf(&sb, "length %d ",
337				    cm->cm_ccb->csio.dxfer_len);
338		}
339	}
340	else {
341		sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ",
342		    cam_sim_name(cm->cm_sc->sassc->sim),
343		    cam_sim_unit(cm->cm_sc->sassc->sim),
344		    cam_sim_bus(cm->cm_sc->sassc->sim),
345		    cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF,
346		    cm->cm_lun);
347	}
348
349	sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID);
350	sbuf_vprintf(&sb, fmt, ap);
351	sbuf_finish(&sb);
352	mps_print_field(cm->cm_sc, "%s", sbuf_data(&sb));
353
354	va_end(ap);
355}
356
357
358static void
359mpssas_remove_volume(struct mps_softc *sc, struct mps_command *tm)
360{
361	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
362	struct mpssas_target *targ;
363	uint16_t handle;
364
365	MPS_FUNCTRACE(sc);
366
367	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
368	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
369	targ = tm->cm_targ;
370
371	if (reply == NULL) {
372		/* XXX retry the remove after the diag reset completes? */
373		mps_dprint(sc, MPS_FAULT,
374		    "%s NULL reply resetting device 0x%04x\n", __func__,
375		    handle);
376		mpssas_free_tm(sc, tm);
377		return;
378	}
379
380	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
381	    MPI2_IOCSTATUS_SUCCESS) {
382		mps_dprint(sc, MPS_ERROR,
383		   "IOCStatus = 0x%x while resetting device 0x%x\n",
384		   le16toh(reply->IOCStatus), handle);
385	}
386
387	mps_dprint(sc, MPS_XINFO,
388	    "Reset aborted %u commands\n", reply->TerminationCount);
389	mps_free_reply(sc, tm->cm_reply_data);
390	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
391
392	mps_dprint(sc, MPS_XINFO,
393	    "clearing target %u handle 0x%04x\n", targ->tid, handle);
394
395	/*
396	 * Don't clear target if remove fails because things will get confusing.
397	 * Leave the devname and sasaddr intact so that we know to avoid reusing
398	 * this target id if possible, and so we can assign the same target id
399	 * to this device if it comes back in the future.
400	 */
401	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
402	    MPI2_IOCSTATUS_SUCCESS) {
403		targ = tm->cm_targ;
404		targ->handle = 0x0;
405		targ->encl_handle = 0x0;
406		targ->encl_slot = 0x0;
407		targ->exp_dev_handle = 0x0;
408		targ->phy_num = 0x0;
409		targ->linkrate = 0x0;
410		targ->devinfo = 0x0;
411		targ->flags = 0x0;
412	}
413
414	mpssas_free_tm(sc, tm);
415}
416
417
418/*
419 * No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal.
420 * Otherwise Volume Delete is same as Bare Drive Removal.
421 */
422void
423mpssas_prepare_volume_remove(struct mpssas_softc *sassc, uint16_t handle)
424{
425	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
426	struct mps_softc *sc;
427	struct mps_command *cm;
428	struct mpssas_target *targ = NULL;
429
430	MPS_FUNCTRACE(sassc->sc);
431	sc = sassc->sc;
432
433#ifdef WD_SUPPORT
434	/*
435	 * If this is a WD controller, determine if the disk should be exposed
436	 * to the OS or not.  If disk should be exposed, return from this
437	 * function without doing anything.
438	 */
439	if (sc->WD_available && (sc->WD_hide_expose ==
440	    MPS_WD_EXPOSE_ALWAYS)) {
441		return;
442	}
443#endif //WD_SUPPORT
444
445	targ = mpssas_find_target_by_handle(sassc, 0, handle);
446	if (targ == NULL) {
447		/* FIXME: what is the action? */
448		/* We don't know about this device? */
449		mps_dprint(sc, MPS_ERROR,
450		   "%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
451		return;
452	}
453
454	targ->flags |= MPSSAS_TARGET_INREMOVAL;
455
456	cm = mpssas_alloc_tm(sc);
457	if (cm == NULL) {
458		mps_dprint(sc, MPS_ERROR,
459		    "%s: command alloc failure\n", __func__);
460		return;
461	}
462
463	mpssas_rescan_target(sc, targ);
464
465	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
466	req->DevHandle = targ->handle;
467	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
468	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
469
470	/* SAS Hard Link Reset / SATA Link Reset */
471	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
472
473	cm->cm_targ = targ;
474	cm->cm_data = NULL;
475	cm->cm_desc.HighPriority.RequestFlags =
476	    MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
477	cm->cm_complete = mpssas_remove_volume;
478	cm->cm_complete_data = (void *)(uintptr_t)handle;
479
480	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
481	    __func__, targ->tid);
482	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
483
484	mps_map_command(sc, cm);
485}
486
487/*
488 * The MPT2 firmware performs debounce on the link to avoid transient link
489 * errors and false removals.  When it does decide that link has been lost
490 * and a device need to go away, it expects that the host will perform a
491 * target reset and then an op remove.  The reset has the side-effect of
492 * aborting any outstanding requests for the device, which is required for
493 * the op-remove to succeed.  It's not clear if the host should check for
494 * the device coming back alive after the reset.
495 */
496void
497mpssas_prepare_remove(struct mpssas_softc *sassc, uint16_t handle)
498{
499	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
500	struct mps_softc *sc;
501	struct mps_command *cm;
502	struct mpssas_target *targ = NULL;
503
504	MPS_FUNCTRACE(sassc->sc);
505
506	sc = sassc->sc;
507
508	targ = mpssas_find_target_by_handle(sassc, 0, handle);
509	if (targ == NULL) {
510		/* FIXME: what is the action? */
511		/* We don't know about this device? */
512		mps_dprint(sc, MPS_ERROR,
513		    "%s : invalid handle 0x%x \n", __func__, handle);
514		return;
515	}
516
517	targ->flags |= MPSSAS_TARGET_INREMOVAL;
518
519	cm = mpssas_alloc_tm(sc);
520	if (cm == NULL) {
521		mps_dprint(sc, MPS_ERROR,
522		    "%s: command alloc failure\n", __func__);
523		return;
524	}
525
526	mpssas_rescan_target(sc, targ);
527
528	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
529	memset(req, 0, sizeof(*req));
530	req->DevHandle = htole16(targ->handle);
531	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
532	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
533
534	/* SAS Hard Link Reset / SATA Link Reset */
535	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
536
537	cm->cm_targ = targ;
538	cm->cm_data = NULL;
539	cm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
540	cm->cm_complete = mpssas_remove_device;
541	cm->cm_complete_data = (void *)(uintptr_t)handle;
542
543	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
544	    __func__, targ->tid);
545	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
546
547	mps_map_command(sc, cm);
548}
549
550static void
551mpssas_remove_device(struct mps_softc *sc, struct mps_command *tm)
552{
553	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
554	MPI2_SAS_IOUNIT_CONTROL_REQUEST *req;
555	struct mpssas_target *targ;
556	struct mps_command *next_cm;
557	uint16_t handle;
558
559	MPS_FUNCTRACE(sc);
560
561	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
562	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
563	targ = tm->cm_targ;
564
565	/*
566	 * Currently there should be no way we can hit this case.  It only
567	 * happens when we have a failure to allocate chain frames, and
568	 * task management commands don't have S/G lists.
569	 */
570	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
571		mps_dprint(sc, MPS_ERROR,
572		    "%s: cm_flags = %#x for remove of handle %#04x! "
573		    "This should not happen!\n", __func__, tm->cm_flags,
574		    handle);
575	}
576
577	if (reply == NULL) {
578		/* XXX retry the remove after the diag reset completes? */
579		mps_dprint(sc, MPS_FAULT,
580		    "%s NULL reply resetting device 0x%04x\n", __func__,
581		    handle);
582		mpssas_free_tm(sc, tm);
583		return;
584	}
585
586	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
587	    MPI2_IOCSTATUS_SUCCESS) {
588		mps_dprint(sc, MPS_ERROR,
589		   "IOCStatus = 0x%x while resetting device 0x%x\n",
590		   le16toh(reply->IOCStatus), handle);
591	}
592
593	mps_dprint(sc, MPS_XINFO, "Reset aborted %u commands\n",
594	    le32toh(reply->TerminationCount));
595	mps_free_reply(sc, tm->cm_reply_data);
596	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
597
598	/* Reuse the existing command */
599	req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req;
600	memset(req, 0, sizeof(*req));
601	req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
602	req->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
603	req->DevHandle = htole16(handle);
604	tm->cm_data = NULL;
605	tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
606	tm->cm_complete = mpssas_remove_complete;
607	tm->cm_complete_data = (void *)(uintptr_t)handle;
608
609	mps_map_command(sc, tm);
610
611	mps_dprint(sc, MPS_XINFO, "clearing target %u handle 0x%04x\n",
612		   targ->tid, handle);
613	TAILQ_FOREACH_SAFE(tm, &targ->commands, cm_link, next_cm) {
614		union ccb *ccb;
615
616		mps_dprint(sc, MPS_XINFO, "Completing missed command %p\n", tm);
617		ccb = tm->cm_complete_data;
618		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
619		mpssas_scsiio_complete(sc, tm);
620	}
621}
622
623static void
624mpssas_remove_complete(struct mps_softc *sc, struct mps_command *tm)
625{
626	MPI2_SAS_IOUNIT_CONTROL_REPLY *reply;
627	uint16_t handle;
628	struct mpssas_target *targ;
629	struct mpssas_lun *lun;
630
631	MPS_FUNCTRACE(sc);
632
633	reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply;
634	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
635
636	/*
637	 * Currently there should be no way we can hit this case.  It only
638	 * happens when we have a failure to allocate chain frames, and
639	 * task management commands don't have S/G lists.
640	 */
641	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
642		mps_dprint(sc, MPS_XINFO,
643			   "%s: cm_flags = %#x for remove of handle %#04x! "
644			   "This should not happen!\n", __func__, tm->cm_flags,
645			   handle);
646		mpssas_free_tm(sc, tm);
647		return;
648	}
649
650	if (reply == NULL) {
651		/* most likely a chip reset */
652		mps_dprint(sc, MPS_FAULT,
653		    "%s NULL reply removing device 0x%04x\n", __func__, handle);
654		mpssas_free_tm(sc, tm);
655		return;
656	}
657
658	mps_dprint(sc, MPS_XINFO,
659	    "%s on handle 0x%04x, IOCStatus= 0x%x\n", __func__,
660	    handle, le16toh(reply->IOCStatus));
661
662	/*
663	 * Don't clear target if remove fails because things will get confusing.
664	 * Leave the devname and sasaddr intact so that we know to avoid reusing
665	 * this target id if possible, and so we can assign the same target id
666	 * to this device if it comes back in the future.
667	 */
668	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
669	    MPI2_IOCSTATUS_SUCCESS) {
670		targ = tm->cm_targ;
671		targ->handle = 0x0;
672		targ->encl_handle = 0x0;
673		targ->encl_slot = 0x0;
674		targ->exp_dev_handle = 0x0;
675		targ->phy_num = 0x0;
676		targ->linkrate = 0x0;
677		targ->devinfo = 0x0;
678		targ->flags = 0x0;
679
680		while(!SLIST_EMPTY(&targ->luns)) {
681			lun = SLIST_FIRST(&targ->luns);
682			SLIST_REMOVE_HEAD(&targ->luns, lun_link);
683			free(lun, M_MPT2);
684		}
685	}
686
687
688	mpssas_free_tm(sc, tm);
689}
690
691static int
692mpssas_register_events(struct mps_softc *sc)
693{
694	u32 events[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];
695
696	bzero(events, 16);
697	setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
698	setbit(events, MPI2_EVENT_SAS_DISCOVERY);
699	setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
700	setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE);
701	setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW);
702	setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
703	setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
704	setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
705	setbit(events, MPI2_EVENT_IR_VOLUME);
706	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
707	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
708	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);
709
710	mps_register_events(sc, events, mpssas_evt_handler, NULL,
711	    &sc->sassc->mpssas_eh);
712
713	return (0);
714}
715
716int
717mps_attach_sas(struct mps_softc *sc)
718{
719	struct mpssas_softc *sassc;
720	cam_status status;
721	int unit, error = 0, reqs;
722
723	MPS_FUNCTRACE(sc);
724	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);
725
726	sassc = malloc(sizeof(struct mpssas_softc), M_MPT2, M_WAITOK|M_ZERO);
727	if(!sassc) {
728		mps_dprint(sc, MPS_INIT|MPS_ERROR,
729		    "Cannot allocate SAS controller memory\n");
730		return (ENOMEM);
731	}
732
733	/*
734	 * XXX MaxTargets could change during a reinit.  Since we don't
735	 * resize the targets[] array during such an event, cache the value
736	 * of MaxTargets here so that we don't get into trouble later.  This
737	 * should move into the reinit logic.
738	 */
739	sassc->maxtargets = sc->facts->MaxTargets + sc->facts->MaxVolumes;
740	sassc->targets = malloc(sizeof(struct mpssas_target) *
741	    sassc->maxtargets, M_MPT2, M_WAITOK|M_ZERO);
742	if(!sassc->targets) {
743		mps_dprint(sc, MPS_INIT|MPS_ERROR,
744		    "Cannot allocate SAS target memory\n");
745		free(sassc, M_MPT2);
746		return (ENOMEM);
747	}
748	sc->sassc = sassc;
749	sassc->sc = sc;
750
751	reqs = sc->num_reqs - sc->num_prireqs - 1;
752	if ((sassc->devq = cam_simq_alloc(reqs)) == NULL) {
753		mps_dprint(sc, MPS_ERROR, "Cannot allocate SIMQ\n");
754		error = ENOMEM;
755		goto out;
756	}
757
758	unit = device_get_unit(sc->mps_dev);
759	sassc->sim = cam_sim_alloc(mpssas_action, mpssas_poll, "mps", sassc,
760	    unit, &sc->mps_mtx, reqs, reqs, sassc->devq);
761	if (sassc->sim == NULL) {
762		mps_dprint(sc, MPS_INIT|MPS_ERROR, "Cannot allocate SIM\n");
763		error = EINVAL;
764		goto out;
765	}
766
767	TAILQ_INIT(&sassc->ev_queue);
768
769	/* Initialize taskqueue for Event Handling */
770	TASK_INIT(&sassc->ev_task, 0, mpssas_firmware_event_work, sc);
771	sassc->ev_tq = taskqueue_create("mps_taskq", M_NOWAIT | M_ZERO,
772	    taskqueue_thread_enqueue, &sassc->ev_tq);
773	taskqueue_start_threads(&sassc->ev_tq, 1, PRIBIO, "%s taskq",
774	    device_get_nameunit(sc->mps_dev));
775
776	mps_lock(sc);
777
778	/*
779	 * XXX There should be a bus for every port on the adapter, but since
780	 * we're just going to fake the topology for now, we'll pretend that
781	 * everything is just a target on a single bus.
782	 */
783	if ((error = xpt_bus_register(sassc->sim, sc->mps_dev, 0)) != 0) {
784		mps_dprint(sc, MPS_INIT|MPS_ERROR,
785		    "Error %d registering SCSI bus\n", error);
786		mps_unlock(sc);
787		goto out;
788	}
789
790	/*
791	 * Assume that discovery events will start right away.
792	 *
793	 * Hold off boot until discovery is complete.
794	 */
795	sassc->flags |= MPSSAS_IN_STARTUP | MPSSAS_IN_DISCOVERY;
796	sc->sassc->startup_refcount = 0;
797	mpssas_startup_increment(sassc);
798
799	callout_init(&sassc->discovery_callout, 1 /*mpsafe*/);
800
801	/*
802	 * Register for async events so we can determine the EEDP
803	 * capabilities of devices.
804	 */
805	status = xpt_create_path(&sassc->path, /*periph*/NULL,
806	    cam_sim_path(sc->sassc->sim), CAM_TARGET_WILDCARD,
807	    CAM_LUN_WILDCARD);
808	if (status != CAM_REQ_CMP) {
809		mps_dprint(sc, MPS_ERROR|MPS_INIT,
810		    "Error %#x creating sim path\n", status);
811		sassc->path = NULL;
812	} else {
813		int event;
814
815#if (__FreeBSD_version >= 1000006) || \
816    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
817		event = AC_ADVINFO_CHANGED;
818#else
819		event = AC_FOUND_DEVICE;
820#endif
821		status = xpt_register_async(event, mpssas_async, sc,
822					    sassc->path);
823		if (status != CAM_REQ_CMP) {
824			mps_dprint(sc, MPS_ERROR,
825			    "Error %#x registering async handler for "
826			    "AC_ADVINFO_CHANGED events\n", status);
827			xpt_free_path(sassc->path);
828			sassc->path = NULL;
829		}
830	}
831	if (status != CAM_REQ_CMP) {
832		/*
833		 * EEDP use is the exception, not the rule.
834		 * Warn the user, but do not fail to attach.
835		 */
836		mps_printf(sc, "EEDP capabilities disabled.\n");
837	}
838
839	mps_unlock(sc);
840
841	mpssas_register_events(sc);
842out:
843	if (error)
844		mps_detach_sas(sc);
845
846	mps_dprint(sc, MPS_INIT, "%s exit error= %d\n", __func__, error);
847	return (error);
848}
849
850int
851mps_detach_sas(struct mps_softc *sc)
852{
853	struct mpssas_softc *sassc;
854	struct mpssas_lun *lun, *lun_tmp;
855	struct mpssas_target *targ;
856	int i;
857
858	MPS_FUNCTRACE(sc);
859
860	if (sc->sassc == NULL)
861		return (0);
862
863	sassc = sc->sassc;
864	mps_deregister_events(sc, sassc->mpssas_eh);
865
866	/*
867	 * Drain and free the event handling taskqueue with the lock
868	 * unheld so that any parallel processing tasks drain properly
869	 * without deadlocking.
870	 */
871	if (sassc->ev_tq != NULL)
872		taskqueue_free(sassc->ev_tq);
873
874	/* Make sure CAM doesn't wedge if we had to bail out early. */
875	mps_lock(sc);
876
877	while (sassc->startup_refcount != 0)
878		mpssas_startup_decrement(sassc);
879
880	/* Deregister our async handler */
881	if (sassc->path != NULL) {
882		xpt_register_async(0, mpssas_async, sc, sassc->path);
883		xpt_free_path(sassc->path);
884		sassc->path = NULL;
885	}
886
887	if (sassc->flags & MPSSAS_IN_STARTUP)
888		xpt_release_simq(sassc->sim, 1);
889
890	if (sassc->sim != NULL) {
891		xpt_bus_deregister(cam_sim_path(sassc->sim));
892		cam_sim_free(sassc->sim, FALSE);
893	}
894
895	mps_unlock(sc);
896
897	if (sassc->devq != NULL)
898		cam_simq_free(sassc->devq);
899
900	for(i=0; i< sassc->maxtargets ;i++) {
901		targ = &sassc->targets[i];
902		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
903			free(lun, M_MPT2);
904		}
905	}
906	free(sassc->targets, M_MPT2);
907	free(sassc, M_MPT2);
908	sc->sassc = NULL;
909
910	return (0);
911}
912
913void
914mpssas_discovery_end(struct mpssas_softc *sassc)
915{
916	struct mps_softc *sc = sassc->sc;
917
918	MPS_FUNCTRACE(sc);
919
920	if (sassc->flags & MPSSAS_DISCOVERY_TIMEOUT_PENDING)
921		callout_stop(&sassc->discovery_callout);
922
923	/*
924	 * After discovery has completed, check the mapping table for any
925	 * missing devices and update their missing counts. Only do this once
926	 * whenever the driver is initialized so that missing counts aren't
927	 * updated unnecessarily. Note that just because discovery has
928	 * completed doesn't mean that events have been processed yet. The
929	 * check_devices function is a callout timer that checks if ALL devices
930	 * are missing. If so, it will wait a little longer for events to
931	 * complete and keep resetting itself until some device in the mapping
932	 * table is not missing, meaning that event processing has started.
933	 */
934	if (sc->track_mapping_events) {
935		mps_dprint(sc, MPS_XINFO | MPS_MAPPING, "Discovery has "
936		    "completed. Check for missing devices in the mapping "
937		    "table.\n");
938		callout_reset(&sc->device_check_callout,
939		    MPS_MISSING_CHECK_DELAY * hz, mps_mapping_check_devices,
940		    sc);
941	}
942}
943
944static void
945mpssas_action(struct cam_sim *sim, union ccb *ccb)
946{
947	struct mpssas_softc *sassc;
948
949	sassc = cam_sim_softc(sim);
950
951	MPS_FUNCTRACE(sassc->sc);
952	mps_dprint(sassc->sc, MPS_TRACE, "ccb func_code 0x%x\n",
953	    ccb->ccb_h.func_code);
954	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
955
956	switch (ccb->ccb_h.func_code) {
957	case XPT_PATH_INQ:
958	{
959		struct ccb_pathinq *cpi = &ccb->cpi;
960		struct mps_softc *sc = sassc->sc;
961
962		cpi->version_num = 1;
963		cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
964		cpi->target_sprt = 0;
965#if __FreeBSD_version >= 1000039
966		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN;
967#else
968		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED;
969#endif
970		cpi->hba_eng_cnt = 0;
971		cpi->max_target = sassc->maxtargets - 1;
972		cpi->max_lun = 255;
973
974		/*
975		 * initiator_id is set here to an ID outside the set of valid
976		 * target IDs (including volumes).
977		 */
978		cpi->initiator_id = sassc->maxtargets;
979		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
980		strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN);
981		strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
982		cpi->unit_number = cam_sim_unit(sim);
983		cpi->bus_id = cam_sim_bus(sim);
984		cpi->base_transfer_speed = 150000;
985		cpi->transport = XPORT_SAS;
986		cpi->transport_version = 0;
987		cpi->protocol = PROTO_SCSI;
988		cpi->protocol_version = SCSI_REV_SPC;
989		cpi->maxio = sc->maxio;
990		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
991		break;
992	}
993	case XPT_GET_TRAN_SETTINGS:
994	{
995		struct ccb_trans_settings	*cts;
996		struct ccb_trans_settings_sas	*sas;
997		struct ccb_trans_settings_scsi	*scsi;
998		struct mpssas_target *targ;
999
1000		cts = &ccb->cts;
1001		sas = &cts->xport_specific.sas;
1002		scsi = &cts->proto_specific.scsi;
1003
1004		KASSERT(cts->ccb_h.target_id < sassc->maxtargets,
1005		    ("Target %d out of bounds in XPT_GET_TRANS_SETTINGS\n",
1006		    cts->ccb_h.target_id));
1007		targ = &sassc->targets[cts->ccb_h.target_id];
1008		if (targ->handle == 0x0) {
1009			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1010			break;
1011		}
1012
1013		cts->protocol_version = SCSI_REV_SPC2;
1014		cts->transport = XPORT_SAS;
1015		cts->transport_version = 0;
1016
1017		sas->valid = CTS_SAS_VALID_SPEED;
1018		switch (targ->linkrate) {
1019		case 0x08:
1020			sas->bitrate = 150000;
1021			break;
1022		case 0x09:
1023			sas->bitrate = 300000;
1024			break;
1025		case 0x0a:
1026			sas->bitrate = 600000;
1027			break;
1028		default:
1029			sas->valid = 0;
1030		}
1031
1032		cts->protocol = PROTO_SCSI;
1033		scsi->valid = CTS_SCSI_VALID_TQ;
1034		scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
1035
1036		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1037		break;
1038	}
1039	case XPT_CALC_GEOMETRY:
1040		cam_calc_geometry(&ccb->ccg, /*extended*/1);
1041		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1042		break;
1043	case XPT_RESET_DEV:
1044		mps_dprint(sassc->sc, MPS_XINFO, "mpssas_action XPT_RESET_DEV\n");
1045		mpssas_action_resetdev(sassc, ccb);
1046		return;
1047	case XPT_RESET_BUS:
1048	case XPT_ABORT:
1049	case XPT_TERM_IO:
1050		mps_dprint(sassc->sc, MPS_XINFO,
1051		    "mpssas_action faking success for abort or reset\n");
1052		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1053		break;
1054	case XPT_SCSI_IO:
1055		mpssas_action_scsiio(sassc, ccb);
1056		return;
1057#if __FreeBSD_version >= 900026
1058	case XPT_SMP_IO:
1059		mpssas_action_smpio(sassc, ccb);
1060		return;
1061#endif
1062	default:
1063		mpssas_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL);
1064		break;
1065	}
1066	xpt_done(ccb);
1067
1068}
1069
1070static void
1071mpssas_announce_reset(struct mps_softc *sc, uint32_t ac_code,
1072    target_id_t target_id, lun_id_t lun_id)
1073{
1074	path_id_t path_id = cam_sim_path(sc->sassc->sim);
1075	struct cam_path *path;
1076
1077	mps_dprint(sc, MPS_XINFO, "%s code %x target %d lun %jx\n", __func__,
1078	    ac_code, target_id, (uintmax_t)lun_id);
1079
1080	if (xpt_create_path(&path, NULL,
1081		path_id, target_id, lun_id) != CAM_REQ_CMP) {
1082		mps_dprint(sc, MPS_ERROR, "unable to create path for reset "
1083			   "notification\n");
1084		return;
1085	}
1086
1087	xpt_async(ac_code, path, NULL);
1088	xpt_free_path(path);
1089}
1090
1091static void
1092mpssas_complete_all_commands(struct mps_softc *sc)
1093{
1094	struct mps_command *cm;
1095	int i;
1096	int completed;
1097
1098	MPS_FUNCTRACE(sc);
1099	mtx_assert(&sc->mps_mtx, MA_OWNED);
1100
1101	/* complete all commands with a NULL reply */
1102	for (i = 1; i < sc->num_reqs; i++) {
1103		cm = &sc->commands[i];
1104		if (cm->cm_state == MPS_CM_STATE_FREE)
1105			continue;
1106
1107		cm->cm_state = MPS_CM_STATE_BUSY;
1108		cm->cm_reply = NULL;
1109		completed = 0;
1110
1111		if (cm->cm_flags & MPS_CM_FLAGS_SATA_ID_TIMEOUT) {
1112			MPASS(cm->cm_data);
1113			free(cm->cm_data, M_MPT2);
1114			cm->cm_data = NULL;
1115		}
1116
1117		if (cm->cm_flags & MPS_CM_FLAGS_POLLED)
1118			cm->cm_flags |= MPS_CM_FLAGS_COMPLETE;
1119
1120		if (cm->cm_complete != NULL) {
1121			mpssas_log_command(cm, MPS_RECOVERY,
1122			    "completing cm %p state %x ccb %p for diag reset\n",
1123			    cm, cm->cm_state, cm->cm_ccb);
1124
1125			cm->cm_complete(sc, cm);
1126			completed = 1;
1127		} else if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {
1128			mpssas_log_command(cm, MPS_RECOVERY,
1129			    "waking up cm %p state %x ccb %p for diag reset\n",
1130			    cm, cm->cm_state, cm->cm_ccb);
1131			wakeup(cm);
1132			completed = 1;
1133		}
1134
1135		if ((completed == 0) && (cm->cm_state != MPS_CM_STATE_FREE)) {
1136			/* this should never happen, but if it does, log */
1137			mpssas_log_command(cm, MPS_RECOVERY,
1138			    "cm %p state %x flags 0x%x ccb %p during diag "
1139			    "reset\n", cm, cm->cm_state, cm->cm_flags,
1140			    cm->cm_ccb);
1141		}
1142	}
1143
1144	sc->io_cmds_active = 0;
1145}
1146
1147void
1148mpssas_handle_reinit(struct mps_softc *sc)
1149{
1150	int i;
1151
1152	/* Go back into startup mode and freeze the simq, so that CAM
1153	 * doesn't send any commands until after we've rediscovered all
1154	 * targets and found the proper device handles for them.
1155	 *
1156	 * After the reset, portenable will trigger discovery, and after all
1157	 * discovery-related activities have finished, the simq will be
1158	 * released.
1159	 */
1160	mps_dprint(sc, MPS_INIT, "%s startup\n", __func__);
1161	sc->sassc->flags |= MPSSAS_IN_STARTUP;
1162	sc->sassc->flags |= MPSSAS_IN_DISCOVERY;
1163	mpssas_startup_increment(sc->sassc);
1164
1165	/* notify CAM of a bus reset */
1166	mpssas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
1167	    CAM_LUN_WILDCARD);
1168
1169	/* complete and cleanup after all outstanding commands */
1170	mpssas_complete_all_commands(sc);
1171
1172	mps_dprint(sc, MPS_INIT,
1173	    "%s startup %u after command completion\n", __func__,
1174	    sc->sassc->startup_refcount);
1175
1176	/* zero all the target handles, since they may change after the
1177	 * reset, and we have to rediscover all the targets and use the new
1178	 * handles.
1179	 */
1180	for (i = 0; i < sc->sassc->maxtargets; i++) {
1181		if (sc->sassc->targets[i].outstanding != 0)
1182			mps_dprint(sc, MPS_INIT, "target %u outstanding %u\n",
1183			    i, sc->sassc->targets[i].outstanding);
1184		sc->sassc->targets[i].handle = 0x0;
1185		sc->sassc->targets[i].exp_dev_handle = 0x0;
1186		sc->sassc->targets[i].outstanding = 0;
1187		sc->sassc->targets[i].flags = MPSSAS_TARGET_INDIAGRESET;
1188	}
1189}
1190
1191static void
1192mpssas_tm_timeout(void *data)
1193{
1194	struct mps_command *tm = data;
1195	struct mps_softc *sc = tm->cm_sc;
1196
1197	mtx_assert(&sc->mps_mtx, MA_OWNED);
1198
1199	mpssas_log_command(tm, MPS_INFO|MPS_RECOVERY,
1200	    "task mgmt %p timed out\n", tm);
1201
1202	KASSERT(tm->cm_state == MPS_CM_STATE_INQUEUE,
1203	    ("command not inqueue\n"));
1204
1205	tm->cm_state = MPS_CM_STATE_BUSY;
1206	mps_reinit(sc);
1207}
1208
1209static void
1210mpssas_logical_unit_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1211{
1212	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1213	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1214	unsigned int cm_count = 0;
1215	struct mps_command *cm;
1216	struct mpssas_target *targ;
1217
1218	callout_stop(&tm->cm_callout);
1219
1220	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1221	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1222	targ = tm->cm_targ;
1223
1224	/*
1225	 * Currently there should be no way we can hit this case.  It only
1226	 * happens when we have a failure to allocate chain frames, and
1227	 * task management commands don't have S/G lists.
1228	 * XXXSL So should it be an assertion?
1229	 */
1230	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1231		mps_dprint(sc, MPS_RECOVERY|MPS_ERROR,
1232		    "%s: cm_flags = %#x for LUN reset! "
1233		   "This should not happen!\n", __func__, tm->cm_flags);
1234		mpssas_free_tm(sc, tm);
1235		return;
1236	}
1237
1238	if (reply == NULL) {
1239		mps_dprint(sc, MPS_RECOVERY, "NULL reset reply for tm %p\n",
1240		    tm);
1241		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1242			/* this completion was due to a reset, just cleanup */
1243			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1244			    "reset, ignoring NULL LUN reset reply\n");
1245			targ->tm = NULL;
1246			mpssas_free_tm(sc, tm);
1247		}
1248		else {
1249			/* we should have gotten a reply. */
1250			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1251			    "LUN reset attempt, resetting controller\n");
1252			mps_reinit(sc);
1253		}
1254		return;
1255	}
1256
1257	mps_dprint(sc, MPS_RECOVERY,
1258	    "logical unit reset status 0x%x code 0x%x count %u\n",
1259	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1260	    le32toh(reply->TerminationCount));
1261
1262	/*
1263	 * See if there are any outstanding commands for this LUN.
1264	 * This could be made more efficient by using a per-LU data
1265	 * structure of some sort.
1266	 */
1267	TAILQ_FOREACH(cm, &targ->commands, cm_link) {
1268		if (cm->cm_lun == tm->cm_lun)
1269			cm_count++;
1270	}
1271
1272	if (cm_count == 0) {
1273		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1274		    "Finished recovery after LUN reset for target %u\n",
1275		    targ->tid);
1276
1277		mpssas_announce_reset(sc, AC_SENT_BDR, targ->tid, tm->cm_lun);
1278
1279		/*
1280		 * We've finished recovery for this logical unit.  check and
1281		 * see if some other logical unit has a timedout command
1282		 * that needs to be processed.
1283		 */
1284		cm = TAILQ_FIRST(&targ->timedout_commands);
1285		if (cm) {
1286			mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1287			    "More commands to abort for target %u\n",
1288			    targ->tid);
1289			mpssas_send_abort(sc, tm, cm);
1290		} else {
1291			targ->tm = NULL;
1292			mpssas_free_tm(sc, tm);
1293		}
1294	} else {
1295		/*
1296		 * If we still have commands for this LUN, the reset
1297		 * effectively failed, regardless of the status reported.
1298		 * Escalate to a target reset.
1299		 */
1300		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1301		    "logical unit reset complete for target %u, but still "
1302		    "have %u command(s), sending target reset\n", targ->tid,
1303		    cm_count);
1304		mpssas_send_reset(sc, tm,
1305		    MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
1306	}
1307}
1308
1309static void
1310mpssas_target_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1311{
1312	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1313	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1314	struct mpssas_target *targ;
1315
1316	callout_stop(&tm->cm_callout);
1317
1318	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1319	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1320	targ = tm->cm_targ;
1321
1322	/*
1323	 * Currently there should be no way we can hit this case.  It only
1324	 * happens when we have a failure to allocate chain frames, and
1325	 * task management commands don't have S/G lists.
1326	 */
1327	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1328		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x for target reset! "
1329			   "This should not happen!\n", __func__, tm->cm_flags);
1330		mpssas_free_tm(sc, tm);
1331		return;
1332	}
1333
1334	if (reply == NULL) {
1335		mps_dprint(sc, MPS_RECOVERY,
1336		    "NULL target reset reply for tm %pi TaskMID %u\n",
1337		    tm, le16toh(req->TaskMID));
1338		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1339			/* this completion was due to a reset, just cleanup */
1340			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1341			    "reset, ignoring NULL target reset reply\n");
1342			targ->tm = NULL;
1343			mpssas_free_tm(sc, tm);
1344		} else {
1345			/* we should have gotten a reply. */
1346			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1347			    "target reset attempt, resetting controller\n");
1348			mps_reinit(sc);
1349		}
1350		return;
1351	}
1352
1353	mps_dprint(sc, MPS_RECOVERY,
1354	    "target reset status 0x%x code 0x%x count %u\n",
1355	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1356	    le32toh(reply->TerminationCount));
1357
1358	if (targ->outstanding == 0) {
1359		/* we've finished recovery for this target and all
1360		 * of its logical units.
1361		 */
1362		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1363		    "Finished reset recovery for target %u\n", targ->tid);
1364
1365		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1366		    CAM_LUN_WILDCARD);
1367
1368		targ->tm = NULL;
1369		mpssas_free_tm(sc, tm);
1370	} else {
1371		/*
1372		 * After a target reset, if this target still has
1373		 * outstanding commands, the reset effectively failed,
1374		 * regardless of the status reported.  escalate.
1375		 */
1376		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1377		    "Target reset complete for target %u, but still have %u "
1378		    "command(s), resetting controller\n", targ->tid,
1379		    targ->outstanding);
1380		mps_reinit(sc);
1381	}
1382}
1383
1384#define MPS_RESET_TIMEOUT 30
1385
1386int
1387mpssas_send_reset(struct mps_softc *sc, struct mps_command *tm, uint8_t type)
1388{
1389	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1390	struct mpssas_target *target;
1391	int err;
1392
1393	target = tm->cm_targ;
1394	if (target->handle == 0) {
1395		mps_dprint(sc, MPS_ERROR,"%s null devhandle for target_id %d\n",
1396		    __func__, target->tid);
1397		return -1;
1398	}
1399
1400	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1401	req->DevHandle = htole16(target->handle);
1402	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1403	req->TaskType = type;
1404
1405	if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) {
1406		/* XXX Need to handle invalid LUNs */
1407		MPS_SET_LUN(req->LUN, tm->cm_lun);
1408		tm->cm_targ->logical_unit_resets++;
1409		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1410		    "Sending logical unit reset to target %u lun %d\n",
1411		    target->tid, tm->cm_lun);
1412		tm->cm_complete = mpssas_logical_unit_reset_complete;
1413		mpssas_prepare_for_tm(sc, tm, target, tm->cm_lun);
1414	} else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) {
1415		/*
1416		 * Target reset method =
1417		 * 	SAS Hard Link Reset / SATA Link Reset
1418		 */
1419		req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
1420		tm->cm_targ->target_resets++;
1421		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1422		    "Sending target reset to target %u\n", target->tid);
1423		tm->cm_complete = mpssas_target_reset_complete;
1424		mpssas_prepare_for_tm(sc, tm, target, CAM_LUN_WILDCARD);
1425	} else {
1426		mps_dprint(sc, MPS_ERROR, "unexpected reset type 0x%x\n", type);
1427		return -1;
1428	}
1429
1430	tm->cm_data = NULL;
1431	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1432	tm->cm_complete_data = (void *)tm;
1433
1434	callout_reset(&tm->cm_callout, MPS_RESET_TIMEOUT * hz,
1435	    mpssas_tm_timeout, tm);
1436
1437	err = mps_map_command(sc, tm);
1438	if (err)
1439		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1440		    "error %d sending reset type %u\n",
1441		    err, type);
1442
1443	return err;
1444}
1445
1446
1447static void
1448mpssas_abort_complete(struct mps_softc *sc, struct mps_command *tm)
1449{
1450	struct mps_command *cm;
1451	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1452	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1453	struct mpssas_target *targ;
1454
1455	callout_stop(&tm->cm_callout);
1456
1457	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1458	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1459	targ = tm->cm_targ;
1460
1461	/*
1462	 * Currently there should be no way we can hit this case.  It only
1463	 * happens when we have a failure to allocate chain frames, and
1464	 * task management commands don't have S/G lists.
1465	 */
1466	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1467		mps_dprint(sc, MPS_RECOVERY,
1468		    "cm_flags = %#x for abort %p TaskMID %u!\n",
1469		    tm->cm_flags, tm, le16toh(req->TaskMID));
1470		mpssas_free_tm(sc, tm);
1471		return;
1472	}
1473
1474	if (reply == NULL) {
1475		mps_dprint(sc, MPS_RECOVERY,
1476		    "NULL abort reply for tm %p TaskMID %u\n",
1477		    tm, le16toh(req->TaskMID));
1478		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1479			/* this completion was due to a reset, just cleanup */
1480			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1481			    "reset, ignoring NULL abort reply\n");
1482			targ->tm = NULL;
1483			mpssas_free_tm(sc, tm);
1484		} else {
1485			/* we should have gotten a reply. */
1486			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1487			    "abort attempt, resetting controller\n");
1488			mps_reinit(sc);
1489		}
1490		return;
1491	}
1492
1493	mps_dprint(sc, MPS_RECOVERY,
1494	    "abort TaskMID %u status 0x%x code 0x%x count %u\n",
1495	    le16toh(req->TaskMID),
1496	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1497	    le32toh(reply->TerminationCount));
1498
1499	cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands);
1500	if (cm == NULL) {
1501		/*
1502		 * If there are no more timedout commands, we're done with
1503		 * error recovery for this target.
1504		 */
1505		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1506		    "Finished abort recovery for target %u\n", targ->tid);
1507
1508		targ->tm = NULL;
1509		mpssas_free_tm(sc, tm);
1510	} else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) {
1511		/* abort success, but we have more timedout commands to abort */
1512		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1513		    "Continuing abort recovery for target %u\n", targ->tid);
1514
1515		mpssas_send_abort(sc, tm, cm);
1516	} else {
1517		/* we didn't get a command completion, so the abort
1518		 * failed as far as we're concerned.  escalate.
1519		 */
1520		mps_dprint(sc, MPS_RECOVERY,
1521		    "Abort failed for target %u, sending logical unit reset\n",
1522		    targ->tid);
1523
1524		mpssas_send_reset(sc, tm,
1525		    MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET);
1526	}
1527}
1528
1529#define MPS_ABORT_TIMEOUT 5
1530
1531static int
1532mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm, struct mps_command *cm)
1533{
1534	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1535	struct mpssas_target *targ;
1536	int err;
1537
1538	targ = cm->cm_targ;
1539	if (targ->handle == 0) {
1540		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1541		    "%s null devhandle for target_id %d\n",
1542		    __func__, cm->cm_ccb->ccb_h.target_id);
1543		return -1;
1544	}
1545
1546	mpssas_log_command(cm, MPS_RECOVERY|MPS_INFO,
1547	    "Aborting command %p\n", cm);
1548
1549	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1550	req->DevHandle = htole16(targ->handle);
1551	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1552	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK;
1553
1554	/* XXX Need to handle invalid LUNs */
1555	MPS_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun);
1556
1557	req->TaskMID = htole16(cm->cm_desc.Default.SMID);
1558
1559	tm->cm_data = NULL;
1560	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1561	tm->cm_complete = mpssas_abort_complete;
1562	tm->cm_complete_data = (void *)tm;
1563	tm->cm_targ = cm->cm_targ;
1564	tm->cm_lun = cm->cm_lun;
1565
1566	callout_reset(&tm->cm_callout, MPS_ABORT_TIMEOUT * hz,
1567	    mpssas_tm_timeout, tm);
1568
1569	targ->aborts++;
1570
1571	mpssas_prepare_for_tm(sc, tm, targ, tm->cm_lun);
1572
1573	err = mps_map_command(sc, tm);
1574	if (err)
1575		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1576		    "error %d sending abort for cm %p SMID %u\n",
1577		    err, cm, req->TaskMID);
1578	return err;
1579}
1580
1581static void
1582mpssas_scsiio_timeout(void *data)
1583{
1584	sbintime_t elapsed, now;
1585	union ccb *ccb;
1586	struct mps_softc *sc;
1587	struct mps_command *cm;
1588	struct mpssas_target *targ;
1589
1590	cm = (struct mps_command *)data;
1591	sc = cm->cm_sc;
1592	ccb = cm->cm_ccb;
1593	now = sbinuptime();
1594
1595	MPS_FUNCTRACE(sc);
1596	mtx_assert(&sc->mps_mtx, MA_OWNED);
1597
1598	mps_dprint(sc, MPS_XINFO|MPS_RECOVERY, "Timeout checking cm %p\n", sc);
1599
1600	/*
1601	 * Run the interrupt handler to make sure it's not pending.  This
1602	 * isn't perfect because the command could have already completed
1603	 * and been re-used, though this is unlikely.
1604	 */
1605	mps_intr_locked(sc);
1606	if (cm->cm_state != MPS_CM_STATE_INQUEUE) {
1607		mpssas_log_command(cm, MPS_XINFO,
1608		    "SCSI command %p almost timed out\n", cm);
1609		return;
1610	}
1611
1612	if (cm->cm_ccb == NULL) {
1613		mps_dprint(sc, MPS_ERROR, "command timeout with NULL ccb\n");
1614		return;
1615	}
1616
1617	targ = cm->cm_targ;
1618	targ->timeouts++;
1619
1620	elapsed = now - ccb->ccb_h.qos.sim_data;
1621	mpssas_log_command(cm, MPS_INFO|MPS_RECOVERY,
1622	    "Command timeout on target %u(0x%04x) %d set, %d.%d elapsed\n",
1623	    targ->tid, targ->handle, ccb->ccb_h.timeout,
1624	    sbintime_getsec(elapsed), elapsed & 0xffffffff);
1625
1626	/* XXX first, check the firmware state, to see if it's still
1627	 * operational.  if not, do a diag reset.
1628	 */
1629	mpssas_set_ccbstatus(cm->cm_ccb, CAM_CMD_TIMEOUT);
1630	cm->cm_state = MPS_CM_STATE_TIMEDOUT;
1631	TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery);
1632
1633	if (targ->tm != NULL) {
1634		/* target already in recovery, just queue up another
1635		 * timedout command to be processed later.
1636		 */
1637		mps_dprint(sc, MPS_RECOVERY,
1638		    "queued timedout cm %p for processing by tm %p\n",
1639		    cm, targ->tm);
1640	} else if ((targ->tm = mpssas_alloc_tm(sc)) != NULL) {
1641		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1642		    "Sending abort to target %u for SMID %d\n", targ->tid,
1643		    cm->cm_desc.Default.SMID);
1644		mps_dprint(sc, MPS_RECOVERY, "timedout cm %p allocated tm %p\n",
1645		    cm, targ->tm);
1646
1647		/* start recovery by aborting the first timedout command */
1648		mpssas_send_abort(sc, targ->tm, cm);
1649	} else {
1650		/* XXX queue this target up for recovery once a TM becomes
1651		 * available.  The firmware only has a limited number of
1652		 * HighPriority credits for the high priority requests used
1653		 * for task management, and we ran out.
1654		 *
1655		 * Isilon: don't worry about this for now, since we have
1656		 * more credits than disks in an enclosure, and limit
1657		 * ourselves to one TM per target for recovery.
1658		 */
1659		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1660		    "timedout cm %p failed to allocate a tm\n", cm);
1661	}
1662
1663}
1664
1665static void
1666mpssas_action_scsiio(struct mpssas_softc *sassc, union ccb *ccb)
1667{
1668	MPI2_SCSI_IO_REQUEST *req;
1669	struct ccb_scsiio *csio;
1670	struct mps_softc *sc;
1671	struct mpssas_target *targ;
1672	struct mpssas_lun *lun;
1673	struct mps_command *cm;
1674	uint8_t i, lba_byte, *ref_tag_addr;
1675	uint16_t eedp_flags;
1676	uint32_t mpi_control;
1677
1678	sc = sassc->sc;
1679	MPS_FUNCTRACE(sc);
1680	mtx_assert(&sc->mps_mtx, MA_OWNED);
1681
1682	csio = &ccb->csio;
1683	KASSERT(csio->ccb_h.target_id < sassc->maxtargets,
1684	    ("Target %d out of bounds in XPT_SCSI_IO\n",
1685	     csio->ccb_h.target_id));
1686	targ = &sassc->targets[csio->ccb_h.target_id];
1687	mps_dprint(sc, MPS_TRACE, "ccb %p target flag %x\n", ccb, targ->flags);
1688	if (targ->handle == 0x0) {
1689		mps_dprint(sc, MPS_ERROR, "%s NULL handle for target %u\n",
1690		    __func__, csio->ccb_h.target_id);
1691		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1692		xpt_done(ccb);
1693		return;
1694	}
1695	if (targ->flags & MPS_TARGET_FLAGS_RAID_COMPONENT) {
1696		mps_dprint(sc, MPS_ERROR, "%s Raid component no SCSI IO "
1697		    "supported %u\n", __func__, csio->ccb_h.target_id);
1698		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1699		xpt_done(ccb);
1700		return;
1701	}
1702	/*
1703	 * Sometimes, it is possible to get a command that is not "In
1704	 * Progress" and was actually aborted by the upper layer.  Check for
1705	 * this here and complete the command without error.
1706	 */
1707	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_INPROG) {
1708		mps_dprint(sc, MPS_TRACE, "%s Command is not in progress for "
1709		    "target %u\n", __func__, csio->ccb_h.target_id);
1710		xpt_done(ccb);
1711		return;
1712	}
1713	/*
1714	 * If devinfo is 0 this will be a volume.  In that case don't tell CAM
1715	 * that the volume has timed out.  We want volumes to be enumerated
1716	 * until they are deleted/removed, not just failed.
1717	 */
1718	if (targ->flags & MPSSAS_TARGET_INREMOVAL) {
1719		if (targ->devinfo == 0)
1720			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1721		else
1722			mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
1723		xpt_done(ccb);
1724		return;
1725	}
1726
1727	if ((sc->mps_flags & MPS_FLAGS_SHUTDOWN) != 0) {
1728		mps_dprint(sc, MPS_INFO, "%s shutting down\n", __func__);
1729		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1730		xpt_done(ccb);
1731		return;
1732	}
1733
1734	/*
1735	 * If target has a reset in progress, freeze the devq and return.  The
1736	 * devq will be released when the TM reset is finished.
1737	 */
1738	if (targ->flags & MPSSAS_TARGET_INRESET) {
1739		ccb->ccb_h.status = CAM_BUSY | CAM_DEV_QFRZN;
1740		mps_dprint(sc, MPS_INFO, "%s: Freezing devq for target ID %d\n",
1741		    __func__, targ->tid);
1742		xpt_freeze_devq(ccb->ccb_h.path, 1);
1743		xpt_done(ccb);
1744		return;
1745	}
1746
1747	cm = mps_alloc_command(sc);
1748	if (cm == NULL || (sc->mps_flags & MPS_FLAGS_DIAGRESET)) {
1749		if (cm != NULL) {
1750			mps_free_command(sc, cm);
1751		}
1752		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
1753			xpt_freeze_simq(sassc->sim, 1);
1754			sassc->flags |= MPSSAS_QUEUE_FROZEN;
1755		}
1756		ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1757		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1758		xpt_done(ccb);
1759		return;
1760	}
1761
1762	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
1763	bzero(req, sizeof(*req));
1764	req->DevHandle = htole16(targ->handle);
1765	req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
1766	req->MsgFlags = 0;
1767	req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);
1768	req->SenseBufferLength = MPS_SENSE_LEN;
1769	req->SGLFlags = 0;
1770	req->ChainOffset = 0;
1771	req->SGLOffset0 = 24;	/* 32bit word offset to the SGL */
1772	req->SGLOffset1= 0;
1773	req->SGLOffset2= 0;
1774	req->SGLOffset3= 0;
1775	req->SkipCount = 0;
1776	req->DataLength = htole32(csio->dxfer_len);
1777	req->BidirectionalDataLength = 0;
1778	req->IoFlags = htole16(csio->cdb_len);
1779	req->EEDPFlags = 0;
1780
1781	/* Note: BiDirectional transfers are not supported */
1782	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1783	case CAM_DIR_IN:
1784		mpi_control = MPI2_SCSIIO_CONTROL_READ;
1785		cm->cm_flags |= MPS_CM_FLAGS_DATAIN;
1786		break;
1787	case CAM_DIR_OUT:
1788		mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
1789		cm->cm_flags |= MPS_CM_FLAGS_DATAOUT;
1790		break;
1791	case CAM_DIR_NONE:
1792	default:
1793		mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
1794		break;
1795	}
1796
1797	if (csio->cdb_len == 32)
1798                mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
1799	/*
1800	 * It looks like the hardware doesn't require an explicit tag
1801	 * number for each transaction.  SAM Task Management not supported
1802	 * at the moment.
1803	 */
1804	switch (csio->tag_action) {
1805	case MSG_HEAD_OF_Q_TAG:
1806		mpi_control |= MPI2_SCSIIO_CONTROL_HEADOFQ;
1807		break;
1808	case MSG_ORDERED_Q_TAG:
1809		mpi_control |= MPI2_SCSIIO_CONTROL_ORDEREDQ;
1810		break;
1811	case MSG_ACA_TASK:
1812		mpi_control |= MPI2_SCSIIO_CONTROL_ACAQ;
1813		break;
1814	case CAM_TAG_ACTION_NONE:
1815	case MSG_SIMPLE_Q_TAG:
1816	default:
1817		mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
1818		break;
1819	}
1820	mpi_control |= sc->mapping_table[csio->ccb_h.target_id].TLR_bits;
1821	req->Control = htole32(mpi_control);
1822	if (MPS_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) {
1823		mps_free_command(sc, cm);
1824		mpssas_set_ccbstatus(ccb, CAM_LUN_INVALID);
1825		xpt_done(ccb);
1826		return;
1827	}
1828
1829	if (csio->ccb_h.flags & CAM_CDB_POINTER)
1830		bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
1831	else
1832		bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
1833	req->IoFlags = htole16(csio->cdb_len);
1834
1835	/*
1836	 * Check if EEDP is supported and enabled.  If it is then check if the
1837	 * SCSI opcode could be using EEDP.  If so, make sure the LUN exists and
1838	 * is formatted for EEDP support.  If all of this is true, set CDB up
1839	 * for EEDP transfer.
1840	 */
1841	eedp_flags = op_code_prot[req->CDB.CDB32[0]];
1842	if (sc->eedp_enabled && eedp_flags) {
1843		SLIST_FOREACH(lun, &targ->luns, lun_link) {
1844			if (lun->lun_id == csio->ccb_h.target_lun) {
1845				break;
1846			}
1847		}
1848
1849		if ((lun != NULL) && (lun->eedp_formatted)) {
1850			req->EEDPBlockSize = htole16(lun->eedp_block_size);
1851			eedp_flags |= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
1852			    MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
1853			    MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
1854			req->EEDPFlags = htole16(eedp_flags);
1855
1856			/*
1857			 * If CDB less than 32, fill in Primary Ref Tag with
1858			 * low 4 bytes of LBA.  If CDB is 32, tag stuff is
1859			 * already there.  Also, set protection bit.  FreeBSD
1860			 * currently does not support CDBs bigger than 16, but
1861			 * the code doesn't hurt, and will be here for the
1862			 * future.
1863			 */
1864			if (csio->cdb_len != 32) {
1865				lba_byte = (csio->cdb_len == 16) ? 6 : 2;
1866				ref_tag_addr = (uint8_t *)&req->CDB.EEDP32.
1867				    PrimaryReferenceTag;
1868				for (i = 0; i < 4; i++) {
1869					*ref_tag_addr =
1870					    req->CDB.CDB32[lba_byte + i];
1871					ref_tag_addr++;
1872				}
1873				req->CDB.EEDP32.PrimaryReferenceTag =
1874					htole32(req->CDB.EEDP32.PrimaryReferenceTag);
1875				req->CDB.EEDP32.PrimaryApplicationTagMask =
1876				    0xFFFF;
1877				req->CDB.CDB32[1] = (req->CDB.CDB32[1] & 0x1F) |
1878				    0x20;
1879			} else {
1880				eedp_flags |=
1881				    MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG;
1882				req->EEDPFlags = htole16(eedp_flags);
1883				req->CDB.CDB32[10] = (req->CDB.CDB32[10] &
1884				    0x1F) | 0x20;
1885			}
1886		}
1887	}
1888
1889	cm->cm_length = csio->dxfer_len;
1890	if (cm->cm_length != 0) {
1891		cm->cm_data = ccb;
1892		cm->cm_flags |= MPS_CM_FLAGS_USE_CCB;
1893	} else {
1894		cm->cm_data = NULL;
1895	}
1896	cm->cm_sge = &req->SGL;
1897	cm->cm_sglsize = (32 - 24) * 4;
1898	cm->cm_desc.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
1899	cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
1900	cm->cm_complete = mpssas_scsiio_complete;
1901	cm->cm_complete_data = ccb;
1902	cm->cm_targ = targ;
1903	cm->cm_lun = csio->ccb_h.target_lun;
1904	cm->cm_ccb = ccb;
1905
1906	/*
1907	 * If HBA is a WD and the command is not for a retry, try to build a
1908	 * direct I/O message. If failed, or the command is for a retry, send
1909	 * the I/O to the IR volume itself.
1910	 */
1911	if (sc->WD_valid_config) {
1912		if (ccb->ccb_h.sim_priv.entries[0].field == MPS_WD_RETRY) {
1913			mpssas_direct_drive_io(sassc, cm, ccb);
1914		} else {
1915			mpssas_set_ccbstatus(ccb, CAM_REQ_INPROG);
1916		}
1917	}
1918
1919#if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
1920	if (csio->bio != NULL)
1921		biotrack(csio->bio, __func__);
1922#endif
1923	csio->ccb_h.qos.sim_data = sbinuptime();
1924	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
1925	    mpssas_scsiio_timeout, cm, 0);
1926
1927	targ->issued++;
1928	targ->outstanding++;
1929	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
1930	ccb->ccb_h.status |= CAM_SIM_QUEUED;
1931
1932	mpssas_log_command(cm, MPS_XINFO, "%s cm %p ccb %p outstanding %u\n",
1933	    __func__, cm, ccb, targ->outstanding);
1934
1935	mps_map_command(sc, cm);
1936	return;
1937}
1938
1939/**
1940 * mps_sc_failed_io_info - translated non-succesfull SCSI_IO request
1941 */
1942static void
1943mps_sc_failed_io_info(struct mps_softc *sc, struct ccb_scsiio *csio,
1944    Mpi2SCSIIOReply_t *mpi_reply)
1945{
1946	u32 response_info;
1947	u8 *response_bytes;
1948	u16 ioc_status = le16toh(mpi_reply->IOCStatus) &
1949	    MPI2_IOCSTATUS_MASK;
1950	u8 scsi_state = mpi_reply->SCSIState;
1951	u8 scsi_status = mpi_reply->SCSIStatus;
1952	u32 log_info = le32toh(mpi_reply->IOCLogInfo);
1953	const char *desc_ioc_state, *desc_scsi_status;
1954
1955	if (log_info == 0x31170000)
1956		return;
1957
1958	desc_ioc_state = mps_describe_table(mps_iocstatus_string,
1959	    ioc_status);
1960	desc_scsi_status = mps_describe_table(mps_scsi_status_string,
1961	    scsi_status);
1962
1963	mps_dprint(sc, MPS_XINFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x)\n",
1964	    le16toh(mpi_reply->DevHandle), desc_ioc_state, ioc_status);
1965
1966	/*
1967	 *We can add more detail about underflow data here
1968	 * TO-DO
1969	 */
1970	mps_dprint(sc, MPS_XINFO, "\tscsi_status(%s)(0x%02x), "
1971	    "scsi_state %b\n", desc_scsi_status, scsi_status,
1972	    scsi_state, "\20" "\1AutosenseValid" "\2AutosenseFailed"
1973	    "\3NoScsiStatus" "\4Terminated" "\5Response InfoValid");
1974
1975	if (sc->mps_debug & MPS_XINFO &&
1976		scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
1977		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : Start :\n");
1978		scsi_sense_print(csio);
1979		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : End :\n");
1980	}
1981
1982	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
1983		response_info = le32toh(mpi_reply->ResponseInfo);
1984		response_bytes = (u8 *)&response_info;
1985		mps_dprint(sc, MPS_XINFO, "response code(0x%1x): %s\n",
1986		    response_bytes[0],
1987		    mps_describe_table(mps_scsi_taskmgmt_string,
1988		    response_bytes[0]));
1989	}
1990}
1991
1992static void
1993mpssas_scsiio_complete(struct mps_softc *sc, struct mps_command *cm)
1994{
1995	MPI2_SCSI_IO_REPLY *rep;
1996	union ccb *ccb;
1997	struct ccb_scsiio *csio;
1998	struct mpssas_softc *sassc;
1999	struct scsi_vpd_supported_page_list *vpd_list = NULL;
2000	u8 *TLR_bits, TLR_on;
2001	int dir = 0, i;
2002	u16 alloc_len;
2003	struct mpssas_target *target;
2004	target_id_t target_id;
2005
2006	MPS_FUNCTRACE(sc);
2007	mps_dprint(sc, MPS_TRACE,
2008	    "cm %p SMID %u ccb %p reply %p outstanding %u\n", cm,
2009	    cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply,
2010	    cm->cm_targ->outstanding);
2011
2012	callout_stop(&cm->cm_callout);
2013	mtx_assert(&sc->mps_mtx, MA_OWNED);
2014
2015	sassc = sc->sassc;
2016	ccb = cm->cm_complete_data;
2017	csio = &ccb->csio;
2018	target_id = csio->ccb_h.target_id;
2019	rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply;
2020	/*
2021	 * XXX KDM if the chain allocation fails, does it matter if we do
2022	 * the sync and unload here?  It is simpler to do it in every case,
2023	 * assuming it doesn't cause problems.
2024	 */
2025	if (cm->cm_data != NULL) {
2026		if (cm->cm_flags & MPS_CM_FLAGS_DATAIN)
2027			dir = BUS_DMASYNC_POSTREAD;
2028		else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT)
2029			dir = BUS_DMASYNC_POSTWRITE;
2030		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
2031		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2032	}
2033
2034	cm->cm_targ->completed++;
2035	cm->cm_targ->outstanding--;
2036	TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link);
2037	ccb->ccb_h.status &= ~(CAM_STATUS_MASK | CAM_SIM_QUEUED);
2038
2039#if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
2040	if (ccb->csio.bio != NULL)
2041		biotrack(ccb->csio.bio, __func__);
2042#endif
2043
2044	if (cm->cm_state == MPS_CM_STATE_TIMEDOUT) {
2045		TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery);
2046		cm->cm_state = MPS_CM_STATE_BUSY;
2047		if (cm->cm_reply != NULL)
2048			mpssas_log_command(cm, MPS_RECOVERY,
2049			    "completed timedout cm %p ccb %p during recovery "
2050			    "ioc %x scsi %x state %x xfer %u\n",
2051			    cm, cm->cm_ccb, le16toh(rep->IOCStatus),
2052			    rep->SCSIStatus, rep->SCSIState,
2053			    le32toh(rep->TransferCount));
2054		else
2055			mpssas_log_command(cm, MPS_RECOVERY,
2056			    "completed timedout cm %p ccb %p during recovery\n",
2057			    cm, cm->cm_ccb);
2058	} else if (cm->cm_targ->tm != NULL) {
2059		if (cm->cm_reply != NULL)
2060			mpssas_log_command(cm, MPS_RECOVERY,
2061			    "completed cm %p ccb %p during recovery "
2062			    "ioc %x scsi %x state %x xfer %u\n",
2063			    cm, cm->cm_ccb, le16toh(rep->IOCStatus),
2064			    rep->SCSIStatus, rep->SCSIState,
2065			    le32toh(rep->TransferCount));
2066		else
2067			mpssas_log_command(cm, MPS_RECOVERY,
2068			    "completed cm %p ccb %p during recovery\n",
2069			    cm, cm->cm_ccb);
2070	} else if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
2071		mpssas_log_command(cm, MPS_RECOVERY,
2072		    "reset completed cm %p ccb %p\n",
2073		    cm, cm->cm_ccb);
2074	}
2075
2076	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2077		/*
2078		 * We ran into an error after we tried to map the command,
2079		 * so we're getting a callback without queueing the command
2080		 * to the hardware.  So we set the status here, and it will
2081		 * be retained below.  We'll go through the "fast path",
2082		 * because there can be no reply when we haven't actually
2083		 * gone out to the hardware.
2084		 */
2085		mpssas_set_ccbstatus(ccb, CAM_REQUEUE_REQ);
2086
2087		/*
2088		 * Currently the only error included in the mask is
2089		 * MPS_CM_FLAGS_CHAIN_FAILED, which means we're out of
2090		 * chain frames.  We need to freeze the queue until we get
2091		 * a command that completed without this error, which will
2092		 * hopefully have some chain frames attached that we can
2093		 * use.  If we wanted to get smarter about it, we would
2094		 * only unfreeze the queue in this condition when we're
2095		 * sure that we're getting some chain frames back.  That's
2096		 * probably unnecessary.
2097		 */
2098		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
2099			xpt_freeze_simq(sassc->sim, 1);
2100			sassc->flags |= MPSSAS_QUEUE_FROZEN;
2101			mps_dprint(sc, MPS_XINFO, "Error sending command, "
2102				   "freezing SIM queue\n");
2103		}
2104	}
2105
2106	/*
2107	 * If this is a Start Stop Unit command and it was issued by the driver
2108	 * during shutdown, decrement the refcount to account for all of the
2109	 * commands that were sent.  All SSU commands should be completed before
2110	 * shutdown completes, meaning SSU_refcount will be 0 after SSU_started
2111	 * is TRUE.
2112	 */
2113	if (sc->SSU_started && (csio->cdb_io.cdb_bytes[0] == START_STOP_UNIT)) {
2114		mps_dprint(sc, MPS_INFO, "Decrementing SSU count.\n");
2115		sc->SSU_refcount--;
2116	}
2117
2118	/* Take the fast path to completion */
2119	if (cm->cm_reply == NULL) {
2120		if (mpssas_get_ccbstatus(ccb) == CAM_REQ_INPROG) {
2121			if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0)
2122				mpssas_set_ccbstatus(ccb, CAM_SCSI_BUS_RESET);
2123			else {
2124				mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2125				ccb->csio.scsi_status = SCSI_STATUS_OK;
2126			}
2127			if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2128				ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2129				sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2130				mps_dprint(sc, MPS_XINFO,
2131				    "Unfreezing SIM queue\n");
2132			}
2133		}
2134
2135		/*
2136		 * There are two scenarios where the status won't be
2137		 * CAM_REQ_CMP.  The first is if MPS_CM_FLAGS_ERROR_MASK is
2138		 * set, the second is in the MPS_FLAGS_DIAGRESET above.
2139		 */
2140		if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2141			/*
2142			 * Freeze the dev queue so that commands are
2143			 * executed in the correct order after error
2144			 * recovery.
2145			 */
2146			ccb->ccb_h.status |= CAM_DEV_QFRZN;
2147			xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2148		}
2149		mps_free_command(sc, cm);
2150		xpt_done(ccb);
2151		return;
2152	}
2153
2154	mpssas_log_command(cm, MPS_XINFO,
2155	    "ioc %x scsi %x state %x xfer %u\n",
2156	    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2157	    le32toh(rep->TransferCount));
2158
2159	/*
2160	 * If this is a Direct Drive I/O, reissue the I/O to the original IR
2161	 * Volume if an error occurred (normal I/O retry).  Use the original
2162	 * CCB, but set a flag that this will be a retry so that it's sent to
2163	 * the original volume.  Free the command but reuse the CCB.
2164	 */
2165	if (cm->cm_flags & MPS_CM_FLAGS_DD_IO) {
2166		mps_free_command(sc, cm);
2167		ccb->ccb_h.sim_priv.entries[0].field = MPS_WD_RETRY;
2168		mpssas_action_scsiio(sassc, ccb);
2169		return;
2170	} else
2171		ccb->ccb_h.sim_priv.entries[0].field = 0;
2172
2173	switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) {
2174	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
2175		csio->resid = cm->cm_length - le32toh(rep->TransferCount);
2176		/* FALLTHROUGH */
2177	case MPI2_IOCSTATUS_SUCCESS:
2178	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
2179
2180		if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
2181		    MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
2182			mpssas_log_command(cm, MPS_XINFO, "recovered error\n");
2183
2184		/* Completion failed at the transport level. */
2185		if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS |
2186		    MPI2_SCSI_STATE_TERMINATED)) {
2187			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2188			break;
2189		}
2190
2191		/* In a modern packetized environment, an autosense failure
2192		 * implies that there's not much else that can be done to
2193		 * recover the command.
2194		 */
2195		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) {
2196			mpssas_set_ccbstatus(ccb, CAM_AUTOSENSE_FAIL);
2197			break;
2198		}
2199
2200		/*
2201		 * CAM doesn't care about SAS Response Info data, but if this is
2202		 * the state check if TLR should be done.  If not, clear the
2203		 * TLR_bits for the target.
2204		 */
2205		if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) &&
2206		    ((le32toh(rep->ResponseInfo) &
2207		    MPI2_SCSI_RI_MASK_REASONCODE) ==
2208		    MPS_SCSI_RI_INVALID_FRAME)) {
2209			sc->mapping_table[target_id].TLR_bits =
2210			    (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2211		}
2212
2213		/*
2214		 * Intentionally override the normal SCSI status reporting
2215		 * for these two cases.  These are likely to happen in a
2216		 * multi-initiator environment, and we want to make sure that
2217		 * CAM retries these commands rather than fail them.
2218		 */
2219		if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) ||
2220		    (rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) {
2221			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2222			break;
2223		}
2224
2225		/* Handle normal status and sense */
2226		csio->scsi_status = rep->SCSIStatus;
2227		if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD)
2228			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2229		else
2230			mpssas_set_ccbstatus(ccb, CAM_SCSI_STATUS_ERROR);
2231
2232		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2233			int sense_len, returned_sense_len;
2234
2235			returned_sense_len = min(le32toh(rep->SenseCount),
2236			    sizeof(struct scsi_sense_data));
2237			if (returned_sense_len < ccb->csio.sense_len)
2238				ccb->csio.sense_resid = ccb->csio.sense_len -
2239					returned_sense_len;
2240			else
2241				ccb->csio.sense_resid = 0;
2242
2243			sense_len = min(returned_sense_len,
2244			    ccb->csio.sense_len - ccb->csio.sense_resid);
2245			bzero(&ccb->csio.sense_data,
2246			      sizeof(ccb->csio.sense_data));
2247			bcopy(cm->cm_sense, &ccb->csio.sense_data, sense_len);
2248			ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2249		}
2250
2251		/*
2252		 * Check if this is an INQUIRY command.  If it's a VPD inquiry,
2253		 * and it's page code 0 (Supported Page List), and there is
2254		 * inquiry data, and this is for a sequential access device, and
2255		 * the device is an SSP target, and TLR is supported by the
2256		 * controller, turn the TLR_bits value ON if page 0x90 is
2257		 * supported.
2258		 */
2259		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2260		    (csio->cdb_io.cdb_bytes[1] & SI_EVPD) &&
2261		    (csio->cdb_io.cdb_bytes[2] == SVPD_SUPPORTED_PAGE_LIST) &&
2262		    ((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) &&
2263		    (csio->data_ptr != NULL) &&
2264		    ((csio->data_ptr[0] & 0x1f) == T_SEQUENTIAL) &&
2265		    (sc->control_TLR) &&
2266		    (sc->mapping_table[target_id].device_info &
2267		    MPI2_SAS_DEVICE_INFO_SSP_TARGET)) {
2268			vpd_list = (struct scsi_vpd_supported_page_list *)
2269			    csio->data_ptr;
2270			TLR_bits = &sc->mapping_table[target_id].TLR_bits;
2271			*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2272			TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
2273			alloc_len = ((u16)csio->cdb_io.cdb_bytes[3] << 8) +
2274			    csio->cdb_io.cdb_bytes[4];
2275			alloc_len -= csio->resid;
2276			for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
2277				if (vpd_list->list[i] == 0x90) {
2278					*TLR_bits = TLR_on;
2279					break;
2280				}
2281			}
2282		}
2283
2284		/*
2285		 * If this is a SATA direct-access end device, mark it so that
2286		 * a SCSI StartStopUnit command will be sent to it when the
2287		 * driver is being shutdown.
2288		 */
2289		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2290		    ((csio->data_ptr[0] & 0x1f) == T_DIRECT) &&
2291		    (sc->mapping_table[target_id].device_info &
2292		    MPI2_SAS_DEVICE_INFO_SATA_DEVICE) &&
2293		    ((sc->mapping_table[target_id].device_info &
2294		    MPI2_SAS_DEVICE_INFO_MASK_DEVICE_TYPE) ==
2295		    MPI2_SAS_DEVICE_INFO_END_DEVICE)) {
2296			target = &sassc->targets[target_id];
2297			target->supports_SSU = TRUE;
2298			mps_dprint(sc, MPS_XINFO, "Target %d supports SSU\n",
2299			    target_id);
2300		}
2301		break;
2302	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
2303	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
2304		/*
2305		 * If devinfo is 0 this will be a volume.  In that case don't
2306		 * tell CAM that the volume is not there.  We want volumes to
2307		 * be enumerated until they are deleted/removed, not just
2308		 * failed.
2309		 */
2310		if (cm->cm_targ->devinfo == 0)
2311			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2312		else
2313			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2314		break;
2315	case MPI2_IOCSTATUS_INVALID_SGL:
2316		mps_print_scsiio_cmd(sc, cm);
2317		mpssas_set_ccbstatus(ccb, CAM_UNREC_HBA_ERROR);
2318		break;
2319	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
2320		/*
2321		 * This is one of the responses that comes back when an I/O
2322		 * has been aborted.  If it is because of a timeout that we
2323		 * initiated, just set the status to CAM_CMD_TIMEOUT.
2324		 * Otherwise set it to CAM_REQ_ABORTED.  The effect on the
2325		 * command is the same (it gets retried, subject to the
2326		 * retry counter), the only difference is what gets printed
2327		 * on the console.
2328		 */
2329		if (cm->cm_state == MPS_CM_STATE_TIMEDOUT)
2330			mpssas_set_ccbstatus(ccb, CAM_CMD_TIMEOUT);
2331		else
2332			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2333		break;
2334	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
2335		/* resid is ignored for this condition */
2336		csio->resid = 0;
2337		mpssas_set_ccbstatus(ccb, CAM_DATA_RUN_ERR);
2338		break;
2339	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
2340	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
2341		/*
2342		 * These can sometimes be transient transport-related
2343		 * errors, and sometimes persistent drive-related errors.
2344		 * We used to retry these without decrementing the retry
2345		 * count by returning CAM_REQUEUE_REQ.  Unfortunately, if
2346		 * we hit a persistent drive problem that returns one of
2347		 * these error codes, we would retry indefinitely.  So,
2348		 * return CAM_REQ_CMP_ERROR so that we decrement the retry
2349		 * count and avoid infinite retries.  We're taking the
2350		 * potential risk of flagging false failures in the event
2351		 * of a topology-related error (e.g. a SAS expander problem
2352		 * causes a command addressed to a drive to fail), but
2353		 * avoiding getting into an infinite retry loop.
2354		 */
2355		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2356		mps_dprint(sc, MPS_INFO,
2357		    "Controller reported %s tgt %u SMID %u loginfo %x\n",
2358		    mps_describe_table(mps_iocstatus_string,
2359		    le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK),
2360		    target_id, cm->cm_desc.Default.SMID,
2361		    le32toh(rep->IOCLogInfo));
2362		mps_dprint(sc, MPS_XINFO,
2363		    "SCSIStatus %x SCSIState %x xfercount %u\n",
2364		    rep->SCSIStatus, rep->SCSIState,
2365		    le32toh(rep->TransferCount));
2366		break;
2367	case MPI2_IOCSTATUS_INVALID_FUNCTION:
2368	case MPI2_IOCSTATUS_INTERNAL_ERROR:
2369	case MPI2_IOCSTATUS_INVALID_VPID:
2370	case MPI2_IOCSTATUS_INVALID_FIELD:
2371	case MPI2_IOCSTATUS_INVALID_STATE:
2372	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
2373	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
2374	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
2375	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
2376	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
2377	default:
2378		mpssas_log_command(cm, MPS_XINFO,
2379		    "completed ioc %x loginfo %x scsi %x state %x xfer %u\n",
2380		    le16toh(rep->IOCStatus), le32toh(rep->IOCLogInfo),
2381		    rep->SCSIStatus, rep->SCSIState,
2382		    le32toh(rep->TransferCount));
2383		csio->resid = cm->cm_length;
2384		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2385		break;
2386	}
2387
2388	mps_sc_failed_io_info(sc,csio,rep);
2389
2390	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2391		ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2392		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2393		mps_dprint(sc, MPS_XINFO, "Command completed, "
2394		    "unfreezing SIM queue\n");
2395	}
2396
2397	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2398		ccb->ccb_h.status |= CAM_DEV_QFRZN;
2399		xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2400	}
2401
2402	mps_free_command(sc, cm);
2403	xpt_done(ccb);
2404}
2405
2406/* All Request reached here are Endian safe */
2407static void
2408mpssas_direct_drive_io(struct mpssas_softc *sassc, struct mps_command *cm,
2409    union ccb *ccb) {
2410	pMpi2SCSIIORequest_t	pIO_req;
2411	struct mps_softc	*sc = sassc->sc;
2412	uint64_t		virtLBA;
2413	uint32_t		physLBA, stripe_offset, stripe_unit;
2414	uint32_t		io_size, column;
2415	uint8_t			*ptrLBA, lba_idx, physLBA_byte, *CDB;
2416
2417	/*
2418	 * If this is a valid SCSI command (Read6, Read10, Read16, Write6,
2419	 * Write10, or Write16), build a direct I/O message.  Otherwise, the I/O
2420	 * will be sent to the IR volume itself.  Since Read6 and Write6 are a
2421	 * bit different than the 10/16 CDBs, handle them separately.
2422	 */
2423	pIO_req = (pMpi2SCSIIORequest_t)cm->cm_req;
2424	CDB = pIO_req->CDB.CDB32;
2425
2426	/*
2427	 * Handle 6 byte CDBs.
2428	 */
2429	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_6) ||
2430	    (CDB[0] == WRITE_6))) {
2431		/*
2432		 * Get the transfer size in blocks.
2433		 */
2434		io_size = (cm->cm_length >> sc->DD_block_exponent);
2435
2436		/*
2437		 * Get virtual LBA given in the CDB.
2438		 */
2439		virtLBA = ((uint64_t)(CDB[1] & 0x1F) << 16) |
2440		    ((uint64_t)CDB[2] << 8) | (uint64_t)CDB[3];
2441
2442		/*
2443		 * Check that LBA range for I/O does not exceed volume's
2444		 * MaxLBA.
2445		 */
2446		if ((virtLBA + (uint64_t)io_size - 1) <=
2447		    sc->DD_max_lba) {
2448			/*
2449			 * Check if the I/O crosses a stripe boundary.  If not,
2450			 * translate the virtual LBA to a physical LBA and set
2451			 * the DevHandle for the PhysDisk to be used.  If it
2452			 * does cross a boundary, do normal I/O.  To get the
2453			 * right DevHandle to use, get the map number for the
2454			 * column, then use that map number to look up the
2455			 * DevHandle of the PhysDisk.
2456			 */
2457			stripe_offset = (uint32_t)virtLBA &
2458			    (sc->DD_stripe_size - 1);
2459			if ((stripe_offset + io_size) <= sc->DD_stripe_size) {
2460				physLBA = (uint32_t)virtLBA >>
2461				    sc->DD_stripe_exponent;
2462				stripe_unit = physLBA / sc->DD_num_phys_disks;
2463				column = physLBA % sc->DD_num_phys_disks;
2464				pIO_req->DevHandle =
2465				    htole16(sc->DD_column_map[column].dev_handle);
2466				/* ???? Is this endian safe*/
2467				cm->cm_desc.SCSIIO.DevHandle =
2468				    pIO_req->DevHandle;
2469
2470				physLBA = (stripe_unit <<
2471				    sc->DD_stripe_exponent) + stripe_offset;
2472				ptrLBA = &pIO_req->CDB.CDB32[1];
2473				physLBA_byte = (uint8_t)(physLBA >> 16);
2474				*ptrLBA = physLBA_byte;
2475				ptrLBA = &pIO_req->CDB.CDB32[2];
2476				physLBA_byte = (uint8_t)(physLBA >> 8);
2477				*ptrLBA = physLBA_byte;
2478				ptrLBA = &pIO_req->CDB.CDB32[3];
2479				physLBA_byte = (uint8_t)physLBA;
2480				*ptrLBA = physLBA_byte;
2481
2482				/*
2483				 * Set flag that Direct Drive I/O is
2484				 * being done.
2485				 */
2486				cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2487			}
2488		}
2489		return;
2490	}
2491
2492	/*
2493	 * Handle 10, 12 or 16 byte CDBs.
2494	 */
2495	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_10) ||
2496	    (CDB[0] == WRITE_10) || (CDB[0] == READ_16) ||
2497	    (CDB[0] == WRITE_16) || (CDB[0] == READ_12) ||
2498	    (CDB[0] == WRITE_12))) {
2499		/*
2500		 * For 16-byte CDB's, verify that the upper 4 bytes of the CDB
2501		 * are 0.  If not, this is accessing beyond 2TB so handle it in
2502		 * the else section.  10-byte and 12-byte CDB's are OK.
2503		 * FreeBSD sends very rare 12 byte READ/WRITE, but driver is
2504		 * ready to accept 12byte CDB for Direct IOs.
2505		 */
2506		if ((CDB[0] == READ_10 || CDB[0] == WRITE_10) ||
2507		    (CDB[0] == READ_12 || CDB[0] == WRITE_12) ||
2508		    !(CDB[2] | CDB[3] | CDB[4] | CDB[5])) {
2509			/*
2510			 * Get the transfer size in blocks.
2511			 */
2512			io_size = (cm->cm_length >> sc->DD_block_exponent);
2513
2514			/*
2515			 * Get virtual LBA.  Point to correct lower 4 bytes of
2516			 * LBA in the CDB depending on command.
2517			 */
2518			lba_idx = ((CDB[0] == READ_12) ||
2519				(CDB[0] == WRITE_12) ||
2520				(CDB[0] == READ_10) ||
2521				(CDB[0] == WRITE_10))? 2 : 6;
2522			virtLBA = ((uint64_t)CDB[lba_idx] << 24) |
2523			    ((uint64_t)CDB[lba_idx + 1] << 16) |
2524			    ((uint64_t)CDB[lba_idx + 2] << 8) |
2525			    (uint64_t)CDB[lba_idx + 3];
2526
2527			/*
2528			 * Check that LBA range for I/O does not exceed volume's
2529			 * MaxLBA.
2530			 */
2531			if ((virtLBA + (uint64_t)io_size - 1) <=
2532			    sc->DD_max_lba) {
2533				/*
2534				 * Check if the I/O crosses a stripe boundary.
2535				 * If not, translate the virtual LBA to a
2536				 * physical LBA and set the DevHandle for the
2537				 * PhysDisk to be used.  If it does cross a
2538				 * boundary, do normal I/O.  To get the right
2539				 * DevHandle to use, get the map number for the
2540				 * column, then use that map number to look up
2541				 * the DevHandle of the PhysDisk.
2542				 */
2543				stripe_offset = (uint32_t)virtLBA &
2544				    (sc->DD_stripe_size - 1);
2545				if ((stripe_offset + io_size) <=
2546				    sc->DD_stripe_size) {
2547					physLBA = (uint32_t)virtLBA >>
2548					    sc->DD_stripe_exponent;
2549					stripe_unit = physLBA /
2550					    sc->DD_num_phys_disks;
2551					column = physLBA %
2552					    sc->DD_num_phys_disks;
2553					pIO_req->DevHandle =
2554					    htole16(sc->DD_column_map[column].
2555					    dev_handle);
2556					cm->cm_desc.SCSIIO.DevHandle =
2557					    pIO_req->DevHandle;
2558
2559					physLBA = (stripe_unit <<
2560					    sc->DD_stripe_exponent) +
2561					    stripe_offset;
2562					ptrLBA =
2563					    &pIO_req->CDB.CDB32[lba_idx];
2564					physLBA_byte = (uint8_t)(physLBA >> 24);
2565					*ptrLBA = physLBA_byte;
2566					ptrLBA =
2567					    &pIO_req->CDB.CDB32[lba_idx + 1];
2568					physLBA_byte = (uint8_t)(physLBA >> 16);
2569					*ptrLBA = physLBA_byte;
2570					ptrLBA =
2571					    &pIO_req->CDB.CDB32[lba_idx + 2];
2572					physLBA_byte = (uint8_t)(physLBA >> 8);
2573					*ptrLBA = physLBA_byte;
2574					ptrLBA =
2575					    &pIO_req->CDB.CDB32[lba_idx + 3];
2576					physLBA_byte = (uint8_t)physLBA;
2577					*ptrLBA = physLBA_byte;
2578
2579					/*
2580					 * Set flag that Direct Drive I/O is
2581					 * being done.
2582					 */
2583					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2584				}
2585			}
2586		} else {
2587			/*
2588			 * 16-byte CDB and the upper 4 bytes of the CDB are not
2589			 * 0.  Get the transfer size in blocks.
2590			 */
2591			io_size = (cm->cm_length >> sc->DD_block_exponent);
2592
2593			/*
2594			 * Get virtual LBA.
2595			 */
2596			virtLBA = ((uint64_t)CDB[2] << 54) |
2597			    ((uint64_t)CDB[3] << 48) |
2598			    ((uint64_t)CDB[4] << 40) |
2599			    ((uint64_t)CDB[5] << 32) |
2600			    ((uint64_t)CDB[6] << 24) |
2601			    ((uint64_t)CDB[7] << 16) |
2602			    ((uint64_t)CDB[8] << 8) |
2603			    (uint64_t)CDB[9];
2604
2605			/*
2606			 * Check that LBA range for I/O does not exceed volume's
2607			 * MaxLBA.
2608			 */
2609			if ((virtLBA + (uint64_t)io_size - 1) <=
2610			    sc->DD_max_lba) {
2611				/*
2612				 * Check if the I/O crosses a stripe boundary.
2613				 * If not, translate the virtual LBA to a
2614				 * physical LBA and set the DevHandle for the
2615				 * PhysDisk to be used.  If it does cross a
2616				 * boundary, do normal I/O.  To get the right
2617				 * DevHandle to use, get the map number for the
2618				 * column, then use that map number to look up
2619				 * the DevHandle of the PhysDisk.
2620				 */
2621				stripe_offset = (uint32_t)virtLBA &
2622				    (sc->DD_stripe_size - 1);
2623				if ((stripe_offset + io_size) <=
2624				    sc->DD_stripe_size) {
2625					physLBA = (uint32_t)(virtLBA >>
2626					    sc->DD_stripe_exponent);
2627					stripe_unit = physLBA /
2628					    sc->DD_num_phys_disks;
2629					column = physLBA %
2630					    sc->DD_num_phys_disks;
2631					pIO_req->DevHandle =
2632					    htole16(sc->DD_column_map[column].
2633					    dev_handle);
2634					cm->cm_desc.SCSIIO.DevHandle =
2635					    pIO_req->DevHandle;
2636
2637					physLBA = (stripe_unit <<
2638					    sc->DD_stripe_exponent) +
2639					    stripe_offset;
2640
2641					/*
2642					 * Set upper 4 bytes of LBA to 0.  We
2643					 * assume that the phys disks are less
2644					 * than 2 TB's in size.  Then, set the
2645					 * lower 4 bytes.
2646					 */
2647					pIO_req->CDB.CDB32[2] = 0;
2648					pIO_req->CDB.CDB32[3] = 0;
2649					pIO_req->CDB.CDB32[4] = 0;
2650					pIO_req->CDB.CDB32[5] = 0;
2651					ptrLBA = &pIO_req->CDB.CDB32[6];
2652					physLBA_byte = (uint8_t)(physLBA >> 24);
2653					*ptrLBA = physLBA_byte;
2654					ptrLBA = &pIO_req->CDB.CDB32[7];
2655					physLBA_byte = (uint8_t)(physLBA >> 16);
2656					*ptrLBA = physLBA_byte;
2657					ptrLBA = &pIO_req->CDB.CDB32[8];
2658					physLBA_byte = (uint8_t)(physLBA >> 8);
2659					*ptrLBA = physLBA_byte;
2660					ptrLBA = &pIO_req->CDB.CDB32[9];
2661					physLBA_byte = (uint8_t)physLBA;
2662					*ptrLBA = physLBA_byte;
2663
2664					/*
2665					 * Set flag that Direct Drive I/O is
2666					 * being done.
2667					 */
2668					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2669				}
2670			}
2671		}
2672	}
2673}
2674
2675#if __FreeBSD_version >= 900026
2676static void
2677mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm)
2678{
2679	MPI2_SMP_PASSTHROUGH_REPLY *rpl;
2680	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2681	uint64_t sasaddr;
2682	union ccb *ccb;
2683
2684	ccb = cm->cm_complete_data;
2685
2686	/*
2687	 * Currently there should be no way we can hit this case.  It only
2688	 * happens when we have a failure to allocate chain frames, and SMP
2689	 * commands require two S/G elements only.  That should be handled
2690	 * in the standard request size.
2691	 */
2692	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2693		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x on SMP request!\n",
2694			   __func__, cm->cm_flags);
2695		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2696		goto bailout;
2697        }
2698
2699	rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply;
2700	if (rpl == NULL) {
2701		mps_dprint(sc, MPS_ERROR, "%s: NULL cm_reply!\n", __func__);
2702		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2703		goto bailout;
2704	}
2705
2706	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2707	sasaddr = le32toh(req->SASAddress.Low);
2708	sasaddr |= ((uint64_t)(le32toh(req->SASAddress.High))) << 32;
2709
2710	if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
2711	    MPI2_IOCSTATUS_SUCCESS ||
2712	    rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) {
2713		mps_dprint(sc, MPS_XINFO, "%s: IOCStatus %04x SASStatus %02x\n",
2714		    __func__, le16toh(rpl->IOCStatus), rpl->SASStatus);
2715		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2716		goto bailout;
2717	}
2718
2719	mps_dprint(sc, MPS_XINFO, "%s: SMP request to SAS address "
2720		   "%#jx completed successfully\n", __func__,
2721		   (uintmax_t)sasaddr);
2722
2723	if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED)
2724		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2725	else
2726		mpssas_set_ccbstatus(ccb, CAM_SMP_STATUS_ERROR);
2727
2728bailout:
2729	/*
2730	 * We sync in both directions because we had DMAs in the S/G list
2731	 * in both directions.
2732	 */
2733	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
2734			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2735	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2736	mps_free_command(sc, cm);
2737	xpt_done(ccb);
2738}
2739
2740static void
2741mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb, uint64_t sasaddr)
2742{
2743	struct mps_command *cm;
2744	uint8_t *request, *response;
2745	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2746	struct mps_softc *sc;
2747	int error;
2748
2749	sc = sassc->sc;
2750	error = 0;
2751
2752	/*
2753	 * XXX We don't yet support physical addresses here.
2754	 */
2755	switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) {
2756	case CAM_DATA_PADDR:
2757	case CAM_DATA_SG_PADDR:
2758		mps_dprint(sc, MPS_ERROR,
2759			   "%s: physical addresses not supported\n", __func__);
2760		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2761		xpt_done(ccb);
2762		return;
2763	case CAM_DATA_SG:
2764		/*
2765		 * The chip does not support more than one buffer for the
2766		 * request or response.
2767		 */
2768	 	if ((ccb->smpio.smp_request_sglist_cnt > 1)
2769		  || (ccb->smpio.smp_response_sglist_cnt > 1)) {
2770			mps_dprint(sc, MPS_ERROR,
2771				   "%s: multiple request or response "
2772				   "buffer segments not supported for SMP\n",
2773				   __func__);
2774			mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2775			xpt_done(ccb);
2776			return;
2777		}
2778
2779		/*
2780		 * The CAM_SCATTER_VALID flag was originally implemented
2781		 * for the XPT_SCSI_IO CCB, which only has one data pointer.
2782		 * We have two.  So, just take that flag to mean that we
2783		 * might have S/G lists, and look at the S/G segment count
2784		 * to figure out whether that is the case for each individual
2785		 * buffer.
2786		 */
2787		if (ccb->smpio.smp_request_sglist_cnt != 0) {
2788			bus_dma_segment_t *req_sg;
2789
2790			req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request;
2791			request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr;
2792		} else
2793			request = ccb->smpio.smp_request;
2794
2795		if (ccb->smpio.smp_response_sglist_cnt != 0) {
2796			bus_dma_segment_t *rsp_sg;
2797
2798			rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response;
2799			response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr;
2800		} else
2801			response = ccb->smpio.smp_response;
2802		break;
2803	case CAM_DATA_VADDR:
2804		request = ccb->smpio.smp_request;
2805		response = ccb->smpio.smp_response;
2806		break;
2807	default:
2808		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2809		xpt_done(ccb);
2810		return;
2811	}
2812
2813	cm = mps_alloc_command(sc);
2814	if (cm == NULL) {
2815		mps_dprint(sc, MPS_ERROR,
2816		    "%s: cannot allocate command\n", __func__);
2817		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2818		xpt_done(ccb);
2819		return;
2820	}
2821
2822	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2823	bzero(req, sizeof(*req));
2824	req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH;
2825
2826	/* Allow the chip to use any route to this SAS address. */
2827	req->PhysicalPort = 0xff;
2828
2829	req->RequestDataLength = htole16(ccb->smpio.smp_request_len);
2830	req->SGLFlags =
2831	    MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE | MPI2_SGLFLAGS_SGL_TYPE_MPI;
2832
2833	mps_dprint(sc, MPS_XINFO, "%s: sending SMP request to SAS "
2834	    "address %#jx\n", __func__, (uintmax_t)sasaddr);
2835
2836	mpi_init_sge(cm, req, &req->SGL);
2837
2838	/*
2839	 * Set up a uio to pass into mps_map_command().  This allows us to
2840	 * do one map command, and one busdma call in there.
2841	 */
2842	cm->cm_uio.uio_iov = cm->cm_iovec;
2843	cm->cm_uio.uio_iovcnt = 2;
2844	cm->cm_uio.uio_segflg = UIO_SYSSPACE;
2845
2846	/*
2847	 * The read/write flag isn't used by busdma, but set it just in
2848	 * case.  This isn't exactly accurate, either, since we're going in
2849	 * both directions.
2850	 */
2851	cm->cm_uio.uio_rw = UIO_WRITE;
2852
2853	cm->cm_iovec[0].iov_base = request;
2854	cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength);
2855	cm->cm_iovec[1].iov_base = response;
2856	cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len;
2857
2858	cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len +
2859			       cm->cm_iovec[1].iov_len;
2860
2861	/*
2862	 * Trigger a warning message in mps_data_cb() for the user if we
2863	 * wind up exceeding two S/G segments.  The chip expects one
2864	 * segment for the request and another for the response.
2865	 */
2866	cm->cm_max_segs = 2;
2867
2868	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
2869	cm->cm_complete = mpssas_smpio_complete;
2870	cm->cm_complete_data = ccb;
2871
2872	/*
2873	 * Tell the mapping code that we're using a uio, and that this is
2874	 * an SMP passthrough request.  There is a little special-case
2875	 * logic there (in mps_data_cb()) to handle the bidirectional
2876	 * transfer.
2877	 */
2878	cm->cm_flags |= MPS_CM_FLAGS_USE_UIO | MPS_CM_FLAGS_SMP_PASS |
2879			MPS_CM_FLAGS_DATAIN | MPS_CM_FLAGS_DATAOUT;
2880
2881	/* The chip data format is little endian. */
2882	req->SASAddress.High = htole32(sasaddr >> 32);
2883	req->SASAddress.Low = htole32(sasaddr);
2884
2885	/*
2886	 * XXX Note that we don't have a timeout/abort mechanism here.
2887	 * From the manual, it looks like task management requests only
2888	 * work for SCSI IO and SATA passthrough requests.  We may need to
2889	 * have a mechanism to retry requests in the event of a chip reset
2890	 * at least.  Hopefully the chip will insure that any errors short
2891	 * of that are relayed back to the driver.
2892	 */
2893	error = mps_map_command(sc, cm);
2894	if ((error != 0) && (error != EINPROGRESS)) {
2895		mps_dprint(sc, MPS_ERROR,
2896			   "%s: error %d returned from mps_map_command()\n",
2897			   __func__, error);
2898		goto bailout_error;
2899	}
2900
2901	return;
2902
2903bailout_error:
2904	mps_free_command(sc, cm);
2905	mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2906	xpt_done(ccb);
2907	return;
2908
2909}
2910
2911static void
2912mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb)
2913{
2914	struct mps_softc *sc;
2915	struct mpssas_target *targ;
2916	uint64_t sasaddr = 0;
2917
2918	sc = sassc->sc;
2919
2920	/*
2921	 * Make sure the target exists.
2922	 */
2923	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
2924	    ("Target %d out of bounds in XPT_SMP_IO\n", ccb->ccb_h.target_id));
2925	targ = &sassc->targets[ccb->ccb_h.target_id];
2926	if (targ->handle == 0x0) {
2927		mps_dprint(sc, MPS_ERROR,
2928			   "%s: target %d does not exist!\n", __func__,
2929			   ccb->ccb_h.target_id);
2930		mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
2931		xpt_done(ccb);
2932		return;
2933	}
2934
2935	/*
2936	 * If this device has an embedded SMP target, we'll talk to it
2937	 * directly.
2938	 * figure out what the expander's address is.
2939	 */
2940	if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0)
2941		sasaddr = targ->sasaddr;
2942
2943	/*
2944	 * If we don't have a SAS address for the expander yet, try
2945	 * grabbing it from the page 0x83 information cached in the
2946	 * transport layer for this target.  LSI expanders report the
2947	 * expander SAS address as the port-associated SAS address in
2948	 * Inquiry VPD page 0x83.  Maxim expanders don't report it in page
2949	 * 0x83.
2950	 *
2951	 * XXX KDM disable this for now, but leave it commented out so that
2952	 * it is obvious that this is another possible way to get the SAS
2953	 * address.
2954	 *
2955	 * The parent handle method below is a little more reliable, and
2956	 * the other benefit is that it works for devices other than SES
2957	 * devices.  So you can send a SMP request to a da(4) device and it
2958	 * will get routed to the expander that device is attached to.
2959	 * (Assuming the da(4) device doesn't contain an SMP target...)
2960	 */
2961#if 0
2962	if (sasaddr == 0)
2963		sasaddr = xpt_path_sas_addr(ccb->ccb_h.path);
2964#endif
2965
2966	/*
2967	 * If we still don't have a SAS address for the expander, look for
2968	 * the parent device of this device, which is probably the expander.
2969	 */
2970	if (sasaddr == 0) {
2971#ifdef OLD_MPS_PROBE
2972		struct mpssas_target *parent_target;
2973#endif
2974
2975		if (targ->parent_handle == 0x0) {
2976			mps_dprint(sc, MPS_ERROR,
2977				   "%s: handle %d does not have a valid "
2978				   "parent handle!\n", __func__, targ->handle);
2979			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2980			goto bailout;
2981		}
2982#ifdef OLD_MPS_PROBE
2983		parent_target = mpssas_find_target_by_handle(sassc, 0,
2984			targ->parent_handle);
2985
2986		if (parent_target == NULL) {
2987			mps_dprint(sc, MPS_ERROR,
2988				   "%s: handle %d does not have a valid "
2989				   "parent target!\n", __func__, targ->handle);
2990			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2991			goto bailout;
2992		}
2993
2994		if ((parent_target->devinfo &
2995		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
2996			mps_dprint(sc, MPS_ERROR,
2997				   "%s: handle %d parent %d does not "
2998				   "have an SMP target!\n", __func__,
2999				   targ->handle, parent_target->handle);
3000			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3001			goto bailout;
3002
3003		}
3004
3005		sasaddr = parent_target->sasaddr;
3006#else /* OLD_MPS_PROBE */
3007		if ((targ->parent_devinfo &
3008		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3009			mps_dprint(sc, MPS_ERROR,
3010				   "%s: handle %d parent %d does not "
3011				   "have an SMP target!\n", __func__,
3012				   targ->handle, targ->parent_handle);
3013			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3014			goto bailout;
3015
3016		}
3017		if (targ->parent_sasaddr == 0x0) {
3018			mps_dprint(sc, MPS_ERROR,
3019				   "%s: handle %d parent handle %d does "
3020				   "not have a valid SAS address!\n",
3021				   __func__, targ->handle, targ->parent_handle);
3022			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3023			goto bailout;
3024		}
3025
3026		sasaddr = targ->parent_sasaddr;
3027#endif /* OLD_MPS_PROBE */
3028
3029	}
3030
3031	if (sasaddr == 0) {
3032		mps_dprint(sc, MPS_INFO,
3033			   "%s: unable to find SAS address for handle %d\n",
3034			   __func__, targ->handle);
3035		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3036		goto bailout;
3037	}
3038	mpssas_send_smpcmd(sassc, ccb, sasaddr);
3039
3040	return;
3041
3042bailout:
3043	xpt_done(ccb);
3044
3045}
3046#endif //__FreeBSD_version >= 900026
3047
3048static void
3049mpssas_action_resetdev(struct mpssas_softc *sassc, union ccb *ccb)
3050{
3051	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3052	struct mps_softc *sc;
3053	struct mps_command *tm;
3054	struct mpssas_target *targ;
3055
3056	MPS_FUNCTRACE(sassc->sc);
3057	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
3058
3059	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
3060	    ("Target %d out of bounds in XPT_RESET_DEV\n",
3061	     ccb->ccb_h.target_id));
3062	sc = sassc->sc;
3063	tm = mps_alloc_command(sc);
3064	if (tm == NULL) {
3065		mps_dprint(sc, MPS_ERROR,
3066		    "command alloc failure in mpssas_action_resetdev\n");
3067		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
3068		xpt_done(ccb);
3069		return;
3070	}
3071
3072	targ = &sassc->targets[ccb->ccb_h.target_id];
3073	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3074	req->DevHandle = htole16(targ->handle);
3075	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
3076	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
3077
3078	/* SAS Hard Link Reset / SATA Link Reset */
3079	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3080
3081	tm->cm_data = NULL;
3082	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
3083	tm->cm_complete = mpssas_resetdev_complete;
3084	tm->cm_complete_data = ccb;
3085	tm->cm_targ = targ;
3086	targ->flags |= MPSSAS_TARGET_INRESET;
3087
3088	mps_map_command(sc, tm);
3089}
3090
3091static void
3092mpssas_resetdev_complete(struct mps_softc *sc, struct mps_command *tm)
3093{
3094	MPI2_SCSI_TASK_MANAGE_REPLY *resp;
3095	union ccb *ccb;
3096
3097	MPS_FUNCTRACE(sc);
3098	mtx_assert(&sc->mps_mtx, MA_OWNED);
3099
3100	resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
3101	ccb = tm->cm_complete_data;
3102
3103	/*
3104	 * Currently there should be no way we can hit this case.  It only
3105	 * happens when we have a failure to allocate chain frames, and
3106	 * task management commands don't have S/G lists.
3107	 */
3108	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3109		MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3110
3111		req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3112
3113		mps_dprint(sc, MPS_ERROR,
3114			   "%s: cm_flags = %#x for reset of handle %#04x! "
3115			   "This should not happen!\n", __func__, tm->cm_flags,
3116			   req->DevHandle);
3117		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3118		goto bailout;
3119	}
3120
3121	mps_dprint(sc, MPS_XINFO,
3122	    "%s: IOCStatus = 0x%x ResponseCode = 0x%x\n", __func__,
3123	    le16toh(resp->IOCStatus), le32toh(resp->ResponseCode));
3124
3125	if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) {
3126		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
3127		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
3128		    CAM_LUN_WILDCARD);
3129	}
3130	else
3131		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3132
3133bailout:
3134
3135	mpssas_free_tm(sc, tm);
3136	xpt_done(ccb);
3137}
3138
3139static void
3140mpssas_poll(struct cam_sim *sim)
3141{
3142	struct mpssas_softc *sassc;
3143
3144	sassc = cam_sim_softc(sim);
3145
3146	if (sassc->sc->mps_debug & MPS_TRACE) {
3147		/* frequent debug messages during a panic just slow
3148		 * everything down too much.
3149		 */
3150		mps_printf(sassc->sc, "%s clearing MPS_TRACE\n", __func__);
3151		sassc->sc->mps_debug &= ~MPS_TRACE;
3152	}
3153
3154	mps_intr_locked(sassc->sc);
3155}
3156
3157static void
3158mpssas_async(void *callback_arg, uint32_t code, struct cam_path *path,
3159	     void *arg)
3160{
3161	struct mps_softc *sc;
3162
3163	sc = (struct mps_softc *)callback_arg;
3164
3165	switch (code) {
3166#if (__FreeBSD_version >= 1000006) || \
3167    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
3168	case AC_ADVINFO_CHANGED: {
3169		struct mpssas_target *target;
3170		struct mpssas_softc *sassc;
3171		struct scsi_read_capacity_data_long rcap_buf;
3172		struct ccb_dev_advinfo cdai;
3173		struct mpssas_lun *lun;
3174		lun_id_t lunid;
3175		int found_lun;
3176		uintptr_t buftype;
3177
3178		buftype = (uintptr_t)arg;
3179
3180		found_lun = 0;
3181		sassc = sc->sassc;
3182
3183		/*
3184		 * We're only interested in read capacity data changes.
3185		 */
3186		if (buftype != CDAI_TYPE_RCAPLONG)
3187			break;
3188
3189		/*
3190		 * We should have a handle for this, but check to make sure.
3191		 */
3192		KASSERT(xpt_path_target_id(path) < sassc->maxtargets,
3193		    ("Target %d out of bounds in mpssas_async\n",
3194		    xpt_path_target_id(path)));
3195		target = &sassc->targets[xpt_path_target_id(path)];
3196		if (target->handle == 0)
3197			break;
3198
3199		lunid = xpt_path_lun_id(path);
3200
3201		SLIST_FOREACH(lun, &target->luns, lun_link) {
3202			if (lun->lun_id == lunid) {
3203				found_lun = 1;
3204				break;
3205			}
3206		}
3207
3208		if (found_lun == 0) {
3209			lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3210				     M_NOWAIT | M_ZERO);
3211			if (lun == NULL) {
3212				mps_dprint(sc, MPS_ERROR, "Unable to alloc "
3213					   "LUN for EEDP support.\n");
3214				break;
3215			}
3216			lun->lun_id = lunid;
3217			SLIST_INSERT_HEAD(&target->luns, lun, lun_link);
3218		}
3219
3220		bzero(&rcap_buf, sizeof(rcap_buf));
3221		xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
3222		cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
3223		cdai.ccb_h.flags = CAM_DIR_IN;
3224		cdai.buftype = CDAI_TYPE_RCAPLONG;
3225#if (__FreeBSD_version >= 1100061) || \
3226    ((__FreeBSD_version >= 1001510) && (__FreeBSD_version < 1100000))
3227		cdai.flags = CDAI_FLAG_NONE;
3228#else
3229		cdai.flags = 0;
3230#endif
3231		cdai.bufsiz = sizeof(rcap_buf);
3232		cdai.buf = (uint8_t *)&rcap_buf;
3233		xpt_action((union ccb *)&cdai);
3234		if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
3235			cam_release_devq(cdai.ccb_h.path,
3236					 0, 0, 0, FALSE);
3237
3238		if ((mpssas_get_ccbstatus((union ccb *)&cdai) == CAM_REQ_CMP)
3239		 && (rcap_buf.prot & SRC16_PROT_EN)) {
3240			switch (rcap_buf.prot & SRC16_P_TYPE) {
3241			case SRC16_PTYPE_1:
3242			case SRC16_PTYPE_3:
3243				lun->eedp_formatted = TRUE;
3244				lun->eedp_block_size =
3245				    scsi_4btoul(rcap_buf.length);
3246				break;
3247			case SRC16_PTYPE_2:
3248			default:
3249				lun->eedp_formatted = FALSE;
3250				lun->eedp_block_size = 0;
3251				break;
3252			}
3253		} else {
3254			lun->eedp_formatted = FALSE;
3255			lun->eedp_block_size = 0;
3256		}
3257		break;
3258	}
3259#else
3260	case AC_FOUND_DEVICE: {
3261		struct ccb_getdev *cgd;
3262
3263		cgd = arg;
3264		mpssas_check_eedp(sc, path, cgd);
3265		break;
3266	}
3267#endif
3268	default:
3269		break;
3270	}
3271}
3272
3273#if (__FreeBSD_version < 901503) || \
3274    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
3275static void
3276mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
3277		  struct ccb_getdev *cgd)
3278{
3279	struct mpssas_softc *sassc = sc->sassc;
3280	struct ccb_scsiio *csio;
3281	struct scsi_read_capacity_16 *scsi_cmd;
3282	struct scsi_read_capacity_eedp *rcap_buf;
3283	path_id_t pathid;
3284	target_id_t targetid;
3285	lun_id_t lunid;
3286	union ccb *ccb;
3287	struct cam_path *local_path;
3288	struct mpssas_target *target;
3289	struct mpssas_lun *lun;
3290	uint8_t	found_lun;
3291	char path_str[64];
3292
3293	sassc = sc->sassc;
3294	pathid = cam_sim_path(sassc->sim);
3295	targetid = xpt_path_target_id(path);
3296	lunid = xpt_path_lun_id(path);
3297
3298	KASSERT(targetid < sassc->maxtargets,
3299	    ("Target %d out of bounds in mpssas_check_eedp\n",
3300	     targetid));
3301	target = &sassc->targets[targetid];
3302	if (target->handle == 0x0)
3303		return;
3304
3305	/*
3306	 * Determine if the device is EEDP capable.
3307	 *
3308	 * If this flag is set in the inquiry data,
3309	 * the device supports protection information,
3310	 * and must support the 16 byte read
3311	 * capacity command, otherwise continue without
3312	 * sending read cap 16
3313	 */
3314	if ((cgd->inq_data.spc3_flags & SPC3_SID_PROTECT) == 0)
3315		return;
3316
3317	/*
3318	 * Issue a READ CAPACITY 16 command.  This info
3319	 * is used to determine if the LUN is formatted
3320	 * for EEDP support.
3321	 */
3322	ccb = xpt_alloc_ccb_nowait();
3323	if (ccb == NULL) {
3324		mps_dprint(sc, MPS_ERROR, "Unable to alloc CCB "
3325		    "for EEDP support.\n");
3326		return;
3327	}
3328
3329	if (xpt_create_path(&local_path, xpt_periph,
3330	    pathid, targetid, lunid) != CAM_REQ_CMP) {
3331		mps_dprint(sc, MPS_ERROR, "Unable to create "
3332		    "path for EEDP support\n");
3333		xpt_free_ccb(ccb);
3334		return;
3335	}
3336
3337	/*
3338	 * If LUN is already in list, don't create a new
3339	 * one.
3340	 */
3341	found_lun = FALSE;
3342	SLIST_FOREACH(lun, &target->luns, lun_link) {
3343		if (lun->lun_id == lunid) {
3344			found_lun = TRUE;
3345			break;
3346		}
3347	}
3348	if (!found_lun) {
3349		lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3350		    M_NOWAIT | M_ZERO);
3351		if (lun == NULL) {
3352			mps_dprint(sc, MPS_ERROR,
3353			    "Unable to alloc LUN for EEDP support.\n");
3354			xpt_free_path(local_path);
3355			xpt_free_ccb(ccb);
3356			return;
3357		}
3358		lun->lun_id = lunid;
3359		SLIST_INSERT_HEAD(&target->luns, lun,
3360		    lun_link);
3361	}
3362
3363	xpt_path_string(local_path, path_str, sizeof(path_str));
3364
3365	mps_dprint(sc, MPS_INFO, "Sending read cap: path %s handle %d\n",
3366	    path_str, target->handle);
3367
3368	/*
3369	 * Issue a READ CAPACITY 16 command for the LUN.
3370	 * The mpssas_read_cap_done function will load
3371	 * the read cap info into the LUN struct.
3372	 */
3373	rcap_buf = malloc(sizeof(struct scsi_read_capacity_eedp),
3374	    M_MPT2, M_NOWAIT | M_ZERO);
3375	if (rcap_buf == NULL) {
3376		mps_dprint(sc, MPS_FAULT,
3377		    "Unable to alloc read capacity buffer for EEDP support.\n");
3378		xpt_free_path(ccb->ccb_h.path);
3379		xpt_free_ccb(ccb);
3380		return;
3381	}
3382	xpt_setup_ccb(&ccb->ccb_h, local_path, CAM_PRIORITY_XPT);
3383	csio = &ccb->csio;
3384	csio->ccb_h.func_code = XPT_SCSI_IO;
3385	csio->ccb_h.flags = CAM_DIR_IN;
3386	csio->ccb_h.retry_count = 4;
3387	csio->ccb_h.cbfcnp = mpssas_read_cap_done;
3388	csio->ccb_h.timeout = 60000;
3389	csio->data_ptr = (uint8_t *)rcap_buf;
3390	csio->dxfer_len = sizeof(struct scsi_read_capacity_eedp);
3391	csio->sense_len = MPS_SENSE_LEN;
3392	csio->cdb_len = sizeof(*scsi_cmd);
3393	csio->tag_action = MSG_SIMPLE_Q_TAG;
3394
3395	scsi_cmd = (struct scsi_read_capacity_16 *)&csio->cdb_io.cdb_bytes;
3396	bzero(scsi_cmd, sizeof(*scsi_cmd));
3397	scsi_cmd->opcode = 0x9E;
3398	scsi_cmd->service_action = SRC16_SERVICE_ACTION;
3399	((uint8_t *)scsi_cmd)[13] = sizeof(struct scsi_read_capacity_eedp);
3400
3401	ccb->ccb_h.ppriv_ptr1 = sassc;
3402	xpt_action(ccb);
3403}
3404
3405static void
3406mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb)
3407{
3408	struct mpssas_softc *sassc;
3409	struct mpssas_target *target;
3410	struct mpssas_lun *lun;
3411	struct scsi_read_capacity_eedp *rcap_buf;
3412
3413	if (done_ccb == NULL)
3414		return;
3415
3416	/* Driver need to release devq, it Scsi command is
3417	 * generated by driver internally.
3418	 * Currently there is a single place where driver
3419	 * calls scsi command internally. In future if driver
3420	 * calls more scsi command internally, it needs to release
3421	 * devq internally, since those command will not go back to
3422	 * cam_periph.
3423	 */
3424	if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) ) {
3425        	done_ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
3426		xpt_release_devq(done_ccb->ccb_h.path,
3427			       	/*count*/ 1, /*run_queue*/TRUE);
3428	}
3429
3430	rcap_buf = (struct scsi_read_capacity_eedp *)done_ccb->csio.data_ptr;
3431
3432	/*
3433	 * Get the LUN ID for the path and look it up in the LUN list for the
3434	 * target.
3435	 */
3436	sassc = (struct mpssas_softc *)done_ccb->ccb_h.ppriv_ptr1;
3437	KASSERT(done_ccb->ccb_h.target_id < sassc->maxtargets,
3438	    ("Target %d out of bounds in mpssas_read_cap_done\n",
3439	     done_ccb->ccb_h.target_id));
3440	target = &sassc->targets[done_ccb->ccb_h.target_id];
3441	SLIST_FOREACH(lun, &target->luns, lun_link) {
3442		if (lun->lun_id != done_ccb->ccb_h.target_lun)
3443			continue;
3444
3445		/*
3446		 * Got the LUN in the target's LUN list.  Fill it in
3447		 * with EEDP info.  If the READ CAP 16 command had some
3448		 * SCSI error (common if command is not supported), mark
3449		 * the lun as not supporting EEDP and set the block size
3450		 * to 0.
3451		 */
3452		if ((mpssas_get_ccbstatus(done_ccb) != CAM_REQ_CMP)
3453		 || (done_ccb->csio.scsi_status != SCSI_STATUS_OK)) {
3454			lun->eedp_formatted = FALSE;
3455			lun->eedp_block_size = 0;
3456			break;
3457		}
3458
3459		if (rcap_buf->protect & 0x01) {
3460			mps_dprint(sassc->sc, MPS_INFO, "LUN %d for "
3461 			    "target ID %d is formatted for EEDP "
3462 			    "support.\n", done_ccb->ccb_h.target_lun,
3463 			    done_ccb->ccb_h.target_id);
3464			lun->eedp_formatted = TRUE;
3465			lun->eedp_block_size = scsi_4btoul(rcap_buf->length);
3466		}
3467		break;
3468	}
3469
3470	// Finished with this CCB and path.
3471	free(rcap_buf, M_MPT2);
3472	xpt_free_path(done_ccb->ccb_h.path);
3473	xpt_free_ccb(done_ccb);
3474}
3475#endif /* (__FreeBSD_version < 901503) || \
3476          ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006)) */
3477
3478void
3479mpssas_prepare_for_tm(struct mps_softc *sc, struct mps_command *tm,
3480    struct mpssas_target *target, lun_id_t lun_id)
3481{
3482	union ccb *ccb;
3483	path_id_t path_id;
3484
3485	/*
3486	 * Set the INRESET flag for this target so that no I/O will be sent to
3487	 * the target until the reset has completed.  If an I/O request does
3488	 * happen, the devq will be frozen.  The CCB holds the path which is
3489	 * used to release the devq.  The devq is released and the CCB is freed
3490	 * when the TM completes.
3491	 */
3492	ccb = xpt_alloc_ccb_nowait();
3493	if (ccb) {
3494		path_id = cam_sim_path(sc->sassc->sim);
3495		if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, path_id,
3496		    target->tid, lun_id) != CAM_REQ_CMP) {
3497			xpt_free_ccb(ccb);
3498		} else {
3499			tm->cm_ccb = ccb;
3500			tm->cm_targ = target;
3501			target->flags |= MPSSAS_TARGET_INRESET;
3502		}
3503	}
3504}
3505
3506int
3507mpssas_startup(struct mps_softc *sc)
3508{
3509
3510	/*
3511	 * Send the port enable message and set the wait_for_port_enable flag.
3512	 * This flag helps to keep the simq frozen until all discovery events
3513	 * are processed.
3514	 */
3515	sc->wait_for_port_enable = 1;
3516	mpssas_send_portenable(sc);
3517	return (0);
3518}
3519
3520static int
3521mpssas_send_portenable(struct mps_softc *sc)
3522{
3523	MPI2_PORT_ENABLE_REQUEST *request;
3524	struct mps_command *cm;
3525
3526	MPS_FUNCTRACE(sc);
3527
3528	if ((cm = mps_alloc_command(sc)) == NULL)
3529		return (EBUSY);
3530	request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req;
3531	request->Function = MPI2_FUNCTION_PORT_ENABLE;
3532	request->MsgFlags = 0;
3533	request->VP_ID = 0;
3534	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3535	cm->cm_complete = mpssas_portenable_complete;
3536	cm->cm_data = NULL;
3537	cm->cm_sge = NULL;
3538
3539	mps_map_command(sc, cm);
3540	mps_dprint(sc, MPS_XINFO,
3541	    "mps_send_portenable finished cm %p req %p complete %p\n",
3542	    cm, cm->cm_req, cm->cm_complete);
3543	return (0);
3544}
3545
3546static void
3547mpssas_portenable_complete(struct mps_softc *sc, struct mps_command *cm)
3548{
3549	MPI2_PORT_ENABLE_REPLY *reply;
3550	struct mpssas_softc *sassc;
3551
3552	MPS_FUNCTRACE(sc);
3553	sassc = sc->sassc;
3554
3555	/*
3556	 * Currently there should be no way we can hit this case.  It only
3557	 * happens when we have a failure to allocate chain frames, and
3558	 * port enable commands don't have S/G lists.
3559	 */
3560	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3561		mps_dprint(sc, MPS_ERROR, "%s: cm_flags = %#x for port enable! "
3562			   "This should not happen!\n", __func__, cm->cm_flags);
3563	}
3564
3565	reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply;
3566	if (reply == NULL)
3567		mps_dprint(sc, MPS_FAULT, "Portenable NULL reply\n");
3568	else if (le16toh(reply->IOCStatus & MPI2_IOCSTATUS_MASK) !=
3569	    MPI2_IOCSTATUS_SUCCESS)
3570		mps_dprint(sc, MPS_FAULT, "Portenable failed\n");
3571
3572	mps_free_command(sc, cm);
3573
3574	/*
3575	 * Get WarpDrive info after discovery is complete but before the scan
3576	 * starts.  At this point, all devices are ready to be exposed to the
3577	 * OS.  If devices should be hidden instead, take them out of the
3578	 * 'targets' array before the scan.  The devinfo for a disk will have
3579	 * some info and a volume's will be 0.  Use that to remove disks.
3580	 */
3581	mps_wd_config_pages(sc);
3582
3583	/*
3584	 * Done waiting for port enable to complete.  Decrement the refcount.
3585	 * If refcount is 0, discovery is complete and a rescan of the bus can
3586	 * take place.  Since the simq was explicitly frozen before port
3587	 * enable, it must be explicitly released here to keep the
3588	 * freeze/release count in sync.
3589	 */
3590	sc->wait_for_port_enable = 0;
3591	sc->port_enable_complete = 1;
3592	wakeup(&sc->port_enable_complete);
3593	mpssas_startup_decrement(sassc);
3594}
3595
3596int
3597mpssas_check_id(struct mpssas_softc *sassc, int id)
3598{
3599	struct mps_softc *sc = sassc->sc;
3600	char *ids;
3601	char *name;
3602
3603	ids = &sc->exclude_ids[0];
3604	while((name = strsep(&ids, ",")) != NULL) {
3605		if (name[0] == '\0')
3606			continue;
3607		if (strtol(name, NULL, 0) == (long)id)
3608			return (1);
3609	}
3610
3611	return (0);
3612}
3613
3614void
3615mpssas_realloc_targets(struct mps_softc *sc, int maxtargets)
3616{
3617	struct mpssas_softc *sassc;
3618	struct mpssas_lun *lun, *lun_tmp;
3619	struct mpssas_target *targ;
3620	int i;
3621
3622	sassc = sc->sassc;
3623	/*
3624	 * The number of targets is based on IOC Facts, so free all of
3625	 * the allocated LUNs for each target and then the target buffer
3626	 * itself.
3627	 */
3628	for (i=0; i< maxtargets; i++) {
3629		targ = &sassc->targets[i];
3630		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
3631			free(lun, M_MPT2);
3632		}
3633	}
3634	free(sassc->targets, M_MPT2);
3635
3636	sassc->targets = malloc(sizeof(struct mpssas_target) * maxtargets,
3637	    M_MPT2, M_WAITOK|M_ZERO);
3638	if (!sassc->targets) {
3639		panic("%s failed to alloc targets with error %d\n",
3640		    __func__, ENOMEM);
3641	}
3642}
3643