mps_sas.c revision 93e4d330694dc61a6ea4f3c3d78656d9cce70698
1/*-
2 * Copyright (c) 2009 Yahoo! Inc.
3 * Copyright (c) 2011-2015 LSI Corp.
4 * Copyright (c) 2013-2015 Avago Technologies
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
29 *
30 * $FreeBSD$
31 */
32
33#include <sys/cdefs.h>
34__FBSDID("$FreeBSD$");
35
36/* Communications core for Avago Technologies (LSI) MPT2 */
37
38/* TODO Move headers to mpsvar */
39#include <sys/types.h>
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/kernel.h>
43#include <sys/selinfo.h>
44#include <sys/module.h>
45#include <sys/bus.h>
46#include <sys/conf.h>
47#include <sys/bio.h>
48#include <sys/malloc.h>
49#include <sys/uio.h>
50#include <sys/sysctl.h>
51#include <sys/endian.h>
52#include <sys/queue.h>
53#include <sys/kthread.h>
54#include <sys/taskqueue.h>
55#include <sys/sbuf.h>
56
57#include <machine/bus.h>
58#include <machine/resource.h>
59#include <sys/rman.h>
60
61#include <machine/stdarg.h>
62
63#include <cam/cam.h>
64#include <cam/cam_ccb.h>
65#include <cam/cam_xpt.h>
66#include <cam/cam_debug.h>
67#include <cam/cam_sim.h>
68#include <cam/cam_xpt_sim.h>
69#include <cam/cam_xpt_periph.h>
70#include <cam/cam_periph.h>
71#include <cam/scsi/scsi_all.h>
72#include <cam/scsi/scsi_message.h>
73#if __FreeBSD_version >= 900026
74#include <cam/scsi/smp_all.h>
75#endif
76
77#include <dev/mps/mpi/mpi2_type.h>
78#include <dev/mps/mpi/mpi2.h>
79#include <dev/mps/mpi/mpi2_ioc.h>
80#include <dev/mps/mpi/mpi2_sas.h>
81#include <dev/mps/mpi/mpi2_cnfg.h>
82#include <dev/mps/mpi/mpi2_init.h>
83#include <dev/mps/mpi/mpi2_tool.h>
84#include <dev/mps/mps_ioctl.h>
85#include <dev/mps/mpsvar.h>
86#include <dev/mps/mps_table.h>
87#include <dev/mps/mps_sas.h>
88
89#define MPSSAS_DISCOVERY_TIMEOUT	20
90#define MPSSAS_MAX_DISCOVERY_TIMEOUTS	10 /* 200 seconds */
91
92/*
93 * static array to check SCSI OpCode for EEDP protection bits
94 */
95#define	PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP
96#define	PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
97#define	PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
98static uint8_t op_code_prot[256] = {
99	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
100	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
101	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
102	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
103	0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
104	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
105	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
106	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
107	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
108	0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
109	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
110	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
111	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
112	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
113	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
114	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
115};
116
117MALLOC_DEFINE(M_MPSSAS, "MPSSAS", "MPS SAS memory");
118
119static void mpssas_remove_device(struct mps_softc *, struct mps_command *);
120static void mpssas_remove_complete(struct mps_softc *, struct mps_command *);
121static void mpssas_action(struct cam_sim *sim, union ccb *ccb);
122static void mpssas_poll(struct cam_sim *sim);
123static int mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm,
124    struct mps_command *cm);
125static void mpssas_scsiio_timeout(void *data);
126static void mpssas_abort_complete(struct mps_softc *sc, struct mps_command *cm);
127static void mpssas_direct_drive_io(struct mpssas_softc *sassc,
128    struct mps_command *cm, union ccb *ccb);
129static void mpssas_action_scsiio(struct mpssas_softc *, union ccb *);
130static void mpssas_scsiio_complete(struct mps_softc *, struct mps_command *);
131static void mpssas_action_resetdev(struct mpssas_softc *, union ccb *);
132#if __FreeBSD_version >= 900026
133static void mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm);
134static void mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb,
135			       uint64_t sasaddr);
136static void mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb);
137#endif //FreeBSD_version >= 900026
138static void mpssas_resetdev_complete(struct mps_softc *, struct mps_command *);
139static void mpssas_async(void *callback_arg, uint32_t code,
140			 struct cam_path *path, void *arg);
141#if (__FreeBSD_version < 901503) || \
142    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
143static void mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
144			      struct ccb_getdev *cgd);
145static void mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb);
146#endif
147static int mpssas_send_portenable(struct mps_softc *sc);
148static void mpssas_portenable_complete(struct mps_softc *sc,
149    struct mps_command *cm);
150
151struct mpssas_target *
152mpssas_find_target_by_handle(struct mpssas_softc *sassc, int start, uint16_t handle)
153{
154	struct mpssas_target *target;
155	int i;
156
157	for (i = start; i < sassc->maxtargets; i++) {
158		target = &sassc->targets[i];
159		if (target->handle == handle)
160			return (target);
161	}
162
163	return (NULL);
164}
165
166/* we need to freeze the simq during attach and diag reset, to avoid failing
167 * commands before device handles have been found by discovery.  Since
168 * discovery involves reading config pages and possibly sending commands,
169 * discovery actions may continue even after we receive the end of discovery
170 * event, so refcount discovery actions instead of assuming we can unfreeze
171 * the simq when we get the event.
172 */
173void
174mpssas_startup_increment(struct mpssas_softc *sassc)
175{
176	MPS_FUNCTRACE(sassc->sc);
177
178	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
179		if (sassc->startup_refcount++ == 0) {
180			/* just starting, freeze the simq */
181			mps_dprint(sassc->sc, MPS_INIT,
182			    "%s freezing simq\n", __func__);
183#if __FreeBSD_version >= 1000039
184			xpt_hold_boot();
185#endif
186			xpt_freeze_simq(sassc->sim, 1);
187		}
188		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
189		    sassc->startup_refcount);
190	}
191}
192
193void
194mpssas_release_simq_reinit(struct mpssas_softc *sassc)
195{
196	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
197		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
198		xpt_release_simq(sassc->sim, 1);
199		mps_dprint(sassc->sc, MPS_INFO, "Unfreezing SIM queue\n");
200	}
201}
202
203void
204mpssas_startup_decrement(struct mpssas_softc *sassc)
205{
206	MPS_FUNCTRACE(sassc->sc);
207
208	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
209		if (--sassc->startup_refcount == 0) {
210			/* finished all discovery-related actions, release
211			 * the simq and rescan for the latest topology.
212			 */
213			mps_dprint(sassc->sc, MPS_INIT,
214			    "%s releasing simq\n", __func__);
215			sassc->flags &= ~MPSSAS_IN_STARTUP;
216			xpt_release_simq(sassc->sim, 1);
217#if __FreeBSD_version >= 1000039
218			xpt_release_boot();
219#else
220			mpssas_rescan_target(sassc->sc, NULL);
221#endif
222		}
223		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
224		    sassc->startup_refcount);
225	}
226}
227
228/* The firmware requires us to stop sending commands when we're doing task
229 * management, so refcount the TMs and keep the simq frozen when any are in
230 * use.
231 */
232struct mps_command *
233mpssas_alloc_tm(struct mps_softc *sc)
234{
235	struct mps_command *tm;
236
237	tm = mps_alloc_high_priority_command(sc);
238	return tm;
239}
240
241void
242mpssas_free_tm(struct mps_softc *sc, struct mps_command *tm)
243{
244	int target_id = 0xFFFFFFFF;
245
246	if (tm == NULL)
247		return;
248
249	/*
250	 * For TM's the devq is frozen for the device.  Unfreeze it here and
251	 * free the resources used for freezing the devq.  Must clear the
252	 * INRESET flag as well or scsi I/O will not work.
253	 */
254	if (tm->cm_targ != NULL) {
255		tm->cm_targ->flags &= ~MPSSAS_TARGET_INRESET;
256		target_id = tm->cm_targ->tid;
257	}
258	if (tm->cm_ccb) {
259		mps_dprint(sc, MPS_INFO, "Unfreezing devq for target ID %d\n",
260		    target_id);
261		xpt_release_devq(tm->cm_ccb->ccb_h.path, 1, TRUE);
262		xpt_free_path(tm->cm_ccb->ccb_h.path);
263		xpt_free_ccb(tm->cm_ccb);
264	}
265
266	mps_free_high_priority_command(sc, tm);
267}
268
269void
270mpssas_rescan_target(struct mps_softc *sc, struct mpssas_target *targ)
271{
272	struct mpssas_softc *sassc = sc->sassc;
273	path_id_t pathid;
274	target_id_t targetid;
275	union ccb *ccb;
276
277	MPS_FUNCTRACE(sc);
278	pathid = cam_sim_path(sassc->sim);
279	if (targ == NULL)
280		targetid = CAM_TARGET_WILDCARD;
281	else
282		targetid = targ - sassc->targets;
283
284	/*
285	 * Allocate a CCB and schedule a rescan.
286	 */
287	ccb = xpt_alloc_ccb_nowait();
288	if (ccb == NULL) {
289		mps_dprint(sc, MPS_ERROR, "unable to alloc CCB for rescan\n");
290		return;
291	}
292
293	if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
294	    targetid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
295		mps_dprint(sc, MPS_ERROR, "unable to create path for rescan\n");
296		xpt_free_ccb(ccb);
297		return;
298	}
299
300	if (targetid == CAM_TARGET_WILDCARD)
301		ccb->ccb_h.func_code = XPT_SCAN_BUS;
302	else
303		ccb->ccb_h.func_code = XPT_SCAN_TGT;
304
305	mps_dprint(sc, MPS_TRACE, "%s targetid %u\n", __func__, targetid);
306	xpt_rescan(ccb);
307}
308
309static void
310mpssas_log_command(struct mps_command *cm, u_int level, const char *fmt, ...)
311{
312	struct sbuf sb;
313	va_list ap;
314	char str[192];
315	char path_str[64];
316
317	if (cm == NULL)
318		return;
319
320	/* No need to be in here if debugging isn't enabled */
321	if ((cm->cm_sc->mps_debug & level) == 0)
322		return;
323
324	sbuf_new(&sb, str, sizeof(str), 0);
325
326	va_start(ap, fmt);
327
328	if (cm->cm_ccb != NULL) {
329		xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str,
330				sizeof(path_str));
331		sbuf_cat(&sb, path_str);
332		if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) {
333			scsi_command_string(&cm->cm_ccb->csio, &sb);
334			sbuf_printf(&sb, "length %d ",
335				    cm->cm_ccb->csio.dxfer_len);
336		}
337	}
338	else {
339		sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ",
340		    cam_sim_name(cm->cm_sc->sassc->sim),
341		    cam_sim_unit(cm->cm_sc->sassc->sim),
342		    cam_sim_bus(cm->cm_sc->sassc->sim),
343		    cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF,
344		    cm->cm_lun);
345	}
346
347	sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID);
348	sbuf_vprintf(&sb, fmt, ap);
349	sbuf_finish(&sb);
350	mps_dprint_field(cm->cm_sc, level, "%s", sbuf_data(&sb));
351
352	va_end(ap);
353}
354
355
356static void
357mpssas_remove_volume(struct mps_softc *sc, struct mps_command *tm)
358{
359	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
360	struct mpssas_target *targ;
361	uint16_t handle;
362
363	MPS_FUNCTRACE(sc);
364
365	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
366	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
367	targ = tm->cm_targ;
368
369	if (reply == NULL) {
370		/* XXX retry the remove after the diag reset completes? */
371		mps_dprint(sc, MPS_FAULT,
372		    "%s NULL reply resetting device 0x%04x\n", __func__,
373		    handle);
374		mpssas_free_tm(sc, tm);
375		return;
376	}
377
378	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
379	    MPI2_IOCSTATUS_SUCCESS) {
380		mps_dprint(sc, MPS_ERROR,
381		   "IOCStatus = 0x%x while resetting device 0x%x\n",
382		   le16toh(reply->IOCStatus), handle);
383	}
384
385	mps_dprint(sc, MPS_XINFO,
386	    "Reset aborted %u commands\n", reply->TerminationCount);
387	mps_free_reply(sc, tm->cm_reply_data);
388	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
389
390	mps_dprint(sc, MPS_XINFO,
391	    "clearing target %u handle 0x%04x\n", targ->tid, handle);
392
393	/*
394	 * Don't clear target if remove fails because things will get confusing.
395	 * Leave the devname and sasaddr intact so that we know to avoid reusing
396	 * this target id if possible, and so we can assign the same target id
397	 * to this device if it comes back in the future.
398	 */
399	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
400	    MPI2_IOCSTATUS_SUCCESS) {
401		targ = tm->cm_targ;
402		targ->handle = 0x0;
403		targ->encl_handle = 0x0;
404		targ->encl_slot = 0x0;
405		targ->exp_dev_handle = 0x0;
406		targ->phy_num = 0x0;
407		targ->linkrate = 0x0;
408		targ->devinfo = 0x0;
409		targ->flags = 0x0;
410	}
411
412	mpssas_free_tm(sc, tm);
413}
414
415
416/*
417 * No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal.
418 * Otherwise Volume Delete is same as Bare Drive Removal.
419 */
420void
421mpssas_prepare_volume_remove(struct mpssas_softc *sassc, uint16_t handle)
422{
423	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
424	struct mps_softc *sc;
425	struct mps_command *cm;
426	struct mpssas_target *targ = NULL;
427
428	MPS_FUNCTRACE(sassc->sc);
429	sc = sassc->sc;
430
431#ifdef WD_SUPPORT
432	/*
433	 * If this is a WD controller, determine if the disk should be exposed
434	 * to the OS or not.  If disk should be exposed, return from this
435	 * function without doing anything.
436	 */
437	if (sc->WD_available && (sc->WD_hide_expose ==
438	    MPS_WD_EXPOSE_ALWAYS)) {
439		return;
440	}
441#endif //WD_SUPPORT
442
443	targ = mpssas_find_target_by_handle(sassc, 0, handle);
444	if (targ == NULL) {
445		/* FIXME: what is the action? */
446		/* We don't know about this device? */
447		mps_dprint(sc, MPS_ERROR,
448		   "%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
449		return;
450	}
451
452	targ->flags |= MPSSAS_TARGET_INREMOVAL;
453
454	cm = mpssas_alloc_tm(sc);
455	if (cm == NULL) {
456		mps_dprint(sc, MPS_ERROR,
457		    "%s: command alloc failure\n", __func__);
458		return;
459	}
460
461	mpssas_rescan_target(sc, targ);
462
463	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
464	req->DevHandle = targ->handle;
465	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
466	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
467
468	/* SAS Hard Link Reset / SATA Link Reset */
469	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
470
471	cm->cm_targ = targ;
472	cm->cm_data = NULL;
473	cm->cm_desc.HighPriority.RequestFlags =
474	    MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
475	cm->cm_complete = mpssas_remove_volume;
476	cm->cm_complete_data = (void *)(uintptr_t)handle;
477
478	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
479	    __func__, targ->tid);
480	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
481
482	mps_map_command(sc, cm);
483}
484
485/*
486 * The MPT2 firmware performs debounce on the link to avoid transient link
487 * errors and false removals.  When it does decide that link has been lost
488 * and a device need to go away, it expects that the host will perform a
489 * target reset and then an op remove.  The reset has the side-effect of
490 * aborting any outstanding requests for the device, which is required for
491 * the op-remove to succeed.  It's not clear if the host should check for
492 * the device coming back alive after the reset.
493 */
494void
495mpssas_prepare_remove(struct mpssas_softc *sassc, uint16_t handle)
496{
497	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
498	struct mps_softc *sc;
499	struct mps_command *cm;
500	struct mpssas_target *targ = NULL;
501
502	MPS_FUNCTRACE(sassc->sc);
503
504	sc = sassc->sc;
505
506	targ = mpssas_find_target_by_handle(sassc, 0, handle);
507	if (targ == NULL) {
508		/* FIXME: what is the action? */
509		/* We don't know about this device? */
510		mps_dprint(sc, MPS_ERROR,
511		    "%s : invalid handle 0x%x \n", __func__, handle);
512		return;
513	}
514
515	targ->flags |= MPSSAS_TARGET_INREMOVAL;
516
517	cm = mpssas_alloc_tm(sc);
518	if (cm == NULL) {
519		mps_dprint(sc, MPS_ERROR,
520		    "%s: command alloc failure\n", __func__);
521		return;
522	}
523
524	mpssas_rescan_target(sc, targ);
525
526	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
527	memset(req, 0, sizeof(*req));
528	req->DevHandle = htole16(targ->handle);
529	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
530	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
531
532	/* SAS Hard Link Reset / SATA Link Reset */
533	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
534
535	cm->cm_targ = targ;
536	cm->cm_data = NULL;
537	cm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
538	cm->cm_complete = mpssas_remove_device;
539	cm->cm_complete_data = (void *)(uintptr_t)handle;
540
541	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
542	    __func__, targ->tid);
543	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
544
545	mps_map_command(sc, cm);
546}
547
548static void
549mpssas_remove_device(struct mps_softc *sc, struct mps_command *tm)
550{
551	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
552	MPI2_SAS_IOUNIT_CONTROL_REQUEST *req;
553	struct mpssas_target *targ;
554	struct mps_command *next_cm;
555	uint16_t handle;
556
557	MPS_FUNCTRACE(sc);
558
559	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
560	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
561	targ = tm->cm_targ;
562
563	/*
564	 * Currently there should be no way we can hit this case.  It only
565	 * happens when we have a failure to allocate chain frames, and
566	 * task management commands don't have S/G lists.
567	 */
568	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
569		mps_dprint(sc, MPS_ERROR,
570		    "%s: cm_flags = %#x for remove of handle %#04x! "
571		    "This should not happen!\n", __func__, tm->cm_flags,
572		    handle);
573	}
574
575	if (reply == NULL) {
576		/* XXX retry the remove after the diag reset completes? */
577		mps_dprint(sc, MPS_FAULT,
578		    "%s NULL reply resetting device 0x%04x\n", __func__,
579		    handle);
580		mpssas_free_tm(sc, tm);
581		return;
582	}
583
584	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
585	    MPI2_IOCSTATUS_SUCCESS) {
586		mps_dprint(sc, MPS_ERROR,
587		   "IOCStatus = 0x%x while resetting device 0x%x\n",
588		   le16toh(reply->IOCStatus), handle);
589	}
590
591	mps_dprint(sc, MPS_XINFO, "Reset aborted %u commands\n",
592	    le32toh(reply->TerminationCount));
593	mps_free_reply(sc, tm->cm_reply_data);
594	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
595
596	/* Reuse the existing command */
597	req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req;
598	memset(req, 0, sizeof(*req));
599	req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
600	req->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
601	req->DevHandle = htole16(handle);
602	tm->cm_data = NULL;
603	tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
604	tm->cm_complete = mpssas_remove_complete;
605	tm->cm_complete_data = (void *)(uintptr_t)handle;
606
607	mps_map_command(sc, tm);
608
609	mps_dprint(sc, MPS_XINFO, "clearing target %u handle 0x%04x\n",
610		   targ->tid, handle);
611	TAILQ_FOREACH_SAFE(tm, &targ->commands, cm_link, next_cm) {
612		union ccb *ccb;
613
614		mps_dprint(sc, MPS_XINFO, "Completing missed command %p\n", tm);
615		ccb = tm->cm_complete_data;
616		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
617		mpssas_scsiio_complete(sc, tm);
618	}
619}
620
621static void
622mpssas_remove_complete(struct mps_softc *sc, struct mps_command *tm)
623{
624	MPI2_SAS_IOUNIT_CONTROL_REPLY *reply;
625	uint16_t handle;
626	struct mpssas_target *targ;
627	struct mpssas_lun *lun;
628
629	MPS_FUNCTRACE(sc);
630
631	reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply;
632	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
633
634	/*
635	 * Currently there should be no way we can hit this case.  It only
636	 * happens when we have a failure to allocate chain frames, and
637	 * task management commands don't have S/G lists.
638	 */
639	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
640		mps_dprint(sc, MPS_XINFO,
641			   "%s: cm_flags = %#x for remove of handle %#04x! "
642			   "This should not happen!\n", __func__, tm->cm_flags,
643			   handle);
644		mpssas_free_tm(sc, tm);
645		return;
646	}
647
648	if (reply == NULL) {
649		/* most likely a chip reset */
650		mps_dprint(sc, MPS_FAULT,
651		    "%s NULL reply removing device 0x%04x\n", __func__, handle);
652		mpssas_free_tm(sc, tm);
653		return;
654	}
655
656	mps_dprint(sc, MPS_XINFO,
657	    "%s on handle 0x%04x, IOCStatus= 0x%x\n", __func__,
658	    handle, le16toh(reply->IOCStatus));
659
660	/*
661	 * Don't clear target if remove fails because things will get confusing.
662	 * Leave the devname and sasaddr intact so that we know to avoid reusing
663	 * this target id if possible, and so we can assign the same target id
664	 * to this device if it comes back in the future.
665	 */
666	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
667	    MPI2_IOCSTATUS_SUCCESS) {
668		targ = tm->cm_targ;
669		targ->handle = 0x0;
670		targ->encl_handle = 0x0;
671		targ->encl_slot = 0x0;
672		targ->exp_dev_handle = 0x0;
673		targ->phy_num = 0x0;
674		targ->linkrate = 0x0;
675		targ->devinfo = 0x0;
676		targ->flags = 0x0;
677
678		while(!SLIST_EMPTY(&targ->luns)) {
679			lun = SLIST_FIRST(&targ->luns);
680			SLIST_REMOVE_HEAD(&targ->luns, lun_link);
681			free(lun, M_MPT2);
682		}
683	}
684
685
686	mpssas_free_tm(sc, tm);
687}
688
689static int
690mpssas_register_events(struct mps_softc *sc)
691{
692	u32 events[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];
693
694	bzero(events, 16);
695	setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
696	setbit(events, MPI2_EVENT_SAS_DISCOVERY);
697	setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
698	setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE);
699	setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW);
700	setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
701	setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
702	setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
703	setbit(events, MPI2_EVENT_IR_VOLUME);
704	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
705	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
706	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);
707
708	mps_register_events(sc, events, mpssas_evt_handler, NULL,
709	    &sc->sassc->mpssas_eh);
710
711	return (0);
712}
713
714int
715mps_attach_sas(struct mps_softc *sc)
716{
717	struct mpssas_softc *sassc;
718	cam_status status;
719	int unit, error = 0;
720
721	MPS_FUNCTRACE(sc);
722
723	sassc = malloc(sizeof(struct mpssas_softc), M_MPT2, M_WAITOK|M_ZERO);
724	if(!sassc) {
725		device_printf(sc->mps_dev, "Cannot allocate memory %s %d\n",
726		__func__, __LINE__);
727		return (ENOMEM);
728	}
729
730	/*
731	 * XXX MaxTargets could change during a reinit.  Since we don't
732	 * resize the targets[] array during such an event, cache the value
733	 * of MaxTargets here so that we don't get into trouble later.  This
734	 * should move into the reinit logic.
735	 */
736	sassc->maxtargets = sc->facts->MaxTargets;
737	sassc->targets = malloc(sizeof(struct mpssas_target) *
738	    sassc->maxtargets, M_MPT2, M_WAITOK|M_ZERO);
739	if(!sassc->targets) {
740		device_printf(sc->mps_dev, "Cannot allocate memory %s %d\n",
741		__func__, __LINE__);
742		free(sassc, M_MPT2);
743		return (ENOMEM);
744	}
745	sc->sassc = sassc;
746	sassc->sc = sc;
747
748	if ((sassc->devq = cam_simq_alloc(sc->num_reqs)) == NULL) {
749		mps_dprint(sc, MPS_ERROR, "Cannot allocate SIMQ\n");
750		error = ENOMEM;
751		goto out;
752	}
753
754	unit = device_get_unit(sc->mps_dev);
755	sassc->sim = cam_sim_alloc(mpssas_action, mpssas_poll, "mps", sassc,
756	    unit, &sc->mps_mtx, sc->num_reqs, sc->num_reqs, sassc->devq);
757	if (sassc->sim == NULL) {
758		mps_dprint(sc, MPS_ERROR, "Cannot allocate SIM\n");
759		error = EINVAL;
760		goto out;
761	}
762
763	TAILQ_INIT(&sassc->ev_queue);
764
765	/* Initialize taskqueue for Event Handling */
766	TASK_INIT(&sassc->ev_task, 0, mpssas_firmware_event_work, sc);
767	sassc->ev_tq = taskqueue_create("mps_taskq", M_NOWAIT | M_ZERO,
768	    taskqueue_thread_enqueue, &sassc->ev_tq);
769	taskqueue_start_threads(&sassc->ev_tq, 1, PRIBIO, "%s taskq",
770	    device_get_nameunit(sc->mps_dev));
771
772	mps_lock(sc);
773
774	/*
775	 * XXX There should be a bus for every port on the adapter, but since
776	 * we're just going to fake the topology for now, we'll pretend that
777	 * everything is just a target on a single bus.
778	 */
779	if ((error = xpt_bus_register(sassc->sim, sc->mps_dev, 0)) != 0) {
780		mps_dprint(sc, MPS_ERROR, "Error %d registering SCSI bus\n",
781		    error);
782		mps_unlock(sc);
783		goto out;
784	}
785
786	/*
787	 * Assume that discovery events will start right away.
788	 *
789	 * Hold off boot until discovery is complete.
790	 */
791	sassc->flags |= MPSSAS_IN_STARTUP | MPSSAS_IN_DISCOVERY;
792	sc->sassc->startup_refcount = 0;
793	mpssas_startup_increment(sassc);
794
795	callout_init(&sassc->discovery_callout, 1 /*mpsafe*/);
796
797	/*
798	 * Register for async events so we can determine the EEDP
799	 * capabilities of devices.
800	 */
801	status = xpt_create_path(&sassc->path, /*periph*/NULL,
802	    cam_sim_path(sc->sassc->sim), CAM_TARGET_WILDCARD,
803	    CAM_LUN_WILDCARD);
804	if (status != CAM_REQ_CMP) {
805		mps_printf(sc, "Error %#x creating sim path\n", status);
806		sassc->path = NULL;
807	} else {
808		int event;
809
810#if (__FreeBSD_version >= 1000006) || \
811    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
812		event = AC_ADVINFO_CHANGED;
813#else
814		event = AC_FOUND_DEVICE;
815#endif
816		status = xpt_register_async(event, mpssas_async, sc,
817					    sassc->path);
818		if (status != CAM_REQ_CMP) {
819			mps_dprint(sc, MPS_ERROR,
820			    "Error %#x registering async handler for "
821			    "AC_ADVINFO_CHANGED events\n", status);
822			xpt_free_path(sassc->path);
823			sassc->path = NULL;
824		}
825	}
826	if (status != CAM_REQ_CMP) {
827		/*
828		 * EEDP use is the exception, not the rule.
829		 * Warn the user, but do not fail to attach.
830		 */
831		mps_printf(sc, "EEDP capabilities disabled.\n");
832	}
833
834	mps_unlock(sc);
835
836	mpssas_register_events(sc);
837out:
838	if (error)
839		mps_detach_sas(sc);
840	return (error);
841}
842
843int
844mps_detach_sas(struct mps_softc *sc)
845{
846	struct mpssas_softc *sassc;
847	struct mpssas_lun *lun, *lun_tmp;
848	struct mpssas_target *targ;
849	int i;
850
851	MPS_FUNCTRACE(sc);
852
853	if (sc->sassc == NULL)
854		return (0);
855
856	sassc = sc->sassc;
857	mps_deregister_events(sc, sassc->mpssas_eh);
858
859	/*
860	 * Drain and free the event handling taskqueue with the lock
861	 * unheld so that any parallel processing tasks drain properly
862	 * without deadlocking.
863	 */
864	if (sassc->ev_tq != NULL)
865		taskqueue_free(sassc->ev_tq);
866
867	/* Make sure CAM doesn't wedge if we had to bail out early. */
868	mps_lock(sc);
869
870	/* Deregister our async handler */
871	if (sassc->path != NULL) {
872		xpt_register_async(0, mpssas_async, sc, sassc->path);
873		xpt_free_path(sassc->path);
874		sassc->path = NULL;
875	}
876
877	if (sassc->flags & MPSSAS_IN_STARTUP)
878		xpt_release_simq(sassc->sim, 1);
879
880	if (sassc->sim != NULL) {
881		xpt_bus_deregister(cam_sim_path(sassc->sim));
882		cam_sim_free(sassc->sim, FALSE);
883	}
884
885	mps_unlock(sc);
886
887	if (sassc->devq != NULL)
888		cam_simq_free(sassc->devq);
889
890	for(i=0; i< sassc->maxtargets ;i++) {
891		targ = &sassc->targets[i];
892		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
893			free(lun, M_MPT2);
894		}
895	}
896	free(sassc->targets, M_MPT2);
897	free(sassc, M_MPT2);
898	sc->sassc = NULL;
899
900	return (0);
901}
902
903void
904mpssas_discovery_end(struct mpssas_softc *sassc)
905{
906	struct mps_softc *sc = sassc->sc;
907
908	MPS_FUNCTRACE(sc);
909
910	if (sassc->flags & MPSSAS_DISCOVERY_TIMEOUT_PENDING)
911		callout_stop(&sassc->discovery_callout);
912
913}
914
915static void
916mpssas_action(struct cam_sim *sim, union ccb *ccb)
917{
918	struct mpssas_softc *sassc;
919
920	sassc = cam_sim_softc(sim);
921
922	MPS_FUNCTRACE(sassc->sc);
923	mps_dprint(sassc->sc, MPS_TRACE, "ccb func_code 0x%x\n",
924	    ccb->ccb_h.func_code);
925	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
926
927	switch (ccb->ccb_h.func_code) {
928	case XPT_PATH_INQ:
929	{
930		struct ccb_pathinq *cpi = &ccb->cpi;
931		struct mps_softc *sc = sassc->sc;
932		uint8_t sges_per_frame;
933
934		cpi->version_num = 1;
935		cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
936		cpi->target_sprt = 0;
937#if __FreeBSD_version >= 1000039
938		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN;
939#else
940		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED;
941#endif
942		cpi->hba_eng_cnt = 0;
943		cpi->max_target = sassc->maxtargets - 1;
944		cpi->max_lun = 255;
945		cpi->initiator_id = sassc->maxtargets - 1;
946		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
947		strncpy(cpi->hba_vid, "Avago Tech (LSI)", HBA_IDLEN);
948		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
949		cpi->unit_number = cam_sim_unit(sim);
950		cpi->bus_id = cam_sim_bus(sim);
951		cpi->base_transfer_speed = 150000;
952		cpi->transport = XPORT_SAS;
953		cpi->transport_version = 0;
954		cpi->protocol = PROTO_SCSI;
955		cpi->protocol_version = SCSI_REV_SPC;
956
957		/*
958		 * Max IO Size is Page Size * the following:
959		 * ((SGEs per frame - 1 for chain element) *
960		 * Max Chain Depth) + 1 for no chain needed in last frame
961		 *
962		 * If user suggests a Max IO size to use, use the smaller of the
963		 * user's value and the calculated value as long as the user's
964		 * value is larger than 0. The user's value is in pages.
965		 */
966		sges_per_frame = ((sc->facts->IOCRequestFrameSize * 4) /
967		    sizeof(MPI2_SGE_SIMPLE64)) - 1;
968		cpi->maxio = (sges_per_frame * sc->facts->MaxChainDepth) + 1;
969		cpi->maxio *= PAGE_SIZE;
970		if ((sc->max_io_pages > 0) && (sc->max_io_pages * PAGE_SIZE <
971		    cpi->maxio))
972			cpi->maxio = sc->max_io_pages * PAGE_SIZE;
973		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
974		break;
975	}
976	case XPT_GET_TRAN_SETTINGS:
977	{
978		struct ccb_trans_settings	*cts;
979		struct ccb_trans_settings_sas	*sas;
980		struct ccb_trans_settings_scsi	*scsi;
981		struct mpssas_target *targ;
982
983		cts = &ccb->cts;
984		sas = &cts->xport_specific.sas;
985		scsi = &cts->proto_specific.scsi;
986
987		KASSERT(cts->ccb_h.target_id < sassc->maxtargets,
988		    ("Target %d out of bounds in XPT_GET_TRANS_SETTINGS\n",
989		    cts->ccb_h.target_id));
990		targ = &sassc->targets[cts->ccb_h.target_id];
991		if (targ->handle == 0x0) {
992			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
993			break;
994		}
995
996		cts->protocol_version = SCSI_REV_SPC2;
997		cts->transport = XPORT_SAS;
998		cts->transport_version = 0;
999
1000		sas->valid = CTS_SAS_VALID_SPEED;
1001		switch (targ->linkrate) {
1002		case 0x08:
1003			sas->bitrate = 150000;
1004			break;
1005		case 0x09:
1006			sas->bitrate = 300000;
1007			break;
1008		case 0x0a:
1009			sas->bitrate = 600000;
1010			break;
1011		default:
1012			sas->valid = 0;
1013		}
1014
1015		cts->protocol = PROTO_SCSI;
1016		scsi->valid = CTS_SCSI_VALID_TQ;
1017		scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
1018
1019		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1020		break;
1021	}
1022	case XPT_CALC_GEOMETRY:
1023		cam_calc_geometry(&ccb->ccg, /*extended*/1);
1024		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1025		break;
1026	case XPT_RESET_DEV:
1027		mps_dprint(sassc->sc, MPS_XINFO, "mpssas_action XPT_RESET_DEV\n");
1028		mpssas_action_resetdev(sassc, ccb);
1029		return;
1030	case XPT_RESET_BUS:
1031	case XPT_ABORT:
1032	case XPT_TERM_IO:
1033		mps_dprint(sassc->sc, MPS_XINFO,
1034		    "mpssas_action faking success for abort or reset\n");
1035		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1036		break;
1037	case XPT_SCSI_IO:
1038		mpssas_action_scsiio(sassc, ccb);
1039		return;
1040#if __FreeBSD_version >= 900026
1041	case XPT_SMP_IO:
1042		mpssas_action_smpio(sassc, ccb);
1043		return;
1044#endif
1045	default:
1046		mpssas_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL);
1047		break;
1048	}
1049	xpt_done(ccb);
1050
1051}
1052
1053static void
1054mpssas_announce_reset(struct mps_softc *sc, uint32_t ac_code,
1055    target_id_t target_id, lun_id_t lun_id)
1056{
1057	path_id_t path_id = cam_sim_path(sc->sassc->sim);
1058	struct cam_path *path;
1059
1060	mps_dprint(sc, MPS_XINFO, "%s code %x target %d lun %jx\n", __func__,
1061	    ac_code, target_id, (uintmax_t)lun_id);
1062
1063	if (xpt_create_path(&path, NULL,
1064		path_id, target_id, lun_id) != CAM_REQ_CMP) {
1065		mps_dprint(sc, MPS_ERROR, "unable to create path for reset "
1066			   "notification\n");
1067		return;
1068	}
1069
1070	xpt_async(ac_code, path, NULL);
1071	xpt_free_path(path);
1072}
1073
1074static void
1075mpssas_complete_all_commands(struct mps_softc *sc)
1076{
1077	struct mps_command *cm;
1078	int i;
1079	int completed;
1080
1081	MPS_FUNCTRACE(sc);
1082	mtx_assert(&sc->mps_mtx, MA_OWNED);
1083
1084	/* complete all commands with a NULL reply */
1085	for (i = 1; i < sc->num_reqs; i++) {
1086		cm = &sc->commands[i];
1087		cm->cm_reply = NULL;
1088		completed = 0;
1089
1090		if (cm->cm_flags & MPS_CM_FLAGS_POLLED)
1091			cm->cm_flags |= MPS_CM_FLAGS_COMPLETE;
1092
1093		if (cm->cm_complete != NULL) {
1094			mpssas_log_command(cm, MPS_RECOVERY,
1095			    "completing cm %p state %x ccb %p for diag reset\n",
1096			    cm, cm->cm_state, cm->cm_ccb);
1097
1098			cm->cm_complete(sc, cm);
1099			completed = 1;
1100		}
1101
1102		if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {
1103			mpssas_log_command(cm, MPS_RECOVERY,
1104			    "waking up cm %p state %x ccb %p for diag reset\n",
1105			    cm, cm->cm_state, cm->cm_ccb);
1106			wakeup(cm);
1107			completed = 1;
1108		}
1109
1110		if (cm->cm_sc->io_cmds_active != 0) {
1111			cm->cm_sc->io_cmds_active--;
1112		} else {
1113			mps_dprint(cm->cm_sc, MPS_INFO, "Warning: "
1114			    "io_cmds_active is out of sync - resynching to "
1115			    "0\n");
1116		}
1117
1118		if ((completed == 0) && (cm->cm_state != MPS_CM_STATE_FREE)) {
1119			/* this should never happen, but if it does, log */
1120			mpssas_log_command(cm, MPS_RECOVERY,
1121			    "cm %p state %x flags 0x%x ccb %p during diag "
1122			    "reset\n", cm, cm->cm_state, cm->cm_flags,
1123			    cm->cm_ccb);
1124		}
1125	}
1126}
1127
1128void
1129mpssas_handle_reinit(struct mps_softc *sc)
1130{
1131	int i;
1132
1133	/* Go back into startup mode and freeze the simq, so that CAM
1134	 * doesn't send any commands until after we've rediscovered all
1135	 * targets and found the proper device handles for them.
1136	 *
1137	 * After the reset, portenable will trigger discovery, and after all
1138	 * discovery-related activities have finished, the simq will be
1139	 * released.
1140	 */
1141	mps_dprint(sc, MPS_INIT, "%s startup\n", __func__);
1142	sc->sassc->flags |= MPSSAS_IN_STARTUP;
1143	sc->sassc->flags |= MPSSAS_IN_DISCOVERY;
1144	mpssas_startup_increment(sc->sassc);
1145
1146	/* notify CAM of a bus reset */
1147	mpssas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
1148	    CAM_LUN_WILDCARD);
1149
1150	/* complete and cleanup after all outstanding commands */
1151	mpssas_complete_all_commands(sc);
1152
1153	mps_dprint(sc, MPS_INIT,
1154	    "%s startup %u after command completion\n", __func__,
1155	    sc->sassc->startup_refcount);
1156
1157	/* zero all the target handles, since they may change after the
1158	 * reset, and we have to rediscover all the targets and use the new
1159	 * handles.
1160	 */
1161	for (i = 0; i < sc->sassc->maxtargets; i++) {
1162		if (sc->sassc->targets[i].outstanding != 0)
1163			mps_dprint(sc, MPS_INIT, "target %u outstanding %u\n",
1164			    i, sc->sassc->targets[i].outstanding);
1165		sc->sassc->targets[i].handle = 0x0;
1166		sc->sassc->targets[i].exp_dev_handle = 0x0;
1167		sc->sassc->targets[i].outstanding = 0;
1168		sc->sassc->targets[i].flags = MPSSAS_TARGET_INDIAGRESET;
1169	}
1170}
1171
1172static void
1173mpssas_tm_timeout(void *data)
1174{
1175	struct mps_command *tm = data;
1176	struct mps_softc *sc = tm->cm_sc;
1177
1178	mtx_assert(&sc->mps_mtx, MA_OWNED);
1179
1180	mpssas_log_command(tm, MPS_INFO|MPS_RECOVERY,
1181	    "task mgmt %p timed out\n", tm);
1182	mps_reinit(sc);
1183}
1184
1185static void
1186mpssas_logical_unit_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1187{
1188	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1189	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1190	unsigned int cm_count = 0;
1191	struct mps_command *cm;
1192	struct mpssas_target *targ;
1193
1194	callout_stop(&tm->cm_callout);
1195
1196	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1197	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1198	targ = tm->cm_targ;
1199
1200	/*
1201	 * Currently there should be no way we can hit this case.  It only
1202	 * happens when we have a failure to allocate chain frames, and
1203	 * task management commands don't have S/G lists.
1204	 * XXXSL So should it be an assertion?
1205	 */
1206	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1207		mps_dprint(sc, MPS_ERROR, "%s: cm_flags = %#x for LUN reset! "
1208			   "This should not happen!\n", __func__, tm->cm_flags);
1209		mpssas_free_tm(sc, tm);
1210		return;
1211	}
1212
1213	if (reply == NULL) {
1214		mpssas_log_command(tm, MPS_RECOVERY,
1215		    "NULL reset reply for tm %p\n", tm);
1216		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1217			/* this completion was due to a reset, just cleanup */
1218			targ->tm = NULL;
1219			mpssas_free_tm(sc, tm);
1220		}
1221		else {
1222			/* we should have gotten a reply. */
1223			mps_reinit(sc);
1224		}
1225		return;
1226	}
1227
1228	mpssas_log_command(tm, MPS_RECOVERY,
1229	    "logical unit reset status 0x%x code 0x%x count %u\n",
1230	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1231	    le32toh(reply->TerminationCount));
1232
1233	/* See if there are any outstanding commands for this LUN.
1234	 * This could be made more efficient by using a per-LU data
1235	 * structure of some sort.
1236	 */
1237	TAILQ_FOREACH(cm, &targ->commands, cm_link) {
1238		if (cm->cm_lun == tm->cm_lun)
1239			cm_count++;
1240	}
1241
1242	if (cm_count == 0) {
1243		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1244		    "logical unit %u finished recovery after reset\n",
1245		    tm->cm_lun, tm);
1246
1247		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1248		    tm->cm_lun);
1249
1250		/* we've finished recovery for this logical unit.  check and
1251		 * see if some other logical unit has a timedout command
1252		 * that needs to be processed.
1253		 */
1254		cm = TAILQ_FIRST(&targ->timedout_commands);
1255		if (cm) {
1256			mpssas_send_abort(sc, tm, cm);
1257		}
1258		else {
1259			targ->tm = NULL;
1260			mpssas_free_tm(sc, tm);
1261		}
1262	}
1263	else {
1264		/* if we still have commands for this LUN, the reset
1265		 * effectively failed, regardless of the status reported.
1266		 * Escalate to a target reset.
1267		 */
1268		mpssas_log_command(tm, MPS_RECOVERY,
1269		    "logical unit reset complete for tm %p, but still have %u command(s)\n",
1270		    tm, cm_count);
1271		mpssas_send_reset(sc, tm,
1272		    MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
1273	}
1274}
1275
1276static void
1277mpssas_target_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1278{
1279	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1280	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1281	struct mpssas_target *targ;
1282
1283	callout_stop(&tm->cm_callout);
1284
1285	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1286	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1287	targ = tm->cm_targ;
1288
1289	/*
1290	 * Currently there should be no way we can hit this case.  It only
1291	 * happens when we have a failure to allocate chain frames, and
1292	 * task management commands don't have S/G lists.
1293	 */
1294	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1295		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x for target reset! "
1296			   "This should not happen!\n", __func__, tm->cm_flags);
1297		mpssas_free_tm(sc, tm);
1298		return;
1299	}
1300
1301	if (reply == NULL) {
1302		mpssas_log_command(tm, MPS_RECOVERY,
1303		    "NULL reset reply for tm %p\n", tm);
1304		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1305			/* this completion was due to a reset, just cleanup */
1306			targ->tm = NULL;
1307			mpssas_free_tm(sc, tm);
1308		}
1309		else {
1310			/* we should have gotten a reply. */
1311			mps_reinit(sc);
1312		}
1313		return;
1314	}
1315
1316	mpssas_log_command(tm, MPS_RECOVERY,
1317	    "target reset status 0x%x code 0x%x count %u\n",
1318	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1319	    le32toh(reply->TerminationCount));
1320
1321	if (targ->outstanding == 0) {
1322		/* we've finished recovery for this target and all
1323		 * of its logical units.
1324		 */
1325		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1326		    "recovery finished after target reset\n");
1327
1328		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1329		    CAM_LUN_WILDCARD);
1330
1331		targ->tm = NULL;
1332		mpssas_free_tm(sc, tm);
1333	}
1334	else {
1335		/* after a target reset, if this target still has
1336		 * outstanding commands, the reset effectively failed,
1337		 * regardless of the status reported.  escalate.
1338		 */
1339		mpssas_log_command(tm, MPS_RECOVERY,
1340		    "target reset complete for tm %p, but still have %u command(s)\n",
1341		    tm, targ->outstanding);
1342		mps_reinit(sc);
1343	}
1344}
1345
1346#define MPS_RESET_TIMEOUT 30
1347
1348int
1349mpssas_send_reset(struct mps_softc *sc, struct mps_command *tm, uint8_t type)
1350{
1351	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1352	struct mpssas_target *target;
1353	int err;
1354
1355	target = tm->cm_targ;
1356	if (target->handle == 0) {
1357		mps_dprint(sc, MPS_ERROR,"%s null devhandle for target_id %d\n",
1358		    __func__, target->tid);
1359		return -1;
1360	}
1361
1362	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1363	req->DevHandle = htole16(target->handle);
1364	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1365	req->TaskType = type;
1366
1367	if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) {
1368		/* XXX Need to handle invalid LUNs */
1369		MPS_SET_LUN(req->LUN, tm->cm_lun);
1370		tm->cm_targ->logical_unit_resets++;
1371		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1372		    "sending logical unit reset\n");
1373		tm->cm_complete = mpssas_logical_unit_reset_complete;
1374		mpssas_prepare_for_tm(sc, tm, target, tm->cm_lun);
1375	}
1376	else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) {
1377		/*
1378		 * Target reset method =
1379		 * 	SAS Hard Link Reset / SATA Link Reset
1380		 */
1381		req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
1382		tm->cm_targ->target_resets++;
1383		mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1384		    "sending target reset\n");
1385		tm->cm_complete = mpssas_target_reset_complete;
1386		mpssas_prepare_for_tm(sc, tm, target, CAM_LUN_WILDCARD);
1387	}
1388	else {
1389		mps_dprint(sc, MPS_ERROR, "unexpected reset type 0x%x\n", type);
1390		return -1;
1391	}
1392
1393	tm->cm_data = NULL;
1394	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1395	tm->cm_complete_data = (void *)tm;
1396
1397	callout_reset(&tm->cm_callout, MPS_RESET_TIMEOUT * hz,
1398	    mpssas_tm_timeout, tm);
1399
1400	err = mps_map_command(sc, tm);
1401	if (err)
1402		mpssas_log_command(tm, MPS_RECOVERY,
1403		    "error %d sending reset type %u\n",
1404		    err, type);
1405
1406	return err;
1407}
1408
1409
1410static void
1411mpssas_abort_complete(struct mps_softc *sc, struct mps_command *tm)
1412{
1413	struct mps_command *cm;
1414	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1415	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1416	struct mpssas_target *targ;
1417
1418	callout_stop(&tm->cm_callout);
1419
1420	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1421	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1422	targ = tm->cm_targ;
1423
1424	/*
1425	 * Currently there should be no way we can hit this case.  It only
1426	 * happens when we have a failure to allocate chain frames, and
1427	 * task management commands don't have S/G lists.
1428	 */
1429	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1430		mpssas_log_command(tm, MPS_RECOVERY,
1431		    "cm_flags = %#x for abort %p TaskMID %u!\n",
1432		    tm->cm_flags, tm, le16toh(req->TaskMID));
1433		mpssas_free_tm(sc, tm);
1434		return;
1435	}
1436
1437	if (reply == NULL) {
1438		mpssas_log_command(tm, MPS_RECOVERY,
1439		    "NULL abort reply for tm %p TaskMID %u\n",
1440		    tm, le16toh(req->TaskMID));
1441		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1442			/* this completion was due to a reset, just cleanup */
1443			targ->tm = NULL;
1444			mpssas_free_tm(sc, tm);
1445		}
1446		else {
1447			/* we should have gotten a reply. */
1448			mps_reinit(sc);
1449		}
1450		return;
1451	}
1452
1453	mpssas_log_command(tm, MPS_RECOVERY,
1454	    "abort TaskMID %u status 0x%x code 0x%x count %u\n",
1455	    le16toh(req->TaskMID),
1456	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1457	    le32toh(reply->TerminationCount));
1458
1459	cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands);
1460	if (cm == NULL) {
1461		/* if there are no more timedout commands, we're done with
1462		 * error recovery for this target.
1463		 */
1464		mpssas_log_command(tm, MPS_RECOVERY,
1465		    "finished recovery after aborting TaskMID %u\n",
1466		    le16toh(req->TaskMID));
1467
1468		targ->tm = NULL;
1469		mpssas_free_tm(sc, tm);
1470	}
1471	else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) {
1472		/* abort success, but we have more timedout commands to abort */
1473		mpssas_log_command(tm, MPS_RECOVERY,
1474		    "continuing recovery after aborting TaskMID %u\n",
1475		    le16toh(req->TaskMID));
1476
1477		mpssas_send_abort(sc, tm, cm);
1478	}
1479	else {
1480		/* we didn't get a command completion, so the abort
1481		 * failed as far as we're concerned.  escalate.
1482		 */
1483		mpssas_log_command(tm, MPS_RECOVERY,
1484		    "abort failed for TaskMID %u tm %p\n",
1485		    le16toh(req->TaskMID), tm);
1486
1487		mpssas_send_reset(sc, tm,
1488		    MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET);
1489	}
1490}
1491
1492#define MPS_ABORT_TIMEOUT 5
1493
1494static int
1495mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm, struct mps_command *cm)
1496{
1497	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1498	struct mpssas_target *targ;
1499	int err;
1500
1501	targ = cm->cm_targ;
1502	if (targ->handle == 0) {
1503		mps_dprint(sc, MPS_ERROR,"%s null devhandle for target_id %d\n",
1504		    __func__, cm->cm_ccb->ccb_h.target_id);
1505		return -1;
1506	}
1507
1508	mpssas_log_command(tm, MPS_RECOVERY|MPS_INFO,
1509	    "Aborting command %p\n", cm);
1510
1511	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1512	req->DevHandle = htole16(targ->handle);
1513	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
1514	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK;
1515
1516	/* XXX Need to handle invalid LUNs */
1517	MPS_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun);
1518
1519	req->TaskMID = htole16(cm->cm_desc.Default.SMID);
1520
1521	tm->cm_data = NULL;
1522	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
1523	tm->cm_complete = mpssas_abort_complete;
1524	tm->cm_complete_data = (void *)tm;
1525	tm->cm_targ = cm->cm_targ;
1526	tm->cm_lun = cm->cm_lun;
1527
1528	callout_reset(&tm->cm_callout, MPS_ABORT_TIMEOUT * hz,
1529	    mpssas_tm_timeout, tm);
1530
1531	targ->aborts++;
1532
1533	mps_dprint(sc, MPS_INFO, "Sending reset from %s for target ID %d\n",
1534	    __func__, targ->tid);
1535	mpssas_prepare_for_tm(sc, tm, targ, tm->cm_lun);
1536
1537	err = mps_map_command(sc, tm);
1538	if (err)
1539		mpssas_log_command(tm, MPS_RECOVERY,
1540		    "error %d sending abort for cm %p SMID %u\n",
1541		    err, cm, req->TaskMID);
1542	return err;
1543}
1544
1545static void
1546mpssas_scsiio_timeout(void *data)
1547{
1548	struct mps_softc *sc;
1549	struct mps_command *cm;
1550	struct mpssas_target *targ;
1551
1552	cm = (struct mps_command *)data;
1553	sc = cm->cm_sc;
1554
1555	MPS_FUNCTRACE(sc);
1556	mtx_assert(&sc->mps_mtx, MA_OWNED);
1557
1558	mps_dprint(sc, MPS_XINFO, "Timeout checking cm %p\n", sc);
1559
1560	/*
1561	 * Run the interrupt handler to make sure it's not pending.  This
1562	 * isn't perfect because the command could have already completed
1563	 * and been re-used, though this is unlikely.
1564	 */
1565	mps_intr_locked(sc);
1566	if (cm->cm_state == MPS_CM_STATE_FREE) {
1567		mpssas_log_command(cm, MPS_XINFO,
1568		    "SCSI command %p almost timed out\n", cm);
1569		return;
1570	}
1571
1572	if (cm->cm_ccb == NULL) {
1573		mps_dprint(sc, MPS_ERROR, "command timeout with NULL ccb\n");
1574		return;
1575	}
1576
1577	mpssas_log_command(cm, MPS_INFO, "command timeout cm %p ccb %p\n",
1578	    cm, cm->cm_ccb);
1579
1580	targ = cm->cm_targ;
1581	targ->timeouts++;
1582
1583	/* XXX first, check the firmware state, to see if it's still
1584	 * operational.  if not, do a diag reset.
1585	 */
1586	mpssas_set_ccbstatus(cm->cm_ccb, CAM_CMD_TIMEOUT);
1587	cm->cm_state = MPS_CM_STATE_TIMEDOUT;
1588	TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery);
1589
1590	if (targ->tm != NULL) {
1591		/* target already in recovery, just queue up another
1592		 * timedout command to be processed later.
1593		 */
1594		mps_dprint(sc, MPS_RECOVERY,
1595		    "queued timedout cm %p for processing by tm %p\n",
1596		    cm, targ->tm);
1597	}
1598	else if ((targ->tm = mpssas_alloc_tm(sc)) != NULL) {
1599		mps_dprint(sc, MPS_RECOVERY, "timedout cm %p allocated tm %p\n",
1600		    cm, targ->tm);
1601
1602		/* start recovery by aborting the first timedout command */
1603		mpssas_send_abort(sc, targ->tm, cm);
1604	}
1605	else {
1606		/* XXX queue this target up for recovery once a TM becomes
1607		 * available.  The firmware only has a limited number of
1608		 * HighPriority credits for the high priority requests used
1609		 * for task management, and we ran out.
1610		 *
1611		 * Isilon: don't worry about this for now, since we have
1612		 * more credits than disks in an enclosure, and limit
1613		 * ourselves to one TM per target for recovery.
1614		 */
1615		mps_dprint(sc, MPS_RECOVERY,
1616		    "timedout cm %p failed to allocate a tm\n", cm);
1617	}
1618
1619}
1620
1621static void
1622mpssas_action_scsiio(struct mpssas_softc *sassc, union ccb *ccb)
1623{
1624	MPI2_SCSI_IO_REQUEST *req;
1625	struct ccb_scsiio *csio;
1626	struct mps_softc *sc;
1627	struct mpssas_target *targ;
1628	struct mpssas_lun *lun;
1629	struct mps_command *cm;
1630	uint8_t i, lba_byte, *ref_tag_addr;
1631	uint16_t eedp_flags;
1632	uint32_t mpi_control;
1633
1634	sc = sassc->sc;
1635	MPS_FUNCTRACE(sc);
1636	mtx_assert(&sc->mps_mtx, MA_OWNED);
1637
1638	csio = &ccb->csio;
1639	KASSERT(csio->ccb_h.target_id < sassc->maxtargets,
1640	    ("Target %d out of bounds in XPT_SCSI_IO\n",
1641	     csio->ccb_h.target_id));
1642	targ = &sassc->targets[csio->ccb_h.target_id];
1643	mps_dprint(sc, MPS_TRACE, "ccb %p target flag %x\n", ccb, targ->flags);
1644	if (targ->handle == 0x0) {
1645		mps_dprint(sc, MPS_ERROR, "%s NULL handle for target %u\n",
1646		    __func__, csio->ccb_h.target_id);
1647		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1648		xpt_done(ccb);
1649		return;
1650	}
1651	if (targ->flags & MPS_TARGET_FLAGS_RAID_COMPONENT) {
1652		mps_dprint(sc, MPS_ERROR, "%s Raid component no SCSI IO "
1653		    "supported %u\n", __func__, csio->ccb_h.target_id);
1654		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1655		xpt_done(ccb);
1656		return;
1657	}
1658	/*
1659	 * Sometimes, it is possible to get a command that is not "In
1660	 * Progress" and was actually aborted by the upper layer.  Check for
1661	 * this here and complete the command without error.
1662	 */
1663	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_INPROG) {
1664		mps_dprint(sc, MPS_TRACE, "%s Command is not in progress for "
1665		    "target %u\n", __func__, csio->ccb_h.target_id);
1666		xpt_done(ccb);
1667		return;
1668	}
1669	/*
1670	 * If devinfo is 0 this will be a volume.  In that case don't tell CAM
1671	 * that the volume has timed out.  We want volumes to be enumerated
1672	 * until they are deleted/removed, not just failed.
1673	 */
1674	if (targ->flags & MPSSAS_TARGET_INREMOVAL) {
1675		if (targ->devinfo == 0)
1676			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1677		else
1678			mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
1679		xpt_done(ccb);
1680		return;
1681	}
1682
1683	if ((sc->mps_flags & MPS_FLAGS_SHUTDOWN) != 0) {
1684		mps_dprint(sc, MPS_INFO, "%s shutting down\n", __func__);
1685		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1686		xpt_done(ccb);
1687		return;
1688	}
1689
1690	/*
1691	 * If target has a reset in progress, freeze the devq and return.  The
1692	 * devq will be released when the TM reset is finished.
1693	 */
1694	if (targ->flags & MPSSAS_TARGET_INRESET) {
1695		ccb->ccb_h.status = CAM_BUSY | CAM_DEV_QFRZN;
1696		mps_dprint(sc, MPS_INFO, "%s: Freezing devq for target ID %d\n",
1697		    __func__, targ->tid);
1698		xpt_freeze_devq(ccb->ccb_h.path, 1);
1699		xpt_done(ccb);
1700		return;
1701	}
1702
1703	cm = mps_alloc_command(sc);
1704	if (cm == NULL || (sc->mps_flags & MPS_FLAGS_DIAGRESET)) {
1705		if (cm != NULL) {
1706			mps_free_command(sc, cm);
1707		}
1708		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
1709			xpt_freeze_simq(sassc->sim, 1);
1710			sassc->flags |= MPSSAS_QUEUE_FROZEN;
1711		}
1712		ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1713		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1714		xpt_done(ccb);
1715		return;
1716	}
1717
1718	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
1719	bzero(req, sizeof(*req));
1720	req->DevHandle = htole16(targ->handle);
1721	req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
1722	req->MsgFlags = 0;
1723	req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);
1724	req->SenseBufferLength = MPS_SENSE_LEN;
1725	req->SGLFlags = 0;
1726	req->ChainOffset = 0;
1727	req->SGLOffset0 = 24;	/* 32bit word offset to the SGL */
1728	req->SGLOffset1= 0;
1729	req->SGLOffset2= 0;
1730	req->SGLOffset3= 0;
1731	req->SkipCount = 0;
1732	req->DataLength = htole32(csio->dxfer_len);
1733	req->BidirectionalDataLength = 0;
1734	req->IoFlags = htole16(csio->cdb_len);
1735	req->EEDPFlags = 0;
1736
1737	/* Note: BiDirectional transfers are not supported */
1738	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1739	case CAM_DIR_IN:
1740		mpi_control = MPI2_SCSIIO_CONTROL_READ;
1741		cm->cm_flags |= MPS_CM_FLAGS_DATAIN;
1742		break;
1743	case CAM_DIR_OUT:
1744		mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
1745		cm->cm_flags |= MPS_CM_FLAGS_DATAOUT;
1746		break;
1747	case CAM_DIR_NONE:
1748	default:
1749		mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
1750		break;
1751	}
1752
1753	if (csio->cdb_len == 32)
1754                mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
1755	/*
1756	 * It looks like the hardware doesn't require an explicit tag
1757	 * number for each transaction.  SAM Task Management not supported
1758	 * at the moment.
1759	 */
1760	switch (csio->tag_action) {
1761	case MSG_HEAD_OF_Q_TAG:
1762		mpi_control |= MPI2_SCSIIO_CONTROL_HEADOFQ;
1763		break;
1764	case MSG_ORDERED_Q_TAG:
1765		mpi_control |= MPI2_SCSIIO_CONTROL_ORDEREDQ;
1766		break;
1767	case MSG_ACA_TASK:
1768		mpi_control |= MPI2_SCSIIO_CONTROL_ACAQ;
1769		break;
1770	case CAM_TAG_ACTION_NONE:
1771	case MSG_SIMPLE_Q_TAG:
1772	default:
1773		mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
1774		break;
1775	}
1776	mpi_control |= sc->mapping_table[csio->ccb_h.target_id].TLR_bits;
1777	req->Control = htole32(mpi_control);
1778	if (MPS_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) {
1779		mps_free_command(sc, cm);
1780		mpssas_set_ccbstatus(ccb, CAM_LUN_INVALID);
1781		xpt_done(ccb);
1782		return;
1783	}
1784
1785	if (csio->ccb_h.flags & CAM_CDB_POINTER)
1786		bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
1787	else
1788		bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
1789	req->IoFlags = htole16(csio->cdb_len);
1790
1791	/*
1792	 * Check if EEDP is supported and enabled.  If it is then check if the
1793	 * SCSI opcode could be using EEDP.  If so, make sure the LUN exists and
1794	 * is formatted for EEDP support.  If all of this is true, set CDB up
1795	 * for EEDP transfer.
1796	 */
1797	eedp_flags = op_code_prot[req->CDB.CDB32[0]];
1798	if (sc->eedp_enabled && eedp_flags) {
1799		SLIST_FOREACH(lun, &targ->luns, lun_link) {
1800			if (lun->lun_id == csio->ccb_h.target_lun) {
1801				break;
1802			}
1803		}
1804
1805		if ((lun != NULL) && (lun->eedp_formatted)) {
1806			req->EEDPBlockSize = htole16(lun->eedp_block_size);
1807			eedp_flags |= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
1808			    MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
1809			    MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
1810			req->EEDPFlags = htole16(eedp_flags);
1811
1812			/*
1813			 * If CDB less than 32, fill in Primary Ref Tag with
1814			 * low 4 bytes of LBA.  If CDB is 32, tag stuff is
1815			 * already there.  Also, set protection bit.  FreeBSD
1816			 * currently does not support CDBs bigger than 16, but
1817			 * the code doesn't hurt, and will be here for the
1818			 * future.
1819			 */
1820			if (csio->cdb_len != 32) {
1821				lba_byte = (csio->cdb_len == 16) ? 6 : 2;
1822				ref_tag_addr = (uint8_t *)&req->CDB.EEDP32.
1823				    PrimaryReferenceTag;
1824				for (i = 0; i < 4; i++) {
1825					*ref_tag_addr =
1826					    req->CDB.CDB32[lba_byte + i];
1827					ref_tag_addr++;
1828				}
1829				req->CDB.EEDP32.PrimaryReferenceTag =
1830					htole32(req->CDB.EEDP32.PrimaryReferenceTag);
1831				req->CDB.EEDP32.PrimaryApplicationTagMask =
1832				    0xFFFF;
1833				req->CDB.CDB32[1] = (req->CDB.CDB32[1] & 0x1F) |
1834				    0x20;
1835			} else {
1836				eedp_flags |=
1837				    MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG;
1838				req->EEDPFlags = htole16(eedp_flags);
1839				req->CDB.CDB32[10] = (req->CDB.CDB32[10] &
1840				    0x1F) | 0x20;
1841			}
1842		}
1843	}
1844
1845	cm->cm_length = csio->dxfer_len;
1846	if (cm->cm_length != 0) {
1847		cm->cm_data = ccb;
1848		cm->cm_flags |= MPS_CM_FLAGS_USE_CCB;
1849	} else {
1850		cm->cm_data = NULL;
1851	}
1852	cm->cm_sge = &req->SGL;
1853	cm->cm_sglsize = (32 - 24) * 4;
1854	cm->cm_desc.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
1855	cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
1856	cm->cm_complete = mpssas_scsiio_complete;
1857	cm->cm_complete_data = ccb;
1858	cm->cm_targ = targ;
1859	cm->cm_lun = csio->ccb_h.target_lun;
1860	cm->cm_ccb = ccb;
1861
1862	/*
1863	 * If HBA is a WD and the command is not for a retry, try to build a
1864	 * direct I/O message. If failed, or the command is for a retry, send
1865	 * the I/O to the IR volume itself.
1866	 */
1867	if (sc->WD_valid_config) {
1868		if (ccb->ccb_h.sim_priv.entries[0].field == MPS_WD_RETRY) {
1869			mpssas_direct_drive_io(sassc, cm, ccb);
1870		} else {
1871			mpssas_set_ccbstatus(ccb, CAM_REQ_INPROG);
1872		}
1873	}
1874
1875	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
1876	    mpssas_scsiio_timeout, cm, 0);
1877
1878	targ->issued++;
1879	targ->outstanding++;
1880	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
1881	ccb->ccb_h.status |= CAM_SIM_QUEUED;
1882
1883	mpssas_log_command(cm, MPS_XINFO, "%s cm %p ccb %p outstanding %u\n",
1884	    __func__, cm, ccb, targ->outstanding);
1885
1886	mps_map_command(sc, cm);
1887	return;
1888}
1889
1890static void
1891mps_response_code(struct mps_softc *sc, u8 response_code)
1892{
1893        char *desc;
1894
1895        switch (response_code) {
1896        case MPI2_SCSITASKMGMT_RSP_TM_COMPLETE:
1897                desc = "task management request completed";
1898                break;
1899        case MPI2_SCSITASKMGMT_RSP_INVALID_FRAME:
1900                desc = "invalid frame";
1901                break;
1902        case MPI2_SCSITASKMGMT_RSP_TM_NOT_SUPPORTED:
1903                desc = "task management request not supported";
1904                break;
1905        case MPI2_SCSITASKMGMT_RSP_TM_FAILED:
1906                desc = "task management request failed";
1907                break;
1908        case MPI2_SCSITASKMGMT_RSP_TM_SUCCEEDED:
1909                desc = "task management request succeeded";
1910                break;
1911        case MPI2_SCSITASKMGMT_RSP_TM_INVALID_LUN:
1912                desc = "invalid lun";
1913                break;
1914        case 0xA:
1915                desc = "overlapped tag attempted";
1916                break;
1917        case MPI2_SCSITASKMGMT_RSP_IO_QUEUED_ON_IOC:
1918                desc = "task queued, however not sent to target";
1919                break;
1920        default:
1921                desc = "unknown";
1922                break;
1923        }
1924		mps_dprint(sc, MPS_XINFO, "response_code(0x%01x): %s\n",
1925                response_code, desc);
1926}
1927/**
1928 * mps_sc_failed_io_info - translated non-succesfull SCSI_IO request
1929 */
1930static void
1931mps_sc_failed_io_info(struct mps_softc *sc, struct ccb_scsiio *csio,
1932    Mpi2SCSIIOReply_t *mpi_reply)
1933{
1934	u32 response_info;
1935	u8 *response_bytes;
1936	u16 ioc_status = le16toh(mpi_reply->IOCStatus) &
1937	    MPI2_IOCSTATUS_MASK;
1938	u8 scsi_state = mpi_reply->SCSIState;
1939	u8 scsi_status = mpi_reply->SCSIStatus;
1940	char *desc_ioc_state = NULL;
1941	char *desc_scsi_status = NULL;
1942	char *desc_scsi_state = sc->tmp_string;
1943	u32 log_info = le32toh(mpi_reply->IOCLogInfo);
1944
1945	if (log_info == 0x31170000)
1946		return;
1947
1948	switch (ioc_status) {
1949	case MPI2_IOCSTATUS_SUCCESS:
1950		desc_ioc_state = "success";
1951		break;
1952	case MPI2_IOCSTATUS_INVALID_FUNCTION:
1953		desc_ioc_state = "invalid function";
1954		break;
1955	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
1956		desc_ioc_state = "scsi recovered error";
1957		break;
1958	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
1959		desc_ioc_state = "scsi invalid dev handle";
1960		break;
1961	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
1962		desc_ioc_state = "scsi device not there";
1963		break;
1964	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
1965		desc_ioc_state = "scsi data overrun";
1966		break;
1967	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
1968		desc_ioc_state = "scsi data underrun";
1969		break;
1970	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
1971		desc_ioc_state = "scsi io data error";
1972		break;
1973	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
1974		desc_ioc_state = "scsi protocol error";
1975		break;
1976	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
1977		desc_ioc_state = "scsi task terminated";
1978		break;
1979	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
1980		desc_ioc_state = "scsi residual mismatch";
1981		break;
1982	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
1983		desc_ioc_state = "scsi task mgmt failed";
1984		break;
1985	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
1986		desc_ioc_state = "scsi ioc terminated";
1987		break;
1988	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
1989		desc_ioc_state = "scsi ext terminated";
1990		break;
1991	case MPI2_IOCSTATUS_EEDP_GUARD_ERROR:
1992		desc_ioc_state = "eedp guard error";
1993		break;
1994	case MPI2_IOCSTATUS_EEDP_REF_TAG_ERROR:
1995		desc_ioc_state = "eedp ref tag error";
1996		break;
1997	case MPI2_IOCSTATUS_EEDP_APP_TAG_ERROR:
1998		desc_ioc_state = "eedp app tag error";
1999		break;
2000	default:
2001		desc_ioc_state = "unknown";
2002		break;
2003	}
2004
2005	switch (scsi_status) {
2006	case MPI2_SCSI_STATUS_GOOD:
2007		desc_scsi_status = "good";
2008		break;
2009	case MPI2_SCSI_STATUS_CHECK_CONDITION:
2010		desc_scsi_status = "check condition";
2011		break;
2012	case MPI2_SCSI_STATUS_CONDITION_MET:
2013		desc_scsi_status = "condition met";
2014		break;
2015	case MPI2_SCSI_STATUS_BUSY:
2016		desc_scsi_status = "busy";
2017		break;
2018	case MPI2_SCSI_STATUS_INTERMEDIATE:
2019		desc_scsi_status = "intermediate";
2020		break;
2021	case MPI2_SCSI_STATUS_INTERMEDIATE_CONDMET:
2022		desc_scsi_status = "intermediate condmet";
2023		break;
2024	case MPI2_SCSI_STATUS_RESERVATION_CONFLICT:
2025		desc_scsi_status = "reservation conflict";
2026		break;
2027	case MPI2_SCSI_STATUS_COMMAND_TERMINATED:
2028		desc_scsi_status = "command terminated";
2029		break;
2030	case MPI2_SCSI_STATUS_TASK_SET_FULL:
2031		desc_scsi_status = "task set full";
2032		break;
2033	case MPI2_SCSI_STATUS_ACA_ACTIVE:
2034		desc_scsi_status = "aca active";
2035		break;
2036	case MPI2_SCSI_STATUS_TASK_ABORTED:
2037		desc_scsi_status = "task aborted";
2038		break;
2039	default:
2040		desc_scsi_status = "unknown";
2041		break;
2042	}
2043
2044	desc_scsi_state[0] = '\0';
2045	if (!scsi_state)
2046		desc_scsi_state = " ";
2047	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID)
2048		strcat(desc_scsi_state, "response info ");
2049	if (scsi_state & MPI2_SCSI_STATE_TERMINATED)
2050		strcat(desc_scsi_state, "state terminated ");
2051	if (scsi_state & MPI2_SCSI_STATE_NO_SCSI_STATUS)
2052		strcat(desc_scsi_state, "no status ");
2053	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_FAILED)
2054		strcat(desc_scsi_state, "autosense failed ");
2055	if (scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID)
2056		strcat(desc_scsi_state, "autosense valid ");
2057
2058	mps_dprint(sc, MPS_XINFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x)\n",
2059	    le16toh(mpi_reply->DevHandle), desc_ioc_state, ioc_status);
2060	/* We can add more detail about underflow data here
2061	 * TO-DO
2062	 * */
2063	mps_dprint(sc, MPS_XINFO, "\tscsi_status(%s)(0x%02x), "
2064	    "scsi_state(%s)(0x%02x)\n", desc_scsi_status, scsi_status,
2065	    desc_scsi_state, scsi_state);
2066
2067	if (sc->mps_debug & MPS_XINFO &&
2068		scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2069		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : Start :\n");
2070		scsi_sense_print(csio);
2071		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : End :\n");
2072	}
2073
2074	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
2075		response_info = le32toh(mpi_reply->ResponseInfo);
2076		response_bytes = (u8 *)&response_info;
2077		mps_response_code(sc,response_bytes[0]);
2078	}
2079}
2080
2081static void
2082mpssas_scsiio_complete(struct mps_softc *sc, struct mps_command *cm)
2083{
2084	MPI2_SCSI_IO_REPLY *rep;
2085	union ccb *ccb;
2086	struct ccb_scsiio *csio;
2087	struct mpssas_softc *sassc;
2088	struct scsi_vpd_supported_page_list *vpd_list = NULL;
2089	u8 *TLR_bits, TLR_on;
2090	int dir = 0, i;
2091	u16 alloc_len;
2092	struct mpssas_target *target;
2093	target_id_t target_id;
2094
2095	MPS_FUNCTRACE(sc);
2096	mps_dprint(sc, MPS_TRACE,
2097	    "cm %p SMID %u ccb %p reply %p outstanding %u\n", cm,
2098	    cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply,
2099	    cm->cm_targ->outstanding);
2100
2101	callout_stop(&cm->cm_callout);
2102	mtx_assert(&sc->mps_mtx, MA_OWNED);
2103
2104	sassc = sc->sassc;
2105	ccb = cm->cm_complete_data;
2106	csio = &ccb->csio;
2107	target_id = csio->ccb_h.target_id;
2108	rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply;
2109	/*
2110	 * XXX KDM if the chain allocation fails, does it matter if we do
2111	 * the sync and unload here?  It is simpler to do it in every case,
2112	 * assuming it doesn't cause problems.
2113	 */
2114	if (cm->cm_data != NULL) {
2115		if (cm->cm_flags & MPS_CM_FLAGS_DATAIN)
2116			dir = BUS_DMASYNC_POSTREAD;
2117		else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT)
2118			dir = BUS_DMASYNC_POSTWRITE;
2119		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
2120		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2121	}
2122
2123	cm->cm_targ->completed++;
2124	cm->cm_targ->outstanding--;
2125	TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link);
2126	ccb->ccb_h.status &= ~(CAM_STATUS_MASK | CAM_SIM_QUEUED);
2127
2128	if (cm->cm_state == MPS_CM_STATE_TIMEDOUT) {
2129		TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery);
2130		if (cm->cm_reply != NULL)
2131			mpssas_log_command(cm, MPS_RECOVERY,
2132			    "completed timedout cm %p ccb %p during recovery "
2133			    "ioc %x scsi %x state %x xfer %u\n",
2134			    cm, cm->cm_ccb,
2135			    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2136			    le32toh(rep->TransferCount));
2137		else
2138			mpssas_log_command(cm, MPS_RECOVERY,
2139			    "completed timedout cm %p ccb %p during recovery\n",
2140			    cm, cm->cm_ccb);
2141	} else if (cm->cm_targ->tm != NULL) {
2142		if (cm->cm_reply != NULL)
2143			mpssas_log_command(cm, MPS_RECOVERY,
2144			    "completed cm %p ccb %p during recovery "
2145			    "ioc %x scsi %x state %x xfer %u\n",
2146			    cm, cm->cm_ccb,
2147			    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2148			    le32toh(rep->TransferCount));
2149		else
2150			mpssas_log_command(cm, MPS_RECOVERY,
2151			    "completed cm %p ccb %p during recovery\n",
2152			    cm, cm->cm_ccb);
2153	} else if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
2154		mpssas_log_command(cm, MPS_RECOVERY,
2155		    "reset completed cm %p ccb %p\n",
2156		    cm, cm->cm_ccb);
2157	}
2158
2159	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2160		/*
2161		 * We ran into an error after we tried to map the command,
2162		 * so we're getting a callback without queueing the command
2163		 * to the hardware.  So we set the status here, and it will
2164		 * be retained below.  We'll go through the "fast path",
2165		 * because there can be no reply when we haven't actually
2166		 * gone out to the hardware.
2167		 */
2168		mpssas_set_ccbstatus(ccb, CAM_REQUEUE_REQ);
2169
2170		/*
2171		 * Currently the only error included in the mask is
2172		 * MPS_CM_FLAGS_CHAIN_FAILED, which means we're out of
2173		 * chain frames.  We need to freeze the queue until we get
2174		 * a command that completed without this error, which will
2175		 * hopefully have some chain frames attached that we can
2176		 * use.  If we wanted to get smarter about it, we would
2177		 * only unfreeze the queue in this condition when we're
2178		 * sure that we're getting some chain frames back.  That's
2179		 * probably unnecessary.
2180		 */
2181		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
2182			xpt_freeze_simq(sassc->sim, 1);
2183			sassc->flags |= MPSSAS_QUEUE_FROZEN;
2184			mps_dprint(sc, MPS_XINFO, "Error sending command, "
2185				   "freezing SIM queue\n");
2186		}
2187	}
2188
2189	/*
2190	 * If this is a Start Stop Unit command and it was issued by the driver
2191	 * during shutdown, decrement the refcount to account for all of the
2192	 * commands that were sent.  All SSU commands should be completed before
2193	 * shutdown completes, meaning SSU_refcount will be 0 after SSU_started
2194	 * is TRUE.
2195	 */
2196	if (sc->SSU_started && (csio->cdb_io.cdb_bytes[0] == START_STOP_UNIT)) {
2197		mps_dprint(sc, MPS_INFO, "Decrementing SSU count.\n");
2198		sc->SSU_refcount--;
2199	}
2200
2201	/* Take the fast path to completion */
2202	if (cm->cm_reply == NULL) {
2203		if (mpssas_get_ccbstatus(ccb) == CAM_REQ_INPROG) {
2204			if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0)
2205				mpssas_set_ccbstatus(ccb, CAM_SCSI_BUS_RESET);
2206			else {
2207				mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2208				ccb->csio.scsi_status = SCSI_STATUS_OK;
2209			}
2210			if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2211				ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2212				sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2213				mps_dprint(sc, MPS_XINFO,
2214				    "Unfreezing SIM queue\n");
2215			}
2216		}
2217
2218		/*
2219		 * There are two scenarios where the status won't be
2220		 * CAM_REQ_CMP.  The first is if MPS_CM_FLAGS_ERROR_MASK is
2221		 * set, the second is in the MPS_FLAGS_DIAGRESET above.
2222		 */
2223		if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2224			/*
2225			 * Freeze the dev queue so that commands are
2226			 * executed in the correct order after error
2227			 * recovery.
2228			 */
2229			ccb->ccb_h.status |= CAM_DEV_QFRZN;
2230			xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2231		}
2232		mps_free_command(sc, cm);
2233		xpt_done(ccb);
2234		return;
2235	}
2236
2237	mpssas_log_command(cm, MPS_XINFO,
2238	    "ioc %x scsi %x state %x xfer %u\n",
2239	    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2240	    le32toh(rep->TransferCount));
2241
2242	/*
2243	 * If this is a Direct Drive I/O, reissue the I/O to the original IR
2244	 * Volume if an error occurred (normal I/O retry).  Use the original
2245	 * CCB, but set a flag that this will be a retry so that it's sent to
2246	 * the original volume.  Free the command but reuse the CCB.
2247	 */
2248	if (cm->cm_flags & MPS_CM_FLAGS_DD_IO) {
2249		mps_free_command(sc, cm);
2250		ccb->ccb_h.sim_priv.entries[0].field = MPS_WD_RETRY;
2251		mpssas_action_scsiio(sassc, ccb);
2252		return;
2253	} else
2254		ccb->ccb_h.sim_priv.entries[0].field = 0;
2255
2256	switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) {
2257	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
2258		csio->resid = cm->cm_length - le32toh(rep->TransferCount);
2259		/* FALLTHROUGH */
2260	case MPI2_IOCSTATUS_SUCCESS:
2261	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
2262
2263		if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
2264		    MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
2265			mpssas_log_command(cm, MPS_XINFO, "recovered error\n");
2266
2267		/* Completion failed at the transport level. */
2268		if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS |
2269		    MPI2_SCSI_STATE_TERMINATED)) {
2270			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2271			break;
2272		}
2273
2274		/* In a modern packetized environment, an autosense failure
2275		 * implies that there's not much else that can be done to
2276		 * recover the command.
2277		 */
2278		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) {
2279			mpssas_set_ccbstatus(ccb, CAM_AUTOSENSE_FAIL);
2280			break;
2281		}
2282
2283		/*
2284		 * CAM doesn't care about SAS Response Info data, but if this is
2285		 * the state check if TLR should be done.  If not, clear the
2286		 * TLR_bits for the target.
2287		 */
2288		if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) &&
2289		    ((le32toh(rep->ResponseInfo) &
2290		    MPI2_SCSI_RI_MASK_REASONCODE) ==
2291		    MPS_SCSI_RI_INVALID_FRAME)) {
2292			sc->mapping_table[target_id].TLR_bits =
2293			    (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2294		}
2295
2296		/*
2297		 * Intentionally override the normal SCSI status reporting
2298		 * for these two cases.  These are likely to happen in a
2299		 * multi-initiator environment, and we want to make sure that
2300		 * CAM retries these commands rather than fail them.
2301		 */
2302		if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) ||
2303		    (rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) {
2304			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2305			break;
2306		}
2307
2308		/* Handle normal status and sense */
2309		csio->scsi_status = rep->SCSIStatus;
2310		if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD)
2311			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2312		else
2313			mpssas_set_ccbstatus(ccb, CAM_SCSI_STATUS_ERROR);
2314
2315		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2316			int sense_len, returned_sense_len;
2317
2318			returned_sense_len = min(le32toh(rep->SenseCount),
2319			    sizeof(struct scsi_sense_data));
2320			if (returned_sense_len < ccb->csio.sense_len)
2321				ccb->csio.sense_resid = ccb->csio.sense_len -
2322					returned_sense_len;
2323			else
2324				ccb->csio.sense_resid = 0;
2325
2326			sense_len = min(returned_sense_len,
2327			    ccb->csio.sense_len - ccb->csio.sense_resid);
2328			bzero(&ccb->csio.sense_data,
2329			      sizeof(ccb->csio.sense_data));
2330			bcopy(cm->cm_sense, &ccb->csio.sense_data, sense_len);
2331			ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2332		}
2333
2334		/*
2335		 * Check if this is an INQUIRY command.  If it's a VPD inquiry,
2336		 * and it's page code 0 (Supported Page List), and there is
2337		 * inquiry data, and this is for a sequential access device, and
2338		 * the device is an SSP target, and TLR is supported by the
2339		 * controller, turn the TLR_bits value ON if page 0x90 is
2340		 * supported.
2341		 */
2342		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2343		    (csio->cdb_io.cdb_bytes[1] & SI_EVPD) &&
2344		    (csio->cdb_io.cdb_bytes[2] == SVPD_SUPPORTED_PAGE_LIST) &&
2345		    ((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) &&
2346		    (csio->data_ptr != NULL) &&
2347		    ((csio->data_ptr[0] & 0x1f) == T_SEQUENTIAL) &&
2348		    (sc->control_TLR) &&
2349		    (sc->mapping_table[target_id].device_info &
2350		    MPI2_SAS_DEVICE_INFO_SSP_TARGET)) {
2351			vpd_list = (struct scsi_vpd_supported_page_list *)
2352			    csio->data_ptr;
2353			TLR_bits = &sc->mapping_table[target_id].TLR_bits;
2354			*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2355			TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
2356			alloc_len = ((u16)csio->cdb_io.cdb_bytes[3] << 8) +
2357			    csio->cdb_io.cdb_bytes[4];
2358			alloc_len -= csio->resid;
2359			for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
2360				if (vpd_list->list[i] == 0x90) {
2361					*TLR_bits = TLR_on;
2362					break;
2363				}
2364			}
2365		}
2366
2367		/*
2368		 * If this is a SATA direct-access end device, mark it so that
2369		 * a SCSI StartStopUnit command will be sent to it when the
2370		 * driver is being shutdown.
2371		 */
2372		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2373		    ((csio->data_ptr[0] & 0x1f) == T_DIRECT) &&
2374		    (sc->mapping_table[target_id].device_info &
2375		    MPI2_SAS_DEVICE_INFO_SATA_DEVICE) &&
2376		    ((sc->mapping_table[target_id].device_info &
2377		    MPI2_SAS_DEVICE_INFO_MASK_DEVICE_TYPE) ==
2378		    MPI2_SAS_DEVICE_INFO_END_DEVICE)) {
2379			target = &sassc->targets[target_id];
2380			target->supports_SSU = TRUE;
2381			mps_dprint(sc, MPS_XINFO, "Target %d supports SSU\n",
2382			    target_id);
2383		}
2384		break;
2385	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
2386	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
2387		/*
2388		 * If devinfo is 0 this will be a volume.  In that case don't
2389		 * tell CAM that the volume is not there.  We want volumes to
2390		 * be enumerated until they are deleted/removed, not just
2391		 * failed.
2392		 */
2393		if (cm->cm_targ->devinfo == 0)
2394			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2395		else
2396			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2397		break;
2398	case MPI2_IOCSTATUS_INVALID_SGL:
2399		mps_print_scsiio_cmd(sc, cm);
2400		mpssas_set_ccbstatus(ccb, CAM_UNREC_HBA_ERROR);
2401		break;
2402	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
2403		/*
2404		 * This is one of the responses that comes back when an I/O
2405		 * has been aborted.  If it is because of a timeout that we
2406		 * initiated, just set the status to CAM_CMD_TIMEOUT.
2407		 * Otherwise set it to CAM_REQ_ABORTED.  The effect on the
2408		 * command is the same (it gets retried, subject to the
2409		 * retry counter), the only difference is what gets printed
2410		 * on the console.
2411		 */
2412		if (cm->cm_state == MPS_CM_STATE_TIMEDOUT)
2413			mpssas_set_ccbstatus(ccb, CAM_CMD_TIMEOUT);
2414		else
2415			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2416		break;
2417	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
2418		/* resid is ignored for this condition */
2419		csio->resid = 0;
2420		mpssas_set_ccbstatus(ccb, CAM_DATA_RUN_ERR);
2421		break;
2422	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
2423	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
2424		/*
2425		 * These can sometimes be transient transport-related
2426		 * errors, and sometimes persistent drive-related errors.
2427		 * We used to retry these without decrementing the retry
2428		 * count by returning CAM_REQUEUE_REQ.  Unfortunately, if
2429		 * we hit a persistent drive problem that returns one of
2430		 * these error codes, we would retry indefinitely.  So,
2431		 * return CAM_REQ_CMP_ERROR so that we decrement the retry
2432		 * count and avoid infinite retries.  We're taking the
2433		 * potential risk of flagging false failures in the event
2434		 * of a topology-related error (e.g. a SAS expander problem
2435		 * causes a command addressed to a drive to fail), but
2436		 * avoiding getting into an infinite retry loop.
2437		 */
2438		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2439		mpssas_log_command(cm, MPS_INFO,
2440		    "terminated ioc %x scsi %x state %x xfer %u\n",
2441		    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2442		    le32toh(rep->TransferCount));
2443		break;
2444	case MPI2_IOCSTATUS_INVALID_FUNCTION:
2445	case MPI2_IOCSTATUS_INTERNAL_ERROR:
2446	case MPI2_IOCSTATUS_INVALID_VPID:
2447	case MPI2_IOCSTATUS_INVALID_FIELD:
2448	case MPI2_IOCSTATUS_INVALID_STATE:
2449	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
2450	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
2451	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
2452	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
2453	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
2454	default:
2455		mpssas_log_command(cm, MPS_XINFO,
2456		    "completed ioc %x scsi %x state %x xfer %u\n",
2457		    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2458		    le32toh(rep->TransferCount));
2459		csio->resid = cm->cm_length;
2460		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2461		break;
2462	}
2463
2464	mps_sc_failed_io_info(sc,csio,rep);
2465
2466	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2467		ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2468		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2469		mps_dprint(sc, MPS_XINFO, "Command completed, "
2470		    "unfreezing SIM queue\n");
2471	}
2472
2473	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2474		ccb->ccb_h.status |= CAM_DEV_QFRZN;
2475		xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2476	}
2477
2478	mps_free_command(sc, cm);
2479	xpt_done(ccb);
2480}
2481
2482/* All Request reached here are Endian safe */
2483static void
2484mpssas_direct_drive_io(struct mpssas_softc *sassc, struct mps_command *cm,
2485    union ccb *ccb) {
2486	pMpi2SCSIIORequest_t	pIO_req;
2487	struct mps_softc	*sc = sassc->sc;
2488	uint64_t		virtLBA;
2489	uint32_t		physLBA, stripe_offset, stripe_unit;
2490	uint32_t		io_size, column;
2491	uint8_t			*ptrLBA, lba_idx, physLBA_byte, *CDB;
2492
2493	/*
2494	 * If this is a valid SCSI command (Read6, Read10, Read16, Write6,
2495	 * Write10, or Write16), build a direct I/O message.  Otherwise, the I/O
2496	 * will be sent to the IR volume itself.  Since Read6 and Write6 are a
2497	 * bit different than the 10/16 CDBs, handle them separately.
2498	 */
2499	pIO_req = (pMpi2SCSIIORequest_t)cm->cm_req;
2500	CDB = pIO_req->CDB.CDB32;
2501
2502	/*
2503	 * Handle 6 byte CDBs.
2504	 */
2505	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_6) ||
2506	    (CDB[0] == WRITE_6))) {
2507		/*
2508		 * Get the transfer size in blocks.
2509		 */
2510		io_size = (cm->cm_length >> sc->DD_block_exponent);
2511
2512		/*
2513		 * Get virtual LBA given in the CDB.
2514		 */
2515		virtLBA = ((uint64_t)(CDB[1] & 0x1F) << 16) |
2516		    ((uint64_t)CDB[2] << 8) | (uint64_t)CDB[3];
2517
2518		/*
2519		 * Check that LBA range for I/O does not exceed volume's
2520		 * MaxLBA.
2521		 */
2522		if ((virtLBA + (uint64_t)io_size - 1) <=
2523		    sc->DD_max_lba) {
2524			/*
2525			 * Check if the I/O crosses a stripe boundary.  If not,
2526			 * translate the virtual LBA to a physical LBA and set
2527			 * the DevHandle for the PhysDisk to be used.  If it
2528			 * does cross a boundry, do normal I/O.  To get the
2529			 * right DevHandle to use, get the map number for the
2530			 * column, then use that map number to look up the
2531			 * DevHandle of the PhysDisk.
2532			 */
2533			stripe_offset = (uint32_t)virtLBA &
2534			    (sc->DD_stripe_size - 1);
2535			if ((stripe_offset + io_size) <= sc->DD_stripe_size) {
2536				physLBA = (uint32_t)virtLBA >>
2537				    sc->DD_stripe_exponent;
2538				stripe_unit = physLBA / sc->DD_num_phys_disks;
2539				column = physLBA % sc->DD_num_phys_disks;
2540				pIO_req->DevHandle =
2541				    htole16(sc->DD_column_map[column].dev_handle);
2542				/* ???? Is this endian safe*/
2543				cm->cm_desc.SCSIIO.DevHandle =
2544				    pIO_req->DevHandle;
2545
2546				physLBA = (stripe_unit <<
2547				    sc->DD_stripe_exponent) + stripe_offset;
2548				ptrLBA = &pIO_req->CDB.CDB32[1];
2549				physLBA_byte = (uint8_t)(physLBA >> 16);
2550				*ptrLBA = physLBA_byte;
2551				ptrLBA = &pIO_req->CDB.CDB32[2];
2552				physLBA_byte = (uint8_t)(physLBA >> 8);
2553				*ptrLBA = physLBA_byte;
2554				ptrLBA = &pIO_req->CDB.CDB32[3];
2555				physLBA_byte = (uint8_t)physLBA;
2556				*ptrLBA = physLBA_byte;
2557
2558				/*
2559				 * Set flag that Direct Drive I/O is
2560				 * being done.
2561				 */
2562				cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2563			}
2564		}
2565		return;
2566	}
2567
2568	/*
2569	 * Handle 10, 12 or 16 byte CDBs.
2570	 */
2571	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_10) ||
2572	    (CDB[0] == WRITE_10) || (CDB[0] == READ_16) ||
2573	    (CDB[0] == WRITE_16) || (CDB[0] == READ_12) ||
2574	    (CDB[0] == WRITE_12))) {
2575		/*
2576		 * For 16-byte CDB's, verify that the upper 4 bytes of the CDB
2577		 * are 0.  If not, this is accessing beyond 2TB so handle it in
2578		 * the else section.  10-byte and 12-byte CDB's are OK.
2579		 * FreeBSD sends very rare 12 byte READ/WRITE, but driver is
2580		 * ready to accept 12byte CDB for Direct IOs.
2581		 */
2582		if ((CDB[0] == READ_10 || CDB[0] == WRITE_10) ||
2583		    (CDB[0] == READ_12 || CDB[0] == WRITE_12) ||
2584		    !(CDB[2] | CDB[3] | CDB[4] | CDB[5])) {
2585			/*
2586			 * Get the transfer size in blocks.
2587			 */
2588			io_size = (cm->cm_length >> sc->DD_block_exponent);
2589
2590			/*
2591			 * Get virtual LBA.  Point to correct lower 4 bytes of
2592			 * LBA in the CDB depending on command.
2593			 */
2594			lba_idx = ((CDB[0] == READ_12) ||
2595				(CDB[0] == WRITE_12) ||
2596				(CDB[0] == READ_10) ||
2597				(CDB[0] == WRITE_10))? 2 : 6;
2598			virtLBA = ((uint64_t)CDB[lba_idx] << 24) |
2599			    ((uint64_t)CDB[lba_idx + 1] << 16) |
2600			    ((uint64_t)CDB[lba_idx + 2] << 8) |
2601			    (uint64_t)CDB[lba_idx + 3];
2602
2603			/*
2604			 * Check that LBA range for I/O does not exceed volume's
2605			 * MaxLBA.
2606			 */
2607			if ((virtLBA + (uint64_t)io_size - 1) <=
2608			    sc->DD_max_lba) {
2609				/*
2610				 * Check if the I/O crosses a stripe boundary.
2611				 * If not, translate the virtual LBA to a
2612				 * physical LBA and set the DevHandle for the
2613				 * PhysDisk to be used.  If it does cross a
2614				 * boundry, do normal I/O.  To get the right
2615				 * DevHandle to use, get the map number for the
2616				 * column, then use that map number to look up
2617				 * the DevHandle of the PhysDisk.
2618				 */
2619				stripe_offset = (uint32_t)virtLBA &
2620				    (sc->DD_stripe_size - 1);
2621				if ((stripe_offset + io_size) <=
2622				    sc->DD_stripe_size) {
2623					physLBA = (uint32_t)virtLBA >>
2624					    sc->DD_stripe_exponent;
2625					stripe_unit = physLBA /
2626					    sc->DD_num_phys_disks;
2627					column = physLBA %
2628					    sc->DD_num_phys_disks;
2629					pIO_req->DevHandle =
2630					    htole16(sc->DD_column_map[column].
2631					    dev_handle);
2632					cm->cm_desc.SCSIIO.DevHandle =
2633					    pIO_req->DevHandle;
2634
2635					physLBA = (stripe_unit <<
2636					    sc->DD_stripe_exponent) +
2637					    stripe_offset;
2638					ptrLBA =
2639					    &pIO_req->CDB.CDB32[lba_idx];
2640					physLBA_byte = (uint8_t)(physLBA >> 24);
2641					*ptrLBA = physLBA_byte;
2642					ptrLBA =
2643					    &pIO_req->CDB.CDB32[lba_idx + 1];
2644					physLBA_byte = (uint8_t)(physLBA >> 16);
2645					*ptrLBA = physLBA_byte;
2646					ptrLBA =
2647					    &pIO_req->CDB.CDB32[lba_idx + 2];
2648					physLBA_byte = (uint8_t)(physLBA >> 8);
2649					*ptrLBA = physLBA_byte;
2650					ptrLBA =
2651					    &pIO_req->CDB.CDB32[lba_idx + 3];
2652					physLBA_byte = (uint8_t)physLBA;
2653					*ptrLBA = physLBA_byte;
2654
2655					/*
2656					 * Set flag that Direct Drive I/O is
2657					 * being done.
2658					 */
2659					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2660				}
2661			}
2662		} else {
2663			/*
2664			 * 16-byte CDB and the upper 4 bytes of the CDB are not
2665			 * 0.  Get the transfer size in blocks.
2666			 */
2667			io_size = (cm->cm_length >> sc->DD_block_exponent);
2668
2669			/*
2670			 * Get virtual LBA.
2671			 */
2672			virtLBA = ((uint64_t)CDB[2] << 54) |
2673			    ((uint64_t)CDB[3] << 48) |
2674			    ((uint64_t)CDB[4] << 40) |
2675			    ((uint64_t)CDB[5] << 32) |
2676			    ((uint64_t)CDB[6] << 24) |
2677			    ((uint64_t)CDB[7] << 16) |
2678			    ((uint64_t)CDB[8] << 8) |
2679			    (uint64_t)CDB[9];
2680
2681			/*
2682			 * Check that LBA range for I/O does not exceed volume's
2683			 * MaxLBA.
2684			 */
2685			if ((virtLBA + (uint64_t)io_size - 1) <=
2686			    sc->DD_max_lba) {
2687				/*
2688				 * Check if the I/O crosses a stripe boundary.
2689				 * If not, translate the virtual LBA to a
2690				 * physical LBA and set the DevHandle for the
2691				 * PhysDisk to be used.  If it does cross a
2692				 * boundry, do normal I/O.  To get the right
2693				 * DevHandle to use, get the map number for the
2694				 * column, then use that map number to look up
2695				 * the DevHandle of the PhysDisk.
2696				 */
2697				stripe_offset = (uint32_t)virtLBA &
2698				    (sc->DD_stripe_size - 1);
2699				if ((stripe_offset + io_size) <=
2700				    sc->DD_stripe_size) {
2701					physLBA = (uint32_t)(virtLBA >>
2702					    sc->DD_stripe_exponent);
2703					stripe_unit = physLBA /
2704					    sc->DD_num_phys_disks;
2705					column = physLBA %
2706					    sc->DD_num_phys_disks;
2707					pIO_req->DevHandle =
2708					    htole16(sc->DD_column_map[column].
2709					    dev_handle);
2710					cm->cm_desc.SCSIIO.DevHandle =
2711					    pIO_req->DevHandle;
2712
2713					physLBA = (stripe_unit <<
2714					    sc->DD_stripe_exponent) +
2715					    stripe_offset;
2716
2717					/*
2718					 * Set upper 4 bytes of LBA to 0.  We
2719					 * assume that the phys disks are less
2720					 * than 2 TB's in size.  Then, set the
2721					 * lower 4 bytes.
2722					 */
2723					pIO_req->CDB.CDB32[2] = 0;
2724					pIO_req->CDB.CDB32[3] = 0;
2725					pIO_req->CDB.CDB32[4] = 0;
2726					pIO_req->CDB.CDB32[5] = 0;
2727					ptrLBA = &pIO_req->CDB.CDB32[6];
2728					physLBA_byte = (uint8_t)(physLBA >> 24);
2729					*ptrLBA = physLBA_byte;
2730					ptrLBA = &pIO_req->CDB.CDB32[7];
2731					physLBA_byte = (uint8_t)(physLBA >> 16);
2732					*ptrLBA = physLBA_byte;
2733					ptrLBA = &pIO_req->CDB.CDB32[8];
2734					physLBA_byte = (uint8_t)(physLBA >> 8);
2735					*ptrLBA = physLBA_byte;
2736					ptrLBA = &pIO_req->CDB.CDB32[9];
2737					physLBA_byte = (uint8_t)physLBA;
2738					*ptrLBA = physLBA_byte;
2739
2740					/*
2741					 * Set flag that Direct Drive I/O is
2742					 * being done.
2743					 */
2744					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2745				}
2746			}
2747		}
2748	}
2749}
2750
2751#if __FreeBSD_version >= 900026
2752static void
2753mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm)
2754{
2755	MPI2_SMP_PASSTHROUGH_REPLY *rpl;
2756	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2757	uint64_t sasaddr;
2758	union ccb *ccb;
2759
2760	ccb = cm->cm_complete_data;
2761
2762	/*
2763	 * Currently there should be no way we can hit this case.  It only
2764	 * happens when we have a failure to allocate chain frames, and SMP
2765	 * commands require two S/G elements only.  That should be handled
2766	 * in the standard request size.
2767	 */
2768	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2769		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x on SMP request!\n",
2770			   __func__, cm->cm_flags);
2771		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2772		goto bailout;
2773        }
2774
2775	rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply;
2776	if (rpl == NULL) {
2777		mps_dprint(sc, MPS_ERROR, "%s: NULL cm_reply!\n", __func__);
2778		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2779		goto bailout;
2780	}
2781
2782	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2783	sasaddr = le32toh(req->SASAddress.Low);
2784	sasaddr |= ((uint64_t)(le32toh(req->SASAddress.High))) << 32;
2785
2786	if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
2787	    MPI2_IOCSTATUS_SUCCESS ||
2788	    rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) {
2789		mps_dprint(sc, MPS_XINFO, "%s: IOCStatus %04x SASStatus %02x\n",
2790		    __func__, le16toh(rpl->IOCStatus), rpl->SASStatus);
2791		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2792		goto bailout;
2793	}
2794
2795	mps_dprint(sc, MPS_XINFO, "%s: SMP request to SAS address "
2796		   "%#jx completed successfully\n", __func__,
2797		   (uintmax_t)sasaddr);
2798
2799	if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED)
2800		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2801	else
2802		mpssas_set_ccbstatus(ccb, CAM_SMP_STATUS_ERROR);
2803
2804bailout:
2805	/*
2806	 * We sync in both directions because we had DMAs in the S/G list
2807	 * in both directions.
2808	 */
2809	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
2810			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2811	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2812	mps_free_command(sc, cm);
2813	xpt_done(ccb);
2814}
2815
2816static void
2817mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb, uint64_t sasaddr)
2818{
2819	struct mps_command *cm;
2820	uint8_t *request, *response;
2821	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2822	struct mps_softc *sc;
2823	struct sglist *sg;
2824	int error;
2825
2826	sc = sassc->sc;
2827	sg = NULL;
2828	error = 0;
2829
2830	/*
2831	 * XXX We don't yet support physical addresses here.
2832	 */
2833	switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) {
2834	case CAM_DATA_PADDR:
2835	case CAM_DATA_SG_PADDR:
2836		mps_dprint(sc, MPS_ERROR,
2837			   "%s: physical addresses not supported\n", __func__);
2838		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2839		xpt_done(ccb);
2840		return;
2841	case CAM_DATA_SG:
2842		/*
2843		 * The chip does not support more than one buffer for the
2844		 * request or response.
2845		 */
2846	 	if ((ccb->smpio.smp_request_sglist_cnt > 1)
2847		  || (ccb->smpio.smp_response_sglist_cnt > 1)) {
2848			mps_dprint(sc, MPS_ERROR,
2849				   "%s: multiple request or response "
2850				   "buffer segments not supported for SMP\n",
2851				   __func__);
2852			mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2853			xpt_done(ccb);
2854			return;
2855		}
2856
2857		/*
2858		 * The CAM_SCATTER_VALID flag was originally implemented
2859		 * for the XPT_SCSI_IO CCB, which only has one data pointer.
2860		 * We have two.  So, just take that flag to mean that we
2861		 * might have S/G lists, and look at the S/G segment count
2862		 * to figure out whether that is the case for each individual
2863		 * buffer.
2864		 */
2865		if (ccb->smpio.smp_request_sglist_cnt != 0) {
2866			bus_dma_segment_t *req_sg;
2867
2868			req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request;
2869			request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr;
2870		} else
2871			request = ccb->smpio.smp_request;
2872
2873		if (ccb->smpio.smp_response_sglist_cnt != 0) {
2874			bus_dma_segment_t *rsp_sg;
2875
2876			rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response;
2877			response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr;
2878		} else
2879			response = ccb->smpio.smp_response;
2880		break;
2881	case CAM_DATA_VADDR:
2882		request = ccb->smpio.smp_request;
2883		response = ccb->smpio.smp_response;
2884		break;
2885	default:
2886		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2887		xpt_done(ccb);
2888		return;
2889	}
2890
2891	cm = mps_alloc_command(sc);
2892	if (cm == NULL) {
2893		mps_dprint(sc, MPS_ERROR,
2894		    "%s: cannot allocate command\n", __func__);
2895		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2896		xpt_done(ccb);
2897		return;
2898	}
2899
2900	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2901	bzero(req, sizeof(*req));
2902	req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH;
2903
2904	/* Allow the chip to use any route to this SAS address. */
2905	req->PhysicalPort = 0xff;
2906
2907	req->RequestDataLength = htole16(ccb->smpio.smp_request_len);
2908	req->SGLFlags =
2909	    MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE | MPI2_SGLFLAGS_SGL_TYPE_MPI;
2910
2911	mps_dprint(sc, MPS_XINFO, "%s: sending SMP request to SAS "
2912	    "address %#jx\n", __func__, (uintmax_t)sasaddr);
2913
2914	mpi_init_sge(cm, req, &req->SGL);
2915
2916	/*
2917	 * Set up a uio to pass into mps_map_command().  This allows us to
2918	 * do one map command, and one busdma call in there.
2919	 */
2920	cm->cm_uio.uio_iov = cm->cm_iovec;
2921	cm->cm_uio.uio_iovcnt = 2;
2922	cm->cm_uio.uio_segflg = UIO_SYSSPACE;
2923
2924	/*
2925	 * The read/write flag isn't used by busdma, but set it just in
2926	 * case.  This isn't exactly accurate, either, since we're going in
2927	 * both directions.
2928	 */
2929	cm->cm_uio.uio_rw = UIO_WRITE;
2930
2931	cm->cm_iovec[0].iov_base = request;
2932	cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength);
2933	cm->cm_iovec[1].iov_base = response;
2934	cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len;
2935
2936	cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len +
2937			       cm->cm_iovec[1].iov_len;
2938
2939	/*
2940	 * Trigger a warning message in mps_data_cb() for the user if we
2941	 * wind up exceeding two S/G segments.  The chip expects one
2942	 * segment for the request and another for the response.
2943	 */
2944	cm->cm_max_segs = 2;
2945
2946	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
2947	cm->cm_complete = mpssas_smpio_complete;
2948	cm->cm_complete_data = ccb;
2949
2950	/*
2951	 * Tell the mapping code that we're using a uio, and that this is
2952	 * an SMP passthrough request.  There is a little special-case
2953	 * logic there (in mps_data_cb()) to handle the bidirectional
2954	 * transfer.
2955	 */
2956	cm->cm_flags |= MPS_CM_FLAGS_USE_UIO | MPS_CM_FLAGS_SMP_PASS |
2957			MPS_CM_FLAGS_DATAIN | MPS_CM_FLAGS_DATAOUT;
2958
2959	/* The chip data format is little endian. */
2960	req->SASAddress.High = htole32(sasaddr >> 32);
2961	req->SASAddress.Low = htole32(sasaddr);
2962
2963	/*
2964	 * XXX Note that we don't have a timeout/abort mechanism here.
2965	 * From the manual, it looks like task management requests only
2966	 * work for SCSI IO and SATA passthrough requests.  We may need to
2967	 * have a mechanism to retry requests in the event of a chip reset
2968	 * at least.  Hopefully the chip will insure that any errors short
2969	 * of that are relayed back to the driver.
2970	 */
2971	error = mps_map_command(sc, cm);
2972	if ((error != 0) && (error != EINPROGRESS)) {
2973		mps_dprint(sc, MPS_ERROR,
2974			   "%s: error %d returned from mps_map_command()\n",
2975			   __func__, error);
2976		goto bailout_error;
2977	}
2978
2979	return;
2980
2981bailout_error:
2982	mps_free_command(sc, cm);
2983	mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2984	xpt_done(ccb);
2985	return;
2986
2987}
2988
2989static void
2990mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb)
2991{
2992	struct mps_softc *sc;
2993	struct mpssas_target *targ;
2994	uint64_t sasaddr = 0;
2995
2996	sc = sassc->sc;
2997
2998	/*
2999	 * Make sure the target exists.
3000	 */
3001	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
3002	    ("Target %d out of bounds in XPT_SMP_IO\n", ccb->ccb_h.target_id));
3003	targ = &sassc->targets[ccb->ccb_h.target_id];
3004	if (targ->handle == 0x0) {
3005		mps_dprint(sc, MPS_ERROR,
3006			   "%s: target %d does not exist!\n", __func__,
3007			   ccb->ccb_h.target_id);
3008		mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
3009		xpt_done(ccb);
3010		return;
3011	}
3012
3013	/*
3014	 * If this device has an embedded SMP target, we'll talk to it
3015	 * directly.
3016	 * figure out what the expander's address is.
3017	 */
3018	if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0)
3019		sasaddr = targ->sasaddr;
3020
3021	/*
3022	 * If we don't have a SAS address for the expander yet, try
3023	 * grabbing it from the page 0x83 information cached in the
3024	 * transport layer for this target.  LSI expanders report the
3025	 * expander SAS address as the port-associated SAS address in
3026	 * Inquiry VPD page 0x83.  Maxim expanders don't report it in page
3027	 * 0x83.
3028	 *
3029	 * XXX KDM disable this for now, but leave it commented out so that
3030	 * it is obvious that this is another possible way to get the SAS
3031	 * address.
3032	 *
3033	 * The parent handle method below is a little more reliable, and
3034	 * the other benefit is that it works for devices other than SES
3035	 * devices.  So you can send a SMP request to a da(4) device and it
3036	 * will get routed to the expander that device is attached to.
3037	 * (Assuming the da(4) device doesn't contain an SMP target...)
3038	 */
3039#if 0
3040	if (sasaddr == 0)
3041		sasaddr = xpt_path_sas_addr(ccb->ccb_h.path);
3042#endif
3043
3044	/*
3045	 * If we still don't have a SAS address for the expander, look for
3046	 * the parent device of this device, which is probably the expander.
3047	 */
3048	if (sasaddr == 0) {
3049#ifdef OLD_MPS_PROBE
3050		struct mpssas_target *parent_target;
3051#endif
3052
3053		if (targ->parent_handle == 0x0) {
3054			mps_dprint(sc, MPS_ERROR,
3055				   "%s: handle %d does not have a valid "
3056				   "parent handle!\n", __func__, targ->handle);
3057			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3058			goto bailout;
3059		}
3060#ifdef OLD_MPS_PROBE
3061		parent_target = mpssas_find_target_by_handle(sassc, 0,
3062			targ->parent_handle);
3063
3064		if (parent_target == NULL) {
3065			mps_dprint(sc, MPS_ERROR,
3066				   "%s: handle %d does not have a valid "
3067				   "parent target!\n", __func__, targ->handle);
3068			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3069			goto bailout;
3070		}
3071
3072		if ((parent_target->devinfo &
3073		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3074			mps_dprint(sc, MPS_ERROR,
3075				   "%s: handle %d parent %d does not "
3076				   "have an SMP target!\n", __func__,
3077				   targ->handle, parent_target->handle);
3078			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3079			goto bailout;
3080
3081		}
3082
3083		sasaddr = parent_target->sasaddr;
3084#else /* OLD_MPS_PROBE */
3085		if ((targ->parent_devinfo &
3086		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3087			mps_dprint(sc, MPS_ERROR,
3088				   "%s: handle %d parent %d does not "
3089				   "have an SMP target!\n", __func__,
3090				   targ->handle, targ->parent_handle);
3091			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3092			goto bailout;
3093
3094		}
3095		if (targ->parent_sasaddr == 0x0) {
3096			mps_dprint(sc, MPS_ERROR,
3097				   "%s: handle %d parent handle %d does "
3098				   "not have a valid SAS address!\n",
3099				   __func__, targ->handle, targ->parent_handle);
3100			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3101			goto bailout;
3102		}
3103
3104		sasaddr = targ->parent_sasaddr;
3105#endif /* OLD_MPS_PROBE */
3106
3107	}
3108
3109	if (sasaddr == 0) {
3110		mps_dprint(sc, MPS_INFO,
3111			   "%s: unable to find SAS address for handle %d\n",
3112			   __func__, targ->handle);
3113		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3114		goto bailout;
3115	}
3116	mpssas_send_smpcmd(sassc, ccb, sasaddr);
3117
3118	return;
3119
3120bailout:
3121	xpt_done(ccb);
3122
3123}
3124#endif //__FreeBSD_version >= 900026
3125
3126static void
3127mpssas_action_resetdev(struct mpssas_softc *sassc, union ccb *ccb)
3128{
3129	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3130	struct mps_softc *sc;
3131	struct mps_command *tm;
3132	struct mpssas_target *targ;
3133
3134	MPS_FUNCTRACE(sassc->sc);
3135	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
3136
3137	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
3138	    ("Target %d out of bounds in XPT_RESET_DEV\n",
3139	     ccb->ccb_h.target_id));
3140	sc = sassc->sc;
3141	tm = mps_alloc_command(sc);
3142	if (tm == NULL) {
3143		mps_dprint(sc, MPS_ERROR,
3144		    "command alloc failure in mpssas_action_resetdev\n");
3145		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
3146		xpt_done(ccb);
3147		return;
3148	}
3149
3150	targ = &sassc->targets[ccb->ccb_h.target_id];
3151	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3152	req->DevHandle = htole16(targ->handle);
3153	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
3154	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
3155
3156	/* SAS Hard Link Reset / SATA Link Reset */
3157	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3158
3159	tm->cm_data = NULL;
3160	tm->cm_desc.HighPriority.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_HIGH_PRIORITY;
3161	tm->cm_complete = mpssas_resetdev_complete;
3162	tm->cm_complete_data = ccb;
3163	tm->cm_targ = targ;
3164	targ->flags |= MPSSAS_TARGET_INRESET;
3165
3166	mps_map_command(sc, tm);
3167}
3168
3169static void
3170mpssas_resetdev_complete(struct mps_softc *sc, struct mps_command *tm)
3171{
3172	MPI2_SCSI_TASK_MANAGE_REPLY *resp;
3173	union ccb *ccb;
3174
3175	MPS_FUNCTRACE(sc);
3176	mtx_assert(&sc->mps_mtx, MA_OWNED);
3177
3178	resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
3179	ccb = tm->cm_complete_data;
3180
3181	/*
3182	 * Currently there should be no way we can hit this case.  It only
3183	 * happens when we have a failure to allocate chain frames, and
3184	 * task management commands don't have S/G lists.
3185	 */
3186	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3187		MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3188
3189		req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3190
3191		mps_dprint(sc, MPS_ERROR,
3192			   "%s: cm_flags = %#x for reset of handle %#04x! "
3193			   "This should not happen!\n", __func__, tm->cm_flags,
3194			   req->DevHandle);
3195		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3196		goto bailout;
3197	}
3198
3199	mps_dprint(sc, MPS_XINFO,
3200	    "%s: IOCStatus = 0x%x ResponseCode = 0x%x\n", __func__,
3201	    le16toh(resp->IOCStatus), le32toh(resp->ResponseCode));
3202
3203	if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) {
3204		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
3205		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
3206		    CAM_LUN_WILDCARD);
3207	}
3208	else
3209		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3210
3211bailout:
3212
3213	mpssas_free_tm(sc, tm);
3214	xpt_done(ccb);
3215}
3216
3217static void
3218mpssas_poll(struct cam_sim *sim)
3219{
3220	struct mpssas_softc *sassc;
3221
3222	sassc = cam_sim_softc(sim);
3223
3224	if (sassc->sc->mps_debug & MPS_TRACE) {
3225		/* frequent debug messages during a panic just slow
3226		 * everything down too much.
3227		 */
3228		mps_printf(sassc->sc, "%s clearing MPS_TRACE\n", __func__);
3229		sassc->sc->mps_debug &= ~MPS_TRACE;
3230	}
3231
3232	mps_intr_locked(sassc->sc);
3233}
3234
3235static void
3236mpssas_async(void *callback_arg, uint32_t code, struct cam_path *path,
3237	     void *arg)
3238{
3239	struct mps_softc *sc;
3240
3241	sc = (struct mps_softc *)callback_arg;
3242
3243	switch (code) {
3244#if (__FreeBSD_version >= 1000006) || \
3245    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
3246	case AC_ADVINFO_CHANGED: {
3247		struct mpssas_target *target;
3248		struct mpssas_softc *sassc;
3249		struct scsi_read_capacity_data_long rcap_buf;
3250		struct ccb_dev_advinfo cdai;
3251		struct mpssas_lun *lun;
3252		lun_id_t lunid;
3253		int found_lun;
3254		uintptr_t buftype;
3255
3256		buftype = (uintptr_t)arg;
3257
3258		found_lun = 0;
3259		sassc = sc->sassc;
3260
3261		/*
3262		 * We're only interested in read capacity data changes.
3263		 */
3264		if (buftype != CDAI_TYPE_RCAPLONG)
3265			break;
3266
3267		/*
3268		 * We should have a handle for this, but check to make sure.
3269		 */
3270		KASSERT(xpt_path_target_id(path) < sassc->maxtargets,
3271		    ("Target %d out of bounds in mpssas_async\n",
3272		    xpt_path_target_id(path)));
3273		target = &sassc->targets[xpt_path_target_id(path)];
3274		if (target->handle == 0)
3275			break;
3276
3277		lunid = xpt_path_lun_id(path);
3278
3279		SLIST_FOREACH(lun, &target->luns, lun_link) {
3280			if (lun->lun_id == lunid) {
3281				found_lun = 1;
3282				break;
3283			}
3284		}
3285
3286		if (found_lun == 0) {
3287			lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3288				     M_NOWAIT | M_ZERO);
3289			if (lun == NULL) {
3290				mps_dprint(sc, MPS_ERROR, "Unable to alloc "
3291					   "LUN for EEDP support.\n");
3292				break;
3293			}
3294			lun->lun_id = lunid;
3295			SLIST_INSERT_HEAD(&target->luns, lun, lun_link);
3296		}
3297
3298		bzero(&rcap_buf, sizeof(rcap_buf));
3299		xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
3300		cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
3301		cdai.ccb_h.flags = CAM_DIR_IN;
3302		cdai.buftype = CDAI_TYPE_RCAPLONG;
3303#if (__FreeBSD_version >= 1100061) || \
3304    ((__FreeBSD_version >= 1001510) && (__FreeBSD_version < 1100000))
3305		cdai.flags = CDAI_FLAG_NONE;
3306#else
3307		cdai.flags = 0;
3308#endif
3309		cdai.bufsiz = sizeof(rcap_buf);
3310		cdai.buf = (uint8_t *)&rcap_buf;
3311		xpt_action((union ccb *)&cdai);
3312		if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
3313			cam_release_devq(cdai.ccb_h.path,
3314					 0, 0, 0, FALSE);
3315
3316		if ((mpssas_get_ccbstatus((union ccb *)&cdai) == CAM_REQ_CMP)
3317		 && (rcap_buf.prot & SRC16_PROT_EN)) {
3318			lun->eedp_formatted = TRUE;
3319			lun->eedp_block_size = scsi_4btoul(rcap_buf.length);
3320		} else {
3321			lun->eedp_formatted = FALSE;
3322			lun->eedp_block_size = 0;
3323		}
3324		break;
3325	}
3326#else
3327	case AC_FOUND_DEVICE: {
3328		struct ccb_getdev *cgd;
3329
3330		cgd = arg;
3331		mpssas_check_eedp(sc, path, cgd);
3332		break;
3333	}
3334#endif
3335	default:
3336		break;
3337	}
3338}
3339
3340#if (__FreeBSD_version < 901503) || \
3341    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
3342static void
3343mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
3344		  struct ccb_getdev *cgd)
3345{
3346	struct mpssas_softc *sassc = sc->sassc;
3347	struct ccb_scsiio *csio;
3348	struct scsi_read_capacity_16 *scsi_cmd;
3349	struct scsi_read_capacity_eedp *rcap_buf;
3350	path_id_t pathid;
3351	target_id_t targetid;
3352	lun_id_t lunid;
3353	union ccb *ccb;
3354	struct cam_path *local_path;
3355	struct mpssas_target *target;
3356	struct mpssas_lun *lun;
3357	uint8_t	found_lun;
3358	char path_str[64];
3359
3360	sassc = sc->sassc;
3361	pathid = cam_sim_path(sassc->sim);
3362	targetid = xpt_path_target_id(path);
3363	lunid = xpt_path_lun_id(path);
3364
3365	KASSERT(targetid < sassc->maxtargets,
3366	    ("Target %d out of bounds in mpssas_check_eedp\n",
3367	     targetid));
3368	target = &sassc->targets[targetid];
3369	if (target->handle == 0x0)
3370		return;
3371
3372	/*
3373	 * Determine if the device is EEDP capable.
3374	 *
3375	 * If this flag is set in the inquiry data,
3376	 * the device supports protection information,
3377	 * and must support the 16 byte read
3378	 * capacity command, otherwise continue without
3379	 * sending read cap 16
3380	 */
3381	if ((cgd->inq_data.spc3_flags & SPC3_SID_PROTECT) == 0)
3382		return;
3383
3384	/*
3385	 * Issue a READ CAPACITY 16 command.  This info
3386	 * is used to determine if the LUN is formatted
3387	 * for EEDP support.
3388	 */
3389	ccb = xpt_alloc_ccb_nowait();
3390	if (ccb == NULL) {
3391		mps_dprint(sc, MPS_ERROR, "Unable to alloc CCB "
3392		    "for EEDP support.\n");
3393		return;
3394	}
3395
3396	if (xpt_create_path(&local_path, xpt_periph,
3397	    pathid, targetid, lunid) != CAM_REQ_CMP) {
3398		mps_dprint(sc, MPS_ERROR, "Unable to create "
3399		    "path for EEDP support\n");
3400		xpt_free_ccb(ccb);
3401		return;
3402	}
3403
3404	/*
3405	 * If LUN is already in list, don't create a new
3406	 * one.
3407	 */
3408	found_lun = FALSE;
3409	SLIST_FOREACH(lun, &target->luns, lun_link) {
3410		if (lun->lun_id == lunid) {
3411			found_lun = TRUE;
3412			break;
3413		}
3414	}
3415	if (!found_lun) {
3416		lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3417		    M_NOWAIT | M_ZERO);
3418		if (lun == NULL) {
3419			mps_dprint(sc, MPS_ERROR,
3420			    "Unable to alloc LUN for EEDP support.\n");
3421			xpt_free_path(local_path);
3422			xpt_free_ccb(ccb);
3423			return;
3424		}
3425		lun->lun_id = lunid;
3426		SLIST_INSERT_HEAD(&target->luns, lun,
3427		    lun_link);
3428	}
3429
3430	xpt_path_string(local_path, path_str, sizeof(path_str));
3431
3432	mps_dprint(sc, MPS_INFO, "Sending read cap: path %s handle %d\n",
3433	    path_str, target->handle);
3434
3435	/*
3436	 * Issue a READ CAPACITY 16 command for the LUN.
3437	 * The mpssas_read_cap_done function will load
3438	 * the read cap info into the LUN struct.
3439	 */
3440	rcap_buf = malloc(sizeof(struct scsi_read_capacity_eedp),
3441	    M_MPT2, M_NOWAIT | M_ZERO);
3442	if (rcap_buf == NULL) {
3443		mps_dprint(sc, MPS_FAULT,
3444		    "Unable to alloc read capacity buffer for EEDP support.\n");
3445		xpt_free_path(ccb->ccb_h.path);
3446		xpt_free_ccb(ccb);
3447		return;
3448	}
3449	xpt_setup_ccb(&ccb->ccb_h, local_path, CAM_PRIORITY_XPT);
3450	csio = &ccb->csio;
3451	csio->ccb_h.func_code = XPT_SCSI_IO;
3452	csio->ccb_h.flags = CAM_DIR_IN;
3453	csio->ccb_h.retry_count = 4;
3454	csio->ccb_h.cbfcnp = mpssas_read_cap_done;
3455	csio->ccb_h.timeout = 60000;
3456	csio->data_ptr = (uint8_t *)rcap_buf;
3457	csio->dxfer_len = sizeof(struct scsi_read_capacity_eedp);
3458	csio->sense_len = MPS_SENSE_LEN;
3459	csio->cdb_len = sizeof(*scsi_cmd);
3460	csio->tag_action = MSG_SIMPLE_Q_TAG;
3461
3462	scsi_cmd = (struct scsi_read_capacity_16 *)&csio->cdb_io.cdb_bytes;
3463	bzero(scsi_cmd, sizeof(*scsi_cmd));
3464	scsi_cmd->opcode = 0x9E;
3465	scsi_cmd->service_action = SRC16_SERVICE_ACTION;
3466	((uint8_t *)scsi_cmd)[13] = sizeof(struct scsi_read_capacity_eedp);
3467
3468	ccb->ccb_h.ppriv_ptr1 = sassc;
3469	xpt_action(ccb);
3470}
3471
3472static void
3473mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb)
3474{
3475	struct mpssas_softc *sassc;
3476	struct mpssas_target *target;
3477	struct mpssas_lun *lun;
3478	struct scsi_read_capacity_eedp *rcap_buf;
3479
3480	if (done_ccb == NULL)
3481		return;
3482
3483	/* Driver need to release devq, it Scsi command is
3484	 * generated by driver internally.
3485	 * Currently there is a single place where driver
3486	 * calls scsi command internally. In future if driver
3487	 * calls more scsi command internally, it needs to release
3488	 * devq internally, since those command will not go back to
3489	 * cam_periph.
3490	 */
3491	if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) ) {
3492        	done_ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
3493		xpt_release_devq(done_ccb->ccb_h.path,
3494			       	/*count*/ 1, /*run_queue*/TRUE);
3495	}
3496
3497	rcap_buf = (struct scsi_read_capacity_eedp *)done_ccb->csio.data_ptr;
3498
3499	/*
3500	 * Get the LUN ID for the path and look it up in the LUN list for the
3501	 * target.
3502	 */
3503	sassc = (struct mpssas_softc *)done_ccb->ccb_h.ppriv_ptr1;
3504	KASSERT(done_ccb->ccb_h.target_id < sassc->maxtargets,
3505	    ("Target %d out of bounds in mpssas_read_cap_done\n",
3506	     done_ccb->ccb_h.target_id));
3507	target = &sassc->targets[done_ccb->ccb_h.target_id];
3508	SLIST_FOREACH(lun, &target->luns, lun_link) {
3509		if (lun->lun_id != done_ccb->ccb_h.target_lun)
3510			continue;
3511
3512		/*
3513		 * Got the LUN in the target's LUN list.  Fill it in
3514		 * with EEDP info.  If the READ CAP 16 command had some
3515		 * SCSI error (common if command is not supported), mark
3516		 * the lun as not supporting EEDP and set the block size
3517		 * to 0.
3518		 */
3519		if ((mpssas_get_ccbstatus(done_ccb) != CAM_REQ_CMP)
3520		 || (done_ccb->csio.scsi_status != SCSI_STATUS_OK)) {
3521			lun->eedp_formatted = FALSE;
3522			lun->eedp_block_size = 0;
3523			break;
3524		}
3525
3526		if (rcap_buf->protect & 0x01) {
3527			mps_dprint(sassc->sc, MPS_INFO, "LUN %d for "
3528 			    "target ID %d is formatted for EEDP "
3529 			    "support.\n", done_ccb->ccb_h.target_lun,
3530 			    done_ccb->ccb_h.target_id);
3531			lun->eedp_formatted = TRUE;
3532			lun->eedp_block_size = scsi_4btoul(rcap_buf->length);
3533		}
3534		break;
3535	}
3536
3537	// Finished with this CCB and path.
3538	free(rcap_buf, M_MPT2);
3539	xpt_free_path(done_ccb->ccb_h.path);
3540	xpt_free_ccb(done_ccb);
3541}
3542#endif /* (__FreeBSD_version < 901503) || \
3543          ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006)) */
3544
3545void
3546mpssas_prepare_for_tm(struct mps_softc *sc, struct mps_command *tm,
3547    struct mpssas_target *target, lun_id_t lun_id)
3548{
3549	union ccb *ccb;
3550	path_id_t path_id;
3551
3552	/*
3553	 * Set the INRESET flag for this target so that no I/O will be sent to
3554	 * the target until the reset has completed.  If an I/O request does
3555	 * happen, the devq will be frozen.  The CCB holds the path which is
3556	 * used to release the devq.  The devq is released and the CCB is freed
3557	 * when the TM completes.
3558	 */
3559	ccb = xpt_alloc_ccb_nowait();
3560	if (ccb) {
3561		path_id = cam_sim_path(sc->sassc->sim);
3562		if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, path_id,
3563		    target->tid, lun_id) != CAM_REQ_CMP) {
3564			xpt_free_ccb(ccb);
3565		} else {
3566			tm->cm_ccb = ccb;
3567			tm->cm_targ = target;
3568			target->flags |= MPSSAS_TARGET_INRESET;
3569		}
3570	}
3571}
3572
3573int
3574mpssas_startup(struct mps_softc *sc)
3575{
3576
3577	/*
3578	 * Send the port enable message and set the wait_for_port_enable flag.
3579	 * This flag helps to keep the simq frozen until all discovery events
3580	 * are processed.
3581	 */
3582	sc->wait_for_port_enable = 1;
3583	mpssas_send_portenable(sc);
3584	return (0);
3585}
3586
3587static int
3588mpssas_send_portenable(struct mps_softc *sc)
3589{
3590	MPI2_PORT_ENABLE_REQUEST *request;
3591	struct mps_command *cm;
3592
3593	MPS_FUNCTRACE(sc);
3594
3595	if ((cm = mps_alloc_command(sc)) == NULL)
3596		return (EBUSY);
3597	request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req;
3598	request->Function = MPI2_FUNCTION_PORT_ENABLE;
3599	request->MsgFlags = 0;
3600	request->VP_ID = 0;
3601	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3602	cm->cm_complete = mpssas_portenable_complete;
3603	cm->cm_data = NULL;
3604	cm->cm_sge = NULL;
3605
3606	mps_map_command(sc, cm);
3607	mps_dprint(sc, MPS_XINFO,
3608	    "mps_send_portenable finished cm %p req %p complete %p\n",
3609	    cm, cm->cm_req, cm->cm_complete);
3610	return (0);
3611}
3612
3613static void
3614mpssas_portenable_complete(struct mps_softc *sc, struct mps_command *cm)
3615{
3616	MPI2_PORT_ENABLE_REPLY *reply;
3617	struct mpssas_softc *sassc;
3618
3619	MPS_FUNCTRACE(sc);
3620	sassc = sc->sassc;
3621
3622	/*
3623	 * Currently there should be no way we can hit this case.  It only
3624	 * happens when we have a failure to allocate chain frames, and
3625	 * port enable commands don't have S/G lists.
3626	 */
3627	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3628		mps_dprint(sc, MPS_ERROR, "%s: cm_flags = %#x for port enable! "
3629			   "This should not happen!\n", __func__, cm->cm_flags);
3630	}
3631
3632	reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply;
3633	if (reply == NULL)
3634		mps_dprint(sc, MPS_FAULT, "Portenable NULL reply\n");
3635	else if (le16toh(reply->IOCStatus & MPI2_IOCSTATUS_MASK) !=
3636	    MPI2_IOCSTATUS_SUCCESS)
3637		mps_dprint(sc, MPS_FAULT, "Portenable failed\n");
3638
3639	mps_free_command(sc, cm);
3640	if (sc->mps_ich.ich_arg != NULL) {
3641		mps_dprint(sc, MPS_XINFO, "disestablish config intrhook\n");
3642		config_intrhook_disestablish(&sc->mps_ich);
3643		sc->mps_ich.ich_arg = NULL;
3644	}
3645
3646	/*
3647	 * Get WarpDrive info after discovery is complete but before the scan
3648	 * starts.  At this point, all devices are ready to be exposed to the
3649	 * OS.  If devices should be hidden instead, take them out of the
3650	 * 'targets' array before the scan.  The devinfo for a disk will have
3651	 * some info and a volume's will be 0.  Use that to remove disks.
3652	 */
3653	mps_wd_config_pages(sc);
3654
3655	/*
3656	 * Done waiting for port enable to complete.  Decrement the refcount.
3657	 * If refcount is 0, discovery is complete and a rescan of the bus can
3658	 * take place.  Since the simq was explicitly frozen before port
3659	 * enable, it must be explicitly released here to keep the
3660	 * freeze/release count in sync.
3661	 */
3662	sc->wait_for_port_enable = 0;
3663	sc->port_enable_complete = 1;
3664	wakeup(&sc->port_enable_complete);
3665	mpssas_startup_decrement(sassc);
3666}
3667
3668int
3669mpssas_check_id(struct mpssas_softc *sassc, int id)
3670{
3671	struct mps_softc *sc = sassc->sc;
3672	char *ids;
3673	char *name;
3674
3675	ids = &sc->exclude_ids[0];
3676	while((name = strsep(&ids, ",")) != NULL) {
3677		if (name[0] == '\0')
3678			continue;
3679		if (strtol(name, NULL, 0) == (long)id)
3680			return (1);
3681	}
3682
3683	return (0);
3684}
3685
3686void
3687mpssas_realloc_targets(struct mps_softc *sc, int maxtargets)
3688{
3689	struct mpssas_softc *sassc;
3690	struct mpssas_lun *lun, *lun_tmp;
3691	struct mpssas_target *targ;
3692	int i;
3693
3694	sassc = sc->sassc;
3695	/*
3696	 * The number of targets is based on IOC Facts, so free all of
3697	 * the allocated LUNs for each target and then the target buffer
3698	 * itself.
3699	 */
3700	for (i=0; i< maxtargets; i++) {
3701		targ = &sassc->targets[i];
3702		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
3703			free(lun, M_MPT2);
3704		}
3705	}
3706	free(sassc->targets, M_MPT2);
3707
3708	sassc->targets = malloc(sizeof(struct mpssas_target) * maxtargets,
3709	    M_MPT2, M_WAITOK|M_ZERO);
3710	if (!sassc->targets) {
3711		panic("%s failed to alloc targets with error %d\n",
3712		    __func__, ENOMEM);
3713	}
3714}
3715