mps_sas.c revision 2c8883e85872d734be7ac52c0a3092a529db8307
1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2009 Yahoo! Inc.
5 * Copyright (c) 2011-2015 LSI Corp.
6 * Copyright (c) 2013-2015 Avago Technologies
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 * Avago Technologies (LSI) MPT-Fusion Host Adapter FreeBSD
31 *
32 * $FreeBSD$
33 */
34
35#include <sys/cdefs.h>
36__FBSDID("$FreeBSD$");
37
38/* Communications core for Avago Technologies (LSI) MPT2 */
39
40/* TODO Move headers to mpsvar */
41#include <sys/types.h>
42#include <sys/param.h>
43#include <sys/systm.h>
44#include <sys/kernel.h>
45#include <sys/selinfo.h>
46#include <sys/module.h>
47#include <sys/bus.h>
48#include <sys/conf.h>
49#include <sys/bio.h>
50#include <sys/malloc.h>
51#include <sys/uio.h>
52#include <sys/sysctl.h>
53#include <sys/endian.h>
54#include <sys/queue.h>
55#include <sys/kthread.h>
56#include <sys/taskqueue.h>
57#include <sys/sbuf.h>
58
59#include <machine/bus.h>
60#include <machine/resource.h>
61#include <sys/rman.h>
62
63#include <machine/stdarg.h>
64
65#include <cam/cam.h>
66#include <cam/cam_ccb.h>
67#include <cam/cam_xpt.h>
68#include <cam/cam_debug.h>
69#include <cam/cam_sim.h>
70#include <cam/cam_xpt_sim.h>
71#include <cam/cam_xpt_periph.h>
72#include <cam/cam_periph.h>
73#include <cam/scsi/scsi_all.h>
74#include <cam/scsi/scsi_message.h>
75#if __FreeBSD_version >= 900026
76#include <cam/scsi/smp_all.h>
77#endif
78
79#include <dev/mps/mpi/mpi2_type.h>
80#include <dev/mps/mpi/mpi2.h>
81#include <dev/mps/mpi/mpi2_ioc.h>
82#include <dev/mps/mpi/mpi2_sas.h>
83#include <dev/mps/mpi/mpi2_cnfg.h>
84#include <dev/mps/mpi/mpi2_init.h>
85#include <dev/mps/mpi/mpi2_tool.h>
86#include <dev/mps/mps_ioctl.h>
87#include <dev/mps/mpsvar.h>
88#include <dev/mps/mps_table.h>
89#include <dev/mps/mps_sas.h>
90
91#define MPSSAS_DISCOVERY_TIMEOUT	20
92#define MPSSAS_MAX_DISCOVERY_TIMEOUTS	10 /* 200 seconds */
93
94/*
95 * static array to check SCSI OpCode for EEDP protection bits
96 */
97#define	PRO_R MPI2_SCSIIO_EEDPFLAGS_CHECK_REMOVE_OP
98#define	PRO_W MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
99#define	PRO_V MPI2_SCSIIO_EEDPFLAGS_INSERT_OP
100static uint8_t op_code_prot[256] = {
101	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
102	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
103	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
104	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
105	0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
106	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
107	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
108	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
109	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
110	0, 0, 0, PRO_W, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
111	0, 0, 0, 0, 0, 0, 0, 0, PRO_R, 0, PRO_W, 0, 0, 0, PRO_W, PRO_V,
112	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
113	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
114	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
115	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
116	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
117};
118
119MALLOC_DEFINE(M_MPSSAS, "MPSSAS", "MPS SAS memory");
120
121static void mpssas_remove_device(struct mps_softc *, struct mps_command *);
122static void mpssas_remove_complete(struct mps_softc *, struct mps_command *);
123static void mpssas_action(struct cam_sim *sim, union ccb *ccb);
124static void mpssas_poll(struct cam_sim *sim);
125static int mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm,
126    struct mps_command *cm);
127static void mpssas_scsiio_timeout(void *data);
128static void mpssas_abort_complete(struct mps_softc *sc, struct mps_command *cm);
129static void mpssas_direct_drive_io(struct mpssas_softc *sassc,
130    struct mps_command *cm, union ccb *ccb);
131static void mpssas_action_scsiio(struct mpssas_softc *, union ccb *);
132static void mpssas_scsiio_complete(struct mps_softc *, struct mps_command *);
133static void mpssas_action_resetdev(struct mpssas_softc *, union ccb *);
134#if __FreeBSD_version >= 900026
135static void mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm);
136static void mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb,
137			       uint64_t sasaddr);
138static void mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb);
139#endif //FreeBSD_version >= 900026
140static void mpssas_resetdev_complete(struct mps_softc *, struct mps_command *);
141static void mpssas_async(void *callback_arg, uint32_t code,
142			 struct cam_path *path, void *arg);
143#if (__FreeBSD_version < 901503) || \
144    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
145static void mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
146			      struct ccb_getdev *cgd);
147static void mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb);
148#endif
149static int mpssas_send_portenable(struct mps_softc *sc);
150static void mpssas_portenable_complete(struct mps_softc *sc,
151    struct mps_command *cm);
152
153struct mpssas_target *
154mpssas_find_target_by_handle(struct mpssas_softc *sassc, int start, uint16_t handle)
155{
156	struct mpssas_target *target;
157	int i;
158
159	for (i = start; i < sassc->maxtargets; i++) {
160		target = &sassc->targets[i];
161		if (target->handle == handle)
162			return (target);
163	}
164
165	return (NULL);
166}
167
168/* we need to freeze the simq during attach and diag reset, to avoid failing
169 * commands before device handles have been found by discovery.  Since
170 * discovery involves reading config pages and possibly sending commands,
171 * discovery actions may continue even after we receive the end of discovery
172 * event, so refcount discovery actions instead of assuming we can unfreeze
173 * the simq when we get the event.
174 */
175void
176mpssas_startup_increment(struct mpssas_softc *sassc)
177{
178	MPS_FUNCTRACE(sassc->sc);
179
180	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
181		if (sassc->startup_refcount++ == 0) {
182			/* just starting, freeze the simq */
183			mps_dprint(sassc->sc, MPS_INIT,
184			    "%s freezing simq\n", __func__);
185#if __FreeBSD_version >= 1000039
186			xpt_hold_boot();
187#endif
188			xpt_freeze_simq(sassc->sim, 1);
189		}
190		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
191		    sassc->startup_refcount);
192	}
193}
194
195void
196mpssas_release_simq_reinit(struct mpssas_softc *sassc)
197{
198	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
199		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
200		xpt_release_simq(sassc->sim, 1);
201		mps_dprint(sassc->sc, MPS_INFO, "Unfreezing SIM queue\n");
202	}
203}
204
205void
206mpssas_startup_decrement(struct mpssas_softc *sassc)
207{
208	MPS_FUNCTRACE(sassc->sc);
209
210	if ((sassc->flags & MPSSAS_IN_STARTUP) != 0) {
211		if (--sassc->startup_refcount == 0) {
212			/* finished all discovery-related actions, release
213			 * the simq and rescan for the latest topology.
214			 */
215			mps_dprint(sassc->sc, MPS_INIT,
216			    "%s releasing simq\n", __func__);
217			sassc->flags &= ~MPSSAS_IN_STARTUP;
218			xpt_release_simq(sassc->sim, 1);
219#if __FreeBSD_version >= 1000039
220			xpt_release_boot();
221#else
222			mpssas_rescan_target(sassc->sc, NULL);
223#endif
224		}
225		mps_dprint(sassc->sc, MPS_INIT, "%s refcount %u\n", __func__,
226		    sassc->startup_refcount);
227	}
228}
229
230/*
231 * The firmware requires us to stop sending commands when we're doing task
232 * management.
233 * XXX The logic for serializing the device has been made lazy and moved to
234 * mpssas_prepare_for_tm().
235 */
236struct mps_command *
237mpssas_alloc_tm(struct mps_softc *sc)
238{
239	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
240	struct mps_command *tm;
241
242	tm = mps_alloc_high_priority_command(sc);
243	if (tm == NULL)
244		return (NULL);
245
246	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
247	req->Function = MPI2_FUNCTION_SCSI_TASK_MGMT;
248	return tm;
249}
250
251void
252mpssas_free_tm(struct mps_softc *sc, struct mps_command *tm)
253{
254	int target_id = 0xFFFFFFFF;
255
256	if (tm == NULL)
257		return;
258
259	/*
260	 * For TM's the devq is frozen for the device.  Unfreeze it here and
261	 * free the resources used for freezing the devq.  Must clear the
262	 * INRESET flag as well or scsi I/O will not work.
263	 */
264	if (tm->cm_targ != NULL) {
265		tm->cm_targ->flags &= ~MPSSAS_TARGET_INRESET;
266		target_id = tm->cm_targ->tid;
267	}
268	if (tm->cm_ccb) {
269		mps_dprint(sc, MPS_INFO, "Unfreezing devq for target ID %d\n",
270		    target_id);
271		xpt_release_devq(tm->cm_ccb->ccb_h.path, 1, TRUE);
272		xpt_free_path(tm->cm_ccb->ccb_h.path);
273		xpt_free_ccb(tm->cm_ccb);
274	}
275
276	mps_free_high_priority_command(sc, tm);
277}
278
279void
280mpssas_rescan_target(struct mps_softc *sc, struct mpssas_target *targ)
281{
282	struct mpssas_softc *sassc = sc->sassc;
283	path_id_t pathid;
284	target_id_t targetid;
285	union ccb *ccb;
286
287	MPS_FUNCTRACE(sc);
288	pathid = cam_sim_path(sassc->sim);
289	if (targ == NULL)
290		targetid = CAM_TARGET_WILDCARD;
291	else
292		targetid = targ - sassc->targets;
293
294	/*
295	 * Allocate a CCB and schedule a rescan.
296	 */
297	ccb = xpt_alloc_ccb_nowait();
298	if (ccb == NULL) {
299		mps_dprint(sc, MPS_ERROR, "unable to alloc CCB for rescan\n");
300		return;
301	}
302
303	if (xpt_create_path(&ccb->ccb_h.path, NULL, pathid,
304	    targetid, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
305		mps_dprint(sc, MPS_ERROR, "unable to create path for rescan\n");
306		xpt_free_ccb(ccb);
307		return;
308	}
309
310	if (targetid == CAM_TARGET_WILDCARD)
311		ccb->ccb_h.func_code = XPT_SCAN_BUS;
312	else
313		ccb->ccb_h.func_code = XPT_SCAN_TGT;
314
315	mps_dprint(sc, MPS_TRACE, "%s targetid %u\n", __func__, targetid);
316	xpt_rescan(ccb);
317}
318
319static void
320mpssas_log_command(struct mps_command *cm, u_int level, const char *fmt, ...)
321{
322	struct sbuf sb;
323	va_list ap;
324	char str[192];
325	char path_str[64];
326
327	if (cm == NULL)
328		return;
329
330	/* No need to be in here if debugging isn't enabled */
331	if ((cm->cm_sc->mps_debug & level) == 0)
332		return;
333
334	sbuf_new(&sb, str, sizeof(str), 0);
335
336	va_start(ap, fmt);
337
338	if (cm->cm_ccb != NULL) {
339		xpt_path_string(cm->cm_ccb->csio.ccb_h.path, path_str,
340				sizeof(path_str));
341		sbuf_cat(&sb, path_str);
342		if (cm->cm_ccb->ccb_h.func_code == XPT_SCSI_IO) {
343			scsi_command_string(&cm->cm_ccb->csio, &sb);
344			sbuf_printf(&sb, "length %d ",
345				    cm->cm_ccb->csio.dxfer_len);
346		}
347	}
348	else {
349		sbuf_printf(&sb, "(noperiph:%s%d:%u:%u:%u): ",
350		    cam_sim_name(cm->cm_sc->sassc->sim),
351		    cam_sim_unit(cm->cm_sc->sassc->sim),
352		    cam_sim_bus(cm->cm_sc->sassc->sim),
353		    cm->cm_targ ? cm->cm_targ->tid : 0xFFFFFFFF,
354		    cm->cm_lun);
355	}
356
357	sbuf_printf(&sb, "SMID %u ", cm->cm_desc.Default.SMID);
358	sbuf_vprintf(&sb, fmt, ap);
359	sbuf_finish(&sb);
360	mps_print_field(cm->cm_sc, "%s", sbuf_data(&sb));
361
362	va_end(ap);
363}
364
365
366static void
367mpssas_remove_volume(struct mps_softc *sc, struct mps_command *tm)
368{
369	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
370	struct mpssas_target *targ;
371	uint16_t handle;
372
373	MPS_FUNCTRACE(sc);
374
375	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
376	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
377	targ = tm->cm_targ;
378
379	if (reply == NULL) {
380		/* XXX retry the remove after the diag reset completes? */
381		mps_dprint(sc, MPS_FAULT,
382		    "%s NULL reply resetting device 0x%04x\n", __func__,
383		    handle);
384		mpssas_free_tm(sc, tm);
385		return;
386	}
387
388	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
389	    MPI2_IOCSTATUS_SUCCESS) {
390		mps_dprint(sc, MPS_ERROR,
391		   "IOCStatus = 0x%x while resetting device 0x%x\n",
392		   le16toh(reply->IOCStatus), handle);
393	}
394
395	mps_dprint(sc, MPS_XINFO,
396	    "Reset aborted %u commands\n", reply->TerminationCount);
397	mps_free_reply(sc, tm->cm_reply_data);
398	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
399
400	mps_dprint(sc, MPS_XINFO,
401	    "clearing target %u handle 0x%04x\n", targ->tid, handle);
402
403	/*
404	 * Don't clear target if remove fails because things will get confusing.
405	 * Leave the devname and sasaddr intact so that we know to avoid reusing
406	 * this target id if possible, and so we can assign the same target id
407	 * to this device if it comes back in the future.
408	 */
409	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
410	    MPI2_IOCSTATUS_SUCCESS) {
411		targ = tm->cm_targ;
412		targ->handle = 0x0;
413		targ->encl_handle = 0x0;
414		targ->encl_slot = 0x0;
415		targ->exp_dev_handle = 0x0;
416		targ->phy_num = 0x0;
417		targ->linkrate = 0x0;
418		targ->devinfo = 0x0;
419		targ->flags = 0x0;
420	}
421
422	mpssas_free_tm(sc, tm);
423}
424
425
426/*
427 * No Need to call "MPI2_SAS_OP_REMOVE_DEVICE" For Volume removal.
428 * Otherwise Volume Delete is same as Bare Drive Removal.
429 */
430void
431mpssas_prepare_volume_remove(struct mpssas_softc *sassc, uint16_t handle)
432{
433	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
434	struct mps_softc *sc;
435	struct mps_command *tm;
436	struct mpssas_target *targ = NULL;
437
438	MPS_FUNCTRACE(sassc->sc);
439	sc = sassc->sc;
440
441#ifdef WD_SUPPORT
442	/*
443	 * If this is a WD controller, determine if the disk should be exposed
444	 * to the OS or not.  If disk should be exposed, return from this
445	 * function without doing anything.
446	 */
447	if (sc->WD_available && (sc->WD_hide_expose ==
448	    MPS_WD_EXPOSE_ALWAYS)) {
449		return;
450	}
451#endif //WD_SUPPORT
452
453	targ = mpssas_find_target_by_handle(sassc, 0, handle);
454	if (targ == NULL) {
455		/* FIXME: what is the action? */
456		/* We don't know about this device? */
457		mps_dprint(sc, MPS_ERROR,
458		   "%s %d : invalid handle 0x%x \n", __func__,__LINE__, handle);
459		return;
460	}
461
462	targ->flags |= MPSSAS_TARGET_INREMOVAL;
463
464	tm = mpssas_alloc_tm(sc);
465	if (tm == NULL) {
466		mps_dprint(sc, MPS_ERROR,
467		    "%s: command alloc failure\n", __func__);
468		return;
469	}
470
471	mpssas_rescan_target(sc, targ);
472
473	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
474	req->DevHandle = targ->handle;
475	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
476
477	/* SAS Hard Link Reset / SATA Link Reset */
478	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
479
480	tm->cm_targ = targ;
481	tm->cm_data = NULL;
482	tm->cm_complete = mpssas_remove_volume;
483	tm->cm_complete_data = (void *)(uintptr_t)handle;
484
485	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
486	    __func__, targ->tid);
487	mpssas_prepare_for_tm(sc, tm, targ, CAM_LUN_WILDCARD);
488
489	mps_map_command(sc, tm);
490}
491
492/*
493 * The MPT2 firmware performs debounce on the link to avoid transient link
494 * errors and false removals.  When it does decide that link has been lost
495 * and a device need to go away, it expects that the host will perform a
496 * target reset and then an op remove.  The reset has the side-effect of
497 * aborting any outstanding requests for the device, which is required for
498 * the op-remove to succeed.  It's not clear if the host should check for
499 * the device coming back alive after the reset.
500 */
501void
502mpssas_prepare_remove(struct mpssas_softc *sassc, uint16_t handle)
503{
504	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
505	struct mps_softc *sc;
506	struct mps_command *cm;
507	struct mpssas_target *targ = NULL;
508
509	MPS_FUNCTRACE(sassc->sc);
510
511	sc = sassc->sc;
512
513	targ = mpssas_find_target_by_handle(sassc, 0, handle);
514	if (targ == NULL) {
515		/* FIXME: what is the action? */
516		/* We don't know about this device? */
517		mps_dprint(sc, MPS_ERROR,
518		    "%s : invalid handle 0x%x \n", __func__, handle);
519		return;
520	}
521
522	targ->flags |= MPSSAS_TARGET_INREMOVAL;
523
524	cm = mpssas_alloc_tm(sc);
525	if (cm == NULL) {
526		mps_dprint(sc, MPS_ERROR,
527		    "%s: command alloc failure\n", __func__);
528		return;
529	}
530
531	mpssas_rescan_target(sc, targ);
532
533	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)cm->cm_req;
534	memset(req, 0, sizeof(*req));
535	req->DevHandle = htole16(targ->handle);
536	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
537
538	/* SAS Hard Link Reset / SATA Link Reset */
539	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
540
541	cm->cm_targ = targ;
542	cm->cm_data = NULL;
543	cm->cm_complete = mpssas_remove_device;
544	cm->cm_complete_data = (void *)(uintptr_t)handle;
545
546	mps_dprint(sc, MPS_INFO, "%s: Sending reset for target ID %d\n",
547	    __func__, targ->tid);
548	mpssas_prepare_for_tm(sc, cm, targ, CAM_LUN_WILDCARD);
549
550	mps_map_command(sc, cm);
551}
552
553static void
554mpssas_remove_device(struct mps_softc *sc, struct mps_command *tm)
555{
556	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
557	MPI2_SAS_IOUNIT_CONTROL_REQUEST *req;
558	struct mpssas_target *targ;
559	struct mps_command *next_cm;
560	uint16_t handle;
561
562	MPS_FUNCTRACE(sc);
563
564	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
565	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
566	targ = tm->cm_targ;
567
568	/*
569	 * Currently there should be no way we can hit this case.  It only
570	 * happens when we have a failure to allocate chain frames, and
571	 * task management commands don't have S/G lists.
572	 */
573	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
574		mps_dprint(sc, MPS_ERROR,
575		    "%s: cm_flags = %#x for remove of handle %#04x! "
576		    "This should not happen!\n", __func__, tm->cm_flags,
577		    handle);
578	}
579
580	if (reply == NULL) {
581		/* XXX retry the remove after the diag reset completes? */
582		mps_dprint(sc, MPS_FAULT,
583		    "%s NULL reply resetting device 0x%04x\n", __func__,
584		    handle);
585		mpssas_free_tm(sc, tm);
586		return;
587	}
588
589	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
590	    MPI2_IOCSTATUS_SUCCESS) {
591		mps_dprint(sc, MPS_ERROR,
592		   "IOCStatus = 0x%x while resetting device 0x%x\n",
593		   le16toh(reply->IOCStatus), handle);
594	}
595
596	mps_dprint(sc, MPS_XINFO, "Reset aborted %u commands\n",
597	    le32toh(reply->TerminationCount));
598	mps_free_reply(sc, tm->cm_reply_data);
599	tm->cm_reply = NULL;	/* Ensures the reply won't get re-freed */
600
601	/* Reuse the existing command */
602	req = (MPI2_SAS_IOUNIT_CONTROL_REQUEST *)tm->cm_req;
603	memset(req, 0, sizeof(*req));
604	req->Function = MPI2_FUNCTION_SAS_IO_UNIT_CONTROL;
605	req->Operation = MPI2_SAS_OP_REMOVE_DEVICE;
606	req->DevHandle = htole16(handle);
607	tm->cm_data = NULL;
608	tm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
609	tm->cm_complete = mpssas_remove_complete;
610	tm->cm_complete_data = (void *)(uintptr_t)handle;
611
612	mps_map_command(sc, tm);
613
614	mps_dprint(sc, MPS_XINFO, "clearing target %u handle 0x%04x\n",
615		   targ->tid, handle);
616	TAILQ_FOREACH_SAFE(tm, &targ->commands, cm_link, next_cm) {
617		union ccb *ccb;
618
619		mps_dprint(sc, MPS_XINFO, "Completing missed command %p\n", tm);
620		ccb = tm->cm_complete_data;
621		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
622		tm->cm_state = MPS_CM_STATE_BUSY;
623		mpssas_scsiio_complete(sc, tm);
624	}
625}
626
627static void
628mpssas_remove_complete(struct mps_softc *sc, struct mps_command *tm)
629{
630	MPI2_SAS_IOUNIT_CONTROL_REPLY *reply;
631	uint16_t handle;
632	struct mpssas_target *targ;
633	struct mpssas_lun *lun;
634
635	MPS_FUNCTRACE(sc);
636
637	reply = (MPI2_SAS_IOUNIT_CONTROL_REPLY *)tm->cm_reply;
638	handle = (uint16_t)(uintptr_t)tm->cm_complete_data;
639
640	/*
641	 * Currently there should be no way we can hit this case.  It only
642	 * happens when we have a failure to allocate chain frames, and
643	 * task management commands don't have S/G lists.
644	 */
645	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
646		mps_dprint(sc, MPS_XINFO,
647			   "%s: cm_flags = %#x for remove of handle %#04x! "
648			   "This should not happen!\n", __func__, tm->cm_flags,
649			   handle);
650		mpssas_free_tm(sc, tm);
651		return;
652	}
653
654	if (reply == NULL) {
655		/* most likely a chip reset */
656		mps_dprint(sc, MPS_FAULT,
657		    "%s NULL reply removing device 0x%04x\n", __func__, handle);
658		mpssas_free_tm(sc, tm);
659		return;
660	}
661
662	mps_dprint(sc, MPS_XINFO,
663	    "%s on handle 0x%04x, IOCStatus= 0x%x\n", __func__,
664	    handle, le16toh(reply->IOCStatus));
665
666	/*
667	 * Don't clear target if remove fails because things will get confusing.
668	 * Leave the devname and sasaddr intact so that we know to avoid reusing
669	 * this target id if possible, and so we can assign the same target id
670	 * to this device if it comes back in the future.
671	 */
672	if ((le16toh(reply->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
673	    MPI2_IOCSTATUS_SUCCESS) {
674		targ = tm->cm_targ;
675		targ->handle = 0x0;
676		targ->encl_handle = 0x0;
677		targ->encl_slot = 0x0;
678		targ->exp_dev_handle = 0x0;
679		targ->phy_num = 0x0;
680		targ->linkrate = 0x0;
681		targ->devinfo = 0x0;
682		targ->flags = 0x0;
683
684		while(!SLIST_EMPTY(&targ->luns)) {
685			lun = SLIST_FIRST(&targ->luns);
686			SLIST_REMOVE_HEAD(&targ->luns, lun_link);
687			free(lun, M_MPT2);
688		}
689	}
690
691
692	mpssas_free_tm(sc, tm);
693}
694
695static int
696mpssas_register_events(struct mps_softc *sc)
697{
698	u32 events[MPI2_EVENT_NOTIFY_EVENTMASK_WORDS];
699
700	bzero(events, 16);
701	setbit(events, MPI2_EVENT_SAS_DEVICE_STATUS_CHANGE);
702	setbit(events, MPI2_EVENT_SAS_DISCOVERY);
703	setbit(events, MPI2_EVENT_SAS_BROADCAST_PRIMITIVE);
704	setbit(events, MPI2_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE);
705	setbit(events, MPI2_EVENT_SAS_INIT_TABLE_OVERFLOW);
706	setbit(events, MPI2_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
707	setbit(events, MPI2_EVENT_SAS_ENCL_DEVICE_STATUS_CHANGE);
708	setbit(events, MPI2_EVENT_IR_CONFIGURATION_CHANGE_LIST);
709	setbit(events, MPI2_EVENT_IR_VOLUME);
710	setbit(events, MPI2_EVENT_IR_PHYSICAL_DISK);
711	setbit(events, MPI2_EVENT_IR_OPERATION_STATUS);
712	setbit(events, MPI2_EVENT_LOG_ENTRY_ADDED);
713
714	mps_register_events(sc, events, mpssas_evt_handler, NULL,
715	    &sc->sassc->mpssas_eh);
716
717	return (0);
718}
719
720int
721mps_attach_sas(struct mps_softc *sc)
722{
723	struct mpssas_softc *sassc;
724	cam_status status;
725	int unit, error = 0, reqs;
726
727	MPS_FUNCTRACE(sc);
728	mps_dprint(sc, MPS_INIT, "%s entered\n", __func__);
729
730	sassc = malloc(sizeof(struct mpssas_softc), M_MPT2, M_WAITOK|M_ZERO);
731	if(!sassc) {
732		mps_dprint(sc, MPS_INIT|MPS_ERROR,
733		    "Cannot allocate SAS controller memory\n");
734		return (ENOMEM);
735	}
736
737	/*
738	 * XXX MaxTargets could change during a reinit.  Since we don't
739	 * resize the targets[] array during such an event, cache the value
740	 * of MaxTargets here so that we don't get into trouble later.  This
741	 * should move into the reinit logic.
742	 */
743	sassc->maxtargets = sc->facts->MaxTargets + sc->facts->MaxVolumes;
744	sassc->targets = malloc(sizeof(struct mpssas_target) *
745	    sassc->maxtargets, M_MPT2, M_WAITOK|M_ZERO);
746	if(!sassc->targets) {
747		mps_dprint(sc, MPS_INIT|MPS_ERROR,
748		    "Cannot allocate SAS target memory\n");
749		free(sassc, M_MPT2);
750		return (ENOMEM);
751	}
752	sc->sassc = sassc;
753	sassc->sc = sc;
754
755	reqs = sc->num_reqs - sc->num_prireqs - 1;
756	if ((sassc->devq = cam_simq_alloc(reqs)) == NULL) {
757		mps_dprint(sc, MPS_ERROR, "Cannot allocate SIMQ\n");
758		error = ENOMEM;
759		goto out;
760	}
761
762	unit = device_get_unit(sc->mps_dev);
763	sassc->sim = cam_sim_alloc(mpssas_action, mpssas_poll, "mps", sassc,
764	    unit, &sc->mps_mtx, reqs, reqs, sassc->devq);
765	if (sassc->sim == NULL) {
766		mps_dprint(sc, MPS_INIT|MPS_ERROR, "Cannot allocate SIM\n");
767		error = EINVAL;
768		goto out;
769	}
770
771	TAILQ_INIT(&sassc->ev_queue);
772
773	/* Initialize taskqueue for Event Handling */
774	TASK_INIT(&sassc->ev_task, 0, mpssas_firmware_event_work, sc);
775	sassc->ev_tq = taskqueue_create("mps_taskq", M_NOWAIT | M_ZERO,
776	    taskqueue_thread_enqueue, &sassc->ev_tq);
777	taskqueue_start_threads(&sassc->ev_tq, 1, PRIBIO, "%s taskq",
778	    device_get_nameunit(sc->mps_dev));
779
780	mps_lock(sc);
781
782	/*
783	 * XXX There should be a bus for every port on the adapter, but since
784	 * we're just going to fake the topology for now, we'll pretend that
785	 * everything is just a target on a single bus.
786	 */
787	if ((error = xpt_bus_register(sassc->sim, sc->mps_dev, 0)) != 0) {
788		mps_dprint(sc, MPS_INIT|MPS_ERROR,
789		    "Error %d registering SCSI bus\n", error);
790		mps_unlock(sc);
791		goto out;
792	}
793
794	/*
795	 * Assume that discovery events will start right away.
796	 *
797	 * Hold off boot until discovery is complete.
798	 */
799	sassc->flags |= MPSSAS_IN_STARTUP | MPSSAS_IN_DISCOVERY;
800	sc->sassc->startup_refcount = 0;
801	mpssas_startup_increment(sassc);
802
803	callout_init(&sassc->discovery_callout, 1 /*mpsafe*/);
804
805	/*
806	 * Register for async events so we can determine the EEDP
807	 * capabilities of devices.
808	 */
809	status = xpt_create_path(&sassc->path, /*periph*/NULL,
810	    cam_sim_path(sc->sassc->sim), CAM_TARGET_WILDCARD,
811	    CAM_LUN_WILDCARD);
812	if (status != CAM_REQ_CMP) {
813		mps_dprint(sc, MPS_ERROR|MPS_INIT,
814		    "Error %#x creating sim path\n", status);
815		sassc->path = NULL;
816	} else {
817		int event;
818
819#if (__FreeBSD_version >= 1000006) || \
820    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
821		event = AC_ADVINFO_CHANGED;
822#else
823		event = AC_FOUND_DEVICE;
824#endif
825		status = xpt_register_async(event, mpssas_async, sc,
826					    sassc->path);
827		if (status != CAM_REQ_CMP) {
828			mps_dprint(sc, MPS_ERROR,
829			    "Error %#x registering async handler for "
830			    "AC_ADVINFO_CHANGED events\n", status);
831			xpt_free_path(sassc->path);
832			sassc->path = NULL;
833		}
834	}
835	if (status != CAM_REQ_CMP) {
836		/*
837		 * EEDP use is the exception, not the rule.
838		 * Warn the user, but do not fail to attach.
839		 */
840		mps_printf(sc, "EEDP capabilities disabled.\n");
841	}
842
843	mps_unlock(sc);
844
845	mpssas_register_events(sc);
846out:
847	if (error)
848		mps_detach_sas(sc);
849
850	mps_dprint(sc, MPS_INIT, "%s exit error= %d\n", __func__, error);
851	return (error);
852}
853
854int
855mps_detach_sas(struct mps_softc *sc)
856{
857	struct mpssas_softc *sassc;
858	struct mpssas_lun *lun, *lun_tmp;
859	struct mpssas_target *targ;
860	int i;
861
862	MPS_FUNCTRACE(sc);
863
864	if (sc->sassc == NULL)
865		return (0);
866
867	sassc = sc->sassc;
868	mps_deregister_events(sc, sassc->mpssas_eh);
869
870	/*
871	 * Drain and free the event handling taskqueue with the lock
872	 * unheld so that any parallel processing tasks drain properly
873	 * without deadlocking.
874	 */
875	if (sassc->ev_tq != NULL)
876		taskqueue_free(sassc->ev_tq);
877
878	/* Make sure CAM doesn't wedge if we had to bail out early. */
879	mps_lock(sc);
880
881	while (sassc->startup_refcount != 0)
882		mpssas_startup_decrement(sassc);
883
884	/* Deregister our async handler */
885	if (sassc->path != NULL) {
886		xpt_register_async(0, mpssas_async, sc, sassc->path);
887		xpt_free_path(sassc->path);
888		sassc->path = NULL;
889	}
890
891	if (sassc->flags & MPSSAS_IN_STARTUP)
892		xpt_release_simq(sassc->sim, 1);
893
894	if (sassc->sim != NULL) {
895		xpt_bus_deregister(cam_sim_path(sassc->sim));
896		cam_sim_free(sassc->sim, FALSE);
897	}
898
899	mps_unlock(sc);
900
901	if (sassc->devq != NULL)
902		cam_simq_free(sassc->devq);
903
904	for(i=0; i< sassc->maxtargets ;i++) {
905		targ = &sassc->targets[i];
906		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
907			free(lun, M_MPT2);
908		}
909	}
910	free(sassc->targets, M_MPT2);
911	free(sassc, M_MPT2);
912	sc->sassc = NULL;
913
914	return (0);
915}
916
917void
918mpssas_discovery_end(struct mpssas_softc *sassc)
919{
920	struct mps_softc *sc = sassc->sc;
921
922	MPS_FUNCTRACE(sc);
923
924	if (sassc->flags & MPSSAS_DISCOVERY_TIMEOUT_PENDING)
925		callout_stop(&sassc->discovery_callout);
926
927	/*
928	 * After discovery has completed, check the mapping table for any
929	 * missing devices and update their missing counts. Only do this once
930	 * whenever the driver is initialized so that missing counts aren't
931	 * updated unnecessarily. Note that just because discovery has
932	 * completed doesn't mean that events have been processed yet. The
933	 * check_devices function is a callout timer that checks if ALL devices
934	 * are missing. If so, it will wait a little longer for events to
935	 * complete and keep resetting itself until some device in the mapping
936	 * table is not missing, meaning that event processing has started.
937	 */
938	if (sc->track_mapping_events) {
939		mps_dprint(sc, MPS_XINFO | MPS_MAPPING, "Discovery has "
940		    "completed. Check for missing devices in the mapping "
941		    "table.\n");
942		callout_reset(&sc->device_check_callout,
943		    MPS_MISSING_CHECK_DELAY * hz, mps_mapping_check_devices,
944		    sc);
945	}
946}
947
948static void
949mpssas_action(struct cam_sim *sim, union ccb *ccb)
950{
951	struct mpssas_softc *sassc;
952
953	sassc = cam_sim_softc(sim);
954
955	MPS_FUNCTRACE(sassc->sc);
956	mps_dprint(sassc->sc, MPS_TRACE, "ccb func_code 0x%x\n",
957	    ccb->ccb_h.func_code);
958	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
959
960	switch (ccb->ccb_h.func_code) {
961	case XPT_PATH_INQ:
962	{
963		struct ccb_pathinq *cpi = &ccb->cpi;
964		struct mps_softc *sc = sassc->sc;
965
966		cpi->version_num = 1;
967		cpi->hba_inquiry = PI_SDTR_ABLE|PI_TAG_ABLE|PI_WIDE_16;
968		cpi->target_sprt = 0;
969#if __FreeBSD_version >= 1000039
970		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED | PIM_NOSCAN;
971#else
972		cpi->hba_misc = PIM_NOBUSRESET | PIM_UNMAPPED;
973#endif
974		cpi->hba_eng_cnt = 0;
975		cpi->max_target = sassc->maxtargets - 1;
976		cpi->max_lun = 255;
977
978		/*
979		 * initiator_id is set here to an ID outside the set of valid
980		 * target IDs (including volumes).
981		 */
982		cpi->initiator_id = sassc->maxtargets;
983		strlcpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
984		strlcpy(cpi->hba_vid, "Avago Tech", HBA_IDLEN);
985		strlcpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
986		cpi->unit_number = cam_sim_unit(sim);
987		cpi->bus_id = cam_sim_bus(sim);
988		cpi->base_transfer_speed = 150000;
989		cpi->transport = XPORT_SAS;
990		cpi->transport_version = 0;
991		cpi->protocol = PROTO_SCSI;
992		cpi->protocol_version = SCSI_REV_SPC;
993		cpi->maxio = sc->maxio;
994		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
995		break;
996	}
997	case XPT_GET_TRAN_SETTINGS:
998	{
999		struct ccb_trans_settings	*cts;
1000		struct ccb_trans_settings_sas	*sas;
1001		struct ccb_trans_settings_scsi	*scsi;
1002		struct mpssas_target *targ;
1003
1004		cts = &ccb->cts;
1005		sas = &cts->xport_specific.sas;
1006		scsi = &cts->proto_specific.scsi;
1007
1008		KASSERT(cts->ccb_h.target_id < sassc->maxtargets,
1009		    ("Target %d out of bounds in XPT_GET_TRANS_SETTINGS\n",
1010		    cts->ccb_h.target_id));
1011		targ = &sassc->targets[cts->ccb_h.target_id];
1012		if (targ->handle == 0x0) {
1013			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1014			break;
1015		}
1016
1017		cts->protocol_version = SCSI_REV_SPC2;
1018		cts->transport = XPORT_SAS;
1019		cts->transport_version = 0;
1020
1021		sas->valid = CTS_SAS_VALID_SPEED;
1022		switch (targ->linkrate) {
1023		case 0x08:
1024			sas->bitrate = 150000;
1025			break;
1026		case 0x09:
1027			sas->bitrate = 300000;
1028			break;
1029		case 0x0a:
1030			sas->bitrate = 600000;
1031			break;
1032		default:
1033			sas->valid = 0;
1034		}
1035
1036		cts->protocol = PROTO_SCSI;
1037		scsi->valid = CTS_SCSI_VALID_TQ;
1038		scsi->flags = CTS_SCSI_FLAGS_TAG_ENB;
1039
1040		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1041		break;
1042	}
1043	case XPT_CALC_GEOMETRY:
1044		cam_calc_geometry(&ccb->ccg, /*extended*/1);
1045		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1046		break;
1047	case XPT_RESET_DEV:
1048		mps_dprint(sassc->sc, MPS_XINFO, "mpssas_action XPT_RESET_DEV\n");
1049		mpssas_action_resetdev(sassc, ccb);
1050		return;
1051	case XPT_RESET_BUS:
1052	case XPT_ABORT:
1053	case XPT_TERM_IO:
1054		mps_dprint(sassc->sc, MPS_XINFO,
1055		    "mpssas_action faking success for abort or reset\n");
1056		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1057		break;
1058	case XPT_SCSI_IO:
1059		mpssas_action_scsiio(sassc, ccb);
1060		return;
1061#if __FreeBSD_version >= 900026
1062	case XPT_SMP_IO:
1063		mpssas_action_smpio(sassc, ccb);
1064		return;
1065#endif
1066	default:
1067		mpssas_set_ccbstatus(ccb, CAM_FUNC_NOTAVAIL);
1068		break;
1069	}
1070	xpt_done(ccb);
1071
1072}
1073
1074static void
1075mpssas_announce_reset(struct mps_softc *sc, uint32_t ac_code,
1076    target_id_t target_id, lun_id_t lun_id)
1077{
1078	path_id_t path_id = cam_sim_path(sc->sassc->sim);
1079	struct cam_path *path;
1080
1081	mps_dprint(sc, MPS_XINFO, "%s code %x target %d lun %jx\n", __func__,
1082	    ac_code, target_id, (uintmax_t)lun_id);
1083
1084	if (xpt_create_path(&path, NULL,
1085		path_id, target_id, lun_id) != CAM_REQ_CMP) {
1086		mps_dprint(sc, MPS_ERROR, "unable to create path for reset "
1087			   "notification\n");
1088		return;
1089	}
1090
1091	xpt_async(ac_code, path, NULL);
1092	xpt_free_path(path);
1093}
1094
1095static void
1096mpssas_complete_all_commands(struct mps_softc *sc)
1097{
1098	struct mps_command *cm;
1099	int i;
1100	int completed;
1101
1102	MPS_FUNCTRACE(sc);
1103	mtx_assert(&sc->mps_mtx, MA_OWNED);
1104
1105	/* complete all commands with a NULL reply */
1106	for (i = 1; i < sc->num_reqs; i++) {
1107		cm = &sc->commands[i];
1108		if (cm->cm_state == MPS_CM_STATE_FREE)
1109			continue;
1110
1111		cm->cm_state = MPS_CM_STATE_BUSY;
1112		cm->cm_reply = NULL;
1113		completed = 0;
1114
1115		if (cm->cm_flags & MPS_CM_FLAGS_SATA_ID_TIMEOUT) {
1116			MPASS(cm->cm_data);
1117			free(cm->cm_data, M_MPT2);
1118			cm->cm_data = NULL;
1119		}
1120
1121		if (cm->cm_flags & MPS_CM_FLAGS_POLLED)
1122			cm->cm_flags |= MPS_CM_FLAGS_COMPLETE;
1123
1124		if (cm->cm_complete != NULL) {
1125			mpssas_log_command(cm, MPS_RECOVERY,
1126			    "completing cm %p state %x ccb %p for diag reset\n",
1127			    cm, cm->cm_state, cm->cm_ccb);
1128
1129			cm->cm_complete(sc, cm);
1130			completed = 1;
1131		} else if (cm->cm_flags & MPS_CM_FLAGS_WAKEUP) {
1132			mpssas_log_command(cm, MPS_RECOVERY,
1133			    "waking up cm %p state %x ccb %p for diag reset\n",
1134			    cm, cm->cm_state, cm->cm_ccb);
1135			wakeup(cm);
1136			completed = 1;
1137		}
1138
1139		if ((completed == 0) && (cm->cm_state != MPS_CM_STATE_FREE)) {
1140			/* this should never happen, but if it does, log */
1141			mpssas_log_command(cm, MPS_RECOVERY,
1142			    "cm %p state %x flags 0x%x ccb %p during diag "
1143			    "reset\n", cm, cm->cm_state, cm->cm_flags,
1144			    cm->cm_ccb);
1145		}
1146	}
1147
1148	sc->io_cmds_active = 0;
1149}
1150
1151void
1152mpssas_handle_reinit(struct mps_softc *sc)
1153{
1154	int i;
1155
1156	/* Go back into startup mode and freeze the simq, so that CAM
1157	 * doesn't send any commands until after we've rediscovered all
1158	 * targets and found the proper device handles for them.
1159	 *
1160	 * After the reset, portenable will trigger discovery, and after all
1161	 * discovery-related activities have finished, the simq will be
1162	 * released.
1163	 */
1164	mps_dprint(sc, MPS_INIT, "%s startup\n", __func__);
1165	sc->sassc->flags |= MPSSAS_IN_STARTUP;
1166	sc->sassc->flags |= MPSSAS_IN_DISCOVERY;
1167	mpssas_startup_increment(sc->sassc);
1168
1169	/* notify CAM of a bus reset */
1170	mpssas_announce_reset(sc, AC_BUS_RESET, CAM_TARGET_WILDCARD,
1171	    CAM_LUN_WILDCARD);
1172
1173	/* complete and cleanup after all outstanding commands */
1174	mpssas_complete_all_commands(sc);
1175
1176	mps_dprint(sc, MPS_INIT,
1177	    "%s startup %u after command completion\n", __func__,
1178	    sc->sassc->startup_refcount);
1179
1180	/* zero all the target handles, since they may change after the
1181	 * reset, and we have to rediscover all the targets and use the new
1182	 * handles.
1183	 */
1184	for (i = 0; i < sc->sassc->maxtargets; i++) {
1185		if (sc->sassc->targets[i].outstanding != 0)
1186			mps_dprint(sc, MPS_INIT, "target %u outstanding %u\n",
1187			    i, sc->sassc->targets[i].outstanding);
1188		sc->sassc->targets[i].handle = 0x0;
1189		sc->sassc->targets[i].exp_dev_handle = 0x0;
1190		sc->sassc->targets[i].outstanding = 0;
1191		sc->sassc->targets[i].flags = MPSSAS_TARGET_INDIAGRESET;
1192	}
1193}
1194
1195static void
1196mpssas_tm_timeout(void *data)
1197{
1198	struct mps_command *tm = data;
1199	struct mps_softc *sc = tm->cm_sc;
1200
1201	mtx_assert(&sc->mps_mtx, MA_OWNED);
1202
1203	mpssas_log_command(tm, MPS_INFO|MPS_RECOVERY,
1204	    "task mgmt %p timed out\n", tm);
1205
1206	KASSERT(tm->cm_state == MPS_CM_STATE_INQUEUE,
1207	    ("command not inqueue\n"));
1208
1209	tm->cm_state = MPS_CM_STATE_BUSY;
1210	mps_reinit(sc);
1211}
1212
1213static void
1214mpssas_logical_unit_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1215{
1216	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1217	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1218	unsigned int cm_count = 0;
1219	struct mps_command *cm;
1220	struct mpssas_target *targ;
1221
1222	callout_stop(&tm->cm_callout);
1223
1224	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1225	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1226	targ = tm->cm_targ;
1227
1228	/*
1229	 * Currently there should be no way we can hit this case.  It only
1230	 * happens when we have a failure to allocate chain frames, and
1231	 * task management commands don't have S/G lists.
1232	 * XXXSL So should it be an assertion?
1233	 */
1234	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1235		mps_dprint(sc, MPS_RECOVERY|MPS_ERROR,
1236		    "%s: cm_flags = %#x for LUN reset! "
1237		   "This should not happen!\n", __func__, tm->cm_flags);
1238		mpssas_free_tm(sc, tm);
1239		return;
1240	}
1241
1242	if (reply == NULL) {
1243		mps_dprint(sc, MPS_RECOVERY, "NULL reset reply for tm %p\n",
1244		    tm);
1245		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1246			/* this completion was due to a reset, just cleanup */
1247			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1248			    "reset, ignoring NULL LUN reset reply\n");
1249			targ->tm = NULL;
1250			mpssas_free_tm(sc, tm);
1251		}
1252		else {
1253			/* we should have gotten a reply. */
1254			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1255			    "LUN reset attempt, resetting controller\n");
1256			mps_reinit(sc);
1257		}
1258		return;
1259	}
1260
1261	mps_dprint(sc, MPS_RECOVERY,
1262	    "logical unit reset status 0x%x code 0x%x count %u\n",
1263	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1264	    le32toh(reply->TerminationCount));
1265
1266	/*
1267	 * See if there are any outstanding commands for this LUN.
1268	 * This could be made more efficient by using a per-LU data
1269	 * structure of some sort.
1270	 */
1271	TAILQ_FOREACH(cm, &targ->commands, cm_link) {
1272		if (cm->cm_lun == tm->cm_lun)
1273			cm_count++;
1274	}
1275
1276	if (cm_count == 0) {
1277		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1278		    "Finished recovery after LUN reset for target %u\n",
1279		    targ->tid);
1280
1281		mpssas_announce_reset(sc, AC_SENT_BDR, targ->tid, tm->cm_lun);
1282
1283		/*
1284		 * We've finished recovery for this logical unit.  check and
1285		 * see if some other logical unit has a timedout command
1286		 * that needs to be processed.
1287		 */
1288		cm = TAILQ_FIRST(&targ->timedout_commands);
1289		if (cm) {
1290			mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1291			    "More commands to abort for target %u\n",
1292			    targ->tid);
1293			mpssas_send_abort(sc, tm, cm);
1294		} else {
1295			targ->tm = NULL;
1296			mpssas_free_tm(sc, tm);
1297		}
1298	} else {
1299		/*
1300		 * If we still have commands for this LUN, the reset
1301		 * effectively failed, regardless of the status reported.
1302		 * Escalate to a target reset.
1303		 */
1304		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1305		    "logical unit reset complete for target %u, but still "
1306		    "have %u command(s), sending target reset\n", targ->tid,
1307		    cm_count);
1308		mpssas_send_reset(sc, tm,
1309		    MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET);
1310	}
1311}
1312
1313static void
1314mpssas_target_reset_complete(struct mps_softc *sc, struct mps_command *tm)
1315{
1316	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1317	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1318	struct mpssas_target *targ;
1319
1320	callout_stop(&tm->cm_callout);
1321
1322	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1323	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1324	targ = tm->cm_targ;
1325
1326	/*
1327	 * Currently there should be no way we can hit this case.  It only
1328	 * happens when we have a failure to allocate chain frames, and
1329	 * task management commands don't have S/G lists.
1330	 */
1331	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1332		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x for target reset! "
1333			   "This should not happen!\n", __func__, tm->cm_flags);
1334		mpssas_free_tm(sc, tm);
1335		return;
1336	}
1337
1338	if (reply == NULL) {
1339		mps_dprint(sc, MPS_RECOVERY,
1340		    "NULL target reset reply for tm %pi TaskMID %u\n",
1341		    tm, le16toh(req->TaskMID));
1342		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1343			/* this completion was due to a reset, just cleanup */
1344			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1345			    "reset, ignoring NULL target reset reply\n");
1346			targ->tm = NULL;
1347			mpssas_free_tm(sc, tm);
1348		} else {
1349			/* we should have gotten a reply. */
1350			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1351			    "target reset attempt, resetting controller\n");
1352			mps_reinit(sc);
1353		}
1354		return;
1355	}
1356
1357	mps_dprint(sc, MPS_RECOVERY,
1358	    "target reset status 0x%x code 0x%x count %u\n",
1359	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1360	    le32toh(reply->TerminationCount));
1361
1362	if (targ->outstanding == 0) {
1363		/* we've finished recovery for this target and all
1364		 * of its logical units.
1365		 */
1366		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1367		    "Finished reset recovery for target %u\n", targ->tid);
1368
1369		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
1370		    CAM_LUN_WILDCARD);
1371
1372		targ->tm = NULL;
1373		mpssas_free_tm(sc, tm);
1374	} else {
1375		/*
1376		 * After a target reset, if this target still has
1377		 * outstanding commands, the reset effectively failed,
1378		 * regardless of the status reported.  escalate.
1379		 */
1380		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1381		    "Target reset complete for target %u, but still have %u "
1382		    "command(s), resetting controller\n", targ->tid,
1383		    targ->outstanding);
1384		mps_reinit(sc);
1385	}
1386}
1387
1388#define MPS_RESET_TIMEOUT 30
1389
1390int
1391mpssas_send_reset(struct mps_softc *sc, struct mps_command *tm, uint8_t type)
1392{
1393	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1394	struct mpssas_target *target;
1395	int err;
1396
1397	target = tm->cm_targ;
1398	if (target->handle == 0) {
1399		mps_dprint(sc, MPS_ERROR,"%s null devhandle for target_id %d\n",
1400		    __func__, target->tid);
1401		return -1;
1402	}
1403
1404	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1405	req->DevHandle = htole16(target->handle);
1406	req->TaskType = type;
1407
1408	if (type == MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET) {
1409		/* XXX Need to handle invalid LUNs */
1410		MPS_SET_LUN(req->LUN, tm->cm_lun);
1411		tm->cm_targ->logical_unit_resets++;
1412		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1413		    "Sending logical unit reset to target %u lun %d\n",
1414		    target->tid, tm->cm_lun);
1415		tm->cm_complete = mpssas_logical_unit_reset_complete;
1416		mpssas_prepare_for_tm(sc, tm, target, tm->cm_lun);
1417	} else if (type == MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET) {
1418		/*
1419		 * Target reset method =
1420		 * 	SAS Hard Link Reset / SATA Link Reset
1421		 */
1422		req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
1423		tm->cm_targ->target_resets++;
1424		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1425		    "Sending target reset to target %u\n", target->tid);
1426		tm->cm_complete = mpssas_target_reset_complete;
1427		mpssas_prepare_for_tm(sc, tm, target, CAM_LUN_WILDCARD);
1428	} else {
1429		mps_dprint(sc, MPS_ERROR, "unexpected reset type 0x%x\n", type);
1430		return -1;
1431	}
1432
1433	tm->cm_data = NULL;
1434	tm->cm_complete_data = (void *)tm;
1435
1436	callout_reset(&tm->cm_callout, MPS_RESET_TIMEOUT * hz,
1437	    mpssas_tm_timeout, tm);
1438
1439	err = mps_map_command(sc, tm);
1440	if (err)
1441		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1442		    "error %d sending reset type %u\n",
1443		    err, type);
1444
1445	return err;
1446}
1447
1448
1449static void
1450mpssas_abort_complete(struct mps_softc *sc, struct mps_command *tm)
1451{
1452	struct mps_command *cm;
1453	MPI2_SCSI_TASK_MANAGE_REPLY *reply;
1454	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1455	struct mpssas_target *targ;
1456
1457	callout_stop(&tm->cm_callout);
1458
1459	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1460	reply = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
1461	targ = tm->cm_targ;
1462
1463	/*
1464	 * Currently there should be no way we can hit this case.  It only
1465	 * happens when we have a failure to allocate chain frames, and
1466	 * task management commands don't have S/G lists.
1467	 */
1468	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
1469		mps_dprint(sc, MPS_RECOVERY,
1470		    "cm_flags = %#x for abort %p TaskMID %u!\n",
1471		    tm->cm_flags, tm, le16toh(req->TaskMID));
1472		mpssas_free_tm(sc, tm);
1473		return;
1474	}
1475
1476	if (reply == NULL) {
1477		mps_dprint(sc, MPS_RECOVERY,
1478		    "NULL abort reply for tm %p TaskMID %u\n",
1479		    tm, le16toh(req->TaskMID));
1480		if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
1481			/* this completion was due to a reset, just cleanup */
1482			mps_dprint(sc, MPS_RECOVERY, "Hardware undergoing "
1483			    "reset, ignoring NULL abort reply\n");
1484			targ->tm = NULL;
1485			mpssas_free_tm(sc, tm);
1486		} else {
1487			/* we should have gotten a reply. */
1488			mps_dprint(sc, MPS_INFO|MPS_RECOVERY, "NULL reply on "
1489			    "abort attempt, resetting controller\n");
1490			mps_reinit(sc);
1491		}
1492		return;
1493	}
1494
1495	mps_dprint(sc, MPS_RECOVERY,
1496	    "abort TaskMID %u status 0x%x code 0x%x count %u\n",
1497	    le16toh(req->TaskMID),
1498	    le16toh(reply->IOCStatus), le32toh(reply->ResponseCode),
1499	    le32toh(reply->TerminationCount));
1500
1501	cm = TAILQ_FIRST(&tm->cm_targ->timedout_commands);
1502	if (cm == NULL) {
1503		/*
1504		 * If there are no more timedout commands, we're done with
1505		 * error recovery for this target.
1506		 */
1507		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1508		    "Finished abort recovery for target %u\n", targ->tid);
1509
1510		targ->tm = NULL;
1511		mpssas_free_tm(sc, tm);
1512	} else if (le16toh(req->TaskMID) != cm->cm_desc.Default.SMID) {
1513		/* abort success, but we have more timedout commands to abort */
1514		mps_dprint(sc, MPS_INFO|MPS_RECOVERY,
1515		    "Continuing abort recovery for target %u\n", targ->tid);
1516
1517		mpssas_send_abort(sc, tm, cm);
1518	} else {
1519		/* we didn't get a command completion, so the abort
1520		 * failed as far as we're concerned.  escalate.
1521		 */
1522		mps_dprint(sc, MPS_RECOVERY,
1523		    "Abort failed for target %u, sending logical unit reset\n",
1524		    targ->tid);
1525
1526		mpssas_send_reset(sc, tm,
1527		    MPI2_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET);
1528	}
1529}
1530
1531#define MPS_ABORT_TIMEOUT 5
1532
1533static int
1534mpssas_send_abort(struct mps_softc *sc, struct mps_command *tm, struct mps_command *cm)
1535{
1536	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
1537	struct mpssas_target *targ;
1538	int err;
1539
1540	targ = cm->cm_targ;
1541	if (targ->handle == 0) {
1542		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1543		    "%s null devhandle for target_id %d\n",
1544		    __func__, cm->cm_ccb->ccb_h.target_id);
1545		return -1;
1546	}
1547
1548	mpssas_log_command(cm, MPS_RECOVERY|MPS_INFO,
1549	    "Aborting command %p\n", cm);
1550
1551	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
1552	req->DevHandle = htole16(targ->handle);
1553	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_ABORT_TASK;
1554
1555	/* XXX Need to handle invalid LUNs */
1556	MPS_SET_LUN(req->LUN, cm->cm_ccb->ccb_h.target_lun);
1557
1558	req->TaskMID = htole16(cm->cm_desc.Default.SMID);
1559
1560	tm->cm_data = NULL;
1561	tm->cm_complete = mpssas_abort_complete;
1562	tm->cm_complete_data = (void *)tm;
1563	tm->cm_targ = cm->cm_targ;
1564	tm->cm_lun = cm->cm_lun;
1565
1566	callout_reset(&tm->cm_callout, MPS_ABORT_TIMEOUT * hz,
1567	    mpssas_tm_timeout, tm);
1568
1569	targ->aborts++;
1570
1571	mpssas_prepare_for_tm(sc, tm, targ, tm->cm_lun);
1572
1573	err = mps_map_command(sc, tm);
1574	if (err)
1575		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1576		    "error %d sending abort for cm %p SMID %u\n",
1577		    err, cm, req->TaskMID);
1578	return err;
1579}
1580
1581static void
1582mpssas_scsiio_timeout(void *data)
1583{
1584	sbintime_t elapsed, now;
1585	union ccb *ccb;
1586	struct mps_softc *sc;
1587	struct mps_command *cm;
1588	struct mpssas_target *targ;
1589
1590	cm = (struct mps_command *)data;
1591	sc = cm->cm_sc;
1592	ccb = cm->cm_ccb;
1593	now = sbinuptime();
1594
1595	MPS_FUNCTRACE(sc);
1596	mtx_assert(&sc->mps_mtx, MA_OWNED);
1597
1598	mps_dprint(sc, MPS_XINFO|MPS_RECOVERY, "Timeout checking cm %p\n", sc);
1599
1600	/*
1601	 * Run the interrupt handler to make sure it's not pending.  This
1602	 * isn't perfect because the command could have already completed
1603	 * and been re-used, though this is unlikely.
1604	 */
1605	mps_intr_locked(sc);
1606	if (cm->cm_flags & MPS_CM_FLAGS_ON_RECOVERY) {
1607		mpssas_log_command(cm, MPS_XINFO,
1608		    "SCSI command %p almost timed out\n", cm);
1609		return;
1610	}
1611
1612	if (cm->cm_ccb == NULL) {
1613		mps_dprint(sc, MPS_ERROR, "command timeout with NULL ccb\n");
1614		return;
1615	}
1616
1617	targ = cm->cm_targ;
1618	targ->timeouts++;
1619
1620	elapsed = now - ccb->ccb_h.qos.sim_data;
1621	mpssas_log_command(cm, MPS_INFO|MPS_RECOVERY,
1622	    "Command timeout on target %u(0x%04x) %d set, %d.%d elapsed\n",
1623	    targ->tid, targ->handle, ccb->ccb_h.timeout,
1624	    sbintime_getsec(elapsed), elapsed & 0xffffffff);
1625
1626	/* XXX first, check the firmware state, to see if it's still
1627	 * operational.  if not, do a diag reset.
1628	 */
1629	mpssas_set_ccbstatus(cm->cm_ccb, CAM_CMD_TIMEOUT);
1630	cm->cm_flags |= MPS_CM_FLAGS_ON_RECOVERY | MPS_CM_FLAGS_TIMEDOUT;
1631	TAILQ_INSERT_TAIL(&targ->timedout_commands, cm, cm_recovery);
1632
1633	if (targ->tm != NULL) {
1634		/* target already in recovery, just queue up another
1635		 * timedout command to be processed later.
1636		 */
1637		mps_dprint(sc, MPS_RECOVERY,
1638		    "queued timedout cm %p for processing by tm %p\n",
1639		    cm, targ->tm);
1640	} else if ((targ->tm = mpssas_alloc_tm(sc)) != NULL) {
1641		mps_dprint(sc, MPS_RECOVERY|MPS_INFO,
1642		    "Sending abort to target %u for SMID %d\n", targ->tid,
1643		    cm->cm_desc.Default.SMID);
1644		mps_dprint(sc, MPS_RECOVERY, "timedout cm %p allocated tm %p\n",
1645		    cm, targ->tm);
1646
1647		/* start recovery by aborting the first timedout command */
1648		mpssas_send_abort(sc, targ->tm, cm);
1649	} else {
1650		/* XXX queue this target up for recovery once a TM becomes
1651		 * available.  The firmware only has a limited number of
1652		 * HighPriority credits for the high priority requests used
1653		 * for task management, and we ran out.
1654		 *
1655		 * Isilon: don't worry about this for now, since we have
1656		 * more credits than disks in an enclosure, and limit
1657		 * ourselves to one TM per target for recovery.
1658		 */
1659		mps_dprint(sc, MPS_ERROR|MPS_RECOVERY,
1660		    "timedout cm %p failed to allocate a tm\n", cm);
1661	}
1662
1663}
1664
1665static void
1666mpssas_action_scsiio(struct mpssas_softc *sassc, union ccb *ccb)
1667{
1668	MPI2_SCSI_IO_REQUEST *req;
1669	struct ccb_scsiio *csio;
1670	struct mps_softc *sc;
1671	struct mpssas_target *targ;
1672	struct mpssas_lun *lun;
1673	struct mps_command *cm;
1674	uint8_t i, lba_byte, *ref_tag_addr;
1675	uint16_t eedp_flags;
1676	uint32_t mpi_control;
1677
1678	sc = sassc->sc;
1679	MPS_FUNCTRACE(sc);
1680	mtx_assert(&sc->mps_mtx, MA_OWNED);
1681
1682	csio = &ccb->csio;
1683	KASSERT(csio->ccb_h.target_id < sassc->maxtargets,
1684	    ("Target %d out of bounds in XPT_SCSI_IO\n",
1685	     csio->ccb_h.target_id));
1686	targ = &sassc->targets[csio->ccb_h.target_id];
1687	mps_dprint(sc, MPS_TRACE, "ccb %p target flag %x\n", ccb, targ->flags);
1688	if (targ->handle == 0x0) {
1689		mps_dprint(sc, MPS_ERROR, "%s NULL handle for target %u\n",
1690		    __func__, csio->ccb_h.target_id);
1691		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1692		xpt_done(ccb);
1693		return;
1694	}
1695	if (targ->flags & MPS_TARGET_FLAGS_RAID_COMPONENT) {
1696		mps_dprint(sc, MPS_ERROR, "%s Raid component no SCSI IO "
1697		    "supported %u\n", __func__, csio->ccb_h.target_id);
1698		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1699		xpt_done(ccb);
1700		return;
1701	}
1702	/*
1703	 * Sometimes, it is possible to get a command that is not "In
1704	 * Progress" and was actually aborted by the upper layer.  Check for
1705	 * this here and complete the command without error.
1706	 */
1707	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_INPROG) {
1708		mps_dprint(sc, MPS_TRACE, "%s Command is not in progress for "
1709		    "target %u\n", __func__, csio->ccb_h.target_id);
1710		xpt_done(ccb);
1711		return;
1712	}
1713	/*
1714	 * If devinfo is 0 this will be a volume.  In that case don't tell CAM
1715	 * that the volume has timed out.  We want volumes to be enumerated
1716	 * until they are deleted/removed, not just failed.
1717	 */
1718	if (targ->flags & MPSSAS_TARGET_INREMOVAL) {
1719		if (targ->devinfo == 0)
1720			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
1721		else
1722			mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
1723		xpt_done(ccb);
1724		return;
1725	}
1726
1727	if ((sc->mps_flags & MPS_FLAGS_SHUTDOWN) != 0) {
1728		mps_dprint(sc, MPS_INFO, "%s shutting down\n", __func__);
1729		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
1730		xpt_done(ccb);
1731		return;
1732	}
1733
1734	/*
1735	 * If target has a reset in progress, freeze the devq and return.  The
1736	 * devq will be released when the TM reset is finished.
1737	 */
1738	if (targ->flags & MPSSAS_TARGET_INRESET) {
1739		ccb->ccb_h.status = CAM_BUSY | CAM_DEV_QFRZN;
1740		mps_dprint(sc, MPS_INFO, "%s: Freezing devq for target ID %d\n",
1741		    __func__, targ->tid);
1742		xpt_freeze_devq(ccb->ccb_h.path, 1);
1743		xpt_done(ccb);
1744		return;
1745	}
1746
1747	cm = mps_alloc_command(sc);
1748	if (cm == NULL || (sc->mps_flags & MPS_FLAGS_DIAGRESET)) {
1749		if (cm != NULL) {
1750			mps_free_command(sc, cm);
1751		}
1752		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
1753			xpt_freeze_simq(sassc->sim, 1);
1754			sassc->flags |= MPSSAS_QUEUE_FROZEN;
1755		}
1756		ccb->ccb_h.status &= ~CAM_SIM_QUEUED;
1757		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
1758		xpt_done(ccb);
1759		return;
1760	}
1761
1762	req = (MPI2_SCSI_IO_REQUEST *)cm->cm_req;
1763	bzero(req, sizeof(*req));
1764	req->DevHandle = htole16(targ->handle);
1765	req->Function = MPI2_FUNCTION_SCSI_IO_REQUEST;
1766	req->MsgFlags = 0;
1767	req->SenseBufferLowAddress = htole32(cm->cm_sense_busaddr);
1768	req->SenseBufferLength = MPS_SENSE_LEN;
1769	req->SGLFlags = 0;
1770	req->ChainOffset = 0;
1771	req->SGLOffset0 = 24;	/* 32bit word offset to the SGL */
1772	req->SGLOffset1= 0;
1773	req->SGLOffset2= 0;
1774	req->SGLOffset3= 0;
1775	req->SkipCount = 0;
1776	req->DataLength = htole32(csio->dxfer_len);
1777	req->BidirectionalDataLength = 0;
1778	req->IoFlags = htole16(csio->cdb_len);
1779	req->EEDPFlags = 0;
1780
1781	/* Note: BiDirectional transfers are not supported */
1782	switch (csio->ccb_h.flags & CAM_DIR_MASK) {
1783	case CAM_DIR_IN:
1784		mpi_control = MPI2_SCSIIO_CONTROL_READ;
1785		cm->cm_flags |= MPS_CM_FLAGS_DATAIN;
1786		break;
1787	case CAM_DIR_OUT:
1788		mpi_control = MPI2_SCSIIO_CONTROL_WRITE;
1789		cm->cm_flags |= MPS_CM_FLAGS_DATAOUT;
1790		break;
1791	case CAM_DIR_NONE:
1792	default:
1793		mpi_control = MPI2_SCSIIO_CONTROL_NODATATRANSFER;
1794		break;
1795	}
1796
1797	if (csio->cdb_len == 32)
1798                mpi_control |= 4 << MPI2_SCSIIO_CONTROL_ADDCDBLEN_SHIFT;
1799	/*
1800	 * It looks like the hardware doesn't require an explicit tag
1801	 * number for each transaction.  SAM Task Management not supported
1802	 * at the moment.
1803	 */
1804	switch (csio->tag_action) {
1805	case MSG_HEAD_OF_Q_TAG:
1806		mpi_control |= MPI2_SCSIIO_CONTROL_HEADOFQ;
1807		break;
1808	case MSG_ORDERED_Q_TAG:
1809		mpi_control |= MPI2_SCSIIO_CONTROL_ORDEREDQ;
1810		break;
1811	case MSG_ACA_TASK:
1812		mpi_control |= MPI2_SCSIIO_CONTROL_ACAQ;
1813		break;
1814	case CAM_TAG_ACTION_NONE:
1815	case MSG_SIMPLE_Q_TAG:
1816	default:
1817		mpi_control |= MPI2_SCSIIO_CONTROL_SIMPLEQ;
1818		break;
1819	}
1820	mpi_control |= sc->mapping_table[csio->ccb_h.target_id].TLR_bits;
1821	req->Control = htole32(mpi_control);
1822	if (MPS_SET_LUN(req->LUN, csio->ccb_h.target_lun) != 0) {
1823		mps_free_command(sc, cm);
1824		mpssas_set_ccbstatus(ccb, CAM_LUN_INVALID);
1825		xpt_done(ccb);
1826		return;
1827	}
1828
1829	if (csio->ccb_h.flags & CAM_CDB_POINTER)
1830		bcopy(csio->cdb_io.cdb_ptr, &req->CDB.CDB32[0], csio->cdb_len);
1831	else
1832		bcopy(csio->cdb_io.cdb_bytes, &req->CDB.CDB32[0],csio->cdb_len);
1833	req->IoFlags = htole16(csio->cdb_len);
1834
1835	/*
1836	 * Check if EEDP is supported and enabled.  If it is then check if the
1837	 * SCSI opcode could be using EEDP.  If so, make sure the LUN exists and
1838	 * is formatted for EEDP support.  If all of this is true, set CDB up
1839	 * for EEDP transfer.
1840	 */
1841	eedp_flags = op_code_prot[req->CDB.CDB32[0]];
1842	if (sc->eedp_enabled && eedp_flags) {
1843		SLIST_FOREACH(lun, &targ->luns, lun_link) {
1844			if (lun->lun_id == csio->ccb_h.target_lun) {
1845				break;
1846			}
1847		}
1848
1849		if ((lun != NULL) && (lun->eedp_formatted)) {
1850			req->EEDPBlockSize = htole16(lun->eedp_block_size);
1851			eedp_flags |= (MPI2_SCSIIO_EEDPFLAGS_INC_PRI_REFTAG |
1852			    MPI2_SCSIIO_EEDPFLAGS_CHECK_REFTAG |
1853			    MPI2_SCSIIO_EEDPFLAGS_CHECK_GUARD);
1854			req->EEDPFlags = htole16(eedp_flags);
1855
1856			/*
1857			 * If CDB less than 32, fill in Primary Ref Tag with
1858			 * low 4 bytes of LBA.  If CDB is 32, tag stuff is
1859			 * already there.  Also, set protection bit.  FreeBSD
1860			 * currently does not support CDBs bigger than 16, but
1861			 * the code doesn't hurt, and will be here for the
1862			 * future.
1863			 */
1864			if (csio->cdb_len != 32) {
1865				lba_byte = (csio->cdb_len == 16) ? 6 : 2;
1866				ref_tag_addr = (uint8_t *)&req->CDB.EEDP32.
1867				    PrimaryReferenceTag;
1868				for (i = 0; i < 4; i++) {
1869					*ref_tag_addr =
1870					    req->CDB.CDB32[lba_byte + i];
1871					ref_tag_addr++;
1872				}
1873				req->CDB.EEDP32.PrimaryReferenceTag =
1874					htole32(req->CDB.EEDP32.PrimaryReferenceTag);
1875				req->CDB.EEDP32.PrimaryApplicationTagMask =
1876				    0xFFFF;
1877				req->CDB.CDB32[1] = (req->CDB.CDB32[1] & 0x1F) |
1878				    0x20;
1879			} else {
1880				eedp_flags |=
1881				    MPI2_SCSIIO_EEDPFLAGS_INC_PRI_APPTAG;
1882				req->EEDPFlags = htole16(eedp_flags);
1883				req->CDB.CDB32[10] = (req->CDB.CDB32[10] &
1884				    0x1F) | 0x20;
1885			}
1886		}
1887	}
1888
1889	cm->cm_length = csio->dxfer_len;
1890	if (cm->cm_length != 0) {
1891		cm->cm_data = ccb;
1892		cm->cm_flags |= MPS_CM_FLAGS_USE_CCB;
1893	} else {
1894		cm->cm_data = NULL;
1895	}
1896	cm->cm_sge = &req->SGL;
1897	cm->cm_sglsize = (32 - 24) * 4;
1898	cm->cm_desc.SCSIIO.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_SCSI_IO;
1899	cm->cm_desc.SCSIIO.DevHandle = htole16(targ->handle);
1900	cm->cm_complete = mpssas_scsiio_complete;
1901	cm->cm_complete_data = ccb;
1902	cm->cm_targ = targ;
1903	cm->cm_lun = csio->ccb_h.target_lun;
1904	cm->cm_ccb = ccb;
1905
1906	/*
1907	 * If HBA is a WD and the command is not for a retry, try to build a
1908	 * direct I/O message. If failed, or the command is for a retry, send
1909	 * the I/O to the IR volume itself.
1910	 */
1911	if (sc->WD_valid_config) {
1912		if (ccb->ccb_h.sim_priv.entries[0].field == MPS_WD_RETRY) {
1913			mpssas_direct_drive_io(sassc, cm, ccb);
1914		} else {
1915			mpssas_set_ccbstatus(ccb, CAM_REQ_INPROG);
1916		}
1917	}
1918
1919#if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
1920	if (csio->bio != NULL)
1921		biotrack(csio->bio, __func__);
1922#endif
1923	csio->ccb_h.qos.sim_data = sbinuptime();
1924	callout_reset_sbt(&cm->cm_callout, SBT_1MS * ccb->ccb_h.timeout, 0,
1925	    mpssas_scsiio_timeout, cm, 0);
1926
1927	targ->issued++;
1928	targ->outstanding++;
1929	TAILQ_INSERT_TAIL(&targ->commands, cm, cm_link);
1930	ccb->ccb_h.status |= CAM_SIM_QUEUED;
1931
1932	mpssas_log_command(cm, MPS_XINFO, "%s cm %p ccb %p outstanding %u\n",
1933	    __func__, cm, ccb, targ->outstanding);
1934
1935	mps_map_command(sc, cm);
1936	return;
1937}
1938
1939/**
1940 * mps_sc_failed_io_info - translated non-succesfull SCSI_IO request
1941 */
1942static void
1943mps_sc_failed_io_info(struct mps_softc *sc, struct ccb_scsiio *csio,
1944    Mpi2SCSIIOReply_t *mpi_reply)
1945{
1946	u32 response_info;
1947	u8 *response_bytes;
1948	u16 ioc_status = le16toh(mpi_reply->IOCStatus) &
1949	    MPI2_IOCSTATUS_MASK;
1950	u8 scsi_state = mpi_reply->SCSIState;
1951	u8 scsi_status = mpi_reply->SCSIStatus;
1952	u32 log_info = le32toh(mpi_reply->IOCLogInfo);
1953	const char *desc_ioc_state, *desc_scsi_status;
1954
1955	if (log_info == 0x31170000)
1956		return;
1957
1958	desc_ioc_state = mps_describe_table(mps_iocstatus_string,
1959	    ioc_status);
1960	desc_scsi_status = mps_describe_table(mps_scsi_status_string,
1961	    scsi_status);
1962
1963	mps_dprint(sc, MPS_XINFO, "\thandle(0x%04x), ioc_status(%s)(0x%04x)\n",
1964	    le16toh(mpi_reply->DevHandle), desc_ioc_state, ioc_status);
1965
1966	/*
1967	 *We can add more detail about underflow data here
1968	 * TO-DO
1969	 */
1970	mps_dprint(sc, MPS_XINFO, "\tscsi_status(%s)(0x%02x), "
1971	    "scsi_state %b\n", desc_scsi_status, scsi_status,
1972	    scsi_state, "\20" "\1AutosenseValid" "\2AutosenseFailed"
1973	    "\3NoScsiStatus" "\4Terminated" "\5Response InfoValid");
1974
1975	if (sc->mps_debug & MPS_XINFO &&
1976		scsi_state & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
1977		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : Start :\n");
1978		scsi_sense_print(csio);
1979		mps_dprint(sc, MPS_XINFO, "-> Sense Buffer Data : End :\n");
1980	}
1981
1982	if (scsi_state & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) {
1983		response_info = le32toh(mpi_reply->ResponseInfo);
1984		response_bytes = (u8 *)&response_info;
1985		mps_dprint(sc, MPS_XINFO, "response code(0x%1x): %s\n",
1986		    response_bytes[0],
1987		    mps_describe_table(mps_scsi_taskmgmt_string,
1988		    response_bytes[0]));
1989	}
1990}
1991
1992static void
1993mpssas_scsiio_complete(struct mps_softc *sc, struct mps_command *cm)
1994{
1995	MPI2_SCSI_IO_REPLY *rep;
1996	union ccb *ccb;
1997	struct ccb_scsiio *csio;
1998	struct mpssas_softc *sassc;
1999	struct scsi_vpd_supported_page_list *vpd_list = NULL;
2000	u8 *TLR_bits, TLR_on;
2001	int dir = 0, i;
2002	u16 alloc_len;
2003	struct mpssas_target *target;
2004	target_id_t target_id;
2005
2006	MPS_FUNCTRACE(sc);
2007	mps_dprint(sc, MPS_TRACE,
2008	    "cm %p SMID %u ccb %p reply %p outstanding %u\n", cm,
2009	    cm->cm_desc.Default.SMID, cm->cm_ccb, cm->cm_reply,
2010	    cm->cm_targ->outstanding);
2011
2012	callout_stop(&cm->cm_callout);
2013	mtx_assert(&sc->mps_mtx, MA_OWNED);
2014
2015	sassc = sc->sassc;
2016	ccb = cm->cm_complete_data;
2017	csio = &ccb->csio;
2018	target_id = csio->ccb_h.target_id;
2019	rep = (MPI2_SCSI_IO_REPLY *)cm->cm_reply;
2020	/*
2021	 * XXX KDM if the chain allocation fails, does it matter if we do
2022	 * the sync and unload here?  It is simpler to do it in every case,
2023	 * assuming it doesn't cause problems.
2024	 */
2025	if (cm->cm_data != NULL) {
2026		if (cm->cm_flags & MPS_CM_FLAGS_DATAIN)
2027			dir = BUS_DMASYNC_POSTREAD;
2028		else if (cm->cm_flags & MPS_CM_FLAGS_DATAOUT)
2029			dir = BUS_DMASYNC_POSTWRITE;
2030		bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap, dir);
2031		bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2032	}
2033
2034	cm->cm_targ->completed++;
2035	cm->cm_targ->outstanding--;
2036	TAILQ_REMOVE(&cm->cm_targ->commands, cm, cm_link);
2037	ccb->ccb_h.status &= ~(CAM_STATUS_MASK | CAM_SIM_QUEUED);
2038
2039#if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
2040	if (ccb->csio.bio != NULL)
2041		biotrack(ccb->csio.bio, __func__);
2042#endif
2043
2044	if (cm->cm_flags & MPS_CM_FLAGS_ON_RECOVERY) {
2045		TAILQ_REMOVE(&cm->cm_targ->timedout_commands, cm, cm_recovery);
2046		KASSERT(cm->cm_state == MPS_CM_STATE_BUSY,
2047		    ("Not busy for CM_FLAGS_TIMEDOUT: %d\n", cm->cm_state));
2048		cm->cm_flags &= ~MPS_CM_FLAGS_ON_RECOVERY;
2049		if (cm->cm_reply != NULL)
2050			mpssas_log_command(cm, MPS_RECOVERY,
2051			    "completed timedout cm %p ccb %p during recovery "
2052			    "ioc %x scsi %x state %x xfer %u\n",
2053			    cm, cm->cm_ccb, le16toh(rep->IOCStatus),
2054			    rep->SCSIStatus, rep->SCSIState,
2055			    le32toh(rep->TransferCount));
2056		else
2057			mpssas_log_command(cm, MPS_RECOVERY,
2058			    "completed timedout cm %p ccb %p during recovery\n",
2059			    cm, cm->cm_ccb);
2060	} else if (cm->cm_targ->tm != NULL) {
2061		if (cm->cm_reply != NULL)
2062			mpssas_log_command(cm, MPS_RECOVERY,
2063			    "completed cm %p ccb %p during recovery "
2064			    "ioc %x scsi %x state %x xfer %u\n",
2065			    cm, cm->cm_ccb, le16toh(rep->IOCStatus),
2066			    rep->SCSIStatus, rep->SCSIState,
2067			    le32toh(rep->TransferCount));
2068		else
2069			mpssas_log_command(cm, MPS_RECOVERY,
2070			    "completed cm %p ccb %p during recovery\n",
2071			    cm, cm->cm_ccb);
2072	} else if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0) {
2073		mpssas_log_command(cm, MPS_RECOVERY,
2074		    "reset completed cm %p ccb %p\n",
2075		    cm, cm->cm_ccb);
2076	}
2077
2078	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2079		/*
2080		 * We ran into an error after we tried to map the command,
2081		 * so we're getting a callback without queueing the command
2082		 * to the hardware.  So we set the status here, and it will
2083		 * be retained below.  We'll go through the "fast path",
2084		 * because there can be no reply when we haven't actually
2085		 * gone out to the hardware.
2086		 */
2087		mpssas_set_ccbstatus(ccb, CAM_REQUEUE_REQ);
2088
2089		/*
2090		 * Currently the only error included in the mask is
2091		 * MPS_CM_FLAGS_CHAIN_FAILED, which means we're out of
2092		 * chain frames.  We need to freeze the queue until we get
2093		 * a command that completed without this error, which will
2094		 * hopefully have some chain frames attached that we can
2095		 * use.  If we wanted to get smarter about it, we would
2096		 * only unfreeze the queue in this condition when we're
2097		 * sure that we're getting some chain frames back.  That's
2098		 * probably unnecessary.
2099		 */
2100		if ((sassc->flags & MPSSAS_QUEUE_FROZEN) == 0) {
2101			xpt_freeze_simq(sassc->sim, 1);
2102			sassc->flags |= MPSSAS_QUEUE_FROZEN;
2103			mps_dprint(sc, MPS_XINFO, "Error sending command, "
2104				   "freezing SIM queue\n");
2105		}
2106	}
2107
2108	/*
2109	 * If this is a Start Stop Unit command and it was issued by the driver
2110	 * during shutdown, decrement the refcount to account for all of the
2111	 * commands that were sent.  All SSU commands should be completed before
2112	 * shutdown completes, meaning SSU_refcount will be 0 after SSU_started
2113	 * is TRUE.
2114	 */
2115	if (sc->SSU_started && (csio->cdb_io.cdb_bytes[0] == START_STOP_UNIT)) {
2116		mps_dprint(sc, MPS_INFO, "Decrementing SSU count.\n");
2117		sc->SSU_refcount--;
2118	}
2119
2120	/* Take the fast path to completion */
2121	if (cm->cm_reply == NULL) {
2122		if (mpssas_get_ccbstatus(ccb) == CAM_REQ_INPROG) {
2123			if ((sc->mps_flags & MPS_FLAGS_DIAGRESET) != 0)
2124				mpssas_set_ccbstatus(ccb, CAM_SCSI_BUS_RESET);
2125			else {
2126				mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2127				ccb->csio.scsi_status = SCSI_STATUS_OK;
2128			}
2129			if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2130				ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2131				sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2132				mps_dprint(sc, MPS_XINFO,
2133				    "Unfreezing SIM queue\n");
2134			}
2135		}
2136
2137		/*
2138		 * There are two scenarios where the status won't be
2139		 * CAM_REQ_CMP.  The first is if MPS_CM_FLAGS_ERROR_MASK is
2140		 * set, the second is in the MPS_FLAGS_DIAGRESET above.
2141		 */
2142		if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2143			/*
2144			 * Freeze the dev queue so that commands are
2145			 * executed in the correct order after error
2146			 * recovery.
2147			 */
2148			ccb->ccb_h.status |= CAM_DEV_QFRZN;
2149			xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2150		}
2151		mps_free_command(sc, cm);
2152		xpt_done(ccb);
2153		return;
2154	}
2155
2156	mpssas_log_command(cm, MPS_XINFO,
2157	    "ioc %x scsi %x state %x xfer %u\n",
2158	    le16toh(rep->IOCStatus), rep->SCSIStatus, rep->SCSIState,
2159	    le32toh(rep->TransferCount));
2160
2161	/*
2162	 * If this is a Direct Drive I/O, reissue the I/O to the original IR
2163	 * Volume if an error occurred (normal I/O retry).  Use the original
2164	 * CCB, but set a flag that this will be a retry so that it's sent to
2165	 * the original volume.  Free the command but reuse the CCB.
2166	 */
2167	if (cm->cm_flags & MPS_CM_FLAGS_DD_IO) {
2168		mps_free_command(sc, cm);
2169		ccb->ccb_h.sim_priv.entries[0].field = MPS_WD_RETRY;
2170		mpssas_action_scsiio(sassc, ccb);
2171		return;
2172	} else
2173		ccb->ccb_h.sim_priv.entries[0].field = 0;
2174
2175	switch (le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) {
2176	case MPI2_IOCSTATUS_SCSI_DATA_UNDERRUN:
2177		csio->resid = cm->cm_length - le32toh(rep->TransferCount);
2178		/* FALLTHROUGH */
2179	case MPI2_IOCSTATUS_SUCCESS:
2180	case MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR:
2181
2182		if ((le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK) ==
2183		    MPI2_IOCSTATUS_SCSI_RECOVERED_ERROR)
2184			mpssas_log_command(cm, MPS_XINFO, "recovered error\n");
2185
2186		/* Completion failed at the transport level. */
2187		if (rep->SCSIState & (MPI2_SCSI_STATE_NO_SCSI_STATUS |
2188		    MPI2_SCSI_STATE_TERMINATED)) {
2189			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2190			break;
2191		}
2192
2193		/* In a modern packetized environment, an autosense failure
2194		 * implies that there's not much else that can be done to
2195		 * recover the command.
2196		 */
2197		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_FAILED) {
2198			mpssas_set_ccbstatus(ccb, CAM_AUTOSENSE_FAIL);
2199			break;
2200		}
2201
2202		/*
2203		 * CAM doesn't care about SAS Response Info data, but if this is
2204		 * the state check if TLR should be done.  If not, clear the
2205		 * TLR_bits for the target.
2206		 */
2207		if ((rep->SCSIState & MPI2_SCSI_STATE_RESPONSE_INFO_VALID) &&
2208		    ((le32toh(rep->ResponseInfo) &
2209		    MPI2_SCSI_RI_MASK_REASONCODE) ==
2210		    MPS_SCSI_RI_INVALID_FRAME)) {
2211			sc->mapping_table[target_id].TLR_bits =
2212			    (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2213		}
2214
2215		/*
2216		 * Intentionally override the normal SCSI status reporting
2217		 * for these two cases.  These are likely to happen in a
2218		 * multi-initiator environment, and we want to make sure that
2219		 * CAM retries these commands rather than fail them.
2220		 */
2221		if ((rep->SCSIStatus == MPI2_SCSI_STATUS_COMMAND_TERMINATED) ||
2222		    (rep->SCSIStatus == MPI2_SCSI_STATUS_TASK_ABORTED)) {
2223			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2224			break;
2225		}
2226
2227		/* Handle normal status and sense */
2228		csio->scsi_status = rep->SCSIStatus;
2229		if (rep->SCSIStatus == MPI2_SCSI_STATUS_GOOD)
2230			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2231		else
2232			mpssas_set_ccbstatus(ccb, CAM_SCSI_STATUS_ERROR);
2233
2234		if (rep->SCSIState & MPI2_SCSI_STATE_AUTOSENSE_VALID) {
2235			int sense_len, returned_sense_len;
2236
2237			returned_sense_len = min(le32toh(rep->SenseCount),
2238			    sizeof(struct scsi_sense_data));
2239			if (returned_sense_len < ccb->csio.sense_len)
2240				ccb->csio.sense_resid = ccb->csio.sense_len -
2241					returned_sense_len;
2242			else
2243				ccb->csio.sense_resid = 0;
2244
2245			sense_len = min(returned_sense_len,
2246			    ccb->csio.sense_len - ccb->csio.sense_resid);
2247			bzero(&ccb->csio.sense_data,
2248			      sizeof(ccb->csio.sense_data));
2249			bcopy(cm->cm_sense, &ccb->csio.sense_data, sense_len);
2250			ccb->ccb_h.status |= CAM_AUTOSNS_VALID;
2251		}
2252
2253		/*
2254		 * Check if this is an INQUIRY command.  If it's a VPD inquiry,
2255		 * and it's page code 0 (Supported Page List), and there is
2256		 * inquiry data, and this is for a sequential access device, and
2257		 * the device is an SSP target, and TLR is supported by the
2258		 * controller, turn the TLR_bits value ON if page 0x90 is
2259		 * supported.
2260		 */
2261		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2262		    (csio->cdb_io.cdb_bytes[1] & SI_EVPD) &&
2263		    (csio->cdb_io.cdb_bytes[2] == SVPD_SUPPORTED_PAGE_LIST) &&
2264		    ((csio->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) &&
2265		    (csio->data_ptr != NULL) &&
2266		    ((csio->data_ptr[0] & 0x1f) == T_SEQUENTIAL) &&
2267		    (sc->control_TLR) &&
2268		    (sc->mapping_table[target_id].device_info &
2269		    MPI2_SAS_DEVICE_INFO_SSP_TARGET)) {
2270			vpd_list = (struct scsi_vpd_supported_page_list *)
2271			    csio->data_ptr;
2272			TLR_bits = &sc->mapping_table[target_id].TLR_bits;
2273			*TLR_bits = (u8)MPI2_SCSIIO_CONTROL_NO_TLR;
2274			TLR_on = (u8)MPI2_SCSIIO_CONTROL_TLR_ON;
2275			alloc_len = ((u16)csio->cdb_io.cdb_bytes[3] << 8) +
2276			    csio->cdb_io.cdb_bytes[4];
2277			alloc_len -= csio->resid;
2278			for (i = 0; i < MIN(vpd_list->length, alloc_len); i++) {
2279				if (vpd_list->list[i] == 0x90) {
2280					*TLR_bits = TLR_on;
2281					break;
2282				}
2283			}
2284		}
2285
2286		/*
2287		 * If this is a SATA direct-access end device, mark it so that
2288		 * a SCSI StartStopUnit command will be sent to it when the
2289		 * driver is being shutdown.
2290		 */
2291		if ((csio->cdb_io.cdb_bytes[0] == INQUIRY) &&
2292		    ((csio->data_ptr[0] & 0x1f) == T_DIRECT) &&
2293		    (sc->mapping_table[target_id].device_info &
2294		    MPI2_SAS_DEVICE_INFO_SATA_DEVICE) &&
2295		    ((sc->mapping_table[target_id].device_info &
2296		    MPI2_SAS_DEVICE_INFO_MASK_DEVICE_TYPE) ==
2297		    MPI2_SAS_DEVICE_INFO_END_DEVICE)) {
2298			target = &sassc->targets[target_id];
2299			target->supports_SSU = TRUE;
2300			mps_dprint(sc, MPS_XINFO, "Target %d supports SSU\n",
2301			    target_id);
2302		}
2303		break;
2304	case MPI2_IOCSTATUS_SCSI_INVALID_DEVHANDLE:
2305	case MPI2_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
2306		/*
2307		 * If devinfo is 0 this will be a volume.  In that case don't
2308		 * tell CAM that the volume is not there.  We want volumes to
2309		 * be enumerated until they are deleted/removed, not just
2310		 * failed.
2311		 */
2312		if (cm->cm_targ->devinfo == 0)
2313			mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2314		else
2315			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2316		break;
2317	case MPI2_IOCSTATUS_INVALID_SGL:
2318		mps_print_scsiio_cmd(sc, cm);
2319		mpssas_set_ccbstatus(ccb, CAM_UNREC_HBA_ERROR);
2320		break;
2321	case MPI2_IOCSTATUS_SCSI_TASK_TERMINATED:
2322		/*
2323		 * This is one of the responses that comes back when an I/O
2324		 * has been aborted.  If it is because of a timeout that we
2325		 * initiated, just set the status to CAM_CMD_TIMEOUT.
2326		 * Otherwise set it to CAM_REQ_ABORTED.  The effect on the
2327		 * command is the same (it gets retried, subject to the
2328		 * retry counter), the only difference is what gets printed
2329		 * on the console.
2330		 */
2331		if (cm->cm_flags & MPS_CM_FLAGS_TIMEDOUT)
2332			mpssas_set_ccbstatus(ccb, CAM_CMD_TIMEOUT);
2333		else
2334			mpssas_set_ccbstatus(ccb, CAM_REQ_ABORTED);
2335		break;
2336	case MPI2_IOCSTATUS_SCSI_DATA_OVERRUN:
2337		/* resid is ignored for this condition */
2338		csio->resid = 0;
2339		mpssas_set_ccbstatus(ccb, CAM_DATA_RUN_ERR);
2340		break;
2341	case MPI2_IOCSTATUS_SCSI_IOC_TERMINATED:
2342	case MPI2_IOCSTATUS_SCSI_EXT_TERMINATED:
2343		/*
2344		 * These can sometimes be transient transport-related
2345		 * errors, and sometimes persistent drive-related errors.
2346		 * We used to retry these without decrementing the retry
2347		 * count by returning CAM_REQUEUE_REQ.  Unfortunately, if
2348		 * we hit a persistent drive problem that returns one of
2349		 * these error codes, we would retry indefinitely.  So,
2350		 * return CAM_REQ_CMP_ERROR so that we decrement the retry
2351		 * count and avoid infinite retries.  We're taking the
2352		 * potential risk of flagging false failures in the event
2353		 * of a topology-related error (e.g. a SAS expander problem
2354		 * causes a command addressed to a drive to fail), but
2355		 * avoiding getting into an infinite retry loop.
2356		 */
2357		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2358		mps_dprint(sc, MPS_INFO,
2359		    "Controller reported %s tgt %u SMID %u loginfo %x\n",
2360		    mps_describe_table(mps_iocstatus_string,
2361		    le16toh(rep->IOCStatus) & MPI2_IOCSTATUS_MASK),
2362		    target_id, cm->cm_desc.Default.SMID,
2363		    le32toh(rep->IOCLogInfo));
2364		mps_dprint(sc, MPS_XINFO,
2365		    "SCSIStatus %x SCSIState %x xfercount %u\n",
2366		    rep->SCSIStatus, rep->SCSIState,
2367		    le32toh(rep->TransferCount));
2368		break;
2369	case MPI2_IOCSTATUS_INVALID_FUNCTION:
2370	case MPI2_IOCSTATUS_INTERNAL_ERROR:
2371	case MPI2_IOCSTATUS_INVALID_VPID:
2372	case MPI2_IOCSTATUS_INVALID_FIELD:
2373	case MPI2_IOCSTATUS_INVALID_STATE:
2374	case MPI2_IOCSTATUS_OP_STATE_NOT_SUPPORTED:
2375	case MPI2_IOCSTATUS_SCSI_IO_DATA_ERROR:
2376	case MPI2_IOCSTATUS_SCSI_PROTOCOL_ERROR:
2377	case MPI2_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
2378	case MPI2_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
2379	default:
2380		mpssas_log_command(cm, MPS_XINFO,
2381		    "completed ioc %x loginfo %x scsi %x state %x xfer %u\n",
2382		    le16toh(rep->IOCStatus), le32toh(rep->IOCLogInfo),
2383		    rep->SCSIStatus, rep->SCSIState,
2384		    le32toh(rep->TransferCount));
2385		csio->resid = cm->cm_length;
2386		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2387		break;
2388	}
2389
2390	mps_sc_failed_io_info(sc,csio,rep);
2391
2392	if (sassc->flags & MPSSAS_QUEUE_FROZEN) {
2393		ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
2394		sassc->flags &= ~MPSSAS_QUEUE_FROZEN;
2395		mps_dprint(sc, MPS_XINFO, "Command completed, "
2396		    "unfreezing SIM queue\n");
2397	}
2398
2399	if (mpssas_get_ccbstatus(ccb) != CAM_REQ_CMP) {
2400		ccb->ccb_h.status |= CAM_DEV_QFRZN;
2401		xpt_freeze_devq(ccb->ccb_h.path, /*count*/ 1);
2402	}
2403
2404	mps_free_command(sc, cm);
2405	xpt_done(ccb);
2406}
2407
2408/* All Request reached here are Endian safe */
2409static void
2410mpssas_direct_drive_io(struct mpssas_softc *sassc, struct mps_command *cm,
2411    union ccb *ccb) {
2412	pMpi2SCSIIORequest_t	pIO_req;
2413	struct mps_softc	*sc = sassc->sc;
2414	uint64_t		virtLBA;
2415	uint32_t		physLBA, stripe_offset, stripe_unit;
2416	uint32_t		io_size, column;
2417	uint8_t			*ptrLBA, lba_idx, physLBA_byte, *CDB;
2418
2419	/*
2420	 * If this is a valid SCSI command (Read6, Read10, Read16, Write6,
2421	 * Write10, or Write16), build a direct I/O message.  Otherwise, the I/O
2422	 * will be sent to the IR volume itself.  Since Read6 and Write6 are a
2423	 * bit different than the 10/16 CDBs, handle them separately.
2424	 */
2425	pIO_req = (pMpi2SCSIIORequest_t)cm->cm_req;
2426	CDB = pIO_req->CDB.CDB32;
2427
2428	/*
2429	 * Handle 6 byte CDBs.
2430	 */
2431	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_6) ||
2432	    (CDB[0] == WRITE_6))) {
2433		/*
2434		 * Get the transfer size in blocks.
2435		 */
2436		io_size = (cm->cm_length >> sc->DD_block_exponent);
2437
2438		/*
2439		 * Get virtual LBA given in the CDB.
2440		 */
2441		virtLBA = ((uint64_t)(CDB[1] & 0x1F) << 16) |
2442		    ((uint64_t)CDB[2] << 8) | (uint64_t)CDB[3];
2443
2444		/*
2445		 * Check that LBA range for I/O does not exceed volume's
2446		 * MaxLBA.
2447		 */
2448		if ((virtLBA + (uint64_t)io_size - 1) <=
2449		    sc->DD_max_lba) {
2450			/*
2451			 * Check if the I/O crosses a stripe boundary.  If not,
2452			 * translate the virtual LBA to a physical LBA and set
2453			 * the DevHandle for the PhysDisk to be used.  If it
2454			 * does cross a boundary, do normal I/O.  To get the
2455			 * right DevHandle to use, get the map number for the
2456			 * column, then use that map number to look up the
2457			 * DevHandle of the PhysDisk.
2458			 */
2459			stripe_offset = (uint32_t)virtLBA &
2460			    (sc->DD_stripe_size - 1);
2461			if ((stripe_offset + io_size) <= sc->DD_stripe_size) {
2462				physLBA = (uint32_t)virtLBA >>
2463				    sc->DD_stripe_exponent;
2464				stripe_unit = physLBA / sc->DD_num_phys_disks;
2465				column = physLBA % sc->DD_num_phys_disks;
2466				pIO_req->DevHandle =
2467				    htole16(sc->DD_column_map[column].dev_handle);
2468				/* ???? Is this endian safe*/
2469				cm->cm_desc.SCSIIO.DevHandle =
2470				    pIO_req->DevHandle;
2471
2472				physLBA = (stripe_unit <<
2473				    sc->DD_stripe_exponent) + stripe_offset;
2474				ptrLBA = &pIO_req->CDB.CDB32[1];
2475				physLBA_byte = (uint8_t)(physLBA >> 16);
2476				*ptrLBA = physLBA_byte;
2477				ptrLBA = &pIO_req->CDB.CDB32[2];
2478				physLBA_byte = (uint8_t)(physLBA >> 8);
2479				*ptrLBA = physLBA_byte;
2480				ptrLBA = &pIO_req->CDB.CDB32[3];
2481				physLBA_byte = (uint8_t)physLBA;
2482				*ptrLBA = physLBA_byte;
2483
2484				/*
2485				 * Set flag that Direct Drive I/O is
2486				 * being done.
2487				 */
2488				cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2489			}
2490		}
2491		return;
2492	}
2493
2494	/*
2495	 * Handle 10, 12 or 16 byte CDBs.
2496	 */
2497	if ((pIO_req->DevHandle == sc->DD_dev_handle) && ((CDB[0] == READ_10) ||
2498	    (CDB[0] == WRITE_10) || (CDB[0] == READ_16) ||
2499	    (CDB[0] == WRITE_16) || (CDB[0] == READ_12) ||
2500	    (CDB[0] == WRITE_12))) {
2501		/*
2502		 * For 16-byte CDB's, verify that the upper 4 bytes of the CDB
2503		 * are 0.  If not, this is accessing beyond 2TB so handle it in
2504		 * the else section.  10-byte and 12-byte CDB's are OK.
2505		 * FreeBSD sends very rare 12 byte READ/WRITE, but driver is
2506		 * ready to accept 12byte CDB for Direct IOs.
2507		 */
2508		if ((CDB[0] == READ_10 || CDB[0] == WRITE_10) ||
2509		    (CDB[0] == READ_12 || CDB[0] == WRITE_12) ||
2510		    !(CDB[2] | CDB[3] | CDB[4] | CDB[5])) {
2511			/*
2512			 * Get the transfer size in blocks.
2513			 */
2514			io_size = (cm->cm_length >> sc->DD_block_exponent);
2515
2516			/*
2517			 * Get virtual LBA.  Point to correct lower 4 bytes of
2518			 * LBA in the CDB depending on command.
2519			 */
2520			lba_idx = ((CDB[0] == READ_12) ||
2521				(CDB[0] == WRITE_12) ||
2522				(CDB[0] == READ_10) ||
2523				(CDB[0] == WRITE_10))? 2 : 6;
2524			virtLBA = ((uint64_t)CDB[lba_idx] << 24) |
2525			    ((uint64_t)CDB[lba_idx + 1] << 16) |
2526			    ((uint64_t)CDB[lba_idx + 2] << 8) |
2527			    (uint64_t)CDB[lba_idx + 3];
2528
2529			/*
2530			 * Check that LBA range for I/O does not exceed volume's
2531			 * MaxLBA.
2532			 */
2533			if ((virtLBA + (uint64_t)io_size - 1) <=
2534			    sc->DD_max_lba) {
2535				/*
2536				 * Check if the I/O crosses a stripe boundary.
2537				 * If not, translate the virtual LBA to a
2538				 * physical LBA and set the DevHandle for the
2539				 * PhysDisk to be used.  If it does cross a
2540				 * boundary, do normal I/O.  To get the right
2541				 * DevHandle to use, get the map number for the
2542				 * column, then use that map number to look up
2543				 * the DevHandle of the PhysDisk.
2544				 */
2545				stripe_offset = (uint32_t)virtLBA &
2546				    (sc->DD_stripe_size - 1);
2547				if ((stripe_offset + io_size) <=
2548				    sc->DD_stripe_size) {
2549					physLBA = (uint32_t)virtLBA >>
2550					    sc->DD_stripe_exponent;
2551					stripe_unit = physLBA /
2552					    sc->DD_num_phys_disks;
2553					column = physLBA %
2554					    sc->DD_num_phys_disks;
2555					pIO_req->DevHandle =
2556					    htole16(sc->DD_column_map[column].
2557					    dev_handle);
2558					cm->cm_desc.SCSIIO.DevHandle =
2559					    pIO_req->DevHandle;
2560
2561					physLBA = (stripe_unit <<
2562					    sc->DD_stripe_exponent) +
2563					    stripe_offset;
2564					ptrLBA =
2565					    &pIO_req->CDB.CDB32[lba_idx];
2566					physLBA_byte = (uint8_t)(physLBA >> 24);
2567					*ptrLBA = physLBA_byte;
2568					ptrLBA =
2569					    &pIO_req->CDB.CDB32[lba_idx + 1];
2570					physLBA_byte = (uint8_t)(physLBA >> 16);
2571					*ptrLBA = physLBA_byte;
2572					ptrLBA =
2573					    &pIO_req->CDB.CDB32[lba_idx + 2];
2574					physLBA_byte = (uint8_t)(physLBA >> 8);
2575					*ptrLBA = physLBA_byte;
2576					ptrLBA =
2577					    &pIO_req->CDB.CDB32[lba_idx + 3];
2578					physLBA_byte = (uint8_t)physLBA;
2579					*ptrLBA = physLBA_byte;
2580
2581					/*
2582					 * Set flag that Direct Drive I/O is
2583					 * being done.
2584					 */
2585					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2586				}
2587			}
2588		} else {
2589			/*
2590			 * 16-byte CDB and the upper 4 bytes of the CDB are not
2591			 * 0.  Get the transfer size in blocks.
2592			 */
2593			io_size = (cm->cm_length >> sc->DD_block_exponent);
2594
2595			/*
2596			 * Get virtual LBA.
2597			 */
2598			virtLBA = ((uint64_t)CDB[2] << 54) |
2599			    ((uint64_t)CDB[3] << 48) |
2600			    ((uint64_t)CDB[4] << 40) |
2601			    ((uint64_t)CDB[5] << 32) |
2602			    ((uint64_t)CDB[6] << 24) |
2603			    ((uint64_t)CDB[7] << 16) |
2604			    ((uint64_t)CDB[8] << 8) |
2605			    (uint64_t)CDB[9];
2606
2607			/*
2608			 * Check that LBA range for I/O does not exceed volume's
2609			 * MaxLBA.
2610			 */
2611			if ((virtLBA + (uint64_t)io_size - 1) <=
2612			    sc->DD_max_lba) {
2613				/*
2614				 * Check if the I/O crosses a stripe boundary.
2615				 * If not, translate the virtual LBA to a
2616				 * physical LBA and set the DevHandle for the
2617				 * PhysDisk to be used.  If it does cross a
2618				 * boundary, do normal I/O.  To get the right
2619				 * DevHandle to use, get the map number for the
2620				 * column, then use that map number to look up
2621				 * the DevHandle of the PhysDisk.
2622				 */
2623				stripe_offset = (uint32_t)virtLBA &
2624				    (sc->DD_stripe_size - 1);
2625				if ((stripe_offset + io_size) <=
2626				    sc->DD_stripe_size) {
2627					physLBA = (uint32_t)(virtLBA >>
2628					    sc->DD_stripe_exponent);
2629					stripe_unit = physLBA /
2630					    sc->DD_num_phys_disks;
2631					column = physLBA %
2632					    sc->DD_num_phys_disks;
2633					pIO_req->DevHandle =
2634					    htole16(sc->DD_column_map[column].
2635					    dev_handle);
2636					cm->cm_desc.SCSIIO.DevHandle =
2637					    pIO_req->DevHandle;
2638
2639					physLBA = (stripe_unit <<
2640					    sc->DD_stripe_exponent) +
2641					    stripe_offset;
2642
2643					/*
2644					 * Set upper 4 bytes of LBA to 0.  We
2645					 * assume that the phys disks are less
2646					 * than 2 TB's in size.  Then, set the
2647					 * lower 4 bytes.
2648					 */
2649					pIO_req->CDB.CDB32[2] = 0;
2650					pIO_req->CDB.CDB32[3] = 0;
2651					pIO_req->CDB.CDB32[4] = 0;
2652					pIO_req->CDB.CDB32[5] = 0;
2653					ptrLBA = &pIO_req->CDB.CDB32[6];
2654					physLBA_byte = (uint8_t)(physLBA >> 24);
2655					*ptrLBA = physLBA_byte;
2656					ptrLBA = &pIO_req->CDB.CDB32[7];
2657					physLBA_byte = (uint8_t)(physLBA >> 16);
2658					*ptrLBA = physLBA_byte;
2659					ptrLBA = &pIO_req->CDB.CDB32[8];
2660					physLBA_byte = (uint8_t)(physLBA >> 8);
2661					*ptrLBA = physLBA_byte;
2662					ptrLBA = &pIO_req->CDB.CDB32[9];
2663					physLBA_byte = (uint8_t)physLBA;
2664					*ptrLBA = physLBA_byte;
2665
2666					/*
2667					 * Set flag that Direct Drive I/O is
2668					 * being done.
2669					 */
2670					cm->cm_flags |= MPS_CM_FLAGS_DD_IO;
2671				}
2672			}
2673		}
2674	}
2675}
2676
2677#if __FreeBSD_version >= 900026
2678static void
2679mpssas_smpio_complete(struct mps_softc *sc, struct mps_command *cm)
2680{
2681	MPI2_SMP_PASSTHROUGH_REPLY *rpl;
2682	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2683	uint64_t sasaddr;
2684	union ccb *ccb;
2685
2686	ccb = cm->cm_complete_data;
2687
2688	/*
2689	 * Currently there should be no way we can hit this case.  It only
2690	 * happens when we have a failure to allocate chain frames, and SMP
2691	 * commands require two S/G elements only.  That should be handled
2692	 * in the standard request size.
2693	 */
2694	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
2695		mps_dprint(sc, MPS_ERROR,"%s: cm_flags = %#x on SMP request!\n",
2696			   __func__, cm->cm_flags);
2697		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2698		goto bailout;
2699        }
2700
2701	rpl = (MPI2_SMP_PASSTHROUGH_REPLY *)cm->cm_reply;
2702	if (rpl == NULL) {
2703		mps_dprint(sc, MPS_ERROR, "%s: NULL cm_reply!\n", __func__);
2704		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2705		goto bailout;
2706	}
2707
2708	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2709	sasaddr = le32toh(req->SASAddress.Low);
2710	sasaddr |= ((uint64_t)(le32toh(req->SASAddress.High))) << 32;
2711
2712	if ((le16toh(rpl->IOCStatus) & MPI2_IOCSTATUS_MASK) !=
2713	    MPI2_IOCSTATUS_SUCCESS ||
2714	    rpl->SASStatus != MPI2_SASSTATUS_SUCCESS) {
2715		mps_dprint(sc, MPS_XINFO, "%s: IOCStatus %04x SASStatus %02x\n",
2716		    __func__, le16toh(rpl->IOCStatus), rpl->SASStatus);
2717		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
2718		goto bailout;
2719	}
2720
2721	mps_dprint(sc, MPS_XINFO, "%s: SMP request to SAS address "
2722		   "%#jx completed successfully\n", __func__,
2723		   (uintmax_t)sasaddr);
2724
2725	if (ccb->smpio.smp_response[2] == SMP_FR_ACCEPTED)
2726		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
2727	else
2728		mpssas_set_ccbstatus(ccb, CAM_SMP_STATUS_ERROR);
2729
2730bailout:
2731	/*
2732	 * We sync in both directions because we had DMAs in the S/G list
2733	 * in both directions.
2734	 */
2735	bus_dmamap_sync(sc->buffer_dmat, cm->cm_dmamap,
2736			BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
2737	bus_dmamap_unload(sc->buffer_dmat, cm->cm_dmamap);
2738	mps_free_command(sc, cm);
2739	xpt_done(ccb);
2740}
2741
2742static void
2743mpssas_send_smpcmd(struct mpssas_softc *sassc, union ccb *ccb, uint64_t sasaddr)
2744{
2745	struct mps_command *cm;
2746	uint8_t *request, *response;
2747	MPI2_SMP_PASSTHROUGH_REQUEST *req;
2748	struct mps_softc *sc;
2749	int error;
2750
2751	sc = sassc->sc;
2752	error = 0;
2753
2754	/*
2755	 * XXX We don't yet support physical addresses here.
2756	 */
2757	switch ((ccb->ccb_h.flags & CAM_DATA_MASK)) {
2758	case CAM_DATA_PADDR:
2759	case CAM_DATA_SG_PADDR:
2760		mps_dprint(sc, MPS_ERROR,
2761			   "%s: physical addresses not supported\n", __func__);
2762		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2763		xpt_done(ccb);
2764		return;
2765	case CAM_DATA_SG:
2766		/*
2767		 * The chip does not support more than one buffer for the
2768		 * request or response.
2769		 */
2770	 	if ((ccb->smpio.smp_request_sglist_cnt > 1)
2771		  || (ccb->smpio.smp_response_sglist_cnt > 1)) {
2772			mps_dprint(sc, MPS_ERROR,
2773				   "%s: multiple request or response "
2774				   "buffer segments not supported for SMP\n",
2775				   __func__);
2776			mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2777			xpt_done(ccb);
2778			return;
2779		}
2780
2781		/*
2782		 * The CAM_SCATTER_VALID flag was originally implemented
2783		 * for the XPT_SCSI_IO CCB, which only has one data pointer.
2784		 * We have two.  So, just take that flag to mean that we
2785		 * might have S/G lists, and look at the S/G segment count
2786		 * to figure out whether that is the case for each individual
2787		 * buffer.
2788		 */
2789		if (ccb->smpio.smp_request_sglist_cnt != 0) {
2790			bus_dma_segment_t *req_sg;
2791
2792			req_sg = (bus_dma_segment_t *)ccb->smpio.smp_request;
2793			request = (uint8_t *)(uintptr_t)req_sg[0].ds_addr;
2794		} else
2795			request = ccb->smpio.smp_request;
2796
2797		if (ccb->smpio.smp_response_sglist_cnt != 0) {
2798			bus_dma_segment_t *rsp_sg;
2799
2800			rsp_sg = (bus_dma_segment_t *)ccb->smpio.smp_response;
2801			response = (uint8_t *)(uintptr_t)rsp_sg[0].ds_addr;
2802		} else
2803			response = ccb->smpio.smp_response;
2804		break;
2805	case CAM_DATA_VADDR:
2806		request = ccb->smpio.smp_request;
2807		response = ccb->smpio.smp_response;
2808		break;
2809	default:
2810		mpssas_set_ccbstatus(ccb, CAM_REQ_INVALID);
2811		xpt_done(ccb);
2812		return;
2813	}
2814
2815	cm = mps_alloc_command(sc);
2816	if (cm == NULL) {
2817		mps_dprint(sc, MPS_ERROR,
2818		    "%s: cannot allocate command\n", __func__);
2819		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2820		xpt_done(ccb);
2821		return;
2822	}
2823
2824	req = (MPI2_SMP_PASSTHROUGH_REQUEST *)cm->cm_req;
2825	bzero(req, sizeof(*req));
2826	req->Function = MPI2_FUNCTION_SMP_PASSTHROUGH;
2827
2828	/* Allow the chip to use any route to this SAS address. */
2829	req->PhysicalPort = 0xff;
2830
2831	req->RequestDataLength = htole16(ccb->smpio.smp_request_len);
2832	req->SGLFlags =
2833	    MPI2_SGLFLAGS_SYSTEM_ADDRESS_SPACE | MPI2_SGLFLAGS_SGL_TYPE_MPI;
2834
2835	mps_dprint(sc, MPS_XINFO, "%s: sending SMP request to SAS "
2836	    "address %#jx\n", __func__, (uintmax_t)sasaddr);
2837
2838	mpi_init_sge(cm, req, &req->SGL);
2839
2840	/*
2841	 * Set up a uio to pass into mps_map_command().  This allows us to
2842	 * do one map command, and one busdma call in there.
2843	 */
2844	cm->cm_uio.uio_iov = cm->cm_iovec;
2845	cm->cm_uio.uio_iovcnt = 2;
2846	cm->cm_uio.uio_segflg = UIO_SYSSPACE;
2847
2848	/*
2849	 * The read/write flag isn't used by busdma, but set it just in
2850	 * case.  This isn't exactly accurate, either, since we're going in
2851	 * both directions.
2852	 */
2853	cm->cm_uio.uio_rw = UIO_WRITE;
2854
2855	cm->cm_iovec[0].iov_base = request;
2856	cm->cm_iovec[0].iov_len = le16toh(req->RequestDataLength);
2857	cm->cm_iovec[1].iov_base = response;
2858	cm->cm_iovec[1].iov_len = ccb->smpio.smp_response_len;
2859
2860	cm->cm_uio.uio_resid = cm->cm_iovec[0].iov_len +
2861			       cm->cm_iovec[1].iov_len;
2862
2863	/*
2864	 * Trigger a warning message in mps_data_cb() for the user if we
2865	 * wind up exceeding two S/G segments.  The chip expects one
2866	 * segment for the request and another for the response.
2867	 */
2868	cm->cm_max_segs = 2;
2869
2870	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
2871	cm->cm_complete = mpssas_smpio_complete;
2872	cm->cm_complete_data = ccb;
2873
2874	/*
2875	 * Tell the mapping code that we're using a uio, and that this is
2876	 * an SMP passthrough request.  There is a little special-case
2877	 * logic there (in mps_data_cb()) to handle the bidirectional
2878	 * transfer.
2879	 */
2880	cm->cm_flags |= MPS_CM_FLAGS_USE_UIO | MPS_CM_FLAGS_SMP_PASS |
2881			MPS_CM_FLAGS_DATAIN | MPS_CM_FLAGS_DATAOUT;
2882
2883	/* The chip data format is little endian. */
2884	req->SASAddress.High = htole32(sasaddr >> 32);
2885	req->SASAddress.Low = htole32(sasaddr);
2886
2887	/*
2888	 * XXX Note that we don't have a timeout/abort mechanism here.
2889	 * From the manual, it looks like task management requests only
2890	 * work for SCSI IO and SATA passthrough requests.  We may need to
2891	 * have a mechanism to retry requests in the event of a chip reset
2892	 * at least.  Hopefully the chip will insure that any errors short
2893	 * of that are relayed back to the driver.
2894	 */
2895	error = mps_map_command(sc, cm);
2896	if ((error != 0) && (error != EINPROGRESS)) {
2897		mps_dprint(sc, MPS_ERROR,
2898			   "%s: error %d returned from mps_map_command()\n",
2899			   __func__, error);
2900		goto bailout_error;
2901	}
2902
2903	return;
2904
2905bailout_error:
2906	mps_free_command(sc, cm);
2907	mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
2908	xpt_done(ccb);
2909	return;
2910
2911}
2912
2913static void
2914mpssas_action_smpio(struct mpssas_softc *sassc, union ccb *ccb)
2915{
2916	struct mps_softc *sc;
2917	struct mpssas_target *targ;
2918	uint64_t sasaddr = 0;
2919
2920	sc = sassc->sc;
2921
2922	/*
2923	 * Make sure the target exists.
2924	 */
2925	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
2926	    ("Target %d out of bounds in XPT_SMP_IO\n", ccb->ccb_h.target_id));
2927	targ = &sassc->targets[ccb->ccb_h.target_id];
2928	if (targ->handle == 0x0) {
2929		mps_dprint(sc, MPS_ERROR,
2930			   "%s: target %d does not exist!\n", __func__,
2931			   ccb->ccb_h.target_id);
2932		mpssas_set_ccbstatus(ccb, CAM_SEL_TIMEOUT);
2933		xpt_done(ccb);
2934		return;
2935	}
2936
2937	/*
2938	 * If this device has an embedded SMP target, we'll talk to it
2939	 * directly.
2940	 * figure out what the expander's address is.
2941	 */
2942	if ((targ->devinfo & MPI2_SAS_DEVICE_INFO_SMP_TARGET) != 0)
2943		sasaddr = targ->sasaddr;
2944
2945	/*
2946	 * If we don't have a SAS address for the expander yet, try
2947	 * grabbing it from the page 0x83 information cached in the
2948	 * transport layer for this target.  LSI expanders report the
2949	 * expander SAS address as the port-associated SAS address in
2950	 * Inquiry VPD page 0x83.  Maxim expanders don't report it in page
2951	 * 0x83.
2952	 *
2953	 * XXX KDM disable this for now, but leave it commented out so that
2954	 * it is obvious that this is another possible way to get the SAS
2955	 * address.
2956	 *
2957	 * The parent handle method below is a little more reliable, and
2958	 * the other benefit is that it works for devices other than SES
2959	 * devices.  So you can send a SMP request to a da(4) device and it
2960	 * will get routed to the expander that device is attached to.
2961	 * (Assuming the da(4) device doesn't contain an SMP target...)
2962	 */
2963#if 0
2964	if (sasaddr == 0)
2965		sasaddr = xpt_path_sas_addr(ccb->ccb_h.path);
2966#endif
2967
2968	/*
2969	 * If we still don't have a SAS address for the expander, look for
2970	 * the parent device of this device, which is probably the expander.
2971	 */
2972	if (sasaddr == 0) {
2973#ifdef OLD_MPS_PROBE
2974		struct mpssas_target *parent_target;
2975#endif
2976
2977		if (targ->parent_handle == 0x0) {
2978			mps_dprint(sc, MPS_ERROR,
2979				   "%s: handle %d does not have a valid "
2980				   "parent handle!\n", __func__, targ->handle);
2981			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2982			goto bailout;
2983		}
2984#ifdef OLD_MPS_PROBE
2985		parent_target = mpssas_find_target_by_handle(sassc, 0,
2986			targ->parent_handle);
2987
2988		if (parent_target == NULL) {
2989			mps_dprint(sc, MPS_ERROR,
2990				   "%s: handle %d does not have a valid "
2991				   "parent target!\n", __func__, targ->handle);
2992			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
2993			goto bailout;
2994		}
2995
2996		if ((parent_target->devinfo &
2997		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
2998			mps_dprint(sc, MPS_ERROR,
2999				   "%s: handle %d parent %d does not "
3000				   "have an SMP target!\n", __func__,
3001				   targ->handle, parent_target->handle);
3002			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3003			goto bailout;
3004
3005		}
3006
3007		sasaddr = parent_target->sasaddr;
3008#else /* OLD_MPS_PROBE */
3009		if ((targ->parent_devinfo &
3010		     MPI2_SAS_DEVICE_INFO_SMP_TARGET) == 0) {
3011			mps_dprint(sc, MPS_ERROR,
3012				   "%s: handle %d parent %d does not "
3013				   "have an SMP target!\n", __func__,
3014				   targ->handle, targ->parent_handle);
3015			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3016			goto bailout;
3017
3018		}
3019		if (targ->parent_sasaddr == 0x0) {
3020			mps_dprint(sc, MPS_ERROR,
3021				   "%s: handle %d parent handle %d does "
3022				   "not have a valid SAS address!\n",
3023				   __func__, targ->handle, targ->parent_handle);
3024			mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3025			goto bailout;
3026		}
3027
3028		sasaddr = targ->parent_sasaddr;
3029#endif /* OLD_MPS_PROBE */
3030
3031	}
3032
3033	if (sasaddr == 0) {
3034		mps_dprint(sc, MPS_INFO,
3035			   "%s: unable to find SAS address for handle %d\n",
3036			   __func__, targ->handle);
3037		mpssas_set_ccbstatus(ccb, CAM_DEV_NOT_THERE);
3038		goto bailout;
3039	}
3040	mpssas_send_smpcmd(sassc, ccb, sasaddr);
3041
3042	return;
3043
3044bailout:
3045	xpt_done(ccb);
3046
3047}
3048#endif //__FreeBSD_version >= 900026
3049
3050static void
3051mpssas_action_resetdev(struct mpssas_softc *sassc, union ccb *ccb)
3052{
3053	MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3054	struct mps_softc *sc;
3055	struct mps_command *tm;
3056	struct mpssas_target *targ;
3057
3058	MPS_FUNCTRACE(sassc->sc);
3059	mtx_assert(&sassc->sc->mps_mtx, MA_OWNED);
3060
3061	KASSERT(ccb->ccb_h.target_id < sassc->maxtargets,
3062	    ("Target %d out of bounds in XPT_RESET_DEV\n",
3063	     ccb->ccb_h.target_id));
3064	sc = sassc->sc;
3065	tm = mpssas_alloc_tm(sc);
3066	if (tm == NULL) {
3067		mps_dprint(sc, MPS_ERROR,
3068		    "command alloc failure in mpssas_action_resetdev\n");
3069		mpssas_set_ccbstatus(ccb, CAM_RESRC_UNAVAIL);
3070		xpt_done(ccb);
3071		return;
3072	}
3073
3074	targ = &sassc->targets[ccb->ccb_h.target_id];
3075	req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3076	req->DevHandle = htole16(targ->handle);
3077	req->TaskType = MPI2_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
3078
3079	/* SAS Hard Link Reset / SATA Link Reset */
3080	req->MsgFlags = MPI2_SCSITASKMGMT_MSGFLAGS_LINK_RESET;
3081
3082	tm->cm_data = NULL;
3083	tm->cm_complete = mpssas_resetdev_complete;
3084	tm->cm_complete_data = ccb;
3085	tm->cm_targ = targ;
3086
3087	mpssas_prepare_for_tm(sc, tm, targ, CAM_LUN_WILDCARD);
3088	mps_map_command(sc, tm);
3089}
3090
3091static void
3092mpssas_resetdev_complete(struct mps_softc *sc, struct mps_command *tm)
3093{
3094	MPI2_SCSI_TASK_MANAGE_REPLY *resp;
3095	union ccb *ccb;
3096
3097	MPS_FUNCTRACE(sc);
3098	mtx_assert(&sc->mps_mtx, MA_OWNED);
3099
3100	resp = (MPI2_SCSI_TASK_MANAGE_REPLY *)tm->cm_reply;
3101	ccb = tm->cm_complete_data;
3102
3103	/*
3104	 * Currently there should be no way we can hit this case.  It only
3105	 * happens when we have a failure to allocate chain frames, and
3106	 * task management commands don't have S/G lists.
3107	 */
3108	if ((tm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3109		MPI2_SCSI_TASK_MANAGE_REQUEST *req;
3110
3111		req = (MPI2_SCSI_TASK_MANAGE_REQUEST *)tm->cm_req;
3112
3113		mps_dprint(sc, MPS_ERROR,
3114			   "%s: cm_flags = %#x for reset of handle %#04x! "
3115			   "This should not happen!\n", __func__, tm->cm_flags,
3116			   req->DevHandle);
3117		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3118		goto bailout;
3119	}
3120
3121	mps_dprint(sc, MPS_XINFO,
3122	    "%s: IOCStatus = 0x%x ResponseCode = 0x%x\n", __func__,
3123	    le16toh(resp->IOCStatus), le32toh(resp->ResponseCode));
3124
3125	if (le32toh(resp->ResponseCode) == MPI2_SCSITASKMGMT_RSP_TM_COMPLETE) {
3126		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP);
3127		mpssas_announce_reset(sc, AC_SENT_BDR, tm->cm_targ->tid,
3128		    CAM_LUN_WILDCARD);
3129	}
3130	else
3131		mpssas_set_ccbstatus(ccb, CAM_REQ_CMP_ERR);
3132
3133bailout:
3134
3135	mpssas_free_tm(sc, tm);
3136	xpt_done(ccb);
3137}
3138
3139static void
3140mpssas_poll(struct cam_sim *sim)
3141{
3142	struct mpssas_softc *sassc;
3143
3144	sassc = cam_sim_softc(sim);
3145
3146	if (sassc->sc->mps_debug & MPS_TRACE) {
3147		/* frequent debug messages during a panic just slow
3148		 * everything down too much.
3149		 */
3150		mps_printf(sassc->sc, "%s clearing MPS_TRACE\n", __func__);
3151		sassc->sc->mps_debug &= ~MPS_TRACE;
3152	}
3153
3154	mps_intr_locked(sassc->sc);
3155}
3156
3157static void
3158mpssas_async(void *callback_arg, uint32_t code, struct cam_path *path,
3159	     void *arg)
3160{
3161	struct mps_softc *sc;
3162
3163	sc = (struct mps_softc *)callback_arg;
3164
3165	switch (code) {
3166#if (__FreeBSD_version >= 1000006) || \
3167    ((__FreeBSD_version >= 901503) && (__FreeBSD_version < 1000000))
3168	case AC_ADVINFO_CHANGED: {
3169		struct mpssas_target *target;
3170		struct mpssas_softc *sassc;
3171		struct scsi_read_capacity_data_long rcap_buf;
3172		struct ccb_dev_advinfo cdai;
3173		struct mpssas_lun *lun;
3174		lun_id_t lunid;
3175		int found_lun;
3176		uintptr_t buftype;
3177
3178		buftype = (uintptr_t)arg;
3179
3180		found_lun = 0;
3181		sassc = sc->sassc;
3182
3183		/*
3184		 * We're only interested in read capacity data changes.
3185		 */
3186		if (buftype != CDAI_TYPE_RCAPLONG)
3187			break;
3188
3189		/*
3190		 * We should have a handle for this, but check to make sure.
3191		 */
3192		KASSERT(xpt_path_target_id(path) < sassc->maxtargets,
3193		    ("Target %d out of bounds in mpssas_async\n",
3194		    xpt_path_target_id(path)));
3195		target = &sassc->targets[xpt_path_target_id(path)];
3196		if (target->handle == 0)
3197			break;
3198
3199		lunid = xpt_path_lun_id(path);
3200
3201		SLIST_FOREACH(lun, &target->luns, lun_link) {
3202			if (lun->lun_id == lunid) {
3203				found_lun = 1;
3204				break;
3205			}
3206		}
3207
3208		if (found_lun == 0) {
3209			lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3210				     M_NOWAIT | M_ZERO);
3211			if (lun == NULL) {
3212				mps_dprint(sc, MPS_ERROR, "Unable to alloc "
3213					   "LUN for EEDP support.\n");
3214				break;
3215			}
3216			lun->lun_id = lunid;
3217			SLIST_INSERT_HEAD(&target->luns, lun, lun_link);
3218		}
3219
3220		bzero(&rcap_buf, sizeof(rcap_buf));
3221		xpt_setup_ccb(&cdai.ccb_h, path, CAM_PRIORITY_NORMAL);
3222		cdai.ccb_h.func_code = XPT_DEV_ADVINFO;
3223		cdai.ccb_h.flags = CAM_DIR_IN;
3224		cdai.buftype = CDAI_TYPE_RCAPLONG;
3225#if (__FreeBSD_version >= 1100061) || \
3226    ((__FreeBSD_version >= 1001510) && (__FreeBSD_version < 1100000))
3227		cdai.flags = CDAI_FLAG_NONE;
3228#else
3229		cdai.flags = 0;
3230#endif
3231		cdai.bufsiz = sizeof(rcap_buf);
3232		cdai.buf = (uint8_t *)&rcap_buf;
3233		xpt_action((union ccb *)&cdai);
3234		if ((cdai.ccb_h.status & CAM_DEV_QFRZN) != 0)
3235			cam_release_devq(cdai.ccb_h.path,
3236					 0, 0, 0, FALSE);
3237
3238		if ((mpssas_get_ccbstatus((union ccb *)&cdai) == CAM_REQ_CMP)
3239		 && (rcap_buf.prot & SRC16_PROT_EN)) {
3240			switch (rcap_buf.prot & SRC16_P_TYPE) {
3241			case SRC16_PTYPE_1:
3242			case SRC16_PTYPE_3:
3243				lun->eedp_formatted = TRUE;
3244				lun->eedp_block_size =
3245				    scsi_4btoul(rcap_buf.length);
3246				break;
3247			case SRC16_PTYPE_2:
3248			default:
3249				lun->eedp_formatted = FALSE;
3250				lun->eedp_block_size = 0;
3251				break;
3252			}
3253		} else {
3254			lun->eedp_formatted = FALSE;
3255			lun->eedp_block_size = 0;
3256		}
3257		break;
3258	}
3259#else
3260	case AC_FOUND_DEVICE: {
3261		struct ccb_getdev *cgd;
3262
3263		cgd = arg;
3264		mpssas_check_eedp(sc, path, cgd);
3265		break;
3266	}
3267#endif
3268	default:
3269		break;
3270	}
3271}
3272
3273#if (__FreeBSD_version < 901503) || \
3274    ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006))
3275static void
3276mpssas_check_eedp(struct mps_softc *sc, struct cam_path *path,
3277		  struct ccb_getdev *cgd)
3278{
3279	struct mpssas_softc *sassc = sc->sassc;
3280	struct ccb_scsiio *csio;
3281	struct scsi_read_capacity_16 *scsi_cmd;
3282	struct scsi_read_capacity_eedp *rcap_buf;
3283	path_id_t pathid;
3284	target_id_t targetid;
3285	lun_id_t lunid;
3286	union ccb *ccb;
3287	struct cam_path *local_path;
3288	struct mpssas_target *target;
3289	struct mpssas_lun *lun;
3290	uint8_t	found_lun;
3291	char path_str[64];
3292
3293	sassc = sc->sassc;
3294	pathid = cam_sim_path(sassc->sim);
3295	targetid = xpt_path_target_id(path);
3296	lunid = xpt_path_lun_id(path);
3297
3298	KASSERT(targetid < sassc->maxtargets,
3299	    ("Target %d out of bounds in mpssas_check_eedp\n",
3300	     targetid));
3301	target = &sassc->targets[targetid];
3302	if (target->handle == 0x0)
3303		return;
3304
3305	/*
3306	 * Determine if the device is EEDP capable.
3307	 *
3308	 * If this flag is set in the inquiry data,
3309	 * the device supports protection information,
3310	 * and must support the 16 byte read
3311	 * capacity command, otherwise continue without
3312	 * sending read cap 16
3313	 */
3314	if ((cgd->inq_data.spc3_flags & SPC3_SID_PROTECT) == 0)
3315		return;
3316
3317	/*
3318	 * Issue a READ CAPACITY 16 command.  This info
3319	 * is used to determine if the LUN is formatted
3320	 * for EEDP support.
3321	 */
3322	ccb = xpt_alloc_ccb_nowait();
3323	if (ccb == NULL) {
3324		mps_dprint(sc, MPS_ERROR, "Unable to alloc CCB "
3325		    "for EEDP support.\n");
3326		return;
3327	}
3328
3329	if (xpt_create_path(&local_path, xpt_periph,
3330	    pathid, targetid, lunid) != CAM_REQ_CMP) {
3331		mps_dprint(sc, MPS_ERROR, "Unable to create "
3332		    "path for EEDP support\n");
3333		xpt_free_ccb(ccb);
3334		return;
3335	}
3336
3337	/*
3338	 * If LUN is already in list, don't create a new
3339	 * one.
3340	 */
3341	found_lun = FALSE;
3342	SLIST_FOREACH(lun, &target->luns, lun_link) {
3343		if (lun->lun_id == lunid) {
3344			found_lun = TRUE;
3345			break;
3346		}
3347	}
3348	if (!found_lun) {
3349		lun = malloc(sizeof(struct mpssas_lun), M_MPT2,
3350		    M_NOWAIT | M_ZERO);
3351		if (lun == NULL) {
3352			mps_dprint(sc, MPS_ERROR,
3353			    "Unable to alloc LUN for EEDP support.\n");
3354			xpt_free_path(local_path);
3355			xpt_free_ccb(ccb);
3356			return;
3357		}
3358		lun->lun_id = lunid;
3359		SLIST_INSERT_HEAD(&target->luns, lun,
3360		    lun_link);
3361	}
3362
3363	xpt_path_string(local_path, path_str, sizeof(path_str));
3364
3365	mps_dprint(sc, MPS_INFO, "Sending read cap: path %s handle %d\n",
3366	    path_str, target->handle);
3367
3368	/*
3369	 * Issue a READ CAPACITY 16 command for the LUN.
3370	 * The mpssas_read_cap_done function will load
3371	 * the read cap info into the LUN struct.
3372	 */
3373	rcap_buf = malloc(sizeof(struct scsi_read_capacity_eedp),
3374	    M_MPT2, M_NOWAIT | M_ZERO);
3375	if (rcap_buf == NULL) {
3376		mps_dprint(sc, MPS_FAULT,
3377		    "Unable to alloc read capacity buffer for EEDP support.\n");
3378		xpt_free_path(ccb->ccb_h.path);
3379		xpt_free_ccb(ccb);
3380		return;
3381	}
3382	xpt_setup_ccb(&ccb->ccb_h, local_path, CAM_PRIORITY_XPT);
3383	csio = &ccb->csio;
3384	csio->ccb_h.func_code = XPT_SCSI_IO;
3385	csio->ccb_h.flags = CAM_DIR_IN;
3386	csio->ccb_h.retry_count = 4;
3387	csio->ccb_h.cbfcnp = mpssas_read_cap_done;
3388	csio->ccb_h.timeout = 60000;
3389	csio->data_ptr = (uint8_t *)rcap_buf;
3390	csio->dxfer_len = sizeof(struct scsi_read_capacity_eedp);
3391	csio->sense_len = MPS_SENSE_LEN;
3392	csio->cdb_len = sizeof(*scsi_cmd);
3393	csio->tag_action = MSG_SIMPLE_Q_TAG;
3394
3395	scsi_cmd = (struct scsi_read_capacity_16 *)&csio->cdb_io.cdb_bytes;
3396	bzero(scsi_cmd, sizeof(*scsi_cmd));
3397	scsi_cmd->opcode = 0x9E;
3398	scsi_cmd->service_action = SRC16_SERVICE_ACTION;
3399	((uint8_t *)scsi_cmd)[13] = sizeof(struct scsi_read_capacity_eedp);
3400
3401	ccb->ccb_h.ppriv_ptr1 = sassc;
3402	xpt_action(ccb);
3403}
3404
3405static void
3406mpssas_read_cap_done(struct cam_periph *periph, union ccb *done_ccb)
3407{
3408	struct mpssas_softc *sassc;
3409	struct mpssas_target *target;
3410	struct mpssas_lun *lun;
3411	struct scsi_read_capacity_eedp *rcap_buf;
3412
3413	if (done_ccb == NULL)
3414		return;
3415
3416	/* Driver need to release devq, it Scsi command is
3417	 * generated by driver internally.
3418	 * Currently there is a single place where driver
3419	 * calls scsi command internally. In future if driver
3420	 * calls more scsi command internally, it needs to release
3421	 * devq internally, since those command will not go back to
3422	 * cam_periph.
3423	 */
3424	if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) ) {
3425        	done_ccb->ccb_h.status &= ~CAM_DEV_QFRZN;
3426		xpt_release_devq(done_ccb->ccb_h.path,
3427			       	/*count*/ 1, /*run_queue*/TRUE);
3428	}
3429
3430	rcap_buf = (struct scsi_read_capacity_eedp *)done_ccb->csio.data_ptr;
3431
3432	/*
3433	 * Get the LUN ID for the path and look it up in the LUN list for the
3434	 * target.
3435	 */
3436	sassc = (struct mpssas_softc *)done_ccb->ccb_h.ppriv_ptr1;
3437	KASSERT(done_ccb->ccb_h.target_id < sassc->maxtargets,
3438	    ("Target %d out of bounds in mpssas_read_cap_done\n",
3439	     done_ccb->ccb_h.target_id));
3440	target = &sassc->targets[done_ccb->ccb_h.target_id];
3441	SLIST_FOREACH(lun, &target->luns, lun_link) {
3442		if (lun->lun_id != done_ccb->ccb_h.target_lun)
3443			continue;
3444
3445		/*
3446		 * Got the LUN in the target's LUN list.  Fill it in
3447		 * with EEDP info.  If the READ CAP 16 command had some
3448		 * SCSI error (common if command is not supported), mark
3449		 * the lun as not supporting EEDP and set the block size
3450		 * to 0.
3451		 */
3452		if ((mpssas_get_ccbstatus(done_ccb) != CAM_REQ_CMP)
3453		 || (done_ccb->csio.scsi_status != SCSI_STATUS_OK)) {
3454			lun->eedp_formatted = FALSE;
3455			lun->eedp_block_size = 0;
3456			break;
3457		}
3458
3459		if (rcap_buf->protect & 0x01) {
3460			mps_dprint(sassc->sc, MPS_INFO, "LUN %d for "
3461 			    "target ID %d is formatted for EEDP "
3462 			    "support.\n", done_ccb->ccb_h.target_lun,
3463 			    done_ccb->ccb_h.target_id);
3464			lun->eedp_formatted = TRUE;
3465			lun->eedp_block_size = scsi_4btoul(rcap_buf->length);
3466		}
3467		break;
3468	}
3469
3470	// Finished with this CCB and path.
3471	free(rcap_buf, M_MPT2);
3472	xpt_free_path(done_ccb->ccb_h.path);
3473	xpt_free_ccb(done_ccb);
3474}
3475#endif /* (__FreeBSD_version < 901503) || \
3476          ((__FreeBSD_version >= 1000000) && (__FreeBSD_version < 1000006)) */
3477
3478/*
3479 * Set the INRESET flag for this target so that no I/O will be sent to
3480 * the target until the reset has completed.  If an I/O request does
3481 * happen, the devq will be frozen.  The CCB holds the path which is
3482 * used to release the devq.  The devq is released and the CCB is freed
3483 * when the TM completes.
3484 */
3485void
3486mpssas_prepare_for_tm(struct mps_softc *sc, struct mps_command *tm,
3487    struct mpssas_target *target, lun_id_t lun_id)
3488{
3489	union ccb *ccb;
3490	path_id_t path_id;
3491
3492	ccb = xpt_alloc_ccb_nowait();
3493	if (ccb) {
3494		path_id = cam_sim_path(sc->sassc->sim);
3495		if (xpt_create_path(&ccb->ccb_h.path, xpt_periph, path_id,
3496		    target->tid, lun_id) != CAM_REQ_CMP) {
3497			xpt_free_ccb(ccb);
3498		} else {
3499			tm->cm_ccb = ccb;
3500			tm->cm_targ = target;
3501			target->flags |= MPSSAS_TARGET_INRESET;
3502		}
3503	}
3504}
3505
3506int
3507mpssas_startup(struct mps_softc *sc)
3508{
3509
3510	/*
3511	 * Send the port enable message and set the wait_for_port_enable flag.
3512	 * This flag helps to keep the simq frozen until all discovery events
3513	 * are processed.
3514	 */
3515	sc->wait_for_port_enable = 1;
3516	mpssas_send_portenable(sc);
3517	return (0);
3518}
3519
3520static int
3521mpssas_send_portenable(struct mps_softc *sc)
3522{
3523	MPI2_PORT_ENABLE_REQUEST *request;
3524	struct mps_command *cm;
3525
3526	MPS_FUNCTRACE(sc);
3527
3528	if ((cm = mps_alloc_command(sc)) == NULL)
3529		return (EBUSY);
3530	request = (MPI2_PORT_ENABLE_REQUEST *)cm->cm_req;
3531	request->Function = MPI2_FUNCTION_PORT_ENABLE;
3532	request->MsgFlags = 0;
3533	request->VP_ID = 0;
3534	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
3535	cm->cm_complete = mpssas_portenable_complete;
3536	cm->cm_data = NULL;
3537	cm->cm_sge = NULL;
3538
3539	mps_map_command(sc, cm);
3540	mps_dprint(sc, MPS_XINFO,
3541	    "mps_send_portenable finished cm %p req %p complete %p\n",
3542	    cm, cm->cm_req, cm->cm_complete);
3543	return (0);
3544}
3545
3546static void
3547mpssas_portenable_complete(struct mps_softc *sc, struct mps_command *cm)
3548{
3549	MPI2_PORT_ENABLE_REPLY *reply;
3550	struct mpssas_softc *sassc;
3551
3552	MPS_FUNCTRACE(sc);
3553	sassc = sc->sassc;
3554
3555	/*
3556	 * Currently there should be no way we can hit this case.  It only
3557	 * happens when we have a failure to allocate chain frames, and
3558	 * port enable commands don't have S/G lists.
3559	 */
3560	if ((cm->cm_flags & MPS_CM_FLAGS_ERROR_MASK) != 0) {
3561		mps_dprint(sc, MPS_ERROR, "%s: cm_flags = %#x for port enable! "
3562			   "This should not happen!\n", __func__, cm->cm_flags);
3563	}
3564
3565	reply = (MPI2_PORT_ENABLE_REPLY *)cm->cm_reply;
3566	if (reply == NULL)
3567		mps_dprint(sc, MPS_FAULT, "Portenable NULL reply\n");
3568	else if (le16toh(reply->IOCStatus & MPI2_IOCSTATUS_MASK) !=
3569	    MPI2_IOCSTATUS_SUCCESS)
3570		mps_dprint(sc, MPS_FAULT, "Portenable failed\n");
3571
3572	mps_free_command(sc, cm);
3573
3574	/*
3575	 * Get WarpDrive info after discovery is complete but before the scan
3576	 * starts.  At this point, all devices are ready to be exposed to the
3577	 * OS.  If devices should be hidden instead, take them out of the
3578	 * 'targets' array before the scan.  The devinfo for a disk will have
3579	 * some info and a volume's will be 0.  Use that to remove disks.
3580	 */
3581	mps_wd_config_pages(sc);
3582
3583	/*
3584	 * Done waiting for port enable to complete.  Decrement the refcount.
3585	 * If refcount is 0, discovery is complete and a rescan of the bus can
3586	 * take place.  Since the simq was explicitly frozen before port
3587	 * enable, it must be explicitly released here to keep the
3588	 * freeze/release count in sync.
3589	 */
3590	sc->wait_for_port_enable = 0;
3591	sc->port_enable_complete = 1;
3592	wakeup(&sc->port_enable_complete);
3593	mpssas_startup_decrement(sassc);
3594}
3595
3596int
3597mpssas_check_id(struct mpssas_softc *sassc, int id)
3598{
3599	struct mps_softc *sc = sassc->sc;
3600	char *ids;
3601	char *name;
3602
3603	ids = &sc->exclude_ids[0];
3604	while((name = strsep(&ids, ",")) != NULL) {
3605		if (name[0] == '\0')
3606			continue;
3607		if (strtol(name, NULL, 0) == (long)id)
3608			return (1);
3609	}
3610
3611	return (0);
3612}
3613
3614void
3615mpssas_realloc_targets(struct mps_softc *sc, int maxtargets)
3616{
3617	struct mpssas_softc *sassc;
3618	struct mpssas_lun *lun, *lun_tmp;
3619	struct mpssas_target *targ;
3620	int i;
3621
3622	sassc = sc->sassc;
3623	/*
3624	 * The number of targets is based on IOC Facts, so free all of
3625	 * the allocated LUNs for each target and then the target buffer
3626	 * itself.
3627	 */
3628	for (i=0; i< maxtargets; i++) {
3629		targ = &sassc->targets[i];
3630		SLIST_FOREACH_SAFE(lun, &targ->luns, lun_link, lun_tmp) {
3631			free(lun, M_MPT2);
3632		}
3633	}
3634	free(sassc->targets, M_MPT2);
3635
3636	sassc->targets = malloc(sizeof(struct mpssas_target) * maxtargets,
3637	    M_MPT2, M_WAITOK|M_ZERO);
3638	if (!sassc->targets) {
3639		panic("%s failed to alloc targets with error %d\n",
3640		    __func__, ENOMEM);
3641	}
3642}
3643