1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright 2018 Nexenta Systems, Inc.  All rights reserved.
25  * Copyright (c) 2018, Joyent, Inc.
26  * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
27  */
28 
29 /*
30  * AHCI (Advanced Host Controller Interface) SATA HBA Driver
31  *
32  * Power Management Support
33  * ------------------------
34  *
35  * At the moment, the ahci driver only implements suspend/resume to
36  * support Suspend to RAM on X86 feature. Device power management isn't
37  * implemented, link power management is disabled, and hot plug isn't
38  * allowed during the period from suspend to resume.
39  *
40  * For s/r support, the ahci driver only need to implement DDI_SUSPEND
41  * and DDI_RESUME entries, and don't need to take care of new requests
42  * sent down after suspend because the target driver (sd) has already
43  * handled these conditions, and blocked these requests. For the detailed
44  * information, please check with sdopen, sdclose and sdioctl routines.
45  *
46  *
47  * Enclosure Management Support
48  * ----------------------------
49  *
50  * The ahci driver has basic support for AHCI Enclosure Management (EM)
51  * services. The AHCI specification provides an area in the primary ahci BAR for
52  * posting data to send out to the enclosure management and provides a register
53  * that provides both information and control about this. While the
54  * specification allows for multiple forms of enclosure management, the only
55  * supported, and commonly found form, is the AHCI specified LED format. The LED
56  * format is often implemented as a one-way communication mechanism. Software
57  * can write out what it cares about into the aforementioned data buffer and
58  * then we wait for the transmission to be sent.
59  *
60  * This has some drawbacks. It means that we cannot know whether or not it has
61  * succeeded. This means we cannot ask hardware what it thinks the LEDs are
62  * set to. There's also the added unfortunate reality that firmware on the
63  * microcontroller driving this will often not show the LEDs if no drive is
64  * present and that actions taken may potentially cause this to get out of sync
65  * with what we expect it to be. For example, the specification does not
66  * describe what should happen if a drive is removed from the enclosure while
67  * this is set and what should happen when it returns. We can only infer that it
68  * should be the same.
69  *
70  * Because only a single command can be sent at any time and we don't want to
71  * interfere with controller I/O, we create a taskq dedicated to this that has a
72  * single thread. Both resets (which occur on attach and resume) and normal
73  * changes to the LED state will be driven through this taskq. Because the taskq
74  * has a single thread, this guarantees serial processing.
75  *
76  * Each userland-submitted task (basically not resets) has a reference counted
77  * task structure. This allows the thread that called it to be cancelled and
78  * have the system clean itself up. The user thread in ioctl blocks on a CV that
79  * can receive signals as it waits for completion.  Note, there is no guarantee
80  * provided by the kernel that the first thread to enter the kernel will be the
81  * first one to change state.
82  */
83 
84 #include <sys/note.h>
85 #include <sys/scsi/scsi.h>
86 #include <sys/pci.h>
87 #include <sys/disp.h>
88 #include <sys/sata/sata_hba.h>
89 #include <sys/sata/adapters/ahci/ahcireg.h>
90 #include <sys/sata/adapters/ahci/ahcivar.h>
91 
92 /*
93  * FMA header files
94  */
95 #include <sys/ddifm.h>
96 #include <sys/fm/protocol.h>
97 #include <sys/fm/util.h>
98 #include <sys/fm/io/ddi.h>
99 
100 /*
101  * EM Control header files
102  */
103 #include <sys/types.h>
104 #include <sys/file.h>
105 #include <sys/errno.h>
106 #include <sys/open.h>
107 #include <sys/cred.h>
108 #include <sys/ddi.h>
109 #include <sys/sunddi.h>
110 
111 /*
112  * This is the string displayed by modinfo, etc.
113  */
114 static char ahci_ident[] = "ahci driver";
115 
116 /*
117  * Function prototypes for driver entry points
118  */
119 static	int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
120 static	int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
121 static	int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
122 static	int ahci_quiesce(dev_info_t *);
123 
124 /*
125  * Function prototypes for SATA Framework interfaces
126  */
127 static	int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
128 static	int ahci_unregister_sata_hba_tran(ahci_ctl_t *);
129 
130 static	int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
131 static	int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
132 static	int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
133 static	int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
134 static	int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
135 static	int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
136 #if defined(__lock_lint)
137 static	int ahci_selftest(dev_info_t *, sata_device_t *);
138 #endif
139 
140 /*
141  * FMA Prototypes
142  */
143 static	void ahci_fm_init(ahci_ctl_t *);
144 static	void ahci_fm_fini(ahci_ctl_t *);
145 static	int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
146 int	ahci_check_acc_handle(ddi_acc_handle_t);
147 int	ahci_check_dma_handle(ddi_dma_handle_t);
148 void	ahci_fm_ereport(ahci_ctl_t *, char *);
149 static	int ahci_check_all_handle(ahci_ctl_t *);
150 static	int ahci_check_ctl_handle(ahci_ctl_t *);
151 static	int ahci_check_port_handle(ahci_ctl_t *, int);
152 static	int ahci_check_slot_handle(ahci_port_t *, int);
153 
154 /*
155  * Local function prototypes
156  */
157 static	int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
158 static	int ahci_alloc_ports_state(ahci_ctl_t *);
159 static	void ahci_dealloc_ports_state(ahci_ctl_t *);
160 static	int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
161 static	void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
162 static	int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
163 static	void ahci_dealloc_rcvd_fis(ahci_port_t *);
164 static	int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
165 static	void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
166 static  int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
167 static  void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
168 static	void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
169 static	void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);
170 
171 static	int ahci_initialize_controller(ahci_ctl_t *);
172 static	void ahci_uninitialize_controller(ahci_ctl_t *);
173 static	int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
174 static	int ahci_config_space_init(ahci_ctl_t *);
175 static	void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);
176 
177 static	void ahci_drain_ports_taskq(ahci_ctl_t *);
178 static	int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
179     uint8_t);
180 static	int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
181 static	int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
182 static	int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
183     sata_device_t *);
184 static	int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
185     sata_pmult_gscr_t *);
186 static	int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
187     sata_device_t *);
188 static	int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
189 static	int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
190 static	int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
191     sata_device_t *);
192 
193 static	void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
194 static	int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
195 static	void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
196     ahci_addr_t *);
197 static	void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
198 static	int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
199     ahci_addr_t *, sata_pkt_t *);
200 static	int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
201     ahci_addr_t *, sata_pkt_t *);
202 static	int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
203     ahci_addr_t *, int);
204 static  void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
205 static	void ahci_copy_ncq_err_page(sata_cmd_t *,
206     struct sata_ncq_error_recovery_page *);
207 static	void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
208 static	void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
209 static	void ahci_flush_doneq(ahci_port_t *);
210 
211 static	int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
212 static	int ahci_hba_reset(ahci_ctl_t *);
213 static	int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
214 static	int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
215 static	void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
216 static	int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
217     ahci_addr_t *);
218 static	int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
219     ahci_addr_t *);
220 static	int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
221     ahci_addr_t *);
222 static	int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
223 static	int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
224     uint8_t);
225 static	int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
226     uint8_t, int, int *);
227 static	void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
228     uint32_t, uint32_t, uint32_t, uint32_t);
229 static	uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
230 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
231     uint8_t, sata_pkt_t *);
232 static	void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
233     ahci_addr_t *, uint32_t);
234 static	void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
235     uint8_t, uint32_t);
236 static	void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
237     uint8_t, uint32_t);
238 static	void ahci_events_handler(void *);
239 static	void ahci_watchdog_handler(ahci_ctl_t *);
240 
241 static	uint_t ahci_intr(caddr_t, caddr_t);
242 static	void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
243 static	int ahci_add_intrs(ahci_ctl_t *, int);
244 static	void ahci_rem_intrs(ahci_ctl_t *);
245 static	void ahci_enable_all_intrs(ahci_ctl_t *);
246 static	void ahci_disable_all_intrs(ahci_ctl_t *);
247 static	void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
248 static	void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);
249 
250 static  int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
251 static	int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
252 static	int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
253 static	int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
254 static	int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
255 static	int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
256     ahci_port_t *, uint8_t);
257 static	int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
258 static	int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
259     uint8_t, uint32_t);
260 static  int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
261     uint8_t, uint32_t);
262 static	int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);
263 
264 static	void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
265 static	int ahci_get_num_implemented_ports(uint32_t);
266 static  void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
267 static	void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
268 static	void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
269 #if AHCI_DEBUG
270 static	void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
271 #endif
272 
273 static	boolean_t ahci_em_init(ahci_ctl_t *);
274 static	void ahci_em_fini(ahci_ctl_t *);
275 static	void ahci_em_suspend(ahci_ctl_t *);
276 static	void ahci_em_resume(ahci_ctl_t *);
277 static	int ahci_em_ioctl(dev_info_t *, int, intptr_t);
278 
279 
280 /*
281  * DMA attributes for the data buffer
282  *
283  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
284  * does not support 64-bit addressing
285  */
286 static ddi_dma_attr_t buffer_dma_attr = {
287 	DMA_ATTR_V0,		/* dma_attr_version */
288 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
289 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
290 	0x3fffffull,		/* dma_attr_count_max i.e. for one cookie */
291 	0x2ull,			/* dma_attr_align: word aligned */
292 	1,			/* dma_attr_burstsizes */
293 	1,			/* dma_attr_minxfer */
294 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
295 	0xffffffffull,		/* dma_attr_seg */
296 	AHCI_PRDT_NUMBER,	/* dma_attr_sgllen */
297 	512,			/* dma_attr_granular */
298 	0,			/* dma_attr_flags */
299 };
300 
301 /*
302  * DMA attributes for the rcvd FIS
303  *
304  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
305  * does not support 64-bit addressing
306  */
307 static ddi_dma_attr_t rcvd_fis_dma_attr = {
308 	DMA_ATTR_V0,		/* dma_attr_version */
309 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
310 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
311 	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
312 	0x100ull,		/* dma_attr_align: 256-byte aligned */
313 	1,			/* dma_attr_burstsizes */
314 	1,			/* dma_attr_minxfer */
315 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
316 	0xffffffffull,		/* dma_attr_seg */
317 	1,			/* dma_attr_sgllen */
318 	1,			/* dma_attr_granular */
319 	0,			/* dma_attr_flags */
320 };
321 
322 /*
323  * DMA attributes for the command list
324  *
325  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
326  * does not support 64-bit addressing
327  */
328 static ddi_dma_attr_t cmd_list_dma_attr = {
329 	DMA_ATTR_V0,		/* dma_attr_version */
330 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
331 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
332 	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
333 	0x400ull,		/* dma_attr_align: 1K-byte aligned */
334 	1,			/* dma_attr_burstsizes */
335 	1,			/* dma_attr_minxfer */
336 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
337 	0xffffffffull,		/* dma_attr_seg */
338 	1,			/* dma_attr_sgllen */
339 	1,			/* dma_attr_granular */
340 	0,			/* dma_attr_flags */
341 };
342 
343 /*
344  * DMA attributes for cmd tables
345  *
346  * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
347  * does not support 64-bit addressing
348  */
349 static ddi_dma_attr_t cmd_table_dma_attr = {
350 	DMA_ATTR_V0,		/* dma_attr_version */
351 	0x0ull,			/* dma_attr_addr_lo: lowest bus address */
352 	0xffffffffffffffffull,	/* dma_attr_addr_hi: highest bus address */
353 	0xffffffffull,		/* dma_attr_count_max i.e. for one cookie */
354 	0x80ull,		/* dma_attr_align: 128-byte aligned */
355 	1,			/* dma_attr_burstsizes */
356 	1,			/* dma_attr_minxfer */
357 	0xffffffffull,		/* dma_attr_maxxfer i.e. includes all cookies */
358 	0xffffffffull,		/* dma_attr_seg */
359 	1,			/* dma_attr_sgllen */
360 	1,			/* dma_attr_granular */
361 	0,			/* dma_attr_flags */
362 };
363 
364 
365 /* Device access attributes */
366 static ddi_device_acc_attr_t accattr = {
367 	DDI_DEVICE_ATTR_V1,
368 	DDI_STRUCTURE_LE_ACC,
369 	DDI_STRICTORDER_ACC,
370 	DDI_DEFAULT_ACC
371 };
372 
373 static struct dev_ops ahcictl_dev_ops = {
374 	DEVO_REV,		/* devo_rev */
375 	0,			/* refcnt  */
376 	ahci_getinfo,		/* info */
377 	nulldev,		/* identify */
378 	nulldev,		/* probe */
379 	ahci_attach,		/* attach */
380 	ahci_detach,		/* detach */
381 	nodev,			/* no reset */
382 	NULL,			/* driver operations */
383 	NULL,			/* bus operations */
384 	NULL,			/* power */
385 	ahci_quiesce,		/* quiesce */
386 };
387 
388 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
389 	SATA_TRAN_HOTPLUG_OPS_REV_1,
390 	ahci_tran_hotplug_port_activate,
391 	ahci_tran_hotplug_port_deactivate
392 };
393 
394 extern struct mod_ops mod_driverops;
395 
396 static  struct modldrv modldrv = {
397 	&mod_driverops,		/* driverops */
398 	ahci_ident,		/* short description */
399 	&ahcictl_dev_ops,	/* driver ops */
400 };
401 
402 static  struct modlinkage modlinkage = {
403 	MODREV_1,
404 	&modldrv,
405 	NULL
406 };
407 
408 /* The following variables are watchdog handler related */
409 static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
410 static clock_t ahci_watchdog_tick;
411 
412 /*
413  * This static variable indicates the size of command table,
414  * and it's changeable with prdt number, which ahci_dma_prdt_number
415  * indicates.
416  */
417 static size_t ahci_cmd_table_size;
418 
419 /*
420  * The below global variables are tunable via /etc/system
421  *
422  *	ahci_dma_prdt_number
423  *	ahci_msi_enabled
424  *	ahci_buf_64bit_dma
425  *	ahci_commu_64bit_dma
426  */
427 
428 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */
429 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;
430 
431 /* AHCI MSI is tunable */
432 boolean_t ahci_msi_enabled = B_TRUE;
433 
434 /*
435  * 64-bit dma addressing for data buffer is tunable
436  *
437  * The variable controls only the below value:
438  *	DBAU (upper 32-bits physical address of data block)
439  */
440 boolean_t ahci_buf_64bit_dma = B_TRUE;
441 
442 /*
443  * 64-bit dma addressing for communication system descriptors is tunable
444  *
445  * The variable controls the below three values:
446  *
447  *	PxCLBU (upper 32-bits for the command list base physical address)
448  *	PxFBU (upper 32-bits for the received FIS base physical address)
449  *	CTBAU (upper 32-bits of command table base)
450  */
451 boolean_t ahci_commu_64bit_dma = B_TRUE;
452 
453 /*
454  * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
455  * chipset. If the users want to have a try with 64-bit dma, please change
456  * the below variable value to enable it.
457  */
458 boolean_t sb600_buf_64bit_dma_disable = B_TRUE;
459 
460 /*
461  * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
462  * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
463  * please change the below value to enable it.
464  */
465 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;
466 
467 /*
468  * These values control the default delay and default number of times to wait
469  * for an enclosure message to complete.
470  */
471 uint_t	ahci_em_reset_delay_ms = 1;
472 uint_t	ahci_em_reset_delay_count = 1000;
473 uint_t	ahci_em_tx_delay_ms = 1;
474 uint_t	ahci_em_tx_delay_count = 1000;
475 
476 
477 /*
478  * End of global tunable variable definition
479  */
480 
481 #if AHCI_DEBUG
482 uint32_t ahci_debug_flags = 0;
483 #else
484 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
485 #endif
486 
487 
488 #if AHCI_DEBUG
489 /* The following is needed for ahci_log() */
490 static kmutex_t ahci_log_mutex;
491 static char ahci_log_buf[512];
492 #endif
493 
494 /* Opaque state pointer initialized by ddi_soft_state_init() */
495 static void *ahci_statep = NULL;
496 
497 /*
498  *  ahci module initialization.
499  */
500 int
_init(void)501 _init(void)
502 {
503 	int	ret;
504 
505 	ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
506 	if (ret != 0) {
507 		goto err_out;
508 	}
509 
510 #if AHCI_DEBUG
511 	mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
512 #endif
513 
514 	if ((ret = sata_hba_init(&modlinkage)) != 0) {
515 #if AHCI_DEBUG
516 		mutex_destroy(&ahci_log_mutex);
517 #endif
518 		ddi_soft_state_fini(&ahci_statep);
519 		goto err_out;
520 	}
521 
522 	/* watchdog tick */
523 	ahci_watchdog_tick = drv_usectohz(
524 	    (clock_t)ahci_watchdog_timeout * 1000000);
525 
526 	ret = mod_install(&modlinkage);
527 	if (ret != 0) {
528 		sata_hba_fini(&modlinkage);
529 #if AHCI_DEBUG
530 		mutex_destroy(&ahci_log_mutex);
531 #endif
532 		ddi_soft_state_fini(&ahci_statep);
533 		goto err_out;
534 	}
535 
536 	return (ret);
537 
538 err_out:
539 	cmn_err(CE_WARN, "!ahci: Module init failed");
540 	return (ret);
541 }
542 
543 /*
544  * ahci module uninitialize.
545  */
546 int
_fini(void)547 _fini(void)
548 {
549 	int	ret;
550 
551 	ret = mod_remove(&modlinkage);
552 	if (ret != 0) {
553 		return (ret);
554 	}
555 
556 	/* Remove the resources allocated in _init(). */
557 	sata_hba_fini(&modlinkage);
558 #if AHCI_DEBUG
559 	mutex_destroy(&ahci_log_mutex);
560 #endif
561 	ddi_soft_state_fini(&ahci_statep);
562 
563 	return (ret);
564 }
565 
566 /*
567  * _info entry point
568  */
569 int
_info(struct modinfo * modinfop)570 _info(struct modinfo *modinfop)
571 {
572 	return (mod_info(&modlinkage, modinfop));
573 }
574 
575 /*
576  * The attach entry point for dev_ops.
577  */
578 static int
ahci_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)579 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
580 {
581 	ahci_ctl_t *ahci_ctlp = NULL;
582 	int instance = ddi_get_instance(dip);
583 	int status;
584 	int attach_state;
585 	uint32_t cap_status, ahci_version;
586 	uint32_t ghc_control;
587 	int intr_types;
588 	int i;
589 	pci_regspec_t *regs;
590 	int regs_length;
591 	int rnumber;
592 #if AHCI_DEBUG
593 	int speed;
594 #endif
595 
596 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
597 	    NULL);
598 
599 	switch (cmd) {
600 	case DDI_ATTACH:
601 		break;
602 
603 	case DDI_RESUME:
604 
605 		/*
606 		 * During DDI_RESUME, the hardware state of the device
607 		 * (power may have been removed from the device) must be
608 		 * restored, allow pending requests to continue, and
609 		 * service new requests.
610 		 */
611 		ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
612 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
613 
614 		/*
615 		 * GHC.AE must be set to 1 before any other AHCI register
616 		 * is accessed
617 		 */
618 		ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
619 		    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
620 		ghc_control |= AHCI_HBA_GHC_AE;
621 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
622 		    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
623 
624 		/* Restart watch thread */
625 		if (ahci_ctlp->ahcictl_timeout_id == 0)
626 			ahci_ctlp->ahcictl_timeout_id = timeout(
627 			    (void (*)(void *))ahci_watchdog_handler,
628 			    (caddr_t)ahci_ctlp, ahci_watchdog_tick);
629 
630 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
631 
632 		/*
633 		 * Re-initialize the controller and enable the interrupts and
634 		 * restart all the ports.
635 		 *
636 		 * Note that so far we don't support hot-plug during
637 		 * suspend/resume.
638 		 */
639 		if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
640 			AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
641 			    "Failed to initialize the controller "
642 			    "during DDI_RESUME", NULL);
643 			return (DDI_FAILURE);
644 		}
645 
646 		/*
647 		 * Reset the enclosure services.
648 		 */
649 		ahci_em_resume(ahci_ctlp);
650 
651 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
652 		ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
653 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
654 
655 		return (DDI_SUCCESS);
656 
657 	default:
658 		return (DDI_FAILURE);
659 	}
660 
661 	attach_state = AHCI_ATTACH_STATE_NONE;
662 
663 	/* Allocate soft state */
664 	status = ddi_soft_state_zalloc(ahci_statep, instance);
665 	if (status != DDI_SUCCESS) {
666 		cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
667 		    instance);
668 		goto err_out;
669 	}
670 
671 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
672 	ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
673 	ahci_ctlp->ahcictl_dip = dip;
674 
675 	/* Initialize the cport/port mapping */
676 	for (i = 0; i < AHCI_MAX_PORTS; i++) {
677 		ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
678 		ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
679 	}
680 
681 	attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;
682 
683 	/* Initialize FMA properties */
684 	ahci_fm_init(ahci_ctlp);
685 
686 	attach_state |= AHCI_ATTACH_STATE_FMA;
687 
688 	/*
689 	 * Now map the AHCI base address; which includes global
690 	 * registers and port control registers
691 	 *
692 	 * According to the spec, the AHCI Base Address is BAR5,
693 	 * but BAR0-BAR4 are optional, so we need to check which
694 	 * rnumber is used for BAR5.
695 	 */
696 
697 	/*
698 	 * search through DDI "reg" property for the AHCI register set
699 	 */
700 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
701 	    DDI_PROP_DONTPASS, "reg", (int **)&regs,
702 	    (uint_t *)&regs_length) != DDI_PROP_SUCCESS) {
703 		cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
704 		    instance);
705 		goto err_out;
706 	}
707 
708 	/* AHCI Base Address is located at 0x24 offset */
709 	for (rnumber = 0; rnumber < regs_length; ++rnumber) {
710 		if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
711 		    == AHCI_PCI_RNUM)
712 			break;
713 	}
714 
715 	ddi_prop_free(regs);
716 
717 	if (rnumber == regs_length) {
718 		cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
719 		    instance);
720 		goto err_out;
721 	}
722 
723 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);
724 
725 	status = ddi_regs_map_setup(dip,
726 	    rnumber,
727 	    (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
728 	    0,
729 	    0,
730 	    &accattr,
731 	    &ahci_ctlp->ahcictl_ahci_acc_handle);
732 	if (status != DDI_SUCCESS) {
733 		cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
734 		    instance);
735 		goto err_out;
736 	}
737 
738 	attach_state |= AHCI_ATTACH_STATE_REG_MAP;
739 
740 	/*
741 	 * GHC.AE must be set to 1 before any other AHCI register
742 	 * is accessed
743 	 */
744 	ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
745 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
746 	ghc_control |= AHCI_HBA_GHC_AE;
747 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
748 	    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
749 
750 	/* Get the AHCI version information */
751 	ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
752 	    (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));
753 
754 	cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
755 	    (ahci_version & 0xffff0000) >> 16,
756 	    ((ahci_version & 0x0000ff00) >> 4 |
757 	    (ahci_version & 0x000000ff)));
758 
759 	/* We don't support controllers whose versions are lower than 1.0 */
760 	if (!(ahci_version & 0xffff0000)) {
761 		cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
762 		    "than version 1.0", instance);
763 		goto err_out;
764 	}
765 
766 	/* Get the HBA capabilities information */
767 	cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
768 	    (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));
769 
770 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
771 	    cap_status);
772 
773 	/* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
774 	if (ahci_version >= 0x00010200) {
775 		uint32_t cap2_status;
776 
777 		/* Get the HBA capabilities extended information */
778 		cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
779 		    (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));
780 
781 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
782 		    "hba capabilities extended = 0x%x", cap2_status);
783 	}
784 
785 	if (cap_status & AHCI_HBA_CAP_EMS) {
786 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_EMS;
787 		ahci_ctlp->ahcictl_em_loc =
788 		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
789 		    (uint32_t *)AHCI_GLOBAL_EM_LOC(ahci_ctlp));
790 		ahci_ctlp->ahcictl_em_ctl =
791 		    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
792 		    (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
793 	}
794 
795 #if AHCI_DEBUG
796 	/* Get the interface speed supported by the HBA */
797 	speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
798 	if (speed == 0x01) {
799 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
800 		    "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
801 	} else if (speed == 0x10) {
802 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
803 		    "hba interface speed support: Gen 2 (3 Gbps)", NULL);
804 	} else if (speed == 0x11) {
805 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
806 		    "hba interface speed support: Gen 3 (6 Gbps)", NULL);
807 	}
808 #endif
809 
810 	/* Get the number of command slots supported by the HBA */
811 	ahci_ctlp->ahcictl_num_cmd_slots =
812 	    ((cap_status & AHCI_HBA_CAP_NCS) >>
813 	    AHCI_HBA_CAP_NCS_SHIFT) + 1;
814 
815 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
816 	    ahci_ctlp->ahcictl_num_cmd_slots);
817 
818 	/* Get the bit map which indicates ports implemented by the HBA */
819 	ahci_ctlp->ahcictl_ports_implemented =
820 	    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
821 	    (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));
822 
823 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
824 	    ahci_ctlp->ahcictl_ports_implemented);
825 
826 	/* Max port number implemented */
827 	ahci_ctlp->ahcictl_num_ports =
828 	    ddi_fls(ahci_ctlp->ahcictl_ports_implemented);
829 
830 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
831 	    (cap_status & AHCI_HBA_CAP_NP) + 1);
832 
833 	/* Get the number of implemented ports by the HBA */
834 	ahci_ctlp->ahcictl_num_implemented_ports =
835 	    ahci_get_num_implemented_ports(
836 	    ahci_ctlp->ahcictl_ports_implemented);
837 
838 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
839 	    "hba number of implemented ports: %d",
840 	    ahci_ctlp->ahcictl_num_implemented_ports);
841 
842 	/* Check whether HBA supports 64bit DMA addressing */
843 	if (!(cap_status & AHCI_HBA_CAP_S64A)) {
844 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
845 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
846 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
847 		    "hba does not support 64-bit addressing", NULL);
848 	}
849 
850 	/* Checking for the support of Port Multiplier */
851 	if (cap_status & AHCI_HBA_CAP_SPM) {
852 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
853 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
854 		    "hba supports port multiplier (CBSS)", NULL);
855 
856 		/* Support FIS-based switching ? */
857 		if (cap_status & AHCI_HBA_CAP_FBSS) {
858 			ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
859 			AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
860 			    "hba supports FIS-based switching (FBSS)", NULL);
861 		}
862 	}
863 
864 	/* Checking for Support Command List Override */
865 	if (cap_status & AHCI_HBA_CAP_SCLO) {
866 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
867 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
868 		    "hba supports command list override.", NULL);
869 	}
870 
871 	/* Checking for Asynchronous Notification */
872 	if (cap_status & AHCI_HBA_CAP_SSNTF) {
873 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
874 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
875 		    "hba supports asynchronous notification.", NULL);
876 	}
877 
878 	if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
879 	    != DDI_SUCCESS) {
880 		cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
881 		    instance);
882 		goto err_out;
883 	}
884 
885 	attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;
886 
887 	/*
888 	 * Check the pci configuration space, and set caps. We also
889 	 * handle the hardware defect in this function.
890 	 *
891 	 * For example, force ATI SB600 to use 32-bit dma addressing
892 	 * since it doesn't support 64-bit dma though its CAP register
893 	 * declares it support.
894 	 */
895 	if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
896 		cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
897 		    instance);
898 		goto err_out;
899 	}
900 
901 	/*
902 	 * Disable the whole controller interrupts before adding
903 	 * interrupt handlers(s).
904 	 */
905 	ahci_disable_all_intrs(ahci_ctlp);
906 
907 	/* Get supported interrupt types */
908 	if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
909 		cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
910 		    instance);
911 		goto err_out;
912 	}
913 
914 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
915 	    "ddi_intr_get_supported_types() returned: 0x%x",
916 	    intr_types);
917 
918 	if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
919 		/*
920 		 * Try MSI first, but fall back to FIXED if failed
921 		 */
922 		if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
923 		    DDI_SUCCESS) {
924 			ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
925 			AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
926 			    "Using MSI interrupt type", NULL);
927 			goto intr_done;
928 		}
929 
930 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
931 		    "MSI registration failed, "
932 		    "trying FIXED interrupts", NULL);
933 	}
934 
935 	if (intr_types & DDI_INTR_TYPE_FIXED) {
936 		if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
937 		    DDI_SUCCESS) {
938 			ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
939 			AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
940 			    "Using FIXED interrupt type", NULL);
941 			goto intr_done;
942 		}
943 
944 		AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
945 		    "FIXED interrupt registration failed", NULL);
946 	}
947 
948 	cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);
949 
950 	goto err_out;
951 
952 intr_done:
953 
954 	attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;
955 
956 	/* Initialize the controller mutex */
957 	mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
958 	    (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);
959 
960 	attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;
961 
962 	if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
963 		ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
964 	} else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
965 		ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
966 	}
967 
968 	ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
969 	    (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
970 	    sizeof (ahci_prdt_item_t));
971 
972 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
973 	    "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
974 	    " ahci_cmd_table_size is 0x%x",
975 	    ahci_dma_prdt_number, ahci_cmd_table_size);
976 
977 	if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
978 		ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
979 		    ahci_dma_prdt_number;
980 
981 	ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
982 	ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
983 	ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
984 	ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;
985 
986 	/*
987 	 * enable 64bit dma for data buffer for SB600 if
988 	 * sb600_buf_64bit_dma_disable is B_FALSE
989 	 */
990 	if ((ahci_buf_64bit_dma == B_FALSE) ||
991 	    ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
992 	    !(sb600_buf_64bit_dma_disable == B_FALSE &&
993 	    ahci_ctlp->ahcictl_venid == 0x1002 &&
994 	    ahci_ctlp->ahcictl_devid == 0x4380))) {
995 		ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
996 		    0xffffffffull;
997 	}
998 
999 	/*
1000 	 * enable 64bit dma for command buffer for SB600/700/710/800
1001 	 * if sbxxx_commu_64bit_dma_disable is B_FALSE
1002 	 */
1003 	if ((ahci_commu_64bit_dma == B_FALSE) ||
1004 	    ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
1005 	    !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
1006 	    ahci_ctlp->ahcictl_venid == 0x1002 &&
1007 	    (ahci_ctlp->ahcictl_devid == 0x4380 ||
1008 	    ahci_ctlp->ahcictl_devid == 0x4391)))) {
1009 		ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
1010 		    0xffffffffull;
1011 		ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
1012 		    0xffffffffull;
1013 		ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
1014 		    0xffffffffull;
1015 	}
1016 
1017 	/* Allocate the ports structure */
1018 	status = ahci_alloc_ports_state(ahci_ctlp);
1019 	if (status != AHCI_SUCCESS) {
1020 		cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
1021 		    instance);
1022 		goto err_out;
1023 	}
1024 
1025 	attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;
1026 
1027 	/*
1028 	 * Initialize the controller and ports.
1029 	 */
1030 	status = ahci_initialize_controller(ahci_ctlp);
1031 	if (status != AHCI_SUCCESS) {
1032 		cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
1033 		    instance);
1034 		goto err_out;
1035 	}
1036 
1037 	attach_state |= AHCI_ATTACH_STATE_HW_INIT;
1038 
1039 	/* Start one thread to check packet timeouts */
1040 	ahci_ctlp->ahcictl_timeout_id = timeout(
1041 	    (void (*)(void *))ahci_watchdog_handler,
1042 	    (caddr_t)ahci_ctlp, ahci_watchdog_tick);
1043 
1044 	attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;
1045 
1046 	if (!ahci_em_init(ahci_ctlp)) {
1047 		cmn_err(CE_WARN, "!ahci%d: failed to initialize enclosure "
1048 		    "services", instance);
1049 		goto err_out;
1050 	}
1051 	attach_state |= AHCI_ATTACH_STATE_ENCLOSURE;
1052 
1053 	if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
1054 		cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
1055 		    instance);
1056 		goto err_out;
1057 	}
1058 
1059 	/* Check all handles at the end of the attach operation. */
1060 	if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
1061 		cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
1062 		    instance);
1063 		goto err_out;
1064 	}
1065 
1066 	ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;
1067 
1068 	AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);
1069 
1070 	return (DDI_SUCCESS);
1071 
1072 err_out:
1073 	/* FMA message */
1074 	ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
1075 	ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);
1076 
1077 	if (attach_state & AHCI_ATTACH_STATE_ENCLOSURE) {
1078 		ahci_em_fini(ahci_ctlp);
1079 	}
1080 
1081 	if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
1082 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
1083 		(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1084 		ahci_ctlp->ahcictl_timeout_id = 0;
1085 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
1086 	}
1087 
1088 	if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
1089 		ahci_uninitialize_controller(ahci_ctlp);
1090 	}
1091 
1092 	if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
1093 		ahci_dealloc_ports_state(ahci_ctlp);
1094 	}
1095 
1096 	if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
1097 		mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1098 	}
1099 
1100 	if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
1101 		ahci_rem_intrs(ahci_ctlp);
1102 	}
1103 
1104 	if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
1105 		pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1106 	}
1107 
1108 	if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
1109 		ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1110 	}
1111 
1112 	if (attach_state & AHCI_ATTACH_STATE_FMA) {
1113 		ahci_fm_fini(ahci_ctlp);
1114 	}
1115 
1116 	if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
1117 		ddi_soft_state_free(ahci_statep, instance);
1118 	}
1119 
1120 	return (DDI_FAILURE);
1121 }
1122 
1123 /*
1124  * The detach entry point for dev_ops.
1125  */
1126 static int
ahci_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)1127 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
1128 {
1129 	ahci_ctl_t *ahci_ctlp;
1130 	int instance;
1131 	int ret;
1132 
1133 	instance = ddi_get_instance(dip);
1134 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
1135 
1136 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);
1137 
1138 	switch (cmd) {
1139 	case DDI_DETACH:
1140 
1141 		/* disable the interrupts for an uninterrupted detach */
1142 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
1143 		ahci_disable_all_intrs(ahci_ctlp);
1144 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
1145 
1146 		/* unregister from the sata framework. */
1147 		ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
1148 		if (ret != AHCI_SUCCESS) {
1149 			mutex_enter(&ahci_ctlp->ahcictl_mutex);
1150 			ahci_enable_all_intrs(ahci_ctlp);
1151 			mutex_exit(&ahci_ctlp->ahcictl_mutex);
1152 			return (DDI_FAILURE);
1153 		}
1154 
1155 		ahci_em_fini(ahci_ctlp);
1156 
1157 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
1158 
1159 		/* stop the watchdog handler */
1160 		(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1161 		ahci_ctlp->ahcictl_timeout_id = 0;
1162 
1163 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
1164 
1165 		/* uninitialize the controller */
1166 		ahci_uninitialize_controller(ahci_ctlp);
1167 
1168 		/* remove the interrupts */
1169 		ahci_rem_intrs(ahci_ctlp);
1170 
1171 		/* deallocate the ports structures */
1172 		ahci_dealloc_ports_state(ahci_ctlp);
1173 
1174 		/* destroy mutex */
1175 		mutex_destroy(&ahci_ctlp->ahcictl_mutex);
1176 
1177 		/* teardown the pci config */
1178 		pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
1179 
1180 		/* remove the reg maps. */
1181 		ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
1182 
1183 		/* release fma resource */
1184 		ahci_fm_fini(ahci_ctlp);
1185 
1186 		/* free the soft state. */
1187 		ddi_soft_state_free(ahci_statep, instance);
1188 
1189 		return (DDI_SUCCESS);
1190 
1191 	case DDI_SUSPEND:
1192 
1193 		/*
1194 		 * The steps associated with suspension must include putting
1195 		 * the underlying device into a quiescent state so that it
1196 		 * will not generate interrupts or modify or access memory.
1197 		 */
1198 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
1199 		if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
1200 			mutex_exit(&ahci_ctlp->ahcictl_mutex);
1201 			return (DDI_SUCCESS);
1202 		}
1203 
1204 		ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;
1205 
1206 		/* stop the watchdog handler */
1207 		if (ahci_ctlp->ahcictl_timeout_id) {
1208 			(void) untimeout(ahci_ctlp->ahcictl_timeout_id);
1209 			ahci_ctlp->ahcictl_timeout_id = 0;
1210 		}
1211 
1212 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
1213 
1214 		ahci_em_suspend(ahci_ctlp);
1215 
1216 		/*
1217 		 * drain the taskq
1218 		 */
1219 		ahci_drain_ports_taskq(ahci_ctlp);
1220 
1221 		/*
1222 		 * Disable the interrupts and stop all the ports.
1223 		 */
1224 		ahci_uninitialize_controller(ahci_ctlp);
1225 
1226 		return (DDI_SUCCESS);
1227 
1228 	default:
1229 		return (DDI_FAILURE);
1230 	}
1231 }
1232 
1233 /*
1234  * The info entry point for dev_ops.
1235  *
1236  */
1237 static int
ahci_getinfo(dev_info_t * dip,ddi_info_cmd_t infocmd,void * arg,void ** result)1238 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
1239     void *arg, void **result)
1240 {
1241 #ifndef __lock_lint
1242 	_NOTE(ARGUNUSED(dip))
1243 #endif /* __lock_lint */
1244 
1245 	ahci_ctl_t *ahci_ctlp;
1246 	int instance;
1247 	dev_t dev;
1248 
1249 	dev = (dev_t)arg;
1250 	instance = getminor(dev);
1251 
1252 	switch (infocmd) {
1253 		case DDI_INFO_DEVT2DEVINFO:
1254 			ahci_ctlp = ddi_get_soft_state(ahci_statep,  instance);
1255 			if (ahci_ctlp != NULL) {
1256 				*result = ahci_ctlp->ahcictl_dip;
1257 				return (DDI_SUCCESS);
1258 			} else {
1259 				*result = NULL;
1260 				return (DDI_FAILURE);
1261 			}
1262 		case DDI_INFO_DEVT2INSTANCE:
1263 			*(int *)result = instance;
1264 			break;
1265 		default:
1266 			break;
1267 	}
1268 
1269 	return (DDI_SUCCESS);
1270 }
1271 
1272 /*
1273  * Registers the ahci with sata framework.
1274  */
1275 static int
ahci_register_sata_hba_tran(ahci_ctl_t * ahci_ctlp,uint32_t cap_status)1276 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
1277 {
1278 	struct	sata_hba_tran	*sata_hba_tran;
1279 
1280 	AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
1281 	    "ahci_register_sata_hba_tran enter", NULL);
1282 
1283 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
1284 
1285 	/* Allocate memory for the sata_hba_tran  */
1286 	sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);
1287 
1288 	sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
1289 	sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
1290 	sata_hba_tran->sata_tran_hba_dma_attr =
1291 	    &ahci_ctlp->ahcictl_buffer_dma_attr;
1292 
1293 	/* Report the number of implemented ports */
1294 	sata_hba_tran->sata_tran_hba_num_cports =
1295 	    ahci_ctlp->ahcictl_num_implemented_ports;
1296 
1297 	/* Support ATAPI device */
1298 	sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;
1299 
1300 	/* Get the data transfer capability for PIO command by the HBA */
1301 	if (cap_status & AHCI_HBA_CAP_PMD) {
1302 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
1303 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
1304 		    "DRQ block data transfer for PIO command protocol", NULL);
1305 	}
1306 
1307 	/*
1308 	 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
1309 	 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
1310 	 * QUEUED (EXT), and SERVICE commands). Queued operations are
1311 	 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
1312 	 * QUEUED commands when the HBA and device support native command
1313 	 * queuing(NCQ).
1314 	 *
1315 	 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
1316 	 * CAP register of the HBA indicates NCQ is supported.
1317 	 *
1318 	 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
1319 	 * set because the previous register content of PxCI can be re-written
1320 	 * in the register write.
1321 	 */
1322 	if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
1323 	    !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
1324 		sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
1325 		ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
1326 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
1327 		    "Command Queuing", NULL);
1328 	}
1329 
1330 	/* Support port multiplier? */
1331 	if (cap_status & AHCI_HBA_CAP_SPM) {
1332 		sata_hba_tran->sata_tran_hba_features_support |=
1333 		    SATA_CTLF_PORT_MULTIPLIER;
1334 
1335 		/* Support FIS-based switching for port multiplier? */
1336 		if (cap_status & AHCI_HBA_CAP_FBSS) {
1337 			sata_hba_tran->sata_tran_hba_features_support |=
1338 			    SATA_CTLF_PMULT_FBS;
1339 		}
1340 	}
1341 
1342 	/* Report the number of command slots */
1343 	sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;
1344 
1345 	sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
1346 	sata_hba_tran->sata_tran_start = ahci_tran_start;
1347 	sata_hba_tran->sata_tran_abort = ahci_tran_abort;
1348 	sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
1349 	sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
1350 #ifdef __lock_lint
1351 	sata_hba_tran->sata_tran_selftest = ahci_selftest;
1352 #endif
1353 	/*
1354 	 * When SATA framework adds support for pwrmgt the
1355 	 * pwrmgt_ops needs to be updated
1356 	 */
1357 	sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
1358 	sata_hba_tran->sata_tran_ioctl = ahci_em_ioctl;
1359 
1360 	ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;
1361 
1362 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
1363 
1364 	/* Attach it to SATA framework */
1365 	if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
1366 	    != DDI_SUCCESS) {
1367 		kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
1368 		mutex_enter(&ahci_ctlp->ahcictl_mutex);
1369 		ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1370 		mutex_exit(&ahci_ctlp->ahcictl_mutex);
1371 		return (AHCI_FAILURE);
1372 	}
1373 
1374 	return (AHCI_SUCCESS);
1375 }
1376 
1377 /*
1378  * Unregisters the ahci with sata framework.
1379  */
1380 static int
ahci_unregister_sata_hba_tran(ahci_ctl_t * ahci_ctlp)1381 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
1382 {
1383 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1384 	    "ahci_unregister_sata_hba_tran enter", NULL);
1385 
1386 	/* Detach from the SATA framework. */
1387 	if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
1388 	    DDI_SUCCESS) {
1389 		return (AHCI_FAILURE);
1390 	}
1391 
1392 	/* Deallocate sata_hba_tran. */
1393 	kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
1394 	    sizeof (sata_hba_tran_t));
1395 
1396 	mutex_enter(&ahci_ctlp->ahcictl_mutex);
1397 	ahci_ctlp->ahcictl_sata_hba_tran = NULL;
1398 	mutex_exit(&ahci_ctlp->ahcictl_mutex);
1399 
1400 	return (AHCI_SUCCESS);
1401 }
1402 
1403 #define	SET_PORTSTR(str, addrp)						\
1404 	if (AHCI_ADDR_IS_PORT(addrp))					\
1405 		(void) sprintf((str), "%d", (addrp)->aa_port);		\
1406 	else if (AHCI_ADDR_IS_PMULT(addrp))				\
1407 		(void) sprintf((str), "%d (pmult)", (addrp)->aa_port);	\
1408 	else								\
1409 		(void) sprintf((str), "%d:%d", (addrp)->aa_port,	\
1410 		    (addrp)->aa_pmport);
1411 
1412 /*
1413  * ahci_tran_probe_port is called by SATA framework. It returns port state,
1414  * port status registers and an attached device type via sata_device
1415  * structure.
1416  *
1417  * We return the cached information from a previous hardware probe. The
1418  * actual hardware probing itself was done either from within
1419  * ahci_initialize_controller() during the driver attach or from a phy
1420  * ready change interrupt handler.
1421  */
1422 static int
ahci_tran_probe_port(dev_info_t * dip,sata_device_t * sd)1423 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
1424 {
1425 	ahci_ctl_t *ahci_ctlp;
1426 	ahci_port_t *ahci_portp;
1427 	ahci_addr_t addr, pmult_addr;
1428 	uint8_t cport = sd->satadev_addr.cport;
1429 	char portstr[10];
1430 	uint8_t device_type;
1431 	uint32_t port_state;
1432 	uint8_t port;
1433 	int rval = SATA_SUCCESS, rval_init;
1434 
1435 	port_state = 0;
1436 	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1437 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
1438 
1439 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
1440 
1441 	mutex_enter(&ahci_portp->ahciport_mutex);
1442 
1443 	ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
1444 	ASSERT(AHCI_ADDR_IS_VALID(&addr));
1445 	SET_PORTSTR(portstr, &addr);
1446 
1447 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1448 	    "ahci_tran_probe_port enter: port %s", portstr);
1449 
1450 	if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
1451 	    (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1452 	    ahci_portp->ahciport_pmult_info == NULL)) {
1453 		/* port mutliplier is removed. */
1454 		AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1455 		    "ahci_tran_probe_port: "
1456 		    "pmult is removed from port %s", portstr);
1457 		mutex_exit(&ahci_portp->ahciport_mutex);
1458 		return (SATA_FAILURE);
1459 	}
1460 
1461 	/*
1462 	 * The sata_device may refer to
1463 	 * 1. A controller port.
1464 	 *    A controller port should be ready here.
1465 	 * 2. A port multiplier.
1466 	 *    SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
1467 	 *    it and register the port multiplier to the framework.
1468 	 *    SATA_ADDR_PMULT - check the status of all its device ports.
1469 	 * 3. A port multiplier port.
1470 	 *    If it has not been initialized, initialized it.
1471 	 *
1472 	 * A port multiplier or a port multiplier port may require some
1473 	 * initialization because we cannot do these time-consuming jobs in an
1474 	 * interrupt context.
1475 	 */
1476 	if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
1477 		AHCI_ADDR_SET_PMULT(&pmult_addr, port);
1478 		/* Initialize registers on a port multiplier */
1479 		rval_init = ahci_initialize_pmult(ahci_ctlp,
1480 		    ahci_portp, &pmult_addr, sd);
1481 		if (rval_init != AHCI_SUCCESS) {
1482 			AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
1483 			    "ahci_tran_probe_port: "
1484 			    "pmult initialization failed.", NULL);
1485 			mutex_exit(&ahci_portp->ahciport_mutex);
1486 			return (SATA_FAILURE);
1487 		}
1488 	} else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
1489 		/* Check pmports hotplug events */
1490 		(void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
1491 	} else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
1492 	    SATA_ADDR_DPMPORT)) {
1493 		if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
1494 		    &addr, sd) != AHCI_SUCCESS) {
1495 			rval = SATA_FAILURE;
1496 			goto out;
1497 		}
1498 	}
1499 
1500 	/* Update port state and device type */
1501 	port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);
1502 
1503 	switch (port_state) {
1504 
1505 	case SATA_PSTATE_FAILED:
1506 		sd->satadev_state = SATA_PSTATE_FAILED;
1507 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1508 		    "ahci_tran_probe_port: port %s PORT FAILED", portstr);
1509 		goto out;
1510 
1511 	case SATA_PSTATE_SHUTDOWN:
1512 		sd->satadev_state = SATA_PSTATE_SHUTDOWN;
1513 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1514 		    "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
1515 		goto out;
1516 
1517 	case SATA_PSTATE_PWROFF:
1518 		sd->satadev_state = SATA_PSTATE_PWROFF;
1519 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1520 		    "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
1521 		goto out;
1522 
1523 	case SATA_PSTATE_PWRON:
1524 		sd->satadev_state = SATA_PSTATE_PWRON;
1525 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1526 		    "ahci_tran_probe_port: port %s PORT PWRON", portstr);
1527 		break;
1528 
1529 	default:
1530 		sd->satadev_state = port_state;
1531 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1532 		    "ahci_tran_probe_port: port %s PORT NORMAL %x",
1533 		    portstr, port_state);
1534 		break;
1535 	}
1536 
1537 	device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1538 
1539 	switch (device_type) {
1540 
1541 	case SATA_DTYPE_ATADISK:
1542 		sd->satadev_type = SATA_DTYPE_ATADISK;
1543 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1544 		    "ahci_tran_probe_port: port %s DISK found", portstr);
1545 		break;
1546 
1547 	case SATA_DTYPE_ATAPI:
1548 		/*
1549 		 * HBA driver only knows it's an ATAPI device, and don't know
1550 		 * it's CD/DVD, tape or ATAPI disk because the ATAPI device
1551 		 * type need to be determined by checking IDENTIFY PACKET
1552 		 * DEVICE data
1553 		 */
1554 		sd->satadev_type = SATA_DTYPE_ATAPI;
1555 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1556 		    "ahci_tran_probe_port: port %s ATAPI found", portstr);
1557 		break;
1558 
1559 	case SATA_DTYPE_PMULT:
1560 		ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
1561 		sd->satadev_type = SATA_DTYPE_PMULT;
1562 
1563 		/* Update the number of pmports. */
1564 		ASSERT(ahci_portp->ahciport_pmult_info != NULL);
1565 		sd->satadev_add_info = ahci_portp->
1566 		    ahciport_pmult_info->ahcipmi_num_dev_ports;
1567 
1568 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1569 		    "ahci_tran_probe_port: port %s Port Multiplier found",
1570 		    portstr);
1571 		break;
1572 
1573 	case SATA_DTYPE_UNKNOWN:
1574 		sd->satadev_type = SATA_DTYPE_UNKNOWN;
1575 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1576 		    "ahci_tran_probe_port: port %s Unknown device found",
1577 		    portstr);
1578 		break;
1579 
1580 	default:
1581 		/* we don't support any other device types */
1582 		sd->satadev_type = SATA_DTYPE_NONE;
1583 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
1584 		    "ahci_tran_probe_port: port %s No device found", portstr);
1585 		break;
1586 	}
1587 
1588 out:
1589 	/* Register update only fails while probing a pmult/pmport */
1590 	if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
1591 		ahci_update_sata_registers(ahci_ctlp, port, sd);
1592 	} else if (AHCI_ADDR_IS_PMPORT(&addr)) {
1593 		if (port_state & SATA_STATE_READY)
1594 			if (ahci_update_pmult_pscr(ahci_ctlp,
1595 			    &addr, sd) != AHCI_SUCCESS)
1596 				rval = SATA_FAILURE;
1597 	}
1598 
1599 	/* Check handles for the sata registers access */
1600 	if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
1601 	    (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
1602 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1603 		    DDI_SERVICE_UNAFFECTED);
1604 		rval = SATA_FAILURE;
1605 	}
1606 
1607 	mutex_exit(&ahci_portp->ahciport_mutex);
1608 	return (rval);
1609 }
1610 
1611 /*
1612  * There are four operation modes in sata framework:
1613  * SATA_OPMODE_INTERRUPTS
1614  * SATA_OPMODE_POLLING
1615  * SATA_OPMODE_ASYNCH
1616  * SATA_OPMODE_SYNCH
1617  *
1618  * Their combined meanings as following:
1619  *
1620  * SATA_OPMODE_SYNCH
1621  * The command has to be completed before sata_tran_start functions returns.
1622  * Either interrupts or polling could be used - it's up to the driver.
1623  * Mode used currently for internal, sata-module initiated operations.
1624  *
1625  * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
1626  * It is the same as the one above.
1627  *
1628  * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
1629  * The command has to be completed before sata_tran_start function returns.
1630  * No interrupt used, polling only. This should be the mode used for scsi
1631  * packets with FLAG_NOINTR.
1632  *
1633  * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
1634  * The command may be queued (callback function specified). Interrupts could
1635  * be used. It's normal operation mode.
1636  */
1637 /*
1638  * Called by sata framework to transport a sata packet down stream.
1639  */
1640 static int
ahci_tran_start(dev_info_t * dip,sata_pkt_t * spkt)1641 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
1642 {
1643 	ahci_ctl_t *ahci_ctlp;
1644 	ahci_port_t *ahci_portp;
1645 	ahci_addr_t addr;
1646 	uint8_t	cport = spkt->satapkt_device.satadev_addr.cport;
1647 	uint8_t port;
1648 	char portstr[10];
1649 
1650 	ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
1651 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
1652 
1653 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
1654 	    "ahci_tran_start enter: cport %d satapkt 0x%p",
1655 	    cport, (void *)spkt);
1656 
1657 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
1658 
1659 	mutex_enter(&ahci_portp->ahciport_mutex);
1660 	ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
1661 	SET_PORTSTR(portstr, &addr);
1662 
1663 	/* Sanity check */
1664 	if (AHCI_ADDR_IS_PMPORT(&addr)) {
1665 		if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
1666 		    ahci_portp->ahciport_pmult_info == NULL) {
1667 
1668 			spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1669 			spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1670 			spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
1671 			ahci_update_sata_registers(ahci_ctlp, port,
1672 			    &spkt->satapkt_device);
1673 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1674 			    "ahci_tran_start returning PORT_ERROR while "
1675 			    "pmult removed: port: %s", portstr);
1676 			mutex_exit(&ahci_portp->ahciport_mutex);
1677 			return (SATA_TRAN_PORT_ERROR);
1678 		}
1679 
1680 		if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
1681 		    SATA_STATE_READY)) {
1682 			if (!ddi_in_panic() ||
1683 			    ahci_initialize_pmport(ahci_ctlp,
1684 			    ahci_portp, &addr) != AHCI_SUCCESS) {
1685 				spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1686 				spkt->satapkt_device.satadev_type =
1687 				    AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1688 				spkt->satapkt_device.satadev_state =
1689 				    AHCIPORT_GET_STATE(ahci_portp, &addr);
1690 				ahci_update_sata_registers(ahci_ctlp, port,
1691 				    &spkt->satapkt_device);
1692 				AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1693 				    "ahci_tran_start returning PORT_ERROR "
1694 				    "while sub-link is not initialized "
1695 				    "at port: %s", portstr);
1696 				mutex_exit(&ahci_portp->ahciport_mutex);
1697 				return (SATA_TRAN_PORT_ERROR);
1698 			}
1699 		}
1700 	}
1701 
1702 	if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
1703 	    AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
1704 	    AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
1705 		/*
1706 		 * In case the target driver would send the packet before
1707 		 * sata framework can have the opportunity to process those
1708 		 * event reports.
1709 		 */
1710 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1711 		spkt->satapkt_device.satadev_state =
1712 		    ahci_portp->ahciport_port_state;
1713 		ahci_update_sata_registers(ahci_ctlp, port,
1714 		    &spkt->satapkt_device);
1715 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1716 		    "ahci_tran_start returning PORT_ERROR while "
1717 		    "port in FAILED/SHUTDOWN/PWROFF state: "
1718 		    "port: %s", portstr);
1719 		mutex_exit(&ahci_portp->ahciport_mutex);
1720 		return (SATA_TRAN_PORT_ERROR);
1721 	}
1722 
1723 	if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
1724 		/*
1725 		 * ahci_intr_phyrdy_change() may have rendered it to
1726 		 * SATA_DTYPE_NONE.
1727 		 */
1728 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
1729 		spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
1730 		spkt->satapkt_device.satadev_state =
1731 		    ahci_portp->ahciport_port_state;
1732 		ahci_update_sata_registers(ahci_ctlp, port,
1733 		    &spkt->satapkt_device);
1734 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1735 		    "ahci_tran_start returning PORT_ERROR while "
1736 		    "no device attached: port: %s", portstr);
1737 		mutex_exit(&ahci_portp->ahciport_mutex);
1738 		return (SATA_TRAN_PORT_ERROR);
1739 	}
1740 
1741 	/* R/W PMULT command will occupy the whole HBA port */
1742 	if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
1743 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1744 		    "ahci_tran_start returning BUSY while "
1745 		    "executing READ/WRITE PORT-MULT command: "
1746 		    "port: %s", portstr);
1747 		spkt->satapkt_reason = SATA_PKT_BUSY;
1748 		mutex_exit(&ahci_portp->ahciport_mutex);
1749 		return (SATA_TRAN_BUSY);
1750 	}
1751 
1752 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
1753 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1754 		    "ahci_tran_start returning BUSY while "
1755 		    "hot-plug in progress: port: %s", portstr);
1756 		spkt->satapkt_reason = SATA_PKT_BUSY;
1757 		mutex_exit(&ahci_portp->ahciport_mutex);
1758 		return (SATA_TRAN_BUSY);
1759 	}
1760 
1761 	/*
1762 	 * SATA HBA driver should remember that a device was reset and it
1763 	 * is supposed to reject any packets which do not specify either
1764 	 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
1765 	 *
1766 	 * This is to prevent a race condition when a device was arbitrarily
1767 	 * reset by the HBA driver (and lost it's setting) and a target
1768 	 * driver sending some commands to a device before the sata framework
1769 	 * has a chance to restore the device setting (such as cache enable/
1770 	 * disable or other resettable stuff).
1771 	 */
1772 	/*
1773 	 * It is unnecessary to use specific flags to indicate
1774 	 * reset_in_progress for a pmport. While mopping, all command will be
1775 	 * mopped so that the entire HBA port is being dealt as a single
1776 	 * object.
1777 	 */
1778 	if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
1779 		ahci_portp->ahciport_reset_in_progress = 0;
1780 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1781 		    "ahci_tran_start [CLEAR] the "
1782 		    "reset_in_progress for port: %d", port);
1783 	}
1784 
1785 	if (ahci_portp->ahciport_reset_in_progress &&
1786 	    ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
1787 	    ! ddi_in_panic()) {
1788 		spkt->satapkt_reason = SATA_PKT_BUSY;
1789 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1790 		    "ahci_tran_start returning BUSY while "
1791 		    "reset in progress: port: %d", port);
1792 		mutex_exit(&ahci_portp->ahciport_mutex);
1793 		return (SATA_TRAN_BUSY);
1794 	}
1795 
1796 #ifdef AHCI_DEBUG
1797 	if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
1798 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1799 		    "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
1800 		    spkt, port);
1801 	}
1802 #endif
1803 
1804 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
1805 		spkt->satapkt_reason = SATA_PKT_BUSY;
1806 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1807 		    "ahci_tran_start returning BUSY while "
1808 		    "mopping in progress: port: %d", port);
1809 		mutex_exit(&ahci_portp->ahciport_mutex);
1810 		return (SATA_TRAN_BUSY);
1811 	}
1812 
1813 	if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
1814 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
1815 		    DDI_SERVICE_UNAFFECTED);
1816 		mutex_exit(&ahci_portp->ahciport_mutex);
1817 		return (SATA_TRAN_BUSY);
1818 	}
1819 
1820 	if (spkt->satapkt_op_mode &
1821 	    (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
1822 		/*
1823 		 * If a SYNC command to be executed in interrupt context,
1824 		 * bounce it back to sata module.
1825 		 */
1826 		if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
1827 		    servicing_interrupt()) {
1828 			spkt->satapkt_reason = SATA_PKT_BUSY;
1829 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
1830 			    "ahci_tran_start returning BUSY while "
1831 			    "sending SYNC mode under interrupt context: "
1832 			    "port : %d", port);
1833 			mutex_exit(&ahci_portp->ahciport_mutex);
1834 			return (SATA_TRAN_BUSY);
1835 		}
1836 
1837 		/* We need to do the sync start now */
1838 		if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
1839 		    spkt) == AHCI_FAILURE) {
1840 			goto fail_out;
1841 		}
1842 	} else {
1843 		/* Async start, using interrupt */
1844 		if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
1845 		    == AHCI_FAILURE) {
1846 			spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
1847 			goto fail_out;
1848 		}
1849 	}
1850 
1851 	AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
1852 	    "sata tran accepted: port %s", portstr);
1853 
1854 	mutex_exit(&ahci_portp->ahciport_mutex);
1855 	return (SATA_TRAN_ACCEPTED);
1856 
1857 fail_out:
1858 	/*
1859 	 * Failed to deliver packet to the controller.
1860 	 * Check if it's caused by invalid handles.
1861 	 */
1862 	if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
1863 	    ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
1864 		spkt->satapkt_device.satadev_type =
1865 		    AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
1866 		spkt->satapkt_device.satadev_state =
1867 		    AHCIPORT_GET_STATE(ahci_portp, &addr);
1868 		spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
1869 		mutex_exit(&ahci_portp->ahciport_mutex);
1870 		return (SATA_TRAN_PORT_ERROR);
1871 	}
1872 
1873 	AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
1874 	    "return QUEUE_FULL: port %d", port);
1875 	mutex_exit(&ahci_portp->ahciport_mutex);
1876 	return (SATA_TRAN_QUEUE_FULL);
1877 }
1878 
1879 /*
1880  * SATA_OPMODE_SYNCH flag is set
1881  *
1882  * If SATA_OPMODE_POLLING flag is set, then we must poll the command
1883  * without interrupt, otherwise we can still use the interrupt.
1884  */
1885 static int
ahci_do_sync_start(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp,sata_pkt_t * spkt)1886 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1887     ahci_addr_t *addrp, sata_pkt_t *spkt)
1888 {
1889 	int pkt_timeout_ticks;
1890 	uint32_t timeout_tags;
1891 	int rval;
1892 	int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
1893 	uint8_t port = addrp->aa_port;
1894 
1895 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1896 
1897 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
1898 	    "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);
1899 
1900 	if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
1901 		ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
1902 		if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1903 		    addrp, spkt)) == AHCI_FAILURE) {
1904 			ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1905 			return (rval);
1906 		}
1907 
1908 		pkt_timeout_ticks =
1909 		    drv_usectohz((clock_t)spkt->satapkt_time * 1000000);
1910 
1911 		while (spkt->satapkt_reason == SATA_PKT_BUSY) {
1912 			/* Simulate the interrupt */
1913 			mutex_exit(&ahci_portp->ahciport_mutex);
1914 			ahci_port_intr(ahci_ctlp, ahci_portp, port);
1915 			mutex_enter(&ahci_portp->ahciport_mutex);
1916 
1917 			if (spkt->satapkt_reason != SATA_PKT_BUSY)
1918 				break;
1919 
1920 			mutex_exit(&ahci_portp->ahciport_mutex);
1921 			drv_usecwait(AHCI_1MS_USECS);
1922 			mutex_enter(&ahci_portp->ahciport_mutex);
1923 
1924 			pkt_timeout_ticks -= AHCI_1MS_TICKS;
1925 			if (pkt_timeout_ticks < 0) {
1926 				cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
1927 				    "port %d satapkt 0x%p timed out\n",
1928 				    instance, port, (void *)spkt);
1929 				timeout_tags = (0x1 << rval);
1930 				mutex_exit(&ahci_portp->ahciport_mutex);
1931 				ahci_timeout_pkts(ahci_ctlp, ahci_portp,
1932 				    port, timeout_tags);
1933 				mutex_enter(&ahci_portp->ahciport_mutex);
1934 			}
1935 		}
1936 
1937 		ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
1938 		return (AHCI_SUCCESS);
1939 
1940 	} else {
1941 		if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
1942 		    addrp, spkt)) == AHCI_FAILURE)
1943 			return (rval);
1944 
1945 #if AHCI_DEBUG
1946 		/*
1947 		 * Note that the driver always uses the slot 0 to deliver
1948 		 * REQUEST SENSE or READ LOG EXT command
1949 		 */
1950 		if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
1951 			ASSERT(rval == 0);
1952 #endif
1953 
1954 		while (spkt->satapkt_reason == SATA_PKT_BUSY)
1955 			cv_wait(&ahci_portp->ahciport_cv,
1956 			    &ahci_portp->ahciport_mutex);
1957 
1958 		return (AHCI_SUCCESS);
1959 	}
1960 }
1961 
1962 /*
1963  * Searches for and claims a free command slot.
1964  *
1965  * Returns value:
1966  *
1967  * AHCI_FAILURE returned only if
1968  *	1. No empty slot left
1969  *	2. Non-queued command requested while queued command(s) is outstanding
1970  *	3. Queued command requested while non-queued command(s) is outstanding
1971  *	4. HBA doesn't support multiple-use of command list while already a
1972  *	   non-queued command is oustanding
1973  *	5. Queued command requested while some queued command(s) has been
1974  *	   outstanding on a different port multiplier port. (AHCI spec 1.2,
1975  *	   9.1.2)
1976  *
1977  * claimed slot number returned if succeeded
1978  *
1979  * NOTE: it will always return slot 0 for following commands to simplify the
1980  * algorithm.
1981  *	1. REQUEST SENSE or READ LOG EXT command during error recovery process
1982  *	2. READ/WRITE PORTMULT command
1983  */
1984 static int
ahci_claim_free_slot(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp,int command_type)1985 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
1986     ahci_addr_t *addrp, int command_type)
1987 {
1988 	uint32_t port_cmd_issue;
1989 	uint32_t free_slots;
1990 	int slot;
1991 
1992 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
1993 
1994 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
1995 	    "ahciport_pending_tags = 0x%x "
1996 	    "ahciport_pending_ncq_tags = 0x%x",
1997 	    ahci_portp->ahciport_pending_tags,
1998 	    ahci_portp->ahciport_pending_ncq_tags);
1999 
2000 	free_slots = 0;
2001 	/*
2002 	 * According to the AHCI spec, system software is responsible to
2003 	 * ensure that queued and non-queued commands are not mixed in
2004 	 * the command list.
2005 	 */
2006 	if (command_type == AHCI_NON_NCQ_CMD) {
2007 		/* Non-NCQ command request */
2008 		if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2009 			AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
2010 			    "ahci_claim_free_slot: there is still pending "
2011 			    "queued command(s) in the command list, "
2012 			    "so no available slot for the non-queued "
2013 			    "command", NULL);
2014 			return (AHCI_FAILURE);
2015 		}
2016 		if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
2017 			AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
2018 			    "ahci_claim_free_slot: there is still pending "
2019 			    "read/write port-mult command(s) in command list, "
2020 			    "so no available slot for the non-queued command",
2021 			    NULL);
2022 			return (AHCI_FAILURE);
2023 		}
2024 		if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
2025 		    NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2026 			AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2027 			    "ahci_claim_free_slot: HBA cannot support multiple-"
2028 			    "use of the command list for non-queued commands",
2029 			    NULL);
2030 			return (AHCI_FAILURE);
2031 		}
2032 		free_slots = (~ahci_portp->ahciport_pending_tags) &
2033 		    AHCI_SLOT_MASK(ahci_ctlp);
2034 	} else if (command_type == AHCI_NCQ_CMD) {
2035 		/* NCQ command request */
2036 		if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2037 			AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
2038 			    "ahci_claim_free_slot: there is still pending "
2039 			    "non-queued command(s) in the command list, "
2040 			    "so no available slot for the queued command",
2041 			    NULL);
2042 			return (AHCI_FAILURE);
2043 		}
2044 
2045 		/*
2046 		 * NCQ commands cannot be sent to different port multiplier
2047 		 * ports in Command-Based Switching mode
2048 		 */
2049 		/*
2050 		 * NOTE: In Command-Based Switching mode, AHCI controller
2051 		 * usually reports a 'Handshake Error' when multiple NCQ
2052 		 * commands are outstanding simultaneously.
2053 		 */
2054 		if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
2055 			ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2056 			if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
2057 			    NCQ_CMD_IN_PROGRESS(ahci_portp) &&
2058 			    AHCIPORT_NCQ_PMPORT(ahci_portp) !=
2059 			    addrp->aa_pmport) {
2060 				AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2061 				    "ahci_claim_free_slot: there is still "
2062 				    "pending queued command(s) in the "
2063 				    "command list for another Port Multiplier "
2064 				    "port, so no available slot.", NULL);
2065 				return (AHCI_FAILURE);
2066 			}
2067 		}
2068 
2069 		free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
2070 		    AHCI_NCQ_SLOT_MASK(ahci_portp);
2071 	} else if (command_type == AHCI_ERR_RETRI_CMD) {
2072 		/* Error retrieval command request */
2073 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2074 		    "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
2075 		    "SENSE or READ LOG EXT command", NULL);
2076 		slot = 0;
2077 		goto out;
2078 	} else if (command_type == AHCI_RDWR_PMULT_CMD) {
2079 		/*
2080 		 * An extra check on PxCI. Sometimes PxCI bits may not be
2081 		 * cleared during hot-plug or error recovery process.
2082 		 */
2083 		port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2084 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));
2085 
2086 		if (port_cmd_issue != 0) {
2087 			AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
2088 			    "ahci_claim_free_slot: there is still pending "
2089 			    "command(s) in command list (0x%x/0x%x, PxCI %x),"
2090 			    "so no available slot for R/W PMULT command.",
2091 			    NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
2092 			    NCQ_CMD_IN_PROGRESS(ahci_portp),
2093 			    port_cmd_issue);
2094 			return (AHCI_FAILURE);
2095 		}
2096 
2097 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2098 		    "ahci_claim_free_slot: slot 0 is allocated for "
2099 		    "READ/WRITE PORTMULT command", NULL);
2100 		slot = 0;
2101 		goto out;
2102 	}
2103 
2104 	slot = ddi_ffs(free_slots) - 1;
2105 	if (slot == -1) {
2106 		AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2107 		    "ahci_claim_free_slot: no empty slots", NULL);
2108 		return (AHCI_FAILURE);
2109 	}
2110 
2111 	/*
2112 	 * According to the AHCI spec, to allow a simple mechanism for the
2113 	 * HBA to map command list slots to queue entries, software must
2114 	 * match the tag number it uses to the slot it is placing the command
2115 	 * in. For example, if a queued command is placed in slot 5, the tag
2116 	 * for that command must be 5.
2117 	 */
2118 	if (command_type == AHCI_NCQ_CMD) {
2119 		ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
2120 		if (AHCI_ADDR_IS_PMPORT(addrp)) {
2121 			ASSERT(ahci_portp->ahciport_pmult_info != NULL);
2122 			AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
2123 		}
2124 	}
2125 
2126 	ahci_portp->ahciport_pending_tags |= (0x1 << slot);
2127 
2128 out:
2129 	AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
2130 	    "ahci_claim_free_slot: found slot: 0x%x", slot);
2131 
2132 	return (slot);
2133 }
2134 
2135 /*
2136  * Builds the Command Table for the sata packet and delivers it to controller.
2137  *
2138  * Returns:
2139  *	slot number if we can obtain a slot successfully
2140  *	otherwise, return AHCI_FAILURE
2141  */
2142 static int
ahci_deliver_satapkt(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp,sata_pkt_t * spkt)2143 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
2144     ahci_addr_t *addrp, sata_pkt_t *spkt)
2145 {
2146 	int cmd_slot;
2147 	sata_cmd_t *scmd;
2148 	ahci_fis_h2d_register_t *h2d_register_fisp;
2149 	ahci_cmd_table_t *cmd_table;
2150 	ahci_cmd_header_t *cmd_header;
2151 	int ncookies;
2152 	int i;
2153 	int command_type = AHCI_NON_NCQ_CMD;
2154 	int ncq_qdepth;
2155 	int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
2156 	uint8_t port, pmport;
2157 #if AHCI_DEBUG
2158 	uint32_t *ptr;
2159 	uint8_t *ptr2;
2160 #endif
2161 
2162 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2163 
2164 	port = addrp->aa_port;
2165 	pmport = addrp->aa_pmport;
2166 
2167 	spkt->satapkt_reason = SATA_PKT_BUSY;
2168 
2169 	scmd = &spkt->satapkt_cmd;
2170 
2171 	/* Check if the command is a NCQ command */
2172 	if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED ||
2173 	    scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) {
2174 		command_type = AHCI_NCQ_CMD;
2175 
2176 		/*
2177 		 * When NCQ is support, system software must determine the
2178 		 * maximum tag allowed by the device and the HBA, and it
2179 		 * must use a value not beyond of the lower bound of the two.
2180 		 *
2181 		 * Sata module is going to calculate the qdepth and send
2182 		 * down to HBA driver via sata_cmd.
2183 		 */
2184 		ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1;
2185 
2186 		/*
2187 		 * At the moment, the driver doesn't support the dynamic
2188 		 * setting of the maximum ncq depth, and the value can be
2189 		 * set either during the attach or after hot-plug insertion.
2190 		 */
2191 		if (ahci_portp->ahciport_max_ncq_tags == 0) {
2192 			ahci_portp->ahciport_max_ncq_tags = ncq_qdepth;
2193 			AHCIDBG(AHCIDBG_NCQ, ahci_ctlp,
2194 			    "ahci_deliver_satapkt: port %d the max tags for "
2195 			    "NCQ command is %d", port, ncq_qdepth);
2196 		} else {
2197 			if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) {
2198 				cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt"
2199 				    " port %d the max tag for NCQ command is "
2200 				    "requested to change from %d to %d, at the"
2201 				    " moment the driver doesn't support the "
2202 				    "dynamic change so it's going to "
2203 				    "still use the previous tag value",
2204 				    instance, port,
2205 				    ahci_portp->ahciport_max_ncq_tags,
2206 				    ncq_qdepth);
2207 			}
2208 		}
2209 	}
2210 
2211 	/* Check if the command is an error retrieval command */
2212 	if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
2213 		command_type = AHCI_ERR_RETRI_CMD;
2214 
2215 	/* Check if the command is an read/write pmult command */
2216 	if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2217 		command_type = AHCI_RDWR_PMULT_CMD;
2218 
2219 	/* Check if there is an empty command slot */
2220 	cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp,
2221 	    addrp, command_type);
2222 	if (cmd_slot == AHCI_FAILURE) {
2223 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL);
2224 		return (AHCI_FAILURE);
2225 	}
2226 
2227 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
2228 	    "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
2229 	    "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
2230 	    cmd_slot, port, (void *)spkt);
2231 
2232 	cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
2233 	bzero((void *)cmd_table, ahci_cmd_table_size);
2234 
2235 	/* For data transfer operations, it is the H2D Register FIS */
2236 	h2d_register_fisp =
2237 	    &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);
2238 
2239 	SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
2240 
2241 	/*
2242 	 * PMP field only make sense when target is a port multiplier or a
2243 	 * device behind a port multiplier. Otherwise should set it to 0.
2244 	 */
2245 	if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2246 		SET_FIS_PMP(h2d_register_fisp, pmport);
2247 
2248 	SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
2249 	SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
2250 	SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
2251 	SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);
2252 
2253 	switch (scmd->satacmd_addr_type) {
2254 
2255 	case 0:
2256 		/*
2257 		 * satacmd_addr_type will be 0 for the commands below:
2258 		 *	ATAPI command
2259 		 *	SATAC_IDLE_IM
2260 		 *	SATAC_STANDBY_IM
2261 		 *	SATAC_DOWNLOAD_MICROCODE
2262 		 *	SATAC_FLUSH_CACHE
2263 		 *	SATAC_SET_FEATURES
2264 		 *	SATAC_SMART
2265 		 *	SATAC_ID_PACKET_DEVICE
2266 		 *	SATAC_ID_DEVICE
2267 		 *	SATAC_READ_PORTMULT
2268 		 *	SATAC_WRITE_PORTMULT
2269 		 */
2270 		/* FALLTHRU */
2271 
2272 	case ATA_ADDR_LBA:
2273 		/* FALLTHRU */
2274 
2275 	case ATA_ADDR_LBA28:
2276 		/* LBA[7:0] */
2277 		SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2278 
2279 		/* LBA[15:8] */
2280 		SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2281 
2282 		/* LBA[23:16] */
2283 		SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2284 
2285 		/* LBA [27:24] (also called dev_head) */
2286 		SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);
2287 
2288 		break;
2289 
2290 	case ATA_ADDR_LBA48:
2291 		/* LBA[7:0] */
2292 		SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);
2293 
2294 		/* LBA[15:8] */
2295 		SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);
2296 
2297 		/* LBA[23:16] */
2298 		SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);
2299 
2300 		/* LBA [31:24] */
2301 		SET_FIS_SECTOR_EXP(h2d_register_fisp,
2302 		    scmd->satacmd_lba_low_msb);
2303 
2304 		/* LBA [39:32] */
2305 		SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
2306 		    scmd->satacmd_lba_mid_msb);
2307 
2308 		/* LBA [47:40] */
2309 		SET_FIS_CYL_HI_EXP(h2d_register_fisp,
2310 		    scmd->satacmd_lba_high_msb);
2311 
2312 		/* Set dev_head */
2313 		SET_FIS_DEV_HEAD(h2d_register_fisp,
2314 		    scmd->satacmd_device_reg);
2315 
2316 		/* Set the extended sector count and features */
2317 		SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
2318 		    scmd->satacmd_sec_count_msb);
2319 		SET_FIS_FEATURES_EXP(h2d_register_fisp,
2320 		    scmd->satacmd_features_reg_ext);
2321 		break;
2322 	}
2323 
2324 	/*
2325 	 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is
2326 	 * filled into features field, and sector count 8:15 is filled into
2327 	 * features (exp) field. The hba driver doesn't need to anything
2328 	 * special with regard to this, since sata framework has already
2329 	 * done so.
2330 	 *
2331 	 * However the driver needs to make sure TAG is filled into sector
2332 	 * field.
2333 	 */
2334 	if (command_type == AHCI_NCQ_CMD) {
2335 		SET_FIS_SECTOR_COUNT(h2d_register_fisp,
2336 		    (cmd_slot << SATA_TAG_QUEUING_SHIFT));
2337 	}
2338 
2339 	ncookies = scmd->satacmd_num_dma_cookies;
2340 	AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2341 	    "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
2342 	    ncookies, ahci_dma_prdt_number);
2343 
2344 	ASSERT(ncookies <= ahci_dma_prdt_number);
2345 	ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0;
2346 
2347 	/* *** now fill the scatter gather list ******* */
2348 	for (i = 0; i < ncookies; i++) {
2349 		cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
2350 		    scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
2351 		cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
2352 		    scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
2353 		cmd_table->ahcict_prdt[i].ahcipi_descr_info =
2354 		    scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
2355 		ahci_portp->ahciport_prd_bytecounts[cmd_slot] +=
2356 		    scmd->satacmd_dma_cookie_list[i].dmac_size;
2357 	}
2358 
2359 	AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
2360 	    "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x",
2361 	    ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot);
2362 
2363 	/* The ACMD field is filled in for ATAPI command */
2364 	if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
2365 		bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
2366 		    SATA_ATAPI_MAX_CDB_LEN);
2367 	}
2368 
2369 	/* Set Command Header in Command List */
2370 	cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
2371 	BZERO_DESCR_INFO(cmd_header);
2372 	BZERO_PRD_BYTE_COUNT(cmd_header);
2373 
2374 	/* Set the number of entries in the PRD table */
2375 	SET_PRD_TABLE_LENGTH(cmd_header, ncookies);
2376 
2377 	/* Set the length of the command in the CFIS area */
2378 	SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);
2379 
2380 	/*
2381 	 * PMP field only make sense when target is a port multiplier or a
2382 	 * device behind a port multiplier. Otherwise should set it to 0.
2383 	 */
2384 	if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
2385 		SET_PORT_MULTI_PORT(cmd_header, pmport);
2386 
2387 	AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
2388 	    "sata_data_direction = 0x%x",
2389 	    scmd->satacmd_flags.sata_data_direction);
2390 
2391 	/* Set A bit if it is an ATAPI command */
2392 	if (scmd->satacmd_cmd_reg == SATAC_PACKET)
2393 		SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);
2394 
2395 	/* Set W bit if data is going to the device */
2396 	if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
2397 		SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);
2398 
2399 	/*
2400 	 * Set the prefetchable bit - this bit is only valid if the PRDTL
2401 	 * field is non-zero or the ATAPI 'A' bit is set in the command
2402 	 * header. This bit cannot be set when using native command
2403 	 * queuing commands or when using FIS-based switching with a Port
2404 	 * multiplier.
2405 	 */
2406 	if (command_type != AHCI_NCQ_CMD)
2407 		SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);
2408 
2409 	/*
2410 	 * Now remember the sata packet in ahciport_slot_pkts[].
2411 	 * Error retrieval command and r/w port multiplier command will
2412 	 * be stored specifically for each port.
2413 	 */
2414 	if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
2415 	    !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
2416 		ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;
2417 
2418 	/*
2419 	 * Keep the timeout value
2420 	 */
2421 	ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time;
2422 
2423 	/*
2424 	 * If the intial timout is less than 1 tick, then make it longer by
2425 	 * 1 tick to avoid immediate timeout
2426 	 */
2427 	if (ahci_portp->ahciport_slot_timeout[cmd_slot] <=
2428 	    ahci_watchdog_timeout)
2429 		ahci_portp->ahciport_slot_timeout[cmd_slot] +=
2430 		    ahci_watchdog_timeout;
2431 
2432 #if AHCI_DEBUG
2433 	if (ahci_debug_flags & AHCIDBG_ATACMD &&
2434 	    scmd->satacmd_cmd_reg != SATAC_PACKET ||
2435 	    ahci_debug_flags & AHCIDBG_ATAPICMD &&
2436 	    scmd->satacmd_cmd_reg == SATAC_PACKET) {
2437 
2438 		/* Dump the command header and table */
2439 		ahci_log(ahci_ctlp, CE_WARN, "\n");
2440 		ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt "
2441 		    "0x%p cmd_reg 0x%x port %d", spkt,
2442 		    scmd->satacmd_cmd_reg, port);
2443 		ptr = (uint32_t *)cmd_header;
2444 		ahci_log(ahci_ctlp, CE_WARN,
2445 		    "  Command Header:%8x %8x %8x %8x",
2446 		    ptr[0], ptr[1], ptr[2], ptr[3]);
2447 
2448 		/* Dump the H2D register FIS */
2449 		ptr = (uint32_t *)h2d_register_fisp;
2450 		ahci_log(ahci_ctlp, CE_WARN,
2451 		    "  Command FIS:   %8x %8x %8x %8x",
2452 		    ptr[0], ptr[1], ptr[2], ptr[3]);
2453 
2454 		/* Dump the ACMD register FIS */
2455 		ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
2456 		for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
2457 			if (ahci_debug_flags & AHCIDBG_ATAPICMD)
2458 				ahci_log(ahci_ctlp, CE_WARN,
2459 				    "  ATAPI command: %2x %2x %2x %2x "
2460 				    "%2x %2x %2x %2x",
2461 				    ptr2[8 * i], ptr2[8 * i + 1],
2462 				    ptr2[8 * i + 2], ptr2[8 * i + 3],
2463 				    ptr2[8 * i + 4], ptr2[8 * i + 5],
2464 				    ptr2[8 * i + 6], ptr2[8 * i + 7]);
2465 
2466 		/* Dump the PRDT */
2467 		for (i = 0; i < ncookies; i++) {
2468 			ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
2469 			ahci_log(ahci_ctlp, CE_WARN,
2470 			    "  Cookie %d:      %8x %8x %8x %8x",
2471 			    i, ptr[0], ptr[1], ptr[2], ptr[3]);
2472 		}
2473 	}
2474 #endif
2475 
2476 	(void) ddi_dma_sync(
2477 	    ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
2478 	    0,
2479 	    ahci_cmd_table_size,
2480 	    DDI_DMA_SYNC_FORDEV);
2481 
2482 	(void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
2483 	    cmd_slot * sizeof (ahci_cmd_header_t),
2484 	    sizeof (ahci_cmd_header_t),
2485 	    DDI_DMA_SYNC_FORDEV);
2486 
2487 	if ((ahci_check_dma_handle(ahci_portp->
2488 	    ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) ||
2489 	    ahci_check_dma_handle(ahci_portp->
2490 	    ahciport_cmd_list_dma_handle) != DDI_FM_OK) {
2491 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2492 		    DDI_SERVICE_UNAFFECTED);
2493 		return (AHCI_FAILURE);
2494 	}
2495 
2496 	/* Set the corresponding bit in the PxSACT.DS for queued command */
2497 	if (command_type == AHCI_NCQ_CMD) {
2498 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2499 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port),
2500 		    (0x1 << cmd_slot));
2501 	}
2502 
2503 	/* Indicate to the HBA that a command is active. */
2504 	ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
2505 	    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
2506 	    (0x1 << cmd_slot));
2507 
2508 	AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
2509 	    "exit: port %d", port);
2510 
2511 	/* Make sure the command is started by the PxSACT/PxCI */
2512 	if (ahci_check_acc_handle(ahci_ctlp->
2513 	    ahcictl_ahci_acc_handle) != DDI_FM_OK) {
2514 		ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
2515 		    DDI_SERVICE_UNAFFECTED);
2516 		return (AHCI_FAILURE);
2517 	}
2518 
2519 	return (cmd_slot);
2520 }
2521 
2522 /*
2523  * Called by the sata framework to abort the previously sent packet(s).
2524  *
2525  * Reset device to abort commands.
2526  */
2527 static int
ahci_tran_abort(dev_info_t * dip,sata_pkt_t * spkt,int flag)2528 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
2529 {
2530 	ahci_ctl_t *ahci_ctlp;
2531 	ahci_port_t *ahci_portp;
2532 	uint32_t slot_status = 0;
2533 	uint32_t aborted_tags = 0;
2534 	uint32_t finished_tags = 0;
2535 	uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
2536 	uint8_t port;
2537 	int tmp_slot;
2538 	int instance = ddi_get_instance(dip);
2539 
2540 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
2541 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
2542 
2543 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2544 	    "ahci_tran_abort enter: port %d", port);
2545 
2546 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
2547 	mutex_enter(&ahci_portp->ahciport_mutex);
2548 
2549 	/*
2550 	 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2551 	 * commands are being mopped, therefore there is nothing else to do
2552 	 */
2553 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2554 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2555 		    "ahci_tran_abort: port %d is in "
2556 		    "mopping process, so just return directly ", port);
2557 		mutex_exit(&ahci_portp->ahciport_mutex);
2558 		return (SATA_SUCCESS);
2559 	}
2560 
2561 	/*
2562 	 * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT
2563 	 * command is being executed so no other commands is outstanding,
2564 	 * nothing to do.
2565 	 */
2566 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) {
2567 		AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2568 		    "ahci_tran_abort: port %d is reading/writing "
2569 		    "port multiplier, so just return directly ", port);
2570 		mutex_exit(&ahci_portp->ahciport_mutex);
2571 		return (SATA_SUCCESS);
2572 	}
2573 
2574 	if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
2575 	    ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
2576 	    ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
2577 		/*
2578 		 * In case the targer driver would send the request before
2579 		 * sata framework can have the opportunity to process those
2580 		 * event reports.
2581 		 */
2582 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2583 		spkt->satapkt_device.satadev_state =
2584 		    ahci_portp->ahciport_port_state;
2585 		ahci_update_sata_registers(ahci_ctlp, port,
2586 		    &spkt->satapkt_device);
2587 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2588 		    "ahci_tran_abort returning SATA_FAILURE while "
2589 		    "port in FAILED/SHUTDOWN/PWROFF state: "
2590 		    "port: %d", port);
2591 		mutex_exit(&ahci_portp->ahciport_mutex);
2592 		return (SATA_FAILURE);
2593 	}
2594 
2595 	if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
2596 		/*
2597 		 * ahci_intr_phyrdy_change() may have rendered it to
2598 		 * AHCI_PORT_TYPE_NODEV.
2599 		 */
2600 		spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
2601 		spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
2602 		spkt->satapkt_device.satadev_state =
2603 		    ahci_portp->ahciport_port_state;
2604 		ahci_update_sata_registers(ahci_ctlp, port,
2605 		    &spkt->satapkt_device);
2606 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2607 		    "ahci_tran_abort returning SATA_FAILURE while "
2608 		    "no device attached: port: %d", port);
2609 		mutex_exit(&ahci_portp->ahciport_mutex);
2610 		return (SATA_FAILURE);
2611 	}
2612 
2613 	if (flag == SATA_ABORT_ALL_PACKETS) {
2614 		if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2615 			aborted_tags = ahci_portp->ahciport_pending_tags;
2616 		else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2617 			aborted_tags = ahci_portp->ahciport_pending_ncq_tags;
2618 
2619 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets",
2620 		    instance, port);
2621 	} else {
2622 		aborted_tags = 0xffffffff;
2623 		/*
2624 		 * Aborting one specific packet, first search the
2625 		 * ahciport_slot_pkts[] list for matching spkt.
2626 		 */
2627 		for (tmp_slot = 0;
2628 		    tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
2629 			if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
2630 				aborted_tags = (0x1 << tmp_slot);
2631 				break;
2632 			}
2633 		}
2634 
2635 		if (aborted_tags == 0xffffffff) {
2636 			/* request packet is not on the pending list */
2637 			AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
2638 			    "Cannot find the aborting pkt 0x%p on the "
2639 			    "pending list", (void *)spkt);
2640 			ahci_update_sata_registers(ahci_ctlp, port,
2641 			    &spkt->satapkt_device);
2642 			mutex_exit(&ahci_portp->ahciport_mutex);
2643 			return (SATA_FAILURE);
2644 		}
2645 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p",
2646 		    instance, port, (void *)spkt);
2647 	}
2648 
2649 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2650 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2651 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2652 	else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2653 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2654 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2655 
2656 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2657 	ahci_portp->ahciport_mop_in_progress++;
2658 
2659 	/*
2660 	 * To abort the packet(s), first we are trying to clear PxCMD.ST
2661 	 * to stop the port, and if the port can be stopped
2662 	 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
2663 	 * then we just send back the aborted packet(s) with ABORTED flag
2664 	 * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
2665 	 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
2666 	 * perform a COMRESET.
2667 	 */
2668 	(void) ahci_restart_port_wait_till_ready(ahci_ctlp,
2669 	    ahci_portp, port, 0, NULL);
2670 
2671 	/*
2672 	 * Compute which have finished and which need to be retried.
2673 	 *
2674 	 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags
2675 	 * minus the slot_status. The aborted_tags has to be deducted by
2676 	 * finished_tags since we can't possibly abort a tag which had finished
2677 	 * already.
2678 	 */
2679 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2680 		finished_tags = ahci_portp->ahciport_pending_tags &
2681 		    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2682 	else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2683 		finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2684 		    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2685 
2686 	aborted_tags &= ~finished_tags;
2687 
2688 	ahci_mop_commands(ahci_ctlp,
2689 	    ahci_portp,
2690 	    slot_status,
2691 	    0, /* failed tags */
2692 	    0, /* timeout tags */
2693 	    aborted_tags,
2694 	    0); /* reset tags */
2695 
2696 	ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
2697 	mutex_exit(&ahci_portp->ahciport_mutex);
2698 
2699 	return (SATA_SUCCESS);
2700 }
2701 
2702 /*
2703  * Used to do device reset and reject all the pending packets on a device
2704  * during the reset operation.
2705  *
2706  * NOTE: ONLY called by ahci_tran_reset_dport
2707  */
2708 static int
ahci_reset_device_reject_pkts(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)2709 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
2710     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2711 {
2712 	uint32_t slot_status = 0;
2713 	uint32_t reset_tags = 0;
2714 	uint32_t finished_tags = 0;
2715 	uint8_t port = addrp->aa_port;
2716 	sata_device_t sdevice;
2717 	int ret;
2718 
2719 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2720 
2721 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2722 	    "ahci_reset_device_reject_pkts on port: %d", port);
2723 
2724 	/*
2725 	 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2726 	 * commands are being mopped, therefore there is nothing else to do
2727 	 */
2728 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2729 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2730 		    "ahci_reset_device_reject_pkts: port %d is in "
2731 		    "mopping process, so return directly ", port);
2732 		return (SATA_SUCCESS);
2733 	}
2734 
2735 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2736 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2737 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2738 		reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2739 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2740 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2741 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2742 		reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2743 	}
2744 
2745 	if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2746 	    != AHCI_SUCCESS) {
2747 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2748 		    "Try to do a port reset after software "
2749 		    "reset failed", port);
2750 		ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
2751 		if (ret != AHCI_SUCCESS) {
2752 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2753 			    "ahci_reset_device_reject_pkts: port %d "
2754 			    "failed", port);
2755 			return (SATA_FAILURE);
2756 		}
2757 	}
2758 	/* Set the reset in progress flag */
2759 	ahci_portp->ahciport_reset_in_progress = 1;
2760 
2761 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2762 	ahci_portp->ahciport_mop_in_progress++;
2763 
2764 	/* Indicate to the framework that a reset has happened */
2765 	bzero((void *)&sdevice, sizeof (sata_device_t));
2766 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2767 	sdevice.satadev_addr.pmport = 0;
2768 	sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
2769 	sdevice.satadev_state = SATA_DSTATE_RESET |
2770 	    SATA_DSTATE_PWR_ACTIVE;
2771 	mutex_exit(&ahci_portp->ahciport_mutex);
2772 	sata_hba_event_notify(
2773 	    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2774 	    &sdevice,
2775 	    SATA_EVNT_DEVICE_RESET);
2776 	mutex_enter(&ahci_portp->ahciport_mutex);
2777 
2778 	AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2779 	    "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
2780 
2781 	/* Next try to mop the pending commands */
2782 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2783 		finished_tags = ahci_portp->ahciport_pending_tags &
2784 		    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2785 	else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2786 		finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2787 		    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2788 
2789 	reset_tags &= ~finished_tags;
2790 
2791 	ahci_mop_commands(ahci_ctlp,
2792 	    ahci_portp,
2793 	    slot_status,
2794 	    0, /* failed tags */
2795 	    0, /* timeout tags */
2796 	    0, /* aborted tags */
2797 	    reset_tags); /* reset tags */
2798 
2799 	return (SATA_SUCCESS);
2800 }
2801 
2802 /*
2803  * Used to do device reset and reject all the pending packets on a device
2804  * during the reset operation.
2805  *
2806  * NOTE: ONLY called by ahci_tran_reset_dport
2807  */
2808 static int
ahci_reset_pmdevice_reject_pkts(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)2809 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp,
2810     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2811 {
2812 	uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
2813 	uint8_t port = addrp->aa_port;
2814 	uint8_t pmport = addrp->aa_pmport;
2815 	sata_device_t sdevice;
2816 
2817 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2818 
2819 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp,
2820 	    "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport);
2821 
2822 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2823 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2824 		    "ahci_reset_pmdevice_reject_pkts: port %d is in "
2825 		    "mopping process, so return directly ", port);
2826 		return (SATA_SUCCESS);
2827 	}
2828 
2829 	/* Checking for outstanding commands */
2830 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2831 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2832 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2833 		reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2834 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2835 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2836 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2837 		reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2838 	}
2839 
2840 	/* Issue SOFTWARE reset command. */
2841 	if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
2842 	    != AHCI_SUCCESS) {
2843 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2844 		    "Try to do a port reset after software "
2845 		    "reset failed", port);
2846 		return (SATA_FAILURE);
2847 	}
2848 
2849 	/* Set the reset in progress flag */
2850 	ahci_portp->ahciport_reset_in_progress = 1;
2851 
2852 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2853 	ahci_portp->ahciport_mop_in_progress++;
2854 
2855 	/* Indicate to the framework that a reset has happened */
2856 	bzero((void *)&sdevice, sizeof (sata_device_t));
2857 	sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
2858 	sdevice.satadev_addr.pmport = pmport;
2859 	if (AHCI_ADDR_IS_PMULT(addrp))
2860 		sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
2861 	else
2862 		sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
2863 	sdevice.satadev_state = SATA_DSTATE_RESET |
2864 	    SATA_DSTATE_PWR_ACTIVE;
2865 	mutex_exit(&ahci_portp->ahciport_mutex);
2866 	sata_hba_event_notify(
2867 	    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
2868 	    &sdevice,
2869 	    SATA_EVNT_DEVICE_RESET);
2870 	mutex_enter(&ahci_portp->ahciport_mutex);
2871 
2872 	AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
2873 	    "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET",
2874 	    port, pmport);
2875 
2876 	/* Next try to mop the pending commands */
2877 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2878 		finished_tags = ahci_portp->ahciport_pending_tags &
2879 		    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2880 	else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2881 		finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2882 		    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2883 	reset_tags &= ~finished_tags;
2884 
2885 	AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
2886 	    "reset_tags = %x, finished_tags = %x, slot_status = %x",
2887 	    reset_tags, finished_tags, slot_status);
2888 
2889 	/*
2890 	 * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we
2891 	 * try to reset a single device behind a port multiplier will
2892 	 * terminate all the commands on that HBA port. We need mop these
2893 	 * commands as well.
2894 	 */
2895 	ahci_mop_commands(ahci_ctlp,
2896 	    ahci_portp,
2897 	    slot_status,
2898 	    0, /* failed tags */
2899 	    0, /* timeout tags */
2900 	    0, /* aborted tags */
2901 	    reset_tags); /* reset tags */
2902 
2903 	return (SATA_SUCCESS);
2904 }
2905 
2906 /*
2907  * Used to do port reset and reject all the pending packets on a port during
2908  * the reset operation.
2909  */
2910 static int
ahci_reset_port_reject_pkts(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,ahci_addr_t * addrp)2911 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
2912     ahci_port_t *ahci_portp, ahci_addr_t *addrp)
2913 {
2914 	uint32_t slot_status = 0;
2915 	uint32_t reset_tags = 0;
2916 	uint32_t finished_tags = 0;
2917 	uint8_t port = addrp->aa_port;
2918 
2919 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
2920 
2921 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2922 	    "ahci_reset_port_reject_pkts at port: %d", port);
2923 
2924 	/*
2925 	 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
2926 	 * commands are being mopped, therefore there is nothing else to do
2927 	 */
2928 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
2929 		AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
2930 		    "ahci_reset_port_reject_pkts: port %d is in "
2931 		    "mopping process, so return directly ", port);
2932 		return (SATA_SUCCESS);
2933 	}
2934 
2935 	ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
2936 	ahci_portp->ahciport_mop_in_progress++;
2937 
2938 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2939 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2940 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
2941 		reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2942 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
2943 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
2944 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
2945 		reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2946 	}
2947 
2948 	if (ahci_restart_port_wait_till_ready(ahci_ctlp,
2949 	    ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
2950 	    NULL) != AHCI_SUCCESS) {
2951 
2952 		/* Clear mop flag */
2953 		ahci_portp->ahciport_mop_in_progress--;
2954 		if (ahci_portp->ahciport_mop_in_progress == 0)
2955 			ahci_portp->ahciport_flags &=
2956 			    ~AHCI_PORT_FLAG_MOPPING;
2957 		return (SATA_FAILURE);
2958 	}
2959 
2960 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
2961 		finished_tags = ahci_portp->ahciport_pending_tags &
2962 		    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
2963 	else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
2964 		finished_tags = ahci_portp->ahciport_pending_ncq_tags &
2965 		    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
2966 
2967 	reset_tags &= ~finished_tags;
2968 
2969 	ahci_mop_commands(ahci_ctlp,
2970 	    ahci_portp,
2971 	    slot_status,
2972 	    0, /* failed tags */
2973 	    0, /* timeout tags */
2974 	    0, /* aborted tags */
2975 	    reset_tags); /* reset tags */
2976 
2977 	return (SATA_SUCCESS);
2978 }
2979 
2980 /*
2981  * Used to do hba reset and reject all the pending packets on all ports
2982  * during the reset operation.
2983  */
2984 static int
ahci_reset_hba_reject_pkts(ahci_ctl_t * ahci_ctlp)2985 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
2986 {
2987 	ahci_port_t *ahci_portp;
2988 	uint32_t slot_status[AHCI_MAX_PORTS];
2989 	uint32_t reset_tags[AHCI_MAX_PORTS];
2990 	uint32_t finished_tags[AHCI_MAX_PORTS];
2991 	int port;
2992 	int ret = SATA_SUCCESS;
2993 
2994 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
2995 	    "ahci_reset_hba_reject_pkts enter", NULL);
2996 
2997 	bzero(slot_status, sizeof (slot_status));
2998 	bzero(reset_tags, sizeof (reset_tags));
2999 	bzero(finished_tags, sizeof (finished_tags));
3000 
3001 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3002 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3003 			continue;
3004 		}
3005 
3006 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
3007 
3008 		mutex_enter(&ahci_portp->ahciport_mutex);
3009 		ahci_portp->ahciport_reset_in_progress = 1;
3010 		if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3011 			slot_status[port] = ddi_get32(
3012 			    ahci_ctlp->ahcictl_ahci_acc_handle,
3013 			    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3014 			reset_tags[port] = slot_status[port] &
3015 			    AHCI_SLOT_MASK(ahci_ctlp);
3016 			AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3017 			    "port %d: reset_tags = 0x%x pending_tags = 0x%x",
3018 			    port, reset_tags[port],
3019 			    ahci_portp->ahciport_pending_tags);
3020 		} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3021 			slot_status[port] = ddi_get32(
3022 			    ahci_ctlp->ahcictl_ahci_acc_handle,
3023 			    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3024 			reset_tags[port] = slot_status[port] &
3025 			    AHCI_NCQ_SLOT_MASK(ahci_portp);
3026 			AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3027 			    "port %d: reset_tags = 0x%x pending_tags = 0x%x",
3028 			    port, reset_tags[port],
3029 			    ahci_portp->ahciport_pending_tags);
3030 		}
3031 		mutex_exit(&ahci_portp->ahciport_mutex);
3032 	}
3033 
3034 	if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
3035 		ret = SATA_FAILURE;
3036 	}
3037 
3038 	for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
3039 		if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
3040 			continue;
3041 		}
3042 
3043 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
3044 
3045 		mutex_enter(&ahci_portp->ahciport_mutex);
3046 		/*
3047 		 * To prevent recursive enter to ahci_mop_commands, we need
3048 		 * check AHCI_PORT_FLAG_MOPPING flag.
3049 		 */
3050 		if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3051 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3052 			    "ahci_reset_hba_reject_pkts: port %d is in "
3053 			    "mopping process, so return directly ", port);
3054 			mutex_exit(&ahci_portp->ahciport_mutex);
3055 			continue;
3056 		}
3057 
3058 		ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3059 		ahci_portp->ahciport_mop_in_progress++;
3060 
3061 		if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
3062 			finished_tags[port]  =
3063 			    ahci_portp->ahciport_pending_tags &
3064 			    ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
3065 		else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
3066 			finished_tags[port] =
3067 			    ahci_portp->ahciport_pending_ncq_tags &
3068 			    ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp);
3069 
3070 		reset_tags[port] &= ~finished_tags[port];
3071 
3072 		ahci_mop_commands(ahci_ctlp,
3073 		    ahci_portp,
3074 		    slot_status[port],
3075 		    0, /* failed tags */
3076 		    0, /* timeout tags */
3077 		    0, /* aborted tags */
3078 		    reset_tags[port]); /* reset tags */
3079 		mutex_exit(&ahci_portp->ahciport_mutex);
3080 	}
3081 out:
3082 	return (ret);
3083 }
3084 
3085 /*
3086  * Called by sata framework to reset a port(s) or device.
3087  */
3088 static int
ahci_tran_reset_dport(dev_info_t * dip,sata_device_t * sd)3089 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
3090 {
3091 	ahci_ctl_t *ahci_ctlp;
3092 	ahci_port_t *ahci_portp;
3093 	ahci_addr_t addr;
3094 	uint8_t cport = sd->satadev_addr.cport;
3095 	uint8_t pmport = sd->satadev_addr.pmport;
3096 	uint8_t port;
3097 	int ret = SATA_SUCCESS;
3098 	int instance = ddi_get_instance(dip);
3099 
3100 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3101 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
3102 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
3103 
3104 	ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
3105 
3106 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3107 	    "ahci_tran_reset_dport enter: cport %d", cport);
3108 
3109 	switch (sd->satadev_addr.qual) {
3110 	case SATA_ADDR_PMPORT:
3111 		/*
3112 		 * If we want to issue a COMRESET on a pmport, we need to
3113 		 * reject the outstanding commands on that pmport. According
3114 		 * to AHCI spec, PxCI register could only be cleared by
3115 		 * clearing PxCMD.ST, which will halt the controller port - as
3116 		 * well as other pmports.
3117 		 *
3118 		 * Therefore we directly reset the controller port for
3119 		 * simplicity. ahci_tran_probe_port() will handle reset stuff
3120 		 * like initializing the given pmport.
3121 		 */
3122 		/* FALLTHRU */
3123 	case SATA_ADDR_CPORT:
3124 		/* Port reset */
3125 		ahci_portp = ahci_ctlp->ahcictl_ports[port];
3126 		cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3127 		    "port %d reset port", instance, port);
3128 
3129 		mutex_enter(&ahci_portp->ahciport_mutex);
3130 		ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr);
3131 		mutex_exit(&ahci_portp->ahciport_mutex);
3132 
3133 		break;
3134 
3135 	case SATA_ADDR_DPMPORT:
3136 		cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3137 		    "port %d:%d reset device", instance, port, pmport);
3138 		/* FALLTHRU */
3139 	case SATA_ADDR_DCPORT:
3140 		/* Device reset */
3141 		if (sd->satadev_addr.qual == SATA_ADDR_DCPORT)
3142 			cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3143 			    "port %d reset device", instance, port);
3144 
3145 		mutex_enter(&ahci_portp->ahciport_mutex);
3146 		/*
3147 		 * software reset request must be sent to SATA_PMULT_HOSTPORT
3148 		 * if target is a port multiplier:
3149 		 */
3150 		if (sd->satadev_addr.qual == SATA_ADDR_DCPORT &&
3151 		    ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT)
3152 			AHCI_ADDR_SET_PMULT(&addr, port);
3153 
3154 		if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
3155 		    ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
3156 		    ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
3157 			/*
3158 			 * In case the targer driver would send the request
3159 			 * before sata framework can have the opportunity to
3160 			 * process those event reports.
3161 			 */
3162 			sd->satadev_state = ahci_portp->ahciport_port_state;
3163 			ahci_update_sata_registers(ahci_ctlp, port, sd);
3164 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3165 			    "ahci_tran_reset_dport returning SATA_FAILURE "
3166 			    "while port in FAILED/SHUTDOWN/PWROFF state: "
3167 			    "port: %d", port);
3168 			mutex_exit(&ahci_portp->ahciport_mutex);
3169 			ret = SATA_FAILURE;
3170 			break;
3171 		}
3172 
3173 		if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) ==
3174 		    SATA_DTYPE_NONE) {
3175 			/*
3176 			 * ahci_intr_phyrdy_change() may have rendered it to
3177 			 * AHCI_PORT_TYPE_NODEV.
3178 			 */
3179 			sd->satadev_type = SATA_DTYPE_NONE;
3180 			sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp,
3181 			    &addr);
3182 			ahci_update_sata_registers(ahci_ctlp, port, sd);
3183 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3184 			    "ahci_tran_reset_dport returning SATA_FAILURE "
3185 			    "while no device attached: port: %d", port);
3186 			mutex_exit(&ahci_portp->ahciport_mutex);
3187 			ret = SATA_FAILURE;
3188 			break;
3189 		}
3190 
3191 		if (AHCI_ADDR_IS_PORT(&addr)) {
3192 			ret = ahci_reset_device_reject_pkts(ahci_ctlp,
3193 			    ahci_portp, &addr);
3194 		} else {
3195 			ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp,
3196 			    ahci_portp, &addr);
3197 		}
3198 
3199 		mutex_exit(&ahci_portp->ahciport_mutex);
3200 		break;
3201 
3202 	case SATA_ADDR_CNTRL:
3203 		/* Reset the whole controller */
3204 		cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
3205 		    "reset the whole hba", instance);
3206 		ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
3207 		break;
3208 
3209 	default:
3210 		ret = SATA_FAILURE;
3211 	}
3212 
3213 	return (ret);
3214 }
3215 
3216 /*
3217  * Called by sata framework to activate a port as part of hotplug.
3218  * (cfgadm -c connect satax/y)
3219  * Support port multiplier.
3220  */
3221 static int
ahci_tran_hotplug_port_activate(dev_info_t * dip,sata_device_t * satadev)3222 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
3223 {
3224 	ahci_ctl_t *ahci_ctlp;
3225 	ahci_port_t *ahci_portp;
3226 	ahci_addr_t addr;
3227 	uint8_t	cport = satadev->satadev_addr.cport;
3228 	uint8_t	pmport = satadev->satadev_addr.pmport;
3229 	uint8_t port;
3230 	int instance = ddi_get_instance(dip);
3231 
3232 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3233 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
3234 
3235 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3236 	    "ahci_tran_hotplug_port_activate enter: cport %d", cport);
3237 
3238 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
3239 
3240 	mutex_enter(&ahci_portp->ahciport_mutex);
3241 	ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3242 	ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3243 
3244 	if (AHCI_ADDR_IS_PORT(&addr)) {
3245 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated",
3246 		    instance, port);
3247 
3248 		/* Enable the interrupts on the port */
3249 		ahci_enable_port_intrs(ahci_ctlp, port);
3250 
3251 		/*
3252 		 * Reset the port so that the PHY communication would be
3253 		 * re-established.  But this reset is an internal operation
3254 		 * and the sata module doesn't need to know about it.
3255 		 * Moreover, the port with a device attached will be started
3256 		 * too.
3257 		 */
3258 		(void) ahci_restart_port_wait_till_ready(ahci_ctlp,
3259 		    ahci_portp, port,
3260 		    AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
3261 		    NULL);
3262 
3263 		/*
3264 		 * Need to check the link status and device status of the port
3265 		 * and consider raising power if the port was in D3 state
3266 		 */
3267 		ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
3268 		ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
3269 		ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
3270 	} else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3271 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated",
3272 		    instance, port, pmport);
3273 		/* AHCI_ADDR_PMPORT */
3274 		AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON;
3275 		AHCIPORT_PMSTATE(ahci_portp, &addr) &=
3276 		    ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN);
3277 	}
3278 
3279 	satadev->satadev_state = ahci_portp->ahciport_port_state;
3280 
3281 	ahci_update_sata_registers(ahci_ctlp, port, satadev);
3282 
3283 	mutex_exit(&ahci_portp->ahciport_mutex);
3284 	return (SATA_SUCCESS);
3285 }
3286 
3287 /*
3288  * Called by sata framework to deactivate a port as part of hotplug.
3289  * (cfgadm -c disconnect satax/y)
3290  * Support port multiplier.
3291  */
3292 static int
ahci_tran_hotplug_port_deactivate(dev_info_t * dip,sata_device_t * satadev)3293 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
3294 {
3295 	ahci_ctl_t *ahci_ctlp;
3296 	ahci_port_t *ahci_portp;
3297 	ahci_addr_t addr;
3298 	uint8_t	cport = satadev->satadev_addr.cport;
3299 	uint8_t	pmport = satadev->satadev_addr.pmport;
3300 	uint8_t port;
3301 	uint32_t port_scontrol;
3302 	int instance = ddi_get_instance(dip);
3303 
3304 	ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
3305 	port = ahci_ctlp->ahcictl_cport_to_port[cport];
3306 
3307 	AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
3308 	    "ahci_tran_hotplug_port_deactivate enter: cport %d", cport);
3309 
3310 	ahci_portp = ahci_ctlp->ahcictl_ports[port];
3311 	mutex_enter(&ahci_portp->ahciport_mutex);
3312 	ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
3313 	ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));
3314 
3315 	if (AHCI_ADDR_IS_PORT(&addr)) {
3316 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated",
3317 		    instance, port);
3318 
3319 		/* Disable the interrupts on the port */
3320 		ahci_disable_port_intrs(ahci_ctlp, port);
3321 
3322 		if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
3323 
3324 			/* First to abort all the pending commands */
3325 			ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3326 
3327 			/* Then stop the port */
3328 			(void) ahci_put_port_into_notrunning_state(ahci_ctlp,
3329 			    ahci_portp, port);
3330 		}
3331 
3332 		/* Next put the PHY offline */
3333 		port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3334 		    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
3335 		SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE);
3336 		ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *)
3337 		    AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
3338 	} else if (AHCI_ADDR_IS_PMPORT(&addr)) {
3339 		cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated",
3340 		    instance, port, pmport);
3341 
3342 		ahci_disable_port_intrs(ahci_ctlp, port);
3343 		if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr)
3344 		    != SATA_DTYPE_NONE)
3345 			ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
3346 
3347 		/* Re-enable the interrupts for the other pmports */
3348 		ahci_enable_port_intrs(ahci_ctlp, port);
3349 	}
3350 
3351 	/* Update port state */
3352 	AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN);
3353 	satadev->satadev_state = SATA_PSTATE_SHUTDOWN;
3354 
3355 	ahci_update_sata_registers(ahci_ctlp, port, satadev);
3356 
3357 	mutex_exit(&ahci_portp->ahciport_mutex);
3358 	return (SATA_SUCCESS);
3359 }
3360 
3361 /*
3362  * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
3363  * when a device is unplugged or a port is deactivated.
3364  */
3365 static void
ahci_reject_all_abort_pkts(ahci_ctl_t * ahci_ctlp,ahci_port_t * ahci_portp,uint8_t port)3366 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
3367     ahci_port_t *ahci_portp, uint8_t port)
3368 {
3369 	uint32_t slot_status = 0;
3370 	uint32_t abort_tags = 0;
3371 
3372 	ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
3373 
3374 	AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
3375 	    "ahci_reject_all_abort_pkts at port: %d", port);
3376 
3377 	/* Read/write port multiplier command takes highest priority */
3378 	if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
3379 		slot_status = 0x1;
3380 		abort_tags = 0x1;
3381 		goto out;
3382 	}
3383 
3384 	/*
3385 	 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a
3386 	 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA
3387 	 * to get the error data, if yes when the device is removed, the
3388 	 * command needs to be aborted too.
3389 	 */
3390 	if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
3391 		if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
3392 			slot_status = 0x1;
3393 			abort_tags = 0x1;
3394 			goto out;
3395 		} else {
3396 			AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
3397 			    "ahci_reject_all_abort_pkts return directly "
3398 			    "port %d no needs to reject any outstanding "
3399 			    "commands", port);
3400 			return;
3401 		}
3402 	}
3403 
3404 	if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3405 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3406 		    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
3407 		abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
3408 	} else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
3409 		slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
3410 		    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
3411 		abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
3412 	}
3413 
3414 out:
3415 	/* No need to do mop when there is no outstanding commands */
3416 	if (slot_status != 0) {
3417 		ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
3418 		ahci_portp->ahciport_mop_in_progress++;
3419 
3420 		ahci_mop_commands(ahci_ctlp,
3421 		    ahci_portp,
3422 		    slot_status,
3423 		    0, /* failed tags */
3424 		    0, /* timeout tags */
3425 		    abort_tags, /* aborting tags */
3426 		    0); /* reset tags */
3427 	}
3428 }
3429 
3430 #if defined(__lock_lint)
3431 static int
ahci_selftest(dev_info_t * dip,sata_device_t * device)3432 ahci_selftest(dev_info_t *dip, sata_device_t *device)
3433 {
3434 	return (SATA_SUCCESS);
3435 }
3436 #endif
3437 
3438 /*
3439  * Initialize fma capabilities and register with IO fault services.
3440  */
3441 static void
ahci_fm_init(ahci_ctl_t * ahci_ctlp)3442 ahci_fm_init(ahci_ctl_t *ahci_ctlp)
3443 {
3444 	/*
3445 	 * Need to change iblock to priority for new MSI intr
3446 	 */
3447 	ddi_iblock_cookie_t fm_ibc;
3448 
3449 	ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY,
3450 	    ahci_ctlp->ahcictl_dip,
3451 	    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
3452 	    DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
3453 	    DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);
3454 
3455 	/* Only register with IO Fault Services if we have some capability */
3456 	if (ahci_ctlp->ahcictl_fm_cap) {
3457 		/* Adjust access and dma attributes for FMA */
3458 		accattr.devacc_attr_access = DDI_FLAGERR_ACC;
3459 		buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3460 		rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3461 		cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3462 		cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
3463 
3464 		/*
3465 		 * Register capabilities with IO Fault Services.
3466 		 * ahcictl_fm_cap will be updated to indicate
3467 		 * capabilities actually supported (not requested.)
3468 		 */
3469 		ddi_fm_init(ahci_ctlp->ahcictl_dip,
3470 		    &ahci_ctlp->ahcictl_fm_cap, &fm_ibc);
3471 
3472 		if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) {
3473 			cmn_err(CE_WARN, "!ahci%d: fma init failed.",
3474 			    ddi_get_instance(ahci_ctlp->ahcictl_dip));
3475 			return;
3476 		}
3477 		/*
3478 		 * Initialize pci ereport capabilities if ereport
3479 		 * capable (should always be.)
3480 		 */
3481 		if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3482 		    DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3483 			pci_ereport_setup(ahci_ctlp->ahcictl_dip);
3484 		}
3485 
3486 		/*
3487 		 * Register error callback if error callback capable.
3488 		 */
3489 		if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3490 			ddi_fm_handler_register(ahci_ctlp->ahcictl_dip,
3491 			    ahci_fm_error_cb, (void *) ahci_ctlp);
3492 		}
3493 
3494 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3495 		    "ahci_fm_fini: fma enabled.", NULL);
3496 	}
3497 }
3498 
3499 /*
3500  * Releases fma capabilities and un-registers with IO fault services.
3501  */
3502 static void
ahci_fm_fini(ahci_ctl_t * ahci_ctlp)3503 ahci_fm_fini(ahci_ctl_t *ahci_ctlp)
3504 {
3505 	/* Only unregister FMA capabilities if registered */
3506 	if (ahci_ctlp->ahcictl_fm_cap) {
3507 		/*
3508 		 * Un-register error callback if error callback capable.
3509 		 */
3510 		if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3511 			ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip);
3512 		}
3513 
3514 		/*
3515 		 * Release any resources allocated by pci_ereport_setup()
3516 		 */
3517 		if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
3518 		    DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
3519 			pci_ereport_teardown(ahci_ctlp->ahcictl_dip);
3520 		}
3521 
3522 		/* Unregister from IO Fault Services */
3523 		ddi_fm_fini(ahci_ctlp->ahcictl_dip);
3524 
3525 		/* Adjust access and dma attributes for FMA */
3526 		accattr.devacc_attr_access = DDI_DEFAULT_ACC;
3527 		buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3528 		rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3529 		cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3530 		cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
3531 
3532 		AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
3533 		    "ahci_fm_fini: fma disabled.", NULL);
3534 	}
3535 }
3536 
3537 /*ARGSUSED*/
3538 static int
ahci_fm_error_cb(dev_info_t * dip,ddi_fm_error_t * err,const void * impl_data)3539 ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *