13c9168faSHans Rosenfeld /* 23c9168faSHans Rosenfeld * This file and its contents are supplied under the terms of the 33c9168faSHans Rosenfeld * Common Development and Distribution License ("CDDL"), version 1.0. 43c9168faSHans Rosenfeld * You may only use this file in accordance with the terms of version 53c9168faSHans Rosenfeld * 1.0 of the CDDL. 63c9168faSHans Rosenfeld * 73c9168faSHans Rosenfeld * A full copy of the text of the CDDL should have accompanied this 83c9168faSHans Rosenfeld * source. A copy of the CDDL is also available via the Internet at 93c9168faSHans Rosenfeld * http://www.illumos.org/license/CDDL. 103c9168faSHans Rosenfeld */ 113c9168faSHans Rosenfeld 123c9168faSHans Rosenfeld /* 136afc9eb2SHans Rosenfeld * Copyright 2016 Nexenta Systems, Inc. All rights reserved. 1434c938c7SPete Shephard * Copyright 2016 Tegile Systems, Inc. All rights reserved. 152f95345bSYouzhong Yang * Copyright (c) 2016 The MathWorks, Inc. All rights reserved. 163c9168faSHans Rosenfeld */ 173c9168faSHans Rosenfeld 183c9168faSHans Rosenfeld /* 193c9168faSHans Rosenfeld * blkdev driver for NVMe compliant storage devices 203c9168faSHans Rosenfeld * 213c9168faSHans Rosenfeld * This driver was written to conform to version 1.0e of the NVMe specification. 223c9168faSHans Rosenfeld * It may work with newer versions, but that is completely untested and disabled 233c9168faSHans Rosenfeld * by default. 243c9168faSHans Rosenfeld * 253c9168faSHans Rosenfeld * The driver has only been tested on x86 systems and will not work on big- 263c9168faSHans Rosenfeld * endian systems without changes to the code accessing registers and data 273c9168faSHans Rosenfeld * structures used by the hardware. 283c9168faSHans Rosenfeld * 293c9168faSHans Rosenfeld * 303c9168faSHans Rosenfeld * Interrupt Usage: 313c9168faSHans Rosenfeld * 323c9168faSHans Rosenfeld * The driver will use a FIXED interrupt while configuring the device as the 333c9168faSHans Rosenfeld * specification requires. Later in the attach process it will switch to MSI-X 343c9168faSHans Rosenfeld * or MSI if supported. The driver wants to have one interrupt vector per CPU, 353c9168faSHans Rosenfeld * but it will work correctly if less are available. Interrupts can be shared 363c9168faSHans Rosenfeld * by queues, the interrupt handler will iterate through the I/O queue array by 373c9168faSHans Rosenfeld * steps of n_intr_cnt. Usually only the admin queue will share an interrupt 383c9168faSHans Rosenfeld * with one I/O queue. The interrupt handler will retrieve completed commands 393c9168faSHans Rosenfeld * from all queues sharing an interrupt vector and will post them to a taskq 403c9168faSHans Rosenfeld * for completion processing. 413c9168faSHans Rosenfeld * 423c9168faSHans Rosenfeld * 433c9168faSHans Rosenfeld * Command Processing: 443c9168faSHans Rosenfeld * 453c9168faSHans Rosenfeld * NVMe devices can have up to 65536 I/O queue pairs, with each queue holding up 463c9168faSHans Rosenfeld * to 65536 I/O commands. The driver will configure one I/O queue pair per 473c9168faSHans Rosenfeld * available interrupt vector, with the queue length usually much smaller than 483c9168faSHans Rosenfeld * the maximum of 65536. If the hardware doesn't provide enough queues, fewer 493c9168faSHans Rosenfeld * interrupt vectors will be used. 503c9168faSHans Rosenfeld * 513c9168faSHans Rosenfeld * Additionally the hardware provides a single special admin queue pair that can 523c9168faSHans Rosenfeld * hold up to 4096 admin commands. 533c9168faSHans Rosenfeld * 543c9168faSHans Rosenfeld * From the hardware perspective both queues of a queue pair are independent, 553c9168faSHans Rosenfeld * but they share some driver state: the command array (holding pointers to 563c9168faSHans Rosenfeld * commands currently being processed by the hardware) and the active command 573c9168faSHans Rosenfeld * counter. Access to the submission side of a queue pair and the shared state 583c9168faSHans Rosenfeld * is protected by nq_mutex. The completion side of a queue pair does not need 593c9168faSHans Rosenfeld * that protection apart from its access to the shared state; it is called only 603c9168faSHans Rosenfeld * in the interrupt handler which does not run concurrently for the same 613c9168faSHans Rosenfeld * interrupt vector. 623c9168faSHans Rosenfeld * 633c9168faSHans Rosenfeld * When a command is submitted to a queue pair the active command counter is 643c9168faSHans Rosenfeld * incremented and a pointer to the command is stored in the command array. The 653c9168faSHans Rosenfeld * array index is used as command identifier (CID) in the submission queue 663c9168faSHans Rosenfeld * entry. Some commands may take a very long time to complete, and if the queue 673c9168faSHans Rosenfeld * wraps around in that time a submission may find the next array slot to still 683c9168faSHans Rosenfeld * be used by a long-running command. In this case the array is sequentially 693c9168faSHans Rosenfeld * searched for the next free slot. The length of the command array is the same 703c9168faSHans Rosenfeld * as the configured queue length. 713c9168faSHans Rosenfeld * 723c9168faSHans Rosenfeld * 733c9168faSHans Rosenfeld * Namespace Support: 743c9168faSHans Rosenfeld * 753c9168faSHans Rosenfeld * NVMe devices can have multiple namespaces, each being a independent data 763c9168faSHans Rosenfeld * store. The driver supports multiple namespaces and creates a blkdev interface 773c9168faSHans Rosenfeld * for each namespace found. Namespaces can have various attributes to support 789cfb65faSHans Rosenfeld * thin provisioning and protection information. This driver does not support 799cfb65faSHans Rosenfeld * any of this and ignores namespaces that have these attributes. 803c9168faSHans Rosenfeld * 813c9168faSHans Rosenfeld * 823c9168faSHans Rosenfeld * Blkdev Interface: 833c9168faSHans Rosenfeld * 843c9168faSHans Rosenfeld * This driver uses blkdev to do all the heavy lifting involved with presenting 853c9168faSHans Rosenfeld * a disk device to the system. As a result, the processing of I/O requests is 863c9168faSHans Rosenfeld * relatively simple as blkdev takes care of partitioning, boundary checks, DMA 873c9168faSHans Rosenfeld * setup, and splitting of transfers into manageable chunks. 883c9168faSHans Rosenfeld * 893c9168faSHans Rosenfeld * I/O requests coming in from blkdev are turned into NVM commands and posted to 903c9168faSHans Rosenfeld * an I/O queue. The queue is selected by taking the CPU id modulo the number of 913c9168faSHans Rosenfeld * queues. There is currently no timeout handling of I/O commands. 923c9168faSHans Rosenfeld * 933c9168faSHans Rosenfeld * Blkdev also supports querying device/media information and generating a 943c9168faSHans Rosenfeld * devid. The driver reports the best block size as determined by the namespace 953c9168faSHans Rosenfeld * format back to blkdev as physical block size to support partition and block 963c9168faSHans Rosenfeld * alignment. The devid is composed using the device vendor ID, model number, 973c9168faSHans Rosenfeld * serial number, and the namespace ID. 983c9168faSHans Rosenfeld * 993c9168faSHans Rosenfeld * 1003c9168faSHans Rosenfeld * Error Handling: 1013c9168faSHans Rosenfeld * 1023c9168faSHans Rosenfeld * Error handling is currently limited to detecting fatal hardware errors, 1033c9168faSHans Rosenfeld * either by asynchronous events, or synchronously through command status or 1043c9168faSHans Rosenfeld * admin command timeouts. In case of severe errors the device is fenced off, 1053c9168faSHans Rosenfeld * all further requests will return EIO. FMA is then called to fault the device. 1063c9168faSHans Rosenfeld * 1073c9168faSHans Rosenfeld * The hardware has a limit for outstanding asynchronous event requests. Before 1083c9168faSHans Rosenfeld * this limit is known the driver assumes it is at least 1 and posts a single 1093c9168faSHans Rosenfeld * asynchronous request. Later when the limit is known more asynchronous event 1103c9168faSHans Rosenfeld * requests are posted to allow quicker reception of error information. When an 1113c9168faSHans Rosenfeld * asynchronous event is posted by the hardware the driver will parse the error 1123c9168faSHans Rosenfeld * status fields and log information or fault the device, depending on the 1133c9168faSHans Rosenfeld * severity of the asynchronous event. The asynchronous event request is then 1143c9168faSHans Rosenfeld * reused and posted to the admin queue again. 1153c9168faSHans Rosenfeld * 1163c9168faSHans Rosenfeld * On command completion the command status is checked for errors. In case of 1173c9168faSHans Rosenfeld * errors indicating a driver bug the driver panics. Almost all other error 1183c9168faSHans Rosenfeld * status values just cause EIO to be returned. 1193c9168faSHans Rosenfeld * 1203c9168faSHans Rosenfeld * Command timeouts are currently detected for all admin commands except 1213c9168faSHans Rosenfeld * asynchronous event requests. If a command times out and the hardware appears 1223c9168faSHans Rosenfeld * to be healthy the driver attempts to abort the command. If this fails the 1233c9168faSHans Rosenfeld * driver assumes the device to be dead, fences it off, and calls FMA to retire 1243c9168faSHans Rosenfeld * it. In general admin commands are issued at attach time only. No timeout 1253c9168faSHans Rosenfeld * handling of normal I/O commands is presently done. 1263c9168faSHans Rosenfeld * 1273c9168faSHans Rosenfeld * In some cases it may be possible that the ABORT command times out, too. In 1283c9168faSHans Rosenfeld * that case the device is also declared dead and fenced off. 1293c9168faSHans Rosenfeld * 1303c9168faSHans Rosenfeld * 1313c9168faSHans Rosenfeld * Quiesce / Fast Reboot: 1323c9168faSHans Rosenfeld * 1333c9168faSHans Rosenfeld * The driver currently does not support fast reboot. A quiesce(9E) entry point 1343c9168faSHans Rosenfeld * is still provided which is used to send a shutdown notification to the 1353c9168faSHans Rosenfeld * device. 1363c9168faSHans Rosenfeld * 1373c9168faSHans Rosenfeld * 1383c9168faSHans Rosenfeld * Driver Configuration: 1393c9168faSHans Rosenfeld * 1403c9168faSHans Rosenfeld * The following driver properties can be changed to control some aspects of the 1413c9168faSHans Rosenfeld * drivers operation: 1423c9168faSHans Rosenfeld * - strict-version: can be set to 0 to allow devices conforming to newer 1433c9168faSHans Rosenfeld * versions to be used 1443c9168faSHans Rosenfeld * - ignore-unknown-vendor-status: can be set to 1 to not handle any vendor 1453c9168faSHans Rosenfeld * specific command status as a fatal error leading device faulting 1463c9168faSHans Rosenfeld * - admin-queue-len: the maximum length of the admin queue (16-4096) 1473c9168faSHans Rosenfeld * - io-queue-len: the maximum length of the I/O queues (16-65536) 1483c9168faSHans Rosenfeld * - async-event-limit: the maximum number of asynchronous event requests to be 1493c9168faSHans Rosenfeld * posted by the driver 1503c9168faSHans Rosenfeld * 1513c9168faSHans Rosenfeld * 1523c9168faSHans Rosenfeld * TODO: 1533c9168faSHans Rosenfeld * - figure out sane default for I/O queue depth reported to blkdev 1543c9168faSHans Rosenfeld * - polled I/O support to support kernel core dumping 1553c9168faSHans Rosenfeld * - FMA handling of media errors 1563c9168faSHans Rosenfeld * - support for the Volatile Write Cache 1573c9168faSHans Rosenfeld * - support for devices supporting very large I/O requests using chained PRPs 1583c9168faSHans Rosenfeld * - support for querying log pages from user space 1593c9168faSHans Rosenfeld * - support for configuring hardware parameters like interrupt coalescing 1603c9168faSHans Rosenfeld * - support for media formatting and hard partitioning into namespaces 1613c9168faSHans Rosenfeld * - support for big-endian systems 1623c9168faSHans Rosenfeld * - support for fast reboot 1633c9168faSHans Rosenfeld */ 1643c9168faSHans Rosenfeld 1653c9168faSHans Rosenfeld #include <sys/byteorder.h> 1663c9168faSHans Rosenfeld #ifdef _BIG_ENDIAN 1673c9168faSHans Rosenfeld #error nvme driver needs porting for big-endian platforms 1683c9168faSHans Rosenfeld #endif 1693c9168faSHans Rosenfeld 1703c9168faSHans Rosenfeld #include <sys/modctl.h> 1713c9168faSHans Rosenfeld #include <sys/conf.h> 1723c9168faSHans Rosenfeld #include <sys/devops.h> 1733c9168faSHans Rosenfeld #include <sys/ddi.h> 1743c9168faSHans Rosenfeld #include <sys/sunddi.h> 1753c9168faSHans Rosenfeld #include <sys/bitmap.h> 1763c9168faSHans Rosenfeld #include <sys/sysmacros.h> 1773c9168faSHans Rosenfeld #include <sys/param.h> 1783c9168faSHans Rosenfeld #include <sys/varargs.h> 1793c9168faSHans Rosenfeld #include <sys/cpuvar.h> 1803c9168faSHans Rosenfeld #include <sys/disp.h> 1813c9168faSHans Rosenfeld #include <sys/blkdev.h> 1823c9168faSHans Rosenfeld #include <sys/atomic.h> 1833c9168faSHans Rosenfeld #include <sys/archsystm.h> 184510a6847SHans Rosenfeld #include <sys/sata/sata_hba.h> 1853c9168faSHans Rosenfeld 1863c9168faSHans Rosenfeld #include "nvme_reg.h" 1873c9168faSHans Rosenfeld #include "nvme_var.h" 1883c9168faSHans Rosenfeld 1893c9168faSHans Rosenfeld 1903c9168faSHans Rosenfeld /* NVMe spec version supported */ 1913c9168faSHans Rosenfeld static const int nvme_version_major = 1; 1923c9168faSHans Rosenfeld static const int nvme_version_minor = 0; 1933c9168faSHans Rosenfeld 194e8ba2a38SHans Rosenfeld /* tunable for admin command timeout in seconds, default is 1s */ 195e8ba2a38SHans Rosenfeld static volatile int nvme_admin_cmd_timeout = 1; 196e8ba2a38SHans Rosenfeld 1973c9168faSHans Rosenfeld static int nvme_attach(dev_info_t *, ddi_attach_cmd_t); 1983c9168faSHans Rosenfeld static int nvme_detach(dev_info_t *, ddi_detach_cmd_t); 1993c9168faSHans Rosenfeld static int nvme_quiesce(dev_info_t *); 2003c9168faSHans Rosenfeld static int nvme_fm_errcb(dev_info_t *, ddi_fm_error_t *, const void *); 2013c9168faSHans Rosenfeld static int nvme_setup_interrupts(nvme_t *, int, int); 2023c9168faSHans Rosenfeld static void nvme_release_interrupts(nvme_t *); 2033c9168faSHans Rosenfeld static uint_t nvme_intr(caddr_t, caddr_t); 2043c9168faSHans Rosenfeld 2053c9168faSHans Rosenfeld static void nvme_shutdown(nvme_t *, int, boolean_t); 2063c9168faSHans Rosenfeld static boolean_t nvme_reset(nvme_t *, boolean_t); 2073c9168faSHans Rosenfeld static int nvme_init(nvme_t *); 2083c9168faSHans Rosenfeld static nvme_cmd_t *nvme_alloc_cmd(nvme_t *, int); 2093c9168faSHans Rosenfeld static void nvme_free_cmd(nvme_cmd_t *); 2103c9168faSHans Rosenfeld static nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t, 2113c9168faSHans Rosenfeld bd_xfer_t *); 2123c9168faSHans Rosenfeld static int nvme_admin_cmd(nvme_cmd_t *, int); 2133c9168faSHans Rosenfeld static int nvme_submit_cmd(nvme_qpair_t *, nvme_cmd_t *); 2143c9168faSHans Rosenfeld static nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *); 2153c9168faSHans Rosenfeld static boolean_t nvme_wait_cmd(nvme_cmd_t *, uint_t); 2163c9168faSHans Rosenfeld static void nvme_wakeup_cmd(void *); 2173c9168faSHans Rosenfeld static void nvme_async_event_task(void *); 2183c9168faSHans Rosenfeld 2193c9168faSHans Rosenfeld static int nvme_check_unknown_cmd_status(nvme_cmd_t *); 2203c9168faSHans Rosenfeld static int nvme_check_vendor_cmd_status(nvme_cmd_t *); 2213c9168faSHans Rosenfeld static int nvme_check_integrity_cmd_status(nvme_cmd_t *); 2223c9168faSHans Rosenfeld static int nvme_check_specific_cmd_status(nvme_cmd_t *); 2233c9168faSHans Rosenfeld static int nvme_check_generic_cmd_status(nvme_cmd_t *); 2243c9168faSHans Rosenfeld static inline int nvme_check_cmd_status(nvme_cmd_t *); 2253c9168faSHans Rosenfeld 2263c9168faSHans Rosenfeld static void nvme_abort_cmd(nvme_cmd_t *); 2273c9168faSHans Rosenfeld static int nvme_async_event(nvme_t *); 2283c9168faSHans Rosenfeld static void *nvme_get_logpage(nvme_t *, uint8_t, ...); 2293c9168faSHans Rosenfeld static void *nvme_identify(nvme_t *, uint32_t); 2303c9168faSHans Rosenfeld static int nvme_set_nqueues(nvme_t *, uint16_t); 2313c9168faSHans Rosenfeld 2323c9168faSHans Rosenfeld static void nvme_free_dma(nvme_dma_t *); 2333c9168faSHans Rosenfeld static int nvme_zalloc_dma(nvme_t *, size_t, uint_t, ddi_dma_attr_t *, 2343c9168faSHans Rosenfeld nvme_dma_t **); 2353c9168faSHans Rosenfeld static int nvme_zalloc_queue_dma(nvme_t *, uint32_t, uint16_t, uint_t, 2363c9168faSHans Rosenfeld nvme_dma_t **); 2373c9168faSHans Rosenfeld static void nvme_free_qpair(nvme_qpair_t *); 2383c9168faSHans Rosenfeld static int nvme_alloc_qpair(nvme_t *, uint32_t, nvme_qpair_t **, int); 2393c9168faSHans Rosenfeld static int nvme_create_io_qpair(nvme_t *, nvme_qpair_t *, uint16_t); 2403c9168faSHans Rosenfeld 2413c9168faSHans Rosenfeld static inline void nvme_put64(nvme_t *, uintptr_t, uint64_t); 2423c9168faSHans Rosenfeld static inline void nvme_put32(nvme_t *, uintptr_t, uint32_t); 2433c9168faSHans Rosenfeld static inline uint64_t nvme_get64(nvme_t *, uintptr_t); 2443c9168faSHans Rosenfeld static inline uint32_t nvme_get32(nvme_t *, uintptr_t); 2453c9168faSHans Rosenfeld 2463c9168faSHans Rosenfeld static boolean_t nvme_check_regs_hdl(nvme_t *); 2473c9168faSHans Rosenfeld static boolean_t nvme_check_dma_hdl(nvme_dma_t *); 2483c9168faSHans Rosenfeld 2493c9168faSHans Rosenfeld static int nvme_fill_prp(nvme_cmd_t *, bd_xfer_t *); 2503c9168faSHans Rosenfeld 2513c9168faSHans Rosenfeld static void nvme_bd_xfer_done(void *); 2523c9168faSHans Rosenfeld static void nvme_bd_driveinfo(void *, bd_drive_t *); 2533c9168faSHans Rosenfeld static int nvme_bd_mediainfo(void *, bd_media_t *); 2543c9168faSHans Rosenfeld static int nvme_bd_cmd(nvme_namespace_t *, bd_xfer_t *, uint8_t); 2553c9168faSHans Rosenfeld static int nvme_bd_read(void *, bd_xfer_t *); 2563c9168faSHans Rosenfeld static int nvme_bd_write(void *, bd_xfer_t *); 2573c9168faSHans Rosenfeld static int nvme_bd_sync(void *, bd_xfer_t *); 2583c9168faSHans Rosenfeld static int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *); 2593c9168faSHans Rosenfeld 260*8834f7acSYouzhong Yang static int nvme_prp_dma_constructor(void *, void *, int); 261*8834f7acSYouzhong Yang static void nvme_prp_dma_destructor(void *, void *); 262*8834f7acSYouzhong Yang 2633c9168faSHans Rosenfeld static void nvme_prepare_devid(nvme_t *, uint32_t); 2643c9168faSHans Rosenfeld 2653c9168faSHans Rosenfeld static void *nvme_state; 2663c9168faSHans Rosenfeld static kmem_cache_t *nvme_cmd_cache; 2673c9168faSHans Rosenfeld 2683c9168faSHans Rosenfeld /* 2693c9168faSHans Rosenfeld * DMA attributes for queue DMA memory 2703c9168faSHans Rosenfeld * 2713c9168faSHans Rosenfeld * Queue DMA memory must be page aligned. The maximum length of a queue is 2723c9168faSHans Rosenfeld * 65536 entries, and an entry can be 64 bytes long. 2733c9168faSHans Rosenfeld */ 2743c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_queue_dma_attr = { 2753c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 2763c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 2773c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 278910f0d12SYouzhong Yang .dma_attr_count_max = (UINT16_MAX + 1) * sizeof (nvme_sqe_t) - 1, 2793c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 2803c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 2813c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 2823c9168faSHans Rosenfeld .dma_attr_maxxfer = (UINT16_MAX + 1) * sizeof (nvme_sqe_t), 2833c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 2843c9168faSHans Rosenfeld .dma_attr_sgllen = 1, 2853c9168faSHans Rosenfeld .dma_attr_granular = 1, 2863c9168faSHans Rosenfeld .dma_attr_flags = 0, 2873c9168faSHans Rosenfeld }; 2883c9168faSHans Rosenfeld 2893c9168faSHans Rosenfeld /* 2903c9168faSHans Rosenfeld * DMA attributes for transfers using Physical Region Page (PRP) entries 2913c9168faSHans Rosenfeld * 2923c9168faSHans Rosenfeld * A PRP entry describes one page of DMA memory using the page size specified 2933c9168faSHans Rosenfeld * in the controller configuration's memory page size register (CC.MPS). It uses 2943c9168faSHans Rosenfeld * a 64bit base address aligned to this page size. There is no limitation on 2953c9168faSHans Rosenfeld * chaining PRPs together for arbitrarily large DMA transfers. 2963c9168faSHans Rosenfeld */ 2973c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_prp_dma_attr = { 2983c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 2993c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 3003c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 3013c9168faSHans Rosenfeld .dma_attr_count_max = 0xfff, 3023c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 3033c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 3043c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 3053c9168faSHans Rosenfeld .dma_attr_maxxfer = 0x1000, 3062f95345bSYouzhong Yang .dma_attr_seg = 0xfff, 3073c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 3083c9168faSHans Rosenfeld .dma_attr_granular = 1, 3093c9168faSHans Rosenfeld .dma_attr_flags = 0, 3103c9168faSHans Rosenfeld }; 3113c9168faSHans Rosenfeld 3123c9168faSHans Rosenfeld /* 3133c9168faSHans Rosenfeld * DMA attributes for transfers using scatter/gather lists 3143c9168faSHans Rosenfeld * 3153c9168faSHans Rosenfeld * A SGL entry describes a chunk of DMA memory using a 64bit base address and a 3163c9168faSHans Rosenfeld * 32bit length field. SGL Segment and SGL Last Segment entries require the 3173c9168faSHans Rosenfeld * length to be a multiple of 16 bytes. 3183c9168faSHans Rosenfeld */ 3193c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_sgl_dma_attr = { 3203c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 3213c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 3223c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 3233c9168faSHans Rosenfeld .dma_attr_count_max = 0xffffffffUL, 3243c9168faSHans Rosenfeld .dma_attr_align = 1, 3253c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 3263c9168faSHans Rosenfeld .dma_attr_minxfer = 0x10, 3273c9168faSHans Rosenfeld .dma_attr_maxxfer = 0xfffffffffULL, 3283c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 3293c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 3303c9168faSHans Rosenfeld .dma_attr_granular = 0x10, 3313c9168faSHans Rosenfeld .dma_attr_flags = 0 3323c9168faSHans Rosenfeld }; 3333c9168faSHans Rosenfeld 3343c9168faSHans Rosenfeld static ddi_device_acc_attr_t nvme_reg_acc_attr = { 3353c9168faSHans Rosenfeld .devacc_attr_version = DDI_DEVICE_ATTR_V0, 3363c9168faSHans Rosenfeld .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 3373c9168faSHans Rosenfeld .devacc_attr_dataorder = DDI_STRICTORDER_ACC 3383c9168faSHans Rosenfeld }; 3393c9168faSHans Rosenfeld 3403c9168faSHans Rosenfeld static struct dev_ops nvme_dev_ops = { 3413c9168faSHans Rosenfeld .devo_rev = DEVO_REV, 3423c9168faSHans Rosenfeld .devo_refcnt = 0, 3433c9168faSHans Rosenfeld .devo_getinfo = ddi_no_info, 3443c9168faSHans Rosenfeld .devo_identify = nulldev, 3453c9168faSHans Rosenfeld .devo_probe = nulldev, 3463c9168faSHans Rosenfeld .devo_attach = nvme_attach, 3473c9168faSHans Rosenfeld .devo_detach = nvme_detach, 3483c9168faSHans Rosenfeld .devo_reset = nodev, 3493c9168faSHans Rosenfeld .devo_cb_ops = NULL, 3503c9168faSHans Rosenfeld .devo_bus_ops = NULL, 3513c9168faSHans Rosenfeld .devo_power = NULL, 3523c9168faSHans Rosenfeld .devo_quiesce = nvme_quiesce, 3533c9168faSHans Rosenfeld }; 3543c9168faSHans Rosenfeld 3553c9168faSHans Rosenfeld static struct modldrv nvme_modldrv = { 3563c9168faSHans Rosenfeld .drv_modops = &mod_driverops, 3573c9168faSHans Rosenfeld .drv_linkinfo = "NVMe v1.0e", 3583c9168faSHans Rosenfeld .drv_dev_ops = &nvme_dev_ops 3593c9168faSHans Rosenfeld }; 3603c9168faSHans Rosenfeld 3613c9168faSHans Rosenfeld static struct modlinkage nvme_modlinkage = { 3623c9168faSHans Rosenfeld .ml_rev = MODREV_1, 3633c9168faSHans Rosenfeld .ml_linkage = { &nvme_modldrv, NULL } 3643c9168faSHans Rosenfeld }; 3653c9168faSHans Rosenfeld 3663c9168faSHans Rosenfeld static bd_ops_t nvme_bd_ops = { 3673c9168faSHans Rosenfeld .o_version = BD_OPS_VERSION_0, 3683c9168faSHans Rosenfeld .o_drive_info = nvme_bd_driveinfo, 3693c9168faSHans Rosenfeld .o_media_info = nvme_bd_mediainfo, 3703c9168faSHans Rosenfeld .o_devid_init = nvme_bd_devid, 3713c9168faSHans Rosenfeld .o_sync_cache = nvme_bd_sync, 3723c9168faSHans Rosenfeld .o_read = nvme_bd_read, 3733c9168faSHans Rosenfeld .o_write = nvme_bd_write, 3743c9168faSHans Rosenfeld }; 3753c9168faSHans Rosenfeld 3763c9168faSHans Rosenfeld int 3773c9168faSHans Rosenfeld _init(void) 3783c9168faSHans Rosenfeld { 3793c9168faSHans Rosenfeld int error; 3803c9168faSHans Rosenfeld 3813c9168faSHans Rosenfeld error = ddi_soft_state_init(&nvme_state, sizeof (nvme_t), 1); 3823c9168faSHans Rosenfeld if (error != DDI_SUCCESS) 3833c9168faSHans Rosenfeld return (error); 3843c9168faSHans Rosenfeld 3853c9168faSHans Rosenfeld nvme_cmd_cache = kmem_cache_create("nvme_cmd_cache", 3863c9168faSHans Rosenfeld sizeof (nvme_cmd_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 3873c9168faSHans Rosenfeld 3883c9168faSHans Rosenfeld bd_mod_init(&nvme_dev_ops); 3893c9168faSHans Rosenfeld 3903c9168faSHans Rosenfeld error = mod_install(&nvme_modlinkage); 3913c9168faSHans Rosenfeld if (error != DDI_SUCCESS) { 3923c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 3933c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 3943c9168faSHans Rosenfeld } 3953c9168faSHans Rosenfeld 3963c9168faSHans Rosenfeld return (error); 3973c9168faSHans Rosenfeld } 3983c9168faSHans Rosenfeld 3993c9168faSHans Rosenfeld int 4003c9168faSHans Rosenfeld _fini(void) 4013c9168faSHans Rosenfeld { 4023c9168faSHans Rosenfeld int error; 4033c9168faSHans Rosenfeld 4043c9168faSHans Rosenfeld error = mod_remove(&nvme_modlinkage); 4053c9168faSHans Rosenfeld if (error == DDI_SUCCESS) { 4063c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 4073c9168faSHans Rosenfeld kmem_cache_destroy(nvme_cmd_cache); 4083c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 4093c9168faSHans Rosenfeld } 4103c9168faSHans Rosenfeld 4113c9168faSHans Rosenfeld return (error); 4123c9168faSHans Rosenfeld } 4133c9168faSHans Rosenfeld 4143c9168faSHans Rosenfeld int 4153c9168faSHans Rosenfeld _info(struct modinfo *modinfop) 4163c9168faSHans Rosenfeld { 4173c9168faSHans Rosenfeld return (mod_info(&nvme_modlinkage, modinfop)); 4183c9168faSHans Rosenfeld } 4193c9168faSHans Rosenfeld 4203c9168faSHans Rosenfeld static inline void 4213c9168faSHans Rosenfeld nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val) 4223c9168faSHans Rosenfeld { 4233c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 4243c9168faSHans Rosenfeld 4253c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4263c9168faSHans Rosenfeld ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val); 4273c9168faSHans Rosenfeld } 4283c9168faSHans Rosenfeld 4293c9168faSHans Rosenfeld static inline void 4303c9168faSHans Rosenfeld nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val) 4313c9168faSHans Rosenfeld { 4323c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 4333c9168faSHans Rosenfeld 4343c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4353c9168faSHans Rosenfeld ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val); 4363c9168faSHans Rosenfeld } 4373c9168faSHans Rosenfeld 4383c9168faSHans Rosenfeld static inline uint64_t 4393c9168faSHans Rosenfeld nvme_get64(nvme_t *nvme, uintptr_t reg) 4403c9168faSHans Rosenfeld { 4413c9168faSHans Rosenfeld uint64_t val; 4423c9168faSHans Rosenfeld 4433c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 4443c9168faSHans Rosenfeld 4453c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4463c9168faSHans Rosenfeld val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg)); 4473c9168faSHans Rosenfeld 4483c9168faSHans Rosenfeld return (val); 4493c9168faSHans Rosenfeld } 4503c9168faSHans Rosenfeld 4513c9168faSHans Rosenfeld static inline uint32_t 4523c9168faSHans Rosenfeld nvme_get32(nvme_t *nvme, uintptr_t reg) 4533c9168faSHans Rosenfeld { 4543c9168faSHans Rosenfeld uint32_t val; 4553c9168faSHans Rosenfeld 4563c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 4573c9168faSHans Rosenfeld 4583c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4593c9168faSHans Rosenfeld val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg)); 4603c9168faSHans Rosenfeld 4613c9168faSHans Rosenfeld return (val); 4623c9168faSHans Rosenfeld } 4633c9168faSHans Rosenfeld 4643c9168faSHans Rosenfeld static boolean_t 4653c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme_t *nvme) 4663c9168faSHans Rosenfeld { 4673c9168faSHans Rosenfeld ddi_fm_error_t error; 4683c9168faSHans Rosenfeld 4693c9168faSHans Rosenfeld ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION); 4703c9168faSHans Rosenfeld 4713c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 4723c9168faSHans Rosenfeld return (B_TRUE); 4733c9168faSHans Rosenfeld 4743c9168faSHans Rosenfeld return (B_FALSE); 4753c9168faSHans Rosenfeld } 4763c9168faSHans Rosenfeld 4773c9168faSHans Rosenfeld static boolean_t 4783c9168faSHans Rosenfeld nvme_check_dma_hdl(nvme_dma_t *dma) 4793c9168faSHans Rosenfeld { 4803c9168faSHans Rosenfeld ddi_fm_error_t error; 4813c9168faSHans Rosenfeld 4823c9168faSHans Rosenfeld if (dma == NULL) 4833c9168faSHans Rosenfeld return (B_FALSE); 4843c9168faSHans Rosenfeld 4853c9168faSHans Rosenfeld ddi_fm_dma_err_get(dma->nd_dmah, &error, DDI_FME_VERSION); 4863c9168faSHans Rosenfeld 4873c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 4883c9168faSHans Rosenfeld return (B_TRUE); 4893c9168faSHans Rosenfeld 4903c9168faSHans Rosenfeld return (B_FALSE); 4913c9168faSHans Rosenfeld } 4923c9168faSHans Rosenfeld 4933c9168faSHans Rosenfeld static void 494*8834f7acSYouzhong Yang nvme_free_dma_common(nvme_dma_t *dma) 4953c9168faSHans Rosenfeld { 4963c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 4973c9168faSHans Rosenfeld (void) ddi_dma_unbind_handle(dma->nd_dmah); 4983c9168faSHans Rosenfeld if (dma->nd_acch != NULL) 4993c9168faSHans Rosenfeld ddi_dma_mem_free(&dma->nd_acch); 5003c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 5013c9168faSHans Rosenfeld ddi_dma_free_handle(&dma->nd_dmah); 5023c9168faSHans Rosenfeld } 5033c9168faSHans Rosenfeld 504*8834f7acSYouzhong Yang static void 505*8834f7acSYouzhong Yang nvme_free_dma(nvme_dma_t *dma) 5063c9168faSHans Rosenfeld { 507*8834f7acSYouzhong Yang nvme_free_dma_common(dma); 508*8834f7acSYouzhong Yang kmem_free(dma, sizeof (*dma)); 509*8834f7acSYouzhong Yang } 510*8834f7acSYouzhong Yang 511*8834f7acSYouzhong Yang static void 512*8834f7acSYouzhong Yang nvme_prp_dma_destructor(void *buf, void *private) 513*8834f7acSYouzhong Yang { 514*8834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 5153c9168faSHans Rosenfeld 516*8834f7acSYouzhong Yang nvme_free_dma_common(dma); 517*8834f7acSYouzhong Yang } 518*8834f7acSYouzhong Yang 519*8834f7acSYouzhong Yang static int 520*8834f7acSYouzhong Yang nvme_alloc_dma_common(nvme_t *nvme, nvme_dma_t *dma, 521*8834f7acSYouzhong Yang size_t len, uint_t flags, ddi_dma_attr_t *dma_attr) 522*8834f7acSYouzhong Yang { 5233c9168faSHans Rosenfeld if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL, 5243c9168faSHans Rosenfeld &dma->nd_dmah) != DDI_SUCCESS) { 5253c9168faSHans Rosenfeld /* 5263c9168faSHans Rosenfeld * Due to DDI_DMA_SLEEP this can't be DDI_DMA_NORESOURCES, and 5273c9168faSHans Rosenfeld * the only other possible error is DDI_DMA_BADATTR which 5283c9168faSHans Rosenfeld * indicates a driver bug which should cause a panic. 5293c9168faSHans Rosenfeld */ 5303c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, 5313c9168faSHans Rosenfeld "!failed to get DMA handle, check DMA attributes"); 5323c9168faSHans Rosenfeld return (DDI_FAILURE); 5333c9168faSHans Rosenfeld } 5343c9168faSHans Rosenfeld 5353c9168faSHans Rosenfeld /* 5363c9168faSHans Rosenfeld * ddi_dma_mem_alloc() can only fail when DDI_DMA_NOSLEEP is specified 5373c9168faSHans Rosenfeld * or the flags are conflicting, which isn't the case here. 5383c9168faSHans Rosenfeld */ 5393c9168faSHans Rosenfeld (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr, 5403c9168faSHans Rosenfeld DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &dma->nd_memp, 5413c9168faSHans Rosenfeld &dma->nd_len, &dma->nd_acch); 5423c9168faSHans Rosenfeld 5433c9168faSHans Rosenfeld if (ddi_dma_addr_bind_handle(dma->nd_dmah, NULL, dma->nd_memp, 5443c9168faSHans Rosenfeld dma->nd_len, flags | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, 5453c9168faSHans Rosenfeld &dma->nd_cookie, &dma->nd_ncookie) != DDI_DMA_MAPPED) { 5463c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 5473c9168faSHans Rosenfeld "!failed to bind DMA memory"); 5483c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_dma_bind_err); 549*8834f7acSYouzhong Yang nvme_free_dma_common(dma); 550*8834f7acSYouzhong Yang return (DDI_FAILURE); 551*8834f7acSYouzhong Yang } 552*8834f7acSYouzhong Yang 553*8834f7acSYouzhong Yang return (DDI_SUCCESS); 554*8834f7acSYouzhong Yang } 555*8834f7acSYouzhong Yang 556*8834f7acSYouzhong Yang static int 557*8834f7acSYouzhong Yang nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags, 558*8834f7acSYouzhong Yang ddi_dma_attr_t *dma_attr, nvme_dma_t **ret) 559*8834f7acSYouzhong Yang { 560*8834f7acSYouzhong Yang nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP); 561*8834f7acSYouzhong Yang 562*8834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, len, flags, dma_attr) != 563*8834f7acSYouzhong Yang DDI_SUCCESS) { 5643c9168faSHans Rosenfeld *ret = NULL; 565*8834f7acSYouzhong Yang kmem_free(dma, sizeof (nvme_dma_t)); 5663c9168faSHans Rosenfeld return (DDI_FAILURE); 5673c9168faSHans Rosenfeld } 5683c9168faSHans Rosenfeld 5693c9168faSHans Rosenfeld bzero(dma->nd_memp, dma->nd_len); 5703c9168faSHans Rosenfeld 5713c9168faSHans Rosenfeld *ret = dma; 5723c9168faSHans Rosenfeld return (DDI_SUCCESS); 5733c9168faSHans Rosenfeld } 5743c9168faSHans Rosenfeld 575*8834f7acSYouzhong Yang static int 576*8834f7acSYouzhong Yang nvme_prp_dma_constructor(void *buf, void *private, int flags) 577*8834f7acSYouzhong Yang { 578*8834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 579*8834f7acSYouzhong Yang nvme_t *nvme = (nvme_t *)private; 580*8834f7acSYouzhong Yang 581*8834f7acSYouzhong Yang dma->nd_dmah = NULL; 582*8834f7acSYouzhong Yang dma->nd_acch = NULL; 583*8834f7acSYouzhong Yang 584*8834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, nvme->n_pagesize, 585*8834f7acSYouzhong Yang DDI_DMA_READ, &nvme->n_prp_dma_attr) != DDI_SUCCESS) { 586*8834f7acSYouzhong Yang return (-1); 587*8834f7acSYouzhong Yang } 588*8834f7acSYouzhong Yang 589*8834f7acSYouzhong Yang ASSERT(dma->nd_ncookie == 1); 590*8834f7acSYouzhong Yang 591*8834f7acSYouzhong Yang dma->nd_cached = B_TRUE; 592*8834f7acSYouzhong Yang 593*8834f7acSYouzhong Yang return (0); 594*8834f7acSYouzhong Yang } 595*8834f7acSYouzhong Yang 5963c9168faSHans Rosenfeld static int 5973c9168faSHans Rosenfeld nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len, 5983c9168faSHans Rosenfeld uint_t flags, nvme_dma_t **dma) 5993c9168faSHans Rosenfeld { 6003c9168faSHans Rosenfeld uint32_t len = nentry * qe_len; 6013c9168faSHans Rosenfeld ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr; 6023c9168faSHans Rosenfeld 6033c9168faSHans Rosenfeld len = roundup(len, nvme->n_pagesize); 6043c9168faSHans Rosenfeld 6053c9168faSHans Rosenfeld q_dma_attr.dma_attr_minxfer = len; 6063c9168faSHans Rosenfeld 6073c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma) 6083c9168faSHans Rosenfeld != DDI_SUCCESS) { 6093c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 6103c9168faSHans Rosenfeld "!failed to get DMA memory for queue"); 6113c9168faSHans Rosenfeld goto fail; 6123c9168faSHans Rosenfeld } 6133c9168faSHans Rosenfeld 6143c9168faSHans Rosenfeld if ((*dma)->nd_ncookie != 1) { 6153c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 6163c9168faSHans Rosenfeld "!got too many cookies for queue DMA"); 6173c9168faSHans Rosenfeld goto fail; 6183c9168faSHans Rosenfeld } 6193c9168faSHans Rosenfeld 6203c9168faSHans Rosenfeld return (DDI_SUCCESS); 6213c9168faSHans Rosenfeld 6223c9168faSHans Rosenfeld fail: 6233c9168faSHans Rosenfeld if (*dma) { 6243c9168faSHans Rosenfeld nvme_free_dma(*dma); 6253c9168faSHans Rosenfeld *dma = NULL; 6263c9168faSHans Rosenfeld } 6273c9168faSHans Rosenfeld 6283c9168faSHans Rosenfeld return (DDI_FAILURE); 6293c9168faSHans Rosenfeld } 6303c9168faSHans Rosenfeld 6313c9168faSHans Rosenfeld static void 6323c9168faSHans Rosenfeld nvme_free_qpair(nvme_qpair_t *qp) 6333c9168faSHans Rosenfeld { 6343c9168faSHans Rosenfeld int i; 6353c9168faSHans Rosenfeld 6363c9168faSHans Rosenfeld mutex_destroy(&qp->nq_mutex); 6373c9168faSHans Rosenfeld 6383c9168faSHans Rosenfeld if (qp->nq_sqdma != NULL) 6393c9168faSHans Rosenfeld nvme_free_dma(qp->nq_sqdma); 6403c9168faSHans Rosenfeld if (qp->nq_cqdma != NULL) 6413c9168faSHans Rosenfeld nvme_free_dma(qp->nq_cqdma); 6423c9168faSHans Rosenfeld 6433c9168faSHans Rosenfeld if (qp->nq_active_cmds > 0) 6443c9168faSHans Rosenfeld for (i = 0; i != qp->nq_nentry; i++) 6453c9168faSHans Rosenfeld if (qp->nq_cmd[i] != NULL) 6463c9168faSHans Rosenfeld nvme_free_cmd(qp->nq_cmd[i]); 6473c9168faSHans Rosenfeld 6483c9168faSHans Rosenfeld if (qp->nq_cmd != NULL) 6493c9168faSHans Rosenfeld kmem_free(qp->nq_cmd, sizeof (nvme_cmd_t *) * qp->nq_nentry); 6503c9168faSHans Rosenfeld 6513c9168faSHans Rosenfeld kmem_free(qp, sizeof (nvme_qpair_t)); 6523c9168faSHans Rosenfeld } 6533c9168faSHans Rosenfeld 6543c9168faSHans Rosenfeld static int 6553c9168faSHans Rosenfeld nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp, 6563c9168faSHans Rosenfeld int idx) 6573c9168faSHans Rosenfeld { 6583c9168faSHans Rosenfeld nvme_qpair_t *qp = kmem_zalloc(sizeof (*qp), KM_SLEEP); 6593c9168faSHans Rosenfeld 6603c9168faSHans Rosenfeld mutex_init(&qp->nq_mutex, NULL, MUTEX_DRIVER, 6613c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 6623c9168faSHans Rosenfeld 6633c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t), 6643c9168faSHans Rosenfeld DDI_DMA_WRITE, &qp->nq_sqdma) != DDI_SUCCESS) 6653c9168faSHans Rosenfeld goto fail; 6663c9168faSHans Rosenfeld 6673c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t), 6683c9168faSHans Rosenfeld DDI_DMA_READ, &qp->nq_cqdma) != DDI_SUCCESS) 6693c9168faSHans Rosenfeld goto fail; 6703c9168faSHans Rosenfeld 6713c9168faSHans Rosenfeld qp->nq_sq = (nvme_sqe_t *)qp->nq_sqdma->nd_memp; 6723c9168faSHans Rosenfeld qp->nq_cq = (nvme_cqe_t *)qp->nq_cqdma->nd_memp; 6733c9168faSHans Rosenfeld qp->nq_nentry = nentry; 6743c9168faSHans Rosenfeld 6753c9168faSHans Rosenfeld qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx); 6763c9168faSHans Rosenfeld qp->nq_cqhdbl = NVME_REG_CQHDBL(nvme, idx); 6773c9168faSHans Rosenfeld 6783c9168faSHans Rosenfeld qp->nq_cmd = kmem_zalloc(sizeof (nvme_cmd_t *) * nentry, KM_SLEEP); 6793c9168faSHans Rosenfeld qp->nq_next_cmd = 0; 6803c9168faSHans Rosenfeld 6813c9168faSHans Rosenfeld *nqp = qp; 6823c9168faSHans Rosenfeld return (DDI_SUCCESS); 6833c9168faSHans Rosenfeld 6843c9168faSHans Rosenfeld fail: 6853c9168faSHans Rosenfeld nvme_free_qpair(qp); 6863c9168faSHans Rosenfeld *nqp = NULL; 6873c9168faSHans Rosenfeld 6883c9168faSHans Rosenfeld return (DDI_FAILURE); 6893c9168faSHans Rosenfeld } 6903c9168faSHans Rosenfeld 6913c9168faSHans Rosenfeld static nvme_cmd_t * 6923c9168faSHans Rosenfeld nvme_alloc_cmd(nvme_t *nvme, int kmflag) 6933c9168faSHans Rosenfeld { 6943c9168faSHans Rosenfeld nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag); 6953c9168faSHans Rosenfeld 6963c9168faSHans Rosenfeld if (cmd == NULL) 6973c9168faSHans Rosenfeld return (cmd); 6983c9168faSHans Rosenfeld 6993c9168faSHans Rosenfeld bzero(cmd, sizeof (nvme_cmd_t)); 7003c9168faSHans Rosenfeld 7013c9168faSHans Rosenfeld cmd->nc_nvme = nvme; 7023c9168faSHans Rosenfeld 7033c9168faSHans Rosenfeld mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER, 7043c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 7053c9168faSHans Rosenfeld cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL); 7063c9168faSHans Rosenfeld 7073c9168faSHans Rosenfeld return (cmd); 7083c9168faSHans Rosenfeld } 7093c9168faSHans Rosenfeld 7103c9168faSHans Rosenfeld static void 7113c9168faSHans Rosenfeld nvme_free_cmd(nvme_cmd_t *cmd) 7123c9168faSHans Rosenfeld { 7133c9168faSHans Rosenfeld if (cmd->nc_dma) { 714*8834f7acSYouzhong Yang if (cmd->nc_dma->nd_cached) 715*8834f7acSYouzhong Yang kmem_cache_free(cmd->nc_nvme->n_prp_cache, 716*8834f7acSYouzhong Yang cmd->nc_dma); 717*8834f7acSYouzhong Yang else 718*8834f7acSYouzhong Yang nvme_free_dma(cmd->nc_dma); 7193c9168faSHans Rosenfeld cmd->nc_dma = NULL; 7203c9168faSHans Rosenfeld } 7213c9168faSHans Rosenfeld 7223c9168faSHans Rosenfeld cv_destroy(&cmd->nc_cv); 7233c9168faSHans Rosenfeld mutex_destroy(&cmd->nc_mutex); 7243c9168faSHans Rosenfeld 7253c9168faSHans Rosenfeld kmem_cache_free(nvme_cmd_cache, cmd); 7263c9168faSHans Rosenfeld } 7273c9168faSHans Rosenfeld 7283c9168faSHans Rosenfeld static int 7293c9168faSHans Rosenfeld nvme_submit_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 7303c9168faSHans Rosenfeld { 7313c9168faSHans Rosenfeld nvme_reg_sqtdbl_t tail = { 0 }; 7323c9168faSHans Rosenfeld 7333c9168faSHans Rosenfeld mutex_enter(&qp->nq_mutex); 7343c9168faSHans Rosenfeld 7353c9168faSHans Rosenfeld if (qp->nq_active_cmds == qp->nq_nentry) { 7363c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 7373c9168faSHans Rosenfeld return (DDI_FAILURE); 7383c9168faSHans Rosenfeld } 7393c9168faSHans Rosenfeld 7403c9168faSHans Rosenfeld cmd->nc_completed = B_FALSE; 7413c9168faSHans Rosenfeld 7423c9168faSHans Rosenfeld /* 7433c9168faSHans Rosenfeld * Try to insert the cmd into the active cmd array at the nq_next_cmd 7443c9168faSHans Rosenfeld * slot. If the slot is already occupied advance to the next slot and 7453c9168faSHans Rosenfeld * try again. This can happen for long running commands like async event 7463c9168faSHans Rosenfeld * requests. 7473c9168faSHans Rosenfeld */ 7483c9168faSHans Rosenfeld while (qp->nq_cmd[qp->nq_next_cmd] != NULL) 7493c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 7503c9168faSHans Rosenfeld qp->nq_cmd[qp->nq_next_cmd] = cmd; 7513c9168faSHans Rosenfeld 7523c9168faSHans Rosenfeld qp->nq_active_cmds++; 7533c9168faSHans Rosenfeld 7543c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cid = qp->nq_next_cmd; 7553c9168faSHans Rosenfeld bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t)); 7563c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_sqdma->nd_dmah, 7573c9168faSHans Rosenfeld sizeof (nvme_sqe_t) * qp->nq_sqtail, 7583c9168faSHans Rosenfeld sizeof (nvme_sqe_t), DDI_DMA_SYNC_FORDEV); 7593c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 7603c9168faSHans Rosenfeld 7613c9168faSHans Rosenfeld tail.b.sqtdbl_sqt = qp->nq_sqtail = (qp->nq_sqtail + 1) % qp->nq_nentry; 7623c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r); 7633c9168faSHans Rosenfeld 7643c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 7653c9168faSHans Rosenfeld return (DDI_SUCCESS); 7663c9168faSHans Rosenfeld } 7673c9168faSHans Rosenfeld 7683c9168faSHans Rosenfeld static nvme_cmd_t * 7693c9168faSHans Rosenfeld nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp) 7703c9168faSHans Rosenfeld { 7713c9168faSHans Rosenfeld nvme_reg_cqhdbl_t head = { 0 }; 7723c9168faSHans Rosenfeld 7733c9168faSHans Rosenfeld nvme_cqe_t *cqe; 7743c9168faSHans Rosenfeld nvme_cmd_t *cmd; 7753c9168faSHans Rosenfeld 7763c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_cqdma->nd_dmah, 0, 7773c9168faSHans Rosenfeld sizeof (nvme_cqe_t) * qp->nq_nentry, DDI_DMA_SYNC_FORKERNEL); 7783c9168faSHans Rosenfeld 7793c9168faSHans Rosenfeld cqe = &qp->nq_cq[qp->nq_cqhead]; 7803c9168faSHans Rosenfeld 7813c9168faSHans Rosenfeld /* Check phase tag of CQE. Hardware inverts it for new entries. */ 7823c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_p == qp->nq_phase) 7833c9168faSHans Rosenfeld return (NULL); 7843c9168faSHans Rosenfeld 7853c9168faSHans Rosenfeld ASSERT(nvme->n_ioq[cqe->cqe_sqid] == qp); 7863c9168faSHans Rosenfeld ASSERT(cqe->cqe_cid < qp->nq_nentry); 7873c9168faSHans Rosenfeld 7883c9168faSHans Rosenfeld mutex_enter(&qp->nq_mutex); 7893c9168faSHans Rosenfeld cmd = qp->nq_cmd[cqe->cqe_cid]; 7903c9168faSHans Rosenfeld qp->nq_cmd[cqe->cqe_cid] = NULL; 7913c9168faSHans Rosenfeld qp->nq_active_cmds--; 7923c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 7933c9168faSHans Rosenfeld 7943c9168faSHans Rosenfeld ASSERT(cmd != NULL); 7953c9168faSHans Rosenfeld ASSERT(cmd->nc_nvme == nvme); 7963c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid == cqe->cqe_sqid); 7973c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_cid == cqe->cqe_cid); 7983c9168faSHans Rosenfeld bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t)); 7993c9168faSHans Rosenfeld 8003c9168faSHans Rosenfeld qp->nq_sqhead = cqe->cqe_sqhd; 8013c9168faSHans Rosenfeld 8023c9168faSHans Rosenfeld head.b.cqhdbl_cqh = qp->nq_cqhead = (qp->nq_cqhead + 1) % qp->nq_nentry; 8033c9168faSHans Rosenfeld 8043c9168faSHans Rosenfeld /* Toggle phase on wrap-around. */ 8053c9168faSHans Rosenfeld if (qp->nq_cqhead == 0) 8063c9168faSHans Rosenfeld qp->nq_phase = qp->nq_phase ? 0 : 1; 8073c9168faSHans Rosenfeld 8083c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_cqhdbl, head.r); 8093c9168faSHans Rosenfeld 8103c9168faSHans Rosenfeld return (cmd); 8113c9168faSHans Rosenfeld } 8123c9168faSHans Rosenfeld 8133c9168faSHans Rosenfeld static int 8143c9168faSHans Rosenfeld nvme_check_unknown_cmd_status(nvme_cmd_t *cmd) 8153c9168faSHans Rosenfeld { 8163c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8173c9168faSHans Rosenfeld 8183c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 8193c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 8203c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 8213c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 8223c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 8233c9168faSHans Rosenfeld 824bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 825bef9e21aSHans Rosenfeld 8263c9168faSHans Rosenfeld if (cmd->nc_nvme->n_strict_version) { 8273c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 8283c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 8293c9168faSHans Rosenfeld } 8303c9168faSHans Rosenfeld 8313c9168faSHans Rosenfeld return (EIO); 8323c9168faSHans Rosenfeld } 8333c9168faSHans Rosenfeld 8343c9168faSHans Rosenfeld static int 8353c9168faSHans Rosenfeld nvme_check_vendor_cmd_status(nvme_cmd_t *cmd) 8363c9168faSHans Rosenfeld { 8373c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8383c9168faSHans Rosenfeld 8393c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 8403c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 8413c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 8423c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 8433c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 84434c938c7SPete Shephard if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) { 8453c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 8463c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 8473c9168faSHans Rosenfeld } 8483c9168faSHans Rosenfeld 8493c9168faSHans Rosenfeld return (EIO); 8503c9168faSHans Rosenfeld } 8513c9168faSHans Rosenfeld 8523c9168faSHans Rosenfeld static int 8533c9168faSHans Rosenfeld nvme_check_integrity_cmd_status(nvme_cmd_t *cmd) 8543c9168faSHans Rosenfeld { 8553c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8563c9168faSHans Rosenfeld 8573c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 8583c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_WRITE: 8593c9168faSHans Rosenfeld /* write fail */ 8603c9168faSHans Rosenfeld /* TODO: post ereport */ 861bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 8623c9168faSHans Rosenfeld return (EIO); 8633c9168faSHans Rosenfeld 8643c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_READ: 8653c9168faSHans Rosenfeld /* read fail */ 8663c9168faSHans Rosenfeld /* TODO: post ereport */ 867bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 8683c9168faSHans Rosenfeld return (EIO); 8693c9168faSHans Rosenfeld 8703c9168faSHans Rosenfeld default: 8713c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 8723c9168faSHans Rosenfeld } 8733c9168faSHans Rosenfeld } 8743c9168faSHans Rosenfeld 8753c9168faSHans Rosenfeld static int 8763c9168faSHans Rosenfeld nvme_check_generic_cmd_status(nvme_cmd_t *cmd) 8773c9168faSHans Rosenfeld { 8783c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8793c9168faSHans Rosenfeld 8803c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 8813c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_SUCCESS: 8823c9168faSHans Rosenfeld return (0); 8833c9168faSHans Rosenfeld 8843c9168faSHans Rosenfeld /* 8853c9168faSHans Rosenfeld * Errors indicating a bug in the driver should cause a panic. 8863c9168faSHans Rosenfeld */ 8873c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_OPC: 8883c9168faSHans Rosenfeld /* Invalid Command Opcode */ 8893c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 8903c9168faSHans Rosenfeld "invalid opcode in cmd %p", (void *)cmd); 8913c9168faSHans Rosenfeld return (0); 8923c9168faSHans Rosenfeld 8933c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_FLD: 8943c9168faSHans Rosenfeld /* Invalid Field in Command */ 8953c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 8963c9168faSHans Rosenfeld "invalid field in cmd %p", (void *)cmd); 8973c9168faSHans Rosenfeld return (0); 8983c9168faSHans Rosenfeld 8993c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ID_CNFL: 9003c9168faSHans Rosenfeld /* Command ID Conflict */ 9013c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9023c9168faSHans Rosenfeld "cmd ID conflict in cmd %p", (void *)cmd); 9033c9168faSHans Rosenfeld return (0); 9043c9168faSHans Rosenfeld 9053c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_NS: 9063c9168faSHans Rosenfeld /* Invalid Namespace or Format */ 9073c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9083c9168faSHans Rosenfeld "invalid NS/format in cmd %p", (void *)cmd); 9093c9168faSHans Rosenfeld return (0); 9103c9168faSHans Rosenfeld 9113c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_LBA_RANGE: 9123c9168faSHans Rosenfeld /* LBA Out Of Range */ 9133c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9143c9168faSHans Rosenfeld "LBA out of range in cmd %p", (void *)cmd); 9153c9168faSHans Rosenfeld return (0); 9163c9168faSHans Rosenfeld 9173c9168faSHans Rosenfeld /* 9183c9168faSHans Rosenfeld * Non-fatal errors, handle gracefully. 9193c9168faSHans Rosenfeld */ 9203c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_DATA_XFR_ERR: 9213c9168faSHans Rosenfeld /* Data Transfer Error (DMA) */ 9223c9168faSHans Rosenfeld /* TODO: post ereport */ 9233c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err); 924bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 9253c9168faSHans Rosenfeld return (EIO); 9263c9168faSHans Rosenfeld 9273c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INTERNAL_ERR: 9283c9168faSHans Rosenfeld /* 9293c9168faSHans Rosenfeld * Internal Error. The spec (v1.0, section 4.5.1.2) says 9303c9168faSHans Rosenfeld * detailed error information is returned as async event, 9313c9168faSHans Rosenfeld * so we pretty much ignore the error here and handle it 9323c9168faSHans Rosenfeld * in the async event handler. 9333c9168faSHans Rosenfeld */ 9343c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_internal_err); 935bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 9363c9168faSHans Rosenfeld return (EIO); 9373c9168faSHans Rosenfeld 9383c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_REQUEST: 9393c9168faSHans Rosenfeld /* 9403c9168faSHans Rosenfeld * Command Abort Requested. This normally happens only when a 9413c9168faSHans Rosenfeld * command times out. 9423c9168faSHans Rosenfeld */ 9433c9168faSHans Rosenfeld /* TODO: post ereport or change blkdev to handle this? */ 9443c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err); 9453c9168faSHans Rosenfeld return (ECANCELED); 9463c9168faSHans Rosenfeld 9473c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_PWRLOSS: 9483c9168faSHans Rosenfeld /* Command Aborted due to Power Loss Notification */ 9493c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 9503c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 9513c9168faSHans Rosenfeld return (EIO); 9523c9168faSHans Rosenfeld 9533c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_SQ_DEL: 9543c9168faSHans Rosenfeld /* Command Aborted due to SQ Deletion */ 9553c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del); 9563c9168faSHans Rosenfeld return (EIO); 9573c9168faSHans Rosenfeld 9583c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_CAP_EXC: 9593c9168faSHans Rosenfeld /* Capacity Exceeded */ 9603c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc); 961bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 9623c9168faSHans Rosenfeld return (EIO); 9633c9168faSHans Rosenfeld 9643c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_NS_NOTRDY: 9653c9168faSHans Rosenfeld /* Namespace Not Ready */ 9663c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy); 967bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 9683c9168faSHans Rosenfeld return (EIO); 9693c9168faSHans Rosenfeld 9703c9168faSHans Rosenfeld default: 9713c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 9723c9168faSHans Rosenfeld } 9733c9168faSHans Rosenfeld } 9743c9168faSHans Rosenfeld 9753c9168faSHans Rosenfeld static int 9763c9168faSHans Rosenfeld nvme_check_specific_cmd_status(nvme_cmd_t *cmd) 9773c9168faSHans Rosenfeld { 9783c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 9793c9168faSHans Rosenfeld 9803c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 9813c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_CQ: 9823c9168faSHans Rosenfeld /* Completion Queue Invalid */ 9833c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE); 9843c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err); 9853c9168faSHans Rosenfeld return (EINVAL); 9863c9168faSHans Rosenfeld 9873c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_QID: 9883c9168faSHans Rosenfeld /* Invalid Queue Identifier */ 9893c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 9903c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE || 9913c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE || 9923c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 9933c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err); 9943c9168faSHans Rosenfeld return (EINVAL); 9953c9168faSHans Rosenfeld 9963c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_MAX_QSZ_EXC: 9973c9168faSHans Rosenfeld /* Max Queue Size Exceeded */ 9983c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 9993c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 10003c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc); 10013c9168faSHans Rosenfeld return (EINVAL); 10023c9168faSHans Rosenfeld 10033c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ABRT_CMD_EXC: 10043c9168faSHans Rosenfeld /* Abort Command Limit Exceeded */ 10053c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT); 10063c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 10073c9168faSHans Rosenfeld "abort command limit exceeded in cmd %p", (void *)cmd); 10083c9168faSHans Rosenfeld return (0); 10093c9168faSHans Rosenfeld 10103c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC: 10113c9168faSHans Rosenfeld /* Async Event Request Limit Exceeded */ 10123c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT); 10133c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 10143c9168faSHans Rosenfeld "async event request limit exceeded in cmd %p", 10153c9168faSHans Rosenfeld (void *)cmd); 10163c9168faSHans Rosenfeld return (0); 10173c9168faSHans Rosenfeld 10183c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_INT_VECT: 10193c9168faSHans Rosenfeld /* Invalid Interrupt Vector */ 10203c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 10213c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect); 10223c9168faSHans Rosenfeld return (EINVAL); 10233c9168faSHans Rosenfeld 10243c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_LOG_PAGE: 10253c9168faSHans Rosenfeld /* Invalid Log Page */ 10263c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE); 10273c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_log_page); 1028bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10293c9168faSHans Rosenfeld return (EINVAL); 10303c9168faSHans Rosenfeld 10313c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_FORMAT: 10323c9168faSHans Rosenfeld /* Invalid Format */ 10333c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT); 10343c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_format); 1035bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10363c9168faSHans Rosenfeld return (EINVAL); 10373c9168faSHans Rosenfeld 10383c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_Q_DEL: 10393c9168faSHans Rosenfeld /* Invalid Queue Deletion */ 10403c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 10413c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_q_del); 10423c9168faSHans Rosenfeld return (EINVAL); 10433c9168faSHans Rosenfeld 10443c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_CNFL_ATTR: 10453c9168faSHans Rosenfeld /* Conflicting Attributes */ 10463c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT || 10473c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 10483c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 10493c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr); 1050bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10513c9168faSHans Rosenfeld return (EINVAL); 10523c9168faSHans Rosenfeld 10533c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_INV_PROT: 10543c9168faSHans Rosenfeld /* Invalid Protection Information */ 10553c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE || 10563c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 10573c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 10583c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_prot); 1059bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10603c9168faSHans Rosenfeld return (EINVAL); 10613c9168faSHans Rosenfeld 10623c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_READONLY: 10633c9168faSHans Rosenfeld /* Write to Read Only Range */ 10643c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 10653c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_readonly); 1066bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10673c9168faSHans Rosenfeld return (EROFS); 10683c9168faSHans Rosenfeld 10693c9168faSHans Rosenfeld default: 10703c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 10713c9168faSHans Rosenfeld } 10723c9168faSHans Rosenfeld } 10733c9168faSHans Rosenfeld 10743c9168faSHans Rosenfeld static inline int 10753c9168faSHans Rosenfeld nvme_check_cmd_status(nvme_cmd_t *cmd) 10763c9168faSHans Rosenfeld { 10773c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 10783c9168faSHans Rosenfeld 10793c9168faSHans Rosenfeld /* take a shortcut if everything is alright */ 10803c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 10813c9168faSHans Rosenfeld cqe->cqe_sf.sf_sc == NVME_CQE_SC_GEN_SUCCESS) 10823c9168faSHans Rosenfeld return (0); 10833c9168faSHans Rosenfeld 10843c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC) 10853c9168faSHans Rosenfeld return (nvme_check_generic_cmd_status(cmd)); 10863c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_SPECIFIC) 10873c9168faSHans Rosenfeld return (nvme_check_specific_cmd_status(cmd)); 10883c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_INTEGRITY) 10893c9168faSHans Rosenfeld return (nvme_check_integrity_cmd_status(cmd)); 10903c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_VENDOR) 10913c9168faSHans Rosenfeld return (nvme_check_vendor_cmd_status(cmd)); 10923c9168faSHans Rosenfeld 10933c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 10943c9168faSHans Rosenfeld } 10953c9168faSHans Rosenfeld 10963c9168faSHans Rosenfeld /* 10973c9168faSHans Rosenfeld * nvme_abort_cmd_cb -- replaces nc_callback of aborted commands 10983c9168faSHans Rosenfeld * 10993c9168faSHans Rosenfeld * This functions takes care of cleaning up aborted commands. The command 11003c9168faSHans Rosenfeld * status is checked to catch any fatal errors. 11013c9168faSHans Rosenfeld */ 11023c9168faSHans Rosenfeld static void 11033c9168faSHans Rosenfeld nvme_abort_cmd_cb(void *arg) 11043c9168faSHans Rosenfeld { 11053c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 11063c9168faSHans Rosenfeld 11073c9168faSHans Rosenfeld /* 11083c9168faSHans Rosenfeld * Grab the command mutex. Once we have it we hold the last reference 11093c9168faSHans Rosenfeld * to the command and can safely free it. 11103c9168faSHans Rosenfeld */ 11113c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 11123c9168faSHans Rosenfeld (void) nvme_check_cmd_status(cmd); 11133c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 11143c9168faSHans Rosenfeld 11153c9168faSHans Rosenfeld nvme_free_cmd(cmd); 11163c9168faSHans Rosenfeld } 11173c9168faSHans Rosenfeld 11183c9168faSHans Rosenfeld static void 11193c9168faSHans Rosenfeld nvme_abort_cmd(nvme_cmd_t *abort_cmd) 11203c9168faSHans Rosenfeld { 11213c9168faSHans Rosenfeld nvme_t *nvme = abort_cmd->nc_nvme; 11223c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 11233c9168faSHans Rosenfeld nvme_abort_cmd_t ac = { 0 }; 11243c9168faSHans Rosenfeld 11253c9168faSHans Rosenfeld sema_p(&nvme->n_abort_sema); 11263c9168faSHans Rosenfeld 11273c9168faSHans Rosenfeld ac.b.ac_cid = abort_cmd->nc_sqe.sqe_cid; 11283c9168faSHans Rosenfeld ac.b.ac_sqid = abort_cmd->nc_sqid; 11293c9168faSHans Rosenfeld 11303c9168faSHans Rosenfeld /* 11313c9168faSHans Rosenfeld * Drop the mutex of the aborted command. From this point on 11323c9168faSHans Rosenfeld * we must assume that the abort callback has freed the command. 11333c9168faSHans Rosenfeld */ 11343c9168faSHans Rosenfeld mutex_exit(&abort_cmd->nc_mutex); 11353c9168faSHans Rosenfeld 11363c9168faSHans Rosenfeld cmd->nc_sqid = 0; 11373c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT; 11383c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 11393c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = ac.r; 11403c9168faSHans Rosenfeld 11413c9168faSHans Rosenfeld /* 11423c9168faSHans Rosenfeld * Send the ABORT to the hardware. The ABORT command will return _after_ 11433c9168faSHans Rosenfeld * the aborted command has completed (aborted or otherwise). 11443c9168faSHans Rosenfeld */ 1145e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 11463c9168faSHans Rosenfeld sema_v(&nvme->n_abort_sema); 11473c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 11483c9168faSHans Rosenfeld "!nvme_admin_cmd failed for ABORT"); 11493c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_abort_failed); 11503c9168faSHans Rosenfeld return; 11513c9168faSHans Rosenfeld } 11523c9168faSHans Rosenfeld sema_v(&nvme->n_abort_sema); 11533c9168faSHans Rosenfeld 11543c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 11553c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 11563c9168faSHans Rosenfeld "!ABORT failed with sct = %x, sc = %x", 11573c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 11583c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_abort_failed); 11593c9168faSHans Rosenfeld } else { 11603c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_aborted); 11613c9168faSHans Rosenfeld } 11623c9168faSHans Rosenfeld 11633c9168faSHans Rosenfeld nvme_free_cmd(cmd); 11643c9168faSHans Rosenfeld } 11653c9168faSHans Rosenfeld 11663c9168faSHans Rosenfeld /* 11673c9168faSHans Rosenfeld * nvme_wait_cmd -- wait for command completion or timeout 11683c9168faSHans Rosenfeld * 11693c9168faSHans Rosenfeld * Returns B_TRUE if the command completed normally. 11703c9168faSHans Rosenfeld * 11713c9168faSHans Rosenfeld * Returns B_FALSE if the command timed out and an abort was attempted. The 11723c9168faSHans Rosenfeld * command mutex will be dropped and the command must be considered freed. The 11733c9168faSHans Rosenfeld * freeing of the command is normally done by the abort command callback. 11743c9168faSHans Rosenfeld * 11753c9168faSHans Rosenfeld * In case of a serious error or a timeout of the abort command the hardware 11763c9168faSHans Rosenfeld * will be declared dead and FMA will be notified. 11773c9168faSHans Rosenfeld */ 11783c9168faSHans Rosenfeld static boolean_t 1179e8ba2a38SHans Rosenfeld nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec) 11803c9168faSHans Rosenfeld { 1181e8ba2a38SHans Rosenfeld clock_t timeout = ddi_get_lbolt() + drv_usectohz(sec * MICROSEC); 11823c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 11833c9168faSHans Rosenfeld nvme_reg_csts_t csts; 11843c9168faSHans Rosenfeld 11853c9168faSHans Rosenfeld ASSERT(mutex_owned(&cmd->nc_mutex)); 11863c9168faSHans Rosenfeld 11873c9168faSHans Rosenfeld while (!cmd->nc_completed) { 11883c9168faSHans Rosenfeld if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1) 11893c9168faSHans Rosenfeld break; 11903c9168faSHans Rosenfeld } 11913c9168faSHans Rosenfeld 11923c9168faSHans Rosenfeld if (cmd->nc_completed) 11933c9168faSHans Rosenfeld return (B_TRUE); 11943c9168faSHans Rosenfeld 11953c9168faSHans Rosenfeld /* 11963c9168faSHans Rosenfeld * The command timed out. Change the callback to the cleanup function. 11973c9168faSHans Rosenfeld */ 11983c9168faSHans Rosenfeld cmd->nc_callback = nvme_abort_cmd_cb; 11993c9168faSHans Rosenfeld 12003c9168faSHans Rosenfeld /* 12013c9168faSHans Rosenfeld * Check controller for fatal status, any errors associated with the 12023c9168faSHans Rosenfeld * register or DMA handle, or for a double timeout (abort command timed 12033c9168faSHans Rosenfeld * out). If necessary log a warning and call FMA. 12043c9168faSHans Rosenfeld */ 12053c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 12063c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!command timeout, " 12073c9168faSHans Rosenfeld "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_opc, csts.b.csts_cfs); 12083c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_timeout); 12093c9168faSHans Rosenfeld 12103c9168faSHans Rosenfeld if (csts.b.csts_cfs || 12113c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme) || 12123c9168faSHans Rosenfeld nvme_check_dma_hdl(cmd->nc_dma) || 12133c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) { 12143c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 12153c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 12163c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 12173c9168faSHans Rosenfeld } else { 12183c9168faSHans Rosenfeld /* 12193c9168faSHans Rosenfeld * Try to abort the command. The command mutex is released by 12203c9168faSHans Rosenfeld * nvme_abort_cmd(). 12213c9168faSHans Rosenfeld * If the abort succeeds it will have freed the aborted command. 12223c9168faSHans Rosenfeld * If the abort fails for other reasons we must assume that the 12233c9168faSHans Rosenfeld * command may complete at any time, and the callback will free 12243c9168faSHans Rosenfeld * it for us. 12253c9168faSHans Rosenfeld */ 12263c9168faSHans Rosenfeld nvme_abort_cmd(cmd); 12273c9168faSHans Rosenfeld } 12283c9168faSHans Rosenfeld 12293c9168faSHans Rosenfeld return (B_FALSE); 12303c9168faSHans Rosenfeld } 12313c9168faSHans Rosenfeld 12323c9168faSHans Rosenfeld static void 12333c9168faSHans Rosenfeld nvme_wakeup_cmd(void *arg) 12343c9168faSHans Rosenfeld { 12353c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 12363c9168faSHans Rosenfeld 12373c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 12383c9168faSHans Rosenfeld /* 12393c9168faSHans Rosenfeld * There is a slight chance that this command completed shortly after 12403c9168faSHans Rosenfeld * the timeout was hit in nvme_wait_cmd() but before the callback was 12413c9168faSHans Rosenfeld * changed. Catch that case here and clean up accordingly. 12423c9168faSHans Rosenfeld */ 12433c9168faSHans Rosenfeld if (cmd->nc_callback == nvme_abort_cmd_cb) { 12443c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 12453c9168faSHans Rosenfeld nvme_abort_cmd_cb(cmd); 12463c9168faSHans Rosenfeld return; 12473c9168faSHans Rosenfeld } 12483c9168faSHans Rosenfeld 12493c9168faSHans Rosenfeld cmd->nc_completed = B_TRUE; 12503c9168faSHans Rosenfeld cv_signal(&cmd->nc_cv); 12513c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 12523c9168faSHans Rosenfeld } 12533c9168faSHans Rosenfeld 12543c9168faSHans Rosenfeld static void 12553c9168faSHans Rosenfeld nvme_async_event_task(void *arg) 12563c9168faSHans Rosenfeld { 12573c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 12583c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 12593c9168faSHans Rosenfeld nvme_error_log_entry_t *error_log = NULL; 12603c9168faSHans Rosenfeld nvme_health_log_t *health_log = NULL; 12613c9168faSHans Rosenfeld nvme_async_event_t event; 12623c9168faSHans Rosenfeld int ret; 12633c9168faSHans Rosenfeld 12643c9168faSHans Rosenfeld /* 12653c9168faSHans Rosenfeld * Check for errors associated with the async request itself. The only 12663c9168faSHans Rosenfeld * command-specific error is "async event limit exceeded", which 12673c9168faSHans Rosenfeld * indicates a programming error in the driver and causes a panic in 12683c9168faSHans Rosenfeld * nvme_check_cmd_status(). 12693c9168faSHans Rosenfeld * 12703c9168faSHans Rosenfeld * Other possible errors are various scenarios where the async request 12713c9168faSHans Rosenfeld * was aborted, or internal errors in the device. Internal errors are 12723c9168faSHans Rosenfeld * reported to FMA, the command aborts need no special handling here. 12733c9168faSHans Rosenfeld */ 12743c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 12753c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 12763c9168faSHans Rosenfeld "!async event request returned failure, sct = %x, " 12773c9168faSHans Rosenfeld "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct, 12783c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr, 12793c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_m); 12803c9168faSHans Rosenfeld 12813c9168faSHans Rosenfeld if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 12823c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) { 12833c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 12843c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, 12853c9168faSHans Rosenfeld DDI_SERVICE_LOST); 12863c9168faSHans Rosenfeld } 12873c9168faSHans Rosenfeld nvme_free_cmd(cmd); 12883c9168faSHans Rosenfeld return; 12893c9168faSHans Rosenfeld } 12903c9168faSHans Rosenfeld 12913c9168faSHans Rosenfeld 12923c9168faSHans Rosenfeld event.r = cmd->nc_cqe.cqe_dw0; 12933c9168faSHans Rosenfeld 12943c9168faSHans Rosenfeld /* Clear CQE and re-submit the async request. */ 12953c9168faSHans Rosenfeld bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t)); 12963c9168faSHans Rosenfeld ret = nvme_submit_cmd(nvme->n_adminq, cmd); 12973c9168faSHans Rosenfeld 12983c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 12993c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13003c9168faSHans Rosenfeld "!failed to resubmit async event request"); 13013c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_async_resubmit_failed); 13023c9168faSHans Rosenfeld nvme_free_cmd(cmd); 13033c9168faSHans Rosenfeld } 13043c9168faSHans Rosenfeld 13053c9168faSHans Rosenfeld switch (event.b.ae_type) { 13063c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_ERROR: 13073c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_ERROR) { 13083c9168faSHans Rosenfeld error_log = (nvme_error_log_entry_t *) 13093c9168faSHans Rosenfeld nvme_get_logpage(nvme, event.b.ae_logpage); 13103c9168faSHans Rosenfeld } else { 13113c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 13123c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 13133c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 13143c9168faSHans Rosenfeld } 13153c9168faSHans Rosenfeld 13163c9168faSHans Rosenfeld switch (event.b.ae_info) { 13173c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_SQ: 13183c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 13193c9168faSHans Rosenfeld "invalid submission queue"); 13203c9168faSHans Rosenfeld return; 13213c9168faSHans Rosenfeld 13223c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_DBL: 13233c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 13243c9168faSHans Rosenfeld "invalid doorbell write value"); 13253c9168faSHans Rosenfeld return; 13263c9168faSHans Rosenfeld 13273c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_DIAGFAIL: 13283c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure"); 13293c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 13303c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 13313c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_diagfail_event); 13323c9168faSHans Rosenfeld break; 13333c9168faSHans Rosenfeld 13343c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_PERSISTENT: 13353c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!persistent internal " 13363c9168faSHans Rosenfeld "device error"); 13373c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 13383c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 13393c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_persistent_event); 13403c9168faSHans Rosenfeld break; 13413c9168faSHans Rosenfeld 13423c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_TRANSIENT: 13433c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!transient internal " 13443c9168faSHans Rosenfeld "device error"); 13453c9168faSHans Rosenfeld /* TODO: send ereport */ 13463c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_transient_event); 13473c9168faSHans Rosenfeld break; 13483c9168faSHans Rosenfeld 13493c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_FW_LOAD: 13503c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13513c9168faSHans Rosenfeld "!firmware image load error"); 13523c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_fw_load_event); 13533c9168faSHans Rosenfeld break; 13543c9168faSHans Rosenfeld } 13553c9168faSHans Rosenfeld break; 13563c9168faSHans Rosenfeld 13573c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_HEALTH: 13583c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_HEALTH) { 13593c9168faSHans Rosenfeld health_log = (nvme_health_log_t *) 13603c9168faSHans Rosenfeld nvme_get_logpage(nvme, event.b.ae_logpage, -1); 13613c9168faSHans Rosenfeld } else { 13623c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 13633c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 13643c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 13653c9168faSHans Rosenfeld } 13663c9168faSHans Rosenfeld 13673c9168faSHans Rosenfeld switch (event.b.ae_info) { 13683c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_RELIABILITY: 13693c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13703c9168faSHans Rosenfeld "!device reliability compromised"); 13713c9168faSHans Rosenfeld /* TODO: send ereport */ 13723c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_reliability_event); 13733c9168faSHans Rosenfeld break; 13743c9168faSHans Rosenfeld 13753c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_TEMPERATURE: 13763c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13773c9168faSHans Rosenfeld "!temperature above threshold"); 13783c9168faSHans Rosenfeld /* TODO: send ereport */ 13793c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_temperature_event); 13803c9168faSHans Rosenfeld break; 13813c9168faSHans Rosenfeld 13823c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_SPARE: 13833c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13843c9168faSHans Rosenfeld "!spare space below threshold"); 13853c9168faSHans Rosenfeld /* TODO: send ereport */ 13863c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_spare_event); 13873c9168faSHans Rosenfeld break; 13883c9168faSHans Rosenfeld } 13893c9168faSHans Rosenfeld break; 13903c9168faSHans Rosenfeld 13913c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_VENDOR: 13923c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event " 13933c9168faSHans Rosenfeld "received, info = %x, logpage = %x", event.b.ae_info, 13943c9168faSHans Rosenfeld event.b.ae_logpage); 13953c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_vendor_event); 13963c9168faSHans Rosenfeld break; 13973c9168faSHans Rosenfeld 13983c9168faSHans Rosenfeld default: 13993c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, " 14003c9168faSHans Rosenfeld "type = %x, info = %x, logpage = %x", event.b.ae_type, 14013c9168faSHans Rosenfeld event.b.ae_info, event.b.ae_logpage); 14023c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_event); 14033c9168faSHans Rosenfeld break; 14043c9168faSHans Rosenfeld } 14053c9168faSHans Rosenfeld 14063c9168faSHans Rosenfeld if (error_log) 14073c9168faSHans Rosenfeld kmem_free(error_log, sizeof (nvme_error_log_entry_t) * 14083c9168faSHans Rosenfeld nvme->n_error_log_len); 14093c9168faSHans Rosenfeld 14103c9168faSHans Rosenfeld if (health_log) 14113c9168faSHans Rosenfeld kmem_free(health_log, sizeof (nvme_health_log_t)); 14123c9168faSHans Rosenfeld } 14133c9168faSHans Rosenfeld 14143c9168faSHans Rosenfeld static int 1415e8ba2a38SHans Rosenfeld nvme_admin_cmd(nvme_cmd_t *cmd, int sec) 14163c9168faSHans Rosenfeld { 14173c9168faSHans Rosenfeld int ret; 14183c9168faSHans Rosenfeld 14193c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 14203c9168faSHans Rosenfeld ret = nvme_submit_cmd(cmd->nc_nvme->n_adminq, cmd); 14213c9168faSHans Rosenfeld 14223c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 14233c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 14243c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 14253c9168faSHans Rosenfeld "!nvme_submit_cmd failed"); 14263c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_admin_queue_full); 14273c9168faSHans Rosenfeld nvme_free_cmd(cmd); 14283c9168faSHans Rosenfeld return (DDI_FAILURE); 14293c9168faSHans Rosenfeld } 14303c9168faSHans Rosenfeld 1431e8ba2a38SHans Rosenfeld if (nvme_wait_cmd(cmd, sec) == B_FALSE) { 14323c9168faSHans Rosenfeld /* 14333c9168faSHans Rosenfeld * The command timed out. An abort command was posted that 14343c9168faSHans Rosenfeld * will take care of the cleanup. 14353c9168faSHans Rosenfeld */ 14363c9168faSHans Rosenfeld return (DDI_FAILURE); 14373c9168faSHans Rosenfeld } 14383c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 14393c9168faSHans Rosenfeld 14403c9168faSHans Rosenfeld return (DDI_SUCCESS); 14413c9168faSHans Rosenfeld } 14423c9168faSHans Rosenfeld 14433c9168faSHans Rosenfeld static int 14443c9168faSHans Rosenfeld nvme_async_event(nvme_t *nvme) 14453c9168faSHans Rosenfeld { 14463c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 14473c9168faSHans Rosenfeld int ret; 14483c9168faSHans Rosenfeld 14493c9168faSHans Rosenfeld cmd->nc_sqid = 0; 14503c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT; 14513c9168faSHans Rosenfeld cmd->nc_callback = nvme_async_event_task; 14523c9168faSHans Rosenfeld 14533c9168faSHans Rosenfeld ret = nvme_submit_cmd(nvme->n_adminq, cmd); 14543c9168faSHans Rosenfeld 14553c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 14563c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 14573c9168faSHans Rosenfeld "!nvme_submit_cmd failed for ASYNCHRONOUS EVENT"); 14583c9168faSHans Rosenfeld nvme_free_cmd(cmd); 14593c9168faSHans Rosenfeld return (DDI_FAILURE); 14603c9168faSHans Rosenfeld } 14613c9168faSHans Rosenfeld 14623c9168faSHans Rosenfeld return (DDI_SUCCESS); 14633c9168faSHans Rosenfeld } 14643c9168faSHans Rosenfeld 14653c9168faSHans Rosenfeld static void * 14663c9168faSHans Rosenfeld nvme_get_logpage(nvme_t *nvme, uint8_t logpage, ...) 14673c9168faSHans Rosenfeld { 14683c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 14693c9168faSHans Rosenfeld void *buf = NULL; 147034c938c7SPete Shephard nvme_getlogpage_t getlogpage = { 0 }; 14713c9168faSHans Rosenfeld size_t bufsize; 14723c9168faSHans Rosenfeld va_list ap; 14733c9168faSHans Rosenfeld 14743c9168faSHans Rosenfeld va_start(ap, logpage); 14753c9168faSHans Rosenfeld 14763c9168faSHans Rosenfeld cmd->nc_sqid = 0; 14773c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 14783c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE; 14793c9168faSHans Rosenfeld 14803c9168faSHans Rosenfeld getlogpage.b.lp_lid = logpage; 14813c9168faSHans Rosenfeld 14823c9168faSHans Rosenfeld switch (logpage) { 14833c9168faSHans Rosenfeld case NVME_LOGPAGE_ERROR: 14843c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 14853c9168faSHans Rosenfeld bufsize = nvme->n_error_log_len * 14863c9168faSHans Rosenfeld sizeof (nvme_error_log_entry_t); 14873c9168faSHans Rosenfeld break; 14883c9168faSHans Rosenfeld 14893c9168faSHans Rosenfeld case NVME_LOGPAGE_HEALTH: 14903c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t); 14913c9168faSHans Rosenfeld bufsize = sizeof (nvme_health_log_t); 14923c9168faSHans Rosenfeld break; 14933c9168faSHans Rosenfeld 14943c9168faSHans Rosenfeld case NVME_LOGPAGE_FWSLOT: 14953c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 14963c9168faSHans Rosenfeld bufsize = sizeof (nvme_fwslot_log_t); 14973c9168faSHans Rosenfeld break; 14983c9168faSHans Rosenfeld 14993c9168faSHans Rosenfeld default: 15003c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown log page requested: %d", 15013c9168faSHans Rosenfeld logpage); 15023c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_logpage); 15033c9168faSHans Rosenfeld goto fail; 15043c9168faSHans Rosenfeld } 15053c9168faSHans Rosenfeld 15063c9168faSHans Rosenfeld va_end(ap); 15073c9168faSHans Rosenfeld 15089898bbdbSHans Rosenfeld getlogpage.b.lp_numd = bufsize / sizeof (uint32_t) - 1; 15093c9168faSHans Rosenfeld 15103c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = getlogpage.r; 15113c9168faSHans Rosenfeld 15123c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, getlogpage.b.lp_numd * sizeof (uint32_t), 15133c9168faSHans Rosenfeld DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 15143c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15153c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for GET LOG PAGE"); 15163c9168faSHans Rosenfeld goto fail; 15173c9168faSHans Rosenfeld } 15183c9168faSHans Rosenfeld 15193c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 15203c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15213c9168faSHans Rosenfeld "!too many DMA cookies for GET LOG PAGE"); 15223c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 15233c9168faSHans Rosenfeld goto fail; 15243c9168faSHans Rosenfeld } 15253c9168faSHans Rosenfeld 15263c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 15273c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 15283c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 15293c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 15303c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 15313c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 15323c9168faSHans Rosenfeld } 15333c9168faSHans Rosenfeld 1534e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 15353c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15363c9168faSHans Rosenfeld "!nvme_admin_cmd failed for GET LOG PAGE"); 15373c9168faSHans Rosenfeld return (NULL); 15383c9168faSHans Rosenfeld } 15393c9168faSHans Rosenfeld 15403c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 15413c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15423c9168faSHans Rosenfeld "!GET LOG PAGE failed with sct = %x, sc = %x", 15433c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 15443c9168faSHans Rosenfeld goto fail; 15453c9168faSHans Rosenfeld } 15463c9168faSHans Rosenfeld 15473c9168faSHans Rosenfeld buf = kmem_alloc(bufsize, KM_SLEEP); 15483c9168faSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, buf, bufsize); 15493c9168faSHans Rosenfeld 15503c9168faSHans Rosenfeld fail: 15513c9168faSHans Rosenfeld nvme_free_cmd(cmd); 15523c9168faSHans Rosenfeld 15533c9168faSHans Rosenfeld return (buf); 15543c9168faSHans Rosenfeld } 15553c9168faSHans Rosenfeld 15563c9168faSHans Rosenfeld static void * 15573c9168faSHans Rosenfeld nvme_identify(nvme_t *nvme, uint32_t nsid) 15583c9168faSHans Rosenfeld { 15593c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 15603c9168faSHans Rosenfeld void *buf = NULL; 15613c9168faSHans Rosenfeld 15623c9168faSHans Rosenfeld cmd->nc_sqid = 0; 15633c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 15643c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY; 15653c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 15663c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL; 15673c9168faSHans Rosenfeld 15683c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ, 15693c9168faSHans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 15703c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15713c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for IDENTIFY"); 15723c9168faSHans Rosenfeld goto fail; 15733c9168faSHans Rosenfeld } 15743c9168faSHans Rosenfeld 15753c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 15763c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15773c9168faSHans Rosenfeld "!too many DMA cookies for IDENTIFY"); 15783c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 15793c9168faSHans Rosenfeld goto fail; 15803c9168faSHans Rosenfeld } 15813c9168faSHans Rosenfeld 15823c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 15833c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 15843c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 15853c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 15863c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 15873c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 15883c9168faSHans Rosenfeld } 15893c9168faSHans Rosenfeld 1590e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 15913c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15923c9168faSHans Rosenfeld "!nvme_admin_cmd failed for IDENTIFY"); 15933c9168faSHans Rosenfeld return (NULL); 15943c9168faSHans Rosenfeld } 15953c9168faSHans Rosenfeld 15963c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 15973c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15983c9168faSHans Rosenfeld "!IDENTIFY failed with sct = %x, sc = %x", 15993c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 16003c9168faSHans Rosenfeld goto fail; 16013c9168faSHans Rosenfeld } 16023c9168faSHans Rosenfeld 16033c9168faSHans Rosenfeld buf = kmem_alloc(NVME_IDENTIFY_BUFSIZE, KM_SLEEP); 16043c9168faSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, buf, NVME_IDENTIFY_BUFSIZE); 16053c9168faSHans Rosenfeld 16063c9168faSHans Rosenfeld fail: 16073c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16083c9168faSHans Rosenfeld 16093c9168faSHans Rosenfeld return (buf); 16103c9168faSHans Rosenfeld } 16113c9168faSHans Rosenfeld 16123c9168faSHans Rosenfeld static int 16133c9168faSHans Rosenfeld nvme_set_nqueues(nvme_t *nvme, uint16_t nqueues) 16143c9168faSHans Rosenfeld { 16153c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 16163c9168faSHans Rosenfeld nvme_nqueue_t nq = { 0 }; 16173c9168faSHans Rosenfeld 161834c938c7SPete Shephard nq.b.nq_nsq = nq.b.nq_ncq = nqueues - 1; 16193c9168faSHans Rosenfeld 16203c9168faSHans Rosenfeld cmd->nc_sqid = 0; 16213c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 16223c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES; 16233c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = NVME_FEAT_NQUEUES; 16243c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = nq.r; 16253c9168faSHans Rosenfeld 1626e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 16273c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16283c9168faSHans Rosenfeld "!nvme_admin_cmd failed for SET FEATURES (NQUEUES)"); 16293c9168faSHans Rosenfeld return (0); 16303c9168faSHans Rosenfeld } 16313c9168faSHans Rosenfeld 16323c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 16333c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16343c9168faSHans Rosenfeld "!SET FEATURES (NQUEUES) failed with sct = %x, sc = %x", 16353c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 16363c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16373c9168faSHans Rosenfeld return (0); 16383c9168faSHans Rosenfeld } 16393c9168faSHans Rosenfeld 16403c9168faSHans Rosenfeld nq.r = cmd->nc_cqe.cqe_dw0; 16413c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16423c9168faSHans Rosenfeld 16433c9168faSHans Rosenfeld /* 16443c9168faSHans Rosenfeld * Always use the same number of submission and completion queues, and 16453c9168faSHans Rosenfeld * never use more than the requested number of queues. 16463c9168faSHans Rosenfeld */ 164734c938c7SPete Shephard return (MIN(nqueues, MIN(nq.b.nq_nsq, nq.b.nq_ncq) + 1)); 16483c9168faSHans Rosenfeld } 16493c9168faSHans Rosenfeld 16503c9168faSHans Rosenfeld static int 16513c9168faSHans Rosenfeld nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx) 16523c9168faSHans Rosenfeld { 16533c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 16543c9168faSHans Rosenfeld nvme_create_queue_dw10_t dw10 = { 0 }; 16553c9168faSHans Rosenfeld nvme_create_cq_dw11_t c_dw11 = { 0 }; 16563c9168faSHans Rosenfeld nvme_create_sq_dw11_t s_dw11 = { 0 }; 16573c9168faSHans Rosenfeld 16583c9168faSHans Rosenfeld dw10.b.q_qid = idx; 16593c9168faSHans Rosenfeld dw10.b.q_qsize = qp->nq_nentry - 1; 16603c9168faSHans Rosenfeld 16613c9168faSHans Rosenfeld c_dw11.b.cq_pc = 1; 16623c9168faSHans Rosenfeld c_dw11.b.cq_ien = 1; 16633c9168faSHans Rosenfeld c_dw11.b.cq_iv = idx % nvme->n_intr_cnt; 16643c9168faSHans Rosenfeld 16653c9168faSHans Rosenfeld cmd->nc_sqid = 0; 16663c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 16673c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE; 16683c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 16693c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = c_dw11.r; 16703c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_cqdma->nd_cookie.dmac_laddress; 16713c9168faSHans Rosenfeld 1672e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 16733c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16743c9168faSHans Rosenfeld "!nvme_admin_cmd failed for CREATE CQUEUE"); 16753c9168faSHans Rosenfeld return (DDI_FAILURE); 16763c9168faSHans Rosenfeld } 16773c9168faSHans Rosenfeld 16783c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 16793c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16803c9168faSHans Rosenfeld "!CREATE CQUEUE failed with sct = %x, sc = %x", 16813c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 16823c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16833c9168faSHans Rosenfeld return (DDI_FAILURE); 16843c9168faSHans Rosenfeld } 16853c9168faSHans Rosenfeld 16863c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16873c9168faSHans Rosenfeld 16883c9168faSHans Rosenfeld s_dw11.b.sq_pc = 1; 16893c9168faSHans Rosenfeld s_dw11.b.sq_cqid = idx; 16903c9168faSHans Rosenfeld 16913c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 16923c9168faSHans Rosenfeld cmd->nc_sqid = 0; 16933c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 16943c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE; 16953c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 16963c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = s_dw11.r; 16973c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress; 16983c9168faSHans Rosenfeld 1699e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 17003c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17013c9168faSHans Rosenfeld "!nvme_admin_cmd failed for CREATE SQUEUE"); 17023c9168faSHans Rosenfeld return (DDI_FAILURE); 17033c9168faSHans Rosenfeld } 17043c9168faSHans Rosenfeld 17053c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 17063c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17073c9168faSHans Rosenfeld "!CREATE SQUEUE failed with sct = %x, sc = %x", 17083c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 17093c9168faSHans Rosenfeld nvme_free_cmd(cmd); 17103c9168faSHans Rosenfeld return (DDI_FAILURE); 17113c9168faSHans Rosenfeld } 17123c9168faSHans Rosenfeld 17133c9168faSHans Rosenfeld nvme_free_cmd(cmd); 17143c9168faSHans Rosenfeld 17153c9168faSHans Rosenfeld return (DDI_SUCCESS); 17163c9168faSHans Rosenfeld } 17173c9168faSHans Rosenfeld 17183c9168faSHans Rosenfeld static boolean_t 17193c9168faSHans Rosenfeld nvme_reset(nvme_t *nvme, boolean_t quiesce) 17203c9168faSHans Rosenfeld { 17213c9168faSHans Rosenfeld nvme_reg_csts_t csts; 17223c9168faSHans Rosenfeld int i; 17233c9168faSHans Rosenfeld 17243c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 17253c9168faSHans Rosenfeld 17263c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 17273c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) { 17283c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 17293c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 17303c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 17313c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) 17323c9168faSHans Rosenfeld break; 17333c9168faSHans Rosenfeld 17343c9168faSHans Rosenfeld if (quiesce) 17353c9168faSHans Rosenfeld drv_usecwait(50000); 17363c9168faSHans Rosenfeld else 17373c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 17383c9168faSHans Rosenfeld } 17393c9168faSHans Rosenfeld } 17403c9168faSHans Rosenfeld 17413c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, 0); 17423c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ASQ, 0); 17433c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ACQ, 0); 17443c9168faSHans Rosenfeld 17453c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 17463c9168faSHans Rosenfeld return (csts.b.csts_rdy == 0 ? B_TRUE : B_FALSE); 17473c9168faSHans Rosenfeld } 17483c9168faSHans Rosenfeld 17493c9168faSHans Rosenfeld static void 17503c9168faSHans Rosenfeld nvme_shutdown(nvme_t *nvme, int mode, boolean_t quiesce) 17513c9168faSHans Rosenfeld { 17523c9168faSHans Rosenfeld nvme_reg_cc_t cc; 17533c9168faSHans Rosenfeld nvme_reg_csts_t csts; 17543c9168faSHans Rosenfeld int i; 17553c9168faSHans Rosenfeld 17563c9168faSHans Rosenfeld ASSERT(mode == NVME_CC_SHN_NORMAL || mode == NVME_CC_SHN_ABRUPT); 17573c9168faSHans Rosenfeld 17583c9168faSHans Rosenfeld cc.r = nvme_get32(nvme, NVME_REG_CC); 17593c9168faSHans Rosenfeld cc.b.cc_shn = mode & 0x3; 17603c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 17613c9168faSHans Rosenfeld 17623c9168faSHans Rosenfeld for (i = 0; i != 10; i++) { 17633c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 17643c9168faSHans Rosenfeld if (csts.b.csts_shst == NVME_CSTS_SHN_COMPLETE) 17653c9168faSHans Rosenfeld break; 17663c9168faSHans Rosenfeld 17673c9168faSHans Rosenfeld if (quiesce) 17683c9168faSHans Rosenfeld drv_usecwait(100000); 17693c9168faSHans Rosenfeld else 17703c9168faSHans Rosenfeld delay(drv_usectohz(100000)); 17713c9168faSHans Rosenfeld } 17723c9168faSHans Rosenfeld } 17733c9168faSHans Rosenfeld 17743c9168faSHans Rosenfeld 17753c9168faSHans Rosenfeld static void 17763c9168faSHans Rosenfeld nvme_prepare_devid(nvme_t *nvme, uint32_t nsid) 17773c9168faSHans Rosenfeld { 17783c9168faSHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 17793c9168faSHans Rosenfeld char serial[sizeof (nvme->n_idctl->id_serial) + 1]; 17803c9168faSHans Rosenfeld 17813c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 17823c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_serial, serial, 17833c9168faSHans Rosenfeld sizeof (nvme->n_idctl->id_serial)); 17843c9168faSHans Rosenfeld 17853c9168faSHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 17863c9168faSHans Rosenfeld serial[sizeof (nvme->n_idctl->id_serial)] = '\0'; 17873c9168faSHans Rosenfeld 17883c9168faSHans Rosenfeld (void) snprintf(nvme->n_ns[nsid - 1].ns_devid, 17893c9168faSHans Rosenfeld sizeof (nvme->n_ns[0].ns_devid), "%4X-%s-%s-%X", 17903c9168faSHans Rosenfeld nvme->n_idctl->id_vid, model, serial, nsid); 17913c9168faSHans Rosenfeld } 17923c9168faSHans Rosenfeld 17933c9168faSHans Rosenfeld static int 17943c9168faSHans Rosenfeld nvme_init(nvme_t *nvme) 17953c9168faSHans Rosenfeld { 17963c9168faSHans Rosenfeld nvme_reg_cc_t cc = { 0 }; 17973c9168faSHans Rosenfeld nvme_reg_aqa_t aqa = { 0 }; 17983c9168faSHans Rosenfeld nvme_reg_asq_t asq = { 0 }; 17993c9168faSHans Rosenfeld nvme_reg_acq_t acq = { 0 }; 18003c9168faSHans Rosenfeld nvme_reg_cap_t cap; 18013c9168faSHans Rosenfeld nvme_reg_vs_t vs; 18023c9168faSHans Rosenfeld nvme_reg_csts_t csts; 18033c9168faSHans Rosenfeld int i = 0; 18043c9168faSHans Rosenfeld int nqueues; 1805510a6847SHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 1806510a6847SHans Rosenfeld char *vendor, *product; 18073c9168faSHans Rosenfeld 18083c9168faSHans Rosenfeld /* Check controller version */ 18093c9168faSHans Rosenfeld vs.r = nvme_get32(nvme, NVME_REG_VS); 18103c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d", 18113c9168faSHans Rosenfeld vs.b.vs_mjr, vs.b.vs_mnr); 18123c9168faSHans Rosenfeld 18132797e2b9SHans Rosenfeld if (nvme_version_major < vs.b.vs_mjr || 18142797e2b9SHans Rosenfeld (nvme_version_major == vs.b.vs_mjr && 18152797e2b9SHans Rosenfeld nvme_version_minor < vs.b.vs_mnr)) { 18163c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.%d", 18173c9168faSHans Rosenfeld nvme_version_major, nvme_version_minor); 18183c9168faSHans Rosenfeld if (nvme->n_strict_version) 18193c9168faSHans Rosenfeld goto fail; 18203c9168faSHans Rosenfeld } 18213c9168faSHans Rosenfeld 18223c9168faSHans Rosenfeld /* retrieve controller configuration */ 18233c9168faSHans Rosenfeld cap.r = nvme_get64(nvme, NVME_REG_CAP); 18243c9168faSHans Rosenfeld 18253c9168faSHans Rosenfeld if ((cap.b.cap_css & NVME_CAP_CSS_NVM) == 0) { 18263c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 18273c9168faSHans Rosenfeld "!NVM command set not supported by hardware"); 18283c9168faSHans Rosenfeld goto fail; 18293c9168faSHans Rosenfeld } 18303c9168faSHans Rosenfeld 18313c9168faSHans Rosenfeld nvme->n_nssr_supported = cap.b.cap_nssrs; 18323c9168faSHans Rosenfeld nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd; 18333c9168faSHans Rosenfeld nvme->n_timeout = cap.b.cap_to; 18343c9168faSHans Rosenfeld nvme->n_arbitration_mechanisms = cap.b.cap_ams; 18353c9168faSHans Rosenfeld nvme->n_cont_queues_reqd = cap.b.cap_cqr; 18363c9168faSHans Rosenfeld nvme->n_max_queue_entries = cap.b.cap_mqes + 1; 18373c9168faSHans Rosenfeld 18383c9168faSHans Rosenfeld /* 18393c9168faSHans Rosenfeld * The MPSMIN and MPSMAX fields in the CAP register use 0 to specify 18403c9168faSHans Rosenfeld * the base page size of 4k (1<<12), so add 12 here to get the real 18413c9168faSHans Rosenfeld * page size value. 18423c9168faSHans Rosenfeld */ 18433c9168faSHans Rosenfeld nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT), 18443c9168faSHans Rosenfeld cap.b.cap_mpsmax + 12); 18453c9168faSHans Rosenfeld nvme->n_pagesize = 1UL << (nvme->n_pageshift); 18463c9168faSHans Rosenfeld 18473c9168faSHans Rosenfeld /* 18483c9168faSHans Rosenfeld * Set up Queue DMA to transfer at least 1 page-aligned page at a time. 18493c9168faSHans Rosenfeld */ 18503c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize; 18513c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 18523c9168faSHans Rosenfeld 18533c9168faSHans Rosenfeld /* 18543c9168faSHans Rosenfeld * Set up PRP DMA to transfer 1 page-aligned page at a time. 18553c9168faSHans Rosenfeld * Maxxfer may be increased after we identified the controller limits. 18563c9168faSHans Rosenfeld */ 18573c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize; 18583c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 18593c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize; 18602f95345bSYouzhong Yang nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1; 18613c9168faSHans Rosenfeld 18623c9168faSHans Rosenfeld /* 18633c9168faSHans Rosenfeld * Reset controller if it's still in ready state. 18643c9168faSHans Rosenfeld */ 18653c9168faSHans Rosenfeld if (nvme_reset(nvme, B_FALSE) == B_FALSE) { 18663c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller"); 18673c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 18683c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 18693c9168faSHans Rosenfeld goto fail; 18703c9168faSHans Rosenfeld } 18713c9168faSHans Rosenfeld 18723c9168faSHans Rosenfeld /* 18733c9168faSHans Rosenfeld * Create the admin queue pair. 18743c9168faSHans Rosenfeld */ 18753c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0) 18763c9168faSHans Rosenfeld != DDI_SUCCESS) { 18773c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 18783c9168faSHans Rosenfeld "!unable to allocate admin qpair"); 18793c9168faSHans Rosenfeld goto fail; 18803c9168faSHans Rosenfeld } 18813c9168faSHans Rosenfeld nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP); 18823c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 18833c9168faSHans Rosenfeld 18843c9168faSHans Rosenfeld nvme->n_progress |= NVME_ADMIN_QUEUE; 18853c9168faSHans Rosenfeld 18863c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 18873c9168faSHans Rosenfeld "admin-queue-len", nvme->n_admin_queue_len); 18883c9168faSHans Rosenfeld 18893c9168faSHans Rosenfeld aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1; 18903c9168faSHans Rosenfeld asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress; 18913c9168faSHans Rosenfeld acq = nvme->n_adminq->nq_cqdma->nd_cookie.dmac_laddress; 18923c9168faSHans Rosenfeld 18933c9168faSHans Rosenfeld ASSERT((asq & (nvme->n_pagesize - 1)) == 0); 18943c9168faSHans Rosenfeld ASSERT((acq & (nvme->n_pagesize - 1)) == 0); 18953c9168faSHans Rosenfeld 18963c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, aqa.r); 18973c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ASQ, asq); 18983c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ACQ, acq); 18993c9168faSHans Rosenfeld 190034c938c7SPete Shephard cc.b.cc_ams = 0; /* use Round-Robin arbitration */ 190134c938c7SPete Shephard cc.b.cc_css = 0; /* use NVM command set */ 19023c9168faSHans Rosenfeld cc.b.cc_mps = nvme->n_pageshift - 12; 190334c938c7SPete Shephard cc.b.cc_shn = 0; /* no shutdown in progress */ 190434c938c7SPete Shephard cc.b.cc_en = 1; /* enable controller */ 190534c938c7SPete Shephard cc.b.cc_iosqes = 6; /* submission queue entry is 2^6 bytes long */ 190634c938c7SPete Shephard cc.b.cc_iocqes = 4; /* completion queue entry is 2^4 bytes long */ 19073c9168faSHans Rosenfeld 19083c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 19093c9168faSHans Rosenfeld 19103c9168faSHans Rosenfeld /* 19113c9168faSHans Rosenfeld * Wait for the controller to become ready. 19123c9168faSHans Rosenfeld */ 19133c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 19143c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 19153c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 19163c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 19173c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 19183c9168faSHans Rosenfeld 19193c9168faSHans Rosenfeld if (csts.b.csts_cfs == 1) { 19203c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19213c9168faSHans Rosenfeld "!controller fatal status at init"); 19223c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, 19233c9168faSHans Rosenfeld DDI_SERVICE_LOST); 19243c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 19253c9168faSHans Rosenfeld goto fail; 19263c9168faSHans Rosenfeld } 19273c9168faSHans Rosenfeld 19283c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) 19293c9168faSHans Rosenfeld break; 19303c9168faSHans Rosenfeld } 19313c9168faSHans Rosenfeld } 19323c9168faSHans Rosenfeld 19333c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 19343c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!controller not ready"); 19353c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 19363c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 19373c9168faSHans Rosenfeld goto fail; 19383c9168faSHans Rosenfeld } 19393c9168faSHans Rosenfeld 19403c9168faSHans Rosenfeld /* 19413c9168faSHans Rosenfeld * Assume an abort command limit of 1. We'll destroy and re-init 19423c9168faSHans Rosenfeld * that later when we know the true abort command limit. 19433c9168faSHans Rosenfeld */ 19443c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL); 19453c9168faSHans Rosenfeld 1946bf26ea4bSHans Rosenfeld /* 1947bf26ea4bSHans Rosenfeld * Setup initial interrupt for admin queue. 1948bf26ea4bSHans Rosenfeld */ 1949bf26ea4bSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1) 1950bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 1951bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1) 1952bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 1953bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1) 1954bf26ea4bSHans Rosenfeld != DDI_SUCCESS)) { 1955bf26ea4bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 1956bf26ea4bSHans Rosenfeld "!failed to setup initial interrupt"); 1957bf26ea4bSHans Rosenfeld goto fail; 1958bf26ea4bSHans Rosenfeld } 1959bf26ea4bSHans Rosenfeld 19603c9168faSHans Rosenfeld /* 19613c9168faSHans Rosenfeld * Post an asynchronous event command to catch errors. 19623c9168faSHans Rosenfeld */ 19633c9168faSHans Rosenfeld if (nvme_async_event(nvme) != DDI_SUCCESS) { 19643c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19653c9168faSHans Rosenfeld "!failed to post async event"); 19663c9168faSHans Rosenfeld goto fail; 19673c9168faSHans Rosenfeld } 19683c9168faSHans Rosenfeld 19693c9168faSHans Rosenfeld /* 19703c9168faSHans Rosenfeld * Identify Controller 19713c9168faSHans Rosenfeld */ 19723c9168faSHans Rosenfeld nvme->n_idctl = nvme_identify(nvme, 0); 19733c9168faSHans Rosenfeld if (nvme->n_idctl == NULL) { 19743c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19753c9168faSHans Rosenfeld "!failed to identify controller"); 19763c9168faSHans Rosenfeld goto fail; 19773c9168faSHans Rosenfeld } 19783c9168faSHans Rosenfeld 1979510a6847SHans Rosenfeld /* 1980510a6847SHans Rosenfeld * Get Vendor & Product ID 1981510a6847SHans Rosenfeld */ 1982510a6847SHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 1983510a6847SHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 1984510a6847SHans Rosenfeld sata_split_model(model, &vendor, &product); 1985510a6847SHans Rosenfeld 1986510a6847SHans Rosenfeld if (vendor == NULL) 1987510a6847SHans Rosenfeld nvme->n_vendor = strdup("NVMe"); 1988510a6847SHans Rosenfeld else 1989510a6847SHans Rosenfeld nvme->n_vendor = strdup(vendor); 1990510a6847SHans Rosenfeld 1991510a6847SHans Rosenfeld nvme->n_product = strdup(product); 1992510a6847SHans Rosenfeld 19933c9168faSHans Rosenfeld /* 19943c9168faSHans Rosenfeld * Get controller limits. 19953c9168faSHans Rosenfeld */ 19963c9168faSHans Rosenfeld nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT, 19973c9168faSHans Rosenfeld MIN(nvme->n_admin_queue_len / 10, 19983c9168faSHans Rosenfeld MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit))); 19993c9168faSHans Rosenfeld 20003c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 20013c9168faSHans Rosenfeld "async-event-limit", nvme->n_async_event_limit); 20023c9168faSHans Rosenfeld 20033c9168faSHans Rosenfeld nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1; 20043c9168faSHans Rosenfeld 20056afc9eb2SHans Rosenfeld /* 20066afc9eb2SHans Rosenfeld * Reinitialize the semaphore with the true abort command limit 20076afc9eb2SHans Rosenfeld * supported by the hardware. It's not necessary to disable interrupts 20086afc9eb2SHans Rosenfeld * as only command aborts use the semaphore, and no commands are 20096afc9eb2SHans Rosenfeld * executed or aborted while we're here. 20106afc9eb2SHans Rosenfeld */ 20113c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 20123c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL, 20133c9168faSHans Rosenfeld SEMA_DRIVER, NULL); 20143c9168faSHans Rosenfeld 20153c9168faSHans Rosenfeld nvme->n_progress |= NVME_CTRL_LIMITS; 20163c9168faSHans Rosenfeld 20173c9168faSHans Rosenfeld if (nvme->n_idctl->id_mdts == 0) 20183c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536; 20193c9168faSHans Rosenfeld else 20203c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = 20213c9168faSHans Rosenfeld 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts); 20223c9168faSHans Rosenfeld 20233c9168faSHans Rosenfeld nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1; 20243c9168faSHans Rosenfeld 20253c9168faSHans Rosenfeld /* 20263c9168faSHans Rosenfeld * Limit n_max_data_transfer_size to what we can handle in one PRP. 20273c9168faSHans Rosenfeld * Chained PRPs are currently unsupported. 20283c9168faSHans Rosenfeld * 20293c9168faSHans Rosenfeld * This is a no-op on hardware which doesn't support a transfer size 20303c9168faSHans Rosenfeld * big enough to require chained PRPs. 20313c9168faSHans Rosenfeld */ 20323c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size, 20333c9168faSHans Rosenfeld (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize)); 20343c9168faSHans Rosenfeld 20353c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size; 20363c9168faSHans Rosenfeld 20373c9168faSHans Rosenfeld /* 20383c9168faSHans Rosenfeld * Make sure the minimum/maximum queue entry sizes are not 20393c9168faSHans Rosenfeld * larger/smaller than the default. 20403c9168faSHans Rosenfeld */ 20413c9168faSHans Rosenfeld 20423c9168faSHans Rosenfeld if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) || 20433c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) || 20443c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) || 20453c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t))) 20463c9168faSHans Rosenfeld goto fail; 20473c9168faSHans Rosenfeld 20483c9168faSHans Rosenfeld /* 20493c9168faSHans Rosenfeld * Check for the presence of a Volatile Write Cache. If present, 20503c9168faSHans Rosenfeld * enable it by default. 20513c9168faSHans Rosenfeld */ 20523c9168faSHans Rosenfeld if (nvme->n_idctl->id_vwc.vwc_present == 0) { 20533c9168faSHans Rosenfeld nvme->n_volatile_write_cache_enabled = B_FALSE; 20543c9168faSHans Rosenfeld nvme_bd_ops.o_sync_cache = NULL; 20553c9168faSHans Rosenfeld } else { 20563c9168faSHans Rosenfeld /* 20573c9168faSHans Rosenfeld * TODO: send SET FEATURES to enable VWC 20583c9168faSHans Rosenfeld * (have no hardware to test this) 20593c9168faSHans Rosenfeld */ 20603c9168faSHans Rosenfeld nvme->n_volatile_write_cache_enabled = B_FALSE; 20613c9168faSHans Rosenfeld nvme_bd_ops.o_sync_cache = NULL; 20623c9168faSHans Rosenfeld } 20633c9168faSHans Rosenfeld 20643c9168faSHans Rosenfeld /* 20653c9168faSHans Rosenfeld * Grab a copy of all mandatory log pages. 20663c9168faSHans Rosenfeld * 20673c9168faSHans Rosenfeld * TODO: should go away once user space tool exists to print logs 20683c9168faSHans Rosenfeld */ 20693c9168faSHans Rosenfeld nvme->n_error_log = (nvme_error_log_entry_t *) 20703c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_ERROR); 20713c9168faSHans Rosenfeld nvme->n_health_log = (nvme_health_log_t *) 20723c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_HEALTH, -1); 20733c9168faSHans Rosenfeld nvme->n_fwslot_log = (nvme_fwslot_log_t *) 20743c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_FWSLOT); 20753c9168faSHans Rosenfeld 20763c9168faSHans Rosenfeld /* 20773c9168faSHans Rosenfeld * Identify Namespaces 20783c9168faSHans Rosenfeld */ 20793c9168faSHans Rosenfeld nvme->n_namespace_count = nvme->n_idctl->id_nn; 20803c9168faSHans Rosenfeld nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) * 20813c9168faSHans Rosenfeld nvme->n_namespace_count, KM_SLEEP); 20823c9168faSHans Rosenfeld 20833c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 20843c9168faSHans Rosenfeld nvme_identify_nsid_t *idns; 20853c9168faSHans Rosenfeld int last_rp; 20863c9168faSHans Rosenfeld 20873c9168faSHans Rosenfeld nvme->n_ns[i].ns_nvme = nvme; 20883c9168faSHans Rosenfeld nvme->n_ns[i].ns_idns = idns = nvme_identify(nvme, i + 1); 20893c9168faSHans Rosenfeld 20903c9168faSHans Rosenfeld if (idns == NULL) { 20913c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20923c9168faSHans Rosenfeld "!failed to identify namespace %d", i + 1); 20933c9168faSHans Rosenfeld goto fail; 20943c9168faSHans Rosenfeld } 20953c9168faSHans Rosenfeld 20963c9168faSHans Rosenfeld nvme->n_ns[i].ns_id = i + 1; 20973c9168faSHans Rosenfeld nvme->n_ns[i].ns_block_count = idns->id_nsize; 20983c9168faSHans Rosenfeld nvme->n_ns[i].ns_block_size = 20993c9168faSHans Rosenfeld 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads; 21003c9168faSHans Rosenfeld nvme->n_ns[i].ns_best_block_size = nvme->n_ns[i].ns_block_size; 21013c9168faSHans Rosenfeld 21023c9168faSHans Rosenfeld nvme_prepare_devid(nvme, nvme->n_ns[i].ns_id); 21033c9168faSHans Rosenfeld 21043c9168faSHans Rosenfeld /* 21053c9168faSHans Rosenfeld * Find the LBA format with no metadata and the best relative 21063c9168faSHans Rosenfeld * performance. A value of 3 means "degraded", 0 is best. 21073c9168faSHans Rosenfeld */ 21083c9168faSHans Rosenfeld last_rp = 3; 21090d140ff9SHans Rosenfeld for (int j = 0; j <= idns->id_nlbaf; j++) { 21103c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_lbads == 0) 21113c9168faSHans Rosenfeld break; 21123c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_ms != 0) 21133c9168faSHans Rosenfeld continue; 21143c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_rp >= last_rp) 21153c9168faSHans Rosenfeld continue; 21163c9168faSHans Rosenfeld last_rp = idns->id_lbaf[j].lbaf_rp; 21173c9168faSHans Rosenfeld nvme->n_ns[i].ns_best_block_size = 21183c9168faSHans Rosenfeld 1 << idns->id_lbaf[j].lbaf_lbads; 21193c9168faSHans Rosenfeld } 21203c9168faSHans Rosenfeld 21213c9168faSHans Rosenfeld /* 21223c9168faSHans Rosenfeld * We currently don't support namespaces that use either: 21233c9168faSHans Rosenfeld * - thin provisioning 21243c9168faSHans Rosenfeld * - protection information 21253c9168faSHans Rosenfeld */ 21263c9168faSHans Rosenfeld if (idns->id_nsfeat.f_thin || 21273c9168faSHans Rosenfeld idns->id_dps.dp_pinfo) { 21283c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21293c9168faSHans Rosenfeld "!ignoring namespace %d, unsupported features: " 21309cfb65faSHans Rosenfeld "thin = %d, pinfo = %d", i + 1, 21319cfb65faSHans Rosenfeld idns->id_nsfeat.f_thin, idns->id_dps.dp_pinfo); 21323c9168faSHans Rosenfeld nvme->n_ns[i].ns_ignore = B_TRUE; 21333c9168faSHans Rosenfeld } 21343c9168faSHans Rosenfeld } 21353c9168faSHans Rosenfeld 21363c9168faSHans Rosenfeld /* 21373c9168faSHans Rosenfeld * Try to set up MSI/MSI-X interrupts. 21383c9168faSHans Rosenfeld */ 21393c9168faSHans Rosenfeld if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX)) 21403c9168faSHans Rosenfeld != 0) { 21413c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 21423c9168faSHans Rosenfeld 21433c9168faSHans Rosenfeld nqueues = MIN(UINT16_MAX, ncpus); 21443c9168faSHans Rosenfeld 21453c9168faSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 21463c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS) && 21473c9168faSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 21483c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS)) { 21493c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21503c9168faSHans Rosenfeld "!failed to setup MSI/MSI-X interrupts"); 21513c9168faSHans Rosenfeld goto fail; 21523c9168faSHans Rosenfeld } 21533c9168faSHans Rosenfeld } 21543c9168faSHans Rosenfeld 21553c9168faSHans Rosenfeld nqueues = nvme->n_intr_cnt; 21563c9168faSHans Rosenfeld 21573c9168faSHans Rosenfeld /* 21583c9168faSHans Rosenfeld * Create I/O queue pairs. 21593c9168faSHans Rosenfeld */ 21603c9168faSHans Rosenfeld nvme->n_ioq_count = nvme_set_nqueues(nvme, nqueues); 21613c9168faSHans Rosenfeld if (nvme->n_ioq_count == 0) { 21623c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21633c9168faSHans Rosenfeld "!failed to set number of I/O queues to %d", nqueues); 21643c9168faSHans Rosenfeld goto fail; 21653c9168faSHans Rosenfeld } 21663c9168faSHans Rosenfeld 21673c9168faSHans Rosenfeld /* 21683c9168faSHans Rosenfeld * Reallocate I/O queue array 21693c9168faSHans Rosenfeld */ 21703c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *)); 21713c9168faSHans Rosenfeld nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) * 21723c9168faSHans Rosenfeld (nvme->n_ioq_count + 1), KM_SLEEP); 21733c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 21743c9168faSHans Rosenfeld 21753c9168faSHans Rosenfeld /* 21763c9168faSHans Rosenfeld * If we got less queues than we asked for we might as well give 21773c9168faSHans Rosenfeld * some of the interrupt vectors back to the system. 21783c9168faSHans Rosenfeld */ 21793c9168faSHans Rosenfeld if (nvme->n_ioq_count < nqueues) { 21803c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 21813c9168faSHans Rosenfeld 218234c938c7SPete Shephard if (nvme_setup_interrupts(nvme, nvme->n_intr_type, 218334c938c7SPete Shephard nvme->n_ioq_count) != DDI_SUCCESS) { 21843c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21853c9168faSHans Rosenfeld "!failed to reduce number of interrupts"); 21863c9168faSHans Rosenfeld goto fail; 21873c9168faSHans Rosenfeld } 21883c9168faSHans Rosenfeld } 21893c9168faSHans Rosenfeld 21903c9168faSHans Rosenfeld /* 21913c9168faSHans Rosenfeld * Alloc & register I/O queue pairs 21923c9168faSHans Rosenfeld */ 21933c9168faSHans Rosenfeld nvme->n_io_queue_len = 21943c9168faSHans Rosenfeld MIN(nvme->n_io_queue_len, nvme->n_max_queue_entries); 21953c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-queue-len", 21963c9168faSHans Rosenfeld nvme->n_io_queue_len); 21973c9168faSHans Rosenfeld 21983c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 21993c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_io_queue_len, 22003c9168faSHans Rosenfeld &nvme->n_ioq[i], i) != DDI_SUCCESS) { 22013c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22023c9168faSHans Rosenfeld "!unable to allocate I/O qpair %d", i); 22033c9168faSHans Rosenfeld goto fail; 22043c9168faSHans Rosenfeld } 22053c9168faSHans Rosenfeld 22063c9168faSHans Rosenfeld if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i) 22073c9168faSHans Rosenfeld != DDI_SUCCESS) { 22083c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22093c9168faSHans Rosenfeld "!unable to create I/O qpair %d", i); 22103c9168faSHans Rosenfeld goto fail; 22113c9168faSHans Rosenfeld } 22123c9168faSHans Rosenfeld } 22133c9168faSHans Rosenfeld 22143c9168faSHans Rosenfeld /* 22153c9168faSHans Rosenfeld * Post more asynchronous events commands to reduce event reporting 22163c9168faSHans Rosenfeld * latency as suggested by the spec. 22173c9168faSHans Rosenfeld */ 22183c9168faSHans Rosenfeld for (i = 1; i != nvme->n_async_event_limit; i++) { 22193c9168faSHans Rosenfeld if (nvme_async_event(nvme) != DDI_SUCCESS) { 22203c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22213c9168faSHans Rosenfeld "!failed to post async event %d", i); 22223c9168faSHans Rosenfeld goto fail; 22233c9168faSHans Rosenfeld } 22243c9168faSHans Rosenfeld } 22253c9168faSHans Rosenfeld 22263c9168faSHans Rosenfeld return (DDI_SUCCESS); 22273c9168faSHans Rosenfeld 22283c9168faSHans Rosenfeld fail: 22293c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 22303c9168faSHans Rosenfeld return (DDI_FAILURE); 22313c9168faSHans Rosenfeld } 22323c9168faSHans Rosenfeld 22333c9168faSHans Rosenfeld static uint_t 22343c9168faSHans Rosenfeld nvme_intr(caddr_t arg1, caddr_t arg2) 22353c9168faSHans Rosenfeld { 22363c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 22373c9168faSHans Rosenfeld nvme_t *nvme = (nvme_t *)arg1; 22383c9168faSHans Rosenfeld int inum = (int)(uintptr_t)arg2; 2239bf26ea4bSHans Rosenfeld int ccnt = 0; 22403c9168faSHans Rosenfeld int qnum; 22413c9168faSHans Rosenfeld nvme_cmd_t *cmd; 22423c9168faSHans Rosenfeld 22433c9168faSHans Rosenfeld if (inum >= nvme->n_intr_cnt) 22443c9168faSHans Rosenfeld return (DDI_INTR_UNCLAIMED); 22453c9168faSHans Rosenfeld 22463c9168faSHans Rosenfeld /* 22473c9168faSHans Rosenfeld * The interrupt vector a queue uses is calculated as queue_idx % 22483c9168faSHans Rosenfeld * intr_cnt in nvme_create_io_qpair(). Iterate through the queue array 22493c9168faSHans Rosenfeld * in steps of n_intr_cnt to process all queues using this vector. 22503c9168faSHans Rosenfeld */ 22513c9168faSHans Rosenfeld for (qnum = inum; 22523c9168faSHans Rosenfeld qnum < nvme->n_ioq_count + 1 && nvme->n_ioq[qnum] != NULL; 22533c9168faSHans Rosenfeld qnum += nvme->n_intr_cnt) { 22543c9168faSHans Rosenfeld while ((cmd = nvme_retrieve_cmd(nvme, nvme->n_ioq[qnum]))) { 22553c9168faSHans Rosenfeld taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq, 22563c9168faSHans Rosenfeld cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent); 2257bf26ea4bSHans Rosenfeld ccnt++; 22583c9168faSHans Rosenfeld } 22593c9168faSHans Rosenfeld } 22603c9168faSHans Rosenfeld 2261bf26ea4bSHans Rosenfeld return (ccnt > 0 ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); 22623c9168faSHans Rosenfeld } 22633c9168faSHans Rosenfeld 22643c9168faSHans Rosenfeld static void 22656afc9eb2SHans Rosenfeld nvme_release_interrupts(nvme_t *nvme) 22663c9168faSHans Rosenfeld { 22673c9168faSHans Rosenfeld int i; 22683c9168faSHans Rosenfeld 22693c9168faSHans Rosenfeld for (i = 0; i < nvme->n_intr_cnt; i++) { 22703c9168faSHans Rosenfeld if (nvme->n_inth[i] == NULL) 22713c9168faSHans Rosenfeld break; 22723c9168faSHans Rosenfeld 22733c9168faSHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 22743c9168faSHans Rosenfeld (void) ddi_intr_block_disable(&nvme->n_inth[i], 1); 22753c9168faSHans Rosenfeld else 22763c9168faSHans Rosenfeld (void) ddi_intr_disable(nvme->n_inth[i]); 22773c9168faSHans Rosenfeld 22783c9168faSHans Rosenfeld (void) ddi_intr_remove_handler(nvme->n_inth[i]); 22793c9168faSHans Rosenfeld (void) ddi_intr_free(nvme->n_inth[i]); 22803c9168faSHans Rosenfeld } 22813c9168faSHans Rosenfeld 22823c9168faSHans Rosenfeld kmem_free(nvme->n_inth, nvme->n_inth_sz); 22833c9168faSHans Rosenfeld nvme->n_inth = NULL; 22843c9168faSHans Rosenfeld nvme->n_inth_sz = 0; 22853c9168faSHans Rosenfeld 22863c9168faSHans Rosenfeld nvme->n_progress &= ~NVME_INTERRUPTS; 22873c9168faSHans Rosenfeld } 22883c9168faSHans Rosenfeld 22893c9168faSHans Rosenfeld static int 22903c9168faSHans Rosenfeld nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs) 22913c9168faSHans Rosenfeld { 22923c9168faSHans Rosenfeld int nintrs, navail, count; 22933c9168faSHans Rosenfeld int ret; 22943c9168faSHans Rosenfeld int i; 22953c9168faSHans Rosenfeld 22963c9168faSHans Rosenfeld if (nvme->n_intr_types == 0) { 22973c9168faSHans Rosenfeld ret = ddi_intr_get_supported_types(nvme->n_dip, 22983c9168faSHans Rosenfeld &nvme->n_intr_types); 22993c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23003c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23013c9168faSHans Rosenfeld "!%s: ddi_intr_get_supported types failed", 23023c9168faSHans Rosenfeld __func__); 23033c9168faSHans Rosenfeld return (ret); 23043c9168faSHans Rosenfeld } 23053c9168faSHans Rosenfeld } 23063c9168faSHans Rosenfeld 23073c9168faSHans Rosenfeld if ((nvme->n_intr_types & intr_type) == 0) 23083c9168faSHans Rosenfeld return (DDI_FAILURE); 23093c9168faSHans Rosenfeld 23103c9168faSHans Rosenfeld ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs); 23113c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23123c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed", 23133c9168faSHans Rosenfeld __func__); 23143c9168faSHans Rosenfeld return (ret); 23153c9168faSHans Rosenfeld } 23163c9168faSHans Rosenfeld 23173c9168faSHans Rosenfeld ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail); 23183c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23193c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed", 23203c9168faSHans Rosenfeld __func__); 23213c9168faSHans Rosenfeld return (ret); 23223c9168faSHans Rosenfeld } 23233c9168faSHans Rosenfeld 23243c9168faSHans Rosenfeld /* We want at most one interrupt per queue pair. */ 23253c9168faSHans Rosenfeld if (navail > nqpairs) 23263c9168faSHans Rosenfeld navail = nqpairs; 23273c9168faSHans Rosenfeld 23283c9168faSHans Rosenfeld nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail; 23293c9168faSHans Rosenfeld nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP); 23303c9168faSHans Rosenfeld 23313c9168faSHans Rosenfeld ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail, 23323c9168faSHans Rosenfeld &count, 0); 23333c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23343c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed", 23353c9168faSHans Rosenfeld __func__); 23363c9168faSHans Rosenfeld goto fail; 23373c9168faSHans Rosenfeld } 23383c9168faSHans Rosenfeld 23393c9168faSHans Rosenfeld nvme->n_intr_cnt = count; 23403c9168faSHans Rosenfeld 23413c9168faSHans Rosenfeld ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri); 23423c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23433c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed", 23443c9168faSHans Rosenfeld __func__); 23453c9168faSHans Rosenfeld goto fail; 23463c9168faSHans Rosenfeld } 23473c9168faSHans Rosenfeld 23483c9168faSHans Rosenfeld for (i = 0; i < count; i++) { 23493c9168faSHans Rosenfeld ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr, 23503c9168faSHans Rosenfeld (void *)nvme, (void *)(uintptr_t)i); 23513c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23523c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23533c9168faSHans Rosenfeld "!%s: ddi_intr_add_handler failed", __func__); 23543c9168faSHans Rosenfeld goto fail; 23553c9168faSHans Rosenfeld } 23563c9168faSHans Rosenfeld } 23573c9168faSHans Rosenfeld 23583c9168faSHans Rosenfeld (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap); 23593c9168faSHans Rosenfeld 23606afc9eb2SHans Rosenfeld for (i = 0; i < count; i++) { 236175b41617SHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 236275b41617SHans Rosenfeld ret = ddi_intr_block_enable(&nvme->n_inth[i], 1); 236375b41617SHans Rosenfeld else 236475b41617SHans Rosenfeld ret = ddi_intr_enable(nvme->n_inth[i]); 23653c9168faSHans Rosenfeld 236675b41617SHans Rosenfeld if (ret != DDI_SUCCESS) { 236775b41617SHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 236875b41617SHans Rosenfeld "!%s: enabling interrupt %d failed", __func__, i); 236975b41617SHans Rosenfeld goto fail; 23706afc9eb2SHans Rosenfeld } 23716afc9eb2SHans Rosenfeld } 23726afc9eb2SHans Rosenfeld 23733c9168faSHans Rosenfeld nvme->n_intr_type = intr_type; 23743c9168faSHans Rosenfeld 23753c9168faSHans Rosenfeld nvme->n_progress |= NVME_INTERRUPTS; 23763c9168faSHans Rosenfeld 23773c9168faSHans Rosenfeld return (DDI_SUCCESS); 23783c9168faSHans Rosenfeld 23793c9168faSHans Rosenfeld fail: 23803c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 23813c9168faSHans Rosenfeld 23823c9168faSHans Rosenfeld return (ret); 23833c9168faSHans Rosenfeld } 23843c9168faSHans Rosenfeld 23853c9168faSHans Rosenfeld static int 23863c9168faSHans Rosenfeld nvme_fm_errcb(dev_info_t *dip, ddi_fm_error_t *fm_error, const void *arg) 23873c9168faSHans Rosenfeld { 23883c9168faSHans Rosenfeld _NOTE(ARGUNUSED(arg)); 23893c9168faSHans Rosenfeld 23903c9168faSHans Rosenfeld pci_ereport_post(dip, fm_error, NULL); 23913c9168faSHans Rosenfeld return (fm_error->fme_status); 23923c9168faSHans Rosenfeld } 23933c9168faSHans Rosenfeld 23943c9168faSHans Rosenfeld static int 23953c9168faSHans Rosenfeld nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 23963c9168faSHans Rosenfeld { 23973c9168faSHans Rosenfeld nvme_t *nvme; 23983c9168faSHans Rosenfeld int instance; 23993c9168faSHans Rosenfeld int nregs; 24003c9168faSHans Rosenfeld off_t regsize; 24013c9168faSHans Rosenfeld int i; 24023c9168faSHans Rosenfeld char name[32]; 24033c9168faSHans Rosenfeld 24043c9168faSHans Rosenfeld if (cmd != DDI_ATTACH) 24053c9168faSHans Rosenfeld return (DDI_FAILURE); 24063c9168faSHans Rosenfeld 24073c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 24083c9168faSHans Rosenfeld 24093c9168faSHans Rosenfeld if (ddi_soft_state_zalloc(nvme_state, instance) != DDI_SUCCESS) 24103c9168faSHans Rosenfeld return (DDI_FAILURE); 24113c9168faSHans Rosenfeld 24123c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 24133c9168faSHans Rosenfeld ddi_set_driver_private(dip, nvme); 24143c9168faSHans Rosenfeld nvme->n_dip = dip; 24153c9168faSHans Rosenfeld 24163c9168faSHans Rosenfeld nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 24173c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "strict-version", 1) == 1 ? B_TRUE : B_FALSE; 24183c9168faSHans Rosenfeld nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY, 24193c9168faSHans Rosenfeld dip, DDI_PROP_DONTPASS, "ignore-unknown-vendor-status", 0) == 1 ? 24203c9168faSHans Rosenfeld B_TRUE : B_FALSE; 24213c9168faSHans Rosenfeld nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 24223c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "admin-queue-len", NVME_DEFAULT_ADMIN_QUEUE_LEN); 24233c9168faSHans Rosenfeld nvme->n_io_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 24243c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "io-queue-len", NVME_DEFAULT_IO_QUEUE_LEN); 24253c9168faSHans Rosenfeld nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 24263c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "async-event-limit", 24273c9168faSHans Rosenfeld NVME_DEFAULT_ASYNC_EVENT_LIMIT); 24283c9168faSHans Rosenfeld 24293c9168faSHans Rosenfeld if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN) 24303c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN; 24313c9168faSHans Rosenfeld else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN) 24323c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN; 24333c9168faSHans Rosenfeld 24343c9168faSHans Rosenfeld if (nvme->n_io_queue_len < NVME_MIN_IO_QUEUE_LEN) 24353c9168faSHans Rosenfeld nvme->n_io_queue_len = NVME_MIN_IO_QUEUE_LEN; 24363c9168faSHans Rosenfeld 24373c9168faSHans Rosenfeld if (nvme->n_async_event_limit < 1) 24383c9168faSHans Rosenfeld nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT; 24393c9168faSHans Rosenfeld 24403c9168faSHans Rosenfeld nvme->n_reg_acc_attr = nvme_reg_acc_attr; 24413c9168faSHans Rosenfeld nvme->n_queue_dma_attr = nvme_queue_dma_attr; 24423c9168faSHans Rosenfeld nvme->n_prp_dma_attr = nvme_prp_dma_attr; 24433c9168faSHans Rosenfeld nvme->n_sgl_dma_attr = nvme_sgl_dma_attr; 24443c9168faSHans Rosenfeld 24453c9168faSHans Rosenfeld /* 24463c9168faSHans Rosenfeld * Setup FMA support. 24473c9168faSHans Rosenfeld */ 24483c9168faSHans Rosenfeld nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip, 24493c9168faSHans Rosenfeld DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 24503c9168faSHans Rosenfeld DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 24513c9168faSHans Rosenfeld DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 24523c9168faSHans Rosenfeld 24533c9168faSHans Rosenfeld ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc); 24543c9168faSHans Rosenfeld 24553c9168faSHans Rosenfeld if (nvme->n_fm_cap) { 24563c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE) 24573c9168faSHans Rosenfeld nvme->n_reg_acc_attr.devacc_attr_access = 24583c9168faSHans Rosenfeld DDI_FLAGERR_ACC; 24593c9168faSHans Rosenfeld 24603c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) { 24613c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 24623c9168faSHans Rosenfeld nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 24633c9168faSHans Rosenfeld } 24643c9168faSHans Rosenfeld 24653c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 24663c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 24673c9168faSHans Rosenfeld pci_ereport_setup(dip); 24683c9168faSHans Rosenfeld 24693c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 24703c9168faSHans Rosenfeld ddi_fm_handler_register(dip, nvme_fm_errcb, 24713c9168faSHans Rosenfeld (void *)nvme); 24723c9168faSHans Rosenfeld } 24733c9168faSHans Rosenfeld 24743c9168faSHans Rosenfeld nvme->n_progress |= NVME_FMA_INIT; 24753c9168faSHans Rosenfeld 24763c9168faSHans Rosenfeld /* 24773c9168faSHans Rosenfeld * The spec defines several register sets. Only the controller 24783c9168faSHans Rosenfeld * registers (set 1) are currently used. 24793c9168faSHans Rosenfeld */ 24803c9168faSHans Rosenfeld if (ddi_dev_nregs(dip, &nregs) == DDI_FAILURE || 24813c9168faSHans Rosenfeld nregs < 2 || 24823c9168faSHans Rosenfeld ddi_dev_regsize(dip, 1, ®size) == DDI_FAILURE) 24833c9168faSHans Rosenfeld goto fail; 24843c9168faSHans Rosenfeld 24853c9168faSHans Rosenfeld if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize, 24863c9168faSHans Rosenfeld &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) { 24873c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to map regset 1"); 24883c9168faSHans Rosenfeld goto fail; 24893c9168faSHans Rosenfeld } 24903c9168faSHans Rosenfeld 24913c9168faSHans Rosenfeld nvme->n_progress |= NVME_REGS_MAPPED; 24923c9168faSHans Rosenfeld 24933c9168faSHans Rosenfeld /* 24943c9168faSHans Rosenfeld * Create taskq for command completion. 24953c9168faSHans Rosenfeld */ 24963c9168faSHans Rosenfeld (void) snprintf(name, sizeof (name), "%s%d_cmd_taskq", 24973c9168faSHans Rosenfeld ddi_driver_name(dip), ddi_get_instance(dip)); 24983c9168faSHans Rosenfeld nvme->n_cmd_taskq = ddi_taskq_create(dip, name, MIN(UINT16_MAX, ncpus), 24993c9168faSHans Rosenfeld TASKQ_DEFAULTPRI, 0); 25003c9168faSHans Rosenfeld if (nvme->n_cmd_taskq == NULL) { 25013c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to create cmd taskq"); 25023c9168faSHans Rosenfeld goto fail; 25033c9168faSHans Rosenfeld } 25043c9168faSHans Rosenfeld 2505*8834f7acSYouzhong Yang /* 2506*8834f7acSYouzhong Yang * Create PRP DMA cache 2507*8834f7acSYouzhong Yang */ 2508*8834f7acSYouzhong Yang (void) snprintf(name, sizeof (name), "%s%d_prp_cache", 2509*8834f7acSYouzhong Yang ddi_driver_name(dip), ddi_get_instance(dip)); 2510*8834f7acSYouzhong Yang nvme->n_prp_cache = kmem_cache_create(name, sizeof (nvme_dma_t), 2511*8834f7acSYouzhong Yang 0, nvme_prp_dma_constructor, nvme_prp_dma_destructor, 2512*8834f7acSYouzhong Yang NULL, (void *)nvme, NULL, 0); 25133c9168faSHans Rosenfeld 25143c9168faSHans Rosenfeld if (nvme_init(nvme) != DDI_SUCCESS) 25153c9168faSHans Rosenfeld goto fail; 25163c9168faSHans Rosenfeld 25173c9168faSHans Rosenfeld /* 25183c9168faSHans Rosenfeld * Attach the blkdev driver for each namespace. 25193c9168faSHans Rosenfeld */ 25203c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 25213c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_ignore) 25223c9168faSHans Rosenfeld continue; 25233c9168faSHans Rosenfeld 25243c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl = bd_alloc_handle(&nvme->n_ns[i], 25253c9168faSHans Rosenfeld &nvme_bd_ops, &nvme->n_prp_dma_attr, KM_SLEEP); 25263c9168faSHans Rosenfeld 25273c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl == NULL) { 25283c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 25293c9168faSHans Rosenfeld "!failed to get blkdev handle for namespace %d", i); 25303c9168faSHans Rosenfeld goto fail; 25313c9168faSHans Rosenfeld } 25323c9168faSHans Rosenfeld 25333c9168faSHans Rosenfeld if (bd_attach_handle(dip, nvme->n_ns[i].ns_bd_hdl) 25343c9168faSHans Rosenfeld != DDI_SUCCESS) { 25353c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 25363c9168faSHans Rosenfeld "!failed to attach blkdev handle for namespace %d", 25373c9168faSHans Rosenfeld i); 25383c9168faSHans Rosenfeld goto fail; 25393c9168faSHans Rosenfeld } 25403c9168faSHans Rosenfeld } 25413c9168faSHans Rosenfeld 25423c9168faSHans Rosenfeld return (DDI_SUCCESS); 25433c9168faSHans Rosenfeld 25443c9168faSHans Rosenfeld fail: 25453c9168faSHans Rosenfeld /* attach successful anyway so that FMA can retire the device */ 25463c9168faSHans Rosenfeld if (nvme->n_dead) 25473c9168faSHans Rosenfeld return (DDI_SUCCESS); 25483c9168faSHans Rosenfeld 25493c9168faSHans Rosenfeld (void) nvme_detach(dip, DDI_DETACH); 25503c9168faSHans Rosenfeld 25513c9168faSHans Rosenfeld return (DDI_FAILURE); 25523c9168faSHans Rosenfeld } 25533c9168faSHans Rosenfeld 25543c9168faSHans Rosenfeld static int 25553c9168faSHans Rosenfeld nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 25563c9168faSHans Rosenfeld { 25573c9168faSHans Rosenfeld int instance, i; 25583c9168faSHans Rosenfeld nvme_t *nvme; 25593c9168faSHans Rosenfeld 25603c9168faSHans Rosenfeld if (cmd != DDI_DETACH) 25613c9168faSHans Rosenfeld return (DDI_FAILURE); 25623c9168faSHans Rosenfeld 25633c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 25643c9168faSHans Rosenfeld 25653c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 25663c9168faSHans Rosenfeld 25673c9168faSHans Rosenfeld if (nvme == NULL) 25683c9168faSHans Rosenfeld return (DDI_FAILURE); 25693c9168faSHans Rosenfeld 25703c9168faSHans Rosenfeld if (nvme->n_ns) { 25713c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 25723c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl) { 25733c9168faSHans Rosenfeld (void) bd_detach_handle( 25743c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl); 25753c9168faSHans Rosenfeld bd_free_handle(nvme->n_ns[i].ns_bd_hdl); 25763c9168faSHans Rosenfeld } 25773c9168faSHans Rosenfeld 25783c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_idns) 25793c9168faSHans Rosenfeld kmem_free(nvme->n_ns[i].ns_idns, 25803c9168faSHans Rosenfeld sizeof (nvme_identify_nsid_t)); 25813c9168faSHans Rosenfeld } 25823c9168faSHans Rosenfeld 25833c9168faSHans Rosenfeld kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) * 25843c9168faSHans Rosenfeld nvme->n_namespace_count); 25853c9168faSHans Rosenfeld } 25863c9168faSHans Rosenfeld 25873c9168faSHans Rosenfeld if (nvme->n_progress & NVME_INTERRUPTS) 25883c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 25893c9168faSHans Rosenfeld 25903c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 25913c9168faSHans Rosenfeld ddi_taskq_wait(nvme->n_cmd_taskq); 25923c9168faSHans Rosenfeld 25933c9168faSHans Rosenfeld if (nvme->n_ioq_count > 0) { 25943c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 25953c9168faSHans Rosenfeld if (nvme->n_ioq[i] != NULL) { 25963c9168faSHans Rosenfeld /* TODO: send destroy queue commands */ 25973c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_ioq[i]); 25983c9168faSHans Rosenfeld } 25993c9168faSHans Rosenfeld } 26003c9168faSHans Rosenfeld 26013c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) * 26023c9168faSHans Rosenfeld (nvme->n_ioq_count + 1)); 26033c9168faSHans Rosenfeld } 26043c9168faSHans Rosenfeld 2605*8834f7acSYouzhong Yang if (nvme->n_prp_cache != NULL) { 2606*8834f7acSYouzhong Yang kmem_cache_destroy(nvme->n_prp_cache); 2607*8834f7acSYouzhong Yang } 2608*8834f7acSYouzhong Yang 26093c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) { 26103c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE); 26113c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 26123c9168faSHans Rosenfeld } 26133c9168faSHans Rosenfeld 26143c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 26153c9168faSHans Rosenfeld ddi_taskq_destroy(nvme->n_cmd_taskq); 26163c9168faSHans Rosenfeld 26173c9168faSHans Rosenfeld if (nvme->n_progress & NVME_CTRL_LIMITS) 26183c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 26193c9168faSHans Rosenfeld 26203c9168faSHans Rosenfeld if (nvme->n_progress & NVME_ADMIN_QUEUE) 26213c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_adminq); 26223c9168faSHans Rosenfeld 26233c9168faSHans Rosenfeld if (nvme->n_idctl) 26243c9168faSHans Rosenfeld kmem_free(nvme->n_idctl, sizeof (nvme_identify_ctrl_t)); 26253c9168faSHans Rosenfeld 26263c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) 26273c9168faSHans Rosenfeld ddi_regs_map_free(&nvme->n_regh); 26283c9168faSHans Rosenfeld 26293c9168faSHans Rosenfeld if (nvme->n_progress & NVME_FMA_INIT) { 26303c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 26313c9168faSHans Rosenfeld ddi_fm_handler_unregister(nvme->n_dip); 26323c9168faSHans Rosenfeld 26333c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 26343c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 26353c9168faSHans Rosenfeld pci_ereport_teardown(nvme->n_dip); 26363c9168faSHans Rosenfeld 26373c9168faSHans Rosenfeld ddi_fm_fini(nvme->n_dip); 26383c9168faSHans Rosenfeld } 26393c9168faSHans Rosenfeld 2640510a6847SHans Rosenfeld if (nvme->n_vendor != NULL) 2641510a6847SHans Rosenfeld strfree(nvme->n_vendor); 2642510a6847SHans Rosenfeld 2643510a6847SHans Rosenfeld if (nvme->n_product != NULL) 2644510a6847SHans Rosenfeld strfree(nvme->n_product); 2645510a6847SHans Rosenfeld 26463c9168faSHans Rosenfeld ddi_soft_state_free(nvme_state, instance); 26473c9168faSHans Rosenfeld 26483c9168faSHans Rosenfeld return (DDI_SUCCESS); 26493c9168faSHans Rosenfeld } 26503c9168faSHans Rosenfeld 26513c9168faSHans Rosenfeld static int 26523c9168faSHans Rosenfeld nvme_quiesce(dev_info_t *dip) 26533c9168faSHans Rosenfeld { 26543c9168faSHans Rosenfeld int instance; 26553c9168faSHans Rosenfeld nvme_t *nvme; 26563c9168faSHans Rosenfeld 26573c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 26583c9168faSHans Rosenfeld 26593c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 26603c9168faSHans Rosenfeld 26613c9168faSHans Rosenfeld if (nvme == NULL) 26623c9168faSHans Rosenfeld return (DDI_FAILURE); 26633c9168faSHans Rosenfeld 26643c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_ABRUPT, B_TRUE); 26653c9168faSHans Rosenfeld 26663c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_TRUE); 26673c9168faSHans Rosenfeld 26683c9168faSHans Rosenfeld return (DDI_FAILURE); 26693c9168faSHans Rosenfeld } 26703c9168faSHans Rosenfeld 26713c9168faSHans Rosenfeld static int 26723c9168faSHans Rosenfeld nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer) 26733c9168faSHans Rosenfeld { 26743c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 26753c9168faSHans Rosenfeld int nprp_page, nprp; 26763c9168faSHans Rosenfeld uint64_t *prp; 26773c9168faSHans Rosenfeld 26783c9168faSHans Rosenfeld if (xfer->x_ndmac == 0) 26793c9168faSHans Rosenfeld return (DDI_FAILURE); 26803c9168faSHans Rosenfeld 26813c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress; 26823c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 26833c9168faSHans Rosenfeld 26843c9168faSHans Rosenfeld if (xfer->x_ndmac == 1) { 26853c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; 26863c9168faSHans Rosenfeld return (DDI_SUCCESS); 26873c9168faSHans Rosenfeld } else if (xfer->x_ndmac == 2) { 26883c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress; 26893c9168faSHans Rosenfeld return (DDI_SUCCESS); 26903c9168faSHans Rosenfeld } 26913c9168faSHans Rosenfeld 26923c9168faSHans Rosenfeld xfer->x_ndmac--; 26933c9168faSHans Rosenfeld 26943c9168faSHans Rosenfeld nprp_page = nvme->n_pagesize / sizeof (uint64_t) - 1; 26953c9168faSHans Rosenfeld ASSERT(nprp_page > 0); 26963c9168faSHans Rosenfeld nprp = (xfer->x_ndmac + nprp_page - 1) / nprp_page; 26973c9168faSHans Rosenfeld 26983c9168faSHans Rosenfeld /* 26993c9168faSHans Rosenfeld * We currently don't support chained PRPs and set up our DMA 27003c9168faSHans Rosenfeld * attributes to reflect that. If we still get an I/O request 27013c9168faSHans Rosenfeld * that needs a chained PRP something is very wrong. 27023c9168faSHans Rosenfeld */ 27033c9168faSHans Rosenfeld VERIFY(nprp == 1); 27043c9168faSHans Rosenfeld 2705*8834f7acSYouzhong Yang cmd->nc_dma = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP); 2706*8834f7acSYouzhong Yang bzero(cmd->nc_dma->nd_memp, cmd->nc_dma->nd_len); 27073c9168faSHans Rosenfeld 27083c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress; 27093c9168faSHans Rosenfeld 27103c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 27113c9168faSHans Rosenfeld for (prp = (uint64_t *)cmd->nc_dma->nd_memp; 27123c9168faSHans Rosenfeld xfer->x_ndmac > 0; 27133c9168faSHans Rosenfeld prp++, xfer->x_ndmac--) { 27143c9168faSHans Rosenfeld *prp = xfer->x_dmac.dmac_laddress; 27153c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 27163c9168faSHans Rosenfeld } 27173c9168faSHans Rosenfeld 27183c9168faSHans Rosenfeld (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len, 27193c9168faSHans Rosenfeld DDI_DMA_SYNC_FORDEV); 27203c9168faSHans Rosenfeld return (DDI_SUCCESS); 27213c9168faSHans Rosenfeld } 27223c9168faSHans Rosenfeld 27233c9168faSHans Rosenfeld static nvme_cmd_t * 27243c9168faSHans Rosenfeld nvme_create_nvm_cmd(nvme_namespace_t *ns, uint8_t opc, bd_xfer_t *xfer) 27253c9168faSHans Rosenfeld { 27263c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 27273c9168faSHans Rosenfeld nvme_cmd_t *cmd; 27283c9168faSHans Rosenfeld 27293c9168faSHans Rosenfeld /* 27303c9168faSHans Rosenfeld * Blkdev only sets BD_XFER_POLL when dumping, so don't sleep. 27313c9168faSHans Rosenfeld */ 27323c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ? 27333c9168faSHans Rosenfeld KM_NOSLEEP : KM_SLEEP); 27343c9168faSHans Rosenfeld 27353c9168faSHans Rosenfeld if (cmd == NULL) 27363c9168faSHans Rosenfeld return (NULL); 27373c9168faSHans Rosenfeld 27383c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = opc; 27393c9168faSHans Rosenfeld cmd->nc_callback = nvme_bd_xfer_done; 27403c9168faSHans Rosenfeld cmd->nc_xfer = xfer; 27413c9168faSHans Rosenfeld 27423c9168faSHans Rosenfeld switch (opc) { 27433c9168faSHans Rosenfeld case NVME_OPC_NVM_WRITE: 27443c9168faSHans Rosenfeld case NVME_OPC_NVM_READ: 27453c9168faSHans Rosenfeld VERIFY(xfer->x_nblks <= 0x10000); 27463c9168faSHans Rosenfeld 27473c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 27483c9168faSHans Rosenfeld 27493c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu; 27503c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32); 27513c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1); 27523c9168faSHans Rosenfeld 27533c9168faSHans Rosenfeld if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS) 27543c9168faSHans Rosenfeld goto fail; 27553c9168faSHans Rosenfeld break; 27563c9168faSHans Rosenfeld 27573c9168faSHans Rosenfeld case NVME_OPC_NVM_FLUSH: 27583c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 27593c9168faSHans Rosenfeld break; 27603c9168faSHans Rosenfeld 27613c9168faSHans Rosenfeld default: 27623c9168faSHans Rosenfeld goto fail; 27633c9168faSHans Rosenfeld } 27643c9168faSHans Rosenfeld 27653c9168faSHans Rosenfeld return (cmd); 27663c9168faSHans Rosenfeld 27673c9168faSHans Rosenfeld fail: 27683c9168faSHans Rosenfeld nvme_free_cmd(cmd); 27693c9168faSHans Rosenfeld return (NULL); 27703c9168faSHans Rosenfeld } 27713c9168faSHans Rosenfeld 27723c9168faSHans Rosenfeld static void 27733c9168faSHans Rosenfeld nvme_bd_xfer_done(void *arg) 27743c9168faSHans Rosenfeld { 27753c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 27763c9168faSHans Rosenfeld bd_xfer_t *xfer = cmd->nc_xfer; 27773c9168faSHans Rosenfeld int error = 0; 27783c9168faSHans Rosenfeld 27793c9168faSHans Rosenfeld error = nvme_check_cmd_status(cmd); 27803c9168faSHans Rosenfeld nvme_free_cmd(cmd); 27813c9168faSHans Rosenfeld 27823c9168faSHans Rosenfeld bd_xfer_done(xfer, error); 27833c9168faSHans Rosenfeld } 27843c9168faSHans Rosenfeld 27853c9168faSHans Rosenfeld static void 27863c9168faSHans Rosenfeld nvme_bd_driveinfo(void *arg, bd_drive_t *drive) 27873c9168faSHans Rosenfeld { 27883c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 27893c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 27903c9168faSHans Rosenfeld 27913c9168faSHans Rosenfeld /* 27923c9168faSHans Rosenfeld * blkdev maintains one queue size per instance (namespace), 27933c9168faSHans Rosenfeld * but all namespace share the I/O queues. 27943c9168faSHans Rosenfeld * TODO: need to figure out a sane default, or use per-NS I/O queues, 27953c9168faSHans Rosenfeld * or change blkdev to handle EAGAIN 27963c9168faSHans Rosenfeld */ 27973c9168faSHans Rosenfeld drive->d_qsize = nvme->n_ioq_count * nvme->n_io_queue_len 27983c9168faSHans Rosenfeld / nvme->n_namespace_count; 27993c9168faSHans Rosenfeld 28003c9168faSHans Rosenfeld /* 28013c9168faSHans Rosenfeld * d_maxxfer is not set, which means the value is taken from the DMA 28023c9168faSHans Rosenfeld * attributes specified to bd_alloc_handle. 28033c9168faSHans Rosenfeld */ 28043c9168faSHans Rosenfeld 28053c9168faSHans Rosenfeld drive->d_removable = B_FALSE; 28063c9168faSHans Rosenfeld drive->d_hotpluggable = B_FALSE; 28073c9168faSHans Rosenfeld 28083c9168faSHans Rosenfeld drive->d_target = ns->ns_id; 28093c9168faSHans Rosenfeld drive->d_lun = 0; 2810510a6847SHans Rosenfeld 2811bef9e21aSHans Rosenfeld drive->d_model = nvme->n_idctl->id_model; 2812bef9e21aSHans Rosenfeld drive->d_model_len = sizeof (nvme->n_idctl->id_model); 2813510a6847SHans Rosenfeld drive->d_vendor = nvme->n_vendor; 2814510a6847SHans Rosenfeld drive->d_vendor_len = strlen(nvme->n_vendor); 2815510a6847SHans Rosenfeld drive->d_product = nvme->n_product; 2816510a6847SHans Rosenfeld drive->d_product_len = strlen(nvme->n_product); 2817510a6847SHans Rosenfeld drive->d_serial = nvme->n_idctl->id_serial; 2818510a6847SHans Rosenfeld drive->d_serial_len = sizeof (nvme->n_idctl->id_serial); 2819510a6847SHans Rosenfeld drive->d_revision = nvme->n_idctl->id_fwrev; 2820510a6847SHans Rosenfeld drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev); 28213c9168faSHans Rosenfeld } 28223c9168faSHans Rosenfeld 28233c9168faSHans Rosenfeld static int 28243c9168faSHans Rosenfeld nvme_bd_mediainfo(void *arg, bd_media_t *media) 28253c9168faSHans Rosenfeld { 28263c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28273c9168faSHans Rosenfeld 28283c9168faSHans Rosenfeld media->m_nblks = ns->ns_block_count; 28293c9168faSHans Rosenfeld media->m_blksize = ns->ns_block_size; 28303c9168faSHans Rosenfeld media->m_readonly = B_FALSE; 28313c9168faSHans Rosenfeld media->m_solidstate = B_TRUE; 28323c9168faSHans Rosenfeld 28333c9168faSHans Rosenfeld media->m_pblksize = ns->ns_best_block_size; 28343c9168faSHans Rosenfeld 28353c9168faSHans Rosenfeld return (0); 28363c9168faSHans Rosenfeld } 28373c9168faSHans Rosenfeld 28383c9168faSHans Rosenfeld static int 28393c9168faSHans Rosenfeld nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc) 28403c9168faSHans Rosenfeld { 28413c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 28423c9168faSHans Rosenfeld nvme_cmd_t *cmd; 28433c9168faSHans Rosenfeld 28443c9168faSHans Rosenfeld if (nvme->n_dead) 28453c9168faSHans Rosenfeld return (EIO); 28463c9168faSHans Rosenfeld 28473c9168faSHans Rosenfeld /* No polling for now */ 28483c9168faSHans Rosenfeld if (xfer->x_flags & BD_XFER_POLL) 28493c9168faSHans Rosenfeld return (EIO); 28503c9168faSHans Rosenfeld 28513c9168faSHans Rosenfeld cmd = nvme_create_nvm_cmd(ns, opc, xfer); 28523c9168faSHans Rosenfeld if (cmd == NULL) 28533c9168faSHans Rosenfeld return (ENOMEM); 28543c9168faSHans Rosenfeld 28553c9168faSHans Rosenfeld cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1; 28563c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); 28573c9168faSHans Rosenfeld 28583c9168faSHans Rosenfeld if (nvme_submit_cmd(nvme->n_ioq[cmd->nc_sqid], cmd) 28593c9168faSHans Rosenfeld != DDI_SUCCESS) 28603c9168faSHans Rosenfeld return (EAGAIN); 28613c9168faSHans Rosenfeld 28623c9168faSHans Rosenfeld return (0); 28633c9168faSHans Rosenfeld } 28643c9168faSHans Rosenfeld 28653c9168faSHans Rosenfeld static int 28663c9168faSHans Rosenfeld nvme_bd_read(void *arg, bd_xfer_t *xfer) 28673c9168faSHans Rosenfeld { 28683c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28693c9168faSHans Rosenfeld 28703c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_READ)); 28713c9168faSHans Rosenfeld } 28723c9168faSHans Rosenfeld 28733c9168faSHans Rosenfeld static int 28743c9168faSHans Rosenfeld nvme_bd_write(void *arg, bd_xfer_t *xfer) 28753c9168faSHans Rosenfeld { 28763c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28773c9168faSHans Rosenfeld 28783c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_WRITE)); 28793c9168faSHans Rosenfeld } 28803c9168faSHans Rosenfeld 28813c9168faSHans Rosenfeld static int 28823c9168faSHans Rosenfeld nvme_bd_sync(void *arg, bd_xfer_t *xfer) 28833c9168faSHans Rosenfeld { 28843c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28853c9168faSHans Rosenfeld 28863c9168faSHans Rosenfeld if (ns->ns_nvme->n_dead) 28873c9168faSHans Rosenfeld return (EIO); 28883c9168faSHans Rosenfeld 28893c9168faSHans Rosenfeld /* 28903c9168faSHans Rosenfeld * If the volatile write cache isn't enabled the FLUSH command is a 28913c9168faSHans Rosenfeld * no-op, so we can take a shortcut here. 28923c9168faSHans Rosenfeld */ 28933c9168faSHans Rosenfeld if (ns->ns_nvme->n_volatile_write_cache_enabled == B_FALSE) { 28943c9168faSHans Rosenfeld bd_xfer_done(xfer, ENOTSUP); 28953c9168faSHans Rosenfeld return (0); 28963c9168faSHans Rosenfeld } 28973c9168faSHans Rosenfeld 28983c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_FLUSH)); 28993c9168faSHans Rosenfeld } 29003c9168faSHans Rosenfeld 29013c9168faSHans Rosenfeld static int 29023c9168faSHans Rosenfeld nvme_bd_devid(void *arg, dev_info_t *devinfo, ddi_devid_t *devid) 29033c9168faSHans Rosenfeld { 29043c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 29053c9168faSHans Rosenfeld 29063c9168faSHans Rosenfeld return (ddi_devid_init(devinfo, DEVID_ENCAP, strlen(ns->ns_devid), 29073c9168faSHans Rosenfeld ns->ns_devid, devid)); 29083c9168faSHans Rosenfeld } 2909