13c9168faSHans Rosenfeld /* 23c9168faSHans Rosenfeld * This file and its contents are supplied under the terms of the 33c9168faSHans Rosenfeld * Common Development and Distribution License ("CDDL"), version 1.0. 43c9168faSHans Rosenfeld * You may only use this file in accordance with the terms of version 53c9168faSHans Rosenfeld * 1.0 of the CDDL. 63c9168faSHans Rosenfeld * 73c9168faSHans Rosenfeld * A full copy of the text of the CDDL should have accompanied this 83c9168faSHans Rosenfeld * source. A copy of the CDDL is also available via the Internet at 93c9168faSHans Rosenfeld * http://www.illumos.org/license/CDDL. 103c9168faSHans Rosenfeld */ 113c9168faSHans Rosenfeld 123c9168faSHans Rosenfeld /* 136afc9eb2SHans Rosenfeld * Copyright 2016 Nexenta Systems, Inc. All rights reserved. 1434c938c7SPete Shephard * Copyright 2016 Tegile Systems, Inc. All rights reserved. 152f95345bSYouzhong Yang * Copyright (c) 2016 The MathWorks, Inc. All rights reserved. 163c9168faSHans Rosenfeld */ 173c9168faSHans Rosenfeld 183c9168faSHans Rosenfeld /* 193c9168faSHans Rosenfeld * blkdev driver for NVMe compliant storage devices 203c9168faSHans Rosenfeld * 213c9168faSHans Rosenfeld * This driver was written to conform to version 1.0e of the NVMe specification. 223c9168faSHans Rosenfeld * It may work with newer versions, but that is completely untested and disabled 233c9168faSHans Rosenfeld * by default. 243c9168faSHans Rosenfeld * 253c9168faSHans Rosenfeld * The driver has only been tested on x86 systems and will not work on big- 263c9168faSHans Rosenfeld * endian systems without changes to the code accessing registers and data 273c9168faSHans Rosenfeld * structures used by the hardware. 283c9168faSHans Rosenfeld * 293c9168faSHans Rosenfeld * 303c9168faSHans Rosenfeld * Interrupt Usage: 313c9168faSHans Rosenfeld * 323c9168faSHans Rosenfeld * The driver will use a FIXED interrupt while configuring the device as the 333c9168faSHans Rosenfeld * specification requires. Later in the attach process it will switch to MSI-X 343c9168faSHans Rosenfeld * or MSI if supported. The driver wants to have one interrupt vector per CPU, 353c9168faSHans Rosenfeld * but it will work correctly if less are available. Interrupts can be shared 363c9168faSHans Rosenfeld * by queues, the interrupt handler will iterate through the I/O queue array by 373c9168faSHans Rosenfeld * steps of n_intr_cnt. Usually only the admin queue will share an interrupt 383c9168faSHans Rosenfeld * with one I/O queue. The interrupt handler will retrieve completed commands 393c9168faSHans Rosenfeld * from all queues sharing an interrupt vector and will post them to a taskq 403c9168faSHans Rosenfeld * for completion processing. 413c9168faSHans Rosenfeld * 423c9168faSHans Rosenfeld * 433c9168faSHans Rosenfeld * Command Processing: 443c9168faSHans Rosenfeld * 453c9168faSHans Rosenfeld * NVMe devices can have up to 65536 I/O queue pairs, with each queue holding up 463c9168faSHans Rosenfeld * to 65536 I/O commands. The driver will configure one I/O queue pair per 473c9168faSHans Rosenfeld * available interrupt vector, with the queue length usually much smaller than 483c9168faSHans Rosenfeld * the maximum of 65536. If the hardware doesn't provide enough queues, fewer 493c9168faSHans Rosenfeld * interrupt vectors will be used. 503c9168faSHans Rosenfeld * 513c9168faSHans Rosenfeld * Additionally the hardware provides a single special admin queue pair that can 523c9168faSHans Rosenfeld * hold up to 4096 admin commands. 533c9168faSHans Rosenfeld * 543c9168faSHans Rosenfeld * From the hardware perspective both queues of a queue pair are independent, 553c9168faSHans Rosenfeld * but they share some driver state: the command array (holding pointers to 563c9168faSHans Rosenfeld * commands currently being processed by the hardware) and the active command 573c9168faSHans Rosenfeld * counter. Access to the submission side of a queue pair and the shared state 583c9168faSHans Rosenfeld * is protected by nq_mutex. The completion side of a queue pair does not need 593c9168faSHans Rosenfeld * that protection apart from its access to the shared state; it is called only 603c9168faSHans Rosenfeld * in the interrupt handler which does not run concurrently for the same 613c9168faSHans Rosenfeld * interrupt vector. 623c9168faSHans Rosenfeld * 633c9168faSHans Rosenfeld * When a command is submitted to a queue pair the active command counter is 643c9168faSHans Rosenfeld * incremented and a pointer to the command is stored in the command array. The 653c9168faSHans Rosenfeld * array index is used as command identifier (CID) in the submission queue 663c9168faSHans Rosenfeld * entry. Some commands may take a very long time to complete, and if the queue 673c9168faSHans Rosenfeld * wraps around in that time a submission may find the next array slot to still 683c9168faSHans Rosenfeld * be used by a long-running command. In this case the array is sequentially 693c9168faSHans Rosenfeld * searched for the next free slot. The length of the command array is the same 703c9168faSHans Rosenfeld * as the configured queue length. 713c9168faSHans Rosenfeld * 723c9168faSHans Rosenfeld * 733c9168faSHans Rosenfeld * Namespace Support: 743c9168faSHans Rosenfeld * 753c9168faSHans Rosenfeld * NVMe devices can have multiple namespaces, each being a independent data 763c9168faSHans Rosenfeld * store. The driver supports multiple namespaces and creates a blkdev interface 773c9168faSHans Rosenfeld * for each namespace found. Namespaces can have various attributes to support 783c9168faSHans Rosenfeld * thin provisioning, extended LBAs, and protection information. This driver 793c9168faSHans Rosenfeld * does not support any of this and ignores namespaces that have these 803c9168faSHans Rosenfeld * attributes. 813c9168faSHans Rosenfeld * 823c9168faSHans Rosenfeld * 833c9168faSHans Rosenfeld * Blkdev Interface: 843c9168faSHans Rosenfeld * 853c9168faSHans Rosenfeld * This driver uses blkdev to do all the heavy lifting involved with presenting 863c9168faSHans Rosenfeld * a disk device to the system. As a result, the processing of I/O requests is 873c9168faSHans Rosenfeld * relatively simple as blkdev takes care of partitioning, boundary checks, DMA 883c9168faSHans Rosenfeld * setup, and splitting of transfers into manageable chunks. 893c9168faSHans Rosenfeld * 903c9168faSHans Rosenfeld * I/O requests coming in from blkdev are turned into NVM commands and posted to 913c9168faSHans Rosenfeld * an I/O queue. The queue is selected by taking the CPU id modulo the number of 923c9168faSHans Rosenfeld * queues. There is currently no timeout handling of I/O commands. 933c9168faSHans Rosenfeld * 943c9168faSHans Rosenfeld * Blkdev also supports querying device/media information and generating a 953c9168faSHans Rosenfeld * devid. The driver reports the best block size as determined by the namespace 963c9168faSHans Rosenfeld * format back to blkdev as physical block size to support partition and block 973c9168faSHans Rosenfeld * alignment. The devid is composed using the device vendor ID, model number, 983c9168faSHans Rosenfeld * serial number, and the namespace ID. 993c9168faSHans Rosenfeld * 1003c9168faSHans Rosenfeld * 1013c9168faSHans Rosenfeld * Error Handling: 1023c9168faSHans Rosenfeld * 1033c9168faSHans Rosenfeld * Error handling is currently limited to detecting fatal hardware errors, 1043c9168faSHans Rosenfeld * either by asynchronous events, or synchronously through command status or 1053c9168faSHans Rosenfeld * admin command timeouts. In case of severe errors the device is fenced off, 1063c9168faSHans Rosenfeld * all further requests will return EIO. FMA is then called to fault the device. 1073c9168faSHans Rosenfeld * 1083c9168faSHans Rosenfeld * The hardware has a limit for outstanding asynchronous event requests. Before 1093c9168faSHans Rosenfeld * this limit is known the driver assumes it is at least 1 and posts a single 1103c9168faSHans Rosenfeld * asynchronous request. Later when the limit is known more asynchronous event 1113c9168faSHans Rosenfeld * requests are posted to allow quicker reception of error information. When an 1123c9168faSHans Rosenfeld * asynchronous event is posted by the hardware the driver will parse the error 1133c9168faSHans Rosenfeld * status fields and log information or fault the device, depending on the 1143c9168faSHans Rosenfeld * severity of the asynchronous event. The asynchronous event request is then 1153c9168faSHans Rosenfeld * reused and posted to the admin queue again. 1163c9168faSHans Rosenfeld * 1173c9168faSHans Rosenfeld * On command completion the command status is checked for errors. In case of 1183c9168faSHans Rosenfeld * errors indicating a driver bug the driver panics. Almost all other error 1193c9168faSHans Rosenfeld * status values just cause EIO to be returned. 1203c9168faSHans Rosenfeld * 1213c9168faSHans Rosenfeld * Command timeouts are currently detected for all admin commands except 1223c9168faSHans Rosenfeld * asynchronous event requests. If a command times out and the hardware appears 1233c9168faSHans Rosenfeld * to be healthy the driver attempts to abort the command. If this fails the 1243c9168faSHans Rosenfeld * driver assumes the device to be dead, fences it off, and calls FMA to retire 1253c9168faSHans Rosenfeld * it. In general admin commands are issued at attach time only. No timeout 1263c9168faSHans Rosenfeld * handling of normal I/O commands is presently done. 1273c9168faSHans Rosenfeld * 1283c9168faSHans Rosenfeld * In some cases it may be possible that the ABORT command times out, too. In 1293c9168faSHans Rosenfeld * that case the device is also declared dead and fenced off. 1303c9168faSHans Rosenfeld * 1313c9168faSHans Rosenfeld * 1323c9168faSHans Rosenfeld * Quiesce / Fast Reboot: 1333c9168faSHans Rosenfeld * 1343c9168faSHans Rosenfeld * The driver currently does not support fast reboot. A quiesce(9E) entry point 1353c9168faSHans Rosenfeld * is still provided which is used to send a shutdown notification to the 1363c9168faSHans Rosenfeld * device. 1373c9168faSHans Rosenfeld * 1383c9168faSHans Rosenfeld * 1393c9168faSHans Rosenfeld * Driver Configuration: 1403c9168faSHans Rosenfeld * 1413c9168faSHans Rosenfeld * The following driver properties can be changed to control some aspects of the 1423c9168faSHans Rosenfeld * drivers operation: 1433c9168faSHans Rosenfeld * - strict-version: can be set to 0 to allow devices conforming to newer 1443c9168faSHans Rosenfeld * versions to be used 1453c9168faSHans Rosenfeld * - ignore-unknown-vendor-status: can be set to 1 to not handle any vendor 1463c9168faSHans Rosenfeld * specific command status as a fatal error leading device faulting 1473c9168faSHans Rosenfeld * - admin-queue-len: the maximum length of the admin queue (16-4096) 1483c9168faSHans Rosenfeld * - io-queue-len: the maximum length of the I/O queues (16-65536) 1493c9168faSHans Rosenfeld * - async-event-limit: the maximum number of asynchronous event requests to be 1503c9168faSHans Rosenfeld * posted by the driver 1513c9168faSHans Rosenfeld * 1523c9168faSHans Rosenfeld * 1533c9168faSHans Rosenfeld * TODO: 1543c9168faSHans Rosenfeld * - figure out sane default for I/O queue depth reported to blkdev 1553c9168faSHans Rosenfeld * - polled I/O support to support kernel core dumping 1563c9168faSHans Rosenfeld * - FMA handling of media errors 1573c9168faSHans Rosenfeld * - support for the Volatile Write Cache 1583c9168faSHans Rosenfeld * - support for devices supporting very large I/O requests using chained PRPs 1593c9168faSHans Rosenfeld * - support for querying log pages from user space 1603c9168faSHans Rosenfeld * - support for configuring hardware parameters like interrupt coalescing 1613c9168faSHans Rosenfeld * - support for media formatting and hard partitioning into namespaces 1623c9168faSHans Rosenfeld * - support for big-endian systems 1633c9168faSHans Rosenfeld * - support for fast reboot 1643c9168faSHans Rosenfeld */ 1653c9168faSHans Rosenfeld 1663c9168faSHans Rosenfeld #include <sys/byteorder.h> 1673c9168faSHans Rosenfeld #ifdef _BIG_ENDIAN 1683c9168faSHans Rosenfeld #error nvme driver needs porting for big-endian platforms 1693c9168faSHans Rosenfeld #endif 1703c9168faSHans Rosenfeld 1713c9168faSHans Rosenfeld #include <sys/modctl.h> 1723c9168faSHans Rosenfeld #include <sys/conf.h> 1733c9168faSHans Rosenfeld #include <sys/devops.h> 1743c9168faSHans Rosenfeld #include <sys/ddi.h> 1753c9168faSHans Rosenfeld #include <sys/sunddi.h> 1763c9168faSHans Rosenfeld #include <sys/bitmap.h> 1773c9168faSHans Rosenfeld #include <sys/sysmacros.h> 1783c9168faSHans Rosenfeld #include <sys/param.h> 1793c9168faSHans Rosenfeld #include <sys/varargs.h> 1803c9168faSHans Rosenfeld #include <sys/cpuvar.h> 1813c9168faSHans Rosenfeld #include <sys/disp.h> 1823c9168faSHans Rosenfeld #include <sys/blkdev.h> 1833c9168faSHans Rosenfeld #include <sys/atomic.h> 1843c9168faSHans Rosenfeld #include <sys/archsystm.h> 185510a6847SHans Rosenfeld #include <sys/sata/sata_hba.h> 1863c9168faSHans Rosenfeld 1873c9168faSHans Rosenfeld #include "nvme_reg.h" 1883c9168faSHans Rosenfeld #include "nvme_var.h" 1893c9168faSHans Rosenfeld 1903c9168faSHans Rosenfeld 1913c9168faSHans Rosenfeld /* NVMe spec version supported */ 1923c9168faSHans Rosenfeld static const int nvme_version_major = 1; 1933c9168faSHans Rosenfeld static const int nvme_version_minor = 0; 1943c9168faSHans Rosenfeld 195*e8ba2a38SHans Rosenfeld /* tunable for admin command timeout in seconds, default is 1s */ 196*e8ba2a38SHans Rosenfeld static volatile int nvme_admin_cmd_timeout = 1; 197*e8ba2a38SHans Rosenfeld 1983c9168faSHans Rosenfeld static int nvme_attach(dev_info_t *, ddi_attach_cmd_t); 1993c9168faSHans Rosenfeld static int nvme_detach(dev_info_t *, ddi_detach_cmd_t); 2003c9168faSHans Rosenfeld static int nvme_quiesce(dev_info_t *); 2013c9168faSHans Rosenfeld static int nvme_fm_errcb(dev_info_t *, ddi_fm_error_t *, const void *); 2023c9168faSHans Rosenfeld static int nvme_setup_interrupts(nvme_t *, int, int); 2033c9168faSHans Rosenfeld static void nvme_release_interrupts(nvme_t *); 2043c9168faSHans Rosenfeld static uint_t nvme_intr(caddr_t, caddr_t); 2053c9168faSHans Rosenfeld 2063c9168faSHans Rosenfeld static void nvme_shutdown(nvme_t *, int, boolean_t); 2073c9168faSHans Rosenfeld static boolean_t nvme_reset(nvme_t *, boolean_t); 2083c9168faSHans Rosenfeld static int nvme_init(nvme_t *); 2093c9168faSHans Rosenfeld static nvme_cmd_t *nvme_alloc_cmd(nvme_t *, int); 2103c9168faSHans Rosenfeld static void nvme_free_cmd(nvme_cmd_t *); 2113c9168faSHans Rosenfeld static nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t, 2123c9168faSHans Rosenfeld bd_xfer_t *); 2133c9168faSHans Rosenfeld static int nvme_admin_cmd(nvme_cmd_t *, int); 2143c9168faSHans Rosenfeld static int nvme_submit_cmd(nvme_qpair_t *, nvme_cmd_t *); 2153c9168faSHans Rosenfeld static nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *); 2163c9168faSHans Rosenfeld static boolean_t nvme_wait_cmd(nvme_cmd_t *, uint_t); 2173c9168faSHans Rosenfeld static void nvme_wakeup_cmd(void *); 2183c9168faSHans Rosenfeld static void nvme_async_event_task(void *); 2193c9168faSHans Rosenfeld 2203c9168faSHans Rosenfeld static int nvme_check_unknown_cmd_status(nvme_cmd_t *); 2213c9168faSHans Rosenfeld static int nvme_check_vendor_cmd_status(nvme_cmd_t *); 2223c9168faSHans Rosenfeld static int nvme_check_integrity_cmd_status(nvme_cmd_t *); 2233c9168faSHans Rosenfeld static int nvme_check_specific_cmd_status(nvme_cmd_t *); 2243c9168faSHans Rosenfeld static int nvme_check_generic_cmd_status(nvme_cmd_t *); 2253c9168faSHans Rosenfeld static inline int nvme_check_cmd_status(nvme_cmd_t *); 2263c9168faSHans Rosenfeld 2273c9168faSHans Rosenfeld static void nvme_abort_cmd(nvme_cmd_t *); 2283c9168faSHans Rosenfeld static int nvme_async_event(nvme_t *); 2293c9168faSHans Rosenfeld static void *nvme_get_logpage(nvme_t *, uint8_t, ...); 2303c9168faSHans Rosenfeld static void *nvme_identify(nvme_t *, uint32_t); 2313c9168faSHans Rosenfeld static int nvme_set_nqueues(nvme_t *, uint16_t); 2323c9168faSHans Rosenfeld 2333c9168faSHans Rosenfeld static void nvme_free_dma(nvme_dma_t *); 2343c9168faSHans Rosenfeld static int nvme_zalloc_dma(nvme_t *, size_t, uint_t, ddi_dma_attr_t *, 2353c9168faSHans Rosenfeld nvme_dma_t **); 2363c9168faSHans Rosenfeld static int nvme_zalloc_queue_dma(nvme_t *, uint32_t, uint16_t, uint_t, 2373c9168faSHans Rosenfeld nvme_dma_t **); 2383c9168faSHans Rosenfeld static void nvme_free_qpair(nvme_qpair_t *); 2393c9168faSHans Rosenfeld static int nvme_alloc_qpair(nvme_t *, uint32_t, nvme_qpair_t **, int); 2403c9168faSHans Rosenfeld static int nvme_create_io_qpair(nvme_t *, nvme_qpair_t *, uint16_t); 2413c9168faSHans Rosenfeld 2423c9168faSHans Rosenfeld static inline void nvme_put64(nvme_t *, uintptr_t, uint64_t); 2433c9168faSHans Rosenfeld static inline void nvme_put32(nvme_t *, uintptr_t, uint32_t); 2443c9168faSHans Rosenfeld static inline uint64_t nvme_get64(nvme_t *, uintptr_t); 2453c9168faSHans Rosenfeld static inline uint32_t nvme_get32(nvme_t *, uintptr_t); 2463c9168faSHans Rosenfeld 2473c9168faSHans Rosenfeld static boolean_t nvme_check_regs_hdl(nvme_t *); 2483c9168faSHans Rosenfeld static boolean_t nvme_check_dma_hdl(nvme_dma_t *); 2493c9168faSHans Rosenfeld 2503c9168faSHans Rosenfeld static int nvme_fill_prp(nvme_cmd_t *, bd_xfer_t *); 2513c9168faSHans Rosenfeld 2523c9168faSHans Rosenfeld static void nvme_bd_xfer_done(void *); 2533c9168faSHans Rosenfeld static void nvme_bd_driveinfo(void *, bd_drive_t *); 2543c9168faSHans Rosenfeld static int nvme_bd_mediainfo(void *, bd_media_t *); 2553c9168faSHans Rosenfeld static int nvme_bd_cmd(nvme_namespace_t *, bd_xfer_t *, uint8_t); 2563c9168faSHans Rosenfeld static int nvme_bd_read(void *, bd_xfer_t *); 2573c9168faSHans Rosenfeld static int nvme_bd_write(void *, bd_xfer_t *); 2583c9168faSHans Rosenfeld static int nvme_bd_sync(void *, bd_xfer_t *); 2593c9168faSHans Rosenfeld static int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *); 2603c9168faSHans Rosenfeld 2613c9168faSHans Rosenfeld static void nvme_prepare_devid(nvme_t *, uint32_t); 2623c9168faSHans Rosenfeld 2633c9168faSHans Rosenfeld static void *nvme_state; 2643c9168faSHans Rosenfeld static kmem_cache_t *nvme_cmd_cache; 2653c9168faSHans Rosenfeld 2663c9168faSHans Rosenfeld /* 2673c9168faSHans Rosenfeld * DMA attributes for queue DMA memory 2683c9168faSHans Rosenfeld * 2693c9168faSHans Rosenfeld * Queue DMA memory must be page aligned. The maximum length of a queue is 2703c9168faSHans Rosenfeld * 65536 entries, and an entry can be 64 bytes long. 2713c9168faSHans Rosenfeld */ 2723c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_queue_dma_attr = { 2733c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 2743c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 2753c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 2763c9168faSHans Rosenfeld .dma_attr_count_max = (UINT16_MAX + 1) * sizeof (nvme_sqe_t), 2773c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 2783c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 2793c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 2803c9168faSHans Rosenfeld .dma_attr_maxxfer = (UINT16_MAX + 1) * sizeof (nvme_sqe_t), 2813c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 2823c9168faSHans Rosenfeld .dma_attr_sgllen = 1, 2833c9168faSHans Rosenfeld .dma_attr_granular = 1, 2843c9168faSHans Rosenfeld .dma_attr_flags = 0, 2853c9168faSHans Rosenfeld }; 2863c9168faSHans Rosenfeld 2873c9168faSHans Rosenfeld /* 2883c9168faSHans Rosenfeld * DMA attributes for transfers using Physical Region Page (PRP) entries 2893c9168faSHans Rosenfeld * 2903c9168faSHans Rosenfeld * A PRP entry describes one page of DMA memory using the page size specified 2913c9168faSHans Rosenfeld * in the controller configuration's memory page size register (CC.MPS). It uses 2923c9168faSHans Rosenfeld * a 64bit base address aligned to this page size. There is no limitation on 2933c9168faSHans Rosenfeld * chaining PRPs together for arbitrarily large DMA transfers. 2943c9168faSHans Rosenfeld */ 2953c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_prp_dma_attr = { 2963c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 2973c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 2983c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 2993c9168faSHans Rosenfeld .dma_attr_count_max = 0xfff, 3003c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 3013c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 3023c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 3033c9168faSHans Rosenfeld .dma_attr_maxxfer = 0x1000, 3042f95345bSYouzhong Yang .dma_attr_seg = 0xfff, 3053c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 3063c9168faSHans Rosenfeld .dma_attr_granular = 1, 3073c9168faSHans Rosenfeld .dma_attr_flags = 0, 3083c9168faSHans Rosenfeld }; 3093c9168faSHans Rosenfeld 3103c9168faSHans Rosenfeld /* 3113c9168faSHans Rosenfeld * DMA attributes for transfers using scatter/gather lists 3123c9168faSHans Rosenfeld * 3133c9168faSHans Rosenfeld * A SGL entry describes a chunk of DMA memory using a 64bit base address and a 3143c9168faSHans Rosenfeld * 32bit length field. SGL Segment and SGL Last Segment entries require the 3153c9168faSHans Rosenfeld * length to be a multiple of 16 bytes. 3163c9168faSHans Rosenfeld */ 3173c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_sgl_dma_attr = { 3183c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 3193c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 3203c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 3213c9168faSHans Rosenfeld .dma_attr_count_max = 0xffffffffUL, 3223c9168faSHans Rosenfeld .dma_attr_align = 1, 3233c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 3243c9168faSHans Rosenfeld .dma_attr_minxfer = 0x10, 3253c9168faSHans Rosenfeld .dma_attr_maxxfer = 0xfffffffffULL, 3263c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 3273c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 3283c9168faSHans Rosenfeld .dma_attr_granular = 0x10, 3293c9168faSHans Rosenfeld .dma_attr_flags = 0 3303c9168faSHans Rosenfeld }; 3313c9168faSHans Rosenfeld 3323c9168faSHans Rosenfeld static ddi_device_acc_attr_t nvme_reg_acc_attr = { 3333c9168faSHans Rosenfeld .devacc_attr_version = DDI_DEVICE_ATTR_V0, 3343c9168faSHans Rosenfeld .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 3353c9168faSHans Rosenfeld .devacc_attr_dataorder = DDI_STRICTORDER_ACC 3363c9168faSHans Rosenfeld }; 3373c9168faSHans Rosenfeld 3383c9168faSHans Rosenfeld static struct dev_ops nvme_dev_ops = { 3393c9168faSHans Rosenfeld .devo_rev = DEVO_REV, 3403c9168faSHans Rosenfeld .devo_refcnt = 0, 3413c9168faSHans Rosenfeld .devo_getinfo = ddi_no_info, 3423c9168faSHans Rosenfeld .devo_identify = nulldev, 3433c9168faSHans Rosenfeld .devo_probe = nulldev, 3443c9168faSHans Rosenfeld .devo_attach = nvme_attach, 3453c9168faSHans Rosenfeld .devo_detach = nvme_detach, 3463c9168faSHans Rosenfeld .devo_reset = nodev, 3473c9168faSHans Rosenfeld .devo_cb_ops = NULL, 3483c9168faSHans Rosenfeld .devo_bus_ops = NULL, 3493c9168faSHans Rosenfeld .devo_power = NULL, 3503c9168faSHans Rosenfeld .devo_quiesce = nvme_quiesce, 3513c9168faSHans Rosenfeld }; 3523c9168faSHans Rosenfeld 3533c9168faSHans Rosenfeld static struct modldrv nvme_modldrv = { 3543c9168faSHans Rosenfeld .drv_modops = &mod_driverops, 3553c9168faSHans Rosenfeld .drv_linkinfo = "NVMe v1.0e", 3563c9168faSHans Rosenfeld .drv_dev_ops = &nvme_dev_ops 3573c9168faSHans Rosenfeld }; 3583c9168faSHans Rosenfeld 3593c9168faSHans Rosenfeld static struct modlinkage nvme_modlinkage = { 3603c9168faSHans Rosenfeld .ml_rev = MODREV_1, 3613c9168faSHans Rosenfeld .ml_linkage = { &nvme_modldrv, NULL } 3623c9168faSHans Rosenfeld }; 3633c9168faSHans Rosenfeld 3643c9168faSHans Rosenfeld static bd_ops_t nvme_bd_ops = { 3653c9168faSHans Rosenfeld .o_version = BD_OPS_VERSION_0, 3663c9168faSHans Rosenfeld .o_drive_info = nvme_bd_driveinfo, 3673c9168faSHans Rosenfeld .o_media_info = nvme_bd_mediainfo, 3683c9168faSHans Rosenfeld .o_devid_init = nvme_bd_devid, 3693c9168faSHans Rosenfeld .o_sync_cache = nvme_bd_sync, 3703c9168faSHans Rosenfeld .o_read = nvme_bd_read, 3713c9168faSHans Rosenfeld .o_write = nvme_bd_write, 3723c9168faSHans Rosenfeld }; 3733c9168faSHans Rosenfeld 3743c9168faSHans Rosenfeld int 3753c9168faSHans Rosenfeld _init(void) 3763c9168faSHans Rosenfeld { 3773c9168faSHans Rosenfeld int error; 3783c9168faSHans Rosenfeld 3793c9168faSHans Rosenfeld error = ddi_soft_state_init(&nvme_state, sizeof (nvme_t), 1); 3803c9168faSHans Rosenfeld if (error != DDI_SUCCESS) 3813c9168faSHans Rosenfeld return (error); 3823c9168faSHans Rosenfeld 3833c9168faSHans Rosenfeld nvme_cmd_cache = kmem_cache_create("nvme_cmd_cache", 3843c9168faSHans Rosenfeld sizeof (nvme_cmd_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 3853c9168faSHans Rosenfeld 3863c9168faSHans Rosenfeld bd_mod_init(&nvme_dev_ops); 3873c9168faSHans Rosenfeld 3883c9168faSHans Rosenfeld error = mod_install(&nvme_modlinkage); 3893c9168faSHans Rosenfeld if (error != DDI_SUCCESS) { 3903c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 3913c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 3923c9168faSHans Rosenfeld } 3933c9168faSHans Rosenfeld 3943c9168faSHans Rosenfeld return (error); 3953c9168faSHans Rosenfeld } 3963c9168faSHans Rosenfeld 3973c9168faSHans Rosenfeld int 3983c9168faSHans Rosenfeld _fini(void) 3993c9168faSHans Rosenfeld { 4003c9168faSHans Rosenfeld int error; 4013c9168faSHans Rosenfeld 4023c9168faSHans Rosenfeld error = mod_remove(&nvme_modlinkage); 4033c9168faSHans Rosenfeld if (error == DDI_SUCCESS) { 4043c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 4053c9168faSHans Rosenfeld kmem_cache_destroy(nvme_cmd_cache); 4063c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 4073c9168faSHans Rosenfeld } 4083c9168faSHans Rosenfeld 4093c9168faSHans Rosenfeld return (error); 4103c9168faSHans Rosenfeld } 4113c9168faSHans Rosenfeld 4123c9168faSHans Rosenfeld int 4133c9168faSHans Rosenfeld _info(struct modinfo *modinfop) 4143c9168faSHans Rosenfeld { 4153c9168faSHans Rosenfeld return (mod_info(&nvme_modlinkage, modinfop)); 4163c9168faSHans Rosenfeld } 4173c9168faSHans Rosenfeld 4183c9168faSHans Rosenfeld static inline void 4193c9168faSHans Rosenfeld nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val) 4203c9168faSHans Rosenfeld { 4213c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 4223c9168faSHans Rosenfeld 4233c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4243c9168faSHans Rosenfeld ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val); 4253c9168faSHans Rosenfeld } 4263c9168faSHans Rosenfeld 4273c9168faSHans Rosenfeld static inline void 4283c9168faSHans Rosenfeld nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val) 4293c9168faSHans Rosenfeld { 4303c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 4313c9168faSHans Rosenfeld 4323c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4333c9168faSHans Rosenfeld ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val); 4343c9168faSHans Rosenfeld } 4353c9168faSHans Rosenfeld 4363c9168faSHans Rosenfeld static inline uint64_t 4373c9168faSHans Rosenfeld nvme_get64(nvme_t *nvme, uintptr_t reg) 4383c9168faSHans Rosenfeld { 4393c9168faSHans Rosenfeld uint64_t val; 4403c9168faSHans Rosenfeld 4413c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 4423c9168faSHans Rosenfeld 4433c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4443c9168faSHans Rosenfeld val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg)); 4453c9168faSHans Rosenfeld 4463c9168faSHans Rosenfeld return (val); 4473c9168faSHans Rosenfeld } 4483c9168faSHans Rosenfeld 4493c9168faSHans Rosenfeld static inline uint32_t 4503c9168faSHans Rosenfeld nvme_get32(nvme_t *nvme, uintptr_t reg) 4513c9168faSHans Rosenfeld { 4523c9168faSHans Rosenfeld uint32_t val; 4533c9168faSHans Rosenfeld 4543c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 4553c9168faSHans Rosenfeld 4563c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4573c9168faSHans Rosenfeld val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg)); 4583c9168faSHans Rosenfeld 4593c9168faSHans Rosenfeld return (val); 4603c9168faSHans Rosenfeld } 4613c9168faSHans Rosenfeld 4623c9168faSHans Rosenfeld static boolean_t 4633c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme_t *nvme) 4643c9168faSHans Rosenfeld { 4653c9168faSHans Rosenfeld ddi_fm_error_t error; 4663c9168faSHans Rosenfeld 4673c9168faSHans Rosenfeld ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION); 4683c9168faSHans Rosenfeld 4693c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 4703c9168faSHans Rosenfeld return (B_TRUE); 4713c9168faSHans Rosenfeld 4723c9168faSHans Rosenfeld return (B_FALSE); 4733c9168faSHans Rosenfeld } 4743c9168faSHans Rosenfeld 4753c9168faSHans Rosenfeld static boolean_t 4763c9168faSHans Rosenfeld nvme_check_dma_hdl(nvme_dma_t *dma) 4773c9168faSHans Rosenfeld { 4783c9168faSHans Rosenfeld ddi_fm_error_t error; 4793c9168faSHans Rosenfeld 4803c9168faSHans Rosenfeld if (dma == NULL) 4813c9168faSHans Rosenfeld return (B_FALSE); 4823c9168faSHans Rosenfeld 4833c9168faSHans Rosenfeld ddi_fm_dma_err_get(dma->nd_dmah, &error, DDI_FME_VERSION); 4843c9168faSHans Rosenfeld 4853c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 4863c9168faSHans Rosenfeld return (B_TRUE); 4873c9168faSHans Rosenfeld 4883c9168faSHans Rosenfeld return (B_FALSE); 4893c9168faSHans Rosenfeld } 4903c9168faSHans Rosenfeld 4913c9168faSHans Rosenfeld static void 4923c9168faSHans Rosenfeld nvme_free_dma(nvme_dma_t *dma) 4933c9168faSHans Rosenfeld { 4943c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 4953c9168faSHans Rosenfeld (void) ddi_dma_unbind_handle(dma->nd_dmah); 4963c9168faSHans Rosenfeld if (dma->nd_acch != NULL) 4973c9168faSHans Rosenfeld ddi_dma_mem_free(&dma->nd_acch); 4983c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 4993c9168faSHans Rosenfeld ddi_dma_free_handle(&dma->nd_dmah); 5003c9168faSHans Rosenfeld kmem_free(dma, sizeof (nvme_dma_t)); 5013c9168faSHans Rosenfeld } 5023c9168faSHans Rosenfeld 5033c9168faSHans Rosenfeld static int 5043c9168faSHans Rosenfeld nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags, 5053c9168faSHans Rosenfeld ddi_dma_attr_t *dma_attr, nvme_dma_t **ret) 5063c9168faSHans Rosenfeld { 5073c9168faSHans Rosenfeld nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP); 5083c9168faSHans Rosenfeld 5093c9168faSHans Rosenfeld if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL, 5103c9168faSHans Rosenfeld &dma->nd_dmah) != DDI_SUCCESS) { 5113c9168faSHans Rosenfeld /* 5123c9168faSHans Rosenfeld * Due to DDI_DMA_SLEEP this can't be DDI_DMA_NORESOURCES, and 5133c9168faSHans Rosenfeld * the only other possible error is DDI_DMA_BADATTR which 5143c9168faSHans Rosenfeld * indicates a driver bug which should cause a panic. 5153c9168faSHans Rosenfeld */ 5163c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, 5173c9168faSHans Rosenfeld "!failed to get DMA handle, check DMA attributes"); 5183c9168faSHans Rosenfeld return (DDI_FAILURE); 5193c9168faSHans Rosenfeld } 5203c9168faSHans Rosenfeld 5213c9168faSHans Rosenfeld /* 5223c9168faSHans Rosenfeld * ddi_dma_mem_alloc() can only fail when DDI_DMA_NOSLEEP is specified 5233c9168faSHans Rosenfeld * or the flags are conflicting, which isn't the case here. 5243c9168faSHans Rosenfeld */ 5253c9168faSHans Rosenfeld (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr, 5263c9168faSHans Rosenfeld DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &dma->nd_memp, 5273c9168faSHans Rosenfeld &dma->nd_len, &dma->nd_acch); 5283c9168faSHans Rosenfeld 5293c9168faSHans Rosenfeld if (ddi_dma_addr_bind_handle(dma->nd_dmah, NULL, dma->nd_memp, 5303c9168faSHans Rosenfeld dma->nd_len, flags | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, 5313c9168faSHans Rosenfeld &dma->nd_cookie, &dma->nd_ncookie) != DDI_DMA_MAPPED) { 5323c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 5333c9168faSHans Rosenfeld "!failed to bind DMA memory"); 5343c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_dma_bind_err); 5353c9168faSHans Rosenfeld *ret = NULL; 5363c9168faSHans Rosenfeld nvme_free_dma(dma); 5373c9168faSHans Rosenfeld return (DDI_FAILURE); 5383c9168faSHans Rosenfeld } 5393c9168faSHans Rosenfeld 5403c9168faSHans Rosenfeld bzero(dma->nd_memp, dma->nd_len); 5413c9168faSHans Rosenfeld 5423c9168faSHans Rosenfeld *ret = dma; 5433c9168faSHans Rosenfeld return (DDI_SUCCESS); 5443c9168faSHans Rosenfeld } 5453c9168faSHans Rosenfeld 5463c9168faSHans Rosenfeld static int 5473c9168faSHans Rosenfeld nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len, 5483c9168faSHans Rosenfeld uint_t flags, nvme_dma_t **dma) 5493c9168faSHans Rosenfeld { 5503c9168faSHans Rosenfeld uint32_t len = nentry * qe_len; 5513c9168faSHans Rosenfeld ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr; 5523c9168faSHans Rosenfeld 5533c9168faSHans Rosenfeld len = roundup(len, nvme->n_pagesize); 5543c9168faSHans Rosenfeld 5553c9168faSHans Rosenfeld q_dma_attr.dma_attr_minxfer = len; 5563c9168faSHans Rosenfeld 5573c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma) 5583c9168faSHans Rosenfeld != DDI_SUCCESS) { 5593c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 5603c9168faSHans Rosenfeld "!failed to get DMA memory for queue"); 5613c9168faSHans Rosenfeld goto fail; 5623c9168faSHans Rosenfeld } 5633c9168faSHans Rosenfeld 5643c9168faSHans Rosenfeld if ((*dma)->nd_ncookie != 1) { 5653c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 5663c9168faSHans Rosenfeld "!got too many cookies for queue DMA"); 5673c9168faSHans Rosenfeld goto fail; 5683c9168faSHans Rosenfeld } 5693c9168faSHans Rosenfeld 5703c9168faSHans Rosenfeld return (DDI_SUCCESS); 5713c9168faSHans Rosenfeld 5723c9168faSHans Rosenfeld fail: 5733c9168faSHans Rosenfeld if (*dma) { 5743c9168faSHans Rosenfeld nvme_free_dma(*dma); 5753c9168faSHans Rosenfeld *dma = NULL; 5763c9168faSHans Rosenfeld } 5773c9168faSHans Rosenfeld 5783c9168faSHans Rosenfeld return (DDI_FAILURE); 5793c9168faSHans Rosenfeld } 5803c9168faSHans Rosenfeld 5813c9168faSHans Rosenfeld static void 5823c9168faSHans Rosenfeld nvme_free_qpair(nvme_qpair_t *qp) 5833c9168faSHans Rosenfeld { 5843c9168faSHans Rosenfeld int i; 5853c9168faSHans Rosenfeld 5863c9168faSHans Rosenfeld mutex_destroy(&qp->nq_mutex); 5873c9168faSHans Rosenfeld 5883c9168faSHans Rosenfeld if (qp->nq_sqdma != NULL) 5893c9168faSHans Rosenfeld nvme_free_dma(qp->nq_sqdma); 5903c9168faSHans Rosenfeld if (qp->nq_cqdma != NULL) 5913c9168faSHans Rosenfeld nvme_free_dma(qp->nq_cqdma); 5923c9168faSHans Rosenfeld 5933c9168faSHans Rosenfeld if (qp->nq_active_cmds > 0) 5943c9168faSHans Rosenfeld for (i = 0; i != qp->nq_nentry; i++) 5953c9168faSHans Rosenfeld if (qp->nq_cmd[i] != NULL) 5963c9168faSHans Rosenfeld nvme_free_cmd(qp->nq_cmd[i]); 5973c9168faSHans Rosenfeld 5983c9168faSHans Rosenfeld if (qp->nq_cmd != NULL) 5993c9168faSHans Rosenfeld kmem_free(qp->nq_cmd, sizeof (nvme_cmd_t *) * qp->nq_nentry); 6003c9168faSHans Rosenfeld 6013c9168faSHans Rosenfeld kmem_free(qp, sizeof (nvme_qpair_t)); 6023c9168faSHans Rosenfeld } 6033c9168faSHans Rosenfeld 6043c9168faSHans Rosenfeld static int 6053c9168faSHans Rosenfeld nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp, 6063c9168faSHans Rosenfeld int idx) 6073c9168faSHans Rosenfeld { 6083c9168faSHans Rosenfeld nvme_qpair_t *qp = kmem_zalloc(sizeof (*qp), KM_SLEEP); 6093c9168faSHans Rosenfeld 6103c9168faSHans Rosenfeld mutex_init(&qp->nq_mutex, NULL, MUTEX_DRIVER, 6113c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 6123c9168faSHans Rosenfeld 6133c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t), 6143c9168faSHans Rosenfeld DDI_DMA_WRITE, &qp->nq_sqdma) != DDI_SUCCESS) 6153c9168faSHans Rosenfeld goto fail; 6163c9168faSHans Rosenfeld 6173c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t), 6183c9168faSHans Rosenfeld DDI_DMA_READ, &qp->nq_cqdma) != DDI_SUCCESS) 6193c9168faSHans Rosenfeld goto fail; 6203c9168faSHans Rosenfeld 6213c9168faSHans Rosenfeld qp->nq_sq = (nvme_sqe_t *)qp->nq_sqdma->nd_memp; 6223c9168faSHans Rosenfeld qp->nq_cq = (nvme_cqe_t *)qp->nq_cqdma->nd_memp; 6233c9168faSHans Rosenfeld qp->nq_nentry = nentry; 6243c9168faSHans Rosenfeld 6253c9168faSHans Rosenfeld qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx); 6263c9168faSHans Rosenfeld qp->nq_cqhdbl = NVME_REG_CQHDBL(nvme, idx); 6273c9168faSHans Rosenfeld 6283c9168faSHans Rosenfeld qp->nq_cmd = kmem_zalloc(sizeof (nvme_cmd_t *) * nentry, KM_SLEEP); 6293c9168faSHans Rosenfeld qp->nq_next_cmd = 0; 6303c9168faSHans Rosenfeld 6313c9168faSHans Rosenfeld *nqp = qp; 6323c9168faSHans Rosenfeld return (DDI_SUCCESS); 6333c9168faSHans Rosenfeld 6343c9168faSHans Rosenfeld fail: 6353c9168faSHans Rosenfeld nvme_free_qpair(qp); 6363c9168faSHans Rosenfeld *nqp = NULL; 6373c9168faSHans Rosenfeld 6383c9168faSHans Rosenfeld return (DDI_FAILURE); 6393c9168faSHans Rosenfeld } 6403c9168faSHans Rosenfeld 6413c9168faSHans Rosenfeld static nvme_cmd_t * 6423c9168faSHans Rosenfeld nvme_alloc_cmd(nvme_t *nvme, int kmflag) 6433c9168faSHans Rosenfeld { 6443c9168faSHans Rosenfeld nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag); 6453c9168faSHans Rosenfeld 6463c9168faSHans Rosenfeld if (cmd == NULL) 6473c9168faSHans Rosenfeld return (cmd); 6483c9168faSHans Rosenfeld 6493c9168faSHans Rosenfeld bzero(cmd, sizeof (nvme_cmd_t)); 6503c9168faSHans Rosenfeld 6513c9168faSHans Rosenfeld cmd->nc_nvme = nvme; 6523c9168faSHans Rosenfeld 6533c9168faSHans Rosenfeld mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER, 6543c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 6553c9168faSHans Rosenfeld cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL); 6563c9168faSHans Rosenfeld 6573c9168faSHans Rosenfeld return (cmd); 6583c9168faSHans Rosenfeld } 6593c9168faSHans Rosenfeld 6603c9168faSHans Rosenfeld static void 6613c9168faSHans Rosenfeld nvme_free_cmd(nvme_cmd_t *cmd) 6623c9168faSHans Rosenfeld { 6633c9168faSHans Rosenfeld if (cmd->nc_dma) { 6643c9168faSHans Rosenfeld nvme_free_dma(cmd->nc_dma); 6653c9168faSHans Rosenfeld cmd->nc_dma = NULL; 6663c9168faSHans Rosenfeld } 6673c9168faSHans Rosenfeld 6683c9168faSHans Rosenfeld cv_destroy(&cmd->nc_cv); 6693c9168faSHans Rosenfeld mutex_destroy(&cmd->nc_mutex); 6703c9168faSHans Rosenfeld 6713c9168faSHans Rosenfeld kmem_cache_free(nvme_cmd_cache, cmd); 6723c9168faSHans Rosenfeld } 6733c9168faSHans Rosenfeld 6743c9168faSHans Rosenfeld static int 6753c9168faSHans Rosenfeld nvme_submit_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 6763c9168faSHans Rosenfeld { 6773c9168faSHans Rosenfeld nvme_reg_sqtdbl_t tail = { 0 }; 6783c9168faSHans Rosenfeld 6793c9168faSHans Rosenfeld mutex_enter(&qp->nq_mutex); 6803c9168faSHans Rosenfeld 6813c9168faSHans Rosenfeld if (qp->nq_active_cmds == qp->nq_nentry) { 6823c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 6833c9168faSHans Rosenfeld return (DDI_FAILURE); 6843c9168faSHans Rosenfeld } 6853c9168faSHans Rosenfeld 6863c9168faSHans Rosenfeld cmd->nc_completed = B_FALSE; 6873c9168faSHans Rosenfeld 6883c9168faSHans Rosenfeld /* 6893c9168faSHans Rosenfeld * Try to insert the cmd into the active cmd array at the nq_next_cmd 6903c9168faSHans Rosenfeld * slot. If the slot is already occupied advance to the next slot and 6913c9168faSHans Rosenfeld * try again. This can happen for long running commands like async event 6923c9168faSHans Rosenfeld * requests. 6933c9168faSHans Rosenfeld */ 6943c9168faSHans Rosenfeld while (qp->nq_cmd[qp->nq_next_cmd] != NULL) 6953c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 6963c9168faSHans Rosenfeld qp->nq_cmd[qp->nq_next_cmd] = cmd; 6973c9168faSHans Rosenfeld 6983c9168faSHans Rosenfeld qp->nq_active_cmds++; 6993c9168faSHans Rosenfeld 7003c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cid = qp->nq_next_cmd; 7013c9168faSHans Rosenfeld bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t)); 7023c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_sqdma->nd_dmah, 7033c9168faSHans Rosenfeld sizeof (nvme_sqe_t) * qp->nq_sqtail, 7043c9168faSHans Rosenfeld sizeof (nvme_sqe_t), DDI_DMA_SYNC_FORDEV); 7053c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 7063c9168faSHans Rosenfeld 7073c9168faSHans Rosenfeld tail.b.sqtdbl_sqt = qp->nq_sqtail = (qp->nq_sqtail + 1) % qp->nq_nentry; 7083c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r); 7093c9168faSHans Rosenfeld 7103c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 7113c9168faSHans Rosenfeld return (DDI_SUCCESS); 7123c9168faSHans Rosenfeld } 7133c9168faSHans Rosenfeld 7143c9168faSHans Rosenfeld static nvme_cmd_t * 7153c9168faSHans Rosenfeld nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp) 7163c9168faSHans Rosenfeld { 7173c9168faSHans Rosenfeld nvme_reg_cqhdbl_t head = { 0 }; 7183c9168faSHans Rosenfeld 7193c9168faSHans Rosenfeld nvme_cqe_t *cqe; 7203c9168faSHans Rosenfeld nvme_cmd_t *cmd; 7213c9168faSHans Rosenfeld 7223c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_cqdma->nd_dmah, 0, 7233c9168faSHans Rosenfeld sizeof (nvme_cqe_t) * qp->nq_nentry, DDI_DMA_SYNC_FORKERNEL); 7243c9168faSHans Rosenfeld 7253c9168faSHans Rosenfeld cqe = &qp->nq_cq[qp->nq_cqhead]; 7263c9168faSHans Rosenfeld 7273c9168faSHans Rosenfeld /* Check phase tag of CQE. Hardware inverts it for new entries. */ 7283c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_p == qp->nq_phase) 7293c9168faSHans Rosenfeld return (NULL); 7303c9168faSHans Rosenfeld 7313c9168faSHans Rosenfeld ASSERT(nvme->n_ioq[cqe->cqe_sqid] == qp); 7323c9168faSHans Rosenfeld ASSERT(cqe->cqe_cid < qp->nq_nentry); 7333c9168faSHans Rosenfeld 7343c9168faSHans Rosenfeld mutex_enter(&qp->nq_mutex); 7353c9168faSHans Rosenfeld cmd = qp->nq_cmd[cqe->cqe_cid]; 7363c9168faSHans Rosenfeld qp->nq_cmd[cqe->cqe_cid] = NULL; 7373c9168faSHans Rosenfeld qp->nq_active_cmds--; 7383c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 7393c9168faSHans Rosenfeld 7403c9168faSHans Rosenfeld ASSERT(cmd != NULL); 7413c9168faSHans Rosenfeld ASSERT(cmd->nc_nvme == nvme); 7423c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid == cqe->cqe_sqid); 7433c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_cid == cqe->cqe_cid); 7443c9168faSHans Rosenfeld bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t)); 7453c9168faSHans Rosenfeld 7463c9168faSHans Rosenfeld qp->nq_sqhead = cqe->cqe_sqhd; 7473c9168faSHans Rosenfeld 7483c9168faSHans Rosenfeld head.b.cqhdbl_cqh = qp->nq_cqhead = (qp->nq_cqhead + 1) % qp->nq_nentry; 7493c9168faSHans Rosenfeld 7503c9168faSHans Rosenfeld /* Toggle phase on wrap-around. */ 7513c9168faSHans Rosenfeld if (qp->nq_cqhead == 0) 7523c9168faSHans Rosenfeld qp->nq_phase = qp->nq_phase ? 0 : 1; 7533c9168faSHans Rosenfeld 7543c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_cqhdbl, head.r); 7553c9168faSHans Rosenfeld 7563c9168faSHans Rosenfeld return (cmd); 7573c9168faSHans Rosenfeld } 7583c9168faSHans Rosenfeld 7593c9168faSHans Rosenfeld static int 7603c9168faSHans Rosenfeld nvme_check_unknown_cmd_status(nvme_cmd_t *cmd) 7613c9168faSHans Rosenfeld { 7623c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 7633c9168faSHans Rosenfeld 7643c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 7653c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 7663c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 7673c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 7683c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 7693c9168faSHans Rosenfeld 770bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 771bef9e21aSHans Rosenfeld 7723c9168faSHans Rosenfeld if (cmd->nc_nvme->n_strict_version) { 7733c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 7743c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 7753c9168faSHans Rosenfeld } 7763c9168faSHans Rosenfeld 7773c9168faSHans Rosenfeld return (EIO); 7783c9168faSHans Rosenfeld } 7793c9168faSHans Rosenfeld 7803c9168faSHans Rosenfeld static int 7813c9168faSHans Rosenfeld nvme_check_vendor_cmd_status(nvme_cmd_t *cmd) 7823c9168faSHans Rosenfeld { 7833c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 7843c9168faSHans Rosenfeld 7853c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 7863c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 7873c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 7883c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 7893c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 79034c938c7SPete Shephard if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) { 7913c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 7923c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 7933c9168faSHans Rosenfeld } 7943c9168faSHans Rosenfeld 7953c9168faSHans Rosenfeld return (EIO); 7963c9168faSHans Rosenfeld } 7973c9168faSHans Rosenfeld 7983c9168faSHans Rosenfeld static int 7993c9168faSHans Rosenfeld nvme_check_integrity_cmd_status(nvme_cmd_t *cmd) 8003c9168faSHans Rosenfeld { 8013c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8023c9168faSHans Rosenfeld 8033c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 8043c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_WRITE: 8053c9168faSHans Rosenfeld /* write fail */ 8063c9168faSHans Rosenfeld /* TODO: post ereport */ 807bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 8083c9168faSHans Rosenfeld return (EIO); 8093c9168faSHans Rosenfeld 8103c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_READ: 8113c9168faSHans Rosenfeld /* read fail */ 8123c9168faSHans Rosenfeld /* TODO: post ereport */ 813bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 8143c9168faSHans Rosenfeld return (EIO); 8153c9168faSHans Rosenfeld 8163c9168faSHans Rosenfeld default: 8173c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 8183c9168faSHans Rosenfeld } 8193c9168faSHans Rosenfeld } 8203c9168faSHans Rosenfeld 8213c9168faSHans Rosenfeld static int 8223c9168faSHans Rosenfeld nvme_check_generic_cmd_status(nvme_cmd_t *cmd) 8233c9168faSHans Rosenfeld { 8243c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8253c9168faSHans Rosenfeld 8263c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 8273c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_SUCCESS: 8283c9168faSHans Rosenfeld return (0); 8293c9168faSHans Rosenfeld 8303c9168faSHans Rosenfeld /* 8313c9168faSHans Rosenfeld * Errors indicating a bug in the driver should cause a panic. 8323c9168faSHans Rosenfeld */ 8333c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_OPC: 8343c9168faSHans Rosenfeld /* Invalid Command Opcode */ 8353c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 8363c9168faSHans Rosenfeld "invalid opcode in cmd %p", (void *)cmd); 8373c9168faSHans Rosenfeld return (0); 8383c9168faSHans Rosenfeld 8393c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_FLD: 8403c9168faSHans Rosenfeld /* Invalid Field in Command */ 8413c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 8423c9168faSHans Rosenfeld "invalid field in cmd %p", (void *)cmd); 8433c9168faSHans Rosenfeld return (0); 8443c9168faSHans Rosenfeld 8453c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ID_CNFL: 8463c9168faSHans Rosenfeld /* Command ID Conflict */ 8473c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 8483c9168faSHans Rosenfeld "cmd ID conflict in cmd %p", (void *)cmd); 8493c9168faSHans Rosenfeld return (0); 8503c9168faSHans Rosenfeld 8513c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_NS: 8523c9168faSHans Rosenfeld /* Invalid Namespace or Format */ 8533c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 8543c9168faSHans Rosenfeld "invalid NS/format in cmd %p", (void *)cmd); 8553c9168faSHans Rosenfeld return (0); 8563c9168faSHans Rosenfeld 8573c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_LBA_RANGE: 8583c9168faSHans Rosenfeld /* LBA Out Of Range */ 8593c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 8603c9168faSHans Rosenfeld "LBA out of range in cmd %p", (void *)cmd); 8613c9168faSHans Rosenfeld return (0); 8623c9168faSHans Rosenfeld 8633c9168faSHans Rosenfeld /* 8643c9168faSHans Rosenfeld * Non-fatal errors, handle gracefully. 8653c9168faSHans Rosenfeld */ 8663c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_DATA_XFR_ERR: 8673c9168faSHans Rosenfeld /* Data Transfer Error (DMA) */ 8683c9168faSHans Rosenfeld /* TODO: post ereport */ 8693c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err); 870bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 8713c9168faSHans Rosenfeld return (EIO); 8723c9168faSHans Rosenfeld 8733c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INTERNAL_ERR: 8743c9168faSHans Rosenfeld /* 8753c9168faSHans Rosenfeld * Internal Error. The spec (v1.0, section 4.5.1.2) says 8763c9168faSHans Rosenfeld * detailed error information is returned as async event, 8773c9168faSHans Rosenfeld * so we pretty much ignore the error here and handle it 8783c9168faSHans Rosenfeld * in the async event handler. 8793c9168faSHans Rosenfeld */ 8803c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_internal_err); 881bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 8823c9168faSHans Rosenfeld return (EIO); 8833c9168faSHans Rosenfeld 8843c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_REQUEST: 8853c9168faSHans Rosenfeld /* 8863c9168faSHans Rosenfeld * Command Abort Requested. This normally happens only when a 8873c9168faSHans Rosenfeld * command times out. 8883c9168faSHans Rosenfeld */ 8893c9168faSHans Rosenfeld /* TODO: post ereport or change blkdev to handle this? */ 8903c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err); 8913c9168faSHans Rosenfeld return (ECANCELED); 8923c9168faSHans Rosenfeld 8933c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_PWRLOSS: 8943c9168faSHans Rosenfeld /* Command Aborted due to Power Loss Notification */ 8953c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 8963c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 8973c9168faSHans Rosenfeld return (EIO); 8983c9168faSHans Rosenfeld 8993c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_SQ_DEL: 9003c9168faSHans Rosenfeld /* Command Aborted due to SQ Deletion */ 9013c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del); 9023c9168faSHans Rosenfeld return (EIO); 9033c9168faSHans Rosenfeld 9043c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_CAP_EXC: 9053c9168faSHans Rosenfeld /* Capacity Exceeded */ 9063c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc); 907bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 9083c9168faSHans Rosenfeld return (EIO); 9093c9168faSHans Rosenfeld 9103c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_NS_NOTRDY: 9113c9168faSHans Rosenfeld /* Namespace Not Ready */ 9123c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy); 913bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 9143c9168faSHans Rosenfeld return (EIO); 9153c9168faSHans Rosenfeld 9163c9168faSHans Rosenfeld default: 9173c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 9183c9168faSHans Rosenfeld } 9193c9168faSHans Rosenfeld } 9203c9168faSHans Rosenfeld 9213c9168faSHans Rosenfeld static int 9223c9168faSHans Rosenfeld nvme_check_specific_cmd_status(nvme_cmd_t *cmd) 9233c9168faSHans Rosenfeld { 9243c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 9253c9168faSHans Rosenfeld 9263c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 9273c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_CQ: 9283c9168faSHans Rosenfeld /* Completion Queue Invalid */ 9293c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE); 9303c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err); 9313c9168faSHans Rosenfeld return (EINVAL); 9323c9168faSHans Rosenfeld 9333c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_QID: 9343c9168faSHans Rosenfeld /* Invalid Queue Identifier */ 9353c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 9363c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE || 9373c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE || 9383c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 9393c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err); 9403c9168faSHans Rosenfeld return (EINVAL); 9413c9168faSHans Rosenfeld 9423c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_MAX_QSZ_EXC: 9433c9168faSHans Rosenfeld /* Max Queue Size Exceeded */ 9443c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 9453c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 9463c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc); 9473c9168faSHans Rosenfeld return (EINVAL); 9483c9168faSHans Rosenfeld 9493c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ABRT_CMD_EXC: 9503c9168faSHans Rosenfeld /* Abort Command Limit Exceeded */ 9513c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT); 9523c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9533c9168faSHans Rosenfeld "abort command limit exceeded in cmd %p", (void *)cmd); 9543c9168faSHans Rosenfeld return (0); 9553c9168faSHans Rosenfeld 9563c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC: 9573c9168faSHans Rosenfeld /* Async Event Request Limit Exceeded */ 9583c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT); 9593c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9603c9168faSHans Rosenfeld "async event request limit exceeded in cmd %p", 9613c9168faSHans Rosenfeld (void *)cmd); 9623c9168faSHans Rosenfeld return (0); 9633c9168faSHans Rosenfeld 9643c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_INT_VECT: 9653c9168faSHans Rosenfeld /* Invalid Interrupt Vector */ 9663c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 9673c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect); 9683c9168faSHans Rosenfeld return (EINVAL); 9693c9168faSHans Rosenfeld 9703c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_LOG_PAGE: 9713c9168faSHans Rosenfeld /* Invalid Log Page */ 9723c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE); 9733c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_log_page); 974bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 9753c9168faSHans Rosenfeld return (EINVAL); 9763c9168faSHans Rosenfeld 9773c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_FORMAT: 9783c9168faSHans Rosenfeld /* Invalid Format */ 9793c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT); 9803c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_format); 981bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 9823c9168faSHans Rosenfeld return (EINVAL); 9833c9168faSHans Rosenfeld 9843c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_Q_DEL: 9853c9168faSHans Rosenfeld /* Invalid Queue Deletion */ 9863c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 9873c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_q_del); 9883c9168faSHans Rosenfeld return (EINVAL); 9893c9168faSHans Rosenfeld 9903c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_CNFL_ATTR: 9913c9168faSHans Rosenfeld /* Conflicting Attributes */ 9923c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT || 9933c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 9943c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 9953c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr); 996bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 9973c9168faSHans Rosenfeld return (EINVAL); 9983c9168faSHans Rosenfeld 9993c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_INV_PROT: 10003c9168faSHans Rosenfeld /* Invalid Protection Information */ 10013c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE || 10023c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 10033c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 10043c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_prot); 1005bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10063c9168faSHans Rosenfeld return (EINVAL); 10073c9168faSHans Rosenfeld 10083c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_READONLY: 10093c9168faSHans Rosenfeld /* Write to Read Only Range */ 10103c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 10113c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_readonly); 1012bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10133c9168faSHans Rosenfeld return (EROFS); 10143c9168faSHans Rosenfeld 10153c9168faSHans Rosenfeld default: 10163c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 10173c9168faSHans Rosenfeld } 10183c9168faSHans Rosenfeld } 10193c9168faSHans Rosenfeld 10203c9168faSHans Rosenfeld static inline int 10213c9168faSHans Rosenfeld nvme_check_cmd_status(nvme_cmd_t *cmd) 10223c9168faSHans Rosenfeld { 10233c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 10243c9168faSHans Rosenfeld 10253c9168faSHans Rosenfeld /* take a shortcut if everything is alright */ 10263c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 10273c9168faSHans Rosenfeld cqe->cqe_sf.sf_sc == NVME_CQE_SC_GEN_SUCCESS) 10283c9168faSHans Rosenfeld return (0); 10293c9168faSHans Rosenfeld 10303c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC) 10313c9168faSHans Rosenfeld return (nvme_check_generic_cmd_status(cmd)); 10323c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_SPECIFIC) 10333c9168faSHans Rosenfeld return (nvme_check_specific_cmd_status(cmd)); 10343c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_INTEGRITY) 10353c9168faSHans Rosenfeld return (nvme_check_integrity_cmd_status(cmd)); 10363c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_VENDOR) 10373c9168faSHans Rosenfeld return (nvme_check_vendor_cmd_status(cmd)); 10383c9168faSHans Rosenfeld 10393c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 10403c9168faSHans Rosenfeld } 10413c9168faSHans Rosenfeld 10423c9168faSHans Rosenfeld /* 10433c9168faSHans Rosenfeld * nvme_abort_cmd_cb -- replaces nc_callback of aborted commands 10443c9168faSHans Rosenfeld * 10453c9168faSHans Rosenfeld * This functions takes care of cleaning up aborted commands. The command 10463c9168faSHans Rosenfeld * status is checked to catch any fatal errors. 10473c9168faSHans Rosenfeld */ 10483c9168faSHans Rosenfeld static void 10493c9168faSHans Rosenfeld nvme_abort_cmd_cb(void *arg) 10503c9168faSHans Rosenfeld { 10513c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 10523c9168faSHans Rosenfeld 10533c9168faSHans Rosenfeld /* 10543c9168faSHans Rosenfeld * Grab the command mutex. Once we have it we hold the last reference 10553c9168faSHans Rosenfeld * to the command and can safely free it. 10563c9168faSHans Rosenfeld */ 10573c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 10583c9168faSHans Rosenfeld (void) nvme_check_cmd_status(cmd); 10593c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 10603c9168faSHans Rosenfeld 10613c9168faSHans Rosenfeld nvme_free_cmd(cmd); 10623c9168faSHans Rosenfeld } 10633c9168faSHans Rosenfeld 10643c9168faSHans Rosenfeld static void 10653c9168faSHans Rosenfeld nvme_abort_cmd(nvme_cmd_t *abort_cmd) 10663c9168faSHans Rosenfeld { 10673c9168faSHans Rosenfeld nvme_t *nvme = abort_cmd->nc_nvme; 10683c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 10693c9168faSHans Rosenfeld nvme_abort_cmd_t ac = { 0 }; 10703c9168faSHans Rosenfeld 10713c9168faSHans Rosenfeld sema_p(&nvme->n_abort_sema); 10723c9168faSHans Rosenfeld 10733c9168faSHans Rosenfeld ac.b.ac_cid = abort_cmd->nc_sqe.sqe_cid; 10743c9168faSHans Rosenfeld ac.b.ac_sqid = abort_cmd->nc_sqid; 10753c9168faSHans Rosenfeld 10763c9168faSHans Rosenfeld /* 10773c9168faSHans Rosenfeld * Drop the mutex of the aborted command. From this point on 10783c9168faSHans Rosenfeld * we must assume that the abort callback has freed the command. 10793c9168faSHans Rosenfeld */ 10803c9168faSHans Rosenfeld mutex_exit(&abort_cmd->nc_mutex); 10813c9168faSHans Rosenfeld 10823c9168faSHans Rosenfeld cmd->nc_sqid = 0; 10833c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT; 10843c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 10853c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = ac.r; 10863c9168faSHans Rosenfeld 10873c9168faSHans Rosenfeld /* 10883c9168faSHans Rosenfeld * Send the ABORT to the hardware. The ABORT command will return _after_ 10893c9168faSHans Rosenfeld * the aborted command has completed (aborted or otherwise). 10903c9168faSHans Rosenfeld */ 1091*e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 10923c9168faSHans Rosenfeld sema_v(&nvme->n_abort_sema); 10933c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 10943c9168faSHans Rosenfeld "!nvme_admin_cmd failed for ABORT"); 10953c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_abort_failed); 10963c9168faSHans Rosenfeld return; 10973c9168faSHans Rosenfeld } 10983c9168faSHans Rosenfeld sema_v(&nvme->n_abort_sema); 10993c9168faSHans Rosenfeld 11003c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 11013c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 11023c9168faSHans Rosenfeld "!ABORT failed with sct = %x, sc = %x", 11033c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 11043c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_abort_failed); 11053c9168faSHans Rosenfeld } else { 11063c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_aborted); 11073c9168faSHans Rosenfeld } 11083c9168faSHans Rosenfeld 11093c9168faSHans Rosenfeld nvme_free_cmd(cmd); 11103c9168faSHans Rosenfeld } 11113c9168faSHans Rosenfeld 11123c9168faSHans Rosenfeld /* 11133c9168faSHans Rosenfeld * nvme_wait_cmd -- wait for command completion or timeout 11143c9168faSHans Rosenfeld * 11153c9168faSHans Rosenfeld * Returns B_TRUE if the command completed normally. 11163c9168faSHans Rosenfeld * 11173c9168faSHans Rosenfeld * Returns B_FALSE if the command timed out and an abort was attempted. The 11183c9168faSHans Rosenfeld * command mutex will be dropped and the command must be considered freed. The 11193c9168faSHans Rosenfeld * freeing of the command is normally done by the abort command callback. 11203c9168faSHans Rosenfeld * 11213c9168faSHans Rosenfeld * In case of a serious error or a timeout of the abort command the hardware 11223c9168faSHans Rosenfeld * will be declared dead and FMA will be notified. 11233c9168faSHans Rosenfeld */ 11243c9168faSHans Rosenfeld static boolean_t 1125*e8ba2a38SHans Rosenfeld nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec) 11263c9168faSHans Rosenfeld { 1127*e8ba2a38SHans Rosenfeld clock_t timeout = ddi_get_lbolt() + drv_usectohz(sec * MICROSEC); 11283c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 11293c9168faSHans Rosenfeld nvme_reg_csts_t csts; 11303c9168faSHans Rosenfeld 11313c9168faSHans Rosenfeld ASSERT(mutex_owned(&cmd->nc_mutex)); 11323c9168faSHans Rosenfeld 11333c9168faSHans Rosenfeld while (!cmd->nc_completed) { 11343c9168faSHans Rosenfeld if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1) 11353c9168faSHans Rosenfeld break; 11363c9168faSHans Rosenfeld } 11373c9168faSHans Rosenfeld 11383c9168faSHans Rosenfeld if (cmd->nc_completed) 11393c9168faSHans Rosenfeld return (B_TRUE); 11403c9168faSHans Rosenfeld 11413c9168faSHans Rosenfeld /* 11423c9168faSHans Rosenfeld * The command timed out. Change the callback to the cleanup function. 11433c9168faSHans Rosenfeld */ 11443c9168faSHans Rosenfeld cmd->nc_callback = nvme_abort_cmd_cb; 11453c9168faSHans Rosenfeld 11463c9168faSHans Rosenfeld /* 11473c9168faSHans Rosenfeld * Check controller for fatal status, any errors associated with the 11483c9168faSHans Rosenfeld * register or DMA handle, or for a double timeout (abort command timed 11493c9168faSHans Rosenfeld * out). If necessary log a warning and call FMA. 11503c9168faSHans Rosenfeld */ 11513c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 11523c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!command timeout, " 11533c9168faSHans Rosenfeld "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_opc, csts.b.csts_cfs); 11543c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_timeout); 11553c9168faSHans Rosenfeld 11563c9168faSHans Rosenfeld if (csts.b.csts_cfs || 11573c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme) || 11583c9168faSHans Rosenfeld nvme_check_dma_hdl(cmd->nc_dma) || 11593c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) { 11603c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 11613c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 11623c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 11633c9168faSHans Rosenfeld } else { 11643c9168faSHans Rosenfeld /* 11653c9168faSHans Rosenfeld * Try to abort the command. The command mutex is released by 11663c9168faSHans Rosenfeld * nvme_abort_cmd(). 11673c9168faSHans Rosenfeld * If the abort succeeds it will have freed the aborted command. 11683c9168faSHans Rosenfeld * If the abort fails for other reasons we must assume that the 11693c9168faSHans Rosenfeld * command may complete at any time, and the callback will free 11703c9168faSHans Rosenfeld * it for us. 11713c9168faSHans Rosenfeld */ 11723c9168faSHans Rosenfeld nvme_abort_cmd(cmd); 11733c9168faSHans Rosenfeld } 11743c9168faSHans Rosenfeld 11753c9168faSHans Rosenfeld return (B_FALSE); 11763c9168faSHans Rosenfeld } 11773c9168faSHans Rosenfeld 11783c9168faSHans Rosenfeld static void 11793c9168faSHans Rosenfeld nvme_wakeup_cmd(void *arg) 11803c9168faSHans Rosenfeld { 11813c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 11823c9168faSHans Rosenfeld 11833c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 11843c9168faSHans Rosenfeld /* 11853c9168faSHans Rosenfeld * There is a slight chance that this command completed shortly after 11863c9168faSHans Rosenfeld * the timeout was hit in nvme_wait_cmd() but before the callback was 11873c9168faSHans Rosenfeld * changed. Catch that case here and clean up accordingly. 11883c9168faSHans Rosenfeld */ 11893c9168faSHans Rosenfeld if (cmd->nc_callback == nvme_abort_cmd_cb) { 11903c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 11913c9168faSHans Rosenfeld nvme_abort_cmd_cb(cmd); 11923c9168faSHans Rosenfeld return; 11933c9168faSHans Rosenfeld } 11943c9168faSHans Rosenfeld 11953c9168faSHans Rosenfeld cmd->nc_completed = B_TRUE; 11963c9168faSHans Rosenfeld cv_signal(&cmd->nc_cv); 11973c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 11983c9168faSHans Rosenfeld } 11993c9168faSHans Rosenfeld 12003c9168faSHans Rosenfeld static void 12013c9168faSHans Rosenfeld nvme_async_event_task(void *arg) 12023c9168faSHans Rosenfeld { 12033c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 12043c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 12053c9168faSHans Rosenfeld nvme_error_log_entry_t *error_log = NULL; 12063c9168faSHans Rosenfeld nvme_health_log_t *health_log = NULL; 12073c9168faSHans Rosenfeld nvme_async_event_t event; 12083c9168faSHans Rosenfeld int ret; 12093c9168faSHans Rosenfeld 12103c9168faSHans Rosenfeld /* 12113c9168faSHans Rosenfeld * Check for errors associated with the async request itself. The only 12123c9168faSHans Rosenfeld * command-specific error is "async event limit exceeded", which 12133c9168faSHans Rosenfeld * indicates a programming error in the driver and causes a panic in 12143c9168faSHans Rosenfeld * nvme_check_cmd_status(). 12153c9168faSHans Rosenfeld * 12163c9168faSHans Rosenfeld * Other possible errors are various scenarios where the async request 12173c9168faSHans Rosenfeld * was aborted, or internal errors in the device. Internal errors are 12183c9168faSHans Rosenfeld * reported to FMA, the command aborts need no special handling here. 12193c9168faSHans Rosenfeld */ 12203c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 12213c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 12223c9168faSHans Rosenfeld "!async event request returned failure, sct = %x, " 12233c9168faSHans Rosenfeld "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct, 12243c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr, 12253c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_m); 12263c9168faSHans Rosenfeld 12273c9168faSHans Rosenfeld if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 12283c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) { 12293c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 12303c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, 12313c9168faSHans Rosenfeld DDI_SERVICE_LOST); 12323c9168faSHans Rosenfeld } 12333c9168faSHans Rosenfeld nvme_free_cmd(cmd); 12343c9168faSHans Rosenfeld return; 12353c9168faSHans Rosenfeld } 12363c9168faSHans Rosenfeld 12373c9168faSHans Rosenfeld 12383c9168faSHans Rosenfeld event.r = cmd->nc_cqe.cqe_dw0; 12393c9168faSHans Rosenfeld 12403c9168faSHans Rosenfeld /* Clear CQE and re-submit the async request. */ 12413c9168faSHans Rosenfeld bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t)); 12423c9168faSHans Rosenfeld ret = nvme_submit_cmd(nvme->n_adminq, cmd); 12433c9168faSHans Rosenfeld 12443c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 12453c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 12463c9168faSHans Rosenfeld "!failed to resubmit async event request"); 12473c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_async_resubmit_failed); 12483c9168faSHans Rosenfeld nvme_free_cmd(cmd); 12493c9168faSHans Rosenfeld } 12503c9168faSHans Rosenfeld 12513c9168faSHans Rosenfeld switch (event.b.ae_type) { 12523c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_ERROR: 12533c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_ERROR) { 12543c9168faSHans Rosenfeld error_log = (nvme_error_log_entry_t *) 12553c9168faSHans Rosenfeld nvme_get_logpage(nvme, event.b.ae_logpage); 12563c9168faSHans Rosenfeld } else { 12573c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 12583c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 12593c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 12603c9168faSHans Rosenfeld } 12613c9168faSHans Rosenfeld 12623c9168faSHans Rosenfeld switch (event.b.ae_info) { 12633c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_SQ: 12643c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 12653c9168faSHans Rosenfeld "invalid submission queue"); 12663c9168faSHans Rosenfeld return; 12673c9168faSHans Rosenfeld 12683c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_DBL: 12693c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 12703c9168faSHans Rosenfeld "invalid doorbell write value"); 12713c9168faSHans Rosenfeld return; 12723c9168faSHans Rosenfeld 12733c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_DIAGFAIL: 12743c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure"); 12753c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 12763c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 12773c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_diagfail_event); 12783c9168faSHans Rosenfeld break; 12793c9168faSHans Rosenfeld 12803c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_PERSISTENT: 12813c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!persistent internal " 12823c9168faSHans Rosenfeld "device error"); 12833c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 12843c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 12853c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_persistent_event); 12863c9168faSHans Rosenfeld break; 12873c9168faSHans Rosenfeld 12883c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_TRANSIENT: 12893c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!transient internal " 12903c9168faSHans Rosenfeld "device error"); 12913c9168faSHans Rosenfeld /* TODO: send ereport */ 12923c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_transient_event); 12933c9168faSHans Rosenfeld break; 12943c9168faSHans Rosenfeld 12953c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_FW_LOAD: 12963c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 12973c9168faSHans Rosenfeld "!firmware image load error"); 12983c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_fw_load_event); 12993c9168faSHans Rosenfeld break; 13003c9168faSHans Rosenfeld } 13013c9168faSHans Rosenfeld break; 13023c9168faSHans Rosenfeld 13033c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_HEALTH: 13043c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_HEALTH) { 13053c9168faSHans Rosenfeld health_log = (nvme_health_log_t *) 13063c9168faSHans Rosenfeld nvme_get_logpage(nvme, event.b.ae_logpage, -1); 13073c9168faSHans Rosenfeld } else { 13083c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 13093c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 13103c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 13113c9168faSHans Rosenfeld } 13123c9168faSHans Rosenfeld 13133c9168faSHans Rosenfeld switch (event.b.ae_info) { 13143c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_RELIABILITY: 13153c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13163c9168faSHans Rosenfeld "!device reliability compromised"); 13173c9168faSHans Rosenfeld /* TODO: send ereport */ 13183c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_reliability_event); 13193c9168faSHans Rosenfeld break; 13203c9168faSHans Rosenfeld 13213c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_TEMPERATURE: 13223c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13233c9168faSHans Rosenfeld "!temperature above threshold"); 13243c9168faSHans Rosenfeld /* TODO: send ereport */ 13253c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_temperature_event); 13263c9168faSHans Rosenfeld break; 13273c9168faSHans Rosenfeld 13283c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_SPARE: 13293c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13303c9168faSHans Rosenfeld "!spare space below threshold"); 13313c9168faSHans Rosenfeld /* TODO: send ereport */ 13323c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_spare_event); 13333c9168faSHans Rosenfeld break; 13343c9168faSHans Rosenfeld } 13353c9168faSHans Rosenfeld break; 13363c9168faSHans Rosenfeld 13373c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_VENDOR: 13383c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event " 13393c9168faSHans Rosenfeld "received, info = %x, logpage = %x", event.b.ae_info, 13403c9168faSHans Rosenfeld event.b.ae_logpage); 13413c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_vendor_event); 13423c9168faSHans Rosenfeld break; 13433c9168faSHans Rosenfeld 13443c9168faSHans Rosenfeld default: 13453c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, " 13463c9168faSHans Rosenfeld "type = %x, info = %x, logpage = %x", event.b.ae_type, 13473c9168faSHans Rosenfeld event.b.ae_info, event.b.ae_logpage); 13483c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_event); 13493c9168faSHans Rosenfeld break; 13503c9168faSHans Rosenfeld } 13513c9168faSHans Rosenfeld 13523c9168faSHans Rosenfeld if (error_log) 13533c9168faSHans Rosenfeld kmem_free(error_log, sizeof (nvme_error_log_entry_t) * 13543c9168faSHans Rosenfeld nvme->n_error_log_len); 13553c9168faSHans Rosenfeld 13563c9168faSHans Rosenfeld if (health_log) 13573c9168faSHans Rosenfeld kmem_free(health_log, sizeof (nvme_health_log_t)); 13583c9168faSHans Rosenfeld } 13593c9168faSHans Rosenfeld 13603c9168faSHans Rosenfeld static int 1361*e8ba2a38SHans Rosenfeld nvme_admin_cmd(nvme_cmd_t *cmd, int sec) 13623c9168faSHans Rosenfeld { 13633c9168faSHans Rosenfeld int ret; 13643c9168faSHans Rosenfeld 13653c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 13663c9168faSHans Rosenfeld ret = nvme_submit_cmd(cmd->nc_nvme->n_adminq, cmd); 13673c9168faSHans Rosenfeld 13683c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 13693c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 13703c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 13713c9168faSHans Rosenfeld "!nvme_submit_cmd failed"); 13723c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_admin_queue_full); 13733c9168faSHans Rosenfeld nvme_free_cmd(cmd); 13743c9168faSHans Rosenfeld return (DDI_FAILURE); 13753c9168faSHans Rosenfeld } 13763c9168faSHans Rosenfeld 1377*e8ba2a38SHans Rosenfeld if (nvme_wait_cmd(cmd, sec) == B_FALSE) { 13783c9168faSHans Rosenfeld /* 13793c9168faSHans Rosenfeld * The command timed out. An abort command was posted that 13803c9168faSHans Rosenfeld * will take care of the cleanup. 13813c9168faSHans Rosenfeld */ 13823c9168faSHans Rosenfeld return (DDI_FAILURE); 13833c9168faSHans Rosenfeld } 13843c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 13853c9168faSHans Rosenfeld 13863c9168faSHans Rosenfeld return (DDI_SUCCESS); 13873c9168faSHans Rosenfeld } 13883c9168faSHans Rosenfeld 13893c9168faSHans Rosenfeld static int 13903c9168faSHans Rosenfeld nvme_async_event(nvme_t *nvme) 13913c9168faSHans Rosenfeld { 13923c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 13933c9168faSHans Rosenfeld int ret; 13943c9168faSHans Rosenfeld 13953c9168faSHans Rosenfeld cmd->nc_sqid = 0; 13963c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT; 13973c9168faSHans Rosenfeld cmd->nc_callback = nvme_async_event_task; 13983c9168faSHans Rosenfeld 13993c9168faSHans Rosenfeld ret = nvme_submit_cmd(nvme->n_adminq, cmd); 14003c9168faSHans Rosenfeld 14013c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 14023c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 14033c9168faSHans Rosenfeld "!nvme_submit_cmd failed for ASYNCHRONOUS EVENT"); 14043c9168faSHans Rosenfeld nvme_free_cmd(cmd); 14053c9168faSHans Rosenfeld return (DDI_FAILURE); 14063c9168faSHans Rosenfeld } 14073c9168faSHans Rosenfeld 14083c9168faSHans Rosenfeld return (DDI_SUCCESS); 14093c9168faSHans Rosenfeld } 14103c9168faSHans Rosenfeld 14113c9168faSHans Rosenfeld static void * 14123c9168faSHans Rosenfeld nvme_get_logpage(nvme_t *nvme, uint8_t logpage, ...) 14133c9168faSHans Rosenfeld { 14143c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 14153c9168faSHans Rosenfeld void *buf = NULL; 141634c938c7SPete Shephard nvme_getlogpage_t getlogpage = { 0 }; 14173c9168faSHans Rosenfeld size_t bufsize; 14183c9168faSHans Rosenfeld va_list ap; 14193c9168faSHans Rosenfeld 14203c9168faSHans Rosenfeld va_start(ap, logpage); 14213c9168faSHans Rosenfeld 14223c9168faSHans Rosenfeld cmd->nc_sqid = 0; 14233c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 14243c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE; 14253c9168faSHans Rosenfeld 14263c9168faSHans Rosenfeld getlogpage.b.lp_lid = logpage; 14273c9168faSHans Rosenfeld 14283c9168faSHans Rosenfeld switch (logpage) { 14293c9168faSHans Rosenfeld case NVME_LOGPAGE_ERROR: 14303c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 14313c9168faSHans Rosenfeld bufsize = nvme->n_error_log_len * 14323c9168faSHans Rosenfeld sizeof (nvme_error_log_entry_t); 14333c9168faSHans Rosenfeld break; 14343c9168faSHans Rosenfeld 14353c9168faSHans Rosenfeld case NVME_LOGPAGE_HEALTH: 14363c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t); 14373c9168faSHans Rosenfeld bufsize = sizeof (nvme_health_log_t); 14383c9168faSHans Rosenfeld break; 14393c9168faSHans Rosenfeld 14403c9168faSHans Rosenfeld case NVME_LOGPAGE_FWSLOT: 14413c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 14423c9168faSHans Rosenfeld bufsize = sizeof (nvme_fwslot_log_t); 14433c9168faSHans Rosenfeld break; 14443c9168faSHans Rosenfeld 14453c9168faSHans Rosenfeld default: 14463c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown log page requested: %d", 14473c9168faSHans Rosenfeld logpage); 14483c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_logpage); 14493c9168faSHans Rosenfeld goto fail; 14503c9168faSHans Rosenfeld } 14513c9168faSHans Rosenfeld 14523c9168faSHans Rosenfeld va_end(ap); 14533c9168faSHans Rosenfeld 14549898bbdbSHans Rosenfeld getlogpage.b.lp_numd = bufsize / sizeof (uint32_t) - 1; 14553c9168faSHans Rosenfeld 14563c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = getlogpage.r; 14573c9168faSHans Rosenfeld 14583c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, getlogpage.b.lp_numd * sizeof (uint32_t), 14593c9168faSHans Rosenfeld DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 14603c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 14613c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for GET LOG PAGE"); 14623c9168faSHans Rosenfeld goto fail; 14633c9168faSHans Rosenfeld } 14643c9168faSHans Rosenfeld 14653c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 14663c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 14673c9168faSHans Rosenfeld "!too many DMA cookies for GET LOG PAGE"); 14683c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 14693c9168faSHans Rosenfeld goto fail; 14703c9168faSHans Rosenfeld } 14713c9168faSHans Rosenfeld 14723c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 14733c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 14743c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 14753c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 14763c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 14773c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 14783c9168faSHans Rosenfeld } 14793c9168faSHans Rosenfeld 1480*e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 14813c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 14823c9168faSHans Rosenfeld "!nvme_admin_cmd failed for GET LOG PAGE"); 14833c9168faSHans Rosenfeld return (NULL); 14843c9168faSHans Rosenfeld } 14853c9168faSHans Rosenfeld 14863c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 14873c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 14883c9168faSHans Rosenfeld "!GET LOG PAGE failed with sct = %x, sc = %x", 14893c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 14903c9168faSHans Rosenfeld goto fail; 14913c9168faSHans Rosenfeld } 14923c9168faSHans Rosenfeld 14933c9168faSHans Rosenfeld buf = kmem_alloc(bufsize, KM_SLEEP); 14943c9168faSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, buf, bufsize); 14953c9168faSHans Rosenfeld 14963c9168faSHans Rosenfeld fail: 14973c9168faSHans Rosenfeld nvme_free_cmd(cmd); 14983c9168faSHans Rosenfeld 14993c9168faSHans Rosenfeld return (buf); 15003c9168faSHans Rosenfeld } 15013c9168faSHans Rosenfeld 15023c9168faSHans Rosenfeld static void * 15033c9168faSHans Rosenfeld nvme_identify(nvme_t *nvme, uint32_t nsid) 15043c9168faSHans Rosenfeld { 15053c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 15063c9168faSHans Rosenfeld void *buf = NULL; 15073c9168faSHans Rosenfeld 15083c9168faSHans Rosenfeld cmd->nc_sqid = 0; 15093c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 15103c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY; 15113c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 15123c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL; 15133c9168faSHans Rosenfeld 15143c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ, 15153c9168faSHans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 15163c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15173c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for IDENTIFY"); 15183c9168faSHans Rosenfeld goto fail; 15193c9168faSHans Rosenfeld } 15203c9168faSHans Rosenfeld 15213c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 15223c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15233c9168faSHans Rosenfeld "!too many DMA cookies for IDENTIFY"); 15243c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 15253c9168faSHans Rosenfeld goto fail; 15263c9168faSHans Rosenfeld } 15273c9168faSHans Rosenfeld 15283c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 15293c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 15303c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 15313c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 15323c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 15333c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 15343c9168faSHans Rosenfeld } 15353c9168faSHans Rosenfeld 1536*e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 15373c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15383c9168faSHans Rosenfeld "!nvme_admin_cmd failed for IDENTIFY"); 15393c9168faSHans Rosenfeld return (NULL); 15403c9168faSHans Rosenfeld } 15413c9168faSHans Rosenfeld 15423c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 15433c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15443c9168faSHans Rosenfeld "!IDENTIFY failed with sct = %x, sc = %x", 15453c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 15463c9168faSHans Rosenfeld goto fail; 15473c9168faSHans Rosenfeld } 15483c9168faSHans Rosenfeld 15493c9168faSHans Rosenfeld buf = kmem_alloc(NVME_IDENTIFY_BUFSIZE, KM_SLEEP); 15503c9168faSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, buf, NVME_IDENTIFY_BUFSIZE); 15513c9168faSHans Rosenfeld 15523c9168faSHans Rosenfeld fail: 15533c9168faSHans Rosenfeld nvme_free_cmd(cmd); 15543c9168faSHans Rosenfeld 15553c9168faSHans Rosenfeld return (buf); 15563c9168faSHans Rosenfeld } 15573c9168faSHans Rosenfeld 15583c9168faSHans Rosenfeld static int 15593c9168faSHans Rosenfeld nvme_set_nqueues(nvme_t *nvme, uint16_t nqueues) 15603c9168faSHans Rosenfeld { 15613c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 15623c9168faSHans Rosenfeld nvme_nqueue_t nq = { 0 }; 15633c9168faSHans Rosenfeld 156434c938c7SPete Shephard nq.b.nq_nsq = nq.b.nq_ncq = nqueues - 1; 15653c9168faSHans Rosenfeld 15663c9168faSHans Rosenfeld cmd->nc_sqid = 0; 15673c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 15683c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES; 15693c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = NVME_FEAT_NQUEUES; 15703c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = nq.r; 15713c9168faSHans Rosenfeld 1572*e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 15733c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15743c9168faSHans Rosenfeld "!nvme_admin_cmd failed for SET FEATURES (NQUEUES)"); 15753c9168faSHans Rosenfeld return (0); 15763c9168faSHans Rosenfeld } 15773c9168faSHans Rosenfeld 15783c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 15793c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15803c9168faSHans Rosenfeld "!SET FEATURES (NQUEUES) failed with sct = %x, sc = %x", 15813c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 15823c9168faSHans Rosenfeld nvme_free_cmd(cmd); 15833c9168faSHans Rosenfeld return (0); 15843c9168faSHans Rosenfeld } 15853c9168faSHans Rosenfeld 15863c9168faSHans Rosenfeld nq.r = cmd->nc_cqe.cqe_dw0; 15873c9168faSHans Rosenfeld nvme_free_cmd(cmd); 15883c9168faSHans Rosenfeld 15893c9168faSHans Rosenfeld /* 15903c9168faSHans Rosenfeld * Always use the same number of submission and completion queues, and 15913c9168faSHans Rosenfeld * never use more than the requested number of queues. 15923c9168faSHans Rosenfeld */ 159334c938c7SPete Shephard return (MIN(nqueues, MIN(nq.b.nq_nsq, nq.b.nq_ncq) + 1)); 15943c9168faSHans Rosenfeld } 15953c9168faSHans Rosenfeld 15963c9168faSHans Rosenfeld static int 15973c9168faSHans Rosenfeld nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx) 15983c9168faSHans Rosenfeld { 15993c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 16003c9168faSHans Rosenfeld nvme_create_queue_dw10_t dw10 = { 0 }; 16013c9168faSHans Rosenfeld nvme_create_cq_dw11_t c_dw11 = { 0 }; 16023c9168faSHans Rosenfeld nvme_create_sq_dw11_t s_dw11 = { 0 }; 16033c9168faSHans Rosenfeld 16043c9168faSHans Rosenfeld dw10.b.q_qid = idx; 16053c9168faSHans Rosenfeld dw10.b.q_qsize = qp->nq_nentry - 1; 16063c9168faSHans Rosenfeld 16073c9168faSHans Rosenfeld c_dw11.b.cq_pc = 1; 16083c9168faSHans Rosenfeld c_dw11.b.cq_ien = 1; 16093c9168faSHans Rosenfeld c_dw11.b.cq_iv = idx % nvme->n_intr_cnt; 16103c9168faSHans Rosenfeld 16113c9168faSHans Rosenfeld cmd->nc_sqid = 0; 16123c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 16133c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE; 16143c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 16153c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = c_dw11.r; 16163c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_cqdma->nd_cookie.dmac_laddress; 16173c9168faSHans Rosenfeld 1618*e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 16193c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16203c9168faSHans Rosenfeld "!nvme_admin_cmd failed for CREATE CQUEUE"); 16213c9168faSHans Rosenfeld return (DDI_FAILURE); 16223c9168faSHans Rosenfeld } 16233c9168faSHans Rosenfeld 16243c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 16253c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16263c9168faSHans Rosenfeld "!CREATE CQUEUE failed with sct = %x, sc = %x", 16273c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 16283c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16293c9168faSHans Rosenfeld return (DDI_FAILURE); 16303c9168faSHans Rosenfeld } 16313c9168faSHans Rosenfeld 16323c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16333c9168faSHans Rosenfeld 16343c9168faSHans Rosenfeld s_dw11.b.sq_pc = 1; 16353c9168faSHans Rosenfeld s_dw11.b.sq_cqid = idx; 16363c9168faSHans Rosenfeld 16373c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 16383c9168faSHans Rosenfeld cmd->nc_sqid = 0; 16393c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 16403c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE; 16413c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 16423c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = s_dw11.r; 16433c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress; 16443c9168faSHans Rosenfeld 1645*e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 16463c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16473c9168faSHans Rosenfeld "!nvme_admin_cmd failed for CREATE SQUEUE"); 16483c9168faSHans Rosenfeld return (DDI_FAILURE); 16493c9168faSHans Rosenfeld } 16503c9168faSHans Rosenfeld 16513c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 16523c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16533c9168faSHans Rosenfeld "!CREATE SQUEUE failed with sct = %x, sc = %x", 16543c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 16553c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16563c9168faSHans Rosenfeld return (DDI_FAILURE); 16573c9168faSHans Rosenfeld } 16583c9168faSHans Rosenfeld 16593c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16603c9168faSHans Rosenfeld 16613c9168faSHans Rosenfeld return (DDI_SUCCESS); 16623c9168faSHans Rosenfeld } 16633c9168faSHans Rosenfeld 16643c9168faSHans Rosenfeld static boolean_t 16653c9168faSHans Rosenfeld nvme_reset(nvme_t *nvme, boolean_t quiesce) 16663c9168faSHans Rosenfeld { 16673c9168faSHans Rosenfeld nvme_reg_csts_t csts; 16683c9168faSHans Rosenfeld int i; 16693c9168faSHans Rosenfeld 16703c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 16713c9168faSHans Rosenfeld 16723c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 16733c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) { 16743c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 16753c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 16763c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 16773c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) 16783c9168faSHans Rosenfeld break; 16793c9168faSHans Rosenfeld 16803c9168faSHans Rosenfeld if (quiesce) 16813c9168faSHans Rosenfeld drv_usecwait(50000); 16823c9168faSHans Rosenfeld else 16833c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 16843c9168faSHans Rosenfeld } 16853c9168faSHans Rosenfeld } 16863c9168faSHans Rosenfeld 16873c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, 0); 16883c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ASQ, 0); 16893c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ACQ, 0); 16903c9168faSHans Rosenfeld 16913c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 16923c9168faSHans Rosenfeld return (csts.b.csts_rdy == 0 ? B_TRUE : B_FALSE); 16933c9168faSHans Rosenfeld } 16943c9168faSHans Rosenfeld 16953c9168faSHans Rosenfeld static void 16963c9168faSHans Rosenfeld nvme_shutdown(nvme_t *nvme, int mode, boolean_t quiesce) 16973c9168faSHans Rosenfeld { 16983c9168faSHans Rosenfeld nvme_reg_cc_t cc; 16993c9168faSHans Rosenfeld nvme_reg_csts_t csts; 17003c9168faSHans Rosenfeld int i; 17013c9168faSHans Rosenfeld 17023c9168faSHans Rosenfeld ASSERT(mode == NVME_CC_SHN_NORMAL || mode == NVME_CC_SHN_ABRUPT); 17033c9168faSHans Rosenfeld 17043c9168faSHans Rosenfeld cc.r = nvme_get32(nvme, NVME_REG_CC); 17053c9168faSHans Rosenfeld cc.b.cc_shn = mode & 0x3; 17063c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 17073c9168faSHans Rosenfeld 17083c9168faSHans Rosenfeld for (i = 0; i != 10; i++) { 17093c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 17103c9168faSHans Rosenfeld if (csts.b.csts_shst == NVME_CSTS_SHN_COMPLETE) 17113c9168faSHans Rosenfeld break; 17123c9168faSHans Rosenfeld 17133c9168faSHans Rosenfeld if (quiesce) 17143c9168faSHans Rosenfeld drv_usecwait(100000); 17153c9168faSHans Rosenfeld else 17163c9168faSHans Rosenfeld delay(drv_usectohz(100000)); 17173c9168faSHans Rosenfeld } 17183c9168faSHans Rosenfeld } 17193c9168faSHans Rosenfeld 17203c9168faSHans Rosenfeld 17213c9168faSHans Rosenfeld static void 17223c9168faSHans Rosenfeld nvme_prepare_devid(nvme_t *nvme, uint32_t nsid) 17233c9168faSHans Rosenfeld { 17243c9168faSHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 17253c9168faSHans Rosenfeld char serial[sizeof (nvme->n_idctl->id_serial) + 1]; 17263c9168faSHans Rosenfeld 17273c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 17283c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_serial, serial, 17293c9168faSHans Rosenfeld sizeof (nvme->n_idctl->id_serial)); 17303c9168faSHans Rosenfeld 17313c9168faSHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 17323c9168faSHans Rosenfeld serial[sizeof (nvme->n_idctl->id_serial)] = '\0'; 17333c9168faSHans Rosenfeld 17343c9168faSHans Rosenfeld (void) snprintf(nvme->n_ns[nsid - 1].ns_devid, 17353c9168faSHans Rosenfeld sizeof (nvme->n_ns[0].ns_devid), "%4X-%s-%s-%X", 17363c9168faSHans Rosenfeld nvme->n_idctl->id_vid, model, serial, nsid); 17373c9168faSHans Rosenfeld } 17383c9168faSHans Rosenfeld 17393c9168faSHans Rosenfeld static int 17403c9168faSHans Rosenfeld nvme_init(nvme_t *nvme) 17413c9168faSHans Rosenfeld { 17423c9168faSHans Rosenfeld nvme_reg_cc_t cc = { 0 }; 17433c9168faSHans Rosenfeld nvme_reg_aqa_t aqa = { 0 }; 17443c9168faSHans Rosenfeld nvme_reg_asq_t asq = { 0 }; 17453c9168faSHans Rosenfeld nvme_reg_acq_t acq = { 0 }; 17463c9168faSHans Rosenfeld nvme_reg_cap_t cap; 17473c9168faSHans Rosenfeld nvme_reg_vs_t vs; 17483c9168faSHans Rosenfeld nvme_reg_csts_t csts; 17493c9168faSHans Rosenfeld int i = 0; 17503c9168faSHans Rosenfeld int nqueues; 1751510a6847SHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 1752510a6847SHans Rosenfeld char *vendor, *product; 17533c9168faSHans Rosenfeld 17543c9168faSHans Rosenfeld /* Check controller version */ 17553c9168faSHans Rosenfeld vs.r = nvme_get32(nvme, NVME_REG_VS); 17563c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d", 17573c9168faSHans Rosenfeld vs.b.vs_mjr, vs.b.vs_mnr); 17583c9168faSHans Rosenfeld 17592797e2b9SHans Rosenfeld if (nvme_version_major < vs.b.vs_mjr || 17602797e2b9SHans Rosenfeld (nvme_version_major == vs.b.vs_mjr && 17612797e2b9SHans Rosenfeld nvme_version_minor < vs.b.vs_mnr)) { 17623c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.%d", 17633c9168faSHans Rosenfeld nvme_version_major, nvme_version_minor); 17643c9168faSHans Rosenfeld if (nvme->n_strict_version) 17653c9168faSHans Rosenfeld goto fail; 17663c9168faSHans Rosenfeld } 17673c9168faSHans Rosenfeld 17683c9168faSHans Rosenfeld /* retrieve controller configuration */ 17693c9168faSHans Rosenfeld cap.r = nvme_get64(nvme, NVME_REG_CAP); 17703c9168faSHans Rosenfeld 17713c9168faSHans Rosenfeld if ((cap.b.cap_css & NVME_CAP_CSS_NVM) == 0) { 17723c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17733c9168faSHans Rosenfeld "!NVM command set not supported by hardware"); 17743c9168faSHans Rosenfeld goto fail; 17753c9168faSHans Rosenfeld } 17763c9168faSHans Rosenfeld 17773c9168faSHans Rosenfeld nvme->n_nssr_supported = cap.b.cap_nssrs; 17783c9168faSHans Rosenfeld nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd; 17793c9168faSHans Rosenfeld nvme->n_timeout = cap.b.cap_to; 17803c9168faSHans Rosenfeld nvme->n_arbitration_mechanisms = cap.b.cap_ams; 17813c9168faSHans Rosenfeld nvme->n_cont_queues_reqd = cap.b.cap_cqr; 17823c9168faSHans Rosenfeld nvme->n_max_queue_entries = cap.b.cap_mqes + 1; 17833c9168faSHans Rosenfeld 17843c9168faSHans Rosenfeld /* 17853c9168faSHans Rosenfeld * The MPSMIN and MPSMAX fields in the CAP register use 0 to specify 17863c9168faSHans Rosenfeld * the base page size of 4k (1<<12), so add 12 here to get the real 17873c9168faSHans Rosenfeld * page size value. 17883c9168faSHans Rosenfeld */ 17893c9168faSHans Rosenfeld nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT), 17903c9168faSHans Rosenfeld cap.b.cap_mpsmax + 12); 17913c9168faSHans Rosenfeld nvme->n_pagesize = 1UL << (nvme->n_pageshift); 17923c9168faSHans Rosenfeld 17933c9168faSHans Rosenfeld /* 17943c9168faSHans Rosenfeld * Set up Queue DMA to transfer at least 1 page-aligned page at a time. 17953c9168faSHans Rosenfeld */ 17963c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize; 17973c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 17983c9168faSHans Rosenfeld 17993c9168faSHans Rosenfeld /* 18003c9168faSHans Rosenfeld * Set up PRP DMA to transfer 1 page-aligned page at a time. 18013c9168faSHans Rosenfeld * Maxxfer may be increased after we identified the controller limits. 18023c9168faSHans Rosenfeld */ 18033c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize; 18043c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 18053c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize; 18062f95345bSYouzhong Yang nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1; 18073c9168faSHans Rosenfeld 18083c9168faSHans Rosenfeld /* 18093c9168faSHans Rosenfeld * Reset controller if it's still in ready state. 18103c9168faSHans Rosenfeld */ 18113c9168faSHans Rosenfeld if (nvme_reset(nvme, B_FALSE) == B_FALSE) { 18123c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller"); 18133c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 18143c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 18153c9168faSHans Rosenfeld goto fail; 18163c9168faSHans Rosenfeld } 18173c9168faSHans Rosenfeld 18183c9168faSHans Rosenfeld /* 18193c9168faSHans Rosenfeld * Create the admin queue pair. 18203c9168faSHans Rosenfeld */ 18213c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0) 18223c9168faSHans Rosenfeld != DDI_SUCCESS) { 18233c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 18243c9168faSHans Rosenfeld "!unable to allocate admin qpair"); 18253c9168faSHans Rosenfeld goto fail; 18263c9168faSHans Rosenfeld } 18273c9168faSHans Rosenfeld nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP); 18283c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 18293c9168faSHans Rosenfeld 18303c9168faSHans Rosenfeld nvme->n_progress |= NVME_ADMIN_QUEUE; 18313c9168faSHans Rosenfeld 18323c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 18333c9168faSHans Rosenfeld "admin-queue-len", nvme->n_admin_queue_len); 18343c9168faSHans Rosenfeld 18353c9168faSHans Rosenfeld aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1; 18363c9168faSHans Rosenfeld asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress; 18373c9168faSHans Rosenfeld acq = nvme->n_adminq->nq_cqdma->nd_cookie.dmac_laddress; 18383c9168faSHans Rosenfeld 18393c9168faSHans Rosenfeld ASSERT((asq & (nvme->n_pagesize - 1)) == 0); 18403c9168faSHans Rosenfeld ASSERT((acq & (nvme->n_pagesize - 1)) == 0); 18413c9168faSHans Rosenfeld 18423c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, aqa.r); 18433c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ASQ, asq); 18443c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ACQ, acq); 18453c9168faSHans Rosenfeld 184634c938c7SPete Shephard cc.b.cc_ams = 0; /* use Round-Robin arbitration */ 184734c938c7SPete Shephard cc.b.cc_css = 0; /* use NVM command set */ 18483c9168faSHans Rosenfeld cc.b.cc_mps = nvme->n_pageshift - 12; 184934c938c7SPete Shephard cc.b.cc_shn = 0; /* no shutdown in progress */ 185034c938c7SPete Shephard cc.b.cc_en = 1; /* enable controller */ 185134c938c7SPete Shephard cc.b.cc_iosqes = 6; /* submission queue entry is 2^6 bytes long */ 185234c938c7SPete Shephard cc.b.cc_iocqes = 4; /* completion queue entry is 2^4 bytes long */ 18533c9168faSHans Rosenfeld 18543c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 18553c9168faSHans Rosenfeld 18563c9168faSHans Rosenfeld /* 18573c9168faSHans Rosenfeld * Wait for the controller to become ready. 18583c9168faSHans Rosenfeld */ 18593c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 18603c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 18613c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 18623c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 18633c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 18643c9168faSHans Rosenfeld 18653c9168faSHans Rosenfeld if (csts.b.csts_cfs == 1) { 18663c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 18673c9168faSHans Rosenfeld "!controller fatal status at init"); 18683c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, 18693c9168faSHans Rosenfeld DDI_SERVICE_LOST); 18703c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 18713c9168faSHans Rosenfeld goto fail; 18723c9168faSHans Rosenfeld } 18733c9168faSHans Rosenfeld 18743c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) 18753c9168faSHans Rosenfeld break; 18763c9168faSHans Rosenfeld } 18773c9168faSHans Rosenfeld } 18783c9168faSHans Rosenfeld 18793c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 18803c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!controller not ready"); 18813c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 18823c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 18833c9168faSHans Rosenfeld goto fail; 18843c9168faSHans Rosenfeld } 18853c9168faSHans Rosenfeld 18863c9168faSHans Rosenfeld /* 18873c9168faSHans Rosenfeld * Assume an abort command limit of 1. We'll destroy and re-init 18883c9168faSHans Rosenfeld * that later when we know the true abort command limit. 18893c9168faSHans Rosenfeld */ 18903c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL); 18913c9168faSHans Rosenfeld 1892bf26ea4bSHans Rosenfeld /* 1893bf26ea4bSHans Rosenfeld * Setup initial interrupt for admin queue. 1894bf26ea4bSHans Rosenfeld */ 1895bf26ea4bSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1) 1896bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 1897bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1) 1898bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 1899bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1) 1900bf26ea4bSHans Rosenfeld != DDI_SUCCESS)) { 1901bf26ea4bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 1902bf26ea4bSHans Rosenfeld "!failed to setup initial interrupt"); 1903bf26ea4bSHans Rosenfeld goto fail; 1904bf26ea4bSHans Rosenfeld } 1905bf26ea4bSHans Rosenfeld 19063c9168faSHans Rosenfeld /* 19073c9168faSHans Rosenfeld * Post an asynchronous event command to catch errors. 19083c9168faSHans Rosenfeld */ 19093c9168faSHans Rosenfeld if (nvme_async_event(nvme) != DDI_SUCCESS) { 19103c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19113c9168faSHans Rosenfeld "!failed to post async event"); 19123c9168faSHans Rosenfeld goto fail; 19133c9168faSHans Rosenfeld } 19143c9168faSHans Rosenfeld 19153c9168faSHans Rosenfeld /* 19163c9168faSHans Rosenfeld * Identify Controller 19173c9168faSHans Rosenfeld */ 19183c9168faSHans Rosenfeld nvme->n_idctl = nvme_identify(nvme, 0); 19193c9168faSHans Rosenfeld if (nvme->n_idctl == NULL) { 19203c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19213c9168faSHans Rosenfeld "!failed to identify controller"); 19223c9168faSHans Rosenfeld goto fail; 19233c9168faSHans Rosenfeld } 19243c9168faSHans Rosenfeld 1925510a6847SHans Rosenfeld /* 1926510a6847SHans Rosenfeld * Get Vendor & Product ID 1927510a6847SHans Rosenfeld */ 1928510a6847SHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 1929510a6847SHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 1930510a6847SHans Rosenfeld sata_split_model(model, &vendor, &product); 1931510a6847SHans Rosenfeld 1932510a6847SHans Rosenfeld if (vendor == NULL) 1933510a6847SHans Rosenfeld nvme->n_vendor = strdup("NVMe"); 1934510a6847SHans Rosenfeld else 1935510a6847SHans Rosenfeld nvme->n_vendor = strdup(vendor); 1936510a6847SHans Rosenfeld 1937510a6847SHans Rosenfeld nvme->n_product = strdup(product); 1938510a6847SHans Rosenfeld 19393c9168faSHans Rosenfeld /* 19403c9168faSHans Rosenfeld * Get controller limits. 19413c9168faSHans Rosenfeld */ 19423c9168faSHans Rosenfeld nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT, 19433c9168faSHans Rosenfeld MIN(nvme->n_admin_queue_len / 10, 19443c9168faSHans Rosenfeld MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit))); 19453c9168faSHans Rosenfeld 19463c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 19473c9168faSHans Rosenfeld "async-event-limit", nvme->n_async_event_limit); 19483c9168faSHans Rosenfeld 19493c9168faSHans Rosenfeld nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1; 19503c9168faSHans Rosenfeld 19516afc9eb2SHans Rosenfeld /* 19526afc9eb2SHans Rosenfeld * Reinitialize the semaphore with the true abort command limit 19536afc9eb2SHans Rosenfeld * supported by the hardware. It's not necessary to disable interrupts 19546afc9eb2SHans Rosenfeld * as only command aborts use the semaphore, and no commands are 19556afc9eb2SHans Rosenfeld * executed or aborted while we're here. 19566afc9eb2SHans Rosenfeld */ 19573c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 19583c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL, 19593c9168faSHans Rosenfeld SEMA_DRIVER, NULL); 19603c9168faSHans Rosenfeld 19613c9168faSHans Rosenfeld nvme->n_progress |= NVME_CTRL_LIMITS; 19623c9168faSHans Rosenfeld 19633c9168faSHans Rosenfeld if (nvme->n_idctl->id_mdts == 0) 19643c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536; 19653c9168faSHans Rosenfeld else 19663c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = 19673c9168faSHans Rosenfeld 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts); 19683c9168faSHans Rosenfeld 19693c9168faSHans Rosenfeld nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1; 19703c9168faSHans Rosenfeld 19713c9168faSHans Rosenfeld /* 19723c9168faSHans Rosenfeld * Limit n_max_data_transfer_size to what we can handle in one PRP. 19733c9168faSHans Rosenfeld * Chained PRPs are currently unsupported. 19743c9168faSHans Rosenfeld * 19753c9168faSHans Rosenfeld * This is a no-op on hardware which doesn't support a transfer size 19763c9168faSHans Rosenfeld * big enough to require chained PRPs. 19773c9168faSHans Rosenfeld */ 19783c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size, 19793c9168faSHans Rosenfeld (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize)); 19803c9168faSHans Rosenfeld 19813c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size; 19823c9168faSHans Rosenfeld 19833c9168faSHans Rosenfeld /* 19843c9168faSHans Rosenfeld * Make sure the minimum/maximum queue entry sizes are not 19853c9168faSHans Rosenfeld * larger/smaller than the default. 19863c9168faSHans Rosenfeld */ 19873c9168faSHans Rosenfeld 19883c9168faSHans Rosenfeld if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) || 19893c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) || 19903c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) || 19913c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t))) 19923c9168faSHans Rosenfeld goto fail; 19933c9168faSHans Rosenfeld 19943c9168faSHans Rosenfeld /* 19953c9168faSHans Rosenfeld * Check for the presence of a Volatile Write Cache. If present, 19963c9168faSHans Rosenfeld * enable it by default. 19973c9168faSHans Rosenfeld */ 19983c9168faSHans Rosenfeld if (nvme->n_idctl->id_vwc.vwc_present == 0) { 19993c9168faSHans Rosenfeld nvme->n_volatile_write_cache_enabled = B_FALSE; 20003c9168faSHans Rosenfeld nvme_bd_ops.o_sync_cache = NULL; 20013c9168faSHans Rosenfeld } else { 20023c9168faSHans Rosenfeld /* 20033c9168faSHans Rosenfeld * TODO: send SET FEATURES to enable VWC 20043c9168faSHans Rosenfeld * (have no hardware to test this) 20053c9168faSHans Rosenfeld */ 20063c9168faSHans Rosenfeld nvme->n_volatile_write_cache_enabled = B_FALSE; 20073c9168faSHans Rosenfeld nvme_bd_ops.o_sync_cache = NULL; 20083c9168faSHans Rosenfeld } 20093c9168faSHans Rosenfeld 20103c9168faSHans Rosenfeld /* 20113c9168faSHans Rosenfeld * Grab a copy of all mandatory log pages. 20123c9168faSHans Rosenfeld * 20133c9168faSHans Rosenfeld * TODO: should go away once user space tool exists to print logs 20143c9168faSHans Rosenfeld */ 20153c9168faSHans Rosenfeld nvme->n_error_log = (nvme_error_log_entry_t *) 20163c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_ERROR); 20173c9168faSHans Rosenfeld nvme->n_health_log = (nvme_health_log_t *) 20183c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_HEALTH, -1); 20193c9168faSHans Rosenfeld nvme->n_fwslot_log = (nvme_fwslot_log_t *) 20203c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_FWSLOT); 20213c9168faSHans Rosenfeld 20223c9168faSHans Rosenfeld /* 20233c9168faSHans Rosenfeld * Identify Namespaces 20243c9168faSHans Rosenfeld */ 20253c9168faSHans Rosenfeld nvme->n_namespace_count = nvme->n_idctl->id_nn; 20263c9168faSHans Rosenfeld nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) * 20273c9168faSHans Rosenfeld nvme->n_namespace_count, KM_SLEEP); 20283c9168faSHans Rosenfeld 20293c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 20303c9168faSHans Rosenfeld nvme_identify_nsid_t *idns; 20313c9168faSHans Rosenfeld int last_rp; 20323c9168faSHans Rosenfeld 20333c9168faSHans Rosenfeld nvme->n_ns[i].ns_nvme = nvme; 20343c9168faSHans Rosenfeld nvme->n_ns[i].ns_idns = idns = nvme_identify(nvme, i + 1); 20353c9168faSHans Rosenfeld 20363c9168faSHans Rosenfeld if (idns == NULL) { 20373c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20383c9168faSHans Rosenfeld "!failed to identify namespace %d", i + 1); 20393c9168faSHans Rosenfeld goto fail; 20403c9168faSHans Rosenfeld } 20413c9168faSHans Rosenfeld 20423c9168faSHans Rosenfeld nvme->n_ns[i].ns_id = i + 1; 20433c9168faSHans Rosenfeld nvme->n_ns[i].ns_block_count = idns->id_nsize; 20443c9168faSHans Rosenfeld nvme->n_ns[i].ns_block_size = 20453c9168faSHans Rosenfeld 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads; 20463c9168faSHans Rosenfeld nvme->n_ns[i].ns_best_block_size = nvme->n_ns[i].ns_block_size; 20473c9168faSHans Rosenfeld 20483c9168faSHans Rosenfeld nvme_prepare_devid(nvme, nvme->n_ns[i].ns_id); 20493c9168faSHans Rosenfeld 20503c9168faSHans Rosenfeld /* 20513c9168faSHans Rosenfeld * Find the LBA format with no metadata and the best relative 20523c9168faSHans Rosenfeld * performance. A value of 3 means "degraded", 0 is best. 20533c9168faSHans Rosenfeld */ 20543c9168faSHans Rosenfeld last_rp = 3; 20550d140ff9SHans Rosenfeld for (int j = 0; j <= idns->id_nlbaf; j++) { 20563c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_lbads == 0) 20573c9168faSHans Rosenfeld break; 20583c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_ms != 0) 20593c9168faSHans Rosenfeld continue; 20603c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_rp >= last_rp) 20613c9168faSHans Rosenfeld continue; 20623c9168faSHans Rosenfeld last_rp = idns->id_lbaf[j].lbaf_rp; 20633c9168faSHans Rosenfeld nvme->n_ns[i].ns_best_block_size = 20643c9168faSHans Rosenfeld 1 << idns->id_lbaf[j].lbaf_lbads; 20653c9168faSHans Rosenfeld } 20663c9168faSHans Rosenfeld 20673c9168faSHans Rosenfeld /* 20683c9168faSHans Rosenfeld * We currently don't support namespaces that use either: 20693c9168faSHans Rosenfeld * - thin provisioning 20703c9168faSHans Rosenfeld * - extended LBAs 20713c9168faSHans Rosenfeld * - protection information 20723c9168faSHans Rosenfeld */ 20733c9168faSHans Rosenfeld if (idns->id_nsfeat.f_thin || 20743c9168faSHans Rosenfeld idns->id_flbas.lba_extlba || 20753c9168faSHans Rosenfeld idns->id_dps.dp_pinfo) { 20763c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20773c9168faSHans Rosenfeld "!ignoring namespace %d, unsupported features: " 20783c9168faSHans Rosenfeld "thin = %d, extlba = %d, pinfo = %d", i + 1, 20793c9168faSHans Rosenfeld idns->id_nsfeat.f_thin, idns->id_flbas.lba_extlba, 20803c9168faSHans Rosenfeld idns->id_dps.dp_pinfo); 20813c9168faSHans Rosenfeld nvme->n_ns[i].ns_ignore = B_TRUE; 20823c9168faSHans Rosenfeld } 20833c9168faSHans Rosenfeld } 20843c9168faSHans Rosenfeld 20853c9168faSHans Rosenfeld /* 20863c9168faSHans Rosenfeld * Try to set up MSI/MSI-X interrupts. 20873c9168faSHans Rosenfeld */ 20883c9168faSHans Rosenfeld if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX)) 20893c9168faSHans Rosenfeld != 0) { 20903c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 20913c9168faSHans Rosenfeld 20923c9168faSHans Rosenfeld nqueues = MIN(UINT16_MAX, ncpus); 20933c9168faSHans Rosenfeld 20943c9168faSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 20953c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS) && 20963c9168faSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 20973c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS)) { 20983c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20993c9168faSHans Rosenfeld "!failed to setup MSI/MSI-X interrupts"); 21003c9168faSHans Rosenfeld goto fail; 21013c9168faSHans Rosenfeld } 21023c9168faSHans Rosenfeld } 21033c9168faSHans Rosenfeld 21043c9168faSHans Rosenfeld nqueues = nvme->n_intr_cnt; 21053c9168faSHans Rosenfeld 21063c9168faSHans Rosenfeld /* 21073c9168faSHans Rosenfeld * Create I/O queue pairs. 21083c9168faSHans Rosenfeld */ 21093c9168faSHans Rosenfeld nvme->n_ioq_count = nvme_set_nqueues(nvme, nqueues); 21103c9168faSHans Rosenfeld if (nvme->n_ioq_count == 0) { 21113c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21123c9168faSHans Rosenfeld "!failed to set number of I/O queues to %d", nqueues); 21133c9168faSHans Rosenfeld goto fail; 21143c9168faSHans Rosenfeld } 21153c9168faSHans Rosenfeld 21163c9168faSHans Rosenfeld /* 21173c9168faSHans Rosenfeld * Reallocate I/O queue array 21183c9168faSHans Rosenfeld */ 21193c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *)); 21203c9168faSHans Rosenfeld nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) * 21213c9168faSHans Rosenfeld (nvme->n_ioq_count + 1), KM_SLEEP); 21223c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 21233c9168faSHans Rosenfeld 21243c9168faSHans Rosenfeld /* 21253c9168faSHans Rosenfeld * If we got less queues than we asked for we might as well give 21263c9168faSHans Rosenfeld * some of the interrupt vectors back to the system. 21273c9168faSHans Rosenfeld */ 21283c9168faSHans Rosenfeld if (nvme->n_ioq_count < nqueues) { 21293c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 21303c9168faSHans Rosenfeld 213134c938c7SPete Shephard if (nvme_setup_interrupts(nvme, nvme->n_intr_type, 213234c938c7SPete Shephard nvme->n_ioq_count) != DDI_SUCCESS) { 21333c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21343c9168faSHans Rosenfeld "!failed to reduce number of interrupts"); 21353c9168faSHans Rosenfeld goto fail; 21363c9168faSHans Rosenfeld } 21373c9168faSHans Rosenfeld } 21383c9168faSHans Rosenfeld 21393c9168faSHans Rosenfeld /* 21403c9168faSHans Rosenfeld * Alloc & register I/O queue pairs 21413c9168faSHans Rosenfeld */ 21423c9168faSHans Rosenfeld nvme->n_io_queue_len = 21433c9168faSHans Rosenfeld MIN(nvme->n_io_queue_len, nvme->n_max_queue_entries); 21443c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-queue-len", 21453c9168faSHans Rosenfeld nvme->n_io_queue_len); 21463c9168faSHans Rosenfeld 21473c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 21483c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_io_queue_len, 21493c9168faSHans Rosenfeld &nvme->n_ioq[i], i) != DDI_SUCCESS) { 21503c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21513c9168faSHans Rosenfeld "!unable to allocate I/O qpair %d", i); 21523c9168faSHans Rosenfeld goto fail; 21533c9168faSHans Rosenfeld } 21543c9168faSHans Rosenfeld 21553c9168faSHans Rosenfeld if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i) 21563c9168faSHans Rosenfeld != DDI_SUCCESS) { 21573c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21583c9168faSHans Rosenfeld "!unable to create I/O qpair %d", i); 21593c9168faSHans Rosenfeld goto fail; 21603c9168faSHans Rosenfeld } 21613c9168faSHans Rosenfeld } 21623c9168faSHans Rosenfeld 21633c9168faSHans Rosenfeld /* 21643c9168faSHans Rosenfeld * Post more asynchronous events commands to reduce event reporting 21653c9168faSHans Rosenfeld * latency as suggested by the spec. 21663c9168faSHans Rosenfeld */ 21673c9168faSHans Rosenfeld for (i = 1; i != nvme->n_async_event_limit; i++) { 21683c9168faSHans Rosenfeld if (nvme_async_event(nvme) != DDI_SUCCESS) { 21693c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21703c9168faSHans Rosenfeld "!failed to post async event %d", i); 21713c9168faSHans Rosenfeld goto fail; 21723c9168faSHans Rosenfeld } 21733c9168faSHans Rosenfeld } 21743c9168faSHans Rosenfeld 21753c9168faSHans Rosenfeld return (DDI_SUCCESS); 21763c9168faSHans Rosenfeld 21773c9168faSHans Rosenfeld fail: 21783c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 21793c9168faSHans Rosenfeld return (DDI_FAILURE); 21803c9168faSHans Rosenfeld } 21813c9168faSHans Rosenfeld 21823c9168faSHans Rosenfeld static uint_t 21833c9168faSHans Rosenfeld nvme_intr(caddr_t arg1, caddr_t arg2) 21843c9168faSHans Rosenfeld { 21853c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 21863c9168faSHans Rosenfeld nvme_t *nvme = (nvme_t *)arg1; 21873c9168faSHans Rosenfeld int inum = (int)(uintptr_t)arg2; 2188bf26ea4bSHans Rosenfeld int ccnt = 0; 21893c9168faSHans Rosenfeld int qnum; 21903c9168faSHans Rosenfeld nvme_cmd_t *cmd; 21913c9168faSHans Rosenfeld 21923c9168faSHans Rosenfeld if (inum >= nvme->n_intr_cnt) 21933c9168faSHans Rosenfeld return (DDI_INTR_UNCLAIMED); 21943c9168faSHans Rosenfeld 21953c9168faSHans Rosenfeld /* 21963c9168faSHans Rosenfeld * The interrupt vector a queue uses is calculated as queue_idx % 21973c9168faSHans Rosenfeld * intr_cnt in nvme_create_io_qpair(). Iterate through the queue array 21983c9168faSHans Rosenfeld * in steps of n_intr_cnt to process all queues using this vector. 21993c9168faSHans Rosenfeld */ 22003c9168faSHans Rosenfeld for (qnum = inum; 22013c9168faSHans Rosenfeld qnum < nvme->n_ioq_count + 1 && nvme->n_ioq[qnum] != NULL; 22023c9168faSHans Rosenfeld qnum += nvme->n_intr_cnt) { 22033c9168faSHans Rosenfeld while ((cmd = nvme_retrieve_cmd(nvme, nvme->n_ioq[qnum]))) { 22043c9168faSHans Rosenfeld taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq, 22053c9168faSHans Rosenfeld cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent); 2206bf26ea4bSHans Rosenfeld ccnt++; 22073c9168faSHans Rosenfeld } 22083c9168faSHans Rosenfeld } 22093c9168faSHans Rosenfeld 2210bf26ea4bSHans Rosenfeld return (ccnt > 0 ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); 22113c9168faSHans Rosenfeld } 22123c9168faSHans Rosenfeld 22133c9168faSHans Rosenfeld static void 22146afc9eb2SHans Rosenfeld nvme_release_interrupts(nvme_t *nvme) 22153c9168faSHans Rosenfeld { 22163c9168faSHans Rosenfeld int i; 22173c9168faSHans Rosenfeld 22183c9168faSHans Rosenfeld for (i = 0; i < nvme->n_intr_cnt; i++) { 22193c9168faSHans Rosenfeld if (nvme->n_inth[i] == NULL) 22203c9168faSHans Rosenfeld break; 22213c9168faSHans Rosenfeld 22223c9168faSHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 22233c9168faSHans Rosenfeld (void) ddi_intr_block_disable(&nvme->n_inth[i], 1); 22243c9168faSHans Rosenfeld else 22253c9168faSHans Rosenfeld (void) ddi_intr_disable(nvme->n_inth[i]); 22263c9168faSHans Rosenfeld 22273c9168faSHans Rosenfeld (void) ddi_intr_remove_handler(nvme->n_inth[i]); 22283c9168faSHans Rosenfeld (void) ddi_intr_free(nvme->n_inth[i]); 22293c9168faSHans Rosenfeld } 22303c9168faSHans Rosenfeld 22313c9168faSHans Rosenfeld kmem_free(nvme->n_inth, nvme->n_inth_sz); 22323c9168faSHans Rosenfeld nvme->n_inth = NULL; 22333c9168faSHans Rosenfeld nvme->n_inth_sz = 0; 22343c9168faSHans Rosenfeld 22353c9168faSHans Rosenfeld nvme->n_progress &= ~NVME_INTERRUPTS; 22363c9168faSHans Rosenfeld } 22373c9168faSHans Rosenfeld 22383c9168faSHans Rosenfeld static int 22393c9168faSHans Rosenfeld nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs) 22403c9168faSHans Rosenfeld { 22413c9168faSHans Rosenfeld int nintrs, navail, count; 22423c9168faSHans Rosenfeld int ret; 22433c9168faSHans Rosenfeld int i; 22443c9168faSHans Rosenfeld 22453c9168faSHans Rosenfeld if (nvme->n_intr_types == 0) { 22463c9168faSHans Rosenfeld ret = ddi_intr_get_supported_types(nvme->n_dip, 22473c9168faSHans Rosenfeld &nvme->n_intr_types); 22483c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 22493c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22503c9168faSHans Rosenfeld "!%s: ddi_intr_get_supported types failed", 22513c9168faSHans Rosenfeld __func__); 22523c9168faSHans Rosenfeld return (ret); 22533c9168faSHans Rosenfeld } 22543c9168faSHans Rosenfeld } 22553c9168faSHans Rosenfeld 22563c9168faSHans Rosenfeld if ((nvme->n_intr_types & intr_type) == 0) 22573c9168faSHans Rosenfeld return (DDI_FAILURE); 22583c9168faSHans Rosenfeld 22593c9168faSHans Rosenfeld ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs); 22603c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 22613c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed", 22623c9168faSHans Rosenfeld __func__); 22633c9168faSHans Rosenfeld return (ret); 22643c9168faSHans Rosenfeld } 22653c9168faSHans Rosenfeld 22663c9168faSHans Rosenfeld ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail); 22673c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 22683c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed", 22693c9168faSHans Rosenfeld __func__); 22703c9168faSHans Rosenfeld return (ret); 22713c9168faSHans Rosenfeld } 22723c9168faSHans Rosenfeld 22733c9168faSHans Rosenfeld /* We want at most one interrupt per queue pair. */ 22743c9168faSHans Rosenfeld if (navail > nqpairs) 22753c9168faSHans Rosenfeld navail = nqpairs; 22763c9168faSHans Rosenfeld 22773c9168faSHans Rosenfeld nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail; 22783c9168faSHans Rosenfeld nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP); 22793c9168faSHans Rosenfeld 22803c9168faSHans Rosenfeld ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail, 22813c9168faSHans Rosenfeld &count, 0); 22823c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 22833c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed", 22843c9168faSHans Rosenfeld __func__); 22853c9168faSHans Rosenfeld goto fail; 22863c9168faSHans Rosenfeld } 22873c9168faSHans Rosenfeld 22883c9168faSHans Rosenfeld nvme->n_intr_cnt = count; 22893c9168faSHans Rosenfeld 22903c9168faSHans Rosenfeld ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri); 22913c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 22923c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed", 22933c9168faSHans Rosenfeld __func__); 22943c9168faSHans Rosenfeld goto fail; 22953c9168faSHans Rosenfeld } 22963c9168faSHans Rosenfeld 22973c9168faSHans Rosenfeld for (i = 0; i < count; i++) { 22983c9168faSHans Rosenfeld ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr, 22993c9168faSHans Rosenfeld (void *)nvme, (void *)(uintptr_t)i); 23003c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23013c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23023c9168faSHans Rosenfeld "!%s: ddi_intr_add_handler failed", __func__); 23033c9168faSHans Rosenfeld goto fail; 23043c9168faSHans Rosenfeld } 23053c9168faSHans Rosenfeld } 23063c9168faSHans Rosenfeld 23073c9168faSHans Rosenfeld (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap); 23083c9168faSHans Rosenfeld 23096afc9eb2SHans Rosenfeld for (i = 0; i < count; i++) { 231075b41617SHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 231175b41617SHans Rosenfeld ret = ddi_intr_block_enable(&nvme->n_inth[i], 1); 231275b41617SHans Rosenfeld else 231375b41617SHans Rosenfeld ret = ddi_intr_enable(nvme->n_inth[i]); 23143c9168faSHans Rosenfeld 231575b41617SHans Rosenfeld if (ret != DDI_SUCCESS) { 231675b41617SHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 231775b41617SHans Rosenfeld "!%s: enabling interrupt %d failed", __func__, i); 231875b41617SHans Rosenfeld goto fail; 23196afc9eb2SHans Rosenfeld } 23206afc9eb2SHans Rosenfeld } 23216afc9eb2SHans Rosenfeld 23223c9168faSHans Rosenfeld nvme->n_intr_type = intr_type; 23233c9168faSHans Rosenfeld 23243c9168faSHans Rosenfeld nvme->n_progress |= NVME_INTERRUPTS; 23253c9168faSHans Rosenfeld 23263c9168faSHans Rosenfeld return (DDI_SUCCESS); 23273c9168faSHans Rosenfeld 23283c9168faSHans Rosenfeld fail: 23293c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 23303c9168faSHans Rosenfeld 23313c9168faSHans Rosenfeld return (ret); 23323c9168faSHans Rosenfeld } 23333c9168faSHans Rosenfeld 23343c9168faSHans Rosenfeld static int 23353c9168faSHans Rosenfeld nvme_fm_errcb(dev_info_t *dip, ddi_fm_error_t *fm_error, const void *arg) 23363c9168faSHans Rosenfeld { 23373c9168faSHans Rosenfeld _NOTE(ARGUNUSED(arg)); 23383c9168faSHans Rosenfeld 23393c9168faSHans Rosenfeld pci_ereport_post(dip, fm_error, NULL); 23403c9168faSHans Rosenfeld return (fm_error->fme_status); 23413c9168faSHans Rosenfeld } 23423c9168faSHans Rosenfeld 23433c9168faSHans Rosenfeld static int 23443c9168faSHans Rosenfeld nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 23453c9168faSHans Rosenfeld { 23463c9168faSHans Rosenfeld nvme_t *nvme; 23473c9168faSHans Rosenfeld int instance; 23483c9168faSHans Rosenfeld int nregs; 23493c9168faSHans Rosenfeld off_t regsize; 23503c9168faSHans Rosenfeld int i; 23513c9168faSHans Rosenfeld char name[32]; 23523c9168faSHans Rosenfeld 23533c9168faSHans Rosenfeld if (cmd != DDI_ATTACH) 23543c9168faSHans Rosenfeld return (DDI_FAILURE); 23553c9168faSHans Rosenfeld 23563c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 23573c9168faSHans Rosenfeld 23583c9168faSHans Rosenfeld if (ddi_soft_state_zalloc(nvme_state, instance) != DDI_SUCCESS) 23593c9168faSHans Rosenfeld return (DDI_FAILURE); 23603c9168faSHans Rosenfeld 23613c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 23623c9168faSHans Rosenfeld ddi_set_driver_private(dip, nvme); 23633c9168faSHans Rosenfeld nvme->n_dip = dip; 23643c9168faSHans Rosenfeld 23653c9168faSHans Rosenfeld nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 23663c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "strict-version", 1) == 1 ? B_TRUE : B_FALSE; 23673c9168faSHans Rosenfeld nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY, 23683c9168faSHans Rosenfeld dip, DDI_PROP_DONTPASS, "ignore-unknown-vendor-status", 0) == 1 ? 23693c9168faSHans Rosenfeld B_TRUE : B_FALSE; 23703c9168faSHans Rosenfeld nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 23713c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "admin-queue-len", NVME_DEFAULT_ADMIN_QUEUE_LEN); 23723c9168faSHans Rosenfeld nvme->n_io_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 23733c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "io-queue-len", NVME_DEFAULT_IO_QUEUE_LEN); 23743c9168faSHans Rosenfeld nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 23753c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "async-event-limit", 23763c9168faSHans Rosenfeld NVME_DEFAULT_ASYNC_EVENT_LIMIT); 23773c9168faSHans Rosenfeld 23783c9168faSHans Rosenfeld if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN) 23793c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN; 23803c9168faSHans Rosenfeld else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN) 23813c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN; 23823c9168faSHans Rosenfeld 23833c9168faSHans Rosenfeld if (nvme->n_io_queue_len < NVME_MIN_IO_QUEUE_LEN) 23843c9168faSHans Rosenfeld nvme->n_io_queue_len = NVME_MIN_IO_QUEUE_LEN; 23853c9168faSHans Rosenfeld 23863c9168faSHans Rosenfeld if (nvme->n_async_event_limit < 1) 23873c9168faSHans Rosenfeld nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT; 23883c9168faSHans Rosenfeld 23893c9168faSHans Rosenfeld nvme->n_reg_acc_attr = nvme_reg_acc_attr; 23903c9168faSHans Rosenfeld nvme->n_queue_dma_attr = nvme_queue_dma_attr; 23913c9168faSHans Rosenfeld nvme->n_prp_dma_attr = nvme_prp_dma_attr; 23923c9168faSHans Rosenfeld nvme->n_sgl_dma_attr = nvme_sgl_dma_attr; 23933c9168faSHans Rosenfeld 23943c9168faSHans Rosenfeld /* 23953c9168faSHans Rosenfeld * Setup FMA support. 23963c9168faSHans Rosenfeld */ 23973c9168faSHans Rosenfeld nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip, 23983c9168faSHans Rosenfeld DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 23993c9168faSHans Rosenfeld DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 24003c9168faSHans Rosenfeld DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 24013c9168faSHans Rosenfeld 24023c9168faSHans Rosenfeld ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc); 24033c9168faSHans Rosenfeld 24043c9168faSHans Rosenfeld if (nvme->n_fm_cap) { 24053c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE) 24063c9168faSHans Rosenfeld nvme->n_reg_acc_attr.devacc_attr_access = 24073c9168faSHans Rosenfeld DDI_FLAGERR_ACC; 24083c9168faSHans Rosenfeld 24093c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) { 24103c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 24113c9168faSHans Rosenfeld nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 24123c9168faSHans Rosenfeld } 24133c9168faSHans Rosenfeld 24143c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 24153c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 24163c9168faSHans Rosenfeld pci_ereport_setup(dip); 24173c9168faSHans Rosenfeld 24183c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 24193c9168faSHans Rosenfeld ddi_fm_handler_register(dip, nvme_fm_errcb, 24203c9168faSHans Rosenfeld (void *)nvme); 24213c9168faSHans Rosenfeld } 24223c9168faSHans Rosenfeld 24233c9168faSHans Rosenfeld nvme->n_progress |= NVME_FMA_INIT; 24243c9168faSHans Rosenfeld 24253c9168faSHans Rosenfeld /* 24263c9168faSHans Rosenfeld * The spec defines several register sets. Only the controller 24273c9168faSHans Rosenfeld * registers (set 1) are currently used. 24283c9168faSHans Rosenfeld */ 24293c9168faSHans Rosenfeld if (ddi_dev_nregs(dip, &nregs) == DDI_FAILURE || 24303c9168faSHans Rosenfeld nregs < 2 || 24313c9168faSHans Rosenfeld ddi_dev_regsize(dip, 1, ®size) == DDI_FAILURE) 24323c9168faSHans Rosenfeld goto fail; 24333c9168faSHans Rosenfeld 24343c9168faSHans Rosenfeld if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize, 24353c9168faSHans Rosenfeld &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) { 24363c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to map regset 1"); 24373c9168faSHans Rosenfeld goto fail; 24383c9168faSHans Rosenfeld } 24393c9168faSHans Rosenfeld 24403c9168faSHans Rosenfeld nvme->n_progress |= NVME_REGS_MAPPED; 24413c9168faSHans Rosenfeld 24423c9168faSHans Rosenfeld /* 24433c9168faSHans Rosenfeld * Create taskq for command completion. 24443c9168faSHans Rosenfeld */ 24453c9168faSHans Rosenfeld (void) snprintf(name, sizeof (name), "%s%d_cmd_taskq", 24463c9168faSHans Rosenfeld ddi_driver_name(dip), ddi_get_instance(dip)); 24473c9168faSHans Rosenfeld nvme->n_cmd_taskq = ddi_taskq_create(dip, name, MIN(UINT16_MAX, ncpus), 24483c9168faSHans Rosenfeld TASKQ_DEFAULTPRI, 0); 24493c9168faSHans Rosenfeld if (nvme->n_cmd_taskq == NULL) { 24503c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to create cmd taskq"); 24513c9168faSHans Rosenfeld goto fail; 24523c9168faSHans Rosenfeld } 24533c9168faSHans Rosenfeld 24543c9168faSHans Rosenfeld 24553c9168faSHans Rosenfeld if (nvme_init(nvme) != DDI_SUCCESS) 24563c9168faSHans Rosenfeld goto fail; 24573c9168faSHans Rosenfeld 24583c9168faSHans Rosenfeld /* 24593c9168faSHans Rosenfeld * Attach the blkdev driver for each namespace. 24603c9168faSHans Rosenfeld */ 24613c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 24623c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_ignore) 24633c9168faSHans Rosenfeld continue; 24643c9168faSHans Rosenfeld 24653c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl = bd_alloc_handle(&nvme->n_ns[i], 24663c9168faSHans Rosenfeld &nvme_bd_ops, &nvme->n_prp_dma_attr, KM_SLEEP); 24673c9168faSHans Rosenfeld 24683c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl == NULL) { 24693c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 24703c9168faSHans Rosenfeld "!failed to get blkdev handle for namespace %d", i); 24713c9168faSHans Rosenfeld goto fail; 24723c9168faSHans Rosenfeld } 24733c9168faSHans Rosenfeld 24743c9168faSHans Rosenfeld if (bd_attach_handle(dip, nvme->n_ns[i].ns_bd_hdl) 24753c9168faSHans Rosenfeld != DDI_SUCCESS) { 24763c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 24773c9168faSHans Rosenfeld "!failed to attach blkdev handle for namespace %d", 24783c9168faSHans Rosenfeld i); 24793c9168faSHans Rosenfeld goto fail; 24803c9168faSHans Rosenfeld } 24813c9168faSHans Rosenfeld } 24823c9168faSHans Rosenfeld 24833c9168faSHans Rosenfeld return (DDI_SUCCESS); 24843c9168faSHans Rosenfeld 24853c9168faSHans Rosenfeld fail: 24863c9168faSHans Rosenfeld /* attach successful anyway so that FMA can retire the device */ 24873c9168faSHans Rosenfeld if (nvme->n_dead) 24883c9168faSHans Rosenfeld return (DDI_SUCCESS); 24893c9168faSHans Rosenfeld 24903c9168faSHans Rosenfeld (void) nvme_detach(dip, DDI_DETACH); 24913c9168faSHans Rosenfeld 24923c9168faSHans Rosenfeld return (DDI_FAILURE); 24933c9168faSHans Rosenfeld } 24943c9168faSHans Rosenfeld 24953c9168faSHans Rosenfeld static int 24963c9168faSHans Rosenfeld nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 24973c9168faSHans Rosenfeld { 24983c9168faSHans Rosenfeld int instance, i; 24993c9168faSHans Rosenfeld nvme_t *nvme; 25003c9168faSHans Rosenfeld 25013c9168faSHans Rosenfeld if (cmd != DDI_DETACH) 25023c9168faSHans Rosenfeld return (DDI_FAILURE); 25033c9168faSHans Rosenfeld 25043c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 25053c9168faSHans Rosenfeld 25063c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 25073c9168faSHans Rosenfeld 25083c9168faSHans Rosenfeld if (nvme == NULL) 25093c9168faSHans Rosenfeld return (DDI_FAILURE); 25103c9168faSHans Rosenfeld 25113c9168faSHans Rosenfeld if (nvme->n_ns) { 25123c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 25133c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl) { 25143c9168faSHans Rosenfeld (void) bd_detach_handle( 25153c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl); 25163c9168faSHans Rosenfeld bd_free_handle(nvme->n_ns[i].ns_bd_hdl); 25173c9168faSHans Rosenfeld } 25183c9168faSHans Rosenfeld 25193c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_idns) 25203c9168faSHans Rosenfeld kmem_free(nvme->n_ns[i].ns_idns, 25213c9168faSHans Rosenfeld sizeof (nvme_identify_nsid_t)); 25223c9168faSHans Rosenfeld } 25233c9168faSHans Rosenfeld 25243c9168faSHans Rosenfeld kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) * 25253c9168faSHans Rosenfeld nvme->n_namespace_count); 25263c9168faSHans Rosenfeld } 25273c9168faSHans Rosenfeld 25283c9168faSHans Rosenfeld if (nvme->n_progress & NVME_INTERRUPTS) 25293c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 25303c9168faSHans Rosenfeld 25313c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 25323c9168faSHans Rosenfeld ddi_taskq_wait(nvme->n_cmd_taskq); 25333c9168faSHans Rosenfeld 25343c9168faSHans Rosenfeld if (nvme->n_ioq_count > 0) { 25353c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 25363c9168faSHans Rosenfeld if (nvme->n_ioq[i] != NULL) { 25373c9168faSHans Rosenfeld /* TODO: send destroy queue commands */ 25383c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_ioq[i]); 25393c9168faSHans Rosenfeld } 25403c9168faSHans Rosenfeld } 25413c9168faSHans Rosenfeld 25423c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) * 25433c9168faSHans Rosenfeld (nvme->n_ioq_count + 1)); 25443c9168faSHans Rosenfeld } 25453c9168faSHans Rosenfeld 25463c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) { 25473c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE); 25483c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 25493c9168faSHans Rosenfeld } 25503c9168faSHans Rosenfeld 25513c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 25523c9168faSHans Rosenfeld ddi_taskq_destroy(nvme->n_cmd_taskq); 25533c9168faSHans Rosenfeld 25543c9168faSHans Rosenfeld if (nvme->n_progress & NVME_CTRL_LIMITS) 25553c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 25563c9168faSHans Rosenfeld 25573c9168faSHans Rosenfeld if (nvme->n_progress & NVME_ADMIN_QUEUE) 25583c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_adminq); 25593c9168faSHans Rosenfeld 25603c9168faSHans Rosenfeld if (nvme->n_idctl) 25613c9168faSHans Rosenfeld kmem_free(nvme->n_idctl, sizeof (nvme_identify_ctrl_t)); 25623c9168faSHans Rosenfeld 25633c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) 25643c9168faSHans Rosenfeld ddi_regs_map_free(&nvme->n_regh); 25653c9168faSHans Rosenfeld 25663c9168faSHans Rosenfeld if (nvme->n_progress & NVME_FMA_INIT) { 25673c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 25683c9168faSHans Rosenfeld ddi_fm_handler_unregister(nvme->n_dip); 25693c9168faSHans Rosenfeld 25703c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 25713c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 25723c9168faSHans Rosenfeld pci_ereport_teardown(nvme->n_dip); 25733c9168faSHans Rosenfeld 25743c9168faSHans Rosenfeld ddi_fm_fini(nvme->n_dip); 25753c9168faSHans Rosenfeld } 25763c9168faSHans Rosenfeld 2577510a6847SHans Rosenfeld if (nvme->n_vendor != NULL) 2578510a6847SHans Rosenfeld strfree(nvme->n_vendor); 2579510a6847SHans Rosenfeld 2580510a6847SHans Rosenfeld if (nvme->n_product != NULL) 2581510a6847SHans Rosenfeld strfree(nvme->n_product); 2582510a6847SHans Rosenfeld 25833c9168faSHans Rosenfeld ddi_soft_state_free(nvme_state, instance); 25843c9168faSHans Rosenfeld 25853c9168faSHans Rosenfeld return (DDI_SUCCESS); 25863c9168faSHans Rosenfeld } 25873c9168faSHans Rosenfeld 25883c9168faSHans Rosenfeld static int 25893c9168faSHans Rosenfeld nvme_quiesce(dev_info_t *dip) 25903c9168faSHans Rosenfeld { 25913c9168faSHans Rosenfeld int instance; 25923c9168faSHans Rosenfeld nvme_t *nvme; 25933c9168faSHans Rosenfeld 25943c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 25953c9168faSHans Rosenfeld 25963c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 25973c9168faSHans Rosenfeld 25983c9168faSHans Rosenfeld if (nvme == NULL) 25993c9168faSHans Rosenfeld return (DDI_FAILURE); 26003c9168faSHans Rosenfeld 26013c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_ABRUPT, B_TRUE); 26023c9168faSHans Rosenfeld 26033c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_TRUE); 26043c9168faSHans Rosenfeld 26053c9168faSHans Rosenfeld return (DDI_FAILURE); 26063c9168faSHans Rosenfeld } 26073c9168faSHans Rosenfeld 26083c9168faSHans Rosenfeld static int 26093c9168faSHans Rosenfeld nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer) 26103c9168faSHans Rosenfeld { 26113c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 26123c9168faSHans Rosenfeld int nprp_page, nprp; 26133c9168faSHans Rosenfeld uint64_t *prp; 26143c9168faSHans Rosenfeld 26153c9168faSHans Rosenfeld if (xfer->x_ndmac == 0) 26163c9168faSHans Rosenfeld return (DDI_FAILURE); 26173c9168faSHans Rosenfeld 26183c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress; 26193c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 26203c9168faSHans Rosenfeld 26213c9168faSHans Rosenfeld if (xfer->x_ndmac == 1) { 26223c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; 26233c9168faSHans Rosenfeld return (DDI_SUCCESS); 26243c9168faSHans Rosenfeld } else if (xfer->x_ndmac == 2) { 26253c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress; 26263c9168faSHans Rosenfeld return (DDI_SUCCESS); 26273c9168faSHans Rosenfeld } 26283c9168faSHans Rosenfeld 26293c9168faSHans Rosenfeld xfer->x_ndmac--; 26303c9168faSHans Rosenfeld 26313c9168faSHans Rosenfeld nprp_page = nvme->n_pagesize / sizeof (uint64_t) - 1; 26323c9168faSHans Rosenfeld ASSERT(nprp_page > 0); 26333c9168faSHans Rosenfeld nprp = (xfer->x_ndmac + nprp_page - 1) / nprp_page; 26343c9168faSHans Rosenfeld 26353c9168faSHans Rosenfeld /* 26363c9168faSHans Rosenfeld * We currently don't support chained PRPs and set up our DMA 26373c9168faSHans Rosenfeld * attributes to reflect that. If we still get an I/O request 26383c9168faSHans Rosenfeld * that needs a chained PRP something is very wrong. 26393c9168faSHans Rosenfeld */ 26403c9168faSHans Rosenfeld VERIFY(nprp == 1); 26413c9168faSHans Rosenfeld 26423c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, nvme->n_pagesize * nprp, DDI_DMA_READ, 26433c9168faSHans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 26443c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: nvme_zalloc_dma failed", 26453c9168faSHans Rosenfeld __func__); 26463c9168faSHans Rosenfeld return (DDI_FAILURE); 26473c9168faSHans Rosenfeld } 26483c9168faSHans Rosenfeld 26493c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress; 26503c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, &cmd->nc_dma->nd_cookie); 26513c9168faSHans Rosenfeld 26523c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 26533c9168faSHans Rosenfeld for (prp = (uint64_t *)cmd->nc_dma->nd_memp; 26543c9168faSHans Rosenfeld xfer->x_ndmac > 0; 26553c9168faSHans Rosenfeld prp++, xfer->x_ndmac--) { 26563c9168faSHans Rosenfeld *prp = xfer->x_dmac.dmac_laddress; 26573c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 26583c9168faSHans Rosenfeld } 26593c9168faSHans Rosenfeld 26603c9168faSHans Rosenfeld (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len, 26613c9168faSHans Rosenfeld DDI_DMA_SYNC_FORDEV); 26623c9168faSHans Rosenfeld return (DDI_SUCCESS); 26633c9168faSHans Rosenfeld } 26643c9168faSHans Rosenfeld 26653c9168faSHans Rosenfeld static nvme_cmd_t * 26663c9168faSHans Rosenfeld nvme_create_nvm_cmd(nvme_namespace_t *ns, uint8_t opc, bd_xfer_t *xfer) 26673c9168faSHans Rosenfeld { 26683c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 26693c9168faSHans Rosenfeld nvme_cmd_t *cmd; 26703c9168faSHans Rosenfeld 26713c9168faSHans Rosenfeld /* 26723c9168faSHans Rosenfeld * Blkdev only sets BD_XFER_POLL when dumping, so don't sleep. 26733c9168faSHans Rosenfeld */ 26743c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ? 26753c9168faSHans Rosenfeld KM_NOSLEEP : KM_SLEEP); 26763c9168faSHans Rosenfeld 26773c9168faSHans Rosenfeld if (cmd == NULL) 26783c9168faSHans Rosenfeld return (NULL); 26793c9168faSHans Rosenfeld 26803c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = opc; 26813c9168faSHans Rosenfeld cmd->nc_callback = nvme_bd_xfer_done; 26823c9168faSHans Rosenfeld cmd->nc_xfer = xfer; 26833c9168faSHans Rosenfeld 26843c9168faSHans Rosenfeld switch (opc) { 26853c9168faSHans Rosenfeld case NVME_OPC_NVM_WRITE: 26863c9168faSHans Rosenfeld case NVME_OPC_NVM_READ: 26873c9168faSHans Rosenfeld VERIFY(xfer->x_nblks <= 0x10000); 26883c9168faSHans Rosenfeld 26893c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 26903c9168faSHans Rosenfeld 26913c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu; 26923c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32); 26933c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1); 26943c9168faSHans Rosenfeld 26953c9168faSHans Rosenfeld if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS) 26963c9168faSHans Rosenfeld goto fail; 26973c9168faSHans Rosenfeld break; 26983c9168faSHans Rosenfeld 26993c9168faSHans Rosenfeld case NVME_OPC_NVM_FLUSH: 27003c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 27013c9168faSHans Rosenfeld break; 27023c9168faSHans Rosenfeld 27033c9168faSHans Rosenfeld default: 27043c9168faSHans Rosenfeld goto fail; 27053c9168faSHans Rosenfeld } 27063c9168faSHans Rosenfeld 27073c9168faSHans Rosenfeld return (cmd); 27083c9168faSHans Rosenfeld 27093c9168faSHans Rosenfeld fail: 27103c9168faSHans Rosenfeld nvme_free_cmd(cmd); 27113c9168faSHans Rosenfeld return (NULL); 27123c9168faSHans Rosenfeld } 27133c9168faSHans Rosenfeld 27143c9168faSHans Rosenfeld static void 27153c9168faSHans Rosenfeld nvme_bd_xfer_done(void *arg) 27163c9168faSHans Rosenfeld { 27173c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 27183c9168faSHans Rosenfeld bd_xfer_t *xfer = cmd->nc_xfer; 27193c9168faSHans Rosenfeld int error = 0; 27203c9168faSHans Rosenfeld 27213c9168faSHans Rosenfeld error = nvme_check_cmd_status(cmd); 27223c9168faSHans Rosenfeld nvme_free_cmd(cmd); 27233c9168faSHans Rosenfeld 27243c9168faSHans Rosenfeld bd_xfer_done(xfer, error); 27253c9168faSHans Rosenfeld } 27263c9168faSHans Rosenfeld 27273c9168faSHans Rosenfeld static void 27283c9168faSHans Rosenfeld nvme_bd_driveinfo(void *arg, bd_drive_t *drive) 27293c9168faSHans Rosenfeld { 27303c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 27313c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 27323c9168faSHans Rosenfeld 27333c9168faSHans Rosenfeld /* 27343c9168faSHans Rosenfeld * blkdev maintains one queue size per instance (namespace), 27353c9168faSHans Rosenfeld * but all namespace share the I/O queues. 27363c9168faSHans Rosenfeld * TODO: need to figure out a sane default, or use per-NS I/O queues, 27373c9168faSHans Rosenfeld * or change blkdev to handle EAGAIN 27383c9168faSHans Rosenfeld */ 27393c9168faSHans Rosenfeld drive->d_qsize = nvme->n_ioq_count * nvme->n_io_queue_len 27403c9168faSHans Rosenfeld / nvme->n_namespace_count; 27413c9168faSHans Rosenfeld 27423c9168faSHans Rosenfeld /* 27433c9168faSHans Rosenfeld * d_maxxfer is not set, which means the value is taken from the DMA 27443c9168faSHans Rosenfeld * attributes specified to bd_alloc_handle. 27453c9168faSHans Rosenfeld */ 27463c9168faSHans Rosenfeld 27473c9168faSHans Rosenfeld drive->d_removable = B_FALSE; 27483c9168faSHans Rosenfeld drive->d_hotpluggable = B_FALSE; 27493c9168faSHans Rosenfeld 27503c9168faSHans Rosenfeld drive->d_target = ns->ns_id; 27513c9168faSHans Rosenfeld drive->d_lun = 0; 2752510a6847SHans Rosenfeld 2753bef9e21aSHans Rosenfeld drive->d_model = nvme->n_idctl->id_model; 2754bef9e21aSHans Rosenfeld drive->d_model_len = sizeof (nvme->n_idctl->id_model); 2755510a6847SHans Rosenfeld drive->d_vendor = nvme->n_vendor; 2756510a6847SHans Rosenfeld drive->d_vendor_len = strlen(nvme->n_vendor); 2757510a6847SHans Rosenfeld drive->d_product = nvme->n_product; 2758510a6847SHans Rosenfeld drive->d_product_len = strlen(nvme->n_product); 2759510a6847SHans Rosenfeld drive->d_serial = nvme->n_idctl->id_serial; 2760510a6847SHans Rosenfeld drive->d_serial_len = sizeof (nvme->n_idctl->id_serial); 2761510a6847SHans Rosenfeld drive->d_revision = nvme->n_idctl->id_fwrev; 2762510a6847SHans Rosenfeld drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev); 27633c9168faSHans Rosenfeld } 27643c9168faSHans Rosenfeld 27653c9168faSHans Rosenfeld static int 27663c9168faSHans Rosenfeld nvme_bd_mediainfo(void *arg, bd_media_t *media) 27673c9168faSHans Rosenfeld { 27683c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 27693c9168faSHans Rosenfeld 27703c9168faSHans Rosenfeld media->m_nblks = ns->ns_block_count; 27713c9168faSHans Rosenfeld media->m_blksize = ns->ns_block_size; 27723c9168faSHans Rosenfeld media->m_readonly = B_FALSE; 27733c9168faSHans Rosenfeld media->m_solidstate = B_TRUE; 27743c9168faSHans Rosenfeld 27753c9168faSHans Rosenfeld media->m_pblksize = ns->ns_best_block_size; 27763c9168faSHans Rosenfeld 27773c9168faSHans Rosenfeld return (0); 27783c9168faSHans Rosenfeld } 27793c9168faSHans Rosenfeld 27803c9168faSHans Rosenfeld static int 27813c9168faSHans Rosenfeld nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc) 27823c9168faSHans Rosenfeld { 27833c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 27843c9168faSHans Rosenfeld nvme_cmd_t *cmd; 27853c9168faSHans Rosenfeld 27863c9168faSHans Rosenfeld if (nvme->n_dead) 27873c9168faSHans Rosenfeld return (EIO); 27883c9168faSHans Rosenfeld 27893c9168faSHans Rosenfeld /* No polling for now */ 27903c9168faSHans Rosenfeld if (xfer->x_flags & BD_XFER_POLL) 27913c9168faSHans Rosenfeld return (EIO); 27923c9168faSHans Rosenfeld 27933c9168faSHans Rosenfeld cmd = nvme_create_nvm_cmd(ns, opc, xfer); 27943c9168faSHans Rosenfeld if (cmd == NULL) 27953c9168faSHans Rosenfeld return (ENOMEM); 27963c9168faSHans Rosenfeld 27973c9168faSHans Rosenfeld cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1; 27983c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); 27993c9168faSHans Rosenfeld 28003c9168faSHans Rosenfeld if (nvme_submit_cmd(nvme->n_ioq[cmd->nc_sqid], cmd) 28013c9168faSHans Rosenfeld != DDI_SUCCESS) 28023c9168faSHans Rosenfeld return (EAGAIN); 28033c9168faSHans Rosenfeld 28043c9168faSHans Rosenfeld return (0); 28053c9168faSHans Rosenfeld } 28063c9168faSHans Rosenfeld 28073c9168faSHans Rosenfeld static int 28083c9168faSHans Rosenfeld nvme_bd_read(void *arg, bd_xfer_t *xfer) 28093c9168faSHans Rosenfeld { 28103c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28113c9168faSHans Rosenfeld 28123c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_READ)); 28133c9168faSHans Rosenfeld } 28143c9168faSHans Rosenfeld 28153c9168faSHans Rosenfeld static int 28163c9168faSHans Rosenfeld nvme_bd_write(void *arg, bd_xfer_t *xfer) 28173c9168faSHans Rosenfeld { 28183c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28193c9168faSHans Rosenfeld 28203c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_WRITE)); 28213c9168faSHans Rosenfeld } 28223c9168faSHans Rosenfeld 28233c9168faSHans Rosenfeld static int 28243c9168faSHans Rosenfeld nvme_bd_sync(void *arg, bd_xfer_t *xfer) 28253c9168faSHans Rosenfeld { 28263c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28273c9168faSHans Rosenfeld 28283c9168faSHans Rosenfeld if (ns->ns_nvme->n_dead) 28293c9168faSHans Rosenfeld return (EIO); 28303c9168faSHans Rosenfeld 28313c9168faSHans Rosenfeld /* 28323c9168faSHans Rosenfeld * If the volatile write cache isn't enabled the FLUSH command is a 28333c9168faSHans Rosenfeld * no-op, so we can take a shortcut here. 28343c9168faSHans Rosenfeld */ 28353c9168faSHans Rosenfeld if (ns->ns_nvme->n_volatile_write_cache_enabled == B_FALSE) { 28363c9168faSHans Rosenfeld bd_xfer_done(xfer, ENOTSUP); 28373c9168faSHans Rosenfeld return (0); 28383c9168faSHans Rosenfeld } 28393c9168faSHans Rosenfeld 28403c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_FLUSH)); 28413c9168faSHans Rosenfeld } 28423c9168faSHans Rosenfeld 28433c9168faSHans Rosenfeld static int 28443c9168faSHans Rosenfeld nvme_bd_devid(void *arg, dev_info_t *devinfo, ddi_devid_t *devid) 28453c9168faSHans Rosenfeld { 28463c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28473c9168faSHans Rosenfeld 28483c9168faSHans Rosenfeld return (ddi_devid_init(devinfo, DEVID_ENCAP, strlen(ns->ns_devid), 28493c9168faSHans Rosenfeld ns->ns_devid, devid)); 28503c9168faSHans Rosenfeld } 2851