13c9168faSHans Rosenfeld /* 23c9168faSHans Rosenfeld * This file and its contents are supplied under the terms of the 33c9168faSHans Rosenfeld * Common Development and Distribution License ("CDDL"), version 1.0. 43c9168faSHans Rosenfeld * You may only use this file in accordance with the terms of version 53c9168faSHans Rosenfeld * 1.0 of the CDDL. 63c9168faSHans Rosenfeld * 73c9168faSHans Rosenfeld * A full copy of the text of the CDDL should have accompanied this 83c9168faSHans Rosenfeld * source. A copy of the CDDL is also available via the Internet at 93c9168faSHans Rosenfeld * http://www.illumos.org/license/CDDL. 103c9168faSHans Rosenfeld */ 113c9168faSHans Rosenfeld 123c9168faSHans Rosenfeld /* 136afc9eb2SHans Rosenfeld * Copyright 2016 Nexenta Systems, Inc. All rights reserved. 1434c938c7SPete Shephard * Copyright 2016 Tegile Systems, Inc. All rights reserved. 152f95345bSYouzhong Yang * Copyright (c) 2016 The MathWorks, Inc. All rights reserved. 163c9168faSHans Rosenfeld */ 173c9168faSHans Rosenfeld 183c9168faSHans Rosenfeld /* 193c9168faSHans Rosenfeld * blkdev driver for NVMe compliant storage devices 203c9168faSHans Rosenfeld * 213c9168faSHans Rosenfeld * This driver was written to conform to version 1.0e of the NVMe specification. 223c9168faSHans Rosenfeld * It may work with newer versions, but that is completely untested and disabled 233c9168faSHans Rosenfeld * by default. 243c9168faSHans Rosenfeld * 253c9168faSHans Rosenfeld * The driver has only been tested on x86 systems and will not work on big- 263c9168faSHans Rosenfeld * endian systems without changes to the code accessing registers and data 273c9168faSHans Rosenfeld * structures used by the hardware. 283c9168faSHans Rosenfeld * 293c9168faSHans Rosenfeld * 303c9168faSHans Rosenfeld * Interrupt Usage: 313c9168faSHans Rosenfeld * 323c9168faSHans Rosenfeld * The driver will use a FIXED interrupt while configuring the device as the 333c9168faSHans Rosenfeld * specification requires. Later in the attach process it will switch to MSI-X 343c9168faSHans Rosenfeld * or MSI if supported. The driver wants to have one interrupt vector per CPU, 353c9168faSHans Rosenfeld * but it will work correctly if less are available. Interrupts can be shared 363c9168faSHans Rosenfeld * by queues, the interrupt handler will iterate through the I/O queue array by 373c9168faSHans Rosenfeld * steps of n_intr_cnt. Usually only the admin queue will share an interrupt 383c9168faSHans Rosenfeld * with one I/O queue. The interrupt handler will retrieve completed commands 393c9168faSHans Rosenfeld * from all queues sharing an interrupt vector and will post them to a taskq 403c9168faSHans Rosenfeld * for completion processing. 413c9168faSHans Rosenfeld * 423c9168faSHans Rosenfeld * 433c9168faSHans Rosenfeld * Command Processing: 443c9168faSHans Rosenfeld * 453c9168faSHans Rosenfeld * NVMe devices can have up to 65536 I/O queue pairs, with each queue holding up 463c9168faSHans Rosenfeld * to 65536 I/O commands. The driver will configure one I/O queue pair per 473c9168faSHans Rosenfeld * available interrupt vector, with the queue length usually much smaller than 483c9168faSHans Rosenfeld * the maximum of 65536. If the hardware doesn't provide enough queues, fewer 493c9168faSHans Rosenfeld * interrupt vectors will be used. 503c9168faSHans Rosenfeld * 513c9168faSHans Rosenfeld * Additionally the hardware provides a single special admin queue pair that can 523c9168faSHans Rosenfeld * hold up to 4096 admin commands. 533c9168faSHans Rosenfeld * 543c9168faSHans Rosenfeld * From the hardware perspective both queues of a queue pair are independent, 553c9168faSHans Rosenfeld * but they share some driver state: the command array (holding pointers to 563c9168faSHans Rosenfeld * commands currently being processed by the hardware) and the active command 573c9168faSHans Rosenfeld * counter. Access to the submission side of a queue pair and the shared state 583c9168faSHans Rosenfeld * is protected by nq_mutex. The completion side of a queue pair does not need 593c9168faSHans Rosenfeld * that protection apart from its access to the shared state; it is called only 603c9168faSHans Rosenfeld * in the interrupt handler which does not run concurrently for the same 613c9168faSHans Rosenfeld * interrupt vector. 623c9168faSHans Rosenfeld * 633c9168faSHans Rosenfeld * When a command is submitted to a queue pair the active command counter is 643c9168faSHans Rosenfeld * incremented and a pointer to the command is stored in the command array. The 653c9168faSHans Rosenfeld * array index is used as command identifier (CID) in the submission queue 663c9168faSHans Rosenfeld * entry. Some commands may take a very long time to complete, and if the queue 673c9168faSHans Rosenfeld * wraps around in that time a submission may find the next array slot to still 683c9168faSHans Rosenfeld * be used by a long-running command. In this case the array is sequentially 693c9168faSHans Rosenfeld * searched for the next free slot. The length of the command array is the same 703c9168faSHans Rosenfeld * as the configured queue length. 713c9168faSHans Rosenfeld * 723c9168faSHans Rosenfeld * 733c9168faSHans Rosenfeld * Namespace Support: 743c9168faSHans Rosenfeld * 753c9168faSHans Rosenfeld * NVMe devices can have multiple namespaces, each being a independent data 763c9168faSHans Rosenfeld * store. The driver supports multiple namespaces and creates a blkdev interface 773c9168faSHans Rosenfeld * for each namespace found. Namespaces can have various attributes to support 789cfb65faSHans Rosenfeld * thin provisioning and protection information. This driver does not support 799cfb65faSHans Rosenfeld * any of this and ignores namespaces that have these attributes. 803c9168faSHans Rosenfeld * 813c9168faSHans Rosenfeld * 823c9168faSHans Rosenfeld * Blkdev Interface: 833c9168faSHans Rosenfeld * 843c9168faSHans Rosenfeld * This driver uses blkdev to do all the heavy lifting involved with presenting 853c9168faSHans Rosenfeld * a disk device to the system. As a result, the processing of I/O requests is 863c9168faSHans Rosenfeld * relatively simple as blkdev takes care of partitioning, boundary checks, DMA 873c9168faSHans Rosenfeld * setup, and splitting of transfers into manageable chunks. 883c9168faSHans Rosenfeld * 893c9168faSHans Rosenfeld * I/O requests coming in from blkdev are turned into NVM commands and posted to 903c9168faSHans Rosenfeld * an I/O queue. The queue is selected by taking the CPU id modulo the number of 913c9168faSHans Rosenfeld * queues. There is currently no timeout handling of I/O commands. 923c9168faSHans Rosenfeld * 933c9168faSHans Rosenfeld * Blkdev also supports querying device/media information and generating a 943c9168faSHans Rosenfeld * devid. The driver reports the best block size as determined by the namespace 953c9168faSHans Rosenfeld * format back to blkdev as physical block size to support partition and block 963c9168faSHans Rosenfeld * alignment. The devid is composed using the device vendor ID, model number, 973c9168faSHans Rosenfeld * serial number, and the namespace ID. 983c9168faSHans Rosenfeld * 993c9168faSHans Rosenfeld * 1003c9168faSHans Rosenfeld * Error Handling: 1013c9168faSHans Rosenfeld * 1023c9168faSHans Rosenfeld * Error handling is currently limited to detecting fatal hardware errors, 1033c9168faSHans Rosenfeld * either by asynchronous events, or synchronously through command status or 1043c9168faSHans Rosenfeld * admin command timeouts. In case of severe errors the device is fenced off, 1053c9168faSHans Rosenfeld * all further requests will return EIO. FMA is then called to fault the device. 1063c9168faSHans Rosenfeld * 1073c9168faSHans Rosenfeld * The hardware has a limit for outstanding asynchronous event requests. Before 1083c9168faSHans Rosenfeld * this limit is known the driver assumes it is at least 1 and posts a single 1093c9168faSHans Rosenfeld * asynchronous request. Later when the limit is known more asynchronous event 1103c9168faSHans Rosenfeld * requests are posted to allow quicker reception of error information. When an 1113c9168faSHans Rosenfeld * asynchronous event is posted by the hardware the driver will parse the error 1123c9168faSHans Rosenfeld * status fields and log information or fault the device, depending on the 1133c9168faSHans Rosenfeld * severity of the asynchronous event. The asynchronous event request is then 1143c9168faSHans Rosenfeld * reused and posted to the admin queue again. 1153c9168faSHans Rosenfeld * 1163c9168faSHans Rosenfeld * On command completion the command status is checked for errors. In case of 1173c9168faSHans Rosenfeld * errors indicating a driver bug the driver panics. Almost all other error 1183c9168faSHans Rosenfeld * status values just cause EIO to be returned. 1193c9168faSHans Rosenfeld * 1203c9168faSHans Rosenfeld * Command timeouts are currently detected for all admin commands except 1213c9168faSHans Rosenfeld * asynchronous event requests. If a command times out and the hardware appears 1223c9168faSHans Rosenfeld * to be healthy the driver attempts to abort the command. If this fails the 1233c9168faSHans Rosenfeld * driver assumes the device to be dead, fences it off, and calls FMA to retire 1243c9168faSHans Rosenfeld * it. In general admin commands are issued at attach time only. No timeout 1253c9168faSHans Rosenfeld * handling of normal I/O commands is presently done. 1263c9168faSHans Rosenfeld * 1273c9168faSHans Rosenfeld * In some cases it may be possible that the ABORT command times out, too. In 1283c9168faSHans Rosenfeld * that case the device is also declared dead and fenced off. 1293c9168faSHans Rosenfeld * 1303c9168faSHans Rosenfeld * 1313c9168faSHans Rosenfeld * Quiesce / Fast Reboot: 1323c9168faSHans Rosenfeld * 1333c9168faSHans Rosenfeld * The driver currently does not support fast reboot. A quiesce(9E) entry point 1343c9168faSHans Rosenfeld * is still provided which is used to send a shutdown notification to the 1353c9168faSHans Rosenfeld * device. 1363c9168faSHans Rosenfeld * 1373c9168faSHans Rosenfeld * 1383c9168faSHans Rosenfeld * Driver Configuration: 1393c9168faSHans Rosenfeld * 1403c9168faSHans Rosenfeld * The following driver properties can be changed to control some aspects of the 1413c9168faSHans Rosenfeld * drivers operation: 1423c9168faSHans Rosenfeld * - strict-version: can be set to 0 to allow devices conforming to newer 1433c9168faSHans Rosenfeld * versions to be used 1443c9168faSHans Rosenfeld * - ignore-unknown-vendor-status: can be set to 1 to not handle any vendor 1453c9168faSHans Rosenfeld * specific command status as a fatal error leading device faulting 1463c9168faSHans Rosenfeld * - admin-queue-len: the maximum length of the admin queue (16-4096) 1473c9168faSHans Rosenfeld * - io-queue-len: the maximum length of the I/O queues (16-65536) 1483c9168faSHans Rosenfeld * - async-event-limit: the maximum number of asynchronous event requests to be 1493c9168faSHans Rosenfeld * posted by the driver 1503c9168faSHans Rosenfeld * 1513c9168faSHans Rosenfeld * 1523c9168faSHans Rosenfeld * TODO: 1533c9168faSHans Rosenfeld * - figure out sane default for I/O queue depth reported to blkdev 1543c9168faSHans Rosenfeld * - polled I/O support to support kernel core dumping 1553c9168faSHans Rosenfeld * - FMA handling of media errors 1563c9168faSHans Rosenfeld * - support for the Volatile Write Cache 1573c9168faSHans Rosenfeld * - support for devices supporting very large I/O requests using chained PRPs 1583c9168faSHans Rosenfeld * - support for querying log pages from user space 1593c9168faSHans Rosenfeld * - support for configuring hardware parameters like interrupt coalescing 1603c9168faSHans Rosenfeld * - support for media formatting and hard partitioning into namespaces 1613c9168faSHans Rosenfeld * - support for big-endian systems 1623c9168faSHans Rosenfeld * - support for fast reboot 1633c9168faSHans Rosenfeld */ 1643c9168faSHans Rosenfeld 1653c9168faSHans Rosenfeld #include <sys/byteorder.h> 1663c9168faSHans Rosenfeld #ifdef _BIG_ENDIAN 1673c9168faSHans Rosenfeld #error nvme driver needs porting for big-endian platforms 1683c9168faSHans Rosenfeld #endif 1693c9168faSHans Rosenfeld 1703c9168faSHans Rosenfeld #include <sys/modctl.h> 1713c9168faSHans Rosenfeld #include <sys/conf.h> 1723c9168faSHans Rosenfeld #include <sys/devops.h> 1733c9168faSHans Rosenfeld #include <sys/ddi.h> 1743c9168faSHans Rosenfeld #include <sys/sunddi.h> 1753c9168faSHans Rosenfeld #include <sys/bitmap.h> 1763c9168faSHans Rosenfeld #include <sys/sysmacros.h> 1773c9168faSHans Rosenfeld #include <sys/param.h> 1783c9168faSHans Rosenfeld #include <sys/varargs.h> 1793c9168faSHans Rosenfeld #include <sys/cpuvar.h> 1803c9168faSHans Rosenfeld #include <sys/disp.h> 1813c9168faSHans Rosenfeld #include <sys/blkdev.h> 1823c9168faSHans Rosenfeld #include <sys/atomic.h> 1833c9168faSHans Rosenfeld #include <sys/archsystm.h> 184510a6847SHans Rosenfeld #include <sys/sata/sata_hba.h> 1853c9168faSHans Rosenfeld 1863c9168faSHans Rosenfeld #include "nvme_reg.h" 1873c9168faSHans Rosenfeld #include "nvme_var.h" 1883c9168faSHans Rosenfeld 1893c9168faSHans Rosenfeld 1903c9168faSHans Rosenfeld /* NVMe spec version supported */ 1913c9168faSHans Rosenfeld static const int nvme_version_major = 1; 1923c9168faSHans Rosenfeld static const int nvme_version_minor = 0; 1933c9168faSHans Rosenfeld 194e8ba2a38SHans Rosenfeld /* tunable for admin command timeout in seconds, default is 1s */ 195e8ba2a38SHans Rosenfeld static volatile int nvme_admin_cmd_timeout = 1; 196e8ba2a38SHans Rosenfeld 1973c9168faSHans Rosenfeld static int nvme_attach(dev_info_t *, ddi_attach_cmd_t); 1983c9168faSHans Rosenfeld static int nvme_detach(dev_info_t *, ddi_detach_cmd_t); 1993c9168faSHans Rosenfeld static int nvme_quiesce(dev_info_t *); 2003c9168faSHans Rosenfeld static int nvme_fm_errcb(dev_info_t *, ddi_fm_error_t *, const void *); 2013c9168faSHans Rosenfeld static int nvme_setup_interrupts(nvme_t *, int, int); 2023c9168faSHans Rosenfeld static void nvme_release_interrupts(nvme_t *); 2033c9168faSHans Rosenfeld static uint_t nvme_intr(caddr_t, caddr_t); 2043c9168faSHans Rosenfeld 2053c9168faSHans Rosenfeld static void nvme_shutdown(nvme_t *, int, boolean_t); 2063c9168faSHans Rosenfeld static boolean_t nvme_reset(nvme_t *, boolean_t); 2073c9168faSHans Rosenfeld static int nvme_init(nvme_t *); 2083c9168faSHans Rosenfeld static nvme_cmd_t *nvme_alloc_cmd(nvme_t *, int); 2093c9168faSHans Rosenfeld static void nvme_free_cmd(nvme_cmd_t *); 2103c9168faSHans Rosenfeld static nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t, 2113c9168faSHans Rosenfeld bd_xfer_t *); 2123c9168faSHans Rosenfeld static int nvme_admin_cmd(nvme_cmd_t *, int); 2133c9168faSHans Rosenfeld static int nvme_submit_cmd(nvme_qpair_t *, nvme_cmd_t *); 2143c9168faSHans Rosenfeld static nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *); 2153c9168faSHans Rosenfeld static boolean_t nvme_wait_cmd(nvme_cmd_t *, uint_t); 2163c9168faSHans Rosenfeld static void nvme_wakeup_cmd(void *); 2173c9168faSHans Rosenfeld static void nvme_async_event_task(void *); 2183c9168faSHans Rosenfeld 2193c9168faSHans Rosenfeld static int nvme_check_unknown_cmd_status(nvme_cmd_t *); 2203c9168faSHans Rosenfeld static int nvme_check_vendor_cmd_status(nvme_cmd_t *); 2213c9168faSHans Rosenfeld static int nvme_check_integrity_cmd_status(nvme_cmd_t *); 2223c9168faSHans Rosenfeld static int nvme_check_specific_cmd_status(nvme_cmd_t *); 2233c9168faSHans Rosenfeld static int nvme_check_generic_cmd_status(nvme_cmd_t *); 2243c9168faSHans Rosenfeld static inline int nvme_check_cmd_status(nvme_cmd_t *); 2253c9168faSHans Rosenfeld 2263c9168faSHans Rosenfeld static void nvme_abort_cmd(nvme_cmd_t *); 2273c9168faSHans Rosenfeld static int nvme_async_event(nvme_t *); 2283c9168faSHans Rosenfeld static void *nvme_get_logpage(nvme_t *, uint8_t, ...); 2293c9168faSHans Rosenfeld static void *nvme_identify(nvme_t *, uint32_t); 2303c9168faSHans Rosenfeld static int nvme_set_nqueues(nvme_t *, uint16_t); 2313c9168faSHans Rosenfeld 2323c9168faSHans Rosenfeld static void nvme_free_dma(nvme_dma_t *); 2333c9168faSHans Rosenfeld static int nvme_zalloc_dma(nvme_t *, size_t, uint_t, ddi_dma_attr_t *, 2343c9168faSHans Rosenfeld nvme_dma_t **); 2353c9168faSHans Rosenfeld static int nvme_zalloc_queue_dma(nvme_t *, uint32_t, uint16_t, uint_t, 2363c9168faSHans Rosenfeld nvme_dma_t **); 2373c9168faSHans Rosenfeld static void nvme_free_qpair(nvme_qpair_t *); 2383c9168faSHans Rosenfeld static int nvme_alloc_qpair(nvme_t *, uint32_t, nvme_qpair_t **, int); 2393c9168faSHans Rosenfeld static int nvme_create_io_qpair(nvme_t *, nvme_qpair_t *, uint16_t); 2403c9168faSHans Rosenfeld 2413c9168faSHans Rosenfeld static inline void nvme_put64(nvme_t *, uintptr_t, uint64_t); 2423c9168faSHans Rosenfeld static inline void nvme_put32(nvme_t *, uintptr_t, uint32_t); 2433c9168faSHans Rosenfeld static inline uint64_t nvme_get64(nvme_t *, uintptr_t); 2443c9168faSHans Rosenfeld static inline uint32_t nvme_get32(nvme_t *, uintptr_t); 2453c9168faSHans Rosenfeld 2463c9168faSHans Rosenfeld static boolean_t nvme_check_regs_hdl(nvme_t *); 2473c9168faSHans Rosenfeld static boolean_t nvme_check_dma_hdl(nvme_dma_t *); 2483c9168faSHans Rosenfeld 2493c9168faSHans Rosenfeld static int nvme_fill_prp(nvme_cmd_t *, bd_xfer_t *); 2503c9168faSHans Rosenfeld 2513c9168faSHans Rosenfeld static void nvme_bd_xfer_done(void *); 2523c9168faSHans Rosenfeld static void nvme_bd_driveinfo(void *, bd_drive_t *); 2533c9168faSHans Rosenfeld static int nvme_bd_mediainfo(void *, bd_media_t *); 2543c9168faSHans Rosenfeld static int nvme_bd_cmd(nvme_namespace_t *, bd_xfer_t *, uint8_t); 2553c9168faSHans Rosenfeld static int nvme_bd_read(void *, bd_xfer_t *); 2563c9168faSHans Rosenfeld static int nvme_bd_write(void *, bd_xfer_t *); 2573c9168faSHans Rosenfeld static int nvme_bd_sync(void *, bd_xfer_t *); 2583c9168faSHans Rosenfeld static int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *); 2593c9168faSHans Rosenfeld 2608834f7acSYouzhong Yang static int nvme_prp_dma_constructor(void *, void *, int); 2618834f7acSYouzhong Yang static void nvme_prp_dma_destructor(void *, void *); 2628834f7acSYouzhong Yang 2633c9168faSHans Rosenfeld static void nvme_prepare_devid(nvme_t *, uint32_t); 2643c9168faSHans Rosenfeld 2653c9168faSHans Rosenfeld static void *nvme_state; 2663c9168faSHans Rosenfeld static kmem_cache_t *nvme_cmd_cache; 2673c9168faSHans Rosenfeld 2683c9168faSHans Rosenfeld /* 2693c9168faSHans Rosenfeld * DMA attributes for queue DMA memory 2703c9168faSHans Rosenfeld * 2713c9168faSHans Rosenfeld * Queue DMA memory must be page aligned. The maximum length of a queue is 2723c9168faSHans Rosenfeld * 65536 entries, and an entry can be 64 bytes long. 2733c9168faSHans Rosenfeld */ 2743c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_queue_dma_attr = { 2753c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 2763c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 2773c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 278910f0d12SYouzhong Yang .dma_attr_count_max = (UINT16_MAX + 1) * sizeof (nvme_sqe_t) - 1, 2793c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 2803c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 2813c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 2823c9168faSHans Rosenfeld .dma_attr_maxxfer = (UINT16_MAX + 1) * sizeof (nvme_sqe_t), 2833c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 2843c9168faSHans Rosenfeld .dma_attr_sgllen = 1, 2853c9168faSHans Rosenfeld .dma_attr_granular = 1, 2863c9168faSHans Rosenfeld .dma_attr_flags = 0, 2873c9168faSHans Rosenfeld }; 2883c9168faSHans Rosenfeld 2893c9168faSHans Rosenfeld /* 2903c9168faSHans Rosenfeld * DMA attributes for transfers using Physical Region Page (PRP) entries 2913c9168faSHans Rosenfeld * 2923c9168faSHans Rosenfeld * A PRP entry describes one page of DMA memory using the page size specified 2933c9168faSHans Rosenfeld * in the controller configuration's memory page size register (CC.MPS). It uses 2943c9168faSHans Rosenfeld * a 64bit base address aligned to this page size. There is no limitation on 2953c9168faSHans Rosenfeld * chaining PRPs together for arbitrarily large DMA transfers. 2963c9168faSHans Rosenfeld */ 2973c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_prp_dma_attr = { 2983c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 2993c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 3003c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 3013c9168faSHans Rosenfeld .dma_attr_count_max = 0xfff, 3023c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 3033c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 3043c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 3053c9168faSHans Rosenfeld .dma_attr_maxxfer = 0x1000, 3062f95345bSYouzhong Yang .dma_attr_seg = 0xfff, 3073c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 3083c9168faSHans Rosenfeld .dma_attr_granular = 1, 3093c9168faSHans Rosenfeld .dma_attr_flags = 0, 3103c9168faSHans Rosenfeld }; 3113c9168faSHans Rosenfeld 3123c9168faSHans Rosenfeld /* 3133c9168faSHans Rosenfeld * DMA attributes for transfers using scatter/gather lists 3143c9168faSHans Rosenfeld * 3153c9168faSHans Rosenfeld * A SGL entry describes a chunk of DMA memory using a 64bit base address and a 3163c9168faSHans Rosenfeld * 32bit length field. SGL Segment and SGL Last Segment entries require the 3173c9168faSHans Rosenfeld * length to be a multiple of 16 bytes. 3183c9168faSHans Rosenfeld */ 3193c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_sgl_dma_attr = { 3203c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 3213c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 3223c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 3233c9168faSHans Rosenfeld .dma_attr_count_max = 0xffffffffUL, 3243c9168faSHans Rosenfeld .dma_attr_align = 1, 3253c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 3263c9168faSHans Rosenfeld .dma_attr_minxfer = 0x10, 3273c9168faSHans Rosenfeld .dma_attr_maxxfer = 0xfffffffffULL, 3283c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 3293c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 3303c9168faSHans Rosenfeld .dma_attr_granular = 0x10, 3313c9168faSHans Rosenfeld .dma_attr_flags = 0 3323c9168faSHans Rosenfeld }; 3333c9168faSHans Rosenfeld 3343c9168faSHans Rosenfeld static ddi_device_acc_attr_t nvme_reg_acc_attr = { 3353c9168faSHans Rosenfeld .devacc_attr_version = DDI_DEVICE_ATTR_V0, 3363c9168faSHans Rosenfeld .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 3373c9168faSHans Rosenfeld .devacc_attr_dataorder = DDI_STRICTORDER_ACC 3383c9168faSHans Rosenfeld }; 3393c9168faSHans Rosenfeld 3403c9168faSHans Rosenfeld static struct dev_ops nvme_dev_ops = { 3413c9168faSHans Rosenfeld .devo_rev = DEVO_REV, 3423c9168faSHans Rosenfeld .devo_refcnt = 0, 3433c9168faSHans Rosenfeld .devo_getinfo = ddi_no_info, 3443c9168faSHans Rosenfeld .devo_identify = nulldev, 3453c9168faSHans Rosenfeld .devo_probe = nulldev, 3463c9168faSHans Rosenfeld .devo_attach = nvme_attach, 3473c9168faSHans Rosenfeld .devo_detach = nvme_detach, 3483c9168faSHans Rosenfeld .devo_reset = nodev, 3493c9168faSHans Rosenfeld .devo_cb_ops = NULL, 3503c9168faSHans Rosenfeld .devo_bus_ops = NULL, 3513c9168faSHans Rosenfeld .devo_power = NULL, 3523c9168faSHans Rosenfeld .devo_quiesce = nvme_quiesce, 3533c9168faSHans Rosenfeld }; 3543c9168faSHans Rosenfeld 3553c9168faSHans Rosenfeld static struct modldrv nvme_modldrv = { 3563c9168faSHans Rosenfeld .drv_modops = &mod_driverops, 3573c9168faSHans Rosenfeld .drv_linkinfo = "NVMe v1.0e", 3583c9168faSHans Rosenfeld .drv_dev_ops = &nvme_dev_ops 3593c9168faSHans Rosenfeld }; 3603c9168faSHans Rosenfeld 3613c9168faSHans Rosenfeld static struct modlinkage nvme_modlinkage = { 3623c9168faSHans Rosenfeld .ml_rev = MODREV_1, 3633c9168faSHans Rosenfeld .ml_linkage = { &nvme_modldrv, NULL } 3643c9168faSHans Rosenfeld }; 3653c9168faSHans Rosenfeld 3663c9168faSHans Rosenfeld static bd_ops_t nvme_bd_ops = { 3673c9168faSHans Rosenfeld .o_version = BD_OPS_VERSION_0, 3683c9168faSHans Rosenfeld .o_drive_info = nvme_bd_driveinfo, 3693c9168faSHans Rosenfeld .o_media_info = nvme_bd_mediainfo, 3703c9168faSHans Rosenfeld .o_devid_init = nvme_bd_devid, 3713c9168faSHans Rosenfeld .o_sync_cache = nvme_bd_sync, 3723c9168faSHans Rosenfeld .o_read = nvme_bd_read, 3733c9168faSHans Rosenfeld .o_write = nvme_bd_write, 3743c9168faSHans Rosenfeld }; 3753c9168faSHans Rosenfeld 3763c9168faSHans Rosenfeld int 3773c9168faSHans Rosenfeld _init(void) 3783c9168faSHans Rosenfeld { 3793c9168faSHans Rosenfeld int error; 3803c9168faSHans Rosenfeld 3813c9168faSHans Rosenfeld error = ddi_soft_state_init(&nvme_state, sizeof (nvme_t), 1); 3823c9168faSHans Rosenfeld if (error != DDI_SUCCESS) 3833c9168faSHans Rosenfeld return (error); 3843c9168faSHans Rosenfeld 3853c9168faSHans Rosenfeld nvme_cmd_cache = kmem_cache_create("nvme_cmd_cache", 3863c9168faSHans Rosenfeld sizeof (nvme_cmd_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 3873c9168faSHans Rosenfeld 3883c9168faSHans Rosenfeld bd_mod_init(&nvme_dev_ops); 3893c9168faSHans Rosenfeld 3903c9168faSHans Rosenfeld error = mod_install(&nvme_modlinkage); 3913c9168faSHans Rosenfeld if (error != DDI_SUCCESS) { 3923c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 3933c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 3943c9168faSHans Rosenfeld } 3953c9168faSHans Rosenfeld 3963c9168faSHans Rosenfeld return (error); 3973c9168faSHans Rosenfeld } 3983c9168faSHans Rosenfeld 3993c9168faSHans Rosenfeld int 4003c9168faSHans Rosenfeld _fini(void) 4013c9168faSHans Rosenfeld { 4023c9168faSHans Rosenfeld int error; 4033c9168faSHans Rosenfeld 4043c9168faSHans Rosenfeld error = mod_remove(&nvme_modlinkage); 4053c9168faSHans Rosenfeld if (error == DDI_SUCCESS) { 4063c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 4073c9168faSHans Rosenfeld kmem_cache_destroy(nvme_cmd_cache); 4083c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 4093c9168faSHans Rosenfeld } 4103c9168faSHans Rosenfeld 4113c9168faSHans Rosenfeld return (error); 4123c9168faSHans Rosenfeld } 4133c9168faSHans Rosenfeld 4143c9168faSHans Rosenfeld int 4153c9168faSHans Rosenfeld _info(struct modinfo *modinfop) 4163c9168faSHans Rosenfeld { 4173c9168faSHans Rosenfeld return (mod_info(&nvme_modlinkage, modinfop)); 4183c9168faSHans Rosenfeld } 4193c9168faSHans Rosenfeld 4203c9168faSHans Rosenfeld static inline void 4213c9168faSHans Rosenfeld nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val) 4223c9168faSHans Rosenfeld { 4233c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 4243c9168faSHans Rosenfeld 4253c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4263c9168faSHans Rosenfeld ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val); 4273c9168faSHans Rosenfeld } 4283c9168faSHans Rosenfeld 4293c9168faSHans Rosenfeld static inline void 4303c9168faSHans Rosenfeld nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val) 4313c9168faSHans Rosenfeld { 4323c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 4333c9168faSHans Rosenfeld 4343c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4353c9168faSHans Rosenfeld ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val); 4363c9168faSHans Rosenfeld } 4373c9168faSHans Rosenfeld 4383c9168faSHans Rosenfeld static inline uint64_t 4393c9168faSHans Rosenfeld nvme_get64(nvme_t *nvme, uintptr_t reg) 4403c9168faSHans Rosenfeld { 4413c9168faSHans Rosenfeld uint64_t val; 4423c9168faSHans Rosenfeld 4433c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 4443c9168faSHans Rosenfeld 4453c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4463c9168faSHans Rosenfeld val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg)); 4473c9168faSHans Rosenfeld 4483c9168faSHans Rosenfeld return (val); 4493c9168faSHans Rosenfeld } 4503c9168faSHans Rosenfeld 4513c9168faSHans Rosenfeld static inline uint32_t 4523c9168faSHans Rosenfeld nvme_get32(nvme_t *nvme, uintptr_t reg) 4533c9168faSHans Rosenfeld { 4543c9168faSHans Rosenfeld uint32_t val; 4553c9168faSHans Rosenfeld 4563c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 4573c9168faSHans Rosenfeld 4583c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4593c9168faSHans Rosenfeld val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg)); 4603c9168faSHans Rosenfeld 4613c9168faSHans Rosenfeld return (val); 4623c9168faSHans Rosenfeld } 4633c9168faSHans Rosenfeld 4643c9168faSHans Rosenfeld static boolean_t 4653c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme_t *nvme) 4663c9168faSHans Rosenfeld { 4673c9168faSHans Rosenfeld ddi_fm_error_t error; 4683c9168faSHans Rosenfeld 4693c9168faSHans Rosenfeld ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION); 4703c9168faSHans Rosenfeld 4713c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 4723c9168faSHans Rosenfeld return (B_TRUE); 4733c9168faSHans Rosenfeld 4743c9168faSHans Rosenfeld return (B_FALSE); 4753c9168faSHans Rosenfeld } 4763c9168faSHans Rosenfeld 4773c9168faSHans Rosenfeld static boolean_t 4783c9168faSHans Rosenfeld nvme_check_dma_hdl(nvme_dma_t *dma) 4793c9168faSHans Rosenfeld { 4803c9168faSHans Rosenfeld ddi_fm_error_t error; 4813c9168faSHans Rosenfeld 4823c9168faSHans Rosenfeld if (dma == NULL) 4833c9168faSHans Rosenfeld return (B_FALSE); 4843c9168faSHans Rosenfeld 4853c9168faSHans Rosenfeld ddi_fm_dma_err_get(dma->nd_dmah, &error, DDI_FME_VERSION); 4863c9168faSHans Rosenfeld 4873c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 4883c9168faSHans Rosenfeld return (B_TRUE); 4893c9168faSHans Rosenfeld 4903c9168faSHans Rosenfeld return (B_FALSE); 4913c9168faSHans Rosenfeld } 4923c9168faSHans Rosenfeld 4933c9168faSHans Rosenfeld static void 4948834f7acSYouzhong Yang nvme_free_dma_common(nvme_dma_t *dma) 4953c9168faSHans Rosenfeld { 4963c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 4973c9168faSHans Rosenfeld (void) ddi_dma_unbind_handle(dma->nd_dmah); 4983c9168faSHans Rosenfeld if (dma->nd_acch != NULL) 4993c9168faSHans Rosenfeld ddi_dma_mem_free(&dma->nd_acch); 5003c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 5013c9168faSHans Rosenfeld ddi_dma_free_handle(&dma->nd_dmah); 5023c9168faSHans Rosenfeld } 5033c9168faSHans Rosenfeld 5048834f7acSYouzhong Yang static void 5058834f7acSYouzhong Yang nvme_free_dma(nvme_dma_t *dma) 5063c9168faSHans Rosenfeld { 5078834f7acSYouzhong Yang nvme_free_dma_common(dma); 5088834f7acSYouzhong Yang kmem_free(dma, sizeof (*dma)); 5098834f7acSYouzhong Yang } 5108834f7acSYouzhong Yang 511*b6bc2fd4SDan McDonald /* ARGSUSED */ 5128834f7acSYouzhong Yang static void 5138834f7acSYouzhong Yang nvme_prp_dma_destructor(void *buf, void *private) 5148834f7acSYouzhong Yang { 5158834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 5163c9168faSHans Rosenfeld 5178834f7acSYouzhong Yang nvme_free_dma_common(dma); 5188834f7acSYouzhong Yang } 5198834f7acSYouzhong Yang 5208834f7acSYouzhong Yang static int 5218834f7acSYouzhong Yang nvme_alloc_dma_common(nvme_t *nvme, nvme_dma_t *dma, 5228834f7acSYouzhong Yang size_t len, uint_t flags, ddi_dma_attr_t *dma_attr) 5238834f7acSYouzhong Yang { 5243c9168faSHans Rosenfeld if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL, 5253c9168faSHans Rosenfeld &dma->nd_dmah) != DDI_SUCCESS) { 5263c9168faSHans Rosenfeld /* 5273c9168faSHans Rosenfeld * Due to DDI_DMA_SLEEP this can't be DDI_DMA_NORESOURCES, and 5283c9168faSHans Rosenfeld * the only other possible error is DDI_DMA_BADATTR which 5293c9168faSHans Rosenfeld * indicates a driver bug which should cause a panic. 5303c9168faSHans Rosenfeld */ 5313c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, 5323c9168faSHans Rosenfeld "!failed to get DMA handle, check DMA attributes"); 5333c9168faSHans Rosenfeld return (DDI_FAILURE); 5343c9168faSHans Rosenfeld } 5353c9168faSHans Rosenfeld 5363c9168faSHans Rosenfeld /* 5373c9168faSHans Rosenfeld * ddi_dma_mem_alloc() can only fail when DDI_DMA_NOSLEEP is specified 5383c9168faSHans Rosenfeld * or the flags are conflicting, which isn't the case here. 5393c9168faSHans Rosenfeld */ 5403c9168faSHans Rosenfeld (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr, 5413c9168faSHans Rosenfeld DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &dma->nd_memp, 5423c9168faSHans Rosenfeld &dma->nd_len, &dma->nd_acch); 5433c9168faSHans Rosenfeld 5443c9168faSHans Rosenfeld if (ddi_dma_addr_bind_handle(dma->nd_dmah, NULL, dma->nd_memp, 5453c9168faSHans Rosenfeld dma->nd_len, flags | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, 5463c9168faSHans Rosenfeld &dma->nd_cookie, &dma->nd_ncookie) != DDI_DMA_MAPPED) { 5473c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 5483c9168faSHans Rosenfeld "!failed to bind DMA memory"); 5493c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_dma_bind_err); 5508834f7acSYouzhong Yang nvme_free_dma_common(dma); 5518834f7acSYouzhong Yang return (DDI_FAILURE); 5528834f7acSYouzhong Yang } 5538834f7acSYouzhong Yang 5548834f7acSYouzhong Yang return (DDI_SUCCESS); 5558834f7acSYouzhong Yang } 5568834f7acSYouzhong Yang 5578834f7acSYouzhong Yang static int 5588834f7acSYouzhong Yang nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags, 5598834f7acSYouzhong Yang ddi_dma_attr_t *dma_attr, nvme_dma_t **ret) 5608834f7acSYouzhong Yang { 5618834f7acSYouzhong Yang nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP); 5628834f7acSYouzhong Yang 5638834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, len, flags, dma_attr) != 5648834f7acSYouzhong Yang DDI_SUCCESS) { 5653c9168faSHans Rosenfeld *ret = NULL; 5668834f7acSYouzhong Yang kmem_free(dma, sizeof (nvme_dma_t)); 5673c9168faSHans Rosenfeld return (DDI_FAILURE); 5683c9168faSHans Rosenfeld } 5693c9168faSHans Rosenfeld 5703c9168faSHans Rosenfeld bzero(dma->nd_memp, dma->nd_len); 5713c9168faSHans Rosenfeld 5723c9168faSHans Rosenfeld *ret = dma; 5733c9168faSHans Rosenfeld return (DDI_SUCCESS); 5743c9168faSHans Rosenfeld } 5753c9168faSHans Rosenfeld 576*b6bc2fd4SDan McDonald /* ARGSUSED */ 5778834f7acSYouzhong Yang static int 5788834f7acSYouzhong Yang nvme_prp_dma_constructor(void *buf, void *private, int flags) 5798834f7acSYouzhong Yang { 5808834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 5818834f7acSYouzhong Yang nvme_t *nvme = (nvme_t *)private; 5828834f7acSYouzhong Yang 5838834f7acSYouzhong Yang dma->nd_dmah = NULL; 5848834f7acSYouzhong Yang dma->nd_acch = NULL; 5858834f7acSYouzhong Yang 5868834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, nvme->n_pagesize, 5878834f7acSYouzhong Yang DDI_DMA_READ, &nvme->n_prp_dma_attr) != DDI_SUCCESS) { 5888834f7acSYouzhong Yang return (-1); 5898834f7acSYouzhong Yang } 5908834f7acSYouzhong Yang 5918834f7acSYouzhong Yang ASSERT(dma->nd_ncookie == 1); 5928834f7acSYouzhong Yang 5938834f7acSYouzhong Yang dma->nd_cached = B_TRUE; 5948834f7acSYouzhong Yang 5958834f7acSYouzhong Yang return (0); 5968834f7acSYouzhong Yang } 5978834f7acSYouzhong Yang 5983c9168faSHans Rosenfeld static int 5993c9168faSHans Rosenfeld nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len, 6003c9168faSHans Rosenfeld uint_t flags, nvme_dma_t **dma) 6013c9168faSHans Rosenfeld { 6023c9168faSHans Rosenfeld uint32_t len = nentry * qe_len; 6033c9168faSHans Rosenfeld ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr; 6043c9168faSHans Rosenfeld 6053c9168faSHans Rosenfeld len = roundup(len, nvme->n_pagesize); 6063c9168faSHans Rosenfeld 6073c9168faSHans Rosenfeld q_dma_attr.dma_attr_minxfer = len; 6083c9168faSHans Rosenfeld 6093c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma) 6103c9168faSHans Rosenfeld != DDI_SUCCESS) { 6113c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 6123c9168faSHans Rosenfeld "!failed to get DMA memory for queue"); 6133c9168faSHans Rosenfeld goto fail; 6143c9168faSHans Rosenfeld } 6153c9168faSHans Rosenfeld 6163c9168faSHans Rosenfeld if ((*dma)->nd_ncookie != 1) { 6173c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 6183c9168faSHans Rosenfeld "!got too many cookies for queue DMA"); 6193c9168faSHans Rosenfeld goto fail; 6203c9168faSHans Rosenfeld } 6213c9168faSHans Rosenfeld 6223c9168faSHans Rosenfeld return (DDI_SUCCESS); 6233c9168faSHans Rosenfeld 6243c9168faSHans Rosenfeld fail: 6253c9168faSHans Rosenfeld if (*dma) { 6263c9168faSHans Rosenfeld nvme_free_dma(*dma); 6273c9168faSHans Rosenfeld *dma = NULL; 6283c9168faSHans Rosenfeld } 6293c9168faSHans Rosenfeld 6303c9168faSHans Rosenfeld return (DDI_FAILURE); 6313c9168faSHans Rosenfeld } 6323c9168faSHans Rosenfeld 6333c9168faSHans Rosenfeld static void 6343c9168faSHans Rosenfeld nvme_free_qpair(nvme_qpair_t *qp) 6353c9168faSHans Rosenfeld { 6363c9168faSHans Rosenfeld int i; 6373c9168faSHans Rosenfeld 6383c9168faSHans Rosenfeld mutex_destroy(&qp->nq_mutex); 6393c9168faSHans Rosenfeld 6403c9168faSHans Rosenfeld if (qp->nq_sqdma != NULL) 6413c9168faSHans Rosenfeld nvme_free_dma(qp->nq_sqdma); 6423c9168faSHans Rosenfeld if (qp->nq_cqdma != NULL) 6433c9168faSHans Rosenfeld nvme_free_dma(qp->nq_cqdma); 6443c9168faSHans Rosenfeld 6453c9168faSHans Rosenfeld if (qp->nq_active_cmds > 0) 6463c9168faSHans Rosenfeld for (i = 0; i != qp->nq_nentry; i++) 6473c9168faSHans Rosenfeld if (qp->nq_cmd[i] != NULL) 6483c9168faSHans Rosenfeld nvme_free_cmd(qp->nq_cmd[i]); 6493c9168faSHans Rosenfeld 6503c9168faSHans Rosenfeld if (qp->nq_cmd != NULL) 6513c9168faSHans Rosenfeld kmem_free(qp->nq_cmd, sizeof (nvme_cmd_t *) * qp->nq_nentry); 6523c9168faSHans Rosenfeld 6533c9168faSHans Rosenfeld kmem_free(qp, sizeof (nvme_qpair_t)); 6543c9168faSHans Rosenfeld } 6553c9168faSHans Rosenfeld 6563c9168faSHans Rosenfeld static int 6573c9168faSHans Rosenfeld nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp, 6583c9168faSHans Rosenfeld int idx) 6593c9168faSHans Rosenfeld { 6603c9168faSHans Rosenfeld nvme_qpair_t *qp = kmem_zalloc(sizeof (*qp), KM_SLEEP); 6613c9168faSHans Rosenfeld 6623c9168faSHans Rosenfeld mutex_init(&qp->nq_mutex, NULL, MUTEX_DRIVER, 6633c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 6643c9168faSHans Rosenfeld 6653c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t), 6663c9168faSHans Rosenfeld DDI_DMA_WRITE, &qp->nq_sqdma) != DDI_SUCCESS) 6673c9168faSHans Rosenfeld goto fail; 6683c9168faSHans Rosenfeld 6693c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t), 6703c9168faSHans Rosenfeld DDI_DMA_READ, &qp->nq_cqdma) != DDI_SUCCESS) 6713c9168faSHans Rosenfeld goto fail; 6723c9168faSHans Rosenfeld 6733c9168faSHans Rosenfeld qp->nq_sq = (nvme_sqe_t *)qp->nq_sqdma->nd_memp; 6743c9168faSHans Rosenfeld qp->nq_cq = (nvme_cqe_t *)qp->nq_cqdma->nd_memp; 6753c9168faSHans Rosenfeld qp->nq_nentry = nentry; 6763c9168faSHans Rosenfeld 6773c9168faSHans Rosenfeld qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx); 6783c9168faSHans Rosenfeld qp->nq_cqhdbl = NVME_REG_CQHDBL(nvme, idx); 6793c9168faSHans Rosenfeld 6803c9168faSHans Rosenfeld qp->nq_cmd = kmem_zalloc(sizeof (nvme_cmd_t *) * nentry, KM_SLEEP); 6813c9168faSHans Rosenfeld qp->nq_next_cmd = 0; 6823c9168faSHans Rosenfeld 6833c9168faSHans Rosenfeld *nqp = qp; 6843c9168faSHans Rosenfeld return (DDI_SUCCESS); 6853c9168faSHans Rosenfeld 6863c9168faSHans Rosenfeld fail: 6873c9168faSHans Rosenfeld nvme_free_qpair(qp); 6883c9168faSHans Rosenfeld *nqp = NULL; 6893c9168faSHans Rosenfeld 6903c9168faSHans Rosenfeld return (DDI_FAILURE); 6913c9168faSHans Rosenfeld } 6923c9168faSHans Rosenfeld 6933c9168faSHans Rosenfeld static nvme_cmd_t * 6943c9168faSHans Rosenfeld nvme_alloc_cmd(nvme_t *nvme, int kmflag) 6953c9168faSHans Rosenfeld { 6963c9168faSHans Rosenfeld nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag); 6973c9168faSHans Rosenfeld 6983c9168faSHans Rosenfeld if (cmd == NULL) 6993c9168faSHans Rosenfeld return (cmd); 7003c9168faSHans Rosenfeld 7013c9168faSHans Rosenfeld bzero(cmd, sizeof (nvme_cmd_t)); 7023c9168faSHans Rosenfeld 7033c9168faSHans Rosenfeld cmd->nc_nvme = nvme; 7043c9168faSHans Rosenfeld 7053c9168faSHans Rosenfeld mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER, 7063c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 7073c9168faSHans Rosenfeld cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL); 7083c9168faSHans Rosenfeld 7093c9168faSHans Rosenfeld return (cmd); 7103c9168faSHans Rosenfeld } 7113c9168faSHans Rosenfeld 7123c9168faSHans Rosenfeld static void 7133c9168faSHans Rosenfeld nvme_free_cmd(nvme_cmd_t *cmd) 7143c9168faSHans Rosenfeld { 7153c9168faSHans Rosenfeld if (cmd->nc_dma) { 7168834f7acSYouzhong Yang if (cmd->nc_dma->nd_cached) 7178834f7acSYouzhong Yang kmem_cache_free(cmd->nc_nvme->n_prp_cache, 7188834f7acSYouzhong Yang cmd->nc_dma); 7198834f7acSYouzhong Yang else 7208834f7acSYouzhong Yang nvme_free_dma(cmd->nc_dma); 7213c9168faSHans Rosenfeld cmd->nc_dma = NULL; 7223c9168faSHans Rosenfeld } 7233c9168faSHans Rosenfeld 7243c9168faSHans Rosenfeld cv_destroy(&cmd->nc_cv); 7253c9168faSHans Rosenfeld mutex_destroy(&cmd->nc_mutex); 7263c9168faSHans Rosenfeld 7273c9168faSHans Rosenfeld kmem_cache_free(nvme_cmd_cache, cmd); 7283c9168faSHans Rosenfeld } 7293c9168faSHans Rosenfeld 7303c9168faSHans Rosenfeld static int 7313c9168faSHans Rosenfeld nvme_submit_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 7323c9168faSHans Rosenfeld { 7333c9168faSHans Rosenfeld nvme_reg_sqtdbl_t tail = { 0 }; 7343c9168faSHans Rosenfeld 7353c9168faSHans Rosenfeld mutex_enter(&qp->nq_mutex); 7363c9168faSHans Rosenfeld 7373c9168faSHans Rosenfeld if (qp->nq_active_cmds == qp->nq_nentry) { 7383c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 7393c9168faSHans Rosenfeld return (DDI_FAILURE); 7403c9168faSHans Rosenfeld } 7413c9168faSHans Rosenfeld 7423c9168faSHans Rosenfeld cmd->nc_completed = B_FALSE; 7433c9168faSHans Rosenfeld 7443c9168faSHans Rosenfeld /* 7453c9168faSHans Rosenfeld * Try to insert the cmd into the active cmd array at the nq_next_cmd 7463c9168faSHans Rosenfeld * slot. If the slot is already occupied advance to the next slot and 7473c9168faSHans Rosenfeld * try again. This can happen for long running commands like async event 7483c9168faSHans Rosenfeld * requests. 7493c9168faSHans Rosenfeld */ 7503c9168faSHans Rosenfeld while (qp->nq_cmd[qp->nq_next_cmd] != NULL) 7513c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 7523c9168faSHans Rosenfeld qp->nq_cmd[qp->nq_next_cmd] = cmd; 7533c9168faSHans Rosenfeld 7543c9168faSHans Rosenfeld qp->nq_active_cmds++; 7553c9168faSHans Rosenfeld 7563c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cid = qp->nq_next_cmd; 7573c9168faSHans Rosenfeld bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t)); 7583c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_sqdma->nd_dmah, 7593c9168faSHans Rosenfeld sizeof (nvme_sqe_t) * qp->nq_sqtail, 7603c9168faSHans Rosenfeld sizeof (nvme_sqe_t), DDI_DMA_SYNC_FORDEV); 7613c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 7623c9168faSHans Rosenfeld 7633c9168faSHans Rosenfeld tail.b.sqtdbl_sqt = qp->nq_sqtail = (qp->nq_sqtail + 1) % qp->nq_nentry; 7643c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r); 7653c9168faSHans Rosenfeld 7663c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 7673c9168faSHans Rosenfeld return (DDI_SUCCESS); 7683c9168faSHans Rosenfeld } 7693c9168faSHans Rosenfeld 7703c9168faSHans Rosenfeld static nvme_cmd_t * 7713c9168faSHans Rosenfeld nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp) 7723c9168faSHans Rosenfeld { 7733c9168faSHans Rosenfeld nvme_reg_cqhdbl_t head = { 0 }; 7743c9168faSHans Rosenfeld 7753c9168faSHans Rosenfeld nvme_cqe_t *cqe; 7763c9168faSHans Rosenfeld nvme_cmd_t *cmd; 7773c9168faSHans Rosenfeld 7783c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_cqdma->nd_dmah, 0, 7793c9168faSHans Rosenfeld sizeof (nvme_cqe_t) * qp->nq_nentry, DDI_DMA_SYNC_FORKERNEL); 7803c9168faSHans Rosenfeld 7813c9168faSHans Rosenfeld cqe = &qp->nq_cq[qp->nq_cqhead]; 7823c9168faSHans Rosenfeld 7833c9168faSHans Rosenfeld /* Check phase tag of CQE. Hardware inverts it for new entries. */ 7843c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_p == qp->nq_phase) 7853c9168faSHans Rosenfeld return (NULL); 7863c9168faSHans Rosenfeld 7873c9168faSHans Rosenfeld ASSERT(nvme->n_ioq[cqe->cqe_sqid] == qp); 7883c9168faSHans Rosenfeld ASSERT(cqe->cqe_cid < qp->nq_nentry); 7893c9168faSHans Rosenfeld 7903c9168faSHans Rosenfeld mutex_enter(&qp->nq_mutex); 7913c9168faSHans Rosenfeld cmd = qp->nq_cmd[cqe->cqe_cid]; 7923c9168faSHans Rosenfeld qp->nq_cmd[cqe->cqe_cid] = NULL; 7933c9168faSHans Rosenfeld qp->nq_active_cmds--; 7943c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 7953c9168faSHans Rosenfeld 7963c9168faSHans Rosenfeld ASSERT(cmd != NULL); 7973c9168faSHans Rosenfeld ASSERT(cmd->nc_nvme == nvme); 7983c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid == cqe->cqe_sqid); 7993c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_cid == cqe->cqe_cid); 8003c9168faSHans Rosenfeld bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t)); 8013c9168faSHans Rosenfeld 8023c9168faSHans Rosenfeld qp->nq_sqhead = cqe->cqe_sqhd; 8033c9168faSHans Rosenfeld 8043c9168faSHans Rosenfeld head.b.cqhdbl_cqh = qp->nq_cqhead = (qp->nq_cqhead + 1) % qp->nq_nentry; 8053c9168faSHans Rosenfeld 8063c9168faSHans Rosenfeld /* Toggle phase on wrap-around. */ 8073c9168faSHans Rosenfeld if (qp->nq_cqhead == 0) 8083c9168faSHans Rosenfeld qp->nq_phase = qp->nq_phase ? 0 : 1; 8093c9168faSHans Rosenfeld 8103c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_cqhdbl, head.r); 8113c9168faSHans Rosenfeld 8123c9168faSHans Rosenfeld return (cmd); 8133c9168faSHans Rosenfeld } 8143c9168faSHans Rosenfeld 8153c9168faSHans Rosenfeld static int 8163c9168faSHans Rosenfeld nvme_check_unknown_cmd_status(nvme_cmd_t *cmd) 8173c9168faSHans Rosenfeld { 8183c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8193c9168faSHans Rosenfeld 8203c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 8213c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 8223c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 8233c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 8243c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 8253c9168faSHans Rosenfeld 826bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 827bef9e21aSHans Rosenfeld 8283c9168faSHans Rosenfeld if (cmd->nc_nvme->n_strict_version) { 8293c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 8303c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 8313c9168faSHans Rosenfeld } 8323c9168faSHans Rosenfeld 8333c9168faSHans Rosenfeld return (EIO); 8343c9168faSHans Rosenfeld } 8353c9168faSHans Rosenfeld 8363c9168faSHans Rosenfeld static int 8373c9168faSHans Rosenfeld nvme_check_vendor_cmd_status(nvme_cmd_t *cmd) 8383c9168faSHans Rosenfeld { 8393c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8403c9168faSHans Rosenfeld 8413c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 8423c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 8433c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 8443c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 8453c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 84634c938c7SPete Shephard if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) { 8473c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 8483c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 8493c9168faSHans Rosenfeld } 8503c9168faSHans Rosenfeld 8513c9168faSHans Rosenfeld return (EIO); 8523c9168faSHans Rosenfeld } 8533c9168faSHans Rosenfeld 8543c9168faSHans Rosenfeld static int 8553c9168faSHans Rosenfeld nvme_check_integrity_cmd_status(nvme_cmd_t *cmd) 8563c9168faSHans Rosenfeld { 8573c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8583c9168faSHans Rosenfeld 8593c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 8603c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_WRITE: 8613c9168faSHans Rosenfeld /* write fail */ 8623c9168faSHans Rosenfeld /* TODO: post ereport */ 863bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 8643c9168faSHans Rosenfeld return (EIO); 8653c9168faSHans Rosenfeld 8663c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_READ: 8673c9168faSHans Rosenfeld /* read fail */ 8683c9168faSHans Rosenfeld /* TODO: post ereport */ 869bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 8703c9168faSHans Rosenfeld return (EIO); 8713c9168faSHans Rosenfeld 8723c9168faSHans Rosenfeld default: 8733c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 8743c9168faSHans Rosenfeld } 8753c9168faSHans Rosenfeld } 8763c9168faSHans Rosenfeld 8773c9168faSHans Rosenfeld static int 8783c9168faSHans Rosenfeld nvme_check_generic_cmd_status(nvme_cmd_t *cmd) 8793c9168faSHans Rosenfeld { 8803c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8813c9168faSHans Rosenfeld 8823c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 8833c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_SUCCESS: 8843c9168faSHans Rosenfeld return (0); 8853c9168faSHans Rosenfeld 8863c9168faSHans Rosenfeld /* 8873c9168faSHans Rosenfeld * Errors indicating a bug in the driver should cause a panic. 8883c9168faSHans Rosenfeld */ 8893c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_OPC: 8903c9168faSHans Rosenfeld /* Invalid Command Opcode */ 8913c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 8923c9168faSHans Rosenfeld "invalid opcode in cmd %p", (void *)cmd); 8933c9168faSHans Rosenfeld return (0); 8943c9168faSHans Rosenfeld 8953c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_FLD: 8963c9168faSHans Rosenfeld /* Invalid Field in Command */ 8973c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 8983c9168faSHans Rosenfeld "invalid field in cmd %p", (void *)cmd); 8993c9168faSHans Rosenfeld return (0); 9003c9168faSHans Rosenfeld 9013c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ID_CNFL: 9023c9168faSHans Rosenfeld /* Command ID Conflict */ 9033c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9043c9168faSHans Rosenfeld "cmd ID conflict in cmd %p", (void *)cmd); 9053c9168faSHans Rosenfeld return (0); 9063c9168faSHans Rosenfeld 9073c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_NS: 9083c9168faSHans Rosenfeld /* Invalid Namespace or Format */ 9093c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9103c9168faSHans Rosenfeld "invalid NS/format in cmd %p", (void *)cmd); 9113c9168faSHans Rosenfeld return (0); 9123c9168faSHans Rosenfeld 9133c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_LBA_RANGE: 9143c9168faSHans Rosenfeld /* LBA Out Of Range */ 9153c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9163c9168faSHans Rosenfeld "LBA out of range in cmd %p", (void *)cmd); 9173c9168faSHans Rosenfeld return (0); 9183c9168faSHans Rosenfeld 9193c9168faSHans Rosenfeld /* 9203c9168faSHans Rosenfeld * Non-fatal errors, handle gracefully. 9213c9168faSHans Rosenfeld */ 9223c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_DATA_XFR_ERR: 9233c9168faSHans Rosenfeld /* Data Transfer Error (DMA) */ 9243c9168faSHans Rosenfeld /* TODO: post ereport */ 9253c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err); 926bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 9273c9168faSHans Rosenfeld return (EIO); 9283c9168faSHans Rosenfeld 9293c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INTERNAL_ERR: 9303c9168faSHans Rosenfeld /* 9313c9168faSHans Rosenfeld * Internal Error. The spec (v1.0, section 4.5.1.2) says 9323c9168faSHans Rosenfeld * detailed error information is returned as async event, 9333c9168faSHans Rosenfeld * so we pretty much ignore the error here and handle it 9343c9168faSHans Rosenfeld * in the async event handler. 9353c9168faSHans Rosenfeld */ 9363c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_internal_err); 937bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 9383c9168faSHans Rosenfeld return (EIO); 9393c9168faSHans Rosenfeld 9403c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_REQUEST: 9413c9168faSHans Rosenfeld /* 9423c9168faSHans Rosenfeld * Command Abort Requested. This normally happens only when a 9433c9168faSHans Rosenfeld * command times out. 9443c9168faSHans Rosenfeld */ 9453c9168faSHans Rosenfeld /* TODO: post ereport or change blkdev to handle this? */ 9463c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err); 9473c9168faSHans Rosenfeld return (ECANCELED); 9483c9168faSHans Rosenfeld 9493c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_PWRLOSS: 9503c9168faSHans Rosenfeld /* Command Aborted due to Power Loss Notification */ 9513c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 9523c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 9533c9168faSHans Rosenfeld return (EIO); 9543c9168faSHans Rosenfeld 9553c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_SQ_DEL: 9563c9168faSHans Rosenfeld /* Command Aborted due to SQ Deletion */ 9573c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del); 9583c9168faSHans Rosenfeld return (EIO); 9593c9168faSHans Rosenfeld 9603c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_CAP_EXC: 9613c9168faSHans Rosenfeld /* Capacity Exceeded */ 9623c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc); 963bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 9643c9168faSHans Rosenfeld return (EIO); 9653c9168faSHans Rosenfeld 9663c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_NS_NOTRDY: 9673c9168faSHans Rosenfeld /* Namespace Not Ready */ 9683c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy); 969bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 9703c9168faSHans Rosenfeld return (EIO); 9713c9168faSHans Rosenfeld 9723c9168faSHans Rosenfeld default: 9733c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 9743c9168faSHans Rosenfeld } 9753c9168faSHans Rosenfeld } 9763c9168faSHans Rosenfeld 9773c9168faSHans Rosenfeld static int 9783c9168faSHans Rosenfeld nvme_check_specific_cmd_status(nvme_cmd_t *cmd) 9793c9168faSHans Rosenfeld { 9803c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 9813c9168faSHans Rosenfeld 9823c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 9833c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_CQ: 9843c9168faSHans Rosenfeld /* Completion Queue Invalid */ 9853c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE); 9863c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err); 9873c9168faSHans Rosenfeld return (EINVAL); 9883c9168faSHans Rosenfeld 9893c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_QID: 9903c9168faSHans Rosenfeld /* Invalid Queue Identifier */ 9913c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 9923c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE || 9933c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE || 9943c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 9953c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err); 9963c9168faSHans Rosenfeld return (EINVAL); 9973c9168faSHans Rosenfeld 9983c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_MAX_QSZ_EXC: 9993c9168faSHans Rosenfeld /* Max Queue Size Exceeded */ 10003c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 10013c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 10023c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc); 10033c9168faSHans Rosenfeld return (EINVAL); 10043c9168faSHans Rosenfeld 10053c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ABRT_CMD_EXC: 10063c9168faSHans Rosenfeld /* Abort Command Limit Exceeded */ 10073c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT); 10083c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 10093c9168faSHans Rosenfeld "abort command limit exceeded in cmd %p", (void *)cmd); 10103c9168faSHans Rosenfeld return (0); 10113c9168faSHans Rosenfeld 10123c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC: 10133c9168faSHans Rosenfeld /* Async Event Request Limit Exceeded */ 10143c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT); 10153c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 10163c9168faSHans Rosenfeld "async event request limit exceeded in cmd %p", 10173c9168faSHans Rosenfeld (void *)cmd); 10183c9168faSHans Rosenfeld return (0); 10193c9168faSHans Rosenfeld 10203c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_INT_VECT: 10213c9168faSHans Rosenfeld /* Invalid Interrupt Vector */ 10223c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 10233c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect); 10243c9168faSHans Rosenfeld return (EINVAL); 10253c9168faSHans Rosenfeld 10263c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_LOG_PAGE: 10273c9168faSHans Rosenfeld /* Invalid Log Page */ 10283c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE); 10293c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_log_page); 1030bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10313c9168faSHans Rosenfeld return (EINVAL); 10323c9168faSHans Rosenfeld 10333c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_FORMAT: 10343c9168faSHans Rosenfeld /* Invalid Format */ 10353c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT); 10363c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_format); 1037bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10383c9168faSHans Rosenfeld return (EINVAL); 10393c9168faSHans Rosenfeld 10403c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_Q_DEL: 10413c9168faSHans Rosenfeld /* Invalid Queue Deletion */ 10423c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 10433c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_q_del); 10443c9168faSHans Rosenfeld return (EINVAL); 10453c9168faSHans Rosenfeld 10463c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_CNFL_ATTR: 10473c9168faSHans Rosenfeld /* Conflicting Attributes */ 10483c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT || 10493c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 10503c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 10513c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr); 1052bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10533c9168faSHans Rosenfeld return (EINVAL); 10543c9168faSHans Rosenfeld 10553c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_INV_PROT: 10563c9168faSHans Rosenfeld /* Invalid Protection Information */ 10573c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE || 10583c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 10593c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 10603c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_prot); 1061bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10623c9168faSHans Rosenfeld return (EINVAL); 10633c9168faSHans Rosenfeld 10643c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_READONLY: 10653c9168faSHans Rosenfeld /* Write to Read Only Range */ 10663c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 10673c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_readonly); 1068bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10693c9168faSHans Rosenfeld return (EROFS); 10703c9168faSHans Rosenfeld 10713c9168faSHans Rosenfeld default: 10723c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 10733c9168faSHans Rosenfeld } 10743c9168faSHans Rosenfeld } 10753c9168faSHans Rosenfeld 10763c9168faSHans Rosenfeld static inline int 10773c9168faSHans Rosenfeld nvme_check_cmd_status(nvme_cmd_t *cmd) 10783c9168faSHans Rosenfeld { 10793c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 10803c9168faSHans Rosenfeld 10813c9168faSHans Rosenfeld /* take a shortcut if everything is alright */ 10823c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 10833c9168faSHans Rosenfeld cqe->cqe_sf.sf_sc == NVME_CQE_SC_GEN_SUCCESS) 10843c9168faSHans Rosenfeld return (0); 10853c9168faSHans Rosenfeld 10863c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC) 10873c9168faSHans Rosenfeld return (nvme_check_generic_cmd_status(cmd)); 10883c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_SPECIFIC) 10893c9168faSHans Rosenfeld return (nvme_check_specific_cmd_status(cmd)); 10903c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_INTEGRITY) 10913c9168faSHans Rosenfeld return (nvme_check_integrity_cmd_status(cmd)); 10923c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_VENDOR) 10933c9168faSHans Rosenfeld return (nvme_check_vendor_cmd_status(cmd)); 10943c9168faSHans Rosenfeld 10953c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 10963c9168faSHans Rosenfeld } 10973c9168faSHans Rosenfeld 10983c9168faSHans Rosenfeld /* 10993c9168faSHans Rosenfeld * nvme_abort_cmd_cb -- replaces nc_callback of aborted commands 11003c9168faSHans Rosenfeld * 11013c9168faSHans Rosenfeld * This functions takes care of cleaning up aborted commands. The command 11023c9168faSHans Rosenfeld * status is checked to catch any fatal errors. 11033c9168faSHans Rosenfeld */ 11043c9168faSHans Rosenfeld static void 11053c9168faSHans Rosenfeld nvme_abort_cmd_cb(void *arg) 11063c9168faSHans Rosenfeld { 11073c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 11083c9168faSHans Rosenfeld 11093c9168faSHans Rosenfeld /* 11103c9168faSHans Rosenfeld * Grab the command mutex. Once we have it we hold the last reference 11113c9168faSHans Rosenfeld * to the command and can safely free it. 11123c9168faSHans Rosenfeld */ 11133c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 11143c9168faSHans Rosenfeld (void) nvme_check_cmd_status(cmd); 11153c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 11163c9168faSHans Rosenfeld 11173c9168faSHans Rosenfeld nvme_free_cmd(cmd); 11183c9168faSHans Rosenfeld } 11193c9168faSHans Rosenfeld 11203c9168faSHans Rosenfeld static void 11213c9168faSHans Rosenfeld nvme_abort_cmd(nvme_cmd_t *abort_cmd) 11223c9168faSHans Rosenfeld { 11233c9168faSHans Rosenfeld nvme_t *nvme = abort_cmd->nc_nvme; 11243c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 11253c9168faSHans Rosenfeld nvme_abort_cmd_t ac = { 0 }; 11263c9168faSHans Rosenfeld 11273c9168faSHans Rosenfeld sema_p(&nvme->n_abort_sema); 11283c9168faSHans Rosenfeld 11293c9168faSHans Rosenfeld ac.b.ac_cid = abort_cmd->nc_sqe.sqe_cid; 11303c9168faSHans Rosenfeld ac.b.ac_sqid = abort_cmd->nc_sqid; 11313c9168faSHans Rosenfeld 11323c9168faSHans Rosenfeld /* 11333c9168faSHans Rosenfeld * Drop the mutex of the aborted command. From this point on 11343c9168faSHans Rosenfeld * we must assume that the abort callback has freed the command. 11353c9168faSHans Rosenfeld */ 11363c9168faSHans Rosenfeld mutex_exit(&abort_cmd->nc_mutex); 11373c9168faSHans Rosenfeld 11383c9168faSHans Rosenfeld cmd->nc_sqid = 0; 11393c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT; 11403c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 11413c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = ac.r; 11423c9168faSHans Rosenfeld 11433c9168faSHans Rosenfeld /* 11443c9168faSHans Rosenfeld * Send the ABORT to the hardware. The ABORT command will return _after_ 11453c9168faSHans Rosenfeld * the aborted command has completed (aborted or otherwise). 11463c9168faSHans Rosenfeld */ 1147e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 11483c9168faSHans Rosenfeld sema_v(&nvme->n_abort_sema); 11493c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 11503c9168faSHans Rosenfeld "!nvme_admin_cmd failed for ABORT"); 11513c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_abort_failed); 11523c9168faSHans Rosenfeld return; 11533c9168faSHans Rosenfeld } 11543c9168faSHans Rosenfeld sema_v(&nvme->n_abort_sema); 11553c9168faSHans Rosenfeld 11563c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 11573c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 11583c9168faSHans Rosenfeld "!ABORT failed with sct = %x, sc = %x", 11593c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 11603c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_abort_failed); 11613c9168faSHans Rosenfeld } else { 11623c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_aborted); 11633c9168faSHans Rosenfeld } 11643c9168faSHans Rosenfeld 11653c9168faSHans Rosenfeld nvme_free_cmd(cmd); 11663c9168faSHans Rosenfeld } 11673c9168faSHans Rosenfeld 11683c9168faSHans Rosenfeld /* 11693c9168faSHans Rosenfeld * nvme_wait_cmd -- wait for command completion or timeout 11703c9168faSHans Rosenfeld * 11713c9168faSHans Rosenfeld * Returns B_TRUE if the command completed normally. 11723c9168faSHans Rosenfeld * 11733c9168faSHans Rosenfeld * Returns B_FALSE if the command timed out and an abort was attempted. The 11743c9168faSHans Rosenfeld * command mutex will be dropped and the command must be considered freed. The 11753c9168faSHans Rosenfeld * freeing of the command is normally done by the abort command callback. 11763c9168faSHans Rosenfeld * 11773c9168faSHans Rosenfeld * In case of a serious error or a timeout of the abort command the hardware 11783c9168faSHans Rosenfeld * will be declared dead and FMA will be notified. 11793c9168faSHans Rosenfeld */ 11803c9168faSHans Rosenfeld static boolean_t 1181e8ba2a38SHans Rosenfeld nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec) 11823c9168faSHans Rosenfeld { 1183e8ba2a38SHans Rosenfeld clock_t timeout = ddi_get_lbolt() + drv_usectohz(sec * MICROSEC); 11843c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 11853c9168faSHans Rosenfeld nvme_reg_csts_t csts; 11863c9168faSHans Rosenfeld 11873c9168faSHans Rosenfeld ASSERT(mutex_owned(&cmd->nc_mutex)); 11883c9168faSHans Rosenfeld 11893c9168faSHans Rosenfeld while (!cmd->nc_completed) { 11903c9168faSHans Rosenfeld if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1) 11913c9168faSHans Rosenfeld break; 11923c9168faSHans Rosenfeld } 11933c9168faSHans Rosenfeld 11943c9168faSHans Rosenfeld if (cmd->nc_completed) 11953c9168faSHans Rosenfeld return (B_TRUE); 11963c9168faSHans Rosenfeld 11973c9168faSHans Rosenfeld /* 11983c9168faSHans Rosenfeld * The command timed out. Change the callback to the cleanup function. 11993c9168faSHans Rosenfeld */ 12003c9168faSHans Rosenfeld cmd->nc_callback = nvme_abort_cmd_cb; 12013c9168faSHans Rosenfeld 12023c9168faSHans Rosenfeld /* 12033c9168faSHans Rosenfeld * Check controller for fatal status, any errors associated with the 12043c9168faSHans Rosenfeld * register or DMA handle, or for a double timeout (abort command timed 12053c9168faSHans Rosenfeld * out). If necessary log a warning and call FMA. 12063c9168faSHans Rosenfeld */ 12073c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 12083c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!command timeout, " 12093c9168faSHans Rosenfeld "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_opc, csts.b.csts_cfs); 12103c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_timeout); 12113c9168faSHans Rosenfeld 12123c9168faSHans Rosenfeld if (csts.b.csts_cfs || 12133c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme) || 12143c9168faSHans Rosenfeld nvme_check_dma_hdl(cmd->nc_dma) || 12153c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) { 12163c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 12173c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 12183c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 12193c9168faSHans Rosenfeld } else { 12203c9168faSHans Rosenfeld /* 12213c9168faSHans Rosenfeld * Try to abort the command. The command mutex is released by 12223c9168faSHans Rosenfeld * nvme_abort_cmd(). 12233c9168faSHans Rosenfeld * If the abort succeeds it will have freed the aborted command. 12243c9168faSHans Rosenfeld * If the abort fails for other reasons we must assume that the 12253c9168faSHans Rosenfeld * command may complete at any time, and the callback will free 12263c9168faSHans Rosenfeld * it for us. 12273c9168faSHans Rosenfeld */ 12283c9168faSHans Rosenfeld nvme_abort_cmd(cmd); 12293c9168faSHans Rosenfeld } 12303c9168faSHans Rosenfeld 12313c9168faSHans Rosenfeld return (B_FALSE); 12323c9168faSHans Rosenfeld } 12333c9168faSHans Rosenfeld 12343c9168faSHans Rosenfeld static void 12353c9168faSHans Rosenfeld nvme_wakeup_cmd(void *arg) 12363c9168faSHans Rosenfeld { 12373c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 12383c9168faSHans Rosenfeld 12393c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 12403c9168faSHans Rosenfeld /* 12413c9168faSHans Rosenfeld * There is a slight chance that this command completed shortly after 12423c9168faSHans Rosenfeld * the timeout was hit in nvme_wait_cmd() but before the callback was 12433c9168faSHans Rosenfeld * changed. Catch that case here and clean up accordingly. 12443c9168faSHans Rosenfeld */ 12453c9168faSHans Rosenfeld if (cmd->nc_callback == nvme_abort_cmd_cb) { 12463c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 12473c9168faSHans Rosenfeld nvme_abort_cmd_cb(cmd); 12483c9168faSHans Rosenfeld return; 12493c9168faSHans Rosenfeld } 12503c9168faSHans Rosenfeld 12513c9168faSHans Rosenfeld cmd->nc_completed = B_TRUE; 12523c9168faSHans Rosenfeld cv_signal(&cmd->nc_cv); 12533c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 12543c9168faSHans Rosenfeld } 12553c9168faSHans Rosenfeld 12563c9168faSHans Rosenfeld static void 12573c9168faSHans Rosenfeld nvme_async_event_task(void *arg) 12583c9168faSHans Rosenfeld { 12593c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 12603c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 12613c9168faSHans Rosenfeld nvme_error_log_entry_t *error_log = NULL; 12623c9168faSHans Rosenfeld nvme_health_log_t *health_log = NULL; 12633c9168faSHans Rosenfeld nvme_async_event_t event; 12643c9168faSHans Rosenfeld int ret; 12653c9168faSHans Rosenfeld 12663c9168faSHans Rosenfeld /* 12673c9168faSHans Rosenfeld * Check for errors associated with the async request itself. The only 12683c9168faSHans Rosenfeld * command-specific error is "async event limit exceeded", which 12693c9168faSHans Rosenfeld * indicates a programming error in the driver and causes a panic in 12703c9168faSHans Rosenfeld * nvme_check_cmd_status(). 12713c9168faSHans Rosenfeld * 12723c9168faSHans Rosenfeld * Other possible errors are various scenarios where the async request 12733c9168faSHans Rosenfeld * was aborted, or internal errors in the device. Internal errors are 12743c9168faSHans Rosenfeld * reported to FMA, the command aborts need no special handling here. 12753c9168faSHans Rosenfeld */ 12763c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 12773c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 12783c9168faSHans Rosenfeld "!async event request returned failure, sct = %x, " 12793c9168faSHans Rosenfeld "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct, 12803c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr, 12813c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_m); 12823c9168faSHans Rosenfeld 12833c9168faSHans Rosenfeld if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 12843c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) { 12853c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 12863c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, 12873c9168faSHans Rosenfeld DDI_SERVICE_LOST); 12883c9168faSHans Rosenfeld } 12893c9168faSHans Rosenfeld nvme_free_cmd(cmd); 12903c9168faSHans Rosenfeld return; 12913c9168faSHans Rosenfeld } 12923c9168faSHans Rosenfeld 12933c9168faSHans Rosenfeld 12943c9168faSHans Rosenfeld event.r = cmd->nc_cqe.cqe_dw0; 12953c9168faSHans Rosenfeld 12963c9168faSHans Rosenfeld /* Clear CQE and re-submit the async request. */ 12973c9168faSHans Rosenfeld bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t)); 12983c9168faSHans Rosenfeld ret = nvme_submit_cmd(nvme->n_adminq, cmd); 12993c9168faSHans Rosenfeld 13003c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 13013c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13023c9168faSHans Rosenfeld "!failed to resubmit async event request"); 13033c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_async_resubmit_failed); 13043c9168faSHans Rosenfeld nvme_free_cmd(cmd); 13053c9168faSHans Rosenfeld } 13063c9168faSHans Rosenfeld 13073c9168faSHans Rosenfeld switch (event.b.ae_type) { 13083c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_ERROR: 13093c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_ERROR) { 13103c9168faSHans Rosenfeld error_log = (nvme_error_log_entry_t *) 13113c9168faSHans Rosenfeld nvme_get_logpage(nvme, event.b.ae_logpage); 13123c9168faSHans Rosenfeld } else { 13133c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 13143c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 13153c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 13163c9168faSHans Rosenfeld } 13173c9168faSHans Rosenfeld 13183c9168faSHans Rosenfeld switch (event.b.ae_info) { 13193c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_SQ: 13203c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 13213c9168faSHans Rosenfeld "invalid submission queue"); 13223c9168faSHans Rosenfeld return; 13233c9168faSHans Rosenfeld 13243c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_DBL: 13253c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 13263c9168faSHans Rosenfeld "invalid doorbell write value"); 13273c9168faSHans Rosenfeld return; 13283c9168faSHans Rosenfeld 13293c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_DIAGFAIL: 13303c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure"); 13313c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 13323c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 13333c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_diagfail_event); 13343c9168faSHans Rosenfeld break; 13353c9168faSHans Rosenfeld 13363c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_PERSISTENT: 13373c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!persistent internal " 13383c9168faSHans Rosenfeld "device error"); 13393c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 13403c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 13413c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_persistent_event); 13423c9168faSHans Rosenfeld break; 13433c9168faSHans Rosenfeld 13443c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_TRANSIENT: 13453c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!transient internal " 13463c9168faSHans Rosenfeld "device error"); 13473c9168faSHans Rosenfeld /* TODO: send ereport */ 13483c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_transient_event); 13493c9168faSHans Rosenfeld break; 13503c9168faSHans Rosenfeld 13513c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_FW_LOAD: 13523c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13533c9168faSHans Rosenfeld "!firmware image load error"); 13543c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_fw_load_event); 13553c9168faSHans Rosenfeld break; 13563c9168faSHans Rosenfeld } 13573c9168faSHans Rosenfeld break; 13583c9168faSHans Rosenfeld 13593c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_HEALTH: 13603c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_HEALTH) { 13613c9168faSHans Rosenfeld health_log = (nvme_health_log_t *) 13623c9168faSHans Rosenfeld nvme_get_logpage(nvme, event.b.ae_logpage, -1); 13633c9168faSHans Rosenfeld } else { 13643c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 13653c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 13663c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 13673c9168faSHans Rosenfeld } 13683c9168faSHans Rosenfeld 13693c9168faSHans Rosenfeld switch (event.b.ae_info) { 13703c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_RELIABILITY: 13713c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13723c9168faSHans Rosenfeld "!device reliability compromised"); 13733c9168faSHans Rosenfeld /* TODO: send ereport */ 13743c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_reliability_event); 13753c9168faSHans Rosenfeld break; 13763c9168faSHans Rosenfeld 13773c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_TEMPERATURE: 13783c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13793c9168faSHans Rosenfeld "!temperature above threshold"); 13803c9168faSHans Rosenfeld /* TODO: send ereport */ 13813c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_temperature_event); 13823c9168faSHans Rosenfeld break; 13833c9168faSHans Rosenfeld 13843c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_SPARE: 13853c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13863c9168faSHans Rosenfeld "!spare space below threshold"); 13873c9168faSHans Rosenfeld /* TODO: send ereport */ 13883c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_spare_event); 13893c9168faSHans Rosenfeld break; 13903c9168faSHans Rosenfeld } 13913c9168faSHans Rosenfeld break; 13923c9168faSHans Rosenfeld 13933c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_VENDOR: 13943c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event " 13953c9168faSHans Rosenfeld "received, info = %x, logpage = %x", event.b.ae_info, 13963c9168faSHans Rosenfeld event.b.ae_logpage); 13973c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_vendor_event); 13983c9168faSHans Rosenfeld break; 13993c9168faSHans Rosenfeld 14003c9168faSHans Rosenfeld default: 14013c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, " 14023c9168faSHans Rosenfeld "type = %x, info = %x, logpage = %x", event.b.ae_type, 14033c9168faSHans Rosenfeld event.b.ae_info, event.b.ae_logpage); 14043c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_event); 14053c9168faSHans Rosenfeld break; 14063c9168faSHans Rosenfeld } 14073c9168faSHans Rosenfeld 14083c9168faSHans Rosenfeld if (error_log) 14093c9168faSHans Rosenfeld kmem_free(error_log, sizeof (nvme_error_log_entry_t) * 14103c9168faSHans Rosenfeld nvme->n_error_log_len); 14113c9168faSHans Rosenfeld 14123c9168faSHans Rosenfeld if (health_log) 14133c9168faSHans Rosenfeld kmem_free(health_log, sizeof (nvme_health_log_t)); 14143c9168faSHans Rosenfeld } 14153c9168faSHans Rosenfeld 14163c9168faSHans Rosenfeld static int 1417e8ba2a38SHans Rosenfeld nvme_admin_cmd(nvme_cmd_t *cmd, int sec) 14183c9168faSHans Rosenfeld { 14193c9168faSHans Rosenfeld int ret; 14203c9168faSHans Rosenfeld 14213c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 14223c9168faSHans Rosenfeld ret = nvme_submit_cmd(cmd->nc_nvme->n_adminq, cmd); 14233c9168faSHans Rosenfeld 14243c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 14253c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 14263c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 14273c9168faSHans Rosenfeld "!nvme_submit_cmd failed"); 14283c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_admin_queue_full); 14293c9168faSHans Rosenfeld nvme_free_cmd(cmd); 14303c9168faSHans Rosenfeld return (DDI_FAILURE); 14313c9168faSHans Rosenfeld } 14323c9168faSHans Rosenfeld 1433e8ba2a38SHans Rosenfeld if (nvme_wait_cmd(cmd, sec) == B_FALSE) { 14343c9168faSHans Rosenfeld /* 14353c9168faSHans Rosenfeld * The command timed out. An abort command was posted that 14363c9168faSHans Rosenfeld * will take care of the cleanup. 14373c9168faSHans Rosenfeld */ 14383c9168faSHans Rosenfeld return (DDI_FAILURE); 14393c9168faSHans Rosenfeld } 14403c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 14413c9168faSHans Rosenfeld 14423c9168faSHans Rosenfeld return (DDI_SUCCESS); 14433c9168faSHans Rosenfeld } 14443c9168faSHans Rosenfeld 14453c9168faSHans Rosenfeld static int 14463c9168faSHans Rosenfeld nvme_async_event(nvme_t *nvme) 14473c9168faSHans Rosenfeld { 14483c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 14493c9168faSHans Rosenfeld int ret; 14503c9168faSHans Rosenfeld 14513c9168faSHans Rosenfeld cmd->nc_sqid = 0; 14523c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT; 14533c9168faSHans Rosenfeld cmd->nc_callback = nvme_async_event_task; 14543c9168faSHans Rosenfeld 14553c9168faSHans Rosenfeld ret = nvme_submit_cmd(nvme->n_adminq, cmd); 14563c9168faSHans Rosenfeld 14573c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 14583c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 14593c9168faSHans Rosenfeld "!nvme_submit_cmd failed for ASYNCHRONOUS EVENT"); 14603c9168faSHans Rosenfeld nvme_free_cmd(cmd); 14613c9168faSHans Rosenfeld return (DDI_FAILURE); 14623c9168faSHans Rosenfeld } 14633c9168faSHans Rosenfeld 14643c9168faSHans Rosenfeld return (DDI_SUCCESS); 14653c9168faSHans Rosenfeld } 14663c9168faSHans Rosenfeld 14673c9168faSHans Rosenfeld static void * 14683c9168faSHans Rosenfeld nvme_get_logpage(nvme_t *nvme, uint8_t logpage, ...) 14693c9168faSHans Rosenfeld { 14703c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 14713c9168faSHans Rosenfeld void *buf = NULL; 147234c938c7SPete Shephard nvme_getlogpage_t getlogpage = { 0 }; 14733c9168faSHans Rosenfeld size_t bufsize; 14743c9168faSHans Rosenfeld va_list ap; 14753c9168faSHans Rosenfeld 14763c9168faSHans Rosenfeld va_start(ap, logpage); 14773c9168faSHans Rosenfeld 14783c9168faSHans Rosenfeld cmd->nc_sqid = 0; 14793c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 14803c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE; 14813c9168faSHans Rosenfeld 14823c9168faSHans Rosenfeld getlogpage.b.lp_lid = logpage; 14833c9168faSHans Rosenfeld 14843c9168faSHans Rosenfeld switch (logpage) { 14853c9168faSHans Rosenfeld case NVME_LOGPAGE_ERROR: 14863c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 14873c9168faSHans Rosenfeld bufsize = nvme->n_error_log_len * 14883c9168faSHans Rosenfeld sizeof (nvme_error_log_entry_t); 14893c9168faSHans Rosenfeld break; 14903c9168faSHans Rosenfeld 14913c9168faSHans Rosenfeld case NVME_LOGPAGE_HEALTH: 14923c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t); 14933c9168faSHans Rosenfeld bufsize = sizeof (nvme_health_log_t); 14943c9168faSHans Rosenfeld break; 14953c9168faSHans Rosenfeld 14963c9168faSHans Rosenfeld case NVME_LOGPAGE_FWSLOT: 14973c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 14983c9168faSHans Rosenfeld bufsize = sizeof (nvme_fwslot_log_t); 14993c9168faSHans Rosenfeld break; 15003c9168faSHans Rosenfeld 15013c9168faSHans Rosenfeld default: 15023c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown log page requested: %d", 15033c9168faSHans Rosenfeld logpage); 15043c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_logpage); 15053c9168faSHans Rosenfeld goto fail; 15063c9168faSHans Rosenfeld } 15073c9168faSHans Rosenfeld 15083c9168faSHans Rosenfeld va_end(ap); 15093c9168faSHans Rosenfeld 15109898bbdbSHans Rosenfeld getlogpage.b.lp_numd = bufsize / sizeof (uint32_t) - 1; 15113c9168faSHans Rosenfeld 15123c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = getlogpage.r; 15133c9168faSHans Rosenfeld 15143c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, getlogpage.b.lp_numd * sizeof (uint32_t), 15153c9168faSHans Rosenfeld DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 15163c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15173c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for GET LOG PAGE"); 15183c9168faSHans Rosenfeld goto fail; 15193c9168faSHans Rosenfeld } 15203c9168faSHans Rosenfeld 15213c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 15223c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15233c9168faSHans Rosenfeld "!too many DMA cookies for GET LOG PAGE"); 15243c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 15253c9168faSHans Rosenfeld goto fail; 15263c9168faSHans Rosenfeld } 15273c9168faSHans Rosenfeld 15283c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 15293c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 15303c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 15313c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 15323c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 15333c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 15343c9168faSHans Rosenfeld } 15353c9168faSHans Rosenfeld 1536e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 15373c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15383c9168faSHans Rosenfeld "!nvme_admin_cmd failed for GET LOG PAGE"); 15393c9168faSHans Rosenfeld return (NULL); 15403c9168faSHans Rosenfeld } 15413c9168faSHans Rosenfeld 15423c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 15433c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15443c9168faSHans Rosenfeld "!GET LOG PAGE failed with sct = %x, sc = %x", 15453c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 15463c9168faSHans Rosenfeld goto fail; 15473c9168faSHans Rosenfeld } 15483c9168faSHans Rosenfeld 15493c9168faSHans Rosenfeld buf = kmem_alloc(bufsize, KM_SLEEP); 15503c9168faSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, buf, bufsize); 15513c9168faSHans Rosenfeld 15523c9168faSHans Rosenfeld fail: 15533c9168faSHans Rosenfeld nvme_free_cmd(cmd); 15543c9168faSHans Rosenfeld 15553c9168faSHans Rosenfeld return (buf); 15563c9168faSHans Rosenfeld } 15573c9168faSHans Rosenfeld 15583c9168faSHans Rosenfeld static void * 15593c9168faSHans Rosenfeld nvme_identify(nvme_t *nvme, uint32_t nsid) 15603c9168faSHans Rosenfeld { 15613c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 15623c9168faSHans Rosenfeld void *buf = NULL; 15633c9168faSHans Rosenfeld 15643c9168faSHans Rosenfeld cmd->nc_sqid = 0; 15653c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 15663c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY; 15673c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 15683c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL; 15693c9168faSHans Rosenfeld 15703c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ, 15713c9168faSHans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 15723c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15733c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for IDENTIFY"); 15743c9168faSHans Rosenfeld goto fail; 15753c9168faSHans Rosenfeld } 15763c9168faSHans Rosenfeld 15773c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 15783c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15793c9168faSHans Rosenfeld "!too many DMA cookies for IDENTIFY"); 15803c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 15813c9168faSHans Rosenfeld goto fail; 15823c9168faSHans Rosenfeld } 15833c9168faSHans Rosenfeld 15843c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 15853c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 15863c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 15873c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 15883c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 15893c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 15903c9168faSHans Rosenfeld } 15913c9168faSHans Rosenfeld 1592e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 15933c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15943c9168faSHans Rosenfeld "!nvme_admin_cmd failed for IDENTIFY"); 15953c9168faSHans Rosenfeld return (NULL); 15963c9168faSHans Rosenfeld } 15973c9168faSHans Rosenfeld 15983c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 15993c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16003c9168faSHans Rosenfeld "!IDENTIFY failed with sct = %x, sc = %x", 16013c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 16023c9168faSHans Rosenfeld goto fail; 16033c9168faSHans Rosenfeld } 16043c9168faSHans Rosenfeld 16053c9168faSHans Rosenfeld buf = kmem_alloc(NVME_IDENTIFY_BUFSIZE, KM_SLEEP); 16063c9168faSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, buf, NVME_IDENTIFY_BUFSIZE); 16073c9168faSHans Rosenfeld 16083c9168faSHans Rosenfeld fail: 16093c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16103c9168faSHans Rosenfeld 16113c9168faSHans Rosenfeld return (buf); 16123c9168faSHans Rosenfeld } 16133c9168faSHans Rosenfeld 16143c9168faSHans Rosenfeld static int 16153c9168faSHans Rosenfeld nvme_set_nqueues(nvme_t *nvme, uint16_t nqueues) 16163c9168faSHans Rosenfeld { 16173c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 16183c9168faSHans Rosenfeld nvme_nqueue_t nq = { 0 }; 16193c9168faSHans Rosenfeld 162034c938c7SPete Shephard nq.b.nq_nsq = nq.b.nq_ncq = nqueues - 1; 16213c9168faSHans Rosenfeld 16223c9168faSHans Rosenfeld cmd->nc_sqid = 0; 16233c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 16243c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES; 16253c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = NVME_FEAT_NQUEUES; 16263c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = nq.r; 16273c9168faSHans Rosenfeld 1628e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 16293c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16303c9168faSHans Rosenfeld "!nvme_admin_cmd failed for SET FEATURES (NQUEUES)"); 16313c9168faSHans Rosenfeld return (0); 16323c9168faSHans Rosenfeld } 16333c9168faSHans Rosenfeld 16343c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 16353c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16363c9168faSHans Rosenfeld "!SET FEATURES (NQUEUES) failed with sct = %x, sc = %x", 16373c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 16383c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16393c9168faSHans Rosenfeld return (0); 16403c9168faSHans Rosenfeld } 16413c9168faSHans Rosenfeld 16423c9168faSHans Rosenfeld nq.r = cmd->nc_cqe.cqe_dw0; 16433c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16443c9168faSHans Rosenfeld 16453c9168faSHans Rosenfeld /* 16463c9168faSHans Rosenfeld * Always use the same number of submission and completion queues, and 16473c9168faSHans Rosenfeld * never use more than the requested number of queues. 16483c9168faSHans Rosenfeld */ 164934c938c7SPete Shephard return (MIN(nqueues, MIN(nq.b.nq_nsq, nq.b.nq_ncq) + 1)); 16503c9168faSHans Rosenfeld } 16513c9168faSHans Rosenfeld 16523c9168faSHans Rosenfeld static int 16533c9168faSHans Rosenfeld nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx) 16543c9168faSHans Rosenfeld { 16553c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 16563c9168faSHans Rosenfeld nvme_create_queue_dw10_t dw10 = { 0 }; 16573c9168faSHans Rosenfeld nvme_create_cq_dw11_t c_dw11 = { 0 }; 16583c9168faSHans Rosenfeld nvme_create_sq_dw11_t s_dw11 = { 0 }; 16593c9168faSHans Rosenfeld 16603c9168faSHans Rosenfeld dw10.b.q_qid = idx; 16613c9168faSHans Rosenfeld dw10.b.q_qsize = qp->nq_nentry - 1; 16623c9168faSHans Rosenfeld 16633c9168faSHans Rosenfeld c_dw11.b.cq_pc = 1; 16643c9168faSHans Rosenfeld c_dw11.b.cq_ien = 1; 16653c9168faSHans Rosenfeld c_dw11.b.cq_iv = idx % nvme->n_intr_cnt; 16663c9168faSHans Rosenfeld 16673c9168faSHans Rosenfeld cmd->nc_sqid = 0; 16683c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 16693c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE; 16703c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 16713c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = c_dw11.r; 16723c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_cqdma->nd_cookie.dmac_laddress; 16733c9168faSHans Rosenfeld 1674e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 16753c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16763c9168faSHans Rosenfeld "!nvme_admin_cmd failed for CREATE CQUEUE"); 16773c9168faSHans Rosenfeld return (DDI_FAILURE); 16783c9168faSHans Rosenfeld } 16793c9168faSHans Rosenfeld 16803c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 16813c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16823c9168faSHans Rosenfeld "!CREATE CQUEUE failed with sct = %x, sc = %x", 16833c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 16843c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16853c9168faSHans Rosenfeld return (DDI_FAILURE); 16863c9168faSHans Rosenfeld } 16873c9168faSHans Rosenfeld 16883c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16893c9168faSHans Rosenfeld 16903c9168faSHans Rosenfeld s_dw11.b.sq_pc = 1; 16913c9168faSHans Rosenfeld s_dw11.b.sq_cqid = idx; 16923c9168faSHans Rosenfeld 16933c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 16943c9168faSHans Rosenfeld cmd->nc_sqid = 0; 16953c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 16963c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE; 16973c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 16983c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = s_dw11.r; 16993c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress; 17003c9168faSHans Rosenfeld 1701e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 17023c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17033c9168faSHans Rosenfeld "!nvme_admin_cmd failed for CREATE SQUEUE"); 17043c9168faSHans Rosenfeld return (DDI_FAILURE); 17053c9168faSHans Rosenfeld } 17063c9168faSHans Rosenfeld 17073c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 17083c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17093c9168faSHans Rosenfeld "!CREATE SQUEUE failed with sct = %x, sc = %x", 17103c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 17113c9168faSHans Rosenfeld nvme_free_cmd(cmd); 17123c9168faSHans Rosenfeld return (DDI_FAILURE); 17133c9168faSHans Rosenfeld } 17143c9168faSHans Rosenfeld 17153c9168faSHans Rosenfeld nvme_free_cmd(cmd); 17163c9168faSHans Rosenfeld 17173c9168faSHans Rosenfeld return (DDI_SUCCESS); 17183c9168faSHans Rosenfeld } 17193c9168faSHans Rosenfeld 17203c9168faSHans Rosenfeld static boolean_t 17213c9168faSHans Rosenfeld nvme_reset(nvme_t *nvme, boolean_t quiesce) 17223c9168faSHans Rosenfeld { 17233c9168faSHans Rosenfeld nvme_reg_csts_t csts; 17243c9168faSHans Rosenfeld int i; 17253c9168faSHans Rosenfeld 17263c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 17273c9168faSHans Rosenfeld 17283c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 17293c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) { 17303c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 17313c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 17323c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 17333c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) 17343c9168faSHans Rosenfeld break; 17353c9168faSHans Rosenfeld 17363c9168faSHans Rosenfeld if (quiesce) 17373c9168faSHans Rosenfeld drv_usecwait(50000); 17383c9168faSHans Rosenfeld else 17393c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 17403c9168faSHans Rosenfeld } 17413c9168faSHans Rosenfeld } 17423c9168faSHans Rosenfeld 17433c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, 0); 17443c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ASQ, 0); 17453c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ACQ, 0); 17463c9168faSHans Rosenfeld 17473c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 17483c9168faSHans Rosenfeld return (csts.b.csts_rdy == 0 ? B_TRUE : B_FALSE); 17493c9168faSHans Rosenfeld } 17503c9168faSHans Rosenfeld 17513c9168faSHans Rosenfeld static void 17523c9168faSHans Rosenfeld nvme_shutdown(nvme_t *nvme, int mode, boolean_t quiesce) 17533c9168faSHans Rosenfeld { 17543c9168faSHans Rosenfeld nvme_reg_cc_t cc; 17553c9168faSHans Rosenfeld nvme_reg_csts_t csts; 17563c9168faSHans Rosenfeld int i; 17573c9168faSHans Rosenfeld 17583c9168faSHans Rosenfeld ASSERT(mode == NVME_CC_SHN_NORMAL || mode == NVME_CC_SHN_ABRUPT); 17593c9168faSHans Rosenfeld 17603c9168faSHans Rosenfeld cc.r = nvme_get32(nvme, NVME_REG_CC); 17613c9168faSHans Rosenfeld cc.b.cc_shn = mode & 0x3; 17623c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 17633c9168faSHans Rosenfeld 17643c9168faSHans Rosenfeld for (i = 0; i != 10; i++) { 17653c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 17663c9168faSHans Rosenfeld if (csts.b.csts_shst == NVME_CSTS_SHN_COMPLETE) 17673c9168faSHans Rosenfeld break; 17683c9168faSHans Rosenfeld 17693c9168faSHans Rosenfeld if (quiesce) 17703c9168faSHans Rosenfeld drv_usecwait(100000); 17713c9168faSHans Rosenfeld else 17723c9168faSHans Rosenfeld delay(drv_usectohz(100000)); 17733c9168faSHans Rosenfeld } 17743c9168faSHans Rosenfeld } 17753c9168faSHans Rosenfeld 17763c9168faSHans Rosenfeld 17773c9168faSHans Rosenfeld static void 17783c9168faSHans Rosenfeld nvme_prepare_devid(nvme_t *nvme, uint32_t nsid) 17793c9168faSHans Rosenfeld { 17803c9168faSHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 17813c9168faSHans Rosenfeld char serial[sizeof (nvme->n_idctl->id_serial) + 1]; 17823c9168faSHans Rosenfeld 17833c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 17843c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_serial, serial, 17853c9168faSHans Rosenfeld sizeof (nvme->n_idctl->id_serial)); 17863c9168faSHans Rosenfeld 17873c9168faSHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 17883c9168faSHans Rosenfeld serial[sizeof (nvme->n_idctl->id_serial)] = '\0'; 17893c9168faSHans Rosenfeld 17903c9168faSHans Rosenfeld (void) snprintf(nvme->n_ns[nsid - 1].ns_devid, 17913c9168faSHans Rosenfeld sizeof (nvme->n_ns[0].ns_devid), "%4X-%s-%s-%X", 17923c9168faSHans Rosenfeld nvme->n_idctl->id_vid, model, serial, nsid); 17933c9168faSHans Rosenfeld } 17943c9168faSHans Rosenfeld 17953c9168faSHans Rosenfeld static int 17963c9168faSHans Rosenfeld nvme_init(nvme_t *nvme) 17973c9168faSHans Rosenfeld { 17983c9168faSHans Rosenfeld nvme_reg_cc_t cc = { 0 }; 17993c9168faSHans Rosenfeld nvme_reg_aqa_t aqa = { 0 }; 18003c9168faSHans Rosenfeld nvme_reg_asq_t asq = { 0 }; 18013c9168faSHans Rosenfeld nvme_reg_acq_t acq = { 0 }; 18023c9168faSHans Rosenfeld nvme_reg_cap_t cap; 18033c9168faSHans Rosenfeld nvme_reg_vs_t vs; 18043c9168faSHans Rosenfeld nvme_reg_csts_t csts; 18053c9168faSHans Rosenfeld int i = 0; 18063c9168faSHans Rosenfeld int nqueues; 1807510a6847SHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 1808510a6847SHans Rosenfeld char *vendor, *product; 18093c9168faSHans Rosenfeld 18103c9168faSHans Rosenfeld /* Check controller version */ 18113c9168faSHans Rosenfeld vs.r = nvme_get32(nvme, NVME_REG_VS); 18123c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d", 18133c9168faSHans Rosenfeld vs.b.vs_mjr, vs.b.vs_mnr); 18143c9168faSHans Rosenfeld 18152797e2b9SHans Rosenfeld if (nvme_version_major < vs.b.vs_mjr || 18162797e2b9SHans Rosenfeld (nvme_version_major == vs.b.vs_mjr && 18172797e2b9SHans Rosenfeld nvme_version_minor < vs.b.vs_mnr)) { 18183c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.%d", 18193c9168faSHans Rosenfeld nvme_version_major, nvme_version_minor); 18203c9168faSHans Rosenfeld if (nvme->n_strict_version) 18213c9168faSHans Rosenfeld goto fail; 18223c9168faSHans Rosenfeld } 18233c9168faSHans Rosenfeld 18243c9168faSHans Rosenfeld /* retrieve controller configuration */ 18253c9168faSHans Rosenfeld cap.r = nvme_get64(nvme, NVME_REG_CAP); 18263c9168faSHans Rosenfeld 18273c9168faSHans Rosenfeld if ((cap.b.cap_css & NVME_CAP_CSS_NVM) == 0) { 18283c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 18293c9168faSHans Rosenfeld "!NVM command set not supported by hardware"); 18303c9168faSHans Rosenfeld goto fail; 18313c9168faSHans Rosenfeld } 18323c9168faSHans Rosenfeld 18333c9168faSHans Rosenfeld nvme->n_nssr_supported = cap.b.cap_nssrs; 18343c9168faSHans Rosenfeld nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd; 18353c9168faSHans Rosenfeld nvme->n_timeout = cap.b.cap_to; 18363c9168faSHans Rosenfeld nvme->n_arbitration_mechanisms = cap.b.cap_ams; 18373c9168faSHans Rosenfeld nvme->n_cont_queues_reqd = cap.b.cap_cqr; 18383c9168faSHans Rosenfeld nvme->n_max_queue_entries = cap.b.cap_mqes + 1; 18393c9168faSHans Rosenfeld 18403c9168faSHans Rosenfeld /* 18413c9168faSHans Rosenfeld * The MPSMIN and MPSMAX fields in the CAP register use 0 to specify 18423c9168faSHans Rosenfeld * the base page size of 4k (1<<12), so add 12 here to get the real 18433c9168faSHans Rosenfeld * page size value. 18443c9168faSHans Rosenfeld */ 18453c9168faSHans Rosenfeld nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT), 18463c9168faSHans Rosenfeld cap.b.cap_mpsmax + 12); 18473c9168faSHans Rosenfeld nvme->n_pagesize = 1UL << (nvme->n_pageshift); 18483c9168faSHans Rosenfeld 18493c9168faSHans Rosenfeld /* 18503c9168faSHans Rosenfeld * Set up Queue DMA to transfer at least 1 page-aligned page at a time. 18513c9168faSHans Rosenfeld */ 18523c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize; 18533c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 18543c9168faSHans Rosenfeld 18553c9168faSHans Rosenfeld /* 18563c9168faSHans Rosenfeld * Set up PRP DMA to transfer 1 page-aligned page at a time. 18573c9168faSHans Rosenfeld * Maxxfer may be increased after we identified the controller limits. 18583c9168faSHans Rosenfeld */ 18593c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize; 18603c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 18613c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize; 18622f95345bSYouzhong Yang nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1; 18633c9168faSHans Rosenfeld 18643c9168faSHans Rosenfeld /* 18653c9168faSHans Rosenfeld * Reset controller if it's still in ready state. 18663c9168faSHans Rosenfeld */ 18673c9168faSHans Rosenfeld if (nvme_reset(nvme, B_FALSE) == B_FALSE) { 18683c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller"); 18693c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 18703c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 18713c9168faSHans Rosenfeld goto fail; 18723c9168faSHans Rosenfeld } 18733c9168faSHans Rosenfeld 18743c9168faSHans Rosenfeld /* 18753c9168faSHans Rosenfeld * Create the admin queue pair. 18763c9168faSHans Rosenfeld */ 18773c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0) 18783c9168faSHans Rosenfeld != DDI_SUCCESS) { 18793c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 18803c9168faSHans Rosenfeld "!unable to allocate admin qpair"); 18813c9168faSHans Rosenfeld goto fail; 18823c9168faSHans Rosenfeld } 18833c9168faSHans Rosenfeld nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP); 18843c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 18853c9168faSHans Rosenfeld 18863c9168faSHans Rosenfeld nvme->n_progress |= NVME_ADMIN_QUEUE; 18873c9168faSHans Rosenfeld 18883c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 18893c9168faSHans Rosenfeld "admin-queue-len", nvme->n_admin_queue_len); 18903c9168faSHans Rosenfeld 18913c9168faSHans Rosenfeld aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1; 18923c9168faSHans Rosenfeld asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress; 18933c9168faSHans Rosenfeld acq = nvme->n_adminq->nq_cqdma->nd_cookie.dmac_laddress; 18943c9168faSHans Rosenfeld 18953c9168faSHans Rosenfeld ASSERT((asq & (nvme->n_pagesize - 1)) == 0); 18963c9168faSHans Rosenfeld ASSERT((acq & (nvme->n_pagesize - 1)) == 0); 18973c9168faSHans Rosenfeld 18983c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, aqa.r); 18993c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ASQ, asq); 19003c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ACQ, acq); 19013c9168faSHans Rosenfeld 190234c938c7SPete Shephard cc.b.cc_ams = 0; /* use Round-Robin arbitration */ 190334c938c7SPete Shephard cc.b.cc_css = 0; /* use NVM command set */ 19043c9168faSHans Rosenfeld cc.b.cc_mps = nvme->n_pageshift - 12; 190534c938c7SPete Shephard cc.b.cc_shn = 0; /* no shutdown in progress */ 190634c938c7SPete Shephard cc.b.cc_en = 1; /* enable controller */ 190734c938c7SPete Shephard cc.b.cc_iosqes = 6; /* submission queue entry is 2^6 bytes long */ 190834c938c7SPete Shephard cc.b.cc_iocqes = 4; /* completion queue entry is 2^4 bytes long */ 19093c9168faSHans Rosenfeld 19103c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 19113c9168faSHans Rosenfeld 19123c9168faSHans Rosenfeld /* 19133c9168faSHans Rosenfeld * Wait for the controller to become ready. 19143c9168faSHans Rosenfeld */ 19153c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 19163c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 19173c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 19183c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 19193c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 19203c9168faSHans Rosenfeld 19213c9168faSHans Rosenfeld if (csts.b.csts_cfs == 1) { 19223c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19233c9168faSHans Rosenfeld "!controller fatal status at init"); 19243c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, 19253c9168faSHans Rosenfeld DDI_SERVICE_LOST); 19263c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 19273c9168faSHans Rosenfeld goto fail; 19283c9168faSHans Rosenfeld } 19293c9168faSHans Rosenfeld 19303c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) 19313c9168faSHans Rosenfeld break; 19323c9168faSHans Rosenfeld } 19333c9168faSHans Rosenfeld } 19343c9168faSHans Rosenfeld 19353c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 19363c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!controller not ready"); 19373c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 19383c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 19393c9168faSHans Rosenfeld goto fail; 19403c9168faSHans Rosenfeld } 19413c9168faSHans Rosenfeld 19423c9168faSHans Rosenfeld /* 19433c9168faSHans Rosenfeld * Assume an abort command limit of 1. We'll destroy and re-init 19443c9168faSHans Rosenfeld * that later when we know the true abort command limit. 19453c9168faSHans Rosenfeld */ 19463c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL); 19473c9168faSHans Rosenfeld 1948bf26ea4bSHans Rosenfeld /* 1949bf26ea4bSHans Rosenfeld * Setup initial interrupt for admin queue. 1950bf26ea4bSHans Rosenfeld */ 1951bf26ea4bSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1) 1952bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 1953bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1) 1954bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 1955bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1) 1956bf26ea4bSHans Rosenfeld != DDI_SUCCESS)) { 1957bf26ea4bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 1958bf26ea4bSHans Rosenfeld "!failed to setup initial interrupt"); 1959bf26ea4bSHans Rosenfeld goto fail; 1960bf26ea4bSHans Rosenfeld } 1961bf26ea4bSHans Rosenfeld 19623c9168faSHans Rosenfeld /* 19633c9168faSHans Rosenfeld * Post an asynchronous event command to catch errors. 19643c9168faSHans Rosenfeld */ 19653c9168faSHans Rosenfeld if (nvme_async_event(nvme) != DDI_SUCCESS) { 19663c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19673c9168faSHans Rosenfeld "!failed to post async event"); 19683c9168faSHans Rosenfeld goto fail; 19693c9168faSHans Rosenfeld } 19703c9168faSHans Rosenfeld 19713c9168faSHans Rosenfeld /* 19723c9168faSHans Rosenfeld * Identify Controller 19733c9168faSHans Rosenfeld */ 19743c9168faSHans Rosenfeld nvme->n_idctl = nvme_identify(nvme, 0); 19753c9168faSHans Rosenfeld if (nvme->n_idctl == NULL) { 19763c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19773c9168faSHans Rosenfeld "!failed to identify controller"); 19783c9168faSHans Rosenfeld goto fail; 19793c9168faSHans Rosenfeld } 19803c9168faSHans Rosenfeld 1981510a6847SHans Rosenfeld /* 1982510a6847SHans Rosenfeld * Get Vendor & Product ID 1983510a6847SHans Rosenfeld */ 1984510a6847SHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 1985510a6847SHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 1986510a6847SHans Rosenfeld sata_split_model(model, &vendor, &product); 1987510a6847SHans Rosenfeld 1988510a6847SHans Rosenfeld if (vendor == NULL) 1989510a6847SHans Rosenfeld nvme->n_vendor = strdup("NVMe"); 1990510a6847SHans Rosenfeld else 1991510a6847SHans Rosenfeld nvme->n_vendor = strdup(vendor); 1992510a6847SHans Rosenfeld 1993510a6847SHans Rosenfeld nvme->n_product = strdup(product); 1994510a6847SHans Rosenfeld 19953c9168faSHans Rosenfeld /* 19963c9168faSHans Rosenfeld * Get controller limits. 19973c9168faSHans Rosenfeld */ 19983c9168faSHans Rosenfeld nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT, 19993c9168faSHans Rosenfeld MIN(nvme->n_admin_queue_len / 10, 20003c9168faSHans Rosenfeld MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit))); 20013c9168faSHans Rosenfeld 20023c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 20033c9168faSHans Rosenfeld "async-event-limit", nvme->n_async_event_limit); 20043c9168faSHans Rosenfeld 20053c9168faSHans Rosenfeld nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1; 20063c9168faSHans Rosenfeld 20076afc9eb2SHans Rosenfeld /* 20086afc9eb2SHans Rosenfeld * Reinitialize the semaphore with the true abort command limit 20096afc9eb2SHans Rosenfeld * supported by the hardware. It's not necessary to disable interrupts 20106afc9eb2SHans Rosenfeld * as only command aborts use the semaphore, and no commands are 20116afc9eb2SHans Rosenfeld * executed or aborted while we're here. 20126afc9eb2SHans Rosenfeld */ 20133c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 20143c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL, 20153c9168faSHans Rosenfeld SEMA_DRIVER, NULL); 20163c9168faSHans Rosenfeld 20173c9168faSHans Rosenfeld nvme->n_progress |= NVME_CTRL_LIMITS; 20183c9168faSHans Rosenfeld 20193c9168faSHans Rosenfeld if (nvme->n_idctl->id_mdts == 0) 20203c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536; 20213c9168faSHans Rosenfeld else 20223c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = 20233c9168faSHans Rosenfeld 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts); 20243c9168faSHans Rosenfeld 20253c9168faSHans Rosenfeld nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1; 20263c9168faSHans Rosenfeld 20273c9168faSHans Rosenfeld /* 20283c9168faSHans Rosenfeld * Limit n_max_data_transfer_size to what we can handle in one PRP. 20293c9168faSHans Rosenfeld * Chained PRPs are currently unsupported. 20303c9168faSHans Rosenfeld * 20313c9168faSHans Rosenfeld * This is a no-op on hardware which doesn't support a transfer size 20323c9168faSHans Rosenfeld * big enough to require chained PRPs. 20333c9168faSHans Rosenfeld */ 20343c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size, 20353c9168faSHans Rosenfeld (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize)); 20363c9168faSHans Rosenfeld 20373c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size; 20383c9168faSHans Rosenfeld 20393c9168faSHans Rosenfeld /* 20403c9168faSHans Rosenfeld * Make sure the minimum/maximum queue entry sizes are not 20413c9168faSHans Rosenfeld * larger/smaller than the default. 20423c9168faSHans Rosenfeld */ 20433c9168faSHans Rosenfeld 20443c9168faSHans Rosenfeld if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) || 20453c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) || 20463c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) || 20473c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t))) 20483c9168faSHans Rosenfeld goto fail; 20493c9168faSHans Rosenfeld 20503c9168faSHans Rosenfeld /* 20513c9168faSHans Rosenfeld * Check for the presence of a Volatile Write Cache. If present, 20523c9168faSHans Rosenfeld * enable it by default. 20533c9168faSHans Rosenfeld */ 20543c9168faSHans Rosenfeld if (nvme->n_idctl->id_vwc.vwc_present == 0) { 20553c9168faSHans Rosenfeld nvme->n_volatile_write_cache_enabled = B_FALSE; 20563c9168faSHans Rosenfeld nvme_bd_ops.o_sync_cache = NULL; 20573c9168faSHans Rosenfeld } else { 20583c9168faSHans Rosenfeld /* 20593c9168faSHans Rosenfeld * TODO: send SET FEATURES to enable VWC 20603c9168faSHans Rosenfeld * (have no hardware to test this) 20613c9168faSHans Rosenfeld */ 20623c9168faSHans Rosenfeld nvme->n_volatile_write_cache_enabled = B_FALSE; 20633c9168faSHans Rosenfeld nvme_bd_ops.o_sync_cache = NULL; 20643c9168faSHans Rosenfeld } 20653c9168faSHans Rosenfeld 20663c9168faSHans Rosenfeld /* 20673c9168faSHans Rosenfeld * Grab a copy of all mandatory log pages. 20683c9168faSHans Rosenfeld * 20693c9168faSHans Rosenfeld * TODO: should go away once user space tool exists to print logs 20703c9168faSHans Rosenfeld */ 20713c9168faSHans Rosenfeld nvme->n_error_log = (nvme_error_log_entry_t *) 20723c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_ERROR); 20733c9168faSHans Rosenfeld nvme->n_health_log = (nvme_health_log_t *) 20743c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_HEALTH, -1); 20753c9168faSHans Rosenfeld nvme->n_fwslot_log = (nvme_fwslot_log_t *) 20763c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_FWSLOT); 20773c9168faSHans Rosenfeld 20783c9168faSHans Rosenfeld /* 20793c9168faSHans Rosenfeld * Identify Namespaces 20803c9168faSHans Rosenfeld */ 20813c9168faSHans Rosenfeld nvme->n_namespace_count = nvme->n_idctl->id_nn; 20823c9168faSHans Rosenfeld nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) * 20833c9168faSHans Rosenfeld nvme->n_namespace_count, KM_SLEEP); 20843c9168faSHans Rosenfeld 20853c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 20863c9168faSHans Rosenfeld nvme_identify_nsid_t *idns; 20873c9168faSHans Rosenfeld int last_rp; 20883c9168faSHans Rosenfeld 20893c9168faSHans Rosenfeld nvme->n_ns[i].ns_nvme = nvme; 20903c9168faSHans Rosenfeld nvme->n_ns[i].ns_idns = idns = nvme_identify(nvme, i + 1); 20913c9168faSHans Rosenfeld 20923c9168faSHans Rosenfeld if (idns == NULL) { 20933c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20943c9168faSHans Rosenfeld "!failed to identify namespace %d", i + 1); 20953c9168faSHans Rosenfeld goto fail; 20963c9168faSHans Rosenfeld } 20973c9168faSHans Rosenfeld 20983c9168faSHans Rosenfeld nvme->n_ns[i].ns_id = i + 1; 20993c9168faSHans Rosenfeld nvme->n_ns[i].ns_block_count = idns->id_nsize; 21003c9168faSHans Rosenfeld nvme->n_ns[i].ns_block_size = 21013c9168faSHans Rosenfeld 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads; 21023c9168faSHans Rosenfeld nvme->n_ns[i].ns_best_block_size = nvme->n_ns[i].ns_block_size; 21033c9168faSHans Rosenfeld 21043c9168faSHans Rosenfeld nvme_prepare_devid(nvme, nvme->n_ns[i].ns_id); 21053c9168faSHans Rosenfeld 21063c9168faSHans Rosenfeld /* 21073c9168faSHans Rosenfeld * Find the LBA format with no metadata and the best relative 21083c9168faSHans Rosenfeld * performance. A value of 3 means "degraded", 0 is best. 21093c9168faSHans Rosenfeld */ 21103c9168faSHans Rosenfeld last_rp = 3; 21110d140ff9SHans Rosenfeld for (int j = 0; j <= idns->id_nlbaf; j++) { 21123c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_lbads == 0) 21133c9168faSHans Rosenfeld break; 21143c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_ms != 0) 21153c9168faSHans Rosenfeld continue; 21163c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_rp >= last_rp) 21173c9168faSHans Rosenfeld continue; 21183c9168faSHans Rosenfeld last_rp = idns->id_lbaf[j].lbaf_rp; 21193c9168faSHans Rosenfeld nvme->n_ns[i].ns_best_block_size = 21203c9168faSHans Rosenfeld 1 << idns->id_lbaf[j].lbaf_lbads; 21213c9168faSHans Rosenfeld } 21223c9168faSHans Rosenfeld 21233c9168faSHans Rosenfeld /* 21243c9168faSHans Rosenfeld * We currently don't support namespaces that use either: 21253c9168faSHans Rosenfeld * - thin provisioning 21263c9168faSHans Rosenfeld * - protection information 21273c9168faSHans Rosenfeld */ 21283c9168faSHans Rosenfeld if (idns->id_nsfeat.f_thin || 21293c9168faSHans Rosenfeld idns->id_dps.dp_pinfo) { 21303c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21313c9168faSHans Rosenfeld "!ignoring namespace %d, unsupported features: " 21329cfb65faSHans Rosenfeld "thin = %d, pinfo = %d", i + 1, 21339cfb65faSHans Rosenfeld idns->id_nsfeat.f_thin, idns->id_dps.dp_pinfo); 21343c9168faSHans Rosenfeld nvme->n_ns[i].ns_ignore = B_TRUE; 21353c9168faSHans Rosenfeld } 21363c9168faSHans Rosenfeld } 21373c9168faSHans Rosenfeld 21383c9168faSHans Rosenfeld /* 21393c9168faSHans Rosenfeld * Try to set up MSI/MSI-X interrupts. 21403c9168faSHans Rosenfeld */ 21413c9168faSHans Rosenfeld if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX)) 21423c9168faSHans Rosenfeld != 0) { 21433c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 21443c9168faSHans Rosenfeld 21453c9168faSHans Rosenfeld nqueues = MIN(UINT16_MAX, ncpus); 21463c9168faSHans Rosenfeld 21473c9168faSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 21483c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS) && 21493c9168faSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 21503c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS)) { 21513c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21523c9168faSHans Rosenfeld "!failed to setup MSI/MSI-X interrupts"); 21533c9168faSHans Rosenfeld goto fail; 21543c9168faSHans Rosenfeld } 21553c9168faSHans Rosenfeld } 21563c9168faSHans Rosenfeld 21573c9168faSHans Rosenfeld nqueues = nvme->n_intr_cnt; 21583c9168faSHans Rosenfeld 21593c9168faSHans Rosenfeld /* 21603c9168faSHans Rosenfeld * Create I/O queue pairs. 21613c9168faSHans Rosenfeld */ 21623c9168faSHans Rosenfeld nvme->n_ioq_count = nvme_set_nqueues(nvme, nqueues); 21633c9168faSHans Rosenfeld if (nvme->n_ioq_count == 0) { 21643c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21653c9168faSHans Rosenfeld "!failed to set number of I/O queues to %d", nqueues); 21663c9168faSHans Rosenfeld goto fail; 21673c9168faSHans Rosenfeld } 21683c9168faSHans Rosenfeld 21693c9168faSHans Rosenfeld /* 21703c9168faSHans Rosenfeld * Reallocate I/O queue array 21713c9168faSHans Rosenfeld */ 21723c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *)); 21733c9168faSHans Rosenfeld nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) * 21743c9168faSHans Rosenfeld (nvme->n_ioq_count + 1), KM_SLEEP); 21753c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 21763c9168faSHans Rosenfeld 21773c9168faSHans Rosenfeld /* 21783c9168faSHans Rosenfeld * If we got less queues than we asked for we might as well give 21793c9168faSHans Rosenfeld * some of the interrupt vectors back to the system. 21803c9168faSHans Rosenfeld */ 21813c9168faSHans Rosenfeld if (nvme->n_ioq_count < nqueues) { 21823c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 21833c9168faSHans Rosenfeld 218434c938c7SPete Shephard if (nvme_setup_interrupts(nvme, nvme->n_intr_type, 218534c938c7SPete Shephard nvme->n_ioq_count) != DDI_SUCCESS) { 21863c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21873c9168faSHans Rosenfeld "!failed to reduce number of interrupts"); 21883c9168faSHans Rosenfeld goto fail; 21893c9168faSHans Rosenfeld } 21903c9168faSHans Rosenfeld } 21913c9168faSHans Rosenfeld 21923c9168faSHans Rosenfeld /* 21933c9168faSHans Rosenfeld * Alloc & register I/O queue pairs 21943c9168faSHans Rosenfeld */ 21953c9168faSHans Rosenfeld nvme->n_io_queue_len = 21963c9168faSHans Rosenfeld MIN(nvme->n_io_queue_len, nvme->n_max_queue_entries); 21973c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-queue-len", 21983c9168faSHans Rosenfeld nvme->n_io_queue_len); 21993c9168faSHans Rosenfeld 22003c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 22013c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_io_queue_len, 22023c9168faSHans Rosenfeld &nvme->n_ioq[i], i) != DDI_SUCCESS) { 22033c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22043c9168faSHans Rosenfeld "!unable to allocate I/O qpair %d", i); 22053c9168faSHans Rosenfeld goto fail; 22063c9168faSHans Rosenfeld } 22073c9168faSHans Rosenfeld 22083c9168faSHans Rosenfeld if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i) 22093c9168faSHans Rosenfeld != DDI_SUCCESS) { 22103c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22113c9168faSHans Rosenfeld "!unable to create I/O qpair %d", i); 22123c9168faSHans Rosenfeld goto fail; 22133c9168faSHans Rosenfeld } 22143c9168faSHans Rosenfeld } 22153c9168faSHans Rosenfeld 22163c9168faSHans Rosenfeld /* 22173c9168faSHans Rosenfeld * Post more asynchronous events commands to reduce event reporting 22183c9168faSHans Rosenfeld * latency as suggested by the spec. 22193c9168faSHans Rosenfeld */ 22203c9168faSHans Rosenfeld for (i = 1; i != nvme->n_async_event_limit; i++) { 22213c9168faSHans Rosenfeld if (nvme_async_event(nvme) != DDI_SUCCESS) { 22223c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22233c9168faSHans Rosenfeld "!failed to post async event %d", i); 22243c9168faSHans Rosenfeld goto fail; 22253c9168faSHans Rosenfeld } 22263c9168faSHans Rosenfeld } 22273c9168faSHans Rosenfeld 22283c9168faSHans Rosenfeld return (DDI_SUCCESS); 22293c9168faSHans Rosenfeld 22303c9168faSHans Rosenfeld fail: 22313c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 22323c9168faSHans Rosenfeld return (DDI_FAILURE); 22333c9168faSHans Rosenfeld } 22343c9168faSHans Rosenfeld 22353c9168faSHans Rosenfeld static uint_t 22363c9168faSHans Rosenfeld nvme_intr(caddr_t arg1, caddr_t arg2) 22373c9168faSHans Rosenfeld { 22383c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 22393c9168faSHans Rosenfeld nvme_t *nvme = (nvme_t *)arg1; 22403c9168faSHans Rosenfeld int inum = (int)(uintptr_t)arg2; 2241bf26ea4bSHans Rosenfeld int ccnt = 0; 22423c9168faSHans Rosenfeld int qnum; 22433c9168faSHans Rosenfeld nvme_cmd_t *cmd; 22443c9168faSHans Rosenfeld 22453c9168faSHans Rosenfeld if (inum >= nvme->n_intr_cnt) 22463c9168faSHans Rosenfeld return (DDI_INTR_UNCLAIMED); 22473c9168faSHans Rosenfeld 22483c9168faSHans Rosenfeld /* 22493c9168faSHans Rosenfeld * The interrupt vector a queue uses is calculated as queue_idx % 22503c9168faSHans Rosenfeld * intr_cnt in nvme_create_io_qpair(). Iterate through the queue array 22513c9168faSHans Rosenfeld * in steps of n_intr_cnt to process all queues using this vector. 22523c9168faSHans Rosenfeld */ 22533c9168faSHans Rosenfeld for (qnum = inum; 22543c9168faSHans Rosenfeld qnum < nvme->n_ioq_count + 1 && nvme->n_ioq[qnum] != NULL; 22553c9168faSHans Rosenfeld qnum += nvme->n_intr_cnt) { 22563c9168faSHans Rosenfeld while ((cmd = nvme_retrieve_cmd(nvme, nvme->n_ioq[qnum]))) { 22573c9168faSHans Rosenfeld taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq, 22583c9168faSHans Rosenfeld cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent); 2259bf26ea4bSHans Rosenfeld ccnt++; 22603c9168faSHans Rosenfeld } 22613c9168faSHans Rosenfeld } 22623c9168faSHans Rosenfeld 2263bf26ea4bSHans Rosenfeld return (ccnt > 0 ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); 22643c9168faSHans Rosenfeld } 22653c9168faSHans Rosenfeld 22663c9168faSHans Rosenfeld static void 22676afc9eb2SHans Rosenfeld nvme_release_interrupts(nvme_t *nvme) 22683c9168faSHans Rosenfeld { 22693c9168faSHans Rosenfeld int i; 22703c9168faSHans Rosenfeld 22713c9168faSHans Rosenfeld for (i = 0; i < nvme->n_intr_cnt; i++) { 22723c9168faSHans Rosenfeld if (nvme->n_inth[i] == NULL) 22733c9168faSHans Rosenfeld break; 22743c9168faSHans Rosenfeld 22753c9168faSHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 22763c9168faSHans Rosenfeld (void) ddi_intr_block_disable(&nvme->n_inth[i], 1); 22773c9168faSHans Rosenfeld else 22783c9168faSHans Rosenfeld (void) ddi_intr_disable(nvme->n_inth[i]); 22793c9168faSHans Rosenfeld 22803c9168faSHans Rosenfeld (void) ddi_intr_remove_handler(nvme->n_inth[i]); 22813c9168faSHans Rosenfeld (void) ddi_intr_free(nvme->n_inth[i]); 22823c9168faSHans Rosenfeld } 22833c9168faSHans Rosenfeld 22843c9168faSHans Rosenfeld kmem_free(nvme->n_inth, nvme->n_inth_sz); 22853c9168faSHans Rosenfeld nvme->n_inth = NULL; 22863c9168faSHans Rosenfeld nvme->n_inth_sz = 0; 22873c9168faSHans Rosenfeld 22883c9168faSHans Rosenfeld nvme->n_progress &= ~NVME_INTERRUPTS; 22893c9168faSHans Rosenfeld } 22903c9168faSHans Rosenfeld 22913c9168faSHans Rosenfeld static int 22923c9168faSHans Rosenfeld nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs) 22933c9168faSHans Rosenfeld { 22943c9168faSHans Rosenfeld int nintrs, navail, count; 22953c9168faSHans Rosenfeld int ret; 22963c9168faSHans Rosenfeld int i; 22973c9168faSHans Rosenfeld 22983c9168faSHans Rosenfeld if (nvme->n_intr_types == 0) { 22993c9168faSHans Rosenfeld ret = ddi_intr_get_supported_types(nvme->n_dip, 23003c9168faSHans Rosenfeld &nvme->n_intr_types); 23013c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23023c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23033c9168faSHans Rosenfeld "!%s: ddi_intr_get_supported types failed", 23043c9168faSHans Rosenfeld __func__); 23053c9168faSHans Rosenfeld return (ret); 23063c9168faSHans Rosenfeld } 23073c9168faSHans Rosenfeld } 23083c9168faSHans Rosenfeld 23093c9168faSHans Rosenfeld if ((nvme->n_intr_types & intr_type) == 0) 23103c9168faSHans Rosenfeld return (DDI_FAILURE); 23113c9168faSHans Rosenfeld 23123c9168faSHans Rosenfeld ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs); 23133c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23143c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed", 23153c9168faSHans Rosenfeld __func__); 23163c9168faSHans Rosenfeld return (ret); 23173c9168faSHans Rosenfeld } 23183c9168faSHans Rosenfeld 23193c9168faSHans Rosenfeld ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail); 23203c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23213c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed", 23223c9168faSHans Rosenfeld __func__); 23233c9168faSHans Rosenfeld return (ret); 23243c9168faSHans Rosenfeld } 23253c9168faSHans Rosenfeld 23263c9168faSHans Rosenfeld /* We want at most one interrupt per queue pair. */ 23273c9168faSHans Rosenfeld if (navail > nqpairs) 23283c9168faSHans Rosenfeld navail = nqpairs; 23293c9168faSHans Rosenfeld 23303c9168faSHans Rosenfeld nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail; 23313c9168faSHans Rosenfeld nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP); 23323c9168faSHans Rosenfeld 23333c9168faSHans Rosenfeld ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail, 23343c9168faSHans Rosenfeld &count, 0); 23353c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23363c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed", 23373c9168faSHans Rosenfeld __func__); 23383c9168faSHans Rosenfeld goto fail; 23393c9168faSHans Rosenfeld } 23403c9168faSHans Rosenfeld 23413c9168faSHans Rosenfeld nvme->n_intr_cnt = count; 23423c9168faSHans Rosenfeld 23433c9168faSHans Rosenfeld ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri); 23443c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23453c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed", 23463c9168faSHans Rosenfeld __func__); 23473c9168faSHans Rosenfeld goto fail; 23483c9168faSHans Rosenfeld } 23493c9168faSHans Rosenfeld 23503c9168faSHans Rosenfeld for (i = 0; i < count; i++) { 23513c9168faSHans Rosenfeld ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr, 23523c9168faSHans Rosenfeld (void *)nvme, (void *)(uintptr_t)i); 23533c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 23543c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23553c9168faSHans Rosenfeld "!%s: ddi_intr_add_handler failed", __func__); 23563c9168faSHans Rosenfeld goto fail; 23573c9168faSHans Rosenfeld } 23583c9168faSHans Rosenfeld } 23593c9168faSHans Rosenfeld 23603c9168faSHans Rosenfeld (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap); 23613c9168faSHans Rosenfeld 23626afc9eb2SHans Rosenfeld for (i = 0; i < count; i++) { 236375b41617SHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 236475b41617SHans Rosenfeld ret = ddi_intr_block_enable(&nvme->n_inth[i], 1); 236575b41617SHans Rosenfeld else 236675b41617SHans Rosenfeld ret = ddi_intr_enable(nvme->n_inth[i]); 23673c9168faSHans Rosenfeld 236875b41617SHans Rosenfeld if (ret != DDI_SUCCESS) { 236975b41617SHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 237075b41617SHans Rosenfeld "!%s: enabling interrupt %d failed", __func__, i); 237175b41617SHans Rosenfeld goto fail; 23726afc9eb2SHans Rosenfeld } 23736afc9eb2SHans Rosenfeld } 23746afc9eb2SHans Rosenfeld 23753c9168faSHans Rosenfeld nvme->n_intr_type = intr_type; 23763c9168faSHans Rosenfeld 23773c9168faSHans Rosenfeld nvme->n_progress |= NVME_INTERRUPTS; 23783c9168faSHans Rosenfeld 23793c9168faSHans Rosenfeld return (DDI_SUCCESS); 23803c9168faSHans Rosenfeld 23813c9168faSHans Rosenfeld fail: 23823c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 23833c9168faSHans Rosenfeld 23843c9168faSHans Rosenfeld return (ret); 23853c9168faSHans Rosenfeld } 23863c9168faSHans Rosenfeld 23873c9168faSHans Rosenfeld static int 23883c9168faSHans Rosenfeld nvme_fm_errcb(dev_info_t *dip, ddi_fm_error_t *fm_error, const void *arg) 23893c9168faSHans Rosenfeld { 23903c9168faSHans Rosenfeld _NOTE(ARGUNUSED(arg)); 23913c9168faSHans Rosenfeld 23923c9168faSHans Rosenfeld pci_ereport_post(dip, fm_error, NULL); 23933c9168faSHans Rosenfeld return (fm_error->fme_status); 23943c9168faSHans Rosenfeld } 23953c9168faSHans Rosenfeld 23963c9168faSHans Rosenfeld static int 23973c9168faSHans Rosenfeld nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 23983c9168faSHans Rosenfeld { 23993c9168faSHans Rosenfeld nvme_t *nvme; 24003c9168faSHans Rosenfeld int instance; 24013c9168faSHans Rosenfeld int nregs; 24023c9168faSHans Rosenfeld off_t regsize; 24033c9168faSHans Rosenfeld int i; 24043c9168faSHans Rosenfeld char name[32]; 24053c9168faSHans Rosenfeld 24063c9168faSHans Rosenfeld if (cmd != DDI_ATTACH) 24073c9168faSHans Rosenfeld return (DDI_FAILURE); 24083c9168faSHans Rosenfeld 24093c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 24103c9168faSHans Rosenfeld 24113c9168faSHans Rosenfeld if (ddi_soft_state_zalloc(nvme_state, instance) != DDI_SUCCESS) 24123c9168faSHans Rosenfeld return (DDI_FAILURE); 24133c9168faSHans Rosenfeld 24143c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 24153c9168faSHans Rosenfeld ddi_set_driver_private(dip, nvme); 24163c9168faSHans Rosenfeld nvme->n_dip = dip; 24173c9168faSHans Rosenfeld 24183c9168faSHans Rosenfeld nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 24193c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "strict-version", 1) == 1 ? B_TRUE : B_FALSE; 24203c9168faSHans Rosenfeld nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY, 24213c9168faSHans Rosenfeld dip, DDI_PROP_DONTPASS, "ignore-unknown-vendor-status", 0) == 1 ? 24223c9168faSHans Rosenfeld B_TRUE : B_FALSE; 24233c9168faSHans Rosenfeld nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 24243c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "admin-queue-len", NVME_DEFAULT_ADMIN_QUEUE_LEN); 24253c9168faSHans Rosenfeld nvme->n_io_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 24263c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "io-queue-len", NVME_DEFAULT_IO_QUEUE_LEN); 24273c9168faSHans Rosenfeld nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 24283c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "async-event-limit", 24293c9168faSHans Rosenfeld NVME_DEFAULT_ASYNC_EVENT_LIMIT); 24303c9168faSHans Rosenfeld 24313c9168faSHans Rosenfeld if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN) 24323c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN; 24333c9168faSHans Rosenfeld else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN) 24343c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN; 24353c9168faSHans Rosenfeld 24363c9168faSHans Rosenfeld if (nvme->n_io_queue_len < NVME_MIN_IO_QUEUE_LEN) 24373c9168faSHans Rosenfeld nvme->n_io_queue_len = NVME_MIN_IO_QUEUE_LEN; 24383c9168faSHans Rosenfeld 24393c9168faSHans Rosenfeld if (nvme->n_async_event_limit < 1) 24403c9168faSHans Rosenfeld nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT; 24413c9168faSHans Rosenfeld 24423c9168faSHans Rosenfeld nvme->n_reg_acc_attr = nvme_reg_acc_attr; 24433c9168faSHans Rosenfeld nvme->n_queue_dma_attr = nvme_queue_dma_attr; 24443c9168faSHans Rosenfeld nvme->n_prp_dma_attr = nvme_prp_dma_attr; 24453c9168faSHans Rosenfeld nvme->n_sgl_dma_attr = nvme_sgl_dma_attr; 24463c9168faSHans Rosenfeld 24473c9168faSHans Rosenfeld /* 24483c9168faSHans Rosenfeld * Setup FMA support. 24493c9168faSHans Rosenfeld */ 24503c9168faSHans Rosenfeld nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip, 24513c9168faSHans Rosenfeld DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 24523c9168faSHans Rosenfeld DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 24533c9168faSHans Rosenfeld DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 24543c9168faSHans Rosenfeld 24553c9168faSHans Rosenfeld ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc); 24563c9168faSHans Rosenfeld 24573c9168faSHans Rosenfeld if (nvme->n_fm_cap) { 24583c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE) 24593c9168faSHans Rosenfeld nvme->n_reg_acc_attr.devacc_attr_access = 24603c9168faSHans Rosenfeld DDI_FLAGERR_ACC; 24613c9168faSHans Rosenfeld 24623c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) { 24633c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 24643c9168faSHans Rosenfeld nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 24653c9168faSHans Rosenfeld } 24663c9168faSHans Rosenfeld 24673c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 24683c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 24693c9168faSHans Rosenfeld pci_ereport_setup(dip); 24703c9168faSHans Rosenfeld 24713c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 24723c9168faSHans Rosenfeld ddi_fm_handler_register(dip, nvme_fm_errcb, 24733c9168faSHans Rosenfeld (void *)nvme); 24743c9168faSHans Rosenfeld } 24753c9168faSHans Rosenfeld 24763c9168faSHans Rosenfeld nvme->n_progress |= NVME_FMA_INIT; 24773c9168faSHans Rosenfeld 24783c9168faSHans Rosenfeld /* 24793c9168faSHans Rosenfeld * The spec defines several register sets. Only the controller 24803c9168faSHans Rosenfeld * registers (set 1) are currently used. 24813c9168faSHans Rosenfeld */ 24823c9168faSHans Rosenfeld if (ddi_dev_nregs(dip, &nregs) == DDI_FAILURE || 24833c9168faSHans Rosenfeld nregs < 2 || 24843c9168faSHans Rosenfeld ddi_dev_regsize(dip, 1, ®size) == DDI_FAILURE) 24853c9168faSHans Rosenfeld goto fail; 24863c9168faSHans Rosenfeld 24873c9168faSHans Rosenfeld if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize, 24883c9168faSHans Rosenfeld &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) { 24893c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to map regset 1"); 24903c9168faSHans Rosenfeld goto fail; 24913c9168faSHans Rosenfeld } 24923c9168faSHans Rosenfeld 24933c9168faSHans Rosenfeld nvme->n_progress |= NVME_REGS_MAPPED; 24943c9168faSHans Rosenfeld 24953c9168faSHans Rosenfeld /* 24963c9168faSHans Rosenfeld * Create taskq for command completion. 24973c9168faSHans Rosenfeld */ 24983c9168faSHans Rosenfeld (void) snprintf(name, sizeof (name), "%s%d_cmd_taskq", 24993c9168faSHans Rosenfeld ddi_driver_name(dip), ddi_get_instance(dip)); 25003c9168faSHans Rosenfeld nvme->n_cmd_taskq = ddi_taskq_create(dip, name, MIN(UINT16_MAX, ncpus), 25013c9168faSHans Rosenfeld TASKQ_DEFAULTPRI, 0); 25023c9168faSHans Rosenfeld if (nvme->n_cmd_taskq == NULL) { 25033c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to create cmd taskq"); 25043c9168faSHans Rosenfeld goto fail; 25053c9168faSHans Rosenfeld } 25063c9168faSHans Rosenfeld 25078834f7acSYouzhong Yang /* 25088834f7acSYouzhong Yang * Create PRP DMA cache 25098834f7acSYouzhong Yang */ 25108834f7acSYouzhong Yang (void) snprintf(name, sizeof (name), "%s%d_prp_cache", 25118834f7acSYouzhong Yang ddi_driver_name(dip), ddi_get_instance(dip)); 25128834f7acSYouzhong Yang nvme->n_prp_cache = kmem_cache_create(name, sizeof (nvme_dma_t), 25138834f7acSYouzhong Yang 0, nvme_prp_dma_constructor, nvme_prp_dma_destructor, 25148834f7acSYouzhong Yang NULL, (void *)nvme, NULL, 0); 25153c9168faSHans Rosenfeld 25163c9168faSHans Rosenfeld if (nvme_init(nvme) != DDI_SUCCESS) 25173c9168faSHans Rosenfeld goto fail; 25183c9168faSHans Rosenfeld 25193c9168faSHans Rosenfeld /* 25203c9168faSHans Rosenfeld * Attach the blkdev driver for each namespace. 25213c9168faSHans Rosenfeld */ 25223c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 25233c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_ignore) 25243c9168faSHans Rosenfeld continue; 25253c9168faSHans Rosenfeld 25263c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl = bd_alloc_handle(&nvme->n_ns[i], 25273c9168faSHans Rosenfeld &nvme_bd_ops, &nvme->n_prp_dma_attr, KM_SLEEP); 25283c9168faSHans Rosenfeld 25293c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl == NULL) { 25303c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 25313c9168faSHans Rosenfeld "!failed to get blkdev handle for namespace %d", i); 25323c9168faSHans Rosenfeld goto fail; 25333c9168faSHans Rosenfeld } 25343c9168faSHans Rosenfeld 25353c9168faSHans Rosenfeld if (bd_attach_handle(dip, nvme->n_ns[i].ns_bd_hdl) 25363c9168faSHans Rosenfeld != DDI_SUCCESS) { 25373c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 25383c9168faSHans Rosenfeld "!failed to attach blkdev handle for namespace %d", 25393c9168faSHans Rosenfeld i); 25403c9168faSHans Rosenfeld goto fail; 25413c9168faSHans Rosenfeld } 25423c9168faSHans Rosenfeld } 25433c9168faSHans Rosenfeld 25443c9168faSHans Rosenfeld return (DDI_SUCCESS); 25453c9168faSHans Rosenfeld 25463c9168faSHans Rosenfeld fail: 25473c9168faSHans Rosenfeld /* attach successful anyway so that FMA can retire the device */ 25483c9168faSHans Rosenfeld if (nvme->n_dead) 25493c9168faSHans Rosenfeld return (DDI_SUCCESS); 25503c9168faSHans Rosenfeld 25513c9168faSHans Rosenfeld (void) nvme_detach(dip, DDI_DETACH); 25523c9168faSHans Rosenfeld 25533c9168faSHans Rosenfeld return (DDI_FAILURE); 25543c9168faSHans Rosenfeld } 25553c9168faSHans Rosenfeld 25563c9168faSHans Rosenfeld static int 25573c9168faSHans Rosenfeld nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 25583c9168faSHans Rosenfeld { 25593c9168faSHans Rosenfeld int instance, i; 25603c9168faSHans Rosenfeld nvme_t *nvme; 25613c9168faSHans Rosenfeld 25623c9168faSHans Rosenfeld if (cmd != DDI_DETACH) 25633c9168faSHans Rosenfeld return (DDI_FAILURE); 25643c9168faSHans Rosenfeld 25653c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 25663c9168faSHans Rosenfeld 25673c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 25683c9168faSHans Rosenfeld 25693c9168faSHans Rosenfeld if (nvme == NULL) 25703c9168faSHans Rosenfeld return (DDI_FAILURE); 25713c9168faSHans Rosenfeld 25723c9168faSHans Rosenfeld if (nvme->n_ns) { 25733c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 25743c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl) { 25753c9168faSHans Rosenfeld (void) bd_detach_handle( 25763c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl); 25773c9168faSHans Rosenfeld bd_free_handle(nvme->n_ns[i].ns_bd_hdl); 25783c9168faSHans Rosenfeld } 25793c9168faSHans Rosenfeld 25803c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_idns) 25813c9168faSHans Rosenfeld kmem_free(nvme->n_ns[i].ns_idns, 25823c9168faSHans Rosenfeld sizeof (nvme_identify_nsid_t)); 25833c9168faSHans Rosenfeld } 25843c9168faSHans Rosenfeld 25853c9168faSHans Rosenfeld kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) * 25863c9168faSHans Rosenfeld nvme->n_namespace_count); 25873c9168faSHans Rosenfeld } 25883c9168faSHans Rosenfeld 25893c9168faSHans Rosenfeld if (nvme->n_progress & NVME_INTERRUPTS) 25903c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 25913c9168faSHans Rosenfeld 25923c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 25933c9168faSHans Rosenfeld ddi_taskq_wait(nvme->n_cmd_taskq); 25943c9168faSHans Rosenfeld 25953c9168faSHans Rosenfeld if (nvme->n_ioq_count > 0) { 25963c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 25973c9168faSHans Rosenfeld if (nvme->n_ioq[i] != NULL) { 25983c9168faSHans Rosenfeld /* TODO: send destroy queue commands */ 25993c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_ioq[i]); 26003c9168faSHans Rosenfeld } 26013c9168faSHans Rosenfeld } 26023c9168faSHans Rosenfeld 26033c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) * 26043c9168faSHans Rosenfeld (nvme->n_ioq_count + 1)); 26053c9168faSHans Rosenfeld } 26063c9168faSHans Rosenfeld 26078834f7acSYouzhong Yang if (nvme->n_prp_cache != NULL) { 26088834f7acSYouzhong Yang kmem_cache_destroy(nvme->n_prp_cache); 26098834f7acSYouzhong Yang } 26108834f7acSYouzhong Yang 26113c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) { 26123c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE); 26133c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 26143c9168faSHans Rosenfeld } 26153c9168faSHans Rosenfeld 26163c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 26173c9168faSHans Rosenfeld ddi_taskq_destroy(nvme->n_cmd_taskq); 26183c9168faSHans Rosenfeld 26193c9168faSHans Rosenfeld if (nvme->n_progress & NVME_CTRL_LIMITS) 26203c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 26213c9168faSHans Rosenfeld 26223c9168faSHans Rosenfeld if (nvme->n_progress & NVME_ADMIN_QUEUE) 26233c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_adminq); 26243c9168faSHans Rosenfeld 26253c9168faSHans Rosenfeld if (nvme->n_idctl) 26263c9168faSHans Rosenfeld kmem_free(nvme->n_idctl, sizeof (nvme_identify_ctrl_t)); 26273c9168faSHans Rosenfeld 26283c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) 26293c9168faSHans Rosenfeld ddi_regs_map_free(&nvme->n_regh); 26303c9168faSHans Rosenfeld 26313c9168faSHans Rosenfeld if (nvme->n_progress & NVME_FMA_INIT) { 26323c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 26333c9168faSHans Rosenfeld ddi_fm_handler_unregister(nvme->n_dip); 26343c9168faSHans Rosenfeld 26353c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 26363c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 26373c9168faSHans Rosenfeld pci_ereport_teardown(nvme->n_dip); 26383c9168faSHans Rosenfeld 26393c9168faSHans Rosenfeld ddi_fm_fini(nvme->n_dip); 26403c9168faSHans Rosenfeld } 26413c9168faSHans Rosenfeld 2642510a6847SHans Rosenfeld if (nvme->n_vendor != NULL) 2643510a6847SHans Rosenfeld strfree(nvme->n_vendor); 2644510a6847SHans Rosenfeld 2645510a6847SHans Rosenfeld if (nvme->n_product != NULL) 2646510a6847SHans Rosenfeld strfree(nvme->n_product); 2647510a6847SHans Rosenfeld 26483c9168faSHans Rosenfeld ddi_soft_state_free(nvme_state, instance); 26493c9168faSHans Rosenfeld 26503c9168faSHans Rosenfeld return (DDI_SUCCESS); 26513c9168faSHans Rosenfeld } 26523c9168faSHans Rosenfeld 26533c9168faSHans Rosenfeld static int 26543c9168faSHans Rosenfeld nvme_quiesce(dev_info_t *dip) 26553c9168faSHans Rosenfeld { 26563c9168faSHans Rosenfeld int instance; 26573c9168faSHans Rosenfeld nvme_t *nvme; 26583c9168faSHans Rosenfeld 26593c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 26603c9168faSHans Rosenfeld 26613c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 26623c9168faSHans Rosenfeld 26633c9168faSHans Rosenfeld if (nvme == NULL) 26643c9168faSHans Rosenfeld return (DDI_FAILURE); 26653c9168faSHans Rosenfeld 26663c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_ABRUPT, B_TRUE); 26673c9168faSHans Rosenfeld 26683c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_TRUE); 26693c9168faSHans Rosenfeld 26703c9168faSHans Rosenfeld return (DDI_FAILURE); 26713c9168faSHans Rosenfeld } 26723c9168faSHans Rosenfeld 26733c9168faSHans Rosenfeld static int 26743c9168faSHans Rosenfeld nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer) 26753c9168faSHans Rosenfeld { 26763c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 26773c9168faSHans Rosenfeld int nprp_page, nprp; 26783c9168faSHans Rosenfeld uint64_t *prp; 26793c9168faSHans Rosenfeld 26803c9168faSHans Rosenfeld if (xfer->x_ndmac == 0) 26813c9168faSHans Rosenfeld return (DDI_FAILURE); 26823c9168faSHans Rosenfeld 26833c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress; 26843c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 26853c9168faSHans Rosenfeld 26863c9168faSHans Rosenfeld if (xfer->x_ndmac == 1) { 26873c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; 26883c9168faSHans Rosenfeld return (DDI_SUCCESS); 26893c9168faSHans Rosenfeld } else if (xfer->x_ndmac == 2) { 26903c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress; 26913c9168faSHans Rosenfeld return (DDI_SUCCESS); 26923c9168faSHans Rosenfeld } 26933c9168faSHans Rosenfeld 26943c9168faSHans Rosenfeld xfer->x_ndmac--; 26953c9168faSHans Rosenfeld 26963c9168faSHans Rosenfeld nprp_page = nvme->n_pagesize / sizeof (uint64_t) - 1; 26973c9168faSHans Rosenfeld ASSERT(nprp_page > 0); 26983c9168faSHans Rosenfeld nprp = (xfer->x_ndmac + nprp_page - 1) / nprp_page; 26993c9168faSHans Rosenfeld 27003c9168faSHans Rosenfeld /* 27013c9168faSHans Rosenfeld * We currently don't support chained PRPs and set up our DMA 27023c9168faSHans Rosenfeld * attributes to reflect that. If we still get an I/O request 27033c9168faSHans Rosenfeld * that needs a chained PRP something is very wrong. 27043c9168faSHans Rosenfeld */ 27053c9168faSHans Rosenfeld VERIFY(nprp == 1); 27063c9168faSHans Rosenfeld 27078834f7acSYouzhong Yang cmd->nc_dma = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP); 27088834f7acSYouzhong Yang bzero(cmd->nc_dma->nd_memp, cmd->nc_dma->nd_len); 27093c9168faSHans Rosenfeld 27103c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress; 27113c9168faSHans Rosenfeld 27123c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 27133c9168faSHans Rosenfeld for (prp = (uint64_t *)cmd->nc_dma->nd_memp; 27143c9168faSHans Rosenfeld xfer->x_ndmac > 0; 27153c9168faSHans Rosenfeld prp++, xfer->x_ndmac--) { 27163c9168faSHans Rosenfeld *prp = xfer->x_dmac.dmac_laddress; 27173c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 27183c9168faSHans Rosenfeld } 27193c9168faSHans Rosenfeld 27203c9168faSHans Rosenfeld (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len, 27213c9168faSHans Rosenfeld DDI_DMA_SYNC_FORDEV); 27223c9168faSHans Rosenfeld return (DDI_SUCCESS); 27233c9168faSHans Rosenfeld } 27243c9168faSHans Rosenfeld 27253c9168faSHans Rosenfeld static nvme_cmd_t * 27263c9168faSHans Rosenfeld nvme_create_nvm_cmd(nvme_namespace_t *ns, uint8_t opc, bd_xfer_t *xfer) 27273c9168faSHans Rosenfeld { 27283c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 27293c9168faSHans Rosenfeld nvme_cmd_t *cmd; 27303c9168faSHans Rosenfeld 27313c9168faSHans Rosenfeld /* 27323c9168faSHans Rosenfeld * Blkdev only sets BD_XFER_POLL when dumping, so don't sleep. 27333c9168faSHans Rosenfeld */ 27343c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ? 27353c9168faSHans Rosenfeld KM_NOSLEEP : KM_SLEEP); 27363c9168faSHans Rosenfeld 27373c9168faSHans Rosenfeld if (cmd == NULL) 27383c9168faSHans Rosenfeld return (NULL); 27393c9168faSHans Rosenfeld 27403c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = opc; 27413c9168faSHans Rosenfeld cmd->nc_callback = nvme_bd_xfer_done; 27423c9168faSHans Rosenfeld cmd->nc_xfer = xfer; 27433c9168faSHans Rosenfeld 27443c9168faSHans Rosenfeld switch (opc) { 27453c9168faSHans Rosenfeld case NVME_OPC_NVM_WRITE: 27463c9168faSHans Rosenfeld case NVME_OPC_NVM_READ: 27473c9168faSHans Rosenfeld VERIFY(xfer->x_nblks <= 0x10000); 27483c9168faSHans Rosenfeld 27493c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 27503c9168faSHans Rosenfeld 27513c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu; 27523c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32); 27533c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1); 27543c9168faSHans Rosenfeld 27553c9168faSHans Rosenfeld if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS) 27563c9168faSHans Rosenfeld goto fail; 27573c9168faSHans Rosenfeld break; 27583c9168faSHans Rosenfeld 27593c9168faSHans Rosenfeld case NVME_OPC_NVM_FLUSH: 27603c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 27613c9168faSHans Rosenfeld break; 27623c9168faSHans Rosenfeld 27633c9168faSHans Rosenfeld default: 27643c9168faSHans Rosenfeld goto fail; 27653c9168faSHans Rosenfeld } 27663c9168faSHans Rosenfeld 27673c9168faSHans Rosenfeld return (cmd); 27683c9168faSHans Rosenfeld 27693c9168faSHans Rosenfeld fail: 27703c9168faSHans Rosenfeld nvme_free_cmd(cmd); 27713c9168faSHans Rosenfeld return (NULL); 27723c9168faSHans Rosenfeld } 27733c9168faSHans Rosenfeld 27743c9168faSHans Rosenfeld static void 27753c9168faSHans Rosenfeld nvme_bd_xfer_done(void *arg) 27763c9168faSHans Rosenfeld { 27773c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 27783c9168faSHans Rosenfeld bd_xfer_t *xfer = cmd->nc_xfer; 27793c9168faSHans Rosenfeld int error = 0; 27803c9168faSHans Rosenfeld 27813c9168faSHans Rosenfeld error = nvme_check_cmd_status(cmd); 27823c9168faSHans Rosenfeld nvme_free_cmd(cmd); 27833c9168faSHans Rosenfeld 27843c9168faSHans Rosenfeld bd_xfer_done(xfer, error); 27853c9168faSHans Rosenfeld } 27863c9168faSHans Rosenfeld 27873c9168faSHans Rosenfeld static void 27883c9168faSHans Rosenfeld nvme_bd_driveinfo(void *arg, bd_drive_t *drive) 27893c9168faSHans Rosenfeld { 27903c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 27913c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 27923c9168faSHans Rosenfeld 27933c9168faSHans Rosenfeld /* 27943c9168faSHans Rosenfeld * blkdev maintains one queue size per instance (namespace), 27953c9168faSHans Rosenfeld * but all namespace share the I/O queues. 27963c9168faSHans Rosenfeld * TODO: need to figure out a sane default, or use per-NS I/O queues, 27973c9168faSHans Rosenfeld * or change blkdev to handle EAGAIN 27983c9168faSHans Rosenfeld */ 27993c9168faSHans Rosenfeld drive->d_qsize = nvme->n_ioq_count * nvme->n_io_queue_len 28003c9168faSHans Rosenfeld / nvme->n_namespace_count; 28013c9168faSHans Rosenfeld 28023c9168faSHans Rosenfeld /* 28033c9168faSHans Rosenfeld * d_maxxfer is not set, which means the value is taken from the DMA 28043c9168faSHans Rosenfeld * attributes specified to bd_alloc_handle. 28053c9168faSHans Rosenfeld */ 28063c9168faSHans Rosenfeld 28073c9168faSHans Rosenfeld drive->d_removable = B_FALSE; 28083c9168faSHans Rosenfeld drive->d_hotpluggable = B_FALSE; 28093c9168faSHans Rosenfeld 28103c9168faSHans Rosenfeld drive->d_target = ns->ns_id; 28113c9168faSHans Rosenfeld drive->d_lun = 0; 2812510a6847SHans Rosenfeld 2813bef9e21aSHans Rosenfeld drive->d_model = nvme->n_idctl->id_model; 2814bef9e21aSHans Rosenfeld drive->d_model_len = sizeof (nvme->n_idctl->id_model); 2815510a6847SHans Rosenfeld drive->d_vendor = nvme->n_vendor; 2816510a6847SHans Rosenfeld drive->d_vendor_len = strlen(nvme->n_vendor); 2817510a6847SHans Rosenfeld drive->d_product = nvme->n_product; 2818510a6847SHans Rosenfeld drive->d_product_len = strlen(nvme->n_product); 2819510a6847SHans Rosenfeld drive->d_serial = nvme->n_idctl->id_serial; 2820510a6847SHans Rosenfeld drive->d_serial_len = sizeof (nvme->n_idctl->id_serial); 2821510a6847SHans Rosenfeld drive->d_revision = nvme->n_idctl->id_fwrev; 2822510a6847SHans Rosenfeld drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev); 28233c9168faSHans Rosenfeld } 28243c9168faSHans Rosenfeld 28253c9168faSHans Rosenfeld static int 28263c9168faSHans Rosenfeld nvme_bd_mediainfo(void *arg, bd_media_t *media) 28273c9168faSHans Rosenfeld { 28283c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28293c9168faSHans Rosenfeld 28303c9168faSHans Rosenfeld media->m_nblks = ns->ns_block_count; 28313c9168faSHans Rosenfeld media->m_blksize = ns->ns_block_size; 28323c9168faSHans Rosenfeld media->m_readonly = B_FALSE; 28333c9168faSHans Rosenfeld media->m_solidstate = B_TRUE; 28343c9168faSHans Rosenfeld 28353c9168faSHans Rosenfeld media->m_pblksize = ns->ns_best_block_size; 28363c9168faSHans Rosenfeld 28373c9168faSHans Rosenfeld return (0); 28383c9168faSHans Rosenfeld } 28393c9168faSHans Rosenfeld 28403c9168faSHans Rosenfeld static int 28413c9168faSHans Rosenfeld nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc) 28423c9168faSHans Rosenfeld { 28433c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 28443c9168faSHans Rosenfeld nvme_cmd_t *cmd; 28453c9168faSHans Rosenfeld 28463c9168faSHans Rosenfeld if (nvme->n_dead) 28473c9168faSHans Rosenfeld return (EIO); 28483c9168faSHans Rosenfeld 28493c9168faSHans Rosenfeld /* No polling for now */ 28503c9168faSHans Rosenfeld if (xfer->x_flags & BD_XFER_POLL) 28513c9168faSHans Rosenfeld return (EIO); 28523c9168faSHans Rosenfeld 28533c9168faSHans Rosenfeld cmd = nvme_create_nvm_cmd(ns, opc, xfer); 28543c9168faSHans Rosenfeld if (cmd == NULL) 28553c9168faSHans Rosenfeld return (ENOMEM); 28563c9168faSHans Rosenfeld 28573c9168faSHans Rosenfeld cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1; 28583c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); 28593c9168faSHans Rosenfeld 28603c9168faSHans Rosenfeld if (nvme_submit_cmd(nvme->n_ioq[cmd->nc_sqid], cmd) 28613c9168faSHans Rosenfeld != DDI_SUCCESS) 28623c9168faSHans Rosenfeld return (EAGAIN); 28633c9168faSHans Rosenfeld 28643c9168faSHans Rosenfeld return (0); 28653c9168faSHans Rosenfeld } 28663c9168faSHans Rosenfeld 28673c9168faSHans Rosenfeld static int 28683c9168faSHans Rosenfeld nvme_bd_read(void *arg, bd_xfer_t *xfer) 28693c9168faSHans Rosenfeld { 28703c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28713c9168faSHans Rosenfeld 28723c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_READ)); 28733c9168faSHans Rosenfeld } 28743c9168faSHans Rosenfeld 28753c9168faSHans Rosenfeld static int 28763c9168faSHans Rosenfeld nvme_bd_write(void *arg, bd_xfer_t *xfer) 28773c9168faSHans Rosenfeld { 28783c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28793c9168faSHans Rosenfeld 28803c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_WRITE)); 28813c9168faSHans Rosenfeld } 28823c9168faSHans Rosenfeld 28833c9168faSHans Rosenfeld static int 28843c9168faSHans Rosenfeld nvme_bd_sync(void *arg, bd_xfer_t *xfer) 28853c9168faSHans Rosenfeld { 28863c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 28873c9168faSHans Rosenfeld 28883c9168faSHans Rosenfeld if (ns->ns_nvme->n_dead) 28893c9168faSHans Rosenfeld return (EIO); 28903c9168faSHans Rosenfeld 28913c9168faSHans Rosenfeld /* 28923c9168faSHans Rosenfeld * If the volatile write cache isn't enabled the FLUSH command is a 28933c9168faSHans Rosenfeld * no-op, so we can take a shortcut here. 28943c9168faSHans Rosenfeld */ 28953c9168faSHans Rosenfeld if (ns->ns_nvme->n_volatile_write_cache_enabled == B_FALSE) { 28963c9168faSHans Rosenfeld bd_xfer_done(xfer, ENOTSUP); 28973c9168faSHans Rosenfeld return (0); 28983c9168faSHans Rosenfeld } 28993c9168faSHans Rosenfeld 29003c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_FLUSH)); 29013c9168faSHans Rosenfeld } 29023c9168faSHans Rosenfeld 29033c9168faSHans Rosenfeld static int 29043c9168faSHans Rosenfeld nvme_bd_devid(void *arg, dev_info_t *devinfo, ddi_devid_t *devid) 29053c9168faSHans Rosenfeld { 29063c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 29073c9168faSHans Rosenfeld 29083c9168faSHans Rosenfeld return (ddi_devid_init(devinfo, DEVID_ENCAP, strlen(ns->ns_devid), 29093c9168faSHans Rosenfeld ns->ns_devid, devid)); 29103c9168faSHans Rosenfeld } 2911