13c9168faSHans Rosenfeld /* 23c9168faSHans Rosenfeld * This file and its contents are supplied under the terms of the 33c9168faSHans Rosenfeld * Common Development and Distribution License ("CDDL"), version 1.0. 43c9168faSHans Rosenfeld * You may only use this file in accordance with the terms of version 53c9168faSHans Rosenfeld * 1.0 of the CDDL. 63c9168faSHans Rosenfeld * 73c9168faSHans Rosenfeld * A full copy of the text of the CDDL should have accompanied this 83c9168faSHans Rosenfeld * source. A copy of the CDDL is also available via the Internet at 93c9168faSHans Rosenfeld * http://www.illumos.org/license/CDDL. 103c9168faSHans Rosenfeld */ 113c9168faSHans Rosenfeld 123c9168faSHans Rosenfeld /* 136afc9eb2SHans Rosenfeld * Copyright 2016 Nexenta Systems, Inc. All rights reserved. 1434c938c7SPete Shephard * Copyright 2016 Tegile Systems, Inc. All rights reserved. 152f95345bSYouzhong Yang * Copyright (c) 2016 The MathWorks, Inc. All rights reserved. 163c9168faSHans Rosenfeld */ 173c9168faSHans Rosenfeld 183c9168faSHans Rosenfeld /* 193c9168faSHans Rosenfeld * blkdev driver for NVMe compliant storage devices 203c9168faSHans Rosenfeld * 2124979ca3SHans Rosenfeld * This driver was written to conform to version 1.1b of the NVMe specification. 223c9168faSHans Rosenfeld * It may work with newer versions, but that is completely untested and disabled 233c9168faSHans Rosenfeld * by default. 243c9168faSHans Rosenfeld * 253c9168faSHans Rosenfeld * The driver has only been tested on x86 systems and will not work on big- 263c9168faSHans Rosenfeld * endian systems without changes to the code accessing registers and data 273c9168faSHans Rosenfeld * structures used by the hardware. 283c9168faSHans Rosenfeld * 293c9168faSHans Rosenfeld * 303c9168faSHans Rosenfeld * Interrupt Usage: 313c9168faSHans Rosenfeld * 323c9168faSHans Rosenfeld * The driver will use a FIXED interrupt while configuring the device as the 333c9168faSHans Rosenfeld * specification requires. Later in the attach process it will switch to MSI-X 343c9168faSHans Rosenfeld * or MSI if supported. The driver wants to have one interrupt vector per CPU, 353c9168faSHans Rosenfeld * but it will work correctly if less are available. Interrupts can be shared 363c9168faSHans Rosenfeld * by queues, the interrupt handler will iterate through the I/O queue array by 373c9168faSHans Rosenfeld * steps of n_intr_cnt. Usually only the admin queue will share an interrupt 383c9168faSHans Rosenfeld * with one I/O queue. The interrupt handler will retrieve completed commands 393c9168faSHans Rosenfeld * from all queues sharing an interrupt vector and will post them to a taskq 403c9168faSHans Rosenfeld * for completion processing. 413c9168faSHans Rosenfeld * 423c9168faSHans Rosenfeld * 433c9168faSHans Rosenfeld * Command Processing: 443c9168faSHans Rosenfeld * 453c9168faSHans Rosenfeld * NVMe devices can have up to 65536 I/O queue pairs, with each queue holding up 463c9168faSHans Rosenfeld * to 65536 I/O commands. The driver will configure one I/O queue pair per 473c9168faSHans Rosenfeld * available interrupt vector, with the queue length usually much smaller than 483c9168faSHans Rosenfeld * the maximum of 65536. If the hardware doesn't provide enough queues, fewer 493c9168faSHans Rosenfeld * interrupt vectors will be used. 503c9168faSHans Rosenfeld * 513c9168faSHans Rosenfeld * Additionally the hardware provides a single special admin queue pair that can 523c9168faSHans Rosenfeld * hold up to 4096 admin commands. 533c9168faSHans Rosenfeld * 543c9168faSHans Rosenfeld * From the hardware perspective both queues of a queue pair are independent, 553c9168faSHans Rosenfeld * but they share some driver state: the command array (holding pointers to 563c9168faSHans Rosenfeld * commands currently being processed by the hardware) and the active command 573c9168faSHans Rosenfeld * counter. Access to the submission side of a queue pair and the shared state 583c9168faSHans Rosenfeld * is protected by nq_mutex. The completion side of a queue pair does not need 593c9168faSHans Rosenfeld * that protection apart from its access to the shared state; it is called only 603c9168faSHans Rosenfeld * in the interrupt handler which does not run concurrently for the same 613c9168faSHans Rosenfeld * interrupt vector. 623c9168faSHans Rosenfeld * 633c9168faSHans Rosenfeld * When a command is submitted to a queue pair the active command counter is 643c9168faSHans Rosenfeld * incremented and a pointer to the command is stored in the command array. The 653c9168faSHans Rosenfeld * array index is used as command identifier (CID) in the submission queue 663c9168faSHans Rosenfeld * entry. Some commands may take a very long time to complete, and if the queue 673c9168faSHans Rosenfeld * wraps around in that time a submission may find the next array slot to still 683c9168faSHans Rosenfeld * be used by a long-running command. In this case the array is sequentially 693c9168faSHans Rosenfeld * searched for the next free slot. The length of the command array is the same 703c9168faSHans Rosenfeld * as the configured queue length. 713c9168faSHans Rosenfeld * 723c9168faSHans Rosenfeld * 733c9168faSHans Rosenfeld * Namespace Support: 743c9168faSHans Rosenfeld * 753c9168faSHans Rosenfeld * NVMe devices can have multiple namespaces, each being a independent data 763c9168faSHans Rosenfeld * store. The driver supports multiple namespaces and creates a blkdev interface 773c9168faSHans Rosenfeld * for each namespace found. Namespaces can have various attributes to support 789cfb65faSHans Rosenfeld * thin provisioning and protection information. This driver does not support 799cfb65faSHans Rosenfeld * any of this and ignores namespaces that have these attributes. 803c9168faSHans Rosenfeld * 8124979ca3SHans Rosenfeld * As of NVMe 1.1 namespaces can have an 64bit Extended Unique Identifier 8224979ca3SHans Rosenfeld * (EUI64). This driver uses the EUI64 if present to generate the devid and 8324979ca3SHans Rosenfeld * passes it to blkdev to use it in the device node names. As this is currently 8424979ca3SHans Rosenfeld * untested namespaces with EUI64 are ignored by default. 8524979ca3SHans Rosenfeld * 863c9168faSHans Rosenfeld * 873c9168faSHans Rosenfeld * Blkdev Interface: 883c9168faSHans Rosenfeld * 893c9168faSHans Rosenfeld * This driver uses blkdev to do all the heavy lifting involved with presenting 903c9168faSHans Rosenfeld * a disk device to the system. As a result, the processing of I/O requests is 913c9168faSHans Rosenfeld * relatively simple as blkdev takes care of partitioning, boundary checks, DMA 923c9168faSHans Rosenfeld * setup, and splitting of transfers into manageable chunks. 933c9168faSHans Rosenfeld * 943c9168faSHans Rosenfeld * I/O requests coming in from blkdev are turned into NVM commands and posted to 953c9168faSHans Rosenfeld * an I/O queue. The queue is selected by taking the CPU id modulo the number of 963c9168faSHans Rosenfeld * queues. There is currently no timeout handling of I/O commands. 973c9168faSHans Rosenfeld * 983c9168faSHans Rosenfeld * Blkdev also supports querying device/media information and generating a 993c9168faSHans Rosenfeld * devid. The driver reports the best block size as determined by the namespace 1003c9168faSHans Rosenfeld * format back to blkdev as physical block size to support partition and block 10124979ca3SHans Rosenfeld * alignment. The devid is either based on the namespace EUI64, if present, or 10224979ca3SHans Rosenfeld * composed using the device vendor ID, model number, serial number, and the 10324979ca3SHans Rosenfeld * namespace ID. 1043c9168faSHans Rosenfeld * 1053c9168faSHans Rosenfeld * 1063c9168faSHans Rosenfeld * Error Handling: 1073c9168faSHans Rosenfeld * 1083c9168faSHans Rosenfeld * Error handling is currently limited to detecting fatal hardware errors, 1093c9168faSHans Rosenfeld * either by asynchronous events, or synchronously through command status or 1103c9168faSHans Rosenfeld * admin command timeouts. In case of severe errors the device is fenced off, 1113c9168faSHans Rosenfeld * all further requests will return EIO. FMA is then called to fault the device. 1123c9168faSHans Rosenfeld * 1133c9168faSHans Rosenfeld * The hardware has a limit for outstanding asynchronous event requests. Before 1143c9168faSHans Rosenfeld * this limit is known the driver assumes it is at least 1 and posts a single 1153c9168faSHans Rosenfeld * asynchronous request. Later when the limit is known more asynchronous event 1163c9168faSHans Rosenfeld * requests are posted to allow quicker reception of error information. When an 1173c9168faSHans Rosenfeld * asynchronous event is posted by the hardware the driver will parse the error 1183c9168faSHans Rosenfeld * status fields and log information or fault the device, depending on the 1193c9168faSHans Rosenfeld * severity of the asynchronous event. The asynchronous event request is then 1203c9168faSHans Rosenfeld * reused and posted to the admin queue again. 1213c9168faSHans Rosenfeld * 1223c9168faSHans Rosenfeld * On command completion the command status is checked for errors. In case of 1233c9168faSHans Rosenfeld * errors indicating a driver bug the driver panics. Almost all other error 1243c9168faSHans Rosenfeld * status values just cause EIO to be returned. 1253c9168faSHans Rosenfeld * 1263c9168faSHans Rosenfeld * Command timeouts are currently detected for all admin commands except 1273c9168faSHans Rosenfeld * asynchronous event requests. If a command times out and the hardware appears 1283c9168faSHans Rosenfeld * to be healthy the driver attempts to abort the command. If this fails the 1293c9168faSHans Rosenfeld * driver assumes the device to be dead, fences it off, and calls FMA to retire 1303c9168faSHans Rosenfeld * it. In general admin commands are issued at attach time only. No timeout 1313c9168faSHans Rosenfeld * handling of normal I/O commands is presently done. 1323c9168faSHans Rosenfeld * 1333c9168faSHans Rosenfeld * In some cases it may be possible that the ABORT command times out, too. In 1343c9168faSHans Rosenfeld * that case the device is also declared dead and fenced off. 1353c9168faSHans Rosenfeld * 1363c9168faSHans Rosenfeld * 1373c9168faSHans Rosenfeld * Quiesce / Fast Reboot: 1383c9168faSHans Rosenfeld * 1393c9168faSHans Rosenfeld * The driver currently does not support fast reboot. A quiesce(9E) entry point 1403c9168faSHans Rosenfeld * is still provided which is used to send a shutdown notification to the 1413c9168faSHans Rosenfeld * device. 1423c9168faSHans Rosenfeld * 1433c9168faSHans Rosenfeld * 1443c9168faSHans Rosenfeld * Driver Configuration: 1453c9168faSHans Rosenfeld * 1463c9168faSHans Rosenfeld * The following driver properties can be changed to control some aspects of the 1473c9168faSHans Rosenfeld * drivers operation: 1483c9168faSHans Rosenfeld * - strict-version: can be set to 0 to allow devices conforming to newer 14924979ca3SHans Rosenfeld * versions or namespaces with EUI64 to be used 1503c9168faSHans Rosenfeld * - ignore-unknown-vendor-status: can be set to 1 to not handle any vendor 1513c9168faSHans Rosenfeld * specific command status as a fatal error leading device faulting 1523c9168faSHans Rosenfeld * - admin-queue-len: the maximum length of the admin queue (16-4096) 1533c9168faSHans Rosenfeld * - io-queue-len: the maximum length of the I/O queues (16-65536) 1543c9168faSHans Rosenfeld * - async-event-limit: the maximum number of asynchronous event requests to be 1553c9168faSHans Rosenfeld * posted by the driver 156d148d46eSHans Rosenfeld * - volatile-write-cache-enable: can be set to 0 to disable the volatile write 157d148d46eSHans Rosenfeld * cache 1586801591eSHans Rosenfeld * - min-phys-block-size: the minimum physical block size to report to blkdev, 1596801591eSHans Rosenfeld * which is among other things the basis for ZFS vdev ashift 1603c9168faSHans Rosenfeld * 1613c9168faSHans Rosenfeld * 1623c9168faSHans Rosenfeld * TODO: 1633c9168faSHans Rosenfeld * - figure out sane default for I/O queue depth reported to blkdev 1643c9168faSHans Rosenfeld * - polled I/O support to support kernel core dumping 1653c9168faSHans Rosenfeld * - FMA handling of media errors 1663c9168faSHans Rosenfeld * - support for devices supporting very large I/O requests using chained PRPs 1673c9168faSHans Rosenfeld * - support for querying log pages from user space 1683c9168faSHans Rosenfeld * - support for configuring hardware parameters like interrupt coalescing 1693c9168faSHans Rosenfeld * - support for media formatting and hard partitioning into namespaces 1703c9168faSHans Rosenfeld * - support for big-endian systems 1713c9168faSHans Rosenfeld * - support for fast reboot 17224979ca3SHans Rosenfeld * - support for firmware updates 17324979ca3SHans Rosenfeld * - support for NVMe Subsystem Reset (1.1) 17424979ca3SHans Rosenfeld * - support for Scatter/Gather lists (1.1) 17524979ca3SHans Rosenfeld * - support for Reservations (1.1) 17624979ca3SHans Rosenfeld * - support for power management 1773c9168faSHans Rosenfeld */ 1783c9168faSHans Rosenfeld 1793c9168faSHans Rosenfeld #include <sys/byteorder.h> 1803c9168faSHans Rosenfeld #ifdef _BIG_ENDIAN 1813c9168faSHans Rosenfeld #error nvme driver needs porting for big-endian platforms 1823c9168faSHans Rosenfeld #endif 1833c9168faSHans Rosenfeld 1843c9168faSHans Rosenfeld #include <sys/modctl.h> 1853c9168faSHans Rosenfeld #include <sys/conf.h> 1863c9168faSHans Rosenfeld #include <sys/devops.h> 1873c9168faSHans Rosenfeld #include <sys/ddi.h> 1883c9168faSHans Rosenfeld #include <sys/sunddi.h> 1893c9168faSHans Rosenfeld #include <sys/bitmap.h> 1903c9168faSHans Rosenfeld #include <sys/sysmacros.h> 1913c9168faSHans Rosenfeld #include <sys/param.h> 1923c9168faSHans Rosenfeld #include <sys/varargs.h> 1933c9168faSHans Rosenfeld #include <sys/cpuvar.h> 1943c9168faSHans Rosenfeld #include <sys/disp.h> 1953c9168faSHans Rosenfeld #include <sys/blkdev.h> 1963c9168faSHans Rosenfeld #include <sys/atomic.h> 1973c9168faSHans Rosenfeld #include <sys/archsystm.h> 198510a6847SHans Rosenfeld #include <sys/sata/sata_hba.h> 1993c9168faSHans Rosenfeld 200*9d08e1f8SHans Rosenfeld #ifdef __x86 201*9d08e1f8SHans Rosenfeld #include <sys/x86_archext.h> 202*9d08e1f8SHans Rosenfeld #endif 203*9d08e1f8SHans Rosenfeld 2043c9168faSHans Rosenfeld #include "nvme_reg.h" 2053c9168faSHans Rosenfeld #include "nvme_var.h" 2063c9168faSHans Rosenfeld 2073c9168faSHans Rosenfeld 2083c9168faSHans Rosenfeld /* NVMe spec version supported */ 2093c9168faSHans Rosenfeld static const int nvme_version_major = 1; 21024979ca3SHans Rosenfeld static const int nvme_version_minor = 1; 2113c9168faSHans Rosenfeld 212e8ba2a38SHans Rosenfeld /* tunable for admin command timeout in seconds, default is 1s */ 213e8ba2a38SHans Rosenfeld static volatile int nvme_admin_cmd_timeout = 1; 214e8ba2a38SHans Rosenfeld 2153c9168faSHans Rosenfeld static int nvme_attach(dev_info_t *, ddi_attach_cmd_t); 2163c9168faSHans Rosenfeld static int nvme_detach(dev_info_t *, ddi_detach_cmd_t); 2173c9168faSHans Rosenfeld static int nvme_quiesce(dev_info_t *); 2183c9168faSHans Rosenfeld static int nvme_fm_errcb(dev_info_t *, ddi_fm_error_t *, const void *); 2193c9168faSHans Rosenfeld static int nvme_setup_interrupts(nvme_t *, int, int); 2203c9168faSHans Rosenfeld static void nvme_release_interrupts(nvme_t *); 2213c9168faSHans Rosenfeld static uint_t nvme_intr(caddr_t, caddr_t); 2223c9168faSHans Rosenfeld 2233c9168faSHans Rosenfeld static void nvme_shutdown(nvme_t *, int, boolean_t); 2243c9168faSHans Rosenfeld static boolean_t nvme_reset(nvme_t *, boolean_t); 2253c9168faSHans Rosenfeld static int nvme_init(nvme_t *); 2263c9168faSHans Rosenfeld static nvme_cmd_t *nvme_alloc_cmd(nvme_t *, int); 2273c9168faSHans Rosenfeld static void nvme_free_cmd(nvme_cmd_t *); 2283c9168faSHans Rosenfeld static nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t, 2293c9168faSHans Rosenfeld bd_xfer_t *); 2303c9168faSHans Rosenfeld static int nvme_admin_cmd(nvme_cmd_t *, int); 2313c9168faSHans Rosenfeld static int nvme_submit_cmd(nvme_qpair_t *, nvme_cmd_t *); 2323c9168faSHans Rosenfeld static nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *); 2333c9168faSHans Rosenfeld static boolean_t nvme_wait_cmd(nvme_cmd_t *, uint_t); 2343c9168faSHans Rosenfeld static void nvme_wakeup_cmd(void *); 2353c9168faSHans Rosenfeld static void nvme_async_event_task(void *); 2363c9168faSHans Rosenfeld 2373c9168faSHans Rosenfeld static int nvme_check_unknown_cmd_status(nvme_cmd_t *); 2383c9168faSHans Rosenfeld static int nvme_check_vendor_cmd_status(nvme_cmd_t *); 2393c9168faSHans Rosenfeld static int nvme_check_integrity_cmd_status(nvme_cmd_t *); 2403c9168faSHans Rosenfeld static int nvme_check_specific_cmd_status(nvme_cmd_t *); 2413c9168faSHans Rosenfeld static int nvme_check_generic_cmd_status(nvme_cmd_t *); 2423c9168faSHans Rosenfeld static inline int nvme_check_cmd_status(nvme_cmd_t *); 2433c9168faSHans Rosenfeld 2443c9168faSHans Rosenfeld static void nvme_abort_cmd(nvme_cmd_t *); 2453c9168faSHans Rosenfeld static int nvme_async_event(nvme_t *); 2463c9168faSHans Rosenfeld static void *nvme_get_logpage(nvme_t *, uint8_t, ...); 2473c9168faSHans Rosenfeld static void *nvme_identify(nvme_t *, uint32_t); 248d148d46eSHans Rosenfeld static boolean_t nvme_set_features(nvme_t *, uint32_t, uint8_t, uint32_t, 249d148d46eSHans Rosenfeld uint32_t *); 250d148d46eSHans Rosenfeld static boolean_t nvme_write_cache_set(nvme_t *, boolean_t); 2513c9168faSHans Rosenfeld static int nvme_set_nqueues(nvme_t *, uint16_t); 2523c9168faSHans Rosenfeld 2533c9168faSHans Rosenfeld static void nvme_free_dma(nvme_dma_t *); 2543c9168faSHans Rosenfeld static int nvme_zalloc_dma(nvme_t *, size_t, uint_t, ddi_dma_attr_t *, 2553c9168faSHans Rosenfeld nvme_dma_t **); 2563c9168faSHans Rosenfeld static int nvme_zalloc_queue_dma(nvme_t *, uint32_t, uint16_t, uint_t, 2573c9168faSHans Rosenfeld nvme_dma_t **); 2583c9168faSHans Rosenfeld static void nvme_free_qpair(nvme_qpair_t *); 2593c9168faSHans Rosenfeld static int nvme_alloc_qpair(nvme_t *, uint32_t, nvme_qpair_t **, int); 2603c9168faSHans Rosenfeld static int nvme_create_io_qpair(nvme_t *, nvme_qpair_t *, uint16_t); 2613c9168faSHans Rosenfeld 2623c9168faSHans Rosenfeld static inline void nvme_put64(nvme_t *, uintptr_t, uint64_t); 2633c9168faSHans Rosenfeld static inline void nvme_put32(nvme_t *, uintptr_t, uint32_t); 2643c9168faSHans Rosenfeld static inline uint64_t nvme_get64(nvme_t *, uintptr_t); 2653c9168faSHans Rosenfeld static inline uint32_t nvme_get32(nvme_t *, uintptr_t); 2663c9168faSHans Rosenfeld 2673c9168faSHans Rosenfeld static boolean_t nvme_check_regs_hdl(nvme_t *); 2683c9168faSHans Rosenfeld static boolean_t nvme_check_dma_hdl(nvme_dma_t *); 2693c9168faSHans Rosenfeld 2703c9168faSHans Rosenfeld static int nvme_fill_prp(nvme_cmd_t *, bd_xfer_t *); 2713c9168faSHans Rosenfeld 2723c9168faSHans Rosenfeld static void nvme_bd_xfer_done(void *); 2733c9168faSHans Rosenfeld static void nvme_bd_driveinfo(void *, bd_drive_t *); 2743c9168faSHans Rosenfeld static int nvme_bd_mediainfo(void *, bd_media_t *); 2753c9168faSHans Rosenfeld static int nvme_bd_cmd(nvme_namespace_t *, bd_xfer_t *, uint8_t); 2763c9168faSHans Rosenfeld static int nvme_bd_read(void *, bd_xfer_t *); 2773c9168faSHans Rosenfeld static int nvme_bd_write(void *, bd_xfer_t *); 2783c9168faSHans Rosenfeld static int nvme_bd_sync(void *, bd_xfer_t *); 2793c9168faSHans Rosenfeld static int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *); 2803c9168faSHans Rosenfeld 2818834f7acSYouzhong Yang static int nvme_prp_dma_constructor(void *, void *, int); 2828834f7acSYouzhong Yang static void nvme_prp_dma_destructor(void *, void *); 2838834f7acSYouzhong Yang 2843c9168faSHans Rosenfeld static void nvme_prepare_devid(nvme_t *, uint32_t); 2853c9168faSHans Rosenfeld 2863c9168faSHans Rosenfeld static void *nvme_state; 2873c9168faSHans Rosenfeld static kmem_cache_t *nvme_cmd_cache; 2883c9168faSHans Rosenfeld 2893c9168faSHans Rosenfeld /* 2903c9168faSHans Rosenfeld * DMA attributes for queue DMA memory 2913c9168faSHans Rosenfeld * 2923c9168faSHans Rosenfeld * Queue DMA memory must be page aligned. The maximum length of a queue is 2933c9168faSHans Rosenfeld * 65536 entries, and an entry can be 64 bytes long. 2943c9168faSHans Rosenfeld */ 2953c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_queue_dma_attr = { 2963c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 2973c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 2983c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 299910f0d12SYouzhong Yang .dma_attr_count_max = (UINT16_MAX + 1) * sizeof (nvme_sqe_t) - 1, 3003c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 3013c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 3023c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 3033c9168faSHans Rosenfeld .dma_attr_maxxfer = (UINT16_MAX + 1) * sizeof (nvme_sqe_t), 3043c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 3053c9168faSHans Rosenfeld .dma_attr_sgllen = 1, 3063c9168faSHans Rosenfeld .dma_attr_granular = 1, 3073c9168faSHans Rosenfeld .dma_attr_flags = 0, 3083c9168faSHans Rosenfeld }; 3093c9168faSHans Rosenfeld 3103c9168faSHans Rosenfeld /* 3113c9168faSHans Rosenfeld * DMA attributes for transfers using Physical Region Page (PRP) entries 3123c9168faSHans Rosenfeld * 3133c9168faSHans Rosenfeld * A PRP entry describes one page of DMA memory using the page size specified 3143c9168faSHans Rosenfeld * in the controller configuration's memory page size register (CC.MPS). It uses 3153c9168faSHans Rosenfeld * a 64bit base address aligned to this page size. There is no limitation on 3163c9168faSHans Rosenfeld * chaining PRPs together for arbitrarily large DMA transfers. 3173c9168faSHans Rosenfeld */ 3183c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_prp_dma_attr = { 3193c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 3203c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 3213c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 3223c9168faSHans Rosenfeld .dma_attr_count_max = 0xfff, 3233c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 3243c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 3253c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 3263c9168faSHans Rosenfeld .dma_attr_maxxfer = 0x1000, 3272f95345bSYouzhong Yang .dma_attr_seg = 0xfff, 3283c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 3293c9168faSHans Rosenfeld .dma_attr_granular = 1, 3303c9168faSHans Rosenfeld .dma_attr_flags = 0, 3313c9168faSHans Rosenfeld }; 3323c9168faSHans Rosenfeld 3333c9168faSHans Rosenfeld /* 3343c9168faSHans Rosenfeld * DMA attributes for transfers using scatter/gather lists 3353c9168faSHans Rosenfeld * 3363c9168faSHans Rosenfeld * A SGL entry describes a chunk of DMA memory using a 64bit base address and a 3373c9168faSHans Rosenfeld * 32bit length field. SGL Segment and SGL Last Segment entries require the 3383c9168faSHans Rosenfeld * length to be a multiple of 16 bytes. 3393c9168faSHans Rosenfeld */ 3403c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_sgl_dma_attr = { 3413c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 3423c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 3433c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 3443c9168faSHans Rosenfeld .dma_attr_count_max = 0xffffffffUL, 3453c9168faSHans Rosenfeld .dma_attr_align = 1, 3463c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 3473c9168faSHans Rosenfeld .dma_attr_minxfer = 0x10, 3483c9168faSHans Rosenfeld .dma_attr_maxxfer = 0xfffffffffULL, 3493c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 3503c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 3513c9168faSHans Rosenfeld .dma_attr_granular = 0x10, 3523c9168faSHans Rosenfeld .dma_attr_flags = 0 3533c9168faSHans Rosenfeld }; 3543c9168faSHans Rosenfeld 3553c9168faSHans Rosenfeld static ddi_device_acc_attr_t nvme_reg_acc_attr = { 3563c9168faSHans Rosenfeld .devacc_attr_version = DDI_DEVICE_ATTR_V0, 3573c9168faSHans Rosenfeld .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 3583c9168faSHans Rosenfeld .devacc_attr_dataorder = DDI_STRICTORDER_ACC 3593c9168faSHans Rosenfeld }; 3603c9168faSHans Rosenfeld 3613c9168faSHans Rosenfeld static struct dev_ops nvme_dev_ops = { 3623c9168faSHans Rosenfeld .devo_rev = DEVO_REV, 3633c9168faSHans Rosenfeld .devo_refcnt = 0, 3643c9168faSHans Rosenfeld .devo_getinfo = ddi_no_info, 3653c9168faSHans Rosenfeld .devo_identify = nulldev, 3663c9168faSHans Rosenfeld .devo_probe = nulldev, 3673c9168faSHans Rosenfeld .devo_attach = nvme_attach, 3683c9168faSHans Rosenfeld .devo_detach = nvme_detach, 3693c9168faSHans Rosenfeld .devo_reset = nodev, 3703c9168faSHans Rosenfeld .devo_cb_ops = NULL, 3713c9168faSHans Rosenfeld .devo_bus_ops = NULL, 3723c9168faSHans Rosenfeld .devo_power = NULL, 3733c9168faSHans Rosenfeld .devo_quiesce = nvme_quiesce, 3743c9168faSHans Rosenfeld }; 3753c9168faSHans Rosenfeld 3763c9168faSHans Rosenfeld static struct modldrv nvme_modldrv = { 3773c9168faSHans Rosenfeld .drv_modops = &mod_driverops, 37824979ca3SHans Rosenfeld .drv_linkinfo = "NVMe v1.1b", 3793c9168faSHans Rosenfeld .drv_dev_ops = &nvme_dev_ops 3803c9168faSHans Rosenfeld }; 3813c9168faSHans Rosenfeld 3823c9168faSHans Rosenfeld static struct modlinkage nvme_modlinkage = { 3833c9168faSHans Rosenfeld .ml_rev = MODREV_1, 3843c9168faSHans Rosenfeld .ml_linkage = { &nvme_modldrv, NULL } 3853c9168faSHans Rosenfeld }; 3863c9168faSHans Rosenfeld 3873c9168faSHans Rosenfeld static bd_ops_t nvme_bd_ops = { 3883c9168faSHans Rosenfeld .o_version = BD_OPS_VERSION_0, 3893c9168faSHans Rosenfeld .o_drive_info = nvme_bd_driveinfo, 3903c9168faSHans Rosenfeld .o_media_info = nvme_bd_mediainfo, 3913c9168faSHans Rosenfeld .o_devid_init = nvme_bd_devid, 3923c9168faSHans Rosenfeld .o_sync_cache = nvme_bd_sync, 3933c9168faSHans Rosenfeld .o_read = nvme_bd_read, 3943c9168faSHans Rosenfeld .o_write = nvme_bd_write, 3953c9168faSHans Rosenfeld }; 3963c9168faSHans Rosenfeld 3973c9168faSHans Rosenfeld int 3983c9168faSHans Rosenfeld _init(void) 3993c9168faSHans Rosenfeld { 4003c9168faSHans Rosenfeld int error; 4013c9168faSHans Rosenfeld 4023c9168faSHans Rosenfeld error = ddi_soft_state_init(&nvme_state, sizeof (nvme_t), 1); 4033c9168faSHans Rosenfeld if (error != DDI_SUCCESS) 4043c9168faSHans Rosenfeld return (error); 4053c9168faSHans Rosenfeld 4063c9168faSHans Rosenfeld nvme_cmd_cache = kmem_cache_create("nvme_cmd_cache", 4073c9168faSHans Rosenfeld sizeof (nvme_cmd_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 4083c9168faSHans Rosenfeld 4093c9168faSHans Rosenfeld bd_mod_init(&nvme_dev_ops); 4103c9168faSHans Rosenfeld 4113c9168faSHans Rosenfeld error = mod_install(&nvme_modlinkage); 4123c9168faSHans Rosenfeld if (error != DDI_SUCCESS) { 4133c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 4143c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 4153c9168faSHans Rosenfeld } 4163c9168faSHans Rosenfeld 4173c9168faSHans Rosenfeld return (error); 4183c9168faSHans Rosenfeld } 4193c9168faSHans Rosenfeld 4203c9168faSHans Rosenfeld int 4213c9168faSHans Rosenfeld _fini(void) 4223c9168faSHans Rosenfeld { 4233c9168faSHans Rosenfeld int error; 4243c9168faSHans Rosenfeld 4253c9168faSHans Rosenfeld error = mod_remove(&nvme_modlinkage); 4263c9168faSHans Rosenfeld if (error == DDI_SUCCESS) { 4273c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 4283c9168faSHans Rosenfeld kmem_cache_destroy(nvme_cmd_cache); 4293c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 4303c9168faSHans Rosenfeld } 4313c9168faSHans Rosenfeld 4323c9168faSHans Rosenfeld return (error); 4333c9168faSHans Rosenfeld } 4343c9168faSHans Rosenfeld 4353c9168faSHans Rosenfeld int 4363c9168faSHans Rosenfeld _info(struct modinfo *modinfop) 4373c9168faSHans Rosenfeld { 4383c9168faSHans Rosenfeld return (mod_info(&nvme_modlinkage, modinfop)); 4393c9168faSHans Rosenfeld } 4403c9168faSHans Rosenfeld 4413c9168faSHans Rosenfeld static inline void 4423c9168faSHans Rosenfeld nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val) 4433c9168faSHans Rosenfeld { 4443c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 4453c9168faSHans Rosenfeld 4463c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4473c9168faSHans Rosenfeld ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val); 4483c9168faSHans Rosenfeld } 4493c9168faSHans Rosenfeld 4503c9168faSHans Rosenfeld static inline void 4513c9168faSHans Rosenfeld nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val) 4523c9168faSHans Rosenfeld { 4533c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 4543c9168faSHans Rosenfeld 4553c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4563c9168faSHans Rosenfeld ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val); 4573c9168faSHans Rosenfeld } 4583c9168faSHans Rosenfeld 4593c9168faSHans Rosenfeld static inline uint64_t 4603c9168faSHans Rosenfeld nvme_get64(nvme_t *nvme, uintptr_t reg) 4613c9168faSHans Rosenfeld { 4623c9168faSHans Rosenfeld uint64_t val; 4633c9168faSHans Rosenfeld 4643c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 4653c9168faSHans Rosenfeld 4663c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4673c9168faSHans Rosenfeld val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg)); 4683c9168faSHans Rosenfeld 4693c9168faSHans Rosenfeld return (val); 4703c9168faSHans Rosenfeld } 4713c9168faSHans Rosenfeld 4723c9168faSHans Rosenfeld static inline uint32_t 4733c9168faSHans Rosenfeld nvme_get32(nvme_t *nvme, uintptr_t reg) 4743c9168faSHans Rosenfeld { 4753c9168faSHans Rosenfeld uint32_t val; 4763c9168faSHans Rosenfeld 4773c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 4783c9168faSHans Rosenfeld 4793c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 4803c9168faSHans Rosenfeld val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg)); 4813c9168faSHans Rosenfeld 4823c9168faSHans Rosenfeld return (val); 4833c9168faSHans Rosenfeld } 4843c9168faSHans Rosenfeld 4853c9168faSHans Rosenfeld static boolean_t 4863c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme_t *nvme) 4873c9168faSHans Rosenfeld { 4883c9168faSHans Rosenfeld ddi_fm_error_t error; 4893c9168faSHans Rosenfeld 4903c9168faSHans Rosenfeld ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION); 4913c9168faSHans Rosenfeld 4923c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 4933c9168faSHans Rosenfeld return (B_TRUE); 4943c9168faSHans Rosenfeld 4953c9168faSHans Rosenfeld return (B_FALSE); 4963c9168faSHans Rosenfeld } 4973c9168faSHans Rosenfeld 4983c9168faSHans Rosenfeld static boolean_t 4993c9168faSHans Rosenfeld nvme_check_dma_hdl(nvme_dma_t *dma) 5003c9168faSHans Rosenfeld { 5013c9168faSHans Rosenfeld ddi_fm_error_t error; 5023c9168faSHans Rosenfeld 5033c9168faSHans Rosenfeld if (dma == NULL) 5043c9168faSHans Rosenfeld return (B_FALSE); 5053c9168faSHans Rosenfeld 5063c9168faSHans Rosenfeld ddi_fm_dma_err_get(dma->nd_dmah, &error, DDI_FME_VERSION); 5073c9168faSHans Rosenfeld 5083c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 5093c9168faSHans Rosenfeld return (B_TRUE); 5103c9168faSHans Rosenfeld 5113c9168faSHans Rosenfeld return (B_FALSE); 5123c9168faSHans Rosenfeld } 5133c9168faSHans Rosenfeld 5143c9168faSHans Rosenfeld static void 5158834f7acSYouzhong Yang nvme_free_dma_common(nvme_dma_t *dma) 5163c9168faSHans Rosenfeld { 5173c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 5183c9168faSHans Rosenfeld (void) ddi_dma_unbind_handle(dma->nd_dmah); 5193c9168faSHans Rosenfeld if (dma->nd_acch != NULL) 5203c9168faSHans Rosenfeld ddi_dma_mem_free(&dma->nd_acch); 5213c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 5223c9168faSHans Rosenfeld ddi_dma_free_handle(&dma->nd_dmah); 5233c9168faSHans Rosenfeld } 5243c9168faSHans Rosenfeld 5258834f7acSYouzhong Yang static void 5268834f7acSYouzhong Yang nvme_free_dma(nvme_dma_t *dma) 5273c9168faSHans Rosenfeld { 5288834f7acSYouzhong Yang nvme_free_dma_common(dma); 5298834f7acSYouzhong Yang kmem_free(dma, sizeof (*dma)); 5308834f7acSYouzhong Yang } 5318834f7acSYouzhong Yang 532b6bc2fd4SDan McDonald /* ARGSUSED */ 5338834f7acSYouzhong Yang static void 5348834f7acSYouzhong Yang nvme_prp_dma_destructor(void *buf, void *private) 5358834f7acSYouzhong Yang { 5368834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 5373c9168faSHans Rosenfeld 5388834f7acSYouzhong Yang nvme_free_dma_common(dma); 5398834f7acSYouzhong Yang } 5408834f7acSYouzhong Yang 5418834f7acSYouzhong Yang static int 5428834f7acSYouzhong Yang nvme_alloc_dma_common(nvme_t *nvme, nvme_dma_t *dma, 5438834f7acSYouzhong Yang size_t len, uint_t flags, ddi_dma_attr_t *dma_attr) 5448834f7acSYouzhong Yang { 5453c9168faSHans Rosenfeld if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL, 5463c9168faSHans Rosenfeld &dma->nd_dmah) != DDI_SUCCESS) { 5473c9168faSHans Rosenfeld /* 5483c9168faSHans Rosenfeld * Due to DDI_DMA_SLEEP this can't be DDI_DMA_NORESOURCES, and 5493c9168faSHans Rosenfeld * the only other possible error is DDI_DMA_BADATTR which 5503c9168faSHans Rosenfeld * indicates a driver bug which should cause a panic. 5513c9168faSHans Rosenfeld */ 5523c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, 5533c9168faSHans Rosenfeld "!failed to get DMA handle, check DMA attributes"); 5543c9168faSHans Rosenfeld return (DDI_FAILURE); 5553c9168faSHans Rosenfeld } 5563c9168faSHans Rosenfeld 5573c9168faSHans Rosenfeld /* 5583c9168faSHans Rosenfeld * ddi_dma_mem_alloc() can only fail when DDI_DMA_NOSLEEP is specified 5593c9168faSHans Rosenfeld * or the flags are conflicting, which isn't the case here. 5603c9168faSHans Rosenfeld */ 5613c9168faSHans Rosenfeld (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr, 5623c9168faSHans Rosenfeld DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &dma->nd_memp, 5633c9168faSHans Rosenfeld &dma->nd_len, &dma->nd_acch); 5643c9168faSHans Rosenfeld 5653c9168faSHans Rosenfeld if (ddi_dma_addr_bind_handle(dma->nd_dmah, NULL, dma->nd_memp, 5663c9168faSHans Rosenfeld dma->nd_len, flags | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, 5673c9168faSHans Rosenfeld &dma->nd_cookie, &dma->nd_ncookie) != DDI_DMA_MAPPED) { 5683c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 5693c9168faSHans Rosenfeld "!failed to bind DMA memory"); 5703c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_dma_bind_err); 5718834f7acSYouzhong Yang nvme_free_dma_common(dma); 5728834f7acSYouzhong Yang return (DDI_FAILURE); 5738834f7acSYouzhong Yang } 5748834f7acSYouzhong Yang 5758834f7acSYouzhong Yang return (DDI_SUCCESS); 5768834f7acSYouzhong Yang } 5778834f7acSYouzhong Yang 5788834f7acSYouzhong Yang static int 5798834f7acSYouzhong Yang nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags, 5808834f7acSYouzhong Yang ddi_dma_attr_t *dma_attr, nvme_dma_t **ret) 5818834f7acSYouzhong Yang { 5828834f7acSYouzhong Yang nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP); 5838834f7acSYouzhong Yang 5848834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, len, flags, dma_attr) != 5858834f7acSYouzhong Yang DDI_SUCCESS) { 5863c9168faSHans Rosenfeld *ret = NULL; 5878834f7acSYouzhong Yang kmem_free(dma, sizeof (nvme_dma_t)); 5883c9168faSHans Rosenfeld return (DDI_FAILURE); 5893c9168faSHans Rosenfeld } 5903c9168faSHans Rosenfeld 5913c9168faSHans Rosenfeld bzero(dma->nd_memp, dma->nd_len); 5923c9168faSHans Rosenfeld 5933c9168faSHans Rosenfeld *ret = dma; 5943c9168faSHans Rosenfeld return (DDI_SUCCESS); 5953c9168faSHans Rosenfeld } 5963c9168faSHans Rosenfeld 597b6bc2fd4SDan McDonald /* ARGSUSED */ 5988834f7acSYouzhong Yang static int 5998834f7acSYouzhong Yang nvme_prp_dma_constructor(void *buf, void *private, int flags) 6008834f7acSYouzhong Yang { 6018834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 6028834f7acSYouzhong Yang nvme_t *nvme = (nvme_t *)private; 6038834f7acSYouzhong Yang 6048834f7acSYouzhong Yang dma->nd_dmah = NULL; 6058834f7acSYouzhong Yang dma->nd_acch = NULL; 6068834f7acSYouzhong Yang 6078834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, nvme->n_pagesize, 6088834f7acSYouzhong Yang DDI_DMA_READ, &nvme->n_prp_dma_attr) != DDI_SUCCESS) { 6098834f7acSYouzhong Yang return (-1); 6108834f7acSYouzhong Yang } 6118834f7acSYouzhong Yang 6128834f7acSYouzhong Yang ASSERT(dma->nd_ncookie == 1); 6138834f7acSYouzhong Yang 6148834f7acSYouzhong Yang dma->nd_cached = B_TRUE; 6158834f7acSYouzhong Yang 6168834f7acSYouzhong Yang return (0); 6178834f7acSYouzhong Yang } 6188834f7acSYouzhong Yang 6193c9168faSHans Rosenfeld static int 6203c9168faSHans Rosenfeld nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len, 6213c9168faSHans Rosenfeld uint_t flags, nvme_dma_t **dma) 6223c9168faSHans Rosenfeld { 6233c9168faSHans Rosenfeld uint32_t len = nentry * qe_len; 6243c9168faSHans Rosenfeld ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr; 6253c9168faSHans Rosenfeld 6263c9168faSHans Rosenfeld len = roundup(len, nvme->n_pagesize); 6273c9168faSHans Rosenfeld 6283c9168faSHans Rosenfeld q_dma_attr.dma_attr_minxfer = len; 6293c9168faSHans Rosenfeld 6303c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma) 6313c9168faSHans Rosenfeld != DDI_SUCCESS) { 6323c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 6333c9168faSHans Rosenfeld "!failed to get DMA memory for queue"); 6343c9168faSHans Rosenfeld goto fail; 6353c9168faSHans Rosenfeld } 6363c9168faSHans Rosenfeld 6373c9168faSHans Rosenfeld if ((*dma)->nd_ncookie != 1) { 6383c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 6393c9168faSHans Rosenfeld "!got too many cookies for queue DMA"); 6403c9168faSHans Rosenfeld goto fail; 6413c9168faSHans Rosenfeld } 6423c9168faSHans Rosenfeld 6433c9168faSHans Rosenfeld return (DDI_SUCCESS); 6443c9168faSHans Rosenfeld 6453c9168faSHans Rosenfeld fail: 6463c9168faSHans Rosenfeld if (*dma) { 6473c9168faSHans Rosenfeld nvme_free_dma(*dma); 6483c9168faSHans Rosenfeld *dma = NULL; 6493c9168faSHans Rosenfeld } 6503c9168faSHans Rosenfeld 6513c9168faSHans Rosenfeld return (DDI_FAILURE); 6523c9168faSHans Rosenfeld } 6533c9168faSHans Rosenfeld 6543c9168faSHans Rosenfeld static void 6553c9168faSHans Rosenfeld nvme_free_qpair(nvme_qpair_t *qp) 6563c9168faSHans Rosenfeld { 6573c9168faSHans Rosenfeld int i; 6583c9168faSHans Rosenfeld 6593c9168faSHans Rosenfeld mutex_destroy(&qp->nq_mutex); 6603c9168faSHans Rosenfeld 6613c9168faSHans Rosenfeld if (qp->nq_sqdma != NULL) 6623c9168faSHans Rosenfeld nvme_free_dma(qp->nq_sqdma); 6633c9168faSHans Rosenfeld if (qp->nq_cqdma != NULL) 6643c9168faSHans Rosenfeld nvme_free_dma(qp->nq_cqdma); 6653c9168faSHans Rosenfeld 6663c9168faSHans Rosenfeld if (qp->nq_active_cmds > 0) 6673c9168faSHans Rosenfeld for (i = 0; i != qp->nq_nentry; i++) 6683c9168faSHans Rosenfeld if (qp->nq_cmd[i] != NULL) 6693c9168faSHans Rosenfeld nvme_free_cmd(qp->nq_cmd[i]); 6703c9168faSHans Rosenfeld 6713c9168faSHans Rosenfeld if (qp->nq_cmd != NULL) 6723c9168faSHans Rosenfeld kmem_free(qp->nq_cmd, sizeof (nvme_cmd_t *) * qp->nq_nentry); 6733c9168faSHans Rosenfeld 6743c9168faSHans Rosenfeld kmem_free(qp, sizeof (nvme_qpair_t)); 6753c9168faSHans Rosenfeld } 6763c9168faSHans Rosenfeld 6773c9168faSHans Rosenfeld static int 6783c9168faSHans Rosenfeld nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp, 6793c9168faSHans Rosenfeld int idx) 6803c9168faSHans Rosenfeld { 6813c9168faSHans Rosenfeld nvme_qpair_t *qp = kmem_zalloc(sizeof (*qp), KM_SLEEP); 6823c9168faSHans Rosenfeld 6833c9168faSHans Rosenfeld mutex_init(&qp->nq_mutex, NULL, MUTEX_DRIVER, 6843c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 6853c9168faSHans Rosenfeld 6863c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t), 6873c9168faSHans Rosenfeld DDI_DMA_WRITE, &qp->nq_sqdma) != DDI_SUCCESS) 6883c9168faSHans Rosenfeld goto fail; 6893c9168faSHans Rosenfeld 6903c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t), 6913c9168faSHans Rosenfeld DDI_DMA_READ, &qp->nq_cqdma) != DDI_SUCCESS) 6923c9168faSHans Rosenfeld goto fail; 6933c9168faSHans Rosenfeld 6943c9168faSHans Rosenfeld qp->nq_sq = (nvme_sqe_t *)qp->nq_sqdma->nd_memp; 6953c9168faSHans Rosenfeld qp->nq_cq = (nvme_cqe_t *)qp->nq_cqdma->nd_memp; 6963c9168faSHans Rosenfeld qp->nq_nentry = nentry; 6973c9168faSHans Rosenfeld 6983c9168faSHans Rosenfeld qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx); 6993c9168faSHans Rosenfeld qp->nq_cqhdbl = NVME_REG_CQHDBL(nvme, idx); 7003c9168faSHans Rosenfeld 7013c9168faSHans Rosenfeld qp->nq_cmd = kmem_zalloc(sizeof (nvme_cmd_t *) * nentry, KM_SLEEP); 7023c9168faSHans Rosenfeld qp->nq_next_cmd = 0; 7033c9168faSHans Rosenfeld 7043c9168faSHans Rosenfeld *nqp = qp; 7053c9168faSHans Rosenfeld return (DDI_SUCCESS); 7063c9168faSHans Rosenfeld 7073c9168faSHans Rosenfeld fail: 7083c9168faSHans Rosenfeld nvme_free_qpair(qp); 7093c9168faSHans Rosenfeld *nqp = NULL; 7103c9168faSHans Rosenfeld 7113c9168faSHans Rosenfeld return (DDI_FAILURE); 7123c9168faSHans Rosenfeld } 7133c9168faSHans Rosenfeld 7143c9168faSHans Rosenfeld static nvme_cmd_t * 7153c9168faSHans Rosenfeld nvme_alloc_cmd(nvme_t *nvme, int kmflag) 7163c9168faSHans Rosenfeld { 7173c9168faSHans Rosenfeld nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag); 7183c9168faSHans Rosenfeld 7193c9168faSHans Rosenfeld if (cmd == NULL) 7203c9168faSHans Rosenfeld return (cmd); 7213c9168faSHans Rosenfeld 7223c9168faSHans Rosenfeld bzero(cmd, sizeof (nvme_cmd_t)); 7233c9168faSHans Rosenfeld 7243c9168faSHans Rosenfeld cmd->nc_nvme = nvme; 7253c9168faSHans Rosenfeld 7263c9168faSHans Rosenfeld mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER, 7273c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 7283c9168faSHans Rosenfeld cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL); 7293c9168faSHans Rosenfeld 7303c9168faSHans Rosenfeld return (cmd); 7313c9168faSHans Rosenfeld } 7323c9168faSHans Rosenfeld 7333c9168faSHans Rosenfeld static void 7343c9168faSHans Rosenfeld nvme_free_cmd(nvme_cmd_t *cmd) 7353c9168faSHans Rosenfeld { 7363c9168faSHans Rosenfeld if (cmd->nc_dma) { 7378834f7acSYouzhong Yang if (cmd->nc_dma->nd_cached) 7388834f7acSYouzhong Yang kmem_cache_free(cmd->nc_nvme->n_prp_cache, 7398834f7acSYouzhong Yang cmd->nc_dma); 7408834f7acSYouzhong Yang else 7418834f7acSYouzhong Yang nvme_free_dma(cmd->nc_dma); 7423c9168faSHans Rosenfeld cmd->nc_dma = NULL; 7433c9168faSHans Rosenfeld } 7443c9168faSHans Rosenfeld 7453c9168faSHans Rosenfeld cv_destroy(&cmd->nc_cv); 7463c9168faSHans Rosenfeld mutex_destroy(&cmd->nc_mutex); 7473c9168faSHans Rosenfeld 7483c9168faSHans Rosenfeld kmem_cache_free(nvme_cmd_cache, cmd); 7493c9168faSHans Rosenfeld } 7503c9168faSHans Rosenfeld 7513c9168faSHans Rosenfeld static int 7523c9168faSHans Rosenfeld nvme_submit_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 7533c9168faSHans Rosenfeld { 7543c9168faSHans Rosenfeld nvme_reg_sqtdbl_t tail = { 0 }; 7553c9168faSHans Rosenfeld 7563c9168faSHans Rosenfeld mutex_enter(&qp->nq_mutex); 7573c9168faSHans Rosenfeld 7583c9168faSHans Rosenfeld if (qp->nq_active_cmds == qp->nq_nentry) { 7593c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 7603c9168faSHans Rosenfeld return (DDI_FAILURE); 7613c9168faSHans Rosenfeld } 7623c9168faSHans Rosenfeld 7633c9168faSHans Rosenfeld cmd->nc_completed = B_FALSE; 7643c9168faSHans Rosenfeld 7653c9168faSHans Rosenfeld /* 7663c9168faSHans Rosenfeld * Try to insert the cmd into the active cmd array at the nq_next_cmd 7673c9168faSHans Rosenfeld * slot. If the slot is already occupied advance to the next slot and 7683c9168faSHans Rosenfeld * try again. This can happen for long running commands like async event 7693c9168faSHans Rosenfeld * requests. 7703c9168faSHans Rosenfeld */ 7713c9168faSHans Rosenfeld while (qp->nq_cmd[qp->nq_next_cmd] != NULL) 7723c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 7733c9168faSHans Rosenfeld qp->nq_cmd[qp->nq_next_cmd] = cmd; 7743c9168faSHans Rosenfeld 7753c9168faSHans Rosenfeld qp->nq_active_cmds++; 7763c9168faSHans Rosenfeld 7773c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cid = qp->nq_next_cmd; 7783c9168faSHans Rosenfeld bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t)); 7793c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_sqdma->nd_dmah, 7803c9168faSHans Rosenfeld sizeof (nvme_sqe_t) * qp->nq_sqtail, 7813c9168faSHans Rosenfeld sizeof (nvme_sqe_t), DDI_DMA_SYNC_FORDEV); 7823c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 7833c9168faSHans Rosenfeld 7843c9168faSHans Rosenfeld tail.b.sqtdbl_sqt = qp->nq_sqtail = (qp->nq_sqtail + 1) % qp->nq_nentry; 7853c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r); 7863c9168faSHans Rosenfeld 7873c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 7883c9168faSHans Rosenfeld return (DDI_SUCCESS); 7893c9168faSHans Rosenfeld } 7903c9168faSHans Rosenfeld 7913c9168faSHans Rosenfeld static nvme_cmd_t * 7923c9168faSHans Rosenfeld nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp) 7933c9168faSHans Rosenfeld { 7943c9168faSHans Rosenfeld nvme_reg_cqhdbl_t head = { 0 }; 7953c9168faSHans Rosenfeld 7963c9168faSHans Rosenfeld nvme_cqe_t *cqe; 7973c9168faSHans Rosenfeld nvme_cmd_t *cmd; 7983c9168faSHans Rosenfeld 7993c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_cqdma->nd_dmah, 0, 8003c9168faSHans Rosenfeld sizeof (nvme_cqe_t) * qp->nq_nentry, DDI_DMA_SYNC_FORKERNEL); 8013c9168faSHans Rosenfeld 8023c9168faSHans Rosenfeld cqe = &qp->nq_cq[qp->nq_cqhead]; 8033c9168faSHans Rosenfeld 8043c9168faSHans Rosenfeld /* Check phase tag of CQE. Hardware inverts it for new entries. */ 8053c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_p == qp->nq_phase) 8063c9168faSHans Rosenfeld return (NULL); 8073c9168faSHans Rosenfeld 8083c9168faSHans Rosenfeld ASSERT(nvme->n_ioq[cqe->cqe_sqid] == qp); 8093c9168faSHans Rosenfeld ASSERT(cqe->cqe_cid < qp->nq_nentry); 8103c9168faSHans Rosenfeld 8113c9168faSHans Rosenfeld mutex_enter(&qp->nq_mutex); 8123c9168faSHans Rosenfeld cmd = qp->nq_cmd[cqe->cqe_cid]; 8133c9168faSHans Rosenfeld qp->nq_cmd[cqe->cqe_cid] = NULL; 8143c9168faSHans Rosenfeld qp->nq_active_cmds--; 8153c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 8163c9168faSHans Rosenfeld 8173c9168faSHans Rosenfeld ASSERT(cmd != NULL); 8183c9168faSHans Rosenfeld ASSERT(cmd->nc_nvme == nvme); 8193c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid == cqe->cqe_sqid); 8203c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_cid == cqe->cqe_cid); 8213c9168faSHans Rosenfeld bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t)); 8223c9168faSHans Rosenfeld 8233c9168faSHans Rosenfeld qp->nq_sqhead = cqe->cqe_sqhd; 8243c9168faSHans Rosenfeld 8253c9168faSHans Rosenfeld head.b.cqhdbl_cqh = qp->nq_cqhead = (qp->nq_cqhead + 1) % qp->nq_nentry; 8263c9168faSHans Rosenfeld 8273c9168faSHans Rosenfeld /* Toggle phase on wrap-around. */ 8283c9168faSHans Rosenfeld if (qp->nq_cqhead == 0) 8293c9168faSHans Rosenfeld qp->nq_phase = qp->nq_phase ? 0 : 1; 8303c9168faSHans Rosenfeld 8313c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_cqhdbl, head.r); 8323c9168faSHans Rosenfeld 8333c9168faSHans Rosenfeld return (cmd); 8343c9168faSHans Rosenfeld } 8353c9168faSHans Rosenfeld 8363c9168faSHans Rosenfeld static int 8373c9168faSHans Rosenfeld nvme_check_unknown_cmd_status(nvme_cmd_t *cmd) 8383c9168faSHans Rosenfeld { 8393c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8403c9168faSHans Rosenfeld 8413c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 8423c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 8433c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 8443c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 8453c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 8463c9168faSHans Rosenfeld 847bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 848bef9e21aSHans Rosenfeld 8493c9168faSHans Rosenfeld if (cmd->nc_nvme->n_strict_version) { 8503c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 8513c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 8523c9168faSHans Rosenfeld } 8533c9168faSHans Rosenfeld 8543c9168faSHans Rosenfeld return (EIO); 8553c9168faSHans Rosenfeld } 8563c9168faSHans Rosenfeld 8573c9168faSHans Rosenfeld static int 8583c9168faSHans Rosenfeld nvme_check_vendor_cmd_status(nvme_cmd_t *cmd) 8593c9168faSHans Rosenfeld { 8603c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8613c9168faSHans Rosenfeld 8623c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 8633c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 8643c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 8653c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 8663c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 86734c938c7SPete Shephard if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) { 8683c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 8693c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 8703c9168faSHans Rosenfeld } 8713c9168faSHans Rosenfeld 8723c9168faSHans Rosenfeld return (EIO); 8733c9168faSHans Rosenfeld } 8743c9168faSHans Rosenfeld 8753c9168faSHans Rosenfeld static int 8763c9168faSHans Rosenfeld nvme_check_integrity_cmd_status(nvme_cmd_t *cmd) 8773c9168faSHans Rosenfeld { 8783c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 8793c9168faSHans Rosenfeld 8803c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 8813c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_WRITE: 8823c9168faSHans Rosenfeld /* write fail */ 8833c9168faSHans Rosenfeld /* TODO: post ereport */ 884bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 8853c9168faSHans Rosenfeld return (EIO); 8863c9168faSHans Rosenfeld 8873c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_READ: 8883c9168faSHans Rosenfeld /* read fail */ 8893c9168faSHans Rosenfeld /* TODO: post ereport */ 890bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 8913c9168faSHans Rosenfeld return (EIO); 8923c9168faSHans Rosenfeld 8933c9168faSHans Rosenfeld default: 8943c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 8953c9168faSHans Rosenfeld } 8963c9168faSHans Rosenfeld } 8973c9168faSHans Rosenfeld 8983c9168faSHans Rosenfeld static int 8993c9168faSHans Rosenfeld nvme_check_generic_cmd_status(nvme_cmd_t *cmd) 9003c9168faSHans Rosenfeld { 9013c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 9023c9168faSHans Rosenfeld 9033c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 9043c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_SUCCESS: 9053c9168faSHans Rosenfeld return (0); 9063c9168faSHans Rosenfeld 9073c9168faSHans Rosenfeld /* 9083c9168faSHans Rosenfeld * Errors indicating a bug in the driver should cause a panic. 9093c9168faSHans Rosenfeld */ 9103c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_OPC: 9113c9168faSHans Rosenfeld /* Invalid Command Opcode */ 9123c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9133c9168faSHans Rosenfeld "invalid opcode in cmd %p", (void *)cmd); 9143c9168faSHans Rosenfeld return (0); 9153c9168faSHans Rosenfeld 9163c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_FLD: 9173c9168faSHans Rosenfeld /* Invalid Field in Command */ 9183c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9193c9168faSHans Rosenfeld "invalid field in cmd %p", (void *)cmd); 9203c9168faSHans Rosenfeld return (0); 9213c9168faSHans Rosenfeld 9223c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ID_CNFL: 9233c9168faSHans Rosenfeld /* Command ID Conflict */ 9243c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9253c9168faSHans Rosenfeld "cmd ID conflict in cmd %p", (void *)cmd); 9263c9168faSHans Rosenfeld return (0); 9273c9168faSHans Rosenfeld 9283c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_NS: 9293c9168faSHans Rosenfeld /* Invalid Namespace or Format */ 9303c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9313c9168faSHans Rosenfeld "invalid NS/format in cmd %p", (void *)cmd); 9323c9168faSHans Rosenfeld return (0); 9333c9168faSHans Rosenfeld 9343c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_LBA_RANGE: 9353c9168faSHans Rosenfeld /* LBA Out Of Range */ 9363c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 9373c9168faSHans Rosenfeld "LBA out of range in cmd %p", (void *)cmd); 9383c9168faSHans Rosenfeld return (0); 9393c9168faSHans Rosenfeld 9403c9168faSHans Rosenfeld /* 9413c9168faSHans Rosenfeld * Non-fatal errors, handle gracefully. 9423c9168faSHans Rosenfeld */ 9433c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_DATA_XFR_ERR: 9443c9168faSHans Rosenfeld /* Data Transfer Error (DMA) */ 9453c9168faSHans Rosenfeld /* TODO: post ereport */ 9463c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err); 947bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 9483c9168faSHans Rosenfeld return (EIO); 9493c9168faSHans Rosenfeld 9503c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INTERNAL_ERR: 9513c9168faSHans Rosenfeld /* 9523c9168faSHans Rosenfeld * Internal Error. The spec (v1.0, section 4.5.1.2) says 9533c9168faSHans Rosenfeld * detailed error information is returned as async event, 9543c9168faSHans Rosenfeld * so we pretty much ignore the error here and handle it 9553c9168faSHans Rosenfeld * in the async event handler. 9563c9168faSHans Rosenfeld */ 9573c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_internal_err); 958bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 9593c9168faSHans Rosenfeld return (EIO); 9603c9168faSHans Rosenfeld 9613c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_REQUEST: 9623c9168faSHans Rosenfeld /* 9633c9168faSHans Rosenfeld * Command Abort Requested. This normally happens only when a 9643c9168faSHans Rosenfeld * command times out. 9653c9168faSHans Rosenfeld */ 9663c9168faSHans Rosenfeld /* TODO: post ereport or change blkdev to handle this? */ 9673c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err); 9683c9168faSHans Rosenfeld return (ECANCELED); 9693c9168faSHans Rosenfeld 9703c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_PWRLOSS: 9713c9168faSHans Rosenfeld /* Command Aborted due to Power Loss Notification */ 9723c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 9733c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 9743c9168faSHans Rosenfeld return (EIO); 9753c9168faSHans Rosenfeld 9763c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_SQ_DEL: 9773c9168faSHans Rosenfeld /* Command Aborted due to SQ Deletion */ 9783c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del); 9793c9168faSHans Rosenfeld return (EIO); 9803c9168faSHans Rosenfeld 9813c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_CAP_EXC: 9823c9168faSHans Rosenfeld /* Capacity Exceeded */ 9833c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc); 984bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 9853c9168faSHans Rosenfeld return (EIO); 9863c9168faSHans Rosenfeld 9873c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_NS_NOTRDY: 9883c9168faSHans Rosenfeld /* Namespace Not Ready */ 9893c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy); 990bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 9913c9168faSHans Rosenfeld return (EIO); 9923c9168faSHans Rosenfeld 9933c9168faSHans Rosenfeld default: 9943c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 9953c9168faSHans Rosenfeld } 9963c9168faSHans Rosenfeld } 9973c9168faSHans Rosenfeld 9983c9168faSHans Rosenfeld static int 9993c9168faSHans Rosenfeld nvme_check_specific_cmd_status(nvme_cmd_t *cmd) 10003c9168faSHans Rosenfeld { 10013c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 10023c9168faSHans Rosenfeld 10033c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 10043c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_CQ: 10053c9168faSHans Rosenfeld /* Completion Queue Invalid */ 10063c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE); 10073c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err); 10083c9168faSHans Rosenfeld return (EINVAL); 10093c9168faSHans Rosenfeld 10103c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_QID: 10113c9168faSHans Rosenfeld /* Invalid Queue Identifier */ 10123c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 10133c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE || 10143c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE || 10153c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 10163c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err); 10173c9168faSHans Rosenfeld return (EINVAL); 10183c9168faSHans Rosenfeld 10193c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_MAX_QSZ_EXC: 10203c9168faSHans Rosenfeld /* Max Queue Size Exceeded */ 10213c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 10223c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 10233c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc); 10243c9168faSHans Rosenfeld return (EINVAL); 10253c9168faSHans Rosenfeld 10263c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ABRT_CMD_EXC: 10273c9168faSHans Rosenfeld /* Abort Command Limit Exceeded */ 10283c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT); 10293c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 10303c9168faSHans Rosenfeld "abort command limit exceeded in cmd %p", (void *)cmd); 10313c9168faSHans Rosenfeld return (0); 10323c9168faSHans Rosenfeld 10333c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC: 10343c9168faSHans Rosenfeld /* Async Event Request Limit Exceeded */ 10353c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT); 10363c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 10373c9168faSHans Rosenfeld "async event request limit exceeded in cmd %p", 10383c9168faSHans Rosenfeld (void *)cmd); 10393c9168faSHans Rosenfeld return (0); 10403c9168faSHans Rosenfeld 10413c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_INT_VECT: 10423c9168faSHans Rosenfeld /* Invalid Interrupt Vector */ 10433c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 10443c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect); 10453c9168faSHans Rosenfeld return (EINVAL); 10463c9168faSHans Rosenfeld 10473c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_LOG_PAGE: 10483c9168faSHans Rosenfeld /* Invalid Log Page */ 10493c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE); 10503c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_log_page); 1051bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10523c9168faSHans Rosenfeld return (EINVAL); 10533c9168faSHans Rosenfeld 10543c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_FORMAT: 10553c9168faSHans Rosenfeld /* Invalid Format */ 10563c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT); 10573c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_format); 1058bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10593c9168faSHans Rosenfeld return (EINVAL); 10603c9168faSHans Rosenfeld 10613c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_Q_DEL: 10623c9168faSHans Rosenfeld /* Invalid Queue Deletion */ 10633c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 10643c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_q_del); 10653c9168faSHans Rosenfeld return (EINVAL); 10663c9168faSHans Rosenfeld 10673c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_CNFL_ATTR: 10683c9168faSHans Rosenfeld /* Conflicting Attributes */ 10693c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT || 10703c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 10713c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 10723c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr); 1073bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10743c9168faSHans Rosenfeld return (EINVAL); 10753c9168faSHans Rosenfeld 10763c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_INV_PROT: 10773c9168faSHans Rosenfeld /* Invalid Protection Information */ 10783c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE || 10793c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 10803c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 10813c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_prot); 1082bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10833c9168faSHans Rosenfeld return (EINVAL); 10843c9168faSHans Rosenfeld 10853c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_READONLY: 10863c9168faSHans Rosenfeld /* Write to Read Only Range */ 10873c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 10883c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_readonly); 1089bef9e21aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 10903c9168faSHans Rosenfeld return (EROFS); 10913c9168faSHans Rosenfeld 10923c9168faSHans Rosenfeld default: 10933c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 10943c9168faSHans Rosenfeld } 10953c9168faSHans Rosenfeld } 10963c9168faSHans Rosenfeld 10973c9168faSHans Rosenfeld static inline int 10983c9168faSHans Rosenfeld nvme_check_cmd_status(nvme_cmd_t *cmd) 10993c9168faSHans Rosenfeld { 11003c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 11013c9168faSHans Rosenfeld 11023c9168faSHans Rosenfeld /* take a shortcut if everything is alright */ 11033c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 11043c9168faSHans Rosenfeld cqe->cqe_sf.sf_sc == NVME_CQE_SC_GEN_SUCCESS) 11053c9168faSHans Rosenfeld return (0); 11063c9168faSHans Rosenfeld 11073c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC) 11083c9168faSHans Rosenfeld return (nvme_check_generic_cmd_status(cmd)); 11093c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_SPECIFIC) 11103c9168faSHans Rosenfeld return (nvme_check_specific_cmd_status(cmd)); 11113c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_INTEGRITY) 11123c9168faSHans Rosenfeld return (nvme_check_integrity_cmd_status(cmd)); 11133c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_VENDOR) 11143c9168faSHans Rosenfeld return (nvme_check_vendor_cmd_status(cmd)); 11153c9168faSHans Rosenfeld 11163c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 11173c9168faSHans Rosenfeld } 11183c9168faSHans Rosenfeld 11193c9168faSHans Rosenfeld /* 11203c9168faSHans Rosenfeld * nvme_abort_cmd_cb -- replaces nc_callback of aborted commands 11213c9168faSHans Rosenfeld * 11223c9168faSHans Rosenfeld * This functions takes care of cleaning up aborted commands. The command 11233c9168faSHans Rosenfeld * status is checked to catch any fatal errors. 11243c9168faSHans Rosenfeld */ 11253c9168faSHans Rosenfeld static void 11263c9168faSHans Rosenfeld nvme_abort_cmd_cb(void *arg) 11273c9168faSHans Rosenfeld { 11283c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 11293c9168faSHans Rosenfeld 11303c9168faSHans Rosenfeld /* 11313c9168faSHans Rosenfeld * Grab the command mutex. Once we have it we hold the last reference 11323c9168faSHans Rosenfeld * to the command and can safely free it. 11333c9168faSHans Rosenfeld */ 11343c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 11353c9168faSHans Rosenfeld (void) nvme_check_cmd_status(cmd); 11363c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 11373c9168faSHans Rosenfeld 11383c9168faSHans Rosenfeld nvme_free_cmd(cmd); 11393c9168faSHans Rosenfeld } 11403c9168faSHans Rosenfeld 11413c9168faSHans Rosenfeld static void 11423c9168faSHans Rosenfeld nvme_abort_cmd(nvme_cmd_t *abort_cmd) 11433c9168faSHans Rosenfeld { 11443c9168faSHans Rosenfeld nvme_t *nvme = abort_cmd->nc_nvme; 11453c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 11463c9168faSHans Rosenfeld nvme_abort_cmd_t ac = { 0 }; 11473c9168faSHans Rosenfeld 11483c9168faSHans Rosenfeld sema_p(&nvme->n_abort_sema); 11493c9168faSHans Rosenfeld 11503c9168faSHans Rosenfeld ac.b.ac_cid = abort_cmd->nc_sqe.sqe_cid; 11513c9168faSHans Rosenfeld ac.b.ac_sqid = abort_cmd->nc_sqid; 11523c9168faSHans Rosenfeld 11533c9168faSHans Rosenfeld /* 11543c9168faSHans Rosenfeld * Drop the mutex of the aborted command. From this point on 11553c9168faSHans Rosenfeld * we must assume that the abort callback has freed the command. 11563c9168faSHans Rosenfeld */ 11573c9168faSHans Rosenfeld mutex_exit(&abort_cmd->nc_mutex); 11583c9168faSHans Rosenfeld 11593c9168faSHans Rosenfeld cmd->nc_sqid = 0; 11603c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT; 11613c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 11623c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = ac.r; 11633c9168faSHans Rosenfeld 11643c9168faSHans Rosenfeld /* 11653c9168faSHans Rosenfeld * Send the ABORT to the hardware. The ABORT command will return _after_ 11663c9168faSHans Rosenfeld * the aborted command has completed (aborted or otherwise). 11673c9168faSHans Rosenfeld */ 1168e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 11693c9168faSHans Rosenfeld sema_v(&nvme->n_abort_sema); 11703c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 11713c9168faSHans Rosenfeld "!nvme_admin_cmd failed for ABORT"); 11723c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_abort_failed); 11733c9168faSHans Rosenfeld return; 11743c9168faSHans Rosenfeld } 11753c9168faSHans Rosenfeld sema_v(&nvme->n_abort_sema); 11763c9168faSHans Rosenfeld 11773c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 11783c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 11793c9168faSHans Rosenfeld "!ABORT failed with sct = %x, sc = %x", 11803c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 11813c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_abort_failed); 11823c9168faSHans Rosenfeld } else { 11833c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_aborted); 11843c9168faSHans Rosenfeld } 11853c9168faSHans Rosenfeld 11863c9168faSHans Rosenfeld nvme_free_cmd(cmd); 11873c9168faSHans Rosenfeld } 11883c9168faSHans Rosenfeld 11893c9168faSHans Rosenfeld /* 11903c9168faSHans Rosenfeld * nvme_wait_cmd -- wait for command completion or timeout 11913c9168faSHans Rosenfeld * 11923c9168faSHans Rosenfeld * Returns B_TRUE if the command completed normally. 11933c9168faSHans Rosenfeld * 11943c9168faSHans Rosenfeld * Returns B_FALSE if the command timed out and an abort was attempted. The 11953c9168faSHans Rosenfeld * command mutex will be dropped and the command must be considered freed. The 11963c9168faSHans Rosenfeld * freeing of the command is normally done by the abort command callback. 11973c9168faSHans Rosenfeld * 11983c9168faSHans Rosenfeld * In case of a serious error or a timeout of the abort command the hardware 11993c9168faSHans Rosenfeld * will be declared dead and FMA will be notified. 12003c9168faSHans Rosenfeld */ 12013c9168faSHans Rosenfeld static boolean_t 1202e8ba2a38SHans Rosenfeld nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec) 12033c9168faSHans Rosenfeld { 1204e8ba2a38SHans Rosenfeld clock_t timeout = ddi_get_lbolt() + drv_usectohz(sec * MICROSEC); 12053c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 12063c9168faSHans Rosenfeld nvme_reg_csts_t csts; 12073c9168faSHans Rosenfeld 12083c9168faSHans Rosenfeld ASSERT(mutex_owned(&cmd->nc_mutex)); 12093c9168faSHans Rosenfeld 12103c9168faSHans Rosenfeld while (!cmd->nc_completed) { 12113c9168faSHans Rosenfeld if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1) 12123c9168faSHans Rosenfeld break; 12133c9168faSHans Rosenfeld } 12143c9168faSHans Rosenfeld 12153c9168faSHans Rosenfeld if (cmd->nc_completed) 12163c9168faSHans Rosenfeld return (B_TRUE); 12173c9168faSHans Rosenfeld 12183c9168faSHans Rosenfeld /* 12193c9168faSHans Rosenfeld * The command timed out. Change the callback to the cleanup function. 12203c9168faSHans Rosenfeld */ 12213c9168faSHans Rosenfeld cmd->nc_callback = nvme_abort_cmd_cb; 12223c9168faSHans Rosenfeld 12233c9168faSHans Rosenfeld /* 12243c9168faSHans Rosenfeld * Check controller for fatal status, any errors associated with the 12253c9168faSHans Rosenfeld * register or DMA handle, or for a double timeout (abort command timed 12263c9168faSHans Rosenfeld * out). If necessary log a warning and call FMA. 12273c9168faSHans Rosenfeld */ 12283c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 12293c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!command timeout, " 12303c9168faSHans Rosenfeld "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_opc, csts.b.csts_cfs); 12313c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_timeout); 12323c9168faSHans Rosenfeld 12333c9168faSHans Rosenfeld if (csts.b.csts_cfs || 12343c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme) || 12353c9168faSHans Rosenfeld nvme_check_dma_hdl(cmd->nc_dma) || 12363c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) { 12373c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 12383c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 12393c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 12403c9168faSHans Rosenfeld } else { 12413c9168faSHans Rosenfeld /* 12423c9168faSHans Rosenfeld * Try to abort the command. The command mutex is released by 12433c9168faSHans Rosenfeld * nvme_abort_cmd(). 12443c9168faSHans Rosenfeld * If the abort succeeds it will have freed the aborted command. 12453c9168faSHans Rosenfeld * If the abort fails for other reasons we must assume that the 12463c9168faSHans Rosenfeld * command may complete at any time, and the callback will free 12473c9168faSHans Rosenfeld * it for us. 12483c9168faSHans Rosenfeld */ 12493c9168faSHans Rosenfeld nvme_abort_cmd(cmd); 12503c9168faSHans Rosenfeld } 12513c9168faSHans Rosenfeld 12523c9168faSHans Rosenfeld return (B_FALSE); 12533c9168faSHans Rosenfeld } 12543c9168faSHans Rosenfeld 12553c9168faSHans Rosenfeld static void 12563c9168faSHans Rosenfeld nvme_wakeup_cmd(void *arg) 12573c9168faSHans Rosenfeld { 12583c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 12593c9168faSHans Rosenfeld 12603c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 12613c9168faSHans Rosenfeld /* 12623c9168faSHans Rosenfeld * There is a slight chance that this command completed shortly after 12633c9168faSHans Rosenfeld * the timeout was hit in nvme_wait_cmd() but before the callback was 12643c9168faSHans Rosenfeld * changed. Catch that case here and clean up accordingly. 12653c9168faSHans Rosenfeld */ 12663c9168faSHans Rosenfeld if (cmd->nc_callback == nvme_abort_cmd_cb) { 12673c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 12683c9168faSHans Rosenfeld nvme_abort_cmd_cb(cmd); 12693c9168faSHans Rosenfeld return; 12703c9168faSHans Rosenfeld } 12713c9168faSHans Rosenfeld 12723c9168faSHans Rosenfeld cmd->nc_completed = B_TRUE; 12733c9168faSHans Rosenfeld cv_signal(&cmd->nc_cv); 12743c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 12753c9168faSHans Rosenfeld } 12763c9168faSHans Rosenfeld 12773c9168faSHans Rosenfeld static void 12783c9168faSHans Rosenfeld nvme_async_event_task(void *arg) 12793c9168faSHans Rosenfeld { 12803c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 12813c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 12823c9168faSHans Rosenfeld nvme_error_log_entry_t *error_log = NULL; 12833c9168faSHans Rosenfeld nvme_health_log_t *health_log = NULL; 12843c9168faSHans Rosenfeld nvme_async_event_t event; 12853c9168faSHans Rosenfeld int ret; 12863c9168faSHans Rosenfeld 12873c9168faSHans Rosenfeld /* 12883c9168faSHans Rosenfeld * Check for errors associated with the async request itself. The only 12893c9168faSHans Rosenfeld * command-specific error is "async event limit exceeded", which 12903c9168faSHans Rosenfeld * indicates a programming error in the driver and causes a panic in 12913c9168faSHans Rosenfeld * nvme_check_cmd_status(). 12923c9168faSHans Rosenfeld * 12933c9168faSHans Rosenfeld * Other possible errors are various scenarios where the async request 12943c9168faSHans Rosenfeld * was aborted, or internal errors in the device. Internal errors are 12953c9168faSHans Rosenfeld * reported to FMA, the command aborts need no special handling here. 12963c9168faSHans Rosenfeld */ 12973c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 12983c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 12993c9168faSHans Rosenfeld "!async event request returned failure, sct = %x, " 13003c9168faSHans Rosenfeld "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct, 13013c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr, 13023c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_m); 13033c9168faSHans Rosenfeld 13043c9168faSHans Rosenfeld if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 13053c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) { 13063c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 13073c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, 13083c9168faSHans Rosenfeld DDI_SERVICE_LOST); 13093c9168faSHans Rosenfeld } 13103c9168faSHans Rosenfeld nvme_free_cmd(cmd); 13113c9168faSHans Rosenfeld return; 13123c9168faSHans Rosenfeld } 13133c9168faSHans Rosenfeld 13143c9168faSHans Rosenfeld 13153c9168faSHans Rosenfeld event.r = cmd->nc_cqe.cqe_dw0; 13163c9168faSHans Rosenfeld 13173c9168faSHans Rosenfeld /* Clear CQE and re-submit the async request. */ 13183c9168faSHans Rosenfeld bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t)); 13193c9168faSHans Rosenfeld ret = nvme_submit_cmd(nvme->n_adminq, cmd); 13203c9168faSHans Rosenfeld 13213c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 13223c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13233c9168faSHans Rosenfeld "!failed to resubmit async event request"); 13243c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_async_resubmit_failed); 13253c9168faSHans Rosenfeld nvme_free_cmd(cmd); 13263c9168faSHans Rosenfeld } 13273c9168faSHans Rosenfeld 13283c9168faSHans Rosenfeld switch (event.b.ae_type) { 13293c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_ERROR: 13303c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_ERROR) { 13313c9168faSHans Rosenfeld error_log = (nvme_error_log_entry_t *) 13323c9168faSHans Rosenfeld nvme_get_logpage(nvme, event.b.ae_logpage); 13333c9168faSHans Rosenfeld } else { 13343c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 13353c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 13363c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 13373c9168faSHans Rosenfeld } 13383c9168faSHans Rosenfeld 13393c9168faSHans Rosenfeld switch (event.b.ae_info) { 13403c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_SQ: 13413c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 13423c9168faSHans Rosenfeld "invalid submission queue"); 13433c9168faSHans Rosenfeld return; 13443c9168faSHans Rosenfeld 13453c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_DBL: 13463c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 13473c9168faSHans Rosenfeld "invalid doorbell write value"); 13483c9168faSHans Rosenfeld return; 13493c9168faSHans Rosenfeld 13503c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_DIAGFAIL: 13513c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure"); 13523c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 13533c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 13543c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_diagfail_event); 13553c9168faSHans Rosenfeld break; 13563c9168faSHans Rosenfeld 13573c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_PERSISTENT: 13583c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!persistent internal " 13593c9168faSHans Rosenfeld "device error"); 13603c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 13613c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 13623c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_persistent_event); 13633c9168faSHans Rosenfeld break; 13643c9168faSHans Rosenfeld 13653c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_TRANSIENT: 13663c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!transient internal " 13673c9168faSHans Rosenfeld "device error"); 13683c9168faSHans Rosenfeld /* TODO: send ereport */ 13693c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_transient_event); 13703c9168faSHans Rosenfeld break; 13713c9168faSHans Rosenfeld 13723c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_FW_LOAD: 13733c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13743c9168faSHans Rosenfeld "!firmware image load error"); 13753c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_fw_load_event); 13763c9168faSHans Rosenfeld break; 13773c9168faSHans Rosenfeld } 13783c9168faSHans Rosenfeld break; 13793c9168faSHans Rosenfeld 13803c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_HEALTH: 13813c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_HEALTH) { 13823c9168faSHans Rosenfeld health_log = (nvme_health_log_t *) 13833c9168faSHans Rosenfeld nvme_get_logpage(nvme, event.b.ae_logpage, -1); 13843c9168faSHans Rosenfeld } else { 13853c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 13863c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 13873c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 13883c9168faSHans Rosenfeld } 13893c9168faSHans Rosenfeld 13903c9168faSHans Rosenfeld switch (event.b.ae_info) { 13913c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_RELIABILITY: 13923c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13933c9168faSHans Rosenfeld "!device reliability compromised"); 13943c9168faSHans Rosenfeld /* TODO: send ereport */ 13953c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_reliability_event); 13963c9168faSHans Rosenfeld break; 13973c9168faSHans Rosenfeld 13983c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_TEMPERATURE: 13993c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 14003c9168faSHans Rosenfeld "!temperature above threshold"); 14013c9168faSHans Rosenfeld /* TODO: send ereport */ 14023c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_temperature_event); 14033c9168faSHans Rosenfeld break; 14043c9168faSHans Rosenfeld 14053c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_SPARE: 14063c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 14073c9168faSHans Rosenfeld "!spare space below threshold"); 14083c9168faSHans Rosenfeld /* TODO: send ereport */ 14093c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_spare_event); 14103c9168faSHans Rosenfeld break; 14113c9168faSHans Rosenfeld } 14123c9168faSHans Rosenfeld break; 14133c9168faSHans Rosenfeld 14143c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_VENDOR: 14153c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event " 14163c9168faSHans Rosenfeld "received, info = %x, logpage = %x", event.b.ae_info, 14173c9168faSHans Rosenfeld event.b.ae_logpage); 14183c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_vendor_event); 14193c9168faSHans Rosenfeld break; 14203c9168faSHans Rosenfeld 14213c9168faSHans Rosenfeld default: 14223c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, " 14233c9168faSHans Rosenfeld "type = %x, info = %x, logpage = %x", event.b.ae_type, 14243c9168faSHans Rosenfeld event.b.ae_info, event.b.ae_logpage); 14253c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_event); 14263c9168faSHans Rosenfeld break; 14273c9168faSHans Rosenfeld } 14283c9168faSHans Rosenfeld 14293c9168faSHans Rosenfeld if (error_log) 14303c9168faSHans Rosenfeld kmem_free(error_log, sizeof (nvme_error_log_entry_t) * 14313c9168faSHans Rosenfeld nvme->n_error_log_len); 14323c9168faSHans Rosenfeld 14333c9168faSHans Rosenfeld if (health_log) 14343c9168faSHans Rosenfeld kmem_free(health_log, sizeof (nvme_health_log_t)); 14353c9168faSHans Rosenfeld } 14363c9168faSHans Rosenfeld 14373c9168faSHans Rosenfeld static int 1438e8ba2a38SHans Rosenfeld nvme_admin_cmd(nvme_cmd_t *cmd, int sec) 14393c9168faSHans Rosenfeld { 14403c9168faSHans Rosenfeld int ret; 14413c9168faSHans Rosenfeld 14423c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 14433c9168faSHans Rosenfeld ret = nvme_submit_cmd(cmd->nc_nvme->n_adminq, cmd); 14443c9168faSHans Rosenfeld 14453c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 14463c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 14473c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 14483c9168faSHans Rosenfeld "!nvme_submit_cmd failed"); 14493c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_admin_queue_full); 14503c9168faSHans Rosenfeld nvme_free_cmd(cmd); 14513c9168faSHans Rosenfeld return (DDI_FAILURE); 14523c9168faSHans Rosenfeld } 14533c9168faSHans Rosenfeld 1454e8ba2a38SHans Rosenfeld if (nvme_wait_cmd(cmd, sec) == B_FALSE) { 14553c9168faSHans Rosenfeld /* 14563c9168faSHans Rosenfeld * The command timed out. An abort command was posted that 14573c9168faSHans Rosenfeld * will take care of the cleanup. 14583c9168faSHans Rosenfeld */ 14593c9168faSHans Rosenfeld return (DDI_FAILURE); 14603c9168faSHans Rosenfeld } 14613c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 14623c9168faSHans Rosenfeld 14633c9168faSHans Rosenfeld return (DDI_SUCCESS); 14643c9168faSHans Rosenfeld } 14653c9168faSHans Rosenfeld 14663c9168faSHans Rosenfeld static int 14673c9168faSHans Rosenfeld nvme_async_event(nvme_t *nvme) 14683c9168faSHans Rosenfeld { 14693c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 14703c9168faSHans Rosenfeld int ret; 14713c9168faSHans Rosenfeld 14723c9168faSHans Rosenfeld cmd->nc_sqid = 0; 14733c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT; 14743c9168faSHans Rosenfeld cmd->nc_callback = nvme_async_event_task; 14753c9168faSHans Rosenfeld 14763c9168faSHans Rosenfeld ret = nvme_submit_cmd(nvme->n_adminq, cmd); 14773c9168faSHans Rosenfeld 14783c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 14793c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 14803c9168faSHans Rosenfeld "!nvme_submit_cmd failed for ASYNCHRONOUS EVENT"); 14813c9168faSHans Rosenfeld nvme_free_cmd(cmd); 14823c9168faSHans Rosenfeld return (DDI_FAILURE); 14833c9168faSHans Rosenfeld } 14843c9168faSHans Rosenfeld 14853c9168faSHans Rosenfeld return (DDI_SUCCESS); 14863c9168faSHans Rosenfeld } 14873c9168faSHans Rosenfeld 14883c9168faSHans Rosenfeld static void * 14893c9168faSHans Rosenfeld nvme_get_logpage(nvme_t *nvme, uint8_t logpage, ...) 14903c9168faSHans Rosenfeld { 14913c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 14923c9168faSHans Rosenfeld void *buf = NULL; 149334c938c7SPete Shephard nvme_getlogpage_t getlogpage = { 0 }; 14943c9168faSHans Rosenfeld size_t bufsize; 14953c9168faSHans Rosenfeld va_list ap; 14963c9168faSHans Rosenfeld 14973c9168faSHans Rosenfeld va_start(ap, logpage); 14983c9168faSHans Rosenfeld 14993c9168faSHans Rosenfeld cmd->nc_sqid = 0; 15003c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 15013c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE; 15023c9168faSHans Rosenfeld 15033c9168faSHans Rosenfeld getlogpage.b.lp_lid = logpage; 15043c9168faSHans Rosenfeld 15053c9168faSHans Rosenfeld switch (logpage) { 15063c9168faSHans Rosenfeld case NVME_LOGPAGE_ERROR: 15073c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 15083c9168faSHans Rosenfeld bufsize = nvme->n_error_log_len * 15093c9168faSHans Rosenfeld sizeof (nvme_error_log_entry_t); 15103c9168faSHans Rosenfeld break; 15113c9168faSHans Rosenfeld 15123c9168faSHans Rosenfeld case NVME_LOGPAGE_HEALTH: 15133c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t); 15143c9168faSHans Rosenfeld bufsize = sizeof (nvme_health_log_t); 15153c9168faSHans Rosenfeld break; 15163c9168faSHans Rosenfeld 15173c9168faSHans Rosenfeld case NVME_LOGPAGE_FWSLOT: 15183c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 15193c9168faSHans Rosenfeld bufsize = sizeof (nvme_fwslot_log_t); 15203c9168faSHans Rosenfeld break; 15213c9168faSHans Rosenfeld 15223c9168faSHans Rosenfeld default: 15233c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown log page requested: %d", 15243c9168faSHans Rosenfeld logpage); 15253c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_logpage); 15263c9168faSHans Rosenfeld goto fail; 15273c9168faSHans Rosenfeld } 15283c9168faSHans Rosenfeld 15293c9168faSHans Rosenfeld va_end(ap); 15303c9168faSHans Rosenfeld 15319898bbdbSHans Rosenfeld getlogpage.b.lp_numd = bufsize / sizeof (uint32_t) - 1; 15323c9168faSHans Rosenfeld 15333c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = getlogpage.r; 15343c9168faSHans Rosenfeld 15353c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, getlogpage.b.lp_numd * sizeof (uint32_t), 15363c9168faSHans Rosenfeld DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 15373c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15383c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for GET LOG PAGE"); 15393c9168faSHans Rosenfeld goto fail; 15403c9168faSHans Rosenfeld } 15413c9168faSHans Rosenfeld 15423c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 15433c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15443c9168faSHans Rosenfeld "!too many DMA cookies for GET LOG PAGE"); 15453c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 15463c9168faSHans Rosenfeld goto fail; 15473c9168faSHans Rosenfeld } 15483c9168faSHans Rosenfeld 15493c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 15503c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 15513c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 15523c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 15533c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 15543c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 15553c9168faSHans Rosenfeld } 15563c9168faSHans Rosenfeld 1557e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 15583c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15593c9168faSHans Rosenfeld "!nvme_admin_cmd failed for GET LOG PAGE"); 15603c9168faSHans Rosenfeld return (NULL); 15613c9168faSHans Rosenfeld } 15623c9168faSHans Rosenfeld 15633c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 15643c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15653c9168faSHans Rosenfeld "!GET LOG PAGE failed with sct = %x, sc = %x", 15663c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 15673c9168faSHans Rosenfeld goto fail; 15683c9168faSHans Rosenfeld } 15693c9168faSHans Rosenfeld 15703c9168faSHans Rosenfeld buf = kmem_alloc(bufsize, KM_SLEEP); 15713c9168faSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, buf, bufsize); 15723c9168faSHans Rosenfeld 15733c9168faSHans Rosenfeld fail: 15743c9168faSHans Rosenfeld nvme_free_cmd(cmd); 15753c9168faSHans Rosenfeld 15763c9168faSHans Rosenfeld return (buf); 15773c9168faSHans Rosenfeld } 15783c9168faSHans Rosenfeld 15793c9168faSHans Rosenfeld static void * 15803c9168faSHans Rosenfeld nvme_identify(nvme_t *nvme, uint32_t nsid) 15813c9168faSHans Rosenfeld { 15823c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 15833c9168faSHans Rosenfeld void *buf = NULL; 15843c9168faSHans Rosenfeld 15853c9168faSHans Rosenfeld cmd->nc_sqid = 0; 15863c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 15873c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY; 15883c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 15893c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL; 15903c9168faSHans Rosenfeld 15913c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ, 15923c9168faSHans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 15933c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15943c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for IDENTIFY"); 15953c9168faSHans Rosenfeld goto fail; 15963c9168faSHans Rosenfeld } 15973c9168faSHans Rosenfeld 15983c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 15993c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16003c9168faSHans Rosenfeld "!too many DMA cookies for IDENTIFY"); 16013c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 16023c9168faSHans Rosenfeld goto fail; 16033c9168faSHans Rosenfeld } 16043c9168faSHans Rosenfeld 16053c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 16063c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 16073c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 16083c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 16093c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 16103c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 16113c9168faSHans Rosenfeld } 16123c9168faSHans Rosenfeld 1613e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 16143c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16153c9168faSHans Rosenfeld "!nvme_admin_cmd failed for IDENTIFY"); 16163c9168faSHans Rosenfeld return (NULL); 16173c9168faSHans Rosenfeld } 16183c9168faSHans Rosenfeld 16193c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 16203c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16213c9168faSHans Rosenfeld "!IDENTIFY failed with sct = %x, sc = %x", 16223c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 16233c9168faSHans Rosenfeld goto fail; 16243c9168faSHans Rosenfeld } 16253c9168faSHans Rosenfeld 16263c9168faSHans Rosenfeld buf = kmem_alloc(NVME_IDENTIFY_BUFSIZE, KM_SLEEP); 16273c9168faSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, buf, NVME_IDENTIFY_BUFSIZE); 16283c9168faSHans Rosenfeld 16293c9168faSHans Rosenfeld fail: 16303c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16313c9168faSHans Rosenfeld 16323c9168faSHans Rosenfeld return (buf); 16333c9168faSHans Rosenfeld } 16343c9168faSHans Rosenfeld 1635d148d46eSHans Rosenfeld static boolean_t 1636d148d46eSHans Rosenfeld nvme_set_features(nvme_t *nvme, uint32_t nsid, uint8_t feature, uint32_t val, 1637d148d46eSHans Rosenfeld uint32_t *res) 16383c9168faSHans Rosenfeld { 1639d148d46eSHans Rosenfeld _NOTE(ARGUNUSED(nsid)); 16403c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1641d148d46eSHans Rosenfeld boolean_t ret = B_FALSE; 16423c9168faSHans Rosenfeld 1643d148d46eSHans Rosenfeld ASSERT(res != NULL); 16443c9168faSHans Rosenfeld 16453c9168faSHans Rosenfeld cmd->nc_sqid = 0; 16463c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 16473c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES; 1648d148d46eSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = feature; 1649d148d46eSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = val; 1650d148d46eSHans Rosenfeld 1651d148d46eSHans Rosenfeld switch (feature) { 1652d148d46eSHans Rosenfeld case NVME_FEAT_WRITE_CACHE: 1653d148d46eSHans Rosenfeld if (!nvme->n_write_cache_present) 1654d148d46eSHans Rosenfeld goto fail; 1655d148d46eSHans Rosenfeld break; 1656d148d46eSHans Rosenfeld 1657d148d46eSHans Rosenfeld case NVME_FEAT_NQUEUES: 1658d148d46eSHans Rosenfeld break; 1659d148d46eSHans Rosenfeld 1660d148d46eSHans Rosenfeld default: 1661d148d46eSHans Rosenfeld goto fail; 1662d148d46eSHans Rosenfeld } 16633c9168faSHans Rosenfeld 1664e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 16653c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 1666d148d46eSHans Rosenfeld "!nvme_admin_cmd failed for SET FEATURES"); 1667d148d46eSHans Rosenfeld return (ret); 16683c9168faSHans Rosenfeld } 16693c9168faSHans Rosenfeld 16703c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 16713c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 1672d148d46eSHans Rosenfeld "!SET FEATURES %d failed with sct = %x, sc = %x", 1673d148d46eSHans Rosenfeld feature, cmd->nc_cqe.cqe_sf.sf_sct, 1674d148d46eSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc); 1675d148d46eSHans Rosenfeld goto fail; 16763c9168faSHans Rosenfeld } 16773c9168faSHans Rosenfeld 1678d148d46eSHans Rosenfeld *res = cmd->nc_cqe.cqe_dw0; 1679d148d46eSHans Rosenfeld ret = B_TRUE; 1680d148d46eSHans Rosenfeld 1681d148d46eSHans Rosenfeld fail: 16823c9168faSHans Rosenfeld nvme_free_cmd(cmd); 1683d148d46eSHans Rosenfeld return (ret); 1684d148d46eSHans Rosenfeld } 1685d148d46eSHans Rosenfeld 1686d148d46eSHans Rosenfeld static boolean_t 1687d148d46eSHans Rosenfeld nvme_write_cache_set(nvme_t *nvme, boolean_t enable) 1688d148d46eSHans Rosenfeld { 1689d148d46eSHans Rosenfeld nvme_write_cache_t nwc = { 0 }; 1690d148d46eSHans Rosenfeld 1691d148d46eSHans Rosenfeld if (enable) 1692d148d46eSHans Rosenfeld nwc.b.wc_wce = 1; 1693d148d46eSHans Rosenfeld 1694d148d46eSHans Rosenfeld if (!nvme_set_features(nvme, 0, NVME_FEAT_WRITE_CACHE, nwc.r, &nwc.r)) 1695d148d46eSHans Rosenfeld return (B_FALSE); 1696d148d46eSHans Rosenfeld 1697d148d46eSHans Rosenfeld return (B_TRUE); 1698d148d46eSHans Rosenfeld } 1699d148d46eSHans Rosenfeld 1700d148d46eSHans Rosenfeld static int 1701d148d46eSHans Rosenfeld nvme_set_nqueues(nvme_t *nvme, uint16_t nqueues) 1702d148d46eSHans Rosenfeld { 1703d148d46eSHans Rosenfeld nvme_nqueue_t nq = { 0 }; 1704d148d46eSHans Rosenfeld 1705d148d46eSHans Rosenfeld nq.b.nq_nsq = nq.b.nq_ncq = nqueues - 1; 1706d148d46eSHans Rosenfeld 1707d148d46eSHans Rosenfeld if (!nvme_set_features(nvme, 0, NVME_FEAT_NQUEUES, nq.r, &nq.r)) { 1708d148d46eSHans Rosenfeld return (0); 1709d148d46eSHans Rosenfeld } 17103c9168faSHans Rosenfeld 17113c9168faSHans Rosenfeld /* 17123c9168faSHans Rosenfeld * Always use the same number of submission and completion queues, and 17133c9168faSHans Rosenfeld * never use more than the requested number of queues. 17143c9168faSHans Rosenfeld */ 171534c938c7SPete Shephard return (MIN(nqueues, MIN(nq.b.nq_nsq, nq.b.nq_ncq) + 1)); 17163c9168faSHans Rosenfeld } 17173c9168faSHans Rosenfeld 17183c9168faSHans Rosenfeld static int 17193c9168faSHans Rosenfeld nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx) 17203c9168faSHans Rosenfeld { 17213c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 17223c9168faSHans Rosenfeld nvme_create_queue_dw10_t dw10 = { 0 }; 17233c9168faSHans Rosenfeld nvme_create_cq_dw11_t c_dw11 = { 0 }; 17243c9168faSHans Rosenfeld nvme_create_sq_dw11_t s_dw11 = { 0 }; 17253c9168faSHans Rosenfeld 17263c9168faSHans Rosenfeld dw10.b.q_qid = idx; 17273c9168faSHans Rosenfeld dw10.b.q_qsize = qp->nq_nentry - 1; 17283c9168faSHans Rosenfeld 17293c9168faSHans Rosenfeld c_dw11.b.cq_pc = 1; 17303c9168faSHans Rosenfeld c_dw11.b.cq_ien = 1; 17313c9168faSHans Rosenfeld c_dw11.b.cq_iv = idx % nvme->n_intr_cnt; 17323c9168faSHans Rosenfeld 17333c9168faSHans Rosenfeld cmd->nc_sqid = 0; 17343c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 17353c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE; 17363c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 17373c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = c_dw11.r; 17383c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_cqdma->nd_cookie.dmac_laddress; 17393c9168faSHans Rosenfeld 1740e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 17413c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17423c9168faSHans Rosenfeld "!nvme_admin_cmd failed for CREATE CQUEUE"); 17433c9168faSHans Rosenfeld return (DDI_FAILURE); 17443c9168faSHans Rosenfeld } 17453c9168faSHans Rosenfeld 17463c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 17473c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17483c9168faSHans Rosenfeld "!CREATE CQUEUE failed with sct = %x, sc = %x", 17493c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 17503c9168faSHans Rosenfeld nvme_free_cmd(cmd); 17513c9168faSHans Rosenfeld return (DDI_FAILURE); 17523c9168faSHans Rosenfeld } 17533c9168faSHans Rosenfeld 17543c9168faSHans Rosenfeld nvme_free_cmd(cmd); 17553c9168faSHans Rosenfeld 17563c9168faSHans Rosenfeld s_dw11.b.sq_pc = 1; 17573c9168faSHans Rosenfeld s_dw11.b.sq_cqid = idx; 17583c9168faSHans Rosenfeld 17593c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 17603c9168faSHans Rosenfeld cmd->nc_sqid = 0; 17613c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 17623c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE; 17633c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 17643c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = s_dw11.r; 17653c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress; 17663c9168faSHans Rosenfeld 1767e8ba2a38SHans Rosenfeld if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 17683c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17693c9168faSHans Rosenfeld "!nvme_admin_cmd failed for CREATE SQUEUE"); 17703c9168faSHans Rosenfeld return (DDI_FAILURE); 17713c9168faSHans Rosenfeld } 17723c9168faSHans Rosenfeld 17733c9168faSHans Rosenfeld if (nvme_check_cmd_status(cmd)) { 17743c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17753c9168faSHans Rosenfeld "!CREATE SQUEUE failed with sct = %x, sc = %x", 17763c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 17773c9168faSHans Rosenfeld nvme_free_cmd(cmd); 17783c9168faSHans Rosenfeld return (DDI_FAILURE); 17793c9168faSHans Rosenfeld } 17803c9168faSHans Rosenfeld 17813c9168faSHans Rosenfeld nvme_free_cmd(cmd); 17823c9168faSHans Rosenfeld 17833c9168faSHans Rosenfeld return (DDI_SUCCESS); 17843c9168faSHans Rosenfeld } 17853c9168faSHans Rosenfeld 17863c9168faSHans Rosenfeld static boolean_t 17873c9168faSHans Rosenfeld nvme_reset(nvme_t *nvme, boolean_t quiesce) 17883c9168faSHans Rosenfeld { 17893c9168faSHans Rosenfeld nvme_reg_csts_t csts; 17903c9168faSHans Rosenfeld int i; 17913c9168faSHans Rosenfeld 17923c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 17933c9168faSHans Rosenfeld 17943c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 17953c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) { 17963c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 17973c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 17983c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 17993c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) 18003c9168faSHans Rosenfeld break; 18013c9168faSHans Rosenfeld 18023c9168faSHans Rosenfeld if (quiesce) 18033c9168faSHans Rosenfeld drv_usecwait(50000); 18043c9168faSHans Rosenfeld else 18053c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 18063c9168faSHans Rosenfeld } 18073c9168faSHans Rosenfeld } 18083c9168faSHans Rosenfeld 18093c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, 0); 18103c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ASQ, 0); 18113c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ACQ, 0); 18123c9168faSHans Rosenfeld 18133c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 18143c9168faSHans Rosenfeld return (csts.b.csts_rdy == 0 ? B_TRUE : B_FALSE); 18153c9168faSHans Rosenfeld } 18163c9168faSHans Rosenfeld 18173c9168faSHans Rosenfeld static void 18183c9168faSHans Rosenfeld nvme_shutdown(nvme_t *nvme, int mode, boolean_t quiesce) 18193c9168faSHans Rosenfeld { 18203c9168faSHans Rosenfeld nvme_reg_cc_t cc; 18213c9168faSHans Rosenfeld nvme_reg_csts_t csts; 18223c9168faSHans Rosenfeld int i; 18233c9168faSHans Rosenfeld 18243c9168faSHans Rosenfeld ASSERT(mode == NVME_CC_SHN_NORMAL || mode == NVME_CC_SHN_ABRUPT); 18253c9168faSHans Rosenfeld 18263c9168faSHans Rosenfeld cc.r = nvme_get32(nvme, NVME_REG_CC); 18273c9168faSHans Rosenfeld cc.b.cc_shn = mode & 0x3; 18283c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 18293c9168faSHans Rosenfeld 18303c9168faSHans Rosenfeld for (i = 0; i != 10; i++) { 18313c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 18323c9168faSHans Rosenfeld if (csts.b.csts_shst == NVME_CSTS_SHN_COMPLETE) 18333c9168faSHans Rosenfeld break; 18343c9168faSHans Rosenfeld 18353c9168faSHans Rosenfeld if (quiesce) 18363c9168faSHans Rosenfeld drv_usecwait(100000); 18373c9168faSHans Rosenfeld else 18383c9168faSHans Rosenfeld delay(drv_usectohz(100000)); 18393c9168faSHans Rosenfeld } 18403c9168faSHans Rosenfeld } 18413c9168faSHans Rosenfeld 18423c9168faSHans Rosenfeld 18433c9168faSHans Rosenfeld static void 18443c9168faSHans Rosenfeld nvme_prepare_devid(nvme_t *nvme, uint32_t nsid) 18453c9168faSHans Rosenfeld { 184624979ca3SHans Rosenfeld /* 184724979ca3SHans Rosenfeld * Section 7.7 of the spec describes how to get a unique ID for 184824979ca3SHans Rosenfeld * the controller: the vendor ID, the model name and the serial 184924979ca3SHans Rosenfeld * number shall be unique when combined. 185024979ca3SHans Rosenfeld * 185124979ca3SHans Rosenfeld * If a namespace has no EUI64 we use the above and add the hex 185224979ca3SHans Rosenfeld * namespace ID to get a unique ID for the namespace. 185324979ca3SHans Rosenfeld */ 18543c9168faSHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 18553c9168faSHans Rosenfeld char serial[sizeof (nvme->n_idctl->id_serial) + 1]; 18563c9168faSHans Rosenfeld 18573c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 18583c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_serial, serial, 18593c9168faSHans Rosenfeld sizeof (nvme->n_idctl->id_serial)); 18603c9168faSHans Rosenfeld 18613c9168faSHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 18623c9168faSHans Rosenfeld serial[sizeof (nvme->n_idctl->id_serial)] = '\0'; 18633c9168faSHans Rosenfeld 186424979ca3SHans Rosenfeld nvme->n_ns[nsid - 1].ns_devid = kmem_asprintf("%4X-%s-%s-%X", 18653c9168faSHans Rosenfeld nvme->n_idctl->id_vid, model, serial, nsid); 18663c9168faSHans Rosenfeld } 18673c9168faSHans Rosenfeld 18683c9168faSHans Rosenfeld static int 18693c9168faSHans Rosenfeld nvme_init(nvme_t *nvme) 18703c9168faSHans Rosenfeld { 18713c9168faSHans Rosenfeld nvme_reg_cc_t cc = { 0 }; 18723c9168faSHans Rosenfeld nvme_reg_aqa_t aqa = { 0 }; 18733c9168faSHans Rosenfeld nvme_reg_asq_t asq = { 0 }; 18743c9168faSHans Rosenfeld nvme_reg_acq_t acq = { 0 }; 18753c9168faSHans Rosenfeld nvme_reg_cap_t cap; 18763c9168faSHans Rosenfeld nvme_reg_vs_t vs; 18773c9168faSHans Rosenfeld nvme_reg_csts_t csts; 18783c9168faSHans Rosenfeld int i = 0; 18793c9168faSHans Rosenfeld int nqueues; 1880510a6847SHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 1881510a6847SHans Rosenfeld char *vendor, *product; 18823c9168faSHans Rosenfeld 18833c9168faSHans Rosenfeld /* Check controller version */ 18843c9168faSHans Rosenfeld vs.r = nvme_get32(nvme, NVME_REG_VS); 188524979ca3SHans Rosenfeld nvme->n_version.v_major = vs.b.vs_mjr; 188624979ca3SHans Rosenfeld nvme->n_version.v_minor = vs.b.vs_mnr; 18873c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d", 188824979ca3SHans Rosenfeld nvme->n_version.v_major, nvme->n_version.v_minor); 18893c9168faSHans Rosenfeld 189024979ca3SHans Rosenfeld if (NVME_VERSION_HIGHER(&nvme->n_version, 189124979ca3SHans Rosenfeld nvme_version_major, nvme_version_minor)) { 18923c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.%d", 18933c9168faSHans Rosenfeld nvme_version_major, nvme_version_minor); 18943c9168faSHans Rosenfeld if (nvme->n_strict_version) 18953c9168faSHans Rosenfeld goto fail; 18963c9168faSHans Rosenfeld } 18973c9168faSHans Rosenfeld 18983c9168faSHans Rosenfeld /* retrieve controller configuration */ 18993c9168faSHans Rosenfeld cap.r = nvme_get64(nvme, NVME_REG_CAP); 19003c9168faSHans Rosenfeld 19013c9168faSHans Rosenfeld if ((cap.b.cap_css & NVME_CAP_CSS_NVM) == 0) { 19023c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19033c9168faSHans Rosenfeld "!NVM command set not supported by hardware"); 19043c9168faSHans Rosenfeld goto fail; 19053c9168faSHans Rosenfeld } 19063c9168faSHans Rosenfeld 19073c9168faSHans Rosenfeld nvme->n_nssr_supported = cap.b.cap_nssrs; 19083c9168faSHans Rosenfeld nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd; 19093c9168faSHans Rosenfeld nvme->n_timeout = cap.b.cap_to; 19103c9168faSHans Rosenfeld nvme->n_arbitration_mechanisms = cap.b.cap_ams; 19113c9168faSHans Rosenfeld nvme->n_cont_queues_reqd = cap.b.cap_cqr; 19123c9168faSHans Rosenfeld nvme->n_max_queue_entries = cap.b.cap_mqes + 1; 19133c9168faSHans Rosenfeld 19143c9168faSHans Rosenfeld /* 19153c9168faSHans Rosenfeld * The MPSMIN and MPSMAX fields in the CAP register use 0 to specify 19163c9168faSHans Rosenfeld * the base page size of 4k (1<<12), so add 12 here to get the real 19173c9168faSHans Rosenfeld * page size value. 19183c9168faSHans Rosenfeld */ 19193c9168faSHans Rosenfeld nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT), 19203c9168faSHans Rosenfeld cap.b.cap_mpsmax + 12); 19213c9168faSHans Rosenfeld nvme->n_pagesize = 1UL << (nvme->n_pageshift); 19223c9168faSHans Rosenfeld 19233c9168faSHans Rosenfeld /* 19243c9168faSHans Rosenfeld * Set up Queue DMA to transfer at least 1 page-aligned page at a time. 19253c9168faSHans Rosenfeld */ 19263c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize; 19273c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 19283c9168faSHans Rosenfeld 19293c9168faSHans Rosenfeld /* 19303c9168faSHans Rosenfeld * Set up PRP DMA to transfer 1 page-aligned page at a time. 19313c9168faSHans Rosenfeld * Maxxfer may be increased after we identified the controller limits. 19323c9168faSHans Rosenfeld */ 19333c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize; 19343c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 19353c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize; 19362f95345bSYouzhong Yang nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1; 19373c9168faSHans Rosenfeld 19383c9168faSHans Rosenfeld /* 19393c9168faSHans Rosenfeld * Reset controller if it's still in ready state. 19403c9168faSHans Rosenfeld */ 19413c9168faSHans Rosenfeld if (nvme_reset(nvme, B_FALSE) == B_FALSE) { 19423c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller"); 19433c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 19443c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 19453c9168faSHans Rosenfeld goto fail; 19463c9168faSHans Rosenfeld } 19473c9168faSHans Rosenfeld 19483c9168faSHans Rosenfeld /* 19493c9168faSHans Rosenfeld * Create the admin queue pair. 19503c9168faSHans Rosenfeld */ 19513c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0) 19523c9168faSHans Rosenfeld != DDI_SUCCESS) { 19533c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19543c9168faSHans Rosenfeld "!unable to allocate admin qpair"); 19553c9168faSHans Rosenfeld goto fail; 19563c9168faSHans Rosenfeld } 19573c9168faSHans Rosenfeld nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP); 19583c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 19593c9168faSHans Rosenfeld 19603c9168faSHans Rosenfeld nvme->n_progress |= NVME_ADMIN_QUEUE; 19613c9168faSHans Rosenfeld 19623c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 19633c9168faSHans Rosenfeld "admin-queue-len", nvme->n_admin_queue_len); 19643c9168faSHans Rosenfeld 19653c9168faSHans Rosenfeld aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1; 19663c9168faSHans Rosenfeld asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress; 19673c9168faSHans Rosenfeld acq = nvme->n_adminq->nq_cqdma->nd_cookie.dmac_laddress; 19683c9168faSHans Rosenfeld 19693c9168faSHans Rosenfeld ASSERT((asq & (nvme->n_pagesize - 1)) == 0); 19703c9168faSHans Rosenfeld ASSERT((acq & (nvme->n_pagesize - 1)) == 0); 19713c9168faSHans Rosenfeld 19723c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, aqa.r); 19733c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ASQ, asq); 19743c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ACQ, acq); 19753c9168faSHans Rosenfeld 197634c938c7SPete Shephard cc.b.cc_ams = 0; /* use Round-Robin arbitration */ 197734c938c7SPete Shephard cc.b.cc_css = 0; /* use NVM command set */ 19783c9168faSHans Rosenfeld cc.b.cc_mps = nvme->n_pageshift - 12; 197934c938c7SPete Shephard cc.b.cc_shn = 0; /* no shutdown in progress */ 198034c938c7SPete Shephard cc.b.cc_en = 1; /* enable controller */ 198134c938c7SPete Shephard cc.b.cc_iosqes = 6; /* submission queue entry is 2^6 bytes long */ 198234c938c7SPete Shephard cc.b.cc_iocqes = 4; /* completion queue entry is 2^4 bytes long */ 19833c9168faSHans Rosenfeld 19843c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 19853c9168faSHans Rosenfeld 19863c9168faSHans Rosenfeld /* 19873c9168faSHans Rosenfeld * Wait for the controller to become ready. 19883c9168faSHans Rosenfeld */ 19893c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 19903c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 19913c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 19923c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 19933c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 19943c9168faSHans Rosenfeld 19953c9168faSHans Rosenfeld if (csts.b.csts_cfs == 1) { 19963c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19973c9168faSHans Rosenfeld "!controller fatal status at init"); 19983c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, 19993c9168faSHans Rosenfeld DDI_SERVICE_LOST); 20003c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 20013c9168faSHans Rosenfeld goto fail; 20023c9168faSHans Rosenfeld } 20033c9168faSHans Rosenfeld 20043c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) 20053c9168faSHans Rosenfeld break; 20063c9168faSHans Rosenfeld } 20073c9168faSHans Rosenfeld } 20083c9168faSHans Rosenfeld 20093c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 20103c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!controller not ready"); 20113c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 20123c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 20133c9168faSHans Rosenfeld goto fail; 20143c9168faSHans Rosenfeld } 20153c9168faSHans Rosenfeld 20163c9168faSHans Rosenfeld /* 20173c9168faSHans Rosenfeld * Assume an abort command limit of 1. We'll destroy and re-init 20183c9168faSHans Rosenfeld * that later when we know the true abort command limit. 20193c9168faSHans Rosenfeld */ 20203c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL); 20213c9168faSHans Rosenfeld 2022bf26ea4bSHans Rosenfeld /* 2023bf26ea4bSHans Rosenfeld * Setup initial interrupt for admin queue. 2024bf26ea4bSHans Rosenfeld */ 2025bf26ea4bSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1) 2026bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 2027bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1) 2028bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 2029bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1) 2030bf26ea4bSHans Rosenfeld != DDI_SUCCESS)) { 2031bf26ea4bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2032bf26ea4bSHans Rosenfeld "!failed to setup initial interrupt"); 2033bf26ea4bSHans Rosenfeld goto fail; 2034bf26ea4bSHans Rosenfeld } 2035bf26ea4bSHans Rosenfeld 20363c9168faSHans Rosenfeld /* 20373c9168faSHans Rosenfeld * Post an asynchronous event command to catch errors. 20383c9168faSHans Rosenfeld */ 20393c9168faSHans Rosenfeld if (nvme_async_event(nvme) != DDI_SUCCESS) { 20403c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20413c9168faSHans Rosenfeld "!failed to post async event"); 20423c9168faSHans Rosenfeld goto fail; 20433c9168faSHans Rosenfeld } 20443c9168faSHans Rosenfeld 20453c9168faSHans Rosenfeld /* 20463c9168faSHans Rosenfeld * Identify Controller 20473c9168faSHans Rosenfeld */ 20483c9168faSHans Rosenfeld nvme->n_idctl = nvme_identify(nvme, 0); 20493c9168faSHans Rosenfeld if (nvme->n_idctl == NULL) { 20503c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20513c9168faSHans Rosenfeld "!failed to identify controller"); 20523c9168faSHans Rosenfeld goto fail; 20533c9168faSHans Rosenfeld } 20543c9168faSHans Rosenfeld 2055510a6847SHans Rosenfeld /* 2056510a6847SHans Rosenfeld * Get Vendor & Product ID 2057510a6847SHans Rosenfeld */ 2058510a6847SHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 2059510a6847SHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 2060510a6847SHans Rosenfeld sata_split_model(model, &vendor, &product); 2061510a6847SHans Rosenfeld 2062510a6847SHans Rosenfeld if (vendor == NULL) 2063510a6847SHans Rosenfeld nvme->n_vendor = strdup("NVMe"); 2064510a6847SHans Rosenfeld else 2065510a6847SHans Rosenfeld nvme->n_vendor = strdup(vendor); 2066510a6847SHans Rosenfeld 2067510a6847SHans Rosenfeld nvme->n_product = strdup(product); 2068510a6847SHans Rosenfeld 20693c9168faSHans Rosenfeld /* 20703c9168faSHans Rosenfeld * Get controller limits. 20713c9168faSHans Rosenfeld */ 20723c9168faSHans Rosenfeld nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT, 20733c9168faSHans Rosenfeld MIN(nvme->n_admin_queue_len / 10, 20743c9168faSHans Rosenfeld MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit))); 20753c9168faSHans Rosenfeld 20763c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 20773c9168faSHans Rosenfeld "async-event-limit", nvme->n_async_event_limit); 20783c9168faSHans Rosenfeld 20793c9168faSHans Rosenfeld nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1; 20803c9168faSHans Rosenfeld 20816afc9eb2SHans Rosenfeld /* 20826afc9eb2SHans Rosenfeld * Reinitialize the semaphore with the true abort command limit 20836afc9eb2SHans Rosenfeld * supported by the hardware. It's not necessary to disable interrupts 20846afc9eb2SHans Rosenfeld * as only command aborts use the semaphore, and no commands are 20856afc9eb2SHans Rosenfeld * executed or aborted while we're here. 20866afc9eb2SHans Rosenfeld */ 20873c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 20883c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL, 20893c9168faSHans Rosenfeld SEMA_DRIVER, NULL); 20903c9168faSHans Rosenfeld 20913c9168faSHans Rosenfeld nvme->n_progress |= NVME_CTRL_LIMITS; 20923c9168faSHans Rosenfeld 20933c9168faSHans Rosenfeld if (nvme->n_idctl->id_mdts == 0) 20943c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536; 20953c9168faSHans Rosenfeld else 20963c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = 20973c9168faSHans Rosenfeld 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts); 20983c9168faSHans Rosenfeld 20993c9168faSHans Rosenfeld nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1; 21003c9168faSHans Rosenfeld 21013c9168faSHans Rosenfeld /* 21023c9168faSHans Rosenfeld * Limit n_max_data_transfer_size to what we can handle in one PRP. 21033c9168faSHans Rosenfeld * Chained PRPs are currently unsupported. 21043c9168faSHans Rosenfeld * 21053c9168faSHans Rosenfeld * This is a no-op on hardware which doesn't support a transfer size 21063c9168faSHans Rosenfeld * big enough to require chained PRPs. 21073c9168faSHans Rosenfeld */ 21083c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size, 21093c9168faSHans Rosenfeld (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize)); 21103c9168faSHans Rosenfeld 21113c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size; 21123c9168faSHans Rosenfeld 21133c9168faSHans Rosenfeld /* 21143c9168faSHans Rosenfeld * Make sure the minimum/maximum queue entry sizes are not 21153c9168faSHans Rosenfeld * larger/smaller than the default. 21163c9168faSHans Rosenfeld */ 21173c9168faSHans Rosenfeld 21183c9168faSHans Rosenfeld if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) || 21193c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) || 21203c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) || 21213c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t))) 21223c9168faSHans Rosenfeld goto fail; 21233c9168faSHans Rosenfeld 21243c9168faSHans Rosenfeld /* 21253c9168faSHans Rosenfeld * Check for the presence of a Volatile Write Cache. If present, 2126d148d46eSHans Rosenfeld * enable or disable based on the value of the property 2127d148d46eSHans Rosenfeld * volatile-write-cache-enable (default is enabled). 21283c9168faSHans Rosenfeld */ 2129d148d46eSHans Rosenfeld nvme->n_write_cache_present = 2130d148d46eSHans Rosenfeld nvme->n_idctl->id_vwc.vwc_present == 0 ? B_FALSE : B_TRUE; 2131d148d46eSHans Rosenfeld 2132d148d46eSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2133d148d46eSHans Rosenfeld "volatile-write-cache-present", 2134d148d46eSHans Rosenfeld nvme->n_write_cache_present ? 1 : 0); 2135d148d46eSHans Rosenfeld 2136d148d46eSHans Rosenfeld if (!nvme->n_write_cache_present) { 2137d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = B_FALSE; 2138d148d46eSHans Rosenfeld } else if (!nvme_write_cache_set(nvme, nvme->n_write_cache_enabled)) { 2139d148d46eSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2140d148d46eSHans Rosenfeld "!failed to %sable volatile write cache", 2141d148d46eSHans Rosenfeld nvme->n_write_cache_enabled ? "en" : "dis"); 21423c9168faSHans Rosenfeld /* 2143d148d46eSHans Rosenfeld * Assume the cache is (still) enabled. 21443c9168faSHans Rosenfeld */ 2145d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = B_TRUE; 21463c9168faSHans Rosenfeld } 21473c9168faSHans Rosenfeld 2148d148d46eSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2149d148d46eSHans Rosenfeld "volatile-write-cache-enable", 2150d148d46eSHans Rosenfeld nvme->n_write_cache_enabled ? 1 : 0); 2151d148d46eSHans Rosenfeld 21523c9168faSHans Rosenfeld /* 21533c9168faSHans Rosenfeld * Grab a copy of all mandatory log pages. 21543c9168faSHans Rosenfeld * 21553c9168faSHans Rosenfeld * TODO: should go away once user space tool exists to print logs 21563c9168faSHans Rosenfeld */ 21573c9168faSHans Rosenfeld nvme->n_error_log = (nvme_error_log_entry_t *) 21583c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_ERROR); 21593c9168faSHans Rosenfeld nvme->n_health_log = (nvme_health_log_t *) 21603c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_HEALTH, -1); 21613c9168faSHans Rosenfeld nvme->n_fwslot_log = (nvme_fwslot_log_t *) 21623c9168faSHans Rosenfeld nvme_get_logpage(nvme, NVME_LOGPAGE_FWSLOT); 21633c9168faSHans Rosenfeld 21643c9168faSHans Rosenfeld /* 21653c9168faSHans Rosenfeld * Identify Namespaces 21663c9168faSHans Rosenfeld */ 21673c9168faSHans Rosenfeld nvme->n_namespace_count = nvme->n_idctl->id_nn; 21683c9168faSHans Rosenfeld nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) * 21693c9168faSHans Rosenfeld nvme->n_namespace_count, KM_SLEEP); 21703c9168faSHans Rosenfeld 21713c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 21723c9168faSHans Rosenfeld nvme_identify_nsid_t *idns; 21733c9168faSHans Rosenfeld int last_rp; 21743c9168faSHans Rosenfeld 21753c9168faSHans Rosenfeld nvme->n_ns[i].ns_nvme = nvme; 21763c9168faSHans Rosenfeld nvme->n_ns[i].ns_idns = idns = nvme_identify(nvme, i + 1); 21773c9168faSHans Rosenfeld 21783c9168faSHans Rosenfeld if (idns == NULL) { 21793c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21803c9168faSHans Rosenfeld "!failed to identify namespace %d", i + 1); 21813c9168faSHans Rosenfeld goto fail; 21823c9168faSHans Rosenfeld } 21833c9168faSHans Rosenfeld 21843c9168faSHans Rosenfeld nvme->n_ns[i].ns_id = i + 1; 21853c9168faSHans Rosenfeld nvme->n_ns[i].ns_block_count = idns->id_nsize; 21863c9168faSHans Rosenfeld nvme->n_ns[i].ns_block_size = 21873c9168faSHans Rosenfeld 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads; 21883c9168faSHans Rosenfeld nvme->n_ns[i].ns_best_block_size = nvme->n_ns[i].ns_block_size; 21893c9168faSHans Rosenfeld 219024979ca3SHans Rosenfeld /* 219124979ca3SHans Rosenfeld * Get the EUI64 if present. If not present prepare the devid 219224979ca3SHans Rosenfeld * from other device data. 219324979ca3SHans Rosenfeld */ 219424979ca3SHans Rosenfeld if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) 219524979ca3SHans Rosenfeld bcopy(idns->id_eui64, nvme->n_ns[i].ns_eui64, 219624979ca3SHans Rosenfeld sizeof (nvme->n_ns[i].ns_eui64)); 219724979ca3SHans Rosenfeld 219824979ca3SHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 219924979ca3SHans Rosenfeld if (*(uint64_t *)nvme->n_ns[i].ns_eui64 == 0) { 220024979ca3SHans Rosenfeld nvme_prepare_devid(nvme, nvme->n_ns[i].ns_id); 220124979ca3SHans Rosenfeld } else { 220224979ca3SHans Rosenfeld /* 220324979ca3SHans Rosenfeld * Until EUI64 support is tested on real hardware we 220424979ca3SHans Rosenfeld * will ignore namespaces with an EUI64. This can 220524979ca3SHans Rosenfeld * be overriden by setting strict-version=0 in nvme.conf 220624979ca3SHans Rosenfeld */ 220724979ca3SHans Rosenfeld if (nvme->n_strict_version) 220824979ca3SHans Rosenfeld nvme->n_ns[i].ns_ignore = B_TRUE; 220924979ca3SHans Rosenfeld } 22103c9168faSHans Rosenfeld 22113c9168faSHans Rosenfeld /* 22123c9168faSHans Rosenfeld * Find the LBA format with no metadata and the best relative 22133c9168faSHans Rosenfeld * performance. A value of 3 means "degraded", 0 is best. 22143c9168faSHans Rosenfeld */ 22153c9168faSHans Rosenfeld last_rp = 3; 22160d140ff9SHans Rosenfeld for (int j = 0; j <= idns->id_nlbaf; j++) { 22173c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_lbads == 0) 22183c9168faSHans Rosenfeld break; 22193c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_ms != 0) 22203c9168faSHans Rosenfeld continue; 22213c9168faSHans Rosenfeld if (idns->id_lbaf[j].lbaf_rp >= last_rp) 22223c9168faSHans Rosenfeld continue; 22233c9168faSHans Rosenfeld last_rp = idns->id_lbaf[j].lbaf_rp; 22243c9168faSHans Rosenfeld nvme->n_ns[i].ns_best_block_size = 22253c9168faSHans Rosenfeld 1 << idns->id_lbaf[j].lbaf_lbads; 22263c9168faSHans Rosenfeld } 22273c9168faSHans Rosenfeld 22286801591eSHans Rosenfeld if (nvme->n_ns[i].ns_best_block_size < nvme->n_min_block_size) 22296801591eSHans Rosenfeld nvme->n_ns[i].ns_best_block_size = 22306801591eSHans Rosenfeld nvme->n_min_block_size; 22316801591eSHans Rosenfeld 22323c9168faSHans Rosenfeld /* 22333c9168faSHans Rosenfeld * We currently don't support namespaces that use either: 22343c9168faSHans Rosenfeld * - thin provisioning 22353c9168faSHans Rosenfeld * - protection information 22363c9168faSHans Rosenfeld */ 22373c9168faSHans Rosenfeld if (idns->id_nsfeat.f_thin || 22383c9168faSHans Rosenfeld idns->id_dps.dp_pinfo) { 22393c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22403c9168faSHans Rosenfeld "!ignoring namespace %d, unsupported features: " 22419cfb65faSHans Rosenfeld "thin = %d, pinfo = %d", i + 1, 22429cfb65faSHans Rosenfeld idns->id_nsfeat.f_thin, idns->id_dps.dp_pinfo); 22433c9168faSHans Rosenfeld nvme->n_ns[i].ns_ignore = B_TRUE; 22443c9168faSHans Rosenfeld } 22453c9168faSHans Rosenfeld } 22463c9168faSHans Rosenfeld 22473c9168faSHans Rosenfeld /* 22483c9168faSHans Rosenfeld * Try to set up MSI/MSI-X interrupts. 22493c9168faSHans Rosenfeld */ 22503c9168faSHans Rosenfeld if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX)) 22513c9168faSHans Rosenfeld != 0) { 22523c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 22533c9168faSHans Rosenfeld 22543c9168faSHans Rosenfeld nqueues = MIN(UINT16_MAX, ncpus); 22553c9168faSHans Rosenfeld 22563c9168faSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 22573c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS) && 22583c9168faSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 22593c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS)) { 22603c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22613c9168faSHans Rosenfeld "!failed to setup MSI/MSI-X interrupts"); 22623c9168faSHans Rosenfeld goto fail; 22633c9168faSHans Rosenfeld } 22643c9168faSHans Rosenfeld } 22653c9168faSHans Rosenfeld 22663c9168faSHans Rosenfeld nqueues = nvme->n_intr_cnt; 22673c9168faSHans Rosenfeld 22683c9168faSHans Rosenfeld /* 22693c9168faSHans Rosenfeld * Create I/O queue pairs. 22703c9168faSHans Rosenfeld */ 22713c9168faSHans Rosenfeld nvme->n_ioq_count = nvme_set_nqueues(nvme, nqueues); 22723c9168faSHans Rosenfeld if (nvme->n_ioq_count == 0) { 22733c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22743c9168faSHans Rosenfeld "!failed to set number of I/O queues to %d", nqueues); 22753c9168faSHans Rosenfeld goto fail; 22763c9168faSHans Rosenfeld } 22773c9168faSHans Rosenfeld 22783c9168faSHans Rosenfeld /* 22793c9168faSHans Rosenfeld * Reallocate I/O queue array 22803c9168faSHans Rosenfeld */ 22813c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *)); 22823c9168faSHans Rosenfeld nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) * 22833c9168faSHans Rosenfeld (nvme->n_ioq_count + 1), KM_SLEEP); 22843c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 22853c9168faSHans Rosenfeld 22863c9168faSHans Rosenfeld /* 22873c9168faSHans Rosenfeld * If we got less queues than we asked for we might as well give 22883c9168faSHans Rosenfeld * some of the interrupt vectors back to the system. 22893c9168faSHans Rosenfeld */ 22903c9168faSHans Rosenfeld if (nvme->n_ioq_count < nqueues) { 22913c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 22923c9168faSHans Rosenfeld 229334c938c7SPete Shephard if (nvme_setup_interrupts(nvme, nvme->n_intr_type, 229434c938c7SPete Shephard nvme->n_ioq_count) != DDI_SUCCESS) { 22953c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22963c9168faSHans Rosenfeld "!failed to reduce number of interrupts"); 22973c9168faSHans Rosenfeld goto fail; 22983c9168faSHans Rosenfeld } 22993c9168faSHans Rosenfeld } 23003c9168faSHans Rosenfeld 23013c9168faSHans Rosenfeld /* 23023c9168faSHans Rosenfeld * Alloc & register I/O queue pairs 23033c9168faSHans Rosenfeld */ 23043c9168faSHans Rosenfeld nvme->n_io_queue_len = 23053c9168faSHans Rosenfeld MIN(nvme->n_io_queue_len, nvme->n_max_queue_entries); 23063c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-queue-len", 23073c9168faSHans Rosenfeld nvme->n_io_queue_len); 23083c9168faSHans Rosenfeld 23093c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 23103c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_io_queue_len, 23113c9168faSHans Rosenfeld &nvme->n_ioq[i], i) != DDI_SUCCESS) { 23123c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23133c9168faSHans Rosenfeld "!unable to allocate I/O qpair %d", i); 23143c9168faSHans Rosenfeld goto fail; 23153c9168faSHans Rosenfeld } 23163c9168faSHans Rosenfeld 23173c9168faSHans Rosenfeld if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i) 23183c9168faSHans Rosenfeld != DDI_SUCCESS) { 23193c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23203c9168faSHans Rosenfeld "!unable to create I/O qpair %d", i); 23213c9168faSHans Rosenfeld goto fail; 23223c9168faSHans Rosenfeld } 23233c9168faSHans Rosenfeld } 23243c9168faSHans Rosenfeld 23253c9168faSHans Rosenfeld /* 23263c9168faSHans Rosenfeld * Post more asynchronous events commands to reduce event reporting 23273c9168faSHans Rosenfeld * latency as suggested by the spec. 23283c9168faSHans Rosenfeld */ 23293c9168faSHans Rosenfeld for (i = 1; i != nvme->n_async_event_limit; i++) { 23303c9168faSHans Rosenfeld if (nvme_async_event(nvme) != DDI_SUCCESS) { 23313c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23323c9168faSHans Rosenfeld "!failed to post async event %d", i); 23333c9168faSHans Rosenfeld goto fail; 23343c9168faSHans Rosenfeld } 23353c9168faSHans Rosenfeld } 23363c9168faSHans Rosenfeld 23373c9168faSHans Rosenfeld return (DDI_SUCCESS); 23383c9168faSHans Rosenfeld 23393c9168faSHans Rosenfeld fail: 23403c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 23413c9168faSHans Rosenfeld return (DDI_FAILURE); 23423c9168faSHans Rosenfeld } 23433c9168faSHans Rosenfeld 23443c9168faSHans Rosenfeld static uint_t 23453c9168faSHans Rosenfeld nvme_intr(caddr_t arg1, caddr_t arg2) 23463c9168faSHans Rosenfeld { 23473c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 23483c9168faSHans Rosenfeld nvme_t *nvme = (nvme_t *)arg1; 23493c9168faSHans Rosenfeld int inum = (int)(uintptr_t)arg2; 2350bf26ea4bSHans Rosenfeld int ccnt = 0; 23513c9168faSHans Rosenfeld int qnum; 23523c9168faSHans Rosenfeld nvme_cmd_t *cmd; 23533c9168faSHans Rosenfeld 23543c9168faSHans Rosenfeld if (inum >= nvme->n_intr_cnt) 23553c9168faSHans Rosenfeld return (DDI_INTR_UNCLAIMED); 23563c9168faSHans Rosenfeld 23573c9168faSHans Rosenfeld /* 23583c9168faSHans Rosenfeld * The interrupt vector a queue uses is calculated as queue_idx % 23593c9168faSHans Rosenfeld * intr_cnt in nvme_create_io_qpair(). Iterate through the queue array 23603c9168faSHans Rosenfeld * in steps of n_intr_cnt to process all queues using this vector. 23613c9168faSHans Rosenfeld */ 23623c9168faSHans Rosenfeld for (qnum = inum; 23633c9168faSHans Rosenfeld qnum < nvme->n_ioq_count + 1 && nvme->n_ioq[qnum] != NULL; 23643c9168faSHans Rosenfeld qnum += nvme->n_intr_cnt) { 23653c9168faSHans Rosenfeld while ((cmd = nvme_retrieve_cmd(nvme, nvme->n_ioq[qnum]))) { 23663c9168faSHans Rosenfeld taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq, 23673c9168faSHans Rosenfeld cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent); 2368bf26ea4bSHans Rosenfeld ccnt++; 23693c9168faSHans Rosenfeld } 23703c9168faSHans Rosenfeld } 23713c9168faSHans Rosenfeld 2372bf26ea4bSHans Rosenfeld return (ccnt > 0 ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); 23733c9168faSHans Rosenfeld } 23743c9168faSHans Rosenfeld 23753c9168faSHans Rosenfeld static void 23766afc9eb2SHans Rosenfeld nvme_release_interrupts(nvme_t *nvme) 23773c9168faSHans Rosenfeld { 23783c9168faSHans Rosenfeld int i; 23793c9168faSHans Rosenfeld 23803c9168faSHans Rosenfeld for (i = 0; i < nvme->n_intr_cnt; i++) { 23813c9168faSHans Rosenfeld if (nvme->n_inth[i] == NULL) 23823c9168faSHans Rosenfeld break; 23833c9168faSHans Rosenfeld 23843c9168faSHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 23853c9168faSHans Rosenfeld (void) ddi_intr_block_disable(&nvme->n_inth[i], 1); 23863c9168faSHans Rosenfeld else 23873c9168faSHans Rosenfeld (void) ddi_intr_disable(nvme->n_inth[i]); 23883c9168faSHans Rosenfeld 23893c9168faSHans Rosenfeld (void) ddi_intr_remove_handler(nvme->n_inth[i]); 23903c9168faSHans Rosenfeld (void) ddi_intr_free(nvme->n_inth[i]); 23913c9168faSHans Rosenfeld } 23923c9168faSHans Rosenfeld 23933c9168faSHans Rosenfeld kmem_free(nvme->n_inth, nvme->n_inth_sz); 23943c9168faSHans Rosenfeld nvme->n_inth = NULL; 23953c9168faSHans Rosenfeld nvme->n_inth_sz = 0; 23963c9168faSHans Rosenfeld 23973c9168faSHans Rosenfeld nvme->n_progress &= ~NVME_INTERRUPTS; 23983c9168faSHans Rosenfeld } 23993c9168faSHans Rosenfeld 24003c9168faSHans Rosenfeld static int 24013c9168faSHans Rosenfeld nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs) 24023c9168faSHans Rosenfeld { 24033c9168faSHans Rosenfeld int nintrs, navail, count; 24043c9168faSHans Rosenfeld int ret; 24053c9168faSHans Rosenfeld int i; 24063c9168faSHans Rosenfeld 24073c9168faSHans Rosenfeld if (nvme->n_intr_types == 0) { 24083c9168faSHans Rosenfeld ret = ddi_intr_get_supported_types(nvme->n_dip, 24093c9168faSHans Rosenfeld &nvme->n_intr_types); 24103c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 24113c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 24123c9168faSHans Rosenfeld "!%s: ddi_intr_get_supported types failed", 24133c9168faSHans Rosenfeld __func__); 24143c9168faSHans Rosenfeld return (ret); 24153c9168faSHans Rosenfeld } 2416*9d08e1f8SHans Rosenfeld #ifdef __x86 2417*9d08e1f8SHans Rosenfeld if (get_hwenv() == HW_VMWARE) 2418*9d08e1f8SHans Rosenfeld nvme->n_intr_types &= ~DDI_INTR_TYPE_MSIX; 2419*9d08e1f8SHans Rosenfeld #endif 24203c9168faSHans Rosenfeld } 24213c9168faSHans Rosenfeld 24223c9168faSHans Rosenfeld if ((nvme->n_intr_types & intr_type) == 0) 24233c9168faSHans Rosenfeld return (DDI_FAILURE); 24243c9168faSHans Rosenfeld 24253c9168faSHans Rosenfeld ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs); 24263c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 24273c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed", 24283c9168faSHans Rosenfeld __func__); 24293c9168faSHans Rosenfeld return (ret); 24303c9168faSHans Rosenfeld } 24313c9168faSHans Rosenfeld 24323c9168faSHans Rosenfeld ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail); 24333c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 24343c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed", 24353c9168faSHans Rosenfeld __func__); 24363c9168faSHans Rosenfeld return (ret); 24373c9168faSHans Rosenfeld } 24383c9168faSHans Rosenfeld 24393c9168faSHans Rosenfeld /* We want at most one interrupt per queue pair. */ 24403c9168faSHans Rosenfeld if (navail > nqpairs) 24413c9168faSHans Rosenfeld navail = nqpairs; 24423c9168faSHans Rosenfeld 24433c9168faSHans Rosenfeld nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail; 24443c9168faSHans Rosenfeld nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP); 24453c9168faSHans Rosenfeld 24463c9168faSHans Rosenfeld ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail, 24473c9168faSHans Rosenfeld &count, 0); 24483c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 24493c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed", 24503c9168faSHans Rosenfeld __func__); 24513c9168faSHans Rosenfeld goto fail; 24523c9168faSHans Rosenfeld } 24533c9168faSHans Rosenfeld 24543c9168faSHans Rosenfeld nvme->n_intr_cnt = count; 24553c9168faSHans Rosenfeld 24563c9168faSHans Rosenfeld ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri); 24573c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 24583c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed", 24593c9168faSHans Rosenfeld __func__); 24603c9168faSHans Rosenfeld goto fail; 24613c9168faSHans Rosenfeld } 24623c9168faSHans Rosenfeld 24633c9168faSHans Rosenfeld for (i = 0; i < count; i++) { 24643c9168faSHans Rosenfeld ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr, 24653c9168faSHans Rosenfeld (void *)nvme, (void *)(uintptr_t)i); 24663c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 24673c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 24683c9168faSHans Rosenfeld "!%s: ddi_intr_add_handler failed", __func__); 24693c9168faSHans Rosenfeld goto fail; 24703c9168faSHans Rosenfeld } 24713c9168faSHans Rosenfeld } 24723c9168faSHans Rosenfeld 24733c9168faSHans Rosenfeld (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap); 24743c9168faSHans Rosenfeld 24756afc9eb2SHans Rosenfeld for (i = 0; i < count; i++) { 247675b41617SHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 247775b41617SHans Rosenfeld ret = ddi_intr_block_enable(&nvme->n_inth[i], 1); 247875b41617SHans Rosenfeld else 247975b41617SHans Rosenfeld ret = ddi_intr_enable(nvme->n_inth[i]); 24803c9168faSHans Rosenfeld 248175b41617SHans Rosenfeld if (ret != DDI_SUCCESS) { 248275b41617SHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 248375b41617SHans Rosenfeld "!%s: enabling interrupt %d failed", __func__, i); 248475b41617SHans Rosenfeld goto fail; 24856afc9eb2SHans Rosenfeld } 24866afc9eb2SHans Rosenfeld } 24876afc9eb2SHans Rosenfeld 24883c9168faSHans Rosenfeld nvme->n_intr_type = intr_type; 24893c9168faSHans Rosenfeld 24903c9168faSHans Rosenfeld nvme->n_progress |= NVME_INTERRUPTS; 24913c9168faSHans Rosenfeld 24923c9168faSHans Rosenfeld return (DDI_SUCCESS); 24933c9168faSHans Rosenfeld 24943c9168faSHans Rosenfeld fail: 24953c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 24963c9168faSHans Rosenfeld 24973c9168faSHans Rosenfeld return (ret); 24983c9168faSHans Rosenfeld } 24993c9168faSHans Rosenfeld 25003c9168faSHans Rosenfeld static int 25013c9168faSHans Rosenfeld nvme_fm_errcb(dev_info_t *dip, ddi_fm_error_t *fm_error, const void *arg) 25023c9168faSHans Rosenfeld { 25033c9168faSHans Rosenfeld _NOTE(ARGUNUSED(arg)); 25043c9168faSHans Rosenfeld 25053c9168faSHans Rosenfeld pci_ereport_post(dip, fm_error, NULL); 25063c9168faSHans Rosenfeld return (fm_error->fme_status); 25073c9168faSHans Rosenfeld } 25083c9168faSHans Rosenfeld 25093c9168faSHans Rosenfeld static int 25103c9168faSHans Rosenfeld nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 25113c9168faSHans Rosenfeld { 25123c9168faSHans Rosenfeld nvme_t *nvme; 25133c9168faSHans Rosenfeld int instance; 25143c9168faSHans Rosenfeld int nregs; 25153c9168faSHans Rosenfeld off_t regsize; 25163c9168faSHans Rosenfeld int i; 25173c9168faSHans Rosenfeld char name[32]; 25183c9168faSHans Rosenfeld 25193c9168faSHans Rosenfeld if (cmd != DDI_ATTACH) 25203c9168faSHans Rosenfeld return (DDI_FAILURE); 25213c9168faSHans Rosenfeld 25223c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 25233c9168faSHans Rosenfeld 25243c9168faSHans Rosenfeld if (ddi_soft_state_zalloc(nvme_state, instance) != DDI_SUCCESS) 25253c9168faSHans Rosenfeld return (DDI_FAILURE); 25263c9168faSHans Rosenfeld 25273c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 25283c9168faSHans Rosenfeld ddi_set_driver_private(dip, nvme); 25293c9168faSHans Rosenfeld nvme->n_dip = dip; 25303c9168faSHans Rosenfeld 25313c9168faSHans Rosenfeld nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 25323c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "strict-version", 1) == 1 ? B_TRUE : B_FALSE; 25333c9168faSHans Rosenfeld nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY, 25343c9168faSHans Rosenfeld dip, DDI_PROP_DONTPASS, "ignore-unknown-vendor-status", 0) == 1 ? 25353c9168faSHans Rosenfeld B_TRUE : B_FALSE; 25363c9168faSHans Rosenfeld nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 25373c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "admin-queue-len", NVME_DEFAULT_ADMIN_QUEUE_LEN); 25383c9168faSHans Rosenfeld nvme->n_io_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 25393c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "io-queue-len", NVME_DEFAULT_IO_QUEUE_LEN); 25403c9168faSHans Rosenfeld nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 25413c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "async-event-limit", 25423c9168faSHans Rosenfeld NVME_DEFAULT_ASYNC_EVENT_LIMIT); 2543d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2544d148d46eSHans Rosenfeld DDI_PROP_DONTPASS, "volatile-write-cache-enable", 1) != 0 ? 2545d148d46eSHans Rosenfeld B_TRUE : B_FALSE; 25466801591eSHans Rosenfeld nvme->n_min_block_size = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 25476801591eSHans Rosenfeld DDI_PROP_DONTPASS, "min-phys-block-size", 25486801591eSHans Rosenfeld NVME_DEFAULT_MIN_BLOCK_SIZE); 25496801591eSHans Rosenfeld 25506801591eSHans Rosenfeld if (!ISP2(nvme->n_min_block_size) || 25516801591eSHans Rosenfeld (nvme->n_min_block_size < NVME_DEFAULT_MIN_BLOCK_SIZE)) { 25526801591eSHans Rosenfeld dev_err(dip, CE_WARN, "!min-phys-block-size %s, " 25536801591eSHans Rosenfeld "using default %d", ISP2(nvme->n_min_block_size) ? 25546801591eSHans Rosenfeld "too low" : "not a power of 2", 25556801591eSHans Rosenfeld NVME_DEFAULT_MIN_BLOCK_SIZE); 25566801591eSHans Rosenfeld nvme->n_min_block_size = NVME_DEFAULT_MIN_BLOCK_SIZE; 25576801591eSHans Rosenfeld } 25583c9168faSHans Rosenfeld 25593c9168faSHans Rosenfeld if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN) 25603c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN; 25613c9168faSHans Rosenfeld else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN) 25623c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN; 25633c9168faSHans Rosenfeld 25643c9168faSHans Rosenfeld if (nvme->n_io_queue_len < NVME_MIN_IO_QUEUE_LEN) 25653c9168faSHans Rosenfeld nvme->n_io_queue_len = NVME_MIN_IO_QUEUE_LEN; 25663c9168faSHans Rosenfeld 25673c9168faSHans Rosenfeld if (nvme->n_async_event_limit < 1) 25683c9168faSHans Rosenfeld nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT; 25693c9168faSHans Rosenfeld 25703c9168faSHans Rosenfeld nvme->n_reg_acc_attr = nvme_reg_acc_attr; 25713c9168faSHans Rosenfeld nvme->n_queue_dma_attr = nvme_queue_dma_attr; 25723c9168faSHans Rosenfeld nvme->n_prp_dma_attr = nvme_prp_dma_attr; 25733c9168faSHans Rosenfeld nvme->n_sgl_dma_attr = nvme_sgl_dma_attr; 25743c9168faSHans Rosenfeld 25753c9168faSHans Rosenfeld /* 25763c9168faSHans Rosenfeld * Setup FMA support. 25773c9168faSHans Rosenfeld */ 25783c9168faSHans Rosenfeld nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip, 25793c9168faSHans Rosenfeld DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 25803c9168faSHans Rosenfeld DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 25813c9168faSHans Rosenfeld DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 25823c9168faSHans Rosenfeld 25833c9168faSHans Rosenfeld ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc); 25843c9168faSHans Rosenfeld 25853c9168faSHans Rosenfeld if (nvme->n_fm_cap) { 25863c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE) 25873c9168faSHans Rosenfeld nvme->n_reg_acc_attr.devacc_attr_access = 25883c9168faSHans Rosenfeld DDI_FLAGERR_ACC; 25893c9168faSHans Rosenfeld 25903c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) { 25913c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 25923c9168faSHans Rosenfeld nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 25933c9168faSHans Rosenfeld } 25943c9168faSHans Rosenfeld 25953c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 25963c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 25973c9168faSHans Rosenfeld pci_ereport_setup(dip); 25983c9168faSHans Rosenfeld 25993c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 26003c9168faSHans Rosenfeld ddi_fm_handler_register(dip, nvme_fm_errcb, 26013c9168faSHans Rosenfeld (void *)nvme); 26023c9168faSHans Rosenfeld } 26033c9168faSHans Rosenfeld 26043c9168faSHans Rosenfeld nvme->n_progress |= NVME_FMA_INIT; 26053c9168faSHans Rosenfeld 26063c9168faSHans Rosenfeld /* 26073c9168faSHans Rosenfeld * The spec defines several register sets. Only the controller 26083c9168faSHans Rosenfeld * registers (set 1) are currently used. 26093c9168faSHans Rosenfeld */ 26103c9168faSHans Rosenfeld if (ddi_dev_nregs(dip, &nregs) == DDI_FAILURE || 26113c9168faSHans Rosenfeld nregs < 2 || 26123c9168faSHans Rosenfeld ddi_dev_regsize(dip, 1, ®size) == DDI_FAILURE) 26133c9168faSHans Rosenfeld goto fail; 26143c9168faSHans Rosenfeld 26153c9168faSHans Rosenfeld if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize, 26163c9168faSHans Rosenfeld &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) { 26173c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to map regset 1"); 26183c9168faSHans Rosenfeld goto fail; 26193c9168faSHans Rosenfeld } 26203c9168faSHans Rosenfeld 26213c9168faSHans Rosenfeld nvme->n_progress |= NVME_REGS_MAPPED; 26223c9168faSHans Rosenfeld 26233c9168faSHans Rosenfeld /* 26243c9168faSHans Rosenfeld * Create taskq for command completion. 26253c9168faSHans Rosenfeld */ 26263c9168faSHans Rosenfeld (void) snprintf(name, sizeof (name), "%s%d_cmd_taskq", 26273c9168faSHans Rosenfeld ddi_driver_name(dip), ddi_get_instance(dip)); 26283c9168faSHans Rosenfeld nvme->n_cmd_taskq = ddi_taskq_create(dip, name, MIN(UINT16_MAX, ncpus), 26293c9168faSHans Rosenfeld TASKQ_DEFAULTPRI, 0); 26303c9168faSHans Rosenfeld if (nvme->n_cmd_taskq == NULL) { 26313c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to create cmd taskq"); 26323c9168faSHans Rosenfeld goto fail; 26333c9168faSHans Rosenfeld } 26343c9168faSHans Rosenfeld 26358834f7acSYouzhong Yang /* 26368834f7acSYouzhong Yang * Create PRP DMA cache 26378834f7acSYouzhong Yang */ 26388834f7acSYouzhong Yang (void) snprintf(name, sizeof (name), "%s%d_prp_cache", 26398834f7acSYouzhong Yang ddi_driver_name(dip), ddi_get_instance(dip)); 26408834f7acSYouzhong Yang nvme->n_prp_cache = kmem_cache_create(name, sizeof (nvme_dma_t), 26418834f7acSYouzhong Yang 0, nvme_prp_dma_constructor, nvme_prp_dma_destructor, 26428834f7acSYouzhong Yang NULL, (void *)nvme, NULL, 0); 26433c9168faSHans Rosenfeld 26443c9168faSHans Rosenfeld if (nvme_init(nvme) != DDI_SUCCESS) 26453c9168faSHans Rosenfeld goto fail; 26463c9168faSHans Rosenfeld 26473c9168faSHans Rosenfeld /* 26483c9168faSHans Rosenfeld * Attach the blkdev driver for each namespace. 26493c9168faSHans Rosenfeld */ 26503c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 26513c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_ignore) 26523c9168faSHans Rosenfeld continue; 26533c9168faSHans Rosenfeld 26543c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl = bd_alloc_handle(&nvme->n_ns[i], 26553c9168faSHans Rosenfeld &nvme_bd_ops, &nvme->n_prp_dma_attr, KM_SLEEP); 26563c9168faSHans Rosenfeld 26573c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl == NULL) { 26583c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 26593c9168faSHans Rosenfeld "!failed to get blkdev handle for namespace %d", i); 26603c9168faSHans Rosenfeld goto fail; 26613c9168faSHans Rosenfeld } 26623c9168faSHans Rosenfeld 26633c9168faSHans Rosenfeld if (bd_attach_handle(dip, nvme->n_ns[i].ns_bd_hdl) 26643c9168faSHans Rosenfeld != DDI_SUCCESS) { 26653c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 26663c9168faSHans Rosenfeld "!failed to attach blkdev handle for namespace %d", 26673c9168faSHans Rosenfeld i); 26683c9168faSHans Rosenfeld goto fail; 26693c9168faSHans Rosenfeld } 26703c9168faSHans Rosenfeld } 26713c9168faSHans Rosenfeld 26723c9168faSHans Rosenfeld return (DDI_SUCCESS); 26733c9168faSHans Rosenfeld 26743c9168faSHans Rosenfeld fail: 26753c9168faSHans Rosenfeld /* attach successful anyway so that FMA can retire the device */ 26763c9168faSHans Rosenfeld if (nvme->n_dead) 26773c9168faSHans Rosenfeld return (DDI_SUCCESS); 26783c9168faSHans Rosenfeld 26793c9168faSHans Rosenfeld (void) nvme_detach(dip, DDI_DETACH); 26803c9168faSHans Rosenfeld 26813c9168faSHans Rosenfeld return (DDI_FAILURE); 26823c9168faSHans Rosenfeld } 26833c9168faSHans Rosenfeld 26843c9168faSHans Rosenfeld static int 26853c9168faSHans Rosenfeld nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 26863c9168faSHans Rosenfeld { 26873c9168faSHans Rosenfeld int instance, i; 26883c9168faSHans Rosenfeld nvme_t *nvme; 26893c9168faSHans Rosenfeld 26903c9168faSHans Rosenfeld if (cmd != DDI_DETACH) 26913c9168faSHans Rosenfeld return (DDI_FAILURE); 26923c9168faSHans Rosenfeld 26933c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 26943c9168faSHans Rosenfeld 26953c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 26963c9168faSHans Rosenfeld 26973c9168faSHans Rosenfeld if (nvme == NULL) 26983c9168faSHans Rosenfeld return (DDI_FAILURE); 26993c9168faSHans Rosenfeld 27003c9168faSHans Rosenfeld if (nvme->n_ns) { 27013c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 27023c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl) { 27033c9168faSHans Rosenfeld (void) bd_detach_handle( 27043c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl); 27053c9168faSHans Rosenfeld bd_free_handle(nvme->n_ns[i].ns_bd_hdl); 27063c9168faSHans Rosenfeld } 27073c9168faSHans Rosenfeld 27083c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_idns) 27093c9168faSHans Rosenfeld kmem_free(nvme->n_ns[i].ns_idns, 27103c9168faSHans Rosenfeld sizeof (nvme_identify_nsid_t)); 271124979ca3SHans Rosenfeld if (nvme->n_ns[i].ns_devid) 271224979ca3SHans Rosenfeld strfree(nvme->n_ns[i].ns_devid); 27133c9168faSHans Rosenfeld } 27143c9168faSHans Rosenfeld 27153c9168faSHans Rosenfeld kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) * 27163c9168faSHans Rosenfeld nvme->n_namespace_count); 27173c9168faSHans Rosenfeld } 27183c9168faSHans Rosenfeld 27193c9168faSHans Rosenfeld if (nvme->n_progress & NVME_INTERRUPTS) 27203c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 27213c9168faSHans Rosenfeld 27223c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 27233c9168faSHans Rosenfeld ddi_taskq_wait(nvme->n_cmd_taskq); 27243c9168faSHans Rosenfeld 27253c9168faSHans Rosenfeld if (nvme->n_ioq_count > 0) { 27263c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 27273c9168faSHans Rosenfeld if (nvme->n_ioq[i] != NULL) { 27283c9168faSHans Rosenfeld /* TODO: send destroy queue commands */ 27293c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_ioq[i]); 27303c9168faSHans Rosenfeld } 27313c9168faSHans Rosenfeld } 27323c9168faSHans Rosenfeld 27333c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) * 27343c9168faSHans Rosenfeld (nvme->n_ioq_count + 1)); 27353c9168faSHans Rosenfeld } 27363c9168faSHans Rosenfeld 27378834f7acSYouzhong Yang if (nvme->n_prp_cache != NULL) { 27388834f7acSYouzhong Yang kmem_cache_destroy(nvme->n_prp_cache); 27398834f7acSYouzhong Yang } 27408834f7acSYouzhong Yang 27413c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) { 27423c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE); 27433c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 27443c9168faSHans Rosenfeld } 27453c9168faSHans Rosenfeld 27463c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 27473c9168faSHans Rosenfeld ddi_taskq_destroy(nvme->n_cmd_taskq); 27483c9168faSHans Rosenfeld 27493c9168faSHans Rosenfeld if (nvme->n_progress & NVME_CTRL_LIMITS) 27503c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 27513c9168faSHans Rosenfeld 27523c9168faSHans Rosenfeld if (nvme->n_progress & NVME_ADMIN_QUEUE) 27533c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_adminq); 27543c9168faSHans Rosenfeld 27553c9168faSHans Rosenfeld if (nvme->n_idctl) 27563c9168faSHans Rosenfeld kmem_free(nvme->n_idctl, sizeof (nvme_identify_ctrl_t)); 27573c9168faSHans Rosenfeld 27583c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) 27593c9168faSHans Rosenfeld ddi_regs_map_free(&nvme->n_regh); 27603c9168faSHans Rosenfeld 27613c9168faSHans Rosenfeld if (nvme->n_progress & NVME_FMA_INIT) { 27623c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 27633c9168faSHans Rosenfeld ddi_fm_handler_unregister(nvme->n_dip); 27643c9168faSHans Rosenfeld 27653c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 27663c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 27673c9168faSHans Rosenfeld pci_ereport_teardown(nvme->n_dip); 27683c9168faSHans Rosenfeld 27693c9168faSHans Rosenfeld ddi_fm_fini(nvme->n_dip); 27703c9168faSHans Rosenfeld } 27713c9168faSHans Rosenfeld 2772510a6847SHans Rosenfeld if (nvme->n_vendor != NULL) 2773510a6847SHans Rosenfeld strfree(nvme->n_vendor); 2774510a6847SHans Rosenfeld 2775510a6847SHans Rosenfeld if (nvme->n_product != NULL) 2776510a6847SHans Rosenfeld strfree(nvme->n_product); 2777510a6847SHans Rosenfeld 27783c9168faSHans Rosenfeld ddi_soft_state_free(nvme_state, instance); 27793c9168faSHans Rosenfeld 27803c9168faSHans Rosenfeld return (DDI_SUCCESS); 27813c9168faSHans Rosenfeld } 27823c9168faSHans Rosenfeld 27833c9168faSHans Rosenfeld static int 27843c9168faSHans Rosenfeld nvme_quiesce(dev_info_t *dip) 27853c9168faSHans Rosenfeld { 27863c9168faSHans Rosenfeld int instance; 27873c9168faSHans Rosenfeld nvme_t *nvme; 27883c9168faSHans Rosenfeld 27893c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 27903c9168faSHans Rosenfeld 27913c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 27923c9168faSHans Rosenfeld 27933c9168faSHans Rosenfeld if (nvme == NULL) 27943c9168faSHans Rosenfeld return (DDI_FAILURE); 27953c9168faSHans Rosenfeld 27963c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_ABRUPT, B_TRUE); 27973c9168faSHans Rosenfeld 27983c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_TRUE); 27993c9168faSHans Rosenfeld 28003c9168faSHans Rosenfeld return (DDI_FAILURE); 28013c9168faSHans Rosenfeld } 28023c9168faSHans Rosenfeld 28033c9168faSHans Rosenfeld static int 28043c9168faSHans Rosenfeld nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer) 28053c9168faSHans Rosenfeld { 28063c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 28073c9168faSHans Rosenfeld int nprp_page, nprp; 28083c9168faSHans Rosenfeld uint64_t *prp; 28093c9168faSHans Rosenfeld 28103c9168faSHans Rosenfeld if (xfer->x_ndmac == 0) 28113c9168faSHans Rosenfeld return (DDI_FAILURE); 28123c9168faSHans Rosenfeld 28133c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress; 28143c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 28153c9168faSHans Rosenfeld 28163c9168faSHans Rosenfeld if (xfer->x_ndmac == 1) { 28173c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; 28183c9168faSHans Rosenfeld return (DDI_SUCCESS); 28193c9168faSHans Rosenfeld } else if (xfer->x_ndmac == 2) { 28203c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress; 28213c9168faSHans Rosenfeld return (DDI_SUCCESS); 28223c9168faSHans Rosenfeld } 28233c9168faSHans Rosenfeld 28243c9168faSHans Rosenfeld xfer->x_ndmac--; 28253c9168faSHans Rosenfeld 28263c9168faSHans Rosenfeld nprp_page = nvme->n_pagesize / sizeof (uint64_t) - 1; 28273c9168faSHans Rosenfeld ASSERT(nprp_page > 0); 28283c9168faSHans Rosenfeld nprp = (xfer->x_ndmac + nprp_page - 1) / nprp_page; 28293c9168faSHans Rosenfeld 28303c9168faSHans Rosenfeld /* 28313c9168faSHans Rosenfeld * We currently don't support chained PRPs and set up our DMA 28323c9168faSHans Rosenfeld * attributes to reflect that. If we still get an I/O request 28333c9168faSHans Rosenfeld * that needs a chained PRP something is very wrong. 28343c9168faSHans Rosenfeld */ 28353c9168faSHans Rosenfeld VERIFY(nprp == 1); 28363c9168faSHans Rosenfeld 28378834f7acSYouzhong Yang cmd->nc_dma = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP); 28388834f7acSYouzhong Yang bzero(cmd->nc_dma->nd_memp, cmd->nc_dma->nd_len); 28393c9168faSHans Rosenfeld 28403c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress; 28413c9168faSHans Rosenfeld 28423c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 28433c9168faSHans Rosenfeld for (prp = (uint64_t *)cmd->nc_dma->nd_memp; 28443c9168faSHans Rosenfeld xfer->x_ndmac > 0; 28453c9168faSHans Rosenfeld prp++, xfer->x_ndmac--) { 28463c9168faSHans Rosenfeld *prp = xfer->x_dmac.dmac_laddress; 28473c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 28483c9168faSHans Rosenfeld } 28493c9168faSHans Rosenfeld 28503c9168faSHans Rosenfeld (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len, 28513c9168faSHans Rosenfeld DDI_DMA_SYNC_FORDEV); 28523c9168faSHans Rosenfeld return (DDI_SUCCESS); 28533c9168faSHans Rosenfeld } 28543c9168faSHans Rosenfeld 28553c9168faSHans Rosenfeld static nvme_cmd_t * 28563c9168faSHans Rosenfeld nvme_create_nvm_cmd(nvme_namespace_t *ns, uint8_t opc, bd_xfer_t *xfer) 28573c9168faSHans Rosenfeld { 28583c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 28593c9168faSHans Rosenfeld nvme_cmd_t *cmd; 28603c9168faSHans Rosenfeld 28613c9168faSHans Rosenfeld /* 28623c9168faSHans Rosenfeld * Blkdev only sets BD_XFER_POLL when dumping, so don't sleep. 28633c9168faSHans Rosenfeld */ 28643c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ? 28653c9168faSHans Rosenfeld KM_NOSLEEP : KM_SLEEP); 28663c9168faSHans Rosenfeld 28673c9168faSHans Rosenfeld if (cmd == NULL) 28683c9168faSHans Rosenfeld return (NULL); 28693c9168faSHans Rosenfeld 28703c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = opc; 28713c9168faSHans Rosenfeld cmd->nc_callback = nvme_bd_xfer_done; 28723c9168faSHans Rosenfeld cmd->nc_xfer = xfer; 28733c9168faSHans Rosenfeld 28743c9168faSHans Rosenfeld switch (opc) { 28753c9168faSHans Rosenfeld case NVME_OPC_NVM_WRITE: 28763c9168faSHans Rosenfeld case NVME_OPC_NVM_READ: 28773c9168faSHans Rosenfeld VERIFY(xfer->x_nblks <= 0x10000); 28783c9168faSHans Rosenfeld 28793c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 28803c9168faSHans Rosenfeld 28813c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu; 28823c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32); 28833c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1); 28843c9168faSHans Rosenfeld 28853c9168faSHans Rosenfeld if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS) 28863c9168faSHans Rosenfeld goto fail; 28873c9168faSHans Rosenfeld break; 28883c9168faSHans Rosenfeld 28893c9168faSHans Rosenfeld case NVME_OPC_NVM_FLUSH: 28903c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 28913c9168faSHans Rosenfeld break; 28923c9168faSHans Rosenfeld 28933c9168faSHans Rosenfeld default: 28943c9168faSHans Rosenfeld goto fail; 28953c9168faSHans Rosenfeld } 28963c9168faSHans Rosenfeld 28973c9168faSHans Rosenfeld return (cmd); 28983c9168faSHans Rosenfeld 28993c9168faSHans Rosenfeld fail: 29003c9168faSHans Rosenfeld nvme_free_cmd(cmd); 29013c9168faSHans Rosenfeld return (NULL); 29023c9168faSHans Rosenfeld } 29033c9168faSHans Rosenfeld 29043c9168faSHans Rosenfeld static void 29053c9168faSHans Rosenfeld nvme_bd_xfer_done(void *arg) 29063c9168faSHans Rosenfeld { 29073c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 29083c9168faSHans Rosenfeld bd_xfer_t *xfer = cmd->nc_xfer; 29093c9168faSHans Rosenfeld int error = 0; 29103c9168faSHans Rosenfeld 29113c9168faSHans Rosenfeld error = nvme_check_cmd_status(cmd); 29123c9168faSHans Rosenfeld nvme_free_cmd(cmd); 29133c9168faSHans Rosenfeld 29143c9168faSHans Rosenfeld bd_xfer_done(xfer, error); 29153c9168faSHans Rosenfeld } 29163c9168faSHans Rosenfeld 29173c9168faSHans Rosenfeld static void 29183c9168faSHans Rosenfeld nvme_bd_driveinfo(void *arg, bd_drive_t *drive) 29193c9168faSHans Rosenfeld { 29203c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 29213c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 29223c9168faSHans Rosenfeld 29233c9168faSHans Rosenfeld /* 29243c9168faSHans Rosenfeld * blkdev maintains one queue size per instance (namespace), 29253c9168faSHans Rosenfeld * but all namespace share the I/O queues. 29263c9168faSHans Rosenfeld * TODO: need to figure out a sane default, or use per-NS I/O queues, 29273c9168faSHans Rosenfeld * or change blkdev to handle EAGAIN 29283c9168faSHans Rosenfeld */ 29293c9168faSHans Rosenfeld drive->d_qsize = nvme->n_ioq_count * nvme->n_io_queue_len 29303c9168faSHans Rosenfeld / nvme->n_namespace_count; 29313c9168faSHans Rosenfeld 29323c9168faSHans Rosenfeld /* 29333c9168faSHans Rosenfeld * d_maxxfer is not set, which means the value is taken from the DMA 29343c9168faSHans Rosenfeld * attributes specified to bd_alloc_handle. 29353c9168faSHans Rosenfeld */ 29363c9168faSHans Rosenfeld 29373c9168faSHans Rosenfeld drive->d_removable = B_FALSE; 29383c9168faSHans Rosenfeld drive->d_hotpluggable = B_FALSE; 29393c9168faSHans Rosenfeld 294024979ca3SHans Rosenfeld bcopy(ns->ns_eui64, drive->d_eui64, sizeof (drive->d_eui64)); 29413c9168faSHans Rosenfeld drive->d_target = ns->ns_id; 29423c9168faSHans Rosenfeld drive->d_lun = 0; 2943510a6847SHans Rosenfeld 2944bef9e21aSHans Rosenfeld drive->d_model = nvme->n_idctl->id_model; 2945bef9e21aSHans Rosenfeld drive->d_model_len = sizeof (nvme->n_idctl->id_model); 2946510a6847SHans Rosenfeld drive->d_vendor = nvme->n_vendor; 2947510a6847SHans Rosenfeld drive->d_vendor_len = strlen(nvme->n_vendor); 2948510a6847SHans Rosenfeld drive->d_product = nvme->n_product; 2949510a6847SHans Rosenfeld drive->d_product_len = strlen(nvme->n_product); 2950510a6847SHans Rosenfeld drive->d_serial = nvme->n_idctl->id_serial; 2951510a6847SHans Rosenfeld drive->d_serial_len = sizeof (nvme->n_idctl->id_serial); 2952510a6847SHans Rosenfeld drive->d_revision = nvme->n_idctl->id_fwrev; 2953510a6847SHans Rosenfeld drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev); 29543c9168faSHans Rosenfeld } 29553c9168faSHans Rosenfeld 29563c9168faSHans Rosenfeld static int 29573c9168faSHans Rosenfeld nvme_bd_mediainfo(void *arg, bd_media_t *media) 29583c9168faSHans Rosenfeld { 29593c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 29603c9168faSHans Rosenfeld 29613c9168faSHans Rosenfeld media->m_nblks = ns->ns_block_count; 29623c9168faSHans Rosenfeld media->m_blksize = ns->ns_block_size; 29633c9168faSHans Rosenfeld media->m_readonly = B_FALSE; 29643c9168faSHans Rosenfeld media->m_solidstate = B_TRUE; 29653c9168faSHans Rosenfeld 29663c9168faSHans Rosenfeld media->m_pblksize = ns->ns_best_block_size; 29673c9168faSHans Rosenfeld 29683c9168faSHans Rosenfeld return (0); 29693c9168faSHans Rosenfeld } 29703c9168faSHans Rosenfeld 29713c9168faSHans Rosenfeld static int 29723c9168faSHans Rosenfeld nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc) 29733c9168faSHans Rosenfeld { 29743c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 29753c9168faSHans Rosenfeld nvme_cmd_t *cmd; 29763c9168faSHans Rosenfeld 29773c9168faSHans Rosenfeld if (nvme->n_dead) 29783c9168faSHans Rosenfeld return (EIO); 29793c9168faSHans Rosenfeld 29803c9168faSHans Rosenfeld /* No polling for now */ 29813c9168faSHans Rosenfeld if (xfer->x_flags & BD_XFER_POLL) 29823c9168faSHans Rosenfeld return (EIO); 29833c9168faSHans Rosenfeld 29843c9168faSHans Rosenfeld cmd = nvme_create_nvm_cmd(ns, opc, xfer); 29853c9168faSHans Rosenfeld if (cmd == NULL) 29863c9168faSHans Rosenfeld return (ENOMEM); 29873c9168faSHans Rosenfeld 29883c9168faSHans Rosenfeld cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1; 29893c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); 29903c9168faSHans Rosenfeld 29913c9168faSHans Rosenfeld if (nvme_submit_cmd(nvme->n_ioq[cmd->nc_sqid], cmd) 29923c9168faSHans Rosenfeld != DDI_SUCCESS) 29933c9168faSHans Rosenfeld return (EAGAIN); 29943c9168faSHans Rosenfeld 29953c9168faSHans Rosenfeld return (0); 29963c9168faSHans Rosenfeld } 29973c9168faSHans Rosenfeld 29983c9168faSHans Rosenfeld static int 29993c9168faSHans Rosenfeld nvme_bd_read(void *arg, bd_xfer_t *xfer) 30003c9168faSHans Rosenfeld { 30013c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 30023c9168faSHans Rosenfeld 30033c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_READ)); 30043c9168faSHans Rosenfeld } 30053c9168faSHans Rosenfeld 30063c9168faSHans Rosenfeld static int 30073c9168faSHans Rosenfeld nvme_bd_write(void *arg, bd_xfer_t *xfer) 30083c9168faSHans Rosenfeld { 30093c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 30103c9168faSHans Rosenfeld 30113c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_WRITE)); 30123c9168faSHans Rosenfeld } 30133c9168faSHans Rosenfeld 30143c9168faSHans Rosenfeld static int 30153c9168faSHans Rosenfeld nvme_bd_sync(void *arg, bd_xfer_t *xfer) 30163c9168faSHans Rosenfeld { 30173c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 30183c9168faSHans Rosenfeld 30193c9168faSHans Rosenfeld if (ns->ns_nvme->n_dead) 30203c9168faSHans Rosenfeld return (EIO); 30213c9168faSHans Rosenfeld 30223c9168faSHans Rosenfeld /* 3023d148d46eSHans Rosenfeld * If the volatile write cache is not present or not enabled the FLUSH 3024d148d46eSHans Rosenfeld * command is a no-op, so we can take a shortcut here. 30253c9168faSHans Rosenfeld */ 3026d148d46eSHans Rosenfeld if (!ns->ns_nvme->n_write_cache_present) { 30273c9168faSHans Rosenfeld bd_xfer_done(xfer, ENOTSUP); 30283c9168faSHans Rosenfeld return (0); 30293c9168faSHans Rosenfeld } 30303c9168faSHans Rosenfeld 3031d148d46eSHans Rosenfeld if (!ns->ns_nvme->n_write_cache_enabled) { 3032d148d46eSHans Rosenfeld bd_xfer_done(xfer, 0); 3033d148d46eSHans Rosenfeld return (0); 3034d148d46eSHans Rosenfeld } 3035d148d46eSHans Rosenfeld 30363c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_FLUSH)); 30373c9168faSHans Rosenfeld } 30383c9168faSHans Rosenfeld 30393c9168faSHans Rosenfeld static int 30403c9168faSHans Rosenfeld nvme_bd_devid(void *arg, dev_info_t *devinfo, ddi_devid_t *devid) 30413c9168faSHans Rosenfeld { 30423c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 30433c9168faSHans Rosenfeld 304424979ca3SHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 304524979ca3SHans Rosenfeld if (*(uint64_t *)ns->ns_eui64 != 0) { 304624979ca3SHans Rosenfeld return (ddi_devid_init(devinfo, DEVID_SCSI3_WWN, 304724979ca3SHans Rosenfeld sizeof (ns->ns_eui64), ns->ns_eui64, devid)); 304824979ca3SHans Rosenfeld } else { 304924979ca3SHans Rosenfeld return (ddi_devid_init(devinfo, DEVID_ENCAP, 305024979ca3SHans Rosenfeld strlen(ns->ns_devid), ns->ns_devid, devid)); 305124979ca3SHans Rosenfeld } 30523c9168faSHans Rosenfeld } 3053