13c9168faSHans Rosenfeld /* 23c9168faSHans Rosenfeld * This file and its contents are supplied under the terms of the 33c9168faSHans Rosenfeld * Common Development and Distribution License ("CDDL"), version 1.0. 43c9168faSHans Rosenfeld * You may only use this file in accordance with the terms of version 53c9168faSHans Rosenfeld * 1.0 of the CDDL. 63c9168faSHans Rosenfeld * 73c9168faSHans Rosenfeld * A full copy of the text of the CDDL should have accompanied this 83c9168faSHans Rosenfeld * source. A copy of the CDDL is also available via the Internet at 93c9168faSHans Rosenfeld * http://www.illumos.org/license/CDDL. 103c9168faSHans Rosenfeld */ 113c9168faSHans Rosenfeld 123c9168faSHans Rosenfeld /* 13f313c178SYuri Pankov * Copyright 2018 Nexenta Systems, Inc. 1434c938c7SPete Shephard * Copyright 2016 Tegile Systems, Inc. All rights reserved. 152f95345bSYouzhong Yang * Copyright (c) 2016 The MathWorks, Inc. All rights reserved. 1648d370f1SRobert Mustacchi * Copyright 2018 Joyent, Inc. 17*0999c112SPaul Winder * Copyright 2019 Western Digital Corporation. 183c9168faSHans Rosenfeld */ 193c9168faSHans Rosenfeld 203c9168faSHans Rosenfeld /* 213c9168faSHans Rosenfeld * blkdev driver for NVMe compliant storage devices 223c9168faSHans Rosenfeld * 23f8dcaea5SRobert Mustacchi * This driver was written to conform to version 1.2.1 of the NVMe 24f8dcaea5SRobert Mustacchi * specification. It may work with newer versions, but that is completely 25f8dcaea5SRobert Mustacchi * untested and disabled by default. 263c9168faSHans Rosenfeld * 273c9168faSHans Rosenfeld * The driver has only been tested on x86 systems and will not work on big- 283c9168faSHans Rosenfeld * endian systems without changes to the code accessing registers and data 293c9168faSHans Rosenfeld * structures used by the hardware. 303c9168faSHans Rosenfeld * 313c9168faSHans Rosenfeld * 323c9168faSHans Rosenfeld * Interrupt Usage: 333c9168faSHans Rosenfeld * 340b466603SHans Rosenfeld * The driver will use a single interrupt while configuring the device as the 350b466603SHans Rosenfeld * specification requires, but contrary to the specification it will try to use 360b466603SHans Rosenfeld * a single-message MSI(-X) or FIXED interrupt. Later in the attach process it 370b466603SHans Rosenfeld * will switch to multiple-message MSI(-X) if supported. The driver wants to 380b466603SHans Rosenfeld * have one interrupt vector per CPU, but it will work correctly if less are 390b466603SHans Rosenfeld * available. Interrupts can be shared by queues, the interrupt handler will 400b466603SHans Rosenfeld * iterate through the I/O queue array by steps of n_intr_cnt. Usually only 410b466603SHans Rosenfeld * the admin queue will share an interrupt with one I/O queue. The interrupt 420b466603SHans Rosenfeld * handler will retrieve completed commands from all queues sharing an interrupt 430b466603SHans Rosenfeld * vector and will post them to a taskq for completion processing. 443c9168faSHans Rosenfeld * 453c9168faSHans Rosenfeld * 463c9168faSHans Rosenfeld * Command Processing: 473c9168faSHans Rosenfeld * 484b324362SHans Rosenfeld * NVMe devices can have up to 65535 I/O queue pairs, with each queue holding up 493c9168faSHans Rosenfeld * to 65536 I/O commands. The driver will configure one I/O queue pair per 503c9168faSHans Rosenfeld * available interrupt vector, with the queue length usually much smaller than 513c9168faSHans Rosenfeld * the maximum of 65536. If the hardware doesn't provide enough queues, fewer 523c9168faSHans Rosenfeld * interrupt vectors will be used. 533c9168faSHans Rosenfeld * 543c9168faSHans Rosenfeld * Additionally the hardware provides a single special admin queue pair that can 553c9168faSHans Rosenfeld * hold up to 4096 admin commands. 563c9168faSHans Rosenfeld * 573c9168faSHans Rosenfeld * From the hardware perspective both queues of a queue pair are independent, 583c9168faSHans Rosenfeld * but they share some driver state: the command array (holding pointers to 593c9168faSHans Rosenfeld * commands currently being processed by the hardware) and the active command 60*0999c112SPaul Winder * counter. Access to a submission queue and the shared state is protected by 61*0999c112SPaul Winder * nq_mutex, completion queue is protected by ncq_mutex. 623c9168faSHans Rosenfeld * 633c9168faSHans Rosenfeld * When a command is submitted to a queue pair the active command counter is 643c9168faSHans Rosenfeld * incremented and a pointer to the command is stored in the command array. The 653c9168faSHans Rosenfeld * array index is used as command identifier (CID) in the submission queue 663c9168faSHans Rosenfeld * entry. Some commands may take a very long time to complete, and if the queue 673c9168faSHans Rosenfeld * wraps around in that time a submission may find the next array slot to still 683c9168faSHans Rosenfeld * be used by a long-running command. In this case the array is sequentially 693c9168faSHans Rosenfeld * searched for the next free slot. The length of the command array is the same 704b324362SHans Rosenfeld * as the configured queue length. Queue overrun is prevented by the semaphore, 714b324362SHans Rosenfeld * so a command submission may block if the queue is full. 723c9168faSHans Rosenfeld * 733c9168faSHans Rosenfeld * 744ac9cfccSHans Rosenfeld * Polled I/O Support: 754ac9cfccSHans Rosenfeld * 764ac9cfccSHans Rosenfeld * For kernel core dump support the driver can do polled I/O. As interrupts are 774ac9cfccSHans Rosenfeld * turned off while dumping the driver will just submit a command in the regular 784ac9cfccSHans Rosenfeld * way, and then repeatedly attempt a command retrieval until it gets the 794ac9cfccSHans Rosenfeld * command back. 804ac9cfccSHans Rosenfeld * 814ac9cfccSHans Rosenfeld * 823c9168faSHans Rosenfeld * Namespace Support: 833c9168faSHans Rosenfeld * 843c9168faSHans Rosenfeld * NVMe devices can have multiple namespaces, each being a independent data 853c9168faSHans Rosenfeld * store. The driver supports multiple namespaces and creates a blkdev interface 863c9168faSHans Rosenfeld * for each namespace found. Namespaces can have various attributes to support 87db083a4dSYuri Pankov * protection information. This driver does not support any of this and ignores 88db083a4dSYuri Pankov * namespaces that have these attributes. 893c9168faSHans Rosenfeld * 9024979ca3SHans Rosenfeld * As of NVMe 1.1 namespaces can have an 64bit Extended Unique Identifier 9124979ca3SHans Rosenfeld * (EUI64). This driver uses the EUI64 if present to generate the devid and 9224979ca3SHans Rosenfeld * passes it to blkdev to use it in the device node names. As this is currently 9324979ca3SHans Rosenfeld * untested namespaces with EUI64 are ignored by default. 9424979ca3SHans Rosenfeld * 953d9b1a2aSHans Rosenfeld * We currently support only (2 << NVME_MINOR_INST_SHIFT) - 2 namespaces in a 963d9b1a2aSHans Rosenfeld * single controller. This is an artificial limit imposed by the driver to be 973d9b1a2aSHans Rosenfeld * able to address a reasonable number of controllers and namespaces using a 983d9b1a2aSHans Rosenfeld * 32bit minor node number. 993d9b1a2aSHans Rosenfeld * 1003d9b1a2aSHans Rosenfeld * 1013d9b1a2aSHans Rosenfeld * Minor nodes: 1023d9b1a2aSHans Rosenfeld * 1033d9b1a2aSHans Rosenfeld * For each NVMe device the driver exposes one minor node for the controller and 1043d9b1a2aSHans Rosenfeld * one minor node for each namespace. The only operations supported by those 1053d9b1a2aSHans Rosenfeld * minor nodes are open(9E), close(9E), and ioctl(9E). This serves as the 1063d9b1a2aSHans Rosenfeld * interface for the nvmeadm(1M) utility. 1073d9b1a2aSHans Rosenfeld * 1083c9168faSHans Rosenfeld * 1093c9168faSHans Rosenfeld * Blkdev Interface: 1103c9168faSHans Rosenfeld * 1113c9168faSHans Rosenfeld * This driver uses blkdev to do all the heavy lifting involved with presenting 1123c9168faSHans Rosenfeld * a disk device to the system. As a result, the processing of I/O requests is 1133c9168faSHans Rosenfeld * relatively simple as blkdev takes care of partitioning, boundary checks, DMA 1143c9168faSHans Rosenfeld * setup, and splitting of transfers into manageable chunks. 1153c9168faSHans Rosenfeld * 1163c9168faSHans Rosenfeld * I/O requests coming in from blkdev are turned into NVM commands and posted to 1173c9168faSHans Rosenfeld * an I/O queue. The queue is selected by taking the CPU id modulo the number of 1183c9168faSHans Rosenfeld * queues. There is currently no timeout handling of I/O commands. 1193c9168faSHans Rosenfeld * 1203c9168faSHans Rosenfeld * Blkdev also supports querying device/media information and generating a 1213c9168faSHans Rosenfeld * devid. The driver reports the best block size as determined by the namespace 1223c9168faSHans Rosenfeld * format back to blkdev as physical block size to support partition and block 12324979ca3SHans Rosenfeld * alignment. The devid is either based on the namespace EUI64, if present, or 12424979ca3SHans Rosenfeld * composed using the device vendor ID, model number, serial number, and the 12524979ca3SHans Rosenfeld * namespace ID. 1263c9168faSHans Rosenfeld * 1273c9168faSHans Rosenfeld * 1283c9168faSHans Rosenfeld * Error Handling: 1293c9168faSHans Rosenfeld * 1303c9168faSHans Rosenfeld * Error handling is currently limited to detecting fatal hardware errors, 1313c9168faSHans Rosenfeld * either by asynchronous events, or synchronously through command status or 1323c9168faSHans Rosenfeld * admin command timeouts. In case of severe errors the device is fenced off, 1333c9168faSHans Rosenfeld * all further requests will return EIO. FMA is then called to fault the device. 1343c9168faSHans Rosenfeld * 1353c9168faSHans Rosenfeld * The hardware has a limit for outstanding asynchronous event requests. Before 1363c9168faSHans Rosenfeld * this limit is known the driver assumes it is at least 1 and posts a single 1373c9168faSHans Rosenfeld * asynchronous request. Later when the limit is known more asynchronous event 1383c9168faSHans Rosenfeld * requests are posted to allow quicker reception of error information. When an 1393c9168faSHans Rosenfeld * asynchronous event is posted by the hardware the driver will parse the error 1403c9168faSHans Rosenfeld * status fields and log information or fault the device, depending on the 1413c9168faSHans Rosenfeld * severity of the asynchronous event. The asynchronous event request is then 1423c9168faSHans Rosenfeld * reused and posted to the admin queue again. 1433c9168faSHans Rosenfeld * 1443c9168faSHans Rosenfeld * On command completion the command status is checked for errors. In case of 1453c9168faSHans Rosenfeld * errors indicating a driver bug the driver panics. Almost all other error 1463c9168faSHans Rosenfeld * status values just cause EIO to be returned. 1473c9168faSHans Rosenfeld * 1483c9168faSHans Rosenfeld * Command timeouts are currently detected for all admin commands except 1493c9168faSHans Rosenfeld * asynchronous event requests. If a command times out and the hardware appears 150e984c70bSHans Rosenfeld * to be healthy the driver attempts to abort the command. The original command 151e984c70bSHans Rosenfeld * timeout is also applied to the abort command. If the abort times out too the 1523c9168faSHans Rosenfeld * driver assumes the device to be dead, fences it off, and calls FMA to retire 153e984c70bSHans Rosenfeld * it. In all other cases the aborted command should return immediately with a 154e984c70bSHans Rosenfeld * status indicating it was aborted, and the driver will wait indefinitely for 155e984c70bSHans Rosenfeld * that to happen. No timeout handling of normal I/O commands is presently done. 1563c9168faSHans Rosenfeld * 157e984c70bSHans Rosenfeld * Any command that times out due to the controller dropping dead will be put on 158e984c70bSHans Rosenfeld * nvme_lost_cmds list if it references DMA memory. This will prevent the DMA 159e984c70bSHans Rosenfeld * memory being reused by the system and later be written to by a "dead" NVMe 160e984c70bSHans Rosenfeld * controller. 161e984c70bSHans Rosenfeld * 162e984c70bSHans Rosenfeld * 163e984c70bSHans Rosenfeld * Locking: 164e984c70bSHans Rosenfeld * 165*0999c112SPaul Winder * Each queue pair has a nq_mutex and ncq_mutex. The nq_mutex must be held 166*0999c112SPaul Winder * when accessing shared state and submission queue registers, ncq_mutex 167*0999c112SPaul Winder * is held when accessing completion queue state and registers. 168*0999c112SPaul Winder * Callers of nvme_unqueue_cmd() must make sure that nq_mutex is held, while 169*0999c112SPaul Winder * nvme_submit_{admin,io}_cmd() and nvme_retrieve_cmd() take care of both 170*0999c112SPaul Winder * mutexes themselves. 171e984c70bSHans Rosenfeld * 172e984c70bSHans Rosenfeld * Each command also has its own nc_mutex, which is associated with the 173e984c70bSHans Rosenfeld * condition variable nc_cv. It is only used on admin commands which are run 174e984c70bSHans Rosenfeld * synchronously. In that case it must be held across calls to 175e984c70bSHans Rosenfeld * nvme_submit_{admin,io}_cmd() and nvme_wait_cmd(), which is taken care of by 176e984c70bSHans Rosenfeld * nvme_admin_cmd(). It must also be held whenever the completion state of the 177e984c70bSHans Rosenfeld * command is changed or while a admin command timeout is handled. 178e984c70bSHans Rosenfeld * 179e984c70bSHans Rosenfeld * If both nc_mutex and nq_mutex must be held, nc_mutex must be acquired first. 180e984c70bSHans Rosenfeld * More than one nc_mutex may only be held when aborting commands. In this case, 181e984c70bSHans Rosenfeld * the nc_mutex of the command to be aborted must be held across the call to 182e984c70bSHans Rosenfeld * nvme_abort_cmd() to prevent the command from completing while the abort is in 183e984c70bSHans Rosenfeld * progress. 184e984c70bSHans Rosenfeld * 185*0999c112SPaul Winder * If both nq_mutex and ncq_mutex need to be held, ncq_mutex must be 186*0999c112SPaul Winder * acquired first. More than one nq_mutex is never held by a single thread. 187*0999c112SPaul Winder * The ncq_mutex is only held by nvme_retrieve_cmd() and 188*0999c112SPaul Winder * nvme_process_iocq(). nvme_process_iocq() is only called from the 189*0999c112SPaul Winder * interrupt thread and nvme_retrieve_cmd() during polled I/O, so the 190*0999c112SPaul Winder * mutex is non-contentious but is required for implementation completeness 191*0999c112SPaul Winder * and safety. 192*0999c112SPaul Winder * 193e984c70bSHans Rosenfeld * Each minor node has its own nm_mutex, which protects the open count nm_ocnt 194e984c70bSHans Rosenfeld * and exclusive-open flag nm_oexcl. 1953c9168faSHans Rosenfeld * 1963c9168faSHans Rosenfeld * 1973c9168faSHans Rosenfeld * Quiesce / Fast Reboot: 1983c9168faSHans Rosenfeld * 1993c9168faSHans Rosenfeld * The driver currently does not support fast reboot. A quiesce(9E) entry point 2003c9168faSHans Rosenfeld * is still provided which is used to send a shutdown notification to the 2013c9168faSHans Rosenfeld * device. 2023c9168faSHans Rosenfeld * 2033c9168faSHans Rosenfeld * 2043c9168faSHans Rosenfeld * Driver Configuration: 2053c9168faSHans Rosenfeld * 2063c9168faSHans Rosenfeld * The following driver properties can be changed to control some aspects of the 2073c9168faSHans Rosenfeld * drivers operation: 2083c9168faSHans Rosenfeld * - strict-version: can be set to 0 to allow devices conforming to newer 20948d370f1SRobert Mustacchi * major versions to be used 2103c9168faSHans Rosenfeld * - ignore-unknown-vendor-status: can be set to 1 to not handle any vendor 2113c9168faSHans Rosenfeld * specific command status as a fatal error leading device faulting 2123c9168faSHans Rosenfeld * - admin-queue-len: the maximum length of the admin queue (16-4096) 213*0999c112SPaul Winder * - io-squeue-len: the maximum length of the I/O submission queues (16-65536) 214*0999c112SPaul Winder * - io-cqueue-len: the maximum length of the I/O completion queues (16-65536) 2153c9168faSHans Rosenfeld * - async-event-limit: the maximum number of asynchronous event requests to be 2163c9168faSHans Rosenfeld * posted by the driver 217d148d46eSHans Rosenfeld * - volatile-write-cache-enable: can be set to 0 to disable the volatile write 218d148d46eSHans Rosenfeld * cache 2196801591eSHans Rosenfeld * - min-phys-block-size: the minimum physical block size to report to blkdev, 2206801591eSHans Rosenfeld * which is among other things the basis for ZFS vdev ashift 221*0999c112SPaul Winder * - max-submission-queues: the maximum number of I/O submission queues. 222*0999c112SPaul Winder * - max-completion-queues: the maximum number of I/O completion queues, 223*0999c112SPaul Winder * can be less than max-submission-queues, in which case the completion 224*0999c112SPaul Winder * queues are shared. 2253c9168faSHans Rosenfeld * 2263c9168faSHans Rosenfeld * 2273c9168faSHans Rosenfeld * TODO: 2283c9168faSHans Rosenfeld * - figure out sane default for I/O queue depth reported to blkdev 2293c9168faSHans Rosenfeld * - FMA handling of media errors 2303c9168faSHans Rosenfeld * - support for devices supporting very large I/O requests using chained PRPs 2313c9168faSHans Rosenfeld * - support for configuring hardware parameters like interrupt coalescing 2323c9168faSHans Rosenfeld * - support for media formatting and hard partitioning into namespaces 2333c9168faSHans Rosenfeld * - support for big-endian systems 2343c9168faSHans Rosenfeld * - support for fast reboot 23524979ca3SHans Rosenfeld * - support for firmware updates 23624979ca3SHans Rosenfeld * - support for NVMe Subsystem Reset (1.1) 23724979ca3SHans Rosenfeld * - support for Scatter/Gather lists (1.1) 23824979ca3SHans Rosenfeld * - support for Reservations (1.1) 23924979ca3SHans Rosenfeld * - support for power management 2403c9168faSHans Rosenfeld */ 2413c9168faSHans Rosenfeld 2423c9168faSHans Rosenfeld #include <sys/byteorder.h> 2433c9168faSHans Rosenfeld #ifdef _BIG_ENDIAN 2443c9168faSHans Rosenfeld #error nvme driver needs porting for big-endian platforms 2453c9168faSHans Rosenfeld #endif 2463c9168faSHans Rosenfeld 2473c9168faSHans Rosenfeld #include <sys/modctl.h> 2483c9168faSHans Rosenfeld #include <sys/conf.h> 2493c9168faSHans Rosenfeld #include <sys/devops.h> 2503c9168faSHans Rosenfeld #include <sys/ddi.h> 2513c9168faSHans Rosenfeld #include <sys/sunddi.h> 2523d9b1a2aSHans Rosenfeld #include <sys/sunndi.h> 2533c9168faSHans Rosenfeld #include <sys/bitmap.h> 2543c9168faSHans Rosenfeld #include <sys/sysmacros.h> 2553c9168faSHans Rosenfeld #include <sys/param.h> 2563c9168faSHans Rosenfeld #include <sys/varargs.h> 2573c9168faSHans Rosenfeld #include <sys/cpuvar.h> 2583c9168faSHans Rosenfeld #include <sys/disp.h> 2593c9168faSHans Rosenfeld #include <sys/blkdev.h> 2603c9168faSHans Rosenfeld #include <sys/atomic.h> 2613c9168faSHans Rosenfeld #include <sys/archsystm.h> 262510a6847SHans Rosenfeld #include <sys/sata/sata_hba.h> 2633d9b1a2aSHans Rosenfeld #include <sys/stat.h> 2643d9b1a2aSHans Rosenfeld #include <sys/policy.h> 265e984c70bSHans Rosenfeld #include <sys/list.h> 2663d9b1a2aSHans Rosenfeld 2673d9b1a2aSHans Rosenfeld #include <sys/nvme.h> 2683c9168faSHans Rosenfeld 2699d08e1f8SHans Rosenfeld #ifdef __x86 2709d08e1f8SHans Rosenfeld #include <sys/x86_archext.h> 2719d08e1f8SHans Rosenfeld #endif 2729d08e1f8SHans Rosenfeld 2733c9168faSHans Rosenfeld #include "nvme_reg.h" 2743c9168faSHans Rosenfeld #include "nvme_var.h" 2753c9168faSHans Rosenfeld 27648d370f1SRobert Mustacchi /* 27748d370f1SRobert Mustacchi * Assertions to make sure that we've properly captured various aspects of the 27848d370f1SRobert Mustacchi * packed structures and haven't broken them during updates. 27948d370f1SRobert Mustacchi */ 28048d370f1SRobert Mustacchi CTASSERT(sizeof (nvme_identify_ctrl_t) == 0x1000); 28148d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_oacs) == 256); 28248d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_sqes) == 512); 28348d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_subnqn) == 768); 28448d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_nvmof) == 1792); 28548d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_psd) == 2048); 28648d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_vs) == 3072); 28748d370f1SRobert Mustacchi 28848d370f1SRobert Mustacchi CTASSERT(sizeof (nvme_identify_nsid_t) == 0x1000); 28948d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_nsid_t, id_fpi) == 32); 29048d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_nsid_t, id_nguid) == 104); 29148d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_nsid_t, id_lbaf) == 128); 29248d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_nsid_t, id_vs) == 384); 29348d370f1SRobert Mustacchi 29448d370f1SRobert Mustacchi CTASSERT(sizeof (nvme_identify_primary_caps_t) == 0x1000); 29548d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_primary_caps_t, nipc_vqfrt) == 32); 29648d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_primary_caps_t, nipc_vifrt) == 64); 29748d370f1SRobert Mustacchi 2983c9168faSHans Rosenfeld 2993c9168faSHans Rosenfeld /* NVMe spec version supported */ 3003c9168faSHans Rosenfeld static const int nvme_version_major = 1; 3013c9168faSHans Rosenfeld 302e8ba2a38SHans Rosenfeld /* tunable for admin command timeout in seconds, default is 1s */ 3033d9b1a2aSHans Rosenfeld int nvme_admin_cmd_timeout = 1; 3043d9b1a2aSHans Rosenfeld 3053d9b1a2aSHans Rosenfeld /* tunable for FORMAT NVM command timeout in seconds, default is 600s */ 3063d9b1a2aSHans Rosenfeld int nvme_format_cmd_timeout = 600; 307e8ba2a38SHans Rosenfeld 3083c9168faSHans Rosenfeld static int nvme_attach(dev_info_t *, ddi_attach_cmd_t); 3093c9168faSHans Rosenfeld static int nvme_detach(dev_info_t *, ddi_detach_cmd_t); 3103c9168faSHans Rosenfeld static int nvme_quiesce(dev_info_t *); 3113c9168faSHans Rosenfeld static int nvme_fm_errcb(dev_info_t *, ddi_fm_error_t *, const void *); 3123c9168faSHans Rosenfeld static int nvme_setup_interrupts(nvme_t *, int, int); 3133c9168faSHans Rosenfeld static void nvme_release_interrupts(nvme_t *); 3143c9168faSHans Rosenfeld static uint_t nvme_intr(caddr_t, caddr_t); 3153c9168faSHans Rosenfeld 3163c9168faSHans Rosenfeld static void nvme_shutdown(nvme_t *, int, boolean_t); 3173c9168faSHans Rosenfeld static boolean_t nvme_reset(nvme_t *, boolean_t); 3183c9168faSHans Rosenfeld static int nvme_init(nvme_t *); 3193c9168faSHans Rosenfeld static nvme_cmd_t *nvme_alloc_cmd(nvme_t *, int); 3203c9168faSHans Rosenfeld static void nvme_free_cmd(nvme_cmd_t *); 3213c9168faSHans Rosenfeld static nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t, 3223c9168faSHans Rosenfeld bd_xfer_t *); 323e984c70bSHans Rosenfeld static void nvme_admin_cmd(nvme_cmd_t *, int); 3244b324362SHans Rosenfeld static void nvme_submit_admin_cmd(nvme_qpair_t *, nvme_cmd_t *); 3254b324362SHans Rosenfeld static int nvme_submit_io_cmd(nvme_qpair_t *, nvme_cmd_t *); 3264b324362SHans Rosenfeld static void nvme_submit_cmd_common(nvme_qpair_t *, nvme_cmd_t *); 327e984c70bSHans Rosenfeld static nvme_cmd_t *nvme_unqueue_cmd(nvme_t *, nvme_qpair_t *, int); 3283c9168faSHans Rosenfeld static nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *); 329e984c70bSHans Rosenfeld static void nvme_wait_cmd(nvme_cmd_t *, uint_t); 3303c9168faSHans Rosenfeld static void nvme_wakeup_cmd(void *); 3313c9168faSHans Rosenfeld static void nvme_async_event_task(void *); 3323c9168faSHans Rosenfeld 3333c9168faSHans Rosenfeld static int nvme_check_unknown_cmd_status(nvme_cmd_t *); 3343c9168faSHans Rosenfeld static int nvme_check_vendor_cmd_status(nvme_cmd_t *); 3353c9168faSHans Rosenfeld static int nvme_check_integrity_cmd_status(nvme_cmd_t *); 3363c9168faSHans Rosenfeld static int nvme_check_specific_cmd_status(nvme_cmd_t *); 3373c9168faSHans Rosenfeld static int nvme_check_generic_cmd_status(nvme_cmd_t *); 3383c9168faSHans Rosenfeld static inline int nvme_check_cmd_status(nvme_cmd_t *); 3393c9168faSHans Rosenfeld 340e984c70bSHans Rosenfeld static int nvme_abort_cmd(nvme_cmd_t *, uint_t); 3414b324362SHans Rosenfeld static void nvme_async_event(nvme_t *); 342bc586359SRobert Mustacchi static int nvme_format_nvm(nvme_t *, boolean_t, uint32_t, uint8_t, boolean_t, 343bc586359SRobert Mustacchi uint8_t, boolean_t, uint8_t); 344bc586359SRobert Mustacchi static int nvme_get_logpage(nvme_t *, boolean_t, void **, size_t *, uint8_t, 345bc586359SRobert Mustacchi ...); 346bc586359SRobert Mustacchi static int nvme_identify(nvme_t *, boolean_t, uint32_t, void **); 347bc586359SRobert Mustacchi static int nvme_set_features(nvme_t *, boolean_t, uint32_t, uint8_t, uint32_t, 348d148d46eSHans Rosenfeld uint32_t *); 349bc586359SRobert Mustacchi static int nvme_get_features(nvme_t *, boolean_t, uint32_t, uint8_t, uint32_t *, 3503d9b1a2aSHans Rosenfeld void **, size_t *); 351e984c70bSHans Rosenfeld static int nvme_write_cache_set(nvme_t *, boolean_t); 352*0999c112SPaul Winder static int nvme_set_nqueues(nvme_t *); 3533c9168faSHans Rosenfeld 3543c9168faSHans Rosenfeld static void nvme_free_dma(nvme_dma_t *); 3553c9168faSHans Rosenfeld static int nvme_zalloc_dma(nvme_t *, size_t, uint_t, ddi_dma_attr_t *, 3563c9168faSHans Rosenfeld nvme_dma_t **); 3573c9168faSHans Rosenfeld static int nvme_zalloc_queue_dma(nvme_t *, uint32_t, uint16_t, uint_t, 3583c9168faSHans Rosenfeld nvme_dma_t **); 3593c9168faSHans Rosenfeld static void nvme_free_qpair(nvme_qpair_t *); 360*0999c112SPaul Winder static int nvme_alloc_qpair(nvme_t *, uint32_t, nvme_qpair_t **, uint_t); 3613c9168faSHans Rosenfeld static int nvme_create_io_qpair(nvme_t *, nvme_qpair_t *, uint16_t); 3623c9168faSHans Rosenfeld 3633c9168faSHans Rosenfeld static inline void nvme_put64(nvme_t *, uintptr_t, uint64_t); 3643c9168faSHans Rosenfeld static inline void nvme_put32(nvme_t *, uintptr_t, uint32_t); 3653c9168faSHans Rosenfeld static inline uint64_t nvme_get64(nvme_t *, uintptr_t); 3663c9168faSHans Rosenfeld static inline uint32_t nvme_get32(nvme_t *, uintptr_t); 3673c9168faSHans Rosenfeld 3683c9168faSHans Rosenfeld static boolean_t nvme_check_regs_hdl(nvme_t *); 3693c9168faSHans Rosenfeld static boolean_t nvme_check_dma_hdl(nvme_dma_t *); 3703c9168faSHans Rosenfeld 3713c9168faSHans Rosenfeld static int nvme_fill_prp(nvme_cmd_t *, bd_xfer_t *); 3723c9168faSHans Rosenfeld 3733c9168faSHans Rosenfeld static void nvme_bd_xfer_done(void *); 3743c9168faSHans Rosenfeld static void nvme_bd_driveinfo(void *, bd_drive_t *); 3753c9168faSHans Rosenfeld static int nvme_bd_mediainfo(void *, bd_media_t *); 3763c9168faSHans Rosenfeld static int nvme_bd_cmd(nvme_namespace_t *, bd_xfer_t *, uint8_t); 3773c9168faSHans Rosenfeld static int nvme_bd_read(void *, bd_xfer_t *); 3783c9168faSHans Rosenfeld static int nvme_bd_write(void *, bd_xfer_t *); 3793c9168faSHans Rosenfeld static int nvme_bd_sync(void *, bd_xfer_t *); 3803c9168faSHans Rosenfeld static int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *); 3813c9168faSHans Rosenfeld 3828834f7acSYouzhong Yang static int nvme_prp_dma_constructor(void *, void *, int); 3838834f7acSYouzhong Yang static void nvme_prp_dma_destructor(void *, void *); 3848834f7acSYouzhong Yang 3853c9168faSHans Rosenfeld static void nvme_prepare_devid(nvme_t *, uint32_t); 3863c9168faSHans Rosenfeld 3873d9b1a2aSHans Rosenfeld static int nvme_open(dev_t *, int, int, cred_t *); 3883d9b1a2aSHans Rosenfeld static int nvme_close(dev_t, int, int, cred_t *); 3893d9b1a2aSHans Rosenfeld static int nvme_ioctl(dev_t, int, intptr_t, int, cred_t *, int *); 3903d9b1a2aSHans Rosenfeld 391dc97a43dSHans Rosenfeld #define NVME_MINOR_INST_SHIFT 9 3923d9b1a2aSHans Rosenfeld #define NVME_MINOR(inst, nsid) (((inst) << NVME_MINOR_INST_SHIFT) | (nsid)) 3933d9b1a2aSHans Rosenfeld #define NVME_MINOR_INST(minor) ((minor) >> NVME_MINOR_INST_SHIFT) 3943d9b1a2aSHans Rosenfeld #define NVME_MINOR_NSID(minor) ((minor) & ((1 << NVME_MINOR_INST_SHIFT) - 1)) 3953d9b1a2aSHans Rosenfeld #define NVME_MINOR_MAX (NVME_MINOR(1, 0) - 2) 3963d9b1a2aSHans Rosenfeld 3973c9168faSHans Rosenfeld static void *nvme_state; 3983c9168faSHans Rosenfeld static kmem_cache_t *nvme_cmd_cache; 3993c9168faSHans Rosenfeld 4003c9168faSHans Rosenfeld /* 4013c9168faSHans Rosenfeld * DMA attributes for queue DMA memory 4023c9168faSHans Rosenfeld * 4033c9168faSHans Rosenfeld * Queue DMA memory must be page aligned. The maximum length of a queue is 4043c9168faSHans Rosenfeld * 65536 entries, and an entry can be 64 bytes long. 4053c9168faSHans Rosenfeld */ 4063c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_queue_dma_attr = { 4073c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 4083c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 4093c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 410910f0d12SYouzhong Yang .dma_attr_count_max = (UINT16_MAX + 1) * sizeof (nvme_sqe_t) - 1, 4113c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 4123c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 4133c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 4143c9168faSHans Rosenfeld .dma_attr_maxxfer = (UINT16_MAX + 1) * sizeof (nvme_sqe_t), 4153c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 4163c9168faSHans Rosenfeld .dma_attr_sgllen = 1, 4173c9168faSHans Rosenfeld .dma_attr_granular = 1, 4183c9168faSHans Rosenfeld .dma_attr_flags = 0, 4193c9168faSHans Rosenfeld }; 4203c9168faSHans Rosenfeld 4213c9168faSHans Rosenfeld /* 4223c9168faSHans Rosenfeld * DMA attributes for transfers using Physical Region Page (PRP) entries 4233c9168faSHans Rosenfeld * 4243c9168faSHans Rosenfeld * A PRP entry describes one page of DMA memory using the page size specified 4253c9168faSHans Rosenfeld * in the controller configuration's memory page size register (CC.MPS). It uses 4263c9168faSHans Rosenfeld * a 64bit base address aligned to this page size. There is no limitation on 4273c9168faSHans Rosenfeld * chaining PRPs together for arbitrarily large DMA transfers. 4283c9168faSHans Rosenfeld */ 4293c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_prp_dma_attr = { 4303c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 4313c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 4323c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 4333c9168faSHans Rosenfeld .dma_attr_count_max = 0xfff, 4343c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 4353c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 4363c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 4373c9168faSHans Rosenfeld .dma_attr_maxxfer = 0x1000, 4382f95345bSYouzhong Yang .dma_attr_seg = 0xfff, 4393c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 4403c9168faSHans Rosenfeld .dma_attr_granular = 1, 4413c9168faSHans Rosenfeld .dma_attr_flags = 0, 4423c9168faSHans Rosenfeld }; 4433c9168faSHans Rosenfeld 4443c9168faSHans Rosenfeld /* 4453c9168faSHans Rosenfeld * DMA attributes for transfers using scatter/gather lists 4463c9168faSHans Rosenfeld * 4473c9168faSHans Rosenfeld * A SGL entry describes a chunk of DMA memory using a 64bit base address and a 4483c9168faSHans Rosenfeld * 32bit length field. SGL Segment and SGL Last Segment entries require the 4493c9168faSHans Rosenfeld * length to be a multiple of 16 bytes. 4503c9168faSHans Rosenfeld */ 4513c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_sgl_dma_attr = { 4523c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 4533c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 4543c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 4553c9168faSHans Rosenfeld .dma_attr_count_max = 0xffffffffUL, 4563c9168faSHans Rosenfeld .dma_attr_align = 1, 4573c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 4583c9168faSHans Rosenfeld .dma_attr_minxfer = 0x10, 4593c9168faSHans Rosenfeld .dma_attr_maxxfer = 0xfffffffffULL, 4603c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 4613c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 4623c9168faSHans Rosenfeld .dma_attr_granular = 0x10, 4633c9168faSHans Rosenfeld .dma_attr_flags = 0 4643c9168faSHans Rosenfeld }; 4653c9168faSHans Rosenfeld 4663c9168faSHans Rosenfeld static ddi_device_acc_attr_t nvme_reg_acc_attr = { 4673c9168faSHans Rosenfeld .devacc_attr_version = DDI_DEVICE_ATTR_V0, 4683c9168faSHans Rosenfeld .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 4693c9168faSHans Rosenfeld .devacc_attr_dataorder = DDI_STRICTORDER_ACC 4703c9168faSHans Rosenfeld }; 4713c9168faSHans Rosenfeld 4723d9b1a2aSHans Rosenfeld static struct cb_ops nvme_cb_ops = { 4733d9b1a2aSHans Rosenfeld .cb_open = nvme_open, 4743d9b1a2aSHans Rosenfeld .cb_close = nvme_close, 4753d9b1a2aSHans Rosenfeld .cb_strategy = nodev, 4763d9b1a2aSHans Rosenfeld .cb_print = nodev, 4773d9b1a2aSHans Rosenfeld .cb_dump = nodev, 4783d9b1a2aSHans Rosenfeld .cb_read = nodev, 4793d9b1a2aSHans Rosenfeld .cb_write = nodev, 4803d9b1a2aSHans Rosenfeld .cb_ioctl = nvme_ioctl, 4813d9b1a2aSHans Rosenfeld .cb_devmap = nodev, 4823d9b1a2aSHans Rosenfeld .cb_mmap = nodev, 4833d9b1a2aSHans Rosenfeld .cb_segmap = nodev, 4843d9b1a2aSHans Rosenfeld .cb_chpoll = nochpoll, 4853d9b1a2aSHans Rosenfeld .cb_prop_op = ddi_prop_op, 4863d9b1a2aSHans Rosenfeld .cb_str = 0, 4873d9b1a2aSHans Rosenfeld .cb_flag = D_NEW | D_MP, 4883d9b1a2aSHans Rosenfeld .cb_rev = CB_REV, 4893d9b1a2aSHans Rosenfeld .cb_aread = nodev, 4903d9b1a2aSHans Rosenfeld .cb_awrite = nodev 4913d9b1a2aSHans Rosenfeld }; 4923d9b1a2aSHans Rosenfeld 4933c9168faSHans Rosenfeld static struct dev_ops nvme_dev_ops = { 4943c9168faSHans Rosenfeld .devo_rev = DEVO_REV, 4953c9168faSHans Rosenfeld .devo_refcnt = 0, 4963c9168faSHans Rosenfeld .devo_getinfo = ddi_no_info, 4973c9168faSHans Rosenfeld .devo_identify = nulldev, 4983c9168faSHans Rosenfeld .devo_probe = nulldev, 4993c9168faSHans Rosenfeld .devo_attach = nvme_attach, 5003c9168faSHans Rosenfeld .devo_detach = nvme_detach, 5013c9168faSHans Rosenfeld .devo_reset = nodev, 5023d9b1a2aSHans Rosenfeld .devo_cb_ops = &nvme_cb_ops, 5033c9168faSHans Rosenfeld .devo_bus_ops = NULL, 5043c9168faSHans Rosenfeld .devo_power = NULL, 5053c9168faSHans Rosenfeld .devo_quiesce = nvme_quiesce, 5063c9168faSHans Rosenfeld }; 5073c9168faSHans Rosenfeld 5083c9168faSHans Rosenfeld static struct modldrv nvme_modldrv = { 5093c9168faSHans Rosenfeld .drv_modops = &mod_driverops, 51024979ca3SHans Rosenfeld .drv_linkinfo = "NVMe v1.1b", 5113c9168faSHans Rosenfeld .drv_dev_ops = &nvme_dev_ops 5123c9168faSHans Rosenfeld }; 5133c9168faSHans Rosenfeld 5143c9168faSHans Rosenfeld static struct modlinkage nvme_modlinkage = { 5153c9168faSHans Rosenfeld .ml_rev = MODREV_1, 5163c9168faSHans Rosenfeld .ml_linkage = { &nvme_modldrv, NULL } 5173c9168faSHans Rosenfeld }; 5183c9168faSHans Rosenfeld 5193c9168faSHans Rosenfeld static bd_ops_t nvme_bd_ops = { 5203c9168faSHans Rosenfeld .o_version = BD_OPS_VERSION_0, 5213c9168faSHans Rosenfeld .o_drive_info = nvme_bd_driveinfo, 5223c9168faSHans Rosenfeld .o_media_info = nvme_bd_mediainfo, 5233c9168faSHans Rosenfeld .o_devid_init = nvme_bd_devid, 5243c9168faSHans Rosenfeld .o_sync_cache = nvme_bd_sync, 5253c9168faSHans Rosenfeld .o_read = nvme_bd_read, 5263c9168faSHans Rosenfeld .o_write = nvme_bd_write, 5273c9168faSHans Rosenfeld }; 5283c9168faSHans Rosenfeld 529e984c70bSHans Rosenfeld /* 530e984c70bSHans Rosenfeld * This list will hold commands that have timed out and couldn't be aborted. 531e984c70bSHans Rosenfeld * As we don't know what the hardware may still do with the DMA memory we can't 532e984c70bSHans Rosenfeld * free them, so we'll keep them forever on this list where we can easily look 533e984c70bSHans Rosenfeld * at them with mdb. 534e984c70bSHans Rosenfeld */ 535e984c70bSHans Rosenfeld static struct list nvme_lost_cmds; 536e984c70bSHans Rosenfeld static kmutex_t nvme_lc_mutex; 537e984c70bSHans Rosenfeld 5383c9168faSHans Rosenfeld int 5393c9168faSHans Rosenfeld _init(void) 5403c9168faSHans Rosenfeld { 5413c9168faSHans Rosenfeld int error; 5423c9168faSHans Rosenfeld 5433c9168faSHans Rosenfeld error = ddi_soft_state_init(&nvme_state, sizeof (nvme_t), 1); 5443c9168faSHans Rosenfeld if (error != DDI_SUCCESS) 5453c9168faSHans Rosenfeld return (error); 5463c9168faSHans Rosenfeld 5473c9168faSHans Rosenfeld nvme_cmd_cache = kmem_cache_create("nvme_cmd_cache", 5483c9168faSHans Rosenfeld sizeof (nvme_cmd_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 5493c9168faSHans Rosenfeld 550e984c70bSHans Rosenfeld mutex_init(&nvme_lc_mutex, NULL, MUTEX_DRIVER, NULL); 551e984c70bSHans Rosenfeld list_create(&nvme_lost_cmds, sizeof (nvme_cmd_t), 552e984c70bSHans Rosenfeld offsetof(nvme_cmd_t, nc_list)); 553e984c70bSHans Rosenfeld 5543c9168faSHans Rosenfeld bd_mod_init(&nvme_dev_ops); 5553c9168faSHans Rosenfeld 5563c9168faSHans Rosenfeld error = mod_install(&nvme_modlinkage); 5573c9168faSHans Rosenfeld if (error != DDI_SUCCESS) { 5583c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 559e984c70bSHans Rosenfeld mutex_destroy(&nvme_lc_mutex); 560e984c70bSHans Rosenfeld list_destroy(&nvme_lost_cmds); 5613c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 5623c9168faSHans Rosenfeld } 5633c9168faSHans Rosenfeld 5643c9168faSHans Rosenfeld return (error); 5653c9168faSHans Rosenfeld } 5663c9168faSHans Rosenfeld 5673c9168faSHans Rosenfeld int 5683c9168faSHans Rosenfeld _fini(void) 5693c9168faSHans Rosenfeld { 5703c9168faSHans Rosenfeld int error; 5713c9168faSHans Rosenfeld 572e984c70bSHans Rosenfeld if (!list_is_empty(&nvme_lost_cmds)) 573e984c70bSHans Rosenfeld return (DDI_FAILURE); 574e984c70bSHans Rosenfeld 5753c9168faSHans Rosenfeld error = mod_remove(&nvme_modlinkage); 5763c9168faSHans Rosenfeld if (error == DDI_SUCCESS) { 5773c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 5783c9168faSHans Rosenfeld kmem_cache_destroy(nvme_cmd_cache); 579e984c70bSHans Rosenfeld mutex_destroy(&nvme_lc_mutex); 580e984c70bSHans Rosenfeld list_destroy(&nvme_lost_cmds); 5813c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 5823c9168faSHans Rosenfeld } 5833c9168faSHans Rosenfeld 5843c9168faSHans Rosenfeld return (error); 5853c9168faSHans Rosenfeld } 5863c9168faSHans Rosenfeld 5873c9168faSHans Rosenfeld int 5883c9168faSHans Rosenfeld _info(struct modinfo *modinfop) 5893c9168faSHans Rosenfeld { 5903c9168faSHans Rosenfeld return (mod_info(&nvme_modlinkage, modinfop)); 5913c9168faSHans Rosenfeld } 5923c9168faSHans Rosenfeld 5933c9168faSHans Rosenfeld static inline void 5943c9168faSHans Rosenfeld nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val) 5953c9168faSHans Rosenfeld { 5963c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 5973c9168faSHans Rosenfeld 5983c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 5993c9168faSHans Rosenfeld ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val); 6003c9168faSHans Rosenfeld } 6013c9168faSHans Rosenfeld 6023c9168faSHans Rosenfeld static inline void 6033c9168faSHans Rosenfeld nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val) 6043c9168faSHans Rosenfeld { 6053c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 6063c9168faSHans Rosenfeld 6073c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 6083c9168faSHans Rosenfeld ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val); 6093c9168faSHans Rosenfeld } 6103c9168faSHans Rosenfeld 6113c9168faSHans Rosenfeld static inline uint64_t 6123c9168faSHans Rosenfeld nvme_get64(nvme_t *nvme, uintptr_t reg) 6133c9168faSHans Rosenfeld { 6143c9168faSHans Rosenfeld uint64_t val; 6153c9168faSHans Rosenfeld 6163c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 6173c9168faSHans Rosenfeld 6183c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 6193c9168faSHans Rosenfeld val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg)); 6203c9168faSHans Rosenfeld 6213c9168faSHans Rosenfeld return (val); 6223c9168faSHans Rosenfeld } 6233c9168faSHans Rosenfeld 6243c9168faSHans Rosenfeld static inline uint32_t 6253c9168faSHans Rosenfeld nvme_get32(nvme_t *nvme, uintptr_t reg) 6263c9168faSHans Rosenfeld { 6273c9168faSHans Rosenfeld uint32_t val; 6283c9168faSHans Rosenfeld 6293c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 6303c9168faSHans Rosenfeld 6313c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 6323c9168faSHans Rosenfeld val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg)); 6333c9168faSHans Rosenfeld 6343c9168faSHans Rosenfeld return (val); 6353c9168faSHans Rosenfeld } 6363c9168faSHans Rosenfeld 6373c9168faSHans Rosenfeld static boolean_t 6383c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme_t *nvme) 6393c9168faSHans Rosenfeld { 6403c9168faSHans Rosenfeld ddi_fm_error_t error; 6413c9168faSHans Rosenfeld 6423c9168faSHans Rosenfeld ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION); 6433c9168faSHans Rosenfeld 6443c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 6453c9168faSHans Rosenfeld return (B_TRUE); 6463c9168faSHans Rosenfeld 6473c9168faSHans Rosenfeld return (B_FALSE); 6483c9168faSHans Rosenfeld } 6493c9168faSHans Rosenfeld 6503c9168faSHans Rosenfeld static boolean_t 6513c9168faSHans Rosenfeld nvme_check_dma_hdl(nvme_dma_t *dma) 6523c9168faSHans Rosenfeld { 6533c9168faSHans Rosenfeld ddi_fm_error_t error; 6543c9168faSHans Rosenfeld 6553c9168faSHans Rosenfeld if (dma == NULL) 6563c9168faSHans Rosenfeld return (B_FALSE); 6573c9168faSHans Rosenfeld 6583c9168faSHans Rosenfeld ddi_fm_dma_err_get(dma->nd_dmah, &error, DDI_FME_VERSION); 6593c9168faSHans Rosenfeld 6603c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 6613c9168faSHans Rosenfeld return (B_TRUE); 6623c9168faSHans Rosenfeld 6633c9168faSHans Rosenfeld return (B_FALSE); 6643c9168faSHans Rosenfeld } 6653c9168faSHans Rosenfeld 6663c9168faSHans Rosenfeld static void 6678834f7acSYouzhong Yang nvme_free_dma_common(nvme_dma_t *dma) 6683c9168faSHans Rosenfeld { 6693c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 6703c9168faSHans Rosenfeld (void) ddi_dma_unbind_handle(dma->nd_dmah); 6713c9168faSHans Rosenfeld if (dma->nd_acch != NULL) 6723c9168faSHans Rosenfeld ddi_dma_mem_free(&dma->nd_acch); 6733c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 6743c9168faSHans Rosenfeld ddi_dma_free_handle(&dma->nd_dmah); 6753c9168faSHans Rosenfeld } 6763c9168faSHans Rosenfeld 6778834f7acSYouzhong Yang static void 6788834f7acSYouzhong Yang nvme_free_dma(nvme_dma_t *dma) 6793c9168faSHans Rosenfeld { 6808834f7acSYouzhong Yang nvme_free_dma_common(dma); 6818834f7acSYouzhong Yang kmem_free(dma, sizeof (*dma)); 6828834f7acSYouzhong Yang } 6838834f7acSYouzhong Yang 684b6bc2fd4SDan McDonald /* ARGSUSED */ 6858834f7acSYouzhong Yang static void 6868834f7acSYouzhong Yang nvme_prp_dma_destructor(void *buf, void *private) 6878834f7acSYouzhong Yang { 6888834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 6893c9168faSHans Rosenfeld 6908834f7acSYouzhong Yang nvme_free_dma_common(dma); 6918834f7acSYouzhong Yang } 6928834f7acSYouzhong Yang 6938834f7acSYouzhong Yang static int 6948834f7acSYouzhong Yang nvme_alloc_dma_common(nvme_t *nvme, nvme_dma_t *dma, 6958834f7acSYouzhong Yang size_t len, uint_t flags, ddi_dma_attr_t *dma_attr) 6968834f7acSYouzhong Yang { 6973c9168faSHans Rosenfeld if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL, 6983c9168faSHans Rosenfeld &dma->nd_dmah) != DDI_SUCCESS) { 6993c9168faSHans Rosenfeld /* 7003c9168faSHans Rosenfeld * Due to DDI_DMA_SLEEP this can't be DDI_DMA_NORESOURCES, and 7013c9168faSHans Rosenfeld * the only other possible error is DDI_DMA_BADATTR which 7023c9168faSHans Rosenfeld * indicates a driver bug which should cause a panic. 7033c9168faSHans Rosenfeld */ 7043c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, 7053c9168faSHans Rosenfeld "!failed to get DMA handle, check DMA attributes"); 7063c9168faSHans Rosenfeld return (DDI_FAILURE); 7073c9168faSHans Rosenfeld } 7083c9168faSHans Rosenfeld 7093c9168faSHans Rosenfeld /* 7103c9168faSHans Rosenfeld * ddi_dma_mem_alloc() can only fail when DDI_DMA_NOSLEEP is specified 7113c9168faSHans Rosenfeld * or the flags are conflicting, which isn't the case here. 7123c9168faSHans Rosenfeld */ 7133c9168faSHans Rosenfeld (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr, 7143c9168faSHans Rosenfeld DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &dma->nd_memp, 7153c9168faSHans Rosenfeld &dma->nd_len, &dma->nd_acch); 7163c9168faSHans Rosenfeld 7173c9168faSHans Rosenfeld if (ddi_dma_addr_bind_handle(dma->nd_dmah, NULL, dma->nd_memp, 7183c9168faSHans Rosenfeld dma->nd_len, flags | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, 7193c9168faSHans Rosenfeld &dma->nd_cookie, &dma->nd_ncookie) != DDI_DMA_MAPPED) { 7203c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 7213c9168faSHans Rosenfeld "!failed to bind DMA memory"); 7223c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_dma_bind_err); 7238834f7acSYouzhong Yang nvme_free_dma_common(dma); 7248834f7acSYouzhong Yang return (DDI_FAILURE); 7258834f7acSYouzhong Yang } 7268834f7acSYouzhong Yang 7278834f7acSYouzhong Yang return (DDI_SUCCESS); 7288834f7acSYouzhong Yang } 7298834f7acSYouzhong Yang 7308834f7acSYouzhong Yang static int 7318834f7acSYouzhong Yang nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags, 7328834f7acSYouzhong Yang ddi_dma_attr_t *dma_attr, nvme_dma_t **ret) 7338834f7acSYouzhong Yang { 7348834f7acSYouzhong Yang nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP); 7358834f7acSYouzhong Yang 7368834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, len, flags, dma_attr) != 7378834f7acSYouzhong Yang DDI_SUCCESS) { 7383c9168faSHans Rosenfeld *ret = NULL; 7398834f7acSYouzhong Yang kmem_free(dma, sizeof (nvme_dma_t)); 7403c9168faSHans Rosenfeld return (DDI_FAILURE); 7413c9168faSHans Rosenfeld } 7423c9168faSHans Rosenfeld 7433c9168faSHans Rosenfeld bzero(dma->nd_memp, dma->nd_len); 7443c9168faSHans Rosenfeld 7453c9168faSHans Rosenfeld *ret = dma; 7463c9168faSHans Rosenfeld return (DDI_SUCCESS); 7473c9168faSHans Rosenfeld } 7483c9168faSHans Rosenfeld 749b6bc2fd4SDan McDonald /* ARGSUSED */ 7508834f7acSYouzhong Yang static int 7518834f7acSYouzhong Yang nvme_prp_dma_constructor(void *buf, void *private, int flags) 7528834f7acSYouzhong Yang { 7538834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 7548834f7acSYouzhong Yang nvme_t *nvme = (nvme_t *)private; 7558834f7acSYouzhong Yang 7568834f7acSYouzhong Yang dma->nd_dmah = NULL; 7578834f7acSYouzhong Yang dma->nd_acch = NULL; 7588834f7acSYouzhong Yang 7598834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, nvme->n_pagesize, 7608834f7acSYouzhong Yang DDI_DMA_READ, &nvme->n_prp_dma_attr) != DDI_SUCCESS) { 7618834f7acSYouzhong Yang return (-1); 7628834f7acSYouzhong Yang } 7638834f7acSYouzhong Yang 7648834f7acSYouzhong Yang ASSERT(dma->nd_ncookie == 1); 7658834f7acSYouzhong Yang 7668834f7acSYouzhong Yang dma->nd_cached = B_TRUE; 7678834f7acSYouzhong Yang 7688834f7acSYouzhong Yang return (0); 7698834f7acSYouzhong Yang } 7708834f7acSYouzhong Yang 7713c9168faSHans Rosenfeld static int 7723c9168faSHans Rosenfeld nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len, 7733c9168faSHans Rosenfeld uint_t flags, nvme_dma_t **dma) 7743c9168faSHans Rosenfeld { 7753c9168faSHans Rosenfeld uint32_t len = nentry * qe_len; 7763c9168faSHans Rosenfeld ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr; 7773c9168faSHans Rosenfeld 7783c9168faSHans Rosenfeld len = roundup(len, nvme->n_pagesize); 7793c9168faSHans Rosenfeld 7803c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma) 7813c9168faSHans Rosenfeld != DDI_SUCCESS) { 7823c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 7833c9168faSHans Rosenfeld "!failed to get DMA memory for queue"); 7843c9168faSHans Rosenfeld goto fail; 7853c9168faSHans Rosenfeld } 7863c9168faSHans Rosenfeld 7873c9168faSHans Rosenfeld if ((*dma)->nd_ncookie != 1) { 7883c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 7893c9168faSHans Rosenfeld "!got too many cookies for queue DMA"); 7903c9168faSHans Rosenfeld goto fail; 7913c9168faSHans Rosenfeld } 7923c9168faSHans Rosenfeld 7933c9168faSHans Rosenfeld return (DDI_SUCCESS); 7943c9168faSHans Rosenfeld 7953c9168faSHans Rosenfeld fail: 7963c9168faSHans Rosenfeld if (*dma) { 7973c9168faSHans Rosenfeld nvme_free_dma(*dma); 7983c9168faSHans Rosenfeld *dma = NULL; 7993c9168faSHans Rosenfeld } 8003c9168faSHans Rosenfeld 8013c9168faSHans Rosenfeld return (DDI_FAILURE); 8023c9168faSHans Rosenfeld } 8033c9168faSHans Rosenfeld 804*0999c112SPaul Winder static void 805*0999c112SPaul Winder nvme_free_cq(nvme_cq_t *cq) 806*0999c112SPaul Winder { 807*0999c112SPaul Winder mutex_destroy(&cq->ncq_mutex); 808*0999c112SPaul Winder 809*0999c112SPaul Winder if (cq->ncq_dma != NULL) 810*0999c112SPaul Winder nvme_free_dma(cq->ncq_dma); 811*0999c112SPaul Winder 812*0999c112SPaul Winder kmem_free(cq, sizeof (*cq)); 813*0999c112SPaul Winder } 814*0999c112SPaul Winder 8153c9168faSHans Rosenfeld static void 8163c9168faSHans Rosenfeld nvme_free_qpair(nvme_qpair_t *qp) 8173c9168faSHans Rosenfeld { 8183c9168faSHans Rosenfeld int i; 8193c9168faSHans Rosenfeld 8203c9168faSHans Rosenfeld mutex_destroy(&qp->nq_mutex); 8214b324362SHans Rosenfeld sema_destroy(&qp->nq_sema); 8223c9168faSHans Rosenfeld 8233c9168faSHans Rosenfeld if (qp->nq_sqdma != NULL) 8243c9168faSHans Rosenfeld nvme_free_dma(qp->nq_sqdma); 8253c9168faSHans Rosenfeld 8263c9168faSHans Rosenfeld if (qp->nq_active_cmds > 0) 8273c9168faSHans Rosenfeld for (i = 0; i != qp->nq_nentry; i++) 8283c9168faSHans Rosenfeld if (qp->nq_cmd[i] != NULL) 8293c9168faSHans Rosenfeld nvme_free_cmd(qp->nq_cmd[i]); 8303c9168faSHans Rosenfeld 8313c9168faSHans Rosenfeld if (qp->nq_cmd != NULL) 8323c9168faSHans Rosenfeld kmem_free(qp->nq_cmd, sizeof (nvme_cmd_t *) * qp->nq_nentry); 8333c9168faSHans Rosenfeld 8343c9168faSHans Rosenfeld kmem_free(qp, sizeof (nvme_qpair_t)); 8353c9168faSHans Rosenfeld } 8363c9168faSHans Rosenfeld 837*0999c112SPaul Winder /* 838*0999c112SPaul Winder * Destroy the pre-allocated cq array, but only free individual completion 839*0999c112SPaul Winder * queues from the given starting index. 840*0999c112SPaul Winder */ 841*0999c112SPaul Winder static void 842*0999c112SPaul Winder nvme_destroy_cq_array(nvme_t *nvme, uint_t start) 843*0999c112SPaul Winder { 844*0999c112SPaul Winder uint_t i; 845*0999c112SPaul Winder 846*0999c112SPaul Winder for (i = start; i < nvme->n_cq_count; i++) 847*0999c112SPaul Winder if (nvme->n_cq[i] != NULL) 848*0999c112SPaul Winder nvme_free_cq(nvme->n_cq[i]); 849*0999c112SPaul Winder 850*0999c112SPaul Winder kmem_free(nvme->n_cq, sizeof (*nvme->n_cq) * nvme->n_cq_count); 851*0999c112SPaul Winder } 852*0999c112SPaul Winder 853*0999c112SPaul Winder static int 854*0999c112SPaul Winder nvme_alloc_cq(nvme_t *nvme, uint32_t nentry, nvme_cq_t **cqp, uint16_t idx) 855*0999c112SPaul Winder { 856*0999c112SPaul Winder nvme_cq_t *cq = kmem_zalloc(sizeof (*cq), KM_SLEEP); 857*0999c112SPaul Winder 858*0999c112SPaul Winder mutex_init(&cq->ncq_mutex, NULL, MUTEX_DRIVER, 859*0999c112SPaul Winder DDI_INTR_PRI(nvme->n_intr_pri)); 860*0999c112SPaul Winder 861*0999c112SPaul Winder if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t), 862*0999c112SPaul Winder DDI_DMA_READ, &cq->ncq_dma) != DDI_SUCCESS) 863*0999c112SPaul Winder goto fail; 864*0999c112SPaul Winder 865*0999c112SPaul Winder cq->ncq_cq = (nvme_cqe_t *)cq->ncq_dma->nd_memp; 866*0999c112SPaul Winder cq->ncq_nentry = nentry; 867*0999c112SPaul Winder cq->ncq_id = idx; 868*0999c112SPaul Winder cq->ncq_hdbl = NVME_REG_CQHDBL(nvme, idx); 869*0999c112SPaul Winder 870*0999c112SPaul Winder *cqp = cq; 871*0999c112SPaul Winder return (DDI_SUCCESS); 872*0999c112SPaul Winder 873*0999c112SPaul Winder fail: 874*0999c112SPaul Winder nvme_free_cq(cq); 875*0999c112SPaul Winder *cqp = NULL; 876*0999c112SPaul Winder 877*0999c112SPaul Winder return (DDI_FAILURE); 878*0999c112SPaul Winder } 879*0999c112SPaul Winder 880*0999c112SPaul Winder /* 881*0999c112SPaul Winder * Create the n_cq array big enough to hold "ncq" completion queues. 882*0999c112SPaul Winder * If the array already exists it will be re-sized (but only larger). 883*0999c112SPaul Winder * The admin queue is included in this array, which boosts the 884*0999c112SPaul Winder * max number of entries to UINT16_MAX + 1. 885*0999c112SPaul Winder */ 886*0999c112SPaul Winder static int 887*0999c112SPaul Winder nvme_create_cq_array(nvme_t *nvme, uint_t ncq, uint32_t nentry) 888*0999c112SPaul Winder { 889*0999c112SPaul Winder nvme_cq_t **cq; 890*0999c112SPaul Winder uint_t i, cq_count; 891*0999c112SPaul Winder 892*0999c112SPaul Winder ASSERT3U(ncq, >, nvme->n_cq_count); 893*0999c112SPaul Winder 894*0999c112SPaul Winder cq = nvme->n_cq; 895*0999c112SPaul Winder cq_count = nvme->n_cq_count; 896*0999c112SPaul Winder 897*0999c112SPaul Winder nvme->n_cq = kmem_zalloc(sizeof (*nvme->n_cq) * ncq, KM_SLEEP); 898*0999c112SPaul Winder nvme->n_cq_count = ncq; 899*0999c112SPaul Winder 900*0999c112SPaul Winder for (i = 0; i < cq_count; i++) 901*0999c112SPaul Winder nvme->n_cq[i] = cq[i]; 902*0999c112SPaul Winder 903*0999c112SPaul Winder for (; i < nvme->n_cq_count; i++) 904*0999c112SPaul Winder if (nvme_alloc_cq(nvme, nentry, &nvme->n_cq[i], i) != 905*0999c112SPaul Winder DDI_SUCCESS) 906*0999c112SPaul Winder goto fail; 907*0999c112SPaul Winder 908*0999c112SPaul Winder if (cq != NULL) 909*0999c112SPaul Winder kmem_free(cq, sizeof (*cq) * cq_count); 910*0999c112SPaul Winder 911*0999c112SPaul Winder return (DDI_SUCCESS); 912*0999c112SPaul Winder 913*0999c112SPaul Winder fail: 914*0999c112SPaul Winder nvme_destroy_cq_array(nvme, cq_count); 915*0999c112SPaul Winder /* 916*0999c112SPaul Winder * Restore the original array 917*0999c112SPaul Winder */ 918*0999c112SPaul Winder nvme->n_cq_count = cq_count; 919*0999c112SPaul Winder nvme->n_cq = cq; 920*0999c112SPaul Winder 921*0999c112SPaul Winder return (DDI_FAILURE); 922*0999c112SPaul Winder } 923*0999c112SPaul Winder 9243c9168faSHans Rosenfeld static int 9253c9168faSHans Rosenfeld nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp, 926*0999c112SPaul Winder uint_t idx) 9273c9168faSHans Rosenfeld { 9283c9168faSHans Rosenfeld nvme_qpair_t *qp = kmem_zalloc(sizeof (*qp), KM_SLEEP); 929*0999c112SPaul Winder uint_t cq_idx; 9303c9168faSHans Rosenfeld 9313c9168faSHans Rosenfeld mutex_init(&qp->nq_mutex, NULL, MUTEX_DRIVER, 9323c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 933*0999c112SPaul Winder 934*0999c112SPaul Winder /* 935*0999c112SPaul Winder * The NVMe spec defines that a full queue has one empty (unused) slot; 936*0999c112SPaul Winder * initialize the semaphore accordingly. 937*0999c112SPaul Winder */ 938*0999c112SPaul Winder sema_init(&qp->nq_sema, nentry - 1, NULL, SEMA_DRIVER, NULL); 9393c9168faSHans Rosenfeld 9403c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t), 9413c9168faSHans Rosenfeld DDI_DMA_WRITE, &qp->nq_sqdma) != DDI_SUCCESS) 9423c9168faSHans Rosenfeld goto fail; 9433c9168faSHans Rosenfeld 944*0999c112SPaul Winder /* 945*0999c112SPaul Winder * idx == 0 is adminq, those above 0 are shared io completion queues. 946*0999c112SPaul Winder */ 947*0999c112SPaul Winder cq_idx = idx == 0 ? 0 : 1 + (idx - 1) % (nvme->n_cq_count - 1); 948*0999c112SPaul Winder qp->nq_cq = nvme->n_cq[cq_idx]; 9493c9168faSHans Rosenfeld qp->nq_sq = (nvme_sqe_t *)qp->nq_sqdma->nd_memp; 9503c9168faSHans Rosenfeld qp->nq_nentry = nentry; 9513c9168faSHans Rosenfeld 9523c9168faSHans Rosenfeld qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx); 9533c9168faSHans Rosenfeld 9543c9168faSHans Rosenfeld qp->nq_cmd = kmem_zalloc(sizeof (nvme_cmd_t *) * nentry, KM_SLEEP); 9553c9168faSHans Rosenfeld qp->nq_next_cmd = 0; 9563c9168faSHans Rosenfeld 9573c9168faSHans Rosenfeld *nqp = qp; 9583c9168faSHans Rosenfeld return (DDI_SUCCESS); 9593c9168faSHans Rosenfeld 9603c9168faSHans Rosenfeld fail: 9613c9168faSHans Rosenfeld nvme_free_qpair(qp); 9623c9168faSHans Rosenfeld *nqp = NULL; 9633c9168faSHans Rosenfeld 9643c9168faSHans Rosenfeld return (DDI_FAILURE); 9653c9168faSHans Rosenfeld } 9663c9168faSHans Rosenfeld 9673c9168faSHans Rosenfeld static nvme_cmd_t * 9683c9168faSHans Rosenfeld nvme_alloc_cmd(nvme_t *nvme, int kmflag) 9693c9168faSHans Rosenfeld { 9703c9168faSHans Rosenfeld nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag); 9713c9168faSHans Rosenfeld 9723c9168faSHans Rosenfeld if (cmd == NULL) 9733c9168faSHans Rosenfeld return (cmd); 9743c9168faSHans Rosenfeld 9753c9168faSHans Rosenfeld bzero(cmd, sizeof (nvme_cmd_t)); 9763c9168faSHans Rosenfeld 9773c9168faSHans Rosenfeld cmd->nc_nvme = nvme; 9783c9168faSHans Rosenfeld 9793c9168faSHans Rosenfeld mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER, 9803c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 9813c9168faSHans Rosenfeld cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL); 9823c9168faSHans Rosenfeld 9833c9168faSHans Rosenfeld return (cmd); 9843c9168faSHans Rosenfeld } 9853c9168faSHans Rosenfeld 9863c9168faSHans Rosenfeld static void 9873c9168faSHans Rosenfeld nvme_free_cmd(nvme_cmd_t *cmd) 9883c9168faSHans Rosenfeld { 989e984c70bSHans Rosenfeld /* Don't free commands on the lost commands list. */ 990e984c70bSHans Rosenfeld if (list_link_active(&cmd->nc_list)) 991e984c70bSHans Rosenfeld return; 992e984c70bSHans Rosenfeld 9933c9168faSHans Rosenfeld if (cmd->nc_dma) { 9948834f7acSYouzhong Yang if (cmd->nc_dma->nd_cached) 9958834f7acSYouzhong Yang kmem_cache_free(cmd->nc_nvme->n_prp_cache, 9968834f7acSYouzhong Yang cmd->nc_dma); 9978834f7acSYouzhong Yang else 9988834f7acSYouzhong Yang nvme_free_dma(cmd->nc_dma); 9993c9168faSHans Rosenfeld cmd->nc_dma = NULL; 10003c9168faSHans Rosenfeld } 10013c9168faSHans Rosenfeld 10023c9168faSHans Rosenfeld cv_destroy(&cmd->nc_cv); 10033c9168faSHans Rosenfeld mutex_destroy(&cmd->nc_mutex); 10043c9168faSHans Rosenfeld 10053c9168faSHans Rosenfeld kmem_cache_free(nvme_cmd_cache, cmd); 10063c9168faSHans Rosenfeld } 10073c9168faSHans Rosenfeld 10084b324362SHans Rosenfeld static void 10094b324362SHans Rosenfeld nvme_submit_admin_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 10104b324362SHans Rosenfeld { 10114b324362SHans Rosenfeld sema_p(&qp->nq_sema); 10124b324362SHans Rosenfeld nvme_submit_cmd_common(qp, cmd); 10134b324362SHans Rosenfeld } 10144b324362SHans Rosenfeld 10153c9168faSHans Rosenfeld static int 10164b324362SHans Rosenfeld nvme_submit_io_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 10173c9168faSHans Rosenfeld { 10184b324362SHans Rosenfeld if (sema_tryp(&qp->nq_sema) == 0) 10194b324362SHans Rosenfeld return (EAGAIN); 10203c9168faSHans Rosenfeld 10214b324362SHans Rosenfeld nvme_submit_cmd_common(qp, cmd); 10224b324362SHans Rosenfeld return (0); 10234b324362SHans Rosenfeld } 10243c9168faSHans Rosenfeld 10254b324362SHans Rosenfeld static void 10264b324362SHans Rosenfeld nvme_submit_cmd_common(nvme_qpair_t *qp, nvme_cmd_t *cmd) 10274b324362SHans Rosenfeld { 10284b324362SHans Rosenfeld nvme_reg_sqtdbl_t tail = { 0 }; 10293c9168faSHans Rosenfeld 10304b324362SHans Rosenfeld mutex_enter(&qp->nq_mutex); 10313c9168faSHans Rosenfeld cmd->nc_completed = B_FALSE; 10323c9168faSHans Rosenfeld 10333c9168faSHans Rosenfeld /* 10343c9168faSHans Rosenfeld * Try to insert the cmd into the active cmd array at the nq_next_cmd 10353c9168faSHans Rosenfeld * slot. If the slot is already occupied advance to the next slot and 10363c9168faSHans Rosenfeld * try again. This can happen for long running commands like async event 10373c9168faSHans Rosenfeld * requests. 10383c9168faSHans Rosenfeld */ 10393c9168faSHans Rosenfeld while (qp->nq_cmd[qp->nq_next_cmd] != NULL) 10403c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 10413c9168faSHans Rosenfeld qp->nq_cmd[qp->nq_next_cmd] = cmd; 10423c9168faSHans Rosenfeld 10433c9168faSHans Rosenfeld qp->nq_active_cmds++; 10443c9168faSHans Rosenfeld 10453c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cid = qp->nq_next_cmd; 10463c9168faSHans Rosenfeld bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t)); 10473c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_sqdma->nd_dmah, 10483c9168faSHans Rosenfeld sizeof (nvme_sqe_t) * qp->nq_sqtail, 10493c9168faSHans Rosenfeld sizeof (nvme_sqe_t), DDI_DMA_SYNC_FORDEV); 10503c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 10513c9168faSHans Rosenfeld 10523c9168faSHans Rosenfeld tail.b.sqtdbl_sqt = qp->nq_sqtail = (qp->nq_sqtail + 1) % qp->nq_nentry; 10533c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r); 10543c9168faSHans Rosenfeld 10553c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 10563c9168faSHans Rosenfeld } 10573c9168faSHans Rosenfeld 1058e984c70bSHans Rosenfeld static nvme_cmd_t * 1059e984c70bSHans Rosenfeld nvme_unqueue_cmd(nvme_t *nvme, nvme_qpair_t *qp, int cid) 1060e984c70bSHans Rosenfeld { 1061e984c70bSHans Rosenfeld nvme_cmd_t *cmd; 1062e984c70bSHans Rosenfeld 1063e984c70bSHans Rosenfeld ASSERT(mutex_owned(&qp->nq_mutex)); 1064e984c70bSHans Rosenfeld ASSERT3S(cid, <, qp->nq_nentry); 1065e984c70bSHans Rosenfeld 1066e984c70bSHans Rosenfeld cmd = qp->nq_cmd[cid]; 1067e984c70bSHans Rosenfeld qp->nq_cmd[cid] = NULL; 1068e984c70bSHans Rosenfeld ASSERT3U(qp->nq_active_cmds, >, 0); 1069e984c70bSHans Rosenfeld qp->nq_active_cmds--; 1070e984c70bSHans Rosenfeld sema_v(&qp->nq_sema); 1071e984c70bSHans Rosenfeld 1072e984c70bSHans Rosenfeld ASSERT3P(cmd, !=, NULL); 1073e984c70bSHans Rosenfeld ASSERT3P(cmd->nc_nvme, ==, nvme); 1074e984c70bSHans Rosenfeld ASSERT3S(cmd->nc_sqe.sqe_cid, ==, cid); 1075e984c70bSHans Rosenfeld 1076e984c70bSHans Rosenfeld return (cmd); 1077e984c70bSHans Rosenfeld } 1078e984c70bSHans Rosenfeld 1079*0999c112SPaul Winder /* 1080*0999c112SPaul Winder * Get the command tied to the next completed cqe and bump along completion 1081*0999c112SPaul Winder * queue head counter. 1082*0999c112SPaul Winder */ 10833c9168faSHans Rosenfeld static nvme_cmd_t * 1084*0999c112SPaul Winder nvme_get_completed(nvme_t *nvme, nvme_cq_t *cq) 10853c9168faSHans Rosenfeld { 1086*0999c112SPaul Winder nvme_qpair_t *qp; 10873c9168faSHans Rosenfeld nvme_cqe_t *cqe; 10883c9168faSHans Rosenfeld nvme_cmd_t *cmd; 10893c9168faSHans Rosenfeld 1090*0999c112SPaul Winder ASSERT(mutex_owned(&cq->ncq_mutex)); 10913c9168faSHans Rosenfeld 1092*0999c112SPaul Winder cqe = &cq->ncq_cq[cq->ncq_head]; 10933c9168faSHans Rosenfeld 10943c9168faSHans Rosenfeld /* Check phase tag of CQE. Hardware inverts it for new entries. */ 1095*0999c112SPaul Winder if (cqe->cqe_sf.sf_p == cq->ncq_phase) 10963c9168faSHans Rosenfeld return (NULL); 10973c9168faSHans Rosenfeld 1098*0999c112SPaul Winder qp = nvme->n_ioq[cqe->cqe_sqid]; 10993c9168faSHans Rosenfeld 1100*0999c112SPaul Winder mutex_enter(&qp->nq_mutex); 1101e984c70bSHans Rosenfeld cmd = nvme_unqueue_cmd(nvme, qp, cqe->cqe_cid); 1102*0999c112SPaul Winder mutex_exit(&qp->nq_mutex); 11033c9168faSHans Rosenfeld 11043c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid == cqe->cqe_sqid); 11053c9168faSHans Rosenfeld bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t)); 11063c9168faSHans Rosenfeld 11073c9168faSHans Rosenfeld qp->nq_sqhead = cqe->cqe_sqhd; 11083c9168faSHans Rosenfeld 1109*0999c112SPaul Winder cq->ncq_head = (cq->ncq_head + 1) % cq->ncq_nentry; 11103c9168faSHans Rosenfeld 11113c9168faSHans Rosenfeld /* Toggle phase on wrap-around. */ 1112*0999c112SPaul Winder if (cq->ncq_head == 0) 1113*0999c112SPaul Winder cq->ncq_phase = cq->ncq_phase ? 0 : 1; 11143c9168faSHans Rosenfeld 1115*0999c112SPaul Winder return (cmd); 1116*0999c112SPaul Winder } 1117*0999c112SPaul Winder 1118*0999c112SPaul Winder /* 1119*0999c112SPaul Winder * Process all completed commands on the io completion queue. 1120*0999c112SPaul Winder */ 1121*0999c112SPaul Winder static uint_t 1122*0999c112SPaul Winder nvme_process_iocq(nvme_t *nvme, nvme_cq_t *cq) 1123*0999c112SPaul Winder { 1124*0999c112SPaul Winder nvme_reg_cqhdbl_t head = { 0 }; 1125*0999c112SPaul Winder nvme_cmd_t *cmd; 1126*0999c112SPaul Winder uint_t completed = 0; 1127*0999c112SPaul Winder 1128*0999c112SPaul Winder if (ddi_dma_sync(cq->ncq_dma->nd_dmah, 0, 0, DDI_DMA_SYNC_FORKERNEL) != 1129*0999c112SPaul Winder DDI_SUCCESS) 1130*0999c112SPaul Winder dev_err(nvme->n_dip, CE_WARN, "!ddi_dma_sync() failed in %s", 1131*0999c112SPaul Winder __func__); 1132*0999c112SPaul Winder 1133*0999c112SPaul Winder mutex_enter(&cq->ncq_mutex); 1134*0999c112SPaul Winder 1135*0999c112SPaul Winder while ((cmd = nvme_get_completed(nvme, cq)) != NULL) { 1136*0999c112SPaul Winder taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq, 1137*0999c112SPaul Winder cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent); 1138*0999c112SPaul Winder 1139*0999c112SPaul Winder completed++; 1140*0999c112SPaul Winder } 1141*0999c112SPaul Winder 1142*0999c112SPaul Winder if (completed > 0) { 1143*0999c112SPaul Winder /* 1144*0999c112SPaul Winder * Update the completion queue head doorbell. 1145*0999c112SPaul Winder */ 1146*0999c112SPaul Winder head.b.cqhdbl_cqh = cq->ncq_head; 1147*0999c112SPaul Winder nvme_put32(nvme, cq->ncq_hdbl, head.r); 1148*0999c112SPaul Winder } 1149*0999c112SPaul Winder 1150*0999c112SPaul Winder mutex_exit(&cq->ncq_mutex); 1151*0999c112SPaul Winder 1152*0999c112SPaul Winder return (completed); 1153*0999c112SPaul Winder } 1154*0999c112SPaul Winder 1155*0999c112SPaul Winder static nvme_cmd_t * 1156*0999c112SPaul Winder nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp) 1157*0999c112SPaul Winder { 1158*0999c112SPaul Winder nvme_cq_t *cq = qp->nq_cq; 1159*0999c112SPaul Winder nvme_reg_cqhdbl_t head = { 0 }; 1160*0999c112SPaul Winder nvme_cmd_t *cmd; 1161*0999c112SPaul Winder 1162*0999c112SPaul Winder if (ddi_dma_sync(cq->ncq_dma->nd_dmah, 0, 0, DDI_DMA_SYNC_FORKERNEL) != 1163*0999c112SPaul Winder DDI_SUCCESS) 1164*0999c112SPaul Winder dev_err(nvme->n_dip, CE_WARN, "!ddi_dma_sync() failed in %s", 1165*0999c112SPaul Winder __func__); 1166*0999c112SPaul Winder 1167*0999c112SPaul Winder mutex_enter(&cq->ncq_mutex); 1168*0999c112SPaul Winder 1169*0999c112SPaul Winder if ((cmd = nvme_get_completed(nvme, cq)) != NULL) { 1170*0999c112SPaul Winder head.b.cqhdbl_cqh = cq->ncq_head; 1171*0999c112SPaul Winder nvme_put32(nvme, cq->ncq_hdbl, head.r); 1172*0999c112SPaul Winder } 1173*0999c112SPaul Winder 1174*0999c112SPaul Winder mutex_exit(&cq->ncq_mutex); 11753c9168faSHans Rosenfeld 11763c9168faSHans Rosenfeld return (cmd); 11773c9168faSHans Rosenfeld } 11783c9168faSHans Rosenfeld 11793c9168faSHans Rosenfeld static int 11803c9168faSHans Rosenfeld nvme_check_unknown_cmd_status(nvme_cmd_t *cmd) 11813c9168faSHans Rosenfeld { 11823c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 11833c9168faSHans Rosenfeld 11843c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 11853c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 11863c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 11873c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 11883c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 11893c9168faSHans Rosenfeld 11903d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 11913d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 1192bef9e21aSHans Rosenfeld 11933c9168faSHans Rosenfeld if (cmd->nc_nvme->n_strict_version) { 11943c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 11953c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 11963c9168faSHans Rosenfeld } 11973c9168faSHans Rosenfeld 11983c9168faSHans Rosenfeld return (EIO); 11993c9168faSHans Rosenfeld } 12003c9168faSHans Rosenfeld 12013c9168faSHans Rosenfeld static int 12023c9168faSHans Rosenfeld nvme_check_vendor_cmd_status(nvme_cmd_t *cmd) 12033c9168faSHans Rosenfeld { 12043c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 12053c9168faSHans Rosenfeld 12063c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 12073c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 12083c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 12093c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 12103c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 121134c938c7SPete Shephard if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) { 12123c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 12133c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 12143c9168faSHans Rosenfeld } 12153c9168faSHans Rosenfeld 12163c9168faSHans Rosenfeld return (EIO); 12173c9168faSHans Rosenfeld } 12183c9168faSHans Rosenfeld 12193c9168faSHans Rosenfeld static int 12203c9168faSHans Rosenfeld nvme_check_integrity_cmd_status(nvme_cmd_t *cmd) 12213c9168faSHans Rosenfeld { 12223c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 12233c9168faSHans Rosenfeld 12243c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 12253c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_WRITE: 12263c9168faSHans Rosenfeld /* write fail */ 12273c9168faSHans Rosenfeld /* TODO: post ereport */ 12283d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 12293d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 12303c9168faSHans Rosenfeld return (EIO); 12313c9168faSHans Rosenfeld 12323c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_READ: 12333c9168faSHans Rosenfeld /* read fail */ 12343c9168faSHans Rosenfeld /* TODO: post ereport */ 12353d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 12363d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 12373c9168faSHans Rosenfeld return (EIO); 12383c9168faSHans Rosenfeld 12393c9168faSHans Rosenfeld default: 12403c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 12413c9168faSHans Rosenfeld } 12423c9168faSHans Rosenfeld } 12433c9168faSHans Rosenfeld 12443c9168faSHans Rosenfeld static int 12453c9168faSHans Rosenfeld nvme_check_generic_cmd_status(nvme_cmd_t *cmd) 12463c9168faSHans Rosenfeld { 12473c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 12483c9168faSHans Rosenfeld 12493c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 12503c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_SUCCESS: 12513c9168faSHans Rosenfeld return (0); 12523c9168faSHans Rosenfeld 12533c9168faSHans Rosenfeld /* 12543c9168faSHans Rosenfeld * Errors indicating a bug in the driver should cause a panic. 12553c9168faSHans Rosenfeld */ 12563c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_OPC: 12573c9168faSHans Rosenfeld /* Invalid Command Opcode */ 125808139162SToomas Soome if (!cmd->nc_dontpanic) 125908139162SToomas Soome dev_err(cmd->nc_nvme->n_dip, CE_PANIC, 126008139162SToomas Soome "programming error: invalid opcode in cmd %p", 126108139162SToomas Soome (void *)cmd); 126208139162SToomas Soome return (EINVAL); 12633c9168faSHans Rosenfeld 12643c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_FLD: 12653c9168faSHans Rosenfeld /* Invalid Field in Command */ 12663d9b1a2aSHans Rosenfeld if (!cmd->nc_dontpanic) 12673d9b1a2aSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, 12683d9b1a2aSHans Rosenfeld "programming error: invalid field in cmd %p", 12693d9b1a2aSHans Rosenfeld (void *)cmd); 12703d9b1a2aSHans Rosenfeld return (EIO); 12713c9168faSHans Rosenfeld 12723c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ID_CNFL: 12733c9168faSHans Rosenfeld /* Command ID Conflict */ 12743c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 12753c9168faSHans Rosenfeld "cmd ID conflict in cmd %p", (void *)cmd); 12763c9168faSHans Rosenfeld return (0); 12773c9168faSHans Rosenfeld 12783c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_NS: 12793c9168faSHans Rosenfeld /* Invalid Namespace or Format */ 12803d9b1a2aSHans Rosenfeld if (!cmd->nc_dontpanic) 12813d9b1a2aSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, 1282f313c178SYuri Pankov "programming error: invalid NS/format in cmd %p", 12833d9b1a2aSHans Rosenfeld (void *)cmd); 12843d9b1a2aSHans Rosenfeld return (EINVAL); 12853c9168faSHans Rosenfeld 12863c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_LBA_RANGE: 12873c9168faSHans Rosenfeld /* LBA Out Of Range */ 12883c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 12893c9168faSHans Rosenfeld "LBA out of range in cmd %p", (void *)cmd); 12903c9168faSHans Rosenfeld return (0); 12913c9168faSHans Rosenfeld 12923c9168faSHans Rosenfeld /* 12933c9168faSHans Rosenfeld * Non-fatal errors, handle gracefully. 12943c9168faSHans Rosenfeld */ 12953c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_DATA_XFR_ERR: 12963c9168faSHans Rosenfeld /* Data Transfer Error (DMA) */ 12973c9168faSHans Rosenfeld /* TODO: post ereport */ 12983c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err); 12993d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 13003d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 13013c9168faSHans Rosenfeld return (EIO); 13023c9168faSHans Rosenfeld 13033c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INTERNAL_ERR: 13043c9168faSHans Rosenfeld /* 13053c9168faSHans Rosenfeld * Internal Error. The spec (v1.0, section 4.5.1.2) says 13063c9168faSHans Rosenfeld * detailed error information is returned as async event, 13073c9168faSHans Rosenfeld * so we pretty much ignore the error here and handle it 13083c9168faSHans Rosenfeld * in the async event handler. 13093c9168faSHans Rosenfeld */ 13103c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_internal_err); 13113d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 13123d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 13133c9168faSHans Rosenfeld return (EIO); 13143c9168faSHans Rosenfeld 13153c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_REQUEST: 13163c9168faSHans Rosenfeld /* 13173c9168faSHans Rosenfeld * Command Abort Requested. This normally happens only when a 13183c9168faSHans Rosenfeld * command times out. 13193c9168faSHans Rosenfeld */ 13203c9168faSHans Rosenfeld /* TODO: post ereport or change blkdev to handle this? */ 13213c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err); 13223c9168faSHans Rosenfeld return (ECANCELED); 13233c9168faSHans Rosenfeld 13243c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_PWRLOSS: 13253c9168faSHans Rosenfeld /* Command Aborted due to Power Loss Notification */ 13263c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 13273c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 13283c9168faSHans Rosenfeld return (EIO); 13293c9168faSHans Rosenfeld 13303c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_SQ_DEL: 13313c9168faSHans Rosenfeld /* Command Aborted due to SQ Deletion */ 13323c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del); 13333c9168faSHans Rosenfeld return (EIO); 13343c9168faSHans Rosenfeld 13353c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_CAP_EXC: 13363c9168faSHans Rosenfeld /* Capacity Exceeded */ 13373c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc); 13383d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 13393d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 13403c9168faSHans Rosenfeld return (EIO); 13413c9168faSHans Rosenfeld 13423c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_NS_NOTRDY: 13433c9168faSHans Rosenfeld /* Namespace Not Ready */ 13443c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy); 13453d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 13463d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 13473c9168faSHans Rosenfeld return (EIO); 13483c9168faSHans Rosenfeld 13493c9168faSHans Rosenfeld default: 13503c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 13513c9168faSHans Rosenfeld } 13523c9168faSHans Rosenfeld } 13533c9168faSHans Rosenfeld 13543c9168faSHans Rosenfeld static int 13553c9168faSHans Rosenfeld nvme_check_specific_cmd_status(nvme_cmd_t *cmd) 13563c9168faSHans Rosenfeld { 13573c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 13583c9168faSHans Rosenfeld 13593c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 13603c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_CQ: 13613c9168faSHans Rosenfeld /* Completion Queue Invalid */ 13623c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE); 13633c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err); 13643c9168faSHans Rosenfeld return (EINVAL); 13653c9168faSHans Rosenfeld 13663c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_QID: 13673c9168faSHans Rosenfeld /* Invalid Queue Identifier */ 13683c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 13693c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE || 13703c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE || 13713c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 13723c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err); 13733c9168faSHans Rosenfeld return (EINVAL); 13743c9168faSHans Rosenfeld 13753c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_MAX_QSZ_EXC: 13763c9168faSHans Rosenfeld /* Max Queue Size Exceeded */ 13773c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 13783c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 13793c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc); 13803c9168faSHans Rosenfeld return (EINVAL); 13813c9168faSHans Rosenfeld 13823c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ABRT_CMD_EXC: 13833c9168faSHans Rosenfeld /* Abort Command Limit Exceeded */ 13843c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT); 13853c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 13863c9168faSHans Rosenfeld "abort command limit exceeded in cmd %p", (void *)cmd); 13873c9168faSHans Rosenfeld return (0); 13883c9168faSHans Rosenfeld 13893c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC: 13903c9168faSHans Rosenfeld /* Async Event Request Limit Exceeded */ 13913c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT); 13923c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 13933c9168faSHans Rosenfeld "async event request limit exceeded in cmd %p", 13943c9168faSHans Rosenfeld (void *)cmd); 13953c9168faSHans Rosenfeld return (0); 13963c9168faSHans Rosenfeld 13973c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_INT_VECT: 13983c9168faSHans Rosenfeld /* Invalid Interrupt Vector */ 13993c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 14003c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect); 14013c9168faSHans Rosenfeld return (EINVAL); 14023c9168faSHans Rosenfeld 14033c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_LOG_PAGE: 14043c9168faSHans Rosenfeld /* Invalid Log Page */ 14053c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE); 14063c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_log_page); 14073c9168faSHans Rosenfeld return (EINVAL); 14083c9168faSHans Rosenfeld 14093c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_FORMAT: 14103c9168faSHans Rosenfeld /* Invalid Format */ 14113c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT); 14123c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_format); 14133d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 14143d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 14153c9168faSHans Rosenfeld return (EINVAL); 14163c9168faSHans Rosenfeld 14173c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_Q_DEL: 14183c9168faSHans Rosenfeld /* Invalid Queue Deletion */ 14193c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 14203c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_q_del); 14213c9168faSHans Rosenfeld return (EINVAL); 14223c9168faSHans Rosenfeld 14233c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_CNFL_ATTR: 14243c9168faSHans Rosenfeld /* Conflicting Attributes */ 14253c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT || 14263c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 14273c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 14283c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr); 14293d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 14303d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 14313c9168faSHans Rosenfeld return (EINVAL); 14323c9168faSHans Rosenfeld 14333c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_INV_PROT: 14343c9168faSHans Rosenfeld /* Invalid Protection Information */ 14353c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE || 14363c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 14373c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 14383c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_prot); 14393d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 14403d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 14413c9168faSHans Rosenfeld return (EINVAL); 14423c9168faSHans Rosenfeld 14433c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_READONLY: 14443c9168faSHans Rosenfeld /* Write to Read Only Range */ 14453c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 14463c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_readonly); 14473d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 14483d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 14493c9168faSHans Rosenfeld return (EROFS); 14503c9168faSHans Rosenfeld 14513c9168faSHans Rosenfeld default: 14523c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 14533c9168faSHans Rosenfeld } 14543c9168faSHans Rosenfeld } 14553c9168faSHans Rosenfeld 14563c9168faSHans Rosenfeld static inline int 14573c9168faSHans Rosenfeld nvme_check_cmd_status(nvme_cmd_t *cmd) 14583c9168faSHans Rosenfeld { 14593c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 14603c9168faSHans Rosenfeld 1461e984c70bSHans Rosenfeld /* 1462e984c70bSHans Rosenfeld * Take a shortcut if the controller is dead, or if 1463e984c70bSHans Rosenfeld * command status indicates no error. 1464e984c70bSHans Rosenfeld */ 1465e984c70bSHans Rosenfeld if (cmd->nc_nvme->n_dead) 1466e984c70bSHans Rosenfeld return (EIO); 1467e984c70bSHans Rosenfeld 14683c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 14693c9168faSHans Rosenfeld cqe->cqe_sf.sf_sc == NVME_CQE_SC_GEN_SUCCESS) 14703c9168faSHans Rosenfeld return (0); 14713c9168faSHans Rosenfeld 14723c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC) 14733c9168faSHans Rosenfeld return (nvme_check_generic_cmd_status(cmd)); 14743c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_SPECIFIC) 14753c9168faSHans Rosenfeld return (nvme_check_specific_cmd_status(cmd)); 14763c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_INTEGRITY) 14773c9168faSHans Rosenfeld return (nvme_check_integrity_cmd_status(cmd)); 14783c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_VENDOR) 14793c9168faSHans Rosenfeld return (nvme_check_vendor_cmd_status(cmd)); 14803c9168faSHans Rosenfeld 14813c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 14823c9168faSHans Rosenfeld } 14833c9168faSHans Rosenfeld 1484e984c70bSHans Rosenfeld static int 1485e984c70bSHans Rosenfeld nvme_abort_cmd(nvme_cmd_t *abort_cmd, uint_t sec) 14863c9168faSHans Rosenfeld { 14873c9168faSHans Rosenfeld nvme_t *nvme = abort_cmd->nc_nvme; 14883c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 14893c9168faSHans Rosenfeld nvme_abort_cmd_t ac = { 0 }; 1490e984c70bSHans Rosenfeld int ret = 0; 14913c9168faSHans Rosenfeld 14923c9168faSHans Rosenfeld sema_p(&nvme->n_abort_sema); 14933c9168faSHans Rosenfeld 14943c9168faSHans Rosenfeld ac.b.ac_cid = abort_cmd->nc_sqe.sqe_cid; 14953c9168faSHans Rosenfeld ac.b.ac_sqid = abort_cmd->nc_sqid; 14963c9168faSHans Rosenfeld 14973c9168faSHans Rosenfeld cmd->nc_sqid = 0; 14983c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT; 14993c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 15003c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = ac.r; 15013c9168faSHans Rosenfeld 15023c9168faSHans Rosenfeld /* 15033c9168faSHans Rosenfeld * Send the ABORT to the hardware. The ABORT command will return _after_ 1504e984c70bSHans Rosenfeld * the aborted command has completed (aborted or otherwise), but since 1505e984c70bSHans Rosenfeld * we still hold the aborted command's mutex its callback hasn't been 1506e984c70bSHans Rosenfeld * processed yet. 15073c9168faSHans Rosenfeld */ 1508e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, sec); 15093c9168faSHans Rosenfeld sema_v(&nvme->n_abort_sema); 15103c9168faSHans Rosenfeld 1511e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 15123c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15133c9168faSHans Rosenfeld "!ABORT failed with sct = %x, sc = %x", 15143c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 15153c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_abort_failed); 15163c9168faSHans Rosenfeld } else { 1517e984c70bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 1518e984c70bSHans Rosenfeld "!ABORT of command %d/%d %ssuccessful", 1519e984c70bSHans Rosenfeld abort_cmd->nc_sqe.sqe_cid, abort_cmd->nc_sqid, 1520e984c70bSHans Rosenfeld cmd->nc_cqe.cqe_dw0 & 1 ? "un" : ""); 1521e984c70bSHans Rosenfeld if ((cmd->nc_cqe.cqe_dw0 & 1) == 0) 1522e984c70bSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_aborted); 15233c9168faSHans Rosenfeld } 15243c9168faSHans Rosenfeld 15253c9168faSHans Rosenfeld nvme_free_cmd(cmd); 1526e984c70bSHans Rosenfeld return (ret); 15273c9168faSHans Rosenfeld } 15283c9168faSHans Rosenfeld 15293c9168faSHans Rosenfeld /* 15303c9168faSHans Rosenfeld * nvme_wait_cmd -- wait for command completion or timeout 15313c9168faSHans Rosenfeld * 15323c9168faSHans Rosenfeld * In case of a serious error or a timeout of the abort command the hardware 15333c9168faSHans Rosenfeld * will be declared dead and FMA will be notified. 15343c9168faSHans Rosenfeld */ 1535e984c70bSHans Rosenfeld static void 1536e8ba2a38SHans Rosenfeld nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec) 15373c9168faSHans Rosenfeld { 1538e8ba2a38SHans Rosenfeld clock_t timeout = ddi_get_lbolt() + drv_usectohz(sec * MICROSEC); 15393c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 15403c9168faSHans Rosenfeld nvme_reg_csts_t csts; 1541e984c70bSHans Rosenfeld nvme_qpair_t *qp; 15423c9168faSHans Rosenfeld 15433c9168faSHans Rosenfeld ASSERT(mutex_owned(&cmd->nc_mutex)); 15443c9168faSHans Rosenfeld 15453c9168faSHans Rosenfeld while (!cmd->nc_completed) { 15463c9168faSHans Rosenfeld if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1) 15473c9168faSHans Rosenfeld break; 15483c9168faSHans Rosenfeld } 15493c9168faSHans Rosenfeld 15503c9168faSHans Rosenfeld if (cmd->nc_completed) 1551e984c70bSHans Rosenfeld return; 15523c9168faSHans Rosenfeld 15533c9168faSHans Rosenfeld /* 1554e984c70bSHans Rosenfeld * The command timed out. 1555e984c70bSHans Rosenfeld * 15563c9168faSHans Rosenfeld * Check controller for fatal status, any errors associated with the 15573c9168faSHans Rosenfeld * register or DMA handle, or for a double timeout (abort command timed 15583c9168faSHans Rosenfeld * out). If necessary log a warning and call FMA. 15593c9168faSHans Rosenfeld */ 15603c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 1561e984c70bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!command %d/%d timeout, " 1562e984c70bSHans Rosenfeld "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_cid, cmd->nc_sqid, 1563e984c70bSHans Rosenfeld cmd->nc_sqe.sqe_opc, csts.b.csts_cfs); 15643c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_timeout); 15653c9168faSHans Rosenfeld 15663c9168faSHans Rosenfeld if (csts.b.csts_cfs || 15673c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme) || 15683c9168faSHans Rosenfeld nvme_check_dma_hdl(cmd->nc_dma) || 15693c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) { 15703c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 15713c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 1572e984c70bSHans Rosenfeld } else if (nvme_abort_cmd(cmd, sec) == 0) { 15733c9168faSHans Rosenfeld /* 1574e984c70bSHans Rosenfeld * If the abort succeeded the command should complete 1575e984c70bSHans Rosenfeld * immediately with an appropriate status. 15763c9168faSHans Rosenfeld */ 1577e984c70bSHans Rosenfeld while (!cmd->nc_completed) 1578e984c70bSHans Rosenfeld cv_wait(&cmd->nc_cv, &cmd->nc_mutex); 1579e984c70bSHans Rosenfeld 1580e984c70bSHans Rosenfeld return; 15813c9168faSHans Rosenfeld } 15823c9168faSHans Rosenfeld 1583e984c70bSHans Rosenfeld qp = nvme->n_ioq[cmd->nc_sqid]; 1584e984c70bSHans Rosenfeld 1585e984c70bSHans Rosenfeld mutex_enter(&qp->nq_mutex); 1586e984c70bSHans Rosenfeld (void) nvme_unqueue_cmd(nvme, qp, cmd->nc_sqe.sqe_cid); 1587e984c70bSHans Rosenfeld mutex_exit(&qp->nq_mutex); 1588e984c70bSHans Rosenfeld 1589e984c70bSHans Rosenfeld /* 1590e984c70bSHans Rosenfeld * As we don't know what the presumed dead hardware might still do with 1591e984c70bSHans Rosenfeld * the DMA memory, we'll put the command on the lost commands list if it 1592e984c70bSHans Rosenfeld * has any DMA memory. 1593e984c70bSHans Rosenfeld */ 1594e984c70bSHans Rosenfeld if (cmd->nc_dma != NULL) { 1595e984c70bSHans Rosenfeld mutex_enter(&nvme_lc_mutex); 1596e984c70bSHans Rosenfeld list_insert_head(&nvme_lost_cmds, cmd); 1597e984c70bSHans Rosenfeld mutex_exit(&nvme_lc_mutex); 1598e984c70bSHans Rosenfeld } 15993c9168faSHans Rosenfeld } 16003c9168faSHans Rosenfeld 16013c9168faSHans Rosenfeld static void 16023c9168faSHans Rosenfeld nvme_wakeup_cmd(void *arg) 16033c9168faSHans Rosenfeld { 16043c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 16053c9168faSHans Rosenfeld 16063c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 16073c9168faSHans Rosenfeld cmd->nc_completed = B_TRUE; 16083c9168faSHans Rosenfeld cv_signal(&cmd->nc_cv); 16093c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 16103c9168faSHans Rosenfeld } 16113c9168faSHans Rosenfeld 16123c9168faSHans Rosenfeld static void 16133c9168faSHans Rosenfeld nvme_async_event_task(void *arg) 16143c9168faSHans Rosenfeld { 16153c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 16163c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 16173c9168faSHans Rosenfeld nvme_error_log_entry_t *error_log = NULL; 16183c9168faSHans Rosenfeld nvme_health_log_t *health_log = NULL; 16193d9b1a2aSHans Rosenfeld size_t logsize = 0; 16203c9168faSHans Rosenfeld nvme_async_event_t event; 16213c9168faSHans Rosenfeld 16223c9168faSHans Rosenfeld /* 16233c9168faSHans Rosenfeld * Check for errors associated with the async request itself. The only 16243c9168faSHans Rosenfeld * command-specific error is "async event limit exceeded", which 16253c9168faSHans Rosenfeld * indicates a programming error in the driver and causes a panic in 16263c9168faSHans Rosenfeld * nvme_check_cmd_status(). 16273c9168faSHans Rosenfeld * 16283c9168faSHans Rosenfeld * Other possible errors are various scenarios where the async request 16293c9168faSHans Rosenfeld * was aborted, or internal errors in the device. Internal errors are 16303c9168faSHans Rosenfeld * reported to FMA, the command aborts need no special handling here. 163108139162SToomas Soome * 163208139162SToomas Soome * And finally, at least qemu nvme does not support async events, 163308139162SToomas Soome * and will return NVME_CQE_SC_GEN_INV_OPC | DNR. If so, we 163408139162SToomas Soome * will avoid posting async events. 16353c9168faSHans Rosenfeld */ 163608139162SToomas Soome 1637e984c70bSHans Rosenfeld if (nvme_check_cmd_status(cmd) != 0) { 16383c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 16393c9168faSHans Rosenfeld "!async event request returned failure, sct = %x, " 16403c9168faSHans Rosenfeld "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct, 16413c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr, 16423c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_m); 16433c9168faSHans Rosenfeld 16443c9168faSHans Rosenfeld if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 16453c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) { 16463c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 16473c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, 16483c9168faSHans Rosenfeld DDI_SERVICE_LOST); 16493c9168faSHans Rosenfeld } 165008139162SToomas Soome 165108139162SToomas Soome if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 165208139162SToomas Soome cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INV_OPC && 165308139162SToomas Soome cmd->nc_cqe.cqe_sf.sf_dnr == 1) { 165408139162SToomas Soome nvme->n_async_event_supported = B_FALSE; 165508139162SToomas Soome } 165608139162SToomas Soome 16573c9168faSHans Rosenfeld nvme_free_cmd(cmd); 16583c9168faSHans Rosenfeld return; 16593c9168faSHans Rosenfeld } 16603c9168faSHans Rosenfeld 16613c9168faSHans Rosenfeld 16623c9168faSHans Rosenfeld event.r = cmd->nc_cqe.cqe_dw0; 16633c9168faSHans Rosenfeld 16643c9168faSHans Rosenfeld /* Clear CQE and re-submit the async request. */ 16653c9168faSHans Rosenfeld bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t)); 16664b324362SHans Rosenfeld nvme_submit_admin_cmd(nvme->n_adminq, cmd); 16673c9168faSHans Rosenfeld 16683c9168faSHans Rosenfeld switch (event.b.ae_type) { 16693c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_ERROR: 16703c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_ERROR) { 1671bc586359SRobert Mustacchi (void) nvme_get_logpage(nvme, B_FALSE, 1672bc586359SRobert Mustacchi (void **)&error_log, &logsize, event.b.ae_logpage); 16733c9168faSHans Rosenfeld } else { 16743c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 16753c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 16763c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 16773c9168faSHans Rosenfeld } 16783c9168faSHans Rosenfeld 16793c9168faSHans Rosenfeld switch (event.b.ae_info) { 16803c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_SQ: 16813c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 16823c9168faSHans Rosenfeld "invalid submission queue"); 16833c9168faSHans Rosenfeld return; 16843c9168faSHans Rosenfeld 16853c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_DBL: 16863c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 16873c9168faSHans Rosenfeld "invalid doorbell write value"); 16883c9168faSHans Rosenfeld return; 16893c9168faSHans Rosenfeld 16903c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_DIAGFAIL: 16913c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure"); 16923c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 16933c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 16943c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_diagfail_event); 16953c9168faSHans Rosenfeld break; 16963c9168faSHans Rosenfeld 16973c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_PERSISTENT: 16983c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!persistent internal " 16993c9168faSHans Rosenfeld "device error"); 17003c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 17013c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 17023c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_persistent_event); 17033c9168faSHans Rosenfeld break; 17043c9168faSHans Rosenfeld 17053c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_TRANSIENT: 17063c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!transient internal " 17073c9168faSHans Rosenfeld "device error"); 17083c9168faSHans Rosenfeld /* TODO: send ereport */ 17093c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_transient_event); 17103c9168faSHans Rosenfeld break; 17113c9168faSHans Rosenfeld 17123c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_FW_LOAD: 17133c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17143c9168faSHans Rosenfeld "!firmware image load error"); 17153c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_fw_load_event); 17163c9168faSHans Rosenfeld break; 17173c9168faSHans Rosenfeld } 17183c9168faSHans Rosenfeld break; 17193c9168faSHans Rosenfeld 17203c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_HEALTH: 17213c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_HEALTH) { 1722bc586359SRobert Mustacchi (void) nvme_get_logpage(nvme, B_FALSE, 1723bc586359SRobert Mustacchi (void **)&health_log, &logsize, event.b.ae_logpage, 1724bc586359SRobert Mustacchi -1); 17253c9168faSHans Rosenfeld } else { 17263c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 17273c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 17283c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 17293c9168faSHans Rosenfeld } 17303c9168faSHans Rosenfeld 17313c9168faSHans Rosenfeld switch (event.b.ae_info) { 17323c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_RELIABILITY: 17333c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17343c9168faSHans Rosenfeld "!device reliability compromised"); 17353c9168faSHans Rosenfeld /* TODO: send ereport */ 17363c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_reliability_event); 17373c9168faSHans Rosenfeld break; 17383c9168faSHans Rosenfeld 17393c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_TEMPERATURE: 17403c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17413c9168faSHans Rosenfeld "!temperature above threshold"); 17423c9168faSHans Rosenfeld /* TODO: send ereport */ 17433c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_temperature_event); 17443c9168faSHans Rosenfeld break; 17453c9168faSHans Rosenfeld 17463c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_SPARE: 17473c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17483c9168faSHans Rosenfeld "!spare space below threshold"); 17493c9168faSHans Rosenfeld /* TODO: send ereport */ 17503c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_spare_event); 17513c9168faSHans Rosenfeld break; 17523c9168faSHans Rosenfeld } 17533c9168faSHans Rosenfeld break; 17543c9168faSHans Rosenfeld 17553c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_VENDOR: 17563c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event " 17573c9168faSHans Rosenfeld "received, info = %x, logpage = %x", event.b.ae_info, 17583c9168faSHans Rosenfeld event.b.ae_logpage); 17593c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_vendor_event); 17603c9168faSHans Rosenfeld break; 17613c9168faSHans Rosenfeld 17623c9168faSHans Rosenfeld default: 17633c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, " 17643c9168faSHans Rosenfeld "type = %x, info = %x, logpage = %x", event.b.ae_type, 17653c9168faSHans Rosenfeld event.b.ae_info, event.b.ae_logpage); 17663c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_event); 17673c9168faSHans Rosenfeld break; 17683c9168faSHans Rosenfeld } 17693c9168faSHans Rosenfeld 17703c9168faSHans Rosenfeld if (error_log) 17713d9b1a2aSHans Rosenfeld kmem_free(error_log, logsize); 17723c9168faSHans Rosenfeld 17733c9168faSHans Rosenfeld if (health_log) 17743d9b1a2aSHans Rosenfeld kmem_free(health_log, logsize); 17753c9168faSHans Rosenfeld } 17763c9168faSHans Rosenfeld 1777e984c70bSHans Rosenfeld static void 1778e8ba2a38SHans Rosenfeld nvme_admin_cmd(nvme_cmd_t *cmd, int sec) 17793c9168faSHans Rosenfeld { 17803c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 17814b324362SHans Rosenfeld nvme_submit_admin_cmd(cmd->nc_nvme->n_adminq, cmd); 1782e984c70bSHans Rosenfeld nvme_wait_cmd(cmd, sec); 17833c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 17843c9168faSHans Rosenfeld } 17853c9168faSHans Rosenfeld 17864b324362SHans Rosenfeld static void 17873c9168faSHans Rosenfeld nvme_async_event(nvme_t *nvme) 17883c9168faSHans Rosenfeld { 178908139162SToomas Soome nvme_cmd_t *cmd; 17903c9168faSHans Rosenfeld 179108139162SToomas Soome cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 17923c9168faSHans Rosenfeld cmd->nc_sqid = 0; 17933c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT; 17943c9168faSHans Rosenfeld cmd->nc_callback = nvme_async_event_task; 179508139162SToomas Soome cmd->nc_dontpanic = B_TRUE; 17963c9168faSHans Rosenfeld 17974b324362SHans Rosenfeld nvme_submit_admin_cmd(nvme->n_adminq, cmd); 17983c9168faSHans Rosenfeld } 17993c9168faSHans Rosenfeld 18003d9b1a2aSHans Rosenfeld static int 1801bc586359SRobert Mustacchi nvme_format_nvm(nvme_t *nvme, boolean_t user, uint32_t nsid, uint8_t lbaf, 1802bc586359SRobert Mustacchi boolean_t ms, uint8_t pi, boolean_t pil, uint8_t ses) 18033d9b1a2aSHans Rosenfeld { 18043d9b1a2aSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 18053d9b1a2aSHans Rosenfeld nvme_format_nvm_t format_nvm = { 0 }; 18063d9b1a2aSHans Rosenfeld int ret; 18073d9b1a2aSHans Rosenfeld 18083d9b1a2aSHans Rosenfeld format_nvm.b.fm_lbaf = lbaf & 0xf; 18093d9b1a2aSHans Rosenfeld format_nvm.b.fm_ms = ms ? 1 : 0; 18103d9b1a2aSHans Rosenfeld format_nvm.b.fm_pi = pi & 0x7; 18113d9b1a2aSHans Rosenfeld format_nvm.b.fm_pil = pil ? 1 : 0; 18123d9b1a2aSHans Rosenfeld format_nvm.b.fm_ses = ses & 0x7; 18133d9b1a2aSHans Rosenfeld 18143d9b1a2aSHans Rosenfeld cmd->nc_sqid = 0; 18153d9b1a2aSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 18163d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 18173d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_NVM_FORMAT; 18183d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = format_nvm.r; 18193d9b1a2aSHans Rosenfeld 18203d9b1a2aSHans Rosenfeld /* 18213d9b1a2aSHans Rosenfeld * Some devices like Samsung SM951 don't allow formatting of all 18223d9b1a2aSHans Rosenfeld * namespaces in one command. Handle that gracefully. 18233d9b1a2aSHans Rosenfeld */ 18243d9b1a2aSHans Rosenfeld if (nsid == (uint32_t)-1) 18253d9b1a2aSHans Rosenfeld cmd->nc_dontpanic = B_TRUE; 1826bc586359SRobert Mustacchi /* 1827bc586359SRobert Mustacchi * If this format request was initiated by the user, then don't allow a 1828bc586359SRobert Mustacchi * programmer error to panic the system. 1829bc586359SRobert Mustacchi */ 1830bc586359SRobert Mustacchi if (user) 1831bc586359SRobert Mustacchi cmd->nc_dontpanic = B_TRUE; 18323d9b1a2aSHans Rosenfeld 1833e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_format_cmd_timeout); 18343d9b1a2aSHans Rosenfeld 18353d9b1a2aSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 18363d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 18373d9b1a2aSHans Rosenfeld "!FORMAT failed with sct = %x, sc = %x", 18383d9b1a2aSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 18393d9b1a2aSHans Rosenfeld } 18403d9b1a2aSHans Rosenfeld 18413d9b1a2aSHans Rosenfeld nvme_free_cmd(cmd); 18423d9b1a2aSHans Rosenfeld return (ret); 18433d9b1a2aSHans Rosenfeld } 18443d9b1a2aSHans Rosenfeld 18453d9b1a2aSHans Rosenfeld static int 1846bc586359SRobert Mustacchi nvme_get_logpage(nvme_t *nvme, boolean_t user, void **buf, size_t *bufsize, 1847bc586359SRobert Mustacchi uint8_t logpage, ...) 18483c9168faSHans Rosenfeld { 18493c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 185034c938c7SPete Shephard nvme_getlogpage_t getlogpage = { 0 }; 18513c9168faSHans Rosenfeld va_list ap; 1852e984c70bSHans Rosenfeld int ret; 18533c9168faSHans Rosenfeld 18543c9168faSHans Rosenfeld va_start(ap, logpage); 18553c9168faSHans Rosenfeld 18563c9168faSHans Rosenfeld cmd->nc_sqid = 0; 18573c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 18583c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE; 18593c9168faSHans Rosenfeld 1860bc586359SRobert Mustacchi if (user) 1861bc586359SRobert Mustacchi cmd->nc_dontpanic = B_TRUE; 1862bc586359SRobert Mustacchi 18633c9168faSHans Rosenfeld getlogpage.b.lp_lid = logpage; 18643c9168faSHans Rosenfeld 18653c9168faSHans Rosenfeld switch (logpage) { 18663c9168faSHans Rosenfeld case NVME_LOGPAGE_ERROR: 18673c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 18683d9b1a2aSHans Rosenfeld /* 18693d9b1a2aSHans Rosenfeld * The GET LOG PAGE command can use at most 2 pages to return 18703d9b1a2aSHans Rosenfeld * data, PRP lists are not supported. 18713d9b1a2aSHans Rosenfeld */ 18723d9b1a2aSHans Rosenfeld *bufsize = MIN(2 * nvme->n_pagesize, 18733d9b1a2aSHans Rosenfeld nvme->n_error_log_len * sizeof (nvme_error_log_entry_t)); 18743c9168faSHans Rosenfeld break; 18753c9168faSHans Rosenfeld 18763c9168faSHans Rosenfeld case NVME_LOGPAGE_HEALTH: 18773c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t); 18783d9b1a2aSHans Rosenfeld *bufsize = sizeof (nvme_health_log_t); 18793c9168faSHans Rosenfeld break; 18803c9168faSHans Rosenfeld 18813c9168faSHans Rosenfeld case NVME_LOGPAGE_FWSLOT: 18823c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 18833d9b1a2aSHans Rosenfeld *bufsize = sizeof (nvme_fwslot_log_t); 18843c9168faSHans Rosenfeld break; 18853c9168faSHans Rosenfeld 18863c9168faSHans Rosenfeld default: 18873c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown log page requested: %d", 18883c9168faSHans Rosenfeld logpage); 18893c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_logpage); 1890e984c70bSHans Rosenfeld ret = EINVAL; 18913c9168faSHans Rosenfeld goto fail; 18923c9168faSHans Rosenfeld } 18933c9168faSHans Rosenfeld 18943c9168faSHans Rosenfeld va_end(ap); 18953c9168faSHans Rosenfeld 18963d9b1a2aSHans Rosenfeld getlogpage.b.lp_numd = *bufsize / sizeof (uint32_t) - 1; 18973c9168faSHans Rosenfeld 18983c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = getlogpage.r; 18993c9168faSHans Rosenfeld 1900*0999c112SPaul Winder if (nvme_zalloc_dma(nvme, *bufsize, 19013c9168faSHans Rosenfeld DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 19023c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19033c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for GET LOG PAGE"); 1904e984c70bSHans Rosenfeld ret = ENOMEM; 19053c9168faSHans Rosenfeld goto fail; 19063c9168faSHans Rosenfeld } 19073c9168faSHans Rosenfeld 19083c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 19093c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19103c9168faSHans Rosenfeld "!too many DMA cookies for GET LOG PAGE"); 19113c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 1912e984c70bSHans Rosenfeld ret = ENOMEM; 19133c9168faSHans Rosenfeld goto fail; 19143c9168faSHans Rosenfeld } 19153c9168faSHans Rosenfeld 19163c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 19173c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 19183c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 19193c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 19203c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 19213c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 19223c9168faSHans Rosenfeld } 19233c9168faSHans Rosenfeld 1924e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 19253c9168faSHans Rosenfeld 1926e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 19273c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19283c9168faSHans Rosenfeld "!GET LOG PAGE failed with sct = %x, sc = %x", 19293c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 19303c9168faSHans Rosenfeld goto fail; 19313c9168faSHans Rosenfeld } 19323c9168faSHans Rosenfeld 19333d9b1a2aSHans Rosenfeld *buf = kmem_alloc(*bufsize, KM_SLEEP); 19343d9b1a2aSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize); 19353d9b1a2aSHans Rosenfeld 19363c9168faSHans Rosenfeld fail: 19373c9168faSHans Rosenfeld nvme_free_cmd(cmd); 19383c9168faSHans Rosenfeld 19393d9b1a2aSHans Rosenfeld return (ret); 19403c9168faSHans Rosenfeld } 19413c9168faSHans Rosenfeld 1942e984c70bSHans Rosenfeld static int 1943bc586359SRobert Mustacchi nvme_identify(nvme_t *nvme, boolean_t user, uint32_t nsid, void **buf) 19443c9168faSHans Rosenfeld { 19453c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1946e984c70bSHans Rosenfeld int ret; 1947e984c70bSHans Rosenfeld 1948e984c70bSHans Rosenfeld if (buf == NULL) 1949e984c70bSHans Rosenfeld return (EINVAL); 19503c9168faSHans Rosenfeld 19513c9168faSHans Rosenfeld cmd->nc_sqid = 0; 19523c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 19533c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY; 19543c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 19553c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL; 19563c9168faSHans Rosenfeld 19573c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ, 19583c9168faSHans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 19593c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19603c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for IDENTIFY"); 1961e984c70bSHans Rosenfeld ret = ENOMEM; 19623c9168faSHans Rosenfeld goto fail; 19633c9168faSHans Rosenfeld } 19643c9168faSHans Rosenfeld 19653c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 19663c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19673c9168faSHans Rosenfeld "!too many DMA cookies for IDENTIFY"); 19683c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 1969e984c70bSHans Rosenfeld ret = ENOMEM; 19703c9168faSHans Rosenfeld goto fail; 19713c9168faSHans Rosenfeld } 19723c9168faSHans Rosenfeld 19733c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 19743c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 19753c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 19763c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 19773c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 19783c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 19793c9168faSHans Rosenfeld } 19803c9168faSHans Rosenfeld 1981bc586359SRobert Mustacchi if (user) 1982bc586359SRobert Mustacchi cmd->nc_dontpanic = B_TRUE; 1983bc586359SRobert Mustacchi 1984e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 19853c9168faSHans Rosenfeld 1986e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 19873c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19883c9168faSHans Rosenfeld "!IDENTIFY failed with sct = %x, sc = %x", 19893c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 19903c9168faSHans Rosenfeld goto fail; 19913c9168faSHans Rosenfeld } 19923c9168faSHans Rosenfeld 1993e984c70bSHans Rosenfeld *buf = kmem_alloc(NVME_IDENTIFY_BUFSIZE, KM_SLEEP); 1994e984c70bSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, *buf, NVME_IDENTIFY_BUFSIZE); 19953c9168faSHans Rosenfeld 19963c9168faSHans Rosenfeld fail: 19973c9168faSHans Rosenfeld nvme_free_cmd(cmd); 19983c9168faSHans Rosenfeld 1999e984c70bSHans Rosenfeld return (ret); 20003c9168faSHans Rosenfeld } 20013c9168faSHans Rosenfeld 2002e984c70bSHans Rosenfeld static int 2003bc586359SRobert Mustacchi nvme_set_features(nvme_t *nvme, boolean_t user, uint32_t nsid, uint8_t feature, 2004bc586359SRobert Mustacchi uint32_t val, uint32_t *res) 20053c9168faSHans Rosenfeld { 2006d148d46eSHans Rosenfeld _NOTE(ARGUNUSED(nsid)); 20073c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 2008e984c70bSHans Rosenfeld int ret = EINVAL; 20093c9168faSHans Rosenfeld 2010d148d46eSHans Rosenfeld ASSERT(res != NULL); 20113c9168faSHans Rosenfeld 20123c9168faSHans Rosenfeld cmd->nc_sqid = 0; 20133c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 20143c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES; 2015d148d46eSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = feature; 2016d148d46eSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = val; 2017d148d46eSHans Rosenfeld 2018bc586359SRobert Mustacchi if (user) 2019bc586359SRobert Mustacchi cmd->nc_dontpanic = B_TRUE; 2020bc586359SRobert Mustacchi 2021d148d46eSHans Rosenfeld switch (feature) { 2022d148d46eSHans Rosenfeld case NVME_FEAT_WRITE_CACHE: 2023d148d46eSHans Rosenfeld if (!nvme->n_write_cache_present) 2024d148d46eSHans Rosenfeld goto fail; 2025d148d46eSHans Rosenfeld break; 2026d148d46eSHans Rosenfeld 2027d148d46eSHans Rosenfeld case NVME_FEAT_NQUEUES: 2028d148d46eSHans Rosenfeld break; 2029d148d46eSHans Rosenfeld 2030d148d46eSHans Rosenfeld default: 2031d148d46eSHans Rosenfeld goto fail; 2032d148d46eSHans Rosenfeld } 20333c9168faSHans Rosenfeld 2034e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 20353c9168faSHans Rosenfeld 2036e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 20373c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2038d148d46eSHans Rosenfeld "!SET FEATURES %d failed with sct = %x, sc = %x", 2039d148d46eSHans Rosenfeld feature, cmd->nc_cqe.cqe_sf.sf_sct, 2040d148d46eSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc); 2041d148d46eSHans Rosenfeld goto fail; 20423c9168faSHans Rosenfeld } 20433c9168faSHans Rosenfeld 2044d148d46eSHans Rosenfeld *res = cmd->nc_cqe.cqe_dw0; 2045d148d46eSHans Rosenfeld 2046d148d46eSHans Rosenfeld fail: 20473c9168faSHans Rosenfeld nvme_free_cmd(cmd); 2048d148d46eSHans Rosenfeld return (ret); 2049d148d46eSHans Rosenfeld } 2050d148d46eSHans Rosenfeld 2051e984c70bSHans Rosenfeld static int 2052bc586359SRobert Mustacchi nvme_get_features(nvme_t *nvme, boolean_t user, uint32_t nsid, uint8_t feature, 2053bc586359SRobert Mustacchi uint32_t *res, void **buf, size_t *bufsize) 20543d9b1a2aSHans Rosenfeld { 20553d9b1a2aSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 2056e984c70bSHans Rosenfeld int ret = EINVAL; 20573d9b1a2aSHans Rosenfeld 20583d9b1a2aSHans Rosenfeld ASSERT(res != NULL); 20593d9b1a2aSHans Rosenfeld 20603d9b1a2aSHans Rosenfeld if (bufsize != NULL) 20613d9b1a2aSHans Rosenfeld *bufsize = 0; 20623d9b1a2aSHans Rosenfeld 20633d9b1a2aSHans Rosenfeld cmd->nc_sqid = 0; 20643d9b1a2aSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 20653d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_GET_FEATURES; 20663d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = feature; 20673d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = *res; 20683d9b1a2aSHans Rosenfeld 2069f313c178SYuri Pankov /* 2070f313c178SYuri Pankov * For some of the optional features there doesn't seem to be a method 2071f313c178SYuri Pankov * of detecting whether it is supported other than using it. This will 2072f313c178SYuri Pankov * cause "Invalid Field in Command" error, which is normally considered 2073f313c178SYuri Pankov * a programming error. Set the nc_dontpanic flag to override the panic 2074f313c178SYuri Pankov * in nvme_check_generic_cmd_status(). 2075f313c178SYuri Pankov */ 20763d9b1a2aSHans Rosenfeld switch (feature) { 20773d9b1a2aSHans Rosenfeld case NVME_FEAT_ARBITRATION: 20783d9b1a2aSHans Rosenfeld case NVME_FEAT_POWER_MGMT: 20793d9b1a2aSHans Rosenfeld case NVME_FEAT_TEMPERATURE: 20803d9b1a2aSHans Rosenfeld case NVME_FEAT_ERROR: 20813d9b1a2aSHans Rosenfeld case NVME_FEAT_NQUEUES: 20823d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_COAL: 20833d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_VECT: 20843d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_ATOM: 20853d9b1a2aSHans Rosenfeld case NVME_FEAT_ASYNC_EVENT: 20863d9b1a2aSHans Rosenfeld break; 20873d9b1a2aSHans Rosenfeld 20883d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_CACHE: 20893d9b1a2aSHans Rosenfeld if (!nvme->n_write_cache_present) 20903d9b1a2aSHans Rosenfeld goto fail; 20913d9b1a2aSHans Rosenfeld break; 20923d9b1a2aSHans Rosenfeld 20933d9b1a2aSHans Rosenfeld case NVME_FEAT_LBA_RANGE: 20943d9b1a2aSHans Rosenfeld if (!nvme->n_lba_range_supported) 20953d9b1a2aSHans Rosenfeld goto fail; 20963d9b1a2aSHans Rosenfeld 20973d9b1a2aSHans Rosenfeld cmd->nc_dontpanic = B_TRUE; 20983d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 20993d9b1a2aSHans Rosenfeld ASSERT(bufsize != NULL); 21003d9b1a2aSHans Rosenfeld *bufsize = NVME_LBA_RANGE_BUFSIZE; 21013d9b1a2aSHans Rosenfeld break; 21023d9b1a2aSHans Rosenfeld 21033d9b1a2aSHans Rosenfeld case NVME_FEAT_AUTO_PST: 21043d9b1a2aSHans Rosenfeld if (!nvme->n_auto_pst_supported) 21053d9b1a2aSHans Rosenfeld goto fail; 21063d9b1a2aSHans Rosenfeld 21073d9b1a2aSHans Rosenfeld ASSERT(bufsize != NULL); 21083d9b1a2aSHans Rosenfeld *bufsize = NVME_AUTO_PST_BUFSIZE; 21093d9b1a2aSHans Rosenfeld break; 21103d9b1a2aSHans Rosenfeld 2111f313c178SYuri Pankov case NVME_FEAT_PROGRESS: 2112f313c178SYuri Pankov if (!nvme->n_progress_supported) 2113f313c178SYuri Pankov goto fail; 2114f313c178SYuri Pankov 2115f313c178SYuri Pankov cmd->nc_dontpanic = B_TRUE; 2116f313c178SYuri Pankov break; 2117f313c178SYuri Pankov 21183d9b1a2aSHans Rosenfeld default: 21193d9b1a2aSHans Rosenfeld goto fail; 21203d9b1a2aSHans Rosenfeld } 21213d9b1a2aSHans Rosenfeld 2122bc586359SRobert Mustacchi if (user) 2123bc586359SRobert Mustacchi cmd->nc_dontpanic = B_TRUE; 2124bc586359SRobert Mustacchi 21253d9b1a2aSHans Rosenfeld if (bufsize != NULL && *bufsize != 0) { 21263d9b1a2aSHans Rosenfeld if (nvme_zalloc_dma(nvme, *bufsize, DDI_DMA_READ, 21273d9b1a2aSHans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 21283d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21293d9b1a2aSHans Rosenfeld "!nvme_zalloc_dma failed for GET FEATURES"); 2130e984c70bSHans Rosenfeld ret = ENOMEM; 21313d9b1a2aSHans Rosenfeld goto fail; 21323d9b1a2aSHans Rosenfeld } 21333d9b1a2aSHans Rosenfeld 21343d9b1a2aSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 21353d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21363d9b1a2aSHans Rosenfeld "!too many DMA cookies for GET FEATURES"); 21373d9b1a2aSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 2138e984c70bSHans Rosenfeld ret = ENOMEM; 21393d9b1a2aSHans Rosenfeld goto fail; 21403d9b1a2aSHans Rosenfeld } 21413d9b1a2aSHans Rosenfeld 21423d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = 21433d9b1a2aSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 21443d9b1a2aSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 21453d9b1a2aSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 21463d9b1a2aSHans Rosenfeld &cmd->nc_dma->nd_cookie); 21473d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 21483d9b1a2aSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 21493d9b1a2aSHans Rosenfeld } 21503d9b1a2aSHans Rosenfeld } 21513d9b1a2aSHans Rosenfeld 2152e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 21533d9b1a2aSHans Rosenfeld 2154e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 2155f313c178SYuri Pankov boolean_t known = B_TRUE; 2156f313c178SYuri Pankov 2157f313c178SYuri Pankov /* Check if this is unsupported optional feature */ 2158f313c178SYuri Pankov if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 2159f313c178SYuri Pankov cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INV_FLD) { 2160f313c178SYuri Pankov switch (feature) { 2161f313c178SYuri Pankov case NVME_FEAT_LBA_RANGE: 2162f313c178SYuri Pankov nvme->n_lba_range_supported = B_FALSE; 2163f313c178SYuri Pankov break; 2164f313c178SYuri Pankov case NVME_FEAT_PROGRESS: 2165f313c178SYuri Pankov nvme->n_progress_supported = B_FALSE; 2166f313c178SYuri Pankov break; 2167f313c178SYuri Pankov default: 2168f313c178SYuri Pankov known = B_FALSE; 2169f313c178SYuri Pankov break; 2170f313c178SYuri Pankov } 2171f313c178SYuri Pankov } else { 2172f313c178SYuri Pankov known = B_FALSE; 2173f313c178SYuri Pankov } 2174f313c178SYuri Pankov 2175f313c178SYuri Pankov /* Report the error otherwise */ 2176f313c178SYuri Pankov if (!known) { 21773d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21783d9b1a2aSHans Rosenfeld "!GET FEATURES %d failed with sct = %x, sc = %x", 21793d9b1a2aSHans Rosenfeld feature, cmd->nc_cqe.cqe_sf.sf_sct, 21803d9b1a2aSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc); 2181f313c178SYuri Pankov } 2182f313c178SYuri Pankov 21833d9b1a2aSHans Rosenfeld goto fail; 21843d9b1a2aSHans Rosenfeld } 21853d9b1a2aSHans Rosenfeld 21863d9b1a2aSHans Rosenfeld if (bufsize != NULL && *bufsize != 0) { 21873d9b1a2aSHans Rosenfeld ASSERT(buf != NULL); 21883d9b1a2aSHans Rosenfeld *buf = kmem_alloc(*bufsize, KM_SLEEP); 21893d9b1a2aSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize); 21903d9b1a2aSHans Rosenfeld } 21913d9b1a2aSHans Rosenfeld 21923d9b1a2aSHans Rosenfeld *res = cmd->nc_cqe.cqe_dw0; 21933d9b1a2aSHans Rosenfeld 21943d9b1a2aSHans Rosenfeld fail: 21953d9b1a2aSHans Rosenfeld nvme_free_cmd(cmd); 21963d9b1a2aSHans Rosenfeld return (ret); 21973d9b1a2aSHans Rosenfeld } 21983d9b1a2aSHans Rosenfeld 2199e984c70bSHans Rosenfeld static int 2200d148d46eSHans Rosenfeld nvme_write_cache_set(nvme_t *nvme, boolean_t enable) 2201d148d46eSHans Rosenfeld { 2202d148d46eSHans Rosenfeld nvme_write_cache_t nwc = { 0 }; 2203d148d46eSHans Rosenfeld 2204d148d46eSHans Rosenfeld if (enable) 2205d148d46eSHans Rosenfeld nwc.b.wc_wce = 1; 2206d148d46eSHans Rosenfeld 2207bc586359SRobert Mustacchi return (nvme_set_features(nvme, B_FALSE, 0, NVME_FEAT_WRITE_CACHE, 2208bc586359SRobert Mustacchi nwc.r, &nwc.r)); 2209d148d46eSHans Rosenfeld } 2210d148d46eSHans Rosenfeld 2211d148d46eSHans Rosenfeld static int 2212*0999c112SPaul Winder nvme_set_nqueues(nvme_t *nvme) 2213d148d46eSHans Rosenfeld { 22143d9b1a2aSHans Rosenfeld nvme_nqueues_t nq = { 0 }; 2215e984c70bSHans Rosenfeld int ret; 2216d148d46eSHans Rosenfeld 2217*0999c112SPaul Winder /* 2218*0999c112SPaul Winder * The default is to allocate one completion queue per vector. 2219*0999c112SPaul Winder */ 2220*0999c112SPaul Winder if (nvme->n_completion_queues == -1) 2221*0999c112SPaul Winder nvme->n_completion_queues = nvme->n_intr_cnt; 2222*0999c112SPaul Winder 2223*0999c112SPaul Winder /* 2224*0999c112SPaul Winder * There is no point in having more compeletion queues than 2225*0999c112SPaul Winder * interrupt vectors. 2226*0999c112SPaul Winder */ 2227*0999c112SPaul Winder nvme->n_completion_queues = MIN(nvme->n_completion_queues, 2228*0999c112SPaul Winder nvme->n_intr_cnt); 2229*0999c112SPaul Winder 2230*0999c112SPaul Winder /* 2231*0999c112SPaul Winder * The default is to use one submission queue per completion queue. 2232*0999c112SPaul Winder */ 2233*0999c112SPaul Winder if (nvme->n_submission_queues == -1) 2234*0999c112SPaul Winder nvme->n_submission_queues = nvme->n_completion_queues; 2235*0999c112SPaul Winder 2236*0999c112SPaul Winder /* 2237*0999c112SPaul Winder * There is no point in having more compeletion queues than 2238*0999c112SPaul Winder * submission queues. 2239*0999c112SPaul Winder */ 2240*0999c112SPaul Winder nvme->n_completion_queues = MIN(nvme->n_completion_queues, 2241*0999c112SPaul Winder nvme->n_submission_queues); 2242*0999c112SPaul Winder 2243*0999c112SPaul Winder ASSERT(nvme->n_submission_queues > 0); 2244*0999c112SPaul Winder ASSERT(nvme->n_completion_queues > 0); 2245*0999c112SPaul Winder 2246*0999c112SPaul Winder nq.b.nq_nsq = nvme->n_submission_queues - 1; 2247*0999c112SPaul Winder nq.b.nq_ncq = nvme->n_completion_queues - 1; 2248d148d46eSHans Rosenfeld 2249bc586359SRobert Mustacchi ret = nvme_set_features(nvme, B_FALSE, 0, NVME_FEAT_NQUEUES, nq.r, 2250bc586359SRobert Mustacchi &nq.r); 2251e984c70bSHans Rosenfeld 2252e984c70bSHans Rosenfeld if (ret == 0) { 2253e984c70bSHans Rosenfeld /* 2254*0999c112SPaul Winder * Never use more than the requested number of queues. 2255e984c70bSHans Rosenfeld */ 2256*0999c112SPaul Winder nvme->n_submission_queues = MIN(nvme->n_submission_queues, 2257*0999c112SPaul Winder nq.b.nq_nsq + 1); 2258*0999c112SPaul Winder nvme->n_completion_queues = MIN(nvme->n_completion_queues, 2259*0999c112SPaul Winder nq.b.nq_ncq + 1); 2260d148d46eSHans Rosenfeld } 22613c9168faSHans Rosenfeld 2262e984c70bSHans Rosenfeld return (ret); 22633c9168faSHans Rosenfeld } 22643c9168faSHans Rosenfeld 22653c9168faSHans Rosenfeld static int 2266*0999c112SPaul Winder nvme_create_completion_queue(nvme_t *nvme, nvme_cq_t *cq) 22673c9168faSHans Rosenfeld { 22683c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 22693c9168faSHans Rosenfeld nvme_create_queue_dw10_t dw10 = { 0 }; 22703c9168faSHans Rosenfeld nvme_create_cq_dw11_t c_dw11 = { 0 }; 2271e984c70bSHans Rosenfeld int ret; 22723c9168faSHans Rosenfeld 2273*0999c112SPaul Winder dw10.b.q_qid = cq->ncq_id; 2274*0999c112SPaul Winder dw10.b.q_qsize = cq->ncq_nentry - 1; 22753c9168faSHans Rosenfeld 22763c9168faSHans Rosenfeld c_dw11.b.cq_pc = 1; 22773c9168faSHans Rosenfeld c_dw11.b.cq_ien = 1; 2278*0999c112SPaul Winder c_dw11.b.cq_iv = cq->ncq_id % nvme->n_intr_cnt; 22793c9168faSHans Rosenfeld 22803c9168faSHans Rosenfeld cmd->nc_sqid = 0; 22813c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 22823c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE; 22833c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 22843c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = c_dw11.r; 2285*0999c112SPaul Winder cmd->nc_sqe.sqe_dptr.d_prp[0] = cq->ncq_dma->nd_cookie.dmac_laddress; 22863c9168faSHans Rosenfeld 2287e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 22883c9168faSHans Rosenfeld 2289e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 22903c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22913c9168faSHans Rosenfeld "!CREATE CQUEUE failed with sct = %x, sc = %x", 22923c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 22933c9168faSHans Rosenfeld } 22943c9168faSHans Rosenfeld 22953c9168faSHans Rosenfeld nvme_free_cmd(cmd); 22963c9168faSHans Rosenfeld 2297*0999c112SPaul Winder return (ret); 2298*0999c112SPaul Winder } 2299*0999c112SPaul Winder 2300*0999c112SPaul Winder static int 2301*0999c112SPaul Winder nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx) 2302*0999c112SPaul Winder { 2303*0999c112SPaul Winder nvme_cq_t *cq = qp->nq_cq; 2304*0999c112SPaul Winder nvme_cmd_t *cmd; 2305*0999c112SPaul Winder nvme_create_queue_dw10_t dw10 = { 0 }; 2306*0999c112SPaul Winder nvme_create_sq_dw11_t s_dw11 = { 0 }; 2307*0999c112SPaul Winder int ret; 2308*0999c112SPaul Winder 2309*0999c112SPaul Winder /* 2310*0999c112SPaul Winder * It is possible to have more qpairs than completion queues, 2311*0999c112SPaul Winder * and when the idx > ncq_id, that completion queue is shared 2312*0999c112SPaul Winder * and has already been created. 2313*0999c112SPaul Winder */ 2314*0999c112SPaul Winder if (idx <= cq->ncq_id && 2315*0999c112SPaul Winder nvme_create_completion_queue(nvme, cq) != DDI_SUCCESS) 2316*0999c112SPaul Winder return (DDI_FAILURE); 2317*0999c112SPaul Winder 2318*0999c112SPaul Winder dw10.b.q_qid = idx; 2319*0999c112SPaul Winder dw10.b.q_qsize = qp->nq_nentry - 1; 2320*0999c112SPaul Winder 23213c9168faSHans Rosenfeld s_dw11.b.sq_pc = 1; 2322*0999c112SPaul Winder s_dw11.b.sq_cqid = cq->ncq_id; 23233c9168faSHans Rosenfeld 23243c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 23253c9168faSHans Rosenfeld cmd->nc_sqid = 0; 23263c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 23273c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE; 23283c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 23293c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = s_dw11.r; 23303c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress; 23313c9168faSHans Rosenfeld 2332e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 23333c9168faSHans Rosenfeld 2334e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 23353c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23363c9168faSHans Rosenfeld "!CREATE SQUEUE failed with sct = %x, sc = %x", 23373c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 23383c9168faSHans Rosenfeld } 23393c9168faSHans Rosenfeld 23403c9168faSHans Rosenfeld nvme_free_cmd(cmd); 23413c9168faSHans Rosenfeld 2342e984c70bSHans Rosenfeld return (ret); 23433c9168faSHans Rosenfeld } 23443c9168faSHans Rosenfeld 23453c9168faSHans Rosenfeld static boolean_t 23463c9168faSHans Rosenfeld nvme_reset(nvme_t *nvme, boolean_t quiesce) 23473c9168faSHans Rosenfeld { 23483c9168faSHans Rosenfeld nvme_reg_csts_t csts; 23493c9168faSHans Rosenfeld int i; 23503c9168faSHans Rosenfeld 23513c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 23523c9168faSHans Rosenfeld 23533c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 23543c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) { 23553c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 23563c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 23573c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 23583c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) 23593c9168faSHans Rosenfeld break; 23603c9168faSHans Rosenfeld 23613c9168faSHans Rosenfeld if (quiesce) 23623c9168faSHans Rosenfeld drv_usecwait(50000); 23633c9168faSHans Rosenfeld else 23643c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 23653c9168faSHans Rosenfeld } 23663c9168faSHans Rosenfeld } 23673c9168faSHans Rosenfeld 23683c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, 0); 23693c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ASQ, 0); 23703c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ACQ, 0); 23713c9168faSHans Rosenfeld 23723c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 23733c9168faSHans Rosenfeld return (csts.b.csts_rdy == 0 ? B_TRUE : B_FALSE); 23743c9168faSHans Rosenfeld } 23753c9168faSHans Rosenfeld 23763c9168faSHans Rosenfeld static void 23773c9168faSHans Rosenfeld nvme_shutdown(nvme_t *nvme, int mode, boolean_t quiesce) 23783c9168faSHans Rosenfeld { 23793c9168faSHans Rosenfeld nvme_reg_cc_t cc; 23803c9168faSHans Rosenfeld nvme_reg_csts_t csts; 23813c9168faSHans Rosenfeld int i; 23823c9168faSHans Rosenfeld 23833c9168faSHans Rosenfeld ASSERT(mode == NVME_CC_SHN_NORMAL || mode == NVME_CC_SHN_ABRUPT); 23843c9168faSHans Rosenfeld 23853c9168faSHans Rosenfeld cc.r = nvme_get32(nvme, NVME_REG_CC); 23863c9168faSHans Rosenfeld cc.b.cc_shn = mode & 0x3; 23873c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 23883c9168faSHans Rosenfeld 23893c9168faSHans Rosenfeld for (i = 0; i != 10; i++) { 23903c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 23913c9168faSHans Rosenfeld if (csts.b.csts_shst == NVME_CSTS_SHN_COMPLETE) 23923c9168faSHans Rosenfeld break; 23933c9168faSHans Rosenfeld 23943c9168faSHans Rosenfeld if (quiesce) 23953c9168faSHans Rosenfeld drv_usecwait(100000); 23963c9168faSHans Rosenfeld else 23973c9168faSHans Rosenfeld delay(drv_usectohz(100000)); 23983c9168faSHans Rosenfeld } 23993c9168faSHans Rosenfeld } 24003c9168faSHans Rosenfeld 24013c9168faSHans Rosenfeld 24023c9168faSHans Rosenfeld static void 24033c9168faSHans Rosenfeld nvme_prepare_devid(nvme_t *nvme, uint32_t nsid) 24043c9168faSHans Rosenfeld { 240524979ca3SHans Rosenfeld /* 240624979ca3SHans Rosenfeld * Section 7.7 of the spec describes how to get a unique ID for 240724979ca3SHans Rosenfeld * the controller: the vendor ID, the model name and the serial 240824979ca3SHans Rosenfeld * number shall be unique when combined. 240924979ca3SHans Rosenfeld * 241024979ca3SHans Rosenfeld * If a namespace has no EUI64 we use the above and add the hex 241124979ca3SHans Rosenfeld * namespace ID to get a unique ID for the namespace. 241224979ca3SHans Rosenfeld */ 24133c9168faSHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 24143c9168faSHans Rosenfeld char serial[sizeof (nvme->n_idctl->id_serial) + 1]; 24153c9168faSHans Rosenfeld 24163c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 24173c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_serial, serial, 24183c9168faSHans Rosenfeld sizeof (nvme->n_idctl->id_serial)); 24193c9168faSHans Rosenfeld 24203c9168faSHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 24213c9168faSHans Rosenfeld serial[sizeof (nvme->n_idctl->id_serial)] = '\0'; 24223c9168faSHans Rosenfeld 242324979ca3SHans Rosenfeld nvme->n_ns[nsid - 1].ns_devid = kmem_asprintf("%4X-%s-%s-%X", 24243c9168faSHans Rosenfeld nvme->n_idctl->id_vid, model, serial, nsid); 24253c9168faSHans Rosenfeld } 24263c9168faSHans Rosenfeld 24273d9b1a2aSHans Rosenfeld static int 24283d9b1a2aSHans Rosenfeld nvme_init_ns(nvme_t *nvme, int nsid) 24293d9b1a2aSHans Rosenfeld { 24303d9b1a2aSHans Rosenfeld nvme_namespace_t *ns = &nvme->n_ns[nsid - 1]; 24313d9b1a2aSHans Rosenfeld nvme_identify_nsid_t *idns; 24323d9b1a2aSHans Rosenfeld int last_rp; 24333d9b1a2aSHans Rosenfeld 24343d9b1a2aSHans Rosenfeld ns->ns_nvme = nvme; 24353d9b1a2aSHans Rosenfeld 2436bc586359SRobert Mustacchi if (nvme_identify(nvme, B_FALSE, nsid, (void **)&idns) != 0) { 24373d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 24383d9b1a2aSHans Rosenfeld "!failed to identify namespace %d", nsid); 24393d9b1a2aSHans Rosenfeld return (DDI_FAILURE); 24403d9b1a2aSHans Rosenfeld } 24413d9b1a2aSHans Rosenfeld 24423d9b1a2aSHans Rosenfeld ns->ns_idns = idns; 24433d9b1a2aSHans Rosenfeld ns->ns_id = nsid; 24443d9b1a2aSHans Rosenfeld ns->ns_block_count = idns->id_nsize; 24453d9b1a2aSHans Rosenfeld ns->ns_block_size = 24463d9b1a2aSHans Rosenfeld 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads; 24473d9b1a2aSHans Rosenfeld ns->ns_best_block_size = ns->ns_block_size; 24483d9b1a2aSHans Rosenfeld 24493d9b1a2aSHans Rosenfeld /* 24503d9b1a2aSHans Rosenfeld * Get the EUI64 if present. Use it for devid and device node names. 24513d9b1a2aSHans Rosenfeld */ 24523d9b1a2aSHans Rosenfeld if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) 24533d9b1a2aSHans Rosenfeld bcopy(idns->id_eui64, ns->ns_eui64, sizeof (ns->ns_eui64)); 24543d9b1a2aSHans Rosenfeld 24553d9b1a2aSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 24563d9b1a2aSHans Rosenfeld if (*(uint64_t *)ns->ns_eui64 != 0) { 24573d9b1a2aSHans Rosenfeld uint8_t *eui64 = ns->ns_eui64; 24583d9b1a2aSHans Rosenfeld 24593d9b1a2aSHans Rosenfeld (void) snprintf(ns->ns_name, sizeof (ns->ns_name), 24603d9b1a2aSHans Rosenfeld "%02x%02x%02x%02x%02x%02x%02x%02x", 24613d9b1a2aSHans Rosenfeld eui64[0], eui64[1], eui64[2], eui64[3], 24623d9b1a2aSHans Rosenfeld eui64[4], eui64[5], eui64[6], eui64[7]); 24633d9b1a2aSHans Rosenfeld } else { 24643d9b1a2aSHans Rosenfeld (void) snprintf(ns->ns_name, sizeof (ns->ns_name), "%d", 24653d9b1a2aSHans Rosenfeld ns->ns_id); 24663d9b1a2aSHans Rosenfeld 24673d9b1a2aSHans Rosenfeld nvme_prepare_devid(nvme, ns->ns_id); 24683d9b1a2aSHans Rosenfeld } 24693d9b1a2aSHans Rosenfeld 24703d9b1a2aSHans Rosenfeld /* 24713d9b1a2aSHans Rosenfeld * Find the LBA format with no metadata and the best relative 24723d9b1a2aSHans Rosenfeld * performance. A value of 3 means "degraded", 0 is best. 24733d9b1a2aSHans Rosenfeld */ 24743d9b1a2aSHans Rosenfeld last_rp = 3; 24753d9b1a2aSHans Rosenfeld for (int j = 0; j <= idns->id_nlbaf; j++) { 24763d9b1a2aSHans Rosenfeld if (idns->id_lbaf[j].lbaf_lbads == 0) 24773d9b1a2aSHans Rosenfeld break; 24783d9b1a2aSHans Rosenfeld if (idns->id_lbaf[j].lbaf_ms != 0) 24793d9b1a2aSHans Rosenfeld continue; 24803d9b1a2aSHans Rosenfeld if (idns->id_lbaf[j].lbaf_rp >= last_rp) 24813d9b1a2aSHans Rosenfeld continue; 24823d9b1a2aSHans Rosenfeld last_rp = idns->id_lbaf[j].lbaf_rp; 24833d9b1a2aSHans Rosenfeld ns->ns_best_block_size = 24843d9b1a2aSHans Rosenfeld 1 << idns->id_lbaf[j].lbaf_lbads; 24853d9b1a2aSHans Rosenfeld } 24863d9b1a2aSHans Rosenfeld 24873d9b1a2aSHans Rosenfeld if (ns->ns_best_block_size < nvme->n_min_block_size) 24883d9b1a2aSHans Rosenfeld ns->ns_best_block_size = nvme->n_min_block_size; 24893d9b1a2aSHans Rosenfeld 24903d9b1a2aSHans Rosenfeld /* 24913d9b1a2aSHans Rosenfeld * We currently don't support namespaces that use either: 24923d9b1a2aSHans Rosenfeld * - protection information 2493621738e2SHans Rosenfeld * - illegal block size (< 512) 24943d9b1a2aSHans Rosenfeld */ 2495db083a4dSYuri Pankov if (idns->id_dps.dp_pinfo) { 24963d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2497db083a4dSYuri Pankov "!ignoring namespace %d, unsupported feature: " 2498db083a4dSYuri Pankov "pinfo = %d", nsid, idns->id_dps.dp_pinfo); 24993d9b1a2aSHans Rosenfeld ns->ns_ignore = B_TRUE; 2500621738e2SHans Rosenfeld } else if (ns->ns_block_size < 512) { 2501621738e2SHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2502621738e2SHans Rosenfeld "!ignoring namespace %d, unsupported block size %"PRIu64, 2503621738e2SHans Rosenfeld nsid, (uint64_t)ns->ns_block_size); 250431c62b25SHans Rosenfeld ns->ns_ignore = B_TRUE; 25053d9b1a2aSHans Rosenfeld } else { 25063d9b1a2aSHans Rosenfeld ns->ns_ignore = B_FALSE; 25073d9b1a2aSHans Rosenfeld } 25083d9b1a2aSHans Rosenfeld 25093d9b1a2aSHans Rosenfeld return (DDI_SUCCESS); 25103d9b1a2aSHans Rosenfeld } 25113d9b1a2aSHans Rosenfeld 25123c9168faSHans Rosenfeld static int 25133c9168faSHans Rosenfeld nvme_init(nvme_t *nvme) 25143c9168faSHans Rosenfeld { 25153c9168faSHans Rosenfeld nvme_reg_cc_t cc = { 0 }; 25163c9168faSHans Rosenfeld nvme_reg_aqa_t aqa = { 0 }; 25173c9168faSHans Rosenfeld nvme_reg_asq_t asq = { 0 }; 25183c9168faSHans Rosenfeld nvme_reg_acq_t acq = { 0 }; 25193c9168faSHans Rosenfeld nvme_reg_cap_t cap; 25203c9168faSHans Rosenfeld nvme_reg_vs_t vs; 25213c9168faSHans Rosenfeld nvme_reg_csts_t csts; 25223c9168faSHans Rosenfeld int i = 0; 2523e984c70bSHans Rosenfeld uint16_t nqueues; 2524510a6847SHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 2525510a6847SHans Rosenfeld char *vendor, *product; 25263c9168faSHans Rosenfeld 25273c9168faSHans Rosenfeld /* Check controller version */ 25283c9168faSHans Rosenfeld vs.r = nvme_get32(nvme, NVME_REG_VS); 252924979ca3SHans Rosenfeld nvme->n_version.v_major = vs.b.vs_mjr; 253024979ca3SHans Rosenfeld nvme->n_version.v_minor = vs.b.vs_mnr; 25313c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d", 253224979ca3SHans Rosenfeld nvme->n_version.v_major, nvme->n_version.v_minor); 25333c9168faSHans Rosenfeld 253448d370f1SRobert Mustacchi if (nvme->n_version.v_major > nvme_version_major) { 253548d370f1SRobert Mustacchi dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.x", 253648d370f1SRobert Mustacchi nvme_version_major); 25373c9168faSHans Rosenfeld if (nvme->n_strict_version) 25383c9168faSHans Rosenfeld goto fail; 25393c9168faSHans Rosenfeld } 25403c9168faSHans Rosenfeld 25413c9168faSHans Rosenfeld /* retrieve controller configuration */ 25423c9168faSHans Rosenfeld cap.r = nvme_get64(nvme, NVME_REG_CAP); 25433c9168faSHans Rosenfeld 25443c9168faSHans Rosenfeld if ((cap.b.cap_css & NVME_CAP_CSS_NVM) == 0) { 25453c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 25463c9168faSHans Rosenfeld "!NVM command set not supported by hardware"); 25473c9168faSHans Rosenfeld goto fail; 25483c9168faSHans Rosenfeld } 25493c9168faSHans Rosenfeld 25503c9168faSHans Rosenfeld nvme->n_nssr_supported = cap.b.cap_nssrs; 25513c9168faSHans Rosenfeld nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd; 25523c9168faSHans Rosenfeld nvme->n_timeout = cap.b.cap_to; 25533c9168faSHans Rosenfeld nvme->n_arbitration_mechanisms = cap.b.cap_ams; 25543c9168faSHans Rosenfeld nvme->n_cont_queues_reqd = cap.b.cap_cqr; 25553c9168faSHans Rosenfeld nvme->n_max_queue_entries = cap.b.cap_mqes + 1; 25563c9168faSHans Rosenfeld 25573c9168faSHans Rosenfeld /* 25583c9168faSHans Rosenfeld * The MPSMIN and MPSMAX fields in the CAP register use 0 to specify 25593c9168faSHans Rosenfeld * the base page size of 4k (1<<12), so add 12 here to get the real 25603c9168faSHans Rosenfeld * page size value. 25613c9168faSHans Rosenfeld */ 25623c9168faSHans Rosenfeld nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT), 25633c9168faSHans Rosenfeld cap.b.cap_mpsmax + 12); 25643c9168faSHans Rosenfeld nvme->n_pagesize = 1UL << (nvme->n_pageshift); 25653c9168faSHans Rosenfeld 25663c9168faSHans Rosenfeld /* 25673c9168faSHans Rosenfeld * Set up Queue DMA to transfer at least 1 page-aligned page at a time. 25683c9168faSHans Rosenfeld */ 25693c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize; 25703c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 25713c9168faSHans Rosenfeld 25723c9168faSHans Rosenfeld /* 25733c9168faSHans Rosenfeld * Set up PRP DMA to transfer 1 page-aligned page at a time. 25743c9168faSHans Rosenfeld * Maxxfer may be increased after we identified the controller limits. 25753c9168faSHans Rosenfeld */ 25763c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize; 25773c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 25783c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize; 25792f95345bSYouzhong Yang nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1; 25803c9168faSHans Rosenfeld 25813c9168faSHans Rosenfeld /* 25823c9168faSHans Rosenfeld * Reset controller if it's still in ready state. 25833c9168faSHans Rosenfeld */ 25843c9168faSHans Rosenfeld if (nvme_reset(nvme, B_FALSE) == B_FALSE) { 25853c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller"); 25863c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 25873c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 25883c9168faSHans Rosenfeld goto fail; 25893c9168faSHans Rosenfeld } 25903c9168faSHans Rosenfeld 2591*0999c112SPaul Winder /* 2592*0999c112SPaul Winder * Create the cq array with one completion queue to be assigned 2593*0999c112SPaul Winder * to the admin queue pair. 2594*0999c112SPaul Winder */ 2595*0999c112SPaul Winder if (nvme_create_cq_array(nvme, 1, nvme->n_admin_queue_len) != 2596*0999c112SPaul Winder DDI_SUCCESS) { 2597*0999c112SPaul Winder dev_err(nvme->n_dip, CE_WARN, 2598*0999c112SPaul Winder "!failed to pre-allocate admin completion queue"); 2599*0999c112SPaul Winder goto fail; 2600*0999c112SPaul Winder } 26013c9168faSHans Rosenfeld /* 26023c9168faSHans Rosenfeld * Create the admin queue pair. 26033c9168faSHans Rosenfeld */ 26043c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0) 26053c9168faSHans Rosenfeld != DDI_SUCCESS) { 26063c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 26073c9168faSHans Rosenfeld "!unable to allocate admin qpair"); 26083c9168faSHans Rosenfeld goto fail; 26093c9168faSHans Rosenfeld } 26103c9168faSHans Rosenfeld nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP); 26113c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 26123c9168faSHans Rosenfeld 26133c9168faSHans Rosenfeld nvme->n_progress |= NVME_ADMIN_QUEUE; 26143c9168faSHans Rosenfeld 26153c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 26163c9168faSHans Rosenfeld "admin-queue-len", nvme->n_admin_queue_len); 26173c9168faSHans Rosenfeld 26183c9168faSHans Rosenfeld aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1; 26193c9168faSHans Rosenfeld asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress; 2620*0999c112SPaul Winder acq = nvme->n_adminq->nq_cq->ncq_dma->nd_cookie.dmac_laddress; 26213c9168faSHans Rosenfeld 26223c9168faSHans Rosenfeld ASSERT((asq & (nvme->n_pagesize - 1)) == 0); 26233c9168faSHans Rosenfeld ASSERT((acq & (nvme->n_pagesize - 1)) == 0); 26243c9168faSHans Rosenfeld 26253c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, aqa.r); 26263c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ASQ, asq); 26273c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ACQ, acq); 26283c9168faSHans Rosenfeld 262934c938c7SPete Shephard cc.b.cc_ams = 0; /* use Round-Robin arbitration */ 263034c938c7SPete Shephard cc.b.cc_css = 0; /* use NVM command set */ 26313c9168faSHans Rosenfeld cc.b.cc_mps = nvme->n_pageshift - 12; 263234c938c7SPete Shephard cc.b.cc_shn = 0; /* no shutdown in progress */ 263334c938c7SPete Shephard cc.b.cc_en = 1; /* enable controller */ 263434c938c7SPete Shephard cc.b.cc_iosqes = 6; /* submission queue entry is 2^6 bytes long */ 263534c938c7SPete Shephard cc.b.cc_iocqes = 4; /* completion queue entry is 2^4 bytes long */ 26363c9168faSHans Rosenfeld 26373c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 26383c9168faSHans Rosenfeld 26393c9168faSHans Rosenfeld /* 26403c9168faSHans Rosenfeld * Wait for the controller to become ready. 26413c9168faSHans Rosenfeld */ 26423c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 26433c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 26443c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 26453c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 26463c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 26473c9168faSHans Rosenfeld 26483c9168faSHans Rosenfeld if (csts.b.csts_cfs == 1) { 26493c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 26503c9168faSHans Rosenfeld "!controller fatal status at init"); 26513c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, 26523c9168faSHans Rosenfeld DDI_SERVICE_LOST); 26533c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 26543c9168faSHans Rosenfeld goto fail; 26553c9168faSHans Rosenfeld } 26563c9168faSHans Rosenfeld 26573c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) 26583c9168faSHans Rosenfeld break; 26593c9168faSHans Rosenfeld } 26603c9168faSHans Rosenfeld } 26613c9168faSHans Rosenfeld 26623c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 26633c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!controller not ready"); 26643c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 26653c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 26663c9168faSHans Rosenfeld goto fail; 26673c9168faSHans Rosenfeld } 26683c9168faSHans Rosenfeld 26693c9168faSHans Rosenfeld /* 26703c9168faSHans Rosenfeld * Assume an abort command limit of 1. We'll destroy and re-init 26713c9168faSHans Rosenfeld * that later when we know the true abort command limit. 26723c9168faSHans Rosenfeld */ 26733c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL); 26743c9168faSHans Rosenfeld 2675bf26ea4bSHans Rosenfeld /* 2676bf26ea4bSHans Rosenfeld * Setup initial interrupt for admin queue. 2677bf26ea4bSHans Rosenfeld */ 2678bf26ea4bSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1) 2679bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 2680bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1) 2681bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 2682bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1) 2683bf26ea4bSHans Rosenfeld != DDI_SUCCESS)) { 2684bf26ea4bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2685bf26ea4bSHans Rosenfeld "!failed to setup initial interrupt"); 2686bf26ea4bSHans Rosenfeld goto fail; 2687bf26ea4bSHans Rosenfeld } 2688bf26ea4bSHans Rosenfeld 26893c9168faSHans Rosenfeld /* 26903c9168faSHans Rosenfeld * Post an asynchronous event command to catch errors. 269108139162SToomas Soome * We assume the asynchronous events are supported as required by 269208139162SToomas Soome * specification (Figure 40 in section 5 of NVMe 1.2). 269308139162SToomas Soome * However, since at least qemu does not follow the specification, 269408139162SToomas Soome * we need a mechanism to protect ourselves. 26953c9168faSHans Rosenfeld */ 269608139162SToomas Soome nvme->n_async_event_supported = B_TRUE; 26974b324362SHans Rosenfeld nvme_async_event(nvme); 26983c9168faSHans Rosenfeld 26993c9168faSHans Rosenfeld /* 27003c9168faSHans Rosenfeld * Identify Controller 27013c9168faSHans Rosenfeld */ 2702bc586359SRobert Mustacchi if (nvme_identify(nvme, B_FALSE, 0, (void **)&nvme->n_idctl) != 0) { 27033c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 27043c9168faSHans Rosenfeld "!failed to identify controller"); 27053c9168faSHans Rosenfeld goto fail; 27063c9168faSHans Rosenfeld } 27073c9168faSHans Rosenfeld 2708510a6847SHans Rosenfeld /* 2709510a6847SHans Rosenfeld * Get Vendor & Product ID 2710510a6847SHans Rosenfeld */ 2711510a6847SHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 2712510a6847SHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 2713510a6847SHans Rosenfeld sata_split_model(model, &vendor, &product); 2714510a6847SHans Rosenfeld 2715510a6847SHans Rosenfeld if (vendor == NULL) 2716510a6847SHans Rosenfeld nvme->n_vendor = strdup("NVMe"); 2717510a6847SHans Rosenfeld else 2718510a6847SHans Rosenfeld nvme->n_vendor = strdup(vendor); 2719510a6847SHans Rosenfeld 2720510a6847SHans Rosenfeld nvme->n_product = strdup(product); 2721510a6847SHans Rosenfeld 27223c9168faSHans Rosenfeld /* 27233c9168faSHans Rosenfeld * Get controller limits. 27243c9168faSHans Rosenfeld */ 27253c9168faSHans Rosenfeld nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT, 27263c9168faSHans Rosenfeld MIN(nvme->n_admin_queue_len / 10, 27273c9168faSHans Rosenfeld MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit))); 27283c9168faSHans Rosenfeld 27293c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 27303c9168faSHans Rosenfeld "async-event-limit", nvme->n_async_event_limit); 27313c9168faSHans Rosenfeld 27323c9168faSHans Rosenfeld nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1; 27333c9168faSHans Rosenfeld 27346afc9eb2SHans Rosenfeld /* 27356afc9eb2SHans Rosenfeld * Reinitialize the semaphore with the true abort command limit 27366afc9eb2SHans Rosenfeld * supported by the hardware. It's not necessary to disable interrupts 27376afc9eb2SHans Rosenfeld * as only command aborts use the semaphore, and no commands are 27386afc9eb2SHans Rosenfeld * executed or aborted while we're here. 27396afc9eb2SHans Rosenfeld */ 27403c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 27413c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL, 27423c9168faSHans Rosenfeld SEMA_DRIVER, NULL); 27433c9168faSHans Rosenfeld 27443c9168faSHans Rosenfeld nvme->n_progress |= NVME_CTRL_LIMITS; 27453c9168faSHans Rosenfeld 27463c9168faSHans Rosenfeld if (nvme->n_idctl->id_mdts == 0) 27473c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536; 27483c9168faSHans Rosenfeld else 27493c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = 27503c9168faSHans Rosenfeld 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts); 27513c9168faSHans Rosenfeld 27523c9168faSHans Rosenfeld nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1; 27533c9168faSHans Rosenfeld 27543c9168faSHans Rosenfeld /* 27553c9168faSHans Rosenfeld * Limit n_max_data_transfer_size to what we can handle in one PRP. 27563c9168faSHans Rosenfeld * Chained PRPs are currently unsupported. 27573c9168faSHans Rosenfeld * 27583c9168faSHans Rosenfeld * This is a no-op on hardware which doesn't support a transfer size 27593c9168faSHans Rosenfeld * big enough to require chained PRPs. 27603c9168faSHans Rosenfeld */ 27613c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size, 27623c9168faSHans Rosenfeld (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize)); 27633c9168faSHans Rosenfeld 27643c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size; 27653c9168faSHans Rosenfeld 27663c9168faSHans Rosenfeld /* 27673c9168faSHans Rosenfeld * Make sure the minimum/maximum queue entry sizes are not 27683c9168faSHans Rosenfeld * larger/smaller than the default. 27693c9168faSHans Rosenfeld */ 27703c9168faSHans Rosenfeld 27713c9168faSHans Rosenfeld if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) || 27723c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) || 27733c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) || 27743c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t))) 27753c9168faSHans Rosenfeld goto fail; 27763c9168faSHans Rosenfeld 27773c9168faSHans Rosenfeld /* 27783c9168faSHans Rosenfeld * Check for the presence of a Volatile Write Cache. If present, 2779d148d46eSHans Rosenfeld * enable or disable based on the value of the property 2780d148d46eSHans Rosenfeld * volatile-write-cache-enable (default is enabled). 27813c9168faSHans Rosenfeld */ 2782d148d46eSHans Rosenfeld nvme->n_write_cache_present = 2783d148d46eSHans Rosenfeld nvme->n_idctl->id_vwc.vwc_present == 0 ? B_FALSE : B_TRUE; 2784d148d46eSHans Rosenfeld 2785d148d46eSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2786d148d46eSHans Rosenfeld "volatile-write-cache-present", 2787d148d46eSHans Rosenfeld nvme->n_write_cache_present ? 1 : 0); 2788d148d46eSHans Rosenfeld 2789d148d46eSHans Rosenfeld if (!nvme->n_write_cache_present) { 2790d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = B_FALSE; 2791e984c70bSHans Rosenfeld } else if (nvme_write_cache_set(nvme, nvme->n_write_cache_enabled) 2792e984c70bSHans Rosenfeld != 0) { 2793d148d46eSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2794d148d46eSHans Rosenfeld "!failed to %sable volatile write cache", 2795d148d46eSHans Rosenfeld nvme->n_write_cache_enabled ? "en" : "dis"); 27963c9168faSHans Rosenfeld /* 2797d148d46eSHans Rosenfeld * Assume the cache is (still) enabled. 27983c9168faSHans Rosenfeld */ 2799d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = B_TRUE; 28003c9168faSHans Rosenfeld } 28013c9168faSHans Rosenfeld 2802d148d46eSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2803d148d46eSHans Rosenfeld "volatile-write-cache-enable", 2804d148d46eSHans Rosenfeld nvme->n_write_cache_enabled ? 1 : 0); 2805d148d46eSHans Rosenfeld 28063c9168faSHans Rosenfeld /* 28073d9b1a2aSHans Rosenfeld * Assume LBA Range Type feature is supported. If it isn't this 28083d9b1a2aSHans Rosenfeld * will be set to B_FALSE by nvme_get_features(). 28093c9168faSHans Rosenfeld */ 28103d9b1a2aSHans Rosenfeld nvme->n_lba_range_supported = B_TRUE; 28113d9b1a2aSHans Rosenfeld 28123d9b1a2aSHans Rosenfeld /* 28133d9b1a2aSHans Rosenfeld * Check support for Autonomous Power State Transition. 28143d9b1a2aSHans Rosenfeld */ 28153d9b1a2aSHans Rosenfeld if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) 28163d9b1a2aSHans Rosenfeld nvme->n_auto_pst_supported = 28173d9b1a2aSHans Rosenfeld nvme->n_idctl->id_apsta.ap_sup == 0 ? B_FALSE : B_TRUE; 28183c9168faSHans Rosenfeld 2819f313c178SYuri Pankov /* 2820f313c178SYuri Pankov * Assume Software Progress Marker feature is supported. If it isn't 2821f313c178SYuri Pankov * this will be set to B_FALSE by nvme_get_features(). 2822f313c178SYuri Pankov */ 2823f313c178SYuri Pankov nvme->n_progress_supported = B_TRUE; 2824f313c178SYuri Pankov 28253c9168faSHans Rosenfeld /* 28263c9168faSHans Rosenfeld * Identify Namespaces 28273c9168faSHans Rosenfeld */ 28283c9168faSHans Rosenfeld nvme->n_namespace_count = nvme->n_idctl->id_nn; 2829bf8c808aSYuri Pankov 2830bf8c808aSYuri Pankov if (nvme->n_namespace_count == 0) { 2831bf8c808aSYuri Pankov dev_err(nvme->n_dip, CE_WARN, 2832bf8c808aSYuri Pankov "!controllers without namespaces are not supported"); 2833bf8c808aSYuri Pankov goto fail; 2834bf8c808aSYuri Pankov } 2835bf8c808aSYuri Pankov 28363d9b1a2aSHans Rosenfeld if (nvme->n_namespace_count > NVME_MINOR_MAX) { 28373d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 28383d9b1a2aSHans Rosenfeld "!too many namespaces: %d, limiting to %d\n", 28393d9b1a2aSHans Rosenfeld nvme->n_namespace_count, NVME_MINOR_MAX); 28403d9b1a2aSHans Rosenfeld nvme->n_namespace_count = NVME_MINOR_MAX; 28413d9b1a2aSHans Rosenfeld } 28423d9b1a2aSHans Rosenfeld 28433c9168faSHans Rosenfeld nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) * 28443c9168faSHans Rosenfeld nvme->n_namespace_count, KM_SLEEP); 28453c9168faSHans Rosenfeld 28463c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 28473d9b1a2aSHans Rosenfeld mutex_init(&nvme->n_ns[i].ns_minor.nm_mutex, NULL, MUTEX_DRIVER, 28483d9b1a2aSHans Rosenfeld NULL); 28493d9b1a2aSHans Rosenfeld if (nvme_init_ns(nvme, i + 1) != DDI_SUCCESS) 28503c9168faSHans Rosenfeld goto fail; 28513c9168faSHans Rosenfeld } 28523c9168faSHans Rosenfeld 28533c9168faSHans Rosenfeld /* 28543c9168faSHans Rosenfeld * Try to set up MSI/MSI-X interrupts. 28553c9168faSHans Rosenfeld */ 28563c9168faSHans Rosenfeld if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX)) 28573c9168faSHans Rosenfeld != 0) { 28583c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 28593c9168faSHans Rosenfeld 28603c9168faSHans Rosenfeld nqueues = MIN(UINT16_MAX, ncpus); 28613c9168faSHans Rosenfeld 28623c9168faSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 28633c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS) && 28643c9168faSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 28653c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS)) { 28663c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 28673c9168faSHans Rosenfeld "!failed to setup MSI/MSI-X interrupts"); 28683c9168faSHans Rosenfeld goto fail; 28693c9168faSHans Rosenfeld } 28703c9168faSHans Rosenfeld } 28713c9168faSHans Rosenfeld 28723c9168faSHans Rosenfeld /* 28733c9168faSHans Rosenfeld * Create I/O queue pairs. 28743c9168faSHans Rosenfeld */ 2875e984c70bSHans Rosenfeld 2876*0999c112SPaul Winder if (nvme_set_nqueues(nvme) != 0) { 28773c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2878e984c70bSHans Rosenfeld "!failed to set number of I/O queues to %d", 2879e984c70bSHans Rosenfeld nvme->n_intr_cnt); 28803c9168faSHans Rosenfeld goto fail; 28813c9168faSHans Rosenfeld } 28823c9168faSHans Rosenfeld 28833c9168faSHans Rosenfeld /* 28843c9168faSHans Rosenfeld * Reallocate I/O queue array 28853c9168faSHans Rosenfeld */ 28863c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *)); 28873c9168faSHans Rosenfeld nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) * 2888*0999c112SPaul Winder (nvme->n_submission_queues + 1), KM_SLEEP); 28893c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 28903c9168faSHans Rosenfeld 2891*0999c112SPaul Winder /* 2892*0999c112SPaul Winder * There should always be at least as many submission queues 2893*0999c112SPaul Winder * as completion queues. 2894*0999c112SPaul Winder */ 2895*0999c112SPaul Winder ASSERT(nvme->n_submission_queues >= nvme->n_completion_queues); 2896*0999c112SPaul Winder 2897*0999c112SPaul Winder nvme->n_ioq_count = nvme->n_submission_queues; 2898*0999c112SPaul Winder 2899*0999c112SPaul Winder nvme->n_io_squeue_len = 2900*0999c112SPaul Winder MIN(nvme->n_io_squeue_len, nvme->n_max_queue_entries); 2901*0999c112SPaul Winder 2902*0999c112SPaul Winder (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-squeue-len", 2903*0999c112SPaul Winder nvme->n_io_squeue_len); 2904*0999c112SPaul Winder 2905*0999c112SPaul Winder /* 2906*0999c112SPaul Winder * Pre-allocate completion queues. 2907*0999c112SPaul Winder * When there are the same number of submission and completion 2908*0999c112SPaul Winder * queues there is no value in having a larger completion 2909*0999c112SPaul Winder * queue length. 2910*0999c112SPaul Winder */ 2911*0999c112SPaul Winder if (nvme->n_submission_queues == nvme->n_completion_queues) 2912*0999c112SPaul Winder nvme->n_io_cqueue_len = MIN(nvme->n_io_cqueue_len, 2913*0999c112SPaul Winder nvme->n_io_squeue_len); 2914*0999c112SPaul Winder 2915*0999c112SPaul Winder nvme->n_io_cqueue_len = MIN(nvme->n_io_cqueue_len, 2916*0999c112SPaul Winder nvme->n_max_queue_entries); 2917*0999c112SPaul Winder 2918*0999c112SPaul Winder (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-cqueue-len", 2919*0999c112SPaul Winder nvme->n_io_cqueue_len); 2920*0999c112SPaul Winder 2921*0999c112SPaul Winder if (nvme_create_cq_array(nvme, nvme->n_completion_queues + 1, 2922*0999c112SPaul Winder nvme->n_io_cqueue_len) != DDI_SUCCESS) { 2923*0999c112SPaul Winder dev_err(nvme->n_dip, CE_WARN, 2924*0999c112SPaul Winder "!failed to pre-allocate completion queues"); 2925*0999c112SPaul Winder goto fail; 2926*0999c112SPaul Winder } 2927e984c70bSHans Rosenfeld 29283c9168faSHans Rosenfeld /* 2929*0999c112SPaul Winder * If we use less completion queues than interrupt vectors return 29303c9168faSHans Rosenfeld * some of the interrupt vectors back to the system. 29313c9168faSHans Rosenfeld */ 2932*0999c112SPaul Winder if (nvme->n_completion_queues + 1 < nvme->n_intr_cnt) { 29333c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 29343c9168faSHans Rosenfeld 293534c938c7SPete Shephard if (nvme_setup_interrupts(nvme, nvme->n_intr_type, 2936*0999c112SPaul Winder nvme->n_completion_queues + 1) != DDI_SUCCESS) { 29373c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 29383c9168faSHans Rosenfeld "!failed to reduce number of interrupts"); 29393c9168faSHans Rosenfeld goto fail; 29403c9168faSHans Rosenfeld } 29413c9168faSHans Rosenfeld } 29423c9168faSHans Rosenfeld 29433c9168faSHans Rosenfeld /* 29443c9168faSHans Rosenfeld * Alloc & register I/O queue pairs 29453c9168faSHans Rosenfeld */ 29463c9168faSHans Rosenfeld 29473c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 2948*0999c112SPaul Winder if (nvme_alloc_qpair(nvme, nvme->n_io_squeue_len, 29493c9168faSHans Rosenfeld &nvme->n_ioq[i], i) != DDI_SUCCESS) { 29503c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 29513c9168faSHans Rosenfeld "!unable to allocate I/O qpair %d", i); 29523c9168faSHans Rosenfeld goto fail; 29533c9168faSHans Rosenfeld } 29543c9168faSHans Rosenfeld 2955e984c70bSHans Rosenfeld if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i) != 0) { 29563c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 29573c9168faSHans Rosenfeld "!unable to create I/O qpair %d", i); 29583c9168faSHans Rosenfeld goto fail; 29593c9168faSHans Rosenfeld } 29603c9168faSHans Rosenfeld } 29613c9168faSHans Rosenfeld 29623c9168faSHans Rosenfeld /* 29633c9168faSHans Rosenfeld * Post more asynchronous events commands to reduce event reporting 29643c9168faSHans Rosenfeld * latency as suggested by the spec. 29653c9168faSHans Rosenfeld */ 296608139162SToomas Soome if (nvme->n_async_event_supported) { 296708139162SToomas Soome for (i = 1; i != nvme->n_async_event_limit; i++) 296808139162SToomas Soome nvme_async_event(nvme); 296908139162SToomas Soome } 29703c9168faSHans Rosenfeld 29713c9168faSHans Rosenfeld return (DDI_SUCCESS); 29723c9168faSHans Rosenfeld 29733c9168faSHans Rosenfeld fail: 29743c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 29753c9168faSHans Rosenfeld return (DDI_FAILURE); 29763c9168faSHans Rosenfeld } 29773c9168faSHans Rosenfeld 29783c9168faSHans Rosenfeld static uint_t 29793c9168faSHans Rosenfeld nvme_intr(caddr_t arg1, caddr_t arg2) 29803c9168faSHans Rosenfeld { 29813c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 29823c9168faSHans Rosenfeld nvme_t *nvme = (nvme_t *)arg1; 29833c9168faSHans Rosenfeld int inum = (int)(uintptr_t)arg2; 2984bf26ea4bSHans Rosenfeld int ccnt = 0; 29853c9168faSHans Rosenfeld int qnum; 29863c9168faSHans Rosenfeld 29873c9168faSHans Rosenfeld if (inum >= nvme->n_intr_cnt) 29883c9168faSHans Rosenfeld return (DDI_INTR_UNCLAIMED); 29893c9168faSHans Rosenfeld 2990e984c70bSHans Rosenfeld if (nvme->n_dead) 2991e984c70bSHans Rosenfeld return (nvme->n_intr_type == DDI_INTR_TYPE_FIXED ? 2992e984c70bSHans Rosenfeld DDI_INTR_UNCLAIMED : DDI_INTR_CLAIMED); 2993e984c70bSHans Rosenfeld 29943c9168faSHans Rosenfeld /* 29953c9168faSHans Rosenfeld * The interrupt vector a queue uses is calculated as queue_idx % 29963c9168faSHans Rosenfeld * intr_cnt in nvme_create_io_qpair(). Iterate through the queue array 29973c9168faSHans Rosenfeld * in steps of n_intr_cnt to process all queues using this vector. 29983c9168faSHans Rosenfeld */ 29993c9168faSHans Rosenfeld for (qnum = inum; 3000*0999c112SPaul Winder qnum < nvme->n_cq_count && nvme->n_cq[qnum] != NULL; 30013c9168faSHans Rosenfeld qnum += nvme->n_intr_cnt) { 3002*0999c112SPaul Winder ccnt += nvme_process_iocq(nvme, nvme->n_cq[qnum]); 30033c9168faSHans Rosenfeld } 30043c9168faSHans Rosenfeld 3005bf26ea4bSHans Rosenfeld return (ccnt > 0 ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); 30063c9168faSHans Rosenfeld } 30073c9168faSHans Rosenfeld 30083c9168faSHans Rosenfeld static void 30096afc9eb2SHans Rosenfeld nvme_release_interrupts(nvme_t *nvme) 30103c9168faSHans Rosenfeld { 30113c9168faSHans Rosenfeld int i; 30123c9168faSHans Rosenfeld 30133c9168faSHans Rosenfeld for (i = 0; i < nvme->n_intr_cnt; i++) { 30143c9168faSHans Rosenfeld if (nvme->n_inth[i] == NULL) 30153c9168faSHans Rosenfeld break; 30163c9168faSHans Rosenfeld 30173c9168faSHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 30183c9168faSHans Rosenfeld (void) ddi_intr_block_disable(&nvme->n_inth[i], 1); 30193c9168faSHans Rosenfeld else 30203c9168faSHans Rosenfeld (void) ddi_intr_disable(nvme->n_inth[i]); 30213c9168faSHans Rosenfeld 30223c9168faSHans Rosenfeld (void) ddi_intr_remove_handler(nvme->n_inth[i]); 30233c9168faSHans Rosenfeld (void) ddi_intr_free(nvme->n_inth[i]); 30243c9168faSHans Rosenfeld } 30253c9168faSHans Rosenfeld 30263c9168faSHans Rosenfeld kmem_free(nvme->n_inth, nvme->n_inth_sz); 30273c9168faSHans Rosenfeld nvme->n_inth = NULL; 30283c9168faSHans Rosenfeld nvme->n_inth_sz = 0; 30293c9168faSHans Rosenfeld 30303c9168faSHans Rosenfeld nvme->n_progress &= ~NVME_INTERRUPTS; 30313c9168faSHans Rosenfeld } 30323c9168faSHans Rosenfeld 30333c9168faSHans Rosenfeld static int 30343c9168faSHans Rosenfeld nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs) 30353c9168faSHans Rosenfeld { 30363c9168faSHans Rosenfeld int nintrs, navail, count; 30373c9168faSHans Rosenfeld int ret; 30383c9168faSHans Rosenfeld int i; 30393c9168faSHans Rosenfeld 30403c9168faSHans Rosenfeld if (nvme->n_intr_types == 0) { 30413c9168faSHans Rosenfeld ret = ddi_intr_get_supported_types(nvme->n_dip, 30423c9168faSHans Rosenfeld &nvme->n_intr_types); 30433c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 30443c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 30453c9168faSHans Rosenfeld "!%s: ddi_intr_get_supported types failed", 30463c9168faSHans Rosenfeld __func__); 30473c9168faSHans Rosenfeld return (ret); 30483c9168faSHans Rosenfeld } 30499d08e1f8SHans Rosenfeld #ifdef __x86 30509d08e1f8SHans Rosenfeld if (get_hwenv() == HW_VMWARE) 30519d08e1f8SHans Rosenfeld nvme->n_intr_types &= ~DDI_INTR_TYPE_MSIX; 30529d08e1f8SHans Rosenfeld #endif 30533c9168faSHans Rosenfeld } 30543c9168faSHans Rosenfeld 30553c9168faSHans Rosenfeld if ((nvme->n_intr_types & intr_type) == 0) 30563c9168faSHans Rosenfeld return (DDI_FAILURE); 30573c9168faSHans Rosenfeld 30583c9168faSHans Rosenfeld ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs); 30593c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 30603c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed", 30613c9168faSHans Rosenfeld __func__); 30623c9168faSHans Rosenfeld return (ret); 30633c9168faSHans Rosenfeld } 30643c9168faSHans Rosenfeld 30653c9168faSHans Rosenfeld ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail); 30663c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 30673c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed", 30683c9168faSHans Rosenfeld __func__); 30693c9168faSHans Rosenfeld return (ret); 30703c9168faSHans Rosenfeld } 30713c9168faSHans Rosenfeld 30723c9168faSHans Rosenfeld /* We want at most one interrupt per queue pair. */ 30733c9168faSHans Rosenfeld if (navail > nqpairs) 30743c9168faSHans Rosenfeld navail = nqpairs; 30753c9168faSHans Rosenfeld 30763c9168faSHans Rosenfeld nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail; 30773c9168faSHans Rosenfeld nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP); 30783c9168faSHans Rosenfeld 30793c9168faSHans Rosenfeld ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail, 30803c9168faSHans Rosenfeld &count, 0); 30813c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 30823c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed", 30833c9168faSHans Rosenfeld __func__); 30843c9168faSHans Rosenfeld goto fail; 30853c9168faSHans Rosenfeld } 30863c9168faSHans Rosenfeld 30873c9168faSHans Rosenfeld nvme->n_intr_cnt = count; 30883c9168faSHans Rosenfeld 30893c9168faSHans Rosenfeld ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri); 30903c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 30913c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed", 30923c9168faSHans Rosenfeld __func__); 30933c9168faSHans Rosenfeld goto fail; 30943c9168faSHans Rosenfeld } 30953c9168faSHans Rosenfeld 30963c9168faSHans Rosenfeld for (i = 0; i < count; i++) { 30973c9168faSHans Rosenfeld ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr, 30983c9168faSHans Rosenfeld (void *)nvme, (void *)(uintptr_t)i); 30993c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 31003c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 31013c9168faSHans Rosenfeld "!%s: ddi_intr_add_handler failed", __func__); 31023c9168faSHans Rosenfeld goto fail; 31033c9168faSHans Rosenfeld } 31043c9168faSHans Rosenfeld } 31053c9168faSHans Rosenfeld 31063c9168faSHans Rosenfeld (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap); 31073c9168faSHans Rosenfeld 31086afc9eb2SHans Rosenfeld for (i = 0; i < count; i++) { 310975b41617SHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 311075b41617SHans Rosenfeld ret = ddi_intr_block_enable(&nvme->n_inth[i], 1); 311175b41617SHans Rosenfeld else 311275b41617SHans Rosenfeld ret = ddi_intr_enable(nvme->n_inth[i]); 31133c9168faSHans Rosenfeld 311475b41617SHans Rosenfeld if (ret != DDI_SUCCESS) { 311575b41617SHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 311675b41617SHans Rosenfeld "!%s: enabling interrupt %d failed", __func__, i); 311775b41617SHans Rosenfeld goto fail; 31186afc9eb2SHans Rosenfeld } 31196afc9eb2SHans Rosenfeld } 31206afc9eb2SHans Rosenfeld 31213c9168faSHans Rosenfeld nvme->n_intr_type = intr_type; 31223c9168faSHans Rosenfeld 31233c9168faSHans Rosenfeld nvme->n_progress |= NVME_INTERRUPTS; 31243c9168faSHans Rosenfeld 31253c9168faSHans Rosenfeld return (DDI_SUCCESS); 31263c9168faSHans Rosenfeld 31273c9168faSHans Rosenfeld fail: 31283c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 31293c9168faSHans Rosenfeld 31303c9168faSHans Rosenfeld return (ret); 31313c9168faSHans Rosenfeld } 31323c9168faSHans Rosenfeld 31333c9168faSHans Rosenfeld static int 31343c9168faSHans Rosenfeld nvme_fm_errcb(dev_info_t *dip, ddi_fm_error_t *fm_error, const void *arg) 31353c9168faSHans Rosenfeld { 31363c9168faSHans Rosenfeld _NOTE(ARGUNUSED(arg)); 31373c9168faSHans Rosenfeld 31383c9168faSHans Rosenfeld pci_ereport_post(dip, fm_error, NULL); 31393c9168faSHans Rosenfeld return (fm_error->fme_status); 31403c9168faSHans Rosenfeld } 31413c9168faSHans Rosenfeld 31423c9168faSHans Rosenfeld static int 31433c9168faSHans Rosenfeld nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 31443c9168faSHans Rosenfeld { 31453c9168faSHans Rosenfeld nvme_t *nvme; 31463c9168faSHans Rosenfeld int instance; 31473c9168faSHans Rosenfeld int nregs; 31483c9168faSHans Rosenfeld off_t regsize; 31493c9168faSHans Rosenfeld int i; 31503c9168faSHans Rosenfeld char name[32]; 31513c9168faSHans Rosenfeld 31523c9168faSHans Rosenfeld if (cmd != DDI_ATTACH) 31533c9168faSHans Rosenfeld return (DDI_FAILURE); 31543c9168faSHans Rosenfeld 31553c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 31563c9168faSHans Rosenfeld 31573c9168faSHans Rosenfeld if (ddi_soft_state_zalloc(nvme_state, instance) != DDI_SUCCESS) 31583c9168faSHans Rosenfeld return (DDI_FAILURE); 31593c9168faSHans Rosenfeld 31603c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 31613c9168faSHans Rosenfeld ddi_set_driver_private(dip, nvme); 31623c9168faSHans Rosenfeld nvme->n_dip = dip; 31633c9168faSHans Rosenfeld 31643d9b1a2aSHans Rosenfeld mutex_init(&nvme->n_minor.nm_mutex, NULL, MUTEX_DRIVER, NULL); 31653d9b1a2aSHans Rosenfeld 31663c9168faSHans Rosenfeld nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 31673c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "strict-version", 1) == 1 ? B_TRUE : B_FALSE; 31683c9168faSHans Rosenfeld nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY, 31693c9168faSHans Rosenfeld dip, DDI_PROP_DONTPASS, "ignore-unknown-vendor-status", 0) == 1 ? 31703c9168faSHans Rosenfeld B_TRUE : B_FALSE; 31713c9168faSHans Rosenfeld nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 31723c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "admin-queue-len", NVME_DEFAULT_ADMIN_QUEUE_LEN); 3173*0999c112SPaul Winder nvme->n_io_squeue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 3174*0999c112SPaul Winder DDI_PROP_DONTPASS, "io-squeue-len", NVME_DEFAULT_IO_QUEUE_LEN); 3175*0999c112SPaul Winder /* 3176*0999c112SPaul Winder * Double up the default for completion queues in case of 3177*0999c112SPaul Winder * queue sharing. 3178*0999c112SPaul Winder */ 3179*0999c112SPaul Winder nvme->n_io_cqueue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 3180*0999c112SPaul Winder DDI_PROP_DONTPASS, "io-cqueue-len", 2 * NVME_DEFAULT_IO_QUEUE_LEN); 31813c9168faSHans Rosenfeld nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 31823c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "async-event-limit", 31833c9168faSHans Rosenfeld NVME_DEFAULT_ASYNC_EVENT_LIMIT); 3184d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 3185d148d46eSHans Rosenfeld DDI_PROP_DONTPASS, "volatile-write-cache-enable", 1) != 0 ? 3186d148d46eSHans Rosenfeld B_TRUE : B_FALSE; 31876801591eSHans Rosenfeld nvme->n_min_block_size = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 31886801591eSHans Rosenfeld DDI_PROP_DONTPASS, "min-phys-block-size", 31896801591eSHans Rosenfeld NVME_DEFAULT_MIN_BLOCK_SIZE); 3190*0999c112SPaul Winder nvme->n_submission_queues = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 3191*0999c112SPaul Winder DDI_PROP_DONTPASS, "max-submission-queues", -1); 3192*0999c112SPaul Winder nvme->n_completion_queues = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 3193*0999c112SPaul Winder DDI_PROP_DONTPASS, "max-completion-queues", -1); 31946801591eSHans Rosenfeld 31956801591eSHans Rosenfeld if (!ISP2(nvme->n_min_block_size) || 31966801591eSHans Rosenfeld (nvme->n_min_block_size < NVME_DEFAULT_MIN_BLOCK_SIZE)) { 31976801591eSHans Rosenfeld dev_err(dip, CE_WARN, "!min-phys-block-size %s, " 31986801591eSHans Rosenfeld "using default %d", ISP2(nvme->n_min_block_size) ? 31996801591eSHans Rosenfeld "too low" : "not a power of 2", 32006801591eSHans Rosenfeld NVME_DEFAULT_MIN_BLOCK_SIZE); 32016801591eSHans Rosenfeld nvme->n_min_block_size = NVME_DEFAULT_MIN_BLOCK_SIZE; 32026801591eSHans Rosenfeld } 32033c9168faSHans Rosenfeld 3204*0999c112SPaul Winder if (nvme->n_submission_queues != -1 && 3205*0999c112SPaul Winder (nvme->n_submission_queues < 1 || 3206*0999c112SPaul Winder nvme->n_submission_queues > UINT16_MAX)) { 3207*0999c112SPaul Winder dev_err(dip, CE_WARN, "!\"submission-queues\"=%d is not " 3208*0999c112SPaul Winder "valid. Must be [1..%d]", nvme->n_submission_queues, 3209*0999c112SPaul Winder UINT16_MAX); 3210*0999c112SPaul Winder nvme->n_submission_queues = -1; 3211*0999c112SPaul Winder } 3212*0999c112SPaul Winder 3213*0999c112SPaul Winder if (nvme->n_completion_queues != -1 && 3214*0999c112SPaul Winder (nvme->n_completion_queues < 1 || 3215*0999c112SPaul Winder nvme->n_completion_queues > UINT16_MAX)) { 3216*0999c112SPaul Winder dev_err(dip, CE_WARN, "!\"completion-queues\"=%d is not " 3217*0999c112SPaul Winder "valid. Must be [1..%d]", nvme->n_completion_queues, 3218*0999c112SPaul Winder UINT16_MAX); 3219*0999c112SPaul Winder nvme->n_completion_queues = -1; 3220*0999c112SPaul Winder } 3221*0999c112SPaul Winder 32223c9168faSHans Rosenfeld if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN) 32233c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN; 32243c9168faSHans Rosenfeld else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN) 32253c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN; 32263c9168faSHans Rosenfeld 3227*0999c112SPaul Winder if (nvme->n_io_squeue_len < NVME_MIN_IO_QUEUE_LEN) 3228*0999c112SPaul Winder nvme->n_io_squeue_len = NVME_MIN_IO_QUEUE_LEN; 3229*0999c112SPaul Winder if (nvme->n_io_cqueue_len < NVME_MIN_IO_QUEUE_LEN) 3230*0999c112SPaul Winder nvme->n_io_cqueue_len = NVME_MIN_IO_QUEUE_LEN; 32313c9168faSHans Rosenfeld 32323c9168faSHans Rosenfeld if (nvme->n_async_event_limit < 1) 32333c9168faSHans Rosenfeld nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT; 32343c9168faSHans Rosenfeld 32353c9168faSHans Rosenfeld nvme->n_reg_acc_attr = nvme_reg_acc_attr; 32363c9168faSHans Rosenfeld nvme->n_queue_dma_attr = nvme_queue_dma_attr; 32373c9168faSHans Rosenfeld nvme->n_prp_dma_attr = nvme_prp_dma_attr; 32383c9168faSHans Rosenfeld nvme->n_sgl_dma_attr = nvme_sgl_dma_attr; 32393c9168faSHans Rosenfeld 32403c9168faSHans Rosenfeld /* 32413c9168faSHans Rosenfeld * Setup FMA support. 32423c9168faSHans Rosenfeld */ 32433c9168faSHans Rosenfeld nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip, 32443c9168faSHans Rosenfeld DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 32453c9168faSHans Rosenfeld DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 32463c9168faSHans Rosenfeld DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 32473c9168faSHans Rosenfeld 32483c9168faSHans Rosenfeld ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc); 32493c9168faSHans Rosenfeld 32503c9168faSHans Rosenfeld if (nvme->n_fm_cap) { 32513c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE) 32523c9168faSHans Rosenfeld nvme->n_reg_acc_attr.devacc_attr_access = 32533c9168faSHans Rosenfeld DDI_FLAGERR_ACC; 32543c9168faSHans Rosenfeld 32553c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) { 32563c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 32573c9168faSHans Rosenfeld nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 32583c9168faSHans Rosenfeld } 32593c9168faSHans Rosenfeld 32603c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 32613c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 32623c9168faSHans Rosenfeld pci_ereport_setup(dip); 32633c9168faSHans Rosenfeld 32643c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 32653c9168faSHans Rosenfeld ddi_fm_handler_register(dip, nvme_fm_errcb, 32663c9168faSHans Rosenfeld (void *)nvme); 32673c9168faSHans Rosenfeld } 32683c9168faSHans Rosenfeld 32693c9168faSHans Rosenfeld nvme->n_progress |= NVME_FMA_INIT; 32703c9168faSHans Rosenfeld 32713c9168faSHans Rosenfeld /* 32723c9168faSHans Rosenfeld * The spec defines several register sets. Only the controller 32733c9168faSHans Rosenfeld * registers (set 1) are currently used. 32743c9168faSHans Rosenfeld */ 32753c9168faSHans Rosenfeld if (ddi_dev_nregs(dip, &nregs) == DDI_FAILURE || 32763c9168faSHans Rosenfeld nregs < 2 || 32773c9168faSHans Rosenfeld ddi_dev_regsize(dip, 1, ®size) == DDI_FAILURE) 32783c9168faSHans Rosenfeld goto fail; 32793c9168faSHans Rosenfeld 32803c9168faSHans Rosenfeld if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize, 32813c9168faSHans Rosenfeld &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) { 32823c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to map regset 1"); 32833c9168faSHans Rosenfeld goto fail; 32843c9168faSHans Rosenfeld } 32853c9168faSHans Rosenfeld 32863c9168faSHans Rosenfeld nvme->n_progress |= NVME_REGS_MAPPED; 32873c9168faSHans Rosenfeld 32883c9168faSHans Rosenfeld /* 32893c9168faSHans Rosenfeld * Create taskq for command completion. 32903c9168faSHans Rosenfeld */ 32913c9168faSHans Rosenfeld (void) snprintf(name, sizeof (name), "%s%d_cmd_taskq", 32923c9168faSHans Rosenfeld ddi_driver_name(dip), ddi_get_instance(dip)); 32933c9168faSHans Rosenfeld nvme->n_cmd_taskq = ddi_taskq_create(dip, name, MIN(UINT16_MAX, ncpus), 32943c9168faSHans Rosenfeld TASKQ_DEFAULTPRI, 0); 32953c9168faSHans Rosenfeld if (nvme->n_cmd_taskq == NULL) { 32963c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to create cmd taskq"); 32973c9168faSHans Rosenfeld goto fail; 32983c9168faSHans Rosenfeld } 32993c9168faSHans Rosenfeld 33008834f7acSYouzhong Yang /* 33018834f7acSYouzhong Yang * Create PRP DMA cache 33028834f7acSYouzhong Yang */ 33038834f7acSYouzhong Yang (void) snprintf(name, sizeof (name), "%s%d_prp_cache", 33048834f7acSYouzhong Yang ddi_driver_name(dip), ddi_get_instance(dip)); 33058834f7acSYouzhong Yang nvme->n_prp_cache = kmem_cache_create(name, sizeof (nvme_dma_t), 33068834f7acSYouzhong Yang 0, nvme_prp_dma_constructor, nvme_prp_dma_destructor, 33078834f7acSYouzhong Yang NULL, (void *)nvme, NULL, 0); 33083c9168faSHans Rosenfeld 33093c9168faSHans Rosenfeld if (nvme_init(nvme) != DDI_SUCCESS) 33103c9168faSHans Rosenfeld goto fail; 33113c9168faSHans Rosenfeld 33123c9168faSHans Rosenfeld /* 33133c9168faSHans Rosenfeld * Attach the blkdev driver for each namespace. 33143c9168faSHans Rosenfeld */ 33153c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 33163d9b1a2aSHans Rosenfeld if (ddi_create_minor_node(nvme->n_dip, nvme->n_ns[i].ns_name, 33173d9b1a2aSHans Rosenfeld S_IFCHR, NVME_MINOR(ddi_get_instance(nvme->n_dip), i + 1), 33183d9b1a2aSHans Rosenfeld DDI_NT_NVME_ATTACHMENT_POINT, 0) != DDI_SUCCESS) { 33193d9b1a2aSHans Rosenfeld dev_err(dip, CE_WARN, 33203d9b1a2aSHans Rosenfeld "!failed to create minor node for namespace %d", i); 33213d9b1a2aSHans Rosenfeld goto fail; 33223d9b1a2aSHans Rosenfeld } 33233d9b1a2aSHans Rosenfeld 33243c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_ignore) 33253c9168faSHans Rosenfeld continue; 33263c9168faSHans Rosenfeld 33273c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl = bd_alloc_handle(&nvme->n_ns[i], 33283c9168faSHans Rosenfeld &nvme_bd_ops, &nvme->n_prp_dma_attr, KM_SLEEP); 33293c9168faSHans Rosenfeld 33303c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl == NULL) { 33313c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 33323c9168faSHans Rosenfeld "!failed to get blkdev handle for namespace %d", i); 33333c9168faSHans Rosenfeld goto fail; 33343c9168faSHans Rosenfeld } 33353c9168faSHans Rosenfeld 33363c9168faSHans Rosenfeld if (bd_attach_handle(dip, nvme->n_ns[i].ns_bd_hdl) 33373c9168faSHans Rosenfeld != DDI_SUCCESS) { 33383c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 33393c9168faSHans Rosenfeld "!failed to attach blkdev handle for namespace %d", 33403c9168faSHans Rosenfeld i); 33413c9168faSHans Rosenfeld goto fail; 33423c9168faSHans Rosenfeld } 33433c9168faSHans Rosenfeld } 33443c9168faSHans Rosenfeld 33453d9b1a2aSHans Rosenfeld if (ddi_create_minor_node(dip, "devctl", S_IFCHR, 33463d9b1a2aSHans Rosenfeld NVME_MINOR(ddi_get_instance(dip), 0), DDI_NT_NVME_NEXUS, 0) 33473d9b1a2aSHans Rosenfeld != DDI_SUCCESS) { 33483d9b1a2aSHans Rosenfeld dev_err(dip, CE_WARN, "nvme_attach: " 33493d9b1a2aSHans Rosenfeld "cannot create devctl minor node"); 33503d9b1a2aSHans Rosenfeld goto fail; 33513d9b1a2aSHans Rosenfeld } 33523d9b1a2aSHans Rosenfeld 33533c9168faSHans Rosenfeld return (DDI_SUCCESS); 33543c9168faSHans Rosenfeld 33553c9168faSHans Rosenfeld fail: 33563c9168faSHans Rosenfeld /* attach successful anyway so that FMA can retire the device */ 33573c9168faSHans Rosenfeld if (nvme->n_dead) 33583c9168faSHans Rosenfeld return (DDI_SUCCESS); 33593c9168faSHans Rosenfeld 33603c9168faSHans Rosenfeld (void) nvme_detach(dip, DDI_DETACH); 33613c9168faSHans Rosenfeld 33623c9168faSHans Rosenfeld return (DDI_FAILURE); 33633c9168faSHans Rosenfeld } 33643c9168faSHans Rosenfeld 33653c9168faSHans Rosenfeld static int 33663c9168faSHans Rosenfeld nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 33673c9168faSHans Rosenfeld { 33683c9168faSHans Rosenfeld int instance, i; 33693c9168faSHans Rosenfeld nvme_t *nvme; 33703c9168faSHans Rosenfeld 33713c9168faSHans Rosenfeld if (cmd != DDI_DETACH) 33723c9168faSHans Rosenfeld return (DDI_FAILURE); 33733c9168faSHans Rosenfeld 33743c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 33753c9168faSHans Rosenfeld 33763c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 33773c9168faSHans Rosenfeld 33783c9168faSHans Rosenfeld if (nvme == NULL) 33793c9168faSHans Rosenfeld return (DDI_FAILURE); 33803c9168faSHans Rosenfeld 33813d9b1a2aSHans Rosenfeld ddi_remove_minor_node(dip, "devctl"); 33823d9b1a2aSHans Rosenfeld mutex_destroy(&nvme->n_minor.nm_mutex); 33833d9b1a2aSHans Rosenfeld 33843c9168faSHans Rosenfeld if (nvme->n_ns) { 33853c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 33863d9b1a2aSHans Rosenfeld ddi_remove_minor_node(dip, nvme->n_ns[i].ns_name); 33873d9b1a2aSHans Rosenfeld mutex_destroy(&nvme->n_ns[i].ns_minor.nm_mutex); 33883d9b1a2aSHans Rosenfeld 33893c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl) { 33903c9168faSHans Rosenfeld (void) bd_detach_handle( 33913c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl); 33923c9168faSHans Rosenfeld bd_free_handle(nvme->n_ns[i].ns_bd_hdl); 33933c9168faSHans Rosenfeld } 33943c9168faSHans Rosenfeld 33953c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_idns) 33963c9168faSHans Rosenfeld kmem_free(nvme->n_ns[i].ns_idns, 33973c9168faSHans Rosenfeld sizeof (nvme_identify_nsid_t)); 339824979ca3SHans Rosenfeld if (nvme->n_ns[i].ns_devid) 339924979ca3SHans Rosenfeld strfree(nvme->n_ns[i].ns_devid); 34003c9168faSHans Rosenfeld } 34013c9168faSHans Rosenfeld 34023c9168faSHans Rosenfeld kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) * 34033c9168faSHans Rosenfeld nvme->n_namespace_count); 34043c9168faSHans Rosenfeld } 34053c9168faSHans Rosenfeld 34063c9168faSHans Rosenfeld if (nvme->n_progress & NVME_INTERRUPTS) 34073c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 34083c9168faSHans Rosenfeld 34093c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 34103c9168faSHans Rosenfeld ddi_taskq_wait(nvme->n_cmd_taskq); 34113c9168faSHans Rosenfeld 34123c9168faSHans Rosenfeld if (nvme->n_ioq_count > 0) { 34133c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 34143c9168faSHans Rosenfeld if (nvme->n_ioq[i] != NULL) { 34153c9168faSHans Rosenfeld /* TODO: send destroy queue commands */ 34163c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_ioq[i]); 34173c9168faSHans Rosenfeld } 34183c9168faSHans Rosenfeld } 34193c9168faSHans Rosenfeld 34203c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) * 34213c9168faSHans Rosenfeld (nvme->n_ioq_count + 1)); 34223c9168faSHans Rosenfeld } 34233c9168faSHans Rosenfeld 34248834f7acSYouzhong Yang if (nvme->n_prp_cache != NULL) { 34258834f7acSYouzhong Yang kmem_cache_destroy(nvme->n_prp_cache); 34268834f7acSYouzhong Yang } 34278834f7acSYouzhong Yang 34283c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) { 34293c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE); 34303c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 34313c9168faSHans Rosenfeld } 34323c9168faSHans Rosenfeld 34333c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 34343c9168faSHans Rosenfeld ddi_taskq_destroy(nvme->n_cmd_taskq); 34353c9168faSHans Rosenfeld 34363c9168faSHans Rosenfeld if (nvme->n_progress & NVME_CTRL_LIMITS) 34373c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 34383c9168faSHans Rosenfeld 34393c9168faSHans Rosenfeld if (nvme->n_progress & NVME_ADMIN_QUEUE) 34403c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_adminq); 34413c9168faSHans Rosenfeld 3442*0999c112SPaul Winder if (nvme->n_cq_count > 0) { 3443*0999c112SPaul Winder nvme_destroy_cq_array(nvme, 0); 3444*0999c112SPaul Winder nvme->n_cq = NULL; 3445*0999c112SPaul Winder nvme->n_cq_count = 0; 3446*0999c112SPaul Winder } 3447*0999c112SPaul Winder 34483c9168faSHans Rosenfeld if (nvme->n_idctl) 34493d9b1a2aSHans Rosenfeld kmem_free(nvme->n_idctl, NVME_IDENTIFY_BUFSIZE); 34503c9168faSHans Rosenfeld 34513c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) 34523c9168faSHans Rosenfeld ddi_regs_map_free(&nvme->n_regh); 34533c9168faSHans Rosenfeld 34543c9168faSHans Rosenfeld if (nvme->n_progress & NVME_FMA_INIT) { 34553c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 34563c9168faSHans Rosenfeld ddi_fm_handler_unregister(nvme->n_dip); 34573c9168faSHans Rosenfeld 34583c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 34593c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 34603c9168faSHans Rosenfeld pci_ereport_teardown(nvme->n_dip); 34613c9168faSHans Rosenfeld 34623c9168faSHans Rosenfeld ddi_fm_fini(nvme->n_dip); 34633c9168faSHans Rosenfeld } 34643c9168faSHans Rosenfeld 3465510a6847SHans Rosenfeld if (nvme->n_vendor != NULL) 3466510a6847SHans Rosenfeld strfree(nvme->n_vendor); 3467510a6847SHans Rosenfeld 3468510a6847SHans Rosenfeld if (nvme->n_product != NULL) 3469510a6847SHans Rosenfeld strfree(nvme->n_product); 3470510a6847SHans Rosenfeld 34713c9168faSHans Rosenfeld ddi_soft_state_free(nvme_state, instance); 34723c9168faSHans Rosenfeld 34733c9168faSHans Rosenfeld return (DDI_SUCCESS); 34743c9168faSHans Rosenfeld } 34753c9168faSHans Rosenfeld 34763c9168faSHans Rosenfeld static int 34773c9168faSHans Rosenfeld nvme_quiesce(dev_info_t *dip) 34783c9168faSHans Rosenfeld { 34793c9168faSHans Rosenfeld int instance; 34803c9168faSHans Rosenfeld nvme_t *nvme; 34813c9168faSHans Rosenfeld 34823c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 34833c9168faSHans Rosenfeld 34843c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 34853c9168faSHans Rosenfeld 34863c9168faSHans Rosenfeld if (nvme == NULL) 34873c9168faSHans Rosenfeld return (DDI_FAILURE); 34883c9168faSHans Rosenfeld 34893c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_ABRUPT, B_TRUE); 34903c9168faSHans Rosenfeld 34913c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_TRUE); 34923c9168faSHans Rosenfeld 34933c9168faSHans Rosenfeld return (DDI_FAILURE); 34943c9168faSHans Rosenfeld } 34953c9168faSHans Rosenfeld 34963c9168faSHans Rosenfeld static int 34973c9168faSHans Rosenfeld nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer) 34983c9168faSHans Rosenfeld { 34993c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 35003c9168faSHans Rosenfeld int nprp_page, nprp; 35013c9168faSHans Rosenfeld uint64_t *prp; 35023c9168faSHans Rosenfeld 35033c9168faSHans Rosenfeld if (xfer->x_ndmac == 0) 35043c9168faSHans Rosenfeld return (DDI_FAILURE); 35053c9168faSHans Rosenfeld 35063c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress; 35073c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 35083c9168faSHans Rosenfeld 35093c9168faSHans Rosenfeld if (xfer->x_ndmac == 1) { 35103c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; 35113c9168faSHans Rosenfeld return (DDI_SUCCESS); 35123c9168faSHans Rosenfeld } else if (xfer->x_ndmac == 2) { 35133c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress; 35143c9168faSHans Rosenfeld return (DDI_SUCCESS); 35153c9168faSHans Rosenfeld } 35163c9168faSHans Rosenfeld 35173c9168faSHans Rosenfeld xfer->x_ndmac--; 35183c9168faSHans Rosenfeld 3519*0999c112SPaul Winder nprp_page = nvme->n_pagesize / sizeof (uint64_t); 35203c9168faSHans Rosenfeld ASSERT(nprp_page > 0); 35213c9168faSHans Rosenfeld nprp = (xfer->x_ndmac + nprp_page - 1) / nprp_page; 35223c9168faSHans Rosenfeld 35233c9168faSHans Rosenfeld /* 35243c9168faSHans Rosenfeld * We currently don't support chained PRPs and set up our DMA 35253c9168faSHans Rosenfeld * attributes to reflect that. If we still get an I/O request 35263c9168faSHans Rosenfeld * that needs a chained PRP something is very wrong. 35273c9168faSHans Rosenfeld */ 35283c9168faSHans Rosenfeld VERIFY(nprp == 1); 35293c9168faSHans Rosenfeld 35308834f7acSYouzhong Yang cmd->nc_dma = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP); 35318834f7acSYouzhong Yang bzero(cmd->nc_dma->nd_memp, cmd->nc_dma->nd_len); 35323c9168faSHans Rosenfeld 35333c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress; 35343c9168faSHans Rosenfeld 35353c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 35363c9168faSHans Rosenfeld for (prp = (uint64_t *)cmd->nc_dma->nd_memp; 35373c9168faSHans Rosenfeld xfer->x_ndmac > 0; 35383c9168faSHans Rosenfeld prp++, xfer->x_ndmac--) { 35393c9168faSHans Rosenfeld *prp = xfer->x_dmac.dmac_laddress; 35403c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 35413c9168faSHans Rosenfeld } 35423c9168faSHans Rosenfeld 35433c9168faSHans Rosenfeld (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len, 35443c9168faSHans Rosenfeld DDI_DMA_SYNC_FORDEV); 35453c9168faSHans Rosenfeld return (DDI_SUCCESS); 35463c9168faSHans Rosenfeld } 35473c9168faSHans Rosenfeld 35483c9168faSHans Rosenfeld static nvme_cmd_t * 35493c9168faSHans Rosenfeld nvme_create_nvm_cmd(nvme_namespace_t *ns, uint8_t opc, bd_xfer_t *xfer) 35503c9168faSHans Rosenfeld { 35513c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 35523c9168faSHans Rosenfeld nvme_cmd_t *cmd; 35533c9168faSHans Rosenfeld 35543c9168faSHans Rosenfeld /* 35553c9168faSHans Rosenfeld * Blkdev only sets BD_XFER_POLL when dumping, so don't sleep. 35563c9168faSHans Rosenfeld */ 35573c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ? 35583c9168faSHans Rosenfeld KM_NOSLEEP : KM_SLEEP); 35593c9168faSHans Rosenfeld 35603c9168faSHans Rosenfeld if (cmd == NULL) 35613c9168faSHans Rosenfeld return (NULL); 35623c9168faSHans Rosenfeld 35633c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = opc; 35643c9168faSHans Rosenfeld cmd->nc_callback = nvme_bd_xfer_done; 35653c9168faSHans Rosenfeld cmd->nc_xfer = xfer; 35663c9168faSHans Rosenfeld 35673c9168faSHans Rosenfeld switch (opc) { 35683c9168faSHans Rosenfeld case NVME_OPC_NVM_WRITE: 35693c9168faSHans Rosenfeld case NVME_OPC_NVM_READ: 35703c9168faSHans Rosenfeld VERIFY(xfer->x_nblks <= 0x10000); 35713c9168faSHans Rosenfeld 35723c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 35733c9168faSHans Rosenfeld 35743c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu; 35753c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32); 35763c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1); 35773c9168faSHans Rosenfeld 35783c9168faSHans Rosenfeld if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS) 35793c9168faSHans Rosenfeld goto fail; 35803c9168faSHans Rosenfeld break; 35813c9168faSHans Rosenfeld 35823c9168faSHans Rosenfeld case NVME_OPC_NVM_FLUSH: 35833c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 35843c9168faSHans Rosenfeld break; 35853c9168faSHans Rosenfeld 35863c9168faSHans Rosenfeld default: 35873c9168faSHans Rosenfeld goto fail; 35883c9168faSHans Rosenfeld } 35893c9168faSHans Rosenfeld 35903c9168faSHans Rosenfeld return (cmd); 35913c9168faSHans Rosenfeld 35923c9168faSHans Rosenfeld fail: 35933c9168faSHans Rosenfeld nvme_free_cmd(cmd); 35943c9168faSHans Rosenfeld return (NULL); 35953c9168faSHans Rosenfeld } 35963c9168faSHans Rosenfeld 35973c9168faSHans Rosenfeld static void 35983c9168faSHans Rosenfeld nvme_bd_xfer_done(void *arg) 35993c9168faSHans Rosenfeld { 36003c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 36013c9168faSHans Rosenfeld bd_xfer_t *xfer = cmd->nc_xfer; 36023c9168faSHans Rosenfeld int error = 0; 36033c9168faSHans Rosenfeld 36043c9168faSHans Rosenfeld error = nvme_check_cmd_status(cmd); 36053c9168faSHans Rosenfeld nvme_free_cmd(cmd); 36063c9168faSHans Rosenfeld 36073c9168faSHans Rosenfeld bd_xfer_done(xfer, error); 36083c9168faSHans Rosenfeld } 36093c9168faSHans Rosenfeld 36103c9168faSHans Rosenfeld static void 36113c9168faSHans Rosenfeld nvme_bd_driveinfo(void *arg, bd_drive_t *drive) 36123c9168faSHans Rosenfeld { 36133c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 36143c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 36153c9168faSHans Rosenfeld 36163c9168faSHans Rosenfeld /* 36173c9168faSHans Rosenfeld * blkdev maintains one queue size per instance (namespace), 36183c9168faSHans Rosenfeld * but all namespace share the I/O queues. 36193c9168faSHans Rosenfeld * TODO: need to figure out a sane default, or use per-NS I/O queues, 36203c9168faSHans Rosenfeld * or change blkdev to handle EAGAIN 36213c9168faSHans Rosenfeld */ 3622*0999c112SPaul Winder drive->d_qsize = nvme->n_ioq_count * nvme->n_io_squeue_len 36233c9168faSHans Rosenfeld / nvme->n_namespace_count; 36243c9168faSHans Rosenfeld 36253c9168faSHans Rosenfeld /* 36263c9168faSHans Rosenfeld * d_maxxfer is not set, which means the value is taken from the DMA 36273c9168faSHans Rosenfeld * attributes specified to bd_alloc_handle. 36283c9168faSHans Rosenfeld */ 36293c9168faSHans Rosenfeld 36303c9168faSHans Rosenfeld drive->d_removable = B_FALSE; 36313c9168faSHans Rosenfeld drive->d_hotpluggable = B_FALSE; 36323c9168faSHans Rosenfeld 363324979ca3SHans Rosenfeld bcopy(ns->ns_eui64, drive->d_eui64, sizeof (drive->d_eui64)); 36343c9168faSHans Rosenfeld drive->d_target = ns->ns_id; 36353c9168faSHans Rosenfeld drive->d_lun = 0; 3636510a6847SHans Rosenfeld 3637bef9e21aSHans Rosenfeld drive->d_model = nvme->n_idctl->id_model; 3638bef9e21aSHans Rosenfeld drive->d_model_len = sizeof (nvme->n_idctl->id_model); 3639510a6847SHans Rosenfeld drive->d_vendor = nvme->n_vendor; 3640510a6847SHans Rosenfeld drive->d_vendor_len = strlen(nvme->n_vendor); 3641510a6847SHans Rosenfeld drive->d_product = nvme->n_product; 3642510a6847SHans Rosenfeld drive->d_product_len = strlen(nvme->n_product); 3643510a6847SHans Rosenfeld drive->d_serial = nvme->n_idctl->id_serial; 3644510a6847SHans Rosenfeld drive->d_serial_len = sizeof (nvme->n_idctl->id_serial); 3645510a6847SHans Rosenfeld drive->d_revision = nvme->n_idctl->id_fwrev; 3646510a6847SHans Rosenfeld drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev); 36473c9168faSHans Rosenfeld } 36483c9168faSHans Rosenfeld 36493c9168faSHans Rosenfeld static int 36503c9168faSHans Rosenfeld nvme_bd_mediainfo(void *arg, bd_media_t *media) 36513c9168faSHans Rosenfeld { 36523c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 36533c9168faSHans Rosenfeld 36543c9168faSHans Rosenfeld media->m_nblks = ns->ns_block_count; 36553c9168faSHans Rosenfeld media->m_blksize = ns->ns_block_size; 36563c9168faSHans Rosenfeld media->m_readonly = B_FALSE; 36573c9168faSHans Rosenfeld media->m_solidstate = B_TRUE; 36583c9168faSHans Rosenfeld 36593c9168faSHans Rosenfeld media->m_pblksize = ns->ns_best_block_size; 36603c9168faSHans Rosenfeld 36613c9168faSHans Rosenfeld return (0); 36623c9168faSHans Rosenfeld } 36633c9168faSHans Rosenfeld 36643c9168faSHans Rosenfeld static int 36653c9168faSHans Rosenfeld nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc) 36663c9168faSHans Rosenfeld { 36673c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 36684b324362SHans Rosenfeld nvme_cmd_t *cmd; 36694ac9cfccSHans Rosenfeld nvme_qpair_t *ioq; 36704ac9cfccSHans Rosenfeld boolean_t poll; 36714b324362SHans Rosenfeld int ret; 36723c9168faSHans Rosenfeld 36733c9168faSHans Rosenfeld if (nvme->n_dead) 36743c9168faSHans Rosenfeld return (EIO); 36753c9168faSHans Rosenfeld 36763c9168faSHans Rosenfeld cmd = nvme_create_nvm_cmd(ns, opc, xfer); 36773c9168faSHans Rosenfeld if (cmd == NULL) 36783c9168faSHans Rosenfeld return (ENOMEM); 36793c9168faSHans Rosenfeld 36803c9168faSHans Rosenfeld cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1; 36813c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); 36824ac9cfccSHans Rosenfeld ioq = nvme->n_ioq[cmd->nc_sqid]; 36834ac9cfccSHans Rosenfeld 36844ac9cfccSHans Rosenfeld /* 36854ac9cfccSHans Rosenfeld * Get the polling flag before submitting the command. The command may 36864ac9cfccSHans Rosenfeld * complete immediately after it was submitted, which means we must 36874ac9cfccSHans Rosenfeld * treat both cmd and xfer as if they have been freed already. 36884ac9cfccSHans Rosenfeld */ 36894ac9cfccSHans Rosenfeld poll = (xfer->x_flags & BD_XFER_POLL) != 0; 36903c9168faSHans Rosenfeld 36914b324362SHans Rosenfeld ret = nvme_submit_io_cmd(ioq, cmd); 36924b324362SHans Rosenfeld 36934b324362SHans Rosenfeld if (ret != 0) 36944b324362SHans Rosenfeld return (ret); 36953c9168faSHans Rosenfeld 36964ac9cfccSHans Rosenfeld if (!poll) 36974ac9cfccSHans Rosenfeld return (0); 36984ac9cfccSHans Rosenfeld 36994ac9cfccSHans Rosenfeld do { 37004b324362SHans Rosenfeld cmd = nvme_retrieve_cmd(nvme, ioq); 37014b324362SHans Rosenfeld if (cmd != NULL) 3702*0999c112SPaul Winder cmd->nc_callback(cmd); 37034ac9cfccSHans Rosenfeld else 37044ac9cfccSHans Rosenfeld drv_usecwait(10); 37054ac9cfccSHans Rosenfeld } while (ioq->nq_active_cmds != 0); 37064ac9cfccSHans Rosenfeld 37073c9168faSHans Rosenfeld return (0); 37083c9168faSHans Rosenfeld } 37093c9168faSHans Rosenfeld 37103c9168faSHans Rosenfeld static int 37113c9168faSHans Rosenfeld nvme_bd_read(void *arg, bd_xfer_t *xfer) 37123c9168faSHans Rosenfeld { 37133c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 37143c9168faSHans Rosenfeld 37153c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_READ)); 37163c9168faSHans Rosenfeld } 37173c9168faSHans Rosenfeld 37183c9168faSHans Rosenfeld static int 37193c9168faSHans Rosenfeld nvme_bd_write(void *arg, bd_xfer_t *xfer) 37203c9168faSHans Rosenfeld { 37213c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 37223c9168faSHans Rosenfeld 37233c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_WRITE)); 37243c9168faSHans Rosenfeld } 37253c9168faSHans Rosenfeld 37263c9168faSHans Rosenfeld static int 37273c9168faSHans Rosenfeld nvme_bd_sync(void *arg, bd_xfer_t *xfer) 37283c9168faSHans Rosenfeld { 37293c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 37303c9168faSHans Rosenfeld 37313c9168faSHans Rosenfeld if (ns->ns_nvme->n_dead) 37323c9168faSHans Rosenfeld return (EIO); 37333c9168faSHans Rosenfeld 37343c9168faSHans Rosenfeld /* 3735d148d46eSHans Rosenfeld * If the volatile write cache is not present or not enabled the FLUSH 3736d148d46eSHans Rosenfeld * command is a no-op, so we can take a shortcut here. 37373c9168faSHans Rosenfeld */ 3738d148d46eSHans Rosenfeld if (!ns->ns_nvme->n_write_cache_present) { 37393c9168faSHans Rosenfeld bd_xfer_done(xfer, ENOTSUP); 37403c9168faSHans Rosenfeld return (0); 37413c9168faSHans Rosenfeld } 37423c9168faSHans Rosenfeld 3743d148d46eSHans Rosenfeld if (!ns->ns_nvme->n_write_cache_enabled) { 3744d148d46eSHans Rosenfeld bd_xfer_done(xfer, 0); 3745d148d46eSHans Rosenfeld return (0); 3746d148d46eSHans Rosenfeld } 3747d148d46eSHans Rosenfeld 37483c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_FLUSH)); 37493c9168faSHans Rosenfeld } 37503c9168faSHans Rosenfeld 37513c9168faSHans Rosenfeld static int 37523c9168faSHans Rosenfeld nvme_bd_devid(void *arg, dev_info_t *devinfo, ddi_devid_t *devid) 37533c9168faSHans Rosenfeld { 37543c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 37553c9168faSHans Rosenfeld 375624979ca3SHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 375724979ca3SHans Rosenfeld if (*(uint64_t *)ns->ns_eui64 != 0) { 375824979ca3SHans Rosenfeld return (ddi_devid_init(devinfo, DEVID_SCSI3_WWN, 375924979ca3SHans Rosenfeld sizeof (ns->ns_eui64), ns->ns_eui64, devid)); 376024979ca3SHans Rosenfeld } else { 376124979ca3SHans Rosenfeld return (ddi_devid_init(devinfo, DEVID_ENCAP, 376224979ca3SHans Rosenfeld strlen(ns->ns_devid), ns->ns_devid, devid)); 376324979ca3SHans Rosenfeld } 37643c9168faSHans Rosenfeld } 37653d9b1a2aSHans Rosenfeld 37663d9b1a2aSHans Rosenfeld static int 37673d9b1a2aSHans Rosenfeld nvme_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 37683d9b1a2aSHans Rosenfeld { 37693d9b1a2aSHans Rosenfeld #ifndef __lock_lint 37703d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 37713d9b1a2aSHans Rosenfeld #endif 37723d9b1a2aSHans Rosenfeld minor_t minor = getminor(*devp); 37733d9b1a2aSHans Rosenfeld nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 37743d9b1a2aSHans Rosenfeld int nsid = NVME_MINOR_NSID(minor); 37753d9b1a2aSHans Rosenfeld nvme_minor_state_t *nm; 37763d9b1a2aSHans Rosenfeld int rv = 0; 37773d9b1a2aSHans Rosenfeld 37783d9b1a2aSHans Rosenfeld if (otyp != OTYP_CHR) 37793d9b1a2aSHans Rosenfeld return (EINVAL); 37803d9b1a2aSHans Rosenfeld 37813d9b1a2aSHans Rosenfeld if (nvme == NULL) 37823d9b1a2aSHans Rosenfeld return (ENXIO); 37833d9b1a2aSHans Rosenfeld 37843d9b1a2aSHans Rosenfeld if (nsid > nvme->n_namespace_count) 37853d9b1a2aSHans Rosenfeld return (ENXIO); 37863d9b1a2aSHans Rosenfeld 3787e984c70bSHans Rosenfeld if (nvme->n_dead) 3788e984c70bSHans Rosenfeld return (EIO); 3789e984c70bSHans Rosenfeld 37903d9b1a2aSHans Rosenfeld nm = nsid == 0 ? &nvme->n_minor : &nvme->n_ns[nsid - 1].ns_minor; 37913d9b1a2aSHans Rosenfeld 37923d9b1a2aSHans Rosenfeld mutex_enter(&nm->nm_mutex); 37933d9b1a2aSHans Rosenfeld if (nm->nm_oexcl) { 37943d9b1a2aSHans Rosenfeld rv = EBUSY; 37953d9b1a2aSHans Rosenfeld goto out; 37963d9b1a2aSHans Rosenfeld } 37973d9b1a2aSHans Rosenfeld 37983d9b1a2aSHans Rosenfeld if (flag & FEXCL) { 37993d9b1a2aSHans Rosenfeld if (nm->nm_ocnt != 0) { 38003d9b1a2aSHans Rosenfeld rv = EBUSY; 38013d9b1a2aSHans Rosenfeld goto out; 38023d9b1a2aSHans Rosenfeld } 38033d9b1a2aSHans Rosenfeld nm->nm_oexcl = B_TRUE; 38043d9b1a2aSHans Rosenfeld } 38053d9b1a2aSHans Rosenfeld 38063d9b1a2aSHans Rosenfeld nm->nm_ocnt++; 38073d9b1a2aSHans Rosenfeld 38083d9b1a2aSHans Rosenfeld out: 38093d9b1a2aSHans Rosenfeld mutex_exit(&nm->nm_mutex); 38103d9b1a2aSHans Rosenfeld return (rv); 38113d9b1a2aSHans Rosenfeld 38123d9b1a2aSHans Rosenfeld } 38133d9b1a2aSHans Rosenfeld 38143d9b1a2aSHans Rosenfeld static int 38153d9b1a2aSHans Rosenfeld nvme_close(dev_t dev, int flag, int otyp, cred_t *cred_p) 38163d9b1a2aSHans Rosenfeld { 38173d9b1a2aSHans Rosenfeld #ifndef __lock_lint 38183d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 38193d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(flag)); 38203d9b1a2aSHans Rosenfeld #endif 38213d9b1a2aSHans Rosenfeld minor_t minor = getminor(dev); 38223d9b1a2aSHans Rosenfeld nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 38233d9b1a2aSHans Rosenfeld int nsid = NVME_MINOR_NSID(minor); 38243d9b1a2aSHans Rosenfeld nvme_minor_state_t *nm; 38253d9b1a2aSHans Rosenfeld 38263d9b1a2aSHans Rosenfeld if (otyp != OTYP_CHR) 38273d9b1a2aSHans Rosenfeld return (ENXIO); 38283d9b1a2aSHans Rosenfeld 38293d9b1a2aSHans Rosenfeld if (nvme == NULL) 38303d9b1a2aSHans Rosenfeld return (ENXIO); 38313d9b1a2aSHans Rosenfeld 38323d9b1a2aSHans Rosenfeld if (nsid > nvme->n_namespace_count) 38333d9b1a2aSHans Rosenfeld return (ENXIO); 38343d9b1a2aSHans Rosenfeld 38353d9b1a2aSHans Rosenfeld nm = nsid == 0 ? &nvme->n_minor : &nvme->n_ns[nsid - 1].ns_minor; 38363d9b1a2aSHans Rosenfeld 38373d9b1a2aSHans Rosenfeld mutex_enter(&nm->nm_mutex); 38383d9b1a2aSHans Rosenfeld if (nm->nm_oexcl) 38393d9b1a2aSHans Rosenfeld nm->nm_oexcl = B_FALSE; 38403d9b1a2aSHans Rosenfeld 38413d9b1a2aSHans Rosenfeld ASSERT(nm->nm_ocnt > 0); 38423d9b1a2aSHans Rosenfeld nm->nm_ocnt--; 38433d9b1a2aSHans Rosenfeld mutex_exit(&nm->nm_mutex); 38443d9b1a2aSHans Rosenfeld 38453d9b1a2aSHans Rosenfeld return (0); 38463d9b1a2aSHans Rosenfeld } 38473d9b1a2aSHans Rosenfeld 38483d9b1a2aSHans Rosenfeld static int 38493d9b1a2aSHans Rosenfeld nvme_ioctl_identify(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 38503d9b1a2aSHans Rosenfeld cred_t *cred_p) 38513d9b1a2aSHans Rosenfeld { 38523d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 38533d9b1a2aSHans Rosenfeld int rv = 0; 38543d9b1a2aSHans Rosenfeld void *idctl; 38553d9b1a2aSHans Rosenfeld 38563d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 38573d9b1a2aSHans Rosenfeld return (EPERM); 38583d9b1a2aSHans Rosenfeld 38593d9b1a2aSHans Rosenfeld if (nioc->n_len < NVME_IDENTIFY_BUFSIZE) 38603d9b1a2aSHans Rosenfeld return (EINVAL); 38613d9b1a2aSHans Rosenfeld 3862bc586359SRobert Mustacchi if ((rv = nvme_identify(nvme, B_TRUE, nsid, (void **)&idctl)) != 0) 3863e984c70bSHans Rosenfeld return (rv); 38643d9b1a2aSHans Rosenfeld 38653d9b1a2aSHans Rosenfeld if (ddi_copyout(idctl, (void *)nioc->n_buf, NVME_IDENTIFY_BUFSIZE, mode) 38663d9b1a2aSHans Rosenfeld != 0) 38673d9b1a2aSHans Rosenfeld rv = EFAULT; 38683d9b1a2aSHans Rosenfeld 38693d9b1a2aSHans Rosenfeld kmem_free(idctl, NVME_IDENTIFY_BUFSIZE); 38703d9b1a2aSHans Rosenfeld 38713d9b1a2aSHans Rosenfeld return (rv); 38723d9b1a2aSHans Rosenfeld } 38733d9b1a2aSHans Rosenfeld 38743d9b1a2aSHans Rosenfeld static int 38753d9b1a2aSHans Rosenfeld nvme_ioctl_capabilities(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 38763d9b1a2aSHans Rosenfeld int mode, cred_t *cred_p) 38773d9b1a2aSHans Rosenfeld { 38783d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nsid, cred_p)); 38793d9b1a2aSHans Rosenfeld int rv = 0; 38803d9b1a2aSHans Rosenfeld nvme_reg_cap_t cap = { 0 }; 38813d9b1a2aSHans Rosenfeld nvme_capabilities_t nc; 38823d9b1a2aSHans Rosenfeld 38833d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 38843d9b1a2aSHans Rosenfeld return (EPERM); 38853d9b1a2aSHans Rosenfeld 38863d9b1a2aSHans Rosenfeld if (nioc->n_len < sizeof (nc)) 38873d9b1a2aSHans Rosenfeld return (EINVAL); 38883d9b1a2aSHans Rosenfeld 38893d9b1a2aSHans Rosenfeld cap.r = nvme_get64(nvme, NVME_REG_CAP); 38903d9b1a2aSHans Rosenfeld 38913d9b1a2aSHans Rosenfeld /* 38923d9b1a2aSHans Rosenfeld * The MPSMIN and MPSMAX fields in the CAP register use 0 to 38933d9b1a2aSHans Rosenfeld * specify the base page size of 4k (1<<12), so add 12 here to 38943d9b1a2aSHans Rosenfeld * get the real page size value. 38953d9b1a2aSHans Rosenfeld */ 38963d9b1a2aSHans Rosenfeld nc.mpsmax = 1 << (12 + cap.b.cap_mpsmax); 38973d9b1a2aSHans Rosenfeld nc.mpsmin = 1 << (12 + cap.b.cap_mpsmin); 38983d9b1a2aSHans Rosenfeld 38993d9b1a2aSHans Rosenfeld if (ddi_copyout(&nc, (void *)nioc->n_buf, sizeof (nc), mode) != 0) 39003d9b1a2aSHans Rosenfeld rv = EFAULT; 39013d9b1a2aSHans Rosenfeld 39023d9b1a2aSHans Rosenfeld return (rv); 39033d9b1a2aSHans Rosenfeld } 39043d9b1a2aSHans Rosenfeld 39053d9b1a2aSHans Rosenfeld static int 39063d9b1a2aSHans Rosenfeld nvme_ioctl_get_logpage(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 39073d9b1a2aSHans Rosenfeld int mode, cred_t *cred_p) 39083d9b1a2aSHans Rosenfeld { 39093d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 39103d9b1a2aSHans Rosenfeld void *log = NULL; 39113d9b1a2aSHans Rosenfeld size_t bufsize = 0; 39123d9b1a2aSHans Rosenfeld int rv = 0; 39133d9b1a2aSHans Rosenfeld 39143d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 39153d9b1a2aSHans Rosenfeld return (EPERM); 39163d9b1a2aSHans Rosenfeld 39173d9b1a2aSHans Rosenfeld switch (nioc->n_arg) { 39183d9b1a2aSHans Rosenfeld case NVME_LOGPAGE_ERROR: 39193d9b1a2aSHans Rosenfeld if (nsid != 0) 39203d9b1a2aSHans Rosenfeld return (EINVAL); 39213d9b1a2aSHans Rosenfeld break; 39223d9b1a2aSHans Rosenfeld case NVME_LOGPAGE_HEALTH: 39233d9b1a2aSHans Rosenfeld if (nsid != 0 && nvme->n_idctl->id_lpa.lp_smart == 0) 39243d9b1a2aSHans Rosenfeld return (EINVAL); 39253d9b1a2aSHans Rosenfeld 39263d9b1a2aSHans Rosenfeld if (nsid == 0) 39273d9b1a2aSHans Rosenfeld nsid = (uint32_t)-1; 39283d9b1a2aSHans Rosenfeld 39293d9b1a2aSHans Rosenfeld break; 39303d9b1a2aSHans Rosenfeld case NVME_LOGPAGE_FWSLOT: 39313d9b1a2aSHans Rosenfeld if (nsid != 0) 39323d9b1a2aSHans Rosenfeld return (EINVAL); 39333d9b1a2aSHans Rosenfeld break; 39343d9b1a2aSHans Rosenfeld default: 39353d9b1a2aSHans Rosenfeld return (EINVAL); 39363d9b1a2aSHans Rosenfeld } 39373d9b1a2aSHans Rosenfeld 3938bc586359SRobert Mustacchi if (nvme_get_logpage(nvme, B_TRUE, &log, &bufsize, nioc->n_arg, nsid) 39393d9b1a2aSHans Rosenfeld != DDI_SUCCESS) 39403d9b1a2aSHans Rosenfeld return (EIO); 39413d9b1a2aSHans Rosenfeld 39423d9b1a2aSHans Rosenfeld if (nioc->n_len < bufsize) { 39433d9b1a2aSHans Rosenfeld kmem_free(log, bufsize); 39443d9b1a2aSHans Rosenfeld return (EINVAL); 39453d9b1a2aSHans Rosenfeld } 39463d9b1a2aSHans Rosenfeld 39473d9b1a2aSHans Rosenfeld if (ddi_copyout(log, (void *)nioc->n_buf, bufsize, mode) != 0) 39483d9b1a2aSHans Rosenfeld rv = EFAULT; 39493d9b1a2aSHans Rosenfeld 39503d9b1a2aSHans Rosenfeld nioc->n_len = bufsize; 39513d9b1a2aSHans Rosenfeld kmem_free(log, bufsize); 39523d9b1a2aSHans Rosenfeld 39533d9b1a2aSHans Rosenfeld return (rv); 39543d9b1a2aSHans Rosenfeld } 39553d9b1a2aSHans Rosenfeld 39563d9b1a2aSHans Rosenfeld static int 39573d9b1a2aSHans Rosenfeld nvme_ioctl_get_features(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 39583d9b1a2aSHans Rosenfeld int mode, cred_t *cred_p) 39593d9b1a2aSHans Rosenfeld { 39603d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 39613d9b1a2aSHans Rosenfeld void *buf = NULL; 39623d9b1a2aSHans Rosenfeld size_t bufsize = 0; 39633d9b1a2aSHans Rosenfeld uint32_t res = 0; 39643d9b1a2aSHans Rosenfeld uint8_t feature; 39653d9b1a2aSHans Rosenfeld int rv = 0; 39663d9b1a2aSHans Rosenfeld 39673d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 39683d9b1a2aSHans Rosenfeld return (EPERM); 39693d9b1a2aSHans Rosenfeld 39703d9b1a2aSHans Rosenfeld if ((nioc->n_arg >> 32) > 0xff) 39713d9b1a2aSHans Rosenfeld return (EINVAL); 39723d9b1a2aSHans Rosenfeld 39733d9b1a2aSHans Rosenfeld feature = (uint8_t)(nioc->n_arg >> 32); 39743d9b1a2aSHans Rosenfeld 39753d9b1a2aSHans Rosenfeld switch (feature) { 39763d9b1a2aSHans Rosenfeld case NVME_FEAT_ARBITRATION: 39773d9b1a2aSHans Rosenfeld case NVME_FEAT_POWER_MGMT: 39783d9b1a2aSHans Rosenfeld case NVME_FEAT_TEMPERATURE: 39793d9b1a2aSHans Rosenfeld case NVME_FEAT_ERROR: 39803d9b1a2aSHans Rosenfeld case NVME_FEAT_NQUEUES: 39813d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_COAL: 39823d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_ATOM: 39833d9b1a2aSHans Rosenfeld case NVME_FEAT_ASYNC_EVENT: 39843d9b1a2aSHans Rosenfeld case NVME_FEAT_PROGRESS: 39853d9b1a2aSHans Rosenfeld if (nsid != 0) 39863d9b1a2aSHans Rosenfeld return (EINVAL); 39873d9b1a2aSHans Rosenfeld break; 39883d9b1a2aSHans Rosenfeld 39893d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_VECT: 39903d9b1a2aSHans Rosenfeld if (nsid != 0) 39913d9b1a2aSHans Rosenfeld return (EINVAL); 39923d9b1a2aSHans Rosenfeld 39933d9b1a2aSHans Rosenfeld res = nioc->n_arg & 0xffffffffUL; 39943d9b1a2aSHans Rosenfeld if (res >= nvme->n_intr_cnt) 39953d9b1a2aSHans Rosenfeld return (EINVAL); 39963d9b1a2aSHans Rosenfeld break; 39973d9b1a2aSHans Rosenfeld 39983d9b1a2aSHans Rosenfeld case NVME_FEAT_LBA_RANGE: 39993d9b1a2aSHans Rosenfeld if (nvme->n_lba_range_supported == B_FALSE) 40003d9b1a2aSHans Rosenfeld return (EINVAL); 40013d9b1a2aSHans Rosenfeld 40023d9b1a2aSHans Rosenfeld if (nsid == 0 || 40033d9b1a2aSHans Rosenfeld nsid > nvme->n_namespace_count) 40043d9b1a2aSHans Rosenfeld return (EINVAL); 40053d9b1a2aSHans Rosenfeld 40063d9b1a2aSHans Rosenfeld break; 40073d9b1a2aSHans Rosenfeld 40083d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_CACHE: 40093d9b1a2aSHans Rosenfeld if (nsid != 0) 40103d9b1a2aSHans Rosenfeld return (EINVAL); 40113d9b1a2aSHans Rosenfeld 40123d9b1a2aSHans Rosenfeld if (!nvme->n_write_cache_present) 40133d9b1a2aSHans Rosenfeld return (EINVAL); 40143d9b1a2aSHans Rosenfeld 40153d9b1a2aSHans Rosenfeld break; 40163d9b1a2aSHans Rosenfeld 40173d9b1a2aSHans Rosenfeld case NVME_FEAT_AUTO_PST: 40183d9b1a2aSHans Rosenfeld if (nsid != 0) 40193d9b1a2aSHans Rosenfeld return (EINVAL); 40203d9b1a2aSHans Rosenfeld 40213d9b1a2aSHans Rosenfeld if (!nvme->n_auto_pst_supported) 40223d9b1a2aSHans Rosenfeld return (EINVAL); 40233d9b1a2aSHans Rosenfeld 40243d9b1a2aSHans Rosenfeld break; 40253d9b1a2aSHans Rosenfeld 40263d9b1a2aSHans Rosenfeld default: 40273d9b1a2aSHans Rosenfeld return (EINVAL); 40283d9b1a2aSHans Rosenfeld } 40293d9b1a2aSHans Rosenfeld 4030bc586359SRobert Mustacchi rv = nvme_get_features(nvme, B_TRUE, nsid, feature, &res, &buf, 4031bc586359SRobert Mustacchi &bufsize); 4032e984c70bSHans Rosenfeld if (rv != 0) 4033e984c70bSHans Rosenfeld return (rv); 40343d9b1a2aSHans Rosenfeld 40353d9b1a2aSHans Rosenfeld if (nioc->n_len < bufsize) { 40363d9b1a2aSHans Rosenfeld kmem_free(buf, bufsize); 40373d9b1a2aSHans Rosenfeld return (EINVAL); 40383d9b1a2aSHans Rosenfeld } 40393d9b1a2aSHans Rosenfeld 40403d9b1a2aSHans Rosenfeld if (buf && ddi_copyout(buf, (void*)nioc->n_buf, bufsize, mode) != 0) 40413d9b1a2aSHans Rosenfeld rv = EFAULT; 40423d9b1a2aSHans Rosenfeld 40433d9b1a2aSHans Rosenfeld kmem_free(buf, bufsize); 40443d9b1a2aSHans Rosenfeld nioc->n_arg = res; 40453d9b1a2aSHans Rosenfeld nioc->n_len = bufsize; 40463d9b1a2aSHans Rosenfeld 40473d9b1a2aSHans Rosenfeld return (rv); 40483d9b1a2aSHans Rosenfeld } 40493d9b1a2aSHans Rosenfeld 40503d9b1a2aSHans Rosenfeld static int 40513d9b1a2aSHans Rosenfeld nvme_ioctl_intr_cnt(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 40523d9b1a2aSHans Rosenfeld cred_t *cred_p) 40533d9b1a2aSHans Rosenfeld { 40543d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nsid, mode, cred_p)); 40553d9b1a2aSHans Rosenfeld 40563d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 40573d9b1a2aSHans Rosenfeld return (EPERM); 40583d9b1a2aSHans Rosenfeld 40593d9b1a2aSHans Rosenfeld nioc->n_arg = nvme->n_intr_cnt; 40603d9b1a2aSHans Rosenfeld return (0); 40613d9b1a2aSHans Rosenfeld } 40623d9b1a2aSHans Rosenfeld 40633d9b1a2aSHans Rosenfeld static int 40643d9b1a2aSHans Rosenfeld nvme_ioctl_version(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 40653d9b1a2aSHans Rosenfeld cred_t *cred_p) 40663d9b1a2aSHans Rosenfeld { 40673d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nsid, cred_p)); 40683d9b1a2aSHans Rosenfeld int rv = 0; 40693d9b1a2aSHans Rosenfeld 40703d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 40713d9b1a2aSHans Rosenfeld return (EPERM); 40723d9b1a2aSHans Rosenfeld 40733d9b1a2aSHans Rosenfeld if (nioc->n_len < sizeof (nvme->n_version)) 40743d9b1a2aSHans Rosenfeld return (ENOMEM); 40753d9b1a2aSHans Rosenfeld 40763d9b1a2aSHans Rosenfeld if (ddi_copyout(&nvme->n_version, (void *)nioc->n_buf, 40773d9b1a2aSHans Rosenfeld sizeof (nvme->n_version), mode) != 0) 40783d9b1a2aSHans Rosenfeld rv = EFAULT; 40793d9b1a2aSHans Rosenfeld 40803d9b1a2aSHans Rosenfeld return (rv); 40813d9b1a2aSHans Rosenfeld } 40823d9b1a2aSHans Rosenfeld 40833d9b1a2aSHans Rosenfeld static int 40843d9b1a2aSHans Rosenfeld nvme_ioctl_format(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 40853d9b1a2aSHans Rosenfeld cred_t *cred_p) 40863d9b1a2aSHans Rosenfeld { 40873d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(mode)); 40883d9b1a2aSHans Rosenfeld nvme_format_nvm_t frmt = { 0 }; 40893d9b1a2aSHans Rosenfeld int c_nsid = nsid != 0 ? nsid - 1 : 0; 40903d9b1a2aSHans Rosenfeld 40913d9b1a2aSHans Rosenfeld if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 40923d9b1a2aSHans Rosenfeld return (EPERM); 40933d9b1a2aSHans Rosenfeld 40943d9b1a2aSHans Rosenfeld frmt.r = nioc->n_arg & 0xffffffff; 40953d9b1a2aSHans Rosenfeld 40963d9b1a2aSHans Rosenfeld /* 40973d9b1a2aSHans Rosenfeld * Check whether the FORMAT NVM command is supported. 40983d9b1a2aSHans Rosenfeld */ 40993d9b1a2aSHans Rosenfeld if (nvme->n_idctl->id_oacs.oa_format == 0) 41003d9b1a2aSHans Rosenfeld return (EINVAL); 41013d9b1a2aSHans Rosenfeld 41023d9b1a2aSHans Rosenfeld /* 41033d9b1a2aSHans Rosenfeld * Don't allow format or secure erase of individual namespace if that 41043d9b1a2aSHans Rosenfeld * would cause a format or secure erase of all namespaces. 41053d9b1a2aSHans Rosenfeld */ 41063d9b1a2aSHans Rosenfeld if (nsid != 0 && nvme->n_idctl->id_fna.fn_format != 0) 41073d9b1a2aSHans Rosenfeld return (EINVAL); 41083d9b1a2aSHans Rosenfeld 41093d9b1a2aSHans Rosenfeld if (nsid != 0 && frmt.b.fm_ses != NVME_FRMT_SES_NONE && 41103d9b1a2aSHans Rosenfeld nvme->n_idctl->id_fna.fn_sec_erase != 0) 41113d9b1a2aSHans Rosenfeld return (EINVAL); 41123d9b1a2aSHans Rosenfeld 41133d9b1a2aSHans Rosenfeld /* 41143d9b1a2aSHans Rosenfeld * Don't allow formatting with Protection Information. 41153d9b1a2aSHans Rosenfeld */ 41163d9b1a2aSHans Rosenfeld if (frmt.b.fm_pi != 0 || frmt.b.fm_pil != 0 || frmt.b.fm_ms != 0) 41173d9b1a2aSHans Rosenfeld return (EINVAL); 41183d9b1a2aSHans Rosenfeld 41193d9b1a2aSHans Rosenfeld /* 41203d9b1a2aSHans Rosenfeld * Don't allow formatting using an illegal LBA format, or any LBA format 41213d9b1a2aSHans Rosenfeld * that uses metadata. 41223d9b1a2aSHans Rosenfeld */ 41233d9b1a2aSHans Rosenfeld if (frmt.b.fm_lbaf > nvme->n_ns[c_nsid].ns_idns->id_nlbaf || 41243d9b1a2aSHans Rosenfeld nvme->n_ns[c_nsid].ns_idns->id_lbaf[frmt.b.fm_lbaf].lbaf_ms != 0) 41253d9b1a2aSHans Rosenfeld return (EINVAL); 41263d9b1a2aSHans Rosenfeld 41273d9b1a2aSHans Rosenfeld /* 41283d9b1a2aSHans Rosenfeld * Don't allow formatting using an illegal Secure Erase setting. 41293d9b1a2aSHans Rosenfeld */ 41303d9b1a2aSHans Rosenfeld if (frmt.b.fm_ses > NVME_FRMT_MAX_SES || 41313d9b1a2aSHans Rosenfeld (frmt.b.fm_ses == NVME_FRMT_SES_CRYPTO && 41323d9b1a2aSHans Rosenfeld nvme->n_idctl->id_fna.fn_crypt_erase == 0)) 41333d9b1a2aSHans Rosenfeld return (EINVAL); 41343d9b1a2aSHans Rosenfeld 41353d9b1a2aSHans Rosenfeld if (nsid == 0) 41363d9b1a2aSHans Rosenfeld nsid = (uint32_t)-1; 41373d9b1a2aSHans Rosenfeld 4138bc586359SRobert Mustacchi return (nvme_format_nvm(nvme, B_TRUE, nsid, frmt.b.fm_lbaf, B_FALSE, 0, 4139bc586359SRobert Mustacchi B_FALSE, frmt.b.fm_ses)); 41403d9b1a2aSHans Rosenfeld } 41413d9b1a2aSHans Rosenfeld 41423d9b1a2aSHans Rosenfeld static int 41433d9b1a2aSHans Rosenfeld nvme_ioctl_detach(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 41443d9b1a2aSHans Rosenfeld cred_t *cred_p) 41453d9b1a2aSHans Rosenfeld { 41463d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nioc, mode)); 41473d9b1a2aSHans Rosenfeld int rv = 0; 41483d9b1a2aSHans Rosenfeld 41493d9b1a2aSHans Rosenfeld if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 41503d9b1a2aSHans Rosenfeld return (EPERM); 41513d9b1a2aSHans Rosenfeld 41523d9b1a2aSHans Rosenfeld if (nsid == 0) 41533d9b1a2aSHans Rosenfeld return (EINVAL); 41543d9b1a2aSHans Rosenfeld 41553d9b1a2aSHans Rosenfeld rv = bd_detach_handle(nvme->n_ns[nsid - 1].ns_bd_hdl); 41563d9b1a2aSHans Rosenfeld if (rv != DDI_SUCCESS) 41573d9b1a2aSHans Rosenfeld rv = EBUSY; 41583d9b1a2aSHans Rosenfeld 41593d9b1a2aSHans Rosenfeld return (rv); 41603d9b1a2aSHans Rosenfeld } 41613d9b1a2aSHans Rosenfeld 41623d9b1a2aSHans Rosenfeld static int 41633d9b1a2aSHans Rosenfeld nvme_ioctl_attach(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 41643d9b1a2aSHans Rosenfeld cred_t *cred_p) 41653d9b1a2aSHans Rosenfeld { 41663d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nioc, mode)); 41673d9b1a2aSHans Rosenfeld nvme_identify_nsid_t *idns; 41683d9b1a2aSHans Rosenfeld int rv = 0; 41693d9b1a2aSHans Rosenfeld 41703d9b1a2aSHans Rosenfeld if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 41713d9b1a2aSHans Rosenfeld return (EPERM); 41723d9b1a2aSHans Rosenfeld 41733d9b1a2aSHans Rosenfeld if (nsid == 0) 41743d9b1a2aSHans Rosenfeld return (EINVAL); 41753d9b1a2aSHans Rosenfeld 41763d9b1a2aSHans Rosenfeld /* 41773d9b1a2aSHans Rosenfeld * Identify namespace again, free old identify data. 41783d9b1a2aSHans Rosenfeld */ 41793d9b1a2aSHans Rosenfeld idns = nvme->n_ns[nsid - 1].ns_idns; 41803d9b1a2aSHans Rosenfeld if (nvme_init_ns(nvme, nsid) != DDI_SUCCESS) 41813d9b1a2aSHans Rosenfeld return (EIO); 41823d9b1a2aSHans Rosenfeld 41833d9b1a2aSHans Rosenfeld kmem_free(idns, sizeof (nvme_identify_nsid_t)); 41843d9b1a2aSHans Rosenfeld 41853d9b1a2aSHans Rosenfeld rv = bd_attach_handle(nvme->n_dip, nvme->n_ns[nsid - 1].ns_bd_hdl); 41863d9b1a2aSHans Rosenfeld if (rv != DDI_SUCCESS) 41873d9b1a2aSHans Rosenfeld rv = EBUSY; 41883d9b1a2aSHans Rosenfeld 41893d9b1a2aSHans Rosenfeld return (rv); 41903d9b1a2aSHans Rosenfeld } 41913d9b1a2aSHans Rosenfeld 41923d9b1a2aSHans Rosenfeld static int 41933d9b1a2aSHans Rosenfeld nvme_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred_p, 41943d9b1a2aSHans Rosenfeld int *rval_p) 41953d9b1a2aSHans Rosenfeld { 41963d9b1a2aSHans Rosenfeld #ifndef __lock_lint 41973d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(rval_p)); 41983d9b1a2aSHans Rosenfeld #endif 41993d9b1a2aSHans Rosenfeld minor_t minor = getminor(dev); 42003d9b1a2aSHans Rosenfeld nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 42013d9b1a2aSHans Rosenfeld int nsid = NVME_MINOR_NSID(minor); 42023d9b1a2aSHans Rosenfeld int rv = 0; 42033d9b1a2aSHans Rosenfeld nvme_ioctl_t nioc; 42043d9b1a2aSHans Rosenfeld 42053d9b1a2aSHans Rosenfeld int (*nvme_ioctl[])(nvme_t *, int, nvme_ioctl_t *, int, cred_t *) = { 42063d9b1a2aSHans Rosenfeld NULL, 42073d9b1a2aSHans Rosenfeld nvme_ioctl_identify, 42083d9b1a2aSHans Rosenfeld nvme_ioctl_identify, 42093d9b1a2aSHans Rosenfeld nvme_ioctl_capabilities, 42103d9b1a2aSHans Rosenfeld nvme_ioctl_get_logpage, 42113d9b1a2aSHans Rosenfeld nvme_ioctl_get_features, 42123d9b1a2aSHans Rosenfeld nvme_ioctl_intr_cnt, 42133d9b1a2aSHans Rosenfeld nvme_ioctl_version, 42143d9b1a2aSHans Rosenfeld nvme_ioctl_format, 42153d9b1a2aSHans Rosenfeld nvme_ioctl_detach, 42163d9b1a2aSHans Rosenfeld nvme_ioctl_attach 42173d9b1a2aSHans Rosenfeld }; 42183d9b1a2aSHans Rosenfeld 42193d9b1a2aSHans Rosenfeld if (nvme == NULL) 42203d9b1a2aSHans Rosenfeld return (ENXIO); 42213d9b1a2aSHans Rosenfeld 42223d9b1a2aSHans Rosenfeld if (nsid > nvme->n_namespace_count) 42233d9b1a2aSHans Rosenfeld return (ENXIO); 42243d9b1a2aSHans Rosenfeld 42253d9b1a2aSHans Rosenfeld if (IS_DEVCTL(cmd)) 42263d9b1a2aSHans Rosenfeld return (ndi_devctl_ioctl(nvme->n_dip, cmd, arg, mode, 0)); 42273d9b1a2aSHans Rosenfeld 42283d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 42293d9b1a2aSHans Rosenfeld switch (ddi_model_convert_from(mode & FMODELS)) { 42303d9b1a2aSHans Rosenfeld case DDI_MODEL_ILP32: { 42313d9b1a2aSHans Rosenfeld nvme_ioctl32_t nioc32; 42323d9b1a2aSHans Rosenfeld if (ddi_copyin((void*)arg, &nioc32, sizeof (nvme_ioctl32_t), 42333d9b1a2aSHans Rosenfeld mode) != 0) 42343d9b1a2aSHans Rosenfeld return (EFAULT); 42353d9b1a2aSHans Rosenfeld nioc.n_len = nioc32.n_len; 42363d9b1a2aSHans Rosenfeld nioc.n_buf = nioc32.n_buf; 42373d9b1a2aSHans Rosenfeld nioc.n_arg = nioc32.n_arg; 42383d9b1a2aSHans Rosenfeld break; 42393d9b1a2aSHans Rosenfeld } 42403d9b1a2aSHans Rosenfeld case DDI_MODEL_NONE: 42413d9b1a2aSHans Rosenfeld #endif 42423d9b1a2aSHans Rosenfeld if (ddi_copyin((void*)arg, &nioc, sizeof (nvme_ioctl_t), mode) 42433d9b1a2aSHans Rosenfeld != 0) 42443d9b1a2aSHans Rosenfeld return (EFAULT); 42453d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 42463d9b1a2aSHans Rosenfeld break; 42473d9b1a2aSHans Rosenfeld } 42483d9b1a2aSHans Rosenfeld #endif 42493d9b1a2aSHans Rosenfeld 4250e984c70bSHans Rosenfeld if (nvme->n_dead && cmd != NVME_IOC_DETACH) 4251e984c70bSHans Rosenfeld return (EIO); 4252e984c70bSHans Rosenfeld 4253e984c70bSHans Rosenfeld 42543d9b1a2aSHans Rosenfeld if (cmd == NVME_IOC_IDENTIFY_CTRL) { 42553d9b1a2aSHans Rosenfeld /* 42563d9b1a2aSHans Rosenfeld * This makes NVME_IOC_IDENTIFY_CTRL work the same on devctl and 42573d9b1a2aSHans Rosenfeld * attachment point nodes. 42583d9b1a2aSHans Rosenfeld */ 42593d9b1a2aSHans Rosenfeld nsid = 0; 42603d9b1a2aSHans Rosenfeld } else if (cmd == NVME_IOC_IDENTIFY_NSID && nsid == 0) { 42613d9b1a2aSHans Rosenfeld /* 42623d9b1a2aSHans Rosenfeld * This makes NVME_IOC_IDENTIFY_NSID work on a devctl node, it 42633d9b1a2aSHans Rosenfeld * will always return identify data for namespace 1. 42643d9b1a2aSHans Rosenfeld */ 42653d9b1a2aSHans Rosenfeld nsid = 1; 42663d9b1a2aSHans Rosenfeld } 42673d9b1a2aSHans Rosenfeld 42683d9b1a2aSHans Rosenfeld if (IS_NVME_IOC(cmd) && nvme_ioctl[NVME_IOC_CMD(cmd)] != NULL) 42693d9b1a2aSHans Rosenfeld rv = nvme_ioctl[NVME_IOC_CMD(cmd)](nvme, nsid, &nioc, mode, 42703d9b1a2aSHans Rosenfeld cred_p); 42713d9b1a2aSHans Rosenfeld else 42723d9b1a2aSHans Rosenfeld rv = EINVAL; 42733d9b1a2aSHans Rosenfeld 42743d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 42753d9b1a2aSHans Rosenfeld switch (ddi_model_convert_from(mode & FMODELS)) { 42763d9b1a2aSHans Rosenfeld case DDI_MODEL_ILP32: { 42773d9b1a2aSHans Rosenfeld nvme_ioctl32_t nioc32; 42783d9b1a2aSHans Rosenfeld 42793d9b1a2aSHans Rosenfeld nioc32.n_len = (size32_t)nioc.n_len; 42803d9b1a2aSHans Rosenfeld nioc32.n_buf = (uintptr32_t)nioc.n_buf; 42813d9b1a2aSHans Rosenfeld nioc32.n_arg = nioc.n_arg; 42823d9b1a2aSHans Rosenfeld 42833d9b1a2aSHans Rosenfeld if (ddi_copyout(&nioc32, (void *)arg, sizeof (nvme_ioctl32_t), 42843d9b1a2aSHans Rosenfeld mode) != 0) 42853d9b1a2aSHans Rosenfeld return (EFAULT); 42863d9b1a2aSHans Rosenfeld break; 42873d9b1a2aSHans Rosenfeld } 42883d9b1a2aSHans Rosenfeld case DDI_MODEL_NONE: 42893d9b1a2aSHans Rosenfeld #endif 42903d9b1a2aSHans Rosenfeld if (ddi_copyout(&nioc, (void *)arg, sizeof (nvme_ioctl_t), mode) 42913d9b1a2aSHans Rosenfeld != 0) 42923d9b1a2aSHans Rosenfeld return (EFAULT); 42933d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 42943d9b1a2aSHans Rosenfeld break; 42953d9b1a2aSHans Rosenfeld } 42963d9b1a2aSHans Rosenfeld #endif 42973d9b1a2aSHans Rosenfeld 42983d9b1a2aSHans Rosenfeld return (rv); 42993d9b1a2aSHans Rosenfeld } 4300