13c9168faSHans Rosenfeld /* 23c9168faSHans Rosenfeld * This file and its contents are supplied under the terms of the 33c9168faSHans Rosenfeld * Common Development and Distribution License ("CDDL"), version 1.0. 43c9168faSHans Rosenfeld * You may only use this file in accordance with the terms of version 53c9168faSHans Rosenfeld * 1.0 of the CDDL. 63c9168faSHans Rosenfeld * 73c9168faSHans Rosenfeld * A full copy of the text of the CDDL should have accompanied this 83c9168faSHans Rosenfeld * source. A copy of the CDDL is also available via the Internet at 93c9168faSHans Rosenfeld * http://www.illumos.org/license/CDDL. 103c9168faSHans Rosenfeld */ 113c9168faSHans Rosenfeld 123c9168faSHans Rosenfeld /* 13f313c178SYuri Pankov * Copyright 2018 Nexenta Systems, Inc. 1434c938c7SPete Shephard * Copyright 2016 Tegile Systems, Inc. All rights reserved. 152f95345bSYouzhong Yang * Copyright (c) 2016 The MathWorks, Inc. All rights reserved. 1648d370f1SRobert Mustacchi * Copyright 2018 Joyent, Inc. 170999c112SPaul Winder * Copyright 2019 Western Digital Corporation. 18*709d065fSPaul Winder * Copyright 2020 Racktop Systems. 193c9168faSHans Rosenfeld */ 203c9168faSHans Rosenfeld 213c9168faSHans Rosenfeld /* 223c9168faSHans Rosenfeld * blkdev driver for NVMe compliant storage devices 233c9168faSHans Rosenfeld * 24f8dcaea5SRobert Mustacchi * This driver was written to conform to version 1.2.1 of the NVMe 25f8dcaea5SRobert Mustacchi * specification. It may work with newer versions, but that is completely 26f8dcaea5SRobert Mustacchi * untested and disabled by default. 273c9168faSHans Rosenfeld * 283c9168faSHans Rosenfeld * The driver has only been tested on x86 systems and will not work on big- 293c9168faSHans Rosenfeld * endian systems without changes to the code accessing registers and data 303c9168faSHans Rosenfeld * structures used by the hardware. 313c9168faSHans Rosenfeld * 323c9168faSHans Rosenfeld * 333c9168faSHans Rosenfeld * Interrupt Usage: 343c9168faSHans Rosenfeld * 350b466603SHans Rosenfeld * The driver will use a single interrupt while configuring the device as the 360b466603SHans Rosenfeld * specification requires, but contrary to the specification it will try to use 370b466603SHans Rosenfeld * a single-message MSI(-X) or FIXED interrupt. Later in the attach process it 380b466603SHans Rosenfeld * will switch to multiple-message MSI(-X) if supported. The driver wants to 390b466603SHans Rosenfeld * have one interrupt vector per CPU, but it will work correctly if less are 400b466603SHans Rosenfeld * available. Interrupts can be shared by queues, the interrupt handler will 410b466603SHans Rosenfeld * iterate through the I/O queue array by steps of n_intr_cnt. Usually only 420b466603SHans Rosenfeld * the admin queue will share an interrupt with one I/O queue. The interrupt 430b466603SHans Rosenfeld * handler will retrieve completed commands from all queues sharing an interrupt 440b466603SHans Rosenfeld * vector and will post them to a taskq for completion processing. 453c9168faSHans Rosenfeld * 463c9168faSHans Rosenfeld * 473c9168faSHans Rosenfeld * Command Processing: 483c9168faSHans Rosenfeld * 494b324362SHans Rosenfeld * NVMe devices can have up to 65535 I/O queue pairs, with each queue holding up 503c9168faSHans Rosenfeld * to 65536 I/O commands. The driver will configure one I/O queue pair per 513c9168faSHans Rosenfeld * available interrupt vector, with the queue length usually much smaller than 523c9168faSHans Rosenfeld * the maximum of 65536. If the hardware doesn't provide enough queues, fewer 533c9168faSHans Rosenfeld * interrupt vectors will be used. 543c9168faSHans Rosenfeld * 553c9168faSHans Rosenfeld * Additionally the hardware provides a single special admin queue pair that can 563c9168faSHans Rosenfeld * hold up to 4096 admin commands. 573c9168faSHans Rosenfeld * 583c9168faSHans Rosenfeld * From the hardware perspective both queues of a queue pair are independent, 593c9168faSHans Rosenfeld * but they share some driver state: the command array (holding pointers to 603c9168faSHans Rosenfeld * commands currently being processed by the hardware) and the active command 610999c112SPaul Winder * counter. Access to a submission queue and the shared state is protected by 620999c112SPaul Winder * nq_mutex, completion queue is protected by ncq_mutex. 633c9168faSHans Rosenfeld * 643c9168faSHans Rosenfeld * When a command is submitted to a queue pair the active command counter is 653c9168faSHans Rosenfeld * incremented and a pointer to the command is stored in the command array. The 663c9168faSHans Rosenfeld * array index is used as command identifier (CID) in the submission queue 673c9168faSHans Rosenfeld * entry. Some commands may take a very long time to complete, and if the queue 683c9168faSHans Rosenfeld * wraps around in that time a submission may find the next array slot to still 693c9168faSHans Rosenfeld * be used by a long-running command. In this case the array is sequentially 703c9168faSHans Rosenfeld * searched for the next free slot. The length of the command array is the same 714b324362SHans Rosenfeld * as the configured queue length. Queue overrun is prevented by the semaphore, 724b324362SHans Rosenfeld * so a command submission may block if the queue is full. 733c9168faSHans Rosenfeld * 743c9168faSHans Rosenfeld * 754ac9cfccSHans Rosenfeld * Polled I/O Support: 764ac9cfccSHans Rosenfeld * 774ac9cfccSHans Rosenfeld * For kernel core dump support the driver can do polled I/O. As interrupts are 784ac9cfccSHans Rosenfeld * turned off while dumping the driver will just submit a command in the regular 794ac9cfccSHans Rosenfeld * way, and then repeatedly attempt a command retrieval until it gets the 804ac9cfccSHans Rosenfeld * command back. 814ac9cfccSHans Rosenfeld * 824ac9cfccSHans Rosenfeld * 833c9168faSHans Rosenfeld * Namespace Support: 843c9168faSHans Rosenfeld * 853c9168faSHans Rosenfeld * NVMe devices can have multiple namespaces, each being a independent data 863c9168faSHans Rosenfeld * store. The driver supports multiple namespaces and creates a blkdev interface 873c9168faSHans Rosenfeld * for each namespace found. Namespaces can have various attributes to support 88db083a4dSYuri Pankov * protection information. This driver does not support any of this and ignores 89db083a4dSYuri Pankov * namespaces that have these attributes. 903c9168faSHans Rosenfeld * 9124979ca3SHans Rosenfeld * As of NVMe 1.1 namespaces can have an 64bit Extended Unique Identifier 9224979ca3SHans Rosenfeld * (EUI64). This driver uses the EUI64 if present to generate the devid and 9324979ca3SHans Rosenfeld * passes it to blkdev to use it in the device node names. As this is currently 9424979ca3SHans Rosenfeld * untested namespaces with EUI64 are ignored by default. 9524979ca3SHans Rosenfeld * 963d9b1a2aSHans Rosenfeld * We currently support only (2 << NVME_MINOR_INST_SHIFT) - 2 namespaces in a 973d9b1a2aSHans Rosenfeld * single controller. This is an artificial limit imposed by the driver to be 983d9b1a2aSHans Rosenfeld * able to address a reasonable number of controllers and namespaces using a 993d9b1a2aSHans Rosenfeld * 32bit minor node number. 1003d9b1a2aSHans Rosenfeld * 1013d9b1a2aSHans Rosenfeld * 1023d9b1a2aSHans Rosenfeld * Minor nodes: 1033d9b1a2aSHans Rosenfeld * 1043d9b1a2aSHans Rosenfeld * For each NVMe device the driver exposes one minor node for the controller and 1053d9b1a2aSHans Rosenfeld * one minor node for each namespace. The only operations supported by those 1063d9b1a2aSHans Rosenfeld * minor nodes are open(9E), close(9E), and ioctl(9E). This serves as the 1073d9b1a2aSHans Rosenfeld * interface for the nvmeadm(1M) utility. 1083d9b1a2aSHans Rosenfeld * 1093c9168faSHans Rosenfeld * 1103c9168faSHans Rosenfeld * Blkdev Interface: 1113c9168faSHans Rosenfeld * 1123c9168faSHans Rosenfeld * This driver uses blkdev to do all the heavy lifting involved with presenting 1133c9168faSHans Rosenfeld * a disk device to the system. As a result, the processing of I/O requests is 1143c9168faSHans Rosenfeld * relatively simple as blkdev takes care of partitioning, boundary checks, DMA 1153c9168faSHans Rosenfeld * setup, and splitting of transfers into manageable chunks. 1163c9168faSHans Rosenfeld * 1173c9168faSHans Rosenfeld * I/O requests coming in from blkdev are turned into NVM commands and posted to 1183c9168faSHans Rosenfeld * an I/O queue. The queue is selected by taking the CPU id modulo the number of 1193c9168faSHans Rosenfeld * queues. There is currently no timeout handling of I/O commands. 1203c9168faSHans Rosenfeld * 1213c9168faSHans Rosenfeld * Blkdev also supports querying device/media information and generating a 1223c9168faSHans Rosenfeld * devid. The driver reports the best block size as determined by the namespace 1233c9168faSHans Rosenfeld * format back to blkdev as physical block size to support partition and block 12424979ca3SHans Rosenfeld * alignment. The devid is either based on the namespace EUI64, if present, or 12524979ca3SHans Rosenfeld * composed using the device vendor ID, model number, serial number, and the 12624979ca3SHans Rosenfeld * namespace ID. 1273c9168faSHans Rosenfeld * 1283c9168faSHans Rosenfeld * 1293c9168faSHans Rosenfeld * Error Handling: 1303c9168faSHans Rosenfeld * 1313c9168faSHans Rosenfeld * Error handling is currently limited to detecting fatal hardware errors, 1323c9168faSHans Rosenfeld * either by asynchronous events, or synchronously through command status or 1333c9168faSHans Rosenfeld * admin command timeouts. In case of severe errors the device is fenced off, 1343c9168faSHans Rosenfeld * all further requests will return EIO. FMA is then called to fault the device. 1353c9168faSHans Rosenfeld * 1363c9168faSHans Rosenfeld * The hardware has a limit for outstanding asynchronous event requests. Before 1373c9168faSHans Rosenfeld * this limit is known the driver assumes it is at least 1 and posts a single 1383c9168faSHans Rosenfeld * asynchronous request. Later when the limit is known more asynchronous event 1393c9168faSHans Rosenfeld * requests are posted to allow quicker reception of error information. When an 1403c9168faSHans Rosenfeld * asynchronous event is posted by the hardware the driver will parse the error 1413c9168faSHans Rosenfeld * status fields and log information or fault the device, depending on the 1423c9168faSHans Rosenfeld * severity of the asynchronous event. The asynchronous event request is then 1433c9168faSHans Rosenfeld * reused and posted to the admin queue again. 1443c9168faSHans Rosenfeld * 1453c9168faSHans Rosenfeld * On command completion the command status is checked for errors. In case of 1463c9168faSHans Rosenfeld * errors indicating a driver bug the driver panics. Almost all other error 1473c9168faSHans Rosenfeld * status values just cause EIO to be returned. 1483c9168faSHans Rosenfeld * 1493c9168faSHans Rosenfeld * Command timeouts are currently detected for all admin commands except 1503c9168faSHans Rosenfeld * asynchronous event requests. If a command times out and the hardware appears 151e984c70bSHans Rosenfeld * to be healthy the driver attempts to abort the command. The original command 152e984c70bSHans Rosenfeld * timeout is also applied to the abort command. If the abort times out too the 1533c9168faSHans Rosenfeld * driver assumes the device to be dead, fences it off, and calls FMA to retire 154e984c70bSHans Rosenfeld * it. In all other cases the aborted command should return immediately with a 155e984c70bSHans Rosenfeld * status indicating it was aborted, and the driver will wait indefinitely for 156e984c70bSHans Rosenfeld * that to happen. No timeout handling of normal I/O commands is presently done. 1573c9168faSHans Rosenfeld * 158e984c70bSHans Rosenfeld * Any command that times out due to the controller dropping dead will be put on 159e984c70bSHans Rosenfeld * nvme_lost_cmds list if it references DMA memory. This will prevent the DMA 160e984c70bSHans Rosenfeld * memory being reused by the system and later be written to by a "dead" NVMe 161e984c70bSHans Rosenfeld * controller. 162e984c70bSHans Rosenfeld * 163e984c70bSHans Rosenfeld * 164e984c70bSHans Rosenfeld * Locking: 165e984c70bSHans Rosenfeld * 1660999c112SPaul Winder * Each queue pair has a nq_mutex and ncq_mutex. The nq_mutex must be held 1670999c112SPaul Winder * when accessing shared state and submission queue registers, ncq_mutex 1680999c112SPaul Winder * is held when accessing completion queue state and registers. 1690999c112SPaul Winder * Callers of nvme_unqueue_cmd() must make sure that nq_mutex is held, while 1700999c112SPaul Winder * nvme_submit_{admin,io}_cmd() and nvme_retrieve_cmd() take care of both 1710999c112SPaul Winder * mutexes themselves. 172e984c70bSHans Rosenfeld * 173e984c70bSHans Rosenfeld * Each command also has its own nc_mutex, which is associated with the 174e984c70bSHans Rosenfeld * condition variable nc_cv. It is only used on admin commands which are run 175e984c70bSHans Rosenfeld * synchronously. In that case it must be held across calls to 176e984c70bSHans Rosenfeld * nvme_submit_{admin,io}_cmd() and nvme_wait_cmd(), which is taken care of by 177e984c70bSHans Rosenfeld * nvme_admin_cmd(). It must also be held whenever the completion state of the 178e984c70bSHans Rosenfeld * command is changed or while a admin command timeout is handled. 179e984c70bSHans Rosenfeld * 180e984c70bSHans Rosenfeld * If both nc_mutex and nq_mutex must be held, nc_mutex must be acquired first. 181e984c70bSHans Rosenfeld * More than one nc_mutex may only be held when aborting commands. In this case, 182e984c70bSHans Rosenfeld * the nc_mutex of the command to be aborted must be held across the call to 183e984c70bSHans Rosenfeld * nvme_abort_cmd() to prevent the command from completing while the abort is in 184e984c70bSHans Rosenfeld * progress. 185e984c70bSHans Rosenfeld * 1860999c112SPaul Winder * If both nq_mutex and ncq_mutex need to be held, ncq_mutex must be 1870999c112SPaul Winder * acquired first. More than one nq_mutex is never held by a single thread. 1880999c112SPaul Winder * The ncq_mutex is only held by nvme_retrieve_cmd() and 1890999c112SPaul Winder * nvme_process_iocq(). nvme_process_iocq() is only called from the 1900999c112SPaul Winder * interrupt thread and nvme_retrieve_cmd() during polled I/O, so the 1910999c112SPaul Winder * mutex is non-contentious but is required for implementation completeness 1920999c112SPaul Winder * and safety. 1930999c112SPaul Winder * 194e984c70bSHans Rosenfeld * Each minor node has its own nm_mutex, which protects the open count nm_ocnt 195e984c70bSHans Rosenfeld * and exclusive-open flag nm_oexcl. 1963c9168faSHans Rosenfeld * 1973c9168faSHans Rosenfeld * 1983c9168faSHans Rosenfeld * Quiesce / Fast Reboot: 1993c9168faSHans Rosenfeld * 2003c9168faSHans Rosenfeld * The driver currently does not support fast reboot. A quiesce(9E) entry point 2013c9168faSHans Rosenfeld * is still provided which is used to send a shutdown notification to the 2023c9168faSHans Rosenfeld * device. 2033c9168faSHans Rosenfeld * 2043c9168faSHans Rosenfeld * 205e89be50aSRob Johnston * DDI UFM Support 206e89be50aSRob Johnston * 207e89be50aSRob Johnston * The driver supports the DDI UFM framework for reporting information about 208e89be50aSRob Johnston * the device's firmware image and slot configuration. This data can be 209e89be50aSRob Johnston * queried by userland software via ioctls to the ufm driver. For more 210e89be50aSRob Johnston * information, see ddi_ufm(9E). 211e89be50aSRob Johnston * 212e89be50aSRob Johnston * 2133c9168faSHans Rosenfeld * Driver Configuration: 2143c9168faSHans Rosenfeld * 2153c9168faSHans Rosenfeld * The following driver properties can be changed to control some aspects of the 2163c9168faSHans Rosenfeld * drivers operation: 2173c9168faSHans Rosenfeld * - strict-version: can be set to 0 to allow devices conforming to newer 21848d370f1SRobert Mustacchi * major versions to be used 2193c9168faSHans Rosenfeld * - ignore-unknown-vendor-status: can be set to 1 to not handle any vendor 2203c9168faSHans Rosenfeld * specific command status as a fatal error leading device faulting 2213c9168faSHans Rosenfeld * - admin-queue-len: the maximum length of the admin queue (16-4096) 2220999c112SPaul Winder * - io-squeue-len: the maximum length of the I/O submission queues (16-65536) 2230999c112SPaul Winder * - io-cqueue-len: the maximum length of the I/O completion queues (16-65536) 2243c9168faSHans Rosenfeld * - async-event-limit: the maximum number of asynchronous event requests to be 2253c9168faSHans Rosenfeld * posted by the driver 226d148d46eSHans Rosenfeld * - volatile-write-cache-enable: can be set to 0 to disable the volatile write 227d148d46eSHans Rosenfeld * cache 2286801591eSHans Rosenfeld * - min-phys-block-size: the minimum physical block size to report to blkdev, 2296801591eSHans Rosenfeld * which is among other things the basis for ZFS vdev ashift 2300999c112SPaul Winder * - max-submission-queues: the maximum number of I/O submission queues. 2310999c112SPaul Winder * - max-completion-queues: the maximum number of I/O completion queues, 2320999c112SPaul Winder * can be less than max-submission-queues, in which case the completion 2330999c112SPaul Winder * queues are shared. 2343c9168faSHans Rosenfeld * 2353c9168faSHans Rosenfeld * 2363c9168faSHans Rosenfeld * TODO: 2373c9168faSHans Rosenfeld * - figure out sane default for I/O queue depth reported to blkdev 2383c9168faSHans Rosenfeld * - FMA handling of media errors 2393c9168faSHans Rosenfeld * - support for devices supporting very large I/O requests using chained PRPs 2403c9168faSHans Rosenfeld * - support for configuring hardware parameters like interrupt coalescing 2413c9168faSHans Rosenfeld * - support for media formatting and hard partitioning into namespaces 2423c9168faSHans Rosenfeld * - support for big-endian systems 2433c9168faSHans Rosenfeld * - support for fast reboot 24424979ca3SHans Rosenfeld * - support for NVMe Subsystem Reset (1.1) 24524979ca3SHans Rosenfeld * - support for Scatter/Gather lists (1.1) 24624979ca3SHans Rosenfeld * - support for Reservations (1.1) 24724979ca3SHans Rosenfeld * - support for power management 2483c9168faSHans Rosenfeld */ 2493c9168faSHans Rosenfeld 2503c9168faSHans Rosenfeld #include <sys/byteorder.h> 2513c9168faSHans Rosenfeld #ifdef _BIG_ENDIAN 2523c9168faSHans Rosenfeld #error nvme driver needs porting for big-endian platforms 2533c9168faSHans Rosenfeld #endif 2543c9168faSHans Rosenfeld 2553c9168faSHans Rosenfeld #include <sys/modctl.h> 2563c9168faSHans Rosenfeld #include <sys/conf.h> 2573c9168faSHans Rosenfeld #include <sys/devops.h> 2583c9168faSHans Rosenfeld #include <sys/ddi.h> 259e89be50aSRob Johnston #include <sys/ddi_ufm.h> 2603c9168faSHans Rosenfeld #include <sys/sunddi.h> 2613d9b1a2aSHans Rosenfeld #include <sys/sunndi.h> 2623c9168faSHans Rosenfeld #include <sys/bitmap.h> 2633c9168faSHans Rosenfeld #include <sys/sysmacros.h> 2643c9168faSHans Rosenfeld #include <sys/param.h> 2653c9168faSHans Rosenfeld #include <sys/varargs.h> 2663c9168faSHans Rosenfeld #include <sys/cpuvar.h> 2673c9168faSHans Rosenfeld #include <sys/disp.h> 2683c9168faSHans Rosenfeld #include <sys/blkdev.h> 2693c9168faSHans Rosenfeld #include <sys/atomic.h> 2703c9168faSHans Rosenfeld #include <sys/archsystm.h> 271510a6847SHans Rosenfeld #include <sys/sata/sata_hba.h> 2723d9b1a2aSHans Rosenfeld #include <sys/stat.h> 2733d9b1a2aSHans Rosenfeld #include <sys/policy.h> 274e984c70bSHans Rosenfeld #include <sys/list.h> 2753d9b1a2aSHans Rosenfeld 2763d9b1a2aSHans Rosenfeld #include <sys/nvme.h> 2773c9168faSHans Rosenfeld 2789d08e1f8SHans Rosenfeld #ifdef __x86 2799d08e1f8SHans Rosenfeld #include <sys/x86_archext.h> 2809d08e1f8SHans Rosenfeld #endif 2819d08e1f8SHans Rosenfeld 2823c9168faSHans Rosenfeld #include "nvme_reg.h" 2833c9168faSHans Rosenfeld #include "nvme_var.h" 2843c9168faSHans Rosenfeld 28548d370f1SRobert Mustacchi /* 28648d370f1SRobert Mustacchi * Assertions to make sure that we've properly captured various aspects of the 28748d370f1SRobert Mustacchi * packed structures and haven't broken them during updates. 28848d370f1SRobert Mustacchi */ 28948d370f1SRobert Mustacchi CTASSERT(sizeof (nvme_identify_ctrl_t) == 0x1000); 29048d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_oacs) == 256); 29148d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_sqes) == 512); 29248d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_subnqn) == 768); 29348d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_nvmof) == 1792); 29448d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_psd) == 2048); 29548d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_ctrl_t, id_vs) == 3072); 29648d370f1SRobert Mustacchi 29748d370f1SRobert Mustacchi CTASSERT(sizeof (nvme_identify_nsid_t) == 0x1000); 29848d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_nsid_t, id_fpi) == 32); 29948d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_nsid_t, id_nguid) == 104); 30048d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_nsid_t, id_lbaf) == 128); 30148d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_nsid_t, id_vs) == 384); 30248d370f1SRobert Mustacchi 30348d370f1SRobert Mustacchi CTASSERT(sizeof (nvme_identify_primary_caps_t) == 0x1000); 30448d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_primary_caps_t, nipc_vqfrt) == 32); 30548d370f1SRobert Mustacchi CTASSERT(offsetof(nvme_identify_primary_caps_t, nipc_vifrt) == 64); 30648d370f1SRobert Mustacchi 3073c9168faSHans Rosenfeld 3083c9168faSHans Rosenfeld /* NVMe spec version supported */ 3093c9168faSHans Rosenfeld static const int nvme_version_major = 1; 3103c9168faSHans Rosenfeld 311e8ba2a38SHans Rosenfeld /* tunable for admin command timeout in seconds, default is 1s */ 3123d9b1a2aSHans Rosenfeld int nvme_admin_cmd_timeout = 1; 3133d9b1a2aSHans Rosenfeld 3143d9b1a2aSHans Rosenfeld /* tunable for FORMAT NVM command timeout in seconds, default is 600s */ 3153d9b1a2aSHans Rosenfeld int nvme_format_cmd_timeout = 600; 316e8ba2a38SHans Rosenfeld 317cf840871SPaul Winder /* tunable for firmware commit with NVME_FWC_SAVE, default is 15s */ 318cf840871SPaul Winder int nvme_commit_save_cmd_timeout = 15; 319cf840871SPaul Winder 3203c9168faSHans Rosenfeld static int nvme_attach(dev_info_t *, ddi_attach_cmd_t); 3213c9168faSHans Rosenfeld static int nvme_detach(dev_info_t *, ddi_detach_cmd_t); 3223c9168faSHans Rosenfeld static int nvme_quiesce(dev_info_t *); 3233c9168faSHans Rosenfeld static int nvme_fm_errcb(dev_info_t *, ddi_fm_error_t *, const void *); 3243c9168faSHans Rosenfeld static int nvme_setup_interrupts(nvme_t *, int, int); 3253c9168faSHans Rosenfeld static void nvme_release_interrupts(nvme_t *); 3263c9168faSHans Rosenfeld static uint_t nvme_intr(caddr_t, caddr_t); 3273c9168faSHans Rosenfeld 3283c9168faSHans Rosenfeld static void nvme_shutdown(nvme_t *, int, boolean_t); 3293c9168faSHans Rosenfeld static boolean_t nvme_reset(nvme_t *, boolean_t); 3303c9168faSHans Rosenfeld static int nvme_init(nvme_t *); 3313c9168faSHans Rosenfeld static nvme_cmd_t *nvme_alloc_cmd(nvme_t *, int); 3323c9168faSHans Rosenfeld static void nvme_free_cmd(nvme_cmd_t *); 3333c9168faSHans Rosenfeld static nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t, 3343c9168faSHans Rosenfeld bd_xfer_t *); 335e984c70bSHans Rosenfeld static void nvme_admin_cmd(nvme_cmd_t *, int); 3364b324362SHans Rosenfeld static void nvme_submit_admin_cmd(nvme_qpair_t *, nvme_cmd_t *); 3374b324362SHans Rosenfeld static int nvme_submit_io_cmd(nvme_qpair_t *, nvme_cmd_t *); 3384b324362SHans Rosenfeld static void nvme_submit_cmd_common(nvme_qpair_t *, nvme_cmd_t *); 339e984c70bSHans Rosenfeld static nvme_cmd_t *nvme_unqueue_cmd(nvme_t *, nvme_qpair_t *, int); 3403c9168faSHans Rosenfeld static nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *); 341e984c70bSHans Rosenfeld static void nvme_wait_cmd(nvme_cmd_t *, uint_t); 3423c9168faSHans Rosenfeld static void nvme_wakeup_cmd(void *); 3433c9168faSHans Rosenfeld static void nvme_async_event_task(void *); 3443c9168faSHans Rosenfeld 3453c9168faSHans Rosenfeld static int nvme_check_unknown_cmd_status(nvme_cmd_t *); 3463c9168faSHans Rosenfeld static int nvme_check_vendor_cmd_status(nvme_cmd_t *); 3473c9168faSHans Rosenfeld static int nvme_check_integrity_cmd_status(nvme_cmd_t *); 3483c9168faSHans Rosenfeld static int nvme_check_specific_cmd_status(nvme_cmd_t *); 3493c9168faSHans Rosenfeld static int nvme_check_generic_cmd_status(nvme_cmd_t *); 3503c9168faSHans Rosenfeld static inline int nvme_check_cmd_status(nvme_cmd_t *); 3513c9168faSHans Rosenfeld 352e984c70bSHans Rosenfeld static int nvme_abort_cmd(nvme_cmd_t *, uint_t); 3534b324362SHans Rosenfeld static void nvme_async_event(nvme_t *); 354bc586359SRobert Mustacchi static int nvme_format_nvm(nvme_t *, boolean_t, uint32_t, uint8_t, boolean_t, 355bc586359SRobert Mustacchi uint8_t, boolean_t, uint8_t); 356bc586359SRobert Mustacchi static int nvme_get_logpage(nvme_t *, boolean_t, void **, size_t *, uint8_t, 357bc586359SRobert Mustacchi ...); 358bc586359SRobert Mustacchi static int nvme_identify(nvme_t *, boolean_t, uint32_t, void **); 359bc586359SRobert Mustacchi static int nvme_set_features(nvme_t *, boolean_t, uint32_t, uint8_t, uint32_t, 360d148d46eSHans Rosenfeld uint32_t *); 361bc586359SRobert Mustacchi static int nvme_get_features(nvme_t *, boolean_t, uint32_t, uint8_t, uint32_t *, 3623d9b1a2aSHans Rosenfeld void **, size_t *); 363e984c70bSHans Rosenfeld static int nvme_write_cache_set(nvme_t *, boolean_t); 3640999c112SPaul Winder static int nvme_set_nqueues(nvme_t *); 3653c9168faSHans Rosenfeld 3663c9168faSHans Rosenfeld static void nvme_free_dma(nvme_dma_t *); 3673c9168faSHans Rosenfeld static int nvme_zalloc_dma(nvme_t *, size_t, uint_t, ddi_dma_attr_t *, 3683c9168faSHans Rosenfeld nvme_dma_t **); 3693c9168faSHans Rosenfeld static int nvme_zalloc_queue_dma(nvme_t *, uint32_t, uint16_t, uint_t, 3703c9168faSHans Rosenfeld nvme_dma_t **); 3713c9168faSHans Rosenfeld static void nvme_free_qpair(nvme_qpair_t *); 3720999c112SPaul Winder static int nvme_alloc_qpair(nvme_t *, uint32_t, nvme_qpair_t **, uint_t); 3733c9168faSHans Rosenfeld static int nvme_create_io_qpair(nvme_t *, nvme_qpair_t *, uint16_t); 3743c9168faSHans Rosenfeld 3753c9168faSHans Rosenfeld static inline void nvme_put64(nvme_t *, uintptr_t, uint64_t); 3763c9168faSHans Rosenfeld static inline void nvme_put32(nvme_t *, uintptr_t, uint32_t); 3773c9168faSHans Rosenfeld static inline uint64_t nvme_get64(nvme_t *, uintptr_t); 3783c9168faSHans Rosenfeld static inline uint32_t nvme_get32(nvme_t *, uintptr_t); 3793c9168faSHans Rosenfeld 3803c9168faSHans Rosenfeld static boolean_t nvme_check_regs_hdl(nvme_t *); 3813c9168faSHans Rosenfeld static boolean_t nvme_check_dma_hdl(nvme_dma_t *); 3823c9168faSHans Rosenfeld 3833c9168faSHans Rosenfeld static int nvme_fill_prp(nvme_cmd_t *, bd_xfer_t *); 3843c9168faSHans Rosenfeld 3853c9168faSHans Rosenfeld static void nvme_bd_xfer_done(void *); 3863c9168faSHans Rosenfeld static void nvme_bd_driveinfo(void *, bd_drive_t *); 3873c9168faSHans Rosenfeld static int nvme_bd_mediainfo(void *, bd_media_t *); 3883c9168faSHans Rosenfeld static int nvme_bd_cmd(nvme_namespace_t *, bd_xfer_t *, uint8_t); 3893c9168faSHans Rosenfeld static int nvme_bd_read(void *, bd_xfer_t *); 3903c9168faSHans Rosenfeld static int nvme_bd_write(void *, bd_xfer_t *); 3913c9168faSHans Rosenfeld static int nvme_bd_sync(void *, bd_xfer_t *); 3923c9168faSHans Rosenfeld static int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *); 3933c9168faSHans Rosenfeld 3948834f7acSYouzhong Yang static int nvme_prp_dma_constructor(void *, void *, int); 3958834f7acSYouzhong Yang static void nvme_prp_dma_destructor(void *, void *); 3968834f7acSYouzhong Yang 3973c9168faSHans Rosenfeld static void nvme_prepare_devid(nvme_t *, uint32_t); 3983c9168faSHans Rosenfeld 399e89be50aSRob Johnston /* DDI UFM callbacks */ 400e89be50aSRob Johnston static int nvme_ufm_fill_image(ddi_ufm_handle_t *, void *, uint_t, 401e89be50aSRob Johnston ddi_ufm_image_t *); 402e89be50aSRob Johnston static int nvme_ufm_fill_slot(ddi_ufm_handle_t *, void *, uint_t, uint_t, 403e89be50aSRob Johnston ddi_ufm_slot_t *); 404e89be50aSRob Johnston static int nvme_ufm_getcaps(ddi_ufm_handle_t *, void *, ddi_ufm_cap_t *); 405e89be50aSRob Johnston 4063d9b1a2aSHans Rosenfeld static int nvme_open(dev_t *, int, int, cred_t *); 4073d9b1a2aSHans Rosenfeld static int nvme_close(dev_t, int, int, cred_t *); 4083d9b1a2aSHans Rosenfeld static int nvme_ioctl(dev_t, int, intptr_t, int, cred_t *, int *); 4093d9b1a2aSHans Rosenfeld 410e89be50aSRob Johnston static ddi_ufm_ops_t nvme_ufm_ops = { 411e89be50aSRob Johnston NULL, 412e89be50aSRob Johnston nvme_ufm_fill_image, 413e89be50aSRob Johnston nvme_ufm_fill_slot, 414e89be50aSRob Johnston nvme_ufm_getcaps 415e89be50aSRob Johnston }; 416e89be50aSRob Johnston 417dc97a43dSHans Rosenfeld #define NVME_MINOR_INST_SHIFT 9 4183d9b1a2aSHans Rosenfeld #define NVME_MINOR(inst, nsid) (((inst) << NVME_MINOR_INST_SHIFT) | (nsid)) 4193d9b1a2aSHans Rosenfeld #define NVME_MINOR_INST(minor) ((minor) >> NVME_MINOR_INST_SHIFT) 4203d9b1a2aSHans Rosenfeld #define NVME_MINOR_NSID(minor) ((minor) & ((1 << NVME_MINOR_INST_SHIFT) - 1)) 4213d9b1a2aSHans Rosenfeld #define NVME_MINOR_MAX (NVME_MINOR(1, 0) - 2) 4223d9b1a2aSHans Rosenfeld 4233c9168faSHans Rosenfeld static void *nvme_state; 4243c9168faSHans Rosenfeld static kmem_cache_t *nvme_cmd_cache; 4253c9168faSHans Rosenfeld 4263c9168faSHans Rosenfeld /* 4273c9168faSHans Rosenfeld * DMA attributes for queue DMA memory 4283c9168faSHans Rosenfeld * 4293c9168faSHans Rosenfeld * Queue DMA memory must be page aligned. The maximum length of a queue is 4303c9168faSHans Rosenfeld * 65536 entries, and an entry can be 64 bytes long. 4313c9168faSHans Rosenfeld */ 4323c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_queue_dma_attr = { 4333c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 4343c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 4353c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 436910f0d12SYouzhong Yang .dma_attr_count_max = (UINT16_MAX + 1) * sizeof (nvme_sqe_t) - 1, 4373c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 4383c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 4393c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 4403c9168faSHans Rosenfeld .dma_attr_maxxfer = (UINT16_MAX + 1) * sizeof (nvme_sqe_t), 4413c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 4423c9168faSHans Rosenfeld .dma_attr_sgllen = 1, 4433c9168faSHans Rosenfeld .dma_attr_granular = 1, 4443c9168faSHans Rosenfeld .dma_attr_flags = 0, 4453c9168faSHans Rosenfeld }; 4463c9168faSHans Rosenfeld 4473c9168faSHans Rosenfeld /* 4483c9168faSHans Rosenfeld * DMA attributes for transfers using Physical Region Page (PRP) entries 4493c9168faSHans Rosenfeld * 4503c9168faSHans Rosenfeld * A PRP entry describes one page of DMA memory using the page size specified 4513c9168faSHans Rosenfeld * in the controller configuration's memory page size register (CC.MPS). It uses 4523c9168faSHans Rosenfeld * a 64bit base address aligned to this page size. There is no limitation on 4533c9168faSHans Rosenfeld * chaining PRPs together for arbitrarily large DMA transfers. 4543c9168faSHans Rosenfeld */ 4553c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_prp_dma_attr = { 4563c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 4573c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 4583c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 4593c9168faSHans Rosenfeld .dma_attr_count_max = 0xfff, 4603c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 4613c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 4623c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 4633c9168faSHans Rosenfeld .dma_attr_maxxfer = 0x1000, 4642f95345bSYouzhong Yang .dma_attr_seg = 0xfff, 4653c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 4663c9168faSHans Rosenfeld .dma_attr_granular = 1, 4673c9168faSHans Rosenfeld .dma_attr_flags = 0, 4683c9168faSHans Rosenfeld }; 4693c9168faSHans Rosenfeld 4703c9168faSHans Rosenfeld /* 4713c9168faSHans Rosenfeld * DMA attributes for transfers using scatter/gather lists 4723c9168faSHans Rosenfeld * 4733c9168faSHans Rosenfeld * A SGL entry describes a chunk of DMA memory using a 64bit base address and a 4743c9168faSHans Rosenfeld * 32bit length field. SGL Segment and SGL Last Segment entries require the 4753c9168faSHans Rosenfeld * length to be a multiple of 16 bytes. 4763c9168faSHans Rosenfeld */ 4773c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_sgl_dma_attr = { 4783c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 4793c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 4803c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 4813c9168faSHans Rosenfeld .dma_attr_count_max = 0xffffffffUL, 4823c9168faSHans Rosenfeld .dma_attr_align = 1, 4833c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 4843c9168faSHans Rosenfeld .dma_attr_minxfer = 0x10, 4853c9168faSHans Rosenfeld .dma_attr_maxxfer = 0xfffffffffULL, 4863c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 4873c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 4883c9168faSHans Rosenfeld .dma_attr_granular = 0x10, 4893c9168faSHans Rosenfeld .dma_attr_flags = 0 4903c9168faSHans Rosenfeld }; 4913c9168faSHans Rosenfeld 4923c9168faSHans Rosenfeld static ddi_device_acc_attr_t nvme_reg_acc_attr = { 4933c9168faSHans Rosenfeld .devacc_attr_version = DDI_DEVICE_ATTR_V0, 4943c9168faSHans Rosenfeld .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 4953c9168faSHans Rosenfeld .devacc_attr_dataorder = DDI_STRICTORDER_ACC 4963c9168faSHans Rosenfeld }; 4973c9168faSHans Rosenfeld 4983d9b1a2aSHans Rosenfeld static struct cb_ops nvme_cb_ops = { 4993d9b1a2aSHans Rosenfeld .cb_open = nvme_open, 5003d9b1a2aSHans Rosenfeld .cb_close = nvme_close, 5013d9b1a2aSHans Rosenfeld .cb_strategy = nodev, 5023d9b1a2aSHans Rosenfeld .cb_print = nodev, 5033d9b1a2aSHans Rosenfeld .cb_dump = nodev, 5043d9b1a2aSHans Rosenfeld .cb_read = nodev, 5053d9b1a2aSHans Rosenfeld .cb_write = nodev, 5063d9b1a2aSHans Rosenfeld .cb_ioctl = nvme_ioctl, 5073d9b1a2aSHans Rosenfeld .cb_devmap = nodev, 5083d9b1a2aSHans Rosenfeld .cb_mmap = nodev, 5093d9b1a2aSHans Rosenfeld .cb_segmap = nodev, 5103d9b1a2aSHans Rosenfeld .cb_chpoll = nochpoll, 5113d9b1a2aSHans Rosenfeld .cb_prop_op = ddi_prop_op, 5123d9b1a2aSHans Rosenfeld .cb_str = 0, 5133d9b1a2aSHans Rosenfeld .cb_flag = D_NEW | D_MP, 5143d9b1a2aSHans Rosenfeld .cb_rev = CB_REV, 5153d9b1a2aSHans Rosenfeld .cb_aread = nodev, 5163d9b1a2aSHans Rosenfeld .cb_awrite = nodev 5173d9b1a2aSHans Rosenfeld }; 5183d9b1a2aSHans Rosenfeld 5193c9168faSHans Rosenfeld static struct dev_ops nvme_dev_ops = { 5203c9168faSHans Rosenfeld .devo_rev = DEVO_REV, 5213c9168faSHans Rosenfeld .devo_refcnt = 0, 5223c9168faSHans Rosenfeld .devo_getinfo = ddi_no_info, 5233c9168faSHans Rosenfeld .devo_identify = nulldev, 5243c9168faSHans Rosenfeld .devo_probe = nulldev, 5253c9168faSHans Rosenfeld .devo_attach = nvme_attach, 5263c9168faSHans Rosenfeld .devo_detach = nvme_detach, 5273c9168faSHans Rosenfeld .devo_reset = nodev, 5283d9b1a2aSHans Rosenfeld .devo_cb_ops = &nvme_cb_ops, 5293c9168faSHans Rosenfeld .devo_bus_ops = NULL, 5303c9168faSHans Rosenfeld .devo_power = NULL, 5313c9168faSHans Rosenfeld .devo_quiesce = nvme_quiesce, 5323c9168faSHans Rosenfeld }; 5333c9168faSHans Rosenfeld 5343c9168faSHans Rosenfeld static struct modldrv nvme_modldrv = { 5353c9168faSHans Rosenfeld .drv_modops = &mod_driverops, 53624979ca3SHans Rosenfeld .drv_linkinfo = "NVMe v1.1b", 5373c9168faSHans Rosenfeld .drv_dev_ops = &nvme_dev_ops 5383c9168faSHans Rosenfeld }; 5393c9168faSHans Rosenfeld 5403c9168faSHans Rosenfeld static struct modlinkage nvme_modlinkage = { 5413c9168faSHans Rosenfeld .ml_rev = MODREV_1, 5423c9168faSHans Rosenfeld .ml_linkage = { &nvme_modldrv, NULL } 5433c9168faSHans Rosenfeld }; 5443c9168faSHans Rosenfeld 5453c9168faSHans Rosenfeld static bd_ops_t nvme_bd_ops = { 5464d95620bSPaul Winder .o_version = BD_OPS_CURRENT_VERSION, 5473c9168faSHans Rosenfeld .o_drive_info = nvme_bd_driveinfo, 5483c9168faSHans Rosenfeld .o_media_info = nvme_bd_mediainfo, 5493c9168faSHans Rosenfeld .o_devid_init = nvme_bd_devid, 5503c9168faSHans Rosenfeld .o_sync_cache = nvme_bd_sync, 5513c9168faSHans Rosenfeld .o_read = nvme_bd_read, 5523c9168faSHans Rosenfeld .o_write = nvme_bd_write, 5533c9168faSHans Rosenfeld }; 5543c9168faSHans Rosenfeld 555e984c70bSHans Rosenfeld /* 556e984c70bSHans Rosenfeld * This list will hold commands that have timed out and couldn't be aborted. 557e984c70bSHans Rosenfeld * As we don't know what the hardware may still do with the DMA memory we can't 558e984c70bSHans Rosenfeld * free them, so we'll keep them forever on this list where we can easily look 559e984c70bSHans Rosenfeld * at them with mdb. 560e984c70bSHans Rosenfeld */ 561e984c70bSHans Rosenfeld static struct list nvme_lost_cmds; 562e984c70bSHans Rosenfeld static kmutex_t nvme_lc_mutex; 563e984c70bSHans Rosenfeld 5643c9168faSHans Rosenfeld int 5653c9168faSHans Rosenfeld _init(void) 5663c9168faSHans Rosenfeld { 5673c9168faSHans Rosenfeld int error; 5683c9168faSHans Rosenfeld 5693c9168faSHans Rosenfeld error = ddi_soft_state_init(&nvme_state, sizeof (nvme_t), 1); 5703c9168faSHans Rosenfeld if (error != DDI_SUCCESS) 5713c9168faSHans Rosenfeld return (error); 5723c9168faSHans Rosenfeld 5733c9168faSHans Rosenfeld nvme_cmd_cache = kmem_cache_create("nvme_cmd_cache", 5743c9168faSHans Rosenfeld sizeof (nvme_cmd_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 5753c9168faSHans Rosenfeld 576e984c70bSHans Rosenfeld mutex_init(&nvme_lc_mutex, NULL, MUTEX_DRIVER, NULL); 577e984c70bSHans Rosenfeld list_create(&nvme_lost_cmds, sizeof (nvme_cmd_t), 578e984c70bSHans Rosenfeld offsetof(nvme_cmd_t, nc_list)); 579e984c70bSHans Rosenfeld 5803c9168faSHans Rosenfeld bd_mod_init(&nvme_dev_ops); 5813c9168faSHans Rosenfeld 5823c9168faSHans Rosenfeld error = mod_install(&nvme_modlinkage); 5833c9168faSHans Rosenfeld if (error != DDI_SUCCESS) { 5843c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 585e984c70bSHans Rosenfeld mutex_destroy(&nvme_lc_mutex); 586e984c70bSHans Rosenfeld list_destroy(&nvme_lost_cmds); 5873c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 5883c9168faSHans Rosenfeld } 5893c9168faSHans Rosenfeld 5903c9168faSHans Rosenfeld return (error); 5913c9168faSHans Rosenfeld } 5923c9168faSHans Rosenfeld 5933c9168faSHans Rosenfeld int 5943c9168faSHans Rosenfeld _fini(void) 5953c9168faSHans Rosenfeld { 5963c9168faSHans Rosenfeld int error; 5973c9168faSHans Rosenfeld 598e984c70bSHans Rosenfeld if (!list_is_empty(&nvme_lost_cmds)) 599e984c70bSHans Rosenfeld return (DDI_FAILURE); 600e984c70bSHans Rosenfeld 6013c9168faSHans Rosenfeld error = mod_remove(&nvme_modlinkage); 6023c9168faSHans Rosenfeld if (error == DDI_SUCCESS) { 6033c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 6043c9168faSHans Rosenfeld kmem_cache_destroy(nvme_cmd_cache); 605e984c70bSHans Rosenfeld mutex_destroy(&nvme_lc_mutex); 606e984c70bSHans Rosenfeld list_destroy(&nvme_lost_cmds); 6073c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 6083c9168faSHans Rosenfeld } 6093c9168faSHans Rosenfeld 6103c9168faSHans Rosenfeld return (error); 6113c9168faSHans Rosenfeld } 6123c9168faSHans Rosenfeld 6133c9168faSHans Rosenfeld int 6143c9168faSHans Rosenfeld _info(struct modinfo *modinfop) 6153c9168faSHans Rosenfeld { 6163c9168faSHans Rosenfeld return (mod_info(&nvme_modlinkage, modinfop)); 6173c9168faSHans Rosenfeld } 6183c9168faSHans Rosenfeld 6193c9168faSHans Rosenfeld static inline void 6203c9168faSHans Rosenfeld nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val) 6213c9168faSHans Rosenfeld { 6223c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 6233c9168faSHans Rosenfeld 6243c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 6253c9168faSHans Rosenfeld ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val); 6263c9168faSHans Rosenfeld } 6273c9168faSHans Rosenfeld 6283c9168faSHans Rosenfeld static inline void 6293c9168faSHans Rosenfeld nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val) 6303c9168faSHans Rosenfeld { 6313c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 6323c9168faSHans Rosenfeld 6333c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 6343c9168faSHans Rosenfeld ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val); 6353c9168faSHans Rosenfeld } 6363c9168faSHans Rosenfeld 6373c9168faSHans Rosenfeld static inline uint64_t 6383c9168faSHans Rosenfeld nvme_get64(nvme_t *nvme, uintptr_t reg) 6393c9168faSHans Rosenfeld { 6403c9168faSHans Rosenfeld uint64_t val; 6413c9168faSHans Rosenfeld 6423c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 6433c9168faSHans Rosenfeld 6443c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 6453c9168faSHans Rosenfeld val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg)); 6463c9168faSHans Rosenfeld 6473c9168faSHans Rosenfeld return (val); 6483c9168faSHans Rosenfeld } 6493c9168faSHans Rosenfeld 6503c9168faSHans Rosenfeld static inline uint32_t 6513c9168faSHans Rosenfeld nvme_get32(nvme_t *nvme, uintptr_t reg) 6523c9168faSHans Rosenfeld { 6533c9168faSHans Rosenfeld uint32_t val; 6543c9168faSHans Rosenfeld 6553c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 6563c9168faSHans Rosenfeld 6573c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 6583c9168faSHans Rosenfeld val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg)); 6593c9168faSHans Rosenfeld 6603c9168faSHans Rosenfeld return (val); 6613c9168faSHans Rosenfeld } 6623c9168faSHans Rosenfeld 6633c9168faSHans Rosenfeld static boolean_t 6643c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme_t *nvme) 6653c9168faSHans Rosenfeld { 6663c9168faSHans Rosenfeld ddi_fm_error_t error; 6673c9168faSHans Rosenfeld 6683c9168faSHans Rosenfeld ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION); 6693c9168faSHans Rosenfeld 6703c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 6713c9168faSHans Rosenfeld return (B_TRUE); 6723c9168faSHans Rosenfeld 6733c9168faSHans Rosenfeld return (B_FALSE); 6743c9168faSHans Rosenfeld } 6753c9168faSHans Rosenfeld 6763c9168faSHans Rosenfeld static boolean_t 6773c9168faSHans Rosenfeld nvme_check_dma_hdl(nvme_dma_t *dma) 6783c9168faSHans Rosenfeld { 6793c9168faSHans Rosenfeld ddi_fm_error_t error; 6803c9168faSHans Rosenfeld 6813c9168faSHans Rosenfeld if (dma == NULL) 6823c9168faSHans Rosenfeld return (B_FALSE); 6833c9168faSHans Rosenfeld 6843c9168faSHans Rosenfeld ddi_fm_dma_err_get(dma->nd_dmah, &error, DDI_FME_VERSION); 6853c9168faSHans Rosenfeld 6863c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 6873c9168faSHans Rosenfeld return (B_TRUE); 6883c9168faSHans Rosenfeld 6893c9168faSHans Rosenfeld return (B_FALSE); 6903c9168faSHans Rosenfeld } 6913c9168faSHans Rosenfeld 6923c9168faSHans Rosenfeld static void 6938834f7acSYouzhong Yang nvme_free_dma_common(nvme_dma_t *dma) 6943c9168faSHans Rosenfeld { 6953c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 6963c9168faSHans Rosenfeld (void) ddi_dma_unbind_handle(dma->nd_dmah); 6973c9168faSHans Rosenfeld if (dma->nd_acch != NULL) 6983c9168faSHans Rosenfeld ddi_dma_mem_free(&dma->nd_acch); 6993c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 7003c9168faSHans Rosenfeld ddi_dma_free_handle(&dma->nd_dmah); 7013c9168faSHans Rosenfeld } 7023c9168faSHans Rosenfeld 7038834f7acSYouzhong Yang static void 7048834f7acSYouzhong Yang nvme_free_dma(nvme_dma_t *dma) 7053c9168faSHans Rosenfeld { 7068834f7acSYouzhong Yang nvme_free_dma_common(dma); 7078834f7acSYouzhong Yang kmem_free(dma, sizeof (*dma)); 7088834f7acSYouzhong Yang } 7098834f7acSYouzhong Yang 710b6bc2fd4SDan McDonald /* ARGSUSED */ 7118834f7acSYouzhong Yang static void 7128834f7acSYouzhong Yang nvme_prp_dma_destructor(void *buf, void *private) 7138834f7acSYouzhong Yang { 7148834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 7153c9168faSHans Rosenfeld 7168834f7acSYouzhong Yang nvme_free_dma_common(dma); 7178834f7acSYouzhong Yang } 7188834f7acSYouzhong Yang 7198834f7acSYouzhong Yang static int 7208834f7acSYouzhong Yang nvme_alloc_dma_common(nvme_t *nvme, nvme_dma_t *dma, 7218834f7acSYouzhong Yang size_t len, uint_t flags, ddi_dma_attr_t *dma_attr) 7228834f7acSYouzhong Yang { 7233c9168faSHans Rosenfeld if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL, 7243c9168faSHans Rosenfeld &dma->nd_dmah) != DDI_SUCCESS) { 7253c9168faSHans Rosenfeld /* 7263c9168faSHans Rosenfeld * Due to DDI_DMA_SLEEP this can't be DDI_DMA_NORESOURCES, and 7273c9168faSHans Rosenfeld * the only other possible error is DDI_DMA_BADATTR which 7283c9168faSHans Rosenfeld * indicates a driver bug which should cause a panic. 7293c9168faSHans Rosenfeld */ 7303c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, 7313c9168faSHans Rosenfeld "!failed to get DMA handle, check DMA attributes"); 7323c9168faSHans Rosenfeld return (DDI_FAILURE); 7333c9168faSHans Rosenfeld } 7343c9168faSHans Rosenfeld 7353c9168faSHans Rosenfeld /* 7363c9168faSHans Rosenfeld * ddi_dma_mem_alloc() can only fail when DDI_DMA_NOSLEEP is specified 7373c9168faSHans Rosenfeld * or the flags are conflicting, which isn't the case here. 7383c9168faSHans Rosenfeld */ 7393c9168faSHans Rosenfeld (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr, 7403c9168faSHans Rosenfeld DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &dma->nd_memp, 7413c9168faSHans Rosenfeld &dma->nd_len, &dma->nd_acch); 7423c9168faSHans Rosenfeld 7433c9168faSHans Rosenfeld if (ddi_dma_addr_bind_handle(dma->nd_dmah, NULL, dma->nd_memp, 7443c9168faSHans Rosenfeld dma->nd_len, flags | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, 7453c9168faSHans Rosenfeld &dma->nd_cookie, &dma->nd_ncookie) != DDI_DMA_MAPPED) { 7463c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 7473c9168faSHans Rosenfeld "!failed to bind DMA memory"); 7483c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_dma_bind_err); 7498834f7acSYouzhong Yang nvme_free_dma_common(dma); 7508834f7acSYouzhong Yang return (DDI_FAILURE); 7518834f7acSYouzhong Yang } 7528834f7acSYouzhong Yang 7538834f7acSYouzhong Yang return (DDI_SUCCESS); 7548834f7acSYouzhong Yang } 7558834f7acSYouzhong Yang 7568834f7acSYouzhong Yang static int 7578834f7acSYouzhong Yang nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags, 7588834f7acSYouzhong Yang ddi_dma_attr_t *dma_attr, nvme_dma_t **ret) 7598834f7acSYouzhong Yang { 7608834f7acSYouzhong Yang nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP); 7618834f7acSYouzhong Yang 7628834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, len, flags, dma_attr) != 7638834f7acSYouzhong Yang DDI_SUCCESS) { 7643c9168faSHans Rosenfeld *ret = NULL; 7658834f7acSYouzhong Yang kmem_free(dma, sizeof (nvme_dma_t)); 7663c9168faSHans Rosenfeld return (DDI_FAILURE); 7673c9168faSHans Rosenfeld } 7683c9168faSHans Rosenfeld 7693c9168faSHans Rosenfeld bzero(dma->nd_memp, dma->nd_len); 7703c9168faSHans Rosenfeld 7713c9168faSHans Rosenfeld *ret = dma; 7723c9168faSHans Rosenfeld return (DDI_SUCCESS); 7733c9168faSHans Rosenfeld } 7743c9168faSHans Rosenfeld 775b6bc2fd4SDan McDonald /* ARGSUSED */ 7768834f7acSYouzhong Yang static int 7778834f7acSYouzhong Yang nvme_prp_dma_constructor(void *buf, void *private, int flags) 7788834f7acSYouzhong Yang { 7798834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 7808834f7acSYouzhong Yang nvme_t *nvme = (nvme_t *)private; 7818834f7acSYouzhong Yang 7828834f7acSYouzhong Yang dma->nd_dmah = NULL; 7838834f7acSYouzhong Yang dma->nd_acch = NULL; 7848834f7acSYouzhong Yang 7858834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, nvme->n_pagesize, 7868834f7acSYouzhong Yang DDI_DMA_READ, &nvme->n_prp_dma_attr) != DDI_SUCCESS) { 7878834f7acSYouzhong Yang return (-1); 7888834f7acSYouzhong Yang } 7898834f7acSYouzhong Yang 7908834f7acSYouzhong Yang ASSERT(dma->nd_ncookie == 1); 7918834f7acSYouzhong Yang 7928834f7acSYouzhong Yang dma->nd_cached = B_TRUE; 7938834f7acSYouzhong Yang 7948834f7acSYouzhong Yang return (0); 7958834f7acSYouzhong Yang } 7968834f7acSYouzhong Yang 7973c9168faSHans Rosenfeld static int 7983c9168faSHans Rosenfeld nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len, 7993c9168faSHans Rosenfeld uint_t flags, nvme_dma_t **dma) 8003c9168faSHans Rosenfeld { 8013c9168faSHans Rosenfeld uint32_t len = nentry * qe_len; 8023c9168faSHans Rosenfeld ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr; 8033c9168faSHans Rosenfeld 8043c9168faSHans Rosenfeld len = roundup(len, nvme->n_pagesize); 8053c9168faSHans Rosenfeld 8063c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma) 8073c9168faSHans Rosenfeld != DDI_SUCCESS) { 8083c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 8093c9168faSHans Rosenfeld "!failed to get DMA memory for queue"); 8103c9168faSHans Rosenfeld goto fail; 8113c9168faSHans Rosenfeld } 8123c9168faSHans Rosenfeld 8133c9168faSHans Rosenfeld if ((*dma)->nd_ncookie != 1) { 8143c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 8153c9168faSHans Rosenfeld "!got too many cookies for queue DMA"); 8163c9168faSHans Rosenfeld goto fail; 8173c9168faSHans Rosenfeld } 8183c9168faSHans Rosenfeld 8193c9168faSHans Rosenfeld return (DDI_SUCCESS); 8203c9168faSHans Rosenfeld 8213c9168faSHans Rosenfeld fail: 8223c9168faSHans Rosenfeld if (*dma) { 8233c9168faSHans Rosenfeld nvme_free_dma(*dma); 8243c9168faSHans Rosenfeld *dma = NULL; 8253c9168faSHans Rosenfeld } 8263c9168faSHans Rosenfeld 8273c9168faSHans Rosenfeld return (DDI_FAILURE); 8283c9168faSHans Rosenfeld } 8293c9168faSHans Rosenfeld 8300999c112SPaul Winder static void 8310999c112SPaul Winder nvme_free_cq(nvme_cq_t *cq) 8320999c112SPaul Winder { 8330999c112SPaul Winder mutex_destroy(&cq->ncq_mutex); 8340999c112SPaul Winder 8354d95620bSPaul Winder if (cq->ncq_cmd_taskq != NULL) 8364d95620bSPaul Winder taskq_destroy(cq->ncq_cmd_taskq); 8374d95620bSPaul Winder 8380999c112SPaul Winder if (cq->ncq_dma != NULL) 8390999c112SPaul Winder nvme_free_dma(cq->ncq_dma); 8400999c112SPaul Winder 8410999c112SPaul Winder kmem_free(cq, sizeof (*cq)); 8420999c112SPaul Winder } 8430999c112SPaul Winder 8443c9168faSHans Rosenfeld static void 8453c9168faSHans Rosenfeld nvme_free_qpair(nvme_qpair_t *qp) 8463c9168faSHans Rosenfeld { 8473c9168faSHans Rosenfeld int i; 8483c9168faSHans Rosenfeld 8493c9168faSHans Rosenfeld mutex_destroy(&qp->nq_mutex); 8504b324362SHans Rosenfeld sema_destroy(&qp->nq_sema); 8513c9168faSHans Rosenfeld 8523c9168faSHans Rosenfeld if (qp->nq_sqdma != NULL) 8533c9168faSHans Rosenfeld nvme_free_dma(qp->nq_sqdma); 8543c9168faSHans Rosenfeld 8553c9168faSHans Rosenfeld if (qp->nq_active_cmds > 0) 8563c9168faSHans Rosenfeld for (i = 0; i != qp->nq_nentry; i++) 8573c9168faSHans Rosenfeld if (qp->nq_cmd[i] != NULL) 8583c9168faSHans Rosenfeld nvme_free_cmd(qp->nq_cmd[i]); 8593c9168faSHans Rosenfeld 8603c9168faSHans Rosenfeld if (qp->nq_cmd != NULL) 8613c9168faSHans Rosenfeld kmem_free(qp->nq_cmd, sizeof (nvme_cmd_t *) * qp->nq_nentry); 8623c9168faSHans Rosenfeld 8633c9168faSHans Rosenfeld kmem_free(qp, sizeof (nvme_qpair_t)); 8643c9168faSHans Rosenfeld } 8653c9168faSHans Rosenfeld 8660999c112SPaul Winder /* 8670999c112SPaul Winder * Destroy the pre-allocated cq array, but only free individual completion 8680999c112SPaul Winder * queues from the given starting index. 8690999c112SPaul Winder */ 8700999c112SPaul Winder static void 8710999c112SPaul Winder nvme_destroy_cq_array(nvme_t *nvme, uint_t start) 8720999c112SPaul Winder { 8730999c112SPaul Winder uint_t i; 8740999c112SPaul Winder 8750999c112SPaul Winder for (i = start; i < nvme->n_cq_count; i++) 8760999c112SPaul Winder if (nvme->n_cq[i] != NULL) 8770999c112SPaul Winder nvme_free_cq(nvme->n_cq[i]); 8780999c112SPaul Winder 8790999c112SPaul Winder kmem_free(nvme->n_cq, sizeof (*nvme->n_cq) * nvme->n_cq_count); 8800999c112SPaul Winder } 8810999c112SPaul Winder 8820999c112SPaul Winder static int 8834d95620bSPaul Winder nvme_alloc_cq(nvme_t *nvme, uint32_t nentry, nvme_cq_t **cqp, uint16_t idx, 8844d95620bSPaul Winder uint_t nthr) 8850999c112SPaul Winder { 8860999c112SPaul Winder nvme_cq_t *cq = kmem_zalloc(sizeof (*cq), KM_SLEEP); 8874d95620bSPaul Winder char name[64]; /* large enough for the taskq name */ 8880999c112SPaul Winder 8890999c112SPaul Winder mutex_init(&cq->ncq_mutex, NULL, MUTEX_DRIVER, 8900999c112SPaul Winder DDI_INTR_PRI(nvme->n_intr_pri)); 8910999c112SPaul Winder 8920999c112SPaul Winder if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t), 8930999c112SPaul Winder DDI_DMA_READ, &cq->ncq_dma) != DDI_SUCCESS) 8940999c112SPaul Winder goto fail; 8950999c112SPaul Winder 8960999c112SPaul Winder cq->ncq_cq = (nvme_cqe_t *)cq->ncq_dma->nd_memp; 8970999c112SPaul Winder cq->ncq_nentry = nentry; 8980999c112SPaul Winder cq->ncq_id = idx; 8990999c112SPaul Winder cq->ncq_hdbl = NVME_REG_CQHDBL(nvme, idx); 9000999c112SPaul Winder 9014d95620bSPaul Winder /* 9024d95620bSPaul Winder * Each completion queue has its own command taskq. 9034d95620bSPaul Winder */ 9044d95620bSPaul Winder (void) snprintf(name, sizeof (name), "%s%d_cmd_taskq%u", 9054d95620bSPaul Winder ddi_driver_name(nvme->n_dip), ddi_get_instance(nvme->n_dip), idx); 9064d95620bSPaul Winder 9074d95620bSPaul Winder cq->ncq_cmd_taskq = taskq_create(name, nthr, minclsyspri, 64, INT_MAX, 9084d95620bSPaul Winder TASKQ_PREPOPULATE); 9094d95620bSPaul Winder 9104d95620bSPaul Winder if (cq->ncq_cmd_taskq == NULL) { 9114d95620bSPaul Winder dev_err(nvme->n_dip, CE_WARN, "!failed to create cmd " 9124d95620bSPaul Winder "taskq for cq %u", idx); 9134d95620bSPaul Winder goto fail; 9144d95620bSPaul Winder } 9154d95620bSPaul Winder 9160999c112SPaul Winder *cqp = cq; 9170999c112SPaul Winder return (DDI_SUCCESS); 9180999c112SPaul Winder 9190999c112SPaul Winder fail: 9200999c112SPaul Winder nvme_free_cq(cq); 9210999c112SPaul Winder *cqp = NULL; 9220999c112SPaul Winder 9230999c112SPaul Winder return (DDI_FAILURE); 9240999c112SPaul Winder } 9250999c112SPaul Winder 9260999c112SPaul Winder /* 9270999c112SPaul Winder * Create the n_cq array big enough to hold "ncq" completion queues. 9280999c112SPaul Winder * If the array already exists it will be re-sized (but only larger). 9290999c112SPaul Winder * The admin queue is included in this array, which boosts the 9300999c112SPaul Winder * max number of entries to UINT16_MAX + 1. 9310999c112SPaul Winder */ 9320999c112SPaul Winder static int 9334d95620bSPaul Winder nvme_create_cq_array(nvme_t *nvme, uint_t ncq, uint32_t nentry, uint_t nthr) 9340999c112SPaul Winder { 9350999c112SPaul Winder nvme_cq_t **cq; 9360999c112SPaul Winder uint_t i, cq_count; 9370999c112SPaul Winder 9380999c112SPaul Winder ASSERT3U(ncq, >, nvme->n_cq_count); 9390999c112SPaul Winder 9400999c112SPaul Winder cq = nvme->n_cq; 9410999c112SPaul Winder cq_count = nvme->n_cq_count; 9420999c112SPaul Winder 9430999c112SPaul Winder nvme->n_cq = kmem_zalloc(sizeof (*nvme->n_cq) * ncq, KM_SLEEP); 9440999c112SPaul Winder nvme->n_cq_count = ncq; 9450999c112SPaul Winder 9460999c112SPaul Winder for (i = 0; i < cq_count; i++) 9470999c112SPaul Winder nvme->n_cq[i] = cq[i]; 9480999c112SPaul Winder 9490999c112SPaul Winder for (; i < nvme->n_cq_count; i++) 9504d95620bSPaul Winder if (nvme_alloc_cq(nvme, nentry, &nvme->n_cq[i], i, nthr) != 9510999c112SPaul Winder DDI_SUCCESS) 9520999c112SPaul Winder goto fail; 9530999c112SPaul Winder 9540999c112SPaul Winder if (cq != NULL) 9550999c112SPaul Winder kmem_free(cq, sizeof (*cq) * cq_count); 9560999c112SPaul Winder 9570999c112SPaul Winder return (DDI_SUCCESS); 9580999c112SPaul Winder 9590999c112SPaul Winder fail: 9600999c112SPaul Winder nvme_destroy_cq_array(nvme, cq_count); 9610999c112SPaul Winder /* 9620999c112SPaul Winder * Restore the original array 9630999c112SPaul Winder */ 9640999c112SPaul Winder nvme->n_cq_count = cq_count; 9650999c112SPaul Winder nvme->n_cq = cq; 9660999c112SPaul Winder 9670999c112SPaul Winder return (DDI_FAILURE); 9680999c112SPaul Winder } 9690999c112SPaul Winder 9703c9168faSHans Rosenfeld static int 9713c9168faSHans Rosenfeld nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp, 9720999c112SPaul Winder uint_t idx) 9733c9168faSHans Rosenfeld { 9743c9168faSHans Rosenfeld nvme_qpair_t *qp = kmem_zalloc(sizeof (*qp), KM_SLEEP); 9750999c112SPaul Winder uint_t cq_idx; 9763c9168faSHans Rosenfeld 9773c9168faSHans Rosenfeld mutex_init(&qp->nq_mutex, NULL, MUTEX_DRIVER, 9783c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 9790999c112SPaul Winder 9800999c112SPaul Winder /* 9810999c112SPaul Winder * The NVMe spec defines that a full queue has one empty (unused) slot; 9820999c112SPaul Winder * initialize the semaphore accordingly. 9830999c112SPaul Winder */ 9840999c112SPaul Winder sema_init(&qp->nq_sema, nentry - 1, NULL, SEMA_DRIVER, NULL); 9853c9168faSHans Rosenfeld 9863c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t), 9873c9168faSHans Rosenfeld DDI_DMA_WRITE, &qp->nq_sqdma) != DDI_SUCCESS) 9883c9168faSHans Rosenfeld goto fail; 9893c9168faSHans Rosenfeld 9900999c112SPaul Winder /* 9910999c112SPaul Winder * idx == 0 is adminq, those above 0 are shared io completion queues. 9920999c112SPaul Winder */ 9930999c112SPaul Winder cq_idx = idx == 0 ? 0 : 1 + (idx - 1) % (nvme->n_cq_count - 1); 9940999c112SPaul Winder qp->nq_cq = nvme->n_cq[cq_idx]; 9953c9168faSHans Rosenfeld qp->nq_sq = (nvme_sqe_t *)qp->nq_sqdma->nd_memp; 9963c9168faSHans Rosenfeld qp->nq_nentry = nentry; 9973c9168faSHans Rosenfeld 9983c9168faSHans Rosenfeld qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx); 9993c9168faSHans Rosenfeld 10003c9168faSHans Rosenfeld qp->nq_cmd = kmem_zalloc(sizeof (nvme_cmd_t *) * nentry, KM_SLEEP); 10013c9168faSHans Rosenfeld qp->nq_next_cmd = 0; 10023c9168faSHans Rosenfeld 10033c9168faSHans Rosenfeld *nqp = qp; 10043c9168faSHans Rosenfeld return (DDI_SUCCESS); 10053c9168faSHans Rosenfeld 10063c9168faSHans Rosenfeld fail: 10073c9168faSHans Rosenfeld nvme_free_qpair(qp); 10083c9168faSHans Rosenfeld *nqp = NULL; 10093c9168faSHans Rosenfeld 10103c9168faSHans Rosenfeld return (DDI_FAILURE); 10113c9168faSHans Rosenfeld } 10123c9168faSHans Rosenfeld 10133c9168faSHans Rosenfeld static nvme_cmd_t * 10143c9168faSHans Rosenfeld nvme_alloc_cmd(nvme_t *nvme, int kmflag) 10153c9168faSHans Rosenfeld { 10163c9168faSHans Rosenfeld nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag); 10173c9168faSHans Rosenfeld 10183c9168faSHans Rosenfeld if (cmd == NULL) 10193c9168faSHans Rosenfeld return (cmd); 10203c9168faSHans Rosenfeld 10213c9168faSHans Rosenfeld bzero(cmd, sizeof (nvme_cmd_t)); 10223c9168faSHans Rosenfeld 10233c9168faSHans Rosenfeld cmd->nc_nvme = nvme; 10243c9168faSHans Rosenfeld 10253c9168faSHans Rosenfeld mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER, 10263c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 10273c9168faSHans Rosenfeld cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL); 10283c9168faSHans Rosenfeld 10293c9168faSHans Rosenfeld return (cmd); 10303c9168faSHans Rosenfeld } 10313c9168faSHans Rosenfeld 10323c9168faSHans Rosenfeld static void 10333c9168faSHans Rosenfeld nvme_free_cmd(nvme_cmd_t *cmd) 10343c9168faSHans Rosenfeld { 1035e984c70bSHans Rosenfeld /* Don't free commands on the lost commands list. */ 1036e984c70bSHans Rosenfeld if (list_link_active(&cmd->nc_list)) 1037e984c70bSHans Rosenfeld return; 1038e984c70bSHans Rosenfeld 10393c9168faSHans Rosenfeld if (cmd->nc_dma) { 10408834f7acSYouzhong Yang if (cmd->nc_dma->nd_cached) 10418834f7acSYouzhong Yang kmem_cache_free(cmd->nc_nvme->n_prp_cache, 10428834f7acSYouzhong Yang cmd->nc_dma); 10438834f7acSYouzhong Yang else 10448834f7acSYouzhong Yang nvme_free_dma(cmd->nc_dma); 10453c9168faSHans Rosenfeld cmd->nc_dma = NULL; 10463c9168faSHans Rosenfeld } 10473c9168faSHans Rosenfeld 10483c9168faSHans Rosenfeld cv_destroy(&cmd->nc_cv); 10493c9168faSHans Rosenfeld mutex_destroy(&cmd->nc_mutex); 10503c9168faSHans Rosenfeld 10513c9168faSHans Rosenfeld kmem_cache_free(nvme_cmd_cache, cmd); 10523c9168faSHans Rosenfeld } 10533c9168faSHans Rosenfeld 10544b324362SHans Rosenfeld static void 10554b324362SHans Rosenfeld nvme_submit_admin_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 10564b324362SHans Rosenfeld { 10574b324362SHans Rosenfeld sema_p(&qp->nq_sema); 10584b324362SHans Rosenfeld nvme_submit_cmd_common(qp, cmd); 10594b324362SHans Rosenfeld } 10604b324362SHans Rosenfeld 10613c9168faSHans Rosenfeld static int 10624b324362SHans Rosenfeld nvme_submit_io_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 10633c9168faSHans Rosenfeld { 10644b324362SHans Rosenfeld if (sema_tryp(&qp->nq_sema) == 0) 10654b324362SHans Rosenfeld return (EAGAIN); 10663c9168faSHans Rosenfeld 10674b324362SHans Rosenfeld nvme_submit_cmd_common(qp, cmd); 10684b324362SHans Rosenfeld return (0); 10694b324362SHans Rosenfeld } 10703c9168faSHans Rosenfeld 10714b324362SHans Rosenfeld static void 10724b324362SHans Rosenfeld nvme_submit_cmd_common(nvme_qpair_t *qp, nvme_cmd_t *cmd) 10734b324362SHans Rosenfeld { 10744b324362SHans Rosenfeld nvme_reg_sqtdbl_t tail = { 0 }; 10753c9168faSHans Rosenfeld 10764b324362SHans Rosenfeld mutex_enter(&qp->nq_mutex); 10773c9168faSHans Rosenfeld cmd->nc_completed = B_FALSE; 10783c9168faSHans Rosenfeld 10793c9168faSHans Rosenfeld /* 10803c9168faSHans Rosenfeld * Try to insert the cmd into the active cmd array at the nq_next_cmd 10813c9168faSHans Rosenfeld * slot. If the slot is already occupied advance to the next slot and 10823c9168faSHans Rosenfeld * try again. This can happen for long running commands like async event 10833c9168faSHans Rosenfeld * requests. 10843c9168faSHans Rosenfeld */ 10853c9168faSHans Rosenfeld while (qp->nq_cmd[qp->nq_next_cmd] != NULL) 10863c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 10873c9168faSHans Rosenfeld qp->nq_cmd[qp->nq_next_cmd] = cmd; 10883c9168faSHans Rosenfeld 10893c9168faSHans Rosenfeld qp->nq_active_cmds++; 10903c9168faSHans Rosenfeld 10913c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cid = qp->nq_next_cmd; 10923c9168faSHans Rosenfeld bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t)); 10933c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_sqdma->nd_dmah, 10943c9168faSHans Rosenfeld sizeof (nvme_sqe_t) * qp->nq_sqtail, 10953c9168faSHans Rosenfeld sizeof (nvme_sqe_t), DDI_DMA_SYNC_FORDEV); 10963c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 10973c9168faSHans Rosenfeld 10983c9168faSHans Rosenfeld tail.b.sqtdbl_sqt = qp->nq_sqtail = (qp->nq_sqtail + 1) % qp->nq_nentry; 10993c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r); 11003c9168faSHans Rosenfeld 11013c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 11023c9168faSHans Rosenfeld } 11033c9168faSHans Rosenfeld 1104e984c70bSHans Rosenfeld static nvme_cmd_t * 1105e984c70bSHans Rosenfeld nvme_unqueue_cmd(nvme_t *nvme, nvme_qpair_t *qp, int cid) 1106e984c70bSHans Rosenfeld { 1107e984c70bSHans Rosenfeld nvme_cmd_t *cmd; 1108e984c70bSHans Rosenfeld 1109e984c70bSHans Rosenfeld ASSERT(mutex_owned(&qp->nq_mutex)); 1110e984c70bSHans Rosenfeld ASSERT3S(cid, <, qp->nq_nentry); 1111e984c70bSHans Rosenfeld 1112e984c70bSHans Rosenfeld cmd = qp->nq_cmd[cid]; 1113e984c70bSHans Rosenfeld qp->nq_cmd[cid] = NULL; 1114e984c70bSHans Rosenfeld ASSERT3U(qp->nq_active_cmds, >, 0); 1115e984c70bSHans Rosenfeld qp->nq_active_cmds--; 1116e984c70bSHans Rosenfeld sema_v(&qp->nq_sema); 1117e984c70bSHans Rosenfeld 1118e984c70bSHans Rosenfeld ASSERT3P(cmd, !=, NULL); 1119e984c70bSHans Rosenfeld ASSERT3P(cmd->nc_nvme, ==, nvme); 1120e984c70bSHans Rosenfeld ASSERT3S(cmd->nc_sqe.sqe_cid, ==, cid); 1121e984c70bSHans Rosenfeld 1122e984c70bSHans Rosenfeld return (cmd); 1123e984c70bSHans Rosenfeld } 1124e984c70bSHans Rosenfeld 11250999c112SPaul Winder /* 11260999c112SPaul Winder * Get the command tied to the next completed cqe and bump along completion 11270999c112SPaul Winder * queue head counter. 11280999c112SPaul Winder */ 11293c9168faSHans Rosenfeld static nvme_cmd_t * 11300999c112SPaul Winder nvme_get_completed(nvme_t *nvme, nvme_cq_t *cq) 11313c9168faSHans Rosenfeld { 11320999c112SPaul Winder nvme_qpair_t *qp; 11333c9168faSHans Rosenfeld nvme_cqe_t *cqe; 11343c9168faSHans Rosenfeld nvme_cmd_t *cmd; 11353c9168faSHans Rosenfeld 11360999c112SPaul Winder ASSERT(mutex_owned(&cq->ncq_mutex)); 11373c9168faSHans Rosenfeld 11380999c112SPaul Winder cqe = &cq->ncq_cq[cq->ncq_head]; 11393c9168faSHans Rosenfeld 11403c9168faSHans Rosenfeld /* Check phase tag of CQE. Hardware inverts it for new entries. */ 11410999c112SPaul Winder if (cqe->cqe_sf.sf_p == cq->ncq_phase) 11423c9168faSHans Rosenfeld return (NULL); 11433c9168faSHans Rosenfeld 11440999c112SPaul Winder qp = nvme->n_ioq[cqe->cqe_sqid]; 11453c9168faSHans Rosenfeld 11460999c112SPaul Winder mutex_enter(&qp->nq_mutex); 1147e984c70bSHans Rosenfeld cmd = nvme_unqueue_cmd(nvme, qp, cqe->cqe_cid); 11480999c112SPaul Winder mutex_exit(&qp->nq_mutex); 11493c9168faSHans Rosenfeld 11503c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid == cqe->cqe_sqid); 11513c9168faSHans Rosenfeld bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t)); 11523c9168faSHans Rosenfeld 11533c9168faSHans Rosenfeld qp->nq_sqhead = cqe->cqe_sqhd; 11543c9168faSHans Rosenfeld 11550999c112SPaul Winder cq->ncq_head = (cq->ncq_head + 1) % cq->ncq_nentry; 11563c9168faSHans Rosenfeld 11573c9168faSHans Rosenfeld /* Toggle phase on wrap-around. */ 11580999c112SPaul Winder if (cq->ncq_head == 0) 11590999c112SPaul Winder cq->ncq_phase = cq->ncq_phase ? 0 : 1; 11603c9168faSHans Rosenfeld 11610999c112SPaul Winder return (cmd); 11620999c112SPaul Winder } 11630999c112SPaul Winder 11640999c112SPaul Winder /* 11650999c112SPaul Winder * Process all completed commands on the io completion queue. 11660999c112SPaul Winder */ 11670999c112SPaul Winder static uint_t 11680999c112SPaul Winder nvme_process_iocq(nvme_t *nvme, nvme_cq_t *cq) 11690999c112SPaul Winder { 11700999c112SPaul Winder nvme_reg_cqhdbl_t head = { 0 }; 11710999c112SPaul Winder nvme_cmd_t *cmd; 11720999c112SPaul Winder uint_t completed = 0; 11730999c112SPaul Winder 11740999c112SPaul Winder if (ddi_dma_sync(cq->ncq_dma->nd_dmah, 0, 0, DDI_DMA_SYNC_FORKERNEL) != 11750999c112SPaul Winder DDI_SUCCESS) 11760999c112SPaul Winder dev_err(nvme->n_dip, CE_WARN, "!ddi_dma_sync() failed in %s", 11770999c112SPaul Winder __func__); 11780999c112SPaul Winder 11790999c112SPaul Winder mutex_enter(&cq->ncq_mutex); 11800999c112SPaul Winder 11810999c112SPaul Winder while ((cmd = nvme_get_completed(nvme, cq)) != NULL) { 11824d95620bSPaul Winder taskq_dispatch_ent(cq->ncq_cmd_taskq, cmd->nc_callback, cmd, 11834d95620bSPaul Winder TQ_NOSLEEP, &cmd->nc_tqent); 11840999c112SPaul Winder 11850999c112SPaul Winder completed++; 11860999c112SPaul Winder } 11870999c112SPaul Winder 11880999c112SPaul Winder if (completed > 0) { 11890999c112SPaul Winder /* 11900999c112SPaul Winder * Update the completion queue head doorbell. 11910999c112SPaul Winder */ 11920999c112SPaul Winder head.b.cqhdbl_cqh = cq->ncq_head; 11930999c112SPaul Winder nvme_put32(nvme, cq->ncq_hdbl, head.r); 11940999c112SPaul Winder } 11950999c112SPaul Winder 11960999c112SPaul Winder mutex_exit(&cq->ncq_mutex); 11970999c112SPaul Winder 11980999c112SPaul Winder return (completed); 11990999c112SPaul Winder } 12000999c112SPaul Winder 12010999c112SPaul Winder static nvme_cmd_t * 12020999c112SPaul Winder nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp) 12030999c112SPaul Winder { 12040999c112SPaul Winder nvme_cq_t *cq = qp->nq_cq; 12050999c112SPaul Winder nvme_reg_cqhdbl_t head = { 0 }; 12060999c112SPaul Winder nvme_cmd_t *cmd; 12070999c112SPaul Winder 12080999c112SPaul Winder if (ddi_dma_sync(cq->ncq_dma->nd_dmah, 0, 0, DDI_DMA_SYNC_FORKERNEL) != 12090999c112SPaul Winder DDI_SUCCESS) 12100999c112SPaul Winder dev_err(nvme->n_dip, CE_WARN, "!ddi_dma_sync() failed in %s", 12110999c112SPaul Winder __func__); 12120999c112SPaul Winder 12130999c112SPaul Winder mutex_enter(&cq->ncq_mutex); 12140999c112SPaul Winder 12150999c112SPaul Winder if ((cmd = nvme_get_completed(nvme, cq)) != NULL) { 12160999c112SPaul Winder head.b.cqhdbl_cqh = cq->ncq_head; 12170999c112SPaul Winder nvme_put32(nvme, cq->ncq_hdbl, head.r); 12180999c112SPaul Winder } 12190999c112SPaul Winder 12200999c112SPaul Winder mutex_exit(&cq->ncq_mutex); 12213c9168faSHans Rosenfeld 12223c9168faSHans Rosenfeld return (cmd); 12233c9168faSHans Rosenfeld } 12243c9168faSHans Rosenfeld 12253c9168faSHans Rosenfeld static int 12263c9168faSHans Rosenfeld nvme_check_unknown_cmd_status(nvme_cmd_t *cmd) 12273c9168faSHans Rosenfeld { 12283c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 12293c9168faSHans Rosenfeld 12303c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 12313c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 12323c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 12333c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 12343c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 12353c9168faSHans Rosenfeld 12363d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 12373d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 1238bef9e21aSHans Rosenfeld 12393c9168faSHans Rosenfeld if (cmd->nc_nvme->n_strict_version) { 12403c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 12413c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 12423c9168faSHans Rosenfeld } 12433c9168faSHans Rosenfeld 12443c9168faSHans Rosenfeld return (EIO); 12453c9168faSHans Rosenfeld } 12463c9168faSHans Rosenfeld 12473c9168faSHans Rosenfeld static int 12483c9168faSHans Rosenfeld nvme_check_vendor_cmd_status(nvme_cmd_t *cmd) 12493c9168faSHans Rosenfeld { 12503c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 12513c9168faSHans Rosenfeld 12523c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 12533c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 12543c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 12553c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 12563c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 125734c938c7SPete Shephard if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) { 12583c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 12593c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 12603c9168faSHans Rosenfeld } 12613c9168faSHans Rosenfeld 12623c9168faSHans Rosenfeld return (EIO); 12633c9168faSHans Rosenfeld } 12643c9168faSHans Rosenfeld 12653c9168faSHans Rosenfeld static int 12663c9168faSHans Rosenfeld nvme_check_integrity_cmd_status(nvme_cmd_t *cmd) 12673c9168faSHans Rosenfeld { 12683c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 12693c9168faSHans Rosenfeld 12703c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 12713c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_WRITE: 12723c9168faSHans Rosenfeld /* write fail */ 12733c9168faSHans Rosenfeld /* TODO: post ereport */ 12743d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 12753d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 12763c9168faSHans Rosenfeld return (EIO); 12773c9168faSHans Rosenfeld 12783c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_READ: 12793c9168faSHans Rosenfeld /* read fail */ 12803c9168faSHans Rosenfeld /* TODO: post ereport */ 12813d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 12823d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 12833c9168faSHans Rosenfeld return (EIO); 12843c9168faSHans Rosenfeld 12853c9168faSHans Rosenfeld default: 12863c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 12873c9168faSHans Rosenfeld } 12883c9168faSHans Rosenfeld } 12893c9168faSHans Rosenfeld 12903c9168faSHans Rosenfeld static int 12913c9168faSHans Rosenfeld nvme_check_generic_cmd_status(nvme_cmd_t *cmd) 12923c9168faSHans Rosenfeld { 12933c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 12943c9168faSHans Rosenfeld 12953c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 12963c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_SUCCESS: 12973c9168faSHans Rosenfeld return (0); 12983c9168faSHans Rosenfeld 12993c9168faSHans Rosenfeld /* 13003c9168faSHans Rosenfeld * Errors indicating a bug in the driver should cause a panic. 13013c9168faSHans Rosenfeld */ 13023c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_OPC: 13033c9168faSHans Rosenfeld /* Invalid Command Opcode */ 130408139162SToomas Soome if (!cmd->nc_dontpanic) 130508139162SToomas Soome dev_err(cmd->nc_nvme->n_dip, CE_PANIC, 130608139162SToomas Soome "programming error: invalid opcode in cmd %p", 130708139162SToomas Soome (void *)cmd); 130808139162SToomas Soome return (EINVAL); 13093c9168faSHans Rosenfeld 13103c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_FLD: 13113c9168faSHans Rosenfeld /* Invalid Field in Command */ 13123d9b1a2aSHans Rosenfeld if (!cmd->nc_dontpanic) 13133d9b1a2aSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, 13143d9b1a2aSHans Rosenfeld "programming error: invalid field in cmd %p", 13153d9b1a2aSHans Rosenfeld (void *)cmd); 13163d9b1a2aSHans Rosenfeld return (EIO); 13173c9168faSHans Rosenfeld 13183c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ID_CNFL: 13193c9168faSHans Rosenfeld /* Command ID Conflict */ 13203c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 13213c9168faSHans Rosenfeld "cmd ID conflict in cmd %p", (void *)cmd); 13223c9168faSHans Rosenfeld return (0); 13233c9168faSHans Rosenfeld 13243c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_NS: 13253c9168faSHans Rosenfeld /* Invalid Namespace or Format */ 13263d9b1a2aSHans Rosenfeld if (!cmd->nc_dontpanic) 13273d9b1a2aSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, 1328f313c178SYuri Pankov "programming error: invalid NS/format in cmd %p", 13293d9b1a2aSHans Rosenfeld (void *)cmd); 13303d9b1a2aSHans Rosenfeld return (EINVAL); 13313c9168faSHans Rosenfeld 13323c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_LBA_RANGE: 13333c9168faSHans Rosenfeld /* LBA Out Of Range */ 13343c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 13353c9168faSHans Rosenfeld "LBA out of range in cmd %p", (void *)cmd); 13363c9168faSHans Rosenfeld return (0); 13373c9168faSHans Rosenfeld 13383c9168faSHans Rosenfeld /* 13393c9168faSHans Rosenfeld * Non-fatal errors, handle gracefully. 13403c9168faSHans Rosenfeld */ 13413c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_DATA_XFR_ERR: 13423c9168faSHans Rosenfeld /* Data Transfer Error (DMA) */ 13433c9168faSHans Rosenfeld /* TODO: post ereport */ 13443c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err); 13453d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 13463d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 13473c9168faSHans Rosenfeld return (EIO); 13483c9168faSHans Rosenfeld 13493c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INTERNAL_ERR: 13503c9168faSHans Rosenfeld /* 13513c9168faSHans Rosenfeld * Internal Error. The spec (v1.0, section 4.5.1.2) says 13523c9168faSHans Rosenfeld * detailed error information is returned as async event, 13533c9168faSHans Rosenfeld * so we pretty much ignore the error here and handle it 13543c9168faSHans Rosenfeld * in the async event handler. 13553c9168faSHans Rosenfeld */ 13563c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_internal_err); 13573d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 13583d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 13593c9168faSHans Rosenfeld return (EIO); 13603c9168faSHans Rosenfeld 13613c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_REQUEST: 13623c9168faSHans Rosenfeld /* 13633c9168faSHans Rosenfeld * Command Abort Requested. This normally happens only when a 13643c9168faSHans Rosenfeld * command times out. 13653c9168faSHans Rosenfeld */ 13663c9168faSHans Rosenfeld /* TODO: post ereport or change blkdev to handle this? */ 13673c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err); 13683c9168faSHans Rosenfeld return (ECANCELED); 13693c9168faSHans Rosenfeld 13703c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_PWRLOSS: 13713c9168faSHans Rosenfeld /* Command Aborted due to Power Loss Notification */ 13723c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 13733c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 13743c9168faSHans Rosenfeld return (EIO); 13753c9168faSHans Rosenfeld 13763c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_SQ_DEL: 13773c9168faSHans Rosenfeld /* Command Aborted due to SQ Deletion */ 13783c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del); 13793c9168faSHans Rosenfeld return (EIO); 13803c9168faSHans Rosenfeld 13813c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_CAP_EXC: 13823c9168faSHans Rosenfeld /* Capacity Exceeded */ 13833c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc); 13843d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 13853d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 13863c9168faSHans Rosenfeld return (EIO); 13873c9168faSHans Rosenfeld 13883c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_NS_NOTRDY: 13893c9168faSHans Rosenfeld /* Namespace Not Ready */ 13903c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy); 13913d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 13923d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 13933c9168faSHans Rosenfeld return (EIO); 13943c9168faSHans Rosenfeld 13953c9168faSHans Rosenfeld default: 13963c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 13973c9168faSHans Rosenfeld } 13983c9168faSHans Rosenfeld } 13993c9168faSHans Rosenfeld 14003c9168faSHans Rosenfeld static int 14013c9168faSHans Rosenfeld nvme_check_specific_cmd_status(nvme_cmd_t *cmd) 14023c9168faSHans Rosenfeld { 14033c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 14043c9168faSHans Rosenfeld 14053c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 14063c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_CQ: 14073c9168faSHans Rosenfeld /* Completion Queue Invalid */ 14083c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE); 14093c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err); 14103c9168faSHans Rosenfeld return (EINVAL); 14113c9168faSHans Rosenfeld 14123c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_QID: 14133c9168faSHans Rosenfeld /* Invalid Queue Identifier */ 14143c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 14153c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE || 14163c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE || 14173c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 14183c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err); 14193c9168faSHans Rosenfeld return (EINVAL); 14203c9168faSHans Rosenfeld 14213c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_MAX_QSZ_EXC: 14223c9168faSHans Rosenfeld /* Max Queue Size Exceeded */ 14233c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 14243c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 14253c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc); 14263c9168faSHans Rosenfeld return (EINVAL); 14273c9168faSHans Rosenfeld 14283c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ABRT_CMD_EXC: 14293c9168faSHans Rosenfeld /* Abort Command Limit Exceeded */ 14303c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT); 14313c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 14323c9168faSHans Rosenfeld "abort command limit exceeded in cmd %p", (void *)cmd); 14333c9168faSHans Rosenfeld return (0); 14343c9168faSHans Rosenfeld 14353c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC: 14363c9168faSHans Rosenfeld /* Async Event Request Limit Exceeded */ 14373c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT); 14383c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 14393c9168faSHans Rosenfeld "async event request limit exceeded in cmd %p", 14403c9168faSHans Rosenfeld (void *)cmd); 14413c9168faSHans Rosenfeld return (0); 14423c9168faSHans Rosenfeld 14433c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_INT_VECT: 14443c9168faSHans Rosenfeld /* Invalid Interrupt Vector */ 14453c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 14463c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect); 14473c9168faSHans Rosenfeld return (EINVAL); 14483c9168faSHans Rosenfeld 14493c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_LOG_PAGE: 14503c9168faSHans Rosenfeld /* Invalid Log Page */ 14513c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE); 14523c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_log_page); 14533c9168faSHans Rosenfeld return (EINVAL); 14543c9168faSHans Rosenfeld 14553c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_FORMAT: 14563c9168faSHans Rosenfeld /* Invalid Format */ 14573c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT); 14583c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_format); 14593d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 14603d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 14613c9168faSHans Rosenfeld return (EINVAL); 14623c9168faSHans Rosenfeld 14633c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_Q_DEL: 14643c9168faSHans Rosenfeld /* Invalid Queue Deletion */ 14653c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 14663c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_q_del); 14673c9168faSHans Rosenfeld return (EINVAL); 14683c9168faSHans Rosenfeld 14693c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_CNFL_ATTR: 14703c9168faSHans Rosenfeld /* Conflicting Attributes */ 14713c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT || 14723c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 14733c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 14743c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr); 14753d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 14763d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 14773c9168faSHans Rosenfeld return (EINVAL); 14783c9168faSHans Rosenfeld 14793c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_INV_PROT: 14803c9168faSHans Rosenfeld /* Invalid Protection Information */ 14813c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE || 14823c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 14833c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 14843c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_prot); 14853d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 14863d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 14873c9168faSHans Rosenfeld return (EINVAL); 14883c9168faSHans Rosenfeld 14893c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_READONLY: 14903c9168faSHans Rosenfeld /* Write to Read Only Range */ 14913c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 14923c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_readonly); 14933d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 14943d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 14953c9168faSHans Rosenfeld return (EROFS); 14963c9168faSHans Rosenfeld 1497cf840871SPaul Winder case NVME_CQE_SC_SPC_INV_FW_SLOT: 1498cf840871SPaul Winder /* Invalid Firmware Slot */ 1499cf840871SPaul Winder ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); 1500cf840871SPaul Winder return (EINVAL); 1501cf840871SPaul Winder 1502cf840871SPaul Winder case NVME_CQE_SC_SPC_INV_FW_IMG: 1503cf840871SPaul Winder /* Invalid Firmware Image */ 1504cf840871SPaul Winder ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); 1505cf840871SPaul Winder return (EINVAL); 1506cf840871SPaul Winder 1507cf840871SPaul Winder case NVME_CQE_SC_SPC_FW_RESET: 1508cf840871SPaul Winder /* Conventional Reset Required */ 1509cf840871SPaul Winder ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); 1510cf840871SPaul Winder return (0); 1511cf840871SPaul Winder 1512cf840871SPaul Winder case NVME_CQE_SC_SPC_FW_NSSR: 1513cf840871SPaul Winder /* NVMe Subsystem Reset Required */ 1514cf840871SPaul Winder ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); 1515cf840871SPaul Winder return (0); 1516cf840871SPaul Winder 1517cf840871SPaul Winder case NVME_CQE_SC_SPC_FW_NEXT_RESET: 1518cf840871SPaul Winder /* Activation Requires Reset */ 1519cf840871SPaul Winder ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); 1520cf840871SPaul Winder return (0); 1521cf840871SPaul Winder 1522cf840871SPaul Winder case NVME_CQE_SC_SPC_FW_MTFA: 1523cf840871SPaul Winder /* Activation Requires Maximum Time Violation */ 1524cf840871SPaul Winder ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); 1525cf840871SPaul Winder return (EAGAIN); 1526cf840871SPaul Winder 1527cf840871SPaul Winder case NVME_CQE_SC_SPC_FW_PROHIBITED: 1528cf840871SPaul Winder /* Activation Prohibited */ 1529cf840871SPaul Winder ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); 1530cf840871SPaul Winder return (EINVAL); 1531cf840871SPaul Winder 1532cf840871SPaul Winder case NVME_CQE_SC_SPC_FW_OVERLAP: 1533cf840871SPaul Winder /* Overlapping Firmware Ranges */ 1534cf840871SPaul Winder ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_IMAGE_LOAD); 1535cf840871SPaul Winder return (EINVAL); 1536cf840871SPaul Winder 15373c9168faSHans Rosenfeld default: 15383c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 15393c9168faSHans Rosenfeld } 15403c9168faSHans Rosenfeld } 15413c9168faSHans Rosenfeld 15423c9168faSHans Rosenfeld static inline int 15433c9168faSHans Rosenfeld nvme_check_cmd_status(nvme_cmd_t *cmd) 15443c9168faSHans Rosenfeld { 15453c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 15463c9168faSHans Rosenfeld 1547e984c70bSHans Rosenfeld /* 1548e984c70bSHans Rosenfeld * Take a shortcut if the controller is dead, or if 1549e984c70bSHans Rosenfeld * command status indicates no error. 1550e984c70bSHans Rosenfeld */ 1551e984c70bSHans Rosenfeld if (cmd->nc_nvme->n_dead) 1552e984c70bSHans Rosenfeld return (EIO); 1553e984c70bSHans Rosenfeld 15543c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 15553c9168faSHans Rosenfeld cqe->cqe_sf.sf_sc == NVME_CQE_SC_GEN_SUCCESS) 15563c9168faSHans Rosenfeld return (0); 15573c9168faSHans Rosenfeld 15583c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC) 15593c9168faSHans Rosenfeld return (nvme_check_generic_cmd_status(cmd)); 15603c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_SPECIFIC) 15613c9168faSHans Rosenfeld return (nvme_check_specific_cmd_status(cmd)); 15623c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_INTEGRITY) 15633c9168faSHans Rosenfeld return (nvme_check_integrity_cmd_status(cmd)); 15643c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_VENDOR) 15653c9168faSHans Rosenfeld return (nvme_check_vendor_cmd_status(cmd)); 15663c9168faSHans Rosenfeld 15673c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 15683c9168faSHans Rosenfeld } 15693c9168faSHans Rosenfeld 1570e984c70bSHans Rosenfeld static int 1571e984c70bSHans Rosenfeld nvme_abort_cmd(nvme_cmd_t *abort_cmd, uint_t sec) 15723c9168faSHans Rosenfeld { 15733c9168faSHans Rosenfeld nvme_t *nvme = abort_cmd->nc_nvme; 15743c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 15753c9168faSHans Rosenfeld nvme_abort_cmd_t ac = { 0 }; 1576e984c70bSHans Rosenfeld int ret = 0; 15773c9168faSHans Rosenfeld 15783c9168faSHans Rosenfeld sema_p(&nvme->n_abort_sema); 15793c9168faSHans Rosenfeld 15803c9168faSHans Rosenfeld ac.b.ac_cid = abort_cmd->nc_sqe.sqe_cid; 15813c9168faSHans Rosenfeld ac.b.ac_sqid = abort_cmd->nc_sqid; 15823c9168faSHans Rosenfeld 15833c9168faSHans Rosenfeld cmd->nc_sqid = 0; 15843c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT; 15853c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 15863c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = ac.r; 15873c9168faSHans Rosenfeld 15883c9168faSHans Rosenfeld /* 15893c9168faSHans Rosenfeld * Send the ABORT to the hardware. The ABORT command will return _after_ 1590e984c70bSHans Rosenfeld * the aborted command has completed (aborted or otherwise), but since 1591e984c70bSHans Rosenfeld * we still hold the aborted command's mutex its callback hasn't been 1592e984c70bSHans Rosenfeld * processed yet. 15933c9168faSHans Rosenfeld */ 1594e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, sec); 15953c9168faSHans Rosenfeld sema_v(&nvme->n_abort_sema); 15963c9168faSHans Rosenfeld 1597e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 15983c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15993c9168faSHans Rosenfeld "!ABORT failed with sct = %x, sc = %x", 16003c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 16013c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_abort_failed); 16023c9168faSHans Rosenfeld } else { 1603e984c70bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 1604e984c70bSHans Rosenfeld "!ABORT of command %d/%d %ssuccessful", 1605e984c70bSHans Rosenfeld abort_cmd->nc_sqe.sqe_cid, abort_cmd->nc_sqid, 1606e984c70bSHans Rosenfeld cmd->nc_cqe.cqe_dw0 & 1 ? "un" : ""); 1607e984c70bSHans Rosenfeld if ((cmd->nc_cqe.cqe_dw0 & 1) == 0) 1608e984c70bSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_aborted); 16093c9168faSHans Rosenfeld } 16103c9168faSHans Rosenfeld 16113c9168faSHans Rosenfeld nvme_free_cmd(cmd); 1612e984c70bSHans Rosenfeld return (ret); 16133c9168faSHans Rosenfeld } 16143c9168faSHans Rosenfeld 16153c9168faSHans Rosenfeld /* 16163c9168faSHans Rosenfeld * nvme_wait_cmd -- wait for command completion or timeout 16173c9168faSHans Rosenfeld * 16183c9168faSHans Rosenfeld * In case of a serious error or a timeout of the abort command the hardware 16193c9168faSHans Rosenfeld * will be declared dead and FMA will be notified. 16203c9168faSHans Rosenfeld */ 1621e984c70bSHans Rosenfeld static void 1622e8ba2a38SHans Rosenfeld nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec) 16233c9168faSHans Rosenfeld { 1624e8ba2a38SHans Rosenfeld clock_t timeout = ddi_get_lbolt() + drv_usectohz(sec * MICROSEC); 16253c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 16263c9168faSHans Rosenfeld nvme_reg_csts_t csts; 1627e984c70bSHans Rosenfeld nvme_qpair_t *qp; 16283c9168faSHans Rosenfeld 16293c9168faSHans Rosenfeld ASSERT(mutex_owned(&cmd->nc_mutex)); 16303c9168faSHans Rosenfeld 16313c9168faSHans Rosenfeld while (!cmd->nc_completed) { 16323c9168faSHans Rosenfeld if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1) 16333c9168faSHans Rosenfeld break; 16343c9168faSHans Rosenfeld } 16353c9168faSHans Rosenfeld 16363c9168faSHans Rosenfeld if (cmd->nc_completed) 1637e984c70bSHans Rosenfeld return; 16383c9168faSHans Rosenfeld 16393c9168faSHans Rosenfeld /* 1640e984c70bSHans Rosenfeld * The command timed out. 1641e984c70bSHans Rosenfeld * 16423c9168faSHans Rosenfeld * Check controller for fatal status, any errors associated with the 16433c9168faSHans Rosenfeld * register or DMA handle, or for a double timeout (abort command timed 16443c9168faSHans Rosenfeld * out). If necessary log a warning and call FMA. 16453c9168faSHans Rosenfeld */ 16463c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 1647e984c70bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!command %d/%d timeout, " 1648e984c70bSHans Rosenfeld "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_cid, cmd->nc_sqid, 1649e984c70bSHans Rosenfeld cmd->nc_sqe.sqe_opc, csts.b.csts_cfs); 16503c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_timeout); 16513c9168faSHans Rosenfeld 16523c9168faSHans Rosenfeld if (csts.b.csts_cfs || 16533c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme) || 16543c9168faSHans Rosenfeld nvme_check_dma_hdl(cmd->nc_dma) || 16553c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) { 16563c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 16573c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 1658e984c70bSHans Rosenfeld } else if (nvme_abort_cmd(cmd, sec) == 0) { 16593c9168faSHans Rosenfeld /* 1660e984c70bSHans Rosenfeld * If the abort succeeded the command should complete 1661e984c70bSHans Rosenfeld * immediately with an appropriate status. 16623c9168faSHans Rosenfeld */ 1663e984c70bSHans Rosenfeld while (!cmd->nc_completed) 1664e984c70bSHans Rosenfeld cv_wait(&cmd->nc_cv, &cmd->nc_mutex); 1665e984c70bSHans Rosenfeld 1666e984c70bSHans Rosenfeld return; 16673c9168faSHans Rosenfeld } 16683c9168faSHans Rosenfeld 1669e984c70bSHans Rosenfeld qp = nvme->n_ioq[cmd->nc_sqid]; 1670e984c70bSHans Rosenfeld 1671e984c70bSHans Rosenfeld mutex_enter(&qp->nq_mutex); 1672e984c70bSHans Rosenfeld (void) nvme_unqueue_cmd(nvme, qp, cmd->nc_sqe.sqe_cid); 1673e984c70bSHans Rosenfeld mutex_exit(&qp->nq_mutex); 1674e984c70bSHans Rosenfeld 1675e984c70bSHans Rosenfeld /* 1676e984c70bSHans Rosenfeld * As we don't know what the presumed dead hardware might still do with 1677e984c70bSHans Rosenfeld * the DMA memory, we'll put the command on the lost commands list if it 1678e984c70bSHans Rosenfeld * has any DMA memory. 1679e984c70bSHans Rosenfeld */ 1680e984c70bSHans Rosenfeld if (cmd->nc_dma != NULL) { 1681e984c70bSHans Rosenfeld mutex_enter(&nvme_lc_mutex); 1682e984c70bSHans Rosenfeld list_insert_head(&nvme_lost_cmds, cmd); 1683e984c70bSHans Rosenfeld mutex_exit(&nvme_lc_mutex); 1684e984c70bSHans Rosenfeld } 16853c9168faSHans Rosenfeld } 16863c9168faSHans Rosenfeld 16873c9168faSHans Rosenfeld static void 16883c9168faSHans Rosenfeld nvme_wakeup_cmd(void *arg) 16893c9168faSHans Rosenfeld { 16903c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 16913c9168faSHans Rosenfeld 16923c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 16933c9168faSHans Rosenfeld cmd->nc_completed = B_TRUE; 16943c9168faSHans Rosenfeld cv_signal(&cmd->nc_cv); 16953c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 16963c9168faSHans Rosenfeld } 16973c9168faSHans Rosenfeld 16983c9168faSHans Rosenfeld static void 16993c9168faSHans Rosenfeld nvme_async_event_task(void *arg) 17003c9168faSHans Rosenfeld { 17013c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 17023c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 17033c9168faSHans Rosenfeld nvme_error_log_entry_t *error_log = NULL; 17043c9168faSHans Rosenfeld nvme_health_log_t *health_log = NULL; 17053d9b1a2aSHans Rosenfeld size_t logsize = 0; 17063c9168faSHans Rosenfeld nvme_async_event_t event; 17073c9168faSHans Rosenfeld 17083c9168faSHans Rosenfeld /* 17093c9168faSHans Rosenfeld * Check for errors associated with the async request itself. The only 17103c9168faSHans Rosenfeld * command-specific error is "async event limit exceeded", which 17113c9168faSHans Rosenfeld * indicates a programming error in the driver and causes a panic in 17123c9168faSHans Rosenfeld * nvme_check_cmd_status(). 17133c9168faSHans Rosenfeld * 17143c9168faSHans Rosenfeld * Other possible errors are various scenarios where the async request 17153c9168faSHans Rosenfeld * was aborted, or internal errors in the device. Internal errors are 17163c9168faSHans Rosenfeld * reported to FMA, the command aborts need no special handling here. 171708139162SToomas Soome * 171808139162SToomas Soome * And finally, at least qemu nvme does not support async events, 171908139162SToomas Soome * and will return NVME_CQE_SC_GEN_INV_OPC | DNR. If so, we 172008139162SToomas Soome * will avoid posting async events. 17213c9168faSHans Rosenfeld */ 172208139162SToomas Soome 1723e984c70bSHans Rosenfeld if (nvme_check_cmd_status(cmd) != 0) { 17243c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 17253c9168faSHans Rosenfeld "!async event request returned failure, sct = %x, " 17263c9168faSHans Rosenfeld "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct, 17273c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr, 17283c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_m); 17293c9168faSHans Rosenfeld 17303c9168faSHans Rosenfeld if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 17313c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) { 17323c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 17333c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, 17343c9168faSHans Rosenfeld DDI_SERVICE_LOST); 17353c9168faSHans Rosenfeld } 173608139162SToomas Soome 173708139162SToomas Soome if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 173808139162SToomas Soome cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INV_OPC && 173908139162SToomas Soome cmd->nc_cqe.cqe_sf.sf_dnr == 1) { 174008139162SToomas Soome nvme->n_async_event_supported = B_FALSE; 174108139162SToomas Soome } 174208139162SToomas Soome 17433c9168faSHans Rosenfeld nvme_free_cmd(cmd); 17443c9168faSHans Rosenfeld return; 17453c9168faSHans Rosenfeld } 17463c9168faSHans Rosenfeld 17473c9168faSHans Rosenfeld 17483c9168faSHans Rosenfeld event.r = cmd->nc_cqe.cqe_dw0; 17493c9168faSHans Rosenfeld 17503c9168faSHans Rosenfeld /* Clear CQE and re-submit the async request. */ 17513c9168faSHans Rosenfeld bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t)); 17524b324362SHans Rosenfeld nvme_submit_admin_cmd(nvme->n_adminq, cmd); 17533c9168faSHans Rosenfeld 17543c9168faSHans Rosenfeld switch (event.b.ae_type) { 17553c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_ERROR: 17563c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_ERROR) { 1757bc586359SRobert Mustacchi (void) nvme_get_logpage(nvme, B_FALSE, 1758bc586359SRobert Mustacchi (void **)&error_log, &logsize, event.b.ae_logpage); 17593c9168faSHans Rosenfeld } else { 17603c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 17613c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 17623c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 17633c9168faSHans Rosenfeld } 17643c9168faSHans Rosenfeld 17653c9168faSHans Rosenfeld switch (event.b.ae_info) { 17663c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_SQ: 17673c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 17683c9168faSHans Rosenfeld "invalid submission queue"); 17693c9168faSHans Rosenfeld return; 17703c9168faSHans Rosenfeld 17713c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_DBL: 17723c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 17733c9168faSHans Rosenfeld "invalid doorbell write value"); 17743c9168faSHans Rosenfeld return; 17753c9168faSHans Rosenfeld 17763c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_DIAGFAIL: 17773c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure"); 17783c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 17793c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 17803c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_diagfail_event); 17813c9168faSHans Rosenfeld break; 17823c9168faSHans Rosenfeld 17833c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_PERSISTENT: 17843c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!persistent internal " 17853c9168faSHans Rosenfeld "device error"); 17863c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 17873c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 17883c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_persistent_event); 17893c9168faSHans Rosenfeld break; 17903c9168faSHans Rosenfeld 17913c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_TRANSIENT: 17923c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!transient internal " 17933c9168faSHans Rosenfeld "device error"); 17943c9168faSHans Rosenfeld /* TODO: send ereport */ 17953c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_transient_event); 17963c9168faSHans Rosenfeld break; 17973c9168faSHans Rosenfeld 17983c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_FW_LOAD: 17993c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 18003c9168faSHans Rosenfeld "!firmware image load error"); 18013c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_fw_load_event); 18023c9168faSHans Rosenfeld break; 18033c9168faSHans Rosenfeld } 18043c9168faSHans Rosenfeld break; 18053c9168faSHans Rosenfeld 18063c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_HEALTH: 18073c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_HEALTH) { 1808bc586359SRobert Mustacchi (void) nvme_get_logpage(nvme, B_FALSE, 1809bc586359SRobert Mustacchi (void **)&health_log, &logsize, event.b.ae_logpage, 1810bc586359SRobert Mustacchi -1); 18113c9168faSHans Rosenfeld } else { 18123c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 18133c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 18143c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 18153c9168faSHans Rosenfeld } 18163c9168faSHans Rosenfeld 18173c9168faSHans Rosenfeld switch (event.b.ae_info) { 18183c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_RELIABILITY: 18193c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 18203c9168faSHans Rosenfeld "!device reliability compromised"); 18213c9168faSHans Rosenfeld /* TODO: send ereport */ 18223c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_reliability_event); 18233c9168faSHans Rosenfeld break; 18243c9168faSHans Rosenfeld 18253c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_TEMPERATURE: 18263c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 18273c9168faSHans Rosenfeld "!temperature above threshold"); 18283c9168faSHans Rosenfeld /* TODO: send ereport */ 18293c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_temperature_event); 18303c9168faSHans Rosenfeld break; 18313c9168faSHans Rosenfeld 18323c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_SPARE: 18333c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 18343c9168faSHans Rosenfeld "!spare space below threshold"); 18353c9168faSHans Rosenfeld /* TODO: send ereport */ 18363c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_spare_event); 18373c9168faSHans Rosenfeld break; 18383c9168faSHans Rosenfeld } 18393c9168faSHans Rosenfeld break; 18403c9168faSHans Rosenfeld 18413c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_VENDOR: 18423c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event " 18433c9168faSHans Rosenfeld "received, info = %x, logpage = %x", event.b.ae_info, 18443c9168faSHans Rosenfeld event.b.ae_logpage); 18453c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_vendor_event); 18463c9168faSHans Rosenfeld break; 18473c9168faSHans Rosenfeld 18483c9168faSHans Rosenfeld default: 18493c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, " 18503c9168faSHans Rosenfeld "type = %x, info = %x, logpage = %x", event.b.ae_type, 18513c9168faSHans Rosenfeld event.b.ae_info, event.b.ae_logpage); 18523c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_event); 18533c9168faSHans Rosenfeld break; 18543c9168faSHans Rosenfeld } 18553c9168faSHans Rosenfeld 18563c9168faSHans Rosenfeld if (error_log) 18573d9b1a2aSHans Rosenfeld kmem_free(error_log, logsize); 18583c9168faSHans Rosenfeld 18593c9168faSHans Rosenfeld if (health_log) 18603d9b1a2aSHans Rosenfeld kmem_free(health_log, logsize); 18613c9168faSHans Rosenfeld } 18623c9168faSHans Rosenfeld 1863e984c70bSHans Rosenfeld static void 1864e8ba2a38SHans Rosenfeld nvme_admin_cmd(nvme_cmd_t *cmd, int sec) 18653c9168faSHans Rosenfeld { 18663c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 18674b324362SHans Rosenfeld nvme_submit_admin_cmd(cmd->nc_nvme->n_adminq, cmd); 1868e984c70bSHans Rosenfeld nvme_wait_cmd(cmd, sec); 18693c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 18703c9168faSHans Rosenfeld } 18713c9168faSHans Rosenfeld 18724b324362SHans Rosenfeld static void 18733c9168faSHans Rosenfeld nvme_async_event(nvme_t *nvme) 18743c9168faSHans Rosenfeld { 187508139162SToomas Soome nvme_cmd_t *cmd; 18763c9168faSHans Rosenfeld 187708139162SToomas Soome cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 18783c9168faSHans Rosenfeld cmd->nc_sqid = 0; 18793c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT; 18803c9168faSHans Rosenfeld cmd->nc_callback = nvme_async_event_task; 188108139162SToomas Soome cmd->nc_dontpanic = B_TRUE; 18823c9168faSHans Rosenfeld 18834b324362SHans Rosenfeld nvme_submit_admin_cmd(nvme->n_adminq, cmd); 18843c9168faSHans Rosenfeld } 18853c9168faSHans Rosenfeld 18863d9b1a2aSHans Rosenfeld static int 1887bc586359SRobert Mustacchi nvme_format_nvm(nvme_t *nvme, boolean_t user, uint32_t nsid, uint8_t lbaf, 1888bc586359SRobert Mustacchi boolean_t ms, uint8_t pi, boolean_t pil, uint8_t ses) 18893d9b1a2aSHans Rosenfeld { 18903d9b1a2aSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 18913d9b1a2aSHans Rosenfeld nvme_format_nvm_t format_nvm = { 0 }; 18923d9b1a2aSHans Rosenfeld int ret; 18933d9b1a2aSHans Rosenfeld 18943d9b1a2aSHans Rosenfeld format_nvm.b.fm_lbaf = lbaf & 0xf; 18953d9b1a2aSHans Rosenfeld format_nvm.b.fm_ms = ms ? 1 : 0; 18963d9b1a2aSHans Rosenfeld format_nvm.b.fm_pi = pi & 0x7; 18973d9b1a2aSHans Rosenfeld format_nvm.b.fm_pil = pil ? 1 : 0; 18983d9b1a2aSHans Rosenfeld format_nvm.b.fm_ses = ses & 0x7; 18993d9b1a2aSHans Rosenfeld 19003d9b1a2aSHans Rosenfeld cmd->nc_sqid = 0; 19013d9b1a2aSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 19023d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 19033d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_NVM_FORMAT; 19043d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = format_nvm.r; 19053d9b1a2aSHans Rosenfeld 19063d9b1a2aSHans Rosenfeld /* 19073d9b1a2aSHans Rosenfeld * Some devices like Samsung SM951 don't allow formatting of all 19083d9b1a2aSHans Rosenfeld * namespaces in one command. Handle that gracefully. 19093d9b1a2aSHans Rosenfeld */ 19103d9b1a2aSHans Rosenfeld if (nsid == (uint32_t)-1) 19113d9b1a2aSHans Rosenfeld cmd->nc_dontpanic = B_TRUE; 1912bc586359SRobert Mustacchi /* 1913bc586359SRobert Mustacchi * If this format request was initiated by the user, then don't allow a 1914bc586359SRobert Mustacchi * programmer error to panic the system. 1915bc586359SRobert Mustacchi */ 1916bc586359SRobert Mustacchi if (user) 1917bc586359SRobert Mustacchi cmd->nc_dontpanic = B_TRUE; 19183d9b1a2aSHans Rosenfeld 1919e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_format_cmd_timeout); 19203d9b1a2aSHans Rosenfeld 19213d9b1a2aSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 19223d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19233d9b1a2aSHans Rosenfeld "!FORMAT failed with sct = %x, sc = %x", 19243d9b1a2aSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 19253d9b1a2aSHans Rosenfeld } 19263d9b1a2aSHans Rosenfeld 19273d9b1a2aSHans Rosenfeld nvme_free_cmd(cmd); 19283d9b1a2aSHans Rosenfeld return (ret); 19293d9b1a2aSHans Rosenfeld } 19303d9b1a2aSHans Rosenfeld 19313d9b1a2aSHans Rosenfeld static int 1932bc586359SRobert Mustacchi nvme_get_logpage(nvme_t *nvme, boolean_t user, void **buf, size_t *bufsize, 1933bc586359SRobert Mustacchi uint8_t logpage, ...) 19343c9168faSHans Rosenfeld { 19353c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 193634c938c7SPete Shephard nvme_getlogpage_t getlogpage = { 0 }; 19373c9168faSHans Rosenfeld va_list ap; 1938e984c70bSHans Rosenfeld int ret; 19393c9168faSHans Rosenfeld 19403c9168faSHans Rosenfeld va_start(ap, logpage); 19413c9168faSHans Rosenfeld 19423c9168faSHans Rosenfeld cmd->nc_sqid = 0; 19433c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 19443c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE; 19453c9168faSHans Rosenfeld 1946bc586359SRobert Mustacchi if (user) 1947bc586359SRobert Mustacchi cmd->nc_dontpanic = B_TRUE; 1948bc586359SRobert Mustacchi 19493c9168faSHans Rosenfeld getlogpage.b.lp_lid = logpage; 19503c9168faSHans Rosenfeld 19513c9168faSHans Rosenfeld switch (logpage) { 19523c9168faSHans Rosenfeld case NVME_LOGPAGE_ERROR: 19533c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 19543d9b1a2aSHans Rosenfeld /* 19553d9b1a2aSHans Rosenfeld * The GET LOG PAGE command can use at most 2 pages to return 19563d9b1a2aSHans Rosenfeld * data, PRP lists are not supported. 19573d9b1a2aSHans Rosenfeld */ 19583d9b1a2aSHans Rosenfeld *bufsize = MIN(2 * nvme->n_pagesize, 19593d9b1a2aSHans Rosenfeld nvme->n_error_log_len * sizeof (nvme_error_log_entry_t)); 19603c9168faSHans Rosenfeld break; 19613c9168faSHans Rosenfeld 19623c9168faSHans Rosenfeld case NVME_LOGPAGE_HEALTH: 19633c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t); 19643d9b1a2aSHans Rosenfeld *bufsize = sizeof (nvme_health_log_t); 19653c9168faSHans Rosenfeld break; 19663c9168faSHans Rosenfeld 19673c9168faSHans Rosenfeld case NVME_LOGPAGE_FWSLOT: 19683c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 19693d9b1a2aSHans Rosenfeld *bufsize = sizeof (nvme_fwslot_log_t); 19703c9168faSHans Rosenfeld break; 19713c9168faSHans Rosenfeld 19723c9168faSHans Rosenfeld default: 19733c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown log page requested: %d", 19743c9168faSHans Rosenfeld logpage); 19753c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_logpage); 1976e984c70bSHans Rosenfeld ret = EINVAL; 19773c9168faSHans Rosenfeld goto fail; 19783c9168faSHans Rosenfeld } 19793c9168faSHans Rosenfeld 19803c9168faSHans Rosenfeld va_end(ap); 19813c9168faSHans Rosenfeld 19823d9b1a2aSHans Rosenfeld getlogpage.b.lp_numd = *bufsize / sizeof (uint32_t) - 1; 19833c9168faSHans Rosenfeld 19843c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = getlogpage.r; 19853c9168faSHans Rosenfeld 19860999c112SPaul Winder if (nvme_zalloc_dma(nvme, *bufsize, 19873c9168faSHans Rosenfeld DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 19883c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19893c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for GET LOG PAGE"); 1990e984c70bSHans Rosenfeld ret = ENOMEM; 19913c9168faSHans Rosenfeld goto fail; 19923c9168faSHans Rosenfeld } 19933c9168faSHans Rosenfeld 19943c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 19953c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19963c9168faSHans Rosenfeld "!too many DMA cookies for GET LOG PAGE"); 19973c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 1998e984c70bSHans Rosenfeld ret = ENOMEM; 19993c9168faSHans Rosenfeld goto fail; 20003c9168faSHans Rosenfeld } 20013c9168faSHans Rosenfeld 20023c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 20033c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 20043c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 20053c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 20063c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 20073c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 20083c9168faSHans Rosenfeld } 20093c9168faSHans Rosenfeld 2010e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 20113c9168faSHans Rosenfeld 2012e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 20133c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20143c9168faSHans Rosenfeld "!GET LOG PAGE failed with sct = %x, sc = %x", 20153c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 20163c9168faSHans Rosenfeld goto fail; 20173c9168faSHans Rosenfeld } 20183c9168faSHans Rosenfeld 20193d9b1a2aSHans Rosenfeld *buf = kmem_alloc(*bufsize, KM_SLEEP); 20203d9b1a2aSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize); 20213d9b1a2aSHans Rosenfeld 20223c9168faSHans Rosenfeld fail: 20233c9168faSHans Rosenfeld nvme_free_cmd(cmd); 20243c9168faSHans Rosenfeld 20253d9b1a2aSHans Rosenfeld return (ret); 20263c9168faSHans Rosenfeld } 20273c9168faSHans Rosenfeld 2028e984c70bSHans Rosenfeld static int 2029bc586359SRobert Mustacchi nvme_identify(nvme_t *nvme, boolean_t user, uint32_t nsid, void **buf) 20303c9168faSHans Rosenfeld { 20313c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 2032e984c70bSHans Rosenfeld int ret; 2033e984c70bSHans Rosenfeld 2034e984c70bSHans Rosenfeld if (buf == NULL) 2035e984c70bSHans Rosenfeld return (EINVAL); 20363c9168faSHans Rosenfeld 20373c9168faSHans Rosenfeld cmd->nc_sqid = 0; 20383c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 20393c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY; 20403c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 20413c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL; 20423c9168faSHans Rosenfeld 20433c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ, 20443c9168faSHans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 20453c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20463c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for IDENTIFY"); 2047e984c70bSHans Rosenfeld ret = ENOMEM; 20483c9168faSHans Rosenfeld goto fail; 20493c9168faSHans Rosenfeld } 20503c9168faSHans Rosenfeld 20513c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 20523c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20533c9168faSHans Rosenfeld "!too many DMA cookies for IDENTIFY"); 20543c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 2055e984c70bSHans Rosenfeld ret = ENOMEM; 20563c9168faSHans Rosenfeld goto fail; 20573c9168faSHans Rosenfeld } 20583c9168faSHans Rosenfeld 20593c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 20603c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 20613c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 20623c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 20633c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 20643c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 20653c9168faSHans Rosenfeld } 20663c9168faSHans Rosenfeld 2067bc586359SRobert Mustacchi if (user) 2068bc586359SRobert Mustacchi cmd->nc_dontpanic = B_TRUE; 2069bc586359SRobert Mustacchi 2070e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 20713c9168faSHans Rosenfeld 2072e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 20733c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20743c9168faSHans Rosenfeld "!IDENTIFY failed with sct = %x, sc = %x", 20753c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 20763c9168faSHans Rosenfeld goto fail; 20773c9168faSHans Rosenfeld } 20783c9168faSHans Rosenfeld 2079e984c70bSHans Rosenfeld *buf = kmem_alloc(NVME_IDENTIFY_BUFSIZE, KM_SLEEP); 2080e984c70bSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, *buf, NVME_IDENTIFY_BUFSIZE); 20813c9168faSHans Rosenfeld 20823c9168faSHans Rosenfeld fail: 20833c9168faSHans Rosenfeld nvme_free_cmd(cmd); 20843c9168faSHans Rosenfeld 2085e984c70bSHans Rosenfeld return (ret); 20863c9168faSHans Rosenfeld } 20873c9168faSHans Rosenfeld 2088e984c70bSHans Rosenfeld static int 2089bc586359SRobert Mustacchi nvme_set_features(nvme_t *nvme, boolean_t user, uint32_t nsid, uint8_t feature, 2090bc586359SRobert Mustacchi uint32_t val, uint32_t *res) 20913c9168faSHans Rosenfeld { 2092d148d46eSHans Rosenfeld _NOTE(ARGUNUSED(nsid)); 20933c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 2094e984c70bSHans Rosenfeld int ret = EINVAL; 20953c9168faSHans Rosenfeld 2096d148d46eSHans Rosenfeld ASSERT(res != NULL); 20973c9168faSHans Rosenfeld 20983c9168faSHans Rosenfeld cmd->nc_sqid = 0; 20993c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 21003c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES; 2101d148d46eSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = feature; 2102d148d46eSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = val; 2103d148d46eSHans Rosenfeld 2104bc586359SRobert Mustacchi if (user) 2105bc586359SRobert Mustacchi cmd->nc_dontpanic = B_TRUE; 2106bc586359SRobert Mustacchi 2107d148d46eSHans Rosenfeld switch (feature) { 2108d148d46eSHans Rosenfeld case NVME_FEAT_WRITE_CACHE: 2109d148d46eSHans Rosenfeld if (!nvme->n_write_cache_present) 2110d148d46eSHans Rosenfeld goto fail; 2111d148d46eSHans Rosenfeld break; 2112d148d46eSHans Rosenfeld 2113d148d46eSHans Rosenfeld case NVME_FEAT_NQUEUES: 2114d148d46eSHans Rosenfeld break; 2115d148d46eSHans Rosenfeld 2116d148d46eSHans Rosenfeld default: 2117d148d46eSHans Rosenfeld goto fail; 2118d148d46eSHans Rosenfeld } 21193c9168faSHans Rosenfeld 2120e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 21213c9168faSHans Rosenfeld 2122e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 21233c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2124d148d46eSHans Rosenfeld "!SET FEATURES %d failed with sct = %x, sc = %x", 2125d148d46eSHans Rosenfeld feature, cmd->nc_cqe.cqe_sf.sf_sct, 2126d148d46eSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc); 2127d148d46eSHans Rosenfeld goto fail; 21283c9168faSHans Rosenfeld } 21293c9168faSHans Rosenfeld 2130d148d46eSHans Rosenfeld *res = cmd->nc_cqe.cqe_dw0; 2131d148d46eSHans Rosenfeld 2132d148d46eSHans Rosenfeld fail: 21333c9168faSHans Rosenfeld nvme_free_cmd(cmd); 2134d148d46eSHans Rosenfeld return (ret); 2135d148d46eSHans Rosenfeld } 2136d148d46eSHans Rosenfeld 2137e984c70bSHans Rosenfeld static int 2138bc586359SRobert Mustacchi nvme_get_features(nvme_t *nvme, boolean_t user, uint32_t nsid, uint8_t feature, 2139bc586359SRobert Mustacchi uint32_t *res, void **buf, size_t *bufsize) 21403d9b1a2aSHans Rosenfeld { 21413d9b1a2aSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 2142e984c70bSHans Rosenfeld int ret = EINVAL; 21433d9b1a2aSHans Rosenfeld 21443d9b1a2aSHans Rosenfeld ASSERT(res != NULL); 21453d9b1a2aSHans Rosenfeld 21463d9b1a2aSHans Rosenfeld if (bufsize != NULL) 21473d9b1a2aSHans Rosenfeld *bufsize = 0; 21483d9b1a2aSHans Rosenfeld 21493d9b1a2aSHans Rosenfeld cmd->nc_sqid = 0; 21503d9b1a2aSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 21513d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_GET_FEATURES; 21523d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = feature; 21533d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = *res; 21543d9b1a2aSHans Rosenfeld 2155f313c178SYuri Pankov /* 2156f313c178SYuri Pankov * For some of the optional features there doesn't seem to be a method 2157f313c178SYuri Pankov * of detecting whether it is supported other than using it. This will 2158f313c178SYuri Pankov * cause "Invalid Field in Command" error, which is normally considered 2159f313c178SYuri Pankov * a programming error. Set the nc_dontpanic flag to override the panic 2160f313c178SYuri Pankov * in nvme_check_generic_cmd_status(). 2161f313c178SYuri Pankov */ 21623d9b1a2aSHans Rosenfeld switch (feature) { 21633d9b1a2aSHans Rosenfeld case NVME_FEAT_ARBITRATION: 21643d9b1a2aSHans Rosenfeld case NVME_FEAT_POWER_MGMT: 21653d9b1a2aSHans Rosenfeld case NVME_FEAT_TEMPERATURE: 21663d9b1a2aSHans Rosenfeld case NVME_FEAT_ERROR: 21673d9b1a2aSHans Rosenfeld case NVME_FEAT_NQUEUES: 21683d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_COAL: 21693d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_VECT: 21703d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_ATOM: 21713d9b1a2aSHans Rosenfeld case NVME_FEAT_ASYNC_EVENT: 21723d9b1a2aSHans Rosenfeld break; 21733d9b1a2aSHans Rosenfeld 21743d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_CACHE: 21753d9b1a2aSHans Rosenfeld if (!nvme->n_write_cache_present) 21763d9b1a2aSHans Rosenfeld goto fail; 21773d9b1a2aSHans Rosenfeld break; 21783d9b1a2aSHans Rosenfeld 21793d9b1a2aSHans Rosenfeld case NVME_FEAT_LBA_RANGE: 21803d9b1a2aSHans Rosenfeld if (!nvme->n_lba_range_supported) 21813d9b1a2aSHans Rosenfeld goto fail; 21823d9b1a2aSHans Rosenfeld 21833d9b1a2aSHans Rosenfeld cmd->nc_dontpanic = B_TRUE; 21843d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 21853d9b1a2aSHans Rosenfeld ASSERT(bufsize != NULL); 21863d9b1a2aSHans Rosenfeld *bufsize = NVME_LBA_RANGE_BUFSIZE; 21873d9b1a2aSHans Rosenfeld break; 21883d9b1a2aSHans Rosenfeld 21893d9b1a2aSHans Rosenfeld case NVME_FEAT_AUTO_PST: 21903d9b1a2aSHans Rosenfeld if (!nvme->n_auto_pst_supported) 21913d9b1a2aSHans Rosenfeld goto fail; 21923d9b1a2aSHans Rosenfeld 21933d9b1a2aSHans Rosenfeld ASSERT(bufsize != NULL); 21943d9b1a2aSHans Rosenfeld *bufsize = NVME_AUTO_PST_BUFSIZE; 21953d9b1a2aSHans Rosenfeld break; 21963d9b1a2aSHans Rosenfeld 2197f313c178SYuri Pankov case NVME_FEAT_PROGRESS: 2198f313c178SYuri Pankov if (!nvme->n_progress_supported) 2199f313c178SYuri Pankov goto fail; 2200f313c178SYuri Pankov 2201f313c178SYuri Pankov cmd->nc_dontpanic = B_TRUE; 2202f313c178SYuri Pankov break; 2203f313c178SYuri Pankov 22043d9b1a2aSHans Rosenfeld default: 22053d9b1a2aSHans Rosenfeld goto fail; 22063d9b1a2aSHans Rosenfeld } 22073d9b1a2aSHans Rosenfeld 2208bc586359SRobert Mustacchi if (user) 2209bc586359SRobert Mustacchi cmd->nc_dontpanic = B_TRUE; 2210bc586359SRobert Mustacchi 22113d9b1a2aSHans Rosenfeld if (bufsize != NULL && *bufsize != 0) { 22123d9b1a2aSHans Rosenfeld if (nvme_zalloc_dma(nvme, *bufsize, DDI_DMA_READ, 22133d9b1a2aSHans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 22143d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22153d9b1a2aSHans Rosenfeld "!nvme_zalloc_dma failed for GET FEATURES"); 2216e984c70bSHans Rosenfeld ret = ENOMEM; 22173d9b1a2aSHans Rosenfeld goto fail; 22183d9b1a2aSHans Rosenfeld } 22193d9b1a2aSHans Rosenfeld 22203d9b1a2aSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 22213d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22223d9b1a2aSHans Rosenfeld "!too many DMA cookies for GET FEATURES"); 22233d9b1a2aSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 2224e984c70bSHans Rosenfeld ret = ENOMEM; 22253d9b1a2aSHans Rosenfeld goto fail; 22263d9b1a2aSHans Rosenfeld } 22273d9b1a2aSHans Rosenfeld 22283d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = 22293d9b1a2aSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 22303d9b1a2aSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 22313d9b1a2aSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 22323d9b1a2aSHans Rosenfeld &cmd->nc_dma->nd_cookie); 22333d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 22343d9b1a2aSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 22353d9b1a2aSHans Rosenfeld } 22363d9b1a2aSHans Rosenfeld } 22373d9b1a2aSHans Rosenfeld 2238e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 22393d9b1a2aSHans Rosenfeld 2240e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 2241f313c178SYuri Pankov boolean_t known = B_TRUE; 2242f313c178SYuri Pankov 2243f313c178SYuri Pankov /* Check if this is unsupported optional feature */ 2244f313c178SYuri Pankov if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 2245f313c178SYuri Pankov cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INV_FLD) { 2246f313c178SYuri Pankov switch (feature) { 2247f313c178SYuri Pankov case NVME_FEAT_LBA_RANGE: 2248f313c178SYuri Pankov nvme->n_lba_range_supported = B_FALSE; 2249f313c178SYuri Pankov break; 2250f313c178SYuri Pankov case NVME_FEAT_PROGRESS: 2251f313c178SYuri Pankov nvme->n_progress_supported = B_FALSE; 2252f313c178SYuri Pankov break; 2253f313c178SYuri Pankov default: 2254f313c178SYuri Pankov known = B_FALSE; 2255f313c178SYuri Pankov break; 2256f313c178SYuri Pankov } 2257f313c178SYuri Pankov } else { 2258f313c178SYuri Pankov known = B_FALSE; 2259f313c178SYuri Pankov } 2260f313c178SYuri Pankov 2261f313c178SYuri Pankov /* Report the error otherwise */ 2262f313c178SYuri Pankov if (!known) { 22633d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22643d9b1a2aSHans Rosenfeld "!GET FEATURES %d failed with sct = %x, sc = %x", 22653d9b1a2aSHans Rosenfeld feature, cmd->nc_cqe.cqe_sf.sf_sct, 22663d9b1a2aSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc); 2267f313c178SYuri Pankov } 2268f313c178SYuri Pankov 22693d9b1a2aSHans Rosenfeld goto fail; 22703d9b1a2aSHans Rosenfeld } 22713d9b1a2aSHans Rosenfeld 22723d9b1a2aSHans Rosenfeld if (bufsize != NULL && *bufsize != 0) { 22733d9b1a2aSHans Rosenfeld ASSERT(buf != NULL); 22743d9b1a2aSHans Rosenfeld *buf = kmem_alloc(*bufsize, KM_SLEEP); 22753d9b1a2aSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize); 22763d9b1a2aSHans Rosenfeld } 22773d9b1a2aSHans Rosenfeld 22783d9b1a2aSHans Rosenfeld *res = cmd->nc_cqe.cqe_dw0; 22793d9b1a2aSHans Rosenfeld 22803d9b1a2aSHans Rosenfeld fail: 22813d9b1a2aSHans Rosenfeld nvme_free_cmd(cmd); 22823d9b1a2aSHans Rosenfeld return (ret); 22833d9b1a2aSHans Rosenfeld } 22843d9b1a2aSHans Rosenfeld 2285e984c70bSHans Rosenfeld static int 2286d148d46eSHans Rosenfeld nvme_write_cache_set(nvme_t *nvme, boolean_t enable) 2287d148d46eSHans Rosenfeld { 2288d148d46eSHans Rosenfeld nvme_write_cache_t nwc = { 0 }; 2289d148d46eSHans Rosenfeld 2290d148d46eSHans Rosenfeld if (enable) 2291d148d46eSHans Rosenfeld nwc.b.wc_wce = 1; 2292d148d46eSHans Rosenfeld 2293bc586359SRobert Mustacchi return (nvme_set_features(nvme, B_FALSE, 0, NVME_FEAT_WRITE_CACHE, 2294bc586359SRobert Mustacchi nwc.r, &nwc.r)); 2295d148d46eSHans Rosenfeld } 2296d148d46eSHans Rosenfeld 2297d148d46eSHans Rosenfeld static int 22980999c112SPaul Winder nvme_set_nqueues(nvme_t *nvme) 2299d148d46eSHans Rosenfeld { 23003d9b1a2aSHans Rosenfeld nvme_nqueues_t nq = { 0 }; 2301e984c70bSHans Rosenfeld int ret; 2302d148d46eSHans Rosenfeld 23030999c112SPaul Winder /* 23040999c112SPaul Winder * The default is to allocate one completion queue per vector. 23050999c112SPaul Winder */ 23060999c112SPaul Winder if (nvme->n_completion_queues == -1) 23070999c112SPaul Winder nvme->n_completion_queues = nvme->n_intr_cnt; 23080999c112SPaul Winder 23090999c112SPaul Winder /* 23100999c112SPaul Winder * There is no point in having more compeletion queues than 23110999c112SPaul Winder * interrupt vectors. 23120999c112SPaul Winder */ 23130999c112SPaul Winder nvme->n_completion_queues = MIN(nvme->n_completion_queues, 23140999c112SPaul Winder nvme->n_intr_cnt); 23150999c112SPaul Winder 23160999c112SPaul Winder /* 23170999c112SPaul Winder * The default is to use one submission queue per completion queue. 23180999c112SPaul Winder */ 23190999c112SPaul Winder if (nvme->n_submission_queues == -1) 23200999c112SPaul Winder nvme->n_submission_queues = nvme->n_completion_queues; 23210999c112SPaul Winder 23220999c112SPaul Winder /* 23230999c112SPaul Winder * There is no point in having more compeletion queues than 23240999c112SPaul Winder * submission queues. 23250999c112SPaul Winder */ 23260999c112SPaul Winder nvme->n_completion_queues = MIN(nvme->n_completion_queues, 23270999c112SPaul Winder nvme->n_submission_queues); 23280999c112SPaul Winder 23290999c112SPaul Winder ASSERT(nvme->n_submission_queues > 0); 23300999c112SPaul Winder ASSERT(nvme->n_completion_queues > 0); 23310999c112SPaul Winder 23320999c112SPaul Winder nq.b.nq_nsq = nvme->n_submission_queues - 1; 23330999c112SPaul Winder nq.b.nq_ncq = nvme->n_completion_queues - 1; 2334d148d46eSHans Rosenfeld 2335bc586359SRobert Mustacchi ret = nvme_set_features(nvme, B_FALSE, 0, NVME_FEAT_NQUEUES, nq.r, 2336bc586359SRobert Mustacchi &nq.r); 2337e984c70bSHans Rosenfeld 2338e984c70bSHans Rosenfeld if (ret == 0) { 2339e984c70bSHans Rosenfeld /* 23400999c112SPaul Winder * Never use more than the requested number of queues. 2341e984c70bSHans Rosenfeld */ 23420999c112SPaul Winder nvme->n_submission_queues = MIN(nvme->n_submission_queues, 23430999c112SPaul Winder nq.b.nq_nsq + 1); 23440999c112SPaul Winder nvme->n_completion_queues = MIN(nvme->n_completion_queues, 23450999c112SPaul Winder nq.b.nq_ncq + 1); 2346d148d46eSHans Rosenfeld } 23473c9168faSHans Rosenfeld 2348e984c70bSHans Rosenfeld return (ret); 23493c9168faSHans Rosenfeld } 23503c9168faSHans Rosenfeld 23513c9168faSHans Rosenfeld static int 23520999c112SPaul Winder nvme_create_completion_queue(nvme_t *nvme, nvme_cq_t *cq) 23533c9168faSHans Rosenfeld { 23543c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 23553c9168faSHans Rosenfeld nvme_create_queue_dw10_t dw10 = { 0 }; 23563c9168faSHans Rosenfeld nvme_create_cq_dw11_t c_dw11 = { 0 }; 2357e984c70bSHans Rosenfeld int ret; 23583c9168faSHans Rosenfeld 23590999c112SPaul Winder dw10.b.q_qid = cq->ncq_id; 23600999c112SPaul Winder dw10.b.q_qsize = cq->ncq_nentry - 1; 23613c9168faSHans Rosenfeld 23623c9168faSHans Rosenfeld c_dw11.b.cq_pc = 1; 23633c9168faSHans Rosenfeld c_dw11.b.cq_ien = 1; 23640999c112SPaul Winder c_dw11.b.cq_iv = cq->ncq_id % nvme->n_intr_cnt; 23653c9168faSHans Rosenfeld 23663c9168faSHans Rosenfeld cmd->nc_sqid = 0; 23673c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 23683c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE; 23693c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 23703c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = c_dw11.r; 23710999c112SPaul Winder cmd->nc_sqe.sqe_dptr.d_prp[0] = cq->ncq_dma->nd_cookie.dmac_laddress; 23723c9168faSHans Rosenfeld 2373e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 23743c9168faSHans Rosenfeld 2375e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 23763c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23773c9168faSHans Rosenfeld "!CREATE CQUEUE failed with sct = %x, sc = %x", 23783c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 23793c9168faSHans Rosenfeld } 23803c9168faSHans Rosenfeld 23813c9168faSHans Rosenfeld nvme_free_cmd(cmd); 23823c9168faSHans Rosenfeld 23830999c112SPaul Winder return (ret); 23840999c112SPaul Winder } 23850999c112SPaul Winder 23860999c112SPaul Winder static int 23870999c112SPaul Winder nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx) 23880999c112SPaul Winder { 23890999c112SPaul Winder nvme_cq_t *cq = qp->nq_cq; 23900999c112SPaul Winder nvme_cmd_t *cmd; 23910999c112SPaul Winder nvme_create_queue_dw10_t dw10 = { 0 }; 23920999c112SPaul Winder nvme_create_sq_dw11_t s_dw11 = { 0 }; 23930999c112SPaul Winder int ret; 23940999c112SPaul Winder 23950999c112SPaul Winder /* 23960999c112SPaul Winder * It is possible to have more qpairs than completion queues, 23970999c112SPaul Winder * and when the idx > ncq_id, that completion queue is shared 23980999c112SPaul Winder * and has already been created. 23990999c112SPaul Winder */ 24000999c112SPaul Winder if (idx <= cq->ncq_id && 24010999c112SPaul Winder nvme_create_completion_queue(nvme, cq) != DDI_SUCCESS) 24020999c112SPaul Winder return (DDI_FAILURE); 24030999c112SPaul Winder 24040999c112SPaul Winder dw10.b.q_qid = idx; 24050999c112SPaul Winder dw10.b.q_qsize = qp->nq_nentry - 1; 24060999c112SPaul Winder 24073c9168faSHans Rosenfeld s_dw11.b.sq_pc = 1; 24080999c112SPaul Winder s_dw11.b.sq_cqid = cq->ncq_id; 24093c9168faSHans Rosenfeld 24103c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 24113c9168faSHans Rosenfeld cmd->nc_sqid = 0; 24123c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 24133c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE; 24143c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 24153c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = s_dw11.r; 24163c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress; 24173c9168faSHans Rosenfeld 2418e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 24193c9168faSHans Rosenfeld 2420e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 24213c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 24223c9168faSHans Rosenfeld "!CREATE SQUEUE failed with sct = %x, sc = %x", 24233c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 24243c9168faSHans Rosenfeld } 24253c9168faSHans Rosenfeld 24263c9168faSHans Rosenfeld nvme_free_cmd(cmd); 24273c9168faSHans Rosenfeld 2428e984c70bSHans Rosenfeld return (ret); 24293c9168faSHans Rosenfeld } 24303c9168faSHans Rosenfeld 24313c9168faSHans Rosenfeld static boolean_t 24323c9168faSHans Rosenfeld nvme_reset(nvme_t *nvme, boolean_t quiesce) 24333c9168faSHans Rosenfeld { 24343c9168faSHans Rosenfeld nvme_reg_csts_t csts; 24353c9168faSHans Rosenfeld int i; 24363c9168faSHans Rosenfeld 24373c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 24383c9168faSHans Rosenfeld 24393c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 24403c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) { 24413c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 24423c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 24433c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 24443c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) 24453c9168faSHans Rosenfeld break; 24463c9168faSHans Rosenfeld 24473c9168faSHans Rosenfeld if (quiesce) 24483c9168faSHans Rosenfeld drv_usecwait(50000); 24493c9168faSHans Rosenfeld else 24503c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 24513c9168faSHans Rosenfeld } 24523c9168faSHans Rosenfeld } 24533c9168faSHans Rosenfeld 24543c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, 0); 24553c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ASQ, 0); 24563c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ACQ, 0); 24573c9168faSHans Rosenfeld 24583c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 24593c9168faSHans Rosenfeld return (csts.b.csts_rdy == 0 ? B_TRUE : B_FALSE); 24603c9168faSHans Rosenfeld } 24613c9168faSHans Rosenfeld 24623c9168faSHans Rosenfeld static void 24633c9168faSHans Rosenfeld nvme_shutdown(nvme_t *nvme, int mode, boolean_t quiesce) 24643c9168faSHans Rosenfeld { 24653c9168faSHans Rosenfeld nvme_reg_cc_t cc; 24663c9168faSHans Rosenfeld nvme_reg_csts_t csts; 24673c9168faSHans Rosenfeld int i; 24683c9168faSHans Rosenfeld 24693c9168faSHans Rosenfeld ASSERT(mode == NVME_CC_SHN_NORMAL || mode == NVME_CC_SHN_ABRUPT); 24703c9168faSHans Rosenfeld 24713c9168faSHans Rosenfeld cc.r = nvme_get32(nvme, NVME_REG_CC); 24723c9168faSHans Rosenfeld cc.b.cc_shn = mode & 0x3; 24733c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 24743c9168faSHans Rosenfeld 24753c9168faSHans Rosenfeld for (i = 0; i != 10; i++) { 24763c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 24773c9168faSHans Rosenfeld if (csts.b.csts_shst == NVME_CSTS_SHN_COMPLETE) 24783c9168faSHans Rosenfeld break; 24793c9168faSHans Rosenfeld 24803c9168faSHans Rosenfeld if (quiesce) 24813c9168faSHans Rosenfeld drv_usecwait(100000); 24823c9168faSHans Rosenfeld else 24833c9168faSHans Rosenfeld delay(drv_usectohz(100000)); 24843c9168faSHans Rosenfeld } 24853c9168faSHans Rosenfeld } 24863c9168faSHans Rosenfeld 24873c9168faSHans Rosenfeld 24883c9168faSHans Rosenfeld static void 24893c9168faSHans Rosenfeld nvme_prepare_devid(nvme_t *nvme, uint32_t nsid) 24903c9168faSHans Rosenfeld { 249124979ca3SHans Rosenfeld /* 249224979ca3SHans Rosenfeld * Section 7.7 of the spec describes how to get a unique ID for 249324979ca3SHans Rosenfeld * the controller: the vendor ID, the model name and the serial 249424979ca3SHans Rosenfeld * number shall be unique when combined. 249524979ca3SHans Rosenfeld * 249624979ca3SHans Rosenfeld * If a namespace has no EUI64 we use the above and add the hex 249724979ca3SHans Rosenfeld * namespace ID to get a unique ID for the namespace. 249824979ca3SHans Rosenfeld */ 24993c9168faSHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 25003c9168faSHans Rosenfeld char serial[sizeof (nvme->n_idctl->id_serial) + 1]; 25013c9168faSHans Rosenfeld 25023c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 25033c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_serial, serial, 25043c9168faSHans Rosenfeld sizeof (nvme->n_idctl->id_serial)); 25053c9168faSHans Rosenfeld 25063c9168faSHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 25073c9168faSHans Rosenfeld serial[sizeof (nvme->n_idctl->id_serial)] = '\0'; 25083c9168faSHans Rosenfeld 250924979ca3SHans Rosenfeld nvme->n_ns[nsid - 1].ns_devid = kmem_asprintf("%4X-%s-%s-%X", 25103c9168faSHans Rosenfeld nvme->n_idctl->id_vid, model, serial, nsid); 25113c9168faSHans Rosenfeld } 25123c9168faSHans Rosenfeld 25133d9b1a2aSHans Rosenfeld static int 25143d9b1a2aSHans Rosenfeld nvme_init_ns(nvme_t *nvme, int nsid) 25153d9b1a2aSHans Rosenfeld { 25163d9b1a2aSHans Rosenfeld nvme_namespace_t *ns = &nvme->n_ns[nsid - 1]; 25173d9b1a2aSHans Rosenfeld nvme_identify_nsid_t *idns; 25184d95620bSPaul Winder boolean_t was_ignored; 25193d9b1a2aSHans Rosenfeld int last_rp; 25203d9b1a2aSHans Rosenfeld 25213d9b1a2aSHans Rosenfeld ns->ns_nvme = nvme; 25223d9b1a2aSHans Rosenfeld 2523bc586359SRobert Mustacchi if (nvme_identify(nvme, B_FALSE, nsid, (void **)&idns) != 0) { 25243d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 25253d9b1a2aSHans Rosenfeld "!failed to identify namespace %d", nsid); 25263d9b1a2aSHans Rosenfeld return (DDI_FAILURE); 25273d9b1a2aSHans Rosenfeld } 25283d9b1a2aSHans Rosenfeld 25293d9b1a2aSHans Rosenfeld ns->ns_idns = idns; 25303d9b1a2aSHans Rosenfeld ns->ns_id = nsid; 25313d9b1a2aSHans Rosenfeld ns->ns_block_count = idns->id_nsize; 25323d9b1a2aSHans Rosenfeld ns->ns_block_size = 25333d9b1a2aSHans Rosenfeld 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads; 25343d9b1a2aSHans Rosenfeld ns->ns_best_block_size = ns->ns_block_size; 25353d9b1a2aSHans Rosenfeld 25363d9b1a2aSHans Rosenfeld /* 25373d9b1a2aSHans Rosenfeld * Get the EUI64 if present. Use it for devid and device node names. 25383d9b1a2aSHans Rosenfeld */ 25393d9b1a2aSHans Rosenfeld if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) 25403d9b1a2aSHans Rosenfeld bcopy(idns->id_eui64, ns->ns_eui64, sizeof (ns->ns_eui64)); 25413d9b1a2aSHans Rosenfeld 25423d9b1a2aSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 25433d9b1a2aSHans Rosenfeld if (*(uint64_t *)ns->ns_eui64 != 0) { 25443d9b1a2aSHans Rosenfeld uint8_t *eui64 = ns->ns_eui64; 25453d9b1a2aSHans Rosenfeld 25463d9b1a2aSHans Rosenfeld (void) snprintf(ns->ns_name, sizeof (ns->ns_name), 25473d9b1a2aSHans Rosenfeld "%02x%02x%02x%02x%02x%02x%02x%02x", 25483d9b1a2aSHans Rosenfeld eui64[0], eui64[1], eui64[2], eui64[3], 25493d9b1a2aSHans Rosenfeld eui64[4], eui64[5], eui64[6], eui64[7]); 25503d9b1a2aSHans Rosenfeld } else { 25513d9b1a2aSHans Rosenfeld (void) snprintf(ns->ns_name, sizeof (ns->ns_name), "%d", 25523d9b1a2aSHans Rosenfeld ns->ns_id); 25533d9b1a2aSHans Rosenfeld 25543d9b1a2aSHans Rosenfeld nvme_prepare_devid(nvme, ns->ns_id); 25553d9b1a2aSHans Rosenfeld } 25563d9b1a2aSHans Rosenfeld 25573d9b1a2aSHans Rosenfeld /* 25583d9b1a2aSHans Rosenfeld * Find the LBA format with no metadata and the best relative 25593d9b1a2aSHans Rosenfeld * performance. A value of 3 means "degraded", 0 is best. 25603d9b1a2aSHans Rosenfeld */ 25613d9b1a2aSHans Rosenfeld last_rp = 3; 25623d9b1a2aSHans Rosenfeld for (int j = 0; j <= idns->id_nlbaf; j++) { 25633d9b1a2aSHans Rosenfeld if (idns->id_lbaf[j].lbaf_lbads == 0) 25643d9b1a2aSHans Rosenfeld break; 25653d9b1a2aSHans Rosenfeld if (idns->id_lbaf[j].lbaf_ms != 0) 25663d9b1a2aSHans Rosenfeld continue; 25673d9b1a2aSHans Rosenfeld if (idns->id_lbaf[j].lbaf_rp >= last_rp) 25683d9b1a2aSHans Rosenfeld continue; 25693d9b1a2aSHans Rosenfeld last_rp = idns->id_lbaf[j].lbaf_rp; 25703d9b1a2aSHans Rosenfeld ns->ns_best_block_size = 25713d9b1a2aSHans Rosenfeld 1 << idns->id_lbaf[j].lbaf_lbads; 25723d9b1a2aSHans Rosenfeld } 25733d9b1a2aSHans Rosenfeld 25743d9b1a2aSHans Rosenfeld if (ns->ns_best_block_size < nvme->n_min_block_size) 25753d9b1a2aSHans Rosenfeld ns->ns_best_block_size = nvme->n_min_block_size; 25763d9b1a2aSHans Rosenfeld 25774d95620bSPaul Winder was_ignored = ns->ns_ignore; 25784d95620bSPaul Winder 25793d9b1a2aSHans Rosenfeld /* 25803d9b1a2aSHans Rosenfeld * We currently don't support namespaces that use either: 25813d9b1a2aSHans Rosenfeld * - protection information 2582621738e2SHans Rosenfeld * - illegal block size (< 512) 25833d9b1a2aSHans Rosenfeld */ 2584db083a4dSYuri Pankov if (idns->id_dps.dp_pinfo) { 25853d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2586db083a4dSYuri Pankov "!ignoring namespace %d, unsupported feature: " 2587db083a4dSYuri Pankov "pinfo = %d", nsid, idns->id_dps.dp_pinfo); 25883d9b1a2aSHans Rosenfeld ns->ns_ignore = B_TRUE; 2589621738e2SHans Rosenfeld } else if (ns->ns_block_size < 512) { 2590621738e2SHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2591621738e2SHans Rosenfeld "!ignoring namespace %d, unsupported block size %"PRIu64, 2592621738e2SHans Rosenfeld nsid, (uint64_t)ns->ns_block_size); 259331c62b25SHans Rosenfeld ns->ns_ignore = B_TRUE; 25943d9b1a2aSHans Rosenfeld } else { 25953d9b1a2aSHans Rosenfeld ns->ns_ignore = B_FALSE; 25963d9b1a2aSHans Rosenfeld } 25973d9b1a2aSHans Rosenfeld 25984d95620bSPaul Winder /* 25994d95620bSPaul Winder * Keep a count of namespaces which are attachable. 26004d95620bSPaul Winder * See comments in nvme_bd_driveinfo() to understand its effect. 26014d95620bSPaul Winder */ 26024d95620bSPaul Winder if (was_ignored) { 26034d95620bSPaul Winder /* 26044d95620bSPaul Winder * Previously ignored, but now not. Count it. 26054d95620bSPaul Winder */ 26064d95620bSPaul Winder if (!ns->ns_ignore) 26074d95620bSPaul Winder nvme->n_namespaces_attachable++; 26084d95620bSPaul Winder } else { 26094d95620bSPaul Winder /* 26104d95620bSPaul Winder * Wasn't ignored previously, but now needs to be. 26114d95620bSPaul Winder * Discount it. 26124d95620bSPaul Winder */ 26134d95620bSPaul Winder if (ns->ns_ignore) 26144d95620bSPaul Winder nvme->n_namespaces_attachable--; 26154d95620bSPaul Winder } 26164d95620bSPaul Winder 26173d9b1a2aSHans Rosenfeld return (DDI_SUCCESS); 26183d9b1a2aSHans Rosenfeld } 26193d9b1a2aSHans Rosenfeld 26203c9168faSHans Rosenfeld static int 26213c9168faSHans Rosenfeld nvme_init(nvme_t *nvme) 26223c9168faSHans Rosenfeld { 26233c9168faSHans Rosenfeld nvme_reg_cc_t cc = { 0 }; 26243c9168faSHans Rosenfeld nvme_reg_aqa_t aqa = { 0 }; 26253c9168faSHans Rosenfeld nvme_reg_asq_t asq = { 0 }; 26263c9168faSHans Rosenfeld nvme_reg_acq_t acq = { 0 }; 26273c9168faSHans Rosenfeld nvme_reg_cap_t cap; 26283c9168faSHans Rosenfeld nvme_reg_vs_t vs; 26293c9168faSHans Rosenfeld nvme_reg_csts_t csts; 26303c9168faSHans Rosenfeld int i = 0; 2631e984c70bSHans Rosenfeld uint16_t nqueues; 26324d95620bSPaul Winder uint_t tq_threads; 2633510a6847SHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 2634510a6847SHans Rosenfeld char *vendor, *product; 26353c9168faSHans Rosenfeld 26363c9168faSHans Rosenfeld /* Check controller version */ 26373c9168faSHans Rosenfeld vs.r = nvme_get32(nvme, NVME_REG_VS); 263824979ca3SHans Rosenfeld nvme->n_version.v_major = vs.b.vs_mjr; 263924979ca3SHans Rosenfeld nvme->n_version.v_minor = vs.b.vs_mnr; 26403c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d", 264124979ca3SHans Rosenfeld nvme->n_version.v_major, nvme->n_version.v_minor); 26423c9168faSHans Rosenfeld 264348d370f1SRobert Mustacchi if (nvme->n_version.v_major > nvme_version_major) { 264448d370f1SRobert Mustacchi dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.x", 264548d370f1SRobert Mustacchi nvme_version_major); 26463c9168faSHans Rosenfeld if (nvme->n_strict_version) 26473c9168faSHans Rosenfeld goto fail; 26483c9168faSHans Rosenfeld } 26493c9168faSHans Rosenfeld 26503c9168faSHans Rosenfeld /* retrieve controller configuration */ 26513c9168faSHans Rosenfeld cap.r = nvme_get64(nvme, NVME_REG_CAP); 26523c9168faSHans Rosenfeld 26533c9168faSHans Rosenfeld if ((cap.b.cap_css & NVME_CAP_CSS_NVM) == 0) { 26543c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 26553c9168faSHans Rosenfeld "!NVM command set not supported by hardware"); 26563c9168faSHans Rosenfeld goto fail; 26573c9168faSHans Rosenfeld } 26583c9168faSHans Rosenfeld 26593c9168faSHans Rosenfeld nvme->n_nssr_supported = cap.b.cap_nssrs; 26603c9168faSHans Rosenfeld nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd; 26613c9168faSHans Rosenfeld nvme->n_timeout = cap.b.cap_to; 26623c9168faSHans Rosenfeld nvme->n_arbitration_mechanisms = cap.b.cap_ams; 26633c9168faSHans Rosenfeld nvme->n_cont_queues_reqd = cap.b.cap_cqr; 26643c9168faSHans Rosenfeld nvme->n_max_queue_entries = cap.b.cap_mqes + 1; 26653c9168faSHans Rosenfeld 26663c9168faSHans Rosenfeld /* 26673c9168faSHans Rosenfeld * The MPSMIN and MPSMAX fields in the CAP register use 0 to specify 26683c9168faSHans Rosenfeld * the base page size of 4k (1<<12), so add 12 here to get the real 26693c9168faSHans Rosenfeld * page size value. 26703c9168faSHans Rosenfeld */ 26713c9168faSHans Rosenfeld nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT), 26723c9168faSHans Rosenfeld cap.b.cap_mpsmax + 12); 26733c9168faSHans Rosenfeld nvme->n_pagesize = 1UL << (nvme->n_pageshift); 26743c9168faSHans Rosenfeld 26753c9168faSHans Rosenfeld /* 26763c9168faSHans Rosenfeld * Set up Queue DMA to transfer at least 1 page-aligned page at a time. 26773c9168faSHans Rosenfeld */ 26783c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize; 26793c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 26803c9168faSHans Rosenfeld 26813c9168faSHans Rosenfeld /* 26823c9168faSHans Rosenfeld * Set up PRP DMA to transfer 1 page-aligned page at a time. 26833c9168faSHans Rosenfeld * Maxxfer may be increased after we identified the controller limits. 26843c9168faSHans Rosenfeld */ 26853c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize; 26863c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 26873c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize; 26882f95345bSYouzhong Yang nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1; 26893c9168faSHans Rosenfeld 26903c9168faSHans Rosenfeld /* 26913c9168faSHans Rosenfeld * Reset controller if it's still in ready state. 26923c9168faSHans Rosenfeld */ 26933c9168faSHans Rosenfeld if (nvme_reset(nvme, B_FALSE) == B_FALSE) { 26943c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller"); 26953c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 26963c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 26973c9168faSHans Rosenfeld goto fail; 26983c9168faSHans Rosenfeld } 26993c9168faSHans Rosenfeld 27000999c112SPaul Winder /* 27010999c112SPaul Winder * Create the cq array with one completion queue to be assigned 27024d95620bSPaul Winder * to the admin queue pair and a limited number of taskqs (4). 27030999c112SPaul Winder */ 27044d95620bSPaul Winder if (nvme_create_cq_array(nvme, 1, nvme->n_admin_queue_len, 4) != 27050999c112SPaul Winder DDI_SUCCESS) { 27060999c112SPaul Winder dev_err(nvme->n_dip, CE_WARN, 27070999c112SPaul Winder "!failed to pre-allocate admin completion queue"); 27080999c112SPaul Winder goto fail; 27090999c112SPaul Winder } 27103c9168faSHans Rosenfeld /* 27113c9168faSHans Rosenfeld * Create the admin queue pair. 27123c9168faSHans Rosenfeld */ 27133c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0) 27143c9168faSHans Rosenfeld != DDI_SUCCESS) { 27153c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 27163c9168faSHans Rosenfeld "!unable to allocate admin qpair"); 27173c9168faSHans Rosenfeld goto fail; 27183c9168faSHans Rosenfeld } 27193c9168faSHans Rosenfeld nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP); 27203c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 27213c9168faSHans Rosenfeld 27223c9168faSHans Rosenfeld nvme->n_progress |= NVME_ADMIN_QUEUE; 27233c9168faSHans Rosenfeld 27243c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 27253c9168faSHans Rosenfeld "admin-queue-len", nvme->n_admin_queue_len); 27263c9168faSHans Rosenfeld 27273c9168faSHans Rosenfeld aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1; 27283c9168faSHans Rosenfeld asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress; 27290999c112SPaul Winder acq = nvme->n_adminq->nq_cq->ncq_dma->nd_cookie.dmac_laddress; 27303c9168faSHans Rosenfeld 27313c9168faSHans Rosenfeld ASSERT((asq & (nvme->n_pagesize - 1)) == 0); 27323c9168faSHans Rosenfeld ASSERT((acq & (nvme->n_pagesize - 1)) == 0); 27333c9168faSHans Rosenfeld 27343c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, aqa.r); 27353c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ASQ, asq); 27363c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ACQ, acq); 27373c9168faSHans Rosenfeld 273834c938c7SPete Shephard cc.b.cc_ams = 0; /* use Round-Robin arbitration */ 273934c938c7SPete Shephard cc.b.cc_css = 0; /* use NVM command set */ 27403c9168faSHans Rosenfeld cc.b.cc_mps = nvme->n_pageshift - 12; 274134c938c7SPete Shephard cc.b.cc_shn = 0; /* no shutdown in progress */ 274234c938c7SPete Shephard cc.b.cc_en = 1; /* enable controller */ 274334c938c7SPete Shephard cc.b.cc_iosqes = 6; /* submission queue entry is 2^6 bytes long */ 274434c938c7SPete Shephard cc.b.cc_iocqes = 4; /* completion queue entry is 2^4 bytes long */ 27453c9168faSHans Rosenfeld 27463c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 27473c9168faSHans Rosenfeld 27483c9168faSHans Rosenfeld /* 27493c9168faSHans Rosenfeld * Wait for the controller to become ready. 27503c9168faSHans Rosenfeld */ 27513c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 27523c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 27533c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 27543c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 27553c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 27563c9168faSHans Rosenfeld 27573c9168faSHans Rosenfeld if (csts.b.csts_cfs == 1) { 27583c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 27593c9168faSHans Rosenfeld "!controller fatal status at init"); 27603c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, 27613c9168faSHans Rosenfeld DDI_SERVICE_LOST); 27623c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 27633c9168faSHans Rosenfeld goto fail; 27643c9168faSHans Rosenfeld } 27653c9168faSHans Rosenfeld 27663c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) 27673c9168faSHans Rosenfeld break; 27683c9168faSHans Rosenfeld } 27693c9168faSHans Rosenfeld } 27703c9168faSHans Rosenfeld 27713c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 27723c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!controller not ready"); 27733c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 27743c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 27753c9168faSHans Rosenfeld goto fail; 27763c9168faSHans Rosenfeld } 27773c9168faSHans Rosenfeld 27783c9168faSHans Rosenfeld /* 27793c9168faSHans Rosenfeld * Assume an abort command limit of 1. We'll destroy and re-init 27803c9168faSHans Rosenfeld * that later when we know the true abort command limit. 27813c9168faSHans Rosenfeld */ 27823c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL); 27833c9168faSHans Rosenfeld 2784bf26ea4bSHans Rosenfeld /* 2785bf26ea4bSHans Rosenfeld * Setup initial interrupt for admin queue. 2786bf26ea4bSHans Rosenfeld */ 2787bf26ea4bSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1) 2788bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 2789bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1) 2790bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 2791bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1) 2792bf26ea4bSHans Rosenfeld != DDI_SUCCESS)) { 2793bf26ea4bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2794bf26ea4bSHans Rosenfeld "!failed to setup initial interrupt"); 2795bf26ea4bSHans Rosenfeld goto fail; 2796bf26ea4bSHans Rosenfeld } 2797bf26ea4bSHans Rosenfeld 27983c9168faSHans Rosenfeld /* 27993c9168faSHans Rosenfeld * Post an asynchronous event command to catch errors. 280008139162SToomas Soome * We assume the asynchronous events are supported as required by 280108139162SToomas Soome * specification (Figure 40 in section 5 of NVMe 1.2). 280208139162SToomas Soome * However, since at least qemu does not follow the specification, 280308139162SToomas Soome * we need a mechanism to protect ourselves. 28043c9168faSHans Rosenfeld */ 280508139162SToomas Soome nvme->n_async_event_supported = B_TRUE; 28064b324362SHans Rosenfeld nvme_async_event(nvme); 28073c9168faSHans Rosenfeld 28083c9168faSHans Rosenfeld /* 28093c9168faSHans Rosenfeld * Identify Controller 28103c9168faSHans Rosenfeld */ 2811bc586359SRobert Mustacchi if (nvme_identify(nvme, B_FALSE, 0, (void **)&nvme->n_idctl) != 0) { 28123c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 28133c9168faSHans Rosenfeld "!failed to identify controller"); 28143c9168faSHans Rosenfeld goto fail; 28153c9168faSHans Rosenfeld } 28163c9168faSHans Rosenfeld 2817510a6847SHans Rosenfeld /* 2818510a6847SHans Rosenfeld * Get Vendor & Product ID 2819510a6847SHans Rosenfeld */ 2820510a6847SHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 2821510a6847SHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 2822510a6847SHans Rosenfeld sata_split_model(model, &vendor, &product); 2823510a6847SHans Rosenfeld 2824510a6847SHans Rosenfeld if (vendor == NULL) 2825510a6847SHans Rosenfeld nvme->n_vendor = strdup("NVMe"); 2826510a6847SHans Rosenfeld else 2827510a6847SHans Rosenfeld nvme->n_vendor = strdup(vendor); 2828510a6847SHans Rosenfeld 2829510a6847SHans Rosenfeld nvme->n_product = strdup(product); 2830510a6847SHans Rosenfeld 28313c9168faSHans Rosenfeld /* 28323c9168faSHans Rosenfeld * Get controller limits. 28333c9168faSHans Rosenfeld */ 28343c9168faSHans Rosenfeld nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT, 28353c9168faSHans Rosenfeld MIN(nvme->n_admin_queue_len / 10, 28363c9168faSHans Rosenfeld MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit))); 28373c9168faSHans Rosenfeld 28383c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 28393c9168faSHans Rosenfeld "async-event-limit", nvme->n_async_event_limit); 28403c9168faSHans Rosenfeld 28413c9168faSHans Rosenfeld nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1; 28423c9168faSHans Rosenfeld 28436afc9eb2SHans Rosenfeld /* 28446afc9eb2SHans Rosenfeld * Reinitialize the semaphore with the true abort command limit 28456afc9eb2SHans Rosenfeld * supported by the hardware. It's not necessary to disable interrupts 28466afc9eb2SHans Rosenfeld * as only command aborts use the semaphore, and no commands are 28476afc9eb2SHans Rosenfeld * executed or aborted while we're here. 28486afc9eb2SHans Rosenfeld */ 28493c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 28503c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL, 28513c9168faSHans Rosenfeld SEMA_DRIVER, NULL); 28523c9168faSHans Rosenfeld 28533c9168faSHans Rosenfeld nvme->n_progress |= NVME_CTRL_LIMITS; 28543c9168faSHans Rosenfeld 28553c9168faSHans Rosenfeld if (nvme->n_idctl->id_mdts == 0) 28563c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536; 28573c9168faSHans Rosenfeld else 28583c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = 28593c9168faSHans Rosenfeld 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts); 28603c9168faSHans Rosenfeld 28613c9168faSHans Rosenfeld nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1; 28623c9168faSHans Rosenfeld 28633c9168faSHans Rosenfeld /* 28643c9168faSHans Rosenfeld * Limit n_max_data_transfer_size to what we can handle in one PRP. 28653c9168faSHans Rosenfeld * Chained PRPs are currently unsupported. 28663c9168faSHans Rosenfeld * 28673c9168faSHans Rosenfeld * This is a no-op on hardware which doesn't support a transfer size 28683c9168faSHans Rosenfeld * big enough to require chained PRPs. 28693c9168faSHans Rosenfeld */ 28703c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size, 28713c9168faSHans Rosenfeld (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize)); 28723c9168faSHans Rosenfeld 28733c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size; 28743c9168faSHans Rosenfeld 28753c9168faSHans Rosenfeld /* 28763c9168faSHans Rosenfeld * Make sure the minimum/maximum queue entry sizes are not 28773c9168faSHans Rosenfeld * larger/smaller than the default. 28783c9168faSHans Rosenfeld */ 28793c9168faSHans Rosenfeld 28803c9168faSHans Rosenfeld if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) || 28813c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) || 28823c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) || 28833c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t))) 28843c9168faSHans Rosenfeld goto fail; 28853c9168faSHans Rosenfeld 28863c9168faSHans Rosenfeld /* 28873c9168faSHans Rosenfeld * Check for the presence of a Volatile Write Cache. If present, 2888d148d46eSHans Rosenfeld * enable or disable based on the value of the property 2889d148d46eSHans Rosenfeld * volatile-write-cache-enable (default is enabled). 28903c9168faSHans Rosenfeld */ 2891d148d46eSHans Rosenfeld nvme->n_write_cache_present = 2892d148d46eSHans Rosenfeld nvme->n_idctl->id_vwc.vwc_present == 0 ? B_FALSE : B_TRUE; 2893d148d46eSHans Rosenfeld 2894d148d46eSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2895d148d46eSHans Rosenfeld "volatile-write-cache-present", 2896d148d46eSHans Rosenfeld nvme->n_write_cache_present ? 1 : 0); 2897d148d46eSHans Rosenfeld 2898d148d46eSHans Rosenfeld if (!nvme->n_write_cache_present) { 2899d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = B_FALSE; 2900e984c70bSHans Rosenfeld } else if (nvme_write_cache_set(nvme, nvme->n_write_cache_enabled) 2901e984c70bSHans Rosenfeld != 0) { 2902d148d46eSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2903d148d46eSHans Rosenfeld "!failed to %sable volatile write cache", 2904d148d46eSHans Rosenfeld nvme->n_write_cache_enabled ? "en" : "dis"); 29053c9168faSHans Rosenfeld /* 2906d148d46eSHans Rosenfeld * Assume the cache is (still) enabled. 29073c9168faSHans Rosenfeld */ 2908d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = B_TRUE; 29093c9168faSHans Rosenfeld } 29103c9168faSHans Rosenfeld 2911d148d46eSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2912d148d46eSHans Rosenfeld "volatile-write-cache-enable", 2913d148d46eSHans Rosenfeld nvme->n_write_cache_enabled ? 1 : 0); 2914d148d46eSHans Rosenfeld 29153c9168faSHans Rosenfeld /* 29163d9b1a2aSHans Rosenfeld * Assume LBA Range Type feature is supported. If it isn't this 29173d9b1a2aSHans Rosenfeld * will be set to B_FALSE by nvme_get_features(). 29183c9168faSHans Rosenfeld */ 29193d9b1a2aSHans Rosenfeld nvme->n_lba_range_supported = B_TRUE; 29203d9b1a2aSHans Rosenfeld 29213d9b1a2aSHans Rosenfeld /* 29223d9b1a2aSHans Rosenfeld * Check support for Autonomous Power State Transition. 29233d9b1a2aSHans Rosenfeld */ 29243d9b1a2aSHans Rosenfeld if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) 29253d9b1a2aSHans Rosenfeld nvme->n_auto_pst_supported = 29263d9b1a2aSHans Rosenfeld nvme->n_idctl->id_apsta.ap_sup == 0 ? B_FALSE : B_TRUE; 29273c9168faSHans Rosenfeld 2928f313c178SYuri Pankov /* 2929f313c178SYuri Pankov * Assume Software Progress Marker feature is supported. If it isn't 2930f313c178SYuri Pankov * this will be set to B_FALSE by nvme_get_features(). 2931f313c178SYuri Pankov */ 2932f313c178SYuri Pankov nvme->n_progress_supported = B_TRUE; 2933f313c178SYuri Pankov 29343c9168faSHans Rosenfeld /* 29353c9168faSHans Rosenfeld * Identify Namespaces 29363c9168faSHans Rosenfeld */ 29373c9168faSHans Rosenfeld nvme->n_namespace_count = nvme->n_idctl->id_nn; 2938bf8c808aSYuri Pankov 2939bf8c808aSYuri Pankov if (nvme->n_namespace_count == 0) { 2940bf8c808aSYuri Pankov dev_err(nvme->n_dip, CE_WARN, 2941bf8c808aSYuri Pankov "!controllers without namespaces are not supported"); 2942bf8c808aSYuri Pankov goto fail; 2943bf8c808aSYuri Pankov } 2944bf8c808aSYuri Pankov 29453d9b1a2aSHans Rosenfeld if (nvme->n_namespace_count > NVME_MINOR_MAX) { 29463d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 29473d9b1a2aSHans Rosenfeld "!too many namespaces: %d, limiting to %d\n", 29483d9b1a2aSHans Rosenfeld nvme->n_namespace_count, NVME_MINOR_MAX); 29493d9b1a2aSHans Rosenfeld nvme->n_namespace_count = NVME_MINOR_MAX; 29503d9b1a2aSHans Rosenfeld } 29513d9b1a2aSHans Rosenfeld 29523c9168faSHans Rosenfeld nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) * 29533c9168faSHans Rosenfeld nvme->n_namespace_count, KM_SLEEP); 29543c9168faSHans Rosenfeld 29553c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 29563d9b1a2aSHans Rosenfeld mutex_init(&nvme->n_ns[i].ns_minor.nm_mutex, NULL, MUTEX_DRIVER, 29573d9b1a2aSHans Rosenfeld NULL); 29584d95620bSPaul Winder nvme->n_ns[i].ns_ignore = B_TRUE; 29593d9b1a2aSHans Rosenfeld if (nvme_init_ns(nvme, i + 1) != DDI_SUCCESS) 29603c9168faSHans Rosenfeld goto fail; 29613c9168faSHans Rosenfeld } 29623c9168faSHans Rosenfeld 29633c9168faSHans Rosenfeld /* 29643c9168faSHans Rosenfeld * Try to set up MSI/MSI-X interrupts. 29653c9168faSHans Rosenfeld */ 29663c9168faSHans Rosenfeld if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX)) 29673c9168faSHans Rosenfeld != 0) { 29683c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 29693c9168faSHans Rosenfeld 29703c9168faSHans Rosenfeld nqueues = MIN(UINT16_MAX, ncpus); 29713c9168faSHans Rosenfeld 29723c9168faSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 29733c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS) && 29743c9168faSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 29753c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS)) { 29763c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 29773c9168faSHans Rosenfeld "!failed to setup MSI/MSI-X interrupts"); 29783c9168faSHans Rosenfeld goto fail; 29793c9168faSHans Rosenfeld } 29803c9168faSHans Rosenfeld } 29813c9168faSHans Rosenfeld 29823c9168faSHans Rosenfeld /* 29833c9168faSHans Rosenfeld * Create I/O queue pairs. 29843c9168faSHans Rosenfeld */ 2985e984c70bSHans Rosenfeld 29860999c112SPaul Winder if (nvme_set_nqueues(nvme) != 0) { 29873c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2988e984c70bSHans Rosenfeld "!failed to set number of I/O queues to %d", 2989e984c70bSHans Rosenfeld nvme->n_intr_cnt); 29903c9168faSHans Rosenfeld goto fail; 29913c9168faSHans Rosenfeld } 29923c9168faSHans Rosenfeld 29933c9168faSHans Rosenfeld /* 29943c9168faSHans Rosenfeld * Reallocate I/O queue array 29953c9168faSHans Rosenfeld */ 29963c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *)); 29973c9168faSHans Rosenfeld nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) * 29980999c112SPaul Winder (nvme->n_submission_queues + 1), KM_SLEEP); 29993c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 30003c9168faSHans Rosenfeld 30010999c112SPaul Winder /* 30020999c112SPaul Winder * There should always be at least as many submission queues 30030999c112SPaul Winder * as completion queues. 30040999c112SPaul Winder */ 30050999c112SPaul Winder ASSERT(nvme->n_submission_queues >= nvme->n_completion_queues); 30060999c112SPaul Winder 30070999c112SPaul Winder nvme->n_ioq_count = nvme->n_submission_queues; 30080999c112SPaul Winder 30090999c112SPaul Winder nvme->n_io_squeue_len = 30100999c112SPaul Winder MIN(nvme->n_io_squeue_len, nvme->n_max_queue_entries); 30110999c112SPaul Winder 30120999c112SPaul Winder (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-squeue-len", 30130999c112SPaul Winder nvme->n_io_squeue_len); 30140999c112SPaul Winder 30150999c112SPaul Winder /* 30160999c112SPaul Winder * Pre-allocate completion queues. 30170999c112SPaul Winder * When there are the same number of submission and completion 30180999c112SPaul Winder * queues there is no value in having a larger completion 30190999c112SPaul Winder * queue length. 30200999c112SPaul Winder */ 30210999c112SPaul Winder if (nvme->n_submission_queues == nvme->n_completion_queues) 30220999c112SPaul Winder nvme->n_io_cqueue_len = MIN(nvme->n_io_cqueue_len, 30230999c112SPaul Winder nvme->n_io_squeue_len); 30240999c112SPaul Winder 30250999c112SPaul Winder nvme->n_io_cqueue_len = MIN(nvme->n_io_cqueue_len, 30260999c112SPaul Winder nvme->n_max_queue_entries); 30270999c112SPaul Winder 30280999c112SPaul Winder (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-cqueue-len", 30290999c112SPaul Winder nvme->n_io_cqueue_len); 30300999c112SPaul Winder 30314d95620bSPaul Winder /* 30324d95620bSPaul Winder * Assign the equal quantity of taskq threads to each completion 30334d95620bSPaul Winder * queue, capping the total number of threads to the number 30344d95620bSPaul Winder * of CPUs. 30354d95620bSPaul Winder */ 30364d95620bSPaul Winder tq_threads = MIN(UINT16_MAX, ncpus) / nvme->n_completion_queues; 30374d95620bSPaul Winder 30384d95620bSPaul Winder /* 30394d95620bSPaul Winder * In case the calculation above is zero, we need at least one 30404d95620bSPaul Winder * thread per completion queue. 30414d95620bSPaul Winder */ 30424d95620bSPaul Winder tq_threads = MAX(1, tq_threads); 30434d95620bSPaul Winder 30440999c112SPaul Winder if (nvme_create_cq_array(nvme, nvme->n_completion_queues + 1, 30454d95620bSPaul Winder nvme->n_io_cqueue_len, tq_threads) != DDI_SUCCESS) { 30460999c112SPaul Winder dev_err(nvme->n_dip, CE_WARN, 30470999c112SPaul Winder "!failed to pre-allocate completion queues"); 30480999c112SPaul Winder goto fail; 30490999c112SPaul Winder } 3050e984c70bSHans Rosenfeld 30513c9168faSHans Rosenfeld /* 30520999c112SPaul Winder * If we use less completion queues than interrupt vectors return 30533c9168faSHans Rosenfeld * some of the interrupt vectors back to the system. 30543c9168faSHans Rosenfeld */ 30550999c112SPaul Winder if (nvme->n_completion_queues + 1 < nvme->n_intr_cnt) { 30563c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 30573c9168faSHans Rosenfeld 305834c938c7SPete Shephard if (nvme_setup_interrupts(nvme, nvme->n_intr_type, 30590999c112SPaul Winder nvme->n_completion_queues + 1) != DDI_SUCCESS) { 30603c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 30613c9168faSHans Rosenfeld "!failed to reduce number of interrupts"); 30623c9168faSHans Rosenfeld goto fail; 30633c9168faSHans Rosenfeld } 30643c9168faSHans Rosenfeld } 30653c9168faSHans Rosenfeld 30663c9168faSHans Rosenfeld /* 30673c9168faSHans Rosenfeld * Alloc & register I/O queue pairs 30683c9168faSHans Rosenfeld */ 30693c9168faSHans Rosenfeld 30703c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 30710999c112SPaul Winder if (nvme_alloc_qpair(nvme, nvme->n_io_squeue_len, 30723c9168faSHans Rosenfeld &nvme->n_ioq[i], i) != DDI_SUCCESS) { 30733c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 30743c9168faSHans Rosenfeld "!unable to allocate I/O qpair %d", i); 30753c9168faSHans Rosenfeld goto fail; 30763c9168faSHans Rosenfeld } 30773c9168faSHans Rosenfeld 3078e984c70bSHans Rosenfeld if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i) != 0) { 30793c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 30803c9168faSHans Rosenfeld "!unable to create I/O qpair %d", i); 30813c9168faSHans Rosenfeld goto fail; 30823c9168faSHans Rosenfeld } 30833c9168faSHans Rosenfeld } 30843c9168faSHans Rosenfeld 30853c9168faSHans Rosenfeld /* 30863c9168faSHans Rosenfeld * Post more asynchronous events commands to reduce event reporting 30873c9168faSHans Rosenfeld * latency as suggested by the spec. 30883c9168faSHans Rosenfeld */ 308908139162SToomas Soome if (nvme->n_async_event_supported) { 309008139162SToomas Soome for (i = 1; i != nvme->n_async_event_limit; i++) 309108139162SToomas Soome nvme_async_event(nvme); 309208139162SToomas Soome } 30933c9168faSHans Rosenfeld 30943c9168faSHans Rosenfeld return (DDI_SUCCESS); 30953c9168faSHans Rosenfeld 30963c9168faSHans Rosenfeld fail: 30973c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 30983c9168faSHans Rosenfeld return (DDI_FAILURE); 30993c9168faSHans Rosenfeld } 31003c9168faSHans Rosenfeld 31013c9168faSHans Rosenfeld static uint_t 31023c9168faSHans Rosenfeld nvme_intr(caddr_t arg1, caddr_t arg2) 31033c9168faSHans Rosenfeld { 31043c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 31053c9168faSHans Rosenfeld nvme_t *nvme = (nvme_t *)arg1; 31063c9168faSHans Rosenfeld int inum = (int)(uintptr_t)arg2; 3107bf26ea4bSHans Rosenfeld int ccnt = 0; 31083c9168faSHans Rosenfeld int qnum; 31093c9168faSHans Rosenfeld 31103c9168faSHans Rosenfeld if (inum >= nvme->n_intr_cnt) 31113c9168faSHans Rosenfeld return (DDI_INTR_UNCLAIMED); 31123c9168faSHans Rosenfeld 3113e984c70bSHans Rosenfeld if (nvme->n_dead) 3114e984c70bSHans Rosenfeld return (nvme->n_intr_type == DDI_INTR_TYPE_FIXED ? 3115e984c70bSHans Rosenfeld DDI_INTR_UNCLAIMED : DDI_INTR_CLAIMED); 3116e984c70bSHans Rosenfeld 31173c9168faSHans Rosenfeld /* 31183c9168faSHans Rosenfeld * The interrupt vector a queue uses is calculated as queue_idx % 31193c9168faSHans Rosenfeld * intr_cnt in nvme_create_io_qpair(). Iterate through the queue array 31203c9168faSHans Rosenfeld * in steps of n_intr_cnt to process all queues using this vector. 31213c9168faSHans Rosenfeld */ 31223c9168faSHans Rosenfeld for (qnum = inum; 31230999c112SPaul Winder qnum < nvme->n_cq_count && nvme->n_cq[qnum] != NULL; 31243c9168faSHans Rosenfeld qnum += nvme->n_intr_cnt) { 31250999c112SPaul Winder ccnt += nvme_process_iocq(nvme, nvme->n_cq[qnum]); 31263c9168faSHans Rosenfeld } 31273c9168faSHans Rosenfeld 3128bf26ea4bSHans Rosenfeld return (ccnt > 0 ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); 31293c9168faSHans Rosenfeld } 31303c9168faSHans Rosenfeld 31313c9168faSHans Rosenfeld static void 31326afc9eb2SHans Rosenfeld nvme_release_interrupts(nvme_t *nvme) 31333c9168faSHans Rosenfeld { 31343c9168faSHans Rosenfeld int i; 31353c9168faSHans Rosenfeld 31363c9168faSHans Rosenfeld for (i = 0; i < nvme->n_intr_cnt; i++) { 31373c9168faSHans Rosenfeld if (nvme->n_inth[i] == NULL) 31383c9168faSHans Rosenfeld break; 31393c9168faSHans Rosenfeld 31403c9168faSHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 31413c9168faSHans Rosenfeld (void) ddi_intr_block_disable(&nvme->n_inth[i], 1); 31423c9168faSHans Rosenfeld else 31433c9168faSHans Rosenfeld (void) ddi_intr_disable(nvme->n_inth[i]); 31443c9168faSHans Rosenfeld 31453c9168faSHans Rosenfeld (void) ddi_intr_remove_handler(nvme->n_inth[i]); 31463c9168faSHans Rosenfeld (void) ddi_intr_free(nvme->n_inth[i]); 31473c9168faSHans Rosenfeld } 31483c9168faSHans Rosenfeld 31493c9168faSHans Rosenfeld kmem_free(nvme->n_inth, nvme->n_inth_sz); 31503c9168faSHans Rosenfeld nvme->n_inth = NULL; 31513c9168faSHans Rosenfeld nvme->n_inth_sz = 0; 31523c9168faSHans Rosenfeld 31533c9168faSHans Rosenfeld nvme->n_progress &= ~NVME_INTERRUPTS; 31543c9168faSHans Rosenfeld } 31553c9168faSHans Rosenfeld 31563c9168faSHans Rosenfeld static int 31573c9168faSHans Rosenfeld nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs) 31583c9168faSHans Rosenfeld { 31593c9168faSHans Rosenfeld int nintrs, navail, count; 31603c9168faSHans Rosenfeld int ret; 31613c9168faSHans Rosenfeld int i; 31623c9168faSHans Rosenfeld 31633c9168faSHans Rosenfeld if (nvme->n_intr_types == 0) { 31643c9168faSHans Rosenfeld ret = ddi_intr_get_supported_types(nvme->n_dip, 31653c9168faSHans Rosenfeld &nvme->n_intr_types); 31663c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 31673c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 31683c9168faSHans Rosenfeld "!%s: ddi_intr_get_supported types failed", 31693c9168faSHans Rosenfeld __func__); 31703c9168faSHans Rosenfeld return (ret); 31713c9168faSHans Rosenfeld } 31729d08e1f8SHans Rosenfeld #ifdef __x86 31739d08e1f8SHans Rosenfeld if (get_hwenv() == HW_VMWARE) 31749d08e1f8SHans Rosenfeld nvme->n_intr_types &= ~DDI_INTR_TYPE_MSIX; 31759d08e1f8SHans Rosenfeld #endif 31763c9168faSHans Rosenfeld } 31773c9168faSHans Rosenfeld 31783c9168faSHans Rosenfeld if ((nvme->n_intr_types & intr_type) == 0) 31793c9168faSHans Rosenfeld return (DDI_FAILURE); 31803c9168faSHans Rosenfeld 31813c9168faSHans Rosenfeld ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs); 31823c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 31833c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed", 31843c9168faSHans Rosenfeld __func__); 31853c9168faSHans Rosenfeld return (ret); 31863c9168faSHans Rosenfeld } 31873c9168faSHans Rosenfeld 31883c9168faSHans Rosenfeld ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail); 31893c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 31903c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed", 31913c9168faSHans Rosenfeld __func__); 31923c9168faSHans Rosenfeld return (ret); 31933c9168faSHans Rosenfeld } 31943c9168faSHans Rosenfeld 31953c9168faSHans Rosenfeld /* We want at most one interrupt per queue pair. */ 31963c9168faSHans Rosenfeld if (navail > nqpairs) 31973c9168faSHans Rosenfeld navail = nqpairs; 31983c9168faSHans Rosenfeld 31993c9168faSHans Rosenfeld nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail; 32003c9168faSHans Rosenfeld nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP); 32013c9168faSHans Rosenfeld 32023c9168faSHans Rosenfeld ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail, 32033c9168faSHans Rosenfeld &count, 0); 32043c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 32053c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed", 32063c9168faSHans Rosenfeld __func__); 32073c9168faSHans Rosenfeld goto fail; 32083c9168faSHans Rosenfeld } 32093c9168faSHans Rosenfeld 32103c9168faSHans Rosenfeld nvme->n_intr_cnt = count; 32113c9168faSHans Rosenfeld 32123c9168faSHans Rosenfeld ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri); 32133c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 32143c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed", 32153c9168faSHans Rosenfeld __func__); 32163c9168faSHans Rosenfeld goto fail; 32173c9168faSHans Rosenfeld } 32183c9168faSHans Rosenfeld 32193c9168faSHans Rosenfeld for (i = 0; i < count; i++) { 32203c9168faSHans Rosenfeld ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr, 32213c9168faSHans Rosenfeld (void *)nvme, (void *)(uintptr_t)i); 32223c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 32233c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 32243c9168faSHans Rosenfeld "!%s: ddi_intr_add_handler failed", __func__); 32253c9168faSHans Rosenfeld goto fail; 32263c9168faSHans Rosenfeld } 32273c9168faSHans Rosenfeld } 32283c9168faSHans Rosenfeld 32293c9168faSHans Rosenfeld (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap); 32303c9168faSHans Rosenfeld 32316afc9eb2SHans Rosenfeld for (i = 0; i < count; i++) { 323275b41617SHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 323375b41617SHans Rosenfeld ret = ddi_intr_block_enable(&nvme->n_inth[i], 1); 323475b41617SHans Rosenfeld else 323575b41617SHans Rosenfeld ret = ddi_intr_enable(nvme->n_inth[i]); 32363c9168faSHans Rosenfeld 323775b41617SHans Rosenfeld if (ret != DDI_SUCCESS) { 323875b41617SHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 323975b41617SHans Rosenfeld "!%s: enabling interrupt %d failed", __func__, i); 324075b41617SHans Rosenfeld goto fail; 32416afc9eb2SHans Rosenfeld } 32426afc9eb2SHans Rosenfeld } 32436afc9eb2SHans Rosenfeld 32443c9168faSHans Rosenfeld nvme->n_intr_type = intr_type; 32453c9168faSHans Rosenfeld 32463c9168faSHans Rosenfeld nvme->n_progress |= NVME_INTERRUPTS; 32473c9168faSHans Rosenfeld 32483c9168faSHans Rosenfeld return (DDI_SUCCESS); 32493c9168faSHans Rosenfeld 32503c9168faSHans Rosenfeld fail: 32513c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 32523c9168faSHans Rosenfeld 32533c9168faSHans Rosenfeld return (ret); 32543c9168faSHans Rosenfeld } 32553c9168faSHans Rosenfeld 32563c9168faSHans Rosenfeld static int 32573c9168faSHans Rosenfeld nvme_fm_errcb(dev_info_t *dip, ddi_fm_error_t *fm_error, const void *arg) 32583c9168faSHans Rosenfeld { 32593c9168faSHans Rosenfeld _NOTE(ARGUNUSED(arg)); 32603c9168faSHans Rosenfeld 32613c9168faSHans Rosenfeld pci_ereport_post(dip, fm_error, NULL); 32623c9168faSHans Rosenfeld return (fm_error->fme_status); 32633c9168faSHans Rosenfeld } 32643c9168faSHans Rosenfeld 32653c9168faSHans Rosenfeld static int 32663c9168faSHans Rosenfeld nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 32673c9168faSHans Rosenfeld { 32683c9168faSHans Rosenfeld nvme_t *nvme; 32693c9168faSHans Rosenfeld int instance; 32703c9168faSHans Rosenfeld int nregs; 32713c9168faSHans Rosenfeld off_t regsize; 32723c9168faSHans Rosenfeld int i; 32733c9168faSHans Rosenfeld char name[32]; 32743c9168faSHans Rosenfeld 32753c9168faSHans Rosenfeld if (cmd != DDI_ATTACH) 32763c9168faSHans Rosenfeld return (DDI_FAILURE); 32773c9168faSHans Rosenfeld 32783c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 32793c9168faSHans Rosenfeld 32803c9168faSHans Rosenfeld if (ddi_soft_state_zalloc(nvme_state, instance) != DDI_SUCCESS) 32813c9168faSHans Rosenfeld return (DDI_FAILURE); 32823c9168faSHans Rosenfeld 32833c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 32843c9168faSHans Rosenfeld ddi_set_driver_private(dip, nvme); 32853c9168faSHans Rosenfeld nvme->n_dip = dip; 32863c9168faSHans Rosenfeld 32873d9b1a2aSHans Rosenfeld mutex_init(&nvme->n_minor.nm_mutex, NULL, MUTEX_DRIVER, NULL); 32883d9b1a2aSHans Rosenfeld 32893c9168faSHans Rosenfeld nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 32903c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "strict-version", 1) == 1 ? B_TRUE : B_FALSE; 32913c9168faSHans Rosenfeld nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY, 32923c9168faSHans Rosenfeld dip, DDI_PROP_DONTPASS, "ignore-unknown-vendor-status", 0) == 1 ? 32933c9168faSHans Rosenfeld B_TRUE : B_FALSE; 32943c9168faSHans Rosenfeld nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 32953c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "admin-queue-len", NVME_DEFAULT_ADMIN_QUEUE_LEN); 32960999c112SPaul Winder nvme->n_io_squeue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 32970999c112SPaul Winder DDI_PROP_DONTPASS, "io-squeue-len", NVME_DEFAULT_IO_QUEUE_LEN); 32980999c112SPaul Winder /* 32990999c112SPaul Winder * Double up the default for completion queues in case of 33000999c112SPaul Winder * queue sharing. 33010999c112SPaul Winder */ 33020999c112SPaul Winder nvme->n_io_cqueue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 33030999c112SPaul Winder DDI_PROP_DONTPASS, "io-cqueue-len", 2 * NVME_DEFAULT_IO_QUEUE_LEN); 33043c9168faSHans Rosenfeld nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 33053c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "async-event-limit", 33063c9168faSHans Rosenfeld NVME_DEFAULT_ASYNC_EVENT_LIMIT); 3307d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 3308d148d46eSHans Rosenfeld DDI_PROP_DONTPASS, "volatile-write-cache-enable", 1) != 0 ? 3309d148d46eSHans Rosenfeld B_TRUE : B_FALSE; 33106801591eSHans Rosenfeld nvme->n_min_block_size = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 33116801591eSHans Rosenfeld DDI_PROP_DONTPASS, "min-phys-block-size", 33126801591eSHans Rosenfeld NVME_DEFAULT_MIN_BLOCK_SIZE); 33130999c112SPaul Winder nvme->n_submission_queues = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 33140999c112SPaul Winder DDI_PROP_DONTPASS, "max-submission-queues", -1); 33150999c112SPaul Winder nvme->n_completion_queues = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 33160999c112SPaul Winder DDI_PROP_DONTPASS, "max-completion-queues", -1); 33176801591eSHans Rosenfeld 33186801591eSHans Rosenfeld if (!ISP2(nvme->n_min_block_size) || 33196801591eSHans Rosenfeld (nvme->n_min_block_size < NVME_DEFAULT_MIN_BLOCK_SIZE)) { 33206801591eSHans Rosenfeld dev_err(dip, CE_WARN, "!min-phys-block-size %s, " 33216801591eSHans Rosenfeld "using default %d", ISP2(nvme->n_min_block_size) ? 33226801591eSHans Rosenfeld "too low" : "not a power of 2", 33236801591eSHans Rosenfeld NVME_DEFAULT_MIN_BLOCK_SIZE); 33246801591eSHans Rosenfeld nvme->n_min_block_size = NVME_DEFAULT_MIN_BLOCK_SIZE; 33256801591eSHans Rosenfeld } 33263c9168faSHans Rosenfeld 33270999c112SPaul Winder if (nvme->n_submission_queues != -1 && 33280999c112SPaul Winder (nvme->n_submission_queues < 1 || 33290999c112SPaul Winder nvme->n_submission_queues > UINT16_MAX)) { 33300999c112SPaul Winder dev_err(dip, CE_WARN, "!\"submission-queues\"=%d is not " 33310999c112SPaul Winder "valid. Must be [1..%d]", nvme->n_submission_queues, 33320999c112SPaul Winder UINT16_MAX); 33330999c112SPaul Winder nvme->n_submission_queues = -1; 33340999c112SPaul Winder } 33350999c112SPaul Winder 33360999c112SPaul Winder if (nvme->n_completion_queues != -1 && 33370999c112SPaul Winder (nvme->n_completion_queues < 1 || 33380999c112SPaul Winder nvme->n_completion_queues > UINT16_MAX)) { 33390999c112SPaul Winder dev_err(dip, CE_WARN, "!\"completion-queues\"=%d is not " 33400999c112SPaul Winder "valid. Must be [1..%d]", nvme->n_completion_queues, 33410999c112SPaul Winder UINT16_MAX); 33420999c112SPaul Winder nvme->n_completion_queues = -1; 33430999c112SPaul Winder } 33440999c112SPaul Winder 33453c9168faSHans Rosenfeld if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN) 33463c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN; 33473c9168faSHans Rosenfeld else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN) 33483c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN; 33493c9168faSHans Rosenfeld 33500999c112SPaul Winder if (nvme->n_io_squeue_len < NVME_MIN_IO_QUEUE_LEN) 33510999c112SPaul Winder nvme->n_io_squeue_len = NVME_MIN_IO_QUEUE_LEN; 33520999c112SPaul Winder if (nvme->n_io_cqueue_len < NVME_MIN_IO_QUEUE_LEN) 33530999c112SPaul Winder nvme->n_io_cqueue_len = NVME_MIN_IO_QUEUE_LEN; 33543c9168faSHans Rosenfeld 33553c9168faSHans Rosenfeld if (nvme->n_async_event_limit < 1) 33563c9168faSHans Rosenfeld nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT; 33573c9168faSHans Rosenfeld 33583c9168faSHans Rosenfeld nvme->n_reg_acc_attr = nvme_reg_acc_attr; 33593c9168faSHans Rosenfeld nvme->n_queue_dma_attr = nvme_queue_dma_attr; 33603c9168faSHans Rosenfeld nvme->n_prp_dma_attr = nvme_prp_dma_attr; 33613c9168faSHans Rosenfeld nvme->n_sgl_dma_attr = nvme_sgl_dma_attr; 33623c9168faSHans Rosenfeld 33633c9168faSHans Rosenfeld /* 33643c9168faSHans Rosenfeld * Setup FMA support. 33653c9168faSHans Rosenfeld */ 33663c9168faSHans Rosenfeld nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip, 33673c9168faSHans Rosenfeld DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 33683c9168faSHans Rosenfeld DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 33693c9168faSHans Rosenfeld DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 33703c9168faSHans Rosenfeld 33713c9168faSHans Rosenfeld ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc); 33723c9168faSHans Rosenfeld 33733c9168faSHans Rosenfeld if (nvme->n_fm_cap) { 33743c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE) 33753c9168faSHans Rosenfeld nvme->n_reg_acc_attr.devacc_attr_access = 33763c9168faSHans Rosenfeld DDI_FLAGERR_ACC; 33773c9168faSHans Rosenfeld 33783c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) { 33793c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 33803c9168faSHans Rosenfeld nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 33813c9168faSHans Rosenfeld } 33823c9168faSHans Rosenfeld 33833c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 33843c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 33853c9168faSHans Rosenfeld pci_ereport_setup(dip); 33863c9168faSHans Rosenfeld 33873c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 33883c9168faSHans Rosenfeld ddi_fm_handler_register(dip, nvme_fm_errcb, 33893c9168faSHans Rosenfeld (void *)nvme); 33903c9168faSHans Rosenfeld } 33913c9168faSHans Rosenfeld 33923c9168faSHans Rosenfeld nvme->n_progress |= NVME_FMA_INIT; 33933c9168faSHans Rosenfeld 33943c9168faSHans Rosenfeld /* 33953c9168faSHans Rosenfeld * The spec defines several register sets. Only the controller 33963c9168faSHans Rosenfeld * registers (set 1) are currently used. 33973c9168faSHans Rosenfeld */ 33983c9168faSHans Rosenfeld if (ddi_dev_nregs(dip, &nregs) == DDI_FAILURE || 33993c9168faSHans Rosenfeld nregs < 2 || 34003c9168faSHans Rosenfeld ddi_dev_regsize(dip, 1, ®size) == DDI_FAILURE) 34013c9168faSHans Rosenfeld goto fail; 34023c9168faSHans Rosenfeld 34033c9168faSHans Rosenfeld if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize, 34043c9168faSHans Rosenfeld &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) { 34053c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to map regset 1"); 34063c9168faSHans Rosenfeld goto fail; 34073c9168faSHans Rosenfeld } 34083c9168faSHans Rosenfeld 34093c9168faSHans Rosenfeld nvme->n_progress |= NVME_REGS_MAPPED; 34103c9168faSHans Rosenfeld 34118834f7acSYouzhong Yang /* 34128834f7acSYouzhong Yang * Create PRP DMA cache 34138834f7acSYouzhong Yang */ 34148834f7acSYouzhong Yang (void) snprintf(name, sizeof (name), "%s%d_prp_cache", 34158834f7acSYouzhong Yang ddi_driver_name(dip), ddi_get_instance(dip)); 34168834f7acSYouzhong Yang nvme->n_prp_cache = kmem_cache_create(name, sizeof (nvme_dma_t), 34178834f7acSYouzhong Yang 0, nvme_prp_dma_constructor, nvme_prp_dma_destructor, 34188834f7acSYouzhong Yang NULL, (void *)nvme, NULL, 0); 34193c9168faSHans Rosenfeld 34203c9168faSHans Rosenfeld if (nvme_init(nvme) != DDI_SUCCESS) 34213c9168faSHans Rosenfeld goto fail; 34223c9168faSHans Rosenfeld 3423e89be50aSRob Johnston /* 3424e89be50aSRob Johnston * Initialize the driver with the UFM subsystem 3425e89be50aSRob Johnston */ 3426e89be50aSRob Johnston if (ddi_ufm_init(dip, DDI_UFM_CURRENT_VERSION, &nvme_ufm_ops, 3427e89be50aSRob Johnston &nvme->n_ufmh, nvme) != 0) { 3428e89be50aSRob Johnston dev_err(dip, CE_WARN, "!failed to initialize UFM subsystem"); 3429e89be50aSRob Johnston goto fail; 3430e89be50aSRob Johnston } 3431e89be50aSRob Johnston mutex_init(&nvme->n_fwslot_mutex, NULL, MUTEX_DRIVER, NULL); 3432e89be50aSRob Johnston ddi_ufm_update(nvme->n_ufmh); 3433e89be50aSRob Johnston nvme->n_progress |= NVME_UFM_INIT; 3434e89be50aSRob Johnston 34353c9168faSHans Rosenfeld /* 34363c9168faSHans Rosenfeld * Attach the blkdev driver for each namespace. 34373c9168faSHans Rosenfeld */ 34383c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 34393d9b1a2aSHans Rosenfeld if (ddi_create_minor_node(nvme->n_dip, nvme->n_ns[i].ns_name, 34403d9b1a2aSHans Rosenfeld S_IFCHR, NVME_MINOR(ddi_get_instance(nvme->n_dip), i + 1), 34413d9b1a2aSHans Rosenfeld DDI_NT_NVME_ATTACHMENT_POINT, 0) != DDI_SUCCESS) { 34423d9b1a2aSHans Rosenfeld dev_err(dip, CE_WARN, 34433d9b1a2aSHans Rosenfeld "!failed to create minor node for namespace %d", i); 34443d9b1a2aSHans Rosenfeld goto fail; 34453d9b1a2aSHans Rosenfeld } 34463d9b1a2aSHans Rosenfeld 34473c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_ignore) 34483c9168faSHans Rosenfeld continue; 34493c9168faSHans Rosenfeld 34503c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl = bd_alloc_handle(&nvme->n_ns[i], 34513c9168faSHans Rosenfeld &nvme_bd_ops, &nvme->n_prp_dma_attr, KM_SLEEP); 34523c9168faSHans Rosenfeld 34533c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl == NULL) { 34543c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 34553c9168faSHans Rosenfeld "!failed to get blkdev handle for namespace %d", i); 34563c9168faSHans Rosenfeld goto fail; 34573c9168faSHans Rosenfeld } 34583c9168faSHans Rosenfeld 34593c9168faSHans Rosenfeld if (bd_attach_handle(dip, nvme->n_ns[i].ns_bd_hdl) 34603c9168faSHans Rosenfeld != DDI_SUCCESS) { 34613c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 34623c9168faSHans Rosenfeld "!failed to attach blkdev handle for namespace %d", 34633c9168faSHans Rosenfeld i); 34643c9168faSHans Rosenfeld goto fail; 34653c9168faSHans Rosenfeld } 34663c9168faSHans Rosenfeld } 34673c9168faSHans Rosenfeld 34683d9b1a2aSHans Rosenfeld if (ddi_create_minor_node(dip, "devctl", S_IFCHR, 34693d9b1a2aSHans Rosenfeld NVME_MINOR(ddi_get_instance(dip), 0), DDI_NT_NVME_NEXUS, 0) 34703d9b1a2aSHans Rosenfeld != DDI_SUCCESS) { 34713d9b1a2aSHans Rosenfeld dev_err(dip, CE_WARN, "nvme_attach: " 34723d9b1a2aSHans Rosenfeld "cannot create devctl minor node"); 34733d9b1a2aSHans Rosenfeld goto fail; 34743d9b1a2aSHans Rosenfeld } 34753d9b1a2aSHans Rosenfeld 34763c9168faSHans Rosenfeld return (DDI_SUCCESS); 34773c9168faSHans Rosenfeld 34783c9168faSHans Rosenfeld fail: 34793c9168faSHans Rosenfeld /* attach successful anyway so that FMA can retire the device */ 34803c9168faSHans Rosenfeld if (nvme->n_dead) 34813c9168faSHans Rosenfeld return (DDI_SUCCESS); 34823c9168faSHans Rosenfeld 34833c9168faSHans Rosenfeld (void) nvme_detach(dip, DDI_DETACH); 34843c9168faSHans Rosenfeld 34853c9168faSHans Rosenfeld return (DDI_FAILURE); 34863c9168faSHans Rosenfeld } 34873c9168faSHans Rosenfeld 34883c9168faSHans Rosenfeld static int 34893c9168faSHans Rosenfeld nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 34903c9168faSHans Rosenfeld { 34913c9168faSHans Rosenfeld int instance, i; 34923c9168faSHans Rosenfeld nvme_t *nvme; 34933c9168faSHans Rosenfeld 34943c9168faSHans Rosenfeld if (cmd != DDI_DETACH) 34953c9168faSHans Rosenfeld return (DDI_FAILURE); 34963c9168faSHans Rosenfeld 34973c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 34983c9168faSHans Rosenfeld 34993c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 35003c9168faSHans Rosenfeld 35013c9168faSHans Rosenfeld if (nvme == NULL) 35023c9168faSHans Rosenfeld return (DDI_FAILURE); 35033c9168faSHans Rosenfeld 35043d9b1a2aSHans Rosenfeld ddi_remove_minor_node(dip, "devctl"); 35053d9b1a2aSHans Rosenfeld mutex_destroy(&nvme->n_minor.nm_mutex); 35063d9b1a2aSHans Rosenfeld 35073c9168faSHans Rosenfeld if (nvme->n_ns) { 35083c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 35093d9b1a2aSHans Rosenfeld ddi_remove_minor_node(dip, nvme->n_ns[i].ns_name); 35103d9b1a2aSHans Rosenfeld mutex_destroy(&nvme->n_ns[i].ns_minor.nm_mutex); 35113d9b1a2aSHans Rosenfeld 35123c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl) { 35133c9168faSHans Rosenfeld (void) bd_detach_handle( 35143c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl); 35153c9168faSHans Rosenfeld bd_free_handle(nvme->n_ns[i].ns_bd_hdl); 35163c9168faSHans Rosenfeld } 35173c9168faSHans Rosenfeld 35183c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_idns) 35193c9168faSHans Rosenfeld kmem_free(nvme->n_ns[i].ns_idns, 35203c9168faSHans Rosenfeld sizeof (nvme_identify_nsid_t)); 352124979ca3SHans Rosenfeld if (nvme->n_ns[i].ns_devid) 352224979ca3SHans Rosenfeld strfree(nvme->n_ns[i].ns_devid); 35233c9168faSHans Rosenfeld } 35243c9168faSHans Rosenfeld 35253c9168faSHans Rosenfeld kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) * 35263c9168faSHans Rosenfeld nvme->n_namespace_count); 35273c9168faSHans Rosenfeld } 3528e89be50aSRob Johnston if (nvme->n_progress & NVME_UFM_INIT) { 3529e89be50aSRob Johnston ddi_ufm_fini(nvme->n_ufmh); 3530e89be50aSRob Johnston mutex_destroy(&nvme->n_fwslot_mutex); 3531e89be50aSRob Johnston } 35323c9168faSHans Rosenfeld 35333c9168faSHans Rosenfeld if (nvme->n_progress & NVME_INTERRUPTS) 35343c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 35353c9168faSHans Rosenfeld 35364d95620bSPaul Winder for (i = 0; i < nvme->n_cq_count; i++) { 35374d95620bSPaul Winder if (nvme->n_cq[i]->ncq_cmd_taskq != NULL) 35384d95620bSPaul Winder taskq_wait(nvme->n_cq[i]->ncq_cmd_taskq); 35394d95620bSPaul Winder } 35403c9168faSHans Rosenfeld 35413c9168faSHans Rosenfeld if (nvme->n_ioq_count > 0) { 35423c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 35433c9168faSHans Rosenfeld if (nvme->n_ioq[i] != NULL) { 35443c9168faSHans Rosenfeld /* TODO: send destroy queue commands */ 35453c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_ioq[i]); 35463c9168faSHans Rosenfeld } 35473c9168faSHans Rosenfeld } 35483c9168faSHans Rosenfeld 35493c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) * 35503c9168faSHans Rosenfeld (nvme->n_ioq_count + 1)); 35513c9168faSHans Rosenfeld } 35523c9168faSHans Rosenfeld 35538834f7acSYouzhong Yang if (nvme->n_prp_cache != NULL) { 35548834f7acSYouzhong Yang kmem_cache_destroy(nvme->n_prp_cache); 35558834f7acSYouzhong Yang } 35568834f7acSYouzhong Yang 35573c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) { 35583c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE); 35593c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 35603c9168faSHans Rosenfeld } 35613c9168faSHans Rosenfeld 35623c9168faSHans Rosenfeld if (nvme->n_progress & NVME_CTRL_LIMITS) 35633c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 35643c9168faSHans Rosenfeld 35653c9168faSHans Rosenfeld if (nvme->n_progress & NVME_ADMIN_QUEUE) 35663c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_adminq); 35673c9168faSHans Rosenfeld 35680999c112SPaul Winder if (nvme->n_cq_count > 0) { 35690999c112SPaul Winder nvme_destroy_cq_array(nvme, 0); 35700999c112SPaul Winder nvme->n_cq = NULL; 35710999c112SPaul Winder nvme->n_cq_count = 0; 35720999c112SPaul Winder } 35730999c112SPaul Winder 35743c9168faSHans Rosenfeld if (nvme->n_idctl) 35753d9b1a2aSHans Rosenfeld kmem_free(nvme->n_idctl, NVME_IDENTIFY_BUFSIZE); 35763c9168faSHans Rosenfeld 35773c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) 35783c9168faSHans Rosenfeld ddi_regs_map_free(&nvme->n_regh); 35793c9168faSHans Rosenfeld 35803c9168faSHans Rosenfeld if (nvme->n_progress & NVME_FMA_INIT) { 35813c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 35823c9168faSHans Rosenfeld ddi_fm_handler_unregister(nvme->n_dip); 35833c9168faSHans Rosenfeld 35843c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 35853c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 35863c9168faSHans Rosenfeld pci_ereport_teardown(nvme->n_dip); 35873c9168faSHans Rosenfeld 35883c9168faSHans Rosenfeld ddi_fm_fini(nvme->n_dip); 35893c9168faSHans Rosenfeld } 35903c9168faSHans Rosenfeld 3591510a6847SHans Rosenfeld if (nvme->n_vendor != NULL) 3592510a6847SHans Rosenfeld strfree(nvme->n_vendor); 3593510a6847SHans Rosenfeld 3594510a6847SHans Rosenfeld if (nvme->n_product != NULL) 3595510a6847SHans Rosenfeld strfree(nvme->n_product); 3596510a6847SHans Rosenfeld 35973c9168faSHans Rosenfeld ddi_soft_state_free(nvme_state, instance); 35983c9168faSHans Rosenfeld 35993c9168faSHans Rosenfeld return (DDI_SUCCESS); 36003c9168faSHans Rosenfeld } 36013c9168faSHans Rosenfeld 36023c9168faSHans Rosenfeld static int 36033c9168faSHans Rosenfeld nvme_quiesce(dev_info_t *dip) 36043c9168faSHans Rosenfeld { 36053c9168faSHans Rosenfeld int instance; 36063c9168faSHans Rosenfeld nvme_t *nvme; 36073c9168faSHans Rosenfeld 36083c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 36093c9168faSHans Rosenfeld 36103c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 36113c9168faSHans Rosenfeld 36123c9168faSHans Rosenfeld if (nvme == NULL) 36133c9168faSHans Rosenfeld return (DDI_FAILURE); 36143c9168faSHans Rosenfeld 36153c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_ABRUPT, B_TRUE); 36163c9168faSHans Rosenfeld 36173c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_TRUE); 36183c9168faSHans Rosenfeld 36193c9168faSHans Rosenfeld return (DDI_FAILURE); 36203c9168faSHans Rosenfeld } 36213c9168faSHans Rosenfeld 36223c9168faSHans Rosenfeld static int 36233c9168faSHans Rosenfeld nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer) 36243c9168faSHans Rosenfeld { 36253c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 36263c9168faSHans Rosenfeld int nprp_page, nprp; 36273c9168faSHans Rosenfeld uint64_t *prp; 36283c9168faSHans Rosenfeld 36293c9168faSHans Rosenfeld if (xfer->x_ndmac == 0) 36303c9168faSHans Rosenfeld return (DDI_FAILURE); 36313c9168faSHans Rosenfeld 36323c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress; 36333c9168faSHans Rosenfeld 36343c9168faSHans Rosenfeld if (xfer->x_ndmac == 1) { 36353c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; 36363c9168faSHans Rosenfeld return (DDI_SUCCESS); 36373c9168faSHans Rosenfeld } else if (xfer->x_ndmac == 2) { 3638*709d065fSPaul Winder ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 36393c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress; 36403c9168faSHans Rosenfeld return (DDI_SUCCESS); 36413c9168faSHans Rosenfeld } 36423c9168faSHans Rosenfeld 36433c9168faSHans Rosenfeld xfer->x_ndmac--; 36443c9168faSHans Rosenfeld 36450999c112SPaul Winder nprp_page = nvme->n_pagesize / sizeof (uint64_t); 36463c9168faSHans Rosenfeld ASSERT(nprp_page > 0); 36473c9168faSHans Rosenfeld nprp = (xfer->x_ndmac + nprp_page - 1) / nprp_page; 36483c9168faSHans Rosenfeld 36493c9168faSHans Rosenfeld /* 36503c9168faSHans Rosenfeld * We currently don't support chained PRPs and set up our DMA 36513c9168faSHans Rosenfeld * attributes to reflect that. If we still get an I/O request 36523c9168faSHans Rosenfeld * that needs a chained PRP something is very wrong. 36533c9168faSHans Rosenfeld */ 36543c9168faSHans Rosenfeld VERIFY(nprp == 1); 36553c9168faSHans Rosenfeld 36568834f7acSYouzhong Yang cmd->nc_dma = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP); 36578834f7acSYouzhong Yang bzero(cmd->nc_dma->nd_memp, cmd->nc_dma->nd_len); 36583c9168faSHans Rosenfeld 36593c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress; 36603c9168faSHans Rosenfeld 36613c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 36623c9168faSHans Rosenfeld for (prp = (uint64_t *)cmd->nc_dma->nd_memp; 36633c9168faSHans Rosenfeld xfer->x_ndmac > 0; 36643c9168faSHans Rosenfeld prp++, xfer->x_ndmac--) { 36653c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 3666*709d065fSPaul Winder *prp = xfer->x_dmac.dmac_laddress; 36673c9168faSHans Rosenfeld } 36683c9168faSHans Rosenfeld 36693c9168faSHans Rosenfeld (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len, 36703c9168faSHans Rosenfeld DDI_DMA_SYNC_FORDEV); 36713c9168faSHans Rosenfeld return (DDI_SUCCESS); 36723c9168faSHans Rosenfeld } 36733c9168faSHans Rosenfeld 36743c9168faSHans Rosenfeld static nvme_cmd_t * 36753c9168faSHans Rosenfeld nvme_create_nvm_cmd(nvme_namespace_t *ns, uint8_t opc, bd_xfer_t *xfer) 36763c9168faSHans Rosenfeld { 36773c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 36783c9168faSHans Rosenfeld nvme_cmd_t *cmd; 36793c9168faSHans Rosenfeld 36803c9168faSHans Rosenfeld /* 36813c9168faSHans Rosenfeld * Blkdev only sets BD_XFER_POLL when dumping, so don't sleep. 36823c9168faSHans Rosenfeld */ 36833c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ? 36843c9168faSHans Rosenfeld KM_NOSLEEP : KM_SLEEP); 36853c9168faSHans Rosenfeld 36863c9168faSHans Rosenfeld if (cmd == NULL) 36873c9168faSHans Rosenfeld return (NULL); 36883c9168faSHans Rosenfeld 36893c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = opc; 36903c9168faSHans Rosenfeld cmd->nc_callback = nvme_bd_xfer_done; 36913c9168faSHans Rosenfeld cmd->nc_xfer = xfer; 36923c9168faSHans Rosenfeld 36933c9168faSHans Rosenfeld switch (opc) { 36943c9168faSHans Rosenfeld case NVME_OPC_NVM_WRITE: 36953c9168faSHans Rosenfeld case NVME_OPC_NVM_READ: 36963c9168faSHans Rosenfeld VERIFY(xfer->x_nblks <= 0x10000); 36973c9168faSHans Rosenfeld 36983c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 36993c9168faSHans Rosenfeld 37003c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu; 37013c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32); 37023c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1); 37033c9168faSHans Rosenfeld 37043c9168faSHans Rosenfeld if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS) 37053c9168faSHans Rosenfeld goto fail; 37063c9168faSHans Rosenfeld break; 37073c9168faSHans Rosenfeld 37083c9168faSHans Rosenfeld case NVME_OPC_NVM_FLUSH: 37093c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 37103c9168faSHans Rosenfeld break; 37113c9168faSHans Rosenfeld 37123c9168faSHans Rosenfeld default: 37133c9168faSHans Rosenfeld goto fail; 37143c9168faSHans Rosenfeld } 37153c9168faSHans Rosenfeld 37163c9168faSHans Rosenfeld return (cmd); 37173c9168faSHans Rosenfeld 37183c9168faSHans Rosenfeld fail: 37193c9168faSHans Rosenfeld nvme_free_cmd(cmd); 37203c9168faSHans Rosenfeld return (NULL); 37213c9168faSHans Rosenfeld } 37223c9168faSHans Rosenfeld 37233c9168faSHans Rosenfeld static void 37243c9168faSHans Rosenfeld nvme_bd_xfer_done(void *arg) 37253c9168faSHans Rosenfeld { 37263c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 37273c9168faSHans Rosenfeld bd_xfer_t *xfer = cmd->nc_xfer; 37283c9168faSHans Rosenfeld int error = 0; 37293c9168faSHans Rosenfeld 37303c9168faSHans Rosenfeld error = nvme_check_cmd_status(cmd); 37313c9168faSHans Rosenfeld nvme_free_cmd(cmd); 37323c9168faSHans Rosenfeld 37333c9168faSHans Rosenfeld bd_xfer_done(xfer, error); 37343c9168faSHans Rosenfeld } 37353c9168faSHans Rosenfeld 37363c9168faSHans Rosenfeld static void 37373c9168faSHans Rosenfeld nvme_bd_driveinfo(void *arg, bd_drive_t *drive) 37383c9168faSHans Rosenfeld { 37393c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 37403c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 37414d95620bSPaul Winder uint_t ns_count = MAX(1, nvme->n_namespaces_attachable); 37424d95620bSPaul Winder 37434d95620bSPaul Winder /* 37444d95620bSPaul Winder * Set the blkdev qcount to the number of submission queues. 37454d95620bSPaul Winder * It will then create one waitq/runq pair for each submission 37464d95620bSPaul Winder * queue and spread I/O requests across the queues. 37474d95620bSPaul Winder */ 37484d95620bSPaul Winder drive->d_qcount = nvme->n_ioq_count; 37494d95620bSPaul Winder 37504d95620bSPaul Winder /* 37514d95620bSPaul Winder * I/O activity to individual namespaces is distributed across 37524d95620bSPaul Winder * each of the d_qcount blkdev queues (which has been set to 37534d95620bSPaul Winder * the number of nvme submission queues). d_qsize is the number 37544d95620bSPaul Winder * of submitted and not completed I/Os within each queue that blkdev 37554d95620bSPaul Winder * will allow before it starts holding them in the waitq. 37564d95620bSPaul Winder * 37574d95620bSPaul Winder * Each namespace will create a child blkdev instance, for each one 37584d95620bSPaul Winder * we try and set the d_qsize so that each namespace gets an 37594d95620bSPaul Winder * equal portion of the submission queue. 37604d95620bSPaul Winder * 37614d95620bSPaul Winder * If post instantiation of the nvme drive, n_namespaces_attachable 37624d95620bSPaul Winder * changes and a namespace is attached it could calculate a 37634d95620bSPaul Winder * different d_qsize. It may even be that the sum of the d_qsizes is 37644d95620bSPaul Winder * now beyond the submission queue size. Should that be the case 37654d95620bSPaul Winder * and the I/O rate is such that blkdev attempts to submit more 37664d95620bSPaul Winder * I/Os than the size of the submission queue, the excess I/Os 37674d95620bSPaul Winder * will be held behind the semaphore nq_sema. 37684d95620bSPaul Winder */ 37694d95620bSPaul Winder drive->d_qsize = nvme->n_io_squeue_len / ns_count; 37703c9168faSHans Rosenfeld 37713c9168faSHans Rosenfeld /* 37724d95620bSPaul Winder * Don't let the queue size drop below the minimum, though. 37733c9168faSHans Rosenfeld */ 37744d95620bSPaul Winder drive->d_qsize = MAX(drive->d_qsize, NVME_MIN_IO_QUEUE_LEN); 37753c9168faSHans Rosenfeld 37763c9168faSHans Rosenfeld /* 37773c9168faSHans Rosenfeld * d_maxxfer is not set, which means the value is taken from the DMA 37783c9168faSHans Rosenfeld * attributes specified to bd_alloc_handle. 37793c9168faSHans Rosenfeld */ 37803c9168faSHans Rosenfeld 37813c9168faSHans Rosenfeld drive->d_removable = B_FALSE; 37823c9168faSHans Rosenfeld drive->d_hotpluggable = B_FALSE; 37833c9168faSHans Rosenfeld 378424979ca3SHans Rosenfeld bcopy(ns->ns_eui64, drive->d_eui64, sizeof (drive->d_eui64)); 37853c9168faSHans Rosenfeld drive->d_target = ns->ns_id; 37863c9168faSHans Rosenfeld drive->d_lun = 0; 3787510a6847SHans Rosenfeld 3788bef9e21aSHans Rosenfeld drive->d_model = nvme->n_idctl->id_model; 3789bef9e21aSHans Rosenfeld drive->d_model_len = sizeof (nvme->n_idctl->id_model); 3790510a6847SHans Rosenfeld drive->d_vendor = nvme->n_vendor; 3791510a6847SHans Rosenfeld drive->d_vendor_len = strlen(nvme->n_vendor); 3792510a6847SHans Rosenfeld drive->d_product = nvme->n_product; 3793510a6847SHans Rosenfeld drive->d_product_len = strlen(nvme->n_product); 3794510a6847SHans Rosenfeld drive->d_serial = nvme->n_idctl->id_serial; 3795510a6847SHans Rosenfeld drive->d_serial_len = sizeof (nvme->n_idctl->id_serial); 3796510a6847SHans Rosenfeld drive->d_revision = nvme->n_idctl->id_fwrev; 3797510a6847SHans Rosenfeld drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev); 37983c9168faSHans Rosenfeld } 37993c9168faSHans Rosenfeld 38003c9168faSHans Rosenfeld static int 38013c9168faSHans Rosenfeld nvme_bd_mediainfo(void *arg, bd_media_t *media) 38023c9168faSHans Rosenfeld { 38033c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 38043c9168faSHans Rosenfeld 38053c9168faSHans Rosenfeld media->m_nblks = ns->ns_block_count; 38063c9168faSHans Rosenfeld media->m_blksize = ns->ns_block_size; 38073c9168faSHans Rosenfeld media->m_readonly = B_FALSE; 38083c9168faSHans Rosenfeld media->m_solidstate = B_TRUE; 38093c9168faSHans Rosenfeld 38103c9168faSHans Rosenfeld media->m_pblksize = ns->ns_best_block_size; 38113c9168faSHans Rosenfeld 38123c9168faSHans Rosenfeld return (0); 38133c9168faSHans Rosenfeld } 38143c9168faSHans Rosenfeld 38153c9168faSHans Rosenfeld static int 38163c9168faSHans Rosenfeld nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc) 38173c9168faSHans Rosenfeld { 38183c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 38194b324362SHans Rosenfeld nvme_cmd_t *cmd; 38204ac9cfccSHans Rosenfeld nvme_qpair_t *ioq; 38214ac9cfccSHans Rosenfeld boolean_t poll; 38224b324362SHans Rosenfeld int ret; 38233c9168faSHans Rosenfeld 38243c9168faSHans Rosenfeld if (nvme->n_dead) 38253c9168faSHans Rosenfeld return (EIO); 38263c9168faSHans Rosenfeld 38273c9168faSHans Rosenfeld cmd = nvme_create_nvm_cmd(ns, opc, xfer); 38283c9168faSHans Rosenfeld if (cmd == NULL) 38293c9168faSHans Rosenfeld return (ENOMEM); 38303c9168faSHans Rosenfeld 38314d95620bSPaul Winder cmd->nc_sqid = xfer->x_qnum + 1; 38323c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); 38334ac9cfccSHans Rosenfeld ioq = nvme->n_ioq[cmd->nc_sqid]; 38344ac9cfccSHans Rosenfeld 38354ac9cfccSHans Rosenfeld /* 38364ac9cfccSHans Rosenfeld * Get the polling flag before submitting the command. The command may 38374ac9cfccSHans Rosenfeld * complete immediately after it was submitted, which means we must 38384ac9cfccSHans Rosenfeld * treat both cmd and xfer as if they have been freed already. 38394ac9cfccSHans Rosenfeld */ 38404ac9cfccSHans Rosenfeld poll = (xfer->x_flags & BD_XFER_POLL) != 0; 38413c9168faSHans Rosenfeld 38424b324362SHans Rosenfeld ret = nvme_submit_io_cmd(ioq, cmd); 38434b324362SHans Rosenfeld 38444b324362SHans Rosenfeld if (ret != 0) 38454b324362SHans Rosenfeld return (ret); 38463c9168faSHans Rosenfeld 38474ac9cfccSHans Rosenfeld if (!poll) 38484ac9cfccSHans Rosenfeld return (0); 38494ac9cfccSHans Rosenfeld 38504ac9cfccSHans Rosenfeld do { 38514b324362SHans Rosenfeld cmd = nvme_retrieve_cmd(nvme, ioq); 38524b324362SHans Rosenfeld if (cmd != NULL) 38530999c112SPaul Winder cmd->nc_callback(cmd); 38544ac9cfccSHans Rosenfeld else 38554ac9cfccSHans Rosenfeld drv_usecwait(10); 38564ac9cfccSHans Rosenfeld } while (ioq->nq_active_cmds != 0); 38574ac9cfccSHans Rosenfeld 38583c9168faSHans Rosenfeld return (0); 38593c9168faSHans Rosenfeld } 38603c9168faSHans Rosenfeld 38613c9168faSHans Rosenfeld static int 38623c9168faSHans Rosenfeld nvme_bd_read(void *arg, bd_xfer_t *xfer) 38633c9168faSHans Rosenfeld { 38643c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 38653c9168faSHans Rosenfeld 38663c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_READ)); 38673c9168faSHans Rosenfeld } 38683c9168faSHans Rosenfeld 38693c9168faSHans Rosenfeld static int 38703c9168faSHans Rosenfeld nvme_bd_write(void *arg, bd_xfer_t *xfer) 38713c9168faSHans Rosenfeld { 38723c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 38733c9168faSHans Rosenfeld 38743c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_WRITE)); 38753c9168faSHans Rosenfeld } 38763c9168faSHans Rosenfeld 38773c9168faSHans Rosenfeld static int 38783c9168faSHans Rosenfeld nvme_bd_sync(void *arg, bd_xfer_t *xfer) 38793c9168faSHans Rosenfeld { 38803c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 38813c9168faSHans Rosenfeld 38823c9168faSHans Rosenfeld if (ns->ns_nvme->n_dead) 38833c9168faSHans Rosenfeld return (EIO); 38843c9168faSHans Rosenfeld 38853c9168faSHans Rosenfeld /* 3886d148d46eSHans Rosenfeld * If the volatile write cache is not present or not enabled the FLUSH 3887d148d46eSHans Rosenfeld * command is a no-op, so we can take a shortcut here. 38883c9168faSHans Rosenfeld */ 3889d148d46eSHans Rosenfeld if (!ns->ns_nvme->n_write_cache_present) { 38903c9168faSHans Rosenfeld bd_xfer_done(xfer, ENOTSUP); 38913c9168faSHans Rosenfeld return (0); 38923c9168faSHans Rosenfeld } 38933c9168faSHans Rosenfeld 3894d148d46eSHans Rosenfeld if (!ns->ns_nvme->n_write_cache_enabled) { 3895d148d46eSHans Rosenfeld bd_xfer_done(xfer, 0); 3896d148d46eSHans Rosenfeld return (0); 3897d148d46eSHans Rosenfeld } 3898d148d46eSHans Rosenfeld 38993c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_FLUSH)); 39003c9168faSHans Rosenfeld } 39013c9168faSHans Rosenfeld 39023c9168faSHans Rosenfeld static int 39033c9168faSHans Rosenfeld nvme_bd_devid(void *arg, dev_info_t *devinfo, ddi_devid_t *devid) 39043c9168faSHans Rosenfeld { 39053c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 39063c9168faSHans Rosenfeld 390724979ca3SHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 390824979ca3SHans Rosenfeld if (*(uint64_t *)ns->ns_eui64 != 0) { 390924979ca3SHans Rosenfeld return (ddi_devid_init(devinfo, DEVID_SCSI3_WWN, 391024979ca3SHans Rosenfeld sizeof (ns->ns_eui64), ns->ns_eui64, devid)); 391124979ca3SHans Rosenfeld } else { 391224979ca3SHans Rosenfeld return (ddi_devid_init(devinfo, DEVID_ENCAP, 391324979ca3SHans Rosenfeld strlen(ns->ns_devid), ns->ns_devid, devid)); 391424979ca3SHans Rosenfeld } 39153c9168faSHans Rosenfeld } 39163d9b1a2aSHans Rosenfeld 39173d9b1a2aSHans Rosenfeld static int 39183d9b1a2aSHans Rosenfeld nvme_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 39193d9b1a2aSHans Rosenfeld { 39203d9b1a2aSHans Rosenfeld #ifndef __lock_lint 39213d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 39223d9b1a2aSHans Rosenfeld #endif 39233d9b1a2aSHans Rosenfeld minor_t minor = getminor(*devp); 39243d9b1a2aSHans Rosenfeld nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 39253d9b1a2aSHans Rosenfeld int nsid = NVME_MINOR_NSID(minor); 39263d9b1a2aSHans Rosenfeld nvme_minor_state_t *nm; 39273d9b1a2aSHans Rosenfeld int rv = 0; 39283d9b1a2aSHans Rosenfeld 39293d9b1a2aSHans Rosenfeld if (otyp != OTYP_CHR) 39303d9b1a2aSHans Rosenfeld return (EINVAL); 39313d9b1a2aSHans Rosenfeld 39323d9b1a2aSHans Rosenfeld if (nvme == NULL) 39333d9b1a2aSHans Rosenfeld return (ENXIO); 39343d9b1a2aSHans Rosenfeld 39353d9b1a2aSHans Rosenfeld if (nsid > nvme->n_namespace_count) 39363d9b1a2aSHans Rosenfeld return (ENXIO); 39373d9b1a2aSHans Rosenfeld 3938e984c70bSHans Rosenfeld if (nvme->n_dead) 3939e984c70bSHans Rosenfeld return (EIO); 3940e984c70bSHans Rosenfeld 39413d9b1a2aSHans Rosenfeld nm = nsid == 0 ? &nvme->n_minor : &nvme->n_ns[nsid - 1].ns_minor; 39423d9b1a2aSHans Rosenfeld 39433d9b1a2aSHans Rosenfeld mutex_enter(&nm->nm_mutex); 39443d9b1a2aSHans Rosenfeld if (nm->nm_oexcl) { 39453d9b1a2aSHans Rosenfeld rv = EBUSY; 39463d9b1a2aSHans Rosenfeld goto out; 39473d9b1a2aSHans Rosenfeld } 39483d9b1a2aSHans Rosenfeld 39493d9b1a2aSHans Rosenfeld if (flag & FEXCL) { 39503d9b1a2aSHans Rosenfeld if (nm->nm_ocnt != 0) { 39513d9b1a2aSHans Rosenfeld rv = EBUSY; 39523d9b1a2aSHans Rosenfeld goto out; 39533d9b1a2aSHans Rosenfeld } 39543d9b1a2aSHans Rosenfeld nm->nm_oexcl = B_TRUE; 39553d9b1a2aSHans Rosenfeld } 39563d9b1a2aSHans Rosenfeld 39573d9b1a2aSHans Rosenfeld nm->nm_ocnt++; 39583d9b1a2aSHans Rosenfeld 39593d9b1a2aSHans Rosenfeld out: 39603d9b1a2aSHans Rosenfeld mutex_exit(&nm->nm_mutex); 39613d9b1a2aSHans Rosenfeld return (rv); 39623d9b1a2aSHans Rosenfeld 39633d9b1a2aSHans Rosenfeld } 39643d9b1a2aSHans Rosenfeld 39653d9b1a2aSHans Rosenfeld static int 39663d9b1a2aSHans Rosenfeld nvme_close(dev_t dev, int flag, int otyp, cred_t *cred_p) 39673d9b1a2aSHans Rosenfeld { 39683d9b1a2aSHans Rosenfeld #ifndef __lock_lint 39693d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 39703d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(flag)); 39713d9b1a2aSHans Rosenfeld #endif 39723d9b1a2aSHans Rosenfeld minor_t minor = getminor(dev); 39733d9b1a2aSHans Rosenfeld nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 39743d9b1a2aSHans Rosenfeld int nsid = NVME_MINOR_NSID(minor); 39753d9b1a2aSHans Rosenfeld nvme_minor_state_t *nm; 39763d9b1a2aSHans Rosenfeld 39773d9b1a2aSHans Rosenfeld if (otyp != OTYP_CHR) 39783d9b1a2aSHans Rosenfeld return (ENXIO); 39793d9b1a2aSHans Rosenfeld 39803d9b1a2aSHans Rosenfeld if (nvme == NULL) 39813d9b1a2aSHans Rosenfeld return (ENXIO); 39823d9b1a2aSHans Rosenfeld 39833d9b1a2aSHans Rosenfeld if (nsid > nvme->n_namespace_count) 39843d9b1a2aSHans Rosenfeld return (ENXIO); 39853d9b1a2aSHans Rosenfeld 39863d9b1a2aSHans Rosenfeld nm = nsid == 0 ? &nvme->n_minor : &nvme->n_ns[nsid - 1].ns_minor; 39873d9b1a2aSHans Rosenfeld 39883d9b1a2aSHans Rosenfeld mutex_enter(&nm->nm_mutex); 39893d9b1a2aSHans Rosenfeld if (nm->nm_oexcl) 39903d9b1a2aSHans Rosenfeld nm->nm_oexcl = B_FALSE; 39913d9b1a2aSHans Rosenfeld 39923d9b1a2aSHans Rosenfeld ASSERT(nm->nm_ocnt > 0); 39933d9b1a2aSHans Rosenfeld nm->nm_ocnt--; 39943d9b1a2aSHans Rosenfeld mutex_exit(&nm->nm_mutex); 39953d9b1a2aSHans Rosenfeld 39963d9b1a2aSHans Rosenfeld return (0); 39973d9b1a2aSHans Rosenfeld } 39983d9b1a2aSHans Rosenfeld 39993d9b1a2aSHans Rosenfeld static int 40003d9b1a2aSHans Rosenfeld nvme_ioctl_identify(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 40013d9b1a2aSHans Rosenfeld cred_t *cred_p) 40023d9b1a2aSHans Rosenfeld { 40033d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 40043d9b1a2aSHans Rosenfeld int rv = 0; 40053d9b1a2aSHans Rosenfeld void *idctl; 40063d9b1a2aSHans Rosenfeld 40073d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 40083d9b1a2aSHans Rosenfeld return (EPERM); 40093d9b1a2aSHans Rosenfeld 40103d9b1a2aSHans Rosenfeld if (nioc->n_len < NVME_IDENTIFY_BUFSIZE) 40113d9b1a2aSHans Rosenfeld return (EINVAL); 40123d9b1a2aSHans Rosenfeld 4013bc586359SRobert Mustacchi if ((rv = nvme_identify(nvme, B_TRUE, nsid, (void **)&idctl)) != 0) 4014e984c70bSHans Rosenfeld return (rv); 40153d9b1a2aSHans Rosenfeld 40163d9b1a2aSHans Rosenfeld if (ddi_copyout(idctl, (void *)nioc->n_buf, NVME_IDENTIFY_BUFSIZE, mode) 40173d9b1a2aSHans Rosenfeld != 0) 40183d9b1a2aSHans Rosenfeld rv = EFAULT; 40193d9b1a2aSHans Rosenfeld 40203d9b1a2aSHans Rosenfeld kmem_free(idctl, NVME_IDENTIFY_BUFSIZE); 40213d9b1a2aSHans Rosenfeld 40223d9b1a2aSHans Rosenfeld return (rv); 40233d9b1a2aSHans Rosenfeld } 40243d9b1a2aSHans Rosenfeld 4025cf840871SPaul Winder /* 4026cf840871SPaul Winder * Execute commands on behalf of the various ioctls. 4027cf840871SPaul Winder */ 4028cf840871SPaul Winder static int 4029cf840871SPaul Winder nvme_ioc_cmd(nvme_t *nvme, nvme_sqe_t *sqe, boolean_t is_admin, void *data_addr, 4030cf840871SPaul Winder uint32_t data_len, int rwk, nvme_cqe_t *cqe, uint_t timeout) 4031cf840871SPaul Winder { 4032cf840871SPaul Winder nvme_cmd_t *cmd; 4033cf840871SPaul Winder nvme_qpair_t *ioq; 4034cf840871SPaul Winder int rv = 0; 4035cf840871SPaul Winder 4036cf840871SPaul Winder cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 4037cf840871SPaul Winder if (is_admin) { 4038cf840871SPaul Winder cmd->nc_sqid = 0; 4039cf840871SPaul Winder ioq = nvme->n_adminq; 4040cf840871SPaul Winder } else { 4041cf840871SPaul Winder cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1; 4042cf840871SPaul Winder ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); 4043cf840871SPaul Winder ioq = nvme->n_ioq[cmd->nc_sqid]; 4044cf840871SPaul Winder } 4045cf840871SPaul Winder 4046cf840871SPaul Winder cmd->nc_callback = nvme_wakeup_cmd; 4047cf840871SPaul Winder cmd->nc_sqe = *sqe; 4048cf840871SPaul Winder 4049cf840871SPaul Winder if ((rwk & (FREAD | FWRITE)) != 0) { 4050cf840871SPaul Winder if (data_addr == NULL) { 4051cf840871SPaul Winder rv = EINVAL; 4052cf840871SPaul Winder goto free_cmd; 4053cf840871SPaul Winder } 4054cf840871SPaul Winder 4055cf840871SPaul Winder /* 4056cf840871SPaul Winder * Because we use PRPs and haven't implemented PRP 4057cf840871SPaul Winder * lists here, the maximum data size is restricted to 4058cf840871SPaul Winder * 2 pages. 4059cf840871SPaul Winder */ 4060cf840871SPaul Winder if (data_len > 2 * nvme->n_pagesize) { 4061cf840871SPaul Winder dev_err(nvme->n_dip, CE_WARN, "!Data size %u is too " 4062cf840871SPaul Winder "large for nvme_ioc_cmd(). Limit is 2 pages " 4063cf840871SPaul Winder "(%u bytes)", data_len, 2 * nvme->n_pagesize); 4064cf840871SPaul Winder 4065cf840871SPaul Winder rv = EINVAL; 4066cf840871SPaul Winder goto free_cmd; 4067cf840871SPaul Winder } 4068cf840871SPaul Winder 4069cf840871SPaul Winder if (nvme_zalloc_dma(nvme, data_len, DDI_DMA_READ, 4070cf840871SPaul Winder &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 4071cf840871SPaul Winder dev_err(nvme->n_dip, CE_WARN, 4072cf840871SPaul Winder "!nvme_zalloc_dma failed for nvme_ioc_cmd()"); 4073cf840871SPaul Winder 4074cf840871SPaul Winder rv = ENOMEM; 4075cf840871SPaul Winder goto free_cmd; 4076cf840871SPaul Winder } 4077cf840871SPaul Winder 4078cf840871SPaul Winder if (cmd->nc_dma->nd_ncookie > 2) { 4079cf840871SPaul Winder dev_err(nvme->n_dip, CE_WARN, 4080cf840871SPaul Winder "!too many DMA cookies for nvme_ioc_cmd()"); 4081cf840871SPaul Winder atomic_inc_32(&nvme->n_too_many_cookies); 4082cf840871SPaul Winder 4083cf840871SPaul Winder rv = E2BIG; 4084cf840871SPaul Winder goto free_cmd; 4085cf840871SPaul Winder } 4086cf840871SPaul Winder 4087cf840871SPaul Winder cmd->nc_sqe.sqe_dptr.d_prp[0] = 4088cf840871SPaul Winder cmd->nc_dma->nd_cookie.dmac_laddress; 4089cf840871SPaul Winder 4090cf840871SPaul Winder if (cmd->nc_dma->nd_ncookie > 1) { 4091cf840871SPaul Winder ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 4092cf840871SPaul Winder &cmd->nc_dma->nd_cookie); 4093cf840871SPaul Winder cmd->nc_sqe.sqe_dptr.d_prp[1] = 4094cf840871SPaul Winder cmd->nc_dma->nd_cookie.dmac_laddress; 4095cf840871SPaul Winder } 4096cf840871SPaul Winder 4097cf840871SPaul Winder if ((rwk & FWRITE) != 0) { 4098cf840871SPaul Winder if (ddi_copyin(data_addr, cmd->nc_dma->nd_memp, 4099cf840871SPaul Winder data_len, rwk & FKIOCTL) != 0) { 4100cf840871SPaul Winder rv = EFAULT; 4101cf840871SPaul Winder goto free_cmd; 4102cf840871SPaul Winder } 4103cf840871SPaul Winder } 4104cf840871SPaul Winder } 4105cf840871SPaul Winder 4106cf840871SPaul Winder if (is_admin) { 4107cf840871SPaul Winder nvme_admin_cmd(cmd, timeout); 4108cf840871SPaul Winder } else { 4109cf840871SPaul Winder mutex_enter(&cmd->nc_mutex); 4110cf840871SPaul Winder 4111cf840871SPaul Winder rv = nvme_submit_io_cmd(ioq, cmd); 4112cf840871SPaul Winder 4113cf840871SPaul Winder if (rv == EAGAIN) { 4114cf840871SPaul Winder mutex_exit(&cmd->nc_mutex); 4115cf840871SPaul Winder dev_err(cmd->nc_nvme->n_dip, CE_WARN, 4116cf840871SPaul Winder "!nvme_ioc_cmd() failed, I/O Q full"); 4117cf840871SPaul Winder goto free_cmd; 4118cf840871SPaul Winder } 4119cf840871SPaul Winder 4120cf840871SPaul Winder nvme_wait_cmd(cmd, timeout); 4121cf840871SPaul Winder 4122cf840871SPaul Winder mutex_exit(&cmd->nc_mutex); 4123cf840871SPaul Winder } 4124cf840871SPaul Winder 4125cf840871SPaul Winder if (cqe != NULL) 4126cf840871SPaul Winder *cqe = cmd->nc_cqe; 4127cf840871SPaul Winder 4128cf840871SPaul Winder if ((rv = nvme_check_cmd_status(cmd)) != 0) { 4129cf840871SPaul Winder dev_err(nvme->n_dip, CE_WARN, 4130cf840871SPaul Winder "!nvme_ioc_cmd() failed with sct = %x, sc = %x", 4131cf840871SPaul Winder cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 4132cf840871SPaul Winder 4133cf840871SPaul Winder goto free_cmd; 4134cf840871SPaul Winder } 4135cf840871SPaul Winder 4136cf840871SPaul Winder if ((rwk & FREAD) != 0) { 4137cf840871SPaul Winder if (ddi_copyout(cmd->nc_dma->nd_memp, 4138cf840871SPaul Winder data_addr, data_len, rwk & FKIOCTL) != 0) 4139cf840871SPaul Winder rv = EFAULT; 4140cf840871SPaul Winder } 4141cf840871SPaul Winder 4142cf840871SPaul Winder free_cmd: 4143cf840871SPaul Winder nvme_free_cmd(cmd); 4144cf840871SPaul Winder 4145cf840871SPaul Winder return (rv); 4146cf840871SPaul Winder } 4147cf840871SPaul Winder 41483d9b1a2aSHans Rosenfeld static int 41493d9b1a2aSHans Rosenfeld nvme_ioctl_capabilities(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 41503d9b1a2aSHans Rosenfeld int mode, cred_t *cred_p) 41513d9b1a2aSHans Rosenfeld { 41523d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nsid, cred_p)); 41533d9b1a2aSHans Rosenfeld int rv = 0; 41543d9b1a2aSHans Rosenfeld nvme_reg_cap_t cap = { 0 }; 41553d9b1a2aSHans Rosenfeld nvme_capabilities_t nc; 41563d9b1a2aSHans Rosenfeld 41573d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 41583d9b1a2aSHans Rosenfeld return (EPERM); 41593d9b1a2aSHans Rosenfeld 41603d9b1a2aSHans Rosenfeld if (nioc->n_len < sizeof (nc)) 41613d9b1a2aSHans Rosenfeld return (EINVAL); 41623d9b1a2aSHans Rosenfeld 41633d9b1a2aSHans Rosenfeld cap.r = nvme_get64(nvme, NVME_REG_CAP); 41643d9b1a2aSHans Rosenfeld 41653d9b1a2aSHans Rosenfeld /* 41663d9b1a2aSHans Rosenfeld * The MPSMIN and MPSMAX fields in the CAP register use 0 to 41673d9b1a2aSHans Rosenfeld * specify the base page size of 4k (1<<12), so add 12 here to 41683d9b1a2aSHans Rosenfeld * get the real page size value. 41693d9b1a2aSHans Rosenfeld */ 41703d9b1a2aSHans Rosenfeld nc.mpsmax = 1 << (12 + cap.b.cap_mpsmax); 41713d9b1a2aSHans Rosenfeld nc.mpsmin = 1 << (12 + cap.b.cap_mpsmin); 41723d9b1a2aSHans Rosenfeld 41733d9b1a2aSHans Rosenfeld if (ddi_copyout(&nc, (void *)nioc->n_buf, sizeof (nc), mode) != 0) 41743d9b1a2aSHans Rosenfeld rv = EFAULT; 41753d9b1a2aSHans Rosenfeld 41763d9b1a2aSHans Rosenfeld return (rv); 41773d9b1a2aSHans Rosenfeld } 41783d9b1a2aSHans Rosenfeld 41793d9b1a2aSHans Rosenfeld static int 41803d9b1a2aSHans Rosenfeld nvme_ioctl_get_logpage(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 41813d9b1a2aSHans Rosenfeld int mode, cred_t *cred_p) 41823d9b1a2aSHans Rosenfeld { 41833d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 41843d9b1a2aSHans Rosenfeld void *log = NULL; 41853d9b1a2aSHans Rosenfeld size_t bufsize = 0; 41863d9b1a2aSHans Rosenfeld int rv = 0; 41873d9b1a2aSHans Rosenfeld 41883d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 41893d9b1a2aSHans Rosenfeld return (EPERM); 41903d9b1a2aSHans Rosenfeld 41913d9b1a2aSHans Rosenfeld switch (nioc->n_arg) { 41923d9b1a2aSHans Rosenfeld case NVME_LOGPAGE_ERROR: 41933d9b1a2aSHans Rosenfeld if (nsid != 0) 41943d9b1a2aSHans Rosenfeld return (EINVAL); 41953d9b1a2aSHans Rosenfeld break; 41963d9b1a2aSHans Rosenfeld case NVME_LOGPAGE_HEALTH: 41973d9b1a2aSHans Rosenfeld if (nsid != 0 && nvme->n_idctl->id_lpa.lp_smart == 0) 41983d9b1a2aSHans Rosenfeld return (EINVAL); 41993d9b1a2aSHans Rosenfeld 42003d9b1a2aSHans Rosenfeld if (nsid == 0) 42013d9b1a2aSHans Rosenfeld nsid = (uint32_t)-1; 42023d9b1a2aSHans Rosenfeld 42033d9b1a2aSHans Rosenfeld break; 42043d9b1a2aSHans Rosenfeld case NVME_LOGPAGE_FWSLOT: 42053d9b1a2aSHans Rosenfeld if (nsid != 0) 42063d9b1a2aSHans Rosenfeld return (EINVAL); 42073d9b1a2aSHans Rosenfeld break; 42083d9b1a2aSHans Rosenfeld default: 42093d9b1a2aSHans Rosenfeld return (EINVAL); 42103d9b1a2aSHans Rosenfeld } 42113d9b1a2aSHans Rosenfeld 4212bc586359SRobert Mustacchi if (nvme_get_logpage(nvme, B_TRUE, &log, &bufsize, nioc->n_arg, nsid) 42133d9b1a2aSHans Rosenfeld != DDI_SUCCESS) 42143d9b1a2aSHans Rosenfeld return (EIO); 42153d9b1a2aSHans Rosenfeld 42163d9b1a2aSHans Rosenfeld if (nioc->n_len < bufsize) { 42173d9b1a2aSHans Rosenfeld kmem_free(log, bufsize); 42183d9b1a2aSHans Rosenfeld return (EINVAL); 42193d9b1a2aSHans Rosenfeld } 42203d9b1a2aSHans Rosenfeld 42213d9b1a2aSHans Rosenfeld if (ddi_copyout(log, (void *)nioc->n_buf, bufsize, mode) != 0) 42223d9b1a2aSHans Rosenfeld rv = EFAULT; 42233d9b1a2aSHans Rosenfeld 42243d9b1a2aSHans Rosenfeld nioc->n_len = bufsize; 42253d9b1a2aSHans Rosenfeld kmem_free(log, bufsize); 42263d9b1a2aSHans Rosenfeld 42273d9b1a2aSHans Rosenfeld return (rv); 42283d9b1a2aSHans Rosenfeld } 42293d9b1a2aSHans Rosenfeld 42303d9b1a2aSHans Rosenfeld static int 42313d9b1a2aSHans Rosenfeld nvme_ioctl_get_features(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 42323d9b1a2aSHans Rosenfeld int mode, cred_t *cred_p) 42333d9b1a2aSHans Rosenfeld { 42343d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 42353d9b1a2aSHans Rosenfeld void *buf = NULL; 42363d9b1a2aSHans Rosenfeld size_t bufsize = 0; 42373d9b1a2aSHans Rosenfeld uint32_t res = 0; 42383d9b1a2aSHans Rosenfeld uint8_t feature; 42393d9b1a2aSHans Rosenfeld int rv = 0; 42403d9b1a2aSHans Rosenfeld 42413d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 42423d9b1a2aSHans Rosenfeld return (EPERM); 42433d9b1a2aSHans Rosenfeld 42443d9b1a2aSHans Rosenfeld if ((nioc->n_arg >> 32) > 0xff) 42453d9b1a2aSHans Rosenfeld return (EINVAL); 42463d9b1a2aSHans Rosenfeld 42473d9b1a2aSHans Rosenfeld feature = (uint8_t)(nioc->n_arg >> 32); 42483d9b1a2aSHans Rosenfeld 42493d9b1a2aSHans Rosenfeld switch (feature) { 42503d9b1a2aSHans Rosenfeld case NVME_FEAT_ARBITRATION: 42513d9b1a2aSHans Rosenfeld case NVME_FEAT_POWER_MGMT: 42523d9b1a2aSHans Rosenfeld case NVME_FEAT_TEMPERATURE: 42533d9b1a2aSHans Rosenfeld case NVME_FEAT_ERROR: 42543d9b1a2aSHans Rosenfeld case NVME_FEAT_NQUEUES: 42553d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_COAL: 42563d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_ATOM: 42573d9b1a2aSHans Rosenfeld case NVME_FEAT_ASYNC_EVENT: 42583d9b1a2aSHans Rosenfeld case NVME_FEAT_PROGRESS: 42593d9b1a2aSHans Rosenfeld if (nsid != 0) 42603d9b1a2aSHans Rosenfeld return (EINVAL); 42613d9b1a2aSHans Rosenfeld break; 42623d9b1a2aSHans Rosenfeld 42633d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_VECT: 42643d9b1a2aSHans Rosenfeld if (nsid != 0) 42653d9b1a2aSHans Rosenfeld return (EINVAL); 42663d9b1a2aSHans Rosenfeld 42673d9b1a2aSHans Rosenfeld res = nioc->n_arg & 0xffffffffUL; 42683d9b1a2aSHans Rosenfeld if (res >= nvme->n_intr_cnt) 42693d9b1a2aSHans Rosenfeld return (EINVAL); 42703d9b1a2aSHans Rosenfeld break; 42713d9b1a2aSHans Rosenfeld 42723d9b1a2aSHans Rosenfeld case NVME_FEAT_LBA_RANGE: 42733d9b1a2aSHans Rosenfeld if (nvme->n_lba_range_supported == B_FALSE) 42743d9b1a2aSHans Rosenfeld return (EINVAL); 42753d9b1a2aSHans Rosenfeld 42763d9b1a2aSHans Rosenfeld if (nsid == 0 || 42773d9b1a2aSHans Rosenfeld nsid > nvme->n_namespace_count) 42783d9b1a2aSHans Rosenfeld return (EINVAL); 42793d9b1a2aSHans Rosenfeld 42803d9b1a2aSHans Rosenfeld break; 42813d9b1a2aSHans Rosenfeld 42823d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_CACHE: 42833d9b1a2aSHans Rosenfeld if (nsid != 0) 42843d9b1a2aSHans Rosenfeld return (EINVAL); 42853d9b1a2aSHans Rosenfeld 42863d9b1a2aSHans Rosenfeld if (!nvme->n_write_cache_present) 42873d9b1a2aSHans Rosenfeld return (EINVAL); 42883d9b1a2aSHans Rosenfeld 42893d9b1a2aSHans Rosenfeld break; 42903d9b1a2aSHans Rosenfeld 42913d9b1a2aSHans Rosenfeld case NVME_FEAT_AUTO_PST: 42923d9b1a2aSHans Rosenfeld if (nsid != 0) 42933d9b1a2aSHans Rosenfeld return (EINVAL); 42943d9b1a2aSHans Rosenfeld 42953d9b1a2aSHans Rosenfeld if (!nvme->n_auto_pst_supported) 42963d9b1a2aSHans Rosenfeld return (EINVAL); 42973d9b1a2aSHans Rosenfeld 42983d9b1a2aSHans Rosenfeld break; 42993d9b1a2aSHans Rosenfeld 43003d9b1a2aSHans Rosenfeld default: 43013d9b1a2aSHans Rosenfeld return (EINVAL); 43023d9b1a2aSHans Rosenfeld } 43033d9b1a2aSHans Rosenfeld 4304bc586359SRobert Mustacchi rv = nvme_get_features(nvme, B_TRUE, nsid, feature, &res, &buf, 4305bc586359SRobert Mustacchi &bufsize); 4306e984c70bSHans Rosenfeld if (rv != 0) 4307e984c70bSHans Rosenfeld return (rv); 43083d9b1a2aSHans Rosenfeld 43093d9b1a2aSHans Rosenfeld if (nioc->n_len < bufsize) { 43103d9b1a2aSHans Rosenfeld kmem_free(buf, bufsize); 43113d9b1a2aSHans Rosenfeld return (EINVAL); 43123d9b1a2aSHans Rosenfeld } 43133d9b1a2aSHans Rosenfeld 43143d9b1a2aSHans Rosenfeld if (buf && ddi_copyout(buf, (void*)nioc->n_buf, bufsize, mode) != 0) 43153d9b1a2aSHans Rosenfeld rv = EFAULT; 43163d9b1a2aSHans Rosenfeld 43173d9b1a2aSHans Rosenfeld kmem_free(buf, bufsize); 43183d9b1a2aSHans Rosenfeld nioc->n_arg = res; 43193d9b1a2aSHans Rosenfeld nioc->n_len = bufsize; 43203d9b1a2aSHans Rosenfeld 43213d9b1a2aSHans Rosenfeld return (rv); 43223d9b1a2aSHans Rosenfeld } 43233d9b1a2aSHans Rosenfeld 43243d9b1a2aSHans Rosenfeld static int 43253d9b1a2aSHans Rosenfeld nvme_ioctl_intr_cnt(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 43263d9b1a2aSHans Rosenfeld cred_t *cred_p) 43273d9b1a2aSHans Rosenfeld { 43283d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nsid, mode, cred_p)); 43293d9b1a2aSHans Rosenfeld 43303d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 43313d9b1a2aSHans Rosenfeld return (EPERM); 43323d9b1a2aSHans Rosenfeld 43333d9b1a2aSHans Rosenfeld nioc->n_arg = nvme->n_intr_cnt; 43343d9b1a2aSHans Rosenfeld return (0); 43353d9b1a2aSHans Rosenfeld } 43363d9b1a2aSHans Rosenfeld 43373d9b1a2aSHans Rosenfeld static int 43383d9b1a2aSHans Rosenfeld nvme_ioctl_version(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 43393d9b1a2aSHans Rosenfeld cred_t *cred_p) 43403d9b1a2aSHans Rosenfeld { 43413d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nsid, cred_p)); 43423d9b1a2aSHans Rosenfeld int rv = 0; 43433d9b1a2aSHans Rosenfeld 43443d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 43453d9b1a2aSHans Rosenfeld return (EPERM); 43463d9b1a2aSHans Rosenfeld 43473d9b1a2aSHans Rosenfeld if (nioc->n_len < sizeof (nvme->n_version)) 43483d9b1a2aSHans Rosenfeld return (ENOMEM); 43493d9b1a2aSHans Rosenfeld 43503d9b1a2aSHans Rosenfeld if (ddi_copyout(&nvme->n_version, (void *)nioc->n_buf, 43513d9b1a2aSHans Rosenfeld sizeof (nvme->n_version), mode) != 0) 43523d9b1a2aSHans Rosenfeld rv = EFAULT; 43533d9b1a2aSHans Rosenfeld 43543d9b1a2aSHans Rosenfeld return (rv); 43553d9b1a2aSHans Rosenfeld } 43563d9b1a2aSHans Rosenfeld 43573d9b1a2aSHans Rosenfeld static int 43583d9b1a2aSHans Rosenfeld nvme_ioctl_format(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 43593d9b1a2aSHans Rosenfeld cred_t *cred_p) 43603d9b1a2aSHans Rosenfeld { 43613d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(mode)); 43623d9b1a2aSHans Rosenfeld nvme_format_nvm_t frmt = { 0 }; 43633d9b1a2aSHans Rosenfeld int c_nsid = nsid != 0 ? nsid - 1 : 0; 43643d9b1a2aSHans Rosenfeld 43653d9b1a2aSHans Rosenfeld if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 43663d9b1a2aSHans Rosenfeld return (EPERM); 43673d9b1a2aSHans Rosenfeld 43683d9b1a2aSHans Rosenfeld frmt.r = nioc->n_arg & 0xffffffff; 43693d9b1a2aSHans Rosenfeld 43703d9b1a2aSHans Rosenfeld /* 43713d9b1a2aSHans Rosenfeld * Check whether the FORMAT NVM command is supported. 43723d9b1a2aSHans Rosenfeld */ 43733d9b1a2aSHans Rosenfeld if (nvme->n_idctl->id_oacs.oa_format == 0) 43743d9b1a2aSHans Rosenfeld return (EINVAL); 43753d9b1a2aSHans Rosenfeld 43763d9b1a2aSHans Rosenfeld /* 43773d9b1a2aSHans Rosenfeld * Don't allow format or secure erase of individual namespace if that 43783d9b1a2aSHans Rosenfeld * would cause a format or secure erase of all namespaces. 43793d9b1a2aSHans Rosenfeld */ 43803d9b1a2aSHans Rosenfeld if (nsid != 0 && nvme->n_idctl->id_fna.fn_format != 0) 43813d9b1a2aSHans Rosenfeld return (EINVAL); 43823d9b1a2aSHans Rosenfeld 43833d9b1a2aSHans Rosenfeld if (nsid != 0 && frmt.b.fm_ses != NVME_FRMT_SES_NONE && 43843d9b1a2aSHans Rosenfeld nvme->n_idctl->id_fna.fn_sec_erase != 0) 43853d9b1a2aSHans Rosenfeld return (EINVAL); 43863d9b1a2aSHans Rosenfeld 43873d9b1a2aSHans Rosenfeld /* 43883d9b1a2aSHans Rosenfeld * Don't allow formatting with Protection Information. 43893d9b1a2aSHans Rosenfeld */ 43903d9b1a2aSHans Rosenfeld if (frmt.b.fm_pi != 0 || frmt.b.fm_pil != 0 || frmt.b.fm_ms != 0) 43913d9b1a2aSHans Rosenfeld return (EINVAL); 43923d9b1a2aSHans Rosenfeld 43933d9b1a2aSHans Rosenfeld /* 43943d9b1a2aSHans Rosenfeld * Don't allow formatting using an illegal LBA format, or any LBA format 43953d9b1a2aSHans Rosenfeld * that uses metadata. 43963d9b1a2aSHans Rosenfeld */ 43973d9b1a2aSHans Rosenfeld if (frmt.b.fm_lbaf > nvme->n_ns[c_nsid].ns_idns->id_nlbaf || 43983d9b1a2aSHans Rosenfeld nvme->n_ns[c_nsid].ns_idns->id_lbaf[frmt.b.fm_lbaf].lbaf_ms != 0) 43993d9b1a2aSHans Rosenfeld return (EINVAL); 44003d9b1a2aSHans Rosenfeld 44013d9b1a2aSHans Rosenfeld /* 44023d9b1a2aSHans Rosenfeld * Don't allow formatting using an illegal Secure Erase setting. 44033d9b1a2aSHans Rosenfeld */ 44043d9b1a2aSHans Rosenfeld if (frmt.b.fm_ses > NVME_FRMT_MAX_SES || 44053d9b1a2aSHans Rosenfeld (frmt.b.fm_ses == NVME_FRMT_SES_CRYPTO && 44063d9b1a2aSHans Rosenfeld nvme->n_idctl->id_fna.fn_crypt_erase == 0)) 44073d9b1a2aSHans Rosenfeld return (EINVAL); 44083d9b1a2aSHans Rosenfeld 44093d9b1a2aSHans Rosenfeld if (nsid == 0) 44103d9b1a2aSHans Rosenfeld nsid = (uint32_t)-1; 44113d9b1a2aSHans Rosenfeld 4412bc586359SRobert Mustacchi return (nvme_format_nvm(nvme, B_TRUE, nsid, frmt.b.fm_lbaf, B_FALSE, 0, 4413bc586359SRobert Mustacchi B_FALSE, frmt.b.fm_ses)); 44143d9b1a2aSHans Rosenfeld } 44153d9b1a2aSHans Rosenfeld 44163d9b1a2aSHans Rosenfeld static int 44173d9b1a2aSHans Rosenfeld nvme_ioctl_detach(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 44183d9b1a2aSHans Rosenfeld cred_t *cred_p) 44193d9b1a2aSHans Rosenfeld { 44203d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nioc, mode)); 44213d9b1a2aSHans Rosenfeld int rv = 0; 44223d9b1a2aSHans Rosenfeld 44233d9b1a2aSHans Rosenfeld if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 44243d9b1a2aSHans Rosenfeld return (EPERM); 44253d9b1a2aSHans Rosenfeld 44263d9b1a2aSHans Rosenfeld if (nsid == 0) 44273d9b1a2aSHans Rosenfeld return (EINVAL); 44283d9b1a2aSHans Rosenfeld 44293d9b1a2aSHans Rosenfeld rv = bd_detach_handle(nvme->n_ns[nsid - 1].ns_bd_hdl); 44303d9b1a2aSHans Rosenfeld if (rv != DDI_SUCCESS) 44313d9b1a2aSHans Rosenfeld rv = EBUSY; 44323d9b1a2aSHans Rosenfeld 44333d9b1a2aSHans Rosenfeld return (rv); 44343d9b1a2aSHans Rosenfeld } 44353d9b1a2aSHans Rosenfeld 44363d9b1a2aSHans Rosenfeld static int 44373d9b1a2aSHans Rosenfeld nvme_ioctl_attach(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 44383d9b1a2aSHans Rosenfeld cred_t *cred_p) 44393d9b1a2aSHans Rosenfeld { 44403d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nioc, mode)); 44413d9b1a2aSHans Rosenfeld nvme_identify_nsid_t *idns; 44423d9b1a2aSHans Rosenfeld int rv = 0; 44433d9b1a2aSHans Rosenfeld 44443d9b1a2aSHans Rosenfeld if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 44453d9b1a2aSHans Rosenfeld return (EPERM); 44463d9b1a2aSHans Rosenfeld 44473d9b1a2aSHans Rosenfeld if (nsid == 0) 44483d9b1a2aSHans Rosenfeld return (EINVAL); 44493d9b1a2aSHans Rosenfeld 44503d9b1a2aSHans Rosenfeld /* 44513d9b1a2aSHans Rosenfeld * Identify namespace again, free old identify data. 44523d9b1a2aSHans Rosenfeld */ 44533d9b1a2aSHans Rosenfeld idns = nvme->n_ns[nsid - 1].ns_idns; 44543d9b1a2aSHans Rosenfeld if (nvme_init_ns(nvme, nsid) != DDI_SUCCESS) 44553d9b1a2aSHans Rosenfeld return (EIO); 44563d9b1a2aSHans Rosenfeld 44573d9b1a2aSHans Rosenfeld kmem_free(idns, sizeof (nvme_identify_nsid_t)); 44583d9b1a2aSHans Rosenfeld 44593d9b1a2aSHans Rosenfeld rv = bd_attach_handle(nvme->n_dip, nvme->n_ns[nsid - 1].ns_bd_hdl); 44603d9b1a2aSHans Rosenfeld if (rv != DDI_SUCCESS) 44613d9b1a2aSHans Rosenfeld rv = EBUSY; 44623d9b1a2aSHans Rosenfeld 44633d9b1a2aSHans Rosenfeld return (rv); 44643d9b1a2aSHans Rosenfeld } 44653d9b1a2aSHans Rosenfeld 4466e89be50aSRob Johnston static void 4467e89be50aSRob Johnston nvme_ufm_update(nvme_t *nvme) 4468e89be50aSRob Johnston { 4469e89be50aSRob Johnston mutex_enter(&nvme->n_fwslot_mutex); 4470e89be50aSRob Johnston ddi_ufm_update(nvme->n_ufmh); 4471e89be50aSRob Johnston if (nvme->n_fwslot != NULL) { 4472e89be50aSRob Johnston kmem_free(nvme->n_fwslot, sizeof (nvme_fwslot_log_t)); 4473e89be50aSRob Johnston nvme->n_fwslot = NULL; 4474e89be50aSRob Johnston } 4475e89be50aSRob Johnston mutex_exit(&nvme->n_fwslot_mutex); 4476e89be50aSRob Johnston } 4477e89be50aSRob Johnston 4478cf840871SPaul Winder static int 4479cf840871SPaul Winder nvme_ioctl_firmware_download(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 4480cf840871SPaul Winder int mode, cred_t *cred_p) 4481cf840871SPaul Winder { 4482cf840871SPaul Winder int rv = 0; 4483cf840871SPaul Winder size_t len, copylen; 4484cf840871SPaul Winder offset_t offset; 4485cf840871SPaul Winder uintptr_t buf; 4486cf840871SPaul Winder nvme_sqe_t sqe = { 4487cf840871SPaul Winder .sqe_opc = NVME_OPC_FW_IMAGE_LOAD 4488cf840871SPaul Winder }; 4489cf840871SPaul Winder 4490cf840871SPaul Winder if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 4491cf840871SPaul Winder return (EPERM); 4492cf840871SPaul Winder 4493cf840871SPaul Winder if (nsid != 0) 4494cf840871SPaul Winder return (EINVAL); 4495cf840871SPaul Winder 4496cf840871SPaul Winder /* 4497cf840871SPaul Winder * The offset (in n_len) is restricted to the number of DWORDs in 4498cf840871SPaul Winder * 32 bits. 4499cf840871SPaul Winder */ 4500cf840871SPaul Winder if (nioc->n_len > NVME_FW_OFFSETB_MAX) 4501cf840871SPaul Winder return (EINVAL); 4502cf840871SPaul Winder 4503cf840871SPaul Winder /* Confirm that both offset and length are a multiple of DWORD bytes */ 4504cf840871SPaul Winder if ((nioc->n_len & NVME_DWORD_MASK) != 0 || 4505cf840871SPaul Winder (nioc->n_arg & NVME_DWORD_MASK) != 0) 4506cf840871SPaul Winder return (EINVAL); 4507cf840871SPaul Winder 4508cf840871SPaul Winder len = nioc->n_len; 4509cf840871SPaul Winder offset = nioc->n_arg; 4510cf840871SPaul Winder buf = (uintptr_t)nioc->n_buf; 4511cf840871SPaul Winder while (len > 0 && rv == 0) { 4512cf840871SPaul Winder /* 4513cf840871SPaul Winder * nvme_ioc_cmd() does not use SGLs or PRP lists. 4514cf840871SPaul Winder * It is limited to 2 PRPs per NVM command, so limit 4515cf840871SPaul Winder * the size of the data to 2 pages. 4516cf840871SPaul Winder */ 4517cf840871SPaul Winder copylen = MIN(2 * nvme->n_pagesize, len); 4518cf840871SPaul Winder 4519cf840871SPaul Winder sqe.sqe_cdw10 = (uint32_t)(copylen >> NVME_DWORD_SHIFT) - 1; 4520cf840871SPaul Winder sqe.sqe_cdw11 = (uint32_t)(offset >> NVME_DWORD_SHIFT); 4521cf840871SPaul Winder 4522cf840871SPaul Winder rv = nvme_ioc_cmd(nvme, &sqe, B_TRUE, (void *)buf, copylen, 4523cf840871SPaul Winder FWRITE, NULL, nvme_admin_cmd_timeout); 4524cf840871SPaul Winder 4525cf840871SPaul Winder buf += copylen; 4526cf840871SPaul Winder offset += copylen; 4527cf840871SPaul Winder len -= copylen; 4528cf840871SPaul Winder } 4529cf840871SPaul Winder 4530e89be50aSRob Johnston /* 4531e89be50aSRob Johnston * Let the DDI UFM subsystem know that the firmware information for 4532e89be50aSRob Johnston * this device has changed. 4533e89be50aSRob Johnston */ 4534e89be50aSRob Johnston nvme_ufm_update(nvme); 4535e89be50aSRob Johnston 4536cf840871SPaul Winder return (rv); 4537cf840871SPaul Winder } 4538cf840871SPaul Winder 4539cf840871SPaul Winder static int 4540cf840871SPaul Winder nvme_ioctl_firmware_commit(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 4541cf840871SPaul Winder int mode, cred_t *cred_p) 4542cf840871SPaul Winder { 4543cf840871SPaul Winder nvme_firmware_commit_dw10_t fc_dw10 = { 0 }; 4544cf840871SPaul Winder uint32_t slot = nioc->n_arg & 0xffffffff; 4545cf840871SPaul Winder uint32_t action = nioc->n_arg >> 32; 4546cf840871SPaul Winder nvme_cqe_t cqe = { 0 }; 4547cf840871SPaul Winder nvme_sqe_t sqe = { 4548cf840871SPaul Winder .sqe_opc = NVME_OPC_FW_ACTIVATE 4549cf840871SPaul Winder }; 4550cf840871SPaul Winder int timeout; 4551cf840871SPaul Winder int rv; 4552cf840871SPaul Winder 4553cf840871SPaul Winder if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 4554cf840871SPaul Winder return (EPERM); 4555cf840871SPaul Winder 4556cf840871SPaul Winder if (nsid != 0) 4557cf840871SPaul Winder return (EINVAL); 4558cf840871SPaul Winder 4559cf840871SPaul Winder /* Validate slot is in range. */ 4560cf840871SPaul Winder if (slot < NVME_FW_SLOT_MIN || slot > NVME_FW_SLOT_MAX) 4561cf840871SPaul Winder return (EINVAL); 4562cf840871SPaul Winder 4563cf840871SPaul Winder switch (action) { 4564cf840871SPaul Winder case NVME_FWC_SAVE: 4565cf840871SPaul Winder case NVME_FWC_SAVE_ACTIVATE: 4566cf840871SPaul Winder timeout = nvme_commit_save_cmd_timeout; 4567cf840871SPaul Winder break; 4568cf840871SPaul Winder case NVME_FWC_ACTIVATE: 4569cf840871SPaul Winder case NVME_FWC_ACTIVATE_IMMED: 4570cf840871SPaul Winder timeout = nvme_admin_cmd_timeout; 4571cf840871SPaul Winder break; 4572cf840871SPaul Winder default: 4573cf840871SPaul Winder return (EINVAL); 4574cf840871SPaul Winder } 4575cf840871SPaul Winder 4576cf840871SPaul Winder fc_dw10.b.fc_slot = slot; 4577cf840871SPaul Winder fc_dw10.b.fc_action = action; 4578cf840871SPaul Winder sqe.sqe_cdw10 = fc_dw10.r; 4579cf840871SPaul Winder 4580cf840871SPaul Winder rv = nvme_ioc_cmd(nvme, &sqe, B_TRUE, NULL, 0, 0, &cqe, timeout); 4581cf840871SPaul Winder 4582cf840871SPaul Winder nioc->n_arg = ((uint64_t)cqe.cqe_sf.sf_sct << 16) | cqe.cqe_sf.sf_sc; 4583cf840871SPaul Winder 4584e89be50aSRob Johnston /* 4585e89be50aSRob Johnston * Let the DDI UFM subsystem know that the firmware information for 4586e89be50aSRob Johnston * this device has changed. 4587e89be50aSRob Johnston */ 4588e89be50aSRob Johnston nvme_ufm_update(nvme); 4589e89be50aSRob Johnston 4590cf840871SPaul Winder return (rv); 4591cf840871SPaul Winder } 4592cf840871SPaul Winder 45933d9b1a2aSHans Rosenfeld static int 45943d9b1a2aSHans Rosenfeld nvme_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred_p, 45953d9b1a2aSHans Rosenfeld int *rval_p) 45963d9b1a2aSHans Rosenfeld { 45973d9b1a2aSHans Rosenfeld #ifndef __lock_lint 45983d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(rval_p)); 45993d9b1a2aSHans Rosenfeld #endif 46003d9b1a2aSHans Rosenfeld minor_t minor = getminor(dev); 46013d9b1a2aSHans Rosenfeld nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 46023d9b1a2aSHans Rosenfeld int nsid = NVME_MINOR_NSID(minor); 46033d9b1a2aSHans Rosenfeld int rv = 0; 46043d9b1a2aSHans Rosenfeld nvme_ioctl_t nioc; 46053d9b1a2aSHans Rosenfeld 46063d9b1a2aSHans Rosenfeld int (*nvme_ioctl[])(nvme_t *, int, nvme_ioctl_t *, int, cred_t *) = { 46073d9b1a2aSHans Rosenfeld NULL, 46083d9b1a2aSHans Rosenfeld nvme_ioctl_identify, 46093d9b1a2aSHans Rosenfeld nvme_ioctl_identify, 46103d9b1a2aSHans Rosenfeld nvme_ioctl_capabilities, 46113d9b1a2aSHans Rosenfeld nvme_ioctl_get_logpage, 46123d9b1a2aSHans Rosenfeld nvme_ioctl_get_features, 46133d9b1a2aSHans Rosenfeld nvme_ioctl_intr_cnt, 46143d9b1a2aSHans Rosenfeld nvme_ioctl_version, 46153d9b1a2aSHans Rosenfeld nvme_ioctl_format, 46163d9b1a2aSHans Rosenfeld nvme_ioctl_detach, 4617cf840871SPaul Winder nvme_ioctl_attach, 4618cf840871SPaul Winder nvme_ioctl_firmware_download, 4619cf840871SPaul Winder nvme_ioctl_firmware_commit 46203d9b1a2aSHans Rosenfeld }; 46213d9b1a2aSHans Rosenfeld 46223d9b1a2aSHans Rosenfeld if (nvme == NULL) 46233d9b1a2aSHans Rosenfeld return (ENXIO); 46243d9b1a2aSHans Rosenfeld 46253d9b1a2aSHans Rosenfeld if (nsid > nvme->n_namespace_count) 46263d9b1a2aSHans Rosenfeld return (ENXIO); 46273d9b1a2aSHans Rosenfeld 46283d9b1a2aSHans Rosenfeld if (IS_DEVCTL(cmd)) 46293d9b1a2aSHans Rosenfeld return (ndi_devctl_ioctl(nvme->n_dip, cmd, arg, mode, 0)); 46303d9b1a2aSHans Rosenfeld 46313d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 46323d9b1a2aSHans Rosenfeld switch (ddi_model_convert_from(mode & FMODELS)) { 46333d9b1a2aSHans Rosenfeld case DDI_MODEL_ILP32: { 46343d9b1a2aSHans Rosenfeld nvme_ioctl32_t nioc32; 46353d9b1a2aSHans Rosenfeld if (ddi_copyin((void*)arg, &nioc32, sizeof (nvme_ioctl32_t), 46363d9b1a2aSHans Rosenfeld mode) != 0) 46373d9b1a2aSHans Rosenfeld return (EFAULT); 46383d9b1a2aSHans Rosenfeld nioc.n_len = nioc32.n_len; 46393d9b1a2aSHans Rosenfeld nioc.n_buf = nioc32.n_buf; 46403d9b1a2aSHans Rosenfeld nioc.n_arg = nioc32.n_arg; 46413d9b1a2aSHans Rosenfeld break; 46423d9b1a2aSHans Rosenfeld } 46433d9b1a2aSHans Rosenfeld case DDI_MODEL_NONE: 46443d9b1a2aSHans Rosenfeld #endif 46453d9b1a2aSHans Rosenfeld if (ddi_copyin((void*)arg, &nioc, sizeof (nvme_ioctl_t), mode) 46463d9b1a2aSHans Rosenfeld != 0) 46473d9b1a2aSHans Rosenfeld return (EFAULT); 46483d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 46493d9b1a2aSHans Rosenfeld break; 46503d9b1a2aSHans Rosenfeld } 46513d9b1a2aSHans Rosenfeld #endif 46523d9b1a2aSHans Rosenfeld 4653e984c70bSHans Rosenfeld if (nvme->n_dead && cmd != NVME_IOC_DETACH) 4654e984c70bSHans Rosenfeld return (EIO); 4655e984c70bSHans Rosenfeld 4656e984c70bSHans Rosenfeld 46573d9b1a2aSHans Rosenfeld if (cmd == NVME_IOC_IDENTIFY_CTRL) { 46583d9b1a2aSHans Rosenfeld /* 46593d9b1a2aSHans Rosenfeld * This makes NVME_IOC_IDENTIFY_CTRL work the same on devctl and 46603d9b1a2aSHans Rosenfeld * attachment point nodes. 46613d9b1a2aSHans Rosenfeld */ 46623d9b1a2aSHans Rosenfeld nsid = 0; 46633d9b1a2aSHans Rosenfeld } else if (cmd == NVME_IOC_IDENTIFY_NSID && nsid == 0) { 46643d9b1a2aSHans Rosenfeld /* 46653d9b1a2aSHans Rosenfeld * This makes NVME_IOC_IDENTIFY_NSID work on a devctl node, it 46663d9b1a2aSHans Rosenfeld * will always return identify data for namespace 1. 46673d9b1a2aSHans Rosenfeld */ 46683d9b1a2aSHans Rosenfeld nsid = 1; 46693d9b1a2aSHans Rosenfeld } 46703d9b1a2aSHans Rosenfeld 46713d9b1a2aSHans Rosenfeld if (IS_NVME_IOC(cmd) && nvme_ioctl[NVME_IOC_CMD(cmd)] != NULL) 46723d9b1a2aSHans Rosenfeld rv = nvme_ioctl[NVME_IOC_CMD(cmd)](nvme, nsid, &nioc, mode, 46733d9b1a2aSHans Rosenfeld cred_p); 46743d9b1a2aSHans Rosenfeld else 46753d9b1a2aSHans Rosenfeld rv = EINVAL; 46763d9b1a2aSHans Rosenfeld 46773d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 46783d9b1a2aSHans Rosenfeld switch (ddi_model_convert_from(mode & FMODELS)) { 46793d9b1a2aSHans Rosenfeld case DDI_MODEL_ILP32: { 46803d9b1a2aSHans Rosenfeld nvme_ioctl32_t nioc32; 46813d9b1a2aSHans Rosenfeld 46823d9b1a2aSHans Rosenfeld nioc32.n_len = (size32_t)nioc.n_len; 46833d9b1a2aSHans Rosenfeld nioc32.n_buf = (uintptr32_t)nioc.n_buf; 46843d9b1a2aSHans Rosenfeld nioc32.n_arg = nioc.n_arg; 46853d9b1a2aSHans Rosenfeld 46863d9b1a2aSHans Rosenfeld if (ddi_copyout(&nioc32, (void *)arg, sizeof (nvme_ioctl32_t), 46873d9b1a2aSHans Rosenfeld mode) != 0) 46883d9b1a2aSHans Rosenfeld return (EFAULT); 46893d9b1a2aSHans Rosenfeld break; 46903d9b1a2aSHans Rosenfeld } 46913d9b1a2aSHans Rosenfeld case DDI_MODEL_NONE: 46923d9b1a2aSHans Rosenfeld #endif 46933d9b1a2aSHans Rosenfeld if (ddi_copyout(&nioc, (void *)arg, sizeof (nvme_ioctl_t), mode) 46943d9b1a2aSHans Rosenfeld != 0) 46953d9b1a2aSHans Rosenfeld return (EFAULT); 46963d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 46973d9b1a2aSHans Rosenfeld break; 46983d9b1a2aSHans Rosenfeld } 46993d9b1a2aSHans Rosenfeld #endif 47003d9b1a2aSHans Rosenfeld 47013d9b1a2aSHans Rosenfeld return (rv); 47023d9b1a2aSHans Rosenfeld } 4703e89be50aSRob Johnston 4704e89be50aSRob Johnston /* 4705e89be50aSRob Johnston * DDI UFM Callbacks 4706e89be50aSRob Johnston */ 4707e89be50aSRob Johnston static int 4708e89be50aSRob Johnston nvme_ufm_fill_image(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno, 4709e89be50aSRob Johnston ddi_ufm_image_t *img) 4710e89be50aSRob Johnston { 4711e89be50aSRob Johnston nvme_t *nvme = arg; 4712e89be50aSRob Johnston 4713e89be50aSRob Johnston if (imgno != 0) 4714e89be50aSRob Johnston return (EINVAL); 4715e89be50aSRob Johnston 4716e89be50aSRob Johnston ddi_ufm_image_set_desc(img, "Firmware"); 4717e89be50aSRob Johnston ddi_ufm_image_set_nslots(img, nvme->n_idctl->id_frmw.fw_nslot); 4718e89be50aSRob Johnston 4719e89be50aSRob Johnston return (0); 4720e89be50aSRob Johnston } 4721e89be50aSRob Johnston 4722e89be50aSRob Johnston /* 4723e89be50aSRob Johnston * Fill out firmware slot information for the requested slot. The firmware 4724e89be50aSRob Johnston * slot information is gathered by requesting the Firmware Slot Information log 4725e89be50aSRob Johnston * page. The format of the page is described in section 5.10.1.3. 4726e89be50aSRob Johnston * 4727e89be50aSRob Johnston * We lazily cache the log page on the first call and then invalidate the cache 4728e89be50aSRob Johnston * data after a successful firmware download or firmware commit command. 4729e89be50aSRob Johnston * The cached data is protected by a mutex as the state can change 4730e89be50aSRob Johnston * asynchronous to this callback. 4731e89be50aSRob Johnston */ 4732e89be50aSRob Johnston static int 4733e89be50aSRob Johnston nvme_ufm_fill_slot(ddi_ufm_handle_t *ufmh, void *arg, uint_t imgno, 4734e89be50aSRob Johnston uint_t slotno, ddi_ufm_slot_t *slot) 4735e89be50aSRob Johnston { 4736e89be50aSRob Johnston nvme_t *nvme = arg; 4737e89be50aSRob Johnston void *log = NULL; 4738e89be50aSRob Johnston size_t bufsize; 4739e89be50aSRob Johnston ddi_ufm_attr_t attr = 0; 4740e89be50aSRob Johnston char fw_ver[NVME_FWVER_SZ + 1]; 4741e89be50aSRob Johnston int ret; 4742e89be50aSRob Johnston 4743e89be50aSRob Johnston if (imgno > 0 || slotno > (nvme->n_idctl->id_frmw.fw_nslot - 1)) 4744e89be50aSRob Johnston return (EINVAL); 4745e89be50aSRob Johnston 4746e89be50aSRob Johnston mutex_enter(&nvme->n_fwslot_mutex); 4747e89be50aSRob Johnston if (nvme->n_fwslot == NULL) { 4748e89be50aSRob Johnston ret = nvme_get_logpage(nvme, B_TRUE, &log, &bufsize, 4749e89be50aSRob Johnston NVME_LOGPAGE_FWSLOT, 0); 4750e89be50aSRob Johnston if (ret != DDI_SUCCESS || 4751e89be50aSRob Johnston bufsize != sizeof (nvme_fwslot_log_t)) { 4752e89be50aSRob Johnston if (log != NULL) 4753e89be50aSRob Johnston kmem_free(log, bufsize); 4754e89be50aSRob Johnston mutex_exit(&nvme->n_fwslot_mutex); 4755e89be50aSRob Johnston return (EIO); 4756e89be50aSRob Johnston } 4757e89be50aSRob Johnston nvme->n_fwslot = (nvme_fwslot_log_t *)log; 4758e89be50aSRob Johnston } 4759e89be50aSRob Johnston 4760e89be50aSRob Johnston /* 4761e89be50aSRob Johnston * NVMe numbers firmware slots starting at 1 4762e89be50aSRob Johnston */ 4763e89be50aSRob Johnston if (slotno == (nvme->n_fwslot->fw_afi - 1)) 4764e89be50aSRob Johnston attr |= DDI_UFM_ATTR_ACTIVE; 4765e89be50aSRob Johnston 4766e89be50aSRob Johnston if (slotno != 0 || nvme->n_idctl->id_frmw.fw_readonly == 0) 4767e89be50aSRob Johnston attr |= DDI_UFM_ATTR_WRITEABLE; 4768e89be50aSRob Johnston 4769e89be50aSRob Johnston if (nvme->n_fwslot->fw_frs[slotno][0] == '\0') { 4770e89be50aSRob Johnston attr |= DDI_UFM_ATTR_EMPTY; 4771e89be50aSRob Johnston } else { 4772e89be50aSRob Johnston (void) strncpy(fw_ver, nvme->n_fwslot->fw_frs[slotno], 4773e89be50aSRob Johnston NVME_FWVER_SZ); 4774e89be50aSRob Johnston fw_ver[NVME_FWVER_SZ] = '\0'; 4775e89be50aSRob Johnston ddi_ufm_slot_set_version(slot, fw_ver); 4776e89be50aSRob Johnston } 4777e89be50aSRob Johnston mutex_exit(&nvme->n_fwslot_mutex); 4778e89be50aSRob Johnston 4779e89be50aSRob Johnston ddi_ufm_slot_set_attrs(slot, attr); 4780e89be50aSRob Johnston 4781e89be50aSRob Johnston return (0); 4782e89be50aSRob Johnston } 4783e89be50aSRob Johnston 4784e89be50aSRob Johnston static int 4785e89be50aSRob Johnston nvme_ufm_getcaps(ddi_ufm_handle_t *ufmh, void *arg, ddi_ufm_cap_t *caps) 4786e89be50aSRob Johnston { 4787e89be50aSRob Johnston *caps = DDI_UFM_CAP_REPORT; 4788e89be50aSRob Johnston return (0); 4789e89be50aSRob Johnston } 4790