13c9168faSHans Rosenfeld /* 23c9168faSHans Rosenfeld * This file and its contents are supplied under the terms of the 33c9168faSHans Rosenfeld * Common Development and Distribution License ("CDDL"), version 1.0. 43c9168faSHans Rosenfeld * You may only use this file in accordance with the terms of version 53c9168faSHans Rosenfeld * 1.0 of the CDDL. 63c9168faSHans Rosenfeld * 73c9168faSHans Rosenfeld * A full copy of the text of the CDDL should have accompanied this 83c9168faSHans Rosenfeld * source. A copy of the CDDL is also available via the Internet at 93c9168faSHans Rosenfeld * http://www.illumos.org/license/CDDL. 103c9168faSHans Rosenfeld */ 113c9168faSHans Rosenfeld 123c9168faSHans Rosenfeld /* 13*f313c178SYuri Pankov * Copyright 2018 Nexenta Systems, Inc. 1434c938c7SPete Shephard * Copyright 2016 Tegile Systems, Inc. All rights reserved. 152f95345bSYouzhong Yang * Copyright (c) 2016 The MathWorks, Inc. All rights reserved. 16621738e2SHans Rosenfeld * Copyright 2017 Joyent, Inc. 173c9168faSHans Rosenfeld */ 183c9168faSHans Rosenfeld 193c9168faSHans Rosenfeld /* 203c9168faSHans Rosenfeld * blkdev driver for NVMe compliant storage devices 213c9168faSHans Rosenfeld * 22f8dcaea5SRobert Mustacchi * This driver was written to conform to version 1.2.1 of the NVMe 23f8dcaea5SRobert Mustacchi * specification. It may work with newer versions, but that is completely 24f8dcaea5SRobert Mustacchi * untested and disabled by default. 253c9168faSHans Rosenfeld * 263c9168faSHans Rosenfeld * The driver has only been tested on x86 systems and will not work on big- 273c9168faSHans Rosenfeld * endian systems without changes to the code accessing registers and data 283c9168faSHans Rosenfeld * structures used by the hardware. 293c9168faSHans Rosenfeld * 303c9168faSHans Rosenfeld * 313c9168faSHans Rosenfeld * Interrupt Usage: 323c9168faSHans Rosenfeld * 330b466603SHans Rosenfeld * The driver will use a single interrupt while configuring the device as the 340b466603SHans Rosenfeld * specification requires, but contrary to the specification it will try to use 350b466603SHans Rosenfeld * a single-message MSI(-X) or FIXED interrupt. Later in the attach process it 360b466603SHans Rosenfeld * will switch to multiple-message MSI(-X) if supported. The driver wants to 370b466603SHans Rosenfeld * have one interrupt vector per CPU, but it will work correctly if less are 380b466603SHans Rosenfeld * available. Interrupts can be shared by queues, the interrupt handler will 390b466603SHans Rosenfeld * iterate through the I/O queue array by steps of n_intr_cnt. Usually only 400b466603SHans Rosenfeld * the admin queue will share an interrupt with one I/O queue. The interrupt 410b466603SHans Rosenfeld * handler will retrieve completed commands from all queues sharing an interrupt 420b466603SHans Rosenfeld * vector and will post them to a taskq for completion processing. 433c9168faSHans Rosenfeld * 443c9168faSHans Rosenfeld * 453c9168faSHans Rosenfeld * Command Processing: 463c9168faSHans Rosenfeld * 474b324362SHans Rosenfeld * NVMe devices can have up to 65535 I/O queue pairs, with each queue holding up 483c9168faSHans Rosenfeld * to 65536 I/O commands. The driver will configure one I/O queue pair per 493c9168faSHans Rosenfeld * available interrupt vector, with the queue length usually much smaller than 503c9168faSHans Rosenfeld * the maximum of 65536. If the hardware doesn't provide enough queues, fewer 513c9168faSHans Rosenfeld * interrupt vectors will be used. 523c9168faSHans Rosenfeld * 533c9168faSHans Rosenfeld * Additionally the hardware provides a single special admin queue pair that can 543c9168faSHans Rosenfeld * hold up to 4096 admin commands. 553c9168faSHans Rosenfeld * 563c9168faSHans Rosenfeld * From the hardware perspective both queues of a queue pair are independent, 573c9168faSHans Rosenfeld * but they share some driver state: the command array (holding pointers to 583c9168faSHans Rosenfeld * commands currently being processed by the hardware) and the active command 59e984c70bSHans Rosenfeld * counter. Access to a queue pair and the shared state is protected by 60e984c70bSHans Rosenfeld * nq_mutex. 613c9168faSHans Rosenfeld * 623c9168faSHans Rosenfeld * When a command is submitted to a queue pair the active command counter is 633c9168faSHans Rosenfeld * incremented and a pointer to the command is stored in the command array. The 643c9168faSHans Rosenfeld * array index is used as command identifier (CID) in the submission queue 653c9168faSHans Rosenfeld * entry. Some commands may take a very long time to complete, and if the queue 663c9168faSHans Rosenfeld * wraps around in that time a submission may find the next array slot to still 673c9168faSHans Rosenfeld * be used by a long-running command. In this case the array is sequentially 683c9168faSHans Rosenfeld * searched for the next free slot. The length of the command array is the same 694b324362SHans Rosenfeld * as the configured queue length. Queue overrun is prevented by the semaphore, 704b324362SHans Rosenfeld * so a command submission may block if the queue is full. 713c9168faSHans Rosenfeld * 723c9168faSHans Rosenfeld * 734ac9cfccSHans Rosenfeld * Polled I/O Support: 744ac9cfccSHans Rosenfeld * 754ac9cfccSHans Rosenfeld * For kernel core dump support the driver can do polled I/O. As interrupts are 764ac9cfccSHans Rosenfeld * turned off while dumping the driver will just submit a command in the regular 774ac9cfccSHans Rosenfeld * way, and then repeatedly attempt a command retrieval until it gets the 784ac9cfccSHans Rosenfeld * command back. 794ac9cfccSHans Rosenfeld * 804ac9cfccSHans Rosenfeld * 813c9168faSHans Rosenfeld * Namespace Support: 823c9168faSHans Rosenfeld * 833c9168faSHans Rosenfeld * NVMe devices can have multiple namespaces, each being a independent data 843c9168faSHans Rosenfeld * store. The driver supports multiple namespaces and creates a blkdev interface 853c9168faSHans Rosenfeld * for each namespace found. Namespaces can have various attributes to support 869cfb65faSHans Rosenfeld * thin provisioning and protection information. This driver does not support 879cfb65faSHans Rosenfeld * any of this and ignores namespaces that have these attributes. 883c9168faSHans Rosenfeld * 8924979ca3SHans Rosenfeld * As of NVMe 1.1 namespaces can have an 64bit Extended Unique Identifier 9024979ca3SHans Rosenfeld * (EUI64). This driver uses the EUI64 if present to generate the devid and 9124979ca3SHans Rosenfeld * passes it to blkdev to use it in the device node names. As this is currently 9224979ca3SHans Rosenfeld * untested namespaces with EUI64 are ignored by default. 9324979ca3SHans Rosenfeld * 943d9b1a2aSHans Rosenfeld * We currently support only (2 << NVME_MINOR_INST_SHIFT) - 2 namespaces in a 953d9b1a2aSHans Rosenfeld * single controller. This is an artificial limit imposed by the driver to be 963d9b1a2aSHans Rosenfeld * able to address a reasonable number of controllers and namespaces using a 973d9b1a2aSHans Rosenfeld * 32bit minor node number. 983d9b1a2aSHans Rosenfeld * 993d9b1a2aSHans Rosenfeld * 1003d9b1a2aSHans Rosenfeld * Minor nodes: 1013d9b1a2aSHans Rosenfeld * 1023d9b1a2aSHans Rosenfeld * For each NVMe device the driver exposes one minor node for the controller and 1033d9b1a2aSHans Rosenfeld * one minor node for each namespace. The only operations supported by those 1043d9b1a2aSHans Rosenfeld * minor nodes are open(9E), close(9E), and ioctl(9E). This serves as the 1053d9b1a2aSHans Rosenfeld * interface for the nvmeadm(1M) utility. 1063d9b1a2aSHans Rosenfeld * 1073c9168faSHans Rosenfeld * 1083c9168faSHans Rosenfeld * Blkdev Interface: 1093c9168faSHans Rosenfeld * 1103c9168faSHans Rosenfeld * This driver uses blkdev to do all the heavy lifting involved with presenting 1113c9168faSHans Rosenfeld * a disk device to the system. As a result, the processing of I/O requests is 1123c9168faSHans Rosenfeld * relatively simple as blkdev takes care of partitioning, boundary checks, DMA 1133c9168faSHans Rosenfeld * setup, and splitting of transfers into manageable chunks. 1143c9168faSHans Rosenfeld * 1153c9168faSHans Rosenfeld * I/O requests coming in from blkdev are turned into NVM commands and posted to 1163c9168faSHans Rosenfeld * an I/O queue. The queue is selected by taking the CPU id modulo the number of 1173c9168faSHans Rosenfeld * queues. There is currently no timeout handling of I/O commands. 1183c9168faSHans Rosenfeld * 1193c9168faSHans Rosenfeld * Blkdev also supports querying device/media information and generating a 1203c9168faSHans Rosenfeld * devid. The driver reports the best block size as determined by the namespace 1213c9168faSHans Rosenfeld * format back to blkdev as physical block size to support partition and block 12224979ca3SHans Rosenfeld * alignment. The devid is either based on the namespace EUI64, if present, or 12324979ca3SHans Rosenfeld * composed using the device vendor ID, model number, serial number, and the 12424979ca3SHans Rosenfeld * namespace ID. 1253c9168faSHans Rosenfeld * 1263c9168faSHans Rosenfeld * 1273c9168faSHans Rosenfeld * Error Handling: 1283c9168faSHans Rosenfeld * 1293c9168faSHans Rosenfeld * Error handling is currently limited to detecting fatal hardware errors, 1303c9168faSHans Rosenfeld * either by asynchronous events, or synchronously through command status or 1313c9168faSHans Rosenfeld * admin command timeouts. In case of severe errors the device is fenced off, 1323c9168faSHans Rosenfeld * all further requests will return EIO. FMA is then called to fault the device. 1333c9168faSHans Rosenfeld * 1343c9168faSHans Rosenfeld * The hardware has a limit for outstanding asynchronous event requests. Before 1353c9168faSHans Rosenfeld * this limit is known the driver assumes it is at least 1 and posts a single 1363c9168faSHans Rosenfeld * asynchronous request. Later when the limit is known more asynchronous event 1373c9168faSHans Rosenfeld * requests are posted to allow quicker reception of error information. When an 1383c9168faSHans Rosenfeld * asynchronous event is posted by the hardware the driver will parse the error 1393c9168faSHans Rosenfeld * status fields and log information or fault the device, depending on the 1403c9168faSHans Rosenfeld * severity of the asynchronous event. The asynchronous event request is then 1413c9168faSHans Rosenfeld * reused and posted to the admin queue again. 1423c9168faSHans Rosenfeld * 1433c9168faSHans Rosenfeld * On command completion the command status is checked for errors. In case of 1443c9168faSHans Rosenfeld * errors indicating a driver bug the driver panics. Almost all other error 1453c9168faSHans Rosenfeld * status values just cause EIO to be returned. 1463c9168faSHans Rosenfeld * 1473c9168faSHans Rosenfeld * Command timeouts are currently detected for all admin commands except 1483c9168faSHans Rosenfeld * asynchronous event requests. If a command times out and the hardware appears 149e984c70bSHans Rosenfeld * to be healthy the driver attempts to abort the command. The original command 150e984c70bSHans Rosenfeld * timeout is also applied to the abort command. If the abort times out too the 1513c9168faSHans Rosenfeld * driver assumes the device to be dead, fences it off, and calls FMA to retire 152e984c70bSHans Rosenfeld * it. In all other cases the aborted command should return immediately with a 153e984c70bSHans Rosenfeld * status indicating it was aborted, and the driver will wait indefinitely for 154e984c70bSHans Rosenfeld * that to happen. No timeout handling of normal I/O commands is presently done. 1553c9168faSHans Rosenfeld * 156e984c70bSHans Rosenfeld * Any command that times out due to the controller dropping dead will be put on 157e984c70bSHans Rosenfeld * nvme_lost_cmds list if it references DMA memory. This will prevent the DMA 158e984c70bSHans Rosenfeld * memory being reused by the system and later be written to by a "dead" NVMe 159e984c70bSHans Rosenfeld * controller. 160e984c70bSHans Rosenfeld * 161e984c70bSHans Rosenfeld * 162e984c70bSHans Rosenfeld * Locking: 163e984c70bSHans Rosenfeld * 164e984c70bSHans Rosenfeld * Each queue pair has its own nq_mutex, which must be held when accessing the 165e984c70bSHans Rosenfeld * associated queue registers or the shared state of the queue pair. Callers of 166e984c70bSHans Rosenfeld * nvme_unqueue_cmd() must make sure that nq_mutex is held, while 167e984c70bSHans Rosenfeld * nvme_submit_{admin,io}_cmd() and nvme_retrieve_cmd() take care of this 168e984c70bSHans Rosenfeld * themselves. 169e984c70bSHans Rosenfeld * 170e984c70bSHans Rosenfeld * Each command also has its own nc_mutex, which is associated with the 171e984c70bSHans Rosenfeld * condition variable nc_cv. It is only used on admin commands which are run 172e984c70bSHans Rosenfeld * synchronously. In that case it must be held across calls to 173e984c70bSHans Rosenfeld * nvme_submit_{admin,io}_cmd() and nvme_wait_cmd(), which is taken care of by 174e984c70bSHans Rosenfeld * nvme_admin_cmd(). It must also be held whenever the completion state of the 175e984c70bSHans Rosenfeld * command is changed or while a admin command timeout is handled. 176e984c70bSHans Rosenfeld * 177e984c70bSHans Rosenfeld * If both nc_mutex and nq_mutex must be held, nc_mutex must be acquired first. 178e984c70bSHans Rosenfeld * More than one nc_mutex may only be held when aborting commands. In this case, 179e984c70bSHans Rosenfeld * the nc_mutex of the command to be aborted must be held across the call to 180e984c70bSHans Rosenfeld * nvme_abort_cmd() to prevent the command from completing while the abort is in 181e984c70bSHans Rosenfeld * progress. 182e984c70bSHans Rosenfeld * 183e984c70bSHans Rosenfeld * Each minor node has its own nm_mutex, which protects the open count nm_ocnt 184e984c70bSHans Rosenfeld * and exclusive-open flag nm_oexcl. 1853c9168faSHans Rosenfeld * 1863c9168faSHans Rosenfeld * 1873c9168faSHans Rosenfeld * Quiesce / Fast Reboot: 1883c9168faSHans Rosenfeld * 1893c9168faSHans Rosenfeld * The driver currently does not support fast reboot. A quiesce(9E) entry point 1903c9168faSHans Rosenfeld * is still provided which is used to send a shutdown notification to the 1913c9168faSHans Rosenfeld * device. 1923c9168faSHans Rosenfeld * 1933c9168faSHans Rosenfeld * 1943c9168faSHans Rosenfeld * Driver Configuration: 1953c9168faSHans Rosenfeld * 1963c9168faSHans Rosenfeld * The following driver properties can be changed to control some aspects of the 1973c9168faSHans Rosenfeld * drivers operation: 1983c9168faSHans Rosenfeld * - strict-version: can be set to 0 to allow devices conforming to newer 19924979ca3SHans Rosenfeld * versions or namespaces with EUI64 to be used 2003c9168faSHans Rosenfeld * - ignore-unknown-vendor-status: can be set to 1 to not handle any vendor 2013c9168faSHans Rosenfeld * specific command status as a fatal error leading device faulting 2023c9168faSHans Rosenfeld * - admin-queue-len: the maximum length of the admin queue (16-4096) 2033c9168faSHans Rosenfeld * - io-queue-len: the maximum length of the I/O queues (16-65536) 2043c9168faSHans Rosenfeld * - async-event-limit: the maximum number of asynchronous event requests to be 2053c9168faSHans Rosenfeld * posted by the driver 206d148d46eSHans Rosenfeld * - volatile-write-cache-enable: can be set to 0 to disable the volatile write 207d148d46eSHans Rosenfeld * cache 2086801591eSHans Rosenfeld * - min-phys-block-size: the minimum physical block size to report to blkdev, 2096801591eSHans Rosenfeld * which is among other things the basis for ZFS vdev ashift 2103c9168faSHans Rosenfeld * 2113c9168faSHans Rosenfeld * 2123c9168faSHans Rosenfeld * TODO: 2133c9168faSHans Rosenfeld * - figure out sane default for I/O queue depth reported to blkdev 2143c9168faSHans Rosenfeld * - FMA handling of media errors 2153c9168faSHans Rosenfeld * - support for devices supporting very large I/O requests using chained PRPs 2163c9168faSHans Rosenfeld * - support for configuring hardware parameters like interrupt coalescing 2173c9168faSHans Rosenfeld * - support for media formatting and hard partitioning into namespaces 2183c9168faSHans Rosenfeld * - support for big-endian systems 2193c9168faSHans Rosenfeld * - support for fast reboot 22024979ca3SHans Rosenfeld * - support for firmware updates 22124979ca3SHans Rosenfeld * - support for NVMe Subsystem Reset (1.1) 22224979ca3SHans Rosenfeld * - support for Scatter/Gather lists (1.1) 22324979ca3SHans Rosenfeld * - support for Reservations (1.1) 22424979ca3SHans Rosenfeld * - support for power management 2253c9168faSHans Rosenfeld */ 2263c9168faSHans Rosenfeld 2273c9168faSHans Rosenfeld #include <sys/byteorder.h> 2283c9168faSHans Rosenfeld #ifdef _BIG_ENDIAN 2293c9168faSHans Rosenfeld #error nvme driver needs porting for big-endian platforms 2303c9168faSHans Rosenfeld #endif 2313c9168faSHans Rosenfeld 2323c9168faSHans Rosenfeld #include <sys/modctl.h> 2333c9168faSHans Rosenfeld #include <sys/conf.h> 2343c9168faSHans Rosenfeld #include <sys/devops.h> 2353c9168faSHans Rosenfeld #include <sys/ddi.h> 2363c9168faSHans Rosenfeld #include <sys/sunddi.h> 2373d9b1a2aSHans Rosenfeld #include <sys/sunndi.h> 2383c9168faSHans Rosenfeld #include <sys/bitmap.h> 2393c9168faSHans Rosenfeld #include <sys/sysmacros.h> 2403c9168faSHans Rosenfeld #include <sys/param.h> 2413c9168faSHans Rosenfeld #include <sys/varargs.h> 2423c9168faSHans Rosenfeld #include <sys/cpuvar.h> 2433c9168faSHans Rosenfeld #include <sys/disp.h> 2443c9168faSHans Rosenfeld #include <sys/blkdev.h> 2453c9168faSHans Rosenfeld #include <sys/atomic.h> 2463c9168faSHans Rosenfeld #include <sys/archsystm.h> 247510a6847SHans Rosenfeld #include <sys/sata/sata_hba.h> 2483d9b1a2aSHans Rosenfeld #include <sys/stat.h> 2493d9b1a2aSHans Rosenfeld #include <sys/policy.h> 250e984c70bSHans Rosenfeld #include <sys/list.h> 2513d9b1a2aSHans Rosenfeld 2523d9b1a2aSHans Rosenfeld #include <sys/nvme.h> 2533c9168faSHans Rosenfeld 2549d08e1f8SHans Rosenfeld #ifdef __x86 2559d08e1f8SHans Rosenfeld #include <sys/x86_archext.h> 2569d08e1f8SHans Rosenfeld #endif 2579d08e1f8SHans Rosenfeld 2583c9168faSHans Rosenfeld #include "nvme_reg.h" 2593c9168faSHans Rosenfeld #include "nvme_var.h" 2603c9168faSHans Rosenfeld 2613c9168faSHans Rosenfeld 2623c9168faSHans Rosenfeld /* NVMe spec version supported */ 2633c9168faSHans Rosenfeld static const int nvme_version_major = 1; 264f8dcaea5SRobert Mustacchi static const int nvme_version_minor = 2; 2653c9168faSHans Rosenfeld 266e8ba2a38SHans Rosenfeld /* tunable for admin command timeout in seconds, default is 1s */ 2673d9b1a2aSHans Rosenfeld int nvme_admin_cmd_timeout = 1; 2683d9b1a2aSHans Rosenfeld 2693d9b1a2aSHans Rosenfeld /* tunable for FORMAT NVM command timeout in seconds, default is 600s */ 2703d9b1a2aSHans Rosenfeld int nvme_format_cmd_timeout = 600; 271e8ba2a38SHans Rosenfeld 2723c9168faSHans Rosenfeld static int nvme_attach(dev_info_t *, ddi_attach_cmd_t); 2733c9168faSHans Rosenfeld static int nvme_detach(dev_info_t *, ddi_detach_cmd_t); 2743c9168faSHans Rosenfeld static int nvme_quiesce(dev_info_t *); 2753c9168faSHans Rosenfeld static int nvme_fm_errcb(dev_info_t *, ddi_fm_error_t *, const void *); 2763c9168faSHans Rosenfeld static int nvme_setup_interrupts(nvme_t *, int, int); 2773c9168faSHans Rosenfeld static void nvme_release_interrupts(nvme_t *); 2783c9168faSHans Rosenfeld static uint_t nvme_intr(caddr_t, caddr_t); 2793c9168faSHans Rosenfeld 2803c9168faSHans Rosenfeld static void nvme_shutdown(nvme_t *, int, boolean_t); 2813c9168faSHans Rosenfeld static boolean_t nvme_reset(nvme_t *, boolean_t); 2823c9168faSHans Rosenfeld static int nvme_init(nvme_t *); 2833c9168faSHans Rosenfeld static nvme_cmd_t *nvme_alloc_cmd(nvme_t *, int); 2843c9168faSHans Rosenfeld static void nvme_free_cmd(nvme_cmd_t *); 2853c9168faSHans Rosenfeld static nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t, 2863c9168faSHans Rosenfeld bd_xfer_t *); 287e984c70bSHans Rosenfeld static void nvme_admin_cmd(nvme_cmd_t *, int); 2884b324362SHans Rosenfeld static void nvme_submit_admin_cmd(nvme_qpair_t *, nvme_cmd_t *); 2894b324362SHans Rosenfeld static int nvme_submit_io_cmd(nvme_qpair_t *, nvme_cmd_t *); 2904b324362SHans Rosenfeld static void nvme_submit_cmd_common(nvme_qpair_t *, nvme_cmd_t *); 291e984c70bSHans Rosenfeld static nvme_cmd_t *nvme_unqueue_cmd(nvme_t *, nvme_qpair_t *, int); 2923c9168faSHans Rosenfeld static nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *); 293e984c70bSHans Rosenfeld static void nvme_wait_cmd(nvme_cmd_t *, uint_t); 2943c9168faSHans Rosenfeld static void nvme_wakeup_cmd(void *); 2953c9168faSHans Rosenfeld static void nvme_async_event_task(void *); 2963c9168faSHans Rosenfeld 2973c9168faSHans Rosenfeld static int nvme_check_unknown_cmd_status(nvme_cmd_t *); 2983c9168faSHans Rosenfeld static int nvme_check_vendor_cmd_status(nvme_cmd_t *); 2993c9168faSHans Rosenfeld static int nvme_check_integrity_cmd_status(nvme_cmd_t *); 3003c9168faSHans Rosenfeld static int nvme_check_specific_cmd_status(nvme_cmd_t *); 3013c9168faSHans Rosenfeld static int nvme_check_generic_cmd_status(nvme_cmd_t *); 3023c9168faSHans Rosenfeld static inline int nvme_check_cmd_status(nvme_cmd_t *); 3033c9168faSHans Rosenfeld 304e984c70bSHans Rosenfeld static int nvme_abort_cmd(nvme_cmd_t *, uint_t); 3054b324362SHans Rosenfeld static void nvme_async_event(nvme_t *); 3063d9b1a2aSHans Rosenfeld static int nvme_format_nvm(nvme_t *, uint32_t, uint8_t, boolean_t, uint8_t, 3073d9b1a2aSHans Rosenfeld boolean_t, uint8_t); 3083d9b1a2aSHans Rosenfeld static int nvme_get_logpage(nvme_t *, void **, size_t *, uint8_t, ...); 309e984c70bSHans Rosenfeld static int nvme_identify(nvme_t *, uint32_t, void **); 310e984c70bSHans Rosenfeld static int nvme_set_features(nvme_t *, uint32_t, uint8_t, uint32_t, 311d148d46eSHans Rosenfeld uint32_t *); 312e984c70bSHans Rosenfeld static int nvme_get_features(nvme_t *, uint32_t, uint8_t, uint32_t *, 3133d9b1a2aSHans Rosenfeld void **, size_t *); 314e984c70bSHans Rosenfeld static int nvme_write_cache_set(nvme_t *, boolean_t); 315e984c70bSHans Rosenfeld static int nvme_set_nqueues(nvme_t *, uint16_t *); 3163c9168faSHans Rosenfeld 3173c9168faSHans Rosenfeld static void nvme_free_dma(nvme_dma_t *); 3183c9168faSHans Rosenfeld static int nvme_zalloc_dma(nvme_t *, size_t, uint_t, ddi_dma_attr_t *, 3193c9168faSHans Rosenfeld nvme_dma_t **); 3203c9168faSHans Rosenfeld static int nvme_zalloc_queue_dma(nvme_t *, uint32_t, uint16_t, uint_t, 3213c9168faSHans Rosenfeld nvme_dma_t **); 3223c9168faSHans Rosenfeld static void nvme_free_qpair(nvme_qpair_t *); 3233c9168faSHans Rosenfeld static int nvme_alloc_qpair(nvme_t *, uint32_t, nvme_qpair_t **, int); 3243c9168faSHans Rosenfeld static int nvme_create_io_qpair(nvme_t *, nvme_qpair_t *, uint16_t); 3253c9168faSHans Rosenfeld 3263c9168faSHans Rosenfeld static inline void nvme_put64(nvme_t *, uintptr_t, uint64_t); 3273c9168faSHans Rosenfeld static inline void nvme_put32(nvme_t *, uintptr_t, uint32_t); 3283c9168faSHans Rosenfeld static inline uint64_t nvme_get64(nvme_t *, uintptr_t); 3293c9168faSHans Rosenfeld static inline uint32_t nvme_get32(nvme_t *, uintptr_t); 3303c9168faSHans Rosenfeld 3313c9168faSHans Rosenfeld static boolean_t nvme_check_regs_hdl(nvme_t *); 3323c9168faSHans Rosenfeld static boolean_t nvme_check_dma_hdl(nvme_dma_t *); 3333c9168faSHans Rosenfeld 3343c9168faSHans Rosenfeld static int nvme_fill_prp(nvme_cmd_t *, bd_xfer_t *); 3353c9168faSHans Rosenfeld 3363c9168faSHans Rosenfeld static void nvme_bd_xfer_done(void *); 3373c9168faSHans Rosenfeld static void nvme_bd_driveinfo(void *, bd_drive_t *); 3383c9168faSHans Rosenfeld static int nvme_bd_mediainfo(void *, bd_media_t *); 3393c9168faSHans Rosenfeld static int nvme_bd_cmd(nvme_namespace_t *, bd_xfer_t *, uint8_t); 3403c9168faSHans Rosenfeld static int nvme_bd_read(void *, bd_xfer_t *); 3413c9168faSHans Rosenfeld static int nvme_bd_write(void *, bd_xfer_t *); 3423c9168faSHans Rosenfeld static int nvme_bd_sync(void *, bd_xfer_t *); 3433c9168faSHans Rosenfeld static int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *); 3443c9168faSHans Rosenfeld 3458834f7acSYouzhong Yang static int nvme_prp_dma_constructor(void *, void *, int); 3468834f7acSYouzhong Yang static void nvme_prp_dma_destructor(void *, void *); 3478834f7acSYouzhong Yang 3483c9168faSHans Rosenfeld static void nvme_prepare_devid(nvme_t *, uint32_t); 3493c9168faSHans Rosenfeld 3503d9b1a2aSHans Rosenfeld static int nvme_open(dev_t *, int, int, cred_t *); 3513d9b1a2aSHans Rosenfeld static int nvme_close(dev_t, int, int, cred_t *); 3523d9b1a2aSHans Rosenfeld static int nvme_ioctl(dev_t, int, intptr_t, int, cred_t *, int *); 3533d9b1a2aSHans Rosenfeld 354dc97a43dSHans Rosenfeld #define NVME_MINOR_INST_SHIFT 9 3553d9b1a2aSHans Rosenfeld #define NVME_MINOR(inst, nsid) (((inst) << NVME_MINOR_INST_SHIFT) | (nsid)) 3563d9b1a2aSHans Rosenfeld #define NVME_MINOR_INST(minor) ((minor) >> NVME_MINOR_INST_SHIFT) 3573d9b1a2aSHans Rosenfeld #define NVME_MINOR_NSID(minor) ((minor) & ((1 << NVME_MINOR_INST_SHIFT) - 1)) 3583d9b1a2aSHans Rosenfeld #define NVME_MINOR_MAX (NVME_MINOR(1, 0) - 2) 3593d9b1a2aSHans Rosenfeld 3603c9168faSHans Rosenfeld static void *nvme_state; 3613c9168faSHans Rosenfeld static kmem_cache_t *nvme_cmd_cache; 3623c9168faSHans Rosenfeld 3633c9168faSHans Rosenfeld /* 3643c9168faSHans Rosenfeld * DMA attributes for queue DMA memory 3653c9168faSHans Rosenfeld * 3663c9168faSHans Rosenfeld * Queue DMA memory must be page aligned. The maximum length of a queue is 3673c9168faSHans Rosenfeld * 65536 entries, and an entry can be 64 bytes long. 3683c9168faSHans Rosenfeld */ 3693c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_queue_dma_attr = { 3703c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 3713c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 3723c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 373910f0d12SYouzhong Yang .dma_attr_count_max = (UINT16_MAX + 1) * sizeof (nvme_sqe_t) - 1, 3743c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 3753c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 3763c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 3773c9168faSHans Rosenfeld .dma_attr_maxxfer = (UINT16_MAX + 1) * sizeof (nvme_sqe_t), 3783c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 3793c9168faSHans Rosenfeld .dma_attr_sgllen = 1, 3803c9168faSHans Rosenfeld .dma_attr_granular = 1, 3813c9168faSHans Rosenfeld .dma_attr_flags = 0, 3823c9168faSHans Rosenfeld }; 3833c9168faSHans Rosenfeld 3843c9168faSHans Rosenfeld /* 3853c9168faSHans Rosenfeld * DMA attributes for transfers using Physical Region Page (PRP) entries 3863c9168faSHans Rosenfeld * 3873c9168faSHans Rosenfeld * A PRP entry describes one page of DMA memory using the page size specified 3883c9168faSHans Rosenfeld * in the controller configuration's memory page size register (CC.MPS). It uses 3893c9168faSHans Rosenfeld * a 64bit base address aligned to this page size. There is no limitation on 3903c9168faSHans Rosenfeld * chaining PRPs together for arbitrarily large DMA transfers. 3913c9168faSHans Rosenfeld */ 3923c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_prp_dma_attr = { 3933c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 3943c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 3953c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 3963c9168faSHans Rosenfeld .dma_attr_count_max = 0xfff, 3973c9168faSHans Rosenfeld .dma_attr_align = 0x1000, 3983c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 3993c9168faSHans Rosenfeld .dma_attr_minxfer = 0x1000, 4003c9168faSHans Rosenfeld .dma_attr_maxxfer = 0x1000, 4012f95345bSYouzhong Yang .dma_attr_seg = 0xfff, 4023c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 4033c9168faSHans Rosenfeld .dma_attr_granular = 1, 4043c9168faSHans Rosenfeld .dma_attr_flags = 0, 4053c9168faSHans Rosenfeld }; 4063c9168faSHans Rosenfeld 4073c9168faSHans Rosenfeld /* 4083c9168faSHans Rosenfeld * DMA attributes for transfers using scatter/gather lists 4093c9168faSHans Rosenfeld * 4103c9168faSHans Rosenfeld * A SGL entry describes a chunk of DMA memory using a 64bit base address and a 4113c9168faSHans Rosenfeld * 32bit length field. SGL Segment and SGL Last Segment entries require the 4123c9168faSHans Rosenfeld * length to be a multiple of 16 bytes. 4133c9168faSHans Rosenfeld */ 4143c9168faSHans Rosenfeld static ddi_dma_attr_t nvme_sgl_dma_attr = { 4153c9168faSHans Rosenfeld .dma_attr_version = DMA_ATTR_V0, 4163c9168faSHans Rosenfeld .dma_attr_addr_lo = 0, 4173c9168faSHans Rosenfeld .dma_attr_addr_hi = 0xffffffffffffffffULL, 4183c9168faSHans Rosenfeld .dma_attr_count_max = 0xffffffffUL, 4193c9168faSHans Rosenfeld .dma_attr_align = 1, 4203c9168faSHans Rosenfeld .dma_attr_burstsizes = 0x7ff, 4213c9168faSHans Rosenfeld .dma_attr_minxfer = 0x10, 4223c9168faSHans Rosenfeld .dma_attr_maxxfer = 0xfffffffffULL, 4233c9168faSHans Rosenfeld .dma_attr_seg = 0xffffffffffffffffULL, 4243c9168faSHans Rosenfeld .dma_attr_sgllen = -1, 4253c9168faSHans Rosenfeld .dma_attr_granular = 0x10, 4263c9168faSHans Rosenfeld .dma_attr_flags = 0 4273c9168faSHans Rosenfeld }; 4283c9168faSHans Rosenfeld 4293c9168faSHans Rosenfeld static ddi_device_acc_attr_t nvme_reg_acc_attr = { 4303c9168faSHans Rosenfeld .devacc_attr_version = DDI_DEVICE_ATTR_V0, 4313c9168faSHans Rosenfeld .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 4323c9168faSHans Rosenfeld .devacc_attr_dataorder = DDI_STRICTORDER_ACC 4333c9168faSHans Rosenfeld }; 4343c9168faSHans Rosenfeld 4353d9b1a2aSHans Rosenfeld static struct cb_ops nvme_cb_ops = { 4363d9b1a2aSHans Rosenfeld .cb_open = nvme_open, 4373d9b1a2aSHans Rosenfeld .cb_close = nvme_close, 4383d9b1a2aSHans Rosenfeld .cb_strategy = nodev, 4393d9b1a2aSHans Rosenfeld .cb_print = nodev, 4403d9b1a2aSHans Rosenfeld .cb_dump = nodev, 4413d9b1a2aSHans Rosenfeld .cb_read = nodev, 4423d9b1a2aSHans Rosenfeld .cb_write = nodev, 4433d9b1a2aSHans Rosenfeld .cb_ioctl = nvme_ioctl, 4443d9b1a2aSHans Rosenfeld .cb_devmap = nodev, 4453d9b1a2aSHans Rosenfeld .cb_mmap = nodev, 4463d9b1a2aSHans Rosenfeld .cb_segmap = nodev, 4473d9b1a2aSHans Rosenfeld .cb_chpoll = nochpoll, 4483d9b1a2aSHans Rosenfeld .cb_prop_op = ddi_prop_op, 4493d9b1a2aSHans Rosenfeld .cb_str = 0, 4503d9b1a2aSHans Rosenfeld .cb_flag = D_NEW | D_MP, 4513d9b1a2aSHans Rosenfeld .cb_rev = CB_REV, 4523d9b1a2aSHans Rosenfeld .cb_aread = nodev, 4533d9b1a2aSHans Rosenfeld .cb_awrite = nodev 4543d9b1a2aSHans Rosenfeld }; 4553d9b1a2aSHans Rosenfeld 4563c9168faSHans Rosenfeld static struct dev_ops nvme_dev_ops = { 4573c9168faSHans Rosenfeld .devo_rev = DEVO_REV, 4583c9168faSHans Rosenfeld .devo_refcnt = 0, 4593c9168faSHans Rosenfeld .devo_getinfo = ddi_no_info, 4603c9168faSHans Rosenfeld .devo_identify = nulldev, 4613c9168faSHans Rosenfeld .devo_probe = nulldev, 4623c9168faSHans Rosenfeld .devo_attach = nvme_attach, 4633c9168faSHans Rosenfeld .devo_detach = nvme_detach, 4643c9168faSHans Rosenfeld .devo_reset = nodev, 4653d9b1a2aSHans Rosenfeld .devo_cb_ops = &nvme_cb_ops, 4663c9168faSHans Rosenfeld .devo_bus_ops = NULL, 4673c9168faSHans Rosenfeld .devo_power = NULL, 4683c9168faSHans Rosenfeld .devo_quiesce = nvme_quiesce, 4693c9168faSHans Rosenfeld }; 4703c9168faSHans Rosenfeld 4713c9168faSHans Rosenfeld static struct modldrv nvme_modldrv = { 4723c9168faSHans Rosenfeld .drv_modops = &mod_driverops, 47324979ca3SHans Rosenfeld .drv_linkinfo = "NVMe v1.1b", 4743c9168faSHans Rosenfeld .drv_dev_ops = &nvme_dev_ops 4753c9168faSHans Rosenfeld }; 4763c9168faSHans Rosenfeld 4773c9168faSHans Rosenfeld static struct modlinkage nvme_modlinkage = { 4783c9168faSHans Rosenfeld .ml_rev = MODREV_1, 4793c9168faSHans Rosenfeld .ml_linkage = { &nvme_modldrv, NULL } 4803c9168faSHans Rosenfeld }; 4813c9168faSHans Rosenfeld 4823c9168faSHans Rosenfeld static bd_ops_t nvme_bd_ops = { 4833c9168faSHans Rosenfeld .o_version = BD_OPS_VERSION_0, 4843c9168faSHans Rosenfeld .o_drive_info = nvme_bd_driveinfo, 4853c9168faSHans Rosenfeld .o_media_info = nvme_bd_mediainfo, 4863c9168faSHans Rosenfeld .o_devid_init = nvme_bd_devid, 4873c9168faSHans Rosenfeld .o_sync_cache = nvme_bd_sync, 4883c9168faSHans Rosenfeld .o_read = nvme_bd_read, 4893c9168faSHans Rosenfeld .o_write = nvme_bd_write, 4903c9168faSHans Rosenfeld }; 4913c9168faSHans Rosenfeld 492e984c70bSHans Rosenfeld /* 493e984c70bSHans Rosenfeld * This list will hold commands that have timed out and couldn't be aborted. 494e984c70bSHans Rosenfeld * As we don't know what the hardware may still do with the DMA memory we can't 495e984c70bSHans Rosenfeld * free them, so we'll keep them forever on this list where we can easily look 496e984c70bSHans Rosenfeld * at them with mdb. 497e984c70bSHans Rosenfeld */ 498e984c70bSHans Rosenfeld static struct list nvme_lost_cmds; 499e984c70bSHans Rosenfeld static kmutex_t nvme_lc_mutex; 500e984c70bSHans Rosenfeld 5013c9168faSHans Rosenfeld int 5023c9168faSHans Rosenfeld _init(void) 5033c9168faSHans Rosenfeld { 5043c9168faSHans Rosenfeld int error; 5053c9168faSHans Rosenfeld 5063c9168faSHans Rosenfeld error = ddi_soft_state_init(&nvme_state, sizeof (nvme_t), 1); 5073c9168faSHans Rosenfeld if (error != DDI_SUCCESS) 5083c9168faSHans Rosenfeld return (error); 5093c9168faSHans Rosenfeld 5103c9168faSHans Rosenfeld nvme_cmd_cache = kmem_cache_create("nvme_cmd_cache", 5113c9168faSHans Rosenfeld sizeof (nvme_cmd_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 5123c9168faSHans Rosenfeld 513e984c70bSHans Rosenfeld mutex_init(&nvme_lc_mutex, NULL, MUTEX_DRIVER, NULL); 514e984c70bSHans Rosenfeld list_create(&nvme_lost_cmds, sizeof (nvme_cmd_t), 515e984c70bSHans Rosenfeld offsetof(nvme_cmd_t, nc_list)); 516e984c70bSHans Rosenfeld 5173c9168faSHans Rosenfeld bd_mod_init(&nvme_dev_ops); 5183c9168faSHans Rosenfeld 5193c9168faSHans Rosenfeld error = mod_install(&nvme_modlinkage); 5203c9168faSHans Rosenfeld if (error != DDI_SUCCESS) { 5213c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 522e984c70bSHans Rosenfeld mutex_destroy(&nvme_lc_mutex); 523e984c70bSHans Rosenfeld list_destroy(&nvme_lost_cmds); 5243c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 5253c9168faSHans Rosenfeld } 5263c9168faSHans Rosenfeld 5273c9168faSHans Rosenfeld return (error); 5283c9168faSHans Rosenfeld } 5293c9168faSHans Rosenfeld 5303c9168faSHans Rosenfeld int 5313c9168faSHans Rosenfeld _fini(void) 5323c9168faSHans Rosenfeld { 5333c9168faSHans Rosenfeld int error; 5343c9168faSHans Rosenfeld 535e984c70bSHans Rosenfeld if (!list_is_empty(&nvme_lost_cmds)) 536e984c70bSHans Rosenfeld return (DDI_FAILURE); 537e984c70bSHans Rosenfeld 5383c9168faSHans Rosenfeld error = mod_remove(&nvme_modlinkage); 5393c9168faSHans Rosenfeld if (error == DDI_SUCCESS) { 5403c9168faSHans Rosenfeld ddi_soft_state_fini(&nvme_state); 5413c9168faSHans Rosenfeld kmem_cache_destroy(nvme_cmd_cache); 542e984c70bSHans Rosenfeld mutex_destroy(&nvme_lc_mutex); 543e984c70bSHans Rosenfeld list_destroy(&nvme_lost_cmds); 5443c9168faSHans Rosenfeld bd_mod_fini(&nvme_dev_ops); 5453c9168faSHans Rosenfeld } 5463c9168faSHans Rosenfeld 5473c9168faSHans Rosenfeld return (error); 5483c9168faSHans Rosenfeld } 5493c9168faSHans Rosenfeld 5503c9168faSHans Rosenfeld int 5513c9168faSHans Rosenfeld _info(struct modinfo *modinfop) 5523c9168faSHans Rosenfeld { 5533c9168faSHans Rosenfeld return (mod_info(&nvme_modlinkage, modinfop)); 5543c9168faSHans Rosenfeld } 5553c9168faSHans Rosenfeld 5563c9168faSHans Rosenfeld static inline void 5573c9168faSHans Rosenfeld nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val) 5583c9168faSHans Rosenfeld { 5593c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 5603c9168faSHans Rosenfeld 5613c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 5623c9168faSHans Rosenfeld ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val); 5633c9168faSHans Rosenfeld } 5643c9168faSHans Rosenfeld 5653c9168faSHans Rosenfeld static inline void 5663c9168faSHans Rosenfeld nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val) 5673c9168faSHans Rosenfeld { 5683c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 5693c9168faSHans Rosenfeld 5703c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 5713c9168faSHans Rosenfeld ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val); 5723c9168faSHans Rosenfeld } 5733c9168faSHans Rosenfeld 5743c9168faSHans Rosenfeld static inline uint64_t 5753c9168faSHans Rosenfeld nvme_get64(nvme_t *nvme, uintptr_t reg) 5763c9168faSHans Rosenfeld { 5773c9168faSHans Rosenfeld uint64_t val; 5783c9168faSHans Rosenfeld 5793c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 5803c9168faSHans Rosenfeld 5813c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 5823c9168faSHans Rosenfeld val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg)); 5833c9168faSHans Rosenfeld 5843c9168faSHans Rosenfeld return (val); 5853c9168faSHans Rosenfeld } 5863c9168faSHans Rosenfeld 5873c9168faSHans Rosenfeld static inline uint32_t 5883c9168faSHans Rosenfeld nvme_get32(nvme_t *nvme, uintptr_t reg) 5893c9168faSHans Rosenfeld { 5903c9168faSHans Rosenfeld uint32_t val; 5913c9168faSHans Rosenfeld 5923c9168faSHans Rosenfeld ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 5933c9168faSHans Rosenfeld 5943c9168faSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 5953c9168faSHans Rosenfeld val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg)); 5963c9168faSHans Rosenfeld 5973c9168faSHans Rosenfeld return (val); 5983c9168faSHans Rosenfeld } 5993c9168faSHans Rosenfeld 6003c9168faSHans Rosenfeld static boolean_t 6013c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme_t *nvme) 6023c9168faSHans Rosenfeld { 6033c9168faSHans Rosenfeld ddi_fm_error_t error; 6043c9168faSHans Rosenfeld 6053c9168faSHans Rosenfeld ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION); 6063c9168faSHans Rosenfeld 6073c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 6083c9168faSHans Rosenfeld return (B_TRUE); 6093c9168faSHans Rosenfeld 6103c9168faSHans Rosenfeld return (B_FALSE); 6113c9168faSHans Rosenfeld } 6123c9168faSHans Rosenfeld 6133c9168faSHans Rosenfeld static boolean_t 6143c9168faSHans Rosenfeld nvme_check_dma_hdl(nvme_dma_t *dma) 6153c9168faSHans Rosenfeld { 6163c9168faSHans Rosenfeld ddi_fm_error_t error; 6173c9168faSHans Rosenfeld 6183c9168faSHans Rosenfeld if (dma == NULL) 6193c9168faSHans Rosenfeld return (B_FALSE); 6203c9168faSHans Rosenfeld 6213c9168faSHans Rosenfeld ddi_fm_dma_err_get(dma->nd_dmah, &error, DDI_FME_VERSION); 6223c9168faSHans Rosenfeld 6233c9168faSHans Rosenfeld if (error.fme_status != DDI_FM_OK) 6243c9168faSHans Rosenfeld return (B_TRUE); 6253c9168faSHans Rosenfeld 6263c9168faSHans Rosenfeld return (B_FALSE); 6273c9168faSHans Rosenfeld } 6283c9168faSHans Rosenfeld 6293c9168faSHans Rosenfeld static void 6308834f7acSYouzhong Yang nvme_free_dma_common(nvme_dma_t *dma) 6313c9168faSHans Rosenfeld { 6323c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 6333c9168faSHans Rosenfeld (void) ddi_dma_unbind_handle(dma->nd_dmah); 6343c9168faSHans Rosenfeld if (dma->nd_acch != NULL) 6353c9168faSHans Rosenfeld ddi_dma_mem_free(&dma->nd_acch); 6363c9168faSHans Rosenfeld if (dma->nd_dmah != NULL) 6373c9168faSHans Rosenfeld ddi_dma_free_handle(&dma->nd_dmah); 6383c9168faSHans Rosenfeld } 6393c9168faSHans Rosenfeld 6408834f7acSYouzhong Yang static void 6418834f7acSYouzhong Yang nvme_free_dma(nvme_dma_t *dma) 6423c9168faSHans Rosenfeld { 6438834f7acSYouzhong Yang nvme_free_dma_common(dma); 6448834f7acSYouzhong Yang kmem_free(dma, sizeof (*dma)); 6458834f7acSYouzhong Yang } 6468834f7acSYouzhong Yang 647b6bc2fd4SDan McDonald /* ARGSUSED */ 6488834f7acSYouzhong Yang static void 6498834f7acSYouzhong Yang nvme_prp_dma_destructor(void *buf, void *private) 6508834f7acSYouzhong Yang { 6518834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 6523c9168faSHans Rosenfeld 6538834f7acSYouzhong Yang nvme_free_dma_common(dma); 6548834f7acSYouzhong Yang } 6558834f7acSYouzhong Yang 6568834f7acSYouzhong Yang static int 6578834f7acSYouzhong Yang nvme_alloc_dma_common(nvme_t *nvme, nvme_dma_t *dma, 6588834f7acSYouzhong Yang size_t len, uint_t flags, ddi_dma_attr_t *dma_attr) 6598834f7acSYouzhong Yang { 6603c9168faSHans Rosenfeld if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL, 6613c9168faSHans Rosenfeld &dma->nd_dmah) != DDI_SUCCESS) { 6623c9168faSHans Rosenfeld /* 6633c9168faSHans Rosenfeld * Due to DDI_DMA_SLEEP this can't be DDI_DMA_NORESOURCES, and 6643c9168faSHans Rosenfeld * the only other possible error is DDI_DMA_BADATTR which 6653c9168faSHans Rosenfeld * indicates a driver bug which should cause a panic. 6663c9168faSHans Rosenfeld */ 6673c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, 6683c9168faSHans Rosenfeld "!failed to get DMA handle, check DMA attributes"); 6693c9168faSHans Rosenfeld return (DDI_FAILURE); 6703c9168faSHans Rosenfeld } 6713c9168faSHans Rosenfeld 6723c9168faSHans Rosenfeld /* 6733c9168faSHans Rosenfeld * ddi_dma_mem_alloc() can only fail when DDI_DMA_NOSLEEP is specified 6743c9168faSHans Rosenfeld * or the flags are conflicting, which isn't the case here. 6753c9168faSHans Rosenfeld */ 6763c9168faSHans Rosenfeld (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr, 6773c9168faSHans Rosenfeld DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &dma->nd_memp, 6783c9168faSHans Rosenfeld &dma->nd_len, &dma->nd_acch); 6793c9168faSHans Rosenfeld 6803c9168faSHans Rosenfeld if (ddi_dma_addr_bind_handle(dma->nd_dmah, NULL, dma->nd_memp, 6813c9168faSHans Rosenfeld dma->nd_len, flags | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, 6823c9168faSHans Rosenfeld &dma->nd_cookie, &dma->nd_ncookie) != DDI_DMA_MAPPED) { 6833c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 6843c9168faSHans Rosenfeld "!failed to bind DMA memory"); 6853c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_dma_bind_err); 6868834f7acSYouzhong Yang nvme_free_dma_common(dma); 6878834f7acSYouzhong Yang return (DDI_FAILURE); 6888834f7acSYouzhong Yang } 6898834f7acSYouzhong Yang 6908834f7acSYouzhong Yang return (DDI_SUCCESS); 6918834f7acSYouzhong Yang } 6928834f7acSYouzhong Yang 6938834f7acSYouzhong Yang static int 6948834f7acSYouzhong Yang nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags, 6958834f7acSYouzhong Yang ddi_dma_attr_t *dma_attr, nvme_dma_t **ret) 6968834f7acSYouzhong Yang { 6978834f7acSYouzhong Yang nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP); 6988834f7acSYouzhong Yang 6998834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, len, flags, dma_attr) != 7008834f7acSYouzhong Yang DDI_SUCCESS) { 7013c9168faSHans Rosenfeld *ret = NULL; 7028834f7acSYouzhong Yang kmem_free(dma, sizeof (nvme_dma_t)); 7033c9168faSHans Rosenfeld return (DDI_FAILURE); 7043c9168faSHans Rosenfeld } 7053c9168faSHans Rosenfeld 7063c9168faSHans Rosenfeld bzero(dma->nd_memp, dma->nd_len); 7073c9168faSHans Rosenfeld 7083c9168faSHans Rosenfeld *ret = dma; 7093c9168faSHans Rosenfeld return (DDI_SUCCESS); 7103c9168faSHans Rosenfeld } 7113c9168faSHans Rosenfeld 712b6bc2fd4SDan McDonald /* ARGSUSED */ 7138834f7acSYouzhong Yang static int 7148834f7acSYouzhong Yang nvme_prp_dma_constructor(void *buf, void *private, int flags) 7158834f7acSYouzhong Yang { 7168834f7acSYouzhong Yang nvme_dma_t *dma = (nvme_dma_t *)buf; 7178834f7acSYouzhong Yang nvme_t *nvme = (nvme_t *)private; 7188834f7acSYouzhong Yang 7198834f7acSYouzhong Yang dma->nd_dmah = NULL; 7208834f7acSYouzhong Yang dma->nd_acch = NULL; 7218834f7acSYouzhong Yang 7228834f7acSYouzhong Yang if (nvme_alloc_dma_common(nvme, dma, nvme->n_pagesize, 7238834f7acSYouzhong Yang DDI_DMA_READ, &nvme->n_prp_dma_attr) != DDI_SUCCESS) { 7248834f7acSYouzhong Yang return (-1); 7258834f7acSYouzhong Yang } 7268834f7acSYouzhong Yang 7278834f7acSYouzhong Yang ASSERT(dma->nd_ncookie == 1); 7288834f7acSYouzhong Yang 7298834f7acSYouzhong Yang dma->nd_cached = B_TRUE; 7308834f7acSYouzhong Yang 7318834f7acSYouzhong Yang return (0); 7328834f7acSYouzhong Yang } 7338834f7acSYouzhong Yang 7343c9168faSHans Rosenfeld static int 7353c9168faSHans Rosenfeld nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len, 7363c9168faSHans Rosenfeld uint_t flags, nvme_dma_t **dma) 7373c9168faSHans Rosenfeld { 7383c9168faSHans Rosenfeld uint32_t len = nentry * qe_len; 7393c9168faSHans Rosenfeld ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr; 7403c9168faSHans Rosenfeld 7413c9168faSHans Rosenfeld len = roundup(len, nvme->n_pagesize); 7423c9168faSHans Rosenfeld 7433c9168faSHans Rosenfeld q_dma_attr.dma_attr_minxfer = len; 7443c9168faSHans Rosenfeld 7453c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma) 7463c9168faSHans Rosenfeld != DDI_SUCCESS) { 7473c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 7483c9168faSHans Rosenfeld "!failed to get DMA memory for queue"); 7493c9168faSHans Rosenfeld goto fail; 7503c9168faSHans Rosenfeld } 7513c9168faSHans Rosenfeld 7523c9168faSHans Rosenfeld if ((*dma)->nd_ncookie != 1) { 7533c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 7543c9168faSHans Rosenfeld "!got too many cookies for queue DMA"); 7553c9168faSHans Rosenfeld goto fail; 7563c9168faSHans Rosenfeld } 7573c9168faSHans Rosenfeld 7583c9168faSHans Rosenfeld return (DDI_SUCCESS); 7593c9168faSHans Rosenfeld 7603c9168faSHans Rosenfeld fail: 7613c9168faSHans Rosenfeld if (*dma) { 7623c9168faSHans Rosenfeld nvme_free_dma(*dma); 7633c9168faSHans Rosenfeld *dma = NULL; 7643c9168faSHans Rosenfeld } 7653c9168faSHans Rosenfeld 7663c9168faSHans Rosenfeld return (DDI_FAILURE); 7673c9168faSHans Rosenfeld } 7683c9168faSHans Rosenfeld 7693c9168faSHans Rosenfeld static void 7703c9168faSHans Rosenfeld nvme_free_qpair(nvme_qpair_t *qp) 7713c9168faSHans Rosenfeld { 7723c9168faSHans Rosenfeld int i; 7733c9168faSHans Rosenfeld 7743c9168faSHans Rosenfeld mutex_destroy(&qp->nq_mutex); 7754b324362SHans Rosenfeld sema_destroy(&qp->nq_sema); 7763c9168faSHans Rosenfeld 7773c9168faSHans Rosenfeld if (qp->nq_sqdma != NULL) 7783c9168faSHans Rosenfeld nvme_free_dma(qp->nq_sqdma); 7793c9168faSHans Rosenfeld if (qp->nq_cqdma != NULL) 7803c9168faSHans Rosenfeld nvme_free_dma(qp->nq_cqdma); 7813c9168faSHans Rosenfeld 7823c9168faSHans Rosenfeld if (qp->nq_active_cmds > 0) 7833c9168faSHans Rosenfeld for (i = 0; i != qp->nq_nentry; i++) 7843c9168faSHans Rosenfeld if (qp->nq_cmd[i] != NULL) 7853c9168faSHans Rosenfeld nvme_free_cmd(qp->nq_cmd[i]); 7863c9168faSHans Rosenfeld 7873c9168faSHans Rosenfeld if (qp->nq_cmd != NULL) 7883c9168faSHans Rosenfeld kmem_free(qp->nq_cmd, sizeof (nvme_cmd_t *) * qp->nq_nentry); 7893c9168faSHans Rosenfeld 7903c9168faSHans Rosenfeld kmem_free(qp, sizeof (nvme_qpair_t)); 7913c9168faSHans Rosenfeld } 7923c9168faSHans Rosenfeld 7933c9168faSHans Rosenfeld static int 7943c9168faSHans Rosenfeld nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp, 7953c9168faSHans Rosenfeld int idx) 7963c9168faSHans Rosenfeld { 7973c9168faSHans Rosenfeld nvme_qpair_t *qp = kmem_zalloc(sizeof (*qp), KM_SLEEP); 7983c9168faSHans Rosenfeld 7993c9168faSHans Rosenfeld mutex_init(&qp->nq_mutex, NULL, MUTEX_DRIVER, 8003c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 8014b324362SHans Rosenfeld sema_init(&qp->nq_sema, nentry, NULL, SEMA_DRIVER, NULL); 8023c9168faSHans Rosenfeld 8033c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t), 8043c9168faSHans Rosenfeld DDI_DMA_WRITE, &qp->nq_sqdma) != DDI_SUCCESS) 8053c9168faSHans Rosenfeld goto fail; 8063c9168faSHans Rosenfeld 8073c9168faSHans Rosenfeld if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t), 8083c9168faSHans Rosenfeld DDI_DMA_READ, &qp->nq_cqdma) != DDI_SUCCESS) 8093c9168faSHans Rosenfeld goto fail; 8103c9168faSHans Rosenfeld 8113c9168faSHans Rosenfeld qp->nq_sq = (nvme_sqe_t *)qp->nq_sqdma->nd_memp; 8123c9168faSHans Rosenfeld qp->nq_cq = (nvme_cqe_t *)qp->nq_cqdma->nd_memp; 8133c9168faSHans Rosenfeld qp->nq_nentry = nentry; 8143c9168faSHans Rosenfeld 8153c9168faSHans Rosenfeld qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx); 8163c9168faSHans Rosenfeld qp->nq_cqhdbl = NVME_REG_CQHDBL(nvme, idx); 8173c9168faSHans Rosenfeld 8183c9168faSHans Rosenfeld qp->nq_cmd = kmem_zalloc(sizeof (nvme_cmd_t *) * nentry, KM_SLEEP); 8193c9168faSHans Rosenfeld qp->nq_next_cmd = 0; 8203c9168faSHans Rosenfeld 8213c9168faSHans Rosenfeld *nqp = qp; 8223c9168faSHans Rosenfeld return (DDI_SUCCESS); 8233c9168faSHans Rosenfeld 8243c9168faSHans Rosenfeld fail: 8253c9168faSHans Rosenfeld nvme_free_qpair(qp); 8263c9168faSHans Rosenfeld *nqp = NULL; 8273c9168faSHans Rosenfeld 8283c9168faSHans Rosenfeld return (DDI_FAILURE); 8293c9168faSHans Rosenfeld } 8303c9168faSHans Rosenfeld 8313c9168faSHans Rosenfeld static nvme_cmd_t * 8323c9168faSHans Rosenfeld nvme_alloc_cmd(nvme_t *nvme, int kmflag) 8333c9168faSHans Rosenfeld { 8343c9168faSHans Rosenfeld nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag); 8353c9168faSHans Rosenfeld 8363c9168faSHans Rosenfeld if (cmd == NULL) 8373c9168faSHans Rosenfeld return (cmd); 8383c9168faSHans Rosenfeld 8393c9168faSHans Rosenfeld bzero(cmd, sizeof (nvme_cmd_t)); 8403c9168faSHans Rosenfeld 8413c9168faSHans Rosenfeld cmd->nc_nvme = nvme; 8423c9168faSHans Rosenfeld 8433c9168faSHans Rosenfeld mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER, 8443c9168faSHans Rosenfeld DDI_INTR_PRI(nvme->n_intr_pri)); 8453c9168faSHans Rosenfeld cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL); 8463c9168faSHans Rosenfeld 8473c9168faSHans Rosenfeld return (cmd); 8483c9168faSHans Rosenfeld } 8493c9168faSHans Rosenfeld 8503c9168faSHans Rosenfeld static void 8513c9168faSHans Rosenfeld nvme_free_cmd(nvme_cmd_t *cmd) 8523c9168faSHans Rosenfeld { 853e984c70bSHans Rosenfeld /* Don't free commands on the lost commands list. */ 854e984c70bSHans Rosenfeld if (list_link_active(&cmd->nc_list)) 855e984c70bSHans Rosenfeld return; 856e984c70bSHans Rosenfeld 8573c9168faSHans Rosenfeld if (cmd->nc_dma) { 8588834f7acSYouzhong Yang if (cmd->nc_dma->nd_cached) 8598834f7acSYouzhong Yang kmem_cache_free(cmd->nc_nvme->n_prp_cache, 8608834f7acSYouzhong Yang cmd->nc_dma); 8618834f7acSYouzhong Yang else 8628834f7acSYouzhong Yang nvme_free_dma(cmd->nc_dma); 8633c9168faSHans Rosenfeld cmd->nc_dma = NULL; 8643c9168faSHans Rosenfeld } 8653c9168faSHans Rosenfeld 8663c9168faSHans Rosenfeld cv_destroy(&cmd->nc_cv); 8673c9168faSHans Rosenfeld mutex_destroy(&cmd->nc_mutex); 8683c9168faSHans Rosenfeld 8693c9168faSHans Rosenfeld kmem_cache_free(nvme_cmd_cache, cmd); 8703c9168faSHans Rosenfeld } 8713c9168faSHans Rosenfeld 8724b324362SHans Rosenfeld static void 8734b324362SHans Rosenfeld nvme_submit_admin_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 8744b324362SHans Rosenfeld { 8754b324362SHans Rosenfeld sema_p(&qp->nq_sema); 8764b324362SHans Rosenfeld nvme_submit_cmd_common(qp, cmd); 8774b324362SHans Rosenfeld } 8784b324362SHans Rosenfeld 8793c9168faSHans Rosenfeld static int 8804b324362SHans Rosenfeld nvme_submit_io_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 8813c9168faSHans Rosenfeld { 8824b324362SHans Rosenfeld if (sema_tryp(&qp->nq_sema) == 0) 8834b324362SHans Rosenfeld return (EAGAIN); 8843c9168faSHans Rosenfeld 8854b324362SHans Rosenfeld nvme_submit_cmd_common(qp, cmd); 8864b324362SHans Rosenfeld return (0); 8874b324362SHans Rosenfeld } 8883c9168faSHans Rosenfeld 8894b324362SHans Rosenfeld static void 8904b324362SHans Rosenfeld nvme_submit_cmd_common(nvme_qpair_t *qp, nvme_cmd_t *cmd) 8914b324362SHans Rosenfeld { 8924b324362SHans Rosenfeld nvme_reg_sqtdbl_t tail = { 0 }; 8933c9168faSHans Rosenfeld 8944b324362SHans Rosenfeld mutex_enter(&qp->nq_mutex); 8953c9168faSHans Rosenfeld cmd->nc_completed = B_FALSE; 8963c9168faSHans Rosenfeld 8973c9168faSHans Rosenfeld /* 8983c9168faSHans Rosenfeld * Try to insert the cmd into the active cmd array at the nq_next_cmd 8993c9168faSHans Rosenfeld * slot. If the slot is already occupied advance to the next slot and 9003c9168faSHans Rosenfeld * try again. This can happen for long running commands like async event 9013c9168faSHans Rosenfeld * requests. 9023c9168faSHans Rosenfeld */ 9033c9168faSHans Rosenfeld while (qp->nq_cmd[qp->nq_next_cmd] != NULL) 9043c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 9053c9168faSHans Rosenfeld qp->nq_cmd[qp->nq_next_cmd] = cmd; 9063c9168faSHans Rosenfeld 9073c9168faSHans Rosenfeld qp->nq_active_cmds++; 9083c9168faSHans Rosenfeld 9093c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cid = qp->nq_next_cmd; 9103c9168faSHans Rosenfeld bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t)); 9113c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_sqdma->nd_dmah, 9123c9168faSHans Rosenfeld sizeof (nvme_sqe_t) * qp->nq_sqtail, 9133c9168faSHans Rosenfeld sizeof (nvme_sqe_t), DDI_DMA_SYNC_FORDEV); 9143c9168faSHans Rosenfeld qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 9153c9168faSHans Rosenfeld 9163c9168faSHans Rosenfeld tail.b.sqtdbl_sqt = qp->nq_sqtail = (qp->nq_sqtail + 1) % qp->nq_nentry; 9173c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r); 9183c9168faSHans Rosenfeld 9193c9168faSHans Rosenfeld mutex_exit(&qp->nq_mutex); 9203c9168faSHans Rosenfeld } 9213c9168faSHans Rosenfeld 922e984c70bSHans Rosenfeld static nvme_cmd_t * 923e984c70bSHans Rosenfeld nvme_unqueue_cmd(nvme_t *nvme, nvme_qpair_t *qp, int cid) 924e984c70bSHans Rosenfeld { 925e984c70bSHans Rosenfeld nvme_cmd_t *cmd; 926e984c70bSHans Rosenfeld 927e984c70bSHans Rosenfeld ASSERT(mutex_owned(&qp->nq_mutex)); 928e984c70bSHans Rosenfeld ASSERT3S(cid, <, qp->nq_nentry); 929e984c70bSHans Rosenfeld 930e984c70bSHans Rosenfeld cmd = qp->nq_cmd[cid]; 931e984c70bSHans Rosenfeld qp->nq_cmd[cid] = NULL; 932e984c70bSHans Rosenfeld ASSERT3U(qp->nq_active_cmds, >, 0); 933e984c70bSHans Rosenfeld qp->nq_active_cmds--; 934e984c70bSHans Rosenfeld sema_v(&qp->nq_sema); 935e984c70bSHans Rosenfeld 936e984c70bSHans Rosenfeld ASSERT3P(cmd, !=, NULL); 937e984c70bSHans Rosenfeld ASSERT3P(cmd->nc_nvme, ==, nvme); 938e984c70bSHans Rosenfeld ASSERT3S(cmd->nc_sqe.sqe_cid, ==, cid); 939e984c70bSHans Rosenfeld 940e984c70bSHans Rosenfeld return (cmd); 941e984c70bSHans Rosenfeld } 942e984c70bSHans Rosenfeld 9433c9168faSHans Rosenfeld static nvme_cmd_t * 9443c9168faSHans Rosenfeld nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp) 9453c9168faSHans Rosenfeld { 9463c9168faSHans Rosenfeld nvme_reg_cqhdbl_t head = { 0 }; 9473c9168faSHans Rosenfeld 9483c9168faSHans Rosenfeld nvme_cqe_t *cqe; 9493c9168faSHans Rosenfeld nvme_cmd_t *cmd; 9503c9168faSHans Rosenfeld 9513c9168faSHans Rosenfeld (void) ddi_dma_sync(qp->nq_cqdma->nd_dmah, 0, 9523c9168faSHans Rosenfeld sizeof (nvme_cqe_t) * qp->nq_nentry, DDI_DMA_SYNC_FORKERNEL); 9533c9168faSHans Rosenfeld 9544ac9cfccSHans Rosenfeld mutex_enter(&qp->nq_mutex); 9553c9168faSHans Rosenfeld cqe = &qp->nq_cq[qp->nq_cqhead]; 9563c9168faSHans Rosenfeld 9573c9168faSHans Rosenfeld /* Check phase tag of CQE. Hardware inverts it for new entries. */ 9584ac9cfccSHans Rosenfeld if (cqe->cqe_sf.sf_p == qp->nq_phase) { 9594ac9cfccSHans Rosenfeld mutex_exit(&qp->nq_mutex); 9603c9168faSHans Rosenfeld return (NULL); 9614ac9cfccSHans Rosenfeld } 9623c9168faSHans Rosenfeld 9633c9168faSHans Rosenfeld ASSERT(nvme->n_ioq[cqe->cqe_sqid] == qp); 9643c9168faSHans Rosenfeld 965e984c70bSHans Rosenfeld cmd = nvme_unqueue_cmd(nvme, qp, cqe->cqe_cid); 9663c9168faSHans Rosenfeld 9673c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid == cqe->cqe_sqid); 9683c9168faSHans Rosenfeld bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t)); 9693c9168faSHans Rosenfeld 9703c9168faSHans Rosenfeld qp->nq_sqhead = cqe->cqe_sqhd; 9713c9168faSHans Rosenfeld 9723c9168faSHans Rosenfeld head.b.cqhdbl_cqh = qp->nq_cqhead = (qp->nq_cqhead + 1) % qp->nq_nentry; 9733c9168faSHans Rosenfeld 9743c9168faSHans Rosenfeld /* Toggle phase on wrap-around. */ 9753c9168faSHans Rosenfeld if (qp->nq_cqhead == 0) 9763c9168faSHans Rosenfeld qp->nq_phase = qp->nq_phase ? 0 : 1; 9773c9168faSHans Rosenfeld 9783c9168faSHans Rosenfeld nvme_put32(cmd->nc_nvme, qp->nq_cqhdbl, head.r); 9794ac9cfccSHans Rosenfeld mutex_exit(&qp->nq_mutex); 9803c9168faSHans Rosenfeld 9813c9168faSHans Rosenfeld return (cmd); 9823c9168faSHans Rosenfeld } 9833c9168faSHans Rosenfeld 9843c9168faSHans Rosenfeld static int 9853c9168faSHans Rosenfeld nvme_check_unknown_cmd_status(nvme_cmd_t *cmd) 9863c9168faSHans Rosenfeld { 9873c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 9883c9168faSHans Rosenfeld 9893c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 9903c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 9913c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 9923c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 9933c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 9943c9168faSHans Rosenfeld 9953d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 9963d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 997bef9e21aSHans Rosenfeld 9983c9168faSHans Rosenfeld if (cmd->nc_nvme->n_strict_version) { 9993c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 10003c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 10013c9168faSHans Rosenfeld } 10023c9168faSHans Rosenfeld 10033c9168faSHans Rosenfeld return (EIO); 10043c9168faSHans Rosenfeld } 10053c9168faSHans Rosenfeld 10063c9168faSHans Rosenfeld static int 10073c9168faSHans Rosenfeld nvme_check_vendor_cmd_status(nvme_cmd_t *cmd) 10083c9168faSHans Rosenfeld { 10093c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 10103c9168faSHans Rosenfeld 10113c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 10123c9168faSHans Rosenfeld "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 10133c9168faSHans Rosenfeld "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 10143c9168faSHans Rosenfeld cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 10153c9168faSHans Rosenfeld cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 101634c938c7SPete Shephard if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) { 10173c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 10183c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 10193c9168faSHans Rosenfeld } 10203c9168faSHans Rosenfeld 10213c9168faSHans Rosenfeld return (EIO); 10223c9168faSHans Rosenfeld } 10233c9168faSHans Rosenfeld 10243c9168faSHans Rosenfeld static int 10253c9168faSHans Rosenfeld nvme_check_integrity_cmd_status(nvme_cmd_t *cmd) 10263c9168faSHans Rosenfeld { 10273c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 10283c9168faSHans Rosenfeld 10293c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 10303c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_WRITE: 10313c9168faSHans Rosenfeld /* write fail */ 10323c9168faSHans Rosenfeld /* TODO: post ereport */ 10333d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 10343d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 10353c9168faSHans Rosenfeld return (EIO); 10363c9168faSHans Rosenfeld 10373c9168faSHans Rosenfeld case NVME_CQE_SC_INT_NVM_READ: 10383c9168faSHans Rosenfeld /* read fail */ 10393c9168faSHans Rosenfeld /* TODO: post ereport */ 10403d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 10413d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 10423c9168faSHans Rosenfeld return (EIO); 10433c9168faSHans Rosenfeld 10443c9168faSHans Rosenfeld default: 10453c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 10463c9168faSHans Rosenfeld } 10473c9168faSHans Rosenfeld } 10483c9168faSHans Rosenfeld 10493c9168faSHans Rosenfeld static int 10503c9168faSHans Rosenfeld nvme_check_generic_cmd_status(nvme_cmd_t *cmd) 10513c9168faSHans Rosenfeld { 10523c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 10533c9168faSHans Rosenfeld 10543c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 10553c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_SUCCESS: 10563c9168faSHans Rosenfeld return (0); 10573c9168faSHans Rosenfeld 10583c9168faSHans Rosenfeld /* 10593c9168faSHans Rosenfeld * Errors indicating a bug in the driver should cause a panic. 10603c9168faSHans Rosenfeld */ 10613c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_OPC: 10623c9168faSHans Rosenfeld /* Invalid Command Opcode */ 106308139162SToomas Soome if (!cmd->nc_dontpanic) 106408139162SToomas Soome dev_err(cmd->nc_nvme->n_dip, CE_PANIC, 106508139162SToomas Soome "programming error: invalid opcode in cmd %p", 106608139162SToomas Soome (void *)cmd); 106708139162SToomas Soome return (EINVAL); 10683c9168faSHans Rosenfeld 10693c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_FLD: 10703c9168faSHans Rosenfeld /* Invalid Field in Command */ 10713d9b1a2aSHans Rosenfeld if (!cmd->nc_dontpanic) 10723d9b1a2aSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, 10733d9b1a2aSHans Rosenfeld "programming error: invalid field in cmd %p", 10743d9b1a2aSHans Rosenfeld (void *)cmd); 10753d9b1a2aSHans Rosenfeld return (EIO); 10763c9168faSHans Rosenfeld 10773c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ID_CNFL: 10783c9168faSHans Rosenfeld /* Command ID Conflict */ 10793c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 10803c9168faSHans Rosenfeld "cmd ID conflict in cmd %p", (void *)cmd); 10813c9168faSHans Rosenfeld return (0); 10823c9168faSHans Rosenfeld 10833c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INV_NS: 10843c9168faSHans Rosenfeld /* Invalid Namespace or Format */ 10853d9b1a2aSHans Rosenfeld if (!cmd->nc_dontpanic) 10863d9b1a2aSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, 1087*f313c178SYuri Pankov "programming error: invalid NS/format in cmd %p", 10883d9b1a2aSHans Rosenfeld (void *)cmd); 10893d9b1a2aSHans Rosenfeld return (EINVAL); 10903c9168faSHans Rosenfeld 10913c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_LBA_RANGE: 10923c9168faSHans Rosenfeld /* LBA Out Of Range */ 10933c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 10943c9168faSHans Rosenfeld "LBA out of range in cmd %p", (void *)cmd); 10953c9168faSHans Rosenfeld return (0); 10963c9168faSHans Rosenfeld 10973c9168faSHans Rosenfeld /* 10983c9168faSHans Rosenfeld * Non-fatal errors, handle gracefully. 10993c9168faSHans Rosenfeld */ 11003c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_DATA_XFR_ERR: 11013c9168faSHans Rosenfeld /* Data Transfer Error (DMA) */ 11023c9168faSHans Rosenfeld /* TODO: post ereport */ 11033c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err); 11043d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 11053d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 11063c9168faSHans Rosenfeld return (EIO); 11073c9168faSHans Rosenfeld 11083c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_INTERNAL_ERR: 11093c9168faSHans Rosenfeld /* 11103c9168faSHans Rosenfeld * Internal Error. The spec (v1.0, section 4.5.1.2) says 11113c9168faSHans Rosenfeld * detailed error information is returned as async event, 11123c9168faSHans Rosenfeld * so we pretty much ignore the error here and handle it 11133c9168faSHans Rosenfeld * in the async event handler. 11143c9168faSHans Rosenfeld */ 11153c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_internal_err); 11163d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 11173d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 11183c9168faSHans Rosenfeld return (EIO); 11193c9168faSHans Rosenfeld 11203c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_REQUEST: 11213c9168faSHans Rosenfeld /* 11223c9168faSHans Rosenfeld * Command Abort Requested. This normally happens only when a 11233c9168faSHans Rosenfeld * command times out. 11243c9168faSHans Rosenfeld */ 11253c9168faSHans Rosenfeld /* TODO: post ereport or change blkdev to handle this? */ 11263c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err); 11273c9168faSHans Rosenfeld return (ECANCELED); 11283c9168faSHans Rosenfeld 11293c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_PWRLOSS: 11303c9168faSHans Rosenfeld /* Command Aborted due to Power Loss Notification */ 11313c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 11323c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 11333c9168faSHans Rosenfeld return (EIO); 11343c9168faSHans Rosenfeld 11353c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_ABORT_SQ_DEL: 11363c9168faSHans Rosenfeld /* Command Aborted due to SQ Deletion */ 11373c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del); 11383c9168faSHans Rosenfeld return (EIO); 11393c9168faSHans Rosenfeld 11403c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_CAP_EXC: 11413c9168faSHans Rosenfeld /* Capacity Exceeded */ 11423c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc); 11433d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 11443d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 11453c9168faSHans Rosenfeld return (EIO); 11463c9168faSHans Rosenfeld 11473c9168faSHans Rosenfeld case NVME_CQE_SC_GEN_NVM_NS_NOTRDY: 11483c9168faSHans Rosenfeld /* Namespace Not Ready */ 11493c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy); 11503d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 11513d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 11523c9168faSHans Rosenfeld return (EIO); 11533c9168faSHans Rosenfeld 11543c9168faSHans Rosenfeld default: 11553c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 11563c9168faSHans Rosenfeld } 11573c9168faSHans Rosenfeld } 11583c9168faSHans Rosenfeld 11593c9168faSHans Rosenfeld static int 11603c9168faSHans Rosenfeld nvme_check_specific_cmd_status(nvme_cmd_t *cmd) 11613c9168faSHans Rosenfeld { 11623c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 11633c9168faSHans Rosenfeld 11643c9168faSHans Rosenfeld switch (cqe->cqe_sf.sf_sc) { 11653c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_CQ: 11663c9168faSHans Rosenfeld /* Completion Queue Invalid */ 11673c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE); 11683c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err); 11693c9168faSHans Rosenfeld return (EINVAL); 11703c9168faSHans Rosenfeld 11713c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_QID: 11723c9168faSHans Rosenfeld /* Invalid Queue Identifier */ 11733c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 11743c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE || 11753c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE || 11763c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 11773c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err); 11783c9168faSHans Rosenfeld return (EINVAL); 11793c9168faSHans Rosenfeld 11803c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_MAX_QSZ_EXC: 11813c9168faSHans Rosenfeld /* Max Queue Size Exceeded */ 11823c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 11833c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 11843c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc); 11853c9168faSHans Rosenfeld return (EINVAL); 11863c9168faSHans Rosenfeld 11873c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ABRT_CMD_EXC: 11883c9168faSHans Rosenfeld /* Abort Command Limit Exceeded */ 11893c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT); 11903c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 11913c9168faSHans Rosenfeld "abort command limit exceeded in cmd %p", (void *)cmd); 11923c9168faSHans Rosenfeld return (0); 11933c9168faSHans Rosenfeld 11943c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC: 11953c9168faSHans Rosenfeld /* Async Event Request Limit Exceeded */ 11963c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT); 11973c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 11983c9168faSHans Rosenfeld "async event request limit exceeded in cmd %p", 11993c9168faSHans Rosenfeld (void *)cmd); 12003c9168faSHans Rosenfeld return (0); 12013c9168faSHans Rosenfeld 12023c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_INT_VECT: 12033c9168faSHans Rosenfeld /* Invalid Interrupt Vector */ 12043c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 12053c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect); 12063c9168faSHans Rosenfeld return (EINVAL); 12073c9168faSHans Rosenfeld 12083c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_LOG_PAGE: 12093c9168faSHans Rosenfeld /* Invalid Log Page */ 12103c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE); 12113c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_log_page); 12123c9168faSHans Rosenfeld return (EINVAL); 12133c9168faSHans Rosenfeld 12143c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_FORMAT: 12153c9168faSHans Rosenfeld /* Invalid Format */ 12163c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT); 12173c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_format); 12183d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 12193d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 12203c9168faSHans Rosenfeld return (EINVAL); 12213c9168faSHans Rosenfeld 12223c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_INV_Q_DEL: 12233c9168faSHans Rosenfeld /* Invalid Queue Deletion */ 12243c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 12253c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_q_del); 12263c9168faSHans Rosenfeld return (EINVAL); 12273c9168faSHans Rosenfeld 12283c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_CNFL_ATTR: 12293c9168faSHans Rosenfeld /* Conflicting Attributes */ 12303c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT || 12313c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 12323c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 12333c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr); 12343d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 12353d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 12363c9168faSHans Rosenfeld return (EINVAL); 12373c9168faSHans Rosenfeld 12383c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_INV_PROT: 12393c9168faSHans Rosenfeld /* Invalid Protection Information */ 12403c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE || 12413c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 12423c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 12433c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_inv_prot); 12443d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 12453d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 12463c9168faSHans Rosenfeld return (EINVAL); 12473c9168faSHans Rosenfeld 12483c9168faSHans Rosenfeld case NVME_CQE_SC_SPC_NVM_READONLY: 12493c9168faSHans Rosenfeld /* Write to Read Only Range */ 12503c9168faSHans Rosenfeld ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 12513c9168faSHans Rosenfeld atomic_inc_32(&cmd->nc_nvme->n_readonly); 12523d9b1a2aSHans Rosenfeld if (cmd->nc_xfer != NULL) 12533d9b1a2aSHans Rosenfeld bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 12543c9168faSHans Rosenfeld return (EROFS); 12553c9168faSHans Rosenfeld 12563c9168faSHans Rosenfeld default: 12573c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 12583c9168faSHans Rosenfeld } 12593c9168faSHans Rosenfeld } 12603c9168faSHans Rosenfeld 12613c9168faSHans Rosenfeld static inline int 12623c9168faSHans Rosenfeld nvme_check_cmd_status(nvme_cmd_t *cmd) 12633c9168faSHans Rosenfeld { 12643c9168faSHans Rosenfeld nvme_cqe_t *cqe = &cmd->nc_cqe; 12653c9168faSHans Rosenfeld 1266e984c70bSHans Rosenfeld /* 1267e984c70bSHans Rosenfeld * Take a shortcut if the controller is dead, or if 1268e984c70bSHans Rosenfeld * command status indicates no error. 1269e984c70bSHans Rosenfeld */ 1270e984c70bSHans Rosenfeld if (cmd->nc_nvme->n_dead) 1271e984c70bSHans Rosenfeld return (EIO); 1272e984c70bSHans Rosenfeld 12733c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 12743c9168faSHans Rosenfeld cqe->cqe_sf.sf_sc == NVME_CQE_SC_GEN_SUCCESS) 12753c9168faSHans Rosenfeld return (0); 12763c9168faSHans Rosenfeld 12773c9168faSHans Rosenfeld if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC) 12783c9168faSHans Rosenfeld return (nvme_check_generic_cmd_status(cmd)); 12793c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_SPECIFIC) 12803c9168faSHans Rosenfeld return (nvme_check_specific_cmd_status(cmd)); 12813c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_INTEGRITY) 12823c9168faSHans Rosenfeld return (nvme_check_integrity_cmd_status(cmd)); 12833c9168faSHans Rosenfeld else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_VENDOR) 12843c9168faSHans Rosenfeld return (nvme_check_vendor_cmd_status(cmd)); 12853c9168faSHans Rosenfeld 12863c9168faSHans Rosenfeld return (nvme_check_unknown_cmd_status(cmd)); 12873c9168faSHans Rosenfeld } 12883c9168faSHans Rosenfeld 1289e984c70bSHans Rosenfeld static int 1290e984c70bSHans Rosenfeld nvme_abort_cmd(nvme_cmd_t *abort_cmd, uint_t sec) 12913c9168faSHans Rosenfeld { 12923c9168faSHans Rosenfeld nvme_t *nvme = abort_cmd->nc_nvme; 12933c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 12943c9168faSHans Rosenfeld nvme_abort_cmd_t ac = { 0 }; 1295e984c70bSHans Rosenfeld int ret = 0; 12963c9168faSHans Rosenfeld 12973c9168faSHans Rosenfeld sema_p(&nvme->n_abort_sema); 12983c9168faSHans Rosenfeld 12993c9168faSHans Rosenfeld ac.b.ac_cid = abort_cmd->nc_sqe.sqe_cid; 13003c9168faSHans Rosenfeld ac.b.ac_sqid = abort_cmd->nc_sqid; 13013c9168faSHans Rosenfeld 13023c9168faSHans Rosenfeld cmd->nc_sqid = 0; 13033c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT; 13043c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 13053c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = ac.r; 13063c9168faSHans Rosenfeld 13073c9168faSHans Rosenfeld /* 13083c9168faSHans Rosenfeld * Send the ABORT to the hardware. The ABORT command will return _after_ 1309e984c70bSHans Rosenfeld * the aborted command has completed (aborted or otherwise), but since 1310e984c70bSHans Rosenfeld * we still hold the aborted command's mutex its callback hasn't been 1311e984c70bSHans Rosenfeld * processed yet. 13123c9168faSHans Rosenfeld */ 1313e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, sec); 13143c9168faSHans Rosenfeld sema_v(&nvme->n_abort_sema); 13153c9168faSHans Rosenfeld 1316e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 13173c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 13183c9168faSHans Rosenfeld "!ABORT failed with sct = %x, sc = %x", 13193c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 13203c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_abort_failed); 13213c9168faSHans Rosenfeld } else { 1322e984c70bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 1323e984c70bSHans Rosenfeld "!ABORT of command %d/%d %ssuccessful", 1324e984c70bSHans Rosenfeld abort_cmd->nc_sqe.sqe_cid, abort_cmd->nc_sqid, 1325e984c70bSHans Rosenfeld cmd->nc_cqe.cqe_dw0 & 1 ? "un" : ""); 1326e984c70bSHans Rosenfeld if ((cmd->nc_cqe.cqe_dw0 & 1) == 0) 1327e984c70bSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_aborted); 13283c9168faSHans Rosenfeld } 13293c9168faSHans Rosenfeld 13303c9168faSHans Rosenfeld nvme_free_cmd(cmd); 1331e984c70bSHans Rosenfeld return (ret); 13323c9168faSHans Rosenfeld } 13333c9168faSHans Rosenfeld 13343c9168faSHans Rosenfeld /* 13353c9168faSHans Rosenfeld * nvme_wait_cmd -- wait for command completion or timeout 13363c9168faSHans Rosenfeld * 13373c9168faSHans Rosenfeld * In case of a serious error or a timeout of the abort command the hardware 13383c9168faSHans Rosenfeld * will be declared dead and FMA will be notified. 13393c9168faSHans Rosenfeld */ 1340e984c70bSHans Rosenfeld static void 1341e8ba2a38SHans Rosenfeld nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec) 13423c9168faSHans Rosenfeld { 1343e8ba2a38SHans Rosenfeld clock_t timeout = ddi_get_lbolt() + drv_usectohz(sec * MICROSEC); 13443c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 13453c9168faSHans Rosenfeld nvme_reg_csts_t csts; 1346e984c70bSHans Rosenfeld nvme_qpair_t *qp; 13473c9168faSHans Rosenfeld 13483c9168faSHans Rosenfeld ASSERT(mutex_owned(&cmd->nc_mutex)); 13493c9168faSHans Rosenfeld 13503c9168faSHans Rosenfeld while (!cmd->nc_completed) { 13513c9168faSHans Rosenfeld if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1) 13523c9168faSHans Rosenfeld break; 13533c9168faSHans Rosenfeld } 13543c9168faSHans Rosenfeld 13553c9168faSHans Rosenfeld if (cmd->nc_completed) 1356e984c70bSHans Rosenfeld return; 13573c9168faSHans Rosenfeld 13583c9168faSHans Rosenfeld /* 1359e984c70bSHans Rosenfeld * The command timed out. 1360e984c70bSHans Rosenfeld * 13613c9168faSHans Rosenfeld * Check controller for fatal status, any errors associated with the 13623c9168faSHans Rosenfeld * register or DMA handle, or for a double timeout (abort command timed 13633c9168faSHans Rosenfeld * out). If necessary log a warning and call FMA. 13643c9168faSHans Rosenfeld */ 13653c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 1366e984c70bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!command %d/%d timeout, " 1367e984c70bSHans Rosenfeld "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_cid, cmd->nc_sqid, 1368e984c70bSHans Rosenfeld cmd->nc_sqe.sqe_opc, csts.b.csts_cfs); 13693c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_cmd_timeout); 13703c9168faSHans Rosenfeld 13713c9168faSHans Rosenfeld if (csts.b.csts_cfs || 13723c9168faSHans Rosenfeld nvme_check_regs_hdl(nvme) || 13733c9168faSHans Rosenfeld nvme_check_dma_hdl(cmd->nc_dma) || 13743c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) { 13753c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 13763c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 1377e984c70bSHans Rosenfeld } else if (nvme_abort_cmd(cmd, sec) == 0) { 13783c9168faSHans Rosenfeld /* 1379e984c70bSHans Rosenfeld * If the abort succeeded the command should complete 1380e984c70bSHans Rosenfeld * immediately with an appropriate status. 13813c9168faSHans Rosenfeld */ 1382e984c70bSHans Rosenfeld while (!cmd->nc_completed) 1383e984c70bSHans Rosenfeld cv_wait(&cmd->nc_cv, &cmd->nc_mutex); 1384e984c70bSHans Rosenfeld 1385e984c70bSHans Rosenfeld return; 13863c9168faSHans Rosenfeld } 13873c9168faSHans Rosenfeld 1388e984c70bSHans Rosenfeld qp = nvme->n_ioq[cmd->nc_sqid]; 1389e984c70bSHans Rosenfeld 1390e984c70bSHans Rosenfeld mutex_enter(&qp->nq_mutex); 1391e984c70bSHans Rosenfeld (void) nvme_unqueue_cmd(nvme, qp, cmd->nc_sqe.sqe_cid); 1392e984c70bSHans Rosenfeld mutex_exit(&qp->nq_mutex); 1393e984c70bSHans Rosenfeld 1394e984c70bSHans Rosenfeld /* 1395e984c70bSHans Rosenfeld * As we don't know what the presumed dead hardware might still do with 1396e984c70bSHans Rosenfeld * the DMA memory, we'll put the command on the lost commands list if it 1397e984c70bSHans Rosenfeld * has any DMA memory. 1398e984c70bSHans Rosenfeld */ 1399e984c70bSHans Rosenfeld if (cmd->nc_dma != NULL) { 1400e984c70bSHans Rosenfeld mutex_enter(&nvme_lc_mutex); 1401e984c70bSHans Rosenfeld list_insert_head(&nvme_lost_cmds, cmd); 1402e984c70bSHans Rosenfeld mutex_exit(&nvme_lc_mutex); 1403e984c70bSHans Rosenfeld } 14043c9168faSHans Rosenfeld } 14053c9168faSHans Rosenfeld 14063c9168faSHans Rosenfeld static void 14073c9168faSHans Rosenfeld nvme_wakeup_cmd(void *arg) 14083c9168faSHans Rosenfeld { 14093c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 14103c9168faSHans Rosenfeld 14113c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 14123c9168faSHans Rosenfeld cmd->nc_completed = B_TRUE; 14133c9168faSHans Rosenfeld cv_signal(&cmd->nc_cv); 14143c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 14153c9168faSHans Rosenfeld } 14163c9168faSHans Rosenfeld 14173c9168faSHans Rosenfeld static void 14183c9168faSHans Rosenfeld nvme_async_event_task(void *arg) 14193c9168faSHans Rosenfeld { 14203c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 14213c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 14223c9168faSHans Rosenfeld nvme_error_log_entry_t *error_log = NULL; 14233c9168faSHans Rosenfeld nvme_health_log_t *health_log = NULL; 14243d9b1a2aSHans Rosenfeld size_t logsize = 0; 14253c9168faSHans Rosenfeld nvme_async_event_t event; 14263c9168faSHans Rosenfeld 14273c9168faSHans Rosenfeld /* 14283c9168faSHans Rosenfeld * Check for errors associated with the async request itself. The only 14293c9168faSHans Rosenfeld * command-specific error is "async event limit exceeded", which 14303c9168faSHans Rosenfeld * indicates a programming error in the driver and causes a panic in 14313c9168faSHans Rosenfeld * nvme_check_cmd_status(). 14323c9168faSHans Rosenfeld * 14333c9168faSHans Rosenfeld * Other possible errors are various scenarios where the async request 14343c9168faSHans Rosenfeld * was aborted, or internal errors in the device. Internal errors are 14353c9168faSHans Rosenfeld * reported to FMA, the command aborts need no special handling here. 143608139162SToomas Soome * 143708139162SToomas Soome * And finally, at least qemu nvme does not support async events, 143808139162SToomas Soome * and will return NVME_CQE_SC_GEN_INV_OPC | DNR. If so, we 143908139162SToomas Soome * will avoid posting async events. 14403c9168faSHans Rosenfeld */ 144108139162SToomas Soome 1442e984c70bSHans Rosenfeld if (nvme_check_cmd_status(cmd) != 0) { 14433c9168faSHans Rosenfeld dev_err(cmd->nc_nvme->n_dip, CE_WARN, 14443c9168faSHans Rosenfeld "!async event request returned failure, sct = %x, " 14453c9168faSHans Rosenfeld "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct, 14463c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr, 14473c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_m); 14483c9168faSHans Rosenfeld 14493c9168faSHans Rosenfeld if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 14503c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) { 14513c9168faSHans Rosenfeld cmd->nc_nvme->n_dead = B_TRUE; 14523c9168faSHans Rosenfeld ddi_fm_service_impact(cmd->nc_nvme->n_dip, 14533c9168faSHans Rosenfeld DDI_SERVICE_LOST); 14543c9168faSHans Rosenfeld } 145508139162SToomas Soome 145608139162SToomas Soome if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 145708139162SToomas Soome cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INV_OPC && 145808139162SToomas Soome cmd->nc_cqe.cqe_sf.sf_dnr == 1) { 145908139162SToomas Soome nvme->n_async_event_supported = B_FALSE; 146008139162SToomas Soome } 146108139162SToomas Soome 14623c9168faSHans Rosenfeld nvme_free_cmd(cmd); 14633c9168faSHans Rosenfeld return; 14643c9168faSHans Rosenfeld } 14653c9168faSHans Rosenfeld 14663c9168faSHans Rosenfeld 14673c9168faSHans Rosenfeld event.r = cmd->nc_cqe.cqe_dw0; 14683c9168faSHans Rosenfeld 14693c9168faSHans Rosenfeld /* Clear CQE and re-submit the async request. */ 14703c9168faSHans Rosenfeld bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t)); 14714b324362SHans Rosenfeld nvme_submit_admin_cmd(nvme->n_adminq, cmd); 14723c9168faSHans Rosenfeld 14733c9168faSHans Rosenfeld switch (event.b.ae_type) { 14743c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_ERROR: 14753c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_ERROR) { 14763d9b1a2aSHans Rosenfeld (void) nvme_get_logpage(nvme, (void **)&error_log, 14773d9b1a2aSHans Rosenfeld &logsize, event.b.ae_logpage); 14783c9168faSHans Rosenfeld } else { 14793c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 14803c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 14813c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 14823c9168faSHans Rosenfeld } 14833c9168faSHans Rosenfeld 14843c9168faSHans Rosenfeld switch (event.b.ae_info) { 14853c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_SQ: 14863c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 14873c9168faSHans Rosenfeld "invalid submission queue"); 14883c9168faSHans Rosenfeld return; 14893c9168faSHans Rosenfeld 14903c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_INV_DBL: 14913c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_PANIC, "programming error: " 14923c9168faSHans Rosenfeld "invalid doorbell write value"); 14933c9168faSHans Rosenfeld return; 14943c9168faSHans Rosenfeld 14953c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_DIAGFAIL: 14963c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure"); 14973c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 14983c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 14993c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_diagfail_event); 15003c9168faSHans Rosenfeld break; 15013c9168faSHans Rosenfeld 15023c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_PERSISTENT: 15033c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!persistent internal " 15043c9168faSHans Rosenfeld "device error"); 15053c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 15063c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 15073c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_persistent_event); 15083c9168faSHans Rosenfeld break; 15093c9168faSHans Rosenfeld 15103c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_TRANSIENT: 15113c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!transient internal " 15123c9168faSHans Rosenfeld "device error"); 15133c9168faSHans Rosenfeld /* TODO: send ereport */ 15143c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_transient_event); 15153c9168faSHans Rosenfeld break; 15163c9168faSHans Rosenfeld 15173c9168faSHans Rosenfeld case NVME_ASYNC_ERROR_FW_LOAD: 15183c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15193c9168faSHans Rosenfeld "!firmware image load error"); 15203c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_fw_load_event); 15213c9168faSHans Rosenfeld break; 15223c9168faSHans Rosenfeld } 15233c9168faSHans Rosenfeld break; 15243c9168faSHans Rosenfeld 15253c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_HEALTH: 15263c9168faSHans Rosenfeld if (event.b.ae_logpage == NVME_LOGPAGE_HEALTH) { 15273d9b1a2aSHans Rosenfeld (void) nvme_get_logpage(nvme, (void **)&health_log, 15283d9b1a2aSHans Rosenfeld &logsize, event.b.ae_logpage, -1); 15293c9168faSHans Rosenfeld } else { 15303c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 15313c9168faSHans Rosenfeld "async event reply: %d", event.b.ae_logpage); 15323c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_wrong_logpage); 15333c9168faSHans Rosenfeld } 15343c9168faSHans Rosenfeld 15353c9168faSHans Rosenfeld switch (event.b.ae_info) { 15363c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_RELIABILITY: 15373c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15383c9168faSHans Rosenfeld "!device reliability compromised"); 15393c9168faSHans Rosenfeld /* TODO: send ereport */ 15403c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_reliability_event); 15413c9168faSHans Rosenfeld break; 15423c9168faSHans Rosenfeld 15433c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_TEMPERATURE: 15443c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15453c9168faSHans Rosenfeld "!temperature above threshold"); 15463c9168faSHans Rosenfeld /* TODO: send ereport */ 15473c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_temperature_event); 15483c9168faSHans Rosenfeld break; 15493c9168faSHans Rosenfeld 15503c9168faSHans Rosenfeld case NVME_ASYNC_HEALTH_SPARE: 15513c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 15523c9168faSHans Rosenfeld "!spare space below threshold"); 15533c9168faSHans Rosenfeld /* TODO: send ereport */ 15543c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_spare_event); 15553c9168faSHans Rosenfeld break; 15563c9168faSHans Rosenfeld } 15573c9168faSHans Rosenfeld break; 15583c9168faSHans Rosenfeld 15593c9168faSHans Rosenfeld case NVME_ASYNC_TYPE_VENDOR: 15603c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event " 15613c9168faSHans Rosenfeld "received, info = %x, logpage = %x", event.b.ae_info, 15623c9168faSHans Rosenfeld event.b.ae_logpage); 15633c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_vendor_event); 15643c9168faSHans Rosenfeld break; 15653c9168faSHans Rosenfeld 15663c9168faSHans Rosenfeld default: 15673c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, " 15683c9168faSHans Rosenfeld "type = %x, info = %x, logpage = %x", event.b.ae_type, 15693c9168faSHans Rosenfeld event.b.ae_info, event.b.ae_logpage); 15703c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_event); 15713c9168faSHans Rosenfeld break; 15723c9168faSHans Rosenfeld } 15733c9168faSHans Rosenfeld 15743c9168faSHans Rosenfeld if (error_log) 15753d9b1a2aSHans Rosenfeld kmem_free(error_log, logsize); 15763c9168faSHans Rosenfeld 15773c9168faSHans Rosenfeld if (health_log) 15783d9b1a2aSHans Rosenfeld kmem_free(health_log, logsize); 15793c9168faSHans Rosenfeld } 15803c9168faSHans Rosenfeld 1581e984c70bSHans Rosenfeld static void 1582e8ba2a38SHans Rosenfeld nvme_admin_cmd(nvme_cmd_t *cmd, int sec) 15833c9168faSHans Rosenfeld { 15843c9168faSHans Rosenfeld mutex_enter(&cmd->nc_mutex); 15854b324362SHans Rosenfeld nvme_submit_admin_cmd(cmd->nc_nvme->n_adminq, cmd); 1586e984c70bSHans Rosenfeld nvme_wait_cmd(cmd, sec); 15873c9168faSHans Rosenfeld mutex_exit(&cmd->nc_mutex); 15883c9168faSHans Rosenfeld } 15893c9168faSHans Rosenfeld 15904b324362SHans Rosenfeld static void 15913c9168faSHans Rosenfeld nvme_async_event(nvme_t *nvme) 15923c9168faSHans Rosenfeld { 159308139162SToomas Soome nvme_cmd_t *cmd; 15943c9168faSHans Rosenfeld 159508139162SToomas Soome cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 15963c9168faSHans Rosenfeld cmd->nc_sqid = 0; 15973c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT; 15983c9168faSHans Rosenfeld cmd->nc_callback = nvme_async_event_task; 159908139162SToomas Soome cmd->nc_dontpanic = B_TRUE; 16003c9168faSHans Rosenfeld 16014b324362SHans Rosenfeld nvme_submit_admin_cmd(nvme->n_adminq, cmd); 16023c9168faSHans Rosenfeld } 16033c9168faSHans Rosenfeld 16043d9b1a2aSHans Rosenfeld static int 16053d9b1a2aSHans Rosenfeld nvme_format_nvm(nvme_t *nvme, uint32_t nsid, uint8_t lbaf, boolean_t ms, 16063d9b1a2aSHans Rosenfeld uint8_t pi, boolean_t pil, uint8_t ses) 16073d9b1a2aSHans Rosenfeld { 16083d9b1a2aSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 16093d9b1a2aSHans Rosenfeld nvme_format_nvm_t format_nvm = { 0 }; 16103d9b1a2aSHans Rosenfeld int ret; 16113d9b1a2aSHans Rosenfeld 16123d9b1a2aSHans Rosenfeld format_nvm.b.fm_lbaf = lbaf & 0xf; 16133d9b1a2aSHans Rosenfeld format_nvm.b.fm_ms = ms ? 1 : 0; 16143d9b1a2aSHans Rosenfeld format_nvm.b.fm_pi = pi & 0x7; 16153d9b1a2aSHans Rosenfeld format_nvm.b.fm_pil = pil ? 1 : 0; 16163d9b1a2aSHans Rosenfeld format_nvm.b.fm_ses = ses & 0x7; 16173d9b1a2aSHans Rosenfeld 16183d9b1a2aSHans Rosenfeld cmd->nc_sqid = 0; 16193d9b1a2aSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 16203d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 16213d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_NVM_FORMAT; 16223d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = format_nvm.r; 16233d9b1a2aSHans Rosenfeld 16243d9b1a2aSHans Rosenfeld /* 16253d9b1a2aSHans Rosenfeld * Some devices like Samsung SM951 don't allow formatting of all 16263d9b1a2aSHans Rosenfeld * namespaces in one command. Handle that gracefully. 16273d9b1a2aSHans Rosenfeld */ 16283d9b1a2aSHans Rosenfeld if (nsid == (uint32_t)-1) 16293d9b1a2aSHans Rosenfeld cmd->nc_dontpanic = B_TRUE; 16303d9b1a2aSHans Rosenfeld 1631e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_format_cmd_timeout); 16323d9b1a2aSHans Rosenfeld 16333d9b1a2aSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 16343d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16353d9b1a2aSHans Rosenfeld "!FORMAT failed with sct = %x, sc = %x", 16363d9b1a2aSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 16373d9b1a2aSHans Rosenfeld } 16383d9b1a2aSHans Rosenfeld 16393d9b1a2aSHans Rosenfeld nvme_free_cmd(cmd); 16403d9b1a2aSHans Rosenfeld return (ret); 16413d9b1a2aSHans Rosenfeld } 16423d9b1a2aSHans Rosenfeld 16433d9b1a2aSHans Rosenfeld static int 16443d9b1a2aSHans Rosenfeld nvme_get_logpage(nvme_t *nvme, void **buf, size_t *bufsize, uint8_t logpage, 16453d9b1a2aSHans Rosenfeld ...) 16463c9168faSHans Rosenfeld { 16473c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 164834c938c7SPete Shephard nvme_getlogpage_t getlogpage = { 0 }; 16493c9168faSHans Rosenfeld va_list ap; 1650e984c70bSHans Rosenfeld int ret; 16513c9168faSHans Rosenfeld 16523c9168faSHans Rosenfeld va_start(ap, logpage); 16533c9168faSHans Rosenfeld 16543c9168faSHans Rosenfeld cmd->nc_sqid = 0; 16553c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 16563c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE; 16573c9168faSHans Rosenfeld 16583c9168faSHans Rosenfeld getlogpage.b.lp_lid = logpage; 16593c9168faSHans Rosenfeld 16603c9168faSHans Rosenfeld switch (logpage) { 16613c9168faSHans Rosenfeld case NVME_LOGPAGE_ERROR: 16623c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 16633d9b1a2aSHans Rosenfeld /* 16643d9b1a2aSHans Rosenfeld * The GET LOG PAGE command can use at most 2 pages to return 16653d9b1a2aSHans Rosenfeld * data, PRP lists are not supported. 16663d9b1a2aSHans Rosenfeld */ 16673d9b1a2aSHans Rosenfeld *bufsize = MIN(2 * nvme->n_pagesize, 16683d9b1a2aSHans Rosenfeld nvme->n_error_log_len * sizeof (nvme_error_log_entry_t)); 16693c9168faSHans Rosenfeld break; 16703c9168faSHans Rosenfeld 16713c9168faSHans Rosenfeld case NVME_LOGPAGE_HEALTH: 16723c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t); 16733d9b1a2aSHans Rosenfeld *bufsize = sizeof (nvme_health_log_t); 16743c9168faSHans Rosenfeld break; 16753c9168faSHans Rosenfeld 16763c9168faSHans Rosenfeld case NVME_LOGPAGE_FWSLOT: 16773c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 16783d9b1a2aSHans Rosenfeld *bufsize = sizeof (nvme_fwslot_log_t); 16793c9168faSHans Rosenfeld break; 16803c9168faSHans Rosenfeld 16813c9168faSHans Rosenfeld default: 16823c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unknown log page requested: %d", 16833c9168faSHans Rosenfeld logpage); 16843c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_unknown_logpage); 1685e984c70bSHans Rosenfeld ret = EINVAL; 16863c9168faSHans Rosenfeld goto fail; 16873c9168faSHans Rosenfeld } 16883c9168faSHans Rosenfeld 16893c9168faSHans Rosenfeld va_end(ap); 16903c9168faSHans Rosenfeld 16913d9b1a2aSHans Rosenfeld getlogpage.b.lp_numd = *bufsize / sizeof (uint32_t) - 1; 16923c9168faSHans Rosenfeld 16933c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = getlogpage.r; 16943c9168faSHans Rosenfeld 16953c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, getlogpage.b.lp_numd * sizeof (uint32_t), 16963c9168faSHans Rosenfeld DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 16973c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 16983c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for GET LOG PAGE"); 1699e984c70bSHans Rosenfeld ret = ENOMEM; 17003c9168faSHans Rosenfeld goto fail; 17013c9168faSHans Rosenfeld } 17023c9168faSHans Rosenfeld 17033c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 17043c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17053c9168faSHans Rosenfeld "!too many DMA cookies for GET LOG PAGE"); 17063c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 1707e984c70bSHans Rosenfeld ret = ENOMEM; 17083c9168faSHans Rosenfeld goto fail; 17093c9168faSHans Rosenfeld } 17103c9168faSHans Rosenfeld 17113c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 17123c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 17133c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 17143c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 17153c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 17163c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 17173c9168faSHans Rosenfeld } 17183c9168faSHans Rosenfeld 1719e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 17203c9168faSHans Rosenfeld 1721e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 17223c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17233c9168faSHans Rosenfeld "!GET LOG PAGE failed with sct = %x, sc = %x", 17243c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 17253c9168faSHans Rosenfeld goto fail; 17263c9168faSHans Rosenfeld } 17273c9168faSHans Rosenfeld 17283d9b1a2aSHans Rosenfeld *buf = kmem_alloc(*bufsize, KM_SLEEP); 17293d9b1a2aSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize); 17303d9b1a2aSHans Rosenfeld 17313c9168faSHans Rosenfeld fail: 17323c9168faSHans Rosenfeld nvme_free_cmd(cmd); 17333c9168faSHans Rosenfeld 17343d9b1a2aSHans Rosenfeld return (ret); 17353c9168faSHans Rosenfeld } 17363c9168faSHans Rosenfeld 1737e984c70bSHans Rosenfeld static int 1738e984c70bSHans Rosenfeld nvme_identify(nvme_t *nvme, uint32_t nsid, void **buf) 17393c9168faSHans Rosenfeld { 17403c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1741e984c70bSHans Rosenfeld int ret; 1742e984c70bSHans Rosenfeld 1743e984c70bSHans Rosenfeld if (buf == NULL) 1744e984c70bSHans Rosenfeld return (EINVAL); 17453c9168faSHans Rosenfeld 17463c9168faSHans Rosenfeld cmd->nc_sqid = 0; 17473c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 17483c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY; 17493c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 17503c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL; 17513c9168faSHans Rosenfeld 17523c9168faSHans Rosenfeld if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ, 17533c9168faSHans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 17543c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17553c9168faSHans Rosenfeld "!nvme_zalloc_dma failed for IDENTIFY"); 1756e984c70bSHans Rosenfeld ret = ENOMEM; 17573c9168faSHans Rosenfeld goto fail; 17583c9168faSHans Rosenfeld } 17593c9168faSHans Rosenfeld 17603c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 17613c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17623c9168faSHans Rosenfeld "!too many DMA cookies for IDENTIFY"); 17633c9168faSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 1764e984c70bSHans Rosenfeld ret = ENOMEM; 17653c9168faSHans Rosenfeld goto fail; 17663c9168faSHans Rosenfeld } 17673c9168faSHans Rosenfeld 17683c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 17693c9168faSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 17703c9168faSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 17713c9168faSHans Rosenfeld &cmd->nc_dma->nd_cookie); 17723c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 17733c9168faSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 17743c9168faSHans Rosenfeld } 17753c9168faSHans Rosenfeld 1776e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 17773c9168faSHans Rosenfeld 1778e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 17793c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 17803c9168faSHans Rosenfeld "!IDENTIFY failed with sct = %x, sc = %x", 17813c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 17823c9168faSHans Rosenfeld goto fail; 17833c9168faSHans Rosenfeld } 17843c9168faSHans Rosenfeld 1785e984c70bSHans Rosenfeld *buf = kmem_alloc(NVME_IDENTIFY_BUFSIZE, KM_SLEEP); 1786e984c70bSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, *buf, NVME_IDENTIFY_BUFSIZE); 17873c9168faSHans Rosenfeld 17883c9168faSHans Rosenfeld fail: 17893c9168faSHans Rosenfeld nvme_free_cmd(cmd); 17903c9168faSHans Rosenfeld 1791e984c70bSHans Rosenfeld return (ret); 17923c9168faSHans Rosenfeld } 17933c9168faSHans Rosenfeld 1794e984c70bSHans Rosenfeld static int 1795d148d46eSHans Rosenfeld nvme_set_features(nvme_t *nvme, uint32_t nsid, uint8_t feature, uint32_t val, 1796d148d46eSHans Rosenfeld uint32_t *res) 17973c9168faSHans Rosenfeld { 1798d148d46eSHans Rosenfeld _NOTE(ARGUNUSED(nsid)); 17993c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1800e984c70bSHans Rosenfeld int ret = EINVAL; 18013c9168faSHans Rosenfeld 1802d148d46eSHans Rosenfeld ASSERT(res != NULL); 18033c9168faSHans Rosenfeld 18043c9168faSHans Rosenfeld cmd->nc_sqid = 0; 18053c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 18063c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES; 1807d148d46eSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = feature; 1808d148d46eSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = val; 1809d148d46eSHans Rosenfeld 1810d148d46eSHans Rosenfeld switch (feature) { 1811d148d46eSHans Rosenfeld case NVME_FEAT_WRITE_CACHE: 1812d148d46eSHans Rosenfeld if (!nvme->n_write_cache_present) 1813d148d46eSHans Rosenfeld goto fail; 1814d148d46eSHans Rosenfeld break; 1815d148d46eSHans Rosenfeld 1816d148d46eSHans Rosenfeld case NVME_FEAT_NQUEUES: 1817d148d46eSHans Rosenfeld break; 1818d148d46eSHans Rosenfeld 1819d148d46eSHans Rosenfeld default: 1820d148d46eSHans Rosenfeld goto fail; 1821d148d46eSHans Rosenfeld } 18223c9168faSHans Rosenfeld 1823e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 18243c9168faSHans Rosenfeld 1825e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 18263c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 1827d148d46eSHans Rosenfeld "!SET FEATURES %d failed with sct = %x, sc = %x", 1828d148d46eSHans Rosenfeld feature, cmd->nc_cqe.cqe_sf.sf_sct, 1829d148d46eSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc); 1830d148d46eSHans Rosenfeld goto fail; 18313c9168faSHans Rosenfeld } 18323c9168faSHans Rosenfeld 1833d148d46eSHans Rosenfeld *res = cmd->nc_cqe.cqe_dw0; 1834d148d46eSHans Rosenfeld 1835d148d46eSHans Rosenfeld fail: 18363c9168faSHans Rosenfeld nvme_free_cmd(cmd); 1837d148d46eSHans Rosenfeld return (ret); 1838d148d46eSHans Rosenfeld } 1839d148d46eSHans Rosenfeld 1840e984c70bSHans Rosenfeld static int 18413d9b1a2aSHans Rosenfeld nvme_get_features(nvme_t *nvme, uint32_t nsid, uint8_t feature, uint32_t *res, 18423d9b1a2aSHans Rosenfeld void **buf, size_t *bufsize) 18433d9b1a2aSHans Rosenfeld { 18443d9b1a2aSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1845e984c70bSHans Rosenfeld int ret = EINVAL; 18463d9b1a2aSHans Rosenfeld 18473d9b1a2aSHans Rosenfeld ASSERT(res != NULL); 18483d9b1a2aSHans Rosenfeld 18493d9b1a2aSHans Rosenfeld if (bufsize != NULL) 18503d9b1a2aSHans Rosenfeld *bufsize = 0; 18513d9b1a2aSHans Rosenfeld 18523d9b1a2aSHans Rosenfeld cmd->nc_sqid = 0; 18533d9b1a2aSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 18543d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_GET_FEATURES; 18553d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = feature; 18563d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = *res; 18573d9b1a2aSHans Rosenfeld 1858*f313c178SYuri Pankov /* 1859*f313c178SYuri Pankov * For some of the optional features there doesn't seem to be a method 1860*f313c178SYuri Pankov * of detecting whether it is supported other than using it. This will 1861*f313c178SYuri Pankov * cause "Invalid Field in Command" error, which is normally considered 1862*f313c178SYuri Pankov * a programming error. Set the nc_dontpanic flag to override the panic 1863*f313c178SYuri Pankov * in nvme_check_generic_cmd_status(). 1864*f313c178SYuri Pankov */ 18653d9b1a2aSHans Rosenfeld switch (feature) { 18663d9b1a2aSHans Rosenfeld case NVME_FEAT_ARBITRATION: 18673d9b1a2aSHans Rosenfeld case NVME_FEAT_POWER_MGMT: 18683d9b1a2aSHans Rosenfeld case NVME_FEAT_TEMPERATURE: 18693d9b1a2aSHans Rosenfeld case NVME_FEAT_ERROR: 18703d9b1a2aSHans Rosenfeld case NVME_FEAT_NQUEUES: 18713d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_COAL: 18723d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_VECT: 18733d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_ATOM: 18743d9b1a2aSHans Rosenfeld case NVME_FEAT_ASYNC_EVENT: 18753d9b1a2aSHans Rosenfeld break; 18763d9b1a2aSHans Rosenfeld 18773d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_CACHE: 18783d9b1a2aSHans Rosenfeld if (!nvme->n_write_cache_present) 18793d9b1a2aSHans Rosenfeld goto fail; 18803d9b1a2aSHans Rosenfeld break; 18813d9b1a2aSHans Rosenfeld 18823d9b1a2aSHans Rosenfeld case NVME_FEAT_LBA_RANGE: 18833d9b1a2aSHans Rosenfeld if (!nvme->n_lba_range_supported) 18843d9b1a2aSHans Rosenfeld goto fail; 18853d9b1a2aSHans Rosenfeld 18863d9b1a2aSHans Rosenfeld cmd->nc_dontpanic = B_TRUE; 18873d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_nsid = nsid; 18883d9b1a2aSHans Rosenfeld ASSERT(bufsize != NULL); 18893d9b1a2aSHans Rosenfeld *bufsize = NVME_LBA_RANGE_BUFSIZE; 18903d9b1a2aSHans Rosenfeld break; 18913d9b1a2aSHans Rosenfeld 18923d9b1a2aSHans Rosenfeld case NVME_FEAT_AUTO_PST: 18933d9b1a2aSHans Rosenfeld if (!nvme->n_auto_pst_supported) 18943d9b1a2aSHans Rosenfeld goto fail; 18953d9b1a2aSHans Rosenfeld 18963d9b1a2aSHans Rosenfeld ASSERT(bufsize != NULL); 18973d9b1a2aSHans Rosenfeld *bufsize = NVME_AUTO_PST_BUFSIZE; 18983d9b1a2aSHans Rosenfeld break; 18993d9b1a2aSHans Rosenfeld 1900*f313c178SYuri Pankov case NVME_FEAT_PROGRESS: 1901*f313c178SYuri Pankov if (!nvme->n_progress_supported) 1902*f313c178SYuri Pankov goto fail; 1903*f313c178SYuri Pankov 1904*f313c178SYuri Pankov cmd->nc_dontpanic = B_TRUE; 1905*f313c178SYuri Pankov break; 1906*f313c178SYuri Pankov 19073d9b1a2aSHans Rosenfeld default: 19083d9b1a2aSHans Rosenfeld goto fail; 19093d9b1a2aSHans Rosenfeld } 19103d9b1a2aSHans Rosenfeld 19113d9b1a2aSHans Rosenfeld if (bufsize != NULL && *bufsize != 0) { 19123d9b1a2aSHans Rosenfeld if (nvme_zalloc_dma(nvme, *bufsize, DDI_DMA_READ, 19133d9b1a2aSHans Rosenfeld &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 19143d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19153d9b1a2aSHans Rosenfeld "!nvme_zalloc_dma failed for GET FEATURES"); 1916e984c70bSHans Rosenfeld ret = ENOMEM; 19173d9b1a2aSHans Rosenfeld goto fail; 19183d9b1a2aSHans Rosenfeld } 19193d9b1a2aSHans Rosenfeld 19203d9b1a2aSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 2) { 19213d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19223d9b1a2aSHans Rosenfeld "!too many DMA cookies for GET FEATURES"); 19233d9b1a2aSHans Rosenfeld atomic_inc_32(&nvme->n_too_many_cookies); 1924e984c70bSHans Rosenfeld ret = ENOMEM; 19253d9b1a2aSHans Rosenfeld goto fail; 19263d9b1a2aSHans Rosenfeld } 19273d9b1a2aSHans Rosenfeld 19283d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = 19293d9b1a2aSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 19303d9b1a2aSHans Rosenfeld if (cmd->nc_dma->nd_ncookie > 1) { 19313d9b1a2aSHans Rosenfeld ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 19323d9b1a2aSHans Rosenfeld &cmd->nc_dma->nd_cookie); 19333d9b1a2aSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 19343d9b1a2aSHans Rosenfeld cmd->nc_dma->nd_cookie.dmac_laddress; 19353d9b1a2aSHans Rosenfeld } 19363d9b1a2aSHans Rosenfeld } 19373d9b1a2aSHans Rosenfeld 1938e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 19393d9b1a2aSHans Rosenfeld 1940e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 1941*f313c178SYuri Pankov boolean_t known = B_TRUE; 1942*f313c178SYuri Pankov 1943*f313c178SYuri Pankov /* Check if this is unsupported optional feature */ 1944*f313c178SYuri Pankov if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 1945*f313c178SYuri Pankov cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INV_FLD) { 1946*f313c178SYuri Pankov switch (feature) { 1947*f313c178SYuri Pankov case NVME_FEAT_LBA_RANGE: 1948*f313c178SYuri Pankov nvme->n_lba_range_supported = B_FALSE; 1949*f313c178SYuri Pankov break; 1950*f313c178SYuri Pankov case NVME_FEAT_PROGRESS: 1951*f313c178SYuri Pankov nvme->n_progress_supported = B_FALSE; 1952*f313c178SYuri Pankov break; 1953*f313c178SYuri Pankov default: 1954*f313c178SYuri Pankov known = B_FALSE; 1955*f313c178SYuri Pankov break; 1956*f313c178SYuri Pankov } 1957*f313c178SYuri Pankov } else { 1958*f313c178SYuri Pankov known = B_FALSE; 1959*f313c178SYuri Pankov } 1960*f313c178SYuri Pankov 1961*f313c178SYuri Pankov /* Report the error otherwise */ 1962*f313c178SYuri Pankov if (!known) { 19633d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 19643d9b1a2aSHans Rosenfeld "!GET FEATURES %d failed with sct = %x, sc = %x", 19653d9b1a2aSHans Rosenfeld feature, cmd->nc_cqe.cqe_sf.sf_sct, 19663d9b1a2aSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sc); 1967*f313c178SYuri Pankov } 1968*f313c178SYuri Pankov 19693d9b1a2aSHans Rosenfeld goto fail; 19703d9b1a2aSHans Rosenfeld } 19713d9b1a2aSHans Rosenfeld 19723d9b1a2aSHans Rosenfeld if (bufsize != NULL && *bufsize != 0) { 19733d9b1a2aSHans Rosenfeld ASSERT(buf != NULL); 19743d9b1a2aSHans Rosenfeld *buf = kmem_alloc(*bufsize, KM_SLEEP); 19753d9b1a2aSHans Rosenfeld bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize); 19763d9b1a2aSHans Rosenfeld } 19773d9b1a2aSHans Rosenfeld 19783d9b1a2aSHans Rosenfeld *res = cmd->nc_cqe.cqe_dw0; 19793d9b1a2aSHans Rosenfeld 19803d9b1a2aSHans Rosenfeld fail: 19813d9b1a2aSHans Rosenfeld nvme_free_cmd(cmd); 19823d9b1a2aSHans Rosenfeld return (ret); 19833d9b1a2aSHans Rosenfeld } 19843d9b1a2aSHans Rosenfeld 1985e984c70bSHans Rosenfeld static int 1986d148d46eSHans Rosenfeld nvme_write_cache_set(nvme_t *nvme, boolean_t enable) 1987d148d46eSHans Rosenfeld { 1988d148d46eSHans Rosenfeld nvme_write_cache_t nwc = { 0 }; 1989d148d46eSHans Rosenfeld 1990d148d46eSHans Rosenfeld if (enable) 1991d148d46eSHans Rosenfeld nwc.b.wc_wce = 1; 1992d148d46eSHans Rosenfeld 1993e984c70bSHans Rosenfeld return (nvme_set_features(nvme, 0, NVME_FEAT_WRITE_CACHE, nwc.r, 1994e984c70bSHans Rosenfeld &nwc.r)); 1995d148d46eSHans Rosenfeld } 1996d148d46eSHans Rosenfeld 1997d148d46eSHans Rosenfeld static int 1998e984c70bSHans Rosenfeld nvme_set_nqueues(nvme_t *nvme, uint16_t *nqueues) 1999d148d46eSHans Rosenfeld { 20003d9b1a2aSHans Rosenfeld nvme_nqueues_t nq = { 0 }; 2001e984c70bSHans Rosenfeld int ret; 2002d148d46eSHans Rosenfeld 2003e984c70bSHans Rosenfeld nq.b.nq_nsq = nq.b.nq_ncq = *nqueues - 1; 2004d148d46eSHans Rosenfeld 2005e984c70bSHans Rosenfeld ret = nvme_set_features(nvme, 0, NVME_FEAT_NQUEUES, nq.r, &nq.r); 2006e984c70bSHans Rosenfeld 2007e984c70bSHans Rosenfeld if (ret == 0) { 2008e984c70bSHans Rosenfeld /* 2009e984c70bSHans Rosenfeld * Always use the same number of submission and completion 2010e984c70bSHans Rosenfeld * queues, and never use more than the requested number of 2011e984c70bSHans Rosenfeld * queues. 2012e984c70bSHans Rosenfeld */ 2013e984c70bSHans Rosenfeld *nqueues = MIN(*nqueues, MIN(nq.b.nq_nsq, nq.b.nq_ncq) + 1); 2014d148d46eSHans Rosenfeld } 20153c9168faSHans Rosenfeld 2016e984c70bSHans Rosenfeld return (ret); 20173c9168faSHans Rosenfeld } 20183c9168faSHans Rosenfeld 20193c9168faSHans Rosenfeld static int 20203c9168faSHans Rosenfeld nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx) 20213c9168faSHans Rosenfeld { 20223c9168faSHans Rosenfeld nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 20233c9168faSHans Rosenfeld nvme_create_queue_dw10_t dw10 = { 0 }; 20243c9168faSHans Rosenfeld nvme_create_cq_dw11_t c_dw11 = { 0 }; 20253c9168faSHans Rosenfeld nvme_create_sq_dw11_t s_dw11 = { 0 }; 2026e984c70bSHans Rosenfeld int ret; 20273c9168faSHans Rosenfeld 20283c9168faSHans Rosenfeld dw10.b.q_qid = idx; 20293c9168faSHans Rosenfeld dw10.b.q_qsize = qp->nq_nentry - 1; 20303c9168faSHans Rosenfeld 20313c9168faSHans Rosenfeld c_dw11.b.cq_pc = 1; 20323c9168faSHans Rosenfeld c_dw11.b.cq_ien = 1; 20333c9168faSHans Rosenfeld c_dw11.b.cq_iv = idx % nvme->n_intr_cnt; 20343c9168faSHans Rosenfeld 20353c9168faSHans Rosenfeld cmd->nc_sqid = 0; 20363c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 20373c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE; 20383c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 20393c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = c_dw11.r; 20403c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_cqdma->nd_cookie.dmac_laddress; 20413c9168faSHans Rosenfeld 2042e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 20433c9168faSHans Rosenfeld 2044e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 20453c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20463c9168faSHans Rosenfeld "!CREATE CQUEUE failed with sct = %x, sc = %x", 20473c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 2048e984c70bSHans Rosenfeld goto fail; 20493c9168faSHans Rosenfeld } 20503c9168faSHans Rosenfeld 20513c9168faSHans Rosenfeld nvme_free_cmd(cmd); 20523c9168faSHans Rosenfeld 20533c9168faSHans Rosenfeld s_dw11.b.sq_pc = 1; 20543c9168faSHans Rosenfeld s_dw11.b.sq_cqid = idx; 20553c9168faSHans Rosenfeld 20563c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 20573c9168faSHans Rosenfeld cmd->nc_sqid = 0; 20583c9168faSHans Rosenfeld cmd->nc_callback = nvme_wakeup_cmd; 20593c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE; 20603c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = dw10.r; 20613c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = s_dw11.r; 20623c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress; 20633c9168faSHans Rosenfeld 2064e984c70bSHans Rosenfeld nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); 20653c9168faSHans Rosenfeld 2066e984c70bSHans Rosenfeld if ((ret = nvme_check_cmd_status(cmd)) != 0) { 20673c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 20683c9168faSHans Rosenfeld "!CREATE SQUEUE failed with sct = %x, sc = %x", 20693c9168faSHans Rosenfeld cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 2070e984c70bSHans Rosenfeld goto fail; 20713c9168faSHans Rosenfeld } 20723c9168faSHans Rosenfeld 2073e984c70bSHans Rosenfeld fail: 20743c9168faSHans Rosenfeld nvme_free_cmd(cmd); 20753c9168faSHans Rosenfeld 2076e984c70bSHans Rosenfeld return (ret); 20773c9168faSHans Rosenfeld } 20783c9168faSHans Rosenfeld 20793c9168faSHans Rosenfeld static boolean_t 20803c9168faSHans Rosenfeld nvme_reset(nvme_t *nvme, boolean_t quiesce) 20813c9168faSHans Rosenfeld { 20823c9168faSHans Rosenfeld nvme_reg_csts_t csts; 20833c9168faSHans Rosenfeld int i; 20843c9168faSHans Rosenfeld 20853c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 20863c9168faSHans Rosenfeld 20873c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 20883c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) { 20893c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, 0); 20903c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 20913c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 20923c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) 20933c9168faSHans Rosenfeld break; 20943c9168faSHans Rosenfeld 20953c9168faSHans Rosenfeld if (quiesce) 20963c9168faSHans Rosenfeld drv_usecwait(50000); 20973c9168faSHans Rosenfeld else 20983c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 20993c9168faSHans Rosenfeld } 21003c9168faSHans Rosenfeld } 21013c9168faSHans Rosenfeld 21023c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, 0); 21033c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ASQ, 0); 21043c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_ACQ, 0); 21053c9168faSHans Rosenfeld 21063c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 21073c9168faSHans Rosenfeld return (csts.b.csts_rdy == 0 ? B_TRUE : B_FALSE); 21083c9168faSHans Rosenfeld } 21093c9168faSHans Rosenfeld 21103c9168faSHans Rosenfeld static void 21113c9168faSHans Rosenfeld nvme_shutdown(nvme_t *nvme, int mode, boolean_t quiesce) 21123c9168faSHans Rosenfeld { 21133c9168faSHans Rosenfeld nvme_reg_cc_t cc; 21143c9168faSHans Rosenfeld nvme_reg_csts_t csts; 21153c9168faSHans Rosenfeld int i; 21163c9168faSHans Rosenfeld 21173c9168faSHans Rosenfeld ASSERT(mode == NVME_CC_SHN_NORMAL || mode == NVME_CC_SHN_ABRUPT); 21183c9168faSHans Rosenfeld 21193c9168faSHans Rosenfeld cc.r = nvme_get32(nvme, NVME_REG_CC); 21203c9168faSHans Rosenfeld cc.b.cc_shn = mode & 0x3; 21213c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 21223c9168faSHans Rosenfeld 21233c9168faSHans Rosenfeld for (i = 0; i != 10; i++) { 21243c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 21253c9168faSHans Rosenfeld if (csts.b.csts_shst == NVME_CSTS_SHN_COMPLETE) 21263c9168faSHans Rosenfeld break; 21273c9168faSHans Rosenfeld 21283c9168faSHans Rosenfeld if (quiesce) 21293c9168faSHans Rosenfeld drv_usecwait(100000); 21303c9168faSHans Rosenfeld else 21313c9168faSHans Rosenfeld delay(drv_usectohz(100000)); 21323c9168faSHans Rosenfeld } 21333c9168faSHans Rosenfeld } 21343c9168faSHans Rosenfeld 21353c9168faSHans Rosenfeld 21363c9168faSHans Rosenfeld static void 21373c9168faSHans Rosenfeld nvme_prepare_devid(nvme_t *nvme, uint32_t nsid) 21383c9168faSHans Rosenfeld { 213924979ca3SHans Rosenfeld /* 214024979ca3SHans Rosenfeld * Section 7.7 of the spec describes how to get a unique ID for 214124979ca3SHans Rosenfeld * the controller: the vendor ID, the model name and the serial 214224979ca3SHans Rosenfeld * number shall be unique when combined. 214324979ca3SHans Rosenfeld * 214424979ca3SHans Rosenfeld * If a namespace has no EUI64 we use the above and add the hex 214524979ca3SHans Rosenfeld * namespace ID to get a unique ID for the namespace. 214624979ca3SHans Rosenfeld */ 21473c9168faSHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 21483c9168faSHans Rosenfeld char serial[sizeof (nvme->n_idctl->id_serial) + 1]; 21493c9168faSHans Rosenfeld 21503c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 21513c9168faSHans Rosenfeld bcopy(nvme->n_idctl->id_serial, serial, 21523c9168faSHans Rosenfeld sizeof (nvme->n_idctl->id_serial)); 21533c9168faSHans Rosenfeld 21543c9168faSHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 21553c9168faSHans Rosenfeld serial[sizeof (nvme->n_idctl->id_serial)] = '\0'; 21563c9168faSHans Rosenfeld 215724979ca3SHans Rosenfeld nvme->n_ns[nsid - 1].ns_devid = kmem_asprintf("%4X-%s-%s-%X", 21583c9168faSHans Rosenfeld nvme->n_idctl->id_vid, model, serial, nsid); 21593c9168faSHans Rosenfeld } 21603c9168faSHans Rosenfeld 21613d9b1a2aSHans Rosenfeld static int 21623d9b1a2aSHans Rosenfeld nvme_init_ns(nvme_t *nvme, int nsid) 21633d9b1a2aSHans Rosenfeld { 21643d9b1a2aSHans Rosenfeld nvme_namespace_t *ns = &nvme->n_ns[nsid - 1]; 21653d9b1a2aSHans Rosenfeld nvme_identify_nsid_t *idns; 21663d9b1a2aSHans Rosenfeld int last_rp; 21673d9b1a2aSHans Rosenfeld 21683d9b1a2aSHans Rosenfeld ns->ns_nvme = nvme; 21693d9b1a2aSHans Rosenfeld 2170e984c70bSHans Rosenfeld if (nvme_identify(nvme, nsid, (void **)&idns) != 0) { 21713d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 21723d9b1a2aSHans Rosenfeld "!failed to identify namespace %d", nsid); 21733d9b1a2aSHans Rosenfeld return (DDI_FAILURE); 21743d9b1a2aSHans Rosenfeld } 21753d9b1a2aSHans Rosenfeld 21763d9b1a2aSHans Rosenfeld ns->ns_idns = idns; 21773d9b1a2aSHans Rosenfeld ns->ns_id = nsid; 21783d9b1a2aSHans Rosenfeld ns->ns_block_count = idns->id_nsize; 21793d9b1a2aSHans Rosenfeld ns->ns_block_size = 21803d9b1a2aSHans Rosenfeld 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads; 21813d9b1a2aSHans Rosenfeld ns->ns_best_block_size = ns->ns_block_size; 21823d9b1a2aSHans Rosenfeld 21833d9b1a2aSHans Rosenfeld /* 21843d9b1a2aSHans Rosenfeld * Get the EUI64 if present. Use it for devid and device node names. 21853d9b1a2aSHans Rosenfeld */ 21863d9b1a2aSHans Rosenfeld if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) 21873d9b1a2aSHans Rosenfeld bcopy(idns->id_eui64, ns->ns_eui64, sizeof (ns->ns_eui64)); 21883d9b1a2aSHans Rosenfeld 21893d9b1a2aSHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 21903d9b1a2aSHans Rosenfeld if (*(uint64_t *)ns->ns_eui64 != 0) { 21913d9b1a2aSHans Rosenfeld uint8_t *eui64 = ns->ns_eui64; 21923d9b1a2aSHans Rosenfeld 21933d9b1a2aSHans Rosenfeld (void) snprintf(ns->ns_name, sizeof (ns->ns_name), 21943d9b1a2aSHans Rosenfeld "%02x%02x%02x%02x%02x%02x%02x%02x", 21953d9b1a2aSHans Rosenfeld eui64[0], eui64[1], eui64[2], eui64[3], 21963d9b1a2aSHans Rosenfeld eui64[4], eui64[5], eui64[6], eui64[7]); 21973d9b1a2aSHans Rosenfeld } else { 21983d9b1a2aSHans Rosenfeld (void) snprintf(ns->ns_name, sizeof (ns->ns_name), "%d", 21993d9b1a2aSHans Rosenfeld ns->ns_id); 22003d9b1a2aSHans Rosenfeld 22013d9b1a2aSHans Rosenfeld nvme_prepare_devid(nvme, ns->ns_id); 22023d9b1a2aSHans Rosenfeld } 22033d9b1a2aSHans Rosenfeld 22043d9b1a2aSHans Rosenfeld /* 22053d9b1a2aSHans Rosenfeld * Find the LBA format with no metadata and the best relative 22063d9b1a2aSHans Rosenfeld * performance. A value of 3 means "degraded", 0 is best. 22073d9b1a2aSHans Rosenfeld */ 22083d9b1a2aSHans Rosenfeld last_rp = 3; 22093d9b1a2aSHans Rosenfeld for (int j = 0; j <= idns->id_nlbaf; j++) { 22103d9b1a2aSHans Rosenfeld if (idns->id_lbaf[j].lbaf_lbads == 0) 22113d9b1a2aSHans Rosenfeld break; 22123d9b1a2aSHans Rosenfeld if (idns->id_lbaf[j].lbaf_ms != 0) 22133d9b1a2aSHans Rosenfeld continue; 22143d9b1a2aSHans Rosenfeld if (idns->id_lbaf[j].lbaf_rp >= last_rp) 22153d9b1a2aSHans Rosenfeld continue; 22163d9b1a2aSHans Rosenfeld last_rp = idns->id_lbaf[j].lbaf_rp; 22173d9b1a2aSHans Rosenfeld ns->ns_best_block_size = 22183d9b1a2aSHans Rosenfeld 1 << idns->id_lbaf[j].lbaf_lbads; 22193d9b1a2aSHans Rosenfeld } 22203d9b1a2aSHans Rosenfeld 22213d9b1a2aSHans Rosenfeld if (ns->ns_best_block_size < nvme->n_min_block_size) 22223d9b1a2aSHans Rosenfeld ns->ns_best_block_size = nvme->n_min_block_size; 22233d9b1a2aSHans Rosenfeld 22243d9b1a2aSHans Rosenfeld /* 22253d9b1a2aSHans Rosenfeld * We currently don't support namespaces that use either: 22263d9b1a2aSHans Rosenfeld * - thin provisioning 22273d9b1a2aSHans Rosenfeld * - protection information 2228621738e2SHans Rosenfeld * - illegal block size (< 512) 22293d9b1a2aSHans Rosenfeld */ 22303d9b1a2aSHans Rosenfeld if (idns->id_nsfeat.f_thin || 22313d9b1a2aSHans Rosenfeld idns->id_dps.dp_pinfo) { 22323d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22333d9b1a2aSHans Rosenfeld "!ignoring namespace %d, unsupported features: " 22343d9b1a2aSHans Rosenfeld "thin = %d, pinfo = %d", nsid, 22353d9b1a2aSHans Rosenfeld idns->id_nsfeat.f_thin, idns->id_dps.dp_pinfo); 22363d9b1a2aSHans Rosenfeld ns->ns_ignore = B_TRUE; 2237621738e2SHans Rosenfeld } else if (ns->ns_block_size < 512) { 2238621738e2SHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2239621738e2SHans Rosenfeld "!ignoring namespace %d, unsupported block size %"PRIu64, 2240621738e2SHans Rosenfeld nsid, (uint64_t)ns->ns_block_size); 224131c62b25SHans Rosenfeld ns->ns_ignore = B_TRUE; 22423d9b1a2aSHans Rosenfeld } else { 22433d9b1a2aSHans Rosenfeld ns->ns_ignore = B_FALSE; 22443d9b1a2aSHans Rosenfeld } 22453d9b1a2aSHans Rosenfeld 22463d9b1a2aSHans Rosenfeld return (DDI_SUCCESS); 22473d9b1a2aSHans Rosenfeld } 22483d9b1a2aSHans Rosenfeld 22493c9168faSHans Rosenfeld static int 22503c9168faSHans Rosenfeld nvme_init(nvme_t *nvme) 22513c9168faSHans Rosenfeld { 22523c9168faSHans Rosenfeld nvme_reg_cc_t cc = { 0 }; 22533c9168faSHans Rosenfeld nvme_reg_aqa_t aqa = { 0 }; 22543c9168faSHans Rosenfeld nvme_reg_asq_t asq = { 0 }; 22553c9168faSHans Rosenfeld nvme_reg_acq_t acq = { 0 }; 22563c9168faSHans Rosenfeld nvme_reg_cap_t cap; 22573c9168faSHans Rosenfeld nvme_reg_vs_t vs; 22583c9168faSHans Rosenfeld nvme_reg_csts_t csts; 22593c9168faSHans Rosenfeld int i = 0; 2260e984c70bSHans Rosenfeld uint16_t nqueues; 2261510a6847SHans Rosenfeld char model[sizeof (nvme->n_idctl->id_model) + 1]; 2262510a6847SHans Rosenfeld char *vendor, *product; 22633c9168faSHans Rosenfeld 22643c9168faSHans Rosenfeld /* Check controller version */ 22653c9168faSHans Rosenfeld vs.r = nvme_get32(nvme, NVME_REG_VS); 226624979ca3SHans Rosenfeld nvme->n_version.v_major = vs.b.vs_mjr; 226724979ca3SHans Rosenfeld nvme->n_version.v_minor = vs.b.vs_mnr; 22683c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d", 226924979ca3SHans Rosenfeld nvme->n_version.v_major, nvme->n_version.v_minor); 22703c9168faSHans Rosenfeld 227124979ca3SHans Rosenfeld if (NVME_VERSION_HIGHER(&nvme->n_version, 227224979ca3SHans Rosenfeld nvme_version_major, nvme_version_minor)) { 22733c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.%d", 22743c9168faSHans Rosenfeld nvme_version_major, nvme_version_minor); 22753c9168faSHans Rosenfeld if (nvme->n_strict_version) 22763c9168faSHans Rosenfeld goto fail; 22773c9168faSHans Rosenfeld } 22783c9168faSHans Rosenfeld 22793c9168faSHans Rosenfeld /* retrieve controller configuration */ 22803c9168faSHans Rosenfeld cap.r = nvme_get64(nvme, NVME_REG_CAP); 22813c9168faSHans Rosenfeld 22823c9168faSHans Rosenfeld if ((cap.b.cap_css & NVME_CAP_CSS_NVM) == 0) { 22833c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 22843c9168faSHans Rosenfeld "!NVM command set not supported by hardware"); 22853c9168faSHans Rosenfeld goto fail; 22863c9168faSHans Rosenfeld } 22873c9168faSHans Rosenfeld 22883c9168faSHans Rosenfeld nvme->n_nssr_supported = cap.b.cap_nssrs; 22893c9168faSHans Rosenfeld nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd; 22903c9168faSHans Rosenfeld nvme->n_timeout = cap.b.cap_to; 22913c9168faSHans Rosenfeld nvme->n_arbitration_mechanisms = cap.b.cap_ams; 22923c9168faSHans Rosenfeld nvme->n_cont_queues_reqd = cap.b.cap_cqr; 22933c9168faSHans Rosenfeld nvme->n_max_queue_entries = cap.b.cap_mqes + 1; 22943c9168faSHans Rosenfeld 22953c9168faSHans Rosenfeld /* 22963c9168faSHans Rosenfeld * The MPSMIN and MPSMAX fields in the CAP register use 0 to specify 22973c9168faSHans Rosenfeld * the base page size of 4k (1<<12), so add 12 here to get the real 22983c9168faSHans Rosenfeld * page size value. 22993c9168faSHans Rosenfeld */ 23003c9168faSHans Rosenfeld nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT), 23013c9168faSHans Rosenfeld cap.b.cap_mpsmax + 12); 23023c9168faSHans Rosenfeld nvme->n_pagesize = 1UL << (nvme->n_pageshift); 23033c9168faSHans Rosenfeld 23043c9168faSHans Rosenfeld /* 23053c9168faSHans Rosenfeld * Set up Queue DMA to transfer at least 1 page-aligned page at a time. 23063c9168faSHans Rosenfeld */ 23073c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize; 23083c9168faSHans Rosenfeld nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 23093c9168faSHans Rosenfeld 23103c9168faSHans Rosenfeld /* 23113c9168faSHans Rosenfeld * Set up PRP DMA to transfer 1 page-aligned page at a time. 23123c9168faSHans Rosenfeld * Maxxfer may be increased after we identified the controller limits. 23133c9168faSHans Rosenfeld */ 23143c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize; 23153c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 23163c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize; 23172f95345bSYouzhong Yang nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1; 23183c9168faSHans Rosenfeld 23193c9168faSHans Rosenfeld /* 23203c9168faSHans Rosenfeld * Reset controller if it's still in ready state. 23213c9168faSHans Rosenfeld */ 23223c9168faSHans Rosenfeld if (nvme_reset(nvme, B_FALSE) == B_FALSE) { 23233c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller"); 23243c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 23253c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 23263c9168faSHans Rosenfeld goto fail; 23273c9168faSHans Rosenfeld } 23283c9168faSHans Rosenfeld 23293c9168faSHans Rosenfeld /* 23303c9168faSHans Rosenfeld * Create the admin queue pair. 23313c9168faSHans Rosenfeld */ 23323c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0) 23333c9168faSHans Rosenfeld != DDI_SUCCESS) { 23343c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23353c9168faSHans Rosenfeld "!unable to allocate admin qpair"); 23363c9168faSHans Rosenfeld goto fail; 23373c9168faSHans Rosenfeld } 23383c9168faSHans Rosenfeld nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP); 23393c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 23403c9168faSHans Rosenfeld 23413c9168faSHans Rosenfeld nvme->n_progress |= NVME_ADMIN_QUEUE; 23423c9168faSHans Rosenfeld 23433c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 23443c9168faSHans Rosenfeld "admin-queue-len", nvme->n_admin_queue_len); 23453c9168faSHans Rosenfeld 23463c9168faSHans Rosenfeld aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1; 23473c9168faSHans Rosenfeld asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress; 23483c9168faSHans Rosenfeld acq = nvme->n_adminq->nq_cqdma->nd_cookie.dmac_laddress; 23493c9168faSHans Rosenfeld 23503c9168faSHans Rosenfeld ASSERT((asq & (nvme->n_pagesize - 1)) == 0); 23513c9168faSHans Rosenfeld ASSERT((acq & (nvme->n_pagesize - 1)) == 0); 23523c9168faSHans Rosenfeld 23533c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_AQA, aqa.r); 23543c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ASQ, asq); 23553c9168faSHans Rosenfeld nvme_put64(nvme, NVME_REG_ACQ, acq); 23563c9168faSHans Rosenfeld 235734c938c7SPete Shephard cc.b.cc_ams = 0; /* use Round-Robin arbitration */ 235834c938c7SPete Shephard cc.b.cc_css = 0; /* use NVM command set */ 23593c9168faSHans Rosenfeld cc.b.cc_mps = nvme->n_pageshift - 12; 236034c938c7SPete Shephard cc.b.cc_shn = 0; /* no shutdown in progress */ 236134c938c7SPete Shephard cc.b.cc_en = 1; /* enable controller */ 236234c938c7SPete Shephard cc.b.cc_iosqes = 6; /* submission queue entry is 2^6 bytes long */ 236334c938c7SPete Shephard cc.b.cc_iocqes = 4; /* completion queue entry is 2^4 bytes long */ 23643c9168faSHans Rosenfeld 23653c9168faSHans Rosenfeld nvme_put32(nvme, NVME_REG_CC, cc.r); 23663c9168faSHans Rosenfeld 23673c9168faSHans Rosenfeld /* 23683c9168faSHans Rosenfeld * Wait for the controller to become ready. 23693c9168faSHans Rosenfeld */ 23703c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 23713c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 23723c9168faSHans Rosenfeld for (i = 0; i != nvme->n_timeout * 10; i++) { 23733c9168faSHans Rosenfeld delay(drv_usectohz(50000)); 23743c9168faSHans Rosenfeld csts.r = nvme_get32(nvme, NVME_REG_CSTS); 23753c9168faSHans Rosenfeld 23763c9168faSHans Rosenfeld if (csts.b.csts_cfs == 1) { 23773c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 23783c9168faSHans Rosenfeld "!controller fatal status at init"); 23793c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, 23803c9168faSHans Rosenfeld DDI_SERVICE_LOST); 23813c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 23823c9168faSHans Rosenfeld goto fail; 23833c9168faSHans Rosenfeld } 23843c9168faSHans Rosenfeld 23853c9168faSHans Rosenfeld if (csts.b.csts_rdy == 1) 23863c9168faSHans Rosenfeld break; 23873c9168faSHans Rosenfeld } 23883c9168faSHans Rosenfeld } 23893c9168faSHans Rosenfeld 23903c9168faSHans Rosenfeld if (csts.b.csts_rdy == 0) { 23913c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!controller not ready"); 23923c9168faSHans Rosenfeld ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 23933c9168faSHans Rosenfeld nvme->n_dead = B_TRUE; 23943c9168faSHans Rosenfeld goto fail; 23953c9168faSHans Rosenfeld } 23963c9168faSHans Rosenfeld 23973c9168faSHans Rosenfeld /* 23983c9168faSHans Rosenfeld * Assume an abort command limit of 1. We'll destroy and re-init 23993c9168faSHans Rosenfeld * that later when we know the true abort command limit. 24003c9168faSHans Rosenfeld */ 24013c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL); 24023c9168faSHans Rosenfeld 2403bf26ea4bSHans Rosenfeld /* 2404bf26ea4bSHans Rosenfeld * Setup initial interrupt for admin queue. 2405bf26ea4bSHans Rosenfeld */ 2406bf26ea4bSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1) 2407bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 2408bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1) 2409bf26ea4bSHans Rosenfeld != DDI_SUCCESS) && 2410bf26ea4bSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1) 2411bf26ea4bSHans Rosenfeld != DDI_SUCCESS)) { 2412bf26ea4bSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2413bf26ea4bSHans Rosenfeld "!failed to setup initial interrupt"); 2414bf26ea4bSHans Rosenfeld goto fail; 2415bf26ea4bSHans Rosenfeld } 2416bf26ea4bSHans Rosenfeld 24173c9168faSHans Rosenfeld /* 24183c9168faSHans Rosenfeld * Post an asynchronous event command to catch errors. 241908139162SToomas Soome * We assume the asynchronous events are supported as required by 242008139162SToomas Soome * specification (Figure 40 in section 5 of NVMe 1.2). 242108139162SToomas Soome * However, since at least qemu does not follow the specification, 242208139162SToomas Soome * we need a mechanism to protect ourselves. 24233c9168faSHans Rosenfeld */ 242408139162SToomas Soome nvme->n_async_event_supported = B_TRUE; 24254b324362SHans Rosenfeld nvme_async_event(nvme); 24263c9168faSHans Rosenfeld 24273c9168faSHans Rosenfeld /* 24283c9168faSHans Rosenfeld * Identify Controller 24293c9168faSHans Rosenfeld */ 2430e984c70bSHans Rosenfeld if (nvme_identify(nvme, 0, (void **)&nvme->n_idctl) != 0) { 24313c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 24323c9168faSHans Rosenfeld "!failed to identify controller"); 24333c9168faSHans Rosenfeld goto fail; 24343c9168faSHans Rosenfeld } 24353c9168faSHans Rosenfeld 2436510a6847SHans Rosenfeld /* 2437510a6847SHans Rosenfeld * Get Vendor & Product ID 2438510a6847SHans Rosenfeld */ 2439510a6847SHans Rosenfeld bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 2440510a6847SHans Rosenfeld model[sizeof (nvme->n_idctl->id_model)] = '\0'; 2441510a6847SHans Rosenfeld sata_split_model(model, &vendor, &product); 2442510a6847SHans Rosenfeld 2443510a6847SHans Rosenfeld if (vendor == NULL) 2444510a6847SHans Rosenfeld nvme->n_vendor = strdup("NVMe"); 2445510a6847SHans Rosenfeld else 2446510a6847SHans Rosenfeld nvme->n_vendor = strdup(vendor); 2447510a6847SHans Rosenfeld 2448510a6847SHans Rosenfeld nvme->n_product = strdup(product); 2449510a6847SHans Rosenfeld 24503c9168faSHans Rosenfeld /* 24513c9168faSHans Rosenfeld * Get controller limits. 24523c9168faSHans Rosenfeld */ 24533c9168faSHans Rosenfeld nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT, 24543c9168faSHans Rosenfeld MIN(nvme->n_admin_queue_len / 10, 24553c9168faSHans Rosenfeld MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit))); 24563c9168faSHans Rosenfeld 24573c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 24583c9168faSHans Rosenfeld "async-event-limit", nvme->n_async_event_limit); 24593c9168faSHans Rosenfeld 24603c9168faSHans Rosenfeld nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1; 24613c9168faSHans Rosenfeld 24626afc9eb2SHans Rosenfeld /* 24636afc9eb2SHans Rosenfeld * Reinitialize the semaphore with the true abort command limit 24646afc9eb2SHans Rosenfeld * supported by the hardware. It's not necessary to disable interrupts 24656afc9eb2SHans Rosenfeld * as only command aborts use the semaphore, and no commands are 24666afc9eb2SHans Rosenfeld * executed or aborted while we're here. 24676afc9eb2SHans Rosenfeld */ 24683c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 24693c9168faSHans Rosenfeld sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL, 24703c9168faSHans Rosenfeld SEMA_DRIVER, NULL); 24713c9168faSHans Rosenfeld 24723c9168faSHans Rosenfeld nvme->n_progress |= NVME_CTRL_LIMITS; 24733c9168faSHans Rosenfeld 24743c9168faSHans Rosenfeld if (nvme->n_idctl->id_mdts == 0) 24753c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536; 24763c9168faSHans Rosenfeld else 24773c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = 24783c9168faSHans Rosenfeld 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts); 24793c9168faSHans Rosenfeld 24803c9168faSHans Rosenfeld nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1; 24813c9168faSHans Rosenfeld 24823c9168faSHans Rosenfeld /* 24833c9168faSHans Rosenfeld * Limit n_max_data_transfer_size to what we can handle in one PRP. 24843c9168faSHans Rosenfeld * Chained PRPs are currently unsupported. 24853c9168faSHans Rosenfeld * 24863c9168faSHans Rosenfeld * This is a no-op on hardware which doesn't support a transfer size 24873c9168faSHans Rosenfeld * big enough to require chained PRPs. 24883c9168faSHans Rosenfeld */ 24893c9168faSHans Rosenfeld nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size, 24903c9168faSHans Rosenfeld (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize)); 24913c9168faSHans Rosenfeld 24923c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size; 24933c9168faSHans Rosenfeld 24943c9168faSHans Rosenfeld /* 24953c9168faSHans Rosenfeld * Make sure the minimum/maximum queue entry sizes are not 24963c9168faSHans Rosenfeld * larger/smaller than the default. 24973c9168faSHans Rosenfeld */ 24983c9168faSHans Rosenfeld 24993c9168faSHans Rosenfeld if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) || 25003c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) || 25013c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) || 25023c9168faSHans Rosenfeld ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t))) 25033c9168faSHans Rosenfeld goto fail; 25043c9168faSHans Rosenfeld 25053c9168faSHans Rosenfeld /* 25063c9168faSHans Rosenfeld * Check for the presence of a Volatile Write Cache. If present, 2507d148d46eSHans Rosenfeld * enable or disable based on the value of the property 2508d148d46eSHans Rosenfeld * volatile-write-cache-enable (default is enabled). 25093c9168faSHans Rosenfeld */ 2510d148d46eSHans Rosenfeld nvme->n_write_cache_present = 2511d148d46eSHans Rosenfeld nvme->n_idctl->id_vwc.vwc_present == 0 ? B_FALSE : B_TRUE; 2512d148d46eSHans Rosenfeld 2513d148d46eSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2514d148d46eSHans Rosenfeld "volatile-write-cache-present", 2515d148d46eSHans Rosenfeld nvme->n_write_cache_present ? 1 : 0); 2516d148d46eSHans Rosenfeld 2517d148d46eSHans Rosenfeld if (!nvme->n_write_cache_present) { 2518d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = B_FALSE; 2519e984c70bSHans Rosenfeld } else if (nvme_write_cache_set(nvme, nvme->n_write_cache_enabled) 2520e984c70bSHans Rosenfeld != 0) { 2521d148d46eSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2522d148d46eSHans Rosenfeld "!failed to %sable volatile write cache", 2523d148d46eSHans Rosenfeld nvme->n_write_cache_enabled ? "en" : "dis"); 25243c9168faSHans Rosenfeld /* 2525d148d46eSHans Rosenfeld * Assume the cache is (still) enabled. 25263c9168faSHans Rosenfeld */ 2527d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = B_TRUE; 25283c9168faSHans Rosenfeld } 25293c9168faSHans Rosenfeld 2530d148d46eSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2531d148d46eSHans Rosenfeld "volatile-write-cache-enable", 2532d148d46eSHans Rosenfeld nvme->n_write_cache_enabled ? 1 : 0); 2533d148d46eSHans Rosenfeld 25343c9168faSHans Rosenfeld /* 25353d9b1a2aSHans Rosenfeld * Assume LBA Range Type feature is supported. If it isn't this 25363d9b1a2aSHans Rosenfeld * will be set to B_FALSE by nvme_get_features(). 25373c9168faSHans Rosenfeld */ 25383d9b1a2aSHans Rosenfeld nvme->n_lba_range_supported = B_TRUE; 25393d9b1a2aSHans Rosenfeld 25403d9b1a2aSHans Rosenfeld /* 25413d9b1a2aSHans Rosenfeld * Check support for Autonomous Power State Transition. 25423d9b1a2aSHans Rosenfeld */ 25433d9b1a2aSHans Rosenfeld if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) 25443d9b1a2aSHans Rosenfeld nvme->n_auto_pst_supported = 25453d9b1a2aSHans Rosenfeld nvme->n_idctl->id_apsta.ap_sup == 0 ? B_FALSE : B_TRUE; 25463c9168faSHans Rosenfeld 2547*f313c178SYuri Pankov /* 2548*f313c178SYuri Pankov * Assume Software Progress Marker feature is supported. If it isn't 2549*f313c178SYuri Pankov * this will be set to B_FALSE by nvme_get_features(). 2550*f313c178SYuri Pankov */ 2551*f313c178SYuri Pankov nvme->n_progress_supported = B_TRUE; 2552*f313c178SYuri Pankov 25533c9168faSHans Rosenfeld /* 25543c9168faSHans Rosenfeld * Identify Namespaces 25553c9168faSHans Rosenfeld */ 25563c9168faSHans Rosenfeld nvme->n_namespace_count = nvme->n_idctl->id_nn; 25573d9b1a2aSHans Rosenfeld if (nvme->n_namespace_count > NVME_MINOR_MAX) { 25583d9b1a2aSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 25593d9b1a2aSHans Rosenfeld "!too many namespaces: %d, limiting to %d\n", 25603d9b1a2aSHans Rosenfeld nvme->n_namespace_count, NVME_MINOR_MAX); 25613d9b1a2aSHans Rosenfeld nvme->n_namespace_count = NVME_MINOR_MAX; 25623d9b1a2aSHans Rosenfeld } 25633d9b1a2aSHans Rosenfeld 25643c9168faSHans Rosenfeld nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) * 25653c9168faSHans Rosenfeld nvme->n_namespace_count, KM_SLEEP); 25663c9168faSHans Rosenfeld 25673c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 25683d9b1a2aSHans Rosenfeld mutex_init(&nvme->n_ns[i].ns_minor.nm_mutex, NULL, MUTEX_DRIVER, 25693d9b1a2aSHans Rosenfeld NULL); 25703d9b1a2aSHans Rosenfeld if (nvme_init_ns(nvme, i + 1) != DDI_SUCCESS) 25713c9168faSHans Rosenfeld goto fail; 25723c9168faSHans Rosenfeld } 25733c9168faSHans Rosenfeld 25743c9168faSHans Rosenfeld /* 25753c9168faSHans Rosenfeld * Try to set up MSI/MSI-X interrupts. 25763c9168faSHans Rosenfeld */ 25773c9168faSHans Rosenfeld if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX)) 25783c9168faSHans Rosenfeld != 0) { 25793c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 25803c9168faSHans Rosenfeld 25813c9168faSHans Rosenfeld nqueues = MIN(UINT16_MAX, ncpus); 25823c9168faSHans Rosenfeld 25833c9168faSHans Rosenfeld if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 25843c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS) && 25853c9168faSHans Rosenfeld (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 25863c9168faSHans Rosenfeld nqueues) != DDI_SUCCESS)) { 25873c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 25883c9168faSHans Rosenfeld "!failed to setup MSI/MSI-X interrupts"); 25893c9168faSHans Rosenfeld goto fail; 25903c9168faSHans Rosenfeld } 25913c9168faSHans Rosenfeld } 25923c9168faSHans Rosenfeld 25933c9168faSHans Rosenfeld nqueues = nvme->n_intr_cnt; 25943c9168faSHans Rosenfeld 25953c9168faSHans Rosenfeld /* 25963c9168faSHans Rosenfeld * Create I/O queue pairs. 25973c9168faSHans Rosenfeld */ 2598e984c70bSHans Rosenfeld 2599e984c70bSHans Rosenfeld if (nvme_set_nqueues(nvme, &nqueues) != 0) { 26003c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 2601e984c70bSHans Rosenfeld "!failed to set number of I/O queues to %d", 2602e984c70bSHans Rosenfeld nvme->n_intr_cnt); 26033c9168faSHans Rosenfeld goto fail; 26043c9168faSHans Rosenfeld } 26053c9168faSHans Rosenfeld 26063c9168faSHans Rosenfeld /* 26073c9168faSHans Rosenfeld * Reallocate I/O queue array 26083c9168faSHans Rosenfeld */ 26093c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *)); 26103c9168faSHans Rosenfeld nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) * 2611e984c70bSHans Rosenfeld (nqueues + 1), KM_SLEEP); 26123c9168faSHans Rosenfeld nvme->n_ioq[0] = nvme->n_adminq; 26133c9168faSHans Rosenfeld 2614e984c70bSHans Rosenfeld nvme->n_ioq_count = nqueues; 2615e984c70bSHans Rosenfeld 26163c9168faSHans Rosenfeld /* 26173c9168faSHans Rosenfeld * If we got less queues than we asked for we might as well give 26183c9168faSHans Rosenfeld * some of the interrupt vectors back to the system. 26193c9168faSHans Rosenfeld */ 2620e984c70bSHans Rosenfeld if (nvme->n_ioq_count < nvme->n_intr_cnt) { 26213c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 26223c9168faSHans Rosenfeld 262334c938c7SPete Shephard if (nvme_setup_interrupts(nvme, nvme->n_intr_type, 262434c938c7SPete Shephard nvme->n_ioq_count) != DDI_SUCCESS) { 26253c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 26263c9168faSHans Rosenfeld "!failed to reduce number of interrupts"); 26273c9168faSHans Rosenfeld goto fail; 26283c9168faSHans Rosenfeld } 26293c9168faSHans Rosenfeld } 26303c9168faSHans Rosenfeld 26313c9168faSHans Rosenfeld /* 26323c9168faSHans Rosenfeld * Alloc & register I/O queue pairs 26333c9168faSHans Rosenfeld */ 26343c9168faSHans Rosenfeld nvme->n_io_queue_len = 26353c9168faSHans Rosenfeld MIN(nvme->n_io_queue_len, nvme->n_max_queue_entries); 26363c9168faSHans Rosenfeld (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-queue-len", 26373c9168faSHans Rosenfeld nvme->n_io_queue_len); 26383c9168faSHans Rosenfeld 26393c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 26403c9168faSHans Rosenfeld if (nvme_alloc_qpair(nvme, nvme->n_io_queue_len, 26413c9168faSHans Rosenfeld &nvme->n_ioq[i], i) != DDI_SUCCESS) { 26423c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 26433c9168faSHans Rosenfeld "!unable to allocate I/O qpair %d", i); 26443c9168faSHans Rosenfeld goto fail; 26453c9168faSHans Rosenfeld } 26463c9168faSHans Rosenfeld 2647e984c70bSHans Rosenfeld if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i) != 0) { 26483c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 26493c9168faSHans Rosenfeld "!unable to create I/O qpair %d", i); 26503c9168faSHans Rosenfeld goto fail; 26513c9168faSHans Rosenfeld } 26523c9168faSHans Rosenfeld } 26533c9168faSHans Rosenfeld 26543c9168faSHans Rosenfeld /* 26553c9168faSHans Rosenfeld * Post more asynchronous events commands to reduce event reporting 26563c9168faSHans Rosenfeld * latency as suggested by the spec. 26573c9168faSHans Rosenfeld */ 265808139162SToomas Soome if (nvme->n_async_event_supported) { 265908139162SToomas Soome for (i = 1; i != nvme->n_async_event_limit; i++) 266008139162SToomas Soome nvme_async_event(nvme); 266108139162SToomas Soome } 26623c9168faSHans Rosenfeld 26633c9168faSHans Rosenfeld return (DDI_SUCCESS); 26643c9168faSHans Rosenfeld 26653c9168faSHans Rosenfeld fail: 26663c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 26673c9168faSHans Rosenfeld return (DDI_FAILURE); 26683c9168faSHans Rosenfeld } 26693c9168faSHans Rosenfeld 26703c9168faSHans Rosenfeld static uint_t 26713c9168faSHans Rosenfeld nvme_intr(caddr_t arg1, caddr_t arg2) 26723c9168faSHans Rosenfeld { 26733c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 26743c9168faSHans Rosenfeld nvme_t *nvme = (nvme_t *)arg1; 26753c9168faSHans Rosenfeld int inum = (int)(uintptr_t)arg2; 2676bf26ea4bSHans Rosenfeld int ccnt = 0; 26773c9168faSHans Rosenfeld int qnum; 26783c9168faSHans Rosenfeld nvme_cmd_t *cmd; 26793c9168faSHans Rosenfeld 26803c9168faSHans Rosenfeld if (inum >= nvme->n_intr_cnt) 26813c9168faSHans Rosenfeld return (DDI_INTR_UNCLAIMED); 26823c9168faSHans Rosenfeld 2683e984c70bSHans Rosenfeld if (nvme->n_dead) 2684e984c70bSHans Rosenfeld return (nvme->n_intr_type == DDI_INTR_TYPE_FIXED ? 2685e984c70bSHans Rosenfeld DDI_INTR_UNCLAIMED : DDI_INTR_CLAIMED); 2686e984c70bSHans Rosenfeld 26873c9168faSHans Rosenfeld /* 26883c9168faSHans Rosenfeld * The interrupt vector a queue uses is calculated as queue_idx % 26893c9168faSHans Rosenfeld * intr_cnt in nvme_create_io_qpair(). Iterate through the queue array 26903c9168faSHans Rosenfeld * in steps of n_intr_cnt to process all queues using this vector. 26913c9168faSHans Rosenfeld */ 26923c9168faSHans Rosenfeld for (qnum = inum; 26933c9168faSHans Rosenfeld qnum < nvme->n_ioq_count + 1 && nvme->n_ioq[qnum] != NULL; 26943c9168faSHans Rosenfeld qnum += nvme->n_intr_cnt) { 26953c9168faSHans Rosenfeld while ((cmd = nvme_retrieve_cmd(nvme, nvme->n_ioq[qnum]))) { 26963c9168faSHans Rosenfeld taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq, 26973c9168faSHans Rosenfeld cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent); 2698bf26ea4bSHans Rosenfeld ccnt++; 26993c9168faSHans Rosenfeld } 27003c9168faSHans Rosenfeld } 27013c9168faSHans Rosenfeld 2702bf26ea4bSHans Rosenfeld return (ccnt > 0 ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); 27033c9168faSHans Rosenfeld } 27043c9168faSHans Rosenfeld 27053c9168faSHans Rosenfeld static void 27066afc9eb2SHans Rosenfeld nvme_release_interrupts(nvme_t *nvme) 27073c9168faSHans Rosenfeld { 27083c9168faSHans Rosenfeld int i; 27093c9168faSHans Rosenfeld 27103c9168faSHans Rosenfeld for (i = 0; i < nvme->n_intr_cnt; i++) { 27113c9168faSHans Rosenfeld if (nvme->n_inth[i] == NULL) 27123c9168faSHans Rosenfeld break; 27133c9168faSHans Rosenfeld 27143c9168faSHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 27153c9168faSHans Rosenfeld (void) ddi_intr_block_disable(&nvme->n_inth[i], 1); 27163c9168faSHans Rosenfeld else 27173c9168faSHans Rosenfeld (void) ddi_intr_disable(nvme->n_inth[i]); 27183c9168faSHans Rosenfeld 27193c9168faSHans Rosenfeld (void) ddi_intr_remove_handler(nvme->n_inth[i]); 27203c9168faSHans Rosenfeld (void) ddi_intr_free(nvme->n_inth[i]); 27213c9168faSHans Rosenfeld } 27223c9168faSHans Rosenfeld 27233c9168faSHans Rosenfeld kmem_free(nvme->n_inth, nvme->n_inth_sz); 27243c9168faSHans Rosenfeld nvme->n_inth = NULL; 27253c9168faSHans Rosenfeld nvme->n_inth_sz = 0; 27263c9168faSHans Rosenfeld 27273c9168faSHans Rosenfeld nvme->n_progress &= ~NVME_INTERRUPTS; 27283c9168faSHans Rosenfeld } 27293c9168faSHans Rosenfeld 27303c9168faSHans Rosenfeld static int 27313c9168faSHans Rosenfeld nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs) 27323c9168faSHans Rosenfeld { 27333c9168faSHans Rosenfeld int nintrs, navail, count; 27343c9168faSHans Rosenfeld int ret; 27353c9168faSHans Rosenfeld int i; 27363c9168faSHans Rosenfeld 27373c9168faSHans Rosenfeld if (nvme->n_intr_types == 0) { 27383c9168faSHans Rosenfeld ret = ddi_intr_get_supported_types(nvme->n_dip, 27393c9168faSHans Rosenfeld &nvme->n_intr_types); 27403c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 27413c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 27423c9168faSHans Rosenfeld "!%s: ddi_intr_get_supported types failed", 27433c9168faSHans Rosenfeld __func__); 27443c9168faSHans Rosenfeld return (ret); 27453c9168faSHans Rosenfeld } 27469d08e1f8SHans Rosenfeld #ifdef __x86 27479d08e1f8SHans Rosenfeld if (get_hwenv() == HW_VMWARE) 27489d08e1f8SHans Rosenfeld nvme->n_intr_types &= ~DDI_INTR_TYPE_MSIX; 27499d08e1f8SHans Rosenfeld #endif 27503c9168faSHans Rosenfeld } 27513c9168faSHans Rosenfeld 27523c9168faSHans Rosenfeld if ((nvme->n_intr_types & intr_type) == 0) 27533c9168faSHans Rosenfeld return (DDI_FAILURE); 27543c9168faSHans Rosenfeld 27553c9168faSHans Rosenfeld ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs); 27563c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 27573c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed", 27583c9168faSHans Rosenfeld __func__); 27593c9168faSHans Rosenfeld return (ret); 27603c9168faSHans Rosenfeld } 27613c9168faSHans Rosenfeld 27623c9168faSHans Rosenfeld ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail); 27633c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 27643c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed", 27653c9168faSHans Rosenfeld __func__); 27663c9168faSHans Rosenfeld return (ret); 27673c9168faSHans Rosenfeld } 27683c9168faSHans Rosenfeld 27693c9168faSHans Rosenfeld /* We want at most one interrupt per queue pair. */ 27703c9168faSHans Rosenfeld if (navail > nqpairs) 27713c9168faSHans Rosenfeld navail = nqpairs; 27723c9168faSHans Rosenfeld 27733c9168faSHans Rosenfeld nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail; 27743c9168faSHans Rosenfeld nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP); 27753c9168faSHans Rosenfeld 27763c9168faSHans Rosenfeld ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail, 27773c9168faSHans Rosenfeld &count, 0); 27783c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 27793c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed", 27803c9168faSHans Rosenfeld __func__); 27813c9168faSHans Rosenfeld goto fail; 27823c9168faSHans Rosenfeld } 27833c9168faSHans Rosenfeld 27843c9168faSHans Rosenfeld nvme->n_intr_cnt = count; 27853c9168faSHans Rosenfeld 27863c9168faSHans Rosenfeld ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri); 27873c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 27883c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed", 27893c9168faSHans Rosenfeld __func__); 27903c9168faSHans Rosenfeld goto fail; 27913c9168faSHans Rosenfeld } 27923c9168faSHans Rosenfeld 27933c9168faSHans Rosenfeld for (i = 0; i < count; i++) { 27943c9168faSHans Rosenfeld ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr, 27953c9168faSHans Rosenfeld (void *)nvme, (void *)(uintptr_t)i); 27963c9168faSHans Rosenfeld if (ret != DDI_SUCCESS) { 27973c9168faSHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 27983c9168faSHans Rosenfeld "!%s: ddi_intr_add_handler failed", __func__); 27993c9168faSHans Rosenfeld goto fail; 28003c9168faSHans Rosenfeld } 28013c9168faSHans Rosenfeld } 28023c9168faSHans Rosenfeld 28033c9168faSHans Rosenfeld (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap); 28043c9168faSHans Rosenfeld 28056afc9eb2SHans Rosenfeld for (i = 0; i < count; i++) { 280675b41617SHans Rosenfeld if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 280775b41617SHans Rosenfeld ret = ddi_intr_block_enable(&nvme->n_inth[i], 1); 280875b41617SHans Rosenfeld else 280975b41617SHans Rosenfeld ret = ddi_intr_enable(nvme->n_inth[i]); 28103c9168faSHans Rosenfeld 281175b41617SHans Rosenfeld if (ret != DDI_SUCCESS) { 281275b41617SHans Rosenfeld dev_err(nvme->n_dip, CE_WARN, 281375b41617SHans Rosenfeld "!%s: enabling interrupt %d failed", __func__, i); 281475b41617SHans Rosenfeld goto fail; 28156afc9eb2SHans Rosenfeld } 28166afc9eb2SHans Rosenfeld } 28176afc9eb2SHans Rosenfeld 28183c9168faSHans Rosenfeld nvme->n_intr_type = intr_type; 28193c9168faSHans Rosenfeld 28203c9168faSHans Rosenfeld nvme->n_progress |= NVME_INTERRUPTS; 28213c9168faSHans Rosenfeld 28223c9168faSHans Rosenfeld return (DDI_SUCCESS); 28233c9168faSHans Rosenfeld 28243c9168faSHans Rosenfeld fail: 28253c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 28263c9168faSHans Rosenfeld 28273c9168faSHans Rosenfeld return (ret); 28283c9168faSHans Rosenfeld } 28293c9168faSHans Rosenfeld 28303c9168faSHans Rosenfeld static int 28313c9168faSHans Rosenfeld nvme_fm_errcb(dev_info_t *dip, ddi_fm_error_t *fm_error, const void *arg) 28323c9168faSHans Rosenfeld { 28333c9168faSHans Rosenfeld _NOTE(ARGUNUSED(arg)); 28343c9168faSHans Rosenfeld 28353c9168faSHans Rosenfeld pci_ereport_post(dip, fm_error, NULL); 28363c9168faSHans Rosenfeld return (fm_error->fme_status); 28373c9168faSHans Rosenfeld } 28383c9168faSHans Rosenfeld 28393c9168faSHans Rosenfeld static int 28403c9168faSHans Rosenfeld nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 28413c9168faSHans Rosenfeld { 28423c9168faSHans Rosenfeld nvme_t *nvme; 28433c9168faSHans Rosenfeld int instance; 28443c9168faSHans Rosenfeld int nregs; 28453c9168faSHans Rosenfeld off_t regsize; 28463c9168faSHans Rosenfeld int i; 28473c9168faSHans Rosenfeld char name[32]; 28483c9168faSHans Rosenfeld 28493c9168faSHans Rosenfeld if (cmd != DDI_ATTACH) 28503c9168faSHans Rosenfeld return (DDI_FAILURE); 28513c9168faSHans Rosenfeld 28523c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 28533c9168faSHans Rosenfeld 28543c9168faSHans Rosenfeld if (ddi_soft_state_zalloc(nvme_state, instance) != DDI_SUCCESS) 28553c9168faSHans Rosenfeld return (DDI_FAILURE); 28563c9168faSHans Rosenfeld 28573c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 28583c9168faSHans Rosenfeld ddi_set_driver_private(dip, nvme); 28593c9168faSHans Rosenfeld nvme->n_dip = dip; 28603c9168faSHans Rosenfeld 28613d9b1a2aSHans Rosenfeld mutex_init(&nvme->n_minor.nm_mutex, NULL, MUTEX_DRIVER, NULL); 28623d9b1a2aSHans Rosenfeld 28633c9168faSHans Rosenfeld nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 28643c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "strict-version", 1) == 1 ? B_TRUE : B_FALSE; 28653c9168faSHans Rosenfeld nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY, 28663c9168faSHans Rosenfeld dip, DDI_PROP_DONTPASS, "ignore-unknown-vendor-status", 0) == 1 ? 28673c9168faSHans Rosenfeld B_TRUE : B_FALSE; 28683c9168faSHans Rosenfeld nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 28693c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "admin-queue-len", NVME_DEFAULT_ADMIN_QUEUE_LEN); 28703c9168faSHans Rosenfeld nvme->n_io_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 28713c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "io-queue-len", NVME_DEFAULT_IO_QUEUE_LEN); 28723c9168faSHans Rosenfeld nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 28733c9168faSHans Rosenfeld DDI_PROP_DONTPASS, "async-event-limit", 28743c9168faSHans Rosenfeld NVME_DEFAULT_ASYNC_EVENT_LIMIT); 2875d148d46eSHans Rosenfeld nvme->n_write_cache_enabled = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2876d148d46eSHans Rosenfeld DDI_PROP_DONTPASS, "volatile-write-cache-enable", 1) != 0 ? 2877d148d46eSHans Rosenfeld B_TRUE : B_FALSE; 28786801591eSHans Rosenfeld nvme->n_min_block_size = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 28796801591eSHans Rosenfeld DDI_PROP_DONTPASS, "min-phys-block-size", 28806801591eSHans Rosenfeld NVME_DEFAULT_MIN_BLOCK_SIZE); 28816801591eSHans Rosenfeld 28826801591eSHans Rosenfeld if (!ISP2(nvme->n_min_block_size) || 28836801591eSHans Rosenfeld (nvme->n_min_block_size < NVME_DEFAULT_MIN_BLOCK_SIZE)) { 28846801591eSHans Rosenfeld dev_err(dip, CE_WARN, "!min-phys-block-size %s, " 28856801591eSHans Rosenfeld "using default %d", ISP2(nvme->n_min_block_size) ? 28866801591eSHans Rosenfeld "too low" : "not a power of 2", 28876801591eSHans Rosenfeld NVME_DEFAULT_MIN_BLOCK_SIZE); 28886801591eSHans Rosenfeld nvme->n_min_block_size = NVME_DEFAULT_MIN_BLOCK_SIZE; 28896801591eSHans Rosenfeld } 28903c9168faSHans Rosenfeld 28913c9168faSHans Rosenfeld if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN) 28923c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN; 28933c9168faSHans Rosenfeld else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN) 28943c9168faSHans Rosenfeld nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN; 28953c9168faSHans Rosenfeld 28963c9168faSHans Rosenfeld if (nvme->n_io_queue_len < NVME_MIN_IO_QUEUE_LEN) 28973c9168faSHans Rosenfeld nvme->n_io_queue_len = NVME_MIN_IO_QUEUE_LEN; 28983c9168faSHans Rosenfeld 28993c9168faSHans Rosenfeld if (nvme->n_async_event_limit < 1) 29003c9168faSHans Rosenfeld nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT; 29013c9168faSHans Rosenfeld 29023c9168faSHans Rosenfeld nvme->n_reg_acc_attr = nvme_reg_acc_attr; 29033c9168faSHans Rosenfeld nvme->n_queue_dma_attr = nvme_queue_dma_attr; 29043c9168faSHans Rosenfeld nvme->n_prp_dma_attr = nvme_prp_dma_attr; 29053c9168faSHans Rosenfeld nvme->n_sgl_dma_attr = nvme_sgl_dma_attr; 29063c9168faSHans Rosenfeld 29073c9168faSHans Rosenfeld /* 29083c9168faSHans Rosenfeld * Setup FMA support. 29093c9168faSHans Rosenfeld */ 29103c9168faSHans Rosenfeld nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip, 29113c9168faSHans Rosenfeld DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 29123c9168faSHans Rosenfeld DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 29133c9168faSHans Rosenfeld DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 29143c9168faSHans Rosenfeld 29153c9168faSHans Rosenfeld ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc); 29163c9168faSHans Rosenfeld 29173c9168faSHans Rosenfeld if (nvme->n_fm_cap) { 29183c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE) 29193c9168faSHans Rosenfeld nvme->n_reg_acc_attr.devacc_attr_access = 29203c9168faSHans Rosenfeld DDI_FLAGERR_ACC; 29213c9168faSHans Rosenfeld 29223c9168faSHans Rosenfeld if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) { 29233c9168faSHans Rosenfeld nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 29243c9168faSHans Rosenfeld nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 29253c9168faSHans Rosenfeld } 29263c9168faSHans Rosenfeld 29273c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 29283c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 29293c9168faSHans Rosenfeld pci_ereport_setup(dip); 29303c9168faSHans Rosenfeld 29313c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 29323c9168faSHans Rosenfeld ddi_fm_handler_register(dip, nvme_fm_errcb, 29333c9168faSHans Rosenfeld (void *)nvme); 29343c9168faSHans Rosenfeld } 29353c9168faSHans Rosenfeld 29363c9168faSHans Rosenfeld nvme->n_progress |= NVME_FMA_INIT; 29373c9168faSHans Rosenfeld 29383c9168faSHans Rosenfeld /* 29393c9168faSHans Rosenfeld * The spec defines several register sets. Only the controller 29403c9168faSHans Rosenfeld * registers (set 1) are currently used. 29413c9168faSHans Rosenfeld */ 29423c9168faSHans Rosenfeld if (ddi_dev_nregs(dip, &nregs) == DDI_FAILURE || 29433c9168faSHans Rosenfeld nregs < 2 || 29443c9168faSHans Rosenfeld ddi_dev_regsize(dip, 1, ®size) == DDI_FAILURE) 29453c9168faSHans Rosenfeld goto fail; 29463c9168faSHans Rosenfeld 29473c9168faSHans Rosenfeld if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize, 29483c9168faSHans Rosenfeld &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) { 29493c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to map regset 1"); 29503c9168faSHans Rosenfeld goto fail; 29513c9168faSHans Rosenfeld } 29523c9168faSHans Rosenfeld 29533c9168faSHans Rosenfeld nvme->n_progress |= NVME_REGS_MAPPED; 29543c9168faSHans Rosenfeld 29553c9168faSHans Rosenfeld /* 29563c9168faSHans Rosenfeld * Create taskq for command completion. 29573c9168faSHans Rosenfeld */ 29583c9168faSHans Rosenfeld (void) snprintf(name, sizeof (name), "%s%d_cmd_taskq", 29593c9168faSHans Rosenfeld ddi_driver_name(dip), ddi_get_instance(dip)); 29603c9168faSHans Rosenfeld nvme->n_cmd_taskq = ddi_taskq_create(dip, name, MIN(UINT16_MAX, ncpus), 29613c9168faSHans Rosenfeld TASKQ_DEFAULTPRI, 0); 29623c9168faSHans Rosenfeld if (nvme->n_cmd_taskq == NULL) { 29633c9168faSHans Rosenfeld dev_err(dip, CE_WARN, "!failed to create cmd taskq"); 29643c9168faSHans Rosenfeld goto fail; 29653c9168faSHans Rosenfeld } 29663c9168faSHans Rosenfeld 29678834f7acSYouzhong Yang /* 29688834f7acSYouzhong Yang * Create PRP DMA cache 29698834f7acSYouzhong Yang */ 29708834f7acSYouzhong Yang (void) snprintf(name, sizeof (name), "%s%d_prp_cache", 29718834f7acSYouzhong Yang ddi_driver_name(dip), ddi_get_instance(dip)); 29728834f7acSYouzhong Yang nvme->n_prp_cache = kmem_cache_create(name, sizeof (nvme_dma_t), 29738834f7acSYouzhong Yang 0, nvme_prp_dma_constructor, nvme_prp_dma_destructor, 29748834f7acSYouzhong Yang NULL, (void *)nvme, NULL, 0); 29753c9168faSHans Rosenfeld 29763c9168faSHans Rosenfeld if (nvme_init(nvme) != DDI_SUCCESS) 29773c9168faSHans Rosenfeld goto fail; 29783c9168faSHans Rosenfeld 29793c9168faSHans Rosenfeld /* 29803c9168faSHans Rosenfeld * Attach the blkdev driver for each namespace. 29813c9168faSHans Rosenfeld */ 29823c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 29833d9b1a2aSHans Rosenfeld if (ddi_create_minor_node(nvme->n_dip, nvme->n_ns[i].ns_name, 29843d9b1a2aSHans Rosenfeld S_IFCHR, NVME_MINOR(ddi_get_instance(nvme->n_dip), i + 1), 29853d9b1a2aSHans Rosenfeld DDI_NT_NVME_ATTACHMENT_POINT, 0) != DDI_SUCCESS) { 29863d9b1a2aSHans Rosenfeld dev_err(dip, CE_WARN, 29873d9b1a2aSHans Rosenfeld "!failed to create minor node for namespace %d", i); 29883d9b1a2aSHans Rosenfeld goto fail; 29893d9b1a2aSHans Rosenfeld } 29903d9b1a2aSHans Rosenfeld 29913c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_ignore) 29923c9168faSHans Rosenfeld continue; 29933c9168faSHans Rosenfeld 29943c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl = bd_alloc_handle(&nvme->n_ns[i], 29953c9168faSHans Rosenfeld &nvme_bd_ops, &nvme->n_prp_dma_attr, KM_SLEEP); 29963c9168faSHans Rosenfeld 29973c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl == NULL) { 29983c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 29993c9168faSHans Rosenfeld "!failed to get blkdev handle for namespace %d", i); 30003c9168faSHans Rosenfeld goto fail; 30013c9168faSHans Rosenfeld } 30023c9168faSHans Rosenfeld 30033c9168faSHans Rosenfeld if (bd_attach_handle(dip, nvme->n_ns[i].ns_bd_hdl) 30043c9168faSHans Rosenfeld != DDI_SUCCESS) { 30053c9168faSHans Rosenfeld dev_err(dip, CE_WARN, 30063c9168faSHans Rosenfeld "!failed to attach blkdev handle for namespace %d", 30073c9168faSHans Rosenfeld i); 30083c9168faSHans Rosenfeld goto fail; 30093c9168faSHans Rosenfeld } 30103c9168faSHans Rosenfeld } 30113c9168faSHans Rosenfeld 30123d9b1a2aSHans Rosenfeld if (ddi_create_minor_node(dip, "devctl", S_IFCHR, 30133d9b1a2aSHans Rosenfeld NVME_MINOR(ddi_get_instance(dip), 0), DDI_NT_NVME_NEXUS, 0) 30143d9b1a2aSHans Rosenfeld != DDI_SUCCESS) { 30153d9b1a2aSHans Rosenfeld dev_err(dip, CE_WARN, "nvme_attach: " 30163d9b1a2aSHans Rosenfeld "cannot create devctl minor node"); 30173d9b1a2aSHans Rosenfeld goto fail; 30183d9b1a2aSHans Rosenfeld } 30193d9b1a2aSHans Rosenfeld 30203c9168faSHans Rosenfeld return (DDI_SUCCESS); 30213c9168faSHans Rosenfeld 30223c9168faSHans Rosenfeld fail: 30233c9168faSHans Rosenfeld /* attach successful anyway so that FMA can retire the device */ 30243c9168faSHans Rosenfeld if (nvme->n_dead) 30253c9168faSHans Rosenfeld return (DDI_SUCCESS); 30263c9168faSHans Rosenfeld 30273c9168faSHans Rosenfeld (void) nvme_detach(dip, DDI_DETACH); 30283c9168faSHans Rosenfeld 30293c9168faSHans Rosenfeld return (DDI_FAILURE); 30303c9168faSHans Rosenfeld } 30313c9168faSHans Rosenfeld 30323c9168faSHans Rosenfeld static int 30333c9168faSHans Rosenfeld nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 30343c9168faSHans Rosenfeld { 30353c9168faSHans Rosenfeld int instance, i; 30363c9168faSHans Rosenfeld nvme_t *nvme; 30373c9168faSHans Rosenfeld 30383c9168faSHans Rosenfeld if (cmd != DDI_DETACH) 30393c9168faSHans Rosenfeld return (DDI_FAILURE); 30403c9168faSHans Rosenfeld 30413c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 30423c9168faSHans Rosenfeld 30433c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 30443c9168faSHans Rosenfeld 30453c9168faSHans Rosenfeld if (nvme == NULL) 30463c9168faSHans Rosenfeld return (DDI_FAILURE); 30473c9168faSHans Rosenfeld 30483d9b1a2aSHans Rosenfeld ddi_remove_minor_node(dip, "devctl"); 30493d9b1a2aSHans Rosenfeld mutex_destroy(&nvme->n_minor.nm_mutex); 30503d9b1a2aSHans Rosenfeld 30513c9168faSHans Rosenfeld if (nvme->n_ns) { 30523c9168faSHans Rosenfeld for (i = 0; i != nvme->n_namespace_count; i++) { 30533d9b1a2aSHans Rosenfeld ddi_remove_minor_node(dip, nvme->n_ns[i].ns_name); 30543d9b1a2aSHans Rosenfeld mutex_destroy(&nvme->n_ns[i].ns_minor.nm_mutex); 30553d9b1a2aSHans Rosenfeld 30563c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_bd_hdl) { 30573c9168faSHans Rosenfeld (void) bd_detach_handle( 30583c9168faSHans Rosenfeld nvme->n_ns[i].ns_bd_hdl); 30593c9168faSHans Rosenfeld bd_free_handle(nvme->n_ns[i].ns_bd_hdl); 30603c9168faSHans Rosenfeld } 30613c9168faSHans Rosenfeld 30623c9168faSHans Rosenfeld if (nvme->n_ns[i].ns_idns) 30633c9168faSHans Rosenfeld kmem_free(nvme->n_ns[i].ns_idns, 30643c9168faSHans Rosenfeld sizeof (nvme_identify_nsid_t)); 306524979ca3SHans Rosenfeld if (nvme->n_ns[i].ns_devid) 306624979ca3SHans Rosenfeld strfree(nvme->n_ns[i].ns_devid); 30673c9168faSHans Rosenfeld } 30683c9168faSHans Rosenfeld 30693c9168faSHans Rosenfeld kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) * 30703c9168faSHans Rosenfeld nvme->n_namespace_count); 30713c9168faSHans Rosenfeld } 30723c9168faSHans Rosenfeld 30733c9168faSHans Rosenfeld if (nvme->n_progress & NVME_INTERRUPTS) 30743c9168faSHans Rosenfeld nvme_release_interrupts(nvme); 30753c9168faSHans Rosenfeld 30763c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 30773c9168faSHans Rosenfeld ddi_taskq_wait(nvme->n_cmd_taskq); 30783c9168faSHans Rosenfeld 30793c9168faSHans Rosenfeld if (nvme->n_ioq_count > 0) { 30803c9168faSHans Rosenfeld for (i = 1; i != nvme->n_ioq_count + 1; i++) { 30813c9168faSHans Rosenfeld if (nvme->n_ioq[i] != NULL) { 30823c9168faSHans Rosenfeld /* TODO: send destroy queue commands */ 30833c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_ioq[i]); 30843c9168faSHans Rosenfeld } 30853c9168faSHans Rosenfeld } 30863c9168faSHans Rosenfeld 30873c9168faSHans Rosenfeld kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) * 30883c9168faSHans Rosenfeld (nvme->n_ioq_count + 1)); 30893c9168faSHans Rosenfeld } 30903c9168faSHans Rosenfeld 30918834f7acSYouzhong Yang if (nvme->n_prp_cache != NULL) { 30928834f7acSYouzhong Yang kmem_cache_destroy(nvme->n_prp_cache); 30938834f7acSYouzhong Yang } 30948834f7acSYouzhong Yang 30953c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) { 30963c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE); 30973c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_FALSE); 30983c9168faSHans Rosenfeld } 30993c9168faSHans Rosenfeld 31003c9168faSHans Rosenfeld if (nvme->n_cmd_taskq) 31013c9168faSHans Rosenfeld ddi_taskq_destroy(nvme->n_cmd_taskq); 31023c9168faSHans Rosenfeld 31033c9168faSHans Rosenfeld if (nvme->n_progress & NVME_CTRL_LIMITS) 31043c9168faSHans Rosenfeld sema_destroy(&nvme->n_abort_sema); 31053c9168faSHans Rosenfeld 31063c9168faSHans Rosenfeld if (nvme->n_progress & NVME_ADMIN_QUEUE) 31073c9168faSHans Rosenfeld nvme_free_qpair(nvme->n_adminq); 31083c9168faSHans Rosenfeld 31093c9168faSHans Rosenfeld if (nvme->n_idctl) 31103d9b1a2aSHans Rosenfeld kmem_free(nvme->n_idctl, NVME_IDENTIFY_BUFSIZE); 31113c9168faSHans Rosenfeld 31123c9168faSHans Rosenfeld if (nvme->n_progress & NVME_REGS_MAPPED) 31133c9168faSHans Rosenfeld ddi_regs_map_free(&nvme->n_regh); 31143c9168faSHans Rosenfeld 31153c9168faSHans Rosenfeld if (nvme->n_progress & NVME_FMA_INIT) { 31163c9168faSHans Rosenfeld if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 31173c9168faSHans Rosenfeld ddi_fm_handler_unregister(nvme->n_dip); 31183c9168faSHans Rosenfeld 31193c9168faSHans Rosenfeld if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 31203c9168faSHans Rosenfeld DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 31213c9168faSHans Rosenfeld pci_ereport_teardown(nvme->n_dip); 31223c9168faSHans Rosenfeld 31233c9168faSHans Rosenfeld ddi_fm_fini(nvme->n_dip); 31243c9168faSHans Rosenfeld } 31253c9168faSHans Rosenfeld 3126510a6847SHans Rosenfeld if (nvme->n_vendor != NULL) 3127510a6847SHans Rosenfeld strfree(nvme->n_vendor); 3128510a6847SHans Rosenfeld 3129510a6847SHans Rosenfeld if (nvme->n_product != NULL) 3130510a6847SHans Rosenfeld strfree(nvme->n_product); 3131510a6847SHans Rosenfeld 31323c9168faSHans Rosenfeld ddi_soft_state_free(nvme_state, instance); 31333c9168faSHans Rosenfeld 31343c9168faSHans Rosenfeld return (DDI_SUCCESS); 31353c9168faSHans Rosenfeld } 31363c9168faSHans Rosenfeld 31373c9168faSHans Rosenfeld static int 31383c9168faSHans Rosenfeld nvme_quiesce(dev_info_t *dip) 31393c9168faSHans Rosenfeld { 31403c9168faSHans Rosenfeld int instance; 31413c9168faSHans Rosenfeld nvme_t *nvme; 31423c9168faSHans Rosenfeld 31433c9168faSHans Rosenfeld instance = ddi_get_instance(dip); 31443c9168faSHans Rosenfeld 31453c9168faSHans Rosenfeld nvme = ddi_get_soft_state(nvme_state, instance); 31463c9168faSHans Rosenfeld 31473c9168faSHans Rosenfeld if (nvme == NULL) 31483c9168faSHans Rosenfeld return (DDI_FAILURE); 31493c9168faSHans Rosenfeld 31503c9168faSHans Rosenfeld nvme_shutdown(nvme, NVME_CC_SHN_ABRUPT, B_TRUE); 31513c9168faSHans Rosenfeld 31523c9168faSHans Rosenfeld (void) nvme_reset(nvme, B_TRUE); 31533c9168faSHans Rosenfeld 31543c9168faSHans Rosenfeld return (DDI_FAILURE); 31553c9168faSHans Rosenfeld } 31563c9168faSHans Rosenfeld 31573c9168faSHans Rosenfeld static int 31583c9168faSHans Rosenfeld nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer) 31593c9168faSHans Rosenfeld { 31603c9168faSHans Rosenfeld nvme_t *nvme = cmd->nc_nvme; 31613c9168faSHans Rosenfeld int nprp_page, nprp; 31623c9168faSHans Rosenfeld uint64_t *prp; 31633c9168faSHans Rosenfeld 31643c9168faSHans Rosenfeld if (xfer->x_ndmac == 0) 31653c9168faSHans Rosenfeld return (DDI_FAILURE); 31663c9168faSHans Rosenfeld 31673c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress; 31683c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 31693c9168faSHans Rosenfeld 31703c9168faSHans Rosenfeld if (xfer->x_ndmac == 1) { 31713c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; 31723c9168faSHans Rosenfeld return (DDI_SUCCESS); 31733c9168faSHans Rosenfeld } else if (xfer->x_ndmac == 2) { 31743c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress; 31753c9168faSHans Rosenfeld return (DDI_SUCCESS); 31763c9168faSHans Rosenfeld } 31773c9168faSHans Rosenfeld 31783c9168faSHans Rosenfeld xfer->x_ndmac--; 31793c9168faSHans Rosenfeld 31803c9168faSHans Rosenfeld nprp_page = nvme->n_pagesize / sizeof (uint64_t) - 1; 31813c9168faSHans Rosenfeld ASSERT(nprp_page > 0); 31823c9168faSHans Rosenfeld nprp = (xfer->x_ndmac + nprp_page - 1) / nprp_page; 31833c9168faSHans Rosenfeld 31843c9168faSHans Rosenfeld /* 31853c9168faSHans Rosenfeld * We currently don't support chained PRPs and set up our DMA 31863c9168faSHans Rosenfeld * attributes to reflect that. If we still get an I/O request 31873c9168faSHans Rosenfeld * that needs a chained PRP something is very wrong. 31883c9168faSHans Rosenfeld */ 31893c9168faSHans Rosenfeld VERIFY(nprp == 1); 31903c9168faSHans Rosenfeld 31918834f7acSYouzhong Yang cmd->nc_dma = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP); 31928834f7acSYouzhong Yang bzero(cmd->nc_dma->nd_memp, cmd->nc_dma->nd_len); 31933c9168faSHans Rosenfeld 31943c9168faSHans Rosenfeld cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress; 31953c9168faSHans Rosenfeld 31963c9168faSHans Rosenfeld /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 31973c9168faSHans Rosenfeld for (prp = (uint64_t *)cmd->nc_dma->nd_memp; 31983c9168faSHans Rosenfeld xfer->x_ndmac > 0; 31993c9168faSHans Rosenfeld prp++, xfer->x_ndmac--) { 32003c9168faSHans Rosenfeld *prp = xfer->x_dmac.dmac_laddress; 32013c9168faSHans Rosenfeld ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 32023c9168faSHans Rosenfeld } 32033c9168faSHans Rosenfeld 32043c9168faSHans Rosenfeld (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len, 32053c9168faSHans Rosenfeld DDI_DMA_SYNC_FORDEV); 32063c9168faSHans Rosenfeld return (DDI_SUCCESS); 32073c9168faSHans Rosenfeld } 32083c9168faSHans Rosenfeld 32093c9168faSHans Rosenfeld static nvme_cmd_t * 32103c9168faSHans Rosenfeld nvme_create_nvm_cmd(nvme_namespace_t *ns, uint8_t opc, bd_xfer_t *xfer) 32113c9168faSHans Rosenfeld { 32123c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 32133c9168faSHans Rosenfeld nvme_cmd_t *cmd; 32143c9168faSHans Rosenfeld 32153c9168faSHans Rosenfeld /* 32163c9168faSHans Rosenfeld * Blkdev only sets BD_XFER_POLL when dumping, so don't sleep. 32173c9168faSHans Rosenfeld */ 32183c9168faSHans Rosenfeld cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ? 32193c9168faSHans Rosenfeld KM_NOSLEEP : KM_SLEEP); 32203c9168faSHans Rosenfeld 32213c9168faSHans Rosenfeld if (cmd == NULL) 32223c9168faSHans Rosenfeld return (NULL); 32233c9168faSHans Rosenfeld 32243c9168faSHans Rosenfeld cmd->nc_sqe.sqe_opc = opc; 32253c9168faSHans Rosenfeld cmd->nc_callback = nvme_bd_xfer_done; 32263c9168faSHans Rosenfeld cmd->nc_xfer = xfer; 32273c9168faSHans Rosenfeld 32283c9168faSHans Rosenfeld switch (opc) { 32293c9168faSHans Rosenfeld case NVME_OPC_NVM_WRITE: 32303c9168faSHans Rosenfeld case NVME_OPC_NVM_READ: 32313c9168faSHans Rosenfeld VERIFY(xfer->x_nblks <= 0x10000); 32323c9168faSHans Rosenfeld 32333c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 32343c9168faSHans Rosenfeld 32353c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu; 32363c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32); 32373c9168faSHans Rosenfeld cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1); 32383c9168faSHans Rosenfeld 32393c9168faSHans Rosenfeld if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS) 32403c9168faSHans Rosenfeld goto fail; 32413c9168faSHans Rosenfeld break; 32423c9168faSHans Rosenfeld 32433c9168faSHans Rosenfeld case NVME_OPC_NVM_FLUSH: 32443c9168faSHans Rosenfeld cmd->nc_sqe.sqe_nsid = ns->ns_id; 32453c9168faSHans Rosenfeld break; 32463c9168faSHans Rosenfeld 32473c9168faSHans Rosenfeld default: 32483c9168faSHans Rosenfeld goto fail; 32493c9168faSHans Rosenfeld } 32503c9168faSHans Rosenfeld 32513c9168faSHans Rosenfeld return (cmd); 32523c9168faSHans Rosenfeld 32533c9168faSHans Rosenfeld fail: 32543c9168faSHans Rosenfeld nvme_free_cmd(cmd); 32553c9168faSHans Rosenfeld return (NULL); 32563c9168faSHans Rosenfeld } 32573c9168faSHans Rosenfeld 32583c9168faSHans Rosenfeld static void 32593c9168faSHans Rosenfeld nvme_bd_xfer_done(void *arg) 32603c9168faSHans Rosenfeld { 32613c9168faSHans Rosenfeld nvme_cmd_t *cmd = arg; 32623c9168faSHans Rosenfeld bd_xfer_t *xfer = cmd->nc_xfer; 32633c9168faSHans Rosenfeld int error = 0; 32643c9168faSHans Rosenfeld 32653c9168faSHans Rosenfeld error = nvme_check_cmd_status(cmd); 32663c9168faSHans Rosenfeld nvme_free_cmd(cmd); 32673c9168faSHans Rosenfeld 32683c9168faSHans Rosenfeld bd_xfer_done(xfer, error); 32693c9168faSHans Rosenfeld } 32703c9168faSHans Rosenfeld 32713c9168faSHans Rosenfeld static void 32723c9168faSHans Rosenfeld nvme_bd_driveinfo(void *arg, bd_drive_t *drive) 32733c9168faSHans Rosenfeld { 32743c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 32753c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 32763c9168faSHans Rosenfeld 32773c9168faSHans Rosenfeld /* 32783c9168faSHans Rosenfeld * blkdev maintains one queue size per instance (namespace), 32793c9168faSHans Rosenfeld * but all namespace share the I/O queues. 32803c9168faSHans Rosenfeld * TODO: need to figure out a sane default, or use per-NS I/O queues, 32813c9168faSHans Rosenfeld * or change blkdev to handle EAGAIN 32823c9168faSHans Rosenfeld */ 32833c9168faSHans Rosenfeld drive->d_qsize = nvme->n_ioq_count * nvme->n_io_queue_len 32843c9168faSHans Rosenfeld / nvme->n_namespace_count; 32853c9168faSHans Rosenfeld 32863c9168faSHans Rosenfeld /* 32873c9168faSHans Rosenfeld * d_maxxfer is not set, which means the value is taken from the DMA 32883c9168faSHans Rosenfeld * attributes specified to bd_alloc_handle. 32893c9168faSHans Rosenfeld */ 32903c9168faSHans Rosenfeld 32913c9168faSHans Rosenfeld drive->d_removable = B_FALSE; 32923c9168faSHans Rosenfeld drive->d_hotpluggable = B_FALSE; 32933c9168faSHans Rosenfeld 329424979ca3SHans Rosenfeld bcopy(ns->ns_eui64, drive->d_eui64, sizeof (drive->d_eui64)); 32953c9168faSHans Rosenfeld drive->d_target = ns->ns_id; 32963c9168faSHans Rosenfeld drive->d_lun = 0; 3297510a6847SHans Rosenfeld 3298bef9e21aSHans Rosenfeld drive->d_model = nvme->n_idctl->id_model; 3299bef9e21aSHans Rosenfeld drive->d_model_len = sizeof (nvme->n_idctl->id_model); 3300510a6847SHans Rosenfeld drive->d_vendor = nvme->n_vendor; 3301510a6847SHans Rosenfeld drive->d_vendor_len = strlen(nvme->n_vendor); 3302510a6847SHans Rosenfeld drive->d_product = nvme->n_product; 3303510a6847SHans Rosenfeld drive->d_product_len = strlen(nvme->n_product); 3304510a6847SHans Rosenfeld drive->d_serial = nvme->n_idctl->id_serial; 3305510a6847SHans Rosenfeld drive->d_serial_len = sizeof (nvme->n_idctl->id_serial); 3306510a6847SHans Rosenfeld drive->d_revision = nvme->n_idctl->id_fwrev; 3307510a6847SHans Rosenfeld drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev); 33083c9168faSHans Rosenfeld } 33093c9168faSHans Rosenfeld 33103c9168faSHans Rosenfeld static int 33113c9168faSHans Rosenfeld nvme_bd_mediainfo(void *arg, bd_media_t *media) 33123c9168faSHans Rosenfeld { 33133c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 33143c9168faSHans Rosenfeld 33153c9168faSHans Rosenfeld media->m_nblks = ns->ns_block_count; 33163c9168faSHans Rosenfeld media->m_blksize = ns->ns_block_size; 33173c9168faSHans Rosenfeld media->m_readonly = B_FALSE; 33183c9168faSHans Rosenfeld media->m_solidstate = B_TRUE; 33193c9168faSHans Rosenfeld 33203c9168faSHans Rosenfeld media->m_pblksize = ns->ns_best_block_size; 33213c9168faSHans Rosenfeld 33223c9168faSHans Rosenfeld return (0); 33233c9168faSHans Rosenfeld } 33243c9168faSHans Rosenfeld 33253c9168faSHans Rosenfeld static int 33263c9168faSHans Rosenfeld nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc) 33273c9168faSHans Rosenfeld { 33283c9168faSHans Rosenfeld nvme_t *nvme = ns->ns_nvme; 33294b324362SHans Rosenfeld nvme_cmd_t *cmd; 33304ac9cfccSHans Rosenfeld nvme_qpair_t *ioq; 33314ac9cfccSHans Rosenfeld boolean_t poll; 33324b324362SHans Rosenfeld int ret; 33333c9168faSHans Rosenfeld 33343c9168faSHans Rosenfeld if (nvme->n_dead) 33353c9168faSHans Rosenfeld return (EIO); 33363c9168faSHans Rosenfeld 33373c9168faSHans Rosenfeld cmd = nvme_create_nvm_cmd(ns, opc, xfer); 33383c9168faSHans Rosenfeld if (cmd == NULL) 33393c9168faSHans Rosenfeld return (ENOMEM); 33403c9168faSHans Rosenfeld 33413c9168faSHans Rosenfeld cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1; 33423c9168faSHans Rosenfeld ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); 33434ac9cfccSHans Rosenfeld ioq = nvme->n_ioq[cmd->nc_sqid]; 33444ac9cfccSHans Rosenfeld 33454ac9cfccSHans Rosenfeld /* 33464ac9cfccSHans Rosenfeld * Get the polling flag before submitting the command. The command may 33474ac9cfccSHans Rosenfeld * complete immediately after it was submitted, which means we must 33484ac9cfccSHans Rosenfeld * treat both cmd and xfer as if they have been freed already. 33494ac9cfccSHans Rosenfeld */ 33504ac9cfccSHans Rosenfeld poll = (xfer->x_flags & BD_XFER_POLL) != 0; 33513c9168faSHans Rosenfeld 33524b324362SHans Rosenfeld ret = nvme_submit_io_cmd(ioq, cmd); 33534b324362SHans Rosenfeld 33544b324362SHans Rosenfeld if (ret != 0) 33554b324362SHans Rosenfeld return (ret); 33563c9168faSHans Rosenfeld 33574ac9cfccSHans Rosenfeld if (!poll) 33584ac9cfccSHans Rosenfeld return (0); 33594ac9cfccSHans Rosenfeld 33604ac9cfccSHans Rosenfeld do { 33614b324362SHans Rosenfeld cmd = nvme_retrieve_cmd(nvme, ioq); 33624b324362SHans Rosenfeld if (cmd != NULL) 33634b324362SHans Rosenfeld nvme_bd_xfer_done(cmd); 33644ac9cfccSHans Rosenfeld else 33654ac9cfccSHans Rosenfeld drv_usecwait(10); 33664ac9cfccSHans Rosenfeld } while (ioq->nq_active_cmds != 0); 33674ac9cfccSHans Rosenfeld 33683c9168faSHans Rosenfeld return (0); 33693c9168faSHans Rosenfeld } 33703c9168faSHans Rosenfeld 33713c9168faSHans Rosenfeld static int 33723c9168faSHans Rosenfeld nvme_bd_read(void *arg, bd_xfer_t *xfer) 33733c9168faSHans Rosenfeld { 33743c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 33753c9168faSHans Rosenfeld 33763c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_READ)); 33773c9168faSHans Rosenfeld } 33783c9168faSHans Rosenfeld 33793c9168faSHans Rosenfeld static int 33803c9168faSHans Rosenfeld nvme_bd_write(void *arg, bd_xfer_t *xfer) 33813c9168faSHans Rosenfeld { 33823c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 33833c9168faSHans Rosenfeld 33843c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_WRITE)); 33853c9168faSHans Rosenfeld } 33863c9168faSHans Rosenfeld 33873c9168faSHans Rosenfeld static int 33883c9168faSHans Rosenfeld nvme_bd_sync(void *arg, bd_xfer_t *xfer) 33893c9168faSHans Rosenfeld { 33903c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 33913c9168faSHans Rosenfeld 33923c9168faSHans Rosenfeld if (ns->ns_nvme->n_dead) 33933c9168faSHans Rosenfeld return (EIO); 33943c9168faSHans Rosenfeld 33953c9168faSHans Rosenfeld /* 3396d148d46eSHans Rosenfeld * If the volatile write cache is not present or not enabled the FLUSH 3397d148d46eSHans Rosenfeld * command is a no-op, so we can take a shortcut here. 33983c9168faSHans Rosenfeld */ 3399d148d46eSHans Rosenfeld if (!ns->ns_nvme->n_write_cache_present) { 34003c9168faSHans Rosenfeld bd_xfer_done(xfer, ENOTSUP); 34013c9168faSHans Rosenfeld return (0); 34023c9168faSHans Rosenfeld } 34033c9168faSHans Rosenfeld 3404d148d46eSHans Rosenfeld if (!ns->ns_nvme->n_write_cache_enabled) { 3405d148d46eSHans Rosenfeld bd_xfer_done(xfer, 0); 3406d148d46eSHans Rosenfeld return (0); 3407d148d46eSHans Rosenfeld } 3408d148d46eSHans Rosenfeld 34093c9168faSHans Rosenfeld return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_FLUSH)); 34103c9168faSHans Rosenfeld } 34113c9168faSHans Rosenfeld 34123c9168faSHans Rosenfeld static int 34133c9168faSHans Rosenfeld nvme_bd_devid(void *arg, dev_info_t *devinfo, ddi_devid_t *devid) 34143c9168faSHans Rosenfeld { 34153c9168faSHans Rosenfeld nvme_namespace_t *ns = arg; 34163c9168faSHans Rosenfeld 341724979ca3SHans Rosenfeld /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 341824979ca3SHans Rosenfeld if (*(uint64_t *)ns->ns_eui64 != 0) { 341924979ca3SHans Rosenfeld return (ddi_devid_init(devinfo, DEVID_SCSI3_WWN, 342024979ca3SHans Rosenfeld sizeof (ns->ns_eui64), ns->ns_eui64, devid)); 342124979ca3SHans Rosenfeld } else { 342224979ca3SHans Rosenfeld return (ddi_devid_init(devinfo, DEVID_ENCAP, 342324979ca3SHans Rosenfeld strlen(ns->ns_devid), ns->ns_devid, devid)); 342424979ca3SHans Rosenfeld } 34253c9168faSHans Rosenfeld } 34263d9b1a2aSHans Rosenfeld 34273d9b1a2aSHans Rosenfeld static int 34283d9b1a2aSHans Rosenfeld nvme_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 34293d9b1a2aSHans Rosenfeld { 34303d9b1a2aSHans Rosenfeld #ifndef __lock_lint 34313d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 34323d9b1a2aSHans Rosenfeld #endif 34333d9b1a2aSHans Rosenfeld minor_t minor = getminor(*devp); 34343d9b1a2aSHans Rosenfeld nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 34353d9b1a2aSHans Rosenfeld int nsid = NVME_MINOR_NSID(minor); 34363d9b1a2aSHans Rosenfeld nvme_minor_state_t *nm; 34373d9b1a2aSHans Rosenfeld int rv = 0; 34383d9b1a2aSHans Rosenfeld 34393d9b1a2aSHans Rosenfeld if (otyp != OTYP_CHR) 34403d9b1a2aSHans Rosenfeld return (EINVAL); 34413d9b1a2aSHans Rosenfeld 34423d9b1a2aSHans Rosenfeld if (nvme == NULL) 34433d9b1a2aSHans Rosenfeld return (ENXIO); 34443d9b1a2aSHans Rosenfeld 34453d9b1a2aSHans Rosenfeld if (nsid > nvme->n_namespace_count) 34463d9b1a2aSHans Rosenfeld return (ENXIO); 34473d9b1a2aSHans Rosenfeld 3448e984c70bSHans Rosenfeld if (nvme->n_dead) 3449e984c70bSHans Rosenfeld return (EIO); 3450e984c70bSHans Rosenfeld 34513d9b1a2aSHans Rosenfeld nm = nsid == 0 ? &nvme->n_minor : &nvme->n_ns[nsid - 1].ns_minor; 34523d9b1a2aSHans Rosenfeld 34533d9b1a2aSHans Rosenfeld mutex_enter(&nm->nm_mutex); 34543d9b1a2aSHans Rosenfeld if (nm->nm_oexcl) { 34553d9b1a2aSHans Rosenfeld rv = EBUSY; 34563d9b1a2aSHans Rosenfeld goto out; 34573d9b1a2aSHans Rosenfeld } 34583d9b1a2aSHans Rosenfeld 34593d9b1a2aSHans Rosenfeld if (flag & FEXCL) { 34603d9b1a2aSHans Rosenfeld if (nm->nm_ocnt != 0) { 34613d9b1a2aSHans Rosenfeld rv = EBUSY; 34623d9b1a2aSHans Rosenfeld goto out; 34633d9b1a2aSHans Rosenfeld } 34643d9b1a2aSHans Rosenfeld nm->nm_oexcl = B_TRUE; 34653d9b1a2aSHans Rosenfeld } 34663d9b1a2aSHans Rosenfeld 34673d9b1a2aSHans Rosenfeld nm->nm_ocnt++; 34683d9b1a2aSHans Rosenfeld 34693d9b1a2aSHans Rosenfeld out: 34703d9b1a2aSHans Rosenfeld mutex_exit(&nm->nm_mutex); 34713d9b1a2aSHans Rosenfeld return (rv); 34723d9b1a2aSHans Rosenfeld 34733d9b1a2aSHans Rosenfeld } 34743d9b1a2aSHans Rosenfeld 34753d9b1a2aSHans Rosenfeld static int 34763d9b1a2aSHans Rosenfeld nvme_close(dev_t dev, int flag, int otyp, cred_t *cred_p) 34773d9b1a2aSHans Rosenfeld { 34783d9b1a2aSHans Rosenfeld #ifndef __lock_lint 34793d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 34803d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(flag)); 34813d9b1a2aSHans Rosenfeld #endif 34823d9b1a2aSHans Rosenfeld minor_t minor = getminor(dev); 34833d9b1a2aSHans Rosenfeld nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 34843d9b1a2aSHans Rosenfeld int nsid = NVME_MINOR_NSID(minor); 34853d9b1a2aSHans Rosenfeld nvme_minor_state_t *nm; 34863d9b1a2aSHans Rosenfeld 34873d9b1a2aSHans Rosenfeld if (otyp != OTYP_CHR) 34883d9b1a2aSHans Rosenfeld return (ENXIO); 34893d9b1a2aSHans Rosenfeld 34903d9b1a2aSHans Rosenfeld if (nvme == NULL) 34913d9b1a2aSHans Rosenfeld return (ENXIO); 34923d9b1a2aSHans Rosenfeld 34933d9b1a2aSHans Rosenfeld if (nsid > nvme->n_namespace_count) 34943d9b1a2aSHans Rosenfeld return (ENXIO); 34953d9b1a2aSHans Rosenfeld 34963d9b1a2aSHans Rosenfeld nm = nsid == 0 ? &nvme->n_minor : &nvme->n_ns[nsid - 1].ns_minor; 34973d9b1a2aSHans Rosenfeld 34983d9b1a2aSHans Rosenfeld mutex_enter(&nm->nm_mutex); 34993d9b1a2aSHans Rosenfeld if (nm->nm_oexcl) 35003d9b1a2aSHans Rosenfeld nm->nm_oexcl = B_FALSE; 35013d9b1a2aSHans Rosenfeld 35023d9b1a2aSHans Rosenfeld ASSERT(nm->nm_ocnt > 0); 35033d9b1a2aSHans Rosenfeld nm->nm_ocnt--; 35043d9b1a2aSHans Rosenfeld mutex_exit(&nm->nm_mutex); 35053d9b1a2aSHans Rosenfeld 35063d9b1a2aSHans Rosenfeld return (0); 35073d9b1a2aSHans Rosenfeld } 35083d9b1a2aSHans Rosenfeld 35093d9b1a2aSHans Rosenfeld static int 35103d9b1a2aSHans Rosenfeld nvme_ioctl_identify(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 35113d9b1a2aSHans Rosenfeld cred_t *cred_p) 35123d9b1a2aSHans Rosenfeld { 35133d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 35143d9b1a2aSHans Rosenfeld int rv = 0; 35153d9b1a2aSHans Rosenfeld void *idctl; 35163d9b1a2aSHans Rosenfeld 35173d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 35183d9b1a2aSHans Rosenfeld return (EPERM); 35193d9b1a2aSHans Rosenfeld 35203d9b1a2aSHans Rosenfeld if (nioc->n_len < NVME_IDENTIFY_BUFSIZE) 35213d9b1a2aSHans Rosenfeld return (EINVAL); 35223d9b1a2aSHans Rosenfeld 3523e984c70bSHans Rosenfeld if ((rv = nvme_identify(nvme, nsid, (void **)&idctl)) != 0) 3524e984c70bSHans Rosenfeld return (rv); 35253d9b1a2aSHans Rosenfeld 35263d9b1a2aSHans Rosenfeld if (ddi_copyout(idctl, (void *)nioc->n_buf, NVME_IDENTIFY_BUFSIZE, mode) 35273d9b1a2aSHans Rosenfeld != 0) 35283d9b1a2aSHans Rosenfeld rv = EFAULT; 35293d9b1a2aSHans Rosenfeld 35303d9b1a2aSHans Rosenfeld kmem_free(idctl, NVME_IDENTIFY_BUFSIZE); 35313d9b1a2aSHans Rosenfeld 35323d9b1a2aSHans Rosenfeld return (rv); 35333d9b1a2aSHans Rosenfeld } 35343d9b1a2aSHans Rosenfeld 35353d9b1a2aSHans Rosenfeld static int 35363d9b1a2aSHans Rosenfeld nvme_ioctl_capabilities(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 35373d9b1a2aSHans Rosenfeld int mode, cred_t *cred_p) 35383d9b1a2aSHans Rosenfeld { 35393d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nsid, cred_p)); 35403d9b1a2aSHans Rosenfeld int rv = 0; 35413d9b1a2aSHans Rosenfeld nvme_reg_cap_t cap = { 0 }; 35423d9b1a2aSHans Rosenfeld nvme_capabilities_t nc; 35433d9b1a2aSHans Rosenfeld 35443d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 35453d9b1a2aSHans Rosenfeld return (EPERM); 35463d9b1a2aSHans Rosenfeld 35473d9b1a2aSHans Rosenfeld if (nioc->n_len < sizeof (nc)) 35483d9b1a2aSHans Rosenfeld return (EINVAL); 35493d9b1a2aSHans Rosenfeld 35503d9b1a2aSHans Rosenfeld cap.r = nvme_get64(nvme, NVME_REG_CAP); 35513d9b1a2aSHans Rosenfeld 35523d9b1a2aSHans Rosenfeld /* 35533d9b1a2aSHans Rosenfeld * The MPSMIN and MPSMAX fields in the CAP register use 0 to 35543d9b1a2aSHans Rosenfeld * specify the base page size of 4k (1<<12), so add 12 here to 35553d9b1a2aSHans Rosenfeld * get the real page size value. 35563d9b1a2aSHans Rosenfeld */ 35573d9b1a2aSHans Rosenfeld nc.mpsmax = 1 << (12 + cap.b.cap_mpsmax); 35583d9b1a2aSHans Rosenfeld nc.mpsmin = 1 << (12 + cap.b.cap_mpsmin); 35593d9b1a2aSHans Rosenfeld 35603d9b1a2aSHans Rosenfeld if (ddi_copyout(&nc, (void *)nioc->n_buf, sizeof (nc), mode) != 0) 35613d9b1a2aSHans Rosenfeld rv = EFAULT; 35623d9b1a2aSHans Rosenfeld 35633d9b1a2aSHans Rosenfeld return (rv); 35643d9b1a2aSHans Rosenfeld } 35653d9b1a2aSHans Rosenfeld 35663d9b1a2aSHans Rosenfeld static int 35673d9b1a2aSHans Rosenfeld nvme_ioctl_get_logpage(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 35683d9b1a2aSHans Rosenfeld int mode, cred_t *cred_p) 35693d9b1a2aSHans Rosenfeld { 35703d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 35713d9b1a2aSHans Rosenfeld void *log = NULL; 35723d9b1a2aSHans Rosenfeld size_t bufsize = 0; 35733d9b1a2aSHans Rosenfeld int rv = 0; 35743d9b1a2aSHans Rosenfeld 35753d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 35763d9b1a2aSHans Rosenfeld return (EPERM); 35773d9b1a2aSHans Rosenfeld 35783d9b1a2aSHans Rosenfeld switch (nioc->n_arg) { 35793d9b1a2aSHans Rosenfeld case NVME_LOGPAGE_ERROR: 35803d9b1a2aSHans Rosenfeld if (nsid != 0) 35813d9b1a2aSHans Rosenfeld return (EINVAL); 35823d9b1a2aSHans Rosenfeld break; 35833d9b1a2aSHans Rosenfeld case NVME_LOGPAGE_HEALTH: 35843d9b1a2aSHans Rosenfeld if (nsid != 0 && nvme->n_idctl->id_lpa.lp_smart == 0) 35853d9b1a2aSHans Rosenfeld return (EINVAL); 35863d9b1a2aSHans Rosenfeld 35873d9b1a2aSHans Rosenfeld if (nsid == 0) 35883d9b1a2aSHans Rosenfeld nsid = (uint32_t)-1; 35893d9b1a2aSHans Rosenfeld 35903d9b1a2aSHans Rosenfeld break; 35913d9b1a2aSHans Rosenfeld case NVME_LOGPAGE_FWSLOT: 35923d9b1a2aSHans Rosenfeld if (nsid != 0) 35933d9b1a2aSHans Rosenfeld return (EINVAL); 35943d9b1a2aSHans Rosenfeld break; 35953d9b1a2aSHans Rosenfeld default: 35963d9b1a2aSHans Rosenfeld return (EINVAL); 35973d9b1a2aSHans Rosenfeld } 35983d9b1a2aSHans Rosenfeld 35993d9b1a2aSHans Rosenfeld if (nvme_get_logpage(nvme, &log, &bufsize, nioc->n_arg, nsid) 36003d9b1a2aSHans Rosenfeld != DDI_SUCCESS) 36013d9b1a2aSHans Rosenfeld return (EIO); 36023d9b1a2aSHans Rosenfeld 36033d9b1a2aSHans Rosenfeld if (nioc->n_len < bufsize) { 36043d9b1a2aSHans Rosenfeld kmem_free(log, bufsize); 36053d9b1a2aSHans Rosenfeld return (EINVAL); 36063d9b1a2aSHans Rosenfeld } 36073d9b1a2aSHans Rosenfeld 36083d9b1a2aSHans Rosenfeld if (ddi_copyout(log, (void *)nioc->n_buf, bufsize, mode) != 0) 36093d9b1a2aSHans Rosenfeld rv = EFAULT; 36103d9b1a2aSHans Rosenfeld 36113d9b1a2aSHans Rosenfeld nioc->n_len = bufsize; 36123d9b1a2aSHans Rosenfeld kmem_free(log, bufsize); 36133d9b1a2aSHans Rosenfeld 36143d9b1a2aSHans Rosenfeld return (rv); 36153d9b1a2aSHans Rosenfeld } 36163d9b1a2aSHans Rosenfeld 36173d9b1a2aSHans Rosenfeld static int 36183d9b1a2aSHans Rosenfeld nvme_ioctl_get_features(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 36193d9b1a2aSHans Rosenfeld int mode, cred_t *cred_p) 36203d9b1a2aSHans Rosenfeld { 36213d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(cred_p)); 36223d9b1a2aSHans Rosenfeld void *buf = NULL; 36233d9b1a2aSHans Rosenfeld size_t bufsize = 0; 36243d9b1a2aSHans Rosenfeld uint32_t res = 0; 36253d9b1a2aSHans Rosenfeld uint8_t feature; 36263d9b1a2aSHans Rosenfeld int rv = 0; 36273d9b1a2aSHans Rosenfeld 36283d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 36293d9b1a2aSHans Rosenfeld return (EPERM); 36303d9b1a2aSHans Rosenfeld 36313d9b1a2aSHans Rosenfeld if ((nioc->n_arg >> 32) > 0xff) 36323d9b1a2aSHans Rosenfeld return (EINVAL); 36333d9b1a2aSHans Rosenfeld 36343d9b1a2aSHans Rosenfeld feature = (uint8_t)(nioc->n_arg >> 32); 36353d9b1a2aSHans Rosenfeld 36363d9b1a2aSHans Rosenfeld switch (feature) { 36373d9b1a2aSHans Rosenfeld case NVME_FEAT_ARBITRATION: 36383d9b1a2aSHans Rosenfeld case NVME_FEAT_POWER_MGMT: 36393d9b1a2aSHans Rosenfeld case NVME_FEAT_TEMPERATURE: 36403d9b1a2aSHans Rosenfeld case NVME_FEAT_ERROR: 36413d9b1a2aSHans Rosenfeld case NVME_FEAT_NQUEUES: 36423d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_COAL: 36433d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_ATOM: 36443d9b1a2aSHans Rosenfeld case NVME_FEAT_ASYNC_EVENT: 36453d9b1a2aSHans Rosenfeld case NVME_FEAT_PROGRESS: 36463d9b1a2aSHans Rosenfeld if (nsid != 0) 36473d9b1a2aSHans Rosenfeld return (EINVAL); 36483d9b1a2aSHans Rosenfeld break; 36493d9b1a2aSHans Rosenfeld 36503d9b1a2aSHans Rosenfeld case NVME_FEAT_INTR_VECT: 36513d9b1a2aSHans Rosenfeld if (nsid != 0) 36523d9b1a2aSHans Rosenfeld return (EINVAL); 36533d9b1a2aSHans Rosenfeld 36543d9b1a2aSHans Rosenfeld res = nioc->n_arg & 0xffffffffUL; 36553d9b1a2aSHans Rosenfeld if (res >= nvme->n_intr_cnt) 36563d9b1a2aSHans Rosenfeld return (EINVAL); 36573d9b1a2aSHans Rosenfeld break; 36583d9b1a2aSHans Rosenfeld 36593d9b1a2aSHans Rosenfeld case NVME_FEAT_LBA_RANGE: 36603d9b1a2aSHans Rosenfeld if (nvme->n_lba_range_supported == B_FALSE) 36613d9b1a2aSHans Rosenfeld return (EINVAL); 36623d9b1a2aSHans Rosenfeld 36633d9b1a2aSHans Rosenfeld if (nsid == 0 || 36643d9b1a2aSHans Rosenfeld nsid > nvme->n_namespace_count) 36653d9b1a2aSHans Rosenfeld return (EINVAL); 36663d9b1a2aSHans Rosenfeld 36673d9b1a2aSHans Rosenfeld break; 36683d9b1a2aSHans Rosenfeld 36693d9b1a2aSHans Rosenfeld case NVME_FEAT_WRITE_CACHE: 36703d9b1a2aSHans Rosenfeld if (nsid != 0) 36713d9b1a2aSHans Rosenfeld return (EINVAL); 36723d9b1a2aSHans Rosenfeld 36733d9b1a2aSHans Rosenfeld if (!nvme->n_write_cache_present) 36743d9b1a2aSHans Rosenfeld return (EINVAL); 36753d9b1a2aSHans Rosenfeld 36763d9b1a2aSHans Rosenfeld break; 36773d9b1a2aSHans Rosenfeld 36783d9b1a2aSHans Rosenfeld case NVME_FEAT_AUTO_PST: 36793d9b1a2aSHans Rosenfeld if (nsid != 0) 36803d9b1a2aSHans Rosenfeld return (EINVAL); 36813d9b1a2aSHans Rosenfeld 36823d9b1a2aSHans Rosenfeld if (!nvme->n_auto_pst_supported) 36833d9b1a2aSHans Rosenfeld return (EINVAL); 36843d9b1a2aSHans Rosenfeld 36853d9b1a2aSHans Rosenfeld break; 36863d9b1a2aSHans Rosenfeld 36873d9b1a2aSHans Rosenfeld default: 36883d9b1a2aSHans Rosenfeld return (EINVAL); 36893d9b1a2aSHans Rosenfeld } 36903d9b1a2aSHans Rosenfeld 3691e984c70bSHans Rosenfeld rv = nvme_get_features(nvme, nsid, feature, &res, &buf, &bufsize); 3692e984c70bSHans Rosenfeld if (rv != 0) 3693e984c70bSHans Rosenfeld return (rv); 36943d9b1a2aSHans Rosenfeld 36953d9b1a2aSHans Rosenfeld if (nioc->n_len < bufsize) { 36963d9b1a2aSHans Rosenfeld kmem_free(buf, bufsize); 36973d9b1a2aSHans Rosenfeld return (EINVAL); 36983d9b1a2aSHans Rosenfeld } 36993d9b1a2aSHans Rosenfeld 37003d9b1a2aSHans Rosenfeld if (buf && ddi_copyout(buf, (void*)nioc->n_buf, bufsize, mode) != 0) 37013d9b1a2aSHans Rosenfeld rv = EFAULT; 37023d9b1a2aSHans Rosenfeld 37033d9b1a2aSHans Rosenfeld kmem_free(buf, bufsize); 37043d9b1a2aSHans Rosenfeld nioc->n_arg = res; 37053d9b1a2aSHans Rosenfeld nioc->n_len = bufsize; 37063d9b1a2aSHans Rosenfeld 37073d9b1a2aSHans Rosenfeld return (rv); 37083d9b1a2aSHans Rosenfeld } 37093d9b1a2aSHans Rosenfeld 37103d9b1a2aSHans Rosenfeld static int 37113d9b1a2aSHans Rosenfeld nvme_ioctl_intr_cnt(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 37123d9b1a2aSHans Rosenfeld cred_t *cred_p) 37133d9b1a2aSHans Rosenfeld { 37143d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nsid, mode, cred_p)); 37153d9b1a2aSHans Rosenfeld 37163d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 37173d9b1a2aSHans Rosenfeld return (EPERM); 37183d9b1a2aSHans Rosenfeld 37193d9b1a2aSHans Rosenfeld nioc->n_arg = nvme->n_intr_cnt; 37203d9b1a2aSHans Rosenfeld return (0); 37213d9b1a2aSHans Rosenfeld } 37223d9b1a2aSHans Rosenfeld 37233d9b1a2aSHans Rosenfeld static int 37243d9b1a2aSHans Rosenfeld nvme_ioctl_version(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 37253d9b1a2aSHans Rosenfeld cred_t *cred_p) 37263d9b1a2aSHans Rosenfeld { 37273d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nsid, cred_p)); 37283d9b1a2aSHans Rosenfeld int rv = 0; 37293d9b1a2aSHans Rosenfeld 37303d9b1a2aSHans Rosenfeld if ((mode & FREAD) == 0) 37313d9b1a2aSHans Rosenfeld return (EPERM); 37323d9b1a2aSHans Rosenfeld 37333d9b1a2aSHans Rosenfeld if (nioc->n_len < sizeof (nvme->n_version)) 37343d9b1a2aSHans Rosenfeld return (ENOMEM); 37353d9b1a2aSHans Rosenfeld 37363d9b1a2aSHans Rosenfeld if (ddi_copyout(&nvme->n_version, (void *)nioc->n_buf, 37373d9b1a2aSHans Rosenfeld sizeof (nvme->n_version), mode) != 0) 37383d9b1a2aSHans Rosenfeld rv = EFAULT; 37393d9b1a2aSHans Rosenfeld 37403d9b1a2aSHans Rosenfeld return (rv); 37413d9b1a2aSHans Rosenfeld } 37423d9b1a2aSHans Rosenfeld 37433d9b1a2aSHans Rosenfeld static int 37443d9b1a2aSHans Rosenfeld nvme_ioctl_format(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 37453d9b1a2aSHans Rosenfeld cred_t *cred_p) 37463d9b1a2aSHans Rosenfeld { 37473d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(mode)); 37483d9b1a2aSHans Rosenfeld nvme_format_nvm_t frmt = { 0 }; 37493d9b1a2aSHans Rosenfeld int c_nsid = nsid != 0 ? nsid - 1 : 0; 37503d9b1a2aSHans Rosenfeld 37513d9b1a2aSHans Rosenfeld if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 37523d9b1a2aSHans Rosenfeld return (EPERM); 37533d9b1a2aSHans Rosenfeld 37543d9b1a2aSHans Rosenfeld frmt.r = nioc->n_arg & 0xffffffff; 37553d9b1a2aSHans Rosenfeld 37563d9b1a2aSHans Rosenfeld /* 37573d9b1a2aSHans Rosenfeld * Check whether the FORMAT NVM command is supported. 37583d9b1a2aSHans Rosenfeld */ 37593d9b1a2aSHans Rosenfeld if (nvme->n_idctl->id_oacs.oa_format == 0) 37603d9b1a2aSHans Rosenfeld return (EINVAL); 37613d9b1a2aSHans Rosenfeld 37623d9b1a2aSHans Rosenfeld /* 37633d9b1a2aSHans Rosenfeld * Don't allow format or secure erase of individual namespace if that 37643d9b1a2aSHans Rosenfeld * would cause a format or secure erase of all namespaces. 37653d9b1a2aSHans Rosenfeld */ 37663d9b1a2aSHans Rosenfeld if (nsid != 0 && nvme->n_idctl->id_fna.fn_format != 0) 37673d9b1a2aSHans Rosenfeld return (EINVAL); 37683d9b1a2aSHans Rosenfeld 37693d9b1a2aSHans Rosenfeld if (nsid != 0 && frmt.b.fm_ses != NVME_FRMT_SES_NONE && 37703d9b1a2aSHans Rosenfeld nvme->n_idctl->id_fna.fn_sec_erase != 0) 37713d9b1a2aSHans Rosenfeld return (EINVAL); 37723d9b1a2aSHans Rosenfeld 37733d9b1a2aSHans Rosenfeld /* 37743d9b1a2aSHans Rosenfeld * Don't allow formatting with Protection Information. 37753d9b1a2aSHans Rosenfeld */ 37763d9b1a2aSHans Rosenfeld if (frmt.b.fm_pi != 0 || frmt.b.fm_pil != 0 || frmt.b.fm_ms != 0) 37773d9b1a2aSHans Rosenfeld return (EINVAL); 37783d9b1a2aSHans Rosenfeld 37793d9b1a2aSHans Rosenfeld /* 37803d9b1a2aSHans Rosenfeld * Don't allow formatting using an illegal LBA format, or any LBA format 37813d9b1a2aSHans Rosenfeld * that uses metadata. 37823d9b1a2aSHans Rosenfeld */ 37833d9b1a2aSHans Rosenfeld if (frmt.b.fm_lbaf > nvme->n_ns[c_nsid].ns_idns->id_nlbaf || 37843d9b1a2aSHans Rosenfeld nvme->n_ns[c_nsid].ns_idns->id_lbaf[frmt.b.fm_lbaf].lbaf_ms != 0) 37853d9b1a2aSHans Rosenfeld return (EINVAL); 37863d9b1a2aSHans Rosenfeld 37873d9b1a2aSHans Rosenfeld /* 37883d9b1a2aSHans Rosenfeld * Don't allow formatting using an illegal Secure Erase setting. 37893d9b1a2aSHans Rosenfeld */ 37903d9b1a2aSHans Rosenfeld if (frmt.b.fm_ses > NVME_FRMT_MAX_SES || 37913d9b1a2aSHans Rosenfeld (frmt.b.fm_ses == NVME_FRMT_SES_CRYPTO && 37923d9b1a2aSHans Rosenfeld nvme->n_idctl->id_fna.fn_crypt_erase == 0)) 37933d9b1a2aSHans Rosenfeld return (EINVAL); 37943d9b1a2aSHans Rosenfeld 37953d9b1a2aSHans Rosenfeld if (nsid == 0) 37963d9b1a2aSHans Rosenfeld nsid = (uint32_t)-1; 37973d9b1a2aSHans Rosenfeld 37983d9b1a2aSHans Rosenfeld return (nvme_format_nvm(nvme, nsid, frmt.b.fm_lbaf, B_FALSE, 0, B_FALSE, 37993d9b1a2aSHans Rosenfeld frmt.b.fm_ses)); 38003d9b1a2aSHans Rosenfeld } 38013d9b1a2aSHans Rosenfeld 38023d9b1a2aSHans Rosenfeld static int 38033d9b1a2aSHans Rosenfeld nvme_ioctl_detach(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 38043d9b1a2aSHans Rosenfeld cred_t *cred_p) 38053d9b1a2aSHans Rosenfeld { 38063d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nioc, mode)); 38073d9b1a2aSHans Rosenfeld int rv = 0; 38083d9b1a2aSHans Rosenfeld 38093d9b1a2aSHans Rosenfeld if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 38103d9b1a2aSHans Rosenfeld return (EPERM); 38113d9b1a2aSHans Rosenfeld 38123d9b1a2aSHans Rosenfeld if (nsid == 0) 38133d9b1a2aSHans Rosenfeld return (EINVAL); 38143d9b1a2aSHans Rosenfeld 38153d9b1a2aSHans Rosenfeld rv = bd_detach_handle(nvme->n_ns[nsid - 1].ns_bd_hdl); 38163d9b1a2aSHans Rosenfeld if (rv != DDI_SUCCESS) 38173d9b1a2aSHans Rosenfeld rv = EBUSY; 38183d9b1a2aSHans Rosenfeld 38193d9b1a2aSHans Rosenfeld return (rv); 38203d9b1a2aSHans Rosenfeld } 38213d9b1a2aSHans Rosenfeld 38223d9b1a2aSHans Rosenfeld static int 38233d9b1a2aSHans Rosenfeld nvme_ioctl_attach(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 38243d9b1a2aSHans Rosenfeld cred_t *cred_p) 38253d9b1a2aSHans Rosenfeld { 38263d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(nioc, mode)); 38273d9b1a2aSHans Rosenfeld nvme_identify_nsid_t *idns; 38283d9b1a2aSHans Rosenfeld int rv = 0; 38293d9b1a2aSHans Rosenfeld 38303d9b1a2aSHans Rosenfeld if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 38313d9b1a2aSHans Rosenfeld return (EPERM); 38323d9b1a2aSHans Rosenfeld 38333d9b1a2aSHans Rosenfeld if (nsid == 0) 38343d9b1a2aSHans Rosenfeld return (EINVAL); 38353d9b1a2aSHans Rosenfeld 38363d9b1a2aSHans Rosenfeld /* 38373d9b1a2aSHans Rosenfeld * Identify namespace again, free old identify data. 38383d9b1a2aSHans Rosenfeld */ 38393d9b1a2aSHans Rosenfeld idns = nvme->n_ns[nsid - 1].ns_idns; 38403d9b1a2aSHans Rosenfeld if (nvme_init_ns(nvme, nsid) != DDI_SUCCESS) 38413d9b1a2aSHans Rosenfeld return (EIO); 38423d9b1a2aSHans Rosenfeld 38433d9b1a2aSHans Rosenfeld kmem_free(idns, sizeof (nvme_identify_nsid_t)); 38443d9b1a2aSHans Rosenfeld 38453d9b1a2aSHans Rosenfeld rv = bd_attach_handle(nvme->n_dip, nvme->n_ns[nsid - 1].ns_bd_hdl); 38463d9b1a2aSHans Rosenfeld if (rv != DDI_SUCCESS) 38473d9b1a2aSHans Rosenfeld rv = EBUSY; 38483d9b1a2aSHans Rosenfeld 38493d9b1a2aSHans Rosenfeld return (rv); 38503d9b1a2aSHans Rosenfeld } 38513d9b1a2aSHans Rosenfeld 38523d9b1a2aSHans Rosenfeld static int 38533d9b1a2aSHans Rosenfeld nvme_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred_p, 38543d9b1a2aSHans Rosenfeld int *rval_p) 38553d9b1a2aSHans Rosenfeld { 38563d9b1a2aSHans Rosenfeld #ifndef __lock_lint 38573d9b1a2aSHans Rosenfeld _NOTE(ARGUNUSED(rval_p)); 38583d9b1a2aSHans Rosenfeld #endif 38593d9b1a2aSHans Rosenfeld minor_t minor = getminor(dev); 38603d9b1a2aSHans Rosenfeld nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 38613d9b1a2aSHans Rosenfeld int nsid = NVME_MINOR_NSID(minor); 38623d9b1a2aSHans Rosenfeld int rv = 0; 38633d9b1a2aSHans Rosenfeld nvme_ioctl_t nioc; 38643d9b1a2aSHans Rosenfeld 38653d9b1a2aSHans Rosenfeld int (*nvme_ioctl[])(nvme_t *, int, nvme_ioctl_t *, int, cred_t *) = { 38663d9b1a2aSHans Rosenfeld NULL, 38673d9b1a2aSHans Rosenfeld nvme_ioctl_identify, 38683d9b1a2aSHans Rosenfeld nvme_ioctl_identify, 38693d9b1a2aSHans Rosenfeld nvme_ioctl_capabilities, 38703d9b1a2aSHans Rosenfeld nvme_ioctl_get_logpage, 38713d9b1a2aSHans Rosenfeld nvme_ioctl_get_features, 38723d9b1a2aSHans Rosenfeld nvme_ioctl_intr_cnt, 38733d9b1a2aSHans Rosenfeld nvme_ioctl_version, 38743d9b1a2aSHans Rosenfeld nvme_ioctl_format, 38753d9b1a2aSHans Rosenfeld nvme_ioctl_detach, 38763d9b1a2aSHans Rosenfeld nvme_ioctl_attach 38773d9b1a2aSHans Rosenfeld }; 38783d9b1a2aSHans Rosenfeld 38793d9b1a2aSHans Rosenfeld if (nvme == NULL) 38803d9b1a2aSHans Rosenfeld return (ENXIO); 38813d9b1a2aSHans Rosenfeld 38823d9b1a2aSHans Rosenfeld if (nsid > nvme->n_namespace_count) 38833d9b1a2aSHans Rosenfeld return (ENXIO); 38843d9b1a2aSHans Rosenfeld 38853d9b1a2aSHans Rosenfeld if (IS_DEVCTL(cmd)) 38863d9b1a2aSHans Rosenfeld return (ndi_devctl_ioctl(nvme->n_dip, cmd, arg, mode, 0)); 38873d9b1a2aSHans Rosenfeld 38883d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 38893d9b1a2aSHans Rosenfeld switch (ddi_model_convert_from(mode & FMODELS)) { 38903d9b1a2aSHans Rosenfeld case DDI_MODEL_ILP32: { 38913d9b1a2aSHans Rosenfeld nvme_ioctl32_t nioc32; 38923d9b1a2aSHans Rosenfeld if (ddi_copyin((void*)arg, &nioc32, sizeof (nvme_ioctl32_t), 38933d9b1a2aSHans Rosenfeld mode) != 0) 38943d9b1a2aSHans Rosenfeld return (EFAULT); 38953d9b1a2aSHans Rosenfeld nioc.n_len = nioc32.n_len; 38963d9b1a2aSHans Rosenfeld nioc.n_buf = nioc32.n_buf; 38973d9b1a2aSHans Rosenfeld nioc.n_arg = nioc32.n_arg; 38983d9b1a2aSHans Rosenfeld break; 38993d9b1a2aSHans Rosenfeld } 39003d9b1a2aSHans Rosenfeld case DDI_MODEL_NONE: 39013d9b1a2aSHans Rosenfeld #endif 39023d9b1a2aSHans Rosenfeld if (ddi_copyin((void*)arg, &nioc, sizeof (nvme_ioctl_t), mode) 39033d9b1a2aSHans Rosenfeld != 0) 39043d9b1a2aSHans Rosenfeld return (EFAULT); 39053d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 39063d9b1a2aSHans Rosenfeld break; 39073d9b1a2aSHans Rosenfeld } 39083d9b1a2aSHans Rosenfeld #endif 39093d9b1a2aSHans Rosenfeld 3910e984c70bSHans Rosenfeld if (nvme->n_dead && cmd != NVME_IOC_DETACH) 3911e984c70bSHans Rosenfeld return (EIO); 3912e984c70bSHans Rosenfeld 3913e984c70bSHans Rosenfeld 39143d9b1a2aSHans Rosenfeld if (cmd == NVME_IOC_IDENTIFY_CTRL) { 39153d9b1a2aSHans Rosenfeld /* 39163d9b1a2aSHans Rosenfeld * This makes NVME_IOC_IDENTIFY_CTRL work the same on devctl and 39173d9b1a2aSHans Rosenfeld * attachment point nodes. 39183d9b1a2aSHans Rosenfeld */ 39193d9b1a2aSHans Rosenfeld nsid = 0; 39203d9b1a2aSHans Rosenfeld } else if (cmd == NVME_IOC_IDENTIFY_NSID && nsid == 0) { 39213d9b1a2aSHans Rosenfeld /* 39223d9b1a2aSHans Rosenfeld * This makes NVME_IOC_IDENTIFY_NSID work on a devctl node, it 39233d9b1a2aSHans Rosenfeld * will always return identify data for namespace 1. 39243d9b1a2aSHans Rosenfeld */ 39253d9b1a2aSHans Rosenfeld nsid = 1; 39263d9b1a2aSHans Rosenfeld } 39273d9b1a2aSHans Rosenfeld 39283d9b1a2aSHans Rosenfeld if (IS_NVME_IOC(cmd) && nvme_ioctl[NVME_IOC_CMD(cmd)] != NULL) 39293d9b1a2aSHans Rosenfeld rv = nvme_ioctl[NVME_IOC_CMD(cmd)](nvme, nsid, &nioc, mode, 39303d9b1a2aSHans Rosenfeld cred_p); 39313d9b1a2aSHans Rosenfeld else 39323d9b1a2aSHans Rosenfeld rv = EINVAL; 39333d9b1a2aSHans Rosenfeld 39343d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 39353d9b1a2aSHans Rosenfeld switch (ddi_model_convert_from(mode & FMODELS)) { 39363d9b1a2aSHans Rosenfeld case DDI_MODEL_ILP32: { 39373d9b1a2aSHans Rosenfeld nvme_ioctl32_t nioc32; 39383d9b1a2aSHans Rosenfeld 39393d9b1a2aSHans Rosenfeld nioc32.n_len = (size32_t)nioc.n_len; 39403d9b1a2aSHans Rosenfeld nioc32.n_buf = (uintptr32_t)nioc.n_buf; 39413d9b1a2aSHans Rosenfeld nioc32.n_arg = nioc.n_arg; 39423d9b1a2aSHans Rosenfeld 39433d9b1a2aSHans Rosenfeld if (ddi_copyout(&nioc32, (void *)arg, sizeof (nvme_ioctl32_t), 39443d9b1a2aSHans Rosenfeld mode) != 0) 39453d9b1a2aSHans Rosenfeld return (EFAULT); 39463d9b1a2aSHans Rosenfeld break; 39473d9b1a2aSHans Rosenfeld } 39483d9b1a2aSHans Rosenfeld case DDI_MODEL_NONE: 39493d9b1a2aSHans Rosenfeld #endif 39503d9b1a2aSHans Rosenfeld if (ddi_copyout(&nioc, (void *)arg, sizeof (nvme_ioctl_t), mode) 39513d9b1a2aSHans Rosenfeld != 0) 39523d9b1a2aSHans Rosenfeld return (EFAULT); 39533d9b1a2aSHans Rosenfeld #ifdef _MULTI_DATAMODEL 39543d9b1a2aSHans Rosenfeld break; 39553d9b1a2aSHans Rosenfeld } 39563d9b1a2aSHans Rosenfeld #endif 39573d9b1a2aSHans Rosenfeld 39583d9b1a2aSHans Rosenfeld return (rv); 39593d9b1a2aSHans Rosenfeld } 3960