1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2016 Nexenta Systems, Inc. All rights reserved. 14 * Copyright 2016 Tegile Systems, Inc. All rights reserved. 15 * Copyright (c) 2016 The MathWorks, Inc. All rights reserved. 16 */ 17 18 /* 19 * blkdev driver for NVMe compliant storage devices 20 * 21 * This driver was written to conform to version 1.2.1 of the NVMe 22 * specification. It may work with newer versions, but that is completely 23 * untested and disabled by default. 24 * 25 * The driver has only been tested on x86 systems and will not work on big- 26 * endian systems without changes to the code accessing registers and data 27 * structures used by the hardware. 28 * 29 * 30 * Interrupt Usage: 31 * 32 * The driver will use a FIXED interrupt while configuring the device as the 33 * specification requires. Later in the attach process it will switch to MSI-X 34 * or MSI if supported. The driver wants to have one interrupt vector per CPU, 35 * but it will work correctly if less are available. Interrupts can be shared 36 * by queues, the interrupt handler will iterate through the I/O queue array by 37 * steps of n_intr_cnt. Usually only the admin queue will share an interrupt 38 * with one I/O queue. The interrupt handler will retrieve completed commands 39 * from all queues sharing an interrupt vector and will post them to a taskq 40 * for completion processing. 41 * 42 * 43 * Command Processing: 44 * 45 * NVMe devices can have up to 65536 I/O queue pairs, with each queue holding up 46 * to 65536 I/O commands. The driver will configure one I/O queue pair per 47 * available interrupt vector, with the queue length usually much smaller than 48 * the maximum of 65536. If the hardware doesn't provide enough queues, fewer 49 * interrupt vectors will be used. 50 * 51 * Additionally the hardware provides a single special admin queue pair that can 52 * hold up to 4096 admin commands. 53 * 54 * From the hardware perspective both queues of a queue pair are independent, 55 * but they share some driver state: the command array (holding pointers to 56 * commands currently being processed by the hardware) and the active command 57 * counter. Access to the submission side of a queue pair and the shared state 58 * is protected by nq_mutex. The completion side of a queue pair does not need 59 * that protection apart from its access to the shared state; it is called only 60 * in the interrupt handler which does not run concurrently for the same 61 * interrupt vector. 62 * 63 * When a command is submitted to a queue pair the active command counter is 64 * incremented and a pointer to the command is stored in the command array. The 65 * array index is used as command identifier (CID) in the submission queue 66 * entry. Some commands may take a very long time to complete, and if the queue 67 * wraps around in that time a submission may find the next array slot to still 68 * be used by a long-running command. In this case the array is sequentially 69 * searched for the next free slot. The length of the command array is the same 70 * as the configured queue length. 71 * 72 * 73 * Namespace Support: 74 * 75 * NVMe devices can have multiple namespaces, each being a independent data 76 * store. The driver supports multiple namespaces and creates a blkdev interface 77 * for each namespace found. Namespaces can have various attributes to support 78 * thin provisioning and protection information. This driver does not support 79 * any of this and ignores namespaces that have these attributes. 80 * 81 * As of NVMe 1.1 namespaces can have an 64bit Extended Unique Identifier 82 * (EUI64). This driver uses the EUI64 if present to generate the devid and 83 * passes it to blkdev to use it in the device node names. As this is currently 84 * untested namespaces with EUI64 are ignored by default. 85 * 86 * We currently support only (2 << NVME_MINOR_INST_SHIFT) - 2 namespaces in a 87 * single controller. This is an artificial limit imposed by the driver to be 88 * able to address a reasonable number of controllers and namespaces using a 89 * 32bit minor node number. 90 * 91 * 92 * Minor nodes: 93 * 94 * For each NVMe device the driver exposes one minor node for the controller and 95 * one minor node for each namespace. The only operations supported by those 96 * minor nodes are open(9E), close(9E), and ioctl(9E). This serves as the 97 * interface for the nvmeadm(1M) utility. 98 * 99 * 100 * Blkdev Interface: 101 * 102 * This driver uses blkdev to do all the heavy lifting involved with presenting 103 * a disk device to the system. As a result, the processing of I/O requests is 104 * relatively simple as blkdev takes care of partitioning, boundary checks, DMA 105 * setup, and splitting of transfers into manageable chunks. 106 * 107 * I/O requests coming in from blkdev are turned into NVM commands and posted to 108 * an I/O queue. The queue is selected by taking the CPU id modulo the number of 109 * queues. There is currently no timeout handling of I/O commands. 110 * 111 * Blkdev also supports querying device/media information and generating a 112 * devid. The driver reports the best block size as determined by the namespace 113 * format back to blkdev as physical block size to support partition and block 114 * alignment. The devid is either based on the namespace EUI64, if present, or 115 * composed using the device vendor ID, model number, serial number, and the 116 * namespace ID. 117 * 118 * 119 * Error Handling: 120 * 121 * Error handling is currently limited to detecting fatal hardware errors, 122 * either by asynchronous events, or synchronously through command status or 123 * admin command timeouts. In case of severe errors the device is fenced off, 124 * all further requests will return EIO. FMA is then called to fault the device. 125 * 126 * The hardware has a limit for outstanding asynchronous event requests. Before 127 * this limit is known the driver assumes it is at least 1 and posts a single 128 * asynchronous request. Later when the limit is known more asynchronous event 129 * requests are posted to allow quicker reception of error information. When an 130 * asynchronous event is posted by the hardware the driver will parse the error 131 * status fields and log information or fault the device, depending on the 132 * severity of the asynchronous event. The asynchronous event request is then 133 * reused and posted to the admin queue again. 134 * 135 * On command completion the command status is checked for errors. In case of 136 * errors indicating a driver bug the driver panics. Almost all other error 137 * status values just cause EIO to be returned. 138 * 139 * Command timeouts are currently detected for all admin commands except 140 * asynchronous event requests. If a command times out and the hardware appears 141 * to be healthy the driver attempts to abort the command. If this fails the 142 * driver assumes the device to be dead, fences it off, and calls FMA to retire 143 * it. In general admin commands are issued at attach time only. No timeout 144 * handling of normal I/O commands is presently done. 145 * 146 * In some cases it may be possible that the ABORT command times out, too. In 147 * that case the device is also declared dead and fenced off. 148 * 149 * 150 * Quiesce / Fast Reboot: 151 * 152 * The driver currently does not support fast reboot. A quiesce(9E) entry point 153 * is still provided which is used to send a shutdown notification to the 154 * device. 155 * 156 * 157 * Driver Configuration: 158 * 159 * The following driver properties can be changed to control some aspects of the 160 * drivers operation: 161 * - strict-version: can be set to 0 to allow devices conforming to newer 162 * versions or namespaces with EUI64 to be used 163 * - ignore-unknown-vendor-status: can be set to 1 to not handle any vendor 164 * specific command status as a fatal error leading device faulting 165 * - admin-queue-len: the maximum length of the admin queue (16-4096) 166 * - io-queue-len: the maximum length of the I/O queues (16-65536) 167 * - async-event-limit: the maximum number of asynchronous event requests to be 168 * posted by the driver 169 * - volatile-write-cache-enable: can be set to 0 to disable the volatile write 170 * cache 171 * - min-phys-block-size: the minimum physical block size to report to blkdev, 172 * which is among other things the basis for ZFS vdev ashift 173 * 174 * 175 * TODO: 176 * - figure out sane default for I/O queue depth reported to blkdev 177 * - polled I/O support to support kernel core dumping 178 * - FMA handling of media errors 179 * - support for devices supporting very large I/O requests using chained PRPs 180 * - support for configuring hardware parameters like interrupt coalescing 181 * - support for media formatting and hard partitioning into namespaces 182 * - support for big-endian systems 183 * - support for fast reboot 184 * - support for firmware updates 185 * - support for NVMe Subsystem Reset (1.1) 186 * - support for Scatter/Gather lists (1.1) 187 * - support for Reservations (1.1) 188 * - support for power management 189 */ 190 191 #include <sys/byteorder.h> 192 #ifdef _BIG_ENDIAN 193 #error nvme driver needs porting for big-endian platforms 194 #endif 195 196 #include <sys/modctl.h> 197 #include <sys/conf.h> 198 #include <sys/devops.h> 199 #include <sys/ddi.h> 200 #include <sys/sunddi.h> 201 #include <sys/sunndi.h> 202 #include <sys/bitmap.h> 203 #include <sys/sysmacros.h> 204 #include <sys/param.h> 205 #include <sys/varargs.h> 206 #include <sys/cpuvar.h> 207 #include <sys/disp.h> 208 #include <sys/blkdev.h> 209 #include <sys/atomic.h> 210 #include <sys/archsystm.h> 211 #include <sys/sata/sata_hba.h> 212 #include <sys/stat.h> 213 #include <sys/policy.h> 214 215 #include <sys/nvme.h> 216 217 #ifdef __x86 218 #include <sys/x86_archext.h> 219 #endif 220 221 #include "nvme_reg.h" 222 #include "nvme_var.h" 223 224 225 /* NVMe spec version supported */ 226 static const int nvme_version_major = 1; 227 static const int nvme_version_minor = 2; 228 229 /* tunable for admin command timeout in seconds, default is 1s */ 230 int nvme_admin_cmd_timeout = 1; 231 232 /* tunable for FORMAT NVM command timeout in seconds, default is 600s */ 233 int nvme_format_cmd_timeout = 600; 234 235 static int nvme_attach(dev_info_t *, ddi_attach_cmd_t); 236 static int nvme_detach(dev_info_t *, ddi_detach_cmd_t); 237 static int nvme_quiesce(dev_info_t *); 238 static int nvme_fm_errcb(dev_info_t *, ddi_fm_error_t *, const void *); 239 static int nvme_setup_interrupts(nvme_t *, int, int); 240 static void nvme_release_interrupts(nvme_t *); 241 static uint_t nvme_intr(caddr_t, caddr_t); 242 243 static void nvme_shutdown(nvme_t *, int, boolean_t); 244 static boolean_t nvme_reset(nvme_t *, boolean_t); 245 static int nvme_init(nvme_t *); 246 static nvme_cmd_t *nvme_alloc_cmd(nvme_t *, int); 247 static void nvme_free_cmd(nvme_cmd_t *); 248 static nvme_cmd_t *nvme_create_nvm_cmd(nvme_namespace_t *, uint8_t, 249 bd_xfer_t *); 250 static int nvme_admin_cmd(nvme_cmd_t *, int); 251 static int nvme_submit_cmd(nvme_qpair_t *, nvme_cmd_t *); 252 static nvme_cmd_t *nvme_retrieve_cmd(nvme_t *, nvme_qpair_t *); 253 static boolean_t nvme_wait_cmd(nvme_cmd_t *, uint_t); 254 static void nvme_wakeup_cmd(void *); 255 static void nvme_async_event_task(void *); 256 257 static int nvme_check_unknown_cmd_status(nvme_cmd_t *); 258 static int nvme_check_vendor_cmd_status(nvme_cmd_t *); 259 static int nvme_check_integrity_cmd_status(nvme_cmd_t *); 260 static int nvme_check_specific_cmd_status(nvme_cmd_t *); 261 static int nvme_check_generic_cmd_status(nvme_cmd_t *); 262 static inline int nvme_check_cmd_status(nvme_cmd_t *); 263 264 static void nvme_abort_cmd(nvme_cmd_t *); 265 static int nvme_async_event(nvme_t *); 266 static int nvme_format_nvm(nvme_t *, uint32_t, uint8_t, boolean_t, uint8_t, 267 boolean_t, uint8_t); 268 static int nvme_get_logpage(nvme_t *, void **, size_t *, uint8_t, ...); 269 static void *nvme_identify(nvme_t *, uint32_t); 270 static boolean_t nvme_set_features(nvme_t *, uint32_t, uint8_t, uint32_t, 271 uint32_t *); 272 static boolean_t nvme_get_features(nvme_t *, uint32_t, uint8_t, uint32_t *, 273 void **, size_t *); 274 static boolean_t nvme_write_cache_set(nvme_t *, boolean_t); 275 static int nvme_set_nqueues(nvme_t *, uint16_t); 276 277 static void nvme_free_dma(nvme_dma_t *); 278 static int nvme_zalloc_dma(nvme_t *, size_t, uint_t, ddi_dma_attr_t *, 279 nvme_dma_t **); 280 static int nvme_zalloc_queue_dma(nvme_t *, uint32_t, uint16_t, uint_t, 281 nvme_dma_t **); 282 static void nvme_free_qpair(nvme_qpair_t *); 283 static int nvme_alloc_qpair(nvme_t *, uint32_t, nvme_qpair_t **, int); 284 static int nvme_create_io_qpair(nvme_t *, nvme_qpair_t *, uint16_t); 285 286 static inline void nvme_put64(nvme_t *, uintptr_t, uint64_t); 287 static inline void nvme_put32(nvme_t *, uintptr_t, uint32_t); 288 static inline uint64_t nvme_get64(nvme_t *, uintptr_t); 289 static inline uint32_t nvme_get32(nvme_t *, uintptr_t); 290 291 static boolean_t nvme_check_regs_hdl(nvme_t *); 292 static boolean_t nvme_check_dma_hdl(nvme_dma_t *); 293 294 static int nvme_fill_prp(nvme_cmd_t *, bd_xfer_t *); 295 296 static void nvme_bd_xfer_done(void *); 297 static void nvme_bd_driveinfo(void *, bd_drive_t *); 298 static int nvme_bd_mediainfo(void *, bd_media_t *); 299 static int nvme_bd_cmd(nvme_namespace_t *, bd_xfer_t *, uint8_t); 300 static int nvme_bd_read(void *, bd_xfer_t *); 301 static int nvme_bd_write(void *, bd_xfer_t *); 302 static int nvme_bd_sync(void *, bd_xfer_t *); 303 static int nvme_bd_devid(void *, dev_info_t *, ddi_devid_t *); 304 305 static int nvme_prp_dma_constructor(void *, void *, int); 306 static void nvme_prp_dma_destructor(void *, void *); 307 308 static void nvme_prepare_devid(nvme_t *, uint32_t); 309 310 static int nvme_open(dev_t *, int, int, cred_t *); 311 static int nvme_close(dev_t, int, int, cred_t *); 312 static int nvme_ioctl(dev_t, int, intptr_t, int, cred_t *, int *); 313 314 #define NVME_MINOR_INST_SHIFT 14 315 #define NVME_MINOR(inst, nsid) (((inst) << NVME_MINOR_INST_SHIFT) | (nsid)) 316 #define NVME_MINOR_INST(minor) ((minor) >> NVME_MINOR_INST_SHIFT) 317 #define NVME_MINOR_NSID(minor) ((minor) & ((1 << NVME_MINOR_INST_SHIFT) - 1)) 318 #define NVME_MINOR_MAX (NVME_MINOR(1, 0) - 2) 319 320 static void *nvme_state; 321 static kmem_cache_t *nvme_cmd_cache; 322 323 /* 324 * DMA attributes for queue DMA memory 325 * 326 * Queue DMA memory must be page aligned. The maximum length of a queue is 327 * 65536 entries, and an entry can be 64 bytes long. 328 */ 329 static ddi_dma_attr_t nvme_queue_dma_attr = { 330 .dma_attr_version = DMA_ATTR_V0, 331 .dma_attr_addr_lo = 0, 332 .dma_attr_addr_hi = 0xffffffffffffffffULL, 333 .dma_attr_count_max = (UINT16_MAX + 1) * sizeof (nvme_sqe_t) - 1, 334 .dma_attr_align = 0x1000, 335 .dma_attr_burstsizes = 0x7ff, 336 .dma_attr_minxfer = 0x1000, 337 .dma_attr_maxxfer = (UINT16_MAX + 1) * sizeof (nvme_sqe_t), 338 .dma_attr_seg = 0xffffffffffffffffULL, 339 .dma_attr_sgllen = 1, 340 .dma_attr_granular = 1, 341 .dma_attr_flags = 0, 342 }; 343 344 /* 345 * DMA attributes for transfers using Physical Region Page (PRP) entries 346 * 347 * A PRP entry describes one page of DMA memory using the page size specified 348 * in the controller configuration's memory page size register (CC.MPS). It uses 349 * a 64bit base address aligned to this page size. There is no limitation on 350 * chaining PRPs together for arbitrarily large DMA transfers. 351 */ 352 static ddi_dma_attr_t nvme_prp_dma_attr = { 353 .dma_attr_version = DMA_ATTR_V0, 354 .dma_attr_addr_lo = 0, 355 .dma_attr_addr_hi = 0xffffffffffffffffULL, 356 .dma_attr_count_max = 0xfff, 357 .dma_attr_align = 0x1000, 358 .dma_attr_burstsizes = 0x7ff, 359 .dma_attr_minxfer = 0x1000, 360 .dma_attr_maxxfer = 0x1000, 361 .dma_attr_seg = 0xfff, 362 .dma_attr_sgllen = -1, 363 .dma_attr_granular = 1, 364 .dma_attr_flags = 0, 365 }; 366 367 /* 368 * DMA attributes for transfers using scatter/gather lists 369 * 370 * A SGL entry describes a chunk of DMA memory using a 64bit base address and a 371 * 32bit length field. SGL Segment and SGL Last Segment entries require the 372 * length to be a multiple of 16 bytes. 373 */ 374 static ddi_dma_attr_t nvme_sgl_dma_attr = { 375 .dma_attr_version = DMA_ATTR_V0, 376 .dma_attr_addr_lo = 0, 377 .dma_attr_addr_hi = 0xffffffffffffffffULL, 378 .dma_attr_count_max = 0xffffffffUL, 379 .dma_attr_align = 1, 380 .dma_attr_burstsizes = 0x7ff, 381 .dma_attr_minxfer = 0x10, 382 .dma_attr_maxxfer = 0xfffffffffULL, 383 .dma_attr_seg = 0xffffffffffffffffULL, 384 .dma_attr_sgllen = -1, 385 .dma_attr_granular = 0x10, 386 .dma_attr_flags = 0 387 }; 388 389 static ddi_device_acc_attr_t nvme_reg_acc_attr = { 390 .devacc_attr_version = DDI_DEVICE_ATTR_V0, 391 .devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC, 392 .devacc_attr_dataorder = DDI_STRICTORDER_ACC 393 }; 394 395 static struct cb_ops nvme_cb_ops = { 396 .cb_open = nvme_open, 397 .cb_close = nvme_close, 398 .cb_strategy = nodev, 399 .cb_print = nodev, 400 .cb_dump = nodev, 401 .cb_read = nodev, 402 .cb_write = nodev, 403 .cb_ioctl = nvme_ioctl, 404 .cb_devmap = nodev, 405 .cb_mmap = nodev, 406 .cb_segmap = nodev, 407 .cb_chpoll = nochpoll, 408 .cb_prop_op = ddi_prop_op, 409 .cb_str = 0, 410 .cb_flag = D_NEW | D_MP, 411 .cb_rev = CB_REV, 412 .cb_aread = nodev, 413 .cb_awrite = nodev 414 }; 415 416 static struct dev_ops nvme_dev_ops = { 417 .devo_rev = DEVO_REV, 418 .devo_refcnt = 0, 419 .devo_getinfo = ddi_no_info, 420 .devo_identify = nulldev, 421 .devo_probe = nulldev, 422 .devo_attach = nvme_attach, 423 .devo_detach = nvme_detach, 424 .devo_reset = nodev, 425 .devo_cb_ops = &nvme_cb_ops, 426 .devo_bus_ops = NULL, 427 .devo_power = NULL, 428 .devo_quiesce = nvme_quiesce, 429 }; 430 431 static struct modldrv nvme_modldrv = { 432 .drv_modops = &mod_driverops, 433 .drv_linkinfo = "NVMe v1.1b", 434 .drv_dev_ops = &nvme_dev_ops 435 }; 436 437 static struct modlinkage nvme_modlinkage = { 438 .ml_rev = MODREV_1, 439 .ml_linkage = { &nvme_modldrv, NULL } 440 }; 441 442 static bd_ops_t nvme_bd_ops = { 443 .o_version = BD_OPS_VERSION_0, 444 .o_drive_info = nvme_bd_driveinfo, 445 .o_media_info = nvme_bd_mediainfo, 446 .o_devid_init = nvme_bd_devid, 447 .o_sync_cache = nvme_bd_sync, 448 .o_read = nvme_bd_read, 449 .o_write = nvme_bd_write, 450 }; 451 452 int 453 _init(void) 454 { 455 int error; 456 457 error = ddi_soft_state_init(&nvme_state, sizeof (nvme_t), 1); 458 if (error != DDI_SUCCESS) 459 return (error); 460 461 nvme_cmd_cache = kmem_cache_create("nvme_cmd_cache", 462 sizeof (nvme_cmd_t), 64, NULL, NULL, NULL, NULL, NULL, 0); 463 464 bd_mod_init(&nvme_dev_ops); 465 466 error = mod_install(&nvme_modlinkage); 467 if (error != DDI_SUCCESS) { 468 ddi_soft_state_fini(&nvme_state); 469 bd_mod_fini(&nvme_dev_ops); 470 } 471 472 return (error); 473 } 474 475 int 476 _fini(void) 477 { 478 int error; 479 480 error = mod_remove(&nvme_modlinkage); 481 if (error == DDI_SUCCESS) { 482 ddi_soft_state_fini(&nvme_state); 483 kmem_cache_destroy(nvme_cmd_cache); 484 bd_mod_fini(&nvme_dev_ops); 485 } 486 487 return (error); 488 } 489 490 int 491 _info(struct modinfo *modinfop) 492 { 493 return (mod_info(&nvme_modlinkage, modinfop)); 494 } 495 496 static inline void 497 nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val) 498 { 499 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 500 501 /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 502 ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val); 503 } 504 505 static inline void 506 nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val) 507 { 508 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 509 510 /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 511 ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val); 512 } 513 514 static inline uint64_t 515 nvme_get64(nvme_t *nvme, uintptr_t reg) 516 { 517 uint64_t val; 518 519 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0); 520 521 /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 522 val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg)); 523 524 return (val); 525 } 526 527 static inline uint32_t 528 nvme_get32(nvme_t *nvme, uintptr_t reg) 529 { 530 uint32_t val; 531 532 ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0); 533 534 /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 535 val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg)); 536 537 return (val); 538 } 539 540 static boolean_t 541 nvme_check_regs_hdl(nvme_t *nvme) 542 { 543 ddi_fm_error_t error; 544 545 ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION); 546 547 if (error.fme_status != DDI_FM_OK) 548 return (B_TRUE); 549 550 return (B_FALSE); 551 } 552 553 static boolean_t 554 nvme_check_dma_hdl(nvme_dma_t *dma) 555 { 556 ddi_fm_error_t error; 557 558 if (dma == NULL) 559 return (B_FALSE); 560 561 ddi_fm_dma_err_get(dma->nd_dmah, &error, DDI_FME_VERSION); 562 563 if (error.fme_status != DDI_FM_OK) 564 return (B_TRUE); 565 566 return (B_FALSE); 567 } 568 569 static void 570 nvme_free_dma_common(nvme_dma_t *dma) 571 { 572 if (dma->nd_dmah != NULL) 573 (void) ddi_dma_unbind_handle(dma->nd_dmah); 574 if (dma->nd_acch != NULL) 575 ddi_dma_mem_free(&dma->nd_acch); 576 if (dma->nd_dmah != NULL) 577 ddi_dma_free_handle(&dma->nd_dmah); 578 } 579 580 static void 581 nvme_free_dma(nvme_dma_t *dma) 582 { 583 nvme_free_dma_common(dma); 584 kmem_free(dma, sizeof (*dma)); 585 } 586 587 /* ARGSUSED */ 588 static void 589 nvme_prp_dma_destructor(void *buf, void *private) 590 { 591 nvme_dma_t *dma = (nvme_dma_t *)buf; 592 593 nvme_free_dma_common(dma); 594 } 595 596 static int 597 nvme_alloc_dma_common(nvme_t *nvme, nvme_dma_t *dma, 598 size_t len, uint_t flags, ddi_dma_attr_t *dma_attr) 599 { 600 if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL, 601 &dma->nd_dmah) != DDI_SUCCESS) { 602 /* 603 * Due to DDI_DMA_SLEEP this can't be DDI_DMA_NORESOURCES, and 604 * the only other possible error is DDI_DMA_BADATTR which 605 * indicates a driver bug which should cause a panic. 606 */ 607 dev_err(nvme->n_dip, CE_PANIC, 608 "!failed to get DMA handle, check DMA attributes"); 609 return (DDI_FAILURE); 610 } 611 612 /* 613 * ddi_dma_mem_alloc() can only fail when DDI_DMA_NOSLEEP is specified 614 * or the flags are conflicting, which isn't the case here. 615 */ 616 (void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr, 617 DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, &dma->nd_memp, 618 &dma->nd_len, &dma->nd_acch); 619 620 if (ddi_dma_addr_bind_handle(dma->nd_dmah, NULL, dma->nd_memp, 621 dma->nd_len, flags | DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, NULL, 622 &dma->nd_cookie, &dma->nd_ncookie) != DDI_DMA_MAPPED) { 623 dev_err(nvme->n_dip, CE_WARN, 624 "!failed to bind DMA memory"); 625 atomic_inc_32(&nvme->n_dma_bind_err); 626 nvme_free_dma_common(dma); 627 return (DDI_FAILURE); 628 } 629 630 return (DDI_SUCCESS); 631 } 632 633 static int 634 nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags, 635 ddi_dma_attr_t *dma_attr, nvme_dma_t **ret) 636 { 637 nvme_dma_t *dma = kmem_zalloc(sizeof (nvme_dma_t), KM_SLEEP); 638 639 if (nvme_alloc_dma_common(nvme, dma, len, flags, dma_attr) != 640 DDI_SUCCESS) { 641 *ret = NULL; 642 kmem_free(dma, sizeof (nvme_dma_t)); 643 return (DDI_FAILURE); 644 } 645 646 bzero(dma->nd_memp, dma->nd_len); 647 648 *ret = dma; 649 return (DDI_SUCCESS); 650 } 651 652 /* ARGSUSED */ 653 static int 654 nvme_prp_dma_constructor(void *buf, void *private, int flags) 655 { 656 nvme_dma_t *dma = (nvme_dma_t *)buf; 657 nvme_t *nvme = (nvme_t *)private; 658 659 dma->nd_dmah = NULL; 660 dma->nd_acch = NULL; 661 662 if (nvme_alloc_dma_common(nvme, dma, nvme->n_pagesize, 663 DDI_DMA_READ, &nvme->n_prp_dma_attr) != DDI_SUCCESS) { 664 return (-1); 665 } 666 667 ASSERT(dma->nd_ncookie == 1); 668 669 dma->nd_cached = B_TRUE; 670 671 return (0); 672 } 673 674 static int 675 nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len, 676 uint_t flags, nvme_dma_t **dma) 677 { 678 uint32_t len = nentry * qe_len; 679 ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr; 680 681 len = roundup(len, nvme->n_pagesize); 682 683 q_dma_attr.dma_attr_minxfer = len; 684 685 if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma) 686 != DDI_SUCCESS) { 687 dev_err(nvme->n_dip, CE_WARN, 688 "!failed to get DMA memory for queue"); 689 goto fail; 690 } 691 692 if ((*dma)->nd_ncookie != 1) { 693 dev_err(nvme->n_dip, CE_WARN, 694 "!got too many cookies for queue DMA"); 695 goto fail; 696 } 697 698 return (DDI_SUCCESS); 699 700 fail: 701 if (*dma) { 702 nvme_free_dma(*dma); 703 *dma = NULL; 704 } 705 706 return (DDI_FAILURE); 707 } 708 709 static void 710 nvme_free_qpair(nvme_qpair_t *qp) 711 { 712 int i; 713 714 mutex_destroy(&qp->nq_mutex); 715 716 if (qp->nq_sqdma != NULL) 717 nvme_free_dma(qp->nq_sqdma); 718 if (qp->nq_cqdma != NULL) 719 nvme_free_dma(qp->nq_cqdma); 720 721 if (qp->nq_active_cmds > 0) 722 for (i = 0; i != qp->nq_nentry; i++) 723 if (qp->nq_cmd[i] != NULL) 724 nvme_free_cmd(qp->nq_cmd[i]); 725 726 if (qp->nq_cmd != NULL) 727 kmem_free(qp->nq_cmd, sizeof (nvme_cmd_t *) * qp->nq_nentry); 728 729 kmem_free(qp, sizeof (nvme_qpair_t)); 730 } 731 732 static int 733 nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp, 734 int idx) 735 { 736 nvme_qpair_t *qp = kmem_zalloc(sizeof (*qp), KM_SLEEP); 737 738 mutex_init(&qp->nq_mutex, NULL, MUTEX_DRIVER, 739 DDI_INTR_PRI(nvme->n_intr_pri)); 740 741 if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t), 742 DDI_DMA_WRITE, &qp->nq_sqdma) != DDI_SUCCESS) 743 goto fail; 744 745 if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t), 746 DDI_DMA_READ, &qp->nq_cqdma) != DDI_SUCCESS) 747 goto fail; 748 749 qp->nq_sq = (nvme_sqe_t *)qp->nq_sqdma->nd_memp; 750 qp->nq_cq = (nvme_cqe_t *)qp->nq_cqdma->nd_memp; 751 qp->nq_nentry = nentry; 752 753 qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx); 754 qp->nq_cqhdbl = NVME_REG_CQHDBL(nvme, idx); 755 756 qp->nq_cmd = kmem_zalloc(sizeof (nvme_cmd_t *) * nentry, KM_SLEEP); 757 qp->nq_next_cmd = 0; 758 759 *nqp = qp; 760 return (DDI_SUCCESS); 761 762 fail: 763 nvme_free_qpair(qp); 764 *nqp = NULL; 765 766 return (DDI_FAILURE); 767 } 768 769 static nvme_cmd_t * 770 nvme_alloc_cmd(nvme_t *nvme, int kmflag) 771 { 772 nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag); 773 774 if (cmd == NULL) 775 return (cmd); 776 777 bzero(cmd, sizeof (nvme_cmd_t)); 778 779 cmd->nc_nvme = nvme; 780 781 mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER, 782 DDI_INTR_PRI(nvme->n_intr_pri)); 783 cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL); 784 785 return (cmd); 786 } 787 788 static void 789 nvme_free_cmd(nvme_cmd_t *cmd) 790 { 791 if (cmd->nc_dma) { 792 if (cmd->nc_dma->nd_cached) 793 kmem_cache_free(cmd->nc_nvme->n_prp_cache, 794 cmd->nc_dma); 795 else 796 nvme_free_dma(cmd->nc_dma); 797 cmd->nc_dma = NULL; 798 } 799 800 cv_destroy(&cmd->nc_cv); 801 mutex_destroy(&cmd->nc_mutex); 802 803 kmem_cache_free(nvme_cmd_cache, cmd); 804 } 805 806 static int 807 nvme_submit_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) 808 { 809 nvme_reg_sqtdbl_t tail = { 0 }; 810 811 mutex_enter(&qp->nq_mutex); 812 813 if (qp->nq_active_cmds == qp->nq_nentry) { 814 mutex_exit(&qp->nq_mutex); 815 return (DDI_FAILURE); 816 } 817 818 cmd->nc_completed = B_FALSE; 819 820 /* 821 * Try to insert the cmd into the active cmd array at the nq_next_cmd 822 * slot. If the slot is already occupied advance to the next slot and 823 * try again. This can happen for long running commands like async event 824 * requests. 825 */ 826 while (qp->nq_cmd[qp->nq_next_cmd] != NULL) 827 qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 828 qp->nq_cmd[qp->nq_next_cmd] = cmd; 829 830 qp->nq_active_cmds++; 831 832 cmd->nc_sqe.sqe_cid = qp->nq_next_cmd; 833 bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t)); 834 (void) ddi_dma_sync(qp->nq_sqdma->nd_dmah, 835 sizeof (nvme_sqe_t) * qp->nq_sqtail, 836 sizeof (nvme_sqe_t), DDI_DMA_SYNC_FORDEV); 837 qp->nq_next_cmd = (qp->nq_next_cmd + 1) % qp->nq_nentry; 838 839 tail.b.sqtdbl_sqt = qp->nq_sqtail = (qp->nq_sqtail + 1) % qp->nq_nentry; 840 nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r); 841 842 mutex_exit(&qp->nq_mutex); 843 return (DDI_SUCCESS); 844 } 845 846 static nvme_cmd_t * 847 nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp) 848 { 849 nvme_reg_cqhdbl_t head = { 0 }; 850 851 nvme_cqe_t *cqe; 852 nvme_cmd_t *cmd; 853 854 (void) ddi_dma_sync(qp->nq_cqdma->nd_dmah, 0, 855 sizeof (nvme_cqe_t) * qp->nq_nentry, DDI_DMA_SYNC_FORKERNEL); 856 857 cqe = &qp->nq_cq[qp->nq_cqhead]; 858 859 /* Check phase tag of CQE. Hardware inverts it for new entries. */ 860 if (cqe->cqe_sf.sf_p == qp->nq_phase) 861 return (NULL); 862 863 ASSERT(nvme->n_ioq[cqe->cqe_sqid] == qp); 864 ASSERT(cqe->cqe_cid < qp->nq_nentry); 865 866 mutex_enter(&qp->nq_mutex); 867 cmd = qp->nq_cmd[cqe->cqe_cid]; 868 qp->nq_cmd[cqe->cqe_cid] = NULL; 869 qp->nq_active_cmds--; 870 mutex_exit(&qp->nq_mutex); 871 872 ASSERT(cmd != NULL); 873 ASSERT(cmd->nc_nvme == nvme); 874 ASSERT(cmd->nc_sqid == cqe->cqe_sqid); 875 ASSERT(cmd->nc_sqe.sqe_cid == cqe->cqe_cid); 876 bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t)); 877 878 qp->nq_sqhead = cqe->cqe_sqhd; 879 880 head.b.cqhdbl_cqh = qp->nq_cqhead = (qp->nq_cqhead + 1) % qp->nq_nentry; 881 882 /* Toggle phase on wrap-around. */ 883 if (qp->nq_cqhead == 0) 884 qp->nq_phase = qp->nq_phase ? 0 : 1; 885 886 nvme_put32(cmd->nc_nvme, qp->nq_cqhdbl, head.r); 887 888 return (cmd); 889 } 890 891 static int 892 nvme_check_unknown_cmd_status(nvme_cmd_t *cmd) 893 { 894 nvme_cqe_t *cqe = &cmd->nc_cqe; 895 896 dev_err(cmd->nc_nvme->n_dip, CE_WARN, 897 "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 898 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 899 cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 900 cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 901 902 if (cmd->nc_xfer != NULL) 903 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 904 905 if (cmd->nc_nvme->n_strict_version) { 906 cmd->nc_nvme->n_dead = B_TRUE; 907 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 908 } 909 910 return (EIO); 911 } 912 913 static int 914 nvme_check_vendor_cmd_status(nvme_cmd_t *cmd) 915 { 916 nvme_cqe_t *cqe = &cmd->nc_cqe; 917 918 dev_err(cmd->nc_nvme->n_dip, CE_WARN, 919 "!unknown command status received: opc = %x, sqid = %d, cid = %d, " 920 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, 921 cqe->cqe_sqid, cqe->cqe_cid, cqe->cqe_sf.sf_sc, cqe->cqe_sf.sf_sct, 922 cqe->cqe_sf.sf_dnr, cqe->cqe_sf.sf_m); 923 if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) { 924 cmd->nc_nvme->n_dead = B_TRUE; 925 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 926 } 927 928 return (EIO); 929 } 930 931 static int 932 nvme_check_integrity_cmd_status(nvme_cmd_t *cmd) 933 { 934 nvme_cqe_t *cqe = &cmd->nc_cqe; 935 936 switch (cqe->cqe_sf.sf_sc) { 937 case NVME_CQE_SC_INT_NVM_WRITE: 938 /* write fail */ 939 /* TODO: post ereport */ 940 if (cmd->nc_xfer != NULL) 941 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 942 return (EIO); 943 944 case NVME_CQE_SC_INT_NVM_READ: 945 /* read fail */ 946 /* TODO: post ereport */ 947 if (cmd->nc_xfer != NULL) 948 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 949 return (EIO); 950 951 default: 952 return (nvme_check_unknown_cmd_status(cmd)); 953 } 954 } 955 956 static int 957 nvme_check_generic_cmd_status(nvme_cmd_t *cmd) 958 { 959 nvme_cqe_t *cqe = &cmd->nc_cqe; 960 961 switch (cqe->cqe_sf.sf_sc) { 962 case NVME_CQE_SC_GEN_SUCCESS: 963 return (0); 964 965 /* 966 * Errors indicating a bug in the driver should cause a panic. 967 */ 968 case NVME_CQE_SC_GEN_INV_OPC: 969 /* Invalid Command Opcode */ 970 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 971 "invalid opcode in cmd %p", (void *)cmd); 972 return (0); 973 974 case NVME_CQE_SC_GEN_INV_FLD: 975 /* Invalid Field in Command */ 976 if (!cmd->nc_dontpanic) 977 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, 978 "programming error: invalid field in cmd %p", 979 (void *)cmd); 980 return (EIO); 981 982 case NVME_CQE_SC_GEN_ID_CNFL: 983 /* Command ID Conflict */ 984 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 985 "cmd ID conflict in cmd %p", (void *)cmd); 986 return (0); 987 988 case NVME_CQE_SC_GEN_INV_NS: 989 /* Invalid Namespace or Format */ 990 if (!cmd->nc_dontpanic) 991 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, 992 "programming error: " "invalid NS/format in cmd %p", 993 (void *)cmd); 994 return (EINVAL); 995 996 case NVME_CQE_SC_GEN_NVM_LBA_RANGE: 997 /* LBA Out Of Range */ 998 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 999 "LBA out of range in cmd %p", (void *)cmd); 1000 return (0); 1001 1002 /* 1003 * Non-fatal errors, handle gracefully. 1004 */ 1005 case NVME_CQE_SC_GEN_DATA_XFR_ERR: 1006 /* Data Transfer Error (DMA) */ 1007 /* TODO: post ereport */ 1008 atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err); 1009 if (cmd->nc_xfer != NULL) 1010 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 1011 return (EIO); 1012 1013 case NVME_CQE_SC_GEN_INTERNAL_ERR: 1014 /* 1015 * Internal Error. The spec (v1.0, section 4.5.1.2) says 1016 * detailed error information is returned as async event, 1017 * so we pretty much ignore the error here and handle it 1018 * in the async event handler. 1019 */ 1020 atomic_inc_32(&cmd->nc_nvme->n_internal_err); 1021 if (cmd->nc_xfer != NULL) 1022 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 1023 return (EIO); 1024 1025 case NVME_CQE_SC_GEN_ABORT_REQUEST: 1026 /* 1027 * Command Abort Requested. This normally happens only when a 1028 * command times out. 1029 */ 1030 /* TODO: post ereport or change blkdev to handle this? */ 1031 atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err); 1032 return (ECANCELED); 1033 1034 case NVME_CQE_SC_GEN_ABORT_PWRLOSS: 1035 /* Command Aborted due to Power Loss Notification */ 1036 ddi_fm_service_impact(cmd->nc_nvme->n_dip, DDI_SERVICE_LOST); 1037 cmd->nc_nvme->n_dead = B_TRUE; 1038 return (EIO); 1039 1040 case NVME_CQE_SC_GEN_ABORT_SQ_DEL: 1041 /* Command Aborted due to SQ Deletion */ 1042 atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del); 1043 return (EIO); 1044 1045 case NVME_CQE_SC_GEN_NVM_CAP_EXC: 1046 /* Capacity Exceeded */ 1047 atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc); 1048 if (cmd->nc_xfer != NULL) 1049 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); 1050 return (EIO); 1051 1052 case NVME_CQE_SC_GEN_NVM_NS_NOTRDY: 1053 /* Namespace Not Ready */ 1054 atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy); 1055 if (cmd->nc_xfer != NULL) 1056 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); 1057 return (EIO); 1058 1059 default: 1060 return (nvme_check_unknown_cmd_status(cmd)); 1061 } 1062 } 1063 1064 static int 1065 nvme_check_specific_cmd_status(nvme_cmd_t *cmd) 1066 { 1067 nvme_cqe_t *cqe = &cmd->nc_cqe; 1068 1069 switch (cqe->cqe_sf.sf_sc) { 1070 case NVME_CQE_SC_SPC_INV_CQ: 1071 /* Completion Queue Invalid */ 1072 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE); 1073 atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err); 1074 return (EINVAL); 1075 1076 case NVME_CQE_SC_SPC_INV_QID: 1077 /* Invalid Queue Identifier */ 1078 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 1079 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE || 1080 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE || 1081 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 1082 atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err); 1083 return (EINVAL); 1084 1085 case NVME_CQE_SC_SPC_MAX_QSZ_EXC: 1086 /* Max Queue Size Exceeded */ 1087 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || 1088 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 1089 atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc); 1090 return (EINVAL); 1091 1092 case NVME_CQE_SC_SPC_ABRT_CMD_EXC: 1093 /* Abort Command Limit Exceeded */ 1094 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT); 1095 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 1096 "abort command limit exceeded in cmd %p", (void *)cmd); 1097 return (0); 1098 1099 case NVME_CQE_SC_SPC_ASYNC_EVREQ_EXC: 1100 /* Async Event Request Limit Exceeded */ 1101 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT); 1102 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " 1103 "async event request limit exceeded in cmd %p", 1104 (void *)cmd); 1105 return (0); 1106 1107 case NVME_CQE_SC_SPC_INV_INT_VECT: 1108 /* Invalid Interrupt Vector */ 1109 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); 1110 atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect); 1111 return (EINVAL); 1112 1113 case NVME_CQE_SC_SPC_INV_LOG_PAGE: 1114 /* Invalid Log Page */ 1115 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE); 1116 atomic_inc_32(&cmd->nc_nvme->n_inv_log_page); 1117 return (EINVAL); 1118 1119 case NVME_CQE_SC_SPC_INV_FORMAT: 1120 /* Invalid Format */ 1121 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT); 1122 atomic_inc_32(&cmd->nc_nvme->n_inv_format); 1123 if (cmd->nc_xfer != NULL) 1124 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 1125 return (EINVAL); 1126 1127 case NVME_CQE_SC_SPC_INV_Q_DEL: 1128 /* Invalid Queue Deletion */ 1129 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); 1130 atomic_inc_32(&cmd->nc_nvme->n_inv_q_del); 1131 return (EINVAL); 1132 1133 case NVME_CQE_SC_SPC_NVM_CNFL_ATTR: 1134 /* Conflicting Attributes */ 1135 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT || 1136 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 1137 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 1138 atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr); 1139 if (cmd->nc_xfer != NULL) 1140 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 1141 return (EINVAL); 1142 1143 case NVME_CQE_SC_SPC_NVM_INV_PROT: 1144 /* Invalid Protection Information */ 1145 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE || 1146 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || 1147 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 1148 atomic_inc_32(&cmd->nc_nvme->n_inv_prot); 1149 if (cmd->nc_xfer != NULL) 1150 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 1151 return (EINVAL); 1152 1153 case NVME_CQE_SC_SPC_NVM_READONLY: 1154 /* Write to Read Only Range */ 1155 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); 1156 atomic_inc_32(&cmd->nc_nvme->n_readonly); 1157 if (cmd->nc_xfer != NULL) 1158 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); 1159 return (EROFS); 1160 1161 default: 1162 return (nvme_check_unknown_cmd_status(cmd)); 1163 } 1164 } 1165 1166 static inline int 1167 nvme_check_cmd_status(nvme_cmd_t *cmd) 1168 { 1169 nvme_cqe_t *cqe = &cmd->nc_cqe; 1170 1171 /* take a shortcut if everything is alright */ 1172 if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 1173 cqe->cqe_sf.sf_sc == NVME_CQE_SC_GEN_SUCCESS) 1174 return (0); 1175 1176 if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC) 1177 return (nvme_check_generic_cmd_status(cmd)); 1178 else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_SPECIFIC) 1179 return (nvme_check_specific_cmd_status(cmd)); 1180 else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_INTEGRITY) 1181 return (nvme_check_integrity_cmd_status(cmd)); 1182 else if (cqe->cqe_sf.sf_sct == NVME_CQE_SCT_VENDOR) 1183 return (nvme_check_vendor_cmd_status(cmd)); 1184 1185 return (nvme_check_unknown_cmd_status(cmd)); 1186 } 1187 1188 /* 1189 * nvme_abort_cmd_cb -- replaces nc_callback of aborted commands 1190 * 1191 * This functions takes care of cleaning up aborted commands. The command 1192 * status is checked to catch any fatal errors. 1193 */ 1194 static void 1195 nvme_abort_cmd_cb(void *arg) 1196 { 1197 nvme_cmd_t *cmd = arg; 1198 1199 /* 1200 * Grab the command mutex. Once we have it we hold the last reference 1201 * to the command and can safely free it. 1202 */ 1203 mutex_enter(&cmd->nc_mutex); 1204 (void) nvme_check_cmd_status(cmd); 1205 mutex_exit(&cmd->nc_mutex); 1206 1207 nvme_free_cmd(cmd); 1208 } 1209 1210 static void 1211 nvme_abort_cmd(nvme_cmd_t *abort_cmd) 1212 { 1213 nvme_t *nvme = abort_cmd->nc_nvme; 1214 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1215 nvme_abort_cmd_t ac = { 0 }; 1216 1217 sema_p(&nvme->n_abort_sema); 1218 1219 ac.b.ac_cid = abort_cmd->nc_sqe.sqe_cid; 1220 ac.b.ac_sqid = abort_cmd->nc_sqid; 1221 1222 /* 1223 * Drop the mutex of the aborted command. From this point on 1224 * we must assume that the abort callback has freed the command. 1225 */ 1226 mutex_exit(&abort_cmd->nc_mutex); 1227 1228 cmd->nc_sqid = 0; 1229 cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT; 1230 cmd->nc_callback = nvme_wakeup_cmd; 1231 cmd->nc_sqe.sqe_cdw10 = ac.r; 1232 1233 /* 1234 * Send the ABORT to the hardware. The ABORT command will return _after_ 1235 * the aborted command has completed (aborted or otherwise). 1236 */ 1237 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1238 sema_v(&nvme->n_abort_sema); 1239 dev_err(nvme->n_dip, CE_WARN, 1240 "!nvme_admin_cmd failed for ABORT"); 1241 atomic_inc_32(&nvme->n_abort_failed); 1242 return; 1243 } 1244 sema_v(&nvme->n_abort_sema); 1245 1246 if (nvme_check_cmd_status(cmd)) { 1247 dev_err(nvme->n_dip, CE_WARN, 1248 "!ABORT failed with sct = %x, sc = %x", 1249 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 1250 atomic_inc_32(&nvme->n_abort_failed); 1251 } else { 1252 atomic_inc_32(&nvme->n_cmd_aborted); 1253 } 1254 1255 nvme_free_cmd(cmd); 1256 } 1257 1258 /* 1259 * nvme_wait_cmd -- wait for command completion or timeout 1260 * 1261 * Returns B_TRUE if the command completed normally. 1262 * 1263 * Returns B_FALSE if the command timed out and an abort was attempted. The 1264 * command mutex will be dropped and the command must be considered freed. The 1265 * freeing of the command is normally done by the abort command callback. 1266 * 1267 * In case of a serious error or a timeout of the abort command the hardware 1268 * will be declared dead and FMA will be notified. 1269 */ 1270 static boolean_t 1271 nvme_wait_cmd(nvme_cmd_t *cmd, uint_t sec) 1272 { 1273 clock_t timeout = ddi_get_lbolt() + drv_usectohz(sec * MICROSEC); 1274 nvme_t *nvme = cmd->nc_nvme; 1275 nvme_reg_csts_t csts; 1276 1277 ASSERT(mutex_owned(&cmd->nc_mutex)); 1278 1279 while (!cmd->nc_completed) { 1280 if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1) 1281 break; 1282 } 1283 1284 if (cmd->nc_completed) 1285 return (B_TRUE); 1286 1287 /* 1288 * The command timed out. Change the callback to the cleanup function. 1289 */ 1290 cmd->nc_callback = nvme_abort_cmd_cb; 1291 1292 /* 1293 * Check controller for fatal status, any errors associated with the 1294 * register or DMA handle, or for a double timeout (abort command timed 1295 * out). If necessary log a warning and call FMA. 1296 */ 1297 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 1298 dev_err(nvme->n_dip, CE_WARN, "!command timeout, " 1299 "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_opc, csts.b.csts_cfs); 1300 atomic_inc_32(&nvme->n_cmd_timeout); 1301 1302 if (csts.b.csts_cfs || 1303 nvme_check_regs_hdl(nvme) || 1304 nvme_check_dma_hdl(cmd->nc_dma) || 1305 cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) { 1306 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 1307 nvme->n_dead = B_TRUE; 1308 mutex_exit(&cmd->nc_mutex); 1309 } else { 1310 /* 1311 * Try to abort the command. The command mutex is released by 1312 * nvme_abort_cmd(). 1313 * If the abort succeeds it will have freed the aborted command. 1314 * If the abort fails for other reasons we must assume that the 1315 * command may complete at any time, and the callback will free 1316 * it for us. 1317 */ 1318 nvme_abort_cmd(cmd); 1319 } 1320 1321 return (B_FALSE); 1322 } 1323 1324 static void 1325 nvme_wakeup_cmd(void *arg) 1326 { 1327 nvme_cmd_t *cmd = arg; 1328 1329 mutex_enter(&cmd->nc_mutex); 1330 /* 1331 * There is a slight chance that this command completed shortly after 1332 * the timeout was hit in nvme_wait_cmd() but before the callback was 1333 * changed. Catch that case here and clean up accordingly. 1334 */ 1335 if (cmd->nc_callback == nvme_abort_cmd_cb) { 1336 mutex_exit(&cmd->nc_mutex); 1337 nvme_abort_cmd_cb(cmd); 1338 return; 1339 } 1340 1341 cmd->nc_completed = B_TRUE; 1342 cv_signal(&cmd->nc_cv); 1343 mutex_exit(&cmd->nc_mutex); 1344 } 1345 1346 static void 1347 nvme_async_event_task(void *arg) 1348 { 1349 nvme_cmd_t *cmd = arg; 1350 nvme_t *nvme = cmd->nc_nvme; 1351 nvme_error_log_entry_t *error_log = NULL; 1352 nvme_health_log_t *health_log = NULL; 1353 size_t logsize = 0; 1354 nvme_async_event_t event; 1355 int ret; 1356 1357 /* 1358 * Check for errors associated with the async request itself. The only 1359 * command-specific error is "async event limit exceeded", which 1360 * indicates a programming error in the driver and causes a panic in 1361 * nvme_check_cmd_status(). 1362 * 1363 * Other possible errors are various scenarios where the async request 1364 * was aborted, or internal errors in the device. Internal errors are 1365 * reported to FMA, the command aborts need no special handling here. 1366 */ 1367 if (nvme_check_cmd_status(cmd)) { 1368 dev_err(cmd->nc_nvme->n_dip, CE_WARN, 1369 "!async event request returned failure, sct = %x, " 1370 "sc = %x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct, 1371 cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr, 1372 cmd->nc_cqe.cqe_sf.sf_m); 1373 1374 if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 1375 cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) { 1376 cmd->nc_nvme->n_dead = B_TRUE; 1377 ddi_fm_service_impact(cmd->nc_nvme->n_dip, 1378 DDI_SERVICE_LOST); 1379 } 1380 nvme_free_cmd(cmd); 1381 return; 1382 } 1383 1384 1385 event.r = cmd->nc_cqe.cqe_dw0; 1386 1387 /* Clear CQE and re-submit the async request. */ 1388 bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t)); 1389 ret = nvme_submit_cmd(nvme->n_adminq, cmd); 1390 1391 if (ret != DDI_SUCCESS) { 1392 dev_err(nvme->n_dip, CE_WARN, 1393 "!failed to resubmit async event request"); 1394 atomic_inc_32(&nvme->n_async_resubmit_failed); 1395 nvme_free_cmd(cmd); 1396 } 1397 1398 switch (event.b.ae_type) { 1399 case NVME_ASYNC_TYPE_ERROR: 1400 if (event.b.ae_logpage == NVME_LOGPAGE_ERROR) { 1401 (void) nvme_get_logpage(nvme, (void **)&error_log, 1402 &logsize, event.b.ae_logpage); 1403 } else { 1404 dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 1405 "async event reply: %d", event.b.ae_logpage); 1406 atomic_inc_32(&nvme->n_wrong_logpage); 1407 } 1408 1409 switch (event.b.ae_info) { 1410 case NVME_ASYNC_ERROR_INV_SQ: 1411 dev_err(nvme->n_dip, CE_PANIC, "programming error: " 1412 "invalid submission queue"); 1413 return; 1414 1415 case NVME_ASYNC_ERROR_INV_DBL: 1416 dev_err(nvme->n_dip, CE_PANIC, "programming error: " 1417 "invalid doorbell write value"); 1418 return; 1419 1420 case NVME_ASYNC_ERROR_DIAGFAIL: 1421 dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure"); 1422 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 1423 nvme->n_dead = B_TRUE; 1424 atomic_inc_32(&nvme->n_diagfail_event); 1425 break; 1426 1427 case NVME_ASYNC_ERROR_PERSISTENT: 1428 dev_err(nvme->n_dip, CE_WARN, "!persistent internal " 1429 "device error"); 1430 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 1431 nvme->n_dead = B_TRUE; 1432 atomic_inc_32(&nvme->n_persistent_event); 1433 break; 1434 1435 case NVME_ASYNC_ERROR_TRANSIENT: 1436 dev_err(nvme->n_dip, CE_WARN, "!transient internal " 1437 "device error"); 1438 /* TODO: send ereport */ 1439 atomic_inc_32(&nvme->n_transient_event); 1440 break; 1441 1442 case NVME_ASYNC_ERROR_FW_LOAD: 1443 dev_err(nvme->n_dip, CE_WARN, 1444 "!firmware image load error"); 1445 atomic_inc_32(&nvme->n_fw_load_event); 1446 break; 1447 } 1448 break; 1449 1450 case NVME_ASYNC_TYPE_HEALTH: 1451 if (event.b.ae_logpage == NVME_LOGPAGE_HEALTH) { 1452 (void) nvme_get_logpage(nvme, (void **)&health_log, 1453 &logsize, event.b.ae_logpage, -1); 1454 } else { 1455 dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in " 1456 "async event reply: %d", event.b.ae_logpage); 1457 atomic_inc_32(&nvme->n_wrong_logpage); 1458 } 1459 1460 switch (event.b.ae_info) { 1461 case NVME_ASYNC_HEALTH_RELIABILITY: 1462 dev_err(nvme->n_dip, CE_WARN, 1463 "!device reliability compromised"); 1464 /* TODO: send ereport */ 1465 atomic_inc_32(&nvme->n_reliability_event); 1466 break; 1467 1468 case NVME_ASYNC_HEALTH_TEMPERATURE: 1469 dev_err(nvme->n_dip, CE_WARN, 1470 "!temperature above threshold"); 1471 /* TODO: send ereport */ 1472 atomic_inc_32(&nvme->n_temperature_event); 1473 break; 1474 1475 case NVME_ASYNC_HEALTH_SPARE: 1476 dev_err(nvme->n_dip, CE_WARN, 1477 "!spare space below threshold"); 1478 /* TODO: send ereport */ 1479 atomic_inc_32(&nvme->n_spare_event); 1480 break; 1481 } 1482 break; 1483 1484 case NVME_ASYNC_TYPE_VENDOR: 1485 dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event " 1486 "received, info = %x, logpage = %x", event.b.ae_info, 1487 event.b.ae_logpage); 1488 atomic_inc_32(&nvme->n_vendor_event); 1489 break; 1490 1491 default: 1492 dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, " 1493 "type = %x, info = %x, logpage = %x", event.b.ae_type, 1494 event.b.ae_info, event.b.ae_logpage); 1495 atomic_inc_32(&nvme->n_unknown_event); 1496 break; 1497 } 1498 1499 if (error_log) 1500 kmem_free(error_log, logsize); 1501 1502 if (health_log) 1503 kmem_free(health_log, logsize); 1504 } 1505 1506 static int 1507 nvme_admin_cmd(nvme_cmd_t *cmd, int sec) 1508 { 1509 int ret; 1510 1511 mutex_enter(&cmd->nc_mutex); 1512 ret = nvme_submit_cmd(cmd->nc_nvme->n_adminq, cmd); 1513 1514 if (ret != DDI_SUCCESS) { 1515 mutex_exit(&cmd->nc_mutex); 1516 dev_err(cmd->nc_nvme->n_dip, CE_WARN, 1517 "!nvme_submit_cmd failed"); 1518 atomic_inc_32(&cmd->nc_nvme->n_admin_queue_full); 1519 nvme_free_cmd(cmd); 1520 return (DDI_FAILURE); 1521 } 1522 1523 if (nvme_wait_cmd(cmd, sec) == B_FALSE) { 1524 /* 1525 * The command timed out. An abort command was posted that 1526 * will take care of the cleanup. 1527 */ 1528 return (DDI_FAILURE); 1529 } 1530 mutex_exit(&cmd->nc_mutex); 1531 1532 return (DDI_SUCCESS); 1533 } 1534 1535 static int 1536 nvme_async_event(nvme_t *nvme) 1537 { 1538 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1539 int ret; 1540 1541 cmd->nc_sqid = 0; 1542 cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT; 1543 cmd->nc_callback = nvme_async_event_task; 1544 1545 ret = nvme_submit_cmd(nvme->n_adminq, cmd); 1546 1547 if (ret != DDI_SUCCESS) { 1548 dev_err(nvme->n_dip, CE_WARN, 1549 "!nvme_submit_cmd failed for ASYNCHRONOUS EVENT"); 1550 nvme_free_cmd(cmd); 1551 return (DDI_FAILURE); 1552 } 1553 1554 return (DDI_SUCCESS); 1555 } 1556 1557 static int 1558 nvme_format_nvm(nvme_t *nvme, uint32_t nsid, uint8_t lbaf, boolean_t ms, 1559 uint8_t pi, boolean_t pil, uint8_t ses) 1560 { 1561 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1562 nvme_format_nvm_t format_nvm = { 0 }; 1563 int ret; 1564 1565 format_nvm.b.fm_lbaf = lbaf & 0xf; 1566 format_nvm.b.fm_ms = ms ? 1 : 0; 1567 format_nvm.b.fm_pi = pi & 0x7; 1568 format_nvm.b.fm_pil = pil ? 1 : 0; 1569 format_nvm.b.fm_ses = ses & 0x7; 1570 1571 cmd->nc_sqid = 0; 1572 cmd->nc_callback = nvme_wakeup_cmd; 1573 cmd->nc_sqe.sqe_nsid = nsid; 1574 cmd->nc_sqe.sqe_opc = NVME_OPC_NVM_FORMAT; 1575 cmd->nc_sqe.sqe_cdw10 = format_nvm.r; 1576 1577 /* 1578 * Some devices like Samsung SM951 don't allow formatting of all 1579 * namespaces in one command. Handle that gracefully. 1580 */ 1581 if (nsid == (uint32_t)-1) 1582 cmd->nc_dontpanic = B_TRUE; 1583 1584 if ((ret = nvme_admin_cmd(cmd, nvme_format_cmd_timeout)) 1585 != DDI_SUCCESS) { 1586 dev_err(nvme->n_dip, CE_WARN, 1587 "!nvme_admin_cmd failed for FORMAT NVM"); 1588 return (EIO); 1589 } 1590 1591 if ((ret = nvme_check_cmd_status(cmd)) != 0) { 1592 dev_err(nvme->n_dip, CE_WARN, 1593 "!FORMAT failed with sct = %x, sc = %x", 1594 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 1595 } 1596 1597 nvme_free_cmd(cmd); 1598 return (ret); 1599 } 1600 1601 static int 1602 nvme_get_logpage(nvme_t *nvme, void **buf, size_t *bufsize, uint8_t logpage, 1603 ...) 1604 { 1605 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1606 nvme_getlogpage_t getlogpage = { 0 }; 1607 va_list ap; 1608 int ret = DDI_FAILURE; 1609 1610 va_start(ap, logpage); 1611 1612 cmd->nc_sqid = 0; 1613 cmd->nc_callback = nvme_wakeup_cmd; 1614 cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE; 1615 1616 getlogpage.b.lp_lid = logpage; 1617 1618 switch (logpage) { 1619 case NVME_LOGPAGE_ERROR: 1620 cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 1621 /* 1622 * The GET LOG PAGE command can use at most 2 pages to return 1623 * data, PRP lists are not supported. 1624 */ 1625 *bufsize = MIN(2 * nvme->n_pagesize, 1626 nvme->n_error_log_len * sizeof (nvme_error_log_entry_t)); 1627 break; 1628 1629 case NVME_LOGPAGE_HEALTH: 1630 cmd->nc_sqe.sqe_nsid = va_arg(ap, uint32_t); 1631 *bufsize = sizeof (nvme_health_log_t); 1632 break; 1633 1634 case NVME_LOGPAGE_FWSLOT: 1635 cmd->nc_sqe.sqe_nsid = (uint32_t)-1; 1636 *bufsize = sizeof (nvme_fwslot_log_t); 1637 break; 1638 1639 default: 1640 dev_err(nvme->n_dip, CE_WARN, "!unknown log page requested: %d", 1641 logpage); 1642 atomic_inc_32(&nvme->n_unknown_logpage); 1643 goto fail; 1644 } 1645 1646 va_end(ap); 1647 1648 getlogpage.b.lp_numd = *bufsize / sizeof (uint32_t) - 1; 1649 1650 cmd->nc_sqe.sqe_cdw10 = getlogpage.r; 1651 1652 if (nvme_zalloc_dma(nvme, getlogpage.b.lp_numd * sizeof (uint32_t), 1653 DDI_DMA_READ, &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 1654 dev_err(nvme->n_dip, CE_WARN, 1655 "!nvme_zalloc_dma failed for GET LOG PAGE"); 1656 goto fail; 1657 } 1658 1659 if (cmd->nc_dma->nd_ncookie > 2) { 1660 dev_err(nvme->n_dip, CE_WARN, 1661 "!too many DMA cookies for GET LOG PAGE"); 1662 atomic_inc_32(&nvme->n_too_many_cookies); 1663 goto fail; 1664 } 1665 1666 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 1667 if (cmd->nc_dma->nd_ncookie > 1) { 1668 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 1669 &cmd->nc_dma->nd_cookie); 1670 cmd->nc_sqe.sqe_dptr.d_prp[1] = 1671 cmd->nc_dma->nd_cookie.dmac_laddress; 1672 } 1673 1674 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1675 dev_err(nvme->n_dip, CE_WARN, 1676 "!nvme_admin_cmd failed for GET LOG PAGE"); 1677 return (ret); 1678 } 1679 1680 if (nvme_check_cmd_status(cmd)) { 1681 dev_err(nvme->n_dip, CE_WARN, 1682 "!GET LOG PAGE failed with sct = %x, sc = %x", 1683 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 1684 goto fail; 1685 } 1686 1687 *buf = kmem_alloc(*bufsize, KM_SLEEP); 1688 bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize); 1689 1690 ret = DDI_SUCCESS; 1691 1692 fail: 1693 nvme_free_cmd(cmd); 1694 1695 return (ret); 1696 } 1697 1698 static void * 1699 nvme_identify(nvme_t *nvme, uint32_t nsid) 1700 { 1701 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1702 void *buf = NULL; 1703 1704 cmd->nc_sqid = 0; 1705 cmd->nc_callback = nvme_wakeup_cmd; 1706 cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY; 1707 cmd->nc_sqe.sqe_nsid = nsid; 1708 cmd->nc_sqe.sqe_cdw10 = nsid ? NVME_IDENTIFY_NSID : NVME_IDENTIFY_CTRL; 1709 1710 if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ, 1711 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 1712 dev_err(nvme->n_dip, CE_WARN, 1713 "!nvme_zalloc_dma failed for IDENTIFY"); 1714 goto fail; 1715 } 1716 1717 if (cmd->nc_dma->nd_ncookie > 2) { 1718 dev_err(nvme->n_dip, CE_WARN, 1719 "!too many DMA cookies for IDENTIFY"); 1720 atomic_inc_32(&nvme->n_too_many_cookies); 1721 goto fail; 1722 } 1723 1724 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; 1725 if (cmd->nc_dma->nd_ncookie > 1) { 1726 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 1727 &cmd->nc_dma->nd_cookie); 1728 cmd->nc_sqe.sqe_dptr.d_prp[1] = 1729 cmd->nc_dma->nd_cookie.dmac_laddress; 1730 } 1731 1732 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1733 dev_err(nvme->n_dip, CE_WARN, 1734 "!nvme_admin_cmd failed for IDENTIFY"); 1735 return (NULL); 1736 } 1737 1738 if (nvme_check_cmd_status(cmd)) { 1739 dev_err(nvme->n_dip, CE_WARN, 1740 "!IDENTIFY failed with sct = %x, sc = %x", 1741 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 1742 goto fail; 1743 } 1744 1745 buf = kmem_alloc(NVME_IDENTIFY_BUFSIZE, KM_SLEEP); 1746 bcopy(cmd->nc_dma->nd_memp, buf, NVME_IDENTIFY_BUFSIZE); 1747 1748 fail: 1749 nvme_free_cmd(cmd); 1750 1751 return (buf); 1752 } 1753 1754 static boolean_t 1755 nvme_set_features(nvme_t *nvme, uint32_t nsid, uint8_t feature, uint32_t val, 1756 uint32_t *res) 1757 { 1758 _NOTE(ARGUNUSED(nsid)); 1759 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1760 boolean_t ret = B_FALSE; 1761 1762 ASSERT(res != NULL); 1763 1764 cmd->nc_sqid = 0; 1765 cmd->nc_callback = nvme_wakeup_cmd; 1766 cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES; 1767 cmd->nc_sqe.sqe_cdw10 = feature; 1768 cmd->nc_sqe.sqe_cdw11 = val; 1769 1770 switch (feature) { 1771 case NVME_FEAT_WRITE_CACHE: 1772 if (!nvme->n_write_cache_present) 1773 goto fail; 1774 break; 1775 1776 case NVME_FEAT_NQUEUES: 1777 break; 1778 1779 default: 1780 goto fail; 1781 } 1782 1783 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1784 dev_err(nvme->n_dip, CE_WARN, 1785 "!nvme_admin_cmd failed for SET FEATURES"); 1786 return (ret); 1787 } 1788 1789 if (nvme_check_cmd_status(cmd)) { 1790 dev_err(nvme->n_dip, CE_WARN, 1791 "!SET FEATURES %d failed with sct = %x, sc = %x", 1792 feature, cmd->nc_cqe.cqe_sf.sf_sct, 1793 cmd->nc_cqe.cqe_sf.sf_sc); 1794 goto fail; 1795 } 1796 1797 *res = cmd->nc_cqe.cqe_dw0; 1798 ret = B_TRUE; 1799 1800 fail: 1801 nvme_free_cmd(cmd); 1802 return (ret); 1803 } 1804 1805 static boolean_t 1806 nvme_get_features(nvme_t *nvme, uint32_t nsid, uint8_t feature, uint32_t *res, 1807 void **buf, size_t *bufsize) 1808 { 1809 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1810 boolean_t ret = B_FALSE; 1811 1812 ASSERT(res != NULL); 1813 1814 if (bufsize != NULL) 1815 *bufsize = 0; 1816 1817 cmd->nc_sqid = 0; 1818 cmd->nc_callback = nvme_wakeup_cmd; 1819 cmd->nc_sqe.sqe_opc = NVME_OPC_GET_FEATURES; 1820 cmd->nc_sqe.sqe_cdw10 = feature; 1821 cmd->nc_sqe.sqe_cdw11 = *res; 1822 1823 switch (feature) { 1824 case NVME_FEAT_ARBITRATION: 1825 case NVME_FEAT_POWER_MGMT: 1826 case NVME_FEAT_TEMPERATURE: 1827 case NVME_FEAT_ERROR: 1828 case NVME_FEAT_NQUEUES: 1829 case NVME_FEAT_INTR_COAL: 1830 case NVME_FEAT_INTR_VECT: 1831 case NVME_FEAT_WRITE_ATOM: 1832 case NVME_FEAT_ASYNC_EVENT: 1833 case NVME_FEAT_PROGRESS: 1834 break; 1835 1836 case NVME_FEAT_WRITE_CACHE: 1837 if (!nvme->n_write_cache_present) 1838 goto fail; 1839 break; 1840 1841 case NVME_FEAT_LBA_RANGE: 1842 if (!nvme->n_lba_range_supported) 1843 goto fail; 1844 1845 /* 1846 * The LBA Range Type feature is optional. There doesn't seem 1847 * be a method of detecting whether it is supported other than 1848 * using it. This will cause a "invalid field in command" error, 1849 * which is normally considered a programming error and causes 1850 * panic in nvme_check_generic_cmd_status(). 1851 */ 1852 cmd->nc_dontpanic = B_TRUE; 1853 cmd->nc_sqe.sqe_nsid = nsid; 1854 ASSERT(bufsize != NULL); 1855 *bufsize = NVME_LBA_RANGE_BUFSIZE; 1856 1857 break; 1858 1859 case NVME_FEAT_AUTO_PST: 1860 if (!nvme->n_auto_pst_supported) 1861 goto fail; 1862 1863 ASSERT(bufsize != NULL); 1864 *bufsize = NVME_AUTO_PST_BUFSIZE; 1865 break; 1866 1867 default: 1868 goto fail; 1869 } 1870 1871 if (bufsize != NULL && *bufsize != 0) { 1872 if (nvme_zalloc_dma(nvme, *bufsize, DDI_DMA_READ, 1873 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { 1874 dev_err(nvme->n_dip, CE_WARN, 1875 "!nvme_zalloc_dma failed for GET FEATURES"); 1876 goto fail; 1877 } 1878 1879 if (cmd->nc_dma->nd_ncookie > 2) { 1880 dev_err(nvme->n_dip, CE_WARN, 1881 "!too many DMA cookies for GET FEATURES"); 1882 atomic_inc_32(&nvme->n_too_many_cookies); 1883 goto fail; 1884 } 1885 1886 cmd->nc_sqe.sqe_dptr.d_prp[0] = 1887 cmd->nc_dma->nd_cookie.dmac_laddress; 1888 if (cmd->nc_dma->nd_ncookie > 1) { 1889 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, 1890 &cmd->nc_dma->nd_cookie); 1891 cmd->nc_sqe.sqe_dptr.d_prp[1] = 1892 cmd->nc_dma->nd_cookie.dmac_laddress; 1893 } 1894 } 1895 1896 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1897 dev_err(nvme->n_dip, CE_WARN, 1898 "!nvme_admin_cmd failed for GET FEATURES"); 1899 return (ret); 1900 } 1901 1902 if (nvme_check_cmd_status(cmd)) { 1903 if (feature == NVME_FEAT_LBA_RANGE && 1904 cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && 1905 cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INV_FLD) 1906 nvme->n_lba_range_supported = B_FALSE; 1907 else 1908 dev_err(nvme->n_dip, CE_WARN, 1909 "!GET FEATURES %d failed with sct = %x, sc = %x", 1910 feature, cmd->nc_cqe.cqe_sf.sf_sct, 1911 cmd->nc_cqe.cqe_sf.sf_sc); 1912 goto fail; 1913 } 1914 1915 if (bufsize != NULL && *bufsize != 0) { 1916 ASSERT(buf != NULL); 1917 *buf = kmem_alloc(*bufsize, KM_SLEEP); 1918 bcopy(cmd->nc_dma->nd_memp, *buf, *bufsize); 1919 } 1920 1921 *res = cmd->nc_cqe.cqe_dw0; 1922 ret = B_TRUE; 1923 1924 fail: 1925 nvme_free_cmd(cmd); 1926 return (ret); 1927 } 1928 1929 static boolean_t 1930 nvme_write_cache_set(nvme_t *nvme, boolean_t enable) 1931 { 1932 nvme_write_cache_t nwc = { 0 }; 1933 1934 if (enable) 1935 nwc.b.wc_wce = 1; 1936 1937 if (!nvme_set_features(nvme, 0, NVME_FEAT_WRITE_CACHE, nwc.r, &nwc.r)) 1938 return (B_FALSE); 1939 1940 return (B_TRUE); 1941 } 1942 1943 static int 1944 nvme_set_nqueues(nvme_t *nvme, uint16_t nqueues) 1945 { 1946 nvme_nqueues_t nq = { 0 }; 1947 1948 nq.b.nq_nsq = nq.b.nq_ncq = nqueues - 1; 1949 1950 if (!nvme_set_features(nvme, 0, NVME_FEAT_NQUEUES, nq.r, &nq.r)) { 1951 return (0); 1952 } 1953 1954 /* 1955 * Always use the same number of submission and completion queues, and 1956 * never use more than the requested number of queues. 1957 */ 1958 return (MIN(nqueues, MIN(nq.b.nq_nsq, nq.b.nq_ncq) + 1)); 1959 } 1960 1961 static int 1962 nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx) 1963 { 1964 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 1965 nvme_create_queue_dw10_t dw10 = { 0 }; 1966 nvme_create_cq_dw11_t c_dw11 = { 0 }; 1967 nvme_create_sq_dw11_t s_dw11 = { 0 }; 1968 1969 dw10.b.q_qid = idx; 1970 dw10.b.q_qsize = qp->nq_nentry - 1; 1971 1972 c_dw11.b.cq_pc = 1; 1973 c_dw11.b.cq_ien = 1; 1974 c_dw11.b.cq_iv = idx % nvme->n_intr_cnt; 1975 1976 cmd->nc_sqid = 0; 1977 cmd->nc_callback = nvme_wakeup_cmd; 1978 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE; 1979 cmd->nc_sqe.sqe_cdw10 = dw10.r; 1980 cmd->nc_sqe.sqe_cdw11 = c_dw11.r; 1981 cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_cqdma->nd_cookie.dmac_laddress; 1982 1983 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 1984 dev_err(nvme->n_dip, CE_WARN, 1985 "!nvme_admin_cmd failed for CREATE CQUEUE"); 1986 return (DDI_FAILURE); 1987 } 1988 1989 if (nvme_check_cmd_status(cmd)) { 1990 dev_err(nvme->n_dip, CE_WARN, 1991 "!CREATE CQUEUE failed with sct = %x, sc = %x", 1992 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 1993 nvme_free_cmd(cmd); 1994 return (DDI_FAILURE); 1995 } 1996 1997 nvme_free_cmd(cmd); 1998 1999 s_dw11.b.sq_pc = 1; 2000 s_dw11.b.sq_cqid = idx; 2001 2002 cmd = nvme_alloc_cmd(nvme, KM_SLEEP); 2003 cmd->nc_sqid = 0; 2004 cmd->nc_callback = nvme_wakeup_cmd; 2005 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE; 2006 cmd->nc_sqe.sqe_cdw10 = dw10.r; 2007 cmd->nc_sqe.sqe_cdw11 = s_dw11.r; 2008 cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress; 2009 2010 if (nvme_admin_cmd(cmd, nvme_admin_cmd_timeout) != DDI_SUCCESS) { 2011 dev_err(nvme->n_dip, CE_WARN, 2012 "!nvme_admin_cmd failed for CREATE SQUEUE"); 2013 return (DDI_FAILURE); 2014 } 2015 2016 if (nvme_check_cmd_status(cmd)) { 2017 dev_err(nvme->n_dip, CE_WARN, 2018 "!CREATE SQUEUE failed with sct = %x, sc = %x", 2019 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); 2020 nvme_free_cmd(cmd); 2021 return (DDI_FAILURE); 2022 } 2023 2024 nvme_free_cmd(cmd); 2025 2026 return (DDI_SUCCESS); 2027 } 2028 2029 static boolean_t 2030 nvme_reset(nvme_t *nvme, boolean_t quiesce) 2031 { 2032 nvme_reg_csts_t csts; 2033 int i; 2034 2035 nvme_put32(nvme, NVME_REG_CC, 0); 2036 2037 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 2038 if (csts.b.csts_rdy == 1) { 2039 nvme_put32(nvme, NVME_REG_CC, 0); 2040 for (i = 0; i != nvme->n_timeout * 10; i++) { 2041 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 2042 if (csts.b.csts_rdy == 0) 2043 break; 2044 2045 if (quiesce) 2046 drv_usecwait(50000); 2047 else 2048 delay(drv_usectohz(50000)); 2049 } 2050 } 2051 2052 nvme_put32(nvme, NVME_REG_AQA, 0); 2053 nvme_put32(nvme, NVME_REG_ASQ, 0); 2054 nvme_put32(nvme, NVME_REG_ACQ, 0); 2055 2056 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 2057 return (csts.b.csts_rdy == 0 ? B_TRUE : B_FALSE); 2058 } 2059 2060 static void 2061 nvme_shutdown(nvme_t *nvme, int mode, boolean_t quiesce) 2062 { 2063 nvme_reg_cc_t cc; 2064 nvme_reg_csts_t csts; 2065 int i; 2066 2067 ASSERT(mode == NVME_CC_SHN_NORMAL || mode == NVME_CC_SHN_ABRUPT); 2068 2069 cc.r = nvme_get32(nvme, NVME_REG_CC); 2070 cc.b.cc_shn = mode & 0x3; 2071 nvme_put32(nvme, NVME_REG_CC, cc.r); 2072 2073 for (i = 0; i != 10; i++) { 2074 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 2075 if (csts.b.csts_shst == NVME_CSTS_SHN_COMPLETE) 2076 break; 2077 2078 if (quiesce) 2079 drv_usecwait(100000); 2080 else 2081 delay(drv_usectohz(100000)); 2082 } 2083 } 2084 2085 2086 static void 2087 nvme_prepare_devid(nvme_t *nvme, uint32_t nsid) 2088 { 2089 /* 2090 * Section 7.7 of the spec describes how to get a unique ID for 2091 * the controller: the vendor ID, the model name and the serial 2092 * number shall be unique when combined. 2093 * 2094 * If a namespace has no EUI64 we use the above and add the hex 2095 * namespace ID to get a unique ID for the namespace. 2096 */ 2097 char model[sizeof (nvme->n_idctl->id_model) + 1]; 2098 char serial[sizeof (nvme->n_idctl->id_serial) + 1]; 2099 2100 bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 2101 bcopy(nvme->n_idctl->id_serial, serial, 2102 sizeof (nvme->n_idctl->id_serial)); 2103 2104 model[sizeof (nvme->n_idctl->id_model)] = '\0'; 2105 serial[sizeof (nvme->n_idctl->id_serial)] = '\0'; 2106 2107 nvme->n_ns[nsid - 1].ns_devid = kmem_asprintf("%4X-%s-%s-%X", 2108 nvme->n_idctl->id_vid, model, serial, nsid); 2109 } 2110 2111 static int 2112 nvme_init_ns(nvme_t *nvme, int nsid) 2113 { 2114 nvme_namespace_t *ns = &nvme->n_ns[nsid - 1]; 2115 nvme_identify_nsid_t *idns; 2116 int last_rp; 2117 2118 ns->ns_nvme = nvme; 2119 idns = nvme_identify(nvme, nsid); 2120 2121 if (idns == NULL) { 2122 dev_err(nvme->n_dip, CE_WARN, 2123 "!failed to identify namespace %d", nsid); 2124 return (DDI_FAILURE); 2125 } 2126 2127 ns->ns_idns = idns; 2128 ns->ns_id = nsid; 2129 ns->ns_block_count = idns->id_nsize; 2130 ns->ns_block_size = 2131 1 << idns->id_lbaf[idns->id_flbas.lba_format].lbaf_lbads; 2132 ns->ns_best_block_size = ns->ns_block_size; 2133 2134 /* 2135 * Get the EUI64 if present. Use it for devid and device node names. 2136 */ 2137 if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) 2138 bcopy(idns->id_eui64, ns->ns_eui64, sizeof (ns->ns_eui64)); 2139 2140 /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 2141 if (*(uint64_t *)ns->ns_eui64 != 0) { 2142 uint8_t *eui64 = ns->ns_eui64; 2143 2144 (void) snprintf(ns->ns_name, sizeof (ns->ns_name), 2145 "%02x%02x%02x%02x%02x%02x%02x%02x", 2146 eui64[0], eui64[1], eui64[2], eui64[3], 2147 eui64[4], eui64[5], eui64[6], eui64[7]); 2148 } else { 2149 (void) snprintf(ns->ns_name, sizeof (ns->ns_name), "%d", 2150 ns->ns_id); 2151 2152 nvme_prepare_devid(nvme, ns->ns_id); 2153 } 2154 2155 /* 2156 * Find the LBA format with no metadata and the best relative 2157 * performance. A value of 3 means "degraded", 0 is best. 2158 */ 2159 last_rp = 3; 2160 for (int j = 0; j <= idns->id_nlbaf; j++) { 2161 if (idns->id_lbaf[j].lbaf_lbads == 0) 2162 break; 2163 if (idns->id_lbaf[j].lbaf_ms != 0) 2164 continue; 2165 if (idns->id_lbaf[j].lbaf_rp >= last_rp) 2166 continue; 2167 last_rp = idns->id_lbaf[j].lbaf_rp; 2168 ns->ns_best_block_size = 2169 1 << idns->id_lbaf[j].lbaf_lbads; 2170 } 2171 2172 if (ns->ns_best_block_size < nvme->n_min_block_size) 2173 ns->ns_best_block_size = nvme->n_min_block_size; 2174 2175 /* 2176 * We currently don't support namespaces that use either: 2177 * - thin provisioning 2178 * - protection information 2179 */ 2180 if (idns->id_nsfeat.f_thin || 2181 idns->id_dps.dp_pinfo) { 2182 dev_err(nvme->n_dip, CE_WARN, 2183 "!ignoring namespace %d, unsupported features: " 2184 "thin = %d, pinfo = %d", nsid, 2185 idns->id_nsfeat.f_thin, idns->id_dps.dp_pinfo); 2186 ns->ns_ignore = B_TRUE; 2187 } else { 2188 ns->ns_ignore = B_FALSE; 2189 } 2190 2191 return (DDI_SUCCESS); 2192 } 2193 2194 static int 2195 nvme_init(nvme_t *nvme) 2196 { 2197 nvme_reg_cc_t cc = { 0 }; 2198 nvme_reg_aqa_t aqa = { 0 }; 2199 nvme_reg_asq_t asq = { 0 }; 2200 nvme_reg_acq_t acq = { 0 }; 2201 nvme_reg_cap_t cap; 2202 nvme_reg_vs_t vs; 2203 nvme_reg_csts_t csts; 2204 int i = 0; 2205 int nqueues; 2206 char model[sizeof (nvme->n_idctl->id_model) + 1]; 2207 char *vendor, *product; 2208 2209 /* Check controller version */ 2210 vs.r = nvme_get32(nvme, NVME_REG_VS); 2211 nvme->n_version.v_major = vs.b.vs_mjr; 2212 nvme->n_version.v_minor = vs.b.vs_mnr; 2213 dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d", 2214 nvme->n_version.v_major, nvme->n_version.v_minor); 2215 2216 if (NVME_VERSION_HIGHER(&nvme->n_version, 2217 nvme_version_major, nvme_version_minor)) { 2218 dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.%d", 2219 nvme_version_major, nvme_version_minor); 2220 if (nvme->n_strict_version) 2221 goto fail; 2222 } 2223 2224 /* retrieve controller configuration */ 2225 cap.r = nvme_get64(nvme, NVME_REG_CAP); 2226 2227 if ((cap.b.cap_css & NVME_CAP_CSS_NVM) == 0) { 2228 dev_err(nvme->n_dip, CE_WARN, 2229 "!NVM command set not supported by hardware"); 2230 goto fail; 2231 } 2232 2233 nvme->n_nssr_supported = cap.b.cap_nssrs; 2234 nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd; 2235 nvme->n_timeout = cap.b.cap_to; 2236 nvme->n_arbitration_mechanisms = cap.b.cap_ams; 2237 nvme->n_cont_queues_reqd = cap.b.cap_cqr; 2238 nvme->n_max_queue_entries = cap.b.cap_mqes + 1; 2239 2240 /* 2241 * The MPSMIN and MPSMAX fields in the CAP register use 0 to specify 2242 * the base page size of 4k (1<<12), so add 12 here to get the real 2243 * page size value. 2244 */ 2245 nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT), 2246 cap.b.cap_mpsmax + 12); 2247 nvme->n_pagesize = 1UL << (nvme->n_pageshift); 2248 2249 /* 2250 * Set up Queue DMA to transfer at least 1 page-aligned page at a time. 2251 */ 2252 nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize; 2253 nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 2254 2255 /* 2256 * Set up PRP DMA to transfer 1 page-aligned page at a time. 2257 * Maxxfer may be increased after we identified the controller limits. 2258 */ 2259 nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize; 2260 nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize; 2261 nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize; 2262 nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1; 2263 2264 /* 2265 * Reset controller if it's still in ready state. 2266 */ 2267 if (nvme_reset(nvme, B_FALSE) == B_FALSE) { 2268 dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller"); 2269 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 2270 nvme->n_dead = B_TRUE; 2271 goto fail; 2272 } 2273 2274 /* 2275 * Create the admin queue pair. 2276 */ 2277 if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0) 2278 != DDI_SUCCESS) { 2279 dev_err(nvme->n_dip, CE_WARN, 2280 "!unable to allocate admin qpair"); 2281 goto fail; 2282 } 2283 nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP); 2284 nvme->n_ioq[0] = nvme->n_adminq; 2285 2286 nvme->n_progress |= NVME_ADMIN_QUEUE; 2287 2288 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2289 "admin-queue-len", nvme->n_admin_queue_len); 2290 2291 aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1; 2292 asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress; 2293 acq = nvme->n_adminq->nq_cqdma->nd_cookie.dmac_laddress; 2294 2295 ASSERT((asq & (nvme->n_pagesize - 1)) == 0); 2296 ASSERT((acq & (nvme->n_pagesize - 1)) == 0); 2297 2298 nvme_put32(nvme, NVME_REG_AQA, aqa.r); 2299 nvme_put64(nvme, NVME_REG_ASQ, asq); 2300 nvme_put64(nvme, NVME_REG_ACQ, acq); 2301 2302 cc.b.cc_ams = 0; /* use Round-Robin arbitration */ 2303 cc.b.cc_css = 0; /* use NVM command set */ 2304 cc.b.cc_mps = nvme->n_pageshift - 12; 2305 cc.b.cc_shn = 0; /* no shutdown in progress */ 2306 cc.b.cc_en = 1; /* enable controller */ 2307 cc.b.cc_iosqes = 6; /* submission queue entry is 2^6 bytes long */ 2308 cc.b.cc_iocqes = 4; /* completion queue entry is 2^4 bytes long */ 2309 2310 nvme_put32(nvme, NVME_REG_CC, cc.r); 2311 2312 /* 2313 * Wait for the controller to become ready. 2314 */ 2315 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 2316 if (csts.b.csts_rdy == 0) { 2317 for (i = 0; i != nvme->n_timeout * 10; i++) { 2318 delay(drv_usectohz(50000)); 2319 csts.r = nvme_get32(nvme, NVME_REG_CSTS); 2320 2321 if (csts.b.csts_cfs == 1) { 2322 dev_err(nvme->n_dip, CE_WARN, 2323 "!controller fatal status at init"); 2324 ddi_fm_service_impact(nvme->n_dip, 2325 DDI_SERVICE_LOST); 2326 nvme->n_dead = B_TRUE; 2327 goto fail; 2328 } 2329 2330 if (csts.b.csts_rdy == 1) 2331 break; 2332 } 2333 } 2334 2335 if (csts.b.csts_rdy == 0) { 2336 dev_err(nvme->n_dip, CE_WARN, "!controller not ready"); 2337 ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST); 2338 nvme->n_dead = B_TRUE; 2339 goto fail; 2340 } 2341 2342 /* 2343 * Assume an abort command limit of 1. We'll destroy and re-init 2344 * that later when we know the true abort command limit. 2345 */ 2346 sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL); 2347 2348 /* 2349 * Setup initial interrupt for admin queue. 2350 */ 2351 if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1) 2352 != DDI_SUCCESS) && 2353 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1) 2354 != DDI_SUCCESS) && 2355 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1) 2356 != DDI_SUCCESS)) { 2357 dev_err(nvme->n_dip, CE_WARN, 2358 "!failed to setup initial interrupt"); 2359 goto fail; 2360 } 2361 2362 /* 2363 * Post an asynchronous event command to catch errors. 2364 */ 2365 if (nvme_async_event(nvme) != DDI_SUCCESS) { 2366 dev_err(nvme->n_dip, CE_WARN, 2367 "!failed to post async event"); 2368 goto fail; 2369 } 2370 2371 /* 2372 * Identify Controller 2373 */ 2374 nvme->n_idctl = nvme_identify(nvme, 0); 2375 if (nvme->n_idctl == NULL) { 2376 dev_err(nvme->n_dip, CE_WARN, 2377 "!failed to identify controller"); 2378 goto fail; 2379 } 2380 2381 /* 2382 * Get Vendor & Product ID 2383 */ 2384 bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model)); 2385 model[sizeof (nvme->n_idctl->id_model)] = '\0'; 2386 sata_split_model(model, &vendor, &product); 2387 2388 if (vendor == NULL) 2389 nvme->n_vendor = strdup("NVMe"); 2390 else 2391 nvme->n_vendor = strdup(vendor); 2392 2393 nvme->n_product = strdup(product); 2394 2395 /* 2396 * Get controller limits. 2397 */ 2398 nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT, 2399 MIN(nvme->n_admin_queue_len / 10, 2400 MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit))); 2401 2402 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2403 "async-event-limit", nvme->n_async_event_limit); 2404 2405 nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1; 2406 2407 /* 2408 * Reinitialize the semaphore with the true abort command limit 2409 * supported by the hardware. It's not necessary to disable interrupts 2410 * as only command aborts use the semaphore, and no commands are 2411 * executed or aborted while we're here. 2412 */ 2413 sema_destroy(&nvme->n_abort_sema); 2414 sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL, 2415 SEMA_DRIVER, NULL); 2416 2417 nvme->n_progress |= NVME_CTRL_LIMITS; 2418 2419 if (nvme->n_idctl->id_mdts == 0) 2420 nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536; 2421 else 2422 nvme->n_max_data_transfer_size = 2423 1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts); 2424 2425 nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1; 2426 2427 /* 2428 * Limit n_max_data_transfer_size to what we can handle in one PRP. 2429 * Chained PRPs are currently unsupported. 2430 * 2431 * This is a no-op on hardware which doesn't support a transfer size 2432 * big enough to require chained PRPs. 2433 */ 2434 nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size, 2435 (nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize)); 2436 2437 nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size; 2438 2439 /* 2440 * Make sure the minimum/maximum queue entry sizes are not 2441 * larger/smaller than the default. 2442 */ 2443 2444 if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) || 2445 ((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) || 2446 ((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) || 2447 ((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t))) 2448 goto fail; 2449 2450 /* 2451 * Check for the presence of a Volatile Write Cache. If present, 2452 * enable or disable based on the value of the property 2453 * volatile-write-cache-enable (default is enabled). 2454 */ 2455 nvme->n_write_cache_present = 2456 nvme->n_idctl->id_vwc.vwc_present == 0 ? B_FALSE : B_TRUE; 2457 2458 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2459 "volatile-write-cache-present", 2460 nvme->n_write_cache_present ? 1 : 0); 2461 2462 if (!nvme->n_write_cache_present) { 2463 nvme->n_write_cache_enabled = B_FALSE; 2464 } else if (!nvme_write_cache_set(nvme, nvme->n_write_cache_enabled)) { 2465 dev_err(nvme->n_dip, CE_WARN, 2466 "!failed to %sable volatile write cache", 2467 nvme->n_write_cache_enabled ? "en" : "dis"); 2468 /* 2469 * Assume the cache is (still) enabled. 2470 */ 2471 nvme->n_write_cache_enabled = B_TRUE; 2472 } 2473 2474 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, 2475 "volatile-write-cache-enable", 2476 nvme->n_write_cache_enabled ? 1 : 0); 2477 2478 /* 2479 * Assume LBA Range Type feature is supported. If it isn't this 2480 * will be set to B_FALSE by nvme_get_features(). 2481 */ 2482 nvme->n_lba_range_supported = B_TRUE; 2483 2484 /* 2485 * Check support for Autonomous Power State Transition. 2486 */ 2487 if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) 2488 nvme->n_auto_pst_supported = 2489 nvme->n_idctl->id_apsta.ap_sup == 0 ? B_FALSE : B_TRUE; 2490 2491 /* 2492 * Identify Namespaces 2493 */ 2494 nvme->n_namespace_count = nvme->n_idctl->id_nn; 2495 if (nvme->n_namespace_count > NVME_MINOR_MAX) { 2496 dev_err(nvme->n_dip, CE_WARN, 2497 "!too many namespaces: %d, limiting to %d\n", 2498 nvme->n_namespace_count, NVME_MINOR_MAX); 2499 nvme->n_namespace_count = NVME_MINOR_MAX; 2500 } 2501 2502 nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) * 2503 nvme->n_namespace_count, KM_SLEEP); 2504 2505 for (i = 0; i != nvme->n_namespace_count; i++) { 2506 mutex_init(&nvme->n_ns[i].ns_minor.nm_mutex, NULL, MUTEX_DRIVER, 2507 NULL); 2508 if (nvme_init_ns(nvme, i + 1) != DDI_SUCCESS) 2509 goto fail; 2510 } 2511 2512 /* 2513 * Try to set up MSI/MSI-X interrupts. 2514 */ 2515 if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX)) 2516 != 0) { 2517 nvme_release_interrupts(nvme); 2518 2519 nqueues = MIN(UINT16_MAX, ncpus); 2520 2521 if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 2522 nqueues) != DDI_SUCCESS) && 2523 (nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 2524 nqueues) != DDI_SUCCESS)) { 2525 dev_err(nvme->n_dip, CE_WARN, 2526 "!failed to setup MSI/MSI-X interrupts"); 2527 goto fail; 2528 } 2529 } 2530 2531 nqueues = nvme->n_intr_cnt; 2532 2533 /* 2534 * Create I/O queue pairs. 2535 */ 2536 nvme->n_ioq_count = nvme_set_nqueues(nvme, nqueues); 2537 if (nvme->n_ioq_count == 0) { 2538 dev_err(nvme->n_dip, CE_WARN, 2539 "!failed to set number of I/O queues to %d", nqueues); 2540 goto fail; 2541 } 2542 2543 /* 2544 * Reallocate I/O queue array 2545 */ 2546 kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *)); 2547 nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) * 2548 (nvme->n_ioq_count + 1), KM_SLEEP); 2549 nvme->n_ioq[0] = nvme->n_adminq; 2550 2551 /* 2552 * If we got less queues than we asked for we might as well give 2553 * some of the interrupt vectors back to the system. 2554 */ 2555 if (nvme->n_ioq_count < nqueues) { 2556 nvme_release_interrupts(nvme); 2557 2558 if (nvme_setup_interrupts(nvme, nvme->n_intr_type, 2559 nvme->n_ioq_count) != DDI_SUCCESS) { 2560 dev_err(nvme->n_dip, CE_WARN, 2561 "!failed to reduce number of interrupts"); 2562 goto fail; 2563 } 2564 } 2565 2566 /* 2567 * Alloc & register I/O queue pairs 2568 */ 2569 nvme->n_io_queue_len = 2570 MIN(nvme->n_io_queue_len, nvme->n_max_queue_entries); 2571 (void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-queue-len", 2572 nvme->n_io_queue_len); 2573 2574 for (i = 1; i != nvme->n_ioq_count + 1; i++) { 2575 if (nvme_alloc_qpair(nvme, nvme->n_io_queue_len, 2576 &nvme->n_ioq[i], i) != DDI_SUCCESS) { 2577 dev_err(nvme->n_dip, CE_WARN, 2578 "!unable to allocate I/O qpair %d", i); 2579 goto fail; 2580 } 2581 2582 if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i) 2583 != DDI_SUCCESS) { 2584 dev_err(nvme->n_dip, CE_WARN, 2585 "!unable to create I/O qpair %d", i); 2586 goto fail; 2587 } 2588 } 2589 2590 /* 2591 * Post more asynchronous events commands to reduce event reporting 2592 * latency as suggested by the spec. 2593 */ 2594 for (i = 1; i != nvme->n_async_event_limit; i++) { 2595 if (nvme_async_event(nvme) != DDI_SUCCESS) { 2596 dev_err(nvme->n_dip, CE_WARN, 2597 "!failed to post async event %d", i); 2598 goto fail; 2599 } 2600 } 2601 2602 return (DDI_SUCCESS); 2603 2604 fail: 2605 (void) nvme_reset(nvme, B_FALSE); 2606 return (DDI_FAILURE); 2607 } 2608 2609 static uint_t 2610 nvme_intr(caddr_t arg1, caddr_t arg2) 2611 { 2612 /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 2613 nvme_t *nvme = (nvme_t *)arg1; 2614 int inum = (int)(uintptr_t)arg2; 2615 int ccnt = 0; 2616 int qnum; 2617 nvme_cmd_t *cmd; 2618 2619 if (inum >= nvme->n_intr_cnt) 2620 return (DDI_INTR_UNCLAIMED); 2621 2622 /* 2623 * The interrupt vector a queue uses is calculated as queue_idx % 2624 * intr_cnt in nvme_create_io_qpair(). Iterate through the queue array 2625 * in steps of n_intr_cnt to process all queues using this vector. 2626 */ 2627 for (qnum = inum; 2628 qnum < nvme->n_ioq_count + 1 && nvme->n_ioq[qnum] != NULL; 2629 qnum += nvme->n_intr_cnt) { 2630 while ((cmd = nvme_retrieve_cmd(nvme, nvme->n_ioq[qnum]))) { 2631 taskq_dispatch_ent((taskq_t *)cmd->nc_nvme->n_cmd_taskq, 2632 cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent); 2633 ccnt++; 2634 } 2635 } 2636 2637 return (ccnt > 0 ? DDI_INTR_CLAIMED : DDI_INTR_UNCLAIMED); 2638 } 2639 2640 static void 2641 nvme_release_interrupts(nvme_t *nvme) 2642 { 2643 int i; 2644 2645 for (i = 0; i < nvme->n_intr_cnt; i++) { 2646 if (nvme->n_inth[i] == NULL) 2647 break; 2648 2649 if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 2650 (void) ddi_intr_block_disable(&nvme->n_inth[i], 1); 2651 else 2652 (void) ddi_intr_disable(nvme->n_inth[i]); 2653 2654 (void) ddi_intr_remove_handler(nvme->n_inth[i]); 2655 (void) ddi_intr_free(nvme->n_inth[i]); 2656 } 2657 2658 kmem_free(nvme->n_inth, nvme->n_inth_sz); 2659 nvme->n_inth = NULL; 2660 nvme->n_inth_sz = 0; 2661 2662 nvme->n_progress &= ~NVME_INTERRUPTS; 2663 } 2664 2665 static int 2666 nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs) 2667 { 2668 int nintrs, navail, count; 2669 int ret; 2670 int i; 2671 2672 if (nvme->n_intr_types == 0) { 2673 ret = ddi_intr_get_supported_types(nvme->n_dip, 2674 &nvme->n_intr_types); 2675 if (ret != DDI_SUCCESS) { 2676 dev_err(nvme->n_dip, CE_WARN, 2677 "!%s: ddi_intr_get_supported types failed", 2678 __func__); 2679 return (ret); 2680 } 2681 #ifdef __x86 2682 if (get_hwenv() == HW_VMWARE) 2683 nvme->n_intr_types &= ~DDI_INTR_TYPE_MSIX; 2684 #endif 2685 } 2686 2687 if ((nvme->n_intr_types & intr_type) == 0) 2688 return (DDI_FAILURE); 2689 2690 ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs); 2691 if (ret != DDI_SUCCESS) { 2692 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed", 2693 __func__); 2694 return (ret); 2695 } 2696 2697 ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail); 2698 if (ret != DDI_SUCCESS) { 2699 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed", 2700 __func__); 2701 return (ret); 2702 } 2703 2704 /* We want at most one interrupt per queue pair. */ 2705 if (navail > nqpairs) 2706 navail = nqpairs; 2707 2708 nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail; 2709 nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP); 2710 2711 ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail, 2712 &count, 0); 2713 if (ret != DDI_SUCCESS) { 2714 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed", 2715 __func__); 2716 goto fail; 2717 } 2718 2719 nvme->n_intr_cnt = count; 2720 2721 ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri); 2722 if (ret != DDI_SUCCESS) { 2723 dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed", 2724 __func__); 2725 goto fail; 2726 } 2727 2728 for (i = 0; i < count; i++) { 2729 ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr, 2730 (void *)nvme, (void *)(uintptr_t)i); 2731 if (ret != DDI_SUCCESS) { 2732 dev_err(nvme->n_dip, CE_WARN, 2733 "!%s: ddi_intr_add_handler failed", __func__); 2734 goto fail; 2735 } 2736 } 2737 2738 (void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap); 2739 2740 for (i = 0; i < count; i++) { 2741 if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK) 2742 ret = ddi_intr_block_enable(&nvme->n_inth[i], 1); 2743 else 2744 ret = ddi_intr_enable(nvme->n_inth[i]); 2745 2746 if (ret != DDI_SUCCESS) { 2747 dev_err(nvme->n_dip, CE_WARN, 2748 "!%s: enabling interrupt %d failed", __func__, i); 2749 goto fail; 2750 } 2751 } 2752 2753 nvme->n_intr_type = intr_type; 2754 2755 nvme->n_progress |= NVME_INTERRUPTS; 2756 2757 return (DDI_SUCCESS); 2758 2759 fail: 2760 nvme_release_interrupts(nvme); 2761 2762 return (ret); 2763 } 2764 2765 static int 2766 nvme_fm_errcb(dev_info_t *dip, ddi_fm_error_t *fm_error, const void *arg) 2767 { 2768 _NOTE(ARGUNUSED(arg)); 2769 2770 pci_ereport_post(dip, fm_error, NULL); 2771 return (fm_error->fme_status); 2772 } 2773 2774 static int 2775 nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 2776 { 2777 nvme_t *nvme; 2778 int instance; 2779 int nregs; 2780 off_t regsize; 2781 int i; 2782 char name[32]; 2783 2784 if (cmd != DDI_ATTACH) 2785 return (DDI_FAILURE); 2786 2787 instance = ddi_get_instance(dip); 2788 2789 if (ddi_soft_state_zalloc(nvme_state, instance) != DDI_SUCCESS) 2790 return (DDI_FAILURE); 2791 2792 nvme = ddi_get_soft_state(nvme_state, instance); 2793 ddi_set_driver_private(dip, nvme); 2794 nvme->n_dip = dip; 2795 2796 mutex_init(&nvme->n_minor.nm_mutex, NULL, MUTEX_DRIVER, NULL); 2797 2798 nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2799 DDI_PROP_DONTPASS, "strict-version", 1) == 1 ? B_TRUE : B_FALSE; 2800 nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY, 2801 dip, DDI_PROP_DONTPASS, "ignore-unknown-vendor-status", 0) == 1 ? 2802 B_TRUE : B_FALSE; 2803 nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2804 DDI_PROP_DONTPASS, "admin-queue-len", NVME_DEFAULT_ADMIN_QUEUE_LEN); 2805 nvme->n_io_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2806 DDI_PROP_DONTPASS, "io-queue-len", NVME_DEFAULT_IO_QUEUE_LEN); 2807 nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2808 DDI_PROP_DONTPASS, "async-event-limit", 2809 NVME_DEFAULT_ASYNC_EVENT_LIMIT); 2810 nvme->n_write_cache_enabled = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2811 DDI_PROP_DONTPASS, "volatile-write-cache-enable", 1) != 0 ? 2812 B_TRUE : B_FALSE; 2813 nvme->n_min_block_size = ddi_prop_get_int(DDI_DEV_T_ANY, dip, 2814 DDI_PROP_DONTPASS, "min-phys-block-size", 2815 NVME_DEFAULT_MIN_BLOCK_SIZE); 2816 2817 if (!ISP2(nvme->n_min_block_size) || 2818 (nvme->n_min_block_size < NVME_DEFAULT_MIN_BLOCK_SIZE)) { 2819 dev_err(dip, CE_WARN, "!min-phys-block-size %s, " 2820 "using default %d", ISP2(nvme->n_min_block_size) ? 2821 "too low" : "not a power of 2", 2822 NVME_DEFAULT_MIN_BLOCK_SIZE); 2823 nvme->n_min_block_size = NVME_DEFAULT_MIN_BLOCK_SIZE; 2824 } 2825 2826 if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN) 2827 nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN; 2828 else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN) 2829 nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN; 2830 2831 if (nvme->n_io_queue_len < NVME_MIN_IO_QUEUE_LEN) 2832 nvme->n_io_queue_len = NVME_MIN_IO_QUEUE_LEN; 2833 2834 if (nvme->n_async_event_limit < 1) 2835 nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT; 2836 2837 nvme->n_reg_acc_attr = nvme_reg_acc_attr; 2838 nvme->n_queue_dma_attr = nvme_queue_dma_attr; 2839 nvme->n_prp_dma_attr = nvme_prp_dma_attr; 2840 nvme->n_sgl_dma_attr = nvme_sgl_dma_attr; 2841 2842 /* 2843 * Setup FMA support. 2844 */ 2845 nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip, 2846 DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable", 2847 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 2848 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 2849 2850 ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc); 2851 2852 if (nvme->n_fm_cap) { 2853 if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE) 2854 nvme->n_reg_acc_attr.devacc_attr_access = 2855 DDI_FLAGERR_ACC; 2856 2857 if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) { 2858 nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 2859 nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR; 2860 } 2861 2862 if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 2863 DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 2864 pci_ereport_setup(dip); 2865 2866 if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 2867 ddi_fm_handler_register(dip, nvme_fm_errcb, 2868 (void *)nvme); 2869 } 2870 2871 nvme->n_progress |= NVME_FMA_INIT; 2872 2873 /* 2874 * The spec defines several register sets. Only the controller 2875 * registers (set 1) are currently used. 2876 */ 2877 if (ddi_dev_nregs(dip, &nregs) == DDI_FAILURE || 2878 nregs < 2 || 2879 ddi_dev_regsize(dip, 1, ®size) == DDI_FAILURE) 2880 goto fail; 2881 2882 if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize, 2883 &nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) { 2884 dev_err(dip, CE_WARN, "!failed to map regset 1"); 2885 goto fail; 2886 } 2887 2888 nvme->n_progress |= NVME_REGS_MAPPED; 2889 2890 /* 2891 * Create taskq for command completion. 2892 */ 2893 (void) snprintf(name, sizeof (name), "%s%d_cmd_taskq", 2894 ddi_driver_name(dip), ddi_get_instance(dip)); 2895 nvme->n_cmd_taskq = ddi_taskq_create(dip, name, MIN(UINT16_MAX, ncpus), 2896 TASKQ_DEFAULTPRI, 0); 2897 if (nvme->n_cmd_taskq == NULL) { 2898 dev_err(dip, CE_WARN, "!failed to create cmd taskq"); 2899 goto fail; 2900 } 2901 2902 /* 2903 * Create PRP DMA cache 2904 */ 2905 (void) snprintf(name, sizeof (name), "%s%d_prp_cache", 2906 ddi_driver_name(dip), ddi_get_instance(dip)); 2907 nvme->n_prp_cache = kmem_cache_create(name, sizeof (nvme_dma_t), 2908 0, nvme_prp_dma_constructor, nvme_prp_dma_destructor, 2909 NULL, (void *)nvme, NULL, 0); 2910 2911 if (nvme_init(nvme) != DDI_SUCCESS) 2912 goto fail; 2913 2914 /* 2915 * Attach the blkdev driver for each namespace. 2916 */ 2917 for (i = 0; i != nvme->n_namespace_count; i++) { 2918 if (ddi_create_minor_node(nvme->n_dip, nvme->n_ns[i].ns_name, 2919 S_IFCHR, NVME_MINOR(ddi_get_instance(nvme->n_dip), i + 1), 2920 DDI_NT_NVME_ATTACHMENT_POINT, 0) != DDI_SUCCESS) { 2921 dev_err(dip, CE_WARN, 2922 "!failed to create minor node for namespace %d", i); 2923 goto fail; 2924 } 2925 2926 if (nvme->n_ns[i].ns_ignore) 2927 continue; 2928 2929 nvme->n_ns[i].ns_bd_hdl = bd_alloc_handle(&nvme->n_ns[i], 2930 &nvme_bd_ops, &nvme->n_prp_dma_attr, KM_SLEEP); 2931 2932 if (nvme->n_ns[i].ns_bd_hdl == NULL) { 2933 dev_err(dip, CE_WARN, 2934 "!failed to get blkdev handle for namespace %d", i); 2935 goto fail; 2936 } 2937 2938 if (bd_attach_handle(dip, nvme->n_ns[i].ns_bd_hdl) 2939 != DDI_SUCCESS) { 2940 dev_err(dip, CE_WARN, 2941 "!failed to attach blkdev handle for namespace %d", 2942 i); 2943 goto fail; 2944 } 2945 } 2946 2947 if (ddi_create_minor_node(dip, "devctl", S_IFCHR, 2948 NVME_MINOR(ddi_get_instance(dip), 0), DDI_NT_NVME_NEXUS, 0) 2949 != DDI_SUCCESS) { 2950 dev_err(dip, CE_WARN, "nvme_attach: " 2951 "cannot create devctl minor node"); 2952 goto fail; 2953 } 2954 2955 return (DDI_SUCCESS); 2956 2957 fail: 2958 /* attach successful anyway so that FMA can retire the device */ 2959 if (nvme->n_dead) 2960 return (DDI_SUCCESS); 2961 2962 (void) nvme_detach(dip, DDI_DETACH); 2963 2964 return (DDI_FAILURE); 2965 } 2966 2967 static int 2968 nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 2969 { 2970 int instance, i; 2971 nvme_t *nvme; 2972 2973 if (cmd != DDI_DETACH) 2974 return (DDI_FAILURE); 2975 2976 instance = ddi_get_instance(dip); 2977 2978 nvme = ddi_get_soft_state(nvme_state, instance); 2979 2980 if (nvme == NULL) 2981 return (DDI_FAILURE); 2982 2983 ddi_remove_minor_node(dip, "devctl"); 2984 mutex_destroy(&nvme->n_minor.nm_mutex); 2985 2986 if (nvme->n_ns) { 2987 for (i = 0; i != nvme->n_namespace_count; i++) { 2988 ddi_remove_minor_node(dip, nvme->n_ns[i].ns_name); 2989 mutex_destroy(&nvme->n_ns[i].ns_minor.nm_mutex); 2990 2991 if (nvme->n_ns[i].ns_bd_hdl) { 2992 (void) bd_detach_handle( 2993 nvme->n_ns[i].ns_bd_hdl); 2994 bd_free_handle(nvme->n_ns[i].ns_bd_hdl); 2995 } 2996 2997 if (nvme->n_ns[i].ns_idns) 2998 kmem_free(nvme->n_ns[i].ns_idns, 2999 sizeof (nvme_identify_nsid_t)); 3000 if (nvme->n_ns[i].ns_devid) 3001 strfree(nvme->n_ns[i].ns_devid); 3002 } 3003 3004 kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) * 3005 nvme->n_namespace_count); 3006 } 3007 3008 if (nvme->n_progress & NVME_INTERRUPTS) 3009 nvme_release_interrupts(nvme); 3010 3011 if (nvme->n_cmd_taskq) 3012 ddi_taskq_wait(nvme->n_cmd_taskq); 3013 3014 if (nvme->n_ioq_count > 0) { 3015 for (i = 1; i != nvme->n_ioq_count + 1; i++) { 3016 if (nvme->n_ioq[i] != NULL) { 3017 /* TODO: send destroy queue commands */ 3018 nvme_free_qpair(nvme->n_ioq[i]); 3019 } 3020 } 3021 3022 kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) * 3023 (nvme->n_ioq_count + 1)); 3024 } 3025 3026 if (nvme->n_prp_cache != NULL) { 3027 kmem_cache_destroy(nvme->n_prp_cache); 3028 } 3029 3030 if (nvme->n_progress & NVME_REGS_MAPPED) { 3031 nvme_shutdown(nvme, NVME_CC_SHN_NORMAL, B_FALSE); 3032 (void) nvme_reset(nvme, B_FALSE); 3033 } 3034 3035 if (nvme->n_cmd_taskq) 3036 ddi_taskq_destroy(nvme->n_cmd_taskq); 3037 3038 if (nvme->n_progress & NVME_CTRL_LIMITS) 3039 sema_destroy(&nvme->n_abort_sema); 3040 3041 if (nvme->n_progress & NVME_ADMIN_QUEUE) 3042 nvme_free_qpair(nvme->n_adminq); 3043 3044 if (nvme->n_idctl) 3045 kmem_free(nvme->n_idctl, NVME_IDENTIFY_BUFSIZE); 3046 3047 if (nvme->n_progress & NVME_REGS_MAPPED) 3048 ddi_regs_map_free(&nvme->n_regh); 3049 3050 if (nvme->n_progress & NVME_FMA_INIT) { 3051 if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 3052 ddi_fm_handler_unregister(nvme->n_dip); 3053 3054 if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) || 3055 DDI_FM_ERRCB_CAP(nvme->n_fm_cap)) 3056 pci_ereport_teardown(nvme->n_dip); 3057 3058 ddi_fm_fini(nvme->n_dip); 3059 } 3060 3061 if (nvme->n_vendor != NULL) 3062 strfree(nvme->n_vendor); 3063 3064 if (nvme->n_product != NULL) 3065 strfree(nvme->n_product); 3066 3067 ddi_soft_state_free(nvme_state, instance); 3068 3069 return (DDI_SUCCESS); 3070 } 3071 3072 static int 3073 nvme_quiesce(dev_info_t *dip) 3074 { 3075 int instance; 3076 nvme_t *nvme; 3077 3078 instance = ddi_get_instance(dip); 3079 3080 nvme = ddi_get_soft_state(nvme_state, instance); 3081 3082 if (nvme == NULL) 3083 return (DDI_FAILURE); 3084 3085 nvme_shutdown(nvme, NVME_CC_SHN_ABRUPT, B_TRUE); 3086 3087 (void) nvme_reset(nvme, B_TRUE); 3088 3089 return (DDI_FAILURE); 3090 } 3091 3092 static int 3093 nvme_fill_prp(nvme_cmd_t *cmd, bd_xfer_t *xfer) 3094 { 3095 nvme_t *nvme = cmd->nc_nvme; 3096 int nprp_page, nprp; 3097 uint64_t *prp; 3098 3099 if (xfer->x_ndmac == 0) 3100 return (DDI_FAILURE); 3101 3102 cmd->nc_sqe.sqe_dptr.d_prp[0] = xfer->x_dmac.dmac_laddress; 3103 ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 3104 3105 if (xfer->x_ndmac == 1) { 3106 cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; 3107 return (DDI_SUCCESS); 3108 } else if (xfer->x_ndmac == 2) { 3109 cmd->nc_sqe.sqe_dptr.d_prp[1] = xfer->x_dmac.dmac_laddress; 3110 return (DDI_SUCCESS); 3111 } 3112 3113 xfer->x_ndmac--; 3114 3115 nprp_page = nvme->n_pagesize / sizeof (uint64_t) - 1; 3116 ASSERT(nprp_page > 0); 3117 nprp = (xfer->x_ndmac + nprp_page - 1) / nprp_page; 3118 3119 /* 3120 * We currently don't support chained PRPs and set up our DMA 3121 * attributes to reflect that. If we still get an I/O request 3122 * that needs a chained PRP something is very wrong. 3123 */ 3124 VERIFY(nprp == 1); 3125 3126 cmd->nc_dma = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP); 3127 bzero(cmd->nc_dma->nd_memp, cmd->nc_dma->nd_len); 3128 3129 cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_dma->nd_cookie.dmac_laddress; 3130 3131 /*LINTED: E_PTR_BAD_CAST_ALIGN*/ 3132 for (prp = (uint64_t *)cmd->nc_dma->nd_memp; 3133 xfer->x_ndmac > 0; 3134 prp++, xfer->x_ndmac--) { 3135 *prp = xfer->x_dmac.dmac_laddress; 3136 ddi_dma_nextcookie(xfer->x_dmah, &xfer->x_dmac); 3137 } 3138 3139 (void) ddi_dma_sync(cmd->nc_dma->nd_dmah, 0, cmd->nc_dma->nd_len, 3140 DDI_DMA_SYNC_FORDEV); 3141 return (DDI_SUCCESS); 3142 } 3143 3144 static nvme_cmd_t * 3145 nvme_create_nvm_cmd(nvme_namespace_t *ns, uint8_t opc, bd_xfer_t *xfer) 3146 { 3147 nvme_t *nvme = ns->ns_nvme; 3148 nvme_cmd_t *cmd; 3149 3150 /* 3151 * Blkdev only sets BD_XFER_POLL when dumping, so don't sleep. 3152 */ 3153 cmd = nvme_alloc_cmd(nvme, (xfer->x_flags & BD_XFER_POLL) ? 3154 KM_NOSLEEP : KM_SLEEP); 3155 3156 if (cmd == NULL) 3157 return (NULL); 3158 3159 cmd->nc_sqe.sqe_opc = opc; 3160 cmd->nc_callback = nvme_bd_xfer_done; 3161 cmd->nc_xfer = xfer; 3162 3163 switch (opc) { 3164 case NVME_OPC_NVM_WRITE: 3165 case NVME_OPC_NVM_READ: 3166 VERIFY(xfer->x_nblks <= 0x10000); 3167 3168 cmd->nc_sqe.sqe_nsid = ns->ns_id; 3169 3170 cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu; 3171 cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32); 3172 cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1); 3173 3174 if (nvme_fill_prp(cmd, xfer) != DDI_SUCCESS) 3175 goto fail; 3176 break; 3177 3178 case NVME_OPC_NVM_FLUSH: 3179 cmd->nc_sqe.sqe_nsid = ns->ns_id; 3180 break; 3181 3182 default: 3183 goto fail; 3184 } 3185 3186 return (cmd); 3187 3188 fail: 3189 nvme_free_cmd(cmd); 3190 return (NULL); 3191 } 3192 3193 static void 3194 nvme_bd_xfer_done(void *arg) 3195 { 3196 nvme_cmd_t *cmd = arg; 3197 bd_xfer_t *xfer = cmd->nc_xfer; 3198 int error = 0; 3199 3200 error = nvme_check_cmd_status(cmd); 3201 nvme_free_cmd(cmd); 3202 3203 bd_xfer_done(xfer, error); 3204 } 3205 3206 static void 3207 nvme_bd_driveinfo(void *arg, bd_drive_t *drive) 3208 { 3209 nvme_namespace_t *ns = arg; 3210 nvme_t *nvme = ns->ns_nvme; 3211 3212 /* 3213 * blkdev maintains one queue size per instance (namespace), 3214 * but all namespace share the I/O queues. 3215 * TODO: need to figure out a sane default, or use per-NS I/O queues, 3216 * or change blkdev to handle EAGAIN 3217 */ 3218 drive->d_qsize = nvme->n_ioq_count * nvme->n_io_queue_len 3219 / nvme->n_namespace_count; 3220 3221 /* 3222 * d_maxxfer is not set, which means the value is taken from the DMA 3223 * attributes specified to bd_alloc_handle. 3224 */ 3225 3226 drive->d_removable = B_FALSE; 3227 drive->d_hotpluggable = B_FALSE; 3228 3229 bcopy(ns->ns_eui64, drive->d_eui64, sizeof (drive->d_eui64)); 3230 drive->d_target = ns->ns_id; 3231 drive->d_lun = 0; 3232 3233 drive->d_model = nvme->n_idctl->id_model; 3234 drive->d_model_len = sizeof (nvme->n_idctl->id_model); 3235 drive->d_vendor = nvme->n_vendor; 3236 drive->d_vendor_len = strlen(nvme->n_vendor); 3237 drive->d_product = nvme->n_product; 3238 drive->d_product_len = strlen(nvme->n_product); 3239 drive->d_serial = nvme->n_idctl->id_serial; 3240 drive->d_serial_len = sizeof (nvme->n_idctl->id_serial); 3241 drive->d_revision = nvme->n_idctl->id_fwrev; 3242 drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev); 3243 } 3244 3245 static int 3246 nvme_bd_mediainfo(void *arg, bd_media_t *media) 3247 { 3248 nvme_namespace_t *ns = arg; 3249 3250 media->m_nblks = ns->ns_block_count; 3251 media->m_blksize = ns->ns_block_size; 3252 media->m_readonly = B_FALSE; 3253 media->m_solidstate = B_TRUE; 3254 3255 media->m_pblksize = ns->ns_best_block_size; 3256 3257 return (0); 3258 } 3259 3260 static int 3261 nvme_bd_cmd(nvme_namespace_t *ns, bd_xfer_t *xfer, uint8_t opc) 3262 { 3263 nvme_t *nvme = ns->ns_nvme; 3264 nvme_cmd_t *cmd; 3265 3266 if (nvme->n_dead) 3267 return (EIO); 3268 3269 /* No polling for now */ 3270 if (xfer->x_flags & BD_XFER_POLL) 3271 return (EIO); 3272 3273 cmd = nvme_create_nvm_cmd(ns, opc, xfer); 3274 if (cmd == NULL) 3275 return (ENOMEM); 3276 3277 cmd->nc_sqid = (CPU->cpu_id % nvme->n_ioq_count) + 1; 3278 ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); 3279 3280 if (nvme_submit_cmd(nvme->n_ioq[cmd->nc_sqid], cmd) 3281 != DDI_SUCCESS) 3282 return (EAGAIN); 3283 3284 return (0); 3285 } 3286 3287 static int 3288 nvme_bd_read(void *arg, bd_xfer_t *xfer) 3289 { 3290 nvme_namespace_t *ns = arg; 3291 3292 return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_READ)); 3293 } 3294 3295 static int 3296 nvme_bd_write(void *arg, bd_xfer_t *xfer) 3297 { 3298 nvme_namespace_t *ns = arg; 3299 3300 return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_WRITE)); 3301 } 3302 3303 static int 3304 nvme_bd_sync(void *arg, bd_xfer_t *xfer) 3305 { 3306 nvme_namespace_t *ns = arg; 3307 3308 if (ns->ns_nvme->n_dead) 3309 return (EIO); 3310 3311 /* 3312 * If the volatile write cache is not present or not enabled the FLUSH 3313 * command is a no-op, so we can take a shortcut here. 3314 */ 3315 if (!ns->ns_nvme->n_write_cache_present) { 3316 bd_xfer_done(xfer, ENOTSUP); 3317 return (0); 3318 } 3319 3320 if (!ns->ns_nvme->n_write_cache_enabled) { 3321 bd_xfer_done(xfer, 0); 3322 return (0); 3323 } 3324 3325 return (nvme_bd_cmd(ns, xfer, NVME_OPC_NVM_FLUSH)); 3326 } 3327 3328 static int 3329 nvme_bd_devid(void *arg, dev_info_t *devinfo, ddi_devid_t *devid) 3330 { 3331 nvme_namespace_t *ns = arg; 3332 3333 /*LINTED: E_BAD_PTR_CAST_ALIGN*/ 3334 if (*(uint64_t *)ns->ns_eui64 != 0) { 3335 return (ddi_devid_init(devinfo, DEVID_SCSI3_WWN, 3336 sizeof (ns->ns_eui64), ns->ns_eui64, devid)); 3337 } else { 3338 return (ddi_devid_init(devinfo, DEVID_ENCAP, 3339 strlen(ns->ns_devid), ns->ns_devid, devid)); 3340 } 3341 } 3342 3343 static int 3344 nvme_open(dev_t *devp, int flag, int otyp, cred_t *cred_p) 3345 { 3346 #ifndef __lock_lint 3347 _NOTE(ARGUNUSED(cred_p)); 3348 #endif 3349 minor_t minor = getminor(*devp); 3350 nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 3351 int nsid = NVME_MINOR_NSID(minor); 3352 nvme_minor_state_t *nm; 3353 int rv = 0; 3354 3355 if (otyp != OTYP_CHR) 3356 return (EINVAL); 3357 3358 if (nvme == NULL) 3359 return (ENXIO); 3360 3361 if (nsid > nvme->n_namespace_count) 3362 return (ENXIO); 3363 3364 nm = nsid == 0 ? &nvme->n_minor : &nvme->n_ns[nsid - 1].ns_minor; 3365 3366 mutex_enter(&nm->nm_mutex); 3367 if (nm->nm_oexcl) { 3368 rv = EBUSY; 3369 goto out; 3370 } 3371 3372 if (flag & FEXCL) { 3373 if (nm->nm_ocnt != 0) { 3374 rv = EBUSY; 3375 goto out; 3376 } 3377 nm->nm_oexcl = B_TRUE; 3378 } 3379 3380 nm->nm_ocnt++; 3381 3382 out: 3383 mutex_exit(&nm->nm_mutex); 3384 return (rv); 3385 3386 } 3387 3388 static int 3389 nvme_close(dev_t dev, int flag, int otyp, cred_t *cred_p) 3390 { 3391 #ifndef __lock_lint 3392 _NOTE(ARGUNUSED(cred_p)); 3393 _NOTE(ARGUNUSED(flag)); 3394 #endif 3395 minor_t minor = getminor(dev); 3396 nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 3397 int nsid = NVME_MINOR_NSID(minor); 3398 nvme_minor_state_t *nm; 3399 3400 if (otyp != OTYP_CHR) 3401 return (ENXIO); 3402 3403 if (nvme == NULL) 3404 return (ENXIO); 3405 3406 if (nsid > nvme->n_namespace_count) 3407 return (ENXIO); 3408 3409 nm = nsid == 0 ? &nvme->n_minor : &nvme->n_ns[nsid - 1].ns_minor; 3410 3411 mutex_enter(&nm->nm_mutex); 3412 if (nm->nm_oexcl) 3413 nm->nm_oexcl = B_FALSE; 3414 3415 ASSERT(nm->nm_ocnt > 0); 3416 nm->nm_ocnt--; 3417 mutex_exit(&nm->nm_mutex); 3418 3419 return (0); 3420 } 3421 3422 static int 3423 nvme_ioctl_identify(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 3424 cred_t *cred_p) 3425 { 3426 _NOTE(ARGUNUSED(cred_p)); 3427 int rv = 0; 3428 void *idctl; 3429 3430 if ((mode & FREAD) == 0) 3431 return (EPERM); 3432 3433 if (nioc->n_len < NVME_IDENTIFY_BUFSIZE) 3434 return (EINVAL); 3435 3436 idctl = nvme_identify(nvme, nsid); 3437 if (idctl == NULL) 3438 return (EIO); 3439 3440 if (ddi_copyout(idctl, (void *)nioc->n_buf, NVME_IDENTIFY_BUFSIZE, mode) 3441 != 0) 3442 rv = EFAULT; 3443 3444 kmem_free(idctl, NVME_IDENTIFY_BUFSIZE); 3445 3446 return (rv); 3447 } 3448 3449 static int 3450 nvme_ioctl_capabilities(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 3451 int mode, cred_t *cred_p) 3452 { 3453 _NOTE(ARGUNUSED(nsid, cred_p)); 3454 int rv = 0; 3455 nvme_reg_cap_t cap = { 0 }; 3456 nvme_capabilities_t nc; 3457 3458 if ((mode & FREAD) == 0) 3459 return (EPERM); 3460 3461 if (nioc->n_len < sizeof (nc)) 3462 return (EINVAL); 3463 3464 cap.r = nvme_get64(nvme, NVME_REG_CAP); 3465 3466 /* 3467 * The MPSMIN and MPSMAX fields in the CAP register use 0 to 3468 * specify the base page size of 4k (1<<12), so add 12 here to 3469 * get the real page size value. 3470 */ 3471 nc.mpsmax = 1 << (12 + cap.b.cap_mpsmax); 3472 nc.mpsmin = 1 << (12 + cap.b.cap_mpsmin); 3473 3474 if (ddi_copyout(&nc, (void *)nioc->n_buf, sizeof (nc), mode) != 0) 3475 rv = EFAULT; 3476 3477 return (rv); 3478 } 3479 3480 static int 3481 nvme_ioctl_get_logpage(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 3482 int mode, cred_t *cred_p) 3483 { 3484 _NOTE(ARGUNUSED(cred_p)); 3485 void *log = NULL; 3486 size_t bufsize = 0; 3487 int rv = 0; 3488 3489 if ((mode & FREAD) == 0) 3490 return (EPERM); 3491 3492 switch (nioc->n_arg) { 3493 case NVME_LOGPAGE_ERROR: 3494 if (nsid != 0) 3495 return (EINVAL); 3496 break; 3497 case NVME_LOGPAGE_HEALTH: 3498 if (nsid != 0 && nvme->n_idctl->id_lpa.lp_smart == 0) 3499 return (EINVAL); 3500 3501 if (nsid == 0) 3502 nsid = (uint32_t)-1; 3503 3504 break; 3505 case NVME_LOGPAGE_FWSLOT: 3506 if (nsid != 0) 3507 return (EINVAL); 3508 break; 3509 default: 3510 return (EINVAL); 3511 } 3512 3513 if (nvme_get_logpage(nvme, &log, &bufsize, nioc->n_arg, nsid) 3514 != DDI_SUCCESS) 3515 return (EIO); 3516 3517 if (nioc->n_len < bufsize) { 3518 kmem_free(log, bufsize); 3519 return (EINVAL); 3520 } 3521 3522 if (ddi_copyout(log, (void *)nioc->n_buf, bufsize, mode) != 0) 3523 rv = EFAULT; 3524 3525 nioc->n_len = bufsize; 3526 kmem_free(log, bufsize); 3527 3528 return (rv); 3529 } 3530 3531 static int 3532 nvme_ioctl_get_features(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, 3533 int mode, cred_t *cred_p) 3534 { 3535 _NOTE(ARGUNUSED(cred_p)); 3536 void *buf = NULL; 3537 size_t bufsize = 0; 3538 uint32_t res = 0; 3539 uint8_t feature; 3540 int rv = 0; 3541 3542 if ((mode & FREAD) == 0) 3543 return (EPERM); 3544 3545 if ((nioc->n_arg >> 32) > 0xff) 3546 return (EINVAL); 3547 3548 feature = (uint8_t)(nioc->n_arg >> 32); 3549 3550 switch (feature) { 3551 case NVME_FEAT_ARBITRATION: 3552 case NVME_FEAT_POWER_MGMT: 3553 case NVME_FEAT_TEMPERATURE: 3554 case NVME_FEAT_ERROR: 3555 case NVME_FEAT_NQUEUES: 3556 case NVME_FEAT_INTR_COAL: 3557 case NVME_FEAT_WRITE_ATOM: 3558 case NVME_FEAT_ASYNC_EVENT: 3559 case NVME_FEAT_PROGRESS: 3560 if (nsid != 0) 3561 return (EINVAL); 3562 break; 3563 3564 case NVME_FEAT_INTR_VECT: 3565 if (nsid != 0) 3566 return (EINVAL); 3567 3568 res = nioc->n_arg & 0xffffffffUL; 3569 if (res >= nvme->n_intr_cnt) 3570 return (EINVAL); 3571 break; 3572 3573 case NVME_FEAT_LBA_RANGE: 3574 if (nvme->n_lba_range_supported == B_FALSE) 3575 return (EINVAL); 3576 3577 if (nsid == 0 || 3578 nsid > nvme->n_namespace_count) 3579 return (EINVAL); 3580 3581 break; 3582 3583 case NVME_FEAT_WRITE_CACHE: 3584 if (nsid != 0) 3585 return (EINVAL); 3586 3587 if (!nvme->n_write_cache_present) 3588 return (EINVAL); 3589 3590 break; 3591 3592 case NVME_FEAT_AUTO_PST: 3593 if (nsid != 0) 3594 return (EINVAL); 3595 3596 if (!nvme->n_auto_pst_supported) 3597 return (EINVAL); 3598 3599 break; 3600 3601 default: 3602 return (EINVAL); 3603 } 3604 3605 if (nvme_get_features(nvme, nsid, feature, &res, &buf, &bufsize) == 3606 B_FALSE) 3607 return (EIO); 3608 3609 if (nioc->n_len < bufsize) { 3610 kmem_free(buf, bufsize); 3611 return (EINVAL); 3612 } 3613 3614 if (buf && ddi_copyout(buf, (void*)nioc->n_buf, bufsize, mode) != 0) 3615 rv = EFAULT; 3616 3617 kmem_free(buf, bufsize); 3618 nioc->n_arg = res; 3619 nioc->n_len = bufsize; 3620 3621 return (rv); 3622 } 3623 3624 static int 3625 nvme_ioctl_intr_cnt(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 3626 cred_t *cred_p) 3627 { 3628 _NOTE(ARGUNUSED(nsid, mode, cred_p)); 3629 3630 if ((mode & FREAD) == 0) 3631 return (EPERM); 3632 3633 nioc->n_arg = nvme->n_intr_cnt; 3634 return (0); 3635 } 3636 3637 static int 3638 nvme_ioctl_version(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 3639 cred_t *cred_p) 3640 { 3641 _NOTE(ARGUNUSED(nsid, cred_p)); 3642 int rv = 0; 3643 3644 if ((mode & FREAD) == 0) 3645 return (EPERM); 3646 3647 if (nioc->n_len < sizeof (nvme->n_version)) 3648 return (ENOMEM); 3649 3650 if (ddi_copyout(&nvme->n_version, (void *)nioc->n_buf, 3651 sizeof (nvme->n_version), mode) != 0) 3652 rv = EFAULT; 3653 3654 return (rv); 3655 } 3656 3657 static int 3658 nvme_ioctl_format(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 3659 cred_t *cred_p) 3660 { 3661 _NOTE(ARGUNUSED(mode)); 3662 nvme_format_nvm_t frmt = { 0 }; 3663 int c_nsid = nsid != 0 ? nsid - 1 : 0; 3664 3665 if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 3666 return (EPERM); 3667 3668 frmt.r = nioc->n_arg & 0xffffffff; 3669 3670 /* 3671 * Check whether the FORMAT NVM command is supported. 3672 */ 3673 if (nvme->n_idctl->id_oacs.oa_format == 0) 3674 return (EINVAL); 3675 3676 /* 3677 * Don't allow format or secure erase of individual namespace if that 3678 * would cause a format or secure erase of all namespaces. 3679 */ 3680 if (nsid != 0 && nvme->n_idctl->id_fna.fn_format != 0) 3681 return (EINVAL); 3682 3683 if (nsid != 0 && frmt.b.fm_ses != NVME_FRMT_SES_NONE && 3684 nvme->n_idctl->id_fna.fn_sec_erase != 0) 3685 return (EINVAL); 3686 3687 /* 3688 * Don't allow formatting with Protection Information. 3689 */ 3690 if (frmt.b.fm_pi != 0 || frmt.b.fm_pil != 0 || frmt.b.fm_ms != 0) 3691 return (EINVAL); 3692 3693 /* 3694 * Don't allow formatting using an illegal LBA format, or any LBA format 3695 * that uses metadata. 3696 */ 3697 if (frmt.b.fm_lbaf > nvme->n_ns[c_nsid].ns_idns->id_nlbaf || 3698 nvme->n_ns[c_nsid].ns_idns->id_lbaf[frmt.b.fm_lbaf].lbaf_ms != 0) 3699 return (EINVAL); 3700 3701 /* 3702 * Don't allow formatting using an illegal Secure Erase setting. 3703 */ 3704 if (frmt.b.fm_ses > NVME_FRMT_MAX_SES || 3705 (frmt.b.fm_ses == NVME_FRMT_SES_CRYPTO && 3706 nvme->n_idctl->id_fna.fn_crypt_erase == 0)) 3707 return (EINVAL); 3708 3709 if (nsid == 0) 3710 nsid = (uint32_t)-1; 3711 3712 return (nvme_format_nvm(nvme, nsid, frmt.b.fm_lbaf, B_FALSE, 0, B_FALSE, 3713 frmt.b.fm_ses)); 3714 } 3715 3716 static int 3717 nvme_ioctl_detach(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 3718 cred_t *cred_p) 3719 { 3720 _NOTE(ARGUNUSED(nioc, mode)); 3721 int rv = 0; 3722 3723 if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 3724 return (EPERM); 3725 3726 if (nsid == 0) 3727 return (EINVAL); 3728 3729 rv = bd_detach_handle(nvme->n_ns[nsid - 1].ns_bd_hdl); 3730 if (rv != DDI_SUCCESS) 3731 rv = EBUSY; 3732 3733 return (rv); 3734 } 3735 3736 static int 3737 nvme_ioctl_attach(nvme_t *nvme, int nsid, nvme_ioctl_t *nioc, int mode, 3738 cred_t *cred_p) 3739 { 3740 _NOTE(ARGUNUSED(nioc, mode)); 3741 nvme_identify_nsid_t *idns; 3742 int rv = 0; 3743 3744 if ((mode & FWRITE) == 0 || secpolicy_sys_config(cred_p, B_FALSE) != 0) 3745 return (EPERM); 3746 3747 if (nsid == 0) 3748 return (EINVAL); 3749 3750 /* 3751 * Identify namespace again, free old identify data. 3752 */ 3753 idns = nvme->n_ns[nsid - 1].ns_idns; 3754 if (nvme_init_ns(nvme, nsid) != DDI_SUCCESS) 3755 return (EIO); 3756 3757 kmem_free(idns, sizeof (nvme_identify_nsid_t)); 3758 3759 rv = bd_attach_handle(nvme->n_dip, nvme->n_ns[nsid - 1].ns_bd_hdl); 3760 if (rv != DDI_SUCCESS) 3761 rv = EBUSY; 3762 3763 return (rv); 3764 } 3765 3766 static int 3767 nvme_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred_p, 3768 int *rval_p) 3769 { 3770 #ifndef __lock_lint 3771 _NOTE(ARGUNUSED(rval_p)); 3772 #endif 3773 minor_t minor = getminor(dev); 3774 nvme_t *nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(minor)); 3775 int nsid = NVME_MINOR_NSID(minor); 3776 int rv = 0; 3777 nvme_ioctl_t nioc; 3778 3779 int (*nvme_ioctl[])(nvme_t *, int, nvme_ioctl_t *, int, cred_t *) = { 3780 NULL, 3781 nvme_ioctl_identify, 3782 nvme_ioctl_identify, 3783 nvme_ioctl_capabilities, 3784 nvme_ioctl_get_logpage, 3785 nvme_ioctl_get_features, 3786 nvme_ioctl_intr_cnt, 3787 nvme_ioctl_version, 3788 nvme_ioctl_format, 3789 nvme_ioctl_detach, 3790 nvme_ioctl_attach 3791 }; 3792 3793 if (nvme == NULL) 3794 return (ENXIO); 3795 3796 if (nsid > nvme->n_namespace_count) 3797 return (ENXIO); 3798 3799 if (IS_DEVCTL(cmd)) 3800 return (ndi_devctl_ioctl(nvme->n_dip, cmd, arg, mode, 0)); 3801 3802 #ifdef _MULTI_DATAMODEL 3803 switch (ddi_model_convert_from(mode & FMODELS)) { 3804 case DDI_MODEL_ILP32: { 3805 nvme_ioctl32_t nioc32; 3806 if (ddi_copyin((void*)arg, &nioc32, sizeof (nvme_ioctl32_t), 3807 mode) != 0) 3808 return (EFAULT); 3809 nioc.n_len = nioc32.n_len; 3810 nioc.n_buf = nioc32.n_buf; 3811 nioc.n_arg = nioc32.n_arg; 3812 break; 3813 } 3814 case DDI_MODEL_NONE: 3815 #endif 3816 if (ddi_copyin((void*)arg, &nioc, sizeof (nvme_ioctl_t), mode) 3817 != 0) 3818 return (EFAULT); 3819 #ifdef _MULTI_DATAMODEL 3820 break; 3821 } 3822 #endif 3823 3824 if (cmd == NVME_IOC_IDENTIFY_CTRL) { 3825 /* 3826 * This makes NVME_IOC_IDENTIFY_CTRL work the same on devctl and 3827 * attachment point nodes. 3828 */ 3829 nsid = 0; 3830 } else if (cmd == NVME_IOC_IDENTIFY_NSID && nsid == 0) { 3831 /* 3832 * This makes NVME_IOC_IDENTIFY_NSID work on a devctl node, it 3833 * will always return identify data for namespace 1. 3834 */ 3835 nsid = 1; 3836 } 3837 3838 if (IS_NVME_IOC(cmd) && nvme_ioctl[NVME_IOC_CMD(cmd)] != NULL) 3839 rv = nvme_ioctl[NVME_IOC_CMD(cmd)](nvme, nsid, &nioc, mode, 3840 cred_p); 3841 else 3842 rv = EINVAL; 3843 3844 #ifdef _MULTI_DATAMODEL 3845 switch (ddi_model_convert_from(mode & FMODELS)) { 3846 case DDI_MODEL_ILP32: { 3847 nvme_ioctl32_t nioc32; 3848 3849 nioc32.n_len = (size32_t)nioc.n_len; 3850 nioc32.n_buf = (uintptr32_t)nioc.n_buf; 3851 nioc32.n_arg = nioc.n_arg; 3852 3853 if (ddi_copyout(&nioc32, (void *)arg, sizeof (nvme_ioctl32_t), 3854 mode) != 0) 3855 return (EFAULT); 3856 break; 3857 } 3858 case DDI_MODEL_NONE: 3859 #endif 3860 if (ddi_copyout(&nioc, (void *)arg, sizeof (nvme_ioctl_t), mode) 3861 != 0) 3862 return (EFAULT); 3863 #ifdef _MULTI_DATAMODEL 3864 break; 3865 } 3866 #endif 3867 3868 return (rv); 3869 } 3870