14c87aefeSPatrick Mooney /*-
2*32640292SAndy Fiddaman * SPDX-License-Identifier: BSD-2-Clause
34c87aefeSPatrick Mooney *
44c87aefeSPatrick Mooney * Copyright (c) 2017 Shunsuke Mie
54c87aefeSPatrick Mooney * Copyright (c) 2018 Leon Dang
66960cd89SAndy Fiddaman * Copyright (c) 2020 Chuck Tuffli
74c87aefeSPatrick Mooney *
84c87aefeSPatrick Mooney * Redistribution and use in source and binary forms, with or without
94c87aefeSPatrick Mooney * modification, are permitted provided that the following conditions
104c87aefeSPatrick Mooney * are met:
114c87aefeSPatrick Mooney * 1. Redistributions of source code must retain the above copyright
124c87aefeSPatrick Mooney * notice, this list of conditions and the following disclaimer.
134c87aefeSPatrick Mooney * 2. Redistributions in binary form must reproduce the above copyright
144c87aefeSPatrick Mooney * notice, this list of conditions and the following disclaimer in the
154c87aefeSPatrick Mooney * documentation and/or other materials provided with the distribution.
164c87aefeSPatrick Mooney *
174c87aefeSPatrick Mooney * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
184c87aefeSPatrick Mooney * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
194c87aefeSPatrick Mooney * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
204c87aefeSPatrick Mooney * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
214c87aefeSPatrick Mooney * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
224c87aefeSPatrick Mooney * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
234c87aefeSPatrick Mooney * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
244c87aefeSPatrick Mooney * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
254c87aefeSPatrick Mooney * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
264c87aefeSPatrick Mooney * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
274c87aefeSPatrick Mooney * SUCH DAMAGE.
284c87aefeSPatrick Mooney */
294c87aefeSPatrick Mooney
304c87aefeSPatrick Mooney /*
314c87aefeSPatrick Mooney * bhyve PCIe-NVMe device emulation.
324c87aefeSPatrick Mooney *
334c87aefeSPatrick Mooney * options:
346960cd89SAndy Fiddaman * -s <n>,nvme,devpath,maxq=#,qsz=#,ioslots=#,sectsz=#,ser=A-Z,eui64=#,dsm=<opt>
354c87aefeSPatrick Mooney *
364c87aefeSPatrick Mooney * accepted devpath:
374c87aefeSPatrick Mooney * /dev/blockdev
384c87aefeSPatrick Mooney * /path/to/image
394c87aefeSPatrick Mooney * ram=size_in_MiB
404c87aefeSPatrick Mooney *
414c87aefeSPatrick Mooney * maxq = max number of queues
424c87aefeSPatrick Mooney * qsz = max elements in each queue
434c87aefeSPatrick Mooney * ioslots = max number of concurrent io requests
444c87aefeSPatrick Mooney * sectsz = sector size (defaults to blockif sector size)
454c87aefeSPatrick Mooney * ser = serial number (20-chars max)
4684659b24SMichael Zeller * eui64 = IEEE Extended Unique Identifier (8 byte value)
476960cd89SAndy Fiddaman * dsm = DataSet Management support. Option is one of auto, enable,disable
484c87aefeSPatrick Mooney *
494c87aefeSPatrick Mooney */
504c87aefeSPatrick Mooney
514c87aefeSPatrick Mooney /* TODO:
524c87aefeSPatrick Mooney - create async event for smart and log
534c87aefeSPatrick Mooney - intr coalesce
544c87aefeSPatrick Mooney */
554c87aefeSPatrick Mooney
564c87aefeSPatrick Mooney #include <sys/cdefs.h>
574c87aefeSPatrick Mooney
586960cd89SAndy Fiddaman #include <sys/errno.h>
594c87aefeSPatrick Mooney #include <sys/types.h>
60*32640292SAndy Fiddaman #ifdef __FreeBSD__
61*32640292SAndy Fiddaman #include <sys/crc16.h>
62*32640292SAndy Fiddaman #else
63*32640292SAndy Fiddaman #include "crc16.h"
64*32640292SAndy Fiddaman #endif
6584659b24SMichael Zeller #include <net/ieee_oui.h>
6684659b24SMichael Zeller #ifndef __FreeBSD__
6784659b24SMichael Zeller #include <endian.h>
6884659b24SMichael Zeller #endif
694c87aefeSPatrick Mooney
704c87aefeSPatrick Mooney #include <assert.h>
714c87aefeSPatrick Mooney #include <pthread.h>
726dc98349SAndy Fiddaman #include <pthread_np.h>
734c87aefeSPatrick Mooney #include <semaphore.h>
744c87aefeSPatrick Mooney #include <stdbool.h>
754c87aefeSPatrick Mooney #include <stddef.h>
764c87aefeSPatrick Mooney #include <stdint.h>
774c87aefeSPatrick Mooney #include <stdio.h>
784c87aefeSPatrick Mooney #include <stdlib.h>
794c87aefeSPatrick Mooney #include <string.h>
804c87aefeSPatrick Mooney
814c87aefeSPatrick Mooney #include <machine/atomic.h>
824c87aefeSPatrick Mooney #include <machine/vmm.h>
834c87aefeSPatrick Mooney #include <vmmapi.h>
844c87aefeSPatrick Mooney
854c87aefeSPatrick Mooney #include <dev/nvme/nvme.h>
864c87aefeSPatrick Mooney
874c87aefeSPatrick Mooney #include "bhyverun.h"
884c87aefeSPatrick Mooney #include "block_if.h"
892b948146SAndy Fiddaman #include "config.h"
90154972afSPatrick Mooney #include "debug.h"
914c87aefeSPatrick Mooney #include "pci_emul.h"
924c87aefeSPatrick Mooney
934c87aefeSPatrick Mooney
944c87aefeSPatrick Mooney static int nvme_debug = 0;
956960cd89SAndy Fiddaman #define DPRINTF(fmt, args...) if (nvme_debug) PRINTLN(fmt, ##args)
966960cd89SAndy Fiddaman #define WPRINTF(fmt, args...) PRINTLN(fmt, ##args)
974c87aefeSPatrick Mooney
984c87aefeSPatrick Mooney /* defaults; can be overridden */
994c87aefeSPatrick Mooney #define NVME_MSIX_BAR 4
1004c87aefeSPatrick Mooney
1014c87aefeSPatrick Mooney #define NVME_IOSLOTS 8
1024c87aefeSPatrick Mooney
1034c87aefeSPatrick Mooney /* The NVMe spec defines bits 13:4 in BAR0 as reserved */
1044c87aefeSPatrick Mooney #define NVME_MMIO_SPACE_MIN (1 << 14)
1054c87aefeSPatrick Mooney
1064c87aefeSPatrick Mooney #define NVME_QUEUES 16
1074c87aefeSPatrick Mooney #define NVME_MAX_QENTRIES 2048
1086960cd89SAndy Fiddaman /* Memory Page size Minimum reported in CAP register */
1096960cd89SAndy Fiddaman #define NVME_MPSMIN 0
1106960cd89SAndy Fiddaman /* MPSMIN converted to bytes */
1116960cd89SAndy Fiddaman #define NVME_MPSMIN_BYTES (1 << (12 + NVME_MPSMIN))
1124c87aefeSPatrick Mooney
1134c87aefeSPatrick Mooney #define NVME_PRP2_ITEMS (PAGE_SIZE/sizeof(uint64_t))
1146960cd89SAndy Fiddaman #define NVME_MDTS 9
1156960cd89SAndy Fiddaman /* Note the + 1 allows for the initial descriptor to not be page aligned */
1166960cd89SAndy Fiddaman #define NVME_MAX_IOVEC ((1 << NVME_MDTS) + 1)
1176960cd89SAndy Fiddaman #define NVME_MAX_DATA_SIZE ((1 << NVME_MDTS) * NVME_MPSMIN_BYTES)
1184c87aefeSPatrick Mooney
11984659b24SMichael Zeller /* This is a synthetic status code to indicate there is no status */
12084659b24SMichael Zeller #define NVME_NO_STATUS 0xffff
12184659b24SMichael Zeller #define NVME_COMPLETION_VALID(c) ((c).status != NVME_NO_STATUS)
12284659b24SMichael Zeller
123d7b72f7bSAndy Fiddaman /* Reported temperature in Kelvin (i.e. room temperature) */
124d7b72f7bSAndy Fiddaman #define NVME_TEMPERATURE 296
125d7b72f7bSAndy Fiddaman
1264c87aefeSPatrick Mooney /* helpers */
1274c87aefeSPatrick Mooney
1284c87aefeSPatrick Mooney /* Convert a zero-based value into a one-based value */
1294c87aefeSPatrick Mooney #define ONE_BASED(zero) ((zero) + 1)
1304c87aefeSPatrick Mooney /* Convert a one-based value into a zero-based value */
1314c87aefeSPatrick Mooney #define ZERO_BASED(one) ((one) - 1)
1324c87aefeSPatrick Mooney
1334c87aefeSPatrick Mooney /* Encode number of SQ's and CQ's for Set/Get Features */
1344c87aefeSPatrick Mooney #define NVME_FEATURE_NUM_QUEUES(sc) \
1354c87aefeSPatrick Mooney (ZERO_BASED((sc)->num_squeues) & 0xffff) | \
136*32640292SAndy Fiddaman (ZERO_BASED((sc)->num_cqueues) & 0xffff) << 16
1374c87aefeSPatrick Mooney
1384c87aefeSPatrick Mooney #define NVME_DOORBELL_OFFSET offsetof(struct nvme_registers, doorbell)
1394c87aefeSPatrick Mooney
1404c87aefeSPatrick Mooney enum nvme_controller_register_offsets {
1414c87aefeSPatrick Mooney NVME_CR_CAP_LOW = 0x00,
1424c87aefeSPatrick Mooney NVME_CR_CAP_HI = 0x04,
1434c87aefeSPatrick Mooney NVME_CR_VS = 0x08,
1444c87aefeSPatrick Mooney NVME_CR_INTMS = 0x0c,
1454c87aefeSPatrick Mooney NVME_CR_INTMC = 0x10,
1464c87aefeSPatrick Mooney NVME_CR_CC = 0x14,
1474c87aefeSPatrick Mooney NVME_CR_CSTS = 0x1c,
1484c87aefeSPatrick Mooney NVME_CR_NSSR = 0x20,
1494c87aefeSPatrick Mooney NVME_CR_AQA = 0x24,
1504c87aefeSPatrick Mooney NVME_CR_ASQ_LOW = 0x28,
1514c87aefeSPatrick Mooney NVME_CR_ASQ_HI = 0x2c,
1524c87aefeSPatrick Mooney NVME_CR_ACQ_LOW = 0x30,
1534c87aefeSPatrick Mooney NVME_CR_ACQ_HI = 0x34,
1544c87aefeSPatrick Mooney };
1554c87aefeSPatrick Mooney
1564c87aefeSPatrick Mooney enum nvme_cmd_cdw11 {
1574c87aefeSPatrick Mooney NVME_CMD_CDW11_PC = 0x0001,
1584c87aefeSPatrick Mooney NVME_CMD_CDW11_IEN = 0x0002,
1594c87aefeSPatrick Mooney NVME_CMD_CDW11_IV = 0xFFFF0000,
1604c87aefeSPatrick Mooney };
1614c87aefeSPatrick Mooney
162154972afSPatrick Mooney enum nvme_copy_dir {
163154972afSPatrick Mooney NVME_COPY_TO_PRP,
164154972afSPatrick Mooney NVME_COPY_FROM_PRP,
165154972afSPatrick Mooney };
166154972afSPatrick Mooney
1674c87aefeSPatrick Mooney #define NVME_CQ_INTEN 0x01
1684c87aefeSPatrick Mooney #define NVME_CQ_INTCOAL 0x02
1694c87aefeSPatrick Mooney
1704c87aefeSPatrick Mooney struct nvme_completion_queue {
1714c87aefeSPatrick Mooney struct nvme_completion *qbase;
1726960cd89SAndy Fiddaman pthread_mutex_t mtx;
1734c87aefeSPatrick Mooney uint32_t size;
1744c87aefeSPatrick Mooney uint16_t tail; /* nvme progress */
1754c87aefeSPatrick Mooney uint16_t head; /* guest progress */
1764c87aefeSPatrick Mooney uint16_t intr_vec;
1774c87aefeSPatrick Mooney uint32_t intr_en;
1784c87aefeSPatrick Mooney };
1794c87aefeSPatrick Mooney
1804c87aefeSPatrick Mooney struct nvme_submission_queue {
1814c87aefeSPatrick Mooney struct nvme_command *qbase;
1826960cd89SAndy Fiddaman pthread_mutex_t mtx;
1834c87aefeSPatrick Mooney uint32_t size;
1844c87aefeSPatrick Mooney uint16_t head; /* nvme progress */
1854c87aefeSPatrick Mooney uint16_t tail; /* guest progress */
1864c87aefeSPatrick Mooney uint16_t cqid; /* completion queue id */
1874c87aefeSPatrick Mooney int qpriority;
1884c87aefeSPatrick Mooney };
1894c87aefeSPatrick Mooney
1904c87aefeSPatrick Mooney enum nvme_storage_type {
1914c87aefeSPatrick Mooney NVME_STOR_BLOCKIF = 0,
1924c87aefeSPatrick Mooney NVME_STOR_RAM = 1,
1934c87aefeSPatrick Mooney };
1944c87aefeSPatrick Mooney
1954c87aefeSPatrick Mooney struct pci_nvme_blockstore {
1964c87aefeSPatrick Mooney enum nvme_storage_type type;
1974c87aefeSPatrick Mooney void *ctx;
1984c87aefeSPatrick Mooney uint64_t size;
1994c87aefeSPatrick Mooney uint32_t sectsz;
2004c87aefeSPatrick Mooney uint32_t sectsz_bits;
20184659b24SMichael Zeller uint64_t eui64;
202154972afSPatrick Mooney uint32_t deallocate:1;
2034c87aefeSPatrick Mooney };
2044c87aefeSPatrick Mooney
2056960cd89SAndy Fiddaman /*
2066960cd89SAndy Fiddaman * Calculate the number of additional page descriptors for guest IO requests
2076960cd89SAndy Fiddaman * based on the advertised Max Data Transfer (MDTS) and given the number of
2086960cd89SAndy Fiddaman * default iovec's in a struct blockif_req.
2096960cd89SAndy Fiddaman */
2106960cd89SAndy Fiddaman #define MDTS_PAD_SIZE \
211b0de25cbSAndy Fiddaman ( NVME_MAX_IOVEC > BLOCKIF_IOV_MAX ? \
212b0de25cbSAndy Fiddaman NVME_MAX_IOVEC - BLOCKIF_IOV_MAX : \
213b0de25cbSAndy Fiddaman 0 )
2146960cd89SAndy Fiddaman
2154c87aefeSPatrick Mooney struct pci_nvme_ioreq {
2164c87aefeSPatrick Mooney struct pci_nvme_softc *sc;
217154972afSPatrick Mooney STAILQ_ENTRY(pci_nvme_ioreq) link;
2184c87aefeSPatrick Mooney struct nvme_submission_queue *nvme_sq;
2194c87aefeSPatrick Mooney uint16_t sqid;
2204c87aefeSPatrick Mooney
2214c87aefeSPatrick Mooney /* command information */
2224c87aefeSPatrick Mooney uint16_t opc;
2234c87aefeSPatrick Mooney uint16_t cid;
2244c87aefeSPatrick Mooney uint32_t nsid;
2254c87aefeSPatrick Mooney
2264c87aefeSPatrick Mooney uint64_t prev_gpaddr;
2274c87aefeSPatrick Mooney size_t prev_size;
2286960cd89SAndy Fiddaman size_t bytes;
2294c87aefeSPatrick Mooney
2304c87aefeSPatrick Mooney struct blockif_req io_req;
2314c87aefeSPatrick Mooney
2326960cd89SAndy Fiddaman struct iovec iovpadding[MDTS_PAD_SIZE];
2334c87aefeSPatrick Mooney };
2344c87aefeSPatrick Mooney
235154972afSPatrick Mooney enum nvme_dsm_type {
236154972afSPatrick Mooney /* Dataset Management bit in ONCS reflects backing storage capability */
237154972afSPatrick Mooney NVME_DATASET_MANAGEMENT_AUTO,
238154972afSPatrick Mooney /* Unconditionally set Dataset Management bit in ONCS */
239154972afSPatrick Mooney NVME_DATASET_MANAGEMENT_ENABLE,
240154972afSPatrick Mooney /* Unconditionally clear Dataset Management bit in ONCS */
241154972afSPatrick Mooney NVME_DATASET_MANAGEMENT_DISABLE,
242154972afSPatrick Mooney };
243154972afSPatrick Mooney
2446960cd89SAndy Fiddaman struct pci_nvme_softc;
2456960cd89SAndy Fiddaman struct nvme_feature_obj;
2466960cd89SAndy Fiddaman
2476960cd89SAndy Fiddaman typedef void (*nvme_feature_cb)(struct pci_nvme_softc *,
2486960cd89SAndy Fiddaman struct nvme_feature_obj *,
2496960cd89SAndy Fiddaman struct nvme_command *,
2506960cd89SAndy Fiddaman struct nvme_completion *);
2516960cd89SAndy Fiddaman
2526960cd89SAndy Fiddaman struct nvme_feature_obj {
2536960cd89SAndy Fiddaman uint32_t cdw11;
2546960cd89SAndy Fiddaman nvme_feature_cb set;
2556960cd89SAndy Fiddaman nvme_feature_cb get;
2566960cd89SAndy Fiddaman bool namespace_specific;
2576960cd89SAndy Fiddaman };
2586960cd89SAndy Fiddaman
2596960cd89SAndy Fiddaman #define NVME_FID_MAX (NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION + 1)
2606960cd89SAndy Fiddaman
2616dc98349SAndy Fiddaman typedef enum {
2626dc98349SAndy Fiddaman PCI_NVME_AE_TYPE_ERROR = 0,
2636dc98349SAndy Fiddaman PCI_NVME_AE_TYPE_SMART,
2646dc98349SAndy Fiddaman PCI_NVME_AE_TYPE_NOTICE,
2656dc98349SAndy Fiddaman PCI_NVME_AE_TYPE_IO_CMD = 6,
2666dc98349SAndy Fiddaman PCI_NVME_AE_TYPE_VENDOR = 7,
2676dc98349SAndy Fiddaman PCI_NVME_AE_TYPE_MAX /* Must be last */
2686dc98349SAndy Fiddaman } pci_nvme_async_type;
2696dc98349SAndy Fiddaman
2706dc98349SAndy Fiddaman /* Asynchronous Event Requests */
2716960cd89SAndy Fiddaman struct pci_nvme_aer {
2726960cd89SAndy Fiddaman STAILQ_ENTRY(pci_nvme_aer) link;
2736960cd89SAndy Fiddaman uint16_t cid; /* Command ID of the submitted AER */
2746960cd89SAndy Fiddaman };
2756960cd89SAndy Fiddaman
276d7b72f7bSAndy Fiddaman /** Asynchronous Event Information - Notice */
2776dc98349SAndy Fiddaman typedef enum {
278d7b72f7bSAndy Fiddaman PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED = 0,
279d7b72f7bSAndy Fiddaman PCI_NVME_AEI_NOTICE_FW_ACTIVATION,
280d7b72f7bSAndy Fiddaman PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE,
281d7b72f7bSAndy Fiddaman PCI_NVME_AEI_NOTICE_ANA_CHANGE,
282d7b72f7bSAndy Fiddaman PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE,
283d7b72f7bSAndy Fiddaman PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT,
284d7b72f7bSAndy Fiddaman PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE,
285d7b72f7bSAndy Fiddaman PCI_NVME_AEI_NOTICE_MAX,
286d7b72f7bSAndy Fiddaman } pci_nvme_async_event_info_notice;
287d7b72f7bSAndy Fiddaman
288d7b72f7bSAndy Fiddaman #define PCI_NVME_AEI_NOTICE_SHIFT 8
289d7b72f7bSAndy Fiddaman #define PCI_NVME_AEI_NOTICE_MASK(event) (1 << (event + PCI_NVME_AEI_NOTICE_SHIFT))
2906dc98349SAndy Fiddaman
2916dc98349SAndy Fiddaman /* Asynchronous Event Notifications */
2926dc98349SAndy Fiddaman struct pci_nvme_aen {
2936dc98349SAndy Fiddaman pci_nvme_async_type atype;
2946dc98349SAndy Fiddaman uint32_t event_data;
2956dc98349SAndy Fiddaman bool posted;
2966dc98349SAndy Fiddaman };
2976dc98349SAndy Fiddaman
298d7b72f7bSAndy Fiddaman /*
299d7b72f7bSAndy Fiddaman * By default, enable all Asynchrnous Event Notifications:
300d7b72f7bSAndy Fiddaman * SMART / Health Critical Warnings
301d7b72f7bSAndy Fiddaman * Namespace Attribute Notices
302d7b72f7bSAndy Fiddaman */
303d7b72f7bSAndy Fiddaman #define PCI_NVME_AEN_DEFAULT_MASK 0x11f
304d7b72f7bSAndy Fiddaman
305d7b72f7bSAndy Fiddaman typedef enum {
306d7b72f7bSAndy Fiddaman NVME_CNTRLTYPE_IO = 1,
307d7b72f7bSAndy Fiddaman NVME_CNTRLTYPE_DISCOVERY = 2,
308d7b72f7bSAndy Fiddaman NVME_CNTRLTYPE_ADMIN = 3,
309d7b72f7bSAndy Fiddaman } pci_nvme_cntrl_type;
310d7b72f7bSAndy Fiddaman
3114c87aefeSPatrick Mooney struct pci_nvme_softc {
3124c87aefeSPatrick Mooney struct pci_devinst *nsc_pi;
3134c87aefeSPatrick Mooney
3144c87aefeSPatrick Mooney pthread_mutex_t mtx;
3154c87aefeSPatrick Mooney
3164c87aefeSPatrick Mooney struct nvme_registers regs;
3174c87aefeSPatrick Mooney
3184c87aefeSPatrick Mooney struct nvme_namespace_data nsdata;
3194c87aefeSPatrick Mooney struct nvme_controller_data ctrldata;
3204c87aefeSPatrick Mooney struct nvme_error_information_entry err_log;
3214c87aefeSPatrick Mooney struct nvme_health_information_page health_log;
3224c87aefeSPatrick Mooney struct nvme_firmware_page fw_log;
3236dc98349SAndy Fiddaman struct nvme_ns_list ns_log;
3244c87aefeSPatrick Mooney
3254c87aefeSPatrick Mooney struct pci_nvme_blockstore nvstore;
3264c87aefeSPatrick Mooney
3274c87aefeSPatrick Mooney uint16_t max_qentries; /* max entries per queue */
3284c87aefeSPatrick Mooney uint32_t max_queues; /* max number of IO SQ's or CQ's */
3294c87aefeSPatrick Mooney uint32_t num_cqueues;
3304c87aefeSPatrick Mooney uint32_t num_squeues;
3316960cd89SAndy Fiddaman bool num_q_is_set; /* Has host set Number of Queues */
3324c87aefeSPatrick Mooney
3334c87aefeSPatrick Mooney struct pci_nvme_ioreq *ioreqs;
334154972afSPatrick Mooney STAILQ_HEAD(, pci_nvme_ioreq) ioreqs_free; /* free list of ioreqs */
3354c87aefeSPatrick Mooney uint32_t pending_ios;
3364c87aefeSPatrick Mooney uint32_t ioslots;
3374c87aefeSPatrick Mooney sem_t iosemlock;
3384c87aefeSPatrick Mooney
3394c87aefeSPatrick Mooney /*
3404c87aefeSPatrick Mooney * Memory mapped Submission and Completion queues
3414c87aefeSPatrick Mooney * Each array includes both Admin and IO queues
3424c87aefeSPatrick Mooney */
3434c87aefeSPatrick Mooney struct nvme_completion_queue *compl_queues;
3444c87aefeSPatrick Mooney struct nvme_submission_queue *submit_queues;
3454c87aefeSPatrick Mooney
3466960cd89SAndy Fiddaman struct nvme_feature_obj feat[NVME_FID_MAX];
347154972afSPatrick Mooney
348154972afSPatrick Mooney enum nvme_dsm_type dataset_management;
3496960cd89SAndy Fiddaman
3506960cd89SAndy Fiddaman /* Accounting for SMART data */
3516960cd89SAndy Fiddaman __uint128_t read_data_units;
3526960cd89SAndy Fiddaman __uint128_t write_data_units;
3536960cd89SAndy Fiddaman __uint128_t read_commands;
3546960cd89SAndy Fiddaman __uint128_t write_commands;
3556960cd89SAndy Fiddaman uint32_t read_dunits_remainder;
3566960cd89SAndy Fiddaman uint32_t write_dunits_remainder;
3576960cd89SAndy Fiddaman
3586960cd89SAndy Fiddaman STAILQ_HEAD(, pci_nvme_aer) aer_list;
3596dc98349SAndy Fiddaman pthread_mutex_t aer_mtx;
3606960cd89SAndy Fiddaman uint32_t aer_count;
3616dc98349SAndy Fiddaman struct pci_nvme_aen aen[PCI_NVME_AE_TYPE_MAX];
3626dc98349SAndy Fiddaman pthread_t aen_tid;
3636dc98349SAndy Fiddaman pthread_mutex_t aen_mtx;
3646dc98349SAndy Fiddaman pthread_cond_t aen_cond;
3654c87aefeSPatrick Mooney };
3664c87aefeSPatrick Mooney
3674c87aefeSPatrick Mooney
3686dc98349SAndy Fiddaman static void pci_nvme_cq_update(struct pci_nvme_softc *sc,
3696dc98349SAndy Fiddaman struct nvme_completion_queue *cq,
3706dc98349SAndy Fiddaman uint32_t cdw0,
3716dc98349SAndy Fiddaman uint16_t cid,
3726dc98349SAndy Fiddaman uint16_t sqid,
3736dc98349SAndy Fiddaman uint16_t status);
3746960cd89SAndy Fiddaman static struct pci_nvme_ioreq *pci_nvme_get_ioreq(struct pci_nvme_softc *);
3756960cd89SAndy Fiddaman static void pci_nvme_release_ioreq(struct pci_nvme_softc *, struct pci_nvme_ioreq *);
3766960cd89SAndy Fiddaman static void pci_nvme_io_done(struct blockif_req *, int);
3774c87aefeSPatrick Mooney
3784c87aefeSPatrick Mooney /* Controller Configuration utils */
3794c87aefeSPatrick Mooney #define NVME_CC_GET_EN(cc) \
3804c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_EN_SHIFT & NVME_CC_REG_EN_MASK)
3814c87aefeSPatrick Mooney #define NVME_CC_GET_CSS(cc) \
3824c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_CSS_SHIFT & NVME_CC_REG_CSS_MASK)
3834c87aefeSPatrick Mooney #define NVME_CC_GET_SHN(cc) \
3844c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_SHN_SHIFT & NVME_CC_REG_SHN_MASK)
3854c87aefeSPatrick Mooney #define NVME_CC_GET_IOSQES(cc) \
3864c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_IOSQES_SHIFT & NVME_CC_REG_IOSQES_MASK)
3874c87aefeSPatrick Mooney #define NVME_CC_GET_IOCQES(cc) \
3884c87aefeSPatrick Mooney ((cc) >> NVME_CC_REG_IOCQES_SHIFT & NVME_CC_REG_IOCQES_MASK)
3894c87aefeSPatrick Mooney
3904c87aefeSPatrick Mooney #define NVME_CC_WRITE_MASK \
3914c87aefeSPatrick Mooney ((NVME_CC_REG_EN_MASK << NVME_CC_REG_EN_SHIFT) | \
3924c87aefeSPatrick Mooney (NVME_CC_REG_IOSQES_MASK << NVME_CC_REG_IOSQES_SHIFT) | \
3934c87aefeSPatrick Mooney (NVME_CC_REG_IOCQES_MASK << NVME_CC_REG_IOCQES_SHIFT))
3944c87aefeSPatrick Mooney
3954c87aefeSPatrick Mooney #define NVME_CC_NEN_WRITE_MASK \
3964c87aefeSPatrick Mooney ((NVME_CC_REG_CSS_MASK << NVME_CC_REG_CSS_SHIFT) | \
3974c87aefeSPatrick Mooney (NVME_CC_REG_MPS_MASK << NVME_CC_REG_MPS_SHIFT) | \
3984c87aefeSPatrick Mooney (NVME_CC_REG_AMS_MASK << NVME_CC_REG_AMS_SHIFT))
3994c87aefeSPatrick Mooney
4004c87aefeSPatrick Mooney /* Controller Status utils */
4014c87aefeSPatrick Mooney #define NVME_CSTS_GET_RDY(sts) \
4024c87aefeSPatrick Mooney ((sts) >> NVME_CSTS_REG_RDY_SHIFT & NVME_CSTS_REG_RDY_MASK)
4034c87aefeSPatrick Mooney
4044c87aefeSPatrick Mooney #define NVME_CSTS_RDY (1 << NVME_CSTS_REG_RDY_SHIFT)
4054f3f3e9aSAndy Fiddaman #define NVME_CSTS_CFS (1 << NVME_CSTS_REG_CFS_SHIFT)
4064c87aefeSPatrick Mooney
4074c87aefeSPatrick Mooney /* Completion Queue status word utils */
4084c87aefeSPatrick Mooney #define NVME_STATUS_P (1 << NVME_STATUS_P_SHIFT)
4094c87aefeSPatrick Mooney #define NVME_STATUS_MASK \
4104c87aefeSPatrick Mooney ((NVME_STATUS_SCT_MASK << NVME_STATUS_SCT_SHIFT) |\
4114c87aefeSPatrick Mooney (NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT))
4124c87aefeSPatrick Mooney
413154972afSPatrick Mooney #define NVME_ONCS_DSM (NVME_CTRLR_DATA_ONCS_DSM_MASK << \
414154972afSPatrick Mooney NVME_CTRLR_DATA_ONCS_DSM_SHIFT)
415154972afSPatrick Mooney
4166960cd89SAndy Fiddaman static void nvme_feature_invalid_cb(struct pci_nvme_softc *,
4176960cd89SAndy Fiddaman struct nvme_feature_obj *,
4186960cd89SAndy Fiddaman struct nvme_command *,
4196960cd89SAndy Fiddaman struct nvme_completion *);
420d7b72f7bSAndy Fiddaman static void nvme_feature_temperature(struct pci_nvme_softc *,
421d7b72f7bSAndy Fiddaman struct nvme_feature_obj *,
422d7b72f7bSAndy Fiddaman struct nvme_command *,
423d7b72f7bSAndy Fiddaman struct nvme_completion *);
4246960cd89SAndy Fiddaman static void nvme_feature_num_queues(struct pci_nvme_softc *,
4256960cd89SAndy Fiddaman struct nvme_feature_obj *,
4266960cd89SAndy Fiddaman struct nvme_command *,
4276960cd89SAndy Fiddaman struct nvme_completion *);
4286960cd89SAndy Fiddaman static void nvme_feature_iv_config(struct pci_nvme_softc *,
4296960cd89SAndy Fiddaman struct nvme_feature_obj *,
4306960cd89SAndy Fiddaman struct nvme_command *,
4316960cd89SAndy Fiddaman struct nvme_completion *);
432d7b72f7bSAndy Fiddaman static void nvme_feature_async_event(struct pci_nvme_softc *,
433d7b72f7bSAndy Fiddaman struct nvme_feature_obj *,
434d7b72f7bSAndy Fiddaman struct nvme_command *,
435d7b72f7bSAndy Fiddaman struct nvme_completion *);
4366960cd89SAndy Fiddaman
4376dc98349SAndy Fiddaman static void *aen_thr(void *arg);
4386dc98349SAndy Fiddaman
4394c87aefeSPatrick Mooney static __inline void
cpywithpad(char * dst,size_t dst_size,const char * src,char pad)4404c87aefeSPatrick Mooney cpywithpad(char *dst, size_t dst_size, const char *src, char pad)
4414c87aefeSPatrick Mooney {
4424c87aefeSPatrick Mooney size_t len;
4434c87aefeSPatrick Mooney
4444c87aefeSPatrick Mooney len = strnlen(src, dst_size);
4454c87aefeSPatrick Mooney memset(dst, pad, dst_size);
4464c87aefeSPatrick Mooney memcpy(dst, src, len);
4474c87aefeSPatrick Mooney }
4484c87aefeSPatrick Mooney
4494c87aefeSPatrick Mooney static __inline void
pci_nvme_status_tc(uint16_t * status,uint16_t type,uint16_t code)4504c87aefeSPatrick Mooney pci_nvme_status_tc(uint16_t *status, uint16_t type, uint16_t code)
4514c87aefeSPatrick Mooney {
4524c87aefeSPatrick Mooney
4534c87aefeSPatrick Mooney *status &= ~NVME_STATUS_MASK;
4544c87aefeSPatrick Mooney *status |= (type & NVME_STATUS_SCT_MASK) << NVME_STATUS_SCT_SHIFT |
4554c87aefeSPatrick Mooney (code & NVME_STATUS_SC_MASK) << NVME_STATUS_SC_SHIFT;
4564c87aefeSPatrick Mooney }
4574c87aefeSPatrick Mooney
4584c87aefeSPatrick Mooney static __inline void
pci_nvme_status_genc(uint16_t * status,uint16_t code)4594c87aefeSPatrick Mooney pci_nvme_status_genc(uint16_t *status, uint16_t code)
4604c87aefeSPatrick Mooney {
4614c87aefeSPatrick Mooney
4624c87aefeSPatrick Mooney pci_nvme_status_tc(status, NVME_SCT_GENERIC, code);
4634c87aefeSPatrick Mooney }
4644c87aefeSPatrick Mooney
4656960cd89SAndy Fiddaman /*
4666960cd89SAndy Fiddaman * Initialize the requested number or IO Submission and Completion Queues.
4676960cd89SAndy Fiddaman * Admin queues are allocated implicitly.
4686960cd89SAndy Fiddaman */
4696960cd89SAndy Fiddaman static void
pci_nvme_init_queues(struct pci_nvme_softc * sc,uint32_t nsq,uint32_t ncq)4706960cd89SAndy Fiddaman pci_nvme_init_queues(struct pci_nvme_softc *sc, uint32_t nsq, uint32_t ncq)
4714c87aefeSPatrick Mooney {
4726960cd89SAndy Fiddaman uint32_t i;
4734c87aefeSPatrick Mooney
4746960cd89SAndy Fiddaman /*
4756960cd89SAndy Fiddaman * Allocate and initialize the Submission Queues
4766960cd89SAndy Fiddaman */
4776960cd89SAndy Fiddaman if (nsq > NVME_QUEUES) {
4786960cd89SAndy Fiddaman WPRINTF("%s: clamping number of SQ from %u to %u",
4796960cd89SAndy Fiddaman __func__, nsq, NVME_QUEUES);
4806960cd89SAndy Fiddaman nsq = NVME_QUEUES;
4816960cd89SAndy Fiddaman }
4826960cd89SAndy Fiddaman
4836960cd89SAndy Fiddaman sc->num_squeues = nsq;
4846960cd89SAndy Fiddaman
4856960cd89SAndy Fiddaman sc->submit_queues = calloc(sc->num_squeues + 1,
4866960cd89SAndy Fiddaman sizeof(struct nvme_submission_queue));
4876960cd89SAndy Fiddaman if (sc->submit_queues == NULL) {
4886960cd89SAndy Fiddaman WPRINTF("%s: SQ allocation failed", __func__);
4896960cd89SAndy Fiddaman sc->num_squeues = 0;
4906960cd89SAndy Fiddaman } else {
4916960cd89SAndy Fiddaman struct nvme_submission_queue *sq = sc->submit_queues;
4926960cd89SAndy Fiddaman
4937bb0eb34SAndy Fiddaman for (i = 0; i < sc->num_squeues + 1; i++)
4947bb0eb34SAndy Fiddaman pthread_mutex_init(&sq[i].mtx, NULL);
4956960cd89SAndy Fiddaman }
4966960cd89SAndy Fiddaman
4976960cd89SAndy Fiddaman /*
4986960cd89SAndy Fiddaman * Allocate and initialize the Completion Queues
4996960cd89SAndy Fiddaman */
5006960cd89SAndy Fiddaman if (ncq > NVME_QUEUES) {
5016960cd89SAndy Fiddaman WPRINTF("%s: clamping number of CQ from %u to %u",
5026960cd89SAndy Fiddaman __func__, ncq, NVME_QUEUES);
5036960cd89SAndy Fiddaman ncq = NVME_QUEUES;
5046960cd89SAndy Fiddaman }
5056960cd89SAndy Fiddaman
5066960cd89SAndy Fiddaman sc->num_cqueues = ncq;
5076960cd89SAndy Fiddaman
5086960cd89SAndy Fiddaman sc->compl_queues = calloc(sc->num_cqueues + 1,
5096960cd89SAndy Fiddaman sizeof(struct nvme_completion_queue));
5106960cd89SAndy Fiddaman if (sc->compl_queues == NULL) {
5116960cd89SAndy Fiddaman WPRINTF("%s: CQ allocation failed", __func__);
5126960cd89SAndy Fiddaman sc->num_cqueues = 0;
5136960cd89SAndy Fiddaman } else {
5146960cd89SAndy Fiddaman struct nvme_completion_queue *cq = sc->compl_queues;
5156960cd89SAndy Fiddaman
5167bb0eb34SAndy Fiddaman for (i = 0; i < sc->num_cqueues + 1; i++)
5177bb0eb34SAndy Fiddaman pthread_mutex_init(&cq[i].mtx, NULL);
5186960cd89SAndy Fiddaman }
5194c87aefeSPatrick Mooney }
5204c87aefeSPatrick Mooney
5214c87aefeSPatrick Mooney static void
pci_nvme_init_ctrldata(struct pci_nvme_softc * sc)5224c87aefeSPatrick Mooney pci_nvme_init_ctrldata(struct pci_nvme_softc *sc)
5234c87aefeSPatrick Mooney {
5244c87aefeSPatrick Mooney struct nvme_controller_data *cd = &sc->ctrldata;
525*32640292SAndy Fiddaman int ret;
5264c87aefeSPatrick Mooney
5274c87aefeSPatrick Mooney cd->vid = 0xFB5D;
5284c87aefeSPatrick Mooney cd->ssvid = 0x0000;
5294c87aefeSPatrick Mooney
5304c87aefeSPatrick Mooney cpywithpad((char *)cd->mn, sizeof(cd->mn), "bhyve-NVMe", ' ');
5314c87aefeSPatrick Mooney cpywithpad((char *)cd->fr, sizeof(cd->fr), "1.0", ' ');
5324c87aefeSPatrick Mooney
5334c87aefeSPatrick Mooney /* Num of submission commands that we can handle at a time (2^rab) */
5344c87aefeSPatrick Mooney cd->rab = 4;
5354c87aefeSPatrick Mooney
5364c87aefeSPatrick Mooney /* FreeBSD OUI */
537*32640292SAndy Fiddaman cd->ieee[0] = 0xfc;
5384c87aefeSPatrick Mooney cd->ieee[1] = 0x9c;
539*32640292SAndy Fiddaman cd->ieee[2] = 0x58;
5404c87aefeSPatrick Mooney
5414c87aefeSPatrick Mooney cd->mic = 0;
5424c87aefeSPatrick Mooney
5436960cd89SAndy Fiddaman cd->mdts = NVME_MDTS; /* max data transfer size (2^mdts * CAP.MPSMIN) */
5444c87aefeSPatrick Mooney
545d7b72f7bSAndy Fiddaman cd->ver = NVME_REV(1,4);
5464c87aefeSPatrick Mooney
547d7b72f7bSAndy Fiddaman cd->cntrltype = NVME_CNTRLTYPE_IO;
5484c87aefeSPatrick Mooney cd->oacs = 1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT;
549d7b72f7bSAndy Fiddaman cd->oaes = NVMEB(NVME_CTRLR_DATA_OAES_NS_ATTR);
5504c87aefeSPatrick Mooney cd->acl = 2;
5514c87aefeSPatrick Mooney cd->aerl = 4;
5524c87aefeSPatrick Mooney
5536960cd89SAndy Fiddaman /* Advertise 1, Read-only firmware slot */
5544f3f3e9aSAndy Fiddaman cd->frmw = NVMEB(NVME_CTRLR_DATA_FRMW_SLOT1_RO) |
5556960cd89SAndy Fiddaman (1 << NVME_CTRLR_DATA_FRMW_NUM_SLOTS_SHIFT);
5564c87aefeSPatrick Mooney cd->lpa = 0; /* TODO: support some simple things like SMART */
5574c87aefeSPatrick Mooney cd->elpe = 0; /* max error log page entries */
5584f3f3e9aSAndy Fiddaman /*
5594f3f3e9aSAndy Fiddaman * Report a single power state (zero-based value)
5604f3f3e9aSAndy Fiddaman * power_state[] values are left as zero to indicate "Not reported"
5614f3f3e9aSAndy Fiddaman */
5624f3f3e9aSAndy Fiddaman cd->npss = 0;
5634c87aefeSPatrick Mooney
5644c87aefeSPatrick Mooney /* Warning Composite Temperature Threshold */
5654c87aefeSPatrick Mooney cd->wctemp = 0x0157;
566d7b72f7bSAndy Fiddaman cd->cctemp = 0x0157;
5674c87aefeSPatrick Mooney
5684f3f3e9aSAndy Fiddaman /* SANICAP must not be 0 for Revision 1.4 and later NVMe Controllers */
5694f3f3e9aSAndy Fiddaman cd->sanicap = (NVME_CTRLR_DATA_SANICAP_NODMMAS_NO <<
5704f3f3e9aSAndy Fiddaman NVME_CTRLR_DATA_SANICAP_NODMMAS_SHIFT);
5714f3f3e9aSAndy Fiddaman
5724c87aefeSPatrick Mooney cd->sqes = (6 << NVME_CTRLR_DATA_SQES_MAX_SHIFT) |
5734c87aefeSPatrick Mooney (6 << NVME_CTRLR_DATA_SQES_MIN_SHIFT);
5744c87aefeSPatrick Mooney cd->cqes = (4 << NVME_CTRLR_DATA_CQES_MAX_SHIFT) |
5754c87aefeSPatrick Mooney (4 << NVME_CTRLR_DATA_CQES_MIN_SHIFT);
5764c87aefeSPatrick Mooney cd->nn = 1; /* number of namespaces */
5774c87aefeSPatrick Mooney
578154972afSPatrick Mooney cd->oncs = 0;
579154972afSPatrick Mooney switch (sc->dataset_management) {
580154972afSPatrick Mooney case NVME_DATASET_MANAGEMENT_AUTO:
581154972afSPatrick Mooney if (sc->nvstore.deallocate)
582154972afSPatrick Mooney cd->oncs |= NVME_ONCS_DSM;
583154972afSPatrick Mooney break;
584154972afSPatrick Mooney case NVME_DATASET_MANAGEMENT_ENABLE:
585154972afSPatrick Mooney cd->oncs |= NVME_ONCS_DSM;
586154972afSPatrick Mooney break;
587154972afSPatrick Mooney default:
588154972afSPatrick Mooney break;
589154972afSPatrick Mooney }
590154972afSPatrick Mooney
591d7b72f7bSAndy Fiddaman cd->fna = NVME_CTRLR_DATA_FNA_FORMAT_ALL_MASK <<
592d7b72f7bSAndy Fiddaman NVME_CTRLR_DATA_FNA_FORMAT_ALL_SHIFT;
593d7b72f7bSAndy Fiddaman
594d7b72f7bSAndy Fiddaman cd->vwc = NVME_CTRLR_DATA_VWC_ALL_NO << NVME_CTRLR_DATA_VWC_ALL_SHIFT;
5954c87aefeSPatrick Mooney
596*32640292SAndy Fiddaman #ifdef __FreeBSD__
597*32640292SAndy Fiddaman ret = snprintf(cd->subnqn, sizeof(cd->subnqn),
598*32640292SAndy Fiddaman "nqn.2013-12.org.freebsd:bhyve-%s-%u-%u-%u",
599*32640292SAndy Fiddaman get_config_value("name"), sc->nsc_pi->pi_bus,
600*32640292SAndy Fiddaman sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
601*32640292SAndy Fiddaman #else
602*32640292SAndy Fiddaman ret = snprintf((char *)cd->subnqn, sizeof (cd->subnqn),
603*32640292SAndy Fiddaman "nqn.2013-12.org.illumos:bhyve-%s-%u-%u-%u",
604*32640292SAndy Fiddaman get_config_value("name"), sc->nsc_pi->pi_bus,
605*32640292SAndy Fiddaman sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
606*32640292SAndy Fiddaman #endif
607*32640292SAndy Fiddaman if ((ret < 0) || ((unsigned)ret > sizeof(cd->subnqn)))
608*32640292SAndy Fiddaman EPRINTLN("%s: error setting subnqn (%d)", __func__, ret);
60984659b24SMichael Zeller }
6104c87aefeSPatrick Mooney
61184659b24SMichael Zeller static void
pci_nvme_init_nsdata_size(struct pci_nvme_blockstore * nvstore,struct nvme_namespace_data * nd)6126dc98349SAndy Fiddaman pci_nvme_init_nsdata_size(struct pci_nvme_blockstore *nvstore,
6136dc98349SAndy Fiddaman struct nvme_namespace_data *nd)
61484659b24SMichael Zeller {
6154c87aefeSPatrick Mooney
616154972afSPatrick Mooney /* Get capacity and block size information from backing store */
617154972afSPatrick Mooney nd->nsze = nvstore->size / nvstore->sectsz;
6184c87aefeSPatrick Mooney nd->ncap = nd->nsze;
6194c87aefeSPatrick Mooney nd->nuse = nd->nsze;
6206dc98349SAndy Fiddaman }
6216dc98349SAndy Fiddaman
6226dc98349SAndy Fiddaman static void
pci_nvme_init_nsdata(struct pci_nvme_softc * sc,struct nvme_namespace_data * nd,uint32_t nsid,struct pci_nvme_blockstore * nvstore)6236dc98349SAndy Fiddaman pci_nvme_init_nsdata(struct pci_nvme_softc *sc,
6246dc98349SAndy Fiddaman struct nvme_namespace_data *nd, uint32_t nsid,
6256dc98349SAndy Fiddaman struct pci_nvme_blockstore *nvstore)
6266dc98349SAndy Fiddaman {
6276dc98349SAndy Fiddaman
6286dc98349SAndy Fiddaman pci_nvme_init_nsdata_size(nvstore, nd);
6294c87aefeSPatrick Mooney
630154972afSPatrick Mooney if (nvstore->type == NVME_STOR_BLOCKIF)
631154972afSPatrick Mooney nvstore->deallocate = blockif_candelete(nvstore->ctx);
632154972afSPatrick Mooney
6334c87aefeSPatrick Mooney nd->nlbaf = 0; /* NLBAF is a 0's based value (i.e. 1 LBA Format) */
63484659b24SMichael Zeller nd->flbas = 0;
63584659b24SMichael Zeller
63684659b24SMichael Zeller /* Create an EUI-64 if user did not provide one */
637154972afSPatrick Mooney if (nvstore->eui64 == 0) {
63884659b24SMichael Zeller char *data = NULL;
639154972afSPatrick Mooney uint64_t eui64 = nvstore->eui64;
64084659b24SMichael Zeller
6412b948146SAndy Fiddaman asprintf(&data, "%s%u%u%u", get_config_value("name"),
6422b948146SAndy Fiddaman sc->nsc_pi->pi_bus, sc->nsc_pi->pi_slot,
6432b948146SAndy Fiddaman sc->nsc_pi->pi_func);
64484659b24SMichael Zeller
64584659b24SMichael Zeller if (data != NULL) {
64684659b24SMichael Zeller eui64 = OUI_FREEBSD_NVME_LOW | crc16(0, data, strlen(data));
64784659b24SMichael Zeller free(data);
64884659b24SMichael Zeller }
649154972afSPatrick Mooney nvstore->eui64 = (eui64 << 16) | (nsid & 0xffff);
65084659b24SMichael Zeller }
651154972afSPatrick Mooney be64enc(nd->eui64, nvstore->eui64);
65284659b24SMichael Zeller
6534c87aefeSPatrick Mooney /* LBA data-sz = 2^lbads */
654154972afSPatrick Mooney nd->lbaf[0] = nvstore->sectsz_bits << NVME_NS_DATA_LBAF_LBADS_SHIFT;
6554c87aefeSPatrick Mooney }
6564c87aefeSPatrick Mooney
6574c87aefeSPatrick Mooney static void
pci_nvme_init_logpages(struct pci_nvme_softc * sc)6584c87aefeSPatrick Mooney pci_nvme_init_logpages(struct pci_nvme_softc *sc)
6594c87aefeSPatrick Mooney {
66059d65d31SAndy Fiddaman __uint128_t power_cycles = 1;
6614c87aefeSPatrick Mooney
6624c87aefeSPatrick Mooney memset(&sc->err_log, 0, sizeof(sc->err_log));
6634c87aefeSPatrick Mooney memset(&sc->health_log, 0, sizeof(sc->health_log));
6644c87aefeSPatrick Mooney memset(&sc->fw_log, 0, sizeof(sc->fw_log));
6656dc98349SAndy Fiddaman memset(&sc->ns_log, 0, sizeof(sc->ns_log));
6666960cd89SAndy Fiddaman
6676960cd89SAndy Fiddaman /* Set read/write remainder to round up according to spec */
6686960cd89SAndy Fiddaman sc->read_dunits_remainder = 999;
6696960cd89SAndy Fiddaman sc->write_dunits_remainder = 999;
6706960cd89SAndy Fiddaman
6716960cd89SAndy Fiddaman /* Set nominal Health values checked by implementations */
672d7b72f7bSAndy Fiddaman sc->health_log.temperature = NVME_TEMPERATURE;
6736960cd89SAndy Fiddaman sc->health_log.available_spare = 100;
6746960cd89SAndy Fiddaman sc->health_log.available_spare_threshold = 10;
6754f3f3e9aSAndy Fiddaman
6764f3f3e9aSAndy Fiddaman /* Set Active Firmware Info to slot 1 */
6774f3f3e9aSAndy Fiddaman sc->fw_log.afi = (1 << NVME_FIRMWARE_PAGE_AFI_SLOT_SHIFT);
6784f3f3e9aSAndy Fiddaman memcpy(&sc->fw_log.revision[0], sc->ctrldata.fr,
6794f3f3e9aSAndy Fiddaman sizeof(sc->fw_log.revision[0]));
68059d65d31SAndy Fiddaman
68159d65d31SAndy Fiddaman memcpy(&sc->health_log.power_cycles, &power_cycles,
68259d65d31SAndy Fiddaman sizeof(sc->health_log.power_cycles));
6836960cd89SAndy Fiddaman }
6846960cd89SAndy Fiddaman
6856960cd89SAndy Fiddaman static void
pci_nvme_init_features(struct pci_nvme_softc * sc)6866960cd89SAndy Fiddaman pci_nvme_init_features(struct pci_nvme_softc *sc)
6876960cd89SAndy Fiddaman {
688d7b72f7bSAndy Fiddaman enum nvme_feature fid;
689d7b72f7bSAndy Fiddaman
690d7b72f7bSAndy Fiddaman for (fid = 0; fid < NVME_FID_MAX; fid++) {
691d7b72f7bSAndy Fiddaman switch (fid) {
692d7b72f7bSAndy Fiddaman case NVME_FEAT_ARBITRATION:
693d7b72f7bSAndy Fiddaman case NVME_FEAT_POWER_MANAGEMENT:
694d7b72f7bSAndy Fiddaman case NVME_FEAT_INTERRUPT_COALESCING: //XXX
695d7b72f7bSAndy Fiddaman case NVME_FEAT_WRITE_ATOMICITY:
696d7b72f7bSAndy Fiddaman /* Mandatory but no special handling required */
697d7b72f7bSAndy Fiddaman //XXX hang - case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG:
698d7b72f7bSAndy Fiddaman //XXX hang - case NVME_FEAT_HOST_BEHAVIOR_SUPPORT:
699d7b72f7bSAndy Fiddaman // this returns a data buffer
700d7b72f7bSAndy Fiddaman break;
701d7b72f7bSAndy Fiddaman case NVME_FEAT_TEMPERATURE_THRESHOLD:
702d7b72f7bSAndy Fiddaman sc->feat[fid].set = nvme_feature_temperature;
703d7b72f7bSAndy Fiddaman break;
704d7b72f7bSAndy Fiddaman case NVME_FEAT_ERROR_RECOVERY:
705d7b72f7bSAndy Fiddaman sc->feat[fid].namespace_specific = true;
706d7b72f7bSAndy Fiddaman break;
707d7b72f7bSAndy Fiddaman case NVME_FEAT_NUMBER_OF_QUEUES:
708d7b72f7bSAndy Fiddaman sc->feat[fid].set = nvme_feature_num_queues;
709d7b72f7bSAndy Fiddaman break;
710d7b72f7bSAndy Fiddaman case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
711d7b72f7bSAndy Fiddaman sc->feat[fid].set = nvme_feature_iv_config;
712d7b72f7bSAndy Fiddaman break;
713d7b72f7bSAndy Fiddaman case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
714d7b72f7bSAndy Fiddaman sc->feat[fid].set = nvme_feature_async_event;
715d7b72f7bSAndy Fiddaman /* Enable all AENs by default */
716d7b72f7bSAndy Fiddaman sc->feat[fid].cdw11 = PCI_NVME_AEN_DEFAULT_MASK;
717d7b72f7bSAndy Fiddaman break;
718d7b72f7bSAndy Fiddaman default:
719d7b72f7bSAndy Fiddaman sc->feat[fid].set = nvme_feature_invalid_cb;
720d7b72f7bSAndy Fiddaman sc->feat[fid].get = nvme_feature_invalid_cb;
721d7b72f7bSAndy Fiddaman }
722d7b72f7bSAndy Fiddaman }
7236960cd89SAndy Fiddaman }
7246960cd89SAndy Fiddaman
7256960cd89SAndy Fiddaman static void
pci_nvme_aer_reset(struct pci_nvme_softc * sc)7266dc98349SAndy Fiddaman pci_nvme_aer_reset(struct pci_nvme_softc *sc)
7276960cd89SAndy Fiddaman {
7286960cd89SAndy Fiddaman
7296960cd89SAndy Fiddaman STAILQ_INIT(&sc->aer_list);
7306960cd89SAndy Fiddaman sc->aer_count = 0;
7314c87aefeSPatrick Mooney }
7324c87aefeSPatrick Mooney
7336dc98349SAndy Fiddaman static void
pci_nvme_aer_init(struct pci_nvme_softc * sc)7346dc98349SAndy Fiddaman pci_nvme_aer_init(struct pci_nvme_softc *sc)
7356dc98349SAndy Fiddaman {
7366dc98349SAndy Fiddaman
7376dc98349SAndy Fiddaman pthread_mutex_init(&sc->aer_mtx, NULL);
7386dc98349SAndy Fiddaman pci_nvme_aer_reset(sc);
7396dc98349SAndy Fiddaman }
7406dc98349SAndy Fiddaman
7416960cd89SAndy Fiddaman static void
pci_nvme_aer_destroy(struct pci_nvme_softc * sc)7426960cd89SAndy Fiddaman pci_nvme_aer_destroy(struct pci_nvme_softc *sc)
7436960cd89SAndy Fiddaman {
7446960cd89SAndy Fiddaman struct pci_nvme_aer *aer = NULL;
7456960cd89SAndy Fiddaman
7466dc98349SAndy Fiddaman pthread_mutex_lock(&sc->aer_mtx);
7476960cd89SAndy Fiddaman while (!STAILQ_EMPTY(&sc->aer_list)) {
7486960cd89SAndy Fiddaman aer = STAILQ_FIRST(&sc->aer_list);
7496960cd89SAndy Fiddaman STAILQ_REMOVE_HEAD(&sc->aer_list, link);
7506960cd89SAndy Fiddaman free(aer);
7516960cd89SAndy Fiddaman }
7526dc98349SAndy Fiddaman pthread_mutex_unlock(&sc->aer_mtx);
7536960cd89SAndy Fiddaman
7546dc98349SAndy Fiddaman pci_nvme_aer_reset(sc);
7556960cd89SAndy Fiddaman }
7566960cd89SAndy Fiddaman
7576960cd89SAndy Fiddaman static bool
pci_nvme_aer_available(struct pci_nvme_softc * sc)7586960cd89SAndy Fiddaman pci_nvme_aer_available(struct pci_nvme_softc *sc)
7596960cd89SAndy Fiddaman {
7606960cd89SAndy Fiddaman
7616dc98349SAndy Fiddaman return (sc->aer_count != 0);
7626960cd89SAndy Fiddaman }
7636960cd89SAndy Fiddaman
7646960cd89SAndy Fiddaman static bool
pci_nvme_aer_limit_reached(struct pci_nvme_softc * sc)7656960cd89SAndy Fiddaman pci_nvme_aer_limit_reached(struct pci_nvme_softc *sc)
7666960cd89SAndy Fiddaman {
7676960cd89SAndy Fiddaman struct nvme_controller_data *cd = &sc->ctrldata;
7686960cd89SAndy Fiddaman
7696960cd89SAndy Fiddaman /* AERL is a zero based value while aer_count is one's based */
77059d65d31SAndy Fiddaman return (sc->aer_count == (cd->aerl + 1U));
7716960cd89SAndy Fiddaman }
7726960cd89SAndy Fiddaman
7736960cd89SAndy Fiddaman /*
7746960cd89SAndy Fiddaman * Add an Async Event Request
7756960cd89SAndy Fiddaman *
7766960cd89SAndy Fiddaman * Stores an AER to be returned later if the Controller needs to notify the
7776960cd89SAndy Fiddaman * host of an event.
7786960cd89SAndy Fiddaman * Note that while the NVMe spec doesn't require Controllers to return AER's
7796960cd89SAndy Fiddaman * in order, this implementation does preserve the order.
7806960cd89SAndy Fiddaman */
7816960cd89SAndy Fiddaman static int
pci_nvme_aer_add(struct pci_nvme_softc * sc,uint16_t cid)7826960cd89SAndy Fiddaman pci_nvme_aer_add(struct pci_nvme_softc *sc, uint16_t cid)
7836960cd89SAndy Fiddaman {
7846960cd89SAndy Fiddaman struct pci_nvme_aer *aer = NULL;
7856960cd89SAndy Fiddaman
7866960cd89SAndy Fiddaman aer = calloc(1, sizeof(struct pci_nvme_aer));
7876960cd89SAndy Fiddaman if (aer == NULL)
7886960cd89SAndy Fiddaman return (-1);
7896960cd89SAndy Fiddaman
7906960cd89SAndy Fiddaman /* Save the Command ID for use in the completion message */
7916960cd89SAndy Fiddaman aer->cid = cid;
7926dc98349SAndy Fiddaman
7936dc98349SAndy Fiddaman pthread_mutex_lock(&sc->aer_mtx);
7946dc98349SAndy Fiddaman sc->aer_count++;
7956960cd89SAndy Fiddaman STAILQ_INSERT_TAIL(&sc->aer_list, aer, link);
7966dc98349SAndy Fiddaman pthread_mutex_unlock(&sc->aer_mtx);
7976960cd89SAndy Fiddaman
7986960cd89SAndy Fiddaman return (0);
7996960cd89SAndy Fiddaman }
8006960cd89SAndy Fiddaman
8016960cd89SAndy Fiddaman /*
8026960cd89SAndy Fiddaman * Get an Async Event Request structure
8036960cd89SAndy Fiddaman *
8046960cd89SAndy Fiddaman * Returns a pointer to an AER previously submitted by the host or NULL if
8056960cd89SAndy Fiddaman * no AER's exist. Caller is responsible for freeing the returned struct.
8066960cd89SAndy Fiddaman */
8076960cd89SAndy Fiddaman static struct pci_nvme_aer *
pci_nvme_aer_get(struct pci_nvme_softc * sc)8086960cd89SAndy Fiddaman pci_nvme_aer_get(struct pci_nvme_softc *sc)
8096960cd89SAndy Fiddaman {
8106960cd89SAndy Fiddaman struct pci_nvme_aer *aer = NULL;
8116960cd89SAndy Fiddaman
8126dc98349SAndy Fiddaman pthread_mutex_lock(&sc->aer_mtx);
8136960cd89SAndy Fiddaman aer = STAILQ_FIRST(&sc->aer_list);
8146960cd89SAndy Fiddaman if (aer != NULL) {
8156960cd89SAndy Fiddaman STAILQ_REMOVE_HEAD(&sc->aer_list, link);
8166960cd89SAndy Fiddaman sc->aer_count--;
8176960cd89SAndy Fiddaman }
8186dc98349SAndy Fiddaman pthread_mutex_unlock(&sc->aer_mtx);
8196dc98349SAndy Fiddaman
8206960cd89SAndy Fiddaman return (aer);
8216960cd89SAndy Fiddaman }
8226dc98349SAndy Fiddaman
8236dc98349SAndy Fiddaman static void
pci_nvme_aen_reset(struct pci_nvme_softc * sc)8246dc98349SAndy Fiddaman pci_nvme_aen_reset(struct pci_nvme_softc *sc)
8256dc98349SAndy Fiddaman {
8266dc98349SAndy Fiddaman uint32_t atype;
8276dc98349SAndy Fiddaman
8286dc98349SAndy Fiddaman memset(sc->aen, 0, PCI_NVME_AE_TYPE_MAX * sizeof(struct pci_nvme_aen));
8296dc98349SAndy Fiddaman
8306dc98349SAndy Fiddaman for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) {
8316dc98349SAndy Fiddaman sc->aen[atype].atype = atype;
8326dc98349SAndy Fiddaman }
8336dc98349SAndy Fiddaman }
8346dc98349SAndy Fiddaman
8356dc98349SAndy Fiddaman static void
pci_nvme_aen_init(struct pci_nvme_softc * sc)8366dc98349SAndy Fiddaman pci_nvme_aen_init(struct pci_nvme_softc *sc)
8376dc98349SAndy Fiddaman {
8386dc98349SAndy Fiddaman char nstr[80];
8396dc98349SAndy Fiddaman
8406dc98349SAndy Fiddaman pci_nvme_aen_reset(sc);
8416dc98349SAndy Fiddaman
8426dc98349SAndy Fiddaman pthread_mutex_init(&sc->aen_mtx, NULL);
8436dc98349SAndy Fiddaman pthread_create(&sc->aen_tid, NULL, aen_thr, sc);
8446dc98349SAndy Fiddaman snprintf(nstr, sizeof(nstr), "nvme-aen-%d:%d", sc->nsc_pi->pi_slot,
8456dc98349SAndy Fiddaman sc->nsc_pi->pi_func);
8466dc98349SAndy Fiddaman pthread_set_name_np(sc->aen_tid, nstr);
8476dc98349SAndy Fiddaman }
8486dc98349SAndy Fiddaman
8496dc98349SAndy Fiddaman static void
pci_nvme_aen_destroy(struct pci_nvme_softc * sc)8506dc98349SAndy Fiddaman pci_nvme_aen_destroy(struct pci_nvme_softc *sc)
8516dc98349SAndy Fiddaman {
8526dc98349SAndy Fiddaman
8536dc98349SAndy Fiddaman pci_nvme_aen_reset(sc);
8546dc98349SAndy Fiddaman }
8556dc98349SAndy Fiddaman
8566dc98349SAndy Fiddaman /* Notify the AEN thread of pending work */
8576dc98349SAndy Fiddaman static void
pci_nvme_aen_notify(struct pci_nvme_softc * sc)8586dc98349SAndy Fiddaman pci_nvme_aen_notify(struct pci_nvme_softc *sc)
8596dc98349SAndy Fiddaman {
8606dc98349SAndy Fiddaman
8616dc98349SAndy Fiddaman pthread_cond_signal(&sc->aen_cond);
8626dc98349SAndy Fiddaman }
8636dc98349SAndy Fiddaman
8646dc98349SAndy Fiddaman /*
8656dc98349SAndy Fiddaman * Post an Asynchronous Event Notification
8666dc98349SAndy Fiddaman */
8676dc98349SAndy Fiddaman static int32_t
pci_nvme_aen_post(struct pci_nvme_softc * sc,pci_nvme_async_type atype,uint32_t event_data)8686dc98349SAndy Fiddaman pci_nvme_aen_post(struct pci_nvme_softc *sc, pci_nvme_async_type atype,
8696dc98349SAndy Fiddaman uint32_t event_data)
8706dc98349SAndy Fiddaman {
8716dc98349SAndy Fiddaman struct pci_nvme_aen *aen;
8726dc98349SAndy Fiddaman
8736dc98349SAndy Fiddaman if (atype >= PCI_NVME_AE_TYPE_MAX) {
8746dc98349SAndy Fiddaman return(EINVAL);
8756dc98349SAndy Fiddaman }
8766dc98349SAndy Fiddaman
8776dc98349SAndy Fiddaman pthread_mutex_lock(&sc->aen_mtx);
8786dc98349SAndy Fiddaman aen = &sc->aen[atype];
8796dc98349SAndy Fiddaman
8806dc98349SAndy Fiddaman /* Has the controller already posted an event of this type? */
8816dc98349SAndy Fiddaman if (aen->posted) {
8826dc98349SAndy Fiddaman pthread_mutex_unlock(&sc->aen_mtx);
8836dc98349SAndy Fiddaman return(EALREADY);
8846dc98349SAndy Fiddaman }
8856dc98349SAndy Fiddaman
8866dc98349SAndy Fiddaman aen->event_data = event_data;
8876dc98349SAndy Fiddaman aen->posted = true;
8886dc98349SAndy Fiddaman pthread_mutex_unlock(&sc->aen_mtx);
8896dc98349SAndy Fiddaman
8906dc98349SAndy Fiddaman pci_nvme_aen_notify(sc);
8916dc98349SAndy Fiddaman
8926dc98349SAndy Fiddaman return(0);
8936dc98349SAndy Fiddaman }
8946dc98349SAndy Fiddaman
8956dc98349SAndy Fiddaman static void
pci_nvme_aen_process(struct pci_nvme_softc * sc)8966dc98349SAndy Fiddaman pci_nvme_aen_process(struct pci_nvme_softc *sc)
8976dc98349SAndy Fiddaman {
8986dc98349SAndy Fiddaman struct pci_nvme_aer *aer;
8996dc98349SAndy Fiddaman struct pci_nvme_aen *aen;
9006dc98349SAndy Fiddaman pci_nvme_async_type atype;
9016dc98349SAndy Fiddaman uint32_t mask;
9026dc98349SAndy Fiddaman uint16_t status;
9036dc98349SAndy Fiddaman uint8_t lid;
9046dc98349SAndy Fiddaman
9056dc98349SAndy Fiddaman #ifndef __FreeBSD__
9066dc98349SAndy Fiddaman lid = 0;
9076960cd89SAndy Fiddaman #endif
9086960cd89SAndy Fiddaman
9096dc98349SAndy Fiddaman assert(pthread_mutex_isowned_np(&sc->aen_mtx));
9106dc98349SAndy Fiddaman for (atype = 0; atype < PCI_NVME_AE_TYPE_MAX; atype++) {
9116dc98349SAndy Fiddaman aen = &sc->aen[atype];
9126dc98349SAndy Fiddaman /* Previous iterations may have depleted the available AER's */
9136dc98349SAndy Fiddaman if (!pci_nvme_aer_available(sc)) {
9146dc98349SAndy Fiddaman DPRINTF("%s: no AER", __func__);
9156dc98349SAndy Fiddaman break;
9166dc98349SAndy Fiddaman }
9176dc98349SAndy Fiddaman
9186dc98349SAndy Fiddaman if (!aen->posted) {
9196dc98349SAndy Fiddaman DPRINTF("%s: no AEN posted for atype=%#x", __func__, atype);
9206dc98349SAndy Fiddaman continue;
9216dc98349SAndy Fiddaman }
9226dc98349SAndy Fiddaman
9236dc98349SAndy Fiddaman status = NVME_SC_SUCCESS;
9246dc98349SAndy Fiddaman
9256dc98349SAndy Fiddaman /* Is the event masked? */
9266dc98349SAndy Fiddaman mask =
9276dc98349SAndy Fiddaman sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11;
9286dc98349SAndy Fiddaman
9296dc98349SAndy Fiddaman DPRINTF("%s: atype=%#x mask=%#x event_data=%#x", __func__, atype, mask, aen->event_data);
9306dc98349SAndy Fiddaman switch (atype) {
9316dc98349SAndy Fiddaman case PCI_NVME_AE_TYPE_ERROR:
9326dc98349SAndy Fiddaman lid = NVME_LOG_ERROR;
9336dc98349SAndy Fiddaman break;
9346dc98349SAndy Fiddaman case PCI_NVME_AE_TYPE_SMART:
9356dc98349SAndy Fiddaman mask &= 0xff;
9366dc98349SAndy Fiddaman if ((mask & aen->event_data) == 0)
9376dc98349SAndy Fiddaman continue;
9386dc98349SAndy Fiddaman lid = NVME_LOG_HEALTH_INFORMATION;
9396dc98349SAndy Fiddaman break;
9406dc98349SAndy Fiddaman case PCI_NVME_AE_TYPE_NOTICE:
941d7b72f7bSAndy Fiddaman if (aen->event_data >= PCI_NVME_AEI_NOTICE_MAX) {
9426dc98349SAndy Fiddaman EPRINTLN("%s unknown AEN notice type %u",
9436dc98349SAndy Fiddaman __func__, aen->event_data);
9446dc98349SAndy Fiddaman status = NVME_SC_INTERNAL_DEVICE_ERROR;
94559d65d31SAndy Fiddaman lid = 0;
9466dc98349SAndy Fiddaman break;
9476dc98349SAndy Fiddaman }
948d7b72f7bSAndy Fiddaman if ((PCI_NVME_AEI_NOTICE_MASK(aen->event_data) & mask) == 0)
9496dc98349SAndy Fiddaman continue;
9506dc98349SAndy Fiddaman switch (aen->event_data) {
951d7b72f7bSAndy Fiddaman case PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED:
9526dc98349SAndy Fiddaman lid = NVME_LOG_CHANGED_NAMESPACE;
9536dc98349SAndy Fiddaman break;
954d7b72f7bSAndy Fiddaman case PCI_NVME_AEI_NOTICE_FW_ACTIVATION:
9556dc98349SAndy Fiddaman lid = NVME_LOG_FIRMWARE_SLOT;
9566dc98349SAndy Fiddaman break;
957d7b72f7bSAndy Fiddaman case PCI_NVME_AEI_NOTICE_TELEMETRY_CHANGE:
9586dc98349SAndy Fiddaman lid = NVME_LOG_TELEMETRY_CONTROLLER_INITIATED;
9596dc98349SAndy Fiddaman break;
960d7b72f7bSAndy Fiddaman case PCI_NVME_AEI_NOTICE_ANA_CHANGE:
961d7b72f7bSAndy Fiddaman lid = NVME_LOG_ASYMMETRIC_NAMESPACE_ACCESS;
9626dc98349SAndy Fiddaman break;
963d7b72f7bSAndy Fiddaman case PCI_NVME_AEI_NOTICE_PREDICT_LATENCY_CHANGE:
9646dc98349SAndy Fiddaman lid = NVME_LOG_PREDICTABLE_LATENCY_EVENT_AGGREGATE;
9656dc98349SAndy Fiddaman break;
966d7b72f7bSAndy Fiddaman case PCI_NVME_AEI_NOTICE_LBA_STATUS_ALERT:
9676dc98349SAndy Fiddaman lid = NVME_LOG_LBA_STATUS_INFORMATION;
9686dc98349SAndy Fiddaman break;
969d7b72f7bSAndy Fiddaman case PCI_NVME_AEI_NOTICE_ENDURANCE_GROUP_CHANGE:
9706dc98349SAndy Fiddaman lid = NVME_LOG_ENDURANCE_GROUP_EVENT_AGGREGATE;
9716dc98349SAndy Fiddaman break;
9726dc98349SAndy Fiddaman default:
9736dc98349SAndy Fiddaman lid = 0;
9746dc98349SAndy Fiddaman }
9756dc98349SAndy Fiddaman break;
9766dc98349SAndy Fiddaman default:
9776dc98349SAndy Fiddaman /* bad type?!? */
9786dc98349SAndy Fiddaman EPRINTLN("%s unknown AEN type %u", __func__, atype);
9796dc98349SAndy Fiddaman status = NVME_SC_INTERNAL_DEVICE_ERROR;
98059d65d31SAndy Fiddaman lid = 0;
9816dc98349SAndy Fiddaman break;
9826dc98349SAndy Fiddaman }
9836dc98349SAndy Fiddaman
9846dc98349SAndy Fiddaman aer = pci_nvme_aer_get(sc);
9856dc98349SAndy Fiddaman assert(aer != NULL);
9866dc98349SAndy Fiddaman
9876dc98349SAndy Fiddaman DPRINTF("%s: CID=%#x CDW0=%#x", __func__, aer->cid, (lid << 16) | (aen->event_data << 8) | atype);
9886dc98349SAndy Fiddaman pci_nvme_cq_update(sc, &sc->compl_queues[0],
9896dc98349SAndy Fiddaman (lid << 16) | (aen->event_data << 8) | atype, /* cdw0 */
9906dc98349SAndy Fiddaman aer->cid,
9916dc98349SAndy Fiddaman 0, /* SQID */
9926dc98349SAndy Fiddaman status);
9936dc98349SAndy Fiddaman
9946dc98349SAndy Fiddaman aen->event_data = 0;
9956dc98349SAndy Fiddaman aen->posted = false;
9966dc98349SAndy Fiddaman
9976dc98349SAndy Fiddaman pci_generate_msix(sc->nsc_pi, 0);
9986dc98349SAndy Fiddaman }
9996dc98349SAndy Fiddaman }
10006dc98349SAndy Fiddaman
10016dc98349SAndy Fiddaman static void *
aen_thr(void * arg)10026dc98349SAndy Fiddaman aen_thr(void *arg)
10036dc98349SAndy Fiddaman {
10046dc98349SAndy Fiddaman struct pci_nvme_softc *sc;
10056dc98349SAndy Fiddaman
10066dc98349SAndy Fiddaman sc = arg;
10076dc98349SAndy Fiddaman
10086dc98349SAndy Fiddaman pthread_mutex_lock(&sc->aen_mtx);
10096dc98349SAndy Fiddaman for (;;) {
10106dc98349SAndy Fiddaman pci_nvme_aen_process(sc);
10116dc98349SAndy Fiddaman pthread_cond_wait(&sc->aen_cond, &sc->aen_mtx);
10126dc98349SAndy Fiddaman }
10134f3f3e9aSAndy Fiddaman #ifdef __FreeBSD__ /* Smatch spots unreachable code */
10146dc98349SAndy Fiddaman pthread_mutex_unlock(&sc->aen_mtx);
10156dc98349SAndy Fiddaman
10166dc98349SAndy Fiddaman pthread_exit(NULL);
10176dc98349SAndy Fiddaman #endif
10186dc98349SAndy Fiddaman return (NULL);
10196dc98349SAndy Fiddaman }
10206dc98349SAndy Fiddaman
10214c87aefeSPatrick Mooney static void
pci_nvme_reset_locked(struct pci_nvme_softc * sc)10224c87aefeSPatrick Mooney pci_nvme_reset_locked(struct pci_nvme_softc *sc)
10234c87aefeSPatrick Mooney {
10246960cd89SAndy Fiddaman uint32_t i;
10256960cd89SAndy Fiddaman
10266960cd89SAndy Fiddaman DPRINTF("%s", __func__);
10274c87aefeSPatrick Mooney
10284c87aefeSPatrick Mooney sc->regs.cap_lo = (ZERO_BASED(sc->max_qentries) & NVME_CAP_LO_REG_MQES_MASK) |
10294c87aefeSPatrick Mooney (1 << NVME_CAP_LO_REG_CQR_SHIFT) |
10304c87aefeSPatrick Mooney (60 << NVME_CAP_LO_REG_TO_SHIFT);
10314c87aefeSPatrick Mooney
10324c87aefeSPatrick Mooney sc->regs.cap_hi = 1 << NVME_CAP_HI_REG_CSS_NVM_SHIFT;
10334c87aefeSPatrick Mooney
1034d7b72f7bSAndy Fiddaman sc->regs.vs = NVME_REV(1,4); /* NVMe v1.4 */
10354c87aefeSPatrick Mooney
10364c87aefeSPatrick Mooney sc->regs.cc = 0;
10374c87aefeSPatrick Mooney
10386960cd89SAndy Fiddaman assert(sc->submit_queues != NULL);
10394c87aefeSPatrick Mooney
10406960cd89SAndy Fiddaman for (i = 0; i < sc->num_squeues + 1; i++) {
10416960cd89SAndy Fiddaman sc->submit_queues[i].qbase = NULL;
10426960cd89SAndy Fiddaman sc->submit_queues[i].size = 0;
10436960cd89SAndy Fiddaman sc->submit_queues[i].cqid = 0;
10446960cd89SAndy Fiddaman sc->submit_queues[i].tail = 0;
10456960cd89SAndy Fiddaman sc->submit_queues[i].head = 0;
10466960cd89SAndy Fiddaman }
10474c87aefeSPatrick Mooney
10486960cd89SAndy Fiddaman assert(sc->compl_queues != NULL);
10496960cd89SAndy Fiddaman
10506960cd89SAndy Fiddaman for (i = 0; i < sc->num_cqueues + 1; i++) {
10516960cd89SAndy Fiddaman sc->compl_queues[i].qbase = NULL;
10526960cd89SAndy Fiddaman sc->compl_queues[i].size = 0;
10536960cd89SAndy Fiddaman sc->compl_queues[i].tail = 0;
10546960cd89SAndy Fiddaman sc->compl_queues[i].head = 0;
10554c87aefeSPatrick Mooney }
10566960cd89SAndy Fiddaman
10576960cd89SAndy Fiddaman sc->num_q_is_set = false;
10586960cd89SAndy Fiddaman
10596960cd89SAndy Fiddaman pci_nvme_aer_destroy(sc);
10606dc98349SAndy Fiddaman pci_nvme_aen_destroy(sc);
1061d7b72f7bSAndy Fiddaman
1062d7b72f7bSAndy Fiddaman /*
1063d7b72f7bSAndy Fiddaman * Clear CSTS.RDY last to prevent the host from enabling Controller
1064d7b72f7bSAndy Fiddaman * before cleanup completes
1065d7b72f7bSAndy Fiddaman */
1066d7b72f7bSAndy Fiddaman sc->regs.csts = 0;
10674c87aefeSPatrick Mooney }
10684c87aefeSPatrick Mooney
10694c87aefeSPatrick Mooney static void
pci_nvme_reset(struct pci_nvme_softc * sc)10704c87aefeSPatrick Mooney pci_nvme_reset(struct pci_nvme_softc *sc)
10714c87aefeSPatrick Mooney {
10724c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx);
10734c87aefeSPatrick Mooney pci_nvme_reset_locked(sc);
10744c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx);
10754c87aefeSPatrick Mooney }
10764c87aefeSPatrick Mooney
10774f3f3e9aSAndy Fiddaman static int
pci_nvme_init_controller(struct pci_nvme_softc * sc)1078*32640292SAndy Fiddaman pci_nvme_init_controller(struct pci_nvme_softc *sc)
10794c87aefeSPatrick Mooney {
10804c87aefeSPatrick Mooney uint16_t acqs, asqs;
10814c87aefeSPatrick Mooney
10826960cd89SAndy Fiddaman DPRINTF("%s", __func__);
10834c87aefeSPatrick Mooney
10844f3f3e9aSAndy Fiddaman /*
10854f3f3e9aSAndy Fiddaman * NVMe 2.0 states that "enabling a controller while this field is
10864f3f3e9aSAndy Fiddaman * cleared to 0h produces undefined results" for both ACQS and
10874f3f3e9aSAndy Fiddaman * ASQS. If zero, set CFS and do not become ready.
10884f3f3e9aSAndy Fiddaman */
10894f3f3e9aSAndy Fiddaman asqs = ONE_BASED(sc->regs.aqa & NVME_AQA_REG_ASQS_MASK);
10904f3f3e9aSAndy Fiddaman if (asqs < 2) {
10914f3f3e9aSAndy Fiddaman EPRINTLN("%s: illegal ASQS value %#x (aqa=%#x)", __func__,
10924f3f3e9aSAndy Fiddaman asqs - 1, sc->regs.aqa);
10934f3f3e9aSAndy Fiddaman sc->regs.csts |= NVME_CSTS_CFS;
10944f3f3e9aSAndy Fiddaman return (-1);
10954f3f3e9aSAndy Fiddaman }
10964c87aefeSPatrick Mooney sc->submit_queues[0].size = asqs;
1097*32640292SAndy Fiddaman sc->submit_queues[0].qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx,
1098*32640292SAndy Fiddaman sc->regs.asq, sizeof(struct nvme_command) * asqs);
10994f3f3e9aSAndy Fiddaman if (sc->submit_queues[0].qbase == NULL) {
11004f3f3e9aSAndy Fiddaman EPRINTLN("%s: ASQ vm_map_gpa(%lx) failed", __func__,
11014f3f3e9aSAndy Fiddaman sc->regs.asq);
11024f3f3e9aSAndy Fiddaman sc->regs.csts |= NVME_CSTS_CFS;
11034f3f3e9aSAndy Fiddaman return (-1);
11044f3f3e9aSAndy Fiddaman }
11054c87aefeSPatrick Mooney
11066960cd89SAndy Fiddaman DPRINTF("%s mapping Admin-SQ guest 0x%lx, host: %p",
11076960cd89SAndy Fiddaman __func__, sc->regs.asq, sc->submit_queues[0].qbase);
11084c87aefeSPatrick Mooney
11094f3f3e9aSAndy Fiddaman acqs = ONE_BASED((sc->regs.aqa >> NVME_AQA_REG_ACQS_SHIFT) &
11104f3f3e9aSAndy Fiddaman NVME_AQA_REG_ACQS_MASK);
11114f3f3e9aSAndy Fiddaman if (acqs < 2) {
11124f3f3e9aSAndy Fiddaman EPRINTLN("%s: illegal ACQS value %#x (aqa=%#x)", __func__,
11134f3f3e9aSAndy Fiddaman acqs - 1, sc->regs.aqa);
11144f3f3e9aSAndy Fiddaman sc->regs.csts |= NVME_CSTS_CFS;
11154f3f3e9aSAndy Fiddaman return (-1);
11164f3f3e9aSAndy Fiddaman }
11174c87aefeSPatrick Mooney sc->compl_queues[0].size = acqs;
1118*32640292SAndy Fiddaman sc->compl_queues[0].qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx,
1119*32640292SAndy Fiddaman sc->regs.acq, sizeof(struct nvme_completion) * acqs);
11204f3f3e9aSAndy Fiddaman if (sc->compl_queues[0].qbase == NULL) {
11214f3f3e9aSAndy Fiddaman EPRINTLN("%s: ACQ vm_map_gpa(%lx) failed", __func__,
11224f3f3e9aSAndy Fiddaman sc->regs.acq);
11234f3f3e9aSAndy Fiddaman sc->regs.csts |= NVME_CSTS_CFS;
11244f3f3e9aSAndy Fiddaman return (-1);
11254f3f3e9aSAndy Fiddaman }
11266960cd89SAndy Fiddaman sc->compl_queues[0].intr_en = NVME_CQ_INTEN;
11276960cd89SAndy Fiddaman
11286960cd89SAndy Fiddaman DPRINTF("%s mapping Admin-CQ guest 0x%lx, host: %p",
11296960cd89SAndy Fiddaman __func__, sc->regs.acq, sc->compl_queues[0].qbase);
11304f3f3e9aSAndy Fiddaman
11314f3f3e9aSAndy Fiddaman return (0);
11324c87aefeSPatrick Mooney }
11334c87aefeSPatrick Mooney
11344c87aefeSPatrick Mooney static int
nvme_prp_memcpy(struct vmctx * ctx,uint64_t prp1,uint64_t prp2,uint8_t * b,size_t len,enum nvme_copy_dir dir)1135154972afSPatrick Mooney nvme_prp_memcpy(struct vmctx *ctx, uint64_t prp1, uint64_t prp2, uint8_t *b,
1136154972afSPatrick Mooney size_t len, enum nvme_copy_dir dir)
11374c87aefeSPatrick Mooney {
1138154972afSPatrick Mooney uint8_t *p;
11394c87aefeSPatrick Mooney size_t bytes;
11404c87aefeSPatrick Mooney
11414c87aefeSPatrick Mooney if (len > (8 * 1024)) {
11424c87aefeSPatrick Mooney return (-1);
11434c87aefeSPatrick Mooney }
11444c87aefeSPatrick Mooney
11454c87aefeSPatrick Mooney /* Copy from the start of prp1 to the end of the physical page */
11464c87aefeSPatrick Mooney bytes = PAGE_SIZE - (prp1 & PAGE_MASK);
11474c87aefeSPatrick Mooney bytes = MIN(bytes, len);
11484c87aefeSPatrick Mooney
1149154972afSPatrick Mooney p = vm_map_gpa(ctx, prp1, bytes);
1150154972afSPatrick Mooney if (p == NULL) {
11514c87aefeSPatrick Mooney return (-1);
11524c87aefeSPatrick Mooney }
11534c87aefeSPatrick Mooney
1154154972afSPatrick Mooney if (dir == NVME_COPY_TO_PRP)
1155154972afSPatrick Mooney memcpy(p, b, bytes);
1156154972afSPatrick Mooney else
1157154972afSPatrick Mooney memcpy(b, p, bytes);
11584c87aefeSPatrick Mooney
1159154972afSPatrick Mooney b += bytes;
11604c87aefeSPatrick Mooney
11614c87aefeSPatrick Mooney len -= bytes;
11624c87aefeSPatrick Mooney if (len == 0) {
11634c87aefeSPatrick Mooney return (0);
11644c87aefeSPatrick Mooney }
11654c87aefeSPatrick Mooney
11664c87aefeSPatrick Mooney len = MIN(len, PAGE_SIZE);
11674c87aefeSPatrick Mooney
1168154972afSPatrick Mooney p = vm_map_gpa(ctx, prp2, len);
1169154972afSPatrick Mooney if (p == NULL) {
11704c87aefeSPatrick Mooney return (-1);
11714c87aefeSPatrick Mooney }
11724c87aefeSPatrick Mooney
1173154972afSPatrick Mooney if (dir == NVME_COPY_TO_PRP)
1174154972afSPatrick Mooney memcpy(p, b, len);
1175154972afSPatrick Mooney else
1176154972afSPatrick Mooney memcpy(b, p, len);
11774c87aefeSPatrick Mooney
11784c87aefeSPatrick Mooney return (0);
11794c87aefeSPatrick Mooney }
11804c87aefeSPatrick Mooney
11816960cd89SAndy Fiddaman /*
11826960cd89SAndy Fiddaman * Write a Completion Queue Entry update
11836960cd89SAndy Fiddaman *
11846960cd89SAndy Fiddaman * Write the completion and update the doorbell value
11856960cd89SAndy Fiddaman */
11866960cd89SAndy Fiddaman static void
pci_nvme_cq_update(struct pci_nvme_softc * sc,struct nvme_completion_queue * cq,uint32_t cdw0,uint16_t cid,uint16_t sqid,uint16_t status)11876960cd89SAndy Fiddaman pci_nvme_cq_update(struct pci_nvme_softc *sc,
11886960cd89SAndy Fiddaman struct nvme_completion_queue *cq,
11896960cd89SAndy Fiddaman uint32_t cdw0,
11906960cd89SAndy Fiddaman uint16_t cid,
11916960cd89SAndy Fiddaman uint16_t sqid,
11926960cd89SAndy Fiddaman uint16_t status)
11936960cd89SAndy Fiddaman {
11946960cd89SAndy Fiddaman struct nvme_submission_queue *sq = &sc->submit_queues[sqid];
11956960cd89SAndy Fiddaman struct nvme_completion *cqe;
11966960cd89SAndy Fiddaman
11976960cd89SAndy Fiddaman assert(cq->qbase != NULL);
11986960cd89SAndy Fiddaman
11996960cd89SAndy Fiddaman pthread_mutex_lock(&cq->mtx);
12006960cd89SAndy Fiddaman
12016960cd89SAndy Fiddaman cqe = &cq->qbase[cq->tail];
12026960cd89SAndy Fiddaman
12036960cd89SAndy Fiddaman /* Flip the phase bit */
12046960cd89SAndy Fiddaman status |= (cqe->status ^ NVME_STATUS_P) & NVME_STATUS_P_MASK;
12056960cd89SAndy Fiddaman
12066960cd89SAndy Fiddaman cqe->cdw0 = cdw0;
12076960cd89SAndy Fiddaman cqe->sqhd = sq->head;
12086960cd89SAndy Fiddaman cqe->sqid = sqid;
12096960cd89SAndy Fiddaman cqe->cid = cid;
12106960cd89SAndy Fiddaman cqe->status = status;
12116960cd89SAndy Fiddaman
12126960cd89SAndy Fiddaman cq->tail++;
12136960cd89SAndy Fiddaman if (cq->tail >= cq->size) {
12146960cd89SAndy Fiddaman cq->tail = 0;
12156960cd89SAndy Fiddaman }
12166960cd89SAndy Fiddaman
12176960cd89SAndy Fiddaman pthread_mutex_unlock(&cq->mtx);
12186960cd89SAndy Fiddaman }
12196960cd89SAndy Fiddaman
12204c87aefeSPatrick Mooney static int
nvme_opc_delete_io_sq(struct pci_nvme_softc * sc,struct nvme_command * command,struct nvme_completion * compl)12214c87aefeSPatrick Mooney nvme_opc_delete_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
12224c87aefeSPatrick Mooney struct nvme_completion* compl)
12234c87aefeSPatrick Mooney {
12244c87aefeSPatrick Mooney uint16_t qid = command->cdw10 & 0xffff;
12254c87aefeSPatrick Mooney
12266960cd89SAndy Fiddaman DPRINTF("%s DELETE_IO_SQ %u", __func__, qid);
12276960cd89SAndy Fiddaman if (qid == 0 || qid > sc->num_squeues ||
12286960cd89SAndy Fiddaman (sc->submit_queues[qid].qbase == NULL)) {
12296960cd89SAndy Fiddaman WPRINTF("%s NOT PERMITTED queue id %u / num_squeues %u",
12306960cd89SAndy Fiddaman __func__, qid, sc->num_squeues);
12314c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
12324c87aefeSPatrick Mooney NVME_SC_INVALID_QUEUE_IDENTIFIER);
12334c87aefeSPatrick Mooney return (1);
12344c87aefeSPatrick Mooney }
12354c87aefeSPatrick Mooney
12364c87aefeSPatrick Mooney sc->submit_queues[qid].qbase = NULL;
12376960cd89SAndy Fiddaman sc->submit_queues[qid].cqid = 0;
12384c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
12394c87aefeSPatrick Mooney return (1);
12404c87aefeSPatrick Mooney }
12414c87aefeSPatrick Mooney
12424c87aefeSPatrick Mooney static int
nvme_opc_create_io_sq(struct pci_nvme_softc * sc,struct nvme_command * command,struct nvme_completion * compl)12434c87aefeSPatrick Mooney nvme_opc_create_io_sq(struct pci_nvme_softc* sc, struct nvme_command* command,
12444c87aefeSPatrick Mooney struct nvme_completion* compl)
12454c87aefeSPatrick Mooney {
12464c87aefeSPatrick Mooney if (command->cdw11 & NVME_CMD_CDW11_PC) {
12474c87aefeSPatrick Mooney uint16_t qid = command->cdw10 & 0xffff;
12484c87aefeSPatrick Mooney struct nvme_submission_queue *nsq;
12494c87aefeSPatrick Mooney
12506960cd89SAndy Fiddaman if ((qid == 0) || (qid > sc->num_squeues) ||
12516960cd89SAndy Fiddaman (sc->submit_queues[qid].qbase != NULL)) {
12526960cd89SAndy Fiddaman WPRINTF("%s queue index %u > num_squeues %u",
12536960cd89SAndy Fiddaman __func__, qid, sc->num_squeues);
12544c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status,
12554c87aefeSPatrick Mooney NVME_SCT_COMMAND_SPECIFIC,
12564c87aefeSPatrick Mooney NVME_SC_INVALID_QUEUE_IDENTIFIER);
12574c87aefeSPatrick Mooney return (1);
12584c87aefeSPatrick Mooney }
12594c87aefeSPatrick Mooney
12604c87aefeSPatrick Mooney nsq = &sc->submit_queues[qid];
12614c87aefeSPatrick Mooney nsq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff);
12626960cd89SAndy Fiddaman DPRINTF("%s size=%u (max=%u)", __func__, nsq->size, sc->max_qentries);
12636960cd89SAndy Fiddaman if ((nsq->size < 2) || (nsq->size > sc->max_qentries)) {
12646960cd89SAndy Fiddaman /*
12656960cd89SAndy Fiddaman * Queues must specify at least two entries
12666960cd89SAndy Fiddaman * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to
12676960cd89SAndy Fiddaman * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec
12686960cd89SAndy Fiddaman */
12696960cd89SAndy Fiddaman pci_nvme_status_tc(&compl->status,
12706960cd89SAndy Fiddaman NVME_SCT_COMMAND_SPECIFIC,
12716960cd89SAndy Fiddaman NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED);
12726960cd89SAndy Fiddaman return (1);
12736960cd89SAndy Fiddaman }
12746960cd89SAndy Fiddaman nsq->head = nsq->tail = 0;
12754c87aefeSPatrick Mooney
12764c87aefeSPatrick Mooney nsq->cqid = (command->cdw11 >> 16) & 0xffff;
12776960cd89SAndy Fiddaman if ((nsq->cqid == 0) || (nsq->cqid > sc->num_cqueues)) {
12786960cd89SAndy Fiddaman pci_nvme_status_tc(&compl->status,
12796960cd89SAndy Fiddaman NVME_SCT_COMMAND_SPECIFIC,
12806960cd89SAndy Fiddaman NVME_SC_INVALID_QUEUE_IDENTIFIER);
12816960cd89SAndy Fiddaman return (1);
12826960cd89SAndy Fiddaman }
12836960cd89SAndy Fiddaman
12846960cd89SAndy Fiddaman if (sc->compl_queues[nsq->cqid].qbase == NULL) {
12856960cd89SAndy Fiddaman pci_nvme_status_tc(&compl->status,
12866960cd89SAndy Fiddaman NVME_SCT_COMMAND_SPECIFIC,
12876960cd89SAndy Fiddaman NVME_SC_COMPLETION_QUEUE_INVALID);
12886960cd89SAndy Fiddaman return (1);
12896960cd89SAndy Fiddaman }
12906960cd89SAndy Fiddaman
12914c87aefeSPatrick Mooney nsq->qpriority = (command->cdw11 >> 1) & 0x03;
12924c87aefeSPatrick Mooney
12936960cd89SAndy Fiddaman nsq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
12946960cd89SAndy Fiddaman sizeof(struct nvme_command) * (size_t)nsq->size);
12956960cd89SAndy Fiddaman
12966960cd89SAndy Fiddaman DPRINTF("%s sq %u size %u gaddr %p cqid %u", __func__,
12976960cd89SAndy Fiddaman qid, nsq->size, nsq->qbase, nsq->cqid);
12984c87aefeSPatrick Mooney
12994c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
13004c87aefeSPatrick Mooney
13016960cd89SAndy Fiddaman DPRINTF("%s completed creating IOSQ qid %u",
13026960cd89SAndy Fiddaman __func__, qid);
13034c87aefeSPatrick Mooney } else {
13046dc98349SAndy Fiddaman /*
13054c87aefeSPatrick Mooney * Guest sent non-cont submission queue request.
13064c87aefeSPatrick Mooney * This setting is unsupported by this emulation.
13074c87aefeSPatrick Mooney */
13086960cd89SAndy Fiddaman WPRINTF("%s unsupported non-contig (list-based) "
13096960cd89SAndy Fiddaman "create i/o submission queue", __func__);
13104c87aefeSPatrick Mooney
13114c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
13124c87aefeSPatrick Mooney }
13134c87aefeSPatrick Mooney return (1);
13144c87aefeSPatrick Mooney }
13154c87aefeSPatrick Mooney
13164c87aefeSPatrick Mooney static int
nvme_opc_delete_io_cq(struct pci_nvme_softc * sc,struct nvme_command * command,struct nvme_completion * compl)13174c87aefeSPatrick Mooney nvme_opc_delete_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
13184c87aefeSPatrick Mooney struct nvme_completion* compl)
13194c87aefeSPatrick Mooney {
13204c87aefeSPatrick Mooney uint16_t qid = command->cdw10 & 0xffff;
13216960cd89SAndy Fiddaman uint16_t sqid;
13224c87aefeSPatrick Mooney
13236960cd89SAndy Fiddaman DPRINTF("%s DELETE_IO_CQ %u", __func__, qid);
13246960cd89SAndy Fiddaman if (qid == 0 || qid > sc->num_cqueues ||
13256960cd89SAndy Fiddaman (sc->compl_queues[qid].qbase == NULL)) {
13266960cd89SAndy Fiddaman WPRINTF("%s queue index %u / num_cqueues %u",
13276960cd89SAndy Fiddaman __func__, qid, sc->num_cqueues);
13284c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
13294c87aefeSPatrick Mooney NVME_SC_INVALID_QUEUE_IDENTIFIER);
13304c87aefeSPatrick Mooney return (1);
13314c87aefeSPatrick Mooney }
13324c87aefeSPatrick Mooney
13336960cd89SAndy Fiddaman /* Deleting an Active CQ is an error */
13346960cd89SAndy Fiddaman for (sqid = 1; sqid < sc->num_squeues + 1; sqid++)
13356960cd89SAndy Fiddaman if (sc->submit_queues[sqid].cqid == qid) {
13366960cd89SAndy Fiddaman pci_nvme_status_tc(&compl->status,
13376960cd89SAndy Fiddaman NVME_SCT_COMMAND_SPECIFIC,
13386960cd89SAndy Fiddaman NVME_SC_INVALID_QUEUE_DELETION);
13396960cd89SAndy Fiddaman return (1);
13406960cd89SAndy Fiddaman }
13416960cd89SAndy Fiddaman
13424c87aefeSPatrick Mooney sc->compl_queues[qid].qbase = NULL;
13434c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
13444c87aefeSPatrick Mooney return (1);
13454c87aefeSPatrick Mooney }
13464c87aefeSPatrick Mooney
13474c87aefeSPatrick Mooney static int
nvme_opc_create_io_cq(struct pci_nvme_softc * sc,struct nvme_command * command,struct nvme_completion * compl)13484c87aefeSPatrick Mooney nvme_opc_create_io_cq(struct pci_nvme_softc* sc, struct nvme_command* command,
13494c87aefeSPatrick Mooney struct nvme_completion* compl)
13504c87aefeSPatrick Mooney {
13516960cd89SAndy Fiddaman struct nvme_completion_queue *ncq;
13526960cd89SAndy Fiddaman uint16_t qid = command->cdw10 & 0xffff;
13534c87aefeSPatrick Mooney
13546960cd89SAndy Fiddaman /* Only support Physically Contiguous queues */
13556960cd89SAndy Fiddaman if ((command->cdw11 & NVME_CMD_CDW11_PC) == 0) {
13566960cd89SAndy Fiddaman WPRINTF("%s unsupported non-contig (list-based) "
13576960cd89SAndy Fiddaman "create i/o completion queue",
13586960cd89SAndy Fiddaman __func__);
13594c87aefeSPatrick Mooney
13606960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
13616960cd89SAndy Fiddaman return (1);
13626960cd89SAndy Fiddaman }
13634c87aefeSPatrick Mooney
13646960cd89SAndy Fiddaman if ((qid == 0) || (qid > sc->num_cqueues) ||
13656960cd89SAndy Fiddaman (sc->compl_queues[qid].qbase != NULL)) {
13666960cd89SAndy Fiddaman WPRINTF("%s queue index %u > num_cqueues %u",
13676960cd89SAndy Fiddaman __func__, qid, sc->num_cqueues);
13686960cd89SAndy Fiddaman pci_nvme_status_tc(&compl->status,
13696960cd89SAndy Fiddaman NVME_SCT_COMMAND_SPECIFIC,
13706960cd89SAndy Fiddaman NVME_SC_INVALID_QUEUE_IDENTIFIER);
13716960cd89SAndy Fiddaman return (1);
13726960cd89SAndy Fiddaman }
13736960cd89SAndy Fiddaman
13746960cd89SAndy Fiddaman ncq = &sc->compl_queues[qid];
13756960cd89SAndy Fiddaman ncq->intr_en = (command->cdw11 & NVME_CMD_CDW11_IEN) >> 1;
13766960cd89SAndy Fiddaman ncq->intr_vec = (command->cdw11 >> 16) & 0xffff;
13776960cd89SAndy Fiddaman if (ncq->intr_vec > (sc->max_queues + 1)) {
13786960cd89SAndy Fiddaman pci_nvme_status_tc(&compl->status,
13796960cd89SAndy Fiddaman NVME_SCT_COMMAND_SPECIFIC,
13806960cd89SAndy Fiddaman NVME_SC_INVALID_INTERRUPT_VECTOR);
13816960cd89SAndy Fiddaman return (1);
13826960cd89SAndy Fiddaman }
13834c87aefeSPatrick Mooney
13846960cd89SAndy Fiddaman ncq->size = ONE_BASED((command->cdw10 >> 16) & 0xffff);
13856960cd89SAndy Fiddaman if ((ncq->size < 2) || (ncq->size > sc->max_qentries)) {
13866960cd89SAndy Fiddaman /*
13876960cd89SAndy Fiddaman * Queues must specify at least two entries
13886960cd89SAndy Fiddaman * NOTE: "MAXIMUM QUEUE SIZE EXCEEDED" was renamed to
13896960cd89SAndy Fiddaman * "INVALID QUEUE SIZE" in the NVM Express 1.3 Spec
13904c87aefeSPatrick Mooney */
13916960cd89SAndy Fiddaman pci_nvme_status_tc(&compl->status,
13926960cd89SAndy Fiddaman NVME_SCT_COMMAND_SPECIFIC,
13936960cd89SAndy Fiddaman NVME_SC_MAXIMUM_QUEUE_SIZE_EXCEEDED);
13946960cd89SAndy Fiddaman return (1);
13954c87aefeSPatrick Mooney }
13966960cd89SAndy Fiddaman ncq->head = ncq->tail = 0;
13976960cd89SAndy Fiddaman ncq->qbase = vm_map_gpa(sc->nsc_pi->pi_vmctx,
13986960cd89SAndy Fiddaman command->prp1,
13996960cd89SAndy Fiddaman sizeof(struct nvme_command) * (size_t)ncq->size);
14006960cd89SAndy Fiddaman
14016960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
14026960cd89SAndy Fiddaman
14034c87aefeSPatrick Mooney
14044c87aefeSPatrick Mooney return (1);
14054c87aefeSPatrick Mooney }
14064c87aefeSPatrick Mooney
14074c87aefeSPatrick Mooney static int
nvme_opc_get_log_page(struct pci_nvme_softc * sc,struct nvme_command * command,struct nvme_completion * compl)14084c87aefeSPatrick Mooney nvme_opc_get_log_page(struct pci_nvme_softc* sc, struct nvme_command* command,
14094c87aefeSPatrick Mooney struct nvme_completion* compl)
14104c87aefeSPatrick Mooney {
1411d7b72f7bSAndy Fiddaman uint64_t logoff;
14122b948146SAndy Fiddaman uint32_t logsize;
14134f3f3e9aSAndy Fiddaman uint8_t logpage;
14144c87aefeSPatrick Mooney
14154c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
14164c87aefeSPatrick Mooney
14176960cd89SAndy Fiddaman /*
14186960cd89SAndy Fiddaman * Command specifies the number of dwords to return in fields NUMDU
14196960cd89SAndy Fiddaman * and NUMDL. This is a zero-based value.
14206960cd89SAndy Fiddaman */
14214f3f3e9aSAndy Fiddaman logpage = command->cdw10 & 0xFF;
14226960cd89SAndy Fiddaman logsize = ((command->cdw11 << 16) | (command->cdw10 >> 16)) + 1;
14236960cd89SAndy Fiddaman logsize *= sizeof(uint32_t);
1424d7b72f7bSAndy Fiddaman logoff = ((uint64_t)(command->cdw13) << 32) | command->cdw12;
14256960cd89SAndy Fiddaman
14264f3f3e9aSAndy Fiddaman DPRINTF("%s log page %u len %u", __func__, logpage, logsize);
14274f3f3e9aSAndy Fiddaman
14284c87aefeSPatrick Mooney switch (logpage) {
14294c87aefeSPatrick Mooney case NVME_LOG_ERROR:
1430d7b72f7bSAndy Fiddaman if (logoff >= sizeof(sc->err_log)) {
1431d7b72f7bSAndy Fiddaman pci_nvme_status_genc(&compl->status,
1432d7b72f7bSAndy Fiddaman NVME_SC_INVALID_FIELD);
1433d7b72f7bSAndy Fiddaman break;
1434d7b72f7bSAndy Fiddaman }
1435d7b72f7bSAndy Fiddaman
14364c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1437d7b72f7bSAndy Fiddaman command->prp2, (uint8_t *)&sc->err_log + logoff,
1438d7b72f7bSAndy Fiddaman MIN(logsize - logoff, sizeof(sc->err_log)),
1439154972afSPatrick Mooney NVME_COPY_TO_PRP);
14404c87aefeSPatrick Mooney break;
14414c87aefeSPatrick Mooney case NVME_LOG_HEALTH_INFORMATION:
1442d7b72f7bSAndy Fiddaman if (logoff >= sizeof(sc->health_log)) {
1443d7b72f7bSAndy Fiddaman pci_nvme_status_genc(&compl->status,
1444d7b72f7bSAndy Fiddaman NVME_SC_INVALID_FIELD);
1445d7b72f7bSAndy Fiddaman break;
1446d7b72f7bSAndy Fiddaman }
1447d7b72f7bSAndy Fiddaman
14486960cd89SAndy Fiddaman pthread_mutex_lock(&sc->mtx);
14496960cd89SAndy Fiddaman memcpy(&sc->health_log.data_units_read, &sc->read_data_units,
14506960cd89SAndy Fiddaman sizeof(sc->health_log.data_units_read));
14516960cd89SAndy Fiddaman memcpy(&sc->health_log.data_units_written, &sc->write_data_units,
14526960cd89SAndy Fiddaman sizeof(sc->health_log.data_units_written));
14536960cd89SAndy Fiddaman memcpy(&sc->health_log.host_read_commands, &sc->read_commands,
14546960cd89SAndy Fiddaman sizeof(sc->health_log.host_read_commands));
14556960cd89SAndy Fiddaman memcpy(&sc->health_log.host_write_commands, &sc->write_commands,
14566960cd89SAndy Fiddaman sizeof(sc->health_log.host_write_commands));
14576960cd89SAndy Fiddaman pthread_mutex_unlock(&sc->mtx);
14586960cd89SAndy Fiddaman
14594c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1460d7b72f7bSAndy Fiddaman command->prp2, (uint8_t *)&sc->health_log + logoff,
1461d7b72f7bSAndy Fiddaman MIN(logsize - logoff, sizeof(sc->health_log)),
1462154972afSPatrick Mooney NVME_COPY_TO_PRP);
14634c87aefeSPatrick Mooney break;
14644c87aefeSPatrick Mooney case NVME_LOG_FIRMWARE_SLOT:
1465d7b72f7bSAndy Fiddaman if (logoff >= sizeof(sc->fw_log)) {
1466d7b72f7bSAndy Fiddaman pci_nvme_status_genc(&compl->status,
1467d7b72f7bSAndy Fiddaman NVME_SC_INVALID_FIELD);
1468d7b72f7bSAndy Fiddaman break;
1469d7b72f7bSAndy Fiddaman }
1470d7b72f7bSAndy Fiddaman
14714c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1472d7b72f7bSAndy Fiddaman command->prp2, (uint8_t *)&sc->fw_log + logoff,
1473d7b72f7bSAndy Fiddaman MIN(logsize - logoff, sizeof(sc->fw_log)),
1474154972afSPatrick Mooney NVME_COPY_TO_PRP);
14754c87aefeSPatrick Mooney break;
14766dc98349SAndy Fiddaman case NVME_LOG_CHANGED_NAMESPACE:
1477d7b72f7bSAndy Fiddaman if (logoff >= sizeof(sc->ns_log)) {
1478d7b72f7bSAndy Fiddaman pci_nvme_status_genc(&compl->status,
1479d7b72f7bSAndy Fiddaman NVME_SC_INVALID_FIELD);
1480d7b72f7bSAndy Fiddaman break;
1481d7b72f7bSAndy Fiddaman }
1482d7b72f7bSAndy Fiddaman
14836dc98349SAndy Fiddaman nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1484d7b72f7bSAndy Fiddaman command->prp2, (uint8_t *)&sc->ns_log + logoff,
1485d7b72f7bSAndy Fiddaman MIN(logsize - logoff, sizeof(sc->ns_log)),
14866dc98349SAndy Fiddaman NVME_COPY_TO_PRP);
14876dc98349SAndy Fiddaman memset(&sc->ns_log, 0, sizeof(sc->ns_log));
14886dc98349SAndy Fiddaman break;
14894c87aefeSPatrick Mooney default:
14906960cd89SAndy Fiddaman DPRINTF("%s get log page %x command not supported",
14916960cd89SAndy Fiddaman __func__, logpage);
14924c87aefeSPatrick Mooney
14934c87aefeSPatrick Mooney pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
14944c87aefeSPatrick Mooney NVME_SC_INVALID_LOG_PAGE);
14954c87aefeSPatrick Mooney }
14964c87aefeSPatrick Mooney
14974c87aefeSPatrick Mooney return (1);
14984c87aefeSPatrick Mooney }
14994c87aefeSPatrick Mooney
15004c87aefeSPatrick Mooney static int
nvme_opc_identify(struct pci_nvme_softc * sc,struct nvme_command * command,struct nvme_completion * compl)15014c87aefeSPatrick Mooney nvme_opc_identify(struct pci_nvme_softc* sc, struct nvme_command* command,
15024c87aefeSPatrick Mooney struct nvme_completion* compl)
15034c87aefeSPatrick Mooney {
15044c87aefeSPatrick Mooney void *dest;
15052b948146SAndy Fiddaman uint16_t status;
15062b948146SAndy Fiddaman
15076960cd89SAndy Fiddaman DPRINTF("%s identify 0x%x nsid 0x%x", __func__,
15086960cd89SAndy Fiddaman command->cdw10 & 0xFF, command->nsid);
15096960cd89SAndy Fiddaman
151059d65d31SAndy Fiddaman status = 0;
15116960cd89SAndy Fiddaman pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
15124c87aefeSPatrick Mooney
15134c87aefeSPatrick Mooney switch (command->cdw10 & 0xFF) {
15144c87aefeSPatrick Mooney case 0x00: /* return Identify Namespace data structure */
1515d7b72f7bSAndy Fiddaman /* Global NS only valid with NS Management */
1516d7b72f7bSAndy Fiddaman if (command->nsid == NVME_GLOBAL_NAMESPACE_TAG) {
1517d7b72f7bSAndy Fiddaman pci_nvme_status_genc(&status,
1518d7b72f7bSAndy Fiddaman NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
1519d7b72f7bSAndy Fiddaman break;
1520d7b72f7bSAndy Fiddaman }
15214c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
1522154972afSPatrick Mooney command->prp2, (uint8_t *)&sc->nsdata, sizeof(sc->nsdata),
1523154972afSPatrick Mooney NVME_COPY_TO_PRP);
15244c87aefeSPatrick Mooney break;
15254c87aefeSPatrick Mooney case 0x01: /* return Identify Controller data structure */
15264c87aefeSPatrick Mooney nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, command->prp1,
15274c87aefeSPatrick Mooney command->prp2, (uint8_t *)&sc->ctrldata,
1528154972afSPatrick Mooney sizeof(sc->ctrldata),
1529154972afSPatrick Mooney NVME_COPY_TO_PRP);
15304c87aefeSPatrick Mooney break;
15314c87aefeSPatrick Mooney case 0x02: /* list of 1024 active NSIDs > CDW1.NSID */
15324c87aefeSPatrick Mooney dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
15334c87aefeSPatrick Mooney sizeof(uint32_t) * 1024);
15346960cd89SAndy Fiddaman /* All unused entries shall be zero */
15354f3f3e9aSAndy Fiddaman memset(dest, 0, sizeof(uint32_t) * 1024);
15364c87aefeSPatrick Mooney ((uint32_t *)dest)[0] = 1;
15374c87aefeSPatrick Mooney break;
15384c87aefeSPatrick Mooney case 0x03: /* list of NSID structures in CDW1.NSID, 4096 bytes */
15396960cd89SAndy Fiddaman if (command->nsid != 1) {
15406960cd89SAndy Fiddaman pci_nvme_status_genc(&status,
15416960cd89SAndy Fiddaman NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
15426960cd89SAndy Fiddaman break;
15436960cd89SAndy Fiddaman }
15446960cd89SAndy Fiddaman dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
15456960cd89SAndy Fiddaman sizeof(uint32_t) * 1024);
15466960cd89SAndy Fiddaman /* All bytes after the descriptor shall be zero */
15474f3f3e9aSAndy Fiddaman memset(dest, 0, sizeof(uint32_t) * 1024);
15486960cd89SAndy Fiddaman
15496960cd89SAndy Fiddaman /* Return NIDT=1 (i.e. EUI64) descriptor */
15506960cd89SAndy Fiddaman ((uint8_t *)dest)[0] = 1;
15516960cd89SAndy Fiddaman ((uint8_t *)dest)[1] = sizeof(uint64_t);
15524f3f3e9aSAndy Fiddaman memcpy(((uint8_t *)dest) + 4, sc->nsdata.eui64, sizeof(uint64_t));
15534f3f3e9aSAndy Fiddaman break;
15544f3f3e9aSAndy Fiddaman case 0x13:
15554f3f3e9aSAndy Fiddaman /*
15564f3f3e9aSAndy Fiddaman * Controller list is optional but used by UNH tests. Return
15574f3f3e9aSAndy Fiddaman * a valid but empty list.
15584f3f3e9aSAndy Fiddaman */
15594f3f3e9aSAndy Fiddaman dest = vm_map_gpa(sc->nsc_pi->pi_vmctx, command->prp1,
15604f3f3e9aSAndy Fiddaman sizeof(uint16_t) * 2048);
15614f3f3e9aSAndy Fiddaman memset(dest, 0, sizeof(uint16_t) * 2048);
15626960cd89SAndy Fiddaman break;
15634c87aefeSPatrick Mooney default:
15646960cd89SAndy Fiddaman DPRINTF("%s unsupported identify command requested 0x%x",
15656960cd89SAndy Fiddaman __func__, command->cdw10 & 0xFF);
15666960cd89SAndy Fiddaman pci_nvme_status_genc(&status, NVME_SC_INVALID_FIELD);
15676960cd89SAndy Fiddaman break;
15684c87aefeSPatrick Mooney }
15694c87aefeSPatrick Mooney
15706960cd89SAndy Fiddaman compl->status = status;
15714c87aefeSPatrick Mooney return (1);
15724c87aefeSPatrick Mooney }
15734c87aefeSPatrick Mooney
15746960cd89SAndy Fiddaman static const char *
nvme_fid_to_name(uint8_t fid)15756960cd89SAndy Fiddaman nvme_fid_to_name(uint8_t fid)
15766960cd89SAndy Fiddaman {
15776960cd89SAndy Fiddaman const char *name;
15786960cd89SAndy Fiddaman
15796960cd89SAndy Fiddaman switch (fid) {
15806960cd89SAndy Fiddaman case NVME_FEAT_ARBITRATION:
15816960cd89SAndy Fiddaman name = "Arbitration";
15826960cd89SAndy Fiddaman break;
15836960cd89SAndy Fiddaman case NVME_FEAT_POWER_MANAGEMENT:
15846960cd89SAndy Fiddaman name = "Power Management";
15856960cd89SAndy Fiddaman break;
15866960cd89SAndy Fiddaman case NVME_FEAT_LBA_RANGE_TYPE:
15876960cd89SAndy Fiddaman name = "LBA Range Type";
15886960cd89SAndy Fiddaman break;
15896960cd89SAndy Fiddaman case NVME_FEAT_TEMPERATURE_THRESHOLD:
15906960cd89SAndy Fiddaman name = "Temperature Threshold";
15916960cd89SAndy Fiddaman break;
15926960cd89SAndy Fiddaman case NVME_FEAT_ERROR_RECOVERY:
15936960cd89SAndy Fiddaman name = "Error Recovery";
15946960cd89SAndy Fiddaman break;
15956960cd89SAndy Fiddaman case NVME_FEAT_VOLATILE_WRITE_CACHE:
15966960cd89SAndy Fiddaman name = "Volatile Write Cache";
15976960cd89SAndy Fiddaman break;
15986960cd89SAndy Fiddaman case NVME_FEAT_NUMBER_OF_QUEUES:
15996960cd89SAndy Fiddaman name = "Number of Queues";
16006960cd89SAndy Fiddaman break;
16016960cd89SAndy Fiddaman case NVME_FEAT_INTERRUPT_COALESCING:
16026960cd89SAndy Fiddaman name = "Interrupt Coalescing";
16036960cd89SAndy Fiddaman break;
16046960cd89SAndy Fiddaman case NVME_FEAT_INTERRUPT_VECTOR_CONFIGURATION:
16056960cd89SAndy Fiddaman name = "Interrupt Vector Configuration";
16066960cd89SAndy Fiddaman break;
16076960cd89SAndy Fiddaman case NVME_FEAT_WRITE_ATOMICITY:
16086960cd89SAndy Fiddaman name = "Write Atomicity Normal";
16096960cd89SAndy Fiddaman break;
16106960cd89SAndy Fiddaman case NVME_FEAT_ASYNC_EVENT_CONFIGURATION:
16116960cd89SAndy Fiddaman name = "Asynchronous Event Configuration";
16126960cd89SAndy Fiddaman break;
16136960cd89SAndy Fiddaman case NVME_FEAT_AUTONOMOUS_POWER_STATE_TRANSITION:
16146960cd89SAndy Fiddaman name = "Autonomous Power State Transition";
16156960cd89SAndy Fiddaman break;
16166960cd89SAndy Fiddaman case NVME_FEAT_HOST_MEMORY_BUFFER:
16176960cd89SAndy Fiddaman name = "Host Memory Buffer";
16186960cd89SAndy Fiddaman break;
16196960cd89SAndy Fiddaman case NVME_FEAT_TIMESTAMP:
16206960cd89SAndy Fiddaman name = "Timestamp";
16216960cd89SAndy Fiddaman break;
16226960cd89SAndy Fiddaman case NVME_FEAT_KEEP_ALIVE_TIMER:
16236960cd89SAndy Fiddaman name = "Keep Alive Timer";
16246960cd89SAndy Fiddaman break;
16256960cd89SAndy Fiddaman case NVME_FEAT_HOST_CONTROLLED_THERMAL_MGMT:
16266960cd89SAndy Fiddaman name = "Host Controlled Thermal Management";
16276960cd89SAndy Fiddaman break;
16286960cd89SAndy Fiddaman case NVME_FEAT_NON_OP_POWER_STATE_CONFIG:
16296960cd89SAndy Fiddaman name = "Non-Operation Power State Config";
16306960cd89SAndy Fiddaman break;
16316960cd89SAndy Fiddaman case NVME_FEAT_READ_RECOVERY_LEVEL_CONFIG:
16326960cd89SAndy Fiddaman name = "Read Recovery Level Config";
16336960cd89SAndy Fiddaman break;
16346960cd89SAndy Fiddaman case NVME_FEAT_PREDICTABLE_LATENCY_MODE_CONFIG:
16356960cd89SAndy Fiddaman name = "Predictable Latency Mode Config";
16366960cd89SAndy Fiddaman break;
16376960cd89SAndy Fiddaman case NVME_FEAT_PREDICTABLE_LATENCY_MODE_WINDOW:
16386960cd89SAndy Fiddaman name = "Predictable Latency Mode Window";
16396960cd89SAndy Fiddaman break;
16406960cd89SAndy Fiddaman case NVME_FEAT_LBA_STATUS_INFORMATION_ATTRIBUTES:
16416960cd89SAndy Fiddaman name = "LBA Status Information Report Interval";
16426960cd89SAndy Fiddaman break;
16436960cd89SAndy Fiddaman case NVME_FEAT_HOST_BEHAVIOR_SUPPORT:
16446960cd89SAndy Fiddaman name = "Host Behavior Support";
16456960cd89SAndy Fiddaman break;
16466960cd89SAndy Fiddaman case NVME_FEAT_SANITIZE_CONFIG:
16476960cd89SAndy Fiddaman name = "Sanitize Config";
16486960cd89SAndy Fiddaman break;
16496960cd89SAndy Fiddaman case NVME_FEAT_ENDURANCE_GROUP_EVENT_CONFIGURATION:
16506960cd89SAndy Fiddaman name = "Endurance Group Event Configuration";
16516960cd89SAndy Fiddaman break;
16526960cd89SAndy Fiddaman case NVME_FEAT_SOFTWARE_PROGRESS_MARKER:
16536960cd89SAndy Fiddaman name = "Software Progress Marker";
16546960cd89SAndy Fiddaman break;
16556960cd89SAndy Fiddaman case NVME_FEAT_HOST_IDENTIFIER:
16566960cd89SAndy Fiddaman name = "Host Identifier";
16576960cd89SAndy Fiddaman break;
16586960cd89SAndy Fiddaman case NVME_FEAT_RESERVATION_NOTIFICATION_MASK:
16596960cd89SAndy Fiddaman name = "Reservation Notification Mask";
16606960cd89SAndy Fiddaman break;
16616960cd89SAndy Fiddaman case NVME_FEAT_RESERVATION_PERSISTENCE:
16626960cd89SAndy Fiddaman name = "Reservation Persistence";
16636960cd89SAndy Fiddaman break;
16646960cd89SAndy Fiddaman case NVME_FEAT_NAMESPACE_WRITE_PROTECTION_CONFIG:
16656960cd89SAndy Fiddaman name = "Namespace Write Protection Config";
16666960cd89SAndy Fiddaman break;
16676960cd89SAndy Fiddaman default:
16686960cd89SAndy Fiddaman name = "Unknown";
16696960cd89SAndy Fiddaman break;
16706960cd89SAndy Fiddaman }
16716960cd89SAndy Fiddaman
16726960cd89SAndy Fiddaman return (name);
16736960cd89SAndy Fiddaman }
16746960cd89SAndy Fiddaman
16756960cd89SAndy Fiddaman static void
nvme_feature_invalid_cb(struct pci_nvme_softc * sc __unused,struct nvme_feature_obj * feat __unused,struct nvme_command * command __unused,struct nvme_completion * compl)16764f3f3e9aSAndy Fiddaman nvme_feature_invalid_cb(struct pci_nvme_softc *sc __unused,
16774f3f3e9aSAndy Fiddaman struct nvme_feature_obj *feat __unused,
16784f3f3e9aSAndy Fiddaman struct nvme_command *command __unused,
16796960cd89SAndy Fiddaman struct nvme_completion *compl)
16806960cd89SAndy Fiddaman {
16816960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
16826960cd89SAndy Fiddaman }
16836960cd89SAndy Fiddaman
16846960cd89SAndy Fiddaman static void
nvme_feature_iv_config(struct pci_nvme_softc * sc,struct nvme_feature_obj * feat __unused,struct nvme_command * command,struct nvme_completion * compl)16856960cd89SAndy Fiddaman nvme_feature_iv_config(struct pci_nvme_softc *sc,
16864f3f3e9aSAndy Fiddaman struct nvme_feature_obj *feat __unused,
16876960cd89SAndy Fiddaman struct nvme_command *command,
16886960cd89SAndy Fiddaman struct nvme_completion *compl)
16896960cd89SAndy Fiddaman {
16906960cd89SAndy Fiddaman uint32_t i;
16916960cd89SAndy Fiddaman uint32_t cdw11 = command->cdw11;
16926960cd89SAndy Fiddaman uint16_t iv;
16936960cd89SAndy Fiddaman bool cd;
16946960cd89SAndy Fiddaman
16956960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
16966960cd89SAndy Fiddaman
16976960cd89SAndy Fiddaman iv = cdw11 & 0xffff;
16986960cd89SAndy Fiddaman cd = cdw11 & (1 << 16);
16996960cd89SAndy Fiddaman
17006960cd89SAndy Fiddaman if (iv > (sc->max_queues + 1)) {
17016960cd89SAndy Fiddaman return;
17026960cd89SAndy Fiddaman }
17036960cd89SAndy Fiddaman
17046960cd89SAndy Fiddaman /* No Interrupt Coalescing (i.e. not Coalescing Disable) for Admin Q */
17056960cd89SAndy Fiddaman if ((iv == 0) && !cd)
17066960cd89SAndy Fiddaman return;
17076960cd89SAndy Fiddaman
17086960cd89SAndy Fiddaman /* Requested Interrupt Vector must be used by a CQ */
17096960cd89SAndy Fiddaman for (i = 0; i < sc->num_cqueues + 1; i++) {
17106960cd89SAndy Fiddaman if (sc->compl_queues[i].intr_vec == iv) {
17116960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
17126960cd89SAndy Fiddaman }
17136960cd89SAndy Fiddaman }
1714d7b72f7bSAndy Fiddaman }
1715d7b72f7bSAndy Fiddaman
1716d7b72f7bSAndy Fiddaman #define NVME_ASYNC_EVENT_ENDURANCE_GROUP (0x4000)
1717d7b72f7bSAndy Fiddaman static void
nvme_feature_async_event(struct pci_nvme_softc * sc __unused,struct nvme_feature_obj * feat __unused,struct nvme_command * command,struct nvme_completion * compl)17184f3f3e9aSAndy Fiddaman nvme_feature_async_event(struct pci_nvme_softc *sc __unused,
17194f3f3e9aSAndy Fiddaman struct nvme_feature_obj *feat __unused,
1720d7b72f7bSAndy Fiddaman struct nvme_command *command,
1721d7b72f7bSAndy Fiddaman struct nvme_completion *compl)
1722d7b72f7bSAndy Fiddaman {
1723d7b72f7bSAndy Fiddaman if (command->cdw11 & NVME_ASYNC_EVENT_ENDURANCE_GROUP)
1724d7b72f7bSAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1725d7b72f7bSAndy Fiddaman }
1726d7b72f7bSAndy Fiddaman
1727d7b72f7bSAndy Fiddaman #define NVME_TEMP_THRESH_OVER 0
1728d7b72f7bSAndy Fiddaman #define NVME_TEMP_THRESH_UNDER 1
1729d7b72f7bSAndy Fiddaman static void
nvme_feature_temperature(struct pci_nvme_softc * sc,struct nvme_feature_obj * feat __unused,struct nvme_command * command,struct nvme_completion * compl)1730d7b72f7bSAndy Fiddaman nvme_feature_temperature(struct pci_nvme_softc *sc,
17314f3f3e9aSAndy Fiddaman struct nvme_feature_obj *feat __unused,
1732d7b72f7bSAndy Fiddaman struct nvme_command *command,
1733d7b72f7bSAndy Fiddaman struct nvme_completion *compl)
1734d7b72f7bSAndy Fiddaman {
1735d7b72f7bSAndy Fiddaman uint16_t tmpth; /* Temperature Threshold */
1736d7b72f7bSAndy Fiddaman uint8_t tmpsel; /* Threshold Temperature Select */
1737d7b72f7bSAndy Fiddaman uint8_t thsel; /* Threshold Type Select */
1738d7b72f7bSAndy Fiddaman bool set_crit = false;
173959d65d31SAndy Fiddaman bool report_crit;
1740d7b72f7bSAndy Fiddaman
1741d7b72f7bSAndy Fiddaman tmpth = command->cdw11 & 0xffff;
1742d7b72f7bSAndy Fiddaman tmpsel = (command->cdw11 >> 16) & 0xf;
1743d7b72f7bSAndy Fiddaman thsel = (command->cdw11 >> 20) & 0x3;
1744d7b72f7bSAndy Fiddaman
1745d7b72f7bSAndy Fiddaman DPRINTF("%s: tmpth=%#x tmpsel=%#x thsel=%#x", __func__, tmpth, tmpsel, thsel);
1746d7b72f7bSAndy Fiddaman
1747d7b72f7bSAndy Fiddaman /* Check for unsupported values */
1748d7b72f7bSAndy Fiddaman if (((tmpsel != 0) && (tmpsel != 0xf)) ||
1749d7b72f7bSAndy Fiddaman (thsel > NVME_TEMP_THRESH_UNDER)) {
1750d7b72f7bSAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1751d7b72f7bSAndy Fiddaman return;
1752d7b72f7bSAndy Fiddaman }
1753d7b72f7bSAndy Fiddaman
1754d7b72f7bSAndy Fiddaman if (((thsel == NVME_TEMP_THRESH_OVER) && (NVME_TEMPERATURE >= tmpth)) ||
1755d7b72f7bSAndy Fiddaman ((thsel == NVME_TEMP_THRESH_UNDER) && (NVME_TEMPERATURE <= tmpth)))
1756d7b72f7bSAndy Fiddaman set_crit = true;
1757d7b72f7bSAndy Fiddaman
1758d7b72f7bSAndy Fiddaman pthread_mutex_lock(&sc->mtx);
1759d7b72f7bSAndy Fiddaman if (set_crit)
1760d7b72f7bSAndy Fiddaman sc->health_log.critical_warning |=
1761d7b72f7bSAndy Fiddaman NVME_CRIT_WARN_ST_TEMPERATURE;
1762d7b72f7bSAndy Fiddaman else
1763d7b72f7bSAndy Fiddaman sc->health_log.critical_warning &=
1764d7b72f7bSAndy Fiddaman ~NVME_CRIT_WARN_ST_TEMPERATURE;
1765d7b72f7bSAndy Fiddaman pthread_mutex_unlock(&sc->mtx);
1766d7b72f7bSAndy Fiddaman
176759d65d31SAndy Fiddaman report_crit = sc->feat[NVME_FEAT_ASYNC_EVENT_CONFIGURATION].cdw11 &
176859d65d31SAndy Fiddaman NVME_CRIT_WARN_ST_TEMPERATURE;
176959d65d31SAndy Fiddaman
177059d65d31SAndy Fiddaman if (set_crit && report_crit)
1771d7b72f7bSAndy Fiddaman pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_SMART,
1772d7b72f7bSAndy Fiddaman sc->health_log.critical_warning);
17736960cd89SAndy Fiddaman
1774d7b72f7bSAndy Fiddaman DPRINTF("%s: set_crit=%c critical_warning=%#x status=%#x", __func__, set_crit ? 'T':'F', sc->health_log.critical_warning, compl->status);
17756960cd89SAndy Fiddaman }
17766960cd89SAndy Fiddaman
17776960cd89SAndy Fiddaman static void
nvme_feature_num_queues(struct pci_nvme_softc * sc,struct nvme_feature_obj * feat __unused,struct nvme_command * command,struct nvme_completion * compl)17786960cd89SAndy Fiddaman nvme_feature_num_queues(struct pci_nvme_softc *sc,
17794f3f3e9aSAndy Fiddaman struct nvme_feature_obj *feat __unused,
17806960cd89SAndy Fiddaman struct nvme_command *command,
17816960cd89SAndy Fiddaman struct nvme_completion *compl)
17824c87aefeSPatrick Mooney {
17834c87aefeSPatrick Mooney uint16_t nqr; /* Number of Queues Requested */
17844c87aefeSPatrick Mooney
17856960cd89SAndy Fiddaman if (sc->num_q_is_set) {
17866960cd89SAndy Fiddaman WPRINTF("%s: Number of Queues already set", __func__);
17876960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status,
17886960cd89SAndy Fiddaman NVME_SC_COMMAND_SEQUENCE_ERROR);
17896960cd89SAndy Fiddaman return;
17906960cd89SAndy Fiddaman }
17916960cd89SAndy Fiddaman
17924c87aefeSPatrick Mooney nqr = command->cdw11 & 0xFFFF;
17934c87aefeSPatrick Mooney if (nqr == 0xffff) {
17946960cd89SAndy Fiddaman WPRINTF("%s: Illegal NSQR value %#x", __func__, nqr);
17954c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
17966960cd89SAndy Fiddaman return;
17974c87aefeSPatrick Mooney }
17984c87aefeSPatrick Mooney
17994c87aefeSPatrick Mooney sc->num_squeues = ONE_BASED(nqr);
18004c87aefeSPatrick Mooney if (sc->num_squeues > sc->max_queues) {
18016960cd89SAndy Fiddaman DPRINTF("NSQR=%u is greater than max %u", sc->num_squeues,
18026960cd89SAndy Fiddaman sc->max_queues);
18034c87aefeSPatrick Mooney sc->num_squeues = sc->max_queues;
18044c87aefeSPatrick Mooney }
18054c87aefeSPatrick Mooney
18064c87aefeSPatrick Mooney nqr = (command->cdw11 >> 16) & 0xFFFF;
18074c87aefeSPatrick Mooney if (nqr == 0xffff) {
18086960cd89SAndy Fiddaman WPRINTF("%s: Illegal NCQR value %#x", __func__, nqr);
18094c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
18106960cd89SAndy Fiddaman return;
18114c87aefeSPatrick Mooney }
18124c87aefeSPatrick Mooney
18134c87aefeSPatrick Mooney sc->num_cqueues = ONE_BASED(nqr);
18144c87aefeSPatrick Mooney if (sc->num_cqueues > sc->max_queues) {
18156960cd89SAndy Fiddaman DPRINTF("NCQR=%u is greater than max %u", sc->num_cqueues,
18166960cd89SAndy Fiddaman sc->max_queues);
18174c87aefeSPatrick Mooney sc->num_cqueues = sc->max_queues;
18184c87aefeSPatrick Mooney }
18194c87aefeSPatrick Mooney
18206960cd89SAndy Fiddaman /* Patch the command value which will be saved on callback's return */
18216960cd89SAndy Fiddaman command->cdw11 = NVME_FEATURE_NUM_QUEUES(sc);
18224c87aefeSPatrick Mooney compl->cdw0 = NVME_FEATURE_NUM_QUEUES(sc);
18234c87aefeSPatrick Mooney
18246960cd89SAndy Fiddaman sc->num_q_is_set = true;
18254c87aefeSPatrick Mooney }
18264c87aefeSPatrick Mooney
18274c87aefeSPatrick Mooney static int
nvme_opc_set_features(struct pci_nvme_softc * sc,struct nvme_command * command,struct nvme_completion * compl)18286960cd89SAndy Fiddaman nvme_opc_set_features(struct pci_nvme_softc *sc, struct nvme_command *command,
18296960cd89SAndy Fiddaman struct nvme_completion *compl)
18304c87aefeSPatrick Mooney {
18316960cd89SAndy Fiddaman struct nvme_feature_obj *feat;
18326960cd89SAndy Fiddaman uint32_t nsid = command->nsid;
183359d65d31SAndy Fiddaman uint8_t fid = NVMEV(NVME_FEAT_SET_FID, command->cdw10);
183459d65d31SAndy Fiddaman bool sv = NVMEV(NVME_FEAT_SET_SV, command->cdw10);
18356960cd89SAndy Fiddaman
18366960cd89SAndy Fiddaman DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid));
18376960cd89SAndy Fiddaman
18386960cd89SAndy Fiddaman if (fid >= NVME_FID_MAX) {
18396960cd89SAndy Fiddaman DPRINTF("%s invalid feature 0x%x", __func__, fid);
18406960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
18416960cd89SAndy Fiddaman return (1);
18426960cd89SAndy Fiddaman }
184359d65d31SAndy Fiddaman
184459d65d31SAndy Fiddaman if (sv) {
184559d65d31SAndy Fiddaman pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
184659d65d31SAndy Fiddaman NVME_SC_FEATURE_NOT_SAVEABLE);
184759d65d31SAndy Fiddaman return (1);
184859d65d31SAndy Fiddaman }
184959d65d31SAndy Fiddaman
18506960cd89SAndy Fiddaman feat = &sc->feat[fid];
18516960cd89SAndy Fiddaman
1852d7b72f7bSAndy Fiddaman if (feat->namespace_specific && (nsid == NVME_GLOBAL_NAMESPACE_TAG)) {
1853d7b72f7bSAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
1854d7b72f7bSAndy Fiddaman return (1);
1855d7b72f7bSAndy Fiddaman }
1856d7b72f7bSAndy Fiddaman
18576960cd89SAndy Fiddaman if (!feat->namespace_specific &&
18586960cd89SAndy Fiddaman !((nsid == 0) || (nsid == NVME_GLOBAL_NAMESPACE_TAG))) {
18596960cd89SAndy Fiddaman pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
18606960cd89SAndy Fiddaman NVME_SC_FEATURE_NOT_NS_SPECIFIC);
18616960cd89SAndy Fiddaman return (1);
18626960cd89SAndy Fiddaman }
18634c87aefeSPatrick Mooney
18644c87aefeSPatrick Mooney compl->cdw0 = 0;
18656960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
18664c87aefeSPatrick Mooney
18676960cd89SAndy Fiddaman if (feat->set)
18686960cd89SAndy Fiddaman feat->set(sc, feat, command, compl);
186959d65d31SAndy Fiddaman else {
187059d65d31SAndy Fiddaman pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
187159d65d31SAndy Fiddaman NVME_SC_FEATURE_NOT_CHANGEABLE);
187259d65d31SAndy Fiddaman return (1);
187359d65d31SAndy Fiddaman }
18744c87aefeSPatrick Mooney
18756dc98349SAndy Fiddaman DPRINTF("%s: status=%#x cdw11=%#x", __func__, compl->status, command->cdw11);
18766dc98349SAndy Fiddaman if (compl->status == NVME_SC_SUCCESS) {
18776960cd89SAndy Fiddaman feat->cdw11 = command->cdw11;
18786dc98349SAndy Fiddaman if ((fid == NVME_FEAT_ASYNC_EVENT_CONFIGURATION) &&
18796dc98349SAndy Fiddaman (command->cdw11 != 0))
18806dc98349SAndy Fiddaman pci_nvme_aen_notify(sc);
18816dc98349SAndy Fiddaman }
18824c87aefeSPatrick Mooney
18836960cd89SAndy Fiddaman return (0);
18846960cd89SAndy Fiddaman }
18856960cd89SAndy Fiddaman
1886d7b72f7bSAndy Fiddaman #define NVME_FEATURES_SEL_SUPPORTED 0x3
1887d7b72f7bSAndy Fiddaman #define NVME_FEATURES_NS_SPECIFIC (1 << 1)
1888d7b72f7bSAndy Fiddaman
18896960cd89SAndy Fiddaman static int
nvme_opc_get_features(struct pci_nvme_softc * sc,struct nvme_command * command,struct nvme_completion * compl)18906960cd89SAndy Fiddaman nvme_opc_get_features(struct pci_nvme_softc* sc, struct nvme_command* command,
18916960cd89SAndy Fiddaman struct nvme_completion* compl)
18926960cd89SAndy Fiddaman {
18936960cd89SAndy Fiddaman struct nvme_feature_obj *feat;
18946960cd89SAndy Fiddaman uint8_t fid = command->cdw10 & 0xFF;
1895d7b72f7bSAndy Fiddaman uint8_t sel = (command->cdw10 >> 8) & 0x7;
18966960cd89SAndy Fiddaman
18976960cd89SAndy Fiddaman DPRINTF("%s: Feature ID 0x%x (%s)", __func__, fid, nvme_fid_to_name(fid));
18986960cd89SAndy Fiddaman
18996960cd89SAndy Fiddaman if (fid >= NVME_FID_MAX) {
19006960cd89SAndy Fiddaman DPRINTF("%s invalid feature 0x%x", __func__, fid);
19014c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
19024c87aefeSPatrick Mooney return (1);
19034c87aefeSPatrick Mooney }
19044c87aefeSPatrick Mooney
19056960cd89SAndy Fiddaman compl->cdw0 = 0;
19064c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
19076960cd89SAndy Fiddaman
19086960cd89SAndy Fiddaman feat = &sc->feat[fid];
19096960cd89SAndy Fiddaman if (feat->get) {
19106960cd89SAndy Fiddaman feat->get(sc, feat, command, compl);
19116960cd89SAndy Fiddaman }
19126960cd89SAndy Fiddaman
19136960cd89SAndy Fiddaman if (compl->status == NVME_SC_SUCCESS) {
1914d7b72f7bSAndy Fiddaman if ((sel == NVME_FEATURES_SEL_SUPPORTED) && feat->namespace_specific)
1915d7b72f7bSAndy Fiddaman compl->cdw0 = NVME_FEATURES_NS_SPECIFIC;
1916d7b72f7bSAndy Fiddaman else
1917d7b72f7bSAndy Fiddaman compl->cdw0 = feat->cdw11;
19186960cd89SAndy Fiddaman }
19196960cd89SAndy Fiddaman
19206960cd89SAndy Fiddaman return (0);
19214c87aefeSPatrick Mooney }
19224c87aefeSPatrick Mooney
19234c87aefeSPatrick Mooney static int
nvme_opc_format_nvm(struct pci_nvme_softc * sc,struct nvme_command * command,struct nvme_completion * compl)19246960cd89SAndy Fiddaman nvme_opc_format_nvm(struct pci_nvme_softc* sc, struct nvme_command* command,
19254c87aefeSPatrick Mooney struct nvme_completion* compl)
19264c87aefeSPatrick Mooney {
19276960cd89SAndy Fiddaman uint8_t ses, lbaf, pi;
19284c87aefeSPatrick Mooney
19296960cd89SAndy Fiddaman /* Only supports Secure Erase Setting - User Data Erase */
19306960cd89SAndy Fiddaman ses = (command->cdw10 >> 9) & 0x7;
19316960cd89SAndy Fiddaman if (ses > 0x1) {
19326960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
19336960cd89SAndy Fiddaman return (1);
19346960cd89SAndy Fiddaman }
19354c87aefeSPatrick Mooney
19366960cd89SAndy Fiddaman /* Only supports a single LBA Format */
19376960cd89SAndy Fiddaman lbaf = command->cdw10 & 0xf;
19386960cd89SAndy Fiddaman if (lbaf != 0) {
19396960cd89SAndy Fiddaman pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
19406960cd89SAndy Fiddaman NVME_SC_INVALID_FORMAT);
19416960cd89SAndy Fiddaman return (1);
19426960cd89SAndy Fiddaman }
19434c87aefeSPatrick Mooney
1944*32640292SAndy Fiddaman /* Doesn't support Protection Information */
19456960cd89SAndy Fiddaman pi = (command->cdw10 >> 5) & 0x7;
19466960cd89SAndy Fiddaman if (pi != 0) {
19476960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_INVALID_FIELD);
19486960cd89SAndy Fiddaman return (1);
19496960cd89SAndy Fiddaman }
19506960cd89SAndy Fiddaman
19516960cd89SAndy Fiddaman if (sc->nvstore.type == NVME_STOR_RAM) {
19526960cd89SAndy Fiddaman if (sc->nvstore.ctx)
19536960cd89SAndy Fiddaman free(sc->nvstore.ctx);
19546960cd89SAndy Fiddaman sc->nvstore.ctx = calloc(1, sc->nvstore.size);
19556960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
19566960cd89SAndy Fiddaman } else {
19576960cd89SAndy Fiddaman struct pci_nvme_ioreq *req;
19586960cd89SAndy Fiddaman int err;
19596960cd89SAndy Fiddaman
19606960cd89SAndy Fiddaman req = pci_nvme_get_ioreq(sc);
19616960cd89SAndy Fiddaman if (req == NULL) {
19624c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status,
19636960cd89SAndy Fiddaman NVME_SC_INTERNAL_DEVICE_ERROR);
19646960cd89SAndy Fiddaman WPRINTF("%s: unable to allocate IO req", __func__);
19654c87aefeSPatrick Mooney return (1);
19664c87aefeSPatrick Mooney }
19676960cd89SAndy Fiddaman req->nvme_sq = &sc->submit_queues[0];
19686960cd89SAndy Fiddaman req->sqid = 0;
19696960cd89SAndy Fiddaman req->opc = command->opc;
19706960cd89SAndy Fiddaman req->cid = command->cid;
19716960cd89SAndy Fiddaman req->nsid = command->nsid;
19726960cd89SAndy Fiddaman
19736960cd89SAndy Fiddaman req->io_req.br_offset = 0;
19746960cd89SAndy Fiddaman req->io_req.br_resid = sc->nvstore.size;
19756960cd89SAndy Fiddaman req->io_req.br_callback = pci_nvme_io_done;
19764c87aefeSPatrick Mooney
19776960cd89SAndy Fiddaman err = blockif_delete(sc->nvstore.ctx, &req->io_req);
19786960cd89SAndy Fiddaman if (err) {
19796960cd89SAndy Fiddaman pci_nvme_status_genc(&compl->status,
19806960cd89SAndy Fiddaman NVME_SC_INTERNAL_DEVICE_ERROR);
19816960cd89SAndy Fiddaman pci_nvme_release_ioreq(sc, req);
1982d7b72f7bSAndy Fiddaman } else
1983d7b72f7bSAndy Fiddaman compl->status = NVME_NO_STATUS;
19844c87aefeSPatrick Mooney }
19854c87aefeSPatrick Mooney
19864c87aefeSPatrick Mooney return (1);
19874c87aefeSPatrick Mooney }
19884c87aefeSPatrick Mooney
19894c87aefeSPatrick Mooney static int
nvme_opc_abort(struct pci_nvme_softc * sc __unused,struct nvme_command * command,struct nvme_completion * compl)19904f3f3e9aSAndy Fiddaman nvme_opc_abort(struct pci_nvme_softc *sc __unused, struct nvme_command *command,
19914f3f3e9aSAndy Fiddaman struct nvme_completion *compl)
19924c87aefeSPatrick Mooney {
19936960cd89SAndy Fiddaman DPRINTF("%s submission queue %u, command ID 0x%x", __func__,
19946960cd89SAndy Fiddaman command->cdw10 & 0xFFFF, (command->cdw10 >> 16) & 0xFFFF);
19954c87aefeSPatrick Mooney
19964c87aefeSPatrick Mooney /* TODO: search for the command ID and abort it */
19974c87aefeSPatrick Mooney
19984c87aefeSPatrick Mooney compl->cdw0 = 1;
19994c87aefeSPatrick Mooney pci_nvme_status_genc(&compl->status, NVME_SC_SUCCESS);
20004c87aefeSPatrick Mooney return (1);
20014c87aefeSPatrick Mooney }
20024c87aefeSPatrick Mooney
20034c87aefeSPatrick Mooney static int
nvme_opc_async_event_req(struct pci_nvme_softc * sc,struct nvme_command * command,struct nvme_completion * compl)20044c87aefeSPatrick Mooney nvme_opc_async_event_req(struct pci_nvme_softc* sc,
20054c87aefeSPatrick Mooney struct nvme_command* command, struct nvme_completion* compl)
20064c87aefeSPatrick Mooney {
20076dc98349SAndy Fiddaman DPRINTF("%s async event request count=%u aerl=%u cid=%#x", __func__,
20086dc98349SAndy Fiddaman sc->aer_count, sc->ctrldata.aerl, command->cid);
20096960cd89SAndy Fiddaman
20106960cd89SAndy Fiddaman /* Don't exceed the Async Event Request Limit (AERL). */
20116960cd89SAndy Fiddaman if (pci_nvme_aer_limit_reached(sc)) {
20126960cd89SAndy Fiddaman pci_nvme_status_tc(&compl->status, NVME_SCT_COMMAND_SPECIFIC,
20136960cd89SAndy Fiddaman NVME_SC_ASYNC_EVENT_REQUEST_LIMIT_EXCEEDED);
20146960cd89SAndy Fiddaman return (1);
20156960cd89SAndy Fiddaman }
20166960cd89SAndy Fiddaman
20176960cd89SAndy Fiddaman if (pci_nvme_aer_add(sc, command->cid)) {
20186960cd89SAndy Fiddaman pci_nvme_status_tc(&compl->status, NVME_SCT_GENERIC,
20196960cd89SAndy Fiddaman NVME_SC_INTERNAL_DEVICE_ERROR);
20206960cd89SAndy Fiddaman return (1);
20216960cd89SAndy Fiddaman }
20224c87aefeSPatrick Mooney
20234c87aefeSPatrick Mooney /*
20246960cd89SAndy Fiddaman * Raise events when they happen based on the Set Features cmd.
20254c87aefeSPatrick Mooney * These events happen async, so only set completion successful if
20264c87aefeSPatrick Mooney * there is an event reflective of the request to get event.
20274c87aefeSPatrick Mooney */
20286960cd89SAndy Fiddaman compl->status = NVME_NO_STATUS;
20296dc98349SAndy Fiddaman pci_nvme_aen_notify(sc);
20306960cd89SAndy Fiddaman
20314c87aefeSPatrick Mooney return (0);
20324c87aefeSPatrick Mooney }
20334c87aefeSPatrick Mooney
20344c87aefeSPatrick Mooney static void
pci_nvme_handle_admin_cmd(struct pci_nvme_softc * sc,uint64_t value)20354c87aefeSPatrick Mooney pci_nvme_handle_admin_cmd(struct pci_nvme_softc* sc, uint64_t value)
20364c87aefeSPatrick Mooney {
20374c87aefeSPatrick Mooney struct nvme_completion compl;
20384c87aefeSPatrick Mooney struct nvme_command *cmd;
20394c87aefeSPatrick Mooney struct nvme_submission_queue *sq;
20404c87aefeSPatrick Mooney struct nvme_completion_queue *cq;
20414c87aefeSPatrick Mooney uint16_t sqhead;
20424c87aefeSPatrick Mooney
20436960cd89SAndy Fiddaman DPRINTF("%s index %u", __func__, (uint32_t)value);
20444c87aefeSPatrick Mooney
20454c87aefeSPatrick Mooney sq = &sc->submit_queues[0];
2046154972afSPatrick Mooney cq = &sc->compl_queues[0];
20474c87aefeSPatrick Mooney
20486960cd89SAndy Fiddaman pthread_mutex_lock(&sq->mtx);
20494c87aefeSPatrick Mooney
20506960cd89SAndy Fiddaman sqhead = sq->head;
20516960cd89SAndy Fiddaman DPRINTF("sqhead %u, tail %u", sqhead, sq->tail);
20526dc98349SAndy Fiddaman
20534c87aefeSPatrick Mooney while (sqhead != atomic_load_acq_short(&sq->tail)) {
20544c87aefeSPatrick Mooney cmd = &(sq->qbase)[sqhead];
205584659b24SMichael Zeller compl.cdw0 = 0;
20564c87aefeSPatrick Mooney compl.status = 0;
20574c87aefeSPatrick Mooney
20584c87aefeSPatrick Mooney switch (cmd->opc) {
20594c87aefeSPatrick Mooney case NVME_OPC_DELETE_IO_SQ:
20606960cd89SAndy Fiddaman DPRINTF("%s command DELETE_IO_SQ", __func__);
2061154972afSPatrick Mooney nvme_opc_delete_io_sq(sc, cmd, &compl);
20624c87aefeSPatrick Mooney break;
20634c87aefeSPatrick Mooney case NVME_OPC_CREATE_IO_SQ:
20646960cd89SAndy Fiddaman DPRINTF("%s command CREATE_IO_SQ", __func__);
2065154972afSPatrick Mooney nvme_opc_create_io_sq(sc, cmd, &compl);
20664c87aefeSPatrick Mooney break;
20674c87aefeSPatrick Mooney case NVME_OPC_DELETE_IO_CQ:
20686960cd89SAndy Fiddaman DPRINTF("%s command DELETE_IO_CQ", __func__);
2069154972afSPatrick Mooney nvme_opc_delete_io_cq(sc, cmd, &compl);
20704c87aefeSPatrick Mooney break;
20714c87aefeSPatrick Mooney case NVME_OPC_CREATE_IO_CQ:
20726960cd89SAndy Fiddaman DPRINTF("%s command CREATE_IO_CQ", __func__);
2073154972afSPatrick Mooney nvme_opc_create_io_cq(sc, cmd, &compl);
20744c87aefeSPatrick Mooney break;
20754c87aefeSPatrick Mooney case NVME_OPC_GET_LOG_PAGE:
20766960cd89SAndy Fiddaman DPRINTF("%s command GET_LOG_PAGE", __func__);
2077154972afSPatrick Mooney nvme_opc_get_log_page(sc, cmd, &compl);
20784c87aefeSPatrick Mooney break;
20794c87aefeSPatrick Mooney case NVME_OPC_IDENTIFY:
20806960cd89SAndy Fiddaman DPRINTF("%s command IDENTIFY", __func__);
2081154972afSPatrick Mooney nvme_opc_identify(sc, cmd, &compl);
20824c87aefeSPatrick Mooney break;
20834c87aefeSPatrick Mooney case NVME_OPC_ABORT:
20846960cd89SAndy Fiddaman DPRINTF("%s command ABORT", __func__);
2085154972afSPatrick Mooney nvme_opc_abort(sc, cmd, &compl);
20864c87aefeSPatrick Mooney break;
20874c87aefeSPatrick Mooney case NVME_OPC_SET_FEATURES:
20886960cd89SAndy Fiddaman DPRINTF("%s command SET_FEATURES", __func__);
2089154972afSPatrick Mooney nvme_opc_set_features(sc, cmd, &compl);
20904c87aefeSPatrick Mooney break;
20914c87aefeSPatrick Mooney case NVME_OPC_GET_FEATURES:
20926960cd89SAndy Fiddaman DPRINTF("%s command GET_FEATURES", __func__);
2093154972afSPatrick Mooney nvme_opc_get_features(sc, cmd, &compl);
20944c87aefeSPatrick Mooney break;
20956960cd89SAndy Fiddaman case NVME_OPC_FIRMWARE_ACTIVATE:
20966960cd89SAndy Fiddaman DPRINTF("%s command FIRMWARE_ACTIVATE", __func__);
20976960cd89SAndy Fiddaman pci_nvme_status_tc(&compl.status,
20986960cd89SAndy Fiddaman NVME_SCT_COMMAND_SPECIFIC,
20996960cd89SAndy Fiddaman NVME_SC_INVALID_FIRMWARE_SLOT);
21006960cd89SAndy Fiddaman break;
21014c87aefeSPatrick Mooney case NVME_OPC_ASYNC_EVENT_REQUEST:
21026960cd89SAndy Fiddaman DPRINTF("%s command ASYNC_EVENT_REQ", __func__);
2103154972afSPatrick Mooney nvme_opc_async_event_req(sc, cmd, &compl);
21046960cd89SAndy Fiddaman break;
21056960cd89SAndy Fiddaman case NVME_OPC_FORMAT_NVM:
21066960cd89SAndy Fiddaman DPRINTF("%s command FORMAT_NVM", __func__);
21076960cd89SAndy Fiddaman if ((sc->ctrldata.oacs &
21086960cd89SAndy Fiddaman (1 << NVME_CTRLR_DATA_OACS_FORMAT_SHIFT)) == 0) {
21096960cd89SAndy Fiddaman pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);
2110d7b72f7bSAndy Fiddaman break;
21116960cd89SAndy Fiddaman }
21126960cd89SAndy Fiddaman nvme_opc_format_nvm(sc, cmd, &compl);
21134c87aefeSPatrick Mooney break;
2114d7b72f7bSAndy Fiddaman case NVME_OPC_SECURITY_SEND:
2115d7b72f7bSAndy Fiddaman case NVME_OPC_SECURITY_RECEIVE:
2116d7b72f7bSAndy Fiddaman case NVME_OPC_SANITIZE:
2117d7b72f7bSAndy Fiddaman case NVME_OPC_GET_LBA_STATUS:
2118d7b72f7bSAndy Fiddaman DPRINTF("%s command OPC=%#x (unsupported)", __func__,
2119d7b72f7bSAndy Fiddaman cmd->opc);
2120d7b72f7bSAndy Fiddaman /* Valid but unsupported opcodes */
2121d7b72f7bSAndy Fiddaman pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_FIELD);
2122d7b72f7bSAndy Fiddaman break;
21234c87aefeSPatrick Mooney default:
2124d7b72f7bSAndy Fiddaman DPRINTF("%s command OPC=%#X (not implemented)",
2125d7b72f7bSAndy Fiddaman __func__,
21266960cd89SAndy Fiddaman cmd->opc);
212784659b24SMichael Zeller pci_nvme_status_genc(&compl.status, NVME_SC_INVALID_OPCODE);
21284c87aefeSPatrick Mooney }
2129154972afSPatrick Mooney sqhead = (sqhead + 1) % sq->size;
2130154972afSPatrick Mooney
213184659b24SMichael Zeller if (NVME_COMPLETION_VALID(compl)) {
21326960cd89SAndy Fiddaman pci_nvme_cq_update(sc, &sc->compl_queues[0],
21336960cd89SAndy Fiddaman compl.cdw0,
21346960cd89SAndy Fiddaman cmd->cid,
21356960cd89SAndy Fiddaman 0, /* SQID */
21366960cd89SAndy Fiddaman compl.status);
21374c87aefeSPatrick Mooney }
21384c87aefeSPatrick Mooney }
21394c87aefeSPatrick Mooney
21406960cd89SAndy Fiddaman DPRINTF("setting sqhead %u", sqhead);
21416960cd89SAndy Fiddaman sq->head = sqhead;
21424c87aefeSPatrick Mooney
2143154972afSPatrick Mooney if (cq->head != cq->tail)
21444c87aefeSPatrick Mooney pci_generate_msix(sc->nsc_pi, 0);
21454c87aefeSPatrick Mooney
21466960cd89SAndy Fiddaman pthread_mutex_unlock(&sq->mtx);
21474c87aefeSPatrick Mooney }
21484c87aefeSPatrick Mooney
21496960cd89SAndy Fiddaman /*
21506960cd89SAndy Fiddaman * Update the Write and Read statistics reported in SMART data
21516960cd89SAndy Fiddaman *
21526960cd89SAndy Fiddaman * NVMe defines "data unit" as thousand's of 512 byte blocks and is rounded up.
21536960cd89SAndy Fiddaman * E.g. 1 data unit is 1 - 1,000 512 byte blocks. 3 data units are 2,001 - 3,000
2154*32640292SAndy Fiddaman * 512 byte blocks. Rounding up is achieved by initializing the remainder to 999.
21556960cd89SAndy Fiddaman */
21566960cd89SAndy Fiddaman static void
pci_nvme_stats_write_read_update(struct pci_nvme_softc * sc,uint8_t opc,size_t bytes,uint16_t status)21576960cd89SAndy Fiddaman pci_nvme_stats_write_read_update(struct pci_nvme_softc *sc, uint8_t opc,
21586960cd89SAndy Fiddaman size_t bytes, uint16_t status)
21594c87aefeSPatrick Mooney {
21604c87aefeSPatrick Mooney
21616960cd89SAndy Fiddaman pthread_mutex_lock(&sc->mtx);
21626960cd89SAndy Fiddaman switch (opc) {
21636960cd89SAndy Fiddaman case NVME_OPC_WRITE:
21646960cd89SAndy Fiddaman sc->write_commands++;
21656960cd89SAndy Fiddaman if (status != NVME_SC_SUCCESS)
21666960cd89SAndy Fiddaman break;
21676960cd89SAndy Fiddaman sc->write_dunits_remainder += (bytes / 512);
21686960cd89SAndy Fiddaman while (sc->write_dunits_remainder >= 1000) {
21696960cd89SAndy Fiddaman sc->write_data_units++;
21706960cd89SAndy Fiddaman sc->write_dunits_remainder -= 1000;
21716960cd89SAndy Fiddaman }
21726960cd89SAndy Fiddaman break;
21736960cd89SAndy Fiddaman case NVME_OPC_READ:
21746960cd89SAndy Fiddaman sc->read_commands++;
21756960cd89SAndy Fiddaman if (status != NVME_SC_SUCCESS)
21766960cd89SAndy Fiddaman break;
21776960cd89SAndy Fiddaman sc->read_dunits_remainder += (bytes / 512);
21786960cd89SAndy Fiddaman while (sc->read_dunits_remainder >= 1000) {
21796960cd89SAndy Fiddaman sc->read_data_units++;
21806960cd89SAndy Fiddaman sc->read_dunits_remainder -= 1000;
21816960cd89SAndy Fiddaman }
21826960cd89SAndy Fiddaman break;
21836960cd89SAndy Fiddaman default:
21846960cd89SAndy Fiddaman DPRINTF("%s: Invalid OPC 0x%02x for stats", __func__, opc);
21856960cd89SAndy Fiddaman break;
21866960cd89SAndy Fiddaman }
21876960cd89SAndy Fiddaman pthread_mutex_unlock(&sc->mtx);
21886960cd89SAndy Fiddaman }
21894c87aefeSPatrick Mooney
21906960cd89SAndy Fiddaman /*
2191d7b72f7bSAndy Fiddaman * Check if the combination of Starting LBA (slba) and number of blocks
2192d7b72f7bSAndy Fiddaman * exceeds the range of the underlying storage.
21936960cd89SAndy Fiddaman *
21946960cd89SAndy Fiddaman * Because NVMe specifies the SLBA in blocks as a uint64_t and blockif stores
21956960cd89SAndy Fiddaman * the capacity in bytes as a uint64_t, care must be taken to avoid integer
21966960cd89SAndy Fiddaman * overflow.
21976960cd89SAndy Fiddaman */
21986960cd89SAndy Fiddaman static bool
pci_nvme_out_of_range(struct pci_nvme_blockstore * nvstore,uint64_t slba,uint32_t nblocks)21996960cd89SAndy Fiddaman pci_nvme_out_of_range(struct pci_nvme_blockstore *nvstore, uint64_t slba,
2200d7b72f7bSAndy Fiddaman uint32_t nblocks)
22016960cd89SAndy Fiddaman {
22026960cd89SAndy Fiddaman size_t offset, bytes;
22034c87aefeSPatrick Mooney
22046960cd89SAndy Fiddaman /* Overflow check of multiplying Starting LBA by the sector size */
22056960cd89SAndy Fiddaman if (slba >> (64 - nvstore->sectsz_bits))
22066960cd89SAndy Fiddaman return (true);
22074c87aefeSPatrick Mooney
22086960cd89SAndy Fiddaman offset = slba << nvstore->sectsz_bits;
2209d7b72f7bSAndy Fiddaman bytes = nblocks << nvstore->sectsz_bits;
22104c87aefeSPatrick Mooney
22116960cd89SAndy Fiddaman /* Overflow check of Number of Logical Blocks */
2212d7b72f7bSAndy Fiddaman if ((nvstore->size <= offset) || ((nvstore->size - offset) < bytes))
22136960cd89SAndy Fiddaman return (true);
22144c87aefeSPatrick Mooney
22156960cd89SAndy Fiddaman return (false);
22166960cd89SAndy Fiddaman }
22174c87aefeSPatrick Mooney
22186960cd89SAndy Fiddaman static int
pci_nvme_append_iov_req(struct pci_nvme_softc * sc __unused,struct pci_nvme_ioreq * req,uint64_t gpaddr,size_t size,uint64_t offset)221959d65d31SAndy Fiddaman pci_nvme_append_iov_req(struct pci_nvme_softc *sc __unused,
222059d65d31SAndy Fiddaman struct pci_nvme_ioreq *req, uint64_t gpaddr, size_t size, uint64_t offset)
22216960cd89SAndy Fiddaman {
22226960cd89SAndy Fiddaman int iovidx;
22234f3f3e9aSAndy Fiddaman bool range_is_contiguous;
22244c87aefeSPatrick Mooney
22256960cd89SAndy Fiddaman if (req == NULL)
22266960cd89SAndy Fiddaman return (-1);
22274c87aefeSPatrick Mooney
22286960cd89SAndy Fiddaman if (req->io_req.br_iovcnt == NVME_MAX_IOVEC) {
22296960cd89SAndy Fiddaman return (-1);
22306960cd89SAndy Fiddaman }
22314c87aefeSPatrick Mooney
22324f3f3e9aSAndy Fiddaman /*
22334f3f3e9aSAndy Fiddaman * Minimize the number of IOVs by concatenating contiguous address
22344f3f3e9aSAndy Fiddaman * ranges. If the IOV count is zero, there is no previous range to
22354f3f3e9aSAndy Fiddaman * concatenate.
22364f3f3e9aSAndy Fiddaman */
22374f3f3e9aSAndy Fiddaman if (req->io_req.br_iovcnt == 0)
22384f3f3e9aSAndy Fiddaman range_is_contiguous = false;
22394f3f3e9aSAndy Fiddaman else
22404f3f3e9aSAndy Fiddaman range_is_contiguous = (req->prev_gpaddr + req->prev_size) == gpaddr;
22414f3f3e9aSAndy Fiddaman
22424f3f3e9aSAndy Fiddaman if (range_is_contiguous) {
22436960cd89SAndy Fiddaman iovidx = req->io_req.br_iovcnt - 1;
22444c87aefeSPatrick Mooney
22456960cd89SAndy Fiddaman req->io_req.br_iov[iovidx].iov_base =
22466960cd89SAndy Fiddaman paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
22476960cd89SAndy Fiddaman req->prev_gpaddr, size);
22484f3f3e9aSAndy Fiddaman if (req->io_req.br_iov[iovidx].iov_base == NULL)
22494f3f3e9aSAndy Fiddaman return (-1);
22504c87aefeSPatrick Mooney
22516960cd89SAndy Fiddaman req->prev_size += size;
22526960cd89SAndy Fiddaman req->io_req.br_resid += size;
22534c87aefeSPatrick Mooney
22546960cd89SAndy Fiddaman req->io_req.br_iov[iovidx].iov_len = req->prev_size;
22556960cd89SAndy Fiddaman } else {
22566960cd89SAndy Fiddaman iovidx = req->io_req.br_iovcnt;
22576960cd89SAndy Fiddaman if (iovidx == 0) {
22584f3f3e9aSAndy Fiddaman req->io_req.br_offset = offset;
22596960cd89SAndy Fiddaman req->io_req.br_resid = 0;
22606960cd89SAndy Fiddaman req->io_req.br_param = req;
22616960cd89SAndy Fiddaman }
22624c87aefeSPatrick Mooney
22636960cd89SAndy Fiddaman req->io_req.br_iov[iovidx].iov_base =
22646960cd89SAndy Fiddaman paddr_guest2host(req->sc->nsc_pi->pi_vmctx,
22656960cd89SAndy Fiddaman gpaddr, size);
22664f3f3e9aSAndy Fiddaman if (req->io_req.br_iov[iovidx].iov_base == NULL)
22674f3f3e9aSAndy Fiddaman return (-1);
22684c87aefeSPatrick Mooney
22696960cd89SAndy Fiddaman req->io_req.br_iov[iovidx].iov_len = size;
22704c87aefeSPatrick Mooney
22716960cd89SAndy Fiddaman req->prev_gpaddr = gpaddr;
22726960cd89SAndy Fiddaman req->prev_size = size;
22736960cd89SAndy Fiddaman req->io_req.br_resid += size;
22744c87aefeSPatrick Mooney
22756960cd89SAndy Fiddaman req->io_req.br_iovcnt++;
22764c87aefeSPatrick Mooney }
22776960cd89SAndy Fiddaman
22784c87aefeSPatrick Mooney return (0);
22794c87aefeSPatrick Mooney }
22804c87aefeSPatrick Mooney
22814c87aefeSPatrick Mooney static void
pci_nvme_set_completion(struct pci_nvme_softc * sc,struct nvme_submission_queue * sq,int sqid,uint16_t cid,uint16_t status)22824c87aefeSPatrick Mooney pci_nvme_set_completion(struct pci_nvme_softc *sc,
22834f3f3e9aSAndy Fiddaman struct nvme_submission_queue *sq, int sqid, uint16_t cid, uint16_t status)
22844c87aefeSPatrick Mooney {
22854c87aefeSPatrick Mooney struct nvme_completion_queue *cq = &sc->compl_queues[sq->cqid];
22864c87aefeSPatrick Mooney
22876960cd89SAndy Fiddaman DPRINTF("%s sqid %d cqid %u cid %u status: 0x%x 0x%x",
22884c87aefeSPatrick Mooney __func__, sqid, sq->cqid, cid, NVME_STATUS_GET_SCT(status),
22896960cd89SAndy Fiddaman NVME_STATUS_GET_SC(status));
22904c87aefeSPatrick Mooney
22914f3f3e9aSAndy Fiddaman pci_nvme_cq_update(sc, cq, 0, cid, sqid, status);
22924c87aefeSPatrick Mooney
2293154972afSPatrick Mooney if (cq->head != cq->tail) {
2294154972afSPatrick Mooney if (cq->intr_en & NVME_CQ_INTEN) {
22954c87aefeSPatrick Mooney pci_generate_msix(sc->nsc_pi, cq->intr_vec);
2296154972afSPatrick Mooney } else {
22976960cd89SAndy Fiddaman DPRINTF("%s: CQ%u interrupt disabled",
22986960cd89SAndy Fiddaman __func__, sq->cqid);
2299154972afSPatrick Mooney }
2300154972afSPatrick Mooney }
23014c87aefeSPatrick Mooney }
23024c87aefeSPatrick Mooney
23034c87aefeSPatrick Mooney static void
pci_nvme_release_ioreq(struct pci_nvme_softc * sc,struct pci_nvme_ioreq * req)23044c87aefeSPatrick Mooney pci_nvme_release_ioreq(struct pci_nvme_softc *sc, struct pci_nvme_ioreq *req)
23054c87aefeSPatrick Mooney {
23064c87aefeSPatrick Mooney req->sc = NULL;
23074c87aefeSPatrick Mooney req->nvme_sq = NULL;
23084c87aefeSPatrick Mooney req->sqid = 0;
23094c87aefeSPatrick Mooney
23104c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx);
23114c87aefeSPatrick Mooney
2312154972afSPatrick Mooney STAILQ_INSERT_TAIL(&sc->ioreqs_free, req, link);
23134c87aefeSPatrick Mooney sc->pending_ios--;
23144c87aefeSPatrick Mooney
23154c87aefeSPatrick Mooney /* when no more IO pending, can set to ready if device reset/enabled */
23164c87aefeSPatrick Mooney if (sc->pending_ios == 0 &&
23174c87aefeSPatrick Mooney NVME_CC_GET_EN(sc->regs.cc) && !(NVME_CSTS_GET_RDY(sc->regs.csts)))
23184c87aefeSPatrick Mooney sc->regs.csts |= NVME_CSTS_RDY;
23194c87aefeSPatrick Mooney
23204c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx);
23214c87aefeSPatrick Mooney
23224c87aefeSPatrick Mooney sem_post(&sc->iosemlock);
23234c87aefeSPatrick Mooney }
23244c87aefeSPatrick Mooney
23254c87aefeSPatrick Mooney static struct pci_nvme_ioreq *
pci_nvme_get_ioreq(struct pci_nvme_softc * sc)23264c87aefeSPatrick Mooney pci_nvme_get_ioreq(struct pci_nvme_softc *sc)
23274c87aefeSPatrick Mooney {
2328b0de25cbSAndy Fiddaman struct pci_nvme_ioreq *req = NULL;
23294c87aefeSPatrick Mooney
23304c87aefeSPatrick Mooney sem_wait(&sc->iosemlock);
23314c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx);
23324c87aefeSPatrick Mooney
2333154972afSPatrick Mooney req = STAILQ_FIRST(&sc->ioreqs_free);
23344c87aefeSPatrick Mooney assert(req != NULL);
2335154972afSPatrick Mooney STAILQ_REMOVE_HEAD(&sc->ioreqs_free, link);
23364c87aefeSPatrick Mooney
23374c87aefeSPatrick Mooney req->sc = sc;
23384c87aefeSPatrick Mooney
23394c87aefeSPatrick Mooney sc->pending_ios++;
23404c87aefeSPatrick Mooney
23414c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx);
23424c87aefeSPatrick Mooney
23434c87aefeSPatrick Mooney req->io_req.br_iovcnt = 0;
23444c87aefeSPatrick Mooney req->io_req.br_offset = 0;
23454c87aefeSPatrick Mooney req->io_req.br_resid = 0;
23464c87aefeSPatrick Mooney req->io_req.br_param = req;
23474c87aefeSPatrick Mooney req->prev_gpaddr = 0;
23484c87aefeSPatrick Mooney req->prev_size = 0;
23494c87aefeSPatrick Mooney
23504c87aefeSPatrick Mooney return req;
23514c87aefeSPatrick Mooney }
23524c87aefeSPatrick Mooney
23534c87aefeSPatrick Mooney static void
pci_nvme_io_done(struct blockif_req * br,int err)23544c87aefeSPatrick Mooney pci_nvme_io_done(struct blockif_req *br, int err)
23554c87aefeSPatrick Mooney {
23564c87aefeSPatrick Mooney struct pci_nvme_ioreq *req = br->br_param;
23574c87aefeSPatrick Mooney struct nvme_submission_queue *sq = req->nvme_sq;
23582b948146SAndy Fiddaman uint16_t code, status;
23592b948146SAndy Fiddaman
23606960cd89SAndy Fiddaman DPRINTF("%s error %d %s", __func__, err, strerror(err));
2361154972afSPatrick Mooney
23624c87aefeSPatrick Mooney /* TODO return correct error */
23634c87aefeSPatrick Mooney code = err ? NVME_SC_DATA_TRANSFER_ERROR : NVME_SC_SUCCESS;
236459d65d31SAndy Fiddaman status = 0;
23654c87aefeSPatrick Mooney pci_nvme_status_genc(&status, code);
23664c87aefeSPatrick Mooney
23674f3f3e9aSAndy Fiddaman pci_nvme_set_completion(req->sc, sq, req->sqid, req->cid, status);
23686960cd89SAndy Fiddaman pci_nvme_stats_write_read_update(req->sc, req->opc,
23696960cd89SAndy Fiddaman req->bytes, status);
23704c87aefeSPatrick Mooney pci_nvme_release_ioreq(req->sc, req);
23714c87aefeSPatrick Mooney }
23724c87aefeSPatrick Mooney
23736960cd89SAndy Fiddaman /*
23746960cd89SAndy Fiddaman * Implements the Flush command. The specification states:
23756960cd89SAndy Fiddaman * If a volatile write cache is not present, Flush commands complete
23766960cd89SAndy Fiddaman * successfully and have no effect
23776960cd89SAndy Fiddaman * in the description of the Volatile Write Cache (VWC) field of the Identify
23786960cd89SAndy Fiddaman * Controller data. Therefore, set status to Success if the command is
23796960cd89SAndy Fiddaman * not supported (i.e. RAM or as indicated by the blockif).
23806960cd89SAndy Fiddaman */
23816960cd89SAndy Fiddaman static bool
nvme_opc_flush(struct pci_nvme_softc * sc __unused,struct nvme_command * cmd __unused,struct pci_nvme_blockstore * nvstore,struct pci_nvme_ioreq * req,uint16_t * status)23824f3f3e9aSAndy Fiddaman nvme_opc_flush(struct pci_nvme_softc *sc __unused,
23834f3f3e9aSAndy Fiddaman struct nvme_command *cmd __unused,
23846960cd89SAndy Fiddaman struct pci_nvme_blockstore *nvstore,
23856960cd89SAndy Fiddaman struct pci_nvme_ioreq *req,
23866960cd89SAndy Fiddaman uint16_t *status)
23874c87aefeSPatrick Mooney {
23886960cd89SAndy Fiddaman bool pending = false;
23896960cd89SAndy Fiddaman
23906960cd89SAndy Fiddaman if (nvstore->type == NVME_STOR_RAM) {
23916960cd89SAndy Fiddaman pci_nvme_status_genc(status, NVME_SC_SUCCESS);
23926960cd89SAndy Fiddaman } else {
23936960cd89SAndy Fiddaman int err;
23946960cd89SAndy Fiddaman
23956960cd89SAndy Fiddaman req->io_req.br_callback = pci_nvme_io_done;
23966960cd89SAndy Fiddaman
23976960cd89SAndy Fiddaman err = blockif_flush(nvstore->ctx, &req->io_req);
23986960cd89SAndy Fiddaman switch (err) {
23996960cd89SAndy Fiddaman case 0:
24006960cd89SAndy Fiddaman pending = true;
24016960cd89SAndy Fiddaman break;
24026960cd89SAndy Fiddaman case EOPNOTSUPP:
24036960cd89SAndy Fiddaman pci_nvme_status_genc(status, NVME_SC_SUCCESS);
24046960cd89SAndy Fiddaman break;
24056960cd89SAndy Fiddaman default:
24066960cd89SAndy Fiddaman pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
24076960cd89SAndy Fiddaman }
24086960cd89SAndy Fiddaman }
24096960cd89SAndy Fiddaman
24106960cd89SAndy Fiddaman return (pending);
24116960cd89SAndy Fiddaman }
24126960cd89SAndy Fiddaman
24136960cd89SAndy Fiddaman static uint16_t
nvme_write_read_ram(struct pci_nvme_softc * sc,struct pci_nvme_blockstore * nvstore,uint64_t prp1,uint64_t prp2,size_t offset,uint64_t bytes,bool is_write)24146960cd89SAndy Fiddaman nvme_write_read_ram(struct pci_nvme_softc *sc,
24156960cd89SAndy Fiddaman struct pci_nvme_blockstore *nvstore,
24166960cd89SAndy Fiddaman uint64_t prp1, uint64_t prp2,
24176960cd89SAndy Fiddaman size_t offset, uint64_t bytes,
24186960cd89SAndy Fiddaman bool is_write)
24196960cd89SAndy Fiddaman {
24206960cd89SAndy Fiddaman uint8_t *buf = nvstore->ctx;
24216960cd89SAndy Fiddaman enum nvme_copy_dir dir;
24222b948146SAndy Fiddaman uint16_t status;
24232b948146SAndy Fiddaman
24246960cd89SAndy Fiddaman if (is_write)
24256960cd89SAndy Fiddaman dir = NVME_COPY_TO_PRP;
24266960cd89SAndy Fiddaman else
24276960cd89SAndy Fiddaman dir = NVME_COPY_FROM_PRP;
24286960cd89SAndy Fiddaman
242959d65d31SAndy Fiddaman status = 0;
24306960cd89SAndy Fiddaman if (nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, prp1, prp2,
24316960cd89SAndy Fiddaman buf + offset, bytes, dir))
24326960cd89SAndy Fiddaman pci_nvme_status_genc(&status,
24336960cd89SAndy Fiddaman NVME_SC_DATA_TRANSFER_ERROR);
24346960cd89SAndy Fiddaman else
24356960cd89SAndy Fiddaman pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
24366960cd89SAndy Fiddaman
24376960cd89SAndy Fiddaman return (status);
24386960cd89SAndy Fiddaman }
24396960cd89SAndy Fiddaman
24406960cd89SAndy Fiddaman static uint16_t
nvme_write_read_blockif(struct pci_nvme_softc * sc,struct pci_nvme_blockstore * nvstore,struct pci_nvme_ioreq * req,uint64_t prp1,uint64_t prp2,size_t offset,uint64_t bytes,bool is_write)24416960cd89SAndy Fiddaman nvme_write_read_blockif(struct pci_nvme_softc *sc,
24426960cd89SAndy Fiddaman struct pci_nvme_blockstore *nvstore,
24436960cd89SAndy Fiddaman struct pci_nvme_ioreq *req,
24446960cd89SAndy Fiddaman uint64_t prp1, uint64_t prp2,
24456960cd89SAndy Fiddaman size_t offset, uint64_t bytes,
24466960cd89SAndy Fiddaman bool is_write)
24476960cd89SAndy Fiddaman {
24486960cd89SAndy Fiddaman uint64_t size;
24496960cd89SAndy Fiddaman int err;
24506960cd89SAndy Fiddaman uint16_t status = NVME_NO_STATUS;
24516960cd89SAndy Fiddaman
24526960cd89SAndy Fiddaman size = MIN(PAGE_SIZE - (prp1 % PAGE_SIZE), bytes);
245359d65d31SAndy Fiddaman if (pci_nvme_append_iov_req(sc, req, prp1, size, offset)) {
24544f3f3e9aSAndy Fiddaman err = -1;
24556960cd89SAndy Fiddaman goto out;
24566960cd89SAndy Fiddaman }
24576960cd89SAndy Fiddaman
24586960cd89SAndy Fiddaman offset += size;
24596960cd89SAndy Fiddaman bytes -= size;
24606960cd89SAndy Fiddaman
24616960cd89SAndy Fiddaman if (bytes == 0) {
24626960cd89SAndy Fiddaman ;
24636960cd89SAndy Fiddaman } else if (bytes <= PAGE_SIZE) {
24646960cd89SAndy Fiddaman size = bytes;
246559d65d31SAndy Fiddaman if (pci_nvme_append_iov_req(sc, req, prp2, size, offset)) {
24664f3f3e9aSAndy Fiddaman err = -1;
24676960cd89SAndy Fiddaman goto out;
24686960cd89SAndy Fiddaman }
24696960cd89SAndy Fiddaman } else {
24706960cd89SAndy Fiddaman void *vmctx = sc->nsc_pi->pi_vmctx;
24716960cd89SAndy Fiddaman uint64_t *prp_list = &prp2;
24726960cd89SAndy Fiddaman uint64_t *last = prp_list;
24736960cd89SAndy Fiddaman
24746960cd89SAndy Fiddaman /* PRP2 is pointer to a physical region page list */
24756960cd89SAndy Fiddaman while (bytes) {
24766960cd89SAndy Fiddaman /* Last entry in list points to the next list */
2477b0de25cbSAndy Fiddaman if ((prp_list == last) && (bytes > PAGE_SIZE)) {
24786960cd89SAndy Fiddaman uint64_t prp = *prp_list;
24796960cd89SAndy Fiddaman
24806960cd89SAndy Fiddaman prp_list = paddr_guest2host(vmctx, prp,
24816960cd89SAndy Fiddaman PAGE_SIZE - (prp % PAGE_SIZE));
24824f3f3e9aSAndy Fiddaman if (prp_list == NULL) {
24834f3f3e9aSAndy Fiddaman err = -1;
24844f3f3e9aSAndy Fiddaman goto out;
24854f3f3e9aSAndy Fiddaman }
24866960cd89SAndy Fiddaman last = prp_list + (NVME_PRP2_ITEMS - 1);
24876960cd89SAndy Fiddaman }
24886960cd89SAndy Fiddaman
24896960cd89SAndy Fiddaman size = MIN(bytes, PAGE_SIZE);
24906960cd89SAndy Fiddaman
249159d65d31SAndy Fiddaman if (pci_nvme_append_iov_req(sc, req, *prp_list, size,
249259d65d31SAndy Fiddaman offset)) {
24934f3f3e9aSAndy Fiddaman err = -1;
24946960cd89SAndy Fiddaman goto out;
24956960cd89SAndy Fiddaman }
24966960cd89SAndy Fiddaman
24976960cd89SAndy Fiddaman offset += size;
24986960cd89SAndy Fiddaman bytes -= size;
24996960cd89SAndy Fiddaman
25006960cd89SAndy Fiddaman prp_list++;
25016960cd89SAndy Fiddaman }
25026960cd89SAndy Fiddaman }
25036960cd89SAndy Fiddaman req->io_req.br_callback = pci_nvme_io_done;
25046960cd89SAndy Fiddaman if (is_write)
25056960cd89SAndy Fiddaman err = blockif_write(nvstore->ctx, &req->io_req);
25066960cd89SAndy Fiddaman else
25076960cd89SAndy Fiddaman err = blockif_read(nvstore->ctx, &req->io_req);
25084f3f3e9aSAndy Fiddaman out:
25096960cd89SAndy Fiddaman if (err)
25106960cd89SAndy Fiddaman pci_nvme_status_genc(&status, NVME_SC_DATA_TRANSFER_ERROR);
25114f3f3e9aSAndy Fiddaman
25126960cd89SAndy Fiddaman return (status);
25136960cd89SAndy Fiddaman }
25146960cd89SAndy Fiddaman
25156960cd89SAndy Fiddaman static bool
nvme_opc_write_read(struct pci_nvme_softc * sc,struct nvme_command * cmd,struct pci_nvme_blockstore * nvstore,struct pci_nvme_ioreq * req,uint16_t * status)25166960cd89SAndy Fiddaman nvme_opc_write_read(struct pci_nvme_softc *sc,
25176960cd89SAndy Fiddaman struct nvme_command *cmd,
25186960cd89SAndy Fiddaman struct pci_nvme_blockstore *nvstore,
25196960cd89SAndy Fiddaman struct pci_nvme_ioreq *req,
25206960cd89SAndy Fiddaman uint16_t *status)
25216960cd89SAndy Fiddaman {
25222b948146SAndy Fiddaman uint64_t lba, nblocks, bytes;
25236960cd89SAndy Fiddaman size_t offset;
25246960cd89SAndy Fiddaman bool is_write = cmd->opc == NVME_OPC_WRITE;
25256960cd89SAndy Fiddaman bool pending = false;
25266960cd89SAndy Fiddaman
25276960cd89SAndy Fiddaman lba = ((uint64_t)cmd->cdw11 << 32) | cmd->cdw10;
25286960cd89SAndy Fiddaman nblocks = (cmd->cdw12 & 0xFFFF) + 1;
25294f3f3e9aSAndy Fiddaman bytes = nblocks << nvstore->sectsz_bits;
25304f3f3e9aSAndy Fiddaman if (bytes > NVME_MAX_DATA_SIZE) {
25314f3f3e9aSAndy Fiddaman WPRINTF("%s command would exceed MDTS", __func__);
25324f3f3e9aSAndy Fiddaman pci_nvme_status_genc(status, NVME_SC_INVALID_FIELD);
25334f3f3e9aSAndy Fiddaman goto out;
25344f3f3e9aSAndy Fiddaman }
25356dc98349SAndy Fiddaman
25366960cd89SAndy Fiddaman if (pci_nvme_out_of_range(nvstore, lba, nblocks)) {
2537d7b72f7bSAndy Fiddaman WPRINTF("%s command would exceed LBA range(slba=%#lx nblocks=%#lx)",
2538d7b72f7bSAndy Fiddaman __func__, lba, nblocks);
25396960cd89SAndy Fiddaman pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE);
25406960cd89SAndy Fiddaman goto out;
25416960cd89SAndy Fiddaman }
25426960cd89SAndy Fiddaman
25436960cd89SAndy Fiddaman offset = lba << nvstore->sectsz_bits;
25446960cd89SAndy Fiddaman
25456960cd89SAndy Fiddaman req->bytes = bytes;
25466960cd89SAndy Fiddaman req->io_req.br_offset = lba;
25476960cd89SAndy Fiddaman
25486960cd89SAndy Fiddaman /* PRP bits 1:0 must be zero */
25496960cd89SAndy Fiddaman cmd->prp1 &= ~0x3UL;
25506960cd89SAndy Fiddaman cmd->prp2 &= ~0x3UL;
25516960cd89SAndy Fiddaman
25526960cd89SAndy Fiddaman if (nvstore->type == NVME_STOR_RAM) {
25536960cd89SAndy Fiddaman *status = nvme_write_read_ram(sc, nvstore, cmd->prp1,
25546960cd89SAndy Fiddaman cmd->prp2, offset, bytes, is_write);
25556960cd89SAndy Fiddaman } else {
25566960cd89SAndy Fiddaman *status = nvme_write_read_blockif(sc, nvstore, req,
25576960cd89SAndy Fiddaman cmd->prp1, cmd->prp2, offset, bytes, is_write);
25584c87aefeSPatrick Mooney
25596960cd89SAndy Fiddaman if (*status == NVME_NO_STATUS)
25606960cd89SAndy Fiddaman pending = true;
25616960cd89SAndy Fiddaman }
25626960cd89SAndy Fiddaman out:
25636960cd89SAndy Fiddaman if (!pending)
25646960cd89SAndy Fiddaman pci_nvme_stats_write_read_update(sc, cmd->opc, bytes, *status);
25654c87aefeSPatrick Mooney
25666960cd89SAndy Fiddaman return (pending);
25674c87aefeSPatrick Mooney }
25684c87aefeSPatrick Mooney
2569154972afSPatrick Mooney static void
pci_nvme_dealloc_sm(struct blockif_req * br,int err)2570154972afSPatrick Mooney pci_nvme_dealloc_sm(struct blockif_req *br, int err)
2571154972afSPatrick Mooney {
2572154972afSPatrick Mooney struct pci_nvme_ioreq *req = br->br_param;
2573154972afSPatrick Mooney struct pci_nvme_softc *sc = req->sc;
2574154972afSPatrick Mooney bool done = true;
2575154972afSPatrick Mooney uint16_t status;
25762b948146SAndy Fiddaman
25772b948146SAndy Fiddaman status = 0;
2578154972afSPatrick Mooney if (err) {
2579154972afSPatrick Mooney pci_nvme_status_genc(&status, NVME_SC_INTERNAL_DEVICE_ERROR);
2580154972afSPatrick Mooney } else if ((req->prev_gpaddr + 1) == (req->prev_size)) {
2581154972afSPatrick Mooney pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
2582154972afSPatrick Mooney } else {
2583154972afSPatrick Mooney struct iovec *iov = req->io_req.br_iov;
2584154972afSPatrick Mooney
2585154972afSPatrick Mooney req->prev_gpaddr++;
2586154972afSPatrick Mooney iov += req->prev_gpaddr;
2587154972afSPatrick Mooney
2588154972afSPatrick Mooney /* The iov_* values already include the sector size */
2589154972afSPatrick Mooney req->io_req.br_offset = (off_t)iov->iov_base;
2590154972afSPatrick Mooney req->io_req.br_resid = iov->iov_len;
2591154972afSPatrick Mooney if (blockif_delete(sc->nvstore.ctx, &req->io_req)) {
2592154972afSPatrick Mooney pci_nvme_status_genc(&status,
2593154972afSPatrick Mooney NVME_SC_INTERNAL_DEVICE_ERROR);
2594154972afSPatrick Mooney } else
2595154972afSPatrick Mooney done = false;
2596154972afSPatrick Mooney }
2597154972afSPatrick Mooney
2598154972afSPatrick Mooney if (done) {
25994f3f3e9aSAndy Fiddaman pci_nvme_set_completion(sc, req->nvme_sq, req->sqid, req->cid,
26004f3f3e9aSAndy Fiddaman status);
2601154972afSPatrick Mooney pci_nvme_release_ioreq(sc, req);
2602154972afSPatrick Mooney }
2603154972afSPatrick Mooney }
2604154972afSPatrick Mooney
26056960cd89SAndy Fiddaman static bool
nvme_opc_dataset_mgmt(struct pci_nvme_softc * sc,struct nvme_command * cmd,struct pci_nvme_blockstore * nvstore,struct pci_nvme_ioreq * req,uint16_t * status)2606154972afSPatrick Mooney nvme_opc_dataset_mgmt(struct pci_nvme_softc *sc,
2607154972afSPatrick Mooney struct nvme_command *cmd,
2608154972afSPatrick Mooney struct pci_nvme_blockstore *nvstore,
2609154972afSPatrick Mooney struct pci_nvme_ioreq *req,
2610154972afSPatrick Mooney uint16_t *status)
2611154972afSPatrick Mooney {
26124f3f3e9aSAndy Fiddaman struct nvme_dsm_range *range = NULL;
26136960cd89SAndy Fiddaman uint32_t nr, r, non_zero, dr;
26146960cd89SAndy Fiddaman int err;
26156960cd89SAndy Fiddaman bool pending = false;
2616154972afSPatrick Mooney
2617154972afSPatrick Mooney if ((sc->ctrldata.oncs & NVME_ONCS_DSM) == 0) {
2618154972afSPatrick Mooney pci_nvme_status_genc(status, NVME_SC_INVALID_OPCODE);
2619154972afSPatrick Mooney goto out;
2620154972afSPatrick Mooney }
2621154972afSPatrick Mooney
26226960cd89SAndy Fiddaman nr = cmd->cdw10 & 0xff;
26236960cd89SAndy Fiddaman
26246960cd89SAndy Fiddaman /* copy locally because a range entry could straddle PRPs */
262559d65d31SAndy Fiddaman #ifdef __FreeBSD__
26266960cd89SAndy Fiddaman range = calloc(1, NVME_MAX_DSM_TRIM);
262759d65d31SAndy Fiddaman #else
262859d65d31SAndy Fiddaman _Static_assert(NVME_MAX_DSM_TRIM % sizeof(struct nvme_dsm_range) == 0,
262959d65d31SAndy Fiddaman "NVME_MAX_DSM_TRIM is not a multiple of struct size");
263059d65d31SAndy Fiddaman range = calloc(NVME_MAX_DSM_TRIM / sizeof (*range), sizeof (*range));
263159d65d31SAndy Fiddaman #endif
26326960cd89SAndy Fiddaman if (range == NULL) {
26336960cd89SAndy Fiddaman pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
26346960cd89SAndy Fiddaman goto out;
26356960cd89SAndy Fiddaman }
26366960cd89SAndy Fiddaman nvme_prp_memcpy(sc->nsc_pi->pi_vmctx, cmd->prp1, cmd->prp2,
26376960cd89SAndy Fiddaman (uint8_t *)range, NVME_MAX_DSM_TRIM, NVME_COPY_FROM_PRP);
26386960cd89SAndy Fiddaman
26396960cd89SAndy Fiddaman /* Check for invalid ranges and the number of non-zero lengths */
26406960cd89SAndy Fiddaman non_zero = 0;
26416960cd89SAndy Fiddaman for (r = 0; r <= nr; r++) {
26426960cd89SAndy Fiddaman if (pci_nvme_out_of_range(nvstore,
26436960cd89SAndy Fiddaman range[r].starting_lba, range[r].length)) {
26446960cd89SAndy Fiddaman pci_nvme_status_genc(status, NVME_SC_LBA_OUT_OF_RANGE);
26456960cd89SAndy Fiddaman goto out;
26466960cd89SAndy Fiddaman }
26476960cd89SAndy Fiddaman if (range[r].length != 0)
26486960cd89SAndy Fiddaman non_zero++;
26496960cd89SAndy Fiddaman }
26506960cd89SAndy Fiddaman
2651154972afSPatrick Mooney if (cmd->cdw11 & NVME_DSM_ATTR_DEALLOCATE) {
26526960cd89SAndy Fiddaman size_t offset, bytes;
26536960cd89SAndy Fiddaman int sectsz_bits = sc->nvstore.sectsz_bits;
2654154972afSPatrick Mooney
2655154972afSPatrick Mooney /*
2656154972afSPatrick Mooney * DSM calls are advisory only, and compliant controllers
2657154972afSPatrick Mooney * may choose to take no actions (i.e. return Success).
2658154972afSPatrick Mooney */
2659154972afSPatrick Mooney if (!nvstore->deallocate) {
2660154972afSPatrick Mooney pci_nvme_status_genc(status, NVME_SC_SUCCESS);
2661154972afSPatrick Mooney goto out;
2662154972afSPatrick Mooney }
2663154972afSPatrick Mooney
26646960cd89SAndy Fiddaman /* If all ranges have a zero length, return Success */
26656960cd89SAndy Fiddaman if (non_zero == 0) {
26666960cd89SAndy Fiddaman pci_nvme_status_genc(status, NVME_SC_SUCCESS);
2667154972afSPatrick Mooney goto out;
2668154972afSPatrick Mooney }
2669154972afSPatrick Mooney
26706960cd89SAndy Fiddaman if (req == NULL) {
2671154972afSPatrick Mooney pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
2672154972afSPatrick Mooney goto out;
2673154972afSPatrick Mooney }
2674154972afSPatrick Mooney
26756960cd89SAndy Fiddaman offset = range[0].starting_lba << sectsz_bits;
26766960cd89SAndy Fiddaman bytes = range[0].length << sectsz_bits;
26776960cd89SAndy Fiddaman
2678154972afSPatrick Mooney /*
2679154972afSPatrick Mooney * If the request is for more than a single range, store
2680154972afSPatrick Mooney * the ranges in the br_iov. Optimize for the common case
2681154972afSPatrick Mooney * of a single range.
2682154972afSPatrick Mooney *
2683154972afSPatrick Mooney * Note that NVMe Number of Ranges is a zero based value
2684154972afSPatrick Mooney */
2685154972afSPatrick Mooney req->io_req.br_iovcnt = 0;
26866960cd89SAndy Fiddaman req->io_req.br_offset = offset;
26876960cd89SAndy Fiddaman req->io_req.br_resid = bytes;
2688154972afSPatrick Mooney
2689154972afSPatrick Mooney if (nr == 0) {
2690154972afSPatrick Mooney req->io_req.br_callback = pci_nvme_io_done;
2691154972afSPatrick Mooney } else {
2692154972afSPatrick Mooney struct iovec *iov = req->io_req.br_iov;
2693154972afSPatrick Mooney
26946960cd89SAndy Fiddaman for (r = 0, dr = 0; r <= nr; r++) {
26956960cd89SAndy Fiddaman offset = range[r].starting_lba << sectsz_bits;
26966960cd89SAndy Fiddaman bytes = range[r].length << sectsz_bits;
26976960cd89SAndy Fiddaman if (bytes == 0)
26986960cd89SAndy Fiddaman continue;
26996960cd89SAndy Fiddaman
27006960cd89SAndy Fiddaman if ((nvstore->size - offset) < bytes) {
27016960cd89SAndy Fiddaman pci_nvme_status_genc(status,
27026960cd89SAndy Fiddaman NVME_SC_LBA_OUT_OF_RANGE);
27036960cd89SAndy Fiddaman goto out;
27046960cd89SAndy Fiddaman }
27056960cd89SAndy Fiddaman iov[dr].iov_base = (void *)offset;
27066960cd89SAndy Fiddaman iov[dr].iov_len = bytes;
27076960cd89SAndy Fiddaman dr++;
2708154972afSPatrick Mooney }
2709154972afSPatrick Mooney req->io_req.br_callback = pci_nvme_dealloc_sm;
2710154972afSPatrick Mooney
2711154972afSPatrick Mooney /*
2712154972afSPatrick Mooney * Use prev_gpaddr to track the current entry and
2713154972afSPatrick Mooney * prev_size to track the number of entries
2714154972afSPatrick Mooney */
2715154972afSPatrick Mooney req->prev_gpaddr = 0;
27166960cd89SAndy Fiddaman req->prev_size = dr;
2717154972afSPatrick Mooney }
2718154972afSPatrick Mooney
2719154972afSPatrick Mooney err = blockif_delete(nvstore->ctx, &req->io_req);
2720154972afSPatrick Mooney if (err)
2721154972afSPatrick Mooney pci_nvme_status_genc(status, NVME_SC_INTERNAL_DEVICE_ERROR);
27226960cd89SAndy Fiddaman else
27236960cd89SAndy Fiddaman pending = true;
2724154972afSPatrick Mooney }
2725154972afSPatrick Mooney out:
27266960cd89SAndy Fiddaman free(range);
27276960cd89SAndy Fiddaman return (pending);
2728154972afSPatrick Mooney }
27294c87aefeSPatrick Mooney
27304c87aefeSPatrick Mooney static void
pci_nvme_handle_io_cmd(struct pci_nvme_softc * sc,uint16_t idx)27314c87aefeSPatrick Mooney pci_nvme_handle_io_cmd(struct pci_nvme_softc* sc, uint16_t idx)
27324c87aefeSPatrick Mooney {
27334c87aefeSPatrick Mooney struct nvme_submission_queue *sq;
27342b948146SAndy Fiddaman uint16_t status;
27354c87aefeSPatrick Mooney uint16_t sqhead;
27364c87aefeSPatrick Mooney
27374c87aefeSPatrick Mooney /* handle all submissions up to sq->tail index */
27384c87aefeSPatrick Mooney sq = &sc->submit_queues[idx];
27394c87aefeSPatrick Mooney
27406960cd89SAndy Fiddaman pthread_mutex_lock(&sq->mtx);
27414c87aefeSPatrick Mooney
27426960cd89SAndy Fiddaman sqhead = sq->head;
27436960cd89SAndy Fiddaman DPRINTF("nvme_handle_io qid %u head %u tail %u cmdlist %p",
27446960cd89SAndy Fiddaman idx, sqhead, sq->tail, sq->qbase);
27454c87aefeSPatrick Mooney
27464c87aefeSPatrick Mooney while (sqhead != atomic_load_acq_short(&sq->tail)) {
27474c87aefeSPatrick Mooney struct nvme_command *cmd;
27486960cd89SAndy Fiddaman struct pci_nvme_ioreq *req;
27496960cd89SAndy Fiddaman uint32_t nsid;
27506960cd89SAndy Fiddaman bool pending;
27514c87aefeSPatrick Mooney
27526960cd89SAndy Fiddaman pending = false;
27536960cd89SAndy Fiddaman req = NULL;
27546960cd89SAndy Fiddaman status = 0;
27554c87aefeSPatrick Mooney
27564c87aefeSPatrick Mooney cmd = &sq->qbase[sqhead];
27574c87aefeSPatrick Mooney sqhead = (sqhead + 1) % sq->size;
27584c87aefeSPatrick Mooney
27596960cd89SAndy Fiddaman nsid = le32toh(cmd->nsid);
27606960cd89SAndy Fiddaman if ((nsid == 0) || (nsid > sc->ctrldata.nn)) {
27616960cd89SAndy Fiddaman pci_nvme_status_genc(&status,
27626960cd89SAndy Fiddaman NVME_SC_INVALID_NAMESPACE_OR_FORMAT);
27636960cd89SAndy Fiddaman status |=
27646960cd89SAndy Fiddaman NVME_STATUS_DNR_MASK << NVME_STATUS_DNR_SHIFT;
27656960cd89SAndy Fiddaman goto complete;
27666960cd89SAndy Fiddaman }
27674c87aefeSPatrick Mooney
27686960cd89SAndy Fiddaman req = pci_nvme_get_ioreq(sc);
27696960cd89SAndy Fiddaman if (req == NULL) {
27706960cd89SAndy Fiddaman pci_nvme_status_genc(&status,
27716960cd89SAndy Fiddaman NVME_SC_INTERNAL_DEVICE_ERROR);
27726960cd89SAndy Fiddaman WPRINTF("%s: unable to allocate IO req", __func__);
27736960cd89SAndy Fiddaman goto complete;
27744c87aefeSPatrick Mooney }
27756960cd89SAndy Fiddaman req->nvme_sq = sq;
27766960cd89SAndy Fiddaman req->sqid = idx;
27776960cd89SAndy Fiddaman req->opc = cmd->opc;
27786960cd89SAndy Fiddaman req->cid = cmd->cid;
27796960cd89SAndy Fiddaman req->nsid = cmd->nsid;
27804c87aefeSPatrick Mooney
27814c87aefeSPatrick Mooney switch (cmd->opc) {
27826960cd89SAndy Fiddaman case NVME_OPC_FLUSH:
27836960cd89SAndy Fiddaman pending = nvme_opc_flush(sc, cmd, &sc->nvstore,
27846960cd89SAndy Fiddaman req, &status);
27856960cd89SAndy Fiddaman break;
27866960cd89SAndy Fiddaman case NVME_OPC_WRITE:
27874c87aefeSPatrick Mooney case NVME_OPC_READ:
27886960cd89SAndy Fiddaman pending = nvme_opc_write_read(sc, cmd, &sc->nvstore,
27896960cd89SAndy Fiddaman req, &status);
27904c87aefeSPatrick Mooney break;
27916960cd89SAndy Fiddaman case NVME_OPC_WRITE_ZEROES:
27926960cd89SAndy Fiddaman /* TODO: write zeroes
27936960cd89SAndy Fiddaman WPRINTF("%s write zeroes lba 0x%lx blocks %u",
27946960cd89SAndy Fiddaman __func__, lba, cmd->cdw12 & 0xFFFF); */
27956960cd89SAndy Fiddaman pci_nvme_status_genc(&status, NVME_SC_SUCCESS);
27964c87aefeSPatrick Mooney break;
27976960cd89SAndy Fiddaman case NVME_OPC_DATASET_MANAGEMENT:
27986960cd89SAndy Fiddaman pending = nvme_opc_dataset_mgmt(sc, cmd, &sc->nvstore,
27996960cd89SAndy Fiddaman req, &status);
28006960cd89SAndy Fiddaman break;
28016960cd89SAndy Fiddaman default:
28026960cd89SAndy Fiddaman WPRINTF("%s unhandled io command 0x%x",
28036960cd89SAndy Fiddaman __func__, cmd->opc);
28046960cd89SAndy Fiddaman pci_nvme_status_genc(&status, NVME_SC_INVALID_OPCODE);
28054c87aefeSPatrick Mooney }
28066960cd89SAndy Fiddaman complete:
28076960cd89SAndy Fiddaman if (!pending) {
28084f3f3e9aSAndy Fiddaman pci_nvme_set_completion(sc, sq, idx, cmd->cid, status);
28096960cd89SAndy Fiddaman if (req != NULL)
28106960cd89SAndy Fiddaman pci_nvme_release_ioreq(sc, req);
28114c87aefeSPatrick Mooney }
28124c87aefeSPatrick Mooney }
28134c87aefeSPatrick Mooney
28146960cd89SAndy Fiddaman sq->head = sqhead;
28156960cd89SAndy Fiddaman
28166960cd89SAndy Fiddaman pthread_mutex_unlock(&sq->mtx);
28174c87aefeSPatrick Mooney }
28184c87aefeSPatrick Mooney
28194c87aefeSPatrick Mooney static void
pci_nvme_handle_doorbell(struct pci_nvme_softc * sc,uint64_t idx,int is_sq,uint64_t value)282059d65d31SAndy Fiddaman pci_nvme_handle_doorbell(struct pci_nvme_softc* sc,
28214c87aefeSPatrick Mooney uint64_t idx, int is_sq, uint64_t value)
28224c87aefeSPatrick Mooney {
28236960cd89SAndy Fiddaman DPRINTF("nvme doorbell %lu, %s, val 0x%lx",
28246960cd89SAndy Fiddaman idx, is_sq ? "SQ" : "CQ", value & 0xFFFF);
28254c87aefeSPatrick Mooney
28264c87aefeSPatrick Mooney if (is_sq) {
28276960cd89SAndy Fiddaman if (idx > sc->num_squeues) {
28286960cd89SAndy Fiddaman WPRINTF("%s queue index %lu overflow from "
28296960cd89SAndy Fiddaman "guest (max %u)",
28306960cd89SAndy Fiddaman __func__, idx, sc->num_squeues);
28316960cd89SAndy Fiddaman return;
28326960cd89SAndy Fiddaman }
28336960cd89SAndy Fiddaman
28344c87aefeSPatrick Mooney atomic_store_short(&sc->submit_queues[idx].tail,
28354c87aefeSPatrick Mooney (uint16_t)value);
28364c87aefeSPatrick Mooney
28374c87aefeSPatrick Mooney if (idx == 0) {
28384c87aefeSPatrick Mooney pci_nvme_handle_admin_cmd(sc, value);
28394c87aefeSPatrick Mooney } else {
28404c87aefeSPatrick Mooney /* submission queue; handle new entries in SQ */
28414c87aefeSPatrick Mooney if (idx > sc->num_squeues) {
28426960cd89SAndy Fiddaman WPRINTF("%s SQ index %lu overflow from "
2843154972afSPatrick Mooney "guest (max %u)",
28446960cd89SAndy Fiddaman __func__, idx, sc->num_squeues);
28454c87aefeSPatrick Mooney return;
28464c87aefeSPatrick Mooney }
28474c87aefeSPatrick Mooney pci_nvme_handle_io_cmd(sc, (uint16_t)idx);
28484c87aefeSPatrick Mooney }
28494c87aefeSPatrick Mooney } else {
28504c87aefeSPatrick Mooney if (idx > sc->num_cqueues) {
28516960cd89SAndy Fiddaman WPRINTF("%s queue index %lu overflow from "
2852154972afSPatrick Mooney "guest (max %u)",
28536960cd89SAndy Fiddaman __func__, idx, sc->num_cqueues);
28544c87aefeSPatrick Mooney return;
28554c87aefeSPatrick Mooney }
28564c87aefeSPatrick Mooney
28576960cd89SAndy Fiddaman atomic_store_short(&sc->compl_queues[idx].head,
28586960cd89SAndy Fiddaman (uint16_t)value);
28594c87aefeSPatrick Mooney }
28604c87aefeSPatrick Mooney }
28614c87aefeSPatrick Mooney
28624c87aefeSPatrick Mooney static void
pci_nvme_bar0_reg_dumps(const char * func,uint64_t offset,int iswrite)28634c87aefeSPatrick Mooney pci_nvme_bar0_reg_dumps(const char *func, uint64_t offset, int iswrite)
28644c87aefeSPatrick Mooney {
28654c87aefeSPatrick Mooney const char *s = iswrite ? "WRITE" : "READ";
28664c87aefeSPatrick Mooney
28674c87aefeSPatrick Mooney switch (offset) {
28684c87aefeSPatrick Mooney case NVME_CR_CAP_LOW:
28696960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_CAP_LOW", func, s);
28704c87aefeSPatrick Mooney break;
28714c87aefeSPatrick Mooney case NVME_CR_CAP_HI:
28726960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_CAP_HI", func, s);
28734c87aefeSPatrick Mooney break;
28744c87aefeSPatrick Mooney case NVME_CR_VS:
28756960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_VS", func, s);
28764c87aefeSPatrick Mooney break;
28774c87aefeSPatrick Mooney case NVME_CR_INTMS:
28786960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_INTMS", func, s);
28794c87aefeSPatrick Mooney break;
28804c87aefeSPatrick Mooney case NVME_CR_INTMC:
28816960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_INTMC", func, s);
28824c87aefeSPatrick Mooney break;
28834c87aefeSPatrick Mooney case NVME_CR_CC:
28846960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_CC", func, s);
28854c87aefeSPatrick Mooney break;
28864c87aefeSPatrick Mooney case NVME_CR_CSTS:
28876960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_CSTS", func, s);
28884c87aefeSPatrick Mooney break;
28894c87aefeSPatrick Mooney case NVME_CR_NSSR:
28906960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_NSSR", func, s);
28914c87aefeSPatrick Mooney break;
28924c87aefeSPatrick Mooney case NVME_CR_AQA:
28936960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_AQA", func, s);
28944c87aefeSPatrick Mooney break;
28954c87aefeSPatrick Mooney case NVME_CR_ASQ_LOW:
28966960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_ASQ_LOW", func, s);
28974c87aefeSPatrick Mooney break;
28984c87aefeSPatrick Mooney case NVME_CR_ASQ_HI:
28996960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_ASQ_HI", func, s);
29004c87aefeSPatrick Mooney break;
29014c87aefeSPatrick Mooney case NVME_CR_ACQ_LOW:
29026960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_ACQ_LOW", func, s);
29034c87aefeSPatrick Mooney break;
29044c87aefeSPatrick Mooney case NVME_CR_ACQ_HI:
29056960cd89SAndy Fiddaman DPRINTF("%s %s NVME_CR_ACQ_HI", func, s);
29064c87aefeSPatrick Mooney break;
29074c87aefeSPatrick Mooney default:
29086960cd89SAndy Fiddaman DPRINTF("unknown nvme bar-0 offset 0x%lx", offset);
29094c87aefeSPatrick Mooney }
29104c87aefeSPatrick Mooney
29114c87aefeSPatrick Mooney }
29124c87aefeSPatrick Mooney
29134c87aefeSPatrick Mooney static void
pci_nvme_write_bar_0(struct pci_nvme_softc * sc,uint64_t offset,int size,uint64_t value)2914*32640292SAndy Fiddaman pci_nvme_write_bar_0(struct pci_nvme_softc *sc, uint64_t offset, int size,
2915*32640292SAndy Fiddaman uint64_t value)
29164c87aefeSPatrick Mooney {
29174c87aefeSPatrick Mooney uint32_t ccreg;
29184c87aefeSPatrick Mooney
29194c87aefeSPatrick Mooney if (offset >= NVME_DOORBELL_OFFSET) {
29204c87aefeSPatrick Mooney uint64_t belloffset = offset - NVME_DOORBELL_OFFSET;
29214c87aefeSPatrick Mooney uint64_t idx = belloffset / 8; /* door bell size = 2*int */
29224c87aefeSPatrick Mooney int is_sq = (belloffset % 8) < 4;
29234c87aefeSPatrick Mooney
29244f3f3e9aSAndy Fiddaman if ((sc->regs.csts & NVME_CSTS_RDY) == 0) {
29254f3f3e9aSAndy Fiddaman WPRINTF("doorbell write prior to RDY (offset=%#lx)\n",
29264f3f3e9aSAndy Fiddaman offset);
29274f3f3e9aSAndy Fiddaman return;
29284f3f3e9aSAndy Fiddaman }
29294f3f3e9aSAndy Fiddaman
29304c87aefeSPatrick Mooney if (belloffset > ((sc->max_queues+1) * 8 - 4)) {
29316960cd89SAndy Fiddaman WPRINTF("guest attempted an overflow write offset "
29324c87aefeSPatrick Mooney "0x%lx, val 0x%lx in %s",
29336960cd89SAndy Fiddaman offset, value, __func__);
29344c87aefeSPatrick Mooney return;
29354c87aefeSPatrick Mooney }
29364c87aefeSPatrick Mooney
29374f3f3e9aSAndy Fiddaman if (is_sq) {
29384f3f3e9aSAndy Fiddaman if (sc->submit_queues[idx].qbase == NULL)
29394f3f3e9aSAndy Fiddaman return;
29404f3f3e9aSAndy Fiddaman } else if (sc->compl_queues[idx].qbase == NULL)
29414f3f3e9aSAndy Fiddaman return;
29424f3f3e9aSAndy Fiddaman
294359d65d31SAndy Fiddaman pci_nvme_handle_doorbell(sc, idx, is_sq, value);
29444c87aefeSPatrick Mooney return;
29454c87aefeSPatrick Mooney }
29464c87aefeSPatrick Mooney
29476960cd89SAndy Fiddaman DPRINTF("nvme-write offset 0x%lx, size %d, value 0x%lx",
29486960cd89SAndy Fiddaman offset, size, value);
29494c87aefeSPatrick Mooney
29504c87aefeSPatrick Mooney if (size != 4) {
29516960cd89SAndy Fiddaman WPRINTF("guest wrote invalid size %d (offset 0x%lx, "
29524c87aefeSPatrick Mooney "val 0x%lx) to bar0 in %s",
29536960cd89SAndy Fiddaman size, offset, value, __func__);
29544c87aefeSPatrick Mooney /* TODO: shutdown device */
29554c87aefeSPatrick Mooney return;
29564c87aefeSPatrick Mooney }
29574c87aefeSPatrick Mooney
29584c87aefeSPatrick Mooney pci_nvme_bar0_reg_dumps(__func__, offset, 1);
29594c87aefeSPatrick Mooney
29604c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx);
29614c87aefeSPatrick Mooney
29624c87aefeSPatrick Mooney switch (offset) {
29634c87aefeSPatrick Mooney case NVME_CR_CAP_LOW:
29644c87aefeSPatrick Mooney case NVME_CR_CAP_HI:
29654c87aefeSPatrick Mooney /* readonly */
29664c87aefeSPatrick Mooney break;
29674c87aefeSPatrick Mooney case NVME_CR_VS:
29684c87aefeSPatrick Mooney /* readonly */
29694c87aefeSPatrick Mooney break;
29704c87aefeSPatrick Mooney case NVME_CR_INTMS:
29714c87aefeSPatrick Mooney /* MSI-X, so ignore */
29724c87aefeSPatrick Mooney break;
29734c87aefeSPatrick Mooney case NVME_CR_INTMC:
29744c87aefeSPatrick Mooney /* MSI-X, so ignore */
29754c87aefeSPatrick Mooney break;
29764c87aefeSPatrick Mooney case NVME_CR_CC:
29774c87aefeSPatrick Mooney ccreg = (uint32_t)value;
29784c87aefeSPatrick Mooney
29796960cd89SAndy Fiddaman DPRINTF("%s NVME_CR_CC en %x css %x shn %x iosqes %u "
2980154972afSPatrick Mooney "iocqes %u",
29814c87aefeSPatrick Mooney __func__,
29824c87aefeSPatrick Mooney NVME_CC_GET_EN(ccreg), NVME_CC_GET_CSS(ccreg),
29834c87aefeSPatrick Mooney NVME_CC_GET_SHN(ccreg), NVME_CC_GET_IOSQES(ccreg),
29846960cd89SAndy Fiddaman NVME_CC_GET_IOCQES(ccreg));
29854c87aefeSPatrick Mooney
29864c87aefeSPatrick Mooney if (NVME_CC_GET_SHN(ccreg)) {
29874c87aefeSPatrick Mooney /* perform shutdown - flush out data to backend */
29884c87aefeSPatrick Mooney sc->regs.csts &= ~(NVME_CSTS_REG_SHST_MASK <<
29894c87aefeSPatrick Mooney NVME_CSTS_REG_SHST_SHIFT);
29904c87aefeSPatrick Mooney sc->regs.csts |= NVME_SHST_COMPLETE <<
29914c87aefeSPatrick Mooney NVME_CSTS_REG_SHST_SHIFT;
29924c87aefeSPatrick Mooney }
29934c87aefeSPatrick Mooney if (NVME_CC_GET_EN(ccreg) != NVME_CC_GET_EN(sc->regs.cc)) {
29944c87aefeSPatrick Mooney if (NVME_CC_GET_EN(ccreg) == 0)
29954c87aefeSPatrick Mooney /* transition 1-> causes controller reset */
29964c87aefeSPatrick Mooney pci_nvme_reset_locked(sc);
29974c87aefeSPatrick Mooney else
2998*32640292SAndy Fiddaman pci_nvme_init_controller(sc);
29994c87aefeSPatrick Mooney }
30004c87aefeSPatrick Mooney
30014c87aefeSPatrick Mooney /* Insert the iocqes, iosqes and en bits from the write */
30024c87aefeSPatrick Mooney sc->regs.cc &= ~NVME_CC_WRITE_MASK;
30034c87aefeSPatrick Mooney sc->regs.cc |= ccreg & NVME_CC_WRITE_MASK;
30044c87aefeSPatrick Mooney if (NVME_CC_GET_EN(ccreg) == 0) {
30054c87aefeSPatrick Mooney /* Insert the ams, mps and css bit fields */
30064c87aefeSPatrick Mooney sc->regs.cc &= ~NVME_CC_NEN_WRITE_MASK;
30074c87aefeSPatrick Mooney sc->regs.cc |= ccreg & NVME_CC_NEN_WRITE_MASK;
30084c87aefeSPatrick Mooney sc->regs.csts &= ~NVME_CSTS_RDY;
30094f3f3e9aSAndy Fiddaman } else if ((sc->pending_ios == 0) &&
30104f3f3e9aSAndy Fiddaman !(sc->regs.csts & NVME_CSTS_CFS)) {
30114c87aefeSPatrick Mooney sc->regs.csts |= NVME_CSTS_RDY;
30124c87aefeSPatrick Mooney }
30134c87aefeSPatrick Mooney break;
30144c87aefeSPatrick Mooney case NVME_CR_CSTS:
30154c87aefeSPatrick Mooney break;
30164c87aefeSPatrick Mooney case NVME_CR_NSSR:
30174c87aefeSPatrick Mooney /* ignore writes; don't support subsystem reset */
30184c87aefeSPatrick Mooney break;
30194c87aefeSPatrick Mooney case NVME_CR_AQA:
30204c87aefeSPatrick Mooney sc->regs.aqa = (uint32_t)value;
30214c87aefeSPatrick Mooney break;
30224c87aefeSPatrick Mooney case NVME_CR_ASQ_LOW:
30234c87aefeSPatrick Mooney sc->regs.asq = (sc->regs.asq & (0xFFFFFFFF00000000)) |
30244c87aefeSPatrick Mooney (0xFFFFF000 & value);
30254c87aefeSPatrick Mooney break;
30264c87aefeSPatrick Mooney case NVME_CR_ASQ_HI:
30274c87aefeSPatrick Mooney sc->regs.asq = (sc->regs.asq & (0x00000000FFFFFFFF)) |
30284c87aefeSPatrick Mooney (value << 32);
30294c87aefeSPatrick Mooney break;
30304c87aefeSPatrick Mooney case NVME_CR_ACQ_LOW:
30314c87aefeSPatrick Mooney sc->regs.acq = (sc->regs.acq & (0xFFFFFFFF00000000)) |
30324c87aefeSPatrick Mooney (0xFFFFF000 & value);
30334c87aefeSPatrick Mooney break;
30344c87aefeSPatrick Mooney case NVME_CR_ACQ_HI:
30354c87aefeSPatrick Mooney sc->regs.acq = (sc->regs.acq & (0x00000000FFFFFFFF)) |
30364c87aefeSPatrick Mooney (value << 32);
30374c87aefeSPatrick Mooney break;
30384c87aefeSPatrick Mooney default:
30396960cd89SAndy Fiddaman DPRINTF("%s unknown offset 0x%lx, value 0x%lx size %d",
30406960cd89SAndy Fiddaman __func__, offset, value, size);
30414c87aefeSPatrick Mooney }
30424c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx);
30434c87aefeSPatrick Mooney }
30444c87aefeSPatrick Mooney
30454c87aefeSPatrick Mooney static void
pci_nvme_write(struct pci_devinst * pi,int baridx,uint64_t offset,int size,uint64_t value)3046*32640292SAndy Fiddaman pci_nvme_write(struct pci_devinst *pi, int baridx, uint64_t offset, int size,
3047*32640292SAndy Fiddaman uint64_t value)
30484c87aefeSPatrick Mooney {
30494c87aefeSPatrick Mooney struct pci_nvme_softc* sc = pi->pi_arg;
30504c87aefeSPatrick Mooney
30514c87aefeSPatrick Mooney if (baridx == pci_msix_table_bar(pi) ||
30524c87aefeSPatrick Mooney baridx == pci_msix_pba_bar(pi)) {
30536960cd89SAndy Fiddaman DPRINTF("nvme-write baridx %d, msix: off 0x%lx, size %d, "
30546960cd89SAndy Fiddaman " value 0x%lx", baridx, offset, size, value);
30554c87aefeSPatrick Mooney
30564c87aefeSPatrick Mooney pci_emul_msix_twrite(pi, offset, size, value);
30574c87aefeSPatrick Mooney return;
30584c87aefeSPatrick Mooney }
30594c87aefeSPatrick Mooney
30604c87aefeSPatrick Mooney switch (baridx) {
30614c87aefeSPatrick Mooney case 0:
3062*32640292SAndy Fiddaman pci_nvme_write_bar_0(sc, offset, size, value);
30634c87aefeSPatrick Mooney break;
30644c87aefeSPatrick Mooney
30654c87aefeSPatrick Mooney default:
30666960cd89SAndy Fiddaman DPRINTF("%s unknown baridx %d, val 0x%lx",
30676960cd89SAndy Fiddaman __func__, baridx, value);
30684c87aefeSPatrick Mooney }
30694c87aefeSPatrick Mooney }
30704c87aefeSPatrick Mooney
pci_nvme_read_bar_0(struct pci_nvme_softc * sc,uint64_t offset,int size)30714c87aefeSPatrick Mooney static uint64_t pci_nvme_read_bar_0(struct pci_nvme_softc* sc,
30724c87aefeSPatrick Mooney uint64_t offset, int size)
30734c87aefeSPatrick Mooney {
30744c87aefeSPatrick Mooney uint64_t value;
30754c87aefeSPatrick Mooney
30764c87aefeSPatrick Mooney pci_nvme_bar0_reg_dumps(__func__, offset, 0);
30774c87aefeSPatrick Mooney
30784c87aefeSPatrick Mooney if (offset < NVME_DOORBELL_OFFSET) {
30794c87aefeSPatrick Mooney void *p = &(sc->regs);
30804c87aefeSPatrick Mooney pthread_mutex_lock(&sc->mtx);
30814c87aefeSPatrick Mooney memcpy(&value, (void *)((uintptr_t)p + offset), size);
30824c87aefeSPatrick Mooney pthread_mutex_unlock(&sc->mtx);
30834c87aefeSPatrick Mooney } else {
30844c87aefeSPatrick Mooney value = 0;
30856960cd89SAndy Fiddaman WPRINTF("pci_nvme: read invalid offset %ld", offset);
30864c87aefeSPatrick Mooney }
30874c87aefeSPatrick Mooney
30884c87aefeSPatrick Mooney switch (size) {
30894c87aefeSPatrick Mooney case 1:
30904c87aefeSPatrick Mooney value &= 0xFF;
30914c87aefeSPatrick Mooney break;
30924c87aefeSPatrick Mooney case 2:
30934c87aefeSPatrick Mooney value &= 0xFFFF;
30944c87aefeSPatrick Mooney break;
30954c87aefeSPatrick Mooney case 4:
30964c87aefeSPatrick Mooney value &= 0xFFFFFFFF;
30974c87aefeSPatrick Mooney break;
30984c87aefeSPatrick Mooney }
30994c87aefeSPatrick Mooney
31006960cd89SAndy Fiddaman DPRINTF(" nvme-read offset 0x%lx, size %d -> value 0x%x",
31016960cd89SAndy Fiddaman offset, size, (uint32_t)value);
31024c87aefeSPatrick Mooney
31034c87aefeSPatrick Mooney return (value);
31044c87aefeSPatrick Mooney }
31054c87aefeSPatrick Mooney
31064c87aefeSPatrick Mooney
31074c87aefeSPatrick Mooney
31084c87aefeSPatrick Mooney static uint64_t
pci_nvme_read(struct pci_devinst * pi,int baridx,uint64_t offset,int size)3109*32640292SAndy Fiddaman pci_nvme_read(struct pci_devinst *pi, int baridx, uint64_t offset, int size)
31104c87aefeSPatrick Mooney {
31114c87aefeSPatrick Mooney struct pci_nvme_softc* sc = pi->pi_arg;
31124c87aefeSPatrick Mooney
31134c87aefeSPatrick Mooney if (baridx == pci_msix_table_bar(pi) ||
31144c87aefeSPatrick Mooney baridx == pci_msix_pba_bar(pi)) {
31156960cd89SAndy Fiddaman DPRINTF("nvme-read bar: %d, msix: regoff 0x%lx, size %d",
31166960cd89SAndy Fiddaman baridx, offset, size);
31174c87aefeSPatrick Mooney
31184c87aefeSPatrick Mooney return pci_emul_msix_tread(pi, offset, size);
31194c87aefeSPatrick Mooney }
31204c87aefeSPatrick Mooney
31214c87aefeSPatrick Mooney switch (baridx) {
31224c87aefeSPatrick Mooney case 0:
31234c87aefeSPatrick Mooney return pci_nvme_read_bar_0(sc, offset, size);
31244c87aefeSPatrick Mooney
31254c87aefeSPatrick Mooney default:
31266960cd89SAndy Fiddaman DPRINTF("unknown bar %d, 0x%lx", baridx, offset);
31274c87aefeSPatrick Mooney }
31284c87aefeSPatrick Mooney
31294c87aefeSPatrick Mooney return (0);
31304c87aefeSPatrick Mooney }
31314c87aefeSPatrick Mooney
31324c87aefeSPatrick Mooney static int
pci_nvme_parse_config(struct pci_nvme_softc * sc,nvlist_t * nvl)31332b948146SAndy Fiddaman pci_nvme_parse_config(struct pci_nvme_softc *sc, nvlist_t *nvl)
31344c87aefeSPatrick Mooney {
313559d65d31SAndy Fiddaman char bident[sizeof("XXX:XXX")];
31362b948146SAndy Fiddaman const char *value;
31374c87aefeSPatrick Mooney uint32_t sectsz;
31384c87aefeSPatrick Mooney
31394c87aefeSPatrick Mooney sc->max_queues = NVME_QUEUES;
31404c87aefeSPatrick Mooney sc->max_qentries = NVME_MAX_QENTRIES;
31414c87aefeSPatrick Mooney sc->ioslots = NVME_IOSLOTS;
31424c87aefeSPatrick Mooney sc->num_squeues = sc->max_queues;
31434c87aefeSPatrick Mooney sc->num_cqueues = sc->max_queues;
3144154972afSPatrick Mooney sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO;
31454c87aefeSPatrick Mooney sectsz = 0;
314659d65d31SAndy Fiddaman #ifdef __FreeBSD__
31474c87aefeSPatrick Mooney snprintf(sc->ctrldata.sn, sizeof(sc->ctrldata.sn),
31484c87aefeSPatrick Mooney "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
314959d65d31SAndy Fiddaman #else
315059d65d31SAndy Fiddaman snprintf((char *)sc->ctrldata.sn, sizeof(sc->ctrldata.sn),
315159d65d31SAndy Fiddaman "NVME-%d-%d", sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
315259d65d31SAndy Fiddaman #endif
31532b948146SAndy Fiddaman
31542b948146SAndy Fiddaman value = get_config_value_node(nvl, "maxq");
31552b948146SAndy Fiddaman if (value != NULL)
31562b948146SAndy Fiddaman sc->max_queues = atoi(value);
31572b948146SAndy Fiddaman value = get_config_value_node(nvl, "qsz");
31582b948146SAndy Fiddaman if (value != NULL) {
31592b948146SAndy Fiddaman sc->max_qentries = atoi(value);
31602b948146SAndy Fiddaman if (sc->max_qentries <= 0) {
31612b948146SAndy Fiddaman EPRINTLN("nvme: Invalid qsz option %d",
31622b948146SAndy Fiddaman sc->max_qentries);
31634c87aefeSPatrick Mooney return (-1);
31644c87aefeSPatrick Mooney }
31654c87aefeSPatrick Mooney }
31662b948146SAndy Fiddaman value = get_config_value_node(nvl, "ioslots");
31672b948146SAndy Fiddaman if (value != NULL) {
31682b948146SAndy Fiddaman sc->ioslots = atoi(value);
31692b948146SAndy Fiddaman if (sc->ioslots <= 0) {
31702b948146SAndy Fiddaman EPRINTLN("Invalid ioslots option %d", sc->ioslots);
31712b948146SAndy Fiddaman return (-1);
31722b948146SAndy Fiddaman }
31734c87aefeSPatrick Mooney }
31742b948146SAndy Fiddaman value = get_config_value_node(nvl, "sectsz");
31752b948146SAndy Fiddaman if (value != NULL)
31762b948146SAndy Fiddaman sectsz = atoi(value);
31772b948146SAndy Fiddaman value = get_config_value_node(nvl, "ser");
31782b948146SAndy Fiddaman if (value != NULL) {
31792b948146SAndy Fiddaman /*
31802b948146SAndy Fiddaman * This field indicates the Product Serial Number in
31812b948146SAndy Fiddaman * 7-bit ASCII, unused bytes should be space characters.
31822b948146SAndy Fiddaman * Ref: NVMe v1.3c.
31832b948146SAndy Fiddaman */
31842b948146SAndy Fiddaman cpywithpad((char *)sc->ctrldata.sn,
31852b948146SAndy Fiddaman sizeof(sc->ctrldata.sn), value, ' ');
31862b948146SAndy Fiddaman }
31872b948146SAndy Fiddaman value = get_config_value_node(nvl, "eui64");
31882b948146SAndy Fiddaman if (value != NULL)
31892b948146SAndy Fiddaman sc->nvstore.eui64 = htobe64(strtoull(value, NULL, 0));
31902b948146SAndy Fiddaman value = get_config_value_node(nvl, "dsm");
31912b948146SAndy Fiddaman if (value != NULL) {
31922b948146SAndy Fiddaman if (strcmp(value, "auto") == 0)
31932b948146SAndy Fiddaman sc->dataset_management = NVME_DATASET_MANAGEMENT_AUTO;
31942b948146SAndy Fiddaman else if (strcmp(value, "enable") == 0)
31952b948146SAndy Fiddaman sc->dataset_management = NVME_DATASET_MANAGEMENT_ENABLE;
31962b948146SAndy Fiddaman else if (strcmp(value, "disable") == 0)
31972b948146SAndy Fiddaman sc->dataset_management = NVME_DATASET_MANAGEMENT_DISABLE;
31982b948146SAndy Fiddaman }
31992b948146SAndy Fiddaman
3200*32640292SAndy Fiddaman value = get_config_value_node(nvl, "bootindex");
3201*32640292SAndy Fiddaman if (value != NULL) {
3202*32640292SAndy Fiddaman if (pci_emul_add_boot_device(sc->nsc_pi, atoi(value))) {
3203*32640292SAndy Fiddaman EPRINTLN("Invalid bootindex %d", atoi(value));
3204*32640292SAndy Fiddaman return (-1);
3205*32640292SAndy Fiddaman }
3206*32640292SAndy Fiddaman }
3207*32640292SAndy Fiddaman
32082b948146SAndy Fiddaman value = get_config_value_node(nvl, "ram");
32092b948146SAndy Fiddaman if (value != NULL) {
32102b948146SAndy Fiddaman uint64_t sz = strtoull(value, NULL, 10);
32112b948146SAndy Fiddaman
32122b948146SAndy Fiddaman sc->nvstore.type = NVME_STOR_RAM;
32132b948146SAndy Fiddaman sc->nvstore.size = sz * 1024 * 1024;
32142b948146SAndy Fiddaman sc->nvstore.ctx = calloc(1, sc->nvstore.size);
32152b948146SAndy Fiddaman sc->nvstore.sectsz = 4096;
32162b948146SAndy Fiddaman sc->nvstore.sectsz_bits = 12;
32172b948146SAndy Fiddaman if (sc->nvstore.ctx == NULL) {
32182b948146SAndy Fiddaman EPRINTLN("nvme: Unable to allocate RAM");
32192b948146SAndy Fiddaman return (-1);
32202b948146SAndy Fiddaman }
32212b948146SAndy Fiddaman } else {
322259d65d31SAndy Fiddaman snprintf(bident, sizeof(bident), "%u:%u",
32232b948146SAndy Fiddaman sc->nsc_pi->pi_slot, sc->nsc_pi->pi_func);
32242b948146SAndy Fiddaman sc->nvstore.ctx = blockif_open(nvl, bident);
32252b948146SAndy Fiddaman if (sc->nvstore.ctx == NULL) {
32262b948146SAndy Fiddaman EPRINTLN("nvme: Could not open backing file: %s",
32272b948146SAndy Fiddaman strerror(errno));
32282b948146SAndy Fiddaman return (-1);
32292b948146SAndy Fiddaman }
32302b948146SAndy Fiddaman sc->nvstore.type = NVME_STOR_BLOCKIF;
32312b948146SAndy Fiddaman sc->nvstore.size = blockif_size(sc->nvstore.ctx);
32322b948146SAndy Fiddaman }
32332b948146SAndy Fiddaman
32344c87aefeSPatrick Mooney if (sectsz == 512 || sectsz == 4096 || sectsz == 8192)
32354c87aefeSPatrick Mooney sc->nvstore.sectsz = sectsz;
32364c87aefeSPatrick Mooney else if (sc->nvstore.type != NVME_STOR_RAM)
32374c87aefeSPatrick Mooney sc->nvstore.sectsz = blockif_sectsz(sc->nvstore.ctx);
32384c87aefeSPatrick Mooney for (sc->nvstore.sectsz_bits = 9;
323959d65d31SAndy Fiddaman (1U << sc->nvstore.sectsz_bits) < sc->nvstore.sectsz;
32404c87aefeSPatrick Mooney sc->nvstore.sectsz_bits++);
32414c87aefeSPatrick Mooney
32424c87aefeSPatrick Mooney if (sc->max_queues <= 0 || sc->max_queues > NVME_QUEUES)
32434c87aefeSPatrick Mooney sc->max_queues = NVME_QUEUES;
32444c87aefeSPatrick Mooney
32454c87aefeSPatrick Mooney return (0);
32464c87aefeSPatrick Mooney }
32474c87aefeSPatrick Mooney
32486dc98349SAndy Fiddaman static void
pci_nvme_resized(struct blockif_ctxt * bctxt __unused,void * arg,size_t new_size)32494f3f3e9aSAndy Fiddaman pci_nvme_resized(struct blockif_ctxt *bctxt __unused, void *arg,
32504f3f3e9aSAndy Fiddaman size_t new_size)
32516dc98349SAndy Fiddaman {
32526dc98349SAndy Fiddaman struct pci_nvme_softc *sc;
32536dc98349SAndy Fiddaman struct pci_nvme_blockstore *nvstore;
32546dc98349SAndy Fiddaman struct nvme_namespace_data *nd;
32556dc98349SAndy Fiddaman
32566dc98349SAndy Fiddaman sc = arg;
32576dc98349SAndy Fiddaman nvstore = &sc->nvstore;
32586dc98349SAndy Fiddaman nd = &sc->nsdata;
32596dc98349SAndy Fiddaman
32606dc98349SAndy Fiddaman nvstore->size = new_size;
32616dc98349SAndy Fiddaman pci_nvme_init_nsdata_size(nvstore, nd);
32626dc98349SAndy Fiddaman
32636dc98349SAndy Fiddaman /* Add changed NSID to list */
32646dc98349SAndy Fiddaman sc->ns_log.ns[0] = 1;
32656dc98349SAndy Fiddaman sc->ns_log.ns[1] = 0;
32666dc98349SAndy Fiddaman
32676dc98349SAndy Fiddaman pci_nvme_aen_post(sc, PCI_NVME_AE_TYPE_NOTICE,
3268d7b72f7bSAndy Fiddaman PCI_NVME_AEI_NOTICE_NS_ATTR_CHANGED);
32696dc98349SAndy Fiddaman }
32706dc98349SAndy Fiddaman
32714c87aefeSPatrick Mooney static int
pci_nvme_init(struct pci_devinst * pi,nvlist_t * nvl)3272*32640292SAndy Fiddaman pci_nvme_init(struct pci_devinst *pi, nvlist_t *nvl)
32734c87aefeSPatrick Mooney {
32744c87aefeSPatrick Mooney struct pci_nvme_softc *sc;
32754c87aefeSPatrick Mooney uint32_t pci_membar_sz;
32764c87aefeSPatrick Mooney int error;
32774c87aefeSPatrick Mooney
32784c87aefeSPatrick Mooney error = 0;
32794c87aefeSPatrick Mooney
32804c87aefeSPatrick Mooney sc = calloc(1, sizeof(struct pci_nvme_softc));
32814c87aefeSPatrick Mooney pi->pi_arg = sc;
32824c87aefeSPatrick Mooney sc->nsc_pi = pi;
32834c87aefeSPatrick Mooney
32842b948146SAndy Fiddaman error = pci_nvme_parse_config(sc, nvl);
32854c87aefeSPatrick Mooney if (error < 0)
32864c87aefeSPatrick Mooney goto done;
32874c87aefeSPatrick Mooney else
32884c87aefeSPatrick Mooney error = 0;
32894c87aefeSPatrick Mooney
3290154972afSPatrick Mooney STAILQ_INIT(&sc->ioreqs_free);
32914c87aefeSPatrick Mooney sc->ioreqs = calloc(sc->ioslots, sizeof(struct pci_nvme_ioreq));
329259d65d31SAndy Fiddaman for (uint32_t i = 0; i < sc->ioslots; i++) {
3293154972afSPatrick Mooney STAILQ_INSERT_TAIL(&sc->ioreqs_free, &sc->ioreqs[i], link);
32944c87aefeSPatrick Mooney }
32954c87aefeSPatrick Mooney
32964c87aefeSPatrick Mooney pci_set_cfgdata16(pi, PCIR_DEVICE, 0x0A0A);
32974c87aefeSPatrick Mooney pci_set_cfgdata16(pi, PCIR_VENDOR, 0xFB5D);
32984c87aefeSPatrick Mooney pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
32994c87aefeSPatrick Mooney pci_set_cfgdata8(pi, PCIR_SUBCLASS, PCIS_STORAGE_NVM);
33004c87aefeSPatrick Mooney pci_set_cfgdata8(pi, PCIR_PROGIF,
33014c87aefeSPatrick Mooney PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0);
33024c87aefeSPatrick Mooney
33034c87aefeSPatrick Mooney /*
33044c87aefeSPatrick Mooney * Allocate size of NVMe registers + doorbell space for all queues.
33054c87aefeSPatrick Mooney *
33064c87aefeSPatrick Mooney * The specification requires a minimum memory I/O window size of 16K.
33074c87aefeSPatrick Mooney * The Windows driver will refuse to start a device with a smaller
33084c87aefeSPatrick Mooney * window.
33094c87aefeSPatrick Mooney */
33104c87aefeSPatrick Mooney pci_membar_sz = sizeof(struct nvme_registers) +
33114c87aefeSPatrick Mooney 2 * sizeof(uint32_t) * (sc->max_queues + 1);
33124c87aefeSPatrick Mooney pci_membar_sz = MAX(pci_membar_sz, NVME_MMIO_SPACE_MIN);
33134c87aefeSPatrick Mooney
33146960cd89SAndy Fiddaman DPRINTF("nvme membar size: %u", pci_membar_sz);
33154c87aefeSPatrick Mooney
33164c87aefeSPatrick Mooney error = pci_emul_alloc_bar(pi, 0, PCIBAR_MEM64, pci_membar_sz);
33174c87aefeSPatrick Mooney if (error) {
33186960cd89SAndy Fiddaman WPRINTF("%s pci alloc mem bar failed", __func__);
33194c87aefeSPatrick Mooney goto done;
33204c87aefeSPatrick Mooney }
33214c87aefeSPatrick Mooney
33224c87aefeSPatrick Mooney error = pci_emul_add_msixcap(pi, sc->max_queues + 1, NVME_MSIX_BAR);
33234c87aefeSPatrick Mooney if (error) {
33246960cd89SAndy Fiddaman WPRINTF("%s pci add msixcap failed", __func__);
33254c87aefeSPatrick Mooney goto done;
33264c87aefeSPatrick Mooney }
33274c87aefeSPatrick Mooney
332884659b24SMichael Zeller error = pci_emul_add_pciecap(pi, PCIEM_TYPE_ROOT_INT_EP);
332984659b24SMichael Zeller if (error) {
33306960cd89SAndy Fiddaman WPRINTF("%s pci add Express capability failed", __func__);
333184659b24SMichael Zeller goto done;
333284659b24SMichael Zeller }
333384659b24SMichael Zeller
33344c87aefeSPatrick Mooney pthread_mutex_init(&sc->mtx, NULL);
33354c87aefeSPatrick Mooney sem_init(&sc->iosemlock, 0, sc->ioslots);
33366dc98349SAndy Fiddaman blockif_register_resize_callback(sc->nvstore.ctx, pci_nvme_resized, sc);
33374c87aefeSPatrick Mooney
33386960cd89SAndy Fiddaman pci_nvme_init_queues(sc, sc->max_queues, sc->max_queues);
3339154972afSPatrick Mooney /*
3340154972afSPatrick Mooney * Controller data depends on Namespace data so initialize Namespace
3341154972afSPatrick Mooney * data first.
3342154972afSPatrick Mooney */
3343154972afSPatrick Mooney pci_nvme_init_nsdata(sc, &sc->nsdata, 1, &sc->nvstore);
33444c87aefeSPatrick Mooney pci_nvme_init_ctrldata(sc);
33454c87aefeSPatrick Mooney pci_nvme_init_logpages(sc);
33466960cd89SAndy Fiddaman pci_nvme_init_features(sc);
33476960cd89SAndy Fiddaman
33486960cd89SAndy Fiddaman pci_nvme_aer_init(sc);
33496dc98349SAndy Fiddaman pci_nvme_aen_init(sc);
33506960cd89SAndy Fiddaman
33516960cd89SAndy Fiddaman pci_nvme_reset(sc);
33524c87aefeSPatrick Mooney done:
33534c87aefeSPatrick Mooney return (error);
33544c87aefeSPatrick Mooney }
33554c87aefeSPatrick Mooney
335676e6cd87SAndy Fiddaman static int
pci_nvme_legacy_config(nvlist_t * nvl,const char * opts)3357b0de25cbSAndy Fiddaman pci_nvme_legacy_config(nvlist_t *nvl, const char *opts)
335876e6cd87SAndy Fiddaman {
3359b0de25cbSAndy Fiddaman char *cp, *ram;
336076e6cd87SAndy Fiddaman
3361b0de25cbSAndy Fiddaman if (opts == NULL)
3362b0de25cbSAndy Fiddaman return (0);
336376e6cd87SAndy Fiddaman
3364b0de25cbSAndy Fiddaman if (strncmp(opts, "ram=", 4) == 0) {
3365b0de25cbSAndy Fiddaman cp = strchr(opts, ',');
3366b0de25cbSAndy Fiddaman if (cp == NULL) {
3367b0de25cbSAndy Fiddaman set_config_value_node(nvl, "ram", opts + 4);
3368b0de25cbSAndy Fiddaman return (0);
3369b0de25cbSAndy Fiddaman }
3370b0de25cbSAndy Fiddaman ram = strndup(opts + 4, cp - opts - 4);
3371b0de25cbSAndy Fiddaman set_config_value_node(nvl, "ram", ram);
3372b0de25cbSAndy Fiddaman free(ram);
3373b0de25cbSAndy Fiddaman return (pci_parse_legacy_config(nvl, cp + 1));
3374b0de25cbSAndy Fiddaman } else
3375b0de25cbSAndy Fiddaman return (blockif_legacy_config(nvl, opts));
337676e6cd87SAndy Fiddaman }
33774c87aefeSPatrick Mooney
33784f3f3e9aSAndy Fiddaman static const struct pci_devemu pci_de_nvme = {
33794c87aefeSPatrick Mooney .pe_emu = "nvme",
33804c87aefeSPatrick Mooney .pe_init = pci_nvme_init,
3381b0de25cbSAndy Fiddaman .pe_legacy_config = pci_nvme_legacy_config,
33824c87aefeSPatrick Mooney .pe_barwrite = pci_nvme_write,
33834c87aefeSPatrick Mooney .pe_barread = pci_nvme_read
33844c87aefeSPatrick Mooney };
33854c87aefeSPatrick Mooney PCI_EMUL_SET(pci_de_nvme);
3386