xref: /illumos-gate/usr/src/uts/common/io/nvme/nvme_var.h (revision 0999c1123c1ab769df080ccc5f1626d50663e7a8)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2018 Nexenta Systems, Inc.
14  * Copyright 2016 The MathWorks, Inc. All rights reserved.
15  * Copyright 2017 Joyent, Inc.
16  * Copyright 2019 Western Digital Corporation.
17  */
18 
19 #ifndef _NVME_VAR_H
20 #define	_NVME_VAR_H
21 
22 #include <sys/ddi.h>
23 #include <sys/sunddi.h>
24 #include <sys/blkdev.h>
25 #include <sys/taskq_impl.h>
26 #include <sys/list.h>
27 
28 /*
29  * NVMe driver state
30  */
31 
32 #ifdef __cplusplus
33 extern "C" {
34 #endif
35 
36 #define	NVME_FMA_INIT			0x1
37 #define	NVME_REGS_MAPPED		0x2
38 #define	NVME_ADMIN_QUEUE		0x4
39 #define	NVME_CTRL_LIMITS		0x8
40 #define	NVME_INTERRUPTS			0x10
41 
42 #define	NVME_MIN_ADMIN_QUEUE_LEN	16
43 #define	NVME_MIN_IO_QUEUE_LEN		16
44 #define	NVME_DEFAULT_ADMIN_QUEUE_LEN	256
45 #define	NVME_DEFAULT_IO_QUEUE_LEN	1024
46 #define	NVME_DEFAULT_ASYNC_EVENT_LIMIT	10
47 #define	NVME_MIN_ASYNC_EVENT_LIMIT	1
48 #define	NVME_DEFAULT_MIN_BLOCK_SIZE	512
49 
50 
51 typedef struct nvme nvme_t;
52 typedef struct nvme_namespace nvme_namespace_t;
53 typedef struct nvme_minor_state nvme_minor_state_t;
54 typedef struct nvme_dma nvme_dma_t;
55 typedef struct nvme_cmd nvme_cmd_t;
56 typedef struct nvme_cq nvme_cq_t;
57 typedef struct nvme_qpair nvme_qpair_t;
58 typedef struct nvme_task_arg nvme_task_arg_t;
59 
60 struct nvme_minor_state {
61 	kmutex_t	nm_mutex;
62 	boolean_t	nm_oexcl;
63 	uint_t		nm_ocnt;
64 };
65 
66 struct nvme_dma {
67 	ddi_dma_handle_t nd_dmah;
68 	ddi_acc_handle_t nd_acch;
69 	ddi_dma_cookie_t nd_cookie;
70 	uint_t nd_ncookie;
71 	caddr_t nd_memp;
72 	size_t nd_len;
73 	boolean_t nd_cached;
74 };
75 
76 struct nvme_cmd {
77 	struct list_node nc_list;
78 
79 	nvme_sqe_t nc_sqe;
80 	nvme_cqe_t nc_cqe;
81 
82 	void (*nc_callback)(void *);
83 	bd_xfer_t *nc_xfer;
84 	boolean_t nc_completed;
85 	boolean_t nc_dontpanic;
86 	uint16_t nc_sqid;
87 
88 	nvme_dma_t *nc_dma;
89 
90 	kmutex_t nc_mutex;
91 	kcondvar_t nc_cv;
92 
93 	taskq_ent_t nc_tqent;
94 	nvme_t *nc_nvme;
95 };
96 
97 struct nvme_cq {
98 	size_t ncq_nentry;
99 	uint16_t ncq_id;
100 
101 	nvme_dma_t *ncq_dma;
102 	nvme_cqe_t *ncq_cq;
103 	uint_t ncq_head;
104 	uint_t ncq_tail;
105 	uintptr_t ncq_hdbl;
106 	int ncq_phase;
107 
108 	kmutex_t ncq_mutex;
109 };
110 
111 struct nvme_qpair {
112 	size_t nq_nentry;
113 
114 	nvme_dma_t *nq_sqdma;
115 	nvme_sqe_t *nq_sq;
116 	uint_t nq_sqhead;
117 	uint_t nq_sqtail;
118 	uintptr_t nq_sqtdbl;
119 
120 	nvme_cq_t *nq_cq;
121 
122 	nvme_cmd_t **nq_cmd;
123 	uint16_t nq_next_cmd;
124 	uint_t nq_active_cmds;
125 
126 	kmutex_t nq_mutex;
127 	ksema_t nq_sema;
128 };
129 
130 struct nvme {
131 	dev_info_t *n_dip;
132 	int n_progress;
133 
134 	caddr_t n_regs;
135 	ddi_acc_handle_t n_regh;
136 
137 	kmem_cache_t *n_cmd_cache;
138 	kmem_cache_t *n_prp_cache;
139 
140 	size_t n_inth_sz;
141 	ddi_intr_handle_t *n_inth;
142 	int n_intr_cnt;
143 	uint_t n_intr_pri;
144 	int n_intr_cap;
145 	int n_intr_type;
146 	int n_intr_types;
147 
148 	char *n_product;
149 	char *n_vendor;
150 
151 	nvme_version_t n_version;
152 	boolean_t n_dead;
153 	boolean_t n_strict_version;
154 	boolean_t n_ignore_unknown_vendor_status;
155 	uint32_t n_admin_queue_len;
156 	uint32_t n_io_squeue_len;
157 	uint32_t n_io_cqueue_len;
158 	uint16_t n_async_event_limit;
159 	uint_t n_min_block_size;
160 	uint16_t n_abort_command_limit;
161 	uint64_t n_max_data_transfer_size;
162 	boolean_t n_write_cache_present;
163 	boolean_t n_write_cache_enabled;
164 	int n_error_log_len;
165 	boolean_t n_lba_range_supported;
166 	boolean_t n_auto_pst_supported;
167 	boolean_t n_async_event_supported;
168 	boolean_t n_progress_supported;
169 	int n_submission_queues;
170 	int n_completion_queues;
171 
172 	int n_nssr_supported;
173 	int n_doorbell_stride;
174 	int n_timeout;
175 	int n_arbitration_mechanisms;
176 	int n_cont_queues_reqd;
177 	int n_max_queue_entries;
178 	int n_pageshift;
179 	int n_pagesize;
180 
181 	int n_namespace_count;
182 	uint_t n_ioq_count;
183 	uint_t n_cq_count;
184 
185 	nvme_identify_ctrl_t *n_idctl;
186 
187 	nvme_qpair_t *n_adminq;
188 	nvme_qpair_t **n_ioq;
189 	nvme_cq_t **n_cq;
190 
191 	nvme_namespace_t *n_ns;
192 
193 	ddi_dma_attr_t n_queue_dma_attr;
194 	ddi_dma_attr_t n_prp_dma_attr;
195 	ddi_dma_attr_t n_sgl_dma_attr;
196 	ddi_device_acc_attr_t n_reg_acc_attr;
197 	ddi_iblock_cookie_t n_fm_ibc;
198 	int n_fm_cap;
199 
200 	ksema_t n_abort_sema;
201 
202 	ddi_taskq_t *n_cmd_taskq;
203 
204 	/* state for devctl minor node */
205 	nvme_minor_state_t n_minor;
206 
207 	/* errors detected by driver */
208 	uint32_t n_dma_bind_err;
209 	uint32_t n_abort_failed;
210 	uint32_t n_cmd_timeout;
211 	uint32_t n_cmd_aborted;
212 	uint32_t n_wrong_logpage;
213 	uint32_t n_unknown_logpage;
214 	uint32_t n_too_many_cookies;
215 
216 	/* errors detected by hardware */
217 	uint32_t n_data_xfr_err;
218 	uint32_t n_internal_err;
219 	uint32_t n_abort_rq_err;
220 	uint32_t n_abort_sq_del;
221 	uint32_t n_nvm_cap_exc;
222 	uint32_t n_nvm_ns_notrdy;
223 	uint32_t n_inv_cq_err;
224 	uint32_t n_inv_qid_err;
225 	uint32_t n_max_qsz_exc;
226 	uint32_t n_inv_int_vect;
227 	uint32_t n_inv_log_page;
228 	uint32_t n_inv_format;
229 	uint32_t n_inv_q_del;
230 	uint32_t n_cnfl_attr;
231 	uint32_t n_inv_prot;
232 	uint32_t n_readonly;
233 
234 	/* errors reported by asynchronous events */
235 	uint32_t n_diagfail_event;
236 	uint32_t n_persistent_event;
237 	uint32_t n_transient_event;
238 	uint32_t n_fw_load_event;
239 	uint32_t n_reliability_event;
240 	uint32_t n_temperature_event;
241 	uint32_t n_spare_event;
242 	uint32_t n_vendor_event;
243 	uint32_t n_unknown_event;
244 
245 };
246 
247 struct nvme_namespace {
248 	nvme_t *ns_nvme;
249 	uint8_t ns_eui64[8];
250 	char	ns_name[17];
251 
252 	bd_handle_t ns_bd_hdl;
253 
254 	uint32_t ns_id;
255 	size_t ns_block_count;
256 	size_t ns_block_size;
257 	size_t ns_best_block_size;
258 
259 	boolean_t ns_ignore;
260 
261 	nvme_identify_nsid_t *ns_idns;
262 
263 	/* state for attachment point minor node */
264 	nvme_minor_state_t ns_minor;
265 
266 	/*
267 	 * If a namespace has no EUI64, we create a devid in
268 	 * nvme_prepare_devid().
269 	 */
270 	char *ns_devid;
271 };
272 
273 struct nvme_task_arg {
274 	nvme_t *nt_nvme;
275 	nvme_cmd_t *nt_cmd;
276 };
277 
278 
279 #ifdef __cplusplus
280 }
281 #endif
282 
283 #endif /* _NVME_VAR_H */
284