1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2019 Joyent, Inc.
14  * Copyright 2022 OmniOS Community Edition (OmniOSce) Association.
15  */
16 
17 #ifndef _VIRTIO_IMPL_H
18 #define	_VIRTIO_IMPL_H
19 
20 /*
21  * VIRTIO FRAMEWORK: FRAMEWORK-PRIVATE DEFINITIONS
22  *
23  * For design and usage documentation, see the comments in "virtio.h".
24  *
25  * NOTE: Client drivers should not use definitions from this file.
26  */
27 
28 #include <sys/types.h>
29 #include <sys/dditypes.h>
30 #include <sys/list.h>
31 #include <sys/ccompile.h>
32 
33 #include "virtio.h"
34 
35 #ifdef __cplusplus
36 extern "C" {
37 #endif
38 
39 extern ddi_device_acc_attr_t virtio_acc_attr;
40 extern ddi_dma_attr_t virtio_dma_attr;
41 
42 typedef struct virtio_vq_desc virtio_vq_desc_t;
43 typedef struct virtio_vq_driver virtio_vq_driver_t;
44 typedef struct virtio_vq_device virtio_vq_device_t;
45 typedef struct virtio_vq_elem virtio_vq_elem_t;
46 
47 int virtio_dma_init(virtio_t *, virtio_dma_t *, size_t, const ddi_dma_attr_t *,
48     int, int);
49 void virtio_dma_fini(virtio_dma_t *);
50 
51 
52 
53 typedef enum virtio_dma_level {
54 	VIRTIO_DMALEVEL_HANDLE_ALLOC =	(1ULL << 0),
55 	VIRTIO_DMALEVEL_MEMORY_ALLOC =	(1ULL << 1),
56 	VIRTIO_DMALEVEL_HANDLE_BOUND =	(1ULL << 2),
57 	VIRTIO_DMALEVEL_COOKIE_ARRAY =	(1ULL << 3),
58 } virtio_dma_level_t;
59 
60 struct virtio_dma {
61 	virtio_dma_level_t		vidma_level;
62 	virtio_t			*vidma_virtio;
63 	caddr_t				vidma_va;
64 	size_t				vidma_size;
65 	size_t				vidma_real_size;
66 	ddi_dma_handle_t		vidma_dma_handle;
67 	ddi_acc_handle_t		vidma_acc_handle;
68 	uint_t				vidma_dma_ncookies;
69 	ddi_dma_cookie_t		*vidma_dma_cookies;
70 };
71 
72 typedef enum virtio_initlevel {
73 	VIRTIO_INITLEVEL_REGS =		(1ULL << 0),
74 	VIRTIO_INITLEVEL_PROVIDER =	(1ULL << 1),
75 	VIRTIO_INITLEVEL_INT_ALLOC =	(1ULL << 2),
76 	VIRTIO_INITLEVEL_INT_ADDED =	(1ULL << 3),
77 	VIRTIO_INITLEVEL_INT_ENABLED =	(1ULL << 4),
78 	VIRTIO_INITLEVEL_SHUTDOWN =	(1ULL << 5),
79 } virtio_initlevel_t;
80 
81 struct virtio {
82 	dev_info_t			*vio_dip;
83 
84 	kmutex_t			vio_mutex;
85 
86 	virtio_initlevel_t		vio_initlevel;
87 
88 	list_t				vio_queues;
89 
90 	ddi_acc_handle_t		vio_barh;
91 	caddr_t				vio_bar;
92 	uint_t				vio_config_offset;
93 
94 	uint32_t			vio_features;
95 	uint32_t			vio_features_device;
96 
97 	ddi_intr_handle_t		*vio_interrupts;
98 	int				vio_ninterrupts;
99 	int				vio_interrupt_type;
100 	int				vio_interrupt_cap;
101 	uint_t				vio_interrupt_priority;
102 
103 	ddi_intr_handler_t		*vio_cfgchange_handler;
104 	void				*vio_cfgchange_handlerarg;
105 	boolean_t			vio_cfgchange_handler_added;
106 	uint_t				vio_cfgchange_handler_index;
107 };
108 
109 struct virtio_queue {
110 	virtio_t			*viq_virtio;
111 	kmutex_t			viq_mutex;
112 	const char			*viq_name;
113 	list_node_t			viq_link;
114 
115 	boolean_t			viq_shutdown;
116 	boolean_t			viq_indirect;
117 	uint_t				viq_max_segs;
118 
119 	/*
120 	 * Each Virtio device type has some set of queues for data transfer to
121 	 * and from the host.  This index is described in the specification for
122 	 * the particular device and queue type, and written to QUEUE_SELECT to
123 	 * allow interaction with the queue.  For example, a network device has
124 	 * at least a receive queue with index 0, and a transmit queue with
125 	 * index 1.
126 	 */
127 	uint16_t			viq_index;
128 
129 	/*
130 	 * For legacy Virtio devices, the size and shape of the queue is
131 	 * determined entirely by the number of queue entries.
132 	 */
133 	uint16_t			viq_size;
134 	id_space_t			*viq_descmap;
135 
136 	/*
137 	 * The memory shared between the device and the driver is allocated as
138 	 * a large phyisically contiguous chunk.  Access to this area is
139 	 * through three pointers to packed structures.
140 	 */
141 	virtio_dma_t			viq_dma;
142 	virtio_vq_desc_t		*viq_dma_descs;
143 	virtio_vq_driver_t		*viq_dma_driver;
144 	virtio_vq_device_t		*viq_dma_device;
145 
146 	uint16_t			viq_device_index;
147 	uint16_t			viq_driver_index;
148 
149 	/*
150 	 * Interrupt handler function, or NULL if not provided.
151 	 */
152 	ddi_intr_handler_t		*viq_func;
153 	void				*viq_funcarg;
154 	boolean_t			viq_handler_added;
155 	uint_t				viq_handler_index;
156 
157 	/*
158 	 * When a chain is submitted to the queue, it is also stored in this
159 	 * AVL tree keyed by the index of the first descriptor in the chain.
160 	 */
161 	avl_tree_t			viq_inflight;
162 };
163 
164 struct virtio_chain {
165 	virtio_queue_t			*vic_vq;
166 	avl_node_t			vic_node;
167 
168 	void				*vic_data;
169 
170 	uint16_t			vic_head;
171 	uint32_t			vic_received_length;
172 
173 	virtio_dma_t			vic_indirect_dma;
174 	uint_t				vic_indirect_capacity;
175 	uint_t				vic_indirect_used;
176 
177 	uint_t				vic_direct_capacity;
178 	uint_t				vic_direct_used;
179 	uint16_t			vic_direct[];
180 };
181 
182 /*
183  * PACKED STRUCTS FOR DEVICE ACCESS
184  */
185 
186 struct virtio_vq_desc {
187 	/*
188 	 * Buffer physical address and length.
189 	 */
190 	uint64_t			vqd_addr;
191 	uint32_t			vqd_len;
192 
193 	/*
194 	 * Flags.  Use with the VIRTQ_DESC_F_* family of constants.  See below.
195 	 */
196 	uint16_t			vqd_flags;
197 
198 	/*
199 	 * If VIRTQ_DESC_F_NEXT is set in flags, this refers to the next
200 	 * descriptor in the chain by table index.
201 	 */
202 	uint16_t			vqd_next;
203 } __packed;
204 
205 /*
206  * VIRTIO DESCRIPTOR FLAGS (vqd_flags)
207  */
208 
209 /*
210  * NEXT:
211  *	Signals that this descriptor (direct or indirect) is part of a chain.
212  *	If populated, "vqd_next" names the next descriptor in the chain by its
213  *	table index.
214  */
215 #define	VIRTQ_DESC_F_NEXT		(1 << 0)
216 
217 /*
218  * WRITE:
219  *	Determines whether this buffer is to be written by the device (WRITE is
220  *	set) or by the driver (WRITE is not set).
221  */
222 #define	VIRTQ_DESC_F_WRITE		(1 << 1)
223 
224 /*
225  * INDIRECT:
226  *	This bit signals that a direct descriptor refers to an indirect
227  *	descriptor list, rather than directly to a buffer.  This bit may only
228  *	be used in a direct descriptor; indirect descriptors are not allowed to
229  *	refer to additional layers of indirect tables.  If this bit is set,
230  *	NEXT must be clear; indirect descriptors may not be chained.
231  */
232 #define	VIRTQ_DESC_F_INDIRECT		(1 << 2)
233 
234 /*
235  * This structure is variously known as the "available" or "avail" ring, or the
236  * driver-owned portion of the queue structure.  It is used by the driver to
237  * submit descriptor chains to the device.
238  */
239 struct virtio_vq_driver {
240 	uint16_t			vqdr_flags;
241 	uint16_t			vqdr_index;
242 	uint16_t			vqdr_ring[];
243 } __packed;
244 
245 #define	VIRTQ_AVAIL_F_NO_INTERRUPT	(1 << 0)
246 
247 /*
248  * We use the sizeof operator on this packed struct to calculate the offset of
249  * subsequent structs.  Ensure the compiler is not adding any padding to the
250  * end of the struct.
251  */
252 CTASSERT(sizeof (virtio_vq_driver_t) ==
253     offsetof(virtio_vq_driver_t, vqdr_ring));
254 
255 struct virtio_vq_elem {
256 	/*
257 	 * The device returns chains of descriptors by specifying the table
258 	 * index of the first descriptor in the chain.
259 	 */
260 	uint32_t			vqe_start;
261 	uint32_t			vqe_len;
262 } __packed;
263 
264 /*
265  * This structure is variously known as the "used" ring, or the device-owned
266  * portion of the queue structure.  It is used by the device to return
267  * completed descriptor chains to the driver.
268  */
269 struct virtio_vq_device {
270 	uint16_t			vqde_flags;
271 	uint16_t			vqde_index;
272 	virtio_vq_elem_t		vqde_ring[];
273 } __packed;
274 
275 #define	VIRTQ_USED_F_NO_NOTIFY		(1 << 0)
276 
277 /*
278  * BASIC CONFIGURATION
279  *
280  * Legacy devices expose both their generic and their device-specific
281  * configuration through PCI BAR0.  This is the second entry in the register
282  * address space set for these devices.
283  */
284 #define	VIRTIO_LEGACY_PCI_BAR0		1
285 
286 /*
287  * These are offsets into the base configuration space available through the
288  * virtio_get*() and virtio_put*() family of functions.  These offsets are for
289  * what the specification describes as the "legacy" mode of device operation.
290  */
291 #define	VIRTIO_LEGACY_FEATURES_DEVICE	0x00	/* 32 R   */
292 #define	VIRTIO_LEGACY_FEATURES_DRIVER	0x04	/* 32 R/W */
293 #define	VIRTIO_LEGACY_QUEUE_ADDRESS	0x08	/* 32 R/W */
294 #define	VIRTIO_LEGACY_QUEUE_SIZE	0x0C	/* 16 R   */
295 #define	VIRTIO_LEGACY_QUEUE_SELECT	0x0E	/* 16 R/W */
296 #define	VIRTIO_LEGACY_QUEUE_NOTIFY	0x10	/* 16 R/W */
297 #define	VIRTIO_LEGACY_DEVICE_STATUS	0x12	/* 8  R/W */
298 #define	VIRTIO_LEGACY_ISR_STATUS	0x13	/* 8  R   */
299 
300 #define	VIRTIO_LEGACY_MSIX_CONFIG	0x14	/* 16 R/W */
301 #define	VIRTIO_LEGACY_MSIX_QUEUE	0x16	/* 16 R/W */
302 
303 #define	VIRTIO_LEGACY_CFG_OFFSET	(VIRTIO_LEGACY_ISR_STATUS + 1)
304 #define	VIRTIO_LEGACY_CFG_OFFSET_MSIX	(VIRTIO_LEGACY_MSIX_QUEUE + 2)
305 
306 #define	VIRTIO_LEGACY_MSI_NO_VECTOR	0xFFFF
307 
308 /*
309  * Bits in the Device Status byte (VIRTIO_LEGACY_DEVICE_STATUS):
310  */
311 #define	VIRTIO_STATUS_RESET		0
312 #define	VIRTIO_STATUS_ACKNOWLEDGE	(1 << 0)
313 #define	VIRTIO_STATUS_DRIVER		(1 << 1)
314 #define	VIRTIO_STATUS_DRIVER_OK		(1 << 2)
315 #define	VIRTIO_STATUS_FAILED		(1 << 7)
316 
317 /*
318  * Bits in the Interrupt Service Routine Status byte
319  * (VIRTIO_LEGACY_ISR_STATUS):
320  */
321 #define	VIRTIO_ISR_CHECK_QUEUES		(1 << 0)
322 #define	VIRTIO_ISR_CHECK_CONFIG		(1 << 1)
323 
324 /*
325  * Bits in the Features fields (VIRTIO_LEGACY_FEATURES_DEVICE,
326  * VIRTIO_LEGACY_FEATURES_DRIVER):
327  */
328 #define	VIRTIO_F_RING_INDIRECT_DESC	(1ULL << 28)
329 
330 /*
331  * For devices operating in the legacy mode, virtqueues must be aligned on a
332  * "page size" of 4096 bytes; this is also called the "Queue Align" value in
333  * newer versions of the specification.
334  */
335 #define	VIRTIO_PAGE_SHIFT		12
336 #define	VIRTIO_PAGE_SIZE		(1 << VIRTIO_PAGE_SHIFT)
337 CTASSERT(VIRTIO_PAGE_SIZE == 4096);
338 CTASSERT(ISP2(VIRTIO_PAGE_SIZE));
339 
340 /*
341  * DMA SYNCHRONISATION WRAPPERS
342  */
343 
344 /*
345  * Synchronise the driver-owned portion of the queue so that the device can see
346  * our writes.  This covers the memory accessed via the "viq_dma_descs" and
347  * "viq_dma_device" members.
348  */
349 #define	VIRTQ_DMA_SYNC_FORDEV(viq)	VERIFY0(ddi_dma_sync( \
350 					    (viq)->viq_dma.vidma_dma_handle, \
351 					    0, \
352 					    (uintptr_t)(viq)->viq_dma_device - \
353 					    (uintptr_t)(viq)->viq_dma_descs, \
354 					    DDI_DMA_SYNC_FORDEV))
355 
356 /*
357  * Synchronise the device-owned portion of the queue so that we can see any
358  * writes from the device.  This covers the memory accessed via the
359  * "viq_dma_device" member.
360  */
361 #define	VIRTQ_DMA_SYNC_FORKERNEL(viq)	VERIFY0(ddi_dma_sync( \
362 					    (viq)->viq_dma.vidma_dma_handle, \
363 					    (uintptr_t)(viq)->viq_dma_device - \
364 					    (uintptr_t)(viq)->viq_dma_descs, \
365 					    (viq)->viq_dma.vidma_size - \
366 					    (uintptr_t)(viq)->viq_dma_device - \
367 					    (uintptr_t)(viq)->viq_dma_descs, \
368 					    DDI_DMA_SYNC_FORKERNEL))
369 
370 #ifdef __cplusplus
371 }
372 #endif
373 
374 #endif /* _VIRTIO_IMPL_H */
375