1 /*
2  * Copyright (c) 2013  Chris Torek <torek @ torek net>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 /*
27  * This file and its contents are supplied under the terms of the
28  * Common Development and Distribution License ("CDDL"), version 1.0.
29  * You may only use this file in accordance with the terms of version
30  * 1.0 of the CDDL.
31  *
32  * A full copy of the text of the CDDL should have accompanied this
33  * source.  A copy of the CDDL is also available via the Internet at
34  * http://www.illumos.org/license/CDDL.
35  *
36  * Copyright 2015 Pluribus Networks Inc.
37  * Copyright 2019 Joyent, Inc.
38  * Copyright 2022 OmniOS Community Edition (OmniOSce) Association.
39  * Copyright 2023 Oxide Computer Company
40  */
41 
42 #ifndef	_VIONA_IMPL_H
43 #define	_VIONA_IMPL_H
44 
45 #include <sys/ddi.h>
46 #include <sys/list.h>
47 #include <sys/sunddi.h>
48 #include <sys/sunndi.h>
49 #include <sys/strsun.h>
50 #include <sys/sysmacros.h>
51 #include <sys/uio.h>
52 
53 #include <sys/mac_client.h>
54 #include <sys/mac_provider.h>
55 #include <sys/mac_client_priv.h>
56 #include <sys/neti.h>
57 #include <inet/ip.h>
58 #include <inet/tcp.h>
59 
60 #include <sys/vmm_drv.h>
61 #include <sys/viona_io.h>
62 
63 struct viona_link;
64 typedef struct viona_link viona_link_t;
65 struct viona_desb;
66 typedef struct viona_desb viona_desb_t;
67 struct viona_net;
68 typedef struct viona_neti viona_neti_t;
69 
70 enum viona_ring_state {
71 	VRS_RESET	= 0x0,	/* just allocated or reset */
72 	VRS_SETUP	= 0x1,	/* addrs setup and starting worker thread */
73 	VRS_INIT	= 0x2,	/* worker thread started & waiting to run */
74 	VRS_RUN		= 0x3,	/* running work routine */
75 	VRS_STOP	= 0x4,	/* worker is exiting */
76 };
77 enum viona_ring_state_flags {
78 	VRSF_REQ_START	= 0x1,	/* start running from INIT state */
79 	VRSF_REQ_STOP	= 0x2,	/* stop running, clean up, goto RESET state */
80 	VRSF_REQ_PAUSE	= 0x4,	/* stop running, goto INIT state */
81 	VRSF_RENEW	= 0x8,	/* ring renewing lease */
82 };
83 
84 typedef struct viona_vring {
85 	viona_link_t	*vr_link;
86 
87 	kmutex_t	vr_lock;
88 	kcondvar_t	vr_cv;
89 	uint16_t	vr_state;
90 	uint16_t	vr_state_flags;
91 	uint_t		vr_xfer_outstanding;
92 	kthread_t	*vr_worker_thread;
93 	vmm_lease_t	*vr_lease;
94 
95 	/* ring-sized resources for TX activity */
96 	viona_desb_t	*vr_txdesb;
97 	struct iovec	*vr_txiov;
98 
99 	uint_t		vr_intr_enabled;
100 	uint64_t	vr_msi_addr;
101 	uint64_t	vr_msi_msg;
102 
103 	/* Internal ring-related state */
104 	kmutex_t	vr_a_mutex;	/* sync consumers of 'avail' */
105 	kmutex_t	vr_u_mutex;	/* sync consumers of 'used' */
106 	uint64_t	vr_pa;
107 	uint16_t	vr_size;
108 	uint16_t	vr_mask;	/* cached from vr_size */
109 	uint16_t	vr_cur_aidx;	/* trails behind 'avail_idx' */
110 	uint16_t	vr_cur_uidx;	/* drives 'used_idx' */
111 
112 	/* Reference to guest pages holding virtqueue */
113 	void		**vr_map_pages;
114 	vmm_page_t	*vr_map_hold;
115 
116 	/* Per-ring error condition statistics */
117 	struct viona_ring_stats {
118 		uint64_t	rs_ndesc_too_high;
119 		uint64_t	rs_bad_idx;
120 		uint64_t	rs_indir_bad_len;
121 		uint64_t	rs_indir_bad_nest;
122 		uint64_t	rs_indir_bad_next;
123 		uint64_t	rs_no_space;
124 		uint64_t	rs_too_many_desc;
125 		uint64_t	rs_desc_bad_len;
126 
127 		uint64_t	rs_bad_ring_addr;
128 
129 		uint64_t	rs_fail_hcksum;
130 		uint64_t	rs_fail_hcksum6;
131 		uint64_t	rs_fail_hcksum_proto;
132 
133 		uint64_t	rs_bad_rx_frame;
134 		uint64_t	rs_rx_merge_overrun;
135 		uint64_t	rs_rx_merge_underrun;
136 		uint64_t	rs_rx_pad_short;
137 		uint64_t	rs_rx_mcast_check;
138 		uint64_t	rs_too_short;
139 		uint64_t	rs_tx_absent;
140 
141 		uint64_t	rs_rx_hookdrop;
142 		uint64_t	rs_tx_hookdrop;
143 	} vr_stats;
144 } viona_vring_t;
145 
146 struct viona_link {
147 	vmm_hold_t		*l_vm_hold;
148 	boolean_t		l_destroyed;
149 
150 	viona_vring_t		l_vrings[VIONA_VQ_MAX];
151 
152 	uint32_t		l_features;
153 	uint32_t		l_features_hw;
154 	uint32_t		l_cap_csum;
155 
156 	uint16_t		l_notify_ioport;
157 	void			*l_notify_cookie;
158 
159 	datalink_id_t		l_linkid;
160 	mac_handle_t		l_mh;
161 	mac_client_handle_t	l_mch;
162 	mac_promisc_handle_t	l_mph;
163 	mac_unicast_handle_t	l_muh;
164 	viona_promisc_t		l_promisc;
165 
166 	pollhead_t		l_pollhead;
167 
168 	viona_neti_t		*l_neti;
169 };
170 
171 typedef struct viona_nethook {
172 	net_handle_t		vnh_neti;
173 	hook_family_t		vnh_family;
174 	hook_event_t		vnh_event_in;
175 	hook_event_t		vnh_event_out;
176 	hook_event_token_t	vnh_token_in;
177 	hook_event_token_t	vnh_token_out;
178 	boolean_t		vnh_hooked;
179 } viona_nethook_t;
180 
181 struct viona_neti {
182 	list_node_t		vni_node;
183 
184 	netid_t			vni_netid;
185 	zoneid_t		vni_zid;
186 
187 	viona_nethook_t		vni_nethook;
188 
189 	kmutex_t		vni_lock;	/* Protects remaining members */
190 	kcondvar_t		vni_ref_change; /* Protected by vni_lock */
191 	uint_t			vni_ref;	/* Protected by vni_lock */
192 	list_t			vni_dev_list;	/* Protected by vni_lock */
193 };
194 
195 typedef struct used_elem {
196 	uint16_t	id;
197 	uint32_t	len;
198 } used_elem_t;
199 
200 typedef struct viona_soft_state {
201 	kmutex_t		ss_lock;
202 	viona_link_t		*ss_link;
203 	list_node_t		ss_node;
204 } viona_soft_state_t;
205 
206 #pragma pack(1)
207 struct virtio_desc {
208 	uint64_t	vd_addr;
209 	uint32_t	vd_len;
210 	uint16_t	vd_flags;
211 	uint16_t	vd_next;
212 };
213 
214 struct virtio_used {
215 	uint32_t	vu_idx;
216 	uint32_t	vu_tlen;
217 };
218 
219 struct virtio_net_mrgrxhdr {
220 	uint8_t		vrh_flags;
221 	uint8_t		vrh_gso_type;
222 	uint16_t	vrh_hdr_len;
223 	uint16_t	vrh_gso_size;
224 	uint16_t	vrh_csum_start;
225 	uint16_t	vrh_csum_offset;
226 	uint16_t	vrh_bufs;
227 };
228 
229 struct virtio_net_hdr {
230 	uint8_t		vrh_flags;
231 	uint8_t		vrh_gso_type;
232 	uint16_t	vrh_hdr_len;
233 	uint16_t	vrh_gso_size;
234 	uint16_t	vrh_csum_start;
235 	uint16_t	vrh_csum_offset;
236 };
237 #pragma pack()
238 
239 #define	VNETHOOK_INTERESTED_IN(neti) \
240 	(neti)->vni_nethook.vnh_event_in.he_interested
241 #define	VNETHOOK_INTERESTED_OUT(neti) \
242 	(neti)->vni_nethook.vnh_event_out.he_interested
243 
244 
245 #define	VIONA_PROBE(name)	DTRACE_PROBE(viona__##name)
246 #define	VIONA_PROBE1(name, arg1, arg2)	\
247 	DTRACE_PROBE1(viona__##name, arg1, arg2)
248 #define	VIONA_PROBE2(name, arg1, arg2, arg3, arg4)	\
249 	DTRACE_PROBE2(viona__##name, arg1, arg2, arg3, arg4)
250 #define	VIONA_PROBE3(name, arg1, arg2, arg3, arg4, arg5, arg6)	\
251 	DTRACE_PROBE3(viona__##name, arg1, arg2, arg3, arg4, arg5, arg6)
252 #define	VIONA_PROBE4(name, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8) \
253 	DTRACE_PROBE4(viona__##name, arg1, arg2, arg3, arg4, arg5, arg6, arg7, \
254 	arg8)
255 #define	VIONA_PROBE5(name, arg1, arg2, arg3, arg4, arg5, arg6, arg7, arg8, \
256 	arg9, arg10) \
257 	DTRACE_PROBE5(viona__##name, arg1, arg2, arg3, arg4, arg5, arg6, arg7, \
258 	arg8, arg9, arg10)
259 #define	VIONA_PROBE_BAD_RING_ADDR(r, a)		\
260 	VIONA_PROBE2(bad_ring_addr, viona_vring_t *, r, void *, (void *)(a))
261 
262 #define	VIONA_RING_STAT_INCR(r, name)	\
263 	(((r)->vr_stats.rs_ ## name)++)
264 
265 
266 #define	VIONA_MAX_HDRS_LEN	(sizeof (struct ether_vlan_header) + \
267 	IP_MAX_HDR_LENGTH + TCP_MAX_HDR_LENGTH)
268 
269 #define	VRING_AVAIL_F_NO_INTERRUPT	1
270 #define	VRING_USED_F_NO_NOTIFY		1
271 
272 #define	VRING_DESC_F_NEXT	(1 << 0)
273 #define	VRING_DESC_F_WRITE	(1 << 1)
274 #define	VRING_DESC_F_INDIRECT	(1 << 2)
275 
276 #define	VIRTIO_NET_HDR_F_NEEDS_CSUM	(1 << 0)
277 #define	VIRTIO_NET_HDR_F_DATA_VALID	(1 << 1)
278 
279 #define	VIRTIO_NET_HDR_GSO_NONE		0
280 #define	VIRTIO_NET_HDR_GSO_TCPV4	1
281 
282 #define	VIRTIO_NET_F_CSUM		(1 << 0)
283 #define	VIRTIO_NET_F_GUEST_CSUM		(1 << 1)
284 #define	VIRTIO_NET_F_MAC		(1 << 5) /* host supplies MAC */
285 #define	VIRTIO_NET_F_GUEST_TSO4		(1 << 7) /* guest can accept TSO */
286 #define	VIRTIO_NET_F_HOST_TSO4		(1 << 11) /* host can accept TSO */
287 #define	VIRTIO_NET_F_MRG_RXBUF		(1 << 15) /* host can merge RX bufs */
288 #define	VIRTIO_NET_F_STATUS		(1 << 16) /* cfg status field present */
289 #define	VIRTIO_F_RING_NOTIFY_ON_EMPTY	(1 << 24)
290 #define	VIRTIO_F_RING_INDIRECT_DESC	(1 << 28)
291 #define	VIRTIO_F_RING_EVENT_IDX		(1 << 29)
292 
293 struct viona_ring_params {
294 	uint64_t	vrp_pa;
295 	uint16_t	vrp_size;
296 	uint16_t	vrp_avail_idx;
297 	uint16_t	vrp_used_idx;
298 };
299 
300 void viona_ring_alloc(viona_link_t *, viona_vring_t *);
301 void viona_ring_free(viona_vring_t *);
302 int viona_ring_get_state(viona_link_t *, uint16_t, struct viona_ring_params *);
303 int viona_ring_set_state(viona_link_t *, uint16_t,
304     const struct viona_ring_params *);
305 int viona_ring_reset(viona_vring_t *, boolean_t);
306 int viona_ring_init(viona_link_t *, uint16_t, const struct viona_ring_params *);
307 boolean_t viona_ring_lease_renew(viona_vring_t *);
308 bool vring_need_bail(const viona_vring_t *);
309 int viona_ring_pause(viona_vring_t *);
310 
311 int vq_popchain(viona_vring_t *, struct iovec *, uint_t, uint16_t *,
312     vmm_page_t **);
313 void vq_pushchain(viona_vring_t *, uint32_t, uint16_t);
314 void vq_pushchain_many(viona_vring_t *, uint_t, used_elem_t *);
315 
316 void viona_intr_ring(viona_vring_t *ring, boolean_t);
317 void viona_ring_set_no_notify(viona_vring_t *, boolean_t);
318 void viona_ring_disable_notify(viona_vring_t *);
319 void viona_ring_enable_notify(viona_vring_t *);
320 uint16_t viona_ring_num_avail(viona_vring_t *);
321 
322 
323 void viona_rx_init(void);
324 void viona_rx_fini(void);
325 int viona_rx_set(viona_link_t *, viona_promisc_t);
326 void viona_rx_clear(viona_link_t *);
327 void viona_worker_rx(viona_vring_t *, viona_link_t *);
328 
329 extern kmutex_t viona_force_copy_lock;
330 void viona_worker_tx(viona_vring_t *, viona_link_t *);
331 void viona_tx_ring_alloc(viona_vring_t *, const uint16_t);
332 void viona_tx_ring_free(viona_vring_t *, const uint16_t);
333 
334 void viona_neti_attach(void);
335 void viona_neti_detach(void);
336 viona_neti_t *viona_neti_lookup_by_zid(zoneid_t);
337 void viona_neti_rele(viona_neti_t *);
338 int viona_hook(viona_link_t *, viona_vring_t *, mblk_t **, boolean_t);
339 
340 #endif	/* _VIONA_IMPL_H */
341