xref: /illumos-gate/usr/src/uts/intel/io/vmm/io/vioapic.c (revision 32640292)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
5  * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
6  * All rights reserved.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 /*
30  * This file and its contents are supplied under the terms of the
31  * Common Development and Distribution License ("CDDL"), version 1.0.
32  * You may only use this file in accordance with the terms of version
33  * 1.0 of the CDDL.
34  *
35  * A full copy of the text of the CDDL should have accompanied this
36  * source.  A copy of the CDDL is also available via the Internet at
37  * http://www.illumos.org/license/CDDL.
38  *
39  * Copyright 2014 Pluribus Networks Inc.
40  * Copyright 2017 Joyent, Inc.
41  * Copyright 2021 Oxide Computer Company
42  */
43 
44 #include <sys/cdefs.h>
45 
46 #include <sys/param.h>
47 #include <sys/queue.h>
48 #include <sys/mutex.h>
49 #include <sys/systm.h>
50 #include <sys/kernel.h>
51 #include <sys/kmem.h>
52 #include <sys/cpuset.h>
53 
54 #include <x86/apicreg.h>
55 #include <machine/vmm.h>
56 #include <sys/vmm_data.h>
57 
58 #include "vmm_lapic.h"
59 #include "vlapic.h"
60 #include "vioapic.h"
61 
62 #define	IOREGSEL	0x00
63 #define	IOWIN		0x10
64 
65 #define	REDIR_ENTRIES	32
66 #define	RTBL_RO_BITS	((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
67 
68 struct ioapic_stats {
69 	uint64_t	is_interrupts;
70 	uint64_t	is_saturate_low;
71 	uint64_t	is_saturate_high;
72 };
73 
74 struct vioapic {
75 	struct vm	*vm;
76 	kmutex_t	lock;
77 	uint32_t	id;
78 	uint32_t	ioregsel;
79 	struct {
80 		uint64_t reg;
81 		/*
82 		 * The sum of pin asserts (+1) and deasserts (-1) are tracked in
83 		 * 'acnt'.  It is clamped to prevent overflow or underflow
84 		 * should emulation consumers feed it an invalid set of
85 		 * transitions.
86 		 */
87 		uint_t acnt;
88 	} rtbl[REDIR_ENTRIES];
89 	struct ioapic_stats stats;
90 };
91 
92 #define	VIOAPIC_LOCK(vioapic)		mutex_enter(&((vioapic)->lock))
93 #define	VIOAPIC_UNLOCK(vioapic)		mutex_exit(&((vioapic)->lock))
94 #define	VIOAPIC_LOCKED(vioapic)		MUTEX_HELD(&((vioapic)->lock))
95 
96 
97 static void
vioapic_send_intr(struct vioapic * vioapic,int pin)98 vioapic_send_intr(struct vioapic *vioapic, int pin)
99 {
100 	int vector, delmode;
101 	uint32_t low, high, dest;
102 	bool level, phys;
103 
104 	VERIFY(pin >= 0 && pin < REDIR_ENTRIES);
105 	ASSERT(VIOAPIC_LOCKED(vioapic));
106 
107 	low = vioapic->rtbl[pin].reg;
108 	high = vioapic->rtbl[pin].reg >> 32;
109 
110 	if ((low & IOART_INTMASK) == IOART_INTMSET) {
111 		/* Pin is masked */
112 		return;
113 	}
114 
115 	phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
116 	delmode = low & IOART_DELMOD;
117 	level = low & IOART_TRGRLVL ? true : false;
118 	if (level) {
119 		if ((low & IOART_REM_IRR) != 0) {
120 			/* IRR already pending */
121 			return;
122 		}
123 		vioapic->rtbl[pin].reg |= IOART_REM_IRR;
124 	}
125 
126 	vector = low & IOART_INTVEC;
127 	dest = high >> APIC_ID_SHIFT;
128 	vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
129 	vioapic->stats.is_interrupts++;
130 }
131 
132 static int
vioapic_set_pinstate(struct vioapic * vioapic,int pin,bool newstate)133 vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
134 {
135 	uint_t oldcnt, newcnt;
136 	bool needintr = false;
137 	int err = 0;
138 
139 	VERIFY(pin >= 0 && pin < REDIR_ENTRIES);
140 	ASSERT(VIOAPIC_LOCKED(vioapic));
141 
142 	oldcnt = newcnt = vioapic->rtbl[pin].acnt;
143 	if (newstate) {
144 		if (newcnt != UINT_MAX) {
145 			newcnt++;
146 		} else {
147 			err = E2BIG;
148 			DTRACE_PROBE2(vioapic__sat_high,
149 			    struct vioapic *, vioapic, int, pin);
150 			vioapic->stats.is_saturate_high++;
151 		}
152 	} else {
153 		if (newcnt != 0) {
154 			newcnt--;
155 		} else {
156 			err = ERANGE;
157 			DTRACE_PROBE2(vioapic__sat_low,
158 			    struct vioapic *, vioapic, int, pin);
159 			vioapic->stats.is_saturate_low++;
160 		}
161 	}
162 	vioapic->rtbl[pin].acnt = newcnt;
163 
164 	if (oldcnt == 0 && newcnt == 1) {
165 		needintr = true;
166 		DTRACE_PROBE2(vioapic__assert, struct vioapic *, vioapic,
167 		    int, pin);
168 	} else if (oldcnt == 1 && newcnt == 0) {
169 		DTRACE_PROBE2(vioapic__deassert, struct vioapic *, vioapic,
170 		    int, pin);
171 	}
172 
173 	if (needintr) {
174 		vioapic_send_intr(vioapic, pin);
175 	}
176 	return (err);
177 }
178 
179 enum irqstate {
180 	IRQSTATE_ASSERT,
181 	IRQSTATE_DEASSERT,
182 	IRQSTATE_PULSE
183 };
184 
185 static int
vioapic_set_irqstate(struct vm * vm,int irq,enum irqstate irqstate)186 vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
187 {
188 	struct vioapic *vioapic;
189 	int err = 0;
190 
191 	if (irq < 0 || irq >= REDIR_ENTRIES)
192 		return (EINVAL);
193 
194 	vioapic = vm_ioapic(vm);
195 
196 	VIOAPIC_LOCK(vioapic);
197 	switch (irqstate) {
198 	case IRQSTATE_ASSERT:
199 		err = vioapic_set_pinstate(vioapic, irq, true);
200 		break;
201 	case IRQSTATE_DEASSERT:
202 		err = vioapic_set_pinstate(vioapic, irq, false);
203 		break;
204 	case IRQSTATE_PULSE:
205 		err = vioapic_set_pinstate(vioapic, irq, true);
206 		if (err == 0) {
207 			err = vioapic_set_pinstate(vioapic, irq, false);
208 		}
209 		break;
210 	default:
211 		panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
212 	}
213 	VIOAPIC_UNLOCK(vioapic);
214 
215 	return (err);
216 }
217 
218 int
vioapic_assert_irq(struct vm * vm,int irq)219 vioapic_assert_irq(struct vm *vm, int irq)
220 {
221 
222 	return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
223 }
224 
225 int
vioapic_deassert_irq(struct vm * vm,int irq)226 vioapic_deassert_irq(struct vm *vm, int irq)
227 {
228 
229 	return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
230 }
231 
232 int
vioapic_pulse_irq(struct vm * vm,int irq)233 vioapic_pulse_irq(struct vm *vm, int irq)
234 {
235 
236 	return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
237 }
238 
239 static uint32_t
vioapic_read(struct vioapic * vioapic,int vcpuid,uint32_t addr)240 vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr)
241 {
242 	int regnum, pin, rshift;
243 
244 	regnum = addr & 0xff;
245 	switch (regnum) {
246 	case IOAPIC_ID:
247 		return (vioapic->id);
248 		break;
249 	case IOAPIC_VER:
250 		return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
251 		break;
252 	case IOAPIC_ARB:
253 		return (vioapic->id);
254 		break;
255 	default:
256 		break;
257 	}
258 
259 	/* redirection table entries */
260 	if (regnum >= IOAPIC_REDTBL &&
261 	    regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
262 		pin = (regnum - IOAPIC_REDTBL) / 2;
263 		if ((regnum - IOAPIC_REDTBL) % 2)
264 			rshift = 32;
265 		else
266 			rshift = 0;
267 
268 		return (vioapic->rtbl[pin].reg >> rshift);
269 	}
270 
271 	return (0);
272 }
273 
274 static void
vioapic_write(struct vioapic * vioapic,int vcpuid,uint32_t addr,uint32_t data)275 vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data)
276 {
277 	uint64_t data64, mask64;
278 	int regnum, pin, lshift;
279 
280 	regnum = addr & 0xff;
281 	switch (regnum) {
282 	case IOAPIC_ID:
283 		vioapic->id = data & APIC_ID_MASK;
284 		break;
285 	case IOAPIC_VER:
286 	case IOAPIC_ARB:
287 		/* readonly */
288 		break;
289 	default:
290 		break;
291 	}
292 
293 	/* redirection table entries */
294 	if (regnum >= IOAPIC_REDTBL &&
295 	    regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
296 		pin = (regnum - IOAPIC_REDTBL) / 2;
297 		if ((regnum - IOAPIC_REDTBL) % 2)
298 			lshift = 32;
299 		else
300 			lshift = 0;
301 
302 		data64 = (uint64_t)data << lshift;
303 		mask64 = (uint64_t)0xffffffff << lshift;
304 		vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
305 		vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
306 
307 		/*
308 		 * Switching from level to edge triggering will clear the IRR
309 		 * bit. This is what FreeBSD will do in order to EOI an
310 		 * interrupt when the IO-APIC doesn't support targeted EOI (see
311 		 * _ioapic_eoi_source).
312 		 */
313 		if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG &&
314 		    (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0)
315 			vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
316 
317 		/*
318 		 * Generate an interrupt if the following conditions are met:
319 		 * - pin trigger mode is level
320 		 * - pin level is asserted
321 		 */
322 		if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL &&
323 		    (vioapic->rtbl[pin].acnt > 0)) {
324 			vioapic_send_intr(vioapic, pin);
325 		}
326 	}
327 }
328 
329 static int
vioapic_mmio_rw(struct vioapic * vioapic,int vcpuid,uint64_t gpa,uint64_t * data,int size,bool doread)330 vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa,
331     uint64_t *data, int size, bool doread)
332 {
333 	uint64_t offset;
334 
335 	offset = gpa - VIOAPIC_BASE;
336 
337 	/*
338 	 * The IOAPIC specification allows 32-bit wide accesses to the
339 	 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
340 	 */
341 	if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
342 		if (doread)
343 			*data = 0;
344 		return (0);
345 	}
346 
347 	VIOAPIC_LOCK(vioapic);
348 	if (offset == IOREGSEL) {
349 		if (doread)
350 			*data = vioapic->ioregsel;
351 		else
352 			vioapic->ioregsel = *data;
353 	} else {
354 		if (doread) {
355 			*data = vioapic_read(vioapic, vcpuid,
356 			    vioapic->ioregsel);
357 		} else {
358 			vioapic_write(vioapic, vcpuid, vioapic->ioregsel,
359 			    *data);
360 		}
361 	}
362 	VIOAPIC_UNLOCK(vioapic);
363 
364 	return (0);
365 }
366 
367 int
vioapic_mmio_read(struct vm * vm,int vcpuid,uint64_t gpa,uint64_t * rval,int size)368 vioapic_mmio_read(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t *rval,
369     int size)
370 {
371 	int error;
372 	struct vioapic *vioapic;
373 
374 	vioapic = vm_ioapic(vm);
375 	error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true);
376 	return (error);
377 }
378 
379 int
vioapic_mmio_write(struct vm * vm,int vcpuid,uint64_t gpa,uint64_t wval,int size)380 vioapic_mmio_write(struct vm *vm, int vcpuid, uint64_t gpa, uint64_t wval,
381     int size)
382 {
383 	int error;
384 	struct vioapic *vioapic;
385 
386 	vioapic = vm_ioapic(vm);
387 	error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false);
388 	return (error);
389 }
390 
391 void
vioapic_process_eoi(struct vm * vm,int vcpuid,int vector)392 vioapic_process_eoi(struct vm *vm, int vcpuid, int vector)
393 {
394 	struct vioapic *vioapic;
395 	int pin;
396 
397 	KASSERT(vector >= 0 && vector < 256,
398 	    ("vioapic_process_eoi: invalid vector %d", vector));
399 
400 	vioapic = vm_ioapic(vm);
401 
402 	/*
403 	 * XXX keep track of the pins associated with this vector instead
404 	 * of iterating on every single pin each time.
405 	 */
406 	VIOAPIC_LOCK(vioapic);
407 	for (pin = 0; pin < REDIR_ENTRIES; pin++) {
408 		if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
409 			continue;
410 		if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
411 			continue;
412 		vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
413 		if (vioapic->rtbl[pin].acnt > 0) {
414 			/* Pin asserted at EOI */
415 			vioapic_send_intr(vioapic, pin);
416 		}
417 	}
418 	VIOAPIC_UNLOCK(vioapic);
419 }
420 
421 struct vioapic *
vioapic_init(struct vm * vm)422 vioapic_init(struct vm *vm)
423 {
424 	int i;
425 	struct vioapic *vioapic;
426 
427 	vioapic = kmem_zalloc(sizeof (struct vioapic), KM_SLEEP);
428 
429 	vioapic->vm = vm;
430 	mutex_init(&vioapic->lock, NULL, MUTEX_ADAPTIVE, NULL);
431 
432 	/* Initialize all redirection entries to mask all interrupts */
433 	for (i = 0; i < REDIR_ENTRIES; i++)
434 		vioapic->rtbl[i].reg = 0x0001000000010000UL;
435 
436 	return (vioapic);
437 }
438 
439 void
vioapic_cleanup(struct vioapic * vioapic)440 vioapic_cleanup(struct vioapic *vioapic)
441 {
442 	mutex_destroy(&vioapic->lock);
443 	kmem_free(vioapic, sizeof (*vioapic));
444 }
445 
446 int
vioapic_pincount(struct vm * vm)447 vioapic_pincount(struct vm *vm)
448 {
449 
450 	return (REDIR_ENTRIES);
451 }
452 
453 static int
vioapic_data_read(void * datap,const vmm_data_req_t * req)454 vioapic_data_read(void *datap, const vmm_data_req_t *req)
455 {
456 	VERIFY3U(req->vdr_class, ==, VDC_IOAPIC);
457 	VERIFY3U(req->vdr_version, ==, 1);
458 	VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_ioapic_v1));
459 
460 	struct vioapic *vioapic = datap;
461 	struct vdi_ioapic_v1 *out = req->vdr_data;
462 
463 	VIOAPIC_LOCK(vioapic);
464 	out->vi_id = vioapic->id;
465 	out->vi_reg_sel = vioapic->ioregsel;
466 	for (uint_t i = 0; i < REDIR_ENTRIES; i++) {
467 		out->vi_pin_reg[i] = vioapic->rtbl[i].reg;
468 		out->vi_pin_level[i] = vioapic->rtbl[i].acnt;
469 	}
470 	VIOAPIC_UNLOCK(vioapic);
471 
472 	return (0);
473 }
474 
475 static int
vioapic_data_write(void * datap,const vmm_data_req_t * req)476 vioapic_data_write(void *datap, const vmm_data_req_t *req)
477 {
478 	VERIFY3U(req->vdr_class, ==, VDC_IOAPIC);
479 	VERIFY3U(req->vdr_version, ==, 1);
480 	VERIFY3U(req->vdr_len, >=, sizeof (struct vdi_ioapic_v1));
481 
482 	struct vioapic *vioapic = datap;
483 	const struct vdi_ioapic_v1 *src = req->vdr_data;
484 
485 	VIOAPIC_LOCK(vioapic);
486 	vioapic->id = src->vi_id;
487 	vioapic->ioregsel = src->vi_reg_sel;
488 	for (uint_t i = 0; i < REDIR_ENTRIES; i++) {
489 		vioapic->rtbl[i].reg = src->vi_pin_reg[i] & ~RTBL_RO_BITS;
490 		vioapic->rtbl[i].acnt = src->vi_pin_level[i];
491 	}
492 	VIOAPIC_UNLOCK(vioapic);
493 
494 	return (0);
495 }
496 
497 static const vmm_data_version_entry_t ioapic_v1 = {
498 	.vdve_class = VDC_IOAPIC,
499 	.vdve_version = 1,
500 	.vdve_len_expect = sizeof (struct vdi_ioapic_v1),
501 	.vdve_readf = vioapic_data_read,
502 	.vdve_writef = vioapic_data_write,
503 };
504 VMM_DATA_VERSION(ioapic_v1);
505