xref: /illumos-gate/usr/src/cmd/bhyve/mem.c (revision e0c0d44e)
1bf21cd93STycho Nightingale /*-
24c87aefeSPatrick Mooney  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
34c87aefeSPatrick Mooney  *
4bf21cd93STycho Nightingale  * Copyright (c) 2012 NetApp, Inc.
5bf21cd93STycho Nightingale  * All rights reserved.
6bf21cd93STycho Nightingale  *
7bf21cd93STycho Nightingale  * Redistribution and use in source and binary forms, with or without
8bf21cd93STycho Nightingale  * modification, are permitted provided that the following conditions
9bf21cd93STycho Nightingale  * are met:
10bf21cd93STycho Nightingale  * 1. Redistributions of source code must retain the above copyright
11bf21cd93STycho Nightingale  *    notice, this list of conditions and the following disclaimer.
12bf21cd93STycho Nightingale  * 2. Redistributions in binary form must reproduce the above copyright
13bf21cd93STycho Nightingale  *    notice, this list of conditions and the following disclaimer in the
14bf21cd93STycho Nightingale  *    documentation and/or other materials provided with the distribution.
15bf21cd93STycho Nightingale  *
16bf21cd93STycho Nightingale  * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
17bf21cd93STycho Nightingale  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18bf21cd93STycho Nightingale  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19bf21cd93STycho Nightingale  * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
20bf21cd93STycho Nightingale  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21bf21cd93STycho Nightingale  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22bf21cd93STycho Nightingale  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23bf21cd93STycho Nightingale  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24bf21cd93STycho Nightingale  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25bf21cd93STycho Nightingale  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26bf21cd93STycho Nightingale  * SUCH DAMAGE.
27bf21cd93STycho Nightingale  *
284c87aefeSPatrick Mooney  * $FreeBSD$
29bf21cd93STycho Nightingale  */
30*e0c0d44eSPatrick Mooney /*
31*e0c0d44eSPatrick Mooney  * This file and its contents are supplied under the terms of the
32*e0c0d44eSPatrick Mooney  * Common Development and Distribution License ("CDDL"), version 1.0.
33*e0c0d44eSPatrick Mooney  * You may only use this file in accordance with the terms of version
34*e0c0d44eSPatrick Mooney  * 1.0 of the CDDL.
35*e0c0d44eSPatrick Mooney  *
36*e0c0d44eSPatrick Mooney  * A full copy of the text of the CDDL should have accompanied this
37*e0c0d44eSPatrick Mooney  * source.  A copy of the CDDL is also available via the Internet at
38*e0c0d44eSPatrick Mooney  * http://www.illumos.org/license/CDDL.
39*e0c0d44eSPatrick Mooney  *
40*e0c0d44eSPatrick Mooney  * Copyright 2020 Oxide Computer Company
41*e0c0d44eSPatrick Mooney  */
42bf21cd93STycho Nightingale 
43bf21cd93STycho Nightingale /*
44bf21cd93STycho Nightingale  * Memory ranges are represented with an RB tree. On insertion, the range
45bf21cd93STycho Nightingale  * is checked for overlaps. On lookup, the key has the same base and limit
46bf21cd93STycho Nightingale  * so it can be searched within the range.
47bf21cd93STycho Nightingale  */
48bf21cd93STycho Nightingale 
49bf21cd93STycho Nightingale #include <sys/cdefs.h>
504c87aefeSPatrick Mooney __FBSDID("$FreeBSD$");
51bf21cd93STycho Nightingale 
52bf21cd93STycho Nightingale #include <sys/types.h>
53bf21cd93STycho Nightingale #include <sys/errno.h>
544c87aefeSPatrick Mooney #include <sys/tree.h>
55bf21cd93STycho Nightingale #include <machine/vmm.h>
56bf21cd93STycho Nightingale 
57bf21cd93STycho Nightingale #include <assert.h>
584c87aefeSPatrick Mooney #include <err.h>
59bf21cd93STycho Nightingale #include <pthread.h>
604c87aefeSPatrick Mooney #include <stdio.h>
614c87aefeSPatrick Mooney #include <stdlib.h>
62bf21cd93STycho Nightingale 
63bf21cd93STycho Nightingale #include "mem.h"
64bf21cd93STycho Nightingale 
65bf21cd93STycho Nightingale struct mmio_rb_range {
66bf21cd93STycho Nightingale 	RB_ENTRY(mmio_rb_range)	mr_link;	/* RB tree links */
67bf21cd93STycho Nightingale 	struct mem_range	mr_param;
68bf21cd93STycho Nightingale 	uint64_t                mr_base;
69bf21cd93STycho Nightingale 	uint64_t                mr_end;
70bf21cd93STycho Nightingale };
71bf21cd93STycho Nightingale 
72bf21cd93STycho Nightingale struct mmio_rb_tree;
73bf21cd93STycho Nightingale RB_PROTOTYPE(mmio_rb_tree, mmio_rb_range, mr_link, mmio_rb_range_compare);
74bf21cd93STycho Nightingale 
75bf21cd93STycho Nightingale RB_HEAD(mmio_rb_tree, mmio_rb_range) mmio_rb_root, mmio_rb_fallback;
76bf21cd93STycho Nightingale 
77bf21cd93STycho Nightingale /*
78bf21cd93STycho Nightingale  * Per-vCPU cache. Since most accesses from a vCPU will be to
79bf21cd93STycho Nightingale  * consecutive addresses in a range, it makes sense to cache the
80bf21cd93STycho Nightingale  * result of a lookup.
81bf21cd93STycho Nightingale  */
82bf21cd93STycho Nightingale static struct mmio_rb_range	*mmio_hint[VM_MAXCPU];
83bf21cd93STycho Nightingale 
84bf21cd93STycho Nightingale static pthread_rwlock_t mmio_rwlock;
85bf21cd93STycho Nightingale 
86bf21cd93STycho Nightingale static int
mmio_rb_range_compare(struct mmio_rb_range * a,struct mmio_rb_range * b)87bf21cd93STycho Nightingale mmio_rb_range_compare(struct mmio_rb_range *a, struct mmio_rb_range *b)
88bf21cd93STycho Nightingale {
89bf21cd93STycho Nightingale 	if (a->mr_end < b->mr_base)
90bf21cd93STycho Nightingale 		return (-1);
91bf21cd93STycho Nightingale 	else if (a->mr_base > b->mr_end)
92bf21cd93STycho Nightingale 		return (1);
93bf21cd93STycho Nightingale 	return (0);
94bf21cd93STycho Nightingale }
95bf21cd93STycho Nightingale 
96bf21cd93STycho Nightingale static int
mmio_rb_lookup(struct mmio_rb_tree * rbt,uint64_t addr,struct mmio_rb_range ** entry)97bf21cd93STycho Nightingale mmio_rb_lookup(struct mmio_rb_tree *rbt, uint64_t addr,
98bf21cd93STycho Nightingale     struct mmio_rb_range **entry)
99bf21cd93STycho Nightingale {
100bf21cd93STycho Nightingale 	struct mmio_rb_range find, *res;
101bf21cd93STycho Nightingale 
102bf21cd93STycho Nightingale 	find.mr_base = find.mr_end = addr;
103bf21cd93STycho Nightingale 
104bf21cd93STycho Nightingale 	res = RB_FIND(mmio_rb_tree, rbt, &find);
105bf21cd93STycho Nightingale 
106bf21cd93STycho Nightingale 	if (res != NULL) {
107bf21cd93STycho Nightingale 		*entry = res;
108bf21cd93STycho Nightingale 		return (0);
109bf21cd93STycho Nightingale 	}
110*e0c0d44eSPatrick Mooney 
111bf21cd93STycho Nightingale 	return (ENOENT);
112bf21cd93STycho Nightingale }
113bf21cd93STycho Nightingale 
114bf21cd93STycho Nightingale static int
mmio_rb_add(struct mmio_rb_tree * rbt,struct mmio_rb_range * new)115bf21cd93STycho Nightingale mmio_rb_add(struct mmio_rb_tree *rbt, struct mmio_rb_range *new)
116bf21cd93STycho Nightingale {
117bf21cd93STycho Nightingale 	struct mmio_rb_range *overlap;
118bf21cd93STycho Nightingale 
119bf21cd93STycho Nightingale 	overlap = RB_INSERT(mmio_rb_tree, rbt, new);
120bf21cd93STycho Nightingale 
121bf21cd93STycho Nightingale 	if (overlap != NULL) {
122bf21cd93STycho Nightingale #ifdef RB_DEBUG
123bf21cd93STycho Nightingale 		printf("overlap detected: new %lx:%lx, tree %lx:%lx\n",
124bf21cd93STycho Nightingale 		       new->mr_base, new->mr_end,
125bf21cd93STycho Nightingale 		       overlap->mr_base, overlap->mr_end);
126bf21cd93STycho Nightingale #endif
127bf21cd93STycho Nightingale 
128bf21cd93STycho Nightingale 		return (EEXIST);
129bf21cd93STycho Nightingale 	}
130bf21cd93STycho Nightingale 
131bf21cd93STycho Nightingale 	return (0);
132bf21cd93STycho Nightingale }
133bf21cd93STycho Nightingale 
134bf21cd93STycho Nightingale #if 0
135bf21cd93STycho Nightingale static void
136bf21cd93STycho Nightingale mmio_rb_dump(struct mmio_rb_tree *rbt)
137bf21cd93STycho Nightingale {
1384c87aefeSPatrick Mooney 	int perror;
139bf21cd93STycho Nightingale 	struct mmio_rb_range *np;
140bf21cd93STycho Nightingale 
141bf21cd93STycho Nightingale 	pthread_rwlock_rdlock(&mmio_rwlock);
142bf21cd93STycho Nightingale 	RB_FOREACH(np, mmio_rb_tree, rbt) {
143bf21cd93STycho Nightingale 		printf(" %lx:%lx, %s\n", np->mr_base, np->mr_end,
144bf21cd93STycho Nightingale 		       np->mr_param.name);
145bf21cd93STycho Nightingale 	}
1464c87aefeSPatrick Mooney 	perror = pthread_rwlock_unlock(&mmio_rwlock);
1474c87aefeSPatrick Mooney 	assert(perror == 0);
148bf21cd93STycho Nightingale }
149bf21cd93STycho Nightingale #endif
150bf21cd93STycho Nightingale 
151bf21cd93STycho Nightingale RB_GENERATE(mmio_rb_tree, mmio_rb_range, mr_link, mmio_rb_range_compare);
152bf21cd93STycho Nightingale 
1534c87aefeSPatrick Mooney typedef int (mem_cb_t)(struct vmctx *ctx, int vcpu, uint64_t gpa,
1544c87aefeSPatrick Mooney     struct mem_range *mr, void *arg);
1554c87aefeSPatrick Mooney 
156bf21cd93STycho Nightingale static int
mem_read(void * ctx,int vcpu,uint64_t gpa,uint64_t * rval,int size,void * arg)157bf21cd93STycho Nightingale mem_read(void *ctx, int vcpu, uint64_t gpa, uint64_t *rval, int size, void *arg)
158bf21cd93STycho Nightingale {
159bf21cd93STycho Nightingale 	int error;
160bf21cd93STycho Nightingale 	struct mem_range *mr = arg;
161bf21cd93STycho Nightingale 
162bf21cd93STycho Nightingale 	error = (*mr->handler)(ctx, vcpu, MEM_F_READ, gpa, size,
163bf21cd93STycho Nightingale 			       rval, mr->arg1, mr->arg2);
164bf21cd93STycho Nightingale 	return (error);
165bf21cd93STycho Nightingale }
166bf21cd93STycho Nightingale 
167bf21cd93STycho Nightingale static int
mem_write(void * ctx,int vcpu,uint64_t gpa,uint64_t wval,int size,void * arg)168bf21cd93STycho Nightingale mem_write(void *ctx, int vcpu, uint64_t gpa, uint64_t wval, int size, void *arg)
169bf21cd93STycho Nightingale {
170bf21cd93STycho Nightingale 	int error;
171bf21cd93STycho Nightingale 	struct mem_range *mr = arg;
172bf21cd93STycho Nightingale 
173bf21cd93STycho Nightingale 	error = (*mr->handler)(ctx, vcpu, MEM_F_WRITE, gpa, size,
174bf21cd93STycho Nightingale 			       &wval, mr->arg1, mr->arg2);
175bf21cd93STycho Nightingale 	return (error);
176bf21cd93STycho Nightingale }
177bf21cd93STycho Nightingale 
1784c87aefeSPatrick Mooney static int
access_memory(struct vmctx * ctx,int vcpu,uint64_t paddr,mem_cb_t * cb,void * arg)1794c87aefeSPatrick Mooney access_memory(struct vmctx *ctx, int vcpu, uint64_t paddr, mem_cb_t *cb,
1804c87aefeSPatrick Mooney     void *arg)
181bf21cd93STycho Nightingale {
182bf21cd93STycho Nightingale 	struct mmio_rb_range *entry;
1834c87aefeSPatrick Mooney 	int err, perror, immutable;
184*e0c0d44eSPatrick Mooney 
185bf21cd93STycho Nightingale 	pthread_rwlock_rdlock(&mmio_rwlock);
186bf21cd93STycho Nightingale 	/*
187bf21cd93STycho Nightingale 	 * First check the per-vCPU cache
188bf21cd93STycho Nightingale 	 */
189bf21cd93STycho Nightingale 	if (mmio_hint[vcpu] &&
190bf21cd93STycho Nightingale 	    paddr >= mmio_hint[vcpu]->mr_base &&
191bf21cd93STycho Nightingale 	    paddr <= mmio_hint[vcpu]->mr_end) {
192bf21cd93STycho Nightingale 		entry = mmio_hint[vcpu];
193bf21cd93STycho Nightingale 	} else
194bf21cd93STycho Nightingale 		entry = NULL;
195bf21cd93STycho Nightingale 
196bf21cd93STycho Nightingale 	if (entry == NULL) {
197bf21cd93STycho Nightingale 		if (mmio_rb_lookup(&mmio_rb_root, paddr, &entry) == 0) {
198bf21cd93STycho Nightingale 			/* Update the per-vCPU cache */
199*e0c0d44eSPatrick Mooney 			mmio_hint[vcpu] = entry;
200bf21cd93STycho Nightingale 		} else if (mmio_rb_lookup(&mmio_rb_fallback, paddr, &entry)) {
2014c87aefeSPatrick Mooney 			perror = pthread_rwlock_unlock(&mmio_rwlock);
2024c87aefeSPatrick Mooney 			assert(perror == 0);
203bf21cd93STycho Nightingale 			return (ESRCH);
204bf21cd93STycho Nightingale 		}
205bf21cd93STycho Nightingale 	}
206bf21cd93STycho Nightingale 
207bf21cd93STycho Nightingale 	assert(entry != NULL);
208bf21cd93STycho Nightingale 
209bf21cd93STycho Nightingale 	/*
210bf21cd93STycho Nightingale 	 * An 'immutable' memory range is guaranteed to be never removed
211bf21cd93STycho Nightingale 	 * so there is no need to hold 'mmio_rwlock' while calling the
212bf21cd93STycho Nightingale 	 * handler.
213bf21cd93STycho Nightingale 	 *
214bf21cd93STycho Nightingale 	 * XXX writes to the PCIR_COMMAND register can cause register_mem()
215bf21cd93STycho Nightingale 	 * to be called. If the guest is using PCI extended config space
216bf21cd93STycho Nightingale 	 * to modify the PCIR_COMMAND register then register_mem() can
217bf21cd93STycho Nightingale 	 * deadlock on 'mmio_rwlock'. However by registering the extended
218bf21cd93STycho Nightingale 	 * config space window as 'immutable' the deadlock can be avoided.
219bf21cd93STycho Nightingale 	 */
220bf21cd93STycho Nightingale 	immutable = (entry->mr_param.flags & MEM_F_IMMUTABLE);
2214c87aefeSPatrick Mooney 	if (immutable) {
2224c87aefeSPatrick Mooney 		perror = pthread_rwlock_unlock(&mmio_rwlock);
2234c87aefeSPatrick Mooney 		assert(perror == 0);
2244c87aefeSPatrick Mooney 	}
225bf21cd93STycho Nightingale 
2264c87aefeSPatrick Mooney 	err = cb(ctx, vcpu, paddr, &entry->mr_param, arg);
2274c87aefeSPatrick Mooney 
2284c87aefeSPatrick Mooney 	if (!immutable) {
2294c87aefeSPatrick Mooney 		perror = pthread_rwlock_unlock(&mmio_rwlock);
2304c87aefeSPatrick Mooney 		assert(perror == 0);
2314c87aefeSPatrick Mooney 	}
232bf21cd93STycho Nightingale 
233bf21cd93STycho Nightingale 
234bf21cd93STycho Nightingale 	return (err);
235bf21cd93STycho Nightingale }
236bf21cd93STycho Nightingale 
2374c87aefeSPatrick Mooney static int
emulate_mem_cb(struct vmctx * ctx,int vcpu,uint64_t paddr,struct mem_range * mr,void * arg)2384c87aefeSPatrick Mooney emulate_mem_cb(struct vmctx *ctx, int vcpu, uint64_t paddr, struct mem_range *mr,
2394c87aefeSPatrick Mooney     void *arg)
2404c87aefeSPatrick Mooney {
241*e0c0d44eSPatrick Mooney 	struct vm_mmio *mmio;
242*e0c0d44eSPatrick Mooney 	int err = 0;
243*e0c0d44eSPatrick Mooney 
244*e0c0d44eSPatrick Mooney 	mmio = arg;
2454c87aefeSPatrick Mooney 
246*e0c0d44eSPatrick Mooney 	if (mmio->read != 0) {
247*e0c0d44eSPatrick Mooney 		err = mem_read(ctx, vcpu, paddr, &mmio->data, mmio->bytes, mr);
248*e0c0d44eSPatrick Mooney 	} else {
249*e0c0d44eSPatrick Mooney 		err = mem_write(ctx, vcpu, paddr, mmio->data, mmio->bytes, mr);
250*e0c0d44eSPatrick Mooney 	}
251*e0c0d44eSPatrick Mooney 
252*e0c0d44eSPatrick Mooney 	return (err);
2534c87aefeSPatrick Mooney }
2544c87aefeSPatrick Mooney 
2554c87aefeSPatrick Mooney int
emulate_mem(struct vmctx * ctx,int vcpu,struct vm_mmio * mmio)256*e0c0d44eSPatrick Mooney emulate_mem(struct vmctx *ctx, int vcpu, struct vm_mmio *mmio)
2574c87aefeSPatrick Mooney {
258*e0c0d44eSPatrick Mooney 	return (access_memory(ctx, vcpu, mmio->gpa, emulate_mem_cb, mmio));
2594c87aefeSPatrick Mooney }
2604c87aefeSPatrick Mooney 
2614c87aefeSPatrick Mooney struct rw_mem_args {
2624c87aefeSPatrick Mooney 	uint64_t *val;
2634c87aefeSPatrick Mooney 	int size;
2644c87aefeSPatrick Mooney 	int operation;
2654c87aefeSPatrick Mooney };
2664c87aefeSPatrick Mooney 
2674c87aefeSPatrick Mooney static int
rw_mem_cb(struct vmctx * ctx,int vcpu,uint64_t paddr,struct mem_range * mr,void * arg)2684c87aefeSPatrick Mooney rw_mem_cb(struct vmctx *ctx, int vcpu, uint64_t paddr, struct mem_range *mr,
2694c87aefeSPatrick Mooney     void *arg)
2704c87aefeSPatrick Mooney {
2714c87aefeSPatrick Mooney 	struct rw_mem_args *rma;
2724c87aefeSPatrick Mooney 
2734c87aefeSPatrick Mooney 	rma = arg;
2744c87aefeSPatrick Mooney 	return (mr->handler(ctx, vcpu, rma->operation, paddr, rma->size,
2754c87aefeSPatrick Mooney 	    rma->val, mr->arg1, mr->arg2));
2764c87aefeSPatrick Mooney }
2774c87aefeSPatrick Mooney 
2784c87aefeSPatrick Mooney int
read_mem(struct vmctx * ctx,int vcpu,uint64_t gpa,uint64_t * rval,int size)2794c87aefeSPatrick Mooney read_mem(struct vmctx *ctx, int vcpu, uint64_t gpa, uint64_t *rval, int size)
2804c87aefeSPatrick Mooney {
2814c87aefeSPatrick Mooney 	struct rw_mem_args rma;
2824c87aefeSPatrick Mooney 
2834c87aefeSPatrick Mooney 	rma.val = rval;
2844c87aefeSPatrick Mooney 	rma.size = size;
2854c87aefeSPatrick Mooney 	rma.operation = MEM_F_READ;
2864c87aefeSPatrick Mooney 	return (access_memory(ctx, vcpu, gpa, rw_mem_cb, &rma));
2874c87aefeSPatrick Mooney }
2884c87aefeSPatrick Mooney 
2894c87aefeSPatrick Mooney int
write_mem(struct vmctx * ctx,int vcpu,uint64_t gpa,uint64_t wval,int size)2904c87aefeSPatrick Mooney write_mem(struct vmctx *ctx, int vcpu, uint64_t gpa, uint64_t wval, int size)
2914c87aefeSPatrick Mooney {
2924c87aefeSPatrick Mooney 	struct rw_mem_args rma;
2934c87aefeSPatrick Mooney 
2944c87aefeSPatrick Mooney 	rma.val = &wval;
2954c87aefeSPatrick Mooney 	rma.size = size;
2964c87aefeSPatrick Mooney 	rma.operation = MEM_F_WRITE;
2974c87aefeSPatrick Mooney 	return (access_memory(ctx, vcpu, gpa, rw_mem_cb, &rma));
2984c87aefeSPatrick Mooney }
2994c87aefeSPatrick Mooney 
300bf21cd93STycho Nightingale static int
register_mem_int(struct mmio_rb_tree * rbt,struct mem_range * memp)301bf21cd93STycho Nightingale register_mem_int(struct mmio_rb_tree *rbt, struct mem_range *memp)
302bf21cd93STycho Nightingale {
303bf21cd93STycho Nightingale 	struct mmio_rb_range *entry, *mrp;
3044c87aefeSPatrick Mooney 	int err, perror;
305bf21cd93STycho Nightingale 
306bf21cd93STycho Nightingale 	err = 0;
307bf21cd93STycho Nightingale 
308bf21cd93STycho Nightingale 	mrp = malloc(sizeof(struct mmio_rb_range));
3094c87aefeSPatrick Mooney 	if (mrp == NULL) {
3104c87aefeSPatrick Mooney 		warn("%s: couldn't allocate memory for mrp\n",
3114c87aefeSPatrick Mooney 		     __func__);
3124c87aefeSPatrick Mooney 		err = ENOMEM;
3134c87aefeSPatrick Mooney 	} else {
314bf21cd93STycho Nightingale 		mrp->mr_param = *memp;
315bf21cd93STycho Nightingale 		mrp->mr_base = memp->base;
316bf21cd93STycho Nightingale 		mrp->mr_end = memp->base + memp->size - 1;
317bf21cd93STycho Nightingale 		pthread_rwlock_wrlock(&mmio_rwlock);
318bf21cd93STycho Nightingale 		if (mmio_rb_lookup(rbt, memp->base, &entry) != 0)
319bf21cd93STycho Nightingale 			err = mmio_rb_add(rbt, mrp);
3204c87aefeSPatrick Mooney 		perror = pthread_rwlock_unlock(&mmio_rwlock);
3214c87aefeSPatrick Mooney 		assert(perror == 0);
322bf21cd93STycho Nightingale 		if (err)
323bf21cd93STycho Nightingale 			free(mrp);
3244c87aefeSPatrick Mooney 	}
325bf21cd93STycho Nightingale 
326bf21cd93STycho Nightingale 	return (err);
327bf21cd93STycho Nightingale }
328bf21cd93STycho Nightingale 
329bf21cd93STycho Nightingale int
register_mem(struct mem_range * memp)330bf21cd93STycho Nightingale register_mem(struct mem_range *memp)
331bf21cd93STycho Nightingale {
332bf21cd93STycho Nightingale 
333bf21cd93STycho Nightingale 	return (register_mem_int(&mmio_rb_root, memp));
334bf21cd93STycho Nightingale }
335bf21cd93STycho Nightingale 
336bf21cd93STycho Nightingale int
register_mem_fallback(struct mem_range * memp)337bf21cd93STycho Nightingale register_mem_fallback(struct mem_range *memp)
338bf21cd93STycho Nightingale {
339bf21cd93STycho Nightingale 
340bf21cd93STycho Nightingale 	return (register_mem_int(&mmio_rb_fallback, memp));
341bf21cd93STycho Nightingale }
342bf21cd93STycho Nightingale 
343*e0c0d44eSPatrick Mooney int
unregister_mem(struct mem_range * memp)344bf21cd93STycho Nightingale unregister_mem(struct mem_range *memp)
345bf21cd93STycho Nightingale {
346bf21cd93STycho Nightingale 	struct mem_range *mr;
347bf21cd93STycho Nightingale 	struct mmio_rb_range *entry = NULL;
3484c87aefeSPatrick Mooney 	int err, perror, i;
349*e0c0d44eSPatrick Mooney 
350bf21cd93STycho Nightingale 	pthread_rwlock_wrlock(&mmio_rwlock);
351bf21cd93STycho Nightingale 	err = mmio_rb_lookup(&mmio_rb_root, memp->base, &entry);
352bf21cd93STycho Nightingale 	if (err == 0) {
353bf21cd93STycho Nightingale 		mr = &entry->mr_param;
354bf21cd93STycho Nightingale 		assert(mr->name == memp->name);
355*e0c0d44eSPatrick Mooney 		assert(mr->base == memp->base && mr->size == memp->size);
356bf21cd93STycho Nightingale 		assert((mr->flags & MEM_F_IMMUTABLE) == 0);
357bf21cd93STycho Nightingale 		RB_REMOVE(mmio_rb_tree, &mmio_rb_root, entry);
358bf21cd93STycho Nightingale 
359*e0c0d44eSPatrick Mooney 		/* flush Per-vCPU cache */
360bf21cd93STycho Nightingale 		for (i=0; i < VM_MAXCPU; i++) {
361bf21cd93STycho Nightingale 			if (mmio_hint[i] == entry)
362bf21cd93STycho Nightingale 				mmio_hint[i] = NULL;
363bf21cd93STycho Nightingale 		}
364bf21cd93STycho Nightingale 	}
3654c87aefeSPatrick Mooney 	perror = pthread_rwlock_unlock(&mmio_rwlock);
3664c87aefeSPatrick Mooney 	assert(perror == 0);
367bf21cd93STycho Nightingale 
368bf21cd93STycho Nightingale 	if (entry)
369bf21cd93STycho Nightingale 		free(entry);
370*e0c0d44eSPatrick Mooney 
371bf21cd93STycho Nightingale 	return (err);
372bf21cd93STycho Nightingale }
373bf21cd93STycho Nightingale 
374bf21cd93STycho Nightingale void
init_mem(void)375bf21cd93STycho Nightingale init_mem(void)
376bf21cd93STycho Nightingale {
377bf21cd93STycho Nightingale 
378bf21cd93STycho Nightingale 	RB_INIT(&mmio_rb_root);
379bf21cd93STycho Nightingale 	RB_INIT(&mmio_rb_fallback);
380bf21cd93STycho Nightingale 	pthread_rwlock_init(&mmio_rwlock, NULL);
381bf21cd93STycho Nightingale }
382