1bf21cd93STycho Nightingale /*- 24c87aefeSPatrick Mooney * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 34c87aefeSPatrick Mooney * 4bf21cd93STycho Nightingale * Copyright (c) 2012 NetApp, Inc. 5bf21cd93STycho Nightingale * All rights reserved. 6bf21cd93STycho Nightingale * 7bf21cd93STycho Nightingale * Redistribution and use in source and binary forms, with or without 8bf21cd93STycho Nightingale * modification, are permitted provided that the following conditions 9bf21cd93STycho Nightingale * are met: 10bf21cd93STycho Nightingale * 1. Redistributions of source code must retain the above copyright 11bf21cd93STycho Nightingale * notice, this list of conditions and the following disclaimer. 12bf21cd93STycho Nightingale * 2. Redistributions in binary form must reproduce the above copyright 13bf21cd93STycho Nightingale * notice, this list of conditions and the following disclaimer in the 14bf21cd93STycho Nightingale * documentation and/or other materials provided with the distribution. 15bf21cd93STycho Nightingale * 16bf21cd93STycho Nightingale * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17bf21cd93STycho Nightingale * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18bf21cd93STycho Nightingale * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19bf21cd93STycho Nightingale * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20bf21cd93STycho Nightingale * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21bf21cd93STycho Nightingale * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22bf21cd93STycho Nightingale * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23bf21cd93STycho Nightingale * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24bf21cd93STycho Nightingale * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25bf21cd93STycho Nightingale * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26bf21cd93STycho Nightingale * SUCH DAMAGE. 27bf21cd93STycho Nightingale * 284c87aefeSPatrick Mooney * $FreeBSD$ 29bf21cd93STycho Nightingale */ 30*e0c0d44eSPatrick Mooney /* 31*e0c0d44eSPatrick Mooney * This file and its contents are supplied under the terms of the 32*e0c0d44eSPatrick Mooney * Common Development and Distribution License ("CDDL"), version 1.0. 33*e0c0d44eSPatrick Mooney * You may only use this file in accordance with the terms of version 34*e0c0d44eSPatrick Mooney * 1.0 of the CDDL. 35*e0c0d44eSPatrick Mooney * 36*e0c0d44eSPatrick Mooney * A full copy of the text of the CDDL should have accompanied this 37*e0c0d44eSPatrick Mooney * source. A copy of the CDDL is also available via the Internet at 38*e0c0d44eSPatrick Mooney * http://www.illumos.org/license/CDDL. 39*e0c0d44eSPatrick Mooney * 40*e0c0d44eSPatrick Mooney * Copyright 2020 Oxide Computer Company 41*e0c0d44eSPatrick Mooney */ 42bf21cd93STycho Nightingale 43bf21cd93STycho Nightingale /* 44bf21cd93STycho Nightingale * Memory ranges are represented with an RB tree. On insertion, the range 45bf21cd93STycho Nightingale * is checked for overlaps. On lookup, the key has the same base and limit 46bf21cd93STycho Nightingale * so it can be searched within the range. 47bf21cd93STycho Nightingale */ 48bf21cd93STycho Nightingale 49bf21cd93STycho Nightingale #include <sys/cdefs.h> 504c87aefeSPatrick Mooney __FBSDID("$FreeBSD$"); 51bf21cd93STycho Nightingale 52bf21cd93STycho Nightingale #include <sys/types.h> 53bf21cd93STycho Nightingale #include <sys/errno.h> 544c87aefeSPatrick Mooney #include <sys/tree.h> 55bf21cd93STycho Nightingale #include <machine/vmm.h> 56bf21cd93STycho Nightingale 57bf21cd93STycho Nightingale #include <assert.h> 584c87aefeSPatrick Mooney #include <err.h> 59bf21cd93STycho Nightingale #include <pthread.h> 604c87aefeSPatrick Mooney #include <stdio.h> 614c87aefeSPatrick Mooney #include <stdlib.h> 62bf21cd93STycho Nightingale 63bf21cd93STycho Nightingale #include "mem.h" 64bf21cd93STycho Nightingale 65bf21cd93STycho Nightingale struct mmio_rb_range { 66bf21cd93STycho Nightingale RB_ENTRY(mmio_rb_range) mr_link; /* RB tree links */ 67bf21cd93STycho Nightingale struct mem_range mr_param; 68bf21cd93STycho Nightingale uint64_t mr_base; 69bf21cd93STycho Nightingale uint64_t mr_end; 70bf21cd93STycho Nightingale }; 71bf21cd93STycho Nightingale 72bf21cd93STycho Nightingale struct mmio_rb_tree; 73bf21cd93STycho Nightingale RB_PROTOTYPE(mmio_rb_tree, mmio_rb_range, mr_link, mmio_rb_range_compare); 74bf21cd93STycho Nightingale 75bf21cd93STycho Nightingale RB_HEAD(mmio_rb_tree, mmio_rb_range) mmio_rb_root, mmio_rb_fallback; 76bf21cd93STycho Nightingale 77bf21cd93STycho Nightingale /* 78bf21cd93STycho Nightingale * Per-vCPU cache. Since most accesses from a vCPU will be to 79bf21cd93STycho Nightingale * consecutive addresses in a range, it makes sense to cache the 80bf21cd93STycho Nightingale * result of a lookup. 81bf21cd93STycho Nightingale */ 82bf21cd93STycho Nightingale static struct mmio_rb_range *mmio_hint[VM_MAXCPU]; 83bf21cd93STycho Nightingale 84bf21cd93STycho Nightingale static pthread_rwlock_t mmio_rwlock; 85bf21cd93STycho Nightingale 86bf21cd93STycho Nightingale static int 87bf21cd93STycho Nightingale mmio_rb_range_compare(struct mmio_rb_range *a, struct mmio_rb_range *b) 88bf21cd93STycho Nightingale { 89bf21cd93STycho Nightingale if (a->mr_end < b->mr_base) 90bf21cd93STycho Nightingale return (-1); 91bf21cd93STycho Nightingale else if (a->mr_base > b->mr_end) 92bf21cd93STycho Nightingale return (1); 93bf21cd93STycho Nightingale return (0); 94bf21cd93STycho Nightingale } 95bf21cd93STycho Nightingale 96bf21cd93STycho Nightingale static int 97bf21cd93STycho Nightingale mmio_rb_lookup(struct mmio_rb_tree *rbt, uint64_t addr, 98bf21cd93STycho Nightingale struct mmio_rb_range **entry) 99bf21cd93STycho Nightingale { 100bf21cd93STycho Nightingale struct mmio_rb_range find, *res; 101bf21cd93STycho Nightingale 102bf21cd93STycho Nightingale find.mr_base = find.mr_end = addr; 103bf21cd93STycho Nightingale 104bf21cd93STycho Nightingale res = RB_FIND(mmio_rb_tree, rbt, &find); 105bf21cd93STycho Nightingale 106bf21cd93STycho Nightingale if (res != NULL) { 107bf21cd93STycho Nightingale *entry = res; 108bf21cd93STycho Nightingale return (0); 109bf21cd93STycho Nightingale } 110*e0c0d44eSPatrick Mooney 111bf21cd93STycho Nightingale return (ENOENT); 112bf21cd93STycho Nightingale } 113bf21cd93STycho Nightingale 114bf21cd93STycho Nightingale static int 115bf21cd93STycho Nightingale mmio_rb_add(struct mmio_rb_tree *rbt, struct mmio_rb_range *new) 116bf21cd93STycho Nightingale { 117bf21cd93STycho Nightingale struct mmio_rb_range *overlap; 118bf21cd93STycho Nightingale 119bf21cd93STycho Nightingale overlap = RB_INSERT(mmio_rb_tree, rbt, new); 120bf21cd93STycho Nightingale 121bf21cd93STycho Nightingale if (overlap != NULL) { 122bf21cd93STycho Nightingale #ifdef RB_DEBUG 123bf21cd93STycho Nightingale printf("overlap detected: new %lx:%lx, tree %lx:%lx\n", 124bf21cd93STycho Nightingale new->mr_base, new->mr_end, 125bf21cd93STycho Nightingale overlap->mr_base, overlap->mr_end); 126bf21cd93STycho Nightingale #endif 127bf21cd93STycho Nightingale 128bf21cd93STycho Nightingale return (EEXIST); 129bf21cd93STycho Nightingale } 130bf21cd93STycho Nightingale 131bf21cd93STycho Nightingale return (0); 132bf21cd93STycho Nightingale } 133bf21cd93STycho Nightingale 134bf21cd93STycho Nightingale #if 0 135bf21cd93STycho Nightingale static void 136bf21cd93STycho Nightingale mmio_rb_dump(struct mmio_rb_tree *rbt) 137bf21cd93STycho Nightingale { 1384c87aefeSPatrick Mooney int perror; 139bf21cd93STycho Nightingale struct mmio_rb_range *np; 140bf21cd93STycho Nightingale 141bf21cd93STycho Nightingale pthread_rwlock_rdlock(&mmio_rwlock); 142bf21cd93STycho Nightingale RB_FOREACH(np, mmio_rb_tree, rbt) { 143bf21cd93STycho Nightingale printf(" %lx:%lx, %s\n", np->mr_base, np->mr_end, 144bf21cd93STycho Nightingale np->mr_param.name); 145bf21cd93STycho Nightingale } 1464c87aefeSPatrick Mooney perror = pthread_rwlock_unlock(&mmio_rwlock); 1474c87aefeSPatrick Mooney assert(perror == 0); 148bf21cd93STycho Nightingale } 149bf21cd93STycho Nightingale #endif 150bf21cd93STycho Nightingale 151bf21cd93STycho Nightingale RB_GENERATE(mmio_rb_tree, mmio_rb_range, mr_link, mmio_rb_range_compare); 152bf21cd93STycho Nightingale 1534c87aefeSPatrick Mooney typedef int (mem_cb_t)(struct vmctx *ctx, int vcpu, uint64_t gpa, 1544c87aefeSPatrick Mooney struct mem_range *mr, void *arg); 1554c87aefeSPatrick Mooney 156bf21cd93STycho Nightingale static int 157bf21cd93STycho Nightingale mem_read(void *ctx, int vcpu, uint64_t gpa, uint64_t *rval, int size, void *arg) 158bf21cd93STycho Nightingale { 159bf21cd93STycho Nightingale int error; 160bf21cd93STycho Nightingale struct mem_range *mr = arg; 161bf21cd93STycho Nightingale 162bf21cd93STycho Nightingale error = (*mr->handler)(ctx, vcpu, MEM_F_READ, gpa, size, 163bf21cd93STycho Nightingale rval, mr->arg1, mr->arg2); 164bf21cd93STycho Nightingale return (error); 165bf21cd93STycho Nightingale } 166bf21cd93STycho Nightingale 167bf21cd93STycho Nightingale static int 168bf21cd93STycho Nightingale mem_write(void *ctx, int vcpu, uint64_t gpa, uint64_t wval, int size, void *arg) 169bf21cd93STycho Nightingale { 170bf21cd93STycho Nightingale int error; 171bf21cd93STycho Nightingale struct mem_range *mr = arg; 172bf21cd93STycho Nightingale 173bf21cd93STycho Nightingale error = (*mr->handler)(ctx, vcpu, MEM_F_WRITE, gpa, size, 174bf21cd93STycho Nightingale &wval, mr->arg1, mr->arg2); 175bf21cd93STycho Nightingale return (error); 176bf21cd93STycho Nightingale } 177bf21cd93STycho Nightingale 1784c87aefeSPatrick Mooney static int 1794c87aefeSPatrick Mooney access_memory(struct vmctx *ctx, int vcpu, uint64_t paddr, mem_cb_t *cb, 1804c87aefeSPatrick Mooney void *arg) 181bf21cd93STycho Nightingale { 182bf21cd93STycho Nightingale struct mmio_rb_range *entry; 1834c87aefeSPatrick Mooney int err, perror, immutable; 184*e0c0d44eSPatrick Mooney 185bf21cd93STycho Nightingale pthread_rwlock_rdlock(&mmio_rwlock); 186bf21cd93STycho Nightingale /* 187bf21cd93STycho Nightingale * First check the per-vCPU cache 188bf21cd93STycho Nightingale */ 189bf21cd93STycho Nightingale if (mmio_hint[vcpu] && 190bf21cd93STycho Nightingale paddr >= mmio_hint[vcpu]->mr_base && 191bf21cd93STycho Nightingale paddr <= mmio_hint[vcpu]->mr_end) { 192bf21cd93STycho Nightingale entry = mmio_hint[vcpu]; 193bf21cd93STycho Nightingale } else 194bf21cd93STycho Nightingale entry = NULL; 195bf21cd93STycho Nightingale 196bf21cd93STycho Nightingale if (entry == NULL) { 197bf21cd93STycho Nightingale if (mmio_rb_lookup(&mmio_rb_root, paddr, &entry) == 0) { 198bf21cd93STycho Nightingale /* Update the per-vCPU cache */ 199*e0c0d44eSPatrick Mooney mmio_hint[vcpu] = entry; 200bf21cd93STycho Nightingale } else if (mmio_rb_lookup(&mmio_rb_fallback, paddr, &entry)) { 2014c87aefeSPatrick Mooney perror = pthread_rwlock_unlock(&mmio_rwlock); 2024c87aefeSPatrick Mooney assert(perror == 0); 203bf21cd93STycho Nightingale return (ESRCH); 204bf21cd93STycho Nightingale } 205bf21cd93STycho Nightingale } 206bf21cd93STycho Nightingale 207bf21cd93STycho Nightingale assert(entry != NULL); 208bf21cd93STycho Nightingale 209bf21cd93STycho Nightingale /* 210bf21cd93STycho Nightingale * An 'immutable' memory range is guaranteed to be never removed 211bf21cd93STycho Nightingale * so there is no need to hold 'mmio_rwlock' while calling the 212bf21cd93STycho Nightingale * handler. 213bf21cd93STycho Nightingale * 214bf21cd93STycho Nightingale * XXX writes to the PCIR_COMMAND register can cause register_mem() 215bf21cd93STycho Nightingale * to be called. If the guest is using PCI extended config space 216bf21cd93STycho Nightingale * to modify the PCIR_COMMAND register then register_mem() can 217bf21cd93STycho Nightingale * deadlock on 'mmio_rwlock'. However by registering the extended 218bf21cd93STycho Nightingale * config space window as 'immutable' the deadlock can be avoided. 219bf21cd93STycho Nightingale */ 220bf21cd93STycho Nightingale immutable = (entry->mr_param.flags & MEM_F_IMMUTABLE); 2214c87aefeSPatrick Mooney if (immutable) { 2224c87aefeSPatrick Mooney perror = pthread_rwlock_unlock(&mmio_rwlock); 2234c87aefeSPatrick Mooney assert(perror == 0); 2244c87aefeSPatrick Mooney } 225bf21cd93STycho Nightingale 2264c87aefeSPatrick Mooney err = cb(ctx, vcpu, paddr, &entry->mr_param, arg); 2274c87aefeSPatrick Mooney 2284c87aefeSPatrick Mooney if (!immutable) { 2294c87aefeSPatrick Mooney perror = pthread_rwlock_unlock(&mmio_rwlock); 2304c87aefeSPatrick Mooney assert(perror == 0); 2314c87aefeSPatrick Mooney } 232bf21cd93STycho Nightingale 233bf21cd93STycho Nightingale 234bf21cd93STycho Nightingale return (err); 235bf21cd93STycho Nightingale } 236bf21cd93STycho Nightingale 2374c87aefeSPatrick Mooney static int 2384c87aefeSPatrick Mooney emulate_mem_cb(struct vmctx *ctx, int vcpu, uint64_t paddr, struct mem_range *mr, 2394c87aefeSPatrick Mooney void *arg) 2404c87aefeSPatrick Mooney { 241*e0c0d44eSPatrick Mooney struct vm_mmio *mmio; 242*e0c0d44eSPatrick Mooney int err = 0; 243*e0c0d44eSPatrick Mooney 244*e0c0d44eSPatrick Mooney mmio = arg; 2454c87aefeSPatrick Mooney 246*e0c0d44eSPatrick Mooney if (mmio->read != 0) { 247*e0c0d44eSPatrick Mooney err = mem_read(ctx, vcpu, paddr, &mmio->data, mmio->bytes, mr); 248*e0c0d44eSPatrick Mooney } else { 249*e0c0d44eSPatrick Mooney err = mem_write(ctx, vcpu, paddr, mmio->data, mmio->bytes, mr); 250*e0c0d44eSPatrick Mooney } 251*e0c0d44eSPatrick Mooney 252*e0c0d44eSPatrick Mooney return (err); 2534c87aefeSPatrick Mooney } 2544c87aefeSPatrick Mooney 2554c87aefeSPatrick Mooney int 256*e0c0d44eSPatrick Mooney emulate_mem(struct vmctx *ctx, int vcpu, struct vm_mmio *mmio) 2574c87aefeSPatrick Mooney { 258*e0c0d44eSPatrick Mooney return (access_memory(ctx, vcpu, mmio->gpa, emulate_mem_cb, mmio)); 2594c87aefeSPatrick Mooney } 2604c87aefeSPatrick Mooney 2614c87aefeSPatrick Mooney struct rw_mem_args { 2624c87aefeSPatrick Mooney uint64_t *val; 2634c87aefeSPatrick Mooney int size; 2644c87aefeSPatrick Mooney int operation; 2654c87aefeSPatrick Mooney }; 2664c87aefeSPatrick Mooney 2674c87aefeSPatrick Mooney static int 2684c87aefeSPatrick Mooney rw_mem_cb(struct vmctx *ctx, int vcpu, uint64_t paddr, struct mem_range *mr, 2694c87aefeSPatrick Mooney void *arg) 2704c87aefeSPatrick Mooney { 2714c87aefeSPatrick Mooney struct rw_mem_args *rma; 2724c87aefeSPatrick Mooney 2734c87aefeSPatrick Mooney rma = arg; 2744c87aefeSPatrick Mooney return (mr->handler(ctx, vcpu, rma->operation, paddr, rma->size, 2754c87aefeSPatrick Mooney rma->val, mr->arg1, mr->arg2)); 2764c87aefeSPatrick Mooney } 2774c87aefeSPatrick Mooney 2784c87aefeSPatrick Mooney int 2794c87aefeSPatrick Mooney read_mem(struct vmctx *ctx, int vcpu, uint64_t gpa, uint64_t *rval, int size) 2804c87aefeSPatrick Mooney { 2814c87aefeSPatrick Mooney struct rw_mem_args rma; 2824c87aefeSPatrick Mooney 2834c87aefeSPatrick Mooney rma.val = rval; 2844c87aefeSPatrick Mooney rma.size = size; 2854c87aefeSPatrick Mooney rma.operation = MEM_F_READ; 2864c87aefeSPatrick Mooney return (access_memory(ctx, vcpu, gpa, rw_mem_cb, &rma)); 2874c87aefeSPatrick Mooney } 2884c87aefeSPatrick Mooney 2894c87aefeSPatrick Mooney int 2904c87aefeSPatrick Mooney write_mem(struct vmctx *ctx, int vcpu, uint64_t gpa, uint64_t wval, int size) 2914c87aefeSPatrick Mooney { 2924c87aefeSPatrick Mooney struct rw_mem_args rma; 2934c87aefeSPatrick Mooney 2944c87aefeSPatrick Mooney rma.val = &wval; 2954c87aefeSPatrick Mooney rma.size = size; 2964c87aefeSPatrick Mooney rma.operation = MEM_F_WRITE; 2974c87aefeSPatrick Mooney return (access_memory(ctx, vcpu, gpa, rw_mem_cb, &rma)); 2984c87aefeSPatrick Mooney } 2994c87aefeSPatrick Mooney 300bf21cd93STycho Nightingale static int 301bf21cd93STycho Nightingale register_mem_int(struct mmio_rb_tree *rbt, struct mem_range *memp) 302bf21cd93STycho Nightingale { 303bf21cd93STycho Nightingale struct mmio_rb_range *entry, *mrp; 3044c87aefeSPatrick Mooney int err, perror; 305bf21cd93STycho Nightingale 306bf21cd93STycho Nightingale err = 0; 307bf21cd93STycho Nightingale 308bf21cd93STycho Nightingale mrp = malloc(sizeof(struct mmio_rb_range)); 3094c87aefeSPatrick Mooney if (mrp == NULL) { 3104c87aefeSPatrick Mooney warn("%s: couldn't allocate memory for mrp\n", 3114c87aefeSPatrick Mooney __func__); 3124c87aefeSPatrick Mooney err = ENOMEM; 3134c87aefeSPatrick Mooney } else { 314bf21cd93STycho Nightingale mrp->mr_param = *memp; 315bf21cd93STycho Nightingale mrp->mr_base = memp->base; 316bf21cd93STycho Nightingale mrp->mr_end = memp->base + memp->size - 1; 317bf21cd93STycho Nightingale pthread_rwlock_wrlock(&mmio_rwlock); 318bf21cd93STycho Nightingale if (mmio_rb_lookup(rbt, memp->base, &entry) != 0) 319bf21cd93STycho Nightingale err = mmio_rb_add(rbt, mrp); 3204c87aefeSPatrick Mooney perror = pthread_rwlock_unlock(&mmio_rwlock); 3214c87aefeSPatrick Mooney assert(perror == 0); 322bf21cd93STycho Nightingale if (err) 323bf21cd93STycho Nightingale free(mrp); 3244c87aefeSPatrick Mooney } 325bf21cd93STycho Nightingale 326bf21cd93STycho Nightingale return (err); 327bf21cd93STycho Nightingale } 328bf21cd93STycho Nightingale 329bf21cd93STycho Nightingale int 330bf21cd93STycho Nightingale register_mem(struct mem_range *memp) 331bf21cd93STycho Nightingale { 332bf21cd93STycho Nightingale 333bf21cd93STycho Nightingale return (register_mem_int(&mmio_rb_root, memp)); 334bf21cd93STycho Nightingale } 335bf21cd93STycho Nightingale 336bf21cd93STycho Nightingale int 337bf21cd93STycho Nightingale register_mem_fallback(struct mem_range *memp) 338bf21cd93STycho Nightingale { 339bf21cd93STycho Nightingale 340bf21cd93STycho Nightingale return (register_mem_int(&mmio_rb_fallback, memp)); 341bf21cd93STycho Nightingale } 342bf21cd93STycho Nightingale 343*e0c0d44eSPatrick Mooney int 344bf21cd93STycho Nightingale unregister_mem(struct mem_range *memp) 345bf21cd93STycho Nightingale { 346bf21cd93STycho Nightingale struct mem_range *mr; 347bf21cd93STycho Nightingale struct mmio_rb_range *entry = NULL; 3484c87aefeSPatrick Mooney int err, perror, i; 349*e0c0d44eSPatrick Mooney 350bf21cd93STycho Nightingale pthread_rwlock_wrlock(&mmio_rwlock); 351bf21cd93STycho Nightingale err = mmio_rb_lookup(&mmio_rb_root, memp->base, &entry); 352bf21cd93STycho Nightingale if (err == 0) { 353bf21cd93STycho Nightingale mr = &entry->mr_param; 354bf21cd93STycho Nightingale assert(mr->name == memp->name); 355*e0c0d44eSPatrick Mooney assert(mr->base == memp->base && mr->size == memp->size); 356bf21cd93STycho Nightingale assert((mr->flags & MEM_F_IMMUTABLE) == 0); 357bf21cd93STycho Nightingale RB_REMOVE(mmio_rb_tree, &mmio_rb_root, entry); 358bf21cd93STycho Nightingale 359*e0c0d44eSPatrick Mooney /* flush Per-vCPU cache */ 360bf21cd93STycho Nightingale for (i=0; i < VM_MAXCPU; i++) { 361bf21cd93STycho Nightingale if (mmio_hint[i] == entry) 362bf21cd93STycho Nightingale mmio_hint[i] = NULL; 363bf21cd93STycho Nightingale } 364bf21cd93STycho Nightingale } 3654c87aefeSPatrick Mooney perror = pthread_rwlock_unlock(&mmio_rwlock); 3664c87aefeSPatrick Mooney assert(perror == 0); 367bf21cd93STycho Nightingale 368bf21cd93STycho Nightingale if (entry) 369bf21cd93STycho Nightingale free(entry); 370*e0c0d44eSPatrick Mooney 371bf21cd93STycho Nightingale return (err); 372bf21cd93STycho Nightingale } 373bf21cd93STycho Nightingale 374bf21cd93STycho Nightingale void 375bf21cd93STycho Nightingale init_mem(void) 376bf21cd93STycho Nightingale { 377bf21cd93STycho Nightingale 378bf21cd93STycho Nightingale RB_INIT(&mmio_rb_root); 379bf21cd93STycho Nightingale RB_INIT(&mmio_rb_fallback); 380bf21cd93STycho Nightingale pthread_rwlock_init(&mmio_rwlock, NULL); 381bf21cd93STycho Nightingale } 382