1843e1988Sjohnlev /* 2843e1988Sjohnlev * CDDL HEADER START 3843e1988Sjohnlev * 4843e1988Sjohnlev * The contents of this file are subject to the terms of the 5843e1988Sjohnlev * Common Development and Distribution License (the "License"). 6843e1988Sjohnlev * You may not use this file except in compliance with the License. 7843e1988Sjohnlev * 8843e1988Sjohnlev * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9843e1988Sjohnlev * or http://www.opensolaris.org/os/licensing. 10843e1988Sjohnlev * See the License for the specific language governing permissions 11843e1988Sjohnlev * and limitations under the License. 12843e1988Sjohnlev * 13843e1988Sjohnlev * When distributing Covered Code, include this CDDL HEADER in each 14843e1988Sjohnlev * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15843e1988Sjohnlev * If applicable, add the following below this CDDL HEADER, with the 16843e1988Sjohnlev * fields enclosed by brackets "[]" replaced with your own identifying 17843e1988Sjohnlev * information: Portions Copyright [yyyy] [name of copyright owner] 18843e1988Sjohnlev * 19843e1988Sjohnlev * CDDL HEADER END 20843e1988Sjohnlev */ 21843e1988Sjohnlev 22843e1988Sjohnlev /* 23a576ab5bSrab * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24843e1988Sjohnlev * Use is subject to license terms. 25843e1988Sjohnlev */ 26843e1988Sjohnlev 27843e1988Sjohnlev #pragma ident "%Z%%M% %I% %E% SMI" 28843e1988Sjohnlev 29843e1988Sjohnlev /* 30843e1988Sjohnlev * gnttab.c 31843e1988Sjohnlev * 32843e1988Sjohnlev * Granting foreign access to our memory reservation. 33843e1988Sjohnlev * 34a576ab5bSrab * Copyright (c) 2005-2006, Christopher Clark 35843e1988Sjohnlev * Copyright (c) 2004-2005, K A Fraser 36843e1988Sjohnlev * 37a576ab5bSrab * This program is free software; you can redistribute it and/or 38a576ab5bSrab * modify it under the terms of the GNU General Public License version 2 39a576ab5bSrab * as published by the Free Software Foundation; or, when distributed 40a576ab5bSrab * separately from the Linux kernel or incorporated into other 41a576ab5bSrab * software packages, subject to the following license: 42843e1988Sjohnlev * 43843e1988Sjohnlev * Permission is hereby granted, free of charge, to any person obtaining a copy 44843e1988Sjohnlev * of this source file (the "Software"), to deal in the Software without 45843e1988Sjohnlev * restriction, including without limitation the rights to use, copy, modify, 46843e1988Sjohnlev * merge, publish, distribute, sublicense, and/or sell copies of the Software, 47843e1988Sjohnlev * and to permit persons to whom the Software is furnished to do so, subject to 48843e1988Sjohnlev * the following conditions: 49843e1988Sjohnlev * 50843e1988Sjohnlev * The above copyright notice and this permission notice shall be included in 51843e1988Sjohnlev * all copies or substantial portions of the Software. 52843e1988Sjohnlev * 53843e1988Sjohnlev * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 54843e1988Sjohnlev * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 55843e1988Sjohnlev * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 56843e1988Sjohnlev * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 57843e1988Sjohnlev * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 58843e1988Sjohnlev * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 59843e1988Sjohnlev * IN THE SOFTWARE. 60843e1988Sjohnlev */ 61843e1988Sjohnlev 62843e1988Sjohnlev #include <sys/types.h> 63843e1988Sjohnlev #include <sys/archsystm.h> 64551bc2a6Smrj #ifdef XPV_HVM_DRIVER 65551bc2a6Smrj #include <sys/xpv_support.h> 66551bc2a6Smrj #include <sys/mman.h> 67551bc2a6Smrj #include <vm/hat.h> 68551bc2a6Smrj #endif 69843e1988Sjohnlev #include <sys/hypervisor.h> 70843e1988Sjohnlev #include <sys/gnttab.h> 71843e1988Sjohnlev #include <sys/sysmacros.h> 72843e1988Sjohnlev #include <sys/machsystm.h> 73843e1988Sjohnlev #include <sys/systm.h> 74843e1988Sjohnlev #include <sys/mutex.h> 75843e1988Sjohnlev #include <sys/atomic.h> 76843e1988Sjohnlev #include <sys/spl.h> 77843e1988Sjohnlev #include <sys/condvar.h> 78843e1988Sjohnlev #include <sys/cpuvar.h> 79843e1988Sjohnlev #include <sys/taskq.h> 80843e1988Sjohnlev #include <sys/panic.h> 81843e1988Sjohnlev #include <sys/cmn_err.h> 82843e1988Sjohnlev #include <sys/promif.h> 83843e1988Sjohnlev #include <sys/cpu.h> 84843e1988Sjohnlev #include <sys/vmem.h> 85843e1988Sjohnlev #include <vm/hat_i86.h> 86843e1988Sjohnlev #include <sys/bootconf.h> 87843e1988Sjohnlev #include <sys/bootsvcs.h> 88551bc2a6Smrj #ifndef XPV_HVM_DRIVER 89843e1988Sjohnlev #include <sys/bootinfo.h> 90843e1988Sjohnlev #include <sys/multiboot.h> 91551bc2a6Smrj #include <vm/kboot_mmu.h> 92551bc2a6Smrj #endif 93843e1988Sjohnlev #include <sys/bootvfs.h> 94843e1988Sjohnlev #include <sys/bootprops.h> 95843e1988Sjohnlev #include <vm/seg_kmem.h> 96a576ab5bSrab #include <sys/mman.h> 97843e1988Sjohnlev 98a576ab5bSrab /* Globals */ 99843e1988Sjohnlev 100a576ab5bSrab static grant_ref_t **gnttab_list; 101a576ab5bSrab static uint_t nr_grant_frames; 102843e1988Sjohnlev static int gnttab_free_count; 103843e1988Sjohnlev static grant_ref_t gnttab_free_head; 104843e1988Sjohnlev static kmutex_t gnttab_list_lock; 105843e1988Sjohnlev static grant_entry_t *shared; 106a576ab5bSrab static struct gnttab_free_callback *gnttab_free_callback_list; 107843e1988Sjohnlev 108a576ab5bSrab /* Macros */ 109843e1988Sjohnlev 110a576ab5bSrab #define GT_PGADDR(i) ((uintptr_t)shared + ((i) << MMU_PAGESHIFT)) 111a576ab5bSrab #define VALID_GRANT_REF(r) ((r) < (nr_grant_frames * GREFS_PER_GRANT_FRAME)) 112a576ab5bSrab #define RPP (PAGESIZE / sizeof (grant_ref_t)) 113a576ab5bSrab #define GNTTAB_ENTRY(entry) (gnttab_list[(entry) / RPP][(entry) % RPP]) 114a576ab5bSrab #define CMPXCHG(t, c, n) atomic_cas_16((t), (c), (n)) 115a576ab5bSrab /* External tools reserve first few grant table entries. */ 116a576ab5bSrab #define NR_RESERVED_ENTRIES 8 117a576ab5bSrab #define GNTTAB_LIST_END 0xffffffff 118a576ab5bSrab #define GREFS_PER_GRANT_FRAME (PAGESIZE / sizeof (grant_entry_t)) 119a576ab5bSrab 120a576ab5bSrab /* Implementation */ 121a576ab5bSrab 122a576ab5bSrab static uint_t 123a576ab5bSrab max_nr_grant_frames(void) 124843e1988Sjohnlev { 125a576ab5bSrab struct gnttab_query_size query; 126a576ab5bSrab int rc; 127843e1988Sjohnlev 128a576ab5bSrab query.dom = DOMID_SELF; 129a576ab5bSrab 130a576ab5bSrab rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 131a576ab5bSrab if ((rc < 0) || (query.status != GNTST_okay)) 132a576ab5bSrab return (4); /* Legacy max supported number of frames */ 133843e1988Sjohnlev 134a576ab5bSrab ASSERT(query.max_nr_frames); 135a576ab5bSrab return (query.max_nr_frames); 136a576ab5bSrab } 137843e1988Sjohnlev 138843e1988Sjohnlev static void 139843e1988Sjohnlev do_free_callbacks(void) 140843e1988Sjohnlev { 141843e1988Sjohnlev struct gnttab_free_callback *callback, *next; 142843e1988Sjohnlev 143843e1988Sjohnlev callback = gnttab_free_callback_list; 144843e1988Sjohnlev gnttab_free_callback_list = NULL; 145843e1988Sjohnlev 146843e1988Sjohnlev while (callback != NULL) { 147843e1988Sjohnlev next = callback->next; 148843e1988Sjohnlev if (gnttab_free_count >= callback->count) { 149843e1988Sjohnlev callback->next = NULL; 150843e1988Sjohnlev callback->fn(callback->arg); 151843e1988Sjohnlev } else { 152843e1988Sjohnlev callback->next = gnttab_free_callback_list; 153843e1988Sjohnlev gnttab_free_callback_list = callback; 154843e1988Sjohnlev } 155843e1988Sjohnlev callback = next; 156843e1988Sjohnlev } 157843e1988Sjohnlev } 158843e1988Sjohnlev 159843e1988Sjohnlev static void 160843e1988Sjohnlev check_free_callbacks(void) 161843e1988Sjohnlev { 162843e1988Sjohnlev if (gnttab_free_callback_list) 163843e1988Sjohnlev do_free_callbacks(); 164843e1988Sjohnlev } 165843e1988Sjohnlev 166a576ab5bSrab static int 167a576ab5bSrab grow_gnttab_list(uint_t more_frames) 168a576ab5bSrab { 169a576ab5bSrab uint_t new_nr_grant_frames, extra_entries, i; 170a576ab5bSrab 171a576ab5bSrab ASSERT(MUTEX_HELD(&gnttab_list_lock)); 172a576ab5bSrab 173a576ab5bSrab new_nr_grant_frames = nr_grant_frames + more_frames; 174a576ab5bSrab extra_entries = more_frames * GREFS_PER_GRANT_FRAME; 175a576ab5bSrab 176a576ab5bSrab for (i = nr_grant_frames; i < new_nr_grant_frames; i++) 177a576ab5bSrab gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP); 178a576ab5bSrab 179a576ab5bSrab for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames; 180a576ab5bSrab i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++) 181a576ab5bSrab GNTTAB_ENTRY(i) = i + 1; 182a576ab5bSrab 183a576ab5bSrab GNTTAB_ENTRY(i) = gnttab_free_head; 184a576ab5bSrab gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames; 185a576ab5bSrab gnttab_free_count += extra_entries; 186a576ab5bSrab 187a576ab5bSrab nr_grant_frames = new_nr_grant_frames; 188a576ab5bSrab 189a576ab5bSrab check_free_callbacks(); 190a576ab5bSrab 191a576ab5bSrab return (0); 192a576ab5bSrab } 193a576ab5bSrab 194a576ab5bSrab static int 195a576ab5bSrab gnttab_expand(uint_t req_entries) 196a576ab5bSrab { 197a576ab5bSrab uint_t cur, extra; 198a576ab5bSrab 199a576ab5bSrab ASSERT(MUTEX_HELD(&gnttab_list_lock)); 200a576ab5bSrab 201a576ab5bSrab cur = nr_grant_frames; 202a576ab5bSrab extra = ((req_entries + (GREFS_PER_GRANT_FRAME - 1)) / 203a576ab5bSrab GREFS_PER_GRANT_FRAME); 204a576ab5bSrab if (cur + extra > max_nr_grant_frames()) 205a576ab5bSrab return (-1); 206a576ab5bSrab 207a576ab5bSrab return (grow_gnttab_list(extra)); 208a576ab5bSrab } 209a576ab5bSrab 210a576ab5bSrab static int 211a576ab5bSrab get_free_entries(int count) 212a576ab5bSrab { 213a576ab5bSrab int ref, rc; 214a576ab5bSrab grant_ref_t head; 215a576ab5bSrab 216a576ab5bSrab mutex_enter(&gnttab_list_lock); 217a576ab5bSrab if (gnttab_free_count < count && 218a576ab5bSrab ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) { 219a576ab5bSrab mutex_exit(&gnttab_list_lock); 220a576ab5bSrab return (rc); 221a576ab5bSrab } 222a576ab5bSrab ref = head = gnttab_free_head; 223a576ab5bSrab gnttab_free_count -= count; 224a576ab5bSrab while (count-- > 1) 225a576ab5bSrab head = GNTTAB_ENTRY(head); 226a576ab5bSrab gnttab_free_head = GNTTAB_ENTRY(head); 227a576ab5bSrab GNTTAB_ENTRY(head) = GNTTAB_LIST_END; 228a576ab5bSrab mutex_exit(&gnttab_list_lock); 229a576ab5bSrab return (ref); 230a576ab5bSrab } 231a576ab5bSrab 232843e1988Sjohnlev static void 233843e1988Sjohnlev put_free_entry(grant_ref_t ref) 234843e1988Sjohnlev { 235843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 236843e1988Sjohnlev 237843e1988Sjohnlev mutex_enter(&gnttab_list_lock); 238a576ab5bSrab GNTTAB_ENTRY(ref) = gnttab_free_head; 239843e1988Sjohnlev gnttab_free_head = ref; 240843e1988Sjohnlev gnttab_free_count++; 241843e1988Sjohnlev check_free_callbacks(); 242843e1988Sjohnlev mutex_exit(&gnttab_list_lock); 243843e1988Sjohnlev } 244843e1988Sjohnlev 245843e1988Sjohnlev /* 246843e1988Sjohnlev * Public grant-issuing interface functions 247843e1988Sjohnlev */ 248843e1988Sjohnlev 249843e1988Sjohnlev int 250843e1988Sjohnlev gnttab_grant_foreign_access(domid_t domid, gnttab_frame_t frame, int readonly) 251843e1988Sjohnlev { 252843e1988Sjohnlev int ref; 253843e1988Sjohnlev 254a576ab5bSrab if ((ref = get_free_entries(1)) == -1) 255843e1988Sjohnlev return (-1); 256843e1988Sjohnlev 257843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 258843e1988Sjohnlev 259843e1988Sjohnlev shared[ref].frame = frame; 260843e1988Sjohnlev shared[ref].domid = domid; 261843e1988Sjohnlev membar_producer(); 262843e1988Sjohnlev shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 263843e1988Sjohnlev 264843e1988Sjohnlev return (ref); 265843e1988Sjohnlev } 266843e1988Sjohnlev 267843e1988Sjohnlev void 268843e1988Sjohnlev gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 269843e1988Sjohnlev gnttab_frame_t frame, int readonly) 270843e1988Sjohnlev { 271843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 272843e1988Sjohnlev 273843e1988Sjohnlev shared[ref].frame = frame; 274843e1988Sjohnlev shared[ref].domid = domid; 275843e1988Sjohnlev membar_producer(); 276843e1988Sjohnlev shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 277843e1988Sjohnlev } 278843e1988Sjohnlev 279843e1988Sjohnlev 280843e1988Sjohnlev int 281843e1988Sjohnlev gnttab_query_foreign_access(grant_ref_t ref) 282843e1988Sjohnlev { 283843e1988Sjohnlev uint16_t nflags; 284843e1988Sjohnlev 285843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 286843e1988Sjohnlev 287843e1988Sjohnlev nflags = shared[ref].flags; 288843e1988Sjohnlev 289843e1988Sjohnlev return (nflags & (GTF_reading|GTF_writing)); 290843e1988Sjohnlev } 291843e1988Sjohnlev 292843e1988Sjohnlev /* ARGSUSED */ 293843e1988Sjohnlev int 294843e1988Sjohnlev gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 295843e1988Sjohnlev { 296843e1988Sjohnlev uint16_t flags, nflags; 297843e1988Sjohnlev 298843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 299843e1988Sjohnlev 300843e1988Sjohnlev nflags = shared[ref].flags; 301843e1988Sjohnlev do { 302843e1988Sjohnlev if ((flags = nflags) & (GTF_reading|GTF_writing)) { 303843e1988Sjohnlev cmn_err(CE_WARN, "g.e. still in use!"); 304843e1988Sjohnlev return (0); 305843e1988Sjohnlev } 306a576ab5bSrab } while ((nflags = CMPXCHG(&shared[ref].flags, flags, 0)) != flags); 307843e1988Sjohnlev 308843e1988Sjohnlev return (1); 309843e1988Sjohnlev } 310843e1988Sjohnlev 311843e1988Sjohnlev void 312843e1988Sjohnlev gnttab_end_foreign_access(grant_ref_t ref, int readonly, gnttab_frame_t page) 313843e1988Sjohnlev { 314843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 315843e1988Sjohnlev 316843e1988Sjohnlev if (gnttab_end_foreign_access_ref(ref, readonly)) { 317843e1988Sjohnlev put_free_entry(ref); 318843e1988Sjohnlev /* 319843e1988Sjohnlev * XXPV - we don't support freeing a page here 320843e1988Sjohnlev */ 321843e1988Sjohnlev if (page != 0) { 322843e1988Sjohnlev cmn_err(CE_WARN, 323843e1988Sjohnlev "gnttab_end_foreign_access_ref: using unsupported free_page interface"); 324843e1988Sjohnlev /* free_page(page); */ 325843e1988Sjohnlev } 326843e1988Sjohnlev } else { 327843e1988Sjohnlev /* 328843e1988Sjohnlev * XXX This needs to be fixed so that the ref and page are 329843e1988Sjohnlev * placed on a list to be freed up later. 330843e1988Sjohnlev */ 331843e1988Sjohnlev cmn_err(CE_WARN, "leaking g.e. and page still in use!"); 332843e1988Sjohnlev } 333843e1988Sjohnlev } 334843e1988Sjohnlev 335843e1988Sjohnlev int 336a576ab5bSrab gnttab_grant_foreign_transfer(domid_t domid, pfn_t pfn) 337843e1988Sjohnlev { 338843e1988Sjohnlev int ref; 339843e1988Sjohnlev 340a576ab5bSrab if ((ref = get_free_entries(1)) == -1) 341843e1988Sjohnlev return (-1); 342843e1988Sjohnlev 343843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 344843e1988Sjohnlev 345a576ab5bSrab gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 346843e1988Sjohnlev 347843e1988Sjohnlev return (ref); 348843e1988Sjohnlev } 349843e1988Sjohnlev 350843e1988Sjohnlev void 351a576ab5bSrab gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, pfn_t pfn) 352843e1988Sjohnlev { 353843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 354843e1988Sjohnlev 355a576ab5bSrab shared[ref].frame = pfn; 356843e1988Sjohnlev shared[ref].domid = domid; 357843e1988Sjohnlev membar_producer(); 358843e1988Sjohnlev shared[ref].flags = GTF_accept_transfer; 359843e1988Sjohnlev } 360843e1988Sjohnlev 361843e1988Sjohnlev gnttab_frame_t 362843e1988Sjohnlev gnttab_end_foreign_transfer_ref(grant_ref_t ref) 363843e1988Sjohnlev { 364843e1988Sjohnlev gnttab_frame_t frame; 365843e1988Sjohnlev uint16_t flags; 366843e1988Sjohnlev 367843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 368843e1988Sjohnlev 369843e1988Sjohnlev /* 370843e1988Sjohnlev * If a transfer is not even yet started, try to reclaim the grant 371843e1988Sjohnlev * reference and return failure (== 0). 372843e1988Sjohnlev */ 373843e1988Sjohnlev while (!((flags = shared[ref].flags) & GTF_transfer_committed)) { 374a576ab5bSrab if (CMPXCHG(&shared[ref].flags, flags, 0) == flags) 375843e1988Sjohnlev return (0); 376843e1988Sjohnlev (void) HYPERVISOR_yield(); 377843e1988Sjohnlev } 378843e1988Sjohnlev 379843e1988Sjohnlev /* If a transfer is in progress then wait until it is completed. */ 380843e1988Sjohnlev while (!(flags & GTF_transfer_completed)) { 381843e1988Sjohnlev flags = shared[ref].flags; 382843e1988Sjohnlev (void) HYPERVISOR_yield(); 383843e1988Sjohnlev } 384843e1988Sjohnlev 385843e1988Sjohnlev /* Read the frame number /after/ reading completion status. */ 386843e1988Sjohnlev membar_consumer(); 387843e1988Sjohnlev frame = shared[ref].frame; 388843e1988Sjohnlev ASSERT(frame != 0); 389843e1988Sjohnlev 390843e1988Sjohnlev return (frame); 391843e1988Sjohnlev } 392843e1988Sjohnlev 393843e1988Sjohnlev gnttab_frame_t 394843e1988Sjohnlev gnttab_end_foreign_transfer(grant_ref_t ref) 395843e1988Sjohnlev { 396843e1988Sjohnlev gnttab_frame_t frame; 397843e1988Sjohnlev 398843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 399843e1988Sjohnlev 400843e1988Sjohnlev frame = gnttab_end_foreign_transfer_ref(ref); 401843e1988Sjohnlev put_free_entry(ref); 402843e1988Sjohnlev return (frame); 403843e1988Sjohnlev } 404843e1988Sjohnlev 405843e1988Sjohnlev void 406843e1988Sjohnlev gnttab_free_grant_reference(grant_ref_t ref) 407843e1988Sjohnlev { 408843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 409843e1988Sjohnlev 410843e1988Sjohnlev put_free_entry(ref); 411843e1988Sjohnlev } 412843e1988Sjohnlev 413843e1988Sjohnlev void 414843e1988Sjohnlev gnttab_free_grant_references(grant_ref_t head) 415843e1988Sjohnlev { 416843e1988Sjohnlev grant_ref_t ref; 417843e1988Sjohnlev int count = 1; 418843e1988Sjohnlev 419843e1988Sjohnlev if (head == GNTTAB_LIST_END) 420843e1988Sjohnlev return; 421843e1988Sjohnlev mutex_enter(&gnttab_list_lock); 422843e1988Sjohnlev ref = head; 423a576ab5bSrab while (GNTTAB_ENTRY(ref) != GNTTAB_LIST_END) { 424a576ab5bSrab ref = GNTTAB_ENTRY(ref); 425843e1988Sjohnlev count++; 426843e1988Sjohnlev } 427a576ab5bSrab GNTTAB_ENTRY(ref) = gnttab_free_head; 428843e1988Sjohnlev gnttab_free_head = head; 429843e1988Sjohnlev gnttab_free_count += count; 430843e1988Sjohnlev check_free_callbacks(); 431843e1988Sjohnlev mutex_exit(&gnttab_list_lock); 432843e1988Sjohnlev } 433843e1988Sjohnlev 434843e1988Sjohnlev int 435843e1988Sjohnlev gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head) 436843e1988Sjohnlev { 437843e1988Sjohnlev int h = get_free_entries(count); 438843e1988Sjohnlev 439843e1988Sjohnlev if (h == -1) 440843e1988Sjohnlev return (-1); 441843e1988Sjohnlev 442843e1988Sjohnlev *head = h; 443843e1988Sjohnlev 444843e1988Sjohnlev return (0); 445843e1988Sjohnlev } 446843e1988Sjohnlev 447a576ab5bSrab int 448a576ab5bSrab gnttab_empty_grant_references(const grant_ref_t *private_head) 449a576ab5bSrab { 450a576ab5bSrab return (*private_head == GNTTAB_LIST_END); 451a576ab5bSrab } 452a576ab5bSrab 453843e1988Sjohnlev int 454843e1988Sjohnlev gnttab_claim_grant_reference(grant_ref_t *private_head) 455843e1988Sjohnlev { 456843e1988Sjohnlev grant_ref_t g = *private_head; 457843e1988Sjohnlev 458843e1988Sjohnlev if (g == GNTTAB_LIST_END) 459843e1988Sjohnlev return (-1); 460a576ab5bSrab *private_head = GNTTAB_ENTRY(g); 461843e1988Sjohnlev return (g); 462843e1988Sjohnlev } 463843e1988Sjohnlev 464843e1988Sjohnlev void 465843e1988Sjohnlev gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t release) 466843e1988Sjohnlev { 467843e1988Sjohnlev ASSERT(VALID_GRANT_REF(release)); 468843e1988Sjohnlev 469a576ab5bSrab GNTTAB_ENTRY(release) = *private_head; 470843e1988Sjohnlev *private_head = release; 471843e1988Sjohnlev } 472843e1988Sjohnlev 473843e1988Sjohnlev void 474843e1988Sjohnlev gnttab_request_free_callback(struct gnttab_free_callback *callback, 475843e1988Sjohnlev void (*fn)(void *), void *arg, uint16_t count) 476843e1988Sjohnlev { 477843e1988Sjohnlev mutex_enter(&gnttab_list_lock); 478843e1988Sjohnlev if (callback->next) 479843e1988Sjohnlev goto out; 480843e1988Sjohnlev callback->fn = fn; 481843e1988Sjohnlev callback->arg = arg; 482843e1988Sjohnlev callback->count = count; 483843e1988Sjohnlev callback->next = gnttab_free_callback_list; 484843e1988Sjohnlev gnttab_free_callback_list = callback; 485843e1988Sjohnlev check_free_callbacks(); 486843e1988Sjohnlev out: 487843e1988Sjohnlev mutex_exit(&gnttab_list_lock); 488843e1988Sjohnlev } 489843e1988Sjohnlev 490a576ab5bSrab void 491a576ab5bSrab gnttab_cancel_free_callback(struct gnttab_free_callback *callback) 492a576ab5bSrab { 493a576ab5bSrab struct gnttab_free_callback **pcb; 494a576ab5bSrab 495a576ab5bSrab mutex_enter(&gnttab_list_lock); 496a576ab5bSrab for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 497a576ab5bSrab if (*pcb == callback) { 498a576ab5bSrab *pcb = callback->next; 499a576ab5bSrab break; 500a576ab5bSrab } 501a576ab5bSrab } 502a576ab5bSrab mutex_exit(&gnttab_list_lock); 503a576ab5bSrab } 504a576ab5bSrab 505a576ab5bSrab static gnttab_frame_t * 506a576ab5bSrab gnttab_setup(gnttab_setup_table_t *pset) 507a576ab5bSrab { 508a576ab5bSrab gnttab_frame_t *frames; 509a576ab5bSrab 510a576ab5bSrab frames = kmem_alloc(pset->nr_frames * sizeof (gnttab_frame_t), 511a576ab5bSrab KM_SLEEP); 512a576ab5bSrab 513a576ab5bSrab /*LINTED: constant in conditional context*/ 514a576ab5bSrab set_xen_guest_handle(pset->frame_list, frames); 515a576ab5bSrab 516a576ab5bSrab /* 517a576ab5bSrab * Take pset->nr_frames pages of grant table space from 518a576ab5bSrab * the hypervisor and map it 519a576ab5bSrab */ 520a576ab5bSrab if ((HYPERVISOR_grant_table_op(GNTTABOP_setup_table, pset, 1) != 0) || 521a576ab5bSrab (pset->status != 0)) { 522a576ab5bSrab cmn_err(CE_PANIC, "Grant Table setup failed"); 523a576ab5bSrab } 524a576ab5bSrab 525a576ab5bSrab return (frames); 526a576ab5bSrab } 527551bc2a6Smrj 528a576ab5bSrab #ifdef XPV_HVM_DRIVER 529551bc2a6Smrj static void 530551bc2a6Smrj gnttab_map(void) 531551bc2a6Smrj { 532551bc2a6Smrj struct xen_add_to_physmap xatp; 533551bc2a6Smrj caddr_t va; 534551bc2a6Smrj pfn_t pfn; 535551bc2a6Smrj int i; 536551bc2a6Smrj 537551bc2a6Smrj va = (caddr_t)shared; 538a576ab5bSrab for (i = 0; i < max_nr_grant_frames(); i++) { 539a576ab5bSrab if ((pfn = hat_getpfnum(kas.a_hat, va)) == PFN_INVALID) 540a576ab5bSrab cmn_err(CE_PANIC, "gnttab_map: Invalid pfn"); 541551bc2a6Smrj 542551bc2a6Smrj xatp.domid = DOMID_SELF; 543551bc2a6Smrj xatp.idx = i; 544551bc2a6Smrj xatp.space = XENMAPSPACE_grant_table; 545551bc2a6Smrj xatp.gpfn = pfn; 546551bc2a6Smrj hat_unload(kas.a_hat, va, MMU_PAGESIZE, HAT_UNLOAD); 547a576ab5bSrab /* 548a576ab5bSrab * This call replaces the existing machine page backing 549a576ab5bSrab * the given gpfn with the page from the allocated grant 550a576ab5bSrab * table at index idx. The existing machine page is 551a576ab5bSrab * returned to the free list. 552a576ab5bSrab */ 553551bc2a6Smrj if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp) != 0) 554551bc2a6Smrj panic("Couldn't map grant table"); 555551bc2a6Smrj hat_devload(kas.a_hat, va, MMU_PAGESIZE, pfn, 556*5b1df0b5Sjohnlev PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 557551bc2a6Smrj HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST); 558551bc2a6Smrj va += MMU_PAGESIZE; 559551bc2a6Smrj } 560551bc2a6Smrj } 561a576ab5bSrab #endif /* XPV_HVM_DRIVER */ 562551bc2a6Smrj 563551bc2a6Smrj void 564551bc2a6Smrj gnttab_init(void) 565551bc2a6Smrj { 566a576ab5bSrab gnttab_setup_table_t set; 567551bc2a6Smrj int i; 568a576ab5bSrab uint_t nr_init_grefs, max_nr_glist_frames; 569a576ab5bSrab gnttab_frame_t *frames; 570551bc2a6Smrj 571a576ab5bSrab /* 572a576ab5bSrab * gnttab_init() should only be invoked once. 573a576ab5bSrab */ 574a576ab5bSrab mutex_enter(&gnttab_list_lock); 575a576ab5bSrab ASSERT(nr_grant_frames == 0); 576a576ab5bSrab nr_grant_frames = 1; 577a576ab5bSrab mutex_exit(&gnttab_list_lock); 578551bc2a6Smrj 579a576ab5bSrab max_nr_glist_frames = (max_nr_grant_frames() * 580a576ab5bSrab GREFS_PER_GRANT_FRAME / (PAGESIZE / sizeof (grant_ref_t))); 581551bc2a6Smrj 582a576ab5bSrab set.dom = DOMID_SELF; 583a576ab5bSrab set.nr_frames = max_nr_grant_frames(); 584a576ab5bSrab frames = gnttab_setup(&set); 585551bc2a6Smrj 586a576ab5bSrab #ifdef XPV_HVM_DRIVER 587a576ab5bSrab shared = (grant_entry_t *)xen_alloc_pages(set.nr_frames); 588551bc2a6Smrj 589551bc2a6Smrj gnttab_map(); 590551bc2a6Smrj #else /* XPV_HVM_DRIVER */ 591a576ab5bSrab shared = vmem_xalloc(heap_arena, set.nr_frames * MMU_PAGESIZE, 592a576ab5bSrab MMU_PAGESIZE, 0, 0, 0, 0, VM_SLEEP); 593a576ab5bSrab for (i = 0; i < set.nr_frames; i++) { 594a576ab5bSrab hat_devload(kas.a_hat, (caddr_t)GT_PGADDR(i), PAGESIZE, 595*5b1df0b5Sjohnlev xen_assign_pfn(frames[i]), 596*5b1df0b5Sjohnlev PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 597a576ab5bSrab HAT_LOAD_LOCK); 598a576ab5bSrab } 599a576ab5bSrab #endif 600551bc2a6Smrj 601a576ab5bSrab gnttab_list = kmem_alloc(max_nr_glist_frames * sizeof (grant_ref_t *), 602a576ab5bSrab KM_SLEEP); 603843e1988Sjohnlev 604a576ab5bSrab for (i = 0; i < nr_grant_frames; i++) { 605a576ab5bSrab gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP); 606843e1988Sjohnlev } 607843e1988Sjohnlev 608a576ab5bSrab kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t)); 609843e1988Sjohnlev 610a576ab5bSrab nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME; 611843e1988Sjohnlev 612a576ab5bSrab for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 613a576ab5bSrab GNTTAB_ENTRY(i) = i + 1; 614843e1988Sjohnlev 615a576ab5bSrab GNTTAB_ENTRY(nr_init_grefs - 1) = GNTTAB_LIST_END; 616a576ab5bSrab gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 617a576ab5bSrab gnttab_free_head = NR_RESERVED_ENTRIES; 618843e1988Sjohnlev } 619843e1988Sjohnlev 620843e1988Sjohnlev void 621843e1988Sjohnlev gnttab_resume(void) 622843e1988Sjohnlev { 623843e1988Sjohnlev gnttab_setup_table_t set; 624843e1988Sjohnlev int i; 625a576ab5bSrab gnttab_frame_t *frames; 626a576ab5bSrab uint_t available_frames = max_nr_grant_frames(); 627843e1988Sjohnlev 628a576ab5bSrab if (available_frames < nr_grant_frames) { 629a576ab5bSrab cmn_err(CE_PANIC, "Hypervisor does not have enough grant " 630a576ab5bSrab "frames: required(%u), available(%u)", nr_grant_frames, 631a576ab5bSrab available_frames); 632843e1988Sjohnlev } 633843e1988Sjohnlev 634a576ab5bSrab #ifdef XPV_HVM_DRIVER 635a576ab5bSrab gnttab_map(); 636a576ab5bSrab #endif /* XPV_HVM_DRIVER */ 637a576ab5bSrab 638a576ab5bSrab set.dom = DOMID_SELF; 639a576ab5bSrab set.nr_frames = available_frames; 640a576ab5bSrab frames = gnttab_setup(&set); 641a576ab5bSrab 642a576ab5bSrab for (i = 0; i < available_frames; i++) { 643843e1988Sjohnlev (void) HYPERVISOR_update_va_mapping(GT_PGADDR(i), 644843e1988Sjohnlev FRAME_TO_MA(frames[i]) | PT_VALID | PT_WRITABLE, 645843e1988Sjohnlev UVMF_INVLPG | UVMF_ALL); 646843e1988Sjohnlev } 647a576ab5bSrab kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t)); 648843e1988Sjohnlev } 649843e1988Sjohnlev 650843e1988Sjohnlev void 651843e1988Sjohnlev gnttab_suspend(void) 652843e1988Sjohnlev { 653843e1988Sjohnlev int i; 654843e1988Sjohnlev 655843e1988Sjohnlev /* 656843e1988Sjohnlev * clear grant table mappings before suspending 657843e1988Sjohnlev */ 658a576ab5bSrab for (i = 0; i < max_nr_grant_frames(); i++) { 659843e1988Sjohnlev (void) HYPERVISOR_update_va_mapping(GT_PGADDR(i), 660843e1988Sjohnlev 0, UVMF_INVLPG); 661843e1988Sjohnlev } 662843e1988Sjohnlev } 663843e1988Sjohnlev 664843e1988Sjohnlev /* 665843e1988Sjohnlev * Local variables: 666843e1988Sjohnlev * c-file-style: "solaris" 667843e1988Sjohnlev * indent-tabs-mode: t 668843e1988Sjohnlev * c-indent-level: 8 669843e1988Sjohnlev * c-basic-offset: 8 670843e1988Sjohnlev * tab-width: 8 671843e1988Sjohnlev * End: 672843e1988Sjohnlev */ 673