1843e1988Sjohnlev /* 2843e1988Sjohnlev * CDDL HEADER START 3843e1988Sjohnlev * 4843e1988Sjohnlev * The contents of this file are subject to the terms of the 5843e1988Sjohnlev * Common Development and Distribution License (the "License"). 6843e1988Sjohnlev * You may not use this file except in compliance with the License. 7843e1988Sjohnlev * 8843e1988Sjohnlev * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9843e1988Sjohnlev * or http://www.opensolaris.org/os/licensing. 10843e1988Sjohnlev * See the License for the specific language governing permissions 11843e1988Sjohnlev * and limitations under the License. 12843e1988Sjohnlev * 13843e1988Sjohnlev * When distributing Covered Code, include this CDDL HEADER in each 14843e1988Sjohnlev * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15843e1988Sjohnlev * If applicable, add the following below this CDDL HEADER, with the 16843e1988Sjohnlev * fields enclosed by brackets "[]" replaced with your own identifying 17843e1988Sjohnlev * information: Portions Copyright [yyyy] [name of copyright owner] 18843e1988Sjohnlev * 19843e1988Sjohnlev * CDDL HEADER END 20843e1988Sjohnlev */ 21843e1988Sjohnlev 22843e1988Sjohnlev /* 23a576ab5bSrab * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24843e1988Sjohnlev * Use is subject to license terms. 25843e1988Sjohnlev */ 26843e1988Sjohnlev 27843e1988Sjohnlev #pragma ident "%Z%%M% %I% %E% SMI" 28843e1988Sjohnlev 29843e1988Sjohnlev /* 30843e1988Sjohnlev * gnttab.c 31843e1988Sjohnlev * 32843e1988Sjohnlev * Granting foreign access to our memory reservation. 33843e1988Sjohnlev * 34a576ab5bSrab * Copyright (c) 2005-2006, Christopher Clark 35843e1988Sjohnlev * Copyright (c) 2004-2005, K A Fraser 36843e1988Sjohnlev * 37a576ab5bSrab * This program is free software; you can redistribute it and/or 38a576ab5bSrab * modify it under the terms of the GNU General Public License version 2 39a576ab5bSrab * as published by the Free Software Foundation; or, when distributed 40a576ab5bSrab * separately from the Linux kernel or incorporated into other 41a576ab5bSrab * software packages, subject to the following license: 42843e1988Sjohnlev * 43843e1988Sjohnlev * Permission is hereby granted, free of charge, to any person obtaining a copy 44843e1988Sjohnlev * of this source file (the "Software"), to deal in the Software without 45843e1988Sjohnlev * restriction, including without limitation the rights to use, copy, modify, 46843e1988Sjohnlev * merge, publish, distribute, sublicense, and/or sell copies of the Software, 47843e1988Sjohnlev * and to permit persons to whom the Software is furnished to do so, subject to 48843e1988Sjohnlev * the following conditions: 49843e1988Sjohnlev * 50843e1988Sjohnlev * The above copyright notice and this permission notice shall be included in 51843e1988Sjohnlev * all copies or substantial portions of the Software. 52843e1988Sjohnlev * 53843e1988Sjohnlev * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 54843e1988Sjohnlev * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 55843e1988Sjohnlev * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 56843e1988Sjohnlev * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 57843e1988Sjohnlev * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 58843e1988Sjohnlev * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 59843e1988Sjohnlev * IN THE SOFTWARE. 60843e1988Sjohnlev */ 61843e1988Sjohnlev 62843e1988Sjohnlev #include <sys/types.h> 63843e1988Sjohnlev #include <sys/archsystm.h> 64551bc2a6Smrj #ifdef XPV_HVM_DRIVER 65551bc2a6Smrj #include <sys/xpv_support.h> 66551bc2a6Smrj #include <sys/mman.h> 67551bc2a6Smrj #include <vm/hat.h> 68551bc2a6Smrj #endif 69843e1988Sjohnlev #include <sys/hypervisor.h> 70843e1988Sjohnlev #include <sys/gnttab.h> 71843e1988Sjohnlev #include <sys/sysmacros.h> 72843e1988Sjohnlev #include <sys/machsystm.h> 73843e1988Sjohnlev #include <sys/systm.h> 74843e1988Sjohnlev #include <sys/mutex.h> 75843e1988Sjohnlev #include <sys/atomic.h> 76843e1988Sjohnlev #include <sys/spl.h> 77843e1988Sjohnlev #include <sys/condvar.h> 78843e1988Sjohnlev #include <sys/cpuvar.h> 79843e1988Sjohnlev #include <sys/taskq.h> 80843e1988Sjohnlev #include <sys/panic.h> 81843e1988Sjohnlev #include <sys/cmn_err.h> 82843e1988Sjohnlev #include <sys/promif.h> 83843e1988Sjohnlev #include <sys/cpu.h> 84843e1988Sjohnlev #include <sys/vmem.h> 85843e1988Sjohnlev #include <vm/hat_i86.h> 86843e1988Sjohnlev #include <sys/bootconf.h> 87843e1988Sjohnlev #include <sys/bootsvcs.h> 88551bc2a6Smrj #ifndef XPV_HVM_DRIVER 89843e1988Sjohnlev #include <sys/bootinfo.h> 90843e1988Sjohnlev #include <sys/multiboot.h> 91551bc2a6Smrj #include <vm/kboot_mmu.h> 92551bc2a6Smrj #endif 93843e1988Sjohnlev #include <sys/bootvfs.h> 94843e1988Sjohnlev #include <sys/bootprops.h> 95843e1988Sjohnlev #include <vm/seg_kmem.h> 96a576ab5bSrab #include <sys/mman.h> 97843e1988Sjohnlev 98a576ab5bSrab /* Globals */ 99843e1988Sjohnlev 100a576ab5bSrab static grant_ref_t **gnttab_list; 101a576ab5bSrab static uint_t nr_grant_frames; 102843e1988Sjohnlev static int gnttab_free_count; 103843e1988Sjohnlev static grant_ref_t gnttab_free_head; 104843e1988Sjohnlev static kmutex_t gnttab_list_lock; 105843e1988Sjohnlev static grant_entry_t *shared; 106a576ab5bSrab static struct gnttab_free_callback *gnttab_free_callback_list; 107843e1988Sjohnlev 108a576ab5bSrab /* Macros */ 109843e1988Sjohnlev 110a576ab5bSrab #define GT_PGADDR(i) ((uintptr_t)shared + ((i) << MMU_PAGESHIFT)) 111a576ab5bSrab #define VALID_GRANT_REF(r) ((r) < (nr_grant_frames * GREFS_PER_GRANT_FRAME)) 112a576ab5bSrab #define RPP (PAGESIZE / sizeof (grant_ref_t)) 113a576ab5bSrab #define GNTTAB_ENTRY(entry) (gnttab_list[(entry) / RPP][(entry) % RPP]) 114a576ab5bSrab #define CMPXCHG(t, c, n) atomic_cas_16((t), (c), (n)) 115a576ab5bSrab /* External tools reserve first few grant table entries. */ 116a576ab5bSrab #define NR_RESERVED_ENTRIES 8 117a576ab5bSrab #define GNTTAB_LIST_END 0xffffffff 118a576ab5bSrab #define GREFS_PER_GRANT_FRAME (PAGESIZE / sizeof (grant_entry_t)) 119a576ab5bSrab 120a576ab5bSrab /* Implementation */ 121a576ab5bSrab 122a576ab5bSrab static uint_t 123a576ab5bSrab max_nr_grant_frames(void) 124843e1988Sjohnlev { 125a576ab5bSrab struct gnttab_query_size query; 126a576ab5bSrab int rc; 127843e1988Sjohnlev 128a576ab5bSrab query.dom = DOMID_SELF; 129a576ab5bSrab 130a576ab5bSrab rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 131a576ab5bSrab if ((rc < 0) || (query.status != GNTST_okay)) 132a576ab5bSrab return (4); /* Legacy max supported number of frames */ 133843e1988Sjohnlev 134a576ab5bSrab ASSERT(query.max_nr_frames); 135a576ab5bSrab return (query.max_nr_frames); 136a576ab5bSrab } 137843e1988Sjohnlev 138843e1988Sjohnlev static void 139843e1988Sjohnlev do_free_callbacks(void) 140843e1988Sjohnlev { 141843e1988Sjohnlev struct gnttab_free_callback *callback, *next; 142843e1988Sjohnlev 143843e1988Sjohnlev callback = gnttab_free_callback_list; 144843e1988Sjohnlev gnttab_free_callback_list = NULL; 145843e1988Sjohnlev 146843e1988Sjohnlev while (callback != NULL) { 147843e1988Sjohnlev next = callback->next; 148843e1988Sjohnlev if (gnttab_free_count >= callback->count) { 149843e1988Sjohnlev callback->next = NULL; 150843e1988Sjohnlev callback->fn(callback->arg); 151843e1988Sjohnlev } else { 152843e1988Sjohnlev callback->next = gnttab_free_callback_list; 153843e1988Sjohnlev gnttab_free_callback_list = callback; 154843e1988Sjohnlev } 155843e1988Sjohnlev callback = next; 156843e1988Sjohnlev } 157843e1988Sjohnlev } 158843e1988Sjohnlev 159843e1988Sjohnlev static void 160843e1988Sjohnlev check_free_callbacks(void) 161843e1988Sjohnlev { 162843e1988Sjohnlev if (gnttab_free_callback_list) 163843e1988Sjohnlev do_free_callbacks(); 164843e1988Sjohnlev } 165843e1988Sjohnlev 166a576ab5bSrab static int 167a576ab5bSrab grow_gnttab_list(uint_t more_frames) 168a576ab5bSrab { 169a576ab5bSrab uint_t new_nr_grant_frames, extra_entries, i; 1701213c83cSgarypen uint_t nr_glist_frames, new_nr_glist_frames; 171a576ab5bSrab 172a576ab5bSrab ASSERT(MUTEX_HELD(&gnttab_list_lock)); 173a576ab5bSrab 174a576ab5bSrab new_nr_grant_frames = nr_grant_frames + more_frames; 175a576ab5bSrab extra_entries = more_frames * GREFS_PER_GRANT_FRAME; 176a576ab5bSrab 1771213c83cSgarypen nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) 1781213c83cSgarypen / RPP; 1791213c83cSgarypen new_nr_glist_frames = (new_nr_grant_frames * GREFS_PER_GRANT_FRAME 1801213c83cSgarypen + RPP - 1) / RPP; 1811213c83cSgarypen for (i = nr_glist_frames; i < new_nr_glist_frames; i++) 182a576ab5bSrab gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP); 183a576ab5bSrab 184a576ab5bSrab for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames; 185a576ab5bSrab i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++) 186a576ab5bSrab GNTTAB_ENTRY(i) = i + 1; 187a576ab5bSrab 188a576ab5bSrab GNTTAB_ENTRY(i) = gnttab_free_head; 189a576ab5bSrab gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames; 190a576ab5bSrab gnttab_free_count += extra_entries; 191a576ab5bSrab 192a576ab5bSrab nr_grant_frames = new_nr_grant_frames; 193a576ab5bSrab 194a576ab5bSrab check_free_callbacks(); 195a576ab5bSrab 196a576ab5bSrab return (0); 197a576ab5bSrab } 198a576ab5bSrab 199a576ab5bSrab static int 200a576ab5bSrab gnttab_expand(uint_t req_entries) 201a576ab5bSrab { 202a576ab5bSrab uint_t cur, extra; 203a576ab5bSrab 204a576ab5bSrab ASSERT(MUTEX_HELD(&gnttab_list_lock)); 205a576ab5bSrab 206a576ab5bSrab cur = nr_grant_frames; 207a576ab5bSrab extra = ((req_entries + (GREFS_PER_GRANT_FRAME - 1)) / 208a576ab5bSrab GREFS_PER_GRANT_FRAME); 209a576ab5bSrab if (cur + extra > max_nr_grant_frames()) 210a576ab5bSrab return (-1); 211a576ab5bSrab 212a576ab5bSrab return (grow_gnttab_list(extra)); 213a576ab5bSrab } 214a576ab5bSrab 215a576ab5bSrab static int 216a576ab5bSrab get_free_entries(int count) 217a576ab5bSrab { 218a576ab5bSrab int ref, rc; 219a576ab5bSrab grant_ref_t head; 220a576ab5bSrab 221a576ab5bSrab mutex_enter(&gnttab_list_lock); 222a576ab5bSrab if (gnttab_free_count < count && 223a576ab5bSrab ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) { 224a576ab5bSrab mutex_exit(&gnttab_list_lock); 225a576ab5bSrab return (rc); 226a576ab5bSrab } 227a576ab5bSrab ref = head = gnttab_free_head; 228a576ab5bSrab gnttab_free_count -= count; 229a576ab5bSrab while (count-- > 1) 230a576ab5bSrab head = GNTTAB_ENTRY(head); 231a576ab5bSrab gnttab_free_head = GNTTAB_ENTRY(head); 232a576ab5bSrab GNTTAB_ENTRY(head) = GNTTAB_LIST_END; 233a576ab5bSrab mutex_exit(&gnttab_list_lock); 234a576ab5bSrab return (ref); 235a576ab5bSrab } 236a576ab5bSrab 237843e1988Sjohnlev static void 238843e1988Sjohnlev put_free_entry(grant_ref_t ref) 239843e1988Sjohnlev { 240843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 241843e1988Sjohnlev 242843e1988Sjohnlev mutex_enter(&gnttab_list_lock); 243a576ab5bSrab GNTTAB_ENTRY(ref) = gnttab_free_head; 244843e1988Sjohnlev gnttab_free_head = ref; 245843e1988Sjohnlev gnttab_free_count++; 246843e1988Sjohnlev check_free_callbacks(); 247843e1988Sjohnlev mutex_exit(&gnttab_list_lock); 248843e1988Sjohnlev } 249843e1988Sjohnlev 250843e1988Sjohnlev /* 251843e1988Sjohnlev * Public grant-issuing interface functions 252843e1988Sjohnlev */ 253843e1988Sjohnlev 254843e1988Sjohnlev int 255843e1988Sjohnlev gnttab_grant_foreign_access(domid_t domid, gnttab_frame_t frame, int readonly) 256843e1988Sjohnlev { 257843e1988Sjohnlev int ref; 258843e1988Sjohnlev 259a576ab5bSrab if ((ref = get_free_entries(1)) == -1) 260843e1988Sjohnlev return (-1); 261843e1988Sjohnlev 262843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 263843e1988Sjohnlev 264843e1988Sjohnlev shared[ref].frame = frame; 265843e1988Sjohnlev shared[ref].domid = domid; 266843e1988Sjohnlev membar_producer(); 267843e1988Sjohnlev shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 268843e1988Sjohnlev 269843e1988Sjohnlev return (ref); 270843e1988Sjohnlev } 271843e1988Sjohnlev 272843e1988Sjohnlev void 273843e1988Sjohnlev gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 274843e1988Sjohnlev gnttab_frame_t frame, int readonly) 275843e1988Sjohnlev { 276843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 277843e1988Sjohnlev 278843e1988Sjohnlev shared[ref].frame = frame; 279843e1988Sjohnlev shared[ref].domid = domid; 280843e1988Sjohnlev membar_producer(); 281843e1988Sjohnlev shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 282843e1988Sjohnlev } 283843e1988Sjohnlev 284843e1988Sjohnlev 285843e1988Sjohnlev int 286843e1988Sjohnlev gnttab_query_foreign_access(grant_ref_t ref) 287843e1988Sjohnlev { 288843e1988Sjohnlev uint16_t nflags; 289843e1988Sjohnlev 290843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 291843e1988Sjohnlev 292843e1988Sjohnlev nflags = shared[ref].flags; 293843e1988Sjohnlev 294843e1988Sjohnlev return (nflags & (GTF_reading|GTF_writing)); 295843e1988Sjohnlev } 296843e1988Sjohnlev 297843e1988Sjohnlev /* ARGSUSED */ 298843e1988Sjohnlev int 299843e1988Sjohnlev gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 300843e1988Sjohnlev { 301843e1988Sjohnlev uint16_t flags, nflags; 302843e1988Sjohnlev 303843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 304843e1988Sjohnlev 305843e1988Sjohnlev nflags = shared[ref].flags; 306843e1988Sjohnlev do { 307843e1988Sjohnlev if ((flags = nflags) & (GTF_reading|GTF_writing)) { 308843e1988Sjohnlev cmn_err(CE_WARN, "g.e. still in use!"); 309843e1988Sjohnlev return (0); 310843e1988Sjohnlev } 311a576ab5bSrab } while ((nflags = CMPXCHG(&shared[ref].flags, flags, 0)) != flags); 312843e1988Sjohnlev 313843e1988Sjohnlev return (1); 314843e1988Sjohnlev } 315843e1988Sjohnlev 316843e1988Sjohnlev void 317843e1988Sjohnlev gnttab_end_foreign_access(grant_ref_t ref, int readonly, gnttab_frame_t page) 318843e1988Sjohnlev { 319843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 320843e1988Sjohnlev 321843e1988Sjohnlev if (gnttab_end_foreign_access_ref(ref, readonly)) { 322843e1988Sjohnlev put_free_entry(ref); 323843e1988Sjohnlev /* 324843e1988Sjohnlev * XXPV - we don't support freeing a page here 325843e1988Sjohnlev */ 326843e1988Sjohnlev if (page != 0) { 327843e1988Sjohnlev cmn_err(CE_WARN, 328843e1988Sjohnlev "gnttab_end_foreign_access_ref: using unsupported free_page interface"); 329843e1988Sjohnlev /* free_page(page); */ 330843e1988Sjohnlev } 331843e1988Sjohnlev } else { 332843e1988Sjohnlev /* 333843e1988Sjohnlev * XXX This needs to be fixed so that the ref and page are 334843e1988Sjohnlev * placed on a list to be freed up later. 335843e1988Sjohnlev */ 336843e1988Sjohnlev cmn_err(CE_WARN, "leaking g.e. and page still in use!"); 337843e1988Sjohnlev } 338843e1988Sjohnlev } 339843e1988Sjohnlev 340843e1988Sjohnlev int 341a576ab5bSrab gnttab_grant_foreign_transfer(domid_t domid, pfn_t pfn) 342843e1988Sjohnlev { 343843e1988Sjohnlev int ref; 344843e1988Sjohnlev 345a576ab5bSrab if ((ref = get_free_entries(1)) == -1) 346843e1988Sjohnlev return (-1); 347843e1988Sjohnlev 348843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 349843e1988Sjohnlev 350a576ab5bSrab gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 351843e1988Sjohnlev 352843e1988Sjohnlev return (ref); 353843e1988Sjohnlev } 354843e1988Sjohnlev 355843e1988Sjohnlev void 356a576ab5bSrab gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, pfn_t pfn) 357843e1988Sjohnlev { 358843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 359843e1988Sjohnlev 360a576ab5bSrab shared[ref].frame = pfn; 361843e1988Sjohnlev shared[ref].domid = domid; 362843e1988Sjohnlev membar_producer(); 363843e1988Sjohnlev shared[ref].flags = GTF_accept_transfer; 364843e1988Sjohnlev } 365843e1988Sjohnlev 366843e1988Sjohnlev gnttab_frame_t 367843e1988Sjohnlev gnttab_end_foreign_transfer_ref(grant_ref_t ref) 368843e1988Sjohnlev { 369843e1988Sjohnlev gnttab_frame_t frame; 370843e1988Sjohnlev uint16_t flags; 371843e1988Sjohnlev 372843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 373843e1988Sjohnlev 374843e1988Sjohnlev /* 375843e1988Sjohnlev * If a transfer is not even yet started, try to reclaim the grant 376843e1988Sjohnlev * reference and return failure (== 0). 377843e1988Sjohnlev */ 378843e1988Sjohnlev while (!((flags = shared[ref].flags) & GTF_transfer_committed)) { 379a576ab5bSrab if (CMPXCHG(&shared[ref].flags, flags, 0) == flags) 380843e1988Sjohnlev return (0); 381843e1988Sjohnlev (void) HYPERVISOR_yield(); 382843e1988Sjohnlev } 383843e1988Sjohnlev 384843e1988Sjohnlev /* If a transfer is in progress then wait until it is completed. */ 385843e1988Sjohnlev while (!(flags & GTF_transfer_completed)) { 386843e1988Sjohnlev flags = shared[ref].flags; 387843e1988Sjohnlev (void) HYPERVISOR_yield(); 388843e1988Sjohnlev } 389843e1988Sjohnlev 390843e1988Sjohnlev /* Read the frame number /after/ reading completion status. */ 391843e1988Sjohnlev membar_consumer(); 392843e1988Sjohnlev frame = shared[ref].frame; 393843e1988Sjohnlev ASSERT(frame != 0); 394843e1988Sjohnlev 395843e1988Sjohnlev return (frame); 396843e1988Sjohnlev } 397843e1988Sjohnlev 398843e1988Sjohnlev gnttab_frame_t 399843e1988Sjohnlev gnttab_end_foreign_transfer(grant_ref_t ref) 400843e1988Sjohnlev { 401843e1988Sjohnlev gnttab_frame_t frame; 402843e1988Sjohnlev 403843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 404843e1988Sjohnlev 405843e1988Sjohnlev frame = gnttab_end_foreign_transfer_ref(ref); 406843e1988Sjohnlev put_free_entry(ref); 407843e1988Sjohnlev return (frame); 408843e1988Sjohnlev } 409843e1988Sjohnlev 410843e1988Sjohnlev void 411843e1988Sjohnlev gnttab_free_grant_reference(grant_ref_t ref) 412843e1988Sjohnlev { 413843e1988Sjohnlev ASSERT(VALID_GRANT_REF(ref)); 414843e1988Sjohnlev 415843e1988Sjohnlev put_free_entry(ref); 416843e1988Sjohnlev } 417843e1988Sjohnlev 418843e1988Sjohnlev void 419843e1988Sjohnlev gnttab_free_grant_references(grant_ref_t head) 420843e1988Sjohnlev { 421843e1988Sjohnlev grant_ref_t ref; 422843e1988Sjohnlev int count = 1; 423843e1988Sjohnlev 424843e1988Sjohnlev if (head == GNTTAB_LIST_END) 425843e1988Sjohnlev return; 426843e1988Sjohnlev mutex_enter(&gnttab_list_lock); 427843e1988Sjohnlev ref = head; 428a576ab5bSrab while (GNTTAB_ENTRY(ref) != GNTTAB_LIST_END) { 429a576ab5bSrab ref = GNTTAB_ENTRY(ref); 430843e1988Sjohnlev count++; 431843e1988Sjohnlev } 432a576ab5bSrab GNTTAB_ENTRY(ref) = gnttab_free_head; 433843e1988Sjohnlev gnttab_free_head = head; 434843e1988Sjohnlev gnttab_free_count += count; 435843e1988Sjohnlev check_free_callbacks(); 436843e1988Sjohnlev mutex_exit(&gnttab_list_lock); 437843e1988Sjohnlev } 438843e1988Sjohnlev 439843e1988Sjohnlev int 440843e1988Sjohnlev gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head) 441843e1988Sjohnlev { 442843e1988Sjohnlev int h = get_free_entries(count); 443843e1988Sjohnlev 444843e1988Sjohnlev if (h == -1) 445843e1988Sjohnlev return (-1); 446843e1988Sjohnlev 447843e1988Sjohnlev *head = h; 448843e1988Sjohnlev 449843e1988Sjohnlev return (0); 450843e1988Sjohnlev } 451843e1988Sjohnlev 452a576ab5bSrab int 453a576ab5bSrab gnttab_empty_grant_references(const grant_ref_t *private_head) 454a576ab5bSrab { 455a576ab5bSrab return (*private_head == GNTTAB_LIST_END); 456a576ab5bSrab } 457a576ab5bSrab 458843e1988Sjohnlev int 459843e1988Sjohnlev gnttab_claim_grant_reference(grant_ref_t *private_head) 460843e1988Sjohnlev { 461843e1988Sjohnlev grant_ref_t g = *private_head; 462843e1988Sjohnlev 463843e1988Sjohnlev if (g == GNTTAB_LIST_END) 464843e1988Sjohnlev return (-1); 465a576ab5bSrab *private_head = GNTTAB_ENTRY(g); 466843e1988Sjohnlev return (g); 467843e1988Sjohnlev } 468843e1988Sjohnlev 469843e1988Sjohnlev void 470843e1988Sjohnlev gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t release) 471843e1988Sjohnlev { 472843e1988Sjohnlev ASSERT(VALID_GRANT_REF(release)); 473843e1988Sjohnlev 474a576ab5bSrab GNTTAB_ENTRY(release) = *private_head; 475843e1988Sjohnlev *private_head = release; 476843e1988Sjohnlev } 477843e1988Sjohnlev 478843e1988Sjohnlev void 479843e1988Sjohnlev gnttab_request_free_callback(struct gnttab_free_callback *callback, 480843e1988Sjohnlev void (*fn)(void *), void *arg, uint16_t count) 481843e1988Sjohnlev { 482843e1988Sjohnlev mutex_enter(&gnttab_list_lock); 483843e1988Sjohnlev if (callback->next) 484843e1988Sjohnlev goto out; 485843e1988Sjohnlev callback->fn = fn; 486843e1988Sjohnlev callback->arg = arg; 487843e1988Sjohnlev callback->count = count; 488843e1988Sjohnlev callback->next = gnttab_free_callback_list; 489843e1988Sjohnlev gnttab_free_callback_list = callback; 490843e1988Sjohnlev check_free_callbacks(); 491843e1988Sjohnlev out: 492843e1988Sjohnlev mutex_exit(&gnttab_list_lock); 493843e1988Sjohnlev } 494843e1988Sjohnlev 495a576ab5bSrab void 496a576ab5bSrab gnttab_cancel_free_callback(struct gnttab_free_callback *callback) 497a576ab5bSrab { 498a576ab5bSrab struct gnttab_free_callback **pcb; 499a576ab5bSrab 500a576ab5bSrab mutex_enter(&gnttab_list_lock); 501a576ab5bSrab for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 502a576ab5bSrab if (*pcb == callback) { 503a576ab5bSrab *pcb = callback->next; 504a576ab5bSrab break; 505a576ab5bSrab } 506a576ab5bSrab } 507a576ab5bSrab mutex_exit(&gnttab_list_lock); 508a576ab5bSrab } 509a576ab5bSrab 510a576ab5bSrab static gnttab_frame_t * 511a576ab5bSrab gnttab_setup(gnttab_setup_table_t *pset) 512a576ab5bSrab { 513a576ab5bSrab gnttab_frame_t *frames; 514a576ab5bSrab 515a576ab5bSrab frames = kmem_alloc(pset->nr_frames * sizeof (gnttab_frame_t), 516a576ab5bSrab KM_SLEEP); 517a576ab5bSrab 518a576ab5bSrab /*LINTED: constant in conditional context*/ 519a576ab5bSrab set_xen_guest_handle(pset->frame_list, frames); 520a576ab5bSrab 521*6eb35ee7Srab #ifndef XPV_HVM_DRIVER 522a576ab5bSrab /* 523a576ab5bSrab * Take pset->nr_frames pages of grant table space from 524a576ab5bSrab * the hypervisor and map it 525a576ab5bSrab */ 526a576ab5bSrab if ((HYPERVISOR_grant_table_op(GNTTABOP_setup_table, pset, 1) != 0) || 527a576ab5bSrab (pset->status != 0)) { 528a576ab5bSrab cmn_err(CE_PANIC, "Grant Table setup failed"); 529a576ab5bSrab } 530*6eb35ee7Srab #endif 531a576ab5bSrab 532a576ab5bSrab return (frames); 533a576ab5bSrab } 534551bc2a6Smrj 535a576ab5bSrab #ifdef XPV_HVM_DRIVER 536551bc2a6Smrj static void 537551bc2a6Smrj gnttab_map(void) 538551bc2a6Smrj { 539551bc2a6Smrj struct xen_add_to_physmap xatp; 540551bc2a6Smrj caddr_t va; 541551bc2a6Smrj pfn_t pfn; 542551bc2a6Smrj int i; 543551bc2a6Smrj 544551bc2a6Smrj va = (caddr_t)shared; 545a576ab5bSrab for (i = 0; i < max_nr_grant_frames(); i++) { 546a576ab5bSrab if ((pfn = hat_getpfnum(kas.a_hat, va)) == PFN_INVALID) 547a576ab5bSrab cmn_err(CE_PANIC, "gnttab_map: Invalid pfn"); 548551bc2a6Smrj 549551bc2a6Smrj xatp.domid = DOMID_SELF; 550551bc2a6Smrj xatp.idx = i; 551551bc2a6Smrj xatp.space = XENMAPSPACE_grant_table; 552551bc2a6Smrj xatp.gpfn = pfn; 553551bc2a6Smrj hat_unload(kas.a_hat, va, MMU_PAGESIZE, HAT_UNLOAD); 554a576ab5bSrab /* 555a576ab5bSrab * This call replaces the existing machine page backing 556a576ab5bSrab * the given gpfn with the page from the allocated grant 557a576ab5bSrab * table at index idx. The existing machine page is 558a576ab5bSrab * returned to the free list. 559a576ab5bSrab */ 560551bc2a6Smrj if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp) != 0) 561551bc2a6Smrj panic("Couldn't map grant table"); 562551bc2a6Smrj hat_devload(kas.a_hat, va, MMU_PAGESIZE, pfn, 5635b1df0b5Sjohnlev PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 564551bc2a6Smrj HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST); 565551bc2a6Smrj va += MMU_PAGESIZE; 566551bc2a6Smrj } 567551bc2a6Smrj } 568a576ab5bSrab #endif /* XPV_HVM_DRIVER */ 569551bc2a6Smrj 570551bc2a6Smrj void 571551bc2a6Smrj gnttab_init(void) 572551bc2a6Smrj { 573a576ab5bSrab gnttab_setup_table_t set; 574551bc2a6Smrj int i; 5751213c83cSgarypen uint_t nr_init_grefs, max_nr_glist_frames, nr_glist_frames; 576a576ab5bSrab gnttab_frame_t *frames; 577551bc2a6Smrj 578a576ab5bSrab /* 579a576ab5bSrab * gnttab_init() should only be invoked once. 580a576ab5bSrab */ 581a576ab5bSrab mutex_enter(&gnttab_list_lock); 582a576ab5bSrab ASSERT(nr_grant_frames == 0); 583a576ab5bSrab nr_grant_frames = 1; 584a576ab5bSrab mutex_exit(&gnttab_list_lock); 585551bc2a6Smrj 586a576ab5bSrab max_nr_glist_frames = (max_nr_grant_frames() * 5871213c83cSgarypen GREFS_PER_GRANT_FRAME / RPP); 588551bc2a6Smrj 589a576ab5bSrab set.dom = DOMID_SELF; 590a576ab5bSrab set.nr_frames = max_nr_grant_frames(); 591a576ab5bSrab frames = gnttab_setup(&set); 592551bc2a6Smrj 593a576ab5bSrab #ifdef XPV_HVM_DRIVER 594a576ab5bSrab shared = (grant_entry_t *)xen_alloc_pages(set.nr_frames); 595551bc2a6Smrj 596551bc2a6Smrj gnttab_map(); 597551bc2a6Smrj #else /* XPV_HVM_DRIVER */ 598a576ab5bSrab shared = vmem_xalloc(heap_arena, set.nr_frames * MMU_PAGESIZE, 599a576ab5bSrab MMU_PAGESIZE, 0, 0, 0, 0, VM_SLEEP); 600a576ab5bSrab for (i = 0; i < set.nr_frames; i++) { 601a576ab5bSrab hat_devload(kas.a_hat, (caddr_t)GT_PGADDR(i), PAGESIZE, 6025b1df0b5Sjohnlev xen_assign_pfn(frames[i]), 6035b1df0b5Sjohnlev PROT_READ | PROT_WRITE | HAT_STORECACHING_OK, 604a576ab5bSrab HAT_LOAD_LOCK); 605a576ab5bSrab } 606a576ab5bSrab #endif 607551bc2a6Smrj 608a576ab5bSrab gnttab_list = kmem_alloc(max_nr_glist_frames * sizeof (grant_ref_t *), 609a576ab5bSrab KM_SLEEP); 610843e1988Sjohnlev 6111213c83cSgarypen nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1) 6121213c83cSgarypen / RPP; 6131213c83cSgarypen for (i = 0; i < nr_glist_frames; i++) { 614a576ab5bSrab gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP); 615843e1988Sjohnlev } 616843e1988Sjohnlev 617a576ab5bSrab kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t)); 618843e1988Sjohnlev 619a576ab5bSrab nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME; 620843e1988Sjohnlev 621a576ab5bSrab for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 622a576ab5bSrab GNTTAB_ENTRY(i) = i + 1; 623843e1988Sjohnlev 624a576ab5bSrab GNTTAB_ENTRY(nr_init_grefs - 1) = GNTTAB_LIST_END; 625a576ab5bSrab gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 626a576ab5bSrab gnttab_free_head = NR_RESERVED_ENTRIES; 627843e1988Sjohnlev } 628843e1988Sjohnlev 629843e1988Sjohnlev void 630843e1988Sjohnlev gnttab_resume(void) 631843e1988Sjohnlev { 632843e1988Sjohnlev gnttab_setup_table_t set; 633843e1988Sjohnlev int i; 634a576ab5bSrab gnttab_frame_t *frames; 635a576ab5bSrab uint_t available_frames = max_nr_grant_frames(); 636843e1988Sjohnlev 637a576ab5bSrab if (available_frames < nr_grant_frames) { 638a576ab5bSrab cmn_err(CE_PANIC, "Hypervisor does not have enough grant " 639a576ab5bSrab "frames: required(%u), available(%u)", nr_grant_frames, 640a576ab5bSrab available_frames); 641843e1988Sjohnlev } 642843e1988Sjohnlev 643a576ab5bSrab #ifdef XPV_HVM_DRIVER 644a576ab5bSrab gnttab_map(); 645a576ab5bSrab #endif /* XPV_HVM_DRIVER */ 646a576ab5bSrab 647a576ab5bSrab set.dom = DOMID_SELF; 648a576ab5bSrab set.nr_frames = available_frames; 649a576ab5bSrab frames = gnttab_setup(&set); 650a576ab5bSrab 651a576ab5bSrab for (i = 0; i < available_frames; i++) { 652843e1988Sjohnlev (void) HYPERVISOR_update_va_mapping(GT_PGADDR(i), 653843e1988Sjohnlev FRAME_TO_MA(frames[i]) | PT_VALID | PT_WRITABLE, 654843e1988Sjohnlev UVMF_INVLPG | UVMF_ALL); 655843e1988Sjohnlev } 656a576ab5bSrab kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t)); 657843e1988Sjohnlev } 658843e1988Sjohnlev 659843e1988Sjohnlev void 660843e1988Sjohnlev gnttab_suspend(void) 661843e1988Sjohnlev { 662843e1988Sjohnlev int i; 663843e1988Sjohnlev 664843e1988Sjohnlev /* 665843e1988Sjohnlev * clear grant table mappings before suspending 666843e1988Sjohnlev */ 667a576ab5bSrab for (i = 0; i < max_nr_grant_frames(); i++) { 668843e1988Sjohnlev (void) HYPERVISOR_update_va_mapping(GT_PGADDR(i), 669843e1988Sjohnlev 0, UVMF_INVLPG); 670843e1988Sjohnlev } 671843e1988Sjohnlev } 672843e1988Sjohnlev 673843e1988Sjohnlev /* 674843e1988Sjohnlev * Local variables: 675843e1988Sjohnlev * c-file-style: "solaris" 676843e1988Sjohnlev * indent-tabs-mode: t 677843e1988Sjohnlev * c-indent-level: 8 678843e1988Sjohnlev * c-basic-offset: 8 679843e1988Sjohnlev * tab-width: 8 680843e1988Sjohnlev * End: 681843e1988Sjohnlev */ 682