1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * gnttab.c 31 * 32 * Granting foreign access to our memory reservation. 33 * 34 * Copyright (c) 2005-2006, Christopher Clark 35 * Copyright (c) 2004-2005, K A Fraser 36 * 37 * This program is free software; you can redistribute it and/or 38 * modify it under the terms of the GNU General Public License version 2 39 * as published by the Free Software Foundation; or, when distributed 40 * separately from the Linux kernel or incorporated into other 41 * software packages, subject to the following license: 42 * 43 * Permission is hereby granted, free of charge, to any person obtaining a copy 44 * of this source file (the "Software"), to deal in the Software without 45 * restriction, including without limitation the rights to use, copy, modify, 46 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 47 * and to permit persons to whom the Software is furnished to do so, subject to 48 * the following conditions: 49 * 50 * The above copyright notice and this permission notice shall be included in 51 * all copies or substantial portions of the Software. 52 * 53 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 54 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 55 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 56 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 57 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 58 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 59 * IN THE SOFTWARE. 60 */ 61 62 #include <sys/types.h> 63 #include <sys/archsystm.h> 64 #ifdef XPV_HVM_DRIVER 65 #include <sys/xpv_support.h> 66 #include <sys/mman.h> 67 #include <vm/hat.h> 68 #endif 69 #include <sys/hypervisor.h> 70 #include <sys/gnttab.h> 71 #include <sys/sysmacros.h> 72 #include <sys/machsystm.h> 73 #include <sys/systm.h> 74 #include <sys/mutex.h> 75 #include <sys/atomic.h> 76 #include <sys/spl.h> 77 #include <sys/condvar.h> 78 #include <sys/cpuvar.h> 79 #include <sys/taskq.h> 80 #include <sys/panic.h> 81 #include <sys/cmn_err.h> 82 #include <sys/promif.h> 83 #include <sys/cpu.h> 84 #include <sys/vmem.h> 85 #include <vm/hat_i86.h> 86 #include <sys/bootconf.h> 87 #include <sys/bootsvcs.h> 88 #ifndef XPV_HVM_DRIVER 89 #include <sys/bootinfo.h> 90 #include <sys/multiboot.h> 91 #include <vm/kboot_mmu.h> 92 #endif 93 #include <sys/bootvfs.h> 94 #include <sys/bootprops.h> 95 #include <vm/seg_kmem.h> 96 #include <sys/mman.h> 97 98 /* Globals */ 99 100 static grant_ref_t **gnttab_list; 101 static uint_t nr_grant_frames; 102 static int gnttab_free_count; 103 static grant_ref_t gnttab_free_head; 104 static kmutex_t gnttab_list_lock; 105 static grant_entry_t *shared; 106 static struct gnttab_free_callback *gnttab_free_callback_list; 107 108 /* Macros */ 109 110 #define GT_PGADDR(i) ((uintptr_t)shared + ((i) << MMU_PAGESHIFT)) 111 #define VALID_GRANT_REF(r) ((r) < (nr_grant_frames * GREFS_PER_GRANT_FRAME)) 112 #define RPP (PAGESIZE / sizeof (grant_ref_t)) 113 #define GNTTAB_ENTRY(entry) (gnttab_list[(entry) / RPP][(entry) % RPP]) 114 #define CMPXCHG(t, c, n) atomic_cas_16((t), (c), (n)) 115 /* External tools reserve first few grant table entries. */ 116 #define NR_RESERVED_ENTRIES 8 117 #define GNTTAB_LIST_END 0xffffffff 118 #define GREFS_PER_GRANT_FRAME (PAGESIZE / sizeof (grant_entry_t)) 119 120 /* Implementation */ 121 122 static uint_t 123 max_nr_grant_frames(void) 124 { 125 struct gnttab_query_size query; 126 int rc; 127 128 query.dom = DOMID_SELF; 129 130 rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1); 131 if ((rc < 0) || (query.status != GNTST_okay)) 132 return (4); /* Legacy max supported number of frames */ 133 134 ASSERT(query.max_nr_frames); 135 return (query.max_nr_frames); 136 } 137 138 static void 139 do_free_callbacks(void) 140 { 141 struct gnttab_free_callback *callback, *next; 142 143 callback = gnttab_free_callback_list; 144 gnttab_free_callback_list = NULL; 145 146 while (callback != NULL) { 147 next = callback->next; 148 if (gnttab_free_count >= callback->count) { 149 callback->next = NULL; 150 callback->fn(callback->arg); 151 } else { 152 callback->next = gnttab_free_callback_list; 153 gnttab_free_callback_list = callback; 154 } 155 callback = next; 156 } 157 } 158 159 static void 160 check_free_callbacks(void) 161 { 162 if (gnttab_free_callback_list) 163 do_free_callbacks(); 164 } 165 166 static int 167 grow_gnttab_list(uint_t more_frames) 168 { 169 uint_t new_nr_grant_frames, extra_entries, i; 170 171 ASSERT(MUTEX_HELD(&gnttab_list_lock)); 172 173 new_nr_grant_frames = nr_grant_frames + more_frames; 174 extra_entries = more_frames * GREFS_PER_GRANT_FRAME; 175 176 for (i = nr_grant_frames; i < new_nr_grant_frames; i++) 177 gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP); 178 179 for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames; 180 i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++) 181 GNTTAB_ENTRY(i) = i + 1; 182 183 GNTTAB_ENTRY(i) = gnttab_free_head; 184 gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames; 185 gnttab_free_count += extra_entries; 186 187 nr_grant_frames = new_nr_grant_frames; 188 189 check_free_callbacks(); 190 191 return (0); 192 } 193 194 static int 195 gnttab_expand(uint_t req_entries) 196 { 197 uint_t cur, extra; 198 199 ASSERT(MUTEX_HELD(&gnttab_list_lock)); 200 201 cur = nr_grant_frames; 202 extra = ((req_entries + (GREFS_PER_GRANT_FRAME - 1)) / 203 GREFS_PER_GRANT_FRAME); 204 if (cur + extra > max_nr_grant_frames()) 205 return (-1); 206 207 return (grow_gnttab_list(extra)); 208 } 209 210 static int 211 get_free_entries(int count) 212 { 213 int ref, rc; 214 grant_ref_t head; 215 216 mutex_enter(&gnttab_list_lock); 217 if (gnttab_free_count < count && 218 ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) { 219 mutex_exit(&gnttab_list_lock); 220 return (rc); 221 } 222 ref = head = gnttab_free_head; 223 gnttab_free_count -= count; 224 while (count-- > 1) 225 head = GNTTAB_ENTRY(head); 226 gnttab_free_head = GNTTAB_ENTRY(head); 227 GNTTAB_ENTRY(head) = GNTTAB_LIST_END; 228 mutex_exit(&gnttab_list_lock); 229 return (ref); 230 } 231 232 static void 233 put_free_entry(grant_ref_t ref) 234 { 235 ASSERT(VALID_GRANT_REF(ref)); 236 237 mutex_enter(&gnttab_list_lock); 238 GNTTAB_ENTRY(ref) = gnttab_free_head; 239 gnttab_free_head = ref; 240 gnttab_free_count++; 241 check_free_callbacks(); 242 mutex_exit(&gnttab_list_lock); 243 } 244 245 /* 246 * Public grant-issuing interface functions 247 */ 248 249 int 250 gnttab_grant_foreign_access(domid_t domid, gnttab_frame_t frame, int readonly) 251 { 252 int ref; 253 254 if ((ref = get_free_entries(1)) == -1) 255 return (-1); 256 257 ASSERT(VALID_GRANT_REF(ref)); 258 259 shared[ref].frame = frame; 260 shared[ref].domid = domid; 261 membar_producer(); 262 shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 263 264 return (ref); 265 } 266 267 void 268 gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid, 269 gnttab_frame_t frame, int readonly) 270 { 271 ASSERT(VALID_GRANT_REF(ref)); 272 273 shared[ref].frame = frame; 274 shared[ref].domid = domid; 275 membar_producer(); 276 shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0); 277 } 278 279 280 int 281 gnttab_query_foreign_access(grant_ref_t ref) 282 { 283 uint16_t nflags; 284 285 ASSERT(VALID_GRANT_REF(ref)); 286 287 nflags = shared[ref].flags; 288 289 return (nflags & (GTF_reading|GTF_writing)); 290 } 291 292 /* ARGSUSED */ 293 int 294 gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly) 295 { 296 uint16_t flags, nflags; 297 298 ASSERT(VALID_GRANT_REF(ref)); 299 300 nflags = shared[ref].flags; 301 do { 302 if ((flags = nflags) & (GTF_reading|GTF_writing)) { 303 cmn_err(CE_WARN, "g.e. still in use!"); 304 return (0); 305 } 306 } while ((nflags = CMPXCHG(&shared[ref].flags, flags, 0)) != flags); 307 308 return (1); 309 } 310 311 void 312 gnttab_end_foreign_access(grant_ref_t ref, int readonly, gnttab_frame_t page) 313 { 314 ASSERT(VALID_GRANT_REF(ref)); 315 316 if (gnttab_end_foreign_access_ref(ref, readonly)) { 317 put_free_entry(ref); 318 /* 319 * XXPV - we don't support freeing a page here 320 */ 321 if (page != 0) { 322 cmn_err(CE_WARN, 323 "gnttab_end_foreign_access_ref: using unsupported free_page interface"); 324 /* free_page(page); */ 325 } 326 } else { 327 /* 328 * XXX This needs to be fixed so that the ref and page are 329 * placed on a list to be freed up later. 330 */ 331 cmn_err(CE_WARN, "leaking g.e. and page still in use!"); 332 } 333 } 334 335 int 336 gnttab_grant_foreign_transfer(domid_t domid, pfn_t pfn) 337 { 338 int ref; 339 340 if ((ref = get_free_entries(1)) == -1) 341 return (-1); 342 343 ASSERT(VALID_GRANT_REF(ref)); 344 345 gnttab_grant_foreign_transfer_ref(ref, domid, pfn); 346 347 return (ref); 348 } 349 350 void 351 gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, pfn_t pfn) 352 { 353 ASSERT(VALID_GRANT_REF(ref)); 354 355 shared[ref].frame = pfn; 356 shared[ref].domid = domid; 357 membar_producer(); 358 shared[ref].flags = GTF_accept_transfer; 359 } 360 361 gnttab_frame_t 362 gnttab_end_foreign_transfer_ref(grant_ref_t ref) 363 { 364 gnttab_frame_t frame; 365 uint16_t flags; 366 367 ASSERT(VALID_GRANT_REF(ref)); 368 369 /* 370 * If a transfer is not even yet started, try to reclaim the grant 371 * reference and return failure (== 0). 372 */ 373 while (!((flags = shared[ref].flags) & GTF_transfer_committed)) { 374 if (CMPXCHG(&shared[ref].flags, flags, 0) == flags) 375 return (0); 376 (void) HYPERVISOR_yield(); 377 } 378 379 /* If a transfer is in progress then wait until it is completed. */ 380 while (!(flags & GTF_transfer_completed)) { 381 flags = shared[ref].flags; 382 (void) HYPERVISOR_yield(); 383 } 384 385 /* Read the frame number /after/ reading completion status. */ 386 membar_consumer(); 387 frame = shared[ref].frame; 388 ASSERT(frame != 0); 389 390 return (frame); 391 } 392 393 gnttab_frame_t 394 gnttab_end_foreign_transfer(grant_ref_t ref) 395 { 396 gnttab_frame_t frame; 397 398 ASSERT(VALID_GRANT_REF(ref)); 399 400 frame = gnttab_end_foreign_transfer_ref(ref); 401 put_free_entry(ref); 402 return (frame); 403 } 404 405 void 406 gnttab_free_grant_reference(grant_ref_t ref) 407 { 408 ASSERT(VALID_GRANT_REF(ref)); 409 410 put_free_entry(ref); 411 } 412 413 void 414 gnttab_free_grant_references(grant_ref_t head) 415 { 416 grant_ref_t ref; 417 int count = 1; 418 419 if (head == GNTTAB_LIST_END) 420 return; 421 mutex_enter(&gnttab_list_lock); 422 ref = head; 423 while (GNTTAB_ENTRY(ref) != GNTTAB_LIST_END) { 424 ref = GNTTAB_ENTRY(ref); 425 count++; 426 } 427 GNTTAB_ENTRY(ref) = gnttab_free_head; 428 gnttab_free_head = head; 429 gnttab_free_count += count; 430 check_free_callbacks(); 431 mutex_exit(&gnttab_list_lock); 432 } 433 434 int 435 gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head) 436 { 437 int h = get_free_entries(count); 438 439 if (h == -1) 440 return (-1); 441 442 *head = h; 443 444 return (0); 445 } 446 447 int 448 gnttab_empty_grant_references(const grant_ref_t *private_head) 449 { 450 return (*private_head == GNTTAB_LIST_END); 451 } 452 453 int 454 gnttab_claim_grant_reference(grant_ref_t *private_head) 455 { 456 grant_ref_t g = *private_head; 457 458 if (g == GNTTAB_LIST_END) 459 return (-1); 460 *private_head = GNTTAB_ENTRY(g); 461 return (g); 462 } 463 464 void 465 gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t release) 466 { 467 ASSERT(VALID_GRANT_REF(release)); 468 469 GNTTAB_ENTRY(release) = *private_head; 470 *private_head = release; 471 } 472 473 void 474 gnttab_request_free_callback(struct gnttab_free_callback *callback, 475 void (*fn)(void *), void *arg, uint16_t count) 476 { 477 mutex_enter(&gnttab_list_lock); 478 if (callback->next) 479 goto out; 480 callback->fn = fn; 481 callback->arg = arg; 482 callback->count = count; 483 callback->next = gnttab_free_callback_list; 484 gnttab_free_callback_list = callback; 485 check_free_callbacks(); 486 out: 487 mutex_exit(&gnttab_list_lock); 488 } 489 490 void 491 gnttab_cancel_free_callback(struct gnttab_free_callback *callback) 492 { 493 struct gnttab_free_callback **pcb; 494 495 mutex_enter(&gnttab_list_lock); 496 for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) { 497 if (*pcb == callback) { 498 *pcb = callback->next; 499 break; 500 } 501 } 502 mutex_exit(&gnttab_list_lock); 503 } 504 505 static gnttab_frame_t * 506 gnttab_setup(gnttab_setup_table_t *pset) 507 { 508 gnttab_frame_t *frames; 509 510 frames = kmem_alloc(pset->nr_frames * sizeof (gnttab_frame_t), 511 KM_SLEEP); 512 513 /*LINTED: constant in conditional context*/ 514 set_xen_guest_handle(pset->frame_list, frames); 515 516 /* 517 * Take pset->nr_frames pages of grant table space from 518 * the hypervisor and map it 519 */ 520 if ((HYPERVISOR_grant_table_op(GNTTABOP_setup_table, pset, 1) != 0) || 521 (pset->status != 0)) { 522 cmn_err(CE_PANIC, "Grant Table setup failed"); 523 } 524 525 return (frames); 526 } 527 528 #ifdef XPV_HVM_DRIVER 529 static void 530 gnttab_map(void) 531 { 532 struct xen_add_to_physmap xatp; 533 caddr_t va; 534 pfn_t pfn; 535 int i; 536 537 va = (caddr_t)shared; 538 for (i = 0; i < max_nr_grant_frames(); i++) { 539 if ((pfn = hat_getpfnum(kas.a_hat, va)) == PFN_INVALID) 540 cmn_err(CE_PANIC, "gnttab_map: Invalid pfn"); 541 542 xatp.domid = DOMID_SELF; 543 xatp.idx = i; 544 xatp.space = XENMAPSPACE_grant_table; 545 xatp.gpfn = pfn; 546 hat_unload(kas.a_hat, va, MMU_PAGESIZE, HAT_UNLOAD); 547 /* 548 * This call replaces the existing machine page backing 549 * the given gpfn with the page from the allocated grant 550 * table at index idx. The existing machine page is 551 * returned to the free list. 552 */ 553 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp) != 0) 554 panic("Couldn't map grant table"); 555 hat_devload(kas.a_hat, va, MMU_PAGESIZE, pfn, 556 PROT_READ | PROT_WRITE, 557 HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST); 558 va += MMU_PAGESIZE; 559 } 560 } 561 #endif /* XPV_HVM_DRIVER */ 562 563 void 564 gnttab_init(void) 565 { 566 gnttab_setup_table_t set; 567 int i; 568 uint_t nr_init_grefs, max_nr_glist_frames; 569 gnttab_frame_t *frames; 570 571 /* 572 * gnttab_init() should only be invoked once. 573 */ 574 mutex_enter(&gnttab_list_lock); 575 ASSERT(nr_grant_frames == 0); 576 nr_grant_frames = 1; 577 mutex_exit(&gnttab_list_lock); 578 579 max_nr_glist_frames = (max_nr_grant_frames() * 580 GREFS_PER_GRANT_FRAME / (PAGESIZE / sizeof (grant_ref_t))); 581 582 set.dom = DOMID_SELF; 583 set.nr_frames = max_nr_grant_frames(); 584 frames = gnttab_setup(&set); 585 586 #ifdef XPV_HVM_DRIVER 587 shared = (grant_entry_t *)xen_alloc_pages(set.nr_frames); 588 589 gnttab_map(); 590 #else /* XPV_HVM_DRIVER */ 591 shared = vmem_xalloc(heap_arena, set.nr_frames * MMU_PAGESIZE, 592 MMU_PAGESIZE, 0, 0, 0, 0, VM_SLEEP); 593 for (i = 0; i < set.nr_frames; i++) { 594 hat_devload(kas.a_hat, (caddr_t)GT_PGADDR(i), PAGESIZE, 595 xen_assign_pfn(frames[i]), PROT_READ | PROT_WRITE, 596 HAT_LOAD_LOCK); 597 } 598 #endif 599 600 gnttab_list = kmem_alloc(max_nr_glist_frames * sizeof (grant_ref_t *), 601 KM_SLEEP); 602 603 for (i = 0; i < nr_grant_frames; i++) { 604 gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP); 605 } 606 607 kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t)); 608 609 nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME; 610 611 for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++) 612 GNTTAB_ENTRY(i) = i + 1; 613 614 GNTTAB_ENTRY(nr_init_grefs - 1) = GNTTAB_LIST_END; 615 gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES; 616 gnttab_free_head = NR_RESERVED_ENTRIES; 617 } 618 619 void 620 gnttab_resume(void) 621 { 622 gnttab_setup_table_t set; 623 int i; 624 gnttab_frame_t *frames; 625 uint_t available_frames = max_nr_grant_frames(); 626 627 if (available_frames < nr_grant_frames) { 628 cmn_err(CE_PANIC, "Hypervisor does not have enough grant " 629 "frames: required(%u), available(%u)", nr_grant_frames, 630 available_frames); 631 } 632 633 #ifdef XPV_HVM_DRIVER 634 gnttab_map(); 635 #endif /* XPV_HVM_DRIVER */ 636 637 set.dom = DOMID_SELF; 638 set.nr_frames = available_frames; 639 frames = gnttab_setup(&set); 640 641 for (i = 0; i < available_frames; i++) { 642 (void) HYPERVISOR_update_va_mapping(GT_PGADDR(i), 643 FRAME_TO_MA(frames[i]) | PT_VALID | PT_WRITABLE, 644 UVMF_INVLPG | UVMF_ALL); 645 } 646 kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t)); 647 } 648 649 void 650 gnttab_suspend(void) 651 { 652 int i; 653 654 /* 655 * clear grant table mappings before suspending 656 */ 657 for (i = 0; i < max_nr_grant_frames(); i++) { 658 (void) HYPERVISOR_update_va_mapping(GT_PGADDR(i), 659 0, UVMF_INVLPG); 660 } 661 } 662 663 /* 664 * Local variables: 665 * c-file-style: "solaris" 666 * indent-tabs-mode: t 667 * c-indent-level: 8 668 * c-basic-offset: 8 669 * tab-width: 8 670 * End: 671 */ 672