xref: /illumos-gate/usr/src/uts/common/xen/os/gnttab.c (revision 6eb35ee7)
1843e1988Sjohnlev /*
2843e1988Sjohnlev  * CDDL HEADER START
3843e1988Sjohnlev  *
4843e1988Sjohnlev  * The contents of this file are subject to the terms of the
5843e1988Sjohnlev  * Common Development and Distribution License (the "License").
6843e1988Sjohnlev  * You may not use this file except in compliance with the License.
7843e1988Sjohnlev  *
8843e1988Sjohnlev  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9843e1988Sjohnlev  * or http://www.opensolaris.org/os/licensing.
10843e1988Sjohnlev  * See the License for the specific language governing permissions
11843e1988Sjohnlev  * and limitations under the License.
12843e1988Sjohnlev  *
13843e1988Sjohnlev  * When distributing Covered Code, include this CDDL HEADER in each
14843e1988Sjohnlev  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15843e1988Sjohnlev  * If applicable, add the following below this CDDL HEADER, with the
16843e1988Sjohnlev  * fields enclosed by brackets "[]" replaced with your own identifying
17843e1988Sjohnlev  * information: Portions Copyright [yyyy] [name of copyright owner]
18843e1988Sjohnlev  *
19843e1988Sjohnlev  * CDDL HEADER END
20843e1988Sjohnlev  */
21843e1988Sjohnlev 
22843e1988Sjohnlev /*
23a576ab5bSrab  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24843e1988Sjohnlev  * Use is subject to license terms.
25843e1988Sjohnlev  */
26843e1988Sjohnlev 
27843e1988Sjohnlev #pragma ident	"%Z%%M%	%I%	%E% SMI"
28843e1988Sjohnlev 
29843e1988Sjohnlev /*
30843e1988Sjohnlev  * gnttab.c
31843e1988Sjohnlev  *
32843e1988Sjohnlev  * Granting foreign access to our memory reservation.
33843e1988Sjohnlev  *
34a576ab5bSrab  * Copyright (c) 2005-2006, Christopher Clark
35843e1988Sjohnlev  * Copyright (c) 2004-2005, K A Fraser
36843e1988Sjohnlev  *
37a576ab5bSrab  * This program is free software; you can redistribute it and/or
38a576ab5bSrab  * modify it under the terms of the GNU General Public License version 2
39a576ab5bSrab  * as published by the Free Software Foundation; or, when distributed
40a576ab5bSrab  * separately from the Linux kernel or incorporated into other
41a576ab5bSrab  * software packages, subject to the following license:
42843e1988Sjohnlev  *
43843e1988Sjohnlev  * Permission is hereby granted, free of charge, to any person obtaining a copy
44843e1988Sjohnlev  * of this source file (the "Software"), to deal in the Software without
45843e1988Sjohnlev  * restriction, including without limitation the rights to use, copy, modify,
46843e1988Sjohnlev  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
47843e1988Sjohnlev  * and to permit persons to whom the Software is furnished to do so, subject to
48843e1988Sjohnlev  * the following conditions:
49843e1988Sjohnlev  *
50843e1988Sjohnlev  * The above copyright notice and this permission notice shall be included in
51843e1988Sjohnlev  * all copies or substantial portions of the Software.
52843e1988Sjohnlev  *
53843e1988Sjohnlev  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
54843e1988Sjohnlev  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
55843e1988Sjohnlev  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
56843e1988Sjohnlev  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
57843e1988Sjohnlev  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
58843e1988Sjohnlev  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
59843e1988Sjohnlev  * IN THE SOFTWARE.
60843e1988Sjohnlev  */
61843e1988Sjohnlev 
62843e1988Sjohnlev #include <sys/types.h>
63843e1988Sjohnlev #include <sys/archsystm.h>
64551bc2a6Smrj #ifdef XPV_HVM_DRIVER
65551bc2a6Smrj #include <sys/xpv_support.h>
66551bc2a6Smrj #include <sys/mman.h>
67551bc2a6Smrj #include <vm/hat.h>
68551bc2a6Smrj #endif
69843e1988Sjohnlev #include <sys/hypervisor.h>
70843e1988Sjohnlev #include <sys/gnttab.h>
71843e1988Sjohnlev #include <sys/sysmacros.h>
72843e1988Sjohnlev #include <sys/machsystm.h>
73843e1988Sjohnlev #include <sys/systm.h>
74843e1988Sjohnlev #include <sys/mutex.h>
75843e1988Sjohnlev #include <sys/atomic.h>
76843e1988Sjohnlev #include <sys/spl.h>
77843e1988Sjohnlev #include <sys/condvar.h>
78843e1988Sjohnlev #include <sys/cpuvar.h>
79843e1988Sjohnlev #include <sys/taskq.h>
80843e1988Sjohnlev #include <sys/panic.h>
81843e1988Sjohnlev #include <sys/cmn_err.h>
82843e1988Sjohnlev #include <sys/promif.h>
83843e1988Sjohnlev #include <sys/cpu.h>
84843e1988Sjohnlev #include <sys/vmem.h>
85843e1988Sjohnlev #include <vm/hat_i86.h>
86843e1988Sjohnlev #include <sys/bootconf.h>
87843e1988Sjohnlev #include <sys/bootsvcs.h>
88551bc2a6Smrj #ifndef XPV_HVM_DRIVER
89843e1988Sjohnlev #include <sys/bootinfo.h>
90843e1988Sjohnlev #include <sys/multiboot.h>
91551bc2a6Smrj #include <vm/kboot_mmu.h>
92551bc2a6Smrj #endif
93843e1988Sjohnlev #include <sys/bootvfs.h>
94843e1988Sjohnlev #include <sys/bootprops.h>
95843e1988Sjohnlev #include <vm/seg_kmem.h>
96a576ab5bSrab #include <sys/mman.h>
97843e1988Sjohnlev 
98a576ab5bSrab /* Globals */
99843e1988Sjohnlev 
100a576ab5bSrab static grant_ref_t **gnttab_list;
101a576ab5bSrab static uint_t nr_grant_frames;
102843e1988Sjohnlev static int gnttab_free_count;
103843e1988Sjohnlev static grant_ref_t gnttab_free_head;
104843e1988Sjohnlev static kmutex_t gnttab_list_lock;
105843e1988Sjohnlev static grant_entry_t *shared;
106a576ab5bSrab static struct gnttab_free_callback *gnttab_free_callback_list;
107843e1988Sjohnlev 
108a576ab5bSrab /* Macros */
109843e1988Sjohnlev 
110a576ab5bSrab #define	GT_PGADDR(i) ((uintptr_t)shared + ((i) << MMU_PAGESHIFT))
111a576ab5bSrab #define	VALID_GRANT_REF(r) ((r) < (nr_grant_frames * GREFS_PER_GRANT_FRAME))
112a576ab5bSrab #define	RPP (PAGESIZE / sizeof (grant_ref_t))
113a576ab5bSrab #define	GNTTAB_ENTRY(entry) (gnttab_list[(entry) / RPP][(entry) % RPP])
114a576ab5bSrab #define	CMPXCHG(t, c, n) atomic_cas_16((t), (c), (n))
115a576ab5bSrab /* External tools reserve first few grant table entries. */
116a576ab5bSrab #define	NR_RESERVED_ENTRIES 8
117a576ab5bSrab #define	GNTTAB_LIST_END 0xffffffff
118a576ab5bSrab #define	GREFS_PER_GRANT_FRAME (PAGESIZE / sizeof (grant_entry_t))
119a576ab5bSrab 
120a576ab5bSrab /* Implementation */
121a576ab5bSrab 
122a576ab5bSrab static uint_t
max_nr_grant_frames(void)123a576ab5bSrab max_nr_grant_frames(void)
124843e1988Sjohnlev {
125a576ab5bSrab 	struct gnttab_query_size query;
126a576ab5bSrab 	int rc;
127843e1988Sjohnlev 
128a576ab5bSrab 	query.dom = DOMID_SELF;
129a576ab5bSrab 
130a576ab5bSrab 	rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
131a576ab5bSrab 	if ((rc < 0) || (query.status != GNTST_okay))
132a576ab5bSrab 		return (4); /* Legacy max supported number of frames */
133843e1988Sjohnlev 
134a576ab5bSrab 	ASSERT(query.max_nr_frames);
135a576ab5bSrab 	return (query.max_nr_frames);
136a576ab5bSrab }
137843e1988Sjohnlev 
138843e1988Sjohnlev static void
do_free_callbacks(void)139843e1988Sjohnlev do_free_callbacks(void)
140843e1988Sjohnlev {
141843e1988Sjohnlev 	struct gnttab_free_callback *callback, *next;
142843e1988Sjohnlev 
143843e1988Sjohnlev 	callback = gnttab_free_callback_list;
144843e1988Sjohnlev 	gnttab_free_callback_list = NULL;
145843e1988Sjohnlev 
146843e1988Sjohnlev 	while (callback != NULL) {
147843e1988Sjohnlev 		next = callback->next;
148843e1988Sjohnlev 		if (gnttab_free_count >= callback->count) {
149843e1988Sjohnlev 			callback->next = NULL;
150843e1988Sjohnlev 			callback->fn(callback->arg);
151843e1988Sjohnlev 		} else {
152843e1988Sjohnlev 			callback->next = gnttab_free_callback_list;
153843e1988Sjohnlev 			gnttab_free_callback_list = callback;
154843e1988Sjohnlev 		}
155843e1988Sjohnlev 		callback = next;
156843e1988Sjohnlev 	}
157843e1988Sjohnlev }
158843e1988Sjohnlev 
159843e1988Sjohnlev static void
check_free_callbacks(void)160843e1988Sjohnlev check_free_callbacks(void)
161843e1988Sjohnlev {
162843e1988Sjohnlev 	if (gnttab_free_callback_list)
163843e1988Sjohnlev 		do_free_callbacks();
164843e1988Sjohnlev }
165843e1988Sjohnlev 
166a576ab5bSrab static int
grow_gnttab_list(uint_t more_frames)167a576ab5bSrab grow_gnttab_list(uint_t more_frames)
168a576ab5bSrab {
169a576ab5bSrab 	uint_t new_nr_grant_frames, extra_entries, i;
1701213c83cSgarypen 	uint_t nr_glist_frames, new_nr_glist_frames;
171a576ab5bSrab 
172a576ab5bSrab 	ASSERT(MUTEX_HELD(&gnttab_list_lock));
173a576ab5bSrab 
174a576ab5bSrab 	new_nr_grant_frames = nr_grant_frames + more_frames;
175a576ab5bSrab 	extra_entries = more_frames * GREFS_PER_GRANT_FRAME;
176a576ab5bSrab 
1771213c83cSgarypen 	nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1)
1781213c83cSgarypen 	    / RPP;
1791213c83cSgarypen 	new_nr_glist_frames = (new_nr_grant_frames * GREFS_PER_GRANT_FRAME
1801213c83cSgarypen 	    + RPP - 1) / RPP;
1811213c83cSgarypen 	for (i = nr_glist_frames; i < new_nr_glist_frames; i++)
182a576ab5bSrab 		gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP);
183a576ab5bSrab 
184a576ab5bSrab 	for (i = GREFS_PER_GRANT_FRAME * nr_grant_frames;
185a576ab5bSrab 	    i < GREFS_PER_GRANT_FRAME * new_nr_grant_frames - 1; i++)
186a576ab5bSrab 		GNTTAB_ENTRY(i) = i + 1;
187a576ab5bSrab 
188a576ab5bSrab 	GNTTAB_ENTRY(i) = gnttab_free_head;
189a576ab5bSrab 	gnttab_free_head = GREFS_PER_GRANT_FRAME * nr_grant_frames;
190a576ab5bSrab 	gnttab_free_count += extra_entries;
191a576ab5bSrab 
192a576ab5bSrab 	nr_grant_frames = new_nr_grant_frames;
193a576ab5bSrab 
194a576ab5bSrab 	check_free_callbacks();
195a576ab5bSrab 
196a576ab5bSrab 	return (0);
197a576ab5bSrab }
198a576ab5bSrab 
199a576ab5bSrab static int
gnttab_expand(uint_t req_entries)200a576ab5bSrab gnttab_expand(uint_t req_entries)
201a576ab5bSrab {
202a576ab5bSrab 	uint_t cur, extra;
203a576ab5bSrab 
204a576ab5bSrab 	ASSERT(MUTEX_HELD(&gnttab_list_lock));
205a576ab5bSrab 
206a576ab5bSrab 	cur = nr_grant_frames;
207a576ab5bSrab 	extra = ((req_entries + (GREFS_PER_GRANT_FRAME - 1)) /
208a576ab5bSrab 	    GREFS_PER_GRANT_FRAME);
209a576ab5bSrab 	if (cur + extra > max_nr_grant_frames())
210a576ab5bSrab 		return (-1);
211a576ab5bSrab 
212a576ab5bSrab 	return (grow_gnttab_list(extra));
213a576ab5bSrab }
214a576ab5bSrab 
215a576ab5bSrab static int
get_free_entries(int count)216a576ab5bSrab get_free_entries(int count)
217a576ab5bSrab {
218a576ab5bSrab 	int ref, rc;
219a576ab5bSrab 	grant_ref_t head;
220a576ab5bSrab 
221a576ab5bSrab 	mutex_enter(&gnttab_list_lock);
222a576ab5bSrab 	if (gnttab_free_count < count &&
223a576ab5bSrab 	    ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
224a576ab5bSrab 		mutex_exit(&gnttab_list_lock);
225a576ab5bSrab 		return (rc);
226a576ab5bSrab 	}
227a576ab5bSrab 	ref = head = gnttab_free_head;
228a576ab5bSrab 	gnttab_free_count -= count;
229a576ab5bSrab 	while (count-- > 1)
230a576ab5bSrab 		head = GNTTAB_ENTRY(head);
231a576ab5bSrab 	gnttab_free_head = GNTTAB_ENTRY(head);
232a576ab5bSrab 	GNTTAB_ENTRY(head) = GNTTAB_LIST_END;
233a576ab5bSrab 	mutex_exit(&gnttab_list_lock);
234a576ab5bSrab 	return (ref);
235a576ab5bSrab }
236a576ab5bSrab 
237843e1988Sjohnlev static void
put_free_entry(grant_ref_t ref)238843e1988Sjohnlev put_free_entry(grant_ref_t ref)
239843e1988Sjohnlev {
240843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(ref));
241843e1988Sjohnlev 
242843e1988Sjohnlev 	mutex_enter(&gnttab_list_lock);
243a576ab5bSrab 	GNTTAB_ENTRY(ref) = gnttab_free_head;
244843e1988Sjohnlev 	gnttab_free_head = ref;
245843e1988Sjohnlev 	gnttab_free_count++;
246843e1988Sjohnlev 	check_free_callbacks();
247843e1988Sjohnlev 	mutex_exit(&gnttab_list_lock);
248843e1988Sjohnlev }
249843e1988Sjohnlev 
250843e1988Sjohnlev /*
251843e1988Sjohnlev  * Public grant-issuing interface functions
252843e1988Sjohnlev  */
253843e1988Sjohnlev 
254843e1988Sjohnlev int
gnttab_grant_foreign_access(domid_t domid,gnttab_frame_t frame,int readonly)255843e1988Sjohnlev gnttab_grant_foreign_access(domid_t domid, gnttab_frame_t frame, int readonly)
256843e1988Sjohnlev {
257843e1988Sjohnlev 	int ref;
258843e1988Sjohnlev 
259a576ab5bSrab 	if ((ref = get_free_entries(1)) == -1)
260843e1988Sjohnlev 		return (-1);
261843e1988Sjohnlev 
262843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(ref));
263843e1988Sjohnlev 
264843e1988Sjohnlev 	shared[ref].frame = frame;
265843e1988Sjohnlev 	shared[ref].domid = domid;
266843e1988Sjohnlev 	membar_producer();
267843e1988Sjohnlev 	shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
268843e1988Sjohnlev 
269843e1988Sjohnlev 	return (ref);
270843e1988Sjohnlev }
271843e1988Sjohnlev 
272843e1988Sjohnlev void
gnttab_grant_foreign_access_ref(grant_ref_t ref,domid_t domid,gnttab_frame_t frame,int readonly)273843e1988Sjohnlev gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
274843e1988Sjohnlev 				gnttab_frame_t frame, int readonly)
275843e1988Sjohnlev {
276843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(ref));
277843e1988Sjohnlev 
278843e1988Sjohnlev 	shared[ref].frame = frame;
279843e1988Sjohnlev 	shared[ref].domid = domid;
280843e1988Sjohnlev 	membar_producer();
281843e1988Sjohnlev 	shared[ref].flags = GTF_permit_access | (readonly ? GTF_readonly : 0);
282843e1988Sjohnlev }
283843e1988Sjohnlev 
284843e1988Sjohnlev 
285843e1988Sjohnlev int
gnttab_query_foreign_access(grant_ref_t ref)286843e1988Sjohnlev gnttab_query_foreign_access(grant_ref_t ref)
287843e1988Sjohnlev {
288843e1988Sjohnlev 	uint16_t nflags;
289843e1988Sjohnlev 
290843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(ref));
291843e1988Sjohnlev 
292843e1988Sjohnlev 	nflags = shared[ref].flags;
293843e1988Sjohnlev 
294843e1988Sjohnlev 	return (nflags & (GTF_reading|GTF_writing));
295843e1988Sjohnlev }
296843e1988Sjohnlev 
297843e1988Sjohnlev /* ARGSUSED */
298843e1988Sjohnlev int
gnttab_end_foreign_access_ref(grant_ref_t ref,int readonly)299843e1988Sjohnlev gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
300843e1988Sjohnlev {
301843e1988Sjohnlev 	uint16_t flags, nflags;
302843e1988Sjohnlev 
303843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(ref));
304843e1988Sjohnlev 
305843e1988Sjohnlev 	nflags = shared[ref].flags;
306843e1988Sjohnlev 	do {
307843e1988Sjohnlev 		if ((flags = nflags) & (GTF_reading|GTF_writing)) {
308843e1988Sjohnlev 			cmn_err(CE_WARN, "g.e. still in use!");
309843e1988Sjohnlev 			return (0);
310843e1988Sjohnlev 		}
311a576ab5bSrab 	} while ((nflags = CMPXCHG(&shared[ref].flags, flags, 0)) != flags);
312843e1988Sjohnlev 
313843e1988Sjohnlev 	return (1);
314843e1988Sjohnlev }
315843e1988Sjohnlev 
316843e1988Sjohnlev void
gnttab_end_foreign_access(grant_ref_t ref,int readonly,gnttab_frame_t page)317843e1988Sjohnlev gnttab_end_foreign_access(grant_ref_t ref, int readonly, gnttab_frame_t page)
318843e1988Sjohnlev {
319843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(ref));
320843e1988Sjohnlev 
321843e1988Sjohnlev 	if (gnttab_end_foreign_access_ref(ref, readonly)) {
322843e1988Sjohnlev 		put_free_entry(ref);
323843e1988Sjohnlev 		/*
324843e1988Sjohnlev 		 * XXPV - we don't support freeing a page here
325843e1988Sjohnlev 		 */
326843e1988Sjohnlev 		if (page != 0) {
327843e1988Sjohnlev 			cmn_err(CE_WARN,
328843e1988Sjohnlev 	"gnttab_end_foreign_access_ref: using unsupported free_page interface");
329843e1988Sjohnlev 			/* free_page(page); */
330843e1988Sjohnlev 		}
331843e1988Sjohnlev 	} else {
332843e1988Sjohnlev 		/*
333843e1988Sjohnlev 		 * XXX This needs to be fixed so that the ref and page are
334843e1988Sjohnlev 		 * placed on a list to be freed up later.
335843e1988Sjohnlev 		 */
336843e1988Sjohnlev 		cmn_err(CE_WARN, "leaking g.e. and page still in use!");
337843e1988Sjohnlev 	}
338843e1988Sjohnlev }
339843e1988Sjohnlev 
340843e1988Sjohnlev int
gnttab_grant_foreign_transfer(domid_t domid,pfn_t pfn)341a576ab5bSrab gnttab_grant_foreign_transfer(domid_t domid, pfn_t pfn)
342843e1988Sjohnlev {
343843e1988Sjohnlev 	int ref;
344843e1988Sjohnlev 
345a576ab5bSrab 	if ((ref = get_free_entries(1)) == -1)
346843e1988Sjohnlev 		return (-1);
347843e1988Sjohnlev 
348843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(ref));
349843e1988Sjohnlev 
350a576ab5bSrab 	gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
351843e1988Sjohnlev 
352843e1988Sjohnlev 	return (ref);
353843e1988Sjohnlev }
354843e1988Sjohnlev 
355843e1988Sjohnlev void
gnttab_grant_foreign_transfer_ref(grant_ref_t ref,domid_t domid,pfn_t pfn)356a576ab5bSrab gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid, pfn_t pfn)
357843e1988Sjohnlev {
358843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(ref));
359843e1988Sjohnlev 
360a576ab5bSrab 	shared[ref].frame = pfn;
361843e1988Sjohnlev 	shared[ref].domid = domid;
362843e1988Sjohnlev 	membar_producer();
363843e1988Sjohnlev 	shared[ref].flags = GTF_accept_transfer;
364843e1988Sjohnlev }
365843e1988Sjohnlev 
366843e1988Sjohnlev gnttab_frame_t
gnttab_end_foreign_transfer_ref(grant_ref_t ref)367843e1988Sjohnlev gnttab_end_foreign_transfer_ref(grant_ref_t ref)
368843e1988Sjohnlev {
369843e1988Sjohnlev 	gnttab_frame_t frame;
370843e1988Sjohnlev 	uint16_t flags;
371843e1988Sjohnlev 
372843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(ref));
373843e1988Sjohnlev 
374843e1988Sjohnlev 	/*
375843e1988Sjohnlev 	 * If a transfer is not even yet started, try to reclaim the grant
376843e1988Sjohnlev 	 * reference and return failure (== 0).
377843e1988Sjohnlev 	 */
378843e1988Sjohnlev 	while (!((flags = shared[ref].flags) & GTF_transfer_committed)) {
379a576ab5bSrab 		if (CMPXCHG(&shared[ref].flags, flags, 0) == flags)
380843e1988Sjohnlev 			return (0);
381843e1988Sjohnlev 		(void) HYPERVISOR_yield();
382843e1988Sjohnlev 	}
383843e1988Sjohnlev 
384843e1988Sjohnlev 	/* If a transfer is in progress then wait until it is completed. */
385843e1988Sjohnlev 	while (!(flags & GTF_transfer_completed)) {
386843e1988Sjohnlev 		flags = shared[ref].flags;
387843e1988Sjohnlev 		(void) HYPERVISOR_yield();
388843e1988Sjohnlev 	}
389843e1988Sjohnlev 
390843e1988Sjohnlev 	/* Read the frame number /after/ reading completion status. */
391843e1988Sjohnlev 	membar_consumer();
392843e1988Sjohnlev 	frame = shared[ref].frame;
393843e1988Sjohnlev 	ASSERT(frame != 0);
394843e1988Sjohnlev 
395843e1988Sjohnlev 	return (frame);
396843e1988Sjohnlev }
397843e1988Sjohnlev 
398843e1988Sjohnlev gnttab_frame_t
gnttab_end_foreign_transfer(grant_ref_t ref)399843e1988Sjohnlev gnttab_end_foreign_transfer(grant_ref_t ref)
400843e1988Sjohnlev {
401843e1988Sjohnlev 	gnttab_frame_t frame;
402843e1988Sjohnlev 
403843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(ref));
404843e1988Sjohnlev 
405843e1988Sjohnlev 	frame = gnttab_end_foreign_transfer_ref(ref);
406843e1988Sjohnlev 	put_free_entry(ref);
407843e1988Sjohnlev 	return (frame);
408843e1988Sjohnlev }
409843e1988Sjohnlev 
410843e1988Sjohnlev void
gnttab_free_grant_reference(grant_ref_t ref)411843e1988Sjohnlev gnttab_free_grant_reference(grant_ref_t ref)
412843e1988Sjohnlev {
413843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(ref));
414843e1988Sjohnlev 
415843e1988Sjohnlev 	put_free_entry(ref);
416843e1988Sjohnlev }
417843e1988Sjohnlev 
418843e1988Sjohnlev void
gnttab_free_grant_references(grant_ref_t head)419843e1988Sjohnlev gnttab_free_grant_references(grant_ref_t head)
420843e1988Sjohnlev {
421843e1988Sjohnlev 	grant_ref_t ref;
422843e1988Sjohnlev 	int count = 1;
423843e1988Sjohnlev 
424843e1988Sjohnlev 	if (head == GNTTAB_LIST_END)
425843e1988Sjohnlev 		return;
426843e1988Sjohnlev 	mutex_enter(&gnttab_list_lock);
427843e1988Sjohnlev 	ref = head;
428a576ab5bSrab 	while (GNTTAB_ENTRY(ref) != GNTTAB_LIST_END) {
429a576ab5bSrab 		ref = GNTTAB_ENTRY(ref);
430843e1988Sjohnlev 		count++;
431843e1988Sjohnlev 	}
432a576ab5bSrab 	GNTTAB_ENTRY(ref) = gnttab_free_head;
433843e1988Sjohnlev 	gnttab_free_head = head;
434843e1988Sjohnlev 	gnttab_free_count += count;
435843e1988Sjohnlev 	check_free_callbacks();
436843e1988Sjohnlev 	mutex_exit(&gnttab_list_lock);
437843e1988Sjohnlev }
438843e1988Sjohnlev 
439843e1988Sjohnlev int
gnttab_alloc_grant_references(uint16_t count,grant_ref_t * head)440843e1988Sjohnlev gnttab_alloc_grant_references(uint16_t count, grant_ref_t *head)
441843e1988Sjohnlev {
442843e1988Sjohnlev 	int h = get_free_entries(count);
443843e1988Sjohnlev 
444843e1988Sjohnlev 	if (h == -1)
445843e1988Sjohnlev 		return (-1);
446843e1988Sjohnlev 
447843e1988Sjohnlev 	*head = h;
448843e1988Sjohnlev 
449843e1988Sjohnlev 	return (0);
450843e1988Sjohnlev }
451843e1988Sjohnlev 
452a576ab5bSrab int
gnttab_empty_grant_references(const grant_ref_t * private_head)453a576ab5bSrab gnttab_empty_grant_references(const grant_ref_t *private_head)
454a576ab5bSrab {
455a576ab5bSrab 	return (*private_head == GNTTAB_LIST_END);
456a576ab5bSrab }
457a576ab5bSrab 
458843e1988Sjohnlev int
gnttab_claim_grant_reference(grant_ref_t * private_head)459843e1988Sjohnlev gnttab_claim_grant_reference(grant_ref_t *private_head)
460843e1988Sjohnlev {
461843e1988Sjohnlev 	grant_ref_t g = *private_head;
462843e1988Sjohnlev 
463843e1988Sjohnlev 	if (g == GNTTAB_LIST_END)
464843e1988Sjohnlev 		return (-1);
465a576ab5bSrab 	*private_head = GNTTAB_ENTRY(g);
466843e1988Sjohnlev 	return (g);
467843e1988Sjohnlev }
468843e1988Sjohnlev 
469843e1988Sjohnlev void
gnttab_release_grant_reference(grant_ref_t * private_head,grant_ref_t release)470843e1988Sjohnlev gnttab_release_grant_reference(grant_ref_t *private_head, grant_ref_t release)
471843e1988Sjohnlev {
472843e1988Sjohnlev 	ASSERT(VALID_GRANT_REF(release));
473843e1988Sjohnlev 
474a576ab5bSrab 	GNTTAB_ENTRY(release) = *private_head;
475843e1988Sjohnlev 	*private_head = release;
476843e1988Sjohnlev }
477843e1988Sjohnlev 
478843e1988Sjohnlev void
gnttab_request_free_callback(struct gnttab_free_callback * callback,void (* fn)(void *),void * arg,uint16_t count)479843e1988Sjohnlev gnttab_request_free_callback(struct gnttab_free_callback *callback,
480843e1988Sjohnlev 	void (*fn)(void *), void *arg, uint16_t count)
481843e1988Sjohnlev {
482843e1988Sjohnlev 	mutex_enter(&gnttab_list_lock);
483843e1988Sjohnlev 	if (callback->next)
484843e1988Sjohnlev 		goto out;
485843e1988Sjohnlev 	callback->fn = fn;
486843e1988Sjohnlev 	callback->arg = arg;
487843e1988Sjohnlev 	callback->count = count;
488843e1988Sjohnlev 	callback->next = gnttab_free_callback_list;
489843e1988Sjohnlev 	gnttab_free_callback_list = callback;
490843e1988Sjohnlev 	check_free_callbacks();
491843e1988Sjohnlev out:
492843e1988Sjohnlev 	mutex_exit(&gnttab_list_lock);
493843e1988Sjohnlev }
494843e1988Sjohnlev 
495a576ab5bSrab void
gnttab_cancel_free_callback(struct gnttab_free_callback * callback)496a576ab5bSrab gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
497a576ab5bSrab {
498a576ab5bSrab 	struct gnttab_free_callback **pcb;
499a576ab5bSrab 
500a576ab5bSrab 	mutex_enter(&gnttab_list_lock);
501a576ab5bSrab 	for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
502a576ab5bSrab 		if (*pcb == callback) {
503a576ab5bSrab 			*pcb = callback->next;
504a576ab5bSrab 			break;
505a576ab5bSrab 		}
506a576ab5bSrab 	}
507a576ab5bSrab 	mutex_exit(&gnttab_list_lock);
508a576ab5bSrab }
509a576ab5bSrab 
510a576ab5bSrab static gnttab_frame_t *
gnttab_setup(gnttab_setup_table_t * pset)511a576ab5bSrab gnttab_setup(gnttab_setup_table_t *pset)
512a576ab5bSrab {
513a576ab5bSrab 	gnttab_frame_t *frames;
514a576ab5bSrab 
515a576ab5bSrab 	frames = kmem_alloc(pset->nr_frames * sizeof (gnttab_frame_t),
516a576ab5bSrab 	    KM_SLEEP);
517a576ab5bSrab 
518a576ab5bSrab 	/*LINTED: constant in conditional context*/
519a576ab5bSrab 	set_xen_guest_handle(pset->frame_list, frames);
520a576ab5bSrab 
521*6eb35ee7Srab #ifndef XPV_HVM_DRIVER
522a576ab5bSrab 	/*
523a576ab5bSrab 	 * Take pset->nr_frames pages of grant table space from
524a576ab5bSrab 	 * the hypervisor and map it
525a576ab5bSrab 	 */
526a576ab5bSrab 	if ((HYPERVISOR_grant_table_op(GNTTABOP_setup_table, pset, 1) != 0) ||
527a576ab5bSrab 	    (pset->status != 0)) {
528a576ab5bSrab 		cmn_err(CE_PANIC, "Grant Table setup failed");
529a576ab5bSrab 	}
530*6eb35ee7Srab #endif
531a576ab5bSrab 
532a576ab5bSrab 	return (frames);
533a576ab5bSrab }
534551bc2a6Smrj 
535a576ab5bSrab #ifdef XPV_HVM_DRIVER
536551bc2a6Smrj static void
gnttab_map(void)537551bc2a6Smrj gnttab_map(void)
538551bc2a6Smrj {
539551bc2a6Smrj 	struct xen_add_to_physmap xatp;
540551bc2a6Smrj 	caddr_t va;
541551bc2a6Smrj 	pfn_t pfn;
542551bc2a6Smrj 	int i;
543551bc2a6Smrj 
544551bc2a6Smrj 	va = (caddr_t)shared;
545a576ab5bSrab 	for (i = 0; i < max_nr_grant_frames(); i++) {
546a576ab5bSrab 		if ((pfn = hat_getpfnum(kas.a_hat, va)) == PFN_INVALID)
547a576ab5bSrab 			cmn_err(CE_PANIC, "gnttab_map: Invalid pfn");
548551bc2a6Smrj 
549551bc2a6Smrj 		xatp.domid = DOMID_SELF;
550551bc2a6Smrj 		xatp.idx = i;
551551bc2a6Smrj 		xatp.space = XENMAPSPACE_grant_table;
552551bc2a6Smrj 		xatp.gpfn = pfn;
553551bc2a6Smrj 		hat_unload(kas.a_hat, va, MMU_PAGESIZE, HAT_UNLOAD);
554a576ab5bSrab 		/*
555a576ab5bSrab 		 * This call replaces the existing machine page backing
556a576ab5bSrab 		 * the given gpfn with the page from the allocated grant
557a576ab5bSrab 		 * table at index idx. The existing machine page is
558a576ab5bSrab 		 * returned to the free list.
559a576ab5bSrab 		 */
560551bc2a6Smrj 		if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp) != 0)
561551bc2a6Smrj 			panic("Couldn't map grant table");
562551bc2a6Smrj 		hat_devload(kas.a_hat, va, MMU_PAGESIZE, pfn,
5635b1df0b5Sjohnlev 		    PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
564551bc2a6Smrj 		    HAT_LOAD | HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
565551bc2a6Smrj 		va += MMU_PAGESIZE;
566551bc2a6Smrj 	}
567551bc2a6Smrj }
568a576ab5bSrab #endif /* XPV_HVM_DRIVER */
569551bc2a6Smrj 
570551bc2a6Smrj void
gnttab_init(void)571551bc2a6Smrj gnttab_init(void)
572551bc2a6Smrj {
573a576ab5bSrab 	gnttab_setup_table_t set;
574551bc2a6Smrj 	int i;
5751213c83cSgarypen 	uint_t nr_init_grefs, max_nr_glist_frames, nr_glist_frames;
576a576ab5bSrab 	gnttab_frame_t *frames;
577551bc2a6Smrj 
578a576ab5bSrab 	/*
579a576ab5bSrab 	 * gnttab_init() should only be invoked once.
580a576ab5bSrab 	 */
581a576ab5bSrab 	mutex_enter(&gnttab_list_lock);
582a576ab5bSrab 	ASSERT(nr_grant_frames == 0);
583a576ab5bSrab 	nr_grant_frames = 1;
584a576ab5bSrab 	mutex_exit(&gnttab_list_lock);
585551bc2a6Smrj 
586a576ab5bSrab 	max_nr_glist_frames = (max_nr_grant_frames() *
5871213c83cSgarypen 	    GREFS_PER_GRANT_FRAME / RPP);
588551bc2a6Smrj 
589a576ab5bSrab 	set.dom = DOMID_SELF;
590a576ab5bSrab 	set.nr_frames = max_nr_grant_frames();
591a576ab5bSrab 	frames = gnttab_setup(&set);
592551bc2a6Smrj 
593a576ab5bSrab #ifdef XPV_HVM_DRIVER
594a576ab5bSrab 	shared = (grant_entry_t *)xen_alloc_pages(set.nr_frames);
595551bc2a6Smrj 
596551bc2a6Smrj 	gnttab_map();
597551bc2a6Smrj #else /* XPV_HVM_DRIVER */
598a576ab5bSrab 	shared = vmem_xalloc(heap_arena, set.nr_frames * MMU_PAGESIZE,
599a576ab5bSrab 	    MMU_PAGESIZE, 0, 0, 0, 0, VM_SLEEP);
600a576ab5bSrab 	for (i = 0; i < set.nr_frames; i++) {
601a576ab5bSrab 		hat_devload(kas.a_hat, (caddr_t)GT_PGADDR(i), PAGESIZE,
6025b1df0b5Sjohnlev 		    xen_assign_pfn(frames[i]),
6035b1df0b5Sjohnlev 		    PROT_READ | PROT_WRITE | HAT_STORECACHING_OK,
604a576ab5bSrab 		    HAT_LOAD_LOCK);
605a576ab5bSrab 	}
606a576ab5bSrab #endif
607551bc2a6Smrj 
608a576ab5bSrab 	gnttab_list = kmem_alloc(max_nr_glist_frames * sizeof (grant_ref_t *),
609a576ab5bSrab 	    KM_SLEEP);
610843e1988Sjohnlev 
6111213c83cSgarypen 	nr_glist_frames = (nr_grant_frames * GREFS_PER_GRANT_FRAME + RPP - 1)
6121213c83cSgarypen 	    / RPP;
6131213c83cSgarypen 	for (i = 0; i < nr_glist_frames; i++) {
614a576ab5bSrab 		gnttab_list[i] = kmem_alloc(PAGESIZE, KM_SLEEP);
615843e1988Sjohnlev 	}
616843e1988Sjohnlev 
617a576ab5bSrab 	kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t));
618843e1988Sjohnlev 
619a576ab5bSrab 	nr_init_grefs = nr_grant_frames * GREFS_PER_GRANT_FRAME;
620843e1988Sjohnlev 
621a576ab5bSrab 	for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
622a576ab5bSrab 		GNTTAB_ENTRY(i) = i + 1;
623843e1988Sjohnlev 
624a576ab5bSrab 	GNTTAB_ENTRY(nr_init_grefs - 1) = GNTTAB_LIST_END;
625a576ab5bSrab 	gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
626a576ab5bSrab 	gnttab_free_head  = NR_RESERVED_ENTRIES;
627843e1988Sjohnlev }
628843e1988Sjohnlev 
629843e1988Sjohnlev void
gnttab_resume(void)630843e1988Sjohnlev gnttab_resume(void)
631843e1988Sjohnlev {
632843e1988Sjohnlev 	gnttab_setup_table_t set;
633843e1988Sjohnlev 	int i;
634a576ab5bSrab 	gnttab_frame_t *frames;
635a576ab5bSrab 	uint_t available_frames = max_nr_grant_frames();
636843e1988Sjohnlev 
637a576ab5bSrab 	if (available_frames < nr_grant_frames) {
638a576ab5bSrab 		cmn_err(CE_PANIC, "Hypervisor does not have enough grant "
639a576ab5bSrab 		    "frames: required(%u), available(%u)", nr_grant_frames,
640a576ab5bSrab 		    available_frames);
641843e1988Sjohnlev 	}
642843e1988Sjohnlev 
643a576ab5bSrab #ifdef XPV_HVM_DRIVER
644a576ab5bSrab 	gnttab_map();
645a576ab5bSrab #endif /* XPV_HVM_DRIVER */
646a576ab5bSrab 
647a576ab5bSrab 	set.dom = DOMID_SELF;
648a576ab5bSrab 	set.nr_frames = available_frames;
649a576ab5bSrab 	frames = gnttab_setup(&set);
650a576ab5bSrab 
651a576ab5bSrab 	for (i = 0; i < available_frames; i++) {
652843e1988Sjohnlev 		(void) HYPERVISOR_update_va_mapping(GT_PGADDR(i),
653843e1988Sjohnlev 		    FRAME_TO_MA(frames[i]) | PT_VALID | PT_WRITABLE,
654843e1988Sjohnlev 		    UVMF_INVLPG | UVMF_ALL);
655843e1988Sjohnlev 	}
656a576ab5bSrab 	kmem_free(frames, set.nr_frames * sizeof (gnttab_frame_t));
657843e1988Sjohnlev }
658843e1988Sjohnlev 
659843e1988Sjohnlev void
gnttab_suspend(void)660843e1988Sjohnlev gnttab_suspend(void)
661843e1988Sjohnlev {
662843e1988Sjohnlev 	int i;
663843e1988Sjohnlev 
664843e1988Sjohnlev 	/*
665843e1988Sjohnlev 	 * clear grant table mappings before suspending
666843e1988Sjohnlev 	 */
667a576ab5bSrab 	for (i = 0; i < max_nr_grant_frames(); i++) {
668843e1988Sjohnlev 		(void) HYPERVISOR_update_va_mapping(GT_PGADDR(i),
669843e1988Sjohnlev 		    0, UVMF_INVLPG);
670843e1988Sjohnlev 	}
671843e1988Sjohnlev }
672843e1988Sjohnlev 
673843e1988Sjohnlev /*
674843e1988Sjohnlev  * Local variables:
675843e1988Sjohnlev  *  c-file-style: "solaris"
676843e1988Sjohnlev  *  indent-tabs-mode: t
677843e1988Sjohnlev  *  c-indent-level: 8
678843e1988Sjohnlev  *  c-basic-offset: 8
679843e1988Sjohnlev  *  tab-width: 8
680843e1988Sjohnlev  * End:
681843e1988Sjohnlev  */
682