xref: /illumos-gate/usr/src/uts/sun4v/io/ldc_shm.c (revision a704fb2d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * sun4v LDC Link Layer Shared Memory Routines
28  */
29 #include <sys/types.h>
30 #include <sys/kmem.h>
31 #include <sys/cmn_err.h>
32 #include <sys/ksynch.h>
33 #include <sys/debug.h>
34 #include <sys/cyclic.h>
35 #include <sys/machsystm.h>
36 #include <sys/vm.h>
37 #include <sys/machcpuvar.h>
38 #include <sys/mmu.h>
39 #include <sys/pte.h>
40 #include <vm/hat.h>
41 #include <vm/as.h>
42 #include <vm/hat_sfmmu.h>
43 #include <sys/vm_machparam.h>
44 #include <vm/seg_kmem.h>
45 #include <vm/seg_kpm.h>
46 #include <sys/hypervisor_api.h>
47 #include <sys/ldc.h>
48 #include <sys/ldc_impl.h>
49 
50 /* LDC variables used by shared memory routines */
51 extern ldc_soft_state_t *ldcssp;
52 extern int ldc_max_retries;
53 extern clock_t ldc_delay;
54 
55 #ifdef DEBUG
56 extern int ldcdbg;
57 #endif
58 
59 /* LDC internal functions used by shared memory routines */
60 extern void i_ldc_reset(ldc_chan_t *ldcp, boolean_t force_reset);
61 extern int i_ldc_h2v_error(int h_error);
62 
63 #ifdef DEBUG
64 extern void ldcdebug(int64_t id, const char *fmt, ...);
65 #endif
66 
67 /* Memory synchronization internal functions */
68 static int i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle,
69     uint8_t direction, uint64_t offset, size_t size);
70 static int i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle,
71     uint8_t direction, uint64_t start, uint64_t end);
72 static int i_ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie,
73     uint32_t ccount, uint8_t mtype, uint8_t perm, caddr_t *vaddr,
74     caddr_t *raddr);
75 static int i_ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr,
76     size_t len, uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie,
77     uint32_t *ccount);
78 
79 /*
80  * LDC framework supports mapping remote domain's memory
81  * either directly or via shadow memory pages. Default
82  * support is currently implemented via shadow copy.
83  * Direct map can be enabled by setting 'ldc_shmem_enabled'
84  */
85 int ldc_shmem_enabled = 1;
86 
87 /*
88  * Use of directly mapped shared memory for LDC descriptor
89  * rings is permitted if this variable is non-zero.
90  */
91 int ldc_dring_shmem_enabled = 1;
92 
93 /*
94  * The major and minor versions required to use directly
95  * mapped shared memory for LDC descriptor rings. The
96  * ldc_dring_shmem_hv_force variable, if set to a non-zero
97  * value, overrides the hypervisor API version check.
98  */
99 static int ldc_dring_shmem_hv_major = 1;
100 static int ldc_dring_shmem_hv_minor = 1;
101 static int ldc_dring_shmem_hv_force = 0;
102 
103 /*
104  * The results of the hypervisor service group API check.
105  * A non-zero value indicates the HV includes support for
106  * descriptor ring shared memory.
107  */
108 static int ldc_dring_shmem_hv_ok = 0;
109 
110 /*
111  * Pages exported for remote access over each channel is
112  * maintained in a table registered with the Hypervisor.
113  * The default number of entries in the table is set to
114  * 'ldc_mtbl_entries'.
115  */
116 uint64_t ldc_maptable_entries = LDC_MTBL_ENTRIES;
117 
118 #define	IDX2COOKIE(idx, pg_szc, pg_shift)				\
119 	(((pg_szc) << LDC_COOKIE_PGSZC_SHIFT) | ((idx) << (pg_shift)))
120 
121 /*
122  * Pages imported over each channel are maintained in a global (per-guest)
123  * mapin table. Starting with HV LDC API version 1.2, HV supports APIs to
124  * obtain information about the total size of the memory that can be direct
125  * mapped through this mapin table. The minimum size of the mapin area that we
126  * expect is defined below.
127  */
128 #define	GIGABYTE		((uint64_t)(1 << 30))
129 uint64_t ldc_mapin_size_min = GIGABYTE;
130 
131 /* HV LDC API version that supports mapin size info */
132 #define	LDC_MAPIN_VER_MAJOR	1
133 #define	LDC_MAPIN_VER_MINOR	2
134 
135 /*
136  * Sets ldc_dring_shmem_hv_ok to a non-zero value if the HV LDC
137  * API version supports directly mapped shared memory or if it has
138  * been explicitly enabled via ldc_dring_shmem_hv_force.
139  */
140 void
i_ldc_mem_set_hsvc_vers(uint64_t major,uint64_t minor)141 i_ldc_mem_set_hsvc_vers(uint64_t major, uint64_t minor)
142 {
143 	if ((major == ldc_dring_shmem_hv_major &&
144 	    minor >= ldc_dring_shmem_hv_minor) ||
145 	    (major > ldc_dring_shmem_hv_major) ||
146 	    (ldc_dring_shmem_hv_force != 0)) {
147 		ldc_dring_shmem_hv_ok = 1;
148 	}
149 }
150 
151 /*
152  * initialize mapin table.
153  */
154 void
i_ldc_init_mapin(ldc_soft_state_t * ldcssp,uint64_t major,uint64_t minor)155 i_ldc_init_mapin(ldc_soft_state_t *ldcssp, uint64_t major, uint64_t minor)
156 {
157 	int		rv;
158 	uint64_t	sz;
159 	uint64_t	table_type = LDC_MAPIN_TYPE_REGULAR;
160 
161 	/* set mapin size to default. */
162 	ldcssp->mapin_size = LDC_DIRECT_MAP_SIZE_DEFAULT;
163 
164 	/* Check if the HV supports mapin size API. */
165 	if ((major == LDC_MAPIN_VER_MAJOR &&
166 	    minor < LDC_MAPIN_VER_MINOR) ||
167 	    (major < LDC_MAPIN_VER_MAJOR)) {
168 		/* Older version of HV. */
169 		return;
170 	}
171 
172 	/* Get info about the mapin size supported by HV */
173 	rv = hv_ldc_mapin_size_max(table_type, &sz);
174 	if (rv != 0) {
175 		cmn_err(CE_NOTE, "Failed to get mapin information\n");
176 		return;
177 	}
178 
179 	/* Save the table size */
180 	ldcssp->mapin_size = sz;
181 
182 	D1(DBG_ALL_LDCS, "%s: mapin_size read from HV is (0x%llx)\n",
183 	    __func__, sz);
184 }
185 
186 /*
187  * Allocate a memory handle for the channel and link it into the list
188  * Also choose which memory table to use if this is the first handle
189  * being assigned to this channel
190  */
191 int
ldc_mem_alloc_handle(ldc_handle_t handle,ldc_mem_handle_t * mhandle)192 ldc_mem_alloc_handle(ldc_handle_t handle, ldc_mem_handle_t *mhandle)
193 {
194 	ldc_chan_t	*ldcp;
195 	ldc_mhdl_t	*mhdl;
196 
197 	if (handle == 0) {
198 		DWARN(DBG_ALL_LDCS,
199 		    "ldc_mem_alloc_handle: invalid channel handle\n");
200 		return (EINVAL);
201 	}
202 	ldcp = (ldc_chan_t *)handle;
203 
204 	mutex_enter(&ldcp->lock);
205 
206 	/* check to see if channel is initalized */
207 	if ((ldcp->tstate & ~TS_IN_RESET) < TS_INIT) {
208 		DWARN(ldcp->id,
209 		    "ldc_mem_alloc_handle: (0x%llx) channel not initialized\n",
210 		    ldcp->id);
211 		mutex_exit(&ldcp->lock);
212 		return (EINVAL);
213 	}
214 
215 	/* allocate handle for channel */
216 	mhdl = kmem_cache_alloc(ldcssp->memhdl_cache, KM_SLEEP);
217 
218 	/* initialize the lock */
219 	mutex_init(&mhdl->lock, NULL, MUTEX_DRIVER, NULL);
220 
221 	mhdl->myshadow = B_FALSE;
222 	mhdl->memseg = NULL;
223 	mhdl->ldcp = ldcp;
224 	mhdl->status = LDC_UNBOUND;
225 
226 	/* insert memory handle (@ head) into list */
227 	if (ldcp->mhdl_list == NULL) {
228 		ldcp->mhdl_list = mhdl;
229 		mhdl->next = NULL;
230 	} else {
231 		/* insert @ head */
232 		mhdl->next = ldcp->mhdl_list;
233 		ldcp->mhdl_list = mhdl;
234 	}
235 
236 	/* return the handle */
237 	*mhandle = (ldc_mem_handle_t)mhdl;
238 
239 	mutex_exit(&ldcp->lock);
240 
241 	D1(ldcp->id, "ldc_mem_alloc_handle: (0x%llx) allocated handle 0x%llx\n",
242 	    ldcp->id, mhdl);
243 
244 	return (0);
245 }
246 
247 /*
248  * Free memory handle for the channel and unlink it from the list
249  */
250 int
ldc_mem_free_handle(ldc_mem_handle_t mhandle)251 ldc_mem_free_handle(ldc_mem_handle_t mhandle)
252 {
253 	ldc_mhdl_t	*mhdl, *phdl;
254 	ldc_chan_t	*ldcp;
255 
256 	if (mhandle == 0) {
257 		DWARN(DBG_ALL_LDCS,
258 		    "ldc_mem_free_handle: invalid memory handle\n");
259 		return (EINVAL);
260 	}
261 	mhdl = (ldc_mhdl_t *)mhandle;
262 
263 	mutex_enter(&mhdl->lock);
264 
265 	ldcp = mhdl->ldcp;
266 
267 	if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) {
268 		DWARN(ldcp->id,
269 		    "ldc_mem_free_handle: cannot free, 0x%llx hdl bound\n",
270 		    mhdl);
271 		mutex_exit(&mhdl->lock);
272 		return (EINVAL);
273 	}
274 	mutex_exit(&mhdl->lock);
275 
276 	mutex_enter(&ldcp->mlist_lock);
277 
278 	phdl = ldcp->mhdl_list;
279 
280 	/* first handle */
281 	if (phdl == mhdl) {
282 		ldcp->mhdl_list = mhdl->next;
283 		mutex_destroy(&mhdl->lock);
284 		kmem_cache_free(ldcssp->memhdl_cache, mhdl);
285 
286 		D1(ldcp->id,
287 		    "ldc_mem_free_handle: (0x%llx) freed handle 0x%llx\n",
288 		    ldcp->id, mhdl);
289 	} else {
290 		/* walk the list - unlink and free */
291 		while (phdl != NULL) {
292 			if (phdl->next == mhdl) {
293 				phdl->next = mhdl->next;
294 				mutex_destroy(&mhdl->lock);
295 				kmem_cache_free(ldcssp->memhdl_cache, mhdl);
296 				D1(ldcp->id,
297 				    "ldc_mem_free_handle: (0x%llx) freed "
298 				    "handle 0x%llx\n", ldcp->id, mhdl);
299 				break;
300 			}
301 			phdl = phdl->next;
302 		}
303 	}
304 
305 	if (phdl == NULL) {
306 		DWARN(ldcp->id,
307 		    "ldc_mem_free_handle: invalid handle 0x%llx\n", mhdl);
308 		mutex_exit(&ldcp->mlist_lock);
309 		return (EINVAL);
310 	}
311 
312 	mutex_exit(&ldcp->mlist_lock);
313 
314 	return (0);
315 }
316 
317 /*
318  * Bind a memory handle to a virtual address.
319  * The virtual address is converted to the corresponding real addresses.
320  * Returns pointer to the first ldc_mem_cookie and the total number
321  * of cookies for this virtual address. Other cookies can be obtained
322  * using the ldc_mem_nextcookie() call. If the pages are stored in
323  * consecutive locations in the table, a single cookie corresponding to
324  * the first location is returned. The cookie size spans all the entries.
325  *
326  * If the VA corresponds to a page that is already being exported, reuse
327  * the page and do not export it again. Bump the page's use count.
328  */
329 int
ldc_mem_bind_handle(ldc_mem_handle_t mhandle,caddr_t vaddr,size_t len,uint8_t mtype,uint8_t perm,ldc_mem_cookie_t * cookie,uint32_t * ccount)330 ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, size_t len,
331     uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount)
332 {
333 	/*
334 	 * Check if direct shared memory map is enabled, if not change
335 	 * the mapping type to SHADOW_MAP.
336 	 */
337 	if (ldc_shmem_enabled == 0)
338 		mtype = LDC_SHADOW_MAP;
339 
340 	return (i_ldc_mem_bind_handle(mhandle, vaddr, len, mtype, perm,
341 	    cookie, ccount));
342 }
343 
344 static int
i_ldc_mem_bind_handle(ldc_mem_handle_t mhandle,caddr_t vaddr,size_t len,uint8_t mtype,uint8_t perm,ldc_mem_cookie_t * cookie,uint32_t * ccount)345 i_ldc_mem_bind_handle(ldc_mem_handle_t mhandle, caddr_t vaddr, size_t len,
346     uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount)
347 {
348 	ldc_mhdl_t	*mhdl;
349 	ldc_chan_t	*ldcp;
350 	ldc_mtbl_t	*mtbl;
351 	ldc_memseg_t	*memseg;
352 	ldc_mte_t	tmp_mte;
353 	uint64_t	index, prev_index = 0;
354 	int64_t		cookie_idx;
355 	uintptr_t	raddr, ra_aligned;
356 	uint64_t	psize, poffset, v_offset;
357 	uint64_t	pg_shift, pg_size, pg_size_code, pg_mask;
358 	pgcnt_t		npages;
359 	caddr_t		v_align, addr;
360 	int		i, rv;
361 
362 	if (mhandle == 0) {
363 		DWARN(DBG_ALL_LDCS,
364 		    "ldc_mem_bind_handle: invalid memory handle\n");
365 		return (EINVAL);
366 	}
367 	mhdl = (ldc_mhdl_t *)mhandle;
368 	ldcp = mhdl->ldcp;
369 
370 	/* clear count */
371 	*ccount = 0;
372 
373 	mutex_enter(&mhdl->lock);
374 
375 	if (mhdl->status == LDC_BOUND || mhdl->memseg != NULL) {
376 		DWARN(ldcp->id,
377 		    "ldc_mem_bind_handle: (0x%x) handle already bound\n",
378 		    mhandle);
379 		mutex_exit(&mhdl->lock);
380 		return (EINVAL);
381 	}
382 
383 	/* Force address and size to be 8-byte aligned */
384 	if ((((uintptr_t)vaddr | len) & 0x7) != 0) {
385 		DWARN(ldcp->id,
386 		    "ldc_mem_bind_handle: addr/size is not 8-byte aligned\n");
387 		mutex_exit(&mhdl->lock);
388 		return (EINVAL);
389 	}
390 
391 	mutex_enter(&ldcp->lock);
392 
393 	/*
394 	 * If this channel is binding a memory handle for the
395 	 * first time allocate it a memory map table and initialize it
396 	 */
397 	if ((mtbl = ldcp->mtbl) == NULL) {
398 
399 		/* Allocate and initialize the map table structure */
400 		mtbl = kmem_zalloc(sizeof (ldc_mtbl_t), KM_SLEEP);
401 		mtbl->num_entries = mtbl->num_avail = ldc_maptable_entries;
402 		mtbl->size = ldc_maptable_entries * sizeof (ldc_mte_slot_t);
403 		mtbl->next_entry = 0;
404 		mtbl->contigmem = B_TRUE;
405 
406 		/* Allocate the table itself */
407 		mtbl->table = (ldc_mte_slot_t *)
408 		    contig_mem_alloc_align(mtbl->size, MMU_PAGESIZE);
409 		if (mtbl->table == NULL) {
410 
411 			/* allocate a page of memory using kmem_alloc */
412 			mtbl->table = kmem_alloc(MMU_PAGESIZE, KM_SLEEP);
413 			mtbl->size = MMU_PAGESIZE;
414 			mtbl->contigmem = B_FALSE;
415 			mtbl->num_entries = mtbl->num_avail =
416 			    mtbl->size / sizeof (ldc_mte_slot_t);
417 			DWARN(ldcp->id,
418 			    "ldc_mem_bind_handle: (0x%llx) reduced tbl size "
419 			    "to %lx entries\n", ldcp->id, mtbl->num_entries);
420 		}
421 
422 		/* zero out the memory */
423 		bzero(mtbl->table, mtbl->size);
424 
425 		/* initialize the lock */
426 		mutex_init(&mtbl->lock, NULL, MUTEX_DRIVER, NULL);
427 
428 		/* register table for this channel */
429 		rv = hv_ldc_set_map_table(ldcp->id,
430 		    va_to_pa(mtbl->table), mtbl->num_entries);
431 		if (rv != 0) {
432 			DWARN(DBG_ALL_LDCS,
433 			    "ldc_mem_bind_handle: (0x%lx) err %d mapping tbl",
434 			    ldcp->id, rv);
435 			if (mtbl->contigmem)
436 				contig_mem_free(mtbl->table, mtbl->size);
437 			else
438 				kmem_free(mtbl->table, mtbl->size);
439 			mutex_destroy(&mtbl->lock);
440 			kmem_free(mtbl, sizeof (ldc_mtbl_t));
441 			mutex_exit(&ldcp->lock);
442 			mutex_exit(&mhdl->lock);
443 			return (EIO);
444 		}
445 
446 		ldcp->mtbl = mtbl;
447 
448 		D1(ldcp->id,
449 		    "ldc_mem_bind_handle: (0x%llx) alloc'd map table 0x%llx\n",
450 		    ldcp->id, ldcp->mtbl->table);
451 	}
452 
453 	mutex_exit(&ldcp->lock);
454 
455 	/* FUTURE: get the page size, pgsz code, and shift */
456 	pg_size = MMU_PAGESIZE;
457 	pg_size_code = page_szc(pg_size);
458 	pg_shift = page_get_shift(pg_size_code);
459 	pg_mask = ~(pg_size - 1);
460 
461 	D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) binding "
462 	    "va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n",
463 	    ldcp->id, vaddr, pg_size, pg_size_code, pg_shift);
464 
465 	/* aligned VA and its offset */
466 	v_align = (caddr_t)(((uintptr_t)vaddr) & ~(pg_size - 1));
467 	v_offset = ((uintptr_t)vaddr) & (pg_size - 1);
468 
469 	npages = (len+v_offset)/pg_size;
470 	npages = ((len+v_offset)%pg_size == 0) ? npages : npages+1;
471 
472 	D1(ldcp->id, "ldc_mem_bind_handle: binding "
473 	    "(0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n",
474 	    ldcp->id, vaddr, v_align, v_offset, npages);
475 
476 	/* lock the memory table - exclusive access to channel */
477 	mutex_enter(&mtbl->lock);
478 
479 	if (npages > mtbl->num_avail) {
480 		D1(ldcp->id, "ldc_mem_bind_handle: (0x%llx) no table entries\n",
481 		    ldcp->id);
482 		mutex_exit(&mtbl->lock);
483 		mutex_exit(&mhdl->lock);
484 		return (ENOMEM);
485 	}
486 
487 	/* Allocate a memseg structure */
488 	memseg = mhdl->memseg =
489 	    kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP);
490 
491 	/* Allocate memory to store all pages and cookies */
492 	memseg->pages = kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP);
493 	memseg->cookies =
494 	    kmem_zalloc((sizeof (ldc_mem_cookie_t) * npages), KM_SLEEP);
495 
496 	D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) processing 0x%llx pages\n",
497 	    ldcp->id, npages);
498 
499 	addr = v_align;
500 
501 	/*
502 	 * Table slots are used in a round-robin manner. The algorithm permits
503 	 * inserting duplicate entries. Slots allocated earlier will typically
504 	 * get freed before we get back to reusing the slot.Inserting duplicate
505 	 * entries should be OK as we only lookup entries using the cookie addr
506 	 * i.e. tbl index, during export, unexport and copy operation.
507 	 *
508 	 * One implementation what was tried was to search for a duplicate
509 	 * page entry first and reuse it. The search overhead is very high and
510 	 * in the vnet case dropped the perf by almost half, 50 to 24 mbps.
511 	 * So it does make sense to avoid searching for duplicates.
512 	 *
513 	 * But during the process of searching for a free slot, if we find a
514 	 * duplicate entry we will go ahead and use it, and bump its use count.
515 	 */
516 
517 	/* index to start searching from */
518 	index = mtbl->next_entry;
519 	cookie_idx = -1;
520 
521 	tmp_mte.ll = 0;	/* initialise fields to 0 */
522 
523 	if (mtype & LDC_DIRECT_MAP) {
524 		tmp_mte.mte_r = (perm & LDC_MEM_R) ? 1 : 0;
525 		tmp_mte.mte_w = (perm & LDC_MEM_W) ? 1 : 0;
526 		tmp_mte.mte_x = (perm & LDC_MEM_X) ? 1 : 0;
527 	}
528 
529 	if (mtype & LDC_SHADOW_MAP) {
530 		tmp_mte.mte_cr = (perm & LDC_MEM_R) ? 1 : 0;
531 		tmp_mte.mte_cw = (perm & LDC_MEM_W) ? 1 : 0;
532 	}
533 
534 	if (mtype & LDC_IO_MAP) {
535 		tmp_mte.mte_ir = (perm & LDC_MEM_R) ? 1 : 0;
536 		tmp_mte.mte_iw = (perm & LDC_MEM_W) ? 1 : 0;
537 	}
538 
539 	D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll);
540 
541 	tmp_mte.mte_pgszc = pg_size_code;
542 
543 	/* initialize each mem table entry */
544 	for (i = 0; i < npages; i++) {
545 
546 		/* check if slot is available in the table */
547 		while (mtbl->table[index].entry.ll != 0) {
548 
549 			index = (index + 1) % mtbl->num_entries;
550 
551 			if (index == mtbl->next_entry) {
552 				/* we have looped around */
553 				DWARN(DBG_ALL_LDCS,
554 				    "ldc_mem_bind_handle: (0x%llx) cannot find "
555 				    "entry\n", ldcp->id);
556 				*ccount = 0;
557 
558 				/* NOTE: free memory, remove previous entries */
559 				/* this shouldnt happen as num_avail was ok */
560 
561 				mutex_exit(&mtbl->lock);
562 				mutex_exit(&mhdl->lock);
563 				return (ENOMEM);
564 			}
565 		}
566 
567 		/* get the real address */
568 		raddr = va_to_pa((void *)addr);
569 		ra_aligned = ((uintptr_t)raddr & pg_mask);
570 
571 		/* build the mte */
572 		tmp_mte.mte_rpfn = ra_aligned >> pg_shift;
573 
574 		D1(ldcp->id, "ldc_mem_bind_handle mte=0x%llx\n", tmp_mte.ll);
575 
576 		/* update entry in table */
577 		mtbl->table[index].entry = tmp_mte;
578 
579 		D2(ldcp->id, "ldc_mem_bind_handle: (0x%llx) stored MTE 0x%llx"
580 		    " into loc 0x%llx\n", ldcp->id, tmp_mte.ll, index);
581 
582 		/* calculate the size and offset for this export range */
583 		if (i == 0) {
584 			/* first page */
585 			psize = min((pg_size - v_offset), len);
586 			poffset = v_offset;
587 
588 		} else if (i == (npages - 1)) {
589 			/* last page */
590 			psize =	(((uintptr_t)(vaddr + len)) &
591 			    ((uint64_t)(pg_size-1)));
592 			if (psize == 0)
593 				psize = pg_size;
594 			poffset = 0;
595 
596 		} else {
597 			/* middle pages */
598 			psize = pg_size;
599 			poffset = 0;
600 		}
601 
602 		/* store entry for this page */
603 		memseg->pages[i].index = index;
604 		memseg->pages[i].raddr = raddr;
605 		memseg->pages[i].mte = &(mtbl->table[index]);
606 
607 		/* create the cookie */
608 		if (i == 0 || (index != prev_index + 1)) {
609 			cookie_idx++;
610 			memseg->cookies[cookie_idx].addr =
611 			    IDX2COOKIE(index, pg_size_code, pg_shift);
612 			memseg->cookies[cookie_idx].addr |= poffset;
613 			memseg->cookies[cookie_idx].size = psize;
614 
615 		} else {
616 			memseg->cookies[cookie_idx].size += psize;
617 		}
618 
619 		D1(ldcp->id, "ldc_mem_bind_handle: bound "
620 		    "(0x%llx) va=0x%llx, idx=0x%llx, "
621 		    "ra=0x%llx(sz=0x%x,off=0x%x)\n",
622 		    ldcp->id, addr, index, raddr, psize, poffset);
623 
624 		/* decrement number of available entries */
625 		mtbl->num_avail--;
626 
627 		/* increment va by page size */
628 		addr += pg_size;
629 
630 		/* increment index */
631 		prev_index = index;
632 		index = (index + 1) % mtbl->num_entries;
633 
634 		/* save the next slot */
635 		mtbl->next_entry = index;
636 	}
637 
638 	mutex_exit(&mtbl->lock);
639 
640 	/* memory handle = bound */
641 	mhdl->mtype = mtype;
642 	mhdl->perm = perm;
643 	mhdl->status = LDC_BOUND;
644 
645 	/* update memseg_t */
646 	memseg->vaddr = vaddr;
647 	memseg->raddr = memseg->pages[0].raddr;
648 	memseg->size = len;
649 	memseg->npages = npages;
650 	memseg->ncookies = cookie_idx + 1;
651 	memseg->next_cookie = (memseg->ncookies > 1) ? 1 : 0;
652 
653 	/* return count and first cookie */
654 	*ccount = memseg->ncookies;
655 	cookie->addr = memseg->cookies[0].addr;
656 	cookie->size = memseg->cookies[0].size;
657 
658 	D1(ldcp->id,
659 	    "ldc_mem_bind_handle: (0x%llx) bound 0x%llx, va=0x%llx, "
660 	    "pgs=0x%llx cookies=0x%llx\n",
661 	    ldcp->id, mhdl, vaddr, npages, memseg->ncookies);
662 
663 	mutex_exit(&mhdl->lock);
664 	return (0);
665 }
666 
667 /*
668  * Return the next cookie associated with the specified memory handle
669  */
670 int
ldc_mem_nextcookie(ldc_mem_handle_t mhandle,ldc_mem_cookie_t * cookie)671 ldc_mem_nextcookie(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie)
672 {
673 	ldc_mhdl_t	*mhdl;
674 	ldc_chan_t	*ldcp;
675 	ldc_memseg_t	*memseg;
676 
677 	if (mhandle == 0) {
678 		DWARN(DBG_ALL_LDCS,
679 		    "ldc_mem_nextcookie: invalid memory handle\n");
680 		return (EINVAL);
681 	}
682 	mhdl = (ldc_mhdl_t *)mhandle;
683 
684 	mutex_enter(&mhdl->lock);
685 
686 	ldcp = mhdl->ldcp;
687 	memseg = mhdl->memseg;
688 
689 	if (cookie == 0) {
690 		DWARN(ldcp->id,
691 		    "ldc_mem_nextcookie:(0x%llx) invalid cookie arg\n",
692 		    ldcp->id);
693 		mutex_exit(&mhdl->lock);
694 		return (EINVAL);
695 	}
696 
697 	if (memseg->next_cookie != 0) {
698 		cookie->addr = memseg->cookies[memseg->next_cookie].addr;
699 		cookie->size = memseg->cookies[memseg->next_cookie].size;
700 		memseg->next_cookie++;
701 		if (memseg->next_cookie == memseg->ncookies)
702 			memseg->next_cookie = 0;
703 
704 	} else {
705 		DWARN(ldcp->id,
706 		    "ldc_mem_nextcookie:(0x%llx) no more cookies\n", ldcp->id);
707 		cookie->addr = 0;
708 		cookie->size = 0;
709 		mutex_exit(&mhdl->lock);
710 		return (EINVAL);
711 	}
712 
713 	D1(ldcp->id,
714 	    "ldc_mem_nextcookie: (0x%llx) cookie addr=0x%llx,sz=0x%llx\n",
715 	    ldcp->id, cookie->addr, cookie->size);
716 
717 	mutex_exit(&mhdl->lock);
718 	return (0);
719 }
720 
721 /*
722  * Unbind the virtual memory region associated with the specified
723  * memory handle. Allassociated cookies are freed and the corresponding
724  * RA space is no longer exported.
725  */
726 int
ldc_mem_unbind_handle(ldc_mem_handle_t mhandle)727 ldc_mem_unbind_handle(ldc_mem_handle_t mhandle)
728 {
729 	ldc_mhdl_t	*mhdl;
730 	ldc_chan_t	*ldcp;
731 	ldc_mtbl_t	*mtbl;
732 	ldc_memseg_t	*memseg;
733 	uint64_t	cookie_addr;
734 	uint64_t	pg_shift, pg_size_code;
735 	int		i, rv, retries;
736 
737 	if (mhandle == 0) {
738 		DWARN(DBG_ALL_LDCS,
739 		    "ldc_mem_unbind_handle: invalid memory handle\n");
740 		return (EINVAL);
741 	}
742 	mhdl = (ldc_mhdl_t *)mhandle;
743 
744 	mutex_enter(&mhdl->lock);
745 
746 	if (mhdl->status == LDC_UNBOUND) {
747 		DWARN(DBG_ALL_LDCS,
748 		    "ldc_mem_unbind_handle: (0x%x) handle is not bound\n",
749 		    mhandle);
750 		mutex_exit(&mhdl->lock);
751 		return (EINVAL);
752 	}
753 
754 	ldcp = mhdl->ldcp;
755 	mtbl = ldcp->mtbl;
756 
757 	memseg = mhdl->memseg;
758 
759 	/* lock the memory table - exclusive access to channel */
760 	mutex_enter(&mtbl->lock);
761 
762 	/* undo the pages exported */
763 	for (i = 0; i < memseg->npages; i++) {
764 
765 		/* clear the entry from the table */
766 		memseg->pages[i].mte->entry.ll = 0;
767 
768 		/* check for mapped pages, revocation cookie != 0 */
769 		if (memseg->pages[i].mte->cookie) {
770 
771 			pg_size_code = page_szc(MMU_PAGESIZE);
772 			pg_shift = page_get_shift(pg_size_code);
773 			cookie_addr = IDX2COOKIE(memseg->pages[i].index,
774 			    pg_size_code, pg_shift);
775 
776 			D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) revoke "
777 			    "cookie 0x%llx, rcookie 0x%llx\n", ldcp->id,
778 			    cookie_addr, memseg->pages[i].mte->cookie);
779 
780 			retries = 0;
781 			do {
782 				rv = hv_ldc_revoke(ldcp->id, cookie_addr,
783 				    memseg->pages[i].mte->cookie);
784 
785 				if (rv != H_EWOULDBLOCK)
786 					break;
787 
788 				drv_usecwait(ldc_delay);
789 
790 			} while (retries++ < ldc_max_retries);
791 
792 			if (rv) {
793 				DWARN(ldcp->id,
794 				    "ldc_mem_unbind_handle: (0x%llx) cannot "
795 				    "revoke mapping, cookie %llx\n", ldcp->id,
796 				    cookie_addr);
797 			}
798 		}
799 
800 		mtbl->num_avail++;
801 	}
802 	mutex_exit(&mtbl->lock);
803 
804 	/* free the allocated memseg and page structures */
805 	kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages));
806 	kmem_free(memseg->cookies,
807 	    (sizeof (ldc_mem_cookie_t) * memseg->npages));
808 	kmem_cache_free(ldcssp->memseg_cache, memseg);
809 
810 	/* uninitialize the memory handle */
811 	mhdl->memseg = NULL;
812 	mhdl->status = LDC_UNBOUND;
813 
814 	D1(ldcp->id, "ldc_mem_unbind_handle: (0x%llx) unbound handle 0x%llx\n",
815 	    ldcp->id, mhdl);
816 
817 	mutex_exit(&mhdl->lock);
818 	return (0);
819 }
820 
821 /*
822  * Get information about the dring. The base address of the descriptor
823  * ring along with the type and permission are returned back.
824  */
825 int
ldc_mem_info(ldc_mem_handle_t mhandle,ldc_mem_info_t * minfo)826 ldc_mem_info(ldc_mem_handle_t mhandle, ldc_mem_info_t *minfo)
827 {
828 	ldc_mhdl_t	*mhdl;
829 
830 	if (mhandle == 0) {
831 		DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid memory handle\n");
832 		return (EINVAL);
833 	}
834 	mhdl = (ldc_mhdl_t *)mhandle;
835 
836 	if (minfo == NULL) {
837 		DWARN(DBG_ALL_LDCS, "ldc_mem_info: invalid args\n");
838 		return (EINVAL);
839 	}
840 
841 	mutex_enter(&mhdl->lock);
842 
843 	minfo->status = mhdl->status;
844 	if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED) {
845 		minfo->vaddr = mhdl->memseg->vaddr;
846 		minfo->raddr = mhdl->memseg->raddr;
847 		minfo->mtype = mhdl->mtype;
848 		minfo->perm = mhdl->perm;
849 	}
850 	mutex_exit(&mhdl->lock);
851 
852 	return (0);
853 }
854 
855 /*
856  * Copy data either from or to the client specified virtual address
857  * space to or from the exported memory associated with the cookies.
858  * The direction argument determines whether the data is read from or
859  * written to exported memory.
860  */
861 int
ldc_mem_copy(ldc_handle_t handle,caddr_t vaddr,uint64_t off,size_t * size,ldc_mem_cookie_t * cookies,uint32_t ccount,uint8_t direction)862 ldc_mem_copy(ldc_handle_t handle, caddr_t vaddr, uint64_t off, size_t *size,
863     ldc_mem_cookie_t *cookies, uint32_t ccount, uint8_t direction)
864 {
865 	ldc_chan_t	*ldcp;
866 	uint64_t	local_voff, local_valign;
867 	uint64_t	cookie_addr, cookie_size;
868 	uint64_t	pg_shift, pg_size, pg_size_code;
869 	uint64_t	export_caddr, export_poff, export_psize, export_size;
870 	uint64_t	local_ra, local_poff, local_psize;
871 	uint64_t	copy_size, copied_len = 0, total_bal = 0, idx = 0;
872 	pgcnt_t		npages;
873 	size_t		len = *size;
874 	int		i, rv = 0;
875 
876 	uint64_t	chid;
877 
878 	if (handle == 0) {
879 		DWARN(DBG_ALL_LDCS, "ldc_mem_copy: invalid channel handle\n");
880 		return (EINVAL);
881 	}
882 	ldcp = (ldc_chan_t *)handle;
883 	chid = ldcp->id;
884 
885 	/* check to see if channel is UP */
886 	if (ldcp->tstate != TS_UP) {
887 		DWARN(chid, "ldc_mem_copy: (0x%llx) channel is not UP\n",
888 		    chid);
889 		return (ECONNRESET);
890 	}
891 
892 	/* Force address and size to be 8-byte aligned */
893 	if ((((uintptr_t)vaddr | len) & 0x7) != 0) {
894 		DWARN(chid,
895 		    "ldc_mem_copy: addr/sz is not 8-byte aligned\n");
896 		return (EINVAL);
897 	}
898 
899 	/* Find the size of the exported memory */
900 	export_size = 0;
901 	for (i = 0; i < ccount; i++)
902 		export_size += cookies[i].size;
903 
904 	/* check to see if offset is valid */
905 	if (off > export_size) {
906 		DWARN(chid,
907 		    "ldc_mem_copy: (0x%llx) start offset > export mem size\n",
908 		    chid);
909 		return (EINVAL);
910 	}
911 
912 	/*
913 	 * Check to see if the export size is smaller than the size we
914 	 * are requesting to copy - if so flag an error
915 	 */
916 	if ((export_size - off) < *size) {
917 		DWARN(chid,
918 		    "ldc_mem_copy: (0x%llx) copy size > export mem size\n",
919 		    chid);
920 		return (EINVAL);
921 	}
922 
923 	total_bal = min(export_size, *size);
924 
925 	/* FUTURE: get the page size, pgsz code, and shift */
926 	pg_size = MMU_PAGESIZE;
927 	pg_size_code = page_szc(pg_size);
928 	pg_shift = page_get_shift(pg_size_code);
929 
930 	D1(chid, "ldc_mem_copy: copying data "
931 	    "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n",
932 	    chid, vaddr, pg_size, pg_size_code, pg_shift);
933 
934 	/* aligned VA and its offset */
935 	local_valign = (((uintptr_t)vaddr) & ~(pg_size - 1));
936 	local_voff = ((uintptr_t)vaddr) & (pg_size - 1);
937 
938 	npages = (len+local_voff)/pg_size;
939 	npages = ((len+local_voff)%pg_size == 0) ? npages : npages+1;
940 
941 	D1(chid,
942 	    "ldc_mem_copy: (0x%llx) v=0x%llx,val=0x%llx,off=0x%x,pgs=0x%x\n",
943 	    chid, vaddr, local_valign, local_voff, npages);
944 
945 	local_ra = va_to_pa((void *)local_valign);
946 	local_poff = local_voff;
947 	local_psize = min(len, (pg_size - local_voff));
948 
949 	len -= local_psize;
950 
951 	/*
952 	 * find the first cookie in the list of cookies
953 	 * if the offset passed in is not zero
954 	 */
955 	for (idx = 0; idx < ccount; idx++) {
956 		cookie_size = cookies[idx].size;
957 		if (off < cookie_size)
958 			break;
959 		off -= cookie_size;
960 	}
961 
962 	cookie_addr = cookies[idx].addr + off;
963 	cookie_size = cookies[idx].size - off;
964 
965 	export_caddr = cookie_addr & ~(pg_size - 1);
966 	export_poff = cookie_addr & (pg_size - 1);
967 	export_psize = min(cookie_size, (pg_size - export_poff));
968 
969 	for (;;) {
970 
971 		copy_size = min(export_psize, local_psize);
972 
973 		D1(chid,
974 		    "ldc_mem_copy:(0x%llx) dir=0x%x, caddr=0x%llx,"
975 		    " loc_ra=0x%llx, exp_poff=0x%llx, loc_poff=0x%llx,"
976 		    " exp_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx,"
977 		    " total_bal=0x%llx\n",
978 		    chid, direction, export_caddr, local_ra, export_poff,
979 		    local_poff, export_psize, local_psize, copy_size,
980 		    total_bal);
981 
982 		rv = hv_ldc_copy(chid, direction,
983 		    (export_caddr + export_poff), (local_ra + local_poff),
984 		    copy_size, &copied_len);
985 
986 		if (rv != 0) {
987 			int		error = EIO;
988 			uint64_t	rx_hd, rx_tl;
989 
990 			DWARN(chid,
991 			    "ldc_mem_copy: (0x%llx) err %d during copy\n",
992 			    (unsigned long long)chid, rv);
993 			DWARN(chid,
994 			    "ldc_mem_copy: (0x%llx) dir=0x%x, caddr=0x%lx, "
995 			    "loc_ra=0x%lx, exp_poff=0x%lx, loc_poff=0x%lx,"
996 			    " exp_psz=0x%lx, loc_psz=0x%lx, copy_sz=0x%lx,"
997 			    " copied_len=0x%lx, total_bal=0x%lx\n",
998 			    chid, direction, export_caddr, local_ra,
999 			    export_poff, local_poff, export_psize, local_psize,
1000 			    copy_size, copied_len, total_bal);
1001 
1002 			*size = *size - total_bal;
1003 
1004 			/*
1005 			 * check if reason for copy error was due to
1006 			 * a channel reset. we need to grab the lock
1007 			 * just in case we have to do a reset.
1008 			 */
1009 			mutex_enter(&ldcp->lock);
1010 			mutex_enter(&ldcp->tx_lock);
1011 
1012 			rv = hv_ldc_rx_get_state(ldcp->id,
1013 			    &rx_hd, &rx_tl, &(ldcp->link_state));
1014 			if (ldcp->link_state == LDC_CHANNEL_DOWN ||
1015 			    ldcp->link_state == LDC_CHANNEL_RESET) {
1016 				i_ldc_reset(ldcp, B_FALSE);
1017 				error = ECONNRESET;
1018 			}
1019 
1020 			mutex_exit(&ldcp->tx_lock);
1021 			mutex_exit(&ldcp->lock);
1022 
1023 			return (error);
1024 		}
1025 
1026 		ASSERT(copied_len <= copy_size);
1027 
1028 		D2(chid, "ldc_mem_copy: copied=0x%llx\n", copied_len);
1029 		export_poff += copied_len;
1030 		local_poff += copied_len;
1031 		export_psize -= copied_len;
1032 		local_psize -= copied_len;
1033 		cookie_size -= copied_len;
1034 
1035 		total_bal -= copied_len;
1036 
1037 		if (copy_size != copied_len)
1038 			continue;
1039 
1040 		if (export_psize == 0 && total_bal != 0) {
1041 
1042 			if (cookie_size == 0) {
1043 				idx++;
1044 				cookie_addr = cookies[idx].addr;
1045 				cookie_size = cookies[idx].size;
1046 
1047 				export_caddr = cookie_addr & ~(pg_size - 1);
1048 				export_poff = cookie_addr & (pg_size - 1);
1049 				export_psize =
1050 				    min(cookie_size, (pg_size-export_poff));
1051 			} else {
1052 				export_caddr += pg_size;
1053 				export_poff = 0;
1054 				export_psize = min(cookie_size, pg_size);
1055 			}
1056 		}
1057 
1058 		if (local_psize == 0 && total_bal != 0) {
1059 			local_valign += pg_size;
1060 			local_ra = va_to_pa((void *)local_valign);
1061 			local_poff = 0;
1062 			local_psize = min(pg_size, len);
1063 			len -= local_psize;
1064 		}
1065 
1066 		/* check if we are all done */
1067 		if (total_bal == 0)
1068 			break;
1069 	}
1070 
1071 
1072 	D1(chid,
1073 	    "ldc_mem_copy: (0x%llx) done copying sz=0x%llx\n",
1074 	    chid, *size);
1075 
1076 	return (0);
1077 }
1078 
1079 /*
1080  * Copy data either from or to the client specified virtual address
1081  * space to or from HV physical memory.
1082  *
1083  * The direction argument determines whether the data is read from or
1084  * written to HV memory. direction values are LDC_COPY_IN/OUT similar
1085  * to the ldc_mem_copy interface
1086  */
1087 int
ldc_mem_rdwr_cookie(ldc_handle_t handle,caddr_t vaddr,size_t * size,caddr_t paddr,uint8_t direction)1088 ldc_mem_rdwr_cookie(ldc_handle_t handle, caddr_t vaddr, size_t *size,
1089     caddr_t paddr, uint8_t direction)
1090 {
1091 	ldc_chan_t	*ldcp;
1092 	uint64_t	local_voff, local_valign;
1093 	uint64_t	pg_shift, pg_size, pg_size_code;
1094 	uint64_t	target_pa, target_poff, target_psize, target_size;
1095 	uint64_t	local_ra, local_poff, local_psize;
1096 	uint64_t	copy_size, copied_len = 0;
1097 	pgcnt_t		npages;
1098 	size_t		len = *size;
1099 	int		rv = 0;
1100 
1101 	if (handle == 0) {
1102 		DWARN(DBG_ALL_LDCS,
1103 		    "ldc_mem_rdwr_cookie: invalid channel handle\n");
1104 		return (EINVAL);
1105 	}
1106 	ldcp = (ldc_chan_t *)handle;
1107 
1108 	mutex_enter(&ldcp->lock);
1109 
1110 	/* check to see if channel is UP */
1111 	if (ldcp->tstate != TS_UP) {
1112 		DWARN(ldcp->id,
1113 		    "ldc_mem_rdwr_cookie: (0x%llx) channel is not UP\n",
1114 		    ldcp->id);
1115 		mutex_exit(&ldcp->lock);
1116 		return (ECONNRESET);
1117 	}
1118 
1119 	/* Force address and size to be 8-byte aligned */
1120 	if ((((uintptr_t)vaddr | len) & 0x7) != 0) {
1121 		DWARN(ldcp->id,
1122 		    "ldc_mem_rdwr_cookie: addr/size is not 8-byte aligned\n");
1123 		mutex_exit(&ldcp->lock);
1124 		return (EINVAL);
1125 	}
1126 
1127 	target_size = *size;
1128 
1129 	/* FUTURE: get the page size, pgsz code, and shift */
1130 	pg_size = MMU_PAGESIZE;
1131 	pg_size_code = page_szc(pg_size);
1132 	pg_shift = page_get_shift(pg_size_code);
1133 
1134 	D1(ldcp->id, "ldc_mem_rdwr_cookie: copying data "
1135 	    "(0x%llx) va 0x%llx pgsz=0x%llx, pgszc=0x%llx, pg_shift=0x%llx\n",
1136 	    ldcp->id, vaddr, pg_size, pg_size_code, pg_shift);
1137 
1138 	/* aligned VA and its offset */
1139 	local_valign = ((uintptr_t)vaddr) & ~(pg_size - 1);
1140 	local_voff = ((uintptr_t)vaddr) & (pg_size - 1);
1141 
1142 	npages = (len + local_voff) / pg_size;
1143 	npages = ((len + local_voff) % pg_size == 0) ? npages : npages+1;
1144 
1145 	D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) v=0x%llx, "
1146 	    "val=0x%llx,off=0x%x,pgs=0x%x\n",
1147 	    ldcp->id, vaddr, local_valign, local_voff, npages);
1148 
1149 	local_ra = va_to_pa((void *)local_valign);
1150 	local_poff = local_voff;
1151 	local_psize = min(len, (pg_size - local_voff));
1152 
1153 	len -= local_psize;
1154 
1155 	target_pa = ((uintptr_t)paddr) & ~(pg_size - 1);
1156 	target_poff = ((uintptr_t)paddr) & (pg_size - 1);
1157 	target_psize = pg_size - target_poff;
1158 
1159 	for (;;) {
1160 
1161 		copy_size = min(target_psize, local_psize);
1162 
1163 		D1(ldcp->id,
1164 		    "ldc_mem_rdwr_cookie: (0x%llx) dir=0x%x, tar_pa=0x%llx,"
1165 		    " loc_ra=0x%llx, tar_poff=0x%llx, loc_poff=0x%llx,"
1166 		    " tar_psz=0x%llx, loc_psz=0x%llx, copy_sz=0x%llx,"
1167 		    " total_bal=0x%llx\n",
1168 		    ldcp->id, direction, target_pa, local_ra, target_poff,
1169 		    local_poff, target_psize, local_psize, copy_size,
1170 		    target_size);
1171 
1172 		rv = hv_ldc_copy(ldcp->id, direction,
1173 		    (target_pa + target_poff), (local_ra + local_poff),
1174 		    copy_size, &copied_len);
1175 
1176 		if (rv != 0) {
1177 			DWARN(DBG_ALL_LDCS,
1178 			    "ldc_mem_rdwr_cookie: (0x%lx) err %d during copy\n",
1179 			    ldcp->id, rv);
1180 			DWARN(DBG_ALL_LDCS,
1181 			    "ldc_mem_rdwr_cookie: (0x%llx) dir=%lld, "
1182 			    "tar_pa=0x%llx, loc_ra=0x%llx, tar_poff=0x%llx, "
1183 			    "loc_poff=0x%llx, tar_psz=0x%llx, loc_psz=0x%llx, "
1184 			    "copy_sz=0x%llx, total_bal=0x%llx\n",
1185 			    ldcp->id, direction, target_pa, local_ra,
1186 			    target_poff, local_poff, target_psize, local_psize,
1187 			    copy_size, target_size);
1188 
1189 			*size = *size - target_size;
1190 			mutex_exit(&ldcp->lock);
1191 			return (i_ldc_h2v_error(rv));
1192 		}
1193 
1194 		D2(ldcp->id, "ldc_mem_rdwr_cookie: copied=0x%llx\n",
1195 		    copied_len);
1196 		target_poff += copied_len;
1197 		local_poff += copied_len;
1198 		target_psize -= copied_len;
1199 		local_psize -= copied_len;
1200 
1201 		target_size -= copied_len;
1202 
1203 		if (copy_size != copied_len)
1204 			continue;
1205 
1206 		if (target_psize == 0 && target_size != 0) {
1207 			target_pa += pg_size;
1208 			target_poff = 0;
1209 			target_psize = min(pg_size, target_size);
1210 		}
1211 
1212 		if (local_psize == 0 && target_size != 0) {
1213 			local_valign += pg_size;
1214 			local_ra = va_to_pa((void *)local_valign);
1215 			local_poff = 0;
1216 			local_psize = min(pg_size, len);
1217 			len -= local_psize;
1218 		}
1219 
1220 		/* check if we are all done */
1221 		if (target_size == 0)
1222 			break;
1223 	}
1224 
1225 	mutex_exit(&ldcp->lock);
1226 
1227 	D1(ldcp->id, "ldc_mem_rdwr_cookie: (0x%llx) done copying sz=0x%llx\n",
1228 	    ldcp->id, *size);
1229 
1230 	return (0);
1231 }
1232 
1233 /*
1234  * Map an exported memory segment into the local address space. If the
1235  * memory range was exported for direct map access, a HV call is made
1236  * to allocate a RA range. If the map is done via a shadow copy, local
1237  * shadow memory is allocated and the base VA is returned in 'vaddr'. If
1238  * the mapping is a direct map then the RA is returned in 'raddr'.
1239  */
1240 int
ldc_mem_map(ldc_mem_handle_t mhandle,ldc_mem_cookie_t * cookie,uint32_t ccount,uint8_t mtype,uint8_t perm,caddr_t * vaddr,caddr_t * raddr)1241 ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie, uint32_t ccount,
1242     uint8_t mtype, uint8_t perm, caddr_t *vaddr, caddr_t *raddr)
1243 {
1244 	/*
1245 	 * Check if direct map over shared memory is enabled, if not change
1246 	 * the mapping type to SHADOW_MAP.
1247 	 */
1248 	if (ldc_shmem_enabled == 0)
1249 		mtype = LDC_SHADOW_MAP;
1250 
1251 	return (i_ldc_mem_map(mhandle, cookie, ccount, mtype, perm,
1252 	    vaddr, raddr));
1253 }
1254 
1255 static int
i_ldc_mem_map(ldc_mem_handle_t mhandle,ldc_mem_cookie_t * cookie,uint32_t ccount,uint8_t mtype,uint8_t perm,caddr_t * vaddr,caddr_t * raddr)1256 i_ldc_mem_map(ldc_mem_handle_t mhandle, ldc_mem_cookie_t *cookie,
1257     uint32_t ccount, uint8_t mtype, uint8_t perm, caddr_t *vaddr,
1258     caddr_t *raddr)
1259 {
1260 
1261 	int		i, j, idx, rv, retries;
1262 	ldc_chan_t	*ldcp;
1263 	ldc_mhdl_t	*mhdl;
1264 	ldc_memseg_t	*memseg;
1265 	caddr_t		tmpaddr;
1266 	uint64_t	map_perm = perm;
1267 	uint64_t	pg_size, pg_shift, pg_size_code, pg_mask;
1268 	uint64_t	exp_size = 0, base_off, map_size, npages;
1269 	uint64_t	cookie_addr, cookie_off, cookie_size;
1270 	tte_t		ldc_tte;
1271 
1272 	if (mhandle == 0) {
1273 		DWARN(DBG_ALL_LDCS, "ldc_mem_map: invalid memory handle\n");
1274 		return (EINVAL);
1275 	}
1276 	mhdl = (ldc_mhdl_t *)mhandle;
1277 
1278 	mutex_enter(&mhdl->lock);
1279 
1280 	if (mhdl->status == LDC_BOUND || mhdl->status == LDC_MAPPED ||
1281 	    mhdl->memseg != NULL) {
1282 		DWARN(DBG_ALL_LDCS,
1283 		    "ldc_mem_map: (0x%llx) handle bound/mapped\n", mhandle);
1284 		mutex_exit(&mhdl->lock);
1285 		return (EINVAL);
1286 	}
1287 
1288 	ldcp = mhdl->ldcp;
1289 
1290 	mutex_enter(&ldcp->lock);
1291 
1292 	if (ldcp->tstate != TS_UP) {
1293 		DWARN(ldcp->id,
1294 		    "ldc_mem_dring_map: (0x%llx) channel is not UP\n",
1295 		    ldcp->id);
1296 		mutex_exit(&ldcp->lock);
1297 		mutex_exit(&mhdl->lock);
1298 		return (ECONNRESET);
1299 	}
1300 
1301 	if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) {
1302 		DWARN(ldcp->id, "ldc_mem_map: invalid map type\n");
1303 		mutex_exit(&ldcp->lock);
1304 		mutex_exit(&mhdl->lock);
1305 		return (EINVAL);
1306 	}
1307 
1308 	D1(ldcp->id, "ldc_mem_map: (0x%llx) cookie = 0x%llx,0x%llx\n",
1309 	    ldcp->id, cookie->addr, cookie->size);
1310 
1311 	/* FUTURE: get the page size, pgsz code, and shift */
1312 	pg_size = MMU_PAGESIZE;
1313 	pg_size_code = page_szc(pg_size);
1314 	pg_shift = page_get_shift(pg_size_code);
1315 	pg_mask = ~(pg_size - 1);
1316 
1317 	/* calculate the number of pages in the exported cookie */
1318 	base_off = cookie[0].addr & (pg_size - 1);
1319 	for (idx = 0; idx < ccount; idx++)
1320 		exp_size += cookie[idx].size;
1321 	map_size = P2ROUNDUP((exp_size + base_off), pg_size);
1322 	npages = (map_size >> pg_shift);
1323 
1324 	/* Allocate memseg structure */
1325 	memseg = mhdl->memseg =
1326 	    kmem_cache_alloc(ldcssp->memseg_cache, KM_SLEEP);
1327 
1328 	/* Allocate memory to store all pages and cookies */
1329 	memseg->pages =	kmem_zalloc((sizeof (ldc_page_t) * npages), KM_SLEEP);
1330 	memseg->cookies =
1331 	    kmem_zalloc((sizeof (ldc_mem_cookie_t) * ccount), KM_SLEEP);
1332 
1333 	D2(ldcp->id, "ldc_mem_map: (0x%llx) exp_size=0x%llx, map_size=0x%llx,"
1334 	    "pages=0x%llx\n", ldcp->id, exp_size, map_size, npages);
1335 
1336 	/*
1337 	 * Check to see if the client is requesting direct or shadow map
1338 	 * If direct map is requested, try to map remote memory first,
1339 	 * and if that fails, revert to shadow map
1340 	 */
1341 	if (mtype == LDC_DIRECT_MAP) {
1342 
1343 		/* Allocate kernel virtual space for mapping */
1344 		memseg->vaddr = vmem_xalloc(heap_arena, map_size,
1345 		    pg_size, 0, 0, NULL, NULL, VM_NOSLEEP);
1346 		if (memseg->vaddr == NULL) {
1347 			DWARN(DBG_ALL_LDCS,
1348 			    "ldc_mem_map: (0x%lx) memory map failed\n",
1349 			    ldcp->id);
1350 			kmem_free(memseg->cookies,
1351 			    (sizeof (ldc_mem_cookie_t) * ccount));
1352 			kmem_free(memseg->pages,
1353 			    (sizeof (ldc_page_t) * npages));
1354 			kmem_cache_free(ldcssp->memseg_cache, memseg);
1355 
1356 			mutex_exit(&ldcp->lock);
1357 			mutex_exit(&mhdl->lock);
1358 			return (ENOMEM);
1359 		}
1360 
1361 		/* Unload previous mapping */
1362 		hat_unload(kas.a_hat, memseg->vaddr, map_size,
1363 		    HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK);
1364 
1365 		/* for each cookie passed in - map into address space */
1366 		idx = 0;
1367 		cookie_size = 0;
1368 		tmpaddr = memseg->vaddr;
1369 
1370 		for (i = 0; i < npages; i++) {
1371 
1372 			if (cookie_size == 0) {
1373 				ASSERT(idx < ccount);
1374 				cookie_addr = cookie[idx].addr & pg_mask;
1375 				cookie_off = cookie[idx].addr & (pg_size - 1);
1376 				cookie_size =
1377 				    P2ROUNDUP((cookie_off + cookie[idx].size),
1378 				    pg_size);
1379 				idx++;
1380 			}
1381 
1382 			D1(ldcp->id, "ldc_mem_map: (0x%llx) mapping "
1383 			    "cookie 0x%llx, bal=0x%llx\n", ldcp->id,
1384 			    cookie_addr, cookie_size);
1385 
1386 			/* map the cookie into address space */
1387 			for (retries = 0; retries < ldc_max_retries;
1388 			    retries++) {
1389 
1390 				rv = hv_ldc_mapin(ldcp->id, cookie_addr,
1391 				    &memseg->pages[i].raddr, &map_perm);
1392 				if (rv != H_EWOULDBLOCK && rv != H_ETOOMANY)
1393 					break;
1394 
1395 				drv_usecwait(ldc_delay);
1396 			}
1397 
1398 			if (rv || memseg->pages[i].raddr == 0) {
1399 				DWARN(ldcp->id,
1400 				    "ldc_mem_map: (0x%llx) hv mapin err %d\n",
1401 				    ldcp->id, rv);
1402 
1403 				/* remove previous mapins */
1404 				hat_unload(kas.a_hat, memseg->vaddr, map_size,
1405 				    HAT_UNLOAD_NOSYNC | HAT_UNLOAD_UNLOCK);
1406 				for (j = 0; j < i; j++) {
1407 					rv = hv_ldc_unmap(
1408 					    memseg->pages[j].raddr);
1409 					if (rv) {
1410 						DWARN(ldcp->id,
1411 						    "ldc_mem_map: (0x%llx) "
1412 						    "cannot unmap ra=0x%llx\n",
1413 						    ldcp->id,
1414 						    memseg->pages[j].raddr);
1415 					}
1416 				}
1417 
1418 				/* free kernel virtual space */
1419 				vmem_free(heap_arena, (void *)memseg->vaddr,
1420 				    map_size);
1421 
1422 				/* direct map failed - revert to shadow map */
1423 				mtype = LDC_SHADOW_MAP;
1424 				break;
1425 
1426 			} else {
1427 
1428 				D1(ldcp->id,
1429 				    "ldc_mem_map: (0x%llx) vtop map 0x%llx -> "
1430 				    "0x%llx, cookie=0x%llx, perm=0x%llx\n",
1431 				    ldcp->id, tmpaddr, memseg->pages[i].raddr,
1432 				    cookie_addr, perm);
1433 
1434 				/*
1435 				 * NOTE: Calling hat_devload directly, causes it
1436 				 * to look for page_t using the pfn. Since this
1437 				 * addr is greater than the memlist, it treates
1438 				 * it as non-memory
1439 				 */
1440 				sfmmu_memtte(&ldc_tte,
1441 				    (pfn_t)(memseg->pages[i].raddr >> pg_shift),
1442 				    PROT_READ | PROT_WRITE | HAT_NOSYNC, TTE8K);
1443 
1444 				D1(ldcp->id,
1445 				    "ldc_mem_map: (0x%llx) ra 0x%llx -> "
1446 				    "tte 0x%llx\n", ldcp->id,
1447 				    memseg->pages[i].raddr, ldc_tte);
1448 
1449 				sfmmu_tteload(kas.a_hat, &ldc_tte, tmpaddr,
1450 				    NULL, HAT_LOAD_LOCK);
1451 
1452 				cookie_size -= pg_size;
1453 				cookie_addr += pg_size;
1454 				tmpaddr += pg_size;
1455 			}
1456 		}
1457 	}
1458 
1459 	if (mtype == LDC_SHADOW_MAP) {
1460 		if (*vaddr == NULL) {
1461 			memseg->vaddr = kmem_zalloc(exp_size, KM_SLEEP);
1462 			mhdl->myshadow = B_TRUE;
1463 
1464 			D1(ldcp->id, "ldc_mem_map: (0x%llx) allocated "
1465 			    "shadow page va=0x%llx\n", ldcp->id, memseg->vaddr);
1466 		} else {
1467 			/*
1468 			 * Use client supplied memory for memseg->vaddr
1469 			 * WARNING: assuming that client mem is >= exp_size
1470 			 */
1471 			memseg->vaddr = *vaddr;
1472 		}
1473 
1474 		/* Save all page and cookie information */
1475 		for (i = 0, tmpaddr = memseg->vaddr; i < npages; i++) {
1476 			memseg->pages[i].raddr = va_to_pa(tmpaddr);
1477 			tmpaddr += pg_size;
1478 		}
1479 
1480 	}
1481 
1482 	/* save all cookies */
1483 	bcopy(cookie, memseg->cookies, ccount * sizeof (ldc_mem_cookie_t));
1484 
1485 	/* update memseg_t */
1486 	memseg->raddr = memseg->pages[0].raddr;
1487 	memseg->size = (mtype == LDC_SHADOW_MAP) ? exp_size : map_size;
1488 	memseg->npages = npages;
1489 	memseg->ncookies = ccount;
1490 	memseg->next_cookie = 0;
1491 
1492 	/* memory handle = mapped */
1493 	mhdl->mtype = mtype;
1494 	mhdl->perm = perm;
1495 	mhdl->status = LDC_MAPPED;
1496 
1497 	D1(ldcp->id, "ldc_mem_map: (0x%llx) mapped 0x%llx, ra=0x%llx, "
1498 	    "va=0x%llx, pgs=0x%llx cookies=0x%llx\n",
1499 	    ldcp->id, mhdl, memseg->raddr, memseg->vaddr,
1500 	    memseg->npages, memseg->ncookies);
1501 
1502 	if (mtype == LDC_SHADOW_MAP)
1503 		base_off = 0;
1504 	if (raddr)
1505 		*raddr = (caddr_t)(memseg->raddr | base_off);
1506 	if (vaddr)
1507 		*vaddr = (caddr_t)((uintptr_t)memseg->vaddr | base_off);
1508 
1509 	mutex_exit(&ldcp->lock);
1510 	mutex_exit(&mhdl->lock);
1511 	return (0);
1512 }
1513 
1514 /*
1515  * Unmap a memory segment. Free shadow memory (if any).
1516  */
1517 int
ldc_mem_unmap(ldc_mem_handle_t mhandle)1518 ldc_mem_unmap(ldc_mem_handle_t mhandle)
1519 {
1520 	int		i, rv;
1521 	ldc_mhdl_t	*mhdl = (ldc_mhdl_t *)mhandle;
1522 	ldc_chan_t	*ldcp;
1523 	ldc_memseg_t	*memseg;
1524 
1525 	if (mhdl == 0 || mhdl->status != LDC_MAPPED) {
1526 		DWARN(DBG_ALL_LDCS,
1527 		    "ldc_mem_unmap: (0x%llx) handle is not mapped\n",
1528 		    mhandle);
1529 		return (EINVAL);
1530 	}
1531 
1532 	mutex_enter(&mhdl->lock);
1533 
1534 	ldcp = mhdl->ldcp;
1535 	memseg = mhdl->memseg;
1536 
1537 	D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapping handle 0x%llx\n",
1538 	    ldcp->id, mhdl);
1539 
1540 	/* if we allocated shadow memory - free it */
1541 	if (mhdl->mtype == LDC_SHADOW_MAP && mhdl->myshadow) {
1542 		kmem_free(memseg->vaddr, memseg->size);
1543 	} else if (mhdl->mtype == LDC_DIRECT_MAP) {
1544 
1545 		/* unmap in the case of DIRECT_MAP */
1546 		hat_unload(kas.a_hat, memseg->vaddr, memseg->size,
1547 		    HAT_UNLOAD_UNLOCK);
1548 
1549 		for (i = 0; i < memseg->npages; i++) {
1550 			rv = hv_ldc_unmap(memseg->pages[i].raddr);
1551 			if (rv) {
1552 				DWARN(DBG_ALL_LDCS,
1553 				    "ldc_mem_map: (0x%lx) hv unmap err %d\n",
1554 				    ldcp->id, rv);
1555 			}
1556 		}
1557 
1558 		vmem_free(heap_arena, (void *)memseg->vaddr, memseg->size);
1559 	}
1560 
1561 	/* free the allocated memseg and page structures */
1562 	kmem_free(memseg->pages, (sizeof (ldc_page_t) * memseg->npages));
1563 	kmem_free(memseg->cookies,
1564 	    (sizeof (ldc_mem_cookie_t) * memseg->ncookies));
1565 	kmem_cache_free(ldcssp->memseg_cache, memseg);
1566 
1567 	/* uninitialize the memory handle */
1568 	mhdl->memseg = NULL;
1569 	mhdl->status = LDC_UNBOUND;
1570 
1571 	D1(ldcp->id, "ldc_mem_unmap: (0x%llx) unmapped handle 0x%llx\n",
1572 	    ldcp->id, mhdl);
1573 
1574 	mutex_exit(&mhdl->lock);
1575 	return (0);
1576 }
1577 
1578 /*
1579  * Internal entry point for LDC mapped memory entry consistency
1580  * semantics. Acquire copies the contents of the remote memory
1581  * into the local shadow copy. The release operation copies the local
1582  * contents into the remote memory. The offset and size specify the
1583  * bounds for the memory range being synchronized.
1584  */
1585 static int
i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle,uint8_t direction,uint64_t offset,size_t size)1586 i_ldc_mem_acquire_release(ldc_mem_handle_t mhandle, uint8_t direction,
1587     uint64_t offset, size_t size)
1588 {
1589 	int		err;
1590 	ldc_mhdl_t	*mhdl;
1591 	ldc_chan_t	*ldcp;
1592 	ldc_memseg_t	*memseg;
1593 	caddr_t		local_vaddr;
1594 	size_t		copy_size;
1595 
1596 	if (mhandle == 0) {
1597 		DWARN(DBG_ALL_LDCS,
1598 		    "i_ldc_mem_acquire_release: invalid memory handle\n");
1599 		return (EINVAL);
1600 	}
1601 	mhdl = (ldc_mhdl_t *)mhandle;
1602 
1603 	mutex_enter(&mhdl->lock);
1604 
1605 	if (mhdl->status != LDC_MAPPED || mhdl->ldcp == NULL) {
1606 		DWARN(DBG_ALL_LDCS,
1607 		    "i_ldc_mem_acquire_release: not mapped memory\n");
1608 		mutex_exit(&mhdl->lock);
1609 		return (EINVAL);
1610 	}
1611 
1612 	/* do nothing for direct map */
1613 	if (mhdl->mtype == LDC_DIRECT_MAP) {
1614 		mutex_exit(&mhdl->lock);
1615 		return (0);
1616 	}
1617 
1618 	/* do nothing if COPY_IN+MEM_W and COPY_OUT+MEM_R */
1619 	if ((direction == LDC_COPY_IN && (mhdl->perm & LDC_MEM_R) == 0) ||
1620 	    (direction == LDC_COPY_OUT && (mhdl->perm & LDC_MEM_W) == 0)) {
1621 		mutex_exit(&mhdl->lock);
1622 		return (0);
1623 	}
1624 
1625 	if (offset >= mhdl->memseg->size ||
1626 	    (offset + size) > mhdl->memseg->size) {
1627 		DWARN(DBG_ALL_LDCS,
1628 		    "i_ldc_mem_acquire_release: memory out of range\n");
1629 		mutex_exit(&mhdl->lock);
1630 		return (EINVAL);
1631 	}
1632 
1633 	/* get the channel handle and memory segment */
1634 	ldcp = mhdl->ldcp;
1635 	memseg = mhdl->memseg;
1636 
1637 	if (mhdl->mtype == LDC_SHADOW_MAP) {
1638 
1639 		local_vaddr = memseg->vaddr + offset;
1640 		copy_size = size;
1641 
1642 		/* copy to/from remote from/to local memory */
1643 		err = ldc_mem_copy((ldc_handle_t)ldcp, local_vaddr, offset,
1644 		    &copy_size, memseg->cookies, memseg->ncookies,
1645 		    direction);
1646 		if (err || copy_size != size) {
1647 			DWARN(ldcp->id,
1648 			    "i_ldc_mem_acquire_release: copy failed\n");
1649 			mutex_exit(&mhdl->lock);
1650 			return (err);
1651 		}
1652 	}
1653 
1654 	mutex_exit(&mhdl->lock);
1655 
1656 	return (0);
1657 }
1658 
1659 /*
1660  * Ensure that the contents in the remote memory seg are consistent
1661  * with the contents if of local segment
1662  */
1663 int
ldc_mem_acquire(ldc_mem_handle_t mhandle,uint64_t offset,uint64_t size)1664 ldc_mem_acquire(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size)
1665 {
1666 	return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_IN, offset, size));
1667 }
1668 
1669 
1670 /*
1671  * Ensure that the contents in the local memory seg are consistent
1672  * with the contents if of remote segment
1673  */
1674 int
ldc_mem_release(ldc_mem_handle_t mhandle,uint64_t offset,uint64_t size)1675 ldc_mem_release(ldc_mem_handle_t mhandle, uint64_t offset, uint64_t size)
1676 {
1677 	return (i_ldc_mem_acquire_release(mhandle, LDC_COPY_OUT, offset, size));
1678 }
1679 
1680 /*
1681  * Allocate a descriptor ring. The size of each each descriptor
1682  * must be 8-byte aligned and the entire ring should be a multiple
1683  * of MMU_PAGESIZE.
1684  */
1685 int
ldc_mem_dring_create(uint32_t len,uint32_t dsize,ldc_dring_handle_t * dhandle)1686 ldc_mem_dring_create(uint32_t len, uint32_t dsize, ldc_dring_handle_t *dhandle)
1687 {
1688 	ldc_dring_t *dringp;
1689 	size_t size = (dsize * len);
1690 
1691 	D1(DBG_ALL_LDCS, "ldc_mem_dring_create: len=0x%x, size=0x%x\n",
1692 	    len, dsize);
1693 
1694 	if (dhandle == 0) {
1695 		DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid dhandle\n");
1696 		return (EINVAL);
1697 	}
1698 
1699 	if (len == 0) {
1700 		DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid length\n");
1701 		return (EINVAL);
1702 	}
1703 
1704 	/* descriptor size should be 8-byte aligned */
1705 	if (dsize == 0 || (dsize & 0x7)) {
1706 		DWARN(DBG_ALL_LDCS, "ldc_mem_dring_create: invalid size\n");
1707 		return (EINVAL);
1708 	}
1709 
1710 	*dhandle = 0;
1711 
1712 	/* Allocate a desc ring structure */
1713 	dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP);
1714 
1715 	/* Initialize dring */
1716 	dringp->length = len;
1717 	dringp->dsize = dsize;
1718 
1719 	/* round off to multiple of pagesize */
1720 	dringp->size = (size & MMU_PAGEMASK);
1721 	if (size & MMU_PAGEOFFSET)
1722 		dringp->size += MMU_PAGESIZE;
1723 
1724 	dringp->status = LDC_UNBOUND;
1725 
1726 	/* allocate descriptor ring memory */
1727 	dringp->base = kmem_zalloc(dringp->size, KM_SLEEP);
1728 
1729 	/* initialize the desc ring lock */
1730 	mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL);
1731 
1732 	/* Add descriptor ring to the head of global list */
1733 	mutex_enter(&ldcssp->lock);
1734 	dringp->next = ldcssp->dring_list;
1735 	ldcssp->dring_list = dringp;
1736 	mutex_exit(&ldcssp->lock);
1737 
1738 	*dhandle = (ldc_dring_handle_t)dringp;
1739 
1740 	D1(DBG_ALL_LDCS, "ldc_mem_dring_create: dring allocated\n");
1741 
1742 	return (0);
1743 }
1744 
1745 
1746 /*
1747  * Destroy a descriptor ring.
1748  */
1749 int
ldc_mem_dring_destroy(ldc_dring_handle_t dhandle)1750 ldc_mem_dring_destroy(ldc_dring_handle_t dhandle)
1751 {
1752 	ldc_dring_t *dringp;
1753 	ldc_dring_t *tmp_dringp;
1754 
1755 	D1(DBG_ALL_LDCS, "ldc_mem_dring_destroy: entered\n");
1756 
1757 	if (dhandle == 0) {
1758 		DWARN(DBG_ALL_LDCS,
1759 		    "ldc_mem_dring_destroy: invalid desc ring handle\n");
1760 		return (EINVAL);
1761 	}
1762 	dringp = (ldc_dring_t *)dhandle;
1763 
1764 	if (dringp->status == LDC_BOUND) {
1765 		DWARN(DBG_ALL_LDCS,
1766 		    "ldc_mem_dring_destroy: desc ring is bound\n");
1767 		return (EACCES);
1768 	}
1769 
1770 	mutex_enter(&dringp->lock);
1771 	mutex_enter(&ldcssp->lock);
1772 
1773 	/* remove from linked list - if not bound */
1774 	tmp_dringp = ldcssp->dring_list;
1775 	if (tmp_dringp == dringp) {
1776 		ldcssp->dring_list = dringp->next;
1777 		dringp->next = NULL;
1778 
1779 	} else {
1780 		while (tmp_dringp != NULL) {
1781 			if (tmp_dringp->next == dringp) {
1782 				tmp_dringp->next = dringp->next;
1783 				dringp->next = NULL;
1784 				break;
1785 			}
1786 			tmp_dringp = tmp_dringp->next;
1787 		}
1788 		if (tmp_dringp == NULL) {
1789 			DWARN(DBG_ALL_LDCS,
1790 			    "ldc_mem_dring_destroy: invalid descriptor\n");
1791 			mutex_exit(&ldcssp->lock);
1792 			mutex_exit(&dringp->lock);
1793 			return (EINVAL);
1794 		}
1795 	}
1796 
1797 	mutex_exit(&ldcssp->lock);
1798 
1799 	/* free the descriptor ring */
1800 	kmem_free(dringp->base, dringp->size);
1801 
1802 	mutex_exit(&dringp->lock);
1803 
1804 	/* destroy dring lock */
1805 	mutex_destroy(&dringp->lock);
1806 
1807 	/* free desc ring object */
1808 	kmem_free(dringp, sizeof (ldc_dring_t));
1809 
1810 	return (0);
1811 }
1812 
1813 /*
1814  * Bind a previously allocated dring to a channel. The channel should
1815  * be OPEN in order to bind the ring to the channel. Returns back a
1816  * descriptor ring cookie. The descriptor ring is exported for remote
1817  * access by the client at the other end of the channel. An entry for
1818  * dring pages is stored in map table (via call to ldc_mem_bind_handle).
1819  */
1820 int
ldc_mem_dring_bind(ldc_handle_t handle,ldc_dring_handle_t dhandle,uint8_t mtype,uint8_t perm,ldc_mem_cookie_t * cookie,uint32_t * ccount)1821 ldc_mem_dring_bind(ldc_handle_t handle, ldc_dring_handle_t dhandle,
1822     uint8_t mtype, uint8_t perm, ldc_mem_cookie_t *cookie, uint32_t *ccount)
1823 {
1824 	int		err;
1825 	ldc_chan_t	*ldcp;
1826 	ldc_dring_t	*dringp;
1827 	ldc_mem_handle_t mhandle;
1828 
1829 	/* check to see if channel is initalized */
1830 	if (handle == 0) {
1831 		DWARN(DBG_ALL_LDCS,
1832 		    "ldc_mem_dring_bind: invalid channel handle\n");
1833 		return (EINVAL);
1834 	}
1835 	ldcp = (ldc_chan_t *)handle;
1836 
1837 	if (dhandle == 0) {
1838 		DWARN(DBG_ALL_LDCS,
1839 		    "ldc_mem_dring_bind: invalid desc ring handle\n");
1840 		return (EINVAL);
1841 	}
1842 	dringp = (ldc_dring_t *)dhandle;
1843 
1844 	if (cookie == NULL) {
1845 		DWARN(ldcp->id,
1846 		    "ldc_mem_dring_bind: invalid cookie arg\n");
1847 		return (EINVAL);
1848 	}
1849 
1850 	/* ensure the mtype is valid */
1851 	if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP)) == 0) {
1852 		DWARN(ldcp->id, "ldc_mem_dring_bind: invalid map type\n");
1853 		return (EINVAL);
1854 	}
1855 
1856 	/* no need to bind as direct map if it's not HV supported or enabled */
1857 	if (!ldc_dring_shmem_hv_ok || !ldc_dring_shmem_enabled) {
1858 		mtype = LDC_SHADOW_MAP;
1859 	}
1860 
1861 	mutex_enter(&dringp->lock);
1862 
1863 	if (dringp->status == LDC_BOUND) {
1864 		DWARN(DBG_ALL_LDCS,
1865 		    "ldc_mem_dring_bind: (0x%llx) descriptor ring is bound\n",
1866 		    ldcp->id);
1867 		mutex_exit(&dringp->lock);
1868 		return (EINVAL);
1869 	}
1870 
1871 	if ((perm & LDC_MEM_RW) == 0) {
1872 		DWARN(DBG_ALL_LDCS,
1873 		    "ldc_mem_dring_bind: invalid permissions\n");
1874 		mutex_exit(&dringp->lock);
1875 		return (EINVAL);
1876 	}
1877 
1878 	if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP|LDC_IO_MAP)) == 0) {
1879 		DWARN(DBG_ALL_LDCS, "ldc_mem_dring_bind: invalid type\n");
1880 		mutex_exit(&dringp->lock);
1881 		return (EINVAL);
1882 	}
1883 
1884 	dringp->ldcp = ldcp;
1885 
1886 	/* create an memory handle */
1887 	err = ldc_mem_alloc_handle(handle, &mhandle);
1888 	if (err || mhandle == 0) {
1889 		DWARN(DBG_ALL_LDCS,
1890 		    "ldc_mem_dring_bind: (0x%llx) error allocating mhandle\n",
1891 		    ldcp->id);
1892 		mutex_exit(&dringp->lock);
1893 		return (err);
1894 	}
1895 	dringp->mhdl = mhandle;
1896 
1897 	/* bind the descriptor ring to channel */
1898 	err = i_ldc_mem_bind_handle(mhandle, dringp->base, dringp->size,
1899 	    mtype, perm, cookie, ccount);
1900 	if (err) {
1901 		DWARN(ldcp->id,
1902 		    "ldc_mem_dring_bind: (0x%llx) error binding mhandle\n",
1903 		    ldcp->id);
1904 		mutex_exit(&dringp->lock);
1905 		return (err);
1906 	}
1907 
1908 	/*
1909 	 * For now return error if we get more than one cookie
1910 	 * FUTURE: Return multiple cookies ..
1911 	 */
1912 	if (*ccount > 1) {
1913 		(void) ldc_mem_unbind_handle(mhandle);
1914 		(void) ldc_mem_free_handle(mhandle);
1915 
1916 		dringp->ldcp = NULL;
1917 		dringp->mhdl = 0;
1918 		*ccount = 0;
1919 
1920 		mutex_exit(&dringp->lock);
1921 		return (EAGAIN);
1922 	}
1923 
1924 	/* Add descriptor ring to channel's exported dring list */
1925 	mutex_enter(&ldcp->exp_dlist_lock);
1926 	dringp->ch_next = ldcp->exp_dring_list;
1927 	ldcp->exp_dring_list = dringp;
1928 	mutex_exit(&ldcp->exp_dlist_lock);
1929 
1930 	dringp->status = LDC_BOUND;
1931 
1932 	mutex_exit(&dringp->lock);
1933 
1934 	return (0);
1935 }
1936 
1937 /*
1938  * Return the next cookie associated with the specified dring handle
1939  */
1940 int
ldc_mem_dring_nextcookie(ldc_dring_handle_t dhandle,ldc_mem_cookie_t * cookie)1941 ldc_mem_dring_nextcookie(ldc_dring_handle_t dhandle, ldc_mem_cookie_t *cookie)
1942 {
1943 	int		rv = 0;
1944 	ldc_dring_t	*dringp;
1945 	ldc_chan_t	*ldcp;
1946 
1947 	if (dhandle == 0) {
1948 		DWARN(DBG_ALL_LDCS,
1949 		    "ldc_mem_dring_nextcookie: invalid desc ring handle\n");
1950 		return (EINVAL);
1951 	}
1952 	dringp = (ldc_dring_t *)dhandle;
1953 	mutex_enter(&dringp->lock);
1954 
1955 	if (dringp->status != LDC_BOUND) {
1956 		DWARN(DBG_ALL_LDCS,
1957 		    "ldc_mem_dring_nextcookie: descriptor ring 0x%llx "
1958 		    "is not bound\n", dringp);
1959 		mutex_exit(&dringp->lock);
1960 		return (EINVAL);
1961 	}
1962 
1963 	ldcp = dringp->ldcp;
1964 
1965 	if (cookie == NULL) {
1966 		DWARN(ldcp->id,
1967 		    "ldc_mem_dring_nextcookie:(0x%llx) invalid cookie arg\n",
1968 		    ldcp->id);
1969 		mutex_exit(&dringp->lock);
1970 		return (EINVAL);
1971 	}
1972 
1973 	rv = ldc_mem_nextcookie((ldc_mem_handle_t)dringp->mhdl, cookie);
1974 	mutex_exit(&dringp->lock);
1975 
1976 	return (rv);
1977 }
1978 
1979 /*
1980  * Unbind a previously bound dring from a channel.
1981  */
1982 int
ldc_mem_dring_unbind(ldc_dring_handle_t dhandle)1983 ldc_mem_dring_unbind(ldc_dring_handle_t dhandle)
1984 {
1985 	ldc_dring_t	*dringp;
1986 	ldc_dring_t	*tmp_dringp;
1987 	ldc_chan_t	*ldcp;
1988 
1989 	if (dhandle == 0) {
1990 		DWARN(DBG_ALL_LDCS,
1991 		    "ldc_mem_dring_unbind: invalid desc ring handle\n");
1992 		return (EINVAL);
1993 	}
1994 	dringp = (ldc_dring_t *)dhandle;
1995 
1996 	mutex_enter(&dringp->lock);
1997 
1998 	if (dringp->status == LDC_UNBOUND) {
1999 		DWARN(DBG_ALL_LDCS,
2000 		    "ldc_mem_dring_bind: descriptor ring 0x%llx is unbound\n",
2001 		    dringp);
2002 		mutex_exit(&dringp->lock);
2003 		return (EINVAL);
2004 	}
2005 	ldcp = dringp->ldcp;
2006 
2007 	mutex_enter(&ldcp->exp_dlist_lock);
2008 
2009 	tmp_dringp = ldcp->exp_dring_list;
2010 	if (tmp_dringp == dringp) {
2011 		ldcp->exp_dring_list = dringp->ch_next;
2012 		dringp->ch_next = NULL;
2013 
2014 	} else {
2015 		while (tmp_dringp != NULL) {
2016 			if (tmp_dringp->ch_next == dringp) {
2017 				tmp_dringp->ch_next = dringp->ch_next;
2018 				dringp->ch_next = NULL;
2019 				break;
2020 			}
2021 			tmp_dringp = tmp_dringp->ch_next;
2022 		}
2023 		if (tmp_dringp == NULL) {
2024 			DWARN(DBG_ALL_LDCS,
2025 			    "ldc_mem_dring_unbind: invalid descriptor\n");
2026 			mutex_exit(&ldcp->exp_dlist_lock);
2027 			mutex_exit(&dringp->lock);
2028 			return (EINVAL);
2029 		}
2030 	}
2031 
2032 	mutex_exit(&ldcp->exp_dlist_lock);
2033 
2034 	(void) ldc_mem_unbind_handle((ldc_mem_handle_t)dringp->mhdl);
2035 	(void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl);
2036 
2037 	dringp->ldcp = NULL;
2038 	dringp->mhdl = 0;
2039 	dringp->status = LDC_UNBOUND;
2040 
2041 	mutex_exit(&dringp->lock);
2042 
2043 	return (0);
2044 }
2045 
2046 #ifdef	DEBUG
2047 void
i_ldc_mem_inject_dring_clear(ldc_chan_t * ldcp)2048 i_ldc_mem_inject_dring_clear(ldc_chan_t *ldcp)
2049 {
2050 	ldc_dring_t	*dp;
2051 	ldc_mhdl_t	*mhdl;
2052 	ldc_mtbl_t	*mtbl;
2053 	ldc_memseg_t	*memseg;
2054 	uint64_t	cookie_addr;
2055 	uint64_t	pg_shift, pg_size_code;
2056 	int		i, rv, retries;
2057 
2058 	/* has a map table been allocated? */
2059 	if ((mtbl = ldcp->mtbl) == NULL)
2060 		return;
2061 
2062 	/* lock the memory table - exclusive access to channel */
2063 	mutex_enter(&mtbl->lock);
2064 
2065 	/* lock the exported dring list */
2066 	mutex_enter(&ldcp->exp_dlist_lock);
2067 
2068 	for (dp = ldcp->exp_dring_list; dp != NULL; dp = dp->ch_next) {
2069 		if ((mhdl = (ldc_mhdl_t *)dp->mhdl) == NULL)
2070 			continue;
2071 
2072 		if ((memseg = mhdl->memseg) == NULL)
2073 			continue;
2074 
2075 		/* undo the pages exported */
2076 		for (i = 0; i < memseg->npages; i++) {
2077 
2078 			/* clear the entry from the table */
2079 			memseg->pages[i].mte->entry.ll = 0;
2080 
2081 			pg_size_code = page_szc(MMU_PAGESIZE);
2082 			pg_shift = page_get_shift(pg_size_code);
2083 			cookie_addr = IDX2COOKIE(memseg->pages[i].index,
2084 			    pg_size_code, pg_shift);
2085 
2086 			retries = 0;
2087 			do {
2088 				rv = hv_ldc_revoke(ldcp->id, cookie_addr,
2089 				    memseg->pages[i].mte->cookie);
2090 
2091 				if (rv != H_EWOULDBLOCK)
2092 					break;
2093 
2094 				drv_usecwait(ldc_delay);
2095 
2096 			} while (retries++ < ldc_max_retries);
2097 
2098 			if (rv != 0) {
2099 				DWARN(ldcp->id,
2100 				    "i_ldc_mem_inject_dring_clear(): "
2101 				    "hv_ldc_revoke failed: "
2102 				    "channel: 0x%lx, cookie addr: 0x%p,"
2103 				    "cookie: 0x%lx, rv: %d",
2104 				    ldcp->id, cookie_addr,
2105 				    memseg->pages[i].mte->cookie, rv);
2106 			}
2107 
2108 			mtbl->num_avail++;
2109 		}
2110 	}
2111 
2112 	mutex_exit(&ldcp->exp_dlist_lock);
2113 	mutex_exit(&mtbl->lock);
2114 }
2115 #endif
2116 
2117 /*
2118  * Get information about the dring. The base address of the descriptor
2119  * ring along with the type and permission are returned back.
2120  */
2121 int
ldc_mem_dring_info(ldc_dring_handle_t dhandle,ldc_mem_info_t * minfo)2122 ldc_mem_dring_info(ldc_dring_handle_t dhandle, ldc_mem_info_t *minfo)
2123 {
2124 	ldc_dring_t	*dringp;
2125 	int		rv;
2126 
2127 	if (dhandle == 0) {
2128 		DWARN(DBG_ALL_LDCS,
2129 		    "ldc_mem_dring_info: invalid desc ring handle\n");
2130 		return (EINVAL);
2131 	}
2132 	dringp = (ldc_dring_t *)dhandle;
2133 
2134 	mutex_enter(&dringp->lock);
2135 
2136 	if (dringp->mhdl) {
2137 		rv = ldc_mem_info(dringp->mhdl, minfo);
2138 		if (rv) {
2139 			DWARN(DBG_ALL_LDCS,
2140 			    "ldc_mem_dring_info: error reading mem info\n");
2141 			mutex_exit(&dringp->lock);
2142 			return (rv);
2143 		}
2144 	} else {
2145 		minfo->vaddr = dringp->base;
2146 		minfo->raddr = 0;
2147 		minfo->status = dringp->status;
2148 	}
2149 
2150 	mutex_exit(&dringp->lock);
2151 
2152 	return (0);
2153 }
2154 
2155 /*
2156  * Map an exported descriptor ring into the local address space. If the
2157  * descriptor ring was exported for direct map access, a HV call is made
2158  * to allocate a RA range. If the map is done via a shadow copy, local
2159  * shadow memory is allocated.
2160  */
2161 int
ldc_mem_dring_map(ldc_handle_t handle,ldc_mem_cookie_t * cookie,uint32_t ccount,uint32_t len,uint32_t dsize,uint8_t mtype,ldc_dring_handle_t * dhandle)2162 ldc_mem_dring_map(ldc_handle_t handle, ldc_mem_cookie_t *cookie,
2163     uint32_t ccount, uint32_t len, uint32_t dsize, uint8_t mtype,
2164     ldc_dring_handle_t *dhandle)
2165 {
2166 	int		err;
2167 	ldc_chan_t	*ldcp = (ldc_chan_t *)handle;
2168 	ldc_mem_handle_t mhandle;
2169 	ldc_dring_t	*dringp;
2170 	size_t		dring_size;
2171 
2172 	if (dhandle == 0) {
2173 		DWARN(DBG_ALL_LDCS,
2174 		    "ldc_mem_dring_map: invalid dhandle\n");
2175 		return (EINVAL);
2176 	}
2177 
2178 	/* check to see if channel is initalized */
2179 	if (handle == 0) {
2180 		DWARN(DBG_ALL_LDCS,
2181 		    "ldc_mem_dring_map: invalid channel handle\n");
2182 		return (EINVAL);
2183 	}
2184 	ldcp = (ldc_chan_t *)handle;
2185 
2186 	if (cookie == NULL) {
2187 		DWARN(ldcp->id,
2188 		    "ldc_mem_dring_map: (0x%llx) invalid cookie\n",
2189 		    ldcp->id);
2190 		return (EINVAL);
2191 	}
2192 
2193 	/* FUTURE: For now we support only one cookie per dring */
2194 	ASSERT(ccount == 1);
2195 
2196 	if (cookie->size < (dsize * len)) {
2197 		DWARN(ldcp->id,
2198 		    "ldc_mem_dring_map: (0x%llx) invalid dsize/len\n",
2199 		    ldcp->id);
2200 		return (EINVAL);
2201 	}
2202 
2203 	/* ensure the mtype is valid */
2204 	if ((mtype & (LDC_SHADOW_MAP|LDC_DIRECT_MAP)) == 0) {
2205 		DWARN(ldcp->id, "ldc_mem_dring_map: invalid map type\n");
2206 		return (EINVAL);
2207 	}
2208 
2209 	/* do not attempt direct map if it's not HV supported or enabled */
2210 	if (!ldc_dring_shmem_hv_ok || !ldc_dring_shmem_enabled) {
2211 		mtype = LDC_SHADOW_MAP;
2212 	}
2213 
2214 	*dhandle = 0;
2215 
2216 	/* Allocate an dring structure */
2217 	dringp = kmem_zalloc(sizeof (ldc_dring_t), KM_SLEEP);
2218 
2219 	D1(ldcp->id,
2220 	    "ldc_mem_dring_map: 0x%x,0x%x,0x%x,0x%llx,0x%llx\n",
2221 	    mtype, len, dsize, cookie->addr, cookie->size);
2222 
2223 	/* Initialize dring */
2224 	dringp->length = len;
2225 	dringp->dsize = dsize;
2226 
2227 	/* round of to multiple of page size */
2228 	dring_size = len * dsize;
2229 	dringp->size = (dring_size & MMU_PAGEMASK);
2230 	if (dring_size & MMU_PAGEOFFSET)
2231 		dringp->size += MMU_PAGESIZE;
2232 
2233 	dringp->ldcp = ldcp;
2234 
2235 	/* create an memory handle */
2236 	err = ldc_mem_alloc_handle(handle, &mhandle);
2237 	if (err || mhandle == 0) {
2238 		DWARN(DBG_ALL_LDCS,
2239 		    "ldc_mem_dring_map: cannot alloc hdl err=%d\n",
2240 		    err);
2241 		kmem_free(dringp, sizeof (ldc_dring_t));
2242 		return (ENOMEM);
2243 	}
2244 
2245 	dringp->mhdl = mhandle;
2246 	dringp->base = NULL;
2247 
2248 	/* map the dring into local memory */
2249 	err = i_ldc_mem_map(mhandle, cookie, ccount, mtype, LDC_MEM_RW,
2250 	    &(dringp->base), NULL);
2251 	if (err || dringp->base == NULL) {
2252 		DWARN(DBG_ALL_LDCS,
2253 		    "ldc_mem_dring_map: cannot map desc ring err=%d\n", err);
2254 		(void) ldc_mem_free_handle(mhandle);
2255 		kmem_free(dringp, sizeof (ldc_dring_t));
2256 		return (ENOMEM);
2257 	}
2258 
2259 	/* initialize the desc ring lock */
2260 	mutex_init(&dringp->lock, NULL, MUTEX_DRIVER, NULL);
2261 
2262 	/* Add descriptor ring to channel's imported dring list */
2263 	mutex_enter(&ldcp->imp_dlist_lock);
2264 	dringp->ch_next = ldcp->imp_dring_list;
2265 	ldcp->imp_dring_list = dringp;
2266 	mutex_exit(&ldcp->imp_dlist_lock);
2267 
2268 	dringp->status = LDC_MAPPED;
2269 
2270 	*dhandle = (ldc_dring_handle_t)dringp;
2271 
2272 	return (0);
2273 }
2274 
2275 /*
2276  * Unmap a descriptor ring. Free shadow memory (if any).
2277  */
2278 int
ldc_mem_dring_unmap(ldc_dring_handle_t dhandle)2279 ldc_mem_dring_unmap(ldc_dring_handle_t dhandle)
2280 {
2281 	ldc_dring_t	*dringp;
2282 	ldc_dring_t	*tmp_dringp;
2283 	ldc_chan_t	*ldcp;
2284 
2285 	if (dhandle == 0) {
2286 		DWARN(DBG_ALL_LDCS,
2287 		    "ldc_mem_dring_unmap: invalid desc ring handle\n");
2288 		return (EINVAL);
2289 	}
2290 	dringp = (ldc_dring_t *)dhandle;
2291 
2292 	if (dringp->status != LDC_MAPPED) {
2293 		DWARN(DBG_ALL_LDCS,
2294 		    "ldc_mem_dring_unmap: not a mapped desc ring\n");
2295 		return (EINVAL);
2296 	}
2297 
2298 	mutex_enter(&dringp->lock);
2299 
2300 	ldcp = dringp->ldcp;
2301 
2302 	mutex_enter(&ldcp->imp_dlist_lock);
2303 
2304 	/* find and unlink the desc ring from channel import list */
2305 	tmp_dringp = ldcp->imp_dring_list;
2306 	if (tmp_dringp == dringp) {
2307 		ldcp->imp_dring_list = dringp->ch_next;
2308 		dringp->ch_next = NULL;
2309 
2310 	} else {
2311 		while (tmp_dringp != NULL) {
2312 			if (tmp_dringp->ch_next == dringp) {
2313 				tmp_dringp->ch_next = dringp->ch_next;
2314 				dringp->ch_next = NULL;
2315 				break;
2316 			}
2317 			tmp_dringp = tmp_dringp->ch_next;
2318 		}
2319 		if (tmp_dringp == NULL) {
2320 			DWARN(DBG_ALL_LDCS,
2321 			    "ldc_mem_dring_unmap: invalid descriptor\n");
2322 			mutex_exit(&ldcp->imp_dlist_lock);
2323 			mutex_exit(&dringp->lock);
2324 			return (EINVAL);
2325 		}
2326 	}
2327 
2328 	mutex_exit(&ldcp->imp_dlist_lock);
2329 
2330 	/* do a LDC memory handle unmap and free */
2331 	(void) ldc_mem_unmap(dringp->mhdl);
2332 	(void) ldc_mem_free_handle((ldc_mem_handle_t)dringp->mhdl);
2333 
2334 	dringp->status = 0;
2335 	dringp->ldcp = NULL;
2336 
2337 	mutex_exit(&dringp->lock);
2338 
2339 	/* destroy dring lock */
2340 	mutex_destroy(&dringp->lock);
2341 
2342 	/* free desc ring object */
2343 	kmem_free(dringp, sizeof (ldc_dring_t));
2344 
2345 	return (0);
2346 }
2347 
2348 /*
2349  * Internal entry point for descriptor ring access entry consistency
2350  * semantics. Acquire copies the contents of the remote descriptor ring
2351  * into the local shadow copy. The release operation copies the local
2352  * contents into the remote dring. The start and end locations specify
2353  * bounds for the entries being synchronized.
2354  */
2355 static int
i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle,uint8_t direction,uint64_t start,uint64_t end)2356 i_ldc_dring_acquire_release(ldc_dring_handle_t dhandle,
2357     uint8_t direction, uint64_t start, uint64_t end)
2358 {
2359 	int			err;
2360 	ldc_dring_t		*dringp;
2361 	ldc_chan_t		*ldcp;
2362 	ldc_mhdl_t		*mhdl;
2363 	uint64_t		soff;
2364 	size_t			copy_size;
2365 
2366 	if (dhandle == 0) {
2367 		DWARN(DBG_ALL_LDCS,
2368 		    "i_ldc_dring_acquire_release: invalid desc ring handle\n");
2369 		return (EINVAL);
2370 	}
2371 	dringp = (ldc_dring_t *)dhandle;
2372 	mutex_enter(&dringp->lock);
2373 
2374 	if (dringp->status != LDC_MAPPED || dringp->ldcp == NULL) {
2375 		DWARN(DBG_ALL_LDCS,
2376 		    "i_ldc_dring_acquire_release: not a mapped desc ring\n");
2377 		mutex_exit(&dringp->lock);
2378 		return (EINVAL);
2379 	}
2380 
2381 	if (start >= dringp->length || end >= dringp->length) {
2382 		DWARN(DBG_ALL_LDCS,
2383 		    "i_ldc_dring_acquire_release: index out of range\n");
2384 		mutex_exit(&dringp->lock);
2385 		return (EINVAL);
2386 	}
2387 
2388 	mhdl = (ldc_mhdl_t *)dringp->mhdl;
2389 	if (mhdl == NULL) {
2390 		DWARN(DBG_ALL_LDCS,
2391 		    "i_ldc_dring_acquire_release: invalid memory handle\n");
2392 		mutex_exit(&dringp->lock);
2393 		return (EINVAL);
2394 	}
2395 
2396 	if (mhdl->mtype != LDC_SHADOW_MAP) {
2397 		DWARN(DBG_ALL_LDCS,
2398 		    "i_ldc_dring_acquire_release: invalid mtype: %d\n",
2399 		    mhdl->mtype);
2400 		mutex_exit(&dringp->lock);
2401 		return (EINVAL);
2402 	}
2403 
2404 	/* get the channel handle */
2405 	ldcp = dringp->ldcp;
2406 
2407 	copy_size = (start <= end) ? (((end - start) + 1) * dringp->dsize) :
2408 	    ((dringp->length - start) * dringp->dsize);
2409 
2410 	/* Calculate the relative offset for the first desc */
2411 	soff = (start * dringp->dsize);
2412 
2413 	/* copy to/from remote from/to local memory */
2414 	D1(ldcp->id, "i_ldc_dring_acquire_release: c1 off=0x%llx sz=0x%llx\n",
2415 	    soff, copy_size);
2416 	err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl,
2417 	    direction, soff, copy_size);
2418 	if (err) {
2419 		DWARN(ldcp->id,
2420 		    "i_ldc_dring_acquire_release: copy failed\n");
2421 		mutex_exit(&dringp->lock);
2422 		return (err);
2423 	}
2424 
2425 	/* do the balance */
2426 	if (start > end) {
2427 		copy_size = ((end + 1) * dringp->dsize);
2428 		soff = 0;
2429 
2430 		/* copy to/from remote from/to local memory */
2431 		D1(ldcp->id, "i_ldc_dring_acquire_release: c2 "
2432 		    "off=0x%llx sz=0x%llx\n", soff, copy_size);
2433 		err = i_ldc_mem_acquire_release((ldc_mem_handle_t)dringp->mhdl,
2434 		    direction, soff, copy_size);
2435 		if (err) {
2436 			DWARN(ldcp->id,
2437 			    "i_ldc_dring_acquire_release: copy failed\n");
2438 			mutex_exit(&dringp->lock);
2439 			return (err);
2440 		}
2441 	}
2442 
2443 	mutex_exit(&dringp->lock);
2444 
2445 	return (0);
2446 }
2447 
2448 /*
2449  * Ensure that the contents in the local dring are consistent
2450  * with the contents if of remote dring
2451  */
2452 int
ldc_mem_dring_acquire(ldc_dring_handle_t dhandle,uint64_t start,uint64_t end)2453 ldc_mem_dring_acquire(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end)
2454 {
2455 	return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_IN, start, end));
2456 }
2457 
2458 /*
2459  * Ensure that the contents in the remote dring are consistent
2460  * with the contents if of local dring
2461  */
2462 int
ldc_mem_dring_release(ldc_dring_handle_t dhandle,uint64_t start,uint64_t end)2463 ldc_mem_dring_release(ldc_dring_handle_t dhandle, uint64_t start, uint64_t end)
2464 {
2465 	return (i_ldc_dring_acquire_release(dhandle, LDC_COPY_OUT, start, end));
2466 }
2467