1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25/*
26 * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
27 */
28
29#include <sys/types.h>
30#include <sys/param.h>
31#include <sys/conf.h>
32#include <sys/ddi.h>
33#include <sys/sunddi.h>
34#include <sys/ddi_impldefs.h>
35#include <sys/cmn_err.h>
36#include <sys/kmem.h>
37#include <sys/vmem.h>
38#include <sys/sysmacros.h>
39
40#include <sys/ddidmareq.h>
41#include <sys/sysiosbus.h>
42#include <sys/iommu.h>
43#include <sys/iocache.h>
44#include <sys/dvma.h>
45
46#include <vm/as.h>
47#include <vm/hat.h>
48#include <vm/page.h>
49#include <vm/hat_sfmmu.h>
50#include <sys/machparam.h>
51#include <sys/machsystm.h>
52#include <sys/vmsystm.h>
53#include <sys/iommutsb.h>
54
55/* Useful debugging Stuff */
56#include <sys/nexusdebug.h>
57#include <sys/debug.h>
58/* Bitfield debugging definitions for this file */
59#define	IOMMU_GETDVMAPAGES_DEBUG	0x1
60#define	IOMMU_DMAMAP_DEBUG		0x2
61#define	IOMMU_DMAMCTL_DEBUG		0x4
62#define	IOMMU_DMAMCTL_SYNC_DEBUG	0x8
63#define	IOMMU_DMAMCTL_HTOC_DEBUG	0x10
64#define	IOMMU_DMAMCTL_KVADDR_DEBUG	0x20
65#define	IOMMU_DMAMCTL_GETERR_DEBUG	0x400
66#define	IOMMU_DMAMCTL_DMA_FREE_DEBUG	0x1000
67#define	IOMMU_REGISTERS_DEBUG		0x2000
68#define	IOMMU_DMA_SETUP_DEBUG		0x4000
69#define	IOMMU_DMA_UNBINDHDL_DEBUG	0x8000
70#define	IOMMU_DMA_BINDHDL_DEBUG		0x10000
71#define	IOMMU_DMA_WIN_DEBUG		0x20000
72#define	IOMMU_DMA_ALLOCHDL_DEBUG	0x40000
73#define	IOMMU_DMA_LIM_SETUP_DEBUG	0x80000
74#define	IOMMU_FASTDMA_RESERVE		0x100000
75#define	IOMMU_FASTDMA_LOAD		0x200000
76#define	IOMMU_INTER_INTRA_XFER		0x400000
77#define	IOMMU_TTE			0x800000
78#define	IOMMU_TLB			0x1000000
79#define	IOMMU_FASTDMA_SYNC		0x2000000
80
81/* Turn on if you need to keep track of outstanding IOMMU usage */
82/* #define	IO_MEMUSAGE */
83/* Turn on to debug IOMMU unmapping code */
84/* #define	IO_MEMDEBUG */
85
86static struct dvma_ops iommu_dvma_ops = {
87	DVMAO_REV,
88	iommu_dvma_kaddr_load,
89	iommu_dvma_unload,
90	iommu_dvma_sync
91};
92
93extern void *sbusp;		/* sbus soft state hook */
94
95#define	DVMA_MAX_CACHE	65536
96
97/*
98 * This is the number of pages that a mapping request needs before we force
99 * the TLB flush code to use diagnostic registers.  This value was determined
100 * through a series of test runs measuring dma mapping settup performance.
101 */
102int tlb_flush_using_diag = 16;
103
104int sysio_iommu_tsb_sizes[] = {
105	IOMMU_TSB_SIZE_8M,
106	IOMMU_TSB_SIZE_16M,
107	IOMMU_TSB_SIZE_32M,
108	IOMMU_TSB_SIZE_64M,
109	IOMMU_TSB_SIZE_128M,
110	IOMMU_TSB_SIZE_256M,
111	IOMMU_TSB_SIZE_512M,
112	IOMMU_TSB_SIZE_1G
113};
114
115static int iommu_map_window(ddi_dma_impl_t *, off_t, size_t);
116
117int
118iommu_init(struct sbus_soft_state *softsp, caddr_t address)
119{
120	int i;
121	char name[40];
122
123#ifdef DEBUG
124	debug_info = 1;
125#endif
126
127	/*
128	 * Simply add each registers offset to the base address
129	 * to calculate the already mapped virtual address of
130	 * the device register...
131	 *
132	 * define a macro for the pointer arithmetic; all registers
133	 * are 64 bits wide and are defined as uint64_t's.
134	 */
135
136#define	REG_ADDR(b, o)	(uint64_t *)((caddr_t)(b) + (o))
137
138	softsp->iommu_ctrl_reg = REG_ADDR(address, OFF_IOMMU_CTRL_REG);
139	softsp->tsb_base_addr = REG_ADDR(address, OFF_TSB_BASE_ADDR);
140	softsp->iommu_flush_reg = REG_ADDR(address, OFF_IOMMU_FLUSH_REG);
141	softsp->iommu_tlb_tag = REG_ADDR(address, OFF_IOMMU_TLB_TAG);
142	softsp->iommu_tlb_data = REG_ADDR(address, OFF_IOMMU_TLB_DATA);
143
144#undef REG_ADDR
145
146	mutex_init(&softsp->dma_pool_lock, NULL, MUTEX_DEFAULT, NULL);
147	mutex_init(&softsp->intr_poll_list_lock, NULL, MUTEX_DEFAULT, NULL);
148
149	/* Set up the DVMA resource sizes */
150	if ((softsp->iommu_tsb_cookie = iommu_tsb_alloc(softsp->upa_id)) ==
151	    IOMMU_TSB_COOKIE_NONE) {
152		cmn_err(CE_WARN, "%s%d: Unable to retrieve IOMMU array.",
153		    ddi_driver_name(softsp->dip),
154		    ddi_get_instance(softsp->dip));
155		return (DDI_FAILURE);
156	}
157	softsp->soft_tsb_base_addr =
158	    iommu_tsb_cookie_to_va(softsp->iommu_tsb_cookie);
159	softsp->iommu_dvma_size =
160	    iommu_tsb_cookie_to_size(softsp->iommu_tsb_cookie) <<
161	    IOMMU_TSB_TO_RNG;
162	softsp->iommu_dvma_base = (ioaddr_t)
163	    (0 - (ioaddr_t)softsp->iommu_dvma_size);
164
165	(void) snprintf(name, sizeof (name), "%s%d_dvma",
166	    ddi_driver_name(softsp->dip), ddi_get_instance(softsp->dip));
167
168	/*
169	 * Initialize the DVMA vmem arena.
170	 */
171	softsp->dvma_arena = vmem_create(name,
172	    (void *)(uintptr_t)softsp->iommu_dvma_base,
173	    softsp->iommu_dvma_size, PAGESIZE, NULL, NULL, NULL,
174	    DVMA_MAX_CACHE, VM_SLEEP);
175
176	/* Set the limit for dvma_reserve() to 1/2 of the total dvma space */
177	softsp->dma_reserve = iommu_btop(softsp->iommu_dvma_size >> 1);
178
179#if defined(DEBUG) && defined(IO_MEMUSAGE)
180	mutex_init(&softsp->iomemlock, NULL, MUTEX_DEFAULT, NULL);
181	softsp->iomem = (struct io_mem_list *)0;
182#endif /* DEBUG && IO_MEMUSAGE */
183	/*
184	 * Get the base address of the TSB table and store it in the hardware
185	 */
186
187	/*
188	 * We plan on the PROM flushing all TLB entries.  If this is not the
189	 * case, this is where we should flush the hardware TLB.
190	 */
191
192	/* Set the IOMMU registers */
193	(void) iommu_resume_init(softsp);
194
195	/* check the convenient copy of TSB base, and flush write buffers */
196	if (*softsp->tsb_base_addr !=
197	    va_to_pa((caddr_t)softsp->soft_tsb_base_addr)) {
198		iommu_tsb_free(softsp->iommu_tsb_cookie);
199		return (DDI_FAILURE);
200	}
201
202	softsp->sbus_io_lo_pfn = UINT32_MAX;
203	softsp->sbus_io_hi_pfn = 0;
204	for (i = 0; i < sysio_pd_getnrng(softsp->dip); i++) {
205		struct rangespec *rangep;
206		uint64_t addr;
207		pfn_t hipfn, lopfn;
208
209		rangep = sysio_pd_getrng(softsp->dip, i);
210		addr = (uint64_t)((uint64_t)rangep->rng_bustype << 32);
211		addr |= (uint64_t)rangep->rng_offset;
212		lopfn = (pfn_t)(addr >> MMU_PAGESHIFT);
213		addr += (uint64_t)(rangep->rng_size - 1);
214		hipfn = (pfn_t)(addr >> MMU_PAGESHIFT);
215
216		softsp->sbus_io_lo_pfn = (lopfn < softsp->sbus_io_lo_pfn) ?
217		    lopfn : softsp->sbus_io_lo_pfn;
218
219		softsp->sbus_io_hi_pfn = (hipfn > softsp->sbus_io_hi_pfn) ?
220		    hipfn : softsp->sbus_io_hi_pfn;
221	}
222
223	DPRINTF(IOMMU_REGISTERS_DEBUG, ("IOMMU Control reg: %p IOMMU TSB "
224	    "base reg: %p IOMMU flush reg: %p TSB base addr %p\n",
225	    (void *)softsp->iommu_ctrl_reg, (void *)softsp->tsb_base_addr,
226	    (void *)softsp->iommu_flush_reg,
227	    (void *)softsp->soft_tsb_base_addr));
228
229	return (DDI_SUCCESS);
230}
231
232/*
233 * function to uninitialize the iommu and release the tsb back to
234 * the spare pool.  See startup.c for tsb spare management.
235 */
236
237int
238iommu_uninit(struct sbus_soft_state *softsp)
239{
240	vmem_destroy(softsp->dvma_arena);
241
242	/* flip off the IOMMU enable switch */
243	*softsp->iommu_ctrl_reg &=
244	    (TSB_SIZE << TSB_SIZE_SHIFT | IOMMU_DISABLE);
245
246	iommu_tsb_free(softsp->iommu_tsb_cookie);
247
248	return (DDI_SUCCESS);
249}
250
251/*
252 * Initialize iommu hardware registers when the system is being resumed.
253 * (Subset of iommu_init())
254 */
255int
256iommu_resume_init(struct sbus_soft_state *softsp)
257{
258	int i;
259	uint_t tsb_size;
260	uint_t tsb_bytes;
261
262	/*
263	 * Reset the base address of the TSB table in the hardware
264	 */
265	*softsp->tsb_base_addr = va_to_pa((caddr_t)softsp->soft_tsb_base_addr);
266
267	/*
268	 * Figure out the correct size of the IOMMU TSB entries.  If we
269	 * end up with a size smaller than that needed for 8M of IOMMU
270	 * space, default the size to 8M.  XXX We could probably panic here
271	 */
272	i = sizeof (sysio_iommu_tsb_sizes) / sizeof (sysio_iommu_tsb_sizes[0])
273	    - 1;
274
275	tsb_bytes = iommu_tsb_cookie_to_size(softsp->iommu_tsb_cookie);
276
277	while (i > 0) {
278		if (tsb_bytes >= sysio_iommu_tsb_sizes[i])
279			break;
280		i--;
281	}
282
283	tsb_size = i;
284
285	/* OK, lets flip the "on" switch of the IOMMU */
286	*softsp->iommu_ctrl_reg = (uint64_t)(tsb_size << TSB_SIZE_SHIFT
287	    | IOMMU_ENABLE | IOMMU_DIAG_ENABLE);
288
289	return (DDI_SUCCESS);
290}
291
292void
293iommu_tlb_flush(struct sbus_soft_state *softsp, ioaddr_t addr, pgcnt_t npages)
294{
295	volatile uint64_t tmpreg;
296	volatile uint64_t *vaddr_reg, *valid_bit_reg;
297	ioaddr_t hiaddr, ioaddr;
298	int i, do_flush = 0;
299
300	if (npages == 1) {
301		*softsp->iommu_flush_reg = (uint64_t)addr;
302		tmpreg = *softsp->sbus_ctrl_reg;
303		return;
304	}
305
306	hiaddr = addr + (ioaddr_t)(npages * IOMMU_PAGESIZE);
307	for (i = 0, vaddr_reg = softsp->iommu_tlb_tag,
308	    valid_bit_reg = softsp->iommu_tlb_data;
309	    i < IOMMU_TLB_ENTRIES; i++, vaddr_reg++, valid_bit_reg++) {
310		tmpreg = *vaddr_reg;
311		ioaddr = (ioaddr_t)((tmpreg & IOMMU_TLBTAG_VA_MASK) <<
312		    IOMMU_TLBTAG_VA_SHIFT);
313
314		DPRINTF(IOMMU_TLB, ("Vaddr reg 0x%p, "
315		    "TLB vaddr reg %lx, IO addr 0x%x "
316		    "Base addr 0x%x, Hi addr 0x%x\n",
317		    (void *)vaddr_reg, tmpreg, ioaddr, addr, hiaddr));
318
319		if (ioaddr >= addr && ioaddr <= hiaddr) {
320			tmpreg = *valid_bit_reg;
321
322			DPRINTF(IOMMU_TLB, ("Valid reg addr 0x%p, "
323			    "TLB valid reg %lx\n",
324			    (void *)valid_bit_reg, tmpreg));
325
326			if (tmpreg & IOMMU_TLB_VALID) {
327				*softsp->iommu_flush_reg = (uint64_t)ioaddr;
328				do_flush = 1;
329			}
330		}
331	}
332
333	if (do_flush)
334		tmpreg = *softsp->sbus_ctrl_reg;
335}
336
337
338/*
339 * Shorthand defines
340 */
341
342#define	ALO		dma_lim->dlim_addr_lo
343#define	AHI		dma_lim->dlim_addr_hi
344#define	OBJSIZE		dmareq->dmar_object.dmao_size
345#define	IOTTE_NDX(vaddr, base) (base + \
346		(int)(iommu_btop((vaddr & ~IOMMU_PAGEMASK) - \
347		softsp->iommu_dvma_base)))
348/*
349 * If DDI_DMA_PARTIAL flag is set and the request is for
350 * less than MIN_DVMA_WIN_SIZE, it's not worth the hassle so
351 * we turn off the DDI_DMA_PARTIAL flag
352 */
353#define	MIN_DVMA_WIN_SIZE	(128)
354
355/* ARGSUSED */
356void
357iommu_remove_mappings(ddi_dma_impl_t *mp)
358{
359#if defined(DEBUG) && defined(IO_MEMDEBUG)
360	pgcnt_t npages;
361	ioaddr_t ioaddr;
362	volatile uint64_t *iotte_ptr;
363	ioaddr_t ioaddr = mp->dmai_mapping & ~IOMMU_PAGEOFFSET;
364	pgcnt_t npages = mp->dmai_ndvmapages;
365	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
366	struct sbus_soft_state *softsp = mppriv->softsp;
367
368#if defined(IO_MEMUSAGE)
369	struct io_mem_list **prevp, *walk;
370#endif /* DEBUG && IO_MEMUSAGE */
371
372	ASSERT(softsp != NULL);
373	/*
374	 * Run thru the mapped entries and free 'em
375	 */
376
377	ioaddr = mp->dmai_mapping & ~IOMMU_PAGEOFFSET;
378	npages = mp->dmai_ndvmapages;
379
380#if defined(IO_MEMUSAGE)
381	mutex_enter(&softsp->iomemlock);
382	prevp = &softsp->iomem;
383	walk = softsp->iomem;
384
385	while (walk) {
386		if (walk->ioaddr == ioaddr) {
387			*prevp = walk->next;
388			break;
389		}
390
391		prevp = &walk->next;
392		walk = walk->next;
393	}
394	mutex_exit(&softsp->iomemlock);
395
396	kmem_free(walk->pfn, sizeof (pfn_t) * (npages + 1));
397	kmem_free(walk, sizeof (struct io_mem_list));
398#endif /* IO_MEMUSAGE */
399
400	iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr);
401
402	while (npages) {
403		DPRINTF(IOMMU_DMAMCTL_DEBUG,
404		    ("dma_mctl: freeing ioaddr %x iotte %p\n",
405		    ioaddr, iotte_ptr));
406		*iotte_ptr = (uint64_t)0;	/* unload tte */
407		iommu_tlb_flush(softsp, ioaddr, 1);
408		npages--;
409		ioaddr += IOMMU_PAGESIZE;
410		iotte_ptr++;
411	}
412#endif /* DEBUG && IO_MEMDEBUG */
413}
414
415
416int
417iommu_create_vaddr_mappings(ddi_dma_impl_t *mp, uintptr_t addr)
418{
419	pfn_t pfn;
420	struct as *as = NULL;
421	pgcnt_t npages;
422	ioaddr_t ioaddr;
423	uint_t offset;
424	volatile uint64_t *iotte_ptr;
425	uint64_t tmp_iotte_flag;
426	int rval = DDI_DMA_MAPPED;
427	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
428	struct sbus_soft_state *softsp = mppriv->softsp;
429	int diag_tlb_flush;
430#if defined(DEBUG) && defined(IO_MEMUSAGE)
431	struct io_mem_list *iomemp;
432	pfn_t *pfnp;
433#endif /* DEBUG && IO_MEMUSAGE */
434
435	ASSERT(softsp != NULL);
436
437	/* Set Valid and Cache for mem xfer */
438	tmp_iotte_flag = IOTTE_VALID | IOTTE_CACHE | IOTTE_WRITE | IOTTE_STREAM;
439
440	offset = (uint_t)(mp->dmai_mapping & IOMMU_PAGEOFFSET);
441	npages = iommu_btopr(mp->dmai_size + offset);
442	ioaddr = (ioaddr_t)(mp->dmai_mapping & ~IOMMU_PAGEOFFSET);
443	iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr);
444	diag_tlb_flush = npages > tlb_flush_using_diag ? 1 : 0;
445
446	as = mp->dmai_object.dmao_obj.virt_obj.v_as;
447	if (as == NULL)
448		as = &kas;
449
450	/*
451	 * Set the per object bits of the TTE here. We optimize this for
452	 * the memory case so that the while loop overhead is minimal.
453	 */
454	/* Turn on NOSYNC if we need consistent mem */
455	if (mp->dmai_rflags & DDI_DMA_CONSISTENT) {
456		mp->dmai_rflags |= DMP_NOSYNC;
457		tmp_iotte_flag ^= IOTTE_STREAM;
458	/* Set streaming mode if not consistent mem */
459	} else if (softsp->stream_buf_off) {
460		tmp_iotte_flag ^= IOTTE_STREAM;
461	}
462
463#if defined(DEBUG) && defined(IO_MEMUSAGE)
464	iomemp = kmem_alloc(sizeof (struct io_mem_list), KM_SLEEP);
465	iomemp->rdip = mp->dmai_rdip;
466	iomemp->ioaddr = ioaddr;
467	iomemp->addr = addr;
468	iomemp->npages = npages;
469	pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) * (npages + 1),
470	    KM_SLEEP);
471#endif /* DEBUG && IO_MEMUSAGE */
472	/*
473	 * Grab the mappings from the dmmu and stick 'em into the
474	 * iommu.
475	 */
476	ASSERT(npages != 0);
477
478	/* If we're going to flush the TLB using diag mode, do it now. */
479	if (diag_tlb_flush)
480		iommu_tlb_flush(softsp, ioaddr, npages);
481
482	do {
483		uint64_t iotte_flag = tmp_iotte_flag;
484
485		/*
486		 * Fetch the pfn for the DMA object
487		 */
488
489		ASSERT(as);
490		pfn = hat_getpfnum(as->a_hat, (caddr_t)addr);
491		ASSERT(pfn != PFN_INVALID);
492
493		if (!pf_is_memory(pfn)) {
494			/* DVMA'ing to IO space */
495
496			/* Turn off cache bit if set */
497			if (iotte_flag & IOTTE_CACHE)
498				iotte_flag ^= IOTTE_CACHE;
499
500			/* Turn off stream bit if set */
501			if (iotte_flag & IOTTE_STREAM)
502				iotte_flag ^= IOTTE_STREAM;
503
504			if (IS_INTRA_SBUS(softsp, pfn)) {
505				/* Intra sbus transfer */
506
507				/* Turn on intra flag */
508				iotte_flag |= IOTTE_INTRA;
509
510				DPRINTF(IOMMU_INTER_INTRA_XFER, (
511				    "Intra xfer pfnum %lx TTE %lx\n",
512				    pfn, iotte_flag));
513			} else {
514				if (pf_is_dmacapable(pfn) == 1) {
515					/*EMPTY*/
516					DPRINTF(IOMMU_INTER_INTRA_XFER,
517					    ("Inter xfer pfnum %lx "
518					    "tte hi %lx\n",
519					    pfn, iotte_flag));
520				} else {
521					rval = DDI_DMA_NOMAPPING;
522#if defined(DEBUG) && defined(IO_MEMDEBUG)
523					goto bad;
524#endif /* DEBUG && IO_MEMDEBUG */
525				}
526			}
527		}
528		addr += IOMMU_PAGESIZE;
529
530		DPRINTF(IOMMU_TTE, ("vaddr mapping: tte index %p pfn %lx "
531		    "tte flag %lx addr %lx ioaddr %x\n",
532		    (void *)iotte_ptr, pfn, iotte_flag, addr, ioaddr));
533
534		/* Flush the IOMMU TLB before loading a new mapping */
535		if (!diag_tlb_flush)
536			iommu_tlb_flush(softsp, ioaddr, 1);
537
538		/* Set the hardware IO TTE */
539		*iotte_ptr = ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag;
540
541		ioaddr += IOMMU_PAGESIZE;
542		npages--;
543		iotte_ptr++;
544#if defined(DEBUG) && defined(IO_MEMUSAGE)
545		*pfnp = pfn;
546		pfnp++;
547#endif /* DEBUG && IO_MEMUSAGE */
548	} while (npages != 0);
549
550#if defined(DEBUG) && defined(IO_MEMUSAGE)
551	mutex_enter(&softsp->iomemlock);
552	iomemp->next = softsp->iomem;
553	softsp->iomem = iomemp;
554	mutex_exit(&softsp->iomemlock);
555#endif /* DEBUG && IO_MEMUSAGE */
556
557	return (rval);
558
559#if defined(DEBUG) && defined(IO_MEMDEBUG)
560bad:
561	/* If we fail a mapping, free up any mapping resources used */
562	iommu_remove_mappings(mp);
563	return (rval);
564#endif /* DEBUG && IO_MEMDEBUG */
565}
566
567
568int
569iommu_create_pp_mappings(ddi_dma_impl_t *mp, page_t *pp, page_t **pplist)
570{
571	pfn_t pfn;
572	pgcnt_t npages;
573	ioaddr_t ioaddr;
574	uint_t offset;
575	volatile uint64_t *iotte_ptr;
576	uint64_t tmp_iotte_flag;
577	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
578	struct sbus_soft_state *softsp = mppriv->softsp;
579	int diag_tlb_flush;
580#if defined(DEBUG) && defined(IO_MEMUSAGE)
581	struct io_mem_list *iomemp;
582	pfn_t *pfnp;
583#endif /* DEBUG && IO_MEMUSAGE */
584	int rval = DDI_DMA_MAPPED;
585
586	/* Set Valid and Cache for mem xfer */
587	tmp_iotte_flag = IOTTE_VALID | IOTTE_CACHE | IOTTE_WRITE | IOTTE_STREAM;
588
589	ASSERT(softsp != NULL);
590
591	offset = (uint_t)(mp->dmai_mapping & IOMMU_PAGEOFFSET);
592	npages = iommu_btopr(mp->dmai_size + offset);
593	ioaddr = (ioaddr_t)(mp->dmai_mapping & ~IOMMU_PAGEOFFSET);
594	iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr);
595	diag_tlb_flush = npages > tlb_flush_using_diag ? 1 : 0;
596
597	/*
598	 * Set the per object bits of the TTE here. We optimize this for
599	 * the memory case so that the while loop overhead is minimal.
600	 */
601	if (mp->dmai_rflags & DDI_DMA_CONSISTENT) {
602		/* Turn on NOSYNC if we need consistent mem */
603		mp->dmai_rflags |= DMP_NOSYNC;
604		tmp_iotte_flag ^= IOTTE_STREAM;
605	} else if (softsp->stream_buf_off) {
606		/* Set streaming mode if not consistent mem */
607		tmp_iotte_flag ^= IOTTE_STREAM;
608	}
609
610#if defined(DEBUG) && defined(IO_MEMUSAGE)
611	iomemp = kmem_alloc(sizeof (struct io_mem_list), KM_SLEEP);
612	iomemp->rdip = mp->dmai_rdip;
613	iomemp->ioaddr = ioaddr;
614	iomemp->npages = npages;
615	pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) * (npages + 1),
616	    KM_SLEEP);
617#endif /* DEBUG && IO_MEMUSAGE */
618	/*
619	 * Grab the mappings from the dmmu and stick 'em into the
620	 * iommu.
621	 */
622	ASSERT(npages != 0);
623
624	/* If we're going to flush the TLB using diag mode, do it now. */
625	if (diag_tlb_flush)
626		iommu_tlb_flush(softsp, ioaddr, npages);
627
628	do {
629		uint64_t iotte_flag;
630
631		iotte_flag = tmp_iotte_flag;
632
633		if (pp != NULL) {
634			pfn = pp->p_pagenum;
635			pp = pp->p_next;
636		} else {
637			pfn = (*pplist)->p_pagenum;
638			pplist++;
639		}
640
641		DPRINTF(IOMMU_TTE, ("pp mapping TTE index %p pfn %lx "
642		    "tte flag %lx ioaddr %x\n", (void *)iotte_ptr,
643		    pfn, iotte_flag, ioaddr));
644
645		/* Flush the IOMMU TLB before loading a new mapping */
646		if (!diag_tlb_flush)
647			iommu_tlb_flush(softsp, ioaddr, 1);
648
649		/* Set the hardware IO TTE */
650		*iotte_ptr = ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag;
651
652		ioaddr += IOMMU_PAGESIZE;
653		npages--;
654		iotte_ptr++;
655
656#if defined(DEBUG) && defined(IO_MEMUSAGE)
657		*pfnp = pfn;
658		pfnp++;
659#endif /* DEBUG && IO_MEMUSAGE */
660
661	} while (npages != 0);
662
663#if defined(DEBUG) && defined(IO_MEMUSAGE)
664	mutex_enter(&softsp->iomemlock);
665	iomemp->next = softsp->iomem;
666	softsp->iomem = iomemp;
667	mutex_exit(&softsp->iomemlock);
668#endif /* DEBUG && IO_MEMUSAGE */
669
670	return (rval);
671}
672
673
674int
675iommu_dma_lim_setup(dev_info_t *dip, dev_info_t *rdip,
676    struct sbus_soft_state *softsp, uint_t *burstsizep, uint_t burstsize64,
677    uint_t *minxferp, uint_t dma_flags)
678{
679	struct regspec *rp;
680
681	/* Take care of 64 bit limits. */
682	if (!(dma_flags & DDI_DMA_SBUS_64BIT)) {
683		/*
684		 * return burst size for 32-bit mode
685		 */
686		*burstsizep &= softsp->sbus_burst_sizes;
687		return (DDI_FAILURE);
688	}
689
690	/*
691	 * check if SBus supports 64 bit and if caller
692	 * is child of SBus. No support through bridges
693	 */
694	if (!softsp->sbus64_burst_sizes || (ddi_get_parent(rdip) != dip)) {
695		/*
696		 * SBus doesn't support it or bridge. Do 32-bit
697		 * xfers
698		 */
699		*burstsizep &= softsp->sbus_burst_sizes;
700		return (DDI_FAILURE);
701	}
702
703	rp = ddi_rnumber_to_regspec(rdip, 0);
704	if (rp == NULL) {
705		*burstsizep &= softsp->sbus_burst_sizes;
706		return (DDI_FAILURE);
707	}
708
709	/* Check for old-style 64 bit burstsizes */
710	if (burstsize64 & SYSIO64_BURST_MASK) {
711		/* Scale back burstsizes if Necessary */
712		*burstsizep &= (softsp->sbus64_burst_sizes |
713		    softsp->sbus_burst_sizes);
714	} else {
715		/* Get the 64 bit burstsizes. */
716		*burstsizep = burstsize64;
717
718		/* Scale back burstsizes if Necessary */
719		*burstsizep &= (softsp->sbus64_burst_sizes >>
720		    SYSIO64_BURST_SHIFT);
721	}
722
723	/*
724	 * Set the largest value of the smallest burstsize that the
725	 * device or the bus can manage.
726	 */
727	*minxferp = MAX(*minxferp,
728	    (1 << (ddi_ffs(softsp->sbus64_burst_sizes) - 1)));
729
730	return (DDI_SUCCESS);
731}
732
733
734int
735iommu_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
736    ddi_dma_attr_t *dma_attr, int (*waitfp)(caddr_t), caddr_t arg,
737    ddi_dma_handle_t *handlep)
738{
739	ioaddr_t addrlow, addrhigh, segalign;
740	ddi_dma_impl_t *mp;
741	struct dma_impl_priv *mppriv;
742	struct sbus_soft_state *softsp = (struct sbus_soft_state *)
743	    ddi_get_soft_state(sbusp, ddi_get_instance(dip));
744
745	/*
746	 * Setup dma burstsizes and min-xfer counts.
747	 */
748	(void) iommu_dma_lim_setup(dip, rdip, softsp,
749	    &dma_attr->dma_attr_burstsizes,
750	    dma_attr->dma_attr_burstsizes, &dma_attr->dma_attr_minxfer,
751	    dma_attr->dma_attr_flags);
752
753	if (dma_attr->dma_attr_burstsizes == 0)
754		return (DDI_DMA_BADATTR);
755
756	addrlow = (ioaddr_t)dma_attr->dma_attr_addr_lo;
757	addrhigh = (ioaddr_t)dma_attr->dma_attr_addr_hi;
758	segalign = (ioaddr_t)dma_attr->dma_attr_seg;
759
760	/*
761	 * Check sanity for hi and lo address limits
762	 */
763	if ((addrhigh <= addrlow) ||
764	    (addrhigh < (ioaddr_t)softsp->iommu_dvma_base)) {
765		return (DDI_DMA_BADATTR);
766	}
767	if (dma_attr->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL)
768		return (DDI_DMA_BADATTR);
769
770	mppriv = kmem_zalloc(sizeof (*mppriv),
771	    (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
772
773	if (mppriv == NULL) {
774		if (waitfp != DDI_DMA_DONTWAIT) {
775			ddi_set_callback(waitfp, arg,
776			    &softsp->dvma_call_list_id);
777		}
778		return (DDI_DMA_NORESOURCES);
779	}
780	mp = (ddi_dma_impl_t *)mppriv;
781
782	DPRINTF(IOMMU_DMA_ALLOCHDL_DEBUG, ("dma_allochdl: (%s) handle %p "
783	    "hi %x lo %x min %x burst %x\n",
784	    ddi_get_name(dip), (void *)mp, addrhigh, addrlow,
785	    dma_attr->dma_attr_minxfer, dma_attr->dma_attr_burstsizes));
786
787	mp->dmai_rdip = rdip;
788	mp->dmai_minxfer = (uint_t)dma_attr->dma_attr_minxfer;
789	mp->dmai_burstsizes = (uint_t)dma_attr->dma_attr_burstsizes;
790	mp->dmai_attr = *dma_attr;
791	/* See if the DMA engine has any limit restrictions. */
792	if (segalign == (ioaddr_t)UINT32_MAX &&
793	    addrhigh == (ioaddr_t)UINT32_MAX &&
794	    (dma_attr->dma_attr_align <= IOMMU_PAGESIZE) && addrlow == 0) {
795		mp->dmai_rflags |= DMP_NOLIMIT;
796	}
797	mppriv->softsp = softsp;
798	mppriv->phys_sync_flag = va_to_pa((caddr_t)&mppriv->sync_flag);
799
800	*handlep = (ddi_dma_handle_t)mp;
801	return (DDI_SUCCESS);
802}
803
804/*ARGSUSED*/
805int
806iommu_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
807{
808	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)handle;
809	struct sbus_soft_state *softsp = mppriv->softsp;
810	ASSERT(softsp != NULL);
811
812	kmem_free(mppriv, sizeof (*mppriv));
813
814	if (softsp->dvma_call_list_id != 0) {
815		ddi_run_callback(&softsp->dvma_call_list_id);
816	}
817	return (DDI_SUCCESS);
818}
819
820static int
821check_dma_attr(struct ddi_dma_req *dmareq, ddi_dma_attr_t *dma_attr,
822    uint32_t *size)
823{
824	ioaddr_t addrlow;
825	ioaddr_t addrhigh;
826	uint32_t segalign;
827	uint32_t smask;
828
829	smask = *size - 1;
830	segalign = dma_attr->dma_attr_seg;
831	if (smask > segalign) {
832		if ((dmareq->dmar_flags & DDI_DMA_PARTIAL) == 0)
833			return (DDI_DMA_TOOBIG);
834		*size = segalign + 1;
835	}
836	addrlow = (ioaddr_t)dma_attr->dma_attr_addr_lo;
837	addrhigh = (ioaddr_t)dma_attr->dma_attr_addr_hi;
838	if (addrlow + smask > addrhigh || addrlow + smask < addrlow) {
839		if (!((addrlow + dmareq->dmar_object.dmao_size == 0) &&
840		    (addrhigh == (ioaddr_t)-1))) {
841			if ((dmareq->dmar_flags & DDI_DMA_PARTIAL) == 0)
842				return (DDI_DMA_TOOBIG);
843			*size = MIN(addrhigh - addrlow + 1, *size);
844		}
845	}
846	return (DDI_DMA_MAPOK);
847}
848
849int
850iommu_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
851    ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
852    ddi_dma_cookie_t *cp, uint_t *ccountp)
853{
854	page_t *pp;
855	uint32_t size;
856	ioaddr_t ioaddr;
857	uint_t offset;
858	uintptr_t addr = 0;
859	pgcnt_t npages;
860	int rval;
861	ddi_dma_attr_t *dma_attr;
862	struct sbus_soft_state *softsp;
863	struct page **pplist = NULL;
864	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
865	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
866
867#ifdef lint
868	dip = dip;
869	rdip = rdip;
870#endif
871
872	if (mp->dmai_inuse)
873		return (DDI_DMA_INUSE);
874
875	dma_attr = &mp->dmai_attr;
876	size = (uint32_t)dmareq->dmar_object.dmao_size;
877	if (!(mp->dmai_rflags & DMP_NOLIMIT)) {
878		rval = check_dma_attr(dmareq, dma_attr, &size);
879		if (rval != DDI_DMA_MAPOK)
880			return (rval);
881	}
882	mp->dmai_inuse = 1;
883	mp->dmai_offset = 0;
884	mp->dmai_rflags = (dmareq->dmar_flags & DMP_DDIFLAGS) |
885	    (mp->dmai_rflags & DMP_NOLIMIT);
886
887	switch (dmareq->dmar_object.dmao_type) {
888	case DMA_OTYP_VADDR:
889	case DMA_OTYP_BUFVADDR:
890		addr = (uintptr_t)dmareq->dmar_object.dmao_obj.virt_obj.v_addr;
891		offset = addr & IOMMU_PAGEOFFSET;
892		pplist = dmareq->dmar_object.dmao_obj.virt_obj.v_priv;
893		npages = iommu_btopr(OBJSIZE + offset);
894
895		DPRINTF(IOMMU_DMAMAP_DEBUG, ("dma_map vaddr: %lx pages "
896		    "req addr %lx off %x OBJSIZE %x\n",
897		    npages, addr, offset, OBJSIZE));
898
899		/* We don't need the addr anymore if we have a shadow list */
900		if (pplist != NULL)
901			addr = (uintptr_t)NULL;
902		pp = NULL;
903		break;
904
905	case DMA_OTYP_PAGES:
906		pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
907		offset = dmareq->dmar_object.dmao_obj.pp_obj.pp_offset;
908		npages = iommu_btopr(OBJSIZE + offset);
909		break;
910
911	case DMA_OTYP_PADDR:
912	default:
913		/*
914		 * Not a supported type for this implementation
915		 */
916		rval = DDI_DMA_NOMAPPING;
917		goto bad;
918	}
919
920	/* Get our soft state once we know we're mapping an object. */
921	softsp = mppriv->softsp;
922	ASSERT(softsp != NULL);
923
924	if (mp->dmai_rflags & DDI_DMA_PARTIAL) {
925		if (size != OBJSIZE) {
926			/*
927			 * If the request is for partial mapping arrangement,
928			 * the device has to be able to address at least the
929			 * size of the window we are establishing.
930			 */
931			if (size < iommu_ptob(MIN_DVMA_WIN_SIZE)) {
932				rval = DDI_DMA_NOMAPPING;
933				goto bad;
934			}
935			npages = iommu_btopr(size + offset);
936		}
937		/*
938		 * If the size requested is less than a moderate amt,
939		 * skip the partial mapping stuff- it's not worth the
940		 * effort.
941		 */
942		if (npages > MIN_DVMA_WIN_SIZE) {
943			npages = MIN_DVMA_WIN_SIZE + iommu_btopr(offset);
944			size = iommu_ptob(MIN_DVMA_WIN_SIZE);
945			DPRINTF(IOMMU_DMA_SETUP_DEBUG, ("dma_setup: SZ %x pg "
946			    "%lx sz %x\n", OBJSIZE, npages, size));
947			if (pplist != NULL) {
948				mp->dmai_minfo = (void *)pplist;
949				mp->dmai_rflags |= DMP_SHADOW;
950			}
951		} else {
952			mp->dmai_rflags ^= DDI_DMA_PARTIAL;
953		}
954	} else {
955		if (npages >= iommu_btop(softsp->iommu_dvma_size) -
956		    MIN_DVMA_WIN_SIZE) {
957			rval = DDI_DMA_TOOBIG;
958			goto bad;
959		}
960	}
961
962	/*
963	 * save dmareq-object, size and npages into mp
964	 */
965	mp->dmai_object = dmareq->dmar_object;
966	mp->dmai_size = size;
967	mp->dmai_ndvmapages = npages;
968
969	if (mp->dmai_rflags & DMP_NOLIMIT) {
970		ioaddr = (ioaddr_t)(uintptr_t)vmem_alloc(softsp->dvma_arena,
971		    iommu_ptob(npages),
972		    dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP);
973		if (ioaddr == 0) {
974			rval = DDI_DMA_NORESOURCES;
975			goto bad;
976		}
977
978		/*
979		 * If we have a 1 page request and we're working with a page
980		 * list, we're going to speed load an IOMMU entry.
981		 */
982		if (npages == 1 && !addr) {
983			uint64_t iotte_flag = IOTTE_VALID | IOTTE_CACHE |
984			    IOTTE_WRITE | IOTTE_STREAM;
985			volatile uint64_t *iotte_ptr;
986			pfn_t pfn;
987#if defined(DEBUG) && defined(IO_MEMUSAGE)
988			struct io_mem_list *iomemp;
989			pfn_t *pfnp;
990#endif /* DEBUG && IO_MEMUSAGE */
991
992			iotte_ptr = IOTTE_NDX(ioaddr,
993			    softsp->soft_tsb_base_addr);
994
995			if (mp->dmai_rflags & DDI_DMA_CONSISTENT) {
996				mp->dmai_rflags |= DMP_NOSYNC;
997				iotte_flag ^= IOTTE_STREAM;
998			} else if (softsp->stream_buf_off)
999				iotte_flag ^= IOTTE_STREAM;
1000
1001			mp->dmai_rflags ^= DDI_DMA_PARTIAL;
1002
1003			if (pp != NULL)
1004				pfn = pp->p_pagenum;
1005			else
1006				pfn = (*pplist)->p_pagenum;
1007
1008			iommu_tlb_flush(softsp, ioaddr, 1);
1009
1010			*iotte_ptr =
1011			    ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag;
1012
1013			mp->dmai_mapping = (ioaddr_t)(ioaddr + offset);
1014			mp->dmai_nwin = 0;
1015			if (cp != NULL) {
1016				cp->dmac_notused = 0;
1017				cp->dmac_address = (ioaddr_t)mp->dmai_mapping;
1018				cp->dmac_size = mp->dmai_size;
1019				cp->dmac_type = 0;
1020				*ccountp = 1;
1021			}
1022
1023			DPRINTF(IOMMU_TTE, ("speed loading: TTE index %p "
1024			    "pfn %lx tte flag %lx addr %lx ioaddr %x\n",
1025			    (void *)iotte_ptr, pfn, iotte_flag, addr, ioaddr));
1026
1027#if defined(DEBUG) && defined(IO_MEMUSAGE)
1028			iomemp = kmem_alloc(sizeof (struct io_mem_list),
1029			    KM_SLEEP);
1030			iomemp->rdip = mp->dmai_rdip;
1031			iomemp->ioaddr = ioaddr;
1032			iomemp->addr = addr;
1033			iomemp->npages = npages;
1034			pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) *
1035			    (npages + 1), KM_SLEEP);
1036			*pfnp = pfn;
1037			mutex_enter(&softsp->iomemlock);
1038			iomemp->next = softsp->iomem;
1039			softsp->iomem = iomemp;
1040			mutex_exit(&softsp->iomemlock);
1041#endif /* DEBUG && IO_MEMUSAGE */
1042
1043			return (DDI_DMA_MAPPED);
1044		}
1045	} else {
1046		ioaddr = (ioaddr_t)(uintptr_t)vmem_xalloc(softsp->dvma_arena,
1047		    iommu_ptob(npages),
1048		    MAX((uint_t)dma_attr->dma_attr_align, IOMMU_PAGESIZE), 0,
1049		    (uint_t)dma_attr->dma_attr_seg + 1,
1050		    (void *)(uintptr_t)(ioaddr_t)dma_attr->dma_attr_addr_lo,
1051		    (void *)(uintptr_t)
1052		    ((ioaddr_t)dma_attr->dma_attr_addr_hi + 1),
1053		    dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP);
1054	}
1055
1056	if (ioaddr == 0) {
1057		if (dmareq->dmar_fp == DDI_DMA_SLEEP)
1058			rval = DDI_DMA_NOMAPPING;
1059		else
1060			rval = DDI_DMA_NORESOURCES;
1061		goto bad;
1062	}
1063
1064	mp->dmai_mapping = ioaddr + offset;
1065	ASSERT(mp->dmai_mapping >= softsp->iommu_dvma_base);
1066
1067	/*
1068	 * At this point we have a range of virtual address allocated
1069	 * with which we now have to map to the requested object.
1070	 */
1071	if (addr) {
1072		rval = iommu_create_vaddr_mappings(mp,
1073		    addr & ~IOMMU_PAGEOFFSET);
1074		if (rval == DDI_DMA_NOMAPPING)
1075			goto bad_nomap;
1076	} else {
1077		rval = iommu_create_pp_mappings(mp, pp, pplist);
1078		if (rval == DDI_DMA_NOMAPPING)
1079			goto bad_nomap;
1080	}
1081
1082	if (cp) {
1083		cp->dmac_notused = 0;
1084		cp->dmac_address = (ioaddr_t)mp->dmai_mapping;
1085		cp->dmac_size = mp->dmai_size;
1086		cp->dmac_type = 0;
1087		*ccountp = 1;
1088	}
1089	if (mp->dmai_rflags & DDI_DMA_PARTIAL) {
1090		size = iommu_ptob(mp->dmai_ndvmapages - iommu_btopr(offset));
1091		mp->dmai_nwin =
1092		    (dmareq->dmar_object.dmao_size + (size - 1)) / size;
1093		return (DDI_DMA_PARTIAL_MAP);
1094	} else {
1095		mp->dmai_nwin = 0;
1096		return (DDI_DMA_MAPPED);
1097	}
1098
1099bad_nomap:
1100	/*
1101	 * Could not create mmu mappings.
1102	 */
1103	if (mp->dmai_rflags & DMP_NOLIMIT) {
1104		vmem_free(softsp->dvma_arena, (void *)(uintptr_t)ioaddr,
1105		    iommu_ptob(npages));
1106	} else {
1107		vmem_xfree(softsp->dvma_arena, (void *)(uintptr_t)ioaddr,
1108		    iommu_ptob(npages));
1109	}
1110
1111bad:
1112	if (rval == DDI_DMA_NORESOURCES &&
1113	    dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
1114		ddi_set_callback(dmareq->dmar_fp,
1115		    dmareq->dmar_arg, &softsp->dvma_call_list_id);
1116	}
1117	mp->dmai_inuse = 0;
1118	return (rval);
1119}
1120
1121/* ARGSUSED */
1122int
1123iommu_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
1124    ddi_dma_handle_t handle)
1125{
1126	ioaddr_t addr;
1127	uint_t npages;
1128	size_t size;
1129	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1130	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
1131	struct sbus_soft_state *softsp = mppriv->softsp;
1132	ASSERT(softsp != NULL);
1133
1134	addr = (ioaddr_t)(mp->dmai_mapping & ~IOMMU_PAGEOFFSET);
1135	npages = mp->dmai_ndvmapages;
1136	size = iommu_ptob(npages);
1137
1138	DPRINTF(IOMMU_DMA_UNBINDHDL_DEBUG, ("iommu_dma_unbindhdl: "
1139	    "unbinding addr %x for %x pages\n", addr, mp->dmai_ndvmapages));
1140
1141	/* sync the entire object */
1142	if (!(mp->dmai_rflags & DDI_DMA_CONSISTENT)) {
1143		/* flush stream write buffers */
1144		sync_stream_buf(softsp, addr, npages, (int *)&mppriv->sync_flag,
1145		    mppriv->phys_sync_flag);
1146	}
1147
1148#if defined(DEBUG) && defined(IO_MEMDEBUG)
1149	/*
1150	 * 'Free' the dma mappings.
1151	 */
1152	iommu_remove_mappings(mp);
1153#endif /* DEBUG && IO_MEMDEBUG */
1154
1155	ASSERT(npages > (uint_t)0);
1156	if (mp->dmai_rflags & DMP_NOLIMIT)
1157		vmem_free(softsp->dvma_arena, (void *)(uintptr_t)addr, size);
1158	else
1159		vmem_xfree(softsp->dvma_arena, (void *)(uintptr_t)addr, size);
1160
1161	mp->dmai_ndvmapages = 0;
1162	mp->dmai_inuse = 0;
1163	mp->dmai_minfo = NULL;
1164
1165	if (softsp->dvma_call_list_id != 0)
1166		ddi_run_callback(&softsp->dvma_call_list_id);
1167
1168	return (DDI_SUCCESS);
1169}
1170
1171/*ARGSUSED*/
1172int
1173iommu_dma_flush(dev_info_t *dip, dev_info_t *rdip,
1174    ddi_dma_handle_t handle, off_t off, size_t len,
1175    uint_t cache_flags)
1176{
1177	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1178	struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
1179
1180	if (!(mp->dmai_rflags & DDI_DMA_CONSISTENT)) {
1181		sync_stream_buf(mppriv->softsp, mp->dmai_mapping,
1182		    mp->dmai_ndvmapages, (int *)&mppriv->sync_flag,
1183		    mppriv->phys_sync_flag);
1184	}
1185	return (DDI_SUCCESS);
1186}
1187
1188/*ARGSUSED*/
1189int
1190iommu_dma_win(dev_info_t *dip, dev_info_t *rdip,
1191    ddi_dma_handle_t handle, uint_t win, off_t *offp,
1192    size_t *lenp, ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1193{
1194	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1195	off_t offset;
1196	uint_t winsize;
1197	uint_t newoff;
1198	int rval;
1199
1200	offset = mp->dmai_mapping & IOMMU_PAGEOFFSET;
1201	winsize = iommu_ptob(mp->dmai_ndvmapages - iommu_btopr(offset));
1202
1203	DPRINTF(IOMMU_DMA_WIN_DEBUG, ("getwin win %d winsize %x\n", win,
1204	    winsize));
1205
1206	/*
1207	 * win is in the range [0 .. dmai_nwin-1]
1208	 */
1209	if (win >= mp->dmai_nwin)
1210		return (DDI_FAILURE);
1211
1212	newoff = win * winsize;
1213	if (newoff > mp->dmai_object.dmao_size - mp->dmai_minxfer)
1214		return (DDI_FAILURE);
1215
1216	ASSERT(cookiep);
1217	cookiep->dmac_notused = 0;
1218	cookiep->dmac_type = 0;
1219	cookiep->dmac_address = (ioaddr_t)mp->dmai_mapping;
1220	cookiep->dmac_size = mp->dmai_size;
1221	*ccountp = 1;
1222	*offp = (off_t)newoff;
1223	*lenp = (uint_t)winsize;
1224
1225	if (newoff == mp->dmai_offset) {
1226		/*
1227		 * Nothing to do...
1228		 */
1229		return (DDI_SUCCESS);
1230	}
1231
1232	if ((rval = iommu_map_window(mp, newoff, winsize)) != DDI_SUCCESS)
1233		return (rval);
1234
1235	/*
1236	 * Set this again in case iommu_map_window() has changed it
1237	 */
1238	cookiep->dmac_size = mp->dmai_size;
1239
1240	return (DDI_SUCCESS);
1241}
1242
1243static int
1244iommu_map_window(ddi_dma_impl_t *mp, off_t newoff, size_t winsize)
1245{
1246	uintptr_t addr = 0;
1247	page_t *pp;
1248	uint_t flags;
1249	struct page **pplist = NULL;
1250
1251#if defined(DEBUG) && defined(IO_MEMDEBUG)
1252	/* Free mappings for current window */
1253	iommu_remove_mappings(mp);
1254#endif /* DEBUG && IO_MEMDEBUG */
1255
1256	mp->dmai_offset = newoff;
1257	mp->dmai_size = mp->dmai_object.dmao_size - newoff;
1258	mp->dmai_size = MIN(mp->dmai_size, winsize);
1259
1260	if (mp->dmai_object.dmao_type == DMA_OTYP_VADDR ||
1261	    mp->dmai_object.dmao_type == DMA_OTYP_BUFVADDR) {
1262		if (mp->dmai_rflags & DMP_SHADOW) {
1263			pplist = (struct page **)mp->dmai_minfo;
1264			ASSERT(pplist != NULL);
1265			pplist = pplist + (newoff >> MMU_PAGESHIFT);
1266		} else {
1267			addr = (uintptr_t)
1268			    mp->dmai_object.dmao_obj.virt_obj.v_addr;
1269			addr = (addr + newoff) & ~IOMMU_PAGEOFFSET;
1270		}
1271		pp = NULL;
1272	} else {
1273		pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp;
1274		flags = 0;
1275		while (flags < newoff) {
1276			pp = pp->p_next;
1277			flags += MMU_PAGESIZE;
1278		}
1279	}
1280
1281	/* Set up mappings for next window */
1282	if (addr) {
1283		if (iommu_create_vaddr_mappings(mp, addr) < 0)
1284			return (DDI_FAILURE);
1285	} else {
1286		if (iommu_create_pp_mappings(mp, pp, pplist) < 0)
1287			return (DDI_FAILURE);
1288	}
1289
1290	/*
1291	 * also invalidate read stream buffer
1292	 */
1293	if (!(mp->dmai_rflags & DDI_DMA_CONSISTENT)) {
1294		struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
1295
1296		sync_stream_buf(mppriv->softsp, mp->dmai_mapping,
1297		    mp->dmai_ndvmapages, (int *)&mppriv->sync_flag,
1298		    mppriv->phys_sync_flag);
1299	}
1300
1301	return (DDI_SUCCESS);
1302
1303}
1304
1305
1306/*ARGSUSED*/
1307int
1308iommu_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
1309    ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
1310    off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags)
1311{
1312	pgcnt_t npages;
1313	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1314
1315	DPRINTF(IOMMU_DMAMCTL_DEBUG, ("dma_mctl: handle %p ", (void *)mp));
1316	switch (request) {
1317
1318	case DDI_DMA_SET_SBUS64:
1319	{
1320		struct dma_impl_priv *mppriv = (struct dma_impl_priv *)mp;
1321
1322		return (iommu_dma_lim_setup(dip, rdip, mppriv->softsp,
1323		    &mp->dmai_burstsizes, (uint_t)*lenp, &mp->dmai_minxfer,
1324		    DDI_DMA_SBUS_64BIT));
1325	}
1326
1327	case DDI_DMA_RESERVE:
1328	{
1329		struct ddi_dma_req *dmareq = (struct ddi_dma_req *)offp;
1330		ddi_dma_lim_t *dma_lim;
1331		ddi_dma_handle_t *handlep;
1332		uint_t np;
1333		ioaddr_t ioaddr;
1334		int i;
1335		struct fast_dvma *iommu_fast_dvma;
1336		struct sbus_soft_state *softsp =
1337		    (struct sbus_soft_state *)ddi_get_soft_state(sbusp,
1338		    ddi_get_instance(dip));
1339
1340		/* Some simple sanity checks */
1341		dma_lim = dmareq->dmar_limits;
1342		if (dma_lim->dlim_burstsizes == 0) {
1343			DPRINTF(IOMMU_FASTDMA_RESERVE,
1344			    ("Reserve: bad burstsizes\n"));
1345			return (DDI_DMA_BADLIMITS);
1346		}
1347		if ((AHI <= ALO) || (AHI < softsp->iommu_dvma_base)) {
1348			DPRINTF(IOMMU_FASTDMA_RESERVE,
1349			    ("Reserve: bad limits\n"));
1350			return (DDI_DMA_BADLIMITS);
1351		}
1352
1353		np = dmareq->dmar_object.dmao_size;
1354		mutex_enter(&softsp->dma_pool_lock);
1355		if (np > softsp->dma_reserve) {
1356			mutex_exit(&softsp->dma_pool_lock);
1357			DPRINTF(IOMMU_FASTDMA_RESERVE,
1358			    ("Reserve: dma_reserve is exhausted\n"));
1359			return (DDI_DMA_NORESOURCES);
1360		}
1361
1362		softsp->dma_reserve -= np;
1363		mutex_exit(&softsp->dma_pool_lock);
1364		mp = kmem_zalloc(sizeof (*mp), KM_SLEEP);
1365		mp->dmai_rflags = DMP_BYPASSNEXUS;
1366		mp->dmai_rdip = rdip;
1367		mp->dmai_minxfer = dma_lim->dlim_minxfer;
1368		mp->dmai_burstsizes = dma_lim->dlim_burstsizes;
1369
1370		ioaddr = (ioaddr_t)(uintptr_t)vmem_xalloc(softsp->dvma_arena,
1371		    iommu_ptob(np), IOMMU_PAGESIZE, 0,
1372		    dma_lim->dlim_cntr_max + 1,
1373		    (void *)(uintptr_t)ALO, (void *)(uintptr_t)(AHI + 1),
1374		    dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP);
1375
1376		if (ioaddr == 0) {
1377			mutex_enter(&softsp->dma_pool_lock);
1378			softsp->dma_reserve += np;
1379			mutex_exit(&softsp->dma_pool_lock);
1380			kmem_free(mp, sizeof (*mp));
1381			DPRINTF(IOMMU_FASTDMA_RESERVE,
1382			    ("Reserve: No dvma resources available\n"));
1383			return (DDI_DMA_NOMAPPING);
1384		}
1385
1386		/* create a per request structure */
1387		iommu_fast_dvma = kmem_alloc(sizeof (struct fast_dvma),
1388		    KM_SLEEP);
1389
1390		/*
1391		 * We need to remember the size of the transfer so that
1392		 * we can figure the virtual pages to sync when the transfer
1393		 * is complete.
1394		 */
1395		iommu_fast_dvma->pagecnt = kmem_zalloc(np *
1396		    sizeof (uint_t), KM_SLEEP);
1397
1398		/* Allocate a streaming cache sync flag for each index */
1399		iommu_fast_dvma->sync_flag = kmem_zalloc(np *
1400		    sizeof (int), KM_SLEEP);
1401
1402		/* Allocate a physical sync flag for each index */
1403		iommu_fast_dvma->phys_sync_flag =
1404		    kmem_zalloc(np * sizeof (uint64_t), KM_SLEEP);
1405
1406		for (i = 0; i < np; i++)
1407			iommu_fast_dvma->phys_sync_flag[i] = va_to_pa((caddr_t)
1408			    &iommu_fast_dvma->sync_flag[i]);
1409
1410		mp->dmai_mapping = ioaddr;
1411		mp->dmai_ndvmapages = np;
1412		iommu_fast_dvma->ops = &iommu_dvma_ops;
1413		iommu_fast_dvma->softsp = (caddr_t)softsp;
1414		mp->dmai_nexus_private = (caddr_t)iommu_fast_dvma;
1415		handlep = (ddi_dma_handle_t *)objp;
1416		*handlep = (ddi_dma_handle_t)mp;
1417
1418		DPRINTF(IOMMU_FASTDMA_RESERVE,
1419		    ("Reserve: mapping object %p base addr %lx size %x\n",
1420		    (void *)mp, mp->dmai_mapping, mp->dmai_ndvmapages));
1421
1422		break;
1423	}
1424
1425	case DDI_DMA_RELEASE:
1426	{
1427		ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1428		uint_t np = npages = mp->dmai_ndvmapages;
1429		ioaddr_t ioaddr = mp->dmai_mapping;
1430		volatile uint64_t *iotte_ptr;
1431		struct fast_dvma *iommu_fast_dvma = (struct fast_dvma *)
1432		    mp->dmai_nexus_private;
1433		struct sbus_soft_state *softsp = (struct sbus_soft_state *)
1434		    iommu_fast_dvma->softsp;
1435
1436		ASSERT(softsp != NULL);
1437
1438		/* Unload stale mappings and flush stale tlb's */
1439		iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr);
1440
1441		while (npages > (uint_t)0) {
1442			*iotte_ptr = (uint64_t)0;	/* unload tte */
1443			iommu_tlb_flush(softsp, ioaddr, 1);
1444
1445			npages--;
1446			iotte_ptr++;
1447			ioaddr += IOMMU_PAGESIZE;
1448		}
1449
1450		ioaddr = (ioaddr_t)mp->dmai_mapping;
1451		mutex_enter(&softsp->dma_pool_lock);
1452		softsp->dma_reserve += np;
1453		mutex_exit(&softsp->dma_pool_lock);
1454
1455		if (mp->dmai_rflags & DMP_NOLIMIT)
1456			vmem_free(softsp->dvma_arena,
1457			    (void *)(uintptr_t)ioaddr, iommu_ptob(np));
1458		else
1459			vmem_xfree(softsp->dvma_arena,
1460			    (void *)(uintptr_t)ioaddr, iommu_ptob(np));
1461
1462		kmem_free(mp, sizeof (*mp));
1463		kmem_free(iommu_fast_dvma->pagecnt, np * sizeof (uint_t));
1464		kmem_free(iommu_fast_dvma->sync_flag, np * sizeof (int));
1465		kmem_free(iommu_fast_dvma->phys_sync_flag, np *
1466		    sizeof (uint64_t));
1467		kmem_free(iommu_fast_dvma, sizeof (struct fast_dvma));
1468
1469
1470		DPRINTF(IOMMU_FASTDMA_RESERVE,
1471		    ("Release: Base addr %x size %x\n", ioaddr, np));
1472		/*
1473		 * Now that we've freed some resource,
1474		 * if there is anybody waiting for it
1475		 * try and get them going.
1476		 */
1477		if (softsp->dvma_call_list_id != 0)
1478			ddi_run_callback(&softsp->dvma_call_list_id);
1479
1480		break;
1481	}
1482
1483	default:
1484		DPRINTF(IOMMU_DMAMCTL_DEBUG, ("iommu_dma_mctl: unknown option "
1485		    "0%x\n", request));
1486
1487		return (DDI_FAILURE);
1488	}
1489	return (DDI_SUCCESS);
1490}
1491
1492/*ARGSUSED*/
1493void
1494iommu_dvma_kaddr_load(ddi_dma_handle_t h, caddr_t a, uint_t len, uint_t index,
1495    ddi_dma_cookie_t *cp)
1496{
1497	uintptr_t addr;
1498	ioaddr_t ioaddr;
1499	uint_t offset;
1500	pfn_t pfn;
1501	int npages;
1502	volatile uint64_t *iotte_ptr;
1503	uint64_t iotte_flag = 0;
1504	struct as *as = NULL;
1505	extern struct as kas;
1506	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h;
1507	struct fast_dvma *iommu_fast_dvma =
1508	    (struct fast_dvma *)mp->dmai_nexus_private;
1509	struct sbus_soft_state *softsp = (struct sbus_soft_state *)
1510	    iommu_fast_dvma->softsp;
1511#if defined(DEBUG) && defined(IO_MEMUSAGE)
1512	struct io_mem_list *iomemp;
1513	pfn_t *pfnp;
1514#endif /* DEBUG && IO_MEMUSAGE */
1515
1516	ASSERT(softsp != NULL);
1517
1518	addr = (uintptr_t)a;
1519	ioaddr = (ioaddr_t)(mp->dmai_mapping + iommu_ptob(index));
1520	offset = (uint_t)(addr & IOMMU_PAGEOFFSET);
1521	iommu_fast_dvma->pagecnt[index] = iommu_btopr(len + offset);
1522	as = &kas;
1523	addr &= ~IOMMU_PAGEOFFSET;
1524	npages = iommu_btopr(len + offset);
1525
1526#if defined(DEBUG) && defined(IO_MEMUSAGE)
1527	iomemp = kmem_alloc(sizeof (struct io_mem_list), KM_SLEEP);
1528	iomemp->rdip = mp->dmai_rdip;
1529	iomemp->ioaddr = ioaddr;
1530	iomemp->addr = addr;
1531	iomemp->npages = npages;
1532	pfnp = iomemp->pfn = kmem_zalloc(sizeof (*pfnp) * (npages + 1),
1533	    KM_SLEEP);
1534#endif /* DEBUG && IO_MEMUSAGE */
1535
1536	cp->dmac_address = ioaddr | offset;
1537	cp->dmac_size = len;
1538
1539	iotte_ptr = IOTTE_NDX(ioaddr, softsp->soft_tsb_base_addr);
1540	/* read/write and streaming io on */
1541	iotte_flag = IOTTE_VALID | IOTTE_WRITE | IOTTE_CACHE;
1542
1543	if (mp->dmai_rflags & DDI_DMA_CONSISTENT)
1544		mp->dmai_rflags |= DMP_NOSYNC;
1545	else if (!softsp->stream_buf_off)
1546		iotte_flag |= IOTTE_STREAM;
1547
1548	DPRINTF(IOMMU_FASTDMA_LOAD, ("kaddr_load: ioaddr %x "
1549	    "size %x offset %x index %x kaddr %lx\n",
1550	    ioaddr, len, offset, index, addr));
1551	ASSERT(npages > 0);
1552	do {
1553		pfn = hat_getpfnum(as->a_hat, (caddr_t)addr);
1554		if (pfn == PFN_INVALID) {
1555			DPRINTF(IOMMU_FASTDMA_LOAD, ("kaddr_load: invalid pfn "
1556			    "from hat_getpfnum()\n"));
1557		}
1558
1559		iommu_tlb_flush(softsp, ioaddr, 1);
1560
1561		/* load tte */
1562		*iotte_ptr = ((uint64_t)pfn << IOMMU_PAGESHIFT) | iotte_flag;
1563
1564		npages--;
1565		iotte_ptr++;
1566
1567		addr += IOMMU_PAGESIZE;
1568		ioaddr += IOMMU_PAGESIZE;
1569
1570#if defined(DEBUG) && defined(IO_MEMUSAGE)
1571		*pfnp = pfn;
1572		pfnp++;
1573#endif /* DEBUG && IO_MEMUSAGE */
1574
1575	} while (npages > 0);
1576
1577#if defined(DEBUG) && defined(IO_MEMUSAGE)
1578	mutex_enter(&softsp->iomemlock);
1579	iomemp->next = softsp->iomem;
1580	softsp->iomem = iomemp;
1581	mutex_exit(&softsp->iomemlock);
1582#endif /* DEBUG && IO_MEMUSAGE */
1583}
1584
1585/*ARGSUSED*/
1586void
1587iommu_dvma_unload(ddi_dma_handle_t h, uint_t index, uint_t view)
1588{
1589	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h;
1590	ioaddr_t ioaddr;
1591	pgcnt_t npages;
1592	struct fast_dvma *iommu_fast_dvma =
1593	    (struct fast_dvma *)mp->dmai_nexus_private;
1594	struct sbus_soft_state *softsp = (struct sbus_soft_state *)
1595	    iommu_fast_dvma->softsp;
1596#if defined(DEBUG) && defined(IO_MEMUSAGE)
1597	struct io_mem_list **prevp, *walk;
1598#endif /* DEBUG && IO_MEMUSAGE */
1599
1600	ASSERT(softsp != NULL);
1601
1602	ioaddr = (ioaddr_t)(mp->dmai_mapping + iommu_ptob(index));
1603	npages = iommu_fast_dvma->pagecnt[index];
1604
1605#if defined(DEBUG) && defined(IO_MEMUSAGE)
1606	mutex_enter(&softsp->iomemlock);
1607	prevp = &softsp->iomem;
1608	walk = softsp->iomem;
1609
1610	while (walk != NULL) {
1611		if (walk->ioaddr == ioaddr) {
1612			*prevp = walk->next;
1613			break;
1614		}
1615		prevp = &walk->next;
1616		walk = walk->next;
1617	}
1618	mutex_exit(&softsp->iomemlock);
1619
1620	kmem_free(walk->pfn, sizeof (pfn_t) * (npages + 1));
1621	kmem_free(walk, sizeof (struct io_mem_list));
1622#endif /* DEBUG && IO_MEMUSAGE */
1623
1624	DPRINTF(IOMMU_FASTDMA_SYNC, ("kaddr_unload: handle %p sync flag "
1625	    "addr %p sync flag pfn %llx index %x page count %lx\n", (void *)mp,
1626	    (void *)&iommu_fast_dvma->sync_flag[index],
1627	    iommu_fast_dvma->phys_sync_flag[index],
1628	    index, npages));
1629
1630	if ((mp->dmai_rflags & DMP_NOSYNC) != DMP_NOSYNC) {
1631		sync_stream_buf(softsp, ioaddr, npages,
1632		    (int *)&iommu_fast_dvma->sync_flag[index],
1633		    iommu_fast_dvma->phys_sync_flag[index]);
1634	}
1635}
1636
1637/*ARGSUSED*/
1638void
1639iommu_dvma_sync(ddi_dma_handle_t h, uint_t index, uint_t view)
1640{
1641	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h;
1642	ioaddr_t ioaddr;
1643	uint_t npages;
1644	struct fast_dvma *iommu_fast_dvma =
1645	    (struct fast_dvma *)mp->dmai_nexus_private;
1646	struct sbus_soft_state *softsp = (struct sbus_soft_state *)
1647	    iommu_fast_dvma->softsp;
1648
1649	if ((mp->dmai_rflags & DMP_NOSYNC) == DMP_NOSYNC)
1650		return;
1651
1652	ASSERT(softsp != NULL);
1653	ioaddr = (ioaddr_t)(mp->dmai_mapping + iommu_ptob(index));
1654	npages = iommu_fast_dvma->pagecnt[index];
1655
1656	DPRINTF(IOMMU_FASTDMA_SYNC, ("kaddr_sync: handle %p, "
1657	    "sync flag addr %p, sync flag pfn %llx\n", (void *)mp,
1658	    (void *)&iommu_fast_dvma->sync_flag[index],
1659	    iommu_fast_dvma->phys_sync_flag[index]));
1660
1661	sync_stream_buf(softsp, ioaddr, npages,
1662	    (int *)&iommu_fast_dvma->sync_flag[index],
1663	    iommu_fast_dvma->phys_sync_flag[index]);
1664}
1665