1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24/*
25 * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
26 * Copyright (c) 2011 Bayard G. Bell.  All rights reserved.
27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
28 * Copyright 2017 Joyent, Inc.
29 * Copyright 2020 Ryan Zezeski
30 */
31
32/*
33 * x86 root nexus driver
34 */
35
36#include <sys/sysmacros.h>
37#include <sys/conf.h>
38#include <sys/autoconf.h>
39#include <sys/sysmacros.h>
40#include <sys/debug.h>
41#include <sys/psw.h>
42#include <sys/ddidmareq.h>
43#include <sys/promif.h>
44#include <sys/devops.h>
45#include <sys/kmem.h>
46#include <sys/cmn_err.h>
47#include <vm/seg.h>
48#include <vm/seg_kmem.h>
49#include <vm/seg_dev.h>
50#include <sys/vmem.h>
51#include <sys/mman.h>
52#include <vm/hat.h>
53#include <vm/as.h>
54#include <vm/page.h>
55#include <sys/avintr.h>
56#include <sys/errno.h>
57#include <sys/modctl.h>
58#include <sys/ddi_impldefs.h>
59#include <sys/sunddi.h>
60#include <sys/sunndi.h>
61#include <sys/mach_intr.h>
62#include <sys/psm.h>
63#include <sys/ontrap.h>
64#include <sys/atomic.h>
65#include <sys/sdt.h>
66#include <sys/rootnex.h>
67#include <vm/hat_i86.h>
68#include <sys/ddifm.h>
69#include <sys/ddi_isa.h>
70#include <sys/apic.h>
71
72#ifdef __xpv
73#include <sys/bootinfo.h>
74#include <sys/hypervisor.h>
75#include <sys/bootconf.h>
76#include <vm/kboot_mmu.h>
77#endif
78
79#if defined(__amd64) && !defined(__xpv)
80#include <sys/immu.h>
81#endif
82
83
84/*
85 * enable/disable extra checking of function parameters. Useful for debugging
86 * drivers.
87 */
88#ifdef	DEBUG
89int rootnex_alloc_check_parms = 1;
90int rootnex_bind_check_parms = 1;
91int rootnex_bind_check_inuse = 1;
92int rootnex_unbind_verify_buffer = 0;
93int rootnex_sync_check_parms = 1;
94#else
95int rootnex_alloc_check_parms = 0;
96int rootnex_bind_check_parms = 0;
97int rootnex_bind_check_inuse = 0;
98int rootnex_unbind_verify_buffer = 0;
99int rootnex_sync_check_parms = 0;
100#endif
101
102boolean_t rootnex_dmar_not_setup;
103
104/* Master Abort and Target Abort panic flag */
105int rootnex_fm_ma_ta_panic_flag = 0;
106
107/* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
108int rootnex_bind_fail = 1;
109int rootnex_bind_warn = 1;
110uint8_t *rootnex_warn_list;
111/* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
112#define	ROOTNEX_BIND_WARNING	(0x1 << 0)
113
114/*
115 * revert back to old broken behavior of always sync'ing entire copy buffer.
116 * This is useful if be have a buggy driver which doesn't correctly pass in
117 * the offset and size into ddi_dma_sync().
118 */
119int rootnex_sync_ignore_params = 0;
120
121/*
122 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
123 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
124 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
125 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
126 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
127 * (< 8K). We will still need to allocate the copy buffer during bind though
128 * (if we need one). These can only be modified in /etc/system before rootnex
129 * attach.
130 */
131#if defined(__amd64)
132int rootnex_prealloc_cookies = 65;
133int rootnex_prealloc_windows = 4;
134int rootnex_prealloc_copybuf = 2;
135#else
136int rootnex_prealloc_cookies = 33;
137int rootnex_prealloc_windows = 4;
138int rootnex_prealloc_copybuf = 2;
139#endif
140
141/* driver global state */
142static rootnex_state_t *rootnex_state;
143
144#ifdef DEBUG
145/* shortcut to rootnex counters */
146static uint64_t *rootnex_cnt;
147#endif
148
149/*
150 * XXX - does x86 even need these or are they left over from the SPARC days?
151 */
152/* statically defined integer/boolean properties for the root node */
153static rootnex_intprop_t rootnex_intprp[] = {
154	{ "PAGESIZE",			PAGESIZE },
155	{ "MMU_PAGESIZE",		MMU_PAGESIZE },
156	{ "MMU_PAGEOFFSET",		MMU_PAGEOFFSET },
157	{ DDI_RELATIVE_ADDRESSING,	1 },
158};
159#define	NROOT_INTPROPS	(sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
160
161/*
162 * If we're dom0, we're using a real device so we need to load
163 * the cookies with MFNs instead of PFNs.
164 */
165#ifdef __xpv
166typedef maddr_t rootnex_addr_t;
167#define	ROOTNEX_PADDR_TO_RBASE(pa)	\
168	(DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
169#else
170typedef paddr_t rootnex_addr_t;
171#define	ROOTNEX_PADDR_TO_RBASE(pa)	(pa)
172#endif
173
174static struct cb_ops rootnex_cb_ops = {
175	nodev,		/* open */
176	nodev,		/* close */
177	nodev,		/* strategy */
178	nodev,		/* print */
179	nodev,		/* dump */
180	nodev,		/* read */
181	nodev,		/* write */
182	nodev,		/* ioctl */
183	nodev,		/* devmap */
184	nodev,		/* mmap */
185	nodev,		/* segmap */
186	nochpoll,	/* chpoll */
187	ddi_prop_op,	/* cb_prop_op */
188	NULL,		/* struct streamtab */
189	D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
190	CB_REV,		/* Rev */
191	nodev,		/* cb_aread */
192	nodev		/* cb_awrite */
193};
194
195static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
196    off_t offset, off_t len, caddr_t *vaddrp);
197static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
198    struct hat *hat, struct seg *seg, caddr_t addr,
199    struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
200static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
201    ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
202    ddi_dma_handle_t *handlep);
203static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
204    ddi_dma_handle_t handle);
205static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
206    ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
207    ddi_dma_cookie_t *cookiep, uint_t *ccountp);
208static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
209    ddi_dma_handle_t handle);
210static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
211    ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
212static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
213    ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
214    ddi_dma_cookie_t *cookiep, uint_t *ccountp);
215static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
216    ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
217    off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
218static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
219    ddi_ctl_enum_t ctlop, void *arg, void *result);
220static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
221    ddi_iblock_cookie_t *ibc);
222static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
223    ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
224static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
225    void *);
226static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
227
228static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
229    ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
230    ddi_dma_handle_t *handlep);
231static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
232    ddi_dma_handle_t handle);
233static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
234    ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
235    ddi_dma_cookie_t *cookiep, uint_t *ccountp);
236static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
237    ddi_dma_handle_t handle);
238#if defined(__amd64) && !defined(__xpv)
239static void rootnex_coredma_reset_cookies(dev_info_t *dip,
240    ddi_dma_handle_t handle);
241static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
242    ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
243static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
244    ddi_dma_cookie_t *cookiep, uint_t ccount);
245static int rootnex_coredma_clear_cookies(dev_info_t *dip,
246    ddi_dma_handle_t handle);
247static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
248#endif
249static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
250    ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
251static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
252    ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
253    ddi_dma_cookie_t *cookiep, uint_t *ccountp);
254
255#if defined(__amd64) && !defined(__xpv)
256static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
257    ddi_dma_handle_t handle, void *v);
258static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
259    ddi_dma_handle_t handle);
260#endif
261
262
263static struct bus_ops rootnex_bus_ops = {
264	BUSO_REV,
265	rootnex_map,
266	NULL,
267	NULL,
268	NULL,
269	rootnex_map_fault,
270	0,
271	rootnex_dma_allochdl,
272	rootnex_dma_freehdl,
273	rootnex_dma_bindhdl,
274	rootnex_dma_unbindhdl,
275	rootnex_dma_sync,
276	rootnex_dma_win,
277	rootnex_dma_mctl,
278	rootnex_ctlops,
279	ddi_bus_prop_op,
280	i_ddi_rootnex_get_eventcookie,
281	i_ddi_rootnex_add_eventcall,
282	i_ddi_rootnex_remove_eventcall,
283	i_ddi_rootnex_post_event,
284	0,			/* bus_intr_ctl */
285	0,			/* bus_config */
286	0,			/* bus_unconfig */
287	rootnex_fm_init,	/* bus_fm_init */
288	NULL,			/* bus_fm_fini */
289	NULL,			/* bus_fm_access_enter */
290	NULL,			/* bus_fm_access_exit */
291	NULL,			/* bus_powr */
292	rootnex_intr_ops	/* bus_intr_op */
293};
294
295static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
296static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
297static int rootnex_quiesce(dev_info_t *dip);
298
299static struct dev_ops rootnex_ops = {
300	DEVO_REV,
301	0,
302	ddi_no_info,
303	nulldev,
304	nulldev,
305	rootnex_attach,
306	rootnex_detach,
307	nulldev,
308	&rootnex_cb_ops,
309	&rootnex_bus_ops,
310	NULL,
311	rootnex_quiesce,		/* quiesce */
312};
313
314static struct modldrv rootnex_modldrv = {
315	&mod_driverops,
316	"i86pc root nexus",
317	&rootnex_ops
318};
319
320static struct modlinkage rootnex_modlinkage = {
321	MODREV_1,
322	(void *)&rootnex_modldrv,
323	NULL
324};
325
326#if defined(__amd64) && !defined(__xpv)
327static iommulib_nexops_t iommulib_nexops = {
328	IOMMU_NEXOPS_VERSION,
329	"Rootnex IOMMU ops Vers 1.1",
330	NULL,
331	rootnex_coredma_allochdl,
332	rootnex_coredma_freehdl,
333	rootnex_coredma_bindhdl,
334	rootnex_coredma_unbindhdl,
335	rootnex_coredma_reset_cookies,
336	rootnex_coredma_get_cookies,
337	rootnex_coredma_set_cookies,
338	rootnex_coredma_clear_cookies,
339	rootnex_coredma_get_sleep_flags,
340	rootnex_coredma_sync,
341	rootnex_coredma_win,
342	rootnex_coredma_hdl_setprivate,
343	rootnex_coredma_hdl_getprivate
344};
345#endif
346
347/*
348 *  extern hacks
349 */
350extern struct seg_ops segdev_ops;
351extern int ignore_hardware_nodes;	/* force flag from ddi_impl.c */
352#ifdef	DDI_MAP_DEBUG
353extern int ddi_map_debug_flag;
354#define	ddi_map_debug	if (ddi_map_debug_flag) prom_printf
355#endif
356extern void i86_pp_map(page_t *pp, caddr_t kaddr);
357extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
358extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
359    psm_intr_op_t, int *);
360extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
361extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
362
363/*
364 * Use device arena to use for device control register mappings.
365 * Various kernel memory walkers (debugger, dtrace) need to know
366 * to avoid this address range to prevent undesired device activity.
367 */
368extern void *device_arena_alloc(size_t size, int vm_flag);
369extern void device_arena_free(void * vaddr, size_t size);
370
371
372/*
373 *  Internal functions
374 */
375static int rootnex_dma_init();
376static void rootnex_add_props(dev_info_t *);
377static int rootnex_ctl_reportdev(dev_info_t *dip);
378static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
379static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
380static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
381static int rootnex_map_handle(ddi_map_req_t *mp);
382static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
383static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
384static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
385    ddi_dma_attr_t *attr);
386static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
387    rootnex_sglinfo_t *sglinfo);
388static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
389    ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
390static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
391    rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
392static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
393    rootnex_dma_t *dma, ddi_dma_attr_t *attr);
394static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
395static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
396    ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
397static void rootnex_teardown_windows(rootnex_dma_t *dma);
398static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
399    rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
400static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
401    rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
402    size_t *copybuf_used, page_t **cur_pp);
403static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
404    rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
405    ddi_dma_attr_t *attr, off_t cur_offset);
406static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
407    rootnex_dma_t *dma, rootnex_window_t **windowp,
408    ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
409static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
410    rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
411static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
412    off_t offset, size_t size, uint_t cache_flags);
413static int rootnex_verify_buffer(rootnex_dma_t *dma);
414static int rootnex_dma_check(dev_info_t *dip, const void *handle,
415    const void *comp_addr, const void *not_used);
416static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
417    rootnex_sglinfo_t *sglinfo);
418static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
419
420/*
421 * _init()
422 *
423 */
424int
425_init(void)
426{
427
428	rootnex_state = NULL;
429	return (mod_install(&rootnex_modlinkage));
430}
431
432
433/*
434 * _info()
435 *
436 */
437int
438_info(struct modinfo *modinfop)
439{
440	return (mod_info(&rootnex_modlinkage, modinfop));
441}
442
443
444/*
445 * _fini()
446 *
447 */
448int
449_fini(void)
450{
451	return (EBUSY);
452}
453
454
455/*
456 * rootnex_attach()
457 *
458 */
459static int
460rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
461{
462	int fmcap;
463	int e;
464
465	switch (cmd) {
466	case DDI_ATTACH:
467		break;
468	case DDI_RESUME:
469#if defined(__amd64) && !defined(__xpv)
470		return (immu_unquiesce());
471#else
472		return (DDI_SUCCESS);
473#endif
474	default:
475		return (DDI_FAILURE);
476	}
477
478	/*
479	 * We should only have one instance of rootnex. Save it away since we
480	 * don't have an easy way to get it back later.
481	 */
482	ASSERT(rootnex_state == NULL);
483	rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
484
485	rootnex_state->r_dip = dip;
486	rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
487	rootnex_state->r_reserved_msg_printed = B_FALSE;
488#ifdef DEBUG
489	rootnex_cnt = &rootnex_state->r_counters[0];
490#endif
491
492	/*
493	 * Set minimum fm capability level for i86pc platforms and then
494	 * initialize error handling. Since we're the rootnex, we don't
495	 * care what's returned in the fmcap field.
496	 */
497	ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
498	    DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
499	fmcap = ddi_system_fmcap;
500	ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
501
502	/* initialize DMA related state */
503	e = rootnex_dma_init();
504	if (e != DDI_SUCCESS) {
505		kmem_free(rootnex_state, sizeof (rootnex_state_t));
506		return (DDI_FAILURE);
507	}
508
509	/* Add static root node properties */
510	rootnex_add_props(dip);
511
512	/* since we can't call ddi_report_dev() */
513	cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
514
515	/* Initialize rootnex event handle */
516	i_ddi_rootnex_init_events(dip);
517
518#if defined(__amd64) && !defined(__xpv)
519	e = iommulib_nexus_register(dip, &iommulib_nexops,
520	    &rootnex_state->r_iommulib_handle);
521
522	ASSERT(e == DDI_SUCCESS);
523#endif
524
525	return (DDI_SUCCESS);
526}
527
528
529/*
530 * rootnex_detach()
531 *
532 */
533/*ARGSUSED*/
534static int
535rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
536{
537	switch (cmd) {
538	case DDI_SUSPEND:
539#if defined(__amd64) && !defined(__xpv)
540		return (immu_quiesce());
541#else
542		return (DDI_SUCCESS);
543#endif
544	default:
545		return (DDI_FAILURE);
546	}
547	/*NOTREACHED*/
548
549}
550
551
552/*
553 * rootnex_dma_init()
554 *
555 */
556/*ARGSUSED*/
557static int
558rootnex_dma_init()
559{
560	size_t bufsize;
561
562
563	/*
564	 * size of our cookie/window/copybuf state needed in dma bind that we
565	 * pre-alloc in dma_alloc_handle
566	 */
567	rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
568	rootnex_state->r_prealloc_size =
569	    (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
570	    (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
571	    (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
572
573	/*
574	 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
575	 * allocate 16 extra bytes for struct pointer alignment
576	 * (p->dmai_private & dma->dp_prealloc_buffer)
577	 */
578	bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
579	    rootnex_state->r_prealloc_size + 0x10;
580	rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
581	    bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
582	if (rootnex_state->r_dmahdl_cache == NULL) {
583		return (DDI_FAILURE);
584	}
585
586	/*
587	 * allocate array to track which major numbers we have printed warnings
588	 * for.
589	 */
590	rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
591	    KM_SLEEP);
592
593	return (DDI_SUCCESS);
594}
595
596
597/*
598 * rootnex_add_props()
599 *
600 */
601static void
602rootnex_add_props(dev_info_t *dip)
603{
604	rootnex_intprop_t *rpp;
605	int i;
606
607	/* Add static integer/boolean properties to the root node */
608	rpp = rootnex_intprp;
609	for (i = 0; i < NROOT_INTPROPS; i++) {
610		(void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
611		    rpp[i].prop_name, rpp[i].prop_value);
612	}
613}
614
615
616
617/*
618 * *************************
619 *  ctlops related routines
620 * *************************
621 */
622
623/*
624 * rootnex_ctlops()
625 *
626 */
627/*ARGSUSED*/
628static int
629rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
630    void *arg, void *result)
631{
632	int n, *ptr;
633	struct ddi_parent_private_data *pdp;
634
635	switch (ctlop) {
636	case DDI_CTLOPS_DMAPMAPC:
637		/*
638		 * Return 'partial' to indicate that dma mapping
639		 * has to be done in the main MMU.
640		 */
641		return (DDI_DMA_PARTIAL);
642
643	case DDI_CTLOPS_BTOP:
644		/*
645		 * Convert byte count input to physical page units.
646		 * (byte counts that are not a page-size multiple
647		 * are rounded down)
648		 */
649		*(ulong_t *)result = btop(*(ulong_t *)arg);
650		return (DDI_SUCCESS);
651
652	case DDI_CTLOPS_PTOB:
653		/*
654		 * Convert size in physical pages to bytes
655		 */
656		*(ulong_t *)result = ptob(*(ulong_t *)arg);
657		return (DDI_SUCCESS);
658
659	case DDI_CTLOPS_BTOPR:
660		/*
661		 * Convert byte count input to physical page units
662		 * (byte counts that are not a page-size multiple
663		 * are rounded up)
664		 */
665		*(ulong_t *)result = btopr(*(ulong_t *)arg);
666		return (DDI_SUCCESS);
667
668	case DDI_CTLOPS_INITCHILD:
669		return (impl_ddi_sunbus_initchild(arg));
670
671	case DDI_CTLOPS_UNINITCHILD:
672		impl_ddi_sunbus_removechild(arg);
673		return (DDI_SUCCESS);
674
675	case DDI_CTLOPS_REPORTDEV:
676		return (rootnex_ctl_reportdev(rdip));
677
678	case DDI_CTLOPS_IOMIN:
679		/*
680		 * Nothing to do here but reflect back..
681		 */
682		return (DDI_SUCCESS);
683
684	case DDI_CTLOPS_REGSIZE:
685	case DDI_CTLOPS_NREGS:
686		break;
687
688	case DDI_CTLOPS_SIDDEV:
689		if (ndi_dev_is_prom_node(rdip))
690			return (DDI_SUCCESS);
691		if (ndi_dev_is_persistent_node(rdip))
692			return (DDI_SUCCESS);
693		return (DDI_FAILURE);
694
695	case DDI_CTLOPS_POWER:
696		return ((*pm_platform_power)((power_req_t *)arg));
697
698	case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
699	case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
700	case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
701	case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
702	case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
703	case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
704		if (!rootnex_state->r_reserved_msg_printed) {
705			rootnex_state->r_reserved_msg_printed = B_TRUE;
706			cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
707			    "1 or more reserved/obsolete operations.");
708		}
709		return (DDI_FAILURE);
710
711	default:
712		return (DDI_FAILURE);
713	}
714	/*
715	 * The rest are for "hardware" properties
716	 */
717	if ((pdp = ddi_get_parent_data(rdip)) == NULL)
718		return (DDI_FAILURE);
719
720	if (ctlop == DDI_CTLOPS_NREGS) {
721		ptr = (int *)result;
722		*ptr = pdp->par_nreg;
723	} else {
724		off_t *size = (off_t *)result;
725
726		ptr = (int *)arg;
727		n = *ptr;
728		if (n >= pdp->par_nreg) {
729			return (DDI_FAILURE);
730		}
731		*size = (off_t)pdp->par_reg[n].regspec_size;
732	}
733	return (DDI_SUCCESS);
734}
735
736
737/*
738 * rootnex_ctl_reportdev()
739 *
740 */
741static int
742rootnex_ctl_reportdev(dev_info_t *dev)
743{
744	int i, n, len, f_len = 0;
745	char *buf;
746
747	buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
748	f_len += snprintf(buf, REPORTDEV_BUFSIZE,
749	    "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
750	len = strlen(buf);
751
752	for (i = 0; i < sparc_pd_getnreg(dev); i++) {
753
754		struct regspec *rp = sparc_pd_getreg(dev, i);
755
756		if (i == 0)
757			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
758			    ": ");
759		else
760			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
761			    " and ");
762		len = strlen(buf);
763
764		switch (rp->regspec_bustype) {
765
766		case BTEISA:
767			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
768			    "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
769			break;
770
771		case BTISA:
772			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
773			    "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
774			break;
775
776		default:
777			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
778			    "space %x offset %x",
779			    rp->regspec_bustype, rp->regspec_addr);
780			break;
781		}
782		len = strlen(buf);
783	}
784	for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
785		int pri;
786
787		if (i != 0) {
788			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
789			    ",");
790			len = strlen(buf);
791		}
792		pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
793		f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
794		    " sparc ipl %d", pri);
795		len = strlen(buf);
796	}
797#ifdef DEBUG
798	if (f_len + 1 >= REPORTDEV_BUFSIZE) {
799		cmn_err(CE_NOTE, "next message is truncated: "
800		    "printed length 1024, real length %d", f_len);
801	}
802#endif /* DEBUG */
803	cmn_err(CE_CONT, "?%s\n", buf);
804	kmem_free(buf, REPORTDEV_BUFSIZE);
805	return (DDI_SUCCESS);
806}
807
808
809/*
810 * ******************
811 *  map related code
812 * ******************
813 */
814
815/*
816 * rootnex_map()
817 *
818 */
819static int
820rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
821    off_t len, caddr_t *vaddrp)
822{
823	struct regspec *orp = NULL;
824	struct regspec64 rp = { 0 };
825	ddi_map_req_t mr = *mp;		/* Get private copy of request */
826
827	mp = &mr;
828
829	switch (mp->map_op)  {
830	case DDI_MO_MAP_LOCKED:
831	case DDI_MO_UNMAP:
832	case DDI_MO_MAP_HANDLE:
833		break;
834	default:
835#ifdef	DDI_MAP_DEBUG
836		cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
837		    mp->map_op);
838#endif	/* DDI_MAP_DEBUG */
839		return (DDI_ME_UNIMPLEMENTED);
840	}
841
842	if (mp->map_flags & DDI_MF_USER_MAPPING)  {
843#ifdef	DDI_MAP_DEBUG
844		cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
845#endif	/* DDI_MAP_DEBUG */
846		return (DDI_ME_UNIMPLEMENTED);
847	}
848
849	/*
850	 * First, we need to get the original regspec out before we convert it
851	 * to the extended format. If we have a register number, then we need to
852	 * convert that to a regspec.
853	 */
854	if (mp->map_type == DDI_MT_RNUMBER)  {
855
856		int rnumber = mp->map_obj.rnumber;
857#ifdef	DDI_MAP_DEBUG
858		static char *out_of_range =
859		    "rootnex_map: Out of range rnumber <%d>, device <%s>";
860#endif	/* DDI_MAP_DEBUG */
861
862		orp = i_ddi_rnumber_to_regspec(rdip, rnumber);
863		if (orp == NULL) {
864#ifdef	DDI_MAP_DEBUG
865			cmn_err(CE_WARN, out_of_range, rnumber,
866			    ddi_get_name(rdip));
867#endif	/* DDI_MAP_DEBUG */
868			return (DDI_ME_RNUMBER_RANGE);
869		}
870	} else if (!(mp->map_flags & DDI_MF_EXT_REGSPEC)) {
871		orp = mp->map_obj.rp;
872	}
873
874	/*
875	 * Ensure that we are always using a 64-bit extended regspec regardless
876	 * of what was passed into us. If the child driver is using a 64-bit
877	 * regspec, then we need to make sure that we copy this to the local
878	 * regspec64, rp.
879	 */
880	if (orp != NULL) {
881		rp.regspec_bustype = orp->regspec_bustype;
882		rp.regspec_addr = orp->regspec_addr;
883		rp.regspec_size = orp->regspec_size;
884	} else {
885		struct regspec64 *rp64;
886		rp64 = (struct regspec64 *)mp->map_obj.rp;
887		rp = *rp64;
888	}
889
890	mp->map_type = DDI_MT_REGSPEC;
891	mp->map_flags |= DDI_MF_EXT_REGSPEC;
892	mp->map_obj.rp = (struct regspec *)&rp;
893
894	/*
895	 * Adjust offset and length correspnding to called values...
896	 * XXX: A non-zero length means override the one in the regspec
897	 * XXX: (regardless of what's in the parent's range?)
898	 */
899
900#ifdef	DDI_MAP_DEBUG
901	cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
902	    "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
903	    rp.regspec_bustype, rp.regspec_addr, rp.regspec_size, offset,
904	    len, mp->map_handlep);
905#endif	/* DDI_MAP_DEBUG */
906
907	/*
908	 * I/O or memory mapping:
909	 *
910	 *	<bustype=0, addr=x, len=x>: memory
911	 *	<bustype=1, addr=x, len=x>: i/o
912	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
913	 */
914
915	if (rp.regspec_bustype > 1 && rp.regspec_addr != 0) {
916		cmn_err(CE_WARN, "<%s,%s> invalid register spec"
917		    " <0x%" PRIx64 ", 0x%" PRIx64 ", 0x%" PRIx64 ">",
918		    ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
919		    rp.regspec_addr, rp.regspec_size);
920		return (DDI_ME_INVAL);
921	}
922
923	if (rp.regspec_bustype > 1 && rp.regspec_addr == 0) {
924		/*
925		 * compatibility i/o mapping
926		 */
927		rp.regspec_bustype += offset;
928	} else {
929		/*
930		 * Normal memory or i/o mapping
931		 */
932		rp.regspec_addr += offset;
933	}
934
935	if (len != 0)
936		rp.regspec_size = len;
937
938#ifdef	DDI_MAP_DEBUG
939	cmn_err(CE_CONT, "             <%s,%s> <0x%" PRIx64 ", 0x%" PRIx64
940	    ", 0x%" PRId64 "> offset %d len %d handle 0x%x\n",
941	    ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
942	    rp.regspec_addr, rp.regspec_size, offset, len, mp->map_handlep);
943#endif	/* DDI_MAP_DEBUG */
944
945
946	/*
947	 * The x86 root nexus does not have any notion of valid ranges of
948	 * addresses. Its children have valid ranges, but because there are none
949	 * for the nexus, we don't need to call i_ddi_apply_range().  Verify
950	 * that is the case.
951	 */
952	ASSERT0(sparc_pd_getnrng(dip));
953
954	switch (mp->map_op)  {
955	case DDI_MO_MAP_LOCKED:
956
957		/*
958		 * Set up the locked down kernel mapping to the regspec...
959		 */
960
961		return (rootnex_map_regspec(mp, vaddrp));
962
963	case DDI_MO_UNMAP:
964
965		/*
966		 * Release mapping...
967		 */
968
969		return (rootnex_unmap_regspec(mp, vaddrp));
970
971	case DDI_MO_MAP_HANDLE:
972
973		return (rootnex_map_handle(mp));
974
975	default:
976		return (DDI_ME_UNIMPLEMENTED);
977	}
978}
979
980
981/*
982 * rootnex_map_fault()
983 *
984 *	fault in mappings for requestors
985 */
986/*ARGSUSED*/
987static int
988rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
989    struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
990    uint_t lock)
991{
992
993#ifdef	DDI_MAP_DEBUG
994	ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
995	ddi_map_debug(" Seg <%s>\n",
996	    seg->s_ops == &segdev_ops ? "segdev" :
997	    seg == &kvseg ? "segkmem" : "NONE!");
998#endif	/* DDI_MAP_DEBUG */
999
1000	/*
1001	 * This is all terribly broken, but it is a start
1002	 *
1003	 * XXX	Note that this test means that segdev_ops
1004	 *	must be exported from seg_dev.c.
1005	 * XXX	What about devices with their own segment drivers?
1006	 */
1007	if (seg->s_ops == &segdev_ops) {
1008		struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1009
1010		if (hat == NULL) {
1011			/*
1012			 * This is one plausible interpretation of
1013			 * a null hat i.e. use the first hat on the
1014			 * address space hat list which by convention is
1015			 * the hat of the system MMU.  At alternative
1016			 * would be to panic .. this might well be better ..
1017			 */
1018			ASSERT(AS_READ_HELD(seg->s_as));
1019			hat = seg->s_as->a_hat;
1020			cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1021		}
1022		hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1023		    (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1024	} else if (seg == &kvseg && dp == NULL) {
1025		hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1026		    HAT_LOAD_LOCK);
1027	} else
1028		return (DDI_FAILURE);
1029	return (DDI_SUCCESS);
1030}
1031
1032
1033static int
1034rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1035{
1036	rootnex_addr_t rbase;
1037	void *cvaddr;
1038	uint64_t npages, pgoffset;
1039	struct regspec64 *rp;
1040	ddi_acc_hdl_t *hp;
1041	ddi_acc_impl_t *ap;
1042	uint_t	hat_acc_flags;
1043	paddr_t pbase;
1044
1045	ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
1046	rp = (struct regspec64 *)mp->map_obj.rp;
1047	hp = mp->map_handlep;
1048
1049#ifdef	DDI_MAP_DEBUG
1050	ddi_map_debug(
1051	    "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1052	    rp->regspec_bustype, rp->regspec_addr,
1053	    rp->regspec_size, mp->map_handlep);
1054#endif	/* DDI_MAP_DEBUG */
1055
1056	/*
1057	 * I/O or memory mapping
1058	 *
1059	 *	<bustype=0, addr=x, len=x>: memory
1060	 *	<bustype=1, addr=x, len=x>: i/o
1061	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1062	 */
1063
1064	if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1065		cmn_err(CE_WARN, "rootnex: invalid register spec"
1066		    " <0x%" PRIx64 ", 0x%" PRIx64", 0x%" PRIx64">",
1067		    rp->regspec_bustype, rp->regspec_addr, rp->regspec_size);
1068		return (DDI_FAILURE);
1069	}
1070
1071	if (rp->regspec_bustype != 0) {
1072		/*
1073		 * I/O space - needs a handle.
1074		 */
1075		if (hp == NULL) {
1076			return (DDI_FAILURE);
1077		}
1078		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1079		ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1080		impl_acc_hdl_init(hp);
1081
1082		if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1083#ifdef  DDI_MAP_DEBUG
1084			ddi_map_debug("rootnex_map_regspec: mmap() "
1085			    "to I/O space is not supported.\n");
1086#endif  /* DDI_MAP_DEBUG */
1087			return (DDI_ME_INVAL);
1088		} else {
1089			/*
1090			 * 1275-compliant vs. compatibility i/o mapping
1091			 */
1092			*vaddrp =
1093			    (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1094			    ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1095			    ((caddr_t)(uintptr_t)rp->regspec_addr);
1096#ifdef __xpv
1097			if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1098				hp->ah_pfn = xen_assign_pfn(
1099				    mmu_btop((ulong_t)rp->regspec_addr &
1100				    MMU_PAGEMASK));
1101			} else {
1102				hp->ah_pfn = mmu_btop(
1103				    (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1104			}
1105#else
1106			hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1107			    MMU_PAGEMASK);
1108#endif
1109			hp->ah_pnum = mmu_btopr(rp->regspec_size +
1110			    (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1111		}
1112
1113#ifdef	DDI_MAP_DEBUG
1114		ddi_map_debug(
1115	    "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1116		    rp->regspec_size, *vaddrp);
1117#endif	/* DDI_MAP_DEBUG */
1118		return (DDI_SUCCESS);
1119	}
1120
1121	/*
1122	 * Memory space
1123	 */
1124
1125	if (hp != NULL) {
1126		/*
1127		 * hat layer ignores
1128		 * hp->ah_acc.devacc_attr_endian_flags.
1129		 */
1130		switch (hp->ah_acc.devacc_attr_dataorder) {
1131		case DDI_STRICTORDER_ACC:
1132			hat_acc_flags = HAT_STRICTORDER;
1133			break;
1134		case DDI_UNORDERED_OK_ACC:
1135			hat_acc_flags = HAT_UNORDERED_OK;
1136			break;
1137		case DDI_MERGING_OK_ACC:
1138			hat_acc_flags = HAT_MERGING_OK;
1139			break;
1140		case DDI_LOADCACHING_OK_ACC:
1141			hat_acc_flags = HAT_LOADCACHING_OK;
1142			break;
1143		case DDI_STORECACHING_OK_ACC:
1144			hat_acc_flags = HAT_STORECACHING_OK;
1145			break;
1146		default:
1147			return (DDI_ME_INVAL);
1148		}
1149		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1150		ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1151		impl_acc_hdl_init(hp);
1152		hp->ah_hat_flags = hat_acc_flags;
1153	} else {
1154		hat_acc_flags = HAT_STRICTORDER;
1155	}
1156
1157	rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1158#ifdef __xpv
1159	/*
1160	 * If we're dom0, we're using a real device so we need to translate
1161	 * the MA to a PA.
1162	 */
1163	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1164		pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1165	} else {
1166		pbase = rbase;
1167	}
1168#else
1169	pbase = rbase;
1170#endif
1171	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1172
1173	if (rp->regspec_size == 0) {
1174#ifdef  DDI_MAP_DEBUG
1175		ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1176#endif  /* DDI_MAP_DEBUG */
1177		return (DDI_ME_INVAL);
1178	}
1179
1180	if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1181		/* extra cast to make gcc happy */
1182		*vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1183	} else {
1184		npages = mmu_btopr(rp->regspec_size + pgoffset);
1185
1186#ifdef	DDI_MAP_DEBUG
1187		ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1188		    "physical %llx", npages, pbase);
1189#endif	/* DDI_MAP_DEBUG */
1190
1191		cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1192		if (cvaddr == NULL)
1193			return (DDI_ME_NORESOURCES);
1194
1195		/*
1196		 * Now map in the pages we've allocated...
1197		 */
1198		hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1199		    mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1200		    HAT_LOAD_LOCK);
1201		*vaddrp = (caddr_t)cvaddr + pgoffset;
1202
1203		/* save away pfn and npages for FMA */
1204		hp = mp->map_handlep;
1205		if (hp) {
1206			hp->ah_pfn = mmu_btop(pbase);
1207			hp->ah_pnum = npages;
1208		}
1209	}
1210
1211#ifdef	DDI_MAP_DEBUG
1212	ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1213#endif	/* DDI_MAP_DEBUG */
1214	return (DDI_SUCCESS);
1215}
1216
1217
1218static int
1219rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1220{
1221	caddr_t addr = (caddr_t)*vaddrp;
1222	uint64_t npages, pgoffset;
1223	struct regspec64 *rp;
1224
1225	if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1226		return (0);
1227
1228	ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
1229	rp = (struct regspec64 *)mp->map_obj.rp;
1230
1231	if (rp->regspec_size == 0) {
1232#ifdef  DDI_MAP_DEBUG
1233		ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1234#endif  /* DDI_MAP_DEBUG */
1235		return (DDI_ME_INVAL);
1236	}
1237
1238	/*
1239	 * I/O or memory mapping:
1240	 *
1241	 *	<bustype=0, addr=x, len=x>: memory
1242	 *	<bustype=1, addr=x, len=x>: i/o
1243	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1244	 */
1245	if (rp->regspec_bustype != 0) {
1246		/*
1247		 * This is I/O space, which requires no particular
1248		 * processing on unmap since it isn't mapped in the
1249		 * first place.
1250		 */
1251		return (DDI_SUCCESS);
1252	}
1253
1254	/*
1255	 * Memory space
1256	 */
1257	pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1258	npages = mmu_btopr(rp->regspec_size + pgoffset);
1259	hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1260	device_arena_free(addr - pgoffset, ptob(npages));
1261
1262	/*
1263	 * Destroy the pointer - the mapping has logically gone
1264	 */
1265	*vaddrp = NULL;
1266
1267	return (DDI_SUCCESS);
1268}
1269
1270static int
1271rootnex_map_handle(ddi_map_req_t *mp)
1272{
1273	rootnex_addr_t rbase;
1274	ddi_acc_hdl_t *hp;
1275	uint64_t pgoffset;
1276	struct regspec64 *rp;
1277	paddr_t pbase;
1278
1279	rp = (struct regspec64 *)mp->map_obj.rp;
1280
1281#ifdef	DDI_MAP_DEBUG
1282	ddi_map_debug(
1283	    "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1284	    rp->regspec_bustype, rp->regspec_addr,
1285	    rp->regspec_size, mp->map_handlep);
1286#endif	/* DDI_MAP_DEBUG */
1287
1288	/*
1289	 * I/O or memory mapping:
1290	 *
1291	 *	<bustype=0, addr=x, len=x>: memory
1292	 *	<bustype=1, addr=x, len=x>: i/o
1293	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1294	 */
1295	if (rp->regspec_bustype != 0) {
1296		/*
1297		 * This refers to I/O space, and we don't support "mapping"
1298		 * I/O space to a user.
1299		 */
1300		return (DDI_FAILURE);
1301	}
1302
1303	/*
1304	 * Set up the hat_flags for the mapping.
1305	 */
1306	hp = mp->map_handlep;
1307
1308	switch (hp->ah_acc.devacc_attr_endian_flags) {
1309	case DDI_NEVERSWAP_ACC:
1310		hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1311		break;
1312	case DDI_STRUCTURE_LE_ACC:
1313		hp->ah_hat_flags = HAT_STRUCTURE_LE;
1314		break;
1315	case DDI_STRUCTURE_BE_ACC:
1316		return (DDI_FAILURE);
1317	default:
1318		return (DDI_REGS_ACC_CONFLICT);
1319	}
1320
1321	switch (hp->ah_acc.devacc_attr_dataorder) {
1322	case DDI_STRICTORDER_ACC:
1323		break;
1324	case DDI_UNORDERED_OK_ACC:
1325		hp->ah_hat_flags |= HAT_UNORDERED_OK;
1326		break;
1327	case DDI_MERGING_OK_ACC:
1328		hp->ah_hat_flags |= HAT_MERGING_OK;
1329		break;
1330	case DDI_LOADCACHING_OK_ACC:
1331		hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1332		break;
1333	case DDI_STORECACHING_OK_ACC:
1334		hp->ah_hat_flags |= HAT_STORECACHING_OK;
1335		break;
1336	default:
1337		return (DDI_FAILURE);
1338	}
1339
1340	rbase = (rootnex_addr_t)rp->regspec_addr &
1341	    (~(rootnex_addr_t)MMU_PAGEOFFSET);
1342	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1343
1344	if (rp->regspec_size == 0)
1345		return (DDI_ME_INVAL);
1346
1347#ifdef __xpv
1348	/*
1349	 * If we're dom0, we're using a real device so we need to translate
1350	 * the MA to a PA.
1351	 */
1352	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1353		pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1354		    (rbase & MMU_PAGEOFFSET);
1355	} else {
1356		pbase = rbase;
1357	}
1358#else
1359	pbase = rbase;
1360#endif
1361
1362	hp->ah_pfn = mmu_btop(pbase);
1363	hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1364
1365	return (DDI_SUCCESS);
1366}
1367
1368
1369
1370/*
1371 * ************************
1372 *  interrupt related code
1373 * ************************
1374 */
1375
1376/*
1377 * rootnex_intr_ops()
1378 *	bus_intr_op() function for interrupt support
1379 */
1380/* ARGSUSED */
1381static int
1382rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1383    ddi_intr_handle_impl_t *hdlp, void *result)
1384{
1385	struct intrspec			*ispec;
1386
1387	DDI_INTR_NEXDBG((CE_CONT,
1388	    "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1389	    (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1390
1391	/* Process the interrupt operation */
1392	switch (intr_op) {
1393	case DDI_INTROP_GETCAP:
1394		/* First check with pcplusmp */
1395		if (psm_intr_ops == NULL)
1396			return (DDI_FAILURE);
1397
1398		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1399			*(int *)result = 0;
1400			return (DDI_FAILURE);
1401		}
1402		break;
1403	case DDI_INTROP_SETCAP:
1404		if (psm_intr_ops == NULL)
1405			return (DDI_FAILURE);
1406
1407		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1408			return (DDI_FAILURE);
1409		break;
1410	case DDI_INTROP_ALLOC:
1411		ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1412		return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1413	case DDI_INTROP_FREE:
1414		ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1415		return (rootnex_free_intr_fixed(rdip, hdlp));
1416	case DDI_INTROP_GETPRI:
1417		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1418			return (DDI_FAILURE);
1419		*(int *)result = ispec->intrspec_pri;
1420		break;
1421	case DDI_INTROP_SETPRI:
1422		/* Validate the interrupt priority passed to us */
1423		if (*(int *)result > LOCK_LEVEL)
1424			return (DDI_FAILURE);
1425
1426		/* Ensure that PSM is all initialized and ispec is ok */
1427		if ((psm_intr_ops == NULL) ||
1428		    ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1429			return (DDI_FAILURE);
1430
1431		/* Change the priority */
1432		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1433		    PSM_FAILURE)
1434			return (DDI_FAILURE);
1435
1436		/* update the ispec with the new priority */
1437		ispec->intrspec_pri =  *(int *)result;
1438		break;
1439	case DDI_INTROP_ADDISR:
1440		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1441			return (DDI_FAILURE);
1442		ispec->intrspec_func = hdlp->ih_cb_func;
1443		break;
1444	case DDI_INTROP_REMISR:
1445		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1446			return (DDI_FAILURE);
1447		ispec->intrspec_func = (uint_t (*)()) 0;
1448		break;
1449	case DDI_INTROP_ENABLE:
1450		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1451			return (DDI_FAILURE);
1452
1453		/* Call psmi to translate irq with the dip */
1454		if (psm_intr_ops == NULL)
1455			return (DDI_FAILURE);
1456
1457		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1458		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1459		    (int *)&hdlp->ih_vector) == PSM_FAILURE)
1460			return (DDI_FAILURE);
1461
1462		/* Add the interrupt handler */
1463		if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1464		    hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1465		    hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1466			return (DDI_FAILURE);
1467		break;
1468	case DDI_INTROP_DISABLE:
1469		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1470			return (DDI_FAILURE);
1471
1472		/* Call psm_ops() to translate irq with the dip */
1473		if (psm_intr_ops == NULL)
1474			return (DDI_FAILURE);
1475
1476		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1477		(void) (*psm_intr_ops)(rdip, hdlp,
1478		    PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1479
1480		/* Remove the interrupt handler */
1481		rem_avintr((void *)hdlp, ispec->intrspec_pri,
1482		    hdlp->ih_cb_func, hdlp->ih_vector);
1483		break;
1484	case DDI_INTROP_SETMASK:
1485		if (psm_intr_ops == NULL)
1486			return (DDI_FAILURE);
1487
1488		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1489			return (DDI_FAILURE);
1490		break;
1491	case DDI_INTROP_CLRMASK:
1492		if (psm_intr_ops == NULL)
1493			return (DDI_FAILURE);
1494
1495		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1496			return (DDI_FAILURE);
1497		break;
1498	case DDI_INTROP_GETPENDING:
1499		if (psm_intr_ops == NULL)
1500			return (DDI_FAILURE);
1501
1502		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1503		    result)) {
1504			*(int *)result = 0;
1505			return (DDI_FAILURE);
1506		}
1507		break;
1508	case DDI_INTROP_NAVAIL:
1509	case DDI_INTROP_NINTRS:
1510		*(int *)result = i_ddi_get_intx_nintrs(rdip);
1511		if (*(int *)result == 0) {
1512			/*
1513			 * Special case for 'pcic' driver' only. This driver
1514			 * driver is a child of 'isa' and 'rootnex' drivers.
1515			 *
1516			 * See detailed comments on this in the function
1517			 * rootnex_get_ispec().
1518			 *
1519			 * Children of 'pcic' send 'NINITR' request all the
1520			 * way to rootnex driver. But, the 'pdp->par_nintr'
1521			 * field may not initialized. So, we fake it here
1522			 * to return 1 (a la what PCMCIA nexus does).
1523			 */
1524			if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1525				*(int *)result = 1;
1526			else
1527				return (DDI_FAILURE);
1528		}
1529		break;
1530	case DDI_INTROP_SUPPORTED_TYPES:
1531		*(int *)result = DDI_INTR_TYPE_FIXED;	/* Always ... */
1532		break;
1533	default:
1534		return (DDI_FAILURE);
1535	}
1536
1537	return (DDI_SUCCESS);
1538}
1539
1540
1541/*
1542 * rootnex_get_ispec()
1543 *	convert an interrupt number to an interrupt specification.
1544 *	The interrupt number determines which interrupt spec will be
1545 *	returned if more than one exists.
1546 *
1547 *	Look into the parent private data area of the 'rdip' to find out
1548 *	the interrupt specification.  First check to make sure there is
1549 *	one that matchs "inumber" and then return a pointer to it.
1550 *
1551 *	Return NULL if one could not be found.
1552 *
1553 *	NOTE: This is needed for rootnex_intr_ops()
1554 */
1555static struct intrspec *
1556rootnex_get_ispec(dev_info_t *rdip, int inum)
1557{
1558	struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1559
1560	/*
1561	 * Special case handling for drivers that provide their own
1562	 * intrspec structures instead of relying on the DDI framework.
1563	 *
1564	 * A broken hardware driver in ON could potentially provide its
1565	 * own intrspec structure, instead of relying on the hardware.
1566	 * If these drivers are children of 'rootnex' then we need to
1567	 * continue to provide backward compatibility to them here.
1568	 *
1569	 * Following check is a special case for 'pcic' driver which
1570	 * was found to have broken hardwre andby provides its own intrspec.
1571	 *
1572	 * Verbatim comments from this driver are shown here:
1573	 * "Don't use the ddi_add_intr since we don't have a
1574	 * default intrspec in all cases."
1575	 *
1576	 * Since an 'ispec' may not be always created for it,
1577	 * check for that and create one if so.
1578	 *
1579	 * NOTE: Currently 'pcic' is the only driver found to do this.
1580	 */
1581	if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1582		pdp->par_nintr = 1;
1583		pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1584		    pdp->par_nintr, KM_SLEEP);
1585	}
1586
1587	/* Validate the interrupt number */
1588	if (inum >= pdp->par_nintr)
1589		return (NULL);
1590
1591	/* Get the interrupt structure pointer and return that */
1592	return ((struct intrspec *)&pdp->par_intr[inum]);
1593}
1594
1595/*
1596 * Allocate interrupt vector for FIXED (legacy) type.
1597 */
1598static int
1599rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1600    void *result)
1601{
1602	struct intrspec		*ispec;
1603	ddi_intr_handle_impl_t	info_hdl;
1604	int			ret;
1605	int			free_phdl = 0;
1606	apic_get_type_t		type_info;
1607
1608	if (psm_intr_ops == NULL)
1609		return (DDI_FAILURE);
1610
1611	if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1612		return (DDI_FAILURE);
1613
1614	/*
1615	 * If the PSM module is "APIX" then pass the request for it
1616	 * to allocate the vector now.
1617	 */
1618	bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1619	info_hdl.ih_private = &type_info;
1620	if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1621	    PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1622		if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1623			free_phdl = 1;
1624			i_ddi_alloc_intr_phdl(hdlp);
1625		}
1626		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1627		ret = (*psm_intr_ops)(rdip, hdlp,
1628		    PSM_INTR_OP_ALLOC_VECTORS, result);
1629		if (free_phdl) { /* free up the phdl structure */
1630			free_phdl = 0;
1631			i_ddi_free_intr_phdl(hdlp);
1632			hdlp->ih_private = NULL;
1633		}
1634	} else {
1635		/*
1636		 * No APIX module; fall back to the old scheme where the
1637		 * interrupt vector is allocated during ddi_enable_intr() call.
1638		 */
1639		hdlp->ih_pri = ispec->intrspec_pri;
1640		*(int *)result = hdlp->ih_scratch1;
1641		ret = DDI_SUCCESS;
1642	}
1643
1644	return (ret);
1645}
1646
1647/*
1648 * Free up interrupt vector for FIXED (legacy) type.
1649 */
1650static int
1651rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1652{
1653	struct intrspec			*ispec;
1654	struct ddi_parent_private_data	*pdp;
1655	ddi_intr_handle_impl_t		info_hdl;
1656	int				ret;
1657	apic_get_type_t			type_info;
1658
1659	if (psm_intr_ops == NULL)
1660		return (DDI_FAILURE);
1661
1662	/*
1663	 * If the PSM module is "APIX" then pass the request for it
1664	 * to free up the vector now.
1665	 */
1666	bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1667	info_hdl.ih_private = &type_info;
1668	if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1669	    PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1670		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1671			return (DDI_FAILURE);
1672		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1673		ret = (*psm_intr_ops)(rdip, hdlp,
1674		    PSM_INTR_OP_FREE_VECTORS, NULL);
1675	} else {
1676		/*
1677		 * No APIX module; fall back to the old scheme where
1678		 * the interrupt vector was already freed during
1679		 * ddi_disable_intr() call.
1680		 */
1681		ret = DDI_SUCCESS;
1682	}
1683
1684	pdp = ddi_get_parent_data(rdip);
1685
1686	/*
1687	 * Special case for 'pcic' driver' only.
1688	 * If an intrspec was created for it, clean it up here
1689	 * See detailed comments on this in the function
1690	 * rootnex_get_ispec().
1691	 */
1692	if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1693		kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1694		    pdp->par_nintr);
1695		/*
1696		 * Set it to zero; so that
1697		 * DDI framework doesn't free it again
1698		 */
1699		pdp->par_intr = NULL;
1700		pdp->par_nintr = 0;
1701	}
1702
1703	return (ret);
1704}
1705
1706
1707/*
1708 * ******************
1709 *  dma related code
1710 * ******************
1711 */
1712
1713/*ARGSUSED*/
1714static int
1715rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1716    ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1717    ddi_dma_handle_t *handlep)
1718{
1719	uint64_t maxsegmentsize_ll;
1720	uint_t maxsegmentsize;
1721	ddi_dma_impl_t *hp;
1722	rootnex_dma_t *dma;
1723	uint64_t count_max;
1724	uint64_t seg;
1725	int kmflag;
1726	int e;
1727
1728
1729	/* convert our sleep flags */
1730	if (waitfp == DDI_DMA_SLEEP) {
1731		kmflag = KM_SLEEP;
1732	} else {
1733		kmflag = KM_NOSLEEP;
1734	}
1735
1736	/*
1737	 * We try to do only one memory allocation here. We'll do a little
1738	 * pointer manipulation later. If the bind ends up taking more than
1739	 * our prealloc's space, we'll have to allocate more memory in the
1740	 * bind operation. Not great, but much better than before and the
1741	 * best we can do with the current bind interfaces.
1742	 */
1743	hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1744	if (hp == NULL)
1745		return (DDI_DMA_NORESOURCES);
1746
1747	/* Do our pointer manipulation now, align the structures */
1748	hp->dmai_private = (void *)(((uintptr_t)hp +
1749	    (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1750	dma = (rootnex_dma_t *)hp->dmai_private;
1751	dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1752	    sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1753
1754	/* setup the handle */
1755	rootnex_clean_dmahdl(hp);
1756	hp->dmai_error.err_fep = NULL;
1757	hp->dmai_error.err_cf = NULL;
1758	dma->dp_dip = rdip;
1759	dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1760	dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1761
1762	/*
1763	 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1764	 * is being used. Set the upper limit to the seg value.
1765	 * There will be enough DVMA space to always get addresses
1766	 * that will match the constraints.
1767	 */
1768	if (IOMMU_USED(rdip) &&
1769	    (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1770		dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1771		dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1772	} else
1773		dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1774
1775	hp->dmai_minxfer = attr->dma_attr_minxfer;
1776	hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1777	hp->dmai_rdip = rdip;
1778	hp->dmai_attr = *attr;
1779
1780	if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1781		dma->dp_sglinfo.si_cancross = B_FALSE;
1782	else
1783		dma->dp_sglinfo.si_cancross = B_TRUE;
1784
1785	/* we don't need to worry about the SPL since we do a tryenter */
1786	mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1787
1788	/*
1789	 * Figure out our maximum segment size. If the segment size is greater
1790	 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1791	 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1792	 * dma_attr_count_max are size-1 type values.
1793	 *
1794	 * Maximum segment size is the largest physically contiguous chunk of
1795	 * memory that we can return from a bind (i.e. the maximum size of a
1796	 * single cookie).
1797	 */
1798
1799	/* handle the rollover cases */
1800	seg = attr->dma_attr_seg + 1;
1801	if (seg < attr->dma_attr_seg) {
1802		seg = attr->dma_attr_seg;
1803	}
1804	count_max = attr->dma_attr_count_max + 1;
1805	if (count_max < attr->dma_attr_count_max) {
1806		count_max = attr->dma_attr_count_max;
1807	}
1808
1809	/*
1810	 * granularity may or may not be a power of two. If it isn't, we can't
1811	 * use a simple mask.
1812	 */
1813	if (!ISP2(attr->dma_attr_granular)) {
1814		dma->dp_granularity_power_2 = B_FALSE;
1815	} else {
1816		dma->dp_granularity_power_2 = B_TRUE;
1817	}
1818
1819	/*
1820	 * maxxfer should be a whole multiple of granularity. If we're going to
1821	 * break up a window because we're greater than maxxfer, we might as
1822	 * well make sure it's maxxfer is a whole multiple so we don't have to
1823	 * worry about triming the window later on for this case.
1824	 */
1825	if (attr->dma_attr_granular > 1) {
1826		if (dma->dp_granularity_power_2) {
1827			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1828			    (attr->dma_attr_maxxfer &
1829			    (attr->dma_attr_granular - 1));
1830		} else {
1831			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1832			    (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1833		}
1834	} else {
1835		dma->dp_maxxfer = attr->dma_attr_maxxfer;
1836	}
1837
1838	maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1839	maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1840	if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1841		maxsegmentsize = 0xFFFFFFFF;
1842	} else {
1843		maxsegmentsize = maxsegmentsize_ll;
1844	}
1845	dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1846	dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1847
1848	/* check the ddi_dma_attr arg to make sure it makes a little sense */
1849	if (rootnex_alloc_check_parms) {
1850		e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1851		if (e != DDI_SUCCESS) {
1852			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1853			(void) rootnex_dma_freehdl(dip, rdip,
1854			    (ddi_dma_handle_t)hp);
1855			return (e);
1856		}
1857	}
1858
1859	*handlep = (ddi_dma_handle_t)hp;
1860
1861	ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1862	ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1863	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1864
1865	return (DDI_SUCCESS);
1866}
1867
1868
1869/*
1870 * rootnex_dma_allochdl()
1871 *    called from ddi_dma_alloc_handle().
1872 */
1873static int
1874rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1875    int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1876{
1877	int retval = DDI_SUCCESS;
1878#if defined(__amd64) && !defined(__xpv)
1879
1880	if (IOMMU_UNITIALIZED(rdip)) {
1881		retval = iommulib_nex_open(dip, rdip);
1882
1883		if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1884			return (retval);
1885	}
1886
1887	if (IOMMU_UNUSED(rdip)) {
1888		retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1889		    handlep);
1890	} else {
1891		retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1892		    waitfp, arg, handlep);
1893	}
1894#else
1895	retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1896	    handlep);
1897#endif
1898	switch (retval) {
1899	case DDI_DMA_NORESOURCES:
1900		if (waitfp != DDI_DMA_DONTWAIT) {
1901			ddi_set_callback(waitfp, arg,
1902			    &rootnex_state->r_dvma_call_list_id);
1903		}
1904		break;
1905	case DDI_SUCCESS:
1906		ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1907		break;
1908	default:
1909		break;
1910	}
1911	return (retval);
1912}
1913
1914/*ARGSUSED*/
1915static int
1916rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1917    ddi_dma_handle_t handle)
1918{
1919	ddi_dma_impl_t *hp;
1920	rootnex_dma_t *dma;
1921
1922
1923	hp = (ddi_dma_impl_t *)handle;
1924	dma = (rootnex_dma_t *)hp->dmai_private;
1925
1926	/* unbind should have been called first */
1927	ASSERT(!dma->dp_inuse);
1928
1929	mutex_destroy(&dma->dp_mutex);
1930	kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1931
1932	ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1933	ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1934	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1935
1936	return (DDI_SUCCESS);
1937}
1938
1939/*
1940 * rootnex_dma_freehdl()
1941 *    called from ddi_dma_free_handle().
1942 */
1943static int
1944rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1945{
1946	int ret;
1947
1948	ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1949#if defined(__amd64) && !defined(__xpv)
1950	if (IOMMU_USED(rdip))
1951		ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1952	else
1953#endif
1954	ret = rootnex_coredma_freehdl(dip, rdip, handle);
1955
1956	if (rootnex_state->r_dvma_call_list_id)
1957		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1958
1959	return (ret);
1960}
1961
1962/*ARGSUSED*/
1963static int
1964rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1965    ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1966    ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1967{
1968	rootnex_sglinfo_t *sinfo;
1969	ddi_dma_obj_t *dmao;
1970#if defined(__amd64) && !defined(__xpv)
1971	struct dvmaseg *dvs;
1972	ddi_dma_cookie_t *cookie;
1973#endif
1974	ddi_dma_attr_t *attr;
1975	ddi_dma_impl_t *hp;
1976	rootnex_dma_t *dma;
1977	int kmflag;
1978	int e;
1979	uint_t ncookies;
1980
1981	hp = (ddi_dma_impl_t *)handle;
1982	dma = (rootnex_dma_t *)hp->dmai_private;
1983	dmao = &dma->dp_dma;
1984	sinfo = &dma->dp_sglinfo;
1985	attr = &hp->dmai_attr;
1986
1987	/* convert the sleep flags */
1988	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1989		dma->dp_sleep_flags = kmflag = KM_SLEEP;
1990	} else {
1991		dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1992	}
1993
1994	hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1995
1996	/*
1997	 * This is useful for debugging a driver. Not as useful in a production
1998	 * system. The only time this will fail is if you have a driver bug.
1999	 */
2000	if (rootnex_bind_check_inuse) {
2001		/*
2002		 * No one else should ever have this lock unless someone else
2003		 * is trying to use this handle. So contention on the lock
2004		 * is the same as inuse being set.
2005		 */
2006		e = mutex_tryenter(&dma->dp_mutex);
2007		if (e == 0) {
2008			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2009			return (DDI_DMA_INUSE);
2010		}
2011		if (dma->dp_inuse) {
2012			mutex_exit(&dma->dp_mutex);
2013			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2014			return (DDI_DMA_INUSE);
2015		}
2016		dma->dp_inuse = B_TRUE;
2017		mutex_exit(&dma->dp_mutex);
2018	}
2019
2020	/* check the ddi_dma_attr arg to make sure it makes a little sense */
2021	if (rootnex_bind_check_parms) {
2022		e = rootnex_valid_bind_parms(dmareq, attr);
2023		if (e != DDI_SUCCESS) {
2024			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2025			rootnex_clean_dmahdl(hp);
2026			return (e);
2027		}
2028	}
2029
2030	/* save away the original bind info */
2031	dma->dp_dma = dmareq->dmar_object;
2032
2033#if defined(__amd64) && !defined(__xpv)
2034	if (IOMMU_USED(rdip)) {
2035		dmao = &dma->dp_dvma;
2036		e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2037		switch (e) {
2038		case DDI_SUCCESS:
2039			if (sinfo->si_cancross ||
2040			    dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2041			    dmao->dmao_size > sinfo->si_max_cookie_size) {
2042				dma->dp_dvma_used = B_TRUE;
2043				break;
2044			}
2045			sinfo->si_sgl_size = 1;
2046			hp->dmai_rflags |= DMP_NOSYNC;
2047
2048			dma->dp_dvma_used = B_TRUE;
2049			dma->dp_need_to_free_cookie = B_FALSE;
2050
2051			dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2052			cookie = hp->dmai_cookie = dma->dp_cookies =
2053			    (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2054			cookie->dmac_laddress = dvs->dvs_start +
2055			    dmao->dmao_obj.dvma_obj.dv_off;
2056			cookie->dmac_size = dvs->dvs_len;
2057			cookie->dmac_type = 0;
2058
2059			ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2060			    rdip);
2061			goto fast;
2062		case DDI_ENOTSUP:
2063			break;
2064		default:
2065			rootnex_clean_dmahdl(hp);
2066			return (e);
2067		}
2068	}
2069#endif
2070
2071	/*
2072	 * Figure out a rough estimate of what maximum number of pages
2073	 * this buffer could use (a high estimate of course).
2074	 */
2075	sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2076
2077	if (dma->dp_dvma_used) {
2078		/*
2079		 * The number of physical pages is the worst case.
2080		 *
2081		 * For DVMA, the worst case is the length divided
2082		 * by the maximum cookie length, plus 1. Add to that
2083		 * the number of segment boundaries potentially crossed, and
2084		 * the additional number of DVMA segments that was returned.
2085		 *
2086		 * In the normal case, for modern devices, si_cancross will
2087		 * be false, and dv_nseg will be 1, and the fast path will
2088		 * have been taken above.
2089		 */
2090		ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2091		    + 1;
2092		if (sinfo->si_cancross)
2093			ncookies +=
2094			    (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2095		ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2096
2097		sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2098	}
2099
2100	/*
2101	 * We'll use the pre-allocated cookies for any bind that will *always*
2102	 * fit (more important to be consistent, we don't want to create
2103	 * additional degenerate cases).
2104	 */
2105	if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2106		dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2107		dma->dp_need_to_free_cookie = B_FALSE;
2108		ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2109		    uint_t, sinfo->si_max_pages);
2110
2111	/*
2112	 * For anything larger than that, we'll go ahead and allocate the
2113	 * maximum number of pages we expect to see. Hopefuly, we won't be
2114	 * seeing this path in the fast path for high performance devices very
2115	 * frequently.
2116	 *
2117	 * a ddi bind interface that allowed the driver to provide storage to
2118	 * the bind interface would speed this case up.
2119	 */
2120	} else {
2121		/*
2122		 * Save away how much memory we allocated. If we're doing a
2123		 * nosleep, the alloc could fail...
2124		 */
2125		dma->dp_cookie_size = sinfo->si_max_pages *
2126		    sizeof (ddi_dma_cookie_t);
2127		dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2128		if (dma->dp_cookies == NULL) {
2129			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2130			rootnex_clean_dmahdl(hp);
2131			return (DDI_DMA_NORESOURCES);
2132		}
2133		dma->dp_need_to_free_cookie = B_TRUE;
2134		ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2135		    uint_t, sinfo->si_max_pages);
2136	}
2137	hp->dmai_cookie = dma->dp_cookies;
2138
2139	/*
2140	 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2141	 * looking at the constraints in the dma structure. It will then put
2142	 * some additional state about the sgl in the dma struct (i.e. is
2143	 * the sgl clean, or do we need to do some munging; how many pages
2144	 * need to be copied, etc.)
2145	 */
2146	if (dma->dp_dvma_used)
2147		rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2148	else
2149		rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2150
2151out:
2152	ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2153	/* if we don't need a copy buffer, we don't need to sync */
2154	if (sinfo->si_copybuf_req == 0) {
2155		hp->dmai_rflags |= DMP_NOSYNC;
2156	}
2157
2158	/*
2159	 * if we don't need the copybuf and we don't need to do a partial,  we
2160	 * hit the fast path. All the high performance devices should be trying
2161	 * to hit this path. To hit this path, a device should be able to reach
2162	 * all of memory, shouldn't try to bind more than it can transfer, and
2163	 * the buffer shouldn't require more cookies than the driver/device can
2164	 * handle [sgllen]).
2165	 *
2166	 * Note that negative values of dma_attr_sgllen are supposed
2167	 * to mean unlimited, but we just cast them to mean a
2168	 * "ridiculous large limit".  This saves some extra checks on
2169	 * hot paths.
2170	 */
2171	if ((sinfo->si_copybuf_req == 0) &&
2172	    (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2173	    (dmao->dmao_size <= dma->dp_maxxfer)) {
2174fast:
2175		/*
2176		 * If the driver supports FMA, insert the handle in the FMA DMA
2177		 * handle cache.
2178		 */
2179		if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2180			hp->dmai_error.err_cf = rootnex_dma_check;
2181
2182		/*
2183		 * copy out the first cookie and ccountp, set the cookie
2184		 * pointer to the second cookie. The first cookie is passed
2185		 * back on the stack. Additional cookies are accessed via
2186		 * ddi_dma_nextcookie()
2187		 */
2188		*cookiep = dma->dp_cookies[0];
2189		*ccountp = sinfo->si_sgl_size;
2190		hp->dmai_cookie++;
2191		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2192		hp->dmai_ncookies = *ccountp;
2193		hp->dmai_curcookie = 1;
2194		ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2195		ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2196		    uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2197		    uint_t, dmao->dmao_size, uint_t, *ccountp);
2198
2199
2200		return (DDI_DMA_MAPPED);
2201	}
2202
2203	/*
2204	 * go to the slow path, we may need to alloc more memory, create
2205	 * multiple windows, and munge up a sgl to make the device happy.
2206	 */
2207
2208	/*
2209	 * With the IOMMU mapobject method used, we should never hit
2210	 * the slow path. If we do, something is seriously wrong.
2211	 * Clean up and return an error.
2212	 */
2213
2214#if defined(__amd64) && !defined(__xpv)
2215
2216	if (dma->dp_dvma_used) {
2217		(void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2218		    &dma->dp_dvma);
2219		e = DDI_DMA_NOMAPPING;
2220	} else {
2221#endif
2222		e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2223		    kmflag);
2224#if defined(__amd64) && !defined(__xpv)
2225	}
2226#endif
2227	if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2228		if (dma->dp_need_to_free_cookie) {
2229			kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2230		}
2231		ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2232		rootnex_clean_dmahdl(hp); /* must be after free cookie */
2233		return (e);
2234	}
2235
2236	/*
2237	 * If the driver supports FMA, insert the handle in the FMA DMA handle
2238	 * cache.
2239	 */
2240	if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2241		hp->dmai_error.err_cf = rootnex_dma_check;
2242
2243	/* if the first window uses the copy buffer, sync it for the device */
2244	if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2245	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
2246		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2247		    DDI_DMA_SYNC_FORDEV);
2248	}
2249
2250	/*
2251	 * copy out the first cookie and ccountp, set the cookie pointer to the
2252	 * second cookie. Make sure the partial flag is set/cleared correctly.
2253	 * If we have a partial map (i.e. multiple windows), the number of
2254	 * cookies we return is the number of cookies in the first window.
2255	 */
2256	if (e == DDI_DMA_MAPPED) {
2257		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2258		*ccountp = sinfo->si_sgl_size;
2259		hp->dmai_nwin = 1;
2260	} else {
2261		hp->dmai_rflags |= DDI_DMA_PARTIAL;
2262		*ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2263		ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2264	}
2265	*cookiep = dma->dp_cookies[0];
2266	hp->dmai_cookie++;
2267	hp->dmai_ncookies = *ccountp;
2268	hp->dmai_curcookie = 1;
2269
2270	ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2271	ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2272	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2273	    dmao->dmao_size, uint_t, *ccountp);
2274	return (e);
2275}
2276
2277/*
2278 * rootnex_dma_bindhdl()
2279 *    called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2280 */
2281static int
2282rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2283    ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2284    ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2285{
2286	int ret;
2287#if defined(__amd64) && !defined(__xpv)
2288	if (IOMMU_USED(rdip))
2289		ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2290		    cookiep, ccountp);
2291	else
2292#endif
2293	ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2294	    cookiep, ccountp);
2295
2296	if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2297		ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2298		    &rootnex_state->r_dvma_call_list_id);
2299	}
2300
2301	return (ret);
2302}
2303
2304
2305
2306/*ARGSUSED*/
2307static int
2308rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2309    ddi_dma_handle_t handle)
2310{
2311	ddi_dma_impl_t *hp;
2312	rootnex_dma_t *dma;
2313	int e;
2314
2315	hp = (ddi_dma_impl_t *)handle;
2316	dma = (rootnex_dma_t *)hp->dmai_private;
2317
2318	/* make sure the buffer wasn't free'd before calling unbind */
2319	if (rootnex_unbind_verify_buffer) {
2320		e = rootnex_verify_buffer(dma);
2321		if (e != DDI_SUCCESS) {
2322			ASSERT(0);
2323			return (DDI_FAILURE);
2324		}
2325	}
2326
2327	/* sync the current window before unbinding the buffer */
2328	if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2329	    (hp->dmai_rflags & DDI_DMA_READ)) {
2330		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2331		    DDI_DMA_SYNC_FORCPU);
2332	}
2333
2334	/*
2335	 * cleanup and copy buffer or window state. if we didn't use the copy
2336	 * buffer or windows, there won't be much to do :-)
2337	 */
2338	rootnex_teardown_copybuf(dma);
2339	rootnex_teardown_windows(dma);
2340
2341#if defined(__amd64) && !defined(__xpv)
2342	if (IOMMU_USED(rdip) && dma->dp_dvma_used)
2343		(void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2344		    &dma->dp_dvma);
2345#endif
2346
2347	/*
2348	 * If we had to allocate space to for the worse case sgl (it didn't
2349	 * fit into our pre-allocate buffer), free that up now
2350	 */
2351	if (dma->dp_need_to_free_cookie) {
2352		kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2353	}
2354
2355	/*
2356	 * clean up the handle so it's ready for the next bind (i.e. if the
2357	 * handle is reused).
2358	 */
2359	rootnex_clean_dmahdl(hp);
2360	hp->dmai_error.err_cf = NULL;
2361
2362	ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2363	ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2364	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2365
2366	return (DDI_SUCCESS);
2367}
2368
2369/*
2370 * rootnex_dma_unbindhdl()
2371 *    called from ddi_dma_unbind_handle()
2372 */
2373/*ARGSUSED*/
2374static int
2375rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2376    ddi_dma_handle_t handle)
2377{
2378	int ret;
2379
2380#if defined(__amd64) && !defined(__xpv)
2381	if (IOMMU_USED(rdip))
2382		ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2383	else
2384#endif
2385	ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2386
2387	if (rootnex_state->r_dvma_call_list_id)
2388		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2389
2390	return (ret);
2391}
2392
2393#if defined(__amd64) && !defined(__xpv)
2394
2395static int
2396rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2397{
2398	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2399	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2400
2401	if (dma->dp_sleep_flags != KM_SLEEP &&
2402	    dma->dp_sleep_flags != KM_NOSLEEP)
2403		cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2404	return (dma->dp_sleep_flags);
2405}
2406/*ARGSUSED*/
2407static void
2408rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2409{
2410	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2411	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2412	rootnex_window_t *window;
2413
2414	if (dma->dp_window) {
2415		window = &dma->dp_window[dma->dp_current_win];
2416		hp->dmai_cookie = window->wd_first_cookie;
2417	} else {
2418		hp->dmai_cookie = dma->dp_cookies;
2419	}
2420	hp->dmai_cookie++;
2421	hp->dmai_curcookie = 1;
2422}
2423
2424/*ARGSUSED*/
2425static int
2426rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2427    ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2428{
2429	int i;
2430	int km_flags;
2431	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2432	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2433	rootnex_window_t *window;
2434	ddi_dma_cookie_t *cp;
2435	ddi_dma_cookie_t *cookie;
2436
2437	ASSERT(*cookiepp == NULL);
2438	ASSERT(*ccountp == 0);
2439
2440	if (dma->dp_window) {
2441		window = &dma->dp_window[dma->dp_current_win];
2442		cp = window->wd_first_cookie;
2443		*ccountp = window->wd_cookie_cnt;
2444	} else {
2445		cp = dma->dp_cookies;
2446		*ccountp = dma->dp_sglinfo.si_sgl_size;
2447	}
2448
2449	km_flags = rootnex_coredma_get_sleep_flags(handle);
2450	cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2451	if (cookie == NULL) {
2452		return (DDI_DMA_NORESOURCES);
2453	}
2454
2455	for (i = 0; i < *ccountp; i++) {
2456		cookie[i].dmac_notused = cp[i].dmac_notused;
2457		cookie[i].dmac_type = cp[i].dmac_type;
2458		cookie[i].dmac_address = cp[i].dmac_address;
2459		cookie[i].dmac_size = cp[i].dmac_size;
2460	}
2461
2462	*cookiepp = cookie;
2463
2464	return (DDI_SUCCESS);
2465}
2466
2467/*ARGSUSED*/
2468static int
2469rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2470    ddi_dma_cookie_t *cookiep, uint_t ccount)
2471{
2472	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2473	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2474	rootnex_window_t *window;
2475	ddi_dma_cookie_t *cur_cookiep;
2476
2477	ASSERT(cookiep);
2478	ASSERT(ccount != 0);
2479	ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2480
2481	if (dma->dp_window) {
2482		window = &dma->dp_window[dma->dp_current_win];
2483		dma->dp_saved_cookies = window->wd_first_cookie;
2484		window->wd_first_cookie = cookiep;
2485		ASSERT(ccount == window->wd_cookie_cnt);
2486		cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2487		    + window->wd_first_cookie;
2488	} else {
2489		dma->dp_saved_cookies = dma->dp_cookies;
2490		dma->dp_cookies = cookiep;
2491		ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2492		cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2493		    + dma->dp_cookies;
2494	}
2495
2496	dma->dp_need_to_switch_cookies = B_TRUE;
2497	hp->dmai_cookie = cur_cookiep;
2498
2499	return (DDI_SUCCESS);
2500}
2501
2502/*ARGSUSED*/
2503static int
2504rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2505{
2506	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2507	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2508	rootnex_window_t *window;
2509	ddi_dma_cookie_t *cur_cookiep;
2510	ddi_dma_cookie_t *cookie_array;
2511	uint_t ccount;
2512
2513	/* check if cookies have not been switched */
2514	if (dma->dp_need_to_switch_cookies == B_FALSE)
2515		return (DDI_SUCCESS);
2516
2517	ASSERT(dma->dp_saved_cookies);
2518
2519	if (dma->dp_window) {
2520		window = &dma->dp_window[dma->dp_current_win];
2521		cookie_array = window->wd_first_cookie;
2522		window->wd_first_cookie = dma->dp_saved_cookies;
2523		dma->dp_saved_cookies = NULL;
2524		ccount = window->wd_cookie_cnt;
2525		cur_cookiep = (hp->dmai_cookie - cookie_array)
2526		    + window->wd_first_cookie;
2527	} else {
2528		cookie_array = dma->dp_cookies;
2529		dma->dp_cookies = dma->dp_saved_cookies;
2530		dma->dp_saved_cookies = NULL;
2531		ccount = dma->dp_sglinfo.si_sgl_size;
2532		cur_cookiep = (hp->dmai_cookie - cookie_array)
2533		    + dma->dp_cookies;
2534	}
2535
2536	kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2537
2538	hp->dmai_cookie = cur_cookiep;
2539
2540	dma->dp_need_to_switch_cookies = B_FALSE;
2541
2542	return (DDI_SUCCESS);
2543}
2544
2545#endif
2546
2547static struct as *
2548rootnex_get_as(ddi_dma_obj_t *dmao)
2549{
2550	struct as *asp;
2551
2552	switch (dmao->dmao_type) {
2553	case DMA_OTYP_VADDR:
2554	case DMA_OTYP_BUFVADDR:
2555		asp = dmao->dmao_obj.virt_obj.v_as;
2556		if (asp == NULL)
2557			asp = &kas;
2558		break;
2559	default:
2560		asp = NULL;
2561		break;
2562	}
2563	return (asp);
2564}
2565
2566/*
2567 * rootnex_verify_buffer()
2568 *   verify buffer wasn't free'd
2569 */
2570static int
2571rootnex_verify_buffer(rootnex_dma_t *dma)
2572{
2573	page_t **pplist;
2574	caddr_t vaddr;
2575	uint_t pcnt;
2576	uint_t poff;
2577	page_t *pp;
2578	char b;
2579	int i;
2580
2581	/* Figure out how many pages this buffer occupies */
2582	if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2583		poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2584	} else {
2585		vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2586		poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2587	}
2588	pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2589
2590	switch (dma->dp_dma.dmao_type) {
2591	case DMA_OTYP_PAGES:
2592		/*
2593		 * for a linked list of pp's walk through them to make sure
2594		 * they're locked and not free.
2595		 */
2596		pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2597		for (i = 0; i < pcnt; i++) {
2598			if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2599				return (DDI_FAILURE);
2600			}
2601			pp = pp->p_next;
2602		}
2603		break;
2604
2605	case DMA_OTYP_VADDR:
2606	case DMA_OTYP_BUFVADDR:
2607		pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2608		/*
2609		 * for an array of pp's walk through them to make sure they're
2610		 * not free. It's possible that they may not be locked.
2611		 */
2612		if (pplist) {
2613			for (i = 0; i < pcnt; i++) {
2614				if (PP_ISFREE(pplist[i])) {
2615					return (DDI_FAILURE);
2616				}
2617			}
2618
2619		/* For a virtual address, try to peek at each page */
2620		} else {
2621			if (rootnex_get_as(&dma->dp_dma) == &kas) {
2622				for (i = 0; i < pcnt; i++) {
2623					if (ddi_peek8(NULL, vaddr, &b) ==
2624					    DDI_FAILURE)
2625						return (DDI_FAILURE);
2626					vaddr += MMU_PAGESIZE;
2627				}
2628			}
2629		}
2630		break;
2631
2632	default:
2633		cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2634		break;
2635	}
2636
2637	return (DDI_SUCCESS);
2638}
2639
2640
2641/*
2642 * rootnex_clean_dmahdl()
2643 *    Clean the dma handle. This should be called on a handle alloc and an
2644 *    unbind handle. Set the handle state to the default settings.
2645 */
2646static void
2647rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2648{
2649	rootnex_dma_t *dma;
2650
2651
2652	dma = (rootnex_dma_t *)hp->dmai_private;
2653
2654	hp->dmai_nwin = 0;
2655	dma->dp_current_cookie = 0;
2656	dma->dp_copybuf_size = 0;
2657	dma->dp_window = NULL;
2658	dma->dp_cbaddr = NULL;
2659	dma->dp_inuse = B_FALSE;
2660	dma->dp_dvma_used = B_FALSE;
2661	dma->dp_need_to_free_cookie = B_FALSE;
2662	dma->dp_need_to_switch_cookies = B_FALSE;
2663	dma->dp_saved_cookies = NULL;
2664	dma->dp_sleep_flags = KM_PANIC;
2665	dma->dp_need_to_free_window = B_FALSE;
2666	dma->dp_partial_required = B_FALSE;
2667	dma->dp_trim_required = B_FALSE;
2668	dma->dp_sglinfo.si_copybuf_req = 0;
2669#if !defined(__amd64)
2670	dma->dp_cb_remaping = B_FALSE;
2671	dma->dp_kva = NULL;
2672#endif
2673
2674	/* FMA related initialization */
2675	hp->dmai_fault = 0;
2676	hp->dmai_fault_check = NULL;
2677	hp->dmai_fault_notify = NULL;
2678	hp->dmai_error.err_ena = 0;
2679	hp->dmai_error.err_status = DDI_FM_OK;
2680	hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2681	hp->dmai_error.err_ontrap = NULL;
2682
2683	/* Cookie tracking */
2684	hp->dmai_ncookies = 0;
2685	hp->dmai_curcookie = 0;
2686}
2687
2688
2689/*
2690 * rootnex_valid_alloc_parms()
2691 *    Called in ddi_dma_alloc_handle path to validate its parameters.
2692 */
2693static int
2694rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2695{
2696	if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2697	    (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2698	    (attr->dma_attr_granular > MMU_PAGESIZE) ||
2699	    (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2700		return (DDI_DMA_BADATTR);
2701	}
2702
2703	if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2704		return (DDI_DMA_BADATTR);
2705	}
2706
2707	if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2708	    MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2709	    attr->dma_attr_sgllen == 0) {
2710		return (DDI_DMA_BADATTR);
2711	}
2712
2713	/* We should be able to DMA into every byte offset in a page */
2714	if (maxsegmentsize < MMU_PAGESIZE) {
2715		return (DDI_DMA_BADATTR);
2716	}
2717
2718	/* if we're bouncing on seg, seg must be <= addr_hi */
2719	if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2720	    (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2721		return (DDI_DMA_BADATTR);
2722	}
2723	return (DDI_SUCCESS);
2724}
2725
2726/*
2727 * rootnex_valid_bind_parms()
2728 *    Called in ddi_dma_*_bind_handle path to validate its parameters.
2729 */
2730/* ARGSUSED */
2731static int
2732rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2733{
2734#if !defined(__amd64)
2735	/*
2736	 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2737	 * we can track the offset for the obsoleted interfaces.
2738	 */
2739	if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2740		return (DDI_DMA_TOOBIG);
2741	}
2742#endif
2743
2744	return (DDI_SUCCESS);
2745}
2746
2747
2748/*
2749 * rootnex_need_bounce_seg()
2750 *    check to see if the buffer lives on both side of the seg.
2751 */
2752static boolean_t
2753rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2754{
2755	ddi_dma_atyp_t buftype;
2756	rootnex_addr_t raddr;
2757	boolean_t lower_addr;
2758	boolean_t upper_addr;
2759	uint64_t offset;
2760	page_t **pplist;
2761	uint64_t paddr;
2762	uint32_t psize;
2763	uint32_t size;
2764	caddr_t vaddr;
2765	uint_t pcnt;
2766	page_t *pp;
2767
2768	pp = NULL;
2769	/* shortcuts */
2770	pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2771	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2772	buftype = dmar_object->dmao_type;
2773	size = dmar_object->dmao_size;
2774
2775	lower_addr = B_FALSE;
2776	upper_addr = B_FALSE;
2777	pcnt = 0;
2778
2779	/*
2780	 * Process the first page to handle the initial offset of the buffer.
2781	 * We'll use the base address we get later when we loop through all
2782	 * the pages.
2783	 */
2784	if (buftype == DMA_OTYP_PAGES) {
2785		pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2786		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2787		    MMU_PAGEOFFSET;
2788		paddr = pfn_to_pa(pp->p_pagenum) + offset;
2789		psize = MIN(size, (MMU_PAGESIZE - offset));
2790		pp = pp->p_next;
2791		sglinfo->si_asp = NULL;
2792	} else if (pplist != NULL) {
2793		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2794		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2795		if (sglinfo->si_asp == NULL) {
2796			sglinfo->si_asp = &kas;
2797		}
2798		paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2799		paddr += offset;
2800		psize = MIN(size, (MMU_PAGESIZE - offset));
2801		pcnt++;
2802	} else {
2803		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2804		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2805		if (sglinfo->si_asp == NULL) {
2806			sglinfo->si_asp = &kas;
2807		}
2808		paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2809		paddr += offset;
2810		psize = MIN(size, (MMU_PAGESIZE - offset));
2811		vaddr += psize;
2812	}
2813
2814	raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2815
2816	if ((raddr + psize) > sglinfo->si_segmask) {
2817		upper_addr = B_TRUE;
2818	} else {
2819		lower_addr = B_TRUE;
2820	}
2821	size -= psize;
2822
2823	/*
2824	 * Walk through the rest of the pages in the buffer. Track to see
2825	 * if we have pages on both sides of the segment boundary.
2826	 */
2827	while (size > 0) {
2828		/* partial or full page */
2829		psize = MIN(size, MMU_PAGESIZE);
2830
2831		if (buftype == DMA_OTYP_PAGES) {
2832			/* get the paddr from the page_t */
2833			ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2834			paddr = pfn_to_pa(pp->p_pagenum);
2835			pp = pp->p_next;
2836		} else if (pplist != NULL) {
2837			/* index into the array of page_t's to get the paddr */
2838			ASSERT(!PP_ISFREE(pplist[pcnt]));
2839			paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2840			pcnt++;
2841		} else {
2842			/* call into the VM to get the paddr */
2843			paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2844			    vaddr));
2845			vaddr += psize;
2846		}
2847
2848		raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2849
2850		if ((raddr + psize) > sglinfo->si_segmask) {
2851			upper_addr = B_TRUE;
2852		} else {
2853			lower_addr = B_TRUE;
2854		}
2855		/*
2856		 * if the buffer lives both above and below the segment
2857		 * boundary, or the current page is the page immediately
2858		 * after the segment, we will use a copy/bounce buffer for
2859		 * all pages > seg.
2860		 */
2861		if ((lower_addr && upper_addr) ||
2862		    (raddr == (sglinfo->si_segmask + 1))) {
2863			return (B_TRUE);
2864		}
2865
2866		size -= psize;
2867	}
2868
2869	return (B_FALSE);
2870}
2871
2872/*
2873 * rootnex_get_sgl()
2874 *    Called in bind fastpath to get the sgl. Most of this will be replaced
2875 *    with a call to the vm layer when vm2.0 comes around...
2876 */
2877static void
2878rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2879    rootnex_sglinfo_t *sglinfo)
2880{
2881	ddi_dma_atyp_t buftype;
2882	rootnex_addr_t raddr;
2883	uint64_t last_page;
2884	uint64_t offset;
2885	uint64_t addrhi;
2886	uint64_t addrlo;
2887	uint64_t maxseg;
2888	page_t **pplist;
2889	uint64_t paddr;
2890	uint32_t psize;
2891	uint32_t size;
2892	caddr_t vaddr;
2893	uint_t pcnt;
2894	page_t *pp;
2895	uint_t cnt;
2896
2897	pp = NULL;
2898	/* shortcuts */
2899	pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2900	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2901	maxseg = sglinfo->si_max_cookie_size;
2902	buftype = dmar_object->dmao_type;
2903	addrhi = sglinfo->si_max_addr;
2904	addrlo = sglinfo->si_min_addr;
2905	size = dmar_object->dmao_size;
2906
2907	pcnt = 0;
2908	cnt = 0;
2909
2910
2911	/*
2912	 * check to see if we need to use the copy buffer for pages over
2913	 * the segment attr.
2914	 */
2915	sglinfo->si_bounce_on_seg = B_FALSE;
2916	if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2917		sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2918		    dmar_object, sglinfo);
2919	}
2920
2921	/*
2922	 * if we were passed down a linked list of pages, i.e. pointer to
2923	 * page_t, use this to get our physical address and buf offset.
2924	 */
2925	if (buftype == DMA_OTYP_PAGES) {
2926		pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2927		ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2928		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2929		    MMU_PAGEOFFSET;
2930		paddr = pfn_to_pa(pp->p_pagenum) + offset;
2931		psize = MIN(size, (MMU_PAGESIZE - offset));
2932		pp = pp->p_next;
2933		sglinfo->si_asp = NULL;
2934
2935	/*
2936	 * We weren't passed down a linked list of pages, but if we were passed
2937	 * down an array of pages, use this to get our physical address and buf
2938	 * offset.
2939	 */
2940	} else if (pplist != NULL) {
2941		ASSERT((buftype == DMA_OTYP_VADDR) ||
2942		    (buftype == DMA_OTYP_BUFVADDR));
2943
2944		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2945		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2946		if (sglinfo->si_asp == NULL) {
2947			sglinfo->si_asp = &kas;
2948		}
2949
2950		ASSERT(!PP_ISFREE(pplist[pcnt]));
2951		paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2952		paddr += offset;
2953		psize = MIN(size, (MMU_PAGESIZE - offset));
2954		pcnt++;
2955
2956	/*
2957	 * All we have is a virtual address, we'll need to call into the VM
2958	 * to get the physical address.
2959	 */
2960	} else {
2961		ASSERT((buftype == DMA_OTYP_VADDR) ||
2962		    (buftype == DMA_OTYP_BUFVADDR));
2963
2964		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2965		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2966		if (sglinfo->si_asp == NULL) {
2967			sglinfo->si_asp = &kas;
2968		}
2969
2970		paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2971		paddr += offset;
2972		psize = MIN(size, (MMU_PAGESIZE - offset));
2973		vaddr += psize;
2974	}
2975
2976	raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2977
2978	/*
2979	 * Setup the first cookie with the physical address of the page and the
2980	 * size of the page (which takes into account the initial offset into
2981	 * the page.
2982	 */
2983	sgl[cnt].dmac_laddress = raddr;
2984	sgl[cnt].dmac_size = psize;
2985	sgl[cnt].dmac_type = 0;
2986
2987	/*
2988	 * Save away the buffer offset into the page. We'll need this later in
2989	 * the copy buffer code to help figure out the page index within the
2990	 * buffer and the offset into the current page.
2991	 */
2992	sglinfo->si_buf_offset = offset;
2993
2994	/*
2995	 * If we are using the copy buffer for anything over the segment
2996	 * boundary, and this page is over the segment boundary.
2997	 *   OR
2998	 * if the DMA engine can't reach the physical address.
2999	 */
3000	if (((sglinfo->si_bounce_on_seg) &&
3001	    ((raddr + psize) > sglinfo->si_segmask)) ||
3002	    ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3003		/*
3004		 * Increase how much copy buffer we use. We always increase by
3005		 * pagesize so we don't have to worry about converting offsets.
3006		 * Set a flag in the cookies dmac_type to indicate that it uses
3007		 * the copy buffer. If this isn't the last cookie, go to the
3008		 * next cookie (since we separate each page which uses the copy
3009		 * buffer in case the copy buffer is not physically contiguous.
3010		 */
3011		sglinfo->si_copybuf_req += MMU_PAGESIZE;
3012		sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3013		if ((cnt + 1) < sglinfo->si_max_pages) {
3014			cnt++;
3015			sgl[cnt].dmac_laddress = 0;
3016			sgl[cnt].dmac_size = 0;
3017			sgl[cnt].dmac_type = 0;
3018		}
3019	}
3020
3021	/*
3022	 * save this page's physical address so we can figure out if the next
3023	 * page is physically contiguous. Keep decrementing size until we are
3024	 * done with the buffer.
3025	 */
3026	last_page = raddr & MMU_PAGEMASK;
3027	size -= psize;
3028
3029	while (size > 0) {
3030		/* Get the size for this page (i.e. partial or full page) */
3031		psize = MIN(size, MMU_PAGESIZE);
3032
3033		if (buftype == DMA_OTYP_PAGES) {
3034			/* get the paddr from the page_t */
3035			ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
3036			paddr = pfn_to_pa(pp->p_pagenum);
3037			pp = pp->p_next;
3038		} else if (pplist != NULL) {
3039			/* index into the array of page_t's to get the paddr */
3040			ASSERT(!PP_ISFREE(pplist[pcnt]));
3041			paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
3042			pcnt++;
3043		} else {
3044			/* call into the VM to get the paddr */
3045			paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
3046			    vaddr));
3047			vaddr += psize;
3048		}
3049
3050		raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3051
3052		/*
3053		 * If we are using the copy buffer for anything over the
3054		 * segment boundary, and this page is over the segment
3055		 * boundary.
3056		 *   OR
3057		 * if the DMA engine can't reach the physical address.
3058		 */
3059		if (((sglinfo->si_bounce_on_seg) &&
3060		    ((raddr + psize) > sglinfo->si_segmask)) ||
3061		    ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3062
3063			sglinfo->si_copybuf_req += MMU_PAGESIZE;
3064
3065			/*
3066			 * if there is something in the current cookie, go to
3067			 * the next one. We only want one page in a cookie which
3068			 * uses the copybuf since the copybuf doesn't have to
3069			 * be physically contiguous.
3070			 */
3071			if (sgl[cnt].dmac_size != 0) {
3072				cnt++;
3073			}
3074			sgl[cnt].dmac_laddress = raddr;
3075			sgl[cnt].dmac_size = psize;
3076#if defined(__amd64)
3077			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3078#else
3079			/*
3080			 * save the buf offset for 32-bit kernel. used in the
3081			 * obsoleted interfaces.
3082			 */
3083			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
3084			    (dmar_object->dmao_size - size);
3085#endif
3086			/* if this isn't the last cookie, go to the next one */
3087			if ((cnt + 1) < sglinfo->si_max_pages) {
3088				cnt++;
3089				sgl[cnt].dmac_laddress = 0;
3090				sgl[cnt].dmac_size = 0;
3091				sgl[cnt].dmac_type = 0;
3092			}
3093
3094		/*
3095		 * this page didn't need the copy buffer, if it's not physically
3096		 * contiguous, or it would put us over a segment boundary, or it
3097		 * puts us over the max cookie size, or the current sgl doesn't
3098		 * have anything in it.
3099		 */
3100		} else if (((last_page + MMU_PAGESIZE) != raddr) ||
3101		    !(raddr & sglinfo->si_segmask) ||
3102		    ((sgl[cnt].dmac_size + psize) > maxseg) ||
3103		    (sgl[cnt].dmac_size == 0)) {
3104			/*
3105			 * if we're not already in a new cookie, go to the next
3106			 * cookie.
3107			 */
3108			if (sgl[cnt].dmac_size != 0) {
3109				cnt++;
3110			}
3111
3112			/* save the cookie information */
3113			sgl[cnt].dmac_laddress = raddr;
3114			sgl[cnt].dmac_size = psize;
3115#if defined(__amd64)
3116			sgl[cnt].dmac_type = 0;
3117#else
3118			/*
3119			 * save the buf offset for 32-bit kernel. used in the
3120			 * obsoleted interfaces.
3121			 */
3122			sgl[cnt].dmac_type = dmar_object->dmao_size - size;
3123#endif
3124
3125		/*
3126		 * this page didn't need the copy buffer, it is physically
3127		 * contiguous with the last page, and it's <= the max cookie
3128		 * size.
3129		 */
3130		} else {
3131			sgl[cnt].dmac_size += psize;
3132
3133			/*
3134			 * If this cookie is used up, and more cookies
3135			 * are available, then move onto the next one.
3136			 */
3137			if ((sgl[cnt].dmac_size == maxseg) &&
3138			    ((cnt + 1) < sglinfo->si_max_pages)) {
3139				cnt++;
3140				sgl[cnt].dmac_laddress = 0;
3141				sgl[cnt].dmac_size = 0;
3142				sgl[cnt].dmac_type = 0;
3143			}
3144		}
3145
3146		/*
3147		 * save this page's physical address so we can figure out if the
3148		 * next page is physically contiguous. Keep decrementing size
3149		 * until we are done with the buffer.
3150		 */
3151		last_page = raddr;
3152		size -= psize;
3153	}
3154
3155	/* we're done, save away how many cookies the sgl has */
3156	if (sgl[cnt].dmac_size == 0) {
3157		ASSERT(cnt < sglinfo->si_max_pages);
3158		sglinfo->si_sgl_size = cnt;
3159	} else {
3160		sglinfo->si_sgl_size = cnt + 1;
3161	}
3162}
3163
3164static void
3165rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3166    rootnex_sglinfo_t *sglinfo)
3167{
3168	uint64_t offset;
3169	uint64_t maxseg;
3170	uint64_t dvaddr;
3171	struct dvmaseg *dvs;
3172	uint64_t paddr;
3173	uint32_t psize, ssize;
3174	uint32_t size;
3175	uint_t cnt;
3176	int physcontig;
3177
3178	ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3179
3180	/* shortcuts */
3181	maxseg = sglinfo->si_max_cookie_size;
3182	size = dmar_object->dmao_size;
3183
3184	cnt = 0;
3185	sglinfo->si_bounce_on_seg = B_FALSE;
3186
3187	dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3188	offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3189	ssize = dvs->dvs_len;
3190	paddr = dvs->dvs_start;
3191	paddr += offset;
3192	psize = MIN(ssize, (maxseg - offset));
3193	dvaddr = paddr + psize;
3194	ssize -= psize;
3195
3196	sgl[cnt].dmac_laddress = paddr;
3197	sgl[cnt].dmac_size = psize;
3198	sgl[cnt].dmac_type = 0;
3199
3200	size -= psize;
3201	while (size > 0) {
3202		if (ssize == 0) {
3203			dvs++;
3204			ssize = dvs->dvs_len;
3205			dvaddr = dvs->dvs_start;
3206			physcontig = 0;
3207		} else
3208			physcontig = 1;
3209
3210		paddr = dvaddr;
3211		psize = MIN(ssize, maxseg);
3212		dvaddr += psize;
3213		ssize -= psize;
3214
3215		if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3216		    ((sgl[cnt].dmac_size + psize) > maxseg) ||
3217		    (sgl[cnt].dmac_size == 0)) {
3218			/*
3219			 * if we're not already in a new cookie, go to the next
3220			 * cookie.
3221			 */
3222			if (sgl[cnt].dmac_size != 0) {
3223				cnt++;
3224			}
3225
3226			/* save the cookie information */
3227			sgl[cnt].dmac_laddress = paddr;
3228			sgl[cnt].dmac_size = psize;
3229			sgl[cnt].dmac_type = 0;
3230		} else {
3231			sgl[cnt].dmac_size += psize;
3232
3233			/*
3234			 * If this cookie is used up, and more cookies
3235			 * are available, then move onto the next one.
3236			 */
3237			if ((sgl[cnt].dmac_size == maxseg) &&
3238			    ((cnt + 1) < sglinfo->si_max_pages)) {
3239				cnt++;
3240				sgl[cnt].dmac_laddress = 0;
3241				sgl[cnt].dmac_size = 0;
3242				sgl[cnt].dmac_type = 0;
3243			}
3244		}
3245		size -= psize;
3246	}
3247
3248	/* we're done, save away how many cookies the sgl has */
3249	if (sgl[cnt].dmac_size == 0) {
3250		sglinfo->si_sgl_size = cnt;
3251	} else {
3252		sglinfo->si_sgl_size = cnt + 1;
3253	}
3254}
3255
3256/*
3257 * rootnex_bind_slowpath()
3258 *    Call in the bind path if the calling driver can't use the sgl without
3259 *    modifying it. We either need to use the copy buffer and/or we will end up
3260 *    with a partial bind.
3261 */
3262static int
3263rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3264    rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3265{
3266	rootnex_sglinfo_t *sinfo;
3267	rootnex_window_t *window;
3268	ddi_dma_cookie_t *cookie;
3269	size_t copybuf_used;
3270	size_t dmac_size;
3271	boolean_t partial;
3272	off_t cur_offset;
3273	page_t *cur_pp;
3274	major_t mnum;
3275	int e;
3276	int i;
3277
3278
3279	sinfo = &dma->dp_sglinfo;
3280	copybuf_used = 0;
3281	partial = B_FALSE;
3282
3283	/*
3284	 * If we're using the copybuf, set the copybuf state in dma struct.
3285	 * Needs to be first since it sets the copy buffer size.
3286	 */
3287	if (sinfo->si_copybuf_req != 0) {
3288		e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3289		if (e != DDI_SUCCESS) {
3290			return (e);
3291		}
3292	} else {
3293		dma->dp_copybuf_size = 0;
3294	}
3295
3296	/*
3297	 * Figure out if we need to do a partial mapping. If so, figure out
3298	 * if we need to trim the buffers when we munge the sgl.
3299	 */
3300	if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3301	    (dmao->dmao_size > dma->dp_maxxfer) ||
3302	    ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3303		dma->dp_partial_required = B_TRUE;
3304		if (attr->dma_attr_granular != 1) {
3305			dma->dp_trim_required = B_TRUE;
3306		}
3307	} else {
3308		dma->dp_partial_required = B_FALSE;
3309		dma->dp_trim_required = B_FALSE;
3310	}
3311
3312	/* If we need to do a partial bind, make sure the driver supports it */
3313	if (dma->dp_partial_required &&
3314	    !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3315
3316		mnum = ddi_driver_major(dma->dp_dip);
3317		/*
3318		 * patchable which allows us to print one warning per major
3319		 * number.
3320		 */
3321		if ((rootnex_bind_warn) &&
3322		    ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3323			rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3324			cmn_err(CE_WARN, "!%s: coding error detected, the "
3325			    "driver is using ddi_dma_attr(9S) incorrectly. "
3326			    "There is a small risk of data corruption in "
3327			    "particular with large I/Os. The driver should be "
3328			    "replaced with a corrected version for proper "
3329			    "system operation. To disable this warning, add "
3330			    "'set rootnex:rootnex_bind_warn=0' to "
3331			    "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3332		}
3333		return (DDI_DMA_TOOBIG);
3334	}
3335
3336	/*
3337	 * we might need multiple windows, setup state to handle them. In this
3338	 * code path, we will have at least one window.
3339	 */
3340	e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3341	if (e != DDI_SUCCESS) {
3342		rootnex_teardown_copybuf(dma);
3343		return (e);
3344	}
3345
3346	window = &dma->dp_window[0];
3347	cookie = &dma->dp_cookies[0];
3348	cur_offset = 0;
3349	rootnex_init_win(hp, dma, window, cookie, cur_offset);
3350	if (dmao->dmao_type == DMA_OTYP_PAGES) {
3351		cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3352	}
3353
3354	/* loop though all the cookies we got back from get_sgl() */
3355	for (i = 0; i < sinfo->si_sgl_size; i++) {
3356		/*
3357		 * If we're using the copy buffer, check this cookie and setup
3358		 * its associated copy buffer state. If this cookie uses the
3359		 * copy buffer, make sure we sync this window during dma_sync.
3360		 */
3361		if (dma->dp_copybuf_size > 0) {
3362			rootnex_setup_cookie(dmao, dma, cookie,
3363			    cur_offset, &copybuf_used, &cur_pp);
3364			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3365				window->wd_dosync = B_TRUE;
3366			}
3367		}
3368
3369		/*
3370		 * save away the cookie size, since it could be modified in
3371		 * the windowing code.
3372		 */
3373		dmac_size = cookie->dmac_size;
3374
3375		/* if we went over max copybuf size */
3376		if (dma->dp_copybuf_size &&
3377		    (copybuf_used > dma->dp_copybuf_size)) {
3378			partial = B_TRUE;
3379			e = rootnex_copybuf_window_boundary(hp, dma, &window,
3380			    cookie, cur_offset, &copybuf_used);
3381			if (e != DDI_SUCCESS) {
3382				rootnex_teardown_copybuf(dma);
3383				rootnex_teardown_windows(dma);
3384				return (e);
3385			}
3386
3387			/*
3388			 * if the coookie uses the copy buffer, make sure the
3389			 * new window we just moved to is set to sync.
3390			 */
3391			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3392				window->wd_dosync = B_TRUE;
3393			}
3394			ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3395			    dma->dp_dip);
3396
3397		/* if the cookie cnt == max sgllen, move to the next window */
3398		} else if (window->wd_cookie_cnt >=
3399		    (unsigned)attr->dma_attr_sgllen) {
3400			partial = B_TRUE;
3401			ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3402			e = rootnex_sgllen_window_boundary(hp, dma, &window,
3403			    cookie, attr, cur_offset);
3404			if (e != DDI_SUCCESS) {
3405				rootnex_teardown_copybuf(dma);
3406				rootnex_teardown_windows(dma);
3407				return (e);
3408			}
3409
3410			/*
3411			 * if the coookie uses the copy buffer, make sure the
3412			 * new window we just moved to is set to sync.
3413			 */
3414			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3415				window->wd_dosync = B_TRUE;
3416			}
3417			ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3418			    dma->dp_dip);
3419
3420		/* else if we will be over maxxfer */
3421		} else if ((window->wd_size + dmac_size) >
3422		    dma->dp_maxxfer) {
3423			partial = B_TRUE;
3424			e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3425			    cookie);
3426			if (e != DDI_SUCCESS) {
3427				rootnex_teardown_copybuf(dma);
3428				rootnex_teardown_windows(dma);
3429				return (e);
3430			}
3431
3432			/*
3433			 * if the coookie uses the copy buffer, make sure the
3434			 * new window we just moved to is set to sync.
3435			 */
3436			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3437				window->wd_dosync = B_TRUE;
3438			}
3439			ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3440			    dma->dp_dip);
3441
3442		/* else this cookie fits in the current window */
3443		} else {
3444			window->wd_cookie_cnt++;
3445			window->wd_size += dmac_size;
3446		}
3447
3448		/* track our offset into the buffer, go to the next cookie */
3449		ASSERT(dmac_size <= dmao->dmao_size);
3450		ASSERT(cookie->dmac_size <= dmac_size);
3451		cur_offset += dmac_size;
3452		cookie++;
3453	}
3454
3455	/* if we ended up with a zero sized window in the end, clean it up */
3456	if (window->wd_size == 0) {
3457		hp->dmai_nwin--;
3458		window--;
3459	}
3460
3461	ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3462
3463	if (!partial) {
3464		return (DDI_DMA_MAPPED);
3465	}
3466
3467	ASSERT(dma->dp_partial_required);
3468	return (DDI_DMA_PARTIAL_MAP);
3469}
3470
3471/*
3472 * rootnex_setup_copybuf()
3473 *    Called in bind slowpath. Figures out if we're going to use the copy
3474 *    buffer, and if we do, sets up the basic state to handle it.
3475 */
3476static int
3477rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3478    rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3479{
3480	rootnex_sglinfo_t *sinfo;
3481	ddi_dma_attr_t lattr;
3482	size_t max_copybuf;
3483	int cansleep;
3484	int e;
3485#if !defined(__amd64)
3486	int vmflag;
3487#endif
3488
3489	ASSERT(!dma->dp_dvma_used);
3490
3491	sinfo = &dma->dp_sglinfo;
3492
3493	/* read this first so it's consistent through the routine  */
3494	max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3495
3496	/* We need to call into the rootnex on ddi_dma_sync() */
3497	hp->dmai_rflags &= ~DMP_NOSYNC;
3498
3499	/* make sure the copybuf size <= the max size */
3500	dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3501	ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3502
3503#if !defined(__amd64)
3504	/*
3505	 * if we don't have kva space to copy to/from, allocate the KVA space
3506	 * now. We only do this for the 32-bit kernel. We use seg kpm space for
3507	 * the 64-bit kernel.
3508	 */
3509	if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
3510	    (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
3511
3512		/* convert the sleep flags */
3513		if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3514			vmflag = VM_SLEEP;
3515		} else {
3516			vmflag = VM_NOSLEEP;
3517		}
3518
3519		/* allocate Kernel VA space that we can bcopy to/from */
3520		dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
3521		    vmflag);
3522		if (dma->dp_kva == NULL) {
3523			return (DDI_DMA_NORESOURCES);
3524		}
3525	}
3526#endif
3527
3528	/* convert the sleep flags */
3529	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3530		cansleep = 1;
3531	} else {
3532		cansleep = 0;
3533	}
3534
3535	/*
3536	 * Allocate the actual copy buffer. This needs to fit within the DMA
3537	 * engine limits, so we can't use kmem_alloc... We don't need
3538	 * contiguous memory (sgllen) since we will be forcing windows on
3539	 * sgllen anyway.
3540	 */
3541	lattr = *attr;
3542	lattr.dma_attr_align = MMU_PAGESIZE;
3543	lattr.dma_attr_sgllen = -1;	/* no limit */
3544	/*
3545	 * if we're using the copy buffer because of seg, use that for our
3546	 * upper address limit.
3547	 */
3548	if (sinfo->si_bounce_on_seg) {
3549		lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3550	}
3551	e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3552	    0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3553	if (e != DDI_SUCCESS) {
3554#if !defined(__amd64)
3555		if (dma->dp_kva != NULL) {
3556			vmem_free(heap_arena, dma->dp_kva,
3557			    dma->dp_copybuf_size);
3558		}
3559#endif
3560		return (DDI_DMA_NORESOURCES);
3561	}
3562
3563	ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3564	    size_t, dma->dp_copybuf_size);
3565
3566	return (DDI_SUCCESS);
3567}
3568
3569
3570/*
3571 * rootnex_setup_windows()
3572 *    Called in bind slowpath to setup the window state. We always have windows
3573 *    in the slowpath. Even if the window count = 1.
3574 */
3575static int
3576rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3577    ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3578{
3579	rootnex_window_t *windowp;
3580	rootnex_sglinfo_t *sinfo;
3581	size_t copy_state_size;
3582	size_t win_state_size;
3583	size_t state_available;
3584	size_t space_needed;
3585	uint_t copybuf_win;
3586	uint_t maxxfer_win;
3587	size_t space_used;
3588	uint_t sglwin;
3589
3590
3591	sinfo = &dma->dp_sglinfo;
3592
3593	dma->dp_current_win = 0;
3594	hp->dmai_nwin = 0;
3595
3596	/* If we don't need to do a partial, we only have one window */
3597	if (!dma->dp_partial_required) {
3598		dma->dp_max_win = 1;
3599
3600	/*
3601	 * we need multiple windows, need to figure out the worse case number
3602	 * of windows.
3603	 */
3604	} else {
3605		/*
3606		 * if we need windows because we need more copy buffer that
3607		 * we allow, the worse case number of windows we could need
3608		 * here would be (copybuf space required / copybuf space that
3609		 * we have) plus one for remainder, and plus 2 to handle the
3610		 * extra pages on the trim for the first and last pages of the
3611		 * buffer (a page is the minimum window size so under the right
3612		 * attr settings, you could have a window for each page).
3613		 * The last page will only be hit here if the size is not a
3614		 * multiple of the granularity (which theoretically shouldn't
3615		 * be the case but never has been enforced, so we could have
3616		 * broken things without it).
3617		 */
3618		if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3619			ASSERT(dma->dp_copybuf_size > 0);
3620			copybuf_win = (sinfo->si_copybuf_req /
3621			    dma->dp_copybuf_size) + 1 + 2;
3622		} else {
3623			copybuf_win = 0;
3624		}
3625
3626		/*
3627		 * if we need windows because we have more cookies than the H/W
3628		 * can handle, the number of windows we would need here would
3629		 * be (cookie count / cookies count H/W supports minus 1[for
3630		 * trim]) plus one for remainder.
3631		 */
3632		if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3633			sglwin = (sinfo->si_sgl_size /
3634			    (attr->dma_attr_sgllen - 1)) + 1;
3635		} else {
3636			sglwin = 0;
3637		}
3638
3639		/*
3640		 * if we need windows because we're binding more memory than the
3641		 * H/W can transfer at once, the number of windows we would need
3642		 * here would be (xfer count / max xfer H/W supports) plus one
3643		 * for remainder, and plus 2 to handle the extra pages on the
3644		 * trim (see above comment about trim)
3645		 */
3646		if (dmao->dmao_size > dma->dp_maxxfer) {
3647			maxxfer_win = (dmao->dmao_size /
3648			    dma->dp_maxxfer) + 1 + 2;
3649		} else {
3650			maxxfer_win = 0;
3651		}
3652		dma->dp_max_win =  copybuf_win + sglwin + maxxfer_win;
3653		ASSERT(dma->dp_max_win > 0);
3654	}
3655	win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3656
3657	/*
3658	 * Get space for window and potential copy buffer state. Before we
3659	 * go and allocate memory, see if we can get away with using what's
3660	 * left in the pre-allocted state or the dynamically allocated sgl.
3661	 */
3662	space_used = (uintptr_t)(sinfo->si_sgl_size *
3663	    sizeof (ddi_dma_cookie_t));
3664
3665	/* if we dynamically allocated space for the cookies */
3666	if (dma->dp_need_to_free_cookie) {
3667		/* if we have more space in the pre-allocted buffer, use it */
3668		ASSERT(space_used <= dma->dp_cookie_size);
3669		if ((dma->dp_cookie_size - space_used) <=
3670		    rootnex_state->r_prealloc_size) {
3671			state_available = rootnex_state->r_prealloc_size;
3672			windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3673
3674		/*
3675		 * else, we have more free space in the dynamically allocated
3676		 * buffer, i.e. the buffer wasn't worse case fragmented so we
3677		 * didn't need a lot of cookies.
3678		 */
3679		} else {
3680			state_available = dma->dp_cookie_size - space_used;
3681			windowp = (rootnex_window_t *)
3682			    &dma->dp_cookies[sinfo->si_sgl_size];
3683		}
3684
3685	/* we used the pre-alloced buffer */
3686	} else {
3687		ASSERT(space_used <= rootnex_state->r_prealloc_size);
3688		state_available = rootnex_state->r_prealloc_size - space_used;
3689		windowp = (rootnex_window_t *)
3690		    &dma->dp_cookies[sinfo->si_sgl_size];
3691	}
3692
3693	/*
3694	 * figure out how much state we need to track the copy buffer. Add an
3695	 * addition 8 bytes for pointer alignemnt later.
3696	 */
3697	if (dma->dp_copybuf_size > 0) {
3698		copy_state_size = sinfo->si_max_pages *
3699		    sizeof (rootnex_pgmap_t);
3700	} else {
3701		copy_state_size = 0;
3702	}
3703	/* add an additional 8 bytes for pointer alignment */
3704	space_needed = win_state_size + copy_state_size + 0x8;
3705
3706	/* if we have enough space already, use it */
3707	if (state_available >= space_needed) {
3708		dma->dp_window = windowp;
3709		dma->dp_need_to_free_window = B_FALSE;
3710
3711	/* not enough space, need to allocate more. */
3712	} else {
3713		dma->dp_window = kmem_alloc(space_needed, kmflag);
3714		if (dma->dp_window == NULL) {
3715			return (DDI_DMA_NORESOURCES);
3716		}
3717		dma->dp_need_to_free_window = B_TRUE;
3718		dma->dp_window_size = space_needed;
3719		ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3720		    dma->dp_dip, size_t, space_needed);
3721	}
3722
3723	/*
3724	 * we allocate copy buffer state and window state at the same time.
3725	 * setup our copy buffer state pointers. Make sure it's aligned.
3726	 */
3727	if (dma->dp_copybuf_size > 0) {
3728		dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3729		    &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3730
3731#if !defined(__amd64)
3732		/*
3733		 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3734		 * false/NULL. Should be quicker to bzero vs loop and set.
3735		 */
3736		bzero(dma->dp_pgmap, copy_state_size);
3737#endif
3738	} else {
3739		dma->dp_pgmap = NULL;
3740	}
3741
3742	return (DDI_SUCCESS);
3743}
3744
3745
3746/*
3747 * rootnex_teardown_copybuf()
3748 *    cleans up after rootnex_setup_copybuf()
3749 */
3750static void
3751rootnex_teardown_copybuf(rootnex_dma_t *dma)
3752{
3753#if !defined(__amd64)
3754	int i;
3755
3756	/*
3757	 * if we allocated kernel heap VMEM space, go through all the pages and
3758	 * map out any of the ones that we're mapped into the kernel heap VMEM
3759	 * arena. Then free the VMEM space.
3760	 */
3761	if (dma->dp_kva != NULL) {
3762		for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3763			if (dma->dp_pgmap[i].pm_mapped) {
3764				hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3765				    MMU_PAGESIZE, HAT_UNLOAD);
3766				dma->dp_pgmap[i].pm_mapped = B_FALSE;
3767			}
3768		}
3769
3770		vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3771	}
3772
3773#endif
3774
3775	/* if we allocated a copy buffer, free it */
3776	if (dma->dp_cbaddr != NULL) {
3777		i_ddi_mem_free(dma->dp_cbaddr, NULL);
3778	}
3779}
3780
3781
3782/*
3783 * rootnex_teardown_windows()
3784 *    cleans up after rootnex_setup_windows()
3785 */
3786static void
3787rootnex_teardown_windows(rootnex_dma_t *dma)
3788{
3789	/*
3790	 * if we had to allocate window state on the last bind (because we
3791	 * didn't have enough pre-allocated space in the handle), free it.
3792	 */
3793	if (dma->dp_need_to_free_window) {
3794		kmem_free(dma->dp_window, dma->dp_window_size);
3795	}
3796}
3797
3798
3799/*
3800 * rootnex_init_win()
3801 *    Called in bind slow path during creation of a new window. Initializes
3802 *    window state to default values.
3803 */
3804/*ARGSUSED*/
3805static void
3806rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3807    rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3808{
3809	hp->dmai_nwin++;
3810	window->wd_dosync = B_FALSE;
3811	window->wd_offset = cur_offset;
3812	window->wd_size = 0;
3813	window->wd_first_cookie = cookie;
3814	window->wd_cookie_cnt = 0;
3815	window->wd_trim.tr_trim_first = B_FALSE;
3816	window->wd_trim.tr_trim_last = B_FALSE;
3817	window->wd_trim.tr_first_copybuf_win = B_FALSE;
3818	window->wd_trim.tr_last_copybuf_win = B_FALSE;
3819#if !defined(__amd64)
3820	window->wd_remap_copybuf = dma->dp_cb_remaping;
3821#endif
3822}
3823
3824
3825/*
3826 * rootnex_setup_cookie()
3827 *    Called in the bind slow path when the sgl uses the copy buffer. If any of
3828 *    the sgl uses the copy buffer, we need to go through each cookie, figure
3829 *    out if it uses the copy buffer, and if it does, save away everything we'll
3830 *    need during sync.
3831 */
3832static void
3833rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3834    ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3835    page_t **cur_pp)
3836{
3837	boolean_t copybuf_sz_power_2;
3838	rootnex_sglinfo_t *sinfo;
3839	paddr_t paddr;
3840	uint_t pidx;
3841	uint_t pcnt;
3842	off_t poff;
3843#if defined(__amd64)
3844	pfn_t pfn;
3845#else
3846	page_t **pplist;
3847#endif
3848
3849	ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3850
3851	sinfo = &dma->dp_sglinfo;
3852
3853	/*
3854	 * Calculate the page index relative to the start of the buffer. The
3855	 * index to the current page for our buffer is the offset into the
3856	 * first page of the buffer plus our current offset into the buffer
3857	 * itself, shifted of course...
3858	 */
3859	pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3860	ASSERT(pidx < sinfo->si_max_pages);
3861
3862	/* if this cookie uses the copy buffer */
3863	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3864		/*
3865		 * NOTE: we know that since this cookie uses the copy buffer, it
3866		 * is <= MMU_PAGESIZE.
3867		 */
3868
3869		/*
3870		 * get the offset into the page. For the 64-bit kernel, get the
3871		 * pfn which we'll use with seg kpm.
3872		 */
3873		poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3874#if defined(__amd64)
3875		/* mfn_to_pfn() is a NOP on i86pc */
3876		pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3877#endif /* __amd64 */
3878
3879		/* figure out if the copybuf size is a power of 2 */
3880		if (!ISP2(dma->dp_copybuf_size)) {
3881			copybuf_sz_power_2 = B_FALSE;
3882		} else {
3883			copybuf_sz_power_2 = B_TRUE;
3884		}
3885
3886		/* This page uses the copy buffer */
3887		dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3888
3889		/*
3890		 * save the copy buffer KVA that we'll use with this page.
3891		 * if we still fit within the copybuf, it's a simple add.
3892		 * otherwise, we need to wrap over using & or % accordingly.
3893		 */
3894		if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3895			dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3896			    *copybuf_used;
3897		} else {
3898			if (copybuf_sz_power_2) {
3899				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3900				    (uintptr_t)dma->dp_cbaddr +
3901				    (*copybuf_used &
3902				    (dma->dp_copybuf_size - 1)));
3903			} else {
3904				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3905				    (uintptr_t)dma->dp_cbaddr +
3906				    (*copybuf_used % dma->dp_copybuf_size));
3907			}
3908		}
3909
3910		/*
3911		 * over write the cookie physical address with the address of
3912		 * the physical address of the copy buffer page that we will
3913		 * use.
3914		 */
3915		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3916		    dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3917
3918		cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3919
3920		/* if we have a kernel VA, it's easy, just save that address */
3921		if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3922		    (sinfo->si_asp == &kas)) {
3923			/*
3924			 * save away the page aligned virtual address of the
3925			 * driver buffer. Offsets are handled in the sync code.
3926			 */
3927			dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3928			    dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3929			    & MMU_PAGEMASK);
3930#if !defined(__amd64)
3931			/*
3932			 * we didn't need to, and will never need to map this
3933			 * page.
3934			 */
3935			dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3936#endif
3937
3938		/* we don't have a kernel VA. We need one for the bcopy. */
3939		} else {
3940#if defined(__amd64)
3941			/*
3942			 * for the 64-bit kernel, it's easy. We use seg kpm to
3943			 * get a Kernel VA for the corresponding pfn.
3944			 */
3945			dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3946#else
3947			/*
3948			 * for the 32-bit kernel, this is a pain. First we'll
3949			 * save away the page_t or user VA for this page. This
3950			 * is needed in rootnex_dma_win() when we switch to a
3951			 * new window which requires us to re-map the copy
3952			 * buffer.
3953			 */
3954			pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3955			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3956				dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3957				dma->dp_pgmap[pidx].pm_vaddr = NULL;
3958			} else if (pplist != NULL) {
3959				dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3960				dma->dp_pgmap[pidx].pm_vaddr = NULL;
3961			} else {
3962				dma->dp_pgmap[pidx].pm_pp = NULL;
3963				dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3964				    (((uintptr_t)
3965				    dmar_object->dmao_obj.virt_obj.v_addr +
3966				    cur_offset) & MMU_PAGEMASK);
3967			}
3968
3969			/*
3970			 * save away the page aligned virtual address which was
3971			 * allocated from the kernel heap arena (taking into
3972			 * account if we need more copy buffer than we alloced
3973			 * and use multiple windows to handle this, i.e. &,%).
3974			 * NOTE: there isn't and physical memory backing up this
3975			 * virtual address space currently.
3976			 */
3977			if ((*copybuf_used + MMU_PAGESIZE) <=
3978			    dma->dp_copybuf_size) {
3979				dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3980				    (((uintptr_t)dma->dp_kva + *copybuf_used) &
3981				    MMU_PAGEMASK);
3982			} else {
3983				if (copybuf_sz_power_2) {
3984					dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3985					    (((uintptr_t)dma->dp_kva +
3986					    (*copybuf_used &
3987					    (dma->dp_copybuf_size - 1))) &
3988					    MMU_PAGEMASK);
3989				} else {
3990					dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3991					    (((uintptr_t)dma->dp_kva +
3992					    (*copybuf_used %
3993					    dma->dp_copybuf_size)) &
3994					    MMU_PAGEMASK);
3995				}
3996			}
3997
3998			/*
3999			 * if we haven't used up the available copy buffer yet,
4000			 * map the kva to the physical page.
4001			 */
4002			if (!dma->dp_cb_remaping && ((*copybuf_used +
4003			    MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
4004				dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
4005				if (dma->dp_pgmap[pidx].pm_pp != NULL) {
4006					i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
4007					    dma->dp_pgmap[pidx].pm_kaddr);
4008				} else {
4009					i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
4010					    sinfo->si_asp,
4011					    dma->dp_pgmap[pidx].pm_kaddr);
4012				}
4013
4014			/*
4015			 * we've used up the available copy buffer, this page
4016			 * will have to be mapped during rootnex_dma_win() when
4017			 * we switch to a new window which requires a re-map
4018			 * the copy buffer. (32-bit kernel only)
4019			 */
4020			} else {
4021				dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4022			}
4023#endif
4024			/* go to the next page_t */
4025			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4026				*cur_pp = (*cur_pp)->p_next;
4027			}
4028		}
4029
4030		/* add to the copy buffer count */
4031		*copybuf_used += MMU_PAGESIZE;
4032
4033	/*
4034	 * This cookie doesn't use the copy buffer. Walk through the pages this
4035	 * cookie occupies to reflect this.
4036	 */
4037	} else {
4038		/*
4039		 * figure out how many pages the cookie occupies. We need to
4040		 * use the original page offset of the buffer and the cookies
4041		 * offset in the buffer to do this.
4042		 */
4043		poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
4044		pcnt = mmu_btopr(cookie->dmac_size + poff);
4045
4046		while (pcnt > 0) {
4047#if !defined(__amd64)
4048			/*
4049			 * the 32-bit kernel doesn't have seg kpm, so we need
4050			 * to map in the driver buffer (if it didn't come down
4051			 * with a kernel VA) on the fly. Since this page doesn't
4052			 * use the copy buffer, it's not, or will it ever, have
4053			 * to be mapped in.
4054			 */
4055			dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4056#endif
4057			dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
4058
4059			/*
4060			 * we need to update pidx and cur_pp or we'll loose
4061			 * track of where we are.
4062			 */
4063			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4064				*cur_pp = (*cur_pp)->p_next;
4065			}
4066			pidx++;
4067			pcnt--;
4068		}
4069	}
4070}
4071
4072
4073/*
4074 * rootnex_sgllen_window_boundary()
4075 *    Called in the bind slow path when the next cookie causes us to exceed (in
4076 *    this case == since we start at 0 and sgllen starts at 1) the maximum sgl
4077 *    length supported by the DMA H/W.
4078 */
4079static int
4080rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4081    rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
4082    off_t cur_offset)
4083{
4084	off_t new_offset;
4085	size_t trim_sz;
4086	off_t coffset;
4087
4088
4089	/*
4090	 * if we know we'll never have to trim, it's pretty easy. Just move to
4091	 * the next window and init it. We're done.
4092	 */
4093	if (!dma->dp_trim_required) {
4094		(*windowp)++;
4095		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4096		(*windowp)->wd_cookie_cnt++;
4097		(*windowp)->wd_size = cookie->dmac_size;
4098		return (DDI_SUCCESS);
4099	}
4100
4101	/* figure out how much we need to trim from the window */
4102	ASSERT(attr->dma_attr_granular != 0);
4103	if (dma->dp_granularity_power_2) {
4104		trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
4105	} else {
4106		trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
4107	}
4108
4109	/* The window's a whole multiple of granularity. We're done */
4110	if (trim_sz == 0) {
4111		(*windowp)++;
4112		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4113		(*windowp)->wd_cookie_cnt++;
4114		(*windowp)->wd_size = cookie->dmac_size;
4115		return (DDI_SUCCESS);
4116	}
4117
4118	/*
4119	 * The window's not a whole multiple of granularity, since we know this
4120	 * is due to the sgllen, we need to go back to the last cookie and trim
4121	 * that one, add the left over part of the old cookie into the new
4122	 * window, and then add in the new cookie into the new window.
4123	 */
4124
4125	/*
4126	 * make sure the driver isn't making us do something bad... Trimming and
4127	 * sgllen == 1 don't go together.
4128	 */
4129	if (attr->dma_attr_sgllen == 1) {
4130		return (DDI_DMA_NOMAPPING);
4131	}
4132
4133	/*
4134	 * first, setup the current window to account for the trim. Need to go
4135	 * back to the last cookie for this.
4136	 */
4137	cookie--;
4138	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
4139	(*windowp)->wd_trim.tr_last_cookie = cookie;
4140	(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4141	ASSERT(cookie->dmac_size > trim_sz);
4142	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4143	(*windowp)->wd_size -= trim_sz;
4144
4145	/* save the buffer offsets for the next window */
4146	coffset = cookie->dmac_size - trim_sz;
4147	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4148
4149	/*
4150	 * set this now in case this is the first window. all other cases are
4151	 * set in dma_win()
4152	 */
4153	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4154
4155	/*
4156	 * initialize the next window using what's left over in the previous
4157	 * cookie.
4158	 */
4159	(*windowp)++;
4160	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4161	(*windowp)->wd_cookie_cnt++;
4162	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
4163	(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4164	(*windowp)->wd_trim.tr_first_size = trim_sz;
4165	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4166		(*windowp)->wd_dosync = B_TRUE;
4167	}
4168
4169	/*
4170	 * now go back to the current cookie and add it to the new window. set
4171	 * the new window size to the what was left over from the previous
4172	 * cookie and what's in the current cookie.
4173	 */
4174	cookie++;
4175	(*windowp)->wd_cookie_cnt++;
4176	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
4177
4178	/*
4179	 * trim plus the next cookie could put us over maxxfer (a cookie can be
4180	 * a max size of maxxfer). Handle that case.
4181	 */
4182	if ((*windowp)->wd_size > dma->dp_maxxfer) {
4183		/*
4184		 * maxxfer is already a whole multiple of granularity, and this
4185		 * trim will be <= the previous trim (since a cookie can't be
4186		 * larger than maxxfer). Make things simple here.
4187		 */
4188		trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
4189		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
4190		(*windowp)->wd_trim.tr_last_cookie = cookie;
4191		(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4192		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4193		(*windowp)->wd_size -= trim_sz;
4194		ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
4195
4196		/* save the buffer offsets for the next window */
4197		coffset = cookie->dmac_size - trim_sz;
4198		new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4199
4200		/* setup the next window */
4201		(*windowp)++;
4202		rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4203		(*windowp)->wd_cookie_cnt++;
4204		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
4205		(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4206		    coffset;
4207		(*windowp)->wd_trim.tr_first_size = trim_sz;
4208	}
4209
4210	return (DDI_SUCCESS);
4211}
4212
4213
4214/*
4215 * rootnex_copybuf_window_boundary()
4216 *    Called in bind slowpath when we get to a window boundary because we used
4217 *    up all the copy buffer that we have.
4218 */
4219static int
4220rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4221    rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
4222    size_t *copybuf_used)
4223{
4224	rootnex_sglinfo_t *sinfo;
4225	off_t new_offset;
4226	size_t trim_sz;
4227	paddr_t paddr;
4228	off_t coffset;
4229	uint_t pidx;
4230	off_t poff;
4231
4232	pidx = 0;
4233	sinfo = &dma->dp_sglinfo;
4234
4235	/*
4236	 * the copy buffer should be a whole multiple of page size. We know that
4237	 * this cookie is <= MMU_PAGESIZE.
4238	 */
4239	ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4240
4241	/*
4242	 * from now on, all new windows in this bind need to be re-mapped during
4243	 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
4244	 * space...
4245	 */
4246#if !defined(__amd64)
4247	dma->dp_cb_remaping = B_TRUE;
4248#endif
4249
4250	/* reset copybuf used */
4251	*copybuf_used = 0;
4252
4253	/*
4254	 * if we don't have to trim (since granularity is set to 1), go to the
4255	 * next window and add the current cookie to it. We know the current
4256	 * cookie uses the copy buffer since we're in this code path.
4257	 */
4258	if (!dma->dp_trim_required) {
4259		(*windowp)++;
4260		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4261
4262		/* Add this cookie to the new window */
4263		(*windowp)->wd_cookie_cnt++;
4264		(*windowp)->wd_size += cookie->dmac_size;
4265		*copybuf_used += MMU_PAGESIZE;
4266		return (DDI_SUCCESS);
4267	}
4268
4269	/*
4270	 * *** may need to trim, figure it out.
4271	 */
4272
4273	/* figure out how much we need to trim from the window */
4274	if (dma->dp_granularity_power_2) {
4275		trim_sz = (*windowp)->wd_size &
4276		    (hp->dmai_attr.dma_attr_granular - 1);
4277	} else {
4278		trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4279	}
4280
4281	/*
4282	 * if the window's a whole multiple of granularity, go to the next
4283	 * window, init it, then add in the current cookie. We know the current
4284	 * cookie uses the copy buffer since we're in this code path.
4285	 */
4286	if (trim_sz == 0) {
4287		(*windowp)++;
4288		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4289
4290		/* Add this cookie to the new window */
4291		(*windowp)->wd_cookie_cnt++;
4292		(*windowp)->wd_size += cookie->dmac_size;
4293		*copybuf_used += MMU_PAGESIZE;
4294		return (DDI_SUCCESS);
4295	}
4296
4297	/*
4298	 * *** We figured it out, we definitly need to trim
4299	 */
4300
4301	/*
4302	 * make sure the driver isn't making us do something bad...
4303	 * Trimming and sgllen == 1 don't go together.
4304	 */
4305	if (hp->dmai_attr.dma_attr_sgllen == 1) {
4306		return (DDI_DMA_NOMAPPING);
4307	}
4308
4309	/*
4310	 * first, setup the current window to account for the trim. Need to go
4311	 * back to the last cookie for this. Some of the last cookie will be in
4312	 * the current window, and some of the last cookie will be in the new
4313	 * window. All of the current cookie will be in the new window.
4314	 */
4315	cookie--;
4316	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
4317	(*windowp)->wd_trim.tr_last_cookie = cookie;
4318	(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4319	ASSERT(cookie->dmac_size > trim_sz);
4320	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4321	(*windowp)->wd_size -= trim_sz;
4322
4323	/*
4324	 * we're trimming the last cookie (not the current cookie). So that
4325	 * last cookie may have or may not have been using the copy buffer (
4326	 * we know the cookie passed in uses the copy buffer since we're in
4327	 * this code path).
4328	 *
4329	 * If the last cookie doesn't use the copy buffer, nothing special to
4330	 * do. However, if it does uses the copy buffer, it will be both the
4331	 * last page in the current window and the first page in the next
4332	 * window. Since we are reusing the copy buffer (and KVA space on the
4333	 * 32-bit kernel), this page will use the end of the copy buffer in the
4334	 * current window, and the start of the copy buffer in the next window.
4335	 * Track that info... The cookie physical address was already set to
4336	 * the copy buffer physical address in setup_cookie..
4337	 */
4338	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4339		pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4340		    (*windowp)->wd_size) >> MMU_PAGESHIFT;
4341		(*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4342		(*windowp)->wd_trim.tr_last_pidx = pidx;
4343		(*windowp)->wd_trim.tr_last_cbaddr =
4344		    dma->dp_pgmap[pidx].pm_cbaddr;
4345#if !defined(__amd64)
4346		(*windowp)->wd_trim.tr_last_kaddr =
4347		    dma->dp_pgmap[pidx].pm_kaddr;
4348#endif
4349	}
4350
4351	/* save the buffer offsets for the next window */
4352	coffset = cookie->dmac_size - trim_sz;
4353	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4354
4355	/*
4356	 * set this now in case this is the first window. all other cases are
4357	 * set in dma_win()
4358	 */
4359	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4360
4361	/*
4362	 * initialize the next window using what's left over in the previous
4363	 * cookie.
4364	 */
4365	(*windowp)++;
4366	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4367	(*windowp)->wd_cookie_cnt++;
4368	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
4369	(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4370	(*windowp)->wd_trim.tr_first_size = trim_sz;
4371
4372	/*
4373	 * again, we're tracking if the last cookie uses the copy buffer.
4374	 * read the comment above for more info on why we need to track
4375	 * additional state.
4376	 *
4377	 * For the first cookie in the new window, we need reset the physical
4378	 * address to DMA into to the start of the copy buffer plus any
4379	 * initial page offset which may be present.
4380	 */
4381	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4382		(*windowp)->wd_dosync = B_TRUE;
4383		(*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4384		(*windowp)->wd_trim.tr_first_pidx = pidx;
4385		(*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4386		poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4387
4388		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4389		    poff;
4390		(*windowp)->wd_trim.tr_first_paddr =
4391		    ROOTNEX_PADDR_TO_RBASE(paddr);
4392
4393#if !defined(__amd64)
4394		(*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
4395#endif
4396		/* account for the cookie copybuf usage in the new window */
4397		*copybuf_used += MMU_PAGESIZE;
4398
4399		/*
4400		 * every piece of code has to have a hack, and here is this
4401		 * ones :-)
4402		 *
4403		 * There is a complex interaction between setup_cookie and the
4404		 * copybuf window boundary. The complexity had to be in either
4405		 * the maxxfer window, or the copybuf window, and I chose the
4406		 * copybuf code.
4407		 *
4408		 * So in this code path, we have taken the last cookie,
4409		 * virtually broken it in half due to the trim, and it happens
4410		 * to use the copybuf which further complicates life. At the
4411		 * same time, we have already setup the current cookie, which
4412		 * is now wrong. More background info: the current cookie uses
4413		 * the copybuf, so it is only a page long max. So we need to
4414		 * fix the current cookies copy buffer address, physical
4415		 * address, and kva for the 32-bit kernel. We due this by
4416		 * bumping them by page size (of course, we can't due this on
4417		 * the physical address since the copy buffer may not be
4418		 * physically contiguous).
4419		 */
4420		cookie++;
4421		dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4422		poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4423
4424		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4425		    dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4426		cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4427
4428#if !defined(__amd64)
4429		ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
4430		dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
4431#endif
4432	} else {
4433		/* go back to the current cookie */
4434		cookie++;
4435	}
4436
4437	/*
4438	 * add the current cookie to the new window. set the new window size to
4439	 * the what was left over from the previous cookie and what's in the
4440	 * current cookie.
4441	 */
4442	(*windowp)->wd_cookie_cnt++;
4443	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
4444	ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4445
4446	/*
4447	 * we know that the cookie passed in always uses the copy buffer. We
4448	 * wouldn't be here if it didn't.
4449	 */
4450	*copybuf_used += MMU_PAGESIZE;
4451
4452	return (DDI_SUCCESS);
4453}
4454
4455
4456/*
4457 * rootnex_maxxfer_window_boundary()
4458 *    Called in bind slowpath when we get to a window boundary because we will
4459 *    go over maxxfer.
4460 */
4461static int
4462rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4463    rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4464{
4465	size_t dmac_size;
4466	off_t new_offset;
4467	size_t trim_sz;
4468	off_t coffset;
4469
4470
4471	/*
4472	 * calculate how much we have to trim off of the current cookie to equal
4473	 * maxxfer. We don't have to account for granularity here since our
4474	 * maxxfer already takes that into account.
4475	 */
4476	trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4477	ASSERT(trim_sz <= cookie->dmac_size);
4478	ASSERT(trim_sz <= dma->dp_maxxfer);
4479
4480	/* save cookie size since we need it later and we might change it */
4481	dmac_size = cookie->dmac_size;
4482
4483	/*
4484	 * if we're not trimming the entire cookie, setup the current window to
4485	 * account for the trim.
4486	 */
4487	if (trim_sz < cookie->dmac_size) {
4488		(*windowp)->wd_cookie_cnt++;
4489		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
4490		(*windowp)->wd_trim.tr_last_cookie = cookie;
4491		(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4492		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4493		(*windowp)->wd_size = dma->dp_maxxfer;
4494
4495		/*
4496		 * set the adjusted cookie size now in case this is the first
4497		 * window. All other windows are taken care of in get win
4498		 */
4499		cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4500	}
4501
4502	/*
4503	 * coffset is the current offset within the cookie, new_offset is the
4504	 * current offset with the entire buffer.
4505	 */
4506	coffset = dmac_size - trim_sz;
4507	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4508
4509	/* initialize the next window */
4510	(*windowp)++;
4511	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4512	(*windowp)->wd_cookie_cnt++;
4513	(*windowp)->wd_size = trim_sz;
4514	if (trim_sz < dmac_size) {
4515		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
4516		(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4517		    coffset;
4518		(*windowp)->wd_trim.tr_first_size = trim_sz;
4519	}
4520
4521	return (DDI_SUCCESS);
4522}
4523
4524
4525/*ARGSUSED*/
4526static int
4527rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4528    off_t off, size_t len, uint_t cache_flags)
4529{
4530	rootnex_sglinfo_t *sinfo;
4531	rootnex_pgmap_t *cbpage;
4532	rootnex_window_t *win;
4533	ddi_dma_impl_t *hp;
4534	rootnex_dma_t *dma;
4535	caddr_t fromaddr;
4536	caddr_t toaddr;
4537	uint_t psize;
4538	off_t offset;
4539	uint_t pidx;
4540	size_t size;
4541	off_t poff;
4542	int e;
4543
4544
4545	hp = (ddi_dma_impl_t *)handle;
4546	dma = (rootnex_dma_t *)hp->dmai_private;
4547	sinfo = &dma->dp_sglinfo;
4548
4549	/*
4550	 * if we don't have any windows, we don't need to sync. A copybuf
4551	 * will cause us to have at least one window.
4552	 */
4553	if (dma->dp_window == NULL) {
4554		return (DDI_SUCCESS);
4555	}
4556
4557	/* This window may not need to be sync'd */
4558	win = &dma->dp_window[dma->dp_current_win];
4559	if (!win->wd_dosync) {
4560		return (DDI_SUCCESS);
4561	}
4562
4563	/* handle off and len special cases */
4564	if ((off == 0) || (rootnex_sync_ignore_params)) {
4565		offset = win->wd_offset;
4566	} else {
4567		offset = off;
4568	}
4569	if ((len == 0) || (rootnex_sync_ignore_params)) {
4570		size = win->wd_size;
4571	} else {
4572		size = len;
4573	}
4574
4575	/* check the sync args to make sure they make a little sense */
4576	if (rootnex_sync_check_parms) {
4577		e = rootnex_valid_sync_parms(hp, win, offset, size,
4578		    cache_flags);
4579		if (e != DDI_SUCCESS) {
4580			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4581			return (DDI_FAILURE);
4582		}
4583	}
4584
4585	/*
4586	 * special case the first page to handle the offset into the page. The
4587	 * offset to the current page for our buffer is the offset into the
4588	 * first page of the buffer plus our current offset into the buffer
4589	 * itself, masked of course.
4590	 */
4591	poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4592	psize = MIN((MMU_PAGESIZE - poff), size);
4593
4594	/* go through all the pages that we want to sync */
4595	while (size > 0) {
4596		/*
4597		 * Calculate the page index relative to the start of the buffer.
4598		 * The index to the current page for our buffer is the offset
4599		 * into the first page of the buffer plus our current offset
4600		 * into the buffer itself, shifted of course...
4601		 */
4602		pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4603		ASSERT(pidx < sinfo->si_max_pages);
4604
4605		/*
4606		 * if this page uses the copy buffer, we need to sync it,
4607		 * otherwise, go on to the next page.
4608		 */
4609		cbpage = &dma->dp_pgmap[pidx];
4610		ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4611		    (cbpage->pm_uses_copybuf == B_FALSE));
4612		if (cbpage->pm_uses_copybuf) {
4613			/* cbaddr and kaddr should be page aligned */
4614			ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4615			    MMU_PAGEOFFSET) == 0);
4616			ASSERT(((uintptr_t)cbpage->pm_kaddr &
4617			    MMU_PAGEOFFSET) == 0);
4618
4619			/*
4620			 * if we're copying for the device, we are going to
4621			 * copy from the drivers buffer and to the rootnex
4622			 * allocated copy buffer.
4623			 */
4624			if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4625				fromaddr = cbpage->pm_kaddr + poff;
4626				toaddr = cbpage->pm_cbaddr + poff;
4627				ROOTNEX_DPROBE2(rootnex__sync__dev,
4628				    dev_info_t *, dma->dp_dip, size_t, psize);
4629
4630			/*
4631			 * if we're copying for the cpu/kernel, we are going to
4632			 * copy from the rootnex allocated copy buffer to the
4633			 * drivers buffer.
4634			 */
4635			} else {
4636				fromaddr = cbpage->pm_cbaddr + poff;
4637				toaddr = cbpage->pm_kaddr + poff;
4638				ROOTNEX_DPROBE2(rootnex__sync__cpu,
4639				    dev_info_t *, dma->dp_dip, size_t, psize);
4640			}
4641
4642			bcopy(fromaddr, toaddr, psize);
4643		}
4644
4645		/*
4646		 * decrement size until we're done, update our offset into the
4647		 * buffer, and get the next page size.
4648		 */
4649		size -= psize;
4650		offset += psize;
4651		psize = MIN(MMU_PAGESIZE, size);
4652
4653		/* page offset is zero for the rest of this loop */
4654		poff = 0;
4655	}
4656
4657	return (DDI_SUCCESS);
4658}
4659
4660/*
4661 * rootnex_dma_sync()
4662 *    called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4663 *    We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4664 *    is set, ddi_dma_sync() returns immediately passing back success.
4665 */
4666/*ARGSUSED*/
4667static int
4668rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4669    off_t off, size_t len, uint_t cache_flags)
4670{
4671#if defined(__amd64) && !defined(__xpv)
4672	if (IOMMU_USED(rdip)) {
4673		return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4674		    cache_flags));
4675	}
4676#endif
4677	return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4678	    cache_flags));
4679}
4680
4681/*
4682 * rootnex_valid_sync_parms()
4683 *    checks the parameters passed to sync to verify they are correct.
4684 */
4685static int
4686rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4687    off_t offset, size_t size, uint_t cache_flags)
4688{
4689	off_t woffset;
4690
4691
4692	/*
4693	 * the first part of the test to make sure the offset passed in is
4694	 * within the window.
4695	 */
4696	if (offset < win->wd_offset) {
4697		return (DDI_FAILURE);
4698	}
4699
4700	/*
4701	 * second and last part of the test to make sure the offset and length
4702	 * passed in is within the window.
4703	 */
4704	woffset = offset - win->wd_offset;
4705	if ((woffset + size) > win->wd_size) {
4706		return (DDI_FAILURE);
4707	}
4708
4709	/*
4710	 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4711	 * be set too.
4712	 */
4713	if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4714	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
4715		return (DDI_SUCCESS);
4716	}
4717
4718	/*
4719	 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4720	 * should be set. Also DDI_DMA_READ should be set in the flags.
4721	 */
4722	if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4723	    (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4724	    (hp->dmai_rflags & DDI_DMA_READ)) {
4725		return (DDI_SUCCESS);
4726	}
4727
4728	return (DDI_FAILURE);
4729}
4730
4731
4732/*ARGSUSED*/
4733static int
4734rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4735    uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4736    uint_t *ccountp)
4737{
4738	rootnex_window_t *window;
4739	rootnex_trim_t *trim;
4740	ddi_dma_impl_t *hp;
4741	rootnex_dma_t *dma;
4742	ddi_dma_obj_t *dmao;
4743#if !defined(__amd64)
4744	rootnex_sglinfo_t *sinfo;
4745	rootnex_pgmap_t *pmap;
4746	uint_t pidx;
4747	uint_t pcnt;
4748	off_t poff;
4749	int i;
4750#endif
4751
4752
4753	hp = (ddi_dma_impl_t *)handle;
4754	dma = (rootnex_dma_t *)hp->dmai_private;
4755#if !defined(__amd64)
4756	sinfo = &dma->dp_sglinfo;
4757#endif
4758
4759	/* If we try and get a window which doesn't exist, return failure */
4760	if (win >= hp->dmai_nwin) {
4761		ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4762		return (DDI_FAILURE);
4763	}
4764
4765	dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4766
4767	/*
4768	 * if we don't have any windows, and they're asking for the first
4769	 * window, setup the cookie pointer to the first cookie in the bind.
4770	 * setup our return values, then increment the cookie since we return
4771	 * the first cookie on the stack.
4772	 */
4773	if (dma->dp_window == NULL) {
4774		if (win != 0) {
4775			ROOTNEX_DPROF_INC(
4776			    &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4777			return (DDI_FAILURE);
4778		}
4779		hp->dmai_cookie = dma->dp_cookies;
4780		*offp = 0;
4781		*lenp = dmao->dmao_size;
4782		*ccountp = dma->dp_sglinfo.si_sgl_size;
4783		*cookiep = hp->dmai_cookie[0];
4784		hp->dmai_cookie++;
4785		hp->dmai_ncookies = *ccountp;
4786		hp->dmai_curcookie = 1;
4787		return (DDI_SUCCESS);
4788	}
4789
4790	/* sync the old window before moving on to the new one */
4791	window = &dma->dp_window[dma->dp_current_win];
4792	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4793		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4794		    DDI_DMA_SYNC_FORCPU);
4795	}
4796
4797#if !defined(__amd64)
4798	/*
4799	 * before we move to the next window, if we need to re-map, unmap all
4800	 * the pages in this window.
4801	 */
4802	if (dma->dp_cb_remaping) {
4803		/*
4804		 * If we switch to this window again, we'll need to map in
4805		 * on the fly next time.
4806		 */
4807		window->wd_remap_copybuf = B_TRUE;
4808
4809		/*
4810		 * calculate the page index into the buffer where this window
4811		 * starts, and the number of pages this window takes up.
4812		 */
4813		pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4814		    MMU_PAGESHIFT;
4815		poff = (sinfo->si_buf_offset + window->wd_offset) &
4816		    MMU_PAGEOFFSET;
4817		pcnt = mmu_btopr(window->wd_size + poff);
4818		ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4819
4820		/* unmap pages which are currently mapped in this window */
4821		for (i = 0; i < pcnt; i++) {
4822			if (dma->dp_pgmap[pidx].pm_mapped) {
4823				hat_unload(kas.a_hat,
4824				    dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4825				    HAT_UNLOAD);
4826				dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4827			}
4828			pidx++;
4829		}
4830	}
4831#endif
4832
4833	/*
4834	 * Move to the new window.
4835	 * NOTE: current_win must be set for sync to work right
4836	 */
4837	dma->dp_current_win = win;
4838	window = &dma->dp_window[win];
4839
4840	/* if needed, adjust the first and/or last cookies for trim */
4841	trim = &window->wd_trim;
4842	if (trim->tr_trim_first) {
4843		window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4844		window->wd_first_cookie->dmac_size = trim->tr_first_size;
4845#if !defined(__amd64)
4846		window->wd_first_cookie->dmac_type =
4847		    (window->wd_first_cookie->dmac_type &
4848		    ROOTNEX_USES_COPYBUF) + window->wd_offset;
4849#endif
4850		if (trim->tr_first_copybuf_win) {
4851			dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4852			    trim->tr_first_cbaddr;
4853#if !defined(__amd64)
4854			dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4855			    trim->tr_first_kaddr;
4856#endif
4857		}
4858	}
4859	if (trim->tr_trim_last) {
4860		trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4861		trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4862		if (trim->tr_last_copybuf_win) {
4863			dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4864			    trim->tr_last_cbaddr;
4865#if !defined(__amd64)
4866			dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4867			    trim->tr_last_kaddr;
4868#endif
4869		}
4870	}
4871
4872	/*
4873	 * setup the cookie pointer to the first cookie in the window. setup
4874	 * our return values, then increment the cookie since we return the
4875	 * first cookie on the stack.
4876	 */
4877	hp->dmai_cookie = window->wd_first_cookie;
4878	*offp = window->wd_offset;
4879	*lenp = window->wd_size;
4880	*ccountp = window->wd_cookie_cnt;
4881	*cookiep = hp->dmai_cookie[0];
4882	hp->dmai_ncookies = *ccountp;
4883	hp->dmai_curcookie = 1;
4884	hp->dmai_cookie++;
4885
4886#if !defined(__amd64)
4887	/* re-map copybuf if required for this window */
4888	if (dma->dp_cb_remaping) {
4889		/*
4890		 * calculate the page index into the buffer where this
4891		 * window starts.
4892		 */
4893		pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4894		    MMU_PAGESHIFT;
4895		ASSERT(pidx < sinfo->si_max_pages);
4896
4897		/*
4898		 * the first page can get unmapped if it's shared with the
4899		 * previous window. Even if the rest of this window is already
4900		 * mapped in, we need to still check this one.
4901		 */
4902		pmap = &dma->dp_pgmap[pidx];
4903		if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4904			if (pmap->pm_pp != NULL) {
4905				pmap->pm_mapped = B_TRUE;
4906				i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4907			} else if (pmap->pm_vaddr != NULL) {
4908				pmap->pm_mapped = B_TRUE;
4909				i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4910				    pmap->pm_kaddr);
4911			}
4912		}
4913		pidx++;
4914
4915		/* map in the rest of the pages if required */
4916		if (window->wd_remap_copybuf) {
4917			window->wd_remap_copybuf = B_FALSE;
4918
4919			/* figure out many pages this window takes up */
4920			poff = (sinfo->si_buf_offset + window->wd_offset) &
4921			    MMU_PAGEOFFSET;
4922			pcnt = mmu_btopr(window->wd_size + poff);
4923			ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4924
4925			/* map pages which require it */
4926			for (i = 1; i < pcnt; i++) {
4927				pmap = &dma->dp_pgmap[pidx];
4928				if (pmap->pm_uses_copybuf) {
4929					ASSERT(pmap->pm_mapped == B_FALSE);
4930					if (pmap->pm_pp != NULL) {
4931						pmap->pm_mapped = B_TRUE;
4932						i86_pp_map(pmap->pm_pp,
4933						    pmap->pm_kaddr);
4934					} else if (pmap->pm_vaddr != NULL) {
4935						pmap->pm_mapped = B_TRUE;
4936						i86_va_map(pmap->pm_vaddr,
4937						    sinfo->si_asp,
4938						    pmap->pm_kaddr);
4939					}
4940				}
4941				pidx++;
4942			}
4943		}
4944	}
4945#endif
4946
4947	/* if the new window uses the copy buffer, sync it for the device */
4948	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4949		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4950		    DDI_DMA_SYNC_FORDEV);
4951	}
4952
4953	return (DDI_SUCCESS);
4954}
4955
4956/*
4957 * rootnex_dma_win()
4958 *    called from ddi_dma_getwin()
4959 */
4960/*ARGSUSED*/
4961static int
4962rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4963    uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4964    uint_t *ccountp)
4965{
4966#if defined(__amd64) && !defined(__xpv)
4967	if (IOMMU_USED(rdip)) {
4968		return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4969		    cookiep, ccountp));
4970	}
4971#endif
4972
4973	return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4974	    cookiep, ccountp));
4975}
4976
4977#if defined(__amd64) && !defined(__xpv)
4978/*ARGSUSED*/
4979static int
4980rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4981    ddi_dma_handle_t handle, void *v)
4982{
4983	ddi_dma_impl_t *hp;
4984	rootnex_dma_t *dma;
4985
4986	hp = (ddi_dma_impl_t *)handle;
4987	dma = (rootnex_dma_t *)hp->dmai_private;
4988	dma->dp_iommu_private = v;
4989
4990	return (DDI_SUCCESS);
4991}
4992
4993/*ARGSUSED*/
4994static void *
4995rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4996    ddi_dma_handle_t handle)
4997{
4998	ddi_dma_impl_t *hp;
4999	rootnex_dma_t *dma;
5000
5001	hp = (ddi_dma_impl_t *)handle;
5002	dma = (rootnex_dma_t *)hp->dmai_private;
5003
5004	return (dma->dp_iommu_private);
5005}
5006#endif
5007
5008/*
5009 * ************************
5010 *  obsoleted dma routines
5011 * ************************
5012 */
5013
5014/*
5015 * rootnex_dma_mctl()
5016 *
5017 * We don't support this legacy interface any more on x86.
5018 */
5019/* ARGSUSED */
5020static int
5021rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
5022    enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
5023    uint_t cache_flags)
5024{
5025	/*
5026	 * The only thing dma_mctl is usef for anymore is legacy SPARC
5027	 * dvma and sbus-specific routines.
5028	 */
5029	return (DDI_FAILURE);
5030}
5031
5032/*
5033 * *********
5034 *  FMA Code
5035 * *********
5036 */
5037
5038/*
5039 * rootnex_fm_init()
5040 *    FMA init busop
5041 */
5042/* ARGSUSED */
5043static int
5044rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
5045    ddi_iblock_cookie_t *ibc)
5046{
5047	*ibc = rootnex_state->r_err_ibc;
5048
5049	return (ddi_system_fmcap);
5050}
5051
5052/*
5053 * rootnex_dma_check()
5054 *    Function called after a dma fault occurred to find out whether the
5055 *    fault address is associated with a driver that is able to handle faults
5056 *    and recover from faults.
5057 */
5058/* ARGSUSED */
5059static int
5060rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
5061    const void *not_used)
5062{
5063	rootnex_window_t *window;
5064	uint64_t start_addr;
5065	uint64_t fault_addr;
5066	ddi_dma_impl_t *hp;
5067	rootnex_dma_t *dma;
5068	uint64_t end_addr;
5069	size_t csize;
5070	int i;
5071	int j;
5072
5073
5074	/* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
5075	hp = (ddi_dma_impl_t *)handle;
5076	ASSERT(hp);
5077
5078	dma = (rootnex_dma_t *)hp->dmai_private;
5079
5080	/* Get the address that we need to search for */
5081	fault_addr = *(uint64_t *)addr;
5082
5083	/*
5084	 * if we don't have any windows, we can just walk through all the
5085	 * cookies.
5086	 */
5087	if (dma->dp_window == NULL) {
5088		/* for each cookie */
5089		for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
5090			/*
5091			 * if the faulted address is within the physical address
5092			 * range of the cookie, return DDI_FM_NONFATAL.
5093			 */
5094			if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
5095			    (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
5096			    dma->dp_cookies[i].dmac_size))) {
5097				return (DDI_FM_NONFATAL);
5098			}
5099		}
5100
5101		/* fault_addr not within this DMA handle */
5102		return (DDI_FM_UNKNOWN);
5103	}
5104
5105	/* we have mutiple windows, walk through each window */
5106	for (i = 0; i < hp->dmai_nwin; i++) {
5107		window = &dma->dp_window[i];
5108
5109		/* Go through all the cookies in the window */
5110		for (j = 0; j < window->wd_cookie_cnt; j++) {
5111
5112			start_addr = window->wd_first_cookie[j].dmac_laddress;
5113			csize = window->wd_first_cookie[j].dmac_size;
5114
5115			/*
5116			 * if we are trimming the first cookie in the window,
5117			 * and this is the first cookie, adjust the start
5118			 * address and size of the cookie to account for the
5119			 * trim.
5120			 */
5121			if (window->wd_trim.tr_trim_first && (j == 0)) {
5122				start_addr = window->wd_trim.tr_first_paddr;
5123				csize = window->wd_trim.tr_first_size;
5124			}
5125
5126			/*
5127			 * if we are trimming the last cookie in the window,
5128			 * and this is the last cookie, adjust the start
5129			 * address and size of the cookie to account for the
5130			 * trim.
5131			 */
5132			if (window->wd_trim.tr_trim_last &&
5133			    (j == (window->wd_cookie_cnt - 1))) {
5134				start_addr = window->wd_trim.tr_last_paddr;
5135				csize = window->wd_trim.tr_last_size;
5136			}
5137
5138			end_addr = start_addr + csize;
5139
5140			/*
5141			 * if the faulted address is within the physical
5142			 * address of the cookie, return DDI_FM_NONFATAL.
5143			 */
5144			if ((fault_addr >= start_addr) &&
5145			    (fault_addr <= end_addr)) {
5146				return (DDI_FM_NONFATAL);
5147			}
5148		}
5149	}
5150
5151	/* fault_addr not within this DMA handle */
5152	return (DDI_FM_UNKNOWN);
5153}
5154
5155/*ARGSUSED*/
5156static int
5157rootnex_quiesce(dev_info_t *dip)
5158{
5159#if defined(__amd64) && !defined(__xpv)
5160	return (immu_quiesce());
5161#else
5162	return (DDI_SUCCESS);
5163#endif
5164}
5165
5166#if defined(__xpv)
5167void
5168immu_init(void)
5169{
5170	;
5171}
5172
5173void
5174immu_startup(void)
5175{
5176	;
5177}
5178/*ARGSUSED*/
5179void
5180immu_physmem_update(uint64_t addr, uint64_t size)
5181{
5182	;
5183}
5184#endif
5185