xref: /illumos-gate/usr/src/uts/i86pc/io/rootnex.c (revision 86ef0a63)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 /*
25  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
26  * Copyright (c) 2011 Bayard G. Bell.  All rights reserved.
27  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
28  * Copyright 2017 Joyent, Inc.
29  * Copyright 2020 Ryan Zezeski
30  */
31 
32 /*
33  * x86 root nexus driver
34  */
35 
36 #include <sys/sysmacros.h>
37 #include <sys/conf.h>
38 #include <sys/autoconf.h>
39 #include <sys/sysmacros.h>
40 #include <sys/debug.h>
41 #include <sys/psw.h>
42 #include <sys/ddidmareq.h>
43 #include <sys/promif.h>
44 #include <sys/devops.h>
45 #include <sys/kmem.h>
46 #include <sys/cmn_err.h>
47 #include <vm/seg.h>
48 #include <vm/seg_kmem.h>
49 #include <vm/seg_dev.h>
50 #include <sys/vmem.h>
51 #include <sys/mman.h>
52 #include <vm/hat.h>
53 #include <vm/as.h>
54 #include <vm/page.h>
55 #include <sys/avintr.h>
56 #include <sys/errno.h>
57 #include <sys/modctl.h>
58 #include <sys/ddi_impldefs.h>
59 #include <sys/sunddi.h>
60 #include <sys/sunndi.h>
61 #include <sys/mach_intr.h>
62 #include <sys/psm.h>
63 #include <sys/ontrap.h>
64 #include <sys/atomic.h>
65 #include <sys/sdt.h>
66 #include <sys/rootnex.h>
67 #include <vm/hat_i86.h>
68 #include <sys/ddifm.h>
69 #include <sys/ddi_isa.h>
70 #include <sys/apic.h>
71 
72 #ifdef __xpv
73 #include <sys/bootinfo.h>
74 #include <sys/hypervisor.h>
75 #include <sys/bootconf.h>
76 #include <vm/kboot_mmu.h>
77 #endif
78 
79 #if !defined(__xpv)
80 #include <sys/immu.h>
81 #endif
82 
83 
84 /*
85  * enable/disable extra checking of function parameters. Useful for debugging
86  * drivers.
87  */
88 #ifdef	DEBUG
89 int rootnex_alloc_check_parms = 1;
90 int rootnex_bind_check_inuse = 1;
91 int rootnex_unbind_verify_buffer = 0;
92 int rootnex_sync_check_parms = 1;
93 #else
94 int rootnex_alloc_check_parms = 0;
95 int rootnex_bind_check_inuse = 0;
96 int rootnex_unbind_verify_buffer = 0;
97 int rootnex_sync_check_parms = 0;
98 #endif
99 
100 boolean_t rootnex_dmar_not_setup;
101 
102 /* Master Abort and Target Abort panic flag */
103 int rootnex_fm_ma_ta_panic_flag = 0;
104 
105 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
106 int rootnex_bind_fail = 1;
107 int rootnex_bind_warn = 1;
108 uint8_t *rootnex_warn_list;
109 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
110 #define	ROOTNEX_BIND_WARNING	(0x1 << 0)
111 
112 /*
113  * revert back to old broken behavior of always sync'ing entire copy buffer.
114  * This is useful if be have a buggy driver which doesn't correctly pass in
115  * the offset and size into ddi_dma_sync().
116  */
117 int rootnex_sync_ignore_params = 0;
118 
119 /*
120  * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
121  * page for alignment.  Allocate enough windows to handle a 256K buffer w/ at
122  * least 65 sgllen DMA engine, and enough copybuf buffer state pages to handle
123  * 2 pages (< 8K). We will still need to allocate the copy buffer during bind
124  * though (if we need one). These can only be modified in /etc/system before
125  * rootnex attach.
126  */
127 int rootnex_prealloc_cookies = 65;
128 int rootnex_prealloc_windows = 4;
129 int rootnex_prealloc_copybuf = 2;
130 
131 /* driver global state */
132 static rootnex_state_t *rootnex_state;
133 
134 #ifdef DEBUG
135 /* shortcut to rootnex counters */
136 static uint64_t *rootnex_cnt;
137 #endif
138 
139 /*
140  * XXX - does x86 even need these or are they left over from the SPARC days?
141  */
142 /* statically defined integer/boolean properties for the root node */
143 static rootnex_intprop_t rootnex_intprp[] = {
144 	{ "PAGESIZE",			PAGESIZE },
145 	{ "MMU_PAGESIZE",		MMU_PAGESIZE },
146 	{ "MMU_PAGEOFFSET",		MMU_PAGEOFFSET },
147 	{ DDI_RELATIVE_ADDRESSING,	1 },
148 };
149 #define	NROOT_INTPROPS	(sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
150 
151 /*
152  * If we're dom0, we're using a real device so we need to load
153  * the cookies with MFNs instead of PFNs.
154  */
155 #ifdef __xpv
156 typedef maddr_t rootnex_addr_t;
157 #define	ROOTNEX_PADDR_TO_RBASE(pa)	\
158 	(DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
159 #else
160 typedef paddr_t rootnex_addr_t;
161 #define	ROOTNEX_PADDR_TO_RBASE(pa)	(pa)
162 #endif
163 
164 static struct cb_ops rootnex_cb_ops = {
165 	nodev,		/* open */
166 	nodev,		/* close */
167 	nodev,		/* strategy */
168 	nodev,		/* print */
169 	nodev,		/* dump */
170 	nodev,		/* read */
171 	nodev,		/* write */
172 	nodev,		/* ioctl */
173 	nodev,		/* devmap */
174 	nodev,		/* mmap */
175 	nodev,		/* segmap */
176 	nochpoll,	/* chpoll */
177 	ddi_prop_op,	/* cb_prop_op */
178 	NULL,		/* struct streamtab */
179 	D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
180 	CB_REV,		/* Rev */
181 	nodev,		/* cb_aread */
182 	nodev		/* cb_awrite */
183 };
184 
185 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
186     off_t offset, off_t len, caddr_t *vaddrp);
187 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
188     struct hat *hat, struct seg *seg, caddr_t addr,
189     struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
190 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
191     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
192     ddi_dma_handle_t *handlep);
193 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
194     ddi_dma_handle_t handle);
195 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
196     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
197     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
198 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
199     ddi_dma_handle_t handle);
200 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
201     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
202 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
203     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
204     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
205 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
206     ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
207     off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
208 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
209     ddi_ctl_enum_t ctlop, void *arg, void *result);
210 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
211     ddi_iblock_cookie_t *ibc);
212 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
213     ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
214 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
215     void *);
216 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
217 
218 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
219     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
220     ddi_dma_handle_t *handlep);
221 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
222     ddi_dma_handle_t handle);
223 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
224     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
225     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
226 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
227     ddi_dma_handle_t handle);
228 #if !defined(__xpv)
229 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
230     ddi_dma_handle_t handle);
231 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
232     ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
233 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
234     ddi_dma_cookie_t *cookiep, uint_t ccount);
235 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
236     ddi_dma_handle_t handle);
237 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
238 #endif
239 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
240     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
241 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
242     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
243     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
244 
245 #if !defined(__xpv)
246 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
247     ddi_dma_handle_t handle, void *v);
248 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
249     ddi_dma_handle_t handle);
250 #endif
251 
252 
253 static struct bus_ops rootnex_bus_ops = {
254 	BUSO_REV,
255 	rootnex_map,
256 	NULL,
257 	NULL,
258 	NULL,
259 	rootnex_map_fault,
260 	0,
261 	rootnex_dma_allochdl,
262 	rootnex_dma_freehdl,
263 	rootnex_dma_bindhdl,
264 	rootnex_dma_unbindhdl,
265 	rootnex_dma_sync,
266 	rootnex_dma_win,
267 	rootnex_dma_mctl,
268 	rootnex_ctlops,
269 	ddi_bus_prop_op,
270 	i_ddi_rootnex_get_eventcookie,
271 	i_ddi_rootnex_add_eventcall,
272 	i_ddi_rootnex_remove_eventcall,
273 	i_ddi_rootnex_post_event,
274 	0,			/* bus_intr_ctl */
275 	0,			/* bus_config */
276 	0,			/* bus_unconfig */
277 	rootnex_fm_init,	/* bus_fm_init */
278 	NULL,			/* bus_fm_fini */
279 	NULL,			/* bus_fm_access_enter */
280 	NULL,			/* bus_fm_access_exit */
281 	NULL,			/* bus_powr */
282 	rootnex_intr_ops	/* bus_intr_op */
283 };
284 
285 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
286 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
287 static int rootnex_quiesce(dev_info_t *dip);
288 
289 static struct dev_ops rootnex_ops = {
290 	DEVO_REV,
291 	0,
292 	ddi_no_info,
293 	nulldev,
294 	nulldev,
295 	rootnex_attach,
296 	rootnex_detach,
297 	nulldev,
298 	&rootnex_cb_ops,
299 	&rootnex_bus_ops,
300 	NULL,
301 	rootnex_quiesce,		/* quiesce */
302 };
303 
304 static struct modldrv rootnex_modldrv = {
305 	&mod_driverops,
306 	"i86pc root nexus",
307 	&rootnex_ops
308 };
309 
310 static struct modlinkage rootnex_modlinkage = {
311 	MODREV_1,
312 	(void *)&rootnex_modldrv,
313 	NULL
314 };
315 
316 #if !defined(__xpv)
317 static iommulib_nexops_t iommulib_nexops = {
318 	IOMMU_NEXOPS_VERSION,
319 	"Rootnex IOMMU ops Vers 1.1",
320 	NULL,
321 	rootnex_coredma_allochdl,
322 	rootnex_coredma_freehdl,
323 	rootnex_coredma_bindhdl,
324 	rootnex_coredma_unbindhdl,
325 	rootnex_coredma_reset_cookies,
326 	rootnex_coredma_get_cookies,
327 	rootnex_coredma_set_cookies,
328 	rootnex_coredma_clear_cookies,
329 	rootnex_coredma_get_sleep_flags,
330 	rootnex_coredma_sync,
331 	rootnex_coredma_win,
332 	rootnex_coredma_hdl_setprivate,
333 	rootnex_coredma_hdl_getprivate
334 };
335 #endif
336 
337 /*
338  *  extern hacks
339  */
340 extern struct seg_ops segdev_ops;
341 extern int ignore_hardware_nodes;	/* force flag from ddi_impl.c */
342 #ifdef	DDI_MAP_DEBUG
343 extern int ddi_map_debug_flag;
344 #define	ddi_map_debug	if (ddi_map_debug_flag) prom_printf
345 #endif
346 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
347 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
348 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
349     psm_intr_op_t, int *);
350 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
351 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
352 
353 /*
354  * Use device arena to use for device control register mappings.
355  * Various kernel memory walkers (debugger, dtrace) need to know
356  * to avoid this address range to prevent undesired device activity.
357  */
358 extern void *device_arena_alloc(size_t size, int vm_flag);
359 extern void device_arena_free(void * vaddr, size_t size);
360 
361 
362 /*
363  *  Internal functions
364  */
365 static int rootnex_dma_init();
366 static void rootnex_add_props(dev_info_t *);
367 static int rootnex_ctl_reportdev(dev_info_t *dip);
368 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
369 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
370 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
371 static int rootnex_map_handle(ddi_map_req_t *mp);
372 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
373 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
374 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
375     rootnex_sglinfo_t *sglinfo);
376 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
377     ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
378 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
379     rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
380 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
381     rootnex_dma_t *dma, ddi_dma_attr_t *attr);
382 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
383 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
384     ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
385 static void rootnex_teardown_windows(rootnex_dma_t *dma);
386 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
387     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
388 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
389     rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
390     size_t *copybuf_used, page_t **cur_pp);
391 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
392     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
393     ddi_dma_attr_t *attr, off_t cur_offset);
394 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
395     rootnex_dma_t *dma, rootnex_window_t **windowp,
396     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
397 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
398     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
399 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
400     off_t offset, size_t size, uint_t cache_flags);
401 static int rootnex_verify_buffer(rootnex_dma_t *dma);
402 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
403     const void *comp_addr, const void *not_used);
404 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
405     rootnex_sglinfo_t *sglinfo);
406 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
407 
408 /*
409  * _init()
410  *
411  */
412 int
_init(void)413 _init(void)
414 {
415 
416 	rootnex_state = NULL;
417 	return (mod_install(&rootnex_modlinkage));
418 }
419 
420 
421 /*
422  * _info()
423  *
424  */
425 int
_info(struct modinfo * modinfop)426 _info(struct modinfo *modinfop)
427 {
428 	return (mod_info(&rootnex_modlinkage, modinfop));
429 }
430 
431 
432 /*
433  * _fini()
434  *
435  */
436 int
_fini(void)437 _fini(void)
438 {
439 	return (EBUSY);
440 }
441 
442 
443 /*
444  * rootnex_attach()
445  *
446  */
447 static int
rootnex_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)448 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
449 {
450 	int fmcap;
451 	int e;
452 
453 	switch (cmd) {
454 	case DDI_ATTACH:
455 		break;
456 	case DDI_RESUME:
457 #if !defined(__xpv)
458 		return (immu_unquiesce());
459 #else
460 		return (DDI_SUCCESS);
461 #endif
462 	default:
463 		return (DDI_FAILURE);
464 	}
465 
466 	/*
467 	 * We should only have one instance of rootnex. Save it away since we
468 	 * don't have an easy way to get it back later.
469 	 */
470 	ASSERT(rootnex_state == NULL);
471 	rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
472 
473 	rootnex_state->r_dip = dip;
474 	rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
475 	rootnex_state->r_reserved_msg_printed = B_FALSE;
476 #ifdef DEBUG
477 	rootnex_cnt = &rootnex_state->r_counters[0];
478 #endif
479 
480 	/*
481 	 * Set minimum fm capability level for i86pc platforms and then
482 	 * initialize error handling. Since we're the rootnex, we don't
483 	 * care what's returned in the fmcap field.
484 	 */
485 	ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
486 	    DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
487 	fmcap = ddi_system_fmcap;
488 	ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
489 
490 	/* initialize DMA related state */
491 	e = rootnex_dma_init();
492 	if (e != DDI_SUCCESS) {
493 		kmem_free(rootnex_state, sizeof (rootnex_state_t));
494 		return (DDI_FAILURE);
495 	}
496 
497 	/* Add static root node properties */
498 	rootnex_add_props(dip);
499 
500 	/* since we can't call ddi_report_dev() */
501 	cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
502 
503 	/* Initialize rootnex event handle */
504 	i_ddi_rootnex_init_events(dip);
505 
506 #if !defined(__xpv)
507 	e = iommulib_nexus_register(dip, &iommulib_nexops,
508 	    &rootnex_state->r_iommulib_handle);
509 
510 	ASSERT(e == DDI_SUCCESS);
511 #endif
512 
513 	return (DDI_SUCCESS);
514 }
515 
516 
517 /*
518  * rootnex_detach()
519  *
520  */
521 /*ARGSUSED*/
522 static int
rootnex_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)523 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
524 {
525 	switch (cmd) {
526 	case DDI_SUSPEND:
527 #if !defined(__xpv)
528 		return (immu_quiesce());
529 #else
530 		return (DDI_SUCCESS);
531 #endif
532 	default:
533 		return (DDI_FAILURE);
534 	}
535 	/*NOTREACHED*/
536 
537 }
538 
539 
540 /*
541  * rootnex_dma_init()
542  *
543  */
544 /*ARGSUSED*/
545 static int
rootnex_dma_init()546 rootnex_dma_init()
547 {
548 	size_t bufsize;
549 
550 
551 	/*
552 	 * size of our cookie/window/copybuf state needed in dma bind that we
553 	 * pre-alloc in dma_alloc_handle
554 	 */
555 	rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
556 	rootnex_state->r_prealloc_size =
557 	    (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
558 	    (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
559 	    (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
560 
561 	/*
562 	 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
563 	 * allocate 16 extra bytes for struct pointer alignment
564 	 * (p->dmai_private & dma->dp_prealloc_buffer)
565 	 */
566 	bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
567 	    rootnex_state->r_prealloc_size + 0x10;
568 	rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
569 	    bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
570 	if (rootnex_state->r_dmahdl_cache == NULL) {
571 		return (DDI_FAILURE);
572 	}
573 
574 	/*
575 	 * allocate array to track which major numbers we have printed warnings
576 	 * for.
577 	 */
578 	rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
579 	    KM_SLEEP);
580 
581 	return (DDI_SUCCESS);
582 }
583 
584 
585 /*
586  * rootnex_add_props()
587  *
588  */
589 static void
rootnex_add_props(dev_info_t * dip)590 rootnex_add_props(dev_info_t *dip)
591 {
592 	rootnex_intprop_t *rpp;
593 	int i;
594 
595 	/* Add static integer/boolean properties to the root node */
596 	rpp = rootnex_intprp;
597 	for (i = 0; i < NROOT_INTPROPS; i++) {
598 		(void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
599 		    rpp[i].prop_name, rpp[i].prop_value);
600 	}
601 }
602 
603 
604 
605 /*
606  * *************************
607  *  ctlops related routines
608  * *************************
609  */
610 
611 /*
612  * rootnex_ctlops()
613  *
614  */
615 /*ARGSUSED*/
616 static int
rootnex_ctlops(dev_info_t * dip,dev_info_t * rdip,ddi_ctl_enum_t ctlop,void * arg,void * result)617 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
618     void *arg, void *result)
619 {
620 	int n, *ptr;
621 	struct ddi_parent_private_data *pdp;
622 
623 	switch (ctlop) {
624 	case DDI_CTLOPS_DMAPMAPC:
625 		/*
626 		 * Return 'partial' to indicate that dma mapping
627 		 * has to be done in the main MMU.
628 		 */
629 		return (DDI_DMA_PARTIAL);
630 
631 	case DDI_CTLOPS_BTOP:
632 		/*
633 		 * Convert byte count input to physical page units.
634 		 * (byte counts that are not a page-size multiple
635 		 * are rounded down)
636 		 */
637 		*(ulong_t *)result = btop(*(ulong_t *)arg);
638 		return (DDI_SUCCESS);
639 
640 	case DDI_CTLOPS_PTOB:
641 		/*
642 		 * Convert size in physical pages to bytes
643 		 */
644 		*(ulong_t *)result = ptob(*(ulong_t *)arg);
645 		return (DDI_SUCCESS);
646 
647 	case DDI_CTLOPS_BTOPR:
648 		/*
649 		 * Convert byte count input to physical page units
650 		 * (byte counts that are not a page-size multiple
651 		 * are rounded up)
652 		 */
653 		*(ulong_t *)result = btopr(*(ulong_t *)arg);
654 		return (DDI_SUCCESS);
655 
656 	case DDI_CTLOPS_INITCHILD:
657 		return (impl_ddi_sunbus_initchild(arg));
658 
659 	case DDI_CTLOPS_UNINITCHILD:
660 		impl_ddi_sunbus_removechild(arg);
661 		return (DDI_SUCCESS);
662 
663 	case DDI_CTLOPS_REPORTDEV:
664 		return (rootnex_ctl_reportdev(rdip));
665 
666 	case DDI_CTLOPS_IOMIN:
667 		/*
668 		 * Nothing to do here but reflect back..
669 		 */
670 		return (DDI_SUCCESS);
671 
672 	case DDI_CTLOPS_REGSIZE:
673 	case DDI_CTLOPS_NREGS:
674 		break;
675 
676 	case DDI_CTLOPS_SIDDEV:
677 		if (ndi_dev_is_prom_node(rdip))
678 			return (DDI_SUCCESS);
679 		if (ndi_dev_is_persistent_node(rdip))
680 			return (DDI_SUCCESS);
681 		return (DDI_FAILURE);
682 
683 	case DDI_CTLOPS_POWER:
684 		return ((*pm_platform_power)((power_req_t *)arg));
685 
686 	case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
687 	case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
688 	case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
689 	case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
690 	case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
691 	case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
692 		if (!rootnex_state->r_reserved_msg_printed) {
693 			rootnex_state->r_reserved_msg_printed = B_TRUE;
694 			cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
695 			    "1 or more reserved/obsolete operations.");
696 		}
697 		return (DDI_FAILURE);
698 
699 	default:
700 		return (DDI_FAILURE);
701 	}
702 	/*
703 	 * The rest are for "hardware" properties
704 	 */
705 	if ((pdp = ddi_get_parent_data(rdip)) == NULL)
706 		return (DDI_FAILURE);
707 
708 	if (ctlop == DDI_CTLOPS_NREGS) {
709 		ptr = (int *)result;
710 		*ptr = pdp->par_nreg;
711 	} else {
712 		off_t *size = (off_t *)result;
713 
714 		ptr = (int *)arg;
715 		n = *ptr;
716 		if (n >= pdp->par_nreg) {
717 			return (DDI_FAILURE);
718 		}
719 		*size = (off_t)pdp->par_reg[n].regspec_size;
720 	}
721 	return (DDI_SUCCESS);
722 }
723 
724 
725 /*
726  * rootnex_ctl_reportdev()
727  *
728  */
729 static int
rootnex_ctl_reportdev(dev_info_t * dev)730 rootnex_ctl_reportdev(dev_info_t *dev)
731 {
732 	int i, n, len, f_len = 0;
733 	char *buf;
734 
735 	buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
736 	f_len += snprintf(buf, REPORTDEV_BUFSIZE,
737 	    "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
738 	len = strlen(buf);
739 
740 	for (i = 0; i < sparc_pd_getnreg(dev); i++) {
741 
742 		struct regspec *rp = sparc_pd_getreg(dev, i);
743 
744 		if (i == 0)
745 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
746 			    ": ");
747 		else
748 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
749 			    " and ");
750 		len = strlen(buf);
751 
752 		switch (rp->regspec_bustype) {
753 
754 		case BTEISA:
755 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
756 			    "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
757 			break;
758 
759 		case BTISA:
760 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
761 			    "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
762 			break;
763 
764 		default:
765 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
766 			    "space %x offset %x",
767 			    rp->regspec_bustype, rp->regspec_addr);
768 			break;
769 		}
770 		len = strlen(buf);
771 	}
772 	for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
773 		int pri;
774 
775 		if (i != 0) {
776 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
777 			    ",");
778 			len = strlen(buf);
779 		}
780 		pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
781 		f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
782 		    " sparc ipl %d", pri);
783 		len = strlen(buf);
784 	}
785 #ifdef DEBUG
786 	if (f_len + 1 >= REPORTDEV_BUFSIZE) {
787 		cmn_err(CE_NOTE, "next message is truncated: "
788 		    "printed length 1024, real length %d", f_len);
789 	}
790 #endif /* DEBUG */
791 	cmn_err(CE_CONT, "?%s\n", buf);
792 	kmem_free(buf, REPORTDEV_BUFSIZE);
793 	return (DDI_SUCCESS);
794 }
795 
796 
797 /*
798  * ******************
799  *  map related code
800  * ******************
801  */
802 
803 /*
804  * rootnex_map()
805  *
806  */
807 static int
rootnex_map(dev_info_t * dip,dev_info_t * rdip,ddi_map_req_t * mp,off_t offset,off_t len,caddr_t * vaddrp)808 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
809     off_t len, caddr_t *vaddrp)
810 {
811 	struct regspec *orp = NULL;
812 	struct regspec64 rp = { 0 };
813 	ddi_map_req_t mr = *mp;		/* Get private copy of request */
814 
815 	mp = &mr;
816 
817 	switch (mp->map_op)  {
818 	case DDI_MO_MAP_LOCKED:
819 	case DDI_MO_UNMAP:
820 	case DDI_MO_MAP_HANDLE:
821 		break;
822 	default:
823 #ifdef	DDI_MAP_DEBUG
824 		cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
825 		    mp->map_op);
826 #endif	/* DDI_MAP_DEBUG */
827 		return (DDI_ME_UNIMPLEMENTED);
828 	}
829 
830 	if (mp->map_flags & DDI_MF_USER_MAPPING)  {
831 #ifdef	DDI_MAP_DEBUG
832 		cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
833 #endif	/* DDI_MAP_DEBUG */
834 		return (DDI_ME_UNIMPLEMENTED);
835 	}
836 
837 	/*
838 	 * First, we need to get the original regspec out before we convert it
839 	 * to the extended format. If we have a register number, then we need to
840 	 * convert that to a regspec.
841 	 */
842 	if (mp->map_type == DDI_MT_RNUMBER)  {
843 
844 		int rnumber = mp->map_obj.rnumber;
845 #ifdef	DDI_MAP_DEBUG
846 		static char *out_of_range =
847 		    "rootnex_map: Out of range rnumber <%d>, device <%s>";
848 #endif	/* DDI_MAP_DEBUG */
849 
850 		orp = i_ddi_rnumber_to_regspec(rdip, rnumber);
851 		if (orp == NULL) {
852 #ifdef	DDI_MAP_DEBUG
853 			cmn_err(CE_WARN, out_of_range, rnumber,
854 			    ddi_get_name(rdip));
855 #endif	/* DDI_MAP_DEBUG */
856 			return (DDI_ME_RNUMBER_RANGE);
857 		}
858 	} else if (!(mp->map_flags & DDI_MF_EXT_REGSPEC)) {
859 		orp = mp->map_obj.rp;
860 	}
861 
862 	/*
863 	 * Ensure that we are always using a 64-bit extended regspec regardless
864 	 * of what was passed into us. If the child driver is using a 64-bit
865 	 * regspec, then we need to make sure that we copy this to the local
866 	 * regspec64, rp.
867 	 */
868 	if (orp != NULL) {
869 		rp.regspec_bustype = orp->regspec_bustype;
870 		rp.regspec_addr = orp->regspec_addr;
871 		rp.regspec_size = orp->regspec_size;
872 	} else {
873 		struct regspec64 *rp64;
874 		rp64 = (struct regspec64 *)mp->map_obj.rp;
875 		rp = *rp64;
876 	}
877 
878 	mp->map_type = DDI_MT_REGSPEC;
879 	mp->map_flags |= DDI_MF_EXT_REGSPEC;
880 	mp->map_obj.rp = (struct regspec *)&rp;
881 
882 	/*
883 	 * Adjust offset and length correspnding to called values...
884 	 * XXX: A non-zero length means override the one in the regspec
885 	 * XXX: (regardless of what's in the parent's range?)
886 	 */
887 
888 #ifdef	DDI_MAP_DEBUG
889 	cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
890 	    "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
891 	    rp.regspec_bustype, rp.regspec_addr, rp.regspec_size, offset,
892 	    len, mp->map_handlep);
893 #endif	/* DDI_MAP_DEBUG */
894 
895 	/*
896 	 * I/O or memory mapping:
897 	 *
898 	 *	<bustype=0, addr=x, len=x>: memory
899 	 *	<bustype=1, addr=x, len=x>: i/o
900 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
901 	 */
902 
903 	if (rp.regspec_bustype > 1 && rp.regspec_addr != 0) {
904 		cmn_err(CE_WARN, "<%s,%s> invalid register spec"
905 		    " <0x%" PRIx64 ", 0x%" PRIx64 ", 0x%" PRIx64 ">",
906 		    ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
907 		    rp.regspec_addr, rp.regspec_size);
908 		return (DDI_ME_INVAL);
909 	}
910 
911 	if (rp.regspec_bustype > 1 && rp.regspec_addr == 0) {
912 		/*
913 		 * compatibility i/o mapping
914 		 */
915 		rp.regspec_bustype += offset;
916 	} else {
917 		/*
918 		 * Normal memory or i/o mapping
919 		 */
920 		rp.regspec_addr += offset;
921 	}
922 
923 	if (len != 0)
924 		rp.regspec_size = len;
925 
926 #ifdef	DDI_MAP_DEBUG
927 	cmn_err(CE_CONT, "             <%s,%s> <0x%" PRIx64 ", 0x%" PRIx64
928 	    ", 0x%" PRId64 "> offset %d len %d handle 0x%x\n",
929 	    ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
930 	    rp.regspec_addr, rp.regspec_size, offset, len, mp->map_handlep);
931 #endif	/* DDI_MAP_DEBUG */
932 
933 
934 	/*
935 	 * The x86 root nexus does not have any notion of valid ranges of
936 	 * addresses. Its children have valid ranges, but because there are none
937 	 * for the nexus, we don't need to call i_ddi_apply_range().  Verify
938 	 * that is the case.
939 	 */
940 	ASSERT0(sparc_pd_getnrng(dip));
941 
942 	switch (mp->map_op)  {
943 	case DDI_MO_MAP_LOCKED:
944 
945 		/*
946 		 * Set up the locked down kernel mapping to the regspec...
947 		 */
948 
949 		return (rootnex_map_regspec(mp, vaddrp));
950 
951 	case DDI_MO_UNMAP:
952 
953 		/*
954 		 * Release mapping...
955 		 */
956 
957 		return (rootnex_unmap_regspec(mp, vaddrp));
958 
959 	case DDI_MO_MAP_HANDLE:
960 
961 		return (rootnex_map_handle(mp));
962 
963 	default:
964 		return (DDI_ME_UNIMPLEMENTED);
965 	}
966 }
967 
968 
969 /*
970  * rootnex_map_fault()
971  *
972  *	fault in mappings for requestors
973  */
974 /*ARGSUSED*/
975 static int
rootnex_map_fault(dev_info_t * dip,dev_info_t * rdip,struct hat * hat,struct seg * seg,caddr_t addr,struct devpage * dp,pfn_t pfn,uint_t prot,uint_t lock)976 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
977     struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
978     uint_t lock)
979 {
980 
981 #ifdef	DDI_MAP_DEBUG
982 	ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
983 	ddi_map_debug(" Seg <%s>\n",
984 	    seg->s_ops == &segdev_ops ? "segdev" :
985 	    seg == &kvseg ? "segkmem" : "NONE!");
986 #endif	/* DDI_MAP_DEBUG */
987 
988 	/*
989 	 * This is all terribly broken, but it is a start
990 	 *
991 	 * XXX	Note that this test means that segdev_ops
992 	 *	must be exported from seg_dev.c.
993 	 * XXX	What about devices with their own segment drivers?
994 	 */
995 	if (seg->s_ops == &segdev_ops) {
996 		struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
997 
998 		if (hat == NULL) {
999 			/*
1000 			 * This is one plausible interpretation of
1001 			 * a null hat i.e. use the first hat on the
1002 			 * address space hat list which by convention is
1003 			 * the hat of the system MMU.  At alternative
1004 			 * would be to panic .. this might well be better ..
1005 			 */
1006 			ASSERT(AS_READ_HELD(seg->s_as));
1007 			hat = seg->s_as->a_hat;
1008 			cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1009 		}
1010 		hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1011 		    (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1012 	} else if (seg == &kvseg && dp == NULL) {
1013 		hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1014 		    HAT_LOAD_LOCK);
1015 	} else
1016 		return (DDI_FAILURE);
1017 	return (DDI_SUCCESS);
1018 }
1019 
1020 
1021 static int
rootnex_map_regspec(ddi_map_req_t * mp,caddr_t * vaddrp)1022 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1023 {
1024 	rootnex_addr_t rbase;
1025 	void *cvaddr;
1026 	uint64_t npages, pgoffset;
1027 	struct regspec64 *rp;
1028 	ddi_acc_hdl_t *hp;
1029 	ddi_acc_impl_t *ap;
1030 	uint_t	hat_acc_flags;
1031 	paddr_t pbase;
1032 
1033 	ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
1034 	rp = (struct regspec64 *)mp->map_obj.rp;
1035 	hp = mp->map_handlep;
1036 
1037 #ifdef	DDI_MAP_DEBUG
1038 	ddi_map_debug(
1039 	    "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1040 	    rp->regspec_bustype, rp->regspec_addr,
1041 	    rp->regspec_size, mp->map_handlep);
1042 #endif	/* DDI_MAP_DEBUG */
1043 
1044 	/*
1045 	 * I/O or memory mapping
1046 	 *
1047 	 *	<bustype=0, addr=x, len=x>: memory
1048 	 *	<bustype=1, addr=x, len=x>: i/o
1049 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1050 	 */
1051 
1052 	if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1053 		cmn_err(CE_WARN, "rootnex: invalid register spec"
1054 		    " <0x%" PRIx64 ", 0x%" PRIx64", 0x%" PRIx64">",
1055 		    rp->regspec_bustype, rp->regspec_addr, rp->regspec_size);
1056 		return (DDI_FAILURE);
1057 	}
1058 
1059 	if (rp->regspec_bustype != 0) {
1060 		/*
1061 		 * I/O space - needs a handle.
1062 		 */
1063 		if (hp == NULL) {
1064 			return (DDI_FAILURE);
1065 		}
1066 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1067 		ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1068 		impl_acc_hdl_init(hp);
1069 
1070 		if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1071 #ifdef  DDI_MAP_DEBUG
1072 			ddi_map_debug("rootnex_map_regspec: mmap() "
1073 			    "to I/O space is not supported.\n");
1074 #endif  /* DDI_MAP_DEBUG */
1075 			return (DDI_ME_INVAL);
1076 		} else {
1077 			/*
1078 			 * 1275-compliant vs. compatibility i/o mapping
1079 			 */
1080 			*vaddrp =
1081 			    (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1082 			    ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1083 			    ((caddr_t)(uintptr_t)rp->regspec_addr);
1084 #ifdef __xpv
1085 			if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1086 				hp->ah_pfn = xen_assign_pfn(
1087 				    mmu_btop((ulong_t)rp->regspec_addr &
1088 				    MMU_PAGEMASK));
1089 			} else {
1090 				hp->ah_pfn = mmu_btop(
1091 				    (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1092 			}
1093 #else
1094 			hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1095 			    MMU_PAGEMASK);
1096 #endif
1097 			hp->ah_pnum = mmu_btopr(rp->regspec_size +
1098 			    (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1099 		}
1100 
1101 #ifdef	DDI_MAP_DEBUG
1102 		ddi_map_debug(
1103 	    "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1104 		    rp->regspec_size, *vaddrp);
1105 #endif	/* DDI_MAP_DEBUG */
1106 		return (DDI_SUCCESS);
1107 	}
1108 
1109 	/*
1110 	 * Memory space
1111 	 */
1112 
1113 	if (hp != NULL) {
1114 		/*
1115 		 * hat layer ignores
1116 		 * hp->ah_acc.devacc_attr_endian_flags.
1117 		 */
1118 		switch (hp->ah_acc.devacc_attr_dataorder) {
1119 		case DDI_STRICTORDER_ACC:
1120 			hat_acc_flags = HAT_STRICTORDER;
1121 			break;
1122 		case DDI_UNORDERED_OK_ACC:
1123 			hat_acc_flags = HAT_UNORDERED_OK;
1124 			break;
1125 		case DDI_MERGING_OK_ACC:
1126 			hat_acc_flags = HAT_MERGING_OK;
1127 			break;
1128 		case DDI_LOADCACHING_OK_ACC:
1129 			hat_acc_flags = HAT_LOADCACHING_OK;
1130 			break;
1131 		case DDI_STORECACHING_OK_ACC:
1132 			hat_acc_flags = HAT_STORECACHING_OK;
1133 			break;
1134 		default:
1135 			return (DDI_ME_INVAL);
1136 		}
1137 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1138 		ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1139 		impl_acc_hdl_init(hp);
1140 		hp->ah_hat_flags = hat_acc_flags;
1141 	} else {
1142 		hat_acc_flags = HAT_STRICTORDER;
1143 	}
1144 
1145 	rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1146 #ifdef __xpv
1147 	/*
1148 	 * If we're dom0, we're using a real device so we need to translate
1149 	 * the MA to a PA.
1150 	 */
1151 	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1152 		pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1153 	} else {
1154 		pbase = rbase;
1155 	}
1156 #else
1157 	pbase = rbase;
1158 #endif
1159 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1160 
1161 	if (rp->regspec_size == 0) {
1162 #ifdef  DDI_MAP_DEBUG
1163 		ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1164 #endif  /* DDI_MAP_DEBUG */
1165 		return (DDI_ME_INVAL);
1166 	}
1167 
1168 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1169 		/* extra cast to make gcc happy */
1170 		*vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1171 	} else {
1172 		npages = mmu_btopr(rp->regspec_size + pgoffset);
1173 
1174 #ifdef	DDI_MAP_DEBUG
1175 		ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1176 		    "physical %llx", npages, pbase);
1177 #endif	/* DDI_MAP_DEBUG */
1178 
1179 		cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1180 		if (cvaddr == NULL)
1181 			return (DDI_ME_NORESOURCES);
1182 
1183 		/*
1184 		 * Now map in the pages we've allocated...
1185 		 */
1186 		hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1187 		    mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1188 		    HAT_LOAD_LOCK);
1189 		*vaddrp = (caddr_t)cvaddr + pgoffset;
1190 
1191 		/* save away pfn and npages for FMA */
1192 		hp = mp->map_handlep;
1193 		if (hp) {
1194 			hp->ah_pfn = mmu_btop(pbase);
1195 			hp->ah_pnum = npages;
1196 		}
1197 	}
1198 
1199 #ifdef	DDI_MAP_DEBUG
1200 	ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1201 #endif	/* DDI_MAP_DEBUG */
1202 	return (DDI_SUCCESS);
1203 }
1204 
1205 
1206 static int
rootnex_unmap_regspec(ddi_map_req_t * mp,caddr_t * vaddrp)1207 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1208 {
1209 	caddr_t addr = (caddr_t)*vaddrp;
1210 	uint64_t npages, pgoffset;
1211 	struct regspec64 *rp;
1212 
1213 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1214 		return (0);
1215 
1216 	ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
1217 	rp = (struct regspec64 *)mp->map_obj.rp;
1218 
1219 	if (rp->regspec_size == 0) {
1220 #ifdef  DDI_MAP_DEBUG
1221 		ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1222 #endif  /* DDI_MAP_DEBUG */
1223 		return (DDI_ME_INVAL);
1224 	}
1225 
1226 	/*
1227 	 * I/O or memory mapping:
1228 	 *
1229 	 *	<bustype=0, addr=x, len=x>: memory
1230 	 *	<bustype=1, addr=x, len=x>: i/o
1231 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1232 	 */
1233 	if (rp->regspec_bustype != 0) {
1234 		/*
1235 		 * This is I/O space, which requires no particular
1236 		 * processing on unmap since it isn't mapped in the
1237 		 * first place.
1238 		 */
1239 		return (DDI_SUCCESS);
1240 	}
1241 
1242 	/*
1243 	 * Memory space
1244 	 */
1245 	pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1246 	npages = mmu_btopr(rp->regspec_size + pgoffset);
1247 	hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1248 	device_arena_free(addr - pgoffset, ptob(npages));
1249 
1250 	/*
1251 	 * Destroy the pointer - the mapping has logically gone
1252 	 */
1253 	*vaddrp = NULL;
1254 
1255 	return (DDI_SUCCESS);
1256 }
1257 
1258 static int
rootnex_map_handle(ddi_map_req_t * mp)1259 rootnex_map_handle(ddi_map_req_t *mp)
1260 {
1261 	rootnex_addr_t rbase;
1262 	ddi_acc_hdl_t *hp;
1263 	uint64_t pgoffset;
1264 	struct regspec64 *rp;
1265 	paddr_t pbase;
1266 
1267 	rp = (struct regspec64 *)mp->map_obj.rp;
1268 
1269 #ifdef	DDI_MAP_DEBUG
1270 	ddi_map_debug(
1271 	    "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1272 	    rp->regspec_bustype, rp->regspec_addr,
1273 	    rp->regspec_size, mp->map_handlep);
1274 #endif	/* DDI_MAP_DEBUG */
1275 
1276 	/*
1277 	 * I/O or memory mapping:
1278 	 *
1279 	 *	<bustype=0, addr=x, len=x>: memory
1280 	 *	<bustype=1, addr=x, len=x>: i/o
1281 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1282 	 */
1283 	if (rp->regspec_bustype != 0) {
1284 		/*
1285 		 * This refers to I/O space, and we don't support "mapping"
1286 		 * I/O space to a user.
1287 		 */
1288 		return (DDI_FAILURE);
1289 	}
1290 
1291 	/*
1292 	 * Set up the hat_flags for the mapping.
1293 	 */
1294 	hp = mp->map_handlep;
1295 
1296 	switch (hp->ah_acc.devacc_attr_endian_flags) {
1297 	case DDI_NEVERSWAP_ACC:
1298 		hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1299 		break;
1300 	case DDI_STRUCTURE_LE_ACC:
1301 		hp->ah_hat_flags = HAT_STRUCTURE_LE;
1302 		break;
1303 	case DDI_STRUCTURE_BE_ACC:
1304 		return (DDI_FAILURE);
1305 	default:
1306 		return (DDI_REGS_ACC_CONFLICT);
1307 	}
1308 
1309 	switch (hp->ah_acc.devacc_attr_dataorder) {
1310 	case DDI_STRICTORDER_ACC:
1311 		break;
1312 	case DDI_UNORDERED_OK_ACC:
1313 		hp->ah_hat_flags |= HAT_UNORDERED_OK;
1314 		break;
1315 	case DDI_MERGING_OK_ACC:
1316 		hp->ah_hat_flags |= HAT_MERGING_OK;
1317 		break;
1318 	case DDI_LOADCACHING_OK_ACC:
1319 		hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1320 		break;
1321 	case DDI_STORECACHING_OK_ACC:
1322 		hp->ah_hat_flags |= HAT_STORECACHING_OK;
1323 		break;
1324 	default:
1325 		return (DDI_FAILURE);
1326 	}
1327 
1328 	rbase = (rootnex_addr_t)rp->regspec_addr &
1329 	    (~(rootnex_addr_t)MMU_PAGEOFFSET);
1330 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1331 
1332 	if (rp->regspec_size == 0)
1333 		return (DDI_ME_INVAL);
1334 
1335 #ifdef __xpv
1336 	/*
1337 	 * If we're dom0, we're using a real device so we need to translate
1338 	 * the MA to a PA.
1339 	 */
1340 	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1341 		pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1342 		    (rbase & MMU_PAGEOFFSET);
1343 	} else {
1344 		pbase = rbase;
1345 	}
1346 #else
1347 	pbase = rbase;
1348 #endif
1349 
1350 	hp->ah_pfn = mmu_btop(pbase);
1351 	hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1352 
1353 	return (DDI_SUCCESS);
1354 }
1355 
1356 
1357 
1358 /*
1359  * ************************
1360  *  interrupt related code
1361  * ************************
1362  */
1363 
1364 /*
1365  * rootnex_intr_ops()
1366  *	bus_intr_op() function for interrupt support
1367  */
1368 /* ARGSUSED */
1369 static int
rootnex_intr_ops(dev_info_t * pdip,dev_info_t * rdip,ddi_intr_op_t intr_op,ddi_intr_handle_impl_t * hdlp,void * result)1370 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1371     ddi_intr_handle_impl_t *hdlp, void *result)
1372 {
1373 	struct intrspec			*ispec;
1374 
1375 	DDI_INTR_NEXDBG((CE_CONT,
1376 	    "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1377 	    (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1378 
1379 	/* Process the interrupt operation */
1380 	switch (intr_op) {
1381 	case DDI_INTROP_GETCAP:
1382 		/* First check with pcplusmp */
1383 		if (psm_intr_ops == NULL)
1384 			return (DDI_FAILURE);
1385 
1386 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1387 			*(int *)result = 0;
1388 			return (DDI_FAILURE);
1389 		}
1390 		break;
1391 	case DDI_INTROP_SETCAP:
1392 		if (psm_intr_ops == NULL)
1393 			return (DDI_FAILURE);
1394 
1395 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1396 			return (DDI_FAILURE);
1397 		break;
1398 	case DDI_INTROP_ALLOC:
1399 		ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1400 		return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1401 	case DDI_INTROP_FREE:
1402 		ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1403 		return (rootnex_free_intr_fixed(rdip, hdlp));
1404 	case DDI_INTROP_GETPRI:
1405 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1406 			return (DDI_FAILURE);
1407 		*(int *)result = ispec->intrspec_pri;
1408 		break;
1409 	case DDI_INTROP_SETPRI:
1410 		/* Validate the interrupt priority passed to us */
1411 		if (*(int *)result > LOCK_LEVEL)
1412 			return (DDI_FAILURE);
1413 
1414 		/* Ensure that PSM is all initialized and ispec is ok */
1415 		if ((psm_intr_ops == NULL) ||
1416 		    ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1417 			return (DDI_FAILURE);
1418 
1419 		/* Change the priority */
1420 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1421 		    PSM_FAILURE)
1422 			return (DDI_FAILURE);
1423 
1424 		/* update the ispec with the new priority */
1425 		ispec->intrspec_pri =  *(int *)result;
1426 		break;
1427 	case DDI_INTROP_ADDISR:
1428 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1429 			return (DDI_FAILURE);
1430 		ispec->intrspec_func = hdlp->ih_cb_func;
1431 		break;
1432 	case DDI_INTROP_REMISR:
1433 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1434 			return (DDI_FAILURE);
1435 		ispec->intrspec_func = (uint_t (*)()) 0;
1436 		break;
1437 	case DDI_INTROP_ENABLE:
1438 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1439 			return (DDI_FAILURE);
1440 
1441 		/* Call psmi to translate irq with the dip */
1442 		if (psm_intr_ops == NULL)
1443 			return (DDI_FAILURE);
1444 
1445 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1446 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1447 		    (int *)&hdlp->ih_vector) == PSM_FAILURE)
1448 			return (DDI_FAILURE);
1449 
1450 		/* Add the interrupt handler */
1451 		if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1452 		    hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1453 		    hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1454 			return (DDI_FAILURE);
1455 		break;
1456 	case DDI_INTROP_DISABLE:
1457 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1458 			return (DDI_FAILURE);
1459 
1460 		/* Call psm_ops() to translate irq with the dip */
1461 		if (psm_intr_ops == NULL)
1462 			return (DDI_FAILURE);
1463 
1464 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1465 		(void) (*psm_intr_ops)(rdip, hdlp,
1466 		    PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1467 
1468 		/* Remove the interrupt handler */
1469 		rem_avintr((void *)hdlp, ispec->intrspec_pri,
1470 		    hdlp->ih_cb_func, hdlp->ih_vector);
1471 		break;
1472 	case DDI_INTROP_SETMASK:
1473 		if (psm_intr_ops == NULL)
1474 			return (DDI_FAILURE);
1475 
1476 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1477 			return (DDI_FAILURE);
1478 		break;
1479 	case DDI_INTROP_CLRMASK:
1480 		if (psm_intr_ops == NULL)
1481 			return (DDI_FAILURE);
1482 
1483 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1484 			return (DDI_FAILURE);
1485 		break;
1486 	case DDI_INTROP_GETPENDING:
1487 		if (psm_intr_ops == NULL)
1488 			return (DDI_FAILURE);
1489 
1490 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1491 		    result)) {
1492 			*(int *)result = 0;
1493 			return (DDI_FAILURE);
1494 		}
1495 		break;
1496 	case DDI_INTROP_NAVAIL:
1497 	case DDI_INTROP_NINTRS:
1498 		*(int *)result = i_ddi_get_intx_nintrs(rdip);
1499 		if (*(int *)result == 0) {
1500 			/*
1501 			 * Special case for 'pcic' driver' only. This driver
1502 			 * driver is a child of 'isa' and 'rootnex' drivers.
1503 			 *
1504 			 * See detailed comments on this in the function
1505 			 * rootnex_get_ispec().
1506 			 *
1507 			 * Children of 'pcic' send 'NINITR' request all the
1508 			 * way to rootnex driver. But, the 'pdp->par_nintr'
1509 			 * field may not initialized. So, we fake it here
1510 			 * to return 1 (a la what PCMCIA nexus does).
1511 			 */
1512 			if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1513 				*(int *)result = 1;
1514 			else
1515 				return (DDI_FAILURE);
1516 		}
1517 		break;
1518 	case DDI_INTROP_SUPPORTED_TYPES:
1519 		*(int *)result = DDI_INTR_TYPE_FIXED;	/* Always ... */
1520 		break;
1521 	default:
1522 		return (DDI_FAILURE);
1523 	}
1524 
1525 	return (DDI_SUCCESS);
1526 }
1527 
1528 
1529 /*
1530  * rootnex_get_ispec()
1531  *	convert an interrupt number to an interrupt specification.
1532  *	The interrupt number determines which interrupt spec will be
1533  *	returned if more than one exists.
1534  *
1535  *	Look into the parent private data area of the 'rdip' to find out
1536  *	the interrupt specification.  First check to make sure there is
1537  *	one that matchs "inumber" and then return a pointer to it.
1538  *
1539  *	Return NULL if one could not be found.
1540  *
1541  *	NOTE: This is needed for rootnex_intr_ops()
1542  */
1543 static struct intrspec *
rootnex_get_ispec(dev_info_t * rdip,int inum)1544 rootnex_get_ispec(dev_info_t *rdip, int inum)
1545 {
1546 	struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1547 
1548 	/*
1549 	 * Special case handling for drivers that provide their own
1550 	 * intrspec structures instead of relying on the DDI framework.
1551 	 *
1552 	 * A broken hardware driver in ON could potentially provide its
1553 	 * own intrspec structure, instead of relying on the hardware.
1554 	 * If these drivers are children of 'rootnex' then we need to
1555 	 * continue to provide backward compatibility to them here.
1556 	 *
1557 	 * Following check is a special case for 'pcic' driver which
1558 	 * was found to have broken hardwre andby provides its own intrspec.
1559 	 *
1560 	 * Verbatim comments from this driver are shown here:
1561 	 * "Don't use the ddi_add_intr since we don't have a
1562 	 * default intrspec in all cases."
1563 	 *
1564 	 * Since an 'ispec' may not be always created for it,
1565 	 * check for that and create one if so.
1566 	 *
1567 	 * NOTE: Currently 'pcic' is the only driver found to do this.
1568 	 */
1569 	if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1570 		pdp->par_nintr = 1;
1571 		pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1572 		    pdp->par_nintr, KM_SLEEP);
1573 	}
1574 
1575 	/* Validate the interrupt number */
1576 	if (inum >= pdp->par_nintr)
1577 		return (NULL);
1578 
1579 	/* Get the interrupt structure pointer and return that */
1580 	return ((struct intrspec *)&pdp->par_intr[inum]);
1581 }
1582 
1583 /*
1584  * Allocate interrupt vector for FIXED (legacy) type.
1585  */
1586 static int
rootnex_alloc_intr_fixed(dev_info_t * rdip,ddi_intr_handle_impl_t * hdlp,void * result)1587 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1588     void *result)
1589 {
1590 	struct intrspec		*ispec;
1591 	ddi_intr_handle_impl_t	info_hdl;
1592 	int			ret;
1593 	int			free_phdl = 0;
1594 	apic_get_type_t		type_info;
1595 
1596 	if (psm_intr_ops == NULL)
1597 		return (DDI_FAILURE);
1598 
1599 	if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1600 		return (DDI_FAILURE);
1601 
1602 	/*
1603 	 * If the PSM module is "APIX" then pass the request for it
1604 	 * to allocate the vector now.
1605 	 */
1606 	bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1607 	info_hdl.ih_private = &type_info;
1608 	if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1609 	    PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1610 		if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1611 			free_phdl = 1;
1612 			i_ddi_alloc_intr_phdl(hdlp);
1613 		}
1614 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1615 		ret = (*psm_intr_ops)(rdip, hdlp,
1616 		    PSM_INTR_OP_ALLOC_VECTORS, result);
1617 		if (free_phdl) { /* free up the phdl structure */
1618 			free_phdl = 0;
1619 			i_ddi_free_intr_phdl(hdlp);
1620 			hdlp->ih_private = NULL;
1621 		}
1622 	} else {
1623 		/*
1624 		 * No APIX module; fall back to the old scheme where the
1625 		 * interrupt vector is allocated during ddi_enable_intr() call.
1626 		 */
1627 		hdlp->ih_pri = ispec->intrspec_pri;
1628 		*(int *)result = hdlp->ih_scratch1;
1629 		ret = DDI_SUCCESS;
1630 	}
1631 
1632 	return (ret);
1633 }
1634 
1635 /*
1636  * Free up interrupt vector for FIXED (legacy) type.
1637  */
1638 static int
rootnex_free_intr_fixed(dev_info_t * rdip,ddi_intr_handle_impl_t * hdlp)1639 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1640 {
1641 	struct intrspec			*ispec;
1642 	struct ddi_parent_private_data	*pdp;
1643 	ddi_intr_handle_impl_t		info_hdl;
1644 	int				ret;
1645 	apic_get_type_t			type_info;
1646 
1647 	if (psm_intr_ops == NULL)
1648 		return (DDI_FAILURE);
1649 
1650 	/*
1651 	 * If the PSM module is "APIX" then pass the request for it
1652 	 * to free up the vector now.
1653 	 */
1654 	bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1655 	info_hdl.ih_private = &type_info;
1656 	if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1657 	    PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1658 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1659 			return (DDI_FAILURE);
1660 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1661 		ret = (*psm_intr_ops)(rdip, hdlp,
1662 		    PSM_INTR_OP_FREE_VECTORS, NULL);
1663 	} else {
1664 		/*
1665 		 * No APIX module; fall back to the old scheme where
1666 		 * the interrupt vector was already freed during
1667 		 * ddi_disable_intr() call.
1668 		 */
1669 		ret = DDI_SUCCESS;
1670 	}
1671 
1672 	pdp = ddi_get_parent_data(rdip);
1673 
1674 	/*
1675 	 * Special case for 'pcic' driver' only.
1676 	 * If an intrspec was created for it, clean it up here
1677 	 * See detailed comments on this in the function
1678 	 * rootnex_get_ispec().
1679 	 */
1680 	if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1681 		kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1682 		    pdp->par_nintr);
1683 		/*
1684 		 * Set it to zero; so that
1685 		 * DDI framework doesn't free it again
1686 		 */
1687 		pdp->par_intr = NULL;
1688 		pdp->par_nintr = 0;
1689 	}
1690 
1691 	return (ret);
1692 }
1693 
1694 
1695 /*
1696  * ******************
1697  *  dma related code
1698  * ******************
1699  */
1700 
1701 /*ARGSUSED*/
1702 static int
rootnex_coredma_allochdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_attr_t * attr,int (* waitfp)(caddr_t),caddr_t arg,ddi_dma_handle_t * handlep)1703 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1704     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1705     ddi_dma_handle_t *handlep)
1706 {
1707 	uint64_t maxsegmentsize_ll;
1708 	uint_t maxsegmentsize;
1709 	ddi_dma_impl_t *hp;
1710 	rootnex_dma_t *dma;
1711 	uint64_t count_max;
1712 	uint64_t seg;
1713 	int kmflag;
1714 	int e;
1715 
1716 
1717 	/* convert our sleep flags */
1718 	if (waitfp == DDI_DMA_SLEEP) {
1719 		kmflag = KM_SLEEP;
1720 	} else {
1721 		kmflag = KM_NOSLEEP;
1722 	}
1723 
1724 	/*
1725 	 * We try to do only one memory allocation here. We'll do a little
1726 	 * pointer manipulation later. If the bind ends up taking more than
1727 	 * our prealloc's space, we'll have to allocate more memory in the
1728 	 * bind operation. Not great, but much better than before and the
1729 	 * best we can do with the current bind interfaces.
1730 	 */
1731 	hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1732 	if (hp == NULL)
1733 		return (DDI_DMA_NORESOURCES);
1734 
1735 	/* Do our pointer manipulation now, align the structures */
1736 	hp->dmai_private = (void *)(((uintptr_t)hp +
1737 	    (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1738 	dma = (rootnex_dma_t *)hp->dmai_private;
1739 	dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1740 	    sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1741 
1742 	/* setup the handle */
1743 	rootnex_clean_dmahdl(hp);
1744 	hp->dmai_error.err_fep = NULL;
1745 	hp->dmai_error.err_cf = NULL;
1746 	dma->dp_dip = rdip;
1747 	dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1748 	dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1749 
1750 	/*
1751 	 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1752 	 * is being used. Set the upper limit to the seg value.
1753 	 * There will be enough DVMA space to always get addresses
1754 	 * that will match the constraints.
1755 	 */
1756 	if (IOMMU_USED(rdip) &&
1757 	    (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1758 		dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1759 		dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1760 	} else
1761 		dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1762 
1763 	hp->dmai_minxfer = attr->dma_attr_minxfer;
1764 	hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1765 	hp->dmai_rdip = rdip;
1766 	hp->dmai_attr = *attr;
1767 
1768 	if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1769 		dma->dp_sglinfo.si_cancross = B_FALSE;
1770 	else
1771 		dma->dp_sglinfo.si_cancross = B_TRUE;
1772 
1773 	/* we don't need to worry about the SPL since we do a tryenter */
1774 	mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1775 
1776 	/*
1777 	 * Figure out our maximum segment size. If the segment size is greater
1778 	 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1779 	 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1780 	 * dma_attr_count_max are size-1 type values.
1781 	 *
1782 	 * Maximum segment size is the largest physically contiguous chunk of
1783 	 * memory that we can return from a bind (i.e. the maximum size of a
1784 	 * single cookie).
1785 	 */
1786 
1787 	/* handle the rollover cases */
1788 	seg = attr->dma_attr_seg + 1;
1789 	if (seg < attr->dma_attr_seg) {
1790 		seg = attr->dma_attr_seg;
1791 	}
1792 	count_max = attr->dma_attr_count_max + 1;
1793 	if (count_max < attr->dma_attr_count_max) {
1794 		count_max = attr->dma_attr_count_max;
1795 	}
1796 
1797 	/*
1798 	 * granularity may or may not be a power of two. If it isn't, we can't
1799 	 * use a simple mask.
1800 	 */
1801 	if (!ISP2(attr->dma_attr_granular)) {
1802 		dma->dp_granularity_power_2 = B_FALSE;
1803 	} else {
1804 		dma->dp_granularity_power_2 = B_TRUE;
1805 	}
1806 
1807 	/*
1808 	 * maxxfer should be a whole multiple of granularity. If we're going to
1809 	 * break up a window because we're greater than maxxfer, we might as
1810 	 * well make sure it's maxxfer is a whole multiple so we don't have to
1811 	 * worry about triming the window later on for this case.
1812 	 */
1813 	if (attr->dma_attr_granular > 1) {
1814 		if (dma->dp_granularity_power_2) {
1815 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1816 			    (attr->dma_attr_maxxfer &
1817 			    (attr->dma_attr_granular - 1));
1818 		} else {
1819 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1820 			    (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1821 		}
1822 	} else {
1823 		dma->dp_maxxfer = attr->dma_attr_maxxfer;
1824 	}
1825 
1826 	maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1827 	maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1828 	if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1829 		maxsegmentsize = 0xFFFFFFFF;
1830 	} else {
1831 		maxsegmentsize = maxsegmentsize_ll;
1832 	}
1833 	dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1834 	dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1835 
1836 	/* check the ddi_dma_attr arg to make sure it makes a little sense */
1837 	if (rootnex_alloc_check_parms) {
1838 		e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1839 		if (e != DDI_SUCCESS) {
1840 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1841 			(void) rootnex_dma_freehdl(dip, rdip,
1842 			    (ddi_dma_handle_t)hp);
1843 			return (e);
1844 		}
1845 	}
1846 
1847 	*handlep = (ddi_dma_handle_t)hp;
1848 
1849 	ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1850 	ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1851 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1852 
1853 	return (DDI_SUCCESS);
1854 }
1855 
1856 
1857 /*
1858  * rootnex_dma_allochdl()
1859  *    called from ddi_dma_alloc_handle().
1860  */
1861 static int
rootnex_dma_allochdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_attr_t * attr,int (* waitfp)(caddr_t),caddr_t arg,ddi_dma_handle_t * handlep)1862 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1863     int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1864 {
1865 	int retval = DDI_SUCCESS;
1866 #if !defined(__xpv)
1867 
1868 	if (IOMMU_UNITIALIZED(rdip)) {
1869 		retval = iommulib_nex_open(dip, rdip);
1870 
1871 		if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1872 			return (retval);
1873 	}
1874 
1875 	if (IOMMU_UNUSED(rdip)) {
1876 		retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1877 		    handlep);
1878 	} else {
1879 		retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1880 		    waitfp, arg, handlep);
1881 	}
1882 #else
1883 	retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1884 	    handlep);
1885 #endif
1886 	switch (retval) {
1887 	case DDI_DMA_NORESOURCES:
1888 		if (waitfp != DDI_DMA_DONTWAIT) {
1889 			ddi_set_callback(waitfp, arg,
1890 			    &rootnex_state->r_dvma_call_list_id);
1891 		}
1892 		break;
1893 	case DDI_SUCCESS:
1894 		ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1895 		break;
1896 	default:
1897 		break;
1898 	}
1899 	return (retval);
1900 }
1901 
1902 /*ARGSUSED*/
1903 static int
rootnex_coredma_freehdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)1904 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1905     ddi_dma_handle_t handle)
1906 {
1907 	ddi_dma_impl_t *hp;
1908 	rootnex_dma_t *dma;
1909 
1910 
1911 	hp = (ddi_dma_impl_t *)handle;
1912 	dma = (rootnex_dma_t *)hp->dmai_private;
1913 
1914 	/* unbind should have been called first */
1915 	ASSERT(!dma->dp_inuse);
1916 
1917 	mutex_destroy(&dma->dp_mutex);
1918 	kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1919 
1920 	ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1921 	ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1922 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1923 
1924 	return (DDI_SUCCESS);
1925 }
1926 
1927 /*
1928  * rootnex_dma_freehdl()
1929  *    called from ddi_dma_free_handle().
1930  */
1931 static int
rootnex_dma_freehdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)1932 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1933 {
1934 	int ret;
1935 
1936 	ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1937 #if !defined(__xpv)
1938 	if (IOMMU_USED(rdip))
1939 		ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1940 	else
1941 #endif
1942 	ret = rootnex_coredma_freehdl(dip, rdip, handle);
1943 
1944 	if (rootnex_state->r_dvma_call_list_id)
1945 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1946 
1947 	return (ret);
1948 }
1949 
1950 /*ARGSUSED*/
1951 static int
rootnex_coredma_bindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,struct ddi_dma_req * dmareq,ddi_dma_cookie_t * cookiep,uint_t * ccountp)1952 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1953     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1954     ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1955 {
1956 	rootnex_sglinfo_t *sinfo;
1957 	ddi_dma_obj_t *dmao;
1958 #if !defined(__xpv)
1959 	struct dvmaseg *dvs;
1960 	ddi_dma_cookie_t *cookie;
1961 #endif
1962 	ddi_dma_attr_t *attr;
1963 	ddi_dma_impl_t *hp;
1964 	rootnex_dma_t *dma;
1965 	int kmflag;
1966 	int e;
1967 	uint_t ncookies;
1968 
1969 	hp = (ddi_dma_impl_t *)handle;
1970 	dma = (rootnex_dma_t *)hp->dmai_private;
1971 	dmao = &dma->dp_dma;
1972 	sinfo = &dma->dp_sglinfo;
1973 	attr = &hp->dmai_attr;
1974 
1975 	/* convert the sleep flags */
1976 	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1977 		dma->dp_sleep_flags = kmflag = KM_SLEEP;
1978 	} else {
1979 		dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1980 	}
1981 
1982 	hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1983 
1984 	/*
1985 	 * This is useful for debugging a driver. Not as useful in a production
1986 	 * system. The only time this will fail is if you have a driver bug.
1987 	 */
1988 	if (rootnex_bind_check_inuse) {
1989 		/*
1990 		 * No one else should ever have this lock unless someone else
1991 		 * is trying to use this handle. So contention on the lock
1992 		 * is the same as inuse being set.
1993 		 */
1994 		e = mutex_tryenter(&dma->dp_mutex);
1995 		if (e == 0) {
1996 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1997 			return (DDI_DMA_INUSE);
1998 		}
1999 		if (dma->dp_inuse) {
2000 			mutex_exit(&dma->dp_mutex);
2001 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2002 			return (DDI_DMA_INUSE);
2003 		}
2004 		dma->dp_inuse = B_TRUE;
2005 		mutex_exit(&dma->dp_mutex);
2006 	}
2007 
2008 	/* save away the original bind info */
2009 	dma->dp_dma = dmareq->dmar_object;
2010 
2011 #if !defined(__xpv)
2012 	if (IOMMU_USED(rdip)) {
2013 		dmao = &dma->dp_dvma;
2014 		e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2015 		switch (e) {
2016 		case DDI_SUCCESS:
2017 			if (sinfo->si_cancross ||
2018 			    dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2019 			    dmao->dmao_size > sinfo->si_max_cookie_size) {
2020 				dma->dp_dvma_used = B_TRUE;
2021 				break;
2022 			}
2023 			sinfo->si_sgl_size = 1;
2024 			hp->dmai_rflags |= DMP_NOSYNC;
2025 
2026 			dma->dp_dvma_used = B_TRUE;
2027 			dma->dp_need_to_free_cookie = B_FALSE;
2028 
2029 			dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2030 			cookie = hp->dmai_cookie = dma->dp_cookies =
2031 			    (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2032 			cookie->dmac_laddress = dvs->dvs_start +
2033 			    dmao->dmao_obj.dvma_obj.dv_off;
2034 			cookie->dmac_size = dvs->dvs_len;
2035 			cookie->dmac_type = 0;
2036 
2037 			ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2038 			    rdip);
2039 			goto fast;
2040 		case DDI_ENOTSUP:
2041 			break;
2042 		default:
2043 			rootnex_clean_dmahdl(hp);
2044 			return (e);
2045 		}
2046 	}
2047 #endif
2048 
2049 	/*
2050 	 * Figure out a rough estimate of what maximum number of pages
2051 	 * this buffer could use (a high estimate of course).
2052 	 */
2053 	sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2054 
2055 	if (dma->dp_dvma_used) {
2056 		/*
2057 		 * The number of physical pages is the worst case.
2058 		 *
2059 		 * For DVMA, the worst case is the length divided
2060 		 * by the maximum cookie length, plus 1. Add to that
2061 		 * the number of segment boundaries potentially crossed, and
2062 		 * the additional number of DVMA segments that was returned.
2063 		 *
2064 		 * In the normal case, for modern devices, si_cancross will
2065 		 * be false, and dv_nseg will be 1, and the fast path will
2066 		 * have been taken above.
2067 		 */
2068 		ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2069 		    + 1;
2070 		if (sinfo->si_cancross)
2071 			ncookies +=
2072 			    (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2073 		ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2074 
2075 		sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2076 	}
2077 
2078 	/*
2079 	 * We'll use the pre-allocated cookies for any bind that will *always*
2080 	 * fit (more important to be consistent, we don't want to create
2081 	 * additional degenerate cases).
2082 	 */
2083 	if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2084 		dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2085 		dma->dp_need_to_free_cookie = B_FALSE;
2086 		ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2087 		    uint_t, sinfo->si_max_pages);
2088 
2089 	/*
2090 	 * For anything larger than that, we'll go ahead and allocate the
2091 	 * maximum number of pages we expect to see. Hopefuly, we won't be
2092 	 * seeing this path in the fast path for high performance devices very
2093 	 * frequently.
2094 	 *
2095 	 * a ddi bind interface that allowed the driver to provide storage to
2096 	 * the bind interface would speed this case up.
2097 	 */
2098 	} else {
2099 		/*
2100 		 * Save away how much memory we allocated. If we're doing a
2101 		 * nosleep, the alloc could fail...
2102 		 */
2103 		dma->dp_cookie_size = sinfo->si_max_pages *
2104 		    sizeof (ddi_dma_cookie_t);
2105 		dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2106 		if (dma->dp_cookies == NULL) {
2107 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2108 			rootnex_clean_dmahdl(hp);
2109 			return (DDI_DMA_NORESOURCES);
2110 		}
2111 		dma->dp_need_to_free_cookie = B_TRUE;
2112 		ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2113 		    uint_t, sinfo->si_max_pages);
2114 	}
2115 	hp->dmai_cookie = dma->dp_cookies;
2116 
2117 	/*
2118 	 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2119 	 * looking at the constraints in the dma structure. It will then put
2120 	 * some additional state about the sgl in the dma struct (i.e. is
2121 	 * the sgl clean, or do we need to do some munging; how many pages
2122 	 * need to be copied, etc.)
2123 	 */
2124 	if (dma->dp_dvma_used)
2125 		rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2126 	else
2127 		rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2128 
2129 out:
2130 	ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2131 	/* if we don't need a copy buffer, we don't need to sync */
2132 	if (sinfo->si_copybuf_req == 0) {
2133 		hp->dmai_rflags |= DMP_NOSYNC;
2134 	}
2135 
2136 	/*
2137 	 * if we don't need the copybuf and we don't need to do a partial,  we
2138 	 * hit the fast path. All the high performance devices should be trying
2139 	 * to hit this path. To hit this path, a device should be able to reach
2140 	 * all of memory, shouldn't try to bind more than it can transfer, and
2141 	 * the buffer shouldn't require more cookies than the driver/device can
2142 	 * handle [sgllen]).
2143 	 *
2144 	 * Note that negative values of dma_attr_sgllen are supposed
2145 	 * to mean unlimited, but we just cast them to mean a
2146 	 * "ridiculous large limit".  This saves some extra checks on
2147 	 * hot paths.
2148 	 */
2149 	if ((sinfo->si_copybuf_req == 0) &&
2150 	    (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2151 	    (dmao->dmao_size <= dma->dp_maxxfer)) {
2152 fast:
2153 		/*
2154 		 * If the driver supports FMA, insert the handle in the FMA DMA
2155 		 * handle cache.
2156 		 */
2157 		if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2158 			hp->dmai_error.err_cf = rootnex_dma_check;
2159 
2160 		/*
2161 		 * copy out the first cookie and ccountp, set the cookie
2162 		 * pointer to the second cookie. The first cookie is passed
2163 		 * back on the stack. Additional cookies are accessed via
2164 		 * ddi_dma_nextcookie()
2165 		 */
2166 		*cookiep = dma->dp_cookies[0];
2167 		*ccountp = sinfo->si_sgl_size;
2168 		hp->dmai_cookie++;
2169 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2170 		hp->dmai_ncookies = *ccountp;
2171 		hp->dmai_curcookie = 1;
2172 		ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2173 		ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2174 		    uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2175 		    uint_t, dmao->dmao_size, uint_t, *ccountp);
2176 
2177 
2178 		return (DDI_DMA_MAPPED);
2179 	}
2180 
2181 	/*
2182 	 * go to the slow path, we may need to alloc more memory, create
2183 	 * multiple windows, and munge up a sgl to make the device happy.
2184 	 */
2185 
2186 	/*
2187 	 * With the IOMMU mapobject method used, we should never hit
2188 	 * the slow path. If we do, something is seriously wrong.
2189 	 * Clean up and return an error.
2190 	 */
2191 
2192 #if !defined(__xpv)
2193 
2194 	if (dma->dp_dvma_used) {
2195 		(void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2196 		    &dma->dp_dvma);
2197 		e = DDI_DMA_NOMAPPING;
2198 	} else {
2199 #endif
2200 		e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2201 		    kmflag);
2202 #if !defined(__xpv)
2203 	}
2204 #endif
2205 	if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2206 		if (dma->dp_need_to_free_cookie) {
2207 			kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2208 		}
2209 		ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2210 		rootnex_clean_dmahdl(hp); /* must be after free cookie */
2211 		return (e);
2212 	}
2213 
2214 	/*
2215 	 * If the driver supports FMA, insert the handle in the FMA DMA handle
2216 	 * cache.
2217 	 */
2218 	if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2219 		hp->dmai_error.err_cf = rootnex_dma_check;
2220 
2221 	/* if the first window uses the copy buffer, sync it for the device */
2222 	if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2223 	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
2224 		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2225 		    DDI_DMA_SYNC_FORDEV);
2226 	}
2227 
2228 	/*
2229 	 * copy out the first cookie and ccountp, set the cookie pointer to the
2230 	 * second cookie. Make sure the partial flag is set/cleared correctly.
2231 	 * If we have a partial map (i.e. multiple windows), the number of
2232 	 * cookies we return is the number of cookies in the first window.
2233 	 */
2234 	if (e == DDI_DMA_MAPPED) {
2235 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2236 		*ccountp = sinfo->si_sgl_size;
2237 		hp->dmai_nwin = 1;
2238 	} else {
2239 		hp->dmai_rflags |= DDI_DMA_PARTIAL;
2240 		*ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2241 		ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2242 	}
2243 	*cookiep = dma->dp_cookies[0];
2244 	hp->dmai_cookie++;
2245 	hp->dmai_ncookies = *ccountp;
2246 	hp->dmai_curcookie = 1;
2247 
2248 	ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2249 	ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2250 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2251 	    dmao->dmao_size, uint_t, *ccountp);
2252 	return (e);
2253 }
2254 
2255 /*
2256  * rootnex_dma_bindhdl()
2257  *    called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2258  */
2259 static int
rootnex_dma_bindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,struct ddi_dma_req * dmareq,ddi_dma_cookie_t * cookiep,uint_t * ccountp)2260 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2261     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2262     ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2263 {
2264 	int ret;
2265 #if !defined(__xpv)
2266 	if (IOMMU_USED(rdip))
2267 		ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2268 		    cookiep, ccountp);
2269 	else
2270 #endif
2271 	ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2272 	    cookiep, ccountp);
2273 
2274 	if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2275 		ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2276 		    &rootnex_state->r_dvma_call_list_id);
2277 	}
2278 
2279 	return (ret);
2280 }
2281 
2282 
2283 
2284 /*ARGSUSED*/
2285 static int
rootnex_coredma_unbindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)2286 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2287     ddi_dma_handle_t handle)
2288 {
2289 	ddi_dma_impl_t *hp;
2290 	rootnex_dma_t *dma;
2291 	int e;
2292 
2293 	hp = (ddi_dma_impl_t *)handle;
2294 	dma = (rootnex_dma_t *)hp->dmai_private;
2295 
2296 	/* make sure the buffer wasn't free'd before calling unbind */
2297 	if (rootnex_unbind_verify_buffer) {
2298 		e = rootnex_verify_buffer(dma);
2299 		if (e != DDI_SUCCESS) {
2300 			ASSERT(0);
2301 			return (DDI_FAILURE);
2302 		}
2303 	}
2304 
2305 	/* sync the current window before unbinding the buffer */
2306 	if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2307 	    (hp->dmai_rflags & DDI_DMA_READ)) {
2308 		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2309 		    DDI_DMA_SYNC_FORCPU);
2310 	}
2311 
2312 	/*
2313 	 * cleanup and copy buffer or window state. if we didn't use the copy
2314 	 * buffer or windows, there won't be much to do :-)
2315 	 */
2316 	rootnex_teardown_copybuf(dma);
2317 	rootnex_teardown_windows(dma);
2318 
2319 #if !defined(__xpv)
2320 	if (IOMMU_USED(rdip) && dma->dp_dvma_used)
2321 		(void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2322 		    &dma->dp_dvma);
2323 #endif
2324 
2325 	/*
2326 	 * If we had to allocate space to for the worse case sgl (it didn't
2327 	 * fit into our pre-allocate buffer), free that up now
2328 	 */
2329 	if (dma->dp_need_to_free_cookie) {
2330 		kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2331 	}
2332 
2333 	/*
2334 	 * clean up the handle so it's ready for the next bind (i.e. if the
2335 	 * handle is reused).
2336 	 */
2337 	rootnex_clean_dmahdl(hp);
2338 	hp->dmai_error.err_cf = NULL;
2339 
2340 	ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2341 	ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2342 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2343 
2344 	return (DDI_SUCCESS);
2345 }
2346 
2347 /*
2348  * rootnex_dma_unbindhdl()
2349  *    called from ddi_dma_unbind_handle()
2350  */
2351 /*ARGSUSED*/
2352 static int
rootnex_dma_unbindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)2353 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2354     ddi_dma_handle_t handle)
2355 {
2356 	int ret;
2357 
2358 #if !defined(__xpv)
2359 	if (IOMMU_USED(rdip))
2360 		ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2361 	else
2362 #endif
2363 	ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2364 
2365 	if (rootnex_state->r_dvma_call_list_id)
2366 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2367 
2368 	return (ret);
2369 }
2370 
2371 #if !defined(__xpv)
2372 
2373 static int
rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)2374 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2375 {
2376 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2377 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2378 
2379 	if (dma->dp_sleep_flags != KM_SLEEP &&
2380 	    dma->dp_sleep_flags != KM_NOSLEEP)
2381 		cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2382 	return (dma->dp_sleep_flags);
2383 }
2384 /*ARGSUSED*/
2385 static void
rootnex_coredma_reset_cookies(dev_info_t * dip,ddi_dma_handle_t handle)2386 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2387 {
2388 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2389 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2390 	rootnex_window_t *window;
2391 
2392 	if (dma->dp_window) {
2393 		window = &dma->dp_window[dma->dp_current_win];
2394 		hp->dmai_cookie = window->wd_first_cookie;
2395 	} else {
2396 		hp->dmai_cookie = dma->dp_cookies;
2397 	}
2398 	hp->dmai_cookie++;
2399 	hp->dmai_curcookie = 1;
2400 }
2401 
2402 /*ARGSUSED*/
2403 static int
rootnex_coredma_get_cookies(dev_info_t * dip,ddi_dma_handle_t handle,ddi_dma_cookie_t ** cookiepp,uint_t * ccountp)2404 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2405     ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2406 {
2407 	int i;
2408 	int km_flags;
2409 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2410 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2411 	rootnex_window_t *window;
2412 	ddi_dma_cookie_t *cp;
2413 	ddi_dma_cookie_t *cookie;
2414 
2415 	ASSERT(*cookiepp == NULL);
2416 	ASSERT(*ccountp == 0);
2417 
2418 	if (dma->dp_window) {
2419 		window = &dma->dp_window[dma->dp_current_win];
2420 		cp = window->wd_first_cookie;
2421 		*ccountp = window->wd_cookie_cnt;
2422 	} else {
2423 		cp = dma->dp_cookies;
2424 		*ccountp = dma->dp_sglinfo.si_sgl_size;
2425 	}
2426 
2427 	km_flags = rootnex_coredma_get_sleep_flags(handle);
2428 	cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2429 	if (cookie == NULL) {
2430 		return (DDI_DMA_NORESOURCES);
2431 	}
2432 
2433 	for (i = 0; i < *ccountp; i++) {
2434 		cookie[i].dmac_notused = cp[i].dmac_notused;
2435 		cookie[i].dmac_type = cp[i].dmac_type;
2436 		cookie[i].dmac_address = cp[i].dmac_address;
2437 		cookie[i].dmac_size = cp[i].dmac_size;
2438 	}
2439 
2440 	*cookiepp = cookie;
2441 
2442 	return (DDI_SUCCESS);
2443 }
2444 
2445 /*ARGSUSED*/
2446 static int
rootnex_coredma_set_cookies(dev_info_t * dip,ddi_dma_handle_t handle,ddi_dma_cookie_t * cookiep,uint_t ccount)2447 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2448     ddi_dma_cookie_t *cookiep, uint_t ccount)
2449 {
2450 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2451 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2452 	rootnex_window_t *window;
2453 	ddi_dma_cookie_t *cur_cookiep;
2454 
2455 	ASSERT(cookiep);
2456 	ASSERT(ccount != 0);
2457 	ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2458 
2459 	if (dma->dp_window) {
2460 		window = &dma->dp_window[dma->dp_current_win];
2461 		dma->dp_saved_cookies = window->wd_first_cookie;
2462 		window->wd_first_cookie = cookiep;
2463 		ASSERT(ccount == window->wd_cookie_cnt);
2464 		cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2465 		    + window->wd_first_cookie;
2466 	} else {
2467 		dma->dp_saved_cookies = dma->dp_cookies;
2468 		dma->dp_cookies = cookiep;
2469 		ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2470 		cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2471 		    + dma->dp_cookies;
2472 	}
2473 
2474 	dma->dp_need_to_switch_cookies = B_TRUE;
2475 	hp->dmai_cookie = cur_cookiep;
2476 
2477 	return (DDI_SUCCESS);
2478 }
2479 
2480 /*ARGSUSED*/
2481 static int
rootnex_coredma_clear_cookies(dev_info_t * dip,ddi_dma_handle_t handle)2482 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2483 {
2484 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2485 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2486 	rootnex_window_t *window;
2487 	ddi_dma_cookie_t *cur_cookiep;
2488 	ddi_dma_cookie_t *cookie_array;
2489 	uint_t ccount;
2490 
2491 	/* check if cookies have not been switched */
2492 	if (dma->dp_need_to_switch_cookies == B_FALSE)
2493 		return (DDI_SUCCESS);
2494 
2495 	ASSERT(dma->dp_saved_cookies);
2496 
2497 	if (dma->dp_window) {
2498 		window = &dma->dp_window[dma->dp_current_win];
2499 		cookie_array = window->wd_first_cookie;
2500 		window->wd_first_cookie = dma->dp_saved_cookies;
2501 		dma->dp_saved_cookies = NULL;
2502 		ccount = window->wd_cookie_cnt;
2503 		cur_cookiep = (hp->dmai_cookie - cookie_array)
2504 		    + window->wd_first_cookie;
2505 	} else {
2506 		cookie_array = dma->dp_cookies;
2507 		dma->dp_cookies = dma->dp_saved_cookies;
2508 		dma->dp_saved_cookies = NULL;
2509 		ccount = dma->dp_sglinfo.si_sgl_size;
2510 		cur_cookiep = (hp->dmai_cookie - cookie_array)
2511 		    + dma->dp_cookies;
2512 	}
2513 
2514 	kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2515 
2516 	hp->dmai_cookie = cur_cookiep;
2517 
2518 	dma->dp_need_to_switch_cookies = B_FALSE;
2519 
2520 	return (DDI_SUCCESS);
2521 }
2522 
2523 #endif
2524 
2525 static struct as *
rootnex_get_as(ddi_dma_obj_t * dmao)2526 rootnex_get_as(ddi_dma_obj_t *dmao)
2527 {
2528 	struct as *asp;
2529 
2530 	switch (dmao->dmao_type) {
2531 	case DMA_OTYP_VADDR:
2532 	case DMA_OTYP_BUFVADDR:
2533 		asp = dmao->dmao_obj.virt_obj.v_as;
2534 		if (asp == NULL)
2535 			asp = &kas;
2536 		break;
2537 	default:
2538 		asp = NULL;
2539 		break;
2540 	}
2541 	return (asp);
2542 }
2543 
2544 /*
2545  * rootnex_verify_buffer()
2546  *   verify buffer wasn't free'd
2547  */
2548 static int
rootnex_verify_buffer(rootnex_dma_t * dma)2549 rootnex_verify_buffer(rootnex_dma_t *dma)
2550 {
2551 	page_t **pplist;
2552 	caddr_t vaddr;
2553 	uint_t pcnt;
2554 	uint_t poff;
2555 	page_t *pp;
2556 	char b;
2557 	int i;
2558 
2559 	/* Figure out how many pages this buffer occupies */
2560 	if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2561 		poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2562 	} else {
2563 		vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2564 		poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2565 	}
2566 	pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2567 
2568 	switch (dma->dp_dma.dmao_type) {
2569 	case DMA_OTYP_PAGES:
2570 		/*
2571 		 * for a linked list of pp's walk through them to make sure
2572 		 * they're locked and not free.
2573 		 */
2574 		pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2575 		for (i = 0; i < pcnt; i++) {
2576 			if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2577 				return (DDI_FAILURE);
2578 			}
2579 			pp = pp->p_next;
2580 		}
2581 		break;
2582 
2583 	case DMA_OTYP_VADDR:
2584 	case DMA_OTYP_BUFVADDR:
2585 		pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2586 		/*
2587 		 * for an array of pp's walk through them to make sure they're
2588 		 * not free. It's possible that they may not be locked.
2589 		 */
2590 		if (pplist) {
2591 			for (i = 0; i < pcnt; i++) {
2592 				if (PP_ISFREE(pplist[i])) {
2593 					return (DDI_FAILURE);
2594 				}
2595 			}
2596 
2597 		/* For a virtual address, try to peek at each page */
2598 		} else {
2599 			if (rootnex_get_as(&dma->dp_dma) == &kas) {
2600 				for (i = 0; i < pcnt; i++) {
2601 					if (ddi_peek8(NULL, vaddr, &b) ==
2602 					    DDI_FAILURE)
2603 						return (DDI_FAILURE);
2604 					vaddr += MMU_PAGESIZE;
2605 				}
2606 			}
2607 		}
2608 		break;
2609 
2610 	default:
2611 		cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2612 		break;
2613 	}
2614 
2615 	return (DDI_SUCCESS);
2616 }
2617 
2618 
2619 /*
2620  * rootnex_clean_dmahdl()
2621  *    Clean the dma handle. This should be called on a handle alloc and an
2622  *    unbind handle. Set the handle state to the default settings.
2623  */
2624 static void
rootnex_clean_dmahdl(ddi_dma_impl_t * hp)2625 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2626 {
2627 	rootnex_dma_t *dma;
2628 
2629 
2630 	dma = (rootnex_dma_t *)hp->dmai_private;
2631 
2632 	hp->dmai_nwin = 0;
2633 	dma->dp_current_cookie = 0;
2634 	dma->dp_copybuf_size = 0;
2635 	dma->dp_window = NULL;
2636 	dma->dp_cbaddr = NULL;
2637 	dma->dp_inuse = B_FALSE;
2638 	dma->dp_dvma_used = B_FALSE;
2639 	dma->dp_need_to_free_cookie = B_FALSE;
2640 	dma->dp_need_to_switch_cookies = B_FALSE;
2641 	dma->dp_saved_cookies = NULL;
2642 	dma->dp_sleep_flags = KM_PANIC;
2643 	dma->dp_need_to_free_window = B_FALSE;
2644 	dma->dp_partial_required = B_FALSE;
2645 	dma->dp_trim_required = B_FALSE;
2646 	dma->dp_sglinfo.si_copybuf_req = 0;
2647 
2648 	/* FMA related initialization */
2649 	hp->dmai_fault = 0;
2650 	hp->dmai_fault_check = NULL;
2651 	hp->dmai_fault_notify = NULL;
2652 	hp->dmai_error.err_ena = 0;
2653 	hp->dmai_error.err_status = DDI_FM_OK;
2654 	hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2655 	hp->dmai_error.err_ontrap = NULL;
2656 
2657 	/* Cookie tracking */
2658 	hp->dmai_ncookies = 0;
2659 	hp->dmai_curcookie = 0;
2660 }
2661 
2662 
2663 /*
2664  * rootnex_valid_alloc_parms()
2665  *    Called in ddi_dma_alloc_handle path to validate its parameters.
2666  */
2667 static int
rootnex_valid_alloc_parms(ddi_dma_attr_t * attr,uint_t maxsegmentsize)2668 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2669 {
2670 	if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2671 	    (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2672 	    (attr->dma_attr_granular > MMU_PAGESIZE) ||
2673 	    (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2674 		return (DDI_DMA_BADATTR);
2675 	}
2676 
2677 	if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2678 		return (DDI_DMA_BADATTR);
2679 	}
2680 
2681 	if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2682 	    MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2683 	    attr->dma_attr_sgllen == 0) {
2684 		return (DDI_DMA_BADATTR);
2685 	}
2686 
2687 	/* We should be able to DMA into every byte offset in a page */
2688 	if (maxsegmentsize < MMU_PAGESIZE) {
2689 		return (DDI_DMA_BADATTR);
2690 	}
2691 
2692 	/* if we're bouncing on seg, seg must be <= addr_hi */
2693 	if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2694 	    (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2695 		return (DDI_DMA_BADATTR);
2696 	}
2697 	return (DDI_SUCCESS);
2698 }
2699 
2700 /*
2701  * rootnex_need_bounce_seg()
2702  *    check to see if the buffer lives on both side of the seg.
2703  */
2704 static boolean_t
rootnex_need_bounce_seg(ddi_dma_obj_t * dmar_object,rootnex_sglinfo_t * sglinfo)2705 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2706 {
2707 	ddi_dma_atyp_t buftype;
2708 	rootnex_addr_t raddr;
2709 	boolean_t lower_addr;
2710 	boolean_t upper_addr;
2711 	uint64_t offset;
2712 	page_t **pplist;
2713 	uint64_t paddr;
2714 	uint32_t psize;
2715 	uint32_t size;
2716 	caddr_t vaddr;
2717 	uint_t pcnt;
2718 	page_t *pp;
2719 
2720 	pp = NULL;
2721 	/* shortcuts */
2722 	pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2723 	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2724 	buftype = dmar_object->dmao_type;
2725 	size = dmar_object->dmao_size;
2726 
2727 	lower_addr = B_FALSE;
2728 	upper_addr = B_FALSE;
2729 	pcnt = 0;
2730 
2731 	/*
2732 	 * Process the first page to handle the initial offset of the buffer.
2733 	 * We'll use the base address we get later when we loop through all
2734 	 * the pages.
2735 	 */
2736 	if (buftype == DMA_OTYP_PAGES) {
2737 		pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2738 		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2739 		    MMU_PAGEOFFSET;
2740 		paddr = pfn_to_pa(pp->p_pagenum) + offset;
2741 		psize = MIN(size, (MMU_PAGESIZE - offset));
2742 		pp = pp->p_next;
2743 		sglinfo->si_asp = NULL;
2744 	} else if (pplist != NULL) {
2745 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2746 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2747 		if (sglinfo->si_asp == NULL) {
2748 			sglinfo->si_asp = &kas;
2749 		}
2750 		paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2751 		paddr += offset;
2752 		psize = MIN(size, (MMU_PAGESIZE - offset));
2753 		pcnt++;
2754 	} else {
2755 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2756 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2757 		if (sglinfo->si_asp == NULL) {
2758 			sglinfo->si_asp = &kas;
2759 		}
2760 		paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2761 		paddr += offset;
2762 		psize = MIN(size, (MMU_PAGESIZE - offset));
2763 		vaddr += psize;
2764 	}
2765 
2766 	raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2767 
2768 	if ((raddr + psize) > sglinfo->si_segmask) {
2769 		upper_addr = B_TRUE;
2770 	} else {
2771 		lower_addr = B_TRUE;
2772 	}
2773 	size -= psize;
2774 
2775 	/*
2776 	 * Walk through the rest of the pages in the buffer. Track to see
2777 	 * if we have pages on both sides of the segment boundary.
2778 	 */
2779 	while (size > 0) {
2780 		/* partial or full page */
2781 		psize = MIN(size, MMU_PAGESIZE);
2782 
2783 		if (buftype == DMA_OTYP_PAGES) {
2784 			/* get the paddr from the page_t */
2785 			ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2786 			paddr = pfn_to_pa(pp->p_pagenum);
2787 			pp = pp->p_next;
2788 		} else if (pplist != NULL) {
2789 			/* index into the array of page_t's to get the paddr */
2790 			ASSERT(!PP_ISFREE(pplist[pcnt]));
2791 			paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2792 			pcnt++;
2793 		} else {
2794 			/* call into the VM to get the paddr */
2795 			paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2796 			    vaddr));
2797 			vaddr += psize;
2798 		}
2799 
2800 		raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2801 
2802 		if ((raddr + psize) > sglinfo->si_segmask) {
2803 			upper_addr = B_TRUE;
2804 		} else {
2805 			lower_addr = B_TRUE;
2806 		}
2807 		/*
2808 		 * if the buffer lives both above and below the segment
2809 		 * boundary, or the current page is the page immediately
2810 		 * after the segment, we will use a copy/bounce buffer for
2811 		 * all pages > seg.
2812 		 */
2813 		if ((lower_addr && upper_addr) ||
2814 		    (raddr == (sglinfo->si_segmask + 1))) {
2815 			return (B_TRUE);
2816 		}
2817 
2818 		size -= psize;
2819 	}
2820 
2821 	return (B_FALSE);
2822 }
2823 
2824 /*
2825  * rootnex_get_sgl()
2826  *    Called in bind fastpath to get the sgl. Most of this will be replaced
2827  *    with a call to the vm layer when vm2.0 comes around...
2828  */
2829 static void
rootnex_get_sgl(ddi_dma_obj_t * dmar_object,ddi_dma_cookie_t * sgl,rootnex_sglinfo_t * sglinfo)2830 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2831     rootnex_sglinfo_t *sglinfo)
2832 {
2833 	ddi_dma_atyp_t buftype;
2834 	rootnex_addr_t raddr;
2835 	uint64_t last_page;
2836 	uint64_t offset;
2837 	uint64_t addrhi;
2838 	uint64_t addrlo;
2839 	uint64_t maxseg;
2840 	page_t **pplist;
2841 	uint64_t paddr;
2842 	uint32_t psize;
2843 	uint32_t size;
2844 	caddr_t vaddr;
2845 	uint_t pcnt;
2846 	page_t *pp;
2847 	uint_t cnt;
2848 
2849 	pp = NULL;
2850 	/* shortcuts */
2851 	pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2852 	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2853 	maxseg = sglinfo->si_max_cookie_size;
2854 	buftype = dmar_object->dmao_type;
2855 	addrhi = sglinfo->si_max_addr;
2856 	addrlo = sglinfo->si_min_addr;
2857 	size = dmar_object->dmao_size;
2858 
2859 	pcnt = 0;
2860 	cnt = 0;
2861 
2862 
2863 	/*
2864 	 * check to see if we need to use the copy buffer for pages over
2865 	 * the segment attr.
2866 	 */
2867 	sglinfo->si_bounce_on_seg = B_FALSE;
2868 	if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2869 		sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2870 		    dmar_object, sglinfo);
2871 	}
2872 
2873 	/*
2874 	 * if we were passed down a linked list of pages, i.e. pointer to
2875 	 * page_t, use this to get our physical address and buf offset.
2876 	 */
2877 	if (buftype == DMA_OTYP_PAGES) {
2878 		pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2879 		ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2880 		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2881 		    MMU_PAGEOFFSET;
2882 		paddr = pfn_to_pa(pp->p_pagenum) + offset;
2883 		psize = MIN(size, (MMU_PAGESIZE - offset));
2884 		pp = pp->p_next;
2885 		sglinfo->si_asp = NULL;
2886 
2887 	/*
2888 	 * We weren't passed down a linked list of pages, but if we were passed
2889 	 * down an array of pages, use this to get our physical address and buf
2890 	 * offset.
2891 	 */
2892 	} else if (pplist != NULL) {
2893 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2894 		    (buftype == DMA_OTYP_BUFVADDR));
2895 
2896 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2897 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2898 		if (sglinfo->si_asp == NULL) {
2899 			sglinfo->si_asp = &kas;
2900 		}
2901 
2902 		ASSERT(!PP_ISFREE(pplist[pcnt]));
2903 		paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2904 		paddr += offset;
2905 		psize = MIN(size, (MMU_PAGESIZE - offset));
2906 		pcnt++;
2907 
2908 	/*
2909 	 * All we have is a virtual address, we'll need to call into the VM
2910 	 * to get the physical address.
2911 	 */
2912 	} else {
2913 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2914 		    (buftype == DMA_OTYP_BUFVADDR));
2915 
2916 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2917 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2918 		if (sglinfo->si_asp == NULL) {
2919 			sglinfo->si_asp = &kas;
2920 		}
2921 
2922 		paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2923 		paddr += offset;
2924 		psize = MIN(size, (MMU_PAGESIZE - offset));
2925 		vaddr += psize;
2926 	}
2927 
2928 	raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2929 
2930 	/*
2931 	 * Setup the first cookie with the physical address of the page and the
2932 	 * size of the page (which takes into account the initial offset into
2933 	 * the page.
2934 	 */
2935 	sgl[cnt].dmac_laddress = raddr;
2936 	sgl[cnt].dmac_size = psize;
2937 	sgl[cnt].dmac_type = 0;
2938 
2939 	/*
2940 	 * Save away the buffer offset into the page. We'll need this later in
2941 	 * the copy buffer code to help figure out the page index within the
2942 	 * buffer and the offset into the current page.
2943 	 */
2944 	sglinfo->si_buf_offset = offset;
2945 
2946 	/*
2947 	 * If we are using the copy buffer for anything over the segment
2948 	 * boundary, and this page is over the segment boundary.
2949 	 *   OR
2950 	 * if the DMA engine can't reach the physical address.
2951 	 */
2952 	if (((sglinfo->si_bounce_on_seg) &&
2953 	    ((raddr + psize) > sglinfo->si_segmask)) ||
2954 	    ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
2955 		/*
2956 		 * Increase how much copy buffer we use. We always increase by
2957 		 * pagesize so we don't have to worry about converting offsets.
2958 		 * Set a flag in the cookies dmac_type to indicate that it uses
2959 		 * the copy buffer. If this isn't the last cookie, go to the
2960 		 * next cookie (since we separate each page which uses the copy
2961 		 * buffer in case the copy buffer is not physically contiguous.
2962 		 */
2963 		sglinfo->si_copybuf_req += MMU_PAGESIZE;
2964 		sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
2965 		if ((cnt + 1) < sglinfo->si_max_pages) {
2966 			cnt++;
2967 			sgl[cnt].dmac_laddress = 0;
2968 			sgl[cnt].dmac_size = 0;
2969 			sgl[cnt].dmac_type = 0;
2970 		}
2971 	}
2972 
2973 	/*
2974 	 * save this page's physical address so we can figure out if the next
2975 	 * page is physically contiguous. Keep decrementing size until we are
2976 	 * done with the buffer.
2977 	 */
2978 	last_page = raddr & MMU_PAGEMASK;
2979 	size -= psize;
2980 
2981 	while (size > 0) {
2982 		/* Get the size for this page (i.e. partial or full page) */
2983 		psize = MIN(size, MMU_PAGESIZE);
2984 
2985 		if (buftype == DMA_OTYP_PAGES) {
2986 			/* get the paddr from the page_t */
2987 			ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2988 			paddr = pfn_to_pa(pp->p_pagenum);
2989 			pp = pp->p_next;
2990 		} else if (pplist != NULL) {
2991 			/* index into the array of page_t's to get the paddr */
2992 			ASSERT(!PP_ISFREE(pplist[pcnt]));
2993 			paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2994 			pcnt++;
2995 		} else {
2996 			/* call into the VM to get the paddr */
2997 			paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2998 			    vaddr));
2999 			vaddr += psize;
3000 		}
3001 
3002 		raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3003 
3004 		/*
3005 		 * If we are using the copy buffer for anything over the
3006 		 * segment boundary, and this page is over the segment
3007 		 * boundary.
3008 		 *   OR
3009 		 * if the DMA engine can't reach the physical address.
3010 		 */
3011 		if (((sglinfo->si_bounce_on_seg) &&
3012 		    ((raddr + psize) > sglinfo->si_segmask)) ||
3013 		    ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3014 
3015 			sglinfo->si_copybuf_req += MMU_PAGESIZE;
3016 
3017 			/*
3018 			 * if there is something in the current cookie, go to
3019 			 * the next one. We only want one page in a cookie which
3020 			 * uses the copybuf since the copybuf doesn't have to
3021 			 * be physically contiguous.
3022 			 */
3023 			if (sgl[cnt].dmac_size != 0) {
3024 				cnt++;
3025 			}
3026 			sgl[cnt].dmac_laddress = raddr;
3027 			sgl[cnt].dmac_size = psize;
3028 			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3029 			/* if this isn't the last cookie, go to the next one */
3030 			if ((cnt + 1) < sglinfo->si_max_pages) {
3031 				cnt++;
3032 				sgl[cnt].dmac_laddress = 0;
3033 				sgl[cnt].dmac_size = 0;
3034 				sgl[cnt].dmac_type = 0;
3035 			}
3036 
3037 		/*
3038 		 * this page didn't need the copy buffer, if it's not physically
3039 		 * contiguous, or it would put us over a segment boundary, or it
3040 		 * puts us over the max cookie size, or the current sgl doesn't
3041 		 * have anything in it.
3042 		 */
3043 		} else if (((last_page + MMU_PAGESIZE) != raddr) ||
3044 		    !(raddr & sglinfo->si_segmask) ||
3045 		    ((sgl[cnt].dmac_size + psize) > maxseg) ||
3046 		    (sgl[cnt].dmac_size == 0)) {
3047 			/*
3048 			 * if we're not already in a new cookie, go to the next
3049 			 * cookie.
3050 			 */
3051 			if (sgl[cnt].dmac_size != 0) {
3052 				cnt++;
3053 			}
3054 
3055 			/* save the cookie information */
3056 			sgl[cnt].dmac_laddress = raddr;
3057 			sgl[cnt].dmac_size = psize;
3058 			sgl[cnt].dmac_type = 0;
3059 
3060 		/*
3061 		 * this page didn't need the copy buffer, it is physically
3062 		 * contiguous with the last page, and it's <= the max cookie
3063 		 * size.
3064 		 */
3065 		} else {
3066 			sgl[cnt].dmac_size += psize;
3067 
3068 			/*
3069 			 * If this cookie is used up, and more cookies
3070 			 * are available, then move onto the next one.
3071 			 */
3072 			if ((sgl[cnt].dmac_size == maxseg) &&
3073 			    ((cnt + 1) < sglinfo->si_max_pages)) {
3074 				cnt++;
3075 				sgl[cnt].dmac_laddress = 0;
3076 				sgl[cnt].dmac_size = 0;
3077 				sgl[cnt].dmac_type = 0;
3078 			}
3079 		}
3080 
3081 		/*
3082 		 * save this page's physical address so we can figure out if the
3083 		 * next page is physically contiguous. Keep decrementing size
3084 		 * until we are done with the buffer.
3085 		 */
3086 		last_page = raddr;
3087 		size -= psize;
3088 	}
3089 
3090 	/* we're done, save away how many cookies the sgl has */
3091 	if (sgl[cnt].dmac_size == 0) {
3092 		ASSERT(cnt < sglinfo->si_max_pages);
3093 		sglinfo->si_sgl_size = cnt;
3094 	} else {
3095 		sglinfo->si_sgl_size = cnt + 1;
3096 	}
3097 }
3098 
3099 static void
rootnex_dvma_get_sgl(ddi_dma_obj_t * dmar_object,ddi_dma_cookie_t * sgl,rootnex_sglinfo_t * sglinfo)3100 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3101     rootnex_sglinfo_t *sglinfo)
3102 {
3103 	uint64_t offset;
3104 	uint64_t maxseg;
3105 	uint64_t dvaddr;
3106 	struct dvmaseg *dvs;
3107 	uint64_t paddr;
3108 	uint32_t psize, ssize;
3109 	uint32_t size;
3110 	uint_t cnt;
3111 	int physcontig;
3112 
3113 	ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3114 
3115 	/* shortcuts */
3116 	maxseg = sglinfo->si_max_cookie_size;
3117 	size = dmar_object->dmao_size;
3118 
3119 	cnt = 0;
3120 	sglinfo->si_bounce_on_seg = B_FALSE;
3121 
3122 	dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3123 	offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3124 	ssize = dvs->dvs_len;
3125 	paddr = dvs->dvs_start;
3126 	paddr += offset;
3127 	psize = MIN(ssize, (maxseg - offset));
3128 	dvaddr = paddr + psize;
3129 	ssize -= psize;
3130 
3131 	sgl[cnt].dmac_laddress = paddr;
3132 	sgl[cnt].dmac_size = psize;
3133 	sgl[cnt].dmac_type = 0;
3134 
3135 	size -= psize;
3136 	while (size > 0) {
3137 		if (ssize == 0) {
3138 			dvs++;
3139 			ssize = dvs->dvs_len;
3140 			dvaddr = dvs->dvs_start;
3141 			physcontig = 0;
3142 		} else
3143 			physcontig = 1;
3144 
3145 		paddr = dvaddr;
3146 		psize = MIN(ssize, maxseg);
3147 		dvaddr += psize;
3148 		ssize -= psize;
3149 
3150 		if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3151 		    ((sgl[cnt].dmac_size + psize) > maxseg) ||
3152 		    (sgl[cnt].dmac_size == 0)) {
3153 			/*
3154 			 * if we're not already in a new cookie, go to the next
3155 			 * cookie.
3156 			 */
3157 			if (sgl[cnt].dmac_size != 0) {
3158 				cnt++;
3159 			}
3160 
3161 			/* save the cookie information */
3162 			sgl[cnt].dmac_laddress = paddr;
3163 			sgl[cnt].dmac_size = psize;
3164 			sgl[cnt].dmac_type = 0;
3165 		} else {
3166 			sgl[cnt].dmac_size += psize;
3167 
3168 			/*
3169 			 * If this cookie is used up, and more cookies
3170 			 * are available, then move onto the next one.
3171 			 */
3172 			if ((sgl[cnt].dmac_size == maxseg) &&
3173 			    ((cnt + 1) < sglinfo->si_max_pages)) {
3174 				cnt++;
3175 				sgl[cnt].dmac_laddress = 0;
3176 				sgl[cnt].dmac_size = 0;
3177 				sgl[cnt].dmac_type = 0;
3178 			}
3179 		}
3180 		size -= psize;
3181 	}
3182 
3183 	/* we're done, save away how many cookies the sgl has */
3184 	if (sgl[cnt].dmac_size == 0) {
3185 		sglinfo->si_sgl_size = cnt;
3186 	} else {
3187 		sglinfo->si_sgl_size = cnt + 1;
3188 	}
3189 }
3190 
3191 /*
3192  * rootnex_bind_slowpath()
3193  *    Call in the bind path if the calling driver can't use the sgl without
3194  *    modifying it. We either need to use the copy buffer and/or we will end up
3195  *    with a partial bind.
3196  */
3197 static int
rootnex_bind_slowpath(ddi_dma_impl_t * hp,struct ddi_dma_req * dmareq,rootnex_dma_t * dma,ddi_dma_attr_t * attr,ddi_dma_obj_t * dmao,int kmflag)3198 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3199     rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3200 {
3201 	rootnex_sglinfo_t *sinfo;
3202 	rootnex_window_t *window;
3203 	ddi_dma_cookie_t *cookie;
3204 	size_t copybuf_used;
3205 	size_t dmac_size;
3206 	boolean_t partial;
3207 	off_t cur_offset;
3208 	page_t *cur_pp;
3209 	major_t mnum;
3210 	int e;
3211 	int i;
3212 
3213 
3214 	sinfo = &dma->dp_sglinfo;
3215 	copybuf_used = 0;
3216 	partial = B_FALSE;
3217 
3218 	/*
3219 	 * If we're using the copybuf, set the copybuf state in dma struct.
3220 	 * Needs to be first since it sets the copy buffer size.
3221 	 */
3222 	if (sinfo->si_copybuf_req != 0) {
3223 		e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3224 		if (e != DDI_SUCCESS) {
3225 			return (e);
3226 		}
3227 	} else {
3228 		dma->dp_copybuf_size = 0;
3229 	}
3230 
3231 	/*
3232 	 * Figure out if we need to do a partial mapping. If so, figure out
3233 	 * if we need to trim the buffers when we munge the sgl.
3234 	 */
3235 	if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3236 	    (dmao->dmao_size > dma->dp_maxxfer) ||
3237 	    ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3238 		dma->dp_partial_required = B_TRUE;
3239 		if (attr->dma_attr_granular != 1) {
3240 			dma->dp_trim_required = B_TRUE;
3241 		}
3242 	} else {
3243 		dma->dp_partial_required = B_FALSE;
3244 		dma->dp_trim_required = B_FALSE;
3245 	}
3246 
3247 	/* If we need to do a partial bind, make sure the driver supports it */
3248 	if (dma->dp_partial_required &&
3249 	    !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3250 
3251 		mnum = ddi_driver_major(dma->dp_dip);
3252 		/*
3253 		 * patchable which allows us to print one warning per major
3254 		 * number.
3255 		 */
3256 		if ((rootnex_bind_warn) &&
3257 		    ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3258 			rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3259 			cmn_err(CE_WARN, "!%s: coding error detected, the "
3260 			    "driver is using ddi_dma_attr(9S) incorrectly. "
3261 			    "There is a small risk of data corruption in "
3262 			    "particular with large I/Os. The driver should be "
3263 			    "replaced with a corrected version for proper "
3264 			    "system operation. To disable this warning, add "
3265 			    "'set rootnex:rootnex_bind_warn=0' to "
3266 			    "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3267 		}
3268 		return (DDI_DMA_TOOBIG);
3269 	}
3270 
3271 	/*
3272 	 * we might need multiple windows, setup state to handle them. In this
3273 	 * code path, we will have at least one window.
3274 	 */
3275 	e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3276 	if (e != DDI_SUCCESS) {
3277 		rootnex_teardown_copybuf(dma);
3278 		return (e);
3279 	}
3280 
3281 	window = &dma->dp_window[0];
3282 	cookie = &dma->dp_cookies[0];
3283 	cur_offset = 0;
3284 	rootnex_init_win(hp, dma, window, cookie, cur_offset);
3285 	if (dmao->dmao_type == DMA_OTYP_PAGES) {
3286 		cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3287 	}
3288 
3289 	/* loop though all the cookies we got back from get_sgl() */
3290 	for (i = 0; i < sinfo->si_sgl_size; i++) {
3291 		/*
3292 		 * If we're using the copy buffer, check this cookie and setup
3293 		 * its associated copy buffer state. If this cookie uses the
3294 		 * copy buffer, make sure we sync this window during dma_sync.
3295 		 */
3296 		if (dma->dp_copybuf_size > 0) {
3297 			rootnex_setup_cookie(dmao, dma, cookie,
3298 			    cur_offset, &copybuf_used, &cur_pp);
3299 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3300 				window->wd_dosync = B_TRUE;
3301 			}
3302 		}
3303 
3304 		/*
3305 		 * save away the cookie size, since it could be modified in
3306 		 * the windowing code.
3307 		 */
3308 		dmac_size = cookie->dmac_size;
3309 
3310 		/* if we went over max copybuf size */
3311 		if (dma->dp_copybuf_size &&
3312 		    (copybuf_used > dma->dp_copybuf_size)) {
3313 			partial = B_TRUE;
3314 			e = rootnex_copybuf_window_boundary(hp, dma, &window,
3315 			    cookie, cur_offset, &copybuf_used);
3316 			if (e != DDI_SUCCESS) {
3317 				rootnex_teardown_copybuf(dma);
3318 				rootnex_teardown_windows(dma);
3319 				return (e);
3320 			}
3321 
3322 			/*
3323 			 * if the coookie uses the copy buffer, make sure the
3324 			 * new window we just moved to is set to sync.
3325 			 */
3326 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3327 				window->wd_dosync = B_TRUE;
3328 			}
3329 			ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3330 			    dma->dp_dip);
3331 
3332 		/* if the cookie cnt == max sgllen, move to the next window */
3333 		} else if (window->wd_cookie_cnt >=
3334 		    (unsigned)attr->dma_attr_sgllen) {
3335 			partial = B_TRUE;
3336 			ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3337 			e = rootnex_sgllen_window_boundary(hp, dma, &window,
3338 			    cookie, attr, cur_offset);
3339 			if (e != DDI_SUCCESS) {
3340 				rootnex_teardown_copybuf(dma);
3341 				rootnex_teardown_windows(dma);
3342 				return (e);
3343 			}
3344 
3345 			/*
3346 			 * if the coookie uses the copy buffer, make sure the
3347 			 * new window we just moved to is set to sync.
3348 			 */
3349 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3350 				window->wd_dosync = B_TRUE;
3351 			}
3352 			ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3353 			    dma->dp_dip);
3354 
3355 		/* else if we will be over maxxfer */
3356 		} else if ((window->wd_size + dmac_size) >
3357 		    dma->dp_maxxfer) {
3358 			partial = B_TRUE;
3359 			e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3360 			    cookie);
3361 			if (e != DDI_SUCCESS) {
3362 				rootnex_teardown_copybuf(dma);
3363 				rootnex_teardown_windows(dma);
3364 				return (e);
3365 			}
3366 
3367 			/*
3368 			 * if the coookie uses the copy buffer, make sure the
3369 			 * new window we just moved to is set to sync.
3370 			 */
3371 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3372 				window->wd_dosync = B_TRUE;
3373 			}
3374 			ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3375 			    dma->