xref: /illumos-gate/usr/src/uts/i86pc/io/rootnex.c (revision bbf21555)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 /*
25  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
26  * Copyright (c) 2011 Bayard G. Bell.  All rights reserved.
27  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
28  * Copyright 2017 Joyent, Inc.
29  * Copyright 2020 Ryan Zezeski
30  */
31 
32 /*
33  * x86 root nexus driver
34  */
35 
36 #include <sys/sysmacros.h>
37 #include <sys/conf.h>
38 #include <sys/autoconf.h>
39 #include <sys/sysmacros.h>
40 #include <sys/debug.h>
41 #include <sys/psw.h>
42 #include <sys/ddidmareq.h>
43 #include <sys/promif.h>
44 #include <sys/devops.h>
45 #include <sys/kmem.h>
46 #include <sys/cmn_err.h>
47 #include <vm/seg.h>
48 #include <vm/seg_kmem.h>
49 #include <vm/seg_dev.h>
50 #include <sys/vmem.h>
51 #include <sys/mman.h>
52 #include <vm/hat.h>
53 #include <vm/as.h>
54 #include <vm/page.h>
55 #include <sys/avintr.h>
56 #include <sys/errno.h>
57 #include <sys/modctl.h>
58 #include <sys/ddi_impldefs.h>
59 #include <sys/sunddi.h>
60 #include <sys/sunndi.h>
61 #include <sys/mach_intr.h>
62 #include <sys/psm.h>
63 #include <sys/ontrap.h>
64 #include <sys/atomic.h>
65 #include <sys/sdt.h>
66 #include <sys/rootnex.h>
67 #include <vm/hat_i86.h>
68 #include <sys/ddifm.h>
69 #include <sys/ddi_isa.h>
70 #include <sys/apic.h>
71 
72 #ifdef __xpv
73 #include <sys/bootinfo.h>
74 #include <sys/hypervisor.h>
75 #include <sys/bootconf.h>
76 #include <vm/kboot_mmu.h>
77 #endif
78 
79 #if !defined(__xpv)
80 #include <sys/immu.h>
81 #endif
82 
83 
84 /*
85  * enable/disable extra checking of function parameters. Useful for debugging
86  * drivers.
87  */
88 #ifdef	DEBUG
89 int rootnex_alloc_check_parms = 1;
90 int rootnex_bind_check_inuse = 1;
91 int rootnex_unbind_verify_buffer = 0;
92 int rootnex_sync_check_parms = 1;
93 #else
94 int rootnex_alloc_check_parms = 0;
95 int rootnex_bind_check_inuse = 0;
96 int rootnex_unbind_verify_buffer = 0;
97 int rootnex_sync_check_parms = 0;
98 #endif
99 
100 boolean_t rootnex_dmar_not_setup;
101 
102 /* Master Abort and Target Abort panic flag */
103 int rootnex_fm_ma_ta_panic_flag = 0;
104 
105 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
106 int rootnex_bind_fail = 1;
107 int rootnex_bind_warn = 1;
108 uint8_t *rootnex_warn_list;
109 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
110 #define	ROOTNEX_BIND_WARNING	(0x1 << 0)
111 
112 /*
113  * revert back to old broken behavior of always sync'ing entire copy buffer.
114  * This is useful if be have a buggy driver which doesn't correctly pass in
115  * the offset and size into ddi_dma_sync().
116  */
117 int rootnex_sync_ignore_params = 0;
118 
119 /*
120  * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
121  * page for alignment.  Allocate enough windows to handle a 256K buffer w/ at
122  * least 65 sgllen DMA engine, and enough copybuf buffer state pages to handle
123  * 2 pages (< 8K). We will still need to allocate the copy buffer during bind
124  * though (if we need one). These can only be modified in /etc/system before
125  * rootnex attach.
126  */
127 int rootnex_prealloc_cookies = 65;
128 int rootnex_prealloc_windows = 4;
129 int rootnex_prealloc_copybuf = 2;
130 
131 /* driver global state */
132 static rootnex_state_t *rootnex_state;
133 
134 #ifdef DEBUG
135 /* shortcut to rootnex counters */
136 static uint64_t *rootnex_cnt;
137 #endif
138 
139 /*
140  * XXX - does x86 even need these or are they left over from the SPARC days?
141  */
142 /* statically defined integer/boolean properties for the root node */
143 static rootnex_intprop_t rootnex_intprp[] = {
144 	{ "PAGESIZE",			PAGESIZE },
145 	{ "MMU_PAGESIZE",		MMU_PAGESIZE },
146 	{ "MMU_PAGEOFFSET",		MMU_PAGEOFFSET },
147 	{ DDI_RELATIVE_ADDRESSING,	1 },
148 };
149 #define	NROOT_INTPROPS	(sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
150 
151 /*
152  * If we're dom0, we're using a real device so we need to load
153  * the cookies with MFNs instead of PFNs.
154  */
155 #ifdef __xpv
156 typedef maddr_t rootnex_addr_t;
157 #define	ROOTNEX_PADDR_TO_RBASE(pa)	\
158 	(DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
159 #else
160 typedef paddr_t rootnex_addr_t;
161 #define	ROOTNEX_PADDR_TO_RBASE(pa)	(pa)
162 #endif
163 
164 static struct cb_ops rootnex_cb_ops = {
165 	nodev,		/* open */
166 	nodev,		/* close */
167 	nodev,		/* strategy */
168 	nodev,		/* print */
169 	nodev,		/* dump */
170 	nodev,		/* read */
171 	nodev,		/* write */
172 	nodev,		/* ioctl */
173 	nodev,		/* devmap */
174 	nodev,		/* mmap */
175 	nodev,		/* segmap */
176 	nochpoll,	/* chpoll */
177 	ddi_prop_op,	/* cb_prop_op */
178 	NULL,		/* struct streamtab */
179 	D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
180 	CB_REV,		/* Rev */
181 	nodev,		/* cb_aread */
182 	nodev		/* cb_awrite */
183 };
184 
185 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
186     off_t offset, off_t len, caddr_t *vaddrp);
187 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
188     struct hat *hat, struct seg *seg, caddr_t addr,
189     struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
190 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
191     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
192     ddi_dma_handle_t *handlep);
193 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
194     ddi_dma_handle_t handle);
195 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
196     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
197     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
198 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
199     ddi_dma_handle_t handle);
200 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
201     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
202 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
203     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
204     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
205 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
206     ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
207     off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
208 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
209     ddi_ctl_enum_t ctlop, void *arg, void *result);
210 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
211     ddi_iblock_cookie_t *ibc);
212 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
213     ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
214 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
215     void *);
216 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
217 
218 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
219     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
220     ddi_dma_handle_t *handlep);
221 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
222     ddi_dma_handle_t handle);
223 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
224     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
225     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
226 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
227     ddi_dma_handle_t handle);
228 #if !defined(__xpv)
229 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
230     ddi_dma_handle_t handle);
231 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
232     ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
233 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
234     ddi_dma_cookie_t *cookiep, uint_t ccount);
235 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
236     ddi_dma_handle_t handle);
237 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
238 #endif
239 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
240     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
241 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
242     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
243     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
244 
245 #if !defined(__xpv)
246 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
247     ddi_dma_handle_t handle, void *v);
248 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
249     ddi_dma_handle_t handle);
250 #endif
251 
252 
253 static struct bus_ops rootnex_bus_ops = {
254 	BUSO_REV,
255 	rootnex_map,
256 	NULL,
257 	NULL,
258 	NULL,
259 	rootnex_map_fault,
260 	0,
261 	rootnex_dma_allochdl,
262 	rootnex_dma_freehdl,
263 	rootnex_dma_bindhdl,
264 	rootnex_dma_unbindhdl,
265 	rootnex_dma_sync,
266 	rootnex_dma_win,
267 	rootnex_dma_mctl,
268 	rootnex_ctlops,
269 	ddi_bus_prop_op,
270 	i_ddi_rootnex_get_eventcookie,
271 	i_ddi_rootnex_add_eventcall,
272 	i_ddi_rootnex_remove_eventcall,
273 	i_ddi_rootnex_post_event,
274 	0,			/* bus_intr_ctl */
275 	0,			/* bus_config */
276 	0,			/* bus_unconfig */
277 	rootnex_fm_init,	/* bus_fm_init */
278 	NULL,			/* bus_fm_fini */
279 	NULL,			/* bus_fm_access_enter */
280 	NULL,			/* bus_fm_access_exit */
281 	NULL,			/* bus_powr */
282 	rootnex_intr_ops	/* bus_intr_op */
283 };
284 
285 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
286 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
287 static int rootnex_quiesce(dev_info_t *dip);
288 
289 static struct dev_ops rootnex_ops = {
290 	DEVO_REV,
291 	0,
292 	ddi_no_info,
293 	nulldev,
294 	nulldev,
295 	rootnex_attach,
296 	rootnex_detach,
297 	nulldev,
298 	&rootnex_cb_ops,
299 	&rootnex_bus_ops,
300 	NULL,
301 	rootnex_quiesce,		/* quiesce */
302 };
303 
304 static struct modldrv rootnex_modldrv = {
305 	&mod_driverops,
306 	"i86pc root nexus",
307 	&rootnex_ops
308 };
309 
310 static struct modlinkage rootnex_modlinkage = {
311 	MODREV_1,
312 	(void *)&rootnex_modldrv,
313 	NULL
314 };
315 
316 #if !defined(__xpv)
317 static iommulib_nexops_t iommulib_nexops = {
318 	IOMMU_NEXOPS_VERSION,
319 	"Rootnex IOMMU ops Vers 1.1",
320 	NULL,
321 	rootnex_coredma_allochdl,
322 	rootnex_coredma_freehdl,
323 	rootnex_coredma_bindhdl,
324 	rootnex_coredma_unbindhdl,
325 	rootnex_coredma_reset_cookies,
326 	rootnex_coredma_get_cookies,
327 	rootnex_coredma_set_cookies,
328 	rootnex_coredma_clear_cookies,
329 	rootnex_coredma_get_sleep_flags,
330 	rootnex_coredma_sync,
331 	rootnex_coredma_win,
332 	rootnex_coredma_hdl_setprivate,
333 	rootnex_coredma_hdl_getprivate
334 };
335 #endif
336 
337 /*
338  *  extern hacks
339  */
340 extern struct seg_ops segdev_ops;
341 extern int ignore_hardware_nodes;	/* force flag from ddi_impl.c */
342 #ifdef	DDI_MAP_DEBUG
343 extern int ddi_map_debug_flag;
344 #define	ddi_map_debug	if (ddi_map_debug_flag) prom_printf
345 #endif
346 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
347 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
348 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
349     psm_intr_op_t, int *);
350 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
351 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
352 
353 /*
354  * Use device arena to use for device control register mappings.
355  * Various kernel memory walkers (debugger, dtrace) need to know
356  * to avoid this address range to prevent undesired device activity.
357  */
358 extern void *device_arena_alloc(size_t size, int vm_flag);
359 extern void device_arena_free(void * vaddr, size_t size);
360 
361 
362 /*
363  *  Internal functions
364  */
365 static int rootnex_dma_init();
366 static void rootnex_add_props(dev_info_t *);
367 static int rootnex_ctl_reportdev(dev_info_t *dip);
368 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
369 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
370 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
371 static int rootnex_map_handle(ddi_map_req_t *mp);
372 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
373 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
374 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
375     rootnex_sglinfo_t *sglinfo);
376 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
377     ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
378 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
379     rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
380 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
381     rootnex_dma_t *dma, ddi_dma_attr_t *attr);
382 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
383 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
384     ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
385 static void rootnex_teardown_windows(rootnex_dma_t *dma);
386 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
387     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
388 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
389     rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
390     size_t *copybuf_used, page_t **cur_pp);
391 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
392     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
393     ddi_dma_attr_t *attr, off_t cur_offset);
394 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
395     rootnex_dma_t *dma, rootnex_window_t **windowp,
396     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
397 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
398     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
399 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
400     off_t offset, size_t size, uint_t cache_flags);
401 static int rootnex_verify_buffer(rootnex_dma_t *dma);
402 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
403     const void *comp_addr, const void *not_used);
404 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
405     rootnex_sglinfo_t *sglinfo);
406 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
407 
408 /*
409  * _init()
410  *
411  */
412 int
_init(void)413 _init(void)
414 {
415 
416 	rootnex_state = NULL;
417 	return (mod_install(&rootnex_modlinkage));
418 }
419 
420 
421 /*
422  * _info()
423  *
424  */
425 int
_info(struct modinfo * modinfop)426 _info(struct modinfo *modinfop)
427 {
428 	return (mod_info(&rootnex_modlinkage, modinfop));
429 }
430 
431 
432 /*
433  * _fini()
434  *
435  */
436 int
_fini(void)437 _fini(void)
438 {
439 	return (EBUSY);
440 }
441 
442 
443 /*
444  * rootnex_attach()
445  *
446  */
447 static int
rootnex_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)448 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
449 {
450 	int fmcap;
451 	int e;
452 
453 	switch (cmd) {
454 	case DDI_ATTACH:
455 		break;
456 	case DDI_RESUME:
457 #if !defined(__xpv)
458 		return (immu_unquiesce());
459 #else
460 		return (DDI_SUCCESS);
461 #endif
462 	default:
463 		return (DDI_FAILURE);
464 	}
465 
466 	/*
467 	 * We should only have one instance of rootnex. Save it away since we
468 	 * don't have an easy way to get it back later.
469 	 */
470 	ASSERT(rootnex_state == NULL);
471 	rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
472 
473 	rootnex_state->r_dip = dip;
474 	rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
475 	rootnex_state->r_reserved_msg_printed = B_FALSE;
476 #ifdef DEBUG
477 	rootnex_cnt = &rootnex_state->r_counters[0];
478 #endif
479 
480 	/*
481 	 * Set minimum fm capability level for i86pc platforms and then
482 	 * initialize error handling. Since we're the rootnex, we don't
483 	 * care what's returned in the fmcap field.
484 	 */
485 	ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
486 	    DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
487 	fmcap = ddi_system_fmcap;
488 	ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
489 
490 	/* initialize DMA related state */
491 	e = rootnex_dma_init();
492 	if (e != DDI_SUCCESS) {
493 		kmem_free(rootnex_state, sizeof (rootnex_state_t));
494 		return (DDI_FAILURE);
495 	}
496 
497 	/* Add static root node properties */
498 	rootnex_add_props(dip);
499 
500 	/* since we can't call ddi_report_dev() */
501 	cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
502 
503 	/* Initialize rootnex event handle */
504 	i_ddi_rootnex_init_events(dip);
505 
506 #if !defined(__xpv)
507 	e = iommulib_nexus_register(dip, &iommulib_nexops,
508 	    &rootnex_state->r_iommulib_handle);
509 
510 	ASSERT(e == DDI_SUCCESS);
511 #endif
512 
513 	return (DDI_SUCCESS);
514 }
515 
516 
517 /*
518  * rootnex_detach()
519  *
520  */
521 /*ARGSUSED*/
522 static int
rootnex_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)523 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
524 {
525 	switch (cmd) {
526 	case DDI_SUSPEND:
527 #if !defined(__xpv)
528 		return (immu_quiesce());
529 #else
530 		return (DDI_SUCCESS);
531 #endif
532 	default:
533 		return (DDI_FAILURE);
534 	}
535 	/*NOTREACHED*/
536 
537 }
538 
539 
540 /*
541  * rootnex_dma_init()
542  *
543  */
544 /*ARGSUSED*/
545 static int
rootnex_dma_init()546 rootnex_dma_init()
547 {
548 	size_t bufsize;
549 
550 
551 	/*
552 	 * size of our cookie/window/copybuf state needed in dma bind that we
553 	 * pre-alloc in dma_alloc_handle
554 	 */
555 	rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
556 	rootnex_state->r_prealloc_size =
557 	    (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
558 	    (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
559 	    (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
560 
561 	/*
562 	 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
563 	 * allocate 16 extra bytes for struct pointer alignment
564 	 * (p->dmai_private & dma->dp_prealloc_buffer)
565 	 */
566 	bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
567 	    rootnex_state->r_prealloc_size + 0x10;
568 	rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
569 	    bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
570 	if (rootnex_state->r_dmahdl_cache == NULL) {
571 		return (DDI_FAILURE);
572 	}
573 
574 	/*
575 	 * allocate array to track which major numbers we have printed warnings
576 	 * for.
577 	 */
578 	rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
579 	    KM_SLEEP);
580 
581 	return (DDI_SUCCESS);
582 }
583 
584 
585 /*
586  * rootnex_add_props()
587  *
588  */
589 static void
rootnex_add_props(dev_info_t * dip)590 rootnex_add_props(dev_info_t *dip)
591 {
592 	rootnex_intprop_t *rpp;
593 	int i;
594 
595 	/* Add static integer/boolean properties to the root node */
596 	rpp = rootnex_intprp;
597 	for (i = 0; i < NROOT_INTPROPS; i++) {
598 		(void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
599 		    rpp[i].prop_name, rpp[i].prop_value);
600 	}
601 }
602 
603 
604 
605 /*
606  * *************************
607  *  ctlops related routines
608  * *************************
609  */
610 
611 /*
612  * rootnex_ctlops()
613  *
614  */
615 /*ARGSUSED*/
616 static int
rootnex_ctlops(dev_info_t * dip,dev_info_t * rdip,ddi_ctl_enum_t ctlop,void * arg,void * result)617 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
618     void *arg, void *result)
619 {
620 	int n, *ptr;
621 	struct ddi_parent_private_data *pdp;
622 
623 	switch (ctlop) {
624 	case DDI_CTLOPS_DMAPMAPC:
625 		/*
626 		 * Return 'partial' to indicate that dma mapping
627 		 * has to be done in the main MMU.
628 		 */
629 		return (DDI_DMA_PARTIAL);
630 
631 	case DDI_CTLOPS_BTOP:
632 		/*
633 		 * Convert byte count input to physical page units.
634 		 * (byte counts that are not a page-size multiple
635 		 * are rounded down)
636 		 */
637 		*(ulong_t *)result = btop(*(ulong_t *)arg);
638 		return (DDI_SUCCESS);
639 
640 	case DDI_CTLOPS_PTOB:
641 		/*
642 		 * Convert size in physical pages to bytes
643 		 */
644 		*(ulong_t *)result = ptob(*(ulong_t *)arg);
645 		return (DDI_SUCCESS);
646 
647 	case DDI_CTLOPS_BTOPR:
648 		/*
649 		 * Convert byte count input to physical page units
650 		 * (byte counts that are not a page-size multiple
651 		 * are rounded up)
652 		 */
653 		*(ulong_t *)result = btopr(*(ulong_t *)arg);
654 		return (DDI_SUCCESS);
655 
656 	case DDI_CTLOPS_INITCHILD:
657 		return (impl_ddi_sunbus_initchild(arg));
658 
659 	case DDI_CTLOPS_UNINITCHILD:
660 		impl_ddi_sunbus_removechild(arg);
661 		return (DDI_SUCCESS);
662 
663 	case DDI_CTLOPS_REPORTDEV:
664 		return (rootnex_ctl_reportdev(rdip));
665 
666 	case DDI_CTLOPS_IOMIN:
667 		/*
668 		 * Nothing to do here but reflect back..
669 		 */
670 		return (DDI_SUCCESS);
671 
672 	case DDI_CTLOPS_REGSIZE:
673 	case DDI_CTLOPS_NREGS:
674 		break;
675 
676 	case DDI_CTLOPS_SIDDEV:
677 		if (ndi_dev_is_prom_node(rdip))
678 			return (DDI_SUCCESS);
679 		if (ndi_dev_is_persistent_node(rdip))
680 			return (DDI_SUCCESS);
681 		return (DDI_FAILURE);
682 
683 	case DDI_CTLOPS_POWER:
684 		return ((*pm_platform_power)((power_req_t *)arg));
685 
686 	case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
687 	case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
688 	case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
689 	case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
690 	case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
691 	case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
692 		if (!rootnex_state->r_reserved_msg_printed) {
693 			rootnex_state->r_reserved_msg_printed = B_TRUE;
694 			cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
695 			    "1 or more reserved/obsolete operations.");
696 		}
697 		return (DDI_FAILURE);
698 
699 	default:
700 		return (DDI_FAILURE);
701 	}
702 	/*
703 	 * The rest are for "hardware" properties
704 	 */
705 	if ((pdp = ddi_get_parent_data(rdip)) == NULL)
706 		return (DDI_FAILURE);
707 
708 	if (ctlop == DDI_CTLOPS_NREGS) {
709 		ptr = (int *)result;
710 		*ptr = pdp->par_nreg;
711 	} else {
712 		off_t *size = (off_t *)result;
713 
714 		ptr = (int *)arg;
715 		n = *ptr;
716 		if (n >= pdp->par_nreg) {
717 			return (DDI_FAILURE);
718 		}
719 		*size = (off_t)pdp->par_reg[n].regspec_size;
720 	}
721 	return (DDI_SUCCESS);
722 }
723 
724 
725 /*
726  * rootnex_ctl_reportdev()
727  *
728  */
729 static int
rootnex_ctl_reportdev(dev_info_t * dev)730 rootnex_ctl_reportdev(dev_info_t *dev)
731 {
732 	int i, n, len, f_len = 0;
733 	char *buf;
734 
735 	buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
736 	f_len += snprintf(buf, REPORTDEV_BUFSIZE,
737 	    "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
738 	len = strlen(buf);
739 
740 	for (i = 0; i < sparc_pd_getnreg(dev); i++) {
741 
742 		struct regspec *rp = sparc_pd_getreg(dev, i);
743 
744 		if (i == 0)
745 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
746 			    ": ");
747 		else
748 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
749 			    " and ");
750 		len = strlen(buf);
751 
752 		switch (rp->regspec_bustype) {
753 
754 		case BTEISA:
755 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
756 			    "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
757 			break;
758 
759 		case BTISA:
760 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
761 			    "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
762 			break;
763 
764 		default:
765 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
766 			    "space %x offset %x",
767 			    rp->regspec_bustype, rp->regspec_addr);
768 			break;
769 		}
770 		len = strlen(buf);
771 	}
772 	for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
773 		int pri;
774 
775 		if (i != 0) {
776 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
777 			    ",");
778 			len = strlen(buf);
779 		}
780 		pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
781 		f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
782 		    " sparc ipl %d", pri);
783 		len = strlen(buf);
784 	}
785 #ifdef DEBUG
786 	if (f_len + 1 >= REPORTDEV_BUFSIZE) {
787 		cmn_err(CE_NOTE, "next message is truncated: "
788 		    "printed length 1024, real length %d", f_len);
789 	}
790 #endif /* DEBUG */
791 	cmn_err(CE_CONT, "?%s\n", buf);
792 	kmem_free(buf, REPORTDEV_BUFSIZE);
793 	return (DDI_SUCCESS);
794 }
795 
796 
797 /*
798  * ******************
799  *  map related code
800  * ******************
801  */
802 
803 /*
804  * rootnex_map()
805  *
806  */
807 static int
rootnex_map(dev_info_t * dip,dev_info_t * rdip,ddi_map_req_t * mp,off_t offset,off_t len,caddr_t * vaddrp)808 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
809     off_t len, caddr_t *vaddrp)
810 {
811 	struct regspec *orp = NULL;
812 	struct regspec64 rp = { 0 };
813 	ddi_map_req_t mr = *mp;		/* Get private copy of request */
814 
815 	mp = &mr;
816 
817 	switch (mp->map_op)  {
818 	case DDI_MO_MAP_LOCKED:
819 	case DDI_MO_UNMAP:
820 	case DDI_MO_MAP_HANDLE:
821 		break;
822 	default:
823 #ifdef	DDI_MAP_DEBUG
824 		cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
825 		    mp->map_op);
826 #endif	/* DDI_MAP_DEBUG */
827 		return (DDI_ME_UNIMPLEMENTED);
828 	}
829 
830 	if (mp->map_flags & DDI_MF_USER_MAPPING)  {
831 #ifdef	DDI_MAP_DEBUG
832 		cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
833 #endif	/* DDI_MAP_DEBUG */
834 		return (DDI_ME_UNIMPLEMENTED);
835 	}
836 
837 	/*
838 	 * First, we need to get the original regspec out before we convert it
839 	 * to the extended format. If we have a register number, then we need to
840 	 * convert that to a regspec.
841 	 */
842 	if (mp->map_type == DDI_MT_RNUMBER)  {
843 
844 		int rnumber = mp->map_obj.rnumber;
845 #ifdef	DDI_MAP_DEBUG
846 		static char *out_of_range =
847 		    "rootnex_map: Out of range rnumber <%d>, device <%s>";
848 #endif	/* DDI_MAP_DEBUG */
849 
850 		orp = i_ddi_rnumber_to_regspec(rdip, rnumber);
851 		if (orp == NULL) {
852 #ifdef	DDI_MAP_DEBUG
853 			cmn_err(CE_WARN, out_of_range, rnumber,
854 			    ddi_get_name(rdip));
855 #endif	/* DDI_MAP_DEBUG */
856 			return (DDI_ME_RNUMBER_RANGE);
857 		}
858 	} else if (!(mp->map_flags & DDI_MF_EXT_REGSPEC)) {
859 		orp = mp->map_obj.rp;
860 	}
861 
862 	/*
863 	 * Ensure that we are always using a 64-bit extended regspec regardless
864 	 * of what was passed into us. If the child driver is using a 64-bit
865 	 * regspec, then we need to make sure that we copy this to the local
866 	 * regspec64, rp.
867 	 */
868 	if (orp != NULL) {
869 		rp.regspec_bustype = orp->regspec_bustype;
870 		rp.regspec_addr = orp->regspec_addr;
871 		rp.regspec_size = orp->regspec_size;
872 	} else {
873 		struct regspec64 *rp64;
874 		rp64 = (struct regspec64 *)mp->map_obj.rp;
875 		rp = *rp64;
876 	}
877 
878 	mp->map_type = DDI_MT_REGSPEC;
879 	mp->map_flags |= DDI_MF_EXT_REGSPEC;
880 	mp->map_obj.rp = (struct regspec *)&rp;
881 
882 	/*
883 	 * Adjust offset and length correspnding to called values...
884 	 * XXX: A non-zero length means override the one in the regspec
885 	 * XXX: (regardless of what's in the parent's range?)
886 	 */
887 
888 #ifdef	DDI_MAP_DEBUG
889 	cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
890 	    "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
891 	    rp.regspec_bustype, rp.regspec_addr, rp.regspec_size, offset,
892 	    len, mp->map_handlep);
893 #endif	/* DDI_MAP_DEBUG */
894 
895 	/*
896 	 * I/O or memory mapping:
897 	 *
898 	 *	<bustype=0, addr=x, len=x>: memory
899 	 *	<bustype=1, addr=x, len=x>: i/o
900 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
901 	 */
902 
903 	if (rp.regspec_bustype > 1 && rp.regspec_addr != 0) {
904 		cmn_err(CE_WARN, "<%s,%s> invalid register spec"
905 		    " <0x%" PRIx64 ", 0x%" PRIx64 ", 0x%" PRIx64 ">",
906 		    ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
907 		    rp.regspec_addr, rp.regspec_size);
908 		return (DDI_ME_INVAL);
909 	}
910 
911 	if (rp.regspec_bustype > 1 && rp.regspec_addr == 0) {
912 		/*
913 		 * compatibility i/o mapping
914 		 */
915 		rp.regspec_bustype += offset;
916 	} else {
917 		/*
918 		 * Normal memory or i/o mapping
919 		 */
920 		rp.regspec_addr += offset;
921 	}
922 
923 	if (len != 0)
924 		rp.regspec_size = len;
925 
926 #ifdef	DDI_MAP_DEBUG
927 	cmn_err(CE_CONT, "             <%s,%s> <0x%" PRIx64 ", 0x%" PRIx64
928 	    ", 0x%" PRId64 "> offset %d len %d handle 0x%x\n",
929 	    ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
930 	    rp.regspec_addr, rp.regspec_size, offset, len, mp->map_handlep);
931 #endif	/* DDI_MAP_DEBUG */
932 
933 
934 	/*
935 	 * The x86 root nexus does not have any notion of valid ranges of
936 	 * addresses. Its children have valid ranges, but because there are none
937 	 * for the nexus, we don't need to call i_ddi_apply_range().  Verify
938 	 * that is the case.
939 	 */
940 	ASSERT0(sparc_pd_getnrng(dip));
941 
942 	switch (mp->map_op)  {
943 	case DDI_MO_MAP_LOCKED:
944 
945 		/*
946 		 * Set up the locked down kernel mapping to the regspec...
947 		 */
948 
949 		return (rootnex_map_regspec(mp, vaddrp));
950 
951 	case DDI_MO_UNMAP:
952 
953 		/*
954 		 * Release mapping...
955 		 */
956 
957 		return (rootnex_unmap_regspec(mp, vaddrp));
958 
959 	case DDI_MO_MAP_HANDLE:
960 
961 		return (rootnex_map_handle(mp));
962 
963 	default:
964 		return (DDI_ME_UNIMPLEMENTED);
965 	}
966 }
967 
968 
969 /*
970  * rootnex_map_fault()
971  *
972  *	fault in mappings for requestors
973  */
974 /*ARGSUSED*/
975 static int
rootnex_map_fault(dev_info_t * dip,dev_info_t * rdip,struct hat * hat,struct seg * seg,caddr_t addr,struct devpage * dp,pfn_t pfn,uint_t prot,uint_t lock)976 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
977     struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
978     uint_t lock)
979 {
980 
981 #ifdef	DDI_MAP_DEBUG
982 	ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
983 	ddi_map_debug(" Seg <%s>\n",
984 	    seg->s_ops == &segdev_ops ? "segdev" :
985 	    seg == &kvseg ? "segkmem" : "NONE!");
986 #endif	/* DDI_MAP_DEBUG */
987 
988 	/*
989 	 * This is all terribly broken, but it is a start
990 	 *
991 	 * XXX	Note that this test means that segdev_ops
992 	 *	must be exported from seg_dev.c.
993 	 * XXX	What about devices with their own segment drivers?
994 	 */
995 	if (seg->s_ops == &segdev_ops) {
996 		struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
997 
998 		if (hat == NULL) {
999 			/*
1000 			 * This is one plausible interpretation of
1001 			 * a null hat i.e. use the first hat on the
1002 			 * address space hat list which by convention is
1003 			 * the hat of the system MMU.  At alternative
1004 			 * would be to panic .. this might well be better ..
1005 			 */
1006 			ASSERT(AS_READ_HELD(seg->s_as));
1007 			hat = seg->s_as->a_hat;
1008 			cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1009 		}
1010 		hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1011 		    (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1012 	} else if (seg == &kvseg && dp == NULL) {
1013 		hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1014 		    HAT_LOAD_LOCK);
1015 	} else
1016 		return (DDI_FAILURE);
1017 	return (DDI_SUCCESS);
1018 }
1019 
1020 
1021 static int
rootnex_map_regspec(ddi_map_req_t * mp,caddr_t * vaddrp)1022 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1023 {
1024 	rootnex_addr_t rbase;
1025 	void *cvaddr;
1026 	uint64_t npages, pgoffset;
1027 	struct regspec64 *rp;
1028 	ddi_acc_hdl_t *hp;
1029 	ddi_acc_impl_t *ap;
1030 	uint_t	hat_acc_flags;
1031 	paddr_t pbase;
1032 
1033 	ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
1034 	rp = (struct regspec64 *)mp->map_obj.rp;
1035 	hp = mp->map_handlep;
1036 
1037 #ifdef	DDI_MAP_DEBUG
1038 	ddi_map_debug(
1039 	    "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1040 	    rp->regspec_bustype, rp->regspec_addr,
1041 	    rp->regspec_size, mp->map_handlep);
1042 #endif	/* DDI_MAP_DEBUG */
1043 
1044 	/*
1045 	 * I/O or memory mapping
1046 	 *
1047 	 *	<bustype=0, addr=x, len=x>: memory
1048 	 *	<bustype=1, addr=x, len=x>: i/o
1049 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1050 	 */
1051 
1052 	if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1053 		cmn_err(CE_WARN, "rootnex: invalid register spec"
1054 		    " <0x%" PRIx64 ", 0x%" PRIx64", 0x%" PRIx64">",
1055 		    rp->regspec_bustype, rp->regspec_addr, rp->regspec_size);
1056 		return (DDI_FAILURE);
1057 	}
1058 
1059 	if (rp->regspec_bustype != 0) {
1060 		/*
1061 		 * I/O space - needs a handle.
1062 		 */
1063 		if (hp == NULL) {
1064 			return (DDI_FAILURE);
1065 		}
1066 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1067 		ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1068 		impl_acc_hdl_init(hp);
1069 
1070 		if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1071 #ifdef  DDI_MAP_DEBUG
1072 			ddi_map_debug("rootnex_map_regspec: mmap() "
1073 			    "to I/O space is not supported.\n");
1074 #endif  /* DDI_MAP_DEBUG */
1075 			return (DDI_ME_INVAL);
1076 		} else {
1077 			/*
1078 			 * 1275-compliant vs. compatibility i/o mapping
1079 			 */
1080 			*vaddrp =
1081 			    (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1082 			    ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1083 			    ((caddr_t)(uintptr_t)rp->regspec_addr);
1084 #ifdef __xpv
1085 			if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1086 				hp->ah_pfn = xen_assign_pfn(
1087 				    mmu_btop((ulong_t)rp->regspec_addr &
1088 				    MMU_PAGEMASK));
1089 			} else {
1090 				hp->ah_pfn = mmu_btop(
1091 				    (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1092 			}
1093 #else
1094 			hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1095 			    MMU_PAGEMASK);
1096 #endif
1097 			hp->ah_pnum = mmu_btopr(rp->regspec_size +
1098 			    (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1099 		}
1100 
1101 #ifdef	DDI_MAP_DEBUG
1102 		ddi_map_debug(
1103 	    "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1104 		    rp->regspec_size, *vaddrp);
1105 #endif	/* DDI_MAP_DEBUG */
1106 		return (DDI_SUCCESS);
1107 	}
1108 
1109 	/*
1110 	 * Memory space
1111 	 */
1112 
1113 	if (hp != NULL) {
1114 		/*
1115 		 * hat layer ignores
1116 		 * hp->ah_acc.devacc_attr_endian_flags.
1117 		 */
1118 		switch (hp->ah_acc.devacc_attr_dataorder) {
1119 		case DDI_STRICTORDER_ACC:
1120 			hat_acc_flags = HAT_STRICTORDER;
1121 			break;
1122 		case DDI_UNORDERED_OK_ACC:
1123 			hat_acc_flags = HAT_UNORDERED_OK;
1124 			break;
1125 		case DDI_MERGING_OK_ACC:
1126 			hat_acc_flags = HAT_MERGING_OK;
1127 			break;
1128 		case DDI_LOADCACHING_OK_ACC:
1129 			hat_acc_flags = HAT_LOADCACHING_OK;
1130 			break;
1131 		case DDI_STORECACHING_OK_ACC:
1132 			hat_acc_flags = HAT_STORECACHING_OK;
1133 			break;
1134 		default:
1135 			return (DDI_ME_INVAL);
1136 		}
1137 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1138 		ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1139 		impl_acc_hdl_init(hp);
1140 		hp->ah_hat_flags = hat_acc_flags;
1141 	} else {
1142 		hat_acc_flags = HAT_STRICTORDER;
1143 	}
1144 
1145 	rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1146 #ifdef __xpv
1147 	/*
1148 	 * If we're dom0, we're using a real device so we need to translate
1149 	 * the MA to a PA.
1150 	 */
1151 	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1152 		pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1153 	} else {
1154 		pbase = rbase;
1155 	}
1156 #else
1157 	pbase = rbase;
1158 #endif
1159 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1160 
1161 	if (rp->regspec_size == 0) {
1162 #ifdef  DDI_MAP_DEBUG
1163 		ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1164 #endif  /* DDI_MAP_DEBUG */
1165 		return (DDI_ME_INVAL);
1166 	}
1167 
1168 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1169 		/* extra cast to make gcc happy */
1170 		*vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1171 	} else {
1172 		npages = mmu_btopr(rp->regspec_size + pgoffset);
1173 
1174 #ifdef	DDI_MAP_DEBUG
1175 		ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1176 		    "physical %llx", npages, pbase);
1177 #endif	/* DDI_MAP_DEBUG */
1178 
1179 		cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1180 		if (cvaddr == NULL)
1181 			return (DDI_ME_NORESOURCES);
1182 
1183 		/*
1184 		 * Now map in the pages we've allocated...
1185 		 */
1186 		hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1187 		    mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1188 		    HAT_LOAD_LOCK);
1189 		*vaddrp = (caddr_t)cvaddr + pgoffset;
1190 
1191 		/* save away pfn and npages for FMA */
1192 		hp = mp->map_handlep;
1193 		if (hp) {
1194 			hp->ah_pfn = mmu_btop(pbase);
1195 			hp->ah_pnum = npages;
1196 		}
1197 	}
1198 
1199 #ifdef	DDI_MAP_DEBUG
1200 	ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1201 #endif	/* DDI_MAP_DEBUG */
1202 	return (DDI_SUCCESS);
1203 }
1204 
1205 
1206 static int
rootnex_unmap_regspec(ddi_map_req_t * mp,caddr_t * vaddrp)1207 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1208 {
1209 	caddr_t addr = (caddr_t)*vaddrp;
1210 	uint64_t npages, pgoffset;
1211 	struct regspec64 *rp;
1212 
1213 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1214 		return (0);
1215 
1216 	ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
1217 	rp = (struct regspec64 *)mp->map_obj.rp;
1218 
1219 	if (rp->regspec_size == 0) {
1220 #ifdef  DDI_MAP_DEBUG
1221 		ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1222 #endif  /* DDI_MAP_DEBUG */
1223 		return (DDI_ME_INVAL);
1224 	}
1225 
1226 	/*
1227 	 * I/O or memory mapping:
1228 	 *
1229 	 *	<bustype=0, addr=x, len=x>: memory
1230 	 *	<bustype=1, addr=x, len=x>: i/o
1231 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1232 	 */
1233 	if (rp->regspec_bustype != 0) {
1234 		/*
1235 		 * This is I/O space, which requires no particular
1236 		 * processing on unmap since it isn't mapped in the
1237 		 * first place.
1238 		 */
1239 		return (DDI_SUCCESS);
1240 	}
1241 
1242 	/*
1243 	 * Memory space
1244 	 */
1245 	pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1246 	npages = mmu_btopr(rp->regspec_size + pgoffset);
1247 	hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1248 	device_arena_free(addr - pgoffset, ptob(npages));
1249 
1250 	/*
1251 	 * Destroy the pointer - the mapping has logically gone
1252 	 */
1253 	*vaddrp = NULL;
1254 
1255 	return (DDI_SUCCESS);
1256 }
1257 
1258 static int
rootnex_map_handle(ddi_map_req_t * mp)1259 rootnex_map_handle(ddi_map_req_t *mp)
1260 {
1261 	rootnex_addr_t rbase;
1262 	ddi_acc_hdl_t *hp;
1263 	uint64_t pgoffset;
1264 	struct regspec64 *rp;
1265 	paddr_t pbase;
1266 
1267 	rp = (struct regspec64 *)mp->map_obj.rp;
1268 
1269 #ifdef	DDI_MAP_DEBUG
1270 	ddi_map_debug(
1271 	    "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1272 	    rp->regspec_bustype, rp->regspec_addr,
1273 	    rp->regspec_size, mp->map_handlep);
1274 #endif	/* DDI_MAP_DEBUG */
1275 
1276 	/*
1277 	 * I/O or memory mapping:
1278 	 *
1279 	 *	<bustype=0, addr=x, len=x>: memory
1280 	 *	<bustype=1, addr=x, len=x>: i/o
1281 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1282 	 */
1283 	if (rp->regspec_bustype != 0) {
1284 		/*
1285 		 * This refers to I/O space, and we don't support "mapping"
1286 		 * I/O space to a user.
1287 		 */
1288 		return (DDI_FAILURE);
1289 	}
1290 
1291 	/*
1292 	 * Set up the hat_flags for the mapping.
1293 	 */
1294 	hp = mp->map_handlep;
1295 
1296 	switch (hp->ah_acc.devacc_attr_endian_flags) {
1297 	case DDI_NEVERSWAP_ACC:
1298 		hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1299 		break;
1300 	case DDI_STRUCTURE_LE_ACC:
1301 		hp->ah_hat_flags = HAT_STRUCTURE_LE;
1302 		break;
1303 	case DDI_STRUCTURE_BE_ACC:
1304 		return (DDI_FAILURE);
1305 	default:
1306 		return (DDI_REGS_ACC_CONFLICT);
1307 	}
1308 
1309 	switch (hp->ah_acc.devacc_attr_dataorder) {
1310 	case DDI_STRICTORDER_ACC:
1311 		break;
1312 	case DDI_UNORDERED_OK_ACC:
1313 		hp->ah_hat_flags |= HAT_UNORDERED_OK;
1314 		break;
1315 	case DDI_MERGING_OK_ACC:
1316 		hp->ah_hat_flags |= HAT_MERGING_OK;
1317 		break;
1318 	case DDI_LOADCACHING_OK_ACC:
1319 		hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1320 		break;
1321 	case DDI_STORECACHING_OK_ACC:
1322 		hp->ah_hat_flags |= HAT_STORECACHING_OK;
1323 		break;
1324 	default:
1325 		return (DDI_FAILURE);
1326 	}
1327 
1328 	rbase = (rootnex_addr_t)rp->regspec_addr &
1329 	    (~(rootnex_addr_t)MMU_PAGEOFFSET);
1330 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1331 
1332 	if (rp->regspec_size == 0)
1333 		return (DDI_ME_INVAL);
1334 
1335 #ifdef __xpv
1336 	/*
1337 	 * If we're dom0, we're using a real device so we need to translate
1338 	 * the MA to a PA.
1339 	 */
1340 	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1341 		pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1342 		    (rbase & MMU_PAGEOFFSET);
1343 	} else {
1344 		pbase = rbase;
1345 	}
1346 #else
1347 	pbase = rbase;
1348 #endif
1349 
1350 	hp->ah_pfn = mmu_btop(pbase);
1351 	hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1352 
1353 	return (DDI_SUCCESS);
1354 }
1355 
1356 
1357 
1358 /*
1359  * ************************
1360  *  interrupt related code
1361  * ************************
1362  */
1363 
1364 /*
1365  * rootnex_intr_ops()
1366  *	bus_intr_op() function for interrupt support
1367  */
1368 /* ARGSUSED */
1369 static int
rootnex_intr_ops(dev_info_t * pdip,dev_info_t * rdip,ddi_intr_op_t intr_op,ddi_intr_handle_impl_t * hdlp,void * result)1370 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1371     ddi_intr_handle_impl_t *hdlp, void *result)
1372 {
1373 	struct intrspec			*ispec;
1374 
1375 	DDI_INTR_NEXDBG((CE_CONT,
1376 	    "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1377 	    (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1378 
1379 	/* Process the interrupt operation */
1380 	switch (intr_op) {
1381 	case DDI_INTROP_GETCAP:
1382 		/* First check with pcplusmp */
1383 		if (psm_intr_ops == NULL)
1384 			return (DDI_FAILURE);
1385 
1386 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1387 			*(int *)result = 0;
1388 			return (DDI_FAILURE);
1389 		}
1390 		break;
1391 	case DDI_INTROP_SETCAP:
1392 		if (psm_intr_ops == NULL)
1393 			return (DDI_FAILURE);
1394 
1395 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1396 			return (DDI_FAILURE);
1397 		break;
1398 	case DDI_INTROP_ALLOC:
1399 		ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1400 		return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1401 	case DDI_INTROP_FREE:
1402 		ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1403 		return (rootnex_free_intr_fixed(rdip, hdlp));
1404 	case DDI_INTROP_GETPRI:
1405 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1406 			return (DDI_FAILURE);
1407 		*(int *)result = ispec->intrspec_pri;
1408 		break;
1409 	case DDI_INTROP_SETPRI:
1410 		/* Validate the interrupt priority passed to us */
1411 		if (*(int *)result > LOCK_LEVEL)
1412 			return (DDI_FAILURE);
1413 
1414 		/* Ensure that PSM is all initialized and ispec is ok */
1415 		if ((psm_intr_ops == NULL) ||
1416 		    ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1417 			return (DDI_FAILURE);
1418 
1419 		/* Change the priority */
1420 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1421 		    PSM_FAILURE)
1422 			return (DDI_FAILURE);
1423 
1424 		/* update the ispec with the new priority */
1425 		ispec->intrspec_pri =  *(int *)result;
1426 		break;
1427 	case DDI_INTROP_ADDISR:
1428 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1429 			return (DDI_FAILURE);
1430 		ispec->intrspec_func = hdlp->ih_cb_func;
1431 		break;
1432 	case DDI_INTROP_REMISR:
1433 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1434 			return (DDI_FAILURE);
1435 		ispec->intrspec_func = (uint_t (*)()) 0;
1436 		break;
1437 	case DDI_INTROP_ENABLE:
1438 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1439 			return (DDI_FAILURE);
1440 
1441 		/* Call psmi to translate irq with the dip */
1442 		if (psm_intr_ops == NULL)
1443 			return (DDI_FAILURE);
1444 
1445 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1446 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1447 		    (int *)&hdlp->ih_vector) == PSM_FAILURE)
1448 			return (DDI_FAILURE);
1449 
1450 		/* Add the interrupt handler */
1451 		if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1452 		    hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1453 		    hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1454 			return (DDI_FAILURE);
1455 		break;
1456 	case DDI_INTROP_DISABLE:
1457 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1458 			return (DDI_FAILURE);
1459 
1460 		/* Call psm_ops() to translate irq with the dip */
1461 		if (psm_intr_ops == NULL)
1462 			return (DDI_FAILURE);
1463 
1464 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1465 		(void) (*psm_intr_ops)(rdip, hdlp,
1466 		    PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1467 
1468 		/* Remove the interrupt handler */
1469 		rem_avintr((void *)hdlp, ispec->intrspec_pri,
1470 		    hdlp->ih_cb_func, hdlp->ih_vector);
1471 		break;
1472 	case DDI_INTROP_SETMASK:
1473 		if (psm_intr_ops == NULL)
1474 			return (DDI_FAILURE);
1475 
1476 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1477 			return (DDI_FAILURE);
1478 		break;
1479 	case DDI_INTROP_CLRMASK:
1480 		if (psm_intr_ops == NULL)
1481 			return (DDI_FAILURE);
1482 
1483 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1484 			return (DDI_FAILURE);
1485 		break;
1486 	case DDI_INTROP_GETPENDING:
1487 		if (psm_intr_ops == NULL)
1488 			return (DDI_FAILURE);
1489 
1490 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1491 		    result)) {
1492 			*(int *)result = 0;
1493 			return (DDI_FAILURE);
1494 		}
1495 		break;
1496 	case DDI_INTROP_NAVAIL:
1497 	case DDI_INTROP_NINTRS:
1498 		*(int *)result = i_ddi_get_intx_nintrs(rdip);
1499 		if (*(int *)result == 0) {
1500 			/*
1501 			 * Special case for 'pcic' driver' only. This driver
1502 			 * driver is a child of 'isa' and 'rootnex' drivers.
1503 			 *
1504 			 * See detailed comments on this in the function
1505 			 * rootnex_get_ispec().
1506 			 *
1507 			 * Children of 'pcic' send 'NINITR' request all the
1508 			 * way to rootnex driver. But, the 'pdp->par_nintr'
1509 			 * field may not initialized. So, we fake it here
1510 			 * to return 1 (a la what PCMCIA nexus does).
1511 			 */
1512 			if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1513 				*(int *)result = 1;
1514 			else
1515 				return (DDI_FAILURE);
1516 		}
1517 		break;
1518 	case DDI_INTROP_SUPPORTED_TYPES:
1519 		*(int *)result = DDI_INTR_TYPE_FIXED;	/* Always ... */
1520 		break;
1521 	default:
1522 		return (DDI_FAILURE);
1523 	}
1524 
1525 	return (DDI_SUCCESS);
1526 }
1527 
1528 
1529 /*
1530  * rootnex_get_ispec()
1531  *	convert an interrupt number to an interrupt specification.
1532  *	The interrupt number determines which interrupt spec will be
1533  *	returned if more than one exists.
1534  *
1535  *	Look into the parent private data area of the 'rdip' to find out
1536  *	the interrupt specification.  First check to make sure there is
1537  *	one that matchs "inumber" and then return a pointer to it.
1538  *
1539  *	Return NULL if one could not be found.
1540  *
1541  *	NOTE: This is needed for rootnex_intr_ops()
1542  */
1543 static struct intrspec *
rootnex_get_ispec(dev_info_t * rdip,int inum)1544 rootnex_get_ispec(dev_info_t *rdip, int inum)
1545 {
1546 	struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1547 
1548 	/*
1549 	 * Special case handling for drivers that provide their own
1550 	 * intrspec structures instead of relying on the DDI framework.
1551 	 *
1552 	 * A broken hardware driver in ON could potentially provide its
1553 	 * own intrspec structure, instead of relying on the hardware.
1554 	 * If these drivers are children of 'rootnex' then we need to
1555 	 * continue to provide backward compatibility to them here.
1556 	 *
1557 	 * Following check is a special case for 'pcic' driver which
1558 	 * was found to have broken hardwre andby provides its own intrspec.
1559 	 *
1560 	 * Verbatim comments from this driver are shown here:
1561 	 * "Don't use the ddi_add_intr since we don't have a
1562 	 * default intrspec in all cases."
1563 	 *
1564 	 * Since an 'ispec' may not be always created for it,
1565 	 * check for that and create one if so.
1566 	 *
1567 	 * NOTE: Currently 'pcic' is the only driver found to do this.
1568 	 */
1569 	if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1570 		pdp->par_nintr = 1;
1571 		pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1572 		    pdp->par_nintr, KM_SLEEP);
1573 	}
1574 
1575 	/* Validate the interrupt number */
1576 	if (inum >= pdp->par_nintr)
1577 		return (NULL);
1578 
1579 	/* Get the interrupt structure pointer and return that */
1580 	return ((struct intrspec *)&pdp->par_intr[inum]);
1581 }
1582 
1583 /*
1584  * Allocate interrupt vector for FIXED (legacy) type.
1585  */
1586 static int
rootnex_alloc_intr_fixed(dev_info_t * rdip,ddi_intr_handle_impl_t * hdlp,void * result)1587 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1588     void *result)
1589 {
1590 	struct intrspec		*ispec;
1591 	ddi_intr_handle_impl_t	info_hdl;
1592 	int			ret;
1593 	int			free_phdl = 0;
1594 	apic_get_type_t		type_info;
1595 
1596 	if (psm_intr_ops == NULL)
1597 		return (DDI_FAILURE);
1598 
1599 	if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1600 		return (DDI_FAILURE);
1601 
1602 	/*
1603 	 * If the PSM module is "APIX" then pass the request for it
1604 	 * to allocate the vector now.
1605 	 */
1606 	bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1607 	info_hdl.ih_private = &type_info;
1608 	if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1609 	    PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1610 		if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1611 			free_phdl = 1;
1612 			i_ddi_alloc_intr_phdl(hdlp);
1613 		}
1614 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1615 		ret = (*psm_intr_ops)(rdip, hdlp,
1616 		    PSM_INTR_OP_ALLOC_VECTORS, result);
1617 		if (free_phdl) { /* free up the phdl structure */
1618 			free_phdl = 0;
1619 			i_ddi_free_intr_phdl(hdlp);
1620 			hdlp->ih_private = NULL;
1621 		}
1622 	} else {
1623 		/*
1624 		 * No APIX module; fall back to the old scheme where the
1625 		 * interrupt vector is allocated during ddi_enable_intr() call.
1626 		 */
1627 		hdlp->ih_pri = ispec->intrspec_pri;
1628 		*(int *)result = hdlp->ih_scratch1;
1629 		ret = DDI_SUCCESS;
1630 	}
1631 
1632 	return (ret);
1633 }
1634 
1635 /*
1636  * Free up interrupt vector for FIXED (legacy) type.
1637  */
1638 static int
rootnex_free_intr_fixed(dev_info_t * rdip,ddi_intr_handle_impl_t * hdlp)1639 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1640 {
1641 	struct intrspec			*ispec;
1642 	struct ddi_parent_private_data	*pdp;
1643 	ddi_intr_handle_impl_t		info_hdl;
1644 	int				ret;
1645 	apic_get_type_t			type_info;
1646 
1647 	if (psm_intr_ops == NULL)
1648 		return (DDI_FAILURE);
1649 
1650 	/*
1651 	 * If the PSM module is "APIX" then pass the request for it
1652 	 * to free up the vector now.
1653 	 */
1654 	bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1655 	info_hdl.ih_private = &type_info;
1656 	if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1657 	    PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1658 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1659 			return (DDI_FAILURE);
1660 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1661 		ret = (*psm_intr_ops)(rdip, hdlp,
1662 		    PSM_INTR_OP_FREE_VECTORS, NULL);
1663 	} else {
1664 		/*
1665 		 * No APIX module; fall back to the old scheme where
1666 		 * the interrupt vector was already freed during
1667 		 * ddi_disable_intr() call.
1668 		 */
1669 		ret = DDI_SUCCESS;
1670 	}
1671 
1672 	pdp = ddi_get_parent_data(rdip);
1673 
1674 	/*
1675 	 * Special case for 'pcic' driver' only.
1676 	 * If an intrspec was created for it, clean it up here
1677 	 * See detailed comments on this in the function
1678 	 * rootnex_get_ispec().
1679 	 */
1680 	if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1681 		kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1682 		    pdp->par_nintr);
1683 		/*
1684 		 * Set it to zero; so that
1685 		 * DDI framework doesn't free it again
1686 		 */
1687 		pdp->par_intr = NULL;
1688 		pdp->par_nintr = 0;
1689 	}
1690 
1691 	return (ret);
1692 }
1693 
1694 
1695 /*
1696  * ******************
1697  *  dma related code
1698  * ******************
1699  */
1700 
1701 /*ARGSUSED*/
1702 static int
rootnex_coredma_allochdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_attr_t * attr,int (* waitfp)(caddr_t),caddr_t arg,ddi_dma_handle_t * handlep)1703 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1704     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1705     ddi_dma_handle_t *handlep)
1706 {
1707 	uint64_t maxsegmentsize_ll;
1708 	uint_t maxsegmentsize;
1709 	ddi_dma_impl_t *hp;
1710 	rootnex_dma_t *dma;
1711 	uint64_t count_max;
1712 	uint64_t seg;
1713 	int kmflag;
1714 	int e;
1715 
1716 
1717 	/* convert our sleep flags */
1718 	if (waitfp == DDI_DMA_SLEEP) {
1719 		kmflag = KM_SLEEP;
1720 	} else {
1721 		kmflag = KM_NOSLEEP;
1722 	}
1723 
1724 	/*
1725 	 * We try to do only one memory allocation here. We'll do a little
1726 	 * pointer manipulation later. If the bind ends up taking more than
1727 	 * our prealloc's space, we'll have to allocate more memory in the
1728 	 * bind operation. Not great, but much better than before and the
1729 	 * best we can do with the current bind interfaces.
1730 	 */
1731 	hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1732 	if (hp == NULL)
1733 		return (DDI_DMA_NORESOURCES);
1734 
1735 	/* Do our pointer manipulation now, align the structures */
1736 	hp->dmai_private = (void *)(((uintptr_t)hp +
1737 	    (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1738 	dma = (rootnex_dma_t *)hp->dmai_private;
1739 	dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1740 	    sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1741 
1742 	/* setup the handle */
1743 	rootnex_clean_dmahdl(hp);
1744 	hp->dmai_error.err_fep = NULL;
1745 	hp->dmai_error.err_cf = NULL;
1746 	dma->dp_dip = rdip;
1747 	dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1748 	dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1749 
1750 	/*
1751 	 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1752 	 * is being used. Set the upper limit to the seg value.
1753 	 * There will be enough DVMA space to always get addresses
1754 	 * that will match the constraints.
1755 	 */
1756 	if (IOMMU_USED(rdip) &&
1757 	    (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1758 		dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1759 		dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1760 	} else
1761 		dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1762 
1763 	hp->dmai_minxfer = attr->dma_attr_minxfer;
1764 	hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1765 	hp->dmai_rdip = rdip;
1766 	hp->dmai_attr = *attr;
1767 
1768 	if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1769 		dma->dp_sglinfo.si_cancross = B_FALSE;
1770 	else
1771 		dma->dp_sglinfo.si_cancross = B_TRUE;
1772 
1773 	/* we don't need to worry about the SPL since we do a tryenter */
1774 	mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1775 
1776 	/*
1777 	 * Figure out our maximum segment size. If the segment size is greater
1778 	 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1779 	 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1780 	 * dma_attr_count_max are size-1 type values.
1781 	 *
1782 	 * Maximum segment size is the largest physically contiguous chunk of
1783 	 * memory that we can return from a bind (i.e. the maximum size of a
1784 	 * single cookie).
1785 	 */
1786 
1787 	/* handle the rollover cases */
1788 	seg = attr->dma_attr_seg + 1;
1789 	if (seg < attr->dma_attr_seg) {
1790 		seg = attr->dma_attr_seg;
1791 	}
1792 	count_max = attr->dma_attr_count_max + 1;
1793 	if (count_max < attr->dma_attr_count_max) {
1794 		count_max = attr->dma_attr_count_max;
1795 	}
1796 
1797 	/*
1798 	 * granularity may or may not be a power of two. If it isn't, we can't
1799 	 * use a simple mask.
1800 	 */
1801 	if (!ISP2(attr->dma_attr_granular)) {
1802 		dma->dp_granularity_power_2 = B_FALSE;
1803 	} else {
1804 		dma->dp_granularity_power_2 = B_TRUE;
1805 	}
1806 
1807 	/*
1808 	 * maxxfer should be a whole multiple of granularity. If we're going to
1809 	 * break up a window because we're greater than maxxfer, we might as
1810 	 * well make sure it's maxxfer is a whole multiple so we don't have to
1811 	 * worry about triming the window later on for this case.
1812 	 */
1813 	if (attr->dma_attr_granular > 1) {
1814 		if (dma->dp_granularity_power_2) {
1815 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1816 			    (attr->dma_attr_maxxfer &
1817 			    (attr->dma_attr_granular - 1));
1818 		} else {
1819 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1820 			    (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1821 		}
1822 	} else {
1823 		dma->dp_maxxfer = attr->dma_attr_maxxfer;
1824 	}
1825 
1826 	maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1827 	maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1828 	if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1829 		maxsegmentsize = 0xFFFFFFFF;
1830 	} else {
1831 		maxsegmentsize = maxsegmentsize_ll;
1832 	}
1833 	dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1834 	dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1835 
1836 	/* check the ddi_dma_attr arg to make sure it makes a little sense */
1837 	if (rootnex_alloc_check_parms) {
1838 		e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1839 		if (e != DDI_SUCCESS) {
1840 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1841 			(void) rootnex_dma_freehdl(dip, rdip,
1842 			    (ddi_dma_handle_t)hp);
1843 			return (e);
1844 		}
1845 	}
1846 
1847 	*handlep = (ddi_dma_handle_t)hp;
1848 
1849 	ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1850 	ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1851 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1852 
1853 	return (DDI_SUCCESS);
1854 }
1855 
1856 
1857 /*
1858  * rootnex_dma_allochdl()
1859  *    called from ddi_dma_alloc_handle().
1860  */
1861 static int
rootnex_dma_allochdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_attr_t * attr,int (* waitfp)(caddr_t),caddr_t arg,ddi_dma_handle_t * handlep)1862 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1863     int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1864 {
1865 	int retval = DDI_SUCCESS;
1866 #if !defined(__xpv)
1867 
1868 	if (IOMMU_UNITIALIZED(rdip)) {
1869 		retval = iommulib_nex_open(dip, rdip);
1870 
1871 		if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1872 			return (retval);
1873 	}
1874 
1875 	if (IOMMU_UNUSED(rdip)) {
1876 		retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1877 		    handlep);
1878 	} else {
1879 		retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1880 		    waitfp, arg, handlep);
1881 	}
1882 #else
1883 	retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1884 	    handlep);
1885 #endif
1886 	switch (retval) {
1887 	case DDI_DMA_NORESOURCES:
1888 		if (waitfp != DDI_DMA_DONTWAIT) {
1889 			ddi_set_callback(waitfp, arg,
1890 			    &rootnex_state->r_dvma_call_list_id);
1891 		}
1892 		break;
1893 	case DDI_SUCCESS:
1894 		ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1895 		break;
1896 	default:
1897 		break;
1898 	}
1899 	return (retval);
1900 }
1901 
1902 /*ARGSUSED*/
1903 static int
rootnex_coredma_freehdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)1904 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1905     ddi_dma_handle_t handle)
1906 {
1907 	ddi_dma_impl_t *hp;
1908 	rootnex_dma_t *dma;
1909 
1910 
1911 	hp = (ddi_dma_impl_t *)handle;
1912 	dma = (rootnex_dma_t *)hp->dmai_private;
1913 
1914 	/* unbind should have been called first */
1915 	ASSERT(!dma->dp_inuse);
1916 
1917 	mutex_destroy(&dma->dp_mutex);
1918 	kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1919 
1920 	ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1921 	ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1922 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1923 
1924 	return (DDI_SUCCESS);
1925 }
1926 
1927 /*
1928  * rootnex_dma_freehdl()
1929  *    called from ddi_dma_free_handle().
1930  */
1931 static int
rootnex_dma_freehdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)1932 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1933 {
1934 	int ret;
1935 
1936 	ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1937 #if !defined(__xpv)
1938 	if (IOMMU_USED(rdip))
1939 		ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1940 	else
1941 #endif
1942 	ret = rootnex_coredma_freehdl(dip, rdip, handle);
1943 
1944 	if (rootnex_state->r_dvma_call_list_id)
1945 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1946 
1947 	return (ret);
1948 }
1949 
1950 /*ARGSUSED*/
1951 static int
rootnex_coredma_bindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,struct ddi_dma_req * dmareq,ddi_dma_cookie_t * cookiep,uint_t * ccountp)1952 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1953     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1954     ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1955 {
1956 	rootnex_sglinfo_t *sinfo;
1957 	ddi_dma_obj_t *dmao;
1958 #if !defined(__xpv)
1959 	struct dvmaseg *dvs;
1960 	ddi_dma_cookie_t *cookie;
1961 #endif
1962 	ddi_dma_attr_t *attr;
1963 	ddi_dma_impl_t *hp;
1964 	rootnex_dma_t *dma;
1965 	int kmflag;
1966 	int e;
1967 	uint_t ncookies;
1968 
1969 	hp = (ddi_dma_impl_t *)handle;
1970 	dma = (rootnex_dma_t *)hp->dmai_private;
1971 	dmao = &dma->dp_dma;
1972 	sinfo = &dma->dp_sglinfo;
1973 	attr = &hp->dmai_attr;
1974 
1975 	/* convert the sleep flags */
1976 	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1977 		dma->dp_sleep_flags = kmflag = KM_SLEEP;
1978 	} else {
1979 		dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1980 	}
1981 
1982 	hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1983 
1984 	/*
1985 	 * This is useful for debugging a driver. Not as useful in a production
1986 	 * system. The only time this will fail is if you have a driver bug.
1987 	 */
1988 	if (rootnex_bind_check_inuse) {
1989 		/*
1990 		 * No one else should ever have this lock unless someone else
1991 		 * is trying to use this handle. So contention on the lock
1992 		 * is the same as inuse being set.
1993 		 */
1994 		e = mutex_tryenter(&dma->dp_mutex);
1995 		if (e == 0) {
1996 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1997 			return (DDI_DMA_INUSE);
1998 		}
1999 		if (dma->dp_inuse) {
2000 			mutex_exit(&dma->dp_mutex);
2001 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2002 			return (DDI_DMA_INUSE);
2003 		}
2004 		dma->dp_inuse = B_TRUE;
2005 		mutex_exit(&dma->dp_mutex);
2006 	}
2007 
2008 	/* save away the original bind info */
2009 	dma->dp_dma = dmareq->dmar_object;
2010 
2011 #if !defined(__xpv)
2012 	if (IOMMU_USED(rdip)) {
2013 		dmao = &dma->dp_dvma;
2014 		e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2015 		switch (e) {
2016 		case DDI_SUCCESS:
2017 			if (sinfo->si_cancross ||
2018 			    dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2019 			    dmao->dmao_size > sinfo->si_max_cookie_size) {
2020 				dma->dp_dvma_used = B_TRUE;
2021 				break;
2022 			}
2023 			sinfo->si_sgl_size = 1;
2024 			hp->dmai_rflags |= DMP_NOSYNC;
2025 
2026 			dma->dp_dvma_used = B_TRUE;
2027 			dma->dp_need_to_free_cookie = B_FALSE;
2028 
2029 			dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2030 			cookie = hp->dmai_cookie = dma->dp_cookies =
2031 			    (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2032 			cookie->dmac_laddress = dvs->dvs_start +
2033 			    dmao->dmao_obj.dvma_obj.dv_off;
2034 			cookie->dmac_size = dvs->dvs_len;
2035 			cookie->dmac_type = 0;
2036 
2037 			ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2038 			    rdip);
2039 			goto fast;
2040 		case DDI_ENOTSUP:
2041 			break;
2042 		default:
2043 			rootnex_clean_dmahdl(hp);
2044 			return (e);
2045 		}
2046 	}
2047 #endif
2048 
2049 	/*
2050 	 * Figure out a rough estimate of what maximum number of pages
2051 	 * this buffer could use (a high estimate of course).
2052 	 */
2053 	sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2054 
2055 	if (dma->dp_dvma_used) {
2056 		/*
2057 		 * The number of physical pages is the worst case.
2058 		 *
2059 		 * For DVMA, the worst case is the length divided
2060 		 * by the maximum cookie length, plus 1. Add to that
2061 		 * the number of segment boundaries potentially crossed, and
2062 		 * the additional number of DVMA segments that was returned.
2063 		 *
2064 		 * In the normal case, for modern devices, si_cancross will
2065 		 * be false, and dv_nseg will be 1, and the fast path will
2066 		 * have been taken above.
2067 		 */
2068 		ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2069 		    + 1;
2070 		if (sinfo->si_cancross)
2071 			ncookies +=
2072 			    (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2073 		ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2074 
2075 		sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2076 	}
2077 
2078 	/*
2079 	 * We'll use the pre-allocated cookies for any bind that will *always*
2080 	 * fit (more important to be consistent, we don't want to create
2081 	 * additional degenerate cases).
2082 	 */
2083 	if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2084 		dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2085 		dma->dp_need_to_free_cookie = B_FALSE;
2086 		ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2087 		    uint_t, sinfo->si_max_pages);
2088 
2089 	/*
2090 	 * For anything larger than that, we'll go ahead and allocate the
2091 	 * maximum number of pages we expect to see. Hopefuly, we won't be
2092 	 * seeing this path in the fast path for high performance devices very
2093 	 * frequently.
2094 	 *
2095 	 * a ddi bind interface that allowed the driver to provide storage to
2096 	 * the bind interface would speed this case up.
2097 	 */
2098 	} else {
2099 		/*
2100 		 * Save away how much memory we allocated. If we're doing a
2101 		 * nosleep, the alloc could fail...
2102 		 */
2103 		dma->dp_cookie_size = sinfo->si_max_pages *
2104 		    sizeof (ddi_dma_cookie_t);
2105 		dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2106 		if (dma->dp_cookies == NULL) {
2107 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2108 			rootnex_clean_dmahdl(hp);
2109 			return (DDI_DMA_NORESOURCES);
2110 		}
2111 		dma->dp_need_to_free_cookie = B_TRUE;
2112 		ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2113 		    uint_t, sinfo->si_max_pages);
2114 	}
2115 	hp->dmai_cookie = dma->dp_cookies;
2116 
2117 	/*
2118 	 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2119 	 * looking at the constraints in the dma structure. It will then put
2120 	 * some additional state about the sgl in the dma struct (i.e. is
2121 	 * the sgl clean, or do we need to do some munging; how many pages
2122 	 * need to be copied, etc.)
2123 	 */
2124 	if (dma->dp_dvma_used)
2125 		rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2126 	else
2127 		rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2128 
2129 out:
2130 	ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2131 	/* if we don't need a copy buffer, we don't need to sync */
2132 	if (sinfo->si_copybuf_req == 0) {
2133 		hp->dmai_rflags |= DMP_NOSYNC;
2134 	}
2135 
2136 	/*
2137 	 * if we don't need the copybuf and we don't need to do a partial,  we
2138 	 * hit the fast path. All the high performance devices should be trying
2139 	 * to hit this path. To hit this path, a device should be able to reach
2140 	 * all of memory, shouldn't try to bind more than it can transfer, and
2141 	 * the buffer shouldn't require more cookies than the driver/device can
2142 	 * handle [sgllen]).
2143 	 *
2144 	 * Note that negative values of dma_attr_sgllen are supposed
2145 	 * to mean unlimited, but we just cast them to mean a
2146 	 * "ridiculous large limit".  This saves some extra checks on
2147 	 * hot paths.
2148 	 */
2149 	if ((sinfo->si_copybuf_req == 0) &&
2150 	    (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2151 	    (dmao->dmao_size <= dma->dp_maxxfer)) {
2152 fast:
2153 		/*
2154 		 * If the driver supports FMA, insert the handle in the FMA DMA
2155 		 * handle cache.
2156 		 */
2157 		if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2158 			hp->dmai_error.err_cf = rootnex_dma_check;
2159 
2160 		/*
2161 		 * copy out the first cookie and ccountp, set the cookie
2162 		 * pointer to the second cookie. The first cookie is passed
2163 		 * back on the stack. Additional cookies are accessed via
2164 		 * ddi_dma_nextcookie()
2165 		 */
2166 		*cookiep = dma->dp_cookies[0];
2167 		*ccountp = sinfo->si_sgl_size;
2168 		hp->dmai_cookie++;
2169 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2170 		hp->dmai_ncookies = *ccountp;
2171 		hp->dmai_curcookie = 1;
2172 		ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2173 		ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2174 		    uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2175 		    uint_t, dmao->dmao_size, uint_t, *ccountp);
2176 
2177 
2178 		return (DDI_DMA_MAPPED);
2179 	}
2180 
2181 	/*
2182 	 * go to the slow path, we may need to alloc more memory, create
2183 	 * multiple windows, and munge up a sgl to make the device happy.
2184 	 */
2185 
2186 	/*
2187 	 * With the IOMMU mapobject method used, we should never hit
2188 	 * the slow path. If we do, something is seriously wrong.
2189 	 * Clean up and return an error.
2190 	 */
2191 
2192 #if !defined(__xpv)
2193 
2194 	if (dma->dp_dvma_used) {
2195 		(void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2196 		    &dma->dp_dvma);
2197 		e = DDI_DMA_NOMAPPING;
2198 	} else {
2199 #endif
2200 		e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2201 		    kmflag);
2202 #if !defined(__xpv)
2203 	}
2204 #endif
2205 	if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2206 		if (dma->dp_need_to_free_cookie) {
2207 			kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2208 		}
2209 		ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2210 		rootnex_clean_dmahdl(hp); /* must be after free cookie */
2211 		return (e);
2212 	}
2213 
2214 	/*
2215 	 * If the driver supports FMA, insert the handle in the FMA DMA handle
2216 	 * cache.
2217 	 */
2218 	if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2219 		hp->dmai_error.err_cf = rootnex_dma_check;
2220 
2221 	/* if the first window uses the copy buffer, sync it for the device */
2222 	if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2223 	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
2224 		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2225 		    DDI_DMA_SYNC_FORDEV);
2226 	}
2227 
2228 	/*
2229 	 * copy out the first cookie and ccountp, set the cookie pointer to the
2230 	 * second cookie. Make sure the partial flag is set/cleared correctly.
2231 	 * If we have a partial map (i.e. multiple windows), the number of
2232 	 * cookies we return is the number of cookies in the first window.
2233 	 */
2234 	if (e == DDI_DMA_MAPPED) {
2235 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2236 		*ccountp = sinfo->si_sgl_size;
2237 		hp->dmai_nwin = 1;
2238 	} else {
2239 		hp->dmai_rflags |= DDI_DMA_PARTIAL;
2240 		*ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2241 		ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2242 	}
2243 	*cookiep = dma->dp_cookies[0];
2244 	hp->dmai_cookie++;
2245 	hp->dmai_ncookies = *ccountp;
2246 	hp->dmai_curcookie = 1;
2247 
2248 	ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2249 	ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2250 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2251 	    dmao->dmao_size, uint_t, *ccountp);
2252 	return (e);
2253 }
2254 
2255 /*
2256  * rootnex_dma_bindhdl()
2257  *    called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2258  */
2259 static int
rootnex_dma_bindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,struct ddi_dma_req * dmareq,ddi_dma_cookie_t * cookiep,uint_t * ccountp)2260 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2261     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2262     ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2263 {
2264 	int ret;
2265 #if !defined(__xpv)
2266 	if (IOMMU_USED(rdip))
2267 		ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2268 		    cookiep, ccountp);
2269 	else
2270 #endif
2271 	ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2272 	    cookiep, ccountp);
2273 
2274 	if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2275 		ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2276 		    &rootnex_state->r_dvma_call_list_id);
2277 	}
2278 
2279 	return (ret);
2280 }
2281 
2282 
2283 
2284 /*ARGSUSED*/
2285 static int
rootnex_coredma_unbindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)2286 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2287     ddi_dma_handle_t handle)
2288 {
2289 	ddi_dma_impl_t *hp;
2290 	rootnex_dma_t *dma;
2291 	int e;
2292 
2293 	hp = (ddi_dma_impl_t *)handle;
2294 	dma = (rootnex_dma_t *)hp->dmai_private;
2295 
2296 	/* make sure the buffer wasn't free'd before calling unbind */
2297 	if (rootnex_unbind_verify_buffer) {
2298 		e = rootnex_verify_buffer(dma);
2299 		if (e != DDI_SUCCESS) {
2300 			ASSERT(0);
2301 			return (DDI_FAILURE);
2302 		}
2303 	}
2304 
2305 	/* sync the current window before unbinding the buffer */
2306 	if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2307 	    (hp->dmai_rflags & DDI_DMA_READ)) {
2308 		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2309 		    DDI_DMA_SYNC_FORCPU);
2310 	}
2311 
2312 	/*
2313 	 * cleanup and copy buffer or window state. if we didn't use the copy
2314 	 * buffer or windows, there won't be much to do :-)
2315 	 */
2316 	rootnex_teardown_copybuf(dma);
2317 	rootnex_teardown_windows(dma);
2318 
2319 #if !defined(__xpv)
2320 	if (IOMMU_USED(rdip) && dma->dp_dvma_used)
2321 		(void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2322 		    &dma->dp_dvma);
2323 #endif
2324 
2325 	/*
2326 	 * If we had to allocate space to for the worse case sgl (it didn't
2327 	 * fit into our pre-allocate buffer), free that up now
2328 	 */
2329 	if (dma->dp_need_to_free_cookie) {
2330 		kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2331 	}
2332 
2333 	/*
2334 	 * clean up the handle so it's ready for the next bind (i.e. if the
2335 	 * handle is reused).
2336 	 */
2337 	rootnex_clean_dmahdl(hp);
2338 	hp->dmai_error.err_cf = NULL;
2339 
2340 	ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2341 	ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2342 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2343 
2344 	return (DDI_SUCCESS);
2345 }
2346 
2347 /*
2348  * rootnex_dma_unbindhdl()
2349  *    called from ddi_dma_unbind_handle()
2350  */
2351 /*ARGSUSED*/
2352 static int
rootnex_dma_unbindhdl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)2353 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2354     ddi_dma_handle_t handle)
2355 {
2356 	int ret;
2357 
2358 #if !defined(__xpv)
2359 	if (IOMMU_USED(rdip))
2360 		ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2361 	else
2362 #endif
2363 	ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2364 
2365 	if (rootnex_state->r_dvma_call_list_id)
2366 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2367 
2368 	return (ret);
2369 }
2370 
2371 #if !defined(__xpv)
2372 
2373 static int
rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)2374 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2375 {
2376 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2377 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2378 
2379 	if (dma->dp_sleep_flags != KM_SLEEP &&
2380 	    dma->dp_sleep_flags != KM_NOSLEEP)
2381 		cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2382 	return (dma->dp_sleep_flags);
2383 }
2384 /*ARGSUSED*/
2385 static void
rootnex_coredma_reset_cookies(dev_info_t * dip,ddi_dma_handle_t handle)2386 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2387 {
2388 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2389 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2390 	rootnex_window_t *window;
2391 
2392 	if (dma->dp_window) {
2393 		window = &dma->dp_window[dma->dp_current_win];
2394 		hp->dmai_cookie = window->wd_first_cookie;
2395 	} else {
2396 		hp->dmai_cookie = dma->dp_cookies;
2397 	}
2398 	hp->dmai_cookie++;
2399 	hp->dmai_curcookie = 1;
2400 }
2401 
2402 /*ARGSUSED*/
2403 static int
rootnex_coredma_get_cookies(dev_info_t * dip,ddi_dma_handle_t handle,ddi_dma_cookie_t ** cookiepp,uint_t * ccountp)2404 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2405     ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2406 {
2407 	int i;
2408 	int km_flags;
2409 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2410 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2411 	rootnex_window_t *window;
2412 	ddi_dma_cookie_t *cp;
2413 	ddi_dma_cookie_t *cookie;
2414 
2415 	ASSERT(*cookiepp == NULL);
2416 	ASSERT(*ccountp == 0);
2417 
2418 	if (dma->dp_window) {
2419 		window = &dma->dp_window[dma->dp_current_win];
2420 		cp = window->wd_first_cookie;
2421 		*ccountp = window->wd_cookie_cnt;
2422 	} else {
2423 		cp = dma->dp_cookies;
2424 		*ccountp = dma->dp_sglinfo.si_sgl_size;
2425 	}
2426 
2427 	km_flags = rootnex_coredma_get_sleep_flags(handle);
2428 	cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2429 	if (cookie == NULL) {
2430 		return (DDI_DMA_NORESOURCES);
2431 	}
2432 
2433 	for (i = 0; i < *ccountp; i++) {
2434 		cookie[i].dmac_notused = cp[i].dmac_notused;
2435 		cookie[i].dmac_type = cp[i].dmac_type;
2436 		cookie[i].dmac_address = cp[i].dmac_address;
2437 		cookie[i].dmac_size = cp[i].dmac_size;
2438 	}
2439 
2440 	*cookiepp = cookie;
2441 
2442 	return (DDI_SUCCESS);
2443 }
2444 
2445 /*ARGSUSED*/
2446 static int
rootnex_coredma_set_cookies(dev_info_t * dip,ddi_dma_handle_t handle,ddi_dma_cookie_t * cookiep,uint_t ccount)2447 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2448     ddi_dma_cookie_t *cookiep, uint_t ccount)
2449 {
2450 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2451 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2452 	rootnex_window_t *window;
2453 	ddi_dma_cookie_t *cur_cookiep;
2454 
2455 	ASSERT(cookiep);
2456 	ASSERT(ccount != 0);
2457 	ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2458 
2459 	if (dma->dp_window) {
2460 		window = &dma->dp_window[dma->dp_current_win];
2461 		dma->dp_saved_cookies = window->wd_first_cookie;
2462 		window->wd_first_cookie = cookiep;
2463 		ASSERT(ccount == window->wd_cookie_cnt);
2464 		cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2465 		    + window->wd_first_cookie;
2466 	} else {
2467 		dma->dp_saved_cookies = dma->dp_cookies;
2468 		dma->dp_cookies = cookiep;
2469 		ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2470 		cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2471 		    + dma->dp_cookies;
2472 	}
2473 
2474 	dma->dp_need_to_switch_cookies = B_TRUE;
2475 	hp->dmai_cookie = cur_cookiep;
2476 
2477 	return (DDI_SUCCESS);
2478 }
2479 
2480 /*ARGSUSED*/
2481 static int
rootnex_coredma_clear_cookies(dev_info_t * dip,ddi_dma_handle_t handle)2482 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2483 {
2484 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2485 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2486 	rootnex_window_t *window;
2487 	ddi_dma_cookie_t *cur_cookiep;
2488 	ddi_dma_cookie_t *cookie_array;
2489 	uint_t ccount;
2490 
2491 	/* check if cookies have not been switched */
2492 	if (dma->dp_need_to_switch_cookies == B_FALSE)
2493 		return (DDI_SUCCESS);
2494 
2495 	ASSERT(dma->dp_saved_cookies);
2496 
2497 	if (dma->dp_window) {
2498 		window = &dma->dp_window[dma->dp_current_win];
2499 		cookie_array = window->wd_first_cookie;
2500 		window->wd_first_cookie = dma->dp_saved_cookies;
2501 		dma->dp_saved_cookies = NULL;
2502 		ccount = window->wd_cookie_cnt;
2503 		cur_cookiep = (hp->dmai_cookie - cookie_array)
2504 		    + window->wd_first_cookie;
2505 	} else {
2506 		cookie_array = dma->dp_cookies;
2507 		dma->dp_cookies = dma->dp_saved_cookies;
2508 		dma->dp_saved_cookies = NULL;
2509 		ccount = dma->dp_sglinfo.si_sgl_size;
2510 		cur_cookiep = (hp->dmai_cookie - cookie_array)
2511 		    + dma->dp_cookies;
2512 	}
2513 
2514 	kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2515 
2516 	hp->dmai_cookie = cur_cookiep;
2517 
2518 	dma->dp_need_to_switch_cookies = B_FALSE;
2519 
2520 	return (DDI_SUCCESS);
2521 }
2522 
2523 #endif
2524 
2525 static struct as *
rootnex_get_as(ddi_dma_obj_t * dmao)2526 rootnex_get_as(ddi_dma_obj_t *dmao)
2527 {
2528 	struct as *asp;
2529 
2530 	switch (dmao->dmao_type) {
2531 	case DMA_OTYP_VADDR:
2532 	case DMA_OTYP_BUFVADDR:
2533 		asp = dmao->dmao_obj.virt_obj.v_as;
2534 		if (asp == NULL)
2535 			asp = &kas;
2536 		break;
2537 	default:
2538 		asp = NULL;
2539 		break;
2540 	}
2541 	return (asp);
2542 }
2543 
2544 /*
2545  * rootnex_verify_buffer()
2546  *   verify buffer wasn't free'd
2547  */
2548 static int
rootnex_verify_buffer(rootnex_dma_t * dma)2549 rootnex_verify_buffer(rootnex_dma_t *dma)
2550 {
2551 	page_t **pplist;
2552 	caddr_t vaddr;
2553 	uint_t pcnt;
2554 	uint_t poff;
2555 	page_t *pp;
2556 	char b;
2557 	int i;
2558 
2559 	/* Figure out how many pages this buffer occupies */
2560 	if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2561 		poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2562 	} else {
2563 		vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2564 		poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2565 	}
2566 	pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2567 
2568 	switch (dma->dp_dma.dmao_type) {
2569 	case DMA_OTYP_PAGES:
2570 		/*
2571 		 * for a linked list of pp's walk through them to make sure
2572 		 * they're locked and not free.
2573 		 */
2574 		pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2575 		for (i = 0; i < pcnt; i++) {
2576 			if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2577 				return (DDI_FAILURE);
2578 			}
2579 			pp = pp->p_next;
2580 		}
2581 		break;
2582 
2583 	case DMA_OTYP_VADDR:
2584 	case DMA_OTYP_BUFVADDR:
2585 		pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2586 		/*
2587 		 * for an array of pp's walk through them to make sure they're
2588 		 * not free. It's possible that they may not be locked.
2589 		 */
2590 		if (pplist) {
2591 			for (i = 0; i < pcnt; i++) {
2592 				if (PP_ISFREE(pplist[i])) {
2593 					return (DDI_FAILURE);
2594 				}
2595 			}
2596 
2597 		/* For a virtual address, try to peek at each page */
2598 		} else {
2599 			if (rootnex_get_as(&dma->dp_dma) == &kas) {
2600 				for (i = 0; i < pcnt; i++) {
2601 					if (ddi_peek8(NULL, vaddr, &b) ==
2602 					    DDI_FAILURE)
2603 						return (DDI_FAILURE);
2604 					vaddr += MMU_PAGESIZE;
2605 				}
2606 			}
2607 		}
2608 		break;
2609 
2610 	default:
2611 		cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2612 		break;
2613 	}
2614 
2615 	return (DDI_SUCCESS);
2616 }
2617 
2618 
2619 /*
2620  * rootnex_clean_dmahdl()
2621  *    Clean the dma handle. This should be called on a handle alloc and an
2622  *    unbind handle. Set the handle state to the default settings.
2623  */
2624 static void
rootnex_clean_dmahdl(ddi_dma_impl_t * hp)2625 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2626 {
2627 	rootnex_dma_t *dma;
2628 
2629 
2630 	dma = (rootnex_dma_t *)hp->dmai_private;
2631 
2632 	hp->dmai_nwin = 0;
2633 	dma->dp_current_cookie = 0;
2634 	dma->dp_copybuf_size = 0;
2635 	dma->dp_window = NULL;
2636 	dma->dp_cbaddr = NULL;
2637 	dma->dp_inuse = B_FALSE;
2638 	dma->dp_dvma_used = B_FALSE;
2639 	dma->dp_need_to_free_cookie = B_FALSE;
2640 	dma->dp_need_to_switch_cookies = B_FALSE;
2641 	dma->dp_saved_cookies = NULL;
2642 	dma->dp_sleep_flags = KM_PANIC;
2643 	dma->dp_need_to_free_window = B_FALSE;
2644 	dma->dp_partial_required = B_FALSE;
2645 	dma->dp_trim_required = B_FALSE;
2646 	dma->dp_sglinfo.si_copybuf_req = 0;
2647 
2648 	/* FMA related initialization */
2649 	hp->dmai_fault = 0;
2650 	hp->dmai_fault_check = NULL;
2651 	hp->dmai_fault_notify = NULL;
2652 	hp->dmai_error.err_ena = 0;
2653 	hp->dmai_error.err_status = DDI_FM_OK;
2654 	hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2655 	hp->dmai_error.err_ontrap = NULL;
2656 
2657 	/* Cookie tracking */
2658 	hp->dmai_ncookies = 0;
2659 	hp->dmai_curcookie = 0;
2660 }
2661 
2662 
2663 /*
2664  * rootnex_valid_alloc_parms()
2665  *    Called in ddi_dma_alloc_handle path to validate its parameters.
2666  */
2667 static int
rootnex_valid_alloc_parms(ddi_dma_attr_t * attr,uint_t maxsegmentsize)2668 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2669 {
2670 	if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2671 	    (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2672 	    (attr->dma_attr_granular > MMU_PAGESIZE) ||
2673 	    (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2674 		return (DDI_DMA_BADATTR);
2675 	}
2676 
2677 	if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2678 		return (DDI_DMA_BADATTR);
2679 	}
2680 
2681 	if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2682 	    MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2683 	    attr->dma_attr_sgllen == 0) {
2684 		return (DDI_DMA_BADATTR);
2685 	}
2686 
2687 	/* We should be able to DMA into every byte offset in a page */
2688 	if (maxsegmentsize < MMU_PAGESIZE) {
2689 		return (DDI_DMA_BADATTR);
2690 	}
2691 
2692 	/* if we're bouncing on seg, seg must be <= addr_hi */
2693 	if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2694 	    (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2695 		return (DDI_DMA_BADATTR);
2696 	}
2697 	return (DDI_SUCCESS);
2698 }
2699 
2700 /*
2701  * rootnex_need_bounce_seg()
2702  *    check to see if the buffer lives on both side of the seg.
2703  */
2704 static boolean_t
rootnex_need_bounce_seg(ddi_dma_obj_t * dmar_object,rootnex_sglinfo_t * sglinfo)2705 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2706 {
2707 	ddi_dma_atyp_t buftype;
2708 	rootnex_addr_t raddr;
2709 	boolean_t lower_addr;
2710 	boolean_t upper_addr;
2711 	uint64_t offset;
2712 	page_t **pplist;
2713 	uint64_t paddr;
2714 	uint32_t psize;
2715 	uint32_t size;
2716 	caddr_t vaddr;
2717 	uint_t pcnt;
2718 	page_t *pp;
2719 
2720 	pp = NULL;
2721 	/* shortcuts */
2722 	pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2723 	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2724 	buftype = dmar_object->dmao_type;
2725 	size = dmar_object->dmao_size;
2726 
2727 	lower_addr = B_FALSE;
2728 	upper_addr = B_FALSE;
2729 	pcnt = 0;
2730 
2731 	/*
2732 	 * Process the first page to handle the initial offset of the buffer.
2733 	 * We'll use the base address we get later when we loop through all
2734 	 * the pages.
2735 	 */
2736 	if (buftype == DMA_OTYP_PAGES) {
2737 		pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2738 		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2739 		    MMU_PAGEOFFSET;
2740 		paddr = pfn_to_pa(pp->p_pagenum) + offset;
2741 		psize = MIN(size, (MMU_PAGESIZE - offset));
2742 		pp = pp->p_next;
2743 		sglinfo->si_asp = NULL;
2744 	} else if (pplist != NULL) {
2745 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2746 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2747 		if (sglinfo->si_asp == NULL) {
2748 			sglinfo->si_asp = &kas;
2749 		}
2750 		paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2751 		paddr += offset;
2752 		psize = MIN(size, (MMU_PAGESIZE - offset));
2753 		pcnt++;
2754 	} else {
2755 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2756 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2757 		if (sglinfo->si_asp == NULL) {
2758 			sglinfo->si_asp = &kas;
2759 		}
2760 		paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2761 		paddr += offset;
2762 		psize = MIN(size, (MMU_PAGESIZE - offset));
2763 		vaddr += psize;
2764 	}
2765 
2766 	raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2767 
2768 	if ((raddr + psize) > sglinfo->si_segmask) {
2769 		upper_addr = B_TRUE;
2770 	} else {
2771 		lower_addr = B_TRUE;
2772 	}
2773 	size -= psize;
2774 
2775 	/*
2776 	 * Walk through the rest of the pages in the buffer. Track to see
2777 	 * if we have pages on both sides of the segment boundary.
2778 	 */
2779 	while (size > 0) {
2780 		/* partial or full page */
2781 		psize = MIN(size, MMU_PAGESIZE);
2782 
2783 		if (buftype == DMA_OTYP_PAGES) {
2784 			/* get the paddr from the page_t */
2785 			ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2786 			paddr = pfn_to_pa(pp->p_pagenum);
2787 			pp = pp->p_next;
2788 		} else if (pplist != NULL) {
2789 			/* index into the array of page_t's to get the paddr */
2790 			ASSERT(!PP_ISFREE(pplist[pcnt]));
2791 			paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2792 			pcnt++;
2793 		} else {
2794 			/* call into the VM to get the paddr */
2795 			paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2796 			    vaddr));
2797 			vaddr += psize;
2798 		}
2799 
2800 		raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2801 
2802 		if ((raddr + psize) > sglinfo->si_segmask) {
2803 			upper_addr = B_TRUE;
2804 		} else {
2805 			lower_addr = B_TRUE;
2806 		}
2807 		/*
2808 		 * if the buffer lives both above and below the segment
2809 		 * boundary, or the current page is the page immediately
2810 		 * after the segment, we will use a copy/bounce buffer for
2811 		 * all pages > seg.
2812 		 */
2813 		if ((lower_addr && upper_addr) ||
2814 		    (raddr == (sglinfo->si_segmask + 1))) {
2815 			return (B_TRUE);
2816 		}
2817 
2818 		size -= psize;
2819 	}
2820 
2821 	return (B_FALSE);
2822 }
2823 
2824 /*
2825  * rootnex_get_sgl()
2826  *    Called in bind fastpath to get the sgl. Most of this will be replaced
2827  *    with a call to the vm layer when vm2.0 comes around...
2828  */
2829 static void
rootnex_get_sgl(ddi_dma_obj_t * dmar_object,ddi_dma_cookie_t * sgl,rootnex_sglinfo_t * sglinfo)2830 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2831     rootnex_sglinfo_t *sglinfo)
2832 {
2833 	ddi_dma_atyp_t buftype;
2834 	rootnex_addr_t raddr;
2835 	uint64_t last_page;
2836 	uint64_t offset;
2837 	uint64_t addrhi;
2838 	uint64_t addrlo;
2839 	uint64_t maxseg;
2840 	page_t **pplist;
2841 	uint64_t paddr;
2842 	uint32_t psize;
2843 	uint32_t size;
2844 	caddr_t vaddr;
2845 	uint_t pcnt;
2846 	page_t *pp;
2847 	uint_t cnt;
2848 
2849 	pp = NULL;
2850 	/* shortcuts */
2851 	pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2852 	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2853 	maxseg = sglinfo->si_max_cookie_size;
2854 	buftype = dmar_object->dmao_type;
2855 	addrhi = sglinfo->si_max_addr;
2856 	addrlo = sglinfo->si_min_addr;
2857 	size = dmar_object->dmao_size;
2858 
2859 	pcnt = 0;
2860 	cnt = 0;
2861 
2862 
2863 	/*
2864 	 * check to see if we need to use the copy buffer for pages over
2865 	 * the segment attr.
2866 	 */
2867 	sglinfo->si_bounce_on_seg = B_FALSE;
2868 	if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2869 		sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2870 		    dmar_object, sglinfo);
2871 	}
2872 
2873 	/*
2874 	 * if we were passed down a linked list of pages, i.e. pointer to
2875 	 * page_t, use this to get our physical address and buf offset.
2876 	 */
2877 	if (buftype == DMA_OTYP_PAGES) {
2878 		pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2879 		ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2880 		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2881 		    MMU_PAGEOFFSET;
2882 		paddr = pfn_to_pa(pp->p_pagenum) + offset;
2883 		psize = MIN(size, (MMU_PAGESIZE - offset));
2884 		pp = pp->p_next;
2885 		sglinfo->si_asp = NULL;
2886 
2887 	/*
2888 	 * We weren't passed down a linked list of pages, but if we were passed
2889 	 * down an array of pages, use this to get our physical address and buf
2890 	 * offset.
2891 	 */
2892 	} else if (pplist != NULL) {
2893 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2894 		    (buftype == DMA_OTYP_BUFVADDR));
2895 
2896 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2897 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2898 		if (sglinfo->si_asp == NULL) {
2899 			sglinfo->si_asp = &kas;
2900 		}
2901 
2902 		ASSERT(!PP_ISFREE(pplist[pcnt]));
2903 		paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2904 		paddr += offset;
2905 		psize = MIN(size, (MMU_PAGESIZE - offset));
2906 		pcnt++;
2907 
2908 	/*
2909 	 * All we have is a virtual address, we'll need to call into the VM
2910 	 * to get the physical address.
2911 	 */
2912 	} else {
2913 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2914 		    (buftype == DMA_OTYP_BUFVADDR));
2915 
2916 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2917 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2918 		if (sglinfo->si_asp == NULL) {
2919 			sglinfo->si_asp = &kas;
2920 		}
2921 
2922 		paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2923 		paddr += offset;
2924 		psize = MIN(size, (MMU_PAGESIZE - offset));
2925 		vaddr += psize;
2926 	}
2927 
2928 	raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2929 
2930 	/*
2931 	 * Setup the first cookie with the physical address of the page and the
2932 	 * size of the page (which takes into account the initial offset into
2933 	 * the page.
2934 	 */
2935 	sgl[cnt].dmac_laddress = raddr;
2936 	sgl[cnt].dmac_size = psize;
2937 	sgl[cnt].dmac_type = 0;
2938 
2939 	/*
2940 	 * Save away the buffer offset into the page. We'll need this later in
2941 	 * the copy buffer code to help figure out the page index within the
2942 	 * buffer and the offset into the current page.
2943 	 */
2944 	sglinfo->si_buf_offset = offset;
2945 
2946 	/*
2947 	 * If we are using the copy buffer for anything over the segment
2948 	 * boundary, and this page is over the segment boundary.
2949 	 *   OR
2950 	 * if the DMA engine can't reach the physical address.
2951 	 */
2952 	if (((sglinfo->si_bounce_on_seg) &&
2953 	    ((raddr + psize) > sglinfo->si_segmask)) ||
2954 	    ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
2955 		/*
2956 		 * Increase how much copy buffer we use. We always increase by
2957 		 * pagesize so we don't have to worry about converting offsets.
2958 		 * Set a flag in the cookies dmac_type to indicate that it uses
2959 		 * the copy buffer. If this isn't the last cookie, go to the
2960 		 * next cookie (since we separate each page which uses the copy
2961 		 * buffer in case the copy buffer is not physically contiguous.
2962 		 */
2963 		sglinfo->si_copybuf_req += MMU_PAGESIZE;
2964 		sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
2965 		if ((cnt + 1) < sglinfo->si_max_pages) {
2966 			cnt++;
2967 			sgl[cnt].dmac_laddress = 0;
2968 			sgl[cnt].dmac_size = 0;
2969 			sgl[cnt].dmac_type = 0;
2970 		}
2971 	}
2972 
2973 	/*
2974 	 * save this page's physical address so we can figure out if the next
2975 	 * page is physically contiguous. Keep decrementing size until we are
2976 	 * done with the buffer.
2977 	 */
2978 	last_page = raddr & MMU_PAGEMASK;
2979 	size -= psize;
2980 
2981 	while (size > 0) {
2982 		/* Get the size for this page (i.e. partial or full page) */
2983 		psize = MIN(size, MMU_PAGESIZE);
2984 
2985 		if (buftype == DMA_OTYP_PAGES) {
2986 			/* get the paddr from the page_t */
2987 			ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2988 			paddr = pfn_to_pa(pp->p_pagenum);
2989 			pp = pp->p_next;
2990 		} else if (pplist != NULL) {
2991 			/* index into the array of page_t's to get the paddr */
2992 			ASSERT(!PP_ISFREE(pplist[pcnt]));
2993 			paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2994 			pcnt++;
2995 		} else {
2996 			/* call into the VM to get the paddr */
2997 			paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2998 			    vaddr));
2999 			vaddr += psize;
3000 		}
3001 
3002 		raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3003 
3004 		/*
3005 		 * If we are using the copy buffer for anything over the
3006 		 * segment boundary, and this page is over the segment
3007 		 * boundary.
3008 		 *   OR
3009 		 * if the DMA engine can't reach the physical address.
3010 		 */
3011 		if (((sglinfo->si_bounce_on_seg) &&
3012 		    ((raddr + psize) > sglinfo->si_segmask)) ||
3013 		    ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3014 
3015 			sglinfo->si_copybuf_req += MMU_PAGESIZE;
3016 
3017 			/*
3018 			 * if there is something in the current cookie, go to
3019 			 * the next one. We only want one page in a cookie which
3020 			 * uses the copybuf since the copybuf doesn't have to
3021 			 * be physically contiguous.
3022 			 */
3023 			if (sgl[cnt].dmac_size != 0) {
3024 				cnt++;
3025 			}
3026 			sgl[cnt].dmac_laddress = raddr;
3027 			sgl[cnt].dmac_size = psize;
3028 			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3029 			/* if this isn't the last cookie, go to the next one */
3030 			if ((cnt + 1) < sglinfo->si_max_pages) {
3031 				cnt++;
3032 				sgl[cnt].dmac_laddress = 0;
3033 				sgl[cnt].dmac_size = 0;
3034 				sgl[cnt].dmac_type = 0;
3035 			}
3036 
3037 		/*
3038 		 * this page didn't need the copy buffer, if it's not physically
3039 		 * contiguous, or it would put us over a segment boundary, or it
3040 		 * puts us over the max cookie size, or the current sgl doesn't
3041 		 * have anything in it.
3042 		 */
3043 		} else if (((last_page + MMU_PAGESIZE) != raddr) ||
3044 		    !(raddr & sglinfo->si_segmask) ||
3045 		    ((sgl[cnt].dmac_size + psize) > maxseg) ||
3046 		    (sgl[cnt].dmac_size == 0)) {
3047 			/*
3048 			 * if we're not already in a new cookie, go to the next
3049 			 * cookie.
3050 			 */
3051 			if (sgl[cnt].dmac_size != 0) {
3052 				cnt++;
3053 			}
3054 
3055 			/* save the cookie information */
3056 			sgl[cnt].dmac_laddress = raddr;
3057 			sgl[cnt].dmac_size = psize;
3058 			sgl[cnt].dmac_type = 0;
3059 
3060 		/*
3061 		 * this page didn't need the copy buffer, it is physically
3062 		 * contiguous with the last page, and it's <= the max cookie
3063 		 * size.
3064 		 */
3065 		} else {
3066 			sgl[cnt].dmac_size += psize;
3067 
3068 			/*
3069 			 * If this cookie is used up, and more cookies
3070 			 * are available, then move onto the next one.
3071 			 */
3072 			if ((sgl[cnt].dmac_size == maxseg) &&
3073 			    ((cnt + 1) < sglinfo->si_max_pages)) {
3074 				cnt++;
3075 				sgl[cnt].dmac_laddress = 0;
3076 				sgl[cnt].dmac_size = 0;
3077 				sgl[cnt].dmac_type = 0;
3078 			}
3079 		}
3080 
3081 		/*
3082 		 * save this page's physical address so we can figure out if the
3083 		 * next page is physically contiguous. Keep decrementing size
3084 		 * until we are done with the buffer.
3085 		 */
3086 		last_page = raddr;
3087 		size -= psize;
3088 	}
3089 
3090 	/* we're done, save away how many cookies the sgl has */
3091 	if (sgl[cnt].dmac_size == 0) {
3092 		ASSERT(cnt < sglinfo->si_max_pages);
3093 		sglinfo->si_sgl_size = cnt;
3094 	} else {
3095 		sglinfo->si_sgl_size = cnt + 1;
3096 	}
3097 }
3098 
3099 static void
rootnex_dvma_get_sgl(ddi_dma_obj_t * dmar_object,ddi_dma_cookie_t * sgl,rootnex_sglinfo_t * sglinfo)3100 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3101     rootnex_sglinfo_t *sglinfo)
3102 {
3103 	uint64_t offset;
3104 	uint64_t maxseg;
3105 	uint64_t dvaddr;
3106 	struct dvmaseg *dvs;
3107 	uint64_t paddr;
3108 	uint32_t psize, ssize;
3109 	uint32_t size;
3110 	uint_t cnt;
3111 	int physcontig;
3112 
3113 	ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3114 
3115 	/* shortcuts */
3116 	maxseg = sglinfo->si_max_cookie_size;
3117 	size = dmar_object->dmao_size;
3118 
3119 	cnt = 0;
3120 	sglinfo->si_bounce_on_seg = B_FALSE;
3121 
3122 	dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3123 	offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3124 	ssize = dvs->dvs_len;
3125 	paddr = dvs->dvs_start;
3126 	paddr += offset;
3127 	psize = MIN(ssize, (maxseg - offset));
3128 	dvaddr = paddr + psize;
3129 	ssize -= psize;
3130 
3131 	sgl[cnt].dmac_laddress = paddr;
3132 	sgl[cnt].dmac_size = psize;
3133 	sgl[cnt].dmac_type = 0;
3134 
3135 	size -= psize;
3136 	while (size > 0) {
3137 		if (ssize == 0) {
3138 			dvs++;
3139 			ssize = dvs->dvs_len;
3140 			dvaddr = dvs->dvs_start;
3141 			physcontig = 0;
3142 		} else
3143 			physcontig = 1;
3144 
3145 		paddr = dvaddr;
3146 		psize = MIN(ssize, maxseg);
3147 		dvaddr += psize;
3148 		ssize -= psize;
3149 
3150 		if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3151 		    ((sgl[cnt].dmac_size + psize) > maxseg) ||
3152 		    (sgl[cnt].dmac_size == 0)) {
3153 			/*
3154 			 * if we're not already in a new cookie, go to the next
3155 			 * cookie.
3156 			 */
3157 			if (sgl[cnt].dmac_size != 0) {
3158 				cnt++;
3159 			}
3160 
3161 			/* save the cookie information */
3162 			sgl[cnt].dmac_laddress = paddr;
3163 			sgl[cnt].dmac_size = psize;
3164 			sgl[cnt].dmac_type = 0;
3165 		} else {
3166 			sgl[cnt].dmac_size += psize;
3167 
3168 			/*
3169 			 * If this cookie is used up, and more cookies
3170 			 * are available, then move onto the next one.
3171 			 */
3172 			if ((sgl[cnt].dmac_size == maxseg) &&
3173 			    ((cnt + 1) < sglinfo->si_max_pages)) {
3174 				cnt++;
3175 				sgl[cnt].dmac_laddress = 0;
3176 				sgl[cnt].dmac_size = 0;
3177 				sgl[cnt].dmac_type = 0;
3178 			}
3179 		}
3180 		size -= psize;
3181 	}
3182 
3183 	/* we're done, save away how many cookies the sgl has */
3184 	if (sgl[cnt].dmac_size == 0) {
3185 		sglinfo->si_sgl_size = cnt;
3186 	} else {
3187 		sglinfo->si_sgl_size = cnt + 1;
3188 	}
3189 }
3190 
3191 /*
3192  * rootnex_bind_slowpath()
3193  *    Call in the bind path if the calling driver can't use the sgl without
3194  *    modifying it. We either need to use the copy buffer and/or we will end up
3195  *    with a partial bind.
3196  */
3197 static int
rootnex_bind_slowpath(ddi_dma_impl_t * hp,struct ddi_dma_req * dmareq,rootnex_dma_t * dma,ddi_dma_attr_t * attr,ddi_dma_obj_t * dmao,int kmflag)3198 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3199     rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3200 {
3201 	rootnex_sglinfo_t *sinfo;
3202 	rootnex_window_t *window;
3203 	ddi_dma_cookie_t *cookie;
3204 	size_t copybuf_used;
3205 	size_t dmac_size;
3206 	boolean_t partial;
3207 	off_t cur_offset;
3208 	page_t *cur_pp;
3209 	major_t mnum;
3210 	int e;
3211 	int i;
3212 
3213 
3214 	sinfo = &dma->dp_sglinfo;
3215 	copybuf_used = 0;
3216 	partial = B_FALSE;
3217 
3218 	/*
3219 	 * If we're using the copybuf, set the copybuf state in dma struct.
3220 	 * Needs to be first since it sets the copy buffer size.
3221 	 */
3222 	if (sinfo->si_copybuf_req != 0) {
3223 		e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3224 		if (e != DDI_SUCCESS) {
3225 			return (e);
3226 		}
3227 	} else {
3228 		dma->dp_copybuf_size = 0;
3229 	}
3230 
3231 	/*
3232 	 * Figure out if we need to do a partial mapping. If so, figure out
3233 	 * if we need to trim the buffers when we munge the sgl.
3234 	 */
3235 	if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3236 	    (dmao->dmao_size > dma->dp_maxxfer) ||
3237 	    ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3238 		dma->dp_partial_required = B_TRUE;
3239 		if (attr->dma_attr_granular != 1) {
3240 			dma->dp_trim_required = B_TRUE;
3241 		}
3242 	} else {
3243 		dma->dp_partial_required = B_FALSE;
3244 		dma->dp_trim_required = B_FALSE;
3245 	}
3246 
3247 	/* If we need to do a partial bind, make sure the driver supports it */
3248 	if (dma->dp_partial_required &&
3249 	    !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3250 
3251 		mnum = ddi_driver_major(dma->dp_dip);
3252 		/*
3253 		 * patchable which allows us to print one warning per major
3254 		 * number.
3255 		 */
3256 		if ((rootnex_bind_warn) &&
3257 		    ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3258 			rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3259 			cmn_err(CE_WARN, "!%s: coding error detected, the "
3260 			    "driver is using ddi_dma_attr(9S) incorrectly. "
3261 			    "There is a small risk of data corruption in "
3262 			    "particular with large I/Os. The driver should be "
3263 			    "replaced with a corrected version for proper "
3264 			    "system operation. To disable this warning, add "
3265 			    "'set rootnex:rootnex_bind_warn=0' to "
3266 			    "/etc/system(5).", ddi_driver_name(dma->dp_dip));
3267 		}
3268 		return (DDI_DMA_TOOBIG);
3269 	}
3270 
3271 	/*
3272 	 * we might need multiple windows, setup state to handle them. In this
3273 	 * code path, we will have at least one window.
3274 	 */
3275 	e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3276 	if (e != DDI_SUCCESS) {
3277 		rootnex_teardown_copybuf(dma);
3278 		return (e);
3279 	}
3280 
3281 	window = &dma->dp_window[0];
3282 	cookie = &dma->dp_cookies[0];
3283 	cur_offset = 0;
3284 	rootnex_init_win(hp, dma, window, cookie, cur_offset);
3285 	if (dmao->dmao_type == DMA_OTYP_PAGES) {
3286 		cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3287 	}
3288 
3289 	/* loop though all the cookies we got back from get_sgl() */
3290 	for (i = 0; i < sinfo->si_sgl_size; i++) {
3291 		/*
3292 		 * If we're using the copy buffer, check this cookie and setup
3293 		 * its associated copy buffer state. If this cookie uses the
3294 		 * copy buffer, make sure we sync this window during dma_sync.
3295 		 */
3296 		if (dma->dp_copybuf_size > 0) {
3297 			rootnex_setup_cookie(dmao, dma, cookie,
3298 			    cur_offset, &copybuf_used, &cur_pp);
3299 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3300 				window->wd_dosync = B_TRUE;
3301 			}
3302 		}
3303 
3304 		/*
3305 		 * save away the cookie size, since it could be modified in
3306 		 * the windowing code.
3307 		 */
3308 		dmac_size = cookie->dmac_size;
3309 
3310 		/* if we went over max copybuf size */
3311 		if (dma->dp_copybuf_size &&
3312 		    (copybuf_used > dma->dp_copybuf_size)) {
3313 			partial = B_TRUE;
3314 			e = rootnex_copybuf_window_boundary(hp, dma, &window,
3315 			    cookie, cur_offset, &copybuf_used);
3316 			if (e != DDI_SUCCESS) {
3317 				rootnex_teardown_copybuf(dma);
3318 				rootnex_teardown_windows(dma);
3319 				return (e);
3320 			}
3321 
3322 			/*
3323 			 * if the coookie uses the copy buffer, make sure the
3324 			 * new window we just moved to is set to sync.
3325 			 */
3326 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3327 				window->wd_dosync = B_TRUE;
3328 			}
3329 			ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3330 			    dma->dp_dip);
3331 
3332 		/* if the cookie cnt == max sgllen, move to the next window */
3333 		} else if (window->wd_cookie_cnt >=
3334 		    (unsigned)attr->dma_attr_sgllen) {
3335 			partial = B_TRUE;
3336 			ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3337 			e = rootnex_sgllen_window_boundary(hp, dma, &window,
3338 			    cookie, attr, cur_offset);
3339 			if (e != DDI_SUCCESS) {
3340 				rootnex_teardown_copybuf(dma);
3341 				rootnex_teardown_windows(dma);
3342 				return (e);
3343 			}
3344 
3345 			/*
3346 			 * if the coookie uses the copy buffer, make sure the
3347 			 * new window we just moved to is set to sync.
3348 			 */
3349 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3350 				window->wd_dosync = B_TRUE;
3351 			}
3352 			ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3353 			    dma->dp_dip);
3354 
3355 		/* else if we will be over maxxfer */
3356 		} else if ((window->wd_size + dmac_size) >
3357 		    dma->dp_maxxfer) {
3358 			partial = B_TRUE;
3359 			e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3360 			    cookie);
3361 			if (e != DDI_SUCCESS) {
3362 				rootnex_teardown_copybuf(dma);
3363 				rootnex_teardown_windows(dma);
3364 				return (e);
3365 			}
3366 
3367 			/*
3368 			 * if the coookie uses the copy buffer, make sure the
3369 			 * new window we just moved to is set to sync.
3370 			 */
3371 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3372 				window->wd_dosync = B_TRUE;
3373 			}
3374 			ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3375 			    dma->dp_dip);
3376 
3377 		/* else this cookie fits in the current window */
3378 		} else {
3379 			window->wd_cookie_cnt++;
3380 			window->wd_size += dmac_size;
3381 		}
3382 
3383 		/* track our offset into the buffer, go to the next cookie */
3384 		ASSERT(dmac_size <= dmao->dmao_size);
3385 		ASSERT(cookie->dmac_size <= dmac_size);
3386 		cur_offset += dmac_size;
3387 		cookie++;
3388 	}
3389 
3390 	/* if we ended up with a zero sized window in the end, clean it up */
3391 	if (window->wd_size == 0) {
3392 		hp->dmai_nwin--;
3393 		window--;
3394 	}
3395 
3396 	ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3397 
3398 	if (!partial) {
3399 		return (DDI_DMA_MAPPED);
3400 	}
3401 
3402 	ASSERT(dma->dp_partial_required);
3403 	return (DDI_DMA_PARTIAL_MAP);
3404 }
3405 
3406 /*
3407  * rootnex_setup_copybuf()
3408  *    Called in bind slowpath. Figures out if we're going to use the copy
3409  *    buffer, and if we do, sets up the basic state to handle it.
3410  */
3411 static int
rootnex_setup_copybuf(ddi_dma_impl_t * hp,struct ddi_dma_req * dmareq,rootnex_dma_t * dma,ddi_dma_attr_t * attr)3412 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3413     rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3414 {
3415 	rootnex_sglinfo_t *sinfo;
3416 	ddi_dma_attr_t lattr;
3417 	size_t max_copybuf;
3418 	int cansleep;
3419 	int e;
3420 
3421 	ASSERT(!dma->dp_dvma_used);
3422 
3423 	sinfo = &dma->dp_sglinfo;
3424 
3425 	/* read this first so it's consistent through the routine  */
3426 	max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3427 
3428 	/* We need to call into the rootnex on ddi_dma_sync() */
3429 	hp->dmai_rflags &= ~DMP_NOSYNC;
3430 
3431 	/* make sure the copybuf size <= the max size */
3432 	dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3433 	ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3434 
3435 
3436 	/* convert the sleep flags */
3437 	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3438 		cansleep = 1;
3439 	} else {
3440 		cansleep = 0;
3441 	}
3442 
3443 	/*
3444 	 * Allocate the actual copy buffer. This needs to fit within the DMA
3445 	 * engine limits, so we can't use kmem_alloc... We don't need
3446 	 * contiguous memory (sgllen) since we will be forcing windows on
3447 	 * sgllen anyway.
3448 	 */
3449 	lattr = *attr;
3450 	lattr.dma_attr_align = MMU_PAGESIZE;
3451 	lattr.dma_attr_sgllen = -1;	/* no limit */
3452 	/*
3453 	 * if we're using the copy buffer because of seg, use that for our
3454 	 * upper address limit.
3455 	 */
3456 	if (sinfo->si_bounce_on_seg) {
3457 		lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3458 	}
3459 	e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3460 	    0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3461 	if (e != DDI_SUCCESS) {
3462 		return (DDI_DMA_NORESOURCES);
3463 	}
3464 
3465 	ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3466 	    size_t, dma->dp_copybuf_size);
3467 
3468 	return (DDI_SUCCESS);
3469 }
3470 
3471 
3472 /*
3473  * rootnex_setup_windows()
3474  *    Called in bind slowpath to setup the window state. We always have windows
3475  *    in the slowpath. Even if the window count = 1.
3476  */
3477 static int
rootnex_setup_windows(ddi_dma_impl_t * hp,rootnex_dma_t * dma,ddi_dma_attr_t * attr,ddi_dma_obj_t * dmao,int kmflag)3478 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3479     ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3480 {
3481 	rootnex_window_t *windowp;
3482 	rootnex_sglinfo_t *sinfo;
3483 	size_t copy_state_size;
3484 	size_t win_state_size;
3485 	size_t state_available;
3486 	size_t space_needed;
3487 	uint_t copybuf_win;
3488 	uint_t maxxfer_win;
3489 	size_t space_used;
3490 	uint_t sglwin;
3491 
3492 
3493 	sinfo = &dma->dp_sglinfo;
3494 
3495 	dma->dp_current_win = 0;
3496 	hp->dmai_nwin = 0;
3497 
3498 	/* If we don't need to do a partial, we only have one window */
3499 	if (!dma->dp_partial_required) {
3500 		dma->dp_max_win = 1;
3501 
3502 	/*
3503 	 * we need multiple windows, need to figure out the worse case number
3504 	 * of windows.
3505 	 */
3506 	} else {
3507 		/*
3508 		 * if we need windows because we need more copy buffer that
3509 		 * we allow, the worse case number of windows we could need
3510 		 * here would be (copybuf space required / copybuf space that
3511 		 * we have) plus one for remainder, and plus 2 to handle the
3512 		 * extra pages on the trim for the first and last pages of the
3513 		 * buffer (a page is the minimum window size so under the right
3514 		 * attr settings, you could have a window for each page).
3515 		 * The last page will only be hit here if the size is not a
3516 		 * multiple of the granularity (which theoretically shouldn't
3517 		 * be the case but never has been enforced, so we could have
3518 		 * broken things without it).
3519 		 */
3520 		if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3521 			ASSERT(dma->dp_copybuf_size > 0);
3522 			copybuf_win = (sinfo->si_copybuf_req /
3523 			    dma->dp_copybuf_size) + 1 + 2;
3524 		} else {
3525 			copybuf_win = 0;
3526 		}
3527 
3528 		/*
3529 		 * if we need windows because we have more cookies than the H/W
3530 		 * can handle, the number of windows we would need here would
3531 		 * be (cookie count / cookies count H/W supports minus 1[for
3532 		 * trim]) plus one for remainder.
3533 		 */
3534 		if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3535 			sglwin = (sinfo->si_sgl_size /
3536 			    (attr->dma_attr_sgllen - 1)) + 1;
3537 		} else {
3538 			sglwin = 0;
3539 		}
3540 
3541 		/*
3542 		 * if we need windows because we're binding more memory than the
3543 		 * H/W can transfer at once, the number of windows we would need
3544 		 * here would be (xfer count / max xfer H/W supports) plus one
3545 		 * for remainder, and plus 2 to handle the extra pages on the
3546 		 * trim (see above comment about trim)
3547 		 */
3548 		if (dmao->dmao_size > dma->dp_maxxfer) {
3549 			maxxfer_win = (dmao->dmao_size /
3550 			    dma->dp_maxxfer) + 1 + 2;
3551 		} else {
3552 			maxxfer_win = 0;
3553 		}
3554 		dma->dp_max_win =  copybuf_win + sglwin + maxxfer_win;
3555 		ASSERT(dma->dp_max_win > 0);
3556 	}
3557 	win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3558 
3559 	/*
3560 	 * Get space for window and potential copy buffer state. Before we
3561 	 * go and allocate memory, see if we can get away with using what's
3562 	 * left in the pre-allocted state or the dynamically allocated sgl.
3563 	 */
3564 	space_used = (uintptr_t)(sinfo->si_sgl_size *
3565 	    sizeof (ddi_dma_cookie_t));
3566 
3567 	/* if we dynamically allocated space for the cookies */
3568 	if (dma->dp_need_to_free_cookie) {
3569 		/* if we have more space in the pre-allocted buffer, use it */
3570 		ASSERT(space_used <= dma->dp_cookie_size);
3571 		if ((dma->dp_cookie_size - space_used) <=
3572 		    rootnex_state->r_prealloc_size) {
3573 			state_available = rootnex_state->r_prealloc_size;
3574 			windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3575 
3576 		/*
3577 		 * else, we have more free space in the dynamically allocated
3578 		 * buffer, i.e. the buffer wasn't worse case fragmented so we
3579 		 * didn't need a lot of cookies.
3580 		 */
3581 		} else {
3582 			state_available = dma->dp_cookie_size - space_used;
3583 			windowp = (rootnex_window_t *)
3584 			    &dma->dp_cookies[sinfo->si_sgl_size];
3585 		}
3586 
3587 	/* we used the pre-alloced buffer */
3588 	} else {
3589 		ASSERT(space_used <= rootnex_state->r_prealloc_size);
3590 		state_available = rootnex_state->r_prealloc_size - space_used;
3591 		windowp = (rootnex_window_t *)
3592 		    &dma->dp_cookies[sinfo->si_sgl_size];
3593 	}
3594 
3595 	/*
3596 	 * figure out how much state we need to track the copy buffer. Add an
3597 	 * addition 8 bytes for pointer alignemnt later.
3598 	 */
3599 	if (dma->dp_copybuf_size > 0) {
3600 		copy_state_size = sinfo->si_max_pages *
3601 		    sizeof (rootnex_pgmap_t);
3602 	} else {
3603 		copy_state_size = 0;
3604 	}
3605 	/* add an additional 8 bytes for pointer alignment */
3606 	space_needed = win_state_size + copy_state_size + 0x8;
3607 
3608 	/* if we have enough space already, use it */
3609 	if (state_available >= space_needed) {
3610 		dma->dp_window = windowp;
3611 		dma->dp_need_to_free_window = B_FALSE;
3612 
3613 	/* not enough space, need to allocate more. */
3614 	} else {
3615 		dma->dp_window = kmem_alloc(space_needed, kmflag);
3616 		if (dma->dp_window == NULL) {
3617 			return (DDI_DMA_NORESOURCES);
3618 		}
3619 		dma->dp_need_to_free_window = B_TRUE;
3620 		dma->dp_window_size = space_needed;
3621 		ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3622 		    dma->dp_dip, size_t, space_needed);
3623 	}
3624 
3625 	/*
3626 	 * we allocate copy buffer state and window state at the same time.
3627 	 * setup our copy buffer state pointers. Make sure it's aligned.
3628 	 */
3629 	if (dma->dp_copybuf_size > 0) {
3630 		dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3631 		    &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3632 
3633 	} else {
3634 		dma->dp_pgmap = NULL;
3635 	}
3636 
3637 	return (DDI_SUCCESS);
3638 }
3639 
3640 
3641 /*
3642  * rootnex_teardown_copybuf()
3643  *    cleans up after rootnex_setup_copybuf()
3644  */
3645 static void
rootnex_teardown_copybuf(rootnex_dma_t * dma)3646 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3647 {
3648 	/* if we allocated a copy buffer, free it */
3649 	if (dma->dp_cbaddr != NULL) {
3650 		i_ddi_mem_free(dma->dp_cbaddr, NULL);
3651 	}
3652 }
3653 
3654 
3655 /*
3656  * rootnex_teardown_windows()
3657  *    cleans up after rootnex_setup_windows()
3658  */
3659 static void
rootnex_teardown_windows(rootnex_dma_t * dma)3660 rootnex_teardown_windows(rootnex_dma_t *dma)
3661 {
3662 	/*
3663 	 * if we had to allocate window state on the last bind (because we
3664 	 * didn't have enough pre-allocated space in the handle), free it.
3665 	 */
3666 	if (dma->dp_need_to_free_window) {
3667 		kmem_free(dma->dp_window, dma->dp_window_size);
3668 	}
3669 }
3670 
3671 
3672 /*
3673  * rootnex_init_win()
3674  *    Called in bind slow path during creation of a new window. Initializes
3675  *    window state to default values.
3676  */
3677 /*ARGSUSED*/
3678 static void
rootnex_init_win(ddi_dma_impl_t * hp,rootnex_dma_t * dma,rootnex_window_t * window,ddi_dma_cookie_t * cookie,off_t cur_offset)3679 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3680     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3681 {
3682 	hp->dmai_nwin++;
3683 	window->wd_dosync = B_FALSE;
3684 	window->wd_offset = cur_offset;
3685 	window->wd_size = 0;
3686 	window->wd_first_cookie = cookie;
3687 	window->wd_cookie_cnt = 0;
3688 	window->wd_trim.tr_trim_first = B_FALSE;
3689 	window->wd_trim.tr_trim_last = B_FALSE;
3690 	window->wd_trim.tr_first_copybuf_win = B_FALSE;
3691 	window->wd_trim.tr_last_copybuf_win = B_FALSE;
3692 }
3693 
3694 
3695 /*
3696  * rootnex_setup_cookie()
3697  *    Called in the bind slow path when the sgl uses the copy buffer. If any of
3698  *    the sgl uses the copy buffer, we need to go through each cookie, figure
3699  *    out if it uses the copy buffer, and if it does, save away everything we'll
3700  *    need during sync.
3701  */
3702 static void
rootnex_setup_cookie(ddi_dma_obj_t * dmar_object,rootnex_dma_t * dma,ddi_dma_cookie_t * cookie,off_t cur_offset,size_t * copybuf_used,page_t ** cur_pp)3703 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3704     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3705     page_t **cur_pp)
3706 {
3707 	boolean_t copybuf_sz_power_2;
3708 	rootnex_sglinfo_t *sinfo;
3709 	paddr_t paddr;
3710 	uint_t pidx;
3711 	uint_t pcnt;
3712 	off_t poff;
3713 	pfn_t pfn;
3714 
3715 	ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3716 
3717 	sinfo = &dma->dp_sglinfo;
3718 
3719 	/*
3720 	 * Calculate the page index relative to the start of the buffer. The
3721 	 * index to the current page for our buffer is the offset into the
3722 	 * first page of the buffer plus our current offset into the buffer
3723 	 * itself, shifted of course...
3724 	 */
3725 	pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3726 	ASSERT(pidx < sinfo->si_max_pages);
3727 
3728 	/* if this cookie uses the copy buffer */
3729 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3730 		/*
3731 		 * NOTE: we know that since this cookie uses the copy buffer, it
3732 		 * is <= MMU_PAGESIZE.
3733 		 */
3734 
3735 		/*
3736 		 * get the offset into the page. For the 64-bit kernel, get the
3737 		 * pfn which we'll use with seg kpm.
3738 		 */
3739 		poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3740 		/* mfn_to_pfn() is a NOP on i86pc */
3741 		pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3742 
3743 		/* figure out if the copybuf size is a power of 2 */
3744 		if (!ISP2(dma->dp_copybuf_size)) {
3745 			copybuf_sz_power_2 = B_FALSE;
3746 		} else {
3747 			copybuf_sz_power_2 = B_TRUE;
3748 		}
3749 
3750 		/* This page uses the copy buffer */
3751 		dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3752 
3753 		/*
3754 		 * save the copy buffer KVA that we'll use with this page.
3755 		 * if we still fit within the copybuf, it's a simple add.
3756 		 * otherwise, we need to wrap over using & or % accordingly.
3757 		 */
3758 		if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3759 			dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3760 			    *copybuf_used;
3761 		} else {
3762 			if (copybuf_sz_power_2) {
3763 				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3764 				    (uintptr_t)dma->dp_cbaddr +
3765 				    (*copybuf_used &
3766 				    (dma->dp_copybuf_size - 1)));
3767 			} else {
3768 				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3769 				    (uintptr_t)dma->dp_cbaddr +
3770 				    (*copybuf_used % dma->dp_copybuf_size));
3771 			}
3772 		}
3773 
3774 		/*
3775 		 * over write the cookie physical address with the address of
3776 		 * the physical address of the copy buffer page that we will
3777 		 * use.
3778 		 */
3779 		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3780 		    dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3781 
3782 		cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3783 
3784 		/* if we have a kernel VA, it's easy, just save that address */
3785 		if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3786 		    (sinfo->si_asp == &kas)) {
3787 			/*
3788 			 * save away the page aligned virtual address of the
3789 			 * driver buffer. Offsets are handled in the sync code.
3790 			 */
3791 			dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3792 			    dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3793 			    & MMU_PAGEMASK);
3794 
3795 		/* we don't have a kernel VA. We need one for the bcopy. */
3796 		} else {
3797 			/*
3798 			 * for the 64-bit kernel, it's easy. We use seg kpm to
3799 			 * get a Kernel VA for the corresponding pfn.
3800 			 */
3801 			dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3802 			/* go to the next page_t */
3803 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3804 				*cur_pp = (*cur_pp)->p_next;
3805 			}
3806 		}
3807 
3808 		/* add to the copy buffer count */
3809 		*copybuf_used += MMU_PAGESIZE;
3810 
3811 	/*
3812 	 * This cookie doesn't use the copy buffer. Walk through the pages this
3813 	 * cookie occupies to reflect this.
3814 	 */
3815 	} else {
3816 		/*
3817 		 * figure out how many pages the cookie occupies. We need to
3818 		 * use the original page offset of the buffer and the cookies
3819 		 * offset in the buffer to do this.
3820 		 */
3821 		poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
3822 		pcnt = mmu_btopr(cookie->dmac_size + poff);
3823 
3824 		while (pcnt > 0) {
3825 			dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
3826 
3827 			/*
3828 			 * we need to update pidx and cur_pp or we'll loose
3829 			 * track of where we are.
3830 			 */
3831 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3832 				*cur_pp = (*cur_pp)->p_next;
3833 			}
3834 			pidx++;
3835 			pcnt--;
3836 		}
3837 	}
3838 }
3839 
3840 
3841 /*
3842  * rootnex_sgllen_window_boundary()
3843  *    Called in the bind slow path when the next cookie causes us to exceed (in
3844  *    this case == since we start at 0 and sgllen starts at 1) the maximum sgl
3845  *    length supported by the DMA H/W.
3846  */
3847 static int
rootnex_sgllen_window_boundary(ddi_dma_impl_t * hp,rootnex_dma_t * dma,rootnex_window_t ** windowp,ddi_dma_cookie_t * cookie,ddi_dma_attr_t * attr,off_t cur_offset)3848 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3849     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
3850     off_t cur_offset)
3851 {
3852 	off_t new_offset;
3853 	size_t trim_sz;
3854 	off_t coffset;
3855 
3856 
3857 	/*
3858 	 * if we know we'll never have to trim, it's pretty easy. Just move to
3859 	 * the next window and init it. We're done.
3860 	 */
3861 	if (!dma->dp_trim_required) {
3862 		(*windowp)++;
3863 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3864 		(*windowp)->wd_cookie_cnt++;
3865 		(*windowp)->wd_size = cookie->dmac_size;
3866 		return (DDI_SUCCESS);
3867 	}
3868 
3869 	/* figure out how much we need to trim from the window */
3870 	ASSERT(attr->dma_attr_granular != 0);
3871 	if (dma->dp_granularity_power_2) {
3872 		trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
3873 	} else {
3874 		trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
3875 	}
3876 
3877 	/* The window's a whole multiple of granularity. We're done */
3878 	if (trim_sz == 0) {
3879 		(*windowp)++;
3880 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3881 		(*windowp)->wd_cookie_cnt++;
3882 		(*windowp)->wd_size = cookie->dmac_size;
3883 		return (DDI_SUCCESS);
3884 	}
3885 
3886 	/*
3887 	 * The window's not a whole multiple of granularity, since we know this
3888 	 * is due to the sgllen, we need to go back to the last cookie and trim
3889 	 * that one, add the left over part of the old cookie into the new
3890 	 * window, and then add in the new cookie into the new window.
3891 	 */
3892 
3893 	/*
3894 	 * make sure the driver isn't making us do something bad... Trimming and
3895 	 * sgllen == 1 don't go together.
3896 	 */
3897 	if (attr->dma_attr_sgllen == 1) {
3898 		return (DDI_DMA_NOMAPPING);
3899 	}
3900 
3901 	/*
3902 	 * first, setup the current window to account for the trim. Need to go
3903 	 * back to the last cookie for this.
3904 	 */
3905 	cookie--;
3906 	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3907 	(*windowp)->wd_trim.tr_last_cookie = cookie;
3908 	(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
3909 	ASSERT(cookie->dmac_size > trim_sz);
3910 	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3911 	(*windowp)->wd_size -= trim_sz;
3912 
3913 	/* save the buffer offsets for the next window */
3914 	coffset = cookie->dmac_size - trim_sz;
3915 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3916 
3917 	/*
3918 	 * set this now in case this is the first window. all other cases are
3919 	 * set in dma_win()
3920 	 */
3921 	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
3922 
3923 	/*
3924 	 * initialize the next window using what's left over in the previous
3925 	 * cookie.
3926 	 */
3927 	(*windowp)++;
3928 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3929 	(*windowp)->wd_cookie_cnt++;
3930 	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3931 	(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
3932 	(*windowp)->wd_trim.tr_first_size = trim_sz;
3933 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3934 		(*windowp)->wd_dosync = B_TRUE;
3935 	}
3936 
3937 	/*
3938 	 * now go back to the current cookie and add it to the new window. set
3939 	 * the new window size to the what was left over from the previous
3940 	 * cookie and what's in the current cookie.
3941 	 */
3942 	cookie++;
3943 	(*windowp)->wd_cookie_cnt++;
3944 	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
3945 
3946 	/*
3947 	 * trim plus the next cookie could put us over maxxfer (a cookie can be
3948 	 * a max size of maxxfer). Handle that case.
3949 	 */
3950 	if ((*windowp)->wd_size > dma->dp_maxxfer) {
3951 		/*
3952 		 * maxxfer is already a whole multiple of granularity, and this
3953 		 * trim will be <= the previous trim (since a cookie can't be
3954 		 * larger than maxxfer). Make things simple here.
3955 		 */
3956 		trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
3957 		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3958 		(*windowp)->wd_trim.tr_last_cookie = cookie;
3959 		(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
3960 		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3961 		(*windowp)->wd_size -= trim_sz;
3962 		ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
3963 
3964 		/* save the buffer offsets for the next window */
3965 		coffset = cookie->dmac_size - trim_sz;
3966 		new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3967 
3968 		/* setup the next window */
3969 		(*windowp)++;
3970 		rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3971 		(*windowp)->wd_cookie_cnt++;
3972 		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3973 		(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
3974 		    coffset;
3975 		(*windowp)->wd_trim.tr_first_size = trim_sz;
3976 	}
3977 
3978 	return (DDI_SUCCESS);
3979 }
3980 
3981 
3982 /*
3983  * rootnex_copybuf_window_boundary()
3984  *    Called in bind slowpath when we get to a window boundary because we used
3985  *    up all the copy buffer that we have.
3986  */
3987 static int
rootnex_copybuf_window_boundary(ddi_dma_impl_t * hp,rootnex_dma_t * dma,rootnex_window_t ** windowp,ddi_dma_cookie_t * cookie,off_t cur_offset,size_t * copybuf_used)3988 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3989     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
3990     size_t *copybuf_used)
3991 {
3992 	rootnex_sglinfo_t *sinfo;
3993 	off_t new_offset;
3994 	size_t trim_sz;
3995 	paddr_t paddr;
3996 	off_t coffset;
3997 	uint_t pidx;
3998 	off_t poff;
3999 
4000 	pidx = 0;
4001 	sinfo = &dma->dp_sglinfo;
4002 
4003 	/*
4004 	 * the copy buffer should be a whole multiple of page size. We know that
4005 	 * this cookie is <= MMU_PAGESIZE.
4006 	 */
4007 	ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4008 
4009 	/* reset copybuf used */
4010 	*copybuf_used = 0;
4011 
4012 	/*
4013 	 * if we don't have to trim (since granularity is set to 1), go to the
4014 	 * next window and add the current cookie to it. We know the current
4015 	 * cookie uses the copy buffer since we're in this code path.
4016 	 */
4017 	if (!dma->dp_trim_required) {
4018 		(*windowp)++;
4019 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4020 
4021 		/* Add this cookie to the new window */
4022 		(*windowp)->wd_cookie_cnt++;
4023 		(*windowp)->wd_size += cookie->dmac_size;
4024 		*copybuf_used += MMU_PAGESIZE;
4025 		return (DDI_SUCCESS);
4026 	}
4027 
4028 	/*
4029 	 * *** may need to trim, figure it out.
4030 	 */
4031 
4032 	/* figure out how much we need to trim from the window */
4033 	if (dma->dp_granularity_power_2) {
4034 		trim_sz = (*windowp)->wd_size &
4035 		    (hp->dmai_attr.dma_attr_granular - 1);
4036 	} else {
4037 		trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4038 	}
4039 
4040 	/*
4041 	 * if the window's a whole multiple of granularity, go to the next
4042 	 * window, init it, then add in the current cookie. We know the current
4043 	 * cookie uses the copy buffer since we're in this code path.
4044 	 */
4045 	if (trim_sz == 0) {
4046 		(*windowp)++;
4047 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4048 
4049 		/* Add this cookie to the new window */
4050 		(*windowp)->wd_cookie_cnt++;
4051 		(*windowp)->wd_size += cookie->dmac_size;
4052 		*copybuf_used += MMU_PAGESIZE;
4053 		return (DDI_SUCCESS);
4054 	}
4055 
4056 	/*
4057 	 * *** We figured it out, we definitly need to trim
4058 	 */
4059 
4060 	/*
4061 	 * make sure the driver isn't making us do something bad...
4062 	 * Trimming and sgllen == 1 don't go together.
4063 	 */
4064 	if (hp->dmai_attr.dma_attr_sgllen == 1) {
4065 		return (DDI_DMA_NOMAPPING);
4066 	}
4067 
4068 	/*
4069 	 * first, setup the current window to account for the trim. Need to go
4070 	 * back to the last cookie for this. Some of the last cookie will be in
4071 	 * the current window, and some of the last cookie will be in the new
4072 	 * window. All of the current cookie will be in the new window.
4073 	 */
4074 	cookie--;
4075 	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
4076 	(*windowp)->wd_trim.tr_last_cookie = cookie;
4077 	(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4078 	ASSERT(cookie->dmac_size > trim_sz);
4079 	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4080 	(*windowp)->wd_size -= trim_sz;
4081 
4082 	/*
4083 	 * we're trimming the last cookie (not the current cookie). So that
4084 	 * last cookie may have or may not have been using the copy buffer (
4085 	 * we know the cookie passed in uses the copy buffer since we're in
4086 	 * this code path).
4087 	 *
4088 	 * If the last cookie doesn't use the copy buffer, nothing special to
4089 	 * do. However, if it does uses the copy buffer, it will be both the
4090 	 * last page in the current window and the first page in the next
4091 	 * window. Since we are reusing the copy buffer (and KVA space on the
4092 	 * 32-bit kernel), this page will use the end of the copy buffer in the
4093 	 * current window, and the start of the copy buffer in the next window.
4094 	 * Track that info... The cookie physical address was already set to
4095 	 * the copy buffer physical address in setup_cookie..
4096 	 */
4097 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4098 		pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4099 		    (*windowp)->wd_size) >> MMU_PAGESHIFT;
4100 		(*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4101 		(*windowp)->wd_trim.tr_last_pidx = pidx;
4102 		(*windowp)->wd_trim.tr_last_cbaddr =
4103 		    dma->dp_pgmap[pidx].pm_cbaddr;
4104 	}
4105 
4106 	/* save the buffer offsets for the next window */
4107 	coffset = cookie->dmac_size - trim_sz;
4108 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4109 
4110 	/*
4111 	 * set this now in case this is the first window. all other cases are
4112 	 * set in dma_win()
4113 	 */
4114 	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4115 
4116 	/*
4117 	 * initialize the next window using what's left over in the previous
4118 	 * cookie.
4119 	 */
4120 	(*windowp)++;
4121 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4122 	(*windowp)->wd_cookie_cnt++;
4123 	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
4124 	(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4125 	(*windowp)->wd_trim.tr_first_size = trim_sz;
4126 
4127 	/*
4128 	 * again, we're tracking if the last cookie uses the copy buffer.
4129 	 * read the comment above for more info on why we need to track
4130 	 * additional state.
4131 	 *
4132 	 * For the first cookie in the new window, we need reset the physical
4133 	 * address to DMA into to the start of the copy buffer plus any
4134 	 * initial page offset which may be present.
4135 	 */
4136 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4137 		(*windowp)->wd_dosync = B_TRUE;
4138 		(*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4139 		(*windowp)->wd_trim.tr_first_pidx = pidx;
4140 		(*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4141 		poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4142 
4143 		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4144 		    poff;
4145 		(*windowp)->wd_trim.tr_first_paddr =
4146 		    ROOTNEX_PADDR_TO_RBASE(paddr);
4147 
4148 		/* account for the cookie copybuf usage in the new window */
4149 		*copybuf_used += MMU_PAGESIZE;
4150 
4151 		/*
4152 		 * every piece of code has to have a hack, and here is this
4153 		 * ones :-)
4154 		 *
4155 		 * There is a complex interaction between setup_cookie and the
4156 		 * copybuf window boundary. The complexity had to be in either
4157 		 * the maxxfer window, or the copybuf window, and I chose the
4158 		 * copybuf code.
4159 		 *
4160 		 * So in this code path, we have taken the last cookie,
4161 		 * virtually broken it in half due to the trim, and it happens
4162 		 * to use the copybuf which further complicates life. At the
4163 		 * same time, we have already setup the current cookie, which
4164 		 * is now wrong. More background info: the current cookie uses
4165 		 * the copybuf, so it is only a page long max. So we need to
4166 		 * fix the current cookies copy buffer address, physical
4167 		 * address, and kva for the 32-bit kernel. We due this by
4168 		 * bumping them by page size (of course, we can't due this on
4169 		 * the physical address since the copy buffer may not be
4170 		 * physically contiguous).
4171 		 */
4172 		cookie++;
4173 		dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4174 		poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4175 
4176 		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4177 		    dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4178 		cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4179 
4180 	} else {
4181 		/* go back to the current cookie */
4182 		cookie++;
4183 	}
4184 
4185 	/*
4186 	 * add the current cookie to the new window. set the new window size to
4187 	 * the what was left over from the previous cookie and what's in the
4188 	 * current cookie.
4189 	 */
4190 	(*windowp)->wd_cookie_cnt++;
4191 	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
4192 	ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4193 
4194 	/*
4195 	 * we know that the cookie passed in always uses the copy buffer. We
4196 	 * wouldn't be here if it didn't.
4197 	 */
4198 	*copybuf_used += MMU_PAGESIZE;
4199 
4200 	return (DDI_SUCCESS);
4201 }
4202 
4203 
4204 /*
4205  * rootnex_maxxfer_window_boundary()
4206  *    Called in bind slowpath when we get to a window boundary because we will
4207  *    go over maxxfer.
4208  */
4209 static int
rootnex_maxxfer_window_boundary(ddi_dma_impl_t * hp,rootnex_dma_t * dma,rootnex_window_t ** windowp,ddi_dma_cookie_t * cookie)4210 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4211     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4212 {
4213 	size_t dmac_size;
4214 	off_t new_offset;
4215 	size_t trim_sz;
4216 	off_t coffset;
4217 
4218 
4219 	/*
4220 	 * calculate how much we have to trim off of the current cookie to equal
4221 	 * maxxfer. We don't have to account for granularity here since our
4222 	 * maxxfer already takes that into account.
4223 	 */
4224 	trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4225 	ASSERT(trim_sz <= cookie->dmac_size);
4226 	ASSERT(trim_sz <= dma->dp_maxxfer);
4227 
4228 	/* save cookie size since we need it later and we might change it */
4229 	dmac_size = cookie->dmac_size;
4230 
4231 	/*
4232 	 * if we're not trimming the entire cookie, setup the current window to
4233 	 * account for the trim.
4234 	 */
4235 	if (trim_sz < cookie->dmac_size) {
4236 		(*windowp)->wd_cookie_cnt++;
4237 		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
4238 		(*windowp)->wd_trim.tr_last_cookie = cookie;
4239 		(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4240 		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4241 		(*windowp)->wd_size = dma->dp_maxxfer;
4242 
4243 		/*
4244 		 * set the adjusted cookie size now in case this is the first
4245 		 * window. All other windows are taken care of in get win
4246 		 */
4247 		cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4248 	}
4249 
4250 	/*
4251 	 * coffset is the current offset within the cookie, new_offset is the
4252 	 * current offset with the entire buffer.
4253 	 */
4254 	coffset = dmac_size - trim_sz;
4255 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4256 
4257 	/* initialize the next window */
4258 	(*windowp)++;
4259 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4260 	(*windowp)->wd_cookie_cnt++;
4261 	(*windowp)->wd_size = trim_sz;
4262 	if (trim_sz < dmac_size) {
4263 		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
4264 		(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4265 		    coffset;
4266 		(*windowp)->wd_trim.tr_first_size = trim_sz;
4267 	}
4268 
4269 	return (DDI_SUCCESS);
4270 }
4271 
4272 
4273 /*ARGSUSED*/
4274 static int
rootnex_coredma_sync(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,off_t off,size_t len,uint_t cache_flags)4275 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4276     off_t off, size_t len, uint_t cache_flags)
4277 {
4278 	rootnex_sglinfo_t *sinfo;
4279 	rootnex_pgmap_t *cbpage;
4280 	rootnex_window_t *win;
4281 	ddi_dma_impl_t *hp;
4282 	rootnex_dma_t *dma;
4283 	caddr_t fromaddr;
4284 	caddr_t toaddr;
4285 	uint_t psize;
4286 	off_t offset;
4287 	uint_t pidx;
4288 	size_t size;
4289 	off_t poff;
4290 	int e;
4291 
4292 
4293 	hp = (ddi_dma_impl_t *)handle;
4294 	dma = (rootnex_dma_t *)hp->dmai_private;
4295 	sinfo = &dma->dp_sglinfo;
4296 
4297 	/*
4298 	 * if we don't have any windows, we don't need to sync. A copybuf
4299 	 * will cause us to have at least one window.
4300 	 */
4301 	if (dma->dp_window == NULL) {
4302 		return (DDI_SUCCESS);
4303 	}
4304 
4305 	/* This window may not need to be sync'd */
4306 	win = &dma->dp_window[dma->dp_current_win];
4307 	if (!win->wd_dosync) {
4308 		return (DDI_SUCCESS);
4309 	}
4310 
4311 	/* handle off and len special cases */
4312 	if ((off == 0) || (rootnex_sync_ignore_params)) {
4313 		offset = win->wd_offset;
4314 	} else {
4315 		offset = off;
4316 	}
4317 	if ((len == 0) || (rootnex_sync_ignore_params)) {
4318 		size = win->wd_size;
4319 	} else {
4320 		size = len;
4321 	}
4322 
4323 	/* check the sync args to make sure they make a little sense */
4324 	if (rootnex_sync_check_parms) {
4325 		e = rootnex_valid_sync_parms(hp, win, offset, size,
4326 		    cache_flags);
4327 		if (e != DDI_SUCCESS) {
4328 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4329 			return (DDI_FAILURE);
4330 		}
4331 	}
4332 
4333 	/*
4334 	 * special case the first page to handle the offset into the page. The
4335 	 * offset to the current page for our buffer is the offset into the
4336 	 * first page of the buffer plus our current offset into the buffer
4337 	 * itself, masked of course.
4338 	 */
4339 	poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4340 	psize = MIN((MMU_PAGESIZE - poff), size);
4341 
4342 	/* go through all the pages that we want to sync */
4343 	while (size > 0) {
4344 		/*
4345 		 * Calculate the page index relative to the start of the buffer.
4346 		 * The index to the current page for our buffer is the offset
4347 		 * into the first page of the buffer plus our current offset
4348 		 * into the buffer itself, shifted of course...
4349 		 */
4350 		pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4351 		ASSERT(pidx < sinfo->si_max_pages);
4352 
4353 		/*
4354 		 * if this page uses the copy buffer, we need to sync it,
4355 		 * otherwise, go on to the next page.
4356 		 */
4357 		cbpage = &dma->dp_pgmap[pidx];
4358 		ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4359 		    (cbpage->pm_uses_copybuf == B_FALSE));
4360 		if (cbpage->pm_uses_copybuf) {
4361 			/* cbaddr and kaddr should be page aligned */
4362 			ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4363 			    MMU_PAGEOFFSET) == 0);
4364 			ASSERT(((uintptr_t)cbpage->pm_kaddr &
4365 			    MMU_PAGEOFFSET) == 0);
4366 
4367 			/*
4368 			 * if we're copying for the device, we are going to
4369 			 * copy from the drivers buffer and to the rootnex
4370 			 * allocated copy buffer.
4371 			 */
4372 			if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4373 				fromaddr = cbpage->pm_kaddr + poff;
4374 				toaddr = cbpage->pm_cbaddr + poff;
4375 				ROOTNEX_DPROBE2(rootnex__sync__dev,
4376 				    dev_info_t *, dma->dp_dip, size_t, psize);
4377 
4378 			/*
4379 			 * if we're copying for the cpu/kernel, we are going to
4380 			 * copy from the rootnex allocated copy buffer to the
4381 			 * drivers buffer.
4382 			 */
4383 			} else {
4384 				fromaddr = cbpage->pm_cbaddr + poff;
4385 				toaddr = cbpage->pm_kaddr + poff;
4386 				ROOTNEX_DPROBE2(rootnex__sync__cpu,
4387 				    dev_info_t *, dma->dp_dip, size_t, psize);
4388 			}
4389 
4390 			bcopy(fromaddr, toaddr, psize);
4391 		}
4392 
4393 		/*
4394 		 * decrement size until we're done, update our offset into the
4395 		 * buffer, and get the next page size.
4396 		 */
4397 		size -= psize;
4398 		offset += psize;
4399 		psize = MIN(MMU_PAGESIZE, size);
4400 
4401 		/* page offset is zero for the rest of this loop */
4402 		poff = 0;
4403 	}
4404 
4405 	return (DDI_SUCCESS);
4406 }
4407 
4408 /*
4409  * rootnex_dma_sync()
4410  *    called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4411  *    We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4412  *    is set, ddi_dma_sync() returns immediately passing back success.
4413  */
4414 /*ARGSUSED*/
4415 static int
rootnex_dma_sync(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,off_t off,size_t len,uint_t cache_flags)4416 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4417     off_t off, size_t len, uint_t cache_flags)
4418 {
4419 #if !defined(__xpv)
4420 	if (IOMMU_USED(rdip)) {
4421 		return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4422 		    cache_flags));
4423 	}
4424 #endif
4425 	return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4426 	    cache_flags));
4427 }
4428 
4429 /*
4430  * rootnex_valid_sync_parms()
4431  *    checks the parameters passed to sync to verify they are correct.
4432  */
4433 static int
rootnex_valid_sync_parms(ddi_dma_impl_t * hp,rootnex_window_t * win,off_t offset,size_t size,uint_t cache_flags)4434 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4435     off_t offset, size_t size, uint_t cache_flags)
4436 {
4437 	off_t woffset;
4438 
4439 
4440 	/*
4441 	 * the first part of the test to make sure the offset passed in is
4442 	 * within the window.
4443 	 */
4444 	if (offset < win->wd_offset) {
4445 		return (DDI_FAILURE);
4446 	}
4447 
4448 	/*
4449 	 * second and last part of the test to make sure the offset and length
4450 	 * passed in is within the window.
4451 	 */
4452 	woffset = offset - win->wd_offset;
4453 	if ((woffset + size) > win->wd_size) {
4454 		return (DDI_FAILURE);
4455 	}
4456 
4457 	/*
4458 	 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4459 	 * be set too.
4460 	 */
4461 	if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4462 	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
4463 		return (DDI_SUCCESS);
4464 	}
4465 
4466 	/*
4467 	 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4468 	 * should be set. Also DDI_DMA_READ should be set in the flags.
4469 	 */
4470 	if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4471 	    (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4472 	    (hp->dmai_rflags & DDI_DMA_READ)) {
4473 		return (DDI_SUCCESS);
4474 	}
4475 
4476 	return (DDI_FAILURE);
4477 }
4478 
4479 
4480 /*ARGSUSED*/
4481 static int
rootnex_coredma_win(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,uint_t win,off_t * offp,size_t * lenp,ddi_dma_cookie_t * cookiep,uint_t * ccountp)4482 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4483     uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4484     uint_t *ccountp)
4485 {
4486 	rootnex_window_t *window;
4487 	rootnex_trim_t *trim;
4488 	ddi_dma_impl_t *hp;
4489 	rootnex_dma_t *dma;
4490 	ddi_dma_obj_t *dmao;
4491 
4492 
4493 	hp = (ddi_dma_impl_t *)handle;
4494 	dma = (rootnex_dma_t *)hp->dmai_private;
4495 
4496 	/* If we try and get a window which doesn't exist, return failure */
4497 	if (win >= hp->dmai_nwin) {
4498 		ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4499 		return (DDI_FAILURE);
4500 	}
4501 
4502 	dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4503 
4504 	/*
4505 	 * if we don't have any windows, and they're asking for the first
4506 	 * window, setup the cookie pointer to the first cookie in the bind.
4507 	 * setup our return values, then increment the cookie since we return
4508 	 * the first cookie on the stack.
4509 	 */
4510 	if (dma->dp_window == NULL) {
4511 		if (win != 0) {
4512 			ROOTNEX_DPROF_INC(
4513 			    &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4514 			return (DDI_FAILURE);
4515 		}
4516 		hp->dmai_cookie = dma->dp_cookies;
4517 		*offp = 0;
4518 		*lenp = dmao->dmao_size;
4519 		*ccountp = dma->dp_sglinfo.si_sgl_size;
4520 		*cookiep = hp->dmai_cookie[0];
4521 		hp->dmai_cookie++;
4522 		hp->dmai_ncookies = *ccountp;
4523 		hp->dmai_curcookie = 1;
4524 		return (DDI_SUCCESS);
4525 	}
4526 
4527 	/* sync the old window before moving on to the new one */
4528 	window = &dma->dp_window[dma->dp_current_win];
4529 	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4530 		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4531 		    DDI_DMA_SYNC_FORCPU);
4532 	}
4533 
4534 
4535 	/*
4536 	 * Move to the new window.
4537 	 * NOTE: current_win must be set for sync to work right
4538 	 */
4539 	dma->dp_current_win = win;
4540 	window = &dma->dp_window[win];
4541 
4542 	/* if needed, adjust the first and/or last cookies for trim */
4543 	trim = &window->wd_trim;
4544 	if (trim->tr_trim_first) {
4545 		window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4546 		window->wd_first_cookie->dmac_size = trim->tr_first_size;
4547 		if (trim->tr_first_copybuf_win) {
4548 			dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4549 			    trim->tr_first_cbaddr;
4550 		}
4551 	}
4552 	if (trim->tr_trim_last) {
4553 		trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4554 		trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4555 		if (trim->tr_last_copybuf_win) {
4556 			dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4557 			    trim->tr_last_cbaddr;
4558 		}
4559 	}
4560 
4561 	/*
4562 	 * setup the cookie pointer to the first cookie in the window. setup
4563 	 * our return values, then increment the cookie since we return the
4564 	 * first cookie on the stack.
4565 	 */
4566 	hp->dmai_cookie = window->wd_first_cookie;
4567 	*offp = window->wd_offset;
4568 	*lenp = window->wd_size;
4569 	*ccountp = window->wd_cookie_cnt;
4570 	*cookiep = hp->dmai_cookie[0];
4571 	hp->dmai_ncookies = *ccountp;
4572 	hp->dmai_curcookie = 1;
4573 	hp->dmai_cookie++;
4574 
4575 
4576 	/* if the new window uses the copy buffer, sync it for the device */
4577 	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4578 		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4579 		    DDI_DMA_SYNC_FORDEV);
4580 	}
4581 
4582 	return (DDI_SUCCESS);
4583 }
4584 
4585 /*
4586  * rootnex_dma_win()
4587  *    called from ddi_dma_getwin()
4588  */
4589 /*ARGSUSED*/
4590 static int
rootnex_dma_win(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,uint_t win,off_t * offp,size_t * lenp,ddi_dma_cookie_t * cookiep,uint_t * ccountp)4591 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4592     uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4593     uint_t *ccountp)
4594 {
4595 #if !defined(__xpv)
4596 	if (IOMMU_USED(rdip)) {
4597 		return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4598 		    cookiep, ccountp));
4599 	}
4600 #endif
4601 
4602 	return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4603 	    cookiep, ccountp));
4604 }
4605 
4606 #if !defined(__xpv)
4607 /*ARGSUSED*/
4608 static int
rootnex_coredma_hdl_setprivate(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,void * v)4609 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4610     ddi_dma_handle_t handle, void *v)
4611 {
4612 	ddi_dma_impl_t *hp;
4613 	rootnex_dma_t *dma;
4614 
4615 	hp = (ddi_dma_impl_t *)handle;
4616 	dma = (rootnex_dma_t *)hp->dmai_private;
4617 	dma->dp_iommu_private = v;
4618 
4619 	return (DDI_SUCCESS);
4620 }
4621 
4622 /*ARGSUSED*/
4623 static void *
rootnex_coredma_hdl_getprivate(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle)4624 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4625     ddi_dma_handle_t handle)
4626 {
4627 	ddi_dma_impl_t *hp;
4628 	rootnex_dma_t *dma;
4629 
4630 	hp = (ddi_dma_impl_t *)handle;
4631 	dma = (rootnex_dma_t *)hp->dmai_private;
4632 
4633 	return (dma->dp_iommu_private);
4634 }
4635 #endif
4636 
4637 /*
4638  * ************************
4639  *  obsoleted dma routines
4640  * ************************
4641  */
4642 
4643 /*
4644  * rootnex_dma_mctl()
4645  *
4646  * We don't support this legacy interface any more on x86.
4647  */
4648 /* ARGSUSED */
4649 static int
rootnex_dma_mctl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,enum ddi_dma_ctlops request,off_t * offp,size_t * lenp,caddr_t * objpp,uint_t cache_flags)4650 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4651     enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
4652     uint_t cache_flags)
4653 {
4654 	/*
4655 	 * The only thing dma_mctl is usef for anymore is legacy SPARC
4656 	 * dvma and sbus-specific routines.
4657 	 */
4658 	return (DDI_FAILURE);
4659 }
4660 
4661 /*
4662  * *********
4663  *  FMA Code
4664  * *********
4665  */
4666 
4667 /*
4668  * rootnex_fm_init()
4669  *    FMA init busop
4670  */
4671 /* ARGSUSED */
4672 static int
rootnex_fm_init(dev_info_t * dip,dev_info_t * tdip,int tcap,ddi_iblock_cookie_t * ibc)4673 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
4674     ddi_iblock_cookie_t *ibc)
4675 {
4676 	*ibc = rootnex_state->r_err_ibc;
4677 
4678 	return (ddi_system_fmcap);
4679 }
4680 
4681 /*
4682  * rootnex_dma_check()
4683  *    Function called after a dma fault occurred to find out whether the
4684  *    fault address is associated with a driver that is able to handle faults
4685  *    and recover from faults.
4686  */
4687 /* ARGSUSED */
4688 static int
rootnex_dma_check(dev_info_t * dip,const void * handle,const void * addr,const void * not_used)4689 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
4690     const void *not_used)
4691 {
4692 	rootnex_window_t *window;
4693 	uint64_t start_addr;
4694 	uint64_t fault_addr;
4695 	ddi_dma_impl_t *hp;
4696 	rootnex_dma_t *dma;
4697 	uint64_t end_addr;
4698 	size_t csize;
4699 	int i;
4700 	int j;
4701 
4702 
4703 	/* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
4704 	hp = (ddi_dma_impl_t *)handle;
4705 	ASSERT(hp);
4706 
4707 	dma = (rootnex_dma_t *)hp->dmai_private;
4708 
4709 	/* Get the address that we need to search for */
4710 	fault_addr = *(uint64_t *)addr;
4711 
4712 	/*
4713 	 * if we don't have any windows, we can just walk through all the
4714 	 * cookies.
4715 	 */
4716 	if (dma->dp_window == NULL) {
4717 		/* for each cookie */
4718 		for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
4719 			/*
4720 			 * if the faulted address is within the physical address
4721 			 * range of the cookie, return DDI_FM_NONFATAL.
4722 			 */
4723 			if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
4724 			    (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
4725 			    dma->dp_cookies[i].dmac_size))) {
4726 				return (DDI_FM_NONFATAL);
4727 			}
4728 		}
4729 
4730 		/* fault_addr not within this DMA handle */
4731 		return (DDI_FM_UNKNOWN);
4732 	}
4733 
4734 	/* we have mutiple windows, walk through each window */
4735 	for (i = 0; i < hp->dmai_nwin; i++) {
4736 		window = &dma->dp_window[i];
4737 
4738 		/* Go through all the cookies in the window */
4739 		for (j = 0; j < window->wd_cookie_cnt; j++) {
4740 
4741 			start_addr = window->wd_first_cookie[j].dmac_laddress;
4742 			csize = window->wd_first_cookie[j].dmac_size;
4743 
4744 			/*
4745 			 * if we are trimming the first cookie in the window,
4746 			 * and this is the first cookie, adjust the start
4747 			 * address and size of the cookie to account for the
4748 			 * trim.
4749 			 */
4750 			if (window->wd_trim.tr_trim_first && (j == 0)) {
4751 				start_addr = window->wd_trim.tr_first_paddr;
4752 				csize = window->wd_trim.tr_first_size;
4753 			}
4754 
4755 			/*
4756 			 * if we are trimming the last cookie in the window,
4757 			 * and this is the last cookie, adjust the start
4758 			 * address and size of the cookie to account for the
4759 			 * trim.
4760 			 */
4761 			if (window->wd_trim.tr_trim_last &&
4762 			    (j == (window->wd_cookie_cnt - 1))) {
4763 				start_addr = window->wd_trim.tr_last_paddr;
4764 				csize = window->wd_trim.tr_last_size;
4765 			}
4766 
4767 			end_addr = start_addr + csize;
4768 
4769 			/*
4770 			 * if the faulted address is within the physical
4771 			 * address of the cookie, return DDI_FM_NONFATAL.
4772 			 */
4773 			if ((fault_addr >= start_addr) &&
4774 			    (fault_addr <= end_addr)) {
4775 				return (DDI_FM_NONFATAL);
4776 			}
4777 		}
4778 	}
4779 
4780 	/* fault_addr not within this DMA handle */
4781 	return (DDI_FM_UNKNOWN);
4782 }
4783 
4784 /*ARGSUSED*/
4785 static int
rootnex_quiesce(dev_info_t * dip)4786 rootnex_quiesce(dev_info_t *dip)
4787 {
4788 #if !defined(__xpv)
4789 	return (immu_quiesce());
4790 #else
4791 	return (DDI_SUCCESS);
4792 #endif
4793 }
4794 
4795 #if defined(__xpv)
4796 void
immu_init(void)4797 immu_init(void)
4798 {
4799 	;
4800 }
4801 
4802 void
immu_startup(void)4803 immu_startup(void)
4804 {
4805 	;
4806 }
4807 /*ARGSUSED*/
4808 void
immu_physmem_update(uint64_t addr,uint64_t size)4809 immu_physmem_update(uint64_t addr, uint64_t size)
4810 {
4811 	;
4812 }
4813 #endif
4814