xref: /illumos-gate/usr/src/uts/sun4/os/ddi_impl.c (revision de710d24)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 /*
27  * Copyright 2014 Garrett D'Amore <garrett@damore.org>
28  */
29 
30 /*
31  * sun4 specific DDI implementation
32  */
33 #include <sys/cpuvar.h>
34 #include <sys/ddi_subrdefs.h>
35 #include <sys/machsystm.h>
36 #include <sys/sunndi.h>
37 #include <sys/sysmacros.h>
38 #include <sys/ontrap.h>
39 #include <vm/seg_kmem.h>
40 #include <sys/membar.h>
41 #include <sys/dditypes.h>
42 #include <sys/ndifm.h>
43 #include <sys/fm/io/ddi.h>
44 #include <sys/ivintr.h>
45 #include <sys/bootconf.h>
46 #include <sys/conf.h>
47 #include <sys/ethernet.h>
48 #include <sys/idprom.h>
49 #include <sys/promif.h>
50 #include <sys/prom_plat.h>
51 #include <sys/systeminfo.h>
52 #include <sys/fpu/fpusystm.h>
53 #include <sys/vm.h>
54 #include <sys/ddi_isa.h>
55 #include <sys/modctl.h>
56 
57 dev_info_t *get_intr_parent(dev_info_t *, dev_info_t *,
58     ddi_intr_handle_impl_t *);
59 #pragma weak get_intr_parent
60 
61 int process_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t,
62     ddi_intr_handle_impl_t *, void *);
63 #pragma weak process_intr_ops
64 
65 void cells_1275_copy(prop_1275_cell_t *, prop_1275_cell_t *, int32_t);
66     prop_1275_cell_t *cells_1275_cmp(prop_1275_cell_t *, prop_1275_cell_t *,
67     int32_t len);
68 #pragma weak cells_1275_copy
69 
70 /*
71  * Wrapper for ddi_prop_lookup_int_array().
72  * This is handy because it returns the prop length in
73  * bytes which is what most of the callers require.
74  */
75 
76 static int
77 get_prop_int_array(dev_info_t *di, char *pname, int **pval, uint_t *plen)
78 {
79 	int ret;
80 
81 	if ((ret = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, di,
82 	    DDI_PROP_DONTPASS, pname, pval, plen)) == DDI_PROP_SUCCESS) {
83 		*plen = (*plen) * (uint_t)sizeof (int);
84 	}
85 	return (ret);
86 }
87 
88 /*
89  * SECTION: DDI Node Configuration
90  */
91 
92 /*
93  * init_regspec_64:
94  *
95  * If the parent #size-cells is 2, convert the upa-style or
96  * safari-style reg property from 2-size cells to 1 size cell
97  * format, ignoring the size_hi, which must be zero for devices.
98  * (It won't be zero in the memory list properties in the memory
99  * nodes, but that doesn't matter here.)
100  */
101 struct ddi_parent_private_data *
102 init_regspec_64(dev_info_t *dip)
103 {
104 	struct ddi_parent_private_data *pd;
105 	dev_info_t *parent;
106 	int size_cells;
107 
108 	/*
109 	 * If there are no "reg"s in the child node, return.
110 	 */
111 	pd = ddi_get_parent_data(dip);
112 	if ((pd == NULL) || (pd->par_nreg == 0)) {
113 		return (pd);
114 	}
115 	parent = ddi_get_parent(dip);
116 
117 	size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
118 	    DDI_PROP_DONTPASS, "#size-cells", 1);
119 
120 	if (size_cells != 1)  {
121 
122 		int n, j;
123 		struct regspec *irp;
124 		struct reg_64 {
125 			uint_t addr_hi, addr_lo, size_hi, size_lo;
126 		};
127 		struct reg_64 *r64_rp;
128 		struct regspec *rp;
129 		uint_t len = 0;
130 		int *reg_prop;
131 
132 		ASSERT(size_cells == 2);
133 
134 		/*
135 		 * We already looked the property up once before if
136 		 * pd is non-NULL.
137 		 */
138 		(void) ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
139 		    DDI_PROP_DONTPASS, OBP_REG, &reg_prop, &len);
140 		ASSERT(len != 0);
141 
142 		n = sizeof (struct reg_64) / sizeof (int);
143 		n = len / n;
144 
145 		/*
146 		 * We're allocating a buffer the size of the PROM's property,
147 		 * but we're only using a smaller portion when we assign it
148 		 * to a regspec.  We do this so that in the
149 		 * impl_ddi_sunbus_removechild function, we will
150 		 * always free the right amount of memory.
151 		 */
152 		irp = rp = (struct regspec *)reg_prop;
153 		r64_rp = (struct reg_64 *)pd->par_reg;
154 
155 		for (j = 0; j < n; ++j, ++rp, ++r64_rp) {
156 			ASSERT(r64_rp->size_hi == 0);
157 			rp->regspec_bustype = r64_rp->addr_hi;
158 			rp->regspec_addr = r64_rp->addr_lo;
159 			rp->regspec_size = r64_rp->size_lo;
160 		}
161 
162 		ddi_prop_free((void *)pd->par_reg);
163 		pd->par_nreg = n;
164 		pd->par_reg = irp;
165 	}
166 	return (pd);
167 }
168 
169 /*
170  * Create a ddi_parent_private_data structure from the ddi properties of
171  * the dev_info node.
172  *
173  * The "reg" is required if the driver wishes to create mappings on behalf
174  * of the device. The "reg" property is assumed to be a list of at least
175  * one triplet
176  *
177  *	<bustype, address, size>*1
178  *
179  * The "interrupt" property is no longer part of parent private data on
180  * sun4u. The interrupt parent is may not be the device tree parent.
181  *
182  * The "ranges" property describes the mapping of child addresses to parent
183  * addresses.
184  *
185  * N.B. struct rangespec is defined for the following default values:
186  *			parent  child
187  *	#address-cells	2	2
188  *	#size-cells	1	1
189  * This function doesn't deal with non-default cells and will not create
190  * ranges in such cases.
191  */
192 void
193 make_ddi_ppd(dev_info_t *child, struct ddi_parent_private_data **ppd)
194 {
195 	struct ddi_parent_private_data *pdptr;
196 	int *reg_prop, *rng_prop;
197 	uint_t reg_len = 0, rng_len = 0;
198 	dev_info_t *parent;
199 	int parent_addr_cells, parent_size_cells;
200 	int child_addr_cells, child_size_cells;
201 
202 	*ppd = pdptr = kmem_zalloc(sizeof (*pdptr), KM_SLEEP);
203 
204 	/*
205 	 * root node has no parent private data, so *ppd should
206 	 * be initialized for naming to work properly.
207 	 */
208 	if ((parent = ddi_get_parent(child)) == NULL)
209 		return;
210 
211 	/*
212 	 * Set reg field of parent data from "reg" property
213 	 */
214 	if ((get_prop_int_array(child, OBP_REG, &reg_prop, &reg_len)
215 	    == DDI_PROP_SUCCESS) && (reg_len != 0)) {
216 		pdptr->par_nreg = (int)(reg_len / sizeof (struct regspec));
217 		pdptr->par_reg = (struct regspec *)reg_prop;
218 	}
219 
220 	/*
221 	 * "ranges" property ...
222 	 *
223 	 * This function does not handle cases where #address-cells != 2
224 	 * and * min(parent, child) #size-cells != 1 (see bugid 4211124).
225 	 *
226 	 * Nexus drivers with such exceptions (e.g. pci ranges)
227 	 * should either create a separate function for handling
228 	 * ranges or not use parent private data to store ranges.
229 	 */
230 
231 	/* root node has no ranges */
232 	if ((parent = ddi_get_parent(child)) == NULL)
233 		return;
234 
235 	child_addr_cells = ddi_prop_get_int(DDI_DEV_T_ANY, child,
236 	    DDI_PROP_DONTPASS, "#address-cells", 2);
237 	child_size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, child,
238 	    DDI_PROP_DONTPASS, "#size-cells", 1);
239 	parent_addr_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
240 	    DDI_PROP_DONTPASS, "#address-cells", 2);
241 	parent_size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
242 	    DDI_PROP_DONTPASS, "#size-cells", 1);
243 	if (child_addr_cells != 2 || parent_addr_cells != 2 ||
244 	    (child_size_cells != 1 && parent_size_cells != 1)) {
245 		NDI_CONFIG_DEBUG((CE_NOTE, "!ranges not made in parent data; "
246 		    "#address-cells or #size-cells have non-default value"));
247 		return;
248 	}
249 
250 	if (get_prop_int_array(child, OBP_RANGES, &rng_prop, &rng_len)
251 	    == DDI_PROP_SUCCESS) {
252 		pdptr->par_nrng = rng_len / (int)(sizeof (struct rangespec));
253 		pdptr->par_rng = (struct rangespec *)rng_prop;
254 	}
255 }
256 
257 /*
258  * Free ddi_parent_private_data structure
259  */
260 void
261 impl_free_ddi_ppd(dev_info_t *dip)
262 {
263 	struct ddi_parent_private_data *pdptr = ddi_get_parent_data(dip);
264 
265 	if (pdptr == NULL)
266 		return;
267 
268 	if (pdptr->par_nrng != 0)
269 		ddi_prop_free((void *)pdptr->par_rng);
270 
271 	if (pdptr->par_nreg != 0)
272 		ddi_prop_free((void *)pdptr->par_reg);
273 
274 	kmem_free(pdptr, sizeof (*pdptr));
275 	ddi_set_parent_data(dip, NULL);
276 }
277 
278 /*
279  * Name a child of sun busses based on the reg spec.
280  * Handles the following properties:
281  *
282  *	Property	value
283  *	Name		type
284  *
285  *	reg		register spec
286  *	interrupts	new (bus-oriented) interrupt spec
287  *	ranges		range spec
288  *
289  * This may be called multiple times, independent of
290  * initchild calls.
291  */
292 static int
293 impl_sunbus_name_child(dev_info_t *child, char *name, int namelen)
294 {
295 	struct ddi_parent_private_data *pdptr;
296 	struct regspec *rp;
297 
298 	/*
299 	 * Fill in parent-private data and this function returns to us
300 	 * an indication if it used "registers" to fill in the data.
301 	 */
302 	if (ddi_get_parent_data(child) == NULL) {
303 		make_ddi_ppd(child, &pdptr);
304 		ddi_set_parent_data(child, pdptr);
305 	}
306 
307 	/*
308 	 * No reg property, return null string as address
309 	 * (e.g. root node)
310 	 */
311 	name[0] = '\0';
312 	if (sparc_pd_getnreg(child) == 0) {
313 		return (DDI_SUCCESS);
314 	}
315 
316 	rp = sparc_pd_getreg(child, 0);
317 	(void) snprintf(name, namelen, "%x,%x",
318 	    rp->regspec_bustype, rp->regspec_addr);
319 	return (DDI_SUCCESS);
320 }
321 
322 
323 /*
324  * Called from the bus_ctl op of some drivers.
325  * to implement the DDI_CTLOPS_INITCHILD operation.
326  *
327  * NEW drivers should NOT use this function, but should declare
328  * there own initchild/uninitchild handlers. (This function assumes
329  * the layout of the parent private data and the format of "reg",
330  * "ranges", "interrupts" properties and that #address-cells and
331  * #size-cells of the parent bus are defined to be default values.)
332  */
333 int
334 impl_ddi_sunbus_initchild(dev_info_t *child)
335 {
336 	char name[MAXNAMELEN];
337 
338 	(void) impl_sunbus_name_child(child, name, MAXNAMELEN);
339 	ddi_set_name_addr(child, name);
340 
341 	/*
342 	 * Try to merge .conf node. If successful, return failure to
343 	 * remove this child.
344 	 */
345 	if ((ndi_dev_is_persistent_node(child) == 0) &&
346 	    (ndi_merge_node(child, impl_sunbus_name_child) == DDI_SUCCESS)) {
347 		impl_ddi_sunbus_removechild(child);
348 		return (DDI_FAILURE);
349 	}
350 	return (DDI_SUCCESS);
351 }
352 
353 /*
354  * A better name for this function would be impl_ddi_sunbus_uninitchild()
355  * It does not remove the child, it uninitializes it, reclaiming the
356  * resources taken by impl_ddi_sunbus_initchild.
357  */
358 void
359 impl_ddi_sunbus_removechild(dev_info_t *dip)
360 {
361 	impl_free_ddi_ppd(dip);
362 	ddi_set_name_addr(dip, NULL);
363 	/*
364 	 * Strip the node to properly convert it back to prototype form
365 	 */
366 	impl_rem_dev_props(dip);
367 }
368 
369 /*
370  * SECTION: DDI Interrupt
371  */
372 
373 void
374 cells_1275_copy(prop_1275_cell_t *from, prop_1275_cell_t *to, int32_t len)
375 {
376 	int i;
377 	for (i = 0; i < len; i++)
378 		*to = *from;
379 }
380 
381 prop_1275_cell_t *
382 cells_1275_cmp(prop_1275_cell_t *cell1, prop_1275_cell_t *cell2, int32_t len)
383 {
384 	prop_1275_cell_t *match_cell = 0;
385 	int32_t i;
386 
387 	for (i = 0; i < len; i++)
388 		if (cell1[i] != cell2[i]) {
389 			match_cell = &cell1[i];
390 			break;
391 		}
392 
393 	return (match_cell);
394 }
395 
396 /*
397  * get_intr_parent() is a generic routine that process a 1275 interrupt
398  * map (imap) property.  This function returns a dev_info_t structure
399  * which claims ownership of the interrupt domain.
400  * It also returns the new interrupt translation within this new domain.
401  * If an interrupt-parent or interrupt-map property are not found,
402  * then we fallback to using the device tree's parent.
403  *
404  * imap entry format:
405  * <reg>,<interrupt>,<phandle>,<translated interrupt>
406  * reg - The register specification in the interrupts domain
407  * interrupt - The interrupt specification
408  * phandle - PROM handle of the device that owns the xlated interrupt domain
409  * translated interrupt - interrupt specifier in the parents domain
410  * note: <reg>,<interrupt> - The reg and interrupt can be combined to create
411  *	a unique entry called a unit interrupt specifier.
412  *
413  * Here's the processing steps:
414  * step1 - If the interrupt-parent property exists, create the ispec and
415  *	return the dip of the interrupt parent.
416  * step2 - Extract the interrupt-map property and the interrupt-map-mask
417  *	If these don't exist, just return the device tree parent.
418  * step3 - build up the unit interrupt specifier to match against the
419  *	interrupt map property
420  * step4 - Scan the interrupt-map property until a match is found
421  * step4a - Extract the interrupt parent
422  * step4b - Compare the unit interrupt specifier
423  */
424 dev_info_t *
425 get_intr_parent(dev_info_t *pdip, dev_info_t *dip, ddi_intr_handle_impl_t *hdlp)
426 {
427 	prop_1275_cell_t *imap, *imap_mask, *scan, *reg_p, *match_req;
428 	int32_t imap_sz, imap_cells, imap_scan_cells, imap_mask_sz,
429 	    addr_cells, intr_cells, reg_len, i, j;
430 	int32_t match_found = 0;
431 	dev_info_t *intr_parent_dip = NULL;
432 	uint32_t *intr = &hdlp->ih_vector;
433 	uint32_t nodeid;
434 #ifdef DEBUG
435 	static int debug = 0;
436 #endif
437 
438 	/*
439 	 * step1
440 	 * If we have an interrupt-parent property, this property represents
441 	 * the nodeid of our interrupt parent.
442 	 */
443 	if ((nodeid = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
444 	    "interrupt-parent", -1)) != -1) {
445 		intr_parent_dip = e_ddi_nodeid_to_dip(nodeid);
446 		ASSERT(intr_parent_dip);
447 
448 		/*
449 		 * Attach the interrupt parent.
450 		 *
451 		 * N.B. e_ddi_nodeid_to_dip() isn't safe under DR.
452 		 *	Also, interrupt parent isn't held. This needs
453 		 *	to be revisited if DR-capable platforms implement
454 		 *	interrupt redirection.
455 		 */
456 		if (i_ddi_attach_node_hierarchy(intr_parent_dip)
457 		    != DDI_SUCCESS) {
458 			ndi_rele_devi(intr_parent_dip);
459 			return (NULL);
460 		}
461 
462 		return (intr_parent_dip);
463 	}
464 
465 	/*
466 	 * step2
467 	 * Get interrupt map structure from PROM property
468 	 */
469 	if (ddi_getlongprop(DDI_DEV_T_ANY, pdip, DDI_PROP_DONTPASS,
470 	    "interrupt-map", (caddr_t)&imap, &imap_sz)
471 	    != DDI_PROP_SUCCESS) {
472 		/*
473 		 * If we don't have an imap property, default to using the
474 		 * device tree.
475 		 */
476 
477 		ndi_hold_devi(pdip);
478 		return (pdip);
479 	}
480 
481 	/* Get the interrupt mask property */
482 	if (ddi_getlongprop(DDI_DEV_T_ANY, pdip, DDI_PROP_DONTPASS,
483 	    "interrupt-map-mask", (caddr_t)&imap_mask, &imap_mask_sz)
484 	    != DDI_PROP_SUCCESS) {
485 		/*
486 		 * If we don't find this property, we have to fail the request
487 		 * because the 1275 imap property wasn't defined correctly.
488 		 */
489 		ASSERT(intr_parent_dip == NULL);
490 		goto exit2;
491 	}
492 
493 	/* Get the address cell size */
494 	addr_cells = ddi_getprop(DDI_DEV_T_ANY, pdip, 0,
495 	    "#address-cells", 2);
496 
497 	/* Get the interrupts cell size */
498 	intr_cells = ddi_getprop(DDI_DEV_T_ANY, pdip, 0,
499 	    "#interrupt-cells", 1);
500 
501 	/*
502 	 * step3
503 	 * Now lets build up the unit interrupt specifier e.g. reg,intr
504 	 * and apply the imap mask.  match_req will hold this when we're
505 	 * through.
506 	 */
507 	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "reg",
508 	    (caddr_t)&reg_p, &reg_len) != DDI_SUCCESS) {
509 		ASSERT(intr_parent_dip == NULL);
510 		goto exit3;
511 	}
512 
513 	match_req = kmem_alloc(CELLS_1275_TO_BYTES(addr_cells) +
514 	    CELLS_1275_TO_BYTES(intr_cells), KM_SLEEP);
515 
516 	for (i = 0; i < addr_cells; i++)
517 		match_req[i] = (reg_p[i] & imap_mask[i]);
518 
519 	for (j = 0; j < intr_cells; i++, j++)
520 		match_req[i] = (intr[j] & imap_mask[i]);
521 
522 	/* Calculate the imap size in cells */
523 	imap_cells = BYTES_TO_1275_CELLS(imap_sz);
524 
525 #ifdef DEBUG
526 	if (debug)
527 		prom_printf("reg cell size 0x%x, intr cell size 0x%x, "
528 		    "match_request 0x%p, imap 0x%p\n", addr_cells, intr_cells,
529 		    (void *)match_req, (void *)imap);
530 #endif
531 
532 	/*
533 	 * Scan the imap property looking for a match of the interrupt unit
534 	 * specifier.  This loop is rather complex since the data within the
535 	 * imap property may vary in size.
536 	 */
537 	for (scan = imap, imap_scan_cells = i = 0;
538 	    imap_scan_cells < imap_cells; scan += i, imap_scan_cells += i) {
539 		int new_intr_cells;
540 
541 		/* Set the index to the nodeid field */
542 		i = addr_cells + intr_cells;
543 
544 		/*
545 		 * step4a
546 		 * Translate the nodeid field to a dip
547 		 */
548 		ASSERT(intr_parent_dip == NULL);
549 		intr_parent_dip = e_ddi_nodeid_to_dip((uint_t)scan[i++]);
550 
551 		ASSERT(intr_parent_dip != 0);
552 #ifdef DEBUG
553 		if (debug)
554 			prom_printf("scan 0x%p\n", (void *)scan);
555 #endif
556 		/*
557 		 * The tmp_dip describes the new domain, get it's interrupt
558 		 * cell size
559 		 */
560 		new_intr_cells = ddi_getprop(DDI_DEV_T_ANY, intr_parent_dip, 0,
561 		    "#interrupts-cells", 1);
562 
563 		/*
564 		 * step4b
565 		 * See if we have a match on the interrupt unit specifier
566 		 */
567 		if (cells_1275_cmp(match_req, scan, addr_cells + intr_cells)
568 		    == 0) {
569 			uint32_t *intr;
570 
571 			match_found = 1;
572 
573 			/*
574 			 * If we have an imap parent whose not in our device
575 			 * tree path, we need to hold and install that driver.
576 			 */
577 			if (i_ddi_attach_node_hierarchy(intr_parent_dip)
578 			    != DDI_SUCCESS) {
579 				ndi_rele_devi(intr_parent_dip);
580 				intr_parent_dip = (dev_info_t *)NULL;
581 				goto exit4;
582 			}
583 
584 			/*
585 			 * We need to handcraft an ispec along with a bus
586 			 * interrupt value, so we can dup it into our
587 			 * standard ispec structure.
588 			 */
589 			/* Extract the translated interrupt information */
590 			intr = kmem_alloc(
591 			    CELLS_1275_TO_BYTES(new_intr_cells), KM_SLEEP);
592 
593 			for (j = 0; j < new_intr_cells; j++, i++)
594 				intr[j] = scan[i];
595 
596 			cells_1275_copy(intr, &hdlp->ih_vector, new_intr_cells);
597 
598 			kmem_free(intr, CELLS_1275_TO_BYTES(new_intr_cells));
599 
600 #ifdef DEBUG
601 			if (debug)
602 				prom_printf("dip 0x%p\n",
603 				    (void *)intr_parent_dip);
604 #endif
605 			break;
606 		} else {
607 #ifdef DEBUG
608 			if (debug)
609 				prom_printf("dip 0x%p\n",
610 				    (void *)intr_parent_dip);
611 #endif
612 			ndi_rele_devi(intr_parent_dip);
613 			intr_parent_dip = NULL;
614 			i += new_intr_cells;
615 		}
616 	}
617 
618 	/*
619 	 * If we haven't found our interrupt parent at this point, fallback
620 	 * to using the device tree.
621 	 */
622 	if (!match_found) {
623 		ndi_hold_devi(pdip);
624 		ASSERT(intr_parent_dip == NULL);
625 		intr_parent_dip = pdip;
626 	}
627 
628 	ASSERT(intr_parent_dip != NULL);
629 
630 exit4:
631 	kmem_free(reg_p, reg_len);
632 	kmem_free(match_req, CELLS_1275_TO_BYTES(addr_cells) +
633 	    CELLS_1275_TO_BYTES(intr_cells));
634 
635 exit3:
636 	kmem_free(imap_mask, imap_mask_sz);
637 
638 exit2:
639 	kmem_free(imap, imap_sz);
640 
641 	return (intr_parent_dip);
642 }
643 
644 /*
645  * process_intr_ops:
646  *
647  * Process the interrupt op via the interrupt parent.
648  */
649 int
650 process_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t op,
651     ddi_intr_handle_impl_t *hdlp, void *result)
652 {
653 	int		ret = DDI_FAILURE;
654 
655 	if (NEXUS_HAS_INTR_OP(pdip)) {
656 		ret = (*(DEVI(pdip)->devi_ops->devo_bus_ops->
657 		    bus_intr_op)) (pdip, rdip, op, hdlp, result);
658 	} else {
659 		cmn_err(CE_WARN, "Failed to process interrupt "
660 		    "for %s%d due to down-rev nexus driver %s%d",
661 		    ddi_get_name(rdip), ddi_get_instance(rdip),
662 		    ddi_get_name(pdip), ddi_get_instance(pdip));
663 	}
664 
665 	return (ret);
666 }
667 
668 /*ARGSUSED*/
669 uint_t
670 softlevel1(caddr_t arg)
671 {
672 	softint();
673 	return (1);
674 }
675 
676 /*
677  * indirection table, to save us some large switch statements
678  * NOTE: This must agree with "INTLEVEL_foo" constants in
679  *	<sys/avintr.h>
680  */
681 struct autovec *const vectorlist[] = { 0 };
682 
683 /*
684  * This value is exported here for the functions in avintr.c
685  */
686 const uint_t maxautovec = (sizeof (vectorlist) / sizeof (vectorlist[0]));
687 
688 /*
689  * Check for machine specific interrupt levels which cannot be reassigned by
690  * settrap(), sun4u version.
691  *
692  * sun4u does not support V8 SPARC "fast trap" handlers.
693  */
694 /*ARGSUSED*/
695 int
696 exclude_settrap(int lvl)
697 {
698 	return (1);
699 }
700 
701 /*
702  * Check for machine specific interrupt levels which cannot have interrupt
703  * handlers added. We allow levels 1 through 15; level 0 is nonsense.
704  */
705 /*ARGSUSED*/
706 int
707 exclude_level(int lvl)
708 {
709 	return ((lvl < 1) || (lvl > 15));
710 }
711 
712 /*
713  * Wrapper functions used by New DDI interrupt framework.
714  */
715 
716 /*
717  * i_ddi_intr_ops:
718  */
719 int
720 i_ddi_intr_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t op,
721     ddi_intr_handle_impl_t *hdlp, void *result)
722 {
723 	dev_info_t	*pdip = ddi_get_parent(dip);
724 	int		ret = DDI_FAILURE;
725 
726 	/*
727 	 * The following check is required to address
728 	 * one of the test case of ADDI test suite.
729 	 */
730 	if (pdip == NULL)
731 		return (DDI_FAILURE);
732 
733 	if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
734 		return (process_intr_ops(pdip, rdip, op, hdlp, result));
735 
736 	if (hdlp->ih_vector == 0)
737 		hdlp->ih_vector = i_ddi_get_inum(rdip, hdlp->ih_inum);
738 
739 	if (hdlp->ih_pri == 0)
740 		hdlp->ih_pri = i_ddi_get_intr_pri(rdip, hdlp->ih_inum);
741 
742 	switch (op) {
743 	case DDI_INTROP_ADDISR:
744 	case DDI_INTROP_REMISR:
745 	case DDI_INTROP_GETTARGET:
746 	case DDI_INTROP_SETTARGET:
747 	case DDI_INTROP_ENABLE:
748 	case DDI_INTROP_DISABLE:
749 	case DDI_INTROP_BLOCKENABLE:
750 	case DDI_INTROP_BLOCKDISABLE:
751 		/*
752 		 * Try and determine our parent and possibly an interrupt
753 		 * translation. intr parent dip returned held
754 		 */
755 		if ((pdip = get_intr_parent(pdip, dip, hdlp)) == NULL)
756 			goto done;
757 	}
758 
759 	ret = process_intr_ops(pdip, rdip, op, hdlp, result);
760 
761 done:
762 	switch (op) {
763 	case DDI_INTROP_ADDISR:
764 	case DDI_INTROP_REMISR:
765 	case DDI_INTROP_ENABLE:
766 	case DDI_INTROP_DISABLE:
767 	case DDI_INTROP_BLOCKENABLE:
768 	case DDI_INTROP_BLOCKDISABLE:
769 		/* Release hold acquired in get_intr_parent() */
770 		if (pdip)
771 			ndi_rele_devi(pdip);
772 	}
773 
774 	hdlp->ih_vector = 0;
775 
776 	return (ret);
777 }
778 
779 /*
780  * i_ddi_add_ivintr:
781  */
782 /*ARGSUSED*/
783 int
784 i_ddi_add_ivintr(ddi_intr_handle_impl_t *hdlp)
785 {
786 	/*
787 	 * If the PIL was set and is valid use it, otherwise
788 	 * default it to 1
789 	 */
790 	if ((hdlp->ih_pri < 1) || (hdlp->ih_pri > PIL_MAX))
791 		hdlp->ih_pri = 1;
792 
793 	VERIFY(add_ivintr(hdlp->ih_vector, hdlp->ih_pri,
794 	    (intrfunc)hdlp->ih_cb_func, hdlp->ih_cb_arg1,
795 	    hdlp->ih_cb_arg2, NULL) == 0);
796 
797 	return (DDI_SUCCESS);
798 }
799 
800 /*
801  * i_ddi_rem_ivintr:
802  */
803 /*ARGSUSED*/
804 void
805 i_ddi_rem_ivintr(ddi_intr_handle_impl_t *hdlp)
806 {
807 	VERIFY(rem_ivintr(hdlp->ih_vector, hdlp->ih_pri) == 0);
808 }
809 
810 /*
811  * i_ddi_get_inum - Get the interrupt number property from the
812  * specified device. Note that this function is called only for
813  * the FIXED interrupt type.
814  */
815 uint32_t
816 i_ddi_get_inum(dev_info_t *dip, uint_t inumber)
817 {
818 	int32_t			intrlen, intr_cells, max_intrs;
819 	prop_1275_cell_t	*ip, intr_sz;
820 	uint32_t		intr = 0;
821 
822 	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS |
823 	    DDI_PROP_CANSLEEP,
824 	    "interrupts", (caddr_t)&ip, &intrlen) == DDI_SUCCESS) {
825 
826 		intr_cells = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
827 		    "#interrupt-cells", 1);
828 
829 		/* adjust for number of bytes */
830 		intr_sz = CELLS_1275_TO_BYTES(intr_cells);
831 
832 		/* Calculate the number of interrupts */
833 		max_intrs = intrlen / intr_sz;
834 
835 		if (inumber < max_intrs) {
836 			prop_1275_cell_t *intrp = ip;
837 
838 			/* Index into interrupt property */
839 			intrp += (inumber * intr_cells);
840 
841 			cells_1275_copy(intrp, &intr, intr_cells);
842 		}
843 
844 		kmem_free(ip, intrlen);
845 	}
846 
847 	return (intr);
848 }
849 
850 /*
851  * i_ddi_get_intr_pri - Get the interrupt-priorities property from
852  * the specified device. Note that this function is called only for
853  * the FIXED interrupt type.
854  */
855 uint32_t
856 i_ddi_get_intr_pri(dev_info_t *dip, uint_t inumber)
857 {
858 	uint32_t	*intr_prio_p;
859 	uint32_t	pri = 0;
860 	int32_t		i;
861 
862 	/*
863 	 * Use the "interrupt-priorities" property to determine the
864 	 * the pil/ipl for the interrupt handler.
865 	 */
866 	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
867 	    "interrupt-priorities", (caddr_t)&intr_prio_p,
868 	    &i) == DDI_SUCCESS) {
869 		if (inumber < (i / sizeof (int32_t)))
870 			pri = intr_prio_p[inumber];
871 		kmem_free(intr_prio_p, i);
872 	}
873 
874 	return (pri);
875 }
876 
877 int
878 i_ddi_get_intx_nintrs(dev_info_t *dip)
879 {
880 	int32_t intrlen;
881 	prop_1275_cell_t intr_sz;
882 	prop_1275_cell_t *ip;
883 	int32_t ret = 0;
884 
885 	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS |
886 	    DDI_PROP_CANSLEEP,
887 	    "interrupts", (caddr_t)&ip, &intrlen) == DDI_SUCCESS) {
888 
889 		intr_sz = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
890 		    "#interrupt-cells", 1);
891 		/* adjust for number of bytes */
892 		intr_sz = CELLS_1275_TO_BYTES(intr_sz);
893 
894 		ret = intrlen / intr_sz;
895 
896 		kmem_free(ip, intrlen);
897 	}
898 
899 	return (ret);
900 }
901 
902 /*
903  * i_ddi_add_softint - allocate and add a software interrupt.
904  *
905  * NOTE: All software interrupts that are registered through DDI
906  *	 should be triggered only on a single target or CPU.
907  */
908 int
909 i_ddi_add_softint(ddi_softint_hdl_impl_t *hdlp)
910 {
911 	if ((hdlp->ih_private = (void *)add_softintr(hdlp->ih_pri,
912 	    hdlp->ih_cb_func, hdlp->ih_cb_arg1, SOFTINT_ST)) == NULL)
913 		return (DDI_FAILURE);
914 
915 	return (DDI_SUCCESS);
916 }
917 
918 /*
919  * i_ddi_remove_softint - remove and free a software interrupt.
920  */
921 void
922 i_ddi_remove_softint(ddi_softint_hdl_impl_t *hdlp)
923 {
924 	ASSERT(hdlp->ih_private != NULL);
925 
926 	if (rem_softintr((uint64_t)hdlp->ih_private) == 0)
927 		hdlp->ih_private = NULL;
928 }
929 
930 /*
931  * i_ddi_trigger_softint - trigger a software interrupt.
932  */
933 int
934 i_ddi_trigger_softint(ddi_softint_hdl_impl_t *hdlp, void *arg2)
935 {
936 	int	ret;
937 
938 	ASSERT(hdlp->ih_private != NULL);
939 
940 	/* Update the second argument for the software interrupt */
941 	if ((ret = update_softint_arg2((uint64_t)hdlp->ih_private, arg2)) == 0)
942 		setsoftint((uint64_t)hdlp->ih_private);
943 
944 	return (ret ? DDI_EPENDING : DDI_SUCCESS);
945 }
946 
947 /*
948  * i_ddi_set_softint_pri - change software interrupt priority.
949  */
950 /* ARGSUSED */
951 int
952 i_ddi_set_softint_pri(ddi_softint_hdl_impl_t *hdlp, uint_t old_pri)
953 {
954 	int	ret;
955 
956 	ASSERT(hdlp->ih_private != NULL);
957 
958 	/* Update the interrupt priority for the software interrupt */
959 	ret = update_softint_pri((uint64_t)hdlp->ih_private, hdlp->ih_pri);
960 
961 	return (ret ? DDI_FAILURE : DDI_SUCCESS);
962 }
963 
964 /*ARGSUSED*/
965 void
966 i_ddi_alloc_intr_phdl(ddi_intr_handle_impl_t *hdlp)
967 {
968 }
969 
970 /*ARGSUSED*/
971 void
972 i_ddi_free_intr_phdl(ddi_intr_handle_impl_t *hdlp)
973 {
974 }
975 
976 /*
977  * SECTION: DDI Memory/DMA
978  */
979 
980 /* set HAT endianess attributes from ddi_device_acc_attr */
981 void
982 i_ddi_devacc_to_hatacc(ddi_device_acc_attr_t *devaccp, uint_t *hataccp)
983 {
984 	if (devaccp != NULL) {
985 		if (devaccp->devacc_attr_endian_flags == DDI_STRUCTURE_LE_ACC) {
986 			*hataccp &= ~HAT_ENDIAN_MASK;
987 			*hataccp |= HAT_STRUCTURE_LE;
988 		}
989 	}
990 }
991 
992 /*
993  * Check if the specified cache attribute is supported on the platform.
994  * This function must be called before i_ddi_cacheattr_to_hatacc().
995  */
996 boolean_t
997 i_ddi_check_cache_attr(uint_t flags)
998 {
999 	/*
1000 	 * The cache attributes are mutually exclusive. Any combination of
1001 	 * the attributes leads to a failure.
1002 	 */
1003 	uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1004 	if ((cache_attr != 0) && !ISP2(cache_attr))
1005 		return (B_FALSE);
1006 
1007 	/*
1008 	 * On the sparc architecture, only IOMEM_DATA_CACHED is meaningful,
1009 	 * but others lead to a failure.
1010 	 */
1011 	if (cache_attr & IOMEM_DATA_CACHED)
1012 		return (B_TRUE);
1013 	else
1014 		return (B_FALSE);
1015 }
1016 
1017 /* set HAT cache attributes from the cache attributes */
1018 void
1019 i_ddi_cacheattr_to_hatacc(uint_t flags, uint_t *hataccp)
1020 {
1021 	uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1022 	static char *fname = "i_ddi_cacheattr_to_hatacc";
1023 #if defined(lint)
1024 	*hataccp = *hataccp;
1025 #endif
1026 	/*
1027 	 * set HAT attrs according to the cache attrs.
1028 	 */
1029 	switch (cache_attr) {
1030 	/*
1031 	 * The cache coherency is always maintained on SPARC, and
1032 	 * nothing is required.
1033 	 */
1034 	case IOMEM_DATA_CACHED:
1035 		break;
1036 	/*
1037 	 * Both IOMEM_DATA_UC_WRITE_COMBINED and IOMEM_DATA_UNCACHED are
1038 	 * not supported on SPARC -- this case must not occur because the
1039 	 * cache attribute is scrutinized before this function is called.
1040 	 */
1041 	case IOMEM_DATA_UNCACHED:
1042 	case IOMEM_DATA_UC_WR_COMBINE:
1043 	default:
1044 		cmn_err(CE_WARN, "%s: cache_attr=0x%x is ignored.",
1045 		    fname, cache_attr);
1046 	}
1047 }
1048 
1049 static vmem_t *little_endian_arena;
1050 static vmem_t *big_endian_arena;
1051 
1052 static void *
1053 segkmem_alloc_le(vmem_t *vmp, size_t size, int flag)
1054 {
1055 	return (segkmem_xalloc(vmp, NULL, size, flag, HAT_STRUCTURE_LE,
1056 	    segkmem_page_create, NULL));
1057 }
1058 
1059 static void *
1060 segkmem_alloc_be(vmem_t *vmp, size_t size, int flag)
1061 {
1062 	return (segkmem_xalloc(vmp, NULL, size, flag, HAT_STRUCTURE_BE,
1063 	    segkmem_page_create, NULL));
1064 }
1065 
1066 void
1067 ka_init(void)
1068 {
1069 	little_endian_arena = vmem_create("little_endian", NULL, 0, 1,
1070 	    segkmem_alloc_le, segkmem_free, heap_arena, 0, VM_SLEEP);
1071 	big_endian_arena = vmem_create("big_endian", NULL, 0, 1,
1072 	    segkmem_alloc_be, segkmem_free, heap_arena, 0, VM_SLEEP);
1073 }
1074 
1075 /*
1076  * Allocate from the system, aligned on a specific boundary.
1077  * The alignment, if non-zero, must be a power of 2.
1078  */
1079 static void *
1080 kalloca(size_t size, size_t align, int cansleep, uint_t endian_flags)
1081 {
1082 	size_t *addr, *raddr, rsize;
1083 	size_t hdrsize = 4 * sizeof (size_t);	/* must be power of 2 */
1084 
1085 	align = MAX(align, hdrsize);
1086 	ASSERT((align & (align - 1)) == 0);
1087 
1088 	/*
1089 	 * We need to allocate
1090 	 *    rsize = size + hdrsize + align - MIN(hdrsize, buffer_alignment)
1091 	 * bytes to be sure we have enough freedom to satisfy the request.
1092 	 * Since the buffer alignment depends on the request size, this is
1093 	 * not straightforward to use directly.
1094 	 *
1095 	 * kmem guarantees that any allocation of a 64-byte multiple will be
1096 	 * 64-byte aligned.  Since rounding up the request could add more
1097 	 * than we save, we compute the size with and without alignment, and
1098 	 * use the smaller of the two.
1099 	 */
1100 	rsize = size + hdrsize + align;
1101 
1102 	if (endian_flags == DDI_STRUCTURE_LE_ACC) {
1103 		raddr = vmem_alloc(little_endian_arena, rsize,
1104 		    cansleep ? VM_SLEEP : VM_NOSLEEP);
1105 	} else {
1106 		raddr = vmem_alloc(big_endian_arena, rsize,
1107 		    cansleep ? VM_SLEEP : VM_NOSLEEP);
1108 	}
1109 
1110 	if (raddr == NULL)
1111 		return (NULL);
1112 
1113 	addr = (size_t *)P2ROUNDUP((uintptr_t)raddr + hdrsize, align);
1114 	ASSERT((uintptr_t)addr + size - (uintptr_t)raddr <= rsize);
1115 
1116 	addr[-3] = (size_t)endian_flags;
1117 	addr[-2] = (size_t)raddr;
1118 	addr[-1] = rsize;
1119 
1120 	return (addr);
1121 }
1122 
1123 static void
1124 kfreea(void *addr)
1125 {
1126 	size_t *saddr = addr;
1127 
1128 	if (saddr[-3] == DDI_STRUCTURE_LE_ACC)
1129 		vmem_free(little_endian_arena, (void *)saddr[-2], saddr[-1]);
1130 	else
1131 		vmem_free(big_endian_arena, (void *)saddr[-2], saddr[-1]);
1132 }
1133 
1134 /*
1135  * This used to be ddi_iomin, but we were the only remaining caller, so
1136  * we've made it private and moved it here.
1137  */
1138 static int
1139 i_ddi_iomin(dev_info_t *a, int i, int stream)
1140 {
1141 	int r;
1142 
1143 	/*
1144 	 * Make sure that the initial value is sane
1145 	 */
1146 	if (!ISP2(i))
1147 		return (0);
1148 	if (i == 0)
1149 		i = (stream) ? 4 : 1;
1150 
1151 	r = ddi_ctlops(a, a,
1152 	    DDI_CTLOPS_IOMIN, (void *)(uintptr_t)stream, (void *)&i);
1153 	if (r != DDI_SUCCESS || !ISP2(i))
1154 		return (0);
1155 	return (i);
1156 }
1157 
1158 int
1159 i_ddi_mem_alloc(dev_info_t *dip, ddi_dma_attr_t *attr,
1160     size_t length, int cansleep, int flags,
1161     ddi_device_acc_attr_t *accattrp,
1162     caddr_t *kaddrp, size_t *real_length, ddi_acc_hdl_t *handlep)
1163 {
1164 	caddr_t a;
1165 	int iomin, align, streaming;
1166 	uint_t endian_flags = DDI_NEVERSWAP_ACC;
1167 
1168 #if defined(lint)
1169 	*handlep = *handlep;
1170 #endif
1171 
1172 	/*
1173 	 * Check legality of arguments
1174 	 */
1175 	if (length == 0 || kaddrp == NULL || attr == NULL) {
1176 		return (DDI_FAILURE);
1177 	}
1178 
1179 	if (attr->dma_attr_minxfer == 0 || attr->dma_attr_align == 0 ||
1180 	    !ISP2(attr->dma_attr_align) || !ISP2(attr->dma_attr_minxfer)) {
1181 		return (DDI_FAILURE);
1182 	}
1183 
1184 	/*
1185 	 * check if a streaming sequential xfer is requested.
1186 	 */
1187 	streaming = (flags & DDI_DMA_STREAMING) ? 1 : 0;
1188 
1189 	/*
1190 	 * Drivers for 64-bit capable SBus devices will encode
1191 	 * the burtsizes for 64-bit xfers in the upper 16-bits.
1192 	 * For DMA alignment, we use the most restrictive
1193 	 * alignment of 32-bit and 64-bit xfers.
1194 	 */
1195 	iomin = (attr->dma_attr_burstsizes & 0xffff) |
1196 	    ((attr->dma_attr_burstsizes >> 16) & 0xffff);
1197 	/*
1198 	 * If a driver set burtsizes to 0, we give him byte alignment.
1199 	 * Otherwise align at the burtsizes boundary.
1200 	 */
1201 	if (iomin == 0)
1202 		iomin = 1;
1203 	else
1204 		iomin = 1 << (ddi_fls(iomin) - 1);
1205 	iomin = maxbit(iomin, attr->dma_attr_minxfer);
1206 	iomin = maxbit(iomin, attr->dma_attr_align);
1207 	iomin = i_ddi_iomin(dip, iomin, streaming);
1208 	if (iomin == 0)
1209 		return (DDI_FAILURE);
1210 
1211 	ASSERT((iomin & (iomin - 1)) == 0);
1212 	ASSERT(iomin >= attr->dma_attr_minxfer);
1213 	ASSERT(iomin >= attr->dma_attr_align);
1214 
1215 	length = P2ROUNDUP(length, iomin);
1216 	align = iomin;
1217 
1218 	if (accattrp != NULL)
1219 		endian_flags = accattrp->devacc_attr_endian_flags;
1220 
1221 	a = kalloca(length, align, cansleep, endian_flags);
1222 	if ((*kaddrp = a) == 0) {
1223 		return (DDI_FAILURE);
1224 	} else {
1225 		if (real_length) {
1226 			*real_length = length;
1227 		}
1228 		if (handlep) {
1229 			/*
1230 			 * assign handle information
1231 			 */
1232 			impl_acc_hdl_init(handlep);
1233 		}
1234 		return (DDI_SUCCESS);
1235 	}
1236 }
1237 
1238 /* ARGSUSED */
1239 void
1240 i_ddi_mem_free(caddr_t kaddr, ddi_acc_hdl_t *ap)
1241 {
1242 	kfreea(kaddr);
1243 }
1244 
1245 /*
1246  * SECTION: DDI Data Access
1247  */
1248 
1249 static uintptr_t impl_acc_hdl_id = 0;
1250 
1251 /*
1252  * access handle allocator
1253  */
1254 ddi_acc_hdl_t *
1255 impl_acc_hdl_get(ddi_acc_handle_t hdl)
1256 {
1257 	/*
1258 	 * Extract the access handle address from the DDI implemented
1259 	 * access handle
1260 	 */
1261 	return (&((ddi_acc_impl_t *)hdl)->ahi_common);
1262 }
1263 
1264 ddi_acc_handle_t
1265 impl_acc_hdl_alloc(int (*waitfp)(caddr_t), caddr_t arg)
1266 {
1267 	ddi_acc_impl_t *hp;
1268 	on_trap_data_t *otp;
1269 	int sleepflag;
1270 
1271 	sleepflag = ((waitfp == (int (*)())KM_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
1272 
1273 	/*
1274 	 * Allocate and initialize the data access handle and error status.
1275 	 */
1276 	if ((hp = kmem_zalloc(sizeof (ddi_acc_impl_t), sleepflag)) == NULL)
1277 		goto fail;
1278 	if ((hp->ahi_err = (ndi_err_t *)kmem_zalloc(
1279 	    sizeof (ndi_err_t), sleepflag)) == NULL) {
1280 		kmem_free(hp, sizeof (ddi_acc_impl_t));
1281 		goto fail;
1282 	}
1283 	if ((otp = (on_trap_data_t *)kmem_zalloc(
1284 	    sizeof (on_trap_data_t), sleepflag)) == NULL) {
1285 		kmem_free(hp->ahi_err, sizeof (ndi_err_t));
1286 		kmem_free(hp, sizeof (ddi_acc_impl_t));
1287 		goto fail;
1288 	}
1289 	hp->ahi_err->err_ontrap = otp;
1290 	hp->ahi_common.ah_platform_private = (void *)hp;
1291 
1292 	return ((ddi_acc_handle_t)hp);
1293 fail:
1294 	if ((waitfp != (int (*)())KM_SLEEP) &&
1295 	    (waitfp != (int (*)())KM_NOSLEEP))
1296 		ddi_set_callback(waitfp, arg, &impl_acc_hdl_id);
1297 	return (NULL);
1298 }
1299 
1300 void
1301 impl_acc_hdl_free(ddi_acc_handle_t handle)
1302 {
1303 	ddi_acc_impl_t *hp;
1304 
1305 	/*
1306 	 * The supplied (ddi_acc_handle_t) is actually a (ddi_acc_impl_t *),
1307 	 * because that's what we allocated in impl_acc_hdl_alloc() above.
1308 	 */
1309 	hp = (ddi_acc_impl_t *)handle;
1310 	if (hp) {
1311 		kmem_free(hp->ahi_err->err_ontrap, sizeof (on_trap_data_t));
1312 		kmem_free(hp->ahi_err, sizeof (ndi_err_t));
1313 		kmem_free(hp, sizeof (ddi_acc_impl_t));
1314 		if (impl_acc_hdl_id)
1315 			ddi_run_callback(&impl_acc_hdl_id);
1316 	}
1317 }
1318 
1319 #define	PCI_GET_MP_PFN(mp, page_no)	((mp)->dmai_ndvmapages == 1 ? \
1320 	(pfn_t)(mp)->dmai_iopte:(((pfn_t *)(mp)->dmai_iopte)[page_no]))
1321 
1322 /*
1323  * Function called after a dma fault occurred to find out whether the
1324  * fault address is associated with a driver that is able to handle faults
1325  * and recover from faults.
1326  */
1327 /* ARGSUSED */
1328 int
1329 impl_dma_check(dev_info_t *dip, const void *handle, const void *addr,
1330     const void *not_used)
1331 {
1332 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1333 	pfn_t fault_pfn = mmu_btop(*(uint64_t *)addr);
1334 	pfn_t comp_pfn;
1335 
1336 	/*
1337 	 * The driver has to set DDI_DMA_FLAGERR to recover from dma faults.
1338 	 */
1339 	int page;
1340 
1341 	ASSERT(mp);
1342 	for (page = 0; page < mp->dmai_ndvmapages; page++) {
1343 		comp_pfn = PCI_GET_MP_PFN(mp, page);
1344 		if (fault_pfn == comp_pfn)
1345 			return (DDI_FM_NONFATAL);
1346 	}
1347 	return (DDI_FM_UNKNOWN);
1348 }
1349 
1350 /*
1351  * Function used to check if a given access handle owns the failing address.
1352  * Called by ndi_fmc_error, when we detect a PIO error.
1353  */
1354 /* ARGSUSED */
1355 static int
1356 impl_acc_check(dev_info_t *dip, const void *handle, const void *addr,
1357     const void *not_used)
1358 {
1359 	pfn_t pfn, fault_pfn;
1360 	ddi_acc_hdl_t *hp;
1361 
1362 	hp = impl_acc_hdl_get((ddi_acc_handle_t)handle);
1363 
1364 	ASSERT(hp);
1365 
1366 	if (addr != NULL) {
1367 		pfn = hp->ah_pfn;
1368 		fault_pfn = mmu_btop(*(uint64_t *)addr);
1369 		if (fault_pfn >= pfn && fault_pfn < (pfn + hp->ah_pnum))
1370 			return (DDI_FM_NONFATAL);
1371 	}
1372 	return (DDI_FM_UNKNOWN);
1373 }
1374 
1375 void
1376 impl_acc_err_init(ddi_acc_hdl_t *handlep)
1377 {
1378 	int fmcap;
1379 	ndi_err_t *errp;
1380 	on_trap_data_t *otp;
1381 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handlep;
1382 
1383 	fmcap = ddi_fm_capable(handlep->ah_dip);
1384 
1385 	if (handlep->ah_acc.devacc_attr_version < DDI_DEVICE_ATTR_V1 ||
1386 	    !DDI_FM_ACC_ERR_CAP(fmcap)) {
1387 		handlep->ah_acc.devacc_attr_access = DDI_DEFAULT_ACC;
1388 	} else if (DDI_FM_ACC_ERR_CAP(fmcap)) {
1389 		if (handlep->ah_acc.devacc_attr_access == DDI_DEFAULT_ACC) {
1390 			if (handlep->ah_xfermodes)
1391 				return;
1392 			i_ddi_drv_ereport_post(handlep->ah_dip, DVR_EFMCAP,
1393 			    NULL, DDI_NOSLEEP);
1394 		} else {
1395 			errp = hp->ahi_err;
1396 			otp = (on_trap_data_t *)errp->err_ontrap;
1397 			otp->ot_handle = (void *)(hp);
1398 			otp->ot_prot = OT_DATA_ACCESS;
1399 			if (handlep->ah_acc.devacc_attr_access ==
1400 			    DDI_CAUTIOUS_ACC)
1401 				otp->ot_trampoline =
1402 				    (uintptr_t)&i_ddi_caut_trampoline;
1403 			else
1404 				otp->ot_trampoline =
1405 				    (uintptr_t)&i_ddi_prot_trampoline;
1406 			errp->err_status = DDI_FM_OK;
1407 			errp->err_expected = DDI_FM_ERR_UNEXPECTED;
1408 			errp->err_cf = impl_acc_check;
1409 		}
1410 	}
1411 }
1412 
1413 void
1414 impl_acc_hdl_init(ddi_acc_hdl_t *handlep)
1415 {
1416 	ddi_acc_impl_t *hp;
1417 
1418 	ASSERT(handlep);
1419 
1420 	hp = (ddi_acc_impl_t *)handlep;
1421 
1422 	/*
1423 	 * check for SW byte-swapping
1424 	 */
1425 	hp->ahi_get8 = i_ddi_get8;
1426 	hp->ahi_put8 = i_ddi_put8;
1427 	hp->ahi_rep_get8 = i_ddi_rep_get8;
1428 	hp->ahi_rep_put8 = i_ddi_rep_put8;
1429 	if (handlep->ah_acc.devacc_attr_endian_flags & DDI_STRUCTURE_LE_ACC) {
1430 		hp->ahi_get16 = i_ddi_swap_get16;
1431 		hp->ahi_get32 = i_ddi_swap_get32;
1432 		hp->ahi_get64 = i_ddi_swap_get64;
1433 		hp->ahi_put16 = i_ddi_swap_put16;
1434 		hp->ahi_put32 = i_ddi_swap_put32;
1435 		hp->ahi_put64 = i_ddi_swap_put64;
1436 		hp->ahi_rep_get16 = i_ddi_swap_rep_get16;
1437 		hp->ahi_rep_get32 = i_ddi_swap_rep_get32;
1438 		hp->ahi_rep_get64 = i_ddi_swap_rep_get64;
1439 		hp->ahi_rep_put16 = i_ddi_swap_rep_put16;
1440 		hp->ahi_rep_put32 = i_ddi_swap_rep_put32;
1441 		hp->ahi_rep_put64 = i_ddi_swap_rep_put64;
1442 	} else {
1443 		hp->ahi_get16 = i_ddi_get16;
1444 		hp->ahi_get32 = i_ddi_get32;
1445 		hp->ahi_get64 = i_ddi_get64;
1446 		hp->ahi_put16 = i_ddi_put16;
1447 		hp->ahi_put32 = i_ddi_put32;
1448 		hp->ahi_put64 = i_ddi_put64;
1449 		hp->ahi_rep_get16 = i_ddi_rep_get16;
1450 		hp->ahi_rep_get32 = i_ddi_rep_get32;
1451 		hp->ahi_rep_get64 = i_ddi_rep_get64;
1452 		hp->ahi_rep_put16 = i_ddi_rep_put16;
1453 		hp->ahi_rep_put32 = i_ddi_rep_put32;
1454 		hp->ahi_rep_put64 = i_ddi_rep_put64;
1455 	}
1456 
1457 	/* Legacy fault flags and support */
1458 	hp->ahi_fault_check = i_ddi_acc_fault_check;
1459 	hp->ahi_fault_notify = i_ddi_acc_fault_notify;
1460 	hp->ahi_fault = 0;
1461 	impl_acc_err_init(handlep);
1462 }
1463 
1464 void
1465 i_ddi_acc_set_fault(ddi_acc_handle_t handle)
1466 {
1467 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle;
1468 
1469 	if (!hp->ahi_fault) {
1470 		hp->ahi_fault = 1;
1471 			(*hp->ahi_fault_notify)(hp);
1472 	}
1473 }
1474 
1475 void
1476 i_ddi_acc_clr_fault(ddi_acc_handle_t handle)
1477 {
1478 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle;
1479 
1480 	if (hp->ahi_fault) {
1481 		hp->ahi_fault = 0;
1482 			(*hp->ahi_fault_notify)(hp);
1483 	}
1484 }
1485 
1486 /* ARGSUSED */
1487 void
1488 i_ddi_acc_fault_notify(ddi_acc_impl_t *hp)
1489 {
1490 	/* Default version, does nothing */
1491 }
1492 
1493 /*
1494  * SECTION: Misc functions
1495  */
1496 
1497 /*
1498  * instance wrappers
1499  */
1500 /*ARGSUSED*/
1501 uint_t
1502 impl_assign_instance(dev_info_t *dip)
1503 {
1504 	return ((uint_t)-1);
1505 }
1506 
1507 /*ARGSUSED*/
1508 int
1509 impl_keep_instance(dev_info_t *dip)
1510 {
1511 	return (DDI_FAILURE);
1512 }
1513 
1514 /*ARGSUSED*/
1515 int
1516 impl_free_instance(dev_info_t *dip)
1517 {
1518 	return (DDI_FAILURE);
1519 }
1520 
1521 /*ARGSUSED*/
1522 int
1523 impl_check_cpu(dev_info_t *devi)
1524 {
1525 	return (DDI_SUCCESS);
1526 }
1527 
1528 
1529 static const char *nocopydevs[] = {
1530 	"SUNW,ffb",
1531 	"SUNW,afb",
1532 	NULL
1533 };
1534 
1535 /*
1536  * Perform a copy from a memory mapped device (whose devinfo pointer is devi)
1537  * separately mapped at devaddr in the kernel to a kernel buffer at kaddr.
1538  */
1539 /*ARGSUSED*/
1540 int
1541 e_ddi_copyfromdev(dev_info_t *devi,
1542     off_t off, const void *devaddr, void *kaddr, size_t len)
1543 {
1544 	const char **argv;
1545 
1546 	for (argv = nocopydevs; *argv; argv++)
1547 		if (strcmp(ddi_binding_name(devi), *argv) == 0) {
1548 			bzero(kaddr, len);
1549 			return (0);
1550 		}
1551 
1552 	bcopy(devaddr, kaddr, len);
1553 	return (0);
1554 }
1555 
1556 /*
1557  * Perform a copy to a memory mapped device (whose devinfo pointer is devi)
1558  * separately mapped at devaddr in the kernel from a kernel buffer at kaddr.
1559  */
1560 /*ARGSUSED*/
1561 int
1562 e_ddi_copytodev(dev_info_t *devi,
1563     off_t off, const void *kaddr, void *devaddr, size_t len)
1564 {
1565 	const char **argv;
1566 
1567 	for (argv = nocopydevs; *argv; argv++)
1568 		if (strcmp(ddi_binding_name(devi), *argv) == 0)
1569 			return (1);
1570 
1571 	bcopy(kaddr, devaddr, len);
1572 	return (0);
1573 }
1574 
1575 /*
1576  * Boot Configuration
1577  */
1578 idprom_t idprom;
1579 
1580 /*
1581  * Configure the hardware on the system.
1582  * Called before the rootfs is mounted
1583  */
1584 void
1585 configure(void)
1586 {
1587 	extern void i_ddi_init_root();
1588 
1589 	/* We better have released boot by this time! */
1590 	ASSERT(!bootops);
1591 
1592 	/*
1593 	 * Determine whether or not to use the fpu, V9 SPARC cpus
1594 	 * always have one. Could check for existence of a fp queue,
1595 	 * Ultra I, II and IIa do not have a fp queue.
1596 	 */
1597 	if (fpu_exists)
1598 		fpu_probe();
1599 	else
1600 		cmn_err(CE_CONT, "FPU not in use\n");
1601 
1602 #if 0 /* XXXQ - not necessary for sun4u */
1603 	/*
1604 	 * This following line fixes bugid 1041296; we need to do a
1605 	 * prom_nextnode(0) because this call ALSO patches the DMA+
1606 	 * bug in Campus-B and Phoenix. The prom uncaches the traptable
1607 	 * page as a side-effect of devr_next(0) (which prom_nextnode calls),
1608 	 * so this *must* be executed early on. (XXX This is untrue for sun4u)
1609 	 */
1610 	(void) prom_nextnode((pnode_t)0);
1611 #endif
1612 
1613 	/*
1614 	 * Initialize devices on the machine.
1615 	 * Uses configuration tree built by the PROMs to determine what
1616 	 * is present, and builds a tree of prototype dev_info nodes
1617 	 * corresponding to the hardware which identified itself.
1618 	 */
1619 	i_ddi_init_root();
1620 
1621 #ifdef	DDI_PROP_DEBUG
1622 	(void) ddi_prop_debug(1);	/* Enable property debugging */
1623 #endif	/* DDI_PROP_DEBUG */
1624 }
1625 
1626 /*
1627  * The "status" property indicates the operational status of a device.
1628  * If this property is present, the value is a string indicating the
1629  * status of the device as follows:
1630  *
1631  *	"okay"		operational.
1632  *	"disabled"	not operational, but might become operational.
1633  *	"fail"		not operational because a fault has been detected,
1634  *			and it is unlikely that the device will become
1635  *			operational without repair. no additional details
1636  *			are available.
1637  *	"fail-xxx"	not operational because a fault has been detected,
1638  *			and it is unlikely that the device will become
1639  *			operational without repair. "xxx" is additional
1640  *			human-readable information about the particular
1641  *			fault condition that was detected.
1642  *
1643  * The absence of this property means that the operational status is
1644  * unknown or okay.
1645  *
1646  * This routine checks the status property of the specified device node
1647  * and returns 0 if the operational status indicates failure, and 1 otherwise.
1648  *
1649  * The property may exist on plug-in cards the existed before IEEE 1275-1994.
1650  * And, in that case, the property may not even be a string. So we carefully
1651  * check for the value "fail", in the beginning of the string, noting
1652  * the property length.
1653  */
1654 int
1655 status_okay(int id, char *buf, int buflen)
1656 {
1657 	char status_buf[OBP_MAXPROPNAME];
1658 	char *bufp = buf;
1659 	int len = buflen;
1660 	int proplen;
1661 	static const char *status = "status";
1662 	static const char *fail = "fail";
1663 	size_t fail_len = strlen(fail);
1664 
1665 	/*
1666 	 * Get the proplen ... if it's smaller than "fail",
1667 	 * or doesn't exist ... then we don't care, since
1668 	 * the value can't begin with the char string "fail".
1669 	 *
1670 	 * NB: proplen, if it's a string, includes the NULL in the
1671 	 * the size of the property, and fail_len does not.
1672 	 */
1673 	proplen = prom_getproplen((pnode_t)id, (caddr_t)status);
1674 	if (proplen <= fail_len)	/* nonexistent or uninteresting len */
1675 		return (1);
1676 
1677 	/*
1678 	 * if a buffer was provided, use it
1679 	 */
1680 	if ((buf == (char *)NULL) || (buflen <= 0)) {
1681 		bufp = status_buf;
1682 		len = sizeof (status_buf);
1683 	}
1684 	*bufp = (char)0;
1685 
1686 	/*
1687 	 * Get the property into the buffer, to the extent of the buffer,
1688 	 * and in case the buffer is smaller than the property size,
1689 	 * NULL terminate the buffer. (This handles the case where
1690 	 * a buffer was passed in and the caller wants to print the
1691 	 * value, but the buffer was too small).
1692 	 */
1693 	(void) prom_bounded_getprop((pnode_t)id, (caddr_t)status,
1694 	    (caddr_t)bufp, len);
1695 	*(bufp + len - 1) = (char)0;
1696 
1697 	/*
1698 	 * If the value begins with the char string "fail",
1699 	 * then it means the node is failed. We don't care
1700 	 * about any other values. We assume the node is ok
1701 	 * although it might be 'disabled'.
1702 	 */
1703 	if (strncmp(bufp, fail, fail_len) == 0)
1704 		return (0);
1705 
1706 	return (1);
1707 }
1708 
1709 
1710 /*
1711  * We set the cpu type from the idprom, if we can.
1712  * Note that we just read out the contents of it, for the most part.
1713  */
1714 void
1715 setcputype(void)
1716 {
1717 	/*
1718 	 * We cache the idprom info early on so that we don't
1719 	 * rummage through the NVRAM unnecessarily later.
1720 	 */
1721 	(void) prom_getidprom((caddr_t)&idprom, sizeof (idprom));
1722 }
1723 
1724 /*
1725  *  Here is where we actually infer meanings to the members of idprom_t
1726  */
1727 void
1728 parse_idprom(void)
1729 {
1730 	if (idprom.id_format == IDFORM_1) {
1731 		(void) localetheraddr((struct ether_addr *)idprom.id_ether,
1732 		    (struct ether_addr *)NULL);
1733 		(void) snprintf(hw_serial, HW_HOSTID_LEN, "%u",
1734 		    (idprom.id_machine << 24) + idprom.id_serial);
1735 	} else
1736 		prom_printf("Invalid format code in IDprom.\n");
1737 }
1738 
1739 /*
1740  * Allow for implementation specific correction of PROM property values.
1741  */
1742 /*ARGSUSED*/
1743 void
1744 impl_fix_props(dev_info_t *dip, dev_info_t *ch_dip, char *name, int len,
1745     caddr_t buffer)
1746 {
1747 	/*
1748 	 * There are no adjustments needed in this implementation.
1749 	 */
1750 }
1751 
1752 /*
1753  * The following functions ready a cautious request to go up to the nexus
1754  * driver.  It is up to the nexus driver to decide how to process the request.
1755  * It may choose to call i_ddi_do_caut_get/put in this file, or do it
1756  * differently.
1757  */
1758 
1759 static void
1760 i_ddi_caut_getput_ctlops(
1761     ddi_acc_impl_t *hp, uint64_t host_addr, uint64_t dev_addr, size_t size,
1762     size_t repcount, uint_t flags, ddi_ctl_enum_t cmd)
1763 {
1764 	peekpoke_ctlops_t	cautacc_ctlops_arg;
1765 
1766 	cautacc_ctlops_arg.size = size;
1767 	cautacc_ctlops_arg.dev_addr = dev_addr;
1768 	cautacc_ctlops_arg.host_addr = host_addr;
1769 	cautacc_ctlops_arg.handle = (ddi_acc_handle_t)hp;
1770 	cautacc_ctlops_arg.repcount = repcount;
1771 	cautacc_ctlops_arg.flags = flags;
1772 
1773 	(void) ddi_ctlops(hp->ahi_common.ah_dip, hp->ahi_common.ah_dip, cmd,
1774 	    &cautacc_ctlops_arg, NULL);
1775 }
1776 
1777 uint8_t
1778 i_ddi_caut_get8(ddi_acc_impl_t *hp, uint8_t *addr)
1779 {
1780 	uint8_t value;
1781 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1782 	    sizeof (uint8_t), 1, 0, DDI_CTLOPS_PEEK);
1783 
1784 	return (value);
1785 }
1786 
1787 uint16_t
1788 i_ddi_caut_get16(ddi_acc_impl_t *hp, uint16_t *addr)
1789 {
1790 	uint16_t value;
1791 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1792 	    sizeof (uint16_t), 1, 0, DDI_CTLOPS_PEEK);
1793 
1794 	return (value);
1795 }
1796 
1797 uint32_t
1798 i_ddi_caut_get32(ddi_acc_impl_t *hp, uint32_t *addr)
1799 {
1800 	uint32_t value;
1801 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1802 	    sizeof (uint32_t), 1, 0, DDI_CTLOPS_PEEK);
1803 
1804 	return (value);
1805 }
1806 
1807 uint64_t
1808 i_ddi_caut_get64(ddi_acc_impl_t *hp, uint64_t *addr)
1809 {
1810 	uint64_t value;
1811 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1812 	    sizeof (uint64_t), 1, 0, DDI_CTLOPS_PEEK);
1813 
1814 	return (value);
1815 }
1816 
1817 void
1818 i_ddi_caut_put8(ddi_acc_impl_t *hp, uint8_t *addr, uint8_t value)
1819 {
1820 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1821 	    sizeof (uint8_t), 1, 0, DDI_CTLOPS_POKE);
1822 }
1823 
1824 void
1825 i_ddi_caut_put16(ddi_acc_impl_t *hp, uint16_t *addr, uint16_t value)
1826 {
1827 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1828 	    sizeof (uint16_t), 1, 0, DDI_CTLOPS_POKE);
1829 }
1830 
1831 void
1832 i_ddi_caut_put32(ddi_acc_impl_t *hp, uint32_t *addr, uint32_t value)
1833 {
1834 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1835 	    sizeof (uint32_t), 1, 0, DDI_CTLOPS_POKE);
1836 }
1837 
1838 void
1839 i_ddi_caut_put64(ddi_acc_impl_t *hp, uint64_t *addr, uint64_t value)
1840 {
1841 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1842 	    sizeof (uint64_t), 1, 0, DDI_CTLOPS_POKE);
1843 }
1844 
1845 void
1846 i_ddi_caut_rep_get8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
1847 	size_t repcount, uint_t flags)
1848 {
1849 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1850 	    sizeof (uint8_t), repcount, flags, DDI_CTLOPS_PEEK);
1851 }
1852 
1853 void
1854 i_ddi_caut_rep_get16(ddi_acc_impl_t *hp, uint16_t *host_addr,
1855     uint16_t *dev_addr, size_t repcount, uint_t flags)
1856 {
1857 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1858 	    sizeof (uint16_t), repcount, flags, DDI_CTLOPS_PEEK);
1859 }
1860 
1861 void
1862 i_ddi_caut_rep_get32(ddi_acc_impl_t *hp, uint32_t *host_addr,
1863     uint32_t *dev_addr, size_t repcount, uint_t flags)
1864 {
1865 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1866 	    sizeof (uint32_t), repcount, flags, DDI_CTLOPS_PEEK);
1867 }
1868 
1869 void
1870 i_ddi_caut_rep_get64(ddi_acc_impl_t *hp, uint64_t *host_addr,
1871     uint64_t *dev_addr, size_t repcount, uint_t flags)
1872 {
1873 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1874 	    sizeof (uint64_t), repcount, flags, DDI_CTLOPS_PEEK);
1875 }
1876 
1877 void
1878 i_ddi_caut_rep_put8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
1879 	size_t repcount, uint_t flags)
1880 {
1881 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1882 	    sizeof (uint8_t), repcount, flags, DDI_CTLOPS_POKE);
1883 }
1884 
1885 void
1886 i_ddi_caut_rep_put16(ddi_acc_impl_t *hp, uint16_t *host_addr,
1887     uint16_t *dev_addr, size_t repcount, uint_t flags)
1888 {
1889 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1890 	    sizeof (uint16_t), repcount, flags, DDI_CTLOPS_POKE);
1891 }
1892 
1893 void
1894 i_ddi_caut_rep_put32(ddi_acc_impl_t *hp, uint32_t *host_addr,
1895     uint32_t *dev_addr, size_t repcount, uint_t flags)
1896 {
1897 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1898 	    sizeof (uint32_t), repcount, flags, DDI_CTLOPS_POKE);
1899 }
1900 
1901 void
1902 i_ddi_caut_rep_put64(ddi_acc_impl_t *hp, uint64_t *host_addr,
1903     uint64_t *dev_addr, size_t repcount, uint_t flags)
1904 {
1905 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1906 	    sizeof (uint64_t), repcount, flags, DDI_CTLOPS_POKE);
1907 }
1908 
1909 /*
1910  * This is called only to process peek/poke when the DIP is NULL.
1911  * Assume that this is for memory, as nexi take care of device safe accesses.
1912  */
1913 int
1914 peekpoke_mem(ddi_ctl_enum_t cmd, peekpoke_ctlops_t *in_args)
1915 {
1916 	int err = DDI_SUCCESS;
1917 	on_trap_data_t otd;
1918 
1919 	/* Set up protected environment. */
1920 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
1921 		uintptr_t tramp = otd.ot_trampoline;
1922 
1923 		if (cmd == DDI_CTLOPS_POKE) {
1924 			otd.ot_trampoline = (uintptr_t)&poke_fault;
1925 			err = do_poke(in_args->size, (void *)in_args->dev_addr,
1926 			    (void *)in_args->host_addr);
1927 		} else {
1928 			otd.ot_trampoline = (uintptr_t)&peek_fault;
1929 			err = do_peek(in_args->size, (void *)in_args->dev_addr,
1930 			    (void *)in_args->host_addr);
1931 		}
1932 		otd.ot_trampoline = tramp;
1933 	} else
1934 		err = DDI_FAILURE;
1935 
1936 	/* Take down protected environment. */
1937 	no_trap();
1938 
1939 	return (err);
1940 }
1941