xref: /illumos-gate/usr/src/uts/sun4/os/ddi_impl.c (revision 48bbca81)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 /*
27  * Copyright 2014 Garrett D'Amore <garrett@damore.org>
28  * Copyright (c) 2016 by Delphix. All rights reserved.
29  */
30 
31 /*
32  * sun4 specific DDI implementation
33  */
34 #include <sys/cpuvar.h>
35 #include <sys/ddi_subrdefs.h>
36 #include <sys/machsystm.h>
37 #include <sys/sunndi.h>
38 #include <sys/sysmacros.h>
39 #include <sys/ontrap.h>
40 #include <vm/seg_kmem.h>
41 #include <sys/membar.h>
42 #include <sys/dditypes.h>
43 #include <sys/ndifm.h>
44 #include <sys/fm/io/ddi.h>
45 #include <sys/ivintr.h>
46 #include <sys/bootconf.h>
47 #include <sys/conf.h>
48 #include <sys/ethernet.h>
49 #include <sys/idprom.h>
50 #include <sys/promif.h>
51 #include <sys/prom_plat.h>
52 #include <sys/systeminfo.h>
53 #include <sys/fpu/fpusystm.h>
54 #include <sys/vm.h>
55 #include <sys/ddi_isa.h>
56 #include <sys/modctl.h>
57 
58 dev_info_t *get_intr_parent(dev_info_t *, dev_info_t *,
59     ddi_intr_handle_impl_t *);
60 #pragma weak get_intr_parent
61 
62 int process_intr_ops(dev_info_t *, dev_info_t *, ddi_intr_op_t,
63     ddi_intr_handle_impl_t *, void *);
64 #pragma weak process_intr_ops
65 
66 void cells_1275_copy(prop_1275_cell_t *, prop_1275_cell_t *, int32_t);
67     prop_1275_cell_t *cells_1275_cmp(prop_1275_cell_t *, prop_1275_cell_t *,
68     int32_t len);
69 #pragma weak cells_1275_copy
70 
71 /*
72  * Wrapper for ddi_prop_lookup_int_array().
73  * This is handy because it returns the prop length in
74  * bytes which is what most of the callers require.
75  */
76 
77 static int
get_prop_int_array(dev_info_t * di,char * pname,int ** pval,uint_t * plen)78 get_prop_int_array(dev_info_t *di, char *pname, int **pval, uint_t *plen)
79 {
80 	int ret;
81 
82 	if ((ret = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, di,
83 	    DDI_PROP_DONTPASS, pname, pval, plen)) == DDI_PROP_SUCCESS) {
84 		*plen = (*plen) * (uint_t)sizeof (int);
85 	}
86 	return (ret);
87 }
88 
89 /*
90  * SECTION: DDI Node Configuration
91  */
92 
93 /*
94  * init_regspec_64:
95  *
96  * If the parent #size-cells is 2, convert the upa-style or
97  * safari-style reg property from 2-size cells to 1 size cell
98  * format, ignoring the size_hi, which must be zero for devices.
99  * (It won't be zero in the memory list properties in the memory
100  * nodes, but that doesn't matter here.)
101  */
102 struct ddi_parent_private_data *
init_regspec_64(dev_info_t * dip)103 init_regspec_64(dev_info_t *dip)
104 {
105 	struct ddi_parent_private_data *pd;
106 	dev_info_t *parent;
107 	int size_cells;
108 
109 	/*
110 	 * If there are no "reg"s in the child node, return.
111 	 */
112 	pd = ddi_get_parent_data(dip);
113 	if ((pd == NULL) || (pd->par_nreg == 0)) {
114 		return (pd);
115 	}
116 	parent = ddi_get_parent(dip);
117 
118 	size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
119 	    DDI_PROP_DONTPASS, "#size-cells", 1);
120 
121 	if (size_cells != 1)  {
122 
123 		int n, j;
124 		struct regspec *irp;
125 		struct reg_64 {
126 			uint_t addr_hi, addr_lo, size_hi, size_lo;
127 		};
128 		struct reg_64 *r64_rp;
129 		struct regspec *rp;
130 		uint_t len = 0;
131 		int *reg_prop;
132 
133 		ASSERT(size_cells == 2);
134 
135 		/*
136 		 * We already looked the property up once before if
137 		 * pd is non-NULL.
138 		 */
139 		(void) ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
140 		    DDI_PROP_DONTPASS, OBP_REG, &reg_prop, &len);
141 		ASSERT(len != 0);
142 
143 		n = sizeof (struct reg_64) / sizeof (int);
144 		n = len / n;
145 
146 		/*
147 		 * We're allocating a buffer the size of the PROM's property,
148 		 * but we're only using a smaller portion when we assign it
149 		 * to a regspec.  We do this so that in the
150 		 * impl_ddi_sunbus_removechild function, we will
151 		 * always free the right amount of memory.
152 		 */
153 		irp = rp = (struct regspec *)reg_prop;
154 		r64_rp = (struct reg_64 *)pd->par_reg;
155 
156 		for (j = 0; j < n; ++j, ++rp, ++r64_rp) {
157 			ASSERT(r64_rp->size_hi == 0);
158 			rp->regspec_bustype = r64_rp->addr_hi;
159 			rp->regspec_addr = r64_rp->addr_lo;
160 			rp->regspec_size = r64_rp->size_lo;
161 		}
162 
163 		ddi_prop_free((void *)pd->par_reg);
164 		pd->par_nreg = n;
165 		pd->par_reg = irp;
166 	}
167 	return (pd);
168 }
169 
170 /*
171  * Create a ddi_parent_private_data structure from the ddi properties of
172  * the dev_info node.
173  *
174  * The "reg" is required if the driver wishes to create mappings on behalf
175  * of the device. The "reg" property is assumed to be a list of at least
176  * one triplet
177  *
178  *	<bustype, address, size>*1
179  *
180  * The "interrupt" property is no longer part of parent private data on
181  * sun4u. The interrupt parent is may not be the device tree parent.
182  *
183  * The "ranges" property describes the mapping of child addresses to parent
184  * addresses.
185  *
186  * N.B. struct rangespec is defined for the following default values:
187  *			parent  child
188  *	#address-cells	2	2
189  *	#size-cells	1	1
190  * This function doesn't deal with non-default cells and will not create
191  * ranges in such cases.
192  */
193 void
make_ddi_ppd(dev_info_t * child,struct ddi_parent_private_data ** ppd)194 make_ddi_ppd(dev_info_t *child, struct ddi_parent_private_data **ppd)
195 {
196 	struct ddi_parent_private_data *pdptr;
197 	int *reg_prop, *rng_prop;
198 	uint_t reg_len = 0, rng_len = 0;
199 	dev_info_t *parent;
200 	int parent_addr_cells, parent_size_cells;
201 	int child_addr_cells, child_size_cells;
202 
203 	*ppd = pdptr = kmem_zalloc(sizeof (*pdptr), KM_SLEEP);
204 
205 	/*
206 	 * root node has no parent private data, so *ppd should
207 	 * be initialized for naming to work properly.
208 	 */
209 	if ((parent = ddi_get_parent(child)) == NULL)
210 		return;
211 
212 	/*
213 	 * Set reg field of parent data from "reg" property
214 	 */
215 	if ((get_prop_int_array(child, OBP_REG, &reg_prop, &reg_len)
216 	    == DDI_PROP_SUCCESS) && (reg_len != 0)) {
217 		pdptr->par_nreg = (int)(reg_len / sizeof (struct regspec));
218 		pdptr->par_reg = (struct regspec *)reg_prop;
219 	}
220 
221 	/*
222 	 * "ranges" property ...
223 	 *
224 	 * This function does not handle cases where #address-cells != 2
225 	 * and * min(parent, child) #size-cells != 1 (see bugid 4211124).
226 	 *
227 	 * Nexus drivers with such exceptions (e.g. pci ranges)
228 	 * should either create a separate function for handling
229 	 * ranges or not use parent private data to store ranges.
230 	 */
231 
232 	/* root node has no ranges */
233 	if ((parent = ddi_get_parent(child)) == NULL)
234 		return;
235 
236 	child_addr_cells = ddi_prop_get_int(DDI_DEV_T_ANY, child,
237 	    DDI_PROP_DONTPASS, "#address-cells", 2);
238 	child_size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, child,
239 	    DDI_PROP_DONTPASS, "#size-cells", 1);
240 	parent_addr_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
241 	    DDI_PROP_DONTPASS, "#address-cells", 2);
242 	parent_size_cells = ddi_prop_get_int(DDI_DEV_T_ANY, parent,
243 	    DDI_PROP_DONTPASS, "#size-cells", 1);
244 	if (child_addr_cells != 2 || parent_addr_cells != 2 ||
245 	    (child_size_cells != 1 && parent_size_cells != 1)) {
246 		NDI_CONFIG_DEBUG((CE_NOTE, "!ranges not made in parent data; "
247 		    "#address-cells or #size-cells have non-default value"));
248 		return;
249 	}
250 
251 	if (get_prop_int_array(child, OBP_RANGES, &rng_prop, &rng_len)
252 	    == DDI_PROP_SUCCESS) {
253 		pdptr->par_nrng = rng_len / (int)(sizeof (struct rangespec));
254 		pdptr->par_rng = (struct rangespec *)rng_prop;
255 	}
256 }
257 
258 /*
259  * Free ddi_parent_private_data structure
260  */
261 void
impl_free_ddi_ppd(dev_info_t * dip)262 impl_free_ddi_ppd(dev_info_t *dip)
263 {
264 	struct ddi_parent_private_data *pdptr = ddi_get_parent_data(dip);
265 
266 	if (pdptr == NULL)
267 		return;
268 
269 	if (pdptr->par_nrng != 0)
270 		ddi_prop_free((void *)pdptr->par_rng);
271 
272 	if (pdptr->par_nreg != 0)
273 		ddi_prop_free((void *)pdptr->par_reg);
274 
275 	kmem_free(pdptr, sizeof (*pdptr));
276 	ddi_set_parent_data(dip, NULL);
277 }
278 
279 /*
280  * Name a child of sun busses based on the reg spec.
281  * Handles the following properties:
282  *
283  *	Property	value
284  *	Name		type
285  *
286  *	reg		register spec
287  *	interrupts	new (bus-oriented) interrupt spec
288  *	ranges		range spec
289  *
290  * This may be called multiple times, independent of
291  * initchild calls.
292  */
293 static int
impl_sunbus_name_child(dev_info_t * child,char * name,int namelen)294 impl_sunbus_name_child(dev_info_t *child, char *name, int namelen)
295 {
296 	struct ddi_parent_private_data *pdptr;
297 	struct regspec *rp;
298 
299 	/*
300 	 * Fill in parent-private data and this function returns to us
301 	 * an indication if it used "registers" to fill in the data.
302 	 */
303 	if (ddi_get_parent_data(child) == NULL) {
304 		make_ddi_ppd(child, &pdptr);
305 		ddi_set_parent_data(child, pdptr);
306 	}
307 
308 	/*
309 	 * No reg property, return null string as address
310 	 * (e.g. root node)
311 	 */
312 	name[0] = '\0';
313 	if (sparc_pd_getnreg(child) == 0) {
314 		return (DDI_SUCCESS);
315 	}
316 
317 	rp = sparc_pd_getreg(child, 0);
318 	(void) snprintf(name, namelen, "%x,%x",
319 	    rp->regspec_bustype, rp->regspec_addr);
320 	return (DDI_SUCCESS);
321 }
322 
323 
324 /*
325  * Called from the bus_ctl op of some drivers.
326  * to implement the DDI_CTLOPS_INITCHILD operation.
327  *
328  * NEW drivers should NOT use this function, but should declare
329  * there own initchild/uninitchild handlers. (This function assumes
330  * the layout of the parent private data and the format of "reg",
331  * "ranges", "interrupts" properties and that #address-cells and
332  * #size-cells of the parent bus are defined to be default values.)
333  */
334 int
impl_ddi_sunbus_initchild(dev_info_t * child)335 impl_ddi_sunbus_initchild(dev_info_t *child)
336 {
337 	char name[MAXNAMELEN];
338 
339 	(void) impl_sunbus_name_child(child, name, MAXNAMELEN);
340 	ddi_set_name_addr(child, name);
341 
342 	/*
343 	 * Try to merge .conf node. If successful, return failure to
344 	 * remove this child.
345 	 */
346 	if ((ndi_dev_is_persistent_node(child) == 0) &&
347 	    (ndi_merge_node(child, impl_sunbus_name_child) == DDI_SUCCESS)) {
348 		impl_ddi_sunbus_removechild(child);
349 		return (DDI_FAILURE);
350 	}
351 	return (DDI_SUCCESS);
352 }
353 
354 /*
355  * A better name for this function would be impl_ddi_sunbus_uninitchild()
356  * It does not remove the child, it uninitializes it, reclaiming the
357  * resources taken by impl_ddi_sunbus_initchild.
358  */
359 void
impl_ddi_sunbus_removechild(dev_info_t * dip)360 impl_ddi_sunbus_removechild(dev_info_t *dip)
361 {
362 	impl_free_ddi_ppd(dip);
363 	ddi_set_name_addr(dip, NULL);
364 	/*
365 	 * Strip the node to properly convert it back to prototype form
366 	 */
367 	impl_rem_dev_props(dip);
368 }
369 
370 /*
371  * SECTION: DDI Interrupt
372  */
373 
374 void
cells_1275_copy(prop_1275_cell_t * from,prop_1275_cell_t * to,int32_t len)375 cells_1275_copy(prop_1275_cell_t *from, prop_1275_cell_t *to, int32_t len)
376 {
377 	int i;
378 	for (i = 0; i < len; i++)
379 		*to = *from;
380 }
381 
382 prop_1275_cell_t *
cells_1275_cmp(prop_1275_cell_t * cell1,prop_1275_cell_t * cell2,int32_t len)383 cells_1275_cmp(prop_1275_cell_t *cell1, prop_1275_cell_t *cell2, int32_t len)
384 {
385 	prop_1275_cell_t *match_cell = 0;
386 	int32_t i;
387 
388 	for (i = 0; i < len; i++)
389 		if (cell1[i] != cell2[i]) {
390 			match_cell = &cell1[i];
391 			break;
392 		}
393 
394 	return (match_cell);
395 }
396 
397 /*
398  * get_intr_parent() is a generic routine that process a 1275 interrupt
399  * map (imap) property.  This function returns a dev_info_t structure
400  * which claims ownership of the interrupt domain.
401  * It also returns the new interrupt translation within this new domain.
402  * If an interrupt-parent or interrupt-map property are not found,
403  * then we fallback to using the device tree's parent.
404  *
405  * imap entry format:
406  * <reg>,<interrupt>,<phandle>,<translated interrupt>
407  * reg - The register specification in the interrupts domain
408  * interrupt - The interrupt specification
409  * phandle - PROM handle of the device that owns the xlated interrupt domain
410  * translated interrupt - interrupt specifier in the parents domain
411  * note: <reg>,<interrupt> - The reg and interrupt can be combined to create
412  *	a unique entry called a unit interrupt specifier.
413  *
414  * Here's the processing steps:
415  * step1 - If the interrupt-parent property exists, create the ispec and
416  *	return the dip of the interrupt parent.
417  * step2 - Extract the interrupt-map property and the interrupt-map-mask
418  *	If these don't exist, just return the device tree parent.
419  * step3 - build up the unit interrupt specifier to match against the
420  *	interrupt map property
421  * step4 - Scan the interrupt-map property until a match is found
422  * step4a - Extract the interrupt parent
423  * step4b - Compare the unit interrupt specifier
424  */
425 dev_info_t *
get_intr_parent(dev_info_t * pdip,dev_info_t * dip,ddi_intr_handle_impl_t * hdlp)426 get_intr_parent(dev_info_t *pdip, dev_info_t *dip, ddi_intr_handle_impl_t *hdlp)
427 {
428 	prop_1275_cell_t *imap, *imap_mask, *scan, *reg_p, *match_req;
429 	int32_t imap_sz, imap_cells, imap_scan_cells, imap_mask_sz,
430 	    addr_cells, intr_cells, reg_len, i, j;
431 	int32_t match_found = 0;
432 	dev_info_t *intr_parent_dip = NULL;
433 	uint32_t *intr = &hdlp->ih_vector;
434 	uint32_t nodeid;
435 #ifdef DEBUG
436 	static int debug = 0;
437 #endif
438 
439 	/*
440 	 * step1
441 	 * If we have an interrupt-parent property, this property represents
442 	 * the nodeid of our interrupt parent.
443 	 */
444 	if ((nodeid = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
445 	    "interrupt-parent", -1)) != -1) {
446 		intr_parent_dip = e_ddi_nodeid_to_dip(nodeid);
447 		ASSERT(intr_parent_dip);
448 
449 		/*
450 		 * Attach the interrupt parent.
451 		 *
452 		 * N.B. e_ddi_nodeid_to_dip() isn't safe under DR.
453 		 *	Also, interrupt parent isn't held. This needs
454 		 *	to be revisited if DR-capable platforms implement
455 		 *	interrupt redirection.
456 		 */
457 		if (i_ddi_attach_node_hierarchy(intr_parent_dip)
458 		    != DDI_SUCCESS) {
459 			ndi_rele_devi(intr_parent_dip);
460 			return (NULL);
461 		}
462 
463 		return (intr_parent_dip);
464 	}
465 
466 	/*
467 	 * step2
468 	 * Get interrupt map structure from PROM property
469 	 */
470 	if (ddi_getlongprop(DDI_DEV_T_ANY, pdip, DDI_PROP_DONTPASS,
471 	    "interrupt-map", (caddr_t)&imap, &imap_sz)
472 	    != DDI_PROP_SUCCESS) {
473 		/*
474 		 * If we don't have an imap property, default to using the
475 		 * device tree.
476 		 */
477 
478 		ndi_hold_devi(pdip);
479 		return (pdip);
480 	}
481 
482 	/* Get the interrupt mask property */
483 	if (ddi_getlongprop(DDI_DEV_T_ANY, pdip, DDI_PROP_DONTPASS,
484 	    "interrupt-map-mask", (caddr_t)&imap_mask, &imap_mask_sz)
485 	    != DDI_PROP_SUCCESS) {
486 		/*
487 		 * If we don't find this property, we have to fail the request
488 		 * because the 1275 imap property wasn't defined correctly.
489 		 */
490 		ASSERT(intr_parent_dip == NULL);
491 		goto exit2;
492 	}
493 
494 	/* Get the address cell size */
495 	addr_cells = ddi_getprop(DDI_DEV_T_ANY, pdip, 0,
496 	    "#address-cells", 2);
497 
498 	/* Get the interrupts cell size */
499 	intr_cells = ddi_getprop(DDI_DEV_T_ANY, pdip, 0,
500 	    "#interrupt-cells", 1);
501 
502 	/*
503 	 * step3
504 	 * Now lets build up the unit interrupt specifier e.g. reg,intr
505 	 * and apply the imap mask.  match_req will hold this when we're
506 	 * through.
507 	 */
508 	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "reg",
509 	    (caddr_t)&reg_p, &reg_len) != DDI_SUCCESS) {
510 		ASSERT(intr_parent_dip == NULL);
511 		goto exit3;
512 	}
513 
514 	match_req = kmem_alloc(CELLS_1275_TO_BYTES(addr_cells) +
515 	    CELLS_1275_TO_BYTES(intr_cells), KM_SLEEP);
516 
517 	for (i = 0; i < addr_cells; i++)
518 		match_req[i] = (reg_p[i] & imap_mask[i]);
519 
520 	for (j = 0; j < intr_cells; i++, j++)
521 		match_req[i] = (intr[j] & imap_mask[i]);
522 
523 	/* Calculate the imap size in cells */
524 	imap_cells = BYTES_TO_1275_CELLS(imap_sz);
525 
526 #ifdef DEBUG
527 	if (debug)
528 		prom_printf("reg cell size 0x%x, intr cell size 0x%x, "
529 		    "match_request 0x%p, imap 0x%p\n", addr_cells, intr_cells,
530 		    (void *)match_req, (void *)imap);
531 #endif
532 
533 	/*
534 	 * Scan the imap property looking for a match of the interrupt unit
535 	 * specifier.  This loop is rather complex since the data within the
536 	 * imap property may vary in size.
537 	 */
538 	for (scan = imap, imap_scan_cells = i = 0;
539 	    imap_scan_cells < imap_cells; scan += i, imap_scan_cells += i) {
540 		int new_intr_cells;
541 
542 		/* Set the index to the nodeid field */
543 		i = addr_cells + intr_cells;
544 
545 		/*
546 		 * step4a
547 		 * Translate the nodeid field to a dip
548 		 */
549 		ASSERT(intr_parent_dip == NULL);
550 		intr_parent_dip = e_ddi_nodeid_to_dip((uint_t)scan[i++]);
551 
552 		ASSERT(intr_parent_dip != 0);
553 #ifdef DEBUG
554 		if (debug)
555 			prom_printf("scan 0x%p\n", (void *)scan);
556 #endif
557 		/*
558 		 * The tmp_dip describes the new domain, get it's interrupt
559 		 * cell size
560 		 */
561 		new_intr_cells = ddi_getprop(DDI_DEV_T_ANY, intr_parent_dip, 0,
562 		    "#interrupts-cells", 1);
563 
564 		/*
565 		 * step4b
566 		 * See if we have a match on the interrupt unit specifier
567 		 */
568 		if (cells_1275_cmp(match_req, scan, addr_cells + intr_cells)
569 		    == 0) {
570 			uint32_t *intr;
571 
572 			match_found = 1;
573 
574 			/*
575 			 * If we have an imap parent whose not in our device
576 			 * tree path, we need to hold and install that driver.
577 			 */
578 			if (i_ddi_attach_node_hierarchy(intr_parent_dip)
579 			    != DDI_SUCCESS) {
580 				ndi_rele_devi(intr_parent_dip);
581 				intr_parent_dip = (dev_info_t *)NULL;
582 				goto exit4;
583 			}
584 
585 			/*
586 			 * We need to handcraft an ispec along with a bus
587 			 * interrupt value, so we can dup it into our
588 			 * standard ispec structure.
589 			 */
590 			/* Extract the translated interrupt information */
591 			intr = kmem_alloc(
592 			    CELLS_1275_TO_BYTES(new_intr_cells), KM_SLEEP);
593 
594 			for (j = 0; j < new_intr_cells; j++, i++)
595 				intr[j] = scan[i];
596 
597 			cells_1275_copy(intr, &hdlp->ih_vector, new_intr_cells);
598 
599 			kmem_free(intr, CELLS_1275_TO_BYTES(new_intr_cells));
600 
601 #ifdef DEBUG
602 			if (debug)
603 				prom_printf("dip 0x%p\n",
604 				    (void *)intr_parent_dip);
605 #endif
606 			break;
607 		} else {
608 #ifdef DEBUG
609 			if (debug)
610 				prom_printf("dip 0x%p\n",
611 				    (void *)intr_parent_dip);
612 #endif
613 			ndi_rele_devi(intr_parent_dip);
614 			intr_parent_dip = NULL;
615 			i += new_intr_cells;
616 		}
617 	}
618 
619 	/*
620 	 * If we haven't found our interrupt parent at this point, fallback
621 	 * to using the device tree.
622 	 */
623 	if (!match_found) {
624 		ndi_hold_devi(pdip);
625 		ASSERT(intr_parent_dip == NULL);
626 		intr_parent_dip = pdip;
627 	}
628 
629 	ASSERT(intr_parent_dip != NULL);
630 
631 exit4:
632 	kmem_free(reg_p, reg_len);
633 	kmem_free(match_req, CELLS_1275_TO_BYTES(addr_cells) +
634 	    CELLS_1275_TO_BYTES(intr_cells));
635 
636 exit3:
637 	kmem_free(imap_mask, imap_mask_sz);
638 
639 exit2:
640 	kmem_free(imap, imap_sz);
641 
642 	return (intr_parent_dip);
643 }
644 
645 /*
646  * process_intr_ops:
647  *
648  * Process the interrupt op via the interrupt parent.
649  */
650 int
process_intr_ops(dev_info_t * pdip,dev_info_t * rdip,ddi_intr_op_t op,ddi_intr_handle_impl_t * hdlp,void * result)651 process_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t op,
652     ddi_intr_handle_impl_t *hdlp, void *result)
653 {
654 	int		ret = DDI_FAILURE;
655 
656 	if (NEXUS_HAS_INTR_OP(pdip)) {
657 		ret = (*(DEVI(pdip)->devi_ops->devo_bus_ops->
658 		    bus_intr_op)) (pdip, rdip, op, hdlp, result);
659 	} else {
660 		cmn_err(CE_WARN, "Failed to process interrupt "
661 		    "for %s%d due to down-rev nexus driver %s%d",
662 		    ddi_get_name(rdip), ddi_get_instance(rdip),
663 		    ddi_get_name(pdip), ddi_get_instance(pdip));
664 	}
665 
666 	return (ret);
667 }
668 
669 /*ARGSUSED*/
670 uint_t
softlevel1(caddr_t arg)671 softlevel1(caddr_t arg)
672 {
673 	softint();
674 	return (1);
675 }
676 
677 /*
678  * indirection table, to save us some large switch statements
679  * NOTE: This must agree with "INTLEVEL_foo" constants in
680  *	<sys/avintr.h>
681  */
682 struct autovec *const vectorlist[] = { 0 };
683 
684 /*
685  * This value is exported here for the functions in avintr.c
686  */
687 const uint_t maxautovec = (sizeof (vectorlist) / sizeof (vectorlist[0]));
688 
689 /*
690  * Check for machine specific interrupt levels which cannot be reassigned by
691  * settrap(), sun4u version.
692  *
693  * sun4u does not support V8 SPARC "fast trap" handlers.
694  */
695 /*ARGSUSED*/
696 int
exclude_settrap(int lvl)697 exclude_settrap(int lvl)
698 {
699 	return (1);
700 }
701 
702 /*
703  * Check for machine specific interrupt levels which cannot have interrupt
704  * handlers added. We allow levels 1 through 15; level 0 is nonsense.
705  */
706 /*ARGSUSED*/
707 int
exclude_level(int lvl)708 exclude_level(int lvl)
709 {
710 	return ((lvl < 1) || (lvl > 15));
711 }
712 
713 /*
714  * Wrapper functions used by New DDI interrupt framework.
715  */
716 
717 /*
718  * i_ddi_intr_ops:
719  */
720 int
i_ddi_intr_ops(dev_info_t * dip,dev_info_t * rdip,ddi_intr_op_t op,ddi_intr_handle_impl_t * hdlp,void * result)721 i_ddi_intr_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t op,
722     ddi_intr_handle_impl_t *hdlp, void *result)
723 {
724 	dev_info_t	*pdip = ddi_get_parent(dip);
725 	int		ret = DDI_FAILURE;
726 
727 	/*
728 	 * The following check is required to address
729 	 * one of the test case of ADDI test suite.
730 	 */
731 	if (pdip == NULL)
732 		return (DDI_FAILURE);
733 
734 	if (hdlp->ih_type != DDI_INTR_TYPE_FIXED)
735 		return (process_intr_ops(pdip, rdip, op, hdlp, result));
736 
737 	if (hdlp->ih_vector == 0)
738 		hdlp->ih_vector = i_ddi_get_inum(rdip, hdlp->ih_inum);
739 
740 	if (hdlp->ih_pri == 0)
741 		hdlp->ih_pri = i_ddi_get_intr_pri(rdip, hdlp->ih_inum);
742 
743 	switch (op) {
744 	case DDI_INTROP_ADDISR:
745 	case DDI_INTROP_REMISR:
746 	case DDI_INTROP_GETTARGET:
747 	case DDI_INTROP_SETTARGET:
748 	case DDI_INTROP_ENABLE:
749 	case DDI_INTROP_DISABLE:
750 	case DDI_INTROP_BLOCKENABLE:
751 	case DDI_INTROP_BLOCKDISABLE:
752 		/*
753 		 * Try and determine our parent and possibly an interrupt
754 		 * translation. intr parent dip returned held
755 		 */
756 		if ((pdip = get_intr_parent(pdip, dip, hdlp)) == NULL)
757 			goto done;
758 	}
759 
760 	ret = process_intr_ops(pdip, rdip, op, hdlp, result);
761 
762 done:
763 	switch (op) {
764 	case DDI_INTROP_ADDISR:
765 	case DDI_INTROP_REMISR:
766 	case DDI_INTROP_ENABLE:
767 	case DDI_INTROP_DISABLE:
768 	case DDI_INTROP_BLOCKENABLE:
769 	case DDI_INTROP_BLOCKDISABLE:
770 		/* Release hold acquired in get_intr_parent() */
771 		if (pdip)
772 			ndi_rele_devi(pdip);
773 	}
774 
775 	hdlp->ih_vector = 0;
776 
777 	return (ret);
778 }
779 
780 /*
781  * i_ddi_add_ivintr:
782  */
783 /*ARGSUSED*/
784 int
i_ddi_add_ivintr(ddi_intr_handle_impl_t * hdlp)785 i_ddi_add_ivintr(ddi_intr_handle_impl_t *hdlp)
786 {
787 	/*
788 	 * If the PIL was set and is valid use it, otherwise
789 	 * default it to 1
790 	 */
791 	if ((hdlp->ih_pri < 1) || (hdlp->ih_pri > PIL_MAX))
792 		hdlp->ih_pri = 1;
793 
794 	VERIFY(add_ivintr(hdlp->ih_vector, hdlp->ih_pri,
795 	    (intrfunc)hdlp->ih_cb_func, hdlp->ih_cb_arg1,
796 	    hdlp->ih_cb_arg2, NULL) == 0);
797 
798 	return (DDI_SUCCESS);
799 }
800 
801 /*
802  * i_ddi_rem_ivintr:
803  */
804 /*ARGSUSED*/
805 void
i_ddi_rem_ivintr(ddi_intr_handle_impl_t * hdlp)806 i_ddi_rem_ivintr(ddi_intr_handle_impl_t *hdlp)
807 {
808 	VERIFY(rem_ivintr(hdlp->ih_vector, hdlp->ih_pri) == 0);
809 }
810 
811 /*
812  * i_ddi_get_inum - Get the interrupt number property from the
813  * specified device. Note that this function is called only for
814  * the FIXED interrupt type.
815  */
816 uint32_t
i_ddi_get_inum(dev_info_t * dip,uint_t inumber)817 i_ddi_get_inum(dev_info_t *dip, uint_t inumber)
818 {
819 	int32_t			intrlen, intr_cells, max_intrs;
820 	prop_1275_cell_t	*ip, intr_sz;
821 	uint32_t		intr = 0;
822 
823 	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS |
824 	    DDI_PROP_CANSLEEP,
825 	    "interrupts", (caddr_t)&ip, &intrlen) == DDI_SUCCESS) {
826 
827 		intr_cells = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
828 		    "#interrupt-cells", 1);
829 
830 		/* adjust for number of bytes */
831 		intr_sz = CELLS_1275_TO_BYTES(intr_cells);
832 
833 		/* Calculate the number of interrupts */
834 		max_intrs = intrlen / intr_sz;
835 
836 		if (inumber < max_intrs) {
837 			prop_1275_cell_t *intrp = ip;
838 
839 			/* Index into interrupt property */
840 			intrp += (inumber * intr_cells);
841 
842 			cells_1275_copy(intrp, &intr, intr_cells);
843 		}
844 
845 		kmem_free(ip, intrlen);
846 	}
847 
848 	return (intr);
849 }
850 
851 /*
852  * i_ddi_get_intr_pri - Get the interrupt-priorities property from
853  * the specified device. Note that this function is called only for
854  * the FIXED interrupt type.
855  */
856 uint32_t
i_ddi_get_intr_pri(dev_info_t * dip,uint_t inumber)857 i_ddi_get_intr_pri(dev_info_t *dip, uint_t inumber)
858 {
859 	uint32_t	*intr_prio_p;
860 	uint32_t	pri = 0;
861 	int32_t		i;
862 
863 	/*
864 	 * Use the "interrupt-priorities" property to determine the
865 	 * the pil/ipl for the interrupt handler.
866 	 */
867 	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS,
868 	    "interrupt-priorities", (caddr_t)&intr_prio_p,
869 	    &i) == DDI_SUCCESS) {
870 		if (inumber < (i / sizeof (int32_t)))
871 			pri = intr_prio_p[inumber];
872 		kmem_free(intr_prio_p, i);
873 	}
874 
875 	return (pri);
876 }
877 
878 int
i_ddi_get_intx_nintrs(dev_info_t * dip)879 i_ddi_get_intx_nintrs(dev_info_t *dip)
880 {
881 	int32_t intrlen;
882 	prop_1275_cell_t intr_sz;
883 	prop_1275_cell_t *ip;
884 	int32_t ret = 0;
885 
886 	if (ddi_getlongprop(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS |
887 	    DDI_PROP_CANSLEEP,
888 	    "interrupts", (caddr_t)&ip, &intrlen) == DDI_SUCCESS) {
889 
890 		intr_sz = ddi_getprop(DDI_DEV_T_ANY, dip, 0,
891 		    "#interrupt-cells", 1);
892 		/* adjust for number of bytes */
893 		intr_sz = CELLS_1275_TO_BYTES(intr_sz);
894 
895 		ret = intrlen / intr_sz;
896 
897 		kmem_free(ip, intrlen);
898 	}
899 
900 	return (ret);
901 }
902 
903 /*
904  * i_ddi_add_softint - allocate and add a software interrupt.
905  *
906  * NOTE: All software interrupts that are registered through DDI
907  *	 should be triggered only on a single target or CPU.
908  */
909 int
i_ddi_add_softint(ddi_softint_hdl_impl_t * hdlp)910 i_ddi_add_softint(ddi_softint_hdl_impl_t *hdlp)
911 {
912 	if ((hdlp->ih_private = (void *)add_softintr(hdlp->ih_pri,
913 	    hdlp->ih_cb_func, hdlp->ih_cb_arg1, SOFTINT_ST)) == NULL)
914 		return (DDI_FAILURE);
915 
916 	return (DDI_SUCCESS);
917 }
918 
919 /*
920  * i_ddi_remove_softint - remove and free a software interrupt.
921  */
922 void
i_ddi_remove_softint(ddi_softint_hdl_impl_t * hdlp)923 i_ddi_remove_softint(ddi_softint_hdl_impl_t *hdlp)
924 {
925 	ASSERT(hdlp->ih_private != NULL);
926 
927 	if (rem_softintr((uint64_t)hdlp->ih_private) == 0)
928 		hdlp->ih_private = NULL;
929 }
930 
931 /*
932  * i_ddi_trigger_softint - trigger a software interrupt.
933  */
934 int
i_ddi_trigger_softint(ddi_softint_hdl_impl_t * hdlp,void * arg2)935 i_ddi_trigger_softint(ddi_softint_hdl_impl_t *hdlp, void *arg2)
936 {
937 	int	ret;
938 
939 	ASSERT(hdlp->ih_private != NULL);
940 
941 	/* Update the second argument for the software interrupt */
942 	if ((ret = update_softint_arg2((uint64_t)hdlp->ih_private, arg2)) == 0)
943 		setsoftint((uint64_t)hdlp->ih_private);
944 
945 	return (ret ? DDI_EPENDING : DDI_SUCCESS);
946 }
947 
948 /*
949  * i_ddi_set_softint_pri - change software interrupt priority.
950  */
951 /* ARGSUSED */
952 int
i_ddi_set_softint_pri(ddi_softint_hdl_impl_t * hdlp,uint_t old_pri)953 i_ddi_set_softint_pri(ddi_softint_hdl_impl_t *hdlp, uint_t old_pri)
954 {
955 	int	ret;
956 
957 	ASSERT(hdlp->ih_private != NULL);
958 
959 	/* Update the interrupt priority for the software interrupt */
960 	ret = update_softint_pri((uint64_t)hdlp->ih_private, hdlp->ih_pri);
961 
962 	return (ret ? DDI_FAILURE : DDI_SUCCESS);
963 }
964 
965 /*ARGSUSED*/
966 void
i_ddi_alloc_intr_phdl(ddi_intr_handle_impl_t * hdlp)967 i_ddi_alloc_intr_phdl(ddi_intr_handle_impl_t *hdlp)
968 {
969 }
970 
971 /*ARGSUSED*/
972 void
i_ddi_free_intr_phdl(ddi_intr_handle_impl_t * hdlp)973 i_ddi_free_intr_phdl(ddi_intr_handle_impl_t *hdlp)
974 {
975 }
976 
977 /*
978  * SECTION: DDI Memory/DMA
979  */
980 
981 /* set HAT endianess attributes from ddi_device_acc_attr */
982 void
i_ddi_devacc_to_hatacc(ddi_device_acc_attr_t * devaccp,uint_t * hataccp)983 i_ddi_devacc_to_hatacc(ddi_device_acc_attr_t *devaccp, uint_t *hataccp)
984 {
985 	if (devaccp != NULL) {
986 		if (devaccp->devacc_attr_endian_flags == DDI_STRUCTURE_LE_ACC) {
987 			*hataccp &= ~HAT_ENDIAN_MASK;
988 			*hataccp |= HAT_STRUCTURE_LE;
989 		}
990 	}
991 }
992 
993 /*
994  * Check if the specified cache attribute is supported on the platform.
995  * This function must be called before i_ddi_cacheattr_to_hatacc().
996  */
997 boolean_t
i_ddi_check_cache_attr(uint_t flags)998 i_ddi_check_cache_attr(uint_t flags)
999 {
1000 	/*
1001 	 * The cache attributes are mutually exclusive. Any combination of
1002 	 * the attributes leads to a failure.
1003 	 */
1004 	uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1005 	if ((cache_attr != 0) && !ISP2(cache_attr))
1006 		return (B_FALSE);
1007 
1008 	/*
1009 	 * On the sparc architecture, only IOMEM_DATA_CACHED is meaningful,
1010 	 * but others lead to a failure.
1011 	 */
1012 	if (cache_attr & IOMEM_DATA_CACHED)
1013 		return (B_TRUE);
1014 	else
1015 		return (B_FALSE);
1016 }
1017 
1018 /* set HAT cache attributes from the cache attributes */
1019 void
i_ddi_cacheattr_to_hatacc(uint_t flags,uint_t * hataccp)1020 i_ddi_cacheattr_to_hatacc(uint_t flags, uint_t *hataccp)
1021 {
1022 	uint_t cache_attr = IOMEM_CACHE_ATTR(flags);
1023 	static char *fname = "i_ddi_cacheattr_to_hatacc";
1024 #if defined(lint)
1025 	*hataccp = *hataccp;
1026 #endif
1027 	/*
1028 	 * set HAT attrs according to the cache attrs.
1029 	 */
1030 	switch (cache_attr) {
1031 	/*
1032 	 * The cache coherency is always maintained on SPARC, and
1033 	 * nothing is required.
1034 	 */
1035 	case IOMEM_DATA_CACHED:
1036 		break;
1037 	/*
1038 	 * Both IOMEM_DATA_UC_WRITE_COMBINED and IOMEM_DATA_UNCACHED are
1039 	 * not supported on SPARC -- this case must not occur because the
1040 	 * cache attribute is scrutinized before this function is called.
1041 	 */
1042 	case IOMEM_DATA_UNCACHED:
1043 	case IOMEM_DATA_UC_WR_COMBINE:
1044 	default:
1045 		cmn_err(CE_WARN, "%s: cache_attr=0x%x is ignored.",
1046 		    fname, cache_attr);
1047 	}
1048 }
1049 
1050 static vmem_t *little_endian_arena;
1051 static vmem_t *big_endian_arena;
1052 
1053 static void *
segkmem_alloc_le(vmem_t * vmp,size_t size,int flag)1054 segkmem_alloc_le(vmem_t *vmp, size_t size, int flag)
1055 {
1056 	return (segkmem_xalloc(vmp, NULL, size, flag, HAT_STRUCTURE_LE,
1057 	    segkmem_page_create, NULL));
1058 }
1059 
1060 static void *
segkmem_alloc_be(vmem_t * vmp,size_t size,int flag)1061 segkmem_alloc_be(vmem_t *vmp, size_t size, int flag)
1062 {
1063 	return (segkmem_xalloc(vmp, NULL, size, flag, HAT_STRUCTURE_BE,
1064 	    segkmem_page_create, NULL));
1065 }
1066 
1067 void
ka_init(void)1068 ka_init(void)
1069 {
1070 	little_endian_arena = vmem_create("little_endian", NULL, 0, 1,
1071 	    segkmem_alloc_le, segkmem_free, heap_arena, 0, VM_SLEEP);
1072 	big_endian_arena = vmem_create("big_endian", NULL, 0, 1,
1073 	    segkmem_alloc_be, segkmem_free, heap_arena, 0, VM_SLEEP);
1074 }
1075 
1076 /*
1077  * Allocate from the system, aligned on a specific boundary.
1078  * The alignment, if non-zero, must be a power of 2.
1079  */
1080 static void *
kalloca(size_t size,size_t align,int cansleep,uint_t endian_flags)1081 kalloca(size_t size, size_t align, int cansleep, uint_t endian_flags)
1082 {
1083 	size_t *addr, *raddr, rsize;
1084 	size_t hdrsize = 4 * sizeof (size_t);	/* must be power of 2 */
1085 
1086 	align = MAX(align, hdrsize);
1087 	ASSERT((align & (align - 1)) == 0);
1088 
1089 	/*
1090 	 * We need to allocate
1091 	 *    rsize = size + hdrsize + align - MIN(hdrsize, buffer_alignment)
1092 	 * bytes to be sure we have enough freedom to satisfy the request.
1093 	 * Since the buffer alignment depends on the request size, this is
1094 	 * not straightforward to use directly.
1095 	 *
1096 	 * kmem guarantees that any allocation of a 64-byte multiple will be
1097 	 * 64-byte aligned.  Since rounding up the request could add more
1098 	 * than we save, we compute the size with and without alignment, and
1099 	 * use the smaller of the two.
1100 	 */
1101 	rsize = size + hdrsize + align;
1102 
1103 	if (endian_flags == DDI_STRUCTURE_LE_ACC) {
1104 		raddr = vmem_alloc(little_endian_arena, rsize,
1105 		    cansleep ? VM_SLEEP : VM_NOSLEEP);
1106 	} else {
1107 		raddr = vmem_alloc(big_endian_arena, rsize,
1108 		    cansleep ? VM_SLEEP : VM_NOSLEEP);
1109 	}
1110 
1111 	if (raddr == NULL)
1112 		return (NULL);
1113 
1114 	addr = (size_t *)P2ROUNDUP((uintptr_t)raddr + hdrsize, align);
1115 	ASSERT((uintptr_t)addr + size - (uintptr_t)raddr <= rsize);
1116 
1117 	addr[-3] = (size_t)endian_flags;
1118 	addr[-2] = (size_t)raddr;
1119 	addr[-1] = rsize;
1120 
1121 	return (addr);
1122 }
1123 
1124 static void
kfreea(void * addr)1125 kfreea(void *addr)
1126 {
1127 	size_t *saddr = addr;
1128 
1129 	if (saddr[-3] == DDI_STRUCTURE_LE_ACC)
1130 		vmem_free(little_endian_arena, (void *)saddr[-2], saddr[-1]);
1131 	else
1132 		vmem_free(big_endian_arena, (void *)saddr[-2], saddr[-1]);
1133 }
1134 
1135 /*
1136  * This used to be ddi_iomin, but we were the only remaining caller, so
1137  * we've made it private and moved it here.
1138  */
1139 static int
i_ddi_iomin(dev_info_t * a,int i,int stream)1140 i_ddi_iomin(dev_info_t *a, int i, int stream)
1141 {
1142 	int r;
1143 
1144 	/*
1145 	 * Make sure that the initial value is sane
1146 	 */
1147 	if (!ISP2(i))
1148 		return (0);
1149 	if (i == 0)
1150 		i = (stream) ? 4 : 1;
1151 
1152 	r = ddi_ctlops(a, a,
1153 	    DDI_CTLOPS_IOMIN, (void *)(uintptr_t)stream, (void *)&i);
1154 	if (r != DDI_SUCCESS || !ISP2(i))
1155 		return (0);
1156 	return (i);
1157 }
1158 
1159 int
i_ddi_mem_alloc(dev_info_t * dip,ddi_dma_attr_t * attr,size_t length,int cansleep,int flags,ddi_device_acc_attr_t * accattrp,caddr_t * kaddrp,size_t * real_length,ddi_acc_hdl_t * handlep)1160 i_ddi_mem_alloc(dev_info_t *dip, ddi_dma_attr_t *attr,
1161     size_t length, int cansleep, int flags,
1162     ddi_device_acc_attr_t *accattrp,
1163     caddr_t *kaddrp, size_t *real_length, ddi_acc_hdl_t *handlep)
1164 {
1165 	caddr_t a;
1166 	int iomin, align, streaming;
1167 	uint_t endian_flags = DDI_NEVERSWAP_ACC;
1168 
1169 #if defined(lint)
1170 	*handlep = *handlep;
1171 #endif
1172 
1173 	/*
1174 	 * Check legality of arguments
1175 	 */
1176 	if (length == 0 || kaddrp == NULL || attr == NULL) {
1177 		return (DDI_FAILURE);
1178 	}
1179 
1180 	if (attr->dma_attr_minxfer == 0 || attr->dma_attr_align == 0 ||
1181 	    !ISP2(attr->dma_attr_align) || !ISP2(attr->dma_attr_minxfer)) {
1182 		return (DDI_FAILURE);
1183 	}
1184 
1185 	/*
1186 	 * check if a streaming sequential xfer is requested.
1187 	 */
1188 	streaming = (flags & DDI_DMA_STREAMING) ? 1 : 0;
1189 
1190 	/*
1191 	 * Drivers for 64-bit capable SBus devices will encode
1192 	 * the burtsizes for 64-bit xfers in the upper 16-bits.
1193 	 * For DMA alignment, we use the most restrictive
1194 	 * alignment of 32-bit and 64-bit xfers.
1195 	 */
1196 	iomin = (attr->dma_attr_burstsizes & 0xffff) |
1197 	    ((attr->dma_attr_burstsizes >> 16) & 0xffff);
1198 	/*
1199 	 * If a driver set burtsizes to 0, we give it byte alignment.
1200 	 * Otherwise align at the burtsizes boundary.
1201 	 */
1202 	if (iomin == 0)
1203 		iomin = 1;
1204 	else
1205 		iomin = 1 << (ddi_fls(iomin) - 1);
1206 	iomin = maxbit(iomin, attr->dma_attr_minxfer);
1207 	iomin = maxbit(iomin, attr->dma_attr_align);
1208 	iomin = i_ddi_iomin(dip, iomin, streaming);
1209 	if (iomin == 0)
1210 		return (DDI_FAILURE);
1211 
1212 	ASSERT((iomin & (iomin - 1)) == 0);
1213 	ASSERT(iomin >= attr->dma_attr_minxfer);
1214 	ASSERT(iomin >= attr->dma_attr_align);
1215 
1216 	length = P2ROUNDUP(length, iomin);
1217 	align = iomin;
1218 
1219 	if (accattrp != NULL)
1220 		endian_flags = accattrp->devacc_attr_endian_flags;
1221 
1222 	a = kalloca(length, align, cansleep, endian_flags);
1223 	if ((*kaddrp = a) == 0) {
1224 		return (DDI_FAILURE);
1225 	} else {
1226 		if (real_length) {
1227 			*real_length = length;
1228 		}
1229 		if (handlep) {
1230 			/*
1231 			 * assign handle information
1232 			 */
1233 			impl_acc_hdl_init(handlep);
1234 		}
1235 		return (DDI_SUCCESS);
1236 	}
1237 }
1238 
1239 /* ARGSUSED */
1240 void
i_ddi_mem_free(caddr_t kaddr,ddi_acc_hdl_t * ap)1241 i_ddi_mem_free(caddr_t kaddr, ddi_acc_hdl_t *ap)
1242 {
1243 	kfreea(kaddr);
1244 }
1245 
1246 /*
1247  * SECTION: DDI Data Access
1248  */
1249 
1250 static uintptr_t impl_acc_hdl_id = 0;
1251 
1252 /*
1253  * access handle allocator
1254  */
1255 ddi_acc_hdl_t *
impl_acc_hdl_get(ddi_acc_handle_t hdl)1256 impl_acc_hdl_get(ddi_acc_handle_t hdl)
1257 {
1258 	/*
1259 	 * Extract the access handle address from the DDI implemented
1260 	 * access handle
1261 	 */
1262 	return (&((ddi_acc_impl_t *)hdl)->ahi_common);
1263 }
1264 
1265 ddi_acc_handle_t
impl_acc_hdl_alloc(int (* waitfp)(caddr_t),caddr_t arg)1266 impl_acc_hdl_alloc(int (*waitfp)(caddr_t), caddr_t arg)
1267 {
1268 	ddi_acc_impl_t *hp;
1269 	on_trap_data_t *otp;
1270 	int sleepflag;
1271 
1272 	sleepflag = ((waitfp == (int (*)())KM_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
1273 
1274 	/*
1275 	 * Allocate and initialize the data access handle and error status.
1276 	 */
1277 	if ((hp = kmem_zalloc(sizeof (ddi_acc_impl_t), sleepflag)) == NULL)
1278 		goto fail;
1279 	if ((hp->ahi_err = (ndi_err_t *)kmem_zalloc(
1280 	    sizeof (ndi_err_t), sleepflag)) == NULL) {
1281 		kmem_free(hp, sizeof (ddi_acc_impl_t));
1282 		goto fail;
1283 	}
1284 	if ((otp = (on_trap_data_t *)kmem_zalloc(
1285 	    sizeof (on_trap_data_t), sleepflag)) == NULL) {
1286 		kmem_free(hp->ahi_err, sizeof (ndi_err_t));
1287 		kmem_free(hp, sizeof (ddi_acc_impl_t));
1288 		goto fail;
1289 	}
1290 	hp->ahi_err->err_ontrap = otp;
1291 	hp->ahi_common.ah_platform_private = (void *)hp;
1292 
1293 	return ((ddi_acc_handle_t)hp);
1294 fail:
1295 	if ((waitfp != (int (*)())KM_SLEEP) &&
1296 	    (waitfp != (int (*)())KM_NOSLEEP))
1297 		ddi_set_callback(waitfp, arg, &impl_acc_hdl_id);
1298 	return (NULL);
1299 }
1300 
1301 void
impl_acc_hdl_free(ddi_acc_handle_t handle)1302 impl_acc_hdl_free(ddi_acc_handle_t handle)
1303 {
1304 	ddi_acc_impl_t *hp;
1305 
1306 	/*
1307 	 * The supplied (ddi_acc_handle_t) is actually a (ddi_acc_impl_t *),
1308 	 * because that's what we allocated in impl_acc_hdl_alloc() above.
1309 	 */
1310 	hp = (ddi_acc_impl_t *)handle;
1311 	if (hp) {
1312 		kmem_free(hp->ahi_err->err_ontrap, sizeof (on_trap_data_t));
1313 		kmem_free(hp->ahi_err, sizeof (ndi_err_t));
1314 		kmem_free(hp, sizeof (ddi_acc_impl_t));
1315 		if (impl_acc_hdl_id)
1316 			ddi_run_callback(&impl_acc_hdl_id);
1317 	}
1318 }
1319 
1320 #define	PCI_GET_MP_PFN(mp, page_no)	((mp)->dmai_ndvmapages == 1 ? \
1321 	(pfn_t)(mp)->dmai_iopte:(((pfn_t *)(mp)->dmai_iopte)[page_no]))
1322 
1323 /*
1324  * Function called after a dma fault occurred to find out whether the
1325  * fault address is associated with a driver that is able to handle faults
1326  * and recover from faults.
1327  */
1328 /* ARGSUSED */
1329 int
impl_dma_check(dev_info_t * dip,const void * handle,const void * addr,const void * not_used)1330 impl_dma_check(dev_info_t *dip, const void *handle, const void *addr,
1331     const void *not_used)
1332 {
1333 	ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
1334 	pfn_t fault_pfn = mmu_btop(*(uint64_t *)addr);
1335 	pfn_t comp_pfn;
1336 
1337 	/*
1338 	 * The driver has to set DDI_DMA_FLAGERR to recover from dma faults.
1339 	 */
1340 	int page;
1341 
1342 	ASSERT(mp);
1343 	for (page = 0; page < mp->dmai_ndvmapages; page++) {
1344 		comp_pfn = PCI_GET_MP_PFN(mp, page);
1345 		if (fault_pfn == comp_pfn)
1346 			return (DDI_FM_NONFATAL);
1347 	}
1348 	return (DDI_FM_UNKNOWN);
1349 }
1350 
1351 /*
1352  * Function used to check if a given access handle owns the failing address.
1353  * Called by ndi_fmc_error, when we detect a PIO error.
1354  */
1355 /* ARGSUSED */
1356 static int
impl_acc_check(dev_info_t * dip,const void * handle,const void * addr,const void * not_used)1357 impl_acc_check(dev_info_t *dip, const void *handle, const void *addr,
1358     const void *not_used)
1359 {
1360 	pfn_t pfn, fault_pfn;
1361 	ddi_acc_hdl_t *hp;
1362 
1363 	hp = impl_acc_hdl_get((ddi_acc_handle_t)handle);
1364 
1365 	ASSERT(hp);
1366 
1367 	if (addr != NULL) {
1368 		pfn = hp->ah_pfn;
1369 		fault_pfn = mmu_btop(*(uint64_t *)addr);
1370 		if (fault_pfn >= pfn && fault_pfn < (pfn + hp->ah_pnum))
1371 			return (DDI_FM_NONFATAL);
1372 	}
1373 	return (DDI_FM_UNKNOWN);
1374 }
1375 
1376 void
impl_acc_err_init(ddi_acc_hdl_t * handlep)1377 impl_acc_err_init(ddi_acc_hdl_t *handlep)
1378 {
1379 	int fmcap;
1380 	ndi_err_t *errp;
1381 	on_trap_data_t *otp;
1382 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handlep;
1383 
1384 	fmcap = ddi_fm_capable(handlep->ah_dip);
1385 
1386 	if (handlep->ah_acc.devacc_attr_version < DDI_DEVICE_ATTR_V1 ||
1387 	    !DDI_FM_ACC_ERR_CAP(fmcap)) {
1388 		handlep->ah_acc.devacc_attr_access = DDI_DEFAULT_ACC;
1389 	} else if (DDI_FM_ACC_ERR_CAP(fmcap)) {
1390 		if (handlep->ah_acc.devacc_attr_access == DDI_DEFAULT_ACC) {
1391 			if (handlep->ah_xfermodes)
1392 				return;
1393 			i_ddi_drv_ereport_post(handlep->ah_dip, DVR_EFMCAP,
1394 			    NULL, DDI_NOSLEEP);
1395 		} else {
1396 			errp = hp->ahi_err;
1397 			otp = (on_trap_data_t *)errp->err_ontrap;
1398 			otp->ot_handle = (void *)(hp);
1399 			otp->ot_prot = OT_DATA_ACCESS;
1400 			if (handlep->ah_acc.devacc_attr_access ==
1401 			    DDI_CAUTIOUS_ACC)
1402 				otp->ot_trampoline =
1403 				    (uintptr_t)&i_ddi_caut_trampoline;
1404 			else
1405 				otp->ot_trampoline =
1406 				    (uintptr_t)&i_ddi_prot_trampoline;
1407 			errp->err_status = DDI_FM_OK;
1408 			errp->err_expected = DDI_FM_ERR_UNEXPECTED;
1409 			errp->err_cf = impl_acc_check;
1410 		}
1411 	}
1412 }
1413 
1414 void
impl_acc_hdl_init(ddi_acc_hdl_t * handlep)1415 impl_acc_hdl_init(ddi_acc_hdl_t *handlep)
1416 {
1417 	ddi_acc_impl_t *hp;
1418 
1419 	ASSERT(handlep);
1420 
1421 	hp = (ddi_acc_impl_t *)handlep;
1422 
1423 	/*
1424 	 * check for SW byte-swapping
1425 	 */
1426 	hp->ahi_get8 = i_ddi_get8;
1427 	hp->ahi_put8 = i_ddi_put8;
1428 	hp->ahi_rep_get8 = i_ddi_rep_get8;
1429 	hp->ahi_rep_put8 = i_ddi_rep_put8;
1430 	if (handlep->ah_acc.devacc_attr_endian_flags & DDI_STRUCTURE_LE_ACC) {
1431 		hp->ahi_get16 = i_ddi_swap_get16;
1432 		hp->ahi_get32 = i_ddi_swap_get32;
1433 		hp->ahi_get64 = i_ddi_swap_get64;
1434 		hp->ahi_put16 = i_ddi_swap_put16;
1435 		hp->ahi_put32 = i_ddi_swap_put32;
1436 		hp->ahi_put64 = i_ddi_swap_put64;
1437 		hp->ahi_rep_get16 = i_ddi_swap_rep_get16;
1438 		hp->ahi_rep_get32 = i_ddi_swap_rep_get32;
1439 		hp->ahi_rep_get64 = i_ddi_swap_rep_get64;
1440 		hp->ahi_rep_put16 = i_ddi_swap_rep_put16;
1441 		hp->ahi_rep_put32 = i_ddi_swap_rep_put32;
1442 		hp->ahi_rep_put64 = i_ddi_swap_rep_put64;
1443 	} else {
1444 		hp->ahi_get16 = i_ddi_get16;
1445 		hp->ahi_get32 = i_ddi_get32;
1446 		hp->ahi_get64 = i_ddi_get64;
1447 		hp->ahi_put16 = i_ddi_put16;
1448 		hp->ahi_put32 = i_ddi_put32;
1449 		hp->ahi_put64 = i_ddi_put64;
1450 		hp->ahi_rep_get16 = i_ddi_rep_get16;
1451 		hp->ahi_rep_get32 = i_ddi_rep_get32;
1452 		hp->ahi_rep_get64 = i_ddi_rep_get64;
1453 		hp->ahi_rep_put16 = i_ddi_rep_put16;
1454 		hp->ahi_rep_put32 = i_ddi_rep_put32;
1455 		hp->ahi_rep_put64 = i_ddi_rep_put64;
1456 	}
1457 
1458 	/* Legacy fault flags and support */
1459 	hp->ahi_fault_check = i_ddi_acc_fault_check;
1460 	hp->ahi_fault_notify = i_ddi_acc_fault_notify;
1461 	hp->ahi_fault = 0;
1462 	impl_acc_err_init(handlep);
1463 }
1464 
1465 void
i_ddi_acc_set_fault(ddi_acc_handle_t handle)1466 i_ddi_acc_set_fault(ddi_acc_handle_t handle)
1467 {
1468 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle;
1469 
1470 	if (!hp->ahi_fault) {
1471 		hp->ahi_fault = 1;
1472 			(*hp->ahi_fault_notify)(hp);
1473 	}
1474 }
1475 
1476 void
i_ddi_acc_clr_fault(ddi_acc_handle_t handle)1477 i_ddi_acc_clr_fault(ddi_acc_handle_t handle)
1478 {
1479 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handle;
1480 
1481 	if (hp->ahi_fault) {
1482 		hp->ahi_fault = 0;
1483 			(*hp->ahi_fault_notify)(hp);
1484 	}
1485 }
1486 
1487 /* ARGSUSED */
1488 void
i_ddi_acc_fault_notify(ddi_acc_impl_t * hp)1489 i_ddi_acc_fault_notify(ddi_acc_impl_t *hp)
1490 {
1491 	/* Default version, does nothing */
1492 }
1493 
1494 /*
1495  * SECTION: Misc functions
1496  */
1497 
1498 /*
1499  * instance wrappers
1500  */
1501 /*ARGSUSED*/
1502 uint_t
impl_assign_instance(dev_info_t * dip)1503 impl_assign_instance(dev_info_t *dip)
1504 {
1505 	return ((uint_t)-1);
1506 }
1507 
1508 /*ARGSUSED*/
1509 int
impl_keep_instance(dev_info_t * dip)1510 impl_keep_instance(dev_info_t *dip)
1511 {
1512 	return (DDI_FAILURE);
1513 }
1514 
1515 /*ARGSUSED*/
1516 int
impl_free_instance(dev_info_t * dip)1517 impl_free_instance(dev_info_t *dip)
1518 {
1519 	return (DDI_FAILURE);
1520 }
1521 
1522 /*ARGSUSED*/
1523 int
impl_check_cpu(dev_info_t * devi)1524 impl_check_cpu(dev_info_t *devi)
1525 {
1526 	return (DDI_SUCCESS);
1527 }
1528 
1529 
1530 static const char *nocopydevs[] = {
1531 	"SUNW,ffb",
1532 	"SUNW,afb",
1533 	NULL
1534 };
1535 
1536 /*
1537  * Perform a copy from a memory mapped device (whose devinfo pointer is devi)
1538  * separately mapped at devaddr in the kernel to a kernel buffer at kaddr.
1539  */
1540 /*ARGSUSED*/
1541 int
e_ddi_copyfromdev(dev_info_t * devi,off_t off,const void * devaddr,void * kaddr,size_t len)1542 e_ddi_copyfromdev(dev_info_t *devi,
1543     off_t off, const void *devaddr, void *kaddr, size_t len)
1544 {
1545 	const char **argv;
1546 
1547 	for (argv = nocopydevs; *argv; argv++)
1548 		if (strcmp(ddi_binding_name(devi), *argv) == 0) {
1549 			bzero(kaddr, len);
1550 			return (0);
1551 		}
1552 
1553 	bcopy(devaddr, kaddr, len);
1554 	return (0);
1555 }
1556 
1557 /*
1558  * Perform a copy to a memory mapped device (whose devinfo pointer is devi)
1559  * separately mapped at devaddr in the kernel from a kernel buffer at kaddr.
1560  */
1561 /*ARGSUSED*/
1562 int
e_ddi_copytodev(dev_info_t * devi,off_t off,const void * kaddr,void * devaddr,size_t len)1563 e_ddi_copytodev(dev_info_t *devi,
1564     off_t off, const void *kaddr, void *devaddr, size_t len)
1565 {
1566 	const char **argv;
1567 
1568 	for (argv = nocopydevs; *argv; argv++)
1569 		if (strcmp(ddi_binding_name(devi), *argv) == 0)
1570 			return (1);
1571 
1572 	bcopy(kaddr, devaddr, len);
1573 	return (0);
1574 }
1575 
1576 /*
1577  * Boot Configuration
1578  */
1579 idprom_t idprom;
1580 
1581 /*
1582  * Configure the hardware on the system.
1583  * Called before the rootfs is mounted
1584  */
1585 void
configure(void)1586 configure(void)
1587 {
1588 	extern void i_ddi_init_root();
1589 
1590 	/* We better have released boot by this time! */
1591 	ASSERT(!bootops);
1592 
1593 	/*
1594 	 * Determine whether or not to use the fpu, V9 SPARC cpus
1595 	 * always have one. Could check for existence of a fp queue,
1596 	 * Ultra I, II and IIa do not have a fp queue.
1597 	 */
1598 	if (fpu_exists)
1599 		fpu_probe();
1600 	else
1601 		cmn_err(CE_CONT, "FPU not in use\n");
1602 
1603 #if 0 /* XXXQ - not necessary for sun4u */
1604 	/*
1605 	 * This following line fixes bugid 1041296; we need to do a
1606 	 * prom_nextnode(0) because this call ALSO patches the DMA+
1607 	 * bug in Campus-B and Phoenix. The prom uncaches the traptable
1608 	 * page as a side-effect of devr_next(0) (which prom_nextnode calls),
1609 	 * so this *must* be executed early on. (XXX This is untrue for sun4u)
1610 	 */
1611 	(void) prom_nextnode((pnode_t)0);
1612 #endif
1613 
1614 	/*
1615 	 * Initialize devices on the machine.
1616 	 * Uses configuration tree built by the PROMs to determine what
1617 	 * is present, and builds a tree of prototype dev_info nodes
1618 	 * corresponding to the hardware which identified itself.
1619 	 */
1620 	i_ddi_init_root();
1621 
1622 #ifdef	DDI_PROP_DEBUG
1623 	(void) ddi_prop_debug(1);	/* Enable property debugging */
1624 #endif	/* DDI_PROP_DEBUG */
1625 }
1626 
1627 /*
1628  * The "status" property indicates the operational status of a device.
1629  * If this property is present, the value is a string indicating the
1630  * status of the device as follows:
1631  *
1632  *	"okay"		operational.
1633  *	"disabled"	not operational, but might become operational.
1634  *	"fail"		not operational because a fault has been detected,
1635  *			and it is unlikely that the device will become
1636  *			operational without repair. no additional details
1637  *			are available.
1638  *	"fail-xxx"	not operational because a fault has been detected,
1639  *			and it is unlikely that the device will become
1640  *			operational without repair. "xxx" is additional
1641  *			human-readable information about the particular
1642  *			fault condition that was detected.
1643  *
1644  * The absence of this property means that the operational status is
1645  * unknown or okay.
1646  *
1647  * This routine checks the status property of the specified device node
1648  * and returns 0 if the operational status indicates failure, and 1 otherwise.
1649  *
1650  * The property may exist on plug-in cards the existed before IEEE 1275-1994.
1651  * And, in that case, the property may not even be a string. So we carefully
1652  * check for the value "fail", in the beginning of the string, noting
1653  * the property length.
1654  */
1655 int
status_okay(int id,char * buf,int buflen)1656 status_okay(int id, char *buf, int buflen)
1657 {
1658 	char status_buf[OBP_MAXPROPNAME];
1659 	char *bufp = buf;
1660 	int len = buflen;
1661 	int proplen;
1662 	static const char *status = "status";
1663 	static const char *fail = "fail";
1664 	size_t fail_len = strlen(fail);
1665 
1666 	/*
1667 	 * Get the proplen ... if it's smaller than "fail",
1668 	 * or doesn't exist ... then we don't care, since
1669 	 * the value can't begin with the char string "fail".
1670 	 *
1671 	 * NB: proplen, if it's a string, includes the NULL in the
1672 	 * the size of the property, and fail_len does not.
1673 	 */
1674 	proplen = prom_getproplen((pnode_t)id, (caddr_t)status);
1675 	if (proplen <= fail_len)	/* nonexistent or uninteresting len */
1676 		return (1);
1677 
1678 	/*
1679 	 * if a buffer was provided, use it
1680 	 */
1681 	if ((buf == (char *)NULL) || (buflen <= 0)) {
1682 		bufp = status_buf;
1683 		len = sizeof (status_buf);
1684 	}
1685 	*bufp = (char)0;
1686 
1687 	/*
1688 	 * Get the property into the buffer, to the extent of the buffer,
1689 	 * and in case the buffer is smaller than the property size,
1690 	 * NULL terminate the buffer. (This handles the case where
1691 	 * a buffer was passed in and the caller wants to print the
1692 	 * value, but the buffer was too small).
1693 	 */
1694 	(void) prom_bounded_getprop((pnode_t)id, (caddr_t)status,
1695 	    (caddr_t)bufp, len);
1696 	*(bufp + len - 1) = (char)0;
1697 
1698 	/*
1699 	 * If the value begins with the char string "fail",
1700 	 * then it means the node is failed. We don't care
1701 	 * about any other values. We assume the node is ok
1702 	 * although it might be 'disabled'.
1703 	 */
1704 	if (strncmp(bufp, fail, fail_len) == 0)
1705 		return (0);
1706 
1707 	return (1);
1708 }
1709 
1710 
1711 /*
1712  * We set the cpu type from the idprom, if we can.
1713  * Note that we just read out the contents of it, for the most part.
1714  */
1715 void
setcputype(void)1716 setcputype(void)
1717 {
1718 	/*
1719 	 * We cache the idprom info early on so that we don't
1720 	 * rummage through the NVRAM unnecessarily later.
1721 	 */
1722 	(void) prom_getidprom((caddr_t)&idprom, sizeof (idprom));
1723 }
1724 
1725 /*
1726  *  Here is where we actually infer meanings to the members of idprom_t
1727  */
1728 void
parse_idprom(void)1729 parse_idprom(void)
1730 {
1731 	if (idprom.id_format == IDFORM_1) {
1732 		(void) localetheraddr((struct ether_addr *)idprom.id_ether,
1733 		    (struct ether_addr *)NULL);
1734 		(void) snprintf(hw_serial, HW_HOSTID_LEN, "%u",
1735 		    (idprom.id_machine << 24) + idprom.id_serial);
1736 	} else
1737 		prom_printf("Invalid format code in IDprom.\n");
1738 }
1739 
1740 /*
1741  * Allow for implementation specific correction of PROM property values.
1742  */
1743 /*ARGSUSED*/
1744 void
impl_fix_props(dev_info_t * dip,dev_info_t * ch_dip,char * name,int len,caddr_t buffer)1745 impl_fix_props(dev_info_t *dip, dev_info_t *ch_dip, char *name, int len,
1746     caddr_t buffer)
1747 {
1748 	/*
1749 	 * There are no adjustments needed in this implementation.
1750 	 */
1751 }
1752 
1753 /*
1754  * The following functions ready a cautious request to go up to the nexus
1755  * driver.  It is up to the nexus driver to decide how to process the request.
1756  * It may choose to call i_ddi_do_caut_get/put in this file, or do it
1757  * differently.
1758  */
1759 
1760 static void
i_ddi_caut_getput_ctlops(ddi_acc_impl_t * hp,uint64_t host_addr,uint64_t dev_addr,size_t size,size_t repcount,uint_t flags,ddi_ctl_enum_t cmd)1761 i_ddi_caut_getput_ctlops(
1762     ddi_acc_impl_t *hp, uint64_t host_addr, uint64_t dev_addr, size_t size,
1763     size_t repcount, uint_t flags, ddi_ctl_enum_t cmd)
1764 {
1765 	peekpoke_ctlops_t	cautacc_ctlops_arg;
1766 
1767 	cautacc_ctlops_arg.size = size;
1768 	cautacc_ctlops_arg.dev_addr = dev_addr;
1769 	cautacc_ctlops_arg.host_addr = host_addr;
1770 	cautacc_ctlops_arg.handle = (ddi_acc_handle_t)hp;
1771 	cautacc_ctlops_arg.repcount = repcount;
1772 	cautacc_ctlops_arg.flags = flags;
1773 
1774 	(void) ddi_ctlops(hp->ahi_common.ah_dip, hp->ahi_common.ah_dip, cmd,
1775 	    &cautacc_ctlops_arg, NULL);
1776 }
1777 
1778 uint8_t
i_ddi_caut_get8(ddi_acc_impl_t * hp,uint8_t * addr)1779 i_ddi_caut_get8(ddi_acc_impl_t *hp, uint8_t *addr)
1780 {
1781 	uint8_t value;
1782 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1783 	    sizeof (uint8_t), 1, 0, DDI_CTLOPS_PEEK);
1784 
1785 	return (value);
1786 }
1787 
1788 uint16_t
i_ddi_caut_get16(ddi_acc_impl_t * hp,uint16_t * addr)1789 i_ddi_caut_get16(ddi_acc_impl_t *hp, uint16_t *addr)
1790 {
1791 	uint16_t value;
1792 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1793 	    sizeof (uint16_t), 1, 0, DDI_CTLOPS_PEEK);
1794 
1795 	return (value);
1796 }
1797 
1798 uint32_t
i_ddi_caut_get32(ddi_acc_impl_t * hp,uint32_t * addr)1799 i_ddi_caut_get32(ddi_acc_impl_t *hp, uint32_t *addr)
1800 {
1801 	uint32_t value;
1802 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1803 	    sizeof (uint32_t), 1, 0, DDI_CTLOPS_PEEK);
1804 
1805 	return (value);
1806 }
1807 
1808 uint64_t
i_ddi_caut_get64(ddi_acc_impl_t * hp,uint64_t * addr)1809 i_ddi_caut_get64(ddi_acc_impl_t *hp, uint64_t *addr)
1810 {
1811 	uint64_t value;
1812 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1813 	    sizeof (uint64_t), 1, 0, DDI_CTLOPS_PEEK);
1814 
1815 	return (value);
1816 }
1817 
1818 void
i_ddi_caut_put8(ddi_acc_impl_t * hp,uint8_t * addr,uint8_t value)1819 i_ddi_caut_put8(ddi_acc_impl_t *hp, uint8_t *addr, uint8_t value)
1820 {
1821 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1822 	    sizeof (uint8_t), 1, 0, DDI_CTLOPS_POKE);
1823 }
1824 
1825 void
i_ddi_caut_put16(ddi_acc_impl_t * hp,uint16_t * addr,uint16_t value)1826 i_ddi_caut_put16(ddi_acc_impl_t *hp, uint16_t *addr, uint16_t value)
1827 {
1828 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1829 	    sizeof (uint16_t), 1, 0, DDI_CTLOPS_POKE);
1830 }
1831 
1832 void
i_ddi_caut_put32(ddi_acc_impl_t * hp,uint32_t * addr,uint32_t value)1833 i_ddi_caut_put32(ddi_acc_impl_t *hp, uint32_t *addr, uint32_t value)
1834 {
1835 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1836 	    sizeof (uint32_t), 1, 0, DDI_CTLOPS_POKE);
1837 }
1838 
1839 void
i_ddi_caut_put64(ddi_acc_impl_t * hp,uint64_t * addr,uint64_t value)1840 i_ddi_caut_put64(ddi_acc_impl_t *hp, uint64_t *addr, uint64_t value)
1841 {
1842 	i_ddi_caut_getput_ctlops(hp, (uint64_t)&value, (uint64_t)addr,
1843 	    sizeof (uint64_t), 1, 0, DDI_CTLOPS_POKE);
1844 }
1845 
1846 void
i_ddi_caut_rep_get8(ddi_acc_impl_t * hp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)1847 i_ddi_caut_rep_get8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
1848 	size_t repcount, uint_t flags)
1849 {
1850 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1851 	    sizeof (uint8_t), repcount, flags, DDI_CTLOPS_PEEK);
1852 }
1853 
1854 void
i_ddi_caut_rep_get16(ddi_acc_impl_t * hp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1855 i_ddi_caut_rep_get16(ddi_acc_impl_t *hp, uint16_t *host_addr,
1856     uint16_t *dev_addr, size_t repcount, uint_t flags)
1857 {
1858 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1859 	    sizeof (uint16_t), repcount, flags, DDI_CTLOPS_PEEK);
1860 }
1861 
1862 void
i_ddi_caut_rep_get32(ddi_acc_impl_t * hp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1863 i_ddi_caut_rep_get32(ddi_acc_impl_t *hp, uint32_t *host_addr,
1864     uint32_t *dev_addr, size_t repcount, uint_t flags)
1865 {
1866 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1867 	    sizeof (uint32_t), repcount, flags, DDI_CTLOPS_PEEK);
1868 }
1869 
1870 void
i_ddi_caut_rep_get64(ddi_acc_impl_t * hp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1871 i_ddi_caut_rep_get64(ddi_acc_impl_t *hp, uint64_t *host_addr,
1872     uint64_t *dev_addr, size_t repcount, uint_t flags)
1873 {
1874 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1875 	    sizeof (uint64_t), repcount, flags, DDI_CTLOPS_PEEK);
1876 }
1877 
1878 void
i_ddi_caut_rep_put8(ddi_acc_impl_t * hp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)1879 i_ddi_caut_rep_put8(ddi_acc_impl_t *hp, uint8_t *host_addr, uint8_t *dev_addr,
1880 	size_t repcount, uint_t flags)
1881 {
1882 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1883 	    sizeof (uint8_t), repcount, flags, DDI_CTLOPS_POKE);
1884 }
1885 
1886 void
i_ddi_caut_rep_put16(ddi_acc_impl_t * hp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1887 i_ddi_caut_rep_put16(ddi_acc_impl_t *hp, uint16_t *host_addr,
1888     uint16_t *dev_addr, size_t repcount, uint_t flags)
1889 {
1890 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1891 	    sizeof (uint16_t), repcount, flags, DDI_CTLOPS_POKE);
1892 }
1893 
1894 void
i_ddi_caut_rep_put32(ddi_acc_impl_t * hp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1895 i_ddi_caut_rep_put32(ddi_acc_impl_t *hp, uint32_t *host_addr,
1896     uint32_t *dev_addr, size_t repcount, uint_t flags)
1897 {
1898 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1899 	    sizeof (uint32_t), repcount, flags, DDI_CTLOPS_POKE);
1900 }
1901 
1902 void
i_ddi_caut_rep_put64(ddi_acc_impl_t * hp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1903 i_ddi_caut_rep_put64(ddi_acc_impl_t *hp, uint64_t *host_addr,
1904     uint64_t *dev_addr, size_t repcount, uint_t flags)
1905 {
1906 	i_ddi_caut_getput_ctlops(hp, (uint64_t)host_addr, (uint64_t)dev_addr,
1907 	    sizeof (uint64_t), repcount, flags, DDI_CTLOPS_POKE);
1908 }
1909 
1910 /*
1911  * This is called only to process peek/poke when the DIP is NULL.
1912  * Assume that this is for memory, as nexi take care of device safe accesses.
1913  */
1914 int
peekpoke_mem(ddi_ctl_enum_t cmd,peekpoke_ctlops_t * in_args)1915 peekpoke_mem(ddi_ctl_enum_t cmd, peekpoke_ctlops_t *in_args)
1916 {
1917 	int err = DDI_SUCCESS;
1918 	on_trap_data_t otd;
1919 
1920 	/* Set up protected environment. */
1921 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
1922 		uintptr_t tramp = otd.ot_trampoline;
1923 
1924 		if (cmd == DDI_CTLOPS_POKE) {
1925 			otd.ot_trampoline = (uintptr_t)&poke_fault;
1926 			err = do_poke(in_args->size, (void *)in_args->dev_addr,
1927 			    (void *)in_args->host_addr);
1928 		} else {
1929 			otd.ot_trampoline = (uintptr_t)&peek_fault;
1930 			err = do_peek(in_args->size, (void *)in_args->dev_addr,
1931 			    (void *)in_args->host_addr);
1932 		}
1933 		otd.ot_trampoline = tramp;
1934 	} else
1935 		err = DDI_FAILURE;
1936 
1937 	/* Take down protected environment. */
1938 	no_trap();
1939 
1940 	return (err);
1941 }
1942