xref: /illumos-gate/usr/src/uts/intel/os/ddi_i86.c (revision f0089e39)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 /*
27  * Copyright 2014 Garrett D'Amore <garrett@damore.org>
28  */
29 
30 #include <sys/conf.h>
31 #include <sys/kmem.h>
32 #include <sys/ddi_impldefs.h>
33 #include <sys/ddi.h>
34 #include <sys/sunddi.h>
35 #include <sys/ddifm.h>
36 #include <sys/fm/io/ddi.h>
37 #include <sys/fm/protocol.h>
38 #include <sys/ontrap.h>
39 
40 
41 /*
42  * DDI DMA Engine functions for x86.
43  * These functions are more naturally generic, but do not apply to SPARC.
44  */
45 
46 int
ddi_dmae_alloc(dev_info_t * dip,int chnl,int (* dmae_waitfp)(),caddr_t arg)47 ddi_dmae_alloc(dev_info_t *dip, int chnl, int (*dmae_waitfp)(), caddr_t arg)
48 {
49 	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_ACQUIRE,
50 	    (off_t *)dmae_waitfp, (size_t *)arg,
51 	    (caddr_t *)(uintptr_t)chnl, 0));
52 }
53 
54 int
ddi_dmae_release(dev_info_t * dip,int chnl)55 ddi_dmae_release(dev_info_t *dip, int chnl)
56 {
57 	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_FREE, 0, 0,
58 	    (caddr_t *)(uintptr_t)chnl, 0));
59 }
60 
61 int
ddi_dmae_getattr(dev_info_t * dip,ddi_dma_attr_t * attrp)62 ddi_dmae_getattr(dev_info_t *dip, ddi_dma_attr_t *attrp)
63 {
64 	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_GETATTR, 0, 0,
65 	    (caddr_t *)attrp, 0));
66 }
67 
68 int
ddi_dmae_1stparty(dev_info_t * dip,int chnl)69 ddi_dmae_1stparty(dev_info_t *dip, int chnl)
70 {
71 	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_1STPTY, 0, 0,
72 	    (caddr_t *)(uintptr_t)chnl, 0));
73 }
74 
75 int
ddi_dmae_prog(dev_info_t * dip,struct ddi_dmae_req * dmaereqp,ddi_dma_cookie_t * cookiep,int chnl)76 ddi_dmae_prog(dev_info_t *dip, struct ddi_dmae_req *dmaereqp,
77     ddi_dma_cookie_t *cookiep, int chnl)
78 {
79 	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_PROG, (off_t *)dmaereqp,
80 	    (size_t *)cookiep, (caddr_t *)(uintptr_t)chnl, 0));
81 }
82 
83 int
ddi_dmae_swsetup(dev_info_t * dip,struct ddi_dmae_req * dmaereqp,ddi_dma_cookie_t * cookiep,int chnl)84 ddi_dmae_swsetup(dev_info_t *dip, struct ddi_dmae_req *dmaereqp,
85     ddi_dma_cookie_t *cookiep, int chnl)
86 {
87 	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_SWSETUP, (off_t *)dmaereqp,
88 	    (size_t *)cookiep, (caddr_t *)(uintptr_t)chnl, 0));
89 }
90 
91 int
ddi_dmae_swstart(dev_info_t * dip,int chnl)92 ddi_dmae_swstart(dev_info_t *dip, int chnl)
93 {
94 	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_SWSTART, 0, 0,
95 	    (caddr_t *)(uintptr_t)chnl, 0));
96 }
97 
98 int
ddi_dmae_stop(dev_info_t * dip,int chnl)99 ddi_dmae_stop(dev_info_t *dip, int chnl)
100 {
101 	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_STOP, 0, 0,
102 	    (caddr_t *)(uintptr_t)chnl, 0));
103 }
104 
105 int
ddi_dmae_enable(dev_info_t * dip,int chnl)106 ddi_dmae_enable(dev_info_t *dip, int chnl)
107 {
108 	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_ENABLE, 0, 0,
109 	    (caddr_t *)(uintptr_t)chnl, 0));
110 }
111 
112 int
ddi_dmae_disable(dev_info_t * dip,int chnl)113 ddi_dmae_disable(dev_info_t *dip, int chnl)
114 {
115 	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_DISABLE, 0, 0,
116 	    (caddr_t *)(uintptr_t)chnl, 0));
117 }
118 
119 int
ddi_dmae_getcnt(dev_info_t * dip,int chnl,int * countp)120 ddi_dmae_getcnt(dev_info_t *dip, int chnl, int *countp)
121 {
122 	return (ddi_dma_mctl(dip, dip, 0, DDI_DMA_E_GETCNT, 0, (size_t *)countp,
123 	    (caddr_t *)(uintptr_t)chnl, 0));
124 }
125 
126 /*
127  * implementation specific access handle and routines:
128  */
129 
130 static uintptr_t impl_acc_hdl_id = 0;
131 
132 /*
133  * access handle allocator
134  */
135 ddi_acc_hdl_t *
impl_acc_hdl_get(ddi_acc_handle_t hdl)136 impl_acc_hdl_get(ddi_acc_handle_t hdl)
137 {
138 	/*
139 	 * recast to ddi_acc_hdl_t instead of
140 	 * casting to ddi_acc_impl_t and then return the ah_platform_private
141 	 *
142 	 * this optimization based on the ddi_acc_hdl_t is the
143 	 * first member of the ddi_acc_impl_t.
144 	 */
145 	return ((ddi_acc_hdl_t *)hdl);
146 }
147 
148 ddi_acc_handle_t
impl_acc_hdl_alloc(int (* waitfp)(caddr_t),caddr_t arg)149 impl_acc_hdl_alloc(int (*waitfp)(caddr_t), caddr_t arg)
150 {
151 	ddi_acc_impl_t *hp;
152 	on_trap_data_t *otp;
153 	int sleepflag;
154 
155 	sleepflag = ((waitfp == (int (*)())KM_SLEEP) ? KM_SLEEP : KM_NOSLEEP);
156 	/*
157 	 * Allocate and initialize the data access handle and error status.
158 	 */
159 	if ((hp = kmem_zalloc(sizeof (ddi_acc_impl_t), sleepflag)) == NULL)
160 		goto fail;
161 	if ((hp->ahi_err = (ndi_err_t *)kmem_zalloc(
162 	    sizeof (ndi_err_t), sleepflag)) == NULL) {
163 		kmem_free(hp, sizeof (ddi_acc_impl_t));
164 		goto fail;
165 	}
166 	if ((otp = (on_trap_data_t *)kmem_zalloc(
167 	    sizeof (on_trap_data_t), sleepflag)) == NULL) {
168 		kmem_free(hp->ahi_err, sizeof (ndi_err_t));
169 		kmem_free(hp, sizeof (ddi_acc_impl_t));
170 		goto fail;
171 	}
172 	hp->ahi_err->err_ontrap = otp;
173 	hp->ahi_common.ah_platform_private = (void *)hp;
174 
175 	return ((ddi_acc_handle_t)hp);
176 fail:
177 	if ((waitfp != (int (*)())KM_SLEEP) &&
178 	    (waitfp != (int (*)())KM_NOSLEEP))
179 		ddi_set_callback(waitfp, arg, &impl_acc_hdl_id);
180 	return (NULL);
181 }
182 
183 void
impl_acc_hdl_free(ddi_acc_handle_t handle)184 impl_acc_hdl_free(ddi_acc_handle_t handle)
185 {
186 	ddi_acc_impl_t *hp;
187 
188 	/*
189 	 * The supplied (ddi_acc_handle_t) is actually a (ddi_acc_impl_t *),
190 	 * because that's what we allocated in impl_acc_hdl_alloc() above.
191 	 */
192 	hp = (ddi_acc_impl_t *)handle;
193 	if (hp) {
194 		kmem_free(hp->ahi_err->err_ontrap, sizeof (on_trap_data_t));
195 		kmem_free(hp->ahi_err, sizeof (ndi_err_t));
196 		kmem_free(hp, sizeof (ddi_acc_impl_t));
197 		if (impl_acc_hdl_id)
198 			ddi_run_callback(&impl_acc_hdl_id);
199 	}
200 }
201 
202 /*
203  * Function used to check if a given access handle owns the failing address.
204  * Called by ndi_fmc_error, when we detect a PIO error.
205  */
206 /* ARGSUSED */
207 static int
impl_acc_check(dev_info_t * dip,const void * handle,const void * addr,const void * not_used)208 impl_acc_check(dev_info_t *dip, const void *handle, const void *addr,
209     const void *not_used)
210 {
211 	pfn_t pfn, fault_pfn;
212 	ddi_acc_hdl_t *hp;
213 
214 	hp = impl_acc_hdl_get((ddi_acc_handle_t)handle);
215 
216 	ASSERT(hp);
217 
218 	if (addr != NULL) {
219 		pfn = hp->ah_pfn;
220 		fault_pfn = mmu_btop(*(uint64_t *)addr);
221 		if (fault_pfn >= pfn && fault_pfn < (pfn + hp->ah_pnum))
222 			return (DDI_FM_NONFATAL);
223 	}
224 	return (DDI_FM_UNKNOWN);
225 }
226 
227 void
impl_acc_err_init(ddi_acc_hdl_t * handlep)228 impl_acc_err_init(ddi_acc_hdl_t *handlep)
229 {
230 	int fmcap;
231 	ndi_err_t *errp;
232 	on_trap_data_t *otp;
233 	ddi_acc_impl_t *hp = (ddi_acc_impl_t *)handlep;
234 
235 	fmcap = ddi_fm_capable(handlep->ah_dip);
236 
237 	if (handlep->ah_acc.devacc_attr_version < DDI_DEVICE_ATTR_V1 ||
238 	    !DDI_FM_ACC_ERR_CAP(fmcap)) {
239 		handlep->ah_acc.devacc_attr_access = DDI_DEFAULT_ACC;
240 	} else if (handlep->ah_acc.devacc_attr_access == DDI_FLAGERR_ACC &&
241 	    hp->ahi_scan == NULL) {
242 		handlep->ah_acc.devacc_attr_access = DDI_DEFAULT_ACC;
243 	} else if (DDI_FM_ACC_ERR_CAP(fmcap)) {
244 		if (handlep->ah_acc.devacc_attr_access == DDI_DEFAULT_ACC) {
245 			if (handlep->ah_xfermodes)
246 				return;
247 			i_ddi_drv_ereport_post(handlep->ah_dip, DVR_EFMCAP,
248 			    NULL, DDI_NOSLEEP);
249 		} else {
250 			errp = hp->ahi_err;
251 			otp = (on_trap_data_t *)errp->err_ontrap;
252 			otp->ot_handle = (void *)(hp);
253 			otp->ot_prot = OT_DATA_ACCESS;
254 			errp->err_status = DDI_FM_OK;
255 			errp->err_expected = DDI_FM_ERR_UNEXPECTED;
256 			errp->err_cf = impl_acc_check;
257 		}
258 	}
259 }
260 
261 /* ARGSUSED */
262 int
impl_dma_check(dev_info_t * dip,const void * handle,const void * pci_hdl,const void * not_used)263 impl_dma_check(dev_info_t *dip, const void *handle, const void *pci_hdl,
264     const void *not_used)
265 {
266 	return (DDI_FM_UNKNOWN);
267 }
268 
269 void
impl_acc_hdl_init(ddi_acc_hdl_t * handlep)270 impl_acc_hdl_init(ddi_acc_hdl_t *handlep)
271 {
272 	ddi_acc_impl_t *hp;
273 	int fmcap;
274 	int devacc_attr_access;
275 
276 	if (!handlep)
277 		return;
278 	fmcap = ddi_fm_capable(handlep->ah_dip);
279 	if (handlep->ah_acc.devacc_attr_version < DDI_DEVICE_ATTR_V1 ||
280 	    !DDI_FM_ACC_ERR_CAP(fmcap))
281 		devacc_attr_access = DDI_DEFAULT_ACC;
282 	else
283 		devacc_attr_access = handlep->ah_acc.devacc_attr_access;
284 
285 	hp = (ddi_acc_impl_t *)handlep->ah_platform_private;
286 
287 	/*
288 	 * Can only do FLAGERR if scan callback is set up. This should
289 	 * also guarantee that the peekpoke_mutex and err_mutex are defined.
290 	 */
291 	if (devacc_attr_access == DDI_FLAGERR_ACC && hp->ahi_scan == NULL)
292 		devacc_attr_access = DDI_DEFAULT_ACC;
293 
294 	switch (devacc_attr_access) {
295 	case DDI_CAUTIOUS_ACC:
296 		hp->ahi_get8 = i_ddi_caut_get8;
297 		hp->ahi_put8 = i_ddi_caut_put8;
298 		hp->ahi_rep_get8 = i_ddi_caut_rep_get8;
299 		hp->ahi_rep_put8 = i_ddi_caut_rep_put8;
300 		hp->ahi_get16 = i_ddi_caut_get16;
301 		hp->ahi_get32 = i_ddi_caut_get32;
302 		hp->ahi_put16 = i_ddi_caut_put16;
303 		hp->ahi_put32 = i_ddi_caut_put32;
304 		hp->ahi_rep_get16 = i_ddi_caut_rep_get16;
305 		hp->ahi_rep_get32 = i_ddi_caut_rep_get32;
306 		hp->ahi_rep_put16 = i_ddi_caut_rep_put16;
307 		hp->ahi_rep_put32 = i_ddi_caut_rep_put32;
308 		hp->ahi_get64 = i_ddi_caut_get64;
309 		hp->ahi_put64 = i_ddi_caut_put64;
310 		hp->ahi_rep_get64 = i_ddi_caut_rep_get64;
311 		hp->ahi_rep_put64 = i_ddi_caut_rep_put64;
312 		break;
313 	case DDI_FLAGERR_ACC:
314 		if (hp->ahi_acc_attr & DDI_ACCATTR_IO_SPACE) {
315 			hp->ahi_get8 = i_ddi_prot_io_get8;
316 			hp->ahi_put8 = i_ddi_prot_io_put8;
317 			hp->ahi_rep_get8 = i_ddi_prot_io_rep_get8;
318 			hp->ahi_rep_put8 = i_ddi_prot_io_rep_put8;
319 
320 			/* temporary set these 64 functions to no-ops */
321 			hp->ahi_get64 = i_ddi_io_get64;
322 			hp->ahi_put64 = i_ddi_io_put64;
323 			hp->ahi_rep_get64 = i_ddi_io_rep_get64;
324 			hp->ahi_rep_put64 = i_ddi_io_rep_put64;
325 
326 			/*
327 			 * check for BIG endian access
328 			 */
329 			if (handlep->ah_acc.devacc_attr_endian_flags ==
330 			    DDI_STRUCTURE_BE_ACC) {
331 				hp->ahi_get16 = i_ddi_prot_io_swap_get16;
332 				hp->ahi_get32 = i_ddi_prot_io_swap_get32;
333 				hp->ahi_put16 = i_ddi_prot_io_swap_put16;
334 				hp->ahi_put32 = i_ddi_prot_io_swap_put32;
335 				hp->ahi_rep_get16 =
336 				    i_ddi_prot_io_swap_rep_get16;
337 				hp->ahi_rep_get32 =
338 				    i_ddi_prot_io_swap_rep_get32;
339 				hp->ahi_rep_put16 =
340 				    i_ddi_prot_io_swap_rep_put16;
341 				hp->ahi_rep_put32 =
342 				    i_ddi_prot_io_swap_rep_put32;
343 			} else {
344 				hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
345 				hp->ahi_get16 = i_ddi_prot_io_get16;
346 				hp->ahi_get32 = i_ddi_prot_io_get32;
347 				hp->ahi_put16 = i_ddi_prot_io_put16;
348 				hp->ahi_put32 = i_ddi_prot_io_put32;
349 				hp->ahi_rep_get16 = i_ddi_prot_io_rep_get16;
350 				hp->ahi_rep_get32 = i_ddi_prot_io_rep_get32;
351 				hp->ahi_rep_put16 = i_ddi_prot_io_rep_put16;
352 				hp->ahi_rep_put32 = i_ddi_prot_io_rep_put32;
353 			}
354 
355 		} else if (hp->ahi_acc_attr & DDI_ACCATTR_CPU_VADDR) {
356 
357 			hp->ahi_get8 = i_ddi_prot_vaddr_get8;
358 			hp->ahi_put8 = i_ddi_prot_vaddr_put8;
359 			hp->ahi_rep_get8 = i_ddi_prot_vaddr_rep_get8;
360 			hp->ahi_rep_put8 = i_ddi_prot_vaddr_rep_put8;
361 
362 			/*
363 			 * check for BIG endian access
364 			 */
365 			if (handlep->ah_acc.devacc_attr_endian_flags ==
366 			    DDI_STRUCTURE_BE_ACC) {
367 
368 				hp->ahi_get16 = i_ddi_prot_vaddr_swap_get16;
369 				hp->ahi_get32 = i_ddi_prot_vaddr_swap_get32;
370 				hp->ahi_get64 = i_ddi_prot_vaddr_swap_get64;
371 				hp->ahi_put16 = i_ddi_prot_vaddr_swap_put16;
372 				hp->ahi_put32 = i_ddi_prot_vaddr_swap_put32;
373 				hp->ahi_put64 = i_ddi_prot_vaddr_swap_put64;
374 				hp->ahi_rep_get16 =
375 				    i_ddi_prot_vaddr_swap_rep_get16;
376 				hp->ahi_rep_get32 =
377 				    i_ddi_prot_vaddr_swap_rep_get32;
378 				hp->ahi_rep_get64 =
379 				    i_ddi_prot_vaddr_swap_rep_get64;
380 				hp->ahi_rep_put16 =
381 				    i_ddi_prot_vaddr_swap_rep_put16;
382 				hp->ahi_rep_put32 =
383 				    i_ddi_prot_vaddr_swap_rep_put32;
384 				hp->ahi_rep_put64 =
385 				    i_ddi_prot_vaddr_swap_rep_put64;
386 			} else {
387 				hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
388 				hp->ahi_get16 = i_ddi_prot_vaddr_get16;
389 				hp->ahi_get32 = i_ddi_prot_vaddr_get32;
390 				hp->ahi_get64 = i_ddi_prot_vaddr_get64;
391 				hp->ahi_put16 = i_ddi_prot_vaddr_put16;
392 				hp->ahi_put32 = i_ddi_prot_vaddr_put32;
393 				hp->ahi_put64 = i_ddi_prot_vaddr_put64;
394 				hp->ahi_rep_get16 = i_ddi_prot_vaddr_rep_get16;
395 				hp->ahi_rep_get32 = i_ddi_prot_vaddr_rep_get32;
396 				hp->ahi_rep_get64 = i_ddi_prot_vaddr_rep_get64;
397 				hp->ahi_rep_put16 = i_ddi_prot_vaddr_rep_put16;
398 				hp->ahi_rep_put32 = i_ddi_prot_vaddr_rep_put32;
399 				hp->ahi_rep_put64 = i_ddi_prot_vaddr_rep_put64;
400 			}
401 		}
402 		break;
403 	case DDI_DEFAULT_ACC:
404 		if (hp->ahi_acc_attr & DDI_ACCATTR_IO_SPACE) {
405 			hp->ahi_get8 = i_ddi_io_get8;
406 			hp->ahi_put8 = i_ddi_io_put8;
407 			hp->ahi_rep_get8 = i_ddi_io_rep_get8;
408 			hp->ahi_rep_put8 = i_ddi_io_rep_put8;
409 
410 			/* temporary set these 64 functions to no-ops */
411 			hp->ahi_get64 = i_ddi_io_get64;
412 			hp->ahi_put64 = i_ddi_io_put64;
413 			hp->ahi_rep_get64 = i_ddi_io_rep_get64;
414 			hp->ahi_rep_put64 = i_ddi_io_rep_put64;
415 
416 			/*
417 			 * check for BIG endian access
418 			 */
419 			if (handlep->ah_acc.devacc_attr_endian_flags ==
420 			    DDI_STRUCTURE_BE_ACC) {
421 				hp->ahi_get16 = i_ddi_io_swap_get16;
422 				hp->ahi_get32 = i_ddi_io_swap_get32;
423 				hp->ahi_put16 = i_ddi_io_swap_put16;
424 				hp->ahi_put32 = i_ddi_io_swap_put32;
425 				hp->ahi_rep_get16 = i_ddi_io_swap_rep_get16;
426 				hp->ahi_rep_get32 = i_ddi_io_swap_rep_get32;
427 				hp->ahi_rep_put16 = i_ddi_io_swap_rep_put16;
428 				hp->ahi_rep_put32 = i_ddi_io_swap_rep_put32;
429 			} else {
430 				hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
431 				hp->ahi_get16 = i_ddi_io_get16;
432 				hp->ahi_get32 = i_ddi_io_get32;
433 				hp->ahi_put16 = i_ddi_io_put16;
434 				hp->ahi_put32 = i_ddi_io_put32;
435 				hp->ahi_rep_get16 = i_ddi_io_rep_get16;
436 				hp->ahi_rep_get32 = i_ddi_io_rep_get32;
437 				hp->ahi_rep_put16 = i_ddi_io_rep_put16;
438 				hp->ahi_rep_put32 = i_ddi_io_rep_put32;
439 			}
440 
441 		} else if (hp->ahi_acc_attr & DDI_ACCATTR_CPU_VADDR) {
442 
443 			hp->ahi_get8 = i_ddi_vaddr_get8;
444 			hp->ahi_put8 = i_ddi_vaddr_put8;
445 			hp->ahi_rep_get8 = i_ddi_vaddr_rep_get8;
446 			hp->ahi_rep_put8 = i_ddi_vaddr_rep_put8;
447 
448 			/*
449 			 * check for BIG endian access
450 			 */
451 			if (handlep->ah_acc.devacc_attr_endian_flags ==
452 			    DDI_STRUCTURE_BE_ACC) {
453 
454 				hp->ahi_get16 = i_ddi_vaddr_swap_get16;
455 				hp->ahi_get32 = i_ddi_vaddr_swap_get32;
456 				hp->ahi_get64 = i_ddi_vaddr_swap_get64;
457 				hp->ahi_put16 = i_ddi_vaddr_swap_put16;
458 				hp->ahi_put32 = i_ddi_vaddr_swap_put32;
459 				hp->ahi_put64 = i_ddi_vaddr_swap_put64;
460 				hp->ahi_rep_get16 = i_ddi_vaddr_swap_rep_get16;
461 				hp->ahi_rep_get32 = i_ddi_vaddr_swap_rep_get32;
462 				hp->ahi_rep_get64 = i_ddi_vaddr_swap_rep_get64;
463 				hp->ahi_rep_put16 = i_ddi_vaddr_swap_rep_put16;
464 				hp->ahi_rep_put32 = i_ddi_vaddr_swap_rep_put32;
465 				hp->ahi_rep_put64 = i_ddi_vaddr_swap_rep_put64;
466 			} else {
467 				hp->ahi_acc_attr |= DDI_ACCATTR_DIRECT;
468 				hp->ahi_get16 = i_ddi_vaddr_get16;
469 				hp->ahi_get32 = i_ddi_vaddr_get32;
470 				hp->ahi_get64 = i_ddi_vaddr_get64;
471 				hp->ahi_put16 = i_ddi_vaddr_put16;
472 				hp->ahi_put32 = i_ddi_vaddr_put32;
473 				hp->ahi_put64 = i_ddi_vaddr_put64;
474 				hp->ahi_rep_get16 = i_ddi_vaddr_rep_get16;
475 				hp->ahi_rep_get32 = i_ddi_vaddr_rep_get32;
476 				hp->ahi_rep_get64 = i_ddi_vaddr_rep_get64;
477 				hp->ahi_rep_put16 = i_ddi_vaddr_rep_put16;
478 				hp->ahi_rep_put32 = i_ddi_vaddr_rep_put32;
479 				hp->ahi_rep_put64 = i_ddi_vaddr_rep_put64;
480 			}
481 		}
482 		break;
483 	}
484 	hp->ahi_fault_check = i_ddi_acc_fault_check;
485 	hp->ahi_fault_notify = i_ddi_acc_fault_notify;
486 	hp->ahi_fault = 0;
487 	impl_acc_err_init(handlep);
488 }
489 
490 /*
491  * The followings are low-level routines for data access.
492  *
493  * All of these routines should be implemented in assembly. Those
494  * that have been rewritten be found in ~ml/ddi_i86_asm.s
495  */
496 
497 /*ARGSUSED*/
498 uint16_t
i_ddi_vaddr_swap_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)499 i_ddi_vaddr_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
500 {
501 	return (ddi_swap16(*addr));
502 }
503 
504 /*ARGSUSED*/
505 uint16_t
i_ddi_io_swap_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)506 i_ddi_io_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
507 {
508 	return (ddi_swap16(inw((uintptr_t)addr)));
509 }
510 
511 /*ARGSUSED*/
512 uint32_t
i_ddi_vaddr_swap_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)513 i_ddi_vaddr_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
514 {
515 	return (ddi_swap32(*addr));
516 }
517 
518 /*ARGSUSED*/
519 uint32_t
i_ddi_io_swap_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)520 i_ddi_io_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
521 {
522 	return (ddi_swap32(inl((uintptr_t)addr)));
523 }
524 
525 /*ARGSUSED*/
526 uint64_t
i_ddi_vaddr_swap_get64(ddi_acc_impl_t * hdlp,uint64_t * addr)527 i_ddi_vaddr_swap_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
528 {
529 	return (ddi_swap64(*addr));
530 }
531 
532 /*ARGSUSED*/
533 void
i_ddi_vaddr_swap_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)534 i_ddi_vaddr_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
535 {
536 	*addr = ddi_swap16(value);
537 }
538 
539 /*ARGSUSED*/
540 void
i_ddi_io_swap_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)541 i_ddi_io_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
542 {
543 	outw((uintptr_t)addr, ddi_swap16(value));
544 }
545 
546 /*ARGSUSED*/
547 void
i_ddi_vaddr_swap_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)548 i_ddi_vaddr_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
549 {
550 	*addr = ddi_swap32(value);
551 }
552 
553 /*ARGSUSED*/
554 void
i_ddi_io_swap_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)555 i_ddi_io_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
556 {
557 	outl((uintptr_t)addr, ddi_swap32(value));
558 }
559 
560 /*ARGSUSED*/
561 void
i_ddi_vaddr_swap_put64(ddi_acc_impl_t * hdlp,uint64_t * addr,uint64_t value)562 i_ddi_vaddr_swap_put64(ddi_acc_impl_t *hdlp, uint64_t *addr, uint64_t value)
563 {
564 	*addr = ddi_swap64(value);
565 }
566 
567 /*ARGSUSED*/
568 void
i_ddi_vaddr_rep_get8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)569 i_ddi_vaddr_rep_get8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
570     uint8_t *dev_addr, size_t repcount, uint_t flags)
571 {
572 	uint8_t	*h, *d;
573 
574 	h = host_addr;
575 	d = dev_addr;
576 
577 	if (flags == DDI_DEV_AUTOINCR)
578 		for (; repcount; repcount--)
579 			*h++ = *d++;
580 	else
581 		for (; repcount; repcount--)
582 			*h++ = *d;
583 }
584 
585 /*ARGSUSED*/
586 void
i_ddi_vaddr_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)587 i_ddi_vaddr_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
588     uint16_t *dev_addr, size_t repcount, uint_t flags)
589 {
590 	uint16_t *h, *d;
591 
592 	h = host_addr;
593 	d = dev_addr;
594 
595 	if (flags == DDI_DEV_AUTOINCR)
596 		for (; repcount; repcount--)
597 			*h++ = *d++;
598 	else
599 		for (; repcount; repcount--)
600 			*h++ = *d;
601 }
602 
603 /*ARGSUSED*/
604 void
i_ddi_vaddr_swap_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)605 i_ddi_vaddr_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
606     uint16_t *dev_addr, size_t repcount, uint_t flags)
607 {
608 	uint16_t *h, *d;
609 
610 	h = host_addr;
611 	d = dev_addr;
612 
613 	if (flags == DDI_DEV_AUTOINCR)
614 		for (; repcount; repcount--)
615 			*h++ = ddi_swap16(*d++);
616 	else
617 		for (; repcount; repcount--)
618 			*h++ = ddi_swap16(*d);
619 }
620 
621 /*ARGSUSED*/
622 void
i_ddi_io_swap_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)623 i_ddi_io_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
624     uint16_t *dev_addr, size_t repcount, uint_t flags)
625 {
626 	uint16_t *h;
627 	uintptr_t port;
628 
629 	h = host_addr;
630 	port = (uintptr_t)dev_addr;
631 
632 	if (flags == DDI_DEV_AUTOINCR)
633 		for (; repcount; repcount--, port += 2)
634 			*h++ = ddi_swap16(inw(port));
635 	else
636 		for (; repcount; repcount--)
637 			*h++ = ddi_swap16(inw(port));
638 }
639 
640 /*ARGSUSED*/
641 void
i_ddi_vaddr_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)642 i_ddi_vaddr_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
643     uint32_t *dev_addr, size_t repcount, uint_t flags)
644 {
645 	uint32_t *h, *d;
646 
647 	h = host_addr;
648 	d = dev_addr;
649 
650 	if (flags == DDI_DEV_AUTOINCR)
651 		for (; repcount; repcount--)
652 			*h++ = *d++;
653 	else
654 		for (; repcount; repcount--)
655 			*h++ = *d;
656 }
657 
658 /*ARGSUSED*/
659 void
i_ddi_vaddr_swap_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)660 i_ddi_vaddr_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
661     uint32_t *dev_addr, size_t repcount, uint_t flags)
662 {
663 	uint32_t *h, *d;
664 
665 	h = host_addr;
666 	d = dev_addr;
667 
668 	if (flags == DDI_DEV_AUTOINCR)
669 		for (; repcount; repcount--)
670 			*h++ = ddi_swap32(*d++);
671 	else
672 		for (; repcount; repcount--)
673 			*h++ = ddi_swap32(*d);
674 }
675 
676 /*ARGSUSED*/
677 void
i_ddi_io_swap_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)678 i_ddi_io_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
679     uint32_t *dev_addr, size_t repcount, uint_t flags)
680 {
681 	uint32_t *h;
682 	uintptr_t port;
683 
684 	h = host_addr;
685 	port = (uintptr_t)dev_addr;
686 
687 	if (flags == DDI_DEV_AUTOINCR)
688 		for (; repcount; repcount--, port += 4)
689 			*h++ = ddi_swap32(inl(port));
690 	else
691 		for (; repcount; repcount--)
692 			*h++ = ddi_swap32(inl(port));
693 }
694 
695 /*ARGSUSED*/
696 void
i_ddi_vaddr_rep_get64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)697 i_ddi_vaddr_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
698     uint64_t *dev_addr, size_t repcount, uint_t flags)
699 {
700 	uint64_t *h, *d;
701 
702 	h = host_addr;
703 	d = dev_addr;
704 
705 	if (flags == DDI_DEV_AUTOINCR)
706 		for (; repcount; repcount--)
707 			*h++ = *d++;
708 	else
709 		for (; repcount; repcount--)
710 			*h++ = *d;
711 }
712 
713 /*ARGSUSED*/
714 void
i_ddi_vaddr_swap_rep_get64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)715 i_ddi_vaddr_swap_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
716     uint64_t *dev_addr, size_t repcount, uint_t flags)
717 {
718 	uint64_t *h, *d;
719 
720 	h = host_addr;
721 	d = dev_addr;
722 
723 	if (flags == DDI_DEV_AUTOINCR)
724 		for (; repcount; repcount--)
725 			*h++ = ddi_swap64(*d++);
726 	else
727 		for (; repcount; repcount--)
728 			*h++ = ddi_swap64(*d);
729 }
730 
731 /*ARGSUSED*/
732 void
i_ddi_vaddr_rep_put8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)733 i_ddi_vaddr_rep_put8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
734     uint8_t *dev_addr, size_t repcount, uint_t flags)
735 {
736 	uint8_t	*h, *d;
737 
738 	h = host_addr;
739 	d = dev_addr;
740 
741 	if (flags == DDI_DEV_AUTOINCR)
742 		for (; repcount; repcount--)
743 			*d++ = *h++;
744 	else
745 		for (; repcount; repcount--)
746 			*d = *h++;
747 }
748 
749 /*ARGSUSED*/
750 void
i_ddi_vaddr_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)751 i_ddi_vaddr_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
752     uint16_t *dev_addr, size_t repcount, uint_t flags)
753 {
754 	uint16_t *h, *d;
755 
756 	h = host_addr;
757 	d = dev_addr;
758 
759 	if (flags == DDI_DEV_AUTOINCR)
760 		for (; repcount; repcount--)
761 			*d++ = *h++;
762 	else
763 		for (; repcount; repcount--)
764 			*d = *h++;
765 }
766 
767 /*ARGSUSED*/
768 void
i_ddi_vaddr_swap_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)769 i_ddi_vaddr_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
770     uint16_t *dev_addr, size_t repcount, uint_t flags)
771 {
772 	uint16_t *h, *d;
773 
774 	h = host_addr;
775 	d = dev_addr;
776 
777 	if (flags == DDI_DEV_AUTOINCR)
778 		for (; repcount; repcount--)
779 			*d++ = ddi_swap16(*h++);
780 	else
781 		for (; repcount; repcount--)
782 			*d = ddi_swap16(*h++);
783 }
784 
785 /*ARGSUSED*/
786 void
i_ddi_io_swap_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)787 i_ddi_io_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
788     uint16_t *dev_addr, size_t repcount, uint_t flags)
789 {
790 	uint16_t *h;
791 	uintptr_t port;
792 
793 	h = host_addr;
794 	port = (uintptr_t)dev_addr;
795 
796 	if (flags == DDI_DEV_AUTOINCR)
797 		for (; repcount; repcount--, port += 2)
798 			outw(port, ddi_swap16(*h++));
799 	else
800 		for (; repcount; repcount--)
801 			outw(port, ddi_swap16(*h++));
802 }
803 
804 /*ARGSUSED*/
805 void
i_ddi_vaddr_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)806 i_ddi_vaddr_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
807     uint32_t *dev_addr, size_t repcount, uint_t flags)
808 {
809 	uint32_t *h, *d;
810 
811 	h = host_addr;
812 	d = dev_addr;
813 
814 	if (flags == DDI_DEV_AUTOINCR)
815 		for (; repcount; repcount--)
816 			*d++ = *h++;
817 	else
818 		for (; repcount; repcount--)
819 			*d = *h++;
820 }
821 
822 /*ARGSUSED*/
823 void
i_ddi_vaddr_swap_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)824 i_ddi_vaddr_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
825     uint32_t *dev_addr, size_t repcount, uint_t flags)
826 {
827 	uint32_t *h, *d;
828 
829 	h = host_addr;
830 	d = dev_addr;
831 
832 	if (flags == DDI_DEV_AUTOINCR)
833 		for (; repcount; repcount--)
834 			*d++ = ddi_swap32(*h++);
835 	else
836 		for (; repcount; repcount--)
837 			*d = ddi_swap32(*h++);
838 }
839 
840 /*ARGSUSED*/
841 void
i_ddi_io_swap_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)842 i_ddi_io_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
843     uint32_t *dev_addr, size_t repcount, uint_t flags)
844 {
845 	uint32_t *h;
846 	uintptr_t port;
847 
848 	h = host_addr;
849 	port = (uintptr_t)dev_addr;
850 
851 	if (flags == DDI_DEV_AUTOINCR)
852 		for (; repcount; repcount--, port += 4)
853 			outl(port, ddi_swap32(*h++));
854 	else
855 		for (; repcount; repcount--)
856 			outl(port, ddi_swap32(*h++));
857 }
858 
859 /*ARGSUSED*/
860 void
i_ddi_vaddr_rep_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)861 i_ddi_vaddr_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
862     uint64_t *dev_addr, size_t repcount, uint_t flags)
863 {
864 	uint64_t *h, *d;
865 
866 	h = host_addr;
867 	d = dev_addr;
868 
869 	if (flags == DDI_DEV_AUTOINCR)
870 		for (; repcount; repcount--)
871 			*d++ = *h++;
872 	else
873 		for (; repcount; repcount--)
874 			*d = *h++;
875 }
876 
877 /*ARGSUSED*/
878 void
i_ddi_vaddr_swap_rep_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)879 i_ddi_vaddr_swap_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
880     uint64_t *dev_addr, size_t repcount, uint_t flags)
881 {
882 	uint64_t *h, *d;
883 
884 	h = host_addr;
885 	d = dev_addr;
886 
887 	if (flags == DDI_DEV_AUTOINCR)
888 		for (; repcount; repcount--)
889 			*d++ = ddi_swap64(*h++);
890 	else
891 		for (; repcount; repcount--)
892 			*d = ddi_swap64(*h++);
893 }
894 
895 /*ARGSUSED*/
896 uint64_t
i_ddi_io_get64(ddi_acc_impl_t * hdlp,uint64_t * addr)897 i_ddi_io_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
898 {
899 	panic("ddi_get64 from i/o space");
900 	/*NOTREACHED*/
901 	return (0);
902 }
903 
904 /*ARGSUSED*/
905 void
i_ddi_io_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t value)906 i_ddi_io_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr, uint64_t value)
907 {
908 	panic("ddi_put64 to i/o space");
909 	/*NOTREACHED*/
910 }
911 
912 void
do_scan(ddi_acc_impl_t * hdlp)913 do_scan(ddi_acc_impl_t *hdlp)
914 {
915 	ddi_fm_error_t de;
916 	ndi_err_t *errp = (ndi_err_t *)hdlp->ahi_err;
917 
918 	bzero(&de, sizeof (ddi_fm_error_t));
919 	de.fme_version = DDI_FME_VERSION;
920 	de.fme_ena = fm_ena_generate(0, FM_ENA_FMT1);
921 	de.fme_flag = DDI_FM_ERR_UNEXPECTED;
922 
923 	mutex_enter(hdlp->ahi_err_mutexp);
924 	hdlp->ahi_scan(hdlp->ahi_scan_dip, &de);
925 	if (de.fme_status != DDI_FM_OK) {
926 		errp->err_ena = de.fme_ena;
927 		errp->err_expected = de.fme_flag;
928 		errp->err_status = DDI_FM_NONFATAL;
929 	}
930 	mutex_exit(hdlp->ahi_err_mutexp);
931 }
932 
933 /*ARGSUSED*/
934 uint8_t
i_ddi_prot_vaddr_get8(ddi_acc_impl_t * hdlp,uint8_t * addr)935 i_ddi_prot_vaddr_get8(ddi_acc_impl_t *hdlp, uint8_t *addr)
936 {
937 	uint8_t val;
938 
939 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
940 	val = *addr;
941 	if (val == 0xff)
942 		do_scan(hdlp);
943 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
944 
945 	return (val);
946 }
947 
948 /*ARGSUSED*/
949 uint16_t
i_ddi_prot_vaddr_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)950 i_ddi_prot_vaddr_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
951 {
952 	uint16_t val;
953 
954 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
955 	val = *addr;
956 	if (val == 0xffff)
957 		do_scan(hdlp);
958 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
959 
960 	return (val);
961 }
962 
963 /*ARGSUSED*/
964 uint32_t
i_ddi_prot_vaddr_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)965 i_ddi_prot_vaddr_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
966 {
967 	uint32_t val;
968 
969 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
970 	val = *addr;
971 	if (val == 0xffffffff)
972 		do_scan(hdlp);
973 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
974 
975 	return (val);
976 }
977 
978 /*ARGSUSED*/
979 uint64_t
i_ddi_prot_vaddr_get64(ddi_acc_impl_t * hdlp,uint64_t * addr)980 i_ddi_prot_vaddr_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
981 {
982 	uint64_t val;
983 
984 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
985 	val = *addr;
986 	if (val == 0xffffffffffffffff)
987 		do_scan(hdlp);
988 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
989 
990 	return (val);
991 }
992 
993 /*ARGSUSED*/
994 uint8_t
i_ddi_prot_io_get8(ddi_acc_impl_t * hdlp,uint8_t * addr)995 i_ddi_prot_io_get8(ddi_acc_impl_t *hdlp, uint8_t *addr)
996 {
997 	uint8_t val;
998 
999 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1000 	val = inb((uintptr_t)addr);
1001 	if (val == 0xff)
1002 		do_scan(hdlp);
1003 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1004 
1005 	return (val);
1006 }
1007 
1008 /*ARGSUSED*/
1009 uint16_t
i_ddi_prot_io_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)1010 i_ddi_prot_io_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
1011 {
1012 	uint16_t val;
1013 
1014 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1015 	val = inw((uintptr_t)addr);
1016 	if (val == 0xffff)
1017 		do_scan(hdlp);
1018 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1019 
1020 	return (val);
1021 }
1022 
1023 /*ARGSUSED*/
1024 uint32_t
i_ddi_prot_io_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)1025 i_ddi_prot_io_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
1026 {
1027 	uint32_t val;
1028 
1029 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1030 	val = inl((uintptr_t)addr);
1031 	if (val == 0xffffffff)
1032 		do_scan(hdlp);
1033 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1034 
1035 	return (val);
1036 }
1037 
1038 /*ARGSUSED*/
1039 uint16_t
i_ddi_prot_vaddr_swap_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)1040 i_ddi_prot_vaddr_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
1041 {
1042 	uint16_t val;
1043 
1044 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1045 	val = ddi_swap16(*addr);
1046 	if (val == 0xffff)
1047 		do_scan(hdlp);
1048 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1049 
1050 	return (val);
1051 }
1052 
1053 /*ARGSUSED*/
1054 uint16_t
i_ddi_prot_io_swap_get16(ddi_acc_impl_t * hdlp,uint16_t * addr)1055 i_ddi_prot_io_swap_get16(ddi_acc_impl_t *hdlp, uint16_t *addr)
1056 {
1057 	uint16_t val;
1058 
1059 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1060 	val = ddi_swap16(inw((uintptr_t)addr));
1061 	if (val == 0xffff)
1062 		do_scan(hdlp);
1063 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1064 
1065 	return (val);
1066 }
1067 
1068 /*ARGSUSED*/
1069 uint32_t
i_ddi_prot_vaddr_swap_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)1070 i_ddi_prot_vaddr_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
1071 {
1072 	uint32_t val;
1073 
1074 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1075 	val = ddi_swap32(*addr);
1076 	if (val == 0xffffffff)
1077 		do_scan(hdlp);
1078 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1079 
1080 	return (val);
1081 }
1082 
1083 /*ARGSUSED*/
1084 uint32_t
i_ddi_prot_io_swap_get32(ddi_acc_impl_t * hdlp,uint32_t * addr)1085 i_ddi_prot_io_swap_get32(ddi_acc_impl_t *hdlp, uint32_t *addr)
1086 {
1087 	uint32_t val;
1088 
1089 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1090 	val = ddi_swap32(inl((uintptr_t)addr));
1091 	if (val == 0xffffffff)
1092 		do_scan(hdlp);
1093 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1094 
1095 	return (val);
1096 }
1097 
1098 /*ARGSUSED*/
1099 uint64_t
i_ddi_prot_vaddr_swap_get64(ddi_acc_impl_t * hdlp,uint64_t * addr)1100 i_ddi_prot_vaddr_swap_get64(ddi_acc_impl_t *hdlp, uint64_t *addr)
1101 {
1102 	uint64_t val;
1103 
1104 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1105 	val = ddi_swap64(*addr);
1106 	if (val == 0xffffffffffffffff)
1107 		do_scan(hdlp);
1108 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1109 
1110 	return (val);
1111 }
1112 
1113 /*ARGSUSED*/
1114 void
i_ddi_prot_vaddr_put8(ddi_acc_impl_t * hdlp,uint8_t * addr,uint8_t value)1115 i_ddi_prot_vaddr_put8(ddi_acc_impl_t *hdlp, uint8_t *addr, uint8_t value)
1116 {
1117 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1118 	*addr = value;
1119 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1120 }
1121 
1122 /*ARGSUSED*/
1123 void
i_ddi_prot_io_put8(ddi_acc_impl_t * hdlp,uint8_t * addr,uint8_t value)1124 i_ddi_prot_io_put8(ddi_acc_impl_t *hdlp, uint8_t *addr, uint8_t value)
1125 {
1126 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1127 	outb((uintptr_t)addr, value);
1128 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1129 }
1130 
1131 /*ARGSUSED*/
1132 void
i_ddi_prot_vaddr_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)1133 i_ddi_prot_vaddr_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
1134 {
1135 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1136 	*addr = value;
1137 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1138 }
1139 
1140 /*ARGSUSED*/
1141 void
i_ddi_prot_io_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)1142 i_ddi_prot_io_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
1143 {
1144 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1145 	outw((uintptr_t)addr, value);
1146 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1147 }
1148 
1149 /*ARGSUSED*/
1150 void
i_ddi_prot_vaddr_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)1151 i_ddi_prot_vaddr_put32(ddi_acc_impl_t *hdlp, uint32_t *addr,
1152     uint32_t value)
1153 {
1154 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1155 	*addr = value;
1156 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1157 }
1158 
1159 /*ARGSUSED*/
1160 void
i_ddi_prot_io_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)1161 i_ddi_prot_io_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
1162 {
1163 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1164 	outl((uintptr_t)addr, value);
1165 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1166 }
1167 
1168 /*ARGSUSED*/
1169 void
i_ddi_prot_vaddr_put64(ddi_acc_impl_t * hdlp,uint64_t * addr,uint64_t value)1170 i_ddi_prot_vaddr_put64(ddi_acc_impl_t *hdlp, uint64_t *addr,
1171     uint64_t value)
1172 {
1173 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1174 	*addr = value;
1175 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1176 }
1177 
1178 /*ARGSUSED*/
1179 void
i_ddi_prot_vaddr_swap_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)1180 i_ddi_prot_vaddr_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr,
1181     uint16_t value)
1182 {
1183 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1184 	*addr = ddi_swap16(value);
1185 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1186 }
1187 
1188 /*ARGSUSED*/
1189 void
i_ddi_prot_io_swap_put16(ddi_acc_impl_t * hdlp,uint16_t * addr,uint16_t value)1190 i_ddi_prot_io_swap_put16(ddi_acc_impl_t *hdlp, uint16_t *addr, uint16_t value)
1191 {
1192 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1193 	outw((uintptr_t)addr, ddi_swap16(value));
1194 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1195 }
1196 
1197 /*ARGSUSED*/
1198 void
i_ddi_prot_vaddr_swap_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)1199 i_ddi_prot_vaddr_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr,
1200     uint32_t value)
1201 {
1202 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1203 	*addr = ddi_swap32(value);
1204 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1205 }
1206 
1207 /*ARGSUSED*/
1208 void
i_ddi_prot_io_swap_put32(ddi_acc_impl_t * hdlp,uint32_t * addr,uint32_t value)1209 i_ddi_prot_io_swap_put32(ddi_acc_impl_t *hdlp, uint32_t *addr, uint32_t value)
1210 {
1211 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1212 	outl((uintptr_t)addr, ddi_swap32(value));
1213 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1214 }
1215 
1216 /*ARGSUSED*/
1217 void
i_ddi_prot_vaddr_swap_put64(ddi_acc_impl_t * hdlp,uint64_t * addr,uint64_t value)1218 i_ddi_prot_vaddr_swap_put64(ddi_acc_impl_t *hdlp, uint64_t *addr,
1219     uint64_t value)
1220 {
1221 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1222 	*addr = ddi_swap64(value);
1223 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1224 }
1225 
1226 /*ARGSUSED*/
1227 void
i_ddi_prot_io_rep_get8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)1228 i_ddi_prot_io_rep_get8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
1229     uint8_t *dev_addr, size_t repcount, uint_t flags)
1230 {
1231 	int fail = 0;
1232 	uint8_t	*h;
1233 	uintptr_t port;
1234 
1235 	h = host_addr;
1236 	port = (uintptr_t)dev_addr;
1237 
1238 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1239 	if (flags == DDI_DEV_AUTOINCR) {
1240 		for (; repcount; repcount--, port++)
1241 			if ((*h++ = inb(port)) == 0xff)
1242 				fail = 1;
1243 	} else {
1244 		for (; repcount; repcount--)
1245 			if ((*h++ = inb(port)) == 0xff)
1246 				fail = 1;
1247 	}
1248 	if (fail == 1)
1249 		do_scan(hdlp);
1250 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1251 }
1252 
1253 /*ARGSUSED*/
1254 void
i_ddi_prot_io_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1255 i_ddi_prot_io_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1256     uint16_t *dev_addr, size_t repcount, uint_t flags)
1257 {
1258 	int fail = 0;
1259 	uint16_t *h;
1260 	uintptr_t port;
1261 
1262 	h = host_addr;
1263 	port = (uintptr_t)dev_addr;
1264 
1265 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1266 	if (flags == DDI_DEV_AUTOINCR) {
1267 		for (; repcount; repcount--, port += 2)
1268 			if ((*h++ = inw(port)) == 0xffff)
1269 				fail = 1;
1270 	} else {
1271 		for (; repcount; repcount--)
1272 			if ((*h++ = inw(port)) == 0xffff)
1273 				fail = 1;
1274 	}
1275 	if (fail == 1)
1276 		do_scan(hdlp);
1277 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1278 }
1279 
1280 /*ARGSUSED*/
1281 void
i_ddi_prot_io_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1282 i_ddi_prot_io_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1283     uint32_t *dev_addr, size_t repcount, uint_t flags)
1284 {
1285 	int fail = 0;
1286 	uint32_t *h;
1287 	uintptr_t port;
1288 
1289 	h = host_addr;
1290 	port = (uintptr_t)dev_addr;
1291 
1292 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1293 	if (flags == DDI_DEV_AUTOINCR) {
1294 		for (; repcount; repcount--, port += 4)
1295 			if ((*h++ = inl(port)) == 0xffffffff)
1296 				fail = 1;
1297 	} else {
1298 		for (; repcount; repcount--)
1299 			if ((*h++ = inl(port)) == 0xffffffff)
1300 				fail = 1;
1301 	}
1302 	if (fail == 1)
1303 		do_scan(hdlp);
1304 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1305 }
1306 
1307 /*ARGSUSED*/
1308 void
i_ddi_prot_vaddr_rep_get8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)1309 i_ddi_prot_vaddr_rep_get8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
1310     uint8_t *dev_addr, size_t repcount, uint_t flags)
1311 {
1312 	int fail = 0;
1313 	uint8_t	*h, *d;
1314 
1315 	h = host_addr;
1316 	d = dev_addr;
1317 
1318 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1319 	if (flags == DDI_DEV_AUTOINCR) {
1320 		for (; repcount; repcount--)
1321 			if ((*h++ = *d++) == 0xff)
1322 				fail = 1;
1323 	} else {
1324 		for (; repcount; repcount--)
1325 			if ((*h++ = *d) == 0xff)
1326 				fail = 1;
1327 	}
1328 	if (fail == 1)
1329 		do_scan(hdlp);
1330 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1331 }
1332 
1333 /*ARGSUSED*/
1334 void
i_ddi_prot_vaddr_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1335 i_ddi_prot_vaddr_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1336     uint16_t *dev_addr, size_t repcount, uint_t flags)
1337 {
1338 	int fail = 0;
1339 	uint16_t *h, *d;
1340 
1341 	h = host_addr;
1342 	d = dev_addr;
1343 
1344 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1345 	if (flags == DDI_DEV_AUTOINCR) {
1346 		for (; repcount; repcount--)
1347 			if ((*h++ = *d++) == 0xffff)
1348 				fail = 1;
1349 	} else {
1350 		for (; repcount; repcount--)
1351 			if ((*h++ = *d) == 0xffff)
1352 				fail = 1;
1353 	}
1354 	if (fail == 1)
1355 		do_scan(hdlp);
1356 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1357 }
1358 
1359 /*ARGSUSED*/
1360 void
i_ddi_prot_vaddr_swap_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1361 i_ddi_prot_vaddr_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1362     uint16_t *dev_addr, size_t repcount, uint_t flags)
1363 {
1364 	int fail = 0;
1365 	uint16_t *h, *d;
1366 
1367 	h = host_addr;
1368 	d = dev_addr;
1369 
1370 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1371 	if (flags == DDI_DEV_AUTOINCR) {
1372 		for (; repcount; repcount--)
1373 			if ((*h++ = ddi_swap16(*d++)) == 0xffff)
1374 				fail = 1;
1375 	} else {
1376 		for (; repcount; repcount--)
1377 			if ((*h++ = ddi_swap16(*d)) == 0xffff)
1378 				fail = 1;
1379 	}
1380 	if (fail == 1)
1381 		do_scan(hdlp);
1382 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1383 }
1384 
1385 /*ARGSUSED*/
1386 void
i_ddi_prot_io_swap_rep_get16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1387 i_ddi_prot_io_swap_rep_get16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1388     uint16_t *dev_addr, size_t repcount, uint_t flags)
1389 {
1390 	int fail = 0;
1391 	uint16_t *h;
1392 	uintptr_t port;
1393 
1394 	h = host_addr;
1395 	port = (uintptr_t)dev_addr;
1396 
1397 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1398 	if (flags == DDI_DEV_AUTOINCR) {
1399 		for (; repcount; repcount--, port += 2)
1400 			if ((*h++ = ddi_swap16(inw(port))) == 0xffff)
1401 				fail = 1;
1402 	} else {
1403 		for (; repcount; repcount--)
1404 			if ((*h++ = ddi_swap16(inw(port))) == 0xffff)
1405 				fail = 1;
1406 	}
1407 	if (fail == 1)
1408 		do_scan(hdlp);
1409 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1410 }
1411 
1412 /*ARGSUSED*/
1413 void
i_ddi_prot_vaddr_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1414 i_ddi_prot_vaddr_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1415     uint32_t *dev_addr, size_t repcount, uint_t flags)
1416 {
1417 	int fail = 0;
1418 	uint32_t *h, *d;
1419 
1420 	h = host_addr;
1421 	d = dev_addr;
1422 
1423 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1424 	if (flags == DDI_DEV_AUTOINCR) {
1425 		for (; repcount; repcount--)
1426 			if ((*h++ = *d++) == 0xffffffff)
1427 				fail = 1;
1428 	} else {
1429 		for (; repcount; repcount--)
1430 			if ((*h++ = *d) == 0xffffffff)
1431 				fail = 1;
1432 	}
1433 	if (fail == 1)
1434 		do_scan(hdlp);
1435 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1436 }
1437 
1438 /*ARGSUSED*/
1439 void
i_ddi_prot_vaddr_swap_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1440 i_ddi_prot_vaddr_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1441     uint32_t *dev_addr, size_t repcount, uint_t flags)
1442 {
1443 	int fail = 0;
1444 	uint32_t *h, *d;
1445 
1446 	h = host_addr;
1447 	d = dev_addr;
1448 
1449 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1450 	if (flags == DDI_DEV_AUTOINCR) {
1451 		for (; repcount; repcount--)
1452 			if ((*h++ = ddi_swap32(*d++)) == 0xffffffff)
1453 				fail = 1;
1454 	} else {
1455 		for (; repcount; repcount--)
1456 			if ((*h++ = ddi_swap32(*d)) == 0xffffffff)
1457 				fail = 1;
1458 	}
1459 	if (fail == 1)
1460 		do_scan(hdlp);
1461 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1462 }
1463 
1464 /*ARGSUSED*/
1465 void
i_ddi_prot_io_swap_rep_get32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1466 i_ddi_prot_io_swap_rep_get32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1467     uint32_t *dev_addr, size_t repcount, uint_t flags)
1468 {
1469 	int fail = 0;
1470 	uint32_t *h;
1471 	uintptr_t port;
1472 
1473 	h = host_addr;
1474 	port = (uintptr_t)dev_addr;
1475 
1476 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1477 	if (flags == DDI_DEV_AUTOINCR) {
1478 		for (; repcount; repcount--, port += 4)
1479 			if ((*h++ = ddi_swap32(inl(port))) == 0xffffffff)
1480 				fail = 1;
1481 	} else {
1482 		for (; repcount; repcount--)
1483 			if ((*h++ = ddi_swap32(inl(port))) == 0xffffffff)
1484 				fail = 1;
1485 	}
1486 	if (fail == 1)
1487 		do_scan(hdlp);
1488 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1489 }
1490 
1491 /*ARGSUSED*/
1492 void
i_ddi_prot_vaddr_rep_get64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1493 i_ddi_prot_vaddr_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1494     uint64_t *dev_addr, size_t repcount, uint_t flags)
1495 {
1496 	int fail = 0;
1497 	uint64_t *h, *d;
1498 
1499 	h = host_addr;
1500 	d = dev_addr;
1501 
1502 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1503 	if (flags == DDI_DEV_AUTOINCR) {
1504 		for (; repcount; repcount--)
1505 			if ((*h++ = *d++) == 0xffffffffffffffff)
1506 				fail = 1;
1507 	} else {
1508 		for (; repcount; repcount--)
1509 			if ((*h++ = *d) == 0xffffffffffffffff)
1510 				fail = 1;
1511 	}
1512 	if (fail == 1)
1513 		do_scan(hdlp);
1514 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1515 }
1516 
1517 /*ARGSUSED*/
1518 void
i_ddi_prot_vaddr_swap_rep_get64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1519 i_ddi_prot_vaddr_swap_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1520     uint64_t *dev_addr, size_t repcount, uint_t flags)
1521 {
1522 	int fail = 0;
1523 	uint64_t *h, *d;
1524 
1525 	h = host_addr;
1526 	d = dev_addr;
1527 
1528 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1529 	if (flags == DDI_DEV_AUTOINCR) {
1530 		for (; repcount; repcount--)
1531 			if ((*h++ = ddi_swap64(*d++)) == 0xffffffffffffffff)
1532 				fail = 1;
1533 	} else {
1534 		for (; repcount; repcount--)
1535 			if ((*h++ = ddi_swap64(*d)) == 0xffffffffffffffff)
1536 				fail = 1;
1537 	}
1538 	if (fail == 1)
1539 		do_scan(hdlp);
1540 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1541 }
1542 
1543 /*ARGSUSED*/
1544 void
i_ddi_prot_vaddr_rep_put8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)1545 i_ddi_prot_vaddr_rep_put8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
1546     uint8_t *dev_addr, size_t repcount, uint_t flags)
1547 {
1548 	uint8_t	*h, *d;
1549 
1550 	h = host_addr;
1551 	d = dev_addr;
1552 
1553 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1554 	if (flags == DDI_DEV_AUTOINCR)
1555 		for (; repcount; repcount--)
1556 			*d++ = *h++;
1557 	else
1558 		for (; repcount; repcount--)
1559 			*d = *h++;
1560 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1561 }
1562 
1563 /*ARGSUSED*/
1564 void
i_ddi_prot_io_rep_put8(ddi_acc_impl_t * hdlp,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount,uint_t flags)1565 i_ddi_prot_io_rep_put8(ddi_acc_impl_t *hdlp, uint8_t *host_addr,
1566     uint8_t *dev_addr, size_t repcount, uint_t flags)
1567 {
1568 	uint8_t	*h;
1569 	uintptr_t port;
1570 
1571 	h = host_addr;
1572 	port = (uintptr_t)dev_addr;
1573 
1574 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1575 	if (flags == DDI_DEV_AUTOINCR)
1576 		for (; repcount; repcount--, port++)
1577 			outb(port, *h++);
1578 	else
1579 		for (; repcount; repcount--)
1580 			outb(port, *h++);
1581 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1582 }
1583 
1584 /*ARGSUSED*/
1585 void
i_ddi_prot_vaddr_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1586 i_ddi_prot_vaddr_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1587     uint16_t *dev_addr, size_t repcount, uint_t flags)
1588 {
1589 	uint16_t *h, *d;
1590 
1591 	h = host_addr;
1592 	d = dev_addr;
1593 
1594 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1595 	if (flags == DDI_DEV_AUTOINCR)
1596 		for (; repcount; repcount--)
1597 			*d++ = *h++;
1598 	else
1599 		for (; repcount; repcount--)
1600 			*d = *h++;
1601 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1602 }
1603 
1604 /*ARGSUSED*/
1605 void
i_ddi_prot_io_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1606 i_ddi_prot_io_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1607     uint16_t *dev_addr, size_t repcount, uint_t flags)
1608 {
1609 	uint16_t *h;
1610 	uintptr_t port;
1611 
1612 	h = host_addr;
1613 	port = (uintptr_t)dev_addr;
1614 
1615 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1616 	if (flags == DDI_DEV_AUTOINCR)
1617 		for (; repcount; repcount--, port += 2)
1618 			outw(port, *h++);
1619 	else
1620 		for (; repcount; repcount--)
1621 			outw(port, *h++);
1622 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1623 }
1624 
1625 /*ARGSUSED*/
1626 void
i_ddi_prot_vaddr_swap_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1627 i_ddi_prot_vaddr_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1628     uint16_t *dev_addr, size_t repcount, uint_t flags)
1629 {
1630 	uint16_t *h, *d;
1631 
1632 	h = host_addr;
1633 	d = dev_addr;
1634 
1635 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1636 	if (flags == DDI_DEV_AUTOINCR)
1637 		for (; repcount; repcount--)
1638 			*d++ = ddi_swap16(*h++);
1639 	else
1640 		for (; repcount; repcount--)
1641 			*d = ddi_swap16(*h++);
1642 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1643 }
1644 
1645 /*ARGSUSED*/
1646 void
i_ddi_prot_io_swap_rep_put16(ddi_acc_impl_t * hdlp,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount,uint_t flags)1647 i_ddi_prot_io_swap_rep_put16(ddi_acc_impl_t *hdlp, uint16_t *host_addr,
1648     uint16_t *dev_addr, size_t repcount, uint_t flags)
1649 {
1650 	uint16_t *h;
1651 	uintptr_t port;
1652 
1653 	h = host_addr;
1654 	port = (uintptr_t)dev_addr;
1655 
1656 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1657 	if (flags == DDI_DEV_AUTOINCR)
1658 		for (; repcount; repcount--, port += 2)
1659 			outw(port, ddi_swap16(*h++));
1660 	else
1661 		for (; repcount; repcount--)
1662 			outw(port, ddi_swap16(*h++));
1663 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1664 }
1665 
1666 /*ARGSUSED*/
1667 void
i_ddi_prot_vaddr_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1668 i_ddi_prot_vaddr_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1669     uint32_t *dev_addr, size_t repcount, uint_t flags)
1670 {
1671 	uint32_t *h, *d;
1672 
1673 	h = host_addr;
1674 	d = dev_addr;
1675 
1676 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1677 	if (flags == DDI_DEV_AUTOINCR)
1678 		for (; repcount; repcount--)
1679 			*d++ = *h++;
1680 	else
1681 		for (; repcount; repcount--)
1682 			*d = *h++;
1683 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1684 }
1685 
1686 /*ARGSUSED*/
1687 void
i_ddi_prot_io_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1688 i_ddi_prot_io_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1689     uint32_t *dev_addr, size_t repcount, uint_t flags)
1690 {
1691 	uint32_t *h;
1692 	uintptr_t port;
1693 
1694 	h = host_addr;
1695 	port = (uintptr_t)dev_addr;
1696 
1697 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1698 	if (flags == DDI_DEV_AUTOINCR)
1699 		for (; repcount; repcount--, port += 4)
1700 			outl(port, *h++);
1701 	else
1702 		for (; repcount; repcount--)
1703 			outl(port, *h++);
1704 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1705 }
1706 
1707 /*ARGSUSED*/
1708 void
i_ddi_prot_vaddr_swap_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1709 i_ddi_prot_vaddr_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1710     uint32_t *dev_addr, size_t repcount, uint_t flags)
1711 {
1712 	uint32_t *h, *d;
1713 
1714 	h = host_addr;
1715 	d = dev_addr;
1716 
1717 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1718 	if (flags == DDI_DEV_AUTOINCR)
1719 		for (; repcount; repcount--)
1720 			*d++ = ddi_swap32(*h++);
1721 	else
1722 		for (; repcount; repcount--)
1723 			*d = ddi_swap32(*h++);
1724 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1725 }
1726 
1727 /*ARGSUSED*/
1728 void
i_ddi_prot_io_swap_rep_put32(ddi_acc_impl_t * hdlp,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount,uint_t flags)1729 i_ddi_prot_io_swap_rep_put32(ddi_acc_impl_t *hdlp, uint32_t *host_addr,
1730     uint32_t *dev_addr, size_t repcount, uint_t flags)
1731 {
1732 	uint32_t *h;
1733 	uintptr_t port;
1734 
1735 	h = host_addr;
1736 	port = (uintptr_t)dev_addr;
1737 
1738 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1739 	if (flags == DDI_DEV_AUTOINCR)
1740 		for (; repcount; repcount--, port += 4)
1741 			outl(port, ddi_swap32(*h++));
1742 	else
1743 		for (; repcount; repcount--)
1744 			outl(port, ddi_swap32(*h++));
1745 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1746 }
1747 
1748 /*ARGSUSED*/
1749 void
i_ddi_prot_vaddr_rep_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1750 i_ddi_prot_vaddr_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1751     uint64_t *dev_addr, size_t repcount, uint_t flags)
1752 {
1753 	uint64_t *h, *d;
1754 
1755 	h = host_addr;
1756 	d = dev_addr;
1757 
1758 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1759 	if (flags == DDI_DEV_AUTOINCR)
1760 		for (; repcount; repcount--)
1761 			*d++ = *h++;
1762 	else
1763 		for (; repcount; repcount--)
1764 			*d = *h++;
1765 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1766 }
1767 
1768 /*ARGSUSED*/
1769 void
i_ddi_prot_vaddr_swap_rep_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1770 i_ddi_prot_vaddr_swap_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1771     uint64_t *dev_addr, size_t repcount, uint_t flags)
1772 {
1773 	uint64_t *h, *d;
1774 
1775 	h = host_addr;
1776 	d = dev_addr;
1777 
1778 	mutex_enter(hdlp->ahi_peekpoke_mutexp);
1779 	if (flags == DDI_DEV_AUTOINCR)
1780 		for (; repcount; repcount--)
1781 			*d++ = ddi_swap64(*h++);
1782 	else
1783 		for (; repcount; repcount--)
1784 			*d = ddi_swap64(*h++);
1785 	mutex_exit(hdlp->ahi_peekpoke_mutexp);
1786 }
1787 
1788 void
ddi_io_rep_get8(ddi_acc_handle_t handle,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount)1789 ddi_io_rep_get8(ddi_acc_handle_t handle,
1790     uint8_t *host_addr, uint8_t *dev_addr, size_t repcount)
1791 {
1792 	(((ddi_acc_impl_t *)handle)->ahi_rep_get8)
1793 	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1794 	    repcount, DDI_DEV_NO_AUTOINCR);
1795 }
1796 
1797 void
ddi_io_rep_get16(ddi_acc_handle_t handle,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount)1798 ddi_io_rep_get16(ddi_acc_handle_t handle,
1799     uint16_t *host_addr, uint16_t *dev_addr, size_t repcount)
1800 {
1801 	(((ddi_acc_impl_t *)handle)->ahi_rep_get16)
1802 	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1803 	    repcount, DDI_DEV_NO_AUTOINCR);
1804 }
1805 
1806 void
ddi_io_rep_get32(ddi_acc_handle_t handle,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount)1807 ddi_io_rep_get32(ddi_acc_handle_t handle,
1808     uint32_t *host_addr, uint32_t *dev_addr, size_t repcount)
1809 {
1810 	(((ddi_acc_impl_t *)handle)->ahi_rep_get32)
1811 	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1812 	    repcount, DDI_DEV_NO_AUTOINCR);
1813 }
1814 
1815 /*ARGSUSED*/
1816 void
i_ddi_io_rep_get64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1817 i_ddi_io_rep_get64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1818     uint64_t *dev_addr, size_t repcount, uint_t flags)
1819 {
1820 	cmn_err(CE_PANIC, "ddi_rep_get64 from i/o space");
1821 }
1822 
1823 void
ddi_io_rep_put8(ddi_acc_handle_t handle,uint8_t * host_addr,uint8_t * dev_addr,size_t repcount)1824 ddi_io_rep_put8(ddi_acc_handle_t handle,
1825     uint8_t *host_addr, uint8_t *dev_addr, size_t repcount)
1826 {
1827 	(((ddi_acc_impl_t *)handle)->ahi_rep_put8)
1828 	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1829 	    repcount, DDI_DEV_NO_AUTOINCR);
1830 }
1831 
1832 void
ddi_io_rep_put16(ddi_acc_handle_t handle,uint16_t * host_addr,uint16_t * dev_addr,size_t repcount)1833 ddi_io_rep_put16(ddi_acc_handle_t handle,
1834     uint16_t *host_addr, uint16_t *dev_addr, size_t repcount)
1835 {
1836 	(((ddi_acc_impl_t *)handle)->ahi_rep_put16)
1837 	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1838 	    repcount, DDI_DEV_NO_AUTOINCR);
1839 }
1840 
1841 void
ddi_io_rep_put32(ddi_acc_handle_t handle,uint32_t * host_addr,uint32_t * dev_addr,size_t repcount)1842 ddi_io_rep_put32(ddi_acc_handle_t handle,
1843     uint32_t *host_addr, uint32_t *dev_addr, size_t repcount)
1844 {
1845 	(((ddi_acc_impl_t *)handle)->ahi_rep_put32)
1846 	    ((ddi_acc_impl_t *)handle, host_addr, dev_addr,
1847 	    repcount, DDI_DEV_NO_AUTOINCR);
1848 }
1849 
1850 /*ARGSUSED*/
1851 void
i_ddi_io_rep_put64(ddi_acc_impl_t * hdlp,uint64_t * host_addr,uint64_t * dev_addr,size_t repcount,uint_t flags)1852 i_ddi_io_rep_put64(ddi_acc_impl_t *hdlp, uint64_t *host_addr,
1853     uint64_t *dev_addr, size_t repcount, uint_t flags)
1854 {
1855 	cmn_err(CE_PANIC, "ddi_rep_put64 to i/o space");
1856 }
1857 
1858 /*
1859  * These next two functions could be translated into assembler someday
1860  */
1861 int
ddi_check_acc_handle(ddi_acc_handle_t handle)1862 ddi_check_acc_handle(ddi_acc_handle_t handle)
1863 {
1864 	ddi_acc_impl_t *hdlp = (ddi_acc_impl_t *)handle;
1865 	return (((*hdlp->ahi_fault_check)(hdlp) == DDI_SUCCESS) ? DDI_SUCCESS :
1866 	    DDI_FAILURE);
1867 }
1868 
1869 int
i_ddi_acc_fault_check(ddi_acc_impl_t * hdlp)1870 i_ddi_acc_fault_check(ddi_acc_impl_t *hdlp)
1871 {
1872 	/* Default version, just returns flag value */
1873 	return (hdlp->ahi_fault);
1874 }
1875 
1876 /*ARGSUSED*/
1877 void
i_ddi_acc_fault_notify(ddi_acc_impl_t * hdlp)1878 i_ddi_acc_fault_notify(ddi_acc_impl_t *hdlp)
1879 {
1880 	/* Default version, does nothing for now */
1881 }
1882 
1883 void
i_ddi_acc_set_fault(ddi_acc_handle_t handle)1884 i_ddi_acc_set_fault(ddi_acc_handle_t handle)
1885 {
1886 	ddi_acc_impl_t *hdlp = (ddi_acc_impl_t *)handle;
1887 
1888 	if (!hdlp->ahi_fault) {
1889 		hdlp->ahi_fault = 1;
1890 		(*hdlp->ahi_fault_notify)(hdlp);
1891 	}
1892 }
1893 
1894 void
i_ddi_acc_clr_fault(ddi_acc_handle_t handle)1895 i_ddi_acc_clr_fault(ddi_acc_handle_t handle)
1896 {
1897 	ddi_acc_impl_t *hdlp = (ddi_acc_impl_t *)handle;
1898 
1899 	if (hdlp->ahi_fault) {
1900 		hdlp->ahi_fault = 0;
1901 		(*hdlp->ahi_fault_notify)(hdlp);
1902 	}
1903 }
1904