1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26#include <sys/asi.h>
27#include <sys/asm_linkage.h>
28#include <sys/machthread.h>
29#include <sys/privregs.h>
30#include <sys/ontrap.h>
31#include <sys/dditypes.h>
32
33#include "assym.h"
34
35/*
36 * This file implements the following ddi common access
37 * functions:
38 *
39 *	ddi_get{8,16,32,64}
40 *	ddi_put{8,16,32,64}
41 *
42 * and the underlying "trivial" implementations
43 *
44 *      i_ddi_{get,put}{8,16,32,64}
45 *
46 * which assume that there is no need to check the access handle -
47 * byte swapping will be done by the mmu and the address is always
48 * accessible via ld/st instructions.
49 */
50
51/*
52 * The functionality of each of the ddi_get/put routines is performed by
53 * the respective indirect function defined in the access handle.  Use of
54 * the access handle functions provides compatibility across platforms for
55 * drivers.
56 *
57 * By default, the indirect access handle functions are initialized to the
58 * i_ddi_get/put routines to perform memory mapped IO.  If memory mapped IO
59 * is not possible or desired, the access handle must be intialized to another
60 * valid routine to perform the sepcified IO operation.
61 *
62 * The alignment and placement of the following functions have been optimized
63 * such that the implementation specific versions, i_ddi*, fall within the
64 * same cache-line of the generic versions, ddi_*.  This insures that an
65 * I-cache hit will occur thus minimizing the performance impact of using the
66 * access handle.
67 */
68
69	.align 32
70	ENTRY(ddi_get8)
71	ALTENTRY(ddi_getb)
72	ALTENTRY(ddi_io_get8)
73	ALTENTRY(ddi_io_getb)
74	ALTENTRY(ddi_mem_get8)
75	ALTENTRY(ddi_mem_getb)
76	ldn      [%o0 + AHI_GET8], %g1   /* hdl->ahi_get8 access hndl */
77	jmpl    %g1, %g0                 /* jump to access handle routine */
78	nop
79	SET_SIZE(ddi_get8)
80	SET_SIZE(ddi_getb)
81	SET_SIZE(ddi_io_get8)
82	SET_SIZE(ddi_io_getb)
83	SET_SIZE(ddi_mem_get8)
84	SET_SIZE(ddi_mem_getb)
85
86	.align 16
87	ENTRY(i_ddi_get8)
88	retl
89	ldub	[%o1], %o0
90	SET_SIZE(i_ddi_get8)
91
92	.align 32
93	ENTRY(ddi_get16)
94	ALTENTRY(ddi_getw)
95	ALTENTRY(ddi_io_get16)
96	ALTENTRY(ddi_io_getw)
97	ALTENTRY(ddi_mem_get16)
98	ALTENTRY(ddi_mem_getw)
99	ldn      [%o0 + AHI_GET16], %g1   /* hdl->ahi_get16 access hndl */
100	jmpl    %g1, %g0                  /* jump to access handle routine */
101	nop
102	SET_SIZE(ddi_get16)
103	SET_SIZE(ddi_getw)
104	SET_SIZE(ddi_io_get16)
105	SET_SIZE(ddi_io_getw)
106	SET_SIZE(ddi_mem_get16)
107	SET_SIZE(ddi_mem_getw)
108
109	.align 16
110	ENTRY(i_ddi_get16)
111	ALTENTRY(i_ddi_swap_get16)
112	retl
113	lduh	[%o1], %o0
114	SET_SIZE(i_ddi_get16)
115	SET_SIZE(i_ddi_swap_get16)
116
117	.align 32
118	ENTRY(ddi_get32)
119	ALTENTRY(ddi_getl)
120	ALTENTRY(ddi_io_get32)
121	ALTENTRY(ddi_io_getl)
122	ALTENTRY(ddi_mem_get32)
123	ALTENTRY(ddi_mem_getl)
124	ldn      [%o0 + AHI_GET32], %g1   /* hdl->ahi_get32 access handle */
125	jmpl    %g1, %g0		  /* jump to access handle routine */
126	nop
127	SET_SIZE(ddi_get32)
128	SET_SIZE(ddi_getl)
129	SET_SIZE(ddi_io_get32)
130	SET_SIZE(ddi_io_getl)
131	SET_SIZE(ddi_mem_get32)
132	SET_SIZE(ddi_mem_getl)
133
134	.align 16
135	ENTRY(i_ddi_get32)
136	ALTENTRY(i_ddi_swap_get32)
137	retl
138	ld	[%o1], %o0
139	SET_SIZE(i_ddi_get32)
140	SET_SIZE(i_ddi_swap_get32)
141
142	.align 32
143	ENTRY(ddi_get64)
144	ALTENTRY(ddi_getll)
145	ALTENTRY(ddi_io_get64)
146	ALTENTRY(ddi_io_getll)
147	ALTENTRY(ddi_mem_get64)
148	ALTENTRY(ddi_mem_getll)
149	ldn      [%o0 + AHI_GET64], %g1   /* hdl->ahi_get64 access handle */
150	jmpl    %g1, %g0                  /* jump to access handle routine */
151	nop
152	SET_SIZE(ddi_get64)
153	SET_SIZE(ddi_getll)
154	SET_SIZE(ddi_io_get64)
155	SET_SIZE(ddi_io_getll)
156	SET_SIZE(ddi_mem_get64)
157	SET_SIZE(ddi_mem_getll)
158
159	.align 16
160	ENTRY(i_ddi_get64)
161	ALTENTRY(i_ddi_swap_get64)
162	retl
163	ldx	[%o1], %o0
164	SET_SIZE(i_ddi_get64)
165	SET_SIZE(i_ddi_swap_get64)
166
167	.align 32
168	ENTRY(ddi_put8)
169	ALTENTRY(ddi_putb)
170	ALTENTRY(ddi_io_put8)
171	ALTENTRY(ddi_io_putb)
172	ALTENTRY(ddi_mem_put8)
173	ALTENTRY(ddi_mem_putb)
174	ldn      [%o0 + AHI_PUT8], %g1   /* hdl->ahi_put8 access handle */
175	jmpl    %g1, %g0                 /* jump to access handle routine */
176	nop
177	SET_SIZE(ddi_put8)
178	SET_SIZE(ddi_putb)
179	SET_SIZE(ddi_io_put8)
180	SET_SIZE(ddi_io_putb)
181	SET_SIZE(ddi_mem_put8)
182	SET_SIZE(ddi_mem_putb)
183
184	.align 16
185	ENTRY(i_ddi_put8)
186	retl
187	stub	%o2, [%o1]
188	SET_SIZE(i_ddi_put8)
189
190	.align 32
191	ENTRY(ddi_put16)
192	ALTENTRY(ddi_putw)
193	ALTENTRY(ddi_io_put16)
194	ALTENTRY(ddi_io_putw)
195	ALTENTRY(ddi_mem_put16)
196	ALTENTRY(ddi_mem_putw)
197	ldn      [%o0 + AHI_PUT16], %g1   /* hdl->ahi_put16 access handle */
198	jmpl    %g1, %g0                  /* jump to access handle routine */
199	nop
200	SET_SIZE(ddi_put16)
201	SET_SIZE(ddi_putw)
202	SET_SIZE(ddi_io_put16)
203	SET_SIZE(ddi_io_putw)
204	SET_SIZE(ddi_mem_put16)
205	SET_SIZE(ddi_mem_putw)
206
207	.align 16
208	ENTRY(i_ddi_put16)
209	ALTENTRY(i_ddi_swap_put16)
210	retl
211	stuh	%o2, [%o1]
212	SET_SIZE(i_ddi_put16)
213	SET_SIZE(i_ddi_swap_put16)
214
215	.align 32
216	ENTRY(ddi_put32)
217	ALTENTRY(ddi_putl)
218	ALTENTRY(ddi_io_put32)
219	ALTENTRY(ddi_io_putl)
220	ALTENTRY(ddi_mem_put32)
221	ALTENTRY(ddi_mem_putl)
222	ldn      [%o0 + AHI_PUT32], %g1   /* hdl->ahi_put16 access handle */
223	jmpl    %g1, %g0                  /* jump to access handle routine */
224	nop
225	SET_SIZE(ddi_put32)
226	SET_SIZE(ddi_putl)
227	SET_SIZE(ddi_io_put32)
228	SET_SIZE(ddi_io_putl)
229	SET_SIZE(ddi_mem_put32)
230	SET_SIZE(ddi_mem_putl)
231
232	.align 16
233	ENTRY(i_ddi_put32)
234	ALTENTRY(i_ddi_swap_put32)
235	retl
236	st	%o2, [%o1]
237	SET_SIZE(i_ddi_put32)
238	SET_SIZE(i_ddi_swap_put32)
239
240	.align 32
241	ENTRY(ddi_put64)
242	ALTENTRY(ddi_putll)
243	ALTENTRY(ddi_io_put64)
244	ALTENTRY(ddi_io_putll)
245	ALTENTRY(ddi_mem_put64)
246	ALTENTRY(ddi_mem_putll)
247	ldn      [%o0 + AHI_PUT64], %g1   /* hdl->ahi_put64 access handle */
248	jmpl    %g1, %g0                  /* jump to access handle routine */
249	nop
250	SET_SIZE(ddi_put64)
251	SET_SIZE(ddi_putll)
252	SET_SIZE(ddi_io_put64)
253	SET_SIZE(ddi_io_putll)
254	SET_SIZE(ddi_mem_put64)
255	SET_SIZE(ddi_mem_putll)
256
257	.align 16
258	ENTRY(i_ddi_put64)
259	ALTENTRY(i_ddi_swap_put64)
260	retl
261	stx	%o2, [%o1]
262	SET_SIZE(i_ddi_put64)
263	SET_SIZE(i_ddi_swap_put64)
264
265/*
266 * The ddi_io_rep_get/put routines don't take a flag argument like the "plain"
267 * and mem versions do.  This flag is used to determine whether or not the
268 * device address or port should be automatically incremented.  For IO space,
269 * the device port is never incremented and as such, the flag is always set
270 * to DDI_DEV_NO_AUTOINCR.
271 *
272 * This define processes the repetitive get functionality.  Automatic
273 * incrementing of the device address is determined by the flag field
274 * %o4.  If this is set for AUTOINCR, %o4 is updated with 1 for the
275 * subsequent increment in 2:.
276 *
277 * If this flag is not set for AUTOINCR, %o4 is update with a value of 0 thus
278 * making the increment operation a non-operation.
279 */
280
281#define DDI_REP_GET(n,s)			\
282	cmp	DDI_DEV_NO_AUTOINCR, %o4;	\
283	mov	%g0, %o4;			\
284	brz,pn	%o3, 1f;			\
285	movnz	%xcc, n, %o4;			\
2862:						\
287	dec	%o3;				\
288	ld/**/s	[%o2], %g4;			\
289	add	%o2, %o4, %o2;			\
290	st/**/s	%g4, [%o1];			\
291	brnz,pt	%o3, 2b;			\
292	add	%o1, n, %o1;			\
2931:
294
295	.align 32
296	ENTRY(ddi_rep_get8)
297	ALTENTRY(ddi_rep_getb)
298	ALTENTRY(ddi_mem_rep_get8)
299	ALTENTRY(ddi_mem_rep_getb)
300	ldn      [%o0 + AHI_REP_GET8], %g1
301	jmpl    %g1, %g0
302	nop
303	SET_SIZE(ddi_rep_get8)
304	SET_SIZE(ddi_rep_getb)
305	SET_SIZE(ddi_mem_rep_get8)
306	SET_SIZE(ddi_mem_rep_getb)
307
308	.align 16
309	ENTRY(i_ddi_rep_get8)
310	DDI_REP_GET(1,ub)
311	retl
312	nop
313	SET_SIZE(i_ddi_rep_get8)
314
315	.align 32
316	ENTRY(ddi_rep_get16)
317	ALTENTRY(ddi_rep_getw)
318	ALTENTRY(ddi_mem_rep_get16)
319	ALTENTRY(ddi_mem_rep_getw)
320	ldn	[%o0 + AHI_REP_GET16], %g1
321	jmpl    %g1, %g0
322	nop
323	SET_SIZE(ddi_rep_get16)
324	SET_SIZE(ddi_rep_getw)
325	SET_SIZE(ddi_mem_rep_get16)
326	SET_SIZE(ddi_mem_rep_getw)
327
328	.align 16
329	ENTRY(i_ddi_rep_get16)
330	ALTENTRY(i_ddi_swap_rep_get16)
331	DDI_REP_GET(2,uh)
332	retl
333	nop
334	SET_SIZE(i_ddi_rep_get16)
335	SET_SIZE(i_ddi_swap_rep_get16)
336
337	.align 32
338	ENTRY(ddi_rep_get32)
339	ALTENTRY(ddi_rep_getl)
340	ALTENTRY(ddi_mem_rep_get32)
341	ALTENTRY(ddi_mem_rep_getl)
342	ldn      [%o0 + AHI_REP_GET32], %g1
343	jmpl    %g1, %g0
344	nop
345	SET_SIZE(ddi_rep_get32)
346	SET_SIZE(ddi_rep_getl)
347	SET_SIZE(ddi_mem_rep_get32)
348	SET_SIZE(ddi_mem_rep_getl)
349
350	.align 16
351	ENTRY(i_ddi_rep_get32)
352	ALTENTRY(i_ddi_swap_rep_get32)
353	DDI_REP_GET(4,/**/)
354	retl
355	nop
356	SET_SIZE(i_ddi_rep_get32)
357	SET_SIZE(i_ddi_swap_rep_get32)
358
359	.align 32
360	ENTRY(ddi_rep_get64)
361	ALTENTRY(ddi_rep_getll)
362	ALTENTRY(ddi_mem_rep_get64)
363	ALTENTRY(ddi_mem_rep_getll)
364	ldn      [%o0 + AHI_REP_GET64], %g1
365	jmpl    %g1, %g0
366	nop
367	SET_SIZE(ddi_rep_get64)
368	SET_SIZE(ddi_rep_getll)
369	SET_SIZE(ddi_mem_rep_get64)
370	SET_SIZE(ddi_mem_rep_getll)
371
372	.align 16
373	ENTRY(i_ddi_rep_get64)
374	ALTENTRY(i_ddi_swap_rep_get64)
375	DDI_REP_GET(8,x)
376	retl
377	nop
378	SET_SIZE(i_ddi_rep_get64)
379	SET_SIZE(i_ddi_swap_rep_get64)
380
381/*
382 * This define processes the repetitive put functionality.  Automatic
383 * incrementing of the device address is determined by the flag field
384 * %o4.  If this is set for AUTOINCR, %o4 is updated with 1 for the
385 * subsequent increment in 2:.
386 *
387 * If this flag is not set for AUTOINCR, %o4 is update with a value of 0 thus
388 * making the increment operation a non-operation.
389 */
390#define DDI_REP_PUT(n,s)			\
391	cmp	DDI_DEV_NO_AUTOINCR, %o4;	\
392	mov	%g0, %o4;			\
393	brz,pn	%o3, 1f;			\
394	movnz	%xcc, n, %o4;			\
3952:						\
396	dec	%o3;				\
397	ld/**/s	[%o1], %g4;			\
398	add	%o1, n, %o1;			\
399	st/**/s	%g4, [%o2];			\
400	brnz,pt	%o3, 2b;			\
401	add	%o2, %o4, %o2;			\
4021:
403
404	.align 32
405	ENTRY(ddi_rep_put8)
406	ALTENTRY(ddi_rep_putb)
407	ALTENTRY(ddi_mem_rep_put8)
408	ALTENTRY(ddi_mem_rep_putb)
409	ldn      [%o0 + AHI_REP_PUT8], %g1
410	jmpl    %g1, %g0
411	nop
412	SET_SIZE(ddi_rep_put8)
413	SET_SIZE(ddi_rep_putb)
414	SET_SIZE(ddi_mem_rep_put8)
415	SET_SIZE(ddi_mem_rep_putb)
416
417	.align 16
418	ENTRY(i_ddi_rep_put8)
419	DDI_REP_PUT(1,ub)
420	retl
421	nop
422	SET_SIZE(i_ddi_rep_put8)
423
424	.align 32
425	ENTRY(ddi_rep_put16)
426	ALTENTRY(ddi_rep_putw)
427	ALTENTRY(ddi_mem_rep_put16)
428	ALTENTRY(ddi_mem_rep_putw)
429	ldn      [%o0 + AHI_REP_PUT16], %g1
430	jmpl    %g1, %g0
431	nop
432	SET_SIZE(ddi_rep_put16)
433	SET_SIZE(ddi_rep_putw)
434	SET_SIZE(ddi_mem_rep_put16)
435	SET_SIZE(ddi_mem_rep_putw)
436
437	.align 16
438	ENTRY(i_ddi_rep_put16)
439	ALTENTRY(i_ddi_swap_rep_put16)
440	DDI_REP_PUT(2,uh)
441	retl
442	nop
443	SET_SIZE(i_ddi_rep_put16)
444	SET_SIZE(i_ddi_swap_rep_put16)
445
446	.align 32
447	ENTRY(ddi_rep_put32)
448	ALTENTRY(ddi_rep_putl)
449	ALTENTRY(ddi_mem_rep_put32)
450	ALTENTRY(ddi_mem_rep_putl)
451	ldn      [%o0 + AHI_REP_PUT32], %g1
452	jmpl    %g1, %g0
453	nop
454	SET_SIZE(ddi_rep_put32)
455	SET_SIZE(ddi_rep_putl)
456	SET_SIZE(ddi_mem_rep_put32)
457	SET_SIZE(ddi_mem_rep_putl)
458
459	.align 16
460	ENTRY(i_ddi_rep_put32)
461	ALTENTRY(i_ddi_swap_rep_put32)
462	DDI_REP_PUT(4,/**/)
463	retl
464	nop
465	SET_SIZE(i_ddi_rep_put32)
466	SET_SIZE(i_ddi_swap_rep_put32)
467
468	.align 32
469	ENTRY(ddi_rep_put64)
470	ALTENTRY(ddi_rep_putll)
471	ALTENTRY(ddi_mem_rep_put64)
472	ALTENTRY(ddi_mem_rep_putll)
473	ldn      [%o0 + AHI_REP_PUT64], %g1
474	jmpl    %g1, %g0
475	nop
476	SET_SIZE(ddi_rep_put64)
477	SET_SIZE(ddi_rep_putll)
478	SET_SIZE(ddi_mem_rep_put64)
479	SET_SIZE(ddi_mem_rep_putll)
480
481	.align 16
482	ENTRY(i_ddi_rep_put64)
483	ALTENTRY(i_ddi_swap_rep_put64)
484	DDI_REP_PUT(8,x)
485	retl
486	nop
487	SET_SIZE(i_ddi_rep_put64)
488	SET_SIZE(i_ddi_swap_rep_put64)
489
490	.align 16
491	ENTRY(ddi_io_rep_get8)
492	ALTENTRY(ddi_io_rep_getb)
493	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
494	ldn	[%o0 + AHI_REP_GET8], %g1
495	jmpl    %g1, %g0
496	nop
497	SET_SIZE(ddi_io_rep_get8)
498	SET_SIZE(ddi_io_rep_getb)
499
500	.align 16
501	ENTRY(ddi_io_rep_get16)
502	ALTENTRY(ddi_io_rep_getw)
503	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
504	ldn	[%o0 + AHI_REP_GET16], %g1
505	jmpl    %g1, %g0
506	nop
507	SET_SIZE(ddi_io_rep_get16)
508	SET_SIZE(ddi_io_rep_getw)
509
510	.align 16
511	ENTRY(ddi_io_rep_get32)
512	ALTENTRY(ddi_io_rep_getl)
513	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
514	ldn	[%o0 + AHI_REP_GET32], %g1
515	jmpl    %g1, %g0
516	nop
517	SET_SIZE(ddi_io_rep_get32)
518	SET_SIZE(ddi_io_rep_getl)
519
520	.align 16
521	ENTRY(ddi_io_rep_get64)
522	ALTENTRY(ddi_io_rep_getll)
523	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
524	ldn	[%o0 + AHI_REP_GET64], %g1
525	jmpl    %g1, %g0
526	nop
527	SET_SIZE(ddi_io_rep_get64)
528	SET_SIZE(ddi_io_rep_getll)
529
530        .align 64
531	ENTRY(ddi_check_acc_handle)
532	save	%sp, -SA(WINDOWSIZE), %sp	! get a new window
533	ldn	[%i0 + AHI_FAULT_CHECK], %g1
534	jmpl	%g1, %o7
535	mov	%i0, %o0
536	brnz,a,pn %o0, 0f			! if (return_value != 0)
537	mov	-1, %o0				! 	return (DDI_FAILURE)
5380:						! else	return (DDI_SUCCESS)
539	sra	%o0, 0, %i0
540	ret
541	restore
542	SET_SIZE(ddi_check_acc_handle)
543
544        .align 16
545        ENTRY(i_ddi_acc_fault_check)
546	retl
547	ld      [%o0 + AHI_FAULT], %o0
548        SET_SIZE(i_ddi_acc_fault_check)
549
550	.align 16
551	ENTRY(ddi_io_rep_put8)
552	ALTENTRY(ddi_io_rep_putb)
553	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
554	ldn	[%o0 + AHI_REP_PUT8], %g1
555	jmpl    %g1, %g0
556	nop
557	SET_SIZE(ddi_io_rep_put8)
558	SET_SIZE(ddi_io_rep_putb)
559
560	.align 16
561	ENTRY(ddi_io_rep_put16)
562	ALTENTRY(ddi_io_rep_putw)
563	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
564	ldn	[%o0 + AHI_REP_PUT16], %g1
565	jmpl    %g1, %g0
566	nop
567	SET_SIZE(ddi_io_rep_put16)
568	SET_SIZE(ddi_io_rep_putw)
569
570	.align 16
571	ENTRY(ddi_io_rep_put32)
572	ALTENTRY(ddi_io_rep_putl)
573	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
574	ldn	[%o0 + AHI_REP_PUT32], %g1
575	jmpl    %g1, %g0
576	nop
577	SET_SIZE(ddi_io_rep_put32)
578	SET_SIZE(ddi_io_rep_putl)
579
580	.align 16
581	ENTRY(ddi_io_rep_put64)
582	ALTENTRY(ddi_io_rep_putll)
583	set	DDI_DEV_NO_AUTOINCR, %o4 /* Set flag to DDI_DEV_NO_AUTOINCR */
584	ldn	[%o0 + AHI_REP_PUT64], %g1
585	jmpl    %g1, %g0
586	nop
587	SET_SIZE(ddi_io_rep_put64)
588	SET_SIZE(ddi_io_rep_putll)
589
590	ENTRY(do_peek)
591	rdpr	%pstate, %o3	! check ints
592	andcc	%o3, PSTATE_IE, %g0
593	bz,a	done
594	or	%g0, 1, %o0	! Return failure if ints are disabled
595	wrpr	%o3, PSTATE_IE, %pstate
596	cmp	%o0, 8		! 64-bit?
597	bne,a	.peek_int
598	cmp	%o0, 4		! 32-bit?
599	ldx	[%o1], %g1
600	ba	.peekdone
601	stx	%g1, [%o2]
602.peek_int:
603	bne,a	.peek_half
604	cmp	%o0, 2		! 16-bit?
605	lduw	[%o1], %g1
606	ba	.peekdone
607	stuw	%g1, [%o2]
608.peek_half:
609	bne,a	.peek_byte
610	ldub	[%o1], %g1	! 8-bit!
611	lduh	[%o1], %g1
612	ba	.peekdone
613	stuh	%g1, [%o2]
614.peek_byte:
615	stub	%g1, [%o2]
616.peekdone:
617	membar	#Sync		! Make sure the loads take
618	rdpr	%pstate, %o3	! check&enable ints
619	andcc	%o3, PSTATE_IE, %g0
620	bnz	1f
621	nop
622	wrpr	%o3, PSTATE_IE, %pstate
6231:
624	mov	%g0, %o0
625done:
626	retl
627	nop
628	SET_SIZE(do_peek)
629
630	ENTRY(do_poke)
631	cmp	%o0, 8		! 64 bit?
632	bne,a	.poke_int
633	cmp	%o0, 4		! 32-bit?
634	ldx	[%o2], %g1
635	ba	.pokedone
636	stx	%g1, [%o1]
637.poke_int:
638	bne,a	.poke_half
639	cmp	%o0, 2		! 16-bit?
640	lduw	[%o2], %g1
641	ba	.pokedone
642	stuw	%g1, [%o1]
643.poke_half:
644	bne,a	.poke_byte
645	ldub	[%o2], %g1	! 8-bit!
646	lduh	[%o2], %g1
647	ba	.pokedone
648	stuh	%g1, [%o1]
649.poke_byte:
650	stub	%g1, [%o1]
651.pokedone:
652	membar	#Sync
653	retl
654	mov	%g0, %o0
655	SET_SIZE(do_poke)
656
657
658/*
659 * The peek_fault() and poke_fault() routines below are used as on_trap()
660 * trampoline routines.  i_ddi_peek and i_ddi_poke execute do_peek and do_poke
661 * under on_trap protection (see <sys/ontrap.h>), but modify ot_trampoline to
662 * refer to the corresponding routine below.  If a trap occurs, the trap code
663 * will bounce back to the trampoline code, which will effectively cause
664 * do_peek or do_poke to return DDI_FAILURE, instead of longjmp'ing back to
665 * on_trap.  In the case of a peek, we may also need to re-enable interrupts.
666 */
667	.seg	".data"
668.peek_panic:
669	.asciz	"peek_fault: missing or invalid on_trap_data"
670.poke_panic:
671	.asciz	"poke_fault: missing or invalid on_trap_data"
672
673	ENTRY(peek_fault)
674	ldn	[THREAD_REG + T_ONTRAP], %o0	! %o0 = on_trap_data pointer
675	brz,pn	%o0, .peekfail			! if (%o0 == NULL) panic
676	nop
677	lduh	[%o0 + OT_PROT], %o1		! %o1 = %o0->ot_prot
678	andcc	%o1, OT_DATA_ACCESS, %g0	! if (!(%o1 & OT_DATA_ACCESS))
679	bz,pn	%icc, .peekfail			!     panic
680	rdpr	%pstate, %o3
681
682	andcc	%o3, PSTATE_IE, %g0		! enable interrupts
683	bnz	1f
684	nop
685	wrpr	%o3, PSTATE_IE, %pstate
6861:
687	retl
688	sub	%g0, 1, %o0			! return (DDI_FAILURE);
689.peekfail:
690	set	.peek_panic, %o0		! Load panic message
691	call	panic				! Panic if bad t_ontrap data
692	nop
693	SET_SIZE(peek_fault)
694
695
696	ENTRY(poke_fault)
697	ldn	[THREAD_REG + T_ONTRAP], %o0	! %o0 = on_trap_data pointer
698	brz,pn	%o0, .pokefail			! if (%o0 == NULL) panic
699	nop
700	lduh	[%o0 + OT_PROT], %o1		! %o1 = %o0->ot_prot
701	andcc	%o1, OT_DATA_ACCESS, %g0	! if (!(%o1 & OT_DATA_ACCESS))
702	bz,pn	%icc, .pokefail			!     panic
703	nop
704	retl
705	sub	%g0, 1, %o0			! return (DDI_FAILURE);
706.pokefail:
707	set	.poke_panic, %o0		! Load panic message
708	call	panic				! Panic if bad t_ontrap data
709	nop
710	SET_SIZE(poke_fault)
711
712
713/*
714 * IO Fault Services
715 *
716 * Support for protected IO accesses is implemented in the following
717 * functions.  A driver may request one of three protection mechanisms
718 * that enable the system to survive an access errors.  The protection
719 * mechansim is set-up during ddi_regs_map_setup time and may be one of:
720 *
721 *	DDI_DEFAULT_ACC	- no error protection requested.  We will
722 *			use the standard ddi_get/ddi_put operations
723 *			defined above.
724 *
725 *	DDI_FLAGERR - Driver requests that errors encountered will
726 *			be flagged by the system.  The driver is
727 *			responsible for checking the error status
728 *			of the access with a call to ddi_acc_err_get()
729 *			upon return of ddi_get or ddi_put.  To prevent
730 *			an access from causing a system we use internal
731 *			on_trap semantics.
732 *
733 *			The system, depending upon the error,
734 *			may or may not panic.
735 *
736 *	DDI_CAUTIOUS_ACC - Driver expects that the access may cause
737 *			an error to occur.  The system will return
738 *			an error status but will not generate an ereport.
739 *			The system will also ensure synchronous and
740 *			exclusive access to the IO space accessed by
741 *			the caller.
742 *
743 *			To prevent an access from causing a system panic,
744 *			we use on_trap semantics to catch the error and
745 *			set error status.
746 *
747 *	If a read access error is detected and DDI_CAUTIOUS_ACC or
748 *	DDI_FLAGERR_ACC	protection was requested, we will trampoline to the
749 *	error handler, i_ddi_trampoline.  i_ddi_trampoline will:
750 *		- check for proper protection semantics
751 *		- set the error status of the access handle to DDI_FM_NONFATAL
752 *		- re-enable interrupts if neccessary
753 *		- longjmp back to the initiating access function.
754
755 *	If a write access error is detected, an interrupt is typically
756 *	generated and claimed by a bus nexus responsible for the write
757 *	transaction.  The nexus error handler is expected to set the
758 *	error status and the IO initiating driver is expected to check
759 *	for a failed transaction via ddi_fm_acc_err_get().
760 *
761 */
762
763	.seg	".data"
764.acc_panic:
765	.asciz	"DDI access: missing or invalid on_trap_data"
766
767	ENTRY(i_ddi_caut_trampoline)
768	ldn	[THREAD_REG + T_ONTRAP], %o5    ! %o5 = curthread->t_ontrap
769	lduh	[%o5 + OT_PROT], %o1		! %o1 = %o0->ot_prot
770	andcc	%o1, OT_DATA_ACCESS, %g0	! if (!(%o1 & OT_DATA_ACCESS))
771	bz,pn	%icc, .cautaccfail		!     panic
772	rdpr	%pstate, %o3
773	andcc	%o3, PSTATE_IE, %g0		! enable interrupts
774	bnz	1f
775	nop
776	wrpr	%o3, PSTATE_IE, %pstate
7771:
778	ldn	[%o5 + OT_HANDLE], %o0		! %o0 = ot_handle
779	brz,pn	%o0, .cautaccfail		! if (ot_handle == NULL) panic
780	nop
781	ldn	[%o0 + AHI_ERR], %o4		! %o4 = hp->ahi_err
782	membar	#Sync
783	stx	%g0, [%o4 + ERR_ENA]		! ahi_err->err_ena = 0
784	mov	-2, %o0
785	st	%o0, [%o4 + ERR_STATUS]		! ahi_err->err_status = NONFATAL
786	b	longjmp                		! longjmp back
787	add	%o5, OT_JMPBUF, %o0		! %o0 = &ot_jmpbuf
788.cautaccfail:
789	set	.acc_panic, %o0			! Load panic message
790	call	panic				! Panic if bad t_ontrap data
791	nop
792	SET_SIZE(i_ddi_caut_trampoline)
793
794/*
795 * DDI on_trap set-up functions,  i_ddi_ontrap() and i_ddinotrap() are used
796 * to protect * ddi_get accesses for DDI_CAUT_ACC.  i_ddi_ontrap() sets
797 * the jumpbuf (setjmp) that will return back to the access routine from
798 * i_ddi_trampoline().  DDI_NOPROTECT() clears the ontrap set-up.
799 */
800	ENTRY(i_ddi_ontrap)
801	ldn	[%o0 + AHI_ERR], %o4
802	ldn	[%o4 + ERR_ONTRAP],  %o4	! %o4 = hp->ahi_err->err_ontrap
803	ldn	[THREAD_REG + T_ONTRAP], %o5	! %o5 = curthread->t_ontrap
804	stn	%o5, [%o4 + OT_PREV]		! ot_prev = t_ontrap
805	membar	#Sync				! force error barrier
806	stn	%o4, [THREAD_REG + T_ONTRAP]	! t_ontrap = err_ontrap
807	b	setjmp
808	add	%o4, OT_JMPBUF, %o0
809	SET_SIZE(i_ddi_ontrap)
810
811	ENTRY(i_ddi_notrap)
812	membar	#Sync				! force error barrier
813	ldn	[%o0 + AHI_ERR], %o4
814	ldn	[%o4 + ERR_ONTRAP],  %o4	! %o4 = hp->ahi_err->err_ontrap
815	ldn	[%o4 + OT_PREV], %o4
816	retl
817	stn	%o4, [THREAD_REG + T_ONTRAP]	! restore curthread->t_ontrap
818	SET_SIZE(i_ddi_notrap)
819
820/*
821 * Internal on_trap set-up macros.  DDI_PROTECT() and DDI_NOPROTECT() are used
822 * to protect * ddi_get accesses for DDI_FLAGERR_ACC.  DDI_NOPROTECT() sets
823 * the jumpbuf that will return back to the access routine from
824 * i_ddi_protect_trampoline().  DDI_NOPROTECT() clears the ontrap set-up.
825 */
826	ENTRY(i_ddi_prot_trampoline)
827	ldn	[THREAD_REG + T_ONTRAP], %o5    ! %o5 = curthread->t_ontrap
828	lduh	[%o5 + OT_PROT], %o1		! %o1 = %o0->ot_prot
829	andcc	%o1, OT_DATA_ACCESS, %g0	! if (!(%o1 & OT_DATA_ACCESS))
830	bz,pn	%icc, .protaccfail		!     panic
831	rdpr	%pstate, %o3
832	andcc	%o3, PSTATE_IE, %g0		! enable interrupts
833	bnz	1f
834	nop
835	wrpr	%o3, PSTATE_IE, %pstate
8361:
837	ldn	[%o5 + OT_HANDLE], %o0		! %o0 = ot_handle
838	brz,pn	%o0, .protaccfail		! if (ot_handle == NULL) panic
839	nop
840	ldn	[%o0 + AHI_ERR], %o4		! %o4 = hp->ahi_err
841	stn	%g0, [%o4 + ERR_ENA]		! ahi_err->err_ena = 0
842	mov	-2, %o0
843	st	%o0, [%o4 + ERR_STATUS]		! ahi_err->err_status = NONFATAL
844	ldn	[%o5 + OT_PREV], %o0		! restore ontrap
845	membar	#Sync				! force error barrier
846	stn	%o0, [THREAD_REG + T_ONTRAP];
847	b	longjmp                		! longjmp back
848	add	%o5, OT_JMPBUF, %o0		! %o0 = &ot_jmpbuf
849.protaccfail:
850	set	.acc_panic, %o0			! Load panic message
851	call	panic				! Panic if bad t_ontrap data
852	nop
853	SET_SIZE(i_ddi_prot_trampoline)
854
855#define	DDI_PROTECT()				\
856	ldn	[%o0 + AHI_ERR], %o4;		\
857	ldn	[%o4 + ERR_ONTRAP],  %o4;	\
858	ldn	[THREAD_REG + T_ONTRAP], %o5;	\
859	stn	%o5, [%o4 + OT_PREV];		\
860	membar	#Sync;				\
861	stn	%o4, [THREAD_REG + T_ONTRAP];	\
862	add     %o4, OT_JMPBUF, %o0;		\
863	stn	%o7, [%o0 + L_PC];		\
864	stn	%sp, [%o0 + L_SP];		\
865	clr	%o0;
866
867#define	DDI_NOPROTECT()				\
868	ldn	[THREAD_REG + T_ONTRAP], %o4;	\
869	ldn	[%o4 + OT_PREV], %o5;		\
870	membar	#Sync;				\
871	stn	%o5, [THREAD_REG + T_ONTRAP];
872
873/*
874 * DDI_FLAGERR_ACC specific get/put routines.
875 */
876	.align 16
877	ENTRY(i_ddi_prot_get8)
878	DDI_PROTECT()				! set ontrap protection
879	ldub	[%o1], %o2			! do the io access
880	DDI_NOPROTECT()				! remove protection & ret
881	retl
882	mov	%o2, %o0			! set return value
883	SET_SIZE(i_ddi_prot_get8)
884
885	.align 16
886	ENTRY(i_ddi_prot_get16)
887	DDI_PROTECT()				! set ontrap protection
888	lduh	[%o1], %o2			! do the io access
889	DDI_NOPROTECT()				! remove protection & ret
890	retl
891	mov	%o2, %o0			! set return value
892	SET_SIZE(i_ddi_prot_get16)
893
894	.align 16
895	ENTRY(i_ddi_prot_get32)
896	DDI_PROTECT()				! set ontrap protection
897	ld	[%o1], %o2			! do the io access
898	DDI_NOPROTECT()				! remove protection & ret
899	retl
900	mov	%o2, %o0			! set return value
901	SET_SIZE(i_ddi_prot_get32)
902
903	.align 16
904	ENTRY(i_ddi_prot_get64)
905	DDI_PROTECT()				! set ontrap protection
906	ldx	[%o1], %o2			! do the io access
907	DDI_NOPROTECT()				! remove protection & ret
908	retl
909	mov	%o2, %o0			! set return value
910	SET_SIZE(i_ddi_prot_get64)
911
912	.align 16
913	ENTRY(i_ddi_prot_put8)
914	stub	%o2, [%o1]			! do the io access
915	retl
916	membar	#Sync;
917	SET_SIZE(i_ddi_prot_put8)
918
919	.align 16
920	ENTRY(i_ddi_prot_put16)
921	stuh	%o2, [%o1]			! do the io access
922	retl
923	membar	#Sync;
924	SET_SIZE(i_ddi_prot_put16)
925
926	.align 16
927	ENTRY(i_ddi_prot_put32)
928	st	%o2, [%o1]			! do the io access
929	retl
930	membar	#Sync;
931	SET_SIZE(i_ddi_prot_put32)
932
933	.align 16
934	ENTRY(i_ddi_prot_put64)
935	stx	%o2, [%o1]			! do the io access
936	retl
937	membar	#Sync;
938	SET_SIZE(i_ddi_prot_put64)
939
940	.align 16
941	ENTRY(i_ddi_prot_rep_get8)
942	DDI_PROTECT()				! set ontrap protection
943	tst	%o0				! check access error
944	bnz,a	1f
945	nop
946	DDI_REP_GET(1,ub)
9471:
948	DDI_NOPROTECT()				! remove protection & ret
949	retl
950	nop
951	SET_SIZE(i_ddi_prot_rep_get8)
952
953	.align 16
954	ENTRY(i_ddi_prot_rep_get16)
955	DDI_PROTECT()				! set ontrap protection
956	tst	%o0				! check access error
957	bnz,a	1f
958	nop
959	DDI_REP_GET(2,uh)
9601:
961	DDI_NOPROTECT()				! remove protection & ret
962	retl
963	nop
964	SET_SIZE(i_ddi_prot_rep_get16)
965
966	.align 16
967	ENTRY(i_ddi_prot_rep_get32)
968	DDI_PROTECT()				! set ontrap protection
969	tst	%o0				! check access error
970	bnz,a	1f
971	nop
972	DDI_REP_GET(4,/**/)
9731:
974	DDI_NOPROTECT()				! remove protection & ret
975	retl
976	nop
977	SET_SIZE(i_ddi_prot_rep_get32)
978
979	.align 16
980	ENTRY(i_ddi_prot_rep_get64)
981	DDI_PROTECT()				! set ontrap protection
982	tst	%o0				! check access error
983	bnz,a	1f
984	nop
985	DDI_REP_GET(8,x)
9861:
987	DDI_NOPROTECT()				! remove protection & ret
988	retl
989	nop
990	SET_SIZE(i_ddi_prot_rep_get64)
991
992	.align 16
993	ENTRY(i_ddi_prot_rep_put8)
994	DDI_REP_PUT(1,ub)
995	retl
996	membar	#Sync;
997	SET_SIZE(i_ddi_prot_rep_put8)
998
999	.align 16
1000	ENTRY(i_ddi_prot_rep_put16)
1001	DDI_REP_PUT(2,uh)
1002	retl
1003	membar	#Sync;
1004	SET_SIZE(i_ddi_prot_rep_put16)
1005
1006	.align 16
1007	ENTRY(i_ddi_prot_rep_put32)
1008	DDI_REP_PUT(4,/**/)
1009	retl
1010	membar	#Sync;
1011	SET_SIZE(i_ddi_prot_rep_put32)
1012
1013	.align 16
1014	ENTRY(i_ddi_prot_rep_put64)
1015	DDI_REP_PUT(8,x)
1016	retl
1017	membar	#Sync;
1018	SET_SIZE(i_ddi_prot_rep_put64)
1019
1020/*
1021 * Common DDI_CAUTIOUS_ACC routine called from cautious access routines
1022 * in ddi_impl.c
1023 */
1024	ENTRY(i_ddi_caut_get)
1025	rdpr	%pstate, %o3	! check ints
1026	andcc	%o3, PSTATE_IE, %g0
1027	bz,a	cautdone
1028	nop
1029	wrpr	%o3, PSTATE_IE, %pstate
1030	cmp	%o0, 8		! 64-bit?
1031	bne,a	.get_int
1032	cmp	%o0, 4		! 32-bit?
1033	ldx	[%o1], %g1
1034	ba	.getdone
1035	stx	%g1, [%o2]
1036.get_int:
1037	bne,a	.get_half
1038	cmp	%o0, 2		! 16-bit?
1039	lduw	[%o1], %g1
1040	ba	.getdone
1041	stuw	%g1, [%o2]
1042.get_half:
1043	bne,a	.get_byte
1044	ldub	[%o1], %g1	! 8-bit!
1045	lduh	[%o1], %g1
1046	ba	.getdone
1047	stuh	%g1, [%o2]
1048.get_byte:
1049	stub	%g1, [%o2]
1050.getdone:
1051	rdpr	%pstate, %o3	! check&enable ints
1052	andcc	%o3, PSTATE_IE, %g0
1053	bnz,a	cautdone
1054	nop
1055	wrpr	%o3, PSTATE_IE, %pstate
1056cautdone:
1057	retl
1058	nop
1059	SET_SIZE(i_ddi_caut_get)
1060
1061