1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25#if !defined(lint)
26#include "assym.h"
27#endif
28
29/*
30 * General assembly language routines.
31 * It is the intent of this file to contain routines that are
32 * specific to cpu architecture.
33 */
34
35/*
36 * WARNING: If you add a fast trap handler which can be invoked by a
37 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
38 * instead of "done" instruction to return back to the user mode. See
39 * comments for the "fast_trap_done" entry point for more information.
40 */
41#define	FAST_TRAP_DONE	\
42	ba,a	fast_trap_done
43
44#include <sys/machclock.h>
45#include <sys/clock.h>
46
47#if defined(lint)
48#include <sys/types.h>
49#include <sys/scb.h>
50#include <sys/systm.h>
51#include <sys/regset.h>
52#include <sys/sunddi.h>
53#include <sys/lockstat.h>
54#endif	/* lint */
55
56
57#include <sys/asm_linkage.h>
58#include <sys/privregs.h>
59#include <vm/hat_sfmmu.h>
60#include <sys/machparam.h>	/* To get SYSBASE and PAGESIZE */
61#include <sys/machthread.h>
62#include <sys/clock.h>
63#include <sys/intreg.h>
64#include <sys/psr_compat.h>
65#include <sys/isa_defs.h>
66#include <sys/dditypes.h>
67#include <sys/intr.h>
68#include <sys/hypervisor_api.h>
69
70#if !defined(lint)
71#include "assym.h"
72#endif
73
74#define	ICACHE_FLUSHSZ	0x20
75
76#if defined(lint)
77/*
78 * Softint generated when counter field of tick reg matches value field
79 * of tick_cmpr reg
80 */
81/*ARGSUSED*/
82void
83tickcmpr_set(uint64_t clock_cycles)
84{}
85
86#else   /* lint */
87
88	ENTRY_NP(tickcmpr_set)
89	! get 64-bit clock_cycles interval
90	mov	%o0, %o2
91	mov	8, %o3			! A reasonable initial step size
921:
93	WR_TICKCMPR(%o2,%o4,%o5,__LINE__)	! Write to TICK_CMPR
94
95	GET_NATIVE_TIME(%o0,%o4,%o5,__LINE__)	! Read %tick to confirm the
96						! value we wrote was in the
97						! future.
98
99	cmp	%o2, %o0		! If the value we wrote was in the
100	bg,pt	%xcc, 2f		!   future, then blow out of here.
101	  sllx	%o3, 1, %o3		! If not, then double our step size,
102	ba,pt	%xcc, 1b		!   and take another lap.
103	  add	%o0, %o3, %o2		!
1042:
105	retl
106	  nop
107	SET_SIZE(tickcmpr_set)
108
109#endif  /* lint */
110
111#if defined(lint)
112
113void
114tickcmpr_disable(void)
115{}
116
117#else
118
119	ENTRY_NP(tickcmpr_disable)
120	mov	1, %g1
121	sllx	%g1, TICKINT_DIS_SHFT, %o0
122	WR_TICKCMPR(%o0,%o4,%o5,__LINE__)	! Write to TICK_CMPR
123	retl
124	  nop
125	SET_SIZE(tickcmpr_disable)
126
127#endif
128
129#if defined(lint)
130
131/*
132 * tick_write_delta() is intended to increment %stick by the specified delta,
133 * but %stick is only writeable in hyperprivileged mode and at present there
134 * is no provision for this. tick_write_delta is called by the cylic subsystem
135 * if a negative %stick delta is observed after cyclic processing is resumed
136 * after an event such as an OS suspend/resume. On sun4v, the suspend/resume
137 * routines should adjust the %stick offset preventing the cyclic subsystem
138 * from detecting a negative delta. If a negative delta is detected, panic the
139 * system. The negative delta could be caused by improper %stick
140 * synchronization after a suspend/resume.
141 */
142
143/*ARGSUSED*/
144void
145tick_write_delta(uint64_t delta)
146{}
147
148#else	/* lint */
149
150	.seg	".text"
151tick_write_delta_panic:
152	.asciz	"tick_write_delta: not supported, delta: 0x%lx"
153
154	ENTRY_NP(tick_write_delta)
155	sethi	%hi(tick_write_delta_panic), %o1
156        save    %sp, -SA(MINFRAME), %sp ! get a new window to preserve caller
157	mov	%i0, %o1
158	call	panic
159	  or	%i1, %lo(tick_write_delta_panic), %o0
160	/*NOTREACHED*/
161	retl
162	  nop
163#endif
164
165#if defined(lint)
166/*
167 *  return 1 if disabled
168 */
169
170int
171tickcmpr_disabled(void)
172{ return (0); }
173
174#else   /* lint */
175
176	ENTRY_NP(tickcmpr_disabled)
177	RD_TICKCMPR(%g1,%o0,%o1,__LINE__)
178	retl
179	  srlx	%g1, TICKINT_DIS_SHFT, %o0
180	SET_SIZE(tickcmpr_disabled)
181
182#endif  /* lint */
183
184/*
185 * Get current tick
186 */
187#if defined(lint)
188
189u_longlong_t
190gettick(void)
191{ return (0); }
192
193u_longlong_t
194randtick(void)
195{ return (0); }
196
197#else   /* lint */
198
199	ENTRY(gettick)
200	ALTENTRY(randtick)
201	GET_NATIVE_TIME(%o0,%o2,%o3,__LINE__)
202	retl
203	  nop
204	SET_SIZE(randtick)
205	SET_SIZE(gettick)
206
207#endif  /* lint */
208
209/*
210 * Get current tick. For trapstat use only.
211 */
212#if defined (lint)
213
214hrtime_t
215rdtick()
216{ return (0); }
217
218#else
219	ENTRY(rdtick)
220	retl
221	RD_TICK_PHYSICAL(%o0)
222	SET_SIZE(rdtick)
223#endif /* lint */
224
225
226/*
227 * Return the counter portion of the tick register.
228 */
229
230#if defined(lint)
231
232uint64_t
233gettick_counter(void)
234{ return(0); }
235
236uint64_t
237gettick_npt(void)
238{ return(0); }
239
240uint64_t
241getstick_npt(void)
242{ return(0); }
243
244#else	/* lint */
245
246	ENTRY_NP(gettick_counter)
247	RD_TICK(%o0,%o1,%o2,__LINE__)
248	retl
249	nop
250	SET_SIZE(gettick_counter)
251
252	ENTRY_NP(gettick_npt)
253	RD_TICK_PHYSICAL(%o0)
254	retl
255	srlx	%o0, 63, %o0
256	SET_SIZE(gettick_npt)
257
258	ENTRY_NP(getstick_npt)
259	RD_STICK_PHYSICAL(%o0)
260	retl
261	srlx	%o0, 63, %o0
262	SET_SIZE(getstick_npt)
263#endif	/* lint */
264
265/*
266 * Provide a C callable interface to the trap that reads the hi-res timer.
267 * Returns 64-bit nanosecond timestamp in %o0 and %o1.
268 */
269
270#if defined(lint)
271
272hrtime_t
273gethrtime(void)
274{
275	return ((hrtime_t)0);
276}
277
278hrtime_t
279gethrtime_unscaled(void)
280{
281	return ((hrtime_t)0);
282}
283
284hrtime_t
285gethrtime_max(void)
286{
287	return ((hrtime_t)0);
288}
289
290void
291scalehrtime(hrtime_t *hrt)
292{
293	*hrt = 0;
294}
295
296void
297gethrestime(timespec_t *tp)
298{
299	tp->tv_sec = 0;
300	tp->tv_nsec = 0;
301}
302
303time_t
304gethrestime_sec(void)
305{
306	return (0);
307}
308
309void
310gethrestime_lasttick(timespec_t *tp)
311{
312	tp->tv_sec = 0;
313	tp->tv_nsec = 0;
314}
315
316/*ARGSUSED*/
317void
318hres_tick(void)
319{
320}
321
322void
323panic_hres_tick(void)
324{
325}
326
327#else	/* lint */
328
329	ENTRY_NP(gethrtime)
330	GET_HRTIME(%g1,%o0,%o1,%o2,%o3,%o4,%o5,%g2,__LINE__)
331							! %g1 = hrtime
332	retl
333	  mov	%g1, %o0
334	SET_SIZE(gethrtime)
335
336	ENTRY_NP(gethrtime_unscaled)
337	GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__)	! %g1 = native time
338	retl
339	  mov	%g1, %o0
340	SET_SIZE(gethrtime_unscaled)
341
342	ENTRY_NP(gethrtime_waitfree)
343	ALTENTRY(dtrace_gethrtime)
344	GET_NATIVE_TIME(%g1,%o2,%o3,__LINE__)	! %g1 = native time
345	NATIVE_TIME_TO_NSEC(%g1, %o2, %o3)
346	retl
347	  mov	%g1, %o0
348	SET_SIZE(dtrace_gethrtime)
349	SET_SIZE(gethrtime_waitfree)
350
351	ENTRY(gethrtime_max)
352	NATIVE_TIME_MAX(%g1)
353	NATIVE_TIME_TO_NSEC(%g1, %o0, %o1)
354
355	! hrtime_t's are signed, max hrtime_t must be positive
356	mov	-1, %o2
357	brlz,a	%g1, 1f
358	  srlx	%o2, 1, %g1
3591:
360	retl
361	  mov	%g1, %o0
362	SET_SIZE(gethrtime_max)
363
364	ENTRY(scalehrtime)
365	ldx	[%o0], %o1
366	NATIVE_TIME_TO_NSEC(%o1, %o2, %o3)
367	retl
368	  stx	%o1, [%o0]
369	SET_SIZE(scalehrtime)
370
371/*
372 * Fast trap to return a timestamp, uses trap window, leaves traps
373 * disabled.  Returns a 64-bit nanosecond timestamp in %o0 and %o1.
374 *
375 * This is the handler for the ST_GETHRTIME trap.
376 */
377
378	ENTRY_NP(get_timestamp)
379	GET_HRTIME(%g1,%g2,%g3,%g4,%g5,%o0,%o1,%o2,__LINE__)
380	! %g1 = hrtime
381	srlx	%g1, 32, %o0				! %o0 = hi32(%g1)
382	srl	%g1, 0, %o1				! %o1 = lo32(%g1)
383	FAST_TRAP_DONE
384	SET_SIZE(get_timestamp)
385
386/*
387 * Macro to convert GET_HRESTIME() bits into a timestamp.
388 *
389 * We use two separate macros so that the platform-dependent GET_HRESTIME()
390 * can be as small as possible; CONV_HRESTIME() implements the generic part.
391 */
392#define	CONV_HRESTIME(hrestsec, hrestnsec, adj, nslt, nano) \
393	brz,pt	adj, 3f;		/* no adjustments, it's easy */	\
394	add	hrestnsec, nslt, hrestnsec; /* hrest.tv_nsec += nslt */	\
395	brlz,pn	adj, 2f;		/* if hrestime_adj negative */	\
396	  srlx	nslt, ADJ_SHIFT, nslt;	/* delay: nslt >>= 4 */		\
397	subcc	adj, nslt, %g0;		/* hrestime_adj - nslt/16 */	\
398	movg	%xcc, nslt, adj;	/* adj by min(adj, nslt/16) */	\
399	ba	3f;			/* go convert to sec/nsec */	\
400	  add	hrestnsec, adj, hrestnsec; /* delay: apply adjustment */ \
4012:	addcc	adj, nslt, %g0;		/* hrestime_adj + nslt/16 */	\
402	bge,a,pt %xcc, 3f;		/* is adj less negative? */	\
403	  add	hrestnsec, adj, hrestnsec; /* yes: hrest.nsec += adj */	\
404	sub	hrestnsec, nslt, hrestnsec; /* no: hrest.nsec -= nslt/16 */ \
4053:	cmp	hrestnsec, nano;	/* more than a billion? */	\
406	bl,pt	%xcc, 4f;		/* if not, we're done */	\
407	  nop;				/* delay: do nothing :( */	\
408	add	hrestsec, 1, hrestsec;	/* hrest.tv_sec++; */		\
409	sub	hrestnsec, nano, hrestnsec; /* hrest.tv_nsec -= NANOSEC; */	\
410	ba,a	3b;			/* check >= billion again */	\
4114:
412
413	ENTRY_NP(gethrestime)
414	GET_HRESTIME(%o1,%o2,%o3,%o4,%o5,%g1,%g2,%g3,%g4,__LINE__)
415	CONV_HRESTIME(%o1, %o2, %o3, %o4, %o5)
416	stn	%o1, [%o0]
417	retl
418	  stn	%o2, [%o0 + CLONGSIZE]
419	SET_SIZE(gethrestime)
420
421/*
422 * Similar to gethrestime(), but gethrestime_sec() returns current hrestime
423 * seconds.
424 */
425	ENTRY_NP(gethrestime_sec)
426	GET_HRESTIME(%o0,%o2,%o3,%o4,%o5,%g1,%g2,%g3,%g4,__LINE__)
427	CONV_HRESTIME(%o0, %o2, %o3, %o4, %o5)
428	retl					! %o0 current hrestime seconds
429	  nop
430	SET_SIZE(gethrestime_sec)
431
432/*
433 * Returns the hrestime on the last tick.  This is simpler than gethrestime()
434 * and gethrestime_sec():  no conversion is required.  gethrestime_lasttick()
435 * follows the same locking algorithm as GET_HRESTIME and GET_HRTIME,
436 * outlined in detail in clock.h.  (Unlike GET_HRESTIME/GET_HRTIME, we don't
437 * rely on load dependencies to effect the membar #LoadLoad, instead declaring
438 * it explicitly.)
439 */
440	ENTRY_NP(gethrestime_lasttick)
441	sethi	%hi(hres_lock), %o1
4420:
443	lduw	[%o1 + %lo(hres_lock)], %o2	! Load lock value
444	membar	#LoadLoad			! Load of lock must complete
445	andn	%o2, 1, %o2			! Mask off lowest bit
446	ldn	[%o1 + %lo(hrestime)], %g1	! Seconds.
447	add	%o1, %lo(hrestime), %o4
448	ldn	[%o4 + CLONGSIZE], %g2		! Nanoseconds.
449	membar	#LoadLoad			! All loads must complete
450	lduw	[%o1 + %lo(hres_lock)], %o3	! Reload lock value
451	cmp	%o3, %o2			! If lock is locked or has
452	bne	0b				!   changed, retry.
453	  stn	%g1, [%o0]			! Delay: store seconds
454	retl
455	  stn	%g2, [%o0 + CLONGSIZE]		! Delay: store nanoseconds
456	SET_SIZE(gethrestime_lasttick)
457
458/*
459 * Fast trap for gettimeofday().  Returns a timestruc_t in %o0 and %o1.
460 *
461 * This is the handler for the ST_GETHRESTIME trap.
462 */
463
464	ENTRY_NP(get_hrestime)
465	GET_HRESTIME(%o0,%o1,%g1,%g2,%g3,%g4,%g5,%o2,%o3,__LINE__)
466	CONV_HRESTIME(%o0, %o1, %g1, %g2, %g3)
467	FAST_TRAP_DONE
468	SET_SIZE(get_hrestime)
469
470/*
471 * Fast trap to return lwp virtual time, uses trap window, leaves traps
472 * disabled.  Returns a 64-bit number in %o0:%o1, which is the number
473 * of nanoseconds consumed.
474 *
475 * This is the handler for the ST_GETHRVTIME trap.
476 *
477 * Register usage:
478 *	%o0, %o1 = return lwp virtual time
479 * 	%o2 = CPU/thread
480 * 	%o3 = lwp
481 * 	%g1 = scratch
482 * 	%g5 = scratch
483 */
484	ENTRY_NP(get_virtime)
485	GET_NATIVE_TIME(%g5,%g1,%g2,__LINE__)	! %g5 = native time in ticks
486	CPU_ADDR(%g2, %g3)			! CPU struct ptr to %g2
487	ldn	[%g2 + CPU_THREAD], %g2		! thread pointer to %g2
488	ldn	[%g2 + T_LWP], %g3		! lwp pointer to %g3
489
490	/*
491	 * Subtract start time of current microstate from time
492	 * of day to get increment for lwp virtual time.
493	 */
494	ldx	[%g3 + LWP_STATE_START], %g1	! ms_state_start
495	sub	%g5, %g1, %g5
496
497	/*
498	 * Add current value of ms_acct[LMS_USER]
499	 */
500	ldx	[%g3 + LWP_ACCT_USER], %g1	! ms_acct[LMS_USER]
501	add	%g5, %g1, %g5
502	NATIVE_TIME_TO_NSEC(%g5, %g1, %o0)
503
504	srl	%g5, 0, %o1			! %o1 = lo32(%g5)
505	srlx	%g5, 32, %o0			! %o0 = hi32(%g5)
506
507	FAST_TRAP_DONE
508	SET_SIZE(get_virtime)
509
510
511
512	.seg	".text"
513hrtime_base_panic:
514	.asciz	"hrtime_base stepping back"
515
516
517	ENTRY_NP(hres_tick)
518	save	%sp, -SA(MINFRAME), %sp	! get a new window
519
520	sethi	%hi(hrestime), %l4
521	ldstub	[%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5	! try locking
5227:	tst	%l5
523	bz,pt	%xcc, 8f			! if we got it, drive on
524	  ld	[%l4 + %lo(nsec_scale)], %l5	! delay: %l5 = scaling factor
525	ldub	[%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5
5269:	tst	%l5
527	bz,a,pn	%xcc, 7b
528	  ldstub	[%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5
529	ba,pt	%xcc, 9b
530	  ldub	[%l4 + %lo(hres_lock + HRES_LOCK_OFFSET)], %l5
5318:
532	membar	#StoreLoad|#StoreStore
533
534	!
535	! update hres_last_tick.  %l5 has the scaling factor (nsec_scale).
536	!
537	ldx	[%l4 + %lo(hrtime_base)], %g1	! load current hrtime_base
538	GET_NATIVE_TIME(%l0,%l3,%l6,__LINE__)	! current native time
539	stx	%l0, [%l4 + %lo(hres_last_tick)]! prev = current
540	! convert native time to nsecs
541	NATIVE_TIME_TO_NSEC_SCALE(%l0, %l5, %l2, NSEC_SHIFT)
542
543	sub	%l0, %g1, %i1			! get accurate nsec delta
544
545	ldx	[%l4 + %lo(hrtime_base)], %l1
546	cmp	%l1, %l0
547	bg,pn	%xcc, 9f
548	  nop
549
550	stx	%l0, [%l4 + %lo(hrtime_base)]	! update hrtime_base
551
552	!
553	! apply adjustment, if any
554	!
555	ldx	[%l4 + %lo(hrestime_adj)], %l0	! %l0 = hrestime_adj
556	brz	%l0, 2f
557						! hrestime_adj == 0 ?
558						! yes, skip adjustments
559	  clr	%l5				! delay: set adj to zero
560	tst	%l0				! is hrestime_adj >= 0 ?
561	bge,pt	%xcc, 1f			! yes, go handle positive case
562	  srl	%i1, ADJ_SHIFT, %l5		! delay: %l5 = adj
563
564	addcc	%l0, %l5, %g0			! hrestime_adj < -adj ?
565	bl,pt	%xcc, 2f			! yes, use current adj
566	  neg	%l5				! delay: %l5 = -adj
567	ba,pt	%xcc, 2f
568	  mov	%l0, %l5			! no, so set adj = hrestime_adj
5691:
570	subcc	%l0, %l5, %g0			! hrestime_adj < adj ?
571	bl,a,pt	%xcc, 2f			! yes, set adj = hrestime_adj
572	  mov	%l0, %l5			! delay: adj = hrestime_adj
5732:
574	ldx	[%l4 + %lo(timedelta)], %l0	! %l0 = timedelta
575	sub	%l0, %l5, %l0			! timedelta -= adj
576
577	stx	%l0, [%l4 + %lo(timedelta)]	! store new timedelta
578	stx	%l0, [%l4 + %lo(hrestime_adj)]	! hrestime_adj = timedelta
579
580	or	%l4, %lo(hrestime), %l2
581	ldn	[%l2], %i2			! %i2:%i3 = hrestime sec:nsec
582	ldn	[%l2 + CLONGSIZE], %i3
583	add	%i3, %l5, %i3			! hrestime.nsec += adj
584	add	%i3, %i1, %i3			! hrestime.nsec += nslt
585
586	set	NANOSEC, %l5			! %l5 = NANOSEC
587	cmp	%i3, %l5
588	bl,pt	%xcc, 5f			! if hrestime.tv_nsec < NANOSEC
589	  sethi	%hi(one_sec), %i1		! delay
590	add	%i2, 0x1, %i2			! hrestime.tv_sec++
591	sub	%i3, %l5, %i3			! hrestime.tv_nsec - NANOSEC
592	mov	0x1, %l5
593	st	%l5, [%i1 + %lo(one_sec)]
5945:
595	stn	%i2, [%l2]
596	stn	%i3, [%l2 + CLONGSIZE]		! store the new hrestime
597
598	membar	#StoreStore
599
600	ld	[%l4 + %lo(hres_lock)], %i1
601	inc	%i1				! release lock
602	st	%i1, [%l4 + %lo(hres_lock)]	! clear hres_lock
603
604	ret
605	restore
606
6079:
608	!
609	! release hres_lock
610	!
611	ld	[%l4 + %lo(hres_lock)], %i1
612	inc	%i1
613	st	%i1, [%l4 + %lo(hres_lock)]
614
615	sethi	%hi(hrtime_base_panic), %o0
616	call	panic
617	  or	%o0, %lo(hrtime_base_panic), %o0
618
619	SET_SIZE(hres_tick)
620
621#endif	/* lint */
622
623#if !defined(lint) && !defined(__lint)
624
625	.seg	".text"
626kstat_q_panic_msg:
627	.asciz	"kstat_q_exit: qlen == 0"
628
629	ENTRY(kstat_q_panic)
630	save	%sp, -SA(MINFRAME), %sp
631	sethi	%hi(kstat_q_panic_msg), %o0
632	call	panic
633	  or	%o0, %lo(kstat_q_panic_msg), %o0
634	/*NOTREACHED*/
635	SET_SIZE(kstat_q_panic)
636
637#define	BRZPN	brz,pn
638#define	BRZPT	brz,pt
639
640#define	KSTAT_Q_UPDATE(QOP, QBR, QZERO, QRETURN, QTYPE) \
641	ld	[%o0 + QTYPE/**/CNT], %o1;	/* %o1 = old qlen */	\
642	QOP	%o1, 1, %o2;			/* %o2 = new qlen */	\
643	QBR	%o1, QZERO;			/* done if qlen == 0 */	\
644	st	%o2, [%o0 + QTYPE/**/CNT];	/* delay: save qlen */	\
645	ldx	[%o0 + QTYPE/**/LASTUPDATE], %o3;			\
646	ldx	[%o0 + QTYPE/**/TIME], %o4;	/* %o4 = old time */	\
647	ldx	[%o0 + QTYPE/**/LENTIME], %o5;	/* %o5 = old lentime */	\
648	sub	%g1, %o3, %o2;			/* %o2 = time delta */	\
649	mulx	%o1, %o2, %o3;			/* %o3 = cur lentime */	\
650	add	%o4, %o2, %o4;			/* %o4 = new time */	\
651	add	%o5, %o3, %o5;			/* %o5 = new lentime */	\
652	stx	%o4, [%o0 + QTYPE/**/TIME];	/* save time */		\
653	stx	%o5, [%o0 + QTYPE/**/LENTIME];	/* save lentime */	\
654QRETURN;								\
655	stx	%g1, [%o0 + QTYPE/**/LASTUPDATE]; /* lastupdate = now */
656
657#if !defined(DEBUG)
658/*
659 * same as KSTAT_Q_UPDATE but without:
660 * QBR     %o1, QZERO;
661 * to be used only with non-debug build. mimics ASSERT() behaviour.
662 */
663#define	KSTAT_Q_UPDATE_ND(QOP, QRETURN, QTYPE) \
664	ld	[%o0 + QTYPE/**/CNT], %o1;	/* %o1 = old qlen */	\
665	QOP	%o1, 1, %o2;			/* %o2 = new qlen */	\
666	st	%o2, [%o0 + QTYPE/**/CNT];	/* delay: save qlen */	\
667	ldx	[%o0 + QTYPE/**/LASTUPDATE], %o3;			\
668	ldx	[%o0 + QTYPE/**/TIME], %o4;	/* %o4 = old time */	\
669	ldx	[%o0 + QTYPE/**/LENTIME], %o5;	/* %o5 = old lentime */	\
670	sub	%g1, %o3, %o2;			/* %o2 = time delta */	\
671	mulx	%o1, %o2, %o3;			/* %o3 = cur lentime */	\
672	add	%o4, %o2, %o4;			/* %o4 = new time */	\
673	add	%o5, %o3, %o5;			/* %o5 = new lentime */	\
674	stx	%o4, [%o0 + QTYPE/**/TIME];	/* save time */		\
675	stx	%o5, [%o0 + QTYPE/**/LENTIME];	/* save lentime */	\
676QRETURN;								\
677	stx	%g1, [%o0 + QTYPE/**/LASTUPDATE]; /* lastupdate = now */
678#endif
679
680	.align 16
681	ENTRY(kstat_waitq_enter)
682	GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
683	KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_W)
684	SET_SIZE(kstat_waitq_enter)
685
686	.align 16
687	ENTRY(kstat_waitq_exit)
688	GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
689#if defined(DEBUG)
690	KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, retl, KSTAT_IO_W)
691#else
692	KSTAT_Q_UPDATE_ND(sub, retl, KSTAT_IO_W)
693#endif
694	SET_SIZE(kstat_waitq_exit)
695
696	.align 16
697	ENTRY(kstat_runq_enter)
698	GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
699	KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_R)
700	SET_SIZE(kstat_runq_enter)
701
702	.align 16
703	ENTRY(kstat_runq_exit)
704	GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
705#if defined(DEBUG)
706	KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, retl, KSTAT_IO_R)
707#else
708	KSTAT_Q_UPDATE_ND(sub, retl, KSTAT_IO_R)
709#endif
710	SET_SIZE(kstat_runq_exit)
711
712	.align 16
713	ENTRY(kstat_waitq_to_runq)
714	GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
715#if defined(DEBUG)
716	KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_W)
717#else
718	KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_W)
719#endif
720	KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_R)
721	SET_SIZE(kstat_waitq_to_runq)
722
723	.align 16
724	ENTRY(kstat_runq_back_to_waitq)
725	GET_NATIVE_TIME(%g1,%g2,%g3,__LINE__)
726#if defined(DEBUG)
727	KSTAT_Q_UPDATE(sub, BRZPN, kstat_q_panic, 1:, KSTAT_IO_R)
728#else
729	KSTAT_Q_UPDATE_ND(sub, 1:, KSTAT_IO_R)
730#endif
731	KSTAT_Q_UPDATE(add, BRZPT, 1f, 1:retl, KSTAT_IO_W)
732	SET_SIZE(kstat_runq_back_to_waitq)
733
734#endif /* lint */
735
736#ifdef lint
737
738int64_t timedelta;
739hrtime_t hres_last_tick;
740volatile timestruc_t hrestime;
741int64_t hrestime_adj;
742volatile int hres_lock;
743uint_t nsec_scale;
744hrtime_t hrtime_base;
745int traptrace_use_stick;
746
747#else
748	/*
749	 *  -- WARNING --
750	 *
751	 * The following variables MUST be together on a 128-byte boundary.
752	 * In addition to the primary performance motivation (having them all
753	 * on the same cache line(s)), code here and in the GET*TIME() macros
754	 * assumes that they all have the same high 22 address bits (so
755	 * there's only one sethi).
756	 */
757	.seg	".data"
758	.global	timedelta, hres_last_tick, hrestime, hrestime_adj
759	.global	hres_lock, nsec_scale, hrtime_base, traptrace_use_stick
760	.global	nsec_shift, adj_shift, native_tick_offset, native_stick_offset
761
762	/* XXX - above comment claims 128-bytes is necessary */
763	.align	64
764timedelta:
765	.word	0, 0		/* int64_t */
766hres_last_tick:
767	.word	0, 0		/* hrtime_t */
768hrestime:
769	.nword	0, 0		/* 2 longs */
770hrestime_adj:
771	.word	0, 0		/* int64_t */
772hres_lock:
773	.word	0
774nsec_scale:
775	.word	0
776hrtime_base:
777	.word	0, 0
778traptrace_use_stick:
779	.word	0
780nsec_shift:
781	.word	NSEC_SHIFT
782adj_shift:
783	.word	ADJ_SHIFT
784	.align	8
785native_tick_offset:
786	.word	0, 0
787	.align	8
788native_stick_offset:
789	.word	0, 0
790
791#endif
792
793
794/*
795 * drv_usecwait(clock_t n)	[DDI/DKI - section 9F]
796 * usec_delay(int n)		[compatibility - should go one day]
797 * Delay by spinning.
798 *
799 * delay for n microseconds.  numbers <= 0 delay 1 usec
800 *
801 * With UltraSPARC-III the combination of supporting mixed-speed CPUs
802 * and variable clock rate for power management requires that we
803 * use %stick to implement this routine.
804 */
805
806#if defined(lint)
807
808/*ARGSUSED*/
809void
810drv_usecwait(clock_t n)
811{}
812
813/*ARGSUSED*/
814void
815usec_delay(int n)
816{}
817
818#else	/* lint */
819
820	ENTRY(drv_usecwait)
821	ALTENTRY(usec_delay)
822	brlez,a,pn %o0, 0f
823	  mov	1, %o0
8240:
825	sethi	%hi(sticks_per_usec), %o1
826	lduw	[%o1 + %lo(sticks_per_usec)], %o1
827	mulx	%o1, %o0, %o1		! Scale usec to ticks
828	inc	%o1			! We don't start on a tick edge
829	GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
830	add	%o1, %o2, %o1
831
8321:	cmp	%o1, %o2
833	GET_NATIVE_TIME(%o2,%o3,%o4,__LINE__)
834	bgeu,pt	%xcc, 1b
835	  nop
836	retl
837	  nop
838	SET_SIZE(usec_delay)
839	SET_SIZE(drv_usecwait)
840#endif	/* lint */
841
842#if defined(lint)
843
844/* ARGSUSED */
845void
846pil14_interrupt(int level)
847{}
848
849#else
850
851/*
852 * Level-14 interrupt prologue.
853 */
854	ENTRY_NP(pil14_interrupt)
855	CPU_ADDR(%g1, %g2)
856	rdpr	%pil, %g6			! %g6 = interrupted PIL
857	stn	%g6, [%g1 + CPU_PROFILE_PIL]	! record interrupted PIL
858	rdpr	%tstate, %g6
859	rdpr	%tpc, %g5
860	btst	TSTATE_PRIV, %g6		! trap from supervisor mode?
861	bnz,a,pt %xcc, 1f
862	  stn	%g5, [%g1 + CPU_PROFILE_PC]	! if so, record kernel PC
863	stn	%g5, [%g1 + CPU_PROFILE_UPC]	! if not, record user PC
864	ba	pil_interrupt_common		! must be large-disp branch
865	  stn	%g0, [%g1 + CPU_PROFILE_PC]	! zero kernel PC
8661:	ba	pil_interrupt_common		! must be large-disp branch
867	  stn	%g0, [%g1 + CPU_PROFILE_UPC]	! zero user PC
868	SET_SIZE(pil14_interrupt)
869
870	ENTRY_NP(tick_rtt)
871	!
872	! Load TICK_COMPARE into %o5; if bit 63 is set, then TICK_COMPARE is
873	! disabled.  If TICK_COMPARE is enabled, we know that we need to
874	! reenqueue the interrupt request structure.  We'll then check TICKINT
875	! in SOFTINT; if it's set, then we know that we were in a TICK_COMPARE
876	! interrupt.  In this case, TICK_COMPARE may have been rewritten
877	! recently; we'll compare %o5 to the current time to verify that it's
878	! in the future.
879	!
880	! Note that %o5 is live until after 1f.
881	! XXX - there is a subroutine call while %o5 is live!
882	!
883	RD_TICKCMPR(%o5,%g1,%g2,__LINE__)
884	srlx	%o5, TICKINT_DIS_SHFT, %g1
885	brnz,pt	%g1, 2f
886	  nop
887
888	rdpr 	%pstate, %g5
889	andn	%g5, PSTATE_IE, %g1
890	wrpr	%g0, %g1, %pstate		! Disable vec interrupts
891
892	sethi	%hi(cbe_level14_inum), %o1
893	ldx	[%o1 + %lo(cbe_level14_inum)], %o1
894	call	intr_enqueue_req ! preserves %o5 and %g5
895	  mov	PIL_14, %o0
896
897	! Check SOFTINT for TICKINT/STICKINT
898	rd	SOFTINT, %o4
899	set	(TICK_INT_MASK | STICK_INT_MASK), %o0
900	andcc	%o4, %o0, %g0
901	bz,a,pn	%icc, 2f
902	  wrpr	%g0, %g5, %pstate		! Enable vec interrupts
903
904	! clear TICKINT/STICKINT
905	wr	%o0, CLEAR_SOFTINT
906
907	!
908	! Now that we've cleared TICKINT, we can reread %tick and confirm
909	! that the value we programmed is still in the future.  If it isn't,
910	! we need to reprogram TICK_COMPARE to fire as soon as possible.
911	!
912	GET_NATIVE_TIME(%o0,%g1,%g2,__LINE__)	! %o0 = tick
913	cmp	%o5, %o0			! In the future?
914	bg,a,pt	%xcc, 2f			! Yes, drive on.
915	  wrpr	%g0, %g5, %pstate		!   delay: enable vec intr
916
917	!
918	! If we're here, then we have programmed TICK_COMPARE with a %tick
919	! which is in the past; we'll now load an initial step size, and loop
920	! until we've managed to program TICK_COMPARE to fire in the future.
921	!
922	mov	8, %o4				! 8 = arbitrary inital step
9231:	add	%o0, %o4, %o5			! Add the step
924	WR_TICKCMPR(%o5,%g1,%g2,__LINE__)	! Write to TICK_CMPR
925	GET_NATIVE_TIME(%o0,%g1,%g2,__LINE__)	! %o0 = tick
926	cmp	%o5, %o0			! In the future?
927	bg,a,pt	%xcc, 2f			! Yes, drive on.
928	  wrpr	%g0, %g5, %pstate		!    delay: enable vec intr
929	ba	1b				! No, try again.
930	  sllx	%o4, 1, %o4			!    delay: double step size
931
9322:	ba	current_thread_complete
933	  nop
934	SET_SIZE(tick_rtt)
935
936#endif /* lint */
937
938#if defined(lint)
939
940/* ARGSUSED */
941void
942pil15_interrupt(int level)
943{}
944
945#else   /* lint */
946
947/*
948 * Level-15 interrupt prologue.
949 */
950       ENTRY_NP(pil15_interrupt)
951       CPU_ADDR(%g1, %g2)
952       rdpr    %tstate, %g6
953       rdpr    %tpc, %g5
954       btst    TSTATE_PRIV, %g6                ! trap from supervisor mode?
955       bnz,a,pt %xcc, 1f
956       stn     %g5, [%g1 + CPU_CPCPROFILE_PC]  ! if so, record kernel PC
957       stn     %g5, [%g1 + CPU_CPCPROFILE_UPC] ! if not, record user PC
958       ba      pil15_epilogue                  ! must be large-disp branch
959       stn     %g0, [%g1 + CPU_CPCPROFILE_PC]  ! zero kernel PC
9601:     ba      pil15_epilogue                  ! must be large-disp branch
961       stn     %g0, [%g1 + CPU_CPCPROFILE_UPC] ! zero user PC
962       SET_SIZE(pil15_interrupt)
963
964#endif  /* lint */
965
966#if defined(lint)
967/*
968 * Prefetch a page_t for write or read, this assumes a linear
969 * scan of sequential page_t's.
970 */
971/*ARGSUSED*/
972void
973prefetch_page_w(void *pp)
974{}
975
976/*ARGSUSED*/
977void
978prefetch_page_r(void *pp)
979{}
980#else	/* lint */
981
982/* XXXQ These should be inline templates, not functions */
983        ENTRY(prefetch_page_w)
984        retl
985	  nop
986        SET_SIZE(prefetch_page_w)
987
988        ENTRY(prefetch_page_r)
989        retl
990	  nop
991        SET_SIZE(prefetch_page_r)
992
993#endif	/* lint */
994
995#if defined(lint)
996/*
997 * Prefetch struct smap for write.
998 */
999/*ARGSUSED*/
1000void
1001prefetch_smap_w(void *smp)
1002{}
1003#else	/* lint */
1004
1005/* XXXQ These should be inline templates, not functions */
1006	ENTRY(prefetch_smap_w)
1007	retl
1008	  nop
1009	SET_SIZE(prefetch_smap_w)
1010
1011#endif	/* lint */
1012
1013/*
1014 * Generic sun4v MMU and Cache operations.
1015 */
1016
1017#if defined(lint)
1018
1019/*ARGSUSED*/
1020void
1021vtag_flushpage(caddr_t vaddr, uint64_t sfmmup)
1022{}
1023
1024/*ARGSUSED*/
1025void
1026vtag_flushall(void)
1027{}
1028
1029/*ARGSUSED*/
1030void
1031vtag_unmap_perm_tl1(uint64_t vaddr, uint64_t ctxnum)
1032{}
1033
1034/*ARGSUSED*/
1035void
1036vtag_flushpage_tl1(uint64_t vaddr, uint64_t sfmmup)
1037{}
1038
1039/*ARGSUSED*/
1040void
1041vtag_flush_pgcnt_tl1(uint64_t vaddr, uint64_t sfmmup_pgcnt)
1042{}
1043
1044/*ARGSUSED*/
1045void
1046vtag_flushall_tl1(uint64_t dummy1, uint64_t dummy2)
1047{}
1048
1049/*ARGSUSED*/
1050void
1051vac_flushpage(pfn_t pfnum, int vcolor)
1052{}
1053
1054/*ARGSUSED*/
1055void
1056vac_flushpage_tl1(uint64_t pfnum, uint64_t vcolor)
1057{}
1058
1059/*ARGSUSED*/
1060void
1061flush_instr_mem(caddr_t vaddr, size_t len)
1062{}
1063
1064#else	/* lint */
1065
1066	ENTRY_NP(vtag_flushpage)
1067	/*
1068	 * flush page from the tlb
1069	 *
1070	 * %o0 = vaddr
1071	 * %o1 = sfmmup
1072	 */
1073	SFMMU_CPU_CNUM(%o1, %g1, %g2)   /* %g1 = sfmmu cnum on this CPU */
1074
1075	mov	%g1, %o1
1076	mov	MAP_ITLB | MAP_DTLB, %o2
1077	ta	MMU_UNMAP_ADDR
1078	brz,pt	%o0, 1f
1079	  nop
1080	ba	panic_bad_hcall
1081	  mov	MMU_UNMAP_ADDR, %o1
10821:
1083 	retl
1084	  nop
1085	SET_SIZE(vtag_flushpage)
1086
1087	ENTRY_NP(vtag_flushall)
1088	mov	%g0, %o0	! XXX no cpu list yet
1089	mov	%g0, %o1	! XXX no cpu list yet
1090	mov	MAP_ITLB | MAP_DTLB, %o2
1091	mov	MMU_DEMAP_ALL, %o5
1092	ta	FAST_TRAP
1093	brz,pt	%o0, 1f
1094	  nop
1095	ba	panic_bad_hcall
1096	  mov	MMU_DEMAP_ALL, %o1
10971:
1098	retl
1099	  nop
1100	SET_SIZE(vtag_flushall)
1101
1102	ENTRY_NP(vtag_unmap_perm_tl1)
1103	/*
1104	 * x-trap to unmap perm map entry
1105	 * %g1 = vaddr
1106	 * %g2 = ctxnum (KCONTEXT only)
1107	 */
1108	mov	%o0, %g3
1109	mov	%o1, %g4
1110	mov	%o2, %g5
1111	mov	%o5, %g6
1112	mov	%g1, %o0
1113	mov	%g2, %o1
1114	mov	MAP_ITLB | MAP_DTLB, %o2
1115	mov	UNMAP_PERM_ADDR, %o5
1116	ta	FAST_TRAP
1117	brz,pt	%o0, 1f
1118	nop
1119
1120	mov	PTL1_BAD_HCALL, %g1
1121
1122	cmp	%o0, H_ENOMAP
1123	move	%xcc, PTL1_BAD_HCALL_UNMAP_PERM_ENOMAP, %g1
1124
1125	cmp	%o0, H_EINVAL
1126	move	%xcc, PTL1_BAD_HCALL_UNMAP_PERM_EINVAL, %g1
1127
1128	ba,a	ptl1_panic
11291:
1130	mov	%g6, %o5
1131	mov	%g5, %o2
1132	mov	%g4, %o1
1133	mov	%g3, %o0
1134	retry
1135	SET_SIZE(vtag_unmap_perm_tl1)
1136
1137	ENTRY_NP(vtag_flushpage_tl1)
1138	/*
1139	 * x-trap to flush page from tlb and tsb
1140	 *
1141	 * %g1 = vaddr, zero-extended on 32-bit kernel
1142	 * %g2 = sfmmup
1143	 *
1144	 * assumes TSBE_TAG = 0
1145	 */
1146	srln	%g1, MMU_PAGESHIFT, %g1
1147	slln	%g1, MMU_PAGESHIFT, %g1			/* g1 = vaddr */
1148	mov	%o0, %g3
1149	mov	%o1, %g4
1150	mov	%o2, %g5
1151	mov	%g1, %o0			/* vaddr */
1152
1153	SFMMU_CPU_CNUM(%g2, %o1, %g6)   /* %o1 = sfmmu cnum on this CPU */
1154
1155	mov	MAP_ITLB | MAP_DTLB, %o2
1156	ta	MMU_UNMAP_ADDR
1157	brz,pt	%o0, 1f
1158	nop
1159	  ba	ptl1_panic
1160	mov	PTL1_BAD_HCALL, %g1
11611:
1162	mov	%g5, %o2
1163	mov	%g4, %o1
1164	mov	%g3, %o0
1165	membar #Sync
1166	retry
1167	SET_SIZE(vtag_flushpage_tl1)
1168
1169	ENTRY_NP(vtag_flush_pgcnt_tl1)
1170	/*
1171	 * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb
1172	 *
1173	 * %g1 = vaddr, zero-extended on 32-bit kernel
1174	 * %g2 = <sfmmup58|pgcnt6>, (pgcnt - 1) is pass'ed in via pgcnt6 bits.
1175	 *
1176	 * NOTE: this handler relies on the fact that no
1177	 *	interrupts or traps can occur during the loop
1178	 *	issuing the TLB_DEMAP operations. It is assumed
1179	 *	that interrupts are disabled and this code is
1180	 *	fetching from the kernel locked text address.
1181	 *
1182	 * assumes TSBE_TAG = 0
1183	 */
1184	srln	%g1, MMU_PAGESHIFT, %g1
1185	slln	%g1, MMU_PAGESHIFT, %g1		/* g1 = vaddr */
1186	mov	%o0, %g3
1187	mov	%o1, %g4
1188	mov	%o2, %g5
1189
1190	and	%g2, SFMMU_PGCNT_MASK, %g7	/* g7 = pgcnt - 1 */
1191	add	%g7, 1, %g7			/* g7 = pgcnt */
1192
1193        andn    %g2, SFMMU_PGCNT_MASK, %o0      /* %o0 = sfmmup */
1194
1195	SFMMU_CPU_CNUM(%o0, %g2, %g6)    /* %g2 = sfmmu cnum on this CPU */
1196
1197	set	MMU_PAGESIZE, %g6		/* g6 = pgsize */
1198
11991:
1200	mov	%g1, %o0			/* vaddr */
1201	mov	%g2, %o1			/* cnum */
1202	mov	MAP_ITLB | MAP_DTLB, %o2
1203	ta	MMU_UNMAP_ADDR
1204	brz,pt	%o0, 2f
1205	  nop
1206	ba	ptl1_panic
1207	  mov	PTL1_BAD_HCALL, %g1
12082:
1209	deccc	%g7				/* decr pgcnt */
1210	bnz,pt	%icc,1b
1211	  add	%g1, %g6, %g1			/* go to nextpage */
1212
1213	mov	%g5, %o2
1214	mov	%g4, %o1
1215	mov	%g3, %o0
1216	membar #Sync
1217	retry
1218	SET_SIZE(vtag_flush_pgcnt_tl1)
1219
1220	! Not implemented on US1/US2
1221	ENTRY_NP(vtag_flushall_tl1)
1222	mov	%o0, %g3
1223	mov	%o1, %g4
1224	mov	%o2, %g5
1225	mov	%o3, %g6	! XXXQ not used?
1226	mov	%o5, %g7
1227	mov	%g0, %o0	! XXX no cpu list yet
1228	mov	%g0, %o1	! XXX no cpu list yet
1229	mov	MAP_ITLB | MAP_DTLB, %o2
1230	mov	MMU_DEMAP_ALL, %o5
1231	ta	FAST_TRAP
1232	brz,pt	%o0, 1f
1233	  nop
1234	ba	ptl1_panic
1235	  mov	PTL1_BAD_HCALL, %g1
12361:
1237	mov	%g7, %o5
1238	mov	%g6, %o3	! XXXQ not used?
1239	mov	%g5, %o2
1240	mov	%g4, %o1
1241	mov	%g3, %o0
1242	retry
1243	SET_SIZE(vtag_flushall_tl1)
1244
1245/*
1246 * flush_instr_mem:
1247 *	Flush a portion of the I-$ starting at vaddr
1248 * 	%o0 vaddr
1249 *	%o1 bytes to be flushed
1250 */
1251
1252	ENTRY(flush_instr_mem)
1253	membar	#StoreStore				! Ensure the stores
1254							! are globally visible
12551:
1256	flush	%o0
1257	subcc	%o1, ICACHE_FLUSHSZ, %o1		! bytes = bytes-0x20
1258	bgu,pt	%ncc, 1b
1259	  add	%o0, ICACHE_FLUSHSZ, %o0		! vaddr = vaddr+0x20
1260
1261	retl
1262	  nop
1263	SET_SIZE(flush_instr_mem)
1264
1265#endif /* !lint */
1266
1267#if !defined(CUSTOM_FPZERO)
1268
1269/*
1270 * fp_zero() - clear all fp data registers and the fsr
1271 */
1272
1273#if defined(lint) || defined(__lint)
1274
1275void
1276fp_zero(void)
1277{}
1278
1279#else	/* lint */
1280
1281.global	fp_zero_zero
1282.align 8
1283fp_zero_zero:
1284	.xword	0
1285
1286	ENTRY_NP(fp_zero)
1287	sethi	%hi(fp_zero_zero), %o0
1288	ldx	[%o0 + %lo(fp_zero_zero)], %fsr
1289	ldd	[%o0 + %lo(fp_zero_zero)], %f0
1290	fmovd	%f0, %f2
1291	fmovd	%f0, %f4
1292	fmovd	%f0, %f6
1293	fmovd	%f0, %f8
1294	fmovd	%f0, %f10
1295	fmovd	%f0, %f12
1296	fmovd	%f0, %f14
1297	fmovd	%f0, %f16
1298	fmovd	%f0, %f18
1299	fmovd	%f0, %f20
1300	fmovd	%f0, %f22
1301	fmovd	%f0, %f24
1302	fmovd	%f0, %f26
1303	fmovd	%f0, %f28
1304	fmovd	%f0, %f30
1305	fmovd	%f0, %f32
1306	fmovd	%f0, %f34
1307	fmovd	%f0, %f36
1308	fmovd	%f0, %f38
1309	fmovd	%f0, %f40
1310	fmovd	%f0, %f42
1311	fmovd	%f0, %f44
1312	fmovd	%f0, %f46
1313	fmovd	%f0, %f48
1314	fmovd	%f0, %f50
1315	fmovd	%f0, %f52
1316	fmovd	%f0, %f54
1317	fmovd	%f0, %f56
1318	fmovd	%f0, %f58
1319	fmovd	%f0, %f60
1320	retl
1321	fmovd	%f0, %f62
1322	SET_SIZE(fp_zero)
1323
1324#endif	/* lint */
1325#endif  /* CUSTOM_FPZERO */
1326