1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 *
25 * Assembly code support for the Cheetah+ module
26 */
27
28#include "assym.h"
29
30#include <sys/asm_linkage.h>
31#include <sys/mmu.h>
32#include <vm/hat_sfmmu.h>
33#include <sys/machparam.h>
34#include <sys/machcpuvar.h>
35#include <sys/machthread.h>
36#include <sys/machtrap.h>
37#include <sys/privregs.h>
38#include <sys/asm_linkage.h>
39#include <sys/trap.h>
40#include <sys/cheetahregs.h>
41#include <sys/xc_impl.h>
42#include <sys/intreg.h>
43#include <sys/async.h>
44#include <sys/clock.h>
45#include <sys/cheetahasm.h>
46#include <sys/cmpregs.h>
47
48#ifdef TRAPTRACE
49#include <sys/traptrace.h>
50#endif /* TRAPTRACE */
51
52
53	.global retire_l2_start
54	.global retire_l2_end
55	.global unretire_l2_start
56	.global unretire_l2_end
57	.global retire_l3_start
58	.global retire_l3_end
59	.global unretire_l3_start
60	.global unretire_l3_end
61
62/*
63 * Panther version to reflush a line from both the L2 cache and L3
64 * cache by the respective indexes. Flushes all ways of the line from
65 * each cache.
66 *
67 * l2_index	Index into the L2$ of the line to be flushed. This
68 *		register will not be modified by this routine.
69 * l3_index	Index into the L3$ of the line to be flushed. This
70 *		register will not be modified by this routine.
71 * scr2		scratch register.
72 * scr3		scratch register.
73 *
74 */
75#define	PN_ECACHE_REFLUSH_LINE(l2_index, l3_index, scr2, scr3)		\
76	set	PN_L2_MAX_SET, scr2;					\
77	set	PN_L2_SET_SIZE, scr3;					\
781:									\
79	ldxa	[l2_index + scr2]ASI_L2_TAG, %g0;			\
80	cmp	scr2, %g0;						\
81	bg,a	1b;							\
82	  sub	scr2, scr3, scr2;					\
83	mov	6, scr2;						\
846:									\
85	cmp	scr2, %g0;						\
86	bg,a	6b;							\
87	  sub	scr2, 1, scr2;						\
88	set	PN_L3_MAX_SET, scr2;					\
89	set	PN_L3_SET_SIZE, scr3;					\
902:									\
91	ldxa	[l3_index + scr2]ASI_EC_DIAG, %g0;			\
92	cmp	scr2, %g0;						\
93	bg,a	2b;							\
94	  sub	scr2, scr3, scr2;
95
96/*
97 * Panther version of ecache_flush_line. Flushes the line corresponding
98 * to physaddr from both the L2 cache and the L3 cache.
99 *
100 * physaddr	Input: Physical address to flush.
101 *              Output: Physical address to flush (preserved).
102 * l2_idx_out	Input: scratch register.
103 *              Output: Index into the L2$ of the line to be flushed.
104 * l3_idx_out	Input: scratch register.
105 *              Output: Index into the L3$ of the line to be flushed.
106 * scr3		scratch register.
107 * scr4		scratch register.
108 *
109 */
110#define	PN_ECACHE_FLUSH_LINE(physaddr, l2_idx_out, l3_idx_out, scr3, scr4)	\
111	set	PN_L3_SET_SIZE, l2_idx_out;					\
112	sub	l2_idx_out, 1, l2_idx_out;					\
113	and	physaddr, l2_idx_out, l3_idx_out;				\
114	set	PN_L3_IDX_DISP_FLUSH, l2_idx_out;				\
115	or	l2_idx_out, l3_idx_out, l3_idx_out;				\
116	set	PN_L2_SET_SIZE, l2_idx_out;					\
117	sub	l2_idx_out, 1, l2_idx_out;					\
118	and	physaddr, l2_idx_out, l2_idx_out;				\
119	set	PN_L2_IDX_DISP_FLUSH, scr3;					\
120	or	l2_idx_out, scr3, l2_idx_out;					\
121	PN_ECACHE_REFLUSH_LINE(l2_idx_out, l3_idx_out, scr3, scr4)
122
123
124	.align 4096
125	ENTRY(retire_l2)
126retire_l2_start:
127
128	! since we disable interrupts, we don't need to do kpreempt_disable()
129	rdpr	%pstate, %o2
130	andn	%o2, PSTATE_IE, %g1
131	wrpr	%g0, %g1, %pstate		! disable interrupts
132	/*
133	 * Save current DCU state.  Turn off IPS
134	 */
135	setx	DCU_IPS_MASK, %g2, %o3
136	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
137	andn	%g1, %o3, %g4
138	stxa	%g4, [%g0]ASI_DCU
139	flush	%g0
140	PARK_SIBLING_CORE(%g1, %o3, %o4)	! %g1 has DCU value
141	clr	%o5	! assume success
1428:
143	PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %g2, %g3)
1441:
145	! Check if line is invalid; if so, NA it.
146	ldxa	[%o0]ASI_L2_TAG, %o3
147	btst	0x7, %o3
148	bnz	%xcc, 2f
149	 nop
150	stxa	%o1, [%o0]ASI_L2_TAG
151	membar #Sync	! still on same cache line
152	! now delay 15 cycles so we don't have hazard when we return
153	mov	16, %o1
1541:
155	brnz,pt	%o1, 1b
156	 dec	%o1
1579:
158	! UNPARK-SIBLING_CORE is 7 instructions, so we cross a cache boundary
159	UNPARK_SIBLING_CORE(%g1, %o3, %o4)	! 7 instructions
160	/*
161	 * Restore the DCU
162	 */
163	stxa	%g1, [%g0]ASI_DCU
164	flush	%g0
165	wrpr	%g0, %o2, %pstate		!restore pstate
166	retl
167	 mov	%o5, %o0
1682:
169	! It is OK to have STATE as NA (if so, nothing to do!)
170	and	%o3, 0x7, %o3
171	cmp	%o3, 0x5
172	be,a,pt	%xcc, 9b
173	 mov	1, %o5	! indicate was already NA
174	! Hmm.	Not INV, not NA.
175	cmp	%o5, 0
176	be,a,pt	%xcc, 8b	! Flush the cacheline again
177	 mov	2, %o5	! indicate retry was done
178	! We already Flushed cacheline second time. Return -1
179	clr	%o5
180	ba	9b
181	 dec	%o5
182retire_l2_end:
183	SET_SIZE(retire_l2)
184
185	ENTRY(unretire_l2)
186unretire_l2_start:
187
188	! since we disable interrupts, we don't need to do kpreempt_disable()
189	rdpr	%pstate, %o2
190	andn	%o2, PSTATE_IE, %g1
191	wrpr	%g0, %g1, %pstate		! disable interrupts
192	/*
193	 * Save current DCU state.  Turn off IPS
194	 */
195	setx	DCU_IPS_MASK, %g2, %o3
196	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
197	andn	%g1, %o3, %g4
198	stxa	%g4, [%g0]ASI_DCU
199	flush	%g0	/* flush required after changing the IC bit */
200	PARK_SIBLING_CORE(%g1, %o3, %o4)	! %g1 has DCU value
201
202	PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
2031:
204	clr	%o5	! assume success
205	! Check that line is in NA state; if so, INV it.
206	ldxa	[%o0]ASI_L2_TAG, %o3
207	and	%o3, 0x7, %o3
208	cmp	%o3, 0x5
209	bne,a,pt %xcc, 9f	! Wasn't NA, so something is wrong
210	 dec	%o5	! indicate not NA
211	stxa	%g0, [%o0]ASI_L2_TAG
212	membar #Sync
213	! now delay 15 cycles so we don't have hazard when we return
214	mov	16, %o1
2151:
216	brnz,pt	%o1, 1b
217	 dec	%o1
2189:
219	! UNPARK-SIBLING_CORE is 7 instructions
220	UNPARK_SIBLING_CORE(%g1, %o3, %o4)	! 7 instructions
221	/*
222	 * Restore the DCU
223	 */
224	stxa	%g1, [%g0]ASI_DCU
225	flush	%g0
226	wrpr	%g0, %o2, %pstate		!restore pstate
227	retl
228	 mov	%o5, %o0
229unretire_l2_end:
230	SET_SIZE(unretire_l2)
231
232	ENTRY(retire_l3)
233retire_l3_start:
234
235	! since we disable interrupts, we don't need to do kpreempt_disable()
236	rdpr	%pstate, %o2
237	andn	%o2, PSTATE_IE, %g1
238	wrpr	%g0, %g1, %pstate		! disable interrupts
239	/*
240	 * Save current DCU state.  Turn off IPS
241	 */
242	setx	DCU_IPS_MASK, %g2, %o3
243	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
244	andn	%g1, %o3, %g4
245	stxa	%g4, [%g0]ASI_DCU
246	flush	%g0	/* flush required after changing the IC bit */
247	PARK_SIBLING_CORE(%g1, %o3, %o4)	! %g1 has DCU value
248
249	! PN-ECACHE-FLUSH_LINE is 30 instructions
250	PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
2511:
252	clr	%o5	! assume success
253	! Check if line is invalid; if so, NA it.
254	ldxa	[%o0]ASI_EC_DIAG, %o3
255	btst	0x7, %o3
256	bnz	%xcc, 2f
257	 nop
258	stxa	%o1, [%o0]ASI_EC_DIAG
259	membar #Sync	! still on same cache line
260	! now delay 15 cycles so we don't have hazard when we return
261	mov	16, %o1
2621:
263	brnz,pt	%o1, 1b
264	 dec	%o1
2659:
266	! UNPARK-SIBLING_CORE is 7 instructions, so we cross a cache boundary
267	UNPARK_SIBLING_CORE(%g1, %o3, %o4)	! 7 instructions
268	/*
269	 * Restore the DCU
270	 */
271	stxa	%g1, [%g0]ASI_DCU
272	flush	%g0
273	wrpr	%g0, %o2, %pstate		!restore pstate
274	retl
275	 mov	%o5, %o0
2762:
277	! It is OK to have STATE as NA (if so, nothing to do!)
278	and	%o3, 0x7, %o3
279	cmp	%o3, 0x5
280	be,a,pt	%xcc, 9b
281	 inc	%o5	! indicate was already NA
282	! Hmm.	Not INV, not NA
283	ba	9b
284	 dec	%o5
285retire_l3_end:
286	SET_SIZE(retire_l3)
287
288	ENTRY(unretire_l3)
289unretire_l3_start:
290
291	! since we disable interrupts, we don't need to do kpreempt_disable()
292	rdpr	%pstate, %o2
293	andn	%o2, PSTATE_IE, %g1
294	wrpr	%g0, %g1, %pstate		! disable interrupts
295	/*
296	 * Save current DCU state.  Turn off IPS
297	 */
298	setx	DCU_IPS_MASK, %g2, %o3
299	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
300	andn	%g1, %o3, %g4
301	stxa	%g4, [%g0]ASI_DCU
302	flush	%g0	/* flush required after changing the IC bit */
303	PARK_SIBLING_CORE(%g1, %o3, %o4)	! %g1 has DCU value
304
305	PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
3061:
307	clr	%o5	! assume success
308	! Check that line is in NA state; if so, INV it.
309	ldxa	[%o0]ASI_EC_DIAG, %o3
310	and	%o3, 0x7, %o3
311	cmp	%o3, 0x5
312	bne,a,pt %xcc, 9f	! Wasn't NA, so something is wrong
313	 dec	%o5	! indicate not NA
314	stxa	%g0, [%o0]ASI_EC_DIAG
315	membar #Sync
316	! now delay 15 cycles so we don't have hazard when we return
317	mov	16, %o1
3181:
319	brnz,pt	%o1, 1b
320	 dec	%o1
3219:
322	! UNPARK-SIBLING_CORE is 7 instructions
323	UNPARK_SIBLING_CORE(%g1, %o3, %o4)	! 7 instructions
324	/*
325	 * Restore the DCU
326	 */
327	stxa	%g1, [%g0]ASI_DCU
328	flush	%g0
329	wrpr	%g0, %o2, %pstate		!restore pstate
330	retl
331	 mov	%o5, %o0
332unretire_l3_end:
333	SET_SIZE(unretire_l3)
334
335	.align 2048
336
337	ENTRY(retire_l2_alternate)
338
339	! since we disable interrupts, we don't need to do kpreempt_disable()
340	rdpr	%pstate, %o2
341	andn	%o2, PSTATE_IE, %g1
342	wrpr	%g0, %g1, %pstate		! disable interrupts
343	/*
344	 * Save current DCU state.  Turn off IPS
345	 */
346	setx	DCU_IPS_MASK, %g2, %o3
347	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
348	andn	%g1, %o3, %g4
349	stxa	%g4, [%g0]ASI_DCU
350	flush	%g0
351	PARK_SIBLING_CORE(%g1, %o3, %o4)	! %g1 has DCU value
352	clr	%o5	! assume success
3538:
354	PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %g2, %g3)
3551:
356	! Check if line is invalid; if so, NA it.
357	ldxa	[%o0]ASI_L2_TAG, %o3
358	btst	0x7, %o3
359	bnz	%xcc, 2f
360	 nop
361	stxa	%o1, [%o0]ASI_L2_TAG
362	membar #Sync	! still on same cache line
363	! now delay 15 cycles so we don't have hazard when we return
364	mov	16, %o1
3651:
366	brnz,pt	%o1, 1b
367	 dec	%o1
3689:
369	! UNPARK-SIBLING_CORE is 7 instructions, so we cross a cache boundary
370	UNPARK_SIBLING_CORE(%g1, %o3, %o4)	! 7 instructions
371	/*
372	 * Restore the DCU
373	 */
374	stxa	%g1, [%g0]ASI_DCU
375	flush	%g0
376	wrpr	%g0, %o2, %pstate		!restore pstate
377	retl
378	 mov	%o5, %o0
3792:
380	! It is OK to have STATE as NA (if so, nothing to do!)
381	and	%o3, 0x7, %o3
382	cmp	%o3, 0x5
383	be,a,pt	%xcc, 9b
384	 mov	1, %o5	! indicate was already NA
385	! Hmm.	Not INV, not NA.
386	cmp	%o5, 0
387	be,a,pt	%xcc, 8b	! Flush the cacheline again
388	 mov	2, %o5	! indicate retry was done
389	! We already Flushed cacheline second time. Return -1
390	clr	%o5
391	ba	9b
392	 dec	%o5
393	SET_SIZE(retire_l2_alternate)
394
395	ENTRY(unretire_l2_alternate)
396
397	! since we disable interrupts, we don't need to do kpreempt_disable()
398	rdpr	%pstate, %o2
399	andn	%o2, PSTATE_IE, %g1
400	wrpr	%g0, %g1, %pstate		! disable interrupts
401	/*
402	 * Save current DCU state.  Turn off IPS
403	 */
404	setx	DCU_IPS_MASK, %g2, %o3
405	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
406	andn	%g1, %o3, %g4
407	stxa	%g4, [%g0]ASI_DCU
408	flush	%g0	/* flush required after changing the IC bit */
409	PARK_SIBLING_CORE(%g1, %o3, %o4)	! %g1 has DCU value
410
411	PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
4121:
413	clr	%o5	! assume success
414	! Check that line is in NA state; if so, INV it.
415	ldxa	[%o0]ASI_L2_TAG, %o3
416	and	%o3, 0x7, %o3
417	cmp	%o3, 0x5
418	bne,a,pt %xcc, 9f	! Wasn't NA, so something is wrong
419	 dec	%o5	! indicate not NA
420	stxa	%g0, [%o0]ASI_L2_TAG
421	membar #Sync
422	! now delay 15 cycles so we don't have hazard when we return
423	mov	16, %o1
4241:
425	brnz,pt	%o1, 1b
426	 dec	%o1
4279:
428	! UNPARK-SIBLING_CORE is 7 instructions
429	UNPARK_SIBLING_CORE(%g1, %o3, %o4)	! 7 instructions
430	/*
431	 * Restore the DCU
432	 */
433	stxa	%g1, [%g0]ASI_DCU
434	flush	%g0
435	wrpr	%g0, %o2, %pstate		!restore pstate
436	retl
437	 mov	%o5, %o0
438	SET_SIZE(unretire_l2_alternate)
439
440	ENTRY(retire_l3_alternate)
441
442	! since we disable interrupts, we don't need to do kpreempt_disable()
443	rdpr	%pstate, %o2
444	andn	%o2, PSTATE_IE, %g1
445	wrpr	%g0, %g1, %pstate		! disable interrupts
446	/*
447	 * Save current DCU state.  Turn off IPS
448	 */
449	setx	DCU_IPS_MASK, %g2, %o3
450	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
451	andn	%g1, %o3, %g4
452	stxa	%g4, [%g0]ASI_DCU
453	flush	%g0	/* flush required after changing the IC bit */
454	PARK_SIBLING_CORE(%g1, %o3, %o4)	! %g1 has DCU value
455
456	! PN-ECACHE-FLUSH_LINE is 30 instructions
457	PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
4581:
459	clr	%o5	! assume success
460	! Check if line is invalid; if so, NA it.
461	ldxa	[%o0]ASI_EC_DIAG, %o3
462	btst	0x7, %o3
463	bnz	%xcc, 2f
464	 nop
465	stxa	%o1, [%o0]ASI_EC_DIAG
466	membar #Sync	! still on same cache line
467	! now delay 15 cycles so we don't have hazard when we return
468	mov	16, %o1
4691:
470	brnz,pt	%o1, 1b
471	 dec	%o1
4729:
473	! UNPARK-SIBLING_CORE is 7 instructions, so we cross a cache boundary
474	UNPARK_SIBLING_CORE(%g1, %o3, %o4)	! 7 instructions
475	/*
476	 * Restore the DCU
477	 */
478	stxa	%g1, [%g0]ASI_DCU
479	flush	%g0
480	wrpr	%g0, %o2, %pstate		!restore pstate
481	retl
482	 mov	%o5, %o0
4832:
484	! It is OK to have STATE as NA (if so, nothing to do!)
485	and	%o3, 0x7, %o3
486	cmp	%o3, 0x5
487	be,a,pt	%xcc, 9b
488	 inc	%o5	! indicate was already NA
489	! Hmm.	Not INV, not NA
490	ba	9b
491	 dec	%o5
492	SET_SIZE(retire_l3_alternate)
493
494	ENTRY(unretire_l3_alternate)
495
496	! since we disable interrupts, we don't need to do kpreempt_disable()
497	rdpr	%pstate, %o2
498	andn	%o2, PSTATE_IE, %g1
499	wrpr	%g0, %g1, %pstate		! disable interrupts
500	/*
501	 * Save current DCU state.  Turn off IPS
502	 */
503	setx	DCU_IPS_MASK, %g2, %o3
504	ldxa	[%g0]ASI_DCU, %g1	! save DCU in %g1
505	andn	%g1, %o3, %g4
506	stxa	%g4, [%g0]ASI_DCU
507	flush	%g0	/* flush required after changing the IC bit */
508	PARK_SIBLING_CORE(%g1, %o3, %o4)	! %g1 has DCU value
509
510	PN_ECACHE_FLUSH_LINE(%o0, %o3, %o4, %o5, %g2)
5111:
512	clr	%o5	! assume success
513	! Check that line is in NA state; if so, INV it.
514	ldxa	[%o0]ASI_EC_DIAG, %o3
515	and	%o3, 0x7, %o3
516	cmp	%o3, 0x5
517	bne,a,pt %xcc, 9f	! Wasn't NA, so something is wrong
518	 dec	%o5	! indicate not NA
519	stxa	%g0, [%o0]ASI_EC_DIAG
520	membar #Sync
521	! now delay 15 cycles so we don't have hazard when we return
522	mov	16, %o1
5231:
524	brnz,pt	%o1, 1b
525	 dec	%o1
5269:
527	! UNPARK-SIBLING_CORE is 7 instructions
528	UNPARK_SIBLING_CORE(%g1, %o3, %o4)	! 7 instructions
529	/*
530	 * Restore the DCU
531	 */
532	stxa	%g1, [%g0]ASI_DCU
533	flush	%g0
534	wrpr	%g0, %o2, %pstate		!restore pstate
535	retl
536	 mov	%o5, %o0
537	SET_SIZE(unretire_l3_alternate)
538
539	ENTRY(get_ecache_dtags_tl1)
540
541
542	PARK_SIBLING_CORE(%g3, %g4, %g5)
543	add	%g2, CH_CLO_DATA + CH_CHD_EC_DATA, %g2
544	rd	%asi, %g4
545	wr	%g0, ASI_N, %asi
546	GET_ECACHE_DTAGS(%g1, %g2, %g5, %g6, %g7)
547	wr	%g4, %asi
548	UNPARK_SIBLING_CORE(%g3, %g4, %g5)	! can use %g3 again
549
550	retry
551	SET_SIZE(get_ecache_dtags_tl1)
552
553	ENTRY(get_l2_tag_tl1)
554
555	/*
556	 * Now read the tag data
557	 */
558	ldxa	[%g1]ASI_L2_TAG, %g4		! save tag_data
559	stx	%g4, [%g2]
560
561	retry
562	SET_SIZE(get_l2_tag_tl1)
563
564	ENTRY(get_l3_tag_tl1)
565
566	/*
567	 * Now read the tag data
568	 */
569	ldxa	[%g1]ASI_EC_DIAG, %g4		! save tag_data
570	stx	%g4, [%g2]
571
572	retry
573	SET_SIZE(get_l3_tag_tl1)
574
575