17c478bd9Sstevel@tonic-gate/* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 51e2e7a75Shuah * Common Development and Distribution License (the "License"). 61e2e7a75Shuah * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate/* 22bd963cb9SEthindra Ramamurthy * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate * 259b0bb795SJohn Levon * Copyright 2020 Joyent, Inc. 269b0bb795SJohn Levon * 277c478bd9Sstevel@tonic-gate * Assembly code support for Cheetah/Cheetah+ modules 287c478bd9Sstevel@tonic-gate */ 297c478bd9Sstevel@tonic-gate 307c478bd9Sstevel@tonic-gate#include "assym.h" 317c478bd9Sstevel@tonic-gate 327c478bd9Sstevel@tonic-gate#include <sys/asm_linkage.h> 337c478bd9Sstevel@tonic-gate#include <sys/mmu.h> 347c478bd9Sstevel@tonic-gate#include <vm/hat_sfmmu.h> 357c478bd9Sstevel@tonic-gate#include <sys/machparam.h> 367c478bd9Sstevel@tonic-gate#include <sys/machcpuvar.h> 377c478bd9Sstevel@tonic-gate#include <sys/machthread.h> 387c478bd9Sstevel@tonic-gate#include <sys/machtrap.h> 397c478bd9Sstevel@tonic-gate#include <sys/privregs.h> 407c478bd9Sstevel@tonic-gate#include <sys/trap.h> 417c478bd9Sstevel@tonic-gate#include <sys/cheetahregs.h> 427c478bd9Sstevel@tonic-gate#include <sys/us3_module.h> 437c478bd9Sstevel@tonic-gate#include <sys/xc_impl.h> 447c478bd9Sstevel@tonic-gate#include <sys/intreg.h> 457c478bd9Sstevel@tonic-gate#include <sys/async.h> 467c478bd9Sstevel@tonic-gate#include <sys/clock.h> 477c478bd9Sstevel@tonic-gate#include <sys/cheetahasm.h> 487c478bd9Sstevel@tonic-gate#include <sys/cmpregs.h> 497c478bd9Sstevel@tonic-gate 507c478bd9Sstevel@tonic-gate#ifdef TRAPTRACE 517c478bd9Sstevel@tonic-gate#include <sys/traptrace.h> 527c478bd9Sstevel@tonic-gate#endif /* TRAPTRACE */ 537c478bd9Sstevel@tonic-gate 547c478bd9Sstevel@tonic-gate/* BEGIN CSTYLED */ 557c478bd9Sstevel@tonic-gate 567c478bd9Sstevel@tonic-gate#define DCACHE_FLUSHPAGE(arg1, arg2, tmp1, tmp2, tmp3) \ 577c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_DCU, tmp1 ;\ 587c478bd9Sstevel@tonic-gate btst DCU_DC, tmp1 /* is dcache enabled? */ ;\ 597c478bd9Sstevel@tonic-gate bz,pn %icc, 1f ;\ 607c478bd9Sstevel@tonic-gate ASM_LD(tmp1, dcache_linesize) ;\ 617c478bd9Sstevel@tonic-gate ASM_LD(tmp2, dflush_type) ;\ 627c478bd9Sstevel@tonic-gate cmp tmp2, FLUSHPAGE_TYPE ;\ 637c478bd9Sstevel@tonic-gate be,pt %icc, 2f ;\ 647c478bd9Sstevel@tonic-gate nop ;\ 657c478bd9Sstevel@tonic-gate sllx arg1, CHEETAH_DC_VBIT_SHIFT, arg1/* tag to compare */ ;\ 667c478bd9Sstevel@tonic-gate ASM_LD(tmp3, dcache_size) ;\ 677c478bd9Sstevel@tonic-gate cmp tmp2, FLUSHMATCH_TYPE ;\ 687c478bd9Sstevel@tonic-gate be,pt %icc, 3f ;\ 697c478bd9Sstevel@tonic-gate nop ;\ 707c478bd9Sstevel@tonic-gate /* \ 717c478bd9Sstevel@tonic-gate * flushtype = FLUSHALL_TYPE, flush the whole thing \ 727c478bd9Sstevel@tonic-gate * tmp3 = cache size \ 737c478bd9Sstevel@tonic-gate * tmp1 = cache line size \ 747c478bd9Sstevel@tonic-gate */ \ 757c478bd9Sstevel@tonic-gate sub tmp3, tmp1, tmp2 ;\ 767c478bd9Sstevel@tonic-gate4: \ 777c478bd9Sstevel@tonic-gate stxa %g0, [tmp2]ASI_DC_TAG ;\ 787c478bd9Sstevel@tonic-gate membar #Sync ;\ 797c478bd9Sstevel@tonic-gate cmp %g0, tmp2 ;\ 807c478bd9Sstevel@tonic-gate bne,pt %icc, 4b ;\ 817c478bd9Sstevel@tonic-gate sub tmp2, tmp1, tmp2 ;\ 827c478bd9Sstevel@tonic-gate ba,pt %icc, 1f ;\ 837c478bd9Sstevel@tonic-gate nop ;\ 847c478bd9Sstevel@tonic-gate /* \ 857c478bd9Sstevel@tonic-gate * flushtype = FLUSHPAGE_TYPE \ 867c478bd9Sstevel@tonic-gate * arg1 = pfn \ 877c478bd9Sstevel@tonic-gate * arg2 = virtual color \ 887c478bd9Sstevel@tonic-gate * tmp1 = cache line size \ 897c478bd9Sstevel@tonic-gate * tmp2 = tag from cache \ 907c478bd9Sstevel@tonic-gate * tmp3 = counter \ 917c478bd9Sstevel@tonic-gate */ \ 927c478bd9Sstevel@tonic-gate2: \ 937c478bd9Sstevel@tonic-gate set MMU_PAGESIZE, tmp3 ;\ 947c478bd9Sstevel@tonic-gate sllx arg1, MMU_PAGESHIFT, arg1 /* pfn to 43 bit PA */ ;\ 957c478bd9Sstevel@tonic-gate sub tmp3, tmp1, tmp3 ;\ 967c478bd9Sstevel@tonic-gate4: \ 977c478bd9Sstevel@tonic-gate stxa %g0, [arg1 + tmp3]ASI_DC_INVAL ;\ 987c478bd9Sstevel@tonic-gate membar #Sync ;\ 997c478bd9Sstevel@tonic-gate5: \ 1007c478bd9Sstevel@tonic-gate cmp %g0, tmp3 ;\ 1017c478bd9Sstevel@tonic-gate bnz,pt %icc, 4b /* branch if not done */ ;\ 1027c478bd9Sstevel@tonic-gate sub tmp3, tmp1, tmp3 ;\ 1037c478bd9Sstevel@tonic-gate ba,pt %icc, 1f ;\ 1047c478bd9Sstevel@tonic-gate nop ;\ 1057c478bd9Sstevel@tonic-gate /* \ 1067c478bd9Sstevel@tonic-gate * flushtype = FLUSHMATCH_TYPE \ 1077c478bd9Sstevel@tonic-gate * arg1 = tag to compare against \ 1087c478bd9Sstevel@tonic-gate * tmp1 = cache line size \ 1097c478bd9Sstevel@tonic-gate * tmp3 = cache size \ 1107c478bd9Sstevel@tonic-gate * arg2 = counter \ 1117c478bd9Sstevel@tonic-gate * tmp2 = cache tag \ 1127c478bd9Sstevel@tonic-gate */ \ 1137c478bd9Sstevel@tonic-gate3: \ 1147c478bd9Sstevel@tonic-gate sub tmp3, tmp1, arg2 ;\ 1157c478bd9Sstevel@tonic-gate4: \ 1167c478bd9Sstevel@tonic-gate ldxa [arg2]ASI_DC_TAG, tmp2 /* read tag */ ;\ 1177c478bd9Sstevel@tonic-gate btst CHEETAH_DC_VBIT_MASK, tmp2 ;\ 1187c478bd9Sstevel@tonic-gate bz,pn %icc, 5f /* br if no valid sub-blocks */ ;\ 1197c478bd9Sstevel@tonic-gate andn tmp2, CHEETAH_DC_VBIT_MASK, tmp2 /* clear out v bits */ ;\ 1207c478bd9Sstevel@tonic-gate cmp tmp2, arg1 ;\ 1217c478bd9Sstevel@tonic-gate bne,pn %icc, 5f /* branch if tag miss */ ;\ 1227c478bd9Sstevel@tonic-gate nop ;\ 1237c478bd9Sstevel@tonic-gate stxa %g0, [arg2]ASI_DC_TAG ;\ 1247c478bd9Sstevel@tonic-gate membar #Sync ;\ 1257c478bd9Sstevel@tonic-gate5: \ 1267c478bd9Sstevel@tonic-gate cmp %g0, arg2 ;\ 1277c478bd9Sstevel@tonic-gate bne,pt %icc, 4b /* branch if not done */ ;\ 1287c478bd9Sstevel@tonic-gate sub arg2, tmp1, arg2 ;\ 1297c478bd9Sstevel@tonic-gate1: 1307c478bd9Sstevel@tonic-gate 131444ce08eSDonghai Qiao/* 132444ce08eSDonghai Qiao * macro that flushes the entire dcache color 133444ce08eSDonghai Qiao * dcache size = 64K, one way 16K 134bd963cb9SEthindra Ramamurthy * 135bd963cb9SEthindra Ramamurthy * In: 136bd963cb9SEthindra Ramamurthy * arg = virtual color register (not clobbered) 137bd963cb9SEthindra Ramamurthy * way = way#, can either be a constant or a register (not clobbered) 138bd963cb9SEthindra Ramamurthy * tmp1, tmp2, tmp3 = scratch registers 139bd963cb9SEthindra Ramamurthy * 140444ce08eSDonghai Qiao */ 141444ce08eSDonghai Qiao#define DCACHE_FLUSHCOLOR(arg, way, tmp1, tmp2, tmp3) \ 142444ce08eSDonghai Qiao ldxa [%g0]ASI_DCU, tmp1; \ 143444ce08eSDonghai Qiao btst DCU_DC, tmp1; /* is dcache enabled? */ \ 144444ce08eSDonghai Qiao bz,pn %icc, 1f; \ 145444ce08eSDonghai Qiao ASM_LD(tmp1, dcache_linesize) \ 146444ce08eSDonghai Qiao /* \ 147444ce08eSDonghai Qiao * arg = virtual color \ 148444ce08eSDonghai Qiao * tmp1 = cache line size \ 149444ce08eSDonghai Qiao */ \ 150bd963cb9SEthindra Ramamurthy sllx arg, MMU_PAGESHIFT, tmp2; /* color to dcache page */ \ 151444ce08eSDonghai Qiao mov way, tmp3; \ 152bd963cb9SEthindra Ramamurthy sllx tmp3, 14, tmp3; /* One way 16K */ \ 153bd963cb9SEthindra Ramamurthy or tmp2, tmp3, tmp3; \ 154bd963cb9SEthindra Ramamurthy set MMU_PAGESIZE, tmp2; \ 155bd963cb9SEthindra Ramamurthy /* \ 156bd963cb9SEthindra Ramamurthy * tmp2 = page size \ 157bd963cb9SEthindra Ramamurthy * tmp3 = cached page in dcache \ 158bd963cb9SEthindra Ramamurthy */ \ 159444ce08eSDonghai Qiao sub tmp2, tmp1, tmp2; \ 160444ce08eSDonghai Qiao2: \ 161bd963cb9SEthindra Ramamurthy stxa %g0, [tmp3 + tmp2]ASI_DC_TAG; \ 162444ce08eSDonghai Qiao membar #Sync; \ 163444ce08eSDonghai Qiao cmp %g0, tmp2; \ 164444ce08eSDonghai Qiao bne,pt %icc, 2b; \ 165bd963cb9SEthindra Ramamurthy sub tmp2, tmp1, tmp2; \ 166444ce08eSDonghai Qiao1: 1677c478bd9Sstevel@tonic-gate 1687c478bd9Sstevel@tonic-gate/* END CSTYLED */ 1697c478bd9Sstevel@tonic-gate 1707c478bd9Sstevel@tonic-gate/* 1717c478bd9Sstevel@tonic-gate * Cheetah MMU and Cache operations. 1727c478bd9Sstevel@tonic-gate */ 1737c478bd9Sstevel@tonic-gate 1747c478bd9Sstevel@tonic-gate ENTRY_NP(vtag_flushpage) 1757c478bd9Sstevel@tonic-gate /* 1767c478bd9Sstevel@tonic-gate * flush page from the tlb 1777c478bd9Sstevel@tonic-gate * 1787c478bd9Sstevel@tonic-gate * %o0 = vaddr 1791e2e7a75Shuah * %o1 = sfmmup 1807c478bd9Sstevel@tonic-gate */ 1817c478bd9Sstevel@tonic-gate rdpr %pstate, %o5 1827c478bd9Sstevel@tonic-gate#ifdef DEBUG 1831e2e7a75Shuah PANIC_IF_INTR_DISABLED_PSTR(%o5, u3_di_label0, %g1) 1847c478bd9Sstevel@tonic-gate#endif /* DEBUG */ 1857c478bd9Sstevel@tonic-gate /* 1867c478bd9Sstevel@tonic-gate * disable ints 1877c478bd9Sstevel@tonic-gate */ 1887c478bd9Sstevel@tonic-gate andn %o5, PSTATE_IE, %o4 1897c478bd9Sstevel@tonic-gate wrpr %o4, 0, %pstate 1907c478bd9Sstevel@tonic-gate 1917c478bd9Sstevel@tonic-gate /* 1927c478bd9Sstevel@tonic-gate * Then, blow out the tlb 1937c478bd9Sstevel@tonic-gate * Interrupts are disabled to prevent the primary ctx register 1947c478bd9Sstevel@tonic-gate * from changing underneath us. 1957c478bd9Sstevel@tonic-gate */ 1961e2e7a75Shuah sethi %hi(ksfmmup), %o3 1971e2e7a75Shuah ldx [%o3 + %lo(ksfmmup)], %o3 1981e2e7a75Shuah cmp %o3, %o1 1991e2e7a75Shuah bne,pt %xcc, 1f ! if not kernel as, go to 1 2001e2e7a75Shuah sethi %hi(FLUSH_ADDR), %o3 2017c478bd9Sstevel@tonic-gate /* 2021e2e7a75Shuah * For Kernel demaps use primary. type = page implicitly 2037c478bd9Sstevel@tonic-gate */ 2047c478bd9Sstevel@tonic-gate stxa %g0, [%o0]ASI_DTLB_DEMAP /* dmmu flush for KCONTEXT */ 2057c478bd9Sstevel@tonic-gate stxa %g0, [%o0]ASI_ITLB_DEMAP /* immu flush for KCONTEXT */ 2067c478bd9Sstevel@tonic-gate flush %o3 2071e2e7a75Shuah retl 2081e2e7a75Shuah wrpr %g0, %o5, %pstate /* enable interrupts */ 2097c478bd9Sstevel@tonic-gate1: 2107c478bd9Sstevel@tonic-gate /* 2117c478bd9Sstevel@tonic-gate * User demap. We need to set the primary context properly. 2127c478bd9Sstevel@tonic-gate * Secondary context cannot be used for Cheetah IMMU. 2137c478bd9Sstevel@tonic-gate * %o0 = vaddr 2141e2e7a75Shuah * %o1 = sfmmup 2157c478bd9Sstevel@tonic-gate * %o3 = FLUSH_ADDR 2167c478bd9Sstevel@tonic-gate */ 2171e2e7a75Shuah SFMMU_CPU_CNUM(%o1, %g1, %g2) ! %g1 = sfmmu cnum on this CPU 218*55fea89dSDan Cross 2191e2e7a75Shuah ldub [%o1 + SFMMU_CEXT], %o4 ! %o4 = sfmmup->sfmmu_cext 2207c478bd9Sstevel@tonic-gate sll %o4, CTXREG_EXT_SHIFT, %o4 221febcc4a5Sjimand or %g1, %o4, %g1 ! %g1 = primary pgsz | cnum 2221e2e7a75Shuah 2237c478bd9Sstevel@tonic-gate wrpr %g0, 1, %tl 2247c478bd9Sstevel@tonic-gate set MMU_PCONTEXT, %o4 2257c478bd9Sstevel@tonic-gate or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %o0, %o0 2261e2e7a75Shuah ldxa [%o4]ASI_DMMU, %o2 ! %o2 = save old ctxnum 227febcc4a5Sjimand srlx %o2, CTXREG_NEXT_SHIFT, %o1 ! need to preserve nucleus pgsz 228febcc4a5Sjimand sllx %o1, CTXREG_NEXT_SHIFT, %o1 ! %o1 = nucleus pgsz 229febcc4a5Sjimand or %g1, %o1, %g1 ! %g1 = nucleus pgsz | primary pgsz | cnum 230*55fea89dSDan Cross stxa %g1, [%o4]ASI_DMMU ! wr new ctxum 2311e2e7a75Shuah 2327c478bd9Sstevel@tonic-gate stxa %g0, [%o0]ASI_DTLB_DEMAP 2337c478bd9Sstevel@tonic-gate stxa %g0, [%o0]ASI_ITLB_DEMAP 2347c478bd9Sstevel@tonic-gate stxa %o2, [%o4]ASI_DMMU /* restore old ctxnum */ 2357c478bd9Sstevel@tonic-gate flush %o3 2367c478bd9Sstevel@tonic-gate wrpr %g0, 0, %tl 2371e2e7a75Shuah 2387c478bd9Sstevel@tonic-gate retl 2397c478bd9Sstevel@tonic-gate wrpr %g0, %o5, %pstate /* enable interrupts */ 2407c478bd9Sstevel@tonic-gate SET_SIZE(vtag_flushpage) 2417c478bd9Sstevel@tonic-gate 2427c478bd9Sstevel@tonic-gate ENTRY_NP2(vtag_flushall, demap_all) 2437c478bd9Sstevel@tonic-gate /* 2447c478bd9Sstevel@tonic-gate * flush the tlb 2457c478bd9Sstevel@tonic-gate */ 2467c478bd9Sstevel@tonic-gate sethi %hi(FLUSH_ADDR), %o3 2477c478bd9Sstevel@tonic-gate set DEMAP_ALL_TYPE, %g1 2487c478bd9Sstevel@tonic-gate stxa %g0, [%g1]ASI_DTLB_DEMAP 2497c478bd9Sstevel@tonic-gate stxa %g0, [%g1]ASI_ITLB_DEMAP 2507c478bd9Sstevel@tonic-gate flush %o3 2517c478bd9Sstevel@tonic-gate retl 2527c478bd9Sstevel@tonic-gate nop 2537c478bd9Sstevel@tonic-gate SET_SIZE(demap_all) 2547c478bd9Sstevel@tonic-gate SET_SIZE(vtag_flushall) 2557c478bd9Sstevel@tonic-gate 2567c478bd9Sstevel@tonic-gate 2577c478bd9Sstevel@tonic-gate ENTRY_NP(vtag_flushpage_tl1) 2587c478bd9Sstevel@tonic-gate /* 2597c478bd9Sstevel@tonic-gate * x-trap to flush page from tlb and tsb 2607c478bd9Sstevel@tonic-gate * 2617c478bd9Sstevel@tonic-gate * %g1 = vaddr, zero-extended on 32-bit kernel 2621e2e7a75Shuah * %g2 = sfmmup 2637c478bd9Sstevel@tonic-gate * 2647c478bd9Sstevel@tonic-gate * assumes TSBE_TAG = 0 2657c478bd9Sstevel@tonic-gate */ 2667c478bd9Sstevel@tonic-gate srln %g1, MMU_PAGESHIFT, %g1 267*55fea89dSDan Cross 2681e2e7a75Shuah sethi %hi(ksfmmup), %g3 2691e2e7a75Shuah ldx [%g3 + %lo(ksfmmup)], %g3 2701e2e7a75Shuah cmp %g3, %g2 2711e2e7a75Shuah bne,pt %xcc, 1f ! if not kernel as, go to 1 2727c478bd9Sstevel@tonic-gate slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ 2737c478bd9Sstevel@tonic-gate 2747c478bd9Sstevel@tonic-gate /* We need to demap in the kernel context */ 2757c478bd9Sstevel@tonic-gate or DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1 2767c478bd9Sstevel@tonic-gate stxa %g0, [%g1]ASI_DTLB_DEMAP 2777c478bd9Sstevel@tonic-gate stxa %g0, [%g1]ASI_ITLB_DEMAP 2787c478bd9Sstevel@tonic-gate retry 2797c478bd9Sstevel@tonic-gate1: 2807c478bd9Sstevel@tonic-gate /* We need to demap in a user context */ 2817c478bd9Sstevel@tonic-gate or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1 2821e2e7a75Shuah 2831e2e7a75Shuah SFMMU_CPU_CNUM(%g2, %g6, %g3) ! %g6 = sfmmu cnum on this CPU 284*55fea89dSDan Cross 2851e2e7a75Shuah ldub [%g2 + SFMMU_CEXT], %g4 ! %g4 = sfmmup->cext 2867c478bd9Sstevel@tonic-gate sll %g4, CTXREG_EXT_SHIFT, %g4 2871e2e7a75Shuah or %g6, %g4, %g6 ! %g6 = pgsz | cnum 2881e2e7a75Shuah 2897c478bd9Sstevel@tonic-gate set MMU_PCONTEXT, %g4 2907c478bd9Sstevel@tonic-gate ldxa [%g4]ASI_DMMU, %g5 /* rd old ctxnum */ 291febcc4a5Sjimand srlx %g5, CTXREG_NEXT_SHIFT, %g2 /* %g2 = nucleus pgsz */ 292febcc4a5Sjimand sllx %g2, CTXREG_NEXT_SHIFT, %g2 /* preserve nucleus pgsz */ 293febcc4a5Sjimand or %g6, %g2, %g6 /* %g6 = nucleus pgsz | primary pgsz | cnum */ 2941e2e7a75Shuah stxa %g6, [%g4]ASI_DMMU /* wr new ctxum */ 2957c478bd9Sstevel@tonic-gate stxa %g0, [%g1]ASI_DTLB_DEMAP 2967c478bd9Sstevel@tonic-gate stxa %g0, [%g1]ASI_ITLB_DEMAP 2977c478bd9Sstevel@tonic-gate stxa %g5, [%g4]ASI_DMMU /* restore old ctxnum */ 2987c478bd9Sstevel@tonic-gate retry 2997c478bd9Sstevel@tonic-gate SET_SIZE(vtag_flushpage_tl1) 3007c478bd9Sstevel@tonic-gate 3017c478bd9Sstevel@tonic-gate 3027c478bd9Sstevel@tonic-gate ENTRY_NP(vtag_flush_pgcnt_tl1) 3037c478bd9Sstevel@tonic-gate /* 3047c478bd9Sstevel@tonic-gate * x-trap to flush pgcnt MMU_PAGESIZE pages from tlb 3057c478bd9Sstevel@tonic-gate * 3067c478bd9Sstevel@tonic-gate * %g1 = vaddr, zero-extended on 32-bit kernel 3071e2e7a75Shuah * %g2 = <sfmmup58|pgcnt6>, (pgcnt - 1) is pass'ed in via pgcnt6 bits. 3087c478bd9Sstevel@tonic-gate * 3097c478bd9Sstevel@tonic-gate * NOTE: this handler relies on the fact that no 3107c478bd9Sstevel@tonic-gate * interrupts or traps can occur during the loop 3117c478bd9Sstevel@tonic-gate * issuing the TLB_DEMAP operations. It is assumed 3127c478bd9Sstevel@tonic-gate * that interrupts are disabled and this code is 3137c478bd9Sstevel@tonic-gate * fetching from the kernel locked text address. 3147c478bd9Sstevel@tonic-gate * 3157c478bd9Sstevel@tonic-gate * assumes TSBE_TAG = 0 3167c478bd9Sstevel@tonic-gate */ 3171e2e7a75Shuah set SFMMU_PGCNT_MASK, %g4 3181e2e7a75Shuah and %g4, %g2, %g3 /* g3 = pgcnt - 1 */ 3191e2e7a75Shuah add %g3, 1, %g3 /* g3 = pgcnt */ 3201e2e7a75Shuah 3211e2e7a75Shuah andn %g2, SFMMU_PGCNT_MASK, %g2 /* g2 = sfmmup */ 3227c478bd9Sstevel@tonic-gate srln %g1, MMU_PAGESHIFT, %g1 3231e2e7a75Shuah 3241e2e7a75Shuah sethi %hi(ksfmmup), %g4 3251e2e7a75Shuah ldx [%g4 + %lo(ksfmmup)], %g4 3261e2e7a75Shuah cmp %g4, %g2 3271e2e7a75Shuah bne,pn %xcc, 1f /* if not kernel as, go to 1 */ 3287c478bd9Sstevel@tonic-gate slln %g1, MMU_PAGESHIFT, %g1 /* g1 = vaddr */ 3297c478bd9Sstevel@tonic-gate 3307c478bd9Sstevel@tonic-gate /* We need to demap in the kernel context */ 3317c478bd9Sstevel@tonic-gate or DEMAP_NUCLEUS | DEMAP_PAGE_TYPE, %g1, %g1 3327c478bd9Sstevel@tonic-gate set MMU_PAGESIZE, %g2 /* g2 = pgsize */ 3331e2e7a75Shuah sethi %hi(FLUSH_ADDR), %g5 3347c478bd9Sstevel@tonic-gate4: 3357c478bd9Sstevel@tonic-gate stxa %g0, [%g1]ASI_DTLB_DEMAP 3367c478bd9Sstevel@tonic-gate stxa %g0, [%g1]ASI_ITLB_DEMAP 3371e2e7a75Shuah flush %g5 ! flush required by immu 3381e2e7a75Shuah 3397c478bd9Sstevel@tonic-gate deccc %g3 /* decr pgcnt */ 3407c478bd9Sstevel@tonic-gate bnz,pt %icc,4b 3417c478bd9Sstevel@tonic-gate add %g1, %g2, %g1 /* next page */ 3427c478bd9Sstevel@tonic-gate retry 3437c478bd9Sstevel@tonic-gate1: 3441e2e7a75Shuah /* 3451e2e7a75Shuah * We need to demap in a user context 3461e2e7a75Shuah * 3471e2e7a75Shuah * g2 = sfmmup 3481e2e7a75Shuah * g3 = pgcnt 3491e2e7a75Shuah */ 3501e2e7a75Shuah SFMMU_CPU_CNUM(%g2, %g5, %g6) ! %g5 = sfmmu cnum on this CPU 351*55fea89dSDan Cross 3521e2e7a75Shuah or DEMAP_PRIMARY | DEMAP_PAGE_TYPE, %g1, %g1 3531e2e7a75Shuah 3541e2e7a75Shuah ldub [%g2 + SFMMU_CEXT], %g4 ! %g4 = sfmmup->cext 3557c478bd9Sstevel@tonic-gate sll %g4, CTXREG_EXT_SHIFT, %g4 3561e2e7a75Shuah or %g5, %g4, %g5 3571e2e7a75Shuah 3587c478bd9Sstevel@tonic-gate set MMU_PCONTEXT, %g4 3591e2e7a75Shuah ldxa [%g4]ASI_DMMU, %g6 /* rd old ctxnum */ 360febcc4a5Sjimand srlx %g6, CTXREG_NEXT_SHIFT, %g2 /* %g2 = nucleus pgsz */ 361febcc4a5Sjimand sllx %g2, CTXREG_NEXT_SHIFT, %g2 /* preserve nucleus pgsz */ 362febcc4a5Sjimand or %g5, %g2, %g5 /* %g5 = nucleus pgsz | primary pgsz | cnum */ 3631e2e7a75Shuah stxa %g5, [%g4]ASI_DMMU /* wr new ctxum */ 3647c478bd9Sstevel@tonic-gate 3657c478bd9Sstevel@tonic-gate set MMU_PAGESIZE, %g2 /* g2 = pgsize */ 3661e2e7a75Shuah sethi %hi(FLUSH_ADDR), %g5 3677c478bd9Sstevel@tonic-gate3: 3687c478bd9Sstevel@tonic-gate stxa %g0, [%g1]ASI_DTLB_DEMAP 3697c478bd9Sstevel@tonic-gate stxa %g0, [%g1]ASI_ITLB_DEMAP 3701e2e7a75Shuah flush %g5 ! flush required by immu 3711e2e7a75Shuah 3727c478bd9Sstevel@tonic-gate deccc %g3 /* decr pgcnt */ 3737c478bd9Sstevel@tonic-gate bnz,pt %icc,3b 3747c478bd9Sstevel@tonic-gate add %g1, %g2, %g1 /* next page */ 3757c478bd9Sstevel@tonic-gate 3761e2e7a75Shuah stxa %g6, [%g4]ASI_DMMU /* restore old ctxnum */ 3777c478bd9Sstevel@tonic-gate retry 3787c478bd9Sstevel@tonic-gate SET_SIZE(vtag_flush_pgcnt_tl1) 3797c478bd9Sstevel@tonic-gate 3807c478bd9Sstevel@tonic-gate ENTRY_NP(vtag_flushall_tl1) 3817c478bd9Sstevel@tonic-gate /* 3827c478bd9Sstevel@tonic-gate * x-trap to flush tlb 3837c478bd9Sstevel@tonic-gate */ 3847c478bd9Sstevel@tonic-gate set DEMAP_ALL_TYPE, %g4 3857c478bd9Sstevel@tonic-gate stxa %g0, [%g4]ASI_DTLB_DEMAP 3867c478bd9Sstevel@tonic-gate stxa %g0, [%g4]ASI_ITLB_DEMAP 3877c478bd9Sstevel@tonic-gate retry 3887c478bd9Sstevel@tonic-gate SET_SIZE(vtag_flushall_tl1) 3897c478bd9Sstevel@tonic-gate 3907c478bd9Sstevel@tonic-gate 3917c478bd9Sstevel@tonic-gate/* 3927c478bd9Sstevel@tonic-gate * vac_flushpage(pfnum, color) 3937c478bd9Sstevel@tonic-gate * Flush 1 8k page of the D-$ with physical page = pfnum 3947c478bd9Sstevel@tonic-gate * Algorithm: 3957c478bd9Sstevel@tonic-gate * The cheetah dcache is a 64k psuedo 4 way accaociative cache. 3967c478bd9Sstevel@tonic-gate * It is virtual indexed, physically tagged cache. 3977c478bd9Sstevel@tonic-gate */ 3987c478bd9Sstevel@tonic-gate .seg ".data" 3997c478bd9Sstevel@tonic-gate .align 8 4007c478bd9Sstevel@tonic-gate .global dflush_type 4017c478bd9Sstevel@tonic-gatedflush_type: 4027c478bd9Sstevel@tonic-gate .word FLUSHPAGE_TYPE 4037c478bd9Sstevel@tonic-gate 4047c478bd9Sstevel@tonic-gate ENTRY(vac_flushpage) 4057c478bd9Sstevel@tonic-gate /* 4067c478bd9Sstevel@tonic-gate * flush page from the d$ 4077c478bd9Sstevel@tonic-gate * 4087c478bd9Sstevel@tonic-gate * %o0 = pfnum, %o1 = color 4097c478bd9Sstevel@tonic-gate */ 4107c478bd9Sstevel@tonic-gate DCACHE_FLUSHPAGE(%o0, %o1, %o2, %o3, %o4) 4117c478bd9Sstevel@tonic-gate retl 4127c478bd9Sstevel@tonic-gate nop 4137c478bd9Sstevel@tonic-gate SET_SIZE(vac_flushpage) 4147c478bd9Sstevel@tonic-gate 4157c478bd9Sstevel@tonic-gate 4167c478bd9Sstevel@tonic-gate ENTRY_NP(vac_flushpage_tl1) 4177c478bd9Sstevel@tonic-gate /* 4187c478bd9Sstevel@tonic-gate * x-trap to flush page from the d$ 4197c478bd9Sstevel@tonic-gate * 4207c478bd9Sstevel@tonic-gate * %g1 = pfnum, %g2 = color 4217c478bd9Sstevel@tonic-gate */ 4227c478bd9Sstevel@tonic-gate DCACHE_FLUSHPAGE(%g1, %g2, %g3, %g4, %g5) 4237c478bd9Sstevel@tonic-gate retry 4247c478bd9Sstevel@tonic-gate SET_SIZE(vac_flushpage_tl1) 4257c478bd9Sstevel@tonic-gate 4267c478bd9Sstevel@tonic-gate 4277c478bd9Sstevel@tonic-gate ENTRY(vac_flushcolor) 4287c478bd9Sstevel@tonic-gate /* 429444ce08eSDonghai Qiao * %o0 = vcolor 4307c478bd9Sstevel@tonic-gate */ 431444ce08eSDonghai Qiao DCACHE_FLUSHCOLOR(%o0, 0, %o1, %o2, %o3) 432444ce08eSDonghai Qiao DCACHE_FLUSHCOLOR(%o0, 1, %o1, %o2, %o3) 433444ce08eSDonghai Qiao DCACHE_FLUSHCOLOR(%o0, 2, %o1, %o2, %o3) 434444ce08eSDonghai Qiao DCACHE_FLUSHCOLOR(%o0, 3, %o1, %o2, %o3) 4357c478bd9Sstevel@tonic-gate retl 4367c478bd9Sstevel@tonic-gate nop 4377c478bd9Sstevel@tonic-gate SET_SIZE(vac_flushcolor) 4387c478bd9Sstevel@tonic-gate 4397c478bd9Sstevel@tonic-gate 4407c478bd9Sstevel@tonic-gate ENTRY(vac_flushcolor_tl1) 4417c478bd9Sstevel@tonic-gate /* 4427c478bd9Sstevel@tonic-gate * %g1 = vcolor 4437c478bd9Sstevel@tonic-gate */ 444444ce08eSDonghai Qiao DCACHE_FLUSHCOLOR(%g1, 0, %g2, %g3, %g4) 445444ce08eSDonghai Qiao DCACHE_FLUSHCOLOR(%g1, 1, %g2, %g3, %g4) 446444ce08eSDonghai Qiao DCACHE_FLUSHCOLOR(%g1, 2, %g2, %g3, %g4) 447444ce08eSDonghai Qiao DCACHE_FLUSHCOLOR(%g1, 3, %g2, %g3, %g4) 4487c478bd9Sstevel@tonic-gate retry 4497c478bd9Sstevel@tonic-gate SET_SIZE(vac_flushcolor_tl1) 4507c478bd9Sstevel@tonic-gate 4517c478bd9Sstevel@tonic-gate/* 4527c478bd9Sstevel@tonic-gate * Determine whether or not the IDSR is busy. 4537c478bd9Sstevel@tonic-gate * Entry: no arguments 4547c478bd9Sstevel@tonic-gate * Returns: 1 if busy, 0 otherwise 4557c478bd9Sstevel@tonic-gate */ 4567c478bd9Sstevel@tonic-gate ENTRY(idsr_busy) 4577c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_INTR_DISPATCH_STATUS, %g1 4587c478bd9Sstevel@tonic-gate clr %o0 4597c478bd9Sstevel@tonic-gate btst IDSR_BUSY, %g1 4607c478bd9Sstevel@tonic-gate bz,a,pt %xcc, 1f 4617c478bd9Sstevel@tonic-gate mov 1, %o0 4627c478bd9Sstevel@tonic-gate1: 4637c478bd9Sstevel@tonic-gate retl 4647c478bd9Sstevel@tonic-gate nop 4657c478bd9Sstevel@tonic-gate SET_SIZE(idsr_busy) 4667c478bd9Sstevel@tonic-gate 4677c478bd9Sstevel@tonic-gate .global _dispatch_status_busy 4687c478bd9Sstevel@tonic-gate_dispatch_status_busy: 4697c478bd9Sstevel@tonic-gate .asciz "ASI_INTR_DISPATCH_STATUS error: busy" 4707c478bd9Sstevel@tonic-gate .align 4 4717c478bd9Sstevel@tonic-gate 4727c478bd9Sstevel@tonic-gate/* 4737c478bd9Sstevel@tonic-gate * Setup interrupt dispatch data registers 4747c478bd9Sstevel@tonic-gate * Entry: 4757c478bd9Sstevel@tonic-gate * %o0 - function or inumber to call 4767c478bd9Sstevel@tonic-gate * %o1, %o2 - arguments (2 uint64_t's) 4777c478bd9Sstevel@tonic-gate */ 4787c478bd9Sstevel@tonic-gate .seg "text" 4797c478bd9Sstevel@tonic-gate 4807c478bd9Sstevel@tonic-gate ENTRY(init_mondo) 4817c478bd9Sstevel@tonic-gate#ifdef DEBUG 4827c478bd9Sstevel@tonic-gate ! 4837c478bd9Sstevel@tonic-gate ! IDSR should not be busy at the moment 4847c478bd9Sstevel@tonic-gate ! 4857c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_INTR_DISPATCH_STATUS, %g1 4867c478bd9Sstevel@tonic-gate btst IDSR_BUSY, %g1 4877c478bd9Sstevel@tonic-gate bz,pt %xcc, 1f 4887c478bd9Sstevel@tonic-gate nop 4897c478bd9Sstevel@tonic-gate sethi %hi(_dispatch_status_busy), %o0 4907c478bd9Sstevel@tonic-gate call panic 4917c478bd9Sstevel@tonic-gate or %o0, %lo(_dispatch_status_busy), %o0 4927c478bd9Sstevel@tonic-gate#endif /* DEBUG */ 4937c478bd9Sstevel@tonic-gate 4947c478bd9Sstevel@tonic-gate ALTENTRY(init_mondo_nocheck) 4957c478bd9Sstevel@tonic-gate ! 4967c478bd9Sstevel@tonic-gate ! interrupt vector dispatch data reg 0 4977c478bd9Sstevel@tonic-gate ! 4987c478bd9Sstevel@tonic-gate1: 4997c478bd9Sstevel@tonic-gate mov IDDR_0, %g1 5007c478bd9Sstevel@tonic-gate mov IDDR_1, %g2 5017c478bd9Sstevel@tonic-gate mov IDDR_2, %g3 5027c478bd9Sstevel@tonic-gate stxa %o0, [%g1]ASI_INTR_DISPATCH 5037c478bd9Sstevel@tonic-gate 5047c478bd9Sstevel@tonic-gate ! 5057c478bd9Sstevel@tonic-gate ! interrupt vector dispatch data reg 1 5067c478bd9Sstevel@tonic-gate ! 5077c478bd9Sstevel@tonic-gate stxa %o1, [%g2]ASI_INTR_DISPATCH 5087c478bd9Sstevel@tonic-gate 5097c478bd9Sstevel@tonic-gate ! 5107c478bd9Sstevel@tonic-gate ! interrupt vector dispatch data reg 2 5117c478bd9Sstevel@tonic-gate ! 5127c478bd9Sstevel@tonic-gate stxa %o2, [%g3]ASI_INTR_DISPATCH 5137c478bd9Sstevel@tonic-gate 5147c478bd9Sstevel@tonic-gate membar #Sync 5157c478bd9Sstevel@tonic-gate retl 5167c478bd9Sstevel@tonic-gate nop 5177c478bd9Sstevel@tonic-gate SET_SIZE(init_mondo_nocheck) 5187c478bd9Sstevel@tonic-gate SET_SIZE(init_mondo) 5197c478bd9Sstevel@tonic-gate 5207c478bd9Sstevel@tonic-gate 5217c478bd9Sstevel@tonic-gate#if !(defined(JALAPENO) || defined(SERRANO)) 5227c478bd9Sstevel@tonic-gate 5237c478bd9Sstevel@tonic-gate/* 5247c478bd9Sstevel@tonic-gate * Ship mondo to aid using busy/nack pair bn 5257c478bd9Sstevel@tonic-gate */ 5267c478bd9Sstevel@tonic-gate ENTRY_NP(shipit) 5277c478bd9Sstevel@tonic-gate sll %o0, IDCR_PID_SHIFT, %g1 ! IDCR<18:14> = agent id 5287c478bd9Sstevel@tonic-gate sll %o1, IDCR_BN_SHIFT, %g2 ! IDCR<28:24> = b/n pair 5297c478bd9Sstevel@tonic-gate or %g1, IDCR_OFFSET, %g1 ! IDCR<13:0> = 0x70 5307c478bd9Sstevel@tonic-gate or %g1, %g2, %g1 5317c478bd9Sstevel@tonic-gate stxa %g0, [%g1]ASI_INTR_DISPATCH ! interrupt vector dispatch 5327c478bd9Sstevel@tonic-gate membar #Sync 5337c478bd9Sstevel@tonic-gate retl 5347c478bd9Sstevel@tonic-gate nop 5357c478bd9Sstevel@tonic-gate SET_SIZE(shipit) 5367c478bd9Sstevel@tonic-gate 5377c478bd9Sstevel@tonic-gate#endif /* !(JALAPENO || SERRANO) */ 5387c478bd9Sstevel@tonic-gate 5397c478bd9Sstevel@tonic-gate 5407c478bd9Sstevel@tonic-gate/* 5417c478bd9Sstevel@tonic-gate * flush_instr_mem: 5427c478bd9Sstevel@tonic-gate * Flush 1 page of the I-$ starting at vaddr 5437c478bd9Sstevel@tonic-gate * %o0 vaddr 5447c478bd9Sstevel@tonic-gate * %o1 bytes to be flushed 5457c478bd9Sstevel@tonic-gate * UltraSPARC-III maintains consistency of the on-chip Instruction Cache with 5467c478bd9Sstevel@tonic-gate * the stores from all processors so that a FLUSH instruction is only needed 5477c478bd9Sstevel@tonic-gate * to ensure pipeline is consistent. This means a single flush is sufficient at 5487c478bd9Sstevel@tonic-gate * the end of a sequence of stores that updates the instruction stream to 5497c478bd9Sstevel@tonic-gate * ensure correct operation. 5507c478bd9Sstevel@tonic-gate */ 5517c478bd9Sstevel@tonic-gate 5527c478bd9Sstevel@tonic-gate ENTRY(flush_instr_mem) 55363360950Smp flush %o0 ! address irrelevant 5547c478bd9Sstevel@tonic-gate retl 5557c478bd9Sstevel@tonic-gate nop 5567c478bd9Sstevel@tonic-gate SET_SIZE(flush_instr_mem) 5577c478bd9Sstevel@tonic-gate 5587c478bd9Sstevel@tonic-gate 5597c478bd9Sstevel@tonic-gate#if defined(CPU_IMP_ECACHE_ASSOC) 5607c478bd9Sstevel@tonic-gate 5617c478bd9Sstevel@tonic-gate ENTRY(get_ecache_ctrl) 5627c478bd9Sstevel@tonic-gate GET_CPU_IMPL(%o0) 5637c478bd9Sstevel@tonic-gate cmp %o0, JAGUAR_IMPL 5647c478bd9Sstevel@tonic-gate ! 5657c478bd9Sstevel@tonic-gate ! Putting an ASI access in the delay slot may 5667c478bd9Sstevel@tonic-gate ! cause it to be accessed, even when annulled. 5677c478bd9Sstevel@tonic-gate ! 5687c478bd9Sstevel@tonic-gate bne 1f 5697c478bd9Sstevel@tonic-gate nop 5707c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_EC_CFG_TIMING, %o0 ! read Jaguar shared E$ ctrl reg 5717c478bd9Sstevel@tonic-gate b 2f 5727c478bd9Sstevel@tonic-gate nop 573*55fea89dSDan Cross1: 5747c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_EC_CTRL, %o0 ! read Ch/Ch+ E$ control reg 5757c478bd9Sstevel@tonic-gate2: 5767c478bd9Sstevel@tonic-gate retl 5777c478bd9Sstevel@tonic-gate nop 5787c478bd9Sstevel@tonic-gate SET_SIZE(get_ecache_ctrl) 5797c478bd9Sstevel@tonic-gate 5807c478bd9Sstevel@tonic-gate#endif /* CPU_IMP_ECACHE_ASSOC */ 5817c478bd9Sstevel@tonic-gate 5827c478bd9Sstevel@tonic-gate 5837c478bd9Sstevel@tonic-gate#if !(defined(JALAPENO) || defined(SERRANO)) 5847c478bd9Sstevel@tonic-gate 5857c478bd9Sstevel@tonic-gate/* 5867c478bd9Sstevel@tonic-gate * flush_ecache: 5877c478bd9Sstevel@tonic-gate * %o0 - 64 bit physical address 5887c478bd9Sstevel@tonic-gate * %o1 - ecache size 5897c478bd9Sstevel@tonic-gate * %o2 - ecache linesize 5907c478bd9Sstevel@tonic-gate */ 5917c478bd9Sstevel@tonic-gate 5927c478bd9Sstevel@tonic-gate ENTRY(flush_ecache) 5937c478bd9Sstevel@tonic-gate 5947c478bd9Sstevel@tonic-gate /* 5957c478bd9Sstevel@tonic-gate * For certain CPU implementations, we have to flush the L2 cache 5967c478bd9Sstevel@tonic-gate * before flushing the ecache. 5977c478bd9Sstevel@tonic-gate */ 5987c478bd9Sstevel@tonic-gate PN_L2_FLUSHALL(%g3, %g4, %g5) 5997c478bd9Sstevel@tonic-gate 6007c478bd9Sstevel@tonic-gate /* 6017c478bd9Sstevel@tonic-gate * Flush the entire Ecache using displacement flush. 6027c478bd9Sstevel@tonic-gate */ 6037c478bd9Sstevel@tonic-gate ECACHE_FLUSHALL(%o1, %o2, %o0, %o4) 6047c478bd9Sstevel@tonic-gate 6057c478bd9Sstevel@tonic-gate retl 6067c478bd9Sstevel@tonic-gate nop 6077c478bd9Sstevel@tonic-gate SET_SIZE(flush_ecache) 6087c478bd9Sstevel@tonic-gate 6097c478bd9Sstevel@tonic-gate#endif /* !(JALAPENO || SERRANO) */ 6107c478bd9Sstevel@tonic-gate 6117c478bd9Sstevel@tonic-gate 6127c478bd9Sstevel@tonic-gate ENTRY(flush_dcache) 6137c478bd9Sstevel@tonic-gate ASM_LD(%o0, dcache_size) 6147c478bd9Sstevel@tonic-gate ASM_LD(%o1, dcache_linesize) 6157c478bd9Sstevel@tonic-gate CH_DCACHE_FLUSHALL(%o0, %o1, %o2) 6167c478bd9Sstevel@tonic-gate retl 6177c478bd9Sstevel@tonic-gate nop 6187c478bd9Sstevel@tonic-gate SET_SIZE(flush_dcache) 6197c478bd9Sstevel@tonic-gate 6207c478bd9Sstevel@tonic-gate 6217c478bd9Sstevel@tonic-gate ENTRY(flush_icache) 6227c478bd9Sstevel@tonic-gate GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, flush_icache_1); 6237c478bd9Sstevel@tonic-gate ld [%o0 + CHPR_ICACHE_LINESIZE], %o1 6247c478bd9Sstevel@tonic-gate ba,pt %icc, 2f 6257c478bd9Sstevel@tonic-gate ld [%o0 + CHPR_ICACHE_SIZE], %o0 6267c478bd9Sstevel@tonic-gateflush_icache_1: 6277c478bd9Sstevel@tonic-gate ASM_LD(%o0, icache_size) 6287c478bd9Sstevel@tonic-gate ASM_LD(%o1, icache_linesize) 6297c478bd9Sstevel@tonic-gate2: 6307c478bd9Sstevel@tonic-gate CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4) 6317c478bd9Sstevel@tonic-gate retl 6327c478bd9Sstevel@tonic-gate nop 6337c478bd9Sstevel@tonic-gate SET_SIZE(flush_icache) 6347c478bd9Sstevel@tonic-gate 6357c478bd9Sstevel@tonic-gate ENTRY(kdi_flush_idcache) 6367c478bd9Sstevel@tonic-gate CH_DCACHE_FLUSHALL(%o0, %o1, %g1) 6377c478bd9Sstevel@tonic-gate CH_ICACHE_FLUSHALL(%o2, %o3, %g1, %g2) 6387c478bd9Sstevel@tonic-gate membar #Sync 6397c478bd9Sstevel@tonic-gate retl 6407c478bd9Sstevel@tonic-gate nop 6417c478bd9Sstevel@tonic-gate SET_SIZE(kdi_flush_idcache) 6427c478bd9Sstevel@tonic-gate 6437c478bd9Sstevel@tonic-gate ENTRY(flush_pcache) 6447c478bd9Sstevel@tonic-gate PCACHE_FLUSHALL(%o0, %o1, %o2) 6457c478bd9Sstevel@tonic-gate retl 6467c478bd9Sstevel@tonic-gate nop 6477c478bd9Sstevel@tonic-gate SET_SIZE(flush_pcache) 6487c478bd9Sstevel@tonic-gate 6497c478bd9Sstevel@tonic-gate 6507c478bd9Sstevel@tonic-gate#if defined(CPU_IMP_L1_CACHE_PARITY) 6517c478bd9Sstevel@tonic-gate 6527c478bd9Sstevel@tonic-gate/* 6537c478bd9Sstevel@tonic-gate * Get dcache data and tag. The Dcache data is a pointer to a ch_dc_data_t 6547c478bd9Sstevel@tonic-gate * structure (see cheetahregs.h): 6557c478bd9Sstevel@tonic-gate * The Dcache *should* be turned off when this code is executed. 6567c478bd9Sstevel@tonic-gate */ 6577c478bd9Sstevel@tonic-gate .align 128 6587c478bd9Sstevel@tonic-gate ENTRY(get_dcache_dtag) 6597c478bd9Sstevel@tonic-gate rdpr %pstate, %o5 6607c478bd9Sstevel@tonic-gate andn %o5, PSTATE_IE | PSTATE_AM, %o3 6617c478bd9Sstevel@tonic-gate wrpr %g0, %o3, %pstate 6627c478bd9Sstevel@tonic-gate b 1f 6637c478bd9Sstevel@tonic-gate stx %o0, [%o1 + CH_DC_IDX] 6647c478bd9Sstevel@tonic-gate 6657c478bd9Sstevel@tonic-gate .align 128 6667c478bd9Sstevel@tonic-gate1: 6677c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_DC_TAG, %o2 6687c478bd9Sstevel@tonic-gate stx %o2, [%o1 + CH_DC_TAG] 6697c478bd9Sstevel@tonic-gate membar #Sync 6707c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_DC_UTAG, %o2 6717c478bd9Sstevel@tonic-gate membar #Sync 6727c478bd9Sstevel@tonic-gate stx %o2, [%o1 + CH_DC_UTAG] 6737c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_DC_SNP_TAG, %o2 6747c478bd9Sstevel@tonic-gate stx %o2, [%o1 + CH_DC_SNTAG] 6757c478bd9Sstevel@tonic-gate add %o1, CH_DC_DATA, %o1 6767c478bd9Sstevel@tonic-gate clr %o3 6777c478bd9Sstevel@tonic-gate2: 6787c478bd9Sstevel@tonic-gate membar #Sync ! required before ASI_DC_DATA 6797c478bd9Sstevel@tonic-gate ldxa [%o0 + %o3]ASI_DC_DATA, %o2 6807c478bd9Sstevel@tonic-gate membar #Sync ! required after ASI_DC_DATA 6817c478bd9Sstevel@tonic-gate stx %o2, [%o1 + %o3] 6827c478bd9Sstevel@tonic-gate cmp %o3, CH_DC_DATA_REG_SIZE - 8 6837c478bd9Sstevel@tonic-gate blt 2b 6847c478bd9Sstevel@tonic-gate add %o3, 8, %o3 6857c478bd9Sstevel@tonic-gate 6867c478bd9Sstevel@tonic-gate /* 6877c478bd9Sstevel@tonic-gate * Unlike other CPUs in the family, D$ data parity bits for Panther 6887c478bd9Sstevel@tonic-gate * do not reside in the microtag. Instead, we have to read them 6897c478bd9Sstevel@tonic-gate * using the DC_data_parity bit of ASI_DCACHE_DATA. Also, instead 6907c478bd9Sstevel@tonic-gate * of just having 8 parity bits to protect all 32 bytes of data 6917c478bd9Sstevel@tonic-gate * per line, we now have 32 bits of parity. 6927c478bd9Sstevel@tonic-gate */ 6937c478bd9Sstevel@tonic-gate GET_CPU_IMPL(%o3) 6947c478bd9Sstevel@tonic-gate cmp %o3, PANTHER_IMPL 6957c478bd9Sstevel@tonic-gate bne 4f 6967c478bd9Sstevel@tonic-gate clr %o3 6977c478bd9Sstevel@tonic-gate 6987c478bd9Sstevel@tonic-gate /* 6997c478bd9Sstevel@tonic-gate * move our pointer to the next field where we store parity bits 7007c478bd9Sstevel@tonic-gate * and add the offset of the last parity byte since we will be 7017c478bd9Sstevel@tonic-gate * storing all 4 parity bytes within one 64 bit field like this: 7027c478bd9Sstevel@tonic-gate * 7037c478bd9Sstevel@tonic-gate * +------+------------+------------+------------+------------+ 7047c478bd9Sstevel@tonic-gate * | - | DC_parity | DC_parity | DC_parity | DC_parity | 7057c478bd9Sstevel@tonic-gate * | - | for word 3 | for word 2 | for word 1 | for word 0 | 7067c478bd9Sstevel@tonic-gate * +------+------------+------------+------------+------------+ 707*55fea89dSDan Cross * 63:32 31:24 23:16 15:8 7:0 7087c478bd9Sstevel@tonic-gate */ 7097c478bd9Sstevel@tonic-gate add %o1, CH_DC_PN_DATA_PARITY - CH_DC_DATA + 7, %o1 7107c478bd9Sstevel@tonic-gate 7117c478bd9Sstevel@tonic-gate /* add the DC_data_parity bit into our working index */ 7127c478bd9Sstevel@tonic-gate mov 1, %o2 7137c478bd9Sstevel@tonic-gate sll %o2, PN_DC_DATA_PARITY_BIT_SHIFT, %o2 7147c478bd9Sstevel@tonic-gate or %o0, %o2, %o0 7157c478bd9Sstevel@tonic-gate3: 7167c478bd9Sstevel@tonic-gate membar #Sync ! required before ASI_DC_DATA 7177c478bd9Sstevel@tonic-gate ldxa [%o0 + %o3]ASI_DC_DATA, %o2 7187c478bd9Sstevel@tonic-gate membar #Sync ! required after ASI_DC_DATA 7197c478bd9Sstevel@tonic-gate stb %o2, [%o1] 7207c478bd9Sstevel@tonic-gate dec %o1 7217c478bd9Sstevel@tonic-gate cmp %o3, CH_DC_DATA_REG_SIZE - 8 7227c478bd9Sstevel@tonic-gate blt 3b 7237c478bd9Sstevel@tonic-gate add %o3, 8, %o3 7247c478bd9Sstevel@tonic-gate4: 7257c478bd9Sstevel@tonic-gate retl 726*55fea89dSDan Cross wrpr %g0, %o5, %pstate 7277c478bd9Sstevel@tonic-gate SET_SIZE(get_dcache_dtag) 7287c478bd9Sstevel@tonic-gate 7297c478bd9Sstevel@tonic-gate 7307c478bd9Sstevel@tonic-gate/* 7317c478bd9Sstevel@tonic-gate * Get icache data and tag. The data argument is a pointer to a ch_ic_data_t 7327c478bd9Sstevel@tonic-gate * structure (see cheetahregs.h): 7337c478bd9Sstevel@tonic-gate * The Icache *Must* be turned off when this function is called. 7347c478bd9Sstevel@tonic-gate * This is because diagnostic accesses to the Icache interfere with cache 7357c478bd9Sstevel@tonic-gate * consistency. 7367c478bd9Sstevel@tonic-gate */ 7377c478bd9Sstevel@tonic-gate .align 128 7387c478bd9Sstevel@tonic-gate ENTRY(get_icache_dtag) 7397c478bd9Sstevel@tonic-gate rdpr %pstate, %o5 7407c478bd9Sstevel@tonic-gate andn %o5, PSTATE_IE | PSTATE_AM, %o3 7417c478bd9Sstevel@tonic-gate wrpr %g0, %o3, %pstate 7427c478bd9Sstevel@tonic-gate 7437c478bd9Sstevel@tonic-gate stx %o0, [%o1 + CH_IC_IDX] 7447c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_IC_TAG, %o2 7457c478bd9Sstevel@tonic-gate stx %o2, [%o1 + CH_IC_PATAG] 7467c478bd9Sstevel@tonic-gate add %o0, CH_ICTAG_UTAG, %o0 7477c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_IC_TAG, %o2 7487c478bd9Sstevel@tonic-gate add %o0, (CH_ICTAG_UPPER - CH_ICTAG_UTAG), %o0 7497c478bd9Sstevel@tonic-gate stx %o2, [%o1 + CH_IC_UTAG] 7507c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_IC_TAG, %o2 7517c478bd9Sstevel@tonic-gate add %o0, (CH_ICTAG_LOWER - CH_ICTAG_UPPER), %o0 7527c478bd9Sstevel@tonic-gate stx %o2, [%o1 + CH_IC_UPPER] 7537c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_IC_TAG, %o2 7547c478bd9Sstevel@tonic-gate andn %o0, CH_ICTAG_TMASK, %o0 7557c478bd9Sstevel@tonic-gate stx %o2, [%o1 + CH_IC_LOWER] 7567c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_IC_SNP_TAG, %o2 7577c478bd9Sstevel@tonic-gate stx %o2, [%o1 + CH_IC_SNTAG] 7587c478bd9Sstevel@tonic-gate add %o1, CH_IC_DATA, %o1 7597c478bd9Sstevel@tonic-gate clr %o3 7607c478bd9Sstevel@tonic-gate2: 7617c478bd9Sstevel@tonic-gate ldxa [%o0 + %o3]ASI_IC_DATA, %o2 7627c478bd9Sstevel@tonic-gate stx %o2, [%o1 + %o3] 7637c478bd9Sstevel@tonic-gate cmp %o3, PN_IC_DATA_REG_SIZE - 8 7647c478bd9Sstevel@tonic-gate blt 2b 7657c478bd9Sstevel@tonic-gate add %o3, 8, %o3 7667c478bd9Sstevel@tonic-gate 7677c478bd9Sstevel@tonic-gate retl 768*55fea89dSDan Cross wrpr %g0, %o5, %pstate 7697c478bd9Sstevel@tonic-gate SET_SIZE(get_icache_dtag) 7707c478bd9Sstevel@tonic-gate 7717c478bd9Sstevel@tonic-gate/* 7727c478bd9Sstevel@tonic-gate * Get pcache data and tags. 7737c478bd9Sstevel@tonic-gate * inputs: 7747c478bd9Sstevel@tonic-gate * pcache_idx - fully constructed VA for for accessing P$ diagnostic 7757c478bd9Sstevel@tonic-gate * registers. Contains PC_way and PC_addr shifted into 7767c478bd9Sstevel@tonic-gate * the correct bit positions. See the PRM for more details. 7777c478bd9Sstevel@tonic-gate * data - pointer to a ch_pc_data_t 7787c478bd9Sstevel@tonic-gate * structure (see cheetahregs.h): 7797c478bd9Sstevel@tonic-gate */ 7807c478bd9Sstevel@tonic-gate .align 128 7817c478bd9Sstevel@tonic-gate ENTRY(get_pcache_dtag) 7827c478bd9Sstevel@tonic-gate rdpr %pstate, %o5 7837c478bd9Sstevel@tonic-gate andn %o5, PSTATE_IE | PSTATE_AM, %o3 7847c478bd9Sstevel@tonic-gate wrpr %g0, %o3, %pstate 7857c478bd9Sstevel@tonic-gate 7867c478bd9Sstevel@tonic-gate stx %o0, [%o1 + CH_PC_IDX] 7877c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_PC_STATUS_DATA, %o2 7887c478bd9Sstevel@tonic-gate stx %o2, [%o1 + CH_PC_STATUS] 7897c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_PC_TAG, %o2 7907c478bd9Sstevel@tonic-gate stx %o2, [%o1 + CH_PC_TAG] 7917c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_PC_SNP_TAG, %o2 7927c478bd9Sstevel@tonic-gate stx %o2, [%o1 + CH_PC_SNTAG] 7937c478bd9Sstevel@tonic-gate add %o1, CH_PC_DATA, %o1 7947c478bd9Sstevel@tonic-gate clr %o3 7957c478bd9Sstevel@tonic-gate2: 7967c478bd9Sstevel@tonic-gate ldxa [%o0 + %o3]ASI_PC_DATA, %o2 7977c478bd9Sstevel@tonic-gate stx %o2, [%o1 + %o3] 7987c478bd9Sstevel@tonic-gate cmp %o3, CH_PC_DATA_REG_SIZE - 8 7997c478bd9Sstevel@tonic-gate blt 2b 8007c478bd9Sstevel@tonic-gate add %o3, 8, %o3 8017c478bd9Sstevel@tonic-gate 8027c478bd9Sstevel@tonic-gate retl 803*55fea89dSDan Cross wrpr %g0, %o5, %pstate 8047c478bd9Sstevel@tonic-gate SET_SIZE(get_pcache_dtag) 8057c478bd9Sstevel@tonic-gate 8067c478bd9Sstevel@tonic-gate#endif /* CPU_IMP_L1_CACHE_PARITY */ 8077c478bd9Sstevel@tonic-gate 8087c478bd9Sstevel@tonic-gate/* 8097c478bd9Sstevel@tonic-gate * re-enable the i$, d$, w$, and p$ according to bootup cache state. 8107c478bd9Sstevel@tonic-gate * Turn on WE, HPE, SPE, PE, IC, and DC bits defined as DCU_CACHE. 8117c478bd9Sstevel@tonic-gate * %o0 - 64 bit constant 8127c478bd9Sstevel@tonic-gate */ 8137c478bd9Sstevel@tonic-gate ENTRY(set_dcu) 8147c478bd9Sstevel@tonic-gate stxa %o0, [%g0]ASI_DCU ! Store to DCU 8157c478bd9Sstevel@tonic-gate flush %g0 /* flush required after changing the IC bit */ 8167c478bd9Sstevel@tonic-gate retl 8177c478bd9Sstevel@tonic-gate nop 8187c478bd9Sstevel@tonic-gate SET_SIZE(set_dcu) 8197c478bd9Sstevel@tonic-gate 8207c478bd9Sstevel@tonic-gate 8217c478bd9Sstevel@tonic-gate/* 8227c478bd9Sstevel@tonic-gate * Return DCU register. 8237c478bd9Sstevel@tonic-gate */ 8247c478bd9Sstevel@tonic-gate ENTRY(get_dcu) 8257c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_DCU, %o0 /* DCU control register */ 8267c478bd9Sstevel@tonic-gate retl 8277c478bd9Sstevel@tonic-gate nop 8287c478bd9Sstevel@tonic-gate SET_SIZE(get_dcu) 8297c478bd9Sstevel@tonic-gate 8307c478bd9Sstevel@tonic-gate/* 8317c478bd9Sstevel@tonic-gate * Cheetah/Cheetah+ level 15 interrupt handler trap table entry. 8327c478bd9Sstevel@tonic-gate * 8337c478bd9Sstevel@tonic-gate * This handler is used to check for softints generated by error trap 8347c478bd9Sstevel@tonic-gate * handlers to report errors. On Cheetah, this mechanism is used by the 8357c478bd9Sstevel@tonic-gate * Fast ECC at TL>0 error trap handler and, on Cheetah+, by both the Fast 8367c478bd9Sstevel@tonic-gate * ECC at TL>0 error and the I$/D$ parity error at TL>0 trap handlers. 8377c478bd9Sstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must 8387c478bd9Sstevel@tonic-gate * be relocatable. 8397c478bd9Sstevel@tonic-gate */ 8407c478bd9Sstevel@tonic-gate 8417c478bd9Sstevel@tonic-gate ENTRY_NP(ch_pil15_interrupt_instr) 8427c478bd9Sstevel@tonic-gate ASM_JMP(%g1, ch_pil15_interrupt) 8437c478bd9Sstevel@tonic-gate SET_SIZE(ch_pil15_interrupt_instr) 8447c478bd9Sstevel@tonic-gate 8457c478bd9Sstevel@tonic-gate 8467c478bd9Sstevel@tonic-gate ENTRY_NP(ch_pil15_interrupt) 8477c478bd9Sstevel@tonic-gate 8487c478bd9Sstevel@tonic-gate /* 8497c478bd9Sstevel@tonic-gate * Since pil_interrupt is hacked to assume that every level 15 8507c478bd9Sstevel@tonic-gate * interrupt is generated by the CPU to indicate a performance 8517c478bd9Sstevel@tonic-gate * counter overflow this gets ugly. Before calling pil_interrupt 8527c478bd9Sstevel@tonic-gate * the Error at TL>0 pending status is inspected. If it is 8537c478bd9Sstevel@tonic-gate * non-zero, then an error has occurred and it is handled. 8547c478bd9Sstevel@tonic-gate * Otherwise control is transfered to pil_interrupt. Note that if 8557c478bd9Sstevel@tonic-gate * an error is detected pil_interrupt will not be called and 8567c478bd9Sstevel@tonic-gate * overflow interrupts may be lost causing erroneous performance 8577c478bd9Sstevel@tonic-gate * measurements. However, error-recovery will have a detrimental 8587c478bd9Sstevel@tonic-gate * effect on performance anyway. 8597c478bd9Sstevel@tonic-gate */ 8607c478bd9Sstevel@tonic-gate CPU_INDEX(%g1, %g4) 8617c478bd9Sstevel@tonic-gate set ch_err_tl1_pending, %g4 8627c478bd9Sstevel@tonic-gate ldub [%g1 + %g4], %g2 8637c478bd9Sstevel@tonic-gate brz %g2, 1f 8647c478bd9Sstevel@tonic-gate nop 8657c478bd9Sstevel@tonic-gate 8667c478bd9Sstevel@tonic-gate /* 8677c478bd9Sstevel@tonic-gate * We have a pending TL>0 error, clear the TL>0 pending status. 8687c478bd9Sstevel@tonic-gate */ 8697c478bd9Sstevel@tonic-gate stb %g0, [%g1 + %g4] 8707c478bd9Sstevel@tonic-gate 8717c478bd9Sstevel@tonic-gate /* 8727c478bd9Sstevel@tonic-gate * Clear the softint. 8737c478bd9Sstevel@tonic-gate */ 8747c478bd9Sstevel@tonic-gate mov 1, %g5 8757c478bd9Sstevel@tonic-gate sll %g5, PIL_15, %g5 8767c478bd9Sstevel@tonic-gate wr %g5, CLEAR_SOFTINT 8777c478bd9Sstevel@tonic-gate 8787c478bd9Sstevel@tonic-gate /* 8797c478bd9Sstevel@tonic-gate * For Cheetah*, call cpu_tl1_error via systrap at PIL 15 8807c478bd9Sstevel@tonic-gate * to process the Fast ECC/Cache Parity at TL>0 error. Clear 8817c478bd9Sstevel@tonic-gate * panic flag (%g2). 8827c478bd9Sstevel@tonic-gate */ 8837c478bd9Sstevel@tonic-gate set cpu_tl1_error, %g1 8847c478bd9Sstevel@tonic-gate clr %g2 8857c478bd9Sstevel@tonic-gate ba sys_trap 8867c478bd9Sstevel@tonic-gate mov PIL_15, %g4 8877c478bd9Sstevel@tonic-gate 8887c478bd9Sstevel@tonic-gate1: 8897c478bd9Sstevel@tonic-gate /* 8907c478bd9Sstevel@tonic-gate * The logout is invalid. 8917c478bd9Sstevel@tonic-gate * 8927c478bd9Sstevel@tonic-gate * Call the default interrupt handler. 8937c478bd9Sstevel@tonic-gate */ 8947c478bd9Sstevel@tonic-gate sethi %hi(pil_interrupt), %g1 8957c478bd9Sstevel@tonic-gate jmp %g1 + %lo(pil_interrupt) 8967c478bd9Sstevel@tonic-gate mov PIL_15, %g4 8977c478bd9Sstevel@tonic-gate 8987c478bd9Sstevel@tonic-gate SET_SIZE(ch_pil15_interrupt) 8997c478bd9Sstevel@tonic-gate 9007c478bd9Sstevel@tonic-gate 9017c478bd9Sstevel@tonic-gate/* 9027c478bd9Sstevel@tonic-gate * Error Handling 9037c478bd9Sstevel@tonic-gate * 9047c478bd9Sstevel@tonic-gate * Cheetah provides error checking for all memory access paths between 9057c478bd9Sstevel@tonic-gate * the CPU, External Cache, Cheetah Data Switch and system bus. Error 9067c478bd9Sstevel@tonic-gate * information is logged in the AFSR, (also AFSR_EXT for Panther) and 9077c478bd9Sstevel@tonic-gate * AFAR and one of the following traps is generated (provided that it 9087c478bd9Sstevel@tonic-gate * is enabled in External Cache Error Enable Register) to handle that 9097c478bd9Sstevel@tonic-gate * error: 910*55fea89dSDan Cross * 1. trap 0x70: Precise trap 9117c478bd9Sstevel@tonic-gate * tt0_fecc for errors at trap level(TL)>=0 9127c478bd9Sstevel@tonic-gate * 2. trap 0x0A and 0x32: Deferred trap 9137c478bd9Sstevel@tonic-gate * async_err for errors at TL>=0 9147c478bd9Sstevel@tonic-gate * 3. trap 0x63: Disrupting trap 9157c478bd9Sstevel@tonic-gate * ce_err for errors at TL=0 9167c478bd9Sstevel@tonic-gate * (Note that trap 0x63 cannot happen at trap level > 0) 9177c478bd9Sstevel@tonic-gate * 9187c478bd9Sstevel@tonic-gate * Trap level one handlers panic the system except for the fast ecc 9197c478bd9Sstevel@tonic-gate * error handler which tries to recover from certain errors. 9207c478bd9Sstevel@tonic-gate */ 9217c478bd9Sstevel@tonic-gate 9227c478bd9Sstevel@tonic-gate/* 9237c478bd9Sstevel@tonic-gate * FAST ECC TRAP STRATEGY: 9247c478bd9Sstevel@tonic-gate * 9257c478bd9Sstevel@tonic-gate * Software must handle single and multi bit errors which occur due to data 9267c478bd9Sstevel@tonic-gate * or instruction cache reads from the external cache. A single or multi bit 9277c478bd9Sstevel@tonic-gate * error occuring in one of these situations results in a precise trap. 9287c478bd9Sstevel@tonic-gate * 9297c478bd9Sstevel@tonic-gate * The basic flow of this trap handler is as follows: 9307c478bd9Sstevel@tonic-gate * 9317c478bd9Sstevel@tonic-gate * 1) Record the state and then turn off the Dcache and Icache. The Dcache 9327c478bd9Sstevel@tonic-gate * is disabled because bad data could have been installed. The Icache is 9337c478bd9Sstevel@tonic-gate * turned off because we want to capture the Icache line related to the 9347c478bd9Sstevel@tonic-gate * AFAR. 9357c478bd9Sstevel@tonic-gate * 2) Disable trapping on CEEN/NCCEN errors during TL=0 processing. 9367c478bd9Sstevel@tonic-gate * 3) Park sibling core if caches are shared (to avoid race condition while 9377c478bd9Sstevel@tonic-gate * accessing shared resources such as L3 data staging register during 9387c478bd9Sstevel@tonic-gate * CPU logout. 9397c478bd9Sstevel@tonic-gate * 4) Read the AFAR and AFSR. 9407c478bd9Sstevel@tonic-gate * 5) If CPU logout structure is not being used, then: 9417c478bd9Sstevel@tonic-gate * 6) Clear all errors from the AFSR. 9427c478bd9Sstevel@tonic-gate * 7) Capture Ecache, Dcache and Icache lines in "CPU log out" structure. 9437c478bd9Sstevel@tonic-gate * 8) Flush Ecache then Flush Dcache and Icache and restore to previous 9447c478bd9Sstevel@tonic-gate * state. 9457c478bd9Sstevel@tonic-gate * 9) Unpark sibling core if we parked it earlier. 9467c478bd9Sstevel@tonic-gate * 10) call cpu_fast_ecc_error via systrap at PIL 14 unless we're already 9477c478bd9Sstevel@tonic-gate * running at PIL 15. 9487c478bd9Sstevel@tonic-gate * 6) Otherwise, if CPU logout structure is being used: 9497c478bd9Sstevel@tonic-gate * 7) Incriment the "logout busy count". 9507c478bd9Sstevel@tonic-gate * 8) Flush Ecache then Flush Dcache and Icache and restore to previous 9517c478bd9Sstevel@tonic-gate * state. 9527c478bd9Sstevel@tonic-gate * 9) Unpark sibling core if we parked it earlier. 9537c478bd9Sstevel@tonic-gate * 10) Issue a retry since the other CPU error logging code will end up 9547c478bd9Sstevel@tonic-gate * finding this error bit and logging information about it later. 9557c478bd9Sstevel@tonic-gate * 7) Alternatively (to 5 and 6 above), if the cpu_private struct is not 9567c478bd9Sstevel@tonic-gate * yet initialized such that we can't even check the logout struct, then 9577c478bd9Sstevel@tonic-gate * we place the clo_flags data into %g2 (sys_trap->have_win arg #1) and 9587c478bd9Sstevel@tonic-gate * call cpu_fast_ecc_error via systrap. The clo_flags parameter is used 9594fd7ecabSdilpreet * to determine information such as TL, TT, CEEN and NCEEN settings, etc 9604fd7ecabSdilpreet * in the high level trap handler since we don't have access to detailed 9614fd7ecabSdilpreet * logout information in cases where the cpu_private struct is not yet 9627c478bd9Sstevel@tonic-gate * initialized. 9637c478bd9Sstevel@tonic-gate * 9647c478bd9Sstevel@tonic-gate * We flush the E$ and D$ here on TL=1 code to prevent getting nested 9657c478bd9Sstevel@tonic-gate * Fast ECC traps in the TL=0 code. If we get a Fast ECC event here in 9667c478bd9Sstevel@tonic-gate * the TL=1 code, we will go to the Fast ECC at TL>0 handler which, 9677c478bd9Sstevel@tonic-gate * since it is uses different code/data from this handler, has a better 9687c478bd9Sstevel@tonic-gate * chance of fixing things up than simply recursing through this code 9697c478bd9Sstevel@tonic-gate * again (this would probably cause an eventual kernel stack overflow). 9707c478bd9Sstevel@tonic-gate * If the Fast ECC at TL>0 handler encounters a Fast ECC error before it 9717c478bd9Sstevel@tonic-gate * can flush the E$ (or the error is a stuck-at bit), we will recurse in 9727c478bd9Sstevel@tonic-gate * the Fast ECC at TL>0 handler and eventually Red Mode. 9737c478bd9Sstevel@tonic-gate * 9747c478bd9Sstevel@tonic-gate * Note that for Cheetah (and only Cheetah), we use alias addresses for 9757c478bd9Sstevel@tonic-gate * flushing rather than ASI accesses (which don't exist on Cheetah). 9767c478bd9Sstevel@tonic-gate * Should we encounter a Fast ECC error within this handler on Cheetah, 9777c478bd9Sstevel@tonic-gate * there's a good chance it's within the ecache_flushaddr buffer (since 9787c478bd9Sstevel@tonic-gate * it's the largest piece of memory we touch in the handler and it is 9797c478bd9Sstevel@tonic-gate * usually kernel text/data). For that reason the Fast ECC at TL>0 9807c478bd9Sstevel@tonic-gate * handler for Cheetah uses an alternate buffer: ecache_tl1_flushaddr. 9817c478bd9Sstevel@tonic-gate */ 9827c478bd9Sstevel@tonic-gate 9837c478bd9Sstevel@tonic-gate/* 9847c478bd9Sstevel@tonic-gate * Cheetah ecc-protected E$ trap (Trap 70) at TL=0 9857c478bd9Sstevel@tonic-gate * tt0_fecc is replaced by fecc_err_instr in cpu_init_trap of the various 986*55fea89dSDan Cross * architecture-specific files. 9877c478bd9Sstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must 9887c478bd9Sstevel@tonic-gate * be relocatable. 9897c478bd9Sstevel@tonic-gate */ 9907c478bd9Sstevel@tonic-gate 9917c478bd9Sstevel@tonic-gate ENTRY_NP(fecc_err_instr) 9927c478bd9Sstevel@tonic-gate membar #Sync ! Cheetah requires membar #Sync 9937c478bd9Sstevel@tonic-gate 9947c478bd9Sstevel@tonic-gate /* 9957c478bd9Sstevel@tonic-gate * Save current DCU state. Turn off the Dcache and Icache. 9967c478bd9Sstevel@tonic-gate */ 9977c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 9987c478bd9Sstevel@tonic-gate andn %g1, DCU_DC + DCU_IC, %g4 9997c478bd9Sstevel@tonic-gate stxa %g4, [%g0]ASI_DCU 10007c478bd9Sstevel@tonic-gate flush %g0 /* flush required after changing the IC bit */ 10017c478bd9Sstevel@tonic-gate 10027c478bd9Sstevel@tonic-gate ASM_JMP(%g4, fast_ecc_err) 10037c478bd9Sstevel@tonic-gate SET_SIZE(fecc_err_instr) 10047c478bd9Sstevel@tonic-gate 10057c478bd9Sstevel@tonic-gate 10067c478bd9Sstevel@tonic-gate#if !(defined(JALAPENO) || defined(SERRANO)) 10077c478bd9Sstevel@tonic-gate 10087c478bd9Sstevel@tonic-gate .section ".text" 10097c478bd9Sstevel@tonic-gate .align 64 10107c478bd9Sstevel@tonic-gate ENTRY_NP(fast_ecc_err) 10117c478bd9Sstevel@tonic-gate 10127c478bd9Sstevel@tonic-gate /* 10137c478bd9Sstevel@tonic-gate * Turn off CEEN and NCEEN. 10147c478bd9Sstevel@tonic-gate */ 10157c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_ESTATE_ERR, %g3 10167c478bd9Sstevel@tonic-gate andn %g3, EN_REG_NCEEN + EN_REG_CEEN, %g4 10177c478bd9Sstevel@tonic-gate stxa %g4, [%g0]ASI_ESTATE_ERR 10187c478bd9Sstevel@tonic-gate membar #Sync ! membar sync required 10197c478bd9Sstevel@tonic-gate 10207c478bd9Sstevel@tonic-gate /* 10217c478bd9Sstevel@tonic-gate * Check to see whether we need to park our sibling core 10227c478bd9Sstevel@tonic-gate * before recording diagnostic information from caches 10237c478bd9Sstevel@tonic-gate * which may be shared by both cores. 10247c478bd9Sstevel@tonic-gate * We use %g1 to store information about whether or not 10257c478bd9Sstevel@tonic-gate * we had to park the core (%g1 holds our DCUCR value and 10267c478bd9Sstevel@tonic-gate * we only use bits from that register which are "reserved" 10277c478bd9Sstevel@tonic-gate * to keep track of core parking) so that we know whether 10287c478bd9Sstevel@tonic-gate * or not to unpark later. %g5 and %g4 are scratch registers. 10297c478bd9Sstevel@tonic-gate */ 10307c478bd9Sstevel@tonic-gate PARK_SIBLING_CORE(%g1, %g5, %g4) 10317c478bd9Sstevel@tonic-gate 10327c478bd9Sstevel@tonic-gate /* 10337c478bd9Sstevel@tonic-gate * Do the CPU log out capture. 10347c478bd9Sstevel@tonic-gate * %g3 = "failed?" return value. 10357c478bd9Sstevel@tonic-gate * %g2 = Input = AFAR. Output the clo_flags info which is passed 10367c478bd9Sstevel@tonic-gate * into this macro via %g4. Output only valid if cpu_private 10377c478bd9Sstevel@tonic-gate * struct has not been initialized. 10387c478bd9Sstevel@tonic-gate * CHPR_FECCTL0_LOGOUT = cpu logout structure offset input 10397c478bd9Sstevel@tonic-gate * %g4 = Trap information stored in the cpu logout flags field 10407c478bd9Sstevel@tonic-gate * %g5 = scr1 10417c478bd9Sstevel@tonic-gate * %g6 = scr2 10427c478bd9Sstevel@tonic-gate * %g3 = scr3 10437c478bd9Sstevel@tonic-gate * %g4 = scr4 10447c478bd9Sstevel@tonic-gate */ 10454fd7ecabSdilpreet /* store the CEEN and NCEEN values, TL=0 */ 10464fd7ecabSdilpreet and %g3, EN_REG_CEEN + EN_REG_NCEEN, %g4 10477c478bd9Sstevel@tonic-gate set CHPR_FECCTL0_LOGOUT, %g6 10487c478bd9Sstevel@tonic-gate DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4) 10497c478bd9Sstevel@tonic-gate 10507c478bd9Sstevel@tonic-gate /* 10517c478bd9Sstevel@tonic-gate * Flush the Ecache (and L2 cache for Panther) to get the error out 10527c478bd9Sstevel@tonic-gate * of the Ecache. If the UCC or UCU is on a dirty line, then the 10537c478bd9Sstevel@tonic-gate * following flush will turn that into a WDC or WDU, respectively. 10547c478bd9Sstevel@tonic-gate */ 10557c478bd9Sstevel@tonic-gate PN_L2_FLUSHALL(%g4, %g5, %g6) 10567c478bd9Sstevel@tonic-gate 10577c478bd9Sstevel@tonic-gate CPU_INDEX(%g4, %g5) 10587c478bd9Sstevel@tonic-gate mulx %g4, CPU_NODE_SIZE, %g4 10597c478bd9Sstevel@tonic-gate set cpunodes, %g5 10607c478bd9Sstevel@tonic-gate add %g4, %g5, %g4 10617c478bd9Sstevel@tonic-gate ld [%g4 + ECACHE_LINESIZE], %g5 10627c478bd9Sstevel@tonic-gate ld [%g4 + ECACHE_SIZE], %g4 10637c478bd9Sstevel@tonic-gate 10647c478bd9Sstevel@tonic-gate ASM_LDX(%g6, ecache_flushaddr) 10657c478bd9Sstevel@tonic-gate ECACHE_FLUSHALL(%g4, %g5, %g6, %g7) 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate /* 10687c478bd9Sstevel@tonic-gate * Flush the Dcache. Since bad data could have been installed in 10697c478bd9Sstevel@tonic-gate * the Dcache we must flush it before re-enabling it. 10707c478bd9Sstevel@tonic-gate */ 10717c478bd9Sstevel@tonic-gate ASM_LD(%g5, dcache_size) 10727c478bd9Sstevel@tonic-gate ASM_LD(%g6, dcache_linesize) 10737c478bd9Sstevel@tonic-gate CH_DCACHE_FLUSHALL(%g5, %g6, %g7) 10747c478bd9Sstevel@tonic-gate 10757c478bd9Sstevel@tonic-gate /* 10767c478bd9Sstevel@tonic-gate * Flush the Icache. Since we turned off the Icache to capture the 10777c478bd9Sstevel@tonic-gate * Icache line it is now stale or corrupted and we must flush it 10787c478bd9Sstevel@tonic-gate * before re-enabling it. 10797c478bd9Sstevel@tonic-gate */ 10807c478bd9Sstevel@tonic-gate GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, fast_ecc_err_5); 10817c478bd9Sstevel@tonic-gate ld [%g5 + CHPR_ICACHE_LINESIZE], %g6 10827c478bd9Sstevel@tonic-gate ba,pt %icc, 6f 10837c478bd9Sstevel@tonic-gate ld [%g5 + CHPR_ICACHE_SIZE], %g5 10847c478bd9Sstevel@tonic-gatefast_ecc_err_5: 10857c478bd9Sstevel@tonic-gate ASM_LD(%g5, icache_size) 10867c478bd9Sstevel@tonic-gate ASM_LD(%g6, icache_linesize) 10877c478bd9Sstevel@tonic-gate6: 10887c478bd9Sstevel@tonic-gate CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4) 10897c478bd9Sstevel@tonic-gate 10907c478bd9Sstevel@tonic-gate /* 10917c478bd9Sstevel@tonic-gate * check to see whether we parked our sibling core at the start 10927c478bd9Sstevel@tonic-gate * of this handler. If so, we need to unpark it here. 10937c478bd9Sstevel@tonic-gate * We use DCUCR reserved bits (stored in %g1) to keep track of 10947c478bd9Sstevel@tonic-gate * whether or not we need to unpark. %g5 and %g4 are scratch registers. 10957c478bd9Sstevel@tonic-gate */ 10967c478bd9Sstevel@tonic-gate UNPARK_SIBLING_CORE(%g1, %g5, %g4) 10977c478bd9Sstevel@tonic-gate 10987c478bd9Sstevel@tonic-gate /* 10997c478bd9Sstevel@tonic-gate * Restore the Dcache and Icache to the previous state. 11007c478bd9Sstevel@tonic-gate */ 11017c478bd9Sstevel@tonic-gate stxa %g1, [%g0]ASI_DCU 11027c478bd9Sstevel@tonic-gate flush %g0 /* flush required after changing the IC bit */ 11037c478bd9Sstevel@tonic-gate 11047c478bd9Sstevel@tonic-gate /* 11057c478bd9Sstevel@tonic-gate * Make sure our CPU logout operation was successful. 11067c478bd9Sstevel@tonic-gate */ 11077c478bd9Sstevel@tonic-gate cmp %g3, %g0 11087c478bd9Sstevel@tonic-gate be 8f 11097c478bd9Sstevel@tonic-gate nop 11107c478bd9Sstevel@tonic-gate 11117c478bd9Sstevel@tonic-gate /* 11127c478bd9Sstevel@tonic-gate * If the logout structure had been busy, how many times have 11137c478bd9Sstevel@tonic-gate * we tried to use it and failed (nesting count)? If we have 11147c478bd9Sstevel@tonic-gate * already recursed a substantial number of times, then we can 11157c478bd9Sstevel@tonic-gate * assume things are not going to get better by themselves and 11167c478bd9Sstevel@tonic-gate * so it would be best to panic. 11177c478bd9Sstevel@tonic-gate */ 11187c478bd9Sstevel@tonic-gate cmp %g3, CLO_NESTING_MAX 11197c478bd9Sstevel@tonic-gate blt 7f 11207c478bd9Sstevel@tonic-gate nop 11217c478bd9Sstevel@tonic-gate 11227c478bd9Sstevel@tonic-gate call ptl1_panic 11237c478bd9Sstevel@tonic-gate mov PTL1_BAD_ECC, %g1 11247c478bd9Sstevel@tonic-gate 1125*55fea89dSDan Cross7: 11267c478bd9Sstevel@tonic-gate /* 11277c478bd9Sstevel@tonic-gate * Otherwise, if the logout structure was busy but we have not 11287c478bd9Sstevel@tonic-gate * nested more times than our maximum value, then we simply 11297c478bd9Sstevel@tonic-gate * issue a retry. Our TL=0 trap handler code will check and 11307c478bd9Sstevel@tonic-gate * clear the AFSR after it is done logging what is currently 11317c478bd9Sstevel@tonic-gate * in the logout struct and handle this event at that time. 11327c478bd9Sstevel@tonic-gate */ 11337c478bd9Sstevel@tonic-gate retry 11347c478bd9Sstevel@tonic-gate8: 11357c478bd9Sstevel@tonic-gate /* 11367c478bd9Sstevel@tonic-gate * Call cpu_fast_ecc_error via systrap at PIL 14 unless we're 11377c478bd9Sstevel@tonic-gate * already at PIL 15. 11387c478bd9Sstevel@tonic-gate */ 11397c478bd9Sstevel@tonic-gate set cpu_fast_ecc_error, %g1 11407c478bd9Sstevel@tonic-gate rdpr %pil, %g4 11417c478bd9Sstevel@tonic-gate cmp %g4, PIL_14 11427c478bd9Sstevel@tonic-gate ba sys_trap 11437c478bd9Sstevel@tonic-gate movl %icc, PIL_14, %g4 11447c478bd9Sstevel@tonic-gate 11457c478bd9Sstevel@tonic-gate SET_SIZE(fast_ecc_err) 11467c478bd9Sstevel@tonic-gate 11477c478bd9Sstevel@tonic-gate#endif /* !(JALAPENO || SERRANO) */ 11487c478bd9Sstevel@tonic-gate 11497c478bd9Sstevel@tonic-gate 11507c478bd9Sstevel@tonic-gate/* 11517c478bd9Sstevel@tonic-gate * Cheetah/Cheetah+ Fast ECC at TL>0 trap strategy: 11527c478bd9Sstevel@tonic-gate * 11537c478bd9Sstevel@tonic-gate * The basic flow of this trap handler is as follows: 11547c478bd9Sstevel@tonic-gate * 11557c478bd9Sstevel@tonic-gate * 1) In the "trap 70" trap table code (fecc_err_tl1_instr), generate a 11567c478bd9Sstevel@tonic-gate * software trap 0 ("ta 0") to buy an extra set of %tpc, etc. which we 11577c478bd9Sstevel@tonic-gate * will use to save %g1 and %g2. 11587c478bd9Sstevel@tonic-gate * 2) At the software trap 0 at TL>0 trap table code (fecc_err_tl1_cont_instr), 11597c478bd9Sstevel@tonic-gate * we save %g1+%g2 using %tpc, %tnpc + %tstate and jump to the fast ecc 11607c478bd9Sstevel@tonic-gate * handler (using the just saved %g1). 11617c478bd9Sstevel@tonic-gate * 3) Turn off the Dcache if it was on and save the state of the Dcache 11627c478bd9Sstevel@tonic-gate * (whether on or off) in Bit2 (CH_ERR_TSTATE_DC_ON) of %tstate. 11637c478bd9Sstevel@tonic-gate * NB: we don't turn off the Icache because bad data is not installed nor 11647c478bd9Sstevel@tonic-gate * will we be doing any diagnostic accesses. 11657c478bd9Sstevel@tonic-gate * 4) compute physical address of the per-cpu/per-tl save area using %g1+%g2 11667c478bd9Sstevel@tonic-gate * 5) Save %g1-%g7 into the per-cpu/per-tl save area (%g1 + %g2 from the 11677c478bd9Sstevel@tonic-gate * %tpc, %tnpc, %tstate values previously saved). 11687c478bd9Sstevel@tonic-gate * 6) set %tl to %tl - 1. 11697c478bd9Sstevel@tonic-gate * 7) Save the appropriate flags and TPC in the ch_err_tl1_data structure. 11707c478bd9Sstevel@tonic-gate * 8) Save the value of CH_ERR_TSTATE_DC_ON in the ch_err_tl1_tmp field. 11717c478bd9Sstevel@tonic-gate * 9) For Cheetah and Jalapeno, read the AFAR and AFSR and clear. For 11727c478bd9Sstevel@tonic-gate * Cheetah+ (and later), read the shadow AFAR and AFSR but don't clear. 11737c478bd9Sstevel@tonic-gate * Save the values in ch_err_tl1_data. For Panther, read the shadow 11747c478bd9Sstevel@tonic-gate * AFSR_EXT and save the value in ch_err_tl1_data. 11757c478bd9Sstevel@tonic-gate * 10) Disable CEEN/NCEEN to prevent any disrupting/deferred errors from 11767c478bd9Sstevel@tonic-gate * being queued. We'll report them via the AFSR/AFAR capture in step 13. 11777c478bd9Sstevel@tonic-gate * 11) Flush the Ecache. 11787c478bd9Sstevel@tonic-gate * NB: the Ecache is flushed assuming the largest possible size with 11797c478bd9Sstevel@tonic-gate * the smallest possible line size since access to the cpu_nodes may 11807c478bd9Sstevel@tonic-gate * cause an unrecoverable DTLB miss. 11817c478bd9Sstevel@tonic-gate * 12) Reenable CEEN/NCEEN with the value saved from step 10. 11827c478bd9Sstevel@tonic-gate * 13) For Cheetah and Jalapeno, read the AFAR and AFSR and clear again. 11837c478bd9Sstevel@tonic-gate * For Cheetah+ (and later), read the primary AFAR and AFSR and now clear. 11847c478bd9Sstevel@tonic-gate * Save the read AFSR/AFAR values in ch_err_tl1_data. For Panther, 11857c478bd9Sstevel@tonic-gate * read and clear the primary AFSR_EXT and save it in ch_err_tl1_data. 11867c478bd9Sstevel@tonic-gate * 14) Flush and re-enable the Dcache if it was on at step 3. 11877c478bd9Sstevel@tonic-gate * 15) Do TRAPTRACE if enabled. 11887c478bd9Sstevel@tonic-gate * 16) Check if a UCU->WDU (or L3_UCU->WDU for Panther) happened, panic if so. 11897c478bd9Sstevel@tonic-gate * 17) Set the event pending flag in ch_err_tl1_pending[CPU] 11907c478bd9Sstevel@tonic-gate * 18) Cause a softint 15. The pil15_interrupt handler will inspect the 11917c478bd9Sstevel@tonic-gate * event pending flag and call cpu_tl1_error via systrap if set. 11927c478bd9Sstevel@tonic-gate * 19) Restore the registers from step 5 and issue retry. 11937c478bd9Sstevel@tonic-gate */ 11947c478bd9Sstevel@tonic-gate 11957c478bd9Sstevel@tonic-gate/* 11967c478bd9Sstevel@tonic-gate * Cheetah ecc-protected E$ trap (Trap 70) at TL>0 11977c478bd9Sstevel@tonic-gate * tt1_fecc is replaced by fecc_err_tl1_instr in cpu_init_trap of the various 11987c478bd9Sstevel@tonic-gate * architecture-specific files. This generates a "Software Trap 0" at TL>0, 11997c478bd9Sstevel@tonic-gate * which goes to fecc_err_tl1_cont_instr, and we continue the handling there. 12007c478bd9Sstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must 12017c478bd9Sstevel@tonic-gate * be relocatable. 12027c478bd9Sstevel@tonic-gate */ 12037c478bd9Sstevel@tonic-gate 12047c478bd9Sstevel@tonic-gate ENTRY_NP(fecc_err_tl1_instr) 12057c478bd9Sstevel@tonic-gate CH_ERR_TL1_TRAPENTRY(SWTRAP_0); 12067c478bd9Sstevel@tonic-gate SET_SIZE(fecc_err_tl1_instr) 12077c478bd9Sstevel@tonic-gate 12087c478bd9Sstevel@tonic-gate/* 12097c478bd9Sstevel@tonic-gate * Software trap 0 at TL>0. 12107c478bd9Sstevel@tonic-gate * tt1_swtrap0 is replaced by fecc_err_tl1_cont_instr in cpu_init_trap of 12117c478bd9Sstevel@tonic-gate * the various architecture-specific files. This is used as a continuation 12127c478bd9Sstevel@tonic-gate * of the fast ecc handling where we've bought an extra TL level, so we can 12137c478bd9Sstevel@tonic-gate * use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1 12147c478bd9Sstevel@tonic-gate * and %g2. Note that %tstate has bits 0-2 and then bits 8-19 as r/w, 12157c478bd9Sstevel@tonic-gate * there's a reserved hole from 3-7. We only use bits 0-1 and 8-9 (the low 12167c478bd9Sstevel@tonic-gate * order two bits from %g1 and %g2 respectively). 12177c478bd9Sstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must 12187c478bd9Sstevel@tonic-gate * be relocatable. 12197c478bd9Sstevel@tonic-gate */ 12207c478bd9Sstevel@tonic-gate 12217c478bd9Sstevel@tonic-gate ENTRY_NP(fecc_err_tl1_cont_instr) 12227c478bd9Sstevel@tonic-gate CH_ERR_TL1_SWTRAPENTRY(fast_ecc_tl1_err) 12237c478bd9Sstevel@tonic-gate SET_SIZE(fecc_err_tl1_cont_instr) 12247c478bd9Sstevel@tonic-gate 12257c478bd9Sstevel@tonic-gate 12267c478bd9Sstevel@tonic-gate/* 12277c478bd9Sstevel@tonic-gate * The ce_err function handles disrupting trap type 0x63 at TL=0. 12287c478bd9Sstevel@tonic-gate * 12297c478bd9Sstevel@tonic-gate * AFSR errors bits which cause this trap are: 12307c478bd9Sstevel@tonic-gate * CE, EMC, EDU:ST, EDC, WDU, WDC, CPU, CPC, IVU, IVC 12317c478bd9Sstevel@tonic-gate * 12327c478bd9Sstevel@tonic-gate * NCEEN Bit of Cheetah External Cache Error Enable Register enables 12337c478bd9Sstevel@tonic-gate * the following AFSR disrupting traps: EDU:ST, WDU, CPU, IVU 12347c478bd9Sstevel@tonic-gate * 12357c478bd9Sstevel@tonic-gate * CEEN Bit of Cheetah External Cache Error Enable Register enables 12367c478bd9Sstevel@tonic-gate * the following AFSR disrupting traps: CE, EMC, EDC, WDC, CPC, IVC 12377c478bd9Sstevel@tonic-gate * 12387c478bd9Sstevel@tonic-gate * Cheetah+ also handles (No additional processing required): 12397c478bd9Sstevel@tonic-gate * DUE, DTO, DBERR (NCEEN controlled) 12407c478bd9Sstevel@tonic-gate * THCE (CEEN and ET_ECC_en controlled) 12417c478bd9Sstevel@tonic-gate * TUE (ET_ECC_en controlled) 12427c478bd9Sstevel@tonic-gate * 12437c478bd9Sstevel@tonic-gate * Panther further adds: 12447c478bd9Sstevel@tonic-gate * IMU, L3_EDU, L3_WDU, L3_CPU (NCEEN controlled) 12457c478bd9Sstevel@tonic-gate * IMC, L3_EDC, L3_WDC, L3_CPC, L3_THCE (CEEN controlled) 12467c478bd9Sstevel@tonic-gate * TUE_SH, TUE (NCEEN and L2_tag_ECC_en controlled) 12477c478bd9Sstevel@tonic-gate * L3_TUE, L3_TUE_SH (NCEEN and ET_ECC_en controlled) 12487c478bd9Sstevel@tonic-gate * THCE (CEEN and L2_tag_ECC_en controlled) 12497c478bd9Sstevel@tonic-gate * L3_THCE (CEEN and ET_ECC_en controlled) 12507c478bd9Sstevel@tonic-gate * 12517c478bd9Sstevel@tonic-gate * Steps: 12527c478bd9Sstevel@tonic-gate * 1. Disable hardware corrected disrupting errors only (CEEN) 12537c478bd9Sstevel@tonic-gate * 2. Park sibling core if caches are shared (to avoid race 12547c478bd9Sstevel@tonic-gate * condition while accessing shared resources such as L3 12557c478bd9Sstevel@tonic-gate * data staging register during CPU logout. 12567c478bd9Sstevel@tonic-gate * 3. If the CPU logout structure is not currently being used: 12577c478bd9Sstevel@tonic-gate * 4. Clear AFSR error bits 12587c478bd9Sstevel@tonic-gate * 5. Capture Ecache, Dcache and Icache lines associated 12597c478bd9Sstevel@tonic-gate * with AFAR. 12607c478bd9Sstevel@tonic-gate * 6. Unpark sibling core if we parked it earlier. 12617c478bd9Sstevel@tonic-gate * 7. call cpu_disrupting_error via sys_trap at PIL 14 12627c478bd9Sstevel@tonic-gate * unless we're already running at PIL 15. 12637c478bd9Sstevel@tonic-gate * 4. Otherwise, if the CPU logout structure is busy: 12647c478bd9Sstevel@tonic-gate * 5. Incriment "logout busy count" and place into %g3 12657c478bd9Sstevel@tonic-gate * 6. Unpark sibling core if we parked it earlier. 12667c478bd9Sstevel@tonic-gate * 7. Issue a retry since the other CPU error logging 12677c478bd9Sstevel@tonic-gate * code will end up finding this error bit and logging 12687c478bd9Sstevel@tonic-gate * information about it later. 12697c478bd9Sstevel@tonic-gate * 5. Alternatively (to 3 and 4 above), if the cpu_private struct is 12707c478bd9Sstevel@tonic-gate * not yet initialized such that we can't even check the logout 12717c478bd9Sstevel@tonic-gate * struct, then we place the clo_flags data into %g2 12727c478bd9Sstevel@tonic-gate * (sys_trap->have_win arg #1) and call cpu_disrupting_error via 12737c478bd9Sstevel@tonic-gate * systrap. The clo_flags parameter is used to determine information 12747c478bd9Sstevel@tonic-gate * such as TL, TT, CEEN settings, etc in the high level trap 12757c478bd9Sstevel@tonic-gate * handler since we don't have access to detailed logout information 12767c478bd9Sstevel@tonic-gate * in cases where the cpu_private struct is not yet initialized. 12777c478bd9Sstevel@tonic-gate * 12787c478bd9Sstevel@tonic-gate * %g3: [ logout busy count ] - arg #2 12797c478bd9Sstevel@tonic-gate * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1 12807c478bd9Sstevel@tonic-gate */ 12817c478bd9Sstevel@tonic-gate 12827c478bd9Sstevel@tonic-gate .align 128 12837c478bd9Sstevel@tonic-gate ENTRY_NP(ce_err) 12847c478bd9Sstevel@tonic-gate membar #Sync ! Cheetah requires membar #Sync 12857c478bd9Sstevel@tonic-gate 12867c478bd9Sstevel@tonic-gate /* 12877c478bd9Sstevel@tonic-gate * Disable trap on hardware corrected errors (CEEN) while at TL=0 12887c478bd9Sstevel@tonic-gate * to prevent recursion. 12897c478bd9Sstevel@tonic-gate */ 12907c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_ESTATE_ERR, %g1 12917c478bd9Sstevel@tonic-gate bclr EN_REG_CEEN, %g1 12927c478bd9Sstevel@tonic-gate stxa %g1, [%g0]ASI_ESTATE_ERR 12937c478bd9Sstevel@tonic-gate membar #Sync ! membar sync required 12947c478bd9Sstevel@tonic-gate 12957c478bd9Sstevel@tonic-gate /* 12967c478bd9Sstevel@tonic-gate * Save current DCU state. Turn off Icache to allow capture of 12977c478bd9Sstevel@tonic-gate * Icache data by DO_CPU_LOGOUT. 12987c478bd9Sstevel@tonic-gate */ 12997c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 13007c478bd9Sstevel@tonic-gate andn %g1, DCU_IC, %g4 13017c478bd9Sstevel@tonic-gate stxa %g4, [%g0]ASI_DCU 13027c478bd9Sstevel@tonic-gate flush %g0 /* flush required after changing the IC bit */ 13037c478bd9Sstevel@tonic-gate 13047c478bd9Sstevel@tonic-gate /* 13057c478bd9Sstevel@tonic-gate * Check to see whether we need to park our sibling core 13067c478bd9Sstevel@tonic-gate * before recording diagnostic information from caches 13077c478bd9Sstevel@tonic-gate * which may be shared by both cores. 13087c478bd9Sstevel@tonic-gate * We use %g1 to store information about whether or not 13097c478bd9Sstevel@tonic-gate * we had to park the core (%g1 holds our DCUCR value and 13107c478bd9Sstevel@tonic-gate * we only use bits from that register which are "reserved" 13117c478bd9Sstevel@tonic-gate * to keep track of core parking) so that we know whether 13127c478bd9Sstevel@tonic-gate * or not to unpark later. %g5 and %g4 are scratch registers. 13137c478bd9Sstevel@tonic-gate */ 13147c478bd9Sstevel@tonic-gate PARK_SIBLING_CORE(%g1, %g5, %g4) 13157c478bd9Sstevel@tonic-gate 13167c478bd9Sstevel@tonic-gate /* 13177c478bd9Sstevel@tonic-gate * Do the CPU log out capture. 13187c478bd9Sstevel@tonic-gate * %g3 = "failed?" return value. 13197c478bd9Sstevel@tonic-gate * %g2 = Input = AFAR. Output the clo_flags info which is passed 13207c478bd9Sstevel@tonic-gate * into this macro via %g4. Output only valid if cpu_private 13217c478bd9Sstevel@tonic-gate * struct has not been initialized. 13227c478bd9Sstevel@tonic-gate * CHPR_CECC_LOGOUT = cpu logout structure offset input 13237c478bd9Sstevel@tonic-gate * %g4 = Trap information stored in the cpu logout flags field 13247c478bd9Sstevel@tonic-gate * %g5 = scr1 13257c478bd9Sstevel@tonic-gate * %g6 = scr2 13267c478bd9Sstevel@tonic-gate * %g3 = scr3 13277c478bd9Sstevel@tonic-gate * %g4 = scr4 13287c478bd9Sstevel@tonic-gate */ 13297c478bd9Sstevel@tonic-gate clr %g4 ! TL=0 bit in afsr 13307c478bd9Sstevel@tonic-gate set CHPR_CECC_LOGOUT, %g6 13317c478bd9Sstevel@tonic-gate DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4) 13327c478bd9Sstevel@tonic-gate 13337c478bd9Sstevel@tonic-gate /* 13347c478bd9Sstevel@tonic-gate * Flush the Icache. Since we turned off the Icache to capture the 13357c478bd9Sstevel@tonic-gate * Icache line it is now stale or corrupted and we must flush it 13367c478bd9Sstevel@tonic-gate * before re-enabling it. 13377c478bd9Sstevel@tonic-gate */ 13387c478bd9Sstevel@tonic-gate GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, ce_err_1); 13397c478bd9Sstevel@tonic-gate ld [%g5 + CHPR_ICACHE_LINESIZE], %g6 13407c478bd9Sstevel@tonic-gate ba,pt %icc, 2f 13417c478bd9Sstevel@tonic-gate ld [%g5 + CHPR_ICACHE_SIZE], %g5 13427c478bd9Sstevel@tonic-gatece_err_1: 13437c478bd9Sstevel@tonic-gate ASM_LD(%g5, icache_size) 13447c478bd9Sstevel@tonic-gate ASM_LD(%g6, icache_linesize) 13457c478bd9Sstevel@tonic-gate2: 13467c478bd9Sstevel@tonic-gate CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4) 13477c478bd9Sstevel@tonic-gate 13487c478bd9Sstevel@tonic-gate /* 13497c478bd9Sstevel@tonic-gate * check to see whether we parked our sibling core at the start 13507c478bd9Sstevel@tonic-gate * of this handler. If so, we need to unpark it here. 13517c478bd9Sstevel@tonic-gate * We use DCUCR reserved bits (stored in %g1) to keep track of 13527c478bd9Sstevel@tonic-gate * whether or not we need to unpark. %g5 and %g4 are scratch registers. 13537c478bd9Sstevel@tonic-gate */ 13547c478bd9Sstevel@tonic-gate UNPARK_SIBLING_CORE(%g1, %g5, %g4) 13557c478bd9Sstevel@tonic-gate 13567c478bd9Sstevel@tonic-gate /* 13577c478bd9Sstevel@tonic-gate * Restore Icache to previous state. 13587c478bd9Sstevel@tonic-gate */ 13597c478bd9Sstevel@tonic-gate stxa %g1, [%g0]ASI_DCU 13607c478bd9Sstevel@tonic-gate flush %g0 /* flush required after changing the IC bit */ 1361*55fea89dSDan Cross 13627c478bd9Sstevel@tonic-gate /* 13637c478bd9Sstevel@tonic-gate * Make sure our CPU logout operation was successful. 13647c478bd9Sstevel@tonic-gate */ 13657c478bd9Sstevel@tonic-gate cmp %g3, %g0 13667c478bd9Sstevel@tonic-gate be 4f 13677c478bd9Sstevel@tonic-gate nop 13687c478bd9Sstevel@tonic-gate 13697c478bd9Sstevel@tonic-gate /* 13707c478bd9Sstevel@tonic-gate * If the logout structure had been busy, how many times have 13717c478bd9Sstevel@tonic-gate * we tried to use it and failed (nesting count)? If we have 13727c478bd9Sstevel@tonic-gate * already recursed a substantial number of times, then we can 13737c478bd9Sstevel@tonic-gate * assume things are not going to get better by themselves and 13747c478bd9Sstevel@tonic-gate * so it would be best to panic. 13757c478bd9Sstevel@tonic-gate */ 13767c478bd9Sstevel@tonic-gate cmp %g3, CLO_NESTING_MAX 13777c478bd9Sstevel@tonic-gate blt 3f 13787c478bd9Sstevel@tonic-gate nop 13797c478bd9Sstevel@tonic-gate 13807c478bd9Sstevel@tonic-gate call ptl1_panic 13817c478bd9Sstevel@tonic-gate mov PTL1_BAD_ECC, %g1 13827c478bd9Sstevel@tonic-gate 13837c478bd9Sstevel@tonic-gate3: 13847c478bd9Sstevel@tonic-gate /* 13857c478bd9Sstevel@tonic-gate * Otherwise, if the logout structure was busy but we have not 13867c478bd9Sstevel@tonic-gate * nested more times than our maximum value, then we simply 13877c478bd9Sstevel@tonic-gate * issue a retry. Our TL=0 trap handler code will check and 13887c478bd9Sstevel@tonic-gate * clear the AFSR after it is done logging what is currently 13897c478bd9Sstevel@tonic-gate * in the logout struct and handle this event at that time. 13907c478bd9Sstevel@tonic-gate */ 13917c478bd9Sstevel@tonic-gate retry 13927c478bd9Sstevel@tonic-gate4: 13937c478bd9Sstevel@tonic-gate /* 13947c478bd9Sstevel@tonic-gate * Call cpu_disrupting_error via systrap at PIL 14 unless we're 13957c478bd9Sstevel@tonic-gate * already at PIL 15. 13967c478bd9Sstevel@tonic-gate */ 13977c478bd9Sstevel@tonic-gate set cpu_disrupting_error, %g1 13987c478bd9Sstevel@tonic-gate rdpr %pil, %g4 13997c478bd9Sstevel@tonic-gate cmp %g4, PIL_14 14007c478bd9Sstevel@tonic-gate ba sys_trap 14017c478bd9Sstevel@tonic-gate movl %icc, PIL_14, %g4 14027c478bd9Sstevel@tonic-gate SET_SIZE(ce_err) 14037c478bd9Sstevel@tonic-gate 14047c478bd9Sstevel@tonic-gate 14057c478bd9Sstevel@tonic-gate/* 14067c478bd9Sstevel@tonic-gate * This trap cannot happen at TL>0 which means this routine will never 14077c478bd9Sstevel@tonic-gate * actually be called and so we treat this like a BAD TRAP panic. 14087c478bd9Sstevel@tonic-gate */ 14097c478bd9Sstevel@tonic-gate .align 64 14107c478bd9Sstevel@tonic-gate ENTRY_NP(ce_err_tl1) 14117c478bd9Sstevel@tonic-gate 14127c478bd9Sstevel@tonic-gate call ptl1_panic 14137c478bd9Sstevel@tonic-gate mov PTL1_BAD_TRAP, %g1 14147c478bd9Sstevel@tonic-gate 14157c478bd9Sstevel@tonic-gate SET_SIZE(ce_err_tl1) 14167c478bd9Sstevel@tonic-gate 1417*55fea89dSDan Cross 14187c478bd9Sstevel@tonic-gate/* 1419*55fea89dSDan Cross * The async_err function handles deferred trap types 0xA 14207c478bd9Sstevel@tonic-gate * (instruction_access_error) and 0x32 (data_access_error) at TL>=0. 14217c478bd9Sstevel@tonic-gate * 14227c478bd9Sstevel@tonic-gate * AFSR errors bits which cause this trap are: 14237c478bd9Sstevel@tonic-gate * UE, EMU, EDU:BLD, L3_EDU:BLD, TO, BERR 14247c478bd9Sstevel@tonic-gate * On some platforms, EMU may causes cheetah to pull the error pin 14257c478bd9Sstevel@tonic-gate * never giving Solaris a chance to take a trap. 14267c478bd9Sstevel@tonic-gate * 14277c478bd9Sstevel@tonic-gate * NCEEN Bit of Cheetah External Cache Error Enable Register enables 14287c478bd9Sstevel@tonic-gate * the following AFSR deferred traps: UE, EMU, EDU:BLD, TO, BERR 14297c478bd9Sstevel@tonic-gate * 14307c478bd9Sstevel@tonic-gate * Steps: 14317c478bd9Sstevel@tonic-gate * 1. Disable CEEN and NCEEN errors to prevent recursive errors. 14327c478bd9Sstevel@tonic-gate * 2. Turn D$ off per Cheetah PRM P.5 Note 6, turn I$ off to capture 14337c478bd9Sstevel@tonic-gate * I$ line in DO_CPU_LOGOUT. 14347c478bd9Sstevel@tonic-gate * 3. Park sibling core if caches are shared (to avoid race 14357c478bd9Sstevel@tonic-gate * condition while accessing shared resources such as L3 14367c478bd9Sstevel@tonic-gate * data staging register during CPU logout. 14377c478bd9Sstevel@tonic-gate * 4. If the CPU logout structure is not currently being used: 14387c478bd9Sstevel@tonic-gate * 5. Clear AFSR error bits 14397c478bd9Sstevel@tonic-gate * 6. Capture Ecache, Dcache and Icache lines associated 14407c478bd9Sstevel@tonic-gate * with AFAR. 14417c478bd9Sstevel@tonic-gate * 7. Unpark sibling core if we parked it earlier. 14427c478bd9Sstevel@tonic-gate * 8. call cpu_deferred_error via sys_trap. 14437c478bd9Sstevel@tonic-gate * 5. Otherwise, if the CPU logout structure is busy: 14447c478bd9Sstevel@tonic-gate * 6. Incriment "logout busy count" 14457c478bd9Sstevel@tonic-gate * 7. Unpark sibling core if we parked it earlier. 14467c478bd9Sstevel@tonic-gate * 8) Issue a retry since the other CPU error logging 14477c478bd9Sstevel@tonic-gate * code will end up finding this error bit and logging 14487c478bd9Sstevel@tonic-gate * information about it later. 14497c478bd9Sstevel@tonic-gate * 6. Alternatively (to 4 and 5 above), if the cpu_private struct is 14507c478bd9Sstevel@tonic-gate * not yet initialized such that we can't even check the logout 14517c478bd9Sstevel@tonic-gate * struct, then we place the clo_flags data into %g2 14527c478bd9Sstevel@tonic-gate * (sys_trap->have_win arg #1) and call cpu_deferred_error via 14537c478bd9Sstevel@tonic-gate * systrap. The clo_flags parameter is used to determine information 14547c478bd9Sstevel@tonic-gate * such as TL, TT, CEEN settings, etc in the high level trap handler 14557c478bd9Sstevel@tonic-gate * since we don't have access to detailed logout information in cases 14567c478bd9Sstevel@tonic-gate * where the cpu_private struct is not yet initialized. 14577c478bd9Sstevel@tonic-gate * 14587c478bd9Sstevel@tonic-gate * %g2: [ clo_flags if cpu_private unavailable ] - sys_trap->have_win: arg #1 14597c478bd9Sstevel@tonic-gate * %g3: [ logout busy count ] - arg #2 14607c478bd9Sstevel@tonic-gate */ 14617c478bd9Sstevel@tonic-gate 14627c478bd9Sstevel@tonic-gate ENTRY_NP(async_err) 14637c478bd9Sstevel@tonic-gate membar #Sync ! Cheetah requires membar #Sync 14647c478bd9Sstevel@tonic-gate 14657c478bd9Sstevel@tonic-gate /* 14667c478bd9Sstevel@tonic-gate * Disable CEEN and NCEEN. 14677c478bd9Sstevel@tonic-gate */ 14687c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_ESTATE_ERR, %g3 14697c478bd9Sstevel@tonic-gate andn %g3, EN_REG_NCEEN + EN_REG_CEEN, %g4 14707c478bd9Sstevel@tonic-gate stxa %g4, [%g0]ASI_ESTATE_ERR 14717c478bd9Sstevel@tonic-gate membar #Sync ! membar sync required 14727c478bd9Sstevel@tonic-gate 14737c478bd9Sstevel@tonic-gate /* 14747c478bd9Sstevel@tonic-gate * Save current DCU state. 14757c478bd9Sstevel@tonic-gate * Disable Icache to allow capture of Icache data by DO_CPU_LOGOUT. 14767c478bd9Sstevel@tonic-gate * Do this regardless of whether this is a Data Access Error or 14777c478bd9Sstevel@tonic-gate * Instruction Access Error Trap. 14787c478bd9Sstevel@tonic-gate * Disable Dcache for both Data Access Error and Instruction Access 14797c478bd9Sstevel@tonic-gate * Error per Cheetah PRM P.5 Note 6. 14807c478bd9Sstevel@tonic-gate */ 14817c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_DCU, %g1 ! save DCU in %g1 14827c478bd9Sstevel@tonic-gate andn %g1, DCU_IC + DCU_DC, %g4 14837c478bd9Sstevel@tonic-gate stxa %g4, [%g0]ASI_DCU 14847c478bd9Sstevel@tonic-gate flush %g0 /* flush required after changing the IC bit */ 14857c478bd9Sstevel@tonic-gate 14867c478bd9Sstevel@tonic-gate /* 14877c478bd9Sstevel@tonic-gate * Check to see whether we need to park our sibling core 14887c478bd9Sstevel@tonic-gate * before recording diagnostic information from caches 14897c478bd9Sstevel@tonic-gate * which may be shared by both cores. 14907c478bd9Sstevel@tonic-gate * We use %g1 to store information about whether or not 14917c478bd9Sstevel@tonic-gate * we had to park the core (%g1 holds our DCUCR value and 14927c478bd9Sstevel@tonic-gate * we only use bits from that register which are "reserved" 14937c478bd9Sstevel@tonic-gate * to keep track of core parking) so that we know whether 14947c478bd9Sstevel@tonic-gate * or not to unpark later. %g6 and %g4 are scratch registers. 14957c478bd9Sstevel@tonic-gate */ 14967c478bd9Sstevel@tonic-gate PARK_SIBLING_CORE(%g1, %g6, %g4) 14977c478bd9Sstevel@tonic-gate 14987c478bd9Sstevel@tonic-gate /* 14997c478bd9Sstevel@tonic-gate * Do the CPU logout capture. 15007c478bd9Sstevel@tonic-gate * 15017c478bd9Sstevel@tonic-gate * %g3 = "failed?" return value. 15027c478bd9Sstevel@tonic-gate * %g2 = Input = AFAR. Output the clo_flags info which is passed 15037c478bd9Sstevel@tonic-gate * into this macro via %g4. Output only valid if cpu_private 15047c478bd9Sstevel@tonic-gate * struct has not been initialized. 15057c478bd9Sstevel@tonic-gate * CHPR_ASYNC_LOGOUT = cpu logout structure offset input 15067c478bd9Sstevel@tonic-gate * %g4 = Trap information stored in the cpu logout flags field 15077c478bd9Sstevel@tonic-gate * %g5 = scr1 15087c478bd9Sstevel@tonic-gate * %g6 = scr2 15097c478bd9Sstevel@tonic-gate * %g3 = scr3 15107c478bd9Sstevel@tonic-gate * %g4 = scr4 15117c478bd9Sstevel@tonic-gate */ 1512*55fea89dSDan Cross andcc %g5, T_TL1, %g0 1513*55fea89dSDan Cross clr %g6 15147c478bd9Sstevel@tonic-gate movnz %xcc, 1, %g6 ! set %g6 if T_TL1 set 15157c478bd9Sstevel@tonic-gate sllx %g6, CLO_FLAGS_TL_SHIFT, %g6 15167c478bd9Sstevel@tonic-gate sllx %g5, CLO_FLAGS_TT_SHIFT, %g4 15177c478bd9Sstevel@tonic-gate set CLO_FLAGS_TT_MASK, %g2 15187c478bd9Sstevel@tonic-gate and %g4, %g2, %g4 ! ttype 15197c478bd9Sstevel@tonic-gate or %g6, %g4, %g4 ! TT and TL 15207c478bd9Sstevel@tonic-gate and %g3, EN_REG_CEEN, %g3 ! CEEN value 15217c478bd9Sstevel@tonic-gate or %g3, %g4, %g4 ! TT and TL and CEEN 15227c478bd9Sstevel@tonic-gate set CHPR_ASYNC_LOGOUT, %g6 15237c478bd9Sstevel@tonic-gate DO_CPU_LOGOUT(%g3, %g2, %g6, %g4, %g5, %g6, %g3, %g4) 15247c478bd9Sstevel@tonic-gate 15257c478bd9Sstevel@tonic-gate /* 15267c478bd9Sstevel@tonic-gate * If the logout struct was busy, we may need to pass the 1527*55fea89dSDan Cross * TT, TL, and CEEN information to the TL=0 handler via 15287c478bd9Sstevel@tonic-gate * systrap parameter so save it off here. 15297c478bd9Sstevel@tonic-gate */ 15307c478bd9Sstevel@tonic-gate cmp %g3, %g0 15317c478bd9Sstevel@tonic-gate be 1f 15327c478bd9Sstevel@tonic-gate nop 15337c478bd9Sstevel@tonic-gate sllx %g4, 32, %g4 15347c478bd9Sstevel@tonic-gate or %g4, %g3, %g3 15357c478bd9Sstevel@tonic-gate1: 15367c478bd9Sstevel@tonic-gate /* 15377c478bd9Sstevel@tonic-gate * Flush the Icache. Since we turned off the Icache to capture the 15387c478bd9Sstevel@tonic-gate * Icache line it is now stale or corrupted and we must flush it 15397c478bd9Sstevel@tonic-gate * before re-enabling it. 15407c478bd9Sstevel@tonic-gate */ 15417c478bd9Sstevel@tonic-gate GET_CPU_PRIVATE_PTR(%g0, %g5, %g7, async_err_1); 15427c478bd9Sstevel@tonic-gate ld [%g5 + CHPR_ICACHE_LINESIZE], %g6 15437c478bd9Sstevel@tonic-gate ba,pt %icc, 2f 15447c478bd9Sstevel@tonic-gate ld [%g5 + CHPR_ICACHE_SIZE], %g5 15457c478bd9Sstevel@tonic-gateasync_err_1: 15467c478bd9Sstevel@tonic-gate ASM_LD(%g5, icache_size) 15477c478bd9Sstevel@tonic-gate ASM_LD(%g6, icache_linesize) 15487c478bd9Sstevel@tonic-gate2: 15497c478bd9Sstevel@tonic-gate CH_ICACHE_FLUSHALL(%g5, %g6, %g7, %g4) 15507c478bd9Sstevel@tonic-gate 15517c478bd9Sstevel@tonic-gate /* 15527c478bd9Sstevel@tonic-gate * XXX - Don't we need to flush the Dcache before turning it back 15537c478bd9Sstevel@tonic-gate * on to avoid stale or corrupt data? Was this broken? 15547c478bd9Sstevel@tonic-gate */ 15557c478bd9Sstevel@tonic-gate /* 15567c478bd9Sstevel@tonic-gate * Flush the Dcache before turning it back on since it may now 15577c478bd9Sstevel@tonic-gate * contain stale or corrupt data. 15587c478bd9Sstevel@tonic-gate */ 15597c478bd9Sstevel@tonic-gate ASM_LD(%g5, dcache_size) 15607c478bd9Sstevel@tonic-gate ASM_LD(%g6, dcache_linesize) 15617c478bd9Sstevel@tonic-gate CH_DCACHE_FLUSHALL(%g5, %g6, %g7) 15627c478bd9Sstevel@tonic-gate 15637c478bd9Sstevel@tonic-gate /* 15647c478bd9Sstevel@tonic-gate * check to see whether we parked our sibling core at the start 15657c478bd9Sstevel@tonic-gate * of this handler. If so, we need to unpark it here. 15667c478bd9Sstevel@tonic-gate * We use DCUCR reserved bits (stored in %g1) to keep track of 15677c478bd9Sstevel@tonic-gate * whether or not we need to unpark. %g5 and %g7 are scratch registers. 15687c478bd9Sstevel@tonic-gate */ 15697c478bd9Sstevel@tonic-gate UNPARK_SIBLING_CORE(%g1, %g5, %g7) 15707c478bd9Sstevel@tonic-gate 15717c478bd9Sstevel@tonic-gate /* 15727c478bd9Sstevel@tonic-gate * Restore Icache and Dcache to previous state. 15737c478bd9Sstevel@tonic-gate */ 15747c478bd9Sstevel@tonic-gate stxa %g1, [%g0]ASI_DCU 15757c478bd9Sstevel@tonic-gate flush %g0 /* flush required after changing the IC bit */ 1576*55fea89dSDan Cross 15777c478bd9Sstevel@tonic-gate /* 15787c478bd9Sstevel@tonic-gate * Make sure our CPU logout operation was successful. 15797c478bd9Sstevel@tonic-gate */ 15807c478bd9Sstevel@tonic-gate cmp %g3, %g0 15817c478bd9Sstevel@tonic-gate be 4f 15827c478bd9Sstevel@tonic-gate nop 15837c478bd9Sstevel@tonic-gate 15847c478bd9Sstevel@tonic-gate /* 15857c478bd9Sstevel@tonic-gate * If the logout structure had been busy, how many times have 15867c478bd9Sstevel@tonic-gate * we tried to use it and failed (nesting count)? If we have 15877c478bd9Sstevel@tonic-gate * already recursed a substantial number of times, then we can 15887c478bd9Sstevel@tonic-gate * assume things are not going to get better by themselves and 15897c478bd9Sstevel@tonic-gate * so it would be best to panic. 15907c478bd9Sstevel@tonic-gate */ 15917c478bd9Sstevel@tonic-gate cmp %g3, CLO_NESTING_MAX 15927c478bd9Sstevel@tonic-gate blt 3f 15937c478bd9Sstevel@tonic-gate nop 15947c478bd9Sstevel@tonic-gate 15957c478bd9Sstevel@tonic-gate call ptl1_panic 15967c478bd9Sstevel@tonic-gate mov PTL1_BAD_ECC, %g1 15977c478bd9Sstevel@tonic-gate 15987c478bd9Sstevel@tonic-gate3: 15997c478bd9Sstevel@tonic-gate /* 16007c478bd9Sstevel@tonic-gate * Otherwise, if the logout structure was busy but we have not 16017c478bd9Sstevel@tonic-gate * nested more times than our maximum value, then we simply 16027c478bd9Sstevel@tonic-gate * issue a retry. Our TL=0 trap handler code will check and 16037c478bd9Sstevel@tonic-gate * clear the AFSR after it is done logging what is currently 16047c478bd9Sstevel@tonic-gate * in the logout struct and handle this event at that time. 16057c478bd9Sstevel@tonic-gate */ 16067c478bd9Sstevel@tonic-gate retry 16077c478bd9Sstevel@tonic-gate4: 16082dd3029aSjimand RESET_USER_RTT_REGS(%g4, %g5, async_err_resetskip) 16092dd3029aSjimandasync_err_resetskip: 16107c478bd9Sstevel@tonic-gate set cpu_deferred_error, %g1 16117c478bd9Sstevel@tonic-gate ba sys_trap 16127c478bd9Sstevel@tonic-gate mov PIL_15, %g4 ! run at pil 15 16137c478bd9Sstevel@tonic-gate SET_SIZE(async_err) 16147c478bd9Sstevel@tonic-gate 16157c478bd9Sstevel@tonic-gate#if defined(CPU_IMP_L1_CACHE_PARITY) 16167c478bd9Sstevel@tonic-gate 16177c478bd9Sstevel@tonic-gate/* 16187c478bd9Sstevel@tonic-gate * D$ parity error trap (trap 71) at TL=0. 16197c478bd9Sstevel@tonic-gate * tt0_dperr is replaced by dcache_parity_instr in cpu_init_trap of 16207c478bd9Sstevel@tonic-gate * the various architecture-specific files. This merely sets up the 16217c478bd9Sstevel@tonic-gate * arguments for cpu_parity_error and calls it via sys_trap. 16227c478bd9Sstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must 16237c478bd9Sstevel@tonic-gate * be relocatable. 16247c478bd9Sstevel@tonic-gate */ 16257c478bd9Sstevel@tonic-gate ENTRY_NP(dcache_parity_instr) 16267c478bd9Sstevel@tonic-gate membar #Sync ! Cheetah+ requires membar #Sync 16277c478bd9Sstevel@tonic-gate set cpu_parity_error, %g1 16287c478bd9Sstevel@tonic-gate or %g0, CH_ERR_DPE, %g2 16297c478bd9Sstevel@tonic-gate rdpr %tpc, %g3 16307c478bd9Sstevel@tonic-gate sethi %hi(sys_trap), %g7 16317c478bd9Sstevel@tonic-gate jmp %g7 + %lo(sys_trap) 16327c478bd9Sstevel@tonic-gate mov PIL_15, %g4 ! run at pil 15 16337c478bd9Sstevel@tonic-gate SET_SIZE(dcache_parity_instr) 16347c478bd9Sstevel@tonic-gate 16357c478bd9Sstevel@tonic-gate 16367c478bd9Sstevel@tonic-gate/* 16377c478bd9Sstevel@tonic-gate * D$ parity error trap (trap 71) at TL>0. 16387c478bd9Sstevel@tonic-gate * tt1_dperr is replaced by dcache_parity_tl1_instr in cpu_init_trap of 16397c478bd9Sstevel@tonic-gate * the various architecture-specific files. This generates a "Software 16407c478bd9Sstevel@tonic-gate * Trap 1" at TL>0, which goes to dcache_parity_tl1_cont_instr, and we 16417c478bd9Sstevel@tonic-gate * continue the handling there. 16427c478bd9Sstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must 16437c478bd9Sstevel@tonic-gate * be relocatable. 16447c478bd9Sstevel@tonic-gate */ 16457c478bd9Sstevel@tonic-gate ENTRY_NP(dcache_parity_tl1_instr) 16467c478bd9Sstevel@tonic-gate CH_ERR_TL1_TRAPENTRY(SWTRAP_1); 16477c478bd9Sstevel@tonic-gate SET_SIZE(dcache_parity_tl1_instr) 16487c478bd9Sstevel@tonic-gate 16497c478bd9Sstevel@tonic-gate 16507c478bd9Sstevel@tonic-gate/* 16517c478bd9Sstevel@tonic-gate * Software trap 1 at TL>0. 16527c478bd9Sstevel@tonic-gate * tt1_swtrap1 is replaced by dcache_parity_tl1_cont_instr in cpu_init_trap 16537c478bd9Sstevel@tonic-gate * of the various architecture-specific files. This is used as a continuation 16547c478bd9Sstevel@tonic-gate * of the dcache parity handling where we've bought an extra TL level, so we 16557c478bd9Sstevel@tonic-gate * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1 16567c478bd9Sstevel@tonic-gate * and %g2. Note that %tstate has bits 0-2 and then bits 8-19 as r/w, 16577c478bd9Sstevel@tonic-gate * there's a reserved hole from 3-7. We only use bits 0-1 and 8-9 (the low 16587c478bd9Sstevel@tonic-gate * order two bits from %g1 and %g2 respectively). 16597c478bd9Sstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must 16607c478bd9Sstevel@tonic-gate * be relocatable. 16617c478bd9Sstevel@tonic-gate */ 16627c478bd9Sstevel@tonic-gate ENTRY_NP(dcache_parity_tl1_cont_instr) 16637c478bd9Sstevel@tonic-gate CH_ERR_TL1_SWTRAPENTRY(dcache_parity_tl1_err); 16647c478bd9Sstevel@tonic-gate SET_SIZE(dcache_parity_tl1_cont_instr) 16657c478bd9Sstevel@tonic-gate 16667c478bd9Sstevel@tonic-gate/* 16677c478bd9Sstevel@tonic-gate * D$ parity error at TL>0 handler 16687c478bd9Sstevel@tonic-gate * We get here via trap 71 at TL>0->Software trap 1 at TL>0. We enter 16697c478bd9Sstevel@tonic-gate * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate. 16707c478bd9Sstevel@tonic-gate */ 16717c478bd9Sstevel@tonic-gate 16727c478bd9Sstevel@tonic-gate ENTRY_NP(dcache_parity_tl1_err) 16737c478bd9Sstevel@tonic-gate 16747c478bd9Sstevel@tonic-gate /* 16757c478bd9Sstevel@tonic-gate * This macro saves all the %g registers in the ch_err_tl1_data 16767c478bd9Sstevel@tonic-gate * structure, updates the ch_err_tl1_flags and saves the %tpc in 16777c478bd9Sstevel@tonic-gate * ch_err_tl1_tpc. At the end of this macro, %g1 will point to 16787c478bd9Sstevel@tonic-gate * the ch_err_tl1_data structure and %g2 will have the original 16797c478bd9Sstevel@tonic-gate * flags in the ch_err_tl1_data structure. All %g registers 16807c478bd9Sstevel@tonic-gate * except for %g1 and %g2 will be available. 16817c478bd9Sstevel@tonic-gate */ 16827c478bd9Sstevel@tonic-gate CH_ERR_TL1_ENTER(CH_ERR_DPE); 16837c478bd9Sstevel@tonic-gate 16847c478bd9Sstevel@tonic-gate#ifdef TRAPTRACE 16857c478bd9Sstevel@tonic-gate /* 16867c478bd9Sstevel@tonic-gate * Get current trap trace entry physical pointer. 16877c478bd9Sstevel@tonic-gate */ 16887c478bd9Sstevel@tonic-gate CPU_INDEX(%g6, %g5) 16897c478bd9Sstevel@tonic-gate sll %g6, TRAPTR_SIZE_SHIFT, %g6 16907c478bd9Sstevel@tonic-gate set trap_trace_ctl, %g5 16917c478bd9Sstevel@tonic-gate add %g6, %g5, %g6 16927c478bd9Sstevel@tonic-gate ld [%g6 + TRAPTR_LIMIT], %g5 16937c478bd9Sstevel@tonic-gate tst %g5 16947c478bd9Sstevel@tonic-gate be %icc, dpe_tl1_skip_tt 16957c478bd9Sstevel@tonic-gate nop 16967c478bd9Sstevel@tonic-gate ldx [%g6 + TRAPTR_PBASE], %g5 16977c478bd9Sstevel@tonic-gate ld [%g6 + TRAPTR_OFFSET], %g4 16987c478bd9Sstevel@tonic-gate add %g5, %g4, %g5 16997c478bd9Sstevel@tonic-gate 17007c478bd9Sstevel@tonic-gate /* 17017c478bd9Sstevel@tonic-gate * Create trap trace entry. 17027c478bd9Sstevel@tonic-gate */ 17037c478bd9Sstevel@tonic-gate rd %asi, %g7 17047c478bd9Sstevel@tonic-gate wr %g0, TRAPTR_ASI, %asi 17057c478bd9Sstevel@tonic-gate rd STICK, %g4 17067c478bd9Sstevel@tonic-gate stxa %g4, [%g5 + TRAP_ENT_TICK]%asi 17077c478bd9Sstevel@tonic-gate rdpr %tl, %g4 17087c478bd9Sstevel@tonic-gate stha %g4, [%g5 + TRAP_ENT_TL]%asi 17097c478bd9Sstevel@tonic-gate rdpr %tt, %g4 17107c478bd9Sstevel@tonic-gate stha %g4, [%g5 + TRAP_ENT_TT]%asi 17117c478bd9Sstevel@tonic-gate rdpr %tpc, %g4 17127c478bd9Sstevel@tonic-gate stna %g4, [%g5 + TRAP_ENT_TPC]%asi 17137c478bd9Sstevel@tonic-gate rdpr %tstate, %g4 17147c478bd9Sstevel@tonic-gate stxa %g4, [%g5 + TRAP_ENT_TSTATE]%asi 17157c478bd9Sstevel@tonic-gate stna %sp, [%g5 + TRAP_ENT_SP]%asi 17167c478bd9Sstevel@tonic-gate stna %g0, [%g5 + TRAP_ENT_TR]%asi 17177c478bd9Sstevel@tonic-gate stna %g0, [%g5 + TRAP_ENT_F1]%asi 17187c478bd9Sstevel@tonic-gate stna %g0, [%g5 + TRAP_ENT_F2]%asi 17197c478bd9Sstevel@tonic-gate stna %g0, [%g5 + TRAP_ENT_F3]%asi 17207c478bd9Sstevel@tonic-gate stna %g0, [%g5 + TRAP_ENT_F4]%asi 17217c478bd9Sstevel@tonic-gate wr %g0, %g7, %asi 17227c478bd9Sstevel@tonic-gate 17237c478bd9Sstevel@tonic-gate /* 17247c478bd9Sstevel@tonic-gate * Advance trap trace pointer. 17257c478bd9Sstevel@tonic-gate */ 17267c478bd9Sstevel@tonic-gate ld [%g6 + TRAPTR_OFFSET], %g5 17277c478bd9Sstevel@tonic-gate ld [%g6 + TRAPTR_LIMIT], %g4 17287c478bd9Sstevel@tonic-gate st %g5, [%g6 + TRAPTR_LAST_OFFSET] 17297c478bd9Sstevel@tonic-gate add %g5, TRAP_ENT_SIZE, %g5 17307c478bd9Sstevel@tonic-gate sub %g4, TRAP_ENT_SIZE, %g4 17317c478bd9Sstevel@tonic-gate cmp %g5, %g4 17327c478bd9Sstevel@tonic-gate movge %icc, 0, %g5 17337c478bd9Sstevel@tonic-gate st %g5, [%g6 + TRAPTR_OFFSET] 17347c478bd9Sstevel@tonic-gatedpe_tl1_skip_tt: 17357c478bd9Sstevel@tonic-gate#endif /* TRAPTRACE */ 17367c478bd9Sstevel@tonic-gate 17377c478bd9Sstevel@tonic-gate /* 17387c478bd9Sstevel@tonic-gate * I$ and D$ are automatically turned off by HW when the CPU hits 17397c478bd9Sstevel@tonic-gate * a dcache or icache parity error so we will just leave those two 17407c478bd9Sstevel@tonic-gate * off for now to avoid repeating this trap. 17417c478bd9Sstevel@tonic-gate * For Panther, however, since we trap on P$ data parity errors 17427c478bd9Sstevel@tonic-gate * and HW does not automatically disable P$, we need to disable it 17437c478bd9Sstevel@tonic-gate * here so that we don't encounter any recursive traps when we 17447c478bd9Sstevel@tonic-gate * issue the retry. 17457c478bd9Sstevel@tonic-gate */ 17467c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_DCU, %g3 17477c478bd9Sstevel@tonic-gate mov 1, %g4 17487c478bd9Sstevel@tonic-gate sllx %g4, DCU_PE_SHIFT, %g4 17497c478bd9Sstevel@tonic-gate andn %g3, %g4, %g3 17507c478bd9Sstevel@tonic-gate stxa %g3, [%g0]ASI_DCU 17517c478bd9Sstevel@tonic-gate membar #Sync 17527c478bd9Sstevel@tonic-gate 17537c478bd9Sstevel@tonic-gate /* 17547c478bd9Sstevel@tonic-gate * We fall into this macro if we've successfully logged the error in 17557c478bd9Sstevel@tonic-gate * the ch_err_tl1_data structure and want the PIL15 softint to pick 17567c478bd9Sstevel@tonic-gate * it up and log it. %g1 must point to the ch_err_tl1_data structure. 17577c478bd9Sstevel@tonic-gate * Restores the %g registers and issues retry. 17587c478bd9Sstevel@tonic-gate */ 17597c478bd9Sstevel@tonic-gate CH_ERR_TL1_EXIT; 17607c478bd9Sstevel@tonic-gate SET_SIZE(dcache_parity_tl1_err) 17617c478bd9Sstevel@tonic-gate 17627c478bd9Sstevel@tonic-gate/* 17637c478bd9Sstevel@tonic-gate * I$ parity error trap (trap 72) at TL=0. 17647c478bd9Sstevel@tonic-gate * tt0_iperr is replaced by icache_parity_instr in cpu_init_trap of 17657c478bd9Sstevel@tonic-gate * the various architecture-specific files. This merely sets up the 17667c478bd9Sstevel@tonic-gate * arguments for cpu_parity_error and calls it via sys_trap. 17677c478bd9Sstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must 17687c478bd9Sstevel@tonic-gate * be relocatable. 17697c478bd9Sstevel@tonic-gate */ 17707c478bd9Sstevel@tonic-gate 17717c478bd9Sstevel@tonic-gate ENTRY_NP(icache_parity_instr) 17727c478bd9Sstevel@tonic-gate membar #Sync ! Cheetah+ requires membar #Sync 17737c478bd9Sstevel@tonic-gate set cpu_parity_error, %g1 17747c478bd9Sstevel@tonic-gate or %g0, CH_ERR_IPE, %g2 17757c478bd9Sstevel@tonic-gate rdpr %tpc, %g3 17767c478bd9Sstevel@tonic-gate sethi %hi(sys_trap), %g7 17777c478bd9Sstevel@tonic-gate jmp %g7 + %lo(sys_trap) 17787c478bd9Sstevel@tonic-gate mov PIL_15, %g4 ! run at pil 15 17797c478bd9Sstevel@tonic-gate SET_SIZE(icache_parity_instr) 17807c478bd9Sstevel@tonic-gate 17817c478bd9Sstevel@tonic-gate/* 17827c478bd9Sstevel@tonic-gate * I$ parity error trap (trap 72) at TL>0. 17837c478bd9Sstevel@tonic-gate * tt1_iperr is replaced by icache_parity_tl1_instr in cpu_init_trap of 17847c478bd9Sstevel@tonic-gate * the various architecture-specific files. This generates a "Software 17857c478bd9Sstevel@tonic-gate * Trap 2" at TL>0, which goes to icache_parity_tl1_cont_instr, and we 17867c478bd9Sstevel@tonic-gate * continue the handling there. 17877c478bd9Sstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must 17887c478bd9Sstevel@tonic-gate * be relocatable. 17897c478bd9Sstevel@tonic-gate */ 17907c478bd9Sstevel@tonic-gate ENTRY_NP(icache_parity_tl1_instr) 17917c478bd9Sstevel@tonic-gate CH_ERR_TL1_TRAPENTRY(SWTRAP_2); 17927c478bd9Sstevel@tonic-gate SET_SIZE(icache_parity_tl1_instr) 17937c478bd9Sstevel@tonic-gate 17947c478bd9Sstevel@tonic-gate/* 17957c478bd9Sstevel@tonic-gate * Software trap 2 at TL>0. 17967c478bd9Sstevel@tonic-gate * tt1_swtrap2 is replaced by icache_parity_tl1_cont_instr in cpu_init_trap 17977c478bd9Sstevel@tonic-gate * of the various architecture-specific files. This is used as a continuation 17987c478bd9Sstevel@tonic-gate * of the icache parity handling where we've bought an extra TL level, so we 17997c478bd9Sstevel@tonic-gate * can use %tpc, %tnpc, %tstate to temporarily save the value of registers %g1 18007c478bd9Sstevel@tonic-gate * and %g2. Note that %tstate has bits 0-2 and then bits 8-19 as r/w, 18017c478bd9Sstevel@tonic-gate * there's a reserved hole from 3-7. We only use bits 0-1 and 8-9 (the low 18027c478bd9Sstevel@tonic-gate * order two bits from %g1 and %g2 respectively). 18037c478bd9Sstevel@tonic-gate * NB: Must be 8 instructions or less to fit in trap table and code must 18047c478bd9Sstevel@tonic-gate * be relocatable. 18057c478bd9Sstevel@tonic-gate */ 18067c478bd9Sstevel@tonic-gate ENTRY_NP(icache_parity_tl1_cont_instr) 18077c478bd9Sstevel@tonic-gate CH_ERR_TL1_SWTRAPENTRY(icache_parity_tl1_err); 18087c478bd9Sstevel@tonic-gate SET_SIZE(icache_parity_tl1_cont_instr) 18097c478bd9Sstevel@tonic-gate 18107c478bd9Sstevel@tonic-gate 18117c478bd9Sstevel@tonic-gate/* 18127c478bd9Sstevel@tonic-gate * I$ parity error at TL>0 handler 18137c478bd9Sstevel@tonic-gate * We get here via trap 72 at TL>0->Software trap 2 at TL>0. We enter 18147c478bd9Sstevel@tonic-gate * this routine with %g1 and %g2 already saved in %tpc, %tnpc and %tstate. 18157c478bd9Sstevel@tonic-gate */ 18167c478bd9Sstevel@tonic-gate 18177c478bd9Sstevel@tonic-gate ENTRY_NP(icache_parity_tl1_err) 18187c478bd9Sstevel@tonic-gate 18197c478bd9Sstevel@tonic-gate /* 18207c478bd9Sstevel@tonic-gate * This macro saves all the %g registers in the ch_err_tl1_data 18217c478bd9Sstevel@tonic-gate * structure, updates the ch_err_tl1_flags and saves the %tpc in 18227c478bd9Sstevel@tonic-gate * ch_err_tl1_tpc. At the end of this macro, %g1 will point to 18237c478bd9Sstevel@tonic-gate * the ch_err_tl1_data structure and %g2 will have the original 18247c478bd9Sstevel@tonic-gate * flags in the ch_err_tl1_data structure. All %g registers 18257c478bd9Sstevel@tonic-gate * except for %g1 and %g2 will be available. 18267c478bd9Sstevel@tonic-gate */ 18277c478bd9Sstevel@tonic-gate CH_ERR_TL1_ENTER(CH_ERR_IPE); 18287c478bd9Sstevel@tonic-gate 18297c478bd9Sstevel@tonic-gate#ifdef TRAPTRACE 18307c478bd9Sstevel@tonic-gate /* 18317c478bd9Sstevel@tonic-gate * Get current trap trace entry physical pointer. 18327c478bd9Sstevel@tonic-gate */ 18337c478bd9Sstevel@tonic-gate CPU_INDEX(%g6, %g5) 18347c478bd9Sstevel@tonic-gate sll %g6, TRAPTR_SIZE_SHIFT, %g6 18357c478bd9Sstevel@tonic-gate set trap_trace_ctl, %g5 18367c478bd9Sstevel@tonic-gate add %g6, %g5, %g6 18377c478bd9Sstevel@tonic-gate ld [%g6 + TRAPTR_LIMIT], %g5 18387c478bd9Sstevel@tonic-gate tst %g5 18397c478bd9Sstevel@tonic-gate be %icc, ipe_tl1_skip_tt 18407c478bd9Sstevel@tonic-gate nop 18417c478bd9Sstevel@tonic-gate ldx [%g6 + TRAPTR_PBASE], %g5 18427c478bd9Sstevel@tonic-gate ld [%g6 + TRAPTR_OFFSET], %g4 18437c478bd9Sstevel@tonic-gate add %g5, %g4, %g5 18447c478bd9Sstevel@tonic-gate 18457c478bd9Sstevel@tonic-gate /* 18467c478bd9Sstevel@tonic-gate * Create trap trace entry. 18477c478bd9Sstevel@tonic-gate */ 18487c478bd9Sstevel@tonic-gate rd %asi, %g7 18497c478bd9Sstevel@tonic-gate wr %g0, TRAPTR_ASI, %asi 18507c478bd9Sstevel@tonic-gate rd STICK, %g4 18517c478bd9Sstevel@tonic-gate stxa %g4, [%g5 + TRAP_ENT_TICK]%asi 18527c478bd9Sstevel@tonic-gate rdpr %tl, %g4 18537c478bd9Sstevel@tonic-gate stha %g4, [%g5 + TRAP_ENT_TL]%asi 18547c478bd9Sstevel@tonic-gate rdpr %tt, %g4 18557c478bd9Sstevel@tonic-gate stha %g4, [%g5 + TRAP_ENT_TT]%asi 18567c478bd9Sstevel@tonic-gate rdpr %tpc, %g4 18577c478bd9Sstevel@tonic-gate stna %g4, [%g5 + TRAP_ENT_TPC]%asi 18587c478bd9Sstevel@tonic-gate rdpr %tstate, %g4 18597c478bd9Sstevel@tonic-gate stxa %g4, [%g5 + TRAP_ENT_TSTATE]%asi 18607c478bd9Sstevel@tonic-gate stna %sp, [%g5 + TRAP_ENT_SP]%asi 18617c478bd9Sstevel@tonic-gate stna %g0, [%g5 + TRAP_ENT_TR]%asi 18627c478bd9Sstevel@tonic-gate stna %g0, [%g5 + TRAP_ENT_F1]%asi 18637c478bd9Sstevel@tonic-gate stna %g0, [%g5 + TRAP_ENT_F2]%asi 18647c478bd9Sstevel@tonic-gate stna %g0, [%g5 + TRAP_ENT_F3]%asi 18657c478bd9Sstevel@tonic-gate stna %g0, [%g5 + TRAP_ENT_F4]%asi 18667c478bd9Sstevel@tonic-gate wr %g0, %g7, %asi 18677c478bd9Sstevel@tonic-gate 18687c478bd9Sstevel@tonic-gate /* 18697c478bd9Sstevel@tonic-gate * Advance trap trace pointer. 18707c478bd9Sstevel@tonic-gate */ 18717c478bd9Sstevel@tonic-gate ld [%g6 + TRAPTR_OFFSET], %g5 18727c478bd9Sstevel@tonic-gate ld [%g6 + TRAPTR_LIMIT], %g4 18737c478bd9Sstevel@tonic-gate st %g5, [%g6 + TRAPTR_LAST_OFFSET] 18747c478bd9Sstevel@tonic-gate add %g5, TRAP_ENT_SIZE, %g5 18757c478bd9Sstevel@tonic-gate sub %g4, TRAP_ENT_SIZE, %g4 18767c478bd9Sstevel@tonic-gate cmp %g5, %g4 18777c478bd9Sstevel@tonic-gate movge %icc, 0, %g5 18787c478bd9Sstevel@tonic-gate st %g5, [%g6 + TRAPTR_OFFSET] 18797c478bd9Sstevel@tonic-gateipe_tl1_skip_tt: 18807c478bd9Sstevel@tonic-gate#endif /* TRAPTRACE */ 18817c478bd9Sstevel@tonic-gate 18827c478bd9Sstevel@tonic-gate /* 18837c478bd9Sstevel@tonic-gate * We fall into this macro if we've successfully logged the error in 18847c478bd9Sstevel@tonic-gate * the ch_err_tl1_data structure and want the PIL15 softint to pick 18857c478bd9Sstevel@tonic-gate * it up and log it. %g1 must point to the ch_err_tl1_data structure. 18867c478bd9Sstevel@tonic-gate * Restores the %g registers and issues retry. 18877c478bd9Sstevel@tonic-gate */ 18887c478bd9Sstevel@tonic-gate CH_ERR_TL1_EXIT; 18897c478bd9Sstevel@tonic-gate 18907c478bd9Sstevel@tonic-gate SET_SIZE(icache_parity_tl1_err) 18917c478bd9Sstevel@tonic-gate 18927c478bd9Sstevel@tonic-gate#endif /* CPU_IMP_L1_CACHE_PARITY */ 18937c478bd9Sstevel@tonic-gate 18947c478bd9Sstevel@tonic-gate 18957c478bd9Sstevel@tonic-gate/* 18967c478bd9Sstevel@tonic-gate * The itlb_rd_entry and dtlb_rd_entry functions return the tag portion of the 18977c478bd9Sstevel@tonic-gate * tte, the virtual address, and the ctxnum of the specified tlb entry. They 18987c478bd9Sstevel@tonic-gate * should only be used in places where you have no choice but to look at the 18997c478bd9Sstevel@tonic-gate * tlb itself. 19007c478bd9Sstevel@tonic-gate * 19017c478bd9Sstevel@tonic-gate * Note: These two routines are required by the Estar "cpr" loadable module. 19027c478bd9Sstevel@tonic-gate */ 19037c478bd9Sstevel@tonic-gate 19047c478bd9Sstevel@tonic-gate ENTRY_NP(itlb_rd_entry) 19057c478bd9Sstevel@tonic-gate sllx %o0, 3, %o0 19067c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_ITLB_ACCESS, %g1 19077c478bd9Sstevel@tonic-gate stx %g1, [%o1] 19087c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_ITLB_TAGREAD, %g2 19097c478bd9Sstevel@tonic-gate set TAGREAD_CTX_MASK, %o4 19107c478bd9Sstevel@tonic-gate andn %g2, %o4, %o5 19117c478bd9Sstevel@tonic-gate retl 19127c478bd9Sstevel@tonic-gate stx %o5, [%o2] 19137c478bd9Sstevel@tonic-gate SET_SIZE(itlb_rd_entry) 19147c478bd9Sstevel@tonic-gate 19157c478bd9Sstevel@tonic-gate 19167c478bd9Sstevel@tonic-gate ENTRY_NP(dtlb_rd_entry) 19177c478bd9Sstevel@tonic-gate sllx %o0, 3, %o0 19187c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_DTLB_ACCESS, %g1 19197c478bd9Sstevel@tonic-gate stx %g1, [%o1] 19207c478bd9Sstevel@tonic-gate ldxa [%o0]ASI_DTLB_TAGREAD, %g2 19217c478bd9Sstevel@tonic-gate set TAGREAD_CTX_MASK, %o4 19227c478bd9Sstevel@tonic-gate andn %g2, %o4, %o5 19237c478bd9Sstevel@tonic-gate retl 19247c478bd9Sstevel@tonic-gate stx %o5, [%o2] 19257c478bd9Sstevel@tonic-gate SET_SIZE(dtlb_rd_entry) 19267c478bd9Sstevel@tonic-gate 19277c478bd9Sstevel@tonic-gate 19287c478bd9Sstevel@tonic-gate#if !(defined(JALAPENO) || defined(SERRANO)) 19297c478bd9Sstevel@tonic-gate 19307c478bd9Sstevel@tonic-gate ENTRY(get_safari_config) 19317c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_SAFARI_CONFIG, %o0 19327c478bd9Sstevel@tonic-gate retl 19337c478bd9Sstevel@tonic-gate nop 19347c478bd9Sstevel@tonic-gate SET_SIZE(get_safari_config) 19357c478bd9Sstevel@tonic-gate 19367c478bd9Sstevel@tonic-gate 19377c478bd9Sstevel@tonic-gate ENTRY(set_safari_config) 19387c478bd9Sstevel@tonic-gate stxa %o0, [%g0]ASI_SAFARI_CONFIG 19397c478bd9Sstevel@tonic-gate membar #Sync 19407c478bd9Sstevel@tonic-gate retl 19417c478bd9Sstevel@tonic-gate nop 19427c478bd9Sstevel@tonic-gate SET_SIZE(set_safari_config) 19437c478bd9Sstevel@tonic-gate 19447c478bd9Sstevel@tonic-gate#endif /* !(JALAPENO || SERRANO) */ 19457c478bd9Sstevel@tonic-gate 19467c478bd9Sstevel@tonic-gate 19477c478bd9Sstevel@tonic-gate /* 19487c478bd9Sstevel@tonic-gate * Clear the NPT (non-privileged trap) bit in the %tick/%stick 19497c478bd9Sstevel@tonic-gate * registers. In an effort to make the change in the 19507c478bd9Sstevel@tonic-gate * tick/stick counter as consistent as possible, we disable 19517c478bd9Sstevel@tonic-gate * all interrupts while we're changing the registers. We also 19527c478bd9Sstevel@tonic-gate * ensure that the read and write instructions are in the same 19537c478bd9Sstevel@tonic-gate * line in the instruction cache. 19547c478bd9Sstevel@tonic-gate */ 19557c478bd9Sstevel@tonic-gate ENTRY_NP(cpu_clearticknpt) 19567c478bd9Sstevel@tonic-gate rdpr %pstate, %g1 /* save processor state */ 19577c478bd9Sstevel@tonic-gate andn %g1, PSTATE_IE, %g3 /* turn off */ 19587c478bd9Sstevel@tonic-gate wrpr %g0, %g3, %pstate /* interrupts */ 19597c478bd9Sstevel@tonic-gate rdpr %tick, %g2 /* get tick register */ 19607c478bd9Sstevel@tonic-gate brgez,pn %g2, 1f /* if NPT bit off, we're done */ 19617c478bd9Sstevel@tonic-gate mov 1, %g3 /* create mask */ 19627c478bd9Sstevel@tonic-gate sllx %g3, 63, %g3 /* for NPT bit */ 19637c478bd9Sstevel@tonic-gate ba,a,pt %xcc, 2f 19647c478bd9Sstevel@tonic-gate .align 8 /* Ensure rd/wr in same i$ line */ 19657c478bd9Sstevel@tonic-gate2: 19667c478bd9Sstevel@tonic-gate rdpr %tick, %g2 /* get tick register */ 19677c478bd9Sstevel@tonic-gate wrpr %g3, %g2, %tick /* write tick register, */ 19687c478bd9Sstevel@tonic-gate /* clearing NPT bit */ 19697c478bd9Sstevel@tonic-gate1: 19707c478bd9Sstevel@tonic-gate rd STICK, %g2 /* get stick register */ 19717c478bd9Sstevel@tonic-gate brgez,pn %g2, 3f /* if NPT bit off, we're done */ 19727c478bd9Sstevel@tonic-gate mov 1, %g3 /* create mask */ 19737c478bd9Sstevel@tonic-gate sllx %g3, 63, %g3 /* for NPT bit */ 19747c478bd9Sstevel@tonic-gate ba,a,pt %xcc, 4f 19757c478bd9Sstevel@tonic-gate .align 8 /* Ensure rd/wr in same i$ line */ 19767c478bd9Sstevel@tonic-gate4: 19777c478bd9Sstevel@tonic-gate rd STICK, %g2 /* get stick register */ 19787c478bd9Sstevel@tonic-gate wr %g3, %g2, STICK /* write stick register, */ 19797c478bd9Sstevel@tonic-gate /* clearing NPT bit */ 19807c478bd9Sstevel@tonic-gate3: 19817c478bd9Sstevel@tonic-gate jmp %g4 + 4 19827c478bd9Sstevel@tonic-gate wrpr %g0, %g1, %pstate /* restore processor state */ 1983*55fea89dSDan Cross 19847c478bd9Sstevel@tonic-gate SET_SIZE(cpu_clearticknpt) 19857c478bd9Sstevel@tonic-gate 19867c478bd9Sstevel@tonic-gate 19877c478bd9Sstevel@tonic-gate#if defined(CPU_IMP_L1_CACHE_PARITY) 19887c478bd9Sstevel@tonic-gate 19897c478bd9Sstevel@tonic-gate/* 19907c478bd9Sstevel@tonic-gate * correct_dcache_parity(size_t size, size_t linesize) 19917c478bd9Sstevel@tonic-gate * 19927c478bd9Sstevel@tonic-gate * Correct D$ data parity by zeroing the data and initializing microtag 19937c478bd9Sstevel@tonic-gate * for all indexes and all ways of the D$. 19949b0bb795SJohn Levon * 19957c478bd9Sstevel@tonic-gate */ 19967c478bd9Sstevel@tonic-gate ENTRY(correct_dcache_parity) 19977c478bd9Sstevel@tonic-gate /* 19987c478bd9Sstevel@tonic-gate * Register Usage: 19997c478bd9Sstevel@tonic-gate * 20007c478bd9Sstevel@tonic-gate * %o0 = input D$ size 20017c478bd9Sstevel@tonic-gate * %o1 = input D$ line size 20027c478bd9Sstevel@tonic-gate * %o2 = scratch 20037c478bd9Sstevel@tonic-gate * %o3 = scratch 20047c478bd9Sstevel@tonic-gate * %o4 = scratch 20057c478bd9Sstevel@tonic-gate */ 20067c478bd9Sstevel@tonic-gate 20077c478bd9Sstevel@tonic-gate sub %o0, %o1, %o0 ! init cache line address 20087c478bd9Sstevel@tonic-gate 20097c478bd9Sstevel@tonic-gate /* 20107c478bd9Sstevel@tonic-gate * For Panther CPUs, we also need to clear the data parity bits 20117c478bd9Sstevel@tonic-gate * using DC_data_parity bit of the ASI_DCACHE_DATA register. 20127c478bd9Sstevel@tonic-gate */ 20137c478bd9Sstevel@tonic-gate GET_CPU_IMPL(%o3) 20147c478bd9Sstevel@tonic-gate cmp %o3, PANTHER_IMPL 20157c478bd9Sstevel@tonic-gate bne 1f 20167c478bd9Sstevel@tonic-gate clr %o3 ! zero for non-Panther 20177c478bd9Sstevel@tonic-gate mov 1, %o3 20187c478bd9Sstevel@tonic-gate sll %o3, PN_DC_DATA_PARITY_BIT_SHIFT, %o3 20197c478bd9Sstevel@tonic-gate 20207c478bd9Sstevel@tonic-gate1: 20217c478bd9Sstevel@tonic-gate /* 20227c478bd9Sstevel@tonic-gate * Set utag = way since it must be unique within an index. 20237c478bd9Sstevel@tonic-gate */ 20247c478bd9Sstevel@tonic-gate srl %o0, 14, %o2 ! get cache way (DC_way) 20257c478bd9Sstevel@tonic-gate membar #Sync ! required before ASI_DC_UTAG 20267c478bd9Sstevel@tonic-gate stxa %o2, [%o0]ASI_DC_UTAG ! set D$ utag = cache way 20277c478bd9Sstevel@tonic-gate membar #Sync ! required after ASI_DC_UTAG 20287c478bd9Sstevel@tonic-gate 20297c478bd9Sstevel@tonic-gate /* 20307c478bd9Sstevel@tonic-gate * Zero line of D$ data (and data parity bits for Panther) 20317c478bd9Sstevel@tonic-gate */ 20327c478bd9Sstevel@tonic-gate sub %o1, 8, %o2 20337c478bd9Sstevel@tonic-gate or %o0, %o3, %o4 ! same address + DC_data_parity 20347c478bd9Sstevel@tonic-gate2: 20357c478bd9Sstevel@tonic-gate membar #Sync ! required before ASI_DC_DATA 20367c478bd9Sstevel@tonic-gate stxa %g0, [%o0 + %o2]ASI_DC_DATA ! zero 8 bytes of D$ data 20377c478bd9Sstevel@tonic-gate membar #Sync ! required after ASI_DC_DATA 20387c478bd9Sstevel@tonic-gate /* 20397c478bd9Sstevel@tonic-gate * We also clear the parity bits if this is a panther. For non-Panther 20407c478bd9Sstevel@tonic-gate * CPUs, we simply end up clearing the $data register twice. 20417c478bd9Sstevel@tonic-gate */ 20427c478bd9Sstevel@tonic-gate stxa %g0, [%o4 + %o2]ASI_DC_DATA 20437c478bd9Sstevel@tonic-gate membar #Sync 20447c478bd9Sstevel@tonic-gate 20457c478bd9Sstevel@tonic-gate subcc %o2, 8, %o2 20467c478bd9Sstevel@tonic-gate bge 2b 20477c478bd9Sstevel@tonic-gate nop 20487c478bd9Sstevel@tonic-gate 20497c478bd9Sstevel@tonic-gate subcc %o0, %o1, %o0 20507c478bd9Sstevel@tonic-gate bge 1b 20517c478bd9Sstevel@tonic-gate nop 20527c478bd9Sstevel@tonic-gate 20537c478bd9Sstevel@tonic-gate retl 20547c478bd9Sstevel@tonic-gate nop 20557c478bd9Sstevel@tonic-gate SET_SIZE(correct_dcache_parity) 20567c478bd9Sstevel@tonic-gate 20577c478bd9Sstevel@tonic-gate#endif /* CPU_IMP_L1_CACHE_PARITY */ 20587c478bd9Sstevel@tonic-gate 20597c478bd9Sstevel@tonic-gate 20607c478bd9Sstevel@tonic-gate ENTRY_NP(stick_timestamp) 20617c478bd9Sstevel@tonic-gate rd STICK, %g1 ! read stick reg 20627c478bd9Sstevel@tonic-gate sllx %g1, 1, %g1 20637c478bd9Sstevel@tonic-gate srlx %g1, 1, %g1 ! clear npt bit 20647c478bd9Sstevel@tonic-gate 20657c478bd9Sstevel@tonic-gate retl 20667c478bd9Sstevel@tonic-gate stx %g1, [%o0] ! store the timestamp 20677c478bd9Sstevel@tonic-gate SET_SIZE(stick_timestamp) 20687c478bd9Sstevel@tonic-gate 20697c478bd9Sstevel@tonic-gate 20707c478bd9Sstevel@tonic-gate ENTRY_NP(stick_adj) 20717c478bd9Sstevel@tonic-gate rdpr %pstate, %g1 ! save processor state 20727c478bd9Sstevel@tonic-gate andn %g1, PSTATE_IE, %g3 20737c478bd9Sstevel@tonic-gate ba 1f ! cache align stick adj 20747c478bd9Sstevel@tonic-gate wrpr %g0, %g3, %pstate ! turn off interrupts 20757c478bd9Sstevel@tonic-gate 20767c478bd9Sstevel@tonic-gate .align 16 20777c478bd9Sstevel@tonic-gate1: nop 20787c478bd9Sstevel@tonic-gate 20797c478bd9Sstevel@tonic-gate rd STICK, %g4 ! read stick reg 20807c478bd9Sstevel@tonic-gate add %g4, %o0, %o1 ! adjust stick with skew 20817c478bd9Sstevel@tonic-gate wr %o1, %g0, STICK ! write stick reg 20827c478bd9Sstevel@tonic-gate 20837c478bd9Sstevel@tonic-gate retl 20847c478bd9Sstevel@tonic-gate wrpr %g1, %pstate ! restore processor state 20857c478bd9Sstevel@tonic-gate SET_SIZE(stick_adj) 20867c478bd9Sstevel@tonic-gate 20877c478bd9Sstevel@tonic-gate ENTRY_NP(kdi_get_stick) 20887c478bd9Sstevel@tonic-gate rd STICK, %g1 20897c478bd9Sstevel@tonic-gate stx %g1, [%o0] 20907c478bd9Sstevel@tonic-gate retl 20917c478bd9Sstevel@tonic-gate mov %g0, %o0 20927c478bd9Sstevel@tonic-gate SET_SIZE(kdi_get_stick) 20937c478bd9Sstevel@tonic-gate 20947c478bd9Sstevel@tonic-gate/* 20957c478bd9Sstevel@tonic-gate * Invalidate the specified line from the D$. 20967c478bd9Sstevel@tonic-gate * 20977c478bd9Sstevel@tonic-gate * Register usage: 20987c478bd9Sstevel@tonic-gate * %o0 - index for the invalidation, specifies DC_way and DC_addr 20997c478bd9Sstevel@tonic-gate * 21007c478bd9Sstevel@tonic-gate * ASI_DC_TAG, 0x47, is used in the following manner. A 64-bit value is 21017c478bd9Sstevel@tonic-gate * stored to a particular DC_way and DC_addr in ASI_DC_TAG. 21027c478bd9Sstevel@tonic-gate * 21037c478bd9Sstevel@tonic-gate * The format of the stored 64-bit value is: 21047c478bd9Sstevel@tonic-gate * 21057c478bd9Sstevel@tonic-gate * +----------+--------+----------+ 21067c478bd9Sstevel@tonic-gate * | Reserved | DC_tag | DC_valid | 21077c478bd9Sstevel@tonic-gate * +----------+--------+----------+ 21087c478bd9Sstevel@tonic-gate * 63 31 30 1 0 21097c478bd9Sstevel@tonic-gate * 21107c478bd9Sstevel@tonic-gate * DC_tag is the 30-bit physical tag of the associated line. 21117c478bd9Sstevel@tonic-gate * DC_valid is the 1-bit valid field for both the physical and snoop tags. 21127c478bd9Sstevel@tonic-gate * 21137c478bd9Sstevel@tonic-gate * The format of the 64-bit DC_way and DC_addr into ASI_DC_TAG is: 21147c478bd9Sstevel@tonic-gate * 21157c478bd9Sstevel@tonic-gate * +----------+--------+----------+----------+ 21167c478bd9Sstevel@tonic-gate * | Reserved | DC_way | DC_addr | Reserved | 21177c478bd9Sstevel@tonic-gate * +----------+--------+----------+----------+ 21187c478bd9Sstevel@tonic-gate * 63 16 15 14 13 5 4 0 21197c478bd9Sstevel@tonic-gate * 21207c478bd9Sstevel@tonic-gate * DC_way is a 2-bit index that selects one of the 4 ways. 21217c478bd9Sstevel@tonic-gate * DC_addr is a 9-bit index that selects one of 512 tag/valid fields. 21227c478bd9Sstevel@tonic-gate * 21237c478bd9Sstevel@tonic-gate * Setting the DC_valid bit to zero for the specified DC_way and 21247c478bd9Sstevel@tonic-gate * DC_addr index into the D$ results in an invalidation of a D$ line. 21257c478bd9Sstevel@tonic-gate */ 21267c478bd9Sstevel@tonic-gate ENTRY(dcache_inval_line) 21277c478bd9Sstevel@tonic-gate sll %o0, 5, %o0 ! shift index into DC_way and DC_addr 21287c478bd9Sstevel@tonic-gate stxa %g0, [%o0]ASI_DC_TAG ! zero the DC_valid and DC_tag bits 21297c478bd9Sstevel@tonic-gate membar #Sync 21307c478bd9Sstevel@tonic-gate retl 21317c478bd9Sstevel@tonic-gate nop 21327c478bd9Sstevel@tonic-gate SET_SIZE(dcache_inval_line) 21337c478bd9Sstevel@tonic-gate 21347c478bd9Sstevel@tonic-gate/* 21357c478bd9Sstevel@tonic-gate * Invalidate the entire I$ 21367c478bd9Sstevel@tonic-gate * 21377c478bd9Sstevel@tonic-gate * Register usage: 21387c478bd9Sstevel@tonic-gate * %o0 - specifies IC_way, IC_addr, IC_tag 21397c478bd9Sstevel@tonic-gate * %o1 - scratch 21407c478bd9Sstevel@tonic-gate * %o2 - used to save and restore DCU value 21417c478bd9Sstevel@tonic-gate * %o3 - scratch 21427c478bd9Sstevel@tonic-gate * %o5 - used to save and restore PSTATE 21437c478bd9Sstevel@tonic-gate * 21447c478bd9Sstevel@tonic-gate * Due to the behavior of the I$ control logic when accessing ASI_IC_TAG, 21457c478bd9Sstevel@tonic-gate * the I$ should be turned off. Accesses to ASI_IC_TAG may collide and 21467c478bd9Sstevel@tonic-gate * block out snoops and invalidates to the I$, causing I$ consistency 21477c478bd9Sstevel@tonic-gate * to be broken. Before turning on the I$, all I$ lines must be invalidated. 21487c478bd9Sstevel@tonic-gate * 21497c478bd9Sstevel@tonic-gate * ASI_IC_TAG, 0x67, is used in the following manner. A 64-bit value is 21507c478bd9Sstevel@tonic-gate * stored to a particular IC_way, IC_addr, IC_tag in ASI_IC_TAG. The 21517c478bd9Sstevel@tonic-gate * info below describes store (write) use of ASI_IC_TAG. Note that read 21527c478bd9Sstevel@tonic-gate * use of ASI_IC_TAG behaves differently. 21537c478bd9Sstevel@tonic-gate * 21547c478bd9Sstevel@tonic-gate * The format of the stored 64-bit value is: 21557c478bd9Sstevel@tonic-gate * 21567c478bd9Sstevel@tonic-gate * +----------+--------+---------------+-----------+ 21577c478bd9Sstevel@tonic-gate * | Reserved | Valid | IC_vpred<7:0> | Undefined | 21587c478bd9Sstevel@tonic-gate * +----------+--------+---------------+-----------+ 21597c478bd9Sstevel@tonic-gate * 63 55 54 53 46 45 0 21607c478bd9Sstevel@tonic-gate * 21617c478bd9Sstevel@tonic-gate * Valid is the 1-bit valid field for both the physical and snoop tags. 21627c478bd9Sstevel@tonic-gate * IC_vpred is the 8-bit LPB bits for 8 instructions starting at 21637c478bd9Sstevel@tonic-gate * the 32-byte boundary aligned address specified by IC_addr. 21647c478bd9Sstevel@tonic-gate * 21657c478bd9Sstevel@tonic-gate * The format of the 64-bit IC_way, IC_addr, IC_tag into ASI_IC_TAG is: 21667c478bd9Sstevel@tonic-gate * 21677c478bd9Sstevel@tonic-gate * +----------+--------+---------+--------+---------+ 21687c478bd9Sstevel@tonic-gate * | Reserved | IC_way | IC_addr | IC_tag |Reserved | 21697c478bd9Sstevel@tonic-gate * +----------+--------+---------+--------+---------+ 21707c478bd9Sstevel@tonic-gate * 63 16 15 14 13 5 4 3 2 0 21717c478bd9Sstevel@tonic-gate * 21727c478bd9Sstevel@tonic-gate * IC_way is a 2-bit index that selects one of the 4 ways. 21737c478bd9Sstevel@tonic-gate * IC_addr[13:6] is an 8-bit index that selects one of 256 valid fields. 21747c478bd9Sstevel@tonic-gate * IC_addr[5] is a "don't care" for a store. 21757c478bd9Sstevel@tonic-gate * IC_tag set to 2 specifies that the stored value is to be interpreted 21767c478bd9Sstevel@tonic-gate * as containing Valid and IC_vpred as described above. 21777c478bd9Sstevel@tonic-gate * 21787c478bd9Sstevel@tonic-gate * Setting the Valid bit to zero for the specified IC_way and 21797c478bd9Sstevel@tonic-gate * IC_addr index into the I$ results in an invalidation of an I$ line. 21807c478bd9Sstevel@tonic-gate */ 21817c478bd9Sstevel@tonic-gate ENTRY(icache_inval_all) 21827c478bd9Sstevel@tonic-gate rdpr %pstate, %o5 21837c478bd9Sstevel@tonic-gate andn %o5, PSTATE_IE, %o3 21847c478bd9Sstevel@tonic-gate wrpr %g0, %o3, %pstate ! clear IE bit 21857c478bd9Sstevel@tonic-gate 21867c478bd9Sstevel@tonic-gate GET_CPU_PRIVATE_PTR(%g0, %o0, %o2, icache_inval_all_1); 21877c478bd9Sstevel@tonic-gate ld [%o0 + CHPR_ICACHE_LINESIZE], %o1 21887c478bd9Sstevel@tonic-gate ba,pt %icc, 2f 21897c478bd9Sstevel@tonic-gate ld [%o0 + CHPR_ICACHE_SIZE], %o0 21907c478bd9Sstevel@tonic-gateicache_inval_all_1: 21917c478bd9Sstevel@tonic-gate ASM_LD(%o0, icache_size) 21927c478bd9Sstevel@tonic-gate ASM_LD(%o1, icache_linesize) 21937c478bd9Sstevel@tonic-gate2: 21947c478bd9Sstevel@tonic-gate CH_ICACHE_FLUSHALL(%o0, %o1, %o2, %o4) 21957c478bd9Sstevel@tonic-gate 21967c478bd9Sstevel@tonic-gate retl 21977c478bd9Sstevel@tonic-gate wrpr %g0, %o5, %pstate ! restore earlier pstate 21987c478bd9Sstevel@tonic-gate SET_SIZE(icache_inval_all) 21997c478bd9Sstevel@tonic-gate 22007c478bd9Sstevel@tonic-gate 22017c478bd9Sstevel@tonic-gate/* 2202*55fea89dSDan Cross * cache_scrubreq_tl1 is the crosstrap handler called on offlined cpus via a 22037c478bd9Sstevel@tonic-gate * crosstrap. It atomically increments the outstanding request counter and, 22047c478bd9Sstevel@tonic-gate * if there was not already an outstanding request, branches to setsoftint_tl1 2205b0fc0e77Sgovinda * to enqueue an intr_vec for the given inum. 22067c478bd9Sstevel@tonic-gate */ 22077c478bd9Sstevel@tonic-gate 22087c478bd9Sstevel@tonic-gate ! Register usage: 22097c478bd9Sstevel@tonic-gate ! 22107c478bd9Sstevel@tonic-gate ! Arguments: 22117c478bd9Sstevel@tonic-gate ! %g1 - inum 22127c478bd9Sstevel@tonic-gate ! %g2 - index into chsm_outstanding array 22137c478bd9Sstevel@tonic-gate ! 22147c478bd9Sstevel@tonic-gate ! Internal: 22157c478bd9Sstevel@tonic-gate ! %g2, %g3, %g5 - scratch 22167c478bd9Sstevel@tonic-gate ! %g4 - ptr. to scrub_misc chsm_outstanding[index]. 22177c478bd9Sstevel@tonic-gate ! %g6 - setsoftint_tl1 address 22187c478bd9Sstevel@tonic-gate 22197c478bd9Sstevel@tonic-gate ENTRY_NP(cache_scrubreq_tl1) 22207c478bd9Sstevel@tonic-gate mulx %g2, CHSM_OUTSTANDING_INCR, %g2 22217c478bd9Sstevel@tonic-gate set CHPR_SCRUB_MISC + CHSM_OUTSTANDING, %g3 22227c478bd9Sstevel@tonic-gate add %g2, %g3, %g2 22237c478bd9Sstevel@tonic-gate GET_CPU_PRIVATE_PTR(%g2, %g4, %g5, 1f); 22247c478bd9Sstevel@tonic-gate ld [%g4], %g2 ! cpu's chsm_outstanding[index] 22257c478bd9Sstevel@tonic-gate ! 22267c478bd9Sstevel@tonic-gate ! no need to use atomic instructions for the following 22277c478bd9Sstevel@tonic-gate ! increment - we're at tl1 22287c478bd9Sstevel@tonic-gate ! 22297c478bd9Sstevel@tonic-gate add %g2, 0x1, %g3 2230b0fc0e77Sgovinda brnz,pn %g2, 1f ! no need to enqueue more intr_vec 22317c478bd9Sstevel@tonic-gate st %g3, [%g4] ! delay - store incremented counter 22327c478bd9Sstevel@tonic-gate ASM_JMP(%g6, setsoftint_tl1) 22337c478bd9Sstevel@tonic-gate ! not reached 22347c478bd9Sstevel@tonic-gate1: 22357c478bd9Sstevel@tonic-gate retry 22367c478bd9Sstevel@tonic-gate SET_SIZE(cache_scrubreq_tl1) 22377c478bd9Sstevel@tonic-gate 22387c478bd9Sstevel@tonic-gate 22397c478bd9Sstevel@tonic-gate/* 22407c478bd9Sstevel@tonic-gate * Get the error state for the processor. 22417c478bd9Sstevel@tonic-gate * Note that this must not be used at TL>0 22427c478bd9Sstevel@tonic-gate */ 22437c478bd9Sstevel@tonic-gate ENTRY(get_cpu_error_state) 22447c478bd9Sstevel@tonic-gate#if defined(CHEETAH_PLUS) 22457c478bd9Sstevel@tonic-gate set ASI_SHADOW_REG_VA, %o2 22467c478bd9Sstevel@tonic-gate ldxa [%o2]ASI_AFSR, %o1 ! shadow afsr reg 22477c478bd9Sstevel@tonic-gate stx %o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR] 22487c478bd9Sstevel@tonic-gate ldxa [%o2]ASI_AFAR, %o1 ! shadow afar reg 22497c478bd9Sstevel@tonic-gate stx %o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR] 22507c478bd9Sstevel@tonic-gate GET_CPU_IMPL(%o3) ! Only panther has AFSR_EXT registers 22517c478bd9Sstevel@tonic-gate cmp %o3, PANTHER_IMPL 22527c478bd9Sstevel@tonic-gate bne,a 1f 22537c478bd9Sstevel@tonic-gate stx %g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT] ! zero for non-PN 22547c478bd9Sstevel@tonic-gate set ASI_AFSR_EXT_VA, %o2 22557c478bd9Sstevel@tonic-gate ldxa [%o2]ASI_AFSR, %o1 ! afsr_ext reg 22567c478bd9Sstevel@tonic-gate stx %o1, [%o0 + CH_CPU_ERRORS_AFSR_EXT] 22577c478bd9Sstevel@tonic-gate set ASI_SHADOW_AFSR_EXT_VA, %o2 22587c478bd9Sstevel@tonic-gate ldxa [%o2]ASI_AFSR, %o1 ! shadow afsr_ext reg 22597c478bd9Sstevel@tonic-gate stx %o1, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT] 22607c478bd9Sstevel@tonic-gate b 2f 22617c478bd9Sstevel@tonic-gate nop 22627c478bd9Sstevel@tonic-gate1: 22637c478bd9Sstevel@tonic-gate stx %g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT] ! zero for non-PN 22647c478bd9Sstevel@tonic-gate2: 22657c478bd9Sstevel@tonic-gate#else /* CHEETAH_PLUS */ 22667c478bd9Sstevel@tonic-gate stx %g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR] 22677c478bd9Sstevel@tonic-gate stx %g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFAR] 22687c478bd9Sstevel@tonic-gate stx %g0, [%o0 + CH_CPU_ERRORS_AFSR_EXT] 22697c478bd9Sstevel@tonic-gate stx %g0, [%o0 + CH_CPU_ERRORS_SHADOW_AFSR_EXT] 22707c478bd9Sstevel@tonic-gate#endif /* CHEETAH_PLUS */ 22717c478bd9Sstevel@tonic-gate#if defined(SERRANO) 22727c478bd9Sstevel@tonic-gate /* 22737c478bd9Sstevel@tonic-gate * Serrano has an afar2 which captures the address on FRC/FRU errors. 22747c478bd9Sstevel@tonic-gate * We save this in the afar2 of the register save area. 22757c478bd9Sstevel@tonic-gate */ 22767c478bd9Sstevel@tonic-gate set ASI_MCU_AFAR2_VA, %o2 22777c478bd9Sstevel@tonic-gate ldxa [%o2]ASI_MCU_CTRL, %o1 22787c478bd9Sstevel@tonic-gate stx %o1, [%o0 + CH_CPU_ERRORS_AFAR2] 22797c478bd9Sstevel@tonic-gate#endif /* SERRANO */ 22807c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_AFSR, %o1 ! primary afsr reg 22817c478bd9Sstevel@tonic-gate stx %o1, [%o0 + CH_CPU_ERRORS_AFSR] 22827c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_AFAR, %o1 ! primary afar reg 22837c478bd9Sstevel@tonic-gate retl 22847c478bd9Sstevel@tonic-gate stx %o1, [%o0 + CH_CPU_ERRORS_AFAR] 22857c478bd9Sstevel@tonic-gate SET_SIZE(get_cpu_error_state) 22867c478bd9Sstevel@tonic-gate 22877c478bd9Sstevel@tonic-gate/* 22887c478bd9Sstevel@tonic-gate * Check a page of memory for errors. 22897c478bd9Sstevel@tonic-gate * 22907c478bd9Sstevel@tonic-gate * Load each 64 byte block from physical memory. 22917c478bd9Sstevel@tonic-gate * Check AFSR after each load to see if an error 22927c478bd9Sstevel@tonic-gate * was caused. If so, log/scrub that error. 22937c478bd9Sstevel@tonic-gate * 22947c478bd9Sstevel@tonic-gate * Used to determine if a page contains 22957c478bd9Sstevel@tonic-gate * CEs when CEEN is disabled. 22967c478bd9Sstevel@tonic-gate */ 22977c478bd9Sstevel@tonic-gate ENTRY(cpu_check_block) 22987c478bd9Sstevel@tonic-gate ! 22997c478bd9Sstevel@tonic-gate ! get a new window with room for the error regs 23007c478bd9Sstevel@tonic-gate ! 23017c478bd9Sstevel@tonic-gate save %sp, -SA(MINFRAME + CH_CPU_ERROR_SIZE), %sp 23027c478bd9Sstevel@tonic-gate srl %i1, 6, %l4 ! clear top bits of psz 23037c478bd9Sstevel@tonic-gate ! and divide by 64 23047c478bd9Sstevel@tonic-gate rd %fprs, %l2 ! store FP 23057c478bd9Sstevel@tonic-gate wr %g0, FPRS_FEF, %fprs ! enable FP 23067c478bd9Sstevel@tonic-gate1: 23077c478bd9Sstevel@tonic-gate ldda [%i0]ASI_BLK_P, %d0 ! load a block 23087c478bd9Sstevel@tonic-gate membar #Sync 23097c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_AFSR, %l3 ! read afsr reg 23107c478bd9Sstevel@tonic-gate brz,a,pt %l3, 2f ! check for error 23117c478bd9Sstevel@tonic-gate nop 23127c478bd9Sstevel@tonic-gate 23137c478bd9Sstevel@tonic-gate ! 23147c478bd9Sstevel@tonic-gate ! if error, read the error regs and log it 23157c478bd9Sstevel@tonic-gate ! 23167c478bd9Sstevel@tonic-gate call get_cpu_error_state 23177c478bd9Sstevel@tonic-gate add %fp, STACK_BIAS - CH_CPU_ERROR_SIZE, %o0 23187c478bd9Sstevel@tonic-gate 23197c478bd9Sstevel@tonic-gate ! 23207c478bd9Sstevel@tonic-gate ! cpu_ce_detected(ch_cpu_errors_t *, flag) 23217c478bd9Sstevel@tonic-gate ! 23227c478bd9Sstevel@tonic-gate call cpu_ce_detected ! log the error 23237c478bd9Sstevel@tonic-gate mov CE_CEEN_TIMEOUT, %o1 23247c478bd9Sstevel@tonic-gate2: 23257c478bd9Sstevel@tonic-gate dec %l4 ! next 64-byte block 23267c478bd9Sstevel@tonic-gate brnz,a,pt %l4, 1b 23277c478bd9Sstevel@tonic-gate add %i0, 64, %i0 ! increment block addr 23287c478bd9Sstevel@tonic-gate 23297c478bd9Sstevel@tonic-gate wr %l2, %g0, %fprs ! restore FP 23307c478bd9Sstevel@tonic-gate ret 23317c478bd9Sstevel@tonic-gate restore 23327c478bd9Sstevel@tonic-gate 23337c478bd9Sstevel@tonic-gate SET_SIZE(cpu_check_block) 23347c478bd9Sstevel@tonic-gate 23357c478bd9Sstevel@tonic-gate/* 23367c478bd9Sstevel@tonic-gate * Perform a cpu logout called from C. This is used where we did not trap 23377c478bd9Sstevel@tonic-gate * for the error but still want to gather "what we can". Caller must make 23387c478bd9Sstevel@tonic-gate * sure cpu private area exists and that the indicated logout area is free 23397c478bd9Sstevel@tonic-gate * for use, and that we are unable to migrate cpus. 23407c478bd9Sstevel@tonic-gate */ 23417c478bd9Sstevel@tonic-gate ENTRY(cpu_delayed_logout) 23427c478bd9Sstevel@tonic-gate rdpr %pstate, %o2 23437c478bd9Sstevel@tonic-gate andn %o2, PSTATE_IE, %o2 23447c478bd9Sstevel@tonic-gate wrpr %g0, %o2, %pstate ! disable interrupts 23457c478bd9Sstevel@tonic-gate PARK_SIBLING_CORE(%o2, %o3, %o4) ! %o2 has DCU value 23467c478bd9Sstevel@tonic-gate add %o1, CH_CLO_DATA + CH_CHD_EC_DATA, %o1 23477c478bd9Sstevel@tonic-gate rd %asi, %g1 23487c478bd9Sstevel@tonic-gate wr %g0, ASI_P, %asi 23497c478bd9Sstevel@tonic-gate GET_ECACHE_DTAGS(%o0, %o1, %o3, %o4, %o5) 23507c478bd9Sstevel@tonic-gate wr %g1, %asi 23517c478bd9Sstevel@tonic-gate UNPARK_SIBLING_CORE(%o2, %o3, %o4) ! can use %o2 again 23527c478bd9Sstevel@tonic-gate rdpr %pstate, %o2 23537c478bd9Sstevel@tonic-gate or %o2, PSTATE_IE, %o2 23547c478bd9Sstevel@tonic-gate wrpr %g0, %o2, %pstate 23557c478bd9Sstevel@tonic-gate retl 23567c478bd9Sstevel@tonic-gate nop 23577c478bd9Sstevel@tonic-gate SET_SIZE(cpu_delayed_logout) 23587c478bd9Sstevel@tonic-gate 23597c478bd9Sstevel@tonic-gate ENTRY(dtrace_blksuword32) 23607c478bd9Sstevel@tonic-gate save %sp, -SA(MINFRAME + 4), %sp 23617c478bd9Sstevel@tonic-gate 23627c478bd9Sstevel@tonic-gate rdpr %pstate, %l1 23637c478bd9Sstevel@tonic-gate andn %l1, PSTATE_IE, %l2 ! disable interrupts to 23647c478bd9Sstevel@tonic-gate wrpr %g0, %l2, %pstate ! protect our FPU diddling 23657c478bd9Sstevel@tonic-gate 23667c478bd9Sstevel@tonic-gate rd %fprs, %l0 23677c478bd9Sstevel@tonic-gate andcc %l0, FPRS_FEF, %g0 23687c478bd9Sstevel@tonic-gate bz,a,pt %xcc, 1f ! if the fpu is disabled 23697c478bd9Sstevel@tonic-gate wr %g0, FPRS_FEF, %fprs ! ... enable the fpu 23707c478bd9Sstevel@tonic-gate 23717c478bd9Sstevel@tonic-gate st %f0, [%fp + STACK_BIAS - 4] ! save %f0 to the stack 23727c478bd9Sstevel@tonic-gate1: 23737c478bd9Sstevel@tonic-gate set 0f, %l5 23747c478bd9Sstevel@tonic-gate /* 23757c478bd9Sstevel@tonic-gate * We're about to write a block full or either total garbage 23767c478bd9Sstevel@tonic-gate * (not kernel data, don't worry) or user floating-point data 23777c478bd9Sstevel@tonic-gate * (so it only _looks_ like garbage). 23787c478bd9Sstevel@tonic-gate */ 23797c478bd9Sstevel@tonic-gate ld [%i1], %f0 ! modify the block 23807c478bd9Sstevel@tonic-gate membar #Sync 23817c478bd9Sstevel@tonic-gate stn %l5, [THREAD_REG + T_LOFAULT] ! set up the lofault handler 23827c478bd9Sstevel@tonic-gate stda %d0, [%i0]ASI_BLK_COMMIT_S ! store the modified block 23837c478bd9Sstevel@tonic-gate membar #Sync 23847c478bd9Sstevel@tonic-gate stn %g0, [THREAD_REG + T_LOFAULT] ! remove the lofault handler 23857c478bd9Sstevel@tonic-gate 23867c478bd9Sstevel@tonic-gate bz,a,pt %xcc, 1f 23877c478bd9Sstevel@tonic-gate wr %g0, %l0, %fprs ! restore %fprs 23887c478bd9Sstevel@tonic-gate 23897c478bd9Sstevel@tonic-gate ld [%fp + STACK_BIAS - 4], %f0 ! restore %f0 23907c478bd9Sstevel@tonic-gate1: 23917c478bd9Sstevel@tonic-gate 23927c478bd9Sstevel@tonic-gate wrpr %g0, %l1, %pstate ! restore interrupts 23937c478bd9Sstevel@tonic-gate 23947c478bd9Sstevel@tonic-gate ret 23957c478bd9Sstevel@tonic-gate restore %g0, %g0, %o0 23967c478bd9Sstevel@tonic-gate 23977c478bd9Sstevel@tonic-gate0: 23987c478bd9Sstevel@tonic-gate membar #Sync 23997c478bd9Sstevel@tonic-gate stn %g0, [THREAD_REG + T_LOFAULT] ! remove the lofault handler 24007c478bd9Sstevel@tonic-gate 24017c478bd9Sstevel@tonic-gate bz,a,pt %xcc, 1f 24027c478bd9Sstevel@tonic-gate wr %g0, %l0, %fprs ! restore %fprs 24037c478bd9Sstevel@tonic-gate 24047c478bd9Sstevel@tonic-gate ld [%fp + STACK_BIAS - 4], %f0 ! restore %f0 24057c478bd9Sstevel@tonic-gate1: 24067c478bd9Sstevel@tonic-gate 24077c478bd9Sstevel@tonic-gate wrpr %g0, %l1, %pstate ! restore interrupts 24087c478bd9Sstevel@tonic-gate 24097c478bd9Sstevel@tonic-gate /* 24107c478bd9Sstevel@tonic-gate * If tryagain is set (%i2) we tail-call dtrace_blksuword32_err() 24117c478bd9Sstevel@tonic-gate * which deals with watchpoints. Otherwise, just return -1. 24127c478bd9Sstevel@tonic-gate */ 24137c478bd9Sstevel@tonic-gate brnz,pt %i2, 1f 24147c478bd9Sstevel@tonic-gate nop 24157c478bd9Sstevel@tonic-gate ret 24167c478bd9Sstevel@tonic-gate restore %g0, -1, %o0 24177c478bd9Sstevel@tonic-gate1: 24187c478bd9Sstevel@tonic-gate call dtrace_blksuword32_err 24197c478bd9Sstevel@tonic-gate restore 24207c478bd9Sstevel@tonic-gate 24217c478bd9Sstevel@tonic-gate SET_SIZE(dtrace_blksuword32) 24227c478bd9Sstevel@tonic-gate 24237c478bd9Sstevel@tonic-gate#ifdef CHEETAHPLUS_ERRATUM_25 24247c478bd9Sstevel@tonic-gate 24259b0bb795SJohn Levon /* Claim a chunk of physical address space. */ 24267c478bd9Sstevel@tonic-gate ENTRY(claimlines) 24277c478bd9Sstevel@tonic-gate1: 24287c478bd9Sstevel@tonic-gate subcc %o1, %o2, %o1 24297c478bd9Sstevel@tonic-gate add %o0, %o1, %o3 24307c478bd9Sstevel@tonic-gate bgeu,a,pt %xcc, 1b 24317c478bd9Sstevel@tonic-gate casxa [%o3]ASI_MEM, %g0, %g0 24327c478bd9Sstevel@tonic-gate membar #Sync 24337c478bd9Sstevel@tonic-gate retl 24347c478bd9Sstevel@tonic-gate nop 24357c478bd9Sstevel@tonic-gate SET_SIZE(claimlines) 24367c478bd9Sstevel@tonic-gate 24379b0bb795SJohn Levon /* 24389b0bb795SJohn Levon * CPU feature initialization, 24399b0bb795SJohn Levon * turn BPE off, 24409b0bb795SJohn Levon * get device id. 24419b0bb795SJohn Levon */ 24427c478bd9Sstevel@tonic-gate ENTRY(cpu_feature_init) 24437c478bd9Sstevel@tonic-gate save %sp, -SA(MINFRAME), %sp 24447c478bd9Sstevel@tonic-gate sethi %hi(cheetah_bpe_off), %o0 24457c478bd9Sstevel@tonic-gate ld [%o0 + %lo(cheetah_bpe_off)], %o0 24467c478bd9Sstevel@tonic-gate brz %o0, 1f 24477c478bd9Sstevel@tonic-gate nop 24487c478bd9Sstevel@tonic-gate rd ASR_DISPATCH_CONTROL, %o0 24497c478bd9Sstevel@tonic-gate andn %o0, ASR_DISPATCH_CONTROL_BPE, %o0 24507c478bd9Sstevel@tonic-gate wr %o0, 0, ASR_DISPATCH_CONTROL 24517c478bd9Sstevel@tonic-gate1: 24527c478bd9Sstevel@tonic-gate ! 24537c478bd9Sstevel@tonic-gate ! get the device_id and store the device_id 24547c478bd9Sstevel@tonic-gate ! in the appropriate cpunodes structure 24557c478bd9Sstevel@tonic-gate ! given the cpus index 24567c478bd9Sstevel@tonic-gate ! 24577c478bd9Sstevel@tonic-gate CPU_INDEX(%o0, %o1) 24587c478bd9Sstevel@tonic-gate mulx %o0, CPU_NODE_SIZE, %o0 24597c478bd9Sstevel@tonic-gate set cpunodes + DEVICE_ID, %o1 24607c478bd9Sstevel@tonic-gate ldxa [%g0] ASI_DEVICE_SERIAL_ID, %o2 24617c478bd9Sstevel@tonic-gate stx %o2, [%o0 + %o1] 24627c478bd9Sstevel@tonic-gate#ifdef CHEETAHPLUS_ERRATUM_34 24637c478bd9Sstevel@tonic-gate ! 24647c478bd9Sstevel@tonic-gate ! apply Cheetah+ erratum 34 workaround 24657c478bd9Sstevel@tonic-gate ! 24667c478bd9Sstevel@tonic-gate call itlb_erratum34_fixup 24677c478bd9Sstevel@tonic-gate nop 246819f938d5Sjfrank call dtlb_erratum34_fixup 246919f938d5Sjfrank nop 24707c478bd9Sstevel@tonic-gate#endif /* CHEETAHPLUS_ERRATUM_34 */ 24717c478bd9Sstevel@tonic-gate ret 24727c478bd9Sstevel@tonic-gate restore 24737c478bd9Sstevel@tonic-gate SET_SIZE(cpu_feature_init) 24747c478bd9Sstevel@tonic-gate 24757c478bd9Sstevel@tonic-gate/* 24767c478bd9Sstevel@tonic-gate * Copy a tsb entry atomically, from src to dest. 24777c478bd9Sstevel@tonic-gate * src must be 128 bit aligned. 24787c478bd9Sstevel@tonic-gate */ 24797c478bd9Sstevel@tonic-gate ENTRY(copy_tsb_entry) 24807c478bd9Sstevel@tonic-gate ldda [%o0]ASI_NQUAD_LD, %o2 ! %o2 = tag, %o3 = data 24817c478bd9Sstevel@tonic-gate stx %o2, [%o1] 2482*55fea89dSDan Cross stx %o3, [%o1 + 8 ] 24837c478bd9Sstevel@tonic-gate retl 24847c478bd9Sstevel@tonic-gate nop 24857c478bd9Sstevel@tonic-gate SET_SIZE(copy_tsb_entry) 24867c478bd9Sstevel@tonic-gate 24877c478bd9Sstevel@tonic-gate#endif /* CHEETAHPLUS_ERRATUM_25 */ 24887c478bd9Sstevel@tonic-gate 24897c478bd9Sstevel@tonic-gate#ifdef CHEETAHPLUS_ERRATUM_34 24907c478bd9Sstevel@tonic-gate 24917c478bd9Sstevel@tonic-gate ! 24927c478bd9Sstevel@tonic-gate ! In Cheetah+ erratum 34, under certain conditions an ITLB locked 24937c478bd9Sstevel@tonic-gate ! index 0 TTE will erroneously be displaced when a new TTE is 24947c478bd9Sstevel@tonic-gate ! loaded via ASI_ITLB_IN. In order to avoid cheetah+ erratum 34, 24957c478bd9Sstevel@tonic-gate ! locked index 0 TTEs must be relocated. 24967c478bd9Sstevel@tonic-gate ! 24977c478bd9Sstevel@tonic-gate ! NOTE: Care must be taken to avoid an ITLB miss in this routine. 24987c478bd9Sstevel@tonic-gate ! 24997c478bd9Sstevel@tonic-gate ENTRY_NP(itlb_erratum34_fixup) 25007c478bd9Sstevel@tonic-gate rdpr %pstate, %o3 25017c478bd9Sstevel@tonic-gate#ifdef DEBUG 25021e2e7a75Shuah PANIC_IF_INTR_DISABLED_PSTR(%o3, u3_di_label1, %g1) 25037c478bd9Sstevel@tonic-gate#endif /* DEBUG */ 25047c478bd9Sstevel@tonic-gate wrpr %o3, PSTATE_IE, %pstate ! Disable interrupts 25057c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_ITLB_ACCESS, %o1 ! %o1 = entry 0 data 25067c478bd9Sstevel@tonic-gate ldxa [%g0]ASI_ITLB_TAGREAD, %o2 ! %o2 = entry 0 tag 25077c478bd9Sstevel@tonic-gate 25087c478bd9Sstevel@tonic-gate cmp %o1, %g0 ! Is this entry valid? 25097c478bd9Sstevel@tonic-gate bge %xcc, 1f 25107c478bd9Sstevel@tonic-gate andcc %o1, TTE_LCK_INT, %g0 ! Is this entry locked? 25117c478bd9Sstevel@tonic-gate bnz %icc, 2f 25127c478bd9Sstevel@tonic-gate nop 25137c478bd9Sstevel@tonic-gate1: 25147c478bd9Sstevel@tonic-gate retl ! Nope, outta here... 25157c478bd9Sstevel@tonic-gate wrpr %g0, %o3, %pstate ! Enable interrupts 25167c478bd9Sstevel@tonic-gate2: 25177c478bd9Sstevel@tonic-gate sethi %hi(FLUSH_ADDR), %o4 25187c478bd9Sstevel@tonic-gate stxa %g0, [%o2]ASI_ITLB_DEMAP ! Flush this mapping 25197c478bd9Sstevel@tonic-gate flush %o4 ! Flush required for I-MMU 25207c478bd9Sstevel@tonic-gate ! 25217c478bd9Sstevel@tonic-gate ! Start search from index 1 up. This is because the kernel force 25227c478bd9Sstevel@tonic-gate ! loads its text page at index 15 in sfmmu_kernel_remap() and we 25237c478bd9Sstevel@tonic-gate ! don't want our relocated entry evicted later. 25247c478bd9Sstevel@tonic-gate ! 25257c478bd9Sstevel@tonic-gate ! NOTE: We assume that we'll be successful in finding an unlocked 25267c478bd9Sstevel@tonic-gate ! or invalid entry. If that isn't the case there are bound to 25277c478bd9Sstevel@tonic-gate ! bigger problems. 25287c478bd9Sstevel@tonic-gate ! 25297c478bd9Sstevel@tonic-gate set (1 << 3), %g3 25307c478bd9Sstevel@tonic-gate3: 25317c478bd9Sstevel@tonic-gate ldxa [%g3]ASI_ITLB_ACCESS, %o4 ! Load TTE from t16 25327c478bd9Sstevel@tonic-gate ! 25337c478bd9Sstevel@tonic-gate ! If this entry isn't valid, we'll choose to displace it (regardless 25347c478bd9Sstevel@tonic-gate ! of the lock bit). 25357c478bd9Sstevel@tonic-gate ! 25367c478bd9Sstevel@tonic-gate cmp %o4, %g0 ! TTE is > 0 iff not valid 25377c478bd9Sstevel@tonic-gate bge %xcc, 4f ! If invalid, go displace 25387c478bd9Sstevel@tonic-gate andcc %o4, TTE_LCK_INT, %g0 ! Check for lock bit 25397c478bd9Sstevel@tonic-gate bnz,a %icc, 3b ! If locked, look at next 25407c478bd9Sstevel@tonic-gate add %g3, (1 << 3), %g3 ! entry 25417c478bd9Sstevel@tonic-gate4: 25427c478bd9Sstevel@tonic-gate ! 25437c478bd9Sstevel@tonic-gate ! We found an unlocked or invalid entry; we'll explicitly load 25447c478bd9Sstevel@tonic-gate ! the former index 0 entry here. 25457c478bd9Sstevel@tonic-gate ! 25467c478bd9Sstevel@tonic-gate sethi %hi(FLUSH_ADDR), %o4 25477c478bd9Sstevel@tonic-gate set MMU_TAG_ACCESS, %g4 25487c478bd9Sstevel@tonic-gate stxa %o2, [%g4]ASI_IMMU 25497c478bd9Sstevel@tonic-gate stxa %o1, [%g3]ASI_ITLB_ACCESS 25507c478bd9Sstevel@tonic-gate flush %o4 ! Flush required for I-MMU 25517c478bd9Sstevel@tonic-gate retl 25527c478bd9Sstevel@tonic-gate wrpr %g0, %o3, %pstate ! Enable interrupts 25537c478bd9Sstevel@tonic-gate SET_SIZE(itlb_erratum34_fixup) 25547c478bd9Sstevel@tonic-gate 255519f938d5Sjfrank ! 255619f938d5Sjfrank ! In Cheetah+ erratum 34, under certain conditions a DTLB locked 255719f938d5Sjfrank ! index 0 TTE will erroneously be displaced when a new TTE is 255819f938d5Sjfrank ! loaded. In order to avoid cheetah+ erratum 34, locked index 0 255919f938d5Sjfrank ! TTEs must be relocated. 256019f938d5Sjfrank ! 256119f938d5Sjfrank ENTRY_NP(dtlb_erratum34_fixup) 256219f938d5Sjfrank rdpr %pstate, %o3 256319f938d5Sjfrank#ifdef DEBUG 256419f938d5Sjfrank PANIC_IF_INTR_DISABLED_PSTR(%o3, u3_di_label2, %g1) 256519f938d5Sjfrank#endif /* DEBUG */ 256619f938d5Sjfrank wrpr %o3, PSTATE_IE, %pstate ! Disable interrupts 256719f938d5Sjfrank ldxa [%g0]ASI_DTLB_ACCESS, %o1 ! %o1 = entry 0 data 256819f938d5Sjfrank ldxa [%g0]ASI_DTLB_TAGREAD, %o2 ! %o2 = entry 0 tag 256919f938d5Sjfrank 257019f938d5Sjfrank cmp %o1, %g0 ! Is this entry valid? 257119f938d5Sjfrank bge %xcc, 1f 257219f938d5Sjfrank andcc %o1, TTE_LCK_INT, %g0 ! Is this entry locked? 257319f938d5Sjfrank bnz %icc, 2f 257419f938d5Sjfrank nop 257519f938d5Sjfrank1: 257619f938d5Sjfrank retl ! Nope, outta here... 257719f938d5Sjfrank wrpr %g0, %o3, %pstate ! Enable interrupts 257819f938d5Sjfrank2: 257919f938d5Sjfrank stxa %g0, [%o2]ASI_DTLB_DEMAP ! Flush this mapping 258019f938d5Sjfrank membar #Sync 258119f938d5Sjfrank ! 258219f938d5Sjfrank ! Start search from index 1 up. 258319f938d5Sjfrank ! 258419f938d5Sjfrank ! NOTE: We assume that we'll be successful in finding an unlocked 258519f938d5Sjfrank ! or invalid entry. If that isn't the case there are bound to 258619f938d5Sjfrank ! bigger problems. 258719f938d5Sjfrank ! 258819f938d5Sjfrank set (1 << 3), %g3 258919f938d5Sjfrank3: 259019f938d5Sjfrank ldxa [%g3]ASI_DTLB_ACCESS, %o4 ! Load TTE from t16 259119f938d5Sjfrank ! 259219f938d5Sjfrank ! If this entry isn't valid, we'll choose to displace it (regardless 259319f938d5Sjfrank ! of the lock bit). 259419f938d5Sjfrank ! 259519f938d5Sjfrank cmp %o4, %g0 ! TTE is > 0 iff not valid 259619f938d5Sjfrank bge %xcc, 4f ! If invalid, go displace 259719f938d5Sjfrank andcc %o4, TTE_LCK_INT, %g0 ! Check for lock bit 259819f938d5Sjfrank bnz,a %icc, 3b ! If locked, look at next 259919f938d5Sjfrank add %g3, (1 << 3), %g3 ! entry 260019f938d5Sjfrank4: 260119f938d5Sjfrank ! 260219f938d5Sjfrank ! We found an unlocked or invalid entry; we'll explicitly load 260319f938d5Sjfrank ! the former index 0 entry here. 260419f938d5Sjfrank ! 260519f938d5Sjfrank set MMU_TAG_ACCESS, %g4 260619f938d5Sjfrank stxa %o2, [%g4]ASI_DMMU 260719f938d5Sjfrank stxa %o1, [%g3]ASI_DTLB_ACCESS 260819f938d5Sjfrank membar #Sync 260919f938d5Sjfrank retl 261019f938d5Sjfrank wrpr %g0, %o3, %pstate ! Enable interrupts 261119f938d5Sjfrank SET_SIZE(dtlb_erratum34_fixup) 261219f938d5Sjfrank 26137c478bd9Sstevel@tonic-gate#endif /* CHEETAHPLUS_ERRATUM_34 */ 26147c478bd9Sstevel@tonic-gate 2615