/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include uint_t root_phys_addr_lo_mask = 0xffffffffU; #if defined(NIAGARA2_IMPL) char cpu_module_name[] = "SUNW,UltraSPARC-T2"; #elif defined(VFALLS_IMPL) char cpu_module_name[] = "SUNW,UltraSPARC-T2+"; #elif defined(KT_IMPL) char cpu_module_name[] = "SPARC-T3"; #endif /* * Hypervisor services information for the NIAGARA2 and Victoria Falls * CPU module */ static boolean_t cpu_hsvc_available = B_TRUE; static uint64_t cpu_sup_minor; /* Supported minor number */ #if defined(NIAGARA2_IMPL) static hsvc_info_t cpu_hsvc = { HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR, NIAGARA2_HSVC_MINOR, cpu_module_name }; #elif defined(VFALLS_IMPL) static hsvc_info_t cpu_hsvc = { HSVC_REV_1, NULL, HSVC_GROUP_VFALLS_CPU, VFALLS_HSVC_MAJOR, VFALLS_HSVC_MINOR, cpu_module_name }; #elif defined(KT_IMPL) static hsvc_info_t cpu_hsvc = { HSVC_REV_1, NULL, HSVC_GROUP_KT_CPU, KT_HSVC_MAJOR, KT_HSVC_MINOR, cpu_module_name }; #endif void cpu_setup(void) { extern int mmu_exported_pagesize_mask; extern int cpc_has_overflow_intr; extern size_t contig_mem_prealloc_base_size; int status; /* * Negotiate the API version for Niagara2 specific hypervisor * services. */ status = hsvc_register(&cpu_hsvc, &cpu_sup_minor); if (status != 0) { cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services " "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d", cpu_hsvc.hsvc_modname, cpu_hsvc.hsvc_group, cpu_hsvc.hsvc_major, cpu_hsvc.hsvc_minor, status); cpu_hsvc_available = B_FALSE; } /* * The setup common to all CPU modules is done in cpu_setup_common * routine. */ cpu_setup_common(NULL); /* * Initialize the cpu_hwcap_flags for N2 and VF if it is not already * set in cpu_setup_common() by the hwcap MD info. Note that this MD * info may not be available for N2/VF. */ if (cpu_hwcap_flags == 0) { #ifdef KT_IMPL /* * This should not happen since hwcap MD info is always * available for KT platforms. */ ASSERT(cpu_hwcap_flags != 0); /* panic in DEBUG mode */ cpu_hwcap_flags |= AV_SPARC_VIS3 | AV_SPARC_HPC | AV_SPARC_FMAF; #endif /* KT_IMPL */ cpu_hwcap_flags |= AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT | AV_SPARC_POPC; } cache |= (CACHE_PTAG | CACHE_IOCOHERENT); if ((mmu_exported_pagesize_mask & DEFAULT_SUN4V_MMU_PAGESIZE_MASK) != DEFAULT_SUN4V_MMU_PAGESIZE_MASK) cmn_err(CE_PANIC, "machine description" " does not have required sun4v page sizes" " 8K, 64K and 4M: MD mask is 0x%x", mmu_exported_pagesize_mask); /* * Niagara2 supports a 48-bit subset of the full 64-bit virtual * address space. Virtual addresses between 0x0000800000000000 * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole" * and must never be mapped. In addition, software must not use * pages within 4GB of the VA hole as instruction pages to * avoid problems with prefetching into the VA hole. */ hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32)); hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32)); /* * Niagara2 has a performance counter overflow interrupt */ cpc_has_overflow_intr = 1; /* * Enable 4M pages for OOB. */ max_uheap_lpsize = MMU_PAGESIZE4M; max_ustack_lpsize = MMU_PAGESIZE4M; max_privmap_lpsize = MMU_PAGESIZE4M; #ifdef SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB /* * Use CPU Makefile specific compile time define (if exists) * to add to the contig preallocation size. */ contig_mem_prealloc_base_size = MB(SUN4V_CONTIG_MEM_PREALLOC_SIZE_MB); #endif } /* * Set the magic constants of the implementation. */ void cpu_fiximp(struct cpu_node *cpunode) { /* * The Cache node is optional in MD. Therefore in case "Cache" * node does not exists in MD, set the default L2 cache associativity, * size, linesize. */ if (cpunode->ecache_size == 0) cpunode->ecache_size = L2CACHE_SIZE; if (cpunode->ecache_linesize == 0) cpunode->ecache_linesize = L2CACHE_LINESIZE; if (cpunode->ecache_associativity == 0) cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY; } void cpu_map_exec_units(struct cpu *cp) { ASSERT(MUTEX_HELD(&cpu_lock)); /* * The cpu_ipipe and cpu_fpu fields are initialized based on * the execution unit sharing information from the MD. They * default to the CPU id in the absence of such information. */ cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping; if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND) cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id); cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping; if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND) cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id); /* * Niagara 2 defines the core to be at the FPU level */ cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu; /* * The cpu_chip field is initialized based on the information * in the MD and assume that all cpus within a chip * share the same L2 cache. If no such info is available, we * set the cpu to belong to the defacto chip 0. */ cp->cpu_m.cpu_mpipe = cpunodes[cp->cpu_id].l2_cache_mapping; if (cp->cpu_m.cpu_mpipe == NO_L2_CACHE_MAPPING_FOUND) cp->cpu_m.cpu_mpipe = CPU_L2_CACHEID_INVALID; cp->cpu_m.cpu_chip = cpunodes[cp->cpu_id].l2_cache_mapping; if (cp->cpu_m.cpu_chip == NO_L2_CACHE_MAPPING_FOUND) cp->cpu_m.cpu_chip = CPU_CHIPID_INVALID; } static int cpucnt; void cpu_init_private(struct cpu *cp) { extern void niagara_kstat_init(void); ASSERT(MUTEX_HELD(&cpu_lock)); cpu_map_exec_units(cp); if ((cpucnt++ == 0) && (cpu_hsvc_available == B_TRUE)) (void) niagara_kstat_init(); mutex_delay = rdccr_delay; } /*ARGSUSED*/ void cpu_uninit_private(struct cpu *cp) { extern void niagara_kstat_fini(void); ASSERT(MUTEX_HELD(&cpu_lock)); if ((--cpucnt == 0) && (cpu_hsvc_available == B_TRUE)) (void) niagara_kstat_fini(); } /* * On Niagara2, any flush will cause all preceding stores to be * synchronized wrt the i$, regardless of address or ASI. In fact, * the address is ignored, so we always flush address 0. */ /*ARGSUSED*/ void dtrace_flush_sec(uintptr_t addr) { doflush(0); } /* * Trapstat support for Niagara2 processor * The Niagara2 provides HWTW support for TSB lookup and with HWTW * enabled no TSB hit information will be available. Therefore setting * the time spent in TLB miss handler for TSB hits to 0. */ int cpu_trapstat_conf(int cmd) { int status = 0; switch (cmd) { case CPU_TSTATCONF_INIT: case CPU_TSTATCONF_FINI: case CPU_TSTATCONF_ENABLE: case CPU_TSTATCONF_DISABLE: break; default: status = EINVAL; break; } return (status); } void cpu_trapstat_data(void *buf, uint_t tstat_pgszs) { tstat_pgszdata_t *tstatp = (tstat_pgszdata_t *)buf; int i; for (i = 0; i < tstat_pgszs; i++, tstatp++) { tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0; tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0; tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0; tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0; tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0; tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0; tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0; tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0; } } /* * Page coloring support for hashed cache index mode */ /* * Node id bits from machine description (MD). Node id distinguishes * local versus remote memory. Because of MPO, page allocation does * not cross node boundaries. Therefore, remove the node id bits from * the color, since they are fixed. Either bit 30, or 31:30 in * Victoria Falls processors. * The number of node id bits is always 0 in Niagara2. */ typedef struct n2color { uchar_t nnbits; /* number of node id bits */ uchar_t nnmask; /* mask for node id bits */ uchar_t lomask; /* mask for bits below node id */ uchar_t lobits; /* number of bits below node id */ } n2color_t; n2color_t n2color[MMU_PAGE_SIZES]; static uchar_t nhbits[] = {7, 7, 6, 5, 5, 5}; /* * Remove node id bits from color bits 32:28. * This will reduce the number of colors. * No change if number of node bits is zero. */ static inline uint_t n2_hash2color(uint_t color, uchar_t szc) { n2color_t m = n2color[szc]; if (m.nnbits > 0) { color = ((color >> m.nnbits) & ~m.lomask) | (color & m.lomask); ASSERT((color & ~(hw_page_array[szc].hp_colors - 1)) == 0); } return (color); } /* * Restore node id bits into page color. * This will increase the number of colors to match N2. * No change if number of node bits is zero. */ static inline uint_t n2_color2hash(uint_t color, uchar_t szc, uint_t node) { n2color_t m = n2color[szc]; if (m.nnbits > 0) { color = ((color & ~m.lomask) << m.nnbits) | (color & m.lomask); color |= (node & m.nnmask) << m.lobits; } return (color); } /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */ /* * iterator NULL means pfn is VA, do not adjust ra_to_pa * iterator (-1) means pfn is RA, need to convert to PA * iterator non-null means pfn is RA, use ra_to_pa */ uint_t page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc, void *cookie) { mem_node_iterator_t *it = cookie; uint_t color; ASSERT(szc <= TTE256M); if (it == ((mem_node_iterator_t *)(-1))) { pfn = plat_rapfn_to_papfn(pfn); } else if (it != NULL) { ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); pfn = pfn + it->mi_ra_to_pa; } pfn = PFN_BASE(pfn, szc); color = ((pfn >> 15) ^ pfn) & 0x1f; if (szc < TTE4M) { /* 19:18 */ color = (color << 2) | ((pfn >> 5) & 0x3); if (szc > TTE64K) color >>= 1; /* 19 */ } return (n2_hash2color(color, szc)); } static uint_t page_papfn_2_color_cpu(pfn_t papfn, uchar_t szc) { uint_t color; ASSERT(szc <= TTE256M); papfn = PFN_BASE(papfn, szc); color = ((papfn >> 15) ^ papfn) & 0x1f; if (szc < TTE4M) { /* 19:18 */ color = (color << 2) | ((papfn >> 5) & 0x3); if (szc > TTE64K) color >>= 1; /* 19 */ } return (color); } #if TTE256M != 5 #error TTE256M is not 5 #endif uint_t page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask) { static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f}; ASSERT(szc < TTE256M); mask = n2_color2hash(mask, szc, 0); mask &= ni2_color_masks[szc]; if (szc == TTE64K || szc == TTE512K) mask >>= 1; return (n2_hash2color(mask, szc + 1)); } uint_t page_get_nsz_color_cpu(uchar_t szc, uint_t color) { ASSERT(szc < TTE256M); color = n2_color2hash(color, szc, 0); if (szc == TTE64K || szc == TTE512K) color >>= 1; return (n2_hash2color(color, szc + 1)); } uint_t page_get_color_shift_cpu(uchar_t szc, uchar_t nszc) { uint_t s; ASSERT(nszc >= szc); ASSERT(nszc <= TTE256M); s = nhbits[szc] - n2color[szc].nnbits; s -= nhbits[nszc] - n2color[nszc].nnbits; return (s); } uint_t page_convert_color_cpu(uint_t ncolor, uchar_t szc, uchar_t nszc) { uint_t color; ASSERT(nszc > szc); ASSERT(nszc <= TTE256M); ncolor = n2_color2hash(ncolor, nszc, 0); color = ncolor << (nhbits[szc] - nhbits[nszc]); color = n2_hash2color(color, szc); return (color); } #define PAPFN_2_MNODE(pfn) \ (((pfn) & it->mi_mnode_pfn_mask) >> it->mi_mnode_pfn_shift) /*ARGSUSED*/ pfn_t page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color, uint_t ceq_mask, uint_t color_mask, void *cookie) { mem_node_iterator_t *it = cookie; pfn_t pstep = PNUM_SIZE(szc); pfn_t npfn, pfn_ceq_mask, pfn_color; pfn_t tmpmask, mask = (pfn_t)-1; uint_t pfnmn; ASSERT((color & ~ceq_mask) == 0); ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); /* convert RA to PA for accurate color calculation */ if (it->mi_init) { /* first call after it, so cache these values */ it->mi_hash_ceq_mask = n2_color2hash(ceq_mask, szc, it->mi_mnode_mask); it->mi_hash_color = n2_color2hash(color, szc, it->mi_mnode); it->mi_init = 0; } else { ASSERT(it->mi_hash_ceq_mask == n2_color2hash(ceq_mask, szc, it->mi_mnode_mask)); ASSERT(it->mi_hash_color == n2_color2hash(color, szc, it->mi_mnode)); } ceq_mask = it->mi_hash_ceq_mask; color = it->mi_hash_color; pfn += it->mi_ra_to_pa; /* restart here when we switch memblocks */ next_mem_block: pfnmn = PAPFN_2_MNODE(pfn); if ((((page_papfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0) && (pfnmn == it->mi_mnode)) { /* we start from the page with correct color and mnode */ if (szc >= TTE512K) { if (szc >= TTE4M) { /* page color is PA[32:28] */ pfn_ceq_mask = ceq_mask << 15; } else { /* page color is PA[32:28].PA[19:19] */ pfn_ceq_mask = ((ceq_mask & 1) << 6) | ((ceq_mask >> 1) << 15); } /* * Preserve mnode bits in case they are not part of the * color mask (eg., 8GB interleave, mnode bits 34:33). */ pfn_ceq_mask |= it->mi_mnode_pfn_mask; npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); goto done; } else { /* * We deal 64K or 8K page. Check if we could the * satisfy the request without changing PA[32:28] */ pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); pfn_ceq_mask |= it->mi_mnode_pfn_mask; npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask); if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) goto done; /* * for next pfn we have to change bits PA[32:28] * set PA[63:28] and PA[19:18] of the next pfn */ npfn = (pfn >> 15) << 15; npfn |= (ceq_mask & color & 3) << 5; pfn_ceq_mask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13; pfn_ceq_mask |= it->mi_mnode_pfn_mask; npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask); /* * set bits PA[17:13] to match the color */ npfn |= ((npfn >> 15) ^ (color >> 2)) & (ceq_mask >> 2); goto done; } } /* * we start from the page with incorrect color - rare case */ if (szc >= TTE512K) { if (szc >= TTE4M) { /* page color is in bits PA[32:28] */ npfn = ((pfn >> 20) << 20) | (color << 15); pfn_ceq_mask = (ceq_mask << 15) | 0x7fff; } else { /* try get the right color by changing bit PA[19:19] */ npfn = pfn + pstep; pfnmn = PAPFN_2_MNODE(npfn); if ((((page_papfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0) && (pfnmn == it->mi_mnode)) goto done; /* page color is PA[32:28].PA[19:19] */ pfn_ceq_mask = ((ceq_mask & 1) << 6) | ((ceq_mask >> 1) << 15) | (0xff << 7); pfn_color = ((color & 1) << 6) | ((color >> 1) << 15); npfn = ((pfn >> 20) << 20) | pfn_color; } /* Fix mnode if necessary */ if ((pfnmn = PAPFN_2_MNODE(npfn)) != it->mi_mnode) npfn += ((it->mi_mnode - pfnmn) & it->mi_mnode_mask) << it->mi_mnode_pfn_shift; /* * Preserve mnode bits in case they are not part of the color * mask eg 8GB interleave, mnode bits 34:33). */ pfn_ceq_mask |= it->mi_mnode_pfn_mask; while (npfn <= pfn) { npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); } goto done; } /* * We deal 64K or 8K page of incorrect color. * Try correcting color without changing PA[32:28] */ pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2); pfn_color = ((color & 3) << 5) | (color >> 2); if (pfnmn == it->mi_mnode) { npfn = (pfn & ~(pfn_t)0x7f); npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; if (((page_papfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0) { /* the color is fixed - find the next page */ pfn_ceq_mask |= it->mi_mnode_pfn_mask; while (npfn <= pfn) { npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask); } if ((((npfn ^ pfn) >> 15) & 0x1f) == 0) goto done; } } /* to fix the color need to touch PA[32:28] */ npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) : (((pfn >> 18) << 18) | ((color & 0x1c) << 13)); /* fix mnode if input pfn is in the wrong mnode. */ if ((pfnmn = PAPFN_2_MNODE(npfn)) != it->mi_mnode) { npfn += ((it->mi_mnode - pfnmn) & it->mi_mnode_mask) << it->mi_mnode_pfn_shift; } tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13; tmpmask |= it->mi_mnode_pfn_mask; while (npfn <= pfn) { npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask); } /* set bits PA[19:13] to match the color */ npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask; npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn; done: ASSERT(((page_papfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0); ASSERT(PAPFN_2_MNODE(npfn) == it->mi_mnode); /* PA to RA */ npfn -= it->mi_ra_to_pa; /* check for possible memblock switch */ if (npfn > it->mi_mblock_end) { pfn = plat_mem_node_iterator_init(npfn, it->mi_mnode, szc, it, 0); if (pfn == (pfn_t)-1) return (pfn); ASSERT(pfn >= it->mi_mblock_base && pfn <= it->mi_mblock_end); pfn += it->mi_ra_to_pa; goto next_mem_block; } return (npfn); } /* * init page coloring * VF encodes node_id for an L-group in either bit 30 or 31:30, * which effectively reduces the number of colors available per mnode. */ void page_coloring_init_cpu() { int i; uchar_t id; uchar_t lo; uchar_t hi; n2color_t m; mem_node_iterator_t it; static uchar_t idmask[] = {0, 0x7, 0x1f, 0x1f, 0x1f, 0x1f}; for (i = 0; i < max_mem_nodes; i++) { memset(&it, 0, sizeof (it)); if (plat_mem_node_iterator_init(0, i, 0, &it, 1) != (pfn_t)-1) break; } ASSERT(i < max_mem_nodes); for (i = 0; i < mmu_page_sizes; i++) { (void) memset(&m, 0, sizeof (m)); id = it.mi_mnode_pfn_mask >> 15; /* node id mask */ id &= idmask[i]; lo = lowbit(id); if (lo > 0) { hi = highbit(id); m.nnbits = hi - lo + 1; m.nnmask = (1 << m.nnbits) - 1; lo += nhbits[i] - 5; m.lomask = (1 << (lo - 1)) - 1; m.lobits = lo - 1; } hw_page_array[i].hp_colors = 1 << (nhbits[i] - m.nnbits); n2color[i] = m; } } /* * group colorequiv colors on N2 by low order bits of the color first */ void page_set_colorequiv_arr_cpu(void) { static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0}; nequiv_shades_log2[1] -= n2color[1].nnbits; if (colorequiv > 1) { int i; uint_t sv_a = lowbit(colorequiv) - 1; if (sv_a > 15) sv_a = 15; for (i = 0; i < MMU_PAGE_SIZES; i++) { uint_t colors; uint_t a = sv_a; if ((colors = hw_page_array[i].hp_colors) <= 1) continue; while ((colors >> a) == 0) a--; if (a > (colorequivszc[i] & 0xf) + (colorequivszc[i] >> 4)) { if (a <= nequiv_shades_log2[i]) { colorequivszc[i] = (uchar_t)a; } else { colorequivszc[i] = ((a - nequiv_shades_log2[i]) << 4) | nequiv_shades_log2[i]; } } } } }