xref: /illumos-gate/usr/src/uts/sun4v/cpu/niagara2.c (revision fe70c9cf90dfc23d18485fb7b4b20a1175d53a8b)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/systm.h>
30 #include <sys/archsystm.h>
31 #include <sys/machparam.h>
32 #include <sys/machsystm.h>
33 #include <sys/cpu.h>
34 #include <sys/elf_SPARC.h>
35 #include <vm/hat_sfmmu.h>
36 #include <vm/page.h>
37 #include <vm/vm_dep.h>
38 #include <sys/cpuvar.h>
39 #include <sys/async.h>
40 #include <sys/cmn_err.h>
41 #include <sys/debug.h>
42 #include <sys/dditypes.h>
43 #include <sys/sunddi.h>
44 #include <sys/cpu_module.h>
45 #include <sys/prom_debug.h>
46 #include <sys/vmsystm.h>
47 #include <sys/prom_plat.h>
48 #include <sys/sysmacros.h>
49 #include <sys/intreg.h>
50 #include <sys/machtrap.h>
51 #include <sys/ontrap.h>
52 #include <sys/ivintr.h>
53 #include <sys/atomic.h>
54 #include <sys/panic.h>
55 #include <sys/dtrace.h>
56 #include <sys/simulate.h>
57 #include <sys/fault.h>
58 #include <sys/niagara2regs.h>
59 #include <sys/hsvc.h>
60 #include <sys/trapstat.h>
61 
62 uint_t root_phys_addr_lo_mask = 0xffffffffU;
63 char cpu_module_name[] = "SUNW,UltraSPARC-T2";
64 
65 /*
66  * Hypervisor services information for the NIAGARA2 CPU module
67  */
68 static boolean_t niagara2_hsvc_available = B_TRUE;
69 static uint64_t niagara2_sup_minor;		/* Supported minor number */
70 static hsvc_info_t niagara2_hsvc = {
71 	HSVC_REV_1, NULL, HSVC_GROUP_NIAGARA2_CPU, NIAGARA2_HSVC_MAJOR,
72 	NIAGARA2_HSVC_MINOR, cpu_module_name
73 };
74 
75 #ifdef N2_1x_CPC_WORKAROUNDS
76 static uint64_t cpu_ver;		/* Niagara2 CPU version reg */
77 uint64_t	ni2_1x_perf_workarounds = 0;
78 
79 /* Niagara2 CPU version register */
80 #define	VER_MASK_MAJOR_SHIFT	28
81 #define	VER_MASK_MAJOR_MASK	0xf
82 
83 extern uint64_t va_to_pa(void *);
84 extern uint64_t ni2_getver();		/* HV code to get %hver */
85 extern uint64_t niagara2_getver(uint64_t ni2_getver_ra, uint64_t *cpu_version);
86 #endif
87 
88 void
89 cpu_setup(void)
90 {
91 	extern int mmu_exported_pagesize_mask;
92 	extern int cpc_has_overflow_intr;
93 	int status;
94 
95 #ifdef N2_1x_CPC_WORKAROUNDS
96 	/*
97 	 * Get CPU version for Niagara2 part.
98 	 */
99 	if (niagara2_getver(va_to_pa((void *)ni2_getver), &cpu_ver) == H_EOK &&
100 	    ((cpu_ver >> VER_MASK_MAJOR_SHIFT) & VER_MASK_MAJOR_MASK) <= 1)
101 		ni2_1x_perf_workarounds = 1;
102 #endif
103 
104 	/*
105 	 * Negotiate the API version for Niagara2 specific hypervisor
106 	 * services.
107 	 */
108 	status = hsvc_register(&niagara2_hsvc, &niagara2_sup_minor);
109 	if (status != 0) {
110 		cmn_err(CE_WARN, "%s: cannot negotiate hypervisor services "
111 		    "group: 0x%lx major: 0x%lx minor: 0x%lx errno: %d",
112 		    niagara2_hsvc.hsvc_modname, niagara2_hsvc.hsvc_group,
113 		    niagara2_hsvc.hsvc_major, niagara2_hsvc.hsvc_minor, status);
114 		niagara2_hsvc_available = B_FALSE;
115 	}
116 
117 	/*
118 	 * The setup common to all CPU modules is done in cpu_setup_common
119 	 * routine.
120 	 */
121 	cpu_setup_common(NULL);
122 
123 	cache |= (CACHE_PTAG | CACHE_IOCOHERENT);
124 
125 	if ((mmu_exported_pagesize_mask &
126 	    DEFAULT_SUN4V_MMU_PAGESIZE_MASK) !=
127 	    DEFAULT_SUN4V_MMU_PAGESIZE_MASK)
128 		cmn_err(CE_PANIC, "machine description"
129 		    " does not have required sun4v page sizes"
130 		    " 8K, 64K and 4M: MD mask is 0x%x",
131 		    mmu_exported_pagesize_mask);
132 
133 	cpu_hwcap_flags = AV_SPARC_VIS | AV_SPARC_VIS2 | AV_SPARC_ASI_BLK_INIT;
134 
135 	/*
136 	 * Niagara2 supports a 48-bit subset of the full 64-bit virtual
137 	 * address space. Virtual addresses between 0x0000800000000000
138 	 * and 0xffff.7fff.ffff.ffff inclusive lie within a "VA Hole"
139 	 * and must never be mapped. In addition, software must not use
140 	 * pages within 4GB of the VA hole as instruction pages to
141 	 * avoid problems with prefetching into the VA hole.
142 	 */
143 	hole_start = (caddr_t)((1ull << (va_bits - 1)) - (1ull << 32));
144 	hole_end = (caddr_t)((0ull - (1ull << (va_bits - 1))) + (1ull << 32));
145 
146 	/*
147 	 * Niagara2 has a performance counter overflow interrupt
148 	 */
149 	cpc_has_overflow_intr = 1;
150 
151 	/*
152 	 * Enable 4M pages for OOB.
153 	 */
154 	max_uheap_lpsize = MMU_PAGESIZE4M;
155 	max_ustack_lpsize = MMU_PAGESIZE4M;
156 	max_privmap_lpsize = MMU_PAGESIZE4M;
157 }
158 
159 /*
160  * Set the magic constants of the implementation.
161  */
162 void
163 cpu_fiximp(struct cpu_node *cpunode)
164 {
165 	/*
166 	 * The Cache node is optional in MD. Therefore in case "Cache"
167 	 * node does not exists in MD, set the default L2 cache associativity,
168 	 * size, linesize.
169 	 */
170 	if (cpunode->ecache_size == 0)
171 		cpunode->ecache_size = L2CACHE_SIZE;
172 	if (cpunode->ecache_linesize == 0)
173 		cpunode->ecache_linesize = L2CACHE_LINESIZE;
174 	if (cpunode->ecache_associativity == 0)
175 		cpunode->ecache_associativity = L2CACHE_ASSOCIATIVITY;
176 }
177 
178 static int niagara2_cpucnt;
179 
180 void
181 cpu_init_private(struct cpu *cp)
182 {
183 	extern int niagara_kstat_init(void);
184 
185 	/*
186 	 * The cpu_ipipe and cpu_fpu fields are initialized based on
187 	 * the execution unit sharing information from the MD. They default
188 	 * to the virtual CPU id in the absence of such information.
189 	 */
190 	cp->cpu_m.cpu_ipipe = cpunodes[cp->cpu_id].exec_unit_mapping;
191 	if (cp->cpu_m.cpu_ipipe == NO_EU_MAPPING_FOUND)
192 		cp->cpu_m.cpu_ipipe = (id_t)(cp->cpu_id);
193 
194 	cp->cpu_m.cpu_fpu = cpunodes[cp->cpu_id].fpu_mapping;
195 	if (cp->cpu_m.cpu_fpu == NO_EU_MAPPING_FOUND)
196 		cp->cpu_m.cpu_fpu = (id_t)(cp->cpu_id);
197 
198 	/*
199 	 * Niagara 2 defines the core to be at the FPU level
200 	 */
201 	cp->cpu_m.cpu_core = cp->cpu_m.cpu_fpu;
202 
203 	ASSERT(MUTEX_HELD(&cpu_lock));
204 	if ((niagara2_cpucnt++ == 0) && (niagara2_hsvc_available == B_TRUE))
205 		(void) niagara_kstat_init();
206 }
207 
208 /*ARGSUSED*/
209 void
210 cpu_uninit_private(struct cpu *cp)
211 {
212 	extern int niagara_kstat_fini(void);
213 
214 	ASSERT(MUTEX_HELD(&cpu_lock));
215 	if ((--niagara2_cpucnt == 0) && (niagara2_hsvc_available == B_TRUE))
216 		(void) niagara_kstat_fini();
217 }
218 
219 /*
220  * On Niagara2, any flush will cause all preceding stores to be
221  * synchronized wrt the i$, regardless of address or ASI.  In fact,
222  * the address is ignored, so we always flush address 0.
223  */
224 /*ARGSUSED*/
225 void
226 dtrace_flush_sec(uintptr_t addr)
227 {
228 	doflush(0);
229 }
230 
231 /*
232  * Trapstat support for Niagara2 processor
233  * The Niagara2 provides HWTW support for TSB lookup and with HWTW
234  * enabled no TSB hit information will be available. Therefore setting
235  * the time spent in TLB miss handler for TSB hits to 0.
236  */
237 int
238 cpu_trapstat_conf(int cmd)
239 {
240 	int status = 0;
241 
242 	switch (cmd) {
243 	case CPU_TSTATCONF_INIT:
244 	case CPU_TSTATCONF_FINI:
245 	case CPU_TSTATCONF_ENABLE:
246 	case CPU_TSTATCONF_DISABLE:
247 		break;
248 	default:
249 		status = EINVAL;
250 		break;
251 	}
252 	return (status);
253 }
254 
255 void
256 cpu_trapstat_data(void *buf, uint_t tstat_pgszs)
257 {
258 	tstat_pgszdata_t	*tstatp = (tstat_pgszdata_t *)buf;
259 	int	i;
260 
261 	for (i = 0; i < tstat_pgszs; i++, tstatp++) {
262 		tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_count = 0;
263 		tstatp->tpgsz_kernel.tmode_itlb.ttlb_tlb.tmiss_time = 0;
264 		tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_count = 0;
265 		tstatp->tpgsz_user.tmode_itlb.ttlb_tlb.tmiss_time = 0;
266 		tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_count = 0;
267 		tstatp->tpgsz_kernel.tmode_dtlb.ttlb_tlb.tmiss_time = 0;
268 		tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_count = 0;
269 		tstatp->tpgsz_user.tmode_dtlb.ttlb_tlb.tmiss_time = 0;
270 	}
271 }
272 
273 /* NI2 L2$ index is pa[32:28]^pa[17:13].pa[19:18]^pa[12:11].pa[10:6] */
274 uint_t
275 page_pfn_2_color_cpu(pfn_t pfn, uchar_t szc)
276 {
277 	uint_t color;
278 
279 	ASSERT(szc <= TTE256M);
280 
281 	pfn = PFN_BASE(pfn, szc);
282 	color = ((pfn >> 15) ^ pfn) & 0x1f;
283 	if (szc >= TTE4M)
284 		return (color);
285 
286 	color = (color << 2) | ((pfn >> 5) & 0x3);
287 
288 	return (szc <= TTE64K ? color : (color >> 1));
289 }
290 
291 #if TTE256M != 5
292 #error TTE256M is not 5
293 #endif
294 
295 uint_t
296 page_get_nsz_color_mask_cpu(uchar_t szc, uint_t mask)
297 {
298 	static uint_t ni2_color_masks[5] = {0x63, 0x1e, 0x3e, 0x1f, 0x1f};
299 	ASSERT(szc < TTE256M);
300 
301 	mask &= ni2_color_masks[szc];
302 	return ((szc == TTE64K || szc == TTE512K) ? (mask >> 1) : mask);
303 }
304 
305 uint_t
306 page_get_nsz_color_cpu(uchar_t szc, uint_t color)
307 {
308 	ASSERT(szc < TTE256M);
309 	return ((szc == TTE64K || szc == TTE512K) ? (color >> 1) : color);
310 }
311 
312 uint_t
313 page_get_color_shift_cpu(uchar_t szc, uchar_t nszc)
314 {
315 	ASSERT(nszc > szc);
316 	ASSERT(nszc <= TTE256M);
317 
318 	if (szc <= TTE64K)
319 		return ((nszc >= TTE4M) ? 2 : ((nszc >= TTE512K) ? 1 : 0));
320 	if (szc == TTE512K)
321 		return (1);
322 
323 	return (0);
324 }
325 
326 /*ARGSUSED*/
327 pfn_t
328 page_next_pfn_for_color_cpu(pfn_t pfn, uchar_t szc, uint_t color,
329     uint_t ceq_mask, uint_t color_mask)
330 {
331 	pfn_t pstep = PNUM_SIZE(szc);
332 	pfn_t npfn, pfn_ceq_mask, pfn_color;
333 	pfn_t tmpmask, mask = (pfn_t)-1;
334 
335 	ASSERT((color & ~ceq_mask) == 0);
336 
337 	if (((page_pfn_2_color_cpu(pfn, szc) ^ color) & ceq_mask) == 0) {
338 
339 		/* we start from the page with correct color */
340 		if (szc >= TTE512K) {
341 			if (szc >= TTE4M) {
342 				/* page color is PA[32:28] */
343 				pfn_ceq_mask = ceq_mask << 15;
344 			} else {
345 				/* page color is PA[32:28].PA[19:19] */
346 				pfn_ceq_mask = ((ceq_mask & 1) << 6) |
347 				    ((ceq_mask >> 1) << 15);
348 			}
349 			pfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask);
350 			return (pfn);
351 		} else {
352 			/*
353 			 * We deal 64K or 8K page. Check if we could the
354 			 * satisfy the request without changing PA[32:28]
355 			 */
356 			pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2);
357 			npfn = ADD_MASKED(pfn, pstep, pfn_ceq_mask, mask);
358 
359 			if ((((npfn ^ pfn) >> 15) & 0x1f) == 0)
360 				return (npfn);
361 
362 			/*
363 			 * for next pfn we have to change bits PA[32:28]
364 			 * set PA[63:28] and PA[19:18] of the next pfn
365 			 */
366 			npfn = (pfn >> 15) << 15;
367 			npfn |= (ceq_mask & color & 3) << 5;
368 			pfn_ceq_mask = (szc == TTE8K) ? 0 :
369 			    (ceq_mask & 0x1c) << 13;
370 			npfn = ADD_MASKED(npfn, (1 << 15), pfn_ceq_mask, mask);
371 
372 			/*
373 			 * set bits PA[17:13] to match the color
374 			 */
375 			ceq_mask >>= 2;
376 			color = (color >> 2) & ceq_mask;
377 			npfn |= ((npfn >> 15) ^ color) & ceq_mask;
378 			return (npfn);
379 		}
380 	}
381 
382 	/*
383 	 * we start from the page with incorrect color - rare case
384 	 */
385 	if (szc >= TTE512K) {
386 		if (szc >= TTE4M) {
387 			/* page color is in bits PA[32:28] */
388 			npfn = ((pfn >> 20) << 20) | (color << 15);
389 			pfn_ceq_mask = (ceq_mask << 15) | 0x7fff;
390 		} else {
391 			/* try get the right color by changing bit PA[19:19] */
392 			npfn = pfn + pstep;
393 			if (((page_pfn_2_color_cpu(npfn, szc) ^ color) &
394 			    ceq_mask) == 0)
395 				return (npfn);
396 
397 			/* page color is PA[32:28].PA[19:19] */
398 			pfn_ceq_mask = ((ceq_mask & 1) << 6) |
399 			    ((ceq_mask >> 1) << 15) | (0xff << 7);
400 			pfn_color = ((color & 1) << 6) | ((color >> 1) << 15);
401 			npfn = ((pfn >> 20) << 20) | pfn_color;
402 		}
403 
404 		while (npfn <= pfn) {
405 			npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask);
406 		}
407 		return (npfn);
408 	}
409 
410 	/*
411 	 * We deal 64K or 8K page of incorrect color.
412 	 * Try correcting color without changing PA[32:28]
413 	 */
414 
415 	pfn_ceq_mask = ((ceq_mask & 3) << 5) | (ceq_mask >> 2);
416 	pfn_color = ((color & 3) << 5) | (color >> 2);
417 	npfn = (pfn & ~(pfn_t)0x7f);
418 	npfn |= (((pfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask;
419 	npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn;
420 
421 	if (((page_pfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0) {
422 
423 		/* the color is fixed - find the next page */
424 		while (npfn <= pfn) {
425 			npfn = ADD_MASKED(npfn, pstep, pfn_ceq_mask, mask);
426 		}
427 		if ((((npfn ^ pfn) >> 15) & 0x1f) == 0)
428 			return (npfn);
429 	}
430 
431 	/* to fix the color need to touch PA[32:28] */
432 	npfn = (szc == TTE8K) ? ((pfn >> 15) << 15) :
433 	    (((pfn >> 18) << 18) | ((color & 0x1c) << 13));
434 	tmpmask = (szc == TTE8K) ? 0 : (ceq_mask & 0x1c) << 13;
435 
436 	while (npfn <= pfn) {
437 		npfn = ADD_MASKED(npfn, (1 << 15), tmpmask, mask);
438 	}
439 
440 	/* set bits PA[19:13] to match the color */
441 	npfn |= (((npfn >> 15) & 0x1f) ^ pfn_color) & pfn_ceq_mask;
442 	npfn = (szc == TTE64K) ? (npfn & ~(pfn_t)0x7) : npfn;
443 
444 	ASSERT(((page_pfn_2_color_cpu(npfn, szc) ^ color) & ceq_mask) == 0);
445 
446 	return (npfn);
447 }
448 
449 /*
450  * init page coloring
451  */
452 void
453 page_coloring_init_cpu()
454 {
455 	int i;
456 
457 	hw_page_array[0].hp_colors = 1 << 7;
458 	hw_page_array[1].hp_colors = 1 << 7;
459 	hw_page_array[2].hp_colors = 1 << 6;
460 
461 	for (i = 3; i < mmu_page_sizes; i++) {
462 		hw_page_array[i].hp_colors = 1 << 5;
463 	}
464 }
465 
466 /*
467  * group colorequiv colors on N2 by low order bits of the color first
468  */
469 void
470 page_set_colorequiv_arr_cpu(void)
471 {
472 	static uint_t nequiv_shades_log2[MMU_PAGE_SIZES] = {2, 5, 0, 0, 0, 0};
473 
474 	if (colorequiv > 1) {
475 		int i;
476 		uint_t sv_a = lowbit(colorequiv) - 1;
477 
478 		if (sv_a > 15)
479 			sv_a = 15;
480 
481 		for (i = 0; i < MMU_PAGE_SIZES; i++) {
482 			uint_t colors;
483 			uint_t a = sv_a;
484 
485 			if ((colors = hw_page_array[i].hp_colors) <= 1)
486 				continue;
487 			while ((colors >> a) == 0)
488 				a--;
489 			if (a > (colorequivszc[i] & 0xf) +
490 			    (colorequivszc[i] >> 4)) {
491 				if (a <= nequiv_shades_log2[i]) {
492 					colorequivszc[i] = a;
493 				} else {
494 					colorequivszc[i] =
495 					    ((a - nequiv_shades_log2[i]) << 4) |
496 					    nequiv_shades_log2[i];
497 				}
498 			}
499 		}
500 	}
501 }
502