xref: /illumos-gate/usr/src/uts/sun4v/cpu/niagara_perfctr.c (revision a588362c0375ebed265afad9dd5a60b5739820df)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/async.h>
28 #include <sys/sunddi.h>
29 #include <sys/sunndi.h>
30 #include <sys/ddi_impldefs.h>
31 #include <sys/machsystm.h>
32 #include <sys/hypervisor_api.h>
33 #include <sys/kstat.h>
34 #if defined(NIAGARA_IMPL)
35 #include <sys/niagararegs.h>
36 #elif defined(NIAGARA2_IMPL) || defined(VFALLS_IMPL)
37 #include <sys/niagara2regs.h>
38 #endif
39 
40 extern char cpu_module_name[];
41 
42 /*
43  * Data structure used to build array of event-names and pcr-mask values
44  */
45 typedef struct ni_kev_mask {
46 	char		*event_name;
47 	uint64_t	pcr_mask;
48 } ni_kev_mask_t;
49 
50 /*
51  * Kstat data structure for DRAM and JBUS performance counters
52  *
53  * Note that these performance counters are only 31 bits wide. Since
54  * the "busstat" command assumes a 32-bit counter, we emulate a 32-bit
55  * counter by detecting overflow on read of these performance counters
56  * and using the least significant bit of the overflow count as the
57  * most significant bit (i.e. bit# 31) of the DRAM and JBUS performance
58  * counters.
59  */
60 #define	NUM_OF_PICS	2
61 
62 typedef struct ni_ksinfo {
63 	uint8_t		pic_no_evs;			/* number of events */
64 	uint8_t		pic_sel_shift[NUM_OF_PICS];
65 	uint8_t		pic_shift[NUM_OF_PICS];
66 	uint64_t	pic_mask[NUM_OF_PICS];
67 	kstat_t		*pic_name_ksp[NUM_OF_PICS];
68 	kstat_t		*cntr_ksp;
69 	uint32_t	pic_reg[NUM_OF_PICS];
70 	uint32_t	pcr_reg;
71 	uint32_t	pic_overflow[NUM_OF_PICS];	/* overflow count */
72 	uint32_t	pic_last_val[NUM_OF_PICS];	/* last PIC value */
73 } ni_ksinfo_t;
74 
75 static ni_ksinfo_t	*ni_dram_kstats[NIAGARA_DRAM_BANKS];
76 
77 #if defined(NIAGARA_IMPL)
78 static ni_ksinfo_t	*ni_jbus_kstat;
79 #endif
80 
81 typedef struct ni_perf_regs {
82 	uint32_t	pcr_reg;
83 	uint32_t	pic_reg;
84 } ni_perf_regs_t;
85 
86 static ni_perf_regs_t dram_perf_regs[] = {
87 	{HV_NIAGARA_DRAM_CTL0, HV_NIAGARA_DRAM_COUNT0},
88 	{HV_NIAGARA_DRAM_CTL1, HV_NIAGARA_DRAM_COUNT1},
89 	{HV_NIAGARA_DRAM_CTL2, HV_NIAGARA_DRAM_COUNT2},
90 	{HV_NIAGARA_DRAM_CTL3, HV_NIAGARA_DRAM_COUNT3},
91 #ifdef VFALLS_IMPL
92 	{HV_NIAGARA_DRAM_CTL4, HV_NIAGARA_DRAM_COUNT4},
93 	{HV_NIAGARA_DRAM_CTL5, HV_NIAGARA_DRAM_COUNT5},
94 	{HV_NIAGARA_DRAM_CTL6, HV_NIAGARA_DRAM_COUNT6},
95 	{HV_NIAGARA_DRAM_CTL7, HV_NIAGARA_DRAM_COUNT7}
96 #endif
97 };
98 
99 #ifdef VFALLS_IMPL
100 /*
101  * Kstat data structure for Zambezi performance counters
102  * These performance counters are 64 bits wide.
103  */
104 static ni_ksinfo_t	*zam_lpu_kstats[ZAMBEZI_LPU_COUNTERS];
105 static ni_ksinfo_t	*zam_gpd_kstats[ZAMBEZI_GPD_COUNTERS];
106 static ni_ksinfo_t	*zam_asu_kstats[ZAMBEZI_ASU_COUNTERS];
107 
108 typedef struct zam_perf_regs {
109 	uint32_t	pcr_reg;
110 	uint32_t	pic_reg[NUM_OF_PICS];
111 } zam_perf_regs_t;
112 
113 static zam_perf_regs_t lpu_perf_regs[] = {
114 	{HV_ZAM0_LPU_A_PCR, HV_ZAM0_LPU_A_PIC0, HV_ZAM0_LPU_A_PIC1},
115 	{HV_ZAM0_LPU_B_PCR, HV_ZAM0_LPU_B_PIC0, HV_ZAM0_LPU_B_PIC1},
116 	{HV_ZAM0_LPU_C_PCR, HV_ZAM0_LPU_C_PIC0, HV_ZAM0_LPU_C_PIC1},
117 	{HV_ZAM0_LPU_D_PCR, HV_ZAM0_LPU_D_PIC0, HV_ZAM0_LPU_D_PIC1},
118 
119 	{HV_ZAM1_LPU_A_PCR, HV_ZAM1_LPU_A_PIC0, HV_ZAM1_LPU_A_PIC1},
120 	{HV_ZAM1_LPU_B_PCR, HV_ZAM1_LPU_B_PIC0, HV_ZAM1_LPU_B_PIC1},
121 	{HV_ZAM1_LPU_C_PCR, HV_ZAM1_LPU_C_PIC0, HV_ZAM1_LPU_C_PIC1},
122 	{HV_ZAM1_LPU_D_PCR, HV_ZAM1_LPU_D_PIC0, HV_ZAM1_LPU_D_PIC1},
123 
124 	{HV_ZAM2_LPU_A_PCR, HV_ZAM2_LPU_A_PIC0, HV_ZAM2_LPU_A_PIC1},
125 	{HV_ZAM2_LPU_B_PCR, HV_ZAM2_LPU_B_PIC0, HV_ZAM2_LPU_B_PIC1},
126 	{HV_ZAM2_LPU_C_PCR, HV_ZAM2_LPU_C_PIC0, HV_ZAM2_LPU_C_PIC1},
127 	{HV_ZAM2_LPU_D_PCR, HV_ZAM2_LPU_D_PIC0, HV_ZAM2_LPU_D_PIC1},
128 
129 	{HV_ZAM3_LPU_A_PCR, HV_ZAM3_LPU_A_PIC0, HV_ZAM3_LPU_A_PIC1},
130 	{HV_ZAM3_LPU_B_PCR, HV_ZAM3_LPU_B_PIC0, HV_ZAM3_LPU_B_PIC1},
131 	{HV_ZAM3_LPU_C_PCR, HV_ZAM3_LPU_C_PIC0, HV_ZAM3_LPU_C_PIC1},
132 	{HV_ZAM3_LPU_D_PCR, HV_ZAM3_LPU_D_PIC0, HV_ZAM3_LPU_D_PIC1}
133 };
134 
135 static zam_perf_regs_t gpd_perf_regs[] = {
136 	{HV_ZAM0_GPD_PCR, HV_ZAM0_GPD_PIC0, HV_ZAM0_GPD_PIC1},
137 	{HV_ZAM1_GPD_PCR, HV_ZAM1_GPD_PIC0, HV_ZAM1_GPD_PIC1},
138 	{HV_ZAM2_GPD_PCR, HV_ZAM2_GPD_PIC0, HV_ZAM2_GPD_PIC1},
139 	{HV_ZAM3_GPD_PCR, HV_ZAM3_GPD_PIC0, HV_ZAM3_GPD_PIC1}
140 };
141 
142 static zam_perf_regs_t asu_perf_regs[] = {
143 	{HV_ZAM0_ASU_PCR, HV_ZAM0_ASU_PIC0, HV_ZAM0_ASU_PIC1},
144 	{HV_ZAM1_ASU_PCR, HV_ZAM1_ASU_PIC0, HV_ZAM1_ASU_PIC1},
145 	{HV_ZAM2_ASU_PCR, HV_ZAM2_ASU_PIC0, HV_ZAM2_ASU_PIC1},
146 	{HV_ZAM3_ASU_PCR, HV_ZAM3_ASU_PIC0, HV_ZAM3_ASU_PIC1}
147 };
148 
149 static int zam_cntr_kstat_update(kstat_t *, int);
150 #endif
151 
152 static void ni_create_name_kstat(char *, ni_ksinfo_t *, ni_kev_mask_t *);
153 static void ni_delete_name_kstat(ni_ksinfo_t *);
154 
155 static kstat_t *ni_create_cntr_kstat(char *, int,
156 	int (*update)(kstat_t *, int), void *);
157 
158 static int ni_cntr_kstat_update(kstat_t *, int);
159 
160 static kstat_t *ni_create_picN_kstat(char *, int, int, int,
161 	ni_kev_mask_t *);
162 
163 #ifdef DEBUG
164 static int	ni_perf_debug;
165 #endif
166 
167 /*
168  * Niagara, Niagara2 and VFalls DRAM Performance Events
169  */
170 static ni_kev_mask_t
171 niagara_dram_events[] = {
172 	{"mem_reads",		0x0},
173 	{"mem_writes",		0x1},
174 	{"mem_read_write",	0x2},
175 #if defined(NIAGARA_IMPL)
176 	{"bank_busy_stalls",	0x3},
177 #endif
178 	{"rd_queue_latency",	0x4},
179 	{"wr_queue_latency",	0x5},
180 	{"rw_queue_latency",	0x6},
181 	{"wb_buf_hits",		0x7},
182 	{"clear_pic",		0xf}
183 };
184 
185 #if defined(VFALLS_IMPL)
186 /*
187  * Zambezi Performance Events
188  */
189 static ni_kev_mask_t
190 zam_lpu_perf_events[] = {
191 	{"none",		0x0},
192 	{"clock_cycles",	0x1},
193 	{"cycles_c2c_portX",	0x2},
194 	{"cycles_mem_portX",	0x3},
195 	{"cycles_WB_portX",	0x4},
196 	{"cycles_NC_portX",	0x5},
197 	{"cycles_c2c_portY",	0x6},
198 	{"cycles_mem_portY",	0x7},
199 	{"cycles_WB_portY",	0x8},
200 	{"cycles_NC_portY",	0x9},
201 	{"cycles_c2c_portZ",	0xa},
202 	{"cycles_mem_portZ",	0xb},
203 	{"cycles_WB_portZ",	0xc},
204 	{"cycles_NC_portZ",	0xd},
205 	{"cycles_TID_WB",	0xe},
206 	{"cycles_TID_INV",	0xf},
207 	{"cycles_TID_RTD",	0x10},
208 	{"cycles_TID_RTO",	0x11},
209 	{"cycles_TID_RTS",	0x12},
210 	{"cycles_IO_WRM",	0x13},
211 	{"cycles_IO_RD",	0x14},
212 	{"cycles_WB_egress",	0x15},
213 	{"cycles_INV_egress",	0x16},
214 	{"cycles_RTO_egress",	0x17},
215 	{"cycles_RTD_egress",	0x18},
216 	{"cycles_RTS_egress",	0x19},
217 	{"cycles_no_WB",	0x1a},
218 	{"cycles_no_read/inv",	0x1b},
219 	{"cycles_HIT_M",	0x1c},
220 	{"cycles_HIT_O",	0x1d},
221 	{"cycles_HIT_S",	0x1e},
222 	{"cycles_WB_HIT",	0x1f},
223 	{"cycles_MISS",		0x20},
224 	{"cycles_READ_or_INV",	0x21},
225 	{"cycles_WB",		0x22},
226 	{"cycles_NDR",		0x23},
227 	{"cycles_cache_miss",	0x24},
228 	{"cycles_cache_hit",	0x25},
229 	{"cycles_CRC_errors",	0x26},
230 	{"cycles_replys_sent",	0x27},
231 	{"cycles_replys_recev",	0x28},
232 	{"cycles_link_retrain",	0x29},
233 	{"clear_pic",		0xff}
234 };
235 
236 static ni_kev_mask_t
237 zam_gpd_perf_events[] = {
238 	{"none",		0x0},
239 	{"clock_cycles",	0x1},
240 	{"clear_pic",		0xf}
241 };
242 
243 static ni_kev_mask_t
244 zam_asu_perf_events[] = {
245 	{"none",		0x0},
246 	{"clock_cycles",	0x1},
247 	{"asu_in_pck",		0x2},
248 	{"asu_out_pck",		0x3},
249 	{"asu_CAM_hit",		0x4},
250 	{"asu_wakeup",		0x5},
251 	{"clear_pic",		0xf}
252 };
253 #endif
254 
255 #if defined(NIAGARA_IMPL)
256 /*
257  * Niagara JBUS Performance Events
258  */
259 static ni_kev_mask_t
260 niagara_jbus_events[] = {
261 	{"jbus_cycles",		0x1},
262 	{"dma_reads",		0x2},
263 	{"dma_read_latency",	0x3},
264 	{"dma_writes",		0x4},
265 	{"dma_write8",		0x5},
266 	{"ordering_waits",	0x6},
267 	{"pio_reads",		0x8},
268 	{"pio_read_latency",	0x9},
269 	{"aok_dok_off_cycles",	0xc},
270 	{"aok_off_cycles",	0xd},
271 	{"dok_off_cycles",	0xe},
272 	{"clear_pic",		0xf}
273 };
274 #endif
275 
276 /*
277  * Create the picN kstats for DRAM, JBUS and Zambezi events
278  */
279 void
280 niagara_kstat_init()
281 {
282 	int i;
283 	ni_ksinfo_t *ksinfop;
284 #ifdef VFALLS_IMPL
285 	uint64_t stat, pcr;
286 #endif
287 
288 #ifdef DEBUG
289 	if (ni_perf_debug)
290 		printf("ni_kstat_init called\n");
291 #endif
292 
293 	/*
294 	 * Create DRAM perf events kstat
295 	 */
296 	for (i = 0; i < NIAGARA_DRAM_BANKS; i++) {
297 #ifdef VFALLS_IMPL
298 		/* check if this dram instance is enabled in the HW */
299 		stat = hv_niagara_getperf(dram_perf_regs[i].pcr_reg, &pcr);
300 		if ((stat != H_EINVAL) && (stat != H_ENOTSUPPORTED)) {
301 #endif
302 			ksinfop = (ni_ksinfo_t *)kmem_zalloc(
303 			    sizeof (ni_ksinfo_t), KM_NOSLEEP);
304 
305 			if (ksinfop == NULL) {
306 				cmn_err(CE_WARN,
307 				    "%s: no space for dram kstat\n",
308 				    cpu_module_name);
309 				break;
310 			}
311 			ksinfop->pic_no_evs =
312 			    sizeof (niagara_dram_events) /
313 			    sizeof (ni_kev_mask_t);
314 			ksinfop->pic_sel_shift[0] = NIAGARA_DRAM_PIC0_SEL_SHIFT;
315 			ksinfop->pic_shift[0] = NIAGARA_DRAM_PIC0_SHIFT;
316 			ksinfop->pic_mask[0] = NIAGARA_DRAM_PIC0_MASK;
317 			ksinfop->pic_sel_shift[1] = NIAGARA_DRAM_PIC1_SEL_SHIFT;
318 			ksinfop->pic_shift[1] = NIAGARA_DRAM_PIC1_SHIFT;
319 			ksinfop->pic_mask[1] = NIAGARA_DRAM_PIC1_MASK;
320 			ksinfop->pic_reg[0] = dram_perf_regs[i].pic_reg;
321 			ksinfop->pcr_reg = dram_perf_regs[i].pcr_reg;
322 			ni_dram_kstats[i] = ksinfop;
323 
324 			/* create basic pic event/mask pair (only once) */
325 			if (i == 0)
326 				ni_create_name_kstat("dram", ksinfop,
327 				    niagara_dram_events);
328 
329 			/* create counter kstats */
330 			ni_dram_kstats[i]->cntr_ksp = ni_create_cntr_kstat(
331 			    "dram", i, ni_cntr_kstat_update, ksinfop);
332 #ifdef VFALLS_IMPL
333 		}
334 #endif
335 	}
336 
337 #ifdef VFALLS_IMPL
338 	/*
339 	 * Create Zambezi LPU perf events kstat
340 	 */
341 	for (i = 0; i < ZAMBEZI_LPU_COUNTERS; i++) {
342 		/* check if this Zambezi LPU instance is enabled in the HW */
343 		stat = hv_niagara_getperf(lpu_perf_regs[i].pcr_reg, &pcr);
344 		if ((stat != H_EINVAL) && (stat != H_ENOTSUPPORTED)) {
345 			ksinfop = (ni_ksinfo_t *)kmem_zalloc(
346 			    sizeof (ni_ksinfo_t), KM_NOSLEEP);
347 
348 			if (ksinfop == NULL) {
349 				cmn_err(CE_WARN,
350 				    "%s: no space for zambezi lpu kstat\n",
351 				    cpu_module_name);
352 				break;
353 			}
354 			ksinfop->pic_no_evs =
355 			    sizeof (zam_lpu_perf_events) /
356 			    sizeof (ni_kev_mask_t);
357 			ksinfop->pic_sel_shift[0] = ZAMBEZI_PIC0_SEL_SHIFT;
358 			ksinfop->pic_reg[0] = lpu_perf_regs[i].pic_reg[0];
359 			ksinfop->pic_sel_shift[1] = ZAMBEZI_PIC1_SEL_SHIFT;
360 			ksinfop->pic_reg[1] = lpu_perf_regs[i].pic_reg[1];
361 			ksinfop->pcr_reg = lpu_perf_regs[i].pcr_reg;
362 			zam_lpu_kstats[i] = ksinfop;
363 
364 			/* create basic pic event/mask pair (only once) */
365 			if (i == 0)
366 				ni_create_name_kstat("lpu", ksinfop,
367 				    zam_lpu_perf_events);
368 
369 			/* create counter kstats */
370 			zam_lpu_kstats[i]->cntr_ksp = ni_create_cntr_kstat(
371 			    "lpu", i, zam_cntr_kstat_update, ksinfop);
372 		}
373 	}
374 	/*
375 	 * Create Zambezi GPD perf events kstat
376 	 */
377 	for (i = 0; i < ZAMBEZI_GPD_COUNTERS; i++) {
378 		/* check if this Zambezi GPD instance is enabled in the HW */
379 		stat = hv_niagara_getperf(gpd_perf_regs[i].pcr_reg, &pcr);
380 		if ((stat != H_EINVAL) && (stat != H_ENOTSUPPORTED)) {
381 			ksinfop = (ni_ksinfo_t *)kmem_zalloc(
382 			    sizeof (ni_ksinfo_t), KM_NOSLEEP);
383 
384 			if (ksinfop == NULL) {
385 				cmn_err(CE_WARN,
386 				    "%s: no space for zambezi gpd kstat\n",
387 				    cpu_module_name);
388 				break;
389 			}
390 			ksinfop->pic_no_evs =
391 			    sizeof (zam_gpd_perf_events) /
392 			    sizeof (ni_kev_mask_t);
393 			ksinfop->pic_sel_shift[0] = ZAMBEZI_PIC0_SEL_SHIFT;
394 			ksinfop->pic_reg[0] = gpd_perf_regs[i].pic_reg[0];
395 			ksinfop->pic_sel_shift[1] = ZAMBEZI_PIC1_SEL_SHIFT;
396 			ksinfop->pic_reg[1] = gpd_perf_regs[i].pic_reg[1];
397 			ksinfop->pcr_reg = gpd_perf_regs[i].pcr_reg;
398 			zam_gpd_kstats[i] = ksinfop;
399 
400 			/* create basic pic event/mask pair (only once) */
401 			if (i == 0)
402 				ni_create_name_kstat("gpd", ksinfop,
403 				    zam_gpd_perf_events);
404 
405 			/* create counter kstats */
406 			zam_lpu_kstats[i]->cntr_ksp = ni_create_cntr_kstat(
407 			    "gpd", i, zam_cntr_kstat_update, ksinfop);
408 		}
409 	}
410 	/*
411 	 * Create Zambezi ASU perf events kstat
412 	 */
413 	for (i = 0; i < ZAMBEZI_ASU_COUNTERS; i++) {
414 		/* check if this Zambezi ASU instance is enabled in the HW */
415 		stat = hv_niagara_getperf(asu_perf_regs[i].pcr_reg, &pcr);
416 		if ((stat != H_EINVAL) && (stat != H_ENOTSUPPORTED)) {
417 			ksinfop = (ni_ksinfo_t *)kmem_zalloc(
418 			    sizeof (ni_ksinfo_t), KM_NOSLEEP);
419 
420 			if (ksinfop == NULL) {
421 				cmn_err(CE_WARN,
422 				    "%s: no space for zambezi asu kstat\n",
423 				    cpu_module_name);
424 				break;
425 			}
426 			ksinfop->pic_no_evs =
427 			    sizeof (zam_asu_perf_events) /
428 			    sizeof (ni_kev_mask_t);
429 			ksinfop->pic_sel_shift[0] = ZAMBEZI_PIC0_SEL_SHIFT;
430 			ksinfop->pic_reg[0] = asu_perf_regs[i].pic_reg[0];
431 			ksinfop->pic_sel_shift[1] = ZAMBEZI_PIC1_SEL_SHIFT;
432 			ksinfop->pic_reg[1] = asu_perf_regs[i].pic_reg[1];
433 			ksinfop->pcr_reg = asu_perf_regs[i].pcr_reg;
434 			zam_asu_kstats[i] = ksinfop;
435 
436 			/* create basic pic event/mask pair (only once) */
437 			if (i == 0)
438 				ni_create_name_kstat("asu", ksinfop,
439 				    zam_asu_perf_events);
440 
441 			/* create counter kstats */
442 			zam_lpu_kstats[i]->cntr_ksp = ni_create_cntr_kstat(
443 			    "asu", i, zam_cntr_kstat_update, ksinfop);
444 		}
445 	}
446 #endif
447 
448 #if defined(NIAGARA_IMPL)
449 	/*
450 	 * Create JBUS perf events kstat
451 	 */
452 	ni_jbus_kstat = (ni_ksinfo_t *)kmem_alloc(sizeof (ni_ksinfo_t),
453 	    KM_NOSLEEP);
454 
455 	if (ni_jbus_kstat == NULL) {
456 		cmn_err(CE_WARN, "%s: no space for niagara jbus kstat\n",
457 		    cpu_module_name);
458 	} else {
459 		ni_jbus_kstat->pic_no_evs =
460 		    sizeof (niagara_jbus_events) / sizeof (ni_kev_mask_t);
461 		ni_jbus_kstat->pic_sel_shift[0] = NIAGARA_JBUS_PIC0_SEL_SHIFT;
462 		ni_jbus_kstat->pic_shift[0] = NIAGARA_JBUS_PIC0_SHIFT;
463 		ni_jbus_kstat->pic_mask[0] = NIAGARA_JBUS_PIC0_MASK;
464 		ni_jbus_kstat->pic_sel_shift[1] = NIAGARA_JBUS_PIC1_SEL_SHIFT;
465 		ni_jbus_kstat->pic_shift[1] = NIAGARA_JBUS_PIC1_SHIFT;
466 		ni_jbus_kstat->pic_mask[1] = NIAGARA_JBUS_PIC1_MASK;
467 		ni_jbus_kstat->pic_reg[0] = HV_NIAGARA_JBUS_COUNT;
468 		ni_jbus_kstat->pcr_reg = HV_NIAGARA_JBUS_CTL;
469 		ni_create_name_kstat("jbus", ni_jbus_kstat,
470 		    niagara_jbus_events);
471 		ni_jbus_kstat->cntr_ksp = ni_create_cntr_kstat("jbus", 0,
472 		    ni_cntr_kstat_update, ni_jbus_kstat);
473 	}
474 #endif
475 }
476 
477 void
478 niagara_kstat_fini()
479 {
480 	int i;
481 
482 #ifdef DEBUG
483 	if (ni_perf_debug)
484 		printf("ni_kstat_fini called\n");
485 #endif
486 
487 	for (i = 0; i < NIAGARA_DRAM_BANKS; i++) {
488 		if (ni_dram_kstats[i] != NULL) {
489 			ni_delete_name_kstat(ni_dram_kstats[i]);
490 			if (ni_dram_kstats[i]->cntr_ksp != NULL)
491 				kstat_delete(ni_dram_kstats[i]->cntr_ksp);
492 			kmem_free(ni_dram_kstats[i], sizeof (ni_ksinfo_t));
493 			ni_dram_kstats[i] = NULL;
494 		}
495 	}
496 
497 #if defined(VFALLS_IMPL)
498 	for (i = 0; i < ZAMBEZI_LPU_COUNTERS; i++) {
499 		if (zam_lpu_kstats[i] != NULL) {
500 			ni_delete_name_kstat(zam_lpu_kstats[i]);
501 			if (zam_lpu_kstats[i]->cntr_ksp != NULL)
502 				kstat_delete(zam_lpu_kstats[i]->cntr_ksp);
503 			kmem_free(zam_lpu_kstats[i], sizeof (ni_ksinfo_t));
504 			zam_lpu_kstats[i] = NULL;
505 		}
506 	}
507 
508 	for (i = 0; i < ZAMBEZI_GPD_COUNTERS; i++) {
509 		if (zam_gpd_kstats[i] != NULL) {
510 			ni_delete_name_kstat(zam_gpd_kstats[i]);
511 			if (zam_gpd_kstats[i]->cntr_ksp != NULL)
512 				kstat_delete(zam_gpd_kstats[i]->cntr_ksp);
513 			kmem_free(zam_gpd_kstats[i], sizeof (ni_ksinfo_t));
514 			zam_gpd_kstats[i] = NULL;
515 		}
516 	}
517 
518 	for (i = 0; i < ZAMBEZI_ASU_COUNTERS; i++) {
519 		if (zam_asu_kstats[i] != NULL) {
520 			ni_delete_name_kstat(zam_asu_kstats[i]);
521 			if (zam_asu_kstats[i]->cntr_ksp != NULL)
522 				kstat_delete(zam_asu_kstats[i]->cntr_ksp);
523 			kmem_free(zam_asu_kstats[i], sizeof (ni_ksinfo_t));
524 			zam_asu_kstats[i] = NULL;
525 		}
526 	}
527 #endif
528 
529 #if defined(NIAGARA_IMPL)
530 	if (ni_jbus_kstat != NULL) {
531 		ni_delete_name_kstat(ni_jbus_kstat);
532 		if (ni_jbus_kstat->cntr_ksp != NULL)
533 			kstat_delete(ni_jbus_kstat->cntr_ksp);
534 		kmem_free(ni_jbus_kstat, sizeof (ni_ksinfo_t));
535 		ni_jbus_kstat = NULL;
536 	}
537 #endif
538 }
539 
540 static void
541 ni_create_name_kstat(char *name, ni_ksinfo_t *pp, ni_kev_mask_t *ev)
542 {
543 	int	i;
544 
545 #ifdef DEBUG
546 	if (ni_perf_debug > 1)
547 		printf("ni_create_name_kstat: name: %s\n", name);
548 #endif
549 	for (i = 0; i < NUM_OF_PICS; i++) {
550 		pp->pic_name_ksp[i] = ni_create_picN_kstat(name,
551 		    i, pp->pic_sel_shift[i], pp->pic_no_evs, ev);
552 
553 		if (pp->pic_name_ksp[i] == NULL) {
554 			cmn_err(CE_WARN, "%s: unable to create name kstat",
555 			    cpu_module_name);
556 		}
557 	}
558 }
559 
560 static void
561 ni_delete_name_kstat(ni_ksinfo_t *pp)
562 {
563 	int	i;
564 
565 	if (pp != NULL) {
566 		for (i = 0; i < NUM_OF_PICS; i++) {
567 			if (pp->pic_name_ksp[i] != NULL)
568 				kstat_delete(pp->pic_name_ksp[i]);
569 		}
570 	}
571 }
572 
573 /*
574  * Create the picN kstat. Returns a pointer to the
575  * kstat which the driver must store to allow it
576  * to be deleted when necessary.
577  */
578 static kstat_t *
579 ni_create_picN_kstat(char *mod_name, int pic, int pic_sel_shift,
580 	int num_ev, ni_kev_mask_t *ev_array)
581 {
582 	struct kstat_named *pic_named_data;
583 	int	inst = 0;
584 	int	event;
585 	char	pic_name[30];
586 	kstat_t	*picN_ksp = NULL;
587 
588 	(void) sprintf(pic_name, "pic%d", pic);
589 	if ((picN_ksp = kstat_create(mod_name, inst, pic_name,
590 	    "bus", KSTAT_TYPE_NAMED, num_ev, NULL)) == NULL) {
591 		cmn_err(CE_WARN, "%s %s : kstat create failed",
592 		    mod_name, pic_name);
593 
594 		/*
595 		 * It is up to the calling function to delete any kstats
596 		 * that may have been created already. We just
597 		 * return NULL to indicate an error has occured.
598 		 */
599 		return (NULL);
600 	}
601 
602 	pic_named_data = (struct kstat_named *)
603 	    picN_ksp->ks_data;
604 
605 	/*
606 	 * Write event names and their associated pcr masks. The
607 	 * last entry in the array (clear_pic) is added seperately
608 	 * below as the pic value must be inverted.
609 	 */
610 	for (event = 0; event < num_ev - 1; event++) {
611 		pic_named_data[event].value.ui64 =
612 		    (ev_array[event].pcr_mask << pic_sel_shift);
613 
614 		kstat_named_init(&pic_named_data[event],
615 		    ev_array[event].event_name,
616 		    KSTAT_DATA_UINT64);
617 	}
618 
619 	/*
620 	 * add the clear_pic entry.
621 	 */
622 	pic_named_data[event].value.ui64 =
623 	    (uint64_t)~(ev_array[event].pcr_mask << pic_sel_shift);
624 
625 	kstat_named_init(&pic_named_data[event], ev_array[event].event_name,
626 	    KSTAT_DATA_UINT64);
627 
628 	kstat_install(picN_ksp);
629 
630 	return (picN_ksp);
631 }
632 
633 /*
634  * Create the "counters" kstat.
635  */
636 static kstat_t *
637 ni_create_cntr_kstat(char *name, int instance, int (*update)(kstat_t *, int),
638 	void *ksinfop)
639 {
640 	struct kstat	*counters_ksp;
641 	struct kstat_named	*counters_named_data;
642 	char		pic_str[10];
643 	int		i;
644 	int		num_pics = NUM_OF_PICS;
645 
646 #ifdef DEBUG
647 	if (ni_perf_debug > 1)
648 		printf("ni_create_cntr_kstat: name: %s instance: %d\n",
649 		    name, instance);
650 #endif
651 
652 	/*
653 	 * Size of kstat is num_pics + 1 as it
654 	 * also contains the %pcr
655 	 */
656 	if ((counters_ksp = kstat_create(name, instance, "counters", "bus",
657 	    KSTAT_TYPE_NAMED, num_pics + 1, KSTAT_FLAG_WRITABLE)) == NULL) {
658 		cmn_err(CE_WARN,
659 		    "%s: kstat_create for %s%d failed", cpu_module_name,
660 		    name, instance);
661 		return (NULL);
662 	}
663 
664 	counters_named_data = (struct kstat_named *)(counters_ksp->ks_data);
665 
666 	/*
667 	 * Iinitialize the named kstats
668 	 */
669 	kstat_named_init(&counters_named_data[0], "pcr", KSTAT_DATA_UINT64);
670 
671 	for (i = 0; i < num_pics; i++) {
672 		(void) sprintf(pic_str, "pic%d", i);
673 
674 		kstat_named_init(&counters_named_data[i+1], pic_str,
675 		    KSTAT_DATA_UINT64);
676 	}
677 
678 	/*
679 	 * Store the register offset's in the kstat's
680 	 * private field so that they are available
681 	 * to the update function.
682 	 */
683 	counters_ksp->ks_private = (void *)ksinfop;
684 	counters_ksp->ks_update = update;
685 
686 	kstat_install(counters_ksp);
687 
688 	return (counters_ksp);
689 }
690 
691 #if defined(VFALLS_IMPL)
692 /*
693  * zambezi kstat update function. Handles reads/writes
694  * from/to kstat.
695  */
696 static int
697 zam_cntr_kstat_update(kstat_t *ksp, int rw)
698 {
699 	struct kstat_named	*data_p;
700 	ni_ksinfo_t	*ksinfop = ksp->ks_private;
701 	uint64_t	pic0, pic1, pcr;
702 	int		stat = 0;
703 	uint64_t	pic0_stat = 0, pic1_stat = 0, pcr_stat = 0;
704 
705 	data_p = (struct kstat_named *)ksp->ks_data;
706 
707 	if (rw == KSTAT_WRITE) {
708 #ifdef DEBUG
709 		if (ni_perf_debug)
710 			printf("zam_cntr_kstat_update: wr pcr-%d: %lx\n",
711 			    ksinfop->pcr_reg, data_p[0].value.ui64);
712 #endif
713 		if (hv_niagara_setperf(ksinfop->pcr_reg, data_p[0].value.ui64))
714 			stat = EACCES;
715 	} else {
716 		do {
717 			pic0_stat = hv_niagara_getperf(ksinfop->pic_reg[0],
718 			    &pic0);
719 		} while (pic0_stat == H_EWOULDBLOCK);
720 		do {
721 			pic1_stat = hv_niagara_getperf(ksinfop->pic_reg[1],
722 			    &pic1);
723 		} while (pic1_stat == H_EWOULDBLOCK);
724 		do {
725 			pcr_stat = hv_niagara_getperf(ksinfop->pcr_reg,
726 			    &pcr);
727 		} while (pcr_stat == H_EWOULDBLOCK);
728 		if (pic0_stat != 0 || pic1_stat != 0 || pcr_stat != 0)
729 			stat = EACCES;
730 		else {
731 			data_p[0].value.ui64 = pcr;
732 			data_p[1].value.ui64 = pic0;
733 			data_p[2].value.ui64 = pic1;
734 		}
735 #ifdef DEBUG
736 		if (ni_perf_debug)
737 			printf("zam_cntr_kstat_update: rd pcr%d: %lx  "
738 			    "pic0: %16lx pic1: %16lx\n",
739 			    ksinfop->pcr_reg, pcr,
740 			    data_p[1].value.ui64, data_p[2].value.ui64);
741 #endif
742 	}
743 
744 	return (stat);
745 }
746 #endif
747 
748 /*
749  * kstat update function. Handles reads/writes
750  * from/to kstat.
751  */
752 static int
753 ni_cntr_kstat_update(kstat_t *ksp, int rw)
754 {
755 	struct kstat_named	*data_p;
756 	ni_ksinfo_t	*ksinfop = ksp->ks_private;
757 	uint64_t	pic, pcr;
758 	int		stat = 0;
759 	uint32_t	pic0, pic1;
760 
761 	data_p = (struct kstat_named *)ksp->ks_data;
762 
763 	if (rw == KSTAT_WRITE) {
764 #ifdef DEBUG
765 		if (ni_perf_debug)
766 			printf("ni_cntr_kstat_update: wr pcr-%d: %lx\n",
767 			    ksinfop->pcr_reg, data_p[0].value.ui64);
768 #endif
769 		if (hv_niagara_setperf(ksinfop->pcr_reg, data_p[0].value.ui64))
770 			stat = EACCES;
771 	} else {
772 		if (hv_niagara_getperf(ksinfop->pic_reg[0], &pic) != 0 ||
773 		    hv_niagara_getperf(ksinfop->pcr_reg, &pcr) != 0)
774 			stat = EACCES;
775 		else {
776 
777 			data_p[0].value.ui64 = pcr;
778 
779 			/*
780 			 * Generate a 32-bit PIC0 value by detecting overflow
781 			 */
782 			pic0 = (uint32_t)((pic >> ksinfop->pic_shift[0]) &
783 			    ksinfop->pic_mask[0]);
784 			if (pic0 < ksinfop->pic_last_val[0])
785 				ksinfop->pic_overflow[0]++;
786 			ksinfop->pic_last_val[0] = pic0;
787 			pic0 += (ksinfop->pic_overflow[0] & 1) << 31;
788 			data_p[1].value.ui64 = (uint64_t)pic0;
789 
790 			/*
791 			 * Generate a 32-bit PIC1 value by detecting overflow
792 			 */
793 			pic1 = (uint32_t)((pic >> ksinfop->pic_shift[1]) &
794 			    ksinfop->pic_mask[1]);
795 			if (pic1 < ksinfop->pic_last_val[1])
796 				ksinfop->pic_overflow[1]++;
797 			ksinfop->pic_last_val[1] = pic1;
798 			pic1 += (ksinfop->pic_overflow[1] & 1) << 31;
799 			data_p[2].value.ui64 = (uint64_t)pic1;
800 		}
801 #ifdef DEBUG
802 		if (ni_perf_debug)
803 			printf("ni_cntr_kstat_update: rd pcr%d: %lx  "
804 			    "pic%d: %16lx pic0: %8lx pic1: %8lx\n",
805 			    ksinfop->pcr_reg, pcr, ksinfop->pic_reg[0], pic,
806 			    data_p[1].value.ui64, data_p[2].value.ui64);
807 #endif
808 	}
809 	return (stat);
810 }
811