1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26/*
27 * This file contains preset event names from the Performance Application
28 * Programming Interface v3.5 which included the following notice:
29 *
30 *                             Copyright (c) 2005,6
31 *                           Innovative Computing Labs
32 *                         Computer Science Department,
33 *                            University of Tennessee,
34 *                                 Knoxville, TN.
35 *                              All Rights Reserved.
36 *
37 *
38 * Redistribution and use in source and binary forms, with or without
39 * modification, are permitted provided that the following conditions are met:
40 *
41 *    * Redistributions of source code must retain the above copyright notice,
42 *      this list of conditions and the following disclaimer.
43 *    * Redistributions in binary form must reproduce the above copyright
44 *      notice, this list of conditions and the following disclaimer in the
45 *      documentation and/or other materials provided with the distribution.
46 *    * Neither the name of the University of Tennessee nor the names of its
47 *      contributors may be used to endorse or promote products derived from
48 *      this software without specific prior written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
51 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
54 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
60 * POSSIBILITY OF SUCH DAMAGE.
61 *
62 *
63 * This open source software license conforms to the BSD License template.
64 */
65
66/*
67 * SPARC64 VI & VII Performance Counter Backend
68 */
69
70#include <sys/cpuvar.h>
71#include <sys/systm.h>
72#include <sys/cmn_err.h>
73#include <sys/cpc_impl.h>
74#include <sys/cpc_pcbe.h>
75#include <sys/modctl.h>
76#include <sys/machsystm.h>
77#include <sys/sdt.h>
78#include <sys/cpu_impl.h>
79
80static int opl_pcbe_init(void);
81static uint_t opl_pcbe_ncounters(void);
82static const char *opl_pcbe_impl_name(void);
83static const char *opl_pcbe_cpuref(void);
84static char *opl_pcbe_list_events(uint_t picnum);
85static char *opl_pcbe_list_attrs(void);
86static uint64_t opl_pcbe_event_coverage(char *event);
87static uint64_t opl_pcbe_overflow_bitmap(void);
88static int opl_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
89    uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
90    void *token);
91static void opl_pcbe_program(void *token);
92static void opl_pcbe_allstop(void);
93static void opl_pcbe_sample(void *token);
94static void opl_pcbe_free(void *config);
95
96extern void ultra_setpcr(uint64_t);
97extern uint64_t ultra_getpcr(void);
98extern void ultra_setpic(uint64_t);
99extern uint64_t ultra_getpic(void);
100extern uint64_t ultra_gettick(void);
101
102pcbe_ops_t opl_pcbe_ops = {
103	PCBE_VER_1,
104	CPC_CAP_OVERFLOW_INTERRUPT,
105	opl_pcbe_ncounters,
106	opl_pcbe_impl_name,
107	opl_pcbe_cpuref,
108	opl_pcbe_list_events,
109	opl_pcbe_list_attrs,
110	opl_pcbe_event_coverage,
111	opl_pcbe_overflow_bitmap,
112	opl_pcbe_configure,
113	opl_pcbe_program,
114	opl_pcbe_allstop,
115	opl_pcbe_sample,
116	opl_pcbe_free
117};
118
119typedef struct _opl_pcbe_config {
120	uint8_t		opl_picno;	/* From 0 to 7 */
121	uint32_t	opl_bits;	/* %pcr event code unshifted */
122	uint32_t	opl_flags;	/* user/system/priv */
123	uint32_t	opl_pic;	/* unshifted raw %pic value */
124} opl_pcbe_config_t;
125
126struct nametable {
127	const uint8_t	bits;
128	const char	*name;
129};
130
131typedef struct _opl_generic_event {
132	char *name;
133	char *event;
134} opl_generic_event_t;
135
136/*
137 * Performance Control Register (PCR)
138 *
139 * +----------+-----+-----+------+----+
140 * |      0   | OVF |  0  | OVR0 | 0  |
141 * +----------+-----+-----+------+----+
142 * 63     48  47:32  31:27   26    25
143 *
144 * +----+----+--- -+----+-----+---+-----+-----+----+----+----+
145 * | NC |  0 | SC  | 0  | SU  | 0 | SL  |ULRO | UT | ST |PRIV|
146 * +----+----+-----+----+-----+---+-----+-----+----+----+----+
147 * 24:22  21  20:18  17  16:11 10  9:4     3    2    1    0
148 *
149 * ULRO and OVRO bits should be on upon accessing pcr unless
150 * those fields need to be updated.
151 * Turn off these bits when updating SU/SL or OVF field
152 * (during initialization, etc.).
153 *
154 *
155 * Performance Instrumentation Counter (PIC)
156 * Four PICs are implemented in SPARC64 VI and VII,
157 * each PIC is accessed using PCR.SC as a select field.
158 *
159 * +------------------------+--------------------------+
160 * |         PICU	    |		PICL	       |
161 * +------------------------+--------------------------+
162 *  63			 32  31			      0
163 */
164
165#define	PIC_MASK (((uint64_t)1 << 32) - 1)
166
167#define	SPARC64_VI_PCR_PRIVPIC  UINT64_C(0)
168
169#define	CPC_SPARC64_VI_PCR_SYS_SHIFT	1
170#define	CPC_SPARC64_VI_PCR_USR_SHIFT	2
171
172#define	CPC_SPARC64_VI_PCR_PICL_SHIFT	4
173#define	CPC_SPARC64_VI_PCR_PICU_SHIFT	11
174#define	CPC_SPARC64_VI_PCR_PIC_MASK	UINT64_C(0x3F)
175
176#define	CPC_SPARC64_VI_NPIC		8
177
178#define	CPC_SPARC64_VI_PCR_ULRO_SHIFT	3
179#define	CPC_SPARC64_VI_PCR_SC_SHIFT	18
180#define	CPC_SPARC64_VI_PCR_SC_MASK	UINT64_C(0x7)
181#define	CPC_SPARC64_VI_PCR_NC_SHIFT	22
182#define	CPC_SPARC64_VI_PCR_NC_MASK	UINT64_C(0x7)
183#define	CPC_SPARC64_VI_PCR_OVRO_SHIFT	26
184#define	CPC_SPARC64_VI_PCR_OVF_SHIFT	32
185#define	CPC_SPARC64_VI_PCR_OVF_MASK	UINT64_C(0xffff)
186
187#define	SPARC64_VI_PCR_SYS	(UINT64_C(1) << CPC_SPARC64_VI_PCR_SYS_SHIFT)
188#define	SPARC64_VI_PCR_USR	(UINT64_C(1) << CPC_SPARC64_VI_PCR_USR_SHIFT)
189#define	SPARC64_VI_PCR_ULRO	(UINT64_C(1) << CPC_SPARC64_VI_PCR_ULRO_SHIFT)
190#define	SPARC64_VI_PCR_OVRO	(UINT64_C(1) << CPC_SPARC64_VI_PCR_OVRO_SHIFT)
191#define	SPARC64_VI_PCR_OVF	(CPC_SPARC64_VI_PCR_OVF_MASK << \
192					CPC_SPARC64_VI_PCR_OVF_SHIFT)
193
194#define	SPARC64_VI_NUM_PIC_PAIRS	4
195
196#define	SPARC64_VI_PCR_SEL_PIC(pcr, picno) {				\
197	pcr &= ~((CPC_SPARC64_VI_PCR_SC_MASK				\
198		<< CPC_SPARC64_VI_PCR_SC_SHIFT));			\
199									\
200	pcr |= (((picno) & CPC_SPARC64_VI_PCR_SC_MASK)			\
201		<< CPC_SPARC64_VI_PCR_SC_SHIFT);			\
202}
203
204#define	SPARC64_VI_PCR_SEL_EVENT(pcr, sl, su) {				\
205	pcr &= ~((CPC_SPARC64_VI_PCR_PIC_MASK				\
206		<< CPC_SPARC64_VI_PCR_PICL_SHIFT)			\
207	    | (CPC_SPARC64_VI_PCR_PIC_MASK				\
208		<< CPC_SPARC64_VI_PCR_PICU_SHIFT));			\
209									\
210	pcr |= (((sl) & CPC_SPARC64_VI_PCR_PIC_MASK)			\
211		<< CPC_SPARC64_VI_PCR_PICL_SHIFT);			\
212	pcr |= (((su) & CPC_SPARC64_VI_PCR_PIC_MASK)			\
213		<< CPC_SPARC64_VI_PCR_PICU_SHIFT);			\
214}
215
216#define	SPARC64_VI_CHK_OVF(pcr, picno)					\
217	((pcr) & (UINT64_C(1) << (CPC_SPARC64_VI_PCR_OVF_SHIFT + picno)))
218
219#define	SPARC64_VI_CLR_OVF(pcr, picno) {				\
220	pcr &= ~(UINT64_C(1) << (CPC_SPARC64_VI_PCR_OVF_SHIFT + picno)); \
221}
222
223#define	NT_END 0xFF
224#define	CPC_GEN_END { NULL, NULL }
225
226static const uint64_t   allstopped = SPARC64_VI_PCR_PRIVPIC |
227	SPARC64_VI_PCR_ULRO | SPARC64_VI_PCR_OVRO;
228
229#define	SPARC64_VI_EVENTS_comm_0		\
230	{0x0,	"cycle_counts"},		\
231	{0x1,	"instruction_counts"}
232
233#define	SPARC64_VI_EVENTS_comm_1		\
234	{0x5,	"op_stv_wait"},			\
235	{0x8,	"load_store_instructions"},	\
236	{0x9,	"branch_instructions"},		\
237	{0xa,	"floating_instructions"},	\
238	{0xb,	"impdep2_instructions"},	\
239	{0xc,	"prefetch_instructions"}
240
241#define	SPARC64_VI_EVENTS_comm_2		\
242	{0x1a,	"active_cycle_count"}
243
244static const struct nametable SPARC64_VI_names_l0[] = {
245	SPARC64_VI_EVENTS_comm_0,
246	{0x2,	"only_this_thread_active"},
247	{0x3,	"w_cse_window_empty"},
248	{0x4,	"w_op_stv_wait_nc_pend"},
249	SPARC64_VI_EVENTS_comm_1,
250	{0x12,	"flush_rs"},
251	{0x13,	"2iid_use"},
252	{0x15,	"toq_rsbr_phantom"},
253	{0x16,	"trap_int_vector"},
254	{0x18,	"ts_by_sxmiss"},
255	{0x18,	"both_threads_active"},
256	SPARC64_VI_EVENTS_comm_2,
257	{0x1d,	"op_stv_wait_sxmiss"},
258	{0x1e,	"eu_comp_wait"},
259	{0x23,	"op_l1_thrashing"},
260	{0x24,	"swpf_fail_all"},
261	{0x30,	"sx_miss_wait_pf"},
262	{0x31,	"jbus_cpi_count"},
263	{0x36,	"jbus_reqbus1_busy"},
264	{NT_END, ""}
265};
266
267static const struct nametable SPARC64_VI_names_u0[] = {
268	SPARC64_VI_EVENTS_comm_0,
269	{0x2,	"instruction_flow_counts"},
270	{0x3,	"iwr_empty"},
271	SPARC64_VI_EVENTS_comm_1,
272	{0x12,	"rs1"},
273	{0x13,	"1iid_use"},
274	{0x16,	"trap_all"},
275	{0x18,	"thread_switch_all"},
276	{0x18,	"only_this_thread_active"},
277	SPARC64_VI_EVENTS_comm_2,
278	{0x1b,	"rsf_pmmi"},
279	{0x1d,	"act_thread_suspend"},
280	{0x1e,	"cse_window_empty"},
281	{0x1f,	"inh_cmit_gpr_2write"},
282	{0x23,	"if_l1_thrashing"},
283	{0x24,	"swpf_success_all"},
284	{0x30,	"sx_miss_wait_dm"},
285	{0x31,	"jbus_bi_count"},
286	{0x34,	"lost_softpf_pfp_full"},
287	{0x36,	"jbus_reqbus0_busy"},
288	{NT_END, ""}
289};
290
291static const struct nametable SPARC64_VI_names_l1[] = {
292	SPARC64_VI_EVENTS_comm_0,
293	{0x2,	"single_mode_instructions"},
294	{0x3,	"w_branch_comp_wait"},
295	{0x4,	"w_op_stv_wait_sxmiss_ex"},
296	SPARC64_VI_EVENTS_comm_1,
297	{0x13,	"4iid_use"},
298	{0x15,	"flush_rs"},
299	{0x16,	"trap_spill"},
300	{0x18,	"ts_by_timer"},
301	SPARC64_VI_EVENTS_comm_2,
302	{0x1b,	"0iid_use"},
303	{0x1d,	"op_stv_wait_nc_pend"},
304	{0x1e,	"0endop"},
305	{0x20,	"write_op_uTLB"},
306	{0x30,	"sx_miss_count_pf"},
307	{0x31,	"jbus_cpd_count"},
308	{0x32,	"snres_64"},
309	{0x36,	"jbus_reqbus3_busy"},
310	{NT_END, ""}
311};
312
313static const struct nametable SPARC64_VI_names_u1[] = {
314	SPARC64_VI_EVENTS_comm_0,
315	{0x2,	"single_mode_cycle_counts"},
316	{0x3,	"w_eu_comp_wait"},
317	{0x4,	"w_op_stv_wait_sxmiss"},
318	SPARC64_VI_EVENTS_comm_1,
319	{0x13,	"3iid_use"},
320	{0x16,	"trap_int_level"},
321	{0x18,	"ts_by_data_arrive"},
322	{0x18,	"both_threads_empty"},
323	SPARC64_VI_EVENTS_comm_2,
324	{0x1b,	"op_stv_wait_nc_pend"},
325	{0x1d,	"op_stv_wait_sxmiss_ex"},
326	{0x1e,	"branch_comp_wait"},
327	{0x20,	"write_if_uTLB"},
328	{0x30,	"sx_miss_count_dm"},
329	{0x31,	"jbus_cpb_count"},
330	{0x32,	"snres_256"},
331	{0x34,	"lost_softpf_by_abort"},
332	{0x36,	"jbus_reqbus2_busy"},
333	{NT_END, ""}
334};
335
336static const struct nametable SPARC64_VI_names_l2[] = {
337	SPARC64_VI_EVENTS_comm_0,
338	{0x2,	"d_move_wait"},
339	{0x3,	"w_op_stv_wait"},
340	{0x4,	"w_fl_comp_wait"},
341	SPARC64_VI_EVENTS_comm_1,
342	{0x13,	"sync_intlk"},
343	{0x16,	"trap_trap_inst"},
344	{0x18,	"ts_by_if"},
345	SPARC64_VI_EVENTS_comm_2,
346	{0x1e,	"fl_comp_wait"},
347	{0x20,	"op_r_iu_req_mi_go"},
348	{0x30,	"sx_read_count_pf"},
349	{0x31,	"jbus_odrbus_busy"},
350	{0x33,	"sx_miss_count_dm_if"},
351	{0x36,	"jbus_odrbus1_busy"},
352	{NT_END, ""}
353};
354
355static const struct nametable SPARC64_VI_names_u2[] = {
356	SPARC64_VI_EVENTS_comm_0,
357	{0x2,	"instruction_flow_counts"},
358	{0x3,	"iwr_empty"},
359	SPARC64_VI_EVENTS_comm_1,
360	{0x16,	"trap_fill"},
361	{0x18,	"ts_by_intr"},
362	SPARC64_VI_EVENTS_comm_2,
363	{0x1b,	"flush_rs"},
364	{0x1d,	"cse_window_empty_sp_full"},
365	{0x1e,	"op_stv_wait_ex"},
366	{0x1f,	"3endop"},
367	{0x20,	"if_r_iu_req_mi_go"},
368	{0x24,	"swpf_lbs_hit"},
369	{0x30,	"sx_read_count_dm"},
370	{0x31,	"jbus_reqbus_busy"},
371	{0x33,	"sx_btc_count"},
372	{0x36,	"jbus_odrbus0_busy"},
373	{NT_END, ""}
374};
375
376static const struct nametable SPARC64_VI_names_l3[] = {
377	SPARC64_VI_EVENTS_comm_0,
378	{0x2,	"xma_inst"},
379	{0x3,	"w_0endop"},
380	{0x4,	"w_op_stv_wait_ex"},
381	SPARC64_VI_EVENTS_comm_1,
382	{0x16,	"trap_DMMU_miss"},
383	{0x18,	"ts_by_suspend"},
384	{0x19,	"ts_by_other"},
385	SPARC64_VI_EVENTS_comm_2,
386	{0x1b,	"decall_intlk"},
387	{0x1e,	"2endop"},
388	{0x1f,	"op_stv_wait_sxmiss"},
389	{0x20,	"op_wait_all"},
390	{0x30,	"dvp_count_pf"},
391	{0x33,	"sx_miss_count_dm_opex"},
392	{0x36,	"jbus_odrbus3_busy"},
393	{NT_END, ""}
394};
395
396static const struct nametable SPARC64_VI_names_u3[] = {
397	SPARC64_VI_EVENTS_comm_0,
398	{0x2,	"cse_priority_wait"},
399	{0x3,	"w_d_move"},
400	{0x4,	"w_cse_window_empty_sp_full"},
401	SPARC64_VI_EVENTS_comm_1,
402	{0x13,	"regwin_intlk"},
403	{0x15,	"rs1"},
404	{0x16,	"trap_IMMU_miss"},
405	SPARC64_VI_EVENTS_comm_2,
406	{0x1d,	"both_threads_suspended"},
407	{0x1e,	"1endop"},
408	{0x1f,	"op_stv_wait_sxmiss_ex"},
409	{0x20,	"if_wait_all"},
410	{0x30,	"dvp_count_dm"},
411	{0x33,	"sx_miss_count_dm_opsh"},
412	{0x36,	"jbus_odrbus2_busy"},
413	{NT_END, ""}
414};
415
416#undef	SPARC64_VI_EVENTS_comm_0
417#undef	SPARC64_VI_EVENTS_comm_1
418#undef	SPARC64_VI_EVENTS_comm_2
419
420static const struct nametable *SPARC64_VI_names[CPC_SPARC64_VI_NPIC] = {
421	SPARC64_VI_names_l0,
422	SPARC64_VI_names_u0,
423	SPARC64_VI_names_l1,
424	SPARC64_VI_names_u1,
425	SPARC64_VI_names_l2,
426	SPARC64_VI_names_u2,
427	SPARC64_VI_names_l3,
428	SPARC64_VI_names_u3
429};
430
431opl_pcbe_config_t nullpic[CPC_SPARC64_VI_NPIC] = {
432	{0, 0x3f, 0, 0},
433	{1, 0x3f, 0, 0},
434	{2, 0x3f, 0, 0},
435	{3, 0x3f, 0, 0},
436	{4, 0x3f, 0, 0},
437	{5, 0x3f, 0, 0},
438	{6, 0x3f, 0, 0},
439	{7, 0x3f, 0, 0}
440};
441
442#define	SPARC64_VI_GENERIC_EVENTS_comm				\
443	{ "PAPI_tot_cyc",	"cycle_counts" },		\
444	{ "PAPI_tot_ins",	"instruction_counts" },		\
445	{ "PAPI_br_tkn",	"branch_instructions" },	\
446	{ "PAPI_fp_ops",	"floating_instructions" },	\
447	{ "PAPI_fma_ins",	"impdep2_instructions" }
448
449static const opl_generic_event_t SPARC64_VI_generic_names_l0[] = {
450	SPARC64_VI_GENERIC_EVENTS_comm,
451	CPC_GEN_END
452};
453
454static const opl_generic_event_t SPARC64_VI_generic_names_u0[] = {
455	SPARC64_VI_GENERIC_EVENTS_comm,
456	CPC_GEN_END
457};
458
459static const opl_generic_event_t SPARC64_VI_generic_names_l1[] = {
460	SPARC64_VI_GENERIC_EVENTS_comm,
461	CPC_GEN_END
462};
463
464static const opl_generic_event_t SPARC64_VI_generic_names_u1[] = {
465	SPARC64_VI_GENERIC_EVENTS_comm,
466	CPC_GEN_END
467};
468
469static const opl_generic_event_t SPARC64_VI_generic_names_l2[] = {
470	SPARC64_VI_GENERIC_EVENTS_comm,
471	{ "PAPI_l1_dcm",	"op_r_iu_req_mi_go" },
472	CPC_GEN_END
473};
474
475static const opl_generic_event_t SPARC64_VI_generic_names_u2[] = {
476	SPARC64_VI_GENERIC_EVENTS_comm,
477	{ "PAPI_l1_icm",	"if_r_iu_req_mi_go" },
478	CPC_GEN_END
479};
480
481static const opl_generic_event_t SPARC64_VI_generic_names_l3[] = {
482	SPARC64_VI_GENERIC_EVENTS_comm,
483	{ "PAPI_tlb_dm",	"trap_DMMU_miss" },
484	CPC_GEN_END
485};
486
487static const opl_generic_event_t SPARC64_VI_generic_names_u3[] = {
488	SPARC64_VI_GENERIC_EVENTS_comm,
489	{ "PAPI_tlb_im",	"trap_IMMU_miss" },
490	CPC_GEN_END
491};
492
493static const opl_generic_event_t
494	*SPARC64_VI_generic_names[CPC_SPARC64_VI_NPIC] = {
495	SPARC64_VI_generic_names_l0,
496	SPARC64_VI_generic_names_u0,
497	SPARC64_VI_generic_names_l1,
498	SPARC64_VI_generic_names_u1,
499	SPARC64_VI_generic_names_l2,
500	SPARC64_VI_generic_names_u2,
501	SPARC64_VI_generic_names_l3,
502	SPARC64_VI_generic_names_u3
503};
504
505static const struct nametable **events;
506static const opl_generic_event_t **generic_events;
507static const char *opl_impl_name;
508static const char *opl_cpuref;
509static char *pic_events[CPC_SPARC64_VI_NPIC];
510
511static const char *sp_6_ref = "See the \"SPARC64 VI extensions\" and "
512	"\"SPARC64 VII extensions\" for descriptions of these events.";
513
514static int
515opl_pcbe_init(void)
516{
517	const struct nametable		*n;
518	const opl_generic_event_t	*gevp;
519	int				i;
520	size_t				size;
521
522	/*
523	 * Discover type of CPU
524	 *
525	 * Point nametable to that CPU's table
526	 */
527	switch (ULTRA_VER_IMPL(ultra_getver())) {
528	case OLYMPUS_C_IMPL:
529	case JUPITER_IMPL:
530		events = SPARC64_VI_names;
531		generic_events = SPARC64_VI_generic_names;
532		opl_impl_name = "SPARC64 VI & VII";
533		opl_cpuref = sp_6_ref;
534		break;
535	default:
536		return (-1);
537	}
538
539	/*
540	 * Initialize the list of events for each PIC.
541	 * Do two passes: one to compute the size necessary and another
542	 * to copy the strings. Need room for event, comma, and NULL terminator.
543	 */
544	for (i = 0; i < CPC_SPARC64_VI_NPIC; i++) {
545		size = 0;
546		for (n = events[i]; n->bits != NT_END; n++)
547			size += strlen(n->name) + 1;
548		for (gevp = generic_events[i]; gevp->name != NULL; gevp++)
549			size += strlen(gevp->name) + 1;
550		pic_events[i] = kmem_alloc(size + 1, KM_SLEEP);
551		*pic_events[i] = '\0';
552		for (n = events[i]; n->bits != NT_END; n++) {
553			(void) strcat(pic_events[i], n->name);
554			(void) strcat(pic_events[i], ",");
555		}
556		for (gevp = generic_events[i]; gevp->name != NULL; gevp++) {
557			(void) strcat(pic_events[i], gevp->name);
558			(void) strcat(pic_events[i], ",");
559		}
560
561		/*
562		 * Remove trailing comma.
563		 */
564		pic_events[i][size - 1] = '\0';
565	}
566
567	return (0);
568}
569
570static uint_t
571opl_pcbe_ncounters(void)
572{
573	return (CPC_SPARC64_VI_NPIC);
574}
575
576static const char *
577opl_pcbe_impl_name(void)
578{
579	return (opl_impl_name);
580}
581
582static const char *
583opl_pcbe_cpuref(void)
584{
585	return (opl_cpuref);
586}
587
588static char *
589opl_pcbe_list_events(uint_t picnum)
590{
591	ASSERT(picnum >= 0 && picnum < cpc_ncounters);
592
593	return (pic_events[picnum]);
594}
595
596static char *
597opl_pcbe_list_attrs(void)
598{
599	return ("");
600}
601
602static const opl_generic_event_t *
603find_generic_event(int regno, char *name)
604{
605	const opl_generic_event_t *gevp;
606
607	for (gevp = generic_events[regno]; gevp->name != NULL; gevp++)
608		if (strcmp(name, gevp->name) == 0)
609			return (gevp);
610
611	return (NULL);
612}
613
614static const struct nametable *
615find_event(int regno, char *name)
616{
617	const struct nametable *n;
618
619	n = events[regno];
620
621	for (; n->bits != NT_END; n++)
622		if (strcmp(name, n->name) == 0)
623			return (n);
624
625	return (NULL);
626}
627
628static uint64_t
629opl_pcbe_event_coverage(char *event)
630{
631	uint64_t bitmap = 0;
632
633	int	i;
634	for (i = 0; i < CPC_SPARC64_VI_NPIC; i++) {
635		if ((find_event(i, event) != NULL) ||
636		    (find_generic_event(i, event) != NULL))
637			bitmap |= (1 << i);
638	}
639
640	return (bitmap);
641}
642
643/*
644 * Check if counter overflow and clear it.
645 */
646static uint64_t
647opl_pcbe_overflow_bitmap(void)
648{
649	uint64_t	pcr;
650
651	pcr = ultra_getpcr();
652	DTRACE_PROBE1(sparc64__getpcr, uint64_t, pcr);
653
654	return ((pcr & SPARC64_VI_PCR_OVF) >> CPC_SPARC64_VI_PCR_OVF_SHIFT);
655}
656
657/*ARGSUSED*/
658static int
659opl_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
660    uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token)
661{
662	opl_pcbe_config_t		*conf;
663	const struct nametable		*n;
664	const opl_generic_event_t	*gevp;
665	opl_pcbe_config_t		*other_config;
666
667	/*
668	 * If we've been handed an existing configuration, we need only preset
669	 * the counter value.
670	 */
671	if (*data != NULL) {
672		conf = *data;
673		conf->opl_pic = (uint32_t)preset;
674		return (0);
675	}
676
677	if (picnum < 0 || picnum >= CPC_SPARC64_VI_NPIC)
678		return (CPC_INVALID_PICNUM);
679
680	if (nattrs != 0)
681		return (CPC_INVALID_ATTRIBUTE);
682
683	/*
684	 * Find other requests that will be programmed with this one, and ensure
685	 * the flags don't conflict.
686	 */
687	if (((other_config = kcpc_next_config(token, NULL, NULL)) != NULL) &&
688	    (other_config->opl_flags != flags))
689		return (CPC_CONFLICTING_REQS);
690
691	if ((n = find_event(picnum, event)) == NULL) {
692		if ((gevp = find_generic_event(picnum, event)) != NULL) {
693			n = find_event(picnum, gevp->event);
694			ASSERT(n != NULL);
695		} else {
696			return (CPC_INVALID_EVENT);
697		}
698	}
699
700	conf = kmem_alloc(sizeof (opl_pcbe_config_t), KM_SLEEP);
701
702	conf->opl_picno = picnum;
703	conf->opl_bits = (uint32_t)n->bits;
704	conf->opl_flags = flags;
705	conf->opl_pic = (uint32_t)preset;
706
707	*data = conf;
708	return (0);
709}
710
711static void
712opl_pcbe_program(void *token)
713{
714	opl_pcbe_config_t	*pic[CPC_SPARC64_VI_NPIC];
715	opl_pcbe_config_t	*firstconfig;
716	opl_pcbe_config_t	*tmp;
717	uint64_t		pcr;
718	uint64_t		curpic;
719	uint8_t			bitmap = 0;	/* for used pic config */
720	int			i;
721	opl_pcbe_config_t	dummypic[CPC_SPARC64_VI_NPIC];
722
723	/* Get next pic config */
724	firstconfig = tmp = kcpc_next_config(token, NULL, NULL);
725
726	while (tmp != NULL) {
727		ASSERT(tmp->opl_picno < CPC_SPARC64_VI_NPIC);
728		ASSERT(firstconfig->opl_flags == tmp->opl_flags);
729		pic[tmp->opl_picno] = tmp;
730		bitmap |= (uint8_t)(1 << tmp->opl_picno);
731		tmp = kcpc_next_config(token, tmp, NULL);
732	}
733	if (bitmap == 0)
734		panic("opl_pcbe: token %p has no configs", token);
735
736	/* Fill in unused pic config */
737	for (i = 0; i < CPC_SPARC64_VI_NPIC; i++) {
738		if (bitmap & (1 << i))
739			continue;
740
741		dummypic[i] = nullpic[i];
742		dummypic[i].opl_flags = firstconfig->opl_flags;
743		pic[i] = &dummypic[i];
744	}
745
746	/*
747	 * For each counter pair, initialize event settings and
748	 * counter values.
749	 */
750	ultra_setpcr(allstopped);
751	pcr = allstopped;
752	pcr &= ~SPARC64_VI_PCR_ULRO;
753	for (i = 0; i < SPARC64_VI_NUM_PIC_PAIRS; i++) {
754		SPARC64_VI_PCR_SEL_PIC(pcr, i);
755		SPARC64_VI_PCR_SEL_EVENT(pcr, pic[i*2]->opl_bits,
756		    pic[i*2 + 1]->opl_bits);
757
758		ultra_setpcr(pcr);
759		curpic = (uint64_t)(pic[i*2]->opl_pic |
760		    ((uint64_t)pic[i*2 + 1]->opl_pic << 32));
761		ultra_setpic(curpic);
762	}
763
764	/*
765	 * For each counter pair, enable the trace flags to start
766	 * counting. Re-read the counters to sample the counter value now
767	 * and use that as the baseline for future samples.
768	 */
769
770	/* Get PCR */
771	pcr = ultra_getpcr();
772	pcr |= SPARC64_VI_PCR_ULRO;
773	pcr &= ~(SPARC64_VI_PCR_OVRO | SPARC64_VI_PCR_OVF);
774
775	if (pic[0]->opl_flags & CPC_COUNT_USER)
776		pcr |= SPARC64_VI_PCR_USR;
777	if (pic[0]->opl_flags & CPC_COUNT_SYSTEM)
778		pcr |= SPARC64_VI_PCR_SYS;
779
780	/* Set counter values */
781
782	for (i = 0; i < SPARC64_VI_NUM_PIC_PAIRS; i++) {
783		SPARC64_VI_PCR_SEL_PIC(pcr, i);
784		SPARC64_VI_PCR_SEL_EVENT(pcr, pic[i*2]->opl_bits,
785		    pic[i*2 + 1]->opl_bits);
786
787		ultra_setpcr(pcr);
788		DTRACE_PROBE1(sparc64__setpcr, uint64_t, pcr);
789
790		curpic = ultra_getpic();
791		DTRACE_PROBE1(sparc64__newpic, uint64_t, curpic);
792		pic[i*2]->opl_pic = (uint32_t)(curpic & PIC_MASK);
793		pic[i*2 + 1]->opl_pic = (uint32_t)(curpic >> 32);
794	}
795	pcr |= SPARC64_VI_PCR_OVRO;
796	ultra_setpcr(pcr);
797}
798
799static void
800opl_pcbe_allstop(void)
801{
802	ultra_setpcr(allstopped);
803}
804
805
806static void
807opl_pcbe_sample(void *token)
808{
809	uint64_t		curpic;
810	uint64_t		pcr;
811	uint64_t		overflow;
812	int64_t			diff;
813	uint64_t		*pic_data[CPC_SPARC64_VI_NPIC];
814	uint64_t		*dtmp;
815	opl_pcbe_config_t	*pic[CPC_SPARC64_VI_NPIC];
816	opl_pcbe_config_t	*ctmp;
817	opl_pcbe_config_t	*firstconfig;
818	uint8_t			bitmap = 0;	/* for used pic config */
819	int			i;
820	opl_pcbe_config_t dummypic[CPC_SPARC64_VI_NPIC];
821	uint64_t dummypic_data[CPC_SPARC64_VI_NPIC];
822
823	/* Get next pic config */
824	firstconfig = ctmp = kcpc_next_config(token, NULL, &dtmp);
825
826	while (ctmp != NULL) {
827		ASSERT(ctmp->opl_picno < CPC_SPARC64_VI_NPIC);
828		ASSERT(firstconfig->opl_flags == ctmp->opl_flags);
829		pic[ctmp->opl_picno] = ctmp;
830		pic_data[ctmp->opl_picno] = dtmp;
831		bitmap |= (uint8_t)(1 << ctmp->opl_picno);
832		ctmp = kcpc_next_config(token, ctmp, &dtmp);
833	}
834	if (bitmap == 0)
835		panic("opl_pcbe: token %p has no configs", token);
836
837	/* Fill in unuse pic config */
838	for (i = 0; i < CPC_SPARC64_VI_NPIC; i++) {
839		if (bitmap & (1 << i))
840			continue;
841
842		dummypic[i] = nullpic[i];
843		dummypic[i].opl_flags = firstconfig->opl_flags;
844		pic[i] = &dummypic[i];
845
846		dummypic_data[i] = 0;
847		pic_data[i] = &dummypic_data[i];
848	}
849
850	pcr = ultra_getpcr();
851	pcr &= ~SPARC64_VI_PCR_OVRO;
852	pcr |= SPARC64_VI_PCR_ULRO;
853
854	for (i = 0; i < SPARC64_VI_NUM_PIC_PAIRS; i++) {
855		SPARC64_VI_PCR_SEL_PIC(pcr, i);
856		SPARC64_VI_PCR_SEL_EVENT(pcr, pic[i*2]->opl_bits,
857		    pic[i*2 + 1]->opl_bits);
858
859		ultra_setpcr(pcr);
860
861		curpic = ultra_getpic();
862		DTRACE_PROBE1(sparc64__getpic, unit64_t, curpic);
863
864		diff = (curpic & PIC_MASK) - (uint64_t)pic[i*2]->opl_pic;
865		overflow = SPARC64_VI_CHK_OVF(pcr, i*2);
866		if (overflow || (diff < 0)) {
867			SPARC64_VI_CLR_OVF(pcr, i*2);
868			ultra_setpcr(pcr);
869			diff += (1ll << 32);
870		}
871		*pic_data[i*2] += diff;
872
873		diff = (curpic >> 32) - (uint64_t)pic[i*2 + 1]->opl_pic;
874		overflow = SPARC64_VI_CHK_OVF(pcr, i*2 + 1);
875		if (overflow || (diff < 0)) {
876			SPARC64_VI_CLR_OVF(pcr, i*2 + 1);
877			ultra_setpcr(pcr);
878			diff += (1ll << 32);
879		}
880		*pic_data[i*2 + 1] += diff;
881
882		pic[i*2]->opl_pic = (uint32_t)(curpic & PIC_MASK);
883		pic[i*2 + 1]->opl_pic = (uint32_t)(curpic >> 32);
884	}
885	pcr = ultra_getpcr();
886	pcr |= SPARC64_VI_PCR_OVRO;
887	ultra_setpcr(pcr);
888}
889
890static void
891opl_pcbe_free(void *config)
892{
893	kmem_free(config, sizeof (opl_pcbe_config_t));
894}
895
896
897static struct modlpcbe modlpcbe = {
898	&mod_pcbeops,
899	"SPARC64 VI&VII Perf Cntrs",
900	&opl_pcbe_ops
901};
902
903static struct modlinkage modl = {
904	MODREV_1,
905	&modlpcbe,
906};
907
908int
909_init(void)
910{
911	if (opl_pcbe_init() != 0)
912		return (ENOTSUP);
913	return (mod_install(&modl));
914}
915
916int
917_fini(void)
918{
919	return (mod_remove(&modl));
920}
921
922int
923_info(struct modinfo *mi)
924{
925	return (mod_info(&modl, mi));
926}
927