core_pcbe.c revision e733453728bef62a6d9431aafae83669291d4c97
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26/*
27 * Performance Counter Back-End for Intel Family 6 Models 15, 23, 26 & 28
28 */
29
30#include <sys/cpuvar.h>
31#include <sys/param.h>
32#include <sys/cpc_impl.h>
33#include <sys/cpc_pcbe.h>
34#include <sys/modctl.h>
35#include <sys/inttypes.h>
36#include <sys/systm.h>
37#include <sys/cmn_err.h>
38#include <sys/x86_archext.h>
39#include <sys/sdt.h>
40#include <sys/archsystm.h>
41#include <sys/privregs.h>
42#include <sys/ddi.h>
43#include <sys/sunddi.h>
44#include <sys/cred.h>
45#include <sys/policy.h>
46
47static int core_pcbe_init(void);
48static uint_t core_pcbe_ncounters(void);
49static const char *core_pcbe_impl_name(void);
50static const char *core_pcbe_cpuref(void);
51static char *core_pcbe_list_events(uint_t picnum);
52static char *core_pcbe_list_attrs(void);
53static uint64_t core_pcbe_event_coverage(char *event);
54static uint64_t core_pcbe_overflow_bitmap(void);
55static int core_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
56    uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
57    void *token);
58static void core_pcbe_program(void *token);
59static void core_pcbe_allstop(void);
60static void core_pcbe_sample(void *token);
61static void core_pcbe_free(void *config);
62
63#define	FALSE	0
64#define	TRUE	1
65
66/* Counter Type */
67#define	CORE_GPC	0	/* General-Purpose Counter (GPC) */
68#define	CORE_FFC	1	/* Fixed-Function Counter (FFC) */
69
70/* MSR Addresses */
71#define	GPC_BASE_PMC		0x00c1	/* First GPC */
72#define	GPC_BASE_PES		0x0186	/* First GPC Event Select register */
73#define	FFC_BASE_PMC		0x0309	/* First FFC */
74#define	PERF_FIXED_CTR_CTRL	0x038d	/* Used to enable/disable FFCs */
75#define	PERF_GLOBAL_STATUS	0x038e	/* Overflow status register */
76#define	PERF_GLOBAL_CTRL	0x038f	/* Used to enable/disable counting */
77#define	PERF_GLOBAL_OVF_CTRL	0x0390	/* Used to clear overflow status */
78
79/*
80 * Processor Event Select register fields
81 */
82#define	CORE_USR	(1ULL << 16)	/* Count while not in ring 0 */
83#define	CORE_OS		(1ULL << 17)	/* Count while in ring 0 */
84#define	CORE_EDGE	(1ULL << 18)	/* Enable edge detection */
85#define	CORE_PC		(1ULL << 19)	/* Enable pin control */
86#define	CORE_INT	(1ULL << 20)	/* Enable interrupt on overflow */
87#define	CORE_EN		(1ULL << 22)	/* Enable counting */
88#define	CORE_INV	(1ULL << 23)	/* Invert the CMASK */
89#define	CORE_ANYTHR	(1ULL << 21)	/* Count event for any thread on core */
90
91#define	CORE_UMASK_SHIFT	8
92#define	CORE_UMASK_MASK		0xffu
93#define	CORE_CMASK_SHIFT	24
94#define	CORE_CMASK_MASK		0xffu
95
96/*
97 * Fixed-function counter attributes
98 */
99#define	CORE_FFC_OS_EN	(1ULL << 0)	/* Count while not in ring 0 */
100#define	CORE_FFC_USR_EN	(1ULL << 1)	/* Count while in ring 1 */
101#define	CORE_FFC_ANYTHR	(1ULL << 2)	/* Count event for any thread on core */
102#define	CORE_FFC_PMI	(1ULL << 3)	/* Enable interrupt on overflow */
103
104/*
105 * Number of bits for specifying each FFC's attributes in the control register
106 */
107#define	CORE_FFC_ATTR_SIZE	4
108
109/*
110 * CondChgd and OvfBuffer fields of global status and overflow control registers
111 */
112#define	CONDCHGD	(1ULL << 63)
113#define	OVFBUFFER	(1ULL << 62)
114#define	MASK_CONDCHGD_OVFBUFFER	(CONDCHGD | OVFBUFFER)
115
116#define	ALL_STOPPED	0ULL
117
118#define	BITMASK_XBITS(x)	((1ull << (x)) - 1ull)
119
120/*
121 * Only the lower 32-bits can be written to in the general-purpose
122 * counters.  The higher bits are extended from bit 31; all ones if
123 * bit 31 is one and all zeros otherwise.
124 *
125 * The fixed-function counters do not have this restriction.
126 */
127#define	BITS_EXTENDED_FROM_31	(BITMASK_XBITS(width_gpc) & ~BITMASK_XBITS(31))
128
129#define	WRMSR(msr, value)						\
130	wrmsr((msr), (value));						\
131	DTRACE_PROBE2(wrmsr, uint64_t, (msr), uint64_t, (value));
132
133#define	RDMSR(msr, value)						\
134	(value) = rdmsr((msr));						\
135	DTRACE_PROBE2(rdmsr, uint64_t, (msr), uint64_t, (value));
136
137typedef struct core_pcbe_config {
138	uint64_t	core_rawpic;
139	uint64_t	core_ctl;	/* Event Select bits */
140	uint64_t	core_pmc;	/* Counter register address */
141	uint64_t	core_pes;	/* Event Select register address */
142	uint_t		core_picno;
143	uint8_t		core_pictype;	/* CORE_GPC or CORE_FFC */
144} core_pcbe_config_t;
145
146pcbe_ops_t core_pcbe_ops = {
147	PCBE_VER_1,			/* pcbe_ver */
148	CPC_CAP_OVERFLOW_INTERRUPT | CPC_CAP_OVERFLOW_PRECISE,	/* pcbe_caps */
149	core_pcbe_ncounters,		/* pcbe_ncounters */
150	core_pcbe_impl_name,		/* pcbe_impl_name */
151	core_pcbe_cpuref,		/* pcbe_cpuref */
152	core_pcbe_list_events,		/* pcbe_list_events */
153	core_pcbe_list_attrs,		/* pcbe_list_attrs */
154	core_pcbe_event_coverage,	/* pcbe_event_coverage */
155	core_pcbe_overflow_bitmap,	/* pcbe_overflow_bitmap */
156	core_pcbe_configure,		/* pcbe_configure */
157	core_pcbe_program,		/* pcbe_program */
158	core_pcbe_allstop,		/* pcbe_allstop */
159	core_pcbe_sample,		/* pcbe_sample */
160	core_pcbe_free			/* pcbe_free */
161};
162
163struct nametable_fam6mod15_23 {
164	const char	*name;
165	uint64_t	restricted_bits;
166	uint8_t		event_num;
167};
168
169#define	NT_END	0xFF
170
171/*
172 * Counting an event for all cores or all bus agents requires cpc_cpu privileges
173 */
174#define	ALL_CORES	(1ULL << 15)
175#define	ALL_AGENTS	(1ULL << 13)
176
177/*
178 * The events listed in the following table can be counted on all
179 * general-purpose counters on processors that are of Family 6 Models 15 or 23
180 */
181static const struct nametable_fam6mod15_23 cmn_gpc_events_f6m15_23[] = {
182	/* Alphabetical order of event name */
183
184	{ "baclears",			0x0,	0xe6 },
185	{ "bogus_br",			0x0,	0xe4 },
186	{ "br_bac_missp_exec",		0x0,	0x8a },
187
188	{ "br_call_exec",		0x0,	0x92 },
189	{ "br_call_missp_exec",		0x0,	0x93 },
190	{ "br_cnd_exec",		0x0,	0x8b },
191
192	{ "br_cnd_missp_exec",		0x0,	0x8c },
193	{ "br_ind_call_exec",		0x0,	0x94 },
194	{ "br_ind_exec",		0x0,	0x8d },
195
196	{ "br_ind_missp_exec",		0x0,	0x8e },
197	{ "br_inst_decoded",		0x0,	0xe0 },
198	{ "br_inst_exec",		0x0,	0x88 },
199
200	{ "br_inst_retired",		0x0,	0xc4 },
201	{ "br_inst_retired_mispred",	0x0,	0xc5 },
202	{ "br_missp_exec",		0x0,	0x89 },
203
204	{ "br_ret_bac_missp_exec",	0x0,	0x91 },
205	{ "br_ret_exec",		0x0,	0x8f },
206	{ "br_ret_missp_exec",		0x0,	0x90 },
207
208	{ "br_tkn_bubble_1",		0x0,	0x97 },
209	{ "br_tkn_bubble_2",		0x0,	0x98 },
210	{ "bus_bnr_drv",		ALL_AGENTS,	0x61 },
211
212	{ "bus_data_rcv",		ALL_CORES,	0x64 },
213	{ "bus_drdy_clocks",		ALL_AGENTS,	0x62 },
214	{ "bus_hit_drv",		ALL_AGENTS,	0x7a },
215
216	{ "bus_hitm_drv",		ALL_AGENTS,	0x7b },
217	{ "bus_io_wait",		ALL_CORES,	0x7f },
218	{ "bus_lock_clocks",		ALL_CORES | ALL_AGENTS,	0x63 },
219
220	{ "bus_request_outstanding",	ALL_CORES | ALL_AGENTS,	0x60 },
221	{ "bus_trans_any",		ALL_CORES | ALL_AGENTS,	0x70 },
222	{ "bus_trans_brd",		ALL_CORES | ALL_AGENTS,	0x65 },
223
224	{ "bus_trans_burst",		ALL_CORES | ALL_AGENTS,	0x6e },
225	{ "bus_trans_def",		ALL_CORES | ALL_AGENTS,	0x6d },
226	{ "bus_trans_ifetch",		ALL_CORES | ALL_AGENTS,	0x68 },
227
228	{ "bus_trans_inval",		ALL_CORES | ALL_AGENTS,	0x69 },
229	{ "bus_trans_io",		ALL_CORES | ALL_AGENTS,	0x6c },
230	{ "bus_trans_mem",		ALL_CORES | ALL_AGENTS,	0x6f },
231
232	{ "bus_trans_p",		ALL_CORES | ALL_AGENTS,	0x6b },
233	{ "bus_trans_pwr",		ALL_CORES | ALL_AGENTS,	0x6a },
234	{ "bus_trans_rfo",		ALL_CORES | ALL_AGENTS,	0x66 },
235
236	{ "bus_trans_wb",		ALL_CORES | ALL_AGENTS,	0x67 },
237	{ "busq_empty",			ALL_CORES,	0x7d },
238	{ "cmp_snoop",			ALL_CORES,	0x78 },
239
240	{ "cpu_clk_unhalted",		0x0,	0x3c },
241	{ "cycles_int",			0x0,	0xc6 },
242	{ "cycles_l1i_mem_stalled",	0x0,	0x86 },
243
244	{ "dtlb_misses",		0x0,	0x08 },
245	{ "eist_trans",			0x0,	0x3a },
246	{ "esp",			0x0,	0xab },
247
248	{ "ext_snoop",			ALL_AGENTS,	0x77 },
249	{ "fp_mmx_trans",		0x0,	0xcc },
250	{ "hw_int_rcv",			0x0,	0xc8 },
251
252	{ "ild_stall",			0x0,	0x87 },
253	{ "inst_queue",			0x0,	0x83 },
254	{ "inst_retired",		0x0,	0xc0 },
255
256	{ "itlb",			0x0,	0x82 },
257	{ "itlb_miss_retired",		0x0,	0xc9 },
258	{ "l1d_all_ref",		0x0,	0x43 },
259
260	{ "l1d_cache_ld",		0x0,	0x40 },
261	{ "l1d_cache_lock",		0x0,	0x42 },
262	{ "l1d_cache_st",		0x0,	0x41 },
263
264	{ "l1d_m_evict",		0x0,	0x47 },
265	{ "l1d_m_repl",			0x0,	0x46 },
266	{ "l1d_pend_miss",		0x0,	0x48 },
267
268	{ "l1d_prefetch",		0x0,	0x4e },
269	{ "l1d_repl",			0x0,	0x45 },
270	{ "l1d_split",			0x0,	0x49 },
271
272	{ "l1i_misses",			0x0,	0x81 },
273	{ "l1i_reads",			0x0,	0x80 },
274	{ "l2_ads",			ALL_CORES,	0x21 },
275
276	{ "l2_dbus_busy_rd",		ALL_CORES,	0x23 },
277	{ "l2_ifetch",			ALL_CORES,	0x28 },
278	{ "l2_ld",			ALL_CORES,	0x29 },
279
280	{ "l2_lines_in",		ALL_CORES,	0x24 },
281	{ "l2_lines_out",		ALL_CORES,	0x26 },
282	{ "l2_lock",			ALL_CORES,	0x2b },
283
284	{ "l2_m_lines_in",		ALL_CORES,	0x25 },
285	{ "l2_m_lines_out",		ALL_CORES,	0x27 },
286	{ "l2_no_req",			ALL_CORES,	0x32 },
287
288	{ "l2_reject_busq",		ALL_CORES,	0x30 },
289	{ "l2_rqsts",			ALL_CORES,	0x2e },
290	{ "l2_st",			ALL_CORES,	0x2a },
291
292	{ "load_block",			0x0,	0x03 },
293	{ "load_hit_pre",		0x0,	0x4c },
294	{ "machine_nukes",		0x0,	0xc3 },
295
296	{ "macro_insts",		0x0,	0xaa },
297	{ "memory_disambiguation",	0x0,	0x09 },
298	{ "misalign_mem_ref",		0x0,	0x05 },
299	{ "page_walks",			0x0,	0x0c },
300
301	{ "pref_rqsts_dn",		0x0,	0xf8 },
302	{ "pref_rqsts_up",		0x0,	0xf0 },
303	{ "rat_stalls",			0x0,	0xd2 },
304
305	{ "resource_stalls",		0x0,	0xdc },
306	{ "rs_uops_dispatched",		0x0,	0xa0 },
307	{ "seg_reg_renames",		0x0,	0xd5 },
308
309	{ "seg_rename_stalls",		0x0,	0xd4 },
310	{ "segment_reg_loads",		0x0,	0x06 },
311	{ "simd_assist",		0x0,	0xcd },
312
313	{ "simd_comp_inst_retired",	0x0,	0xca },
314	{ "simd_inst_retired",		0x0,	0xc7 },
315	{ "simd_instr_retired",		0x0,	0xce },
316
317	{ "simd_sat_instr_retired",	0x0,	0xcf },
318	{ "simd_sat_uop_exec",		0x0,	0xb1 },
319	{ "simd_uop_type_exec",		0x0,	0xb3 },
320
321	{ "simd_uops_exec",		0x0,	0xb0 },
322	{ "snoop_stall_drv",		ALL_CORES | ALL_AGENTS,	0x7e },
323	{ "sse_pre_exec",		0x0,	0x07 },
324
325	{ "sse_pre_miss",		0x0,	0x4b },
326	{ "store_block",		0x0,	0x04 },
327	{ "thermal_trip",		0x0,	0x3b },
328
329	{ "uops_retired",		0x0,	0xc2 },
330	{ "x87_ops_retired",		0x0,	0xc1 },
331	{ "",				0x0,	NT_END }
332};
333
334/*
335 * If any of the pic specific events require privileges, make sure to add a
336 * check in configure_gpc() to find whether an event hard-coded as a number by
337 * the user has any privilege requirements
338 */
339static const struct nametable_fam6mod15_23 pic0_events[] = {
340	/* Alphabetical order of event name */
341
342	{ "cycles_div_busy",		0x0,	0x14 },
343	{ "fp_comp_ops_exe",		0x0,	0x10 },
344	{ "idle_during_div",		0x0,	0x18 },
345
346	{ "mem_load_retired",		0x0,	0xcb },
347	{ "rs_uops_dispatched_port",	0x0,	0xa1 },
348	{ "",				0x0,	NT_END }
349};
350
351static const struct nametable_fam6mod15_23 pic1_events[] = {
352	/* Alphabetical order of event name */
353
354	{ "delayed_bypass",	0x0,	0x19 },
355	{ "div",		0x0,	0x13 },
356	{ "fp_assist",		0x0,	0x11 },
357
358	{ "mul",		0x0,	0x12 },
359	{ "",			0x0,	NT_END }
360};
361
362/* FFC entries must be in order */
363char *ffc_names_non_htt[] = {
364	"instr_retired.any",
365	"cpu_clk_unhalted.core",
366	"cpu_clk_unhalted.ref",
367	NULL
368};
369
370char *ffc_names_htt[] = {
371	"instr_retired.any",
372	"cpu_clk_unhalted.thread",
373	"cpu_clk_unhalted.ref",
374	NULL
375};
376
377char **ffc_names = NULL;
378
379static char	**gpc_names = NULL;
380static uint32_t	versionid;
381static uint64_t	num_gpc;
382static uint64_t	width_gpc;
383static uint64_t	mask_gpc;
384static uint64_t	num_ffc;
385static uint64_t	width_ffc;
386static uint64_t	mask_ffc;
387static uint_t	total_pmc;
388static uint64_t	control_ffc;
389static uint64_t	control_gpc;
390static uint64_t	control_mask;
391static uint32_t	arch_events_vector;
392
393#define	IMPL_NAME_LEN 100
394static char core_impl_name[IMPL_NAME_LEN];
395
396static const char *core_cpuref =
397	"See Appendix A of the \"Intel 64 and IA-32 Architectures Software" \
398	" Developer's Manual Volume 3B: System Programming Guide, Part 2\"" \
399	" Order Number: 253669-026US, Februrary 2008";
400
401struct events_table_t {
402	uint8_t		eventselect;
403	uint8_t		unitmask;
404	uint64_t	supported_counters;
405	const char	*name;
406};
407
408/* Used to describe which counters support an event */
409#define	C(x) (1 << (x))
410#define	C0 C(0)
411#define	C1 C(1)
412#define	C2 C(2)
413#define	C3 C(3)
414#define	C_ALL 0xFFFFFFFFFFFFFFFF
415
416/* Architectural events */
417#define	ARCH_EVENTS_COMMON					\
418	{ 0xc0, 0x00, C_ALL, "inst_retired.any_p" },		\
419	{ 0x3c, 0x01, C_ALL, "cpu_clk_unhalted.ref_p" },	\
420	{ 0x2e, 0x4f, C_ALL, "longest_lat_cache.reference" },	\
421	{ 0x2e, 0x41, C_ALL, "longest_lat_cache.miss" },	\
422	{ 0xc4, 0x00, C_ALL, "br_inst_retired.all_branches" },	\
423	{ 0xc5, 0x00, C_ALL, "br_misp_retired.all_branches" }
424
425const struct events_table_t arch_events_table_non_htt[] = {
426	{ 0x3c, 0x00, C_ALL, "cpu_clk_unhalted.core" },
427	ARCH_EVENTS_COMMON
428};
429
430const struct events_table_t arch_events_table_htt[] = {
431	{ 0x3c, 0x00, C_ALL, "cpu_clk_unhalted.thread_p" },
432	ARCH_EVENTS_COMMON
433};
434
435const struct events_table_t *arch_events_table = NULL;
436static uint64_t known_arch_events;
437static uint64_t known_ffc_num;
438
439#define	EVENTS_FAM6_MOD26						\
440									\
441{ 0x80, 0x04, C0|C1|C2|C3, "l1i.cycles_stalled" },			\
442{ 0x80, 0x01, C0|C1|C2|C3, "l1i.hits" },				\
443{ 0x80, 0x02, C0|C1|C2|C3, "l1i.misses" },				\
444									\
445{ 0x80, 0x03, C0|C1|C2|C3, "l1i.reads" },				\
446{ 0x82, 0x01, C0|C1|C2|C3, "large_itlb.hit" },				\
447{ 0x87, 0x0F, C0|C1|C2|C3, "ild_stall.any" },				\
448									\
449{ 0x87, 0x04, C0|C1|C2|C3, "ild_stall.iq_full" },			\
450{ 0x87, 0x01, C0|C1|C2|C3, "ild_stall.lcp" },				\
451{ 0x87, 0x02, C0|C1|C2|C3, "ild_stall.mru" },				\
452									\
453{ 0x87, 0x08, C0|C1|C2|C3, "ild_stall.regen" },				\
454{ 0xE6, 0x02, C0|C1|C2|C3, "baclear.bad_target" },			\
455{ 0xE6, 0x01, C0|C1|C2|C3, "baclear.clear" },				\
456									\
457{ 0xE8, 0x01, C0|C1|C2|C3, "bpu_clears.early" },			\
458{ 0xE8, 0x02, C0|C1|C2|C3, "bpu_clears.late" },				\
459{ 0xE5, 0x01, C0|C1|C2|C3, "bpu_missed_call_ret" },			\
460									\
461{ 0xE0, 0x01, C0|C1|C2|C3, "br_inst_decoded" },				\
462{ 0x88, 0x7F, C0|C1|C2|C3, "br_inst_exec.any" },			\
463{ 0x88, 0x01, C0|C1|C2|C3, "br_inst_exec.cond" },			\
464									\
465{ 0x88, 0x02, C0|C1|C2|C3, "br_inst_exec.direct" },			\
466{ 0x88, 0x10, C0|C1|C2|C3, "br_inst_exec.direct_near_call" },		\
467{ 0x88, 0x20, C0|C1|C2|C3, "br_inst_exec.indirect_near_call" },		\
468									\
469{ 0x88, 0x04, C0|C1|C2|C3, "br_inst_exec.indirect_non_call" },		\
470{ 0x88, 0x30, C0|C1|C2|C3, "br_inst_exec.near_calls" },			\
471{ 0x88, 0x07, C0|C1|C2|C3, "br_inst_exec.non_calls" },			\
472									\
473{ 0x88, 0x08, C0|C1|C2|C3, "br_inst_exec.return_near" },		\
474{ 0x88, 0x40, C0|C1|C2|C3, "br_inst_exec.taken" },			\
475{ 0x89, 0x7F, C0|C1|C2|C3, "br_misp_exec.any" },			\
476									\
477{ 0x89, 0x01, C0|C1|C2|C3, "br_misp_exec.cond" },			\
478{ 0x89, 0x02, C0|C1|C2|C3, "br_misp_exec.direct" },			\
479{ 0x89, 0x10, C0|C1|C2|C3, "br_misp_exec.direct_near_call" },		\
480									\
481{ 0x89, 0x20, C0|C1|C2|C3, "br_misp_exec.indirect_near_call" },		\
482{ 0x89, 0x04, C0|C1|C2|C3, "br_misp_exec.indirect_non_call" },		\
483{ 0x89, 0x30, C0|C1|C2|C3, "br_misp_exec.near_calls" },			\
484									\
485{ 0x89, 0x07, C0|C1|C2|C3, "br_misp_exec.non_calls" },			\
486{ 0x89, 0x08, C0|C1|C2|C3, "br_misp_exec.return_near" },		\
487{ 0x89, 0x40, C0|C1|C2|C3, "br_misp_exec.taken" },			\
488									\
489{ 0x17, 0x01, C0|C1|C2|C3, "inst_queue_writes" },			\
490{ 0x1E, 0x01, C0|C1|C2|C3, "inst_queue_write_cycles" },			\
491{ 0xA7, 0x01, C0|C1|C2|C3, "baclear_force_iq" },			\
492									\
493{ 0xD0, 0x01, C0|C1|C2|C3, "macro_insts.decoded" },			\
494{ 0xA6, 0x01, C0|C1|C2|C3, "macro_insts.fusions_decoded" },		\
495{ 0x19, 0x01, C0|C1|C2|C3, "two_uop_insts_decoded" },			\
496									\
497{ 0x18, 0x01, C0|C1|C2|C3, "inst_decoded.dec0" },			\
498{ 0xD1, 0x04, C0|C1|C2|C3, "uops_decoded.esp_folding" },		\
499{ 0xD1, 0x08, C0|C1|C2|C3, "uops_decoded.esp_sync" },			\
500									\
501{ 0xD1, 0x02, C0|C1|C2|C3, "uops_decoded.ms" },				\
502{ 0x20, 0x01, C0|C1|C2|C3, "lsd_overflow" },				\
503{ 0x0E, 0x01, C0|C1|C2|C3, "uops_issued.any" },				\
504									\
505{ 0x0E, 0x02, C0|C1|C2|C3, "uops_issued.fused" },			\
506{ 0xA2, 0x20, C0|C1|C2|C3, "resource_stalls.fpcw" },			\
507{ 0xA2, 0x02, C0|C1|C2|C3, "resource_stalls.load" },			\
508									\
509{ 0xA2, 0x40, C0|C1|C2|C3, "resource_stalls.mxcsr" },			\
510{ 0xA2, 0x04, C0|C1|C2|C3, "resource_stalls.rs_full" },			\
511{ 0xA2, 0x08, C0|C1|C2|C3, "resource_stalls.store" },			\
512									\
513{ 0xA2, 0x01, C0|C1|C2|C3, "resource_stalls.any" },			\
514{ 0xD2, 0x01, C0|C1|C2|C3, "rat_stalls.flags" },			\
515{ 0xD2, 0x02, C0|C1|C2|C3, "rat_stalls.registers" },			\
516									\
517{ 0xD2, 0x04, C0|C1|C2|C3, "rat_stalls.rob_read_port" },		\
518{ 0xD2, 0x08, C0|C1|C2|C3, "rat_stalls.scoreboard" },			\
519{ 0xD2, 0x0F, C0|C1|C2|C3, "rat_stalls.any" },				\
520									\
521{ 0xD4, 0x01, C0|C1|C2|C3, "seg_rename_stalls" },			\
522{ 0xD5, 0x01, C0|C1|C2|C3, "es_reg_renames" },				\
523{ 0x10, 0x02, C0|C1|C2|C3, "fp_comp_ops_exe.mmx" },			\
524									\
525{ 0x10, 0x80, C0|C1|C2|C3, "fp_comp_ops_exe.sse_double_precision" },	\
526{ 0x10, 0x04, C0|C1|C2|C3, "fp_comp_ops_exe.sse_fp" },			\
527{ 0x10, 0x10, C0|C1|C2|C3, "fp_comp_ops_exe.sse_fp_packed" },		\
528									\
529{ 0x10, 0x20, C0|C1|C2|C3, "fp_comp_ops_exe.sse_fp_scalar" },		\
530{ 0x10, 0x40, C0|C1|C2|C3, "fp_comp_ops_exe.sse_single_precision" },	\
531{ 0x10, 0x08, C0|C1|C2|C3, "fp_comp_ops_exe.sse2_integer" },		\
532									\
533{ 0x10, 0x01, C0|C1|C2|C3, "fp_comp_ops_exe.x87" },			\
534{ 0x14, 0x01, C0|C1|C2|C3, "arith.cycles_div_busy" },			\
535{ 0x14, 0x02, C0|C1|C2|C3, "arith.mul" },				\
536									\
537{ 0x12, 0x04, C0|C1|C2|C3, "simd_int_128.pack" },			\
538{ 0x12, 0x20, C0|C1|C2|C3, "simd_int_128.packed_arith" },		\
539{ 0x12, 0x10, C0|C1|C2|C3, "simd_int_128.packed_logical" },		\
540									\
541{ 0x12, 0x01, C0|C1|C2|C3, "simd_int_128.packed_mpy" },			\
542{ 0x12, 0x02, C0|C1|C2|C3, "simd_int_128.packed_shift" },		\
543{ 0x12, 0x40, C0|C1|C2|C3, "simd_int_128.shuffle_move" },		\
544									\
545{ 0x12, 0x08, C0|C1|C2|C3, "simd_int_128.unpack" },			\
546{ 0xFD, 0x04, C0|C1|C2|C3, "simd_int_64.pack" },			\
547{ 0xFD, 0x20, C0|C1|C2|C3, "simd_int_64.packed_arith" },		\
548									\
549{ 0xFD, 0x10, C0|C1|C2|C3, "simd_int_64.packed_logical" },		\
550{ 0xFD, 0x01, C0|C1|C2|C3, "simd_int_64.packed_mpy" },			\
551{ 0xFD, 0x02, C0|C1|C2|C3, "simd_int_64.packed_shift" },		\
552									\
553{ 0xFD, 0x40, C0|C1|C2|C3, "simd_int_64.shuffle_move" },		\
554{ 0xFD, 0x08, C0|C1|C2|C3, "simd_int_64.unpack" },			\
555{ 0xB1, 0x01, C0|C1|C2|C3, "uops_executed.port0" },			\
556									\
557{ 0xB1, 0x02, C0|C1|C2|C3, "uops_executed.port1" },			\
558{ 0x40, 0x04, C0|C1, "l1d_cache_ld.e_state" },				\
559{ 0x40, 0x01, C0|C1, "l1d_cache_ld.i_state" },				\
560									\
561{ 0x40, 0x08, C0|C1, "l1d_cache_ld.m_state" },				\
562{ 0x40, 0x0F, C0|C1, "l1d_cache_ld.mesi" },				\
563{ 0x40, 0x02, C0|C1, "l1d_cache_ld.s_state" },				\
564									\
565{ 0x41, 0x04, C0|C1, "l1d_cache_st.e_state" },				\
566{ 0x41, 0x08, C0|C1, "l1d_cache_st.m_state" },				\
567{ 0x41, 0x0F, C0|C1, "l1d_cache_st.mesi" },				\
568									\
569{ 0x41, 0x02, C0|C1, "l1d_cache_st.s_state" },				\
570{ 0x42, 0x04, C0|C1, "l1d_cache_lock.e_state" },			\
571{ 0x42, 0x01, C0|C1, "l1d_cache_lock.hit" },				\
572									\
573{ 0x42, 0x08, C0|C1, "l1d_cache_lock.m_state" },			\
574{ 0x42, 0x02, C0|C1, "l1d_cache_lock.s_state" },			\
575{ 0x43, 0x01, C0|C1, "l1d_all_ref.any" },				\
576									\
577{ 0x43, 0x02, C0|C1, "l1d_all_ref.cacheable" },				\
578{ 0x4B, 0x01, C0|C1, "mmx2_mem_exec.nta" },				\
579{ 0x4C, 0x01, C0|C1, "load_hit_pre" },					\
580									\
581{ 0x4E, 0x02, C0|C1, "l1d_prefetch.miss" },				\
582{ 0x4E, 0x01, C0|C1, "l1d_prefetch.requests" },				\
583{ 0x4E, 0x04, C0|C1, "l1d_prefetch.triggers" },				\
584									\
585{ 0x51, 0x04, C0|C1, "l1d.m_evict" },					\
586{ 0x51, 0x02, C0|C1, "l1d.m_repl" },					\
587{ 0x51, 0x08, C0|C1, "l1d.m_snoop_evict" },				\
588									\
589{ 0x51, 0x01, C0|C1, "l1d.repl" },					\
590{ 0x52, 0x01, C0|C1, "l1d_cache_prefetch_lock_fb_hit" },		\
591{ 0x53, 0x01, C0|C1, "l1d_cache_lock_fb_hit" },				\
592									\
593{ 0x63, 0x02, C0|C1, "cache_lock_cycles.l1d" },				\
594{ 0x63, 0x01, C0|C1, "cache_lock_cycles.l1d_l2" },			\
595{ 0x06, 0x04, C0|C1|C2|C3, "store_blocks.at_ret" },			\
596									\
597{ 0x06, 0x08, C0|C1|C2|C3, "store_blocks.l1d_block" },			\
598{ 0x06, 0x01, C0|C1|C2|C3, "store_blocks.not_sta" },			\
599{ 0x06, 0x02, C0|C1|C2|C3, "store_blocks.sta" },			\
600									\
601{ 0x13, 0x07, C0|C1|C2|C3, "load_dispatch.any" },			\
602{ 0x13, 0x04, C0|C1|C2|C3, "load_dispatch.mob" },			\
603{ 0x13, 0x01, C0|C1|C2|C3, "load_dispatch.rs" },			\
604									\
605{ 0x13, 0x02, C0|C1|C2|C3, "load_dispatch.rs_delayed" },		\
606{ 0x08, 0x01, C0|C1|C2|C3, "dtlb_load_misses.any" },			\
607{ 0x08, 0x20, C0|C1|C2|C3, "dtlb_load_misses.pde_miss" },		\
608									\
609{ 0x08, 0x02, C0|C1|C2|C3, "dtlb_load_misses.walk_completed" },		\
610{ 0x49, 0x01, C0|C1|C2|C3, "dtlb_misses.any" },				\
611{ 0x49, 0x10, C0|C1|C2|C3, "dtlb_misses.stlb_hit" },			\
612									\
613{ 0x49, 0x02, C0|C1|C2|C3, "dtlb_misses.walk_completed" },		\
614{ 0x4F, 0x02, C0|C1|C2|C3, "ept.epde_miss" },				\
615{ 0x4F, 0x08, C0|C1|C2|C3, "ept.epdpe_miss" },				\
616									\
617{ 0x85, 0x01, C0|C1|C2|C3, "itlb_misses.any" },				\
618{ 0x85, 0x02, C0|C1|C2|C3, "itlb_misses.walk_completed" },		\
619{ 0x24, 0xAA, C0|C1|C2|C3, "l2_rqsts.miss" },				\
620									\
621{ 0x24, 0xFF, C0|C1|C2|C3, "l2_rqsts.references" },			\
622{ 0x24, 0x10, C0|C1|C2|C3, "l2_rqsts.ifetch_hit" },			\
623{ 0x24, 0x20, C0|C1|C2|C3, "l2_rqsts.ifetch_miss" },			\
624									\
625{ 0x24, 0x30, C0|C1|C2|C3, "l2_rqsts.ifetches" },			\
626{ 0x24, 0x01, C0|C1|C2|C3, "l2_rqsts.ld_hit" },				\
627{ 0x24, 0x02, C0|C1|C2|C3, "l2_rqsts.ld_miss" },			\
628									\
629{ 0x24, 0x03, C0|C1|C2|C3, "l2_rqsts.loads" },				\
630{ 0x24, 0x40, C0|C1|C2|C3, "l2_rqsts.prefetch_hit" },			\
631{ 0x24, 0x80, C0|C1|C2|C3, "l2_rqsts.prefetch_miss" },			\
632									\
633{ 0x24, 0xC0, C0|C1|C2|C3, "l2_rqsts.prefetches" },			\
634{ 0x24, 0x04, C0|C1|C2|C3, "l2_rqsts.rfo_hit" },			\
635{ 0x24, 0x08, C0|C1|C2|C3, "l2_rqsts.rfo_miss" },			\
636									\
637{ 0x24, 0x0C, C0|C1|C2|C3, "l2_rqsts.rfos" },				\
638{ 0x26, 0xFF, C0|C1|C2|C3, "l2_data_rqsts.any" },			\
639{ 0x26, 0x04, C0|C1|C2|C3, "l2_data_rqsts.demand.e_state" },		\
640									\
641{ 0x26, 0x01, C0|C1|C2|C3, "l2_data_rqsts.demand.i_state" },		\
642{ 0x26, 0x08, C0|C1|C2|C3, "l2_data_rqsts.demand.m_state" },		\
643{ 0x26, 0x0F, C0|C1|C2|C3, "l2_data_rqsts.demand.mesi" },		\
644									\
645{ 0x26, 0x02, C0|C1|C2|C3, "l2_data_rqsts.demand.s_state" },		\
646{ 0x26, 0x40, C0|C1|C2|C3, "l2_data_rqsts.prefetch.e_state" },		\
647{ 0x26, 0x10, C0|C1|C2|C3, "l2_data_rqsts.prefetch.i_state" },		\
648									\
649{ 0x26, 0x80, C0|C1|C2|C3, "l2_data_rqsts.prefetch.m_state" },		\
650{ 0x26, 0xF0, C0|C1|C2|C3, "l2_data_rqsts.prefetch.mesi" },		\
651{ 0x26, 0x20, C0|C1|C2|C3, "l2_data_rqsts.prefetch.s_state" },		\
652									\
653{ 0x27, 0x40, C0|C1|C2|C3, "l2_write.lock.e_state" },			\
654{ 0x27, 0x10, C0|C1|C2|C3, "l2_write.lock.i_state" },			\
655{ 0x27, 0x20, C0|C1|C2|C3, "l2_write.lock.s_state" },			\
656									\
657{ 0x27, 0x0E, C0|C1|C2|C3, "l2_write.rfo.hit" },			\
658{ 0x27, 0x01, C0|C1|C2|C3, "l2_write.rfo.i_state" },			\
659{ 0x27, 0x08, C0|C1|C2|C3, "l2_write.rfo.m_state" },			\
660									\
661{ 0x27, 0x0F, C0|C1|C2|C3, "l2_write.rfo.mesi" },			\
662{ 0x27, 0x02, C0|C1|C2|C3, "l2_write.rfo.s_state" },			\
663{ 0x28, 0x04, C0|C1|C2|C3, "l1d_wb_l2.e_state" },			\
664									\
665{ 0x28, 0x01, C0|C1|C2|C3, "l1d_wb_l2.i_state" },			\
666{ 0x28, 0x08, C0|C1|C2|C3, "l1d_wb_l2.m_state" },			\
667{ 0xF0, 0x80, C0|C1|C2|C3, "l2_transactions.any" },			\
668									\
669{ 0xF0, 0x20, C0|C1|C2|C3, "l2_transactions.fill" },			\
670{ 0xF0, 0x04, C0|C1|C2|C3, "l2_transactions.ifetch" },			\
671{ 0xF0, 0x10, C0|C1|C2|C3, "l2_transactions.l1d_wb" },			\
672									\
673{ 0xF0, 0x01, C0|C1|C2|C3, "l2_transactions.load" },			\
674{ 0xF0, 0x08, C0|C1|C2|C3, "l2_transactions.prefetch" },		\
675{ 0xF0, 0x02, C0|C1|C2|C3, "l2_transactions.rfo" },			\
676									\
677{ 0xF0, 0x40, C0|C1|C2|C3, "l2_transactions.wb" },			\
678{ 0xF1, 0x07, C0|C1|C2|C3, "l2_lines_in.any" },				\
679{ 0xF1, 0x04, C0|C1|C2|C3, "l2_lines_in.e_state" },			\
680									\
681{ 0xF1, 0x02, C0|C1|C2|C3, "l2_lines_in.s_state" },			\
682{ 0xF2, 0x0F, C0|C1|C2|C3, "l2_lines_out.any" },			\
683{ 0xF2, 0x01, C0|C1|C2|C3, "l2_lines_out.demand_clean" },		\
684									\
685{ 0xF2, 0x02, C0|C1|C2|C3, "l2_lines_out.demand_dirty" },		\
686{ 0xF2, 0x04, C0|C1|C2|C3, "l2_lines_out.prefetch_clean" },		\
687{ 0x6C, 0x01, C0|C1|C2|C3, "io_transactions" },				\
688									\
689{ 0xB0, 0x80, C0|C1|C2|C3, "offcore_requests.any" },			\
690{ 0xB0, 0x10, C0|C1|C2|C3, "offcore_requests.any.rfo" },		\
691{ 0xB0, 0x40, C0|C1|C2|C3, "offcore_requests.l1d_writeback" },		\
692									\
693{ 0xB8, 0x01, C0|C1|C2|C3, "snoop_response.hit" },			\
694{ 0xB8, 0x02, C0|C1|C2|C3, "snoop_response.hite" },			\
695{ 0xB8, 0x04, C0|C1|C2|C3, "snoop_response.hitm" },			\
696									\
697{ 0xF4, 0x10, C0|C1|C2|C3, "sq_misc.split_lock" },			\
698{ 0x0B, 0x01, C0|C1|C2|C3, "mem_inst_retired.loads" },			\
699{ 0x0B, 0x02, C0|C1|C2|C3, "mem_inst_retired.stores" },			\
700									\
701{ 0xC0, 0x04, C0|C1|C2|C3, "inst_retired.mmx" },			\
702{ 0xC0, 0x02, C0|C1|C2|C3, "inst_retired.x87" },			\
703{ 0xC7, 0x04, C0|C1|C2|C3, "ssex_uops_retired.packed_double" },		\
704									\
705{ 0xC7, 0x01, C0|C1|C2|C3, "ssex_uops_retired.packed_single" },		\
706{ 0xC7, 0x08, C0|C1|C2|C3, "ssex_uops_retired.scalar_double" },		\
707{ 0xC7, 0x02, C0|C1|C2|C3, "ssex_uops_retired.scalar_single" },		\
708									\
709{ 0xC7, 0x10, C0|C1|C2|C3, "ssex_uops_retired.vector_integer" },	\
710{ 0xC2, 0x01, C0|C1|C2|C3, "uops_retired.any" },			\
711{ 0xC2, 0x04, C0|C1|C2|C3, "uops_retired.macro_fused" },		\
712									\
713{ 0xC8, 0x20, C0|C1|C2|C3, "itlb_miss_retired" },			\
714{ 0xCB, 0x80, C0|C1|C2|C3, "mem_load_retired.dtlb_miss" },		\
715{ 0xCB, 0x40, C0|C1|C2|C3, "mem_load_retired.hit_lfb" },		\
716									\
717{ 0xCB, 0x01, C0|C1|C2|C3, "mem_load_retired.l1d_hit" },		\
718{ 0xCB, 0x02, C0|C1|C2|C3, "mem_load_retired.l2_hit" },			\
719{ 0xCB, 0x10, C0|C1|C2|C3, "mem_load_retired.llc_miss" },		\
720									\
721{ 0xCB, 0x04, C0|C1|C2|C3, "mem_load_retired.llc_unshared_hit" },	\
722{ 0xCB, 0x08, C0|C1|C2|C3, "mem_load_retired.other_core_l2_hit_hitm" },	\
723{ 0x0F, 0x02, C0|C1|C2|C3, "mem_uncore_retired.other_core_l2_hitm" },	\
724									\
725{ 0x0F, 0x08, C0|C1|C2|C3, "mem_uncore_retired.remote_cache_local_home_hit" },\
726{ 0x0F, 0x10, C0|C1|C2|C3, "mem_uncore_retired.remote_dram" },		\
727{ 0x0F, 0x20, C0|C1|C2|C3, "mem_uncore_retired.local_dram" },		\
728									\
729{ 0x0C, 0x01, C0|C1|C2|C3, "mem_store_retired.dtlb_miss" },		\
730{ 0xC4, 0x01, C0|C1|C2|C3, "br_inst_retired.conditional" },		\
731{ 0xC4, 0x02, C0|C1|C2|C3, "br_inst_retired.near_call" },		\
732									\
733{ 0xC5, 0x02, C0|C1|C2|C3, "br_misp_retired.near_call" },		\
734{ 0xDB, 0x01, C0|C1|C2|C3, "uop_unfusion" },				\
735{ 0xF7, 0x01, C0|C1|C2|C3, "fp_assist.all" },				\
736									\
737{ 0xF7, 0x04, C0|C1|C2|C3, "fp_assist.input" },				\
738{ 0xF7, 0x02, C0|C1|C2|C3, "fp_assist.output" },			\
739{ 0xCC, 0x03, C0|C1|C2|C3, "fp_mmx_trans.any" },			\
740									\
741{ 0xCC, 0x01, C0|C1|C2|C3, "fp_mmx_trans.to_fp" },			\
742{ 0xCC, 0x02, C0|C1|C2|C3, "fp_mmx_trans.to_mmx" },			\
743{ 0xC3, 0x04, C0|C1|C2|C3, "machine_clears.smc" }
744
745#define	EVENTS_FAM6_MOD28						\
746	{ 0x2,  0x81, C0|C1, "store_forwards.good" },                   \
747	{ 0x6,  0x0,  C0|C1, "segment_reg_loads.any" },                 \
748	{ 0x7,  0x1,  C0|C1, "prefetch.prefetcht0" },                   \
749	{ 0x7,  0x6,  C0|C1, "prefetch.sw_l2" },                        \
750	{ 0x7,  0x8,  C0|C1, "prefetch.prefetchnta" },                  \
751	{ 0x8,  0x7,  C0|C1, "data_tlb_misses.dtlb_miss" },             \
752	{ 0x8,  0x5,  C0|C1, "data_tlb_misses.dtlb_miss_ld" },          \
753	{ 0x8,  0x9,  C0|C1, "data_tlb_misses.l0_dtlb_miss_ld" },	\
754	{ 0x8,  0x6,  C0|C1, "data_tlb_misses.dtlb_miss_st" },          \
755	{ 0xC,  0x3,  C0|C1, "page_walks.cycles" },                     \
756	{ 0x10, 0x1,  C0|C1, "x87_comp_ops_exe.any.s" },                \
757	{ 0x10, 0x81, C0|C1, "x87_comp_ops_exe.any.ar" },               \
758	{ 0x11, 0x1,  C0|C1, "fp_assist" },                             \
759	{ 0x11, 0x81, C0|C1, "fp_assist.ar" },                          \
760	{ 0x12, 0x1,  C0|C1, "mul.s" },                                 \
761	{ 0x12, 0x81, C0|C1, "mul.ar" },                                \
762	{ 0x13, 0x1,  C0|C1, "div.s" },                                 \
763	{ 0x13, 0x81, C0|C1, "div.ar" },                                \
764	{ 0x14, 0x1,  C0|C1, "cycles_div_busy" },                       \
765	{ 0x21, 0x0,  C0|C1, "l2_ads" },                      		\
766	{ 0x22, 0x0,  C0|C1, "l2_dbus_busy" },                		\
767	{ 0x24, 0x0,  C0|C1, "l2_lines_in" },   			\
768	{ 0x25, 0x0,  C0|C1, "l2_m_lines_in" },               		\
769	{ 0x26, 0x0,  C0|C1, "l2_lines_out" },  			\
770	{ 0x27, 0x0,  C0|C1, "l2_m_lines_out" },			\
771	{ 0x28, 0x0,  C0|C1, "l2_ifetch" },  				\
772	{ 0x29, 0x0,  C0|C1, "l2_ld" },					\
773	{ 0x2A, 0x0,  C0|C1, "l2_st" },      				\
774	{ 0x2B, 0x0,  C0|C1, "l2_lock" },    				\
775	{ 0x2E, 0x0,  C0|C1, "l2_rqsts" },             			\
776	{ 0x2E, 0x41, C0|C1, "l2_rqsts.self.demand.i_state" },		\
777	{ 0x2E, 0x4F, C0|C1, "l2_rqsts.self.demand.mesi" },		\
778	{ 0x30, 0x0,  C0|C1, "l2_reject_bus_q" },			\
779	{ 0x32, 0x0,  C0|C1, "l2_no_req" },                   		\
780	{ 0x3A, 0x0,  C0|C1, "eist_trans" },                            \
781	{ 0x3B, 0xC0, C0|C1, "thermal_trip" },                          \
782	{ 0x3C, 0x0,  C0|C1, "cpu_clk_unhalted.core_p" },               \
783	{ 0x3C, 0x1,  C0|C1, "cpu_clk_unhalted.bus" },                  \
784	{ 0x3C, 0x2,  C0|C1, "cpu_clk_unhalted.no_other" },             \
785	{ 0x40, 0x21, C0|C1, "l1d_cache.ld" },                          \
786	{ 0x40, 0x22, C0|C1, "l1d_cache.st" },                          \
787	{ 0x60, 0x0,  C0|C1, "bus_request_outstanding" },		\
788	{ 0x61, 0x0,  C0|C1, "bus_bnr_drv" },                		\
789	{ 0x62, 0x0,  C0|C1, "bus_drdy_clocks" },            		\
790	{ 0x63, 0x0,  C0|C1, "bus_lock_clocks" },  			\
791	{ 0x64, 0x0,  C0|C1, "bus_data_rcv" },                		\
792	{ 0x65, 0x0,  C0|C1, "bus_trans_brd" },    			\
793	{ 0x66, 0x0,  C0|C1, "bus_trans_rfo" },    			\
794	{ 0x67, 0x0,  C0|C1, "bus_trans_wb" },     			\
795	{ 0x68, 0x0,  C0|C1, "bus_trans_ifetch" }, 			\
796	{ 0x69, 0x0,  C0|C1, "bus_trans_inval" },  			\
797	{ 0x6A, 0x0,  C0|C1, "bus_trans_pwr" },				\
798	{ 0x6B, 0x0,  C0|C1, "bus_trans_p" },      			\
799	{ 0x6C, 0x0,  C0|C1, "bus_trans_io" },     			\
800	{ 0x6D, 0x0,  C0|C1, "bus_trans_def" },    			\
801	{ 0x6E, 0x0,  C0|C1, "bus_trans_burst" },  			\
802	{ 0x6F, 0x0,  C0|C1, "bus_trans_mem" },    			\
803	{ 0x70, 0x0,  C0|C1, "bus_trans_any" },    			\
804	{ 0x77, 0x0,  C0|C1, "ext_snoop" },     			\
805	{ 0x7A, 0x0,  C0|C1, "bus_hit_drv" },                		\
806	{ 0x7B, 0x0,  C0|C1, "bus_hitm_drv" },               		\
807	{ 0x7D, 0x0,  C0|C1, "busq_empty" },                  		\
808	{ 0x7E, 0x0,  C0|C1, "snoop_stall_drv" },  			\
809	{ 0x7F, 0x0,  C0|C1, "bus_io_wait" },				\
810	{ 0x80, 0x3,  C0|C1, "icache.accesses" },                       \
811	{ 0x80, 0x2,  C0|C1, "icache.misses" },                         \
812	{ 0x82, 0x4,  C0|C1, "itlb.flush" },                            \
813	{ 0x82, 0x2,  C0|C1, "itlb.misses" },                           \
814	{ 0xAA, 0x2,  C0|C1, "macro_insts.cisc_decoded" },              \
815	{ 0xAA, 0x3,  C0|C1, "macro_insts.all_decoded" },               \
816	{ 0xB0, 0x0,  C0|C1, "simd_uops_exec.s" },                      \
817	{ 0xB0, 0x80, C0|C1, "simd_uops_exec.ar" },                     \
818	{ 0xB1, 0x0,  C0|C1, "simd_sat_uop_exec.s" },                   \
819	{ 0xB1, 0x80, C0|C1, "simd_sat_uop_exec.ar" },                  \
820	{ 0xB3, 0x1,  C0|C1, "simd_uop_type_exec.mul.s" },              \
821	{ 0xB3, 0x81, C0|C1, "simd_uop_type_exec.mul.ar" },             \
822	{ 0xB3, 0x02, C0|C1, "simd_uop_type_exec.shift.s" },            \
823	{ 0xB3, 0x82, C0|C1, "simd_uop_type_exec.shift.ar" },           \
824	{ 0xB3, 0x04, C0|C1, "simd_uop_type_exec.pack.s" },             \
825	{ 0xB3, 0x84, C0|C1, "simd_uop_type_exec.pack.ar" },            \
826	{ 0xB3, 0x08, C0|C1, "simd_uop_type_exec.unpack.s" },           \
827	{ 0xB3, 0x88, C0|C1, "simd_uop_type_exec.unpack.ar" },          \
828	{ 0xB3, 0x10, C0|C1, "simd_uop_type_exec.logical.s" },          \
829	{ 0xB3, 0x90, C0|C1, "simd_uop_type_exec.logical.ar" },         \
830	{ 0xB3, 0x20, C0|C1, "simd_uop_type_exec.arithmetic.s" },       \
831	{ 0xB3, 0xA0, C0|C1, "simd_uop_type_exec.arithmetic.ar" },      \
832	{ 0xC2, 0x10, C0|C1, "uops_retired.any" },                      \
833	{ 0xC3, 0x1,  C0|C1, "machine_clears.smc" },                    \
834	{ 0xC4, 0x0,  C0|C1, "br_inst_retired.any" },                   \
835	{ 0xC4, 0x1,  C0|C1, "br_inst_retired.pred_not_taken" },        \
836	{ 0xC4, 0x2,  C0|C1, "br_inst_retired.mispred_not_taken" },     \
837	{ 0xC4, 0x4,  C0|C1, "br_inst_retired.pred_taken" },            \
838	{ 0xC4, 0x8,  C0|C1, "br_inst_retired.mispred_taken" },         \
839	{ 0xC4, 0xA,  C0|C1, "br_inst_retired.mispred" },               \
840	{ 0xC4, 0xC,  C0|C1, "br_inst_retired.taken" },                 \
841	{ 0xC4, 0xF,  C0|C1, "br_inst_retired.any1" },                  \
842	{ 0xC6, 0x1,  C0|C1, "cycles_int_masked.cycles_int_masked" },   \
843	{ 0xC6, 0x2,  C0|C1,						\
844		"cycles_int_masked.cycles_int_pending_and_masked" },	\
845	{ 0xC7, 0x1,  C0|C1, "simd_inst_retired.packed_single" },       \
846	{ 0xC7, 0x2,  C0|C1, "simd_inst_retired.scalar_single" },      	\
847	{ 0xC7, 0x4,  C0|C1, "simd_inst_retired.packed_double" },       \
848	{ 0xC7, 0x8,  C0|C1, "simd_inst_retired.scalar_double" },       \
849	{ 0xC7, 0x10, C0|C1, "simd_inst_retired.vector" },              \
850	{ 0xC7, 0x1F, C0|C1, "simd_inst_retired.any" },                 \
851	{ 0xC8, 0x00, C0|C1, "hw_int_rcv" },                            \
852	{ 0xCA, 0x1,  C0|C1, "simd_comp_inst_retired.packed_single" },  \
853	{ 0xCA, 0x2,  C0|C1, "simd_comp_inst_retired.scalar_single" }, 	\
854	{ 0xCA, 0x4,  C0|C1, "simd_comp_inst_retired.packed_double" },  \
855	{ 0xCA, 0x8,  C0|C1, "simd_comp_inst_retired.scalar_double" },  \
856	{ 0xCB, 0x1,  C0|C1, "mem_load_retired.l2_hit" },               \
857	{ 0xCB, 0x2,  C0|C1, "mem_load_retired.l2_miss" },              \
858	{ 0xCB, 0x4,  C0|C1, "mem_load_retired.dtlb_miss" },           	\
859	{ 0xCD, 0x0,  C0|C1, "simd_assist" },                           \
860	{ 0xCE, 0x0,  C0|C1, "simd_instr_retired" },                    \
861	{ 0xCF, 0x0,  C0|C1, "simd_sat_instr_retired" },                \
862	{ 0xE0, 0x1,  C0|C1, "br_inst_decoded" },                       \
863	{ 0xE4, 0x1,  C0|C1, "bogus_br" },                             	\
864	{ 0xE6, 0x1,  C0|C1, "baclears.any" }
865
866#define	EVENTS_FAM6_MOD37						\
867{ 0xB0, 0x08, C0|C1|C2|C3, "offcore_requests.any.read" },		\
868{ 0xB0, 0x01, C0|C1|C2|C3, "offcore_requests.demand.read_data" },	\
869{ 0xB0, 0x04, C0|C1|C2|C3, "offcore_requests.demand.rfo" }
870
871static const struct events_table_t *events_table = NULL;
872
873const struct events_table_t events_fam6_mod26[] = {
874	EVENTS_FAM6_MOD26,
875	{ NT_END, 0, 0, "" }
876};
877
878const struct events_table_t events_fam6_mod28[] = {
879	EVENTS_FAM6_MOD28,
880	{ NT_END, 0, 0, "" }
881};
882
883const struct events_table_t events_fam6_mod37[] = {
884	EVENTS_FAM6_MOD26,
885	EVENTS_FAM6_MOD37,
886	{ NT_END, 0, 0, "" }
887};
888
889/*
890 * Initialize string containing list of supported general-purpose counter
891 * events for processors of Family 6 Models 15 and 23
892 */
893static void
894pcbe_init_fam6_model15_23()
895{
896	const struct nametable_fam6mod15_23	*n;
897	const struct nametable_fam6mod15_23	*picspecific_events;
898	size_t			common_size;
899	size_t			size;
900	uint64_t		i;
901
902	gpc_names = kmem_alloc(num_gpc * sizeof (char *), KM_SLEEP);
903
904	/* Calculate space needed to save all the common event names */
905	common_size = 0;
906	for (n = cmn_gpc_events_f6m15_23; n->event_num != NT_END; n++) {
907		common_size += strlen(n->name) + 1;
908	}
909
910	for (i = 0; i < num_gpc; i++) {
911		size = 0;
912		switch (i) {
913			case 0:
914				picspecific_events = pic0_events;
915				break;
916			case 1:
917				picspecific_events = pic1_events;
918				break;
919			default:
920				picspecific_events = NULL;
921				break;
922		}
923		if (picspecific_events != NULL) {
924			for (n = picspecific_events;
925			    n->event_num != NT_END;
926			    n++) {
927				size += strlen(n->name) + 1;
928			}
929		}
930
931		gpc_names[i] =
932		    kmem_alloc(size + common_size + 1, KM_SLEEP);
933
934		gpc_names[i][0] = '\0';
935		if (picspecific_events != NULL) {
936			for (n = picspecific_events;
937			    n->event_num != NT_END;
938			    n++) {
939				(void) strcat(gpc_names[i], n->name);
940				(void) strcat(gpc_names[i], ",");
941			}
942		}
943		for (n = cmn_gpc_events_f6m15_23; n->event_num != NT_END;
944		    n++) {
945			(void) strcat(gpc_names[i], n->name);
946			(void) strcat(gpc_names[i], ",");
947		}
948		/*
949		 * Remove trailing comma.
950		 */
951		gpc_names[i][common_size + size - 1] = '\0';
952	}
953}
954
955static int
956core_pcbe_init(void)
957{
958	struct cpuid_regs	cp;
959	size_t			size;
960	uint64_t		i;
961	uint64_t		j;
962	uint64_t		arch_events_vector_length;
963	size_t			arch_events_string_length;
964
965	if (cpuid_getvendor(CPU) != X86_VENDOR_Intel)
966		return (-1);
967
968	/* Obtain Basic CPUID information */
969	cp.cp_eax = 0x0;
970	(void) __cpuid_insn(&cp);
971
972	/* No Architectural Performance Monitoring Leaf returned by CPUID */
973	if (cp.cp_eax < 0xa) {
974		return (-1);
975	}
976
977	/* Obtain the Architectural Performance Monitoring Leaf */
978	cp.cp_eax = 0xa;
979	(void) __cpuid_insn(&cp);
980
981	versionid = cp.cp_eax & 0xFF;
982
983	/*
984	 * Fixed-Function Counters (FFC)
985	 *
986	 * All Family 6 Model 15 and Model 23 processors have fixed-function
987	 * counters.  These counters were made Architectural with
988	 * Family 6 Model 15 Stepping 9.
989	 */
990	switch (versionid) {
991
992		case 0:
993			return (-1);
994
995		case 2:
996			num_ffc = cp.cp_edx & 0x1F;
997			width_ffc = (cp.cp_edx >> 5) & 0xFF;
998
999			/*
1000			 * Some processors have an errata (AW34) where
1001			 * versionid is reported as 2 when actually 1.
1002			 * In this case, fixed-function counters are
1003			 * model-specific as in Version 1.
1004			 */
1005			if (num_ffc != 0) {
1006				break;
1007			}
1008			/* FALLTHROUGH */
1009		case 1:
1010			num_ffc = 3;
1011			width_ffc = 40;
1012			versionid = 1;
1013			break;
1014
1015		default:
1016			num_ffc = cp.cp_edx & 0x1F;
1017			width_ffc = (cp.cp_edx >> 5) & 0xFF;
1018			break;
1019	}
1020
1021
1022	if (num_ffc >= 64)
1023		return (-1);
1024
1025	/* Set HTT-specific names of architectural & FFC events */
1026	if (x86_feature & X86_HTT) {
1027		ffc_names = ffc_names_htt;
1028		arch_events_table = arch_events_table_htt;
1029		known_arch_events =
1030		    sizeof (arch_events_table_htt) /
1031		    sizeof (struct events_table_t);
1032		known_ffc_num =
1033		    sizeof (ffc_names_htt) / sizeof (char *);
1034	} else {
1035		ffc_names = ffc_names_non_htt;
1036		arch_events_table = arch_events_table_non_htt;
1037		known_arch_events =
1038		    sizeof (arch_events_table_non_htt) /
1039		    sizeof (struct events_table_t);
1040		known_ffc_num =
1041		    sizeof (ffc_names_non_htt) / sizeof (char *);
1042	}
1043
1044	if (num_ffc >= known_ffc_num) {
1045		/*
1046		 * The system seems to have more fixed-function counters than
1047		 * what this PCBE is able to handle correctly.  Default to the
1048		 * maximum number of fixed-function counters that this driver
1049		 * is aware of.
1050		 */
1051		num_ffc = known_ffc_num - 1;
1052	}
1053
1054	mask_ffc = BITMASK_XBITS(width_ffc);
1055	control_ffc = BITMASK_XBITS(num_ffc);
1056
1057	/*
1058	 * General Purpose Counters (GPC)
1059	 */
1060	num_gpc = (cp.cp_eax >> 8) & 0xFF;
1061	width_gpc = (cp.cp_eax >> 16) & 0xFF;
1062
1063	if (num_gpc >= 64)
1064		return (-1);
1065
1066	mask_gpc = BITMASK_XBITS(width_gpc);
1067
1068	control_gpc = BITMASK_XBITS(num_gpc);
1069
1070	control_mask = (control_ffc << 32) | control_gpc;
1071
1072	total_pmc = num_gpc + num_ffc;
1073	if (total_pmc > 64) {
1074		/* Too wide for the overflow bitmap */
1075		return (-1);
1076	}
1077
1078	/* GPC events for Family 6 Models 15 & 23 only */
1079	if ((cpuid_getfamily(CPU) == 6) &&
1080	    ((cpuid_getmodel(CPU) == 15) || (cpuid_getmodel(CPU) == 23))) {
1081		(void) snprintf(core_impl_name, IMPL_NAME_LEN,
1082		    "Core Microarchitecture");
1083		pcbe_init_fam6_model15_23();
1084		return (0);
1085	}
1086
1087	(void) snprintf(core_impl_name, IMPL_NAME_LEN,
1088	    "Intel Arch PerfMon v%d on Family %d Model %d",
1089	    versionid, cpuid_getfamily(CPU), cpuid_getmodel(CPU));
1090
1091	/*
1092	 * Architectural events
1093	 */
1094	arch_events_vector_length = (cp.cp_eax >> 24) & 0xFF;
1095
1096	ASSERT(known_arch_events == arch_events_vector_length);
1097
1098	/*
1099	 * To handle the case where a new performance monitoring setup is run
1100	 * on a non-debug kernel
1101	 */
1102	if (known_arch_events > arch_events_vector_length) {
1103		known_arch_events = arch_events_vector_length;
1104	} else {
1105		arch_events_vector_length = known_arch_events;
1106	}
1107
1108	arch_events_vector = cp.cp_ebx &
1109	    BITMASK_XBITS(arch_events_vector_length);
1110
1111	/*
1112	 * Process architectural and non-architectural events using GPC
1113	 */
1114	if (num_gpc > 0) {
1115
1116		gpc_names = kmem_alloc(num_gpc * sizeof (char *), KM_SLEEP);
1117
1118		/* Calculate space required for the architectural gpc events */
1119		arch_events_string_length = 0;
1120		for (i = 0; i < known_arch_events; i++) {
1121			if (((1U << i) & arch_events_vector) == 0) {
1122				arch_events_string_length +=
1123				    strlen(arch_events_table[i].name) + 1;
1124			}
1125		}
1126
1127		/* Non-architectural events list */
1128		if (cpuid_getmodel(CPU) == 26) {
1129			events_table = events_fam6_mod26;
1130		} else if (cpuid_getmodel(CPU) == 28) {
1131			events_table = events_fam6_mod28;
1132		} else if (cpuid_getmodel(CPU) == 37) {
1133			events_table = events_fam6_mod37;
1134		}
1135
1136		for (i = 0; i < num_gpc; i++) {
1137
1138			/*
1139			 * Determine length of all supported event names
1140			 * (architectural + non-architectural)
1141			 */
1142			size = arch_events_string_length;
1143			for (j = 0; events_table != NULL &&
1144			    events_table[j].eventselect != NT_END;
1145			    j++) {
1146				if (C(i) & events_table[j].supported_counters) {
1147					size += strlen(events_table[j].name) +
1148					    1;
1149				}
1150			}
1151
1152			/* Allocate memory for this pics list */
1153			gpc_names[i] = kmem_alloc(size + 1, KM_SLEEP);
1154			gpc_names[i][0] = '\0';
1155			if (size == 0) {
1156				continue;
1157			}
1158
1159			/*
1160			 * Create the list of all supported events
1161			 * (architectural + non-architectural)
1162			 */
1163			for (j = 0; j < known_arch_events; j++) {
1164				if (((1U << j) & arch_events_vector) == 0) {
1165					(void) strcat(gpc_names[i],
1166					    arch_events_table[j].name);
1167					(void) strcat(gpc_names[i], ",");
1168				}
1169			}
1170
1171			for (j = 0; events_table != NULL &&
1172			    events_table[j].eventselect != NT_END;
1173			    j++) {
1174				if (C(i) & events_table[j].supported_counters) {
1175					(void) strcat(gpc_names[i],
1176					    events_table[j].name);
1177					(void) strcat(gpc_names[i], ",");
1178				}
1179			}
1180
1181			/* Remove trailing comma */
1182			gpc_names[i][size - 1] = '\0';
1183		}
1184	}
1185	/*
1186	 * Fixed-function Counters (FFC) are already listed individually in
1187	 * ffc_names[]
1188	 */
1189	return (0);
1190}
1191
1192static uint_t core_pcbe_ncounters()
1193{
1194	return (total_pmc);
1195}
1196
1197static const char *core_pcbe_impl_name(void)
1198{
1199	return (core_impl_name);
1200}
1201
1202static const char *core_pcbe_cpuref(void)
1203{
1204	return (core_cpuref);
1205}
1206
1207static char *core_pcbe_list_events(uint_t picnum)
1208{
1209	ASSERT(picnum < cpc_ncounters);
1210
1211	if (picnum < num_gpc) {
1212		return (gpc_names[picnum]);
1213	} else {
1214		return (ffc_names[picnum - num_gpc]);
1215	}
1216}
1217
1218static char *core_pcbe_list_attrs(void)
1219{
1220	if (versionid >= 3) {
1221		return ("edge,inv,umask,cmask,anythr");
1222	} else {
1223		return ("edge,pc,inv,umask,cmask");
1224	}
1225}
1226
1227static const struct nametable_fam6mod15_23 *
1228find_gpcevent_f6m15_23(char *name,
1229    const struct nametable_fam6mod15_23 *nametable)
1230{
1231	const struct nametable_fam6mod15_23 *n;
1232	int compare_result = -1;
1233
1234	for (n = nametable; n->event_num != NT_END; n++) {
1235		compare_result = strcmp(name, n->name);
1236		if (compare_result <= 0) {
1237			break;
1238		}
1239	}
1240
1241	if (compare_result == 0) {
1242		return (n);
1243	}
1244
1245	return (NULL);
1246}
1247
1248static const struct events_table_t *
1249find_gpcevent(char *name)
1250{
1251	int i;
1252
1253	/* Search architectural events */
1254	for (i = 0; i < known_arch_events; i++) {
1255		if (strcmp(name, arch_events_table[i].name) == 0) {
1256			if (((1U << i) & arch_events_vector) == 0) {
1257				return (&arch_events_table[i]);
1258			}
1259		}
1260	}
1261
1262	/* Search non-architectural events */
1263	if (events_table != NULL) {
1264		for (i = 0; events_table[i].eventselect != NT_END; i++) {
1265			if (strcmp(name, events_table[i].name) == 0) {
1266				return (&events_table[i]);
1267			}
1268		}
1269	}
1270
1271	return (NULL);
1272}
1273static uint64_t
1274core_pcbe_event_coverage(char *event)
1275{
1276	uint64_t bitmap;
1277	uint64_t bitmask;
1278	const struct events_table_t *n;
1279	int i;
1280
1281	bitmap = 0;
1282
1283	/* Is it an event that a GPC can track? */
1284	if (versionid >= 3) {
1285		n = find_gpcevent(event);
1286		if (n != NULL) {
1287			bitmap |= (n->supported_counters &
1288			    BITMASK_XBITS(num_gpc));
1289		}
1290	} else {
1291		if (find_gpcevent_f6m15_23(event, cmn_gpc_events_f6m15_23) !=
1292		    NULL) {
1293			bitmap |= BITMASK_XBITS(num_gpc);
1294		} else if (find_gpcevent_f6m15_23(event, pic0_events) != NULL) {
1295			bitmap |= 1ULL;
1296		} else if (find_gpcevent_f6m15_23(event, pic1_events) != NULL) {
1297			bitmap |= 1ULL << 1;
1298		}
1299	}
1300
1301	/* Check if the event can be counted in the fixed-function counters */
1302	if (num_ffc > 0) {
1303		bitmask = 1ULL << num_gpc;
1304		for (i = 0; i < num_ffc; i++) {
1305			if (strcmp(event, ffc_names[i]) == 0) {
1306				bitmap |= bitmask;
1307			}
1308			bitmask = bitmask << 1;
1309		}
1310	}
1311
1312	return (bitmap);
1313}
1314
1315static uint64_t
1316core_pcbe_overflow_bitmap(void)
1317{
1318	uint64_t interrupt_status;
1319	uint64_t intrbits_ffc;
1320	uint64_t intrbits_gpc;
1321	extern int kcpc_hw_overflow_intr_installed;
1322	uint64_t overflow_bitmap;
1323
1324	RDMSR(PERF_GLOBAL_STATUS, interrupt_status);
1325	WRMSR(PERF_GLOBAL_OVF_CTRL, interrupt_status);
1326
1327	interrupt_status = interrupt_status & control_mask;
1328	intrbits_ffc = (interrupt_status >> 32) & control_ffc;
1329	intrbits_gpc = interrupt_status & control_gpc;
1330	overflow_bitmap = (intrbits_ffc << num_gpc) | intrbits_gpc;
1331
1332	ASSERT(kcpc_hw_overflow_intr_installed);
1333	(*kcpc_hw_enable_cpc_intr)();
1334
1335	return (overflow_bitmap);
1336}
1337
1338static int
1339check_cpc_securitypolicy(core_pcbe_config_t *conf,
1340    const struct nametable_fam6mod15_23 *n)
1341{
1342	if (conf->core_ctl & n->restricted_bits) {
1343		if (secpolicy_cpc_cpu(crgetcred()) != 0) {
1344			return (CPC_ATTR_REQUIRES_PRIVILEGE);
1345		}
1346	}
1347	return (0);
1348}
1349
1350static int
1351configure_gpc(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
1352    uint_t nattrs, kcpc_attr_t *attrs, void **data)
1353{
1354	core_pcbe_config_t	conf;
1355	const struct nametable_fam6mod15_23	*n;
1356	const struct nametable_fam6mod15_23	*m;
1357	const struct nametable_fam6mod15_23	*picspecific_events;
1358	struct nametable_fam6mod15_23	nt_raw = { "", 0x0, 0x0 };
1359	uint_t			i;
1360	long			event_num;
1361	const struct events_table_t *eventcode;
1362	int			umask_known;
1363
1364	if (((preset & BITS_EXTENDED_FROM_31) != 0) &&
1365	    ((preset & BITS_EXTENDED_FROM_31) !=
1366	    BITS_EXTENDED_FROM_31)) {
1367
1368		/*
1369		 * Bits beyond bit-31 in the general-purpose counters can only
1370		 * be written to by extension of bit 31.  We cannot preset
1371		 * these bits to any value other than all 1s or all 0s.
1372		 */
1373		return (CPC_ATTRIBUTE_OUT_OF_RANGE);
1374	}
1375
1376	if (versionid >= 3) {
1377		eventcode = find_gpcevent(event);
1378		if (eventcode != NULL) {
1379			if ((C(picnum) & eventcode->supported_counters) == 0) {
1380				return (CPC_PIC_NOT_CAPABLE);
1381			}
1382			conf.core_ctl = eventcode->eventselect;
1383			conf.core_ctl |= eventcode->unitmask <<
1384			    CORE_UMASK_SHIFT;
1385			umask_known = 1;
1386		} else {
1387			/* Event specified as raw event code */
1388			if (ddi_strtol(event, NULL, 0, &event_num) != 0) {
1389				return (CPC_INVALID_EVENT);
1390			}
1391			conf.core_ctl = event_num & 0xFF;
1392			umask_known = 0;
1393		}
1394	} else {
1395		umask_known = 0;
1396		n = find_gpcevent_f6m15_23(event, cmn_gpc_events_f6m15_23);
1397		if (n == NULL) {
1398			switch (picnum) {
1399				case 0:
1400					picspecific_events = pic0_events;
1401					break;
1402				case 1:
1403					picspecific_events = pic1_events;
1404					break;
1405				default:
1406					picspecific_events = NULL;
1407					break;
1408			}
1409			if (picspecific_events != NULL) {
1410				n = find_gpcevent_f6m15_23(event,
1411				    picspecific_events);
1412			}
1413		}
1414		if (n == NULL) {
1415			/*
1416			 * Check if this is a case where the event was
1417			 * specified directly by its event number instead of
1418			 * its name string.
1419			 */
1420			if (ddi_strtol(event, NULL, 0, &event_num) != 0) {
1421				return (CPC_INVALID_EVENT);
1422			}
1423
1424			event_num = event_num & 0xFF;
1425
1426			/*
1427			 * Search the event table to find out if the event
1428			 * specified has an privilege requirements.  Currently
1429			 * none of the pic-specific counters have any privilege
1430			 * requirements.  Hence only the table
1431			 * cmn_gpc_events_f6m15_23 is searched.
1432			 */
1433			for (m = cmn_gpc_events_f6m15_23;
1434			    m->event_num != NT_END;
1435			    m++) {
1436				if (event_num == m->event_num) {
1437					break;
1438				}
1439			}
1440			if (m->event_num == NT_END) {
1441				nt_raw.event_num = (uint8_t)event_num;
1442				n = &nt_raw;
1443			} else {
1444				n = m;
1445			}
1446		}
1447		conf.core_ctl = n->event_num; /* Event Select */
1448	}
1449
1450
1451	conf.core_picno = picnum;
1452	conf.core_pictype = CORE_GPC;
1453	conf.core_rawpic = preset & mask_gpc;
1454
1455	conf.core_pes = GPC_BASE_PES + picnum;
1456	conf.core_pmc = GPC_BASE_PMC + picnum;
1457
1458	for (i = 0; i < nattrs; i++) {
1459		if (strncmp(attrs[i].ka_name, "umask", 6) == 0) {
1460			if ((umask_known == 1) ||
1461			    ((attrs[i].ka_val | CORE_UMASK_MASK)
1462			    != CORE_UMASK_MASK)) {
1463				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
1464			}
1465
1466			conf.core_ctl |= attrs[i].ka_val <<
1467			    CORE_UMASK_SHIFT;
1468		} else  if (strncmp(attrs[i].ka_name, "edge", 6) == 0) {
1469			if (attrs[i].ka_val != 0)
1470				conf.core_ctl |= CORE_EDGE;
1471		} else if (strncmp(attrs[i].ka_name, "inv", 4) == 0) {
1472			if (attrs[i].ka_val != 0)
1473				conf.core_ctl |= CORE_INV;
1474		} else if (strncmp(attrs[i].ka_name, "cmask", 6) == 0) {
1475			if ((attrs[i].ka_val | CORE_CMASK_MASK) !=
1476			    CORE_CMASK_MASK) {
1477				return (CPC_ATTRIBUTE_OUT_OF_RANGE);
1478			}
1479			conf.core_ctl |= attrs[i].ka_val <<
1480			    CORE_CMASK_SHIFT;
1481		} else if (strncmp(attrs[i].ka_name, "anythr", 7) ==
1482		    0) {
1483			if (versionid < 3)
1484				return (CPC_INVALID_ATTRIBUTE);
1485			if (secpolicy_cpc_cpu(crgetcred()) != 0) {
1486				return (CPC_ATTR_REQUIRES_PRIVILEGE);
1487			}
1488			if (attrs[i].ka_val != 0)
1489				conf.core_ctl |= CORE_ANYTHR;
1490		} else {
1491			return (CPC_INVALID_ATTRIBUTE);
1492		}
1493	}
1494
1495	if (flags & CPC_COUNT_USER)
1496		conf.core_ctl |= CORE_USR;
1497	if (flags & CPC_COUNT_SYSTEM)
1498		conf.core_ctl |= CORE_OS;
1499	if (flags & CPC_OVF_NOTIFY_EMT)
1500		conf.core_ctl |= CORE_INT;
1501	conf.core_ctl |= CORE_EN;
1502
1503	if (versionid < 3) {
1504		if (check_cpc_securitypolicy(&conf, n) != 0) {
1505			return (CPC_ATTR_REQUIRES_PRIVILEGE);
1506		}
1507	}
1508
1509	*data = kmem_alloc(sizeof (core_pcbe_config_t), KM_SLEEP);
1510	*((core_pcbe_config_t *)*data) = conf;
1511
1512	return (0);
1513}
1514
1515static int
1516configure_ffc(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
1517    uint_t nattrs, kcpc_attr_t *attrs, void **data)
1518{
1519	core_pcbe_config_t	*conf;
1520	uint_t			i;
1521
1522	if (picnum - num_gpc >= num_ffc) {
1523		return (CPC_INVALID_PICNUM);
1524	}
1525
1526	if (strcmp(ffc_names[picnum-num_gpc], event) != 0) {
1527		return (CPC_INVALID_EVENT);
1528	}
1529
1530	if ((versionid < 3) && (nattrs != 0)) {
1531		return (CPC_INVALID_ATTRIBUTE);
1532	}
1533
1534	conf = kmem_alloc(sizeof (core_pcbe_config_t), KM_SLEEP);
1535	conf->core_ctl = 0;
1536
1537	for (i = 0; i < nattrs; i++) {
1538		if (strncmp(attrs[i].ka_name, "anythr", 7) == 0) {
1539			if (secpolicy_cpc_cpu(crgetcred()) != 0) {
1540				return (CPC_ATTR_REQUIRES_PRIVILEGE);
1541			}
1542			if (attrs[i].ka_val != 0) {
1543				conf->core_ctl |= CORE_FFC_ANYTHR;
1544			}
1545		} else {
1546			kmem_free(conf, sizeof (core_pcbe_config_t));
1547			return (CPC_INVALID_ATTRIBUTE);
1548		}
1549	}
1550
1551	conf->core_picno = picnum;
1552	conf->core_pictype = CORE_FFC;
1553	conf->core_rawpic = preset & mask_ffc;
1554	conf->core_pmc = FFC_BASE_PMC + (picnum - num_gpc);
1555
1556	/* All fixed-function counters have the same control register */
1557	conf->core_pes = PERF_FIXED_CTR_CTRL;
1558
1559	if (flags & CPC_COUNT_USER)
1560		conf->core_ctl |= CORE_FFC_USR_EN;
1561	if (flags & CPC_COUNT_SYSTEM)
1562		conf->core_ctl |= CORE_FFC_OS_EN;
1563	if (flags & CPC_OVF_NOTIFY_EMT)
1564		conf->core_ctl |= CORE_FFC_PMI;
1565
1566	*data = conf;
1567	return (0);
1568}
1569
1570/*ARGSUSED*/
1571static int
1572core_pcbe_configure(uint_t picnum, char *event, uint64_t preset,
1573    uint32_t flags, uint_t nattrs, kcpc_attr_t *attrs, void **data,
1574    void *token)
1575{
1576	int			ret;
1577	core_pcbe_config_t	*conf;
1578
1579	/*
1580	 * If we've been handed an existing configuration, we need only preset
1581	 * the counter value.
1582	 */
1583	if (*data != NULL) {
1584		conf = *data;
1585		ASSERT(conf->core_pictype == CORE_GPC ||
1586		    conf->core_pictype == CORE_FFC);
1587		if (conf->core_pictype == CORE_GPC)
1588			conf->core_rawpic = preset & mask_gpc;
1589		else /* CORE_FFC */
1590			conf->core_rawpic = preset & mask_ffc;
1591		return (0);
1592	}
1593
1594	if (picnum >= total_pmc) {
1595		return (CPC_INVALID_PICNUM);
1596	}
1597
1598	if (picnum < num_gpc) {
1599		ret = configure_gpc(picnum, event, preset, flags,
1600		    nattrs, attrs, data);
1601	} else {
1602		ret = configure_ffc(picnum, event, preset, flags,
1603		    nattrs, attrs, data);
1604	}
1605	return (ret);
1606}
1607
1608static void
1609core_pcbe_program(void *token)
1610{
1611	core_pcbe_config_t	*cfg;
1612	uint64_t		perf_global_ctrl;
1613	uint64_t		perf_fixed_ctr_ctrl;
1614	uint64_t		curcr4;
1615
1616	core_pcbe_allstop();
1617
1618	curcr4 = getcr4();
1619	if (kcpc_allow_nonpriv(token))
1620		/* Allow RDPMC at any ring level */
1621		setcr4(curcr4 | CR4_PCE);
1622	else
1623		/* Allow RDPMC only at ring 0 */
1624		setcr4(curcr4 & ~CR4_PCE);
1625
1626	/* Clear any overflow indicators before programming the counters */
1627	WRMSR(PERF_GLOBAL_OVF_CTRL, MASK_CONDCHGD_OVFBUFFER | control_mask);
1628
1629	cfg = NULL;
1630	perf_global_ctrl = 0;
1631	perf_fixed_ctr_ctrl = 0;
1632	cfg = (core_pcbe_config_t *)kcpc_next_config(token, cfg, NULL);
1633	while (cfg != NULL) {
1634		ASSERT(cfg->core_pictype == CORE_GPC ||
1635		    cfg->core_pictype == CORE_FFC);
1636
1637		if (cfg->core_pictype == CORE_GPC) {
1638			/*
1639			 * General-purpose counter registers have write
1640			 * restrictions where only the lower 32-bits can be
1641			 * written to.  The rest of the relevant bits are
1642			 * written to by extension from bit 31 (all ZEROS if
1643			 * bit-31 is ZERO and all ONE if bit-31 is ONE).  This
1644			 * makes it possible to write to the counter register
1645			 * only values that have all ONEs or all ZEROs in the
1646			 * higher bits.
1647			 */
1648			if (((cfg->core_rawpic & BITS_EXTENDED_FROM_31) == 0) ||
1649			    ((cfg->core_rawpic & BITS_EXTENDED_FROM_31) ==
1650			    BITS_EXTENDED_FROM_31)) {
1651				/*
1652				 * Straighforward case where the higher bits
1653				 * are all ZEROs or all ONEs.
1654				 */
1655				WRMSR(cfg->core_pmc,
1656				    (cfg->core_rawpic & mask_gpc));
1657			} else {
1658				/*
1659				 * The high order bits are not all the same.
1660				 * We save what is currently in the registers
1661				 * and do not write to it.  When we want to do
1662				 * a read from this register later (in
1663				 * core_pcbe_sample()), we subtract the value
1664				 * we save here to get the actual event count.
1665				 *
1666				 * NOTE: As a result, we will not get overflow
1667				 * interrupts as expected.
1668				 */
1669				RDMSR(cfg->core_pmc, cfg->core_rawpic);
1670				cfg->core_rawpic = cfg->core_rawpic & mask_gpc;
1671			}
1672			WRMSR(cfg->core_pes, cfg->core_ctl);
1673			perf_global_ctrl |= 1ull << cfg->core_picno;
1674		} else {
1675			/*
1676			 * Unlike the general-purpose counters, all relevant
1677			 * bits of fixed-function counters can be written to.
1678			 */
1679			WRMSR(cfg->core_pmc, cfg->core_rawpic & mask_ffc);
1680
1681			/*
1682			 * Collect the control bits for all the
1683			 * fixed-function counters and write it at one shot
1684			 * later in this function
1685			 */
1686			perf_fixed_ctr_ctrl |= cfg->core_ctl <<
1687			    ((cfg->core_picno - num_gpc) * CORE_FFC_ATTR_SIZE);
1688			perf_global_ctrl |=
1689			    1ull << (cfg->core_picno - num_gpc + 32);
1690		}
1691
1692		cfg = (core_pcbe_config_t *)
1693		    kcpc_next_config(token, cfg, NULL);
1694	}
1695
1696	/* Enable all the counters */
1697	WRMSR(PERF_FIXED_CTR_CTRL, perf_fixed_ctr_ctrl);
1698	WRMSR(PERF_GLOBAL_CTRL, perf_global_ctrl);
1699}
1700
1701static void
1702core_pcbe_allstop(void)
1703{
1704	/* Disable all the counters together */
1705	WRMSR(PERF_GLOBAL_CTRL, ALL_STOPPED);
1706
1707	setcr4(getcr4() & ~CR4_PCE);
1708}
1709
1710static void
1711core_pcbe_sample(void *token)
1712{
1713	uint64_t		*daddr;
1714	uint64_t		curpic;
1715	core_pcbe_config_t	*cfg;
1716	uint64_t			counter_mask;
1717
1718	cfg = (core_pcbe_config_t *)kcpc_next_config(token, NULL, &daddr);
1719	while (cfg != NULL) {
1720		ASSERT(cfg->core_pictype == CORE_GPC ||
1721		    cfg->core_pictype == CORE_FFC);
1722
1723		curpic = rdmsr(cfg->core_pmc);
1724
1725		DTRACE_PROBE4(core__pcbe__sample,
1726		    uint64_t, cfg->core_pmc,
1727		    uint64_t, curpic,
1728		    uint64_t, cfg->core_rawpic,
1729		    uint64_t, *daddr);
1730
1731		if (cfg->core_pictype == CORE_GPC) {
1732			counter_mask = mask_gpc;
1733		} else {
1734			counter_mask = mask_ffc;
1735		}
1736		curpic = curpic & counter_mask;
1737		if (curpic >= cfg->core_rawpic) {
1738			*daddr += curpic - cfg->core_rawpic;
1739		} else {
1740			/* Counter overflowed since our last sample */
1741			*daddr += counter_mask - (cfg->core_rawpic - curpic) +
1742			    1;
1743		}
1744		cfg->core_rawpic = *daddr & counter_mask;
1745
1746		cfg =
1747		    (core_pcbe_config_t *)kcpc_next_config(token, cfg, &daddr);
1748	}
1749}
1750
1751static void
1752core_pcbe_free(void *config)
1753{
1754	kmem_free(config, sizeof (core_pcbe_config_t));
1755}
1756
1757static struct modlpcbe core_modlpcbe = {
1758	&mod_pcbeops,
1759	"Core Performance Counters",
1760	&core_pcbe_ops
1761};
1762
1763static struct modlinkage core_modl = {
1764	MODREV_1,
1765	&core_modlpcbe,
1766};
1767
1768int
1769_init(void)
1770{
1771	if (core_pcbe_init() != 0) {
1772		return (ENOTSUP);
1773	}
1774	return (mod_install(&core_modl));
1775}
1776
1777int
1778_fini(void)
1779{
1780	return (mod_remove(&core_modl));
1781}
1782
1783int
1784_info(struct modinfo *mi)
1785{
1786	return (mod_info(&core_modl, mi));
1787}
1788