opteron_pcbe.c (b885580b) opteron_pcbe.c (d0e58ef5)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 51 unchanged lines hidden (view full) ---

60 * POSSIBILITY OF SUCH DAMAGE.
61 *
62 *
63 * This open source software license conforms to the BSD License template.
64 */
65
66/*
67 * Portions Copyright 2009 Advanced Micro Devices, Inc.
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE

--- 51 unchanged lines hidden (view full) ---

60 * POSSIBILITY OF SUCH DAMAGE.
61 *
62 *
63 * This open source software license conforms to the BSD License template.
64 */
65
66/*
67 * Portions Copyright 2009 Advanced Micro Devices, Inc.
68 * Copyright 2019 Joyent, Inc.
68 */
69
70/*
71 * Performance Counter Back-End for AMD Opteron and AMD Athlon 64 processors.
72 */
73
74#include <sys/cpuvar.h>
75#include <sys/param.h>

--- 5 unchanged lines hidden (view full) ---

81#include <sys/errno.h>
82#include <sys/debug.h>
83#include <sys/archsystm.h>
84#include <sys/x86_archext.h>
85#include <sys/privregs.h>
86#include <sys/ddi.h>
87#include <sys/sunddi.h>
88
69 */
70
71/*
72 * Performance Counter Back-End for AMD Opteron and AMD Athlon 64 processors.
73 */
74
75#include <sys/cpuvar.h>
76#include <sys/param.h>

--- 5 unchanged lines hidden (view full) ---

82#include <sys/errno.h>
83#include <sys/debug.h>
84#include <sys/archsystm.h>
85#include <sys/x86_archext.h>
86#include <sys/privregs.h>
87#include <sys/ddi.h>
88#include <sys/sunddi.h>
89
90#include "opteron_pcbe_table.h"
91#include <opteron_pcbe_cpcgen.h>
92
89static int opt_pcbe_init(void);
90static uint_t opt_pcbe_ncounters(void);
91static const char *opt_pcbe_impl_name(void);
92static const char *opt_pcbe_cpuref(void);
93static char *opt_pcbe_list_events(uint_t picnum);
94static char *opt_pcbe_list_attrs(void);
95static uint64_t opt_pcbe_event_coverage(char *event);
96static uint64_t opt_pcbe_overflow_bitmap(void);

--- 18 unchanged lines hidden (view full) ---

115 opt_pcbe_configure,
116 opt_pcbe_program,
117 opt_pcbe_allstop,
118 opt_pcbe_sample,
119 opt_pcbe_free
120};
121
122/*
93static int opt_pcbe_init(void);
94static uint_t opt_pcbe_ncounters(void);
95static const char *opt_pcbe_impl_name(void);
96static const char *opt_pcbe_cpuref(void);
97static char *opt_pcbe_list_events(uint_t picnum);
98static char *opt_pcbe_list_attrs(void);
99static uint64_t opt_pcbe_event_coverage(char *event);
100static uint64_t opt_pcbe_overflow_bitmap(void);

--- 18 unchanged lines hidden (view full) ---

119 opt_pcbe_configure,
120 opt_pcbe_program,
121 opt_pcbe_allstop,
122 opt_pcbe_sample,
123 opt_pcbe_free
124};
125
126/*
127 * Base MSR addresses for the PerfEvtSel registers and the counters themselves.
128 * Add counter number to base address to get corresponding MSR address.
129 */
130#define PES_BASE_ADDR 0xC0010000
131#define PIC_BASE_ADDR 0xC0010004
132
133/*
134 * Base MSR addresses for the PerfEvtSel registers and counters. The counter and
135 * event select registers are interleaved, so one needs to multiply the counter
136 * number by two to determine what they should be set to.
137 */
138#define PES_EXT_BASE_ADDR 0xC0010200
139#define PIC_EXT_BASE_ADDR 0xC0010201
140
141/*
142 * The number of counters present depends on which CPU features are present.
143 */
144#define OPT_PCBE_DEF_NCOUNTERS 4
145#define OPT_PCBE_EXT_NCOUNTERS 6
146
147/*
123 * Define offsets and masks for the fields in the Performance
124 * Event-Select (PES) registers.
125 */
126#define OPT_PES_HOST_SHIFT 41
127#define OPT_PES_GUEST_SHIFT 40
148 * Define offsets and masks for the fields in the Performance
149 * Event-Select (PES) registers.
150 */
151#define OPT_PES_HOST_SHIFT 41
152#define OPT_PES_GUEST_SHIFT 40
153#define OPT_PES_EVSELHI_SHIFT 32
128#define OPT_PES_CMASK_SHIFT 24
129#define OPT_PES_CMASK_MASK 0xFF
130#define OPT_PES_INV_SHIFT 23
131#define OPT_PES_ENABLE_SHIFT 22
132#define OPT_PES_INT_SHIFT 20
133#define OPT_PES_PC_SHIFT 19
134#define OPT_PES_EDGE_SHIFT 18
135#define OPT_PES_OS_SHIFT 17

--- 12 unchanged lines hidden (view full) ---

148#define OPT_PES_GUEST (1ULL << OPT_PES_GUEST_SHIFT)
149
150typedef struct _opt_pcbe_config {
151 uint8_t opt_picno; /* Counter number: 0, 1, 2, or 3 */
152 uint64_t opt_evsel; /* Event Selection register */
153 uint64_t opt_rawpic; /* Raw counter value */
154} opt_pcbe_config_t;
155
154#define OPT_PES_CMASK_SHIFT 24
155#define OPT_PES_CMASK_MASK 0xFF
156#define OPT_PES_INV_SHIFT 23
157#define OPT_PES_ENABLE_SHIFT 22
158#define OPT_PES_INT_SHIFT 20
159#define OPT_PES_PC_SHIFT 19
160#define OPT_PES_EDGE_SHIFT 18
161#define OPT_PES_OS_SHIFT 17

--- 12 unchanged lines hidden (view full) ---

174#define OPT_PES_GUEST (1ULL << OPT_PES_GUEST_SHIFT)
175
176typedef struct _opt_pcbe_config {
177 uint8_t opt_picno; /* Counter number: 0, 1, 2, or 3 */
178 uint64_t opt_evsel; /* Event Selection register */
179 uint64_t opt_rawpic; /* Raw counter value */
180} opt_pcbe_config_t;
181
156opt_pcbe_config_t nullcfgs[4] = {
182opt_pcbe_config_t nullcfgs[OPT_PCBE_EXT_NCOUNTERS] = {
157 { 0, 0, 0 },
158 { 1, 0, 0 },
159 { 2, 0, 0 },
183 { 0, 0, 0 },
184 { 1, 0, 0 },
185 { 2, 0, 0 },
160 { 3, 0, 0 }
186 { 3, 0, 0 },
187 { 4, 0, 0 },
188 { 5, 0, 0 },
161};
162
189};
190
163typedef struct _amd_event {
164 char *name;
165 uint16_t emask; /* Event mask setting */
166} amd_event_t;
191typedef uint64_t (*opt_pcbe_addr_f)(uint_t);
167
192
168typedef struct _amd_generic_event {
169 char *name;
170 char *event;
171 uint8_t umask;
172} amd_generic_event_t;
193typedef struct opt_pcbe_data {
194 uint_t opd_ncounters;
195 uint_t opd_cmask;
196 opt_pcbe_addr_f opd_pesf;
197 opt_pcbe_addr_f opd_picf;
198} opt_pcbe_data_t;
173
199
174/*
175 * Base MSR addresses for the PerfEvtSel registers and the counters themselves.
176 * Add counter number to base address to get corresponding MSR address.
177 */
178#define PES_BASE_ADDR 0xC0010000
179#define PIC_BASE_ADDR 0xC0010004
200opt_pcbe_data_t opd;
180
181#define MASK48 0xFFFFFFFFFFFF
182
183#define EV_END {NULL, 0}
184#define GEN_EV_END {NULL, NULL, 0 }
185
201
202#define MASK48 0xFFFFFFFFFFFF
203
204#define EV_END {NULL, 0}
205#define GEN_EV_END {NULL, NULL, 0 }
206
207/*
208 * The following Macros are used to define tables of events that are used by
209 * various families and some generic classes of events.
210 *
211 * When programming a performance counter there are two different values that we
212 * need to set:
213 *
214 * o Event - Determines the general class of event that is being used.
215 * o Unit - A further breakdown that gives more specific value.
216 *
217 * Prior to the introduction of family 17h support, all family specific events
218 * were programmed based on their event. The generic events, which tried to
219 * provide PAPI mappings to events specified an additional unit mask.
220 *
221 * Starting with Family 17h, CPU performance counters default to using both the
222 * unit mask and the event select. Generic events are always aliases to a
223 * specific event/unit pair, hence why the units for them are always zero. In
224 * addition, the naming of events in family 17h has been changed to reflect
225 * AMD's guide. While this is a departure from what people are used to, it is
226 * believed that matching the more detailed literature that folks are told to
227 * reference is more valuable.
228 */
229
186#define AMD_cmn_events \
187 { "FP_dispatched_fpu_ops", 0x0 }, \
188 { "FP_cycles_no_fpu_ops_retired", 0x1 }, \
189 { "FP_dispatched_fpu_ops_ff", 0x2 }, \
190 { "LS_seg_reg_load", 0x20 }, \
191 { "LS_uarch_resync_self_modify", 0x21 }, \
192 { "LS_uarch_resync_snoop", 0x22 }, \
193 { "LS_buffer_2_full", 0x23 }, \

--- 173 unchanged lines hidden (view full) ---

367 { "PAPI_tlb_im", "IC_itlb_L1_miss_L2_miss", 0x3 }, \
368 { "PAPI_l3_dcr", "L3_read_req", 0xf1 }, \
369 { "PAPI_l3_icr", "L3_read_req", 0xf2 }, \
370 { "PAPI_l3_tcr", "L3_read_req", 0xf7 }, \
371 { "PAPI_l3_stm", "L3_miss", 0xf4 }, \
372 { "PAPI_l3_ldm", "L3_miss", 0xf3 }, \
373 { "PAPI_l3_tcm", "L3_miss", 0xf7 }
374
230#define AMD_cmn_events \
231 { "FP_dispatched_fpu_ops", 0x0 }, \
232 { "FP_cycles_no_fpu_ops_retired", 0x1 }, \
233 { "FP_dispatched_fpu_ops_ff", 0x2 }, \
234 { "LS_seg_reg_load", 0x20 }, \
235 { "LS_uarch_resync_self_modify", 0x21 }, \
236 { "LS_uarch_resync_snoop", 0x22 }, \
237 { "LS_buffer_2_full", 0x23 }, \

--- 173 unchanged lines hidden (view full) ---

411 { "PAPI_tlb_im", "IC_itlb_L1_miss_L2_miss", 0x3 }, \
412 { "PAPI_l3_dcr", "L3_read_req", 0xf1 }, \
413 { "PAPI_l3_icr", "L3_read_req", 0xf2 }, \
414 { "PAPI_l3_tcr", "L3_read_req", 0xf7 }, \
415 { "PAPI_l3_stm", "L3_miss", 0xf4 }, \
416 { "PAPI_l3_ldm", "L3_miss", 0xf3 }, \
417 { "PAPI_l3_tcm", "L3_miss", 0xf7 }
418
375#define AMD_PCBE_SUPPORTED(family) (((family) >= 0xf) && ((family) <= 0x11))
376
377static amd_event_t family_f_events[] = {
419static const amd_event_t family_f_events[] = {
378 AMD_cmn_events,
379 AMD_FAMILY_f_events,
380 EV_END
381};
382
420 AMD_cmn_events,
421 AMD_FAMILY_f_events,
422 EV_END
423};
424
383static amd_event_t family_10h_events[] = {
425static const amd_event_t family_10h_events[] = {
384 AMD_cmn_events,
385 AMD_FAMILY_10h_events,
386 EV_END
387};
388
426 AMD_cmn_events,
427 AMD_FAMILY_10h_events,
428 EV_END
429};
430
389static amd_event_t family_11h_events[] = {
431static const amd_event_t family_11h_events[] = {
390 AMD_cmn_events,
391 AMD_FAMILY_11h_events,
392 EV_END
393};
394
432 AMD_cmn_events,
433 AMD_FAMILY_11h_events,
434 EV_END
435};
436
395static amd_generic_event_t opt_generic_events[] = {
437static const amd_generic_event_t opt_generic_events[] = {
396 AMD_cmn_generic_events,
397 OPT_cmn_generic_events,
398 GEN_EV_END
399};
400
438 AMD_cmn_generic_events,
439 OPT_cmn_generic_events,
440 GEN_EV_END
441};
442
401static amd_generic_event_t family_10h_generic_events[] = {
443static const amd_generic_event_t family_10h_generic_events[] = {
402 AMD_cmn_generic_events,
403 AMD_FAMILY_10h_generic_events,
404 GEN_EV_END
405};
406
444 AMD_cmn_generic_events,
445 AMD_FAMILY_10h_generic_events,
446 GEN_EV_END
447};
448
449/*
450 * For Family 17h, the cpcgen utility generates all of our events including ones
451 * that need specific unit codes, therefore we leave all unit codes out of
452 * these.
453 */
454static const amd_generic_event_t family_17h_papi_events[] = {
455 { "PAPI_br_cn", "ExRetCond" },
456 { "PAPI_br_ins", "ExRetBrnMis" },
457 { "PAPI_fpu_idl", "FpSchedEmpty" },
458 { "PAPI_tot_cyc", "LsNotHaltedCyc" },
459 { "PAPI_tot_ins", "ExRetInstr" },
460 { "PAPI_tlb_dm", "LsL1DTlbMiss" },
461 { "PAPI_tlb_im", "BpL1TlbMissL2Miss" },
462 { "PAPI_tot_cyc", "LsNotHaltedCyc" },
463 GEN_EV_END
464};
465
407static char *evlist;
408static size_t evlist_sz;
466static char *evlist;
467static size_t evlist_sz;
409static amd_event_t *amd_events = NULL;
410static uint_t amd_family;
411static amd_generic_event_t *amd_generic_events = NULL;
468static const amd_event_t *amd_events = NULL;
469static uint_t amd_family, amd_model;
470static const amd_generic_event_t *amd_generic_events = NULL;
412
471
413#define AMD_CPUREF_SIZE 256
414static char amd_generic_bkdg[AMD_CPUREF_SIZE];
415static char amd_fam_f_rev_ae_bkdg[] = "See \"BIOS and Kernel Developer's " \
472static char amd_fam_f_rev_ae_bkdg[] = "See \"BIOS and Kernel Developer's "
416"Guide for AMD Athlon 64 and AMD Opteron Processors\" (AMD publication 26094)";
473"Guide for AMD Athlon 64 and AMD Opteron Processors\" (AMD publication 26094)";
417static char amd_fam_f_NPT_bkdg[] = "See \"BIOS and Kernel Developer's Guide " \
474static char amd_fam_f_NPT_bkdg[] = "See \"BIOS and Kernel Developer's Guide "
418"for AMD NPT Family 0Fh Processors\" (AMD publication 32559)";
475"for AMD NPT Family 0Fh Processors\" (AMD publication 32559)";
419static char amd_fam_10h_bkdg[] = "See \"BIOS and Kernel Developer's Guide " \
476static char amd_fam_10h_bkdg[] = "See \"BIOS and Kernel Developer's Guide "
420"(BKDG) For AMD Family 10h Processors\" (AMD publication 31116)";
477"(BKDG) For AMD Family 10h Processors\" (AMD publication 31116)";
421static char amd_fam_11h_bkdg[] = "See \"BIOS and Kernel Developer's Guide " \
478static char amd_fam_11h_bkdg[] = "See \"BIOS and Kernel Developer's Guide "
422"(BKDG) For AMD Family 11h Processors\" (AMD publication 41256)";
479"(BKDG) For AMD Family 11h Processors\" (AMD publication 41256)";
480static char amd_fam_17h_reg[] = "See \"Open-Source Register Reference For "
481"AMD Family 17h Processors Models 00h-2Fh\" (AMD publication 56255) and "
482"amd_f17h_events(3CPC)";
423
424static char amd_pcbe_impl_name[64];
425static char *amd_pcbe_cpuref;
426
427
428#define BITS(v, u, l) \
429 (((v) >> (l)) & ((1 << (1 + (u) - (l))) - 1))
430
483
484static char amd_pcbe_impl_name[64];
485static char *amd_pcbe_cpuref;
486
487
488#define BITS(v, u, l) \
489 (((v) >> (l)) & ((1 << (1 + (u) - (l))) - 1))
490
491static uint64_t
492opt_pcbe_pes_addr(uint_t counter)
493{
494 ASSERT3U(counter, <, opd.opd_ncounters);
495 return (PES_BASE_ADDR + counter);
496}
431
497
498static uint64_t
499opt_pcbe_pes_ext_addr(uint_t counter)
500{
501 ASSERT3U(counter, <, opd.opd_ncounters);
502 return (PES_EXT_BASE_ADDR + 2 * counter);
503}
504
505static uint64_t
506opt_pcbe_pic_addr(uint_t counter)
507{
508 ASSERT3U(counter, <, opd.opd_ncounters);
509 return (PIC_BASE_ADDR + 2 * counter);
510}
511
512static uint64_t
513opt_pcbe_pic_ext_addr(uint_t counter)
514{
515 ASSERT3U(counter, <, opd.opd_ncounters);
516 return (PIC_EXT_BASE_ADDR + 2 * counter);
517}
518
432static int
433opt_pcbe_init(void)
434{
519static int
520opt_pcbe_init(void)
521{
435 amd_event_t *evp;
436 amd_generic_event_t *gevp;
522 const amd_event_t *evp;
523 const amd_generic_event_t *gevp;
437
438 amd_family = cpuid_getfamily(CPU);
524
525 amd_family = cpuid_getfamily(CPU);
526 amd_model = cpuid_getmodel(CPU);
439
440 /*
441 * Make sure this really _is_ an Opteron or Athlon 64 system. The kernel
442 * loads this module based on its name in the module directory, but it
443 * could have been renamed.
444 */
445 if (cpuid_getvendor(CPU) != X86_VENDOR_AMD || amd_family < 0xf)
446 return (-1);
447
527
528 /*
529 * Make sure this really _is_ an Opteron or Athlon 64 system. The kernel
530 * loads this module based on its name in the module directory, but it
531 * could have been renamed.
532 */
533 if (cpuid_getvendor(CPU) != X86_VENDOR_AMD || amd_family < 0xf)
534 return (-1);
535
448 if (amd_family == 0xf)
536 if (amd_family == 0xf) {
449 /* Some tools expect this string for family 0fh */
450 (void) snprintf(amd_pcbe_impl_name, sizeof (amd_pcbe_impl_name),
451 "AMD Opteron & Athlon64");
537 /* Some tools expect this string for family 0fh */
538 (void) snprintf(amd_pcbe_impl_name, sizeof (amd_pcbe_impl_name),
539 "AMD Opteron & Athlon64");
452 else
540 } else {
453 (void) snprintf(amd_pcbe_impl_name, sizeof (amd_pcbe_impl_name),
541 (void) snprintf(amd_pcbe_impl_name, sizeof (amd_pcbe_impl_name),
454 "AMD Family %02xh%s", amd_family,
455 AMD_PCBE_SUPPORTED(amd_family) ? "" :" (unsupported)");
542 "AMD Family %02xh", amd_family);
543 }
456
457 /*
544
545 /*
546 * Determine whether or not the extended counter set is supported on
547 * this processor.
548 */
549 if (is_x86_feature(x86_featureset, X86FSET_AMD_PCEC)) {
550 opd.opd_ncounters = OPT_PCBE_EXT_NCOUNTERS;
551 opd.opd_pesf = opt_pcbe_pes_ext_addr;
552 opd.opd_picf = opt_pcbe_pic_ext_addr;
553 } else {
554 opd.opd_ncounters = OPT_PCBE_DEF_NCOUNTERS;
555 opd.opd_pesf = opt_pcbe_pes_addr;
556 opd.opd_picf = opt_pcbe_pic_addr;
557 }
558 opd.opd_cmask = (1 << opd.opd_ncounters) - 1;
559
560 /*
458 * Figure out processor revision here and assign appropriate
459 * event configuration.
460 */
461
462 if (amd_family == 0xf) {
463 uint32_t rev;
464
465 rev = cpuid_getchiprev(CPU);

--- 7 unchanged lines hidden (view full) ---

473 } else if (amd_family == 0x10) {
474 amd_pcbe_cpuref = amd_fam_10h_bkdg;
475 amd_events = family_10h_events;
476 amd_generic_events = family_10h_generic_events;
477 } else if (amd_family == 0x11) {
478 amd_pcbe_cpuref = amd_fam_11h_bkdg;
479 amd_events = family_11h_events;
480 amd_generic_events = opt_generic_events;
561 * Figure out processor revision here and assign appropriate
562 * event configuration.
563 */
564
565 if (amd_family == 0xf) {
566 uint32_t rev;
567
568 rev = cpuid_getchiprev(CPU);

--- 7 unchanged lines hidden (view full) ---

576 } else if (amd_family == 0x10) {
577 amd_pcbe_cpuref = amd_fam_10h_bkdg;
578 amd_events = family_10h_events;
579 amd_generic_events = family_10h_generic_events;
580 } else if (amd_family == 0x11) {
581 amd_pcbe_cpuref = amd_fam_11h_bkdg;
582 amd_events = family_11h_events;
583 amd_generic_events = opt_generic_events;
584 } else if (amd_family == 0x17 && amd_model <= 0x2f) {
585 amd_pcbe_cpuref = amd_fam_17h_reg;
586 amd_events = opteron_pcbe_f17h_events;
587 amd_generic_events = family_17h_papi_events;
481 } else {
588 } else {
482
483 amd_pcbe_cpuref = amd_generic_bkdg;
484 (void) snprintf(amd_pcbe_cpuref, AMD_CPUREF_SIZE,
485 "See BIOS and Kernel Developer's Guide " \
486 "(BKDG) For AMD Family %02xh Processors. " \
487 "(Note that this pcbe does not explicitly " \
488 "support this family)", amd_family);
489
490 /*
589 /*
491 * For families that are not explicitly supported we'll use
492 * events for family 0xf. Even if they are not quite right,
493 * it's OK --- we state that pcbe is unsupported.
590 * Different families have different meanings on events and even
591 * worse (like family 15h), different constraints around
592 * programming these values.
494 */
593 */
495 amd_events = family_f_events;
496 amd_generic_events = opt_generic_events;
594 return (-1);
497 }
498
499 /*
500 * Construct event list.
501 *
502 * First pass: Calculate size needed. We'll need an additional byte
503 * for the NULL pointer during the last strcat.
504 *

--- 24 unchanged lines hidden (view full) ---

529 evlist[evlist_sz - 1] = '\0';
530
531 return (0);
532}
533
534static uint_t
535opt_pcbe_ncounters(void)
536{
595 }
596
597 /*
598 * Construct event list.
599 *
600 * First pass: Calculate size needed. We'll need an additional byte
601 * for the NULL pointer during the last strcat.
602 *

--- 24 unchanged lines hidden (view full) ---

627 evlist[evlist_sz - 1] = '\0';
628
629 return (0);
630}
631
632static uint_t
633opt_pcbe_ncounters(void)
634{
537 return (4);
635 return (opd.opd_ncounters);
538}
539
540static const char *
541opt_pcbe_impl_name(void)
542{
543 return (amd_pcbe_impl_name);
544}
545

--- 12 unchanged lines hidden (view full) ---

558}
559
560static char *
561opt_pcbe_list_attrs(void)
562{
563 return ("edge,pc,inv,cmask,umask");
564}
565
636}
637
638static const char *
639opt_pcbe_impl_name(void)
640{
641 return (amd_pcbe_impl_name);
642}
643

--- 12 unchanged lines hidden (view full) ---

656}
657
658static char *
659opt_pcbe_list_attrs(void)
660{
661 return ("edge,pc,inv,cmask,umask");
662}
663
566static amd_generic_event_t *
664static const amd_generic_event_t *
567find_generic_event(char *name)
568{
665find_generic_event(char *name)
666{
569 amd_generic_event_t *gevp;
667 const amd_generic_event_t *gevp;
570
571 for (gevp = amd_generic_events; gevp->name != NULL; gevp++)
572 if (strcmp(name, gevp->name) == 0)
573 return (gevp);
574
575 return (NULL);
576}
577
668
669 for (gevp = amd_generic_events; gevp->name != NULL; gevp++)
670 if (strcmp(name, gevp->name) == 0)
671 return (gevp);
672
673 return (NULL);
674}
675
578static amd_event_t *
676static const amd_event_t *
579find_event(char *name)
580{
677find_event(char *name)
678{
581 amd_event_t *evp;
679 const amd_event_t *evp;
582
583 for (evp = amd_events; evp->name != NULL; evp++)
584 if (strcmp(name, evp->name) == 0)
585 return (evp);
586
587 return (NULL);
588}
589

--- 5 unchanged lines hidden (view full) ---

595 * Check whether counter event is supported
596 */
597 if (find_event(event) == NULL && find_generic_event(event) == NULL)
598 return (0);
599
600 /*
601 * Fortunately, all counters can count all events.
602 */
680
681 for (evp = amd_events; evp->name != NULL; evp++)
682 if (strcmp(name, evp->name) == 0)
683 return (evp);
684
685 return (NULL);
686}
687

--- 5 unchanged lines hidden (view full) ---

693 * Check whether counter event is supported
694 */
695 if (find_event(event) == NULL && find_generic_event(event) == NULL)
696 return (0);
697
698 /*
699 * Fortunately, all counters can count all events.
700 */
603 return (0xF);
701 return (opd.opd_cmask);
604}
605
606static uint64_t
607opt_pcbe_overflow_bitmap(void)
608{
609 /*
610 * Unfortunately, this chip cannot detect which counter overflowed, so
611 * we must act as if they all did.
612 */
702}
703
704static uint64_t
705opt_pcbe_overflow_bitmap(void)
706{
707 /*
708 * Unfortunately, this chip cannot detect which counter overflowed, so
709 * we must act as if they all did.
710 */
613 return (0xF);
711 return (opd.opd_cmask);
614}
615
616/*ARGSUSED*/
617static int
618opt_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
619 uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token)
620{
712}
713
714/*ARGSUSED*/
715static int
716opt_pcbe_configure(uint_t picnum, char *event, uint64_t preset, uint32_t flags,
717 uint_t nattrs, kcpc_attr_t *attrs, void **data, void *token)
718{
621 opt_pcbe_config_t *cfg;
622 amd_event_t *evp;
623 amd_event_t ev_raw = { "raw", 0};
624 amd_generic_event_t *gevp;
625 int i;
626 uint64_t evsel = 0, evsel_tmp = 0;
719 opt_pcbe_config_t *cfg;
720 const amd_event_t *evp;
721 amd_event_t ev_raw = { "raw", 0};
722 const amd_generic_event_t *gevp;
723 int i;
724 uint64_t evsel = 0, evsel_tmp = 0;
627
628 /*
629 * If we've been handed an existing configuration, we need only preset
630 * the counter value.
631 */
632 if (*data != NULL) {
633 cfg = *data;
634 cfg->opt_rawpic = preset & MASK48;
635 return (0);
636 }
637
725
726 /*
727 * If we've been handed an existing configuration, we need only preset
728 * the counter value.
729 */
730 if (*data != NULL) {
731 cfg = *data;
732 cfg->opt_rawpic = preset & MASK48;
733 return (0);
734 }
735
638 if (picnum >= 4)
736 if (picnum >= opd.opd_ncounters)
639 return (CPC_INVALID_PICNUM);
640
641 if ((evp = find_event(event)) == NULL) {
642 if ((gevp = find_generic_event(event)) != NULL) {
643 evp = find_event(gevp->event);
644 ASSERT(evp != NULL);
645
646 if (nattrs > 0)

--- 22 unchanged lines hidden (view full) ---

669 */
670
671 /* Set GuestOnly bit to 0 and HostOnly bit to 1 */
672 evsel &= ~OPT_PES_HOST;
673 evsel &= ~OPT_PES_GUEST;
674
675 /* Set bits [35:32] for extended part of Event Select field */
676 evsel_tmp = evp->emask & 0x0f00;
737 return (CPC_INVALID_PICNUM);
738
739 if ((evp = find_event(event)) == NULL) {
740 if ((gevp = find_generic_event(event)) != NULL) {
741 evp = find_event(gevp->event);
742 ASSERT(evp != NULL);
743
744 if (nattrs > 0)

--- 22 unchanged lines hidden (view full) ---

767 */
768
769 /* Set GuestOnly bit to 0 and HostOnly bit to 1 */
770 evsel &= ~OPT_PES_HOST;
771 evsel &= ~OPT_PES_GUEST;
772
773 /* Set bits [35:32] for extended part of Event Select field */
774 evsel_tmp = evp->emask & 0x0f00;
677 evsel |= evsel_tmp << 24;
775 evsel |= evsel_tmp << OPT_PES_EVSELHI_SHIFT;
678
679 evsel |= evp->emask & 0x00ff;
776
777 evsel |= evp->emask & 0x00ff;
778 evsel |= evp->unit << OPT_PES_UMASK_SHIFT;
680
681 if (flags & CPC_COUNT_USER)
682 evsel |= OPT_PES_USR;
683 if (flags & CPC_COUNT_SYSTEM)
684 evsel |= OPT_PES_OS;
685 if (flags & CPC_OVF_NOTIFY_EMT)
686 evsel |= OPT_PES_INT;
687

--- 29 unchanged lines hidden (view full) ---

717
718 *data = cfg;
719 return (0);
720}
721
722static void
723opt_pcbe_program(void *token)
724{
779
780 if (flags & CPC_COUNT_USER)
781 evsel |= OPT_PES_USR;
782 if (flags & CPC_COUNT_SYSTEM)
783 evsel |= OPT_PES_OS;
784 if (flags & CPC_OVF_NOTIFY_EMT)
785 evsel |= OPT_PES_INT;
786

--- 29 unchanged lines hidden (view full) ---

816
817 *data = cfg;
818 return (0);
819}
820
821static void
822opt_pcbe_program(void *token)
823{
725 opt_pcbe_config_t *cfgs[4] = { &nullcfgs[0], &nullcfgs[1],
726 &nullcfgs[2], &nullcfgs[3] };
824 opt_pcbe_config_t *cfgs[OPT_PCBE_EXT_NCOUNTERS] = { &nullcfgs[0],
825 &nullcfgs[1], &nullcfgs[2],
826 &nullcfgs[3], &nullcfgs[4],
827 &nullcfgs[5] };
727 opt_pcbe_config_t *pcfg = NULL;
728 int i;
729 ulong_t curcr4 = getcr4();
730
731 /*
732 * Allow nonprivileged code to read the performance counters if desired.
733 */
734 if (kcpc_allow_nonpriv(token))
735 setcr4(curcr4 | CR4_PCE);
736 else
737 setcr4(curcr4 & ~CR4_PCE);
738
739 /*
740 * Query kernel for all configs which will be co-programmed.
741 */
742 do {
743 pcfg = (opt_pcbe_config_t *)kcpc_next_config(token, pcfg, NULL);
744
745 if (pcfg != NULL) {
828 opt_pcbe_config_t *pcfg = NULL;
829 int i;
830 ulong_t curcr4 = getcr4();
831
832 /*
833 * Allow nonprivileged code to read the performance counters if desired.
834 */
835 if (kcpc_allow_nonpriv(token))
836 setcr4(curcr4 | CR4_PCE);
837 else
838 setcr4(curcr4 & ~CR4_PCE);
839
840 /*
841 * Query kernel for all configs which will be co-programmed.
842 */
843 do {
844 pcfg = (opt_pcbe_config_t *)kcpc_next_config(token, pcfg, NULL);
845
846 if (pcfg != NULL) {
746 ASSERT(pcfg->opt_picno < 4);
847 ASSERT(pcfg->opt_picno < opd.opd_ncounters);
747 cfgs[pcfg->opt_picno] = pcfg;
748 }
749 } while (pcfg != NULL);
750
751 /*
752 * Program in two loops. The first configures and presets the counter,
753 * and the second loop enables the counters. This ensures that the
754 * counters are all enabled as closely together in time as possible.
755 */
756
848 cfgs[pcfg->opt_picno] = pcfg;
849 }
850 } while (pcfg != NULL);
851
852 /*
853 * Program in two loops. The first configures and presets the counter,
854 * and the second loop enables the counters. This ensures that the
855 * counters are all enabled as closely together in time as possible.
856 */
857
757 for (i = 0; i < 4; i++) {
758 wrmsr(PES_BASE_ADDR + i, cfgs[i]->opt_evsel);
759 wrmsr(PIC_BASE_ADDR + i, cfgs[i]->opt_rawpic);
858 for (i = 0; i < opd.opd_ncounters; i++) {
859 wrmsr(opd.opd_pesf(i), cfgs[i]->opt_evsel);
860 wrmsr(opd.opd_picf(i), cfgs[i]->opt_rawpic);
760 }
761
861 }
862
762 for (i = 0; i < 4; i++) {
763 wrmsr(PES_BASE_ADDR + i, cfgs[i]->opt_evsel |
863 for (i = 0; i < opd.opd_ncounters; i++) {
864 wrmsr(opd.opd_pesf(i), cfgs[i]->opt_evsel |
764 (uint64_t)(uintptr_t)OPT_PES_ENABLE);
765 }
766}
767
768static void
769opt_pcbe_allstop(void)
770{
771 int i;
772
865 (uint64_t)(uintptr_t)OPT_PES_ENABLE);
866 }
867}
868
869static void
870opt_pcbe_allstop(void)
871{
872 int i;
873
773 for (i = 0; i < 4; i++)
774 wrmsr(PES_BASE_ADDR + i, 0ULL);
874 for (i = 0; i < opd.opd_ncounters; i++)
875 wrmsr(opd.opd_pesf(i), 0ULL);
775
776 /*
777 * Disable non-privileged access to the counter registers.
778 */
779 setcr4(getcr4() & ~CR4_PCE);
780}
781
782static void
783opt_pcbe_sample(void *token)
784{
876
877 /*
878 * Disable non-privileged access to the counter registers.
879 */
880 setcr4(getcr4() & ~CR4_PCE);
881}
882
883static void
884opt_pcbe_sample(void *token)
885{
785 opt_pcbe_config_t *cfgs[4] = { NULL, NULL, NULL, NULL };
886 opt_pcbe_config_t *cfgs[OPT_PCBE_EXT_NCOUNTERS] = { NULL, NULL,
887 NULL, NULL, NULL, NULL };
786 opt_pcbe_config_t *pcfg = NULL;
787 int i;
888 opt_pcbe_config_t *pcfg = NULL;
889 int i;
788 uint64_t curpic[4];
789 uint64_t *addrs[4];
890 uint64_t curpic[OPT_PCBE_EXT_NCOUNTERS];
891 uint64_t *addrs[OPT_PCBE_EXT_NCOUNTERS];
790 uint64_t *tmp;
791 int64_t diff;
792
892 uint64_t *tmp;
893 int64_t diff;
894
793 for (i = 0; i < 4; i++)
794 curpic[i] = rdmsr(PIC_BASE_ADDR + i);
895 for (i = 0; i < opd.opd_ncounters; i++)
896 curpic[i] = rdmsr(opd.opd_picf(i));
795
796 /*
797 * Query kernel for all configs which are co-programmed.
798 */
799 do {
800 pcfg = (opt_pcbe_config_t *)kcpc_next_config(token, pcfg, &tmp);
801
802 if (pcfg != NULL) {
897
898 /*
899 * Query kernel for all configs which are co-programmed.
900 */
901 do {
902 pcfg = (opt_pcbe_config_t *)kcpc_next_config(token, pcfg, &tmp);
903
904 if (pcfg != NULL) {
803 ASSERT(pcfg->opt_picno < 4);
905 ASSERT3U(pcfg->opt_picno, <, opd.opd_ncounters);
804 cfgs[pcfg->opt_picno] = pcfg;
805 addrs[pcfg->opt_picno] = tmp;
806 }
807 } while (pcfg != NULL);
808
906 cfgs[pcfg->opt_picno] = pcfg;
907 addrs[pcfg->opt_picno] = tmp;
908 }
909 } while (pcfg != NULL);
910
809 for (i = 0; i < 4; i++) {
911 for (i = 0; i < opd.opd_ncounters; i++) {
810 if (cfgs[i] == NULL)
811 continue;
812
813 diff = (curpic[i] - cfgs[i]->opt_rawpic) & MASK48;
814 *addrs[i] += diff;
815 DTRACE_PROBE4(opt__pcbe__sample, int, i, uint64_t, *addrs[i],
816 uint64_t, curpic[i], uint64_t, cfgs[i]->opt_rawpic);
817 cfgs[i]->opt_rawpic = *addrs[i] & MASK48;

--- 50 unchanged lines hidden --- 		912 if (cfgs[i] == NULL)
913 continue;
914
915 diff = (curpic[i] - cfgs[i]->opt_rawpic) & MASK48;
916 *addrs[i] += diff;
917 DTRACE_PROBE4(opt__pcbe__sample, int, i, uint64_t, *addrs[i],
918 uint64_t, curpic[i], uint64_t, cfgs[i]->opt_rawpic);
919 cfgs[i]->opt_rawpic = *addrs[i] & MASK48;

--- 50 unchanged lines hidden ---