1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2021 Joyent, Inc.
24 */
25
26 /*
27 * x86-specific routines used by the CPU Performance counter driver.
28 */
29
30 #include <sys/types.h>
31 #include <sys/time.h>
32 #include <sys/atomic.h>
33 #include <sys/regset.h>
34 #include <sys/privregs.h>
35 #include <sys/x86_archext.h>
36 #include <sys/cpuvar.h>
37 #include <sys/machcpuvar.h>
38 #include <sys/archsystm.h>
39 #include <sys/cpc_pcbe.h>
40 #include <sys/cpc_impl.h>
41 #include <sys/x_call.h>
42 #include <sys/cmn_err.h>
43 #include <sys/cmt.h>
44 #include <sys/spl.h>
45 #include <sys/apic.h>
46
47 static kcpc_ctx_t *(*overflow_intr_handler)(caddr_t);
48
49 /* Do threads share performance monitoring hardware? */
50 static int strands_perfmon_shared = 0;
51
52 int kcpc_hw_overflow_intr_installed; /* set by APIC code */
53 extern kcpc_ctx_t *kcpc_overflow_intr(caddr_t arg, uint64_t bitmap);
54
55 extern int kcpc_counts_include_idle; /* Project Private /etc/system variable */
56
57 void (*kcpc_hw_enable_cpc_intr)(void); /* set by APIC code */
58
59 int
kcpc_hw_add_ovf_intr(kcpc_ctx_t * (* handler)(caddr_t))60 kcpc_hw_add_ovf_intr(kcpc_ctx_t *(*handler)(caddr_t))
61 {
62 if (x86_type != X86_TYPE_P6)
63 return (0);
64 overflow_intr_handler = handler;
65 return (ipltospl(APIC_PCINT_IPL));
66 }
67
68 void
kcpc_hw_rem_ovf_intr(void)69 kcpc_hw_rem_ovf_intr(void)
70 {
71 overflow_intr_handler = NULL;
72 }
73
74 /*
75 * Hook used on P4 systems to catch online/offline events.
76 */
77 /*ARGSUSED*/
78 static int
kcpc_cpu_setup(cpu_setup_t what,int cpuid,void * arg)79 kcpc_cpu_setup(cpu_setup_t what, int cpuid, void *arg)
80 {
81 pg_cmt_t *chip_pg;
82 int active_cpus_cnt;
83
84 if (what != CPU_ON)
85 return (0);
86
87 /*
88 * If any CPU-bound contexts exist, we don't need to invalidate
89 * anything, as no per-LWP contexts can coexist.
90 */
91 if (kcpc_cpuctx || dtrace_cpc_in_use)
92 return (0);
93
94 /*
95 * If this chip now has more than 1 active cpu, we must invalidate all
96 * contexts in the system.
97 */
98 chip_pg = (pg_cmt_t *)pghw_find_pg(cpu[cpuid], PGHW_CHIP);
99 if (chip_pg != NULL) {
100 active_cpus_cnt = GROUP_SIZE(&chip_pg->cmt_cpus_actv);
101 if (active_cpus_cnt > 1)
102 kcpc_invalidate_all();
103 }
104
105 return (0);
106 }
107
108 static kmutex_t cpu_setup_lock; /* protects setup_registered */
109 static int setup_registered;
110
111
112 void
kcpc_hw_init(cpu_t * cp)113 kcpc_hw_init(cpu_t *cp)
114 {
115 kthread_t *t = cp->cpu_idle_thread;
116 uint32_t versionid;
117 struct cpuid_regs cpuid;
118
119 strands_perfmon_shared = 0;
120 if (is_x86_feature(x86_featureset, X86FSET_HTT)) {
121 if (cpuid_getvendor(cpu[0]) == X86_VENDOR_Intel) {
122 /*
123 * Intel processors that support Architectural
124 * Performance Monitoring Version 3 have per strand
125 * performance monitoring hardware.
126 * Hence we can allow use of performance counters on
127 * multiple strands on the same core simultaneously.
128 */
129 cpuid.cp_eax = 0x0;
130 (void) __cpuid_insn(&cpuid);
131 if (cpuid.cp_eax < 0xa) {
132 strands_perfmon_shared = 1;
133 } else {
134 cpuid.cp_eax = 0xa;
135 (void) __cpuid_insn(&cpuid);
136
137 versionid = cpuid.cp_eax & 0xFF;
138 if (versionid < 3) {
139 strands_perfmon_shared = 1;
140 }
141 }
142 } else if (cpuid_getvendor(cpu[0]) == X86_VENDOR_AMD ||
143 cpuid_getvendor(cpu[0]) == X86_VENDOR_HYGON) {
144 /*
145 * On AMD systems with HT, all of the performance
146 * monitors exist on a per-logical CPU basis.
147 */
148 strands_perfmon_shared = 0;
149 } else {
150 strands_perfmon_shared = 1;
151 }
152 }
153
154 if (strands_perfmon_shared) {
155 mutex_enter(&cpu_setup_lock);
156 if (setup_registered == 0) {
157 mutex_enter(&cpu_lock);
158 register_cpu_setup_func(kcpc_cpu_setup, NULL);
159 mutex_exit(&cpu_lock);
160 setup_registered = 1;
161 }
162 mutex_exit(&cpu_setup_lock);
163 }
164
165 mutex_init(&cp->cpu_cpc_ctxlock, "cpu_cpc_ctxlock", MUTEX_DEFAULT, 0);
166
167 if (kcpc_counts_include_idle)
168 return;
169
170 kcpc_idle_ctxop_install(t, cp);
171 }
172
173 void
kcpc_hw_fini(cpu_t * cp)174 kcpc_hw_fini(cpu_t *cp)
175 {
176 ASSERT(cp->cpu_idle_thread == NULL);
177
178 mutex_destroy(&cp->cpu_cpc_ctxlock);
179 }
180
181 #define BITS(v, u, l) \
182 (((v) >> (l)) & ((1 << (1 + (u) - (l))) - 1))
183
184 #define PCBE_NAMELEN 30 /* Enough Room for pcbe.manuf.model.family.stepping */
185
186 /*
187 * Examine the processor and load an appropriate PCBE.
188 */
189 int
kcpc_hw_load_pcbe(void)190 kcpc_hw_load_pcbe(void)
191 {
192 return (kcpc_pcbe_tryload(cpuid_getvendorstr(CPU), cpuid_getfamily(CPU),
193 cpuid_getmodel(CPU), cpuid_getstep(CPU)));
194 }
195
196 /*
197 * Called by the generic framework to check if it's OK to bind a set to a CPU.
198 */
199 int
kcpc_hw_cpu_hook(processorid_t cpuid,ulong_t * kcpc_cpumap)200 kcpc_hw_cpu_hook(processorid_t cpuid, ulong_t *kcpc_cpumap)
201 {
202 cpu_t *cpu, *p;
203 pg_t *chip_pg;
204 pg_cpu_itr_t itr;
205
206 if (!strands_perfmon_shared)
207 return (0);
208
209 /*
210 * Only one logical CPU on each Pentium 4 HT CPU may be bound to at
211 * once.
212 *
213 * This loop is protected by holding cpu_lock, in order to properly
214 * access the cpu_t of the desired cpu.
215 */
216 mutex_enter(&cpu_lock);
217 if ((cpu = cpu_get(cpuid)) == NULL) {
218 mutex_exit(&cpu_lock);
219 return (-1);
220 }
221
222 chip_pg = (pg_t *)pghw_find_pg(cpu, PGHW_CHIP);
223
224 PG_CPU_ITR_INIT(chip_pg, itr);
225 while ((p = pg_cpu_next(&itr)) != NULL) {
226 if (p == cpu)
227 continue;
228 if (BT_TEST(kcpc_cpumap, p->cpu_id)) {
229 mutex_exit(&cpu_lock);
230 return (-1);
231 }
232 }
233
234 mutex_exit(&cpu_lock);
235 return (0);
236 }
237
238 /*
239 * Called by the generic framework to check if it's OK to bind a set to an LWP.
240 */
241 int
kcpc_hw_lwp_hook(void)242 kcpc_hw_lwp_hook(void)
243 {
244 pg_cmt_t *chip;
245 group_t *chips;
246 group_iter_t i;
247
248 if (!strands_perfmon_shared)
249 return (0);
250
251 /*
252 * Only one CPU per chip may be online.
253 */
254 mutex_enter(&cpu_lock);
255
256 chips = pghw_set_lookup(PGHW_CHIP);
257 if (chips == NULL) {
258 mutex_exit(&cpu_lock);
259 return (0);
260 }
261
262 group_iter_init(&i);
263 while ((chip = group_iterate(chips, &i)) != NULL) {
264 if (GROUP_SIZE(&chip->cmt_cpus_actv) > 1) {
265 mutex_exit(&cpu_lock);
266 return (-1);
267 }
268 }
269
270 mutex_exit(&cpu_lock);
271 return (0);
272 }
273