xref: /illumos-gate/usr/src/uts/common/os/cap_util.c (revision bbf21555)
1b885580bSAlexander Kolbasov /*
2b885580bSAlexander Kolbasov  * CDDL HEADER START
3b885580bSAlexander Kolbasov  *
4b885580bSAlexander Kolbasov  * The contents of this file are subject to the terms of the
5b885580bSAlexander Kolbasov  * Common Development and Distribution License (the "License").
6b885580bSAlexander Kolbasov  * You may not use this file except in compliance with the License.
7b885580bSAlexander Kolbasov  *
8b885580bSAlexander Kolbasov  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9b885580bSAlexander Kolbasov  * or http://www.opensolaris.org/os/licensing.
10b885580bSAlexander Kolbasov  * See the License for the specific language governing permissions
11b885580bSAlexander Kolbasov  * and limitations under the License.
12b885580bSAlexander Kolbasov  *
13b885580bSAlexander Kolbasov  * When distributing Covered Code, include this CDDL HEADER in each
14b885580bSAlexander Kolbasov  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15b885580bSAlexander Kolbasov  * If applicable, add the following below this CDDL HEADER, with the
16b885580bSAlexander Kolbasov  * fields enclosed by brackets "[]" replaced with your own identifying
17b885580bSAlexander Kolbasov  * information: Portions Copyright [yyyy] [name of copyright owner]
18b885580bSAlexander Kolbasov  *
19b885580bSAlexander Kolbasov  * CDDL HEADER END
20b885580bSAlexander Kolbasov  */
21b885580bSAlexander Kolbasov 
22b885580bSAlexander Kolbasov /*
23d3c97224SAlexander Kolbasov  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24b885580bSAlexander Kolbasov  */
25b885580bSAlexander Kolbasov 
26b885580bSAlexander Kolbasov /*
27b885580bSAlexander Kolbasov  * Support for determining capacity and utilization of performance relevant
28b885580bSAlexander Kolbasov  * hardware components in a computer
29b885580bSAlexander Kolbasov  *
30b885580bSAlexander Kolbasov  * THEORY
31b885580bSAlexander Kolbasov  * ------
32b885580bSAlexander Kolbasov  * The capacity and utilization of the performance relevant hardware components
33b885580bSAlexander Kolbasov  * is needed to be able to optimize performance while minimizing the amount of
34b885580bSAlexander Kolbasov  * power used on a system.  The idea is to use hardware performance counters
35b885580bSAlexander Kolbasov  * and potentially other means to determine the capacity and utilization of
36b885580bSAlexander Kolbasov  * performance relevant hardware components (eg. execution pipeline, cache,
37b885580bSAlexander Kolbasov  * memory, etc.) and attribute the utilization to the responsible CPU and the
38b885580bSAlexander Kolbasov  * thread running there.
39b885580bSAlexander Kolbasov  *
40b885580bSAlexander Kolbasov  * This will help characterize the utilization of performance relevant
41b885580bSAlexander Kolbasov  * components and how much is used by each CPU and each thread.  With
42b885580bSAlexander Kolbasov  * that data, the utilization can be aggregated to all the CPUs sharing each
43b885580bSAlexander Kolbasov  * performance relevant hardware component to calculate the total utilization
44b885580bSAlexander Kolbasov  * of each component and compare that with the component's capacity to
45b885580bSAlexander Kolbasov  * essentially determine the actual hardware load of the component.  The
46b885580bSAlexander Kolbasov  * hardware utilization attributed to each running thread can also be
47b885580bSAlexander Kolbasov  * aggregated to determine the total hardware utilization of each component to
48b885580bSAlexander Kolbasov  * a workload.
49b885580bSAlexander Kolbasov  *
50b885580bSAlexander Kolbasov  * Once that is done, one can determine how much of each performance relevant
51b885580bSAlexander Kolbasov  * hardware component is needed by a given thread or set of threads (eg. a
52b885580bSAlexander Kolbasov  * workload) and size up exactly what hardware is needed by the threads and how
53b885580bSAlexander Kolbasov  * much.  With this info, we can better place threads among CPUs to match their
54b885580bSAlexander Kolbasov  * exact hardware resource needs and potentially lower or raise the power based
55b885580bSAlexander Kolbasov  * on their utilization or pack threads onto the fewest hardware components
56b885580bSAlexander Kolbasov  * needed and power off any remaining unused components to minimize power
57b885580bSAlexander Kolbasov  * without sacrificing performance.
58b885580bSAlexander Kolbasov  *
59b885580bSAlexander Kolbasov  * IMPLEMENTATION
60b885580bSAlexander Kolbasov  * --------------
61b885580bSAlexander Kolbasov  * The code has been designed and implemented to make (un)programming and
62b885580bSAlexander Kolbasov  * reading the counters for a given CPU as lightweight and fast as possible.
63b885580bSAlexander Kolbasov  * This is very important because we need to read and potentially (un)program
64b885580bSAlexander Kolbasov  * the counters very often and in performance sensitive code.  Specifically,
65b885580bSAlexander Kolbasov  * the counters may need to be (un)programmed during context switch and/or a
66b885580bSAlexander Kolbasov  * cyclic handler when there are more counter events to count than existing
67b885580bSAlexander Kolbasov  * counters.
68b885580bSAlexander Kolbasov  *
69b885580bSAlexander Kolbasov  * Consequently, the code has been split up to allow allocating and
70b885580bSAlexander Kolbasov  * initializing everything needed to program and read the counters on a given
71b885580bSAlexander Kolbasov  * CPU once and make (un)programming and reading the counters for a given CPU
72b885580bSAlexander Kolbasov  * not have to allocate/free memory or grab any locks.  To do this, all the
73b885580bSAlexander Kolbasov  * state needed to (un)program and read the counters on a CPU is kept per CPU
74b885580bSAlexander Kolbasov  * and is made lock free by forcing any code that reads or manipulates the
75b885580bSAlexander Kolbasov  * counters or the state needed to (un)program or read the counters to run on
76b885580bSAlexander Kolbasov  * the target CPU and disable preemption while running on the target CPU to
77b885580bSAlexander Kolbasov  * protect any critical sections. All counter manipulation on the target CPU is
78b885580bSAlexander Kolbasov  * happening either from a cross-call to the target CPU or at the same PIL as
79b885580bSAlexander Kolbasov  * used by the cross-call subsystem. This guarantees that counter manipulation
80b885580bSAlexander Kolbasov  * is not interrupted by cross-calls from other CPUs.
81b885580bSAlexander Kolbasov  *
82b885580bSAlexander Kolbasov  * The synchronization has been made lock free or as simple as possible for
83b885580bSAlexander Kolbasov  * performance and to avoid getting the locking all tangled up when we interpose
84b885580bSAlexander Kolbasov  * on the CPC routines that (un)program the counters to manage the counters
85b885580bSAlexander Kolbasov  * between the kernel and user on each CPU.  When the user starts using the
86b885580bSAlexander Kolbasov  * counters on a given CPU, the kernel will unprogram the counters that it is
87b885580bSAlexander Kolbasov  * using on that CPU just before they are programmed for the user.  Then the
88b885580bSAlexander Kolbasov  * kernel will program the counters on a given CPU for its own use when the user
89b885580bSAlexander Kolbasov  * stops using them.
90b885580bSAlexander Kolbasov  *
91b885580bSAlexander Kolbasov  * There is a special interaction with DTrace cpc provider (dcpc). Before dcpc
92b885580bSAlexander Kolbasov  * enables any probe, it requests to disable and unprogram all counters used for
93b885580bSAlexander Kolbasov  * capacity and utilizations. These counters are never re-programmed back until
94b885580bSAlexander Kolbasov  * dcpc completes. When all DTrace cpc probes are removed, dcpc notifies CU
95b885580bSAlexander Kolbasov  * framework and it re-programs the counters.
96b885580bSAlexander Kolbasov  *
97b885580bSAlexander Kolbasov  * When a CPU is going offline, its CU counters are unprogrammed and disabled,
98b885580bSAlexander Kolbasov  * so that they would not be re-programmed again by some other activity on the
99b885580bSAlexander Kolbasov  * CPU that is going offline.
100b885580bSAlexander Kolbasov  *
101b885580bSAlexander Kolbasov  * The counters are programmed during boot.  However, a flag is available to
102b885580bSAlexander Kolbasov  * disable this if necessary (see cu_flag below).  A handler is provided to
103b885580bSAlexander Kolbasov  * (un)program the counters during CPU on/offline.  Basic routines are provided
104b885580bSAlexander Kolbasov  * to initialize and tear down this module, initialize and tear down any state
105b885580bSAlexander Kolbasov  * needed for a given CPU, and (un)program the counters for a given CPU.
106b885580bSAlexander Kolbasov  * Lastly, a handler is provided to read the counters and attribute the
107b885580bSAlexander Kolbasov  * utilization to the responsible CPU.
108b885580bSAlexander Kolbasov  */
109b885580bSAlexander Kolbasov #include <sys/types.h>
110b885580bSAlexander Kolbasov #include <sys/cmn_err.h>
111b885580bSAlexander Kolbasov #include <sys/cpuvar.h>
112b885580bSAlexander Kolbasov #include <sys/ddi.h>
113d3c97224SAlexander Kolbasov #include <sys/systm.h>
114b885580bSAlexander Kolbasov #include <sys/disp.h>
115b885580bSAlexander Kolbasov #include <sys/sdt.h>
116b885580bSAlexander Kolbasov #include <sys/sunddi.h>
117b885580bSAlexander Kolbasov #include <sys/thread.h>
118b885580bSAlexander Kolbasov #include <sys/pghw.h>
119b885580bSAlexander Kolbasov #include <sys/cmt.h>
120d3c97224SAlexander Kolbasov #include <sys/policy.h>
121b885580bSAlexander Kolbasov #include <sys/x_call.h>
122b885580bSAlexander Kolbasov #include <sys/cap_util.h>
123b885580bSAlexander Kolbasov 
124b885580bSAlexander Kolbasov #include <sys/archsystm.h>
125b885580bSAlexander Kolbasov #include <sys/promif.h>
126b885580bSAlexander Kolbasov 
127b885580bSAlexander Kolbasov #if defined(__x86)
128b885580bSAlexander Kolbasov #include <sys/xc_levels.h>
129b885580bSAlexander Kolbasov #endif
130b885580bSAlexander Kolbasov 
131b885580bSAlexander Kolbasov 
132b885580bSAlexander Kolbasov /*
133b885580bSAlexander Kolbasov  * Default CPU hardware performance counter flags to use for measuring capacity
134b885580bSAlexander Kolbasov  * and utilization
135b885580bSAlexander Kolbasov  */
136b885580bSAlexander Kolbasov #define	CU_CPC_FLAGS_DEFAULT	\
137b885580bSAlexander Kolbasov 	(CPC_COUNT_USER|CPC_COUNT_SYSTEM|CPC_OVF_NOTIFY_EMT)
138b885580bSAlexander Kolbasov 
139b885580bSAlexander Kolbasov /*
140b885580bSAlexander Kolbasov  * Possible Flags for controlling this module.
141b885580bSAlexander Kolbasov  */
142b885580bSAlexander Kolbasov #define	CU_FLAG_ENABLE		1	/* Enable module */
143b885580bSAlexander Kolbasov #define	CU_FLAG_READY		2	/* Ready to setup module */
144b885580bSAlexander Kolbasov #define	CU_FLAG_ON		4	/* Module is on */
145b885580bSAlexander Kolbasov 
146b885580bSAlexander Kolbasov /*
147b885580bSAlexander Kolbasov  * pg_cpu kstats calculate utilization rate and maximum utilization rate for
148b885580bSAlexander Kolbasov  * some CPUs. The rate is calculated based on data from two subsequent
149b885580bSAlexander Kolbasov  * snapshots. When the time between such two snapshots is too small, the
150b885580bSAlexander Kolbasov  * resulting rate may have low accuracy, so we only consider snapshots which
151b885580bSAlexander Kolbasov  * are separated by SAMPLE_INTERVAL nanoseconds from one another. We do not
152b885580bSAlexander Kolbasov  * update the rate if the interval is smaller than that.
153b885580bSAlexander Kolbasov  *
154b885580bSAlexander Kolbasov  * Use one tenth of a second as the minimum interval for utilization rate
155b885580bSAlexander Kolbasov  * calculation.
156b885580bSAlexander Kolbasov  *
157b885580bSAlexander Kolbasov  * NOTE: The CU_SAMPLE_INTERVAL_MIN should be higher than the scaling factor in
158b885580bSAlexander Kolbasov  * the CU_RATE() macro below to guarantee that we never divide by zero.
159b885580bSAlexander Kolbasov  *
160b885580bSAlexander Kolbasov  * Rate is the number of events per second. The rate is the number of events
161b885580bSAlexander Kolbasov  * divided by time and multiplied by the number of nanoseconds in a second. We
162b885580bSAlexander Kolbasov  * do not want time to be too small since it will cause large errors in
163b885580bSAlexander Kolbasov  * division.
164b885580bSAlexander Kolbasov  *
165b885580bSAlexander Kolbasov  * We do not want to multiply two large numbers (the instruction count and
166b885580bSAlexander Kolbasov  * NANOSEC) either since it may cause integer overflow. So we divide both the
167b885580bSAlexander Kolbasov  * numerator and the denominator by the same value.
168b885580bSAlexander Kolbasov  *
169b885580bSAlexander Kolbasov  * NOTE: The scaling factor below should be less than CU_SAMPLE_INTERVAL_MIN
170b885580bSAlexander Kolbasov  * above to guarantee that time divided by this value is always non-zero.
171b885580bSAlexander Kolbasov  */
172b885580bSAlexander Kolbasov #define	CU_RATE(val, time) \
173b885580bSAlexander Kolbasov 	(((val) * (NANOSEC / CU_SCALE)) / ((time) / CU_SCALE))
174b885580bSAlexander Kolbasov 
175b885580bSAlexander Kolbasov #define	CU_SAMPLE_INTERVAL_MIN	(NANOSEC / 10)
176b885580bSAlexander Kolbasov 
177b885580bSAlexander Kolbasov #define	CU_SCALE (CU_SAMPLE_INTERVAL_MIN / 10000)
178b885580bSAlexander Kolbasov 
179b885580bSAlexander Kolbasov /*
180b885580bSAlexander Kolbasov  * When the time between two kstat reads for the same CPU is less than
181b885580bSAlexander Kolbasov  * CU_UPDATE_THRESHOLD use the old counter data and skip updating counter values
182b885580bSAlexander Kolbasov  * for the CPU. This helps reduce cross-calls when kstat consumers read data
183b885580bSAlexander Kolbasov  * very often or when they read PG utilization data and then CPU utilization
184b885580bSAlexander Kolbasov  * data quickly after that.
185b885580bSAlexander Kolbasov  */
186b885580bSAlexander Kolbasov #define	CU_UPDATE_THRESHOLD (NANOSEC / 10)
187b885580bSAlexander Kolbasov 
188b885580bSAlexander Kolbasov /*
189b885580bSAlexander Kolbasov  * The IS_HIPIL() macro verifies that the code is executed either from a
190b885580bSAlexander Kolbasov  * cross-call or from high-PIL interrupt
191b885580bSAlexander Kolbasov  */
192b885580bSAlexander Kolbasov #ifdef DEBUG
193b885580bSAlexander Kolbasov #define	IS_HIPIL() (getpil() >= XCALL_PIL)
194b885580bSAlexander Kolbasov #else
195b885580bSAlexander Kolbasov #define	IS_HIPIL()
196b885580bSAlexander Kolbasov #endif	/* DEBUG */
197b885580bSAlexander Kolbasov 
198b885580bSAlexander Kolbasov 
199b885580bSAlexander Kolbasov typedef void (*cu_cpu_func_t)(uintptr_t, int *);
200b885580bSAlexander Kolbasov 
201b885580bSAlexander Kolbasov 
202b885580bSAlexander Kolbasov /*
203b885580bSAlexander Kolbasov  * Flags to use for programming CPU hardware performance counters to measure
204b885580bSAlexander Kolbasov  * capacity and utilization
205b885580bSAlexander Kolbasov  */
206b885580bSAlexander Kolbasov int				cu_cpc_flags = CU_CPC_FLAGS_DEFAULT;
207b885580bSAlexander Kolbasov 
208b885580bSAlexander Kolbasov /*
209b885580bSAlexander Kolbasov  * Initial value used for programming hardware counters
210b885580bSAlexander Kolbasov  */
211b885580bSAlexander Kolbasov uint64_t			cu_cpc_preset_value = 0;
212b885580bSAlexander Kolbasov 
213b885580bSAlexander Kolbasov /*
214b885580bSAlexander Kolbasov  * List of CPC event requests for capacity and utilization.
215b885580bSAlexander Kolbasov  */
216b885580bSAlexander Kolbasov static kcpc_request_list_t	*cu_cpc_reqs = NULL;
217b885580bSAlexander Kolbasov 
218b885580bSAlexander Kolbasov /*
219b885580bSAlexander Kolbasov  * When a CPU is a member of PG with a sharing relationship that is supported
220b885580bSAlexander Kolbasov  * by the capacity/utilization framework, a kstat is created for that CPU and
221b885580bSAlexander Kolbasov  * sharing relationship.
222b885580bSAlexander Kolbasov  *
223b885580bSAlexander Kolbasov  * These kstats are updated one at a time, so we can have a single scratch
224b885580bSAlexander Kolbasov  * space to fill the data.
225b885580bSAlexander Kolbasov  *
226b885580bSAlexander Kolbasov  * CPU counter kstats fields:
227b885580bSAlexander Kolbasov  *
228b885580bSAlexander Kolbasov  *   cu_cpu_id		CPU ID for this kstat
229b885580bSAlexander Kolbasov  *
230d3c97224SAlexander Kolbasov  *   cu_pg_id		PG ID for this kstat
231d3c97224SAlexander Kolbasov  *
232b885580bSAlexander Kolbasov  *   cu_generation	Generation value that increases whenever any CPU goes
233b885580bSAlexander Kolbasov  *			  offline or online. Two kstat snapshots for the same
234b885580bSAlexander Kolbasov  *			  CPU may only be compared if they have the same
235b885580bSAlexander Kolbasov  *			  generation.
236b885580bSAlexander Kolbasov  *
237b885580bSAlexander Kolbasov  *   cu_pg_id		PG ID for the relationship described by this kstat
238b885580bSAlexander Kolbasov  *
239b885580bSAlexander Kolbasov  *   cu_cpu_util	Running value of CPU utilization for the sharing
240b885580bSAlexander Kolbasov  *			  relationship
241b885580bSAlexander Kolbasov  *
242b885580bSAlexander Kolbasov  *   cu_cpu_time_running Total time spent collecting CU data. The time may be
243b885580bSAlexander Kolbasov  *			   less than wall time if CU counters were stopped for
244b885580bSAlexander Kolbasov  *			   some time.
245b885580bSAlexander Kolbasov  *
246b885580bSAlexander Kolbasov  *   cu_cpu_time_stopped Total time the CU counters were stopped.
247b885580bSAlexander Kolbasov  *
248b885580bSAlexander Kolbasov  *   cu_cpu_rate	Utilization rate, expressed in operations per second.
249b885580bSAlexander Kolbasov  *
250b885580bSAlexander Kolbasov  *   cu_cpu_rate_max	Maximum observed value of utilization rate.
251d3c97224SAlexander Kolbasov  *
252d3c97224SAlexander Kolbasov  *   cu_cpu_relationship Name of sharing relationship for the PG in this kstat
253b885580bSAlexander Kolbasov  */
254b885580bSAlexander Kolbasov struct cu_cpu_kstat {
255b885580bSAlexander Kolbasov 	kstat_named_t	cu_cpu_id;
256b885580bSAlexander Kolbasov 	kstat_named_t	cu_pg_id;
257d3c97224SAlexander Kolbasov 	kstat_named_t	cu_generation;
258b885580bSAlexander Kolbasov 	kstat_named_t	cu_cpu_util;
259b885580bSAlexander Kolbasov 	kstat_named_t	cu_cpu_time_running;
260b885580bSAlexander Kolbasov 	kstat_named_t	cu_cpu_time_stopped;
261b885580bSAlexander Kolbasov 	kstat_named_t	cu_cpu_rate;
262b885580bSAlexander Kolbasov 	kstat_named_t	cu_cpu_rate_max;
263d3c97224SAlexander Kolbasov 	kstat_named_t	cu_cpu_relationship;
264b885580bSAlexander Kolbasov } cu_cpu_kstat = {
265d3c97224SAlexander Kolbasov 	{ "cpu_id",			KSTAT_DATA_UINT32 },
266d3c97224SAlexander Kolbasov 	{ "pg_id",			KSTAT_DATA_INT32 },
267b885580bSAlexander Kolbasov 	{ "generation",			KSTAT_DATA_UINT32 },
268b885580bSAlexander Kolbasov 	{ "hw_util",			KSTAT_DATA_UINT64 },
269b885580bSAlexander Kolbasov 	{ "hw_util_time_running",	KSTAT_DATA_UINT64 },
270b885580bSAlexander Kolbasov 	{ "hw_util_time_stopped",	KSTAT_DATA_UINT64 },
271b885580bSAlexander Kolbasov 	{ "hw_util_rate",		KSTAT_DATA_UINT64 },
272b885580bSAlexander Kolbasov 	{ "hw_util_rate_max",		KSTAT_DATA_UINT64 },
273d3c97224SAlexander Kolbasov 	{ "relationship",		KSTAT_DATA_STRING },
274b885580bSAlexander Kolbasov };
275b885580bSAlexander Kolbasov 
276b885580bSAlexander Kolbasov /*
277b885580bSAlexander Kolbasov  * Flags for controlling this module
278b885580bSAlexander Kolbasov  */
279b885580bSAlexander Kolbasov uint_t				cu_flags = CU_FLAG_ENABLE;
280b885580bSAlexander Kolbasov 
281b885580bSAlexander Kolbasov /*
282b885580bSAlexander Kolbasov  * Error return value for cu_init() since it can't return anything to be called
283b885580bSAlexander Kolbasov  * from mp_init_tbl[] (:-(
284b885580bSAlexander Kolbasov  */
285b885580bSAlexander Kolbasov static int			cu_init_error = 0;
286b885580bSAlexander Kolbasov 
287b885580bSAlexander Kolbasov hrtime_t			cu_sample_interval_min = CU_SAMPLE_INTERVAL_MIN;
288b885580bSAlexander Kolbasov 
289b885580bSAlexander Kolbasov hrtime_t			cu_update_threshold = CU_UPDATE_THRESHOLD;
290b885580bSAlexander Kolbasov 
291b885580bSAlexander Kolbasov static kmutex_t			pg_cpu_kstat_lock;
292b885580bSAlexander Kolbasov 
293b885580bSAlexander Kolbasov 
294b885580bSAlexander Kolbasov /*
295b885580bSAlexander Kolbasov  * Forward declaration of interface routines
296b885580bSAlexander Kolbasov  */
297b885580bSAlexander Kolbasov void		cu_disable(void);
298b885580bSAlexander Kolbasov void		cu_enable(void);
299b885580bSAlexander Kolbasov void		cu_init(void);
300b885580bSAlexander Kolbasov void		cu_cpc_program(cpu_t *cp, int *err);
301b885580bSAlexander Kolbasov void		cu_cpc_unprogram(cpu_t *cp, int *err);
302b885580bSAlexander Kolbasov int		cu_cpu_update(struct cpu *cp, boolean_t move_to);
303b885580bSAlexander Kolbasov void		cu_pg_update(pghw_t *pg);
304b885580bSAlexander Kolbasov 
305b885580bSAlexander Kolbasov 
306b885580bSAlexander Kolbasov /*
307b885580bSAlexander Kolbasov  * Forward declaration of private routines
308b885580bSAlexander Kolbasov  */
309b885580bSAlexander Kolbasov static int	cu_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs);
310b885580bSAlexander Kolbasov static void	cu_cpc_program_xcall(uintptr_t arg, int *err);
311b885580bSAlexander Kolbasov static int	cu_cpc_req_add(char *event, kcpc_request_list_t *reqs,
312b885580bSAlexander Kolbasov     int nreqs, cu_cntr_stats_t *stats, int kmem_flags, int *nevents);
313b885580bSAlexander Kolbasov static int	cu_cpu_callback(cpu_setup_t what, int id, void *arg);
314b885580bSAlexander Kolbasov static void	cu_cpu_disable(cpu_t *cp);
315b885580bSAlexander Kolbasov static void	cu_cpu_enable(cpu_t *cp);
316b885580bSAlexander Kolbasov static int	cu_cpu_init(cpu_t *cp, kcpc_request_list_t *reqs);
317b885580bSAlexander Kolbasov static int	cu_cpu_fini(cpu_t *cp);
318b885580bSAlexander Kolbasov static void	cu_cpu_kstat_create(pghw_t *pg, cu_cntr_info_t *cntr_info);
319b885580bSAlexander Kolbasov static int	cu_cpu_kstat_update(kstat_t *ksp, int rw);
320b885580bSAlexander Kolbasov static int	cu_cpu_run(cpu_t *cp, cu_cpu_func_t func, uintptr_t arg);
321b885580bSAlexander Kolbasov static int	cu_cpu_update_stats(cu_cntr_stats_t *stats,
322b885580bSAlexander Kolbasov     uint64_t cntr_value);
323b885580bSAlexander Kolbasov static void cu_cpu_info_detach_xcall(void);
324b885580bSAlexander Kolbasov 
325b885580bSAlexander Kolbasov /*
326b885580bSAlexander Kolbasov  * Disable or enable Capacity Utilization counters on all CPUs.
327b885580bSAlexander Kolbasov  */
328b885580bSAlexander Kolbasov void
cu_disable(void)329b885580bSAlexander Kolbasov cu_disable(void)
330b885580bSAlexander Kolbasov {
331b885580bSAlexander Kolbasov 	cpu_t *cp;
332b885580bSAlexander Kolbasov 
333b885580bSAlexander Kolbasov 	ASSERT(MUTEX_HELD(&cpu_lock));
334b885580bSAlexander Kolbasov 
335b885580bSAlexander Kolbasov 	cp = cpu_active;
336b885580bSAlexander Kolbasov 	do {
337b885580bSAlexander Kolbasov 		if (!(cp->cpu_flags & CPU_OFFLINE))
338b885580bSAlexander Kolbasov 			cu_cpu_disable(cp);
339b885580bSAlexander Kolbasov 	} while ((cp = cp->cpu_next_onln) != cpu_active);
340b885580bSAlexander Kolbasov }
341b885580bSAlexander Kolbasov 
342b885580bSAlexander Kolbasov 
343b885580bSAlexander Kolbasov void
cu_enable(void)344b885580bSAlexander Kolbasov cu_enable(void)
345b885580bSAlexander Kolbasov {
346b885580bSAlexander Kolbasov 	cpu_t *cp;
347b885580bSAlexander Kolbasov 
348b885580bSAlexander Kolbasov 	ASSERT(MUTEX_HELD(&cpu_lock));
349b885580bSAlexander Kolbasov 
350b885580bSAlexander Kolbasov 	cp = cpu_active;
351b885580bSAlexander Kolbasov 	do {
352b885580bSAlexander Kolbasov 		if (!(cp->cpu_flags & CPU_OFFLINE))
353b885580bSAlexander Kolbasov 			cu_cpu_enable(cp);
354b885580bSAlexander Kolbasov 	} while ((cp = cp->cpu_next_onln) != cpu_active);
355b885580bSAlexander Kolbasov }
356b885580bSAlexander Kolbasov 
357b885580bSAlexander Kolbasov 
358b885580bSAlexander Kolbasov /*
359b885580bSAlexander Kolbasov  * Setup capacity and utilization support
360b885580bSAlexander Kolbasov  */
361b885580bSAlexander Kolbasov void
cu_init(void)362b885580bSAlexander Kolbasov cu_init(void)
363b885580bSAlexander Kolbasov {
364b885580bSAlexander Kolbasov 	cpu_t	*cp;
365b885580bSAlexander Kolbasov 
366b885580bSAlexander Kolbasov 	cu_init_error = 0;
367b885580bSAlexander Kolbasov 	if (!(cu_flags & CU_FLAG_ENABLE) || (cu_flags & CU_FLAG_ON)) {
368b885580bSAlexander Kolbasov 		cu_init_error = -1;
369b885580bSAlexander Kolbasov 		return;
370b885580bSAlexander Kolbasov 	}
371b885580bSAlexander Kolbasov 
372b885580bSAlexander Kolbasov 	if (kcpc_init() != 0) {
373b885580bSAlexander Kolbasov 		cu_init_error = -2;
374b885580bSAlexander Kolbasov 		return;
375b885580bSAlexander Kolbasov 	}
376b885580bSAlexander Kolbasov 
377b885580bSAlexander Kolbasov 	/*
378b885580bSAlexander Kolbasov 	 * Can't measure hardware capacity and utilization without CPU
379b885580bSAlexander Kolbasov 	 * hardware performance counters
380b885580bSAlexander Kolbasov 	 */
381b885580bSAlexander Kolbasov 	if (cpc_ncounters <= 0) {
382b885580bSAlexander Kolbasov 		cu_init_error = -3;
383b885580bSAlexander Kolbasov 		return;
384b885580bSAlexander Kolbasov 	}
385b885580bSAlexander Kolbasov 
386b885580bSAlexander Kolbasov 	/*
387b885580bSAlexander Kolbasov 	 * Setup CPC event request queue
388b885580bSAlexander Kolbasov 	 */
389b885580bSAlexander Kolbasov 	cu_cpc_reqs = kcpc_reqs_init(cpc_ncounters, KM_SLEEP);
390b885580bSAlexander Kolbasov 
391b885580bSAlexander Kolbasov 	mutex_enter(&cpu_lock);
392b885580bSAlexander Kolbasov 
393b885580bSAlexander Kolbasov 	/*
394b885580bSAlexander Kolbasov 	 * Mark flags to say that module is ready to be setup
395b885580bSAlexander Kolbasov 	 */
396b885580bSAlexander Kolbasov 	cu_flags |= CU_FLAG_READY;
397b885580bSAlexander Kolbasov 
398b885580bSAlexander Kolbasov 	cp = cpu_active;
399b885580bSAlexander Kolbasov 	do {
400b885580bSAlexander Kolbasov 		/*
401b885580bSAlexander Kolbasov 		 * Allocate and setup state needed to measure capacity and
402b885580bSAlexander Kolbasov 		 * utilization
403b885580bSAlexander Kolbasov 		 */
404b885580bSAlexander Kolbasov 		if (cu_cpu_init(cp, cu_cpc_reqs) != 0)
405b885580bSAlexander Kolbasov 			cu_init_error = -5;
406b885580bSAlexander Kolbasov 
407b885580bSAlexander Kolbasov 		/*
408b885580bSAlexander Kolbasov 		 * Reset list of counter event requests so its space can be
409b885580bSAlexander Kolbasov 		 * reused for a different set of requests for next CPU
410b885580bSAlexander Kolbasov 		 */
411b885580bSAlexander Kolbasov 		(void) kcpc_reqs_reset(cu_cpc_reqs);
412b885580bSAlexander Kolbasov 
413b885580bSAlexander Kolbasov 		cp = cp->cpu_next_onln;
414b885580bSAlexander Kolbasov 	} while (cp != cpu_active);
415b885580bSAlexander Kolbasov 
416b885580bSAlexander Kolbasov 	/*
417b885580bSAlexander Kolbasov 	 * Mark flags to say that module is on now and counters are ready to be
418b885580bSAlexander Kolbasov 	 * programmed on all active CPUs
419b885580bSAlexander Kolbasov 	 */
420b885580bSAlexander Kolbasov 	cu_flags |= CU_FLAG_ON;
421b885580bSAlexander Kolbasov 
422b885580bSAlexander Kolbasov 	/*
423b885580bSAlexander Kolbasov 	 * Program counters on currently active CPUs
424b885580bSAlexander Kolbasov 	 */
425b885580bSAlexander Kolbasov 	cp = cpu_active;
426b885580bSAlexander Kolbasov 	do {
427b885580bSAlexander Kolbasov 		if (cu_cpu_run(cp, cu_cpc_program_xcall,
428b885580bSAlexander Kolbasov 		    (uintptr_t)B_FALSE) != 0)
429b885580bSAlexander Kolbasov 			cu_init_error = -6;
430b885580bSAlexander Kolbasov 
431b885580bSAlexander Kolbasov 		cp = cp->cpu_next_onln;
432b885580bSAlexander Kolbasov 	} while (cp != cpu_active);
433b885580bSAlexander Kolbasov 
434b885580bSAlexander Kolbasov 	/*
435b885580bSAlexander Kolbasov 	 * Register callback for CPU state changes to enable and disable
436b885580bSAlexander Kolbasov 	 * CPC counters as CPUs come on and offline
437b885580bSAlexander Kolbasov 	 */
438b885580bSAlexander Kolbasov 	register_cpu_setup_func(cu_cpu_callback, NULL);
439b885580bSAlexander Kolbasov 
440b885580bSAlexander Kolbasov 	mutex_exit(&cpu_lock);
441b885580bSAlexander Kolbasov }
442b885580bSAlexander Kolbasov 
443b885580bSAlexander Kolbasov 
444b885580bSAlexander Kolbasov /*
445b885580bSAlexander Kolbasov  * Return number of counter events needed to measure capacity and utilization
446b885580bSAlexander Kolbasov  * for specified CPU and fill in list of CPC requests with each counter event
447b885580bSAlexander Kolbasov  * needed if list where to add CPC requests is given
448b885580bSAlexander Kolbasov  *
449b885580bSAlexander Kolbasov  * NOTE: Use KM_NOSLEEP for kmem_{,z}alloc() since cpu_lock is held and free
450b885580bSAlexander Kolbasov  *	 everything that has been successfully allocated if any memory
451b885580bSAlexander Kolbasov  *	 allocation fails
452b885580bSAlexander Kolbasov  */
453b885580bSAlexander Kolbasov static int
cu_cpc_init(cpu_t * cp,kcpc_request_list_t * reqs,int nreqs)454b885580bSAlexander Kolbasov cu_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs)
455b885580bSAlexander Kolbasov {
456b885580bSAlexander Kolbasov 	group_t		*cmt_pgs;
457b885580bSAlexander Kolbasov 	cu_cntr_info_t	**cntr_info_array;
458b885580bSAlexander Kolbasov 	cpu_pg_t	*cpu_pgs;
459b885580bSAlexander Kolbasov 	cu_cpu_info_t	*cu_cpu_info;
460b885580bSAlexander Kolbasov 	pg_cmt_t	*pg_cmt;
461b885580bSAlexander Kolbasov 	pghw_t		*pg_hw;
462b885580bSAlexander Kolbasov 	cu_cntr_stats_t	*stats;
463b885580bSAlexander Kolbasov 	int		nevents;
464b885580bSAlexander Kolbasov 	pghw_type_t	pg_hw_type;
465b885580bSAlexander Kolbasov 	group_iter_t	iter;
466b885580bSAlexander Kolbasov 
467b885580bSAlexander Kolbasov 	ASSERT(MUTEX_HELD(&cpu_lock));
468b885580bSAlexander Kolbasov 
469b885580bSAlexander Kolbasov 	/*
470b885580bSAlexander Kolbasov 	 * There has to be a target CPU for this
471b885580bSAlexander Kolbasov 	 */
472b885580bSAlexander Kolbasov 	if (cp == NULL)
473b885580bSAlexander Kolbasov 		return (-1);
474b885580bSAlexander Kolbasov 
475b885580bSAlexander Kolbasov 	/*
476b885580bSAlexander Kolbasov 	 * Return 0 when CPU doesn't belong to any group
477b885580bSAlexander Kolbasov 	 */
478b885580bSAlexander Kolbasov 	cpu_pgs = cp->cpu_pg;
479b885580bSAlexander Kolbasov 	if (cpu_pgs == NULL || GROUP_SIZE(&cpu_pgs->cmt_pgs) < 1)
480b885580bSAlexander Kolbasov 		return (0);
481b885580bSAlexander Kolbasov 
482b885580bSAlexander Kolbasov 	cmt_pgs = &cpu_pgs->cmt_pgs;
483b885580bSAlexander Kolbasov 	cu_cpu_info = cp->cpu_cu_info;
484b885580bSAlexander Kolbasov 
485b885580bSAlexander Kolbasov 	/*
486b885580bSAlexander Kolbasov 	 * Grab counter statistics and info
487b885580bSAlexander Kolbasov 	 */
488b885580bSAlexander Kolbasov 	if (reqs == NULL) {
489b885580bSAlexander Kolbasov 		stats = NULL;
490b885580bSAlexander Kolbasov 		cntr_info_array = NULL;
491b885580bSAlexander Kolbasov 	} else {
492b885580bSAlexander Kolbasov 		if (cu_cpu_info == NULL || cu_cpu_info->cu_cntr_stats == NULL)
493b885580bSAlexander Kolbasov 			return (-2);
494b885580bSAlexander Kolbasov 
495b885580bSAlexander Kolbasov 		stats = cu_cpu_info->cu_cntr_stats;
496b885580bSAlexander Kolbasov 		cntr_info_array = cu_cpu_info->cu_cntr_info;
497b885580bSAlexander Kolbasov 	}
498b885580bSAlexander Kolbasov 
499b885580bSAlexander Kolbasov 	/*
500b885580bSAlexander Kolbasov 	 * See whether platform (or processor) specific code knows which CPC
501b885580bSAlexander Kolbasov 	 * events to request, etc. are needed to measure hardware capacity and
502b885580bSAlexander Kolbasov 	 * utilization on this machine
503b885580bSAlexander Kolbasov 	 */
504b885580bSAlexander Kolbasov 	nevents = cu_plat_cpc_init(cp, reqs, nreqs);
505b885580bSAlexander Kolbasov 	if (nevents >= 0)
506b885580bSAlexander Kolbasov 		return (nevents);
507b885580bSAlexander Kolbasov 
508b885580bSAlexander Kolbasov 	/*
509b885580bSAlexander Kolbasov 	 * Let common code decide which CPC events to request, etc. to measure
510b885580bSAlexander Kolbasov 	 * capacity and utilization since platform (or processor) specific does
511b885580bSAlexander Kolbasov 	 * not know....
512b885580bSAlexander Kolbasov 	 *
513b885580bSAlexander Kolbasov 	 * Walk CPU's PG lineage and do following:
514b885580bSAlexander Kolbasov 	 *
515b885580bSAlexander Kolbasov 	 * - Setup CPC request, counter info, and stats needed for each counter
516b885580bSAlexander Kolbasov 	 *   event to measure capacity and and utilization for each of CPU's PG
517b885580bSAlexander Kolbasov 	 *   hardware sharing relationships
518b885580bSAlexander Kolbasov 	 *
519b885580bSAlexander Kolbasov 	 * - Create PG CPU kstats to export capacity and utilization for each PG
520b885580bSAlexander Kolbasov 	 */
521b885580bSAlexander Kolbasov 	nevents = 0;
522b885580bSAlexander Kolbasov 	group_iter_init(&iter);
523b885580bSAlexander Kolbasov 	while ((pg_cmt = group_iterate(cmt_pgs, &iter)) != NULL) {
524b885580bSAlexander Kolbasov 		cu_cntr_info_t	*cntr_info;
525b885580bSAlexander Kolbasov 		int		nevents_save;
526b885580bSAlexander Kolbasov 		int		nstats;
527b885580bSAlexander Kolbasov 
528b885580bSAlexander Kolbasov 		pg_hw = (pghw_t *)pg_cmt;
529b885580bSAlexander Kolbasov 		pg_hw_type = pg_hw->pghw_hw;
530b885580bSAlexander Kolbasov 		nevents_save = nevents;
531b885580bSAlexander Kolbasov 		nstats = 0;
532b885580bSAlexander Kolbasov 
533b885580bSAlexander Kolbasov 		switch (pg_hw_type) {
534b885580bSAlexander Kolbasov 		case PGHW_IPIPE:
535b885580bSAlexander Kolbasov 			if (cu_cpc_req_add("PAPI_tot_ins", reqs, nreqs, stats,
536b885580bSAlexander Kolbasov 			    KM_NOSLEEP, &nevents) != 0)
537b885580bSAlexander Kolbasov 				continue;
538b885580bSAlexander Kolbasov 			nstats = 1;
539b885580bSAlexander Kolbasov 			break;
540b885580bSAlexander Kolbasov 
541b885580bSAlexander Kolbasov 		case PGHW_FPU:
542b885580bSAlexander Kolbasov 			if (cu_cpc_req_add("PAPI_fp_ins", reqs, nreqs, stats,
543b885580bSAlexander Kolbasov 			    KM_NOSLEEP, &nevents) != 0)
544b885580bSAlexander Kolbasov 				continue;
545b885580bSAlexander Kolbasov 			nstats = 1;
546b885580bSAlexander Kolbasov 			break;
547b885580bSAlexander Kolbasov 
548b885580bSAlexander Kolbasov 		default:
549b885580bSAlexander Kolbasov 			/*
550b885580bSAlexander Kolbasov 			 * Don't measure capacity and utilization for this kind
551b885580bSAlexander Kolbasov 			 * of PG hardware relationship so skip to next PG in
552b885580bSAlexander Kolbasov 			 * CPU's PG lineage
553b885580bSAlexander Kolbasov 			 */
554b885580bSAlexander Kolbasov 			continue;
555b885580bSAlexander Kolbasov 		}
556b885580bSAlexander Kolbasov 
557b885580bSAlexander Kolbasov 		cntr_info = cntr_info_array[pg_hw_type];
558b885580bSAlexander Kolbasov 
559b885580bSAlexander Kolbasov 		/*
560b885580bSAlexander Kolbasov 		 * Nothing to measure for this hardware sharing relationship
561b885580bSAlexander Kolbasov 		 */
562b885580bSAlexander Kolbasov 		if (nevents - nevents_save == 0) {
563c18b20adSToomas Soome 			if (cntr_info != NULL) {
564b885580bSAlexander Kolbasov 				kmem_free(cntr_info, sizeof (cu_cntr_info_t));
565b885580bSAlexander Kolbasov 				cntr_info_array[pg_hw_type] = NULL;
566c18b20adSToomas Soome 			}
567b885580bSAlexander Kolbasov 			continue;
568b885580bSAlexander Kolbasov 		}
569b885580bSAlexander Kolbasov 
570b885580bSAlexander Kolbasov 		/*
571b885580bSAlexander Kolbasov 		 * Fill in counter info for this PG hardware relationship
572b885580bSAlexander Kolbasov 		 */
573b885580bSAlexander Kolbasov 		if (cntr_info == NULL) {
574b885580bSAlexander Kolbasov 			cntr_info = kmem_zalloc(sizeof (cu_cntr_info_t),
575b885580bSAlexander Kolbasov 			    KM_NOSLEEP);
576b885580bSAlexander Kolbasov 			if (cntr_info == NULL)
577b885580bSAlexander Kolbasov 				continue;
578b885580bSAlexander Kolbasov 			cntr_info_array[pg_hw_type] = cntr_info;
579b885580bSAlexander Kolbasov 		}
580b885580bSAlexander Kolbasov 		cntr_info->ci_cpu = cp;
581b885580bSAlexander Kolbasov 		cntr_info->ci_pg = pg_hw;
582b885580bSAlexander Kolbasov 		cntr_info->ci_stats = &stats[nevents_save];
583b885580bSAlexander Kolbasov 		cntr_info->ci_nstats = nstats;
584b885580bSAlexander Kolbasov 
585b885580bSAlexander Kolbasov 		/*
586b885580bSAlexander Kolbasov 		 * Create PG CPU kstats for this hardware relationship
587b885580bSAlexander Kolbasov 		 */
588b885580bSAlexander Kolbasov 		cu_cpu_kstat_create(pg_hw, cntr_info);
589b885580bSAlexander Kolbasov 	}
590b885580bSAlexander Kolbasov 
591b885580bSAlexander Kolbasov 	return (nevents);
592b885580bSAlexander Kolbasov }
593b885580bSAlexander Kolbasov 
594b885580bSAlexander Kolbasov 
595b885580bSAlexander Kolbasov /*
596b885580bSAlexander Kolbasov  * Program counters for capacity and utilization on given CPU
597b885580bSAlexander Kolbasov  *
598b885580bSAlexander Kolbasov  * If any of the following conditions is true, the counters are not programmed:
599b885580bSAlexander Kolbasov  *
600b885580bSAlexander Kolbasov  * - CU framework is disabled
601b885580bSAlexander Kolbasov  * - The cpu_cu_info field of the cpu structure is NULL
602b885580bSAlexander Kolbasov  * - DTrace is active
603b885580bSAlexander Kolbasov  * - Counters are programmed already
604b885580bSAlexander Kolbasov  * - Counters are disabled (by calls to cu_cpu_disable())
605b885580bSAlexander Kolbasov  */
606b885580bSAlexander Kolbasov void
cu_cpc_program(cpu_t * cp,int * err)607b885580bSAlexander Kolbasov cu_cpc_program(cpu_t *cp, int *err)
608b885580bSAlexander Kolbasov {
609b885580bSAlexander Kolbasov 	cu_cpc_ctx_t	*cpu_ctx;
610b885580bSAlexander Kolbasov 	kcpc_ctx_t	*ctx;
611b885580bSAlexander Kolbasov 	cu_cpu_info_t	*cu_cpu_info;
612b885580bSAlexander Kolbasov 
613b885580bSAlexander Kolbasov 	ASSERT(IS_HIPIL());
614b885580bSAlexander Kolbasov 	/*
615b885580bSAlexander Kolbasov 	 * Should be running on given CPU. We disable preemption to keep CPU
616b885580bSAlexander Kolbasov 	 * from disappearing and make sure flags and CPC context don't change
617b885580bSAlexander Kolbasov 	 * from underneath us
618b885580bSAlexander Kolbasov 	 */
619b885580bSAlexander Kolbasov 	kpreempt_disable();
620b885580bSAlexander Kolbasov 	ASSERT(cp == CPU);
621b885580bSAlexander Kolbasov 
622b885580bSAlexander Kolbasov 	/*
623b885580bSAlexander Kolbasov 	 * Module not ready to program counters
624b885580bSAlexander Kolbasov 	 */
625b885580bSAlexander Kolbasov 	if (!(cu_flags & CU_FLAG_ON)) {
626b885580bSAlexander Kolbasov 		*err = -1;
627b885580bSAlexander Kolbasov 		kpreempt_enable();
628b885580bSAlexander Kolbasov 		return;
629b885580bSAlexander Kolbasov 	}
630b885580bSAlexander Kolbasov 
631b885580bSAlexander Kolbasov 	if (cp == NULL) {
632b885580bSAlexander Kolbasov 		*err = -2;
633b885580bSAlexander Kolbasov 		kpreempt_enable();
634b885580bSAlexander Kolbasov 		return;
635b885580bSAlexander Kolbasov 	}
636b885580bSAlexander Kolbasov 
637b885580bSAlexander Kolbasov 	cu_cpu_info = cp->cpu_cu_info;
638b885580bSAlexander Kolbasov 	if (cu_cpu_info == NULL) {
639b885580bSAlexander Kolbasov 		*err = -3;
640b885580bSAlexander Kolbasov 		kpreempt_enable();
641b885580bSAlexander Kolbasov 		return;
642b885580bSAlexander Kolbasov 	}
643b885580bSAlexander Kolbasov 
644b885580bSAlexander Kolbasov 	/*
645b885580bSAlexander Kolbasov 	 * If DTrace CPC is active or counters turned on already or are
646b885580bSAlexander Kolbasov 	 * disabled, just return.
647b885580bSAlexander Kolbasov 	 */
648b885580bSAlexander Kolbasov 	if (dtrace_cpc_in_use || (cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON) ||
649b885580bSAlexander Kolbasov 	    cu_cpu_info->cu_disabled) {
650b885580bSAlexander Kolbasov 		*err = 1;
651b885580bSAlexander Kolbasov 		kpreempt_enable();
652b885580bSAlexander Kolbasov 		return;
653b885580bSAlexander Kolbasov 	}
654b885580bSAlexander Kolbasov 
655b885580bSAlexander Kolbasov 	if ((CPU->cpu_cpc_ctx != NULL) &&
656b885580bSAlexander Kolbasov 	    !(CPU->cpu_cpc_ctx->kc_flags & KCPC_CTX_INVALID_STOPPED)) {
657b885580bSAlexander Kolbasov 		*err = -4;
658b885580bSAlexander Kolbasov 		kpreempt_enable();
659b885580bSAlexander Kolbasov 		return;
660b885580bSAlexander Kolbasov 	}
661b885580bSAlexander Kolbasov 
662b885580bSAlexander Kolbasov 	/*
663b885580bSAlexander Kolbasov 	 * Get CPU's CPC context needed for capacity and utilization
664b885580bSAlexander Kolbasov 	 */
665b885580bSAlexander Kolbasov 	cpu_ctx = &cu_cpu_info->cu_cpc_ctx;
666b885580bSAlexander Kolbasov 	ASSERT(cpu_ctx != NULL);
667b885580bSAlexander Kolbasov 	ASSERT(cpu_ctx->nctx >= 0);
668b885580bSAlexander Kolbasov 
669b885580bSAlexander Kolbasov 	ASSERT(cpu_ctx->ctx_ptr_array == NULL || cpu_ctx->ctx_ptr_array_sz > 0);
670b885580bSAlexander Kolbasov 	ASSERT(cpu_ctx->nctx <= cpu_ctx->ctx_ptr_array_sz);
671b885580bSAlexander Kolbasov 	if (cpu_ctx->nctx <= 0 || cpu_ctx->ctx_ptr_array == NULL ||
672b885580bSAlexander Kolbasov 	    cpu_ctx->ctx_ptr_array_sz <= 0) {
673b885580bSAlexander Kolbasov 		*err = -5;
674b885580bSAlexander Kolbasov 		kpreempt_enable();
675b885580bSAlexander Kolbasov 		return;
676b885580bSAlexander Kolbasov 	}
677b885580bSAlexander Kolbasov 
678b885580bSAlexander Kolbasov 	/*
679b885580bSAlexander Kolbasov 	 * Increment index in CPU's CPC context info to point at next context
680b885580bSAlexander Kolbasov 	 * to program
681b885580bSAlexander Kolbasov 	 *
682b885580bSAlexander Kolbasov 	 * NOTE: Do this now instead of after programming counters to ensure
683b885580bSAlexander Kolbasov 	 *	 that index will always point at *current* context so we will
684b885580bSAlexander Kolbasov 	 *	 always be able to unprogram *current* context if necessary
685b885580bSAlexander Kolbasov 	 */
686b885580bSAlexander Kolbasov 	cpu_ctx->cur_index = (cpu_ctx->cur_index + 1) % cpu_ctx->nctx;
687b885580bSAlexander Kolbasov 
688b885580bSAlexander Kolbasov 	ctx = cpu_ctx->ctx_ptr_array[cpu_ctx->cur_index];
689b885580bSAlexander Kolbasov 
690b885580bSAlexander Kolbasov 	/*
691b885580bSAlexander Kolbasov 	 * Clear KCPC_CTX_INVALID and KCPC_CTX_INVALID_STOPPED from CPU's CPC
692b885580bSAlexander Kolbasov 	 * context before programming counters
693b885580bSAlexander Kolbasov 	 *
694b885580bSAlexander Kolbasov 	 * Context is marked with KCPC_CTX_INVALID_STOPPED when context is
695b885580bSAlexander Kolbasov 	 * unprogrammed and may be marked with KCPC_CTX_INVALID when
696*bbf21555SRichard Lowe 	 * kcpc_invalidate_all() is called by cpustat(8) and dtrace CPC to
697b885580bSAlexander Kolbasov 	 * invalidate all CPC contexts before they take over all the counters.
698b885580bSAlexander Kolbasov 	 *
699b885580bSAlexander Kolbasov 	 * This isn't necessary since these flags are only used for thread bound
700b885580bSAlexander Kolbasov 	 * CPC contexts not CPU bound CPC contexts like ones used for capacity
701b885580bSAlexander Kolbasov 	 * and utilization.
702b885580bSAlexander Kolbasov 	 *
703b885580bSAlexander Kolbasov 	 * There is no need to protect the flag update since no one is using
704b885580bSAlexander Kolbasov 	 * this context now.
705b885580bSAlexander Kolbasov 	 */
706b885580bSAlexander Kolbasov 	ctx->kc_flags &= ~(KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED);
707b885580bSAlexander Kolbasov 
708b885580bSAlexander Kolbasov 	/*
709b885580bSAlexander Kolbasov 	 * Program counters on this CPU
710b885580bSAlexander Kolbasov 	 */
711b885580bSAlexander Kolbasov 	kcpc_program(ctx, B_FALSE, B_FALSE);
712b885580bSAlexander Kolbasov 
713b885580bSAlexander Kolbasov 	cp->cpu_cpc_ctx = ctx;
714b885580bSAlexander Kolbasov 
715b885580bSAlexander Kolbasov 	/*
716b885580bSAlexander Kolbasov 	 * Set state in CPU structure to say that CPU's counters are programmed
717b885580bSAlexander Kolbasov 	 * for capacity and utilization now and that they are transitioning from
718b885580bSAlexander Kolbasov 	 * off to on state. This will cause cu_cpu_update to update stop times
719b885580bSAlexander Kolbasov 	 * for all programmed counters.
720b885580bSAlexander Kolbasov 	 */
721b885580bSAlexander Kolbasov 	cu_cpu_info->cu_flag |= CU_CPU_CNTRS_ON | CU_CPU_CNTRS_OFF_ON;
722b885580bSAlexander Kolbasov 
723b885580bSAlexander Kolbasov 	/*
724b885580bSAlexander Kolbasov 	 * Update counter statistics
725b885580bSAlexander Kolbasov 	 */
726b885580bSAlexander Kolbasov 	(void) cu_cpu_update(cp, B_FALSE);
727b885580bSAlexander Kolbasov 
728b885580bSAlexander Kolbasov 	cu_cpu_info->cu_flag &= ~CU_CPU_CNTRS_OFF_ON;
729b885580bSAlexander Kolbasov 
730b885580bSAlexander Kolbasov 	*err = 0;
731b885580bSAlexander Kolbasov 	kpreempt_enable();
732b885580bSAlexander Kolbasov }
733b885580bSAlexander Kolbasov 
734b885580bSAlexander Kolbasov 
735b885580bSAlexander Kolbasov /*
736b885580bSAlexander Kolbasov  * Cross call wrapper routine for cu_cpc_program()
737b885580bSAlexander Kolbasov  *
738b885580bSAlexander Kolbasov  * Checks to make sure that counters on CPU aren't being used by someone else
739b885580bSAlexander Kolbasov  * before calling cu_cpc_program() since cu_cpc_program() needs to assert that
740b885580bSAlexander Kolbasov  * nobody else is using the counters to catch and prevent any broken code.
741b885580bSAlexander Kolbasov  * Also, this check needs to happen on the target CPU since the CPU's CPC
742b885580bSAlexander Kolbasov  * context can only be changed while running on the CPU.
743b885580bSAlexander Kolbasov  *
744b885580bSAlexander Kolbasov  * If the first argument is TRUE, cu_cpc_program_xcall also checks that there is
745b885580bSAlexander Kolbasov  * no valid thread bound cpc context. This is important to check to prevent
746b885580bSAlexander Kolbasov  * re-programming thread counters with CU counters when CPU is coming on-line.
747b885580bSAlexander Kolbasov  */
748b885580bSAlexander Kolbasov static void
cu_cpc_program_xcall(uintptr_t arg,int * err)749b885580bSAlexander Kolbasov cu_cpc_program_xcall(uintptr_t arg, int *err)
750b885580bSAlexander Kolbasov {
751b885580bSAlexander Kolbasov 	boolean_t	avoid_thread_context = (boolean_t)arg;
752b885580bSAlexander Kolbasov 
753b885580bSAlexander Kolbasov 	kpreempt_disable();
754b885580bSAlexander Kolbasov 
755b885580bSAlexander Kolbasov 	if (CPU->cpu_cpc_ctx != NULL &&
756b885580bSAlexander Kolbasov 	    !(CPU->cpu_cpc_ctx->kc_flags & KCPC_CTX_INVALID_STOPPED)) {
757b885580bSAlexander Kolbasov 		*err = -100;
758b885580bSAlexander Kolbasov 		kpreempt_enable();
759b885580bSAlexander Kolbasov 		return;
760b885580bSAlexander Kolbasov 	}
761b885580bSAlexander Kolbasov 
762b885580bSAlexander Kolbasov 	if (avoid_thread_context && (curthread->t_cpc_ctx != NULL) &&
763b885580bSAlexander Kolbasov 	    !(curthread->t_cpc_ctx->kc_flags & KCPC_CTX_INVALID_STOPPED)) {
764b885580bSAlexander Kolbasov 		*err = -200;
765b885580bSAlexander Kolbasov 		kpreempt_enable();
766b885580bSAlexander Kolbasov 		return;
767b885580bSAlexander Kolbasov 	}
768b885580bSAlexander Kolbasov 
769b885580bSAlexander Kolbasov 	cu_cpc_program(CPU, err);
770b885580bSAlexander Kolbasov 	kpreempt_enable();
771b885580bSAlexander Kolbasov }
772b885580bSAlexander Kolbasov 
773b885580bSAlexander Kolbasov 
774b885580bSAlexander Kolbasov /*
775b885580bSAlexander Kolbasov  * Unprogram counters for capacity and utilization on given CPU
776b885580bSAlexander Kolbasov  * This function should be always executed on the target CPU at high PIL
777b885580bSAlexander Kolbasov  */
778b885580bSAlexander Kolbasov void
cu_cpc_unprogram(cpu_t * cp,int * err)779b885580bSAlexander Kolbasov cu_cpc_unprogram(cpu_t *cp, int *err)
780b885580bSAlexander Kolbasov {
781b885580bSAlexander Kolbasov 	cu_cpc_ctx_t	*cpu_ctx;
782b885580bSAlexander Kolbasov 	kcpc_ctx_t	*ctx;
783b885580bSAlexander Kolbasov 	cu_cpu_info_t	*cu_cpu_info;
784b885580bSAlexander Kolbasov 
785b885580bSAlexander Kolbasov 	ASSERT(IS_HIPIL());
786b885580bSAlexander Kolbasov 	/*
787b885580bSAlexander Kolbasov 	 * Should be running on given CPU with preemption disabled to keep CPU
788b885580bSAlexander Kolbasov 	 * from disappearing and make sure flags and CPC context don't change
789b885580bSAlexander Kolbasov 	 * from underneath us
790b885580bSAlexander Kolbasov 	 */
791b885580bSAlexander Kolbasov 	kpreempt_disable();
792b885580bSAlexander Kolbasov 	ASSERT(cp == CPU);
793b885580bSAlexander Kolbasov 
794b885580bSAlexander Kolbasov 	/*
795b885580bSAlexander Kolbasov 	 * Module not on
796b885580bSAlexander Kolbasov 	 */
797b885580bSAlexander Kolbasov 	if (!(cu_flags & CU_FLAG_ON)) {
798b885580bSAlexander Kolbasov 		*err = -1;
799b885580bSAlexander Kolbasov 		kpreempt_enable();
800b885580bSAlexander Kolbasov 		return;
801b885580bSAlexander Kolbasov 	}
802b885580bSAlexander Kolbasov 
803b885580bSAlexander Kolbasov 	cu_cpu_info = cp->cpu_cu_info;
804b885580bSAlexander Kolbasov 	if (cu_cpu_info == NULL) {
805b885580bSAlexander Kolbasov 		*err = -3;
806b885580bSAlexander Kolbasov 		kpreempt_enable();
807b885580bSAlexander Kolbasov 		return;
808b885580bSAlexander Kolbasov 	}
809b885580bSAlexander Kolbasov 
810b885580bSAlexander Kolbasov 	/*
811b885580bSAlexander Kolbasov 	 * Counters turned off already
812b885580bSAlexander Kolbasov 	 */
813b885580bSAlexander Kolbasov 	if (!(cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON)) {
814b885580bSAlexander Kolbasov 		*err = 1;
815b885580bSAlexander Kolbasov 		kpreempt_enable();
816b885580bSAlexander Kolbasov 		return;
817b885580bSAlexander Kolbasov 	}
818b885580bSAlexander Kolbasov 
819b885580bSAlexander Kolbasov 	/*
820b885580bSAlexander Kolbasov 	 * Update counter statistics
821b885580bSAlexander Kolbasov 	 */
822b885580bSAlexander Kolbasov 	(void) cu_cpu_update(cp, B_FALSE);
823b885580bSAlexander Kolbasov 
824b885580bSAlexander Kolbasov 	/*
825b885580bSAlexander Kolbasov 	 * Get CPU's CPC context needed for capacity and utilization
826b885580bSAlexander Kolbasov 	 */
827b885580bSAlexander Kolbasov 	cpu_ctx = &cu_cpu_info->cu_cpc_ctx;
828b885580bSAlexander Kolbasov 	if (cpu_ctx->nctx <= 0 || cpu_ctx->ctx_ptr_array == NULL ||
829b885580bSAlexander Kolbasov 	    cpu_ctx->ctx_ptr_array_sz <= 0) {
830b885580bSAlexander Kolbasov 		*err = -5;
831b885580bSAlexander Kolbasov 		kpreempt_enable();
832b885580bSAlexander Kolbasov 		return;
833b885580bSAlexander Kolbasov 	}
834b885580bSAlexander Kolbasov 	ctx = cpu_ctx->ctx_ptr_array[cpu_ctx->cur_index];
835b885580bSAlexander Kolbasov 
836b885580bSAlexander Kolbasov 	/*
837b885580bSAlexander Kolbasov 	 * CPU's CPC context should be current capacity and utilization CPC
838b885580bSAlexander Kolbasov 	 * context
839b885580bSAlexander Kolbasov 	 */
840b885580bSAlexander Kolbasov 	ASSERT(cp->cpu_cpc_ctx == ctx);
841b885580bSAlexander Kolbasov 	if (cp->cpu_cpc_ctx != ctx) {
842b885580bSAlexander Kolbasov 		*err = -6;
843b885580bSAlexander Kolbasov 		kpreempt_enable();
844b885580bSAlexander Kolbasov 		return;
845b885580bSAlexander Kolbasov 	}
846b885580bSAlexander Kolbasov 
847b885580bSAlexander Kolbasov 	/*
848b885580bSAlexander Kolbasov 	 * Unprogram counters on CPU.
849b885580bSAlexander Kolbasov 	 */
850b885580bSAlexander Kolbasov 	kcpc_unprogram(ctx, B_FALSE);
851b885580bSAlexander Kolbasov 
852b885580bSAlexander Kolbasov 	ASSERT(ctx->kc_flags & KCPC_CTX_INVALID_STOPPED);
853b885580bSAlexander Kolbasov 
854b885580bSAlexander Kolbasov 	/*
855b885580bSAlexander Kolbasov 	 * Unset state in CPU structure saying that CPU's counters are
856b885580bSAlexander Kolbasov 	 * programmed
857b885580bSAlexander Kolbasov 	 */
858b885580bSAlexander Kolbasov 	cp->cpu_cpc_ctx = NULL;
859b885580bSAlexander Kolbasov 	cu_cpu_info->cu_flag &= ~CU_CPU_CNTRS_ON;
860b885580bSAlexander Kolbasov 
861b885580bSAlexander Kolbasov 	*err = 0;
862b885580bSAlexander Kolbasov 	kpreempt_enable();
863b885580bSAlexander Kolbasov }
864b885580bSAlexander Kolbasov 
865b885580bSAlexander Kolbasov 
866b885580bSAlexander Kolbasov /*
867b885580bSAlexander Kolbasov  * Add given counter event to list of CPC requests
868b885580bSAlexander Kolbasov  */
869b885580bSAlexander Kolbasov static int
cu_cpc_req_add(char * event,kcpc_request_list_t * reqs,int nreqs,cu_cntr_stats_t * stats,int kmem_flags,int * nevents)870b885580bSAlexander Kolbasov cu_cpc_req_add(char *event, kcpc_request_list_t *reqs, int nreqs,
871b885580bSAlexander Kolbasov     cu_cntr_stats_t *stats, int kmem_flags, int *nevents)
872b885580bSAlexander Kolbasov {
873b885580bSAlexander Kolbasov 	int	n;
874b885580bSAlexander Kolbasov 	int	retval;
875b885580bSAlexander Kolbasov 	uint_t  flags;
876b885580bSAlexander Kolbasov 
877b885580bSAlexander Kolbasov 	/*
878b885580bSAlexander Kolbasov 	 * Return error when no counter event specified, counter event not
879b885580bSAlexander Kolbasov 	 * supported by CPC's PCBE, or number of events not given
880b885580bSAlexander Kolbasov 	 */
881b885580bSAlexander Kolbasov 	if (event == NULL || kcpc_event_supported(event) == B_FALSE ||
882b885580bSAlexander Kolbasov 	    nevents == NULL)
883b885580bSAlexander Kolbasov 		return (-1);
884b885580bSAlexander Kolbasov 
885b885580bSAlexander Kolbasov 	n = *nevents;
886b885580bSAlexander Kolbasov 
887b885580bSAlexander Kolbasov 	/*
888b885580bSAlexander Kolbasov 	 * Only count number of counter events needed if list
889b885580bSAlexander Kolbasov 	 * where to add CPC requests not given
890b885580bSAlexander Kolbasov 	 */
891b885580bSAlexander Kolbasov 	if (reqs == NULL) {
892b885580bSAlexander Kolbasov 		n++;
893b885580bSAlexander Kolbasov 		*nevents = n;
894b885580bSAlexander Kolbasov 		return (-3);
895b885580bSAlexander Kolbasov 	}
896b885580bSAlexander Kolbasov 
897b885580bSAlexander Kolbasov 	/*
898b885580bSAlexander Kolbasov 	 * Return error when stats not given or not enough room on list of CPC
899b885580bSAlexander Kolbasov 	 * requests for more counter events
900b885580bSAlexander Kolbasov 	 */
901b885580bSAlexander Kolbasov 	if (stats == NULL || (nreqs <= 0 && n >= nreqs))
902b885580bSAlexander Kolbasov 		return (-4);
903b885580bSAlexander Kolbasov 
904b885580bSAlexander Kolbasov 	/*
905b885580bSAlexander Kolbasov 	 * Use flags in cu_cpc_flags to program counters and enable overflow
906b885580bSAlexander Kolbasov 	 * interrupts/traps (unless PCBE can't handle overflow interrupts) so
907b885580bSAlexander Kolbasov 	 * PCBE can catch counters before they wrap to hopefully give us an
908b885580bSAlexander Kolbasov 	 * accurate (64-bit) virtualized counter
909b885580bSAlexander Kolbasov 	 */
910b885580bSAlexander Kolbasov 	flags = cu_cpc_flags;
911b885580bSAlexander Kolbasov 	if ((kcpc_pcbe_capabilities() & CPC_CAP_OVERFLOW_INTERRUPT) == 0)
912b885580bSAlexander Kolbasov 		flags &= ~CPC_OVF_NOTIFY_EMT;
913b885580bSAlexander Kolbasov 
914b885580bSAlexander Kolbasov 	/*
915b885580bSAlexander Kolbasov 	 * Add CPC request to list
916b885580bSAlexander Kolbasov 	 */
917b885580bSAlexander Kolbasov 	retval = kcpc_reqs_add(reqs, event, cu_cpc_preset_value,
918b885580bSAlexander Kolbasov 	    flags, 0, NULL, &stats[n], kmem_flags);
919b885580bSAlexander Kolbasov 
920b885580bSAlexander Kolbasov 	if (retval != 0)
921b885580bSAlexander Kolbasov 		return (-5);
922b885580bSAlexander Kolbasov 
923b885580bSAlexander Kolbasov 	n++;
924b885580bSAlexander Kolbasov 	*nevents = n;
925b885580bSAlexander Kolbasov 	return (0);
926b885580bSAlexander Kolbasov }
927b885580bSAlexander Kolbasov 
928b885580bSAlexander Kolbasov static void
cu_cpu_info_detach_xcall(void)929b885580bSAlexander Kolbasov cu_cpu_info_detach_xcall(void)
930b885580bSAlexander Kolbasov {
931b885580bSAlexander Kolbasov 	ASSERT(IS_HIPIL());
932b885580bSAlexander Kolbasov 
933b885580bSAlexander Kolbasov 	CPU->cpu_cu_info = NULL;
934b885580bSAlexander Kolbasov }
935b885580bSAlexander Kolbasov 
936b885580bSAlexander Kolbasov 
937b885580bSAlexander Kolbasov /*
938b885580bSAlexander Kolbasov  * Enable or disable collection of capacity/utilization data for a current CPU.
939b885580bSAlexander Kolbasov  * Counters are enabled if 'on' argument is True and disabled if it is False.
940b885580bSAlexander Kolbasov  * This function should be always executed at high PIL
941b885580bSAlexander Kolbasov  */
942b885580bSAlexander Kolbasov static void
cu_cpc_trigger(uintptr_t arg1,uintptr_t arg2)943b885580bSAlexander Kolbasov cu_cpc_trigger(uintptr_t arg1, uintptr_t arg2)
944b885580bSAlexander Kolbasov {
945b885580bSAlexander Kolbasov 	cpu_t		*cp = (cpu_t *)arg1;
946b885580bSAlexander Kolbasov 	boolean_t	on = (boolean_t)arg2;
947b885580bSAlexander Kolbasov 	int		error;
948b885580bSAlexander Kolbasov 	cu_cpu_info_t	*cu_cpu_info;
949b885580bSAlexander Kolbasov 
950b885580bSAlexander Kolbasov 	ASSERT(IS_HIPIL());
951b885580bSAlexander Kolbasov 	kpreempt_disable();
952b885580bSAlexander Kolbasov 	ASSERT(cp == CPU);
953b885580bSAlexander Kolbasov 
954b885580bSAlexander Kolbasov 	if (!(cu_flags & CU_FLAG_ON)) {
955b885580bSAlexander Kolbasov 		kpreempt_enable();
956b885580bSAlexander Kolbasov 		return;
957b885580bSAlexander Kolbasov 	}
958b885580bSAlexander Kolbasov 
959b885580bSAlexander Kolbasov 	cu_cpu_info = cp->cpu_cu_info;
960b885580bSAlexander Kolbasov 	if (cu_cpu_info == NULL) {
961b885580bSAlexander Kolbasov 		kpreempt_enable();
962b885580bSAlexander Kolbasov 		return;
963b885580bSAlexander Kolbasov 	}
964b885580bSAlexander Kolbasov 
965b885580bSAlexander Kolbasov 	ASSERT(!cu_cpu_info->cu_disabled ||
966b885580bSAlexander Kolbasov 	    !(cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON));
967b885580bSAlexander Kolbasov 
968b885580bSAlexander Kolbasov 	if (on) {
969b885580bSAlexander Kolbasov 		/*
970b885580bSAlexander Kolbasov 		 * Decrement the cu_disabled counter.
971b885580bSAlexander Kolbasov 		 * Once it drops to zero, call cu_cpc_program.
972b885580bSAlexander Kolbasov 		 */
973b885580bSAlexander Kolbasov 		if (cu_cpu_info->cu_disabled > 0)
974b885580bSAlexander Kolbasov 			cu_cpu_info->cu_disabled--;
975b885580bSAlexander Kolbasov 		if (cu_cpu_info->cu_disabled == 0)
976b885580bSAlexander Kolbasov 			cu_cpc_program(CPU, &error);
977b885580bSAlexander Kolbasov 	} else if (cu_cpu_info->cu_disabled++ == 0) {
978b885580bSAlexander Kolbasov 		/*
979b885580bSAlexander Kolbasov 		 * This is the first attempt to disable CU, so turn it off
980b885580bSAlexander Kolbasov 		 */
981b885580bSAlexander Kolbasov 		cu_cpc_unprogram(cp, &error);
982b885580bSAlexander Kolbasov 		ASSERT(!(cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON));
983b885580bSAlexander Kolbasov 	}
984b885580bSAlexander Kolbasov 
985b885580bSAlexander Kolbasov 	kpreempt_enable();
986b885580bSAlexander Kolbasov }
987b885580bSAlexander Kolbasov 
988b885580bSAlexander Kolbasov 
989b885580bSAlexander Kolbasov /*
990b885580bSAlexander Kolbasov  * Callback for changes in CPU states
991b885580bSAlexander Kolbasov  * Used to enable or disable hardware performance counters on CPUs that are
992b885580bSAlexander Kolbasov  * turned on or off
993b885580bSAlexander Kolbasov  *
994b885580bSAlexander Kolbasov  * NOTE: cpc should be programmed/unprogrammed while running on the target CPU.
995b885580bSAlexander Kolbasov  * We have to use thread_affinity_set to hop to the right CPU because these
996b885580bSAlexander Kolbasov  * routines expect cpu_lock held, so we can't cross-call other CPUs while
997b885580bSAlexander Kolbasov  * holding CPU lock.
998b885580bSAlexander Kolbasov  */
999b885580bSAlexander Kolbasov static int
1000b885580bSAlexander Kolbasov /* LINTED E_FUNC_ARG_UNUSED */
cu_cpu_callback(cpu_setup_t what,int id,void * arg)1001b885580bSAlexander Kolbasov cu_cpu_callback(cpu_setup_t what, int id, void *arg)
1002b885580bSAlexander Kolbasov {
1003b885580bSAlexander Kolbasov 	cpu_t	*cp;
1004b885580bSAlexander Kolbasov 	int	retval = 0;
1005b885580bSAlexander Kolbasov 
1006b885580bSAlexander Kolbasov 	ASSERT(MUTEX_HELD(&cpu_lock));
1007b885580bSAlexander Kolbasov 
1008b885580bSAlexander Kolbasov 	if (!(cu_flags & CU_FLAG_ON))
1009b885580bSAlexander Kolbasov 		return (-1);
1010b885580bSAlexander Kolbasov 
1011b885580bSAlexander Kolbasov 	cp = cpu_get(id);
1012b885580bSAlexander Kolbasov 	if (cp == NULL)
1013b885580bSAlexander Kolbasov 		return (-2);
1014b885580bSAlexander Kolbasov 
1015b885580bSAlexander Kolbasov 	switch (what) {
1016b885580bSAlexander Kolbasov 	case CPU_ON:
1017b885580bSAlexander Kolbasov 		/*
1018b885580bSAlexander Kolbasov 		 * Setup counters on CPU being turned on
1019b885580bSAlexander Kolbasov 		 */
1020b885580bSAlexander Kolbasov 		retval = cu_cpu_init(cp, cu_cpc_reqs);
1021b885580bSAlexander Kolbasov 
1022b885580bSAlexander Kolbasov 		/*
1023b885580bSAlexander Kolbasov 		 * Reset list of counter event requests so its space can be
1024b885580bSAlexander Kolbasov 		 * reused for a different set of requests for next CPU
1025b885580bSAlexander Kolbasov 		 */
1026b885580bSAlexander Kolbasov 		(void) kcpc_reqs_reset(cu_cpc_reqs);
1027b885580bSAlexander Kolbasov 		break;
1028b885580bSAlexander Kolbasov 	case CPU_INTR_ON:
1029b885580bSAlexander Kolbasov 		/*
1030b885580bSAlexander Kolbasov 		 * Setup counters on CPU being turned on.
1031b885580bSAlexander Kolbasov 		 */
1032b885580bSAlexander Kolbasov 		retval = cu_cpu_run(cp, cu_cpc_program_xcall,
1033b885580bSAlexander Kolbasov 		    (uintptr_t)B_TRUE);
1034b885580bSAlexander Kolbasov 		break;
1035b885580bSAlexander Kolbasov 	case CPU_OFF:
1036b885580bSAlexander Kolbasov 		/*
1037b885580bSAlexander Kolbasov 		 * Disable counters on CPU being turned off. Counters will not
1038b885580bSAlexander Kolbasov 		 * be re-enabled on this CPU until it comes back online.
1039b885580bSAlexander Kolbasov 		 */
1040b885580bSAlexander Kolbasov 		cu_cpu_disable(cp);
1041b885580bSAlexander Kolbasov 		ASSERT(!CU_CPC_ON(cp));
1042b885580bSAlexander Kolbasov 		retval = cu_cpu_fini(cp);
1043b885580bSAlexander Kolbasov 		break;
1044b885580bSAlexander Kolbasov 	default:
1045b885580bSAlexander Kolbasov 		break;
1046b885580bSAlexander Kolbasov 	}
1047b885580bSAlexander Kolbasov 	return (retval);
1048b885580bSAlexander Kolbasov }
1049b885580bSAlexander Kolbasov 
1050b885580bSAlexander Kolbasov 
1051b885580bSAlexander Kolbasov /*
1052b885580bSAlexander Kolbasov  * Disable or enable Capacity Utilization counters on a given CPU. This function
1053b885580bSAlexander Kolbasov  * can be called from any CPU to disable counters on the given CPU.
1054b885580bSAlexander Kolbasov  */
1055b885580bSAlexander Kolbasov static void
cu_cpu_disable(cpu_t * cp)1056b885580bSAlexander Kolbasov cu_cpu_disable(cpu_t *cp)
1057b885580bSAlexander Kolbasov {
1058b885580bSAlexander Kolbasov 	cpu_call(cp, cu_cpc_trigger, (uintptr_t)cp, (uintptr_t)B_FALSE);
1059b885580bSAlexander Kolbasov }
1060b885580bSAlexander Kolbasov 
1061b885580bSAlexander Kolbasov 
1062b885580bSAlexander Kolbasov static void
cu_cpu_enable(cpu_t * cp)1063b885580bSAlexander Kolbasov cu_cpu_enable(cpu_t *cp)
1064b885580bSAlexander Kolbasov {
1065b885580bSAlexander Kolbasov 	cpu_call(cp, cu_cpc_trigger, (uintptr_t)cp, (uintptr_t)B_TRUE);
1066b885580bSAlexander Kolbasov }
1067b885580bSAlexander Kolbasov 
1068b885580bSAlexander Kolbasov 
1069b885580bSAlexander Kolbasov /*
1070b885580bSAlexander Kolbasov  * Setup capacity and utilization support for given CPU
1071b885580bSAlexander Kolbasov  *
1072b885580bSAlexander Kolbasov  * NOTE: Use KM_NOSLEEP for kmem_{,z}alloc() since cpu_lock is held and free
1073b885580bSAlexander Kolbasov  *	 everything that has been successfully allocated including cpu_cu_info
1074b885580bSAlexander Kolbasov  *	if any memory allocation fails
1075b885580bSAlexander Kolbasov  */
1076b885580bSAlexander Kolbasov static int
cu_cpu_init(cpu_t * cp,kcpc_request_list_t * reqs)1077b885580bSAlexander Kolbasov cu_cpu_init(cpu_t *cp, kcpc_request_list_t *reqs)
1078b885580bSAlexander Kolbasov {
1079b885580bSAlexander Kolbasov 	kcpc_ctx_t	**ctx_ptr_array;
1080b885580bSAlexander Kolbasov 	size_t		ctx_ptr_array_sz;
1081b885580bSAlexander Kolbasov 	cu_cpc_ctx_t	*cpu_ctx;
1082b885580bSAlexander Kolbasov 	cu_cpu_info_t	*cu_cpu_info;
1083b885580bSAlexander Kolbasov 	int		n;
1084b885580bSAlexander Kolbasov 
1085b885580bSAlexander Kolbasov 	/*
1086b885580bSAlexander Kolbasov 	 * cpu_lock should be held and protect against CPU going away and races
1087b885580bSAlexander Kolbasov 	 * with cu_{init,fini,cpu_fini}()
1088b885580bSAlexander Kolbasov 	 */
1089b885580bSAlexander Kolbasov 	ASSERT(MUTEX_HELD(&cpu_lock));
1090b885580bSAlexander Kolbasov 
1091b885580bSAlexander Kolbasov 	/*
1092b885580bSAlexander Kolbasov 	 * Return if not ready to setup counters yet
1093b885580bSAlexander Kolbasov 	 */
1094b885580bSAlexander Kolbasov 	if (!(cu_flags & CU_FLAG_READY))
1095b885580bSAlexander Kolbasov 		return (-1);
1096b885580bSAlexander Kolbasov 
1097b885580bSAlexander Kolbasov 	if (cp->cpu_cu_info == NULL) {
1098b885580bSAlexander Kolbasov 		cp->cpu_cu_info = kmem_zalloc(sizeof (cu_cpu_info_t),
1099b885580bSAlexander Kolbasov 		    KM_NOSLEEP);
1100b885580bSAlexander Kolbasov 		if (cp->cpu_cu_info == NULL)
1101b885580bSAlexander Kolbasov 			return (-2);
1102b885580bSAlexander Kolbasov 	}
1103b885580bSAlexander Kolbasov 
1104b885580bSAlexander Kolbasov 	/*
1105b885580bSAlexander Kolbasov 	 * Get capacity and utilization CPC context for CPU and check to see
1106b885580bSAlexander Kolbasov 	 * whether it has been setup already
1107b885580bSAlexander Kolbasov 	 */
1108b885580bSAlexander Kolbasov 	cu_cpu_info = cp->cpu_cu_info;
1109b885580bSAlexander Kolbasov 	cu_cpu_info->cu_cpu = cp;
1110b885580bSAlexander Kolbasov 	cu_cpu_info->cu_disabled = dtrace_cpc_in_use ? 1 : 0;
1111b885580bSAlexander Kolbasov 
1112b885580bSAlexander Kolbasov 	cpu_ctx = &cu_cpu_info->cu_cpc_ctx;
1113b885580bSAlexander Kolbasov 	if (cpu_ctx->nctx > 0 && cpu_ctx->ctx_ptr_array != NULL &&
1114b885580bSAlexander Kolbasov 	    cpu_ctx->ctx_ptr_array_sz > 0) {
1115b885580bSAlexander Kolbasov 		return (1);
1116b885580bSAlexander Kolbasov 	}
1117b885580bSAlexander Kolbasov 
1118b885580bSAlexander Kolbasov 	/*
1119b885580bSAlexander Kolbasov 	 * Should have no contexts since it hasn't been setup already
1120b885580bSAlexander Kolbasov 	 */
1121b885580bSAlexander Kolbasov 	ASSERT(cpu_ctx->nctx == 0 && cpu_ctx->ctx_ptr_array == NULL &&
1122b885580bSAlexander Kolbasov 	    cpu_ctx->ctx_ptr_array_sz == 0);
1123b885580bSAlexander Kolbasov 
1124b885580bSAlexander Kolbasov 	/*
1125b885580bSAlexander Kolbasov 	 * Determine how many CPC events needed to measure capacity and
1126b885580bSAlexander Kolbasov 	 * utilization for this CPU, allocate space for counter statistics for
1127b885580bSAlexander Kolbasov 	 * each event, and fill in list of CPC event requests with corresponding
1128b885580bSAlexander Kolbasov 	 * counter stats for each request to make attributing counter data
1129b885580bSAlexander Kolbasov 	 * easier later....
1130b885580bSAlexander Kolbasov 	 */
1131b885580bSAlexander Kolbasov 	n = cu_cpc_init(cp, NULL, 0);
1132b885580bSAlexander Kolbasov 	if (n <= 0) {
1133b885580bSAlexander Kolbasov 		(void) cu_cpu_fini(cp);
1134b885580bSAlexander Kolbasov 		return (-3);
1135b885580bSAlexander Kolbasov 	}
1136b885580bSAlexander Kolbasov 
1137b885580bSAlexander Kolbasov 	cu_cpu_info->cu_cntr_stats = kmem_zalloc(n * sizeof (cu_cntr_stats_t),
1138b885580bSAlexander Kolbasov 	    KM_NOSLEEP);
1139b885580bSAlexander Kolbasov 	if (cu_cpu_info->cu_cntr_stats == NULL) {
1140b885580bSAlexander Kolbasov 		(void) cu_cpu_fini(cp);
1141b885580bSAlexander Kolbasov 		return (-4);
1142b885580bSAlexander Kolbasov 	}
1143b885580bSAlexander Kolbasov 
1144b885580bSAlexander Kolbasov 	cu_cpu_info->cu_ncntr_stats = n;
1145b885580bSAlexander Kolbasov 
1146b885580bSAlexander Kolbasov 	n = cu_cpc_init(cp, reqs, n);
1147b885580bSAlexander Kolbasov 	if (n <= 0) {
1148b885580bSAlexander Kolbasov 		(void) cu_cpu_fini(cp);
1149b885580bSAlexander Kolbasov 		return (-5);
1150b885580bSAlexander Kolbasov 	}
1151b885580bSAlexander Kolbasov 
1152b885580bSAlexander Kolbasov 	/*
1153b885580bSAlexander Kolbasov 	 * Create CPC context with given requests
1154b885580bSAlexander Kolbasov 	 */
1155b885580bSAlexander Kolbasov 	ctx_ptr_array = NULL;
1156b885580bSAlexander Kolbasov 	ctx_ptr_array_sz = 0;
1157b885580bSAlexander Kolbasov 	n = kcpc_cpu_ctx_create(cp, reqs, KM_NOSLEEP, &ctx_ptr_array,
1158b885580bSAlexander Kolbasov 	    &ctx_ptr_array_sz);
1159b885580bSAlexander Kolbasov 	if (n <= 0) {
1160b885580bSAlexander Kolbasov 		(void) cu_cpu_fini(cp);
1161b885580bSAlexander Kolbasov 		return (-6);
1162b885580bSAlexander Kolbasov 	}
1163b885580bSAlexander Kolbasov 
1164b885580bSAlexander Kolbasov 	/*
1165b885580bSAlexander Kolbasov 	 * Should have contexts
1166b885580bSAlexander Kolbasov 	 */
1167b885580bSAlexander Kolbasov 	ASSERT(n > 0 && ctx_ptr_array != NULL && ctx_ptr_array_sz > 0);
1168b885580bSAlexander Kolbasov 	if (ctx_ptr_array == NULL || ctx_ptr_array_sz <= 0) {
1169b885580bSAlexander Kolbasov 		(void) cu_cpu_fini(cp);
1170b885580bSAlexander Kolbasov 		return (-7);
1171b885580bSAlexander Kolbasov 	}
1172b885580bSAlexander Kolbasov 
1173b885580bSAlexander Kolbasov 	/*
1174b885580bSAlexander Kolbasov 	 * Fill in CPC context info for CPU needed for capacity and utilization
1175b885580bSAlexander Kolbasov 	 */
1176b885580bSAlexander Kolbasov 	cpu_ctx->cur_index = 0;
1177b885580bSAlexander Kolbasov 	cpu_ctx->nctx = n;
1178b885580bSAlexander Kolbasov 	cpu_ctx->ctx_ptr_array = ctx_ptr_array;
1179b885580bSAlexander Kolbasov 	cpu_ctx->ctx_ptr_array_sz = ctx_ptr_array_sz;
1180b885580bSAlexander Kolbasov 	return (0);
1181b885580bSAlexander Kolbasov }
1182b885580bSAlexander Kolbasov 
1183b885580bSAlexander Kolbasov /*
1184b885580bSAlexander Kolbasov  * Tear down capacity and utilization support for given CPU
1185b885580bSAlexander Kolbasov  */
1186b885580bSAlexander Kolbasov static int
cu_cpu_fini(cpu_t * cp)1187b885580bSAlexander Kolbasov cu_cpu_fini(cpu_t *cp)
1188b885580bSAlexander Kolbasov {
1189b885580bSAlexander Kolbasov 	kcpc_ctx_t	*ctx;
1190b885580bSAlexander Kolbasov 	cu_cpc_ctx_t	*cpu_ctx;
1191b885580bSAlexander Kolbasov 	cu_cpu_info_t	*cu_cpu_info;
1192b885580bSAlexander Kolbasov 	int		i;
1193b885580bSAlexander Kolbasov 	pghw_type_t	pg_hw_type;
1194b885580bSAlexander Kolbasov 
1195b885580bSAlexander Kolbasov 	/*
1196b885580bSAlexander Kolbasov 	 * cpu_lock should be held and protect against CPU going away and races
1197b885580bSAlexander Kolbasov 	 * with cu_{init,fini,cpu_init}()
1198b885580bSAlexander Kolbasov 	 */
1199b885580bSAlexander Kolbasov 	ASSERT(MUTEX_HELD(&cpu_lock));
1200b885580bSAlexander Kolbasov 
1201b885580bSAlexander Kolbasov 	/*
1202b885580bSAlexander Kolbasov 	 * Have to at least be ready to setup counters to have allocated
1203b885580bSAlexander Kolbasov 	 * anything that needs to be deallocated now
1204b885580bSAlexander Kolbasov 	 */
1205b885580bSAlexander Kolbasov 	if (!(cu_flags & CU_FLAG_READY))
1206b885580bSAlexander Kolbasov 		return (-1);
1207b885580bSAlexander Kolbasov 
1208b885580bSAlexander Kolbasov 	/*
1209b885580bSAlexander Kolbasov 	 * Nothing to do if CPU's capacity and utilization info doesn't exist
1210b885580bSAlexander Kolbasov 	 */
1211b885580bSAlexander Kolbasov 	cu_cpu_info = cp->cpu_cu_info;
1212b885580bSAlexander Kolbasov 	if (cu_cpu_info == NULL)
1213b885580bSAlexander Kolbasov 		return (1);
1214b885580bSAlexander Kolbasov 
1215b885580bSAlexander Kolbasov 	/*
1216b885580bSAlexander Kolbasov 	 * Tear down any existing kstats and counter info for each hardware
1217b885580bSAlexander Kolbasov 	 * sharing relationship
1218b885580bSAlexander Kolbasov 	 */
1219b885580bSAlexander Kolbasov 	for (pg_hw_type = PGHW_START; pg_hw_type < PGHW_NUM_COMPONENTS;
1220b885580bSAlexander Kolbasov 	    pg_hw_type++) {
1221b885580bSAlexander Kolbasov 		cu_cntr_info_t	*cntr_info;
1222b885580bSAlexander Kolbasov 
1223b885580bSAlexander Kolbasov 		cntr_info = cu_cpu_info->cu_cntr_info[pg_hw_type];
1224b885580bSAlexander Kolbasov 		if (cntr_info == NULL)
1225b885580bSAlexander Kolbasov 			continue;
1226b885580bSAlexander Kolbasov 
1227b885580bSAlexander Kolbasov 		if (cntr_info->ci_kstat != NULL) {
1228b885580bSAlexander Kolbasov 			kstat_delete(cntr_info->ci_kstat);
1229b885580bSAlexander Kolbasov 			cntr_info->ci_kstat = NULL;
1230b885580bSAlexander Kolbasov 		}
1231b885580bSAlexander Kolbasov 		kmem_free(cntr_info, sizeof (cu_cntr_info_t));
1232b885580bSAlexander Kolbasov 	}
1233b885580bSAlexander Kolbasov 
1234b885580bSAlexander Kolbasov 	/*
1235b885580bSAlexander Kolbasov 	 * Free counter statistics for CPU
1236b885580bSAlexander Kolbasov 	 */
1237b885580bSAlexander Kolbasov 	ASSERT(cu_cpu_info->cu_cntr_stats == NULL ||
1238b885580bSAlexander Kolbasov 	    cu_cpu_info->cu_ncntr_stats > 0);
1239b885580bSAlexander Kolbasov 	if (cu_cpu_info->cu_cntr_stats != NULL &&
1240b885580bSAlexander Kolbasov 	    cu_cpu_info->cu_ncntr_stats > 0) {
1241b885580bSAlexander Kolbasov 		kmem_free(cu_cpu_info->cu_cntr_stats,
1242b885580bSAlexander Kolbasov 		    cu_cpu_info->cu_ncntr_stats * sizeof (cu_cntr_stats_t));
1243b885580bSAlexander Kolbasov 		cu_cpu_info->cu_cntr_stats = NULL;
1244b885580bSAlexander Kolbasov 		cu_cpu_info->cu_ncntr_stats = 0;
1245b885580bSAlexander Kolbasov 	}
1246b885580bSAlexander Kolbasov 
1247b885580bSAlexander Kolbasov 	/*
1248b885580bSAlexander Kolbasov 	 * Get capacity and utilization CPC contexts for given CPU and check to
1249b885580bSAlexander Kolbasov 	 * see whether they have been freed already
1250b885580bSAlexander Kolbasov 	 */
1251b885580bSAlexander Kolbasov 	cpu_ctx = &cu_cpu_info->cu_cpc_ctx;
1252b885580bSAlexander Kolbasov 	if (cpu_ctx != NULL && cpu_ctx->ctx_ptr_array != NULL &&
1253b885580bSAlexander Kolbasov 	    cpu_ctx->ctx_ptr_array_sz > 0) {
1254b885580bSAlexander Kolbasov 		/*
1255b885580bSAlexander Kolbasov 		 * Free CPC contexts for given CPU
1256b885580bSAlexander Kolbasov 		 */
1257b885580bSAlexander Kolbasov 		for (i = 0; i < cpu_ctx->nctx; i++) {
1258b885580bSAlexander Kolbasov 			ctx = cpu_ctx->ctx_ptr_array[i];
1259b885580bSAlexander Kolbasov 			if (ctx == NULL)
1260b885580bSAlexander Kolbasov 				continue;
12615a469116SPatrick Mooney 			kcpc_free_cpu(ctx);
1262b885580bSAlexander Kolbasov 		}
1263b885580bSAlexander Kolbasov 
1264b885580bSAlexander Kolbasov 		/*
1265b885580bSAlexander Kolbasov 		 * Free CPC context pointer array
1266b885580bSAlexander Kolbasov 		 */
1267b885580bSAlexander Kolbasov 		kmem_free(cpu_ctx->ctx_ptr_array, cpu_ctx->ctx_ptr_array_sz);
1268b885580bSAlexander Kolbasov 
1269b885580bSAlexander Kolbasov 		/*
1270b885580bSAlexander Kolbasov 		 * Zero CPC info for CPU
1271b885580bSAlexander Kolbasov 		 */
1272b885580bSAlexander Kolbasov 		bzero(cpu_ctx, sizeof (cu_cpc_ctx_t));
1273b885580bSAlexander Kolbasov 	}
1274b885580bSAlexander Kolbasov 
1275b885580bSAlexander Kolbasov 	/*
1276b885580bSAlexander Kolbasov 	 * Set cp->cpu_cu_info pointer to NULL. Go through cross-call to ensure
1277b885580bSAlexander Kolbasov 	 * that no one is going to access the cpu_cu_info whicch we are going to
1278b885580bSAlexander Kolbasov 	 * free.
1279b885580bSAlexander Kolbasov 	 */
1280b885580bSAlexander Kolbasov 	if (cpu_is_online(cp))
1281b885580bSAlexander Kolbasov 		cpu_call(cp, (cpu_call_func_t)cu_cpu_info_detach_xcall, 0, 0);
1282b885580bSAlexander Kolbasov 	else
1283b885580bSAlexander Kolbasov 		cp->cpu_cu_info = NULL;
1284b885580bSAlexander Kolbasov 
1285b885580bSAlexander Kolbasov 	/*
1286b885580bSAlexander Kolbasov 	 * Free CPU's capacity and utilization info
1287b885580bSAlexander Kolbasov 	 */
1288b885580bSAlexander Kolbasov 	kmem_free(cu_cpu_info, sizeof (cu_cpu_info_t));
1289b885580bSAlexander Kolbasov 
1290b885580bSAlexander Kolbasov 	return (0);
1291b885580bSAlexander Kolbasov }
1292b885580bSAlexander Kolbasov 
1293b885580bSAlexander Kolbasov /*
1294b885580bSAlexander Kolbasov  * Create capacity & utilization kstats for given PG CPU hardware sharing
1295b885580bSAlexander Kolbasov  * relationship
1296b885580bSAlexander Kolbasov  */
1297b885580bSAlexander Kolbasov static void
cu_cpu_kstat_create(pghw_t * pg,cu_cntr_info_t * cntr_info)1298b885580bSAlexander Kolbasov cu_cpu_kstat_create(pghw_t *pg, cu_cntr_info_t *cntr_info)
1299b885580bSAlexander Kolbasov {
1300b885580bSAlexander Kolbasov 	kstat_t		*ks;
1301027bcc9fSToomas Soome 	char		*sharing = pghw_type_string(pg->pghw_hw);
1302d3c97224SAlexander Kolbasov 	char		name[KSTAT_STRLEN + 1];
1303b885580bSAlexander Kolbasov 
1304b885580bSAlexander Kolbasov 	/*
1305b885580bSAlexander Kolbasov 	 * Just return when no counter info or CPU
1306b885580bSAlexander Kolbasov 	 */
1307b885580bSAlexander Kolbasov 	if (cntr_info == NULL || cntr_info->ci_cpu == NULL)
1308b885580bSAlexander Kolbasov 		return;
1309b885580bSAlexander Kolbasov 
1310b885580bSAlexander Kolbasov 	/*
1311d3c97224SAlexander Kolbasov 	 * Canonify PG name to conform to kstat name rules
1312b885580bSAlexander Kolbasov 	 */
1313d3c97224SAlexander Kolbasov 	(void) strncpy(name, pghw_type_string(pg->pghw_hw), KSTAT_STRLEN + 1);
1314d3c97224SAlexander Kolbasov 	strident_canon(name, TASKQ_NAMELEN + 1);
1315b885580bSAlexander Kolbasov 
1316d3c97224SAlexander Kolbasov 	if ((ks = kstat_create_zone("pg_hw_perf_cpu",
1317d3c97224SAlexander Kolbasov 	    cntr_info->ci_cpu->cpu_id,
1318d3c97224SAlexander Kolbasov 	    name, "processor_group", KSTAT_TYPE_NAMED,
1319b885580bSAlexander Kolbasov 	    sizeof (cu_cpu_kstat) / sizeof (kstat_named_t),
1320b885580bSAlexander Kolbasov 	    KSTAT_FLAG_VIRTUAL, GLOBAL_ZONEID)) == NULL)
1321b885580bSAlexander Kolbasov 		return;
1322b885580bSAlexander Kolbasov 
1323b885580bSAlexander Kolbasov 	ks->ks_lock = &pg_cpu_kstat_lock;
1324b885580bSAlexander Kolbasov 	ks->ks_data = &cu_cpu_kstat;
1325b885580bSAlexander Kolbasov 	ks->ks_update = cu_cpu_kstat_update;
1326d3c97224SAlexander Kolbasov 	ks->ks_data_size += strlen(sharing) + 1;
1327b885580bSAlexander Kolbasov 
1328b885580bSAlexander Kolbasov 	ks->ks_private = cntr_info;
1329b885580bSAlexander Kolbasov 	cntr_info->ci_kstat = ks;
1330b885580bSAlexander Kolbasov 	kstat_install(cntr_info->ci_kstat);
1331b885580bSAlexander Kolbasov }
1332b885580bSAlexander Kolbasov 
1333b885580bSAlexander Kolbasov 
1334b885580bSAlexander Kolbasov /*
1335b885580bSAlexander Kolbasov  * Propagate values from CPU capacity & utilization stats to kstats
1336b885580bSAlexander Kolbasov  */
1337b885580bSAlexander Kolbasov static int
cu_cpu_kstat_update(kstat_t * ksp,int rw)1338b885580bSAlexander Kolbasov cu_cpu_kstat_update(kstat_t *ksp, int rw)
1339b885580bSAlexander Kolbasov {
1340b885580bSAlexander Kolbasov 	cpu_t		*cp;
1341b885580bSAlexander Kolbasov 	cu_cntr_info_t	*cntr_info = ksp->ks_private;
1342b885580bSAlexander Kolbasov 	struct cu_cpu_kstat	*kstat = &cu_cpu_kstat;
1343b885580bSAlexander Kolbasov 	pghw_t		*pg;
1344b885580bSAlexander Kolbasov 	cu_cntr_stats_t	*stats;
1345b885580bSAlexander Kolbasov 
1346b885580bSAlexander Kolbasov 	if (rw == KSTAT_WRITE)
1347b885580bSAlexander Kolbasov 		return (EACCES);
1348b885580bSAlexander Kolbasov 
1349d3c97224SAlexander Kolbasov 	cp = cntr_info->ci_cpu;
1350d3c97224SAlexander Kolbasov 	pg = cntr_info->ci_pg;
1351d3c97224SAlexander Kolbasov 	kstat->cu_cpu_id.value.ui32 = cp->cpu_id;
1352d3c97224SAlexander Kolbasov 	kstat->cu_pg_id.value.i32 = ((pg_t *)pg)->pg_id;
1353d3c97224SAlexander Kolbasov 
1354d3c97224SAlexander Kolbasov 	/*
1355d3c97224SAlexander Kolbasov 	 * The caller should have priv_cpc_cpu privilege to get utilization
1356d3c97224SAlexander Kolbasov 	 * data. Callers who do not have the privilege will see zeroes as the
1357d3c97224SAlexander Kolbasov 	 * values.
1358d3c97224SAlexander Kolbasov 	 */
1359d3c97224SAlexander Kolbasov 	if (secpolicy_cpc_cpu(crgetcred()) != 0) {
1360d3c97224SAlexander Kolbasov 		kstat->cu_generation.value.ui32 = cp->cpu_generation;
1361d3c97224SAlexander Kolbasov 		kstat_named_setstr(&kstat->cu_cpu_relationship,
1362d3c97224SAlexander Kolbasov 		    pghw_type_string(pg->pghw_hw));
1363d3c97224SAlexander Kolbasov 
1364d3c97224SAlexander Kolbasov 		kstat->cu_cpu_util.value.ui64 = 0;
1365d3c97224SAlexander Kolbasov 		kstat->cu_cpu_rate.value.ui64 = 0;
1366d3c97224SAlexander Kolbasov 		kstat->cu_cpu_rate_max.value.ui64 = 0;
1367d3c97224SAlexander Kolbasov 		kstat->cu_cpu_time_running.value.ui64 = 0;
1368d3c97224SAlexander Kolbasov 		kstat->cu_cpu_time_stopped.value.ui64 = 0;
1369d3c97224SAlexander Kolbasov 
1370d3c97224SAlexander Kolbasov 		return (0);
1371d3c97224SAlexander Kolbasov 	}
1372d3c97224SAlexander Kolbasov 
1373b885580bSAlexander Kolbasov 	kpreempt_disable();
1374b885580bSAlexander Kolbasov 
1375b885580bSAlexander Kolbasov 	/*
1376b885580bSAlexander Kolbasov 	 * Update capacity and utilization statistics needed for CPU's PG (CPU)
1377b885580bSAlexander Kolbasov 	 * kstats
1378b885580bSAlexander Kolbasov 	 */
1379d3c97224SAlexander Kolbasov 
1380b885580bSAlexander Kolbasov 	(void) cu_cpu_update(cp, B_TRUE);
1381b885580bSAlexander Kolbasov 
1382b885580bSAlexander Kolbasov 	stats = cntr_info->ci_stats;
1383b885580bSAlexander Kolbasov 	kstat->cu_generation.value.ui32 = cp->cpu_generation;
1384d3c97224SAlexander Kolbasov 	kstat_named_setstr(&kstat->cu_cpu_relationship,
1385d3c97224SAlexander Kolbasov 	    pghw_type_string(pg->pghw_hw));
1386b885580bSAlexander Kolbasov 
1387b885580bSAlexander Kolbasov 	kstat->cu_cpu_util.value.ui64 = stats->cs_value_total;
1388b885580bSAlexander Kolbasov 	kstat->cu_cpu_rate.value.ui64 = stats->cs_rate;
1389b885580bSAlexander Kolbasov 	kstat->cu_cpu_rate_max.value.ui64 = stats->cs_rate_max;
1390b885580bSAlexander Kolbasov 	kstat->cu_cpu_time_running.value.ui64 = stats->cs_time_running;
1391b885580bSAlexander Kolbasov 	kstat->cu_cpu_time_stopped.value.ui64 = stats->cs_time_stopped;
1392d3c97224SAlexander Kolbasov 
1393b885580bSAlexander Kolbasov 	/*
1394b885580bSAlexander Kolbasov 	 * Counters are stopped now, so the cs_time_stopped was last
1395b885580bSAlexander Kolbasov 	 * updated at cs_time_start time. Add the time passed since then
1396b885580bSAlexander Kolbasov 	 * to the stopped time.
1397b885580bSAlexander Kolbasov 	 */
1398b885580bSAlexander Kolbasov 	if (!(cp->cpu_cu_info->cu_flag & CU_CPU_CNTRS_ON))
1399b885580bSAlexander Kolbasov 		kstat->cu_cpu_time_stopped.value.ui64 +=
1400b885580bSAlexander Kolbasov 		    gethrtime() - stats->cs_time_start;
1401b885580bSAlexander Kolbasov 
1402b885580bSAlexander Kolbasov 	kpreempt_enable();
1403b885580bSAlexander Kolbasov 
1404b885580bSAlexander Kolbasov 	return (0);
1405b885580bSAlexander Kolbasov }
1406b885580bSAlexander Kolbasov 
1407b885580bSAlexander Kolbasov /*
1408b885580bSAlexander Kolbasov  * Run specified function with specified argument on a given CPU and return
1409b885580bSAlexander Kolbasov  * whatever the function returns
1410b885580bSAlexander Kolbasov  */
1411b885580bSAlexander Kolbasov static int
cu_cpu_run(cpu_t * cp,cu_cpu_func_t func,uintptr_t arg)1412b885580bSAlexander Kolbasov cu_cpu_run(cpu_t *cp, cu_cpu_func_t func, uintptr_t arg)
1413b885580bSAlexander Kolbasov {
1414b885580bSAlexander Kolbasov 	int error = 0;
1415b885580bSAlexander Kolbasov 
1416b885580bSAlexander Kolbasov 	/*
1417b885580bSAlexander Kolbasov 	 * cpu_call() will call func on the CPU specified with given argument
1418b885580bSAlexander Kolbasov 	 * and return func's return value in last argument
1419b885580bSAlexander Kolbasov 	 */
1420027bcc9fSToomas Soome 	cpu_call(cp, (cpu_call_func_t)(uintptr_t)func, arg, (uintptr_t)&error);
1421b885580bSAlexander Kolbasov 	return (error);
1422b885580bSAlexander Kolbasov }
1423b885580bSAlexander Kolbasov 
1424b885580bSAlexander Kolbasov 
1425b885580bSAlexander Kolbasov /*
1426b885580bSAlexander Kolbasov  * Update counter statistics on a given CPU.
1427b885580bSAlexander Kolbasov  *
1428b885580bSAlexander Kolbasov  * If move_to argument is True, execute the function on the CPU specified
1429b885580bSAlexander Kolbasov  * Otherwise, assume that it is already runninng on the right CPU
1430b885580bSAlexander Kolbasov  *
1431b885580bSAlexander Kolbasov  * If move_to is specified, the caller should hold cpu_lock or have preemption
1432b885580bSAlexander Kolbasov  * disabled. Otherwise it is up to the caller to guarantee that things do not
1433b885580bSAlexander Kolbasov  * change in the process.
1434b885580bSAlexander Kolbasov  */
1435b885580bSAlexander Kolbasov int
cu_cpu_update(struct cpu * cp,boolean_t move_to)1436b885580bSAlexander Kolbasov cu_cpu_update(struct cpu *cp, boolean_t move_to)
1437b885580bSAlexander Kolbasov {
1438b885580bSAlexander Kolbasov 	int	retval;
1439b885580bSAlexander Kolbasov 	cu_cpu_info_t	*cu_cpu_info = cp->cpu_cu_info;
1440b885580bSAlexander Kolbasov 	hrtime_t	time_snap;
1441b885580bSAlexander Kolbasov 
1442b885580bSAlexander Kolbasov 	ASSERT(!move_to || MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0);
1443b885580bSAlexander Kolbasov 
1444b885580bSAlexander Kolbasov 	/*
1445b885580bSAlexander Kolbasov 	 * Nothing to do if counters are not programmed
1446b885580bSAlexander Kolbasov 	 */
1447b885580bSAlexander Kolbasov 	if (!(cu_flags & CU_FLAG_ON) ||
1448b885580bSAlexander Kolbasov 	    (cu_cpu_info == NULL) ||
1449b885580bSAlexander Kolbasov 	    !(cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON))
1450b885580bSAlexander Kolbasov 		return (0);
1451b885580bSAlexander Kolbasov 
1452b885580bSAlexander Kolbasov 	/*
1453b885580bSAlexander Kolbasov 	 * Don't update CPU statistics if it was updated recently
1454b885580bSAlexander Kolbasov 	 * and provide old results instead
1455b885580bSAlexander Kolbasov 	 */
1456b885580bSAlexander Kolbasov 	time_snap = gethrtime();
1457b885580bSAlexander Kolbasov 	if ((time_snap - cu_cpu_info->cu_sample_time) < cu_update_threshold) {
1458b885580bSAlexander Kolbasov 		DTRACE_PROBE1(cu__drop__sample, cpu_t *, cp);
1459b885580bSAlexander Kolbasov 		return (0);
1460b885580bSAlexander Kolbasov 	}
1461b885580bSAlexander Kolbasov 
1462b885580bSAlexander Kolbasov 	cu_cpu_info->cu_sample_time = time_snap;
1463b885580bSAlexander Kolbasov 
1464b885580bSAlexander Kolbasov 	/*
1465b885580bSAlexander Kolbasov 	 * CPC counter should be read on the CPU that is running the counter. We
1466b885580bSAlexander Kolbasov 	 * either have to move ourselves to the target CPU or insure that we
1467b885580bSAlexander Kolbasov 	 * already run there.
1468b885580bSAlexander Kolbasov 	 *
1469b885580bSAlexander Kolbasov 	 * We use cross-call to the target CPU to execute kcpc_read() and
1470b885580bSAlexander Kolbasov 	 * cu_cpu_update_stats() there.
1471b885580bSAlexander Kolbasov 	 */
1472b885580bSAlexander Kolbasov 	retval = 0;
1473b885580bSAlexander Kolbasov 	if (move_to)
1474027bcc9fSToomas Soome 		(void) cu_cpu_run(cp, (cu_cpu_func_t)(uintptr_t)kcpc_read,
1475b885580bSAlexander Kolbasov 		    (uintptr_t)cu_cpu_update_stats);
1476b885580bSAlexander Kolbasov 	else {
1477b885580bSAlexander Kolbasov 		retval = kcpc_read((kcpc_update_func_t)cu_cpu_update_stats);
1478b885580bSAlexander Kolbasov 		/*
1479b885580bSAlexander Kolbasov 		 * Offset negative return value by -10 so we can distinguish it
1480b885580bSAlexander Kolbasov 		 * from error return values of this routine vs kcpc_read()
1481b885580bSAlexander Kolbasov 		 */
1482b885580bSAlexander Kolbasov 		if (retval < 0)
1483b885580bSAlexander Kolbasov 			retval -= 10;
1484b885580bSAlexander Kolbasov 	}
1485b885580bSAlexander Kolbasov 
1486b885580bSAlexander Kolbasov 	return (retval);
1487b885580bSAlexander Kolbasov }
1488b885580bSAlexander Kolbasov 
1489b885580bSAlexander Kolbasov 
1490b885580bSAlexander Kolbasov /*
1491b885580bSAlexander Kolbasov  * Update CPU counter statistics for current CPU.
1492b885580bSAlexander Kolbasov  * This function may be called from a cross-call
1493b885580bSAlexander Kolbasov  */
1494b885580bSAlexander Kolbasov static int
cu_cpu_update_stats(cu_cntr_stats_t * stats,uint64_t cntr_value)1495b885580bSAlexander Kolbasov cu_cpu_update_stats(cu_cntr_stats_t *stats, uint64_t cntr_value)
1496b885580bSAlexander Kolbasov {
1497b885580bSAlexander Kolbasov 	cu_cpu_info_t	*cu_cpu_info = CPU->cpu_cu_info;
1498b885580bSAlexander Kolbasov 	uint_t		flags;
1499b885580bSAlexander Kolbasov 	uint64_t	delta;
1500b885580bSAlexander Kolbasov 	hrtime_t	time_delta;
1501b885580bSAlexander Kolbasov 	hrtime_t	time_snap;
1502b885580bSAlexander Kolbasov 
1503b885580bSAlexander Kolbasov 	if (stats == NULL)
1504b885580bSAlexander Kolbasov 		return (-1);
1505b885580bSAlexander Kolbasov 
1506b885580bSAlexander Kolbasov 	/*
1507b885580bSAlexander Kolbasov 	 * Nothing to do if counters are not programmed. This should not happen,
1508b885580bSAlexander Kolbasov 	 * but we check just in case.
1509b885580bSAlexander Kolbasov 	 */
1510b885580bSAlexander Kolbasov 	ASSERT(cu_flags & CU_FLAG_ON);
1511b885580bSAlexander Kolbasov 	ASSERT(cu_cpu_info != NULL);
1512b885580bSAlexander Kolbasov 	if (!(cu_flags & CU_FLAG_ON) ||
1513b885580bSAlexander Kolbasov 	    (cu_cpu_info == NULL))
1514b885580bSAlexander Kolbasov 		return (-2);
1515b885580bSAlexander Kolbasov 
1516b885580bSAlexander Kolbasov 	flags = cu_cpu_info->cu_flag;
1517b885580bSAlexander Kolbasov 	ASSERT(flags & CU_CPU_CNTRS_ON);
1518b885580bSAlexander Kolbasov 	if (!(flags & CU_CPU_CNTRS_ON))
1519b885580bSAlexander Kolbasov 		return (-2);
1520b885580bSAlexander Kolbasov 
1521b885580bSAlexander Kolbasov 	/*
1522b885580bSAlexander Kolbasov 	 * Take snapshot of high resolution timer
1523b885580bSAlexander Kolbasov 	 */
1524b885580bSAlexander Kolbasov 	time_snap = gethrtime();
1525b885580bSAlexander Kolbasov 
1526b885580bSAlexander Kolbasov 	/*
1527b885580bSAlexander Kolbasov 	 * CU counters have just been programmed. We cannot assume that the new
1528b885580bSAlexander Kolbasov 	 * cntr_value continues from where we left off, so use the cntr_value as
1529b885580bSAlexander Kolbasov 	 * the new initial value.
1530b885580bSAlexander Kolbasov 	 */
1531b885580bSAlexander Kolbasov 	if (flags & CU_CPU_CNTRS_OFF_ON)
1532b885580bSAlexander Kolbasov 		stats->cs_value_start = cntr_value;
1533b885580bSAlexander Kolbasov 
1534b885580bSAlexander Kolbasov 	/*
1535b885580bSAlexander Kolbasov 	 * Calculate delta in counter values between start of sampling period
1536b885580bSAlexander Kolbasov 	 * and now
1537b885580bSAlexander Kolbasov 	 */
1538b885580bSAlexander Kolbasov 	delta = cntr_value - stats->cs_value_start;
1539b885580bSAlexander Kolbasov 
1540b885580bSAlexander Kolbasov 	/*
1541b885580bSAlexander Kolbasov 	 * Calculate time between start of sampling period and now
1542b885580bSAlexander Kolbasov 	 */
1543b885580bSAlexander Kolbasov 	time_delta = stats->cs_time_start ?
1544b885580bSAlexander Kolbasov 	    time_snap - stats->cs_time_start :
1545b885580bSAlexander Kolbasov 	    0;
1546b885580bSAlexander Kolbasov 	stats->cs_time_start = time_snap;
1547b885580bSAlexander Kolbasov 	stats->cs_value_start = cntr_value;
1548b885580bSAlexander Kolbasov 
1549b885580bSAlexander Kolbasov 	if (time_delta > 0) { /* wrap shouldn't happen */
1550b885580bSAlexander Kolbasov 		/*
1551b885580bSAlexander Kolbasov 		 * Update either running or stopped time based on the transition
1552b885580bSAlexander Kolbasov 		 * state
1553b885580bSAlexander Kolbasov 		 */
1554b885580bSAlexander Kolbasov 		if (flags & CU_CPU_CNTRS_OFF_ON)
1555b885580bSAlexander Kolbasov 			stats->cs_time_stopped += time_delta;
1556b885580bSAlexander Kolbasov 		else
1557b885580bSAlexander Kolbasov 			stats->cs_time_running += time_delta;
1558b885580bSAlexander Kolbasov 	}
1559b885580bSAlexander Kolbasov 
1560b885580bSAlexander Kolbasov 	/*
1561b885580bSAlexander Kolbasov 	 * Update rest of counter statistics if counter value didn't wrap
1562b885580bSAlexander Kolbasov 	 */
1563b885580bSAlexander Kolbasov 	if (delta > 0) {
1564b885580bSAlexander Kolbasov 		/*
1565b885580bSAlexander Kolbasov 		 * Update utilization rate if the interval between samples is
1566b885580bSAlexander Kolbasov 		 * sufficient.
1567b885580bSAlexander Kolbasov 		 */
1568b885580bSAlexander Kolbasov 		ASSERT(cu_sample_interval_min > CU_SCALE);
1569b885580bSAlexander Kolbasov 		if (time_delta > cu_sample_interval_min)
1570b885580bSAlexander Kolbasov 			stats->cs_rate = CU_RATE(delta, time_delta);
1571b885580bSAlexander Kolbasov 		if (stats->cs_rate_max < stats->cs_rate)
1572b885580bSAlexander Kolbasov 			stats->cs_rate_max = stats->cs_rate;
1573b885580bSAlexander Kolbasov 
1574b885580bSAlexander Kolbasov 		stats->cs_value_last = delta;
1575b885580bSAlexander Kolbasov 		stats->cs_value_total += delta;
1576b885580bSAlexander Kolbasov 	}
1577b885580bSAlexander Kolbasov 
1578b885580bSAlexander Kolbasov 	return (0);
1579b885580bSAlexander Kolbasov }
1580b885580bSAlexander Kolbasov 
1581b885580bSAlexander Kolbasov /*
1582b885580bSAlexander Kolbasov  * Update CMT PG utilization data.
1583b885580bSAlexander Kolbasov  *
1584b885580bSAlexander Kolbasov  * This routine computes the running total utilization and times for the
1585b885580bSAlexander Kolbasov  * specified PG by adding up the total utilization and counter running and
1586b885580bSAlexander Kolbasov  * stopped times of all CPUs in the PG and calculates the utilization rate and
1587b885580bSAlexander Kolbasov  * maximum rate for all CPUs in the PG.
1588b885580bSAlexander Kolbasov  */
1589b885580bSAlexander Kolbasov void
cu_pg_update(pghw_t * pg)1590b885580bSAlexander Kolbasov cu_pg_update(pghw_t *pg)
1591b885580bSAlexander Kolbasov {
1592b885580bSAlexander Kolbasov 	pg_cpu_itr_t	cpu_iter;
1593b885580bSAlexander Kolbasov 	pghw_type_t	pg_hwtype;
1594b885580bSAlexander Kolbasov 	cpu_t		*cpu;
1595b885580bSAlexander Kolbasov 	pghw_util_t	*hw_util = &pg->pghw_stats;
1596b885580bSAlexander Kolbasov 	uint64_t	old_utilization = hw_util->pghw_util;
1597b885580bSAlexander Kolbasov 	hrtime_t	now;
1598b885580bSAlexander Kolbasov 	hrtime_t	time_delta;
1599b885580bSAlexander Kolbasov 	uint64_t	utilization_delta;
1600b885580bSAlexander Kolbasov 
1601b885580bSAlexander Kolbasov 	ASSERT(MUTEX_HELD(&cpu_lock));
1602b885580bSAlexander Kolbasov 
1603b885580bSAlexander Kolbasov 	now = gethrtime();
1604b885580bSAlexander Kolbasov 
1605b885580bSAlexander Kolbasov 	pg_hwtype = pg->pghw_hw;
1606b885580bSAlexander Kolbasov 
1607b885580bSAlexander Kolbasov 	/*
1608b885580bSAlexander Kolbasov 	 * Initialize running total utilization and times for PG to 0
1609b885580bSAlexander Kolbasov 	 */
1610b885580bSAlexander Kolbasov 	hw_util->pghw_util = 0;
1611b885580bSAlexander Kolbasov 	hw_util->pghw_time_running = 0;
1612b885580bSAlexander Kolbasov 	hw_util->pghw_time_stopped = 0;
1613b885580bSAlexander Kolbasov 
1614b885580bSAlexander Kolbasov 	/*
1615b885580bSAlexander Kolbasov 	 * Iterate over all CPUs in the PG and aggregate utilization, running
1616b885580bSAlexander Kolbasov 	 * time and stopped time.
1617b885580bSAlexander Kolbasov 	 */
1618b885580bSAlexander Kolbasov 	PG_CPU_ITR_INIT(pg, cpu_iter);
1619b885580bSAlexander Kolbasov 	while ((cpu = pg_cpu_next(&cpu_iter)) != NULL) {
1620b885580bSAlexander Kolbasov 		cu_cpu_info_t	*cu_cpu_info = cpu->cpu_cu_info;
1621b885580bSAlexander Kolbasov 		cu_cntr_info_t	*cntr_info;
1622b885580bSAlexander Kolbasov 		cu_cntr_stats_t	*stats;
1623b885580bSAlexander Kolbasov 
1624b885580bSAlexander Kolbasov 		if (cu_cpu_info == NULL)
1625b885580bSAlexander Kolbasov 			continue;
1626b885580bSAlexander Kolbasov 
1627b885580bSAlexander Kolbasov 		/*
1628b885580bSAlexander Kolbasov 		 * Update utilization data for the CPU and then
1629b885580bSAlexander Kolbasov 		 * aggregate per CPU running totals for PG
1630b885580bSAlexander Kolbasov 		 */
1631b885580bSAlexander Kolbasov 		(void) cu_cpu_update(cpu, B_TRUE);
1632b885580bSAlexander Kolbasov 		cntr_info = cu_cpu_info->cu_cntr_info[pg_hwtype];
1633b885580bSAlexander Kolbasov 
1634b885580bSAlexander Kolbasov 		if (cntr_info == NULL || (stats = cntr_info->ci_stats) == NULL)
1635b885580bSAlexander Kolbasov 			continue;
1636b885580bSAlexander Kolbasov 
1637b885580bSAlexander Kolbasov 		hw_util->pghw_util += stats->cs_value_total;
1638b885580bSAlexander Kolbasov 		hw_util->pghw_time_running += stats->cs_time_running;
1639b885580bSAlexander Kolbasov 		hw_util->pghw_time_stopped += stats->cs_time_stopped;
1640b885580bSAlexander Kolbasov 
1641b885580bSAlexander Kolbasov 		/*
1642b885580bSAlexander Kolbasov 		 * If counters are stopped now, the pg_time_stopped was last
1643b885580bSAlexander Kolbasov 		 * updated at cs_time_start time. Add the time passed since then
1644b885580bSAlexander Kolbasov 		 * to the stopped time.
1645b885580bSAlexander Kolbasov 		 */
1646b885580bSAlexander Kolbasov 		if (!(cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON))
1647b885580bSAlexander Kolbasov 			hw_util->pghw_time_stopped +=
1648b885580bSAlexander Kolbasov 			    now - stats->cs_time_start;
1649b885580bSAlexander Kolbasov 	}
1650b885580bSAlexander Kolbasov 
1651b885580bSAlexander Kolbasov 	/*
1652b885580bSAlexander Kolbasov 	 * Compute per PG instruction rate and maximum rate
1653b885580bSAlexander Kolbasov 	 */
1654b885580bSAlexander Kolbasov 	time_delta = now - hw_util->pghw_time_stamp;
1655b885580bSAlexander Kolbasov 	hw_util->pghw_time_stamp = now;
1656b885580bSAlexander Kolbasov 
1657b885580bSAlexander Kolbasov 	if (old_utilization == 0)
1658b885580bSAlexander Kolbasov 		return;
1659b885580bSAlexander Kolbasov 
1660b885580bSAlexander Kolbasov 	/*
1661b885580bSAlexander Kolbasov 	 * Calculate change in utilization over sampling period and set this to
1662b885580bSAlexander Kolbasov 	 * 0 if the delta would be 0 or negative which may happen if any CPUs go
1663b885580bSAlexander Kolbasov 	 * offline during the sampling period
1664b885580bSAlexander Kolbasov 	 */
1665b885580bSAlexander Kolbasov 	if (hw_util->pghw_util > old_utilization)
1666b885580bSAlexander Kolbasov 		utilization_delta = hw_util->pghw_util - old_utilization;
1667b885580bSAlexander Kolbasov 	else
1668b885580bSAlexander Kolbasov 		utilization_delta = 0;
1669b885580bSAlexander Kolbasov 
1670b885580bSAlexander Kolbasov 	/*
1671b885580bSAlexander Kolbasov 	 * Update utilization rate if the interval between samples is
1672b885580bSAlexander Kolbasov 	 * sufficient.
1673b885580bSAlexander Kolbasov 	 */
1674b885580bSAlexander Kolbasov 	ASSERT(cu_sample_interval_min > CU_SCALE);
1675b885580bSAlexander Kolbasov 	if (time_delta > CU_SAMPLE_INTERVAL_MIN)
1676b885580bSAlexander Kolbasov 		hw_util->pghw_rate = CU_RATE(utilization_delta, time_delta);
1677b885580bSAlexander Kolbasov 
1678b885580bSAlexander Kolbasov 	/*
1679b885580bSAlexander Kolbasov 	 * Update the maximum observed rate
1680b885580bSAlexander Kolbasov 	 */
1681b885580bSAlexander Kolbasov 	if (hw_util->pghw_rate_max < hw_util->pghw_rate)
1682b885580bSAlexander Kolbasov 		hw_util->pghw_rate_max = hw_util->pghw_rate;
1683b885580bSAlexander Kolbasov }
1684