1fb2caebeSRandy Fishel /*
2fb2caebeSRandy Fishel * CDDL HEADER START
3fb2caebeSRandy Fishel *
4fb2caebeSRandy Fishel * The contents of this file are subject to the terms of the
5fb2caebeSRandy Fishel * Common Development and Distribution License (the "License").
6fb2caebeSRandy Fishel * You may not use this file except in compliance with the License.
7fb2caebeSRandy Fishel *
8fb2caebeSRandy Fishel * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fb2caebeSRandy Fishel * or http://www.opensolaris.org/os/licensing.
10fb2caebeSRandy Fishel * See the License for the specific language governing permissions
11fb2caebeSRandy Fishel * and limitations under the License.
12fb2caebeSRandy Fishel *
13fb2caebeSRandy Fishel * When distributing Covered Code, include this CDDL HEADER in each
14fb2caebeSRandy Fishel * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fb2caebeSRandy Fishel * If applicable, add the following below this CDDL HEADER, with the
16fb2caebeSRandy Fishel * fields enclosed by brackets "[]" replaced with your own identifying
17fb2caebeSRandy Fishel * information: Portions Copyright [yyyy] [name of copyright owner]
18fb2caebeSRandy Fishel *
19fb2caebeSRandy Fishel * CDDL HEADER END
20fb2caebeSRandy Fishel */
21fb2caebeSRandy Fishel /*
22a3114836SGerry Liu * Copyright (c) 2009-2010, Intel Corporation.
23fb2caebeSRandy Fishel * All rights reserved.
24fb2caebeSRandy Fishel */
25fb2caebeSRandy Fishel
26fb2caebeSRandy Fishel /*
27fb2caebeSRandy Fishel * Introduction
28fb2caebeSRandy Fishel * This file implements a CPU event notification mechanism to signal clients
29fb2caebeSRandy Fishel * which are interested in CPU related events.
30fb2caebeSRandy Fishel * Currently it only supports CPU idle state change events which will be
31fb2caebeSRandy Fishel * triggered just before CPU entering hardware idle state and just after CPU
32fb2caebeSRandy Fishel * wakes up from hardware idle state.
33fb2caebeSRandy Fishel * Please refer to PSARC/2009/115 for detail information.
34fb2caebeSRandy Fishel *
35fb2caebeSRandy Fishel * Lock Strategy
36fb2caebeSRandy Fishel * 1) cpu_idle_prop_busy/free are protected by cpu_idle_prop_lock.
37fb2caebeSRandy Fishel * 2) No protection for cpu_idle_cb_state because it's per-CPU data.
38fb2caebeSRandy Fishel * 3) cpu_idle_cb_busy is protected by cpu_idle_cb_lock.
39fb2caebeSRandy Fishel * 4) cpu_idle_cb_array is protected by pause_cpus/start_cpus logic.
40fb2caebeSRandy Fishel * 5) cpu_idle_cb_max/curr are protected by both cpu_idle_cb_lock and
41fb2caebeSRandy Fishel * pause_cpus/start_cpus logic.
42fb2caebeSRandy Fishel * We have optimized the algorithm for hot path on read side access.
43fb2caebeSRandy Fishel * In the current algorithm, it's lock free on read side access.
44fb2caebeSRandy Fishel * On write side, we use pause_cpus() to keep other CPUs in the pause thread,
45fb2caebeSRandy Fishel * which will guarantee that no other threads will access
46fb2caebeSRandy Fishel * cpu_idle_cb_max/curr/array data structure.
47fb2caebeSRandy Fishel */
48fb2caebeSRandy Fishel
49fb2caebeSRandy Fishel #include <sys/types.h>
50fb2caebeSRandy Fishel #include <sys/cmn_err.h>
51fb2caebeSRandy Fishel #include <sys/cpuvar.h>
52fb2caebeSRandy Fishel #include <sys/cpu.h>
53fb2caebeSRandy Fishel #include <sys/kmem.h>
54fb2caebeSRandy Fishel #include <sys/machcpuvar.h>
55fb2caebeSRandy Fishel #include <sys/sdt.h>
56fb2caebeSRandy Fishel #include <sys/sysmacros.h>
57fb2caebeSRandy Fishel #include <sys/synch.h>
58fb2caebeSRandy Fishel #include <sys/systm.h>
59fb2caebeSRandy Fishel #include <sys/sunddi.h>
60fb2caebeSRandy Fishel #if defined(__sparc)
61fb2caebeSRandy Fishel #include <sys/machsystm.h>
62fb2caebeSRandy Fishel #elif defined(__x86)
63fb2caebeSRandy Fishel #include <sys/archsystm.h>
64fb2caebeSRandy Fishel #endif
65fb2caebeSRandy Fishel #include <sys/cpu_event.h>
66fb2caebeSRandy Fishel
67fb2caebeSRandy Fishel /* Define normal state for CPU on different platforms. */
68fb2caebeSRandy Fishel #if defined(__x86)
69fb2caebeSRandy Fishel #define CPU_IDLE_STATE_NORMAL IDLE_STATE_C0
70fb2caebeSRandy Fishel #elif defined(__sparc)
71fb2caebeSRandy Fishel /*
72fb2caebeSRandy Fishel * At the time of this implementation IDLE_STATE_NORMAL is defined
73fb2caebeSRandy Fishel * in mach_startup.c, and not in a header file. So if we find it is
74fb2caebeSRandy Fishel * undefined, then we set it to the value as defined in mach_startup.c
75fb2caebeSRandy Fishel * Should it eventually be defined, we will pick it up.
76fb2caebeSRandy Fishel */
77fb2caebeSRandy Fishel #ifndef IDLE_STATE_NORMAL
78fb2caebeSRandy Fishel #define IDLE_STATE_NORMAL 0
79fb2caebeSRandy Fishel #endif
80fb2caebeSRandy Fishel #define CPU_IDLE_STATE_NORMAL IDLE_STATE_NORMAL
81fb2caebeSRandy Fishel #endif
82fb2caebeSRandy Fishel
83fb2caebeSRandy Fishel /*
84fb2caebeSRandy Fishel * To improve cache efficiency and avoid cache false sharing, CPU idle
85fb2caebeSRandy Fishel * properties are grouped into cache lines as below:
86fb2caebeSRandy Fishel * | CPU0 | CPU1 |.........| CPUn |
87fb2caebeSRandy Fishel * | cache line 0 | cache line 1 |.........| cache line n |
88fb2caebeSRandy Fishel * | v0 | ... | vm | v0 | ... | vm |.........| v0 | ... | vm |
89fb2caebeSRandy Fishel * To access value of property m for CPU n, using following value as index:
90fb2caebeSRandy Fishel * index = seq_id_of_CPUn * CPU_IDLE_VALUE_GROUP_SIZE + m.
91fb2caebeSRandy Fishel */
92fb2caebeSRandy Fishel #define CPU_IDLE_VALUE_GROUP_SIZE \
93fb2caebeSRandy Fishel (CPU_CACHE_COHERENCE_SIZE / sizeof (cpu_idle_prop_value_t))
94fb2caebeSRandy Fishel
95fb2caebeSRandy Fishel /* Get callback context handle for current CPU. */
96fb2caebeSRandy Fishel #define CPU_IDLE_GET_CTX(cp) \
97fb2caebeSRandy Fishel ((cpu_idle_callback_context_t)(intptr_t)((cp)->cpu_seqid))
98fb2caebeSRandy Fishel
99fb2caebeSRandy Fishel /* Get CPU sequential id from ctx. */
100fb2caebeSRandy Fishel #define CPU_IDLE_CTX2CPUID(ctx) ((processorid_t)(intptr_t)(ctx))
101fb2caebeSRandy Fishel
102fb2caebeSRandy Fishel /* Compute index from callback context handle. */
103fb2caebeSRandy Fishel #define CPU_IDLE_CTX2IDX(ctx) \
104fb2caebeSRandy Fishel (((int)(intptr_t)(ctx)) * CPU_IDLE_VALUE_GROUP_SIZE)
105fb2caebeSRandy Fishel
106fb2caebeSRandy Fishel #define CPU_IDLE_HDL2VALP(hdl, idx) \
107fb2caebeSRandy Fishel (&((cpu_idle_prop_impl_t *)(hdl))->value[(idx)])
108fb2caebeSRandy Fishel
109fb2caebeSRandy Fishel /*
110fb2caebeSRandy Fishel * When cpu_idle_cb_array is NULL or full, increase CPU_IDLE_ARRAY_CAPACITY_INC
111fb2caebeSRandy Fishel * entries every time. Here we prefer linear growth instead of exponential.
112fb2caebeSRandy Fishel */
113fb2caebeSRandy Fishel #define CPU_IDLE_ARRAY_CAPACITY_INC 0x10
114fb2caebeSRandy Fishel
115fb2caebeSRandy Fishel typedef struct cpu_idle_prop_impl {
116fb2caebeSRandy Fishel cpu_idle_prop_value_t *value;
117fb2caebeSRandy Fishel struct cpu_idle_prop_impl *next;
118fb2caebeSRandy Fishel char *name;
119fb2caebeSRandy Fishel cpu_idle_prop_update_t update;
120fb2caebeSRandy Fishel void *private;
121fb2caebeSRandy Fishel cpu_idle_prop_type_t type;
122fb2caebeSRandy Fishel uint32_t refcnt;
123fb2caebeSRandy Fishel } cpu_idle_prop_impl_t;
124fb2caebeSRandy Fishel
125fb2caebeSRandy Fishel typedef struct cpu_idle_prop_item {
126fb2caebeSRandy Fishel cpu_idle_prop_type_t type;
127fb2caebeSRandy Fishel char *name;
128fb2caebeSRandy Fishel cpu_idle_prop_update_t update;
129fb2caebeSRandy Fishel void *arg;
130fb2caebeSRandy Fishel cpu_idle_prop_handle_t handle;
131fb2caebeSRandy Fishel } cpu_idle_prop_item_t;
132fb2caebeSRandy Fishel
133fb2caebeSRandy Fishel /* Structure to maintain registered callbacks in list. */
134fb2caebeSRandy Fishel typedef struct cpu_idle_cb_impl {
135fb2caebeSRandy Fishel struct cpu_idle_cb_impl *next;
136fb2caebeSRandy Fishel cpu_idle_callback_t *callback;
137fb2caebeSRandy Fishel void *argument;
138fb2caebeSRandy Fishel int priority;
139fb2caebeSRandy Fishel } cpu_idle_cb_impl_t;
140fb2caebeSRandy Fishel
141fb2caebeSRandy Fishel /*
142fb2caebeSRandy Fishel * Structure to maintain registered callbacks in priority order and also
143fb2caebeSRandy Fishel * optimized for cache efficiency for reading access.
144fb2caebeSRandy Fishel */
145fb2caebeSRandy Fishel typedef struct cpu_idle_cb_item {
146fb2caebeSRandy Fishel cpu_idle_enter_cbfn_t enter;
147fb2caebeSRandy Fishel cpu_idle_exit_cbfn_t exit;
148fb2caebeSRandy Fishel void *arg;
149fb2caebeSRandy Fishel cpu_idle_cb_impl_t *impl;
150fb2caebeSRandy Fishel } cpu_idle_cb_item_t;
151fb2caebeSRandy Fishel
152fb2caebeSRandy Fishel /* Per-CPU state aligned to CPU_CACHE_COHERENCE_SIZE to avoid false sharing. */
153fb2caebeSRandy Fishel typedef union cpu_idle_cb_state {
154fb2caebeSRandy Fishel struct {
155a3114836SGerry Liu /* Index of already invoked callbacks. */
156fb2caebeSRandy Fishel int index;
157a3114836SGerry Liu /* Invoke registered callbacks if true. */
158a3114836SGerry Liu boolean_t enabled;
159a3114836SGerry Liu /* Property values are valid if true. */
160fb2caebeSRandy Fishel boolean_t ready;
161a3114836SGerry Liu /* Pointers to per-CPU properties. */
162fb2caebeSRandy Fishel cpu_idle_prop_value_t *idle_state;
163fb2caebeSRandy Fishel cpu_idle_prop_value_t *enter_ts;
164fb2caebeSRandy Fishel cpu_idle_prop_value_t *exit_ts;
165fb2caebeSRandy Fishel cpu_idle_prop_value_t *last_idle;
166fb2caebeSRandy Fishel cpu_idle_prop_value_t *last_busy;
167fb2caebeSRandy Fishel cpu_idle_prop_value_t *total_idle;
168fb2caebeSRandy Fishel cpu_idle_prop_value_t *total_busy;
169fb2caebeSRandy Fishel cpu_idle_prop_value_t *intr_cnt;
170fb2caebeSRandy Fishel } v;
171fb2caebeSRandy Fishel #ifdef _LP64
172fb2caebeSRandy Fishel char align[2 * CPU_CACHE_COHERENCE_SIZE];
173fb2caebeSRandy Fishel #else
174fb2caebeSRandy Fishel char align[CPU_CACHE_COHERENCE_SIZE];
175fb2caebeSRandy Fishel #endif
176fb2caebeSRandy Fishel } cpu_idle_cb_state_t;
177fb2caebeSRandy Fishel
178fb2caebeSRandy Fishel static kmutex_t cpu_idle_prop_lock;
179fb2caebeSRandy Fishel static cpu_idle_prop_impl_t *cpu_idle_prop_busy = NULL;
180fb2caebeSRandy Fishel static cpu_idle_prop_impl_t *cpu_idle_prop_free = NULL;
181fb2caebeSRandy Fishel
182fb2caebeSRandy Fishel static kmutex_t cpu_idle_cb_lock;
183fb2caebeSRandy Fishel static cpu_idle_cb_impl_t *cpu_idle_cb_busy = NULL;
184fb2caebeSRandy Fishel static cpu_idle_cb_item_t *cpu_idle_cb_array = NULL;
185fb2caebeSRandy Fishel static int cpu_idle_cb_curr = 0;
186fb2caebeSRandy Fishel static int cpu_idle_cb_max = 0;
187fb2caebeSRandy Fishel
188fb2caebeSRandy Fishel static cpu_idle_cb_state_t *cpu_idle_cb_state;
189fb2caebeSRandy Fishel
190a3114836SGerry Liu #ifdef __x86
191a3114836SGerry Liu /*
192a3114836SGerry Liu * cpuset used to intercept CPUs before powering them off.
193a3114836SGerry Liu * The control CPU sets the bit corresponding to the target CPU and waits
194a3114836SGerry Liu * until the bit is cleared.
195a3114836SGerry Liu * The target CPU disables interrupts before clearing corresponding bit and
196a3114836SGerry Liu * then loops for ever.
197a3114836SGerry Liu */
198a3114836SGerry Liu static cpuset_t cpu_idle_intercept_set;
199a3114836SGerry Liu #endif
200a3114836SGerry Liu
201fb2caebeSRandy Fishel static int cpu_idle_prop_update_intr_cnt(void *arg, uint64_t seqnum,
202fb2caebeSRandy Fishel cpu_idle_prop_value_t *valp);
203fb2caebeSRandy Fishel
204fb2caebeSRandy Fishel static cpu_idle_prop_item_t cpu_idle_prop_array[] = {
205fb2caebeSRandy Fishel {
206fb2caebeSRandy Fishel CPU_IDLE_PROP_TYPE_INTPTR, CPU_IDLE_PROP_IDLE_STATE,
207fb2caebeSRandy Fishel NULL, NULL, NULL
208fb2caebeSRandy Fishel },
209fb2caebeSRandy Fishel {
210fb2caebeSRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_ENTER_TIMESTAMP,
211fb2caebeSRandy Fishel NULL, NULL, NULL
212fb2caebeSRandy Fishel },
213fb2caebeSRandy Fishel {
214fb2caebeSRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_EXIT_TIMESTAMP,
215fb2caebeSRandy Fishel NULL, NULL, NULL
216fb2caebeSRandy Fishel },
217fb2caebeSRandy Fishel {
218fb2caebeSRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_LAST_IDLE_TIME,
219fb2caebeSRandy Fishel NULL, NULL, NULL
220fb2caebeSRandy Fishel },
221fb2caebeSRandy Fishel {
222fb2caebeSRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_LAST_BUSY_TIME,
223fb2caebeSRandy Fishel NULL, NULL, NULL
224fb2caebeSRandy Fishel },
225fb2caebeSRandy Fishel {
226fb2caebeSRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_TOTAL_IDLE_TIME,
227fb2caebeSRandy Fishel NULL, NULL, NULL
228fb2caebeSRandy Fishel },
229fb2caebeSRandy Fishel {
230fb2caebeSRandy Fishel CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_TOTAL_BUSY_TIME,
231fb2caebeSRandy Fishel NULL, NULL, NULL
232fb2caebeSRandy Fishel },
233fb2caebeSRandy Fishel {
234fb2caebeSRandy Fishel CPU_IDLE_PROP_TYPE_UINT64, CPU_IDLE_PROP_INTERRUPT_COUNT,
235fb2caebeSRandy Fishel cpu_idle_prop_update_intr_cnt, NULL, NULL
236fb2caebeSRandy Fishel },
237fb2caebeSRandy Fishel };
238fb2caebeSRandy Fishel
239fb2caebeSRandy Fishel #define CPU_IDLE_PROP_IDX_IDLE_STATE 0
240fb2caebeSRandy Fishel #define CPU_IDLE_PROP_IDX_ENTER_TS 1
241fb2caebeSRandy Fishel #define CPU_IDLE_PROP_IDX_EXIT_TS 2
242fb2caebeSRandy Fishel #define CPU_IDLE_PROP_IDX_LAST_IDLE 3
243fb2caebeSRandy Fishel #define CPU_IDLE_PROP_IDX_LAST_BUSY 4
244fb2caebeSRandy Fishel #define CPU_IDLE_PROP_IDX_TOTAL_IDLE 5
245fb2caebeSRandy Fishel #define CPU_IDLE_PROP_IDX_TOTAL_BUSY 6
246fb2caebeSRandy Fishel #define CPU_IDLE_PROP_IDX_INTR_CNT 7
247fb2caebeSRandy Fishel
248fb2caebeSRandy Fishel /*ARGSUSED*/
249fb2caebeSRandy Fishel static void
cpu_idle_dtrace_enter(void * arg,cpu_idle_callback_context_t ctx,cpu_idle_check_wakeup_t check_func,void * check_arg)250fb2caebeSRandy Fishel cpu_idle_dtrace_enter(void *arg, cpu_idle_callback_context_t ctx,
251fb2caebeSRandy Fishel cpu_idle_check_wakeup_t check_func, void *check_arg)
252fb2caebeSRandy Fishel {
253fb2caebeSRandy Fishel int state;
254fb2caebeSRandy Fishel
255fb2caebeSRandy Fishel state = cpu_idle_prop_get_intptr(
256fb2caebeSRandy Fishel cpu_idle_prop_array[CPU_IDLE_PROP_IDX_IDLE_STATE].handle, ctx);
257fb2caebeSRandy Fishel DTRACE_PROBE1(idle__state__transition, uint_t, state);
258fb2caebeSRandy Fishel }
259fb2caebeSRandy Fishel
260fb2caebeSRandy Fishel /*ARGSUSED*/
261fb2caebeSRandy Fishel static void
cpu_idle_dtrace_exit(void * arg,cpu_idle_callback_context_t ctx,int flag)262fb2caebeSRandy Fishel cpu_idle_dtrace_exit(void *arg, cpu_idle_callback_context_t ctx, int flag)
263fb2caebeSRandy Fishel {
264fb2caebeSRandy Fishel DTRACE_PROBE1(idle__state__transition, uint_t, CPU_IDLE_STATE_NORMAL);
265fb2caebeSRandy Fishel }
266fb2caebeSRandy Fishel
267fb2caebeSRandy Fishel static cpu_idle_callback_handle_t cpu_idle_cb_handle_dtrace;
268fb2caebeSRandy Fishel static cpu_idle_callback_t cpu_idle_callback_dtrace = {
269fb2caebeSRandy Fishel CPU_IDLE_CALLBACK_VERS,
270fb2caebeSRandy Fishel cpu_idle_dtrace_enter,
271fb2caebeSRandy Fishel cpu_idle_dtrace_exit,
272fb2caebeSRandy Fishel };
273fb2caebeSRandy Fishel
274fb2caebeSRandy Fishel #if defined(__x86) && !defined(__xpv)
275fb2caebeSRandy Fishel extern void tlb_going_idle(void);
276fb2caebeSRandy Fishel extern void tlb_service(void);
277fb2caebeSRandy Fishel
278fb2caebeSRandy Fishel static cpu_idle_callback_handle_t cpu_idle_cb_handle_tlb;
279fb2caebeSRandy Fishel static cpu_idle_callback_t cpu_idle_callback_tlb = {
280fb2caebeSRandy Fishel CPU_IDLE_CALLBACK_VERS,
281fb2caebeSRandy Fishel (cpu_idle_enter_cbfn_t)tlb_going_idle,
282fb2caebeSRandy Fishel (cpu_idle_exit_cbfn_t)tlb_service,
283fb2caebeSRandy Fishel };
284fb2caebeSRandy Fishel #endif
285fb2caebeSRandy Fishel
286fb2caebeSRandy Fishel void
cpu_event_init(void)287fb2caebeSRandy Fishel cpu_event_init(void)
288fb2caebeSRandy Fishel {
289fb2caebeSRandy Fishel int i, idx;
290fb2caebeSRandy Fishel size_t sz;
291fb2caebeSRandy Fishel intptr_t buf;
292fb2caebeSRandy Fishel cpu_idle_cb_state_t *sp;
293fb2caebeSRandy Fishel cpu_idle_prop_item_t *ip;
294fb2caebeSRandy Fishel
295fb2caebeSRandy Fishel mutex_init(&cpu_idle_cb_lock, NULL, MUTEX_DRIVER, NULL);
296fb2caebeSRandy Fishel mutex_init(&cpu_idle_prop_lock, NULL, MUTEX_DRIVER, NULL);
297fb2caebeSRandy Fishel
298fb2caebeSRandy Fishel /* Create internal properties. */
299fb2caebeSRandy Fishel for (i = 0, ip = cpu_idle_prop_array;
300fb2caebeSRandy Fishel i < sizeof (cpu_idle_prop_array) / sizeof (cpu_idle_prop_array[0]);
301fb2caebeSRandy Fishel i++, ip++) {
302fb2caebeSRandy Fishel (void) cpu_idle_prop_create_property(ip->name, ip->type,
303fb2caebeSRandy Fishel ip->update, ip->arg, &ip->handle);
304fb2caebeSRandy Fishel ASSERT(ip->handle != NULL);
305fb2caebeSRandy Fishel }
306fb2caebeSRandy Fishel
307fb2caebeSRandy Fishel /* Allocate buffer and align to CPU_CACHE_COHERENCE_SIZE. */
308fb2caebeSRandy Fishel sz = sizeof (cpu_idle_cb_state_t) * max_ncpus;
309fb2caebeSRandy Fishel sz += CPU_CACHE_COHERENCE_SIZE;
310fb2caebeSRandy Fishel buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
311fb2caebeSRandy Fishel cpu_idle_cb_state = (cpu_idle_cb_state_t *)P2ROUNDUP(buf,
312fb2caebeSRandy Fishel CPU_CACHE_COHERENCE_SIZE);
313fb2caebeSRandy Fishel
314fb2caebeSRandy Fishel /* Cache frequently used property value pointers. */
315fb2caebeSRandy Fishel for (sp = cpu_idle_cb_state, i = 0; i < max_ncpus; i++, sp++) {
316fb2caebeSRandy Fishel idx = CPU_IDLE_CTX2IDX(i);
317fb2caebeSRandy Fishel #define ___INIT_P(f, i) \
318fb2caebeSRandy Fishel sp->v.f = CPU_IDLE_HDL2VALP(cpu_idle_prop_array[(i)].handle, idx)
319fb2caebeSRandy Fishel ___INIT_P(idle_state, CPU_IDLE_PROP_IDX_IDLE_STATE);
320fb2caebeSRandy Fishel ___INIT_P(enter_ts, CPU_IDLE_PROP_IDX_ENTER_TS);
321fb2caebeSRandy Fishel ___INIT_P(exit_ts, CPU_IDLE_PROP_IDX_EXIT_TS);
322fb2caebeSRandy Fishel ___INIT_P(last_idle, CPU_IDLE_PROP_IDX_LAST_IDLE);
323fb2caebeSRandy Fishel ___INIT_P(last_busy, CPU_IDLE_PROP_IDX_LAST_BUSY);
324fb2caebeSRandy Fishel ___INIT_P(total_idle, CPU_IDLE_PROP_IDX_TOTAL_IDLE);
325fb2caebeSRandy Fishel ___INIT_P(total_busy, CPU_IDLE_PROP_IDX_TOTAL_BUSY);
326fb2caebeSRandy Fishel ___INIT_P(last_idle, CPU_IDLE_PROP_IDX_INTR_CNT);
327fb2caebeSRandy Fishel #undef ___INIT_P
328fb2caebeSRandy Fishel }
329fb2caebeSRandy Fishel
330fb2caebeSRandy Fishel /* Register built-in callbacks. */
331fb2caebeSRandy Fishel if (cpu_idle_register_callback(CPU_IDLE_CB_PRIO_DTRACE,
332fb2caebeSRandy Fishel &cpu_idle_callback_dtrace, NULL, &cpu_idle_cb_handle_dtrace) != 0) {
333fb2caebeSRandy Fishel cmn_err(CE_PANIC,
334fb2caebeSRandy Fishel "cpu_idle: failed to register callback for dtrace.");
335fb2caebeSRandy Fishel }
336fb2caebeSRandy Fishel #if defined(__x86) && !defined(__xpv)
337fb2caebeSRandy Fishel if (cpu_idle_register_callback(CPU_IDLE_CB_PRIO_TLB,
338fb2caebeSRandy Fishel &cpu_idle_callback_tlb, NULL, &cpu_idle_cb_handle_tlb) != 0) {
339fb2caebeSRandy Fishel cmn_err(CE_PANIC,
340fb2caebeSRandy Fishel "cpu_idle: failed to register callback for tlb_flush.");
341fb2caebeSRandy Fishel }
342fb2caebeSRandy Fishel #endif
343fb2caebeSRandy Fishel }
344fb2caebeSRandy Fishel
345a3114836SGerry Liu /*
346a3114836SGerry Liu * This function is called to initialize per CPU state when starting CPUs.
347a3114836SGerry Liu */
348fb2caebeSRandy Fishel void
cpu_event_init_cpu(cpu_t * cp)349fb2caebeSRandy Fishel cpu_event_init_cpu(cpu_t *cp)
350fb2caebeSRandy Fishel {
351fb2caebeSRandy Fishel ASSERT(cp->cpu_seqid < max_ncpus);
352a3114836SGerry Liu cpu_idle_cb_state[cp->cpu_seqid].v.index = 0;
353fb2caebeSRandy Fishel cpu_idle_cb_state[cp->cpu_seqid].v.ready = B_FALSE;
354a3114836SGerry Liu cpu_idle_cb_state[cp->cpu_seqid].v.enabled = B_TRUE;
355fb2caebeSRandy Fishel }
356fb2caebeSRandy Fishel
357a3114836SGerry Liu /*
358a3114836SGerry Liu * This function is called to clean up per CPU state when stopping CPUs.
359a3114836SGerry Liu */
360fb2caebeSRandy Fishel void
cpu_event_fini_cpu(cpu_t * cp)361fb2caebeSRandy Fishel cpu_event_fini_cpu(cpu_t *cp)
362fb2caebeSRandy Fishel {
363fb2caebeSRandy Fishel ASSERT(cp->cpu_seqid < max_ncpus);
364a3114836SGerry Liu cpu_idle_cb_state[cp->cpu_seqid].v.enabled = B_FALSE;
365fb2caebeSRandy Fishel cpu_idle_cb_state[cp->cpu_seqid].v.ready = B_FALSE;
366fb2caebeSRandy Fishel }
367fb2caebeSRandy Fishel
368fb2caebeSRandy Fishel static void
cpu_idle_insert_callback(cpu_idle_cb_impl_t * cip)369fb2caebeSRandy Fishel cpu_idle_insert_callback(cpu_idle_cb_impl_t *cip)
370fb2caebeSRandy Fishel {
371fb2caebeSRandy Fishel int unlock = 0, unpause = 0;
372fb2caebeSRandy Fishel int i, cnt_new = 0, cnt_old = 0;
373fb2caebeSRandy Fishel char *buf_new = NULL, *buf_old = NULL;
374fb2caebeSRandy Fishel
375fb2caebeSRandy Fishel ASSERT(MUTEX_HELD(&cpu_idle_cb_lock));
376fb2caebeSRandy Fishel
377fb2caebeSRandy Fishel /*
378fb2caebeSRandy Fishel * Expand array if it's full.
379fb2caebeSRandy Fishel * Memory must be allocated out of pause/start_cpus() scope because
380fb2caebeSRandy Fishel * kmem_zalloc() can't be called with KM_SLEEP flag within that scope.
381fb2caebeSRandy Fishel */
382fb2caebeSRandy Fishel if (cpu_idle_cb_curr == cpu_idle_cb_max) {
383fb2caebeSRandy Fishel cnt_new = cpu_idle_cb_max + CPU_IDLE_ARRAY_CAPACITY_INC;
384fb2caebeSRandy Fishel buf_new = (char *)kmem_zalloc(cnt_new *
385fb2caebeSRandy Fishel sizeof (cpu_idle_cb_item_t), KM_SLEEP);
386fb2caebeSRandy Fishel }
387fb2caebeSRandy Fishel
388fb2caebeSRandy Fishel /* Try to acquire cpu_lock if not held yet. */
389fb2caebeSRandy Fishel if (!MUTEX_HELD(&cpu_lock)) {
390fb2caebeSRandy Fishel mutex_enter(&cpu_lock);
391fb2caebeSRandy Fishel unlock = 1;
392fb2caebeSRandy Fishel }
393fb2caebeSRandy Fishel /*
394fb2caebeSRandy Fishel * Pause all other CPUs (and let them run pause thread).
395fb2caebeSRandy Fishel * It's guaranteed that no other threads will access cpu_idle_cb_array
396fb2caebeSRandy Fishel * after pause_cpus().
397fb2caebeSRandy Fishel */
398fb2caebeSRandy Fishel if (!cpus_paused()) {
399*0ed5c46eSJosef 'Jeff' Sipek pause_cpus(NULL, NULL);
400fb2caebeSRandy Fishel unpause = 1;
401fb2caebeSRandy Fishel }
402fb2caebeSRandy Fishel
403fb2caebeSRandy Fishel /* Copy content to new buffer if needed. */
404fb2caebeSRandy Fishel if (buf_new != NULL) {
405fb2caebeSRandy Fishel buf_old = (char *)cpu_idle_cb_array;
406fb2caebeSRandy Fishel cnt_old = cpu_idle_cb_max;
407fb2caebeSRandy Fishel if (buf_old != NULL) {
408fb2caebeSRandy Fishel ASSERT(cnt_old != 0);
409fb2caebeSRandy Fishel bcopy(cpu_idle_cb_array, buf_new,
410fb2caebeSRandy Fishel sizeof (cpu_idle_cb_item_t) * cnt_old);
411fb2caebeSRandy Fishel }
412fb2caebeSRandy Fishel cpu_idle_cb_array = (cpu_idle_cb_item_t *)buf_new;
413fb2caebeSRandy Fishel cpu_idle_cb_max = cnt_new;
414fb2caebeSRandy Fishel }
415fb2caebeSRandy Fishel
416fb2caebeSRandy Fishel /* Insert into array according to priority. */
417fb2caebeSRandy Fishel ASSERT(cpu_idle_cb_curr < cpu_idle_cb_max);
418fb2caebeSRandy Fishel for (i = cpu_idle_cb_curr; i > 0; i--) {
419fb2caebeSRandy Fishel if (cpu_idle_cb_array[i - 1].impl->priority >= cip->priority) {
420fb2caebeSRandy Fishel break;
421fb2caebeSRandy Fishel }
422fb2caebeSRandy Fishel cpu_idle_cb_array[i] = cpu_idle_cb_array[i - 1];
423fb2caebeSRandy Fishel }
424fb2caebeSRandy Fishel cpu_idle_cb_array[i].arg = cip->argument;
425fb2caebeSRandy Fishel cpu_idle_cb_array[i].enter = cip->callback->idle_enter;
426fb2caebeSRandy Fishel cpu_idle_cb_array[i].exit = cip->callback->idle_exit;
427fb2caebeSRandy Fishel cpu_idle_cb_array[i].impl = cip;
428fb2caebeSRandy Fishel cpu_idle_cb_curr++;
429fb2caebeSRandy Fishel
430fb2caebeSRandy Fishel /* Resume other CPUs from paused state if needed. */
431fb2caebeSRandy Fishel if (unpause) {
432fb2caebeSRandy Fishel start_cpus();
433fb2caebeSRandy Fishel }
434fb2caebeSRandy Fishel if (unlock) {
435fb2caebeSRandy Fishel mutex_exit(&cpu_lock);
436fb2caebeSRandy Fishel }
437fb2caebeSRandy Fishel
438fb2caebeSRandy Fishel /* Free old resource if needed. */
439fb2caebeSRandy Fishel if (buf_old != NULL) {
440fb2caebeSRandy Fishel ASSERT(cnt_old != 0);
441fb2caebeSRandy Fishel kmem_free(buf_old, cnt_old * sizeof (cpu_idle_cb_item_t));
442fb2caebeSRandy Fishel }
443fb2caebeSRandy Fishel }
444fb2caebeSRandy Fishel
445fb2caebeSRandy Fishel static void
cpu_idle_remove_callback(cpu_idle_cb_impl_t * cip)446fb2caebeSRandy Fishel cpu_idle_remove_callback(cpu_idle_cb_impl_t *cip)
447fb2caebeSRandy Fishel {
448fb2caebeSRandy Fishel int i, found = 0;
449fb2caebeSRandy Fishel int unlock = 0, unpause = 0;
450fb2caebeSRandy Fishel cpu_idle_cb_state_t *sp;
451fb2caebeSRandy Fishel
452fb2caebeSRandy Fishel ASSERT(MUTEX_HELD(&cpu_idle_cb_lock));
453fb2caebeSRandy Fishel
454fb2caebeSRandy Fishel /* Try to acquire cpu_lock if not held yet. */
455fb2caebeSRandy Fishel if (!MUTEX_HELD(&cpu_lock)) {
456fb2caebeSRandy Fishel mutex_enter(&cpu_lock);
457fb2caebeSRandy Fishel unlock = 1;
458fb2caebeSRandy Fishel }
459fb2caebeSRandy Fishel /*
460fb2caebeSRandy Fishel * Pause all other CPUs.
461fb2caebeSRandy Fishel * It's guaranteed that no other threads will access cpu_idle_cb_array
462fb2caebeSRandy Fishel * after pause_cpus().
463fb2caebeSRandy Fishel */
464fb2caebeSRandy Fishel if (!cpus_paused()) {
465*0ed5c46eSJosef 'Jeff' Sipek pause_cpus(NULL, NULL);
466fb2caebeSRandy Fishel unpause = 1;
467fb2caebeSRandy Fishel }
468fb2caebeSRandy Fishel
469fb2caebeSRandy Fishel /* Remove cip from array. */
470fb2caebeSRandy Fishel for (i = 0; i < cpu_idle_cb_curr; i++) {
471fb2caebeSRandy Fishel if (found == 0) {
472fb2caebeSRandy Fishel if (cpu_idle_cb_array[i].impl == cip) {
473fb2caebeSRandy Fishel found = 1;
474fb2caebeSRandy Fishel }
475fb2caebeSRandy Fishel } else {
476fb2caebeSRandy Fishel cpu_idle_cb_array[i - 1] = cpu_idle_cb_array[i];
477fb2caebeSRandy Fishel }
478fb2caebeSRandy Fishel }
479fb2caebeSRandy Fishel ASSERT(found != 0);
480fb2caebeSRandy Fishel cpu_idle_cb_curr--;
481fb2caebeSRandy Fishel
482fb2caebeSRandy Fishel /*
483fb2caebeSRandy Fishel * Reset property ready flag for all CPUs if no registered callback
484fb2caebeSRandy Fishel * left because cpu_idle_enter/exit will stop updating property if
485fb2caebeSRandy Fishel * there's no callback registered.
486fb2caebeSRandy Fishel */
487fb2caebeSRandy Fishel if (cpu_idle_cb_curr == 0) {
488fb2caebeSRandy Fishel for (sp = cpu_idle_cb_state, i = 0; i < max_ncpus; i++, sp++) {
489fb2caebeSRandy Fishel sp->v.ready = B_FALSE;
490fb2caebeSRandy Fishel }
491fb2caebeSRandy Fishel }
492fb2caebeSRandy Fishel
493fb2caebeSRandy Fishel /* Resume other CPUs from paused state if needed. */
494fb2caebeSRandy Fishel if (unpause) {
495fb2caebeSRandy Fishel start_cpus();
496fb2caebeSRandy Fishel }
497fb2caebeSRandy Fishel if (unlock) {
498fb2caebeSRandy Fishel mutex_exit(&cpu_lock);
499fb2caebeSRandy Fishel }
500fb2caebeSRandy Fishel }
501fb2caebeSRandy Fishel
502fb2caebeSRandy Fishel int
cpu_idle_register_callback(uint_t prio,cpu_idle_callback_t * cbp,void * arg,cpu_idle_callback_handle_t * hdlp)503fb2caebeSRandy Fishel cpu_idle_register_callback(uint_t prio, cpu_idle_callback_t *cbp,
504fb2caebeSRandy Fishel void *arg, cpu_idle_callback_handle_t *hdlp)
505fb2caebeSRandy Fishel {
506fb2caebeSRandy Fishel cpu_idle_cb_state_t *sp;
507fb2caebeSRandy Fishel cpu_idle_cb_impl_t *cip = NULL;
508fb2caebeSRandy Fishel
509fb2caebeSRandy Fishel /* First validate parameters. */
510fb2caebeSRandy Fishel ASSERT(!CPU_ON_INTR(CPU));
511fb2caebeSRandy Fishel ASSERT(CPU->cpu_seqid < max_ncpus);
512fb2caebeSRandy Fishel sp = &cpu_idle_cb_state[CPU->cpu_seqid];
513fb2caebeSRandy Fishel if (sp->v.index != 0) {
514fb2caebeSRandy Fishel cmn_err(CE_NOTE,
515fb2caebeSRandy Fishel "!cpu_event: register_callback called from callback.");
516fb2caebeSRandy Fishel return (EBUSY);
517fb2caebeSRandy Fishel } else if (cbp == NULL || hdlp == NULL) {
518fb2caebeSRandy Fishel cmn_err(CE_NOTE,
519fb2caebeSRandy Fishel "!cpu_event: NULL parameters in register_callback.");
520fb2caebeSRandy Fishel return (EINVAL);
521fb2caebeSRandy Fishel } else if (prio < CPU_IDLE_CB_PRIO_LOW_BASE ||
522fb2caebeSRandy Fishel prio >= CPU_IDLE_CB_PRIO_RESV_BASE) {
523fb2caebeSRandy Fishel cmn_err(CE_NOTE,
524fb2caebeSRandy Fishel "!cpu_event: priority 0x%x out of range.", prio);
525fb2caebeSRandy Fishel return (EINVAL);
526fb2caebeSRandy Fishel } else if (cbp->version != CPU_IDLE_CALLBACK_VERS) {
527fb2caebeSRandy Fishel cmn_err(CE_NOTE,
528fb2caebeSRandy Fishel "!cpu_event: callback version %d is not supported.",
529fb2caebeSRandy Fishel cbp->version);
530fb2caebeSRandy Fishel return (EINVAL);
531fb2caebeSRandy Fishel }
532fb2caebeSRandy Fishel
533fb2caebeSRandy Fishel mutex_enter(&cpu_idle_cb_lock);
534fb2caebeSRandy Fishel /* Check whether callback with priority exists if not dynamic. */
535fb2caebeSRandy Fishel if (prio != CPU_IDLE_CB_PRIO_DYNAMIC) {
536fb2caebeSRandy Fishel for (cip = cpu_idle_cb_busy; cip != NULL;
537fb2caebeSRandy Fishel cip = cip->next) {
538fb2caebeSRandy Fishel if (cip->priority == prio) {
539fb2caebeSRandy Fishel mutex_exit(&cpu_idle_cb_lock);
540fb2caebeSRandy Fishel cmn_err(CE_NOTE, "!cpu_event: callback with "
541fb2caebeSRandy Fishel "priority 0x%x already exists.", prio);
542fb2caebeSRandy Fishel return (EEXIST);
543fb2caebeSRandy Fishel }
544fb2caebeSRandy Fishel }
545fb2caebeSRandy Fishel }
546fb2caebeSRandy Fishel
547fb2caebeSRandy Fishel cip = kmem_zalloc(sizeof (*cip), KM_SLEEP);
548fb2caebeSRandy Fishel cip->callback = cbp;
549fb2caebeSRandy Fishel cip->argument = arg;
550fb2caebeSRandy Fishel cip->priority = prio;
551fb2caebeSRandy Fishel cip->next = cpu_idle_cb_busy;
552fb2caebeSRandy Fishel cpu_idle_cb_busy = cip;
553fb2caebeSRandy Fishel cpu_idle_insert_callback(cip);
554fb2caebeSRandy Fishel mutex_exit(&cpu_idle_cb_lock);
555fb2caebeSRandy Fishel
556fb2caebeSRandy Fishel *hdlp = (cpu_idle_callback_handle_t)cip;
557fb2caebeSRandy Fishel
558fb2caebeSRandy Fishel return (0);
559fb2caebeSRandy Fishel }
560fb2caebeSRandy Fishel
561fb2caebeSRandy Fishel int
cpu_idle_unregister_callback(cpu_idle_callback_handle_t hdl)562fb2caebeSRandy Fishel cpu_idle_unregister_callback(cpu_idle_callback_handle_t hdl)
563fb2caebeSRandy Fishel {
564fb2caebeSRandy Fishel int rc = ENODEV;
565fb2caebeSRandy Fishel cpu_idle_cb_state_t *sp;
566fb2caebeSRandy Fishel cpu_idle_cb_impl_t *ip, **ipp;
567fb2caebeSRandy Fishel
568fb2caebeSRandy Fishel ASSERT(!CPU_ON_INTR(CPU));
569fb2caebeSRandy Fishel ASSERT(CPU->cpu_seqid < max_ncpus);
570fb2caebeSRandy Fishel sp = &cpu_idle_cb_state[CPU->cpu_seqid];
571fb2caebeSRandy Fishel if (sp->v.index != 0) {
572fb2caebeSRandy Fishel cmn_err(CE_NOTE,
573fb2caebeSRandy Fishel "!cpu_event: unregister_callback called from callback.");
574fb2caebeSRandy Fishel return (EBUSY);
575fb2caebeSRandy Fishel } else if (hdl == NULL) {
576fb2caebeSRandy Fishel cmn_err(CE_NOTE,
577fb2caebeSRandy Fishel "!cpu_event: hdl is NULL in unregister_callback.");
578fb2caebeSRandy Fishel return (EINVAL);
579fb2caebeSRandy Fishel }
580fb2caebeSRandy Fishel
581fb2caebeSRandy Fishel ip = (cpu_idle_cb_impl_t *)hdl;
582fb2caebeSRandy Fishel mutex_enter(&cpu_idle_cb_lock);
583fb2caebeSRandy Fishel for (ipp = &cpu_idle_cb_busy; *ipp != NULL; ipp = &(*ipp)->next) {
584fb2caebeSRandy Fishel if (*ipp == ip) {
585fb2caebeSRandy Fishel *ipp = ip->next;
586fb2caebeSRandy Fishel cpu_idle_remove_callback(ip);
587fb2caebeSRandy Fishel rc = 0;
588fb2caebeSRandy Fishel break;
589fb2caebeSRandy Fishel }
590fb2caebeSRandy Fishel }
591fb2caebeSRandy Fishel mutex_exit(&cpu_idle_cb_lock);
592fb2caebeSRandy Fishel
593fb2caebeSRandy Fishel if (rc == 0) {
594fb2caebeSRandy Fishel kmem_free(ip, sizeof (*ip));
595fb2caebeSRandy Fishel } else {
596fb2caebeSRandy Fishel cmn_err(CE_NOTE,
597fb2caebeSRandy Fishel "!cpu_event: callback handle %p not found.", (void *)hdl);
598fb2caebeSRandy Fishel }
599fb2caebeSRandy Fishel
600fb2caebeSRandy Fishel return (rc);
601fb2caebeSRandy Fishel }
602fb2caebeSRandy Fishel
603fb2caebeSRandy Fishel static int
cpu_idle_enter_state(cpu_idle_cb_state_t * sp,intptr_t state)604fb2caebeSRandy Fishel cpu_idle_enter_state(cpu_idle_cb_state_t *sp, intptr_t state)
605fb2caebeSRandy Fishel {
606fb2caebeSRandy Fishel sp->v.idle_state->cipv_intptr = state;
607fb2caebeSRandy Fishel sp->v.enter_ts->cipv_hrtime = gethrtime_unscaled();
608fb2caebeSRandy Fishel sp->v.last_busy->cipv_hrtime = sp->v.enter_ts->cipv_hrtime -
609fb2caebeSRandy Fishel sp->v.exit_ts->cipv_hrtime;
610fb2caebeSRandy Fishel sp->v.total_busy->cipv_hrtime += sp->v.last_busy->cipv_hrtime;
611fb2caebeSRandy Fishel if (sp->v.ready == B_FALSE) {
612fb2caebeSRandy Fishel sp->v.ready = B_TRUE;
613fb2caebeSRandy Fishel return (0);
614fb2caebeSRandy Fishel }
615fb2caebeSRandy Fishel
616fb2caebeSRandy Fishel return (1);
617fb2caebeSRandy Fishel }
618fb2caebeSRandy Fishel
619fb2caebeSRandy Fishel static void
cpu_idle_exit_state(cpu_idle_cb_state_t * sp)620fb2caebeSRandy Fishel cpu_idle_exit_state(cpu_idle_cb_state_t *sp)
621fb2caebeSRandy Fishel {
622fb2caebeSRandy Fishel sp->v.idle_state->cipv_intptr = CPU_IDLE_STATE_NORMAL;
623fb2caebeSRandy Fishel sp->v.exit_ts->cipv_hrtime = gethrtime_unscaled();
624fb2caebeSRandy Fishel sp->v.last_idle->cipv_hrtime = sp->v.exit_ts->cipv_hrtime -
625fb2caebeSRandy Fishel sp->v.enter_ts->cipv_hrtime;
626fb2caebeSRandy Fishel sp->v.total_idle->cipv_hrtime += sp->v.last_idle->cipv_hrtime;
627fb2caebeSRandy Fishel }
628fb2caebeSRandy Fishel
629fb2caebeSRandy Fishel /*ARGSUSED*/
630fb2caebeSRandy Fishel int
cpu_idle_enter(int state,int flag,cpu_idle_check_wakeup_t check_func,void * check_arg)631fb2caebeSRandy Fishel cpu_idle_enter(int state, int flag,
632fb2caebeSRandy Fishel cpu_idle_check_wakeup_t check_func, void *check_arg)
633fb2caebeSRandy Fishel {
634fb2caebeSRandy Fishel int i;
635fb2caebeSRandy Fishel cpu_idle_cb_item_t *cip;
636fb2caebeSRandy Fishel cpu_idle_cb_state_t *sp;
637fb2caebeSRandy Fishel cpu_idle_callback_context_t ctx;
638fb2caebeSRandy Fishel #if defined(__x86)
639fb2caebeSRandy Fishel ulong_t iflags;
640fb2caebeSRandy Fishel #endif
641fb2caebeSRandy Fishel
642fb2caebeSRandy Fishel ctx = CPU_IDLE_GET_CTX(CPU);
643fb2caebeSRandy Fishel ASSERT(CPU->cpu_seqid < max_ncpus);
644fb2caebeSRandy Fishel sp = &cpu_idle_cb_state[CPU->cpu_seqid];
645fb2caebeSRandy Fishel ASSERT(sp->v.index == 0);
646a3114836SGerry Liu if (sp->v.enabled == B_FALSE) {
647a3114836SGerry Liu #if defined(__x86)
648a3114836SGerry Liu /* Intercept CPU at a safe point before powering off it. */
649a3114836SGerry Liu if (CPU_IN_SET(cpu_idle_intercept_set, CPU->cpu_id)) {
650a3114836SGerry Liu iflags = intr_clear();
651a3114836SGerry Liu CPUSET_ATOMIC_DEL(cpu_idle_intercept_set, CPU->cpu_id);
652a3114836SGerry Liu /*CONSTCOND*/
653a3114836SGerry Liu while (1) {
654a3114836SGerry Liu SMT_PAUSE();
655a3114836SGerry Liu }
656a3114836SGerry Liu }
657a3114836SGerry Liu #endif
658a3114836SGerry Liu
659a3114836SGerry Liu return (0);
660a3114836SGerry Liu }
661fb2caebeSRandy Fishel
662fb2caebeSRandy Fishel /*
663fb2caebeSRandy Fishel * On x86, cpu_idle_enter can be called from idle thread with either
664fb2caebeSRandy Fishel * interrupts enabled or disabled, so we need to make sure interrupts
665fb2caebeSRandy Fishel * are disabled here.
666fb2caebeSRandy Fishel * On SPARC, cpu_idle_enter will be called from idle thread with
667fb2caebeSRandy Fishel * interrupt disabled, so no special handling necessary.
668fb2caebeSRandy Fishel */
669fb2caebeSRandy Fishel #if defined(__x86)
670fb2caebeSRandy Fishel iflags = intr_clear();
671fb2caebeSRandy Fishel #endif
672fb2caebeSRandy Fishel
673fb2caebeSRandy Fishel /* Skip calling callback if state is not ready for current CPU. */
674fb2caebeSRandy Fishel if (cpu_idle_enter_state(sp, state) == 0) {
675fb2caebeSRandy Fishel #if defined(__x86)
676fb2caebeSRandy Fishel intr_restore(iflags);
677fb2caebeSRandy Fishel #endif
678fb2caebeSRandy Fishel return (0);
679fb2caebeSRandy Fishel }
680fb2caebeSRandy Fishel
681fb2caebeSRandy Fishel for (i = 0, cip = cpu_idle_cb_array; i < cpu_idle_cb_curr; i++, cip++) {
682fb2caebeSRandy Fishel /*
683fb2caebeSRandy Fishel * Increase index so corresponding idle_exit callback
684fb2caebeSRandy Fishel * will be invoked should interrupt happen during
685fb2caebeSRandy Fishel * idle_enter callback.
686fb2caebeSRandy Fishel */
687fb2caebeSRandy Fishel sp->v.index++;
688fb2caebeSRandy Fishel
689fb2caebeSRandy Fishel /* Call idle_enter callback function if it's not NULL. */
690fb2caebeSRandy Fishel if (cip->enter != NULL) {
691fb2caebeSRandy Fishel cip->enter(cip->arg, ctx, check_func, check_arg);
692fb2caebeSRandy Fishel
693fb2caebeSRandy Fishel /*
694fb2caebeSRandy Fishel * cpu_idle_enter runs with interrupts
695fb2caebeSRandy Fishel * disabled, so the idle_enter callbacks will
696fb2caebeSRandy Fishel * also be called with interrupts disabled.
697fb2caebeSRandy Fishel * It is permissible for the callbacks to
698fb2caebeSRandy Fishel * enable the interrupts, if they can also
699fb2caebeSRandy Fishel * handle the condition if the interrupt
700fb2caebeSRandy Fishel * occurs.
701fb2caebeSRandy Fishel *
702fb2caebeSRandy Fishel * However, if an interrupt occurs and we
703fb2caebeSRandy Fishel * return here without dealing with it, we
704fb2caebeSRandy Fishel * return to the cpu_idle_enter() caller
705fb2caebeSRandy Fishel * with an EBUSY, and the caller will not
706fb2caebeSRandy Fishel * enter the idle state.
707fb2caebeSRandy Fishel *
708fb2caebeSRandy Fishel * We detect the interrupt, by checking the
709fb2caebeSRandy Fishel * index value of the state pointer. If it
710fb2caebeSRandy Fishel * is not the index we incremented above,
711fb2caebeSRandy Fishel * then it was cleared while processing
712fb2caebeSRandy Fishel * the interrupt.
713fb2caebeSRandy Fishel *
714fb2caebeSRandy Fishel * Also note, that at this point of the code
715fb2caebeSRandy Fishel * the normal index value will be one greater
716fb2caebeSRandy Fishel * than the variable 'i' in the loop, as it
717fb2caebeSRandy Fishel * hasn't yet been incremented.
718fb2caebeSRandy Fishel */
719fb2caebeSRandy Fishel if (sp->v.index != i + 1) {
720fb2caebeSRandy Fishel #if defined(__x86)
721fb2caebeSRandy Fishel intr_restore(iflags);
722fb2caebeSRandy Fishel #endif
723fb2caebeSRandy Fishel return (EBUSY);
724fb2caebeSRandy Fishel }
725fb2caebeSRandy Fishel }
726fb2caebeSRandy Fishel }
727fb2caebeSRandy Fishel #if defined(__x86)
728fb2caebeSRandy Fishel intr_restore(iflags);
729fb2caebeSRandy Fishel #endif
730fb2caebeSRandy Fishel
731fb2caebeSRandy Fishel return (0);
732fb2caebeSRandy Fishel }
733fb2caebeSRandy Fishel
734fb2caebeSRandy Fishel void
cpu_idle_exit(int flag)735fb2caebeSRandy Fishel cpu_idle_exit(int flag)
736fb2caebeSRandy Fishel {
737fb2caebeSRandy Fishel int i;
738fb2caebeSRandy Fishel cpu_idle_cb_item_t *cip;
739fb2caebeSRandy Fishel cpu_idle_cb_state_t *sp;
740fb2caebeSRandy Fishel cpu_idle_callback_context_t ctx;
741fb2caebeSRandy Fishel #if defined(__x86)
742fb2caebeSRandy Fishel ulong_t iflags;
743fb2caebeSRandy Fishel #endif
744fb2caebeSRandy Fishel
745fb2caebeSRandy Fishel ASSERT(CPU->cpu_seqid < max_ncpus);
746fb2caebeSRandy Fishel sp = &cpu_idle_cb_state[CPU->cpu_seqid];
747fb2caebeSRandy Fishel
748fb2caebeSRandy Fishel #if defined(__sparc)
749fb2caebeSRandy Fishel /*
750fb2caebeSRandy Fishel * On SPARC, cpu_idle_exit will only be called from idle thread
751fb2caebeSRandy Fishel * with interrupt disabled.
752fb2caebeSRandy Fishel */
753fb2caebeSRandy Fishel
754fb2caebeSRandy Fishel if (sp->v.index != 0) {
755fb2caebeSRandy Fishel ctx = CPU_IDLE_GET_CTX(CPU);
756fb2caebeSRandy Fishel cpu_idle_exit_state(sp);
757fb2caebeSRandy Fishel for (i = sp->v.index - 1; i >= 0; i--) {
758fb2caebeSRandy Fishel cip = &cpu_idle_cb_array[i];
759fb2caebeSRandy Fishel if (cip->exit != NULL) {
760fb2caebeSRandy Fishel cip->exit(cip->arg, ctx, flag);
761fb2caebeSRandy Fishel }
762fb2caebeSRandy Fishel }
763fb2caebeSRandy Fishel sp->v.index = 0;
764fb2caebeSRandy Fishel }
765fb2caebeSRandy Fishel #elif defined(__x86)
766fb2caebeSRandy Fishel /*
767fb2caebeSRandy Fishel * On x86, cpu_idle_exit will be called from idle thread or interrupt
768fb2caebeSRandy Fishel * handler. When called from interrupt handler, interrupts will be
769fb2caebeSRandy Fishel * disabled. When called from idle thread, interrupts may be disabled
770fb2caebeSRandy Fishel * or enabled.
771fb2caebeSRandy Fishel */
772fb2caebeSRandy Fishel
773fb2caebeSRandy Fishel /* Called from interrupt, interrupts are already disabled. */
774fb2caebeSRandy Fishel if (flag & CPU_IDLE_CB_FLAG_INTR) {
775fb2caebeSRandy Fishel /*
776fb2caebeSRandy Fishel * return if cpu_idle_exit already called or
777fb2caebeSRandy Fishel * there is no registered callback.
778fb2caebeSRandy Fishel */
779fb2caebeSRandy Fishel if (sp->v.index == 0) {
780fb2caebeSRandy Fishel return;
781fb2caebeSRandy Fishel }
782fb2caebeSRandy Fishel ctx = CPU_IDLE_GET_CTX(CPU);
783fb2caebeSRandy Fishel cpu_idle_exit_state(sp);
784fb2caebeSRandy Fishel for (i = sp->v.index - 1; i >= 0; i--) {
785fb2caebeSRandy Fishel cip = &cpu_idle_cb_array[i];
786fb2caebeSRandy Fishel if (cip->exit != NULL) {
787fb2caebeSRandy Fishel cip->exit(cip->arg, ctx, flag);
788fb2caebeSRandy Fishel }
789fb2caebeSRandy Fishel }
790fb2caebeSRandy Fishel sp->v.index = 0;
791fb2caebeSRandy Fishel
792fb2caebeSRandy Fishel /* Called from idle thread, need to disable interrupt. */
793fb2caebeSRandy Fishel } else {
794fb2caebeSRandy Fishel iflags = intr_clear();
795fb2caebeSRandy Fishel if (sp->v.index != 0) {
796fb2caebeSRandy Fishel ctx = CPU_IDLE_GET_CTX(CPU);
797fb2caebeSRandy Fishel cpu_idle_exit_state(sp);
798fb2caebeSRandy Fishel for (i = sp->v.index - 1; i >= 0; i--) {
799fb2caebeSRandy Fishel cip = &cpu_idle_cb_array[i];
800fb2caebeSRandy Fishel if (cip->exit != NULL) {
801fb2caebeSRandy Fishel cip->exit(cip->arg, ctx, flag);
802fb2caebeSRandy Fishel }
803fb2caebeSRandy Fishel }
804fb2caebeSRandy Fishel sp->v.index = 0;
805fb2caebeSRandy Fishel }
806fb2caebeSRandy Fishel intr_restore(iflags);
807fb2caebeSRandy Fishel }
808fb2caebeSRandy Fishel #endif
809fb2caebeSRandy Fishel }
810fb2caebeSRandy Fishel
811fb2caebeSRandy Fishel cpu_idle_callback_context_t
cpu_idle_get_context(void)812fb2caebeSRandy Fishel cpu_idle_get_context(void)
813fb2caebeSRandy Fishel {
814fb2caebeSRandy Fishel return (CPU_IDLE_GET_CTX(CPU));
815fb2caebeSRandy Fishel }
816fb2caebeSRandy Fishel
817fb2caebeSRandy Fishel /*
818fb2caebeSRandy Fishel * Allocate property structure in group of CPU_IDLE_VALUE_GROUP_SIZE to improve
819fb2caebeSRandy Fishel * cache efficiency. To simplify implementation, allocated memory for property
820fb2caebeSRandy Fishel * structure won't be freed.
821fb2caebeSRandy Fishel */
822fb2caebeSRandy Fishel static void
cpu_idle_prop_allocate_impl(void)823fb2caebeSRandy Fishel cpu_idle_prop_allocate_impl(void)
824fb2caebeSRandy Fishel {
825fb2caebeSRandy Fishel int i;
826fb2caebeSRandy Fishel size_t sz;
827fb2caebeSRandy Fishel intptr_t buf;
828fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop;
829fb2caebeSRandy Fishel cpu_idle_prop_value_t *valp;
830fb2caebeSRandy Fishel
831fb2caebeSRandy Fishel ASSERT(!CPU_ON_INTR(CPU));
832fb2caebeSRandy Fishel prop = kmem_zalloc(sizeof (*prop) * CPU_IDLE_VALUE_GROUP_SIZE,
833fb2caebeSRandy Fishel KM_SLEEP);
834fb2caebeSRandy Fishel sz = sizeof (*valp) * CPU_IDLE_VALUE_GROUP_SIZE * max_ncpus;
835fb2caebeSRandy Fishel sz += CPU_CACHE_COHERENCE_SIZE;
836fb2caebeSRandy Fishel buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
837fb2caebeSRandy Fishel valp = (cpu_idle_prop_value_t *)P2ROUNDUP(buf,
838fb2caebeSRandy Fishel CPU_CACHE_COHERENCE_SIZE);
839fb2caebeSRandy Fishel
840fb2caebeSRandy Fishel for (i = 0; i < CPU_IDLE_VALUE_GROUP_SIZE; i++, prop++, valp++) {
841fb2caebeSRandy Fishel prop->value = valp;
842fb2caebeSRandy Fishel prop->next = cpu_idle_prop_free;
843fb2caebeSRandy Fishel cpu_idle_prop_free = prop;
844fb2caebeSRandy Fishel }
845fb2caebeSRandy Fishel }
846fb2caebeSRandy Fishel
847fb2caebeSRandy Fishel int
cpu_idle_prop_create_property(const char * name,cpu_idle_prop_type_t type,cpu_idle_prop_update_t update,void * arg,cpu_idle_prop_handle_t * hdlp)848fb2caebeSRandy Fishel cpu_idle_prop_create_property(const char *name, cpu_idle_prop_type_t type,
849fb2caebeSRandy Fishel cpu_idle_prop_update_t update, void *arg, cpu_idle_prop_handle_t *hdlp)
850fb2caebeSRandy Fishel {
851fb2caebeSRandy Fishel int rc = EEXIST;
852fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop;
853fb2caebeSRandy Fishel
854fb2caebeSRandy Fishel ASSERT(!CPU_ON_INTR(CPU));
855fb2caebeSRandy Fishel if (name == NULL || hdlp == NULL) {
856fb2caebeSRandy Fishel cmn_err(CE_WARN,
857fb2caebeSRandy Fishel "!cpu_event: NULL parameters in create_property.");
858fb2caebeSRandy Fishel return (EINVAL);
859fb2caebeSRandy Fishel }
860fb2caebeSRandy Fishel
861fb2caebeSRandy Fishel mutex_enter(&cpu_idle_prop_lock);
862fb2caebeSRandy Fishel for (prop = cpu_idle_prop_busy; prop != NULL; prop = prop->next) {
863fb2caebeSRandy Fishel if (strcmp(prop->name, name) == 0) {
864fb2caebeSRandy Fishel cmn_err(CE_NOTE,
865fb2caebeSRandy Fishel "!cpu_event: property %s already exists.", name);
866fb2caebeSRandy Fishel break;
867fb2caebeSRandy Fishel }
868fb2caebeSRandy Fishel }
869fb2caebeSRandy Fishel if (prop == NULL) {
870fb2caebeSRandy Fishel if (cpu_idle_prop_free == NULL) {
871fb2caebeSRandy Fishel cpu_idle_prop_allocate_impl();
872fb2caebeSRandy Fishel }
873fb2caebeSRandy Fishel ASSERT(cpu_idle_prop_free != NULL);
874fb2caebeSRandy Fishel prop = cpu_idle_prop_free;
875fb2caebeSRandy Fishel cpu_idle_prop_free = prop->next;
876fb2caebeSRandy Fishel prop->next = cpu_idle_prop_busy;
877fb2caebeSRandy Fishel cpu_idle_prop_busy = prop;
878fb2caebeSRandy Fishel
879fb2caebeSRandy Fishel ASSERT(prop->value != NULL);
880fb2caebeSRandy Fishel prop->name = strdup(name);
881fb2caebeSRandy Fishel prop->type = type;
882fb2caebeSRandy Fishel prop->update = update;
883fb2caebeSRandy Fishel prop->private = arg;
884fb2caebeSRandy Fishel prop->refcnt = 1;
885fb2caebeSRandy Fishel *hdlp = prop;
886fb2caebeSRandy Fishel rc = 0;
887fb2caebeSRandy Fishel }
888fb2caebeSRandy Fishel mutex_exit(&cpu_idle_prop_lock);
889fb2caebeSRandy Fishel
890fb2caebeSRandy Fishel return (rc);
891fb2caebeSRandy Fishel }
892fb2caebeSRandy Fishel
893fb2caebeSRandy Fishel int
cpu_idle_prop_destroy_property(cpu_idle_prop_handle_t hdl)894fb2caebeSRandy Fishel cpu_idle_prop_destroy_property(cpu_idle_prop_handle_t hdl)
895fb2caebeSRandy Fishel {
896fb2caebeSRandy Fishel int rc = ENODEV;
897fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop, **propp;
898fb2caebeSRandy Fishel cpu_idle_prop_value_t *valp;
899fb2caebeSRandy Fishel
900fb2caebeSRandy Fishel ASSERT(!CPU_ON_INTR(CPU));
901fb2caebeSRandy Fishel if (hdl == NULL) {
902fb2caebeSRandy Fishel cmn_err(CE_WARN,
903fb2caebeSRandy Fishel "!cpu_event: hdl is NULL in destroy_property.");
904fb2caebeSRandy Fishel return (EINVAL);
905fb2caebeSRandy Fishel }
906fb2caebeSRandy Fishel
907fb2caebeSRandy Fishel prop = (cpu_idle_prop_impl_t *)hdl;
908fb2caebeSRandy Fishel mutex_enter(&cpu_idle_prop_lock);
909fb2caebeSRandy Fishel for (propp = &cpu_idle_prop_busy; *propp != NULL;
910fb2caebeSRandy Fishel propp = &(*propp)->next) {
911fb2caebeSRandy Fishel if (*propp == prop) {
912fb2caebeSRandy Fishel ASSERT(prop->refcnt > 0);
913fb2caebeSRandy Fishel if (atomic_cas_32(&prop->refcnt, 1, 0) == 1) {
914fb2caebeSRandy Fishel *propp = prop->next;
915fb2caebeSRandy Fishel strfree(prop->name);
916fb2caebeSRandy Fishel valp = prop->value;
917fb2caebeSRandy Fishel bzero(prop, sizeof (*prop));
918fb2caebeSRandy Fishel prop->value = valp;
919fb2caebeSRandy Fishel prop->next = cpu_idle_prop_free;
920fb2caebeSRandy Fishel cpu_idle_prop_free = prop;
921fb2caebeSRandy Fishel rc = 0;
922fb2caebeSRandy Fishel } else {
923fb2caebeSRandy Fishel rc = EBUSY;
924fb2caebeSRandy Fishel }
925fb2caebeSRandy Fishel break;
926fb2caebeSRandy Fishel }
927fb2caebeSRandy Fishel }
928fb2caebeSRandy Fishel mutex_exit(&cpu_idle_prop_lock);
929fb2caebeSRandy Fishel
930fb2caebeSRandy Fishel return (rc);
931fb2caebeSRandy Fishel }
932fb2caebeSRandy Fishel
933fb2caebeSRandy Fishel int
cpu_idle_prop_create_handle(const char * name,cpu_idle_prop_handle_t * hdlp)934fb2caebeSRandy Fishel cpu_idle_prop_create_handle(const char *name, cpu_idle_prop_handle_t *hdlp)
935fb2caebeSRandy Fishel {
936fb2caebeSRandy Fishel int rc = ENODEV;
937fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop;
938fb2caebeSRandy Fishel
939fb2caebeSRandy Fishel ASSERT(!CPU_ON_INTR(CPU));
940fb2caebeSRandy Fishel if (name == NULL || hdlp == NULL) {
941fb2caebeSRandy Fishel cmn_err(CE_WARN,
942fb2caebeSRandy Fishel "!cpu_event: NULL parameters in create_handle.");
943fb2caebeSRandy Fishel return (EINVAL);
944fb2caebeSRandy Fishel }
945fb2caebeSRandy Fishel
946fb2caebeSRandy Fishel mutex_enter(&cpu_idle_prop_lock);
947fb2caebeSRandy Fishel for (prop = cpu_idle_prop_busy; prop != NULL; prop = prop->next) {
948fb2caebeSRandy Fishel if (strcmp(prop->name, name) == 0) {
949fb2caebeSRandy Fishel /* Hold one refcount on object. */
950fb2caebeSRandy Fishel ASSERT(prop->refcnt > 0);
951fb2caebeSRandy Fishel atomic_inc_32(&prop->refcnt);
952fb2caebeSRandy Fishel *hdlp = (cpu_idle_prop_handle_t)prop;
953fb2caebeSRandy Fishel rc = 0;
954fb2caebeSRandy Fishel break;
955fb2caebeSRandy Fishel }
956fb2caebeSRandy Fishel }
957fb2caebeSRandy Fishel mutex_exit(&cpu_idle_prop_lock);
958fb2caebeSRandy Fishel
959fb2caebeSRandy Fishel return (rc);
960fb2caebeSRandy Fishel }
961fb2caebeSRandy Fishel
962fb2caebeSRandy Fishel int
cpu_idle_prop_destroy_handle(cpu_idle_prop_handle_t hdl)963fb2caebeSRandy Fishel cpu_idle_prop_destroy_handle(cpu_idle_prop_handle_t hdl)
964fb2caebeSRandy Fishel {
965fb2caebeSRandy Fishel int rc = ENODEV;
966fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop;
967fb2caebeSRandy Fishel
968fb2caebeSRandy Fishel ASSERT(!CPU_ON_INTR(CPU));
969fb2caebeSRandy Fishel if (hdl == NULL) {
970fb2caebeSRandy Fishel cmn_err(CE_WARN,
971fb2caebeSRandy Fishel "!cpu_event: hdl is NULL in destroy_handle.");
972fb2caebeSRandy Fishel return (EINVAL);
973fb2caebeSRandy Fishel }
974fb2caebeSRandy Fishel
975fb2caebeSRandy Fishel mutex_enter(&cpu_idle_prop_lock);
976fb2caebeSRandy Fishel for (prop = cpu_idle_prop_busy; prop != NULL; prop = prop->next) {
977fb2caebeSRandy Fishel if (prop == hdl) {
978fb2caebeSRandy Fishel /* Release refcnt held in create_handle. */
979fb2caebeSRandy Fishel ASSERT(prop->refcnt > 1);
980fb2caebeSRandy Fishel atomic_dec_32(&prop->refcnt);
981fb2caebeSRandy Fishel rc = 0;
982fb2caebeSRandy Fishel break;
983fb2caebeSRandy Fishel }
984fb2caebeSRandy Fishel }
985fb2caebeSRandy Fishel mutex_exit(&cpu_idle_prop_lock);
986fb2caebeSRandy Fishel
987fb2caebeSRandy Fishel return (rc);
988fb2caebeSRandy Fishel }
989fb2caebeSRandy Fishel
990fb2caebeSRandy Fishel cpu_idle_prop_type_t
cpu_idle_prop_get_type(cpu_idle_prop_handle_t hdl)991fb2caebeSRandy Fishel cpu_idle_prop_get_type(cpu_idle_prop_handle_t hdl)
992fb2caebeSRandy Fishel {
993fb2caebeSRandy Fishel ASSERT(hdl != NULL);
994fb2caebeSRandy Fishel return (((cpu_idle_prop_impl_t *)hdl)->type);
995fb2caebeSRandy Fishel }
996fb2caebeSRandy Fishel
997fb2caebeSRandy Fishel const char *
cpu_idle_prop_get_name(cpu_idle_prop_handle_t hdl)998fb2caebeSRandy Fishel cpu_idle_prop_get_name(cpu_idle_prop_handle_t hdl)
999fb2caebeSRandy Fishel {
1000fb2caebeSRandy Fishel ASSERT(hdl != NULL);
1001fb2caebeSRandy Fishel return (((cpu_idle_prop_impl_t *)hdl)->name);
1002fb2caebeSRandy Fishel }
1003fb2caebeSRandy Fishel
1004fb2caebeSRandy Fishel int
cpu_idle_prop_get_value(cpu_idle_prop_handle_t hdl,cpu_idle_callback_context_t ctx,cpu_idle_prop_value_t * valp)1005fb2caebeSRandy Fishel cpu_idle_prop_get_value(cpu_idle_prop_handle_t hdl,
1006fb2caebeSRandy Fishel cpu_idle_callback_context_t ctx, cpu_idle_prop_value_t *valp)
1007fb2caebeSRandy Fishel {
1008fb2caebeSRandy Fishel int idx, rc = 0;
1009fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1010fb2caebeSRandy Fishel
1011fb2caebeSRandy Fishel ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1012fb2caebeSRandy Fishel if (hdl == NULL || valp == NULL) {
1013fb2caebeSRandy Fishel cmn_err(CE_NOTE, "!cpu_event: NULL parameters in prop_get.");
1014fb2caebeSRandy Fishel return (EINVAL);
1015fb2caebeSRandy Fishel }
1016fb2caebeSRandy Fishel idx = CPU_IDLE_CTX2IDX(ctx);
1017fb2caebeSRandy Fishel if (prop->update != NULL) {
1018fb2caebeSRandy Fishel cpu_idle_cb_state_t *sp;
1019fb2caebeSRandy Fishel
1020fb2caebeSRandy Fishel ASSERT(CPU->cpu_seqid < max_ncpus);
1021fb2caebeSRandy Fishel sp = &cpu_idle_cb_state[CPU->cpu_seqid];
1022fb2caebeSRandy Fishel /* CPU's idle enter timestamp as sequence number. */
1023fb2caebeSRandy Fishel rc = prop->update(prop->private,
1024fb2caebeSRandy Fishel (uint64_t)sp->v.enter_ts->cipv_hrtime, &prop->value[idx]);
1025fb2caebeSRandy Fishel }
1026fb2caebeSRandy Fishel if (rc == 0) {
1027fb2caebeSRandy Fishel *valp = prop->value[idx];
1028fb2caebeSRandy Fishel }
1029fb2caebeSRandy Fishel
1030fb2caebeSRandy Fishel return (rc);
1031fb2caebeSRandy Fishel }
1032fb2caebeSRandy Fishel
1033fb2caebeSRandy Fishel uint32_t
cpu_idle_prop_get_uint32(cpu_idle_prop_handle_t hdl,cpu_idle_callback_context_t ctx)1034fb2caebeSRandy Fishel cpu_idle_prop_get_uint32(cpu_idle_prop_handle_t hdl,
1035fb2caebeSRandy Fishel cpu_idle_callback_context_t ctx)
1036fb2caebeSRandy Fishel {
1037fb2caebeSRandy Fishel int idx;
1038fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1039fb2caebeSRandy Fishel
1040fb2caebeSRandy Fishel ASSERT(hdl != NULL);
1041fb2caebeSRandy Fishel ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1042fb2caebeSRandy Fishel idx = CPU_IDLE_CTX2IDX(ctx);
1043fb2caebeSRandy Fishel return (prop->value[idx].cipv_uint32);
1044fb2caebeSRandy Fishel }
1045fb2caebeSRandy Fishel
1046fb2caebeSRandy Fishel uint64_t
cpu_idle_prop_get_uint64(cpu_idle_prop_handle_t hdl,cpu_idle_callback_context_t ctx)1047fb2caebeSRandy Fishel cpu_idle_prop_get_uint64(cpu_idle_prop_handle_t hdl,
1048fb2caebeSRandy Fishel cpu_idle_callback_context_t ctx)
1049fb2caebeSRandy Fishel {
1050fb2caebeSRandy Fishel int idx;
1051fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1052fb2caebeSRandy Fishel
1053fb2caebeSRandy Fishel ASSERT(hdl != NULL);
1054fb2caebeSRandy Fishel ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1055fb2caebeSRandy Fishel idx = CPU_IDLE_CTX2IDX(ctx);
1056fb2caebeSRandy Fishel return (prop->value[idx].cipv_uint64);
1057fb2caebeSRandy Fishel }
1058fb2caebeSRandy Fishel
1059fb2caebeSRandy Fishel intptr_t
cpu_idle_prop_get_intptr(cpu_idle_prop_handle_t hdl,cpu_idle_callback_context_t ctx)1060fb2caebeSRandy Fishel cpu_idle_prop_get_intptr(cpu_idle_prop_handle_t hdl,
1061fb2caebeSRandy Fishel cpu_idle_callback_context_t ctx)
1062fb2caebeSRandy Fishel {
1063fb2caebeSRandy Fishel int idx;
1064fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1065fb2caebeSRandy Fishel
1066fb2caebeSRandy Fishel ASSERT(hdl != NULL);
1067fb2caebeSRandy Fishel ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1068fb2caebeSRandy Fishel idx = CPU_IDLE_CTX2IDX(ctx);
1069fb2caebeSRandy Fishel return (prop->value[idx].cipv_intptr);
1070fb2caebeSRandy Fishel }
1071fb2caebeSRandy Fishel
1072fb2caebeSRandy Fishel hrtime_t
cpu_idle_prop_get_hrtime(cpu_idle_prop_handle_t hdl,cpu_idle_callback_context_t ctx)1073fb2caebeSRandy Fishel cpu_idle_prop_get_hrtime(cpu_idle_prop_handle_t hdl,
1074fb2caebeSRandy Fishel cpu_idle_callback_context_t ctx)
1075fb2caebeSRandy Fishel {
1076fb2caebeSRandy Fishel int idx;
1077fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1078fb2caebeSRandy Fishel
1079fb2caebeSRandy Fishel ASSERT(hdl != NULL);
1080fb2caebeSRandy Fishel ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1081fb2caebeSRandy Fishel idx = CPU_IDLE_CTX2IDX(ctx);
1082fb2caebeSRandy Fishel return (prop->value[idx].cipv_hrtime);
1083fb2caebeSRandy Fishel }
1084fb2caebeSRandy Fishel
1085fb2caebeSRandy Fishel void
cpu_idle_prop_set_value(cpu_idle_prop_handle_t hdl,cpu_idle_callback_context_t ctx,cpu_idle_prop_value_t val)1086fb2caebeSRandy Fishel cpu_idle_prop_set_value(cpu_idle_prop_handle_t hdl,
1087fb2caebeSRandy Fishel cpu_idle_callback_context_t ctx, cpu_idle_prop_value_t val)
1088fb2caebeSRandy Fishel {
1089fb2caebeSRandy Fishel int idx;
1090fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1091fb2caebeSRandy Fishel
1092fb2caebeSRandy Fishel ASSERT(hdl != NULL);
1093fb2caebeSRandy Fishel ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1094fb2caebeSRandy Fishel idx = CPU_IDLE_CTX2IDX(ctx);
1095fb2caebeSRandy Fishel prop->value[idx] = val;
1096fb2caebeSRandy Fishel }
1097fb2caebeSRandy Fishel
1098fb2caebeSRandy Fishel void
cpu_idle_prop_set_all(cpu_idle_prop_handle_t hdl,cpu_idle_prop_value_t val)1099fb2caebeSRandy Fishel cpu_idle_prop_set_all(cpu_idle_prop_handle_t hdl, cpu_idle_prop_value_t val)
1100fb2caebeSRandy Fishel {
1101fb2caebeSRandy Fishel int i, idx;
1102fb2caebeSRandy Fishel cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1103fb2caebeSRandy Fishel
1104fb2caebeSRandy Fishel ASSERT(hdl != NULL);
1105fb2caebeSRandy Fishel for (i = 0; i < max_ncpus; i++) {
1106fb2caebeSRandy Fishel idx = CPU_IDLE_CTX2IDX(i);
1107fb2caebeSRandy Fishel prop->value[idx] = val;
1108fb2caebeSRandy Fishel }
1109fb2caebeSRandy Fishel }
1110fb2caebeSRandy Fishel
1111fb2caebeSRandy Fishel /*ARGSUSED*/
cpu_idle_prop_update_intr_cnt(void * arg,uint64_t seqnum,cpu_idle_prop_value_t * valp)1112fb2caebeSRandy Fishel static int cpu_idle_prop_update_intr_cnt(void *arg, uint64_t seqnum,
1113fb2caebeSRandy Fishel cpu_idle_prop_value_t *valp)
1114fb2caebeSRandy Fishel {
1115fb2caebeSRandy Fishel int i;
1116fb2caebeSRandy Fishel uint64_t val;
1117fb2caebeSRandy Fishel
1118fb2caebeSRandy Fishel for (val = 0, i = 0; i < PIL_MAX; i++) {
1119fb2caebeSRandy Fishel val += CPU->cpu_stats.sys.intr[i];
1120fb2caebeSRandy Fishel }
1121fb2caebeSRandy Fishel valp->cipv_uint64 = val;
1122fb2caebeSRandy Fishel
1123fb2caebeSRandy Fishel return (0);
1124fb2caebeSRandy Fishel }
1125fb2caebeSRandy Fishel
1126fb2caebeSRandy Fishel uint_t
cpu_idle_get_cpu_state(cpu_t * cp)1127fb2caebeSRandy Fishel cpu_idle_get_cpu_state(cpu_t *cp)
1128fb2caebeSRandy Fishel {
1129fb2caebeSRandy Fishel ASSERT(cp != NULL && cp->cpu_seqid < max_ncpus);
1130fb2caebeSRandy Fishel return ((uint_t)cpu_idle_prop_get_uint32(
1131fb2caebeSRandy Fishel cpu_idle_prop_array[CPU_IDLE_PROP_IDX_IDLE_STATE].handle,
1132fb2caebeSRandy Fishel CPU_IDLE_GET_CTX(cp)));
1133fb2caebeSRandy Fishel }
1134a3114836SGerry Liu
1135a3114836SGerry Liu #if defined(__x86)
1136a3114836SGerry Liu /*
1137a3114836SGerry Liu * Intercept CPU at a safe point in idle() before powering it off.
1138a3114836SGerry Liu */
1139a3114836SGerry Liu void
cpu_idle_intercept_cpu(cpu_t * cp)1140a3114836SGerry Liu cpu_idle_intercept_cpu(cpu_t *cp)
1141a3114836SGerry Liu {
1142a3114836SGerry Liu ASSERT(cp->cpu_seqid < max_ncpus);
1143a3114836SGerry Liu ASSERT(cpu_idle_cb_state[cp->cpu_seqid].v.enabled == B_FALSE);
1144a3114836SGerry Liu
1145a3114836SGerry Liu /* Set flag to intercept CPU. */
1146a3114836SGerry Liu CPUSET_ATOMIC_ADD(cpu_idle_intercept_set, cp->cpu_id);
1147a3114836SGerry Liu /* Wake up CPU from possible sleep state. */
1148a3114836SGerry Liu poke_cpu(cp->cpu_id);
1149a3114836SGerry Liu while (CPU_IN_SET(cpu_idle_intercept_set, cp->cpu_id)) {
1150a3114836SGerry Liu DELAY(1);
1151a3114836SGerry Liu }
1152a3114836SGerry Liu /*
1153a3114836SGerry Liu * Now target CPU is spinning in a pause loop with interrupts disabled.
1154a3114836SGerry Liu */
1155a3114836SGerry Liu }
1156a3114836SGerry Liu #endif
1157