xref: /illumos-gate/usr/src/uts/common/os/msacct.c (revision 2918c4a3)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5f2bd4627Sjohansen  * Common Development and Distribution License (the "License").
6f2bd4627Sjohansen  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
2235a5a358SJonathan Adams  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
24*2918c4a3SJohn Levon  * Copyright (c) 2018, Joyent, Inc.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
277c478bd9Sstevel@tonic-gate #include <sys/types.h>
287c478bd9Sstevel@tonic-gate #include <sys/param.h>
297c478bd9Sstevel@tonic-gate #include <sys/systm.h>
307c478bd9Sstevel@tonic-gate #include <sys/user.h>
317c478bd9Sstevel@tonic-gate #include <sys/proc.h>
327c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
337c478bd9Sstevel@tonic-gate #include <sys/thread.h>
347c478bd9Sstevel@tonic-gate #include <sys/debug.h>
357c478bd9Sstevel@tonic-gate #include <sys/msacct.h>
367c478bd9Sstevel@tonic-gate #include <sys/time.h>
37542a813cSJerry Jelinek #include <sys/zone.h>
387c478bd9Sstevel@tonic-gate 
397c478bd9Sstevel@tonic-gate /*
407c478bd9Sstevel@tonic-gate  * Mega-theory block comment:
417c478bd9Sstevel@tonic-gate  *
427c478bd9Sstevel@tonic-gate  * Microstate accounting uses finite states and the transitions between these
437c478bd9Sstevel@tonic-gate  * states to measure timing and accounting information.  The state information
447c478bd9Sstevel@tonic-gate  * is presently tracked for threads (via microstate accounting) and cpus (via
457c478bd9Sstevel@tonic-gate  * cpu microstate accounting).  In each case, these accounting mechanisms use
467c478bd9Sstevel@tonic-gate  * states and transitions to measure time spent in each state instead of
477c478bd9Sstevel@tonic-gate  * clock-based sampling methodologies.
487c478bd9Sstevel@tonic-gate  *
497c478bd9Sstevel@tonic-gate  * For microstate accounting:
507c478bd9Sstevel@tonic-gate  * state transitions are accomplished by calling new_mstate() to switch between
517c478bd9Sstevel@tonic-gate  * states.  Transitions from a sleeping state (LMS_SLEEP and LMS_STOPPED) occur
527c478bd9Sstevel@tonic-gate  * by calling restore_mstate() which restores a thread to its previously running
537c478bd9Sstevel@tonic-gate  * state.  This code is primarialy executed by the dispatcher in disp() before
547c478bd9Sstevel@tonic-gate  * running a process that was put to sleep.  If the thread was not in a sleeping
557c478bd9Sstevel@tonic-gate  * state, this call has little effect other than to update the count of time the
567c478bd9Sstevel@tonic-gate  * thread has spent waiting on run-queues in its lifetime.
577c478bd9Sstevel@tonic-gate  *
587c478bd9Sstevel@tonic-gate  * For cpu microstate accounting:
597c478bd9Sstevel@tonic-gate  * Cpu microstate accounting is similar to the microstate accounting for threads
607c478bd9Sstevel@tonic-gate  * but it tracks user, system, and idle time for cpus.  Cpu microstate
617c478bd9Sstevel@tonic-gate  * accounting does not track interrupt times as there is a pre-existing
627c478bd9Sstevel@tonic-gate  * interrupt accounting mechanism for this purpose.  Cpu microstate accounting
637c478bd9Sstevel@tonic-gate  * tracks time that user threads have spent active, idle, or in the system on a
647c478bd9Sstevel@tonic-gate  * given cpu.  Cpu microstate accounting has fewer states which allows it to
657c478bd9Sstevel@tonic-gate  * have better defined transitions.  The states transition in the following
667c478bd9Sstevel@tonic-gate  * order:
677c478bd9Sstevel@tonic-gate  *
687c478bd9Sstevel@tonic-gate  *  CMS_USER <-> CMS_SYSTEM <-> CMS_IDLE
697c478bd9Sstevel@tonic-gate  *
707c478bd9Sstevel@tonic-gate  * In order to get to the idle state, the cpu microstate must first go through
717c478bd9Sstevel@tonic-gate  * the system state, and vice-versa for the user state from idle.  The switching
727c478bd9Sstevel@tonic-gate  * of the microstates from user to system is done as part of the regular thread
737c478bd9Sstevel@tonic-gate  * microstate accounting code, except for the idle state which is switched by
747c478bd9Sstevel@tonic-gate  * the dispatcher before it runs the idle loop.
757c478bd9Sstevel@tonic-gate  *
767c478bd9Sstevel@tonic-gate  * Cpu percentages:
777c478bd9Sstevel@tonic-gate  * Cpu percentages are now handled by and based upon microstate accounting
787c478bd9Sstevel@tonic-gate  * information (the same is true for load averages).  The routines which handle
797c478bd9Sstevel@tonic-gate  * the growing/shrinking and exponentiation of cpu percentages have been moved
807c478bd9Sstevel@tonic-gate  * here as it now makes more sense for them to be generated from the microstate
817c478bd9Sstevel@tonic-gate  * code.  Cpu percentages are generated similarly to the way they were before;
827c478bd9Sstevel@tonic-gate  * however, now they are based upon high-resolution timestamps and the
837c478bd9Sstevel@tonic-gate  * timestamps are modified at various state changes instead of during a clock()
847c478bd9Sstevel@tonic-gate  * interrupt.  This allows us to generate more accurate cpu percentages which
857c478bd9Sstevel@tonic-gate  * are also in-sync with microstate data.
867c478bd9Sstevel@tonic-gate  */
877c478bd9Sstevel@tonic-gate 
887c478bd9Sstevel@tonic-gate /*
897c478bd9Sstevel@tonic-gate  * Initialize the microstate level and the
907c478bd9Sstevel@tonic-gate  * associated accounting information for an LWP.
917c478bd9Sstevel@tonic-gate  */
927c478bd9Sstevel@tonic-gate void
init_mstate(kthread_t * t,int init_state)937c478bd9Sstevel@tonic-gate init_mstate(
947c478bd9Sstevel@tonic-gate 	kthread_t	*t,
957c478bd9Sstevel@tonic-gate 	int		init_state)
967c478bd9Sstevel@tonic-gate {
977c478bd9Sstevel@tonic-gate 	struct mstate *ms;
987c478bd9Sstevel@tonic-gate 	klwp_t *lwp;
997c478bd9Sstevel@tonic-gate 	hrtime_t curtime;
1007c478bd9Sstevel@tonic-gate 
1017c478bd9Sstevel@tonic-gate 	ASSERT(init_state != LMS_WAIT_CPU);
1027c478bd9Sstevel@tonic-gate 	ASSERT((unsigned)init_state < NMSTATES);
1037c478bd9Sstevel@tonic-gate 
1047c478bd9Sstevel@tonic-gate 	if ((lwp = ttolwp(t)) != NULL) {
1057c478bd9Sstevel@tonic-gate 		ms = &lwp->lwp_mstate;
1067c478bd9Sstevel@tonic-gate 		curtime = gethrtime_unscaled();
1077c478bd9Sstevel@tonic-gate 		ms->ms_prev = LMS_SYSTEM;
1087c478bd9Sstevel@tonic-gate 		ms->ms_start = curtime;
1097c478bd9Sstevel@tonic-gate 		ms->ms_term = 0;
1107c478bd9Sstevel@tonic-gate 		ms->ms_state_start = curtime;
1117c478bd9Sstevel@tonic-gate 		t->t_mstate = init_state;
1127c478bd9Sstevel@tonic-gate 		t->t_waitrq = 0;
1137c478bd9Sstevel@tonic-gate 		t->t_hrtime = curtime;
1147c478bd9Sstevel@tonic-gate 		if ((t->t_proc_flag & TP_MSACCT) == 0)
1157c478bd9Sstevel@tonic-gate 			t->t_proc_flag |= TP_MSACCT;
1167c478bd9Sstevel@tonic-gate 		bzero((caddr_t)&ms->ms_acct[0], sizeof (ms->ms_acct));
1177c478bd9Sstevel@tonic-gate 	}
1187c478bd9Sstevel@tonic-gate }
1197c478bd9Sstevel@tonic-gate 
1207c478bd9Sstevel@tonic-gate /*
1217c478bd9Sstevel@tonic-gate  * Initialize the microstate level and associated accounting information
1227c478bd9Sstevel@tonic-gate  * for the specified cpu
1237c478bd9Sstevel@tonic-gate  */
1247c478bd9Sstevel@tonic-gate 
1257c478bd9Sstevel@tonic-gate void
init_cpu_mstate(cpu_t * cpu,int init_state)1267c478bd9Sstevel@tonic-gate init_cpu_mstate(
1277c478bd9Sstevel@tonic-gate 	cpu_t *cpu,
1287c478bd9Sstevel@tonic-gate 	int init_state)
1297c478bd9Sstevel@tonic-gate {
1307c478bd9Sstevel@tonic-gate 	ASSERT(init_state != CMS_DISABLED);
1317c478bd9Sstevel@tonic-gate 
1327c478bd9Sstevel@tonic-gate 	cpu->cpu_mstate = init_state;
1337c478bd9Sstevel@tonic-gate 	cpu->cpu_mstate_start = gethrtime_unscaled();
1347c478bd9Sstevel@tonic-gate 	cpu->cpu_waitrq = 0;
1357c478bd9Sstevel@tonic-gate 	bzero((caddr_t)&cpu->cpu_acct[0], sizeof (cpu->cpu_acct));
1367c478bd9Sstevel@tonic-gate }
1377c478bd9Sstevel@tonic-gate 
1387c478bd9Sstevel@tonic-gate /*
1397c478bd9Sstevel@tonic-gate  * sets cpu state to OFFLINE.  We don't actually track this time,
1407c478bd9Sstevel@tonic-gate  * but it serves as a useful placeholder state for when we're not
1417c478bd9Sstevel@tonic-gate  * doing anything.
1427c478bd9Sstevel@tonic-gate  */
1437c478bd9Sstevel@tonic-gate 
1447c478bd9Sstevel@tonic-gate void
term_cpu_mstate(struct cpu * cpu)1457c478bd9Sstevel@tonic-gate term_cpu_mstate(struct cpu *cpu)
1467c478bd9Sstevel@tonic-gate {
1477c478bd9Sstevel@tonic-gate 	ASSERT(cpu->cpu_mstate != CMS_DISABLED);
1487c478bd9Sstevel@tonic-gate 	cpu->cpu_mstate = CMS_DISABLED;
1497c478bd9Sstevel@tonic-gate 	cpu->cpu_mstate_start = 0;
1507c478bd9Sstevel@tonic-gate }
1517c478bd9Sstevel@tonic-gate 
1529102d475Sesolom /* NEW_CPU_MSTATE comments inline in new_cpu_mstate below. */
1539102d475Sesolom 
1549102d475Sesolom #define	NEW_CPU_MSTATE(state)						\
1559102d475Sesolom 	gen = cpu->cpu_mstate_gen;					\
1569102d475Sesolom 	cpu->cpu_mstate_gen = 0;					\
1579102d475Sesolom 	/* Need membar_producer() here if stores not ordered / TSO */	\
1589102d475Sesolom 	cpu->cpu_acct[cpu->cpu_mstate] += curtime - cpu->cpu_mstate_start; \
1599102d475Sesolom 	cpu->cpu_mstate = state;					\
1609102d475Sesolom 	cpu->cpu_mstate_start = curtime;				\
1619102d475Sesolom 	/* Need membar_producer() here if stores not ordered / TSO */	\
1629102d475Sesolom 	cpu->cpu_mstate_gen = (++gen == 0) ? 1 : gen;
1639102d475Sesolom 
1647c478bd9Sstevel@tonic-gate void
new_cpu_mstate(int cmstate,hrtime_t curtime)165eda89462Sesolom new_cpu_mstate(int cmstate, hrtime_t curtime)
1667c478bd9Sstevel@tonic-gate {
167eda89462Sesolom 	cpu_t *cpu = CPU;
168eda89462Sesolom 	uint16_t gen;
1697c478bd9Sstevel@tonic-gate 
1707c478bd9Sstevel@tonic-gate 	ASSERT(cpu->cpu_mstate != CMS_DISABLED);
1717c478bd9Sstevel@tonic-gate 	ASSERT(cmstate < NCMSTATES);
1727c478bd9Sstevel@tonic-gate 	ASSERT(cmstate != CMS_DISABLED);
173eda89462Sesolom 
174eda89462Sesolom 	/*
175eda89462Sesolom 	 * This function cannot be re-entrant on a given CPU. As such,
176eda89462Sesolom 	 * we ASSERT and panic if we are called on behalf of an interrupt.
177eda89462Sesolom 	 * The one exception is for an interrupt which has previously
178eda89462Sesolom 	 * blocked. Such an interrupt is being scheduled by the dispatcher
179eda89462Sesolom 	 * just like a normal thread, and as such cannot arrive here
180eda89462Sesolom 	 * in a re-entrant manner.
181eda89462Sesolom 	 */
182eda89462Sesolom 
183eda89462Sesolom 	ASSERT(!CPU_ON_INTR(cpu) && curthread->t_intr == NULL);
1847c478bd9Sstevel@tonic-gate 	ASSERT(curthread->t_preempt > 0 || curthread == cpu->cpu_idle_thread);
1857c478bd9Sstevel@tonic-gate 
186eda89462Sesolom 	/*
187eda89462Sesolom 	 * LOCKING, or lack thereof:
188eda89462Sesolom 	 *
189eda89462Sesolom 	 * Updates to CPU mstate can only be made by the CPU
190eda89462Sesolom 	 * itself, and the above check to ignore interrupts
191eda89462Sesolom 	 * should prevent recursion into this function on a given
192eda89462Sesolom 	 * processor. i.e. no possible write contention.
193eda89462Sesolom 	 *
194eda89462Sesolom 	 * However, reads of CPU mstate can occur at any time
195eda89462Sesolom 	 * from any CPU. Any locking added to this code path
196eda89462Sesolom 	 * would seriously impact syscall performance. So,
197eda89462Sesolom 	 * instead we have a best-effort protection for readers.
198eda89462Sesolom 	 * The reader will want to account for any time between
199eda89462Sesolom 	 * cpu_mstate_start and the present time. This requires
200eda89462Sesolom 	 * some guarantees that the reader is getting coherent
201eda89462Sesolom 	 * information.
202eda89462Sesolom 	 *
203eda89462Sesolom 	 * We use a generation counter, which is set to 0 before
204eda89462Sesolom 	 * we start making changes, and is set to a new value
205eda89462Sesolom 	 * after we're done. Someone reading the CPU mstate
206eda89462Sesolom 	 * should check for the same non-zero value of this
207eda89462Sesolom 	 * counter both before and after reading all state. The
208eda89462Sesolom 	 * important point is that the reader is not a
209eda89462Sesolom 	 * performance-critical path, but this function is.
2109102d475Sesolom 	 *
2119102d475Sesolom 	 * The ordering of writes is critical. cpu_mstate_gen must
2129102d475Sesolom 	 * be visibly zero on all CPUs before we change cpu_mstate
2139102d475Sesolom 	 * and cpu_mstate_start. Additionally, cpu_mstate_gen must
2149102d475Sesolom 	 * not be restored to oldgen+1 until after all of the other
2159102d475Sesolom 	 * writes have become visible.
2169102d475Sesolom 	 *
2179102d475Sesolom 	 * Normally one puts membar_producer() calls to accomplish
2189102d475Sesolom 	 * this. Unfortunately this routine is extremely performance
2199102d475Sesolom 	 * critical (esp. in syscall_mstate below) and we cannot
2209102d475Sesolom 	 * afford the additional time, particularly on some x86
2219102d475Sesolom 	 * architectures with extremely slow sfence calls. On a
2229102d475Sesolom 	 * CPU which guarantees write ordering (including sparc, x86,
2239102d475Sesolom 	 * and amd64) this is not a problem. The compiler could still
2249102d475Sesolom 	 * reorder the writes, so we make the four cpu fields
2259102d475Sesolom 	 * volatile to prevent this.
2269102d475Sesolom 	 *
2279102d475Sesolom 	 * TSO warning: should we port to a non-TSO (or equivalent)
2289102d475Sesolom 	 * CPU, this will break.
2299102d475Sesolom 	 *
2309102d475Sesolom 	 * The reader stills needs the membar_consumer() calls because,
2319102d475Sesolom 	 * although the volatiles prevent the compiler from reordering
2329102d475Sesolom 	 * loads, the CPU can still do so.
233eda89462Sesolom 	 */
234eda89462Sesolom 
2359102d475Sesolom 	NEW_CPU_MSTATE(cmstate);
2367c478bd9Sstevel@tonic-gate }
2377c478bd9Sstevel@tonic-gate 
238c97ad5cdSakolb /*
239c97ad5cdSakolb  * Return an aggregation of user and system CPU time consumed by
240c97ad5cdSakolb  * the specified thread in scaled nanoseconds.
241c97ad5cdSakolb  */
242c97ad5cdSakolb hrtime_t
mstate_thread_onproc_time(kthread_t * t)243c97ad5cdSakolb mstate_thread_onproc_time(kthread_t *t)
244c97ad5cdSakolb {
245c97ad5cdSakolb 	hrtime_t aggr_time;
246c97ad5cdSakolb 	hrtime_t now;
24735a5a358SJonathan Adams 	hrtime_t waitrq;
248c97ad5cdSakolb 	hrtime_t state_start;
249c97ad5cdSakolb 	struct mstate *ms;
250c97ad5cdSakolb 	klwp_t *lwp;
251c97ad5cdSakolb 	int	mstate;
252c97ad5cdSakolb 
253c97ad5cdSakolb 	ASSERT(THREAD_LOCK_HELD(t));
254c97ad5cdSakolb 
255c97ad5cdSakolb 	if ((lwp = ttolwp(t)) == NULL)
256c97ad5cdSakolb 		return (0);
257c97ad5cdSakolb 
258c97ad5cdSakolb 	mstate = t->t_mstate;
25935a5a358SJonathan Adams 	waitrq = t->t_waitrq;
260c97ad5cdSakolb 	ms = &lwp->lwp_mstate;
261c97ad5cdSakolb 	state_start = ms->ms_state_start;
262c97ad5cdSakolb 
263c97ad5cdSakolb 	aggr_time = ms->ms_acct[LMS_USER] +
264c97ad5cdSakolb 	    ms->ms_acct[LMS_SYSTEM] + ms->ms_acct[LMS_TRAP];
265c97ad5cdSakolb 
266c97ad5cdSakolb 	now = gethrtime_unscaled();
267c97ad5cdSakolb 
268c97ad5cdSakolb 	/*
269c97ad5cdSakolb 	 * NOTE: gethrtime_unscaled on X86 taken on different CPUs is
270c97ad5cdSakolb 	 * inconsistent, so it is possible that now < state_start.
271c97ad5cdSakolb 	 */
27235a5a358SJonathan Adams 	if (mstate == LMS_USER || mstate == LMS_SYSTEM || mstate == LMS_TRAP) {
27335a5a358SJonathan Adams 		/* if waitrq is zero, count all of the time. */
27435a5a358SJonathan Adams 		if (waitrq == 0) {
27535a5a358SJonathan Adams 			waitrq = now;
27635a5a358SJonathan Adams 		}
27735a5a358SJonathan Adams 
27835a5a358SJonathan Adams 		if (waitrq > state_start) {
27935a5a358SJonathan Adams 			aggr_time += waitrq - state_start;
28035a5a358SJonathan Adams 		}
281c97ad5cdSakolb 	}
282c97ad5cdSakolb 
283c97ad5cdSakolb 	scalehrtime(&aggr_time);
284c97ad5cdSakolb 	return (aggr_time);
285c97ad5cdSakolb }
286c97ad5cdSakolb 
28735a5a358SJonathan Adams /*
28835a5a358SJonathan Adams  * Return the amount of onproc and runnable time this thread has experienced.
28935a5a358SJonathan Adams  *
29035a5a358SJonathan Adams  * Because the fields we read are not protected by locks when updated
29135a5a358SJonathan Adams  * by the thread itself, this is an inherently racey interface.  In
29235a5a358SJonathan Adams  * particular, the ASSERT(THREAD_LOCK_HELD(t)) doesn't guarantee as much
29335a5a358SJonathan Adams  * as it might appear to.
29435a5a358SJonathan Adams  *
29535a5a358SJonathan Adams  * The implication for users of this interface is that onproc and runnable
29635a5a358SJonathan Adams  * are *NOT* monotonically increasing; they may temporarily be larger than
29735a5a358SJonathan Adams  * they should be.
29835a5a358SJonathan Adams  */
29935a5a358SJonathan Adams void
mstate_systhread_times(kthread_t * t,hrtime_t * onproc,hrtime_t * runnable)30035a5a358SJonathan Adams mstate_systhread_times(kthread_t *t, hrtime_t *onproc, hrtime_t *runnable)
30135a5a358SJonathan Adams {
30235a5a358SJonathan Adams 	struct mstate	*const	ms = &ttolwp(t)->lwp_mstate;
30335a5a358SJonathan Adams 
30435a5a358SJonathan Adams 	int		mstate;
30535a5a358SJonathan Adams 	hrtime_t	now;
30635a5a358SJonathan Adams 	hrtime_t	state_start;
30735a5a358SJonathan Adams 	hrtime_t	waitrq;
30835a5a358SJonathan Adams 	hrtime_t	aggr_onp;
30935a5a358SJonathan Adams 	hrtime_t	aggr_run;
31035a5a358SJonathan Adams 
31135a5a358SJonathan Adams 	ASSERT(THREAD_LOCK_HELD(t));
31235a5a358SJonathan Adams 	ASSERT(t->t_procp->p_flag & SSYS);
31335a5a358SJonathan Adams 	ASSERT(ttolwp(t) != NULL);
31435a5a358SJonathan Adams 
31535a5a358SJonathan Adams 	/* shouldn't be any non-SYSTEM on-CPU time */
31635a5a358SJonathan Adams 	ASSERT(ms->ms_acct[LMS_USER] == 0);
31735a5a358SJonathan Adams 	ASSERT(ms->ms_acct[LMS_TRAP] == 0);
31835a5a358SJonathan Adams 
31935a5a358SJonathan Adams 	mstate = t->t_mstate;
32035a5a358SJonathan Adams 	waitrq = t->t_waitrq;
32135a5a358SJonathan Adams 	state_start = ms->ms_state_start;
32235a5a358SJonathan Adams 
32335a5a358SJonathan Adams 	aggr_onp = ms->ms_acct[LMS_SYSTEM];
32435a5a358SJonathan Adams 	aggr_run = ms->ms_acct[LMS_WAIT_CPU];
32535a5a358SJonathan Adams 
32635a5a358SJonathan Adams 	now = gethrtime_unscaled();
32735a5a358SJonathan Adams 
32835a5a358SJonathan Adams 	/* if waitrq == 0, then there is no time to account to TS_RUN */
32935a5a358SJonathan Adams 	if (waitrq == 0)
33035a5a358SJonathan Adams 		waitrq = now;
33135a5a358SJonathan Adams 
33235a5a358SJonathan Adams 	/* If there is system time to accumulate, do so */
33335a5a358SJonathan Adams 	if (mstate == LMS_SYSTEM && state_start < waitrq)
33435a5a358SJonathan Adams 		aggr_onp += waitrq - state_start;
33535a5a358SJonathan Adams 
33635a5a358SJonathan Adams 	if (waitrq < now)
33735a5a358SJonathan Adams 		aggr_run += now - waitrq;
33835a5a358SJonathan Adams 
33935a5a358SJonathan Adams 	scalehrtime(&aggr_onp);
34035a5a358SJonathan Adams 	scalehrtime(&aggr_run);
34135a5a358SJonathan Adams 
34235a5a358SJonathan Adams 	*onproc = aggr_onp;
34335a5a358SJonathan Adams 	*runnable = aggr_run;
34435a5a358SJonathan Adams }
34535a5a358SJonathan Adams 
3467c478bd9Sstevel@tonic-gate /*
3477c478bd9Sstevel@tonic-gate  * Return an aggregation of microstate times in scaled nanoseconds (high-res
3487c478bd9Sstevel@tonic-gate  * time).  This keeps in mind that p_acct is already scaled, and ms_acct is
3497c478bd9Sstevel@tonic-gate  * not.
3507c478bd9Sstevel@tonic-gate  */
3517c478bd9Sstevel@tonic-gate hrtime_t
mstate_aggr_state(proc_t * p,int a_state)3527c478bd9Sstevel@tonic-gate mstate_aggr_state(proc_t *p, int a_state)
3537c478bd9Sstevel@tonic-gate {
3547c478bd9Sstevel@tonic-gate 	struct mstate *ms;
3557c478bd9Sstevel@tonic-gate 	kthread_t *t;
3567c478bd9Sstevel@tonic-gate 	klwp_t *lwp;
3577c478bd9Sstevel@tonic-gate 	hrtime_t aggr_time;
3587c478bd9Sstevel@tonic-gate 	hrtime_t scaledtime;
3597c478bd9Sstevel@tonic-gate 
3607c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&p->p_lock));
3617c478bd9Sstevel@tonic-gate 	ASSERT((unsigned)a_state < NMSTATES);
3627c478bd9Sstevel@tonic-gate 
3637c478bd9Sstevel@tonic-gate 	aggr_time = p->p_acct[a_state];
3647c478bd9Sstevel@tonic-gate 	if (a_state == LMS_SYSTEM)
3657c478bd9Sstevel@tonic-gate 		aggr_time += p->p_acct[LMS_TRAP];
3667c478bd9Sstevel@tonic-gate 
3677c478bd9Sstevel@tonic-gate 	t = p->p_tlist;
3687c478bd9Sstevel@tonic-gate 	if (t == NULL)
3697c478bd9Sstevel@tonic-gate 		return (aggr_time);
3707c478bd9Sstevel@tonic-gate 
3717c478bd9Sstevel@tonic-gate 	do {
3727c478bd9Sstevel@tonic-gate 		if (t->t_proc_flag & TP_LWPEXIT)
3737c478bd9Sstevel@tonic-gate 			continue;
3747c478bd9Sstevel@tonic-gate 
3757c478bd9Sstevel@tonic-gate 		lwp = ttolwp(t);
3767c478bd9Sstevel@tonic-gate 		ms = &lwp->lwp_mstate;
3777c478bd9Sstevel@tonic-gate 		scaledtime = ms->ms_acct[a_state];
3787c478bd9Sstevel@tonic-gate 		scalehrtime(&scaledtime);
3797c478bd9Sstevel@tonic-gate 		aggr_time += scaledtime;
3807c478bd9Sstevel@tonic-gate 		if (a_state == LMS_SYSTEM) {
3817c478bd9Sstevel@tonic-gate 			scaledtime = ms->ms_acct[LMS_TRAP];
3827c478bd9Sstevel@tonic-gate 			scalehrtime(&scaledtime);
3837c478bd9Sstevel@tonic-gate 			aggr_time += scaledtime;
3847c478bd9Sstevel@tonic-gate 		}
3857c478bd9Sstevel@tonic-gate 	} while ((t = t->t_forw) != p->p_tlist);
3867c478bd9Sstevel@tonic-gate 
3877c478bd9Sstevel@tonic-gate 	return (aggr_time);
3887c478bd9Sstevel@tonic-gate }
3897c478bd9Sstevel@tonic-gate 
3909102d475Sesolom 
3917c478bd9Sstevel@tonic-gate void
syscall_mstate(int fromms,int toms)3927c478bd9Sstevel@tonic-gate syscall_mstate(int fromms, int toms)
3937c478bd9Sstevel@tonic-gate {
3947c478bd9Sstevel@tonic-gate 	kthread_t *t = curthread;
395542a813cSJerry Jelinek 	zone_t *z = ttozone(t);
3967c478bd9Sstevel@tonic-gate 	struct mstate *ms;
3977c478bd9Sstevel@tonic-gate 	hrtime_t *mstimep;
3987c478bd9Sstevel@tonic-gate 	hrtime_t curtime;
3997c478bd9Sstevel@tonic-gate 	klwp_t *lwp;
4007c478bd9Sstevel@tonic-gate 	hrtime_t newtime;
4019102d475Sesolom 	cpu_t *cpu;
4029102d475Sesolom 	uint16_t gen;
4037c478bd9Sstevel@tonic-gate 
4047c478bd9Sstevel@tonic-gate 	if ((lwp = ttolwp(t)) == NULL)
4057c478bd9Sstevel@tonic-gate 		return;
4067c478bd9Sstevel@tonic-gate 
4077c478bd9Sstevel@tonic-gate 	ASSERT(fromms < NMSTATES);
4087c478bd9Sstevel@tonic-gate 	ASSERT(toms < NMSTATES);
4097c478bd9Sstevel@tonic-gate 
4107c478bd9Sstevel@tonic-gate 	ms = &lwp->lwp_mstate;
4117c478bd9Sstevel@tonic-gate 	mstimep = &ms->ms_acct[fromms];
4127c478bd9Sstevel@tonic-gate 	curtime = gethrtime_unscaled();
4137c478bd9Sstevel@tonic-gate 	newtime = curtime - ms->ms_state_start;
4147c478bd9Sstevel@tonic-gate 	while (newtime < 0) {
4157c478bd9Sstevel@tonic-gate 		curtime = gethrtime_unscaled();
4167c478bd9Sstevel@tonic-gate 		newtime = curtime - ms->ms_state_start;
4177c478bd9Sstevel@tonic-gate 	}
4187c478bd9Sstevel@tonic-gate 	*mstimep += newtime;
4197c478bd9Sstevel@tonic-gate 	t->t_mstate = toms;
4207c478bd9Sstevel@tonic-gate 	ms->ms_state_start = curtime;
4217c478bd9Sstevel@tonic-gate 	ms->ms_prev = fromms;
422eda89462Sesolom 	kpreempt_disable(); /* don't change CPU while changing CPU's state */
4239102d475Sesolom 	cpu = CPU;
4249102d475Sesolom 	ASSERT(cpu == t->t_cpu);
425*2918c4a3SJohn Levon 
426*2918c4a3SJohn Levon 	if (fromms == LMS_USER) {
427*2918c4a3SJohn Levon 		CPU_UARRAY_VAL(z->zone_ustate, cpu->cpu_id,
428*2918c4a3SJohn Levon 		    ZONE_USTATE_UTIME) += newtime;
429*2918c4a3SJohn Levon 	} else if (fromms == LMS_SYSTEM) {
430*2918c4a3SJohn Levon 		CPU_UARRAY_VAL(z->zone_ustate, cpu->cpu_id,
431*2918c4a3SJohn Levon 		    ZONE_USTATE_STIME) += newtime;
432*2918c4a3SJohn Levon 	}
433*2918c4a3SJohn Levon 
4349102d475Sesolom 	if ((toms != LMS_USER) && (cpu->cpu_mstate != CMS_SYSTEM)) {
4359102d475Sesolom 		NEW_CPU_MSTATE(CMS_SYSTEM);
4369102d475Sesolom 	} else if ((toms == LMS_USER) && (cpu->cpu_mstate != CMS_USER)) {
4379102d475Sesolom 		NEW_CPU_MSTATE(CMS_USER);
4389102d475Sesolom 	}
4397c478bd9Sstevel@tonic-gate 	kpreempt_enable();
4407c478bd9Sstevel@tonic-gate }
4417c478bd9Sstevel@tonic-gate 
4429102d475Sesolom #undef NEW_CPU_MSTATE
4439102d475Sesolom 
4447c478bd9Sstevel@tonic-gate /*
4457c478bd9Sstevel@tonic-gate  * The following is for computing the percentage of cpu time used recently
4467c478bd9Sstevel@tonic-gate  * by an lwp.  The function cpu_decay() is also called from /proc code.
4477c478bd9Sstevel@tonic-gate  *
4487c478bd9Sstevel@tonic-gate  * exp_x(x):
4497c478bd9Sstevel@tonic-gate  * Given x as a 64-bit non-negative scaled integer of arbitrary magnitude,
4507c478bd9Sstevel@tonic-gate  * Return exp(-x) as a 64-bit scaled integer in the range [0 .. 1].
4517c478bd9Sstevel@tonic-gate  *
4527c478bd9Sstevel@tonic-gate  * Scaling for 64-bit scaled integer:
4537c478bd9Sstevel@tonic-gate  * The binary point is to the right of the high-order bit
4547c478bd9Sstevel@tonic-gate  * of the low-order 32-bit word.
4557c478bd9Sstevel@tonic-gate  */
4567c478bd9Sstevel@tonic-gate 
4577c478bd9Sstevel@tonic-gate #define	LSHIFT	31
4587c478bd9Sstevel@tonic-gate #define	LSI_ONE	((uint32_t)1 << LSHIFT)	/* 32-bit scaled integer 1 */
4597c478bd9Sstevel@tonic-gate 
4607c478bd9Sstevel@tonic-gate #ifdef DEBUG
4617c478bd9Sstevel@tonic-gate uint_t expx_cnt = 0;	/* number of calls to exp_x() */
4627c478bd9Sstevel@tonic-gate uint_t expx_mul = 0;	/* number of long multiplies in exp_x() */
4637c478bd9Sstevel@tonic-gate #endif
4647c478bd9Sstevel@tonic-gate 
4657c478bd9Sstevel@tonic-gate static uint64_t
exp_x(uint64_t x)4667c478bd9Sstevel@tonic-gate exp_x(uint64_t x)
4677c478bd9Sstevel@tonic-gate {
4687c478bd9Sstevel@tonic-gate 	int i;
4697c478bd9Sstevel@tonic-gate 	uint64_t ull;
4707c478bd9Sstevel@tonic-gate 	uint32_t ui;
4717c478bd9Sstevel@tonic-gate 
4727c478bd9Sstevel@tonic-gate #ifdef DEBUG
4737c478bd9Sstevel@tonic-gate 	expx_cnt++;
4747c478bd9Sstevel@tonic-gate #endif
4757c478bd9Sstevel@tonic-gate 	/*
4767c478bd9Sstevel@tonic-gate 	 * By the formula:
4777c478bd9Sstevel@tonic-gate 	 *	exp(-x) = exp(-x/2) * exp(-x/2)
4787c478bd9Sstevel@tonic-gate 	 * we keep halving x until it becomes small enough for
4797c478bd9Sstevel@tonic-gate 	 * the following approximation to be accurate enough:
4807c478bd9Sstevel@tonic-gate 	 *	exp(-x) = 1 - x
4817c478bd9Sstevel@tonic-gate 	 * We reduce x until it is less than 1/4 (the 2 in LSHIFT-2 below).
4827c478bd9Sstevel@tonic-gate 	 * Our final error will be smaller than 4% .
4837c478bd9Sstevel@tonic-gate 	 */
4847c478bd9Sstevel@tonic-gate 
4857c478bd9Sstevel@tonic-gate 	/*
4867c478bd9Sstevel@tonic-gate 	 * Use a uint64_t for the initial shift calculation.
4877c478bd9Sstevel@tonic-gate 	 */
4887c478bd9Sstevel@tonic-gate 	ull = x >> (LSHIFT-2);
4897c478bd9Sstevel@tonic-gate 
4907c478bd9Sstevel@tonic-gate 	/*
4917c478bd9Sstevel@tonic-gate 	 * Short circuit:
4927c478bd9Sstevel@tonic-gate 	 * A number this large produces effectively 0 (actually .005).
4937c478bd9Sstevel@tonic-gate 	 * This way, we will never do more than 5 multiplies.
4947c478bd9Sstevel@tonic-gate 	 */
4957c478bd9Sstevel@tonic-gate 	if (ull >= (1 << 5))
4967c478bd9Sstevel@tonic-gate 		return (0);
4977c478bd9Sstevel@tonic-gate 
4987c478bd9Sstevel@tonic-gate 	ui = ull;	/* OK.  Now we can use a uint_t. */
4997c478bd9Sstevel@tonic-gate 	for (i = 0; ui != 0; i++)
5007c478bd9Sstevel@tonic-gate 		ui >>= 1;
5017c478bd9Sstevel@tonic-gate 
5027c478bd9Sstevel@tonic-gate 	if (i != 0) {
5037c478bd9Sstevel@tonic-gate #ifdef DEBUG
5047c478bd9Sstevel@tonic-gate 		expx_mul += i;	/* seldom happens */
5057c478bd9Sstevel@tonic-gate #endif
5067c478bd9Sstevel@tonic-gate 		x >>= i;
5077c478bd9Sstevel@tonic-gate 	}
5087c478bd9Sstevel@tonic-gate 
5097c478bd9Sstevel@tonic-gate 	/*
5107c478bd9Sstevel@tonic-gate 	 * Now we compute 1 - x and square it the number of times
5117c478bd9Sstevel@tonic-gate 	 * that we halved x above to produce the final result:
5127c478bd9Sstevel@tonic-gate 	 */
5137c478bd9Sstevel@tonic-gate 	x = LSI_ONE - x;
5147c478bd9Sstevel@tonic-gate 	while (i--)
5157c478bd9Sstevel@tonic-gate 		x = (x * x) >> LSHIFT;
5167c478bd9Sstevel@tonic-gate 
5177c478bd9Sstevel@tonic-gate 	return (x);
5187c478bd9Sstevel@tonic-gate }
5197c478bd9Sstevel@tonic-gate 
5207c478bd9Sstevel@tonic-gate /*
5217c478bd9Sstevel@tonic-gate  * Given the old percent cpu and a time delta in nanoseconds,
5227c478bd9Sstevel@tonic-gate  * return the new decayed percent cpu:  pct * exp(-tau),
5237c478bd9Sstevel@tonic-gate  * where 'tau' is the time delta multiplied by a decay factor.
5247c478bd9Sstevel@tonic-gate  * We have chosen the decay factor (cpu_decay_factor in param.c)
5257c478bd9Sstevel@tonic-gate  * to make the decay over five seconds be approximately 20%.
5267c478bd9Sstevel@tonic-gate  *
5277c478bd9Sstevel@tonic-gate  * 'pct' is a 32-bit scaled integer <= 1
5287c478bd9Sstevel@tonic-gate  * The binary point is to the right of the high-order bit
5297c478bd9Sstevel@tonic-gate  * of the 32-bit word.
5307c478bd9Sstevel@tonic-gate  */
5317c478bd9Sstevel@tonic-gate static uint32_t
cpu_decay(uint32_t pct,hrtime_t nsec)5327c478bd9Sstevel@tonic-gate cpu_decay(uint32_t pct, hrtime_t nsec)
5337c478bd9Sstevel@tonic-gate {
5347c478bd9Sstevel@tonic-gate 	uint64_t delta = (uint64_t)nsec;
5357c478bd9Sstevel@tonic-gate 
5367c478bd9Sstevel@tonic-gate 	delta /= cpu_decay_factor;
5377c478bd9Sstevel@tonic-gate 	return ((pct * exp_x(delta)) >> LSHIFT);
5387c478bd9Sstevel@tonic-gate }
5397c478bd9Sstevel@tonic-gate 
5407c478bd9Sstevel@tonic-gate /*
5417c478bd9Sstevel@tonic-gate  * Given the old percent cpu and a time delta in nanoseconds,
5427c478bd9Sstevel@tonic-gate  * return the new grown percent cpu:  1 - ( 1 - pct ) * exp(-tau)
5437c478bd9Sstevel@tonic-gate  */
5447c478bd9Sstevel@tonic-gate static uint32_t
cpu_grow(uint32_t pct,hrtime_t nsec)5457c478bd9Sstevel@tonic-gate cpu_grow(uint32_t pct, hrtime_t nsec)
5467c478bd9Sstevel@tonic-gate {
5477c478bd9Sstevel@tonic-gate 	return (LSI_ONE - cpu_decay(LSI_ONE - pct, nsec));
5487c478bd9Sstevel@tonic-gate }
5497c478bd9Sstevel@tonic-gate 
5507c478bd9Sstevel@tonic-gate 
5517c478bd9Sstevel@tonic-gate /*
5527c478bd9Sstevel@tonic-gate  * Defined to determine whether a lwp is still on a processor.
5537c478bd9Sstevel@tonic-gate  */
5547c478bd9Sstevel@tonic-gate 
5557c478bd9Sstevel@tonic-gate #define	T_ONPROC(kt)	\
5567c478bd9Sstevel@tonic-gate 	((kt)->t_mstate < LMS_SLEEP)
5577c478bd9Sstevel@tonic-gate #define	T_OFFPROC(kt)	\
5587c478bd9Sstevel@tonic-gate 	((kt)->t_mstate >= LMS_SLEEP)
5597c478bd9Sstevel@tonic-gate 
5607c478bd9Sstevel@tonic-gate uint_t
cpu_update_pct(kthread_t * t,hrtime_t newtime)5617c478bd9Sstevel@tonic-gate cpu_update_pct(kthread_t *t, hrtime_t newtime)
5627c478bd9Sstevel@tonic-gate {
5637c478bd9Sstevel@tonic-gate 	hrtime_t delta;
5647c478bd9Sstevel@tonic-gate 	hrtime_t hrlb;
5657c478bd9Sstevel@tonic-gate 	uint_t pctcpu;
5667c478bd9Sstevel@tonic-gate 	uint_t npctcpu;
5677c478bd9Sstevel@tonic-gate 
5687c478bd9Sstevel@tonic-gate 	/*
5697c478bd9Sstevel@tonic-gate 	 * This routine can get called at PIL > 0, this *has* to be
5707c478bd9Sstevel@tonic-gate 	 * done atomically. Holding locks here causes bad things to happen.
5717c478bd9Sstevel@tonic-gate 	 * (read: deadlock).
5727c478bd9Sstevel@tonic-gate 	 */
5737c478bd9Sstevel@tonic-gate 
5747c478bd9Sstevel@tonic-gate 	do {
575b7671ee8SJerry Jelinek 		pctcpu = t->t_pctcpu;
576b7671ee8SJerry Jelinek 		hrlb = t->t_hrtime;
577b7671ee8SJerry Jelinek 		delta = newtime - hrlb;
578b7671ee8SJerry Jelinek 		if (delta < 0) {
579b7671ee8SJerry Jelinek 			newtime = gethrtime_unscaled();
5807c478bd9Sstevel@tonic-gate 			delta = newtime - hrlb;
581b7671ee8SJerry Jelinek 		}
582b7671ee8SJerry Jelinek 		t->t_hrtime = newtime;
583b7671ee8SJerry Jelinek 		scalehrtime(&delta);
584b7671ee8SJerry Jelinek 		if (T_ONPROC(t) && t->t_waitrq == 0) {
5857c478bd9Sstevel@tonic-gate 			npctcpu = cpu_grow(pctcpu, delta);
5867c478bd9Sstevel@tonic-gate 		} else {
5877c478bd9Sstevel@tonic-gate 			npctcpu = cpu_decay(pctcpu, delta);
5887c478bd9Sstevel@tonic-gate 		}
58975d94465SJosef 'Jeff' Sipek 	} while (atomic_cas_32(&t->t_pctcpu, pctcpu, npctcpu) != pctcpu);
5907c478bd9Sstevel@tonic-gate 
5917c478bd9Sstevel@tonic-gate 	return (npctcpu);
5927c478bd9Sstevel@tonic-gate }
5937c478bd9Sstevel@tonic-gate 
5947c478bd9Sstevel@tonic-gate /*
5957c478bd9Sstevel@tonic-gate  * Change the microstate level for the LWP and update the
5967c478bd9Sstevel@tonic-gate  * associated accounting information.  Return the previous
5977c478bd9Sstevel@tonic-gate  * LWP state.
5987c478bd9Sstevel@tonic-gate  */
5997c478bd9Sstevel@tonic-gate int
new_mstate(kthread_t * t,int new_state)6007c478bd9Sstevel@tonic-gate new_mstate(kthread_t *t, int new_state)
6017c478bd9Sstevel@tonic-gate {
6027c478bd9Sstevel@tonic-gate 	struct mstate *ms;
6037c478bd9Sstevel@tonic-gate 	unsigned state;
6047c478bd9Sstevel@tonic-gate 	hrtime_t *mstimep;
6057c478bd9Sstevel@tonic-gate 	hrtime_t curtime;
6067c478bd9Sstevel@tonic-gate 	hrtime_t newtime;
6077c478bd9Sstevel@tonic-gate 	hrtime_t oldtime;
608542a813cSJerry Jelinek 	hrtime_t ztime;
609542a813cSJerry Jelinek 	hrtime_t origstart;
6107c478bd9Sstevel@tonic-gate 	klwp_t *lwp;
611542a813cSJerry Jelinek 	zone_t *z;
6127c478bd9Sstevel@tonic-gate 
6137c478bd9Sstevel@tonic-gate 	ASSERT(new_state != LMS_WAIT_CPU);
6147c478bd9Sstevel@tonic-gate 	ASSERT((unsigned)new_state < NMSTATES);
6157c478bd9Sstevel@tonic-gate 	ASSERT(t == curthread || THREAD_LOCK_HELD(t));
6167c478bd9Sstevel@tonic-gate 
61714e7cc63Sjohansen 	/*
61814e7cc63Sjohansen 	 * Don't do microstate processing for threads without a lwp (kernel
61914e7cc63Sjohansen 	 * threads).  Also, if we're an interrupt thread that is pinning another
62014e7cc63Sjohansen 	 * thread, our t_mstate hasn't been initialized.  We'd be modifying the
62114e7cc63Sjohansen 	 * microstate of the underlying lwp which doesn't realize that it's
62214e7cc63Sjohansen 	 * pinned.  In this case, also don't change the microstate.
62314e7cc63Sjohansen 	 */
62414e7cc63Sjohansen 	if (((lwp = ttolwp(t)) == NULL) || t->t_intr)
6257c478bd9Sstevel@tonic-gate 		return (LMS_SYSTEM);
6267c478bd9Sstevel@tonic-gate 
6277c478bd9Sstevel@tonic-gate 	curtime = gethrtime_unscaled();
6287c478bd9Sstevel@tonic-gate 
6297c478bd9Sstevel@tonic-gate 	/* adjust cpu percentages before we go any further */
6307c478bd9Sstevel@tonic-gate 	(void) cpu_update_pct(t, curtime);
6317c478bd9Sstevel@tonic-gate 
6327c478bd9Sstevel@tonic-gate 	ms = &lwp->lwp_mstate;
6337c478bd9Sstevel@tonic-gate 	state = t->t_mstate;
634542a813cSJerry Jelinek 	origstart = ms->ms_state_start;
6357c478bd9Sstevel@tonic-gate 	do {
6367c478bd9Sstevel@tonic-gate 		switch (state) {
6377c478bd9Sstevel@tonic-gate 		case LMS_TFAULT:
6387c478bd9Sstevel@tonic-gate 		case LMS_DFAULT:
6397c478bd9Sstevel@tonic-gate 		case LMS_KFAULT:
6407c478bd9Sstevel@tonic-gate 		case LMS_USER_LOCK:
6417c478bd9Sstevel@tonic-gate 			mstimep = &ms->ms_acct[LMS_SYSTEM];
6427c478bd9Sstevel@tonic-gate 			break;
6437c478bd9Sstevel@tonic-gate 		default:
6447c478bd9Sstevel@tonic-gate 			mstimep = &ms->ms_acct[state];
6457c478bd9Sstevel@tonic-gate 			break;
6467c478bd9Sstevel@tonic-gate 		}
647542a813cSJerry Jelinek 		ztime = newtime = curtime - ms->ms_state_start;
6487c478bd9Sstevel@tonic-gate 		if (newtime < 0) {
6497c478bd9Sstevel@tonic-gate 			curtime = gethrtime_unscaled();
6507c478bd9Sstevel@tonic-gate 			oldtime = *mstimep - 1; /* force CAS to fail */
6517c478bd9Sstevel@tonic-gate 			continue;
6527c478bd9Sstevel@tonic-gate 		}
6537c478bd9Sstevel@tonic-gate 		oldtime = *mstimep;
6547c478bd9Sstevel@tonic-gate 		newtime += oldtime;
6557c478bd9Sstevel@tonic-gate 		t->t_mstate = new_state;
6567c478bd9Sstevel@tonic-gate 		ms->ms_state_start = curtime;
65775d94465SJosef 'Jeff' Sipek 	} while (atomic_cas_64((uint64_t *)mstimep, oldtime, newtime) !=
65875d94465SJosef 'Jeff' Sipek 	    oldtime);
659542a813cSJerry Jelinek 
6607c478bd9Sstevel@tonic-gate 	/*
6617c478bd9Sstevel@tonic-gate 	 * Remember the previous running microstate.
6627c478bd9Sstevel@tonic-gate 	 */
6637c478bd9Sstevel@tonic-gate 	if (state != LMS_SLEEP && state != LMS_STOPPED)
6647c478bd9Sstevel@tonic-gate 		ms->ms_prev = state;
6657c478bd9Sstevel@tonic-gate 
6667c478bd9Sstevel@tonic-gate 	/*
6677c478bd9Sstevel@tonic-gate 	 * Switch CPU microstate if appropriate
6687c478bd9Sstevel@tonic-gate 	 */
669eda89462Sesolom 
6707c478bd9Sstevel@tonic-gate 	kpreempt_disable(); /* MUST disable kpreempt before touching t->cpu */
671*2918c4a3SJohn Levon 
672eda89462Sesolom 	ASSERT(t->t_cpu == CPU);
673*2918c4a3SJohn Levon 
674*2918c4a3SJohn Levon 	/*
675*2918c4a3SJohn Levon 	 * When the system boots the initial startup thread will have a
676*2918c4a3SJohn Levon 	 * ms_state_start of 0 which would add a huge system time to the global
677*2918c4a3SJohn Levon 	 * zone.  We want to skip aggregating that initial bit of work.
678*2918c4a3SJohn Levon 	 */
679*2918c4a3SJohn Levon 	if (origstart != 0) {
680*2918c4a3SJohn Levon 		z = ttozone(t);
681*2918c4a3SJohn Levon 		if (state == LMS_USER) {
682*2918c4a3SJohn Levon 			CPU_UARRAY_VAL(z->zone_ustate, t->t_cpu->cpu_id,
683*2918c4a3SJohn Levon 			    ZONE_USTATE_UTIME) += ztime;
684*2918c4a3SJohn Levon 		} else if (state == LMS_SYSTEM) {
685*2918c4a3SJohn Levon 			CPU_UARRAY_VAL(z->zone_ustate, t->t_cpu->cpu_id,
686*2918c4a3SJohn Levon 			    ZONE_USTATE_STIME) += ztime;
687*2918c4a3SJohn Levon 		}
688*2918c4a3SJohn Levon 	}
689*2918c4a3SJohn Levon 
690eda89462Sesolom 	if (!CPU_ON_INTR(t->t_cpu) && curthread->t_intr == NULL) {
691eda89462Sesolom 		if (new_state == LMS_USER && t->t_cpu->cpu_mstate != CMS_USER)
692eda89462Sesolom 			new_cpu_mstate(CMS_USER, curtime);
693eda89462Sesolom 		else if (new_state != LMS_USER &&
694eda89462Sesolom 		    t->t_cpu->cpu_mstate != CMS_SYSTEM)
695eda89462Sesolom 			new_cpu_mstate(CMS_SYSTEM, curtime);
6967c478bd9Sstevel@tonic-gate 	}
6977c478bd9Sstevel@tonic-gate 	kpreempt_enable();
6987c478bd9Sstevel@tonic-gate 
6997c478bd9Sstevel@tonic-gate 	return (ms->ms_prev);
7007c478bd9Sstevel@tonic-gate }
7017c478bd9Sstevel@tonic-gate 
7027c478bd9Sstevel@tonic-gate /*
7037c478bd9Sstevel@tonic-gate  * Restore the LWP microstate to the previous runnable state.
7047c478bd9Sstevel@tonic-gate  * Called from disp() with the newly selected lwp.
7057c478bd9Sstevel@tonic-gate  */
7067c478bd9Sstevel@tonic-gate void
restore_mstate(kthread_t * t)7077c478bd9Sstevel@tonic-gate restore_mstate(kthread_t *t)
7087c478bd9Sstevel@tonic-gate {
7097c478bd9Sstevel@tonic-gate 	struct mstate *ms;
7107c478bd9Sstevel@tonic-gate 	hrtime_t *mstimep;
7117c478bd9Sstevel@tonic-gate 	klwp_t *lwp;
7127c478bd9Sstevel@tonic-gate 	hrtime_t curtime;
7137c478bd9Sstevel@tonic-gate 	hrtime_t waitrq;
7147c478bd9Sstevel@tonic-gate 	hrtime_t newtime;
7157c478bd9Sstevel@tonic-gate 	hrtime_t oldtime;
716542a813cSJerry Jelinek 	hrtime_t waittime;
717542a813cSJerry Jelinek 	zone_t *z;
7187c478bd9Sstevel@tonic-gate 
71914e7cc63Sjohansen 	/*
72014e7cc63Sjohansen 	 * Don't call restore mstate of threads without lwps.  (Kernel threads)
72114e7cc63Sjohansen 	 *
72214e7cc63Sjohansen 	 * threads with t_intr set shouldn't be in the dispatcher, so assert
72314e7cc63Sjohansen 	 * that nobody here has t_intr.
72414e7cc63Sjohansen 	 */
72514e7cc63Sjohansen 	ASSERT(t->t_intr == NULL);
72614e7cc63Sjohansen 
7277c478bd9Sstevel@tonic-gate 	if ((lwp = ttolwp(t)) == NULL)
7287c478bd9Sstevel@tonic-gate 		return;
7297c478bd9Sstevel@tonic-gate 
7307c478bd9Sstevel@tonic-gate 	curtime = gethrtime_unscaled();
7317c478bd9Sstevel@tonic-gate 	(void) cpu_update_pct(t, curtime);
7327c478bd9Sstevel@tonic-gate 	ms = &lwp->lwp_mstate;
7337c478bd9Sstevel@tonic-gate 	ASSERT((unsigned)t->t_mstate < NMSTATES);
7347c478bd9Sstevel@tonic-gate 	do {
7357c478bd9Sstevel@tonic-gate 		switch (t->t_mstate) {
7367c478bd9Sstevel@tonic-gate 		case LMS_SLEEP:
7377c478bd9Sstevel@tonic-gate 			/*
7387c478bd9Sstevel@tonic-gate 			 * Update the timer for the current sleep state.
7397c478bd9Sstevel@tonic-gate 			 */
7407c478bd9Sstevel@tonic-gate 			ASSERT((unsigned)ms->ms_prev < NMSTATES);
7417c478bd9Sstevel@tonic-gate 			switch (ms->ms_prev) {
7427c478bd9Sstevel@tonic-gate 			case LMS_TFAULT:
7437c478bd9Sstevel@tonic-gate 			case LMS_DFAULT:
7447c478bd9Sstevel@tonic-gate 			case LMS_KFAULT:
7457c478bd9Sstevel@tonic-gate 			case LMS_USER_LOCK:
7467c478bd9Sstevel@tonic-gate 				mstimep = &ms->ms_acct[ms->ms_prev];
7477c478bd9Sstevel@tonic-gate 				break;
7487c478bd9Sstevel@tonic-gate 			default:
7497c478bd9Sstevel@tonic-gate 				mstimep = &ms->ms_acct[LMS_SLEEP];
7507c478bd9Sstevel@tonic-gate 				break;
7517c478bd9Sstevel@tonic-gate 			}
7527c478bd9Sstevel@tonic-gate 			/*
7537c478bd9Sstevel@tonic-gate 			 * Return to the previous run state.
7547c478bd9Sstevel@tonic-gate 			 */
7557c478bd9Sstevel@tonic-gate 			t->t_mstate = ms->ms_prev;
7567c478bd9Sstevel@tonic-gate 			break;
7577c478bd9Sstevel@tonic-gate 		case LMS_STOPPED:
7587c478bd9Sstevel@tonic-gate 			mstimep = &ms->ms_acct[LMS_STOPPED];
7597c478bd9Sstevel@tonic-gate 			/*
7607c478bd9Sstevel@tonic-gate 			 * Return to the previous run state.
7617c478bd9Sstevel@tonic-gate 			 */
7627c478bd9Sstevel@tonic-gate 			t->t_mstate = ms->ms_prev;
7637c478bd9Sstevel@tonic-gate 			break;
7647c478bd9Sstevel@tonic-gate 		case LMS_TFAULT:
7657c478bd9Sstevel@tonic-gate 		case LMS_DFAULT:
7667c478bd9Sstevel@tonic-gate 		case LMS_KFAULT:
7677c478bd9Sstevel@tonic-gate 		case LMS_USER_LOCK:
7687c478bd9Sstevel@tonic-gate 			mstimep = &ms->ms_acct[LMS_SYSTEM];
7697c478bd9Sstevel@tonic-gate 			break;
7707c478bd9Sstevel@tonic-gate 		default:
7717c478bd9Sstevel@tonic-gate 			mstimep = &ms->ms_acct[t->t_mstate];
7727c478bd9Sstevel@tonic-gate 			break;
7737c478bd9Sstevel@tonic-gate 		}
7747c478bd9Sstevel@tonic-gate 		waitrq = t->t_waitrq;	/* hopefully atomic */
775f2bd4627Sjohansen 		if (waitrq == 0) {
7767c478bd9Sstevel@tonic-gate 			waitrq = curtime;
7777c478bd9Sstevel@tonic-gate 		}
778f2bd4627Sjohansen 		t->t_waitrq = 0;
7797c478bd9Sstevel@tonic-gate 		newtime = waitrq - ms->ms_state_start;
7807c478bd9Sstevel@tonic-gate 		if (newtime < 0) {
7817c478bd9Sstevel@tonic-gate 			curtime = gethrtime_unscaled();
7827c478bd9Sstevel@tonic-gate 			oldtime = *mstimep - 1; /* force CAS to fail */
7837c478bd9Sstevel@tonic-gate 			continue;
7847c478bd9Sstevel@tonic-gate 		}
7857c478bd9Sstevel@tonic-gate 		oldtime = *mstimep;
7867c478bd9Sstevel@tonic-gate 		newtime += oldtime;
78775d94465SJosef 'Jeff' Sipek 	} while (atomic_cas_64((uint64_t *)mstimep, oldtime, newtime) !=
78875d94465SJosef 'Jeff' Sipek 	    oldtime);
789542a813cSJerry Jelinek 
7907c478bd9Sstevel@tonic-gate 	/*
7917c478bd9Sstevel@tonic-gate 	 * Update the WAIT_CPU timer and per-cpu waitrq total.
7927c478bd9Sstevel@tonic-gate 	 */
793542a813cSJerry Jelinek 	z = ttozone(t);
794542a813cSJerry Jelinek 	waittime = curtime - waitrq;
795542a813cSJerry Jelinek 	ms->ms_acct[LMS_WAIT_CPU] += waittime;
796*2918c4a3SJohn Levon 
797*2918c4a3SJohn Levon 	/*
798*2918c4a3SJohn Levon 	 * We are in a disp context where we're not going to migrate CPUs.
799*2918c4a3SJohn Levon 	 */
800*2918c4a3SJohn Levon 	CPU_UARRAY_VAL(z->zone_ustate, CPU->cpu_id,
801*2918c4a3SJohn Levon 	    ZONE_USTATE_WTIME) += waittime;
802*2918c4a3SJohn Levon 
803542a813cSJerry Jelinek 	CPU->cpu_waitrq += waittime;
8047c478bd9Sstevel@tonic-gate 	ms->ms_state_start = curtime;
8057c478bd9Sstevel@tonic-gate }
8067c478bd9Sstevel@tonic-gate 
8077c478bd9Sstevel@tonic-gate /*
8087c478bd9Sstevel@tonic-gate  * Copy lwp microstate accounting and resource usage information
8097c478bd9Sstevel@tonic-gate  * to the process.  (lwp is terminating)
8107c478bd9Sstevel@tonic-gate  */
8117c478bd9Sstevel@tonic-gate void
term_mstate(kthread_t * t)8127c478bd9Sstevel@tonic-gate term_mstate(kthread_t *t)
8137c478bd9Sstevel@tonic-gate {
8147c478bd9Sstevel@tonic-gate 	struct mstate *ms;
8157c478bd9Sstevel@tonic-gate 	proc_t *p = ttoproc(t);
8167c478bd9Sstevel@tonic-gate 	klwp_t *lwp = ttolwp(t);
8177c478bd9Sstevel@tonic-gate 	int i;
8187c478bd9Sstevel@tonic-gate 	hrtime_t tmp;
8197c478bd9Sstevel@tonic-gate 
8207c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&p->p_lock));
8217c478bd9Sstevel@tonic-gate 
8227c478bd9Sstevel@tonic-gate 	ms = &lwp->lwp_mstate;
8237c478bd9Sstevel@tonic-gate 	(void) new_mstate(t, LMS_STOPPED);
8247c478bd9Sstevel@tonic-gate 	ms->ms_term = ms->ms_state_start;
8257c478bd9Sstevel@tonic-gate 	tmp = ms->ms_term - ms->ms_start;
8267c478bd9Sstevel@tonic-gate 	scalehrtime(&tmp);
8277c478bd9Sstevel@tonic-gate 	p->p_mlreal += tmp;
8287c478bd9Sstevel@tonic-gate 	for (i = 0; i < NMSTATES; i++) {
8297c478bd9Sstevel@tonic-gate 		tmp = ms->ms_acct[i];
8307c478bd9Sstevel@tonic-gate 		scalehrtime(&tmp);
8317c478bd9Sstevel@tonic-gate 		p->p_acct[i] += tmp;
8327c478bd9Sstevel@tonic-gate 	}
8337c478bd9Sstevel@tonic-gate 	p->p_ru.minflt   += lwp->lwp_ru.minflt;
8347c478bd9Sstevel@tonic-gate 	p->p_ru.majflt   += lwp->lwp_ru.majflt;
8357c478bd9Sstevel@tonic-gate 	p->p_ru.nswap    += lwp->lwp_ru.nswap;
8367c478bd9Sstevel@tonic-gate 	p->p_ru.inblock  += lwp->lwp_ru.inblock;
8377c478bd9Sstevel@tonic-gate 	p->p_ru.oublock  += lwp->lwp_ru.oublock;
8387c478bd9Sstevel@tonic-gate 	p->p_ru.msgsnd   += lwp->lwp_ru.msgsnd;
8397c478bd9Sstevel@tonic-gate 	p->p_ru.msgrcv   += lwp->lwp_ru.msgrcv;
8407c478bd9Sstevel@tonic-gate 	p->p_ru.nsignals += lwp->lwp_ru.nsignals;
8417c478bd9Sstevel@tonic-gate 	p->p_ru.nvcsw    += lwp->lwp_ru.nvcsw;
8427c478bd9Sstevel@tonic-gate 	p->p_ru.nivcsw   += lwp->lwp_ru.nivcsw;
8437c478bd9Sstevel@tonic-gate 	p->p_ru.sysc	 += lwp->lwp_ru.sysc;
8447c478bd9Sstevel@tonic-gate 	p->p_ru.ioch	 += lwp->lwp_ru.ioch;
8457c478bd9Sstevel@tonic-gate 	p->p_defunct++;
8467c478bd9Sstevel@tonic-gate }
847