xref: /illumos-gate/usr/src/uts/common/os/cap_util.c (revision bbf21555)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * Support for determining capacity and utilization of performance relevant
 * hardware components in a computer
 *
 * THEORY
 * ------
 * The capacity and utilization of the performance relevant hardware components
 * is needed to be able to optimize performance while minimizing the amount of
 * power used on a system.  The idea is to use hardware performance counters
 * and potentially other means to determine the capacity and utilization of
 * performance relevant hardware components (eg. execution pipeline, cache,
 * memory, etc.) and attribute the utilization to the responsible CPU and the
 * thread running there.
 *
 * This will help characterize the utilization of performance relevant
 * components and how much is used by each CPU and each thread.  With
 * that data, the utilization can be aggregated to all the CPUs sharing each
 * performance relevant hardware component to calculate the total utilization
 * of each component and compare that with the component's capacity to
 * essentially determine the actual hardware load of the component.  The
 * hardware utilization attributed to each running thread can also be
 * aggregated to determine the total hardware utilization of each component to
 * a workload.
 *
 * Once that is done, one can determine how much of each performance relevant
 * hardware component is needed by a given thread or set of threads (eg. a
 * workload) and size up exactly what hardware is needed by the threads and how
 * much.  With this info, we can better place threads among CPUs to match their
 * exact hardware resource needs and potentially lower or raise the power based
 * on their utilization or pack threads onto the fewest hardware components
 * needed and power off any remaining unused components to minimize power
 * without sacrificing performance.
 *
 * IMPLEMENTATION
 * --------------
 * The code has been designed and implemented to make (un)programming and
 * reading the counters for a given CPU as lightweight and fast as possible.
 * This is very important because we need to read and potentially (un)program
 * the counters very often and in performance sensitive code.  Specifically,
 * the counters may need to be (un)programmed during context switch and/or a
 * cyclic handler when there are more counter events to count than existing
 * counters.
 *
 * Consequently, the code has been split up to allow allocating and
 * initializing everything needed to program and read the counters on a given
 * CPU once and make (un)programming and reading the counters for a given CPU
 * not have to allocate/free memory or grab any locks.  To do this, all the
 * state needed to (un)program and read the counters on a CPU is kept per CPU
 * and is made lock free by forcing any code that reads or manipulates the
 * counters or the state needed to (un)program or read the counters to run on
 * the target CPU and disable preemption while running on the target CPU to
 * protect any critical sections. All counter manipulation on the target CPU is
 * happening either from a cross-call to the target CPU or at the same PIL as
 * used by the cross-call subsystem. This guarantees that counter manipulation
 * is not interrupted by cross-calls from other CPUs.
 *
 * The synchronization has been made lock free or as simple as possible for
 * performance and to avoid getting the locking all tangled up when we interpose
 * on the CPC routines that (un)program the counters to manage the counters
 * between the kernel and user on each CPU.  When the user starts using the
 * counters on a given CPU, the kernel will unprogram the counters that it is
 * using on that CPU just before they are programmed for the user.  Then the
 * kernel will program the counters on a given CPU for its own use when the user
 * stops using them.
 *
 * There is a special interaction with DTrace cpc provider (dcpc). Before dcpc
 * enables any probe, it requests to disable and unprogram all counters used for
 * capacity and utilizations. These counters are never re-programmed back until
 * dcpc completes. When all DTrace cpc probes are removed, dcpc notifies CU
 * framework and it re-programs the counters.
 *
 * When a CPU is going offline, its CU counters are unprogrammed and disabled,
 * so that they would not be re-programmed again by some other activity on the
 * CPU that is going offline.
 *
 * The counters are programmed during boot.  However, a flag is available to
 * disable this if necessary (see cu_flag below).  A handler is provided to
 * (un)program the counters during CPU on/offline.  Basic routines are provided
 * to initialize and tear down this module, initialize and tear down any state
 * needed for a given CPU, and (un)program the counters for a given CPU.
 * Lastly, a handler is provided to read the counters and attribute the
 * utilization to the responsible CPU.
 */
#include <sys/types.h>
#include <sys/cmn_err.h>
#include <sys/cpuvar.h>
#include <sys/ddi.h>
#include <sys/systm.h>
#include <sys/disp.h>
#include <sys/sdt.h>
#include <sys/sunddi.h>
#include <sys/thread.h>
#include <sys/pghw.h>
#include <sys/cmt.h>
#include <sys/policy.h>
#include <sys/x_call.h>
#include <sys/cap_util.h>

#include <sys/archsystm.h>
#include <sys/promif.h>

#if defined(__x86)
#include <sys/xc_levels.h>
#endif


/*
 * Default CPU hardware performance counter flags to use for measuring capacity
 * and utilization
 */
#define	CU_CPC_FLAGS_DEFAULT	\
	(CPC_COUNT_USER|CPC_COUNT_SYSTEM|CPC_OVF_NOTIFY_EMT)

/*
 * Possible Flags for controlling this module.
 */
#define	CU_FLAG_ENABLE		1	/* Enable module */
#define	CU_FLAG_READY		2	/* Ready to setup module */
#define	CU_FLAG_ON		4	/* Module is on */

/*
 * pg_cpu kstats calculate utilization rate and maximum utilization rate for
 * some CPUs. The rate is calculated based on data from two subsequent
 * snapshots. When the time between such two snapshots is too small, the
 * resulting rate may have low accuracy, so we only consider snapshots which
 * are separated by SAMPLE_INTERVAL nanoseconds from one another. We do not
 * update the rate if the interval is smaller than that.
 *
 * Use one tenth of a second as the minimum interval for utilization rate
 * calculation.
 *
 * NOTE: The CU_SAMPLE_INTERVAL_MIN should be higher than the scaling factor in
 * the CU_RATE() macro below to guarantee that we never divide by zero.
 *
 * Rate is the number of events per second. The rate is the number of events
 * divided by time and multiplied by the number of nanoseconds in a second. We
 * do not want time to be too small since it will cause large errors in
 * division.
 *
 * We do not want to multiply two large numbers (the instruction count and
 * NANOSEC) either since it may cause integer overflow. So we divide both the
 * numerator and the denominator by the same value.
 *
 * NOTE: The scaling factor below should be less than CU_SAMPLE_INTERVAL_MIN
 * above to guarantee that time divided by this value is always non-zero.
 */
#define	CU_RATE(val, time) \
	(((val) * (NANOSEC / CU_SCALE)) / ((time) / CU_SCALE))

#define	CU_SAMPLE_INTERVAL_MIN	(NANOSEC / 10)

#define	CU_SCALE (CU_SAMPLE_INTERVAL_MIN / 10000)

/*
 * When the time between two kstat reads for the same CPU is less than
 * CU_UPDATE_THRESHOLD use the old counter data and skip updating counter values
 * for the CPU. This helps reduce cross-calls when kstat consumers read data
 * very often or when they read PG utilization data and then CPU utilization
 * data quickly after that.
 */
#define	CU_UPDATE_THRESHOLD (NANOSEC / 10)

/*
 * The IS_HIPIL() macro verifies that the code is executed either from a
 * cross-call or from high-PIL interrupt
 */
#ifdef DEBUG
#define	IS_HIPIL() (getpil() >= XCALL_PIL)
#else
#define	IS_HIPIL()
#endif	/* DEBUG */


typedef void (*cu_cpu_func_t)(uintptr_t, int *);


/*
 * Flags to use for programming CPU hardware performance counters to measure
 * capacity and utilization
 */
int				cu_cpc_flags = CU_CPC_FLAGS_DEFAULT;

/*
 * Initial value used for programming hardware counters
 */
uint64_t			cu_cpc_preset_value = 0;

/*
 * List of CPC event requests for capacity and utilization.
 */
static kcpc_request_list_t	*cu_cpc_reqs = NULL;

/*
 * When a CPU is a member of PG with a sharing relationship that is supported
 * by the capacity/utilization framework, a kstat is created for that CPU and
 * sharing relationship.
 *
 * These kstats are updated one at a time, so we can have a single scratch
 * space to fill the data.
 *
 * CPU counter kstats fields:
 *
 *   cu_cpu_id		CPU ID for this kstat
 *
 *   cu_pg_id		PG ID for this kstat
 *
 *   cu_generation	Generation value that increases whenever any CPU goes
 *			  offline or online. Two kstat snapshots for the same
 *			  CPU may only be compared if they have the same
 *			  generation.
 *
 *   cu_pg_id		PG ID for the relationship described by this kstat
 *
 *   cu_cpu_util	Running value of CPU utilization for the sharing
 *			  relationship
 *
 *   cu_cpu_time_running Total time spent collecting CU data. The time may be
 *			   less than wall time if CU counters were stopped for
 *			   some time.
 *
 *   cu_cpu_time_stopped Total time the CU counters were stopped.
 *
 *   cu_cpu_rate	Utilization rate, expressed in operations per second.
 *
 *   cu_cpu_rate_max	Maximum observed value of utilization rate.
 *
 *   cu_cpu_relationship Name of sharing relationship for the PG in this kstat
 */
struct cu_cpu_kstat {
	kstat_named_t	cu_cpu_id;
	kstat_named_t	cu_pg_id;
	kstat_named_t	cu_generation;
	kstat_named_t	cu_cpu_util;
	kstat_named_t	cu_cpu_time_running;
	kstat_named_t	cu_cpu_time_stopped;
	kstat_named_t	cu_cpu_rate;
	kstat_named_t	cu_cpu_rate_max;
	kstat_named_t	cu_cpu_relationship;
} cu_cpu_kstat = {
	{ "cpu_id",			KSTAT_DATA_UINT32 },
	{ "pg_id",			KSTAT_DATA_INT32 },
	{ "generation",			KSTAT_DATA_UINT32 },
	{ "hw_util",			KSTAT_DATA_UINT64 },
	{ "hw_util_time_running",	KSTAT_DATA_UINT64 },
	{ "hw_util_time_stopped",	KSTAT_DATA_UINT64 },
	{ "hw_util_rate",		KSTAT_DATA_UINT64 },
	{ "hw_util_rate_max",		KSTAT_DATA_UINT64 },
	{ "relationship",		KSTAT_DATA_STRING },
};

/*
 * Flags for controlling this module
 */
uint_t				cu_flags = CU_FLAG_ENABLE;

/*
 * Error return value for cu_init() since it can't return anything to be called
 * from mp_init_tbl[] (:-(
 */
static int			cu_init_error = 0;

hrtime_t			cu_sample_interval_min = CU_SAMPLE_INTERVAL_MIN;

hrtime_t			cu_update_threshold = CU_UPDATE_THRESHOLD;

static kmutex_t			pg_cpu_kstat_lock;


/*
 * Forward declaration of interface routines
 */
void		cu_disable(void);
void		cu_enable(void);
void		cu_init(void);
void		cu_cpc_program(cpu_t *cp, int *err);
void		cu_cpc_unprogram(cpu_t *cp, int *err);
int		cu_cpu_update(struct cpu *cp, boolean_t move_to);
void		cu_pg_update(pghw_t *pg);


/*
 * Forward declaration of private routines
 */
static int	cu_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs);
static void	cu_cpc_program_xcall(uintptr_t arg, int *err);
static int	cu_cpc_req_add(char *event, kcpc_request_list_t *reqs,
    int nreqs, cu_cntr_stats_t *stats, int kmem_flags, int *nevents);
static int	cu_cpu_callback(cpu_setup_t what, int id, void *arg);
static void	cu_cpu_disable(cpu_t *cp);
static void	cu_cpu_enable(cpu_t *cp);
static int	cu_cpu_init(cpu_t *cp, kcpc_request_list_t *reqs);
static int	cu_cpu_fini(cpu_t *cp);
static void	cu_cpu_kstat_create(pghw_t *pg, cu_cntr_info_t *cntr_info);
static int	cu_cpu_kstat_update(kstat_t *ksp, int rw);
static int	cu_cpu_run(cpu_t *cp, cu_cpu_func_t func, uintptr_t arg);
static int	cu_cpu_update_stats(cu_cntr_stats_t *stats,
    uint64_t cntr_value);
static void cu_cpu_info_detach_xcall(void);

/*
 * Disable or enable Capacity Utilization counters on all CPUs.
 */
void
cu_disable(void)
{
	cpu_t *cp;

	ASSERT(MUTEX_HELD(&cpu_lock));

	cp = cpu_active;
	do {
		if (!(cp->cpu_flags & CPU_OFFLINE))
			cu_cpu_disable(cp);
	} while ((cp = cp->cpu_next_onln) != cpu_active);
}


void
cu_enable(void)
{
	cpu_t *cp;

	ASSERT(MUTEX_HELD(&cpu_lock));

	cp = cpu_active;
	do {
		if (!(cp->cpu_flags & CPU_OFFLINE))
			cu_cpu_enable(cp);
	} while ((cp = cp->cpu_next_onln) != cpu_active);
}


/*
 * Setup capacity and utilization support
 */
void
cu_init(void)
{
	cpu_t	*cp;

	cu_init_error = 0;
	if (!(cu_flags & CU_FLAG_ENABLE) || (cu_flags & CU_FLAG_ON)) {
		cu_init_error = -1;
		return;
	}

	if (kcpc_init() != 0) {
		cu_init_error = -2;
		return;
	}

	/*
	 * Can't measure hardware capacity and utilization without CPU
	 * hardware performance counters
	 */
	if (cpc_ncounters <= 0) {
		cu_init_error = -3;
		return;
	}

	/*
	 * Setup CPC event request queue
	 */
	cu_cpc_reqs = kcpc_reqs_init(cpc_ncounters, KM_SLEEP);

	mutex_enter(&cpu_lock);

	/*
	 * Mark flags to say that module is ready to be setup
	 */
	cu_flags |= CU_FLAG_READY;

	cp = cpu_active;
	do {
		/*
		 * Allocate and setup state needed to measure capacity and
		 * utilization
		 */
		if (cu_cpu_init(cp, cu_cpc_reqs) != 0)
			cu_init_error = -5;

		/*
		 * Reset list of counter event requests so its space can be
		 * reused for a different set of requests for next CPU
		 */
		(void) kcpc_reqs_reset(cu_cpc_reqs);

		cp = cp->cpu_next_onln;
	} while (cp != cpu_active);

	/*
	 * Mark flags to say that module is on now and counters are ready to be
	 * programmed on all active CPUs
	 */
	cu_flags |= CU_FLAG_ON;

	/*
	 * Program counters on currently active CPUs
	 */
	cp = cpu_active;
	do {
		if (cu_cpu_run(cp, cu_cpc_program_xcall,
		    (uintptr_t)B_FALSE) != 0)
			cu_init_error = -6;

		cp = cp->cpu_next_onln;
	} while (cp != cpu_active);

	/*
	 * Register callback for CPU state changes to enable and disable
	 * CPC counters as CPUs come on and offline
	 */
	register_cpu_setup_func(cu_cpu_callback, NULL);

	mutex_exit(&cpu_lock);
}


/*
 * Return number of counter events needed to measure capacity and utilization
 * for specified CPU and fill in list of CPC requests with each counter event
 * needed if list where to add CPC requests is given
 *
 * NOTE: Use KM_NOSLEEP for kmem_{,z}alloc() since cpu_lock is held and free
 *	 everything that has been successfully allocated if any memory
 *	 allocation fails
 */
static int
cu_cpc_init(cpu_t *cp, kcpc_request_list_t *reqs, int nreqs)
{
	group_t		*cmt_pgs;
	cu_cntr_info_t	**cntr_info_array;
	cpu_pg_t	*cpu_pgs;
	cu_cpu_info_t	*cu_cpu_info;
	pg_cmt_t	*pg_cmt;
	pghw_t		*pg_hw;
	cu_cntr_stats_t	*stats;
	int		nevents;
	pghw_type_t	pg_hw_type;
	group_iter_t	iter;

	ASSERT(MUTEX_HELD(&cpu_lock));

	/*
	 * There has to be a target CPU for this
	 */
	if (cp == NULL)
		return (-1);

	/*
	 * Return 0 when CPU doesn't belong to any group
	 */
	cpu_pgs = cp->cpu_pg;
	if (cpu_pgs == NULL || GROUP_SIZE(&cpu_pgs->cmt_pgs) < 1)
		return (0);

	cmt_pgs = &cpu_pgs->cmt_pgs;
	cu_cpu_info = cp->cpu_cu_info;

	/*
	 * Grab counter statistics and info
	 */
	if (reqs == NULL) {
		stats = NULL;
		cntr_info_array = NULL;
	} else {
		if (cu_cpu_info == NULL || cu_cpu_info->cu_cntr_stats == NULL)
			return (-2);

		stats = cu_cpu_info->cu_cntr_stats;
		cntr_info_array = cu_cpu_info->cu_cntr_info;
	}

	/*
	 * See whether platform (or processor) specific code knows which CPC
	 * events to request, etc. are needed to measure hardware capacity and
	 * utilization on this machine
	 */
	nevents = cu_plat_cpc_init(cp, reqs, nreqs);
	if (nevents >= 0)
		return (nevents);

	/*
	 * Let common code decide which CPC events to request, etc. to measure
	 * capacity and utilization since platform (or processor) specific does
	 * not know....
	 *
	 * Walk CPU's PG lineage and do following:
	 *
	 * - Setup CPC request, counter info, and stats needed for each counter
	 *   event to measure capacity and and utilization for each of CPU's PG
	 *   hardware sharing relationships
	 *
	 * - Create PG CPU kstats to export capacity and utilization for each PG
	 */
	nevents = 0;
	group_iter_init(&iter);
	while ((pg_cmt = group_iterate(cmt_pgs, &iter)) != NULL) {
		cu_cntr_info_t	*cntr_info;
		int		nevents_save;
		int		nstats;

		pg_hw = (pghw_t *)pg_cmt;
		pg_hw_type = pg_hw->pghw_hw;
		nevents_save = nevents;
		nstats = 0;

		switch (pg_hw_type) {
		case PGHW_IPIPE:
			if (cu_cpc_req_add("PAPI_tot_ins", reqs, nreqs, stats,
			    KM_NOSLEEP, &nevents) != 0)
				continue;
			nstats = 1;
			break;

		case PGHW_FPU:
			if (cu_cpc_req_add("PAPI_fp_ins", reqs, nreqs, stats,
			    KM_NOSLEEP, &nevents) != 0)
				continue;
			nstats = 1;
			break;

		default:
			/*
			 * Don't measure capacity and utilization for this kind
			 * of PG hardware relationship so skip to next PG in
			 * CPU's PG lineage
			 */
			continue;
		}

		cntr_info = cntr_info_array[pg_hw_type];

		/*
		 * Nothing to measure for this hardware sharing relationship
		 */
		if (nevents - nevents_save == 0) {
			if (cntr_info != NULL) {
				kmem_free(cntr_info, sizeof (cu_cntr_info_t));
				cntr_info_array[pg_hw_type] = NULL;
			}
			continue;
		}

		/*
		 * Fill in counter info for this PG hardware relationship
		 */
		if (cntr_info == NULL) {
			cntr_info = kmem_zalloc(sizeof (cu_cntr_info_t),
			    KM_NOSLEEP);
			if (cntr_info == NULL)
				continue;
			cntr_info_array[pg_hw_type] = cntr_info;
		}
		cntr_info->ci_cpu = cp;
		cntr_info->ci_pg = pg_hw;
		cntr_info->ci_stats = &stats[nevents_save];
		cntr_info->ci_nstats = nstats;

		/*
		 * Create PG CPU kstats for this hardware relationship
		 */
		cu_cpu_kstat_create(pg_hw, cntr_info);
	}

	return (nevents);
}


/*
 * Program counters for capacity and utilization on given CPU
 *
 * If any of the following conditions is true, the counters are not programmed:
 *
 * - CU framework is disabled
 * - The cpu_cu_info field of the cpu structure is NULL
 * - DTrace is active
 * - Counters are programmed already
 * - Counters are disabled (by calls to cu_cpu_disable())
 */
void
cu_cpc_program(cpu_t *cp, int *err)
{
	cu_cpc_ctx_t	*cpu_ctx;
	kcpc_ctx_t	*ctx;
	cu_cpu_info_t	*cu_cpu_info;

	ASSERT(IS_HIPIL());
	/*
	 * Should be running on given CPU. We disable preemption to keep CPU
	 * from disappearing and make sure flags and CPC context don't change
	 * from underneath us
	 */
	kpreempt_disable();
	ASSERT(cp == CPU);

	/*
	 * Module not ready to program counters
	 */
	if (!(cu_flags & CU_FLAG_ON)) {
		*err = -1;
		kpreempt_enable();
		return;
	}

	if (cp == NULL) {
		*err = -2;
		kpreempt_enable();
		return;
	}

	cu_cpu_info = cp->cpu_cu_info;
	if (cu_cpu_info == NULL) {
		*err = -3;
		kpreempt_enable();
		return;
	}

	/*
	 * If DTrace CPC is active or counters turned on already or are
	 * disabled, just return.
	 */
	if (dtrace_cpc_in_use || (cu_cpu_info->cu_flag & CU_CPU_CNTRS_ON) ||
	    cu_cpu_info->cu_disabled) {
		*err = 1;
		kpreempt_enable();
		return;
	}

	if ((CPU->cpu_cpc_ctx != NULL) &&
	    !(CPU->cpu_cpc_ctx->kc_flags & KCPC_CTX_INVALID_STOPPED)) {
		*err = -4;
		kpreempt_enable();
		return;
	}

	/*
	 * Get CPU's CPC context needed for capacity and utilization
	 */
	cpu_ctx = &cu_cpu_info->cu_cpc_ctx;
	ASSERT(cpu_ctx != NULL);
	ASSERT(cpu_ctx->nctx >= 0);

	ASSERT(cpu_ctx->ctx_ptr_array == NULL || cpu_ctx->ctx_ptr_array_sz > 0);
	ASSERT(cpu_ctx->nctx <= cpu_ctx->ctx_ptr_array_sz);
	if (cpu_ctx->nctx <= 0 || cpu_ctx->ctx_ptr_array == NULL ||
	    cpu_ctx->ctx_ptr_array_sz <= 0) {
		*err = -5;
		kpreempt_enable();
		return;
	}

	/*
	 * Increment index in CPU's CPC context info to point at next context
	 * to program
	 *
	 * NOTE: Do this now instead of after programming counters to ensure
	 *	 that index will always point at *current* context so we will
	 *	 always be able to unprogram *current* context if necessary
	 */
	cpu_ctx->cur_index = (cpu_ctx->cur_index + 1) % cpu_ctx->nctx;

	ctx = cpu_ctx->ctx_ptr_array[cpu_ctx->cur_index];

	/*
	 * Clear KCPC_CTX_INVALID and KCPC_CTX_INVALID_STOPPED from CPU's CPC
	 * context before programming counters
	 *
	 * Context is marked with KCPC_CTX_INVALID_STOPPED when context is
	 * unprogrammed and may be marked with KCPC_CTX_INVALID when
	 * kcpc_invalidate_all() is called by cpustat(8) and dtrace CPC to
	 * invalidate all CPC contexts before they take over all the counters.
	 *
	 * This isn't necessary since these flags are only used for thread bound
	 * CPC contexts not CPU bound CPC contexts like ones used for capacity
	 * and utilization.
	 *
	 * There is no need to protect the flag update since no one is using
	 * this context now.
	 */
	ctx->kc_flags &= ~(KCPC_CTX_INVALID | KCPC_CTX_INVALID_STOPPED);

	/*
	 * Program counters on this CPU
	 */
	kcpc_program(ctx, B_FALSE, B_FALSE);

	cp->cpu_cpc_ctx = ctx;

	/*
	 * Set state in CPU structure to say that CPU's counters are programmed
	 * for capacity and utilization now and that they are transitioning from
	 * off to on state. This will cause cu_cpu_update to update stop times
	 * for all programmed counters.
	 */
	cu_cpu_info->cu_flag |= CU_CPU_CNTRS_ON | CU_CPU_CNTRS_OFF_ON;

	/*
	 * Update counter statistics
	 */
	(void) cu_cpu_update(cp, B_FALSE);

	cu_cpu_info->cu_flag &= ~CU_CPU_CNTRS_OFF_ON;

	*err = 0;
	kpreempt_enable();
}


/*
 * Cross call wrapper routine for cu_cpc_program()
 *
 * Checks to make sure that counters on CPU aren't being used by someone else
 * before calling cu_cpc_program() since cu_cpc_program() needs to assert that
 * nobody else is using the counters to catch and prevent any broken code.
 * Also, this check needs to happen on the target CPU since the CPU's CPC
 * context can only be changed while running on the CPU.
 *
 * If the first argument is TRUE, cu_cpc_program_xcall also checks that there is
 * no valid thread bound cpc context. This is important to check to prevent
 * re-programming thread counters with CU counters when CPU is coming on-line.
 */
static void
cu_cpc_program_xcall(uintptr_t arg, int *err)
{
	boolean_t	avoid_thread_context = (boolean_t)arg;

	kpreempt_disable();

	if (CPU->cpu_cpc_ctx != NULL &&
	    !(CPU->cpu_cpc_ctx->kc_flags & KCPC_CTX_INVALID_STOPPED)) {
		*err = -100;
		kpreempt_enable();
		return;
	}

	if (avoid_thread_context && (curthread->t_cpc_ctx != NULL) &&
	    !(curthread->t_cpc_ctx->kc_flags & KCPC_CTX_INVALID_STOPPED)) {
		*err = -200;
		kpreempt_enable();
		return;
	}

	cu_cpc_program(CPU, err);
	kpreempt_enable();
}


/*
 * Unprogram counters for capacity and utilization on given CPU
 * This function should be always executed on the target CPU at high PIL
 */
void
cu_cpc_unprogram(cpu_t *cp, int *err)
{
	cu_cpc_ctx_t	*cpu_ctx;
	kcpc_ctx_t	*ctx;
	cu_cpu_info_t	*cu_cpu_info;

	ASSERT(IS_HIPIL());
	/*
	 * Should be running on given CPU with preemption disabled to keep CPU
	 * from disappearing and make sure flags and CPC context don't change
	 * from underneath us
	 */
	kpreempt_disable();
	ASSERT(cp == CPU);

	/*
	 * Module not on
	 */
	if (!(cu_flags & CU_FLAG_ON)) {
		*err = -1;
		kpreempt_enable();
		return;
	}

	cu_cpu_info = cp->cpu_cu_info;
	if (cu_cpu_info == NULL) {
		*err = -3;
		kpreempt_enable();
		return;