27c478bdstevel@tonic-gate * CDDL HEADER START
37c478bdstevel@tonic-gate *
47c478bdstevel@tonic-gate * The contents of this file are subject to the terms of the
52b616c6wesolows * Common Development and Distribution License (the "License").
62b616c6wesolows * You may not use this file except in compliance with the License.
77c478bdstevel@tonic-gate *
87c478bdstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bdstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bdstevel@tonic-gate * See the License for the specific language governing permissions
117c478bdstevel@tonic-gate * and limitations under the License.
127c478bdstevel@tonic-gate *
137c478bdstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bdstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bdstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bdstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bdstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bdstevel@tonic-gate *
197c478bdstevel@tonic-gate * CDDL HEADER END
207c478bdstevel@tonic-gate */
23b52a336Pavel Tatashin * Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
241f9f06cMatthew Ahrens * Copyright (c) 2012 by Delphix. All rights reserved.
256481fd4Igor Kozhukhov * Copyright 2014 Igor Kozhukhov <ikozhukhov@gmail.com>.
26455e370John Levon * Copyright 2018 Joyent, Inc.
27f06dce2Andrew Stormont * Copyright 2017 RackTop Systems.
2889574a1Patrick Mooney * Copyright 2019 Joyent, Inc.
297c478bdstevel@tonic-gate */
317c478bdstevel@tonic-gate#ifndef _SYS_CPUVAR_H
327c478bdstevel@tonic-gate#define	_SYS_CPUVAR_H
347c478bdstevel@tonic-gate#include <sys/thread.h>
357c478bdstevel@tonic-gate#include <sys/sysinfo.h>	/* has cpu_stat_t definition */
367c478bdstevel@tonic-gate#include <sys/disp.h>
377c478bdstevel@tonic-gate#include <sys/processor.h>
38f06dce2Andrew Stormont#include <sys/kcpc.h>		/* has kcpc_ctx_t definition */
406481fd4Igor Kozhukhov#include <sys/loadavg.h>
417c478bdstevel@tonic-gate#if (defined(_KERNEL) || defined(_KMEMUSER)) && defined(_MACHDEP)
427c478bdstevel@tonic-gate#include <sys/machcpuvar.h>
457c478bdstevel@tonic-gate#include <sys/types.h>
467c478bdstevel@tonic-gate#include <sys/file.h>
477c478bdstevel@tonic-gate#include <sys/bitmap.h>
487c478bdstevel@tonic-gate#include <sys/rwlock.h>
497c478bdstevel@tonic-gate#include <sys/msacct.h>
502b616c6wesolows#if defined(__GNUC__) && defined(_ASM_INLINES) && defined(_KERNEL) && \
512b616c6wesolows	(defined(__i386) || defined(__amd64))
527c478bdstevel@tonic-gate#include <asm/cpuvar.h>
557c478bdstevel@tonic-gate#ifdef	__cplusplus
567c478bdstevel@tonic-gateextern "C" {
597c478bdstevel@tonic-gatestruct squeue_set_s;
617c478bdstevel@tonic-gate#define	CPU_CACHE_COHERENCE_SIZE	64
647c478bdstevel@tonic-gate * For fast event tracing.
657c478bdstevel@tonic-gate */
667c478bdstevel@tonic-gatestruct ftrace_record;
677c478bdstevel@tonic-gatetypedef struct ftrace_data {
687c478bdstevel@tonic-gate	int			ftd_state;	/* ftrace flags */
694df4bd6bs	kmutex_t		ftd_unused;	/* ftrace buffer lock, unused */
707c478bdstevel@tonic-gate	struct ftrace_record	*ftd_cur;	/* current record */
717c478bdstevel@tonic-gate	struct ftrace_record	*ftd_first;	/* first record */
727c478bdstevel@tonic-gate	struct ftrace_record	*ftd_last;	/* last record */
737c478bdstevel@tonic-gate} ftrace_data_t;
757c478bdstevel@tonic-gatestruct cyc_cpu;
767c478bdstevel@tonic-gatestruct nvlist;
797c478bdstevel@tonic-gate * Per-CPU data.
80e824d57johnlev *
81e824d57johnlev * Be careful adding new members: if they are not the same in all modules (e.g.
82e824d57johnlev * change size depending on a #define), CTF uniquification can fail to work
83e824d57johnlev * properly.  Furthermore, this is transitive in that it applies recursively to
84e824d57johnlev * all types pointed to by cpu_t.
857c478bdstevel@tonic-gate */
867c478bdstevel@tonic-gatetypedef struct cpu {
877c478bdstevel@tonic-gate	processorid_t	cpu_id;			/* CPU number */
887c478bdstevel@tonic-gate	processorid_t	cpu_seqid;	/* sequential CPU id (0..ncpus-1) */
897c478bdstevel@tonic-gate	volatile cpu_flag_t cpu_flags;		/* flags indicating CPU state */
907c478bdstevel@tonic-gate	struct cpu	*cpu_self;		/* pointer to itself */
917c478bdstevel@tonic-gate	kthread_t	*cpu_thread;		/* current thread */
927c478bdstevel@tonic-gate	kthread_t	*cpu_idle_thread;	/* idle thread for this CPU */
937c478bdstevel@tonic-gate	kthread_t	*cpu_pause_thread;	/* pause thread for this CPU */
947c478bdstevel@tonic-gate	klwp_id_t	cpu_lwp;		/* current lwp (if any) */
957c478bdstevel@tonic-gate	klwp_id_t	cpu_fpowner;		/* currently loaded fpu owner */
967c478bdstevel@tonic-gate	struct cpupart	*cpu_part;		/* partition with this CPU */
977c478bdstevel@tonic-gate	struct lgrp_ld	*cpu_lpl;		/* pointer to this cpu's load */
987c478bdstevel@tonic-gate	int		cpu_cache_offset;	/* see kmem.c for details */
1007c478bdstevel@tonic-gate	/*
1017c478bdstevel@tonic-gate	 * Links to other CPUs.  It is safe to walk these lists if
1027c478bdstevel@tonic-gate	 * one of the following is true:
10389574a1Patrick Mooney	 *	- cpu_lock held
10489574a1Patrick Mooney	 *	- preemption disabled via kpreempt_disable
10589574a1Patrick Mooney	 *	- PIL >= DISP_LEVEL
10689574a1Patrick Mooney	 *	- acting thread is an interrupt thread
10789574a1Patrick Mooney	 *	- all other CPUs are paused
1087c478bdstevel@tonic-gate	 */
1097c478bdstevel@tonic-gate	struct cpu	*cpu_next;		/* next existing CPU */
1107c478bdstevel@tonic-gate	struct cpu	*cpu_prev;		/* prev existing CPU */
1117c478bdstevel@tonic-gate	struct cpu	*cpu_next_onln;		/* next online (enabled) CPU */
1127c478bdstevel@tonic-gate	struct cpu	*cpu_prev_onln;		/* prev online (enabled) CPU */
1137c478bdstevel@tonic-gate	struct cpu	*cpu_next_part;		/* next CPU in partition */
1147c478bdstevel@tonic-gate	struct cpu	*cpu_prev_part;		/* prev CPU in partition */
1157c478bdstevel@tonic-gate	struct cpu	*cpu_next_lgrp;		/* next CPU in latency group */
1167c478bdstevel@tonic-gate	struct cpu	*cpu_prev_lgrp;		/* prev CPU in latency group */
1177c478bdstevel@tonic-gate	struct cpu	*cpu_next_lpl;		/* next CPU in lgrp partition */
1187c478bdstevel@tonic-gate	struct cpu	*cpu_prev_lpl;
120fb2f18fesaxe	struct cpu_pg	*cpu_pg;		/* cpu's processor groups */
1227c478bdstevel@tonic-gate	void		*cpu_reserved[4];	/* reserved for future use */
1247c478bdstevel@tonic-gate	/*
1257c478bdstevel@tonic-gate	 * Scheduling variables.
1267c478bdstevel@tonic-gate	 */
1277c478bdstevel@tonic-gate	disp_t		*cpu_disp;		/* dispatch queue data */
1287c478bdstevel@tonic-gate	/*
1297c478bdstevel@tonic-gate	 * Note that cpu_disp is set before the CPU is added to the system
1307c478bdstevel@tonic-gate	 * and is never modified.  Hence, no additional locking is needed
1317c478bdstevel@tonic-gate	 * beyond what's necessary to access the cpu_t structure.
1327c478bdstevel@tonic-gate	 */
1337c478bdstevel@tonic-gate	char		cpu_runrun;	/* scheduling flag - set to preempt */
1347c478bdstevel@tonic-gate	char		cpu_kprunrun;		/* force kernel preemption */
13589574a1Patrick Mooney	pri_t		cpu_chosen_level;	/* priority at which cpu */
1367c478bdstevel@tonic-gate						/* was chosen for scheduling */
1377c478bdstevel@tonic-gate	kthread_t	*cpu_dispthread; /* thread selected for dispatch */
1387c478bdstevel@tonic-gate	disp_lock_t	cpu_thread_lock; /* dispatcher lock on current thread */
1397c478bdstevel@tonic-gate	uint8_t		cpu_disp_flags;	/* flags used by dispatcher */
1407c478bdstevel@tonic-gate	/*
1417c478bdstevel@tonic-gate	 * The following field is updated when ever the cpu_dispthread
1427c478bdstevel@tonic-gate	 * changes. Also in places, where the current thread(cpu_dispthread)
1437c478bdstevel@tonic-gate	 * priority changes. This is used in disp_lowpri_cpu()
1447c478bdstevel@tonic-gate	 */
1457c478bdstevel@tonic-gate	pri_t		cpu_dispatch_pri; /* priority of cpu_dispthread */
1467c478bdstevel@tonic-gate	clock_t		cpu_last_swtch;	/* last time switched to new thread */
1487c478bdstevel@tonic-gate	/*
1497c478bdstevel@tonic-gate	 * Interrupt data.
1507c478bdstevel@tonic-gate	 */
1517c478bdstevel@tonic-gate	caddr_t		cpu_intr_stack;	/* interrupt stack */
1527c478bdstevel@tonic-gate	kthread_t	*cpu_intr_thread; /* interrupt thread list */
1537c478bdstevel@tonic-gate	uint_t		cpu_intr_actv;	/* interrupt levels active (bitmask) */
1547c478bdstevel@tonic-gate	int		cpu_base_spl;	/* priority for highest rupt active */
1567c478bdstevel@tonic-gate	/*
1577c478bdstevel@tonic-gate	 * Statistics.
1587c478bdstevel@tonic-gate	 */
1597c478bdstevel@tonic-gate	cpu_stats_t	cpu_stats;		/* per-CPU statistics */
1607c478bdstevel@tonic-gate	struct kstat	*cpu_info_kstat;	/* kstat for cpu info */
1627c478bdstevel@tonic-gate	uintptr_t	cpu_profile_pc;	/* kernel PC in profile interrupt */
1637c478bdstevel@tonic-gate	uintptr_t	cpu_profile_upc; /* user PC in profile interrupt */
1647c478bdstevel@tonic-gate	uintptr_t	cpu_profile_pil; /* PIL when profile interrupted */
1667c478bdstevel@tonic-gate	ftrace_data_t	cpu_ftrace;		/* per cpu ftrace data */
168d3d5073Rafael Vanoni	clock_t		cpu_deadman_counter;	/* used by deadman() */
1697c478bdstevel@tonic-gate	uint_t		cpu_deadman_countdown;	/* used by deadman() */
1717c478bdstevel@tonic-gate	kmutex_t	cpu_cpc_ctxlock; /* protects context for idle thread */
1727c478bdstevel@tonic-gate	kcpc_ctx_t	*cpu_cpc_ctx;	/* performance counter context */
1747c478bdstevel@tonic-gate	/*
1757c478bdstevel@tonic-gate	 * Configuration information for the processor_info system call.
1767c478bdstevel@tonic-gate	 */
1777c478bdstevel@tonic-gate	processor_info_t cpu_type_info;	/* config info */
1787c478bdstevel@tonic-gate	time_t		cpu_state_begin; /* when CPU entered current state */
1797c478bdstevel@tonic-gate	char		cpu_cpr_flags;	/* CPR related info */
1807c478bdstevel@tonic-gate	struct cyc_cpu	*cpu_cyclic;	/* per cpu cyclic subsystem data */
1817c478bdstevel@tonic-gate	struct squeue_set_s *cpu_squeue_set;	/* per cpu squeue set */
1827c478bdstevel@tonic-gate	struct nvlist	*cpu_props;	/* pool-related properties */
1847c478bdstevel@tonic-gate	krwlock_t	cpu_ft_lock;		/* DTrace: fasttrap lock */
1857c478bdstevel@tonic-gate	uintptr_t	cpu_dtrace_caller;	/* DTrace: caller, if any */
1867c478bdstevel@tonic-gate	hrtime_t	cpu_dtrace_chillmark;	/* DTrace: chill mark time */
1877c478bdstevel@tonic-gate	hrtime_t	cpu_dtrace_chilled;	/* DTrace: total chill time */
1881f9f06cMatthew Ahrens	uint64_t	cpu_dtrace_probes;	/* DTrace: total probes fired */
1891f9f06cMatthew Ahrens	hrtime_t	cpu_dtrace_nsec;	/* DTrace: ns in dtrace_probe */
1901f9f06cMatthew Ahrens
1919102d47esolom	volatile uint16_t cpu_mstate;		/* cpu microstate */
1929102d47esolom	volatile uint16_t cpu_mstate_gen;	/* generation counter */
1939102d47esolom	volatile hrtime_t cpu_mstate_start;	/* cpu microstate start time */
1949102d47esolom	volatile hrtime_t cpu_acct[NCMSTATES];	/* cpu microstate data */
195eda8946esolom	hrtime_t	cpu_intracct[NCMSTATES]; /* interrupt mstate data */
1967c478bdstevel@tonic-gate	hrtime_t	cpu_waitrq;		/* cpu run-queue wait time */
1977c478bdstevel@tonic-gate	struct loadavg_s cpu_loadavg;		/* loadavg info for this cpu */
1997c478bdstevel@tonic-gate	char		*cpu_idstr;	/* for printing and debugging */
2007c478bdstevel@tonic-gate	char		*cpu_brandstr;	/* for printing */
2027c478bdstevel@tonic-gate	/*
2037c478bdstevel@tonic-gate	 * Sum of all device interrupt weights that are currently directed at
2047c478bdstevel@tonic-gate	 * this cpu. Cleared at start of interrupt redistribution.
2057c478bdstevel@tonic-gate	 */
2067c478bdstevel@tonic-gate	int32_t		cpu_intr_weight;
207affbd3ckchow	void		*cpu_vm_data;
209ab76139esaxe	struct cpu_physid *cpu_physid;	/* physical associations */
211cf74e62mh	uint64_t	cpu_curr_clock;		/* current clock freq in Hz */
212cf74e62mh	char		*cpu_supp_freqs;	/* supported freqs in Hz */
214b9e93c1Jonathan Haslam	uintptr_t	cpu_cpcprofile_pc;	/* kernel PC in cpc interrupt */
215b9e93c1Jonathan Haslam	uintptr_t	cpu_cpcprofile_upc;	/* user PC in cpc interrupt */
216b9e93c1Jonathan Haslam
2177c478bdstevel@tonic-gate	/*
2183aedfe0mishra	 * Interrupt load factor used by dispatcher & softcall
2193aedfe0mishra	 */
2203aedfe0mishra	hrtime_t	cpu_intrlast;   /* total interrupt time (nsec) */
2213aedfe0mishra	int		cpu_intrload;   /* interrupt load factor (0-99%) */
2236890d02Eric Saxe	uint_t		cpu_rotor;	/* for cheap pseudo-random numbers */
2246890d02Eric Saxe
225b885580Alexander Kolbasov	struct cu_cpu_info	*cpu_cu_info;	/* capacity & util. info */
226b885580Alexander Kolbasov
227b885580Alexander Kolbasov	/*
228b885580Alexander Kolbasov	 * cpu_generation is updated whenever CPU goes on-line or off-line.
229b885580Alexander Kolbasov	 * Updates to cpu_generation are protected by cpu_lock.
230b885580Alexander Kolbasov	 *
231b885580Alexander Kolbasov	 * See CPU_NEW_GENERATION() macro below.
232b885580Alexander Kolbasov	 */
233b885580Alexander Kolbasov	volatile uint_t		cpu_generation;	/* tracking on/off-line */
234b885580Alexander Kolbasov
2353aedfe0mishra	/*
236e824d57johnlev	 * New members must be added /before/ this member, as the CTF tools
237e824d57johnlev	 * rely on this being the last field before cpu_m, so they can
238e824d57johnlev	 * correctly calculate the offset when synthetically adding the cpu_m
239e824d57johnlev	 * member in objects that do not have it.  This fixup is required for
240e824d57johnlev	 * uniquification to work correctly.
2417c478bdstevel@tonic-gate	 */
242e824d57johnlev	uintptr_t	cpu_m_pad;
244e824d57johnlev#if (defined(_KERNEL) || defined(_KMEMUSER)) && defined(_MACHDEP)
2457c478bdstevel@tonic-gate	struct machcpu	cpu_m;		/* per architecture info */
2477c478bdstevel@tonic-gate} cpu_t;
2507c478bdstevel@tonic-gate * The cpu_core structure consists of per-CPU state available in any context.
2517c478bdstevel@tonic-gate * On some architectures, this may mean that the page(s) containing the
2527c478bdstevel@tonic-gate * NCPU-sized array of cpu_core structures must be locked in the TLB -- it
2537c478bdstevel@tonic-gate * is up to the platform to assure that this is performed properly.  Note that
2547c478bdstevel@tonic-gate * the structure is sized to avoid false sharing.
2557c478bdstevel@tonic-gate */
256b9e93c1Jonathan Haslam#define	CPUC_SIZE		(sizeof (uint16_t) + sizeof (uint8_t) + \
257b9e93c1Jonathan Haslam				sizeof (uintptr_t) + sizeof (kmutex_t))
2587c478bdstevel@tonic-gate#define	CPUC_PADSIZE		CPU_CACHE_COHERENCE_SIZE - CPUC_SIZE
2607c478bdstevel@tonic-gatetypedef struct cpu_core {
2617c478bdstevel@tonic-gate	uint16_t	cpuc_dtrace_flags;	/* DTrace flags */
262b9e93c1Jonathan Haslam	uint8_t		cpuc_dcpc_intr_state;	/* DCPC provider intr state */
2637c478bdstevel@tonic-gate	uint8_t		cpuc_pad[CPUC_PADSIZE];	/* padding */
2647c478bdstevel@tonic-gate	uintptr_t	cpuc_dtrace_illval;	/* DTrace illegal value */
2657c478bdstevel@tonic-gate	kmutex_t	cpuc_pid_lock;		/* DTrace pid provider lock */
2667c478bdstevel@tonic-gate} cpu_core_t;
2687c478bdstevel@tonic-gate#ifdef _KERNEL
2697c478bdstevel@tonic-gateextern cpu_core_t cpu_core[];
2707c478bdstevel@tonic-gate#endif /* _KERNEL */
2737c478bdstevel@tonic-gate * CPU_ON_INTR() macro. Returns non-zero if currently on interrupt stack.
2747c478bdstevel@tonic-gate * Note that this isn't a test for a high PIL.  For example, cpu_intr_actv
2757c478bdstevel@tonic-gate * does not get updated when we go through sys_trap from TL>0 at high PIL.
2767c478bdstevel@tonic-gate * getpil() should be used instead to check for PIL levels.
2777c478bdstevel@tonic-gate */
2787c478bdstevel@tonic-gate#define	CPU_ON_INTR(cpup) ((cpup)->cpu_intr_actv >> (LOCK_LEVEL + 1))
2806890d02Eric Saxe/*
281e8bb33dColin Yi * Check to see if an interrupt thread might be active at a given ipl.
282e8bb33dColin Yi * If so return true.
283e8bb33dColin Yi * We must be conservative--it is ok to give a false yes, but a false no
284e8bb33dColin Yi * will cause disaster.  (But if the situation changes after we check it is
285e8bb33dColin Yi * ok--the caller is trying to ensure that an interrupt routine has been
286e8bb33dColin Yi * exited).
287e8bb33dColin Yi * This is used when trying to remove an interrupt handler from an autovector
288e8bb33dColin Yi * list in avintr.c.
289e8bb33dColin Yi */
290e8bb33dColin Yi#define	INTR_ACTIVE(cpup, level)	\
29189574a1Patrick Mooney	((level) <= LOCK_LEVEL ?	\
292e8bb33dColin Yi	((cpup)->cpu_intr_actv & (1 << (level))) : (CPU_ON_INTR(cpup)))
293e8bb33dColin Yi
294e8bb33dColin Yi/*
2956890d02Eric Saxe * CPU_PSEUDO_RANDOM() returns a per CPU value that changes each time one
2966890d02Eric Saxe * looks at it. It's meant as a cheap mechanism to be incorporated in routines
2976890d02Eric Saxe * wanting to avoid biasing, but where true randomness isn't needed (just
2986890d02Eric Saxe * something that changes).
2996890d02Eric Saxe */
3006890d02Eric Saxe#define	CPU_PSEUDO_RANDOM() (CPU->cpu_rotor++)
3016890d02Eric Saxe
3023c4c592Gordon Ross#if defined(_KERNEL) || defined(_KMEMUSER) || defined(_BOOT)
3047c478bdstevel@tonic-gate#define	INTR_STACK_SIZE	MAX(DEFAULTSTKSZ, PAGESIZE)
3067c478bdstevel@tonic-gate/* MEMBERS PROTECTED BY "atomicity": cpu_flags */
3097c478bdstevel@tonic-gate * Flags in the CPU structure.
3107c478bdstevel@tonic-gate *
3117c478bdstevel@tonic-gate * These are protected by cpu_lock (except during creation).
3127c478bdstevel@tonic-gate *
3137c478bdstevel@tonic-gate * Offlined-CPUs have three stages of being offline:
3147c478bdstevel@tonic-gate *
3157c478bdstevel@tonic-gate * CPU_ENABLE indicates that the CPU is participating in I/O interrupts
3167c478bdstevel@tonic-gate * that can be directed at a number of different CPUs.  If CPU_ENABLE
3177c478bdstevel@tonic-gate * is off, the CPU will not be given interrupts that can be sent elsewhere,
3187c478bdstevel@tonic-gate * but will still get interrupts from devices associated with that CPU only,
3197c478bdstevel@tonic-gate * and from other CPUs.
3207c478bdstevel@tonic-gate *
3217c478bdstevel@tonic-gate * CPU_OFFLINE indicates that the dispatcher should not allow any threads
3227c478bdstevel@tonic-gate * other than interrupt threads to run on that CPU.  A CPU will not have
3237c478bdstevel@tonic-gate * CPU_OFFLINE set if there are any bound threads (besides interrupts).
3247c478bdstevel@tonic-gate *
3257c478bdstevel@tonic-gate * CPU_QUIESCED is set if p_offline was able to completely turn idle the
3267c478bdstevel@tonic-gate * CPU and it will not have to run interrupt threads.  In this case it'll
3277c478bdstevel@tonic-gate * stay in the idle loop until CPU_QUIESCED is turned off.
3287c478bdstevel@tonic-gate *
3297c478bdstevel@tonic-gate * CPU_FROZEN is used only by CPR to mark CPUs that have been successfully
3307c478bdstevel@tonic-gate * suspended (in the suspend path), or have yet to be resumed (in the resume
3317c478bdstevel@tonic-gate * case).
3327c478bdstevel@tonic-gate *
333c3377eeJohn Levon * CPU_DISABLED is used for disabling SMT. It is similar to CPU_OFFLINE, but
334c3377eeJohn Levon * cannot be onlined without being forced.
335c3377eeJohn Levon *
3367c478bdstevel@tonic-gate * On some platforms CPUs can be individually powered off.
3377c478bdstevel@tonic-gate * The following flags are set for powered off CPUs: CPU_QUIESCED,
3387c478bdstevel@tonic-gate * CPU_OFFLINE, and CPU_POWEROFF.  The following flags are cleared:
3407c478bdstevel@tonic-gate */
3417c478bdstevel@tonic-gate#define	CPU_RUNNING	0x001		/* CPU running */
3427c478bdstevel@tonic-gate#define	CPU_READY	0x002		/* CPU ready for cross-calls */
3437c478bdstevel@tonic-gate#define	CPU_QUIESCED	0x004		/* CPU will stay in idle */
3447c478bdstevel@tonic-gate#define	CPU_EXISTS	0x008		/* CPU is configured */
3457c478bdstevel@tonic-gate#define	CPU_ENABLE	0x010		/* CPU enabled for interrupts */
3467c478bdstevel@tonic-gate#define	CPU_OFFLINE	0x020		/* CPU offline via p_online */
3477c478bdstevel@tonic-gate#define	CPU_POWEROFF	0x040		/* CPU is powered off */
3487c478bdstevel@tonic-gate#define	CPU_FROZEN	0x080		/* CPU is frozen via CPR suspend */
3497c478bdstevel@tonic-gate#define	CPU_SPARE	0x100		/* CPU offline available for use */
3507c478bdstevel@tonic-gate#define	CPU_FAULTED	0x200		/* CPU offline diagnosed faulty */
351c3377eeJohn Levon#define	CPU_DISABLED	0x400		/* CPU explicitly disabled (HT) */
3537c478bdstevel@tonic-gate#define	CPU_ACTIVE(cpu)	(((cpu)->cpu_flags & CPU_OFFLINE) == 0)
3567c478bdstevel@tonic-gate * Flags for cpu_offline(), cpu_faulted(), and cpu_spare().
3577c478bdstevel@tonic-gate */
3587c478bdstevel@tonic-gate#define	CPU_FORCED	0x0001		/* Force CPU offline */
3617c478bdstevel@tonic-gate * DTrace flags.
3627c478bdstevel@tonic-gate */
3637c478bdstevel@tonic-gate#define	CPU_DTRACE_NOFAULT	0x0001	/* Don't fault */
3647c478bdstevel@tonic-gate#define	CPU_DTRACE_DROP		0x0002	/* Drop this ECB */
3657c478bdstevel@tonic-gate#define	CPU_DTRACE_BADADDR	0x0004	/* DTrace fault: bad address */
3667c478bdstevel@tonic-gate#define	CPU_DTRACE_BADALIGN	0x0008	/* DTrace fault: bad alignment */
3677c478bdstevel@tonic-gate#define	CPU_DTRACE_DIVZERO	0x0010	/* DTrace fault: divide by zero */
3687c478bdstevel@tonic-gate#define	CPU_DTRACE_ILLOP	0x0020	/* DTrace fault: illegal operation */
3697c478bdstevel@tonic-gate#define	CPU_DTRACE_NOSCRATCH	0x0040	/* DTrace fault: out of scratch */
3707c478bdstevel@tonic-gate#define	CPU_DTRACE_KPRIV	0x0080	/* DTrace fault: bad kernel access */
3717c478bdstevel@tonic-gate#define	CPU_DTRACE_UPRIV	0x0100	/* DTrace fault: bad user access */
3727c478bdstevel@tonic-gate#define	CPU_DTRACE_TUPOFLOW	0x0200	/* DTrace fault: tuple stack overflow */
3737c478bdstevel@tonic-gate#if defined(__sparc)
3747c478bdstevel@tonic-gate#define	CPU_DTRACE_FAKERESTORE	0x0400	/* pid provider hint to getreg */
3767c478bdstevel@tonic-gate#define	CPU_DTRACE_ENTRY	0x0800	/* pid provider hint to ustack() */
377b8fac8ejhaslam#define	CPU_DTRACE_BADSTACK	0x1000	/* DTrace fault: bad stack */
3797c478bdstevel@tonic-gate#define	CPU_DTRACE_FAULT	(CPU_DTRACE_BADADDR | CPU_DTRACE_BADALIGN | \
3807c478bdstevel@tonic-gate				CPU_DTRACE_DIVZERO | CPU_DTRACE_ILLOP | \
3817c478bdstevel@tonic-gate				CPU_DTRACE_NOSCRATCH | CPU_DTRACE_KPRIV | \
382b8fac8ejhaslam				CPU_DTRACE_UPRIV | CPU_DTRACE_TUPOFLOW | \
383b8fac8ejhaslam				CPU_DTRACE_BADSTACK)
3847c478bdstevel@tonic-gate#define	CPU_DTRACE_ERROR	(CPU_DTRACE_FAULT | CPU_DTRACE_DROP)
3877c478bdstevel@tonic-gate * Dispatcher flags
3887c478bdstevel@tonic-gate * These flags must be changed only by the current CPU.
3897c478bdstevel@tonic-gate */
3907c478bdstevel@tonic-gate#define	CPU_DISP_DONTSTEAL	0x01	/* CPU undergoing context swtch */
3917c478bdstevel@tonic-gate#define	CPU_DISP_HALTED		0x02	/* CPU halted waiting for interrupt */
3947c478bdstevel@tonic-gate * Macros for manipulating sets of CPUs as a bitmap.  Note that this
3957c478bdstevel@tonic-gate * bitmap may vary in size depending on the maximum CPU id a specific
3967c478bdstevel@tonic-gate * platform supports.  This may be different than the number of CPUs
3977c478bdstevel@tonic-gate * the platform supports, since CPU ids can be sparse.  We define two
3987c478bdstevel@tonic-gate * sets of macros; one for platforms where the maximum CPU id is less
3997c478bdstevel@tonic-gate * than the number of bits in a single word (32 in a 32-bit kernel,
4007c478bdstevel@tonic-gate * 64 in a 64-bit kernel), and one for platforms that require bitmaps
4017c478bdstevel@tonic-gate * of more than one word.
4027c478bdstevel@tonic-gate */
4047c478bdstevel@tonic-gate#define	CPUSET_WORDS	BT_BITOUL(NCPU)
4057c478bdstevel@tonic-gate#define	CPUSET_NOTINSET	((uint_t)-1)
40789574a1Patrick Mooney#if defined(_MACHDEP)
40889574a1Patrick Mooneystruct cpuset {
4097c478bdstevel@tonic-gate	ulong_t	cpub[CPUSET_WORDS];
41089574a1Patrick Mooney};
41189574a1Patrick Mooney#else
41289574a1Patrick Mooneystruct cpuset;
41389574a1Patrick Mooney#endif
41489574a1Patrick Mooney
41589574a1Patrick Mooneytypedef struct cpuset cpuset_t;
41689574a1Patrick Mooney
41789574a1Patrick Mooneyextern cpuset_t	*cpuset_alloc(int);
41889574a1Patrick Mooneyextern void	cpuset_free(cpuset_t *);
42189574a1Patrick Mooney * Functions for manipulating cpusets.  These were previously considered
42289574a1Patrick Mooney * private when some cpuset_t handling was performed in the CPUSET_* macros.
42389574a1Patrick Mooney * They are now acceptable to use in non-_MACHDEP code.
42489574a1Patrick Mooney */
42589574a1Patrick Mooneyextern void	cpuset_all(cpuset_t *);
42689574a1Patrick Mooneyextern void	cpuset_all_but(cpuset_t *, const uint_t);
42789574a1Patrick Mooneyextern int	cpuset_isnull(const cpuset_t *);
42889574a1Patrick Mooneyextern int	cpuset_isequal(const cpuset_t *, const cpuset_t *);
42989574a1Patrick Mooneyextern void	cpuset_only(cpuset_t *, const uint_t);
43089574a1Patrick Mooneyextern long	cpu_in_set(const cpuset_t *, const uint_t);
43189574a1Patrick Mooneyextern void	cpuset_add(cpuset_t *, const uint_t);
43289574a1Patrick Mooneyextern void	cpuset_del(cpuset_t *, const uint_t);
43389574a1Patrick Mooneyextern uint_t	cpuset_find(const cpuset_t *);
43489574a1Patrick Mooneyextern void	cpuset_bounds(const cpuset_t *, uint_t *, uint_t *);
43589574a1Patrick Mooneyextern void	cpuset_atomic_del(cpuset_t *, const uint_t);
43689574a1Patrick Mooneyextern void	cpuset_atomic_add(cpuset_t *, const uint_t);
43789574a1Patrick Mooneyextern long	cpuset_atomic_xadd(cpuset_t *, const uint_t);
43889574a1Patrick Mooneyextern long	cpuset_atomic_xdel(cpuset_t *, const uint_t);
43989574a1Patrick Mooneyextern void	cpuset_or(cpuset_t *, cpuset_t *);
44089574a1Patrick Mooneyextern void	cpuset_xor(cpuset_t *, cpuset_t *);
44189574a1Patrick Mooneyextern void	cpuset_and(cpuset_t *, cpuset_t *);
44289574a1Patrick Mooneyextern void	cpuset_zero(cpuset_t *);
44389574a1Patrick Mooney
44489574a1Patrick Mooney
44589574a1Patrick Mooney#if defined(_MACHDEP)
44689574a1Patrick Mooney
44789574a1Patrick Mooney/*
44889574a1Patrick Mooney * Prior to the cpuset_t restructuring, the CPUSET_* macros contained
44989574a1Patrick Mooney * significant logic, rather than directly invoking the backend functions.
45089574a1Patrick Mooney * They are maintained here so that existing _MACHDEP code can use them.
4517c478bdstevel@tonic-gate */
4537c478bdstevel@tonic-gate#define	CPUSET_ALL(set)			cpuset_all(&(set))
4547c478bdstevel@tonic-gate#define	CPUSET_ALL_BUT(set, cpu)	cpuset_all_but(&(set), cpu)
4557c478bdstevel@tonic-gate#define	CPUSET_ONLY(set, cpu)		cpuset_only(&(set), cpu)
45689574a1Patrick Mooney#define	CPU_IN_SET(set, cpu)		cpu_in_set(&(set), cpu)
45789574a1Patrick Mooney#define	CPUSET_ADD(set, cpu)		cpuset_add(&(set), cpu)
45889574a1Patrick Mooney#define	CPUSET_DEL(set, cpu)		cpuset_del(&(set), cpu)
4597c478bdstevel@tonic-gate#define	CPUSET_ISNULL(set)		cpuset_isnull(&(set))
46089574a1Patrick Mooney#define	CPUSET_ISEQUAL(set1, set2)	cpuset_isequal(&(set1), &(set2))
4637c478bdstevel@tonic-gate * Find one CPU in the cpuset.
4647c478bdstevel@tonic-gate * Sets "cpu" to the id of the found CPU, or CPUSET_NOTINSET if no cpu
4657c478bdstevel@tonic-gate * could be found. (i.e. empty set)
4667c478bdstevel@tonic-gate */
4677c478bdstevel@tonic-gate#define	CPUSET_FIND(set, cpu)		{		\
4687c478bdstevel@tonic-gate	cpu = cpuset_find(&(set));			\
4720042319ha * Determine the smallest and largest CPU id in the set. Returns
4730042319ha * CPUSET_NOTINSET in smallest and largest when set is empty.
4740042319ha */
4750042319ha#define	CPUSET_BOUNDS(set, smallest, largest)	{		\
4760042319ha	cpuset_bounds(&(set), &(smallest), &(largest));		\
4807c478bdstevel@tonic-gate * Atomic cpuset operations
4817c478bdstevel@tonic-gate * These are safe to use for concurrent cpuset manipulations.
4827c478bdstevel@tonic-gate * "xdel" and "xadd" are exclusive operations, that set "result" to "0"
4837c478bdstevel@tonic-gate * if the add or del was successful, or "-1" if not successful.
4847c478bdstevel@tonic-gate * (e.g. attempting to add a cpu to a cpuset that's already there, or
4857c478bdstevel@tonic-gate * deleting a cpu that's not in the cpuset)
4867c478bdstevel@tonic-gate */
48889574a1Patrick Mooney#define	CPUSET_ATOMIC_DEL(set, cpu)	cpuset_atomic_del(&(set), cpu)
48989574a1Patrick Mooney#define	CPUSET_ATOMIC_ADD(set, cpu)	cpuset_atomic_add(&(set), cpu)
49189574a1Patrick Mooney#define	CPUSET_ATOMIC_XADD(set, cpu, result)	\
49289574a1Patrick Mooney	(result) = cpuset_atomic_xadd(&(set), cpu)
49489574a1Patrick Mooney#define	CPUSET_ATOMIC_XDEL(set, cpu, result)	\
49589574a1Patrick Mooney	(result) = cpuset_atomic_xdel(&(set), cpu)
49789574a1Patrick Mooney#define	CPUSET_OR(set1, set2)	cpuset_or(&(set1), &(set2))
49989574a1Patrick Mooney#define	CPUSET_XOR(set1, set2)	cpuset_xor(&(set1), &(set2))
50189574a1Patrick Mooney#define	CPUSET_AND(set1, set2)	cpuset_and(&(set1), &(set2))
50389574a1Patrick Mooney#define	CPUSET_ZERO(set)	cpuset_zero(&(set))
50589574a1Patrick Mooney#endif /* defined(_MACHDEP) */
5087c478bdstevel@tonic-gateextern cpuset_t cpu_seqid_inuse;
5107c478bdstevel@tonic-gateextern struct cpu	*cpu[];		/* indexed by CPU number */
5116890d02Eric Saxeextern struct cpu	**cpu_seq;	/* indexed by sequential CPU id */
5127c478bdstevel@tonic-gateextern cpu_t		*cpu_list;	/* list of CPUs */
5132850d85mvextern cpu_t		*cpu_active;	/* list of active CPUs */
51489574a1Patrick Mooneyextern cpuset_t		cpu_active_set;	/* cached set of active CPUs */
5157c478bdstevel@tonic-gateextern int		ncpus;		/* number of CPUs present */
5167c478bdstevel@tonic-gateextern int		ncpus_online;	/* number of CPUs not quiesced */
517c3377eeJohn Levonextern int		ncpus_intr_enabled; /* nr of CPUs taking I/O intrs */
5187c478bdstevel@tonic-gateextern int		max_ncpus;	/* max present before ncpus is known */
5197c478bdstevel@tonic-gateextern int		boot_max_ncpus;	/* like max_ncpus but for real */
52006fb6a3dvextern int		boot_ncpus;	/* # cpus present @ boot */
5217c478bdstevel@tonic-gateextern processorid_t	max_cpuid;	/* maximum CPU number */
5227c478bdstevel@tonic-gateextern struct cpu	*cpu_inmotion;	/* offline or partition move target */
523c97ad5cakolbextern cpu_t		*clock_cpu_list;
524b52a336Pavel Tatashinextern processorid_t	max_cpu_seqid_ever;	/* maximum seqid ever given */
5267c478bdstevel@tonic-gate#if defined(__i386) || defined(__amd64)
5277c478bdstevel@tonic-gateextern struct cpu *curcpup(void);
5287c478bdstevel@tonic-gate#define	CPU		(curcpup())	/* Pointer to current CPU */
5307c478bdstevel@tonic-gate#define	CPU		(curthread->t_cpu)	/* Pointer to current CPU */
534455e370John Levon * CPU_CURRENT indicates to thread_affinity_set() to use whatever curthread's
535455e370John Levon * current CPU is; holding cpu_lock is not required.
5367c478bdstevel@tonic-gate */
5377c478bdstevel@tonic-gate#define	CPU_CURRENT	-3
540455e370John Levon * CPU_BEST can be used by thread_affinity_set() callers to set affinity to a
541455e370John Levon * good CPU (in particular, an ht_acquire()-friendly choice); holding cpu_lock
542455e370John Levon * is not required.
543455e370John Levon */
544455e370John Levon#define	CPU_BEST	-4
545455e370John Levon
546455e370John Levon/*
5477c478bdstevel@tonic-gate * Per-CPU statistics
5487c478bdstevel@tonic-gate *
5497c478bdstevel@tonic-gate * cpu_stats_t contains numerous system and VM-related statistics, in the form
5507c478bdstevel@tonic-gate * of gauges or monotonically-increasing event occurrence counts.
5517c478bdstevel@tonic-gate */
5537c478bdstevel@tonic-gate#define	CPU_STATS_ENTER_K()	kpreempt_disable()
5547c478bdstevel@tonic-gate#define	CPU_STATS_EXIT_K()	kpreempt_enable()
5567c478bdstevel@tonic-gate#define	CPU_STATS_ADD_K(class, stat, amount) \
5577c478bdstevel@tonic-gate	{	kpreempt_disable(); /* keep from switching CPUs */\
5587c478bdstevel@tonic-gate		CPU_STATS_ADDQ(CPU, class, stat, amount); \
5597c478bdstevel@tonic-gate		kpreempt_enable(); \
5607c478bdstevel@tonic-gate	}
5627c478bdstevel@tonic-gate#define	CPU_STATS_ADDQ(cp, class, stat, amount)	{			\
5637c478bdstevel@tonic-gate	extern void __dtrace_probe___cpu_##class##info_##stat(uint_t,	\
5647c478bdstevel@tonic-gate	    uint64_t *, cpu_t *);					\
5657c478bdstevel@tonic-gate	uint64_t *stataddr = &((cp)->cpu_stats.class.stat);		\
5667c478bdstevel@tonic-gate	__dtrace_probe___cpu_##class##info_##stat((amount),		\
5677c478bdstevel@tonic-gate	    stataddr, cp);						\
5687c478bdstevel@tonic-gate	*(stataddr) += (amount);					\
5717c478bdstevel@tonic-gate#define	CPU_STATS(cp, stat)                                       \
5727c478bdstevel@tonic-gate	((cp)->cpu_stats.stat)
574b885580Alexander Kolbasov/*
575b885580Alexander Kolbasov * Increment CPU generation value.
576b885580Alexander Kolbasov * This macro should be called whenever CPU goes on-line or off-line.
577b885580Alexander Kolbasov * Updates to cpu_generation should be protected by cpu_lock.
578b885580Alexander Kolbasov */
579b885580Alexander Kolbasov#define	CPU_NEW_GENERATION(cp)	((cp)->cpu_generation++)
580b885580Alexander Kolbasov
58189574a1Patrick Mooney#endif /* defined(_KERNEL) || defined(_KMEMUSER) */
58289574a1Patrick Mooney
58389574a1Patrick Mooney#define	CPU_CPR_OFFLINE		0x0
58489574a1Patrick Mooney#define	CPU_CPR_ONLINE		0x1
58589574a1Patrick Mooney#define	CPU_CPR_IS_OFFLINE(cpu)	(((cpu)->cpu_cpr_flags & CPU_CPR_ONLINE) == 0)
58689574a1Patrick Mooney#define	CPU_CPR_IS_ONLINE(cpu)	((cpu)->cpu_cpr_flags & CPU_CPR_ONLINE)
58789574a1Patrick Mooney#define	CPU_SET_CPR_FLAGS(cpu, flag)	((cpu)->cpu_cpr_flags |= flag)
590f06dce2Andrew Stormont * CPU support routines (not for genassym.c)
5917c478bdstevel@tonic-gate */
592f06dce2Andrew Stormont#if	(defined(_KERNEL) || defined(_FAKE_KERNEL)) && defined(__STDC__)
5947c478bdstevel@tonic-gatestruct zone;
5967c478bdstevel@tonic-gatevoid	cpu_list_init(cpu_t *);
5977c478bdstevel@tonic-gatevoid	cpu_add_unit(cpu_t *);
5987c478bdstevel@tonic-gatevoid	cpu_del_unit(int cpuid);
5997c478bdstevel@tonic-gatevoid	cpu_add_active(cpu_t *);
6007c478bdstevel@tonic-gatevoid	cpu_kstat_init(cpu_t *);
6017c478bdstevel@tonic-gatevoid	cpu_visibility_add(cpu_t *, struct zone *);
6027c478bdstevel@tonic-gatevoid	cpu_visibility_remove(cpu_t *, struct zone *);
6037c478bdstevel@tonic-gatevoid	cpu_visibility_configure(cpu_t *, struct zone *);
6047c478bdstevel@tonic-gatevoid	cpu_visibility_unconfigure(cpu_t *, struct zone *);
6057c478bdstevel@tonic-gatevoid	cpu_visibility_online(cpu_t *, struct zone *);
6067c478bdstevel@tonic-gatevoid	cpu_visibility_offline(cpu_t *, struct zone *);
6077c478bdstevel@tonic-gatevoid	cpu_create_intrstat(cpu_t *);
6087c478bdstevel@tonic-gatevoid	cpu_delete_intrstat(cpu_t *);
6097c478bdstevel@tonic-gateint	cpu_kstat_intrstat_update(kstat_t *, int);
6107c478bdstevel@tonic-gatevoid	cpu_intr_swtch_enter(kthread_t *);
6117c478bdstevel@tonic-gatevoid	cpu_intr_swtch_exit(kthread_t *);
6137c478bdstevel@tonic-gatevoid	mbox_lock_init(void);	 /* initialize cross-call locks */
6147c478bdstevel@tonic-gatevoid	mbox_init(int cpun);	 /* initialize cross-calls */
6157c478bdstevel@tonic-gatevoid	poke_cpu(int cpun);	 /* interrupt another CPU (to preempt) */
618ae115bcmrj * values for safe_list.  Pause state that CPUs are in.
619ae115bcmrj */
620ae115bcmrj#define	PAUSE_IDLE	0		/* normal state */
621ae115bcmrj#define	PAUSE_READY	1		/* paused thread ready to spl */
622ae115bcmrj#define	PAUSE_WAIT	2		/* paused thread is spl-ed high */
623ae115bcmrj#define	PAUSE_DIE	3		/* tell pause thread to leave */
624ae115bcmrj#define	PAUSE_DEAD	4		/* pause thread has left */