xref: /illumos-gate/usr/src/uts/common/sys/cpuvar.h (revision 5ed99ae7)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
52b616c6cSwesolows  * Common Development and Distribution License (the "License").
62b616c6cSwesolows  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
212b616c6cSwesolows 
227c478bd9Sstevel@tonic-gate /*
23b52a336eSPavel Tatashin  * Copyright (c) 1989, 2010, Oracle and/or its affiliates. All rights reserved.
241f9f06cfSMatthew Ahrens  * Copyright (c) 2012 by Delphix. All rights reserved.
256481fd49SIgor Kozhukhov  * Copyright 2014 Igor Kozhukhov <ikozhukhov@gmail.com>.
26f06dce2cSAndrew Stormont  * Copyright 2017 RackTop Systems.
2789574a1fSPatrick Mooney  * Copyright 2019 Joyent, Inc.
28*5ed99ae7SPatrick Mooney  * Copyright 2021 Oxide Computer Company
297c478bd9Sstevel@tonic-gate  */
307c478bd9Sstevel@tonic-gate 
317c478bd9Sstevel@tonic-gate #ifndef _SYS_CPUVAR_H
327c478bd9Sstevel@tonic-gate #define	_SYS_CPUVAR_H
337c478bd9Sstevel@tonic-gate 
347c478bd9Sstevel@tonic-gate #include <sys/thread.h>
357c478bd9Sstevel@tonic-gate #include <sys/sysinfo.h>	/* has cpu_stat_t definition */
367c478bd9Sstevel@tonic-gate #include <sys/disp.h>
377c478bd9Sstevel@tonic-gate #include <sys/processor.h>
38f06dce2cSAndrew Stormont #include <sys/kcpc.h>		/* has kcpc_ctx_t definition */
397c478bd9Sstevel@tonic-gate 
406481fd49SIgor Kozhukhov #include <sys/loadavg.h>
417c478bd9Sstevel@tonic-gate #if (defined(_KERNEL) || defined(_KMEMUSER)) && defined(_MACHDEP)
427c478bd9Sstevel@tonic-gate #include <sys/machcpuvar.h>
437c478bd9Sstevel@tonic-gate #endif
447c478bd9Sstevel@tonic-gate 
457c478bd9Sstevel@tonic-gate #include <sys/types.h>
467c478bd9Sstevel@tonic-gate #include <sys/file.h>
477c478bd9Sstevel@tonic-gate #include <sys/bitmap.h>
487c478bd9Sstevel@tonic-gate #include <sys/rwlock.h>
497c478bd9Sstevel@tonic-gate #include <sys/msacct.h>
502b616c6cSwesolows #if defined(__GNUC__) && defined(_ASM_INLINES) && defined(_KERNEL) && \
512b616c6cSwesolows 	(defined(__i386) || defined(__amd64))
527c478bd9Sstevel@tonic-gate #include <asm/cpuvar.h>
537c478bd9Sstevel@tonic-gate #endif
547c478bd9Sstevel@tonic-gate 
557c478bd9Sstevel@tonic-gate #ifdef	__cplusplus
567c478bd9Sstevel@tonic-gate extern "C" {
577c478bd9Sstevel@tonic-gate #endif
587c478bd9Sstevel@tonic-gate 
597c478bd9Sstevel@tonic-gate struct squeue_set_s;
607c478bd9Sstevel@tonic-gate 
617c478bd9Sstevel@tonic-gate #define	CPU_CACHE_COHERENCE_SIZE	64
627c478bd9Sstevel@tonic-gate 
637c478bd9Sstevel@tonic-gate /*
647c478bd9Sstevel@tonic-gate  * For fast event tracing.
657c478bd9Sstevel@tonic-gate  */
667c478bd9Sstevel@tonic-gate struct ftrace_record;
677c478bd9Sstevel@tonic-gate typedef struct ftrace_data {
687c478bd9Sstevel@tonic-gate 	int			ftd_state;	/* ftrace flags */
694df4bd60Sbs 	kmutex_t		ftd_unused;	/* ftrace buffer lock, unused */
707c478bd9Sstevel@tonic-gate 	struct ftrace_record	*ftd_cur;	/* current record */
717c478bd9Sstevel@tonic-gate 	struct ftrace_record	*ftd_first;	/* first record */
727c478bd9Sstevel@tonic-gate 	struct ftrace_record	*ftd_last;	/* last record */
737c478bd9Sstevel@tonic-gate } ftrace_data_t;
747c478bd9Sstevel@tonic-gate 
757c478bd9Sstevel@tonic-gate struct cyc_cpu;
767c478bd9Sstevel@tonic-gate struct nvlist;
777c478bd9Sstevel@tonic-gate 
787c478bd9Sstevel@tonic-gate /*
797c478bd9Sstevel@tonic-gate  * Per-CPU data.
80e824d57fSjohnlev  *
81e824d57fSjohnlev  * Be careful adding new members: if they are not the same in all modules (e.g.
82e824d57fSjohnlev  * change size depending on a #define), CTF uniquification can fail to work
83e824d57fSjohnlev  * properly.  Furthermore, this is transitive in that it applies recursively to
84e824d57fSjohnlev  * all types pointed to by cpu_t.
857c478bd9Sstevel@tonic-gate  */
867c478bd9Sstevel@tonic-gate typedef struct cpu {
877c478bd9Sstevel@tonic-gate 	processorid_t	cpu_id;			/* CPU number */
887c478bd9Sstevel@tonic-gate 	processorid_t	cpu_seqid;	/* sequential CPU id (0..ncpus-1) */
897c478bd9Sstevel@tonic-gate 	volatile cpu_flag_t cpu_flags;		/* flags indicating CPU state */
907c478bd9Sstevel@tonic-gate 	struct cpu	*cpu_self;		/* pointer to itself */
917c478bd9Sstevel@tonic-gate 	kthread_t	*cpu_thread;		/* current thread */
927c478bd9Sstevel@tonic-gate 	kthread_t	*cpu_idle_thread;	/* idle thread for this CPU */
937c478bd9Sstevel@tonic-gate 	kthread_t	*cpu_pause_thread;	/* pause thread for this CPU */
947c478bd9Sstevel@tonic-gate 	klwp_id_t	cpu_lwp;		/* current lwp (if any) */
957c478bd9Sstevel@tonic-gate 	klwp_id_t	cpu_fpowner;		/* currently loaded fpu owner */
967c478bd9Sstevel@tonic-gate 	struct cpupart	*cpu_part;		/* partition with this CPU */
977c478bd9Sstevel@tonic-gate 	struct lgrp_ld	*cpu_lpl;		/* pointer to this cpu's load */
987c478bd9Sstevel@tonic-gate 	int		cpu_cache_offset;	/* see kmem.c for details */
997c478bd9Sstevel@tonic-gate 
1007c478bd9Sstevel@tonic-gate 	/*
1017c478bd9Sstevel@tonic-gate 	 * Links to other CPUs.  It is safe to walk these lists if
1027c478bd9Sstevel@tonic-gate 	 * one of the following is true:
10389574a1fSPatrick Mooney 	 *	- cpu_lock held
10489574a1fSPatrick Mooney 	 *	- preemption disabled via kpreempt_disable
10589574a1fSPatrick Mooney 	 *	- PIL >= DISP_LEVEL
10689574a1fSPatrick Mooney 	 *	- acting thread is an interrupt thread
10789574a1fSPatrick Mooney 	 *	- all other CPUs are paused
1087c478bd9Sstevel@tonic-gate 	 */
1097c478bd9Sstevel@tonic-gate 	struct cpu	*cpu_next;		/* next existing CPU */
1107c478bd9Sstevel@tonic-gate 	struct cpu	*cpu_prev;		/* prev existing CPU */
1117c478bd9Sstevel@tonic-gate 	struct cpu	*cpu_next_onln;		/* next online (enabled) CPU */
1127c478bd9Sstevel@tonic-gate 	struct cpu	*cpu_prev_onln;		/* prev online (enabled) CPU */
1137c478bd9Sstevel@tonic-gate 	struct cpu	*cpu_next_part;		/* next CPU in partition */
1147c478bd9Sstevel@tonic-gate 	struct cpu	*cpu_prev_part;		/* prev CPU in partition */
1157c478bd9Sstevel@tonic-gate 	struct cpu	*cpu_next_lgrp;		/* next CPU in latency group */
1167c478bd9Sstevel@tonic-gate 	struct cpu	*cpu_prev_lgrp;		/* prev CPU in latency group */
1177c478bd9Sstevel@tonic-gate 	struct cpu	*cpu_next_lpl;		/* next CPU in lgrp partition */
1187c478bd9Sstevel@tonic-gate 	struct cpu	*cpu_prev_lpl;
119fb2f18f8Sesaxe 
120fb2f18f8Sesaxe 	struct cpu_pg	*cpu_pg;		/* cpu's processor groups */
121fb2f18f8Sesaxe 
1227c478bd9Sstevel@tonic-gate 	void		*cpu_reserved[4];	/* reserved for future use */
1237c478bd9Sstevel@tonic-gate 
1247c478bd9Sstevel@tonic-gate 	/*
1257c478bd9Sstevel@tonic-gate 	 * Scheduling variables.
1267c478bd9Sstevel@tonic-gate 	 */
1277c478bd9Sstevel@tonic-gate 	disp_t		*cpu_disp;		/* dispatch queue data */
1287c478bd9Sstevel@tonic-gate 	/*
1297c478bd9Sstevel@tonic-gate 	 * Note that cpu_disp is set before the CPU is added to the system
1307c478bd9Sstevel@tonic-gate 	 * and is never modified.  Hence, no additional locking is needed
1317c478bd9Sstevel@tonic-gate 	 * beyond what's necessary to access the cpu_t structure.
1327c478bd9Sstevel@tonic-gate 	 */
1337c478bd9Sstevel@tonic-gate 	char		cpu_runrun;	/* scheduling flag - set to preempt */
1347c478bd9Sstevel@tonic-gate 	char		cpu_kprunrun;		/* force kernel preemption */
13589574a1fSPatrick Mooney 	pri_t		cpu_chosen_level;	/* priority at which cpu */
1367c478bd9Sstevel@tonic-gate 						/* was chosen for scheduling */
1377c478bd9Sstevel@tonic-gate 	kthread_t	*cpu_dispthread; /* thread selected for dispatch */
1387c478bd9Sstevel@tonic-gate 	disp_lock_t	cpu_thread_lock; /* dispatcher lock on current thread */
1397c478bd9Sstevel@tonic-gate 	uint8_t		cpu_disp_flags;	/* flags used by dispatcher */
1407c478bd9Sstevel@tonic-gate 	/*
1417c478bd9Sstevel@tonic-gate 	 * The following field is updated when ever the cpu_dispthread
1427c478bd9Sstevel@tonic-gate 	 * changes. Also in places, where the current thread(cpu_dispthread)
1437c478bd9Sstevel@tonic-gate 	 * priority changes. This is used in disp_lowpri_cpu()
1447c478bd9Sstevel@tonic-gate 	 */
1457c478bd9Sstevel@tonic-gate 	pri_t		cpu_dispatch_pri; /* priority of cpu_dispthread */
1467c478bd9Sstevel@tonic-gate 	clock_t		cpu_last_swtch;	/* last time switched to new thread */
1477c478bd9Sstevel@tonic-gate 
1487c478bd9Sstevel@tonic-gate 	/*
1497c478bd9Sstevel@tonic-gate 	 * Interrupt data.
1507c478bd9Sstevel@tonic-gate 	 */
1517c478bd9Sstevel@tonic-gate 	caddr_t		cpu_intr_stack;	/* interrupt stack */
1527c478bd9Sstevel@tonic-gate 	kthread_t	*cpu_intr_thread; /* interrupt thread list */
1537c478bd9Sstevel@tonic-gate 	uint_t		cpu_intr_actv;	/* interrupt levels active (bitmask) */
1547c478bd9Sstevel@tonic-gate 	int		cpu_base_spl;	/* priority for highest rupt active */
1557c478bd9Sstevel@tonic-gate 
1567c478bd9Sstevel@tonic-gate 	/*
1577c478bd9Sstevel@tonic-gate 	 * Statistics.
1587c478bd9Sstevel@tonic-gate 	 */
1597c478bd9Sstevel@tonic-gate 	cpu_stats_t	cpu_stats;		/* per-CPU statistics */
1607c478bd9Sstevel@tonic-gate 	struct kstat	*cpu_info_kstat;	/* kstat for cpu info */
1617c478bd9Sstevel@tonic-gate 
1627c478bd9Sstevel@tonic-gate 	uintptr_t	cpu_profile_pc;	/* kernel PC in profile interrupt */
1637c478bd9Sstevel@tonic-gate 	uintptr_t	cpu_profile_upc; /* user PC in profile interrupt */
1647c478bd9Sstevel@tonic-gate 	uintptr_t	cpu_profile_pil; /* PIL when profile interrupted */
1657c478bd9Sstevel@tonic-gate 
1667c478bd9Sstevel@tonic-gate 	ftrace_data_t	cpu_ftrace;		/* per cpu ftrace data */
1677c478bd9Sstevel@tonic-gate 
168d3d50737SRafael Vanoni 	clock_t		cpu_deadman_counter;	/* used by deadman() */
1697c478bd9Sstevel@tonic-gate 	uint_t		cpu_deadman_countdown;	/* used by deadman() */
1707c478bd9Sstevel@tonic-gate 
1717c478bd9Sstevel@tonic-gate 	kmutex_t	cpu_cpc_ctxlock; /* protects context for idle thread */
1727c478bd9Sstevel@tonic-gate 	kcpc_ctx_t	*cpu_cpc_ctx;	/* performance counter context */
1737c478bd9Sstevel@tonic-gate 
1747c478bd9Sstevel@tonic-gate 	/*
1757c478bd9Sstevel@tonic-gate 	 * Configuration information for the processor_info system call.
1767c478bd9Sstevel@tonic-gate 	 */
1777c478bd9Sstevel@tonic-gate 	processor_info_t cpu_type_info;	/* config info */
1787c478bd9Sstevel@tonic-gate 	time_t		cpu_state_begin; /* when CPU entered current state */
1797c478bd9Sstevel@tonic-gate 	char		cpu_cpr_flags;	/* CPR related info */
1807c478bd9Sstevel@tonic-gate 	struct cyc_cpu	*cpu_cyclic;	/* per cpu cyclic subsystem data */
1817c478bd9Sstevel@tonic-gate 	struct squeue_set_s *cpu_squeue_set;	/* per cpu squeue set */
1827c478bd9Sstevel@tonic-gate 	struct nvlist	*cpu_props;	/* pool-related properties */
1837c478bd9Sstevel@tonic-gate 
1847c478bd9Sstevel@tonic-gate 	krwlock_t	cpu_ft_lock;		/* DTrace: fasttrap lock */
1857c478bd9Sstevel@tonic-gate 	uintptr_t	cpu_dtrace_caller;	/* DTrace: caller, if any */
1867c478bd9Sstevel@tonic-gate 	hrtime_t	cpu_dtrace_chillmark;	/* DTrace: chill mark time */
1877c478bd9Sstevel@tonic-gate 	hrtime_t	cpu_dtrace_chilled;	/* DTrace: total chill time */
1881f9f06cfSMatthew Ahrens 	uint64_t	cpu_dtrace_probes;	/* DTrace: total probes fired */
1891f9f06cfSMatthew Ahrens 	hrtime_t	cpu_dtrace_nsec;	/* DTrace: ns in dtrace_probe */
1901f9f06cfSMatthew Ahrens 
1919102d475Sesolom 	volatile uint16_t cpu_mstate;		/* cpu microstate */
1929102d475Sesolom 	volatile uint16_t cpu_mstate_gen;	/* generation counter */
1939102d475Sesolom 	volatile hrtime_t cpu_mstate_start;	/* cpu microstate start time */
1949102d475Sesolom 	volatile hrtime_t cpu_acct[NCMSTATES];	/* cpu microstate data */
195eda89462Sesolom 	hrtime_t	cpu_intracct[NCMSTATES]; /* interrupt mstate data */
1967c478bd9Sstevel@tonic-gate 	hrtime_t	cpu_waitrq;		/* cpu run-queue wait time */
1977c478bd9Sstevel@tonic-gate 	struct loadavg_s cpu_loadavg;		/* loadavg info for this cpu */
1987c478bd9Sstevel@tonic-gate 
1997c478bd9Sstevel@tonic-gate 	char		*cpu_idstr;	/* for printing and debugging */
2007c478bd9Sstevel@tonic-gate 	char		*cpu_brandstr;	/* for printing */
2017c478bd9Sstevel@tonic-gate 
2027c478bd9Sstevel@tonic-gate 	/*
2037c478bd9Sstevel@tonic-gate 	 * Sum of all device interrupt weights that are currently directed at
2047c478bd9Sstevel@tonic-gate 	 * this cpu. Cleared at start of interrupt redistribution.
2057c478bd9Sstevel@tonic-gate 	 */
2067c478bd9Sstevel@tonic-gate 	int32_t		cpu_intr_weight;
207affbd3ccSkchow 	void		*cpu_vm_data;
2087c478bd9Sstevel@tonic-gate 
209ab761399Sesaxe 	struct cpu_physid *cpu_physid;	/* physical associations */
210ab761399Sesaxe 
211cf74e62bSmh 	uint64_t	cpu_curr_clock;		/* current clock freq in Hz */
212cf74e62bSmh 	char		*cpu_supp_freqs;	/* supported freqs in Hz */
213cf74e62bSmh 
214b9e93c10SJonathan Haslam 	uintptr_t	cpu_cpcprofile_pc;	/* kernel PC in cpc interrupt */
215b9e93c10SJonathan Haslam 	uintptr_t	cpu_cpcprofile_upc;	/* user PC in cpc interrupt */
216b9e93c10SJonathan Haslam 
2173aedfe0bSmishra 	/*
2183aedfe0bSmishra 	 * Interrupt load factor used by dispatcher & softcall
2193aedfe0bSmishra 	 */
2203aedfe0bSmishra 	hrtime_t	cpu_intrlast;   /* total interrupt time (nsec) */
2213aedfe0bSmishra 	int		cpu_intrload;   /* interrupt load factor (0-99%) */
2223aedfe0bSmishra 
2236890d023SEric Saxe 	uint_t		cpu_rotor;	/* for cheap pseudo-random numbers */
2246890d023SEric Saxe 
225b885580bSAlexander Kolbasov 	struct cu_cpu_info	*cpu_cu_info;	/* capacity & util. info */
226b885580bSAlexander Kolbasov 
227b885580bSAlexander Kolbasov 	/*
228b885580bSAlexander Kolbasov 	 * cpu_generation is updated whenever CPU goes on-line or off-line.
229b885580bSAlexander Kolbasov 	 * Updates to cpu_generation are protected by cpu_lock.
230b885580bSAlexander Kolbasov 	 *
231b885580bSAlexander Kolbasov 	 * See CPU_NEW_GENERATION() macro below.
232b885580bSAlexander Kolbasov 	 */
233b885580bSAlexander Kolbasov 	volatile uint_t		cpu_generation;	/* tracking on/off-line */
234b885580bSAlexander Kolbasov 
2357c478bd9Sstevel@tonic-gate 	/*
236e824d57fSjohnlev 	 * New members must be added /before/ this member, as the CTF tools
237e824d57fSjohnlev 	 * rely on this being the last field before cpu_m, so they can
238e824d57fSjohnlev 	 * correctly calculate the offset when synthetically adding the cpu_m
239e824d57fSjohnlev 	 * member in objects that do not have it.  This fixup is required for
240e824d57fSjohnlev 	 * uniquification to work correctly.
2417c478bd9Sstevel@tonic-gate 	 */
242e824d57fSjohnlev 	uintptr_t	cpu_m_pad;
243e824d57fSjohnlev 
244e824d57fSjohnlev #if (defined(_KERNEL) || defined(_KMEMUSER)) && defined(_MACHDEP)
2457c478bd9Sstevel@tonic-gate 	struct machcpu	cpu_m;		/* per architecture info */
2467c478bd9Sstevel@tonic-gate #endif
2477c478bd9Sstevel@tonic-gate } cpu_t;
2487c478bd9Sstevel@tonic-gate 
2497c478bd9Sstevel@tonic-gate /*
2507c478bd9Sstevel@tonic-gate  * The cpu_core structure consists of per-CPU state available in any context.
2517c478bd9Sstevel@tonic-gate  * On some architectures, this may mean that the page(s) containing the
2527c478bd9Sstevel@tonic-gate  * NCPU-sized array of cpu_core structures must be locked in the TLB -- it
2537c478bd9Sstevel@tonic-gate  * is up to the platform to assure that this is performed properly.  Note that
2547c478bd9Sstevel@tonic-gate  * the structure is sized to avoid false sharing.
2557c478bd9Sstevel@tonic-gate  */
256b9e93c10SJonathan Haslam #define	CPUC_SIZE		(sizeof (uint16_t) + sizeof (uint8_t) + \
257b9e93c10SJonathan Haslam 				sizeof (uintptr_t) + sizeof (kmutex_t))
2587c478bd9Sstevel@tonic-gate #define	CPUC_PADSIZE		CPU_CACHE_COHERENCE_SIZE - CPUC_SIZE
2597c478bd9Sstevel@tonic-gate 
2607c478bd9Sstevel@tonic-gate typedef struct cpu_core {
2617c478bd9Sstevel@tonic-gate 	uint16_t	cpuc_dtrace_flags;	/* DTrace flags */
262b9e93c10SJonathan Haslam 	uint8_t		cpuc_dcpc_intr_state;	/* DCPC provider intr state */
2637c478bd9Sstevel@tonic-gate 	uint8_t		cpuc_pad[CPUC_PADSIZE];	/* padding */
2647c478bd9Sstevel@tonic-gate 	uintptr_t	cpuc_dtrace_illval;	/* DTrace illegal value */
2657c478bd9Sstevel@tonic-gate 	kmutex_t	cpuc_pid_lock;		/* DTrace pid provider lock */
2667c478bd9Sstevel@tonic-gate } cpu_core_t;
2677c478bd9Sstevel@tonic-gate 
2687c478bd9Sstevel@tonic-gate #ifdef _KERNEL
2697c478bd9Sstevel@tonic-gate extern cpu_core_t cpu_core[];
2707c478bd9Sstevel@tonic-gate #endif /* _KERNEL */
2717c478bd9Sstevel@tonic-gate 
2727c478bd9Sstevel@tonic-gate /*
2737c478bd9Sstevel@tonic-gate  * CPU_ON_INTR() macro. Returns non-zero if currently on interrupt stack.
2747c478bd9Sstevel@tonic-gate  * Note that this isn't a test for a high PIL.  For example, cpu_intr_actv
2757c478bd9Sstevel@tonic-gate  * does not get updated when we go through sys_trap from TL>0 at high PIL.
2767c478bd9Sstevel@tonic-gate  * getpil() should be used instead to check for PIL levels.
2777c478bd9Sstevel@tonic-gate  */
2787c478bd9Sstevel@tonic-gate #define	CPU_ON_INTR(cpup) ((cpup)->cpu_intr_actv >> (LOCK_LEVEL + 1))
2797c478bd9Sstevel@tonic-gate 
280e8bb33d8SColin Yi /*
281e8bb33d8SColin Yi  * Check to see if an interrupt thread might be active at a given ipl.
282e8bb33d8SColin Yi  * If so return true.
283e8bb33d8SColin Yi  * We must be conservative--it is ok to give a false yes, but a false no
284e8bb33d8SColin Yi  * will cause disaster.  (But if the situation changes after we check it is
285e8bb33d8SColin Yi  * ok--the caller is trying to ensure that an interrupt routine has been
286e8bb33d8SColin Yi  * exited).
287e8bb33d8SColin Yi  * This is used when trying to remove an interrupt handler from an autovector
288e8bb33d8SColin Yi  * list in avintr.c.
289e8bb33d8SColin Yi  */
290e8bb33d8SColin Yi #define	INTR_ACTIVE(cpup, level)	\
29189574a1fSPatrick Mooney 	((level) <= LOCK_LEVEL ?	\
292e8bb33d8SColin Yi 	((cpup)->cpu_intr_actv & (1 << (level))) : (CPU_ON_INTR(cpup)))
293e8bb33d8SColin Yi 
2946890d023SEric Saxe /*
2956890d023SEric Saxe  * CPU_PSEUDO_RANDOM() returns a per CPU value that changes each time one
2966890d023SEric Saxe  * looks at it. It's meant as a cheap mechanism to be incorporated in routines
2976890d023SEric Saxe  * wanting to avoid biasing, but where true randomness isn't needed (just
2986890d023SEric Saxe  * something that changes).
2996890d023SEric Saxe  */
3006890d023SEric Saxe #define	CPU_PSEUDO_RANDOM() (CPU->cpu_rotor++)
3016890d023SEric Saxe 
3023c4c5929SGordon Ross #if defined(_KERNEL) || defined(_KMEMUSER) || defined(_BOOT)
3037c478bd9Sstevel@tonic-gate 
3047c478bd9Sstevel@tonic-gate #define	INTR_STACK_SIZE	MAX(DEFAULTSTKSZ, PAGESIZE)
3057c478bd9Sstevel@tonic-gate 
3067c478bd9Sstevel@tonic-gate /* MEMBERS PROTECTED BY "atomicity": cpu_flags */
3077c478bd9Sstevel@tonic-gate 
3087c478bd9Sstevel@tonic-gate /*
3097c478bd9Sstevel@tonic-gate  * Flags in the CPU structure.
3107c478bd9Sstevel@tonic-gate  *
3117c478bd9Sstevel@tonic-gate  * These are protected by cpu_lock (except during creation).
3127c478bd9Sstevel@tonic-gate  *
3137c478bd9Sstevel@tonic-gate  * Offlined-CPUs have three stages of being offline:
3147c478bd9Sstevel@tonic-gate  *
3157c478bd9Sstevel@tonic-gate  * CPU_ENABLE indicates that the CPU is participating in I/O interrupts
3167c478bd9Sstevel@tonic-gate  * that can be directed at a number of different CPUs.  If CPU_ENABLE
3177c478bd9Sstevel@tonic-gate  * is off, the CPU will not be given interrupts that can be sent elsewhere,
3187c478bd9Sstevel@tonic-gate  * but will still get interrupts from devices associated with that CPU only,
3197c478bd9Sstevel@tonic-gate  * and from other CPUs.
3207c478bd9Sstevel@tonic-gate  *
3217c478bd9Sstevel@tonic-gate  * CPU_OFFLINE indicates that the dispatcher should not allow any threads
3227c478bd9Sstevel@tonic-gate  * other than interrupt threads to run on that CPU.  A CPU will not have
3237c478bd9Sstevel@tonic-gate  * CPU_OFFLINE set if there are any bound threads (besides interrupts).
3247c478bd9Sstevel@tonic-gate  *
3257c478bd9Sstevel@tonic-gate  * CPU_QUIESCED is set if p_offline was able to completely turn idle the
3267c478bd9Sstevel@tonic-gate  * CPU and it will not have to run interrupt threads.  In this case it'll
3277c478bd9Sstevel@tonic-gate  * stay in the idle loop until CPU_QUIESCED is turned off.
3287c478bd9Sstevel@tonic-gate  *
3297c478bd9Sstevel@tonic-gate  * CPU_FROZEN is used only by CPR to mark CPUs that have been successfully
3307c478bd9Sstevel@tonic-gate  * suspended (in the suspend path), or have yet to be resumed (in the resume
3317c478bd9Sstevel@tonic-gate  * case).
3327c478bd9Sstevel@tonic-gate  *
333c3377ee9SJohn Levon  * CPU_DISABLED is used for disabling SMT. It is similar to CPU_OFFLINE, but
334c3377ee9SJohn Levon  * cannot be onlined without being forced.
335c3377ee9SJohn Levon  *
3367c478bd9Sstevel@tonic-gate  * On some platforms CPUs can be individually powered off.
3377c478bd9Sstevel@tonic-gate  * The following flags are set for powered off CPUs: CPU_QUIESCED,
3387c478bd9Sstevel@tonic-gate  * CPU_OFFLINE, and CPU_POWEROFF.  The following flags are cleared:
339c3377ee9SJohn Levon  * CPU_RUNNING, CPU_READY, CPU_EXISTS, CPU_DISABLED and CPU_ENABLE.
3407c478bd9Sstevel@tonic-gate  */
3417c478bd9Sstevel@tonic-gate #define	CPU_RUNNING	0x001		/* CPU running */
3427c478bd9Sstevel@tonic-gate #define	CPU_READY	0x002		/* CPU ready for cross-calls */
3437c478bd9Sstevel@tonic-gate #define	CPU_QUIESCED	0x004		/* CPU will stay in idle */
3447c478bd9Sstevel@tonic-gate #define	CPU_EXISTS	0x008		/* CPU is configured */
3457c478bd9Sstevel@tonic-gate #define	CPU_ENABLE	0x010		/* CPU enabled for interrupts */
3467c478bd9Sstevel@tonic-gate #define	CPU_OFFLINE	0x020		/* CPU offline via p_online */
3477c478bd9Sstevel@tonic-gate #define	CPU_POWEROFF	0x040		/* CPU is powered off */
3487c478bd9Sstevel@tonic-gate #define	CPU_FROZEN	0x080		/* CPU is frozen via CPR suspend */
3497c478bd9Sstevel@tonic-gate #define	CPU_SPARE	0x100		/* CPU offline available for use */
3507c478bd9Sstevel@tonic-gate #define	CPU_FAULTED	0x200		/* CPU offline diagnosed faulty */
351c3377ee9SJohn Levon #define	CPU_DISABLED	0x400		/* CPU explicitly disabled (HT) */
352ae115bc7Smrj 
3537c478bd9Sstevel@tonic-gate #define	CPU_ACTIVE(cpu)	(((cpu)->cpu_flags & CPU_OFFLINE) == 0)
3547c478bd9Sstevel@tonic-gate 
3557c478bd9Sstevel@tonic-gate /*
3567c478bd9Sstevel@tonic-gate  * Flags for cpu_offline(), cpu_faulted(), and cpu_spare().
3577c478bd9Sstevel@tonic-gate  */
3587c478bd9Sstevel@tonic-gate #define	CPU_FORCED	0x0001		/* Force CPU offline */
3597c478bd9Sstevel@tonic-gate 
3607c478bd9Sstevel@tonic-gate /*
3617c478bd9Sstevel@tonic-gate  * DTrace flags.
3627c478bd9Sstevel@tonic-gate  */
3637c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_NOFAULT	0x0001	/* Don't fault */
3647c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_DROP		0x0002	/* Drop this ECB */
3657c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_BADADDR	0x0004	/* DTrace fault: bad address */
3667c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_BADALIGN	0x0008	/* DTrace fault: bad alignment */
3677c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_DIVZERO	0x0010	/* DTrace fault: divide by zero */
3687c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_ILLOP	0x0020	/* DTrace fault: illegal operation */
3697c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_NOSCRATCH	0x0040	/* DTrace fault: out of scratch */
3707c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_KPRIV	0x0080	/* DTrace fault: bad kernel access */
3717c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_UPRIV	0x0100	/* DTrace fault: bad user access */
3727c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_TUPOFLOW	0x0200	/* DTrace fault: tuple stack overflow */
3737c478bd9Sstevel@tonic-gate #if defined(__sparc)
3747c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_FAKERESTORE	0x0400	/* pid provider hint to getreg */
3757c478bd9Sstevel@tonic-gate #endif
3767c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_ENTRY	0x0800	/* pid provider hint to ustack() */
377b8fac8e1Sjhaslam #define	CPU_DTRACE_BADSTACK	0x1000	/* DTrace fault: bad stack */
3787c478bd9Sstevel@tonic-gate 
3797c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_FAULT	(CPU_DTRACE_BADADDR | CPU_DTRACE_BADALIGN | \
3807c478bd9Sstevel@tonic-gate 				CPU_DTRACE_DIVZERO | CPU_DTRACE_ILLOP | \
3817c478bd9Sstevel@tonic-gate 				CPU_DTRACE_NOSCRATCH | CPU_DTRACE_KPRIV | \
382b8fac8e1Sjhaslam 				CPU_DTRACE_UPRIV | CPU_DTRACE_TUPOFLOW | \
383b8fac8e1Sjhaslam 				CPU_DTRACE_BADSTACK)
3847c478bd9Sstevel@tonic-gate #define	CPU_DTRACE_ERROR	(CPU_DTRACE_FAULT | CPU_DTRACE_DROP)
3857c478bd9Sstevel@tonic-gate 
3867c478bd9Sstevel@tonic-gate /*
3877c478bd9Sstevel@tonic-gate  * Dispatcher flags
3887c478bd9Sstevel@tonic-gate  * These flags must be changed only by the current CPU.
3897c478bd9Sstevel@tonic-gate  */
3907c478bd9Sstevel@tonic-gate #define	CPU_DISP_DONTSTEAL	0x01	/* CPU undergoing context swtch */
3917c478bd9Sstevel@tonic-gate #define	CPU_DISP_HALTED		0x02	/* CPU halted waiting for interrupt */
3927c478bd9Sstevel@tonic-gate 
3937c478bd9Sstevel@tonic-gate /*
3947c478bd9Sstevel@tonic-gate  * Macros for manipulating sets of CPUs as a bitmap.  Note that this
3957c478bd9Sstevel@tonic-gate  * bitmap may vary in size depending on the maximum CPU id a specific
3967c478bd9Sstevel@tonic-gate  * platform supports.  This may be different than the number of CPUs
3977c478bd9Sstevel@tonic-gate  * the platform supports, since CPU ids can be sparse.  We define two
3987c478bd9Sstevel@tonic-gate  * sets of macros; one for platforms where the maximum CPU id is less
3997c478bd9Sstevel@tonic-gate  * than the number of bits in a single word (32 in a 32-bit kernel,
4007c478bd9Sstevel@tonic-gate  * 64 in a 64-bit kernel), and one for platforms that require bitmaps
4017c478bd9Sstevel@tonic-gate  * of more than one word.
4027c478bd9Sstevel@tonic-gate  */
4037c478bd9Sstevel@tonic-gate 
4047c478bd9Sstevel@tonic-gate #define	CPUSET_WORDS	BT_BITOUL(NCPU)
4057c478bd9Sstevel@tonic-gate #define	CPUSET_NOTINSET	((uint_t)-1)
4067c478bd9Sstevel@tonic-gate 
40789574a1fSPatrick Mooney #if defined(_MACHDEP)
40889574a1fSPatrick Mooney struct cpuset {
4097c478bd9Sstevel@tonic-gate 	ulong_t	cpub[CPUSET_WORDS];
41089574a1fSPatrick Mooney };
41189574a1fSPatrick Mooney #else
41289574a1fSPatrick Mooney struct cpuset;
41389574a1fSPatrick Mooney #endif
41489574a1fSPatrick Mooney 
41589574a1fSPatrick Mooney typedef struct cpuset cpuset_t;
41689574a1fSPatrick Mooney 
41789574a1fSPatrick Mooney extern cpuset_t	*cpuset_alloc(int);
41889574a1fSPatrick Mooney extern void	cpuset_free(cpuset_t *);
4197c478bd9Sstevel@tonic-gate 
4207c478bd9Sstevel@tonic-gate /*
42189574a1fSPatrick Mooney  * Functions for manipulating cpusets.  These were previously considered
42289574a1fSPatrick Mooney  * private when some cpuset_t handling was performed in the CPUSET_* macros.
42389574a1fSPatrick Mooney  * They are now acceptable to use in non-_MACHDEP code.
42489574a1fSPatrick Mooney  */
42589574a1fSPatrick Mooney extern void	cpuset_all(cpuset_t *);
42689574a1fSPatrick Mooney extern void	cpuset_all_but(cpuset_t *, const uint_t);
42789574a1fSPatrick Mooney extern int	cpuset_isnull(const cpuset_t *);
42889574a1fSPatrick Mooney extern int	cpuset_isequal(const cpuset_t *, const cpuset_t *);
42989574a1fSPatrick Mooney extern void	cpuset_only(cpuset_t *, const uint_t);
43089574a1fSPatrick Mooney extern long	cpu_in_set(const cpuset_t *, const uint_t);
43189574a1fSPatrick Mooney extern void	cpuset_add(cpuset_t *, const uint_t);
43289574a1fSPatrick Mooney extern void	cpuset_del(cpuset_t *, const uint_t);
43389574a1fSPatrick Mooney extern uint_t	cpuset_find(const cpuset_t *);
43489574a1fSPatrick Mooney extern void	cpuset_bounds(const cpuset_t *, uint_t *, uint_t *);
43589574a1fSPatrick Mooney extern void	cpuset_atomic_del(cpuset_t *, const uint_t);
43689574a1fSPatrick Mooney extern void	cpuset_atomic_add(cpuset_t *, const uint_t);
43789574a1fSPatrick Mooney extern long	cpuset_atomic_xadd(cpuset_t *, const uint_t);
43889574a1fSPatrick Mooney extern long	cpuset_atomic_xdel(cpuset_t *, const uint_t);
439*5ed99ae7SPatrick Mooney extern void	cpuset_or(cpuset_t *, const cpuset_t *);
440*5ed99ae7SPatrick Mooney extern void	cpuset_xor(cpuset_t *, const cpuset_t *);
441*5ed99ae7SPatrick Mooney extern void	cpuset_and(cpuset_t *, const cpuset_t *);
44289574a1fSPatrick Mooney extern void	cpuset_zero(cpuset_t *);
44389574a1fSPatrick Mooney 
44489574a1fSPatrick Mooney 
44589574a1fSPatrick Mooney #if defined(_MACHDEP)
44689574a1fSPatrick Mooney 
44789574a1fSPatrick Mooney /*
44889574a1fSPatrick Mooney  * Prior to the cpuset_t restructuring, the CPUSET_* macros contained
44989574a1fSPatrick Mooney  * significant logic, rather than directly invoking the backend functions.
45089574a1fSPatrick Mooney  * They are maintained here so that existing _MACHDEP code can use them.
4517c478bd9Sstevel@tonic-gate  */
4527c478bd9Sstevel@tonic-gate 
4537c478bd9Sstevel@tonic-gate #define	CPUSET_ALL(set)			cpuset_all(&(set))
4547c478bd9Sstevel@tonic-gate #define	CPUSET_ALL_BUT(set, cpu)	cpuset_all_but(&(set), cpu)
4557c478bd9Sstevel@tonic-gate #define	CPUSET_ONLY(set, cpu)		cpuset_only(&(set), cpu)
45689574a1fSPatrick Mooney #define	CPU_IN_SET(set, cpu)		cpu_in_set(&(set), cpu)
45789574a1fSPatrick Mooney #define	CPUSET_ADD(set, cpu)		cpuset_add(&(set), cpu)
45889574a1fSPatrick Mooney #define	CPUSET_DEL(set, cpu)		cpuset_del(&(set), cpu)
4597c478bd9Sstevel@tonic-gate #define	CPUSET_ISNULL(set)		cpuset_isnull(&(set))
46089574a1fSPatrick Mooney #define	CPUSET_ISEQUAL(set1, set2)	cpuset_isequal(&(set1), &(set2))
4617c478bd9Sstevel@tonic-gate 
4627c478bd9Sstevel@tonic-gate /*
4637c478bd9Sstevel@tonic-gate  * Find one CPU in the cpuset.
4647c478bd9Sstevel@tonic-gate  * Sets "cpu" to the id of the found CPU, or CPUSET_NOTINSET if no cpu
4657c478bd9Sstevel@tonic-gate  * could be found. (i.e. empty set)
4667c478bd9Sstevel@tonic-gate  */
4677c478bd9Sstevel@tonic-gate #define	CPUSET_FIND(set, cpu)		{		\
4687c478bd9Sstevel@tonic-gate 	cpu = cpuset_find(&(set));			\
4697c478bd9Sstevel@tonic-gate }
4707c478bd9Sstevel@tonic-gate 
47100423197Sha /*
47200423197Sha  * Determine the smallest and largest CPU id in the set. Returns
47300423197Sha  * CPUSET_NOTINSET in smallest and largest when set is empty.
47400423197Sha  */
47500423197Sha #define	CPUSET_BOUNDS(set, smallest, largest)	{		\
47600423197Sha 	cpuset_bounds(&(set), &(smallest), &(largest));		\
47700423197Sha }
47800423197Sha 
4797c478bd9Sstevel@tonic-gate /*
4807c478bd9Sstevel@tonic-gate  * Atomic cpuset operations
4817c478bd9Sstevel@tonic-gate  * These are safe to use for concurrent cpuset manipulations.
4827c478bd9Sstevel@tonic-gate  * "xdel" and "xadd" are exclusive operations, that set "result" to "0"
4837c478bd9Sstevel@tonic-gate  * if the add or del was successful, or "-1" if not successful.
4847c478bd9Sstevel@tonic-gate  * (e.g. attempting to add a cpu to a cpuset that's already there, or
4857c478bd9Sstevel@tonic-gate  * deleting a cpu that's not in the cpuset)
4867c478bd9Sstevel@tonic-gate  */
4877c478bd9Sstevel@tonic-gate 
48889574a1fSPatrick Mooney #define	CPUSET_ATOMIC_DEL(set, cpu)	cpuset_atomic_del(&(set), cpu)
48989574a1fSPatrick Mooney #define	CPUSET_ATOMIC_ADD(set, cpu)	cpuset_atomic_add(&(set), cpu)
4907c478bd9Sstevel@tonic-gate 
49189574a1fSPatrick Mooney #define	CPUSET_ATOMIC_XADD(set, cpu, result)	\
49289574a1fSPatrick Mooney 	(result) = cpuset_atomic_xadd(&(set), cpu)
4937c478bd9Sstevel@tonic-gate 
49489574a1fSPatrick Mooney #define	CPUSET_ATOMIC_XDEL(set, cpu, result)	\
49589574a1fSPatrick Mooney 	(result) = cpuset_atomic_xdel(&(set), cpu)
49600423197Sha 
49789574a1fSPatrick Mooney #define	CPUSET_OR(set1, set2)	cpuset_or(&(set1), &(set2))
4987c478bd9Sstevel@tonic-gate 
49989574a1fSPatrick Mooney #define	CPUSET_XOR(set1, set2)	cpuset_xor(&(set1), &(set2))
5007c478bd9Sstevel@tonic-gate 
50189574a1fSPatrick Mooney #define	CPUSET_AND(set1, set2)	cpuset_and(&(set1), &(set2))
5027c478bd9Sstevel@tonic-gate 
50389574a1fSPatrick Mooney #define	CPUSET_ZERO(set)	cpuset_zero(&(set))
5047c478bd9Sstevel@tonic-gate 
50589574a1fSPatrick Mooney #endif /* defined(_MACHDEP) */
5067c478bd9Sstevel@tonic-gate 
5077c478bd9Sstevel@tonic-gate 
5087c478bd9Sstevel@tonic-gate extern cpuset_t cpu_seqid_inuse;
5097c478bd9Sstevel@tonic-gate 
5107c478bd9Sstevel@tonic-gate extern struct cpu	*cpu[];		/* indexed by CPU number */
5116890d023SEric Saxe extern struct cpu	**cpu_seq;	/* indexed by sequential CPU id */
5127c478bd9Sstevel@tonic-gate extern cpu_t		*cpu_list;	/* list of CPUs */
5132850d85bSmv extern cpu_t		*cpu_active;	/* list of active CPUs */
51489574a1fSPatrick Mooney extern cpuset_t		cpu_active_set;	/* cached set of active CPUs */
5157c478bd9Sstevel@tonic-gate extern int		ncpus;		/* number of CPUs present */
5167c478bd9Sstevel@tonic-gate extern int		ncpus_online;	/* number of CPUs not quiesced */
517c3377ee9SJohn Levon extern int		ncpus_intr_enabled; /* nr of CPUs taking I/O intrs */
5187c478bd9Sstevel@tonic-gate extern int		max_ncpus;	/* max present before ncpus is known */
5197c478bd9Sstevel@tonic-gate extern int		boot_max_ncpus;	/* like max_ncpus but for real */
52006fb6a36Sdv extern int		boot_ncpus;	/* # cpus present @ boot */
5217c478bd9Sstevel@tonic-gate extern processorid_t	max_cpuid;	/* maximum CPU number */
5227c478bd9Sstevel@tonic-gate extern struct cpu	*cpu_inmotion;	/* offline or partition move target */
523c97ad5cdSakolb extern cpu_t		*clock_cpu_list;
524b52a336eSPavel Tatashin extern processorid_t	max_cpu_seqid_ever;	/* maximum seqid ever given */
5257c478bd9Sstevel@tonic-gate 
5267c478bd9Sstevel@tonic-gate #if defined(__i386) || defined(__amd64)
5277c478bd9Sstevel@tonic-gate extern struct cpu *curcpup(void);
5287c478bd9Sstevel@tonic-gate #define	CPU		(curcpup())	/* Pointer to current CPU */
5297c478bd9Sstevel@tonic-gate #else
5307c478bd9Sstevel@tonic-gate #define	CPU		(curthread->t_cpu)	/* Pointer to current CPU */
5317c478bd9Sstevel@tonic-gate #endif
5327c478bd9Sstevel@tonic-gate 
5337c478bd9Sstevel@tonic-gate /*
534455e370cSJohn Levon  * CPU_CURRENT indicates to thread_affinity_set() to use whatever curthread's
535455e370cSJohn Levon  * current CPU is; holding cpu_lock is not required.
5367c478bd9Sstevel@tonic-gate  */
5377c478bd9Sstevel@tonic-gate #define	CPU_CURRENT	-3
5387c478bd9Sstevel@tonic-gate 
539455e370cSJohn Levon /*
540455e370cSJohn Levon  * CPU_BEST can be used by thread_affinity_set() callers to set affinity to a
541455e370cSJohn Levon  * good CPU (in particular, an ht_acquire()-friendly choice); holding cpu_lock
542455e370cSJohn Levon  * is not required.
543455e370cSJohn Levon  */
544455e370cSJohn Levon #define	CPU_BEST	-4
545455e370cSJohn Levon 
5467c478bd9Sstevel@tonic-gate /*
5477c478bd9Sstevel@tonic-gate  * Per-CPU statistics
5487c478bd9Sstevel@tonic-gate  *
5497c478bd9Sstevel@tonic-gate  * cpu_stats_t contains numerous system and VM-related statistics, in the form
5507c478bd9Sstevel@tonic-gate  * of gauges or monotonically-increasing event occurrence counts.
5517c478bd9Sstevel@tonic-gate  */
5527c478bd9Sstevel@tonic-gate 
5537c478bd9Sstevel@tonic-gate #define	CPU_STATS_ENTER_K()	kpreempt_disable()
5547c478bd9Sstevel@tonic-gate #define	CPU_STATS_EXIT_K()	kpreempt_enable()
5557c478bd9Sstevel@tonic-gate 
5567c478bd9Sstevel@tonic-gate #define	CPU_STATS_ADD_K(class, stat, amount) \
5577c478bd9Sstevel@tonic-gate 	{	kpreempt_disable(); /* keep from switching CPUs */\
5587c478bd9Sstevel@tonic-gate 		CPU_STATS_ADDQ(CPU, class, stat, amount); \
5597c478bd9Sstevel@tonic-gate 		kpreempt_enable(); \
5607c478bd9Sstevel@tonic-gate 	}
5617c478bd9Sstevel@tonic-gate 
5627c478bd9Sstevel@tonic-gate #define	CPU_STATS_ADDQ(cp, class, stat, amount)	{			\
5637c478bd9Sstevel@tonic-gate 	extern void __dtrace_probe___cpu_##class##info_##stat(uint_t,	\
5647c478bd9Sstevel@tonic-gate 	    uint64_t *, cpu_t *);					\
5657c478bd9Sstevel@tonic-gate 	uint64_t *stataddr = &((cp)->cpu_stats.class.stat);		\
5667c478bd9Sstevel@tonic-gate 	__dtrace_probe___cpu_##class##info_##stat((amount),		\
5677c478bd9Sstevel@tonic-gate 	    stataddr, cp);						\
5687c478bd9Sstevel@tonic-gate 	*(stataddr) += (amount);					\
5697c478bd9Sstevel@tonic-gate }
5707c478bd9Sstevel@tonic-gate 
5717c478bd9Sstevel@tonic-gate #define	CPU_STATS(cp, stat)                                       \
5727c478bd9Sstevel@tonic-gate 	((cp)->cpu_stats.stat)
5737c478bd9Sstevel@tonic-gate 
574b885580bSAlexander Kolbasov /*
575b885580bSAlexander Kolbasov  * Increment CPU generation value.
576b885580bSAlexander Kolbasov  * This macro should be called whenever CPU goes on-line or off-line.
577b885580bSAlexander Kolbasov  * Updates to cpu_generation should be protected by cpu_lock.
578b885580bSAlexander Kolbasov  */
579b885580bSAlexander Kolbasov #define	CPU_NEW_GENERATION(cp)	((cp)->cpu_generation++)
580b885580bSAlexander Kolbasov 
58189574a1fSPatrick Mooney #endif /* defined(_KERNEL) || defined(_KMEMUSER) */
58289574a1fSPatrick Mooney 
58389574a1fSPatrick Mooney #define	CPU_CPR_OFFLINE		0x0
58489574a1fSPatrick Mooney #define	CPU_CPR_ONLINE		0x1
58589574a1fSPatrick Mooney #define	CPU_CPR_IS_OFFLINE(cpu)	(((cpu)->cpu_cpr_flags & CPU_CPR_ONLINE) == 0)
58689574a1fSPatrick Mooney #define	CPU_CPR_IS_ONLINE(cpu)	((cpu)->cpu_cpr_flags & CPU_CPR_ONLINE)
58789574a1fSPatrick Mooney #define	CPU_SET_CPR_FLAGS(cpu, flag)	((cpu)->cpu_cpr_flags |= flag)
5887c478bd9Sstevel@tonic-gate 
5897c478bd9Sstevel@tonic-gate /*
590f06dce2cSAndrew Stormont  * CPU support routines (not for genassym.c)
5917c478bd9Sstevel@tonic-gate  */
592f06dce2cSAndrew Stormont #if	(defined(_KERNEL) || defined(_FAKE_KERNEL)) && defined(__STDC__)
5937c478bd9Sstevel@tonic-gate 
5947c478bd9Sstevel@tonic-gate struct zone;
5957c478bd9Sstevel@tonic-gate 
5967c478bd9Sstevel@tonic-gate void	cpu_list_init(cpu_t *);
5977c478bd9Sstevel@tonic-gate void	cpu_add_unit(cpu_t *);
5987c478bd9Sstevel@tonic-gate void	cpu_del_unit(int cpuid);
5997c478bd9Sstevel@tonic-gate void	cpu_add_active(cpu_t *);
6007c478bd9Sstevel@tonic-gate void	cpu_kstat_init(cpu_t *);
6017c478bd9Sstevel@tonic-gate void	cpu_visibility_add(cpu_t *, struct zone *);
6027c478bd9Sstevel@tonic-gate void	cpu_visibility_remove(cpu_t *, struct zone *);
6037c478bd9Sstevel@tonic-gate void	cpu_visibility_configure(cpu_t *, struct zone *);
6047c478bd9Sstevel@tonic-gate void	cpu_visibility_unconfigure(cpu_t *, struct zone *);
6057c478bd9Sstevel@tonic-gate void	cpu_visibility_online(cpu_t *, struct zone *);
6067c478bd9Sstevel@tonic-gate void	cpu_visibility_offline(cpu_t *, struct zone *);
6077c478bd9Sstevel@tonic-gate void	cpu_create_intrstat(cpu_t *);
6087c478bd9Sstevel@tonic-gate void	cpu_delete_intrstat(cpu_t *);
6097c478bd9Sstevel@tonic-gate int	cpu_kstat_intrstat_update(kstat_t *, int);
6107c478bd9Sstevel@tonic-gate void	cpu_intr_swtch_enter(kthread_t *);
6117c478bd9Sstevel@tonic-gate void	cpu_intr_swtch_exit(kthread_t *);
6127c478bd9Sstevel@tonic-gate 
6137c478bd9Sstevel@tonic-gate void	mbox_lock_init(void);	 /* initialize cross-call locks */
6147c478bd9Sstevel@tonic-gate void	mbox_init(int cpun);	 /* initialize cross-calls */
6157c478bd9Sstevel@tonic-gate void	poke_cpu(int cpun);	 /* interrupt another CPU (to preempt) */
6167c478bd9Sstevel@tonic-gate 
617ae115bc7Smrj /*
618ae115bc7Smrj  * values for safe_list.  Pause state that CPUs are in.
619ae115bc7Smrj  */
620ae115bc7Smrj #define	PAUSE_IDLE	0		/* normal state */
621ae115bc7Smrj #define	PAUSE_READY	1		/* paused thread ready to spl */
622