1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright (c) 2012 by Delphix. All rights reserved.
24 */
25
26/*
27 * Architecture-independent CPU control functions.
28 */
29
30#include <sys/types.h>
31#include <sys/param.h>
32#include <sys/var.h>
33#include <sys/thread.h>
34#include <sys/cpuvar.h>
35#include <sys/cpu_event.h>
36#include <sys/kstat.h>
37#include <sys/uadmin.h>
38#include <sys/systm.h>
39#include <sys/errno.h>
40#include <sys/cmn_err.h>
41#include <sys/procset.h>
42#include <sys/processor.h>
43#include <sys/debug.h>
44#include <sys/cpupart.h>
45#include <sys/lgrp.h>
46#include <sys/pset.h>
47#include <sys/pghw.h>
48#include <sys/kmem.h>
49#include <sys/kmem_impl.h>	/* to set per-cpu kmem_cache offset */
50#include <sys/atomic.h>
51#include <sys/callb.h>
52#include <sys/vtrace.h>
53#include <sys/cyclic.h>
54#include <sys/bitmap.h>
55#include <sys/nvpair.h>
56#include <sys/pool_pset.h>
57#include <sys/msacct.h>
58#include <sys/time.h>
59#include <sys/archsystm.h>
60#include <sys/sdt.h>
61#if defined(__x86) || defined(__amd64)
62#include <sys/x86_archext.h>
63#endif
64#include <sys/callo.h>
65
66extern int	mp_cpu_start(cpu_t *);
67extern int	mp_cpu_stop(cpu_t *);
68extern int	mp_cpu_poweron(cpu_t *);
69extern int	mp_cpu_poweroff(cpu_t *);
70extern int	mp_cpu_configure(int);
71extern int	mp_cpu_unconfigure(int);
72extern void	mp_cpu_faulted_enter(cpu_t *);
73extern void	mp_cpu_faulted_exit(cpu_t *);
74
75extern int cmp_cpu_to_chip(processorid_t cpuid);
76#ifdef __sparcv9
77extern char *cpu_fru_fmri(cpu_t *cp);
78#endif
79
80static void cpu_add_active_internal(cpu_t *cp);
81static void cpu_remove_active(cpu_t *cp);
82static void cpu_info_kstat_create(cpu_t *cp);
83static void cpu_info_kstat_destroy(cpu_t *cp);
84static void cpu_stats_kstat_create(cpu_t *cp);
85static void cpu_stats_kstat_destroy(cpu_t *cp);
86
87static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw);
88static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw);
89static int cpu_stat_ks_update(kstat_t *ksp, int rw);
90static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t);
91
92/*
93 * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
94 * max_cpu_seqid_ever, and dispatch queue reallocations.  The lock ordering with
95 * respect to related locks is:
96 *
97 *	cpu_lock --> thread_free_lock  --->  p_lock  --->  thread_lock()
98 *
99 * Warning:  Certain sections of code do not use the cpu_lock when
100 * traversing the cpu_list (e.g. mutex_vector_enter(), clock()).  Since
101 * all cpus are paused during modifications to this list, a solution
102 * to protect the list is too either disable kernel preemption while
103 * walking the list, *or* recheck the cpu_next pointer at each
104 * iteration in the loop.  Note that in no cases can any cached
105 * copies of the cpu pointers be kept as they may become invalid.
106 */
107kmutex_t	cpu_lock;
108cpu_t		*cpu_list;		/* list of all CPUs */
109cpu_t		*clock_cpu_list;	/* used by clock to walk CPUs */
110cpu_t		*cpu_active;		/* list of active CPUs */
111static cpuset_t	cpu_available;		/* set of available CPUs */
112cpuset_t	cpu_seqid_inuse;	/* which cpu_seqids are in use */
113
114cpu_t		**cpu_seq;		/* ptrs to CPUs, indexed by seq_id */
115
116/*
117 * max_ncpus keeps the max cpus the system can have. Initially
118 * it's NCPU, but since most archs scan the devtree for cpus
119 * fairly early on during boot, the real max can be known before
120 * ncpus is set (useful for early NCPU based allocations).
121 */
122int max_ncpus = NCPU;
123/*
124 * platforms that set max_ncpus to maxiumum number of cpus that can be
125 * dynamically added will set boot_max_ncpus to the number of cpus found
126 * at device tree scan time during boot.
127 */
128int boot_max_ncpus = -1;
129int boot_ncpus = -1;
130/*
131 * Maximum possible CPU id.  This can never be >= NCPU since NCPU is
132 * used to size arrays that are indexed by CPU id.
133 */
134processorid_t max_cpuid = NCPU - 1;
135
136/*
137 * Maximum cpu_seqid was given. This number can only grow and never shrink. It
138 * can be used to optimize NCPU loops to avoid going through CPUs which were
139 * never on-line.
140 */
141processorid_t max_cpu_seqid_ever = 0;
142
143int ncpus = 1;
144int ncpus_online = 1;
145
146/*
147 * CPU that we're trying to offline.  Protected by cpu_lock.
148 */
149cpu_t *cpu_inmotion;
150
151/*
152 * Can be raised to suppress further weakbinding, which are instead
153 * satisfied by disabling preemption.  Must be raised/lowered under cpu_lock,
154 * while individual thread weakbinding synchronization is done under thread
155 * lock.
156 */
157int weakbindingbarrier;
158
159/*
160 * Variables used in pause_cpus().
161 */
162static volatile char safe_list[NCPU];
163
164static struct _cpu_pause_info {
165	int		cp_spl;		/* spl saved in pause_cpus() */
166	volatile int	cp_go;		/* Go signal sent after all ready */
167	int		cp_count;	/* # of CPUs to pause */
168	ksema_t		cp_sem;		/* synch pause_cpus & cpu_pause */
169	kthread_id_t	cp_paused;
170	void		*(*cp_func)(void *);
171} cpu_pause_info;
172
173static kmutex_t pause_free_mutex;
174static kcondvar_t pause_free_cv;
175
176
177static struct cpu_sys_stats_ks_data {
178	kstat_named_t cpu_ticks_idle;
179	kstat_named_t cpu_ticks_user;
180	kstat_named_t cpu_ticks_kernel;
181	kstat_named_t cpu_ticks_wait;
182	kstat_named_t cpu_nsec_idle;
183	kstat_named_t cpu_nsec_user;
184	kstat_named_t cpu_nsec_kernel;
185	kstat_named_t cpu_nsec_dtrace;
186	kstat_named_t cpu_nsec_intr;
187	kstat_named_t cpu_load_intr;
188	kstat_named_t wait_ticks_io;
189	kstat_named_t dtrace_probes;
190	kstat_named_t bread;
191	kstat_named_t bwrite;
192	kstat_named_t lread;
193	kstat_named_t lwrite;
194	kstat_named_t phread;
195	kstat_named_t phwrite;
196	kstat_named_t pswitch;
197	kstat_named_t trap;
198	kstat_named_t intr;
199	kstat_named_t syscall;
200	kstat_named_t sysread;
201	kstat_named_t syswrite;
202	kstat_named_t sysfork;
203	kstat_named_t sysvfork;
204	kstat_named_t sysexec;
205	kstat_named_t readch;
206	kstat_named_t writech;
207	kstat_named_t rcvint;
208	kstat_named_t xmtint;
209	kstat_named_t mdmint;
210	kstat_named_t rawch;
211	kstat_named_t canch;
212	kstat_named_t outch;
213	kstat_named_t msg;
214	kstat_named_t sema;
215	kstat_named_t namei;
216	kstat_named_t ufsiget;
217	kstat_named_t ufsdirblk;
218	kstat_named_t ufsipage;
219	kstat_named_t ufsinopage;
220	kstat_named_t procovf;
221	kstat_named_t intrthread;
222	kstat_named_t intrblk;
223	kstat_named_t intrunpin;
224	kstat_named_t idlethread;
225	kstat_named_t inv_swtch;
226	kstat_named_t nthreads;
227	kstat_named_t cpumigrate;
228	kstat_named_t xcalls;
229	kstat_named_t mutex_adenters;
230	kstat_named_t rw_rdfails;
231	kstat_named_t rw_wrfails;
232	kstat_named_t modload;
233	kstat_named_t modunload;
234	kstat_named_t bawrite;
235	kstat_named_t iowait;
236} cpu_sys_stats_ks_data_template = {
237	{ "cpu_ticks_idle",	KSTAT_DATA_UINT64 },
238	{ "cpu_ticks_user",	KSTAT_DATA_UINT64 },
239	{ "cpu_ticks_kernel",	KSTAT_DATA_UINT64 },
240	{ "cpu_ticks_wait",	KSTAT_DATA_UINT64 },
241	{ "cpu_nsec_idle",	KSTAT_DATA_UINT64 },
242	{ "cpu_nsec_user",	KSTAT_DATA_UINT64 },
243	{ "cpu_nsec_kernel",	KSTAT_DATA_UINT64 },
244	{ "cpu_nsec_dtrace",	KSTAT_DATA_UINT64 },
245	{ "cpu_nsec_intr",	KSTAT_DATA_UINT64 },
246	{ "cpu_load_intr",	KSTAT_DATA_UINT64 },
247	{ "wait_ticks_io",	KSTAT_DATA_UINT64 },
248	{ "dtrace_probes",	KSTAT_DATA_UINT64 },
249	{ "bread",		KSTAT_DATA_UINT64 },
250	{ "bwrite",		KSTAT_DATA_UINT64 },
251	{ "lread",		KSTAT_DATA_UINT64 },
252	{ "lwrite",		KSTAT_DATA_UINT64 },
253	{ "phread",		KSTAT_DATA_UINT64 },
254	{ "phwrite",		KSTAT_DATA_UINT64 },
255	{ "pswitch",		KSTAT_DATA_UINT64 },
256	{ "trap",		KSTAT_DATA_UINT64 },
257	{ "intr",		KSTAT_DATA_UINT64 },
258	{ "syscall",		KSTAT_DATA_UINT64 },
259	{ "sysread",		KSTAT_DATA_UINT64 },
260	{ "syswrite",		KSTAT_DATA_UINT64 },
261	{ "sysfork",		KSTAT_DATA_UINT64 },
262	{ "sysvfork",		KSTAT_DATA_UINT64 },
263	{ "sysexec",		KSTAT_DATA_UINT64 },
264	{ "readch",		KSTAT_DATA_UINT64 },
265	{ "writech",		KSTAT_DATA_UINT64 },
266	{ "rcvint",		KSTAT_DATA_UINT64 },
267	{ "xmtint",		KSTAT_DATA_UINT64 },
268	{ "mdmint",		KSTAT_DATA_UINT64 },
269	{ "rawch",		KSTAT_DATA_UINT64 },
270	{ "canch",		KSTAT_DATA_UINT64 },
271	{ "outch",		KSTAT_DATA_UINT64 },
272	{ "msg",		KSTAT_DATA_UINT64 },
273	{ "sema",		KSTAT_DATA_UINT64 },
274	{ "namei",		KSTAT_DATA_UINT64 },
275	{ "ufsiget",		KSTAT_DATA_UINT64 },
276	{ "ufsdirblk",		KSTAT_DATA_UINT64 },
277	{ "ufsipage",		KSTAT_DATA_UINT64 },
278	{ "ufsinopage",		KSTAT_DATA_UINT64 },
279	{ "procovf",		KSTAT_DATA_UINT64 },
280	{ "intrthread",		KSTAT_DATA_UINT64 },
281	{ "intrblk",		KSTAT_DATA_UINT64 },
282	{ "intrunpin",		KSTAT_DATA_UINT64 },
283	{ "idlethread",		KSTAT_DATA_UINT64 },
284	{ "inv_swtch",		KSTAT_DATA_UINT64 },
285	{ "nthreads",		KSTAT_DATA_UINT64 },
286	{ "cpumigrate",		KSTAT_DATA_UINT64 },
287	{ "xcalls",		KSTAT_DATA_UINT64 },
288	{ "mutex_adenters",	KSTAT_DATA_UINT64 },
289	{ "rw_rdfails",		KSTAT_DATA_UINT64 },
290	{ "rw_wrfails",		KSTAT_DATA_UINT64 },
291	{ "modload",		KSTAT_DATA_UINT64 },
292	{ "modunload",		KSTAT_DATA_UINT64 },
293	{ "bawrite",		KSTAT_DATA_UINT64 },
294	{ "iowait",		KSTAT_DATA_UINT64 },
295};
296
297static struct cpu_vm_stats_ks_data {
298	kstat_named_t pgrec;
299	kstat_named_t pgfrec;
300	kstat_named_t pgin;
301	kstat_named_t pgpgin;
302	kstat_named_t pgout;
303	kstat_named_t pgpgout;
304	kstat_named_t swapin;
305	kstat_named_t pgswapin;
306	kstat_named_t swapout;
307	kstat_named_t pgswapout;
308	kstat_named_t zfod;
309	kstat_named_t dfree;
310	kstat_named_t scan;
311	kstat_named_t rev;
312	kstat_named_t hat_fault;
313	kstat_named_t as_fault;
314	kstat_named_t maj_fault;
315	kstat_named_t cow_fault;
316	kstat_named_t prot_fault;
317	kstat_named_t softlock;
318	kstat_named_t kernel_asflt;
319	kstat_named_t pgrrun;
320	kstat_named_t execpgin;
321	kstat_named_t execpgout;
322	kstat_named_t execfree;
323	kstat_named_t anonpgin;
324	kstat_named_t anonpgout;
325	kstat_named_t anonfree;
326	kstat_named_t fspgin;
327	kstat_named_t fspgout;
328	kstat_named_t fsfree;
329} cpu_vm_stats_ks_data_template = {
330	{ "pgrec",		KSTAT_DATA_UINT64 },
331	{ "pgfrec",		KSTAT_DATA_UINT64 },
332	{ "pgin",		KSTAT_DATA_UINT64 },
333	{ "pgpgin",		KSTAT_DATA_UINT64 },
334	{ "pgout",		KSTAT_DATA_UINT64 },
335	{ "pgpgout",		KSTAT_DATA_UINT64 },
336	{ "swapin",		KSTAT_DATA_UINT64 },
337	{ "pgswapin",		KSTAT_DATA_UINT64 },
338	{ "swapout",		KSTAT_DATA_UINT64 },
339	{ "pgswapout",		KSTAT_DATA_UINT64 },
340	{ "zfod",		KSTAT_DATA_UINT64 },
341	{ "dfree",		KSTAT_DATA_UINT64 },
342	{ "scan",		KSTAT_DATA_UINT64 },
343	{ "rev",		KSTAT_DATA_UINT64 },
344	{ "hat_fault",		KSTAT_DATA_UINT64 },
345	{ "as_fault",		KSTAT_DATA_UINT64 },
346	{ "maj_fault",		KSTAT_DATA_UINT64 },
347	{ "cow_fault",		KSTAT_DATA_UINT64 },
348	{ "prot_fault",		KSTAT_DATA_UINT64 },
349	{ "softlock",		KSTAT_DATA_UINT64 },
350	{ "kernel_asflt",	KSTAT_DATA_UINT64 },
351	{ "pgrrun",		KSTAT_DATA_UINT64 },
352	{ "execpgin",		KSTAT_DATA_UINT64 },
353	{ "execpgout",		KSTAT_DATA_UINT64 },
354	{ "execfree",		KSTAT_DATA_UINT64 },
355	{ "anonpgin",		KSTAT_DATA_UINT64 },
356	{ "anonpgout",		KSTAT_DATA_UINT64 },
357	{ "anonfree",		KSTAT_DATA_UINT64 },
358	{ "fspgin",		KSTAT_DATA_UINT64 },
359	{ "fspgout",		KSTAT_DATA_UINT64 },
360	{ "fsfree",		KSTAT_DATA_UINT64 },
361};
362
363/*
364 * Force the specified thread to migrate to the appropriate processor.
365 * Called with thread lock held, returns with it dropped.
366 */
367static void
368force_thread_migrate(kthread_id_t tp)
369{
370	ASSERT(THREAD_LOCK_HELD(tp));
371	if (tp == curthread) {
372		THREAD_TRANSITION(tp);
373		CL_SETRUN(tp);
374		thread_unlock_nopreempt(tp);
375		swtch();
376	} else {
377		if (tp->t_state == TS_ONPROC) {
378			cpu_surrender(tp);
379		} else if (tp->t_state == TS_RUN) {
380			(void) dispdeq(tp);
381			setbackdq(tp);
382		}
383		thread_unlock(tp);
384	}
385}
386
387/*
388 * Set affinity for a specified CPU.
389 * A reference count is incremented and the affinity is held until the
390 * reference count is decremented to zero by thread_affinity_clear().
391 * This is so regions of code requiring affinity can be nested.
392 * Caller needs to ensure that cpu_id remains valid, which can be
393 * done by holding cpu_lock across this call, unless the caller
394 * specifies CPU_CURRENT in which case the cpu_lock will be acquired
395 * by thread_affinity_set and CPU->cpu_id will be the target CPU.
396 */
397void
398thread_affinity_set(kthread_id_t t, int cpu_id)
399{
400	cpu_t		*cp;
401	int		c;
402
403	ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL));
404
405	if ((c = cpu_id) == CPU_CURRENT) {
406		mutex_enter(&cpu_lock);
407		cpu_id = CPU->cpu_id;
408	}
409	/*
410	 * We should be asserting that cpu_lock is held here, but
411	 * the NCA code doesn't acquire it.  The following assert
412	 * should be uncommented when the NCA code is fixed.
413	 *
414	 * ASSERT(MUTEX_HELD(&cpu_lock));
415	 */
416	ASSERT((cpu_id >= 0) && (cpu_id < NCPU));
417	cp = cpu[cpu_id];
418	ASSERT(cp != NULL);		/* user must provide a good cpu_id */
419	/*
420	 * If there is already a hard affinity requested, and this affinity
421	 * conflicts with that, panic.
422	 */
423	thread_lock(t);
424	if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) {
425		panic("affinity_set: setting %p but already bound to %p",
426		    (void *)cp, (void *)t->t_bound_cpu);
427	}
428	t->t_affinitycnt++;
429	t->t_bound_cpu = cp;
430
431	/*
432	 * Make sure we're running on the right CPU.
433	 */
434	if (cp != t->t_cpu || t != curthread) {
435		force_thread_migrate(t);	/* drops thread lock */
436	} else {
437		thread_unlock(t);
438	}
439
440	if (c == CPU_CURRENT)
441		mutex_exit(&cpu_lock);
442}
443
444/*
445 *	Wrapper for backward compatibility.
446 */
447void
448affinity_set(int cpu_id)
449{
450	thread_affinity_set(curthread, cpu_id);
451}
452
453/*
454 * Decrement the affinity reservation count and if it becomes zero,
455 * clear the CPU affinity for the current thread, or set it to the user's
456 * software binding request.
457 */
458void
459thread_affinity_clear(kthread_id_t t)
460{
461	register processorid_t binding;
462
463	thread_lock(t);
464	if (--t->t_affinitycnt == 0) {
465		if ((binding = t->t_bind_cpu) == PBIND_NONE) {
466			/*
467			 * Adjust disp_max_unbound_pri if necessary.
468			 */
469			disp_adjust_unbound_pri(t);
470			t->t_bound_cpu = NULL;
471			if (t->t_cpu->cpu_part != t->t_cpupart) {
472				force_thread_migrate(t);
473				return;
474			}
475		} else {
476			t->t_bound_cpu = cpu[binding];
477			/*
478			 * Make sure the thread is running on the bound CPU.
479			 */
480			if (t->t_cpu != t->t_bound_cpu) {
481				force_thread_migrate(t);
482				return;		/* already dropped lock */
483			}
484		}
485	}
486	thread_unlock(t);
487}
488
489/*
490 * Wrapper for backward compatibility.
491 */
492void
493affinity_clear(void)
494{
495	thread_affinity_clear(curthread);
496}
497
498/*
499 * Weak cpu affinity.  Bind to the "current" cpu for short periods
500 * of time during which the thread must not block (but may be preempted).
501 * Use this instead of kpreempt_disable() when it is only "no migration"
502 * rather than "no preemption" semantics that are required - disabling
503 * preemption holds higher priority threads off of cpu and if the
504 * operation that is protected is more than momentary this is not good
505 * for realtime etc.
506 *
507 * Weakly bound threads will not prevent a cpu from being offlined -
508 * we'll only run them on the cpu to which they are weakly bound but
509 * (because they do not block) we'll always be able to move them on to
510 * another cpu at offline time if we give them just a short moment to
511 * run during which they will unbind.  To give a cpu a chance of offlining,
512 * however, we require a barrier to weak bindings that may be raised for a
513 * given cpu (offline/move code may set this and then wait a short time for
514 * existing weak bindings to drop); the cpu_inmotion pointer is that barrier.
515 *
516 * There are few restrictions on the calling context of thread_nomigrate.
517 * The caller must not hold the thread lock.  Calls may be nested.
518 *
519 * After weakbinding a thread must not perform actions that may block.
520 * In particular it must not call thread_affinity_set; calling that when
521 * already weakbound is nonsensical anyway.
522 *
523 * If curthread is prevented from migrating for other reasons
524 * (kernel preemption disabled; high pil; strongly bound; interrupt thread)
525 * then the weak binding will succeed even if this cpu is the target of an
526 * offline/move request.
527 */
528void
529thread_nomigrate(void)
530{
531	cpu_t *cp;
532	kthread_id_t t = curthread;
533
534again:
535	kpreempt_disable();
536	cp = CPU;
537
538	/*
539	 * A highlevel interrupt must not modify t_nomigrate or
540	 * t_weakbound_cpu of the thread it has interrupted.  A lowlevel
541	 * interrupt thread cannot migrate and we can avoid the
542	 * thread_lock call below by short-circuiting here.  In either
543	 * case we can just return since no migration is possible and
544	 * the condition will persist (ie, when we test for these again
545	 * in thread_allowmigrate they can't have changed).   Migration
546	 * is also impossible if we're at or above DISP_LEVEL pil.
547	 */
548	if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD ||
549	    getpil() >= DISP_LEVEL) {
550		kpreempt_enable();
551		return;
552	}
553
554	/*
555	 * We must be consistent with existing weak bindings.  Since we
556	 * may be interrupted between the increment of t_nomigrate and
557	 * the store to t_weakbound_cpu below we cannot assume that
558	 * t_weakbound_cpu will be set if t_nomigrate is.  Note that we
559	 * cannot assert t_weakbound_cpu == t_bind_cpu since that is not
560	 * always the case.
561	 */
562	if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) {
563		if (!panicstr)
564			panic("thread_nomigrate: binding to %p but already "
565			    "bound to %p", (void *)cp,
566			    (void *)t->t_weakbound_cpu);
567	}
568
569	/*
570	 * At this point we have preemption disabled and we don't yet hold
571	 * the thread lock.  So it's possible that somebody else could
572	 * set t_bind_cpu here and not be able to force us across to the
573	 * new cpu (since we have preemption disabled).
574	 */
575	thread_lock(curthread);
576
577	/*
578	 * If further weak bindings are being (temporarily) suppressed then
579	 * we'll settle for disabling kernel preemption (which assures
580	 * no migration provided the thread does not block which it is
581	 * not allowed to if using thread_nomigrate).  We must remember
582	 * this disposition so we can take appropriate action in
583	 * thread_allowmigrate.  If this is a nested call and the
584	 * thread is already weakbound then fall through as normal.
585	 * We remember the decision to settle for kpreempt_disable through
586	 * negative nesting counting in t_nomigrate.  Once a thread has had one
587	 * weakbinding request satisfied in this way any further (nested)
588	 * requests will continue to be satisfied in the same way,
589	 * even if weak bindings have recommenced.
590	 */
591	if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) {
592		--t->t_nomigrate;
593		thread_unlock(curthread);
594		return;		/* with kpreempt_disable still active */
595	}
596
597	/*
598	 * We hold thread_lock so t_bind_cpu cannot change.  We could,
599	 * however, be running on a different cpu to which we are t_bound_cpu
600	 * to (as explained above).  If we grant the weak binding request
601	 * in that case then the dispatcher must favour our weak binding
602	 * over our strong (in which case, just as when preemption is
603	 * disabled, we can continue to run on a cpu other than the one to
604	 * which we are strongbound; the difference in this case is that
605	 * this thread can be preempted and so can appear on the dispatch
606	 * queues of a cpu other than the one it is strongbound to).
607	 *
608	 * If the cpu we are running on does not appear to be a current
609	 * offline target (we check cpu_inmotion to determine this - since
610	 * we don't hold cpu_lock we may not see a recent store to that,
611	 * so it's possible that we at times can grant a weak binding to a
612	 * cpu that is an offline target, but that one request will not
613	 * prevent the offline from succeeding) then we will always grant
614	 * the weak binding request.  This includes the case above where
615	 * we grant a weakbinding not commensurate with our strong binding.
616	 *
617	 * If our cpu does appear to be an offline target then we're inclined
618	 * not to grant the weakbinding request just yet - we'd prefer to
619	 * migrate to another cpu and grant the request there.  The
620	 * exceptions are those cases where going through preemption code
621	 * will not result in us changing cpu:
622	 *
623	 *	. interrupts have already bypassed this case (see above)
624	 *	. we are already weakbound to this cpu (dispatcher code will
625	 *	  always return us to the weakbound cpu)
626	 *	. preemption was disabled even before we disabled it above
627	 *	. we are strongbound to this cpu (if we're strongbound to
628	 *	another and not yet running there the trip through the
629	 *	dispatcher will move us to the strongbound cpu and we
630	 *	will grant the weak binding there)
631	 */
632	if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 ||
633	    t->t_bound_cpu == cp) {
634		/*
635		 * Don't be tempted to store to t_weakbound_cpu only on
636		 * the first nested bind request - if we're interrupted
637		 * after the increment of t_nomigrate and before the
638		 * store to t_weakbound_cpu and the interrupt calls
639		 * thread_nomigrate then the assertion in thread_allowmigrate
640		 * would fail.
641		 */
642		t->t_nomigrate++;
643		t->t_weakbound_cpu = cp;
644		membar_producer();
645		thread_unlock(curthread);
646		/*
647		 * Now that we have dropped the thread_lock another thread
648		 * can set our t_weakbound_cpu, and will try to migrate us
649		 * to the strongbound cpu (which will not be prevented by
650		 * preemption being disabled since we're about to enable
651		 * preemption).  We have granted the weakbinding to the current
652		 * cpu, so again we are in the position that is is is possible
653		 * that our weak and strong bindings differ.  Again this
654		 * is catered for by dispatcher code which will favour our
655		 * weak binding.
656		 */
657		kpreempt_enable();
658	} else {
659		/*
660		 * Move to another cpu before granting the request by
661		 * forcing this thread through preemption code.  When we
662		 * get to set{front,back}dq called from CL_PREEMPT()
663		 * cpu_choose() will be used to select a cpu to queue
664		 * us on - that will see cpu_inmotion and take
665		 * steps to avoid returning us to this cpu.
666		 */
667		cp->cpu_kprunrun = 1;
668		thread_unlock(curthread);
669		kpreempt_enable();	/* will call preempt() */
670		goto again;
671	}
672}
673
674void
675thread_allowmigrate(void)
676{
677	kthread_id_t t = curthread;
678
679	ASSERT(t->t_weakbound_cpu == CPU ||
680	    (t->t_nomigrate < 0 && t->t_preempt > 0) ||
681	    CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD ||
682	    getpil() >= DISP_LEVEL);
683
684	if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) ||
685	    getpil() >= DISP_LEVEL)
686		return;
687
688	if (t->t_nomigrate < 0) {
689		/*
690		 * This thread was granted "weak binding" in the
691		 * stronger form of kernel preemption disabling.
692		 * Undo a level of nesting for both t_nomigrate
693		 * and t_preempt.
694		 */
695		++t->t_nomigrate;
696		kpreempt_enable();
697	} else if (--t->t_nomigrate == 0) {
698		/*
699		 * Time to drop the weak binding.  We need to cater
700		 * for the case where we're weakbound to a different
701		 * cpu than that to which we're strongbound (a very
702		 * temporary arrangement that must only persist until
703		 * weak binding drops).  We don't acquire thread_lock
704		 * here so even as this code executes t_bound_cpu
705		 * may be changing.  So we disable preemption and
706		 * a) in the case that t_bound_cpu changes while we
707		 * have preemption disabled kprunrun will be set
708		 * asynchronously, and b) if before disabling
709		 * preemption we were already on a different cpu to
710		 * our t_bound_cpu then we set kprunrun ourselves
711		 * to force a trip through the dispatcher when
712		 * preemption is enabled.
713		 */
714		kpreempt_disable();
715		if (t->t_bound_cpu &&
716		    t->t_weakbound_cpu != t->t_bound_cpu)
717			CPU->cpu_kprunrun = 1;
718		t->t_weakbound_cpu = NULL;
719		membar_producer();
720		kpreempt_enable();
721	}
722}
723
724/*
725 * weakbinding_stop can be used to temporarily cause weakbindings made
726 * with thread_nomigrate to be satisfied through the stronger action of
727 * kpreempt_disable.  weakbinding_start recommences normal weakbinding.
728 */
729
730void
731weakbinding_stop(void)
732{
733	ASSERT(MUTEX_HELD(&cpu_lock));
734	weakbindingbarrier = 1;
735	membar_producer();	/* make visible before subsequent thread_lock */
736}
737
738void
739weakbinding_start(void)
740{
741	ASSERT(MUTEX_HELD(&cpu_lock));
742	weakbindingbarrier = 0;
743}
744
745void
746null_xcall(void)
747{
748}
749
750/*
751 * This routine is called to place the CPUs in a safe place so that
752 * one of them can be taken off line or placed on line.  What we are
753 * trying to do here is prevent a thread from traversing the list
754 * of active CPUs while we are changing it or from getting placed on
755 * the run queue of a CPU that has just gone off line.  We do this by
756 * creating a thread with the highest possible prio for each CPU and
757 * having it call this routine.  The advantage of this method is that
758 * we can eliminate all checks for CPU_ACTIVE in the disp routines.
759 * This makes disp faster at the expense of making p_online() slower
760 * which is a good trade off.
761 */
762static void
763cpu_pause(int index)
764{
765	int s;
766	struct _cpu_pause_info *cpi = &cpu_pause_info;
767	volatile char *safe = &safe_list[index];
768	long    lindex = index;
769
770	ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE));
771
772	while (*safe != PAUSE_DIE) {
773		*safe = PAUSE_READY;
774		membar_enter();		/* make sure stores are flushed */
775		sema_v(&cpi->cp_sem);	/* signal requesting thread */
776
777		/*
778		 * Wait here until all pause threads are running.  That
779		 * indicates that it's safe to do the spl.  Until
780		 * cpu_pause_info.cp_go is set, we don't want to spl
781		 * because that might block clock interrupts needed
782		 * to preempt threads on other CPUs.
783		 */
784		while (cpi->cp_go == 0)
785			;
786		/*
787		 * Even though we are at the highest disp prio, we need
788		 * to block out all interrupts below LOCK_LEVEL so that
789		 * an intr doesn't come in, wake up a thread, and call
790		 * setbackdq/setfrontdq.
791		 */
792		s = splhigh();
793		/*
794		 * if cp_func has been set then call it using index as the
795		 * argument, currently only used by cpr_suspend_cpus().
796		 * This function is used as the code to execute on the
797		 * "paused" cpu's when a machine comes out of a sleep state
798		 * and CPU's were powered off.  (could also be used for
799		 * hotplugging CPU's).
800		 */
801		if (cpi->cp_func != NULL)
802			(*cpi->cp_func)((void *)lindex);
803
804		mach_cpu_pause(safe);
805
806		splx(s);
807		/*
808		 * Waiting is at an end. Switch out of cpu_pause
809		 * loop and resume useful work.
810		 */
811		swtch();
812	}
813
814	mutex_enter(&pause_free_mutex);
815	*safe = PAUSE_DEAD;
816	cv_broadcast(&pause_free_cv);
817	mutex_exit(&pause_free_mutex);
818}
819
820/*
821 * Allow the cpus to start running again.
822 */
823void
824start_cpus()
825{
826	int i;
827
828	ASSERT(MUTEX_HELD(&cpu_lock));
829	ASSERT(cpu_pause_info.cp_paused);
830	cpu_pause_info.cp_paused = NULL;
831	for (i = 0; i < NCPU; i++)
832		safe_list[i] = PAUSE_IDLE;
833	membar_enter();			/* make sure stores are flushed */
834	affinity_clear();
835	splx(cpu_pause_info.cp_spl);
836	kpreempt_enable();
837}
838
839/*
840 * Allocate a pause thread for a CPU.
841 */
842static void
843cpu_pause_alloc(cpu_t *cp)
844{
845	kthread_id_t	t;
846	long		cpun = cp->cpu_id;
847
848	/*
849	 * Note, v.v_nglobpris will not change value as long as I hold
850	 * cpu_lock.
851	 */
852	t = thread_create(NULL, 0, cpu_pause, (void *)cpun,
853	    0, &p0, TS_STOPPED, v.v_nglobpris - 1);
854	thread_lock(t);
855	t->t_bound_cpu = cp;
856	t->t_disp_queue = cp->cpu_disp;
857	t->t_affinitycnt = 1;
858	t->t_preempt = 1;
859	thread_unlock(t);
860	cp->cpu_pause_thread = t;
861	/*
862	 * Registering a thread in the callback table is usually done
863	 * in the initialization code of the thread.  In this
864	 * case, we do it right after thread creation because the
865	 * thread itself may never run, and we need to register the
866	 * fact that it is safe for cpr suspend.
867	 */
868	CALLB_CPR_INIT_SAFE(t, "cpu_pause");
869}
870
871/*
872 * Free a pause thread for a CPU.
873 */
874static void
875cpu_pause_free(cpu_t *cp)
876{
877	kthread_id_t	t;
878	int		cpun = cp->cpu_id;
879
880	ASSERT(MUTEX_HELD(&cpu_lock));
881	/*
882	 * We have to get the thread and tell it to die.
883	 */
884	if ((t = cp->cpu_pause_thread) == NULL) {
885		ASSERT(safe_list[cpun] == PAUSE_IDLE);
886		return;
887	}
888	thread_lock(t);
889	t->t_cpu = CPU;		/* disp gets upset if last cpu is quiesced. */
890	t->t_bound_cpu = NULL;	/* Must un-bind; cpu may not be running. */
891	t->t_pri = v.v_nglobpris - 1;
892	ASSERT(safe_list[cpun] == PAUSE_IDLE);
893	safe_list[cpun] = PAUSE_DIE;
894	THREAD_TRANSITION(t);
895	setbackdq(t);
896	thread_unlock_nopreempt(t);
897
898	/*
899	 * If we don't wait for the thread to actually die, it may try to
900	 * run on the wrong cpu as part of an actual call to pause_cpus().
901	 */
902	mutex_enter(&pause_free_mutex);
903	while (safe_list[cpun] != PAUSE_DEAD) {
904		cv_wait(&pause_free_cv, &pause_free_mutex);
905	}
906	mutex_exit(&pause_free_mutex);
907	safe_list[cpun] = PAUSE_IDLE;
908
909	cp->cpu_pause_thread = NULL;
910}
911
912/*
913 * Initialize basic structures for pausing CPUs.
914 */
915void
916cpu_pause_init()
917{
918	sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL);
919	/*
920	 * Create initial CPU pause thread.
921	 */
922	cpu_pause_alloc(CPU);
923}
924
925/*
926 * Start the threads used to pause another CPU.
927 */
928static int
929cpu_pause_start(processorid_t cpu_id)
930{
931	int	i;
932	int	cpu_count = 0;
933
934	for (i = 0; i < NCPU; i++) {
935		cpu_t		*cp;
936		kthread_id_t	t;
937
938		cp = cpu[i];
939		if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) {
940			safe_list[i] = PAUSE_WAIT;
941			continue;
942		}
943
944		/*
945		 * Skip CPU if it is quiesced or not yet started.
946		 */
947		if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) {
948			safe_list[i] = PAUSE_WAIT;
949			continue;
950		}
951
952		/*
953		 * Start this CPU's pause thread.
954		 */
955		t = cp->cpu_pause_thread;
956		thread_lock(t);
957		/*
958		 * Reset the priority, since nglobpris may have
959		 * changed since the thread was created, if someone
960		 * has loaded the RT (or some other) scheduling
961		 * class.
962		 */
963		t->t_pri = v.v_nglobpris - 1;
964		THREAD_TRANSITION(t);
965		setbackdq(t);
966		thread_unlock_nopreempt(t);
967		++cpu_count;
968	}
969	return (cpu_count);
970}
971
972
973/*
974 * Pause all of the CPUs except the one we are on by creating a high
975 * priority thread bound to those CPUs.
976 *
977 * Note that one must be extremely careful regarding code
978 * executed while CPUs are paused.  Since a CPU may be paused
979 * while a thread scheduling on that CPU is holding an adaptive
980 * lock, code executed with CPUs paused must not acquire adaptive
981 * (or low-level spin) locks.  Also, such code must not block,
982 * since the thread that is supposed to initiate the wakeup may
983 * never run.
984 *
985 * With a few exceptions, the restrictions on code executed with CPUs
986 * paused match those for code executed at high-level interrupt
987 * context.
988 */
989void
990pause_cpus(cpu_t *off_cp, void *(*func)(void *))
991{
992	processorid_t	cpu_id;
993	int		i;
994	struct _cpu_pause_info	*cpi = &cpu_pause_info;
995
996	ASSERT(MUTEX_HELD(&cpu_lock));
997	ASSERT(cpi->cp_paused == NULL);
998	cpi->cp_count = 0;
999	cpi->cp_go = 0;
1000	for (i = 0; i < NCPU; i++)
1001		safe_list[i] = PAUSE_IDLE;
1002	kpreempt_disable();
1003
1004	cpi->cp_func = func;
1005
1006	/*
1007	 * If running on the cpu that is going offline, get off it.
1008	 * This is so that it won't be necessary to rechoose a CPU
1009	 * when done.
1010	 */
1011	if (CPU == off_cp)
1012		cpu_id = off_cp->cpu_next_part->cpu_id;
1013	else
1014		cpu_id = CPU->cpu_id;
1015	affinity_set(cpu_id);
1016
1017	/*
1018	 * Start the pause threads and record how many were started
1019	 */
1020	cpi->cp_count = cpu_pause_start(cpu_id);
1021
1022	/*
1023	 * Now wait for all CPUs to be running the pause thread.
1024	 */
1025	while (cpi->cp_count > 0) {
1026		/*
1027		 * Spin reading the count without grabbing the disp
1028		 * lock to make sure we don't prevent the pause
1029		 * threads from getting the lock.
1030		 */
1031		while (sema_held(&cpi->cp_sem))
1032			;
1033		if (sema_tryp(&cpi->cp_sem))
1034			--cpi->cp_count;
1035	}
1036	cpi->cp_go = 1;			/* all have reached cpu_pause */
1037
1038	/*
1039	 * Now wait for all CPUs to spl. (Transition from PAUSE_READY
1040	 * to PAUSE_WAIT.)
1041	 */
1042	for (i = 0; i < NCPU; i++) {
1043		while (safe_list[i] != PAUSE_WAIT)
1044			;
1045	}
1046	cpi->cp_spl = splhigh();	/* block dispatcher on this CPU */
1047	cpi->cp_paused = curthread;
1048}
1049
1050/*
1051 * Check whether the current thread has CPUs paused
1052 */
1053int
1054cpus_paused(void)
1055{
1056	if (cpu_pause_info.cp_paused != NULL) {
1057		ASSERT(cpu_pause_info.cp_paused == curthread);
1058		return (1);
1059	}
1060	return (0);
1061}
1062
1063static cpu_t *
1064cpu_get_all(processorid_t cpun)
1065{
1066	ASSERT(MUTEX_HELD(&cpu_lock));
1067
1068	if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun))
1069		return (NULL);
1070	return (cpu[cpun]);
1071}
1072
1073/*
1074 * Check whether cpun is a valid processor id and whether it should be
1075 * visible from the current zone. If it is, return a pointer to the
1076 * associated CPU structure.
1077 */
1078cpu_t *
1079cpu_get(processorid_t cpun)
1080{
1081	cpu_t *c;
1082
1083	ASSERT(MUTEX_HELD(&cpu_lock));
1084	c = cpu_get_all(cpun);
1085	if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() &&
1086	    zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c))
1087		return (NULL);
1088	return (c);
1089}
1090
1091/*
1092 * The following functions should be used to check CPU states in the kernel.
1093 * They should be invoked with cpu_lock held.  Kernel subsystems interested
1094 * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc
1095 * states.  Those are for user-land (and system call) use only.
1096 */
1097
1098/*
1099 * Determine whether the CPU is online and handling interrupts.
1100 */
1101int
1102cpu_is_online(cpu_t *cpu)
1103{
1104	ASSERT(MUTEX_HELD(&cpu_lock));
1105	return (cpu_flagged_online(cpu->cpu_flags));
1106}
1107
1108/*
1109 * Determine whether the CPU is offline (this includes spare and faulted).
1110 */
1111int
1112cpu_is_offline(cpu_t *cpu)
1113{
1114	ASSERT(MUTEX_HELD(&cpu_lock));
1115	return (cpu_flagged_offline(cpu->cpu_flags));
1116}
1117
1118/*
1119 * Determine whether the CPU is powered off.
1120 */
1121int
1122cpu_is_poweredoff(cpu_t *cpu)
1123{
1124	ASSERT(MUTEX_HELD(&cpu_lock));
1125	return (cpu_flagged_poweredoff(cpu->cpu_flags));
1126}
1127
1128/*
1129 * Determine whether the CPU is handling interrupts.
1130 */
1131int
1132cpu_is_nointr(cpu_t *cpu)
1133{
1134	ASSERT(MUTEX_HELD(&cpu_lock));
1135	return (cpu_flagged_nointr(cpu->cpu_flags));
1136}
1137
1138/*
1139 * Determine whether the CPU is active (scheduling threads).
1140 */
1141int
1142cpu_is_active(cpu_t *cpu)
1143{
1144	ASSERT(MUTEX_HELD(&cpu_lock));
1145	return (cpu_flagged_active(cpu->cpu_flags));
1146}
1147
1148/*
1149 * Same as above, but these require cpu_flags instead of cpu_t pointers.
1150 */
1151int
1152cpu_flagged_online(cpu_flag_t cpu_flags)
1153{
1154	return (cpu_flagged_active(cpu_flags) &&
1155	    (cpu_flags & CPU_ENABLE));
1156}
1157
1158int
1159cpu_flagged_offline(cpu_flag_t cpu_flags)
1160{
1161	return (((cpu_flags & CPU_POWEROFF) == 0) &&
1162	    ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY));
1163}
1164
1165int
1166cpu_flagged_poweredoff(cpu_flag_t cpu_flags)
1167{
1168	return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF);
1169}
1170
1171int
1172cpu_flagged_nointr(cpu_flag_t cpu_flags)
1173{
1174	return (cpu_flagged_active(cpu_flags) &&
1175	    (cpu_flags & CPU_ENABLE) == 0);
1176}
1177
1178int
1179cpu_flagged_active(cpu_flag_t cpu_flags)
1180{
1181	return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) &&
1182	    ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY));
1183}
1184
1185/*
1186 * Bring the indicated CPU online.
1187 */
1188int
1189cpu_online(cpu_t *cp)
1190{
1191	int	error = 0;
1192
1193	/*
1194	 * Handle on-line request.
1195	 *	This code must put the new CPU on the active list before
1196	 *	starting it because it will not be paused, and will start
1197	 * 	using the active list immediately.  The real start occurs
1198	 *	when the CPU_QUIESCED flag is turned off.
1199	 */
1200
1201	ASSERT(MUTEX_HELD(&cpu_lock));
1202
1203	/*
1204	 * Put all the cpus into a known safe place.
1205	 * No mutexes can be entered while CPUs are paused.
1206	 */
1207	error = mp_cpu_start(cp);	/* arch-dep hook */
1208	if (error == 0) {
1209		pg_cpupart_in(cp, cp->cpu_part);
1210		pause_cpus(NULL, NULL);
1211		cpu_add_active_internal(cp);
1212		if (cp->cpu_flags & CPU_FAULTED) {
1213			cp->cpu_flags &= ~CPU_FAULTED;
1214			mp_cpu_faulted_exit(cp);
1215		}
1216		cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN |
1217		    CPU_SPARE);
1218		CPU_NEW_GENERATION(cp);
1219		start_cpus();
1220		cpu_stats_kstat_create(cp);
1221		cpu_create_intrstat(cp);
1222		lgrp_kstat_create(cp);
1223		cpu_state_change_notify(cp->cpu_id, CPU_ON);
1224		cpu_intr_enable(cp);	/* arch-dep hook */
1225		cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1226		cpu_set_state(cp);
1227		cyclic_online(cp);
1228		/*
1229		 * This has to be called only after cyclic_online(). This
1230		 * function uses cyclics.
1231		 */
1232		callout_cpu_online(cp);
1233		poke_cpu(cp->cpu_id);
1234	}
1235
1236	return (error);
1237}
1238
1239/*
1240 * Take the indicated CPU offline.
1241 */
1242int
1243cpu_offline(cpu_t *cp, int flags)
1244{
1245	cpupart_t *pp;
1246	int	error = 0;
1247	cpu_t	*ncp;
1248	int	intr_enable;
1249	int	cyclic_off = 0;
1250	int	callout_off = 0;
1251	int	loop_count;
1252	int	no_quiesce = 0;
1253	int	(*bound_func)(struct cpu *, int);
1254	kthread_t *t;
1255	lpl_t	*cpu_lpl;
1256	proc_t	*p;
1257	int	lgrp_diff_lpl;
1258	boolean_t unbind_all_threads = (flags & CPU_FORCED) != 0;
1259
1260	ASSERT(MUTEX_HELD(&cpu_lock));
1261
1262	/*
1263	 * If we're going from faulted or spare to offline, just
1264	 * clear these flags and update CPU state.
1265	 */
1266	if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) {
1267		if (cp->cpu_flags & CPU_FAULTED) {
1268			cp->cpu_flags &= ~CPU_FAULTED;
1269			mp_cpu_faulted_exit(cp);
1270		}
1271		cp->cpu_flags &= ~CPU_SPARE;
1272		cpu_set_state(cp);
1273		return (0);
1274	}
1275
1276	/*
1277	 * Handle off-line request.
1278	 */
1279	pp = cp->cpu_part;
1280	/*
1281	 * Don't offline last online CPU in partition
1282	 */
1283	if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2)
1284		return (EBUSY);
1285	/*
1286	 * Unbind all soft-bound threads bound to our CPU and hard bound threads
1287	 * if we were asked to.
1288	 */
1289	error = cpu_unbind(cp->cpu_id, unbind_all_threads);
1290	if (error != 0)
1291		return (error);
1292	/*
1293	 * We shouldn't be bound to this CPU ourselves.
1294	 */
1295	if (curthread->t_bound_cpu == cp)
1296		return (EBUSY);
1297
1298	/*
1299	 * Tell interested parties that this CPU is going offline.
1300	 */
1301	CPU_NEW_GENERATION(cp);
1302	cpu_state_change_notify(cp->cpu_id, CPU_OFF);
1303
1304	/*
1305	 * Tell the PG subsystem that the CPU is leaving the partition
1306	 */
1307	pg_cpupart_out(cp, pp);
1308
1309	/*
1310	 * Take the CPU out of interrupt participation so we won't find
1311	 * bound kernel threads.  If the architecture cannot completely
1312	 * shut off interrupts on the CPU, don't quiesce it, but don't
1313	 * run anything but interrupt thread... this is indicated by
1314	 * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being
1315	 * off.
1316	 */
1317	intr_enable = cp->cpu_flags & CPU_ENABLE;
1318	if (intr_enable)
1319		no_quiesce = cpu_intr_disable(cp);
1320
1321	/*
1322	 * Record that we are aiming to offline this cpu.  This acts as
1323	 * a barrier to further weak binding requests in thread_nomigrate
1324	 * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to
1325	 * lean away from this cpu.  Further strong bindings are already
1326	 * avoided since we hold cpu_lock.  Since threads that are set
1327	 * runnable around now and others coming off the target cpu are
1328	 * directed away from the target, existing strong and weak bindings
1329	 * (especially the latter) to the target cpu stand maximum chance of
1330	 * being able to unbind during the short delay loop below (if other
1331	 * unbound threads compete they may not see cpu in time to unbind
1332	 * even if they would do so immediately.
1333	 */
1334	cpu_inmotion = cp;
1335	membar_enter();
1336
1337	/*
1338	 * Check for kernel threads (strong or weak) bound to that CPU.
1339	 * Strongly bound threads may not unbind, and we'll have to return
1340	 * EBUSY.  Weakly bound threads should always disappear - we've
1341	 * stopped more weak binding with cpu_inmotion and existing
1342	 * bindings will drain imminently (they may not block).  Nonetheless
1343	 * we will wait for a fixed period for all bound threads to disappear.
1344	 * Inactive interrupt threads are OK (they'll be in TS_FREE
1345	 * state).  If test finds some bound threads, wait a few ticks
1346	 * to give short-lived threads (such as interrupts) chance to
1347	 * complete.  Note that if no_quiesce is set, i.e. this cpu
1348	 * is required to service interrupts, then we take the route
1349	 * that permits interrupt threads to be active (or bypassed).
1350	 */
1351	bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads;
1352
1353again:	for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) {
1354		if (loop_count >= 5) {
1355			error = EBUSY;	/* some threads still bound */
1356			break;
1357		}
1358
1359		/*
1360		 * If some threads were assigned, give them
1361		 * a chance to complete or move.
1362		 *
1363		 * This assumes that the clock_thread is not bound
1364		 * to any CPU, because the clock_thread is needed to
1365		 * do the delay(hz/100).
1366		 *
1367		 * Note: we still hold the cpu_lock while waiting for
1368		 * the next clock tick.  This is OK since it isn't
1369		 * needed for anything else except processor_bind(2),
1370		 * and system initialization.  If we drop the lock,
1371		 * we would risk another p_online disabling the last
1372		 * processor.
1373		 */
1374		delay(hz/100);
1375	}
1376
1377	if (error == 0 && callout_off == 0) {
1378		callout_cpu_offline(cp);
1379		callout_off = 1;
1380	}
1381
1382	if (error == 0 && cyclic_off == 0) {
1383		if (!cyclic_offline(cp)) {
1384			/*
1385			 * We must have bound cyclics...
1386			 */
1387			error = EBUSY;
1388			goto out;
1389		}
1390		cyclic_off = 1;
1391	}
1392
1393	/*
1394	 * Call mp_cpu_stop() to perform any special operations
1395	 * needed for this machine architecture to offline a CPU.
1396	 */
1397	if (error == 0)
1398		error = mp_cpu_stop(cp);	/* arch-dep hook */
1399
1400	/*
1401	 * If that all worked, take the CPU offline and decrement
1402	 * ncpus_online.
1403	 */
1404	if (error == 0) {
1405		/*
1406		 * Put all the cpus into a known safe place.
1407		 * No mutexes can be entered while CPUs are paused.
1408		 */
1409		pause_cpus(cp, NULL);
1410		/*
1411		 * Repeat the operation, if necessary, to make sure that
1412		 * all outstanding low-level interrupts run to completion
1413		 * before we set the CPU_QUIESCED flag.  It's also possible
1414		 * that a thread has weak bound to the cpu despite our raising
1415		 * cpu_inmotion above since it may have loaded that
1416		 * value before the barrier became visible (this would have
1417		 * to be the thread that was on the target cpu at the time
1418		 * we raised the barrier).
1419		 */
1420		if ((!no_quiesce && cp->cpu_intr_actv != 0) ||
1421		    (*bound_func)(cp, 1)) {
1422			start_cpus();
1423			(void) mp_cpu_start(cp);
1424			goto again;
1425		}
1426		ncp = cp->cpu_next_part;
1427		cpu_lpl = cp->cpu_lpl;
1428		ASSERT(cpu_lpl != NULL);
1429
1430		/*
1431		 * Remove the CPU from the list of active CPUs.
1432		 */
1433		cpu_remove_active(cp);
1434
1435		/*
1436		 * Walk the active process list and look for threads
1437		 * whose home lgroup needs to be updated, or
1438		 * the last CPU they run on is the one being offlined now.
1439		 */
1440
1441		ASSERT(curthread->t_cpu != cp);
1442		for (p = practive; p != NULL; p = p->p_next) {
1443
1444			t = p->p_tlist;
1445
1446			if (t == NULL)
1447				continue;
1448
1449			lgrp_diff_lpl = 0;
1450
1451			do {
1452				ASSERT(t->t_lpl != NULL);
1453				/*
1454				 * Taking last CPU in lpl offline
1455				 * Rehome thread if it is in this lpl
1456				 * Otherwise, update the count of how many
1457				 * threads are in this CPU's lgroup but have
1458				 * a different lpl.
1459				 */
1460
1461				if (cpu_lpl->lpl_ncpu == 0) {
1462					if (t->t_lpl == cpu_lpl)
1463						lgrp_move_thread(t,
1464						    lgrp_choose(t,
1465						    t->t_cpupart), 0);
1466					else if (t->t_lpl->lpl_lgrpid ==
1467					    cpu_lpl->lpl_lgrpid)
1468						lgrp_diff_lpl++;
1469				}
1470				ASSERT(t->t_lpl->lpl_ncpu > 0);
1471
1472				/*
1473				 * Update CPU last ran on if it was this CPU
1474				 */
1475				if (t->t_cpu == cp && t->t_bound_cpu != cp)
1476					t->t_cpu = disp_lowpri_cpu(ncp,
1477					    t->t_lpl, t->t_pri, NULL);
1478				ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1479				    t->t_weakbound_cpu == cp);
1480
1481				t = t->t_forw;
1482			} while (t != p->p_tlist);
1483
1484			/*
1485			 * Didn't find any threads in the same lgroup as this
1486			 * CPU with a different lpl, so remove the lgroup from
1487			 * the process lgroup bitmask.
1488			 */
1489
1490			if (lgrp_diff_lpl == 0)
1491				klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid);
1492		}
1493
1494		/*
1495		 * Walk thread list looking for threads that need to be
1496		 * rehomed, since there are some threads that are not in
1497		 * their process's p_tlist.
1498		 */
1499
1500		t = curthread;
1501		do {
1502			ASSERT(t != NULL && t->t_lpl != NULL);
1503
1504			/*
1505			 * Rehome threads with same lpl as this CPU when this
1506			 * is the last CPU in the lpl.
1507			 */
1508
1509			if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl))
1510				lgrp_move_thread(t,
1511				    lgrp_choose(t, t->t_cpupart), 1);
1512
1513			ASSERT(t->t_lpl->lpl_ncpu > 0);
1514
1515			/*
1516			 * Update CPU last ran on if it was this CPU
1517			 */
1518
1519			if (t->t_cpu == cp && t->t_bound_cpu != cp) {
1520				t->t_cpu = disp_lowpri_cpu(ncp,
1521				    t->t_lpl, t->t_pri, NULL);
1522			}
1523			ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1524			    t->t_weakbound_cpu == cp);
1525			t = t->t_next;
1526
1527		} while (t != curthread);
1528		ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0);
1529		cp->cpu_flags |= CPU_OFFLINE;
1530		disp_cpu_inactive(cp);
1531		if (!no_quiesce)
1532			cp->cpu_flags |= CPU_QUIESCED;
1533		ncpus_online--;
1534		cpu_set_state(cp);
1535		cpu_inmotion = NULL;
1536		start_cpus();
1537		cpu_stats_kstat_destroy(cp);
1538		cpu_delete_intrstat(cp);
1539		lgrp_kstat_destroy(cp);
1540	}
1541
1542out:
1543	cpu_inmotion = NULL;
1544
1545	/*
1546	 * If we failed, re-enable interrupts.
1547	 * Do this even if cpu_intr_disable returned an error, because
1548	 * it may have partially disabled interrupts.
1549	 */
1550	if (error && intr_enable)
1551		cpu_intr_enable(cp);
1552
1553	/*
1554	 * If we failed, but managed to offline the cyclic subsystem on this
1555	 * CPU, bring it back online.
1556	 */
1557	if (error && cyclic_off)
1558		cyclic_online(cp);
1559
1560	/*
1561	 * If we failed, but managed to offline callouts on this CPU,
1562	 * bring it back online.
1563	 */
1564	if (error && callout_off)
1565		callout_cpu_online(cp);
1566
1567	/*
1568	 * If we failed, tell the PG subsystem that the CPU is back
1569	 */
1570	pg_cpupart_in(cp, pp);
1571
1572	/*
1573	 * If we failed, we need to notify everyone that this CPU is back on.
1574	 */
1575	if (error != 0) {
1576		CPU_NEW_GENERATION(cp);
1577		cpu_state_change_notify(cp->cpu_id, CPU_ON);
1578		cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1579	}
1580
1581	return (error);
1582}
1583
1584/*
1585 * Mark the indicated CPU as faulted, taking it offline.
1586 */
1587int
1588cpu_faulted(cpu_t *cp, int flags)
1589{
1590	int	error = 0;
1591
1592	ASSERT(MUTEX_HELD(&cpu_lock));
1593	ASSERT(!cpu_is_poweredoff(cp));
1594
1595	if (cpu_is_offline(cp)) {
1596		cp->cpu_flags &= ~CPU_SPARE;
1597		cp->cpu_flags |= CPU_FAULTED;
1598		mp_cpu_faulted_enter(cp);
1599		cpu_set_state(cp);
1600		return (0);
1601	}
1602
1603	if ((error = cpu_offline(cp, flags)) == 0) {
1604		cp->cpu_flags |= CPU_FAULTED;
1605		mp_cpu_faulted_enter(cp);
1606		cpu_set_state(cp);
1607	}
1608
1609	return (error);
1610}
1611
1612/*
1613 * Mark the indicated CPU as a spare, taking it offline.
1614 */
1615int
1616cpu_spare(cpu_t *cp, int flags)
1617{
1618	int	error = 0;
1619
1620	ASSERT(MUTEX_HELD(&cpu_lock));
1621	ASSERT(!cpu_is_poweredoff(cp));
1622
1623	if (cpu_is_offline(cp)) {
1624		if (cp->cpu_flags & CPU_FAULTED) {
1625			cp->cpu_flags &= ~CPU_FAULTED;
1626			mp_cpu_faulted_exit(cp);
1627		}
1628		cp->cpu_flags |= CPU_SPARE;
1629		cpu_set_state(cp);
1630		return (0);
1631	}
1632
1633	if ((error = cpu_offline(cp, flags)) == 0) {
1634		cp->cpu_flags |= CPU_SPARE;
1635		cpu_set_state(cp);
1636	}
1637
1638	return (error);
1639}
1640
1641/*
1642 * Take the indicated CPU from poweroff to offline.
1643 */
1644int
1645cpu_poweron(cpu_t *cp)
1646{
1647	int	error = ENOTSUP;
1648
1649	ASSERT(MUTEX_HELD(&cpu_lock));
1650	ASSERT(cpu_is_poweredoff(cp));
1651
1652	error = mp_cpu_poweron(cp);	/* arch-dep hook */
1653	if (error == 0)
1654		cpu_set_state(cp);
1655
1656	return (error);
1657}
1658
1659/*
1660 * Take the indicated CPU from any inactive state to powered off.
1661 */
1662int
1663cpu_poweroff(cpu_t *cp)
1664{
1665	int	error = ENOTSUP;
1666
1667	ASSERT(MUTEX_HELD(&cpu_lock));
1668	ASSERT(cpu_is_offline(cp));
1669
1670	if (!(cp->cpu_flags & CPU_QUIESCED))
1671		return (EBUSY);		/* not completely idle */
1672
1673	error = mp_cpu_poweroff(cp);	/* arch-dep hook */
1674	if (error == 0)
1675		cpu_set_state(cp);
1676
1677	return (error);
1678}
1679
1680/*
1681 * Initialize the Sequential CPU id lookup table
1682 */
1683void
1684cpu_seq_tbl_init()
1685{
1686	cpu_t	**tbl;
1687
1688	tbl = kmem_zalloc(sizeof (struct cpu *) * max_ncpus, KM_SLEEP);
1689	tbl[0] = CPU;
1690
1691	cpu_seq = tbl;
1692}
1693
1694/*
1695 * Initialize the CPU lists for the first CPU.
1696 */
1697void
1698cpu_list_init(cpu_t *cp)
1699{
1700	cp->cpu_next = cp;
1701	cp->cpu_prev = cp;
1702	cpu_list = cp;
1703	clock_cpu_list = cp;
1704
1705	cp->cpu_next_onln = cp;
1706	cp->cpu_prev_onln = cp;
1707	cpu_active = cp;
1708
1709	cp->cpu_seqid = 0;
1710	CPUSET_ADD(cpu_seqid_inuse, 0);
1711
1712	/*
1713	 * Bootstrap cpu_seq using cpu_list
1714	 * The cpu_seq[] table will be dynamically allocated
1715	 * when kmem later becomes available (but before going MP)
1716	 */
1717	cpu_seq = &cpu_list;
1718
1719	cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1720	cp_default.cp_cpulist = cp;
1721	cp_default.cp_ncpus = 1;
1722	cp->cpu_next_part = cp;
1723	cp->cpu_prev_part = cp;
1724	cp->cpu_part = &cp_default;
1725
1726	CPUSET_ADD(cpu_available, cp->cpu_id);
1727}
1728
1729/*
1730 * Insert a CPU into the list of available CPUs.
1731 */
1732void
1733cpu_add_unit(cpu_t *cp)
1734{
1735	int seqid;
1736
1737	ASSERT(MUTEX_HELD(&cpu_lock));
1738	ASSERT(cpu_list != NULL);	/* list started in cpu_list_init */
1739
1740	lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0);
1741
1742	/*
1743	 * Note: most users of the cpu_list will grab the
1744	 * cpu_lock to insure that it isn't modified.  However,
1745	 * certain users can't or won't do that.  To allow this
1746	 * we pause the other cpus.  Users who walk the list
1747	 * without cpu_lock, must disable kernel preemption
1748	 * to insure that the list isn't modified underneath
1749	 * them.  Also, any cached pointers to cpu structures
1750	 * must be revalidated by checking to see if the
1751	 * cpu_next pointer points to itself.  This check must
1752	 * be done with the cpu_lock held or kernel preemption
1753	 * disabled.  This check relies upon the fact that
1754	 * old cpu structures are not free'ed or cleared after
1755	 * then are removed from the cpu_list.
1756	 *
1757	 * Note that the clock code walks the cpu list dereferencing
1758	 * the cpu_part pointer, so we need to initialize it before
1759	 * adding the cpu to the list.
1760	 */
1761	cp->cpu_part = &cp_default;
1762	pause_cpus(NULL, NULL);
1763	cp->cpu_next = cpu_list;
1764	cp->cpu_prev = cpu_list->cpu_prev;
1765	cpu_list->cpu_prev->cpu_next = cp;
1766	cpu_list->cpu_prev = cp;
1767	start_cpus();
1768
1769	for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++)
1770		continue;
1771	CPUSET_ADD(cpu_seqid_inuse, seqid);
1772	cp->cpu_seqid = seqid;
1773
1774	if (seqid > max_cpu_seqid_ever)
1775		max_cpu_seqid_ever = seqid;
1776
1777	ASSERT(ncpus < max_ncpus);
1778	ncpus++;
1779	cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1780	cpu[cp->cpu_id] = cp;
1781	CPUSET_ADD(cpu_available, cp->cpu_id);
1782	cpu_seq[cp->cpu_seqid] = cp;
1783
1784	/*
1785	 * allocate a pause thread for this CPU.
1786	 */
1787	cpu_pause_alloc(cp);
1788
1789	/*
1790	 * So that new CPUs won't have NULL prev_onln and next_onln pointers,
1791	 * link them into a list of just that CPU.
1792	 * This is so that disp_lowpri_cpu will work for thread_create in
1793	 * pause_cpus() when called from the startup thread in a new CPU.
1794	 */
1795	cp->cpu_next_onln = cp;
1796	cp->cpu_prev_onln = cp;
1797	cpu_info_kstat_create(cp);
1798	cp->cpu_next_part = cp;
1799	cp->cpu_prev_part = cp;
1800
1801	init_cpu_mstate(cp, CMS_SYSTEM);
1802
1803	pool_pset_mod = gethrtime();
1804}
1805
1806/*
1807 * Do the opposite of cpu_add_unit().
1808 */
1809void
1810cpu_del_unit(int cpuid)
1811{
1812	struct cpu	*cp, *cpnext;
1813
1814	ASSERT(MUTEX_HELD(&cpu_lock));
1815	cp = cpu[cpuid];
1816	ASSERT(cp != NULL);
1817
1818	ASSERT(cp->cpu_next_onln == cp);
1819	ASSERT(cp->cpu_prev_onln == cp);
1820	ASSERT(cp->cpu_next_part == cp);
1821	ASSERT(cp->cpu_prev_part == cp);
1822
1823	/*
1824	 * Tear down the CPU's physical ID cache, and update any
1825	 * processor groups
1826	 */
1827	pg_cpu_fini(cp, NULL);
1828	pghw_physid_destroy(cp);
1829
1830	/*
1831	 * Destroy kstat stuff.
1832	 */
1833	cpu_info_kstat_destroy(cp);
1834	term_cpu_mstate(cp);
1835	/*
1836	 * Free up pause thread.
1837	 */
1838	cpu_pause_free(cp);
1839	CPUSET_DEL(cpu_available, cp->cpu_id);
1840	cpu[cp->cpu_id] = NULL;
1841	cpu_seq[cp->cpu_seqid] = NULL;
1842
1843	/*
1844	 * The clock thread and mutex_vector_enter cannot hold the
1845	 * cpu_lock while traversing the cpu list, therefore we pause
1846	 * all other threads by pausing the other cpus. These, and any
1847	 * other routines holding cpu pointers while possibly sleeping
1848	 * must be sure to call kpreempt_disable before processing the
1849	 * list and be sure to check that the cpu has not been deleted
1850	 * after any sleeps (check cp->cpu_next != NULL). We guarantee
1851	 * to keep the deleted cpu structure around.
1852	 *
1853	 * Note that this MUST be done AFTER cpu_available
1854	 * has been updated so that we don't waste time
1855	 * trying to pause the cpu we're trying to delete.
1856	 */
1857	pause_cpus(NULL, NULL);
1858
1859	cpnext = cp->cpu_next;
1860	cp->cpu_prev->cpu_next = cp->cpu_next;
1861	cp->cpu_next->cpu_prev = cp->cpu_prev;
1862	if (cp == cpu_list)
1863		cpu_list = cpnext;
1864
1865	/*
1866	 * Signals that the cpu has been deleted (see above).
1867	 */
1868	cp->cpu_next = NULL;
1869	cp->cpu_prev = NULL;
1870
1871	start_cpus();
1872
1873	CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid);
1874	ncpus--;
1875	lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0);
1876
1877	pool_pset_mod = gethrtime();
1878}
1879
1880/*
1881 * Add a CPU to the list of active CPUs.
1882 *	This routine must not get any locks, because other CPUs are paused.
1883 */
1884static void
1885cpu_add_active_internal(cpu_t *cp)
1886{
1887	cpupart_t	*pp = cp->cpu_part;
1888
1889	ASSERT(MUTEX_HELD(&cpu_lock));
1890	ASSERT(cpu_list != NULL);	/* list started in cpu_list_init */
1891
1892	ncpus_online++;
1893	cpu_set_state(cp);
1894	cp->cpu_next_onln = cpu_active;
1895	cp->cpu_prev_onln = cpu_active->cpu_prev_onln;
1896	cpu_active->cpu_prev_onln->cpu_next_onln = cp;
1897	cpu_active->cpu_prev_onln = cp;
1898
1899	if (pp->cp_cpulist) {
1900		cp->cpu_next_part = pp->cp_cpulist;
1901		cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part;
1902		pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp;
1903		pp->cp_cpulist->cpu_prev_part = cp;
1904	} else {
1905		ASSERT(pp->cp_ncpus == 0);
1906		pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp;
1907	}
1908	pp->cp_ncpus++;
1909	if (pp->cp_ncpus == 1) {
1910		cp_numparts_nonempty++;
1911		ASSERT(cp_numparts_nonempty != 0);
1912	}
1913
1914	pg_cpu_active(cp);
1915	lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0);
1916
1917	bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg));
1918}
1919
1920/*
1921 * Add a CPU to the list of active CPUs.
1922 *	This is called from machine-dependent layers when a new CPU is started.
1923 */
1924void
1925cpu_add_active(cpu_t *cp)
1926{
1927	pg_cpupart_in(cp, cp->cpu_part);
1928
1929	pause_cpus(NULL, NULL);
1930	cpu_add_active_internal(cp);
1931	start_cpus();
1932
1933	cpu_stats_kstat_create(cp);
1934	cpu_create_intrstat(cp);
1935	lgrp_kstat_create(cp);
1936	cpu_state_change_notify(cp->cpu_id, CPU_INIT);
1937}
1938
1939
1940/*
1941 * Remove a CPU from the list of active CPUs.
1942 *	This routine must not get any locks, because other CPUs are paused.
1943 */
1944/* ARGSUSED */
1945static void
1946cpu_remove_active(cpu_t *cp)
1947{
1948	cpupart_t	*pp = cp->cpu_part;
1949
1950	ASSERT(MUTEX_HELD(&cpu_lock));
1951	ASSERT(cp->cpu_next_onln != cp);	/* not the last one */
1952	ASSERT(cp->cpu_prev_onln != cp);	/* not the last one */
1953
1954	pg_cpu_inactive(cp);
1955
1956	lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0);
1957
1958	if (cp == clock_cpu_list)
1959		clock_cpu_list = cp->cpu_next_onln;
1960
1961	cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln;
1962	cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln;
1963	if (cpu_active == cp) {
1964		cpu_active = cp->cpu_next_onln;
1965	}
1966	cp->cpu_next_onln = cp;
1967	cp->cpu_prev_onln = cp;
1968
1969	cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part;
1970	cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part;
1971	if (pp->cp_cpulist == cp) {
1972		pp->cp_cpulist = cp->cpu_next_part;
1973		ASSERT(pp->cp_cpulist != cp);
1974	}
1975	cp->cpu_next_part = cp;
1976	cp->cpu_prev_part = cp;
1977	pp->cp_ncpus--;
1978	if (pp->cp_ncpus == 0) {
1979		cp_numparts_nonempty--;
1980		ASSERT(cp_numparts_nonempty != 0);
1981	}
1982}
1983
1984/*
1985 * Routine used to setup a newly inserted CPU in preparation for starting
1986 * it running code.
1987 */
1988int
1989cpu_configure(int cpuid)
1990{
1991	int retval = 0;
1992
1993	ASSERT(MUTEX_HELD(&cpu_lock));
1994
1995	/*
1996	 * Some structures are statically allocated based upon
1997	 * the maximum number of cpus the system supports.  Do not
1998	 * try to add anything beyond this limit.
1999	 */
2000	if (cpuid < 0 || cpuid >= NCPU) {
2001		return (EINVAL);
2002	}
2003
2004	if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) {
2005		return (EALREADY);
2006	}
2007
2008	if ((retval = mp_cpu_configure(cpuid)) != 0) {
2009		return (retval);
2010	}
2011
2012	cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF;
2013	cpu_set_state(cpu[cpuid]);
2014	retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG);
2015	if (retval != 0)
2016		(void) mp_cpu_unconfigure(cpuid);
2017
2018	return (retval);
2019}
2020
2021/*
2022 * Routine used to cleanup a CPU that has been powered off.  This will
2023 * destroy all per-cpu information related to this cpu.
2024 */
2025int
2026cpu_unconfigure(int cpuid)
2027{
2028	int error;
2029
2030	ASSERT(MUTEX_HELD(&cpu_lock));
2031
2032	if (cpu[cpuid] == NULL) {
2033		return (ENODEV);
2034	}
2035
2036	if (cpu[cpuid]->cpu_flags == 0) {
2037		return (EALREADY);
2038	}
2039
2040	if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) {
2041		return (EBUSY);
2042	}
2043
2044	if (cpu[cpuid]->cpu_props != NULL) {
2045		(void) nvlist_free(cpu[cpuid]->cpu_props);
2046		cpu[cpuid]->cpu_props = NULL;
2047	}
2048
2049	error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG);
2050
2051	if (error != 0)
2052		return (error);
2053
2054	return (mp_cpu_unconfigure(cpuid));
2055}
2056
2057/*
2058 * Routines for registering and de-registering cpu_setup callback functions.
2059 *
2060 * Caller's context
2061 *	These routines must not be called from a driver's attach(9E) or
2062 *	detach(9E) entry point.
2063 *
2064 * NOTE: CPU callbacks should not block. They are called with cpu_lock held.
2065 */
2066
2067/*
2068 * Ideally, these would be dynamically allocated and put into a linked
2069 * list; however that is not feasible because the registration routine
2070 * has to be available before the kmem allocator is working (in fact,
2071 * it is called by the kmem allocator init code).  In any case, there
2072 * are quite a few extra entries for future users.
2073 */
2074#define	NCPU_SETUPS	20
2075
2076struct cpu_setup {
2077	cpu_setup_func_t *func;
2078	void *arg;
2079} cpu_setups[NCPU_SETUPS];
2080
2081void
2082register_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2083{
2084	int i;
2085
2086	ASSERT(MUTEX_HELD(&cpu_lock));
2087
2088	for (i = 0; i < NCPU_SETUPS; i++)
2089		if (cpu_setups[i].func == NULL)
2090			break;
2091	if (i >= NCPU_SETUPS)
2092		cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries");
2093
2094	cpu_setups[i].func = func;
2095	cpu_setups[i].arg = arg;
2096}
2097
2098void
2099unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2100{
2101	int i;
2102
2103	ASSERT(MUTEX_HELD(&cpu_lock));
2104
2105	for (i = 0; i < NCPU_SETUPS; i++)
2106		if ((cpu_setups[i].func == func) &&
2107		    (cpu_setups[i].arg == arg))
2108			break;
2109	if (i >= NCPU_SETUPS)
2110		cmn_err(CE_PANIC, "Could not find cpu_setup callback to "
2111		    "deregister");
2112
2113	cpu_setups[i].func = NULL;
2114	cpu_setups[i].arg = 0;
2115}
2116
2117/*
2118 * Call any state change hooks for this CPU, ignore any errors.
2119 */
2120void
2121cpu_state_change_notify(int id, cpu_setup_t what)
2122{
2123	int i;
2124
2125	ASSERT(MUTEX_HELD(&cpu_lock));
2126
2127	for (i = 0; i < NCPU_SETUPS; i++) {
2128		if (cpu_setups[i].func != NULL) {
2129			cpu_setups[i].func(what, id, cpu_setups[i].arg);
2130		}
2131	}
2132}
2133
2134/*
2135 * Call any state change hooks for this CPU, undo it if error found.
2136 */
2137static int
2138cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo)
2139{
2140	int i;
2141	int retval = 0;
2142
2143	ASSERT(MUTEX_HELD(&cpu_lock));
2144
2145	for (i = 0; i < NCPU_SETUPS; i++) {
2146		if (cpu_setups[i].func != NULL) {
2147			retval = cpu_setups[i].func(what, id,
2148			    cpu_setups[i].arg);
2149			if (retval) {
2150				for (i--; i >= 0; i--) {
2151					if (cpu_setups[i].func != NULL)
2152						cpu_setups[i].func(undo,
2153						    id, cpu_setups[i].arg);
2154				}
2155				break;
2156			}
2157		}
2158	}
2159	return (retval);
2160}
2161
2162/*
2163 * Export information about this CPU via the kstat mechanism.
2164 */
2165static struct {
2166	kstat_named_t ci_state;
2167	kstat_named_t ci_state_begin;
2168	kstat_named_t ci_cpu_type;
2169	kstat_named_t ci_fpu_type;
2170	kstat_named_t ci_clock_MHz;
2171	kstat_named_t ci_chip_id;
2172	kstat_named_t ci_implementation;
2173	kstat_named_t ci_brandstr;
2174	kstat_named_t ci_core_id;
2175	kstat_named_t ci_curr_clock_Hz;
2176	kstat_named_t ci_supp_freq_Hz;
2177	kstat_named_t ci_pg_id;
2178#if defined(__sparcv9)
2179	kstat_named_t ci_device_ID;
2180	kstat_named_t ci_cpu_fru;
2181#endif
2182#if defined(__x86)
2183	kstat_named_t ci_vendorstr;
2184	kstat_named_t ci_family;
2185	kstat_named_t ci_model;
2186	kstat_named_t ci_step;
2187	kstat_named_t ci_clogid;
2188	kstat_named_t ci_pkg_core_id;
2189	kstat_named_t ci_ncpuperchip;
2190	kstat_named_t ci_ncoreperchip;
2191	kstat_named_t ci_max_cstates;
2192	kstat_named_t ci_curr_cstate;
2193	kstat_named_t ci_cacheid;
2194	kstat_named_t ci_sktstr;
2195#endif
2196} cpu_info_template = {
2197	{ "state",			KSTAT_DATA_CHAR },
2198	{ "state_begin",		KSTAT_DATA_LONG },
2199	{ "cpu_type",			KSTAT_DATA_CHAR },
2200	{ "fpu_type",			KSTAT_DATA_CHAR },
2201	{ "clock_MHz",			KSTAT_DATA_LONG },
2202	{ "chip_id",			KSTAT_DATA_LONG },
2203	{ "implementation",		KSTAT_DATA_STRING },
2204	{ "brand",			KSTAT_DATA_STRING },
2205	{ "core_id",			KSTAT_DATA_LONG },
2206	{ "current_clock_Hz",		KSTAT_DATA_UINT64 },
2207	{ "supported_frequencies_Hz",	KSTAT_DATA_STRING },
2208	{ "pg_id",			KSTAT_DATA_LONG },
2209#if defined(__sparcv9)
2210	{ "device_ID",			KSTAT_DATA_UINT64 },
2211	{ "cpu_fru",			KSTAT_DATA_STRING },
2212#endif
2213#if defined(__x86)
2214	{ "vendor_id",			KSTAT_DATA_STRING },
2215	{ "family",			KSTAT_DATA_INT32 },
2216	{ "model",			KSTAT_DATA_INT32 },
2217	{ "stepping",			KSTAT_DATA_INT32 },
2218	{ "clog_id",			KSTAT_DATA_INT32 },
2219	{ "pkg_core_id",		KSTAT_DATA_LONG },
2220	{ "ncpu_per_chip",		KSTAT_DATA_INT32 },
2221	{ "ncore_per_chip",		KSTAT_DATA_INT32 },
2222	{ "supported_max_cstates",	KSTAT_DATA_INT32 },
2223	{ "current_cstate",		KSTAT_DATA_INT32 },
2224	{ "cache_id",			KSTAT_DATA_INT32 },
2225	{ "socket_type",		KSTAT_DATA_STRING },
2226#endif
2227};
2228
2229static kmutex_t cpu_info_template_lock;
2230
2231static int
2232cpu_info_kstat_update(kstat_t *ksp, int rw)
2233{
2234	cpu_t	*cp = ksp->ks_private;
2235	const char *pi_state;
2236
2237	if (rw == KSTAT_WRITE)
2238		return (EACCES);
2239
2240#if defined(__x86)
2241	/* Is the cpu still initialising itself? */
2242	if (cpuid_checkpass(cp, 1) == 0)
2243		return (ENXIO);
2244#endif
2245	switch (cp->cpu_type_info.pi_state) {
2246	case P_ONLINE:
2247		pi_state = PS_ONLINE;
2248		break;
2249	case P_POWEROFF:
2250		pi_state = PS_POWEROFF;
2251		break;
2252	case P_NOINTR:
2253		pi_state = PS_NOINTR;
2254		break;
2255	case P_FAULTED:
2256		pi_state = PS_FAULTED;
2257		break;
2258	case P_SPARE:
2259		pi_state = PS_SPARE;
2260		break;
2261	case P_OFFLINE:
2262		pi_state = PS_OFFLINE;
2263		break;
2264	default:
2265		pi_state = "unknown";
2266	}
2267	(void) strcpy(cpu_info_template.ci_state.value.c, pi_state);
2268	cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin;
2269	(void) strncpy(cpu_info_template.ci_cpu_type.value.c,
2270	    cp->cpu_type_info.pi_processor_type, 15);
2271	(void) strncpy(cpu_info_template.ci_fpu_type.value.c,
2272	    cp->cpu_type_info.pi_fputypes, 15);
2273	cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock;
2274	cpu_info_template.ci_chip_id.value.l =
2275	    pg_plat_hw_instance_id(cp, PGHW_CHIP);
2276	kstat_named_setstr(&cpu_info_template.ci_implementation,
2277	    cp->cpu_idstr);
2278	kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr);
2279	cpu_info_template.ci_core_id.value.l = pg_plat_get_core_id(cp);
2280	cpu_info_template.ci_curr_clock_Hz.value.ui64 =
2281	    cp->cpu_curr_clock;
2282	cpu_info_template.ci_pg_id.value.l =
2283	    cp->cpu_pg && cp->cpu_pg->cmt_lineage ?
2284	    cp->cpu_pg->cmt_lineage->pg_id : -1;
2285	kstat_named_setstr(&cpu_info_template.ci_supp_freq_Hz,
2286	    cp->cpu_supp_freqs);
2287#if defined(__sparcv9)
2288	cpu_info_template.ci_device_ID.value.ui64 =
2289	    cpunodes[cp->cpu_id].device_id;
2290	kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp));
2291#endif
2292#if defined(__x86)
2293	kstat_named_setstr(&cpu_info_template.ci_vendorstr,
2294	    cpuid_getvendorstr(cp));
2295	cpu_info_template.ci_family.value.l = cpuid_getfamily(cp);
2296	cpu_info_template.ci_model.value.l = cpuid_getmodel(cp);
2297	cpu_info_template.ci_step.value.l = cpuid_getstep(cp);
2298	cpu_info_template.ci_clogid.value.l = cpuid_get_clogid(cp);
2299	cpu_info_template.ci_ncpuperchip.value.l = cpuid_get_ncpu_per_chip(cp);
2300	cpu_info_template.ci_ncoreperchip.value.l =
2301	    cpuid_get_ncore_per_chip(cp);
2302	cpu_info_template.ci_pkg_core_id.value.l = cpuid_get_pkgcoreid(cp);
2303	cpu_info_template.ci_max_cstates.value.l = cp->cpu_m.max_cstates;
2304	cpu_info_template.ci_curr_cstate.value.l = cpu_idle_get_cpu_state(cp);
2305	cpu_info_template.ci_cacheid.value.i32 = cpuid_get_cacheid(cp);
2306	kstat_named_setstr(&cpu_info_template.ci_sktstr,
2307	    cpuid_getsocketstr(cp));
2308#endif
2309
2310	return (0);
2311}
2312
2313static void
2314cpu_info_kstat_create(cpu_t *cp)
2315{
2316	zoneid_t zoneid;
2317
2318	ASSERT(MUTEX_HELD(&cpu_lock));
2319
2320	if (pool_pset_enabled())
2321		zoneid = GLOBAL_ZONEID;
2322	else
2323		zoneid = ALL_ZONES;
2324	if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id,
2325	    NULL, "misc", KSTAT_TYPE_NAMED,
2326	    sizeof (cpu_info_template) / sizeof (kstat_named_t),
2327	    KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_VAR_SIZE, zoneid)) != NULL) {
2328		cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN;
2329#if defined(__sparcv9)
2330		cp->cpu_info_kstat->ks_data_size +=
2331		    strlen(cpu_fru_fmri(cp)) + 1;
2332#endif
2333#if defined(__x86)
2334		cp->cpu_info_kstat->ks_data_size += X86_VENDOR_STRLEN;
2335#endif
2336		if (cp->cpu_supp_freqs != NULL)
2337			cp->cpu_info_kstat->ks_data_size +=
2338			    strlen(cp->cpu_supp_freqs) + 1;
2339		cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock;
2340		cp->cpu_info_kstat->ks_data = &cpu_info_template;
2341		cp->cpu_info_kstat->ks_private = cp;
2342		cp->cpu_info_kstat->ks_update = cpu_info_kstat_update;
2343		kstat_install(cp->cpu_info_kstat);
2344	}
2345}
2346
2347static void
2348cpu_info_kstat_destroy(cpu_t *cp)
2349{
2350	ASSERT(MUTEX_HELD(&cpu_lock));
2351
2352	kstat_delete(cp->cpu_info_kstat);
2353	cp->cpu_info_kstat = NULL;
2354}
2355
2356/*
2357 * Create and install kstats for the boot CPU.
2358 */
2359void
2360cpu_kstat_init(cpu_t *cp)
2361{
2362	mutex_enter(&cpu_lock);
2363	cpu_info_kstat_create(cp);
2364	cpu_stats_kstat_create(cp);
2365	cpu_create_intrstat(cp);
2366	cpu_set_state(cp);
2367	mutex_exit(&cpu_lock);
2368}
2369
2370/*
2371 * Make visible to the zone that subset of the cpu information that would be
2372 * initialized when a cpu is configured (but still offline).
2373 */
2374void
2375cpu_visibility_configure(cpu_t *cp, zone_t *zone)
2376{
2377	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2378
2379	ASSERT(MUTEX_HELD(&cpu_lock));
2380	ASSERT(pool_pset_enabled());
2381	ASSERT(cp != NULL);
2382
2383	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2384		zone->zone_ncpus++;
2385		ASSERT(zone->zone_ncpus <= ncpus);
2386	}
2387	if (cp->cpu_info_kstat != NULL)
2388		kstat_zone_add(cp->cpu_info_kstat, zoneid);
2389}
2390
2391/*
2392 * Make visible to the zone that subset of the cpu information that would be
2393 * initialized when a previously configured cpu is onlined.
2394 */
2395void
2396cpu_visibility_online(cpu_t *cp, zone_t *zone)
2397{
2398	kstat_t *ksp;
2399	char name[sizeof ("cpu_stat") + 10];	/* enough for 32-bit cpuids */
2400	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2401	processorid_t cpun;
2402
2403	ASSERT(MUTEX_HELD(&cpu_lock));
2404	ASSERT(pool_pset_enabled());
2405	ASSERT(cp != NULL);
2406	ASSERT(cpu_is_active(cp));
2407
2408	cpun = cp->cpu_id;
2409	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2410		zone->zone_ncpus_online++;
2411		ASSERT(zone->zone_ncpus_online <= ncpus_online);
2412	}
2413	(void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2414	if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2415	    != NULL) {
2416		kstat_zone_add(ksp, zoneid);
2417		kstat_rele(ksp);
2418	}
2419	if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2420		kstat_zone_add(ksp, zoneid);
2421		kstat_rele(ksp);
2422	}
2423	if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2424		kstat_zone_add(ksp, zoneid);
2425		kstat_rele(ksp);
2426	}
2427	if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2428	    NULL) {
2429		kstat_zone_add(ksp, zoneid);
2430		kstat_rele(ksp);
2431	}
2432}
2433
2434/*
2435 * Update relevant kstats such that cpu is now visible to processes
2436 * executing in specified zone.
2437 */
2438void
2439cpu_visibility_add(cpu_t *cp, zone_t *zone)
2440{
2441	cpu_visibility_configure(cp, zone);
2442	if (cpu_is_active(cp))
2443		cpu_visibility_online(cp, zone);
2444}
2445
2446/*
2447 * Make invisible to the zone that subset of the cpu information that would be
2448 * torn down when a previously offlined cpu is unconfigured.
2449 */
2450void
2451cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone)
2452{
2453	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2454
2455	ASSERT(MUTEX_HELD(&cpu_lock));
2456	ASSERT(pool_pset_enabled());
2457	ASSERT(cp != NULL);
2458
2459	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2460		ASSERT(zone->zone_ncpus != 0);
2461		zone->zone_ncpus--;
2462	}
2463	if (cp->cpu_info_kstat)
2464		kstat_zone_remove(cp->cpu_info_kstat, zoneid);
2465}
2466
2467/*
2468 * Make invisible to the zone that subset of the cpu information that would be
2469 * torn down when a cpu is offlined (but still configured).
2470 */
2471void
2472cpu_visibility_offline(cpu_t *cp, zone_t *zone)
2473{
2474	kstat_t *ksp;
2475	char name[sizeof ("cpu_stat") + 10];	/* enough for 32-bit cpuids */
2476	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2477	processorid_t cpun;
2478
2479	ASSERT(MUTEX_HELD(&cpu_lock));
2480	ASSERT(pool_pset_enabled());
2481	ASSERT(cp != NULL);
2482	ASSERT(cpu_is_active(cp));
2483
2484	cpun = cp->cpu_id;
2485	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2486		ASSERT(zone->zone_ncpus_online != 0);
2487		zone->zone_ncpus_online--;
2488	}
2489
2490	if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2491	    NULL) {
2492		kstat_zone_remove(ksp, zoneid);
2493		kstat_rele(ksp);
2494	}
2495	if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2496		kstat_zone_remove(ksp, zoneid);
2497		kstat_rele(ksp);
2498	}
2499	if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2500		kstat_zone_remove(ksp, zoneid);
2501		kstat_rele(ksp);
2502	}
2503	(void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2504	if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2505	    != NULL) {
2506		kstat_zone_remove(ksp, zoneid);
2507		kstat_rele(ksp);
2508	}
2509}
2510
2511/*
2512 * Update relevant kstats such that cpu is no longer visible to processes
2513 * executing in specified zone.
2514 */
2515void
2516cpu_visibility_remove(cpu_t *cp, zone_t *zone)
2517{
2518	if (cpu_is_active(cp))
2519		cpu_visibility_offline(cp, zone);
2520	cpu_visibility_unconfigure(cp, zone);
2521}
2522
2523/*
2524 * Bind a thread to a CPU as requested.
2525 */
2526int
2527cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind,
2528    int *error)
2529{
2530	processorid_t	binding;
2531	cpu_t		*cp = NULL;
2532
2533	ASSERT(MUTEX_HELD(&cpu_lock));
2534	ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock));
2535
2536	thread_lock(tp);
2537
2538	/*
2539	 * Record old binding, but change the obind, which was initialized
2540	 * to PBIND_NONE, only if this thread has a binding.  This avoids
2541	 * reporting PBIND_NONE for a process when some LWPs are bound.
2542	 */
2543	binding = tp->t_bind_cpu;
2544	if (binding != PBIND_NONE)
2545		*obind = binding;	/* record old binding */
2546
2547	switch (bind) {
2548	case PBIND_QUERY:
2549		/* Just return the old binding */
2550		thread_unlock(tp);
2551		return (0);
2552
2553	case PBIND_QUERY_TYPE:
2554		/* Return the binding type */
2555		*obind = TB_CPU_IS_SOFT(tp) ? PBIND_SOFT : PBIND_HARD;
2556		thread_unlock(tp);
2557		return (0);
2558
2559	case PBIND_SOFT:
2560		/*
2561		 *  Set soft binding for this thread and return the actual
2562		 *  binding
2563		 */
2564		TB_CPU_SOFT_SET(tp);
2565		thread_unlock(tp);
2566		return (0);
2567
2568	case PBIND_HARD:
2569		/*
2570		 *  Set hard binding for this thread and return the actual
2571		 *  binding
2572		 */
2573		TB_CPU_HARD_SET(tp);
2574		thread_unlock(tp);
2575		return (0);
2576
2577	default:
2578		break;
2579	}
2580
2581	/*
2582	 * If this thread/LWP cannot be bound because of permission
2583	 * problems, just note that and return success so that the
2584	 * other threads/LWPs will be bound.  This is the way
2585	 * processor_bind() is defined to work.
2586	 *
2587	 * Binding will get EPERM if the thread is of system class
2588	 * or hasprocperm() fails.
2589	 */
2590	if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) {
2591		*error = EPERM;
2592		thread_unlock(tp);
2593		return (0);
2594	}
2595
2596	binding = bind;
2597	if (binding != PBIND_NONE) {
2598		cp = cpu_get((processorid_t)binding);
2599		/*
2600		 * Make sure binding is valid and is in right partition.
2601		 */
2602		if (cp == NULL || tp->t_cpupart != cp->cpu_part) {
2603			*error = EINVAL;
2604			thread_unlock(tp);
2605			return (0);
2606		}
2607	}
2608	tp->t_bind_cpu = binding;	/* set new binding */
2609
2610	/*
2611	 * If there is no system-set reason for affinity, set
2612	 * the t_bound_cpu field to reflect the binding.
2613	 */
2614	if (tp->t_affinitycnt == 0) {
2615		if (binding == PBIND_NONE) {
2616			/*
2617			 * We may need to adjust disp_max_unbound_pri
2618			 * since we're becoming unbound.
2619			 */
2620			disp_adjust_unbound_pri(tp);
2621
2622			tp->t_bound_cpu = NULL;	/* set new binding */
2623
2624			/*
2625			 * Move thread to lgroup with strongest affinity
2626			 * after unbinding
2627			 */
2628			if (tp->t_lgrp_affinity)
2629				lgrp_move_thread(tp,
2630				    lgrp_choose(tp, tp->t_cpupart), 1);
2631
2632			if (tp->t_state == TS_ONPROC &&
2633			    tp->t_cpu->cpu_part != tp->t_cpupart)
2634				cpu_surrender(tp);
2635		} else {
2636			lpl_t	*lpl;
2637
2638			tp->t_bound_cpu = cp;
2639			ASSERT(cp->cpu_lpl != NULL);
2640
2641			/*
2642			 * Set home to lgroup with most affinity containing CPU
2643			 * that thread is being bound or minimum bounding
2644			 * lgroup if no affinities set
2645			 */
2646			if (tp->t_lgrp_affinity)
2647				lpl = lgrp_affinity_best(tp, tp->t_cpupart,
2648				    LGRP_NONE, B_FALSE);
2649			else
2650				lpl = cp->cpu_lpl;
2651
2652			if (tp->t_lpl != lpl) {
2653				/* can't grab cpu_lock */
2654				lgrp_move_thread(tp, lpl, 1);
2655			}
2656
2657			/*
2658			 * Make the thread switch to the bound CPU.
2659			 * If the thread is runnable, we need to
2660			 * requeue it even if t_cpu is already set
2661			 * to the right CPU, since it may be on a
2662			 * kpreempt queue and need to move to a local
2663			 * queue.  We could check t_disp_queue to
2664			 * avoid unnecessary overhead if it's already
2665			 * on the right queue, but since this isn't
2666			 * a performance-critical operation it doesn't
2667			 * seem worth the extra code and complexity.
2668			 *
2669			 * If the thread is weakbound to the cpu then it will
2670			 * resist the new binding request until the weak
2671			 * binding drops.  The cpu_surrender or requeueing
2672			 * below could be skipped in such cases (since it
2673			 * will have no effect), but that would require
2674			 * thread_allowmigrate to acquire thread_lock so
2675			 * we'll take the very occasional hit here instead.
2676			 */
2677			if (tp->t_state == TS_ONPROC) {
2678				cpu_surrender(tp);
2679			} else if (tp->t_state == TS_RUN) {
2680				cpu_t *ocp = tp->t_cpu;
2681
2682				(void) dispdeq(tp);
2683				setbackdq(tp);
2684				/*
2685				 * Either on the bound CPU's disp queue now,
2686				 * or swapped out or on the swap queue.
2687				 */
2688				ASSERT(tp->t_disp_queue == cp->cpu_disp ||
2689				    tp->t_weakbound_cpu == ocp ||
2690				    (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ))
2691				    != TS_LOAD);
2692			}
2693		}
2694	}
2695
2696	/*
2697	 * Our binding has changed; set TP_CHANGEBIND.
2698	 */
2699	tp->t_proc_flag |= TP_CHANGEBIND;
2700	aston(tp);
2701
2702	thread_unlock(tp);
2703
2704	return (0);
2705}
2706
2707#if CPUSET_WORDS > 1
2708
2709/*
2710 * Functions for implementing cpuset operations when a cpuset is more
2711 * than one word.  On platforms where a cpuset is a single word these
2712 * are implemented as macros in cpuvar.h.
2713 */
2714
2715void
2716cpuset_all(cpuset_t *s)
2717{
2718	int i;
2719
2720	for (i = 0; i < CPUSET_WORDS; i++)
2721		s->cpub[i] = ~0UL;
2722}
2723
2724void
2725cpuset_all_but(cpuset_t *s, uint_t cpu)
2726{
2727	cpuset_all(s);
2728	CPUSET_DEL(*s, cpu);
2729}
2730
2731void
2732cpuset_only(cpuset_t *s, uint_t cpu)
2733{
2734	CPUSET_ZERO(*s);
2735	CPUSET_ADD(*s, cpu);
2736}
2737
2738int
2739cpuset_isnull(cpuset_t *s)
2740{
2741	int i;
2742
2743	for (i = 0; i < CPUSET_WORDS; i++)
2744		if (s->cpub[i] != 0)
2745			return (0);
2746	return (1);
2747}
2748
2749int
2750cpuset_cmp(cpuset_t *s1, cpuset_t *s2)
2751{
2752	int i;
2753
2754	for (i = 0; i < CPUSET_WORDS; i++)
2755		if (s1->cpub[i] != s2->cpub[i])
2756			return (0);
2757	return (1);
2758}
2759
2760uint_t
2761cpuset_find(cpuset_t *s)
2762{
2763
2764	uint_t	i;
2765	uint_t	cpu = (uint_t)-1;
2766
2767	/*
2768	 * Find a cpu in the cpuset
2769	 */
2770	for (i = 0; i < CPUSET_WORDS; i++) {
2771		cpu = (uint_t)(lowbit(s->cpub[i]) - 1);
2772		if (cpu != (uint_t)-1) {
2773			cpu += i * BT_NBIPUL;
2774			break;
2775		}
2776	}
2777	return (cpu);
2778}
2779
2780void
2781cpuset_bounds(cpuset_t *s, uint_t *smallestid, uint_t *largestid)
2782{
2783	int	i, j;
2784	uint_t	bit;
2785
2786	/*
2787	 * First, find the smallest cpu id in the set.
2788	 */
2789	for (i = 0; i < CPUSET_WORDS; i++) {
2790		if (s->cpub[i] != 0) {
2791			bit = (uint_t)(lowbit(s->cpub[i]) - 1);
2792			ASSERT(bit != (uint_t)-1);
2793			*smallestid = bit + (i * BT_NBIPUL);
2794
2795			/*
2796			 * Now find the largest cpu id in
2797			 * the set and return immediately.
2798			 * Done in an inner loop to avoid
2799			 * having to break out of the first
2800			 * loop.
2801			 */
2802			for (j = CPUSET_WORDS - 1; j >= i; j--) {
2803				if (s->cpub[j] != 0) {
2804					bit = (uint_t)(highbit(s->cpub[j]) - 1);
2805					ASSERT(bit != (uint_t)-1);
2806					*largestid = bit + (j * BT_NBIPUL);
2807					ASSERT(*largestid >= *smallestid);
2808					return;
2809				}
2810			}
2811
2812			/*
2813			 * If this code is reached, a
2814			 * smallestid was found, but not a
2815			 * largestid. The cpuset must have
2816			 * been changed during the course
2817			 * of this function call.
2818			 */
2819			ASSERT(0);
2820		}
2821	}
2822	*smallestid = *largestid = CPUSET_NOTINSET;
2823}
2824
2825#endif	/* CPUSET_WORDS */
2826
2827/*
2828 * Unbind threads bound to specified CPU.
2829 *
2830 * If `unbind_all_threads' is true, unbind all user threads bound to a given
2831 * CPU. Otherwise unbind all soft-bound user threads.
2832 */
2833int
2834cpu_unbind(processorid_t cpu, boolean_t unbind_all_threads)
2835{
2836	processorid_t obind;
2837	kthread_t *tp;
2838	int ret = 0;
2839	proc_t *pp;
2840	int err, berr = 0;
2841
2842	ASSERT(MUTEX_HELD(&cpu_lock));
2843
2844	mutex_enter(&pidlock);
2845	for (pp = practive; pp != NULL; pp = pp->p_next) {
2846		mutex_enter(&pp->p_lock);
2847		tp = pp->p_tlist;
2848		/*
2849		 * Skip zombies, kernel processes, and processes in
2850		 * other zones, if called from a non-global zone.
2851		 */
2852		if (tp == NULL || (pp->p_flag & SSYS) ||
2853		    !HASZONEACCESS(curproc, pp->p_zone->zone_id)) {
2854			mutex_exit(&pp->p_lock);
2855			continue;
2856		}
2857		do {
2858			if (tp->t_bind_cpu != cpu)
2859				continue;
2860			/*
2861			 * Skip threads with hard binding when
2862			 * `unbind_all_threads' is not specified.
2863			 */
2864			if (!unbind_all_threads && TB_CPU_IS_HARD(tp))
2865				continue;
2866			err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr);
2867			if (ret == 0)
2868				ret = err;
2869		} while ((tp = tp->t_forw) != pp->p_tlist);
2870		mutex_exit(&pp->p_lock);
2871	}
2872	mutex_exit(&pidlock);
2873	if (ret == 0)
2874		ret = berr;
2875	return (ret);
2876}
2877
2878
2879/*
2880 * Destroy all remaining bound threads on a cpu.
2881 */
2882void
2883cpu_destroy_bound_threads(cpu_t *cp)
2884{
2885	extern id_t syscid;
2886	register kthread_id_t	t, tlist, tnext;
2887
2888	/*
2889	 * Destroy all remaining bound threads on the cpu.  This
2890	 * should include both the interrupt threads and the idle thread.
2891	 * This requires some care, since we need to traverse the
2892	 * thread list with the pidlock mutex locked, but thread_free
2893	 * also locks the pidlock mutex.  So, we collect the threads
2894	 * we're going to reap in a list headed by "tlist", then we
2895	 * unlock the pidlock mutex and traverse the tlist list,
2896	 * doing thread_free's on the thread's.	 Simple, n'est pas?
2897	 * Also, this depends on thread_free not mucking with the
2898	 * t_next and t_prev links of the thread.
2899	 */
2900
2901	if ((t = curthread) != NULL) {
2902
2903		tlist = NULL;
2904		mutex_enter(&pidlock);
2905		do {
2906			tnext = t->t_next;
2907			if (t->t_bound_cpu == cp) {
2908
2909				/*
2910				 * We've found a bound thread, carefully unlink
2911				 * it out of the thread list, and add it to
2912				 * our "tlist".	 We "know" we don't have to
2913				 * worry about unlinking curthread (the thread
2914				 * that is executing this code).
2915				 */
2916				t->t_next->t_prev = t->t_prev;
2917				t->t_prev->t_next = t->t_next;
2918				t->t_next = tlist;
2919				tlist = t;
2920				ASSERT(t->t_cid == syscid);
2921				/* wake up anyone blocked in thread_join */
2922				cv_broadcast(&t->t_joincv);
2923				/*
2924				 * t_lwp set by interrupt threads and not
2925				 * cleared.
2926				 */
2927				t->t_lwp = NULL;
2928				/*
2929				 * Pause and idle threads always have
2930				 * t_state set to TS_ONPROC.
2931				 */
2932				t->t_state = TS_FREE;
2933				t->t_prev = NULL;	/* Just in case */
2934			}
2935
2936		} while ((t = tnext) != curthread);
2937
2938		mutex_exit(&pidlock);
2939
2940		mutex_sync();
2941		for (t = tlist; t != NULL; t = tnext) {
2942			tnext = t->t_next;
2943			thread_free(t);
2944		}
2945	}
2946}
2947
2948/*
2949 * Update the cpu_supp_freqs of this cpu. This information is returned
2950 * as part of cpu_info kstats. If the cpu_info_kstat exists already, then
2951 * maintain the kstat data size.
2952 */
2953void
2954cpu_set_supp_freqs(cpu_t *cp, const char *freqs)
2955{
2956	char clkstr[sizeof ("18446744073709551615") + 1]; /* ui64 MAX */
2957	const char *lfreqs = clkstr;
2958	boolean_t kstat_exists = B_FALSE;
2959	kstat_t *ksp;
2960	size_t len;
2961
2962	/*
2963	 * A NULL pointer means we only support one speed.
2964	 */
2965	if (freqs == NULL)
2966		(void) snprintf(clkstr, sizeof (clkstr), "%"PRIu64,
2967		    cp->cpu_curr_clock);
2968	else
2969		lfreqs = freqs;
2970
2971	/*
2972	 * Make sure the frequency doesn't change while a snapshot is
2973	 * going on. Of course, we only need to worry about this if
2974	 * the kstat exists.
2975	 */
2976	if ((ksp = cp->cpu_info_kstat) != NULL) {
2977		mutex_enter(ksp->ks_lock);
2978		kstat_exists = B_TRUE;
2979	}
2980
2981	/*
2982	 * Free any previously allocated string and if the kstat
2983	 * already exists, then update its data size.
2984	 */
2985	if (cp->cpu_supp_freqs != NULL) {
2986		len = strlen(cp->cpu_supp_freqs) + 1;
2987		kmem_free(cp->cpu_supp_freqs, len);
2988		if (kstat_exists)
2989			ksp->ks_data_size -= len;
2990	}
2991
2992	/*
2993	 * Allocate the new string and set the pointer.
2994	 */
2995	len = strlen(lfreqs) + 1;
2996	cp->cpu_supp_freqs = kmem_alloc(len, KM_SLEEP);
2997	(void) strcpy(cp->cpu_supp_freqs, lfreqs);
2998
2999	/*
3000	 * If the kstat already exists then update the data size and
3001	 * free the lock.
3002	 */
3003	if (kstat_exists) {
3004		ksp->ks_data_size += len;
3005		mutex_exit(ksp->ks_lock);
3006	}
3007}
3008
3009/*
3010 * Indicate the current CPU's clock freqency (in Hz).
3011 * The calling context must be such that CPU references are safe.
3012 */
3013void
3014cpu_set_curr_clock(uint64_t new_clk)
3015{
3016	uint64_t old_clk;
3017
3018	old_clk = CPU->cpu_curr_clock;
3019	CPU->cpu_curr_clock = new_clk;
3020
3021	/*
3022	 * The cpu-change-speed DTrace probe exports the frequency in Hz
3023	 */
3024	DTRACE_PROBE3(cpu__change__speed, processorid_t, CPU->cpu_id,
3025	    uint64_t, old_clk, uint64_t, new_clk);
3026}
3027
3028/*
3029 * processor_info(2) and p_online(2) status support functions
3030 *   The constants returned by the cpu_get_state() and cpu_get_state_str() are
3031 *   for use in communicating processor state information to userland.  Kernel
3032 *   subsystems should only be using the cpu_flags value directly.  Subsystems
3033 *   modifying cpu_flags should record the state change via a call to the
3034 *   cpu_set_state().
3035 */
3036
3037/*
3038 * Update the pi_state of this CPU.  This function provides the CPU status for
3039 * the information returned by processor_info(2).
3040 */
3041void
3042cpu_set_state(cpu_t *cpu)
3043{
3044	ASSERT(MUTEX_HELD(&cpu_lock));
3045	cpu->cpu_type_info.pi_state = cpu_get_state(cpu);
3046	cpu->cpu_state_begin = gethrestime_sec();
3047	pool_cpu_mod = gethrtime();
3048}
3049
3050/*
3051 * Return offline/online/other status for the indicated CPU.  Use only for
3052 * communication with user applications; cpu_flags provides the in-kernel
3053 * interface.
3054 */
3055int
3056cpu_get_state(cpu_t *cpu)
3057{
3058	ASSERT(MUTEX_HELD(&cpu_lock));
3059	if (cpu->cpu_flags & CPU_POWEROFF)
3060		return (P_POWEROFF);
3061	else if (cpu->cpu_flags & CPU_FAULTED)
3062		return (P_FAULTED);
3063	else if (cpu->cpu_flags & CPU_SPARE)
3064		return (P_SPARE);
3065	else if ((cpu->cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
3066		return (P_OFFLINE);
3067	else if (cpu->cpu_flags & CPU_ENABLE)
3068		return (P_ONLINE);
3069	else
3070		return (P_NOINTR);
3071}
3072
3073/*
3074 * Return processor_info(2) state as a string.
3075 */
3076const char *
3077cpu_get_state_str(cpu_t *cpu)
3078{
3079	const char *string;
3080
3081	switch (cpu_get_state(cpu)) {
3082	case P_ONLINE:
3083		string = PS_ONLINE;
3084		break;
3085	case P_POWEROFF:
3086		string = PS_POWEROFF;
3087		break;
3088	case P_NOINTR:
3089		string = PS_NOINTR;
3090		break;
3091	case P_SPARE:
3092		string = PS_SPARE;
3093		break;
3094	case P_FAULTED:
3095		string = PS_FAULTED;
3096		break;
3097	case P_OFFLINE:
3098		string = PS_OFFLINE;
3099		break;
3100	default:
3101		string = "unknown";
3102		break;
3103	}
3104	return (string);
3105}
3106
3107/*
3108 * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named
3109 * kstats, respectively.  This is done when a CPU is initialized or placed
3110 * online via p_online(2).
3111 */
3112static void
3113cpu_stats_kstat_create(cpu_t *cp)
3114{
3115	int 	instance = cp->cpu_id;
3116	char 	*module = "cpu";
3117	char 	*class = "misc";
3118	kstat_t	*ksp;
3119	zoneid_t zoneid;
3120
3121	ASSERT(MUTEX_HELD(&cpu_lock));
3122
3123	if (pool_pset_enabled())
3124		zoneid = GLOBAL_ZONEID;
3125	else
3126		zoneid = ALL_ZONES;
3127	/*
3128	 * Create named kstats
3129	 */
3130#define	CPU_STATS_KS_CREATE(name, tsize, update_func)                    \
3131	ksp = kstat_create_zone(module, instance, (name), class,         \
3132	    KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0,       \
3133	    zoneid);                                                     \
3134	if (ksp != NULL) {                                               \
3135		ksp->ks_private = cp;                                    \
3136		ksp->ks_update = (update_func);                          \
3137		kstat_install(ksp);                                      \
3138	} else                                                           \
3139		cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \
3140		    module, instance, (name));
3141
3142	CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template),
3143	    cpu_sys_stats_ks_update);
3144	CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template),
3145	    cpu_vm_stats_ks_update);
3146
3147	/*
3148	 * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat.
3149	 */
3150	ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL,
3151	    "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid);
3152	if (ksp != NULL) {
3153		ksp->ks_update = cpu_stat_ks_update;
3154		ksp->ks_private = cp;
3155		kstat_install(ksp);
3156	}
3157}
3158
3159static void
3160cpu_stats_kstat_destroy(cpu_t *cp)
3161{
3162	char ks_name[KSTAT_STRLEN];
3163
3164	(void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id);
3165	kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name);
3166
3167	kstat_delete_byname("cpu", cp->cpu_id, "sys");
3168	kstat_delete_byname("cpu", cp->cpu_id, "vm");
3169}
3170
3171static int
3172cpu_sys_stats_ks_update(kstat_t *ksp, int rw)
3173{
3174	cpu_t *cp = (cpu_t *)ksp->ks_private;
3175	struct cpu_sys_stats_ks_data *csskd;
3176	cpu_sys_stats_t *css;
3177	hrtime_t msnsecs[NCMSTATES];
3178	int	i;
3179
3180	if (rw == KSTAT_WRITE)
3181		return (EACCES);
3182
3183	csskd = ksp->ks_data;
3184	css = &cp->cpu_stats.sys;
3185
3186	/*
3187	 * Read CPU mstate, but compare with the last values we
3188	 * received to make sure that the returned kstats never
3189	 * decrease.
3190	 */
3191
3192	get_cpu_mstate(cp, msnsecs);
3193	if (csskd->cpu_nsec_idle.value.ui64 > msnsecs[CMS_IDLE])
3194		msnsecs[CMS_IDLE] = csskd->cpu_nsec_idle.value.ui64;
3195	if (csskd->cpu_nsec_user.value.ui64 > msnsecs[CMS_USER])
3196		msnsecs[CMS_USER] = csskd->cpu_nsec_user.value.ui64;
3197	if (csskd->cpu_nsec_kernel.value.ui64 > msnsecs[CMS_SYSTEM])
3198		msnsecs[CMS_SYSTEM] = csskd->cpu_nsec_kernel.value.ui64;
3199
3200	bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data,
3201	    sizeof (cpu_sys_stats_ks_data_template));
3202
3203	csskd->cpu_ticks_wait.value.ui64 = 0;
3204	csskd->wait_ticks_io.value.ui64 = 0;
3205
3206	csskd->cpu_nsec_idle.value.ui64 = msnsecs[CMS_IDLE];
3207	csskd->cpu_nsec_user.value.ui64 = msnsecs[CMS_USER];
3208	csskd->cpu_nsec_kernel.value.ui64 = msnsecs[CMS_SYSTEM];
3209	csskd->cpu_ticks_idle.value.ui64 =
3210	    NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64);
3211	csskd->cpu_ticks_user.value.ui64 =
3212	    NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64);
3213	csskd->cpu_ticks_kernel.value.ui64 =
3214	    NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64);
3215	csskd->cpu_nsec_dtrace.value.ui64 = cp->cpu_dtrace_nsec;
3216	csskd->dtrace_probes.value.ui64 = cp->cpu_dtrace_probes;
3217	csskd->cpu_nsec_intr.value.ui64 = cp->cpu_intrlast;
3218	csskd->cpu_load_intr.value.ui64 = cp->cpu_intrload;
3219	csskd->bread.value.ui64 = css->bread;
3220	csskd->bwrite.value.ui64 = css->bwrite;
3221	csskd->lread.value.ui64 = css->lread;
3222	csskd->lwrite.value.ui64 = css->lwrite;
3223	csskd->phread.value.ui64 = css->phread;
3224	csskd->phwrite.value.ui64 = css->phwrite;
3225	csskd->pswitch.value.ui64 = css->pswitch;
3226	csskd->trap.value.ui64 = css->trap;
3227	csskd->intr.value.ui64 = 0;
3228	for (i = 0; i < PIL_MAX; i++)
3229		csskd->intr.value.ui64 += css->intr[i];
3230	csskd->syscall.value.ui64 = css->syscall;
3231	csskd->sysread.value.ui64 = css->sysread;
3232	csskd->syswrite.value.ui64 = css->syswrite;
3233	csskd->sysfork.value.ui64 = css->sysfork;
3234	csskd->sysvfork.value.ui64 = css->sysvfork;
3235	csskd->sysexec.value.ui64 = css->sysexec;
3236	csskd->readch.value.ui64 = css->readch;
3237	csskd->writech.value.ui64 = css->writech;
3238	csskd->rcvint.value.ui64 = css->rcvint;
3239	csskd->xmtint.value.ui64 = css->xmtint;
3240	csskd->mdmint.value.ui64 = css->mdmint;
3241	csskd->rawch.value.ui64 = css->rawch;
3242	csskd->canch.value.ui64 = css->canch;
3243	csskd->outch.value.ui64 = css->outch;
3244	csskd->msg.value.ui64 = css->msg;
3245	csskd->sema.value.ui64 = css->sema;
3246	csskd->namei.value.ui64 = css->namei;
3247	csskd->ufsiget.value.ui64 = css->ufsiget;
3248	csskd->ufsdirblk.value.ui64 = css->ufsdirblk;
3249	csskd->ufsipage.value.ui64 = css->ufsipage;
3250	csskd->ufsinopage.value.ui64 = css->ufsinopage;
3251	csskd->procovf.value.ui64 = css->procovf;
3252	csskd->intrthread.value.ui64 = 0;
3253	for (i = 0; i < LOCK_LEVEL - 1; i++)
3254		csskd->intrthread.value.ui64 += css->intr[i];
3255	csskd->intrblk.value.ui64 = css->intrblk;
3256	csskd->intrunpin.value.ui64 = css->intrunpin;
3257	csskd->idlethread.value.ui64 = css->idlethread;
3258	csskd->inv_swtch.value.ui64 = css->inv_swtch;
3259	csskd->nthreads.value.ui64 = css->nthreads;
3260	csskd->cpumigrate.value.ui64 = css->cpumigrate;
3261	csskd->xcalls.value.ui64 = css->xcalls;
3262	csskd->mutex_adenters.value.ui64 = css->mutex_adenters;
3263	csskd->rw_rdfails.value.ui64 = css->rw_rdfails;
3264	csskd->rw_wrfails.value.ui64 = css->rw_wrfails;
3265	csskd->modload.value.ui64 = css->modload;
3266	csskd->modunload.value.ui64 = css->modunload;
3267	csskd->bawrite.value.ui64 = css->bawrite;
3268	csskd->iowait.value.ui64 = css->iowait;
3269
3270	return (0);
3271}
3272
3273static int
3274cpu_vm_stats_ks_update(kstat_t *ksp, int rw)
3275{
3276	cpu_t *cp = (cpu_t *)ksp->ks_private;
3277	struct cpu_vm_stats_ks_data *cvskd;
3278	cpu_vm_stats_t *cvs;
3279
3280	if (rw == KSTAT_WRITE)
3281		return (EACCES);
3282
3283	cvs = &cp->cpu_stats.vm;
3284	cvskd = ksp->ks_data;
3285
3286	bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data,
3287	    sizeof (cpu_vm_stats_ks_data_template));
3288	cvskd->pgrec.value.ui64 = cvs->pgrec;
3289	cvskd->pgfrec.value.ui64 = cvs->pgfrec;
3290	cvskd->pgin.value.ui64 = cvs->pgin;
3291	cvskd->pgpgin.value.ui64 = cvs->pgpgin;
3292	cvskd->pgout.value.ui64 = cvs->pgout;
3293	cvskd->pgpgout.value.ui64 = cvs->pgpgout;
3294	cvskd->swapin.value.ui64 = cvs->swapin;
3295	cvskd->pgswapin.value.ui64 = cvs->pgswapin;
3296	cvskd->swapout.value.ui64 = cvs->swapout;
3297	cvskd->pgswapout.value.ui64 = cvs->pgswapout;
3298	cvskd->zfod.value.ui64 = cvs->zfod;
3299	cvskd->dfree.value.ui64 = cvs->dfree;
3300	cvskd->scan.value.ui64 = cvs->scan;
3301	cvskd->rev.value.ui64 = cvs->rev;
3302	cvskd->hat_fault.value.ui64 = cvs->hat_fault;
3303	cvskd->as_fault.value.ui64 = cvs->as_fault;
3304	cvskd->maj_fault.value.ui64 = cvs->maj_fault;
3305	cvskd->cow_fault.value.ui64 = cvs->cow_fault;
3306	cvskd->prot_fault.value.ui64 = cvs->prot_fault;
3307	cvskd->softlock.value.ui64 = cvs->softlock;
3308	cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt;
3309	cvskd->pgrrun.value.ui64 = cvs->pgrrun;
3310	cvskd->execpgin.value.ui64 = cvs->execpgin;
3311	cvskd->execpgout.value.ui64 = cvs->execpgout;
3312	cvskd->execfree.value.ui64 = cvs->execfree;
3313	cvskd->anonpgin.value.ui64 = cvs->anonpgin;
3314	cvskd->anonpgout.value.ui64 = cvs->anonpgout;
3315	cvskd->anonfree.value.ui64 = cvs->anonfree;
3316	cvskd->fspgin.value.ui64 = cvs->fspgin;
3317	cvskd->fspgout.value.ui64 = cvs->fspgout;
3318	cvskd->fsfree.value.ui64 = cvs->fsfree;
3319
3320	return (0);
3321}
3322
3323static int
3324cpu_stat_ks_update(kstat_t *ksp, int rw)
3325{
3326	cpu_stat_t *cso;
3327	cpu_t *cp;
3328	int i;
3329	hrtime_t msnsecs[NCMSTATES];
3330
3331	cso = (cpu_stat_t *)ksp->ks_data;
3332	cp = (cpu_t *)ksp->ks_private;
3333
3334	if (rw == KSTAT_WRITE)
3335		return (EACCES);
3336
3337	/*
3338	 * Read CPU mstate, but compare with the last values we
3339	 * received to make sure that the returned kstats never
3340	 * decrease.
3341	 */
3342
3343	get_cpu_mstate(cp, msnsecs);
3344	msnsecs[CMS_IDLE] = NSEC_TO_TICK(msnsecs[CMS_IDLE]);
3345	msnsecs[CMS_USER] = NSEC_TO_TICK(msnsecs[CMS_USER]);
3346	msnsecs[CMS_SYSTEM] = NSEC_TO_TICK(msnsecs[CMS_SYSTEM]);
3347	if (cso->cpu_sysinfo.cpu[CPU_IDLE] < msnsecs[CMS_IDLE])
3348		cso->cpu_sysinfo.cpu[CPU_IDLE] = msnsecs[CMS_IDLE];
3349	if (cso->cpu_sysinfo.cpu[CPU_USER] < msnsecs[CMS_USER])
3350		cso->cpu_sysinfo.cpu[CPU_USER] = msnsecs[CMS_USER];
3351	if (cso->cpu_sysinfo.cpu[CPU_KERNEL] < msnsecs[CMS_SYSTEM])
3352		cso->cpu_sysinfo.cpu[CPU_KERNEL] = msnsecs[CMS_SYSTEM];
3353	cso->cpu_sysinfo.cpu[CPU_WAIT] 	= 0;
3354	cso->cpu_sysinfo.wait[W_IO] 	= 0;
3355	cso->cpu_sysinfo.wait[W_SWAP]	= 0;
3356	cso->cpu_sysinfo.wait[W_PIO]	= 0;
3357	cso->cpu_sysinfo.bread 		= CPU_STATS(cp, sys.bread);
3358	cso->cpu_sysinfo.bwrite 	= CPU_STATS(cp, sys.bwrite);
3359	cso->cpu_sysinfo.lread 		= CPU_STATS(cp, sys.lread);
3360	cso->cpu_sysinfo.lwrite 	= CPU_STATS(cp, sys.lwrite);
3361	cso->cpu_sysinfo.phread 	= CPU_STATS(cp, sys.phread);
3362	cso->cpu_sysinfo.phwrite 	= CPU_STATS(cp, sys.phwrite);
3363	cso->cpu_sysinfo.pswitch 	= CPU_STATS(cp, sys.pswitch);
3364	cso->cpu_sysinfo.trap 		= CPU_STATS(cp, sys.trap);
3365	cso->cpu_sysinfo.intr		= 0;
3366	for (i = 0; i < PIL_MAX; i++)
3367		cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]);
3368	cso->cpu_sysinfo.syscall	= CPU_STATS(cp, sys.syscall);
3369	cso->cpu_sysinfo.sysread	= CPU_STATS(cp, sys.sysread);
3370	cso->cpu_sysinfo.syswrite	= CPU_STATS(cp, sys.syswrite);
3371	cso->cpu_sysinfo.sysfork	= CPU_STATS(cp, sys.sysfork);
3372	cso->cpu_sysinfo.sysvfork	= CPU_STATS(cp, sys.sysvfork);
3373	cso->cpu_sysinfo.sysexec	= CPU_STATS(cp, sys.sysexec);
3374	cso->cpu_sysinfo.readch		= CPU_STATS(cp, sys.readch);
3375	cso->cpu_sysinfo.writech	= CPU_STATS(cp, sys.writech);
3376	cso->cpu_sysinfo.rcvint		= CPU_STATS(cp, sys.rcvint);
3377	cso->cpu_sysinfo.xmtint		= CPU_STATS(cp, sys.xmtint);
3378	cso->cpu_sysinfo.mdmint		= CPU_STATS(cp, sys.mdmint);
3379	cso->cpu_sysinfo.rawch		= CPU_STATS(cp, sys.rawch);
3380	cso->cpu_sysinfo.canch		= CPU_STATS(cp, sys.canch);
3381	cso->cpu_sysinfo.outch		= CPU_STATS(cp, sys.outch);
3382	cso->cpu_sysinfo.msg		= CPU_STATS(cp, sys.msg);
3383	cso->cpu_sysinfo.sema		= CPU_STATS(cp, sys.sema);
3384	cso->cpu_sysinfo.namei		= CPU_STATS(cp, sys.namei);
3385	cso->cpu_sysinfo.ufsiget	= CPU_STATS(cp, sys.ufsiget);
3386	cso->cpu_sysinfo.ufsdirblk	= CPU_STATS(cp, sys.ufsdirblk);
3387	cso->cpu_sysinfo.ufsipage	= CPU_STATS(cp, sys.ufsipage);
3388	cso->cpu_sysinfo.ufsinopage	= CPU_STATS(cp, sys.ufsinopage);
3389	cso->cpu_sysinfo.inodeovf	= 0;
3390	cso->cpu_sysinfo.fileovf	= 0;
3391	cso->cpu_sysinfo.procovf	= CPU_STATS(cp, sys.procovf);
3392	cso->cpu_sysinfo.intrthread	= 0;
3393	for (i = 0; i < LOCK_LEVEL - 1; i++)
3394		cso->cpu_sysinfo.intrthread += CPU_STATS(cp, sys.intr[i]);
3395	cso->cpu_sysinfo.intrblk	= CPU_STATS(cp, sys.intrblk);
3396	cso->cpu_sysinfo.idlethread	= CPU_STATS(cp, sys.idlethread);
3397	cso->cpu_sysinfo.inv_swtch	= CPU_STATS(cp, sys.inv_swtch);
3398	cso->cpu_sysinfo.nthreads	= CPU_STATS(cp, sys.nthreads);
3399	cso->cpu_sysinfo.cpumigrate	= CPU_STATS(cp, sys.cpumigrate);
3400	cso->cpu_sysinfo.xcalls		= CPU_STATS(cp, sys.xcalls);
3401	cso->cpu_sysinfo.mutex_adenters	= CPU_STATS(cp, sys.mutex_adenters);
3402	cso->cpu_sysinfo.rw_rdfails	= CPU_STATS(cp, sys.rw_rdfails);
3403	cso->cpu_sysinfo.rw_wrfails	= CPU_STATS(cp, sys.rw_wrfails);
3404	cso->cpu_sysinfo.modload	= CPU_STATS(cp, sys.modload);
3405	cso->cpu_sysinfo.modunload	= CPU_STATS(cp, sys.modunload);
3406	cso->cpu_sysinfo.bawrite	= CPU_STATS(cp, sys.bawrite);
3407	cso->cpu_sysinfo.rw_enters	= 0;
3408	cso->cpu_sysinfo.win_uo_cnt	= 0;
3409	cso->cpu_sysinfo.win_uu_cnt	= 0;
3410	cso->cpu_sysinfo.win_so_cnt	= 0;
3411	cso->cpu_sysinfo.win_su_cnt	= 0;
3412	cso->cpu_sysinfo.win_suo_cnt	= 0;
3413
3414	cso->cpu_syswait.iowait		= CPU_STATS(cp, sys.iowait);
3415	cso->cpu_syswait.swap		= 0;
3416	cso->cpu_syswait.physio		= 0;
3417
3418	cso->cpu_vminfo.pgrec		= CPU_STATS(cp, vm.pgrec);
3419	cso->cpu_vminfo.pgfrec		= CPU_STATS(cp, vm.pgfrec);
3420	cso->cpu_vminfo.pgin		= CPU_STATS(cp, vm.pgin);
3421	cso->cpu_vminfo.pgpgin		= CPU_STATS(cp, vm.pgpgin);
3422	cso->cpu_vminfo.pgout		= CPU_STATS(cp, vm.pgout);
3423	cso->cpu_vminfo.pgpgout		= CPU_STATS(cp, vm.pgpgout);
3424	cso->cpu_vminfo.swapin		= CPU_STATS(cp, vm.swapin);
3425	cso->cpu_vminfo.pgswapin	= CPU_STATS(cp, vm.pgswapin);
3426	cso->cpu_vminfo.swapout		= CPU_STATS(cp, vm.swapout);
3427	cso->cpu_vminfo.pgswapout	= CPU_STATS(cp, vm.pgswapout);
3428	cso->cpu_vminfo.zfod		= CPU_STATS(cp, vm.zfod);
3429	cso->cpu_vminfo.dfree		= CPU_STATS(cp, vm.dfree);
3430	cso->cpu_vminfo.scan		= CPU_STATS(cp, vm.scan);
3431	cso->cpu_vminfo.rev		= CPU_STATS(cp, vm.rev);
3432	cso->cpu_vminfo.hat_fault	= CPU_STATS(cp, vm.hat_fault);
3433	cso->cpu_vminfo.as_fault	= CPU_STATS(cp, vm.as_fault);
3434	cso->cpu_vminfo.maj_fault	= CPU_STATS(cp, vm.maj_fault);
3435	cso->cpu_vminfo.cow_fault	= CPU_STATS(cp, vm.cow_fault);
3436	cso->cpu_vminfo.prot_fault	= CPU_STATS(cp, vm.prot_fault);
3437	cso->cpu_vminfo.softlock	= CPU_STATS(cp, vm.softlock);
3438	cso->cpu_vminfo.kernel_asflt	= CPU_STATS(cp, vm.kernel_asflt);
3439	cso->cpu_vminfo.pgrrun		= CPU_STATS(cp, vm.pgrrun);
3440	cso->cpu_vminfo.execpgin	= CPU_STATS(cp, vm.execpgin);
3441	cso->cpu_vminfo.execpgout	= CPU_STATS(cp, vm.execpgout);
3442	cso->cpu_vminfo.execfree	= CPU_STATS(cp, vm.execfree);
3443	cso->cpu_vminfo.anonpgin	= CPU_STATS(cp, vm.anonpgin);
3444	cso->cpu_vminfo.anonpgout	= CPU_STATS(cp, vm.anonpgout);
3445	cso->cpu_vminfo.anonfree	= CPU_STATS(cp, vm.anonfree);
3446	cso->cpu_vminfo.fspgin		= CPU_STATS(cp, vm.fspgin);
3447	cso->cpu_vminfo.fspgout		= CPU_STATS(cp, vm.fspgout);
3448	cso->cpu_vminfo.fsfree		= CPU_STATS(cp, vm.fsfree);
3449
3450	return (0);
3451}
3452