xref: /illumos-gate/usr/src/uts/common/os/cpu.c (revision 86ef0a63)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2012 by Delphix. All rights reserved.
24  * Copyright 2019 Joyent, Inc.
25  * Copyright 2021 Oxide Computer Company
26  */
27 
28 /*
29  * Architecture-independent CPU control functions.
30  */
31 
32 #include <sys/types.h>
33 #include <sys/param.h>
34 #include <sys/var.h>
35 #include <sys/thread.h>
36 #include <sys/cpuvar.h>
37 #include <sys/cpu_event.h>
38 #include <sys/kstat.h>
39 #include <sys/uadmin.h>
40 #include <sys/systm.h>
41 #include <sys/errno.h>
42 #include <sys/cmn_err.h>
43 #include <sys/procset.h>
44 #include <sys/processor.h>
45 #include <sys/debug.h>
46 #include <sys/cpupart.h>
47 #include <sys/lgrp.h>
48 #include <sys/pset.h>
49 #include <sys/pghw.h>
50 #include <sys/kmem.h>
51 #include <sys/kmem_impl.h>	/* to set per-cpu kmem_cache offset */
52 #include <sys/atomic.h>
53 #include <sys/callb.h>
54 #include <sys/vtrace.h>
55 #include <sys/cyclic.h>
56 #include <sys/bitmap.h>
57 #include <sys/nvpair.h>
58 #include <sys/pool_pset.h>
59 #include <sys/msacct.h>
60 #include <sys/time.h>
61 #include <sys/archsystm.h>
62 #include <sys/sdt.h>
63 #include <sys/smt.h>
64 #if defined(__x86)
65 #include <sys/x86_archext.h>
66 #endif
67 #include <sys/callo.h>
68 
69 extern int	mp_cpu_start(cpu_t *);
70 extern int	mp_cpu_stop(cpu_t *);
71 extern int	mp_cpu_poweron(cpu_t *);
72 extern int	mp_cpu_poweroff(cpu_t *);
73 extern int	mp_cpu_configure(int);
74 extern int	mp_cpu_unconfigure(int);
75 extern void	mp_cpu_faulted_enter(cpu_t *);
76 extern void	mp_cpu_faulted_exit(cpu_t *);
77 
78 extern int cmp_cpu_to_chip(processorid_t cpuid);
79 #ifdef __sparcv9
80 extern char *cpu_fru_fmri(cpu_t *cp);
81 #endif
82 
83 static void cpu_add_active_internal(cpu_t *cp);
84 static void cpu_remove_active(cpu_t *cp);
85 static void cpu_info_kstat_create(cpu_t *cp);
86 static void cpu_info_kstat_destroy(cpu_t *cp);
87 static void cpu_stats_kstat_create(cpu_t *cp);
88 static void cpu_stats_kstat_destroy(cpu_t *cp);
89 
90 static int cpu_sys_stats_ks_update(kstat_t *ksp, int rw);
91 static int cpu_vm_stats_ks_update(kstat_t *ksp, int rw);
92 static int cpu_stat_ks_update(kstat_t *ksp, int rw);
93 static int cpu_state_change_hooks(int, cpu_setup_t, cpu_setup_t);
94 
95 /*
96  * cpu_lock protects ncpus, ncpus_online, cpu_flag, cpu_list, cpu_active,
97  * max_cpu_seqid_ever, and dispatch queue reallocations.  The lock ordering with
98  * respect to related locks is:
99  *
100  *	cpu_lock --> thread_free_lock  --->  p_lock  --->  thread_lock()
101  *
102  * Warning:  Certain sections of code do not use the cpu_lock when
103  * traversing the cpu_list (e.g. mutex_vector_enter(), clock()).  Since
104  * all cpus are paused during modifications to this list, a solution
105  * to protect the list is too either disable kernel preemption while
106  * walking the list, *or* recheck the cpu_next pointer at each
107  * iteration in the loop.  Note that in no cases can any cached
108  * copies of the cpu pointers be kept as they may become invalid.
109  */
110 kmutex_t	cpu_lock;
111 cpu_t		*cpu_list;		/* list of all CPUs */
112 cpu_t		*clock_cpu_list;	/* used by clock to walk CPUs */
113 cpu_t		*cpu_active;		/* list of active CPUs */
114 cpuset_t	cpu_active_set;		/* cached set of active CPUs */
115 static cpuset_t	cpu_available;		/* set of available CPUs */
116 cpuset_t	cpu_seqid_inuse;	/* which cpu_seqids are in use */
117 
118 cpu_t		**cpu_seq;		/* ptrs to CPUs, indexed by seq_id */
119 
120 /*
121  * max_ncpus keeps the max cpus the system can have. Initially
122  * it's NCPU, but since most archs scan the devtree for cpus
123  * fairly early on during boot, the real max can be known before
124  * ncpus is set (useful for early NCPU based allocations).
125  */
126 int max_ncpus = NCPU;
127 /*
128  * platforms that set max_ncpus to maxiumum number of cpus that can be
129  * dynamically added will set boot_max_ncpus to the number of cpus found
130  * at device tree scan time during boot.
131  */
132 int boot_max_ncpus = -1;
133 int boot_ncpus = -1;
134 /*
135  * Maximum possible CPU id.  This can never be >= NCPU since NCPU is
136  * used to size arrays that are indexed by CPU id.
137  */
138 processorid_t max_cpuid = NCPU - 1;
139 
140 /*
141  * Maximum cpu_seqid was given. This number can only grow and never shrink. It
142  * can be used to optimize NCPU loops to avoid going through CPUs which were
143  * never on-line.
144  */
145 processorid_t max_cpu_seqid_ever = 0;
146 
147 int ncpus = 1;
148 int ncpus_online = 1;
149 int ncpus_intr_enabled = 1;
150 
151 /*
152  * CPU that we're trying to offline.  Protected by cpu_lock.
153  */
154 cpu_t *cpu_inmotion;
155 
156 /*
157  * Can be raised to suppress further weakbinding, which are instead
158  * satisfied by disabling preemption.  Must be raised/lowered under cpu_lock,
159  * while individual thread weakbinding synchronization is done under thread
160  * lock.
161  */
162 int weakbindingbarrier;
163 
164 /*
165  * Variables used in pause_cpus().
166  */
167 static volatile char safe_list[NCPU];
168 
169 static struct _cpu_pause_info {
170 	int		cp_spl;		/* spl saved in pause_cpus() */
171 	volatile int	cp_go;		/* Go signal sent after all ready */
172 	int		cp_count;	/* # of CPUs to pause */
173 	ksema_t		cp_sem;		/* synch pause_cpus & cpu_pause */
174 	kthread_id_t	cp_paused;
175 	void		*(*cp_func)(void *);
176 } cpu_pause_info;
177 
178 static kmutex_t pause_free_mutex;
179 static kcondvar_t pause_free_cv;
180 
181 
182 static struct cpu_sys_stats_ks_data {
183 	kstat_named_t cpu_ticks_idle;
184 	kstat_named_t cpu_ticks_user;
185 	kstat_named_t cpu_ticks_kernel;
186 	kstat_named_t cpu_ticks_wait;
187 	kstat_named_t cpu_nsec_idle;
188 	kstat_named_t cpu_nsec_user;
189 	kstat_named_t cpu_nsec_kernel;
190 	kstat_named_t cpu_nsec_dtrace;
191 	kstat_named_t cpu_nsec_intr;
192 	kstat_named_t cpu_load_intr;
193 	kstat_named_t wait_ticks_io;
194 	kstat_named_t dtrace_probes;
195 	kstat_named_t bread;
196 	kstat_named_t bwrite;
197 	kstat_named_t lread;
198 	kstat_named_t lwrite;
199 	kstat_named_t phread;
200 	kstat_named_t phwrite;
201 	kstat_named_t pswitch;
202 	kstat_named_t trap;
203 	kstat_named_t intr;
204 	kstat_named_t syscall;
205 	kstat_named_t sysread;
206 	kstat_named_t syswrite;
207 	kstat_named_t sysfork;
208 	kstat_named_t sysvfork;
209 	kstat_named_t sysexec;
210 	kstat_named_t readch;
211 	kstat_named_t writech;
212 	kstat_named_t rcvint;
213 	kstat_named_t xmtint;
214 	kstat_named_t mdmint;
215 	kstat_named_t rawch;
216 	kstat_named_t canch;
217 	kstat_named_t outch;
218 	kstat_named_t msg;
219 	kstat_named_t sema;
220 	kstat_named_t namei;
221 	kstat_named_t ufsiget;
222 	kstat_named_t ufsdirblk;
223 	kstat_named_t ufsipage;
224 	kstat_named_t ufsinopage;
225 	kstat_named_t procovf;
226 	kstat_named_t intrthread;
227 	kstat_named_t intrblk;
228 	kstat_named_t intrunpin;
229 	kstat_named_t idlethread;
230 	kstat_named_t inv_swtch;
231 	kstat_named_t nthreads;
232 	kstat_named_t cpumigrate;
233 	kstat_named_t xcalls;
234 	kstat_named_t mutex_adenters;
235 	kstat_named_t rw_rdfails;
236 	kstat_named_t rw_wrfails;
237 	kstat_named_t modload;
238 	kstat_named_t modunload;
239 	kstat_named_t bawrite;
240 	kstat_named_t iowait;
241 } cpu_sys_stats_ks_data_template = {
242 	{ "cpu_ticks_idle",	KSTAT_DATA_UINT64 },
243 	{ "cpu_ticks_user",	KSTAT_DATA_UINT64 },
244 	{ "cpu_ticks_kernel",	KSTAT_DATA_UINT64 },
245 	{ "cpu_ticks_wait",	KSTAT_DATA_UINT64 },
246 	{ "cpu_nsec_idle",	KSTAT_DATA_UINT64 },
247 	{ "cpu_nsec_user",	KSTAT_DATA_UINT64 },
248 	{ "cpu_nsec_kernel",	KSTAT_DATA_UINT64 },
249 	{ "cpu_nsec_dtrace",	KSTAT_DATA_UINT64 },
250 	{ "cpu_nsec_intr",	KSTAT_DATA_UINT64 },
251 	{ "cpu_load_intr",	KSTAT_DATA_UINT64 },
252 	{ "wait_ticks_io",	KSTAT_DATA_UINT64 },
253 	{ "dtrace_probes",	KSTAT_DATA_UINT64 },
254 	{ "bread",		KSTAT_DATA_UINT64 },
255 	{ "bwrite",		KSTAT_DATA_UINT64 },
256 	{ "lread",		KSTAT_DATA_UINT64 },
257 	{ "lwrite",		KSTAT_DATA_UINT64 },
258 	{ "phread",		KSTAT_DATA_UINT64 },
259 	{ "phwrite",		KSTAT_DATA_UINT64 },
260 	{ "pswitch",		KSTAT_DATA_UINT64 },
261 	{ "trap",		KSTAT_DATA_UINT64 },
262 	{ "intr",		KSTAT_DATA_UINT64 },
263 	{ "syscall",		KSTAT_DATA_UINT64 },
264 	{ "sysread",		KSTAT_DATA_UINT64 },
265 	{ "syswrite",		KSTAT_DATA_UINT64 },
266 	{ "sysfork",		KSTAT_DATA_UINT64 },
267 	{ "sysvfork",		KSTAT_DATA_UINT64 },
268 	{ "sysexec",		KSTAT_DATA_UINT64 },
269 	{ "readch",		KSTAT_DATA_UINT64 },
270 	{ "writech",		KSTAT_DATA_UINT64 },
271 	{ "rcvint",		KSTAT_DATA_UINT64 },
272 	{ "xmtint",		KSTAT_DATA_UINT64 },
273 	{ "mdmint",		KSTAT_DATA_UINT64 },
274 	{ "rawch",		KSTAT_DATA_UINT64 },
275 	{ "canch",		KSTAT_DATA_UINT64 },
276 	{ "outch",		KSTAT_DATA_UINT64 },
277 	{ "msg",		KSTAT_DATA_UINT64 },
278 	{ "sema",		KSTAT_DATA_UINT64 },
279 	{ "namei",		KSTAT_DATA_UINT64 },
280 	{ "ufsiget",		KSTAT_DATA_UINT64 },
281 	{ "ufsdirblk",		KSTAT_DATA_UINT64 },
282 	{ "ufsipage",		KSTAT_DATA_UINT64 },
283 	{ "ufsinopage",		KSTAT_DATA_UINT64 },
284 	{ "procovf",		KSTAT_DATA_UINT64 },
285 	{ "intrthread",		KSTAT_DATA_UINT64 },
286 	{ "intrblk",		KSTAT_DATA_UINT64 },
287 	{ "intrunpin",		KSTAT_DATA_UINT64 },
288 	{ "idlethread",		KSTAT_DATA_UINT64 },
289 	{ "inv_swtch",		KSTAT_DATA_UINT64 },
290 	{ "nthreads",		KSTAT_DATA_UINT64 },
291 	{ "cpumigrate",		KSTAT_DATA_UINT64 },
292 	{ "xcalls",		KSTAT_DATA_UINT64 },
293 	{ "mutex_adenters",	KSTAT_DATA_UINT64 },
294 	{ "rw_rdfails",		KSTAT_DATA_UINT64 },
295 	{ "rw_wrfails",		KSTAT_DATA_UINT64 },
296 	{ "modload",		KSTAT_DATA_UINT64 },
297 	{ "modunload",		KSTAT_DATA_UINT64 },
298 	{ "bawrite",		KSTAT_DATA_UINT64 },
299 	{ "iowait",		KSTAT_DATA_UINT64 },
300 };
301 
302 static struct cpu_vm_stats_ks_data {
303 	kstat_named_t pgrec;
304 	kstat_named_t pgfrec;
305 	kstat_named_t pgin;
306 	kstat_named_t pgpgin;
307 	kstat_named_t pgout;
308 	kstat_named_t pgpgout;
309 	kstat_named_t swapin;
310 	kstat_named_t pgswapin;
311 	kstat_named_t swapout;
312 	kstat_named_t pgswapout;
313 	kstat_named_t zfod;
314 	kstat_named_t dfree;
315 	kstat_named_t scan;
316 	kstat_named_t rev;
317 	kstat_named_t hat_fault;
318 	kstat_named_t as_fault;
319 	kstat_named_t maj_fault;
320 	kstat_named_t cow_fault;
321 	kstat_named_t prot_fault;
322 	kstat_named_t softlock;
323 	kstat_named_t kernel_asflt;
324 	kstat_named_t pgrrun;
325 	kstat_named_t execpgin;
326 	kstat_named_t execpgout;
327 	kstat_named_t execfree;
328 	kstat_named_t anonpgin;
329 	kstat_named_t anonpgout;
330 	kstat_named_t anonfree;
331 	kstat_named_t fspgin;
332 	kstat_named_t fspgout;
333 	kstat_named_t fsfree;
334 } cpu_vm_stats_ks_data_template = {
335 	{ "pgrec",		KSTAT_DATA_UINT64 },
336 	{ "pgfrec",		KSTAT_DATA_UINT64 },
337 	{ "pgin",		KSTAT_DATA_UINT64 },
338 	{ "pgpgin",		KSTAT_DATA_UINT64 },
339 	{ "pgout",		KSTAT_DATA_UINT64 },
340 	{ "pgpgout",		KSTAT_DATA_UINT64 },
341 	{ "swapin",		KSTAT_DATA_UINT64 },
342 	{ "pgswapin",		KSTAT_DATA_UINT64 },
343 	{ "swapout",		KSTAT_DATA_UINT64 },
344 	{ "pgswapout",		KSTAT_DATA_UINT64 },
345 	{ "zfod",		KSTAT_DATA_UINT64 },
346 	{ "dfree",		KSTAT_DATA_UINT64 },
347 	{ "scan",		KSTAT_DATA_UINT64 },
348 	{ "rev",		KSTAT_DATA_UINT64 },
349 	{ "hat_fault",		KSTAT_DATA_UINT64 },
350 	{ "as_fault",		KSTAT_DATA_UINT64 },
351 	{ "maj_fault",		KSTAT_DATA_UINT64 },
352 	{ "cow_fault",		KSTAT_DATA_UINT64 },
353 	{ "prot_fault",		KSTAT_DATA_UINT64 },
354 	{ "softlock",		KSTAT_DATA_UINT64 },
355 	{ "kernel_asflt",	KSTAT_DATA_UINT64 },
356 	{ "pgrrun",		KSTAT_DATA_UINT64 },
357 	{ "execpgin",		KSTAT_DATA_UINT64 },
358 	{ "execpgout",		KSTAT_DATA_UINT64 },
359 	{ "execfree",		KSTAT_DATA_UINT64 },
360 	{ "anonpgin",		KSTAT_DATA_UINT64 },
361 	{ "anonpgout",		KSTAT_DATA_UINT64 },
362 	{ "anonfree",		KSTAT_DATA_UINT64 },
363 	{ "fspgin",		KSTAT_DATA_UINT64 },
364 	{ "fspgout",		KSTAT_DATA_UINT64 },
365 	{ "fsfree",		KSTAT_DATA_UINT64 },
366 };
367 
368 /*
369  * Force the specified thread to migrate to the appropriate processor.
370  * Called with thread lock held, returns with it dropped.
371  */
372 static void
force_thread_migrate(kthread_id_t tp)373 force_thread_migrate(kthread_id_t tp)
374 {
375 	ASSERT(THREAD_LOCK_HELD(tp));
376 	if (tp == curthread) {
377 		THREAD_TRANSITION(tp);
378 		CL_SETRUN(tp);
379 		thread_unlock_nopreempt(tp);
380 		swtch();
381 	} else {
382 		if (tp->t_state == TS_ONPROC) {
383 			cpu_surrender(tp);
384 		} else if (tp->t_state == TS_RUN) {
385 			(void) dispdeq(tp);
386 			setbackdq(tp);
387 		}
388 		thread_unlock(tp);
389 	}
390 }
391 
392 /*
393  * Set affinity for a specified CPU.
394  *
395  * Specifying a cpu_id of CPU_CURRENT, allowed _only_ when setting affinity for
396  * curthread, will set affinity to the CPU on which the thread is currently
397  * running.  For other cpu_id values, the caller must ensure that the
398  * referenced CPU remains valid, which can be done by holding cpu_lock across
399  * this call.
400  *
401  * CPU affinity is guaranteed after return of thread_affinity_set().  If a
402  * caller setting affinity to CPU_CURRENT requires that its thread not migrate
403  * CPUs prior to a successful return, it should take extra precautions (such as
404  * their own call to kpreempt_disable) to ensure that safety.
405  *
406  * CPU_BEST can be used to pick a "best" CPU to migrate to, including
407  * potentially the current CPU.
408  *
409  * A CPU affinity reference count is maintained by thread_affinity_set and
410  * thread_affinity_clear (incrementing and decrementing it, respectively),
411  * maintaining CPU affinity while the count is non-zero, and allowing regions
412  * of code which require affinity to be nested.
413  */
414 void
thread_affinity_set(kthread_id_t t,int cpu_id)415 thread_affinity_set(kthread_id_t t, int cpu_id)
416 {
417 	cpu_t *cp;
418 
419 	ASSERT(!(t == curthread && t->t_weakbound_cpu != NULL));
420 
421 	if (cpu_id == CPU_CURRENT) {
422 		VERIFY3P(t, ==, curthread);
423 		kpreempt_disable();
424 		cp = CPU;
425 	} else if (cpu_id == CPU_BEST) {
426 		VERIFY3P(t, ==, curthread);
427 		kpreempt_disable();
428 		cp = disp_choose_best_cpu();
429 	} else {
430 		/*
431 		 * We should be asserting that cpu_lock is held here, but
432 		 * the NCA code doesn't acquire it.  The following assert
433 		 * should be uncommented when the NCA code is fixed.
434 		 *
435 		 * ASSERT(MUTEX_HELD(&cpu_lock));
436 		 */
437 		VERIFY((cpu_id >= 0) && (cpu_id < NCPU));
438 		cp = cpu[cpu_id];
439 
440 		/* user must provide a good cpu_id */
441 		VERIFY(cp != NULL);
442 	}
443 
444 	/*
445 	 * If there is already a hard affinity requested, and this affinity
446 	 * conflicts with that, panic.
447 	 */
448 	thread_lock(t);
449 	if (t->t_affinitycnt > 0 && t->t_bound_cpu != cp) {
450 		panic("affinity_set: setting %p but already bound to %p",
451 		    (void *)cp, (void *)t->t_bound_cpu);
452 	}
453 	t->t_affinitycnt++;
454 	t->t_bound_cpu = cp;
455 
456 	/*
457 	 * Make sure we're running on the right CPU.
458 	 */
459 	if (cp != t->t_cpu || t != curthread) {
460 		ASSERT(cpu_id != CPU_CURRENT);
461 		force_thread_migrate(t);	/* drops thread lock */
462 	} else {
463 		thread_unlock(t);
464 	}
465 
466 	if (cpu_id == CPU_CURRENT || cpu_id == CPU_BEST)
467 		kpreempt_enable();
468 }
469 
470 /*
471  *	Wrapper for backward compatibility.
472  */
473 void
affinity_set(int cpu_id)474 affinity_set(int cpu_id)
475 {
476 	thread_affinity_set(curthread, cpu_id);
477 }
478 
479 /*
480  * Decrement the affinity reservation count and if it becomes zero,
481  * clear the CPU affinity for the current thread, or set it to the user's
482  * software binding request.
483  */
484 void
thread_affinity_clear(kthread_id_t t)485 thread_affinity_clear(kthread_id_t t)
486 {
487 	register processorid_t binding;
488 
489 	thread_lock(t);
490 	if (--t->t_affinitycnt == 0) {
491 		if ((binding = t->t_bind_cpu) == PBIND_NONE) {
492 			/*
493 			 * Adjust disp_max_unbound_pri if necessary.
494 			 */
495 			disp_adjust_unbound_pri(t);
496 			t->t_bound_cpu = NULL;
497 			if (t->t_cpu->cpu_part != t->t_cpupart) {
498 				force_thread_migrate(t);
499 				return;
500 			}
501 		} else {
502 			t->t_bound_cpu = cpu[binding];
503 			/*
504 			 * Make sure the thread is running on the bound CPU.
505 			 */
506 			if (t->t_cpu != t->t_bound_cpu) {
507 				force_thread_migrate(t);
508 				return;		/* already dropped lock */
509 			}
510 		}
511 	}
512 	thread_unlock(t);
513 }
514 
515 /*
516  * Wrapper for backward compatibility.
517  */
518 void
affinity_clear(void)519 affinity_clear(void)
520 {
521 	thread_affinity_clear(curthread);
522 }
523 
524 /*
525  * Weak cpu affinity.  Bind to the "current" cpu for short periods
526  * of time during which the thread must not block (but may be preempted).
527  * Use this instead of kpreempt_disable() when it is only "no migration"
528  * rather than "no preemption" semantics that are required - disabling
529  * preemption holds higher priority threads off of cpu and if the
530  * operation that is protected is more than momentary this is not good
531  * for realtime etc.
532  *
533  * Weakly bound threads will not prevent a cpu from being offlined -
534  * we'll only run them on the cpu to which they are weakly bound but
535  * (because they do not block) we'll always be able to move them on to
536  * another cpu at offline time if we give them just a short moment to
537  * run during which they will unbind.  To give a cpu a chance of offlining,
538  * however, we require a barrier to weak bindings that may be raised for a
539  * given cpu (offline/move code may set this and then wait a short time for
540  * existing weak bindings to drop); the cpu_inmotion pointer is that barrier.
541  *
542  * There are few restrictions on the calling context of thread_nomigrate.
543  * The caller must not hold the thread lock.  Calls may be nested.
544  *
545  * After weakbinding a thread must not perform actions that may block.
546  * In particular it must not call thread_affinity_set; calling that when
547  * already weakbound is nonsensical anyway.
548  *
549  * If curthread is prevented from migrating for other reasons
550  * (kernel preemption disabled; high pil; strongly bound; interrupt thread)
551  * then the weak binding will succeed even if this cpu is the target of an
552  * offline/move request.
553  */
554 void
thread_nomigrate(void)555 thread_nomigrate(void)
556 {
557 	cpu_t *cp;
558 	kthread_id_t t = curthread;
559 
560 again:
561 	kpreempt_disable();
562 	cp = CPU;
563 
564 	/*
565 	 * A highlevel interrupt must not modify t_nomigrate or
566 	 * t_weakbound_cpu of the thread it has interrupted.  A lowlevel
567 	 * interrupt thread cannot migrate and we can avoid the
568 	 * thread_lock call below by short-circuiting here.  In either
569 	 * case we can just return since no migration is possible and
570 	 * the condition will persist (ie, when we test for these again
571 	 * in thread_allowmigrate they can't have changed).   Migration
572 	 * is also impossible if we're at or above DISP_LEVEL pil.
573 	 */
574 	if (CPU_ON_INTR(cp) || t->t_flag & T_INTR_THREAD ||
575 	    getpil() >= DISP_LEVEL) {
576 		kpreempt_enable();
577 		return;
578 	}
579 
580 	/*
581 	 * We must be consistent with existing weak bindings.  Since we
582 	 * may be interrupted between the increment of t_nomigrate and
583 	 * the store to t_weakbound_cpu below we cannot assume that
584 	 * t_weakbound_cpu will be set if t_nomigrate is.  Note that we
585 	 * cannot assert t_weakbound_cpu == t_bind_cpu since that is not
586 	 * always the case.
587 	 */
588 	if (t->t_nomigrate && t->t_weakbound_cpu && t->t_weakbound_cpu != cp) {
589 		if (!panicstr)
590 			panic("thread_nomigrate: binding to %p but already "
591 			    "bound to %p", (void *)cp,
592 			    (void *)t->t_weakbound_cpu);
593 	}
594 
595 	/*
596 	 * At this point we have preemption disabled and we don't yet hold
597 	 * the thread lock.  So it's possible that somebody else could
598 	 * set t_bind_cpu here and not be able to force us across to the
599 	 * new cpu (since we have preemption disabled).
600 	 */
601 	thread_lock(curthread);
602 
603 	/*
604 	 * If further weak bindings are being (temporarily) suppressed then
605 	 * we'll settle for disabling kernel preemption (which assures
606 	 * no migration provided the thread does not block which it is
607 	 * not allowed to if using thread_nomigrate).  We must remember
608 	 * this disposition so we can take appropriate action in
609 	 * thread_allowmigrate.  If this is a nested call and the
610 	 * thread is already weakbound then fall through as normal.
611 	 * We remember the decision to settle for kpreempt_disable through
612 	 * negative nesting counting in t_nomigrate.  Once a thread has had one
613 	 * weakbinding request satisfied in this way any further (nested)
614 	 * requests will continue to be satisfied in the same way,
615 	 * even if weak bindings have recommenced.
616 	 */
617 	if (t->t_nomigrate < 0 || weakbindingbarrier && t->t_nomigrate == 0) {
618 		--t->t_nomigrate;
619 		thread_unlock(curthread);
620 		return;		/* with kpreempt_disable still active */
621 	}
622 
623 	/*
624 	 * We hold thread_lock so t_bind_cpu cannot change.  We could,
625 	 * however, be running on a different cpu to which we are t_bound_cpu
626 	 * to (as explained above).  If we grant the weak binding request
627 	 * in that case then the dispatcher must favour our weak binding
628 	 * over our strong (in which case, just as when preemption is
629 	 * disabled, we can continue to run on a cpu other than the one to
630 	 * which we are strongbound; the difference in this case is that
631 	 * this thread can be preempted and so can appear on the dispatch
632 	 * queues of a cpu other than the one it is strongbound to).
633 	 *
634 	 * If the cpu we are running on does not appear to be a current
635 	 * offline target (we check cpu_inmotion to determine this - since
636 	 * we don't hold cpu_lock we may not see a recent store to that,
637 	 * so it's possible that we at times can grant a weak binding to a
638 	 * cpu that is an offline target, but that one request will not
639 	 * prevent the offline from succeeding) then we will always grant
640 	 * the weak binding request.  This includes the case above where
641 	 * we grant a weakbinding not commensurate with our strong binding.
642 	 *
643 	 * If our cpu does appear to be an offline target then we're inclined
644 	 * not to grant the weakbinding request just yet - we'd prefer to
645 	 * migrate to another cpu and grant the request there.  The
646 	 * exceptions are those cases where going through preemption code
647 	 * will not result in us changing cpu:
648 	 *
649 	 *	. interrupts have already bypassed this case (see above)
650 	 *	. we are already weakbound to this cpu (dispatcher code will
651 	 *	  always return us to the weakbound cpu)
652 	 *	. preemption was disabled even before we disabled it above
653 	 *	. we are strongbound to this cpu (if we're strongbound to
654 	 *	another and not yet running there the trip through the
655 	 *	dispatcher will move us to the strongbound cpu and we
656 	 *	will grant the weak binding there)
657 	 */
658 	if (cp != cpu_inmotion || t->t_nomigrate > 0 || t->t_preempt > 1 ||
659 	    t->t_bound_cpu == cp) {
660 		/*
661 		 * Don't be tempted to store to t_weakbound_cpu only on
662 		 * the first nested bind request - if we're interrupted
663 		 * after the increment of t_nomigrate and before the
664 		 * store to t_weakbound_cpu and the interrupt calls
665 		 * thread_nomigrate then the assertion in thread_allowmigrate
666 		 * would fail.
667 		 */
668 		t->t_nomigrate++;
669 		t->t_weakbound_cpu = cp;
670 		membar_producer();
671 		thread_unlock(curthread);
672 		/*
673 		 * Now that we have dropped the thread_lock another thread
674 		 * can set our t_weakbound_cpu, and will try to migrate us
675 		 * to the strongbound cpu (which will not be prevented by
676 		 * preemption being disabled since we're about to enable
677 		 * preemption).  We have granted the weakbinding to the current
678 		 * cpu, so again we are in the position that is is is possible
679 		 * that our weak and strong bindings differ.  Again this
680 		 * is catered for by dispatcher code which will favour our
681 		 * weak binding.
682 		 */
683 		kpreempt_enable();
684 	} else {
685 		/*
686 		 * Move to another cpu before granting the request by
687 		 * forcing this thread through preemption code.  When we
688 		 * get to set{front,back}dq called from CL_PREEMPT()
689 		 * cpu_choose() will be used to select a cpu to queue
690 		 * us on - that will see cpu_inmotion and take
691 		 * steps to avoid returning us to this cpu.
692 		 */
693 		cp->cpu_kprunrun = 1;
694 		thread_unlock(curthread);
695 		kpreempt_enable();	/* will call preempt() */
696 		goto again;
697 	}
698 }
699 
700 void
thread_allowmigrate(void)701 thread_allowmigrate(void)
702 {
703 	kthread_id_t t = curthread;
704 
705 	ASSERT(t->t_weakbound_cpu == CPU ||
706 	    (t->t_nomigrate < 0 && t->t_preempt > 0) ||
707 	    CPU_ON_INTR(CPU) || t->t_flag & T_INTR_THREAD ||
708 	    getpil() >= DISP_LEVEL);
709 
710 	if (CPU_ON_INTR(CPU) || (t->t_flag & T_INTR_THREAD) ||
711 	    getpil() >= DISP_LEVEL)
712 		return;
713 
714 	if (t->t_nomigrate < 0) {
715 		/*
716 		 * This thread was granted "weak binding" in the
717 		 * stronger form of kernel preemption disabling.
718 		 * Undo a level of nesting for both t_nomigrate
719 		 * and t_preempt.
720 		 */
721 		++t->t_nomigrate;
722 		kpreempt_enable();
723 	} else if (--t->t_nomigrate == 0) {
724 		/*
725 		 * Time to drop the weak binding.  We need to cater
726 		 * for the case where we're weakbound to a different
727 		 * cpu than that to which we're strongbound (a very
728 		 * temporary arrangement that must only persist until
729 		 * weak binding drops).  We don't acquire thread_lock
730 		 * here so even as this code executes t_bound_cpu
731 		 * may be changing.  So we disable preemption and
732 		 * a) in the case that t_bound_cpu changes while we
733 		 * have preemption disabled kprunrun will be set
734 		 * asynchronously, and b) if before disabling
735 		 * preemption we were already on a different cpu to
736 		 * our t_bound_cpu then we set kprunrun ourselves
737 		 * to force a trip through the dispatcher when
738 		 * preemption is enabled.
739 		 */
740 		kpreempt_disable();
741 		if (t->t_bound_cpu &&
742 		    t->t_weakbound_cpu != t->t_bound_cpu)
743 			CPU->cpu_kprunrun = 1;
744 		t->t_weakbound_cpu = NULL;
745 		membar_producer();
746 		kpreempt_enable();
747 	}
748 }
749 
750 /*
751  * weakbinding_stop can be used to temporarily cause weakbindings made
752  * with thread_nomigrate to be satisfied through the stronger action of
753  * kpreempt_disable.  weakbinding_start recommences normal weakbinding.
754  */
755 
756 void
weakbinding_stop(void)757 weakbinding_stop(void)
758 {
759 	ASSERT(MUTEX_HELD(&cpu_lock));
760 	weakbindingbarrier = 1;
761 	membar_producer();	/* make visible before subsequent thread_lock */
762 }
763 
764 void
weakbinding_start(void)765 weakbinding_start(void)
766 {
767 	ASSERT(MUTEX_HELD(&cpu_lock));
768 	weakbindingbarrier = 0;
769 }
770 
771 void
null_xcall(void)772 null_xcall(void)
773 {
774 }
775 
776 /*
777  * This routine is called to place the CPUs in a safe place so that
778  * one of them can be taken off line or placed on line.  What we are
779  * trying to do here is prevent a thread from traversing the list
780  * of active CPUs while we are changing it or from getting placed on
781  * the run queue of a CPU that has just gone off line.  We do this by
782  * creating a thread with the highest possible prio for each CPU and
783  * having it call this routine.  The advantage of this method is that
784  * we can eliminate all checks for CPU_ACTIVE in the disp routines.
785  * This makes disp faster at the expense of making p_online() slower
786  * which is a good trade off.
787  */
788 static void
cpu_pause(int index)789 cpu_pause(int index)
790 {
791 	int s;
792 	struct _cpu_pause_info *cpi = &cpu_pause_info;
793 	volatile char *safe = &safe_list[index];
794 	long    lindex = index;
795 
796 	ASSERT((curthread->t_bound_cpu != NULL) || (*safe == PAUSE_DIE));
797 
798 	while (*safe != PAUSE_DIE) {
799 		*safe = PAUSE_READY;
800 		membar_enter();		/* make sure stores are flushed */
801 		sema_v(&cpi->cp_sem);	/* signal requesting thread */
802 
803 		/*
804 		 * Wait here until all pause threads are running.  That
805 		 * indicates that it's safe to do the spl.  Until
806 		 * cpu_pause_info.cp_go is set, we don't want to spl
807 		 * because that might block clock interrupts needed
808 		 * to preempt threads on other CPUs.
809 		 */
810 		while (cpi->cp_go == 0)
811 			;
812 		/*
813 		 * Even though we are at the highest disp prio, we need
814 		 * to block out all interrupts below LOCK_LEVEL so that
815 		 * an intr doesn't come in, wake up a thread, and call
816 		 * setbackdq/setfrontdq.
817 		 */
818 		s = splhigh();
819 		/*
820 		 * if cp_func has been set then call it using index as the
821 		 * argument, currently only used by cpr_suspend_cpus().
822 		 * This function is used as the code to execute on the
823 		 * "paused" cpu's when a machine comes out of a sleep state
824 		 * and CPU's were powered off.  (could also be used for
825 		 * hotplugging CPU's).
826 		 */
827 		if (cpi->cp_func != NULL)
828 			(*cpi->cp_func)((void *)lindex);
829 
830 		mach_cpu_pause(safe);
831 
832 		splx(s);
833 		/*
834 		 * Waiting is at an end. Switch out of cpu_pause
835 		 * loop and resume useful work.
836 		 */
837 		swtch();
838 	}
839 
840 	mutex_enter(&pause_free_mutex);
841 	*safe = PAUSE_DEAD;
842 	cv_broadcast(&pause_free_cv);
843 	mutex_exit(&pause_free_mutex);
844 }
845 
846 /*
847  * Allow the cpus to start running again.
848  */
849 void
start_cpus()850 start_cpus()
851 {
852 	int i;
853 
854 	ASSERT(MUTEX_HELD(&cpu_lock));
855 	ASSERT(cpu_pause_info.cp_paused);
856 	cpu_pause_info.cp_paused = NULL;
857 	for (i = 0; i < NCPU; i++)
858 		safe_list[i] = PAUSE_IDLE;
859 	membar_enter();			/* make sure stores are flushed */
860 	affinity_clear();
861 	splx(cpu_pause_info.cp_spl);
862 	kpreempt_enable();
863 }
864 
865 /*
866  * Allocate a pause thread for a CPU.
867  */
868 static void
cpu_pause_alloc(cpu_t * cp)869 cpu_pause_alloc(cpu_t *cp)
870 {
871 	kthread_id_t	t;
872 	long		cpun = cp->cpu_id;
873 
874 	/*
875 	 * Note, v.v_nglobpris will not change value as long as I hold
876 	 * cpu_lock.
877 	 */
878 	t = thread_create(NULL, 0, cpu_pause, (void *)cpun,
879 	    0, &p0, TS_STOPPED, v.v_nglobpris - 1);
880 	thread_lock(t);
881 	t->t_bound_cpu = cp;
882 	t->t_disp_queue = cp->cpu_disp;
883 	t->t_affinitycnt = 1;
884 	t->t_preempt = 1;
885 	thread_unlock(t);
886 	cp->cpu_pause_thread = t;
887 	/*
888 	 * Registering a thread in the callback table is usually done
889 	 * in the initialization code of the thread.  In this
890 	 * case, we do it right after thread creation because the
891 	 * thread itself may never run, and we need to register the
892 	 * fact that it is safe for cpr suspend.
893 	 */
894 	CALLB_CPR_INIT_SAFE(t, "cpu_pause");
895 }
896 
897 /*
898  * Free a pause thread for a CPU.
899  */
900 static void
cpu_pause_free(cpu_t * cp)901 cpu_pause_free(cpu_t *cp)
902 {
903 	kthread_id_t	t;
904 	int		cpun = cp->cpu_id;
905 
906 	ASSERT(MUTEX_HELD(&cpu_lock));
907 	/*
908 	 * We have to get the thread and tell it to die.
909 	 */
910 	if ((t = cp->cpu_pause_thread) == NULL) {
911 		ASSERT(safe_list[cpun] == PAUSE_IDLE);
912 		return;
913 	}
914 	thread_lock(t);
915 	t->t_cpu = CPU;		/* disp gets upset if last cpu is quiesced. */
916 	t->t_bound_cpu = NULL;	/* Must un-bind; cpu may not be running. */
917 	t->t_pri = v.v_nglobpris - 1;
918 	ASSERT(safe_list[cpun] == PAUSE_IDLE);
919 	safe_list[cpun] = PAUSE_DIE;
920 	THREAD_TRANSITION(t);
921 	setbackdq(t);
922 	thread_unlock_nopreempt(t);
923 
924 	/*
925 	 * If we don't wait for the thread to actually die, it may try to
926 	 * run on the wrong cpu as part of an actual call to pause_cpus().
927 	 */
928 	mutex_enter(&pause_free_mutex);
929 	while (safe_list[cpun] != PAUSE_DEAD) {
930 		cv_wait(&pause_free_cv, &pause_free_mutex);
931 	}
932 	mutex_exit(&pause_free_mutex);
933 	safe_list[cpun] = PAUSE_IDLE;
934 
935 	cp->cpu_pause_thread = NULL;
936 }
937 
938 /*
939  * Initialize basic structures for pausing CPUs.
940  */
941 void
cpu_pause_init()942 cpu_pause_init()
943 {
944 	sema_init(&cpu_pause_info.cp_sem, 0, NULL, SEMA_DEFAULT, NULL);
945 	/*
946 	 * Create initial CPU pause thread.
947 	 */
948 	cpu_pause_alloc(CPU);
949 }
950 
951 /*
952  * Start the threads used to pause another CPU.
953  */
954 static int
cpu_pause_start(processorid_t cpu_id)955 cpu_pause_start(processorid_t cpu_id)
956 {
957 	int	i;
958 	int	cpu_count = 0;
959 
960 	for (i = 0; i < NCPU; i++) {
961 		cpu_t		*cp;
962 		kthread_id_t	t;
963 
964 		cp = cpu[i];
965 		if (!CPU_IN_SET(cpu_available, i) || (i == cpu_id)) {
966 			safe_list[i] = PAUSE_WAIT;
967 			continue;
968 		}
969 
970 		/*
971 		 * Skip CPU if it is quiesced or not yet started.
972 		 */
973 		if ((cp->cpu_flags & (CPU_QUIESCED | CPU_READY)) != CPU_READY) {
974 			safe_list[i] = PAUSE_WAIT;
975 			continue;
976 		}
977 
978 		/*
979 		 * Start this CPU's pause thread.
980 		 */
981 		t = cp->cpu_pause_thread;
982 		thread_lock(t);
983 		/*
984 		 * Reset the priority, since nglobpris may have
985 		 * changed since the thread was created, if someone
986 		 * has loaded the RT (or some other) scheduling
987 		 * class.
988 		 */
989 		t->t_pri = v.v_nglobpris - 1;
990 		THREAD_TRANSITION(t);
991 		setbackdq(t);
992 		thread_unlock_nopreempt(t);
993 		++cpu_count;
994 	}
995 	return (cpu_count);
996 }
997 
998 
999 /*
1000  * Pause all of the CPUs except the one we are on by creating a high
1001  * priority thread bound to those CPUs.
1002  *
1003  * Note that one must be extremely careful regarding code
1004  * executed while CPUs are paused.  Since a CPU may be paused
1005  * while a thread scheduling on that CPU is holding an adaptive
1006  * lock, code executed with CPUs paused must not acquire adaptive
1007  * (or low-level spin) locks.  Also, such code must not block,
1008  * since the thread that is supposed to initiate the wakeup may
1009  * never run.
1010  *
1011  * With a few exceptions, the restrictions on code executed with CPUs
1012  * paused match those for code executed at high-level interrupt
1013  * context.
1014  */
1015 void
pause_cpus(cpu_t * off_cp,void * (* func)(void *))1016 pause_cpus(cpu_t *off_cp, void *(*func)(void *))
1017 {
1018 	processorid_t	cpu_id;
1019 	int		i;
1020 	struct _cpu_pause_info	*cpi = &cpu_pause_info;
1021 
1022 	ASSERT(MUTEX_HELD(&cpu_lock));
1023 	ASSERT(cpi->cp_paused == NULL);
1024 	cpi->cp_count = 0;
1025 	cpi->cp_go = 0;
1026 	for (i = 0; i < NCPU; i++)
1027 		safe_list[i] = PAUSE_IDLE;
1028 	kpreempt_disable();
1029 
1030 	cpi->cp_func = func;
1031 
1032 	/*
1033 	 * If running on the cpu that is going offline, get off it.
1034 	 * This is so that it won't be necessary to rechoose a CPU
1035 	 * when done.
1036 	 */
1037 	if (CPU == off_cp)
1038 		cpu_id = off_cp->cpu_next_part->cpu_id;
1039 	else
1040 		cpu_id = CPU->cpu_id;
1041 	affinity_set(cpu_id);
1042 
1043 	/*
1044 	 * Start the pause threads and record how many were started
1045 	 */
1046 	cpi->cp_count = cpu_pause_start(cpu_id);
1047 
1048 	/*
1049 	 * Now wait for all CPUs to be running the pause thread.
1050 	 */
1051 	while (cpi->cp_count > 0) {
1052 		/*
1053 		 * Spin reading the count without grabbing the disp
1054 		 * lock to make sure we don't prevent the pause
1055 		 * threads from getting the lock.
1056 		 */
1057 		while (sema_held(&cpi->cp_sem))
1058 			;
1059 		if (sema_tryp(&cpi->cp_sem))
1060 			--cpi->cp_count;
1061 	}
1062 	cpi->cp_go = 1;			/* all have reached cpu_pause */
1063 
1064 	/*
1065 	 * Now wait for all CPUs to spl. (Transition from PAUSE_READY
1066 	 * to PAUSE_WAIT.)
1067 	 */
1068 	for (i = 0; i < NCPU; i++) {
1069 		while (safe_list[i] != PAUSE_WAIT)
1070 			;
1071 	}
1072 	cpi->cp_spl = splhigh();	/* block dispatcher on this CPU */
1073 	cpi->cp_paused = curthread;
1074 }
1075 
1076 /*
1077  * Check whether the current thread has CPUs paused
1078  */
1079 int
cpus_paused(void)1080 cpus_paused(void)
1081 {
1082 	if (cpu_pause_info.cp_paused != NULL) {
1083 		ASSERT(cpu_pause_info.cp_paused == curthread);
1084 		return (1);
1085 	}
1086 	return (0);
1087 }
1088 
1089 static cpu_t *
cpu_get_all(processorid_t cpun)1090 cpu_get_all(processorid_t cpun)
1091 {
1092 	ASSERT(MUTEX_HELD(&cpu_lock));
1093 
1094 	if (cpun >= NCPU || cpun < 0 || !CPU_IN_SET(cpu_available, cpun))
1095 		return (NULL);
1096 	return (cpu[cpun]);
1097 }
1098 
1099 /*
1100  * Check whether cpun is a valid processor id and whether it should be
1101  * visible from the current zone. If it is, return a pointer to the
1102  * associated CPU structure.
1103  */
1104 cpu_t *
cpu_get(processorid_t cpun)1105 cpu_get(processorid_t cpun)
1106 {
1107 	cpu_t *c;
1108 
1109 	ASSERT(MUTEX_HELD(&cpu_lock));
1110 	c = cpu_get_all(cpun);
1111 	if (c != NULL && !INGLOBALZONE(curproc) && pool_pset_enabled() &&
1112 	    zone_pset_get(curproc->p_zone) != cpupart_query_cpu(c))
1113 		return (NULL);
1114 	return (c);
1115 }
1116 
1117 /*
1118  * The following functions should be used to check CPU states in the kernel.
1119  * They should be invoked with cpu_lock held.  Kernel subsystems interested
1120  * in CPU states should *not* use cpu_get_state() and various P_ONLINE/etc
1121  * states.  Those are for user-land (and system call) use only.
1122  */
1123 
1124 /*
1125  * Determine whether the CPU is online and handling interrupts.
1126  */
1127 int
cpu_is_online(cpu_t * cpu)1128 cpu_is_online(cpu_t *cpu)
1129 {
1130 	ASSERT(MUTEX_HELD(&cpu_lock));
1131 	return (cpu_flagged_online(cpu->cpu_flags));
1132 }
1133 
1134 /*
1135  * Determine whether the CPU is offline (this includes spare and faulted).
1136  */
1137 int
cpu_is_offline(cpu_t * cpu)1138 cpu_is_offline(cpu_t *cpu)
1139 {
1140 	ASSERT(MUTEX_HELD(&cpu_lock));
1141 	return (cpu_flagged_offline(cpu->cpu_flags));
1142 }
1143 
1144 /*
1145  * Determine whether the CPU is powered off.
1146  */
1147 int
cpu_is_poweredoff(cpu_t * cpu)1148 cpu_is_poweredoff(cpu_t *cpu)
1149 {
1150 	ASSERT(MUTEX_HELD(&cpu_lock));
1151 	return (cpu_flagged_poweredoff(cpu->cpu_flags));
1152 }
1153 
1154 /*
1155  * Determine whether the CPU is handling interrupts.
1156  */
1157 int
cpu_is_nointr(cpu_t * cpu)1158 cpu_is_nointr(cpu_t *cpu)
1159 {
1160 	ASSERT(MUTEX_HELD(&cpu_lock));
1161 	return (cpu_flagged_nointr(cpu->cpu_flags));
1162 }
1163 
1164 /*
1165  * Determine whether the CPU is active (scheduling threads).
1166  */
1167 int
cpu_is_active(cpu_t * cpu)1168 cpu_is_active(cpu_t *cpu)
1169 {
1170 	ASSERT(MUTEX_HELD(&cpu_lock));
1171 	return (cpu_flagged_active(cpu->cpu_flags));
1172 }
1173 
1174 /*
1175  * Same as above, but these require cpu_flags instead of cpu_t pointers.
1176  */
1177 int
cpu_flagged_online(cpu_flag_t cpu_flags)1178 cpu_flagged_online(cpu_flag_t cpu_flags)
1179 {
1180 	return (cpu_flagged_active(cpu_flags) &&
1181 	    (cpu_flags & CPU_ENABLE));
1182 }
1183 
1184 int
cpu_flagged_offline(cpu_flag_t cpu_flags)1185 cpu_flagged_offline(cpu_flag_t cpu_flags)
1186 {
1187 	return (((cpu_flags & CPU_POWEROFF) == 0) &&
1188 	    ((cpu_flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY));
1189 }
1190 
1191 int
cpu_flagged_poweredoff(cpu_flag_t cpu_flags)1192 cpu_flagged_poweredoff(cpu_flag_t cpu_flags)
1193 {
1194 	return ((cpu_flags & CPU_POWEROFF) == CPU_POWEROFF);
1195 }
1196 
1197 int
cpu_flagged_nointr(cpu_flag_t cpu_flags)1198 cpu_flagged_nointr(cpu_flag_t cpu_flags)
1199 {
1200 	return (cpu_flagged_active(cpu_flags) &&
1201 	    (cpu_flags & CPU_ENABLE) == 0);
1202 }
1203 
1204 int
cpu_flagged_active(cpu_flag_t cpu_flags)1205 cpu_flagged_active(cpu_flag_t cpu_flags)
1206 {
1207 	return (((cpu_flags & (CPU_POWEROFF | CPU_FAULTED | CPU_SPARE)) == 0) &&
1208 	    ((cpu_flags & (CPU_READY | CPU_OFFLINE)) == CPU_READY));
1209 }
1210 
1211 /*
1212  * Bring the indicated CPU online.
1213  */
1214 int
cpu_online(cpu_t * cp,int flags)1215 cpu_online(cpu_t *cp, int flags)
1216 {
1217 	int	error = 0;
1218 
1219 	/*
1220 	 * Handle on-line request.
1221 	 *	This code must put the new CPU on the active list before
1222 	 *	starting it because it will not be paused, and will start
1223 	 *	using the active list immediately.  The real start occurs
1224 	 *	when the CPU_QUIESCED flag is turned off.
1225 	 */
1226 
1227 	ASSERT(MUTEX_HELD(&cpu_lock));
1228 
1229 	if ((cp->cpu_flags & CPU_DISABLED) && !smt_can_enable(cp, flags))
1230 		return (EINVAL);
1231 
1232 	/*
1233 	 * Put all the cpus into a known safe place.
1234 	 * No mutexes can be entered while CPUs are paused.
1235 	 */
1236 	error = mp_cpu_start(cp);	/* arch-dep hook */
1237 	if (error == 0) {
1238 		pg_cpupart_in(cp, cp->cpu_part);
1239 		pause_cpus(NULL, NULL);
1240 		cpu_add_active_internal(cp);
1241 		if (cp->cpu_flags & CPU_FAULTED) {
1242 			cp->cpu_flags &= ~CPU_FAULTED;
1243 			mp_cpu_faulted_exit(cp);
1244 		}
1245 
1246 		if (cp->cpu_flags & CPU_DISABLED)
1247 			smt_force_enabled();
1248 
1249 		cp->cpu_flags &= ~(CPU_QUIESCED | CPU_OFFLINE | CPU_FROZEN |
1250 		    CPU_SPARE | CPU_DISABLED);
1251 		CPU_NEW_GENERATION(cp);
1252 		start_cpus();
1253 		cpu_stats_kstat_create(cp);
1254 		cpu_create_intrstat(cp);
1255 		lgrp_kstat_create(cp);
1256 		cpu_state_change_notify(cp->cpu_id, CPU_ON);
1257 		cpu_intr_enable(cp);	/* arch-dep hook */
1258 		cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1259 		cpu_set_state(cp);
1260 		cyclic_online(cp);
1261 		/*
1262 		 * This has to be called only after cyclic_online(). This
1263 		 * function uses cyclics.
1264 		 */
1265 		callout_cpu_online(cp);
1266 		poke_cpu(cp->cpu_id);
1267 	}
1268 
1269 	return (error);
1270 }
1271 
1272 /*
1273  * Take the indicated CPU offline.
1274  */
1275 int
cpu_offline(cpu_t * cp,int flags)1276 cpu_offline(cpu_t *cp, int flags)
1277 {
1278 	cpupart_t *pp;
1279 	int	error = 0;
1280 	cpu_t	*ncp;
1281 	int	intr_enable;
1282 	int	cyclic_off = 0;
1283 	int	callout_off = 0;
1284 	int	loop_count;
1285 	int	no_quiesce = 0;
1286 	int	(*bound_func)(struct cpu *, int);
1287 	kthread_t *t;
1288 	lpl_t	*cpu_lpl;
1289 	proc_t	*p;
1290 	int	lgrp_diff_lpl;
1291 	boolean_t forced = (flags & CPU_FORCED) != 0;
1292 
1293 	ASSERT(MUTEX_HELD(&cpu_lock));
1294 
1295 	if (cp->cpu_flags & CPU_DISABLED)
1296 		return (EINVAL);
1297 
1298 	/*
1299 	 * If we're going from faulted or spare to offline, just
1300 	 * clear these flags and update CPU state.
1301 	 */
1302 	if (cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) {
1303 		if (cp->cpu_flags & CPU_FAULTED) {
1304 			cp->cpu_flags &= ~CPU_FAULTED;
1305 			mp_cpu_faulted_exit(cp);
1306 		}
1307 		cp->cpu_flags &= ~CPU_SPARE;
1308 		cpu_set_state(cp);
1309 		return (0);
1310 	}
1311 
1312 	/*
1313 	 * Handle off-line request.
1314 	 */
1315 	pp = cp->cpu_part;
1316 	/*
1317 	 * Don't offline last online CPU in partition
1318 	 */
1319 	if (ncpus_online <= 1 || pp->cp_ncpus <= 1 || cpu_intr_count(cp) < 2)
1320 		return (EBUSY);
1321 	/*
1322 	 * Unbind all soft-bound threads bound to our CPU and hard bound threads
1323 	 * if we were asked to.
1324 	 */
1325 	error = cpu_unbind(cp->cpu_id, forced);
1326 	if (error != 0)
1327 		return (error);
1328 	/*
1329 	 * We shouldn't be bound to this CPU ourselves.
1330 	 */
1331 	if (curthread->t_bound_cpu == cp)
1332 		return (EBUSY);
1333 
1334 	/*
1335 	 * Tell interested parties that this CPU is going offline.
1336 	 */
1337 	CPU_NEW_GENERATION(cp);
1338 	cpu_state_change_notify(cp->cpu_id, CPU_OFF);
1339 
1340 	/*
1341 	 * Tell the PG subsystem that the CPU is leaving the partition
1342 	 */
1343 	pg_cpupart_out(cp, pp);
1344 
1345 	/*
1346 	 * Take the CPU out of interrupt participation so we won't find
1347 	 * bound kernel threads.  If the architecture cannot completely
1348 	 * shut off interrupts on the CPU, don't quiesce it, but don't
1349 	 * run anything but interrupt thread... this is indicated by
1350 	 * the CPU_OFFLINE flag being on but the CPU_QUIESCE flag being
1351 	 * off.
1352 	 */
1353 	intr_enable = cp->cpu_flags & CPU_ENABLE;
1354 	if (intr_enable)
1355 		no_quiesce = cpu_intr_disable(cp);
1356 
1357 	/*
1358 	 * Record that we are aiming to offline this cpu.  This acts as
1359 	 * a barrier to further weak binding requests in thread_nomigrate
1360 	 * and also causes cpu_choose, disp_lowpri_cpu and setfrontdq to
1361 	 * lean away from this cpu.  Further strong bindings are already
1362 	 * avoided since we hold cpu_lock.  Since threads that are set
1363 	 * runnable around now and others coming off the target cpu are
1364 	 * directed away from the target, existing strong and weak bindings
1365 	 * (especially the latter) to the target cpu stand maximum chance of
1366 	 * being able to unbind during the short delay loop below (if other
1367 	 * unbound threads compete they may not see cpu in time to unbind
1368 	 * even if they would do so immediately.
1369 	 */
1370 	cpu_inmotion = cp;
1371 	membar_enter();
1372 
1373 	/*
1374 	 * Check for kernel threads (strong or weak) bound to that CPU.
1375 	 * Strongly bound threads may not unbind, and we'll have to return
1376 	 * EBUSY.  Weakly bound threads should always disappear - we've
1377 	 * stopped more weak binding with cpu_inmotion and existing
1378 	 * bindings will drain imminently (they may not block).  Nonetheless
1379 	 * we will wait for a fixed period for all bound threads to disappear.
1380 	 * Inactive interrupt threads are OK (they'll be in TS_FREE
1381 	 * state).  If test finds some bound threads, wait a few ticks
1382 	 * to give short-lived threads (such as interrupts) chance to
1383 	 * complete.  Note that if no_quiesce is set, i.e. this cpu
1384 	 * is required to service interrupts, then we take the route
1385 	 * that permits interrupt threads to be active (or bypassed).
1386 	 */
1387 	bound_func = no_quiesce ? disp_bound_threads : disp_bound_anythreads;
1388 
1389 again:	for (loop_count = 0; (*bound_func)(cp, 0); loop_count++) {
1390 		if (loop_count >= 5) {
1391 			error = EBUSY;	/* some threads still bound */
1392 			break;
1393 		}
1394 
1395 		/*
1396 		 * If some threads were assigned, give them
1397 		 * a chance to complete or move.
1398 		 *
1399 		 * This assumes that the clock_thread is not bound
1400 		 * to any CPU, because the clock_thread is needed to
1401 		 * do the delay(hz/100).
1402 		 *
1403 		 * Note: we still hold the cpu_lock while waiting for
1404 		 * the next clock tick.  This is OK since it isn't
1405 		 * needed for anything else except processor_bind(2),
1406 		 * and system initialization.  If we drop the lock,
1407 		 * we would risk another p_online disabling the last
1408 		 * processor.
1409 		 */
1410 		delay(hz/100);
1411 	}
1412 
1413 	if (error == 0 && callout_off == 0) {
1414 		callout_cpu_offline(cp);
1415 		callout_off = 1;
1416 	}
1417 
1418 	if (error == 0 && cyclic_off == 0) {
1419 		if (!cyclic_offline(cp)) {
1420 			/*
1421 			 * We must have bound cyclics...
1422 			 */
1423 			error = EBUSY;
1424 			goto out;
1425 		}
1426 		cyclic_off = 1;
1427 	}
1428 
1429 	/*
1430 	 * Call mp_cpu_stop() to perform any special operations
1431 	 * needed for this machine architecture to offline a CPU.
1432 	 */
1433 	if (error == 0)
1434 		error = mp_cpu_stop(cp);	/* arch-dep hook */
1435 
1436 	/*
1437 	 * If that all worked, take the CPU offline and decrement
1438 	 * ncpus_online.
1439 	 */
1440 	if (error == 0) {
1441 		/*
1442 		 * Put all the cpus into a known safe place.
1443 		 * No mutexes can be entered while CPUs are paused.
1444 		 */
1445 		pause_cpus(cp, NULL);
1446 		/*
1447 		 * Repeat the operation, if necessary, to make sure that
1448 		 * all outstanding low-level interrupts run to completion
1449 		 * before we set the CPU_QUIESCED flag.  It's also possible
1450 		 * that a thread has weak bound to the cpu despite our raising
1451 		 * cpu_inmotion above since it may have loaded that
1452 		 * value before the barrier became visible (this would have
1453 		 * to be the thread that was on the target cpu at the time
1454 		 * we raised the barrier).
1455 		 */
1456 		if ((!no_quiesce && cp->cpu_intr_actv != 0) ||
1457 		    (*bound_func)(cp, 1)) {
1458 			start_cpus();
1459 			(void) mp_cpu_start(cp);
1460 			goto again;
1461 		}
1462 		ncp = cp->cpu_next_part;
1463 		cpu_lpl = cp->cpu_lpl;
1464 		ASSERT(cpu_lpl != NULL);
1465 
1466 		/*
1467 		 * Remove the CPU from the list of active CPUs.
1468 		 */
1469 		cpu_remove_active(cp);
1470 
1471 		/*
1472 		 * Walk the active process list and look for threads
1473 		 * whose home lgroup needs to be updated, or
1474 		 * the last CPU they run on is the one being offlined now.
1475 		 */
1476 
1477 		ASSERT(curthread->t_cpu != cp);
1478 		for (p = practive; p != NULL; p = p->p_next) {
1479 
1480 			t = p->p_tlist;
1481 
1482 			if (t == NULL)
1483 				continue;
1484 
1485 			lgrp_diff_lpl = 0;
1486 
1487 			do {
1488 				ASSERT(t->t_lpl != NULL);
1489 				/*
1490 				 * Taking last CPU in lpl offline
1491 				 * Rehome thread if it is in this lpl
1492 				 * Otherwise, update the count of how many
1493 				 * threads are in this CPU's lgroup but have
1494 				 * a different lpl.
1495 				 */
1496 
1497 				if (cpu_lpl->lpl_ncpu == 0) {
1498 					if (t->t_lpl == cpu_lpl)
1499 						lgrp_move_thread(t,
1500 						    lgrp_choose(t,
1501 						    t->t_cpupart), 0);
1502 					else if (t->t_lpl->lpl_lgrpid ==
1503 					    cpu_lpl->lpl_lgrpid)
1504 						lgrp_diff_lpl++;
1505 				}
1506 				ASSERT(t->t_lpl->lpl_ncpu > 0);
1507 
1508 				/*
1509 				 * Update CPU last ran on if it was this CPU
1510 				 */
1511 				if (t->t_cpu == cp && t->t_bound_cpu != cp)
1512 					t->t_cpu = disp_lowpri_cpu(ncp, t,
1513 					    t->t_pri);
1514 				ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1515 				    t->t_weakbound_cpu == cp);
1516 
1517 				t = t->t_forw;
1518 			} while (t != p->p_tlist);
1519 
1520 			/*
1521 			 * Didn't find any threads in the same lgroup as this
1522 			 * CPU with a different lpl, so remove the lgroup from
1523 			 * the process lgroup bitmask.
1524 			 */
1525 
1526 			if (lgrp_diff_lpl == 0)
1527 				klgrpset_del(p->p_lgrpset, cpu_lpl->lpl_lgrpid);
1528 		}
1529 
1530 		/*
1531 		 * Walk thread list looking for threads that need to be
1532 		 * rehomed, since there are some threads that are not in
1533 		 * their process's p_tlist.
1534 		 */
1535 
1536 		t = curthread;
1537 		do {
1538 			ASSERT(t != NULL && t->t_lpl != NULL);
1539 
1540 			/*
1541 			 * Rehome threads with same lpl as this CPU when this
1542 			 * is the last CPU in the lpl.
1543 			 */
1544 
1545 			if ((cpu_lpl->lpl_ncpu == 0) && (t->t_lpl == cpu_lpl))
1546 				lgrp_move_thread(t,
1547 				    lgrp_choose(t, t->t_cpupart), 1);
1548 
1549 			ASSERT(t->t_lpl->lpl_ncpu > 0);
1550 
1551 			/*
1552 			 * Update CPU last ran on if it was this CPU
1553 			 */
1554 
1555 			if (t->t_cpu == cp && t->t_bound_cpu != cp)
1556 				t->t_cpu = disp_lowpri_cpu(ncp, t, t->t_pri);
1557 
1558 			ASSERT(t->t_cpu != cp || t->t_bound_cpu == cp ||
1559 			    t->t_weakbound_cpu == cp);
1560 			t = t->t_next;
1561 
1562 		} while (t != curthread);
1563 		ASSERT((cp->cpu_flags & (CPU_FAULTED | CPU_SPARE)) == 0);
1564 		cp->cpu_flags |= CPU_OFFLINE;
1565 		disp_cpu_inactive(cp);
1566 		if (!no_quiesce)
1567 			cp->cpu_flags |= CPU_QUIESCED;
1568 		ncpus_online--;
1569 		cpu_set_state(cp);
1570 		cpu_inmotion = NULL;
1571 		start_cpus();
1572 		cpu_stats_kstat_destroy(cp);
1573 		cpu_delete_intrstat(cp);
1574 		lgrp_kstat_destroy(cp);
1575 	}
1576 
1577 out:
1578 	cpu_inmotion = NULL;
1579 
1580 	/*
1581 	 * If we failed, re-enable interrupts.
1582 	 * Do this even if cpu_intr_disable returned an error, because
1583 	 * it may have partially disabled interrupts.
1584 	 */
1585 	if (error && intr_enable)
1586 		cpu_intr_enable(cp);
1587 
1588 	/*
1589 	 * If we failed, but managed to offline the cyclic subsystem on this
1590 	 * CPU, bring it back online.
1591 	 */
1592 	if (error && cyclic_off)
1593 		cyclic_online(cp);
1594 
1595 	/*
1596 	 * If we failed, but managed to offline callouts on this CPU,
1597 	 * bring it back online.
1598 	 */
1599 	if (error && callout_off)
1600 		callout_cpu_online(cp);
1601 
1602 	/*
1603 	 * If we failed, tell the PG subsystem that the CPU is back
1604 	 */
1605 	pg_cpupart_in(cp, pp);
1606 
1607 	/*
1608 	 * If we failed, we need to notify everyone that this CPU is back on.
1609 	 */
1610 	if (error != 0) {
1611 		CPU_NEW_GENERATION(cp);
1612 		cpu_state_change_notify(cp->cpu_id, CPU_ON);
1613 		cpu_state_change_notify(cp->cpu_id, CPU_INTR_ON);
1614 	}
1615 
1616 	return (error);
1617 }
1618 
1619 /*
1620  * Mark the indicated CPU as faulted, taking it offline.
1621  */
1622 int
cpu_faulted(cpu_t * cp,int flags)1623 cpu_faulted(cpu_t *cp, int flags)
1624 {
1625 	int	error = 0;
1626 
1627 	ASSERT(MUTEX_HELD(&cpu_lock));
1628 	ASSERT(!cpu_is_poweredoff(cp));
1629 
1630 	if (cp->cpu_flags & CPU_DISABLED)
1631 		return (EINVAL);
1632 
1633 	if (cpu_is_offline(cp)) {
1634 		cp->cpu_flags &= ~CPU_SPARE;
1635 		cp->cpu_flags |= CPU_FAULTED;
1636 		mp_cpu_faulted_enter(cp);
1637 		cpu_set_state(cp);
1638 		return (0);
1639 	}
1640 
1641 	if ((error = cpu_offline(cp, flags)) == 0) {
1642 		cp->cpu_flags |= CPU_FAULTED;
1643 		mp_cpu_faulted_enter(cp);
1644 		cpu_set_state(cp);
1645 	}
1646 
1647 	return (error);
1648 }
1649 
1650 /*
1651  * Mark the indicated CPU as a spare, taking it offline.
1652  */
1653 int
cpu_spare(cpu_t * cp,int flags)1654 cpu_spare(cpu_t *cp, int flags)
1655 {
1656 	int	error = 0;
1657 
1658 	ASSERT(MUTEX_HELD(&cpu_lock));
1659 	ASSERT(!cpu_is_poweredoff(cp));
1660 
1661 	if (cp->cpu_flags & CPU_DISABLED)
1662 		return (EINVAL);
1663 
1664 	if (cpu_is_offline(cp)) {
1665 		if (cp->cpu_flags & CPU_FAULTED) {
1666 			cp->cpu_flags &= ~CPU_FAULTED;
1667 			mp_cpu_faulted_exit(cp);
1668 		}
1669 		cp->cpu_flags |= CPU_SPARE;
1670 		cpu_set_state(cp);
1671 		return (0);
1672 	}
1673 
1674 	if ((error = cpu_offline(cp, flags)) == 0) {
1675 		cp->cpu_flags |= CPU_SPARE;
1676 		cpu_set_state(cp);
1677 	}
1678 
1679 	return (error);
1680 }
1681 
1682 /*
1683  * Take the indicated CPU from poweroff to offline.
1684  */
1685 int
cpu_poweron(cpu_t * cp)1686 cpu_poweron(cpu_t *cp)
1687 {
1688 	int	error = ENOTSUP;
1689 
1690 	ASSERT(MUTEX_HELD(&cpu_lock));
1691 	ASSERT(cpu_is_poweredoff(cp));
1692 
1693 	error = mp_cpu_poweron(cp);	/* arch-dep hook */
1694 	if (error == 0)
1695 		cpu_set_state(cp);
1696 
1697 	return (error);
1698 }
1699 
1700 /*
1701  * Take the indicated CPU from any inactive state to powered off.
1702  */
1703 int
cpu_poweroff(cpu_t * cp)1704 cpu_poweroff(cpu_t *cp)
1705 {
1706 	int	error = ENOTSUP;
1707 
1708 	ASSERT(MUTEX_HELD(&cpu_lock));
1709 	ASSERT(cpu_is_offline(cp));
1710 
1711 	if (!(cp->cpu_flags & CPU_QUIESCED))
1712 		return (EBUSY);		/* not completely idle */
1713 
1714 	error = mp_cpu_poweroff(cp);	/* arch-dep hook */
1715 	if (error == 0)
1716 		cpu_set_state(cp);
1717 
1718 	return (error);
1719 }
1720 
1721 /*
1722  * Initialize the Sequential CPU id lookup table
1723  */
1724 void
cpu_seq_tbl_init()1725 cpu_seq_tbl_init()
1726 {
1727 	cpu_t	**tbl;
1728 
1729 	tbl = kmem_zalloc(sizeof (struct cpu *) * max_ncpus, KM_SLEEP);
1730 	tbl[0] = CPU;
1731 
1732 	cpu_seq = tbl;
1733 }
1734 
1735 /*
1736  * Initialize the CPU lists for the first CPU.
1737  */
1738 void
cpu_list_init(cpu_t * cp)1739 cpu_list_init(cpu_t *cp)
1740 {
1741 	cp->cpu_next = cp;
1742 	cp->cpu_prev = cp;
1743 	cpu_list = cp;
1744 	clock_cpu_list = cp;
1745 
1746 	cp->cpu_next_onln = cp;
1747 	cp->cpu_prev_onln = cp;
1748 	cpu_active = cp;
1749 
1750 	cp->cpu_seqid = 0;
1751 	CPUSET_ADD(cpu_seqid_inuse, 0);
1752 
1753 	/*
1754 	 * Bootstrap cpu_seq using cpu_list
1755 	 * The cpu_seq[] table will be dynamically allocated
1756 	 * when kmem later becomes available (but before going MP)
1757 	 */
1758 	cpu_seq = &cpu_list;
1759 
1760 	cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1761 	cp_default.cp_cpulist = cp;
1762 	cp_default.cp_ncpus = 1;
1763 	cp->cpu_next_part = cp;
1764 	cp->cpu_prev_part = cp;
1765 	cp->cpu_part = &cp_default;
1766 
1767 	CPUSET_ADD(cpu_available, cp->cpu_id);
1768 	CPUSET_ADD(cpu_active_set, cp->cpu_id);
1769 }
1770 
1771 /*
1772  * Insert a CPU into the list of available CPUs.
1773  */
1774 void
cpu_add_unit(cpu_t * cp)1775 cpu_add_unit(cpu_t *cp)
1776 {
1777 	int seqid;
1778 
1779 	ASSERT(MUTEX_HELD(&cpu_lock));
1780 	ASSERT(cpu_list != NULL);	/* list started in cpu_list_init */
1781 
1782 	lgrp_config(LGRP_CONFIG_CPU_ADD, (uintptr_t)cp, 0);
1783 
1784 	/*
1785 	 * Note: most users of the cpu_list will grab the
1786 	 * cpu_lock to insure that it isn't modified.  However,
1787 	 * certain users can't or won't do that.  To allow this
1788 	 * we pause the other cpus.  Users who walk the list
1789 	 * without cpu_lock, must disable kernel preemption
1790 	 * to insure that the list isn't modified underneath
1791 	 * them.  Also, any cached pointers to cpu structures
1792 	 * must be revalidated by checking to see if the
1793 	 * cpu_next pointer points to itself.  This check must
1794 	 * be done with the cpu_lock held or kernel preemption
1795 	 * disabled.  This check relies upon the fact that
1796 	 * old cpu structures are not free'ed or cleared after
1797 	 * then are removed from the cpu_list.
1798 	 *
1799 	 * Note that the clock code walks the cpu list dereferencing
1800 	 * the cpu_part pointer, so we need to initialize it before
1801 	 * adding the cpu to the list.
1802 	 */
1803 	cp->cpu_part = &cp_default;
1804 	pause_cpus(NULL, NULL);
1805 	cp->cpu_next = cpu_list;
1806 	cp->cpu_prev = cpu_list->cpu_prev;
1807 	cpu_list->cpu_prev->cpu_next = cp;
1808 	cpu_list->cpu_prev = cp;
1809 	start_cpus();
1810 
1811 	for (seqid = 0; CPU_IN_SET(cpu_seqid_inuse, seqid); seqid++)
1812 		continue;
1813 	CPUSET_ADD(cpu_seqid_inuse, seqid);
1814 	cp->cpu_seqid = seqid;
1815 
1816 	if (seqid > max_cpu_seqid_ever)
1817 		max_cpu_seqid_ever = seqid;
1818 
1819 	ASSERT(ncpus < max_ncpus);
1820 	ncpus++;
1821 	cp->cpu_cache_offset = KMEM_CPU_CACHE_OFFSET(cp->cpu_seqid);
1822 	cpu[cp->cpu_id] = cp;
1823 	CPUSET_ADD(cpu_available, cp->cpu_id);
1824 	cpu_seq[cp->cpu_seqid] = cp;
1825 
1826 	/*
1827 	 * allocate a pause thread for this CPU.
1828 	 */
1829 	cpu_pause_alloc(cp);
1830 
1831 	/*
1832 	 * So that new CPUs won't have NULL prev_onln and next_onln pointers,
1833 	 * link them into a list of just that CPU.
1834 	 * This is so that disp_lowpri_cpu will work for thread_create in
1835 	 * pause_cpus() when called from the startup thread in a new CPU.
1836 	 */
1837 	cp->cpu_next_onln = cp;
1838 	cp->cpu_prev_onln = cp;
1839 	cpu_info_kstat_create(cp);
1840 	cp->cpu_next_part = cp;
1841 	cp->cpu_prev_part = cp;
1842 
1843 	init_cpu_mstate(cp, CMS_SYSTEM);
1844 
1845 	pool_pset_mod = gethrtime();
1846 }
1847 
1848 /*
1849  * Do the opposite of cpu_add_unit().
1850  */
1851 void
cpu_del_unit(int cpuid)1852 cpu_del_unit(int cpuid)
1853 {
1854 	struct cpu	*cp, *cpnext;
1855 
1856 	ASSERT(MUTEX_HELD(&cpu_lock));
1857 	cp = cpu[cpuid];
1858 	ASSERT(cp != NULL);
1859 
1860 	ASSERT(cp->cpu_next_onln == cp);
1861 	ASSERT(cp->cpu_prev_onln == cp);
1862 	ASSERT(cp->cpu_next_part == cp);
1863 	ASSERT(cp->cpu_prev_part == cp);
1864 
1865 	/*
1866 	 * Tear down the CPU's physical ID cache, and update any
1867 	 * processor groups
1868 	 */
1869 	pg_cpu_fini(cp, NULL);
1870 	pghw_physid_destroy(cp);
1871 
1872 	/*
1873 	 * Destroy kstat stuff.
1874 	 */
1875 	cpu_info_kstat_destroy(cp);
1876 	term_cpu_mstate(cp);
1877 	/*
1878 	 * Free up pause thread.
1879 	 */
1880 	cpu_pause_free(cp);
1881 	CPUSET_DEL(cpu_available, cp->cpu_id);
1882 	cpu[cp->cpu_id] = NULL;
1883 	cpu_seq[cp->cpu_seqid] = NULL;
1884 
1885 	/*
1886 	 * The clock thread and mutex_vector_enter cannot hold the
1887 	 * cpu_lock while traversing the cpu list, therefore we pause
1888 	 * all other threads by pausing the other cpus. These, and any
1889 	 * other routines holding cpu pointers while possibly sleeping
1890 	 * must be sure to call kpreempt_disable before processing the
1891 	 * list and be sure to check that the cpu has not been deleted
1892 	 * after any sleeps (check cp->cpu_next != NULL). We guarantee
1893 	 * to keep the deleted cpu structure around.
1894 	 *
1895 	 * Note that this MUST be done AFTER cpu_available
1896 	 * has been updated so that we don't waste time
1897 	 * trying to pause the cpu we're trying to delete.
1898 	 */
1899 	pause_cpus(NULL, NULL);
1900 
1901 	cpnext = cp->cpu_next;
1902 	cp->cpu_prev->cpu_next = cp->cpu_next;
1903 	cp->cpu_next->cpu_prev = cp->cpu_prev;
1904 	if (cp == cpu_list)
1905 		cpu_list = cpnext;
1906 
1907 	/*
1908 	 * Signals that the cpu has been deleted (see above).
1909 	 */
1910 	cp->cpu_next = NULL;
1911 	cp->cpu_prev = NULL;
1912 
1913 	start_cpus();
1914 
1915 	CPUSET_DEL(cpu_seqid_inuse, cp->cpu_seqid);
1916 	ncpus--;
1917 	lgrp_config(LGRP_CONFIG_CPU_DEL, (uintptr_t)cp, 0);
1918 
1919 	pool_pset_mod = gethrtime();
1920 }
1921 
1922 /*
1923  * Add a CPU to the list of active CPUs.
1924  *	This routine must not get any locks, because other CPUs are paused.
1925  */
1926 static void
cpu_add_active_internal(cpu_t * cp)1927 cpu_add_active_internal(cpu_t *cp)
1928 {
1929 	cpupart_t	*pp = cp->cpu_part;
1930 
1931 	ASSERT(MUTEX_HELD(&cpu_lock));
1932 	ASSERT(cpu_list != NULL);	/* list started in cpu_list_init */
1933 
1934 	ncpus_online++;
1935 	cpu_set_state(cp);
1936 	cp->cpu_next_onln = cpu_active;
1937 	cp->cpu_prev_onln = cpu_active->cpu_prev_onln;
1938 	cpu_active->cpu_prev_onln->cpu_next_onln = cp;
1939 	cpu_active->cpu_prev_onln = cp;
1940 	CPUSET_ADD(cpu_active_set, cp->cpu_id);
1941 
1942 	if (pp->cp_cpulist) {
1943 		cp->cpu_next_part = pp->cp_cpulist;
1944 		cp->cpu_prev_part = pp->cp_cpulist->cpu_prev_part;
1945 		pp->cp_cpulist->cpu_prev_part->cpu_next_part = cp;
1946 		pp->cp_cpulist->cpu_prev_part = cp;
1947 	} else {
1948 		ASSERT(pp->cp_ncpus == 0);
1949 		pp->cp_cpulist = cp->cpu_next_part = cp->cpu_prev_part = cp;
1950 	}
1951 	pp->cp_ncpus++;
1952 	if (pp->cp_ncpus == 1) {
1953 		cp_numparts_nonempty++;
1954 		ASSERT(cp_numparts_nonempty != 0);
1955 	}
1956 
1957 	pg_cpu_active(cp);
1958 	lgrp_config(LGRP_CONFIG_CPU_ONLINE, (uintptr_t)cp, 0);
1959 
1960 	bzero(&cp->cpu_loadavg, sizeof (cp->cpu_loadavg));
1961 }
1962 
1963 /*
1964  * Add a CPU to the list of active CPUs.
1965  *	This is called from machine-dependent layers when a new CPU is started.
1966  */
1967 void
cpu_add_active(cpu_t * cp)1968 cpu_add_active(cpu_t *cp)
1969 {
1970 	pg_cpupart_in(cp, cp->cpu_part);
1971 
1972 	pause_cpus(NULL, NULL);
1973 	cpu_add_active_internal(cp);
1974 	start_cpus();
1975 
1976 	cpu_stats_kstat_create(cp);
1977 	cpu_create_intrstat(cp);
1978 	lgrp_kstat_create(cp);
1979 	cpu_state_change_notify(cp->cpu_id, CPU_INIT);
1980 }
1981 
1982 
1983 /*
1984  * Remove a CPU from the list of active CPUs.
1985  *	This routine must not get any locks, because other CPUs are paused.
1986  */
1987 /* ARGSUSED */
1988 static void
cpu_remove_active(cpu_t * cp)1989 cpu_remove_active(cpu_t *cp)
1990 {
1991 	cpupart_t	*pp = cp->cpu_part;
1992 
1993 	ASSERT(MUTEX_HELD(&cpu_lock));
1994 	ASSERT(cp->cpu_next_onln != cp);	/* not the last one */
1995 	ASSERT(cp->cpu_prev_onln != cp);	/* not the last one */
1996 
1997 	pg_cpu_inactive(cp);
1998 
1999 	lgrp_config(LGRP_CONFIG_CPU_OFFLINE, (uintptr_t)cp, 0);
2000 
2001 	if (cp == clock_cpu_list)
2002 		clock_cpu_list = cp->cpu_next_onln;
2003 
2004 	cp->cpu_prev_onln->cpu_next_onln = cp->cpu_next_onln;
2005 	cp->cpu_next_onln->cpu_prev_onln = cp->cpu_prev_onln;
2006 	if (cpu_active == cp) {
2007 		cpu_active = cp->cpu_next_onln;
2008 	}
2009 	cp->cpu_next_onln = cp;
2010 	cp->cpu_prev_onln = cp;
2011 	CPUSET_DEL(cpu_active_set, cp->cpu_id);
2012 
2013 	cp->cpu_prev_part->cpu_next_part = cp->cpu_next_part;
2014 	cp->cpu_next_part->cpu_prev_part = cp->cpu_prev_part;
2015 	if (pp->cp_cpulist == cp) {
2016 		pp->cp_cpulist = cp->cpu_next_part;
2017 		ASSERT(pp->cp_cpulist != cp);
2018 	}
2019 	cp->cpu_next_part = cp;
2020 	cp->cpu_prev_part = cp;
2021 	pp->cp_ncpus--;
2022 	if (pp->cp_ncpus == 0) {
2023 		cp_numparts_nonempty--;
2024 		ASSERT(cp_numparts_nonempty != 0);
2025 	}
2026 }
2027 
2028 /*
2029  * Routine used to setup a newly inserted CPU in preparation for starting
2030  * it running code.
2031  */
2032 int
cpu_configure(int cpuid)2033 cpu_configure(int cpuid)
2034 {
2035 	int retval = 0;
2036 
2037 	ASSERT(MUTEX_HELD(&cpu_lock));
2038 
2039 	/*
2040 	 * Some structures are statically allocated based upon
2041 	 * the maximum number of cpus the system supports.  Do not
2042 	 * try to add anything beyond this limit.
2043 	 */
2044 	if (cpuid < 0 || cpuid >= NCPU) {
2045 		return (EINVAL);
2046 	}
2047 
2048 	if ((cpu[cpuid] != NULL) && (cpu[cpuid]->cpu_flags != 0)) {
2049 		return (EALREADY);
2050 	}
2051 
2052 	if ((retval = mp_cpu_configure(cpuid)) != 0) {
2053 		return (retval);
2054 	}
2055 
2056 	cpu[cpuid]->cpu_flags = CPU_QUIESCED | CPU_OFFLINE | CPU_POWEROFF;
2057 	cpu_set_state(cpu[cpuid]);
2058 	retval = cpu_state_change_hooks(cpuid, CPU_CONFIG, CPU_UNCONFIG);
2059 	if (retval != 0)
2060 		(void) mp_cpu_unconfigure(cpuid);
2061 
2062 	return (retval);
2063 }
2064 
2065 /*
2066  * Routine used to cleanup a CPU that has been powered off.  This will
2067  * destroy all per-cpu information related to this cpu.
2068  */
2069 int
cpu_unconfigure(int cpuid)2070 cpu_unconfigure(int cpuid)
2071 {
2072 	int error;
2073 
2074 	ASSERT(MUTEX_HELD(&cpu_lock));
2075 
2076 	if (cpu[cpuid] == NULL) {
2077 		return (ENODEV);
2078 	}
2079 
2080 	if (cpu[cpuid]->cpu_flags == 0) {
2081 		return (EALREADY);
2082 	}
2083 
2084 	if ((cpu[cpuid]->cpu_flags & CPU_POWEROFF) == 0) {
2085 		return (EBUSY);
2086 	}
2087 
2088 	if (cpu[cpuid]->cpu_props != NULL) {
2089 		(void) nvlist_free(cpu[cpuid]->cpu_props);
2090 		cpu[cpuid]->cpu_props = NULL;
2091 	}
2092 
2093 	error = cpu_state_change_hooks(cpuid, CPU_UNCONFIG, CPU_CONFIG);
2094 
2095 	if (error != 0)
2096 		return (error);
2097 
2098 	return (mp_cpu_unconfigure(cpuid));
2099 }
2100 
2101 /*
2102  * Routines for registering and de-registering cpu_setup callback functions.
2103  *
2104  * Caller's context
2105  *	These routines must not be called from a driver's attach(9E) or
2106  *	detach(9E) entry point.
2107  *
2108  * NOTE: CPU callbacks should not block. They are called with cpu_lock held.
2109  */
2110 
2111 /*
2112  * Ideally, these would be dynamically allocated and put into a linked
2113  * list; however that is not feasible because the registration routine
2114  * has to be available before the kmem allocator is working (in fact,
2115  * it is called by the kmem allocator init code).  In any case, there
2116  * are quite a few extra entries for future users.
2117  */
2118 #define	NCPU_SETUPS	20
2119 
2120 struct cpu_setup {
2121 	cpu_setup_func_t *func;
2122 	void *arg;
2123 } cpu_setups[NCPU_SETUPS];
2124 
2125 void
register_cpu_setup_func(cpu_setup_func_t * func,void * arg)2126 register_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2127 {
2128 	int i;
2129 
2130 	ASSERT(MUTEX_HELD(&cpu_lock));
2131 
2132 	for (i = 0; i < NCPU_SETUPS; i++)
2133 		if (cpu_setups[i].func == NULL)
2134 			break;
2135 	if (i >= NCPU_SETUPS)
2136 		cmn_err(CE_PANIC, "Ran out of cpu_setup callback entries");
2137 
2138 	cpu_setups[i].func = func;
2139 	cpu_setups[i].arg = arg;
2140 }
2141 
2142 void
unregister_cpu_setup_func(cpu_setup_func_t * func,void * arg)2143 unregister_cpu_setup_func(cpu_setup_func_t *func, void *arg)
2144 {
2145 	int i;
2146 
2147 	ASSERT(MUTEX_HELD(&cpu_lock));
2148 
2149 	for (i = 0; i < NCPU_SETUPS; i++)
2150 		if ((cpu_setups[i].func == func) &&
2151 		    (cpu_setups[i].arg == arg))
2152 			break;
2153 	if (i >= NCPU_SETUPS)
2154 		cmn_err(CE_PANIC, "Could not find cpu_setup callback to "
2155 		    "deregister");
2156 
2157 	cpu_setups[i].func = NULL;
2158 	cpu_setups[i].arg = 0;
2159 }
2160 
2161 /*
2162  * Call any state change hooks for this CPU, ignore any errors.
2163  */
2164 void
cpu_state_change_notify(int id,cpu_setup_t what)2165 cpu_state_change_notify(int id, cpu_setup_t what)
2166 {
2167 	int i;
2168 
2169 	ASSERT(MUTEX_HELD(&cpu_lock));
2170 
2171 	for (i = 0; i < NCPU_SETUPS; i++) {
2172 		if (cpu_setups[i].func != NULL) {
2173 			cpu_setups[i].func(what, id, cpu_setups[i].arg);
2174 		}
2175 	}
2176 }
2177 
2178 /*
2179  * Call any state change hooks for this CPU, undo it if error found.
2180  */
2181 static int
cpu_state_change_hooks(int id,cpu_setup_t what,cpu_setup_t undo)2182 cpu_state_change_hooks(int id, cpu_setup_t what, cpu_setup_t undo)
2183 {
2184 	int i;
2185 	int retval = 0;
2186 
2187 	ASSERT(MUTEX_HELD(&cpu_lock));
2188 
2189 	for (i = 0; i < NCPU_SETUPS; i++) {
2190 		if (cpu_setups[i].func != NULL) {
2191 			retval = cpu_setups[i].func(what, id,
2192 			    cpu_setups[i].arg);
2193 			if (retval) {
2194 				for (i--; i >= 0; i--) {
2195 					if (cpu_setups[i].func != NULL)
2196 						cpu_setups[i].func(undo,
2197 						    id, cpu_setups[i].arg);
2198 				}
2199 				break;
2200 			}
2201 		}
2202 	}
2203 	return (retval);
2204 }
2205 
2206 /*
2207  * Export information about this CPU via the kstat mechanism.
2208  */
2209 static struct {
2210 	kstat_named_t ci_state;
2211 	kstat_named_t ci_state_begin;
2212 	kstat_named_t ci_cpu_type;
2213 	kstat_named_t ci_fpu_type;
2214 	kstat_named_t ci_clock_MHz;
2215 	kstat_named_t ci_chip_id;
2216 	kstat_named_t ci_implementation;
2217 	kstat_named_t ci_brandstr;
2218 	kstat_named_t ci_core_id;
2219 	kstat_named_t ci_curr_clock_Hz;
2220 	kstat_named_t ci_supp_freq_Hz;
2221 	kstat_named_t ci_pg_id;
2222 #if defined(__sparcv9)
2223 	kstat_named_t ci_device_ID;
2224 	kstat_named_t ci_cpu_fru;
2225 #endif
2226 #if defined(__x86)
2227 	kstat_named_t ci_vendorstr;
2228 	kstat_named_t ci_family;
2229 	kstat_named_t ci_model;
2230 	kstat_named_t ci_step;
2231 	kstat_named_t ci_clogid;
2232 	kstat_named_t ci_pkg_core_id;
2233 	kstat_named_t ci_ncpuperchip;
2234 	kstat_named_t ci_ncoreperchip;
2235 	kstat_named_t ci_max_cstates;
2236 	kstat_named_t ci_curr_cstate;
2237 	kstat_named_t ci_cacheid;
2238 	kstat_named_t ci_sktstr;
2239 #endif
2240 } cpu_info_template = {
2241 	{ "state",			KSTAT_DATA_CHAR },
2242 	{ "state_begin",		KSTAT_DATA_LONG },
2243 	{ "cpu_type",			KSTAT_DATA_CHAR },
2244 	{ "fpu_type",			KSTAT_DATA_CHAR },
2245 	{ "clock_MHz",			KSTAT_DATA_LONG },
2246 	{ "chip_id",			KSTAT_DATA_LONG },
2247 	{ "implementation",		KSTAT_DATA_STRING },
2248 	{ "brand",			KSTAT_DATA_STRING },
2249 	{ "core_id",			KSTAT_DATA_LONG },
2250 	{ "current_clock_Hz",		KSTAT_DATA_UINT64 },
2251 	{ "supported_frequencies_Hz",	KSTAT_DATA_STRING },
2252 	{ "pg_id",			KSTAT_DATA_LONG },
2253 #if defined(__sparcv9)
2254 	{ "device_ID",			KSTAT_DATA_UINT64 },
2255 	{ "cpu_fru",			KSTAT_DATA_STRING },
2256 #endif
2257 #if defined(__x86)
2258 	{ "vendor_id",			KSTAT_DATA_STRING },
2259 	{ "family",			KSTAT_DATA_INT32 },
2260 	{ "model",			KSTAT_DATA_INT32 },
2261 	{ "stepping",			KSTAT_DATA_INT32 },
2262 	{ "clog_id",			KSTAT_DATA_INT32 },
2263 	{ "pkg_core_id",		KSTAT_DATA_LONG },
2264 	{ "ncpu_per_chip",		KSTAT_DATA_INT32 },
2265 	{ "ncore_per_chip",		KSTAT_DATA_INT32 },
2266 	{ "supported_max_cstates",	KSTAT_DATA_INT32 },
2267 	{ "current_cstate",		KSTAT_DATA_INT32 },
2268 	{ "cache_id",			KSTAT_DATA_INT32 },
2269 	{ "socket_type",		KSTAT_DATA_STRING },
2270 #endif
2271 };
2272 
2273 static kmutex_t cpu_info_template_lock;
2274 
2275 static int
cpu_info_kstat_update(kstat_t * ksp,int rw)2276 cpu_info_kstat_update(kstat_t *ksp, int rw)
2277 {
2278 	cpu_t	*cp = ksp->ks_private;
2279 	const char *pi_state;
2280 
2281 	if (rw == KSTAT_WRITE)
2282 		return (EACCES);
2283 
2284 #if defined(__x86)
2285 	/* Is the cpu still initialising itself? */
2286 	if (cpuid_checkpass(cp, 1) == 0)
2287 		return (ENXIO);
2288 #endif
2289 
2290 	pi_state = cpu_get_state_str(cp->cpu_flags);
2291 
2292 	(void) strcpy(cpu_info_template.ci_state.value.c, pi_state);
2293 	cpu_info_template.ci_state_begin.value.l = cp->cpu_state_begin;
2294 	(void) strncpy(cpu_info_template.ci_cpu_type.value.c,
2295 	    cp->cpu_type_info.pi_processor_type, 15);
2296 	(void) strncpy(cpu_info_template.ci_fpu_type.value.c,
2297 	    cp->cpu_type_info.pi_fputypes, 15);
2298 	cpu_info_template.ci_clock_MHz.value.l = cp->cpu_type_info.pi_clock;
2299 	cpu_info_template.ci_chip_id.value.l =
2300 	    pg_plat_hw_instance_id(cp, PGHW_CHIP);
2301 	kstat_named_setstr(&cpu_info_template.ci_implementation,
2302 	    cp->cpu_idstr);
2303 	kstat_named_setstr(&cpu_info_template.ci_brandstr, cp->cpu_brandstr);
2304 	cpu_info_template.ci_core_id.value.l = pg_plat_get_core_id(cp);
2305 	cpu_info_template.ci_curr_clock_Hz.value.ui64 =
2306 	    cp->cpu_curr_clock;
2307 	cpu_info_template.ci_pg_id.value.l =
2308 	    cp->cpu_pg && cp->cpu_pg->cmt_lineage ?
2309 	    cp->cpu_pg->cmt_lineage->pg_id : -1;
2310 	kstat_named_setstr(&cpu_info_template.ci_supp_freq_Hz,
2311 	    cp->cpu_supp_freqs);
2312 #if defined(__sparcv9)
2313 	cpu_info_template.ci_device_ID.value.ui64 =
2314 	    cpunodes[cp->cpu_id].device_id;
2315 	kstat_named_setstr(&cpu_info_template.ci_cpu_fru, cpu_fru_fmri(cp));
2316 #endif
2317 #if defined(__x86)
2318 	kstat_named_setstr(&cpu_info_template.ci_vendorstr,
2319 	    cpuid_getvendorstr(cp));
2320 	cpu_info_template.ci_family.value.l = cpuid_getfamily(cp);
2321 	cpu_info_template.ci_model.value.l = cpuid_getmodel(cp);
2322 	cpu_info_template.ci_step.value.l = cpuid_getstep(cp);
2323 	cpu_info_template.ci_clogid.value.l = cpuid_get_clogid(cp);
2324 	cpu_info_template.ci_ncpuperchip.value.l = cpuid_get_ncpu_per_chip(cp);
2325 	cpu_info_template.ci_ncoreperchip.value.l =
2326 	    cpuid_get_ncore_per_chip(cp);
2327 	cpu_info_template.ci_pkg_core_id.value.l = cpuid_get_pkgcoreid(cp);
2328 	cpu_info_template.ci_max_cstates.value.l = cp->cpu_m.max_cstates;
2329 	cpu_info_template.ci_curr_cstate.value.l = cpu_idle_get_cpu_state(cp);
2330 	cpu_info_template.ci_cacheid.value.i32 = cpuid_get_cacheid(cp);
2331 	kstat_named_setstr(&cpu_info_template.ci_sktstr,
2332 	    cpuid_getsocketstr(cp));
2333 #endif
2334 
2335 	return (0);
2336 }
2337 
2338 static void
cpu_info_kstat_create(cpu_t * cp)2339 cpu_info_kstat_create(cpu_t *cp)
2340 {
2341 	zoneid_t zoneid;
2342 
2343 	ASSERT(MUTEX_HELD(&cpu_lock));
2344 
2345 	if (pool_pset_enabled())
2346 		zoneid = GLOBAL_ZONEID;
2347 	else
2348 		zoneid = ALL_ZONES;
2349 	if ((cp->cpu_info_kstat = kstat_create_zone("cpu_info", cp->cpu_id,
2350 	    NULL, "misc", KSTAT_TYPE_NAMED,
2351 	    sizeof (cpu_info_template) / sizeof (kstat_named_t),
2352 	    KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_VAR_SIZE, zoneid)) != NULL) {
2353 		cp->cpu_info_kstat->ks_data_size += 2 * CPU_IDSTRLEN;
2354 #if defined(__sparcv9)
2355 		cp->cpu_info_kstat->ks_data_size +=
2356 		    strlen(cpu_fru_fmri(cp)) + 1;
2357 #endif
2358 #if defined(__x86)
2359 		cp->cpu_info_kstat->ks_data_size += X86_VENDOR_STRLEN;
2360 #endif
2361 		if (cp->cpu_supp_freqs != NULL)
2362 			cp->cpu_info_kstat->ks_data_size +=
2363 			    strlen(cp->cpu_supp_freqs) + 1;
2364 		cp->cpu_info_kstat->ks_lock = &cpu_info_template_lock;
2365 		cp->cpu_info_kstat->ks_data = &cpu_info_template;
2366 		cp->cpu_info_kstat->ks_private = cp;
2367 		cp->cpu_info_kstat->ks_update = cpu_info_kstat_update;
2368 		kstat_install(cp->cpu_info_kstat);
2369 	}
2370 }
2371 
2372 static void
cpu_info_kstat_destroy(cpu_t * cp)2373 cpu_info_kstat_destroy(cpu_t *cp)
2374 {
2375 	ASSERT(MUTEX_HELD(&cpu_lock));
2376 
2377 	kstat_delete(cp->cpu_info_kstat);
2378 	cp->cpu_info_kstat = NULL;
2379 }
2380 
2381 /*
2382  * Create and install kstats for the boot CPU.
2383  */
2384 void
cpu_kstat_init(cpu_t * cp)2385 cpu_kstat_init(cpu_t *cp)
2386 {
2387 	mutex_enter(&cpu_lock);
2388 	cpu_info_kstat_create(cp);
2389 	cpu_stats_kstat_create(cp);
2390 	cpu_create_intrstat(cp);
2391 	cpu_set_state(cp);
2392 	mutex_exit(&cpu_lock);
2393 }
2394 
2395 /*
2396  * Make visible to the zone that subset of the cpu information that would be
2397  * initialized when a cpu is configured (but still offline).
2398  */
2399 void
cpu_visibility_configure(cpu_t * cp,zone_t * zone)2400 cpu_visibility_configure(cpu_t *cp, zone_t *zone)
2401 {
2402 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2403 
2404 	ASSERT(MUTEX_HELD(&cpu_lock));
2405 	ASSERT(pool_pset_enabled());
2406 	ASSERT(cp != NULL);
2407 
2408 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2409 		zone->zone_ncpus++;
2410 		ASSERT(zone->zone_ncpus <= ncpus);
2411 	}
2412 	if (cp->cpu_info_kstat != NULL)
2413 		kstat_zone_add(cp->cpu_info_kstat, zoneid);
2414 }
2415 
2416 /*
2417  * Make visible to the zone that subset of the cpu information that would be
2418  * initialized when a previously configured cpu is onlined.
2419  */
2420 void
cpu_visibility_online(cpu_t * cp,zone_t * zone)2421 cpu_visibility_online(cpu_t *cp, zone_t *zone)
2422 {
2423 	kstat_t *ksp;
2424 	char name[sizeof ("cpu_stat") + 10];	/* enough for 32-bit cpuids */
2425 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2426 	processorid_t cpun;
2427 
2428 	ASSERT(MUTEX_HELD(&cpu_lock));
2429 	ASSERT(pool_pset_enabled());
2430 	ASSERT(cp != NULL);
2431 	ASSERT(cpu_is_active(cp));
2432 
2433 	cpun = cp->cpu_id;
2434 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2435 		zone->zone_ncpus_online++;
2436 		ASSERT(zone->zone_ncpus_online <= ncpus_online);
2437 	}
2438 	(void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2439 	if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2440 	    != NULL) {
2441 		kstat_zone_add(ksp, zoneid);
2442 		kstat_rele(ksp);
2443 	}
2444 	if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2445 		kstat_zone_add(ksp, zoneid);
2446 		kstat_rele(ksp);
2447 	}
2448 	if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2449 		kstat_zone_add(ksp, zoneid);
2450 		kstat_rele(ksp);
2451 	}
2452 	if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2453 	    NULL) {
2454 		kstat_zone_add(ksp, zoneid);
2455 		kstat_rele(ksp);
2456 	}
2457 }
2458 
2459 /*
2460  * Update relevant kstats such that cpu is now visible to processes
2461  * executing in specified zone.
2462  */
2463 void
cpu_visibility_add(cpu_t * cp,zone_t * zone)2464 cpu_visibility_add(cpu_t *cp, zone_t *zone)
2465 {
2466 	cpu_visibility_configure(cp, zone);
2467 	if (cpu_is_active(cp))
2468 		cpu_visibility_online(cp, zone);
2469 }
2470 
2471 /*
2472  * Make invisible to the zone that subset of the cpu information that would be
2473  * torn down when a previously offlined cpu is unconfigured.
2474  */
2475 void
cpu_visibility_unconfigure(cpu_t * cp,zone_t * zone)2476 cpu_visibility_unconfigure(cpu_t *cp, zone_t *zone)
2477 {
2478 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2479 
2480 	ASSERT(MUTEX_HELD(&cpu_lock));
2481 	ASSERT(pool_pset_enabled());
2482 	ASSERT(cp != NULL);
2483 
2484 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2485 		ASSERT(zone->zone_ncpus != 0);
2486 		zone->zone_ncpus--;
2487 	}
2488 	if (cp->cpu_info_kstat)
2489 		kstat_zone_remove(cp->cpu_info_kstat, zoneid);
2490 }
2491 
2492 /*
2493  * Make invisible to the zone that subset of the cpu information that would be
2494  * torn down when a cpu is offlined (but still configured).
2495  */
2496 void
cpu_visibility_offline(cpu_t * cp,zone_t * zone)2497 cpu_visibility_offline(cpu_t *cp, zone_t *zone)
2498 {
2499 	kstat_t *ksp;
2500 	char name[sizeof ("cpu_stat") + 10];	/* enough for 32-bit cpuids */
2501 	zoneid_t zoneid = zone ? zone->zone_id : ALL_ZONES;
2502 	processorid_t cpun;
2503 
2504 	ASSERT(MUTEX_HELD(&cpu_lock));
2505 	ASSERT(pool_pset_enabled());
2506 	ASSERT(cp != NULL);
2507 	ASSERT(cpu_is_active(cp));
2508 
2509 	cpun = cp->cpu_id;
2510 	if (zoneid != ALL_ZONES && zoneid != GLOBAL_ZONEID) {
2511 		ASSERT(zone->zone_ncpus_online != 0);
2512 		zone->zone_ncpus_online--;
2513 	}
2514 
2515 	if ((ksp = kstat_hold_byname("cpu", cpun, "intrstat", ALL_ZONES)) !=
2516 	    NULL) {
2517 		kstat_zone_remove(ksp, zoneid);
2518 		kstat_rele(ksp);
2519 	}
2520 	if ((ksp = kstat_hold_byname("cpu", cpun, "vm", ALL_ZONES)) != NULL) {
2521 		kstat_zone_remove(ksp, zoneid);
2522 		kstat_rele(ksp);
2523 	}
2524 	if ((ksp = kstat_hold_byname("cpu", cpun, "sys", ALL_ZONES)) != NULL) {
2525 		kstat_zone_remove(ksp, zoneid);
2526 		kstat_rele(ksp);
2527 	}
2528 	(void) snprintf(name, sizeof (name), "cpu_stat%d", cpun);
2529 	if ((ksp = kstat_hold_byname("cpu_stat", cpun, name, ALL_ZONES))
2530 	    != NULL) {
2531 		kstat_zone_remove(ksp, zoneid);
2532 		kstat_rele(ksp);
2533 	}
2534 }
2535 
2536 /*
2537  * Update relevant kstats such that cpu is no longer visible to processes
2538  * executing in specified zone.
2539  */
2540 void
cpu_visibility_remove(cpu_t * cp,zone_t * zone)2541 cpu_visibility_remove(cpu_t *cp, zone_t *zone)
2542 {
2543 	if (cpu_is_active(cp))
2544 		cpu_visibility_offline(cp, zone);
2545 	cpu_visibility_unconfigure(cp, zone);
2546 }
2547 
2548 /*
2549  * Bind a thread to a CPU as requested.
2550  */
2551 int
cpu_bind_thread(kthread_id_t tp,processorid_t bind,processorid_t * obind,int * error)2552 cpu_bind_thread(kthread_id_t tp, processorid_t bind, processorid_t *obind,
2553     int *error)
2554 {
2555 	processorid_t	binding;
2556 	cpu_t		*cp = NULL;
2557 
2558 	ASSERT(MUTEX_HELD(&cpu_lock));
2559 	ASSERT(MUTEX_HELD(&ttoproc(tp)->p_lock));
2560 
2561 	thread_lock(tp);
2562 
2563 	/*
2564 	 * Record old binding, but change the obind, which was initialized
2565 	 * to PBIND_NONE, only if this thread has a binding.  This avoids
2566 	 * reporting PBIND_NONE for a process when some LWPs are bound.
2567 	 */
2568 	binding = tp->t_bind_cpu;
2569 	if (binding != PBIND_NONE)
2570 		*obind = binding;	/* record old binding */
2571 
2572 	switch (bind) {
2573 	case PBIND_QUERY:
2574 		/* Just return the old binding */
2575 		thread_unlock(tp);
2576 		return (0);
2577 
2578 	case PBIND_QUERY_TYPE:
2579 		/* Return the binding type */
2580 		*obind = TB_CPU_IS_SOFT(tp) ? PBIND_SOFT : PBIND_HARD;
2581 		thread_unlock(tp);
2582 		return (0);
2583 
2584 	case PBIND_SOFT:
2585 		/*
2586 		 *  Set soft binding for this thread and return the actual
2587 		 *  binding
2588 		 */
2589 		TB_CPU_SOFT_SET(tp);
2590 		thread_unlock(tp);
2591 		return (0);
2592 
2593 	case PBIND_HARD:
2594 		/*
2595 		 *  Set hard binding for this thread and return the actual
2596 		 *  binding
2597 		 */
2598 		TB_CPU_HARD_SET(tp);
2599 		thread_unlock(tp);
2600 		return (0);
2601 
2602 	default:
2603 		break;
2604 	}
2605 
2606 	/*
2607 	 * If this thread/LWP cannot be bound because of permission
2608 	 * problems, just note that and return success so that the
2609 	 * other threads/LWPs will be bound.  This is the way
2610 	 * processor_bind() is defined to work.
2611 	 *
2612 	 * Binding will get EPERM if the thread is of system class
2613 	 * or hasprocperm() fails.
2614 	 */
2615 	if (tp->t_cid == 0 || !hasprocperm(tp->t_cred, CRED())) {
2616 		*error = EPERM;
2617 		thread_unlock(tp);
2618 		return (0);
2619 	}
2620 
2621 	binding = bind;
2622 	if (binding != PBIND_NONE) {
2623 		cp = cpu_get((processorid_t)binding);
2624 		/*
2625 		 * Make sure binding is valid and is in right partition.
2626 		 */
2627 		if (cp == NULL || tp->t_cpupart != cp->cpu_part) {
2628 			*error = EINVAL;
2629 			thread_unlock(tp);
2630 			return (0);
2631 		}
2632 	}
2633 	tp->t_bind_cpu = binding;	/* set new binding */
2634 
2635 	/*
2636 	 * If there is no system-set reason for affinity, set
2637 	 * the t_bound_cpu field to reflect the binding.
2638 	 */
2639 	if (tp->t_affinitycnt == 0) {
2640 		if (binding == PBIND_NONE) {
2641 			/*
2642 			 * We may need to adjust disp_max_unbound_pri
2643 			 * since we're becoming unbound.
2644 			 */
2645 			disp_adjust_unbound_pri(tp);
2646 
2647 			tp->t_bound_cpu = NULL;	/* set new binding */
2648 
2649 			/*
2650 			 * Move thread to lgroup with strongest affinity
2651 			 * after unbinding
2652 			 */
2653 			if (tp->t_lgrp_affinity)
2654 				lgrp_move_thread(tp,
2655 				    lgrp_choose(tp, tp->t_cpupart), 1);
2656 
2657 			if (tp->t_state == TS_ONPROC &&
2658 			    tp->t_cpu->cpu_part != tp->t_cpupart)
2659 				cpu_surrender(tp);
2660 		} else {
2661 			lpl_t	*lpl;
2662 
2663 			tp->t_bound_cpu = cp;
2664 			ASSERT(cp->cpu_lpl != NULL);
2665 
2666 			/*
2667 			 * Set home to lgroup with most affinity containing CPU
2668 			 * that thread is being bound or minimum bounding
2669 			 * lgroup if no affinities set
2670 			 */
2671 			if (tp->t_lgrp_affinity)
2672 				lpl = lgrp_affinity_best(tp, tp->t_cpupart,
2673 				    LGRP_NONE, B_FALSE);
2674 			else
2675 				lpl = cp->cpu_lpl;
2676 
2677 			if (tp->t_lpl != lpl) {
2678 				/* can't grab cpu_lock */
2679 				lgrp_move_thread(tp, lpl, 1);
2680 			}
2681 
2682 			/*
2683 			 * Make the thread switch to the bound CPU.
2684 			 * If the thread is runnable, we need to
2685 			 * requeue it even if t_cpu is already set
2686 			 * to the right CPU, since it may be on a
2687 			 * kpreempt queue and need to move to a local
2688 			 * queue.  We could check t_disp_queue to
2689 			 * avoid unnecessary overhead if it's already
2690 			 * on the right queue, but since this isn't
2691 			 * a performance-critical operation it doesn't
2692 			 * seem worth the extra code and complexity.
2693 			 *
2694 			 * If the thread is weakbound to the cpu then it will
2695 			 * resist the new binding request until the weak
2696 			 * binding drops.  The cpu_surrender or requeueing
2697 			 * below could be skipped in such cases (since it
2698 			 * will have no effect), but that would require
2699 			 * thread_allowmigrate to acquire thread_lock so
2700 			 * we'll take the very occasional hit here instead.
2701 			 */
2702 			if (tp->t_state == TS_ONPROC) {
2703 				cpu_surrender(tp);
2704 			} else if (tp->t_state == TS_RUN) {
2705 				cpu_t *ocp = tp->t_cpu;
2706 
2707 				(void) dispdeq(tp);
2708 				setbackdq(tp);
2709 				/*
2710 				 * Either on the bound CPU's disp queue now,
2711 				 * or swapped out or on the swap queue.
2712 				 */
2713 				ASSERT(tp->t_disp_queue == cp->cpu_disp ||
2714 				    tp->t_weakbound_cpu == ocp ||
2715 				    (tp->t_schedflag & (TS_LOAD | TS_ON_SWAPQ))
2716 				    != TS_LOAD);
2717 			}
2718 		}
2719 	}
2720 
2721 	/*
2722 	 * Our binding has changed; set TP_CHANGEBIND.
2723 	 */
2724 	tp->t_proc_flag |= TP_CHANGEBIND;
2725 	aston(tp);
2726 
2727 	thread_unlock(tp);
2728 
2729 	return (0);
2730 }
2731 
2732 
2733 cpuset_t *
cpuset_alloc(int kmflags)2734 cpuset_alloc(int kmflags)
2735 {
2736 	return (kmem_alloc(sizeof (cpuset_t), kmflags));
2737 }
2738 
2739 void
cpuset_free(cpuset_t * s)2740 cpuset_free(cpuset_t *s)
2741 {
2742 	kmem_free(s, sizeof (cpuset_t));
2743 }
2744 
2745 void
cpuset_all(cpuset_t * s)2746 cpuset_all(cpuset_t *s)
2747 {
2748 	int i;
2749 
2750 	for (i = 0; i < CPUSET_WORDS; i++)
2751 		s->cpub[i] = ~0UL;
2752 }
2753 
2754 void
cpuset_all_but(cpuset_t * s,const uint_t cpu)2755 cpuset_all_but(cpuset_t *s, const uint_t cpu)
2756 {
2757 	cpuset_all(s);
2758 	CPUSET_DEL(*s, cpu);
2759 }
2760 
2761 void
cpuset_only(cpuset_t * s,const uint_t cpu)2762 cpuset_only(cpuset_t *s, const uint_t cpu)
2763 {
2764 	CPUSET_ZERO(*s);
2765 	CPUSET_ADD(*s, cpu);
2766 }
2767 
2768 long
cpu_in_set(const cpuset_t * s,const uint_t cpu)2769 cpu_in_set(const cpuset_t *s, const uint_t cpu)
2770 {
2771 	VERIFY(cpu < NCPU);
2772 	return (BT_TEST(s->cpub, cpu));
2773 }
2774 
2775 void
cpuset_add(cpuset_t * s,const uint_t cpu)2776 cpuset_add(cpuset_t *s, const uint_t cpu)
2777 {
2778 	VERIFY(cpu < NCPU);
2779 	BT_SET(s->cpub, cpu);
2780 }
2781 
2782 void
cpuset_del(cpuset_t * s,const uint_t cpu)2783 cpuset_del(cpuset_t *s, const uint_t cpu)
2784 {
2785 	VERIFY(cpu < NCPU);
2786 	BT_CLEAR(s->cpub, cpu);
2787 }
2788 
2789 int
cpuset_isnull(const cpuset_t * s)2790 cpuset_isnull(const cpuset_t *s)
2791 {
2792 	int i;
2793 
2794 	for (i = 0; i < CPUSET_WORDS; i++) {
2795 		if (s->cpub[i] != 0)
2796 			return (0);
2797 	}
2798 	return (1);
2799 }
2800 
2801 int
cpuset_isequal(const cpuset_t * s1,const cpuset_t * s2)2802 cpuset_isequal(const cpuset_t *s1, const cpuset_t *s2)
2803 {
2804 	int i;
2805 
2806 	for (i = 0; i < CPUSET_WORDS; i++) {
2807 		if (s1->cpub[i] != s2->cpub[i])
2808 			return (0);
2809 	}
2810 	return (1);
2811 }
2812 
2813 uint_t
cpuset_find(const cpuset_t * s)2814 cpuset_find(const cpuset_t *s)
2815 {
2816 
2817 	uint_t	i;
2818 	uint_t	cpu = (uint_t)-1;
2819 
2820 	/*
2821 	 * Find a cpu in the cpuset
2822 	 */
2823 	for (i = 0; i < CPUSET_WORDS; i++) {
2824 		cpu = (uint_t)(lowbit(s->cpub[i]) - 1);
2825 		if (cpu != (uint_t)-1) {
2826 			cpu += i * BT_NBIPUL;
2827 			break;
2828 		}
2829 	}
2830 	return (cpu);
2831 }
2832 
2833 void
cpuset_bounds(const cpuset_t * s,uint_t * smallestid,uint_t * largestid)2834 cpuset_bounds(const cpuset_t *s, uint_t *smallestid, uint_t *largestid)
2835 {
2836 	int	i, j;
2837 	uint_t	bit;
2838 
2839 	/*
2840 	 * First, find the smallest cpu id in the set.
2841 	 */
2842 	for (i = 0; i < CPUSET_WORDS; i++) {
2843 		if (s->cpub[i] != 0) {
2844 			bit = (uint_t)(lowbit(s->cpub[i]) - 1);
2845 			ASSERT(bit != (uint_t)-1);
2846 			*smallestid = bit + (i * BT_NBIPUL);
2847 
2848 			/*
2849 			 * Now find the largest cpu id in
2850 			 * the set and return immediately.
2851 			 * Done in an inner loop to avoid
2852 			 * having to break out of the first
2853 			 * loop.
2854 			 */
2855 			for (j = CPUSET_WORDS - 1; j >= i; j--) {
2856 				if (s->cpub[j] != 0) {
2857 					bit = (uint_t)(highbit(s->cpub[j]) - 1);
2858 					ASSERT(bit != (uint_t)-1);
2859 					*largestid = bit + (j * BT_NBIPUL);
2860 					ASSERT(*largestid >= *smallestid);
2861 					return;
2862 				}
2863 			}
2864 
2865 			/*
2866 			 * If this code is reached, a
2867 			 * smallestid was found, but not a
2868 			 * largestid. The cpuset must have
2869 			 * been changed during the course
2870 			 * of this function call.
2871 			 */
2872 			ASSERT(0);
2873 		}
2874 	}
2875 	*smallestid = *largestid = CPUSET_NOTINSET;
2876 }
2877 
2878 void
cpuset_atomic_del(cpuset_t * s,const uint_t cpu)2879 cpuset_atomic_del(cpuset_t *s, const uint_t cpu)
2880 {
2881 	VERIFY(cpu < NCPU);
2882 	BT_ATOMIC_CLEAR(s->cpub, (cpu))
2883 }
2884 
2885 void
cpuset_atomic_add(cpuset_t * s,const uint_t cpu)2886 cpuset_atomic_add(cpuset_t *s, const uint_t cpu)
2887 {
2888 	VERIFY(cpu < NCPU);
2889 	BT_ATOMIC_SET(s->cpub, (cpu))
2890 }
2891 
2892 long
cpuset_atomic_xadd(cpuset_t * s,const uint_t cpu)2893 cpuset_atomic_xadd(cpuset_t *s, const uint_t cpu)
2894 {
2895 	long res;
2896 
2897 	VERIFY(cpu < NCPU);
2898 	BT_ATOMIC_SET_EXCL(s->cpub, cpu, res);
2899 	return (res);
2900 }
2901 
2902 long
cpuset_atomic_xdel(cpuset_t * s,const uint_t cpu)2903 cpuset_atomic_xdel(cpuset_t *s, const uint_t cpu)
2904 {
2905 	long res;
2906 
2907 	VERIFY(cpu < NCPU);
2908 	BT_ATOMIC_CLEAR_EXCL(s->cpub, cpu, res);
2909 	return (res);
2910 }
2911 
2912 void
cpuset_or(cpuset_t * dst,const cpuset_t * src)2913 cpuset_or(cpuset_t *dst, const cpuset_t *src)
2914 {
2915 	for (int i = 0; i < CPUSET_WORDS; i++) {
2916 		dst->cpub[i] |= src->cpub[i];
2917 	}
2918 }
2919 
2920 void
cpuset_xor(cpuset_t * dst,const cpuset_t * src)2921 cpuset_xor(cpuset_t *dst, const cpuset_t *src)
2922 {
2923 	for (int i = 0; i < CPUSET_WORDS; i++) {
2924 		dst->cpub[i] ^= src->cpub[i];
2925 	}
2926 }
2927 
2928 void
cpuset_and(cpuset_t * dst,const cpuset_t * src)2929 cpuset_and(cpuset_t *dst, const cpuset_t *src)
2930 {
2931 	for (int i = 0; i < CPUSET_WORDS; i++) {
2932 		dst->cpub[i] &= src->cpub[i];
2933 	}
2934 }
2935 
2936 void
cpuset_zero(cpuset_t * dst)2937 cpuset_zero(cpuset_t *dst)
2938 {
2939 	for (int i = 0; i < CPUSET_WORDS; i++) {
2940 		dst->cpub[i] = 0;
2941 	}
2942 }
2943 
2944 
2945 /*
2946  * Unbind threads bound to specified CPU.
2947  *
2948  * If `unbind_all_threads' is true, unbind all user threads bound to a given
2949  * CPU. Otherwise unbind all soft-bound user threads.
2950  */
2951 int
cpu_unbind(processorid_t cpu,boolean_t unbind_all_threads)2952 cpu_unbind(processorid_t cpu, boolean_t unbind_all_threads)
2953 {
2954 	processorid_t obind;
2955 	kthread_t *tp;
2956 	int ret = 0;
2957 	proc_t *pp;
2958 	int err, berr = 0;
2959 
2960 	ASSERT(MUTEX_HELD(&cpu_lock));
2961 
2962 	mutex_enter(&pidlock);
2963 	for (pp = practive; pp != NULL; pp = pp->p_next) {
2964 		mutex_enter(&pp->p_lock);
2965 		tp = pp->p_tlist;
2966 		/*
2967 		 * Skip zombies, kernel processes, and processes in
2968 		 * other zones, if called from a non-global zone.
2969 		 */
2970 		if (tp == NULL || (pp->p_flag & SSYS) ||
2971 		    !HASZONEACCESS(curproc, pp->p_zone->zone_id)) {
2972 			mutex_exit(&pp->p_lock);
2973 			continue;
2974 		}
2975 		do {
2976 			if (tp->t_bind_cpu != cpu)
2977 				continue;
2978 			/*
2979 			 * Skip threads with hard binding when
2980 			 * `unbind_all_threads' is not specified.
2981 			 */
2982 			if (!unbind_all_threads && TB_CPU_IS_HARD(tp))
2983 				continue;
2984 			err = cpu_bind_thread(tp, PBIND_NONE, &obind, &berr);
2985 			if (ret == 0)
2986 				ret = err;
2987 		} while ((tp = tp->t_forw) != pp->p_tlist);
2988 		mutex_exit(&pp->p_lock);
2989 	}
2990 	mutex_exit(&pidlock);
2991 	if (ret == 0)
2992 		ret = berr;
2993 	return (ret);
2994 }
2995 
2996 
2997 /*
2998  * Destroy all remaining bound threads on a cpu.
2999  */
3000 void
cpu_destroy_bound_threads(cpu_t * cp)3001 cpu_destroy_bound_threads(cpu_t *cp)
3002 {
3003 	extern id_t syscid;
3004 	register kthread_id_t	t, tlist, tnext;
3005 
3006 	/*
3007 	 * Destroy all remaining bound threads on the cpu.  This
3008 	 * should include both the interrupt threads and the idle thread.
3009 	 * This requires some care, since we need to traverse the
3010 	 * thread list with the pidlock mutex locked, but thread_free
3011 	 * also locks the pidlock mutex.  So, we collect the threads
3012 	 * we're going to reap in a list headed by "tlist", then we
3013 	 * unlock the pidlock mutex and traverse the tlist list,
3014 	 * doing thread_free's on the thread's.	 Simple, n'est pas?
3015 	 * Also, this depends on thread_free not mucking with the
3016 	 * t_next and t_prev links of the thread.
3017 	 */
3018 
3019 	if ((t = curthread) != NULL) {
3020 
3021 		tlist = NULL;
3022 		mutex_enter(&pidlock);
3023 		do {
3024 			tnext = t->t_next;
3025 			if (t->t_bound_cpu == cp) {
3026 
3027 				/*
3028 				 * We've found a bound thread, carefully unlink
3029 				 * it out of the thread list, and add it to
3030 				 * our "tlist".	 We "know" we don't have to
3031 				 * worry about unlinking curthread (the thread
3032 				 * that is executing this code).
3033 				 */
3034 				t->t_next->t_prev = t->t_prev;
3035 				t->t_prev->t_next = t->t_next;
3036 				t->t_next = tlist;
3037 				tlist = t;
3038 				ASSERT(t->t_cid == syscid);
3039 				/* wake up anyone blocked in thread_join */
3040 				cv_broadcast(&t->t_joincv);
3041 				/*
3042 				 * t_lwp set by interrupt threads and not
3043 				 * cleared.
3044 				 */
3045 				t->t_lwp = NULL;
3046 				/*
3047 				 * Pause and idle threads always have
3048 				 * t_state set to TS_ONPROC.
3049 				 */
3050 				t->t_state = TS_FREE;
3051 				t->t_prev = NULL;	/* Just in case */
3052 			}
3053 
3054 		} while ((t = tnext) != curthread);
3055 
3056 		mutex_exit(&pidlock);
3057 
3058 		mutex_sync();
3059 		for (t = tlist; t != NULL; t = tnext) {
3060 			tnext = t->t_next;
3061 			thread_free(t);
3062 		}
3063 	}
3064 }
3065 
3066 /*
3067  * Update the cpu_supp_freqs of this cpu. This information is returned
3068  * as part of cpu_info kstats. If the cpu_info_kstat exists already, then
3069  * maintain the kstat data size.
3070  */
3071 void
cpu_set_supp_freqs(cpu_t * cp,const char * freqs)3072 cpu_set_supp_freqs(cpu_t *cp, const char *freqs)
3073 {
3074 	char clkstr[sizeof ("18446744073709551615") + 1]; /* ui64 MAX */
3075 	const char *lfreqs = clkstr;
3076 	boolean_t kstat_exists = B_FALSE;
3077 	kstat_t *ksp;
3078 	size_t len;
3079 
3080 	/*
3081 	 * A NULL pointer means we only support one speed.
3082 	 */
3083 	if (freqs == NULL)
3084 		(void) snprintf(clkstr, sizeof (clkstr), "%"PRIu64,
3085 		    cp->cpu_curr_clock);
3086 	else
3087 		lfreqs = freqs;
3088 
3089 	/*
3090 	 * Make sure the frequency doesn't change while a snapshot is
3091 	 * going on. Of course, we only need to worry about this if
3092 	 * the kstat exists.
3093 	 */
3094 	if ((ksp = cp->cpu_info_kstat) != NULL) {
3095 		mutex_enter(ksp->ks_lock);
3096 		kstat_exists = B_TRUE;
3097 	}
3098 
3099 	/*
3100 	 * Free any previously allocated string and if the kstat
3101 	 * already exists, then update its data size.
3102 	 */
3103 	if (cp->cpu_supp_freqs != NULL) {
3104 		len = strlen(cp->cpu_supp_freqs) + 1;
3105 		kmem_free(cp->cpu_supp_freqs, len);
3106 		if (kstat_exists)
3107 			ksp->ks_data_size -= len;
3108 	}
3109 
3110 	/*
3111 	 * Allocate the new string and set the pointer.
3112 	 */
3113 	len = strlen(lfreqs) + 1;
3114 	cp->cpu_supp_freqs = kmem_alloc(len, KM_SLEEP);
3115 	(void) strcpy(cp->cpu_supp_freqs, lfreqs);
3116 
3117 	/*
3118 	 * If the kstat already exists then update the data size and
3119 	 * free the lock.
3120 	 */
3121 	if (kstat_exists) {
3122 		ksp->ks_data_size += len;
3123 		mutex_exit(ksp->ks_lock);
3124 	}
3125 }
3126 
3127 /*
3128  * Indicate the current CPU's clock freqency (in Hz).
3129  * The calling context must be such that CPU references are safe.
3130  */
3131 void
cpu_set_curr_clock(uint64_t new_clk)3132 cpu_set_curr_clock(uint64_t new_clk)
3133 {
3134 	uint64_t old_clk;
3135 
3136 	old_clk = CPU->cpu_curr_clock;
3137 	CPU->cpu_curr_clock = new_clk;
3138 
3139 	/*
3140 	 * The cpu-change-speed DTrace probe exports the frequency in Hz
3141 	 */
3142 	DTRACE_PROBE3(cpu__change__speed, processorid_t, CPU->cpu_id,
3143 	    uint64_t, old_clk, uint64_t, new_clk);
3144 }
3145 
3146 /*
3147  * processor_info(2) and p_online(2) status support functions
3148  *   The constants returned by the cpu_get_state() and cpu_get_state_str() are
3149  *   for use in communicating processor state information to userland.  Kernel
3150  *   subsystems should only be using the cpu_flags value directly.  Subsystems
3151  *   modifying cpu_flags should record the state change via a call to the
3152  *   cpu_set_state().
3153  */
3154 
3155 /*
3156  * Update the pi_state of this CPU.  This function provides the CPU status for
3157  * the information returned by processor_info(2).
3158  */
3159 void
cpu_set_state(cpu_t * cpu)3160 cpu_set_state(cpu_t *cpu)
3161 {
3162 	ASSERT(MUTEX_HELD(&cpu_lock));
3163 	cpu->cpu_type_info.pi_state = cpu_get_state(cpu);
3164 	cpu->cpu_state_begin = gethrestime_sec();
3165 	pool_cpu_mod = gethrtime();
3166 }
3167 
3168 /*
3169  * Return offline/online/other status for the indicated CPU.  Use only for
3170  * communication with user applications; cpu_flags provides the in-kernel
3171  * interface.
3172  */
3173 static int
cpu_flags_to_state(cpu_flag_t flags)3174 cpu_flags_to_state(cpu_flag_t flags)
3175 {
3176 	if (flags & CPU_DISABLED)
3177 		return (P_DISABLED);
3178 	else if (flags & CPU_POWEROFF)
3179 		return (P_POWEROFF);
3180 	else if (flags & CPU_FAULTED)
3181 		return (P_FAULTED);
3182 	else if (flags & CPU_SPARE)
3183 		return (P_SPARE);
3184 	else if ((flags & (CPU_READY | CPU_OFFLINE)) != CPU_READY)
3185 		return (P_OFFLINE);
3186 	else if (flags & CPU_ENABLE)
3187 		return (P_ONLINE);
3188 	else
3189 		return (P_NOINTR);
3190 }
3191 
3192 int
cpu_get_state(cpu_t * cpu)3193 cpu_get_state(cpu_t *cpu)
3194 {
3195 	ASSERT(MUTEX_HELD(&cpu_lock));
3196 	return (cpu_flags_to_state(cpu->cpu_flags));
3197 }
3198 
3199 /*
3200  * Return processor_info(2) state as a string.
3201  */
3202 const char *
cpu_get_state_str(cpu_flag_t flags)3203 cpu_get_state_str(cpu_flag_t flags)
3204 {
3205 	const char *string;
3206 
3207 	switch (cpu_flags_to_state(flags)) {
3208 	case P_ONLINE:
3209 		string = PS_ONLINE;
3210 		break;
3211 	case P_POWEROFF:
3212 		string = PS_POWEROFF;
3213 		break;
3214 	case P_NOINTR:
3215 		string = PS_NOINTR;
3216 		break;
3217 	case P_SPARE:
3218 		string = PS_SPARE;
3219 		break;
3220 	case P_FAULTED:
3221 		string = PS_FAULTED;
3222 		break;
3223 	case P_OFFLINE:
3224 		string = PS_OFFLINE;
3225 		break;
3226 	case P_DISABLED:
3227 		string = PS_DISABLED;
3228 		break;
3229 	default:
3230 		string = "unknown";
3231 		break;
3232 	}
3233 	return (string);
3234 }
3235 
3236 /*
3237  * Export this CPU's statistics (cpu_stat_t and cpu_stats_t) as raw and named
3238  * kstats, respectively.  This is done when a CPU is initialized or placed
3239  * online via p_online(2).
3240  */
3241 static void
cpu_stats_kstat_create(cpu_t * cp)3242 cpu_stats_kstat_create(cpu_t *cp)
3243 {
3244 	int	instance = cp->cpu_id;
3245 	char	*module = "cpu";
3246 	char	*class = "misc";
3247 	kstat_t	*ksp;
3248 	zoneid_t zoneid;
3249 
3250 	ASSERT(MUTEX_HELD(&cpu_lock));
3251 
3252 	if (pool_pset_enabled())
3253 		zoneid = GLOBAL_ZONEID;
3254 	else
3255 		zoneid = ALL_ZONES;
3256 	/*
3257 	 * Create named kstats
3258 	 */
3259 #define	CPU_STATS_KS_CREATE(name, tsize, update_func)                    \
3260 	ksp = kstat_create_zone(module, instance, (name), class,         \
3261 	    KSTAT_TYPE_NAMED, (tsize) / sizeof (kstat_named_t), 0,       \
3262 	    zoneid);                                                     \
3263 	if (ksp != NULL) {                                               \
3264 		ksp->ks_private = cp;                                    \
3265 		ksp->ks_update = (update_func);                          \
3266 		kstat_install(ksp);                                      \
3267 	} else                                                           \
3268 		cmn_err(CE_WARN, "cpu: unable to create %s:%d:%s kstat", \
3269 		    module, instance, (name));
3270 
3271 	CPU_STATS_KS_CREATE("sys", sizeof (cpu_sys_stats_ks_data_template),
3272 	    cpu_sys_stats_ks_update);
3273 	CPU_STATS_KS_CREATE("vm", sizeof (cpu_vm_stats_ks_data_template),
3274 	    cpu_vm_stats_ks_update);
3275 
3276 	/*
3277 	 * Export the familiar cpu_stat_t KSTAT_TYPE_RAW kstat.
3278 	 */
3279 	ksp = kstat_create_zone("cpu_stat", cp->cpu_id, NULL,
3280 	    "misc", KSTAT_TYPE_RAW, sizeof (cpu_stat_t), 0, zoneid);
3281 	if (ksp != NULL) {
3282 		ksp->ks_update = cpu_stat_ks_update;
3283 		ksp->ks_private = cp;
3284 		kstat_install(ksp);
3285 	}
3286 }
3287 
3288 static void
cpu_stats_kstat_destroy(cpu_t * cp)3289 cpu_stats_kstat_destroy(cpu_t *cp)
3290 {
3291 	char ks_name[KSTAT_STRLEN];
3292 
3293 	(void) sprintf(ks_name, "cpu_stat%d", cp->cpu_id);
3294 	kstat_delete_byname("cpu_stat", cp->cpu_id, ks_name);
3295 
3296 	kstat_delete_byname("cpu", cp->cpu_id, "sys");
3297 	kstat_delete_byname("cpu", cp->cpu_id, "vm");
3298 }
3299 
3300 static int
cpu_sys_stats_ks_update(kstat_t * ksp,int rw)3301 cpu_sys_stats_ks_update(kstat_t *ksp, int rw)
3302 {
3303 	cpu_t *cp = (cpu_t *)ksp->ks_private;
3304 	struct cpu_sys_stats_ks_data *csskd;
3305 	cpu_sys_stats_t *css;
3306 	hrtime_t msnsecs[NCMSTATES];
3307 	int	i;
3308 
3309 	if (rw == KSTAT_WRITE)
3310 		return (EACCES);
3311 
3312 	csskd = ksp->ks_data;
3313 	css = &cp->cpu_stats.sys;
3314 
3315 	/*
3316 	 * Read CPU mstate, but compare with the last values we
3317 	 * received to make sure that the returned kstats never
3318 	 * decrease.
3319 	 */
3320 
3321 	get_cpu_mstate(cp, msnsecs);
3322 	if (csskd->cpu_nsec_idle.value.ui64 > msnsecs[CMS_IDLE])
3323 		msnsecs[CMS_IDLE] = csskd->cpu_nsec_idle.value.ui64;
3324 	if (csskd->cpu_nsec_user.value.ui64 > msnsecs[CMS_USER])
3325 		msnsecs[CMS_USER] = csskd->cpu_nsec_user.value.ui64;
3326 	if (csskd->cpu_nsec_kernel.value.ui64 > msnsecs[CMS_SYSTEM])
3327 		msnsecs[CMS_SYSTEM] = csskd->cpu_nsec_kernel.value.ui64;
3328 
3329 	bcopy(&cpu_sys_stats_ks_data_template, ksp->ks_data,
3330 	    sizeof (cpu_sys_stats_ks_data_template));
3331 
3332 	csskd->cpu_ticks_wait.value.ui64 = 0;
3333 	csskd->wait_ticks_io.value.ui64 = 0;
3334 
3335 	csskd->cpu_nsec_idle.value.ui64 = msnsecs[CMS_IDLE];
3336 	csskd->cpu_nsec_user.value.ui64 = msnsecs[CMS_USER];
3337 	csskd->cpu_nsec_kernel.value.ui64 = msnsecs[CMS_SYSTEM];
3338 	csskd->cpu_ticks_idle.value.ui64 =
3339 	    NSEC_TO_TICK(csskd->cpu_nsec_idle.value.ui64);
3340 	csskd->cpu_ticks_user.value.ui64 =
3341 	    NSEC_TO_TICK(csskd->cpu_nsec_user.value.ui64);
3342 	csskd->cpu_ticks_kernel.value.ui64 =
3343 	    NSEC_TO_TICK(csskd->cpu_nsec_kernel.value.ui64);
3344 	csskd->cpu_nsec_dtrace.value.ui64 = cp->cpu_dtrace_nsec;
3345 	csskd->dtrace_probes.value.ui64 = cp->cpu_dtrace_probes;
3346 	csskd->cpu_nsec_intr.value.ui64 = cp->cpu_intrlast;
3347 	csskd->cpu_load_intr.value.ui64 = cp->cpu_intrload;
3348 	csskd->bread.value.ui64 = css->bread;
3349 	csskd->bwrite.value.ui64 = css->bwrite;
3350 	csskd->lread.value.ui64 = css->lread;
3351 	csskd->lwrite.value.ui64 = css->lwrite;
3352 	csskd->phread.value.ui64 = css->phread;
3353 	csskd->phwrite.value.ui64 = css->phwrite;
3354 	csskd->pswitch.value.ui64 = css->pswitch;
3355 	csskd->trap.value.ui64 = css->trap;
3356 	csskd->intr.value.ui64 = 0;
3357 	for (i = 0; i < PIL_MAX; i++)
3358 		csskd->intr.value.ui64 += css->intr[i];
3359 	csskd->syscall.value.ui64 = css->syscall;
3360 	csskd->sysread.value.ui64 = css->sysread;
3361 	csskd->syswrite.value.ui64 = css->syswrite;
3362 	csskd->sysfork.value.ui64 = css->sysfork;
3363 	csskd->sysvfork.value.ui64 = css->sysvfork;
3364 	csskd->sysexec.value.ui64 = css->sysexec;
3365 	csskd->readch.value.ui64 = css->readch;
3366 	csskd->writech.value.ui64 = css->writech;
3367 	csskd->rcvint.value.ui64 = css->rcvint;
3368 	csskd->xmtint.value.ui64 = css->xmtint;
3369 	csskd->mdmint.value.ui64 = css->mdmint;
3370 	csskd->rawch.value.ui64 = css->rawch;
3371 	csskd->canch.value.ui64 = css->canch;
3372 	csskd->outch.value.ui64 = css->outch;
3373 	csskd->msg.value.ui64 = css->msg;
3374 	csskd->sema.value.ui64 = css->sema;
3375 	csskd->namei.value.ui64 = css->namei;
3376 	csskd->ufsiget.value.ui64 = css->ufsiget;
3377 	csskd->ufsdirblk.value.ui64 = css->ufsdirblk;
3378 	csskd->ufsipage.value.ui64 = css->ufsipage;
3379 	csskd->ufsinopage.value.ui64 = css->ufsinopage;
3380 	csskd->procovf.value.ui64 = css->procovf;
3381 	csskd->intrthread.value.ui64 = 0;
3382 	for (i = 0; i < LOCK_LEVEL - 1; i++)
3383 		csskd->intrthread.value.ui64 += css->intr[i];
3384 	csskd->intrblk.value.ui64 = css->intrblk;
3385 	csskd->intrunpin.value.ui64 = css->intrunpin;
3386 	csskd->idlethread.value.ui64 = css->idlethread;
3387 	csskd->inv_swtch.value.ui64 = css->inv_swtch;
3388 	csskd->nthreads.value.ui64 = css->nthreads;
3389 	csskd->cpumigrate.value.ui64 = css->cpumigrate;
3390 	csskd->xcalls.value.ui64 = css->xcalls;
3391 	csskd->mutex_adenters.value.ui64 = css->mutex_adenters;
3392 	csskd->rw_rdfails.value.ui64 = css->rw_rdfails;
3393 	csskd->rw_wrfails.value.ui64 = css->rw_wrfails;
3394 	csskd->modload.value.ui64 = css->modload;
3395 	csskd->modunload.value.ui64 = css->modunload;
3396 	csskd->bawrite.value.ui64 = css->bawrite;
3397 	csskd->iowait.value.ui64 = css->iowait;
3398 
3399 	return (0);
3400 }
3401 
3402 static int
cpu_vm_stats_ks_update(kstat_t * ksp,int rw)3403 cpu_vm_stats_ks_update(kstat_t *ksp, int rw)
3404 {
3405 	cpu_t *cp = (cpu_t *)ksp->ks_private;
3406 	struct cpu_vm_stats_ks_data *cvskd;
3407 	cpu_vm_stats_t *cvs;
3408 
3409 	if (rw == KSTAT_WRITE)
3410 		return (EACCES);
3411 
3412 	cvs = &cp->cpu_stats.vm;
3413 	cvskd = ksp->ks_data;
3414 
3415 	bcopy(&cpu_vm_stats_ks_data_template, ksp->ks_data,
3416 	    sizeof (cpu_vm_stats_ks_data_template));
3417 	cvskd->pgrec.value.ui64 = cvs->pgrec;
3418 	cvskd->pgfrec.value.ui64 = cvs->pgfrec;
3419 	cvskd->pgin.value.ui64 = cvs->pgin;
3420 	cvskd->pgpgin.value.ui64 = cvs->pgpgin;
3421 	cvskd->pgout.value.ui64 = cvs->pgout;
3422 	cvskd->pgpgout.value.ui64 = cvs->pgpgout;
3423 	cvskd->swapin.value.ui64 = cvs->swapin;
3424 	cvskd->pgswapin.value.ui64 = cvs->pgswapin;
3425 	cvskd->swapout.value.ui64 = cvs->swapout;
3426 	cvskd->pgswapout.value.ui64 = cvs->pgswapout;
3427 	cvskd->zfod.value.ui64 = cvs->zfod;
3428 	cvskd->dfree.value.ui64 = cvs->dfree;
3429 	cvskd->scan.value.ui64 = cvs->scan;
3430 	cvskd->rev.value.ui64 = cvs->rev;
3431 	cvskd->hat_fault.value.ui64 = cvs->hat_fault;
3432 	cvskd->as_fault.value.ui64 = cvs->as_fault;
3433 	cvskd->maj_fault.value.ui64 = cvs->maj_fault;
3434 	cvskd->cow_fault.value.ui64 = cvs->cow_fault;
3435 	cvskd->prot_fault.value.ui64 = cvs->prot_fault;
3436 	cvskd->softlock.value.ui64 = cvs->softlock;
3437 	cvskd->kernel_asflt.value.ui64 = cvs->kernel_asflt;
3438 	cvskd->pgrrun.value.ui64 = cvs->pgrrun;
3439 	cvskd->execpgin.value.ui64 = cvs->execpgin;
3440 	cvskd->execpgout.value.ui64 = cvs->execpgout;
3441 	cvskd->execfree.value.ui64 = cvs->execfree;
3442 	cvskd->anonpgin.value.ui64 = cvs->anonpgin;
3443 	cvskd->anonpgout.value.ui64 = cvs->anonpgout;
3444 	cvskd->anonfree.value.ui64 = cvs->anonfree;
3445 	cvskd->fspgin.value.ui64 = cvs->fspgin;
3446 	cvskd->fspgout.value.ui64 = cvs->fspgout;
3447 	cvskd->fsfree.value.ui64 = cvs->fsfree;
3448 
3449 	return (0);
3450 }
3451 
3452 static int
cpu_stat_ks_update(kstat_t * ksp,int rw)3453 cpu_stat_ks_update(kstat_t *ksp, int rw)
3454 {
3455 	cpu_stat_t *cso;
3456 	cpu_t *cp;
3457 	int i;
3458 	hrtime_t msnsecs[NCMSTATES];
3459 
3460 	cso = (cpu_stat_t *)ksp->ks_data;
3461 	cp = (cpu_t *)ksp->ks_private;
3462 
3463 	if (rw == KSTAT_WRITE)
3464 		return (EACCES);
3465 
3466 	/*
3467 	 * Read CPU mstate, but compare with the last values we
3468 	 * received to make sure that the returned kstats never
3469 	 * decrease.
3470 	 */
3471 
3472 	get_cpu_mstate(cp, msnsecs);
3473 	msnsecs[CMS_IDLE] = NSEC_TO_TICK(msnsecs[CMS_IDLE]);
3474 	msnsecs[CMS_USER] = NSEC_TO_TICK(msnsecs[CMS_USER]);
3475 	msnsecs[CMS_SYSTEM] = NSEC_TO_TICK(msnsecs[CMS_SYSTEM]);
3476 	if (cso->cpu_sysinfo.cpu[CPU_IDLE] < msnsecs[CMS_IDLE])
3477 		cso->cpu_sysinfo.cpu[CPU_IDLE] = msnsecs[CMS_IDLE];
3478 	if (cso->cpu_sysinfo.cpu[CPU_USER] < msnsecs[CMS_USER])
3479 		cso->cpu_sysinfo.cpu[CPU_USER] = msnsecs[CMS_USER];
3480 	if (cso->cpu_sysinfo.cpu[CPU_KERNEL] < msnsecs[CMS_SYSTEM])
3481 		cso->cpu_sysinfo.cpu[CPU_KERNEL] = msnsecs[CMS_SYSTEM];
3482 	cso->cpu_sysinfo.cpu[CPU_WAIT]	= 0;
3483 	cso->cpu_sysinfo.wait[W_IO]	= 0;
3484 	cso->cpu_sysinfo.wait[W_SWAP]	= 0;
3485 	cso->cpu_sysinfo.wait[W_PIO]	= 0;
3486 	cso->cpu_sysinfo.bread		= CPU_STATS(cp, sys.bread);
3487 	cso->cpu_sysinfo.bwrite		= CPU_STATS(cp, sys.bwrite);
3488 	cso->cpu_sysinfo.lread		= CPU_STATS(cp, sys.lread);
3489 	cso->cpu_sysinfo.lwrite		= CPU_STATS(cp, sys.lwrite);
3490 	cso->cpu_sysinfo.phread		= CPU_STATS(cp, sys.phread);
3491 	cso->cpu_sysinfo.phwrite	= CPU_STATS(cp, sys.phwrite);
3492 	cso->cpu_sysinfo.pswitch	= CPU_STATS(cp, sys.pswitch);
3493 	cso->cpu_sysinfo.trap		= CPU_STATS(cp, sys.trap);
3494 	cso->cpu_sysinfo.intr		= 0;
3495 	for (i = 0; i < PIL_MAX; i++)
3496 		cso->cpu_sysinfo.intr += CPU_STATS(cp, sys.intr[i]);
3497 	cso->cpu_sysinfo.syscall	= CPU_STATS(cp, sys.syscall);
3498 	cso->cpu_sysinfo.sysread	= CPU_STATS(cp, sys.sysread);
3499 	cso->cpu_sysinfo.syswrite	= CPU_STATS(cp, sys.syswrite);
3500 	cso->cpu_sysinfo.sysfork	= CPU_STATS(cp, sys.sysfork);
3501 	cso->cpu_sysinfo.sysvfork	= CPU_STATS(cp, sys.sysvfork);
3502 	cso->cpu_sysinfo.sysexec	= CPU_STATS(cp, sys.sysexec);
3503 	cso->cpu_sysinfo.readch		= CPU_STATS(cp, sys.readch);
3504 	cso->cpu_sysinfo.writech	= CPU_STATS(cp, sys.writech);
3505 	cso->cpu_sysinfo.rcvint		= CPU_STATS(cp, sys.rcvint);