1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25/*
26 * Process switching routines.
27 */
28
29#include "assym.h"
30
31#include <sys/param.h>
32#include <sys/asm_linkage.h>
33#include <sys/mmu.h>
34#include <sys/pcb.h>
35#include <sys/machthread.h>
36#include <sys/machclock.h>
37#include <sys/privregs.h>
38#include <sys/vtrace.h>
39#include <vm/hat_sfmmu.h>
40
41/*
42 * resume(kthread_id_t)
43 *
44 * a thread can only run on one processor at a time. there
45 * exists a window on MPs where the current thread on one
46 * processor is capable of being dispatched by another processor.
47 * some overlap between outgoing and incoming threads can happen
48 * when they are the same thread. in this case where the threads
49 * are the same, resume() on one processor will spin on the incoming
50 * thread until resume() on the other processor has finished with
51 * the outgoing thread.
52 *
53 * The MMU context changes when the resuming thread resides in a different
54 * process.  Kernel threads are known by resume to reside in process 0.
55 * The MMU context, therefore, only changes when resuming a thread in
56 * a process different from curproc.
57 *
58 * resume_from_intr() is called when the thread being resumed was not
59 * passivated by resume (e.g. was interrupted).  This means that the
60 * resume lock is already held and that a restore context is not needed.
61 * Also, the MMU context is not changed on the resume in this case.
62 *
63 * resume_from_zombie() is the same as resume except the calling thread
64 * is a zombie and must be put on the deathrow list after the CPU is
65 * off the stack.
66 */
67
68	ENTRY(resume)
69	save	%sp, -SA(MINFRAME), %sp		! save ins and locals
70
71	call	__dtrace_probe___sched_off__cpu	! DTrace probe
72	mov	%i0, %o0			! arg for DTrace probe
73
74	membar	#Sync				! flush writebuffers
75	flushw					! flushes all but this window
76
77	stn	%i7, [THREAD_REG + T_PC]	! save return address
78	stn	%fp, [THREAD_REG + T_SP]	! save sp
79
80	!
81	! Save GSR (Graphics Status Register).
82	!
83	! Read fprs, call fp_save if FPRS_FEF set.
84	! This handles floating-point state saving.
85	! The fprs could be turned on by hw bcopy software,
86	! *or* by fp_disabled. Handle it either way.
87	!
88	ldn	[THREAD_REG + T_LWP], %o4	! get lwp pointer
89	rd	%fprs, %g4			! read fprs
90	brnz,pt	%o4, 0f				! if user thread skip
91	  ldn	[THREAD_REG + T_CPU], %i1	! get CPU pointer
92
93	!
94	! kernel thread
95	!
96	! we save fprs at the beginning the stack so we know
97	! where to check at resume time
98	ldn	[THREAD_REG + T_STACK], %i2
99	ldn	[THREAD_REG + T_CTX], %g3	! get ctx pointer
100	andcc	%g4, FPRS_FEF, %g0		! is FPRS_FEF set
101	bz,pt	%icc, 1f			! nope, skip
102	  st	%g4, [%i2 + SA(MINFRAME) + FPU_FPRS]	! save fprs
103
104	! save kernel fp state in stack
105	add	%i2, SA(MINFRAME), %o0		! o0 = kfpu_t ptr
106	rd	%gsr, %g5
107	call	fp_save
108	stx	%g5, [%o0 + FPU_GSR]		! store GSR
109	ba,a,pt	%icc, 1f
110	  nop
111
1120:
113	! user thread
114	! o4 = lwp ptr
115	! g4 = fprs
116	! i1 = CPU ptr
117	ldn	[%o4 + LWP_FPU], %o0		! fp pointer
118	stn	%fp, [THREAD_REG + T_SP]	! save sp
119	andcc	%g4, FPRS_FEF, %g0		! is FPRS_FEF set
120	st	%g4, [%o0 + FPU_FPRS]		! store FPRS
121#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
122	sethi	%hi(fpu_exists), %g5
123	ld	[%g5 + %lo(fpu_exists)], %g5
124	brz,pn	%g5, 1f
125	  ldn	[THREAD_REG + T_CTX], %g3	! get ctx pointer
126#endif
127	bz,pt	%icc, 1f			! most apps don't use fp
128	  ldn	[THREAD_REG + T_CTX], %g3	! get ctx pointer
129	ldn	[%o4 + LWP_FPU], %o0		! fp pointer
130	rd	%gsr, %g5
131	call	fp_save				! doesn't touch globals
132	stx	%g5, [%o0 + FPU_GSR]		! store GSR
1331:
134	!
135	! Perform context switch callback if set.
136	! This handles coprocessor state saving.
137	! i1 = cpu ptr
138	! g3 = ctx pointer
139	!
140	wr	%g0, %g0, %fprs			! disable fpu and clear fprs
141	brz,pt	%g3, 2f				! skip call when zero
142	ldn	[%i0 + T_PROCP], %i3		! delay slot - get proc pointer
143	call	savectx
144	mov	THREAD_REG, %o0			! delay - arg = thread pointer
1452:
146	ldn	[THREAD_REG + T_PROCP], %i2	! load old curproc - for mmu
147
148	!
149	! Temporarily switch to idle thread's stack
150	!
151	ldn	[%i1 + CPU_IDLE_THREAD], %o0	! idle thread pointer
152	ldn	[%o0 + T_SP], %o1		! get onto idle thread stack
153	sub	%o1, SA(MINFRAME), %sp		! save room for ins and locals
154	clr	%fp
155
156	!
157	! Set the idle thread as the current thread
158	!
159	mov	THREAD_REG, %l3			! save %g7 (current thread)
160	mov	%o0, THREAD_REG			! set %g7 to idle
161	stn	%o0, [%i1 + CPU_THREAD]		! set CPU's thread to idle
162
163	!
164	! Clear and unlock previous thread's t_lock
165	! to allow it to be dispatched by another processor.
166	!
167	clrb	[%l3 + T_LOCK]			! clear tp->t_lock
168
169	!
170	! IMPORTANT: Registers at this point must be:
171	!	%i0 = new thread
172	!	%i1 = cpu pointer
173	!	%i2 = old proc pointer
174	!	%i3 = new proc pointer
175	!
176	! Here we are in the idle thread, have dropped the old thread.
177	!
178	ALTENTRY(_resume_from_idle)
179
180	! SET_KCONTEXTREG(reg0, reg1, reg2, reg3, reg4, label1, label2, label3)
181	SET_KCONTEXTREG(%o0, %g1, %g2, %g3, %o3, l1, l2, l3)
182
183	cmp 	%i2, %i3		! resuming the same process?
184	be,pt	%xcc, 5f		! yes.
185	  nop
186
187	ldx	[%i3 + P_AS], %o0	! load p->p_as
188	ldx	[%o0 + A_HAT], %i5	! %i5 = new proc hat
189
190	!
191	! update cpusran field
192	!
193	ld	[%i1 + CPU_ID], %o4
194	add	%i5, SFMMU_CPUSRAN, %o5
195	CPU_INDEXTOSET(%o5, %o4, %g1)
196	ldx	[%o5], %o2		! %o2 = cpusran field
197	mov	1, %g2
198	sllx	%g2, %o4, %o4		! %o4 = bit for this cpu
199	andcc	%o4, %o2, %g0
200	bnz,pn	%xcc, 0f		! bit already set, go to 0
201	  nop
2023:
203	or	%o2, %o4, %o1		! or in this cpu's bit mask
204	casx	[%o5], %o2, %o1
205	cmp	%o2, %o1
206	bne,a,pn %xcc, 3b
207	  ldx	[%o5], %o2		! o2 = cpusran field
208	membar	#LoadLoad|#StoreLoad
209
2100:
211	!
212	! disable interrupts
213	!
214	! if resume from user to kernel thread
215	!	call sfmmu_setctx_sec
216	! if resume from kernel (or a different user) thread to user thread
217	!	call sfmmu_alloc_ctx
218	! sfmmu_load_mmustate
219	!
220	! enable interrupts
221	!
222	! %i5 = new proc hat
223	!
224
225	sethi	%hi(ksfmmup), %o2
226        ldx	[%o2 + %lo(ksfmmup)], %o2
227
228	rdpr	%pstate, %i4
229        cmp	%i5, %o2		! new proc hat == ksfmmup ?
230	bne,pt	%xcc, 3f		! new proc is not kernel as, go to 3
231	  wrpr	%i4, PSTATE_IE, %pstate
232
233	SET_KAS_CTXSEC_ARGS(%i5, %o0, %o1)
234
235	! new proc is kernel as
236
237	call	sfmmu_setctx_sec		! switch to kernel context
238	  or	%o0, %o1, %o0
239
240	ba,a,pt	%icc, 4f
241
242	!
243	! Switch to user address space.
244	!
2453:
246	mov	%i5, %o0			! %o0 = sfmmup
247	mov	%i1, %o2			! %o2 = CPU
248	set	SFMMU_PRIVATE, %o3		! %o3 = sfmmu private flag
249	call	sfmmu_alloc_ctx
250	  mov	%g0, %o1			! %o1 = allocate flag = 0
251
252	brz,a,pt %o0, 4f			! %o0 == 0, no private alloc'ed
253          nop
254
255        ldn     [%i5 + SFMMU_SCDP], %o0         ! using shared contexts?
256        brz,a,pt %o0, 4f
257          nop
258
259	ldn   [%o0 + SCD_SFMMUP], %o0		! %o0 = scdp->scd_sfmmup
260	mov	%i1, %o2			! %o2 = CPU
261	set	SFMMU_SHARED, %o3		! %o3 = sfmmu shared flag
262	call	sfmmu_alloc_ctx
263	  mov	1, %o1				! %o1 = allocate flag = 1
264
2654:
266	call	sfmmu_load_mmustate		! program MMU registers
267	  mov	%i5, %o0
268
269	wrpr    %g0, %i4, %pstate               ! enable interrupts
270
2715:
272	!
273	! spin until dispatched thread's mutex has
274	! been unlocked. this mutex is unlocked when
275	! it becomes safe for the thread to run.
276	!
277	ldstub	[%i0 + T_LOCK], %o0	! lock curthread's t_lock
2786:
279	brnz,pn	%o0, 7f			! lock failed
280	  ldx	[%i0 + T_PC], %i7	! delay - restore resuming thread's pc
281
282	!
283	! Fix CPU structure to indicate new running thread.
284	! Set pointer in new thread to the CPU structure.
285	! XXX - Move migration statistic out of here
286	!
287        ldx	[%i0 + T_CPU], %g2	! last CPU to run the new thread
288        cmp     %g2, %i1		! test for migration
289        be,pt	%xcc, 4f		! no migration
290          ldn	[%i0 + T_LWP], %o1	! delay - get associated lwp (if any)
291        ldx	[%i1 + CPU_STATS_SYS_CPUMIGRATE], %g2
292        inc     %g2
293        stx	%g2, [%i1 + CPU_STATS_SYS_CPUMIGRATE]
294	stx	%i1, [%i0 + T_CPU]	! set new thread's CPU pointer
2954:
296	stx	%i0, [%i1 + CPU_THREAD]	! set CPU's thread pointer
297	membar	#StoreLoad		! synchronize with mutex_exit()
298	mov	%i0, THREAD_REG		! update global thread register
299	stx	%o1, [%i1 + CPU_LWP]	! set CPU's lwp ptr
300	brz,a,pn %o1, 1f		! if no lwp, branch and clr mpcb
301	  stx	%g0, [%i1 + CPU_MPCB]
302	!
303	! user thread
304	! o1 = lwp
305	! i0 = new thread
306	!
307	ldx	[%i0 + T_STACK], %o0
308	stx	%o0, [%i1 + CPU_MPCB]	! set CPU's mpcb pointer
309#ifdef CPU_MPCB_PA
310	ldx	[%o0 + MPCB_PA], %o0
311	stx	%o0, [%i1 + CPU_MPCB_PA]
312#endif
313	! Switch to new thread's stack
314	ldx	[%i0 + T_SP], %o0	! restore resuming thread's sp
315	sub	%o0, SA(MINFRAME), %sp	! in case of intr or trap before restore
316	mov	%o0, %fp
317	!
318	! Restore resuming thread's GSR reg and floating-point regs
319	! Note that the ld to the gsr register ensures that the loading of
320	! the floating point saved state has completed without necessity
321	! of a membar #Sync.
322	!
323#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
324	sethi	%hi(fpu_exists), %g3
325	ld	[%g3 + %lo(fpu_exists)], %g3
326	brz,pn	%g3, 2f
327	  ldx	[%i0 + T_CTX], %i5	! should resumed thread restorectx?
328#endif
329	ldx	[%o1 + LWP_FPU], %o0		! fp pointer
330	ld	[%o0 + FPU_FPRS], %g5		! get fpu_fprs
331	andcc	%g5, FPRS_FEF, %g0		! is FPRS_FEF set?
332	bz,a,pt	%icc, 9f			! no, skip fp_restore
333	  wr	%g0, FPRS_FEF, %fprs		! enable fprs so fp_zero works
334
335	ldx	[THREAD_REG + T_CPU], %o4	! cpu pointer
336	call	fp_restore
337	  wr	%g5, %g0, %fprs			! enable fpu and restore fprs
338
339	ldx	[%o0 + FPU_GSR], %g5		! load saved GSR data
340	wr	%g5, %g0, %gsr			! restore %gsr data
341	ba,pt	%icc,2f
342	  ldx	[%i0 + T_CTX], %i5	! should resumed thread restorectx?
343
3449:
345	!
346	! Zero resuming thread's fp registers, for *all* non-fp program
347	! Remove all possibility of using the fp regs as a "covert channel".
348	!
349	call	fp_zero
350	  wr	%g0, %g0, %gsr
351	ldx	[%i0 + T_CTX], %i5	! should resumed thread restorectx?
352	ba,pt	%icc, 2f
353	  wr	%g0, %g0, %fprs			! disable fprs
354
3551:
356#ifdef CPU_MPCB_PA
357	mov	-1, %o1
358	stx	%o1, [%i1 + CPU_MPCB_PA]
359#endif
360	!
361	! kernel thread
362	! i0 = new thread
363	!
364	! Switch to new thread's stack
365	!
366	ldx	[%i0 + T_SP], %o0	! restore resuming thread's sp
367	sub	%o0, SA(MINFRAME), %sp	! in case of intr or trap before restore
368	mov	%o0, %fp
369	!
370	! Restore resuming thread's GSR reg and floating-point regs
371	! Note that the ld to the gsr register ensures that the loading of
372	! the floating point saved state has completed without necessity
373	! of a membar #Sync.
374	!
375	ldx	[%i0 + T_STACK], %o0
376	ld	[%o0 + SA(MINFRAME) + FPU_FPRS], %g5	! load fprs
377	ldx	[%i0 + T_CTX], %i5		! should thread restorectx?
378	andcc	%g5, FPRS_FEF, %g0		! did we save fp in stack?
379	bz,a,pt	%icc, 2f
380	  wr	%g0, %g0, %fprs			! clr fprs
381
382	wr	%g5, %g0, %fprs			! enable fpu and restore fprs
383	call	fp_restore
384	add	%o0, SA(MINFRAME), %o0		! o0 = kpu_t ptr
385	ldx	[%o0 + FPU_GSR], %g5		! load saved GSR data
386	wr	%g5, %g0, %gsr			! restore %gsr data
387
3882:
389	!
390	! Restore resuming thread's context
391	! i5 = ctx ptr
392	!
393	brz,a,pt %i5, 8f		! skip restorectx() when zero
394	  ld	[%i1 + CPU_BASE_SPL], %o0
395	call	restorectx		! thread can not sleep on temp stack
396	  mov	THREAD_REG, %o0		! delay slot - arg = thread pointer
397	!
398	! Set priority as low as possible, blocking all interrupt threads
399	! that may be active.
400	!
401	ld	[%i1 + CPU_BASE_SPL], %o0
4028:
403	wrpr	%o0, 0, %pil
404	wrpr	%g0, WSTATE_KERN, %wstate
405	!
406	! If we are resuming an interrupt thread, store a starting timestamp
407	! in the thread structure.
408	!
409	lduh	[THREAD_REG + T_FLAGS], %o0
410	andcc	%o0, T_INTR_THREAD, %g0
411	bnz,pn	%xcc, 0f
412	  nop
4135:
414	call	__dtrace_probe___sched_on__cpu	! DTrace probe
415	nop
416
417	ret				! resume curthread
418	restore
4190:
420	add	THREAD_REG, T_INTR_START, %o2
4211:
422	ldx	[%o2], %o1
423	RD_CLOCK_TICK(%o0,%o3,%g5,__LINE__)
424	casx	[%o2], %o1, %o0
425	cmp	%o0, %o1
426	be,pt	%xcc, 5b
427	  nop
428	! If an interrupt occurred while we were attempting to store
429	! the timestamp, try again.
430	ba,pt	%xcc, 1b
431	  nop
432
433	!
434	! lock failed - spin with regular load to avoid cache-thrashing.
435	!
4367:
437	brnz,a,pt %o0, 7b		! spin while locked
438	  ldub	[%i0 + T_LOCK], %o0
439	ba	%xcc, 6b
440	  ldstub  [%i0 + T_LOCK], %o0	! delay - lock curthread's mutex
441	SET_SIZE(_resume_from_idle)
442	SET_SIZE(resume)
443
444	ENTRY(resume_from_zombie)
445	save	%sp, -SA(MINFRAME), %sp		! save ins and locals
446
447	call	__dtrace_probe___sched_off__cpu	! DTrace probe
448	mov	%i0, %o0			! arg for DTrace probe
449
450	ldn	[THREAD_REG + T_CPU], %i1	! cpu pointer
451
452	flushw					! flushes all but this window
453	ldn	[THREAD_REG + T_PROCP], %i2	! old procp for mmu ctx
454
455	!
456	! Temporarily switch to the idle thread's stack so that
457	! the zombie thread's stack can be reclaimed by the reaper.
458	!
459	ldn	[%i1 + CPU_IDLE_THREAD], %o2	! idle thread pointer
460	ldn	[%o2 + T_SP], %o1		! get onto idle thread stack
461	sub	%o1, SA(MINFRAME), %sp		! save room for ins and locals
462	clr	%fp
463	!
464	! Set the idle thread as the current thread.
465	! Put the zombie on death-row.
466	!
467	mov	THREAD_REG, %o0			! save %g7 = curthread for arg
468	mov	%o2, THREAD_REG			! set %g7 to idle
469	stn	%g0, [%i1 + CPU_MPCB]		! clear mpcb
470#ifdef CPU_MPCB_PA
471	mov	-1, %o1
472	stx	%o1, [%i1 + CPU_MPCB_PA]
473#endif
474	call	reapq_add			! reapq_add(old_thread);
475	stn	%o2, [%i1 + CPU_THREAD]		! delay - CPU's thread = idle
476
477	!
478	! resume_from_idle args:
479	!	%i0 = new thread
480	!	%i1 = cpu
481	!	%i2 = old proc
482	!	%i3 = new proc
483	!
484	b	_resume_from_idle		! finish job of resume
485	ldn	[%i0 + T_PROCP], %i3		! new process
486	SET_SIZE(resume_from_zombie)
487
488	ENTRY(resume_from_intr)
489	save	%sp, -SA(MINFRAME), %sp		! save ins and locals
490
491	!
492	! We read in the fprs and call fp_save if FPRS_FEF is set
493	! to save the floating-point state if fprs has been
494	! modified by operations such as hw bcopy or fp_disabled.
495	! This is to resolve an issue where an interrupting thread
496	! doesn't retain their floating-point registers when
497	! switching out of the interrupt context.
498	!
499	rd	%fprs, %g4
500	ldn	[THREAD_REG + T_STACK], %i2
501	andcc	%g4, FPRS_FEF, %g0		! is FPRS_FEF set
502	bz,pt	%icc, 4f
503	  st	%g4, [%i2 + SA(MINFRAME) + FPU_FPRS]	! save fprs
504
505	! save kernel fp state in stack
506	add	%i2, SA(MINFRAME), %o0		! %o0 = kfpu_t ptr
507	rd	%gsr, %g5
508	call fp_save
509	stx	%g5, [%o0 + FPU_GSR]		! store GSR
510
5114:
512
513	flushw					! flushes all but this window
514	stn	%fp, [THREAD_REG + T_SP]	! delay - save sp
515	stn	%i7, [THREAD_REG + T_PC]	! save return address
516
517	ldn	[%i0 + T_PC], %i7		! restore resuming thread's pc
518	ldn	[THREAD_REG + T_CPU], %i1	! cpu pointer
519
520	!
521	! Fix CPU structure to indicate new running thread.
522	! The pinned thread we're resuming already has the CPU pointer set.
523	!
524	mov	THREAD_REG, %l3		! save old thread
525	stn	%i0, [%i1 + CPU_THREAD]	! set CPU's thread pointer
526	membar	#StoreLoad		! synchronize with mutex_exit()
527	mov	%i0, THREAD_REG		! update global thread register
528
529	!
530	! Switch to new thread's stack
531	!
532	ldn	[THREAD_REG + T_SP], %o0	! restore resuming thread's sp
533	sub	%o0, SA(MINFRAME), %sp ! in case of intr or trap before restore
534	mov	%o0, %fp
535	clrb	[%l3 + T_LOCK]		! clear intr thread's tp->t_lock
536
537	!
538	! If we are resuming an interrupt thread, store a timestamp in the
539	! thread structure.
540	!
541	lduh	[THREAD_REG + T_FLAGS], %o0
542	andcc	%o0, T_INTR_THREAD, %g0
543	bnz,pn	%xcc, 0f
544	!
545	! We're resuming a non-interrupt thread.
546	! Clear CPU_INTRCNT and check if cpu_kprunrun set?
547	!
548	ldub	[%i1 + CPU_KPRUNRUN], %o5	! delay
549	brnz,pn	%o5, 3f				! call kpreempt(KPREEMPT_SYNC);
550	stub	%g0, [%i1 + CPU_INTRCNT]
5511:
552	ret				! resume curthread
553	restore
5540:
555	!
556	! We're an interrupt thread. Update t_intr_start and cpu_intrcnt
557	!
558	add	THREAD_REG, T_INTR_START, %o2
5592:
560	ldx	[%o2], %o1
561	RD_CLOCK_TICK(%o0,%o3,%l1,__LINE__)
562	casx	[%o2], %o1, %o0
563	cmp	%o0, %o1
564	bne,pn	%xcc, 2b
565	ldn	[THREAD_REG + T_INTR], %l1	! delay
566	! Reset cpu_intrcnt if we aren't pinning anyone
567	brz,a,pt %l1, 2f
568	stub	%g0, [%i1 + CPU_INTRCNT]
5692:
570	ba,pt	%xcc, 1b
571	nop
5723:
573	!
574	! We're a non-interrupt thread and cpu_kprunrun is set. call kpreempt.
575	!
576	call	kpreempt
577	mov	KPREEMPT_SYNC, %o0
578	ba,pt	%xcc, 1b
579	nop
580	SET_SIZE(resume_from_intr)
581
582
583/*
584 * thread_start()
585 *
586 * the current register window was crafted by thread_run() to contain
587 * an address of a procedure (in register %i7), and its args in registers
588 * %i0 through %i5. a stack trace of this thread will show the procedure
589 * that thread_start() invoked at the bottom of the stack. an exit routine
590 * is stored in %l0 and called when started thread returns from its called
591 * procedure.
592 */
593
594	ENTRY(thread_start)
595	mov	%i0, %o0
596	jmpl 	%i7, %o7	! call thread_run()'s start() procedure.
597	mov	%i1, %o1
598
599	call	thread_exit	! destroy thread if it returns.
600	nop
601	unimp 0
602	SET_SIZE(thread_start)
603