xref: /illumos-gate/usr/src/uts/sparc/v9/ml/lock_prim.S (revision 5d9d9091)
1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 *
25 * Copyright 2020 Joyent, Inc.
26 */
27
28#include "assym.h"
29
30#include <sys/t_lock.h>
31#include <sys/mutex.h>
32#include <sys/mutex_impl.h>
33#include <sys/rwlock_impl.h>
34#include <sys/asm_linkage.h>
35#include <sys/machlock.h>
36#include <sys/machthread.h>
37#include <sys/lockstat.h>
38
39/* #define DEBUG */
40
41#ifdef DEBUG
42#include <sys/machparam.h>
43#endif /* DEBUG */
44
45/************************************************************************
46 *		ATOMIC OPERATIONS
47 */
48
49/*
50 * uint8_t	ldstub(uint8_t *cp)
51 *
52 * Store 0xFF at the specified location, and return its previous content.
53 */
54
55	ENTRY(ldstub)
56	retl
57	ldstub	[%o0], %o0
58	SET_SIZE(ldstub)
59
60/************************************************************************
61 *		MEMORY BARRIERS -- see atomic.h for full descriptions.
62 */
63
64#ifdef SF_ERRATA_51
65	.align 32
66	ENTRY(membar_return)
67	retl
68	nop
69	SET_SIZE(membar_return)
70#define	MEMBAR_RETURN	ba,pt %icc, membar_return
71#else
72#define	MEMBAR_RETURN	retl
73#endif
74
75	ENTRY(membar_enter)
76	MEMBAR_RETURN
77	membar	#StoreLoad|#StoreStore
78	SET_SIZE(membar_enter)
79
80	ENTRY(membar_exit)
81	MEMBAR_RETURN
82	membar	#LoadStore|#StoreStore
83	SET_SIZE(membar_exit)
84
85	ENTRY(membar_producer)
86	MEMBAR_RETURN
87	membar	#StoreStore
88	SET_SIZE(membar_producer)
89
90	ENTRY(membar_consumer)
91	MEMBAR_RETURN
92	membar	#LoadLoad
93	SET_SIZE(membar_consumer)
94
95/************************************************************************
96 *		MINIMUM LOCKS
97 */
98
99/*
100 * lock_try(lp), ulock_try(lp)
101 * - returns non-zero on success.
102 * - doesn't block interrupts so don't use this to spin on a lock.
103 * - uses "0xFF is busy, anything else is free" model.
104 *
105 * ulock_try() is for a lock in the user address space.
106 */
107
108	.align	32
109	ENTRY(lock_try)
110	ldstub	[%o0], %o1		! try to set lock, get value in %o1
111	brnz,pn	%o1, 1f
112	membar	#LoadLoad
113.lock_try_lockstat_patch_point:
114	retl
115	or	%o0, 1, %o0		! ensure lo32 != 0
1161:
117	retl
118	clr	%o0
119	SET_SIZE(lock_try)
120
121	.align	32
122	ENTRY(lock_spin_try)
123	ldstub	[%o0], %o1		! try to set lock, get value in %o1
124	brnz,pn	%o1, 1f
125	membar	#LoadLoad
126	retl
127	or	%o0, 1, %o0		! ensure lo32 != 0
1281:
129	retl
130	clr	%o0
131	SET_SIZE(lock_spin_try)
132
133	.align	32
134	ENTRY(lock_set)
135	ldstub	[%o0], %o1
136	brnz,pn	%o1, 1f			! go to C for the hard case
137	membar	#LoadLoad
138.lock_set_lockstat_patch_point:
139	retl
140	nop
1411:
142	sethi	%hi(lock_set_spin), %o2	! load up for jump to C
143	jmp	%o2 + %lo(lock_set_spin)
144	nop				! delay: do nothing
145	SET_SIZE(lock_set)
146
147	ENTRY(lock_clear)
148	membar	#LoadStore|#StoreStore
149.lock_clear_lockstat_patch_point:
150	retl
151	clrb	[%o0]
152	SET_SIZE(lock_clear)
153
154	.align	32
155	ENTRY(ulock_try)
156	ldstuba	[%o0]ASI_USER, %o1	! try to set lock, get value in %o1
157	xor	%o1, 0xff, %o0		! delay - return non-zero if success
158	retl
159	  membar	#LoadLoad
160	SET_SIZE(ulock_try)
161
162	ENTRY(ulock_clear)
163	membar  #LoadStore|#StoreStore
164	retl
165	  stba	%g0, [%o0]ASI_USER	! clear lock
166	SET_SIZE(ulock_clear)
167
168
169/*
170 * lock_set_spl(lp, new_pil, *old_pil_addr)
171 *	Sets pil to new_pil, grabs lp, stores old pil in *old_pil_addr.
172 */
173
174	ENTRY(lock_set_spl)
175	rdpr	%pil, %o3			! %o3 = current pil
176	cmp	%o3, %o1			! is current pil high enough?
177	bl,a,pt %icc, 1f			! if not, write %pil in delay
178	wrpr	%g0, %o1, %pil
1791:
180	ldstub	[%o0], %o4			! try the lock
181	brnz,pn	%o4, 2f				! go to C for the miss case
182	membar	#LoadLoad
183.lock_set_spl_lockstat_patch_point:
184	retl
185	sth	%o3, [%o2]			! delay - save original pil
1862:
187	sethi	%hi(lock_set_spl_spin), %o5	! load up jmp to C
188	jmp	%o5 + %lo(lock_set_spl_spin)	! jmp to lock_set_spl_spin
189	nop					! delay: do nothing
190	SET_SIZE(lock_set_spl)
191
192/*
193 * lock_clear_splx(lp, s)
194 */
195
196	ENTRY(lock_clear_splx)
197	ldn	[THREAD_REG + T_CPU], %o2	! get CPU pointer
198	membar	#LoadStore|#StoreStore
199	ld	[%o2 + CPU_BASE_SPL], %o2
200	clrb	[%o0]				! clear lock
201	cmp	%o2, %o1			! compare new to base
202	movl	%xcc, %o1, %o2			! use new pri if base is less
203.lock_clear_splx_lockstat_patch_point:
204	retl
205	wrpr	%g0, %o2, %pil
206	SET_SIZE(lock_clear_splx)
207
208/*
209 * mutex_enter() and mutex_exit().
210 *
211 * These routines handle the simple cases of mutex_enter() (adaptive
212 * lock, not held) and mutex_exit() (adaptive lock, held, no waiters).
213 * If anything complicated is going on we punt to mutex_vector_enter().
214 *
215 * mutex_tryenter() is similar to mutex_enter() but returns zero if
216 * the lock cannot be acquired, nonzero on success.
217 *
218 * If mutex_exit() gets preempted in the window between checking waiters
219 * and clearing the lock, we can miss wakeups.  Disabling preemption
220 * in the mutex code is prohibitively expensive, so instead we detect
221 * mutex preemption by examining the trapped PC in the interrupt path.
222 * If we interrupt a thread in mutex_exit() that has not yet cleared
223 * the lock, pil_interrupt() resets its PC back to the beginning of
224 * mutex_exit() so it will check again for waiters when it resumes.
225 *
226 * The lockstat code below is activated when the lockstat driver
227 * calls lockstat_hot_patch() to hot-patch the kernel mutex code.
228 * Note that we don't need to test lockstat_event_mask here -- we won't
229 * patch this code in unless we're gathering ADAPTIVE_HOLD lockstats.
230 */
231
232	.align	32
233	ENTRY(mutex_enter)
234	mov	THREAD_REG, %o1
235	casx	[%o0], %g0, %o1			! try to acquire as adaptive
236	brnz,pn	%o1, 1f				! locked or wrong type
237	membar	#LoadLoad
238.mutex_enter_lockstat_patch_point:
239	retl
240	nop
2411:
242	sethi	%hi(mutex_vector_enter), %o2	! load up for jump to C
243	jmp	%o2 + %lo(mutex_vector_enter)
244	nop
245	SET_SIZE(mutex_enter)
246
247	ENTRY(mutex_tryenter)
248	mov	THREAD_REG, %o1
249	casx	[%o0], %g0, %o1			! try to acquire as adaptive
250	brnz,pn	%o1, 1f				! locked or wrong type continue
251	membar	#LoadLoad
252.mutex_tryenter_lockstat_patch_point:
253	retl
254	or	%o0, 1, %o0			! ensure lo32 != 0
2551:
256	sethi	%hi(mutex_vector_tryenter), %o2		! hi bits
257	jmp	%o2 + %lo(mutex_vector_tryenter)	! go to C
258	nop
259	SET_SIZE(mutex_tryenter)
260
261	ENTRY(mutex_adaptive_tryenter)
262	mov	THREAD_REG, %o1
263	casx	[%o0], %g0, %o1			! try to acquire as adaptive
264	brnz,pn	%o1, 0f				! locked or wrong type
265	membar	#LoadLoad
266	retl
267	or	%o0, 1, %o0			! ensure lo32 != 0
2680:
269	retl
270	mov	%g0, %o0
271	SET_SIZE(mutex_adaptive_tryenter)
272
273	! these need to be together and cache aligned for performance.
274	.align 64
275	.global	mutex_exit_critical_size
276	.global	mutex_exit_critical_start
277	.global mutex_owner_running_critical_size
278	.global mutex_owner_running_critical_start
279
280mutex_exit_critical_size = .mutex_exit_critical_end - mutex_exit_critical_start
281
282	.align	32
283
284	ENTRY(mutex_exit)
285mutex_exit_critical_start:		! If we are interrupted, restart here
286	ldn	[%o0], %o1		! get the owner field
287	membar	#LoadStore|#StoreStore
288	cmp	THREAD_REG, %o1		! do we own lock with no waiters?
289	be,a,pt	%ncc, 1f		! if so, drive on ...
290	stn	%g0, [%o0]		! delay: clear lock if we owned it
291.mutex_exit_critical_end:		! for pil_interrupt() hook
292	ba,a,pt	%xcc, mutex_vector_exit	! go to C for the hard cases
2931:
294.mutex_exit_lockstat_patch_point:
295	retl
296	nop
297	SET_SIZE(mutex_exit)
298
299mutex_owner_running_critical_size = .mutex_owner_running_critical_end - mutex_owner_running_critical_start
300
301	.align  32
302
303	ENTRY(mutex_owner_running)
304mutex_owner_running_critical_start:	! If interrupted restart here
305	ldn	[%o0], %o1		! get the owner field
306	and	%o1, MUTEX_THREAD, %o1	! remove the waiters bit if any
307	brz,pn	%o1, 1f			! if so, drive on ...
308	nop
309	ldn	[%o1+T_CPU], %o2	! get owner->t_cpu
310	ldn	[%o2+CPU_THREAD], %o3	! get owner->t_cpu->cpu_thread
311.mutex_owner_running_critical_end:	! for pil_interrupt() hook
312	cmp	%o1, %o3		! owner == running thread?
313	be,a,pt	%xcc, 2f		! yes, go return cpu
314	nop
3151:
316	retl
317	mov	%g0, %o0		! return 0 (owner not running)
3182:
319	retl
320	mov	%o2, %o0		! owner running, return cpu
321	SET_SIZE(mutex_owner_running)
322
323/*
324 * rw_enter() and rw_exit().
325 *
326 * These routines handle the simple cases of rw_enter (write-locking an unheld
327 * lock or read-locking a lock that's neither write-locked nor write-wanted)
328 * and rw_exit (no waiters or not the last reader).  If anything complicated
329 * is going on we punt to rw_enter_sleep() and rw_exit_wakeup(), respectively.
330 */
331
332	.align	16
333	ENTRY(rw_enter)
334	cmp	%o1, RW_WRITER			! entering as writer?
335	be,a,pn	%icc, 2f			! if so, go do it ...
336	or	THREAD_REG, RW_WRITE_LOCKED, %o5 ! delay: %o5 = owner
337	ldn	[%o0], %o4			! %o4 = old lock value
3381:
339	andcc	%o4, RW_WRITE_CLAIMED, %g0	! write-locked or write-wanted?
340	bz,pt	%xcc, 3f			! if so, prepare to block
341	add	%o4, RW_READ_LOCK, %o5		! delay: increment hold count
342	sethi	%hi(rw_enter_sleep), %o2	! load up jump
343	jmp	%o2 + %lo(rw_enter_sleep)	! jmp to rw_enter_sleep
344	nop					! delay: do nothing
3453:
346	casx	[%o0], %o4, %o5			! try to grab read lock
347	cmp	%o4, %o5			! did we get it?
348#ifdef sun4v
349	be,a,pt %xcc, 0f
350	membar  #LoadLoad
351	sethi	%hi(rw_enter_sleep), %o2	! load up jump
352	jmp	%o2 + %lo(rw_enter_sleep)	! jmp to rw_enter_sleep
353	nop					! delay: do nothing
3540:
355#else /* sun4v */
356	bne,pn	%xcc, 1b			! if not, try again
357	mov	%o5, %o4			! delay: %o4 = old lock value
358	membar	#LoadLoad
359#endif /* sun4v */
360.rw_read_enter_lockstat_patch_point:
361	retl
362	nop
3632:
364	casx	[%o0], %g0, %o5			! try to grab write lock
365	brz,pt %o5, 4f				! branch around if we got it
366	membar	#LoadLoad			! done regardless of where we go
367	sethi	%hi(rw_enter_sleep), %o2
368	jmp	%o2 + %lo(rw_enter_sleep)	! jump to rw_enter_sleep if not
369	nop					! delay: do nothing
3704:
371.rw_write_enter_lockstat_patch_point:
372	retl
373	nop
374	SET_SIZE(rw_enter)
375
376	.align	16
377	ENTRY(rw_exit)
378	ldn	[%o0], %o4			! %o4 = old lock value
379	membar	#LoadStore|#StoreStore		! membar_exit()
380	subcc	%o4, RW_READ_LOCK, %o5		! %o5 = new lock value if reader
381	bnz,pn	%xcc, 2f			! single reader, no waiters?
382	clr	%o1
3831:
384	srl	%o4, RW_HOLD_COUNT_SHIFT, %o3	! %o3 = hold count (lockstat)
385	casx	[%o0], %o4, %o5			! try to drop lock
386	cmp	%o4, %o5			! did we succeed?
387	bne,pn	%xcc, rw_exit_wakeup		! if not, go to C
388	nop					! delay: do nothing
389.rw_read_exit_lockstat_patch_point:
390	retl
391	nop					! delay: do nothing
3922:
393	andcc	%o4, RW_WRITE_LOCKED, %g0	! are we a writer?
394	bnz,a,pt %xcc, 3f
395	or	THREAD_REG, RW_WRITE_LOCKED, %o4 ! delay: %o4 = owner
396	cmp	%o5, RW_READ_LOCK		! would lock still be held?
397	bge,pt	%xcc, 1b			! if so, go ahead and drop it
398	nop
399	ba,pt	%xcc, rw_exit_wakeup		! otherwise, wake waiters
400	nop
4013:
402	casx	[%o0], %o4, %o1			! try to drop write lock
403	cmp	%o4, %o1			! did we succeed?
404	bne,pn	%xcc, rw_exit_wakeup		! if not, go to C
405	nop
406.rw_write_exit_lockstat_patch_point:
407	retl
408	nop
409	SET_SIZE(rw_exit)
410
411#define	RETL			0x81c3e008
412#define	NOP			0x01000000
413#define BA			0x10800000
414
415#define	DISP22			((1 << 22) - 1)
416#define	ANNUL			0x20000000
417
418#define	HOT_PATCH_COMMON(addr, event, normal_instr, annul, rs)		\
419	ba	1f;							\
420	rd	%pc, %o0;						\
421	save	%sp, -SA(MINFRAME), %sp;				\
422	set	lockstat_probemap, %l1;					\
423	ld	[%l1 + (event * DTRACE_IDSIZE)], %o0;			\
424	brz,pn	%o0, 0f;						\
425	ldub	[THREAD_REG + T_LOCKSTAT], %l0;				\
426	add	%l0, 1, %l2;						\
427	stub	%l2, [THREAD_REG + T_LOCKSTAT];				\
428	set	lockstat_probe, %g1;					\
429	ld	[%l1 + (event * DTRACE_IDSIZE)], %o0;			\
430	brz,a,pn %o0, 0f;						\
431	stub	%l0, [THREAD_REG + T_LOCKSTAT];				\
432	ldn	[%g1], %g2;						\
433	mov	rs, %o2;						\
434	jmpl	%g2, %o7;						\
435	mov	%i0, %o1;						\
436	stub	%l0, [THREAD_REG + T_LOCKSTAT];				\
4370:	ret;								\
438	restore	%g0, 1, %o0;	/* for mutex_tryenter / lock_try */	\
4391:	set	addr, %o1;						\
440	sub	%o0, %o1, %o0;						\
441	srl	%o0, 2, %o0;						\
442	inc	%o0;							\
443	set	DISP22, %o1;						\
444	and	%o1, %o0, %o0;						\
445	set	BA, %o1;						\
446	or	%o1, %o0, %o0;						\
447	sethi	%hi(annul), %o2;					\
448	add	%o0, %o2, %o2;						\
449	set	addr, %o0;						\
450	set	normal_instr, %o1;					\
451	ld	[%i0 + (event * DTRACE_IDSIZE)], %o3;			\
452	tst	%o3;							\
453	movnz	%icc, %o2, %o1;						\
454	call	hot_patch_kernel_text;					\
455	mov	4, %o2;							\
456	membar	#Sync
457
458#define	HOT_PATCH(addr, event, normal_instr)	\
459	HOT_PATCH_COMMON(addr, event, normal_instr, 0, %i1)
460
461#define	HOT_PATCH_ARG(addr, event, normal_instr, arg)	\
462	HOT_PATCH_COMMON(addr, event, normal_instr, 0, arg)
463
464#define HOT_PATCH_ANNULLED(addr, event, normal_instr)	\
465	HOT_PATCH_COMMON(addr, event, normal_instr, ANNUL, %i1)
466
467	ENTRY(lockstat_hot_patch)
468	save	%sp, -SA(MINFRAME), %sp
469	set	lockstat_probemap, %i0
470	HOT_PATCH(.mutex_enter_lockstat_patch_point,
471		LS_MUTEX_ENTER_ACQUIRE, RETL)
472	HOT_PATCH_ANNULLED(.mutex_tryenter_lockstat_patch_point,
473		LS_MUTEX_TRYENTER_ACQUIRE, RETL)
474	HOT_PATCH(.mutex_exit_lockstat_patch_point,
475		LS_MUTEX_EXIT_RELEASE, RETL)
476	HOT_PATCH(.rw_write_enter_lockstat_patch_point,
477		LS_RW_ENTER_ACQUIRE, RETL)
478	HOT_PATCH(.rw_read_enter_lockstat_patch_point,
479		LS_RW_ENTER_ACQUIRE, RETL)
480	HOT_PATCH_ARG(.rw_write_exit_lockstat_patch_point,
481		LS_RW_EXIT_RELEASE, RETL, RW_WRITER)
482	HOT_PATCH_ARG(.rw_read_exit_lockstat_patch_point,
483		LS_RW_EXIT_RELEASE, RETL, RW_READER)
484	HOT_PATCH(.lock_set_lockstat_patch_point,
485		LS_LOCK_SET_ACQUIRE, RETL)
486	HOT_PATCH_ANNULLED(.lock_try_lockstat_patch_point,
487		LS_LOCK_TRY_ACQUIRE, RETL)
488	HOT_PATCH(.lock_clear_lockstat_patch_point,
489		LS_LOCK_CLEAR_RELEASE, RETL)
490	HOT_PATCH(.lock_set_spl_lockstat_patch_point,
491		LS_LOCK_SET_SPL_ACQUIRE, RETL)
492	HOT_PATCH(.lock_clear_splx_lockstat_patch_point,
493		LS_LOCK_CLEAR_SPLX_RELEASE, RETL)
494	ret
495	restore
496	SET_SIZE(lockstat_hot_patch)
497
498/*
499 * asm_mutex_spin_enter(mutex_t *)
500 *
501 * For use by assembly interrupt handler only.
502 * Does not change spl, since the interrupt handler is assumed to be
503 * running at high level already.
504 * Traps may be off, so cannot panic.
505 * Does not keep statistics on the lock.
506 *
507 * Entry:	%l6 - points to mutex
508 *		%l7 - address of call (returns to %l7+8)
509 * Uses:	%l6, %l5
510 */
511	.align 16
512	ENTRY_NP(asm_mutex_spin_enter)
513	ldstub	[%l6 + M_SPINLOCK], %l5	! try to set lock, get value in %l5
5141:
515	tst	%l5
516	bnz	3f			! lock already held - go spin
517	nop
5182:
519	jmp	%l7 + 8			! return
520	membar	#LoadLoad
521	!
522	! Spin on lock without using an atomic operation to prevent the caches
523	! from unnecessarily moving ownership of the line around.
524	!
5253:
526	ldub	[%l6 + M_SPINLOCK], %l5
5274:
528	tst	%l5
529	bz,a	1b			! lock appears to be free, try again
530	ldstub	[%l6 + M_SPINLOCK], %l5	! delay slot - try to set lock
531
532	sethi	%hi(panicstr) , %l5
533	ldn	[%l5 + %lo(panicstr)], %l5
534	tst	%l5
535	bnz	2b			! after panic, feign success
536	nop
537	b	4b
538	ldub	[%l6 + M_SPINLOCK], %l5	! delay - reload lock
539	SET_SIZE(asm_mutex_spin_enter)
540
541/*
542 * asm_mutex_spin_exit(mutex_t *)
543 *
544 * For use by assembly interrupt handler only.
545 * Does not change spl, since the interrupt handler is assumed to be
546 * running at high level already.
547 *
548 * Entry:	%l6 - points to mutex
549 *		%l7 - address of call (returns to %l7+8)
550 * Uses:	none
551 */
552	ENTRY_NP(asm_mutex_spin_exit)
553	membar	#LoadStore|#StoreStore
554	jmp	%l7 + 8			! return
555	clrb	[%l6 + M_SPINLOCK]	! delay - clear lock
556	SET_SIZE(asm_mutex_spin_exit)
557
558/*
559 * thread_onproc()
560 * Set thread in onproc state for the specified CPU.
561 * Also set the thread lock pointer to the CPU's onproc lock.
562 * Since the new lock isn't held, the store ordering is important.
563 * If not done in assembler, the compiler could reorder the stores.
564 */
565
566	ENTRY(thread_onproc)
567	set	TS_ONPROC, %o2		! TS_ONPROC state
568	st	%o2, [%o0 + T_STATE]	! store state
569	add	%o1, CPU_THREAD_LOCK, %o3 ! pointer to disp_lock while running
570	retl				! return
571	stn	%o3, [%o0 + T_LOCKP]	! delay - store new lock pointer
572	SET_SIZE(thread_onproc)
573
574/* delay function used in some mutex code - just do 3 nop cas ops */
575	ENTRY(cas_delay)
576	casx [%o0], %g0, %g0
577	casx [%o0], %g0, %g0
578	retl
579	casx [%o0], %g0, %g0
580	SET_SIZE(cas_delay)
581
582/*
583 * alternative delay function for some niagara processors.   The rd
584 * instruction uses less resources than casx on those cpus.
585 */
586	ENTRY(rdccr_delay)
587	rd	%ccr, %g0
588	rd	%ccr, %g0
589	retl
590	rd	%ccr, %g0
591	SET_SIZE(rdccr_delay)
592
593/*
594 * mutex_delay_default(void)
595 * Spins for approx a few hundred processor cycles and returns to caller.
596 */
597
598	ENTRY(mutex_delay_default)
599	mov	72,%o0
6001:	brgz	%o0, 1b
601	dec	%o0
602	retl
603	nop
604	SET_SIZE(mutex_delay_default)
605
606