xref: /illumos-gate/usr/src/lib/libc/port/threads/rwlock.c (revision 5d2ed727)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2016 by Delphix. All rights reserved.
25  */
26 
27 #include "lint.h"
28 #include "thr_uberdata.h"
29 #include <sys/sdt.h>
30 
31 #define	TRY_FLAG		0x10
32 #define	READ_LOCK		0
33 #define	WRITE_LOCK		1
34 #define	READ_LOCK_TRY		(READ_LOCK | TRY_FLAG)
35 #define	WRITE_LOCK_TRY		(WRITE_LOCK | TRY_FLAG)
36 
37 #define	NLOCKS	4	/* initial number of readlock_t structs allocated */
38 
39 #define	ASSERT_CONSISTENT_STATE(readers)		\
40 	ASSERT(!((readers) & URW_WRITE_LOCKED) ||	\
41 		((readers) & ~URW_HAS_WAITERS) == URW_WRITE_LOCKED)
42 
43 /*
44  * Find/allocate an entry for rwlp in our array of rwlocks held for reading.
45  * We must be deferring signals for this to be safe.
46  * Else if we are returning an entry with ul_rdlockcnt == 0,
47  * it could be reassigned behind our back in a signal handler.
48  */
49 static readlock_t *
rwl_entry(rwlock_t * rwlp)50 rwl_entry(rwlock_t *rwlp)
51 {
52 	ulwp_t *self = curthread;
53 	readlock_t *remembered = NULL;
54 	readlock_t *readlockp;
55 	uint_t nlocks;
56 
57 	/* we must be deferring signals */
58 	ASSERT((self->ul_critical + self->ul_sigdefer) != 0);
59 
60 	if ((nlocks = self->ul_rdlockcnt) != 0)
61 		readlockp = self->ul_readlock.array;
62 	else {
63 		nlocks = 1;
64 		readlockp = &self->ul_readlock.single;
65 	}
66 
67 	for (; nlocks; nlocks--, readlockp++) {
68 		if (readlockp->rd_rwlock == rwlp)
69 			return (readlockp);
70 		if (readlockp->rd_count == 0 && remembered == NULL)
71 			remembered = readlockp;
72 	}
73 	if (remembered != NULL) {
74 		remembered->rd_rwlock = rwlp;
75 		return (remembered);
76 	}
77 
78 	/*
79 	 * No entry available.  Allocate more space, converting the single
80 	 * readlock_t entry into an array of readlock_t entries if necessary.
81 	 */
82 	if ((nlocks = self->ul_rdlockcnt) == 0) {
83 		/*
84 		 * Initial allocation of the readlock_t array.
85 		 * Convert the single entry into an array.
86 		 */
87 		self->ul_rdlockcnt = nlocks = NLOCKS;
88 		readlockp = lmalloc(nlocks * sizeof (readlock_t));
89 		/*
90 		 * The single readlock_t becomes the first entry in the array.
91 		 */
92 		*readlockp = self->ul_readlock.single;
93 		self->ul_readlock.single.rd_count = 0;
94 		self->ul_readlock.array = readlockp;
95 		/*
96 		 * Return the next available entry in the array.
97 		 */
98 		(++readlockp)->rd_rwlock = rwlp;
99 		return (readlockp);
100 	}
101 	/*
102 	 * Reallocate the array, double the size each time.
103 	 */
104 	readlockp = lmalloc(nlocks * 2 * sizeof (readlock_t));
105 	(void) memcpy(readlockp, self->ul_readlock.array,
106 	    nlocks * sizeof (readlock_t));
107 	lfree(self->ul_readlock.array, nlocks * sizeof (readlock_t));
108 	self->ul_readlock.array = readlockp;
109 	self->ul_rdlockcnt *= 2;
110 	/*
111 	 * Return the next available entry in the newly allocated array.
112 	 */
113 	(readlockp += nlocks)->rd_rwlock = rwlp;
114 	return (readlockp);
115 }
116 
117 /*
118  * Free the array of rwlocks held for reading.
119  */
120 void
rwl_free(ulwp_t * ulwp)121 rwl_free(ulwp_t *ulwp)
122 {
123 	uint_t nlocks;
124 
125 	if ((nlocks = ulwp->ul_rdlockcnt) != 0)
126 		lfree(ulwp->ul_readlock.array, nlocks * sizeof (readlock_t));
127 	ulwp->ul_rdlockcnt = 0;
128 	ulwp->ul_readlock.single.rd_rwlock = NULL;
129 	ulwp->ul_readlock.single.rd_count = 0;
130 }
131 
132 /*
133  * Check if a reader version of the lock is held by the current thread.
134  */
135 #pragma weak _rw_read_held = rw_read_held
136 int
rw_read_held(rwlock_t * rwlp)137 rw_read_held(rwlock_t *rwlp)
138 {
139 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
140 	uint32_t readers;
141 	ulwp_t *self = curthread;
142 	readlock_t *readlockp;
143 	uint_t nlocks;
144 	int rval = 0;
145 
146 	no_preempt(self);
147 
148 	readers = *rwstate;
149 	ASSERT_CONSISTENT_STATE(readers);
150 	if (!(readers & URW_WRITE_LOCKED) &&
151 	    (readers & URW_READERS_MASK) != 0) {
152 		/*
153 		 * The lock is held for reading by some thread.
154 		 * Search our array of rwlocks held for reading for a match.
155 		 */
156 		if ((nlocks = self->ul_rdlockcnt) != 0)
157 			readlockp = self->ul_readlock.array;
158 		else {
159 			nlocks = 1;
160 			readlockp = &self->ul_readlock.single;
161 		}
162 		for (; nlocks; nlocks--, readlockp++) {
163 			if (readlockp->rd_rwlock == rwlp) {
164 				if (readlockp->rd_count)
165 					rval = 1;
166 				break;
167 			}
168 		}
169 	}
170 
171 	preempt(self);
172 	return (rval);
173 }
174 
175 /*
176  * Check if a writer version of the lock is held by the current thread.
177  */
178 #pragma weak _rw_write_held = rw_write_held
179 int
rw_write_held(rwlock_t * rwlp)180 rw_write_held(rwlock_t *rwlp)
181 {
182 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
183 	uint32_t readers;
184 	ulwp_t *self = curthread;
185 	int rval;
186 
187 	no_preempt(self);
188 
189 	readers = *rwstate;
190 	ASSERT_CONSISTENT_STATE(readers);
191 	rval = ((readers & URW_WRITE_LOCKED) &&
192 	    rwlp->rwlock_owner == (uintptr_t)self &&
193 	    (rwlp->rwlock_type == USYNC_THREAD ||
194 	    rwlp->rwlock_ownerpid == self->ul_uberdata->pid));
195 
196 	preempt(self);
197 	return (rval);
198 }
199 
200 #pragma weak _rwlock_init = rwlock_init
201 /* ARGSUSED2 */
202 int
rwlock_init(rwlock_t * rwlp,int type,void * arg)203 rwlock_init(rwlock_t *rwlp, int type, void *arg)
204 {
205 	ulwp_t *self = curthread;
206 
207 	if (type != USYNC_THREAD && type != USYNC_PROCESS)
208 		return (EINVAL);
209 	/*
210 	 * Once reinitialized, we can no longer be holding a read or write lock.
211 	 * We can do nothing about other threads that are holding read locks.
212 	 */
213 	sigoff(self);
214 	rwl_entry(rwlp)->rd_count = 0;
215 	sigon(self);
216 	(void) memset(rwlp, 0, sizeof (*rwlp));
217 	rwlp->rwlock_type = (uint16_t)type;
218 	rwlp->rwlock_magic = RWL_MAGIC;
219 	rwlp->mutex.mutex_type = (uint8_t)type;
220 	rwlp->mutex.mutex_flag = LOCK_INITED;
221 	rwlp->mutex.mutex_magic = MUTEX_MAGIC;
222 
223 	/*
224 	 * This should be at the beginning of the function,
225 	 * but for the sake of old broken applications that
226 	 * do not have proper alignment for their rwlocks
227 	 * (and don't check the return code from rwlock_init),
228 	 * we put it here, after initializing the rwlock regardless.
229 	 */
230 	if (((uintptr_t)rwlp & (_LONG_LONG_ALIGNMENT - 1)) &&
231 	    self->ul_misaligned == 0)
232 		return (EINVAL);
233 
234 	return (0);
235 }
236 
237 #pragma weak pthread_rwlock_destroy = rwlock_destroy
238 #pragma weak _rwlock_destroy = rwlock_destroy
239 int
rwlock_destroy(rwlock_t * rwlp)240 rwlock_destroy(rwlock_t *rwlp)
241 {
242 	ulwp_t *self = curthread;
243 
244 	/*
245 	 * Once destroyed, we can no longer be holding a read or write lock.
246 	 * We can do nothing about other threads that are holding read locks.
247 	 */
248 	sigoff(self);
249 	rwl_entry(rwlp)->rd_count = 0;
250 	sigon(self);
251 	rwlp->rwlock_magic = 0;
252 	tdb_sync_obj_deregister(rwlp);
253 	return (0);
254 }
255 
256 /*
257  * The following four functions:
258  *	read_lock_try()
259  *	read_unlock_try()
260  *	write_lock_try()
261  *	write_unlock_try()
262  * lie at the heart of the fast-path code for rwlocks,
263  * both process-private and process-shared.
264  *
265  * They are called once without recourse to any other locking primitives.
266  * If they succeed, we are done and the fast-path code was successful.
267  * If they fail, we have to deal with lock queues, either to enqueue
268  * ourself and sleep or to dequeue and wake up someone else (slow paths).
269  *
270  * Unless 'ignore_waiters_flag' is true (a condition that applies only
271  * when read_lock_try() or write_lock_try() is called from code that
272  * is already in the slow path and has already acquired the queue lock),
273  * these functions will always fail if the waiters flag, URW_HAS_WAITERS,
274  * is set in the 'rwstate' word.  Thus, setting the waiters flag on the
275  * rwlock and acquiring the queue lock guarantees exclusive access to
276  * the rwlock (and is the only way to guarantee exclusive access).
277  */
278 
279 /*
280  * Attempt to acquire a readers lock.  Return true on success.
281  */
282 static int
read_lock_try(rwlock_t * rwlp,int ignore_waiters_flag)283 read_lock_try(rwlock_t *rwlp, int ignore_waiters_flag)
284 {
285 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
286 	uint32_t mask = ignore_waiters_flag?
287 	    URW_WRITE_LOCKED : (URW_HAS_WAITERS | URW_WRITE_LOCKED);
288 	uint32_t readers;
289 	ulwp_t *self = curthread;
290 
291 	no_preempt(self);
292 	while (((readers = *rwstate) & mask) == 0) {
293 		if (atomic_cas_32(rwstate, readers, readers + 1) == readers) {
294 			preempt(self);
295 			return (1);
296 		}
297 	}
298 	preempt(self);
299 	return (0);
300 }
301 
302 /*
303  * Attempt to release a reader lock.  Return true on success.
304  */
305 static int
read_unlock_try(rwlock_t * rwlp)306 read_unlock_try(rwlock_t *rwlp)
307 {
308 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
309 	uint32_t readers;
310 	ulwp_t *self = curthread;
311 
312 	no_preempt(self);
313 	while (((readers = *rwstate) & URW_HAS_WAITERS) == 0) {
314 		if (atomic_cas_32(rwstate, readers, readers - 1) == readers) {
315 			preempt(self);
316 			return (1);
317 		}
318 	}
319 	preempt(self);
320 	return (0);
321 }
322 
323 /*
324  * Attempt to acquire a writer lock.  Return true on success.
325  */
326 static int
write_lock_try(rwlock_t * rwlp,int ignore_waiters_flag)327 write_lock_try(rwlock_t *rwlp, int ignore_waiters_flag)
328 {
329 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
330 	uint32_t mask = ignore_waiters_flag?
331 	    (URW_WRITE_LOCKED | URW_READERS_MASK) :
332 	    (URW_HAS_WAITERS | URW_WRITE_LOCKED | URW_READERS_MASK);
333 	ulwp_t *self = curthread;
334 	uint32_t readers;
335 
336 	no_preempt(self);
337 	while (((readers = *rwstate) & mask) == 0) {
338 		if (atomic_cas_32(rwstate, readers, readers | URW_WRITE_LOCKED)
339 		    == readers) {
340 			preempt(self);
341 			return (1);
342 		}
343 	}
344 	preempt(self);
345 	return (0);
346 }
347 
348 /*
349  * Attempt to release a writer lock.  Return true on success.
350  */
351 static int
write_unlock_try(rwlock_t * rwlp)352 write_unlock_try(rwlock_t *rwlp)
353 {
354 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
355 	uint32_t readers;
356 	ulwp_t *self = curthread;
357 
358 	no_preempt(self);
359 	while (((readers = *rwstate) & URW_HAS_WAITERS) == 0) {
360 		if (atomic_cas_32(rwstate, readers, 0) == readers) {
361 			preempt(self);
362 			return (1);
363 		}
364 	}
365 	preempt(self);
366 	return (0);
367 }
368 
369 /*
370  * Release a process-private rwlock and wake up any thread(s) sleeping on it.
371  * This is called when a thread releases a lock that appears to have waiters.
372  */
373 static void
rw_queue_release(rwlock_t * rwlp)374 rw_queue_release(rwlock_t *rwlp)
375 {
376 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
377 	queue_head_t *qp;
378 	uint32_t readers;
379 	uint32_t writer;
380 	ulwp_t **ulwpp;
381 	ulwp_t *ulwp;
382 	ulwp_t *prev;
383 	int nlwpid = 0;
384 	int more;
385 	int maxlwps = MAXLWPS;
386 	lwpid_t buffer[MAXLWPS];
387 	lwpid_t *lwpid = buffer;
388 
389 	qp = queue_lock(rwlp, MX);
390 
391 	/*
392 	 * Here is where we actually drop the lock,
393 	 * but we retain the URW_HAS_WAITERS flag, if it is already set.
394 	 */
395 	readers = *rwstate;
396 	ASSERT_CONSISTENT_STATE(readers);
397 	if (readers & URW_WRITE_LOCKED)	/* drop the writer lock */
398 		atomic_and_32(rwstate, ~URW_WRITE_LOCKED);
399 	else				/* drop the readers lock */
400 		atomic_dec_32(rwstate);
401 	if (!(readers & URW_HAS_WAITERS)) {	/* no waiters */
402 		queue_unlock(qp);
403 		return;
404 	}
405 
406 	/*
407 	 * The presence of the URW_HAS_WAITERS flag causes all rwlock
408 	 * code to go through the slow path, acquiring queue_lock(qp).
409 	 * Therefore, the rest of this code is safe because we are
410 	 * holding the queue lock and the URW_HAS_WAITERS flag is set.
411 	 */
412 
413 	readers = *rwstate;		/* must fetch the value again */
414 	ASSERT_CONSISTENT_STATE(readers);
415 	ASSERT(readers & URW_HAS_WAITERS);
416 	readers &= URW_READERS_MASK;	/* count of current readers */
417 	writer = 0;			/* no current writer */
418 
419 	/*
420 	 * Examine the queue of waiters in priority order and prepare
421 	 * to wake up as many readers as we encounter before encountering
422 	 * a writer.  If the highest priority thread on the queue is a
423 	 * writer, stop there and wake it up.
424 	 *
425 	 * We keep track of lwpids that are to be unparked in lwpid[].
426 	 * __lwp_unpark_all() is called to unpark all of them after
427 	 * they have been removed from the sleep queue and the sleep
428 	 * queue lock has been dropped.  If we run out of space in our
429 	 * on-stack buffer, we need to allocate more but we can't call
430 	 * lmalloc() because we are holding a queue lock when the overflow
431 	 * occurs and lmalloc() acquires a lock.  We can't use alloca()
432 	 * either because the application may have allocated a small
433 	 * stack and we don't want to overrun the stack.  So we call
434 	 * alloc_lwpids() to allocate a bigger buffer using the mmap()
435 	 * system call directly since that path acquires no locks.
436 	 */
437 	while ((ulwpp = queue_slot(qp, &prev, &more)) != NULL) {
438 		ulwp = *ulwpp;
439 		ASSERT(ulwp->ul_wchan == rwlp);
440 		if (ulwp->ul_writer) {
441 			if (writer != 0 || readers != 0)
442 				break;
443 			/* one writer to wake */
444 			writer++;
445 		} else {
446 			if (writer != 0)
447 				break;
448 			/* at least one reader to wake */
449 			readers++;
450 			if (nlwpid == maxlwps)
451 				lwpid = alloc_lwpids(lwpid, &nlwpid, &maxlwps);
452 		}
453 		queue_unlink(qp, ulwpp, prev);
454 		ulwp->ul_sleepq = NULL;
455 		ulwp->ul_wchan = NULL;
456 		if (writer) {
457 			/*
458 			 * Hand off the lock to the writer we will be waking.
459 			 */
460 			ASSERT((*rwstate & ~URW_HAS_WAITERS) == 0);
461 			atomic_or_32(rwstate, URW_WRITE_LOCKED);
462 			rwlp->rwlock_owner = (uintptr_t)ulwp;
463 		}
464 		lwpid[nlwpid++] = ulwp->ul_lwpid;
465 	}
466 
467 	/*
468 	 * This modification of rwstate must be done last.
469 	 * The presence of the URW_HAS_WAITERS flag causes all rwlock
470 	 * code to go through the slow path, acquiring queue_lock(qp).
471 	 * Otherwise the read_lock_try() and write_lock_try() fast paths
472 	 * are effective.
473 	 */
474 	if (ulwpp == NULL)
475 		atomic_and_32(rwstate, ~URW_HAS_WAITERS);
476 
477 	if (nlwpid == 0) {
478 		queue_unlock(qp);
479 	} else {
480 		ulwp_t *self = curthread;
481 		no_preempt(self);
482 		queue_unlock(qp);
483 		if (nlwpid == 1)
484 			(void) __lwp_unpark(lwpid[0]);
485 		else
486 			(void) __lwp_unpark_all(lwpid, nlwpid);
487 		preempt(self);
488 	}
489 	if (lwpid != buffer)
490 		(void) munmap((caddr_t)lwpid, maxlwps * sizeof (lwpid_t));
491 }
492 
493 /*
494  * Common code for rdlock, timedrdlock, wrlock, timedwrlock, tryrdlock,
495  * and trywrlock for process-shared (USYNC_PROCESS) rwlocks.
496  *
497  * Note: if the lock appears to be contended we call __lwp_rwlock_rdlock()
498  * or __lwp_rwlock_wrlock() holding the mutex. These return with the mutex
499  * released, and if they need to sleep will release the mutex first. In the
500  * event of a spurious wakeup, these will return EAGAIN (because it is much
501  * easier for us to re-acquire the mutex here).
502  */
503 int
shared_rwlock_lock(rwlock_t * rwlp,timespec_t * tsp,int rd_wr)504 shared_rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
505 {
506 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
507 	mutex_t *mp = &rwlp->mutex;
508 	int try_flag;
509 	int error;
510 
511 	try_flag = (rd_wr & TRY_FLAG);
512 	rd_wr &= ~TRY_FLAG;
513 	ASSERT(rd_wr == READ_LOCK || rd_wr == WRITE_LOCK);
514 
515 	if (!try_flag) {
516 		DTRACE_PROBE2(plockstat, rw__block, rwlp, rd_wr);
517 	}
518 
519 	do {
520 		if (try_flag && (*rwstate & URW_WRITE_LOCKED)) {
521 			error = EBUSY;
522 			break;
523 		}
524 		if ((error = mutex_lock(mp)) != 0)
525 			break;
526 		if (rd_wr == READ_LOCK) {
527 			if (read_lock_try(rwlp, 0)) {
528 				(void) mutex_unlock(mp);
529 				break;
530 			}
531 		} else {
532 			if (write_lock_try(rwlp, 0)) {
533 				(void) mutex_unlock(mp);
534 				break;
535 			}
536 		}
537 		atomic_or_32(rwstate, URW_HAS_WAITERS);
538 
539 #ifdef THREAD_DEBUG
540 		uint32_t readers;
541 		readers = *rwstate;
542 		ASSERT_CONSISTENT_STATE(readers);
543 #endif
544 		/*
545 		 * The calls to __lwp_rwlock_*() below will release the mutex,
546 		 * so we need a dtrace probe here.  The owner field of the
547 		 * mutex is cleared in the kernel when the mutex is released,
548 		 * so we should not clear it here.
549 		 */
550 		DTRACE_PROBE2(plockstat, mutex__release, mp, 0);
551 		/*
552 		 * The waiters bit may be inaccurate.
553 		 * Only the kernel knows for sure.
554 		 */
555 		if (rd_wr == READ_LOCK) {
556 			if (try_flag)
557 				error = __lwp_rwlock_tryrdlock(rwlp);
558 			else
559 				error = __lwp_rwlock_rdlock(rwlp, tsp);
560 		} else {
561 			if (try_flag)
562 				error = __lwp_rwlock_trywrlock(rwlp);
563 			else
564 				error = __lwp_rwlock_wrlock(rwlp, tsp);
565 		}
566 	} while (error == EAGAIN || error == EINTR);
567 
568 	if (!try_flag) {
569 		DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, error == 0);
570 	}
571 
572 	return (error);
573 }
574 
575 /*
576  * Common code for rdlock, timedrdlock, wrlock, timedwrlock, tryrdlock,
577  * and trywrlock for process-private (USYNC_THREAD) rwlocks.
578  */
579 int
rwlock_lock(rwlock_t * rwlp,timespec_t * tsp,int rd_wr)580 rwlock_lock(rwlock_t *rwlp, timespec_t *tsp, int rd_wr)
581 {
582 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
583 	uint32_t readers;
584 	ulwp_t *self = curthread;
585 	queue_head_t *qp;
586 	ulwp_t *ulwp;
587 	int try_flag;
588 	int ignore_waiters_flag;
589 	int error = 0;
590 
591 	try_flag = (rd_wr & TRY_FLAG);
592 	rd_wr &= ~TRY_FLAG;
593 	ASSERT(rd_wr == READ_LOCK || rd_wr == WRITE_LOCK);
594 
595 	if (!try_flag) {
596 		DTRACE_PROBE2(plockstat, rw__block, rwlp, rd_wr);
597 	}
598 
599 	qp = queue_lock(rwlp, MX);
600 	/* initial attempt to acquire the lock fails if there are waiters */
601 	ignore_waiters_flag = 0;
602 	while (error == 0) {
603 		if (rd_wr == READ_LOCK) {
604 			if (read_lock_try(rwlp, ignore_waiters_flag))
605 				break;
606 		} else {
607 			if (write_lock_try(rwlp, ignore_waiters_flag))
608 				break;
609 		}
610 		/* subsequent attempts do not fail due to waiters */
611 		ignore_waiters_flag = 1;
612 		atomic_or_32(rwstate, URW_HAS_WAITERS);
613 		readers = *rwstate;
614 		ASSERT_CONSISTENT_STATE(readers);
615 		if ((readers & URW_WRITE_LOCKED) ||
616 		    (rd_wr == WRITE_LOCK &&
617 		    (readers & URW_READERS_MASK) != 0))
618 			/* EMPTY */;	/* somebody holds the lock */
619 		else if ((ulwp = queue_waiter(qp)) == NULL) {
620 			atomic_and_32(rwstate, ~URW_HAS_WAITERS);
621 			ignore_waiters_flag = 0;
622 			continue;	/* no queued waiters, start over */
623 		} else {
624 			/*
625 			 * Do a priority check on the queued waiter (the
626 			 * highest priority thread on the queue) to see
627 			 * if we should defer to it or just grab the lock.
628 			 */
629 			int our_pri = real_priority(self);
630 			int his_pri = real_priority(ulwp);
631 
632 			if (rd_wr == WRITE_LOCK) {
633 				/*
634 				 * We defer to a queued thread that has
635 				 * a higher priority than ours.
636 				 */
637 				if (his_pri <= our_pri) {
638 					/*
639 					 * Don't defer, just grab the lock.
640 					 */
641 					continue;
642 				}
643 			} else {
644 				/*
645 				 * We defer to a queued thread that has
646 				 * a higher priority than ours or that
647 				 * is a writer whose priority equals ours.
648 				 */
649 				if (his_pri < our_pri ||
650 				    (his_pri == our_pri && !ulwp->ul_writer)) {
651 					/*
652 					 * Don't defer, just grab the lock.
653 					 */
654 					continue;
655 				}
656 			}
657 		}
658 		/*
659 		 * We are about to block.
660 		 * If we're doing a trylock, return EBUSY instead.
661 		 */
662 		if (try_flag) {
663 			error = EBUSY;
664 			break;
665 		}
666 		/*
667 		 * Enqueue writers ahead of readers.
668 		 */
669 		self->ul_writer = rd_wr;	/* *must* be 0 or 1 */
670 		enqueue(qp, self, 0);
671 		set_parking_flag(self, 1);
672 		queue_unlock(qp);
673 		if ((error = __lwp_park(tsp, 0)) == EINTR)
674 			error = 0;
675 		set_parking_flag(self, 0);
676 		qp = queue_lock(rwlp, MX);
677 		if (self->ul_sleepq && dequeue_self(qp) == 0) {
678 			atomic_and_32(rwstate, ~URW_HAS_WAITERS);
679 			ignore_waiters_flag = 0;
680 		}
681 		self->ul_writer = 0;
682 		if (rd_wr == WRITE_LOCK &&
683 		    (*rwstate & URW_WRITE_LOCKED) &&
684 		    rwlp->rwlock_owner == (uintptr_t)self) {
685 			/*
686 			 * We acquired the lock by hand-off
687 			 * from the previous owner,
688 			 */
689 			error = 0;	/* timedlock did not fail */
690 			break;
691 		}
692 	}
693 
694 	/*
695 	 * Make one final check to see if there are any threads left
696 	 * on the rwlock queue.  Clear the URW_HAS_WAITERS flag if not.
697 	 */
698 	if (qp->qh_root == NULL || qp->qh_root->qr_head == NULL)
699 		atomic_and_32(rwstate, ~URW_HAS_WAITERS);
700 
701 	queue_unlock(qp);
702 
703 	if (!try_flag) {
704 		DTRACE_PROBE3(plockstat, rw__blocked, rwlp, rd_wr, error == 0);
705 	}
706 
707 	return (error);
708 }
709 
710 int
rw_rdlock_impl(rwlock_t * rwlp,timespec_t * tsp)711 rw_rdlock_impl(rwlock_t *rwlp, timespec_t *tsp)
712 {
713 	ulwp_t *self = curthread;
714 	uberdata_t *udp = self->ul_uberdata;
715 	readlock_t *readlockp;
716 	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
717 	int error;
718 
719 	/*
720 	 * If we already hold a readers lock on this rwlock,
721 	 * just increment our reference count and return.
722 	 */
723 	sigoff(self);
724 	readlockp = rwl_entry(rwlp);
725 	if (readlockp->rd_count != 0) {
726 		if (readlockp->rd_count == READ_LOCK_MAX) {
727 			sigon(self);
728 			error = EAGAIN;
729 			goto out;
730 		}
731 		sigon(self);
732 		error = 0;
733 		goto out;
734 	}
735 	sigon(self);
736 
737 	/*
738 	 * If we hold the writer lock, bail out.
739 	 */
740 	if (rw_write_held(rwlp)) {
741 		if (self->ul_error_detection)
742 			rwlock_error(rwlp, "rwlock_rdlock",
743 			    "calling thread owns the writer lock");
744 		error = EDEADLK;
745 		goto out;
746 	}
747 
748 	if (read_lock_try(rwlp, 0))
749 		error = 0;
750 	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
751 		error = shared_rwlock_lock(rwlp, tsp, READ_LOCK);
752 	else						/* user-level */
753 		error = rwlock_lock(rwlp, tsp, READ_LOCK);
754 
755 out:
756 	if (error == 0) {
757 		sigoff(self);
758 		rwl_entry(rwlp)->rd_count++;
759 		sigon(self);
760 		if (rwsp)
761 			tdb_incr(rwsp->rw_rdlock);
762 		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK);
763 	} else {
764 		DTRACE_PROBE3(plockstat, rw__error, rwlp, READ_LOCK, error);
765 	}
766 
767 	return (error);
768 }
769 
770 #pragma weak pthread_rwlock_rdlock = rw_rdlock
771 #pragma weak _rw_rdlock = rw_rdlock
772 int
rw_rdlock(rwlock_t * rwlp)773 rw_rdlock(rwlock_t *rwlp)
774 {
775 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
776 	return (rw_rdlock_impl(rwlp, NULL));
777 }
778 
779 void
lrw_rdlock(rwlock_t * rwlp)780 lrw_rdlock(rwlock_t *rwlp)
781 {
782 	enter_critical(curthread);
783 	(void) rw_rdlock_impl(rwlp, NULL);
784 }
785 
786 int
pthread_rwlock_reltimedrdlock_np(pthread_rwlock_t * _RESTRICT_KYWD rwlp,const struct timespec * _RESTRICT_KYWD reltime)787 pthread_rwlock_reltimedrdlock_np(pthread_rwlock_t *_RESTRICT_KYWD rwlp,
788     const struct timespec *_RESTRICT_KYWD reltime)
789 {
790 	timespec_t tslocal = *reltime;
791 	int error;
792 
793 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
794 	error = rw_rdlock_impl((rwlock_t *)rwlp, &tslocal);
795 	if (error == ETIME)
796 		error = ETIMEDOUT;
797 	return (error);
798 }
799 
800 int
pthread_rwlock_timedrdlock(pthread_rwlock_t * _RESTRICT_KYWD rwlp,const struct timespec * _RESTRICT_KYWD abstime)801 pthread_rwlock_timedrdlock(pthread_rwlock_t *_RESTRICT_KYWD rwlp,
802     const struct timespec *_RESTRICT_KYWD abstime)
803 {
804 	timespec_t tslocal;
805 	int error;
806 
807 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
808 	abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal);
809 	error = rw_rdlock_impl((rwlock_t *)rwlp, &tslocal);
810 	if (error == ETIME)
811 		error = ETIMEDOUT;
812 	return (error);
813 }
814 
815 int
rw_wrlock_impl(rwlock_t * rwlp,timespec_t * tsp)816 rw_wrlock_impl(rwlock_t *rwlp, timespec_t *tsp)
817 {
818 	ulwp_t *self = curthread;
819 	uberdata_t *udp = self->ul_uberdata;
820 	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
821 	int error;
822 
823 	/*
824 	 * If we hold a readers lock on this rwlock, bail out.
825 	 */
826 	if (rw_read_held(rwlp)) {
827 		if (self->ul_error_detection)
828 			rwlock_error(rwlp, "rwlock_wrlock",
829 			    "calling thread owns the readers lock");
830 		error = EDEADLK;
831 		goto out;
832 	}
833 
834 	/*
835 	 * If we hold the writer lock, bail out.
836 	 */
837 	if (rw_write_held(rwlp)) {
838 		if (self->ul_error_detection)
839 			rwlock_error(rwlp, "rwlock_wrlock",
840 			    "calling thread owns the writer lock");
841 		error = EDEADLK;
842 		goto out;
843 	}
844 
845 	if (write_lock_try(rwlp, 0))
846 		error = 0;
847 	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
848 		error = shared_rwlock_lock(rwlp, tsp, WRITE_LOCK);
849 	else						/* user-level */
850 		error = rwlock_lock(rwlp, tsp, WRITE_LOCK);
851 
852 out:
853 	if (error == 0) {
854 		rwlp->rwlock_owner = (uintptr_t)self;
855 		if (rwlp->rwlock_type == USYNC_PROCESS)
856 			rwlp->rwlock_ownerpid = udp->pid;
857 		if (rwsp) {
858 			tdb_incr(rwsp->rw_wrlock);
859 			rwsp->rw_wrlock_begin_hold = gethrtime();
860 		}
861 		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, WRITE_LOCK);
862 	} else {
863 		DTRACE_PROBE3(plockstat, rw__error, rwlp, WRITE_LOCK, error);
864 	}
865 	return (error);
866 }
867 
868 #pragma weak pthread_rwlock_wrlock = rw_wrlock
869 #pragma weak _rw_wrlock = rw_wrlock
870 int
rw_wrlock(rwlock_t * rwlp)871 rw_wrlock(rwlock_t *rwlp)
872 {
873 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
874 	return (rw_wrlock_impl(rwlp, NULL));
875 }
876 
877 void
lrw_wrlock(rwlock_t * rwlp)878 lrw_wrlock(rwlock_t *rwlp)
879 {
880 	enter_critical(curthread);
881 	(void) rw_wrlock_impl(rwlp, NULL);
882 }
883 
884 int
pthread_rwlock_reltimedwrlock_np(pthread_rwlock_t * _RESTRICT_KYWD rwlp,const struct timespec * _RESTRICT_KYWD reltime)885 pthread_rwlock_reltimedwrlock_np(pthread_rwlock_t *_RESTRICT_KYWD rwlp,
886     const struct timespec *_RESTRICT_KYWD reltime)
887 {
888 	timespec_t tslocal = *reltime;
889 	int error;
890 
891 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
892 	error = rw_wrlock_impl((rwlock_t *)rwlp, &tslocal);
893 	if (error == ETIME)
894 		error = ETIMEDOUT;
895 	return (error);
896 }
897 
898 int
pthread_rwlock_timedwrlock(pthread_rwlock_t * rwlp,const timespec_t * abstime)899 pthread_rwlock_timedwrlock(pthread_rwlock_t *rwlp, const timespec_t *abstime)
900 {
901 	timespec_t tslocal;
902 	int error;
903 
904 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
905 	abstime_to_reltime(CLOCK_REALTIME, abstime, &tslocal);
906 	error = rw_wrlock_impl((rwlock_t *)rwlp, &tslocal);
907 	if (error == ETIME)
908 		error = ETIMEDOUT;
909 	return (error);
910 }
911 
912 #pragma weak pthread_rwlock_tryrdlock = rw_tryrdlock
913 int
rw_tryrdlock(rwlock_t * rwlp)914 rw_tryrdlock(rwlock_t *rwlp)
915 {
916 	ulwp_t *self = curthread;
917 	uberdata_t *udp = self->ul_uberdata;
918 	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
919 	readlock_t *readlockp;
920 	int error;
921 
922 	ASSERT(!curthread->ul_critical || curthread->ul_bindflags);
923 
924 	if (rwsp)
925 		tdb_incr(rwsp->rw_rdlock_try);
926 
927 	/*
928 	 * If we already hold a readers lock on this rwlock,
929 	 * just increment our reference count and return.
930 	 */
931 	sigoff(self);
932 	readlockp = rwl_entry(rwlp);
933 	if (readlockp->rd_count != 0) {
934 		if (readlockp->rd_count == READ_LOCK_MAX) {
935 			sigon(self);
936 			error = EAGAIN;
937 			goto out;
938 		}
939 		sigon(self);
940 		error = 0;
941 		goto out;
942 	}
943 	sigon(self);
944 
945 	if (read_lock_try(rwlp, 0))
946 		error = 0;
947 	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
948 		error = shared_rwlock_lock(rwlp, NULL, READ_LOCK_TRY);
949 	else						/* user-level */
950 		error = rwlock_lock(rwlp, NULL, READ_LOCK_TRY);
951 
952 out:
953 	if (error == 0) {
954 		sigoff(self);
955 		rwl_entry(rwlp)->rd_count++;
956 		sigon(self);
957 		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, READ_LOCK);
958 	} else {
959 		if (rwsp)
960 			tdb_incr(rwsp->rw_rdlock_try_fail);
961 		if (error != EBUSY) {
962 			DTRACE_PROBE3(plockstat, rw__error, rwlp, READ_LOCK,
963 			    error);
964 		}
965 	}
966 
967 	return (error);
968 }
969 
970 #pragma weak pthread_rwlock_trywrlock = rw_trywrlock
971 int
rw_trywrlock(rwlock_t * rwlp)972 rw_trywrlock(rwlock_t *rwlp)
973 {
974 	ulwp_t *self = curthread;
975 	uberdata_t *udp = self->ul_uberdata;
976 	tdb_rwlock_stats_t *rwsp = RWLOCK_STATS(rwlp, udp);
977 	int error;
978 
979 	ASSERT(!self->ul_critical || self->ul_bindflags);
980 
981 	if (rwsp)
982 		tdb_incr(rwsp->rw_wrlock_try);
983 
984 	if (write_lock_try(rwlp, 0))
985 		error = 0;
986 	else if (rwlp->rwlock_type == USYNC_PROCESS)	/* kernel-level */
987 		error = shared_rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY);
988 	else						/* user-level */
989 		error = rwlock_lock(rwlp, NULL, WRITE_LOCK_TRY);
990 
991 	if (error == 0) {
992 		rwlp->rwlock_owner = (uintptr_t)self;
993 		if (rwlp->rwlock_type == USYNC_PROCESS)
994 			rwlp->rwlock_ownerpid = udp->pid;
995 		if (rwsp)
996 			rwsp->rw_wrlock_begin_hold = gethrtime();
997 		DTRACE_PROBE2(plockstat, rw__acquire, rwlp, WRITE_LOCK);
998 	} else {
999 		if (rwsp)
1000 			tdb_incr(rwsp->rw_wrlock_try_fail);
1001 		if (error != EBUSY) {
1002 			DTRACE_PROBE3(plockstat, rw__error, rwlp, WRITE_LOCK,
1003 			    error);
1004 		}
1005 	}
1006 	return (error);
1007 }
1008 
1009 #pragma weak pthread_rwlock_unlock = rw_unlock
1010 #pragma weak _rw_unlock = rw_unlock
1011 int
rw_unlock(rwlock_t * rwlp)1012 rw_unlock(rwlock_t *rwlp)
1013 {
1014 	volatile uint32_t *rwstate = (volatile uint32_t *)&rwlp->rwlock_readers;
1015 	uint32_t readers;
1016 	ulwp_t *self = curthread;
1017 	uberdata_t *udp = self->ul_uberdata;
1018 	tdb_rwlock_stats_t *rwsp;
1019 	int rd_wr;
1020 
1021 	readers = *rwstate;
1022 	ASSERT_CONSISTENT_STATE(readers);
1023 	if (readers & URW_WRITE_LOCKED) {
1024 		rd_wr = WRITE_LOCK;
1025 		readers = 0;
1026 	} else {
1027 		rd_wr = READ_LOCK;
1028 		readers &= URW_READERS_MASK;
1029 	}
1030 
1031 	if (rd_wr == WRITE_LOCK) {
1032 		/*
1033 		 * Since the writer lock is held, we'd better be
1034 		 * holding it, else we cannot legitimately be here.
1035 		 */
1036 		if (!rw_write_held(rwlp)) {
1037 			if (self->ul_error_detection)
1038 				rwlock_error(rwlp, "rwlock_unlock",
1039 				    "writer lock held, "
1040 				    "but not by the calling thread");
1041 			return (EPERM);
1042 		}
1043 		if ((rwsp = RWLOCK_STATS(rwlp, udp)) != NULL) {
1044 			if (rwsp->rw_wrlock_begin_hold)
1045 				rwsp->rw_wrlock_hold_time +=
1046 				    gethrtime() - rwsp->rw_wrlock_begin_hold;
1047 			rwsp->rw_wrlock_begin_hold = 0;
1048 		}
1049 		rwlp->rwlock_owner = 0;
1050 		rwlp->rwlock_ownerpid = 0;
1051 	} else if (readers > 0) {
1052 		/*
1053 		 * A readers lock is held; if we don't hold one, bail out.
1054 		 */
1055 		readlock_t *readlockp;
1056 
1057 		sigoff(self);
1058 		readlockp = rwl_entry(rwlp);
1059 		if (readlockp->rd_count == 0) {
1060 			sigon(self);
1061 			if (self->ul_error_detection)
1062 				rwlock_error(rwlp, "rwlock_unlock",
1063 				    "readers lock held, "
1064 				    "but not by the calling thread");
1065 			return (EPERM);
1066 		}
1067 		/*
1068 		 * If we hold more than one readers lock on this rwlock,
1069 		 * just decrement our reference count and return.
1070 		 */
1071 		if (--readlockp->rd_count != 0) {
1072 			sigon(self);
1073 			goto out;
1074 		}
1075 		sigon(self);
1076 	} else {
1077 		/*
1078 		 * This is a usage error.
1079 		 * No thread should release an unowned lock.
1080 		 */
1081 		if (self->ul_error_detection)
1082 			rwlock_error(rwlp, "rwlock_unlock", "lock not owned");
1083 		return (EPERM);
1084 	}
1085 
1086 	if (rd_wr == WRITE_LOCK && write_unlock_try(rwlp)) {
1087 		/* EMPTY */;
1088 	} else if (rd_wr == READ_LOCK && read_unlock_try(rwlp)) {
1089 		/* EMPTY */;
1090 	} else if (rwlp->rwlock_type == USYNC_PROCESS) {
1091 		(void) mutex_lock(&rwlp->mutex);
1092 		(void) __lwp_rwlock_unlock(rwlp);
1093 		(void) mutex_unlock(&rwlp->mutex);
1094 	} else {
1095 		rw_queue_release(rwlp);
1096 	}
1097 
1098 out:
1099 	DTRACE_PROBE2(plockstat, rw__release, rwlp, rd_wr);
1100 	return (0);
1101 }
1102 
1103 void
lrw_unlock(rwlock_t * rwlp)1104 lrw_unlock(rwlock_t *rwlp)
1105 {
1106 	(void) rw_unlock(rwlp);
1107 	exit_critical(curthread);
1108 }
1109