1744947dcSTom Erickson /* 2744947dcSTom Erickson * CDDL HEADER START 3744947dcSTom Erickson * 4744947dcSTom Erickson * The contents of this file are subject to the terms of the 5744947dcSTom Erickson * Common Development and Distribution License (the "License"). 6744947dcSTom Erickson * You may not use this file except in compliance with the License. 7744947dcSTom Erickson * 8744947dcSTom Erickson * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9744947dcSTom Erickson * or http://www.opensolaris.org/os/licensing. 10744947dcSTom Erickson * See the License for the specific language governing permissions 11744947dcSTom Erickson * and limitations under the License. 12744947dcSTom Erickson * 13744947dcSTom Erickson * When distributing Covered Code, include this CDDL HEADER in each 14744947dcSTom Erickson * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15744947dcSTom Erickson * If applicable, add the following below this CDDL HEADER, with the 16744947dcSTom Erickson * fields enclosed by brackets "[]" replaced with your own identifying 17744947dcSTom Erickson * information: Portions Copyright [yyyy] [name of copyright owner] 18744947dcSTom Erickson * 19744947dcSTom Erickson * CDDL HEADER END 20744947dcSTom Erickson */ 21744947dcSTom Erickson /* 22744947dcSTom Erickson * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 23*9a686fbcSPaul Dagnelie * Copyright (c) 2014, 2015 by Delphix. All rights reserved. 24744947dcSTom Erickson */ 25744947dcSTom Erickson 26744947dcSTom Erickson /* 27744947dcSTom Erickson * A Zero Reference Lock (ZRL) is a reference count that can lock out new 28744947dcSTom Erickson * references only when the count is zero and only without waiting if the count 29744947dcSTom Erickson * is not already zero. It is similar to a read-write lock in that it allows 30744947dcSTom Erickson * multiple readers and only a single writer, but it does not allow a writer to 31744947dcSTom Erickson * block while waiting for readers to exit, and therefore the question of 32744947dcSTom Erickson * reader/writer priority is moot (no WRWANT bit). Since the equivalent of 33744947dcSTom Erickson * rw_enter(&lock, RW_WRITER) is disallowed and only tryenter() is allowed, it 34744947dcSTom Erickson * is perfectly safe for the same reader to acquire the same lock multiple 35744947dcSTom Erickson * times. The fact that a ZRL is reentrant for readers (through multiple calls 36744947dcSTom Erickson * to zrl_add()) makes it convenient for determining whether something is 37744947dcSTom Erickson * actively referenced without the fuss of flagging lock ownership across 38744947dcSTom Erickson * function calls. 39744947dcSTom Erickson */ 40744947dcSTom Erickson #include <sys/zrlock.h> 41744947dcSTom Erickson 42744947dcSTom Erickson /* 43744947dcSTom Erickson * A ZRL can be locked only while there are zero references, so ZRL_LOCKED is 44744947dcSTom Erickson * treated as zero references. 45744947dcSTom Erickson */ 468df17305SMatthew Ahrens #define ZRL_LOCKED -1 47744947dcSTom Erickson #define ZRL_DESTROYED -2 48744947dcSTom Erickson 49744947dcSTom Erickson void 50744947dcSTom Erickson zrl_init(zrlock_t *zrl) 51744947dcSTom Erickson { 52744947dcSTom Erickson mutex_init(&zrl->zr_mtx, NULL, MUTEX_DEFAULT, NULL); 53744947dcSTom Erickson zrl->zr_refcount = 0; 54744947dcSTom Erickson cv_init(&zrl->zr_cv, NULL, CV_DEFAULT, NULL); 55744947dcSTom Erickson #ifdef ZFS_DEBUG 56744947dcSTom Erickson zrl->zr_owner = NULL; 57744947dcSTom Erickson zrl->zr_caller = NULL; 58744947dcSTom Erickson #endif 59744947dcSTom Erickson } 60744947dcSTom Erickson 61744947dcSTom Erickson void 62744947dcSTom Erickson zrl_destroy(zrlock_t *zrl) 63744947dcSTom Erickson { 648df17305SMatthew Ahrens ASSERT0(zrl->zr_refcount); 65744947dcSTom Erickson 66744947dcSTom Erickson mutex_destroy(&zrl->zr_mtx); 67744947dcSTom Erickson zrl->zr_refcount = ZRL_DESTROYED; 68744947dcSTom Erickson cv_destroy(&zrl->zr_cv); 69744947dcSTom Erickson } 70744947dcSTom Erickson 71744947dcSTom Erickson void 72*9a686fbcSPaul Dagnelie zrl_add_impl(zrlock_t *zrl, const char *zc) 73744947dcSTom Erickson { 74744947dcSTom Erickson uint32_t n = (uint32_t)zrl->zr_refcount; 75744947dcSTom Erickson 76744947dcSTom Erickson while (n != ZRL_LOCKED) { 77744947dcSTom Erickson uint32_t cas = atomic_cas_32( 78744947dcSTom Erickson (uint32_t *)&zrl->zr_refcount, n, n + 1); 79744947dcSTom Erickson if (cas == n) { 808df17305SMatthew Ahrens ASSERT3S((int32_t)n, >=, 0); 81744947dcSTom Erickson #ifdef ZFS_DEBUG 82744947dcSTom Erickson if (zrl->zr_owner == curthread) { 83744947dcSTom Erickson DTRACE_PROBE2(zrlock__reentry, 84744947dcSTom Erickson zrlock_t *, zrl, uint32_t, n); 85744947dcSTom Erickson } 86744947dcSTom Erickson zrl->zr_owner = curthread; 87744947dcSTom Erickson zrl->zr_caller = zc; 88744947dcSTom Erickson #endif 89744947dcSTom Erickson return; 90744947dcSTom Erickson } 91744947dcSTom Erickson n = cas; 92744947dcSTom Erickson } 93744947dcSTom Erickson 94744947dcSTom Erickson mutex_enter(&zrl->zr_mtx); 95744947dcSTom Erickson while (zrl->zr_refcount == ZRL_LOCKED) { 96744947dcSTom Erickson cv_wait(&zrl->zr_cv, &zrl->zr_mtx); 97744947dcSTom Erickson } 988df17305SMatthew Ahrens ASSERT3S(zrl->zr_refcount, >=, 0); 99744947dcSTom Erickson zrl->zr_refcount++; 100744947dcSTom Erickson #ifdef ZFS_DEBUG 101744947dcSTom Erickson zrl->zr_owner = curthread; 102744947dcSTom Erickson zrl->zr_caller = zc; 103744947dcSTom Erickson #endif 104744947dcSTom Erickson mutex_exit(&zrl->zr_mtx); 105744947dcSTom Erickson } 106744947dcSTom Erickson 107744947dcSTom Erickson void 108744947dcSTom Erickson zrl_remove(zrlock_t *zrl) 109744947dcSTom Erickson { 110744947dcSTom Erickson uint32_t n; 111744947dcSTom Erickson 112744947dcSTom Erickson #ifdef ZFS_DEBUG 113744947dcSTom Erickson if (zrl->zr_owner == curthread) { 114744947dcSTom Erickson zrl->zr_owner = NULL; 115744947dcSTom Erickson zrl->zr_caller = NULL; 116744947dcSTom Erickson } 117744947dcSTom Erickson #endif 1188df17305SMatthew Ahrens n = atomic_dec_32_nv((uint32_t *)&zrl->zr_refcount); 1198df17305SMatthew Ahrens ASSERT3S((int32_t)n, >=, 0); 120744947dcSTom Erickson } 121744947dcSTom Erickson 122744947dcSTom Erickson int 123744947dcSTom Erickson zrl_tryenter(zrlock_t *zrl) 124744947dcSTom Erickson { 125744947dcSTom Erickson uint32_t n = (uint32_t)zrl->zr_refcount; 126744947dcSTom Erickson 127744947dcSTom Erickson if (n == 0) { 128744947dcSTom Erickson uint32_t cas = atomic_cas_32( 129744947dcSTom Erickson (uint32_t *)&zrl->zr_refcount, 0, ZRL_LOCKED); 130744947dcSTom Erickson if (cas == 0) { 131744947dcSTom Erickson #ifdef ZFS_DEBUG 1328df17305SMatthew Ahrens ASSERT3P(zrl->zr_owner, ==, NULL); 133744947dcSTom Erickson zrl->zr_owner = curthread; 134744947dcSTom Erickson #endif 135744947dcSTom Erickson return (1); 136744947dcSTom Erickson } 137744947dcSTom Erickson } 138744947dcSTom Erickson 1398df17305SMatthew Ahrens ASSERT3S((int32_t)n, >, ZRL_DESTROYED); 140744947dcSTom Erickson 141744947dcSTom Erickson return (0); 142744947dcSTom Erickson } 143744947dcSTom Erickson 144744947dcSTom Erickson void 145744947dcSTom Erickson zrl_exit(zrlock_t *zrl) 146744947dcSTom Erickson { 1478df17305SMatthew Ahrens ASSERT3S(zrl->zr_refcount, ==, ZRL_LOCKED); 148744947dcSTom Erickson 149744947dcSTom Erickson mutex_enter(&zrl->zr_mtx); 150744947dcSTom Erickson #ifdef ZFS_DEBUG 1518df17305SMatthew Ahrens ASSERT3P(zrl->zr_owner, ==, curthread); 152744947dcSTom Erickson zrl->zr_owner = NULL; 153744947dcSTom Erickson membar_producer(); /* make sure the owner store happens first */ 154744947dcSTom Erickson #endif 155744947dcSTom Erickson zrl->zr_refcount = 0; 156744947dcSTom Erickson cv_broadcast(&zrl->zr_cv); 157744947dcSTom Erickson mutex_exit(&zrl->zr_mtx); 158744947dcSTom Erickson } 159744947dcSTom Erickson 160744947dcSTom Erickson int 161744947dcSTom Erickson zrl_refcount(zrlock_t *zrl) 162744947dcSTom Erickson { 1638df17305SMatthew Ahrens ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED); 164744947dcSTom Erickson 165744947dcSTom Erickson int n = (int)zrl->zr_refcount; 166744947dcSTom Erickson return (n <= 0 ? 0 : n); 167744947dcSTom Erickson } 168744947dcSTom Erickson 169744947dcSTom Erickson int 170744947dcSTom Erickson zrl_is_zero(zrlock_t *zrl) 171744947dcSTom Erickson { 1728df17305SMatthew Ahrens ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED); 173744947dcSTom Erickson 174744947dcSTom Erickson return (zrl->zr_refcount <= 0); 175744947dcSTom Erickson } 176744947dcSTom Erickson 177744947dcSTom Erickson int 178744947dcSTom Erickson zrl_is_locked(zrlock_t *zrl) 179744947dcSTom Erickson { 1808df17305SMatthew Ahrens ASSERT3S(zrl->zr_refcount, >, ZRL_DESTROYED); 181744947dcSTom Erickson 182744947dcSTom Erickson return (zrl->zr_refcount == ZRL_LOCKED); 183744947dcSTom Erickson } 184744947dcSTom Erickson 185744947dcSTom Erickson #ifdef ZFS_DEBUG 186744947dcSTom Erickson kthread_t * 187744947dcSTom Erickson zrl_owner(zrlock_t *zrl) 188744947dcSTom Erickson { 189744947dcSTom Erickson return (zrl->zr_owner); 190744947dcSTom Erickson } 191744947dcSTom Erickson #endif 192