xref: /illumos-gate/usr/src/uts/common/os/ipc.c (revision f4421060)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T		*/
/*	All Rights Reserved					*/


/*
 * Common Inter-Process Communication routines.
 *
 * Overview
 * --------
 *
 * The System V inter-process communication (IPC) facilities provide
 * three services, message queues, semaphore arrays, and shared memory
 * segments, which are mananged using filesystem-like namespaces.
 * Unlike a filesystem, these namespaces aren't mounted and accessible
 * via a path -- a special API is used to interact with the different
 * facilities (nothing precludes a VFS-based interface, but the
 * standards require the special APIs).  Furthermore, these special
 * APIs don't use file descriptors, nor do they have an equivalent.
 * This means that every operation which acts on an object needs to
 * perform the quivalent of a lookup, which in turn means that every
 * operation can fail if the specified object doesn't exist in the
 * facility's namespace.
 *
 * Objects
 * -------
 *
 * Each object in a namespace has a unique ID, which is assigned by the
 * system and is used to identify the object when performing operations
 * on it.  An object can also have a key, which is selected by the user
 * at allocation time and is used as a primitive rendezvous mechanism.
 * An object without a key is said to have a "private" key.
 *
 * To perform an operation on an object given its key, one must first
 * perform a lookup and obtain its ID.  The ID is then used to identify
 * the object when performing the operation.  If the object has a
 * private key, the ID must be known or obtained by other means.
 *
 * Each object in the namespace has a creator uid and gid, as well as
 * an owner uid and gid.  Both are initialized with the ruid and rgid
 * of the process which created the object.  The creator or current
 * owner has the ability to change the owner of the object.
 *
 * Each object in the namespace has a set of file-like permissions,
 * which, in conjunction with the creator and owner uid and gid,
 * control read and write access to the object (execute is ignored).
 *
 * Each object also has a creator project and zone, which are used to
 * account for its resource usage.
 *
 * Operations
 * ----------
 *
 * There are five operations which all three facilities have in
 * common: GET, SET, STAT, RMID, and IDS.
 *
 * GET, like open, is used to allocate a new object or obtain an
 * existing one (using its key).  It takes a key, a set of flags and
 * mode bits, and optionally facility-specific arguments.  If the key
 * is IPC_PRIVATE, a new object with the requested mode bits and
 * facility-specific attributes is created.  If the key isn't
 * IPC_PRIVATE, the GET will attempt to look up the specified key and
 * either return that or create a new key depending on the state of the
 * IPC_CREAT and IPC_EXCL flags, much like open.  If GET needs to
 * allocate an object, it can fail if there is insufficient space in
 * the namespace (the maximum number of ids for the facility has been
 * exceeded) or if the facility-specific initialization fails.  If GET
 * finds an object it can return, it can still fail if that object's
 * permissions or facility-specific attributes are less than those
 * requested.
 *
 * SET is used to adjust facility-specific parameters of an object, in
 * addition to the owner uid and gid, and mode bits.  It can fail if
 * the caller isn't the creator or owner.
 *
 * STAT is used to obtain information about an object including the
 * general attributes object described as well as facility-specific
 * information.  It can fail if the caller doesn't have read
 * permission.
 *
 * RMID removes an object from the namespace.  Subsequent operations
 * using the object's ID or key will fail (until another object is
 * created with the same key or ID).  Since an RMID may be performed
 * asynchronously with other operations, it is possible that other
 * threads and/or processes will have references to the object.  While
 * a facility may have actions which need to be performed at RMID time,
 * only when all references are dropped can the object be destroyed.
 * RMID will fail if the caller isn't the creator or owner.
 *
 * IDS obtains a list of all IDs in a facility's namespace.  There are
 * no facility-specific behaviors of IDS.
 *
 * Design
 * ------
 *
 * Because some IPC facilities provide services whose operations must
 * scale, a mechanism which allows fast, concurrent access to
 * individual objects is needed.  Of primary importance is object
 * lookup based on ID (SET, STAT, others).  Allocation (GET),
 * deallocation (RMID), ID enumeration (IDS), and key lookups (GET) are
 * lesser concerns, but should be implemented in such a way that ID
 * lookup isn't affected (at least not in the common case).
 *
 * Starting from the bottom up, each object is represented by a
 * structure, the first member of which must be a kipc_perm_t.  The
 * kipc_perm_t contains the information described above in "Objects", a
 * reference count (since the object may continue to exist after it has
 * been removed from the namespace), as well as some additional
 * metadata used to manage data structure membership.  These objects
 * are dynamically allocated.
 *
 * Above the objects is a power-of-two sized table of ID slots.  Each
 * slot contains a pointer to an object, a sequence number, and a
 * lock.  An object's ID is a function of its slot's index in the table
 * and its slot's sequence number.  Every time a slot is released (via
 * RMID) its sequence number is increased.  Strictly speaking, the
 * sequence number is unnecessary.  However, checking the sequence
 * number after a lookup provides a certain degree of robustness
 * against the use of stale IDs (useful since nothing else does).  When
 * the table fills up, it is resized (see Locking, below).
 *
 * Of an ID's 31 bits (an ID is, as defined by the standards, a signed
 * int) the top IPC_SEQ_BITS are used for the sequence number with the
 * remainder holding the index into the table.  The size of the table
 * is therefore bounded at 2 ^ (31 - IPC_SEQ_BITS) slots.
 *
 * Managing this table is the ipc_service structure.  It contains a
 * pointer to the dynamically allocated ID table, a namespace-global
 * lock, an id_space for managing the free space in the table, and
 * sundry other metadata necessary for the maintenance of the
 * namespace.  An AVL tree of all keyed objects in the table (sorted by
 * key) is used for key lookups.  An unordered doubly linked list of
 * all objects in the namespace (keyed or not) is maintained to
 * facilitate ID enumeration.
 *
 * To help visualize these relationships, here's a picture of a
 * namespace with a table of size 8 containing three objects
 * (IPC_SEQ_BITS = 28):
 *
 *
 * +-ipc_service_t--+
 * | table          *---\
 * | keys           *---+----------------------\
 * | all ids        *--\|                      |
 * |                |  ||                      |
 * +----------------+  ||                      |
 *                     ||                      |
 * /-------------------/|                      |
 * |    /---------------/                      |
 * |    |                                      |
 * |    v                                      |
 * |  +-0------+-1------+-2------+-3------+-4--+---+-5------+-6------+-7------+
 * |  | Seq=3  |        |        | Seq=1  |    :   |        |        | Seq=6  |
 * |  |        |        |        |        |    :   |        |        |        |
 * |  +-*------+--------+--------+-*------+----+---+--------+--------+-*------+
 * |    |                          |           |                       |
 * |    |                      /---/           |      /----------------/
 * |    |                      |               |      |
 * |    v                      v               |      v
 * |  +-kipc_perm_t-+        +-kipc_perm_t-+   |    +-kipc_perm_t-+
 * |  | id=0x30     |        | id=0x13     |   |    | id=0x67     |
 * |  | key=0xfeed  |        | key=0xbeef  |   |    | key=0xcafe  |
 * \->| [list]      |<------>| [list]      |<------>| [list]      |
 * /->| [avl left]  x   /--->| [avl left]  x   \--->| [avl left]  *---\
 * |  | [avl right] x   |    | [avl right] x        | [avl right] *---+-\
 * |  |             |   |    |             |        |             |   | |
 * |  +-------------+   |    +-------------+        +-------------+   | |
 * |                    \---------------------------------------------/ |
 * \--------------------------------------------------------------------/
 *
 * Locking
 * -------
 *
 * There are three locks (or sets of locks) which are used to ensure
 * correctness: the slot locks, the namespace lock, and p_lock (needed
 * when checking resource controls).  Their ordering is
 *
 *   namespace lock -> slot lock 0 -> ... -> slot lock t -> p_lock
 *
 * Generally speaking, the namespace lock is used to protect allocation
 * and removal from the namespace, ID enumeration, and resizing the ID
 * table.  Specifically:
 *
 * - write access to all fields of the ipc_service structure
 * - read access to all variable fields of ipc_service except
 *   ipcs_tabsz (table size) and ipcs_table (the table pointer)
 * - read/write access to ipc_avl, ipc_list in visible objects'
 *   kipc_perm structures (i.e. objects which have been removed from
 *   the namespace don't have this restriction)
 * - write access to ipct_seq and ipct_data in the table entries
 *
 * A slot lock by itself is meaningless (except when resizing).  Of
 * greater interest conceptually is the notion of an ID lock -- a
 * "virtual lock" which refers to whichever slot lock an object's ID
 * currently hashes to.
 *
 * An ID lock protects all objects with that ID.  Normally there will
 * only be one such object: the one pointed to by the locked slot.
 * However, if an object is removed from the namespace but retains
 * references (e.g. an attached shared memory segment which has been
 * RMIDed), it continues to use the lock associated with its original
 * ID.  While this can result in increased contention, operations which
 * require taking the ID lock of removed objects are infrequent.
 *
 * Specifically, an ID lock protects the contents of an object's
 * structure, including the contents of the embedded kipc_perm
 * structure (but excluding those fields protected by the namespace
 * lock).  It also protects the ipct_seq and ipct_data fields in its
 * slot (it is really a slot lock, after all).
 *
 * Recall that the table is resizable.  To avoid requiring every ID
 * lookup to take a global lock, a scheme much like that employed for
 * file descriptors (see the comment above UF_ENTER in user.h) is
 * used.  Note that the sequence number and data pointer are protected
 * by both the namespace lock and their slot lock.  When the table is
 * resized, the following operations take place:
 *
 *   1) A new table is allocated.
 *   2) The global lock is taken.
 *   3) All old slots are locked, in order.
 *   4) The first half of the new slots are locked.
 *   5) All table entries are copied to the new table, and cleared from
 *	the old table.
 *   6) The ipc_service structure is updated to point to the new table.
 *   7) The ipc_service structure is updated with the new table size.
 *   8) All slot locks (old and new) are dropped.
 *
 * Because the slot locks are embedded in the table, ID lookups and
 * other operations which require taking an slot lock need to verify
 * that the lock taken wasn't part of a stale table.  This is
 * accomplished by checking the table size before and after
 * dereferencing the table pointer and taking the lock: if the size
 * changes, the lock must be dropped and reacquired.  It is this
 * additional work which distinguishes an ID lock from a slot lock.
 *
 * Because we can't guarantee that threads aren't accessing the old
 * tables' locks, they are never deallocated.  To prevent spurious
 * reports of memory leaks, a pointer to the discarded table is stored
 * in the new one in step 5.  (Theoretically ipcs_destroy will delete
 * the discarded tables, but it is only ever called from a failed _init
 * invocation; i.e. when there aren't any.)
 *
 * Interfaces
 * ----------
 *
 * The following interfaces are provided by the ipc module for use by
 * the individual IPC facilities:
 *
 * ipcperm_access
 *
 *   Given an object and a cred structure, determines if the requested
 *   access type is allowed.
 *
 * ipcperm_set, ipcperm_stat,
 * ipcperm_set64, ipcperm_stat64
 *
 *   Performs the common portion of an STAT or SET operation.  All
 *   (except stat and stat64) can fail, so they should be called before
 *   any facility-specific non-reversible changes are made to an
 *   object.  Similarly, the set operations have side effects, so they
 *   should only be called once the possibility of a facility-specific
 *   failure is eliminated.
 *
 * ipcs_create
 *
 *   Creates an IPC namespace for use by an IPC facility.
 *
 * ipcs_destroy
 *
 *   Destroys an IPC namespace.
 *
 * ipcs_lock, ipcs_unlock
 *
 *   Takes the namespace lock.  Ideally such access wouldn't be
 *   necessary, but there may be facility-specific data protected by
 *   this lock (e.g. project-wide resource consumption).
 *
 * ipc_lock
 *
 *   Takes the lock associated with an ID.  Can't fail.
 *
 * ipc_relock
 *
 *   Like ipc_lock, but takes a pointer to a held lock.  Drops the lock
 *   unless it is the one that would have been returned by ipc_lock.
 *   Used after calls to cv_wait.
 *
 * ipc_lookup
 *
 *   Performs an ID lookup, returns with the ID lock held.  Fails if
 *   the ID doesn't exist in the namespace.
 *
 * ipc_hold
 *
 *   Takes a reference on an object.
 *
 * ipc_rele
 *
 *   Releases a reference on an object, and drops the object's lock.
 *   Calls the object's destructor if last reference is being
 *   released.
 *
 * ipc_rele_locked
 *
 *   Releases a reference on an object.  Doesn't drop lock, and may
 *   only be called when there is more than one reference to the
 *   object.
 *
 * ipc_get, ipc_commit_begin, ipc_commit_end, ipc_cleanup
 *
 *   Components of a GET operation.  ipc_get performs a key lookup,
 *   allocating an object if the key isn't found (returning with the
 *   namespace lock and p_lock held), and returning the existing object
 *   if it is (with the object lock held).  ipc_get doesn't modify the
 *   namespace.
 *
 *   ipc_commit_begin begins the process of inserting an object
 *   allocated by ipc_get into the namespace, and can fail.  If
 *   successful, it returns with the namespace lock and p_lock held.
 *   ipc_commit_end completes the process of inserting an object into
 *   the namespace and can't fail.  The facility can call ipc_cleanup
 *   at any time following a successful ipc_get and before
 *   ipc_commit_end or a failed ipc_commit_begin to fail the
 *   allocation.  Pseudocode for the suggested GET implementation:
 *
 *   top:
 *
 *     ipc_get
 *
 *     if failure
 *       return
 *
 *     if found {
 *
 *	 if object meets criteria
 *	   unlock object and return success
 *       else
 *	   unlock object and return failure
 *
 *     } else {
 *
 *	 perform resource control tests
 *	 drop namespace lock, p_lock
 *	 if failure
 *	   ipc_cleanup
 *
 *       perform facility-specific initialization
 *	 if failure {
 *	   facility-specific cleanup
 *	   ipc_cleanup
 *       }
 *
 *	 ( At this point the object should be destructible using the
 *	   destructor given to ipcs_create )
 *
 *       ipc_commit_begin
 *	 if retry
 *	   goto top
 *       else if failure
 *         return
 *
 *       perform facility-specific resource control tests/allocations
 *	 if failure
 *	   ipc_cleanup
 *
 *	 ipc_commit_end
 *	 perform any infallible post-creation actions, unlock, and return
 *
 *     }
 *
 * ipc_rmid
 *
 *   Performs the common portion of an RMID operation -- looks up an ID
 *   removes it, and calls the a facility-specific function to do
 *   RMID-time cleanup on the private portions of the object.
 *
 * ipc_ids
 *
 *   Performs the common portion of an IDS operation.
 *
 */

#include <sys/types.h>
#include <sys/param.h>
#include <sys/cred.h>
#include <sys/policy.h>
#include <sys/proc.h>
#include <sys/user.h>
#include <sys/ipc.h>
#include <sys/ipc_impl.h>
#include <sys/errno.h>
#include <sys/systm.h>
#include <sys/list.h>
#include <sys/atomic.h>
#include <sys/zone.h>
#include <sys/task.h>
#include <sys/modctl.h>

#include <c2/audit.h>

static struct modlmisc modlmisc = {
	&mod_miscops,
	"common ipc code",
};

static struct modlinkage modlinkage = {
	MODREV_1, (void *)&modlmisc, NULL
};


int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}


/*
 * Check message, semaphore, or shared memory access permissions.
 *
 * This routine verifies the requested access permission for the current
 * process.  The zone ids are compared, and the appropriate bits are
 * checked corresponding to owner, group (including the list of
 * supplementary groups), or everyone.  Zero is returned on success.
 * On failure, the security policy is asked to check to override the
 * permissions check; the policy will either return 0 for access granted
 * or EACCES.
 *
 * Access to objects in other zones requires that the caller be in the
 * global zone and have the appropriate IPC_DAC_* privilege, regardless
 * of whether the uid or gid match those of the object.  Note that
 * cross-zone accesses will normally never get here since they'll
 * fail in ipc_lookup or ipc_get.
 *
 * The arguments must be set up as follows:
 *	p - Pointer to permission structure to verify
 *	mode - Desired access permissions
 */
int
ipcperm_access(kipc_perm_t *p, int mode, cred_t *cr)
{
	int shifts = 0;
	uid_t uid = crgetuid(cr);
	zoneid_t zoneid = getzoneid();

	if (p->ipc_zoneid == zoneid) {
		if (uid != p->ipc_uid && uid != p->ipc_cuid) {
			shifts += 3;
			if (!groupmember(p->ipc_gid, cr) &&
			    !groupmember(p->ipc_cgid, cr))
				shifts += 3;
		}

		mode &= ~(p->ipc_mode << shifts);

		if (mode == 0)
			return (0);
	} else if (zoneid != GLOBAL_ZONEID)
		return (EACCES);

	return (secpolicy_ipc_access(cr, p, mode));
}

/*
 * There are two versions of the ipcperm_set/stat functions:
 *   ipcperm_???        - for use with IPC_SET/STAT
 *   ipcperm_???_64     - for use with IPC_SET64/STAT64
 *
 * These functions encapsulate the common portions (copying, permission
 * checks, and auditing) of the set/stat operations.  All, except for
 * stat and stat_64 which are void, return 0 on success or a non-zero
 * errno value on error.
 */

int
ipcperm_set(ipc_service_t *service, struct cred *cr,
    kipc_perm_t *kperm, struct ipc_perm *perm, model_t model)
{
	STRUCT_HANDLE(ipc_perm, lperm);
	uid_t uid;
	gid_t gid;
	mode_t mode;
	zone_t *zone;

	ASSERT(IPC_LOCKED(service, kperm));

	STRUCT_SET_HANDLE(lperm, model, perm);
	uid = STRUCT_FGET(lperm, uid);
	gid = STRUCT_FGET(lperm, gid);
	mode = STRUCT_FGET(lperm, mode);

	if (secpolicy_ipc_owner(cr, kperm) != 0)
		return (EPERM);

	zone = crgetzone(cr);
	if (!VALID_UID(uid, zone) || !VALID_GID(gid, zone))
		return (EINVAL);

	kperm->ipc_uid = uid;
	kperm->ipc_gid = gid;
	kperm->ipc_mode = (mode & 0777) | (kperm->ipc_mode & ~0777);

	if (AU_AUDITING())
		audit_ipcget(service->ipcs_atype, kperm);

	return (0);
}

void
ipcperm_stat(struct ipc_perm *perm, kipc_perm_t *kperm, model_t model)
{
	STRUCT_HANDLE(ipc_perm, lperm);

	STRUCT_SET_HANDLE(lperm, model, perm);
	STRUCT_FSET(lperm, uid, kperm->ipc_uid);
	STRUCT_FSET(lperm, gid, kperm->ipc_gid);
	STRUCT_FSET(lperm, cuid, kperm->ipc_cuid);
	STRUCT_FSET(lperm, cgid, kperm->ipc_cgid);
	STRUCT_FSET(lperm, mode, kperm->ipc_mode);
	STRUCT_FSET(lperm, seq, 0);
	STRUCT_FSET(lperm, key, kperm->ipc_key);
}

int
ipcperm_set64(ipc_service_t *service, struct cred *cr,
    kipc_perm_t *kperm, ipc_perm64_t *perm64)
{
	zone_t *zone;

	ASSERT(IPC_LOCKED(service, kperm));

	if (secpolicy_ipc_owner(cr, kperm) != 0)
		return (EPERM);

	zone = crgetzone(cr);
	if (!VALID_UID(perm64->ipcx_uid, zone) ||
	    !VALID_GID(perm64->ipcx_gid, zone))
		return (EINVAL);

	kperm->ipc_uid = perm64->ipcx_uid;
	kperm->ipc_gid = perm64->ipcx_gid;
	kperm->ipc_mode = (perm64->ipcx_mode & 0777) |
	    (kperm->ipc_mode & ~0777);

	if (AU_AUDITING())
		audit_ipcget(service->ipcs_atype, kperm);

	return (0);
}

void
ipcperm_stat64(ipc_perm64_t *perm64, kipc_perm_t *kperm)
{
	perm64->ipcx_uid = kperm->ipc_uid;
	perm64->ipcx_gid = kperm->ipc_gid;
	perm64->ipcx_cuid = kperm->ipc_cuid;
	perm64->ipcx_cgid = kperm->ipc_cgid;
	perm64->ipcx_mode = kperm->ipc_mode;
	perm64->ipcx_key = kperm->ipc_key;
	perm64->ipcx_projid = kperm->ipc_proj->kpj_id;
	perm64->ipcx_zoneid = kperm->ipc_zoneid;
}


/*
 * ipc key comparator.
 */
static int
ipc_key_compar(const void *a, const void *b)
{
	kipc_perm_t *aperm = (kipc_perm_t *)a;
	kipc_perm_t *bperm = (kipc_perm_t *)b;
	int ak = aperm->ipc_key;
	int bk = bperm->ipc_key;
	zoneid_t az;
	zoneid_t bz;

	ASSERT(ak != IPC_PRIVATE);
	ASSERT(bk != IPC_PRIVATE);

	/*
	 * Compare key first, then zoneid.  This optimizes performance for
	 * systems with only one zone, since the zone checks will only be
	 * made when the keys match.
	 */
	if (ak < bk)
		return (-1);
	if (ak > bk)
		return (1);

	/* keys match */
	az = aperm->ipc_zoneid;
	bz = bperm->ipc_zoneid;
	if (az < bz)
		return (-1);
	if (az > bz)
		return (1);
	return (0);
}

/*
 * Create an ipc service.
 */
ipc_service_t *
ipcs_create(const char *name, rctl_hndl_t proj_rctl, rctl_hndl_t zone_rctl,
    size_t size, ipc_func_t *dtor, ipc_func_t *rmid, int audit_type,
    size_t rctl_offset)
{
	ipc_service_t *result;

	result = kmem_alloc(sizeof (ipc_service_t), KM_SLEEP);

	mutex_init(&result->ipcs_lock, NULL, MUTEX_ADAPTIVE, NULL);
	result->ipcs_count = 0;
	avl_create(&result->ipcs_keys, ipc_key_compar, size, 0);
	result->ipcs_tabsz = IPC_IDS_MIN;
	result->ipcs_table =
	    kmem_zalloc(IPC_IDS_MIN * sizeof (ipc_slot_t), KM_SLEEP);
	result->ipcs_ssize = size;
	result->ipcs_ids = id_space_create(name, 0, IPC_IDS_MIN);
	result->ipcs_dtor = dtor;
	result->ipcs_rmid = rmid;
	result->ipcs_proj_rctl = proj_rctl;
	result->ipcs_zone_rctl = zone_rctl;
	result->ipcs_atype = audit_type;
	ASSERT(rctl_offset < sizeof (ipc_rqty_t));
	result->ipcs_rctlofs = rctl_offset;
	list_create(&result->ipcs_usedids, sizeof (kipc_perm_t),
	    offsetof(kipc_perm_t, ipc_list));

	return (result);
}

/*
 * Destroy an ipc service.
 */
void
ipcs_destroy(ipc_service_t *service)
{
	ipc_slot_t *slot, *next;

	mutex_enter(&service->ipcs_lock);

	ASSERT(service->ipcs_count == 0);
	avl_destroy(&service->ipcs_keys);
	list_destroy(&service->ipcs_usedids);
	id_space_destroy(service->ipcs_ids);

	for (slot = service->ipcs_table; slot; slot = next) {
		next = slot[0].ipct_chain;
		kmem_free(slot, service->ipcs_tabsz * sizeof (ipc_slot_t));
		service->ipcs_tabsz >>= 1;
	}

	mutex_destroy(&service->ipcs_lock);
	kmem_free(service, sizeof (ipc_service_t));
}

/*
 * Takes the service lock.
 */
void
ipcs_lock(ipc_service_t *service)
{
	mutex_enter(&service->ipcs_lock);
}

/*
 * Releases the service lock.
 */
void
ipcs_unlock(ipc_service_t *service)
{
	mutex_exit(&service->ipcs_lock);
}


/*
 * Locks the specified ID.  Returns the ID's ID table index.
 */
static int
ipc_lock_internal(ipc_service_t *service, uint_t id)
{
	uint_t	tabsz;
	uint_t	index;
	kmutex_t *mutex;

	for (;;) {
		tabsz = service->ipcs_tabsz;
		membar_consumer();
		index = id & (tabsz - 1);
		mutex = &service->ipcs_table[index].ipct_lock;
		mutex_enter(mutex);
		if (tabsz == service->ipcs_tabsz)
			break;
		mutex_exit(mutex);
	}

	return (index);
}

/*
 * Locks the specified ID.  Returns a pointer to the ID's lock.
 */
kmutex_t *
ipc_lock(ipc_service_t *service, int id)
{
	uint_t index;

	/*
	 * These assertions don't reflect requirements of the code
	 * which follows, but they should never fail nonetheless.
	 */
	ASSERT(id >= 0);
	ASSERT(IPC_INDEX(id) < service->ipcs_tabsz);
	index = ipc_lock_internal(service, id);

	return (&service->ipcs_table[index].ipct_lock);
}

/*
 * Checks to see if the held lock provided is the current lock for the
 * specified id.  If so, we return it instead of dropping it and
 * returning the result of ipc_lock.  This is intended to speed up cv
 * wakeups where we are left holding a lock which could be stale, but
 * probably isn't.
 */
kmutex_t *
ipc_relock(ipc_service_t *service, int id, kmutex_t *lock)
{
	ASSERT(id >= 0);
	ASSERT(IPC_INDEX(id) < service->ipcs_tabsz);
	ASSERT(MUTEX_HELD(lock));

	if (&service->ipcs_table[IPC_INDEX(id)].ipct_lock == lock)
		return (lock);

	mutex_exit(lock);
	return (ipc_lock(service, id));
}

/*
 * Performs an ID lookup.  If the ID doesn't exist or has been removed,
 * or isn't visible to the caller (because of zones), NULL is returned.
 * Otherwise, a pointer to the ID's perm structure and held ID lock are
 * returned.
 */
kmutex_t *
ipc_lookup(ipc_service_t *service, int id, kipc_perm_t **perm)
{
	kipc_perm_t *result;
	uint_t index;

	/*
	 * There is no need to check to see if id is in-range (i.e.
	 * positive and fits into the table).  If it is out-of-range,
	 * the id simply won't match the object's.
	 */

	index = ipc_lock_internal(service, id);
	result = service->ipcs_table[index].ipct_data;
	if (result == NULL || result->ipc_id != (uint_t)id ||
	    !HASZONEACCESS(curproc, result->ipc_zoneid)) {
		mutex_exit(&service->ipcs_table[index].ipct_lock);
		return (NULL);
	}

	ASSERT(IPC_SEQ(id) == service->ipcs_table[index].ipct_seq);

	*perm = result;
	if (AU_AUDITING())
		audit_ipc(service->ipcs_atype, id, result);

	return (&service->ipcs_table[index].ipct_lock);
}

/*
 * Increase the reference count on an ID.
 */
/*ARGSUSED*/
void
ipc_hold(ipc_service_t *s, kipc_perm_t *perm)
{
	ASSERT(IPC_INDEX(perm->ipc_id) < s->ipcs_tabsz);
	ASSERT(IPC_LOCKED(s, perm));
	perm->ipc_ref++;
}

/*
 * Decrease the reference count on an ID and drops the ID's lock.
 * Destroys the ID if the new reference count is zero.
 */
void
ipc_rele(ipc_service_t *s, kipc_perm_t *perm)
{
	int nref;

	ASSERT(IPC_INDEX(perm->ipc_id) < s->ipcs_tabsz);
	ASSERT(IPC_LOCKED(s, perm));
	ASSERT(perm->ipc_ref > 0);

	nref = --perm->ipc_ref;
	mutex_exit(&s->ipcs_table[IPC_INDEX(perm->ipc_id)].ipct_lock);

	if (nref == 0) {
		ASSERT(IPC_FREE(perm));		/* ipc_rmid clears IPC_ALLOC */
		s->ipcs_dtor(perm);
		project_rele(perm->ipc_proj);
		zone_rele_ref(&perm->ipc_zone_ref, ZONE_REF_IPC);
		kmem_free(perm, s->ipcs_ssize);
	}
}

/*
 * Decrease the reference count on an ID, but don't drop the ID lock.
 * Used in cases where one thread needs to remove many references (on
 * behalf of other parties).
 */
void
ipc_rele_locked(ipc_service_t *s, kipc_perm_t *perm)
{
	ASSERT(perm->ipc_ref > 1);
	ASSERT(IPC_INDEX(perm->ipc_id) < s->ipcs_tabsz);
	ASSERT(IPC_LOCKED(s, perm));

	perm->ipc_ref--;
}


/*
 * Internal function to grow the service ID table.
 */
static int
ipc_grow(ipc_service_t *service)
{
	ipc_slot_t *new, *old;
	int i, oldsize, newsize;

	ASSERT(MUTEX_HELD(&service->ipcs_lock));
	ASSERT(MUTEX_NOT_HELD(&curproc->p_lock));

	if (service->ipcs_tabsz == IPC_IDS_MAX)
		return (ENOSPC);

	oldsize = service->ipcs_tabsz;
	newsize = oldsize << 1;
	new = kmem_zalloc(newsize * sizeof (ipc_slot_t), KM_NOSLEEP);
	if (new == NULL)
		return (ENOSPC);

	old = service->ipcs_table;
	for (i = 0; i < oldsize; i++) {
		mutex_enter(&old[i].ipct_lock);
		mutex_enter(&new[i].ipct_lock);

		new[i].ipct_seq = old[i].ipct_seq;
		new[i].ipct_data = old[i].ipct_data;
		old[i].ipct_data = NULL;
	}

	new[0].ipct_chain = old;
	service->ipcs_table = new;
	membar_producer();
	service->ipcs_tabsz = newsize;

	for (i = 0; i < oldsize; i++) {
		mutex_exit(&old[i].ipct_lock);
		mutex_exit(&new[i].ipct_lock);
	}

	id_space_extend(service->ipcs_ids, oldsize, service->ipcs_tabsz);

	return (0);
}


static int
ipc_keylookup(ipc_service_t *service, key_t key, int flag, kipc_perm_t **permp)
{
	kipc_perm_t *perm = NULL;
	avl_index_t where;
	kipc_perm_t template;

	ASSERT(MUTEX_HELD(&service->ipcs_lock));

	template.ipc_key = key;
	template.ipc_zoneid = getzoneid();
	if (perm = avl_find(&service->ipcs_keys, &template, &where)) {
		ASSERT(!IPC_FREE(perm));
		if ((flag & (IPC_CREAT | IPC_EXCL)) == (IPC_CREAT | IPC_EXCL))
			return (EEXIST);
		if ((flag & 0777) & ~perm->ipc_mode) {
			if (AU_AUDITING())
				audit_ipcget(0, (void *)perm);
			return (EACCES);
		}
		*permp = perm;
		return (0);
	} else if (flag & IPC_CREAT) {
		*permp = NULL;
		return (0);
	}
	return (ENOENT);
}

static int
ipc_alloc_test(ipc_service_t *service, proc_t *pp)
{
	ASSERT(MUTEX_HELD(&service->ipcs_lock));

	/*
	 * Resizing the table first would result in a cleaner code
	 * path, but would also allow a user to (permanently) double
	 * the id table size in cases where the allocation would be
	 * denied.  Hence we test the rctl first.
	 */
retry:
	mutex_enter(&pp->p_lock);
	if ((rctl_test(service->ipcs_proj_rctl, pp->p_task->tk_proj->kpj_rctls,
	    pp, 1, RCA_SAFE) & RCT_DENY) ||
	    (rctl_test(service->ipcs_zone_rctl, pp->p_zone->zone_rctls,
	    pp, 1, RCA_SAFE) & RCT_DENY)) {
		mutex_exit(&pp->p_lock);
		return (ENOSPC);
	}

	if (service->ipcs_count == service->ipcs_tabsz) {
		int error;

		mutex_exit(&pp->p_lock);
		if (error = ipc_grow(service))
			return (error);
		goto retry;
	}

	return (0);
}

/*
 * Given a key, search for or create the associated identifier.
 *
 * If IPC_CREAT is specified and the key isn't found, or if the key is
 * equal to IPC_PRIVATE, we return 0 and place a pointer to a newly
 * allocated object structure in permp.  A pointer to the held service
 * lock is placed in lockp.  ipc_mode's IPC_ALLOC bit is clear.
 *
 * If the key is found and no error conditions arise, we return 0 and
 * place a pointer to the existing object structure in permp.  A
 * pointer to the held ID lock is placed in lockp.  ipc_mode's
 * IPC_ALLOC bit is set.
 *
 * Otherwise, a non-zero errno value is returned.
 */
int
ipc_get(ipc_service_t *service, key_t key, int flag, kipc_perm_t **permp,
    kmutex_t **lockp)
{
	kipc_perm_t	*perm = NULL;
	proc_t		*pp = curproc;
	int		error, index;
	cred_t		*cr = CRED();

	if (key != IPC_PRIVATE) {

		mutex_enter(&service->ipcs_lock);
		error = ipc_keylookup(service, key, flag, &perm);
		if (perm != NULL)
			index = ipc_lock_internal(service, perm->ipc_id);
		mutex_exit(&service->ipcs_lock);

		if (error) {
			ASSERT(perm == NULL);
			return (error);
		}

		if (perm) {
			ASSERT(!IPC_FREE(perm));
			*permp = perm;
			*lockp = &service->ipcs_table[index].ipct_lock;
			return (0);
		}

		/* Key not found; fall through */
	}

	perm = kmem_zalloc(service->ipcs_ssize, KM_SLEEP);

	mutex_enter(&service->ipcs_lock);
	if (error = ipc_alloc_test(service, pp)) {
		mutex_exit(&service->ipcs_lock);
		kmem_free(perm, service->ipcs_ssize);
		return (error);
	}

	perm->ipc_cuid = perm->ipc_uid = crgetuid(cr);
	perm->ipc_cgid = perm->ipc_gid = crgetgid(cr);
	perm->ipc_zoneid = getzoneid();
	perm->ipc_mode = flag & 0777;
	perm->ipc_key = key;
	perm->ipc_ref = 1;
	perm->ipc_id = IPC_ID_INVAL;
	*permp = perm;
	*lockp = &service->ipcs_lock;

	return (0);
}

/*
 * Attempts to add the a newly created ID to the global namespace.  If
 * creating it would cause an error, we return the error.  If there is
 * the possibility that we could obtain the existing ID and return it
 * to the user, we return EAGAIN.  Otherwise, we return 0 with p_lock
 * and the service lock held.
 *
 * Since this should be only called after all initialization has been
 * completed, on failure we automatically invoke the destructor for the
 * object and deallocate the memory associated with it.
 */
int
ipc_commit_begin(ipc_service_t *service, key_t key, int flag,
    kipc_perm_t *newperm)
{
	kipc_perm_t *perm;
	int error;
	proc_t *pp = curproc;

	ASSERT(newperm->ipc_ref == 1);
	ASSERT(IPC_FREE(newperm));

	/*
	 * Set ipc_proj and ipc_zone_ref so that future calls to ipc_cleanup()
	 * clean up the necessary state.  This must be done before the
	 * potential call to ipcs_dtor() below.
	 */
	newperm->ipc_proj = pp->p_task->tk_proj;
	zone_init_ref(&newperm->ipc_zone_ref);
	zone_hold_ref(pp->p_zone, &newperm->ipc_zone_ref, ZONE_REF_IPC);

	mutex_enter(&service->ipcs_lock);
	/*
	 * Ensure that no-one has raced with us and created the key.
	 */
	if ((key != IPC_PRIVATE) &&
	    (((error = ipc_keylookup(service, key, flag, &perm)) != 0) ||
	    (perm != NULL))) {
		error = error ? error : EAGAIN;
		goto errout;
	}

	/*
	 * Ensure that no-one has raced with us and used the last of
	 * the permissible ids, or the last of the free spaces in the
	 * id table.
	 */
	if (error = ipc_alloc_test(service, pp))
		goto errout;

	ASSERT(MUTEX_HELD(&service->ipcs_lock));
	ASSERT(MUTEX_HELD(&pp->p_lock));

	return (0);
errout:
	mutex_exit(&service->ipcs_lock);
	service->ipcs_dtor(newperm);
	zone_rele_ref(&newperm->ipc_zone_ref, ZONE_REF_IPC);
	kmem_free(newperm, service->ipcs_ssize);
	return (error);
}

/*
 * Commit the ID allocation transaction.  Called with p_lock and the
 * service lock held, both of which are dropped.  Returns the held ID
 * lock so the caller can extract the ID and perform ipcget auditing.
 */
kmutex_t *
ipc_commit_end(ipc_service_t *service, kipc_perm_t *perm)
{
	ipc_slot_t *slot;
	avl_index_t where;
	int index;
	void *loc;

	ASSERT(MUTEX_HELD(&service->ipcs_lock));
	ASSERT(MUTEX_HELD(&curproc->p_lock));

	(void) project_hold(perm->ipc_proj);
	mutex_exit(&curproc->p_lock);

	/*
	 * Pick out our slot.
	 */
	service->ipcs_count++;
	index = id_alloc(service->ipcs_ids);
	ASSERT(index < service->ipcs_tabsz);
	slot = &service->ipcs_table[index];
	mutex_enter(&slot->ipct_lock);
	ASSERT(slot->ipct_data == NULL);

	/*
	 * Update the perm structure.
	 */
	perm->ipc_mode |= IPC_ALLOC;
	perm->ipc_id = (slot->ipct_seq << IPC_SEQ_SHIFT) | index;

	/*
	 * Push into global visibility.
	 */
	slot->ipct_data = perm;
	if (perm->ipc_key != IPC_PRIVATE) {
		loc = avl_find(&service->ipcs_keys, perm, &where);
		ASSERT(loc == NULL);
		avl_insert(&service->ipcs_keys, perm, where);
	}
	list_insert_head(&service->ipcs_usedids, perm);

	/*
	 * Update resource consumption.
	 */
	IPC_PROJ_USAGE(perm, service) += 1;
	IPC_ZONE_USAGE(perm, service) += 1;

	mutex_exit(&service->ipcs_lock);
	return (&slot->ipct_lock);
}

/*
 * Clean up function, in case the allocation fails.  If called between
 * ipc_lookup and ipc_commit_begin, perm->ipc_proj will be 0 and we
 * merely free the perm structure.  If called after ipc_commit_begin,
 * we also drop locks and call the ID's destructor.
 */
void
ipc_cleanup(ipc_service_t *service, kipc_perm_t *perm)
{
	ASSERT(IPC_FREE(perm));
	if (perm->ipc_proj) {
		mutex_exit(&curproc->p_lock);
		mutex_exit(&service->ipcs_lock);
		service->ipcs_dtor(perm);
	}
	if (perm->ipc_zone_ref.zref_zone != NULL)
		zone_rele_ref(&perm->ipc_zone_ref, ZONE_REF_IPC);
	kmem_free(perm, service->ipcs_ssize);
}


/*
 * Common code to remove an IPC object.  This should be called after
 * all permissions checks have been performed, and with the service
 * and ID locked.  Note that this does not remove the object from
 * the ipcs_usedids list (this needs to be done by the caller before
 * dropping the service lock).
 */
static void
ipc_remove(ipc_service_t *service, kipc_perm_t *perm)
{
	int id = perm->ipc_id;
	int index;

	ASSERT(MUTEX_HELD(&service->ipcs_lock));
	ASSERT(IPC_LOCKED(service, perm));

	index = IPC_INDEX(id);

	service->ipcs_table[index].ipct_data = NULL;

	if (perm->ipc_key != IPC_PRIVATE)
		avl_remove(&service->ipcs_keys, perm);
	list_remove(&service->ipcs_usedids, perm);
	perm->ipc_mode &= ~IPC_ALLOC;

	id_free(service->ipcs_ids, index);

	if (service->ipcs_table[index].ipct_seq++ == IPC_SEQ_MASK)
		service->ipcs_table[index].ipct_seq = 0;
	service->ipcs_count--;
	ASSERT(IPC_PROJ_USAGE(perm, service) > 0);
	ASSERT(IPC_ZONE_USAGE(perm, service) > 0);
	IPC_PROJ_USAGE(perm, service) -= 1;
	IPC_ZONE_USAGE(perm, service) -= 1;
	ASSERT(service->ipcs_count || ((IPC_PROJ_USAGE(perm, service) == 0) &&
	    (IPC_ZONE_USAGE(perm, service) == 0)));
}


/*
 * Common code to perform an IPC_RMID.  Returns an errno value on
 * failure, 0 on success.
 */
int
ipc_rmid(ipc_service_t *service, int id, cred_t *cr)
{
	kipc_perm_t *perm;
	kmutex_t *lock;

	mutex_enter(&service->ipcs_lock);

	lock = ipc_lookup(service, id, &perm);
	if (lock == NULL) {
		mutex_exit(&service->ipcs_lock);
		return (EINVAL);
	}

	ASSERT(service->ipcs_count > 0);

	if (secpolicy_ipc_owner(cr, perm) != 0) {
		mutex_exit(lock);
		mutex_exit(&service->ipcs_lock);
		return (EPERM);
	}

	/*
	 * Nothing can fail from this point on.
	 */
	ipc_remove(service, perm);
	mutex_exit(&service->ipcs_lock);

	/* perform any per-service removal actions */
	service->ipcs_rmid(perm);

	ipc_rele(service, perm);

	return (0);
}

/*
 * Implementation for shmids, semids, and msgids.  buf is the address
 * of the user buffer, nids is the size, and pnids is a pointer to
 * where we write the actual number of ids that [would] have been
 * copied out.
 */
int
ipc_ids(ipc_service_t *service, int *buf, uint_t nids, uint_t *pnids)
{
	kipc_perm_t *perm;
	size_t	idsize = 0;
	int	error = 0;
	int	idcount;
	int	*ids;
	int	numids = 0;
	zoneid_t zoneid = getzoneid();
	int	global = INGLOBALZONE(curproc);

	if (buf == NULL)
		nids = 0;

	/*
	 * Get an accurate count of the total number of ids, and allocate a
	 * staging buffer.  Since ipcs_count is always sane, we don't have
	 * to take ipcs_lock for our first guess.  If there are no ids, or
	 * we're in the global zone and the number of ids is greater than
	 * the size of the specified buffer, we shunt to the end.  Otherwise,
	 * we go through the id list looking for (and counting) what is
	 * visible in the specified zone.
	 */
	idcount = service->ipcs_count;
	for (;;) {
		if ((global && idcount > nids) || idcount == 0) {
			numids = idcount;
			nids = 0;
			goto out;
		}

		idsize = idcount * sizeof (int);
		ids = kmem_alloc(idsize, KM_SLEEP);

		mutex_enter(&service->ipcs_lock);
		if (idcount >= service->ipcs_count)
			break;
		idcount = service->ipcs_count;
		mutex_exit(&service->ipcs_lock);

		if (idsize != 0) {
			kmem_free(ids, idsize);
			idsize = 0;
		}
	}

	for (perm = list_head(&service->ipcs_usedids); perm != NULL;
	    perm = list_next(&service->ipcs_usedids, perm)) {
		ASSERT(!IPC_FREE(perm));
		if (global || perm->ipc_zoneid == zoneid)
			ids[numids++] = perm->ipc_id;
	}
	mutex_exit(&service->ipcs_lock);

	/*
	 * If there isn't enough space to hold all of the ids, just
	 * return the number of ids without copying out any of them.
	 */
	if (nids < numids)
		nids = 0;

out:
	if (suword32(pnids, (uint32_t)numids) ||
	    (nids != 0 && copyout(ids, buf, numids * sizeof (int))))
		error = EFAULT;
	if (idsize != 0)
		kmem_free(ids, idsize);
	return (error);
}

/*
 * Destroy IPC objects from the given service that are associated with
 * the given zone.
 *
 * We can't hold on to the service lock when freeing objects, so we
 * first search the service and move all the objects to a private
 * list, then walk through and free them after dropping the lock.
 */
void
ipc_remove_zone(ipc_service_t *service, zoneid_t zoneid)
{
	kipc_perm_t *perm, *next;
	list_t rmlist;
	kmutex_t *lock;

	list_create(&rmlist, sizeof (kipc_perm_t),
	    offsetof(kipc_perm_t, ipc_list));

	mutex_enter(&service->ipcs_lock);
	for (perm = list_head(&service->ipcs_usedids); perm != NULL;
	    perm = next) {
		next = list_next(&service->ipcs_usedids, perm);
		if (perm->ipc_zoneid != zoneid)
			continue;

		/*
		 * Remove the object from the service, then put it on
		 * the removal list so we can defer the call to
		 * ipc_rele (which will actually free the structure).
		 * We need to do this since the destructor may grab
		 * the service lock.
		 */
		ASSERT(!IPC_FREE(perm));
		lock = ipc_lock(service, perm->ipc_id);
		ipc_remove(service, perm);
		mutex_exit(lock);
		list_insert_tail(&rmlist, perm);
	}
	mutex_exit(&service->ipcs_lock);

	/*
	 * Now that we've dropped the service lock, loop through the
	 * private list freeing removed objects.
	 */
	for (perm = list_head(&rmlist); perm != NULL; perm = next) {
		next = list_next(&rmlist, perm);
		list_remove(&rmlist, perm);

		(void) ipc_lock(service, perm->ipc_id);

		/* perform any per-service removal actions */
		service->ipcs_rmid(perm);

		/* release reference */
		ipc_rele(service, perm);
	}

	list_destroy(&rmlist);
}