/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2019 Joyent, Inc.
 * Copyright (c) 2016 by Delphix. All rights reserved.
 */

#include <sys/param.h>
#include <sys/user.h>
#include <sys/mman.h>
#include <sys/kmem.h>
#include <sys/sysmacros.h>
#include <sys/cmn_err.h>
#include <sys/systm.h>
#include <sys/tuneable.h>
#include <vm/hat.h>
#include <vm/seg.h>
#include <vm/as.h>
#include <vm/anon.h>
#include <vm/page.h>
#include <sys/buf.h>
#include <sys/swap.h>
#include <sys/atomic.h>
#include <vm/seg_spt.h>
#include <sys/debug.h>
#include <sys/vtrace.h>
#include <sys/shm.h>
#include <sys/shm_impl.h>
#include <sys/lgrp.h>
#include <sys/vmsystm.h>
#include <sys/policy.h>
#include <sys/project.h>
#include <sys/zone.h>

#define	SEGSPTADDR	(caddr_t)0x0

/*
 * # pages used for spt
 */
size_t	spt_used;

/*
 * See spt_setminfree().
 */
pgcnt_t segspt_minfree = 0;
size_t segspt_minfree_clamp = (1UL << 30); /* 1GB in bytes */

static int segspt_create(struct seg **segpp, void *argsp);
static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize);
static void segspt_free(struct seg *seg);
static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len);
static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr);

/* ARGSUSED */
__NORETURN static int
segspt_badop_dup(struct seg *seg __unused, struct seg *newseg __unused)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static faultcode_t
segspt_badop_fault(struct hat *hat, struct seg *seg, caddr_t addr,
    size_t len, enum fault_type type, enum seg_rw rw)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static faultcode_t
segspt_badop_faulta(struct seg *seg __unused, caddr_t addr __unused)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_prot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static size_t
segspt_badop_swapout(struct seg *seg)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_sync(struct seg *seg, caddr_t addr, size_t len, int attr,
    uint_t flags)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN
static size_t
segspt_badop_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_lockop(struct seg *seg, caddr_t addr, size_t len, int attr,
    int op, ulong_t *lockmap, size_t pos)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static u_offset_t
segspt_badop_getoffset(struct seg *seg, caddr_t addr)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_gettype(struct seg *seg, caddr_t addr)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static void
segspt_badop_dump(struct seg *seg)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_pagelock(struct seg *seg, caddr_t addr, size_t len,
    struct page ***ppp, enum lock_type type, enum seg_rw rw)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_setpgsz(struct seg *seg, caddr_t addr, size_t len, uint_t szc)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
{
	panic("%s called", __func__);
}

/* ARGSUSED */
__NORETURN static int
segspt_badop_capable(struct seg *seg, segcapability_t capability)
{
	panic("%s called", __func__);
}

struct seg_ops segspt_ops = {
	segspt_badop_dup,		/* dup */
	segspt_unmap,
	segspt_free,
	segspt_badop_fault,		/* fault */
	segspt_badop_faulta,		/* faulta */
	segspt_badop_prot,		/* setprot */
	segspt_badop_checkprot,		/* checkprot */
	segspt_badop_kluster,		/* kluster */
	segspt_badop_swapout,		/* swapout */
	segspt_badop_sync,		/* sync */
	segspt_badop_incore,		/* incore */
	segspt_badop_lockop,		/* lockop */
	segspt_badop_getprot,		/* getprot */
	segspt_badop_getoffset,		/* getoffset */
	segspt_badop_gettype,		/* gettype */
	segspt_badop_getvp,		/* getvp */
	segspt_badop_advise,		/* advise */
	segspt_badop_dump,		/* dump */
	segspt_badop_pagelock,		/* pagelock */
	segspt_badop_setpgsz,		/* setpgsz */
	segspt_badop_getmemid,		/* getmemid */
	segspt_getpolicy,		/* getpolicy */
	segspt_badop_capable,		/* capable */
	seg_inherit_notsup		/* inherit */
};

static int segspt_shmdup(struct seg *seg, struct seg *newseg);
static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize);
static void segspt_shmfree(struct seg *seg);
static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg,
		caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw);
static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr);
static int segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len,
		uint_t prot);
static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size,
		uint_t prot);
static int	segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta);
static size_t	segspt_shmswapout(struct seg *seg);
static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len,
		char *vec);
static int segspt_shmsync(struct seg *seg, caddr_t addr, size_t len,
		int attr, uint_t flags);
static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
		int attr, int op, ulong_t *lockmap, size_t pos);
static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len,
		uint_t *protv);
static u_offset_t segspt_shmgetoffset(struct seg *seg, caddr_t addr);
static int segspt_shmgettype(struct seg *seg, caddr_t addr);
static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len,
		uint_t behav);
static void segspt_shmdump(struct seg *seg);
static int segspt_shmpagelock(struct seg *, caddr_t, size_t,
		struct page ***, enum lock_type, enum seg_rw);
static int segspt_shmsetpgsz(struct seg *, caddr_t, size_t, uint_t);
static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *);
static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t);
static int segspt_shmcapable(struct seg *, segcapability_t);

struct seg_ops segspt_shmops = {
	segspt_shmdup,
	segspt_shmunmap,
	segspt_shmfree,
	segspt_shmfault,
	segspt_shmfaulta,
	segspt_shmsetprot,
	segspt_shmcheckprot,
	segspt_shmkluster,
	segspt_shmswapout,
	segspt_shmsync,
	segspt_shmincore,
	segspt_shmlockop,
	segspt_shmgetprot,
	segspt_shmgetoffset,
	segspt_shmgettype,
	segspt_shmgetvp,
	segspt_shmadvise,	/* advise */
	segspt_shmdump,
	segspt_shmpagelock,
	segspt_shmsetpgsz,
	segspt_shmgetmemid,
	segspt_shmgetpolicy,
	segspt_shmcapable,
	seg_inherit_notsup
};

static void segspt_purge(struct seg *seg);
static int segspt_reclaim(void *, caddr_t, size_t, struct page **,
		enum seg_rw, int);
static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len,
		page_t **ppa);

/*
 * This value corresponds to headroom in availrmem that ISM can never allocate
 * (but others can).  The original intent here was to prevent ISM from locking
 * all of the remaining availrmem into memory, making forward progress
 * difficult. It's not clear how much this matters on modern systems.
 *
 * The traditional default value of 5% of total memory is used, except on
 * systems where that quickly gets ridiculous: in that case we clamp at a rather
 * arbitrary value of 1GB.
 *
 * Note that since this is called lazily on the first sptcreate(), in theory,
 * this could represent a very small value if the system is heavily loaded
 * already. In practice, the first ISM user is pretty likely to come along
 * earlier during the system's operation.
 *
 * This never gets re-figured.
 */
static void
spt_setminfree(void)
{
	segspt_minfree = availrmem / 20;

	if (segspt_minfree_clamp != 0 &&
	    segspt_minfree > (segspt_minfree_clamp / PAGESIZE))
		segspt_minfree = segspt_minfree_clamp / PAGESIZE;
}

int
sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp,
    uint_t prot, uint_t flags, uint_t share_szc)
{
	int	err;
	struct	as	*newas;
	struct	segspt_crargs sptcargs;

	if (segspt_minfree == 0)
		spt_setminfree();

	if (!hat_supported(HAT_SHARED_PT, (void *)0))
		return (EINVAL);

	/*
	 * get a new as for this shared memory segment
	 */
	newas = as_alloc();
	newas->a_proc = NULL;
	sptcargs.amp = amp;
	sptcargs.prot = prot;
	sptcargs.flags = flags;
	sptcargs.szc = share_szc;
	/*
	 * create a shared page table (spt) segment
	 */

	if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) {
		as_free(newas);
		return (err);
	}
	*sptseg = sptcargs.seg_spt;
	return (0);
}

void
sptdestroy(struct as *as, struct anon_map *amp)
{

	(void) as_unmap(as, SEGSPTADDR, amp->size);
	as_free(as);
}

/*
 * called from seg_free().
 * free (i.e., unlock, unmap, return to free list)
 *  all the pages in the given seg.
 */
void
segspt_free(struct seg	*seg)
{
	struct spt_data *sptd = (struct spt_data *)seg->s_data;

	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));

	if (sptd != NULL) {
		if (sptd->spt_realsize)
			segspt_free_pages(seg, seg->s_base, sptd->spt_realsize);

		if (sptd->spt_ppa_lckcnt) {
			kmem_free(sptd->spt_ppa_lckcnt,
			    sizeof (*sptd->spt_ppa_lckcnt)
			    * btopr(sptd->spt_amp->size));
		}
		kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp));
		cv_destroy(&sptd->spt_cv);
		mutex_destroy(&sptd->spt_lock);
		kmem_free(sptd, sizeof (*sptd));
	}
}

/*ARGSUSED*/
static int
segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr,
    uint_t flags)
{
	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));

	return (0);
}

/*ARGSUSED*/
static size_t
segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec)
{
	caddr_t	eo_seg;
	pgcnt_t	npages;
	struct shm_data *shmd = (struct shm_data *)seg->s_data;
	struct seg	*sptseg;
	struct spt_data *sptd;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
#ifdef lint
	seg = seg;
#endif
	sptseg = shmd->shm_sptseg;
	sptd = sptseg->s_data;

	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
		eo_seg = addr + len;
		while (addr < eo_seg) {
			/* page exists, and it's locked. */
			*vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED |
			    SEG_PAGE_ANON;
			addr += PAGESIZE;
		}
		return (len);
	} else {
		struct  anon_map *amp = shmd->shm_amp;
		struct  anon	*ap;
		page_t		*pp;
		pgcnt_t		anon_index;
		struct vnode	*vp;
		u_offset_t	off;
		ulong_t		i;
		int		ret;
		anon_sync_obj_t	cookie;

		addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
		anon_index = seg_page(seg, addr);
		npages = btopr(len);
		if (anon_index + npages > btopr(shmd->shm_amp->size)) {
			return (EINVAL);
		}
		ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
		for (i = 0; i < npages; i++, anon_index++) {
			ret = 0;
			anon_array_enter(amp, anon_index, &cookie);
			ap = anon_get_ptr(amp->ahp, anon_index);
			if (ap != NULL) {
				swap_xlate(ap, &vp, &off);
				anon_array_exit(&cookie);
				pp = page_lookup_nowait(vp, off, SE_SHARED);
				if (pp != NULL) {
					ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON;
					page_unlock(pp);
				}
			} else {
				anon_array_exit(&cookie);
			}
			if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
				ret |= SEG_PAGE_LOCKED;
			}
			*vec++ = (char)ret;
		}
		ANON_LOCK_EXIT(&amp->a_rwlock);
		return (len);
	}
}

static int
segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize)
{
	size_t share_size;

	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));

	/*
	 * seg.s_size may have been rounded up to the largest page size
	 * in shmat().
	 * XXX This should be cleanedup. sptdestroy should take a length
	 * argument which should be the same as sptcreate. Then
	 * this rounding would not be needed (or is done in shm.c)
	 * Only the check for full segment will be needed.
	 *
	 * XXX -- shouldn't raddr == 0 always? These tests don't seem
	 * to be useful at all.
	 */
	share_size = page_get_pagesize(seg->s_szc);
	ssize = P2ROUNDUP(ssize, share_size);

	if (raddr == seg->s_base && ssize == seg->s_size) {
		seg_free(seg);
		return (0);
	} else
		return (EINVAL);
}

int
segspt_create(struct seg **segpp, void *argsp)
{
	struct seg	*seg = *segpp;
	int		err;
	caddr_t		addr = seg->s_base;
	struct spt_data *sptd;
	struct segspt_crargs *sptcargs = (struct segspt_crargs *)argsp;
	struct anon_map *amp = sptcargs->amp;
	struct kshmid	*sp = amp->a_sp;
	struct	cred	*cred = CRED();
	ulong_t		i, j, anon_index = 0;
	pgcnt_t		npages = btopr(amp->size);
	struct vnode	*vp;
	page_t		**ppa;
	uint_t		hat_flags;
	size_t		pgsz;
	pgcnt_t		pgcnt;
	caddr_t		a;
	pgcnt_t		pidx;
	size_t		sz;
	proc_t		*procp = curproc;
	rctl_qty_t	lockedbytes = 0;
	kproject_t	*proj;

	/*
	 * We are holding the a_lock on the underlying dummy as,
	 * so we can make calls to the HAT layer.
	 */
	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
	ASSERT(sp != NULL);

	if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
		if (err = anon_swap_adjust(npages))
			return (err);
	}
	err = ENOMEM;

	if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL)
		goto out1;

	ppa = NULL;
	if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
		if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages),
		    KM_NOSLEEP)) == NULL)
			goto out2;
	}

	mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL);

	if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL)
		goto out3;

	seg->s_ops = &segspt_ops;
	sptd->spt_vp = vp;
	sptd->spt_amp = amp;
	sptd->spt_prot = sptcargs->prot;
	sptd->spt_flags = sptcargs->flags;
	seg->s_data = (caddr_t)sptd;
	sptd->spt_ppa = NULL;
	sptd->spt_ppa_lckcnt = NULL;
	seg->s_szc = sptcargs->szc;
	cv_init(&sptd->spt_cv, NULL, CV_DEFAULT, NULL);
	sptd->spt_gen = 0;

	ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
	if (seg->s_szc > amp->a_szc) {
		amp->a_szc = seg->s_szc;
	}
	ANON_LOCK_EXIT(&amp->a_rwlock);

	/*
	 * Set policy to affect initial allocation of pages in
	 * anon_map_createpages()
	 */
	(void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index,
	    NULL, 0, ptob(npages));

	if (sptcargs->flags & SHM_PAGEABLE) {
		size_t  share_sz;
		pgcnt_t new_npgs, more_pgs;
		struct anon_hdr *nahp;
		zone_t *zone;

		share_sz = page_get_pagesize(seg->s_szc);
		if (!IS_P2ALIGNED(amp->size, share_sz)) {
			/*
			 * We are rounding up the size of the anon array
			 * on 4 M boundary because we always create 4 M
			 * of page(s) when locking, faulting pages and we
			 * don't have to check for all corner cases e.g.
			 * if there is enough space to allocate 4 M
			 * page.
			 */
			new_npgs = btop(P2ROUNDUP(amp->size, share_sz));
			more_pgs = new_npgs - npages;

			/*
			 * The zone will never be NULL, as a fully created
			 * shm always has an owning zone.
			 */
			zone = sp->shm_perm.ipc_zone_ref.zref_zone;
			ASSERT(zone != NULL);
			if (anon_resv_zone(ptob(more_pgs), zone) == 0) {
				err = ENOMEM;
				goto out4;
			}

			nahp = anon_create(new_npgs, ANON_SLEEP);
			ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
			(void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages,
			    ANON_SLEEP);
			anon_release(amp->ahp, npages);
			amp->ahp = nahp;
			ASSERT(amp->swresv == ptob(npages));
			amp->swresv = amp->size = ptob(new_npgs);
			ANON_LOCK_EXIT(&amp->a_rwlock);
			npages = new_npgs;
		}

		sptd->spt_ppa_lckcnt = kmem_zalloc(npages *
		    sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP);
		sptd->spt_pcachecnt = 0;
		sptd->spt_realsize = ptob(npages);
		sptcargs->seg_spt = seg;
		return (0);
	}

	/*
	 * get array of pages for each anon slot in amp
	 */
	if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa,
	    seg, addr, S_CREATE, cred)) != 0)
		goto out4;

	mutex_enter(&sp->shm_mlock);

	/* May be partially locked, so, count bytes to charge for locking */
	for (i = 0; i < npages; i++)
		if (ppa[i]->p_lckcnt == 0)
			lockedbytes += PAGESIZE;

	proj = sp->shm_perm.ipc_proj;

	if (lockedbytes > 0) {
		mutex_enter(&procp->p_lock);
		if (rctl_incr_locked_mem(procp, proj, lockedbytes, 0)) {
			mutex_exit(&procp->p_lock);
			mutex_exit(&sp->shm_mlock);
			for (i = 0; i < npages; i++)
				page_unlock(ppa[i]);
			err = ENOMEM;
			goto out4;
		}
		mutex_exit(&procp->p_lock);
	}

	/*
	 * addr is initial address corresponding to the first page on ppa list
	 */
	for (i = 0; i < npages; i++) {
		/* attempt to lock all pages */
		if (page_pp_lock(ppa[i], 0, 1) == 0) {
			/*
			 * if unable to lock any page, unlock all
			 * of them and return error
			 */
			for (j = 0; j < i; j++)
				page_pp_unlock(ppa[j], 0, 1);
			for (i = 0; i < npages; i++)
				page_unlock(ppa[i]);
			rctl_decr_locked_mem(NULL, proj, lockedbytes, 0);
			mutex_exit(&sp->shm_mlock);
			err = ENOMEM;
			goto out4;
		}
	}
	mutex_exit(&sp->shm_mlock);

	/*
	 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
	 * for the entire life of the segment. For example platforms
	 * that do not support Dynamic Reconfiguration.
	 */
	hat_flags = HAT_LOAD_SHARE;
	if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL))
		hat_flags |= HAT_LOAD_LOCK;

	/*
	 * Load translations one lare page at a time
	 * to make sure we don't create mappings bigger than
	 * segment's size code in case underlying pages
	 * are shared with segvn's segment that uses bigger
	 * size code than we do.
	 */
	pgsz = page_get_pagesize(seg->s_szc);
	pgcnt = page_get_pagecnt(seg->s_szc);
	for (a = addr, pidx = 0; pidx < npages; a += pgsz, pidx += pgcnt) {
		sz = MIN(pgsz, ptob(npages - pidx));
		hat_memload_array(seg->s_as->a_hat, a, sz,
		    &ppa[pidx], sptd->spt_prot, hat_flags);
	}

	/*
	 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
	 * we will leave the pages locked SE_SHARED for the life
	 * of the ISM segment. This will prevent any calls to
	 * hat_pageunload() on this ISM segment for those platforms.
	 */
	if (!(hat_flags & HAT_LOAD_LOCK)) {
		/*
		 * On platforms that support HAT_DYNAMIC_ISM_UNMAP,
		 * we no longer need to hold the SE_SHARED lock on the pages,
		 * since L_PAGELOCK and F_SOFTLOCK calls will grab the
		 * SE_SHARED lock on the pages as necessary.
		 */
		for (i = 0; i < npages; i++)
			page_unlock(ppa[i]);
	}
	sptd->spt_pcachecnt = 0;
	kmem_free(ppa, ((sizeof (page_t *)) * npages));
	sptd->spt_realsize = ptob(npages);
	atomic_add_long(&spt_used, npages);
	sptcargs->seg_spt = seg;
	return (0);

out4:
	seg->s_data = NULL;
	kmem_free(vp, sizeof (*vp));
	cv_destroy(&sptd->spt_cv);
out3:
	mutex_destroy(&sptd->spt_lock);
	if ((sptcargs->flags & SHM_PAGEABLE) == 0)
		kmem_free(ppa, (sizeof (*ppa) * npages));
out2:
	kmem_free(sptd, sizeof (*sptd));
out1:
	if ((sptcargs->flags & SHM_PAGEABLE) == 0)
		anon_swap_restore(npages);
	return (err);
}

/*ARGSUSED*/
void
segspt_free_pages(struct seg *seg, caddr_t addr, size_t len)
{
	struct page	*pp;
	struct spt_data *sptd = (struct spt_data *)seg->s_data;
	pgcnt_t		npages;
	ulong_t		anon_idx;
	struct anon_map *amp;
	struct anon	*ap;
	struct vnode	*vp;
	u_offset_t	off;
	uint_t		hat_flags;
	int		root = 0;
	pgcnt_t		pgs, curnpgs = 0;
	page_t		*rootpp;
	rctl_qty_t	unlocked_bytes = 0;
	kproject_t	*proj;
	kshmid_t	*sp;

	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));

	len = P2ROUNDUP(len, PAGESIZE);

	npages = btop(len);

	hat_flags = HAT_UNLOAD_UNLOCK | HAT_UNLOAD_UNMAP;
	if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) ||
	    (sptd->spt_flags & SHM_PAGEABLE)) {
		hat_flags = HAT_UNLOAD_UNMAP;
	}

	hat_unload(seg->s_as->a_hat, addr, len, hat_flags);

	amp = sptd->spt_amp;
	if (sptd->spt_flags & SHM_PAGEABLE)
		npages = btop(amp->size);

	ASSERT(amp != NULL);

	proj = NULL;
	rootpp = NULL;
	sp = NULL;
	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
		sp = amp->a_sp;
		proj = sp->shm_perm.ipc_proj;
		mutex_enter(&sp->shm_mlock);
	}
	for (anon_idx = 0; anon_idx < npages; anon_idx++) {
		if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
			if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) {
				panic("segspt_free_pages: null app");
				/*NOTREACHED*/
			}
		} else {
			if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx))
			    == NULL)
				continue;
		}
		ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0);
		swap_xlate(ap, &vp, &off);

		/*
		 * If this platform supports HAT_DYNAMIC_ISM_UNMAP,
		 * the pages won't be having SE_SHARED lock at this
		 * point.
		 *
		 * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
		 * the pages are still held SE_SHARED locked from the
		 * original segspt_create()
		 *
		 * Our goal is to get SE_EXCL lock on each page, remove
		 * permanent lock on it and invalidate the page.
		 */
		if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
			if (hat_flags == HAT_UNLOAD_UNMAP)
				pp = page_lookup(vp, off, SE_EXCL);
			else {
				if ((pp = page_find(vp, off)) == NULL) {
					panic("segspt_free_pages: "
					    "page not locked");
					/*NOTREACHED*/
				}
				if (!page_tryupgrade(pp)) {
					page_unlock(pp);
					pp = page_lookup(vp, off, SE_EXCL);
				}
			}
			if (pp == NULL) {
				panic("segspt_free_pages: "
				    "page not in the system");
				/*NOTREACHED*/
			}
			ASSERT(pp->p_lckcnt > 0);
			page_pp_unlock(pp, 0, 1);
			if (pp->p_lckcnt == 0)
				unlocked_bytes += PAGESIZE;
		} else {
			if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL)
				continue;
		}
		/*
		 * It's logical to invalidate the pages here as in most cases
		 * these were created by segspt.
		 */
		if (pp->p_szc != 0) {
			if (root == 0) {
				ASSERT(curnpgs == 0);
				root = 1;
				rootpp = pp;
				pgs = curnpgs = page_get_pagecnt(pp->p_szc);
				ASSERT(pgs > 1);
				ASSERT(IS_P2ALIGNED(pgs, pgs));
				ASSERT(!(page_pptonum(pp) & (pgs - 1)));
				curnpgs--;
			} else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) {
				ASSERT(curnpgs == 1);
				ASSERT(page_pptonum(pp) ==
				    page_pptonum(rootpp) + (pgs - 1));
				page_destroy_pages(rootpp);
				root = 0;
				curnpgs = 0;
			} else {
				ASSERT(curnpgs > 1);
				ASSERT(page_pptonum(pp) ==
				    page_pptonum(rootpp) + (pgs - curnpgs));
				curnpgs--;
			}
		} else {
			if (root != 0 || curnpgs != 0) {
				panic("segspt_free_pages: bad large page");
				/*NOTREACHED*/
			}
			/*
			 * Before destroying the pages, we need to take care
			 * of the rctl locked memory accounting. For that
			 * we need to calculte the unlocked_bytes.
			 */
			if (pp->p_lckcnt > 0)
				unlocked_bytes += PAGESIZE;
			/*LINTED: constant in conditional context */
			VN_DISPOSE(pp, B_INVAL, 0, kcred);
		}
	}
	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
		if (unlocked_bytes > 0)
			rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0);
		mutex_exit(&sp->shm_mlock);
	}
	if (root != 0 || curnpgs != 0) {
		panic("segspt_free_pages: bad large page");
		/*NOTREACHED*/
	}

	/*
	 * mark that pages have been released
	 */
	sptd->spt_realsize = 0;

	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
		atomic_add_long(&spt_used, -npages);
		anon_swap_restore(npages);
	}
}

/*
 * Get memory allocation policy info for specified address in given segment
 */
static lgrp_mem_policy_info_t *
segspt_getpolicy(struct seg *seg, caddr_t addr)
{
	struct anon_map		*amp;
	ulong_t			anon_index;
	lgrp_mem_policy_info_t	*policy_info;
	struct spt_data		*spt_data;

	ASSERT(seg != NULL);

	/*
	 * Get anon_map from segspt
	 *
	 * Assume that no lock needs to be held on anon_map, since
	 * it should be protected by its reference count which must be
	 * nonzero for an existing segment
	 * Need to grab readers lock on policy tree though
	 */
	spt_data = (struct spt_data *)seg->s_data;
	if (spt_data == NULL)
		return (NULL);
	amp = spt_data->spt_amp;
	ASSERT(amp->refcnt != 0);

	/*
	 * Get policy info
	 *
	 * Assume starting anon index of 0
	 */
	anon_index = seg_page(seg, addr);
	policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);

	return (policy_info);
}

/*
 * DISM only.
 * Return locked pages over a given range.
 *
 * We will cache all DISM locked pages and save the pplist for the
 * entire segment in the ppa field of the underlying DISM segment structure.
 * Later, during a call to segspt_reclaim() we will use this ppa array
 * to page_unlock() all of the pages and then we will free this ppa list.
 */
/*ARGSUSED*/
static int
segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len,
    struct page ***ppp, enum lock_type type, enum seg_rw rw)
{
	struct  shm_data *shmd = (struct shm_data *)seg->s_data;
	struct  seg	*sptseg = shmd->shm_sptseg;
	struct  spt_data *sptd = sptseg->s_data;
	pgcnt_t pg_idx, npages, tot_npages, npgs;
	struct  page **pplist, **pl, **ppa, *pp;
	struct  anon_map *amp;
	spgcnt_t	an_idx;
	int	ret = ENOTSUP;
	uint_t	pl_built = 0;
	struct  anon *ap;
	struct  vnode *vp;
	u_offset_t off;
	pgcnt_t claim_availrmem = 0;
	uint_t	szc;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
	ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK);

	/*
	 * We want to lock/unlock the entire ISM segment. Therefore,
	 * we will be using the underlying sptseg and it's base address
	 * and length for the caching arguments.
	 */
	ASSERT(sptseg);
	ASSERT(sptd);

	pg_idx = seg_page(seg, addr);
	npages = btopr(len);

	/*
	 * check if the request is larger than number of pages covered
	 * by amp
	 */
	if (pg_idx + npages > btopr(sptd->spt_amp->size)) {
		*ppp = NULL;
		return (ENOTSUP);
	}

	if (type == L_PAGEUNLOCK) {
		ASSERT(sptd->spt_ppa != NULL);

		seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size,
		    sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);

		/*
		 * If someone is blocked while unmapping, we purge
		 * segment page cache and thus reclaim pplist synchronously
		 * without waiting for seg_pasync_thread. This speeds up
		 * unmapping in cases where munmap(2) is called, while
		 * raw async i/o is still in progress or where a thread
		 * exits on data fault in a multithreaded application.
		 */
		if ((sptd->spt_flags & DISM_PPA_CHANGED) ||
		    (AS_ISUNMAPWAIT(seg->s_as) &&
		    shmd->shm_softlockcnt > 0)) {
			segspt_purge(seg);
		}
		return (0);
	}

	/* The L_PAGELOCK case ... */

	if (sptd->spt_flags & DISM_PPA_CHANGED) {
		segspt_purge(seg);
		/*
		 * for DISM ppa needs to be rebuild since
		 * number of locked pages could be changed
		 */
		*ppp = NULL;
		return (ENOTSUP);
	}

	/*
	 * First try to find pages in segment page cache, without
	 * holding the segment lock.
	 */
	pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
	    S_WRITE, SEGP_FORCE_WIRED);
	if (pplist != NULL) {
		ASSERT(sptd->spt_ppa != NULL);
		ASSERT(sptd->spt_ppa == pplist);
		ppa = sptd->spt_ppa;
		for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
			if (ppa[an_idx] == NULL) {
				seg_pinactive(seg, NULL, seg->s_base,
				    sptd->spt_amp->size, ppa,
				    S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
				*ppp = NULL;
				return (ENOTSUP);
			}
			if ((szc = ppa[an_idx]->p_szc) != 0) {
				npgs = page_get_pagecnt(szc);
				an_idx = P2ROUNDUP(an_idx + 1, npgs);
			} else {
				an_idx++;
			}
		}
		/*
		 * Since we cache the entire DISM segment, we want to
		 * set ppp to point to the first slot that corresponds
		 * to the requested addr, i.e. pg_idx.
		 */
		*ppp = &(sptd->spt_ppa[pg_idx]);
		return (0);
	}

	mutex_enter(&sptd->spt_lock);
	/*
	 * try to find pages in segment page cache with mutex
	 */
	pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
	    S_WRITE, SEGP_FORCE_WIRED);
	if (pplist != NULL) {
		ASSERT(sptd->spt_ppa != NULL);
		ASSERT(sptd->spt_ppa == pplist);
		ppa = sptd->spt_ppa;
		for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
			if (ppa[an_idx] == NULL) {
				mutex_exit(&sptd->spt_lock);
				seg_pinactive(seg, NULL, seg->s_base,
				    sptd->spt_amp->size, ppa,
				    S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
				*ppp = NULL;
				return (ENOTSUP);
			}
			if ((szc = ppa[an_idx]->p_szc) != 0) {
				npgs = page_get_pagecnt(szc);
				an_idx = P2ROUNDUP(an_idx + 1, npgs);
			} else {
				an_idx++;
			}
		}
		/*
		 * Since we cache the entire DISM segment, we want to
		 * set ppp to point to the first slot that corresponds
		 * to the requested addr, i.e. pg_idx.
		 */
		mutex_exit(&sptd->spt_lock);
		*ppp = &(sptd->spt_ppa[pg_idx]);
		return (0);
	}
	if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size,
	    SEGP_FORCE_WIRED) == SEGP_FAIL) {
		mutex_exit(&sptd->spt_lock);
		*ppp = NULL;
		return (ENOTSUP);
	}

	/*
	 * No need to worry about protections because DISM pages are always rw.
	 */
	pl = pplist = NULL;
	amp = sptd->spt_amp;

	/*
	 * Do we need to build the ppa array?
	 */
	if (sptd->spt_ppa == NULL) {
		pgcnt_t lpg_cnt = 0;

		pl_built = 1;
		tot_npages = btopr(sptd->spt_amp->size);

		ASSERT(sptd->spt_pcachecnt == 0);
		pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP);
		pl = pplist;

		ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
		for (an_idx = 0; an_idx < tot_npages; ) {
			ap = anon_get_ptr(amp->ahp, an_idx);
			/*
			 * Cache only mlocked pages. For large pages
			 * if one (constituent) page is mlocked
			 * all pages for that large page
			 * are cached also. This is for quick
			 * lookups of ppa array;
			 */
			if ((ap != NULL) && (lpg_cnt != 0 ||
			    (sptd->spt_ppa_lckcnt[an_idx] != 0))) {

				swap_xlate(ap, &vp, &off);
				pp = page_lookup(vp, off, SE_SHARED);
				ASSERT(pp != NULL);
				if (lpg_cnt == 0) {
					lpg_cnt++;
					/*
					 * For a small page, we are done --
					 * lpg_count is reset to 0 below.
					 *
					 * For a large page, we are guaranteed
					 * to find the anon structures of all
					 * constituent pages and a non-zero
					 * lpg_cnt ensures that we don't test
					 * for mlock for these. We are done
					 * when lpg_count reaches (npgs + 1).
					 * If we are not the first constituent
					 * page, restart at the first one.
					 */
					npgs = page_get_pagecnt(pp->p_szc);
					if (!IS_P2ALIGNED(an_idx, npgs)) {
						an_idx = P2ALIGN(an_idx, npgs);
						page_unlock(pp);
						continue;
					}
				}
				if (++lpg_cnt > npgs)
					lpg_cnt = 0;

				/*
				 * availrmem is decremented only
				 * for unlocked pages
				 */
				if (sptd->spt_ppa_lckcnt[an_idx] == 0)
					claim_availrmem++;
				pplist[an_idx] = pp;
			}
			an_idx++;
		}
		ANON_LOCK_EXIT(&amp->a_rwlock);

		if (claim_availrmem) {
			mutex_enter(&freemem_lock);
			if (availrmem < tune.t_minarmem + claim_availrmem) {
				mutex_exit(&freemem_lock);
				ret = ENOTSUP;
				claim_availrmem = 0;
				goto insert_fail;
			} else {
				availrmem -= claim_availrmem;
			}
			mutex_exit(&freemem_lock);
		}

		sptd->spt_ppa = pl;
	} else {
		/*
		 * We already have a valid ppa[].
		 */
		pl = sptd->spt_ppa;
	}

	ASSERT(pl != NULL);

	ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size,
	    sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED,
	    segspt_reclaim);
	if (ret == SEGP_FAIL) {
		/*
		 * seg_pinsert failed. We return
		 * ENOTSUP, so that the as_pagelock() code will
		 * then try the slower F_SOFTLOCK path.
		 */
		if (pl_built) {
			/*
			 * No one else has referenced the ppa[].
			 * We created it and we need to destroy it.
			 */
			sptd->spt_ppa = NULL;
		}
		ret = ENOTSUP;
		goto insert_fail;
	}

	/*
	 * In either case, we increment softlockcnt on the 'real' segment.
	 */
	sptd->spt_pcachecnt++;
	atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));

	ppa = sptd->spt_ppa;
	for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
		if (ppa[an_idx] == NULL) {
			mutex_exit(&sptd->spt_lock);
			seg_pinactive(seg, NULL, seg->s_base,
			    sptd->spt_amp->size,
			    pl, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
			*ppp = NULL;
			return (ENOTSUP);
		}
		if ((szc = ppa[an_idx]->p_szc) != 0) {
			npgs = page_get_pagecnt(szc);
			an_idx = P2ROUNDUP(an_idx + 1, npgs);
		} else {
			an_idx++;
		}
	}
	/*
	 * We can now drop the sptd->spt_lock since the ppa[]
	 * exists and we have incremented pacachecnt.
	 */
	mutex_exit(&sptd->spt_lock);

	/*
	 * Since we cache the entire segment, we want to
	 * set ppp to point to the first slot that corresponds
	 * to the requested addr, i.e. pg_idx.
	 */
	*ppp = &(sptd->spt_ppa[pg_idx]);
	return (0);

insert_fail:
	/*
	 * We will only reach this code if we tried and failed.
	 *
	 * And we can drop the lock on the dummy seg, once we've failed
	 * to set up a new ppa[].
	 */
	mutex_exit(&sptd->spt_lock);

	if (pl_built) {
		if (claim_availrmem) {
			mutex_enter(&freemem_lock);
			availrmem += claim_availrmem;
			mutex_exit(&freemem_lock);
		}

		/*
		 * We created pl and we need to destroy it.
		 */
		pplist = pl;
		for (an_idx = 0; an_idx < tot_npages; an_idx++) {
			if (pplist[an_idx] != NULL)
				page_unlock(pplist[an_idx]);
		}
		kmem_free(pl, sizeof (page_t *) * tot_npages);
	}

	if (shmd->shm_softlockcnt <= 0) {
		if (AS_ISUNMAPWAIT(seg->s_as)) {
			mutex_enter(&seg->s_as->a_contents);
			if (AS_ISUNMAPWAIT(seg->s_as)) {
				AS_CLRUNMAPWAIT(seg->s_as);
				cv_broadcast(&seg->s_as->a_cv);
			}
			mutex_exit(&seg->s_as->a_contents);
		}
	}
	*ppp = NULL;
	return (ret);
}



/*
 * return locked pages over a given range.
 *
 * We will cache the entire ISM segment and save the pplist for the
 * entire segment in the ppa field of the underlying ISM segment structure.
 * Later, during a call to segspt_reclaim() we will use this ppa array
 * to page_unlock() all of the pages and then we will free this ppa list.
 */
/*ARGSUSED*/
static int
segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len,
    struct page ***ppp, enum lock_type type, enum seg_rw rw)
{
	struct shm_data *shmd = (struct shm_data *)seg->s_data;
	struct seg	*sptseg = shmd->shm_sptseg;
	struct spt_data *sptd = sptseg->s_data;
	pgcnt_t np, page_index, npages;
	caddr_t a, spt_base;
	struct page **pplist, **pl, *pp;
	struct anon_map *amp;
	ulong_t anon_index;
	int ret = ENOTSUP;
	uint_t	pl_built = 0;
	struct anon *ap;
	struct vnode *vp;
	u_offset_t off;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
	ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK);


	/*
	 * We want to lock/unlock the entire ISM segment. Therefore,
	 * we will be using the underlying sptseg and it's base address
	 * and length for the caching arguments.
	 */
	ASSERT(sptseg);
	ASSERT(sptd);

	if (sptd->spt_flags & SHM_PAGEABLE) {
		return (segspt_dismpagelock(seg, addr, len, ppp, type, rw));
	}

	page_index = seg_page(seg, addr);
	npages = btopr(len);

	/*
	 * check if the request is larger than number of pages covered
	 * by amp
	 */
	if (page_index + npages > btopr(sptd->spt_amp->size)) {
		*ppp = NULL;
		return (ENOTSUP);
	}

	if (type == L_PAGEUNLOCK) {

		ASSERT(sptd->spt_ppa != NULL);

		seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size,
		    sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);

		/*
		 * If someone is blocked while unmapping, we purge
		 * segment page cache and thus reclaim pplist synchronously
		 * without waiting for seg_pasync_thread. This speeds up
		 * unmapping in cases where munmap(2) is called, while
		 * raw async i/o is still in progress or where a thread
		 * exits on data fault in a multithreaded application.
		 */
		if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) {
			segspt_purge(seg);
		}
		return (0);
	}

	/* The L_PAGELOCK case... */

	/*
	 * First try to find pages in segment page cache, without
	 * holding the segment lock.
	 */
	pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
	    S_WRITE, SEGP_FORCE_WIRED);
	if (pplist != NULL) {
		ASSERT(sptd->spt_ppa == pplist);
		ASSERT(sptd->spt_ppa[page_index]);
		/*
		 * Since we cache the entire ISM segment, we want to
		 * set ppp to point to the first slot that corresponds
		 * to the requested addr, i.e. page_index.
		 */
		*ppp = &(sptd->spt_ppa[page_index]);
		return (0);
	}

	mutex_enter(&sptd->spt_lock);

	/*
	 * try to find pages in segment page cache
	 */
	pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
	    S_WRITE, SEGP_FORCE_WIRED);
	if (pplist != NULL) {
		ASSERT(sptd->spt_ppa == pplist);
		/*
		 * Since we cache the entire segment, we want to
		 * set ppp to point to the first slot that corresponds
		 * to the requested addr, i.e. page_index.
		 */
		mutex_exit(&sptd->spt_lock);
		*ppp = &(sptd->spt_ppa[page_index]);
		return (0);
	}

	if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size,
	    SEGP_FORCE_WIRED) == SEGP_FAIL) {
		mutex_exit(&sptd->spt_lock);
		*ppp = NULL;
		return (ENOTSUP);
	}

	/*
	 * No need to worry about protections because ISM pages
	 * are always rw.
	 */
	pl = pplist = NULL;

	/*
	 * Do we need to build the ppa array?
	 */
	if (sptd->spt_ppa == NULL) {
		ASSERT(sptd->spt_ppa == pplist);

		spt_base = sptseg->s_base;
		pl_built = 1;

		/*
		 * availrmem is decremented once during anon_swap_adjust()
		 * and is incremented during the anon_unresv(), which is
		 * called from shm_rm_amp() when the segment is destroyed.
		 */
		amp = sptd->spt_amp;
		ASSERT(amp != NULL);

		/* pcachecnt is protected by sptd->spt_lock */
		ASSERT(sptd->spt_pcachecnt == 0);
		pplist = kmem_zalloc(sizeof (page_t *)
		    * btopr(sptd->spt_amp->size), KM_SLEEP);
		pl = pplist;

		anon_index = seg_page(sptseg, spt_base);

		ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
		for (a = spt_base; a < (spt_base + sptd->spt_amp->size);
		    a += PAGESIZE, anon_index++, pplist++) {
			ap = anon_get_ptr(amp->ahp, anon_index);
			ASSERT(ap != NULL);
			swap_xlate(ap, &vp, &off);
			pp = page_lookup(vp, off, SE_SHARED);
			ASSERT(pp != NULL);
			*pplist = pp;
		}
		ANON_LOCK_EXIT(&amp->a_rwlock);

		if (a < (spt_base + sptd->spt_amp->size)) {
			ret = ENOTSUP;
			goto insert_fail;
		}
		sptd->spt_ppa = pl;
	} else {
		/*
		 * We already have a valid ppa[].
		 */
		pl = sptd->spt_ppa;
	}

	ASSERT(pl != NULL);

	ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size,
	    sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED,
	    segspt_reclaim);
	if (ret == SEGP_FAIL) {
		/*
		 * seg_pinsert failed. We return
		 * ENOTSUP, so that the as_pagelock() code will
		 * then try the slower F_SOFTLOCK path.
		 */
		if (pl_built) {
			/*
			 * No one else has referenced the ppa[].
			 * We created it and we need to destroy it.
			 */
			sptd->spt_ppa = NULL;
		}
		ret = ENOTSUP;
		goto insert_fail;
	}

	/*
	 * In either case, we increment softlockcnt on the 'real' segment.
	 */
	sptd->spt_pcachecnt++;
	atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));

	/*
	 * We can now drop the sptd->spt_lock since the ppa[]
	 * exists and we have incremented pacachecnt.
	 */
	mutex_exit(&sptd->spt_lock);

	/*
	 * Since we cache the entire segment, we want to
	 * set ppp to point to the first slot that corresponds
	 * to the requested addr, i.e. page_index.
	 */
	*ppp = &(sptd->spt_ppa[page_index]);
	return (0);

insert_fail:
	/*
	 * We will only reach this code if we tried and failed.
	 *
	 * And we can drop the lock on the dummy seg, once we've failed
	 * to set up a new ppa[].
	 */
	mutex_exit(&sptd->spt_lock);

	if (pl_built) {
		/*
		 * We created pl and we need to destroy it.
		 */
		pplist = pl;
		np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT);
		while (np) {
			page_unlock(*pplist);
			np--;
			pplist++;
		}
		kmem_free(pl, sizeof (page_t *) * btopr(sptd->spt_amp->size));
	}
	if (shmd->shm_softlockcnt <= 0) {
		if (AS_ISUNMAPWAIT(seg->s_as)) {
			mutex_enter(&seg->s_as->a_contents);
			if (AS_ISUNMAPWAIT(seg->s_as)) {
				AS_CLRUNMAPWAIT(seg->s_as);
				cv_broadcast(&seg->s_as->a_cv);
			}
			mutex_exit(&seg->s_as->a_contents);
		}
	}
	*ppp = NULL;
	return (ret);
}

/*
 * purge any cached pages in the I/O page cache
 */
static void
segspt_purge(struct seg *seg)
{
	seg_ppurge(seg, NULL, SEGP_FORCE_WIRED);
}

static int
segspt_reclaim(void *ptag, caddr_t addr, size_t len, struct page **pplist,
    enum seg_rw rw, int async)
{
	struct seg *seg = (struct seg *)ptag;
	struct	shm_data *shmd = (struct shm_data *)seg->s_data;
	struct	seg	*sptseg;
	struct	spt_data *sptd;
	pgcnt_t npages, i, free_availrmem = 0;
	int	done = 0;

#ifdef lint
	addr = addr;
#endif
	sptseg = shmd->shm_sptseg;
	sptd = sptseg->s_data;
	npages = (len >> PAGESHIFT);
	ASSERT(npages);
	ASSERT(sptd->spt_pcachecnt != 0);
	ASSERT(sptd->spt_ppa == pplist);
	ASSERT(npages == btopr(sptd->spt_amp->size));
	ASSERT(async || AS_LOCK_HELD(seg->s_as));

	/*
	 * Acquire the lock on the dummy seg and destroy the
	 * ppa array IF this is the last pcachecnt.
	 */
	mutex_enter(&sptd->spt_lock);
	if (--sptd->spt_pcachecnt == 0) {
		for (i = 0; i < npages; i++) {
			if (pplist[i] == NULL) {
				continue;
			}
			if (rw == S_WRITE) {
				hat_setrefmod(pplist[i]);
			} else {
				hat_setref(pplist[i]);
			}
			if ((sptd->spt_flags & SHM_PAGEABLE) &&
			    (sptd->spt_ppa_lckcnt[i] == 0))
				free_availrmem++;
			page_unlock(pplist[i]);
		}
		if ((sptd->spt_flags & SHM_PAGEABLE) && free_availrmem) {
			mutex_enter(&freemem_lock);
			availrmem += free_availrmem;
			mutex_exit(&freemem_lock);
		}
		/*
		 * Since we want to cach/uncache the entire ISM segment,
		 * we will track the pplist in a segspt specific field
		 * ppa, that is initialized at the time we add an entry to
		 * the cache.
		 */
		ASSERT(sptd->spt_pcachecnt == 0);
		kmem_free(pplist, sizeof (page_t *) * npages);
		sptd->spt_ppa = NULL;
		sptd->spt_flags &= ~DISM_PPA_CHANGED;
		sptd->spt_gen++;
		cv_broadcast(&sptd->spt_cv);
		done = 1;
	}
	mutex_exit(&sptd->spt_lock);

	/*
	 * If we are pcache async thread or called via seg_ppurge_wiredpp() we
	 * may not hold AS lock (in this case async argument is not 0). This
	 * means if softlockcnt drops to 0 after the decrement below address
	 * space may get freed. We can't allow it since after softlock
	 * derement to 0 we still need to access as structure for possible
	 * wakeup of unmap waiters. To prevent the disappearance of as we take
	 * this segment's shm_segfree_syncmtx. segspt_shmfree() also takes
	 * this mutex as a barrier to make sure this routine completes before
	 * segment is freed.
	 *
	 * The second complication we have to deal with in async case is a
	 * possibility of missed wake up of unmap wait thread. When we don't
	 * hold as lock here we may take a_contents lock before unmap wait
	 * thread that was first to see softlockcnt was still not 0. As a
	 * result we'll fail to wake up an unmap wait thread. To avoid this
	 * race we set nounmapwait flag in as structure if we drop softlockcnt
	 * to 0 if async is not 0.  unmapwait thread
	 * will not block if this flag is set.
	 */
	if (async)
		mutex_enter(&shmd->shm_segfree_syncmtx);

	/*
	 * Now decrement softlockcnt.
	 */
	ASSERT(shmd->shm_softlockcnt > 0);
	atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));

	if (shmd->shm_softlockcnt <= 0) {
		if (async || AS_ISUNMAPWAIT(seg->s_as)) {
			mutex_enter(&seg->s_as->a_contents);
			if (async)
				AS_SETNOUNMAPWAIT(seg->s_as);
			if (AS_ISUNMAPWAIT(seg->s_as)) {
				AS_CLRUNMAPWAIT(seg->s_as);
				cv_broadcast(&seg->s_as->a_cv);
			}
			mutex_exit(&seg->s_as->a_contents);
		}
	}

	if (async)
		mutex_exit(&shmd->shm_segfree_syncmtx);

	return (done);
}

/*
 * Do a F_SOFTUNLOCK call over the range requested.
 * The range must have already been F_SOFTLOCK'ed.
 *
 * The calls to acquire and release the anon map lock mutex were
 * removed in order to avoid a deadly embrace during a DR
 * memory delete operation.  (Eg. DR blocks while waiting for a
 * exclusive lock on a page that is being used for kaio; the
 * thread that will complete the kaio and call segspt_softunlock
 * blocks on the anon map lock; another thread holding the anon
 * map lock blocks on another page lock via the segspt_shmfault
 * -> page_lookup -> page_lookup_create -> page_lock_es code flow.)
 *
 * The appropriateness of the removal is based upon the following:
 * 1. If we are holding a segment's reader lock and the page is held
 * shared, then the corresponding element in anonmap which points to
 * anon struct cannot change and there is no need to acquire the
 * anonymous map lock.
 * 2. Threads in segspt_softunlock have a reader lock on the segment
 * and already have the shared page lock, so we are guaranteed that
 * the anon map slot cannot change and therefore can call anon_get_ptr()
 * without grabbing the anonymous map lock.
 * 3. Threads that softlock a shared page break copy-on-write, even if
 * its a read.  Thus cow faults can be ignored with respect to soft
 * unlocking, since the breaking of cow means that the anon slot(s) will
 * not be shared.
 */
static void
segspt_softunlock(struct seg *seg, caddr_t sptseg_addr,
    size_t len, enum seg_rw rw)
{
	struct shm_data *shmd = (struct shm_data *)seg->s_data;
	struct seg	*sptseg;
	struct spt_data *sptd;
	page_t *pp;
	caddr_t adr;
	struct vnode *vp;
	u_offset_t offset;
	ulong_t anon_index;
	struct anon_map *amp;		/* XXX - for locknest */
	struct anon *ap = NULL;
	pgcnt_t npages;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));

	sptseg = shmd->shm_sptseg;
	sptd = sptseg->s_data;

	/*
	 * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
	 * and therefore their pages are SE_SHARED locked
	 * for the entire life of the segment.
	 */
	if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) &&
	    ((sptd->spt_flags & SHM_PAGEABLE) == 0)) {
		goto softlock_decrement;
	}

	/*
	 * Any thread is free to do a page_find and
	 * page_unlock() on the pages within this seg.
	 *
	 * We are already holding the as->a_lock on the user's
	 * real segment, but we need to hold the a_lock on the
	 * underlying dummy as. This is mostly to satisfy the
	 * underlying HAT layer.
	 */
	AS_LOCK_ENTER(sptseg->s_as, RW_READER);
	hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len);
	AS_LOCK_EXIT(sptseg->s_as);

	amp = sptd->spt_amp;
	ASSERT(amp != NULL);
	anon_index = seg_page(sptseg, sptseg_addr);

	for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) {
		ap = anon_get_ptr(amp->ahp, anon_index++);
		ASSERT(ap != NULL);
		swap_xlate(ap, &vp, &offset);

		/*
		 * Use page_find() instead of page_lookup() to
		 * find the page since we know that it has a
		 * "shared" lock.
		 */
		pp = page_find(vp, offset);
		ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1));
		if (pp == NULL) {
			panic("segspt_softunlock: "
			    "addr %p, ap %p, vp %p, off %llx",
			    (void *)adr, (void *)ap, (void *)vp, offset);
			/*NOTREACHED*/
		}

		if (rw == S_WRITE) {
			hat_setrefmod(pp);
		} else if (rw != S_OTHER) {
			hat_setref(pp);
		}
		page_unlock(pp);
	}

softlock_decrement:
	npages = btopr(len);
	ASSERT(shmd->shm_softlockcnt >= npages);
	atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages);
	if (shmd->shm_softlockcnt == 0) {
		/*
		 * All SOFTLOCKS are gone. Wakeup any waiting
		 * unmappers so they can try again to unmap.
		 * Check for waiters first without the mutex
		 * held so we don't always grab the mutex on
		 * softunlocks.
		 */
		if (AS_ISUNMAPWAIT(seg->s_as)) {
			mutex_enter(&seg->s_as->a_contents);
			if (AS_ISUNMAPWAIT(seg->s_as)) {
				AS_CLRUNMAPWAIT(seg->s_as);
				cv_broadcast(&seg->s_as->a_cv);
			}
			mutex_exit(&seg->s_as->a_contents);
		}
	}
}

int
segspt_shmattach(struct seg **segpp, void *argsp)
{
	struct seg *seg = *segpp;
	struct shm_data *shmd_arg = (struct shm_data *)argsp;
	struct shm_data *shmd;
	struct anon_map *shm_amp = shmd_arg->shm_amp;
	struct spt_data *sptd;
	int error = 0;

	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));

	shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP);
	if (shmd == NULL)
		return (ENOMEM);

	shmd->shm_sptas = shmd_arg->shm_sptas;
	shmd->shm_amp = shm_amp;
	shmd->shm_sptseg = shmd_arg->shm_sptseg;

	(void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0,
	    NULL, 0, seg->s_size);

	mutex_init(&shmd->shm_segfree_syncmtx, NULL, MUTEX_DEFAULT, NULL);

	seg->s_data = (void *)shmd;
	seg->s_ops = &segspt_shmops;
	seg->s_szc = shmd->shm_sptseg->s_szc;
	sptd = shmd->shm_sptseg->s_data;

	if (sptd->spt_flags & SHM_PAGEABLE) {
		if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size),
		    KM_NOSLEEP)) == NULL) {
			seg->s_data = (void *)NULL;
			kmem_free(shmd, (sizeof (*shmd)));
			return (ENOMEM);
		}
		shmd->shm_lckpgs = 0;
		if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
			if ((error = hat_share(seg->s_as->a_hat, seg->s_base,
			    shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
			    seg->s_size, seg->s_szc)) != 0) {
				kmem_free(shmd->shm_vpage,
				    btopr(shm_amp->size));
			}
		}
	} else {
		error = hat_share(seg->s_as->a_hat, seg->s_base,
		    shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
		    seg->s_size, seg->s_szc);
	}
	if (error) {
		seg->s_szc = 0;
		seg->s_data = (void *)NULL;
		kmem_free(shmd, (sizeof (*shmd)));
	} else {
		ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
		shm_amp->refcnt++;
		ANON_LOCK_EXIT(&shm_amp->a_rwlock);
	}
	return (error);
}

int
segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize)
{
	struct shm_data *shmd = (struct shm_data *)seg->s_data;
	int reclaim = 1;

	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
retry:
	if (shmd->shm_softlockcnt > 0) {
		if (reclaim == 1) {
			segspt_purge(seg);
			reclaim = 0;
			goto retry;
		}
		return (EAGAIN);
	}

	if (ssize != seg->s_size) {
#ifdef DEBUG
		cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n",
		    ssize, seg->s_size);
#endif
		return (EINVAL);
	}

	(void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK,
	    NULL, 0);
	hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc);

	seg_free(seg);

	return (0);
}

void
segspt_shmfree(struct seg *seg)
{
	struct shm_data *shmd = (struct shm_data *)seg->s_data;
	struct anon_map *shm_amp = shmd->shm_amp;

	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));

	(void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0,
	    MC_UNLOCK, NULL, 0);

	/*
	 * Need to increment refcnt when attaching
	 * and decrement when detaching because of dup().
	 */
	ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
	shm_amp->refcnt--;
	ANON_LOCK_EXIT(&shm_amp->a_rwlock);

	if (shmd->shm_vpage) {	/* only for DISM */
		kmem_free(shmd->shm_vpage, btopr(shm_amp->size));
		shmd->shm_vpage = NULL;
	}

	/*
	 * Take shm_segfree_syncmtx lock to let segspt_reclaim() finish if it's
	 * still working with this segment without holding as lock.
	 */
	ASSERT(shmd->shm_softlockcnt == 0);
	mutex_enter(&shmd->shm_segfree_syncmtx);
	mutex_destroy(&shmd->shm_segfree_syncmtx);

	kmem_free(shmd, sizeof (*shmd));
}

/*ARGSUSED*/
int
segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
{
	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));

	/*
	 * Shared page table is more than shared mapping.
	 *  Individual process sharing page tables can't change prot
	 *  because there is only one set of page tables.
	 *  This will be allowed after private page table is
	 *  supported.
	 */
/* need to return correct status error? */
	return (0);
}


faultcode_t
segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr,
    size_t len, enum fault_type type, enum seg_rw rw)
{
	struct  shm_data	*shmd = (struct shm_data *)seg->s_data;
	struct  seg		*sptseg = shmd->shm_sptseg;
	struct  as		*curspt = shmd->shm_sptas;
	struct  spt_data	*sptd = sptseg->s_data;
	pgcnt_t npages;
	size_t  size;
	caddr_t segspt_addr, shm_addr;
	page_t  **ppa;
	int	i;
	ulong_t an_idx = 0;
	int	err = 0;
	int	dyn_ism_unmap = hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0);
	size_t	pgsz;
	pgcnt_t	pgcnt;
	caddr_t	a;
	pgcnt_t	pidx;

#ifdef lint
	hat = hat;
#endif
	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));

	/*
	 * Because of the way spt is implemented
	 * the realsize of the segment does not have to be
	 * equal to the segment size itself. The segment size is
	 * often in multiples of a page size larger than PAGESIZE.
	 * The realsize is rounded up to the nearest PAGESIZE
	 * based on what the user requested. This is a bit of
	 * ungliness that is historical but not easily fixed
	 * without re-designing the higher levels of ISM.
	 */
	ASSERT(addr >= seg->s_base);
	if (((addr + len) - seg->s_base) > sptd->spt_realsize)
		return (FC_NOMAP);
	/*
	 * For all of the following cases except F_PROT, we need to
	 * make any necessary adjustments to addr and len
	 * and get all of the necessary page_t's into an array called ppa[].
	 *
	 * The code in shmat() forces base addr and len of ISM segment
	 * to be aligned to largest page size supported. Therefore,
	 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
	 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
	 * in large pagesize chunks, or else we will screw up the HAT
	 * layer by calling hat_memload_array() with differing page sizes
	 * over a given virtual range.
	 */
	pgsz = page_get_pagesize(sptseg->s_szc);
	pgcnt = page_get_pagecnt(sptseg->s_szc);
	shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
	size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
	npages = btopr(size);

	/*
	 * Now we need to convert from addr in segshm to addr in segspt.
	 */
	an_idx = seg_page(seg, shm_addr);
	segspt_addr = sptseg->s_base + ptob(an_idx);

	ASSERT((segspt_addr + ptob(npages)) <=
	    (sptseg->s_base + sptd->spt_realsize));
	ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size));

	switch (type) {

	case F_SOFTLOCK:

		atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
		/*
		 * Fall through to the F_INVAL case to load up the hat layer
		 * entries with the HAT_LOAD_LOCK flag.
		 */
		/* FALLTHRU */
	case F_INVAL:

		if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
			return (FC_NOMAP);

		ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP);

		err = spt_anon_getpages(sptseg, segspt_addr, size, ppa);
		if (err != 0) {
			if (type == F_SOFTLOCK) {
				atomic_add_long((ulong_t *)(
				    &(shmd->shm_softlockcnt)), -npages);
			}
			goto dism_err;
		}
		AS_LOCK_ENTER(sptseg->s_as, RW_READER);
		a = segspt_addr;
		pidx = 0;
		if (type == F_SOFTLOCK) {

			/*
			 * Load up the translation keeping it
			 * locked and don't unlock the page.
			 */
			for (; pidx < npages; a += pgsz, pidx += pgcnt) {
				hat_memload_array(sptseg->s_as->a_hat,
				    a, pgsz, &ppa[pidx], sptd->spt_prot,
				    HAT_LOAD_LOCK | HAT_LOAD_SHARE);
			}
		} else {
			/*
			 * Migrate pages marked for migration
			 */
			if (lgrp_optimizations())
				page_migrate(seg, shm_addr, ppa, npages);

			for (; pidx < npages; a += pgsz, pidx += pgcnt) {
				hat_memload_array(sptseg->s_as->a_hat,
				    a, pgsz, &ppa[pidx],
				    sptd->spt_prot,
				    HAT_LOAD_SHARE);
			}

			/*
			 * And now drop the SE_SHARED lock(s).
			 */
			if (dyn_ism_unmap) {
				for (i = 0; i < npages; i++) {
					page_unlock(ppa[i]);
				}
			}
		}

		if (!dyn_ism_unmap) {
			if (hat_share(seg->s_as->a_hat, shm_addr,
			    curspt->a_hat, segspt_addr, ptob(npages),
			    seg->s_szc) != 0) {
				panic("hat_share err in DISM fault");
				/* NOTREACHED */
			}
			if (type == F_INVAL) {
				for (i = 0; i < npages; i++) {
					page_unlock(ppa[i]);
				}
			}
		}
		AS_LOCK_EXIT(sptseg->s_as);
dism_err:
		kmem_free(ppa, npages * sizeof (page_t *));
		return (err);

	case F_SOFTUNLOCK:

		/*
		 * This is a bit ugly, we pass in the real seg pointer,
		 * but the segspt_addr is the virtual address within the
		 * dummy seg.
		 */
		segspt_softunlock(seg, segspt_addr, size, rw);
		return (0);

	case F_PROT:

		/*
		 * This takes care of the unusual case where a user
		 * allocates a stack in shared memory and a register
		 * window overflow is written to that stack page before
		 * it is otherwise modified.
		 *
		 * We can get away with this because ISM segments are
		 * always rw. Other than this unusual case, there
		 * should be no instances of protection violations.
		 */
		return (0);

	default:
#ifdef DEBUG
		panic("segspt_dismfault default type?");
#else
		return (FC_NOMAP);
#endif
	}
}


faultcode_t
segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr,
    size_t len, enum fault_type type, enum seg_rw rw)
{
	struct shm_data		*shmd = (struct shm_data *)seg->s_data;
	struct seg		*sptseg = shmd->shm_sptseg;
	struct as		*curspt = shmd->shm_sptas;
	struct spt_data		*sptd = sptseg->s_data;
	pgcnt_t npages;
	size_t size;
	caddr_t sptseg_addr, shm_addr;
	page_t *pp, **ppa;
	int	i;
	u_offset_t offset;
	ulong_t anon_index = 0;
	struct vnode *vp;
	struct anon_map *amp;		/* XXX - for locknest */
	struct anon *ap = NULL;
	size_t		pgsz;
	pgcnt_t		pgcnt;
	caddr_t		a;
	pgcnt_t		pidx;
	size_t		sz;

#ifdef lint
	hat = hat;
#endif

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));

	if (sptd->spt_flags & SHM_PAGEABLE) {
		return (segspt_dismfault(hat, seg, addr, len, type, rw));
	}

	/*
	 * Because of the way spt is implemented
	 * the realsize of the segment does not have to be
	 * equal to the segment size itself. The segment size is
	 * often in multiples of a page size larger than PAGESIZE.
	 * The realsize is rounded up to the nearest PAGESIZE
	 * based on what the user requested. This is a bit of
	 * ungliness that is historical but not easily fixed
	 * without re-designing the higher levels of ISM.
	 */
	ASSERT(addr >= seg->s_base);
	if (((addr + len) - seg->s_base) > sptd->spt_realsize)
		return (FC_NOMAP);
	/*
	 * For all of the following cases except F_PROT, we need to
	 * make any necessary adjustments to addr and len
	 * and get all of the necessary page_t's into an array called ppa[].
	 *
	 * The code in shmat() forces base addr and len of ISM segment
	 * to be aligned to largest page size supported. Therefore,
	 * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
	 * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
	 * in large pagesize chunks, or else we will screw up the HAT
	 * layer by calling hat_memload_array() with differing page sizes
	 * over a given virtual range.
	 */
	pgsz = page_get_pagesize(sptseg->s_szc);
	pgcnt = page_get_pagecnt(sptseg->s_szc);
	shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
	size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
	npages = btopr(size);

	/*
	 * Now we need to convert from addr in segshm to addr in segspt.
	 */
	anon_index = seg_page(seg, shm_addr);
	sptseg_addr = sptseg->s_base + ptob(anon_index);

	/*
	 * And now we may have to adjust npages downward if we have
	 * exceeded the realsize of the segment or initial anon
	 * allocations.
	 */
	if ((sptseg_addr + ptob(npages)) >
	    (sptseg->s_base + sptd->spt_realsize))
		size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr;

	npages = btopr(size);

	ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size));
	ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0);

	switch (type) {

	case F_SOFTLOCK:

		/*
		 * availrmem is decremented once during anon_swap_adjust()
		 * and is incremented during the anon_unresv(), which is
		 * called from shm_rm_amp() when the segment is destroyed.
		 */
		atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
		/*
		 * Some platforms assume that ISM pages are SE_SHARED
		 * locked for the entire life of the segment.
		 */
		if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0))
			return (0);
		/*
		 * Fall through to the F_INVAL case to load up the hat layer
		 * entries with the HAT_LOAD_LOCK flag.
		 */

		/* FALLTHRU */
	case F_INVAL:

		if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
			return (FC_NOMAP);

		/*
		 * Some platforms that do NOT support DYNAMIC_ISM_UNMAP
		 * may still rely on this call to hat_share(). That
		 * would imply that those hat's can fault on a
		 * HAT_LOAD_LOCK translation, which would seem
		 * contradictory.
		 */
		if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
			if (hat_share(seg->s_as->a_hat, seg->s_base,
			    curspt->a_hat, sptseg->s_base,
			    sptseg->s_size, sptseg->s_szc) != 0) {
				panic("hat_share error in ISM fault");
				/*NOTREACHED*/
			}
			return (0);
		}
		ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP);

		/*
		 * I see no need to lock the real seg,
		 * here, because all of our work will be on the underlying
		 * dummy seg.
		 *
		 * sptseg_addr and npages now account for large pages.
		 */
		amp = sptd->spt_amp;
		ASSERT(amp != NULL);
		anon_index = seg_page(sptseg, sptseg_addr);

		ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
		for (i = 0; i < npages; i++) {
			ap = anon_get_ptr(amp->ahp, anon_index++);
			ASSERT(ap != NULL);
			swap_xlate(ap, &vp, &offset);
			pp = page_lookup(vp, offset, SE_SHARED);
			ASSERT(pp != NULL);
			ppa[i] = pp;
		}
		ANON_LOCK_EXIT(&amp->a_rwlock);
		ASSERT(i == npages);

		/*
		 * We are already holding the as->a_lock on the user's
		 * real segment, but we need to hold the a_lock on the
		 * underlying dummy as. This is mostly to satisfy the
		 * underlying HAT layer.
		 */
		AS_LOCK_ENTER(sptseg->s_as, RW_READER);
		a = sptseg_addr;
		pidx = 0;
		if (type == F_SOFTLOCK) {
			/*
			 * Load up the translation keeping it
			 * locked and don't unlock the page.
			 */
			for (; pidx < npages; a += pgsz, pidx += pgcnt) {
				sz = MIN(pgsz, ptob(npages - pidx));
				hat_memload_array(sptseg->s_as->a_hat, a,
				    sz, &ppa[pidx], sptd->spt_prot,
				    HAT_LOAD_LOCK | HAT_LOAD_SHARE);
			}
		} else {
			/*
			 * Migrate pages marked for migration.
			 */
			if (lgrp_optimizations())
				page_migrate(seg, shm_addr, ppa, npages);

			for (; pidx < npages; a += pgsz, pidx += pgcnt) {
				sz = MIN(pgsz, ptob(npages - pidx));
				hat_memload_array(sptseg->s_as->a_hat,
				    a, sz, &ppa[pidx],
				    sptd->spt_prot, HAT_LOAD_SHARE);
			}

			/*
			 * And now drop the SE_SHARED lock(s).
			 */
			for (i = 0; i < npages; i++)
				page_unlock(ppa[i]);
		}
		AS_LOCK_EXIT(sptseg->s_as);

		kmem_free(ppa, sizeof (page_t *) * npages);
		return (0);
	case F_SOFTUNLOCK:

		/*
		 * This is a bit ugly, we pass in the real seg pointer,
		 * but the sptseg_addr is the virtual address within the
		 * dummy seg.
		 */
		segspt_softunlock(seg, sptseg_addr, ptob(npages), rw);
		return (0);

	case F_PROT:

		/*
		 * This takes care of the unusual case where a user
		 * allocates a stack in shared memory and a register
		 * window overflow is written to that stack page before
		 * it is otherwise modified.
		 *
		 * We can get away with this because ISM segments are
		 * always rw. Other than this unusual case, there
		 * should be no instances of protection violations.
		 */
		return (0);

	default:
#ifdef DEBUG
		cmn_err(CE_WARN, "segspt_shmfault default type?");
#endif
		return (FC_NOMAP);
	}
}

/*ARGSUSED*/
static faultcode_t
segspt_shmfaulta(struct seg *seg, caddr_t addr)
{
	return (0);
}

/*ARGSUSED*/
static int
segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta)
{
	return (0);
}

/*ARGSUSED*/
static size_t
segspt_shmswapout(struct seg *seg)
{
	return (0);
}

/*
 * duplicate the shared page tables
 */
int
segspt_shmdup(struct seg *seg, struct seg *newseg)
{
	struct shm_data		*shmd = (struct shm_data *)seg->s_data;
	struct anon_map		*amp = shmd->shm_amp;
	struct shm_data		*shmd_new;
	struct seg		*spt_seg = shmd->shm_sptseg;
	struct spt_data		*sptd = spt_seg->s_data;
	int			error = 0;

	ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));

	shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP);
	newseg->s_data = (void *)shmd_new;
	shmd_new->shm_sptas = shmd->shm_sptas;
	shmd_new->shm_amp = amp;
	shmd_new->shm_sptseg = shmd->shm_sptseg;
	newseg->s_ops = &segspt_shmops;
	newseg->s_szc = seg->s_szc;
	ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc);

	ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
	amp->refcnt++;
	ANON_LOCK_EXIT(&amp->a_rwlock);

	if (sptd->spt_flags & SHM_PAGEABLE) {
		shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP);
		shmd_new->shm_lckpgs = 0;
		if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
			if ((error = hat_share(newseg->s_as->a_hat,
			    newseg->s_base, shmd->shm_sptas->a_hat, SEGSPTADDR,
			    seg->s_size, seg->s_szc)) != 0) {
				kmem_free(shmd_new->shm_vpage,
				    btopr(amp->size));
			}
		}
		return (error);
	} else {
		return (hat_share(newseg->s_as->a_hat, newseg->s_base,
		    shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size,
		    seg->s_szc));

	}
}

/*ARGSUSED*/
int
segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
{
	struct shm_data *shmd = (struct shm_data *)seg->s_data;
	struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));

	/*
	 * ISM segment is always rw.
	 */
	return (((sptd->spt_prot & prot) != prot) ? EACCES : 0);
}

/*
 * Return an array of locked large pages, for empty slots allocate
 * private zero-filled anon pages.
 */
static int
spt_anon_getpages(
	struct seg *sptseg,
	caddr_t sptaddr,
	size_t len,
	page_t *ppa[])
{
	struct  spt_data *sptd = sptseg->s_data;
	struct  anon_map *amp = sptd->spt_amp;
	enum	seg_rw rw = sptd->spt_prot;
	uint_t	szc = sptseg->s_szc;
	size_t	pg_sz, share_sz = page_get_pagesize(szc);
	pgcnt_t	lp_npgs;
	caddr_t	lp_addr, e_sptaddr;
	uint_t	vpprot, ppa_szc = 0;
	struct  vpage *vpage = NULL;
	ulong_t	j, ppa_idx;
	int	err, ierr = 0;
	pgcnt_t	an_idx;
	anon_sync_obj_t cookie;
	int anon_locked = 0;
	pgcnt_t amp_pgs;


	ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz));
	ASSERT(len != 0);

	pg_sz = share_sz;
	lp_npgs = btop(pg_sz);
	lp_addr = sptaddr;
	e_sptaddr = sptaddr + len;
	an_idx = seg_page(sptseg, sptaddr);
	ppa_idx = 0;

	ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);

	amp_pgs = page_get_pagecnt(amp->a_szc);

	/*CONSTCOND*/
	while (1) {
		for (; lp_addr < e_sptaddr;
		    an_idx += lp_npgs, lp_addr += pg_sz, ppa_idx += lp_npgs) {

			/*
			 * If we're currently locked, and we get to a new
			 * page, unlock our current anon chunk.
			 */
			if (anon_locked && P2PHASE(an_idx, amp_pgs) == 0) {
				anon_array_exit(&cookie);
				anon_locked = 0;
			}
			if (!anon_locked) {
				anon_array_enter(amp, an_idx, &cookie);
				anon_locked = 1;
			}
			ppa_szc = (uint_t)-1;
			ierr = anon_map_getpages(amp, an_idx, szc, sptseg,
			    lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx],
			    &ppa_szc, vpage, rw, 0, segvn_anypgsz, 0, kcred);

			if (ierr != 0) {
				if (ierr > 0) {
					err = FC_MAKE_ERR(ierr);
					goto lpgs_err;
				}
				break;
			}
		}
		if (lp_addr == e_sptaddr) {
			break;
		}
		ASSERT(lp_addr < e_sptaddr);

		/*
		 * ierr == -1 means we failed to allocate a large page.
		 * so do a size down operation.
		 *
		 * ierr == -2 means some other process that privately shares
		 * pages with this process has allocated a larger page and we
		 * need to retry with larger pages. So do a size up
		 * operation. This relies on the fact that large pages are
		 * never partially shared i.e. if we share any constituent
		 * page of a large page with another process we must share the
		 * entire large page. Note this cannot happen for SOFTLOCK
		 * case, unless current address (lpaddr) is at the beginning
		 * of the next page size boundary because the other process
		 * couldn't have relocated locked pages.
		 */
		ASSERT(ierr == -1 || ierr == -2);
		if (segvn_anypgsz) {
			ASSERT(ierr == -2 || szc != 0);
			ASSERT(ierr == -1 || szc < sptseg->s_szc);
			szc = (ierr == -1) ? szc - 1 : szc + 1;
		} else {
			/*
			 * For faults and segvn_anypgsz == 0
			 * we need to be careful not to loop forever
			 * if existing page is found with szc other
			 * than 0 or seg->s_szc. This could be due
			 * to page relocations on behalf of DR or
			 * more likely large page creation. For this
			 * case simply re-size to existing page's szc
			 * if returned by anon_map_getpages().
			 */
			if (ppa_szc == (uint_t)-1) {
				szc = (ierr == -1) ? 0 : sptseg->s_szc;
			} else {
				ASSERT(ppa_szc <= sptseg->s_szc);
				ASSERT(ierr == -2 || ppa_szc < szc);
				ASSERT(ierr == -1 || ppa_szc > szc);
				szc = ppa_szc;
			}
		}
		pg_sz = page_get_pagesize(szc);
		lp_npgs = btop(pg_sz);
		ASSERT(IS_P2ALIGNED(lp_addr, pg_sz));
	}
	if (anon_locked) {
		anon_array_exit(&cookie);
	}
	ANON_LOCK_EXIT(&amp->a_rwlock);
	return (0);

lpgs_err:
	if (anon_locked) {
		anon_array_exit(&cookie);
	}
	ANON_LOCK_EXIT(&amp->a_rwlock);
	for (j = 0; j < ppa_idx; j++)
		page_unlock(ppa[j]);
	return (err);
}

/*
 * count the number of bytes in a set of spt pages that are currently not
 * locked
 */
static rctl_qty_t
spt_unlockedbytes(pgcnt_t npages, page_t **ppa)
{
	ulong_t	i;
	rctl_qty_t unlocked = 0;

	for (i = 0; i < npages; i++) {
		if (ppa[i]->p_lckcnt == 0)
			unlocked += PAGESIZE;
	}
	return (unlocked);
}

extern	u_longlong_t randtick(void);
/* number of locks to reserve/skip by spt_lockpages() and spt_unlockpages() */
#define	NLCK	(NCPU_P2)
/* Random number with a range [0, n-1], n must be power of two */
#define	RAND_P2(n)	\
	((((long)curthread >> PTR24_LSB) ^ (long)randtick()) & ((n) - 1))

int
spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
    page_t **ppa, ulong_t *lockmap, size_t pos,
    rctl_qty_t *locked)
{
	struct	shm_data *shmd = seg->s_data;
	struct	spt_data *sptd = shmd->shm_sptseg->s_data;
	ulong_t	i;
	int	kernel;
	pgcnt_t	nlck = 0;
	int	rv = 0;
	int	use_reserved = 1;

	/* return the number of bytes actually locked */
	*locked = 0;

	/*
	 * To avoid contention on freemem_lock, availrmem and pages_locked
	 * global counters are updated only every nlck locked pages instead of
	 * every time.  Reserve nlck locks up front and deduct from this
	 * reservation for each page that requires a lock.  When the reservation
	 * is consumed, reserve again.  nlck is randomized, so the competing
	 * threads do not fall into a cyclic lock contention pattern. When
	 * memory is low, the lock ahead is disabled, and instead page_pp_lock()
	 * is used to lock pages.
	 */
	for (i = 0; i < npages; anon_index++, pos++, i++) {
		if (nlck == 0 && use_reserved == 1) {
			nlck = NLCK + RAND_P2(NLCK);
			/* if fewer loops left, decrease nlck */
			nlck = MIN(nlck, npages - i);
			/*
			 * Reserve nlck locks up front and deduct from this
			 * reservation for each page that requires a lock.  When
			 * the reservation is consumed, reserve again.
			 */
			mutex_enter(&freemem_lock);
			if ((availrmem - nlck) < pages_pp_maximum) {
				/* Do not do advance memory reserves */
				use_reserved = 0;
			} else {
				availrmem	-= nlck;
				pages_locked	+= nlck;
			}
			mutex_exit(&freemem_lock);
		}
		if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) {
			if (sptd->spt_ppa_lckcnt[anon_index] <
			    (ushort_t)DISM_LOCK_MAX) {
				if (++sptd->spt_ppa_lckcnt[anon_index] ==
				    (ushort_t)DISM_LOCK_MAX) {
					cmn_err(CE_WARN,
					    "DISM page lock limit "
					    "reached on DISM offset 0x%lx\n",
					    anon_index << PAGESHIFT);
				}
				kernel = (sptd->spt_ppa &&
				    sptd->spt_ppa[anon_index]);
				if (!page_pp_lock(ppa[i], 0, kernel ||
				    use_reserved)) {
					sptd->spt_ppa_lckcnt[anon_index]--;
					rv = EAGAIN;
					break;
				}
				/* if this is a newly locked page, count it */
				if (ppa[i]->p_lckcnt == 1) {
					if (kernel == 0 && use_reserved == 1)
						nlck--;
					*locked += PAGESIZE;
				}
				shmd->shm_lckpgs++;
				shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED;
				if (lockmap != NULL)
					BT_SET(lockmap, pos);
			}
		}
	}
	/* Return unused lock reservation */
	if (nlck != 0 && use_reserved == 1) {
		mutex_enter(&freemem_lock);
		availrmem	+= nlck;
		pages_locked	-= nlck;
		mutex_exit(&freemem_lock);
	}

	return (rv);
}

int
spt_unlockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
    rctl_qty_t *unlocked)
{
	struct shm_data	*shmd = seg->s_data;
	struct spt_data	*sptd = shmd->shm_sptseg->s_data;
	struct anon_map	*amp = sptd->spt_amp;
	struct anon	*ap;
	struct vnode	*vp;
	u_offset_t	off;
	struct page	*pp;
	int		kernel;
	anon_sync_obj_t	cookie;
	ulong_t		i;
	pgcnt_t		nlck = 0;
	pgcnt_t		nlck_limit = NLCK;

	ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
	for (i = 0; i < npages; i++, anon_index++) {
		if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
			anon_array_enter(amp, anon_index, &cookie);
			ap = anon_get_ptr(amp->ahp, anon_index);
			ASSERT(ap);

			swap_xlate(ap, &vp, &off);
			anon_array_exit(&cookie);
			pp = page_lookup(vp, off, SE_SHARED);
			ASSERT(pp);
			/*
			 * availrmem is decremented only for pages which are not
			 * in seg pcache, for pages in seg pcache availrmem was
			 * decremented in _dismpagelock()
			 */
			kernel = (sptd->spt_ppa && sptd->spt_ppa[anon_index]);
			ASSERT(pp->p_lckcnt > 0);

			/*
			 * lock page but do not change availrmem, we do it
			 * ourselves every nlck loops.
			 */
			page_pp_unlock(pp, 0, 1);
			if (pp->p_lckcnt == 0) {
				if (kernel == 0)
					nlck++;
				*unlocked += PAGESIZE;
			}
			page_unlock(pp);
			shmd->shm_vpage[anon_index] &= ~DISM_PG_LOCKED;
			sptd->spt_ppa_lckcnt[anon_index]--;
			shmd->shm_lckpgs--;
		}

		/*
		 * To reduce freemem_lock contention, do not update availrmem
		 * until at least NLCK pages have been unlocked.
		 * 1. No need to update if nlck is zero
		 * 2. Always update if the last iteration
		 */
		if (nlck > 0 && (nlck == nlck_limit || i == npages - 1)) {
			mutex_enter(&freemem_lock);
			availrmem	+= nlck;
			pages_locked	-= nlck;
			mutex_exit(&freemem_lock);
			nlck = 0;
			nlck_limit = NLCK + RAND_P2(NLCK);
		}
	}
	ANON_LOCK_EXIT(&amp->a_rwlock);

	return (0);
}

/*ARGSUSED*/
static int
segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
    int attr, int op, ulong_t *lockmap, size_t pos)
{
	struct shm_data *shmd = seg->s_data;
	struct seg	*sptseg = shmd->shm_sptseg;
	struct spt_data *sptd = sptseg->s_data;
	struct kshmid	*sp = sptd->spt_amp->a_sp;
	pgcnt_t		npages, a_npages;
	page_t		**ppa;
	pgcnt_t		an_idx, a_an_idx, ppa_idx;
	caddr_t		spt_addr, a_addr;	/* spt and aligned address */
	size_t		a_len;			/* aligned len */
	size_t		share_sz;
	ulong_t		i;
	int		sts = 0;
	rctl_qty_t	unlocked = 0;
	rctl_qty_t	locked = 0;
	struct proc	*p = curproc;
	kproject_t	*proj;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
	ASSERT(sp != NULL);

	if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
		return (0);
	}

	addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
	an_idx = seg_page(seg, addr);
	npages = btopr(len);

	if (an_idx + npages > btopr(shmd->shm_amp->size)) {
		return (ENOMEM);
	}

	/*
	 * A shm's project never changes, so no lock needed.
	 * The shm has a hold on the project, so it will not go away.
	 * Since we have a mapping to shm within this zone, we know
	 * that the zone will not go away.
	 */
	proj = sp->shm_perm.ipc_proj;

	if (op == MC_LOCK) {

		/*
		 * Need to align addr and size request if they are not
		 * aligned so we can always allocate large page(s) however
		 * we only lock what was requested in initial request.
		 */
		share_sz = page_get_pagesize(sptseg->s_szc);
		a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz);
		a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)),
		    share_sz);
		a_npages = btop(a_len);
		a_an_idx = seg_page(seg, a_addr);
		spt_addr = sptseg->s_base + ptob(a_an_idx);
		ppa_idx = an_idx - a_an_idx;

		if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages),
		    KM_NOSLEEP)) == NULL) {
			return (ENOMEM);
		}

		/*
		 * Don't cache any new pages for IO and
		 * flush any cached pages.
		 */
		mutex_enter(&sptd->spt_lock);
		if (sptd->spt_ppa != NULL)
			sptd->spt_flags |= DISM_PPA_CHANGED;

		sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa);
		if (sts != 0) {
			mutex_exit(&sptd->spt_lock);
			kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
			return (sts);
		}

		mutex_enter(&sp->shm_mlock);
		/* enforce locked memory rctl */
		unlocked = spt_unlockedbytes(npages, &ppa[ppa_idx]);

		mutex_enter(&p->p_lock);
		if (rctl_incr_locked_mem(p, proj, unlocked, 0)) {
			mutex_exit(&p->p_lock);
			sts = EAGAIN;
		} else {
			mutex_exit(&p->p_lock);
			sts = spt_lockpages(seg, an_idx, npages,
			    &ppa[ppa_idx], lockmap, pos, &locked);

			/*
			 * correct locked count if not all pages could be
			 * locked
			 */
			if ((unlocked - locked) > 0) {
				rctl_decr_locked_mem(NULL, proj,
				    (unlocked - locked), 0);
			}
		}
		/*
		 * unlock pages
		 */
		for (i = 0; i < a_npages; i++)
			page_unlock(ppa[i]);
		if (sptd->spt_ppa != NULL)
			sptd->spt_flags |= DISM_PPA_CHANGED;
		mutex_exit(&sp->shm_mlock);
		mutex_exit(&sptd->spt_lock);

		kmem_free(ppa, ((sizeof (page_t *)) * a_npages));

	} else if (op == MC_UNLOCK) { /* unlock */
		page_t		**ppa;

		mutex_enter(&sptd->spt_lock);
		if (shmd->shm_lckpgs == 0) {
			mutex_exit(&sptd->spt_lock);
			return (0);
		}
		/*
		 * Don't cache new IO pages.
		 */
		if (sptd->spt_ppa != NULL)
			sptd->spt_flags |= DISM_PPA_CHANGED;

		mutex_enter(&sp->shm_mlock);
		sts = spt_unlockpages(seg, an_idx, npages, &unlocked);
		if ((ppa = sptd->spt_ppa) != NULL)
			sptd->spt_flags |= DISM_PPA_CHANGED;
		mutex_exit(&sptd->spt_lock);

		rctl_decr_locked_mem(NULL, proj, unlocked, 0);
		mutex_exit(&sp->shm_mlock);

		if (ppa != NULL)
			seg_ppurge_wiredpp(ppa);
	}
	return (sts);
}

/*ARGSUSED*/
int
segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
{
	struct shm_data *shmd = (struct shm_data *)seg->s_data;
	struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
	spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));

	/*
	 * ISM segment is always rw.
	 */
	while (--pgno >= 0)
		*protv++ = sptd->spt_prot;
	return (0);
}

/*ARGSUSED*/
u_offset_t
segspt_shmgetoffset(struct seg *seg, caddr_t addr)
{
	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));

	/* Offset does not matter in ISM memory */

	return ((u_offset_t)0);
}

/* ARGSUSED */
int
segspt_shmgettype(struct seg *seg, caddr_t addr)
{
	struct shm_data *shmd = (struct shm_data *)seg->s_data;
	struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));

	/*
	 * The shared memory mapping is always MAP_SHARED, SWAP is only
	 * reserved for DISM
	 */
	return (MAP_SHARED |
	    ((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE));
}

/*ARGSUSED*/
int
segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
{
	struct shm_data *shmd = (struct shm_data *)seg->s_data;
	struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));

	*vpp = sptd->spt_vp;
	return (0);
}

/*
 * We need to wait for pending IO to complete to a DISM segment in order for
 * pages to get kicked out of the seg_pcache.  120 seconds should be more
 * than enough time to wait.
 */
static clock_t spt_pcache_wait = 120;

/*ARGSUSED*/
static int
segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
{
	struct shm_data	*shmd = (struct shm_data *)seg->s_data;
	struct spt_data	*sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
	struct anon_map	*amp;
	pgcnt_t pg_idx;
	ushort_t gen;
	clock_t	end_lbolt;
	int writer;
	page_t **ppa;

	ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));

	if (behav == MADV_FREE || behav == MADV_PURGE) {
		if ((sptd->spt_flags & SHM_PAGEABLE) == 0)
			return (0);

		amp = sptd->spt_amp;
		pg_idx = seg_page(seg, addr);

		mutex_enter(&sptd->spt_lock);
		if ((ppa = sptd->spt_ppa) == NULL) {
			mutex_exit(&sptd->spt_lock);
			ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
			(void) anon_disclaim(amp, pg_idx, len, behav, NULL);
			ANON_LOCK_EXIT(&amp->a_rwlock);
			return (0);
		}

		sptd->spt_flags |= DISM_PPA_CHANGED;
		gen = sptd->spt_gen;

		mutex_exit(&sptd->spt_lock);

		/*
		 * Purge all DISM cached pages
		 */
		seg_ppurge_wiredpp(ppa);

		/*
		 * Drop the AS_LOCK so that other threads can grab it
		 * in the as_pageunlock path and hopefully get the segment
		 * kicked out of the seg_pcache.  We bump the shm_softlockcnt
		 * to keep this segment resident.
		 */
		writer = AS_WRITE_HELD(seg->s_as);
		atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
		AS_LOCK_EXIT(seg->s_as);

		mutex_enter(&sptd->spt_lock);

		end_lbolt = ddi_get_lbolt() + (hz * spt_pcache_wait);

		/*
		 * Try to wait for pages to get kicked out of the seg_pcache.
		 */
		while (sptd->spt_gen == gen &&
		    (sptd->spt_flags & DISM_PPA_CHANGED) &&
		    ddi_get_lbolt() < end_lbolt) {
			if (!cv_timedwait_sig(&sptd->spt_cv,
			    &sptd->spt_lock, end_lbolt)) {
				break;
			}
		}

		mutex_exit(&sptd->spt_lock);

		/* Regrab the AS_LOCK and release our hold on the segment */
		AS_LOCK_ENTER(seg->s_as, writer ? RW_WRITER : RW_READER);
		atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
		if (shmd->shm_softlockcnt <= 0) {
			if (AS_ISUNMAPWAIT(seg->s_as)) {
				mutex_enter(&seg->s_as->a_contents);
				if (AS_ISUNMAPWAIT(seg->s_as)) {
					AS_CLRUNMAPWAIT(seg->s_as);
					cv_broadcast(&seg->s_as->a_cv);
				}
				mutex_exit(&seg->s_as->a_contents);
			}
		}

		ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
		(void) anon_disclaim(amp, pg_idx, len, behav, NULL);
		ANON_LOCK_EXIT(&amp->a_rwlock);
	} else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP ||
	    behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) {
		int			already_set;
		ulong_t			anon_index;
		lgrp_mem_policy_t	policy;
		caddr_t			shm_addr;
		size_t			share_size;
		size_t			size;
		struct seg		*sptseg = shmd->shm_sptseg;
		caddr_t			sptseg_addr;

		/*
		 * Align address and length to page size of underlying segment
		 */
		share_size = page_get_pagesize(shmd->shm_sptseg->s_szc);
		shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size);
		size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)),
		    share_size);

		amp = shmd->shm_amp;
		anon_index = seg_page(seg, shm_addr);

		/*
		 * And now we may have to adjust size downward if we have
		 * exceeded the realsize of the segment or initial anon
		 * allocations.
		 */
		sptseg_addr = sptseg->s_base + ptob(anon_index);
		if ((sptseg_addr + size) >
		    (sptseg->s_base + sptd->spt_realsize))
			size = (sptseg->s_base + sptd->spt_realsize) -
			    sptseg_addr;

		/*
		 * Set memory allocation policy for this segment
		 */
		policy = lgrp_madv_to_policy(behav, len, MAP_SHARED);
		already_set = lgrp_shm_policy_set(policy, amp, anon_index,
		    NULL, 0, len);

		/*
		 * If random memory allocation policy set already,
		 * don't bother reapplying it.
		 */
		if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy))
			return (0);

		/*
		 * Mark any existing pages in the given range for
		 * migration, flushing the I/O page cache, and using
		 * underlying segment to calculate anon index and get
		 * anonmap and vnode pointer from
		 */
		if (shmd->shm_softlockcnt > 0)
			segspt_purge(seg);

		page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0);
	}

	return (0);
}

/*ARGSUSED*/
void
segspt_shmdump(struct seg *seg)
{
	/* no-op for ISM segment */
}

/*ARGSUSED*/
static int
segspt_shmsetpgsz(struct seg *seg, caddr_t addr, size_t len, uint_t szc)
{
	return (ENOTSUP);
}

/*
 * get a memory ID for an addr in a given segment
 */
static int
segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
{
	struct shm_data *shmd = (struct shm_data *)seg->s_data;
	struct anon	*ap;
	size_t		anon_index;
	struct anon_map	*amp = shmd->shm_amp;
	struct spt_data	*sptd = shmd->shm_sptseg->s_data;
	struct seg	*sptseg = shmd->shm_sptseg;
	anon_sync_obj_t	cookie;

	anon_index = seg_page(seg, addr);

	if (addr > (seg->s_base + sptd->spt_realsize)) {
		return (EFAULT);
	}

	ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
	anon_array_enter(amp, anon_index, &cookie);
	ap = anon_get_ptr(amp->ahp, anon_index);
	if (ap == NULL) {
		struct page *pp;
		caddr_t spt_addr = sptseg->s_base + ptob(anon_index);

		pp = anon_zero(sptseg, spt_addr, &ap, kcred);
		if (pp == NULL) {
			anon_array_exit(&cookie);
			ANON_LOCK_EXIT(&amp->a_rwlock);
			return (ENOMEM);
		}
		(void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP);
		page_unlock(pp);
	}
	anon_array_exit(&cookie);
	ANON_LOCK_EXIT(&amp->a_rwlock);
	memidp->val[0] = (uintptr_t)ap;
	memidp->val[1] = (uintptr_t)addr & PAGEOFFSET;
	return (0);
}

/*
 * Get memory allocation policy info for specified address in given segment
 */
static lgrp_mem_policy_info_t *
segspt_shmgetpolicy(struct seg *seg, caddr_t addr)
{
	struct anon_map		*amp;
	ulong_t			anon_index;
	lgrp_mem_policy_info_t	*policy_info;
	struct shm_data		*shm_data;

	ASSERT(seg != NULL);

	/*
	 * Get anon_map from segshm
	 *
	 * Assume that no lock needs to be held on anon_map, since
	 * it should be protected by its reference count which must be
	 * nonzero for an existing segment
	 * Need to grab readers lock on policy tree though
	 */
	shm_data = (struct shm_data *)seg->s_data;
	if (shm_data == NULL)
		return (NULL);
	amp = shm_data->shm_amp;
	ASSERT(amp->refcnt != 0);

	/*
	 * Get policy info
	 *
	 * Assume starting anon index of 0
	 */
	anon_index = seg_page(seg, addr);
	policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);

	return (policy_info);
}

/*ARGSUSED*/
static int
segspt_shmcapable(struct seg *seg, segcapability_t capability)
{
	return (0);
}