xref: /illumos-gate/usr/src/uts/i86xpv/vm/seg_mf.c (revision 284ce987)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2018 Joyent, Inc.
 */

/*
 * Machine frame segment driver.  This segment driver allows dom0 processes to
 * map pages of other domains or Xen (e.g. during save/restore).  ioctl()s on
 * the privcmd driver provide the MFN values backing each mapping, and we map
 * them into the process's address space at this time.  Demand-faulting is not
 * supported by this driver due to the requirements upon some of the ioctl()s.
 */


#include <sys/types.h>
#include <sys/systm.h>
#include <sys/vmsystm.h>
#include <sys/mman.h>
#include <sys/errno.h>
#include <sys/kmem.h>
#include <sys/cmn_err.h>
#include <sys/vnode.h>
#include <sys/conf.h>
#include <sys/debug.h>
#include <sys/lgrp.h>
#include <sys/hypervisor.h>

#include <vm/page.h>
#include <vm/hat.h>
#include <vm/as.h>
#include <vm/seg.h>

#include <vm/hat_pte.h>
#include <vm/hat_i86.h>
#include <vm/seg_mf.h>

#include <sys/fs/snode.h>

#define	VTOCVP(vp)	(VTOS(vp)->s_commonvp)

typedef struct segmf_mfn_s {
	mfn_t		m_mfn;
} segmf_mfn_t;

/* g_flags */
#define	SEGMF_GFLAGS_WR		0x1
#define	SEGMF_GFLAGS_MAPPED	0x2
typedef struct segmf_gref_s {
	uint64_t	g_ptep;
	grant_ref_t	g_gref;
	uint32_t	g_flags;
	grant_handle_t	g_handle;
} segmf_gref_t;

typedef union segmf_mu_u {
	segmf_mfn_t	m;
	segmf_gref_t	g;
} segmf_mu_t;

typedef enum {
	SEGMF_MAP_EMPTY = 0,
	SEGMF_MAP_MFN,
	SEGMF_MAP_GREF
} segmf_map_type_t;

typedef struct segmf_map_s {
	segmf_map_type_t	t_type;
	segmf_mu_t		u;
} segmf_map_t;

struct segmf_data {
	kmutex_t	lock;
	struct vnode	*vp;
	uchar_t		prot;
	uchar_t		maxprot;
	size_t		softlockcnt;
	domid_t		domid;
	segmf_map_t	*map;
};

static struct seg_ops segmf_ops;

static int segmf_fault_gref_range(struct seg *seg, caddr_t addr, size_t len);

static struct segmf_data *
segmf_data_zalloc(struct seg *seg)
{
	struct segmf_data *data = kmem_zalloc(sizeof (*data), KM_SLEEP);

	mutex_init(&data->lock, "segmf.lock", MUTEX_DEFAULT, NULL);
	seg->s_ops = &segmf_ops;
	seg->s_data = data;
	return (data);
}

int
segmf_create(struct seg **segpp, void *args)
{
	struct seg *seg = *segpp;
	struct segmf_crargs *a = args;
	struct segmf_data *data;
	struct as *as = seg->s_as;
	pgcnt_t i, npages = seg_pages(seg);
	int error;

	hat_map(as->a_hat, seg->s_base, seg->s_size, HAT_MAP);

	data = segmf_data_zalloc(seg);
	data->vp = specfind(a->dev, VCHR);
	data->prot = a->prot;
	data->maxprot = a->maxprot;

	data->map = kmem_alloc(npages * sizeof (segmf_map_t), KM_SLEEP);
	for (i = 0; i < npages; i++) {
		data->map[i].t_type = SEGMF_MAP_EMPTY;
	}

	error = VOP_ADDMAP(VTOCVP(data->vp), 0, as, seg->s_base, seg->s_size,
	    data->prot, data->maxprot, MAP_SHARED, CRED(), NULL);

	if (error != 0)
		hat_unload(as->a_hat,
		    seg->s_base, seg->s_size, HAT_UNLOAD_UNMAP);
	return (error);
}

/*
 * Duplicate a seg and return new segment in newseg.
 */
static int
segmf_dup(struct seg *seg, struct seg *newseg)
{
	struct segmf_data *data = seg->s_data;
	struct segmf_data *ndata;
	pgcnt_t npages = seg_pages(newseg);
	size_t sz;

	ndata = segmf_data_zalloc(newseg);

	VN_HOLD(data->vp);
	ndata->vp = data->vp;
	ndata->prot = data->prot;
	ndata->maxprot = data->maxprot;
	ndata->domid = data->domid;

	sz = npages * sizeof (segmf_map_t);
	ndata->map = kmem_alloc(sz, KM_SLEEP);
	bcopy(data->map, ndata->map, sz);

	return (VOP_ADDMAP(VTOCVP(ndata->vp), 0, newseg->s_as,
	    newseg->s_base, newseg->s_size, ndata->prot, ndata->maxprot,
	    MAP_SHARED, CRED(), NULL));
}

/*
 * We only support unmapping the whole segment, and we automatically unlock
 * what we previously soft-locked.
 */
static int
segmf_unmap(struct seg *seg, caddr_t addr, size_t len)
{
	struct segmf_data *data = seg->s_data;
	offset_t off;

	if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
	    (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
		panic("segmf_unmap");

	if (addr != seg->s_base || len != seg->s_size)
		return (ENOTSUP);

	hat_unload(seg->s_as->a_hat, addr, len,
	    HAT_UNLOAD_UNMAP | HAT_UNLOAD_UNLOCK);

	off = (offset_t)seg_page(seg, addr);

	ASSERT(data->vp != NULL);

	(void) VOP_DELMAP(VTOCVP(data->vp), off, seg->s_as, addr, len,
	    data->prot, data->maxprot, MAP_SHARED, CRED(), NULL);

	seg_free(seg);
	return (0);
}

static void
segmf_free(struct seg *seg)
{
	struct segmf_data *data = seg->s_data;
	pgcnt_t npages = seg_pages(seg);

	kmem_free(data->map, npages * sizeof (segmf_map_t));
	VN_RELE(data->vp);
	mutex_destroy(&data->lock);
	kmem_free(data, sizeof (*data));
}

static int segmf_faultpage_debug = 0;
/*ARGSUSED*/
static int
segmf_faultpage(struct hat *hat, struct seg *seg, caddr_t addr,
    enum fault_type type, uint_t prot)
{
	struct segmf_data *data = seg->s_data;
	uint_t hat_flags = HAT_LOAD_NOCONSIST;
	mfn_t mfn;
	x86pte_t pte;
	segmf_map_t *map;
	uint_t idx;


	idx = seg_page(seg, addr);
	map = &data->map[idx];
	ASSERT(map->t_type == SEGMF_MAP_MFN);

	mfn = map->u.m.m_mfn;

	if (type == F_SOFTLOCK) {
		mutex_enter(&freemem_lock);
		data->softlockcnt++;
		mutex_exit(&freemem_lock);
		hat_flags |= HAT_LOAD_LOCK;
	} else
		hat_flags |= HAT_LOAD;

	if (segmf_faultpage_debug > 0) {
		uprintf("segmf_faultpage: addr %p domid %x mfn %lx prot %x\n",
		    (void *)addr, data->domid, mfn, prot);
		segmf_faultpage_debug--;
	}

	/*
	 * Ask the HAT to load a throwaway mapping to page zero, then
	 * overwrite it with our foreign domain mapping. It gets removed
	 * later via hat_unload()
	 */
	hat_devload(hat, addr, MMU_PAGESIZE, (pfn_t)0,
	    PROT_READ | HAT_UNORDERED_OK, hat_flags);

	pte = mmu_ptob((x86pte_t)mfn) | PT_VALID | PT_USER | PT_FOREIGN;
	if (prot & PROT_WRITE)
		pte |= PT_WRITABLE;

	if (HYPERVISOR_update_va_mapping_otherdomain((uintptr_t)addr, pte,
	    UVMF_INVLPG | UVMF_ALL, data->domid) != 0) {
		hat_flags = HAT_UNLOAD_UNMAP;

		if (type == F_SOFTLOCK) {
			hat_flags |= HAT_UNLOAD_UNLOCK;
			mutex_enter(&freemem_lock);
			data->softlockcnt--;
			mutex_exit(&freemem_lock);
		}

		hat_unload(hat, addr, MMU_PAGESIZE, hat_flags);
		return (FC_MAKE_ERR(EFAULT));
	}

	return (0);
}

static int
seg_rw_to_prot(enum seg_rw rw)
{
	switch (rw) {
	case S_READ:
		return (PROT_READ);
	case S_WRITE:
		return (PROT_WRITE);
	case S_EXEC:
		return (PROT_EXEC);
	case S_OTHER:
	default:
		break;
	}
	return (PROT_READ | PROT_WRITE | PROT_EXEC);
}

static void
segmf_softunlock(struct hat *hat, struct seg *seg, caddr_t addr, size_t len)
{
	struct segmf_data *data = seg->s_data;

	hat_unlock(hat, addr, len);

	mutex_enter(&freemem_lock);
	ASSERT(data->softlockcnt >= btopr(len));
	data->softlockcnt -= btopr(len);
	mutex_exit(&freemem_lock);

	if (data->softlockcnt == 0) {
		struct as *as = seg->s_as;

		if (AS_ISUNMAPWAIT(as)) {
			mutex_enter(&as->a_contents);
			if (AS_ISUNMAPWAIT(as)) {
				AS_CLRUNMAPWAIT(as);
				cv_broadcast(&as->a_cv);
			}
			mutex_exit(&as->a_contents);
		}
	}
}

static int
segmf_fault_range(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
    enum fault_type type, enum seg_rw rw)
{
	struct segmf_data *data = seg->s_data;
	int error = 0;
	caddr_t a;

	if ((data->prot & seg_rw_to_prot(rw)) == 0)
		return (FC_PROT);

	/* loop over the address range handling each fault */

	for (a = addr; a < addr + len; a += PAGESIZE) {
		error = segmf_faultpage(hat, seg, a, type, data->prot);
		if (error != 0)
			break;
	}

	if (error != 0 && type == F_SOFTLOCK) {
		size_t done = (size_t)(a - addr);

		/*
		 * Undo what's been done so far.
		 */
		if (done > 0)
			segmf_softunlock(hat, seg, addr, done);
	}

	return (error);
}

/*
 * We never demand-fault for seg_mf.
 */
/*ARGSUSED*/
static int
segmf_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
    enum fault_type type, enum seg_rw rw)
{
	return (FC_MAKE_ERR(EFAULT));
}

/*ARGSUSED*/
static int
segmf_faulta(struct seg *seg, caddr_t addr)
{
	return (0);
}

/*ARGSUSED*/
static int
segmf_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
{
	return (EINVAL);
}

/*ARGSUSED*/
static int
segmf_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
{
	return (EINVAL);
}

/*ARGSUSED*/
static int
segmf_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
{
	return (-1);
}

/*ARGSUSED*/
static int
segmf_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
{
	return (0);
}

/*
 * XXPV	Hmm.  Should we say that mf mapping are "in core?"
 */

/*ARGSUSED*/
static size_t
segmf_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
{
	size_t v;

	for (v = 0, len = (len + PAGEOFFSET) & PAGEMASK; len;
	    len -= PAGESIZE, v += PAGESIZE)
		*vec++ = 1;
	return (v);
}

/*ARGSUSED*/
static int
segmf_lockop(struct seg *seg, caddr_t addr,
    size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
{
	return (0);
}

static int
segmf_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
{
	struct segmf_data *data = seg->s_data;
	pgcnt_t pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;

	if (pgno != 0) {
		do
			protv[--pgno] = data->prot;
		while (pgno != 0)
			;
	}
	return (0);
}

static u_offset_t
segmf_getoffset(struct seg *seg, caddr_t addr)
{
	return (addr - seg->s_base);
}

/*ARGSUSED*/
static int
segmf_gettype(struct seg *seg, caddr_t addr)
{
	return (MAP_SHARED);
}

/*ARGSUSED1*/
static int
segmf_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
{
	struct segmf_data *data = seg->s_data;

	*vpp = VTOCVP(data->vp);
	return (0);
}

/*ARGSUSED*/
static int
segmf_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
{
	return (0);
}

/*ARGSUSED*/
static void
segmf_dump(struct seg *seg)
{}

/*ARGSUSED*/
static int
segmf_pagelock(struct seg *seg, caddr_t addr, size_t len,
    struct page ***ppp, enum lock_type type, enum seg_rw rw)
{
	return (ENOTSUP);
}

/*ARGSUSED*/
static int
segmf_setpagesize(struct seg *seg, caddr_t addr, size_t len, uint_t szc)
{
	return (ENOTSUP);
}

static int
segmf_getmemid(struct seg *seg, caddr_t addr, memid_t *memid)
{
	struct segmf_data *data = seg->s_data;

	memid->val[0] = (uintptr_t)VTOCVP(data->vp);
	memid->val[1] = (uintptr_t)seg_page(seg, addr);
	return (0);
}

/*ARGSUSED*/
static lgrp_mem_policy_info_t *
segmf_getpolicy(struct seg *seg, caddr_t addr)
{
	return (NULL);
}

/*ARGSUSED*/
static int
segmf_capable(struct seg *seg, segcapability_t capability)
{
	return (0);
}

/*
 * Add a set of contiguous foreign MFNs to the segment. soft-locking them.  The
 * pre-faulting is necessary due to live migration; in particular we must
 * return an error in response to IOCTL_PRIVCMD_MMAPBATCH rather than faulting
 * later on a bad MFN.  Whilst this isn't necessary for the other MMAP
 * ioctl()s, we lock them too, as they should be transitory.
 */
int
segmf_add_mfns(struct seg *seg, caddr_t addr, mfn_t mfn,
    pgcnt_t pgcnt, domid_t domid)
{
	struct segmf_data *data = seg->s_data;
	pgcnt_t base;
	faultcode_t fc;
	pgcnt_t i;
	int error = 0;

	if (seg->s_ops != &segmf_ops)
		return (EINVAL);

	/*
	 * Don't mess with dom0.
	 *
	 * Only allow the domid to be set once for the segment.
	 * After that attempts to add mappings to this segment for
	 * other domains explicitly fails.
	 */

	if (domid == 0 || domid == DOMID_SELF)
		return (EACCES);

	mutex_enter(&data->lock);

	if (data->domid == 0)
		data->domid = domid;

	if (data->domid != domid) {
		error = EINVAL;
		goto out;
	}

	base = seg_page(seg, addr);

	for (i = 0; i < pgcnt; i++) {
		data->map[base + i].t_type = SEGMF_MAP_MFN;
		data->map[base + i].u.m.m_mfn = mfn++;
	}

	fc = segmf_fault_range(seg->s_as->a_hat, seg, addr,
	    pgcnt * MMU_PAGESIZE, F_SOFTLOCK, S_OTHER);

	if (fc != 0) {
		error = fc_decode(fc);
		for (i = 0; i < pgcnt; i++) {
			data->map[base + i].t_type = SEGMF_MAP_EMPTY;
		}
	}

out:
	mutex_exit(&data->lock);
	return (error);
}

int
segmf_add_grefs(struct seg *seg, caddr_t addr, uint_t flags,
    grant_ref_t *grefs, uint_t cnt, domid_t domid)
{
	struct segmf_data *data;
	segmf_map_t *map;
	faultcode_t fc;
	uint_t idx;
	uint_t i;
	int e;

	if (seg->s_ops != &segmf_ops)
		return (EINVAL);

	/*
	 * Don't mess with dom0.
	 *
	 * Only allow the domid to be set once for the segment.
	 * After that attempts to add mappings to this segment for
	 * other domains explicitly fails.
	 */

	if (domid == 0 || domid == DOMID_SELF)
		return (EACCES);

	data = seg->s_data;
	idx = seg_page(seg, addr);
	map = &data->map[idx];
	e = 0;

	mutex_enter(&data->lock);

	if (data->domid == 0)
		data->domid = domid;

	if (data->domid != domid) {
		e = EINVAL;
		goto out;
	}

	/* store away the grefs passed in then fault in the pages */
	for (i = 0; i < cnt; i++) {
		map[i].t_type = SEGMF_MAP_GREF;
		map[i].u.g.g_gref = grefs[i];
		map[i].u.g.g_handle = 0;
		map[i].u.g.g_flags = 0;
		if (flags & SEGMF_GREF_WR) {
			map[i].u.g.g_flags |= SEGMF_GFLAGS_WR;
		}
	}
	fc = segmf_fault_gref_range(seg, addr, cnt);
	if (fc != 0) {
		e = fc_decode(fc);
		for (i = 0; i < cnt; i++) {
			data->map[i].t_type = SEGMF_MAP_EMPTY;
		}
	}

out:
	mutex_exit(&data->lock);
	return (e);
}

int
segmf_release_grefs(struct seg *seg, caddr_t addr, uint_t cnt)
{
	gnttab_unmap_grant_ref_t mapop[SEGMF_MAX_GREFS];
	struct segmf_data *data;
	segmf_map_t *map;
	uint_t idx;
	long e;
	int i;
	int n;


	if (cnt > SEGMF_MAX_GREFS) {
		return (-1);
	}

	idx = seg_page(seg, addr);
	data = seg->s_data;
	map = &data->map[idx];

	bzero(mapop, sizeof (gnttab_unmap_grant_ref_t) * cnt);

	/*
	 * for each entry which isn't empty and is currently mapped,
	 * set it up for an unmap then mark them empty.
	 */
	n = 0;
	for (i = 0; i < cnt; i++) {
		ASSERT(map[i].t_type != SEGMF_MAP_MFN);
		if ((map[i].t_type == SEGMF_MAP_GREF) &&
		    (map[i].u.g.g_flags & SEGMF_GFLAGS_MAPPED)) {
			mapop[n].handle = map[i].u.g.g_handle;
			mapop[n].host_addr = map[i].u.g.g_ptep;
			mapop[n].dev_bus_addr = 0;
			n++;
		}
		map[i].t_type = SEGMF_MAP_EMPTY;
	}

	/* if there's nothing to unmap, just return */
	if (n == 0) {
		return (0);
	}

	e = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, &mapop, n);
	if (e != 0) {
		return (-1);
	}

	return (0);
}


void
segmf_add_gref_pte(struct seg *seg, caddr_t addr, uint64_t pte_ma)
{
	struct segmf_data *data;
	uint_t idx;

	idx = seg_page(seg, addr);
	data = seg->s_data;

	data->map[idx].u.g.g_ptep = pte_ma;
}


static int
segmf_fault_gref_range(struct seg *seg, caddr_t addr, size_t cnt)
{
	gnttab_map_grant_ref_t mapop[SEGMF_MAX_GREFS];
	struct segmf_data *data;
	segmf_map_t *map;
	uint_t idx;
	int e;
	int i;


	if (cnt > SEGMF_MAX_GREFS) {
		return (-1);
	}

	data = seg->s_data;
	idx = seg_page(seg, addr);
	map = &data->map[idx];

	bzero(mapop, sizeof (gnttab_map_grant_ref_t) * cnt);

	ASSERT(map->t_type == SEGMF_MAP_GREF);

	/*
	 * map in each page passed in into the user apps AS. We do this by
	 * passing the MA of the actual pte of the mapping to the hypervisor.
	 */
	for (i = 0; i < cnt; i++) {
		mapop[i].host_addr = map[i].u.g.g_ptep;
		mapop[i].dom = data->domid;
		mapop[i].ref = map[i].u.g.g_gref;
		mapop[i].flags = GNTMAP_host_map | GNTMAP_application_map |
		    GNTMAP_contains_pte;
		if (!(map[i].u.g.g_flags & SEGMF_GFLAGS_WR)) {
			mapop[i].flags |= GNTMAP_readonly;
		}
	}
	e = xen_map_gref(GNTTABOP_map_grant_ref, mapop, cnt, B_TRUE);
	if ((e != 0) || (mapop[0].status != GNTST_okay)) {
		return (FC_MAKE_ERR(EFAULT));
	}

	/* save handle for segmf_release_grefs() and mark it as mapped */
	for (i = 0; i < cnt; i++) {
		ASSERT(mapop[i].status == GNTST_okay);
		map[i].u.g.g_handle = mapop[i].handle;
		map[i].u.g.g_flags |= SEGMF_GFLAGS_MAPPED;
	}

	return (0);
}

static struct seg_ops segmf_ops = {
	segmf_dup,
	segmf_unmap,
	segmf_free,
	segmf_fault,
	segmf_faulta,
	segmf_setprot,
	segmf_checkprot,
	(int (*)())segmf_kluster,
	(size_t (*)(struct seg *))NULL,	/* swapout */
	segmf_sync,
	segmf_incore,
	segmf_lockop,
	segmf_getprot,
	segmf_getoffset,
	segmf_gettype,
	segmf_getvp,
	segmf_advise,
	segmf_dump,
	segmf_pagelock,
	segmf_setpagesize,
	segmf_getmemid,
	segmf_getpolicy,
	segmf_capable,
	seg_inherit_notsup
};