xref: /illumos-gate/usr/src/uts/common/io/mlxcx/mlxcx_dma.c (revision ebb7c6fd)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2020, The University of Queensland
 * Copyright (c) 2018, Joyent, Inc.
 */

/*
 * DMA allocation and tear down routines.
 */

#include <mlxcx.h>

void
mlxcx_dma_acc_attr(mlxcx_t *mlxp, ddi_device_acc_attr_t *accp)
{
	bzero(accp, sizeof (*accp));
	accp->devacc_attr_version = DDI_DEVICE_ATTR_V0;
	accp->devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
	accp->devacc_attr_dataorder = DDI_STRICTORDER_ACC;

	if (DDI_FM_DMA_ERR_CAP(mlxp->mlx_fm_caps)) {
		accp->devacc_attr_access = DDI_FLAGERR_ACC;
	} else {
		accp->devacc_attr_access = DDI_DEFAULT_ACC;
	}
}

void
mlxcx_dma_page_attr(mlxcx_t *mlxp, ddi_dma_attr_t *attrp)
{
	bzero(attrp, sizeof (*attrp));
	attrp->dma_attr_version = DMA_ATTR_V0;

	/*
	 * This is a 64-bit PCIe device. We can use the entire address space.
	 */
	attrp->dma_attr_addr_lo = 0x0;
	attrp->dma_attr_addr_hi = UINT64_MAX;

	/*
	 * The count max indicates the total amount that can fit into one
	 * cookie. Because we're creating a single page for tracking purposes,
	 * this can be a page in size. The alignment and segment are related to
	 * this same requirement. The alignment needs to be page aligned and the
	 * segment is the boundary that this can't cross, aka a 4k page.
	 */
	attrp->dma_attr_count_max = MLXCX_CMD_DMA_PAGE_SIZE - 1;
	attrp->dma_attr_align = MLXCX_CMD_DMA_PAGE_SIZE;
	attrp->dma_attr_seg = MLXCX_CMD_DMA_PAGE_SIZE - 1;

	attrp->dma_attr_burstsizes = 0xfff;

	/*
	 * The minimum and and maximum sizes that we can send. We cap this based
	 * on the use of this, which is a page size.
	 */
	attrp->dma_attr_minxfer = 0x1;
	attrp->dma_attr_maxxfer = MLXCX_CMD_DMA_PAGE_SIZE;

	/*
	 * This is supposed to be used for static data structures, therefore we
	 * keep this just to a page.
	 */
	attrp->dma_attr_sgllen = 1;

	/*
	 * The granularity describe the addressing graularity. That is, the
	 * hardware can ask for chunks in this units of bytes.
	 */
	attrp->dma_attr_granular = MLXCX_CMD_DMA_PAGE_SIZE;

	if (DDI_FM_DMA_ERR_CAP(mlxp->mlx_fm_caps)) {
		attrp->dma_attr_flags = DDI_DMA_FLAGERR;
	} else {
		attrp->dma_attr_flags = 0;
	}
}

/*
 * DMA attributes for queue memory (EQ, CQ, WQ etc)
 *
 * These have to allocate in units of whole pages, but can be multiple
 * pages and don't have to be physically contiguous.
 */
void
mlxcx_dma_queue_attr(mlxcx_t *mlxp, ddi_dma_attr_t *attrp)
{
	bzero(attrp, sizeof (*attrp));
	attrp->dma_attr_version = DMA_ATTR_V0;

	/*
	 * This is a 64-bit PCIe device. We can use the entire address space.
	 */
	attrp->dma_attr_addr_lo = 0x0;
	attrp->dma_attr_addr_hi = UINT64_MAX;

	attrp->dma_attr_count_max = MLXCX_QUEUE_DMA_PAGE_SIZE - 1;

	attrp->dma_attr_align = MLXCX_QUEUE_DMA_PAGE_SIZE;

	attrp->dma_attr_burstsizes = 0xfff;

	/*
	 * The minimum and and maximum sizes that we can send. We cap this based
	 * on the use of this, which is a page size.
	 */
	attrp->dma_attr_minxfer = MLXCX_QUEUE_DMA_PAGE_SIZE;
	attrp->dma_attr_maxxfer = UINT32_MAX;

	attrp->dma_attr_seg = UINT64_MAX;

	attrp->dma_attr_granular = MLXCX_QUEUE_DMA_PAGE_SIZE;

	/* But we can have more than one. */
	attrp->dma_attr_sgllen = MLXCX_CREATE_QUEUE_MAX_PAGES;

	if (DDI_FM_DMA_ERR_CAP(mlxp->mlx_fm_caps)) {
		attrp->dma_attr_flags = DDI_DMA_FLAGERR;
	} else {
		attrp->dma_attr_flags = 0;
	}
}

/*
 * DMA attributes for packet buffers
 */
void
mlxcx_dma_buf_attr(mlxcx_t *mlxp, ddi_dma_attr_t *attrp)
{
	bzero(attrp, sizeof (*attrp));
	attrp->dma_attr_version = DMA_ATTR_V0;

	/*
	 * This is a 64-bit PCIe device. We can use the entire address space.
	 */
	attrp->dma_attr_addr_lo = 0x0;
	attrp->dma_attr_addr_hi = UINT64_MAX;

	/*
	 * Each scatter pointer has a 32-bit length field.
	 */
	attrp->dma_attr_count_max = UINT32_MAX;

	/*
	 * The PRM gives us no alignment requirements for scatter pointers,
	 * but it implies that units < 16bytes are a bad idea.
	 */
	attrp->dma_attr_align = 16;
	attrp->dma_attr_granular = 1;

	attrp->dma_attr_burstsizes = 0xfff;

	attrp->dma_attr_minxfer = 1;
	attrp->dma_attr_maxxfer = UINT64_MAX;

	attrp->dma_attr_seg = UINT64_MAX;

	/*
	 * We choose how many scatter pointers we're allowed per packet when
	 * we set the recv queue stride. This macro is from mlxcx_reg.h where
	 * we fix that for all of our receive queues.
	 */
	attrp->dma_attr_sgllen = MLXCX_RECVQ_MAX_PTRS;

	if (DDI_FM_DMA_ERR_CAP(mlxp->mlx_fm_caps)) {
		attrp->dma_attr_flags = DDI_DMA_FLAGERR;
	} else {
		attrp->dma_attr_flags = 0;
	}
}

/*
 * DMA attributes for queue doorbells
 */
void
mlxcx_dma_qdbell_attr(mlxcx_t *mlxp, ddi_dma_attr_t *attrp)
{
	bzero(attrp, sizeof (*attrp));
	attrp->dma_attr_version = DMA_ATTR_V0;

	/*
	 * This is a 64-bit PCIe device. We can use the entire address space.
	 */
	attrp->dma_attr_addr_lo = 0x0;
	attrp->dma_attr_addr_hi = UINT64_MAX;

	/*
	 * Queue doorbells are always exactly 16 bytes in length, but
	 * the ddi_dma functions don't like such small values of count_max.
	 *
	 * We tell some lies here.
	 */
	attrp->dma_attr_count_max = MLXCX_QUEUE_DMA_PAGE_SIZE - 1;
	attrp->dma_attr_align = 8;
	attrp->dma_attr_burstsizes = 0x8;
	attrp->dma_attr_minxfer = 1;
	attrp->dma_attr_maxxfer = UINT16_MAX;
	attrp->dma_attr_seg = MLXCX_QUEUE_DMA_PAGE_SIZE - 1;
	attrp->dma_attr_granular = 1;
	attrp->dma_attr_sgllen = 1;

	if (DDI_FM_DMA_ERR_CAP(mlxp->mlx_fm_caps)) {
		attrp->dma_attr_flags = DDI_DMA_FLAGERR;
	} else {
		attrp->dma_attr_flags = 0;
	}
}

void
mlxcx_dma_free(mlxcx_dma_buffer_t *mxdb)
{
	int ret;

	if (mxdb->mxdb_flags & MLXCX_DMABUF_BOUND) {
		VERIFY(mxdb->mxdb_dma_handle != NULL);
		ret = ddi_dma_unbind_handle(mxdb->mxdb_dma_handle);
		VERIFY3S(ret, ==, DDI_SUCCESS);
		mxdb->mxdb_flags &= ~MLXCX_DMABUF_BOUND;
		mxdb->mxdb_ncookies = 0;
	}

	if (mxdb->mxdb_flags & MLXCX_DMABUF_MEM_ALLOC) {
		ddi_dma_mem_free(&mxdb->mxdb_acc_handle);
		mxdb->mxdb_acc_handle = NULL;
		mxdb->mxdb_va = NULL;
		mxdb->mxdb_len = 0;
		mxdb->mxdb_flags &= ~MLXCX_DMABUF_MEM_ALLOC;
	}

	if (mxdb->mxdb_flags & MLXCX_DMABUF_FOREIGN) {
		/* The mblk will be freed separately */
		mxdb->mxdb_va = NULL;
		mxdb->mxdb_len = 0;
		mxdb->mxdb_flags &= ~MLXCX_DMABUF_FOREIGN;
	}

	if (mxdb->mxdb_flags & MLXCX_DMABUF_HDL_ALLOC) {
		ddi_dma_free_handle(&mxdb->mxdb_dma_handle);
		mxdb->mxdb_dma_handle = NULL;
		mxdb->mxdb_flags &= ~MLXCX_DMABUF_HDL_ALLOC;
	}

	ASSERT3U(mxdb->mxdb_flags, ==, 0);
	ASSERT3P(mxdb->mxdb_dma_handle, ==, NULL);
	ASSERT3P(mxdb->mxdb_va, ==, NULL);
	ASSERT3U(mxdb->mxdb_len, ==, 0);
	ASSERT3U(mxdb->mxdb_ncookies, ==, 0);
}

void
mlxcx_dma_unbind(mlxcx_t *mlxp, mlxcx_dma_buffer_t *mxdb)
{
	int ret;

	ASSERT(mxdb->mxdb_flags & MLXCX_DMABUF_HDL_ALLOC);
	ASSERT(mxdb->mxdb_flags & MLXCX_DMABUF_BOUND);

	if (mxdb->mxdb_flags & MLXCX_DMABUF_FOREIGN) {
		/* The mblk will be freed separately */
		mxdb->mxdb_va = NULL;
		mxdb->mxdb_len = 0;
		mxdb->mxdb_flags &= ~MLXCX_DMABUF_FOREIGN;
	}

	ret = ddi_dma_unbind_handle(mxdb->mxdb_dma_handle);
	VERIFY3S(ret, ==, DDI_SUCCESS);
	mxdb->mxdb_flags &= ~MLXCX_DMABUF_BOUND;
	mxdb->mxdb_ncookies = 0;
}

boolean_t
mlxcx_dma_init(mlxcx_t *mlxp, mlxcx_dma_buffer_t *mxdb,
    ddi_dma_attr_t *attrp, boolean_t wait)
{
	int ret;
	int (*memcb)(caddr_t);

	if (wait == B_TRUE) {
		memcb = DDI_DMA_SLEEP;
	} else {
		memcb = DDI_DMA_DONTWAIT;
	}

	ASSERT3S(mxdb->mxdb_flags, ==, 0);

	ret = ddi_dma_alloc_handle(mlxp->mlx_dip, attrp, memcb, NULL,
	    &mxdb->mxdb_dma_handle);
	if (ret != 0) {
		mlxcx_warn(mlxp, "!failed to allocate DMA handle: %d", ret);
		mxdb->mxdb_dma_handle = NULL;
		return (B_FALSE);
	}
	mxdb->mxdb_flags |= MLXCX_DMABUF_HDL_ALLOC;

	return (B_TRUE);
}

boolean_t
mlxcx_dma_bind_mblk(mlxcx_t *mlxp, mlxcx_dma_buffer_t *mxdb,
    const mblk_t *mp, size_t off, boolean_t wait)
{
	int ret;
	uint_t flags = DDI_DMA_STREAMING;
	int (*memcb)(caddr_t);

	if (wait == B_TRUE) {
		memcb = DDI_DMA_SLEEP;
	} else {
		memcb = DDI_DMA_DONTWAIT;
	}

	ASSERT(mxdb->mxdb_flags & MLXCX_DMABUF_HDL_ALLOC);
	ASSERT0(mxdb->mxdb_flags &
	    (MLXCX_DMABUF_FOREIGN | MLXCX_DMABUF_MEM_ALLOC));
	ASSERT0(mxdb->mxdb_flags & MLXCX_DMABUF_BOUND);

	ASSERT3U(off, <=, MBLKL(mp));
	mxdb->mxdb_va = (caddr_t)(mp->b_rptr + off);
	mxdb->mxdb_len = MBLKL(mp) - off;
	mxdb->mxdb_flags |= MLXCX_DMABUF_FOREIGN;

	ret = ddi_dma_addr_bind_handle(mxdb->mxdb_dma_handle, NULL,
	    mxdb->mxdb_va, mxdb->mxdb_len, DDI_DMA_WRITE | flags, memcb, NULL,
	    NULL, NULL);
	if (ret != DDI_DMA_MAPPED) {
		mxdb->mxdb_va = NULL;
		mxdb->mxdb_len = 0;
		mxdb->mxdb_flags &= ~MLXCX_DMABUF_FOREIGN;
		return (B_FALSE);
	}
	mxdb->mxdb_flags |= MLXCX_DMABUF_BOUND;
	mxdb->mxdb_ncookies = ddi_dma_ncookies(mxdb->mxdb_dma_handle);

	return (B_TRUE);
}

boolean_t
mlxcx_dma_alloc(mlxcx_t *mlxp, mlxcx_dma_buffer_t *mxdb,
    ddi_dma_attr_t *attrp, ddi_device_acc_attr_t *accp, boolean_t zero,
    size_t size, boolean_t wait)
{
	int ret;
	uint_t flags = DDI_DMA_CONSISTENT;
	size_t len;
	int (*memcb)(caddr_t);

	if (wait == B_TRUE) {
		memcb = DDI_DMA_SLEEP;
	} else {
		memcb = DDI_DMA_DONTWAIT;
	}

	ASSERT3U(mxdb->mxdb_flags, ==, 0);

	ret = ddi_dma_alloc_handle(mlxp->mlx_dip, attrp, memcb, NULL,
	    &mxdb->mxdb_dma_handle);
	if (ret != 0) {
		mlxcx_warn(mlxp, "!failed to allocate DMA handle: %d", ret);
		mxdb->mxdb_dma_handle = NULL;
		return (B_FALSE);
	}
	mxdb->mxdb_flags |= MLXCX_DMABUF_HDL_ALLOC;

	ret = ddi_dma_mem_alloc(mxdb->mxdb_dma_handle, size, accp, flags, memcb,
	    NULL, &mxdb->mxdb_va, &len, &mxdb->mxdb_acc_handle);
	if (ret != DDI_SUCCESS) {
		mlxcx_warn(mlxp, "!failed to allocate DMA memory: %d", ret);
		mxdb->mxdb_va = NULL;
		mxdb->mxdb_acc_handle = NULL;
		mlxcx_dma_free(mxdb);
		return (B_FALSE);
	}
	mxdb->mxdb_len = size;
	mxdb->mxdb_flags |= MLXCX_DMABUF_MEM_ALLOC;

	if (zero == B_TRUE)
		bzero(mxdb->mxdb_va, len);

	ret = ddi_dma_addr_bind_handle(mxdb->mxdb_dma_handle, NULL,
	    mxdb->mxdb_va, len, DDI_DMA_RDWR | flags, memcb, NULL, NULL,
	    NULL);
	if (ret != 0) {
		mlxcx_warn(mlxp, "!failed to bind DMA memory: %d", ret);
		mlxcx_dma_free(mxdb);
		return (B_FALSE);
	}
	mxdb->mxdb_flags |= MLXCX_DMABUF_BOUND;
	mxdb->mxdb_ncookies = ddi_dma_ncookies(mxdb->mxdb_dma_handle);

	return (B_TRUE);
}

boolean_t
mlxcx_dma_alloc_offset(mlxcx_t *mlxp, mlxcx_dma_buffer_t *mxdb,
    ddi_dma_attr_t *attrp, ddi_device_acc_attr_t *accp, boolean_t zero,
    size_t size, size_t offset, boolean_t wait)
{
	int ret;
	uint_t flags = DDI_DMA_STREAMING;
	size_t len;
	int (*memcb)(caddr_t);

	if (wait == B_TRUE) {
		memcb = DDI_DMA_SLEEP;
	} else {
		memcb = DDI_DMA_DONTWAIT;
	}

	ASSERT3U(mxdb->mxdb_flags, ==, 0);

	ret = ddi_dma_alloc_handle(mlxp->mlx_dip, attrp, memcb, NULL,
	    &mxdb->mxdb_dma_handle);
	if (ret != 0) {
		mlxcx_warn(mlxp, "!failed to allocate DMA handle: %d", ret);
		mxdb->mxdb_dma_handle = NULL;
		return (B_FALSE);
	}
	mxdb->mxdb_flags |= MLXCX_DMABUF_HDL_ALLOC;

	ret = ddi_dma_mem_alloc(mxdb->mxdb_dma_handle, size + offset, accp,
	    flags, memcb, NULL, &mxdb->mxdb_va, &len, &mxdb->mxdb_acc_handle);
	if (ret != DDI_SUCCESS) {
		mlxcx_warn(mlxp, "!failed to allocate DMA memory: %d", ret);
		mxdb->mxdb_va = NULL;
		mxdb->mxdb_acc_handle = NULL;
		mlxcx_dma_free(mxdb);
		return (B_FALSE);
	}

	if (zero == B_TRUE)
		bzero(mxdb->mxdb_va, len);

	mxdb->mxdb_va += offset;
	len -= offset;
	mxdb->mxdb_len = len;
	mxdb->mxdb_flags |= MLXCX_DMABUF_MEM_ALLOC;

	ret = ddi_dma_addr_bind_handle(mxdb->mxdb_dma_handle, NULL,
	    mxdb->mxdb_va, len, DDI_DMA_RDWR | flags, memcb, NULL, NULL,
	    NULL);
	if (ret != 0) {
		mlxcx_warn(mlxp, "!failed to bind DMA memory: %d", ret);
		mlxcx_dma_free(mxdb);
		return (B_FALSE);
	}
	mxdb->mxdb_flags |= MLXCX_DMABUF_BOUND;
	mxdb->mxdb_ncookies = ddi_dma_ncookies(mxdb->mxdb_dma_handle);

	return (B_TRUE);
}