1fcf3ce44SJohn Forte /*
2fcf3ce44SJohn Forte  * CDDL HEADER START
3fcf3ce44SJohn Forte  *
4fcf3ce44SJohn Forte  * The contents of this file are subject to the terms of the
5fcf3ce44SJohn Forte  * Common Development and Distribution License (the "License").
6fcf3ce44SJohn Forte  * You may not use this file except in compliance with the License.
7fcf3ce44SJohn Forte  *
8fcf3ce44SJohn Forte  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fcf3ce44SJohn Forte  * or http://www.opensolaris.org/os/licensing.
10fcf3ce44SJohn Forte  * See the License for the specific language governing permissions
11fcf3ce44SJohn Forte  * and limitations under the License.
12fcf3ce44SJohn Forte  *
13fcf3ce44SJohn Forte  * When distributing Covered Code, include this CDDL HEADER in each
14fcf3ce44SJohn Forte  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fcf3ce44SJohn Forte  * If applicable, add the following below this CDDL HEADER, with the
16fcf3ce44SJohn Forte  * fields enclosed by brackets "[]" replaced with your own identifying
17fcf3ce44SJohn Forte  * information: Portions Copyright [yyyy] [name of copyright owner]
18fcf3ce44SJohn Forte  *
19fcf3ce44SJohn Forte  * CDDL HEADER END
20fcf3ce44SJohn Forte  */
21c4ddbbe1SDaniel Beauregard 
22c4ddbbe1SDaniel Beauregard /*
23c4ddbbe1SDaniel Beauregard  * Copyright 2009 QLogic Corporation.  All rights reserved.
24c4ddbbe1SDaniel Beauregard  * Use is subject to license terms.
25c4ddbbe1SDaniel Beauregard  */
26c4ddbbe1SDaniel Beauregard 
27fcf3ce44SJohn Forte /*
283fb517f7SJames Moore  * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
29fcf3ce44SJohn Forte  */
30fcf3ce44SJohn Forte 
31fcf3ce44SJohn Forte #include <sys/conf.h>
32fcf3ce44SJohn Forte #include <sys/ddi.h>
33fcf3ce44SJohn Forte #include <sys/sunddi.h>
34fcf3ce44SJohn Forte #include <sys/modctl.h>
35fcf3ce44SJohn Forte 
36*4558d122SViswanathan Kannappan #include <sys/stmf_defines.h>
37*4558d122SViswanathan Kannappan #include <sys/fct_defines.h>
38*4558d122SViswanathan Kannappan #include <sys/stmf.h>
39*4558d122SViswanathan Kannappan #include <sys/portif.h>
40*4558d122SViswanathan Kannappan #include <sys/fct.h>
41*4558d122SViswanathan Kannappan 
42*4558d122SViswanathan Kannappan #include "qlt.h"
43*4558d122SViswanathan Kannappan #include "qlt_dma.h"
44fcf3ce44SJohn Forte 
453fb517f7SJames Moore /*
463fb517f7SJames Moore  *  Local Function Prototypes.
473fb517f7SJames Moore  */
483fb517f7SJames Moore static void
493fb517f7SJames Moore qlt_dma_free_handles(qlt_state_t *qlt, qlt_dma_handle_t *first_handle);
503fb517f7SJames Moore 
51fcf3ce44SJohn Forte #define	BUF_COUNT_2K		2048
52fcf3ce44SJohn Forte #define	BUF_COUNT_8K		512
533f3ce7b9SDaniel Beauregard #define	BUF_COUNT_64K		256
543f3ce7b9SDaniel Beauregard #define	BUF_COUNT_128K		1024
55fcf3ce44SJohn Forte #define	BUF_COUNT_256K		8
56fcf3ce44SJohn Forte 
57fcf3ce44SJohn Forte #define	QLT_DMEM_MAX_BUF_SIZE	(4 * 65536)
58fcf3ce44SJohn Forte #define	QLT_DMEM_NBUCKETS	5
59fcf3ce44SJohn Forte static qlt_dmem_bucket_t bucket2K	= { 2048, BUF_COUNT_2K },
60fcf3ce44SJohn Forte 			bucket8K	= { 8192, BUF_COUNT_8K },
61fcf3ce44SJohn Forte 			bucket64K	= { 65536, BUF_COUNT_64K },
62fcf3ce44SJohn Forte 			bucket128k	= { (2 * 65536), BUF_COUNT_128K },
63fcf3ce44SJohn Forte 			bucket256k	= { (4 * 65536), BUF_COUNT_256K };
64fcf3ce44SJohn Forte 
65fcf3ce44SJohn Forte static qlt_dmem_bucket_t *dmem_buckets[] = { &bucket2K, &bucket8K,
66fcf3ce44SJohn Forte 			&bucket64K, &bucket128k, &bucket256k, NULL };
67fcf3ce44SJohn Forte static ddi_device_acc_attr_t acc;
68fcf3ce44SJohn Forte static ddi_dma_attr_t qlt_scsi_dma_attr = {
69fcf3ce44SJohn Forte 	DMA_ATTR_V0,		/* dma_attr_version */
70fcf3ce44SJohn Forte 	0,			/* low DMA address range */
71fcf3ce44SJohn Forte 	0xffffffffffffffff,	/* high DMA address range */
72fcf3ce44SJohn Forte 	0xffffffff,		/* DMA counter register */
73fcf3ce44SJohn Forte 	8192,			/* DMA address alignment */
74fcf3ce44SJohn Forte 	0xff,			/* DMA burstsizes */
75fcf3ce44SJohn Forte 	1,			/* min effective DMA size */
76fcf3ce44SJohn Forte 	0xffffffff,		/* max DMA xfer size */
77fcf3ce44SJohn Forte 	0xffffffff,		/* segment boundary */
78fcf3ce44SJohn Forte 	1,			/* s/g list length */
79fcf3ce44SJohn Forte 	1,			/* granularity of device */
80fcf3ce44SJohn Forte 	0			/* DMA transfer flags */
81fcf3ce44SJohn Forte };
82fcf3ce44SJohn Forte 
83fcf3ce44SJohn Forte fct_status_t
qlt_dmem_init(qlt_state_t * qlt)84fcf3ce44SJohn Forte qlt_dmem_init(qlt_state_t *qlt)
85fcf3ce44SJohn Forte {
863fb517f7SJames Moore 	qlt_dmem_bucket_t	*p;
873fb517f7SJames Moore 	qlt_dmem_bctl_t		*bctl, *bc;
883fb517f7SJames Moore 	qlt_dmem_bctl_t		*prev;
893fb517f7SJames Moore 	int			ndx, i;
903fb517f7SJames Moore 	uint32_t		total_mem;
913fb517f7SJames Moore 	uint8_t			*addr;
923fb517f7SJames Moore 	uint8_t			*host_addr;
933fb517f7SJames Moore 	uint64_t		dev_addr;
943fb517f7SJames Moore 	ddi_dma_cookie_t	cookie;
953fb517f7SJames Moore 	uint32_t		ncookie;
963fb517f7SJames Moore 	uint32_t		bsize;
973fb517f7SJames Moore 	size_t			len;
98fcf3ce44SJohn Forte 
99a2255df3SDaniel Beauregard 	if (qlt->qlt_bucketcnt[0] != 0) {
100a2255df3SDaniel Beauregard 		bucket2K.dmem_nbufs = qlt->qlt_bucketcnt[0];
101a2255df3SDaniel Beauregard 	}
102a2255df3SDaniel Beauregard 	if (qlt->qlt_bucketcnt[1] != 0) {
103a2255df3SDaniel Beauregard 		bucket8K.dmem_nbufs = qlt->qlt_bucketcnt[1];
104a2255df3SDaniel Beauregard 	}
105a2255df3SDaniel Beauregard 	if (qlt->qlt_bucketcnt[2] != 0) {
106a2255df3SDaniel Beauregard 		bucket64K.dmem_nbufs = qlt->qlt_bucketcnt[2];
107a2255df3SDaniel Beauregard 	}
108a2255df3SDaniel Beauregard 	if (qlt->qlt_bucketcnt[3] != 0) {
109a2255df3SDaniel Beauregard 		bucket128k.dmem_nbufs = qlt->qlt_bucketcnt[3];
110fcf3ce44SJohn Forte 	}
111a2255df3SDaniel Beauregard 	if (qlt->qlt_bucketcnt[4] != 0) {
112a2255df3SDaniel Beauregard 		bucket256k.dmem_nbufs = qlt->qlt_bucketcnt[4];
113a2255df3SDaniel Beauregard 	}
114a2255df3SDaniel Beauregard 
115fcf3ce44SJohn Forte 	bsize = sizeof (dmem_buckets);
116c4ddbbe1SDaniel Beauregard 	ndx = (int)(bsize / sizeof (void *));
117fcf3ce44SJohn Forte 	/*
118fcf3ce44SJohn Forte 	 * The reason it is ndx - 1 everywhere is becasue the last bucket
119fcf3ce44SJohn Forte 	 * pointer is NULL.
120fcf3ce44SJohn Forte 	 */
121fcf3ce44SJohn Forte 	qlt->dmem_buckets = (qlt_dmem_bucket_t **)kmem_zalloc(bsize +
122c4ddbbe1SDaniel Beauregard 	    ((ndx - 1) * (int)sizeof (qlt_dmem_bucket_t)), KM_SLEEP);
123fcf3ce44SJohn Forte 	for (i = 0; i < (ndx - 1); i++) {
124fcf3ce44SJohn Forte 		qlt->dmem_buckets[i] = (qlt_dmem_bucket_t *)
125c4ddbbe1SDaniel Beauregard 		    ((uint8_t *)qlt->dmem_buckets + bsize +
126c4ddbbe1SDaniel Beauregard 		    (i * (int)sizeof (qlt_dmem_bucket_t)));
127fcf3ce44SJohn Forte 		bcopy(dmem_buckets[i], qlt->dmem_buckets[i],
128c4ddbbe1SDaniel Beauregard 		    sizeof (qlt_dmem_bucket_t));
129fcf3ce44SJohn Forte 	}
130fcf3ce44SJohn Forte 	bzero(&acc, sizeof (acc));
131fcf3ce44SJohn Forte 	acc.devacc_attr_version = DDI_DEVICE_ATTR_V0;
132fcf3ce44SJohn Forte 	acc.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
133fcf3ce44SJohn Forte 	acc.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
134fcf3ce44SJohn Forte 	for (ndx = 0; (p = qlt->dmem_buckets[ndx]) != NULL; ndx++) {
135fcf3ce44SJohn Forte 		bctl = (qlt_dmem_bctl_t *)kmem_zalloc(p->dmem_nbufs *
136c4ddbbe1SDaniel Beauregard 		    sizeof (qlt_dmem_bctl_t), KM_NOSLEEP);
137c4ddbbe1SDaniel Beauregard 		if (bctl == NULL) {
138c4ddbbe1SDaniel Beauregard 			EL(qlt, "bctl==NULL\n");
139fcf3ce44SJohn Forte 			goto alloc_bctl_failed;
140c4ddbbe1SDaniel Beauregard 		}
141fcf3ce44SJohn Forte 		p->dmem_bctls_mem = bctl;
142fcf3ce44SJohn Forte 		mutex_init(&p->dmem_lock, NULL, MUTEX_DRIVER, NULL);
143c4ddbbe1SDaniel Beauregard 		if ((i = ddi_dma_alloc_handle(qlt->dip, &qlt_scsi_dma_attr,
144c4ddbbe1SDaniel Beauregard 		    DDI_DMA_SLEEP, 0, &p->dmem_dma_handle)) != DDI_SUCCESS) {
145c4ddbbe1SDaniel Beauregard 			EL(qlt, "ddi_dma_alloc_handle status=%xh\n", i);
146fcf3ce44SJohn Forte 			goto alloc_handle_failed;
147c4ddbbe1SDaniel Beauregard 		}
148fcf3ce44SJohn Forte 
149fcf3ce44SJohn Forte 		total_mem = p->dmem_buf_size * p->dmem_nbufs;
150fcf3ce44SJohn Forte 
151c4ddbbe1SDaniel Beauregard 		if ((i = ddi_dma_mem_alloc(p->dmem_dma_handle, total_mem, &acc,
152fcf3ce44SJohn Forte 		    DDI_DMA_STREAMING, DDI_DMA_DONTWAIT, 0, (caddr_t *)&addr,
153c4ddbbe1SDaniel Beauregard 		    &len, &p->dmem_acc_handle)) != DDI_SUCCESS) {
154c4ddbbe1SDaniel Beauregard 			EL(qlt, "ddi_dma_mem_alloc status=%xh\n", i);
155fcf3ce44SJohn Forte 			goto mem_alloc_failed;
156c4ddbbe1SDaniel Beauregard 		}
157fcf3ce44SJohn Forte 
158c4ddbbe1SDaniel Beauregard 		if ((i = ddi_dma_addr_bind_handle(p->dmem_dma_handle, NULL,
159fcf3ce44SJohn Forte 		    (caddr_t)addr, total_mem, DDI_DMA_RDWR | DDI_DMA_STREAMING,
160c4ddbbe1SDaniel Beauregard 		    DDI_DMA_DONTWAIT, 0, &cookie, &ncookie)) != DDI_SUCCESS) {
161c4ddbbe1SDaniel Beauregard 			EL(qlt, "ddi_dma_addr_bind_handle status=%xh\n", i);
162fcf3ce44SJohn Forte 			goto addr_bind_handle_failed;
163c4ddbbe1SDaniel Beauregard 		}
164c4ddbbe1SDaniel Beauregard 		if (ncookie != 1) {
165c4ddbbe1SDaniel Beauregard 			EL(qlt, "ncookie=%d\n", ncookie);
166fcf3ce44SJohn Forte 			goto dmem_init_failed;
167c4ddbbe1SDaniel Beauregard 		}
168fcf3ce44SJohn Forte 
169fcf3ce44SJohn Forte 		p->dmem_host_addr = host_addr = addr;
170fcf3ce44SJohn Forte 		p->dmem_dev_addr = dev_addr = (uint64_t)cookie.dmac_laddress;
171fcf3ce44SJohn Forte 		bsize = p->dmem_buf_size;
172fcf3ce44SJohn Forte 		p->dmem_bctl_free_list = bctl;
173fcf3ce44SJohn Forte 		p->dmem_nbufs_free = p->dmem_nbufs;
174fcf3ce44SJohn Forte 		for (i = 0; i < p->dmem_nbufs; i++) {
1753fb517f7SJames Moore 			stmf_data_buf_t	*db;
176fcf3ce44SJohn Forte 			prev = bctl;
177fcf3ce44SJohn Forte 			bctl->bctl_bucket = p;
178fcf3ce44SJohn Forte 			bctl->bctl_buf = db = stmf_alloc(STMF_STRUCT_DATA_BUF,
179c4ddbbe1SDaniel Beauregard 			    0, 0);
180fcf3ce44SJohn Forte 			db->db_port_private = bctl;
181fcf3ce44SJohn Forte 			db->db_sglist[0].seg_addr = host_addr;
182fcf3ce44SJohn Forte 			bctl->bctl_dev_addr = dev_addr;
183fcf3ce44SJohn Forte 			db->db_sglist[0].seg_length = db->db_buf_size = bsize;
184fcf3ce44SJohn Forte 			db->db_sglist_length = 1;
185fcf3ce44SJohn Forte 			host_addr += bsize;
186fcf3ce44SJohn Forte 			dev_addr += bsize;
187fcf3ce44SJohn Forte 			bctl++;
188fcf3ce44SJohn Forte 			prev->bctl_next = bctl;
189fcf3ce44SJohn Forte 		}
190fcf3ce44SJohn Forte 		prev->bctl_next = NULL;
191fcf3ce44SJohn Forte 	}
192fcf3ce44SJohn Forte 
193fcf3ce44SJohn Forte 	return (QLT_SUCCESS);
194fcf3ce44SJohn Forte 
195fcf3ce44SJohn Forte dmem_failure_loop:;
196fcf3ce44SJohn Forte 	bc = bctl;
197fcf3ce44SJohn Forte 	while (bc) {
198fcf3ce44SJohn Forte 		stmf_free(bc->bctl_buf);
199fcf3ce44SJohn Forte 		bc = bc->bctl_next;
200fcf3ce44SJohn Forte 	}
201fcf3ce44SJohn Forte dmem_init_failed:;
202fcf3ce44SJohn Forte 	(void) ddi_dma_unbind_handle(p->dmem_dma_handle);
203fcf3ce44SJohn Forte addr_bind_handle_failed:;
204fcf3ce44SJohn Forte 	ddi_dma_mem_free(&p->dmem_acc_handle);
205fcf3ce44SJohn Forte mem_alloc_failed:;
206fcf3ce44SJohn Forte 	ddi_dma_free_handle(&p->dmem_dma_handle);
207fcf3ce44SJohn Forte alloc_handle_failed:;
208fcf3ce44SJohn Forte 	kmem_free(p->dmem_bctls_mem, p->dmem_nbufs * sizeof (qlt_dmem_bctl_t));
209fcf3ce44SJohn Forte 	mutex_destroy(&p->dmem_lock);
210fcf3ce44SJohn Forte alloc_bctl_failed:;
211fcf3ce44SJohn Forte 	if (--ndx >= 0) {
212fcf3ce44SJohn Forte 		p = qlt->dmem_buckets[ndx];
213fcf3ce44SJohn Forte 		bctl = p->dmem_bctl_free_list;
214fcf3ce44SJohn Forte 		goto dmem_failure_loop;
215fcf3ce44SJohn Forte 	}
216fcf3ce44SJohn Forte 	kmem_free(qlt->dmem_buckets, sizeof (dmem_buckets) +
217fcf3ce44SJohn Forte 	    ((sizeof (dmem_buckets)/sizeof (void *))
218fcf3ce44SJohn Forte 	    *sizeof (qlt_dmem_bucket_t)));
219fcf3ce44SJohn Forte 	qlt->dmem_buckets = NULL;
220fcf3ce44SJohn Forte 
221fcf3ce44SJohn Forte 	return (QLT_FAILURE);
222fcf3ce44SJohn Forte }
223fcf3ce44SJohn Forte 
2243fb517f7SJames Moore void
qlt_dma_handle_pool_init(qlt_state_t * qlt)2253fb517f7SJames Moore qlt_dma_handle_pool_init(qlt_state_t *qlt)
2263fb517f7SJames Moore {
2273fb517f7SJames Moore 	qlt_dma_handle_pool_t *pool;
2283fb517f7SJames Moore 
2293fb517f7SJames Moore 	pool = kmem_zalloc(sizeof (*pool), KM_SLEEP);
2303fb517f7SJames Moore 	mutex_init(&pool->pool_lock, NULL, MUTEX_DRIVER, NULL);
2313fb517f7SJames Moore 	qlt->qlt_dma_handle_pool = pool;
2323fb517f7SJames Moore }
2333fb517f7SJames Moore 
2343fb517f7SJames Moore void
qlt_dma_handle_pool_fini(qlt_state_t * qlt)2353fb517f7SJames Moore qlt_dma_handle_pool_fini(qlt_state_t *qlt)
2363fb517f7SJames Moore {
2373fb517f7SJames Moore 	qlt_dma_handle_pool_t	*pool;
2383fb517f7SJames Moore 	qlt_dma_handle_t	*handle, *next_handle;
2393fb517f7SJames Moore 
2403fb517f7SJames Moore 	pool = qlt->qlt_dma_handle_pool;
2413fb517f7SJames Moore 	mutex_enter(&pool->pool_lock);
2423fb517f7SJames Moore 	/*
2433fb517f7SJames Moore 	 * XXX Need to wait for free == total elements
2443fb517f7SJames Moore 	 * XXX Not sure how other driver shutdown stuff is done.
2453fb517f7SJames Moore 	 */
2463fb517f7SJames Moore 	ASSERT(pool->num_free == pool->num_total);
2473fb517f7SJames Moore 	if (pool->num_free != pool->num_total)
2483fb517f7SJames Moore 		cmn_err(CE_WARN,
2493fb517f7SJames Moore 		    "num_free %d != num_total %d\n",
2503fb517f7SJames Moore 		    pool->num_free, pool->num_total);
2513fb517f7SJames Moore 	handle = pool->free_list;
2523fb517f7SJames Moore 	while (handle) {
2533fb517f7SJames Moore 		next_handle = handle->next;
2543fb517f7SJames Moore 		kmem_free(handle, sizeof (*handle));
2553fb517f7SJames Moore 		handle = next_handle;
2563fb517f7SJames Moore 	}
2573fb517f7SJames Moore 	qlt->qlt_dma_handle_pool = NULL;
2583fb517f7SJames Moore 	mutex_destroy(&pool->pool_lock);
2593fb517f7SJames Moore 	kmem_free(pool, sizeof (*pool));
2603fb517f7SJames Moore }
2613fb517f7SJames Moore 
262fcf3ce44SJohn Forte void
qlt_dmem_fini(qlt_state_t * qlt)263fcf3ce44SJohn Forte qlt_dmem_fini(qlt_state_t *qlt)
264fcf3ce44SJohn Forte {
265fcf3ce44SJohn Forte 	qlt_dmem_bucket_t *p;
266fcf3ce44SJohn Forte 	qlt_dmem_bctl_t *bctl;
267fcf3ce44SJohn Forte 	int ndx;
268fcf3ce44SJohn Forte 
269fcf3ce44SJohn Forte 	for (ndx = 0; (p = qlt->dmem_buckets[ndx]) != NULL; ndx++) {
270fcf3ce44SJohn Forte 		bctl = p->dmem_bctl_free_list;
271fcf3ce44SJohn Forte 		while (bctl) {
272fcf3ce44SJohn Forte 			stmf_free(bctl->bctl_buf);
273fcf3ce44SJohn Forte 			bctl = bctl->bctl_next;
274fcf3ce44SJohn Forte 		}
275fcf3ce44SJohn Forte 		bctl = p->dmem_bctl_free_list;
276fcf3ce44SJohn Forte 		(void) ddi_dma_unbind_handle(p->dmem_dma_handle);
277fcf3ce44SJohn Forte 		ddi_dma_mem_free(&p->dmem_acc_handle);
278fcf3ce44SJohn Forte 		ddi_dma_free_handle(&p->dmem_dma_handle);
279fcf3ce44SJohn Forte 		kmem_free(p->dmem_bctls_mem,
280c4ddbbe1SDaniel Beauregard 		    p->dmem_nbufs * sizeof (qlt_dmem_bctl_t));
281fcf3ce44SJohn Forte 		mutex_destroy(&p->dmem_lock);
282fcf3ce44SJohn Forte 	}
283fcf3ce44SJohn Forte 	kmem_free(qlt->dmem_buckets, sizeof (dmem_buckets) +
284fcf3ce44SJohn Forte 	    (((sizeof (dmem_buckets)/sizeof (void *))-1)*
285c4ddbbe1SDaniel Beauregard 	    sizeof (qlt_dmem_bucket_t)));
286fcf3ce44SJohn Forte 	qlt->dmem_buckets = NULL;
287fcf3ce44SJohn Forte }
288fcf3ce44SJohn Forte 
289fcf3ce44SJohn Forte stmf_data_buf_t *
qlt_dmem_alloc(fct_local_port_t * port,uint32_t size,uint32_t * pminsize,uint32_t flags)290fcf3ce44SJohn Forte qlt_dmem_alloc(fct_local_port_t *port, uint32_t size, uint32_t *pminsize,
291fcf3ce44SJohn Forte     uint32_t flags)
292fcf3ce44SJohn Forte {
293fcf3ce44SJohn Forte 	return (qlt_i_dmem_alloc((qlt_state_t *)
294fcf3ce44SJohn Forte 	    port->port_fca_private, size, pminsize,
295fcf3ce44SJohn Forte 	    flags));
296fcf3ce44SJohn Forte }
297fcf3ce44SJohn Forte 
298fcf3ce44SJohn Forte /* ARGSUSED */
299fcf3ce44SJohn Forte stmf_data_buf_t *
qlt_i_dmem_alloc(qlt_state_t * qlt,uint32_t size,uint32_t * pminsize,uint32_t flags)300fcf3ce44SJohn Forte qlt_i_dmem_alloc(qlt_state_t *qlt, uint32_t size, uint32_t *pminsize,
3013fb517f7SJames Moore     uint32_t flags)
302fcf3ce44SJohn Forte {
303a2255df3SDaniel Beauregard 	qlt_dmem_bucket_t	*p;
3043f3ce7b9SDaniel Beauregard 	qlt_dmem_bctl_t 	*bctl;
305a2255df3SDaniel Beauregard 	int			i;
3063f3ce7b9SDaniel Beauregard 	uint32_t		size_possible = 0;
307fcf3ce44SJohn Forte 
3083f3ce7b9SDaniel Beauregard 	if (size > QLT_DMEM_MAX_BUF_SIZE) {
3093f3ce7b9SDaniel Beauregard 		goto qlt_try_partial_alloc;
3103f3ce7b9SDaniel Beauregard 	}
3113f3ce7b9SDaniel Beauregard 
3123f3ce7b9SDaniel Beauregard 	/* 1st try to do a full allocation */
3133f3ce7b9SDaniel Beauregard 	for (i = 0; (p = qlt->dmem_buckets[i]) != NULL; i++) {
3143f3ce7b9SDaniel Beauregard 		if (p->dmem_buf_size >= size) {
3153f3ce7b9SDaniel Beauregard 			if (p->dmem_nbufs_free) {
3163f3ce7b9SDaniel Beauregard 				mutex_enter(&p->dmem_lock);
3173f3ce7b9SDaniel Beauregard 				bctl = p->dmem_bctl_free_list;
3183f3ce7b9SDaniel Beauregard 				if (bctl == NULL) {
319a2255df3SDaniel Beauregard 					mutex_exit(&p->dmem_lock);
3203f3ce7b9SDaniel Beauregard 					continue;
321a2255df3SDaniel Beauregard 				}
3223f3ce7b9SDaniel Beauregard 				p->dmem_bctl_free_list =
3233f3ce7b9SDaniel Beauregard 				    bctl->bctl_next;
3243f3ce7b9SDaniel Beauregard 				p->dmem_nbufs_free--;
3253f3ce7b9SDaniel Beauregard 				qlt->qlt_bufref[i]++;
3263f3ce7b9SDaniel Beauregard 				mutex_exit(&p->dmem_lock);
3273f3ce7b9SDaniel Beauregard 				bctl->bctl_buf->db_data_size = size;
3283f3ce7b9SDaniel Beauregard 				return (bctl->bctl_buf);
3293f3ce7b9SDaniel Beauregard 			} else {
3303f3ce7b9SDaniel Beauregard 				qlt->qlt_bumpbucket++;
331fcf3ce44SJohn Forte 			}
332fcf3ce44SJohn Forte 		}
333fcf3ce44SJohn Forte 	}
334fcf3ce44SJohn Forte 
3353f3ce7b9SDaniel Beauregard qlt_try_partial_alloc:
3363f3ce7b9SDaniel Beauregard 
3373f3ce7b9SDaniel Beauregard 	qlt->qlt_pmintry++;
3383f3ce7b9SDaniel Beauregard 
3393f3ce7b9SDaniel Beauregard 	/* Now go from high to low */
3403f3ce7b9SDaniel Beauregard 	for (i = QLT_DMEM_NBUCKETS - 1; i >= 0; i--) {
3413f3ce7b9SDaniel Beauregard 		p = qlt->dmem_buckets[i];
3423f3ce7b9SDaniel Beauregard 		if (p->dmem_nbufs_free == 0)
3433f3ce7b9SDaniel Beauregard 			continue;
3443f3ce7b9SDaniel Beauregard 		if (!size_possible) {
3453f3ce7b9SDaniel Beauregard 			size_possible = p->dmem_buf_size;
3463f3ce7b9SDaniel Beauregard 		}
3473f3ce7b9SDaniel Beauregard 		if (*pminsize > p->dmem_buf_size) {
3483f3ce7b9SDaniel Beauregard 			/* At this point we know the request is failing. */
3493f3ce7b9SDaniel Beauregard 			if (size_possible) {
3503f3ce7b9SDaniel Beauregard 				/*
3513f3ce7b9SDaniel Beauregard 				 * This caller is asking too much. We already
3523f3ce7b9SDaniel Beauregard 				 * know what we can give, so get out.
3533f3ce7b9SDaniel Beauregard 				 */
3543f3ce7b9SDaniel Beauregard 				break;
3553f3ce7b9SDaniel Beauregard 			} else {
3563f3ce7b9SDaniel Beauregard 				/*
3573f3ce7b9SDaniel Beauregard 				 * Lets continue to find out and tell what
3583f3ce7b9SDaniel Beauregard 				 * we can give.
3593f3ce7b9SDaniel Beauregard 				 */
3603f3ce7b9SDaniel Beauregard 				continue;
3613f3ce7b9SDaniel Beauregard 			}
3623f3ce7b9SDaniel Beauregard 		}
3633f3ce7b9SDaniel Beauregard 		mutex_enter(&p->dmem_lock);
3643f3ce7b9SDaniel Beauregard 		if (*pminsize <= p->dmem_buf_size) {
3653f3ce7b9SDaniel Beauregard 			bctl = p->dmem_bctl_free_list;
3663f3ce7b9SDaniel Beauregard 			if (bctl == NULL) {
3673f3ce7b9SDaniel Beauregard 				/* Someone took it. */
3683f3ce7b9SDaniel Beauregard 				size_possible = 0;
3693f3ce7b9SDaniel Beauregard 				mutex_exit(&p->dmem_lock);
3703f3ce7b9SDaniel Beauregard 				continue;
3713f3ce7b9SDaniel Beauregard 			}
3723f3ce7b9SDaniel Beauregard 			p->dmem_bctl_free_list = bctl->bctl_next;
3733f3ce7b9SDaniel Beauregard 			p->dmem_nbufs_free--;
3743f3ce7b9SDaniel Beauregard 			mutex_exit(&p->dmem_lock);
3753f3ce7b9SDaniel Beauregard 			bctl->bctl_buf->db_data_size = p->dmem_buf_size;
3763f3ce7b9SDaniel Beauregard 			qlt->qlt_pmin_ok++;
3773f3ce7b9SDaniel Beauregard 			return (bctl->bctl_buf);
3783f3ce7b9SDaniel Beauregard 		}
379fcf3ce44SJohn Forte 	}
380a2255df3SDaniel Beauregard 
3813f3ce7b9SDaniel Beauregard 	*pminsize = size_possible;
3823f3ce7b9SDaniel Beauregard 
383fcf3ce44SJohn Forte 	return (NULL);
384fcf3ce44SJohn Forte }
385fcf3ce44SJohn Forte 
386fcf3ce44SJohn Forte /* ARGSUSED */
387fcf3ce44SJohn Forte void
qlt_i_dmem_free(qlt_state_t * qlt,stmf_data_buf_t * dbuf)388fcf3ce44SJohn Forte qlt_i_dmem_free(qlt_state_t *qlt, stmf_data_buf_t *dbuf)
389fcf3ce44SJohn Forte {
390fcf3ce44SJohn Forte 	qlt_dmem_free(0, dbuf);
391fcf3ce44SJohn Forte }
392fcf3ce44SJohn Forte 
393fcf3ce44SJohn Forte /* ARGSUSED */
394fcf3ce44SJohn Forte void
qlt_dmem_free(fct_dbuf_store_t * fds,stmf_data_buf_t * dbuf)395fcf3ce44SJohn Forte qlt_dmem_free(fct_dbuf_store_t *fds, stmf_data_buf_t *dbuf)
396fcf3ce44SJohn Forte {
3973fb517f7SJames Moore 	qlt_dmem_bctl_t		*bctl;
3983fb517f7SJames Moore 	qlt_dmem_bucket_t	*p;
399fcf3ce44SJohn Forte 
4003fb517f7SJames Moore 	ASSERT((dbuf->db_flags & DB_LU_DATA_BUF) == 0);
4013fb517f7SJames Moore 
4023fb517f7SJames Moore 	bctl = (qlt_dmem_bctl_t *)dbuf->db_port_private;
4033fb517f7SJames Moore 	p = bctl->bctl_bucket;
404fcf3ce44SJohn Forte 	mutex_enter(&p->dmem_lock);
405fcf3ce44SJohn Forte 	bctl->bctl_next = p->dmem_bctl_free_list;
406fcf3ce44SJohn Forte 	p->dmem_bctl_free_list = bctl;
407fcf3ce44SJohn Forte 	p->dmem_nbufs_free++;
408fcf3ce44SJohn Forte 	mutex_exit(&p->dmem_lock);
409fcf3ce44SJohn Forte }
410fcf3ce44SJohn Forte 
411fcf3ce44SJohn Forte void
qlt_dmem_dma_sync(stmf_data_buf_t * dbuf,uint_t sync_type)412fcf3ce44SJohn Forte qlt_dmem_dma_sync(stmf_data_buf_t *dbuf, uint_t sync_type)
413fcf3ce44SJohn Forte {
4143fb517f7SJames Moore 	qlt_dmem_bctl_t		*bctl;
4153fb517f7SJames Moore 	qlt_dma_sgl_t		*qsgl;
4163fb517f7SJames Moore 	qlt_dmem_bucket_t	*p;
4173fb517f7SJames Moore 	qlt_dma_handle_t	*th;
4183fb517f7SJames Moore 	int			rv;
4193fb517f7SJames Moore 
4203fb517f7SJames Moore 	if (dbuf->db_flags & DB_LU_DATA_BUF) {
4213fb517f7SJames Moore 		/*
4223fb517f7SJames Moore 		 * go through ddi handle list
4233fb517f7SJames Moore 		 */
4243fb517f7SJames Moore 		qsgl = (qlt_dma_sgl_t *)dbuf->db_port_private;
4253fb517f7SJames Moore 		th = qsgl->handle_list;
4263fb517f7SJames Moore 		while (th) {
4273fb517f7SJames Moore 			rv = ddi_dma_sync(th->dma_handle,
4283fb517f7SJames Moore 			    0, 0, sync_type);
4293fb517f7SJames Moore 			if (rv != DDI_SUCCESS) {
4303fb517f7SJames Moore 				cmn_err(CE_WARN, "ddi_dma_sync FAILED\n");
4313fb517f7SJames Moore 			}
4323fb517f7SJames Moore 			th = th->next;
4333fb517f7SJames Moore 		}
4343fb517f7SJames Moore 	} else {
4353fb517f7SJames Moore 		bctl = (qlt_dmem_bctl_t *)dbuf->db_port_private;
4363fb517f7SJames Moore 		p = bctl->bctl_bucket;
4373fb517f7SJames Moore 		(void) ddi_dma_sync(p->dmem_dma_handle, (off_t)
4383fb517f7SJames Moore 		    (bctl->bctl_dev_addr - p->dmem_dev_addr),
4393fb517f7SJames Moore 		    dbuf->db_data_size, sync_type);
4403fb517f7SJames Moore 	}
4413fb517f7SJames Moore }
4423fb517f7SJames Moore 
4433fb517f7SJames Moore /*
4443fb517f7SJames Moore  * A very lite version of ddi_dma_addr_bind_handle()
4453fb517f7SJames Moore  */
4463fb517f7SJames Moore uint64_t
qlt_ddi_vtop(caddr_t vaddr)4473fb517f7SJames Moore qlt_ddi_vtop(caddr_t vaddr)
4483fb517f7SJames Moore {
4493fb517f7SJames Moore 	uint64_t offset, paddr;
4503fb517f7SJames Moore 	pfn_t pfn;
4513fb517f7SJames Moore 
4523fb517f7SJames Moore 	pfn = hat_getpfnum(kas.a_hat, vaddr);
4533fb517f7SJames Moore 	ASSERT(pfn != PFN_INVALID && pfn != PFN_SUSPENDED);
4543fb517f7SJames Moore 	offset = ((uintptr_t)vaddr) & MMU_PAGEOFFSET;
4553fb517f7SJames Moore 	paddr = mmu_ptob(pfn);
4563fb517f7SJames Moore 	return (paddr+offset);
4573fb517f7SJames Moore }
4583fb517f7SJames Moore 
4593fb517f7SJames Moore static ddi_dma_attr_t 	qlt_sgl_dma_attr = {
4603fb517f7SJames Moore 	DMA_ATTR_V0,		/* dma_attr_version */
4613fb517f7SJames Moore 	0,				/* low DMA address range */
4623fb517f7SJames Moore 	0xffffffffffffffff,		/* high DMA address range */
4633fb517f7SJames Moore 	0xffffffff,			/* DMA counter register */
4643fb517f7SJames Moore 	64,				/* DMA address alignment */
4653fb517f7SJames Moore 	0xff,			/* DMA burstsizes */
4663fb517f7SJames Moore 	1,				/* min effective DMA size */
4673fb517f7SJames Moore 	0xffffffff,			/* max DMA xfer size */
4683fb517f7SJames Moore 	0xffffffff,			/* segment boundary */
4693fb517f7SJames Moore 	QLT_DMA_SG_LIST_LENGTH,	/* s/g list length */
4703fb517f7SJames Moore 	1,				/* granularity of device */
4713fb517f7SJames Moore 	0				/* DMA transfer flags */
4723fb517f7SJames Moore };
4733fb517f7SJames Moore 
4743fb517f7SJames Moore /*
4753fb517f7SJames Moore  * Allocate a qlt_dma_handle container and fill it with a ddi_dma_handle
4763fb517f7SJames Moore  */
4773fb517f7SJames Moore static qlt_dma_handle_t *
qlt_dma_alloc_handle(qlt_state_t * qlt)4783fb517f7SJames Moore qlt_dma_alloc_handle(qlt_state_t *qlt)
4793fb517f7SJames Moore {
4803fb517f7SJames Moore 	ddi_dma_handle_t ddi_handle;
4813fb517f7SJames Moore 	qlt_dma_handle_t *qlt_handle;
4823fb517f7SJames Moore 	int rv;
4833fb517f7SJames Moore 
4843fb517f7SJames Moore 	rv = ddi_dma_alloc_handle(qlt->dip, &qlt_sgl_dma_attr,
4853fb517f7SJames Moore 	    DDI_DMA_SLEEP, 0, &ddi_handle);
4863fb517f7SJames Moore 	if (rv != DDI_SUCCESS) {
4873fb517f7SJames Moore 		EL(qlt, "ddi_dma_alloc_handle status=%xh\n", rv);
4883fb517f7SJames Moore 		return (NULL);
4893fb517f7SJames Moore 	}
4903fb517f7SJames Moore 	qlt_handle = kmem_zalloc(sizeof (qlt_dma_handle_t), KM_SLEEP);
4913fb517f7SJames Moore 	qlt_handle->dma_handle = ddi_handle;
4923fb517f7SJames Moore 	return (qlt_handle);
4933fb517f7SJames Moore }
4943fb517f7SJames Moore 
4953fb517f7SJames Moore /*
4963fb517f7SJames Moore  * Allocate a list of qlt_dma_handle containers from the free list
4973fb517f7SJames Moore  */
4983fb517f7SJames Moore static qlt_dma_handle_t *
qlt_dma_alloc_handle_list(qlt_state_t * qlt,int handle_count)4993fb517f7SJames Moore qlt_dma_alloc_handle_list(qlt_state_t *qlt, int handle_count)
5003fb517f7SJames Moore {
5013fb517f7SJames Moore 	qlt_dma_handle_pool_t	*pool;
5023fb517f7SJames Moore 	qlt_dma_handle_t	*tmp_handle, *first_handle, *last_handle;
5033fb517f7SJames Moore 	int i;
5043fb517f7SJames Moore 
5053fb517f7SJames Moore 	/*
5063fb517f7SJames Moore 	 * Make sure the free list can satisfy the request.
5073fb517f7SJames Moore 	 * Once the free list is primed, it should satisfy most requests.
5083fb517f7SJames Moore 	 * XXX Should there be a limit on pool size?
5093fb517f7SJames Moore 	 */
5103fb517f7SJames Moore 	pool = qlt->qlt_dma_handle_pool;
5113fb517f7SJames Moore 	mutex_enter(&pool->pool_lock);
5123fb517f7SJames Moore 	while (handle_count > pool->num_free) {
5133fb517f7SJames Moore 		mutex_exit(&pool->pool_lock);
5143fb517f7SJames Moore 		if ((tmp_handle = qlt_dma_alloc_handle(qlt)) == NULL)
5153fb517f7SJames Moore 			return (NULL);
5163fb517f7SJames Moore 		mutex_enter(&pool->pool_lock);
5173fb517f7SJames Moore 		tmp_handle->next = pool->free_list;
5183fb517f7SJames Moore 		pool->free_list = tmp_handle;
5193fb517f7SJames Moore 		pool->num_free++;
5203fb517f7SJames Moore 		pool->num_total++;
5213fb517f7SJames Moore 	}
5223fb517f7SJames Moore 
5233fb517f7SJames Moore 	/*
5243fb517f7SJames Moore 	 * The free list lock is held and the list is large enough to
5253fb517f7SJames Moore 	 * satisfy this request. Run down the freelist and snip off
5263fb517f7SJames Moore 	 * the number of elements needed for this request.
5273fb517f7SJames Moore 	 */
5283fb517f7SJames Moore 	first_handle = pool->free_list;
5293fb517f7SJames Moore 	tmp_handle = first_handle;
5303fb517f7SJames Moore 	for (i = 0; i < handle_count; i++) {
5313fb517f7SJames Moore 		last_handle = tmp_handle;
5323fb517f7SJames Moore 		tmp_handle = tmp_handle->next;
5333fb517f7SJames Moore 	}
5343fb517f7SJames Moore 	pool->free_list = tmp_handle;
5353fb517f7SJames Moore 	pool->num_free -= handle_count;
5363fb517f7SJames Moore 	mutex_exit(&pool->pool_lock);
5373fb517f7SJames Moore 	last_handle->next = NULL;	/* sanity */
5383fb517f7SJames Moore 	return (first_handle);
5393fb517f7SJames Moore }
5403fb517f7SJames Moore 
5413fb517f7SJames Moore /*
5423fb517f7SJames Moore  * Return a list of qlt_dma_handle containers to the free list.
5433fb517f7SJames Moore  */
5443fb517f7SJames Moore static void
qlt_dma_free_handles(qlt_state_t * qlt,qlt_dma_handle_t * first_handle)5453fb517f7SJames Moore qlt_dma_free_handles(qlt_state_t *qlt, qlt_dma_handle_t *first_handle)
5463fb517f7SJames Moore {
5473fb517f7SJames Moore 	qlt_dma_handle_pool_t *pool;
5483fb517f7SJames Moore 	qlt_dma_handle_t *tmp_handle, *last_handle;
5493fb517f7SJames Moore 	int rv, handle_count;
5503fb517f7SJames Moore 
5513fb517f7SJames Moore 	/*
5523fb517f7SJames Moore 	 * Traverse the list and unbind the handles
5533fb517f7SJames Moore 	 */
5543fb517f7SJames Moore 	ASSERT(first_handle);
5553fb517f7SJames Moore 	tmp_handle = first_handle;
5563fb517f7SJames Moore 	handle_count = 0;
5573fb517f7SJames Moore 	while (tmp_handle != NULL) {
5583fb517f7SJames Moore 		last_handle = tmp_handle;
5593fb517f7SJames Moore 		/*
5603fb517f7SJames Moore 		 * If the handle is bound, unbind the handle so it can be
5613fb517f7SJames Moore 		 * reused. It may not be bound if there was a bind failure.
5623fb517f7SJames Moore 		 */
5633fb517f7SJames Moore 		if (tmp_handle->num_cookies != 0) {
5643fb517f7SJames Moore 			rv = ddi_dma_unbind_handle(tmp_handle->dma_handle);
5653fb517f7SJames Moore 			ASSERT(rv == DDI_SUCCESS);
5663fb517f7SJames Moore 			tmp_handle->num_cookies = 0;
5673fb517f7SJames Moore 			tmp_handle->num_cookies_fetched = 0;
5683fb517f7SJames Moore 		}
5693fb517f7SJames Moore 		tmp_handle = tmp_handle->next;
5703fb517f7SJames Moore 		handle_count++;
5713fb517f7SJames Moore 	}
5723fb517f7SJames Moore 	/*
5733fb517f7SJames Moore 	 * Insert this list into the free list
5743fb517f7SJames Moore 	 */
5753fb517f7SJames Moore 	pool = qlt->qlt_dma_handle_pool;
5763fb517f7SJames Moore 	mutex_enter(&pool->pool_lock);
5773fb517f7SJames Moore 	last_handle->next = pool->free_list;
5783fb517f7SJames Moore 	pool->free_list = first_handle;
5793fb517f7SJames Moore 	pool->num_free += handle_count;
5803fb517f7SJames Moore 	mutex_exit(&pool->pool_lock);
5813fb517f7SJames Moore }
5823fb517f7SJames Moore 
5833fb517f7SJames Moore /*
5843fb517f7SJames Moore  * cookies produced by mapping this dbuf
5853fb517f7SJames Moore  */
5863fb517f7SJames Moore uint16_t
qlt_get_cookie_count(stmf_data_buf_t * dbuf)5873fb517f7SJames Moore qlt_get_cookie_count(stmf_data_buf_t *dbuf)
5883fb517f7SJames Moore {
5893fb517f7SJames Moore 	qlt_dma_sgl_t *qsgl = dbuf->db_port_private;
5903fb517f7SJames Moore 
5913fb517f7SJames Moore 	ASSERT(dbuf->db_flags & DB_LU_DATA_BUF);
5923fb517f7SJames Moore 	return (qsgl->cookie_count);
5933fb517f7SJames Moore }
594fcf3ce44SJohn Forte 
5953fb517f7SJames Moore ddi_dma_cookie_t
qlt_get_cookie_array(stmf_data_buf_t * dbuf)5963fb517f7SJames Moore *qlt_get_cookie_array(stmf_data_buf_t *dbuf)
5973fb517f7SJames Moore {
5983fb517f7SJames Moore 	qlt_dma_sgl_t *qsgl = dbuf->db_port_private;
5993fb517f7SJames Moore 
6003fb517f7SJames Moore 	ASSERT(dbuf->db_flags & DB_LU_DATA_BUF);
6013fb517f7SJames Moore 
6023fb517f7SJames Moore 	if (qsgl->cookie_prefetched)
6033fb517f7SJames Moore 		return (&qsgl->cookie[0]);
6043fb517f7SJames Moore 	else
6053fb517f7SJames Moore 		return (NULL);
6063fb517f7SJames Moore }
6073fb517f7SJames Moore 
6083fb517f7SJames Moore /*
6093fb517f7SJames Moore  * Wrapper around ddi_dma_nextcookie that hides the ddi_dma_handle usage.
6103fb517f7SJames Moore  */
6113fb517f7SJames Moore void
qlt_ddi_dma_nextcookie(stmf_data_buf_t * dbuf,ddi_dma_cookie_t * cookiep)6123fb517f7SJames Moore qlt_ddi_dma_nextcookie(stmf_data_buf_t *dbuf, ddi_dma_cookie_t *cookiep)
6133fb517f7SJames Moore {
6143fb517f7SJames Moore 	qlt_dma_sgl_t *qsgl = dbuf->db_port_private;
6153fb517f7SJames Moore 
6163fb517f7SJames Moore 	ASSERT(dbuf->db_flags & DB_LU_DATA_BUF);
6173fb517f7SJames Moore 
6183fb517f7SJames Moore 	if (qsgl->cookie_prefetched) {
6193fb517f7SJames Moore 		ASSERT(qsgl->cookie_next_fetch < qsgl->cookie_count);
6203fb517f7SJames Moore 		*cookiep = qsgl->cookie[qsgl->cookie_next_fetch++];
6213fb517f7SJames Moore 	} else {
6223fb517f7SJames Moore 		qlt_dma_handle_t *fetch;
6233fb517f7SJames Moore 		qlt_dma_handle_t *FETCH_DONE = (qlt_dma_handle_t *)0xbad;
6243fb517f7SJames Moore 
6253fb517f7SJames Moore 		ASSERT(qsgl->handle_list != NULL);
6263fb517f7SJames Moore 		ASSERT(qsgl->handle_next_fetch != FETCH_DONE);
6273fb517f7SJames Moore 
6283fb517f7SJames Moore 		fetch = qsgl->handle_next_fetch;
6293fb517f7SJames Moore 		if (fetch->num_cookies_fetched == 0) {
6303fb517f7SJames Moore 			*cookiep = fetch->first_cookie;
6313fb517f7SJames Moore 		} else {
6323fb517f7SJames Moore 			ddi_dma_nextcookie(fetch->dma_handle, cookiep);
6333fb517f7SJames Moore 		}
6343fb517f7SJames Moore 		if (++fetch->num_cookies_fetched == fetch->num_cookies) {
6353fb517f7SJames Moore 			if (fetch->next == NULL)
6363fb517f7SJames Moore 				qsgl->handle_next_fetch = FETCH_DONE;
6373fb517f7SJames Moore 			else
6383fb517f7SJames Moore 				qsgl->handle_next_fetch = fetch->next;
6393fb517f7SJames Moore 		}
6403fb517f7SJames Moore 	}
6413fb517f7SJames Moore }
6423fb517f7SJames Moore 
6433fb517f7SJames Moore /*
6443fb517f7SJames Moore  * Set this flag to fetch the DDI dma cookies from the handles here and
6453fb517f7SJames Moore  * store them in the port private area of the dbuf. This will allow
6463fb517f7SJames Moore  * faster access to the cookies in qlt_xfer_scsi_data() at the expense of
6473fb517f7SJames Moore  * an extra copy. If the qlt->req_lock is hot, this may help.
6483fb517f7SJames Moore  */
6493fb517f7SJames Moore int qlt_sgl_prefetch = 0;
6503fb517f7SJames Moore 
6513fb517f7SJames Moore /*ARGSUSED*/
6523fb517f7SJames Moore stmf_status_t
qlt_dma_setup_dbuf(fct_local_port_t * port,stmf_data_buf_t * dbuf,uint32_t flags)6533fb517f7SJames Moore qlt_dma_setup_dbuf(fct_local_port_t *port, stmf_data_buf_t *dbuf,
6543fb517f7SJames Moore     uint32_t flags)
6553fb517f7SJames Moore {
6563fb517f7SJames Moore 	qlt_state_t		*qlt = port->port_fca_private;
6573fb517f7SJames Moore 	qlt_dma_sgl_t		*qsgl;
6583fb517f7SJames Moore 	struct stmf_sglist_ent	*sglp;
6593fb517f7SJames Moore 	qlt_dma_handle_t	*handle_list, *th;
6603fb517f7SJames Moore 	int			i, rv;
6613fb517f7SJames Moore 	ddi_dma_cookie_t	*cookie_p;
6623fb517f7SJames Moore 	int			cookie_count, numbufs;
6633fb517f7SJames Moore 	int			prefetch;
6643fb517f7SJames Moore 	size_t			qsize;
6653fb517f7SJames Moore 
6663fb517f7SJames Moore 	/*
6673fb517f7SJames Moore 	 * psuedo code:
6683fb517f7SJames Moore 	 * get dma handle list from cache - one per sglist entry
6693fb517f7SJames Moore 	 * foreach sglist entry
6703fb517f7SJames Moore 	 *	bind dma handle to sglist vaddr
6713fb517f7SJames Moore 	 * allocate space for DMA state to store in db_port_private
6723fb517f7SJames Moore 	 * fill in port private object
6733fb517f7SJames Moore 	 * if prefetching
6743fb517f7SJames Moore 	 *	move all dma cookies into db_port_private
6753fb517f7SJames Moore 	 */
6763fb517f7SJames Moore 	dbuf->db_port_private = NULL;
6773fb517f7SJames Moore 	numbufs = dbuf->db_sglist_length;
6783fb517f7SJames Moore 	handle_list = qlt_dma_alloc_handle_list(qlt, numbufs);
6793fb517f7SJames Moore 	if (handle_list == NULL) {
6803fb517f7SJames Moore 		EL(qlt, "handle_list==NULL\n");
6813fb517f7SJames Moore 		return (STMF_FAILURE);
6823fb517f7SJames Moore 	}
6833fb517f7SJames Moore 	/*
6843fb517f7SJames Moore 	 * Loop through sglist and bind each entry to a handle
6853fb517f7SJames Moore 	 */
6863fb517f7SJames Moore 	th = handle_list;
6873fb517f7SJames Moore 	sglp = &dbuf->db_sglist[0];
6883fb517f7SJames Moore 	cookie_count = 0;
6893fb517f7SJames Moore 	for (i = 0; i < numbufs; i++, sglp++) {
6903fb517f7SJames Moore 
6913fb517f7SJames Moore 		/*
6923fb517f7SJames Moore 		 * Bind this sgl entry to a DDI dma handle
6933fb517f7SJames Moore 		 */
6943fb517f7SJames Moore 		if ((rv = ddi_dma_addr_bind_handle(
6953fb517f7SJames Moore 		    th->dma_handle,
6963fb517f7SJames Moore 		    NULL,
6973fb517f7SJames Moore 		    (caddr_t)(sglp->seg_addr),
6983fb517f7SJames Moore 		    (size_t)sglp->seg_length,
6993fb517f7SJames Moore 		    DDI_DMA_RDWR | DDI_DMA_STREAMING,
7003fb517f7SJames Moore 		    DDI_DMA_DONTWAIT,
7013fb517f7SJames Moore 		    NULL,
7023fb517f7SJames Moore 		    &th->first_cookie,
7033fb517f7SJames Moore 		    &th->num_cookies)) != DDI_DMA_MAPPED) {
7043fb517f7SJames Moore 			cmn_err(CE_NOTE, "ddi_dma_addr_bind_handle %d", rv);
7053fb517f7SJames Moore 			qlt_dma_free_handles(qlt, handle_list);
7063fb517f7SJames Moore 			return (STMF_FAILURE);
7073fb517f7SJames Moore 		}
7083fb517f7SJames Moore 
7093fb517f7SJames Moore 		/*
7103fb517f7SJames Moore 		 * Add to total cookie count
7113fb517f7SJames Moore 		 */
7123fb517f7SJames Moore 		cookie_count += th->num_cookies;
7133fb517f7SJames Moore 		if (cookie_count > QLT_DMA_SG_LIST_LENGTH) {
7143fb517f7SJames Moore 			/*
7153fb517f7SJames Moore 			 * Request exceeds HBA limit
7163fb517f7SJames Moore 			 */
7173fb517f7SJames Moore 			qlt_dma_free_handles(qlt, handle_list);
7183fb517f7SJames Moore 			return (STMF_FAILURE);
7193fb517f7SJames Moore 		}
7203fb517f7SJames Moore 		/* move to next ddi_dma_handle */
7213fb517f7SJames Moore 		th = th->next;
7223fb517f7SJames Moore 	}
7233fb517f7SJames Moore 
7243fb517f7SJames Moore 	/*
7253fb517f7SJames Moore 	 * Allocate our port private object for DMA mapping state.
7263fb517f7SJames Moore 	 */
7273fb517f7SJames Moore 	prefetch =  qlt_sgl_prefetch;
7283fb517f7SJames Moore 	qsize = sizeof (qlt_dma_sgl_t);
7293fb517f7SJames Moore 	if (prefetch) {
7303fb517f7SJames Moore 		/* one extra ddi_dma_cookie allocated for alignment padding */
7313fb517f7SJames Moore 		qsize += cookie_count * sizeof (ddi_dma_cookie_t);
7323fb517f7SJames Moore 	}
7333fb517f7SJames Moore 	qsgl = kmem_alloc(qsize, KM_SLEEP);
7343fb517f7SJames Moore 	/*
7353fb517f7SJames Moore 	 * Fill in the sgl
7363fb517f7SJames Moore 	 */
7373fb517f7SJames Moore 	dbuf->db_port_private = qsgl;
7383fb517f7SJames Moore 	qsgl->qsize = qsize;
7393fb517f7SJames Moore 	qsgl->handle_count = dbuf->db_sglist_length;
7403fb517f7SJames Moore 	qsgl->cookie_prefetched = prefetch;
7413fb517f7SJames Moore 	qsgl->cookie_count = cookie_count;
7423fb517f7SJames Moore 	qsgl->cookie_next_fetch = 0;
7433fb517f7SJames Moore 	qsgl->handle_list = handle_list;
7443fb517f7SJames Moore 	qsgl->handle_next_fetch = handle_list;
7453fb517f7SJames Moore 	if (prefetch) {
7463fb517f7SJames Moore 		/*
7473fb517f7SJames Moore 		 * traverse handle list and move cookies to db_port_private
7483fb517f7SJames Moore 		 */
7493fb517f7SJames Moore 		th = handle_list;
7503fb517f7SJames Moore 		cookie_p = &qsgl->cookie[0];
7513fb517f7SJames Moore 		for (i = 0; i < numbufs; i++) {
7523fb517f7SJames Moore 			uint_t cc = th->num_cookies;
7533fb517f7SJames Moore 
7543fb517f7SJames Moore 			*cookie_p++ = th->first_cookie;
7553fb517f7SJames Moore 			while (--cc > 0) {
7563fb517f7SJames Moore 				ddi_dma_nextcookie(th->dma_handle, cookie_p++);
7573fb517f7SJames Moore 			}
7583fb517f7SJames Moore 			th->num_cookies_fetched = th->num_cookies;
7593fb517f7SJames Moore 			th = th->next;
7603fb517f7SJames Moore 		}
7613fb517f7SJames Moore 	}
7623fb517f7SJames Moore 
7633fb517f7SJames Moore 	return (STMF_SUCCESS);
7643fb517f7SJames Moore }
7653fb517f7SJames Moore 
7663fb517f7SJames Moore void
qlt_dma_teardown_dbuf(fct_dbuf_store_t * fds,stmf_data_buf_t * dbuf)7673fb517f7SJames Moore qlt_dma_teardown_dbuf(fct_dbuf_store_t *fds, stmf_data_buf_t *dbuf)
7683fb517f7SJames Moore {
7693fb517f7SJames Moore 	qlt_state_t		*qlt = fds->fds_fca_private;
7703fb517f7SJames Moore 	qlt_dma_sgl_t		*qsgl = dbuf->db_port_private;
7713fb517f7SJames Moore 
7723fb517f7SJames Moore 	ASSERT(qlt);
7733fb517f7SJames Moore 	ASSERT(qsgl);
7743fb517f7SJames Moore 	ASSERT(dbuf->db_flags & DB_LU_DATA_BUF);
7753fb517f7SJames Moore 
7763fb517f7SJames Moore 	/*
7773fb517f7SJames Moore 	 * unbind and free the dma handles
7783fb517f7SJames Moore 	 */
7793fb517f7SJames Moore 	if (qsgl->handle_list) {
7803fb517f7SJames Moore 		/* go through ddi handle list */
7813fb517f7SJames Moore 		qlt_dma_free_handles(qlt, qsgl->handle_list);
7823fb517f7SJames Moore 	}
7833fb517f7SJames Moore 	kmem_free(qsgl, qsgl->qsize);
7843fb517f7SJames Moore }
7853fb517f7SJames Moore 
7863fb517f7SJames Moore uint8_t
qlt_get_iocb_count(uint32_t cookie_count)7873fb517f7SJames Moore qlt_get_iocb_count(uint32_t cookie_count)
7883fb517f7SJames Moore {
7893fb517f7SJames Moore 	uint32_t	cnt, cont_segs;
790