1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2009 Emulex.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * Source file containing Queue handling functions
29  *
30  */
31 
32 #include <oce_impl.h>
33 
34 int oce_destroy_q(struct oce_dev  *oce, struct oce_mbx  *mbx, size_t req_size,
35     enum qtype  qtype);
36 /* MAil box Queue functions */
37 struct oce_mq *
38 oce_mq_create(struct oce_dev *dev, struct oce_eq *eq, uint32_t q_len);
39 
40 /* event queue handling */
41 struct oce_eq *
42 oce_eq_create(struct oce_dev *dev, uint32_t q_len, uint32_t item_size,
43     uint32_t eq_delay);
44 
45 /* completion queue handling */
46 struct oce_cq *
47 oce_cq_create(struct oce_dev *dev, struct oce_eq *eq, uint32_t q_len,
48     uint32_t item_size, boolean_t sol_event, boolean_t is_eventable,
49     boolean_t nodelay, uint32_t ncoalesce);
50 
51 
52 /* Tx  WQ functions */
53 static struct oce_wq *oce_wq_init(struct oce_dev *dev,  uint32_t q_len,
54     int wq_type);
55 static void oce_wq_fini(struct oce_dev *dev, struct oce_wq *wq);
56 static int oce_wq_create(struct oce_wq *wq, struct oce_eq *eq);
57 static void oce_wq_del(struct oce_dev *dev, struct oce_wq *wq);
58 /* Rx Queue functions */
59 static struct oce_rq *oce_rq_init(struct oce_dev *dev, uint32_t q_len,
60     uint32_t frag_size, uint32_t mtu,
61     boolean_t rss);
62 static void oce_rq_fini(struct oce_dev *dev, struct oce_rq *rq);
63 static int oce_rq_create(struct oce_rq *rq, uint32_t if_id, struct oce_eq *eq);
64 static void oce_rq_del(struct oce_dev *dev, struct oce_rq *rq);
65 
66 /*
67  * function to create an event queue
68  *
69  * dev - software handle to the device
70  * eqcfg - pointer to a config structure containg the eq parameters
71  *
72  * return pointer to EQ; NULL on failure
73  */
74 struct oce_eq *
75 oce_eq_create(struct oce_dev *dev, uint32_t q_len, uint32_t item_size,
76     uint32_t eq_delay)
77 {
78 	struct oce_eq *eq;
79 	struct oce_mbx mbx;
80 	struct mbx_create_common_eq *fwcmd;
81 	int ret = 0;
82 
83 	/* allocate an eq */
84 	eq = kmem_zalloc(sizeof (struct oce_eq), KM_NOSLEEP);
85 
86 	if (eq == NULL) {
87 		return (NULL);
88 	}
89 
90 	bzero(&mbx, sizeof (struct oce_mbx));
91 	/* allocate mbx */
92 	fwcmd = (struct mbx_create_common_eq *)&mbx.payload;
93 
94 	eq->ring = create_ring_buffer(dev, q_len,
95 	    item_size, DDI_DMA_CONSISTENT);
96 
97 	if (eq->ring == NULL) {
98 		oce_log(dev, CE_WARN, MOD_CONFIG,
99 		    "EQ ring alloc failed:0x%p",
100 		    (void *)eq->ring);
101 		kmem_free(eq, sizeof (struct oce_eq));
102 		return (NULL);
103 	}
104 
105 	mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0,
106 	    MBX_SUBSYSTEM_COMMON,
107 	    OPCODE_CREATE_COMMON_EQ, MBX_TIMEOUT_SEC,
108 	    sizeof (struct mbx_create_common_eq));
109 
110 	fwcmd->params.req.num_pages = eq->ring->dbuf->num_pages;
111 	oce_page_list(eq->ring->dbuf, &fwcmd->params.req.pages[0],
112 	    eq->ring->dbuf->num_pages);
113 
114 	/* dw 0 */
115 	fwcmd->params.req.eq_ctx.size = (item_size == 4) ? 0 : 1;
116 	fwcmd->params.req.eq_ctx.valid = 1;
117 	/* dw 1 */
118 	fwcmd->params.req.eq_ctx.armed = 0;
119 	fwcmd->params.req.eq_ctx.pd = 0;
120 	fwcmd->params.req.eq_ctx.count = OCE_LOG2(q_len/256);
121 
122 	/* dw 2 */
123 	fwcmd->params.req.eq_ctx.function = dev->fn;
124 	fwcmd->params.req.eq_ctx.nodelay  = 0;
125 	fwcmd->params.req.eq_ctx.phase = 0;
126 	/* todo: calculate multiplier from max min and cur */
127 	fwcmd->params.req.eq_ctx.delay_mult = eq_delay;
128 
129 	/* fill rest of mbx */
130 	mbx.u0.s.embedded = 1;
131 	mbx.payload_length = sizeof (struct mbx_create_common_eq);
132 	DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ);
133 
134 	/* now post the command */
135 	ret = oce_mbox_post(dev, &mbx, NULL);
136 
137 	if (ret != 0) {
138 		oce_log(dev, CE_WARN, MOD_CONFIG,
139 		    "EQ create failed: %d", ret);
140 		destroy_ring_buffer(dev, eq->ring);
141 		kmem_free(eq, sizeof (struct oce_eq));
142 		return (NULL);
143 	}
144 
145 	/* interpret the response */
146 	eq->eq_id = LE_16(fwcmd->params.rsp.eq_id);
147 	eq->eq_cfg.q_len = q_len;
148 	eq->eq_cfg.item_size = item_size;
149 	eq->eq_cfg.cur_eqd = (uint8_t)eq_delay;
150 	eq->parent = (void *)dev;
151 	atomic_inc_32(&dev->neqs);
152 	oce_log(dev, CE_NOTE, MOD_CONFIG,
153 	    "EQ created, eq=0x%p eq_id=0x%x", (void *)eq, eq->eq_id);
154 	/* Save the eq pointer */
155 	return (eq);
156 } /* oce_eq_create */
157 
158 /*
159  * function to delete an event queue
160  *
161  * dev - software handle to the device
162  * eq - handle to the eq to be deleted
163  *
164  * return 0=>success, failure otherwise
165  */
166 void
167 oce_eq_del(struct oce_dev *dev, struct oce_eq *eq)
168 {
169 	struct oce_mbx mbx;
170 	struct mbx_destroy_common_eq *fwcmd;
171 
172 	/* drain the residual events */
173 	oce_drain_eq(eq);
174 
175 	/* destroy the ring */
176 	destroy_ring_buffer(dev, eq->ring);
177 	eq->ring = NULL;
178 
179 	/* send a command to delete the EQ */
180 	fwcmd = (struct mbx_destroy_common_eq *)&mbx.payload;
181 	fwcmd->params.req.id = eq->eq_id;
182 	(void) oce_destroy_q(dev, &mbx,
183 	    sizeof (struct mbx_destroy_common_eq),
184 	    QTYPE_EQ);
185 	kmem_free(eq, sizeof (struct oce_eq));
186 	atomic_dec_32(&dev->neqs);
187 }
188 
189 /*
190  * function to create a completion queue
191  *
192  * dev - software handle to the device
193  * eq - optional eq to be associated with to the cq
194  * cqcfg - configuration for this queue
195  *
196  * return pointer to the cq created. NULL on failure
197  */
198 struct oce_cq *
199 oce_cq_create(struct oce_dev *dev, struct oce_eq *eq, uint32_t q_len,
200     uint32_t item_size, boolean_t sol_event, boolean_t is_eventable,
201     boolean_t nodelay, uint32_t ncoalesce)
202 {
203 	struct oce_cq *cq = NULL;
204 	struct oce_mbx mbx;
205 	struct mbx_create_common_cq *fwcmd;
206 	int ret = 0;
207 
208 	/* create cq */
209 	cq = kmem_zalloc(sizeof (struct oce_cq), KM_NOSLEEP);
210 	if (cq == NULL) {
211 		oce_log(dev, CE_NOTE, MOD_CONFIG, "%s",
212 		    "CQ allocation failed");
213 		return (NULL);
214 	}
215 
216 	/* create the ring buffer for this queue */
217 	cq->ring = create_ring_buffer(dev, q_len,
218 	    item_size, DDI_DMA_CONSISTENT);
219 	if (cq->ring == NULL) {
220 		oce_log(dev, CE_WARN, MOD_CONFIG,
221 		    "CQ ring alloc failed:0x%p",
222 		    (void *)cq->ring);
223 		kmem_free(cq, sizeof (struct oce_cq));
224 		return (NULL);
225 	}
226 	/* initialize mailbox */
227 	bzero(&mbx, sizeof (struct oce_mbx));
228 	fwcmd = (struct mbx_create_common_cq *)&mbx.payload;
229 
230 	/* fill the command header */
231 	mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0,
232 	    MBX_SUBSYSTEM_COMMON,
233 	    OPCODE_CREATE_COMMON_CQ, MBX_TIMEOUT_SEC,
234 	    sizeof (struct mbx_create_common_cq));
235 
236 	/* fill command context */
237 	/* dw0 */
238 	fwcmd->params.req.cq_ctx.eventable = is_eventable;
239 	fwcmd->params.req.cq_ctx.sol_event = sol_event;
240 	fwcmd->params.req.cq_ctx.valid = 1;
241 	fwcmd->params.req.cq_ctx.count = OCE_LOG2(q_len/256);
242 	fwcmd->params.req.cq_ctx.nodelay = nodelay;
243 	fwcmd->params.req.cq_ctx.coalesce_wm = ncoalesce;
244 
245 	/* dw1 */
246 	fwcmd->params.req.cq_ctx.armed = B_FALSE;
247 	fwcmd->params.req.cq_ctx.eq_id = eq->eq_id;
248 	fwcmd->params.req.cq_ctx.pd = 0;
249 	/* dw2 */
250 	fwcmd->params.req.cq_ctx.function = dev->fn;
251 
252 	/* fill the rest of the command */
253 	fwcmd->params.req.num_pages = cq->ring->dbuf->num_pages;
254 	oce_page_list(cq->ring->dbuf, &fwcmd->params.req.pages[0],
255 	    cq->ring->dbuf->num_pages);
256 
257 	/* fill rest of mbx */
258 	mbx.u0.s.embedded = 1;
259 	mbx.payload_length = sizeof (struct mbx_create_common_cq);
260 	DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ);
261 
262 	/* now send the mail box */
263 	ret = oce_mbox_post(dev, &mbx, NULL);
264 
265 	if (ret != 0) {
266 		oce_log(dev, CE_WARN, MOD_CONFIG,
267 		    "CQ create failed: 0x%x", ret);
268 		destroy_ring_buffer(dev, cq->ring);
269 		kmem_free(cq, sizeof (struct oce_cq));
270 		return (NULL);
271 	}
272 
273 	cq->parent = dev;
274 	cq->eq = eq; /* eq array index */
275 	cq->cq_cfg.q_len = q_len;
276 	cq->cq_cfg.item_size = item_size;
277 	cq->cq_cfg.sol_eventable = (uint8_t)sol_event;
278 	cq->cq_cfg.nodelay = (uint8_t)nodelay;
279 	/* interpret the response */
280 	cq->cq_id = LE_16(fwcmd->params.rsp.cq_id);
281 	dev->cq[cq->cq_id] = cq;
282 	atomic_inc_32(&eq->ref_count);
283 	return (cq);
284 } /* oce_cq_create */
285 
286 /*
287  * function to delete a completion queue
288  *
289  * dev - software handle to the device
290  * cq - handle to the CQ to delete
291  *
292  * return none
293  */
294 static void
295 oce_cq_del(struct oce_dev *dev, struct oce_cq *cq)
296 {
297 	struct oce_mbx mbx;
298 	struct mbx_destroy_common_cq *fwcmd;
299 
300 	/* destroy the ring */
301 	destroy_ring_buffer(dev, cq->ring);
302 	cq->ring = NULL;
303 
304 	bzero(&mbx, sizeof (struct oce_mbx));
305 	/* send a command to delete the CQ */
306 	fwcmd = (struct mbx_destroy_common_cq *)&mbx.payload;
307 	fwcmd->params.req.id = cq->cq_id;
308 	(void) oce_destroy_q(dev, &mbx,
309 	    sizeof (struct mbx_destroy_common_cq),
310 	    QTYPE_CQ);
311 
312 	/* Reset the handler */
313 	cq->cq_handler = NULL;
314 	dev->cq[cq->cq_id] = NULL;
315 	atomic_dec_32(&cq->eq->ref_count);
316 	mutex_destroy(&cq->lock);
317 
318 	/* release the eq */
319 	kmem_free(cq, sizeof (struct oce_cq));
320 } /* oce_cq_del */
321 
322 /*
323  * function to create an MQ
324  *
325  * dev - software handle to the device
326  * eq - the EQ to associate with the MQ for event notification
327  * q_len - the number of entries to create in the MQ
328  *
329  * return pointer to the created MQ, failure otherwise
330  */
331 struct oce_mq *
332 oce_mq_create(struct oce_dev *dev, struct oce_eq *eq, uint32_t q_len)
333 {
334 	struct oce_mbx mbx;
335 	struct mbx_create_common_mq *fwcmd;
336 	struct oce_mq *mq = NULL;
337 	int ret = 0;
338 	struct oce_cq  *cq;
339 
340 	/* Create the Completion Q */
341 	cq = oce_cq_create(dev, eq, CQ_LEN_256,
342 	    sizeof (struct oce_mq_cqe),
343 	    B_FALSE, B_TRUE, B_TRUE, 0);
344 	if (cq == NULL) {
345 		return (NULL);
346 	}
347 
348 
349 	/* allocate the mq */
350 	mq = kmem_zalloc(sizeof (struct oce_mq), KM_NOSLEEP);
351 
352 	if (mq == NULL) {
353 		goto mq_alloc_fail;
354 	}
355 
356 	bzero(&mbx, sizeof (struct oce_mbx));
357 	/* allocate mbx */
358 	fwcmd = (struct mbx_create_common_mq *)&mbx.payload;
359 
360 	/* create the ring buffer for this queue */
361 	mq->ring = create_ring_buffer(dev, q_len,
362 	    sizeof (struct oce_mbx), DDI_DMA_CONSISTENT);
363 	if (mq->ring == NULL) {
364 		oce_log(dev, CE_WARN, MOD_CONFIG,
365 		    "MQ ring alloc failed:0x%p",
366 		    (void *)mq->ring);
367 		goto mq_ring_alloc;
368 	}
369 
370 	mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0,
371 	    MBX_SUBSYSTEM_COMMON,
372 	    OPCODE_CREATE_COMMON_MQ, MBX_TIMEOUT_SEC,
373 	    sizeof (struct mbx_create_common_mq));
374 
375 	fwcmd->params.req.num_pages = mq->ring->dbuf->num_pages;
376 	oce_page_list(mq->ring->dbuf, fwcmd->params.req.pages,
377 	    mq->ring->dbuf->num_pages);
378 	fwcmd->params.req.context.u0.s.cq_id = cq->cq_id;
379 	fwcmd->params.req.context.u0.s.ring_size =
380 	    OCE_LOG2(q_len) + 1;
381 	fwcmd->params.req.context.u0.s.valid = 1;
382 	fwcmd->params.req.context.u0.s.fid = dev->fn;
383 
384 	/* fill rest of mbx */
385 	mbx.u0.s.embedded = 1;
386 	mbx.payload_length = sizeof (struct mbx_create_common_mq);
387 	DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ);
388 
389 	/* now send the mail box */
390 	ret = oce_mbox_post(dev, &mbx, NULL);
391 	if (ret != DDI_SUCCESS) {
392 		oce_log(dev, CE_WARN, MOD_CONFIG,
393 		    "MQ create failed: 0x%x", ret);
394 		goto mq_fail;
395 	}
396 
397 	/* interpret the response */
398 	mq->mq_id = LE_16(fwcmd->params.rsp.mq_id);
399 	mq->cq = cq;
400 	mq->cfg.q_len = (uint8_t)q_len;
401 	mq->cfg.eqd = 0;
402 
403 	/* fill rest of the mq */
404 	mq->parent = dev;
405 
406 	/* set the MQCQ handlers */
407 	cq->cq_handler = oce_drain_mq_cq;
408 	cq->cb_arg = (void *)mq;
409 	mutex_init(&mq->lock, NULL, MUTEX_DRIVER,
410 	    DDI_INTR_PRI(dev->intr_pri));
411 	return (mq);
412 
413 mq_fail:
414 	destroy_ring_buffer(dev, mq->ring);
415 mq_ring_alloc:
416 	kmem_free(mq, sizeof (struct oce_mq));
417 mq_alloc_fail:
418 	oce_cq_del(dev, cq);
419 	return (NULL);
420 } /* oce_mq_create */
421 
422 /*
423  * function to delete an MQ
424  *
425  * dev - software handle to the device
426  * mq - pointer to the MQ to delete
427  *
428  * return none
429  */
430 static void
431 oce_mq_del(struct oce_dev *dev, struct oce_mq *mq)
432 {
433 	struct oce_mbx mbx;
434 	struct mbx_destroy_common_mq *fwcmd;
435 
436 	/* destroy the ring */
437 	destroy_ring_buffer(dev, mq->ring);
438 	mq->ring = NULL;
439 	bzero(&mbx, sizeof (struct oce_mbx));
440 	fwcmd = (struct mbx_destroy_common_mq *)&mbx.payload;
441 	fwcmd->params.req.id = mq->mq_id;
442 	(void) oce_destroy_q(dev, &mbx,
443 	    sizeof (struct mbx_destroy_common_mq),
444 	    QTYPE_MQ);
445 	oce_cq_del(dev, mq->cq);
446 	mq->cq = NULL;
447 	mutex_destroy(&mq->lock);
448 	kmem_free(mq, sizeof (struct oce_mq));
449 } /* oce_mq_del */
450 
451 /*
452  * function to create a WQ for NIC Tx
453  *
454  * dev - software handle to the device
455  * wqcfg - configuration structure providing WQ config parameters
456  *
457  * return pointer to the WQ created. NULL on failure
458  */
459 static struct oce_wq *
460 oce_wq_init(struct oce_dev *dev,  uint32_t q_len, int wq_type)
461 {
462 	struct oce_wq *wq;
463 	char str[MAX_POOL_NAME];
464 	int ret;
465 
466 	ASSERT(dev != NULL);
467 	/* q_len must be min 256 and max 2k */
468 	if (q_len < 256 || q_len > 2048) {
469 		oce_log(dev, CE_WARN, MOD_CONFIG,
470 		    "Invalid q length. Must be "
471 		    "[256, 2000]: 0x%x", q_len);
472 		return (NULL);
473 	}
474 
475 	/* allocate wq */
476 	wq = kmem_zalloc(sizeof (struct oce_wq), KM_NOSLEEP);
477 	if (wq == NULL) {
478 		oce_log(dev, CE_WARN, MOD_CONFIG, "%s",
479 		    "WQ allocation failed");
480 	}
481 
482 	/* Set the wq config */
483 	wq->cfg.q_len = q_len;
484 	wq->cfg.wq_type = (uint8_t)wq_type;
485 	wq->cfg.eqd = OCE_DEFAULT_WQ_EQD;
486 	wq->cfg.nbufs = 2 * wq->cfg.q_len;
487 	wq->cfg.nhdl = 2 * wq->cfg.q_len;
488 
489 	/* assign parent */
490 	wq->parent = (void *)dev;
491 
492 	/* Create the WQ Buffer pool */
493 	ret  = oce_wqb_cache_create(wq, dev->tx_bcopy_limit);
494 	if (ret != DDI_SUCCESS) {
495 		oce_log(dev, CE_WARN, MOD_CONFIG, "%s",
496 		    "WQ Buffer Pool create failed ");
497 		goto wqb_fail;
498 	}
499 
500 	/* Create a pool of memory handles */
501 	ret = oce_wqm_cache_create(wq);
502 	if (ret != DDI_SUCCESS) {
503 		oce_log(dev, CE_WARN, MOD_CONFIG, "%s",
504 		    "WQ MAP Handles Pool create failed ");
505 		goto wqm_fail;
506 	}
507 
508 	(void) snprintf(str, MAX_POOL_NAME, "%s%d", "oce_wqed_", dev->dev_id);
509 	wq->wqed_cache = kmem_cache_create(str, sizeof (oce_wqe_desc_t),
510 	    0, oce_wqe_desc_ctor,
511 	    oce_wqe_desc_dtor, NULL, NULL, NULL, 0);
512 	if (wq->wqed_cache == NULL) {
513 		oce_log(dev, CE_WARN, MOD_CONFIG, "%s",
514 		    "WQ Packet Desc Pool create failed ");
515 		goto wqed_fail;
516 	}
517 
518 	/* create the ring buffer */
519 	wq->ring = create_ring_buffer(dev, q_len,
520 	    NIC_WQE_SIZE, DDI_DMA_CONSISTENT);
521 	if (wq->ring == NULL) {
522 		oce_log(dev, CE_WARN, MOD_CONFIG, "%s",
523 		    "Failed to create WQ ring ");
524 		goto wq_ringfail;
525 	}
526 
527 	/* Initialize WQ lock */
528 	mutex_init(&wq->tx_lock, NULL, MUTEX_DRIVER,
529 	    DDI_INTR_PRI(dev->intr_pri));
530 	/* Initialize WQ lock */
531 	mutex_init(&wq->txc_lock, NULL, MUTEX_DRIVER,
532 	    DDI_INTR_PRI(dev->intr_pri));
533 	atomic_inc_32(&dev->nwqs);
534 
535 	OCE_LIST_CREATE(&wq->wqe_desc_list, DDI_INTR_PRI(dev->intr_pri));
536 	return (wq);
537 
538 wqcq_fail:
539 	destroy_ring_buffer(dev, wq->ring);
540 wq_ringfail:
541 	kmem_cache_destroy(wq->wqed_cache);
542 wqed_fail:
543 	oce_wqm_cache_destroy(wq);
544 wqm_fail:
545 	oce_wqb_cache_destroy(wq);
546 wqb_fail:
547 	kmem_free(wq, sizeof (struct oce_wq));
548 	return (NULL);
549 } /* oce_wq_create */
550 
551 /*
552  * function to delete a WQ
553  *
554  * dev - software handle to the device
555  * wq - WQ to delete
556  *
557  * return 0 => success, failure otherwise
558  */
559 static void
560 oce_wq_fini(struct oce_dev *dev, struct oce_wq *wq)
561 {
562 	/* destroy cq */
563 	oce_wqb_cache_destroy(wq);
564 	oce_wqm_cache_destroy(wq);
565 	kmem_cache_destroy(wq->wqed_cache);
566 
567 	/* Free the packet descriptor list */
568 	OCE_LIST_DESTROY(&wq->wqe_desc_list);
569 	destroy_ring_buffer(dev, wq->ring);
570 	wq->ring = NULL;
571 	/* Destroy the Mutex */
572 	mutex_destroy(&wq->tx_lock);
573 	mutex_destroy(&wq->txc_lock);
574 	kmem_free(wq, sizeof (struct oce_wq));
575 	atomic_dec_32(&dev->nwqs);
576 } /* oce_wq_del */
577 
578 
579 static int
580 oce_wq_create(struct oce_wq *wq, struct oce_eq *eq)
581 {
582 
583 	struct oce_mbx mbx;
584 	struct mbx_create_nic_wq *fwcmd;
585 	struct oce_dev *dev = wq->parent;
586 	struct oce_cq *cq;
587 	int ret;
588 
589 	/* create the CQ */
590 	cq = oce_cq_create(dev, eq, CQ_LEN_1024,
591 	    sizeof (struct oce_nic_tx_cqe),
592 	    B_FALSE, B_TRUE, B_FALSE, 3);
593 	if (cq == NULL) {
594 		oce_log(dev, CE_WARN, MOD_CONFIG, "%s",
595 		    "WCCQ create failed ");
596 		return (DDI_FAILURE);
597 	}
598 	/* now fill the command */
599 	bzero(&mbx, sizeof (struct oce_mbx));
600 	fwcmd = (struct mbx_create_nic_wq *)&mbx.payload;
601 	mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0,
602 	    MBX_SUBSYSTEM_NIC,
603 	    OPCODE_CREATE_NIC_WQ, MBX_TIMEOUT_SEC,
604 	    sizeof (struct mbx_create_nic_wq));
605 
606 	fwcmd->params.req.nic_wq_type = (uint8_t)wq->cfg.wq_type;
607 	fwcmd->params.req.num_pages = wq->ring->dbuf->num_pages;
608 	oce_log(dev, CE_NOTE, MOD_CONFIG, "NUM_PAGES = 0x%d size = %lu",
609 	    (uint32_t)wq->ring->dbuf->num_pages,
610 	    wq->ring->dbuf->size);
611 
612 	/* workaround: fill 0x01 for ulp_mask in rsvd0 */
613 	fwcmd->params.req.rsvd0 = 0x01;
614 	fwcmd->params.req.wq_size = OCE_LOG2(wq->cfg.q_len) + 1;
615 	fwcmd->params.req.valid = 1;
616 	fwcmd->params.req.pd_id = 0;
617 	fwcmd->params.req.pci_function_id = dev->fn;
618 	fwcmd->params.req.cq_id = cq->cq_id;
619 
620 	oce_page_list(wq->ring->dbuf, fwcmd->params.req.pages,
621 	    wq->ring->dbuf->num_pages);
622 
623 	/* fill rest of mbx */
624 	mbx.u0.s.embedded = 1;
625 	mbx.payload_length = sizeof (struct mbx_create_nic_wq);
626 	DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ);
627 
628 	/* now post the command */
629 	ret = oce_mbox_post(dev, &mbx, NULL);
630 	if (ret != DDI_SUCCESS) {
631 		oce_log(dev, CE_WARN, MOD_CONFIG,
632 		    "WQ create failed: %d", ret);
633 		oce_cq_del(dev, cq);
634 		return (ret);
635 
636 	}
637 
638 	/* interpret the response */
639 	wq->wq_id = LE_16(fwcmd->params.rsp.wq_id);
640 	wq->qstate = QCREATED;
641 	wq->cq = cq;
642 	/* set the WQCQ handlers */
643 	wq->cq->cq_handler = oce_drain_wq_cq;
644 	wq->cq->cb_arg = (void *)wq;
645 	/* All are free to start with */
646 	wq->wq_free = wq->cfg.q_len;
647 	/* reset indicies */
648 	wq->ring->cidx = 0;
649 	wq->ring->pidx = 0;
650 	return (0);
651 }
652 
653 /*
654  * function to delete a WQ
655  *
656  * dev - software handle to the device
657  * wq - WQ to delete
658  *
659  * return none
660  */
661 static void
662 oce_wq_del(struct oce_dev *dev, struct oce_wq *wq)
663 {
664 	struct oce_mbx mbx;
665 	struct mbx_delete_nic_wq *fwcmd;
666 
667 
668 	ASSERT(dev != NULL);
669 	ASSERT(wq != NULL);
670 	if (wq->qstate == QCREATED) {
671 		bzero(&mbx, sizeof (struct oce_mbx));
672 		/* now fill the command */
673 		fwcmd = (struct mbx_delete_nic_wq *)&mbx.payload;
674 		fwcmd->params.req.wq_id = wq->wq_id;
675 		(void) oce_destroy_q(dev, &mbx,
676 		    sizeof (struct mbx_delete_nic_wq),
677 		    QTYPE_WQ);
678 		wq->qstate = QDELETED;
679 		oce_cq_del(dev, wq->cq);
680 		wq->cq = NULL;
681 	}
682 } /* oce_wq_del */
683 
684 /*
685  * function to allocate RQ resources
686  *
687  * dev - software handle to the device
688  * rqcfg - configuration structure providing RQ config parameters
689  *
690  * return pointer to the RQ created. NULL on failure
691  */
692 /* ARGSUSED */
693 static struct oce_rq *
694 oce_rq_init(struct oce_dev *dev, uint32_t q_len,
695     uint32_t frag_size, uint32_t mtu,
696     boolean_t rss)
697 {
698 
699 	struct oce_rq *rq;
700 	int ret;
701 
702 	/* validate q creation parameters */
703 	if (!OCE_LOG2(frag_size))
704 		return (NULL);
705 	if ((q_len == 0) || (q_len > 1024))
706 		return (NULL);
707 
708 	/* allocate the rq */
709 	rq = kmem_zalloc(sizeof (struct oce_rq), KM_NOSLEEP);
710 	if (rq == NULL) {
711 		oce_log(dev, CE_WARN, MOD_CONFIG, "%s",
712 		    "RQ allocation failed");
713 		return (NULL);
714 	}
715 
716 	rq->cfg.q_len = q_len;
717 	rq->cfg.frag_size = frag_size;
718 	rq->cfg.mtu = mtu;
719 	rq->cfg.eqd = 0;
720 	rq->cfg.nbufs = 8 * 1024;
721 
722 	/* assign parent */
723 	rq->parent = (void *)dev;
724 
725 	/* create the cache */
726 	ret  =  oce_rqb_cache_create(rq, OCE_RQ_BUF_SIZE +
727 	    OCE_RQE_BUF_HEADROOM);
728 	if (ret != DDI_SUCCESS) {
729 		goto rqb_fail;
730 	}
731 
732 	/* create the ring buffer */
733 	rq->ring = create_ring_buffer(dev, q_len,
734 	    sizeof (struct oce_nic_rqe), DDI_DMA_CONSISTENT);
735 	if (rq->ring == NULL) {
736 		oce_log(dev, CE_WARN, MOD_CONFIG, "%s",
737 		    "RQ ring create failed ");
738 		goto rq_ringfail;
739 	}
740 
741 	/* allocate mbx */
742 	rq->shadow_ring = kmem_zalloc(sizeof (struct rq_shadow_entry) *
743 	    q_len, KM_SLEEP);
744 
745 	/* Initialize the RQ lock */
746 	mutex_init(&rq->rx_lock, NULL, MUTEX_DRIVER,
747 	    DDI_INTR_PRI(dev->intr_pri));
748 	/* Initialize the recharge  lock */
749 	mutex_init(&rq->rc_lock, NULL, MUTEX_DRIVER,
750 	    DDI_INTR_PRI(dev->intr_pri));
751 	atomic_inc_32(&dev->nrqs);
752 	return (rq);
753 
754 rqcq_fail:
755 	kmem_free(rq->shadow_ring,
756 	    sizeof (struct rq_shadow_entry) * q_len);
757 	destroy_ring_buffer(dev, rq->ring);
758 rq_ringfail:
759 	oce_rqb_cache_destroy(rq);
760 rqb_fail:
761 	kmem_free(rq, sizeof (struct oce_rq));
762 	return (NULL);
763 } /* oce_rq_create */
764 
765 /*
766  * function to delete an RQ
767  *
768  * dev - software handle to the device
769  * rq - RQ to delete
770  *
771  * return none
772  */
773 static void
774 oce_rq_fini(struct oce_dev *dev, struct oce_rq *rq)
775 {
776 	/* Destroy buffer cache */
777 	oce_rqb_cache_destroy(rq);
778 	destroy_ring_buffer(dev, rq->ring);
779 	rq->ring = NULL;
780 	kmem_free(rq->shadow_ring,
781 	    sizeof (struct rq_shadow_entry) * rq->cfg.q_len);
782 	rq->shadow_ring = NULL;
783 	mutex_destroy(&rq->rx_lock);
784 	mutex_destroy(&rq->rc_lock);
785 	kmem_free(rq, sizeof (struct oce_rq));
786 	atomic_dec_32(&dev->nrqs);
787 } /* oce_rq_del */
788 
789 
790 static int
791 oce_rq_create(struct oce_rq *rq, uint32_t if_id, struct oce_eq *eq)
792 {
793 	struct oce_mbx mbx;
794 	struct mbx_create_nic_rq *fwcmd;
795 	struct oce_dev *dev = rq->parent;
796 	struct oce_cq *cq;
797 	int ret;
798 
799 	cq = oce_cq_create(dev, eq, CQ_LEN_1024, sizeof (struct oce_nic_rx_cqe),
800 	    B_FALSE, B_TRUE, B_FALSE, 3);
801 
802 	if (cq == NULL) {
803 		return (DDI_FAILURE);
804 	}
805 
806 	/* now fill the command */
807 	bzero(&mbx, sizeof (struct oce_mbx));
808 	fwcmd = (struct mbx_create_nic_rq *)&mbx.payload;
809 	mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0,
810 	    MBX_SUBSYSTEM_NIC,
811 	    OPCODE_CREATE_NIC_RQ, MBX_TIMEOUT_SEC,
812 	    sizeof (struct mbx_create_nic_rq));
813 
814 	fwcmd->params.req.num_pages = rq->ring->dbuf->num_pages;
815 	fwcmd->params.req.frag_size = OCE_LOG2(rq->cfg.frag_size);
816 	fwcmd->params.req.cq_id = cq->cq_id;
817 	oce_page_list(rq->ring->dbuf, fwcmd->params.req.pages,
818 	    rq->ring->dbuf->num_pages);
819 
820 	fwcmd->params.req.if_id = if_id;
821 	fwcmd->params.req.max_frame_size = (uint16_t)rq->cfg.mtu;
822 	fwcmd->params.req.is_rss_queue = rq->cfg.is_rss_queue;
823 
824 	/* fill rest of mbx */
825 	mbx.u0.s.embedded = 1;
826 	mbx.payload_length = sizeof (struct mbx_create_nic_rq);
827 	DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ);
828 
829 	/* now post the command */
830 	ret = oce_mbox_post(dev, &mbx, NULL);
831 	if (ret != 0) {
832 		oce_log(dev, CE_WARN, MOD_CONFIG,
833 		    "RQ create failed: %d", ret);
834 		oce_cq_del(dev, cq);
835 		return (ret);
836 	}
837 
838 	/* interpret the response */
839 	rq->rq_id = LE_16(fwcmd->params.rsp.u0.s.rq_id);
840 	/* rq->rss_cpuid = fwcmd->params.rsp.u0.bits.rss_cpuid; */
841 	rq->cfg.if_id = if_id;
842 	rq->qstate = QCREATED;
843 	rq->cq = cq;
844 
845 	/* set the Completion Handler */
846 	rq->cq->cq_handler = oce_drain_rq_cq;
847 	rq->cq->cb_arg  = (void *)rq;
848 	/* reset the indicies */
849 	rq->ring->cidx = 0;
850 	rq->ring->pidx = 0;
851 	rq->buf_avail = 0;
852 	return (0);
853 
854 }
855 
856 /*
857  * function to delete an RQ
858  *
859  * dev - software handle to the device
860  * rq - RQ to delete
861  *
862  * return none
863  */
864 static void
865 oce_rq_del(struct oce_dev *dev, struct oce_rq *rq)
866 {
867 	struct oce_mbx mbx;
868 	struct mbx_delete_nic_rq *fwcmd;
869 
870 	ASSERT(dev != NULL);
871 	ASSERT(rq != NULL);
872 
873 	bzero(&mbx, sizeof (struct oce_mbx));
874 
875 	/* delete the Queue  */
876 	if (rq->qstate == QCREATED) {
877 		fwcmd = (struct mbx_delete_nic_rq *)&mbx.payload;
878 		fwcmd->params.req.rq_id = rq->rq_id;
879 		(void) oce_destroy_q(dev, &mbx,
880 		    sizeof (struct mbx_delete_nic_rq), QTYPE_RQ);
881 		rq->qstate = QDELETED;
882 		oce_clean_rq(rq);
883 		/* Delete the associated CQ */
884 		oce_cq_del(dev, rq->cq);
885 		rq->cq = NULL;
886 		/* free up the posted buffers */
887 		oce_rq_discharge(dev->rq[0]);
888 	}
889 } /* oce_rq_del */
890 
891 /*
892  * function to arm an EQ so that it can generate events
893  *
894  * dev - software handle to the device
895  * qid - id of the EQ returned by the fw at the time of creation
896  * npopped - number of EQEs to arm with
897  * rearm - rearm bit
898  * clearint - bit to clear the interrupt condition because of which
899  *	EQEs are generated
900  *
901  * return none
902  */
903 void
904 oce_arm_eq(struct oce_dev *dev, int16_t qid, int npopped,
905     boolean_t rearm, boolean_t clearint)
906 {
907 	eq_db_t eq_db = {0};
908 
909 	eq_db.bits.rearm = rearm;
910 	eq_db.bits.event  = B_TRUE;
911 	eq_db.bits.num_popped = npopped;
912 	eq_db.bits.clrint = clearint;
913 	eq_db.bits.qid = qid;
914 	OCE_DB_WRITE32(dev, PD_EQ_DB, eq_db.dw0);
915 }
916 
917 /*
918  * function to arm a CQ with CQEs
919  *
920  * dev - software handle to the device
921  * qid - the id of the CQ returned by the fw at the time of creation
922  * npopped - number of CQEs to arm with
923  * rearm - rearm bit enable/disable
924  *
925  * return none
926  */
927 void
928 oce_arm_cq(struct oce_dev *dev, int16_t qid, int npopped,
929     boolean_t rearm)
930 {
931 	cq_db_t cq_db = {0};
932 	cq_db.bits.rearm = rearm;
933 	cq_db.bits.num_popped = npopped;
934 	cq_db.bits.event = 0;
935 	cq_db.bits.qid = qid;
936 	OCE_DB_WRITE32(dev, PD_CQ_DB, cq_db.dw0);
937 }
938 
939 
940 /*
941  * function to delete a EQ, CQ, MQ, WQ or RQ
942  *
943  * dev - sofware handle to the device
944  * mbx - mbox command to send to the fw to delete the queue
945  *	mbx contains the queue information to delete
946  * req_size - the size of the mbx payload dependent on the qtype
947  * qtype - the type of queue i.e. EQ, CQ, MQ, WQ or RQ
948  *
949  * return DDI_SUCCESS => success, failure otherwise
950  */
951 int
952 oce_destroy_q(struct oce_dev *dev, struct oce_mbx  *mbx, size_t req_size,
953     enum qtype qtype)
954 {
955 	struct mbx_hdr *hdr = (struct mbx_hdr *)&mbx->payload;
956 	int opcode;
957 	int subsys;
958 	int ret;
959 
960 	switch (qtype) {
961 	case QTYPE_EQ: {
962 		opcode = OPCODE_DESTROY_COMMON_EQ;
963 		subsys = MBX_SUBSYSTEM_COMMON;
964 		break;
965 	}
966 	case QTYPE_CQ: {
967 		opcode = OPCODE_DESTROY_COMMON_CQ;
968 		subsys = MBX_SUBSYSTEM_COMMON;
969 		break;
970 	}
971 	case QTYPE_MQ: {
972 		opcode = OPCODE_DESTROY_COMMON_MQ;
973 		subsys = MBX_SUBSYSTEM_COMMON;
974 		break;
975 	}
976 	case QTYPE_WQ: {
977 		opcode = OPCODE_DELETE_NIC_WQ;
978 		subsys = MBX_SUBSYSTEM_NIC;
979 		break;
980 	}
981 	case QTYPE_RQ: {
982 		opcode = OPCODE_DELETE_NIC_RQ;
983 		subsys = MBX_SUBSYSTEM_NIC;
984 		break;
985 	}
986 	default: {
987 		ASSERT(0);
988 		break;
989 	}
990 	}
991 
992 	mbx_common_req_hdr_init(hdr, 0, 0, subsys,
993 	    opcode, MBX_TIMEOUT_SEC, req_size);
994 
995 	/* fill rest of mbx */
996 	mbx->u0.s.embedded = 1;
997 	mbx->payload_length = (uint32_t)req_size;
998 	DW_SWAP(u32ptr(mbx), mbx->payload_length + OCE_BMBX_RHDR_SZ);
999 
1000 	/* send command */
1001 	ret = oce_mbox_post(dev, mbx, NULL);
1002 
1003 	if (ret != 0) {
1004 		oce_log(dev, CE_WARN, MOD_CONFIG, "%s",
1005 		    "Failed to del q ");
1006 	}
1007 	return (ret);
1008 }
1009 
1010 /*
1011  * function to set the delay parameter in the EQ for interrupt coalescing
1012  *
1013  * dev - software handle to the device
1014  * eq_arr - array of EQ ids to delete
1015  * eq_cnt - number of elements in eq_arr
1016  * eq_delay - delay parameter
1017  *
1018  * return DDI_SUCCESS => success, failure otherwise
1019  */
1020 int
1021 oce_set_eq_delay(struct oce_dev *dev, uint32_t *eq_arr,
1022     uint32_t eq_cnt, uint32_t eq_delay)
1023 {
1024 	struct oce_mbx mbx;
1025 	struct mbx_modify_common_eq_delay *fwcmd;
1026 	int ret;
1027 	int neq;
1028 
1029 	bzero(&mbx, sizeof (struct oce_mbx));
1030 	fwcmd = (struct mbx_modify_common_eq_delay *)&mbx.payload;
1031 
1032 	/* fill the command */
1033 	fwcmd->params.req.num_eq = eq_cnt;
1034 	for (neq = 0; neq < eq_cnt; neq++) {
1035 		fwcmd->params.req.delay[neq].eq_id = eq_arr[neq];
1036 		fwcmd->params.req.delay[neq].phase = 0;
1037 		fwcmd->params.req.delay[neq].dm = eq_delay;
1038 
1039 	}
1040 
1041 	/* initialize the ioctl header */
1042 	mbx_common_req_hdr_init(&fwcmd->hdr, 0, 0,
1043 	    MBX_SUBSYSTEM_COMMON,
1044 	    OPCODE_MODIFY_COMMON_EQ_DELAY,
1045 	    MBX_TIMEOUT_SEC,
1046 	    sizeof (struct mbx_modify_common_eq_delay));
1047 
1048 	/* fill rest of mbx */
1049 	mbx.u0.s.embedded = 1;
1050 	mbx.payload_length = sizeof (struct mbx_modify_common_eq_delay);
1051 	DW_SWAP(u32ptr(&mbx), mbx.payload_length + OCE_BMBX_RHDR_SZ);
1052 
1053 	/* post the command */
1054 	ret = oce_mbox_post(dev, &mbx, NULL);
1055 	if (ret != 0) {
1056 		oce_log(dev, CE_WARN, MOD_CONFIG,
1057 		    "Failed to set EQ delay %d", ret);
1058 	}
1059 
1060 	return (ret);
1061 } /* oce_set_eq_delay */
1062 
1063 /*
1064  * function to cleanup the eqs used during stop
1065  *
1066  * eq - pointer to event queue structure
1067  *
1068  * return none
1069  */
1070 void
1071 oce_drain_eq(struct oce_eq *eq)
1072 {
1073 	struct oce_eqe *eqe;
1074 	uint16_t num_eqe = 0;
1075 	struct oce_dev *dev;
1076 
1077 	dev = eq->parent;
1078 	/* get the first item in eq to process */
1079 	eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe);
1080 
1081 	while (eqe->u0.dw0) {
1082 		eqe->u0.dw0 = LE_32(eqe->u0.dw0);
1083 
1084 		/* clear valid bit */
1085 		eqe->u0.dw0 = 0;
1086 
1087 		/* process next eqe */
1088 		RING_GET(eq->ring, 1);
1089 
1090 		eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe);
1091 		num_eqe++;
1092 	} /* for all EQEs */
1093 	if (num_eqe) {
1094 		oce_arm_eq(dev, eq->eq_id, num_eqe, B_FALSE, B_TRUE);
1095 	}
1096 } /* oce_drain_eq */
1097 
1098 
1099 int
1100 oce_init_txrx(struct oce_dev  *dev)
1101 {
1102 	dev->wq[0] = oce_wq_init(dev, dev->tx_ring_size, NIC_WQ_TYPE_STANDARD);
1103 
1104 	if (dev->wq[0] == NULL) {
1105 		goto queue_fail;
1106 	}
1107 
1108 	dev->rq[0] = oce_rq_init(dev, dev->rx_ring_size, OCE_RQ_BUF_SIZE,
1109 	    OCE_RQ_MAX_FRAME_SZ, B_FALSE);
1110 	if (dev->rq[0] == NULL) {
1111 		goto queue_fail;
1112 	}
1113 	return (DDI_SUCCESS);
1114 queue_fail:
1115 	oce_fini_txrx(dev);
1116 	return (DDI_FAILURE);
1117 }
1118 void
1119 oce_fini_txrx(struct oce_dev *dev)
1120 {
1121 	if (dev->wq[0] != NULL) {
1122 		oce_wq_fini(dev, dev->wq[0]);
1123 		dev->wq[0] = NULL;
1124 	}
1125 	if (dev->rq[0] != NULL) {
1126 		oce_rq_fini(dev, dev->rq[0]);
1127 		dev->rq[0] = NULL;
1128 	}
1129 	return;
1130 
1131 }
1132 
1133 int
1134 oce_create_queues(struct oce_dev *dev)
1135 {
1136 
1137 	int i;
1138 	struct oce_eq *eq;
1139 	struct oce_mq *mq;
1140 
1141 	for (i = 0; i < dev->num_vectors; i++) {
1142 		eq = oce_eq_create(dev, EQ_LEN_1024, EQE_SIZE_4, 0);
1143 		if (eq == NULL) {
1144 			goto rings_fail;
1145 		}
1146 		dev->eq[i] = eq;
1147 	}
1148 	if (oce_wq_create(dev->wq[0], dev->eq[0]) != 0)
1149 		goto rings_fail;
1150 	if (oce_rq_create(dev->rq[0], dev->if_id,
1151 	    dev->neqs > 1 ? dev->eq[1] : dev->eq[0]) != 0)
1152 		goto rings_fail;
1153 	mq = oce_mq_create(dev, dev->eq[0], 64);
1154 	if (mq == NULL)
1155 		goto rings_fail;
1156 	dev->mq = mq;
1157 	return (DDI_SUCCESS);
1158 rings_fail:
1159 	oce_delete_queues(dev);
1160 	return (DDI_FAILURE);
1161 
1162 }
1163 
1164 void
1165 oce_delete_queues(struct oce_dev *dev)
1166 {
1167 	int i;
1168 	if (dev->mq != NULL) {
1169 		oce_mq_del(dev, dev->mq);
1170 		dev->mq = NULL;
1171 	}
1172 
1173 	for (i = 0; i < dev->nrqs; i++) {
1174 		oce_rq_del(dev, dev->rq[i]);
1175 	}
1176 	for (i = 0; i < dev->nwqs; i++) {
1177 		oce_wq_del(dev, dev->wq[i]);
1178 	}
1179 	/* create as many eqs as the number of vectors */
1180 	for (i = 0; i < dev->num_vectors; i++) {
1181 		oce_eq_del(dev, dev->eq[i]);
1182 		dev->eq[i] = NULL;
1183 	}
1184 }
1185