1 /******************************************************************************
2 
3   Copyright (c) 2013-2018, Intel Corporation
4   All rights reserved.
5 
6   Redistribution and use in source and binary forms, with or without
7   modification, are permitted provided that the following conditions are met:
8 
9    1. Redistributions of source code must retain the above copyright notice,
10       this list of conditions and the following disclaimer.
11 
12    2. Redistributions in binary form must reproduce the above copyright
13       notice, this list of conditions and the following disclaimer in the
14       documentation and/or other materials provided with the distribution.
15 
16    3. Neither the name of the Intel Corporation nor the names of its
17       contributors may be used to endorse or promote products derived from
18       this software without specific prior written permission.
19 
20   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22   IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23   ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24   LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25   CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26   SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27   INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28   CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29   ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30   POSSIBILITY OF SUCH DAMAGE.
31 
32 ******************************************************************************/
33 /*$FreeBSD$*/
34 
35 #include "i40e_status.h"
36 #include "i40e_type.h"
37 #include "i40e_register.h"
38 #include "i40e_adminq.h"
39 #include "i40e_prototype.h"
40 
41 /**
42  *  i40e_adminq_init_regs - Initialize AdminQ registers
43  *  @hw: pointer to the hardware structure
44  *
45  *  This assumes the alloc_asq and alloc_arq functions have already been called
46  **/
i40e_adminq_init_regs(struct i40e_hw * hw)47 static void i40e_adminq_init_regs(struct i40e_hw *hw)
48 {
49 	/* set head and tail registers in our local struct */
50 	if (i40e_is_vf(hw)) {
51 		hw->aq.asq.tail = I40E_VF_ATQT1;
52 		hw->aq.asq.head = I40E_VF_ATQH1;
53 		hw->aq.asq.len  = I40E_VF_ATQLEN1;
54 		hw->aq.asq.bal  = I40E_VF_ATQBAL1;
55 		hw->aq.asq.bah  = I40E_VF_ATQBAH1;
56 		hw->aq.arq.tail = I40E_VF_ARQT1;
57 		hw->aq.arq.head = I40E_VF_ARQH1;
58 		hw->aq.arq.len  = I40E_VF_ARQLEN1;
59 		hw->aq.arq.bal  = I40E_VF_ARQBAL1;
60 		hw->aq.arq.bah  = I40E_VF_ARQBAH1;
61 	} else {
62 		hw->aq.asq.tail = I40E_PF_ATQT;
63 		hw->aq.asq.head = I40E_PF_ATQH;
64 		hw->aq.asq.len  = I40E_PF_ATQLEN;
65 		hw->aq.asq.bal  = I40E_PF_ATQBAL;
66 		hw->aq.asq.bah  = I40E_PF_ATQBAH;
67 		hw->aq.arq.tail = I40E_PF_ARQT;
68 		hw->aq.arq.head = I40E_PF_ARQH;
69 		hw->aq.arq.len  = I40E_PF_ARQLEN;
70 		hw->aq.arq.bal  = I40E_PF_ARQBAL;
71 		hw->aq.arq.bah  = I40E_PF_ARQBAH;
72 	}
73 }
74 
75 /**
76  *  i40e_alloc_adminq_asq_ring - Allocate Admin Queue send rings
77  *  @hw: pointer to the hardware structure
78  **/
i40e_alloc_adminq_asq_ring(struct i40e_hw * hw)79 enum i40e_status_code i40e_alloc_adminq_asq_ring(struct i40e_hw *hw)
80 {
81 	enum i40e_status_code ret_code;
82 
83 	ret_code = i40e_allocate_dma_mem(hw, &hw->aq.asq.desc_buf,
84 					 i40e_mem_atq_ring,
85 					 (hw->aq.num_asq_entries *
86 					 sizeof(struct i40e_aq_desc)),
87 					 I40E_ADMINQ_DESC_ALIGNMENT);
88 	if (ret_code)
89 		return ret_code;
90 
91 	ret_code = i40e_allocate_virt_mem(hw, &hw->aq.asq.cmd_buf,
92 					  (hw->aq.num_asq_entries *
93 					  sizeof(struct i40e_asq_cmd_details)));
94 	if (ret_code) {
95 		i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf);
96 		return ret_code;
97 	}
98 
99 	return ret_code;
100 }
101 
102 /**
103  *  i40e_alloc_adminq_arq_ring - Allocate Admin Queue receive rings
104  *  @hw: pointer to the hardware structure
105  **/
i40e_alloc_adminq_arq_ring(struct i40e_hw * hw)106 enum i40e_status_code i40e_alloc_adminq_arq_ring(struct i40e_hw *hw)
107 {
108 	enum i40e_status_code ret_code;
109 
110 	ret_code = i40e_allocate_dma_mem(hw, &hw->aq.arq.desc_buf,
111 					 i40e_mem_arq_ring,
112 					 (hw->aq.num_arq_entries *
113 					 sizeof(struct i40e_aq_desc)),
114 					 I40E_ADMINQ_DESC_ALIGNMENT);
115 
116 	return ret_code;
117 }
118 
119 /**
120  *  i40e_free_adminq_asq - Free Admin Queue send rings
121  *  @hw: pointer to the hardware structure
122  *
123  *  This assumes the posted send buffers have already been cleaned
124  *  and de-allocated
125  **/
i40e_free_adminq_asq(struct i40e_hw * hw)126 void i40e_free_adminq_asq(struct i40e_hw *hw)
127 {
128 	i40e_free_virt_mem(hw, &hw->aq.asq.cmd_buf);
129 	i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf);
130 }
131 
132 /**
133  *  i40e_free_adminq_arq - Free Admin Queue receive rings
134  *  @hw: pointer to the hardware structure
135  *
136  *  This assumes the posted receive buffers have already been cleaned
137  *  and de-allocated
138  **/
i40e_free_adminq_arq(struct i40e_hw * hw)139 void i40e_free_adminq_arq(struct i40e_hw *hw)
140 {
141 	i40e_free_dma_mem(hw, &hw->aq.arq.desc_buf);
142 }
143 
144 /**
145  *  i40e_alloc_arq_bufs - Allocate pre-posted buffers for the receive queue
146  *  @hw: pointer to the hardware structure
147  **/
i40e_alloc_arq_bufs(struct i40e_hw * hw)148 static enum i40e_status_code i40e_alloc_arq_bufs(struct i40e_hw *hw)
149 {
150 	enum i40e_status_code ret_code;
151 	struct i40e_aq_desc *desc;
152 	struct i40e_dma_mem *bi;
153 	int i;
154 
155 	/* We'll be allocating the buffer info memory first, then we can
156 	 * allocate the mapped buffers for the event processing
157 	 */
158 
159 	/* buffer_info structures do not need alignment */
160 	ret_code = i40e_allocate_virt_mem(hw, &hw->aq.arq.dma_head,
161 		(hw->aq.num_arq_entries * sizeof(struct i40e_dma_mem)));
162 	if (ret_code)
163 		goto alloc_arq_bufs;
164 	hw->aq.arq.r.arq_bi = (struct i40e_dma_mem *)hw->aq.arq.dma_head.va;
165 
166 	/* allocate the mapped buffers */
167 	for (i = 0; i < hw->aq.num_arq_entries; i++) {
168 		bi = &hw->aq.arq.r.arq_bi[i];
169 		ret_code = i40e_allocate_dma_mem(hw, bi,
170 						 i40e_mem_arq_buf,
171 						 hw->aq.arq_buf_size,
172 						 I40E_ADMINQ_DESC_ALIGNMENT);
173 		if (ret_code)
174 			goto unwind_alloc_arq_bufs;
175 
176 		/* now configure the descriptors for use */
177 		desc = I40E_ADMINQ_DESC(hw->aq.arq, i);
178 
179 		desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_BUF);
180 		if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF)
181 			desc->flags |= CPU_TO_LE16(I40E_AQ_FLAG_LB);
182 		desc->opcode = 0;
183 		/* This is in accordance with Admin queue design, there is no
184 		 * register for buffer size configuration
185 		 */
186 		desc->datalen = CPU_TO_LE16((u16)bi->size);
187 		desc->retval = 0;
188 		desc->cookie_high = 0;
189 		desc->cookie_low = 0;
190 		desc->params.external.addr_high =
191 			CPU_TO_LE32(I40E_HI_DWORD(bi->pa));
192 		desc->params.external.addr_low =
193 			CPU_TO_LE32(I40E_LO_DWORD(bi->pa));
194 		desc->params.external.param0 = 0;
195 		desc->params.external.param1 = 0;
196 	}
197 
198 alloc_arq_bufs:
199 	return ret_code;
200 
201 unwind_alloc_arq_bufs:
202 	/* don't try to free the one that failed... */
203 	i--;
204 	for (; i >= 0; i--)
205 		i40e_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);
206 	i40e_free_virt_mem(hw, &hw->aq.arq.dma_head);
207 
208 	return ret_code;
209 }
210 
211 /**
212  *  i40e_alloc_asq_bufs - Allocate empty buffer structs for the send queue
213  *  @hw: pointer to the hardware structure
214  **/
i40e_alloc_asq_bufs(struct i40e_hw * hw)215 static enum i40e_status_code i40e_alloc_asq_bufs(struct i40e_hw *hw)
216 {
217 	enum i40e_status_code ret_code;
218 	struct i40e_dma_mem *bi;
219 	int i;
220 
221 	/* No mapped memory needed yet, just the buffer info structures */
222 	ret_code = i40e_allocate_virt_mem(hw, &hw->aq.asq.dma_head,
223 		(hw->aq.num_asq_entries * sizeof(struct i40e_dma_mem)));
224 	if (ret_code)
225 		goto alloc_asq_bufs;
226 	hw->aq.asq.r.asq_bi = (struct i40e_dma_mem *)hw->aq.asq.dma_head.va;
227 
228 	/* allocate the mapped buffers */
229 	for (i = 0; i < hw->aq.num_asq_entries; i++) {
230 		bi = &hw->aq.asq.r.asq_bi[i];
231 		ret_code = i40e_allocate_dma_mem(hw, bi,
232 						 i40e_mem_asq_buf,
233 						 hw->aq.asq_buf_size,
234 						 I40E_ADMINQ_DESC_ALIGNMENT);
235 		if (ret_code)
236 			goto unwind_alloc_asq_bufs;
237 	}
238 alloc_asq_bufs:
239 	return ret_code;
240 
241 unwind_alloc_asq_bufs:
242 	/* don't try to free the one that failed... */
243 	i--;
244 	for (; i >= 0; i--)
245 		i40e_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);
246 	i40e_free_virt_mem(hw, &hw->aq.asq.dma_head);
247 
248 	return ret_code;
249 }
250 
251 /**
252  *  i40e_free_arq_bufs - Free receive queue buffer info elements
253  *  @hw: pointer to the hardware structure
254  **/
i40e_free_arq_bufs(struct i40e_hw * hw)255 static void i40e_free_arq_bufs(struct i40e_hw *hw)
256 {
257 	int i;
258 
259 	/* free descriptors */
260 	for (i = 0; i < hw->aq.num_arq_entries; i++)
261 		i40e_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);
262 
263 	/* free the descriptor memory */
264 	i40e_free_dma_mem(hw, &hw->aq.arq.desc_buf);
265 
266 	/* free the dma header */
267 	i40e_free_virt_mem(hw, &hw->aq.arq.dma_head);
268 }
269 
270 /**
271  *  i40e_free_asq_bufs - Free send queue buffer info elements
272  *  @hw: pointer to the hardware structure
273  **/
i40e_free_asq_bufs(struct i40e_hw * hw)274 static void i40e_free_asq_bufs(struct i40e_hw *hw)
275 {
276 	int i;
277 
278 	/* only unmap if the address is non-NULL */
279 	for (i = 0; i < hw->aq.num_asq_entries; i++)
280 		if (hw->aq.asq.r.asq_bi[i].pa)
281 			i40e_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);
282 
283 	/* free the buffer info list */
284 	i40e_free_virt_mem(hw, &hw->aq.asq.cmd_buf);
285 
286 	/* free the descriptor memory */
287 	i40e_free_dma_mem(hw, &hw->aq.asq.desc_buf);
288 
289 	/* free the dma header */
290 	i40e_free_virt_mem(hw, &hw->aq.asq.dma_head);
291 }
292 
293 /**
294  *  i40e_config_asq_regs - configure ASQ registers
295  *  @hw: pointer to the hardware structure
296  *
297  *  Configure base address and length registers for the transmit queue
298  **/
i40e_config_asq_regs(struct i40e_hw * hw)299 static enum i40e_status_code i40e_config_asq_regs(struct i40e_hw *hw)
300 {
301 	enum i40e_status_code ret_code = I40E_SUCCESS;
302 	u32 reg = 0;
303 
304 	/* Clear Head and Tail */
305 	wr32(hw, hw->aq.asq.head, 0);
306 	wr32(hw, hw->aq.asq.tail, 0);
307 
308 	/* set starting point */
309 	if (!i40e_is_vf(hw))
310 		wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries |
311 					  I40E_PF_ATQLEN_ATQENABLE_MASK));
312 	if (i40e_is_vf(hw))
313 		wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries |
314 					  I40E_VF_ATQLEN1_ATQENABLE_MASK));
315 	wr32(hw, hw->aq.asq.bal, I40E_LO_DWORD(hw->aq.asq.desc_buf.pa));
316 	wr32(hw, hw->aq.asq.bah, I40E_HI_DWORD(hw->aq.asq.desc_buf.pa));
317 
318 	/* Check one register to verify that config was applied */
319 	reg = rd32(hw, hw->aq.asq.bal);
320 	if (reg != I40E_LO_DWORD(hw->aq.asq.desc_buf.pa))
321 		ret_code = I40E_ERR_ADMIN_QUEUE_ERROR;
322 
323 	return ret_code;
324 }
325 
326 /**
327  *  i40e_config_arq_regs - ARQ register configuration
328  *  @hw: pointer to the hardware structure
329  *
330  * Configure base address and length registers for the receive (event queue)
331  **/
i40e_config_arq_regs(struct i40e_hw * hw)332 static enum i40e_status_code i40e_config_arq_regs(struct i40e_hw *hw)
333 {
334 	enum i40e_status_code ret_code = I40E_SUCCESS;
335 	u32 reg = 0;
336 
337 	/* Clear Head and Tail */
338 	wr32(hw, hw->aq.arq.head, 0);
339 	wr32(hw, hw->aq.arq.tail, 0);
340 
341 	/* set starting point */
342 	if (!i40e_is_vf(hw))
343 		wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries |
344 					  I40E_PF_ARQLEN_ARQENABLE_MASK));
345 	if (i40e_is_vf(hw))
346 		wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries |
347 					  I40E_VF_ARQLEN1_ARQENABLE_MASK));
348 	wr32(hw, hw->aq.arq.bal, I40E_LO_DWORD(hw->aq.arq.desc_buf.pa));
349 	wr32(hw, hw->aq.arq.bah, I40E_HI_DWORD(hw->aq.arq.desc_buf.pa));
350 
351 	/* Update tail in the HW to post pre-allocated buffers */
352 	wr32(hw, hw->aq.arq.tail, hw->aq.num_arq_entries - 1);
353 
354 	/* Check one register to verify that config was applied */
355 	reg = rd32(hw, hw->aq.arq.bal);
356 	if (reg != I40E_LO_DWORD(hw->aq.arq.desc_buf.pa))
357 		ret_code = I40E_ERR_ADMIN_QUEUE_ERROR;
358 
359 	return ret_code;
360 }
361 
362 /**
363  *  i40e_init_asq - main initialization routine for ASQ
364  *  @hw: pointer to the hardware structure
365  *
366  *  This is the main initialization routine for the Admin Send Queue
367  *  Prior to calling this function, drivers *MUST* set the following fields
368  *  in the hw->aq structure:
369  *     - hw->aq.num_asq_entries
370  *     - hw->aq.arq_buf_size
371  *
372  *  Do *NOT* hold the lock when calling this as the memory allocation routines
373  *  called are not going to be atomic context safe
374  **/
i40e_init_asq(struct i40e_hw * hw)375 enum i40e_status_code i40e_init_asq(struct i40e_hw *hw)
376 {
377 	enum i40e_status_code ret_code = I40E_SUCCESS;
378 
379 	if (hw->aq.asq.count > 0) {
380 		/* queue already initialized */
381 		ret_code = I40E_ERR_NOT_READY;
382 		goto init_adminq_exit;
383 	}
384 
385 	/* verify input for valid configuration */
386 	if ((hw->aq.num_asq_entries == 0) ||
387 	    (hw->aq.asq_buf_size == 0)) {
388 		ret_code = I40E_ERR_CONFIG;
389 		goto init_adminq_exit;
390 	}
391 
392 	hw->aq.asq.next_to_use = 0;
393 	hw->aq.asq.next_to_clean = 0;
394 
395 	/* allocate the ring memory */
396 	ret_code = i40e_alloc_adminq_asq_ring(hw);
397 	if (ret_code != I40E_SUCCESS)
398 		goto init_adminq_exit;
399 
400 	/* allocate buffers in the rings */
401 	ret_code = i40e_alloc_asq_bufs(hw);
402 	if (ret_code != I40E_SUCCESS)
403 		goto init_adminq_free_rings;
404 
405 	/* initialize base registers */
406 	ret_code = i40e_config_asq_regs(hw);
407 	if (ret_code != I40E_SUCCESS)
408 		goto init_config_regs;
409 
410 	/* success! */
411 	hw->aq.asq.count = hw->aq.num_asq_entries;
412 	goto init_adminq_exit;
413 
414 init_adminq_free_rings:
415 	i40e_free_adminq_asq(hw);
416 	return ret_code;
417 
418 init_config_regs:
419 	i40e_free_asq_bufs(hw);
420 
421 init_adminq_exit:
422 	return ret_code;
423 }
424 
425 /**
426  *  i40e_init_arq - initialize ARQ
427  *  @hw: pointer to the hardware structure
428  *
429  *  The main initialization routine for the Admin Receive (Event) Queue.
430  *  Prior to calling this function, drivers *MUST* set the following fields
431  *  in the hw->aq structure:
432  *     - hw->aq.num_asq_entries
433  *     - hw->aq.arq_buf_size
434  *
435  *  Do *NOT* hold the lock when calling this as the memory allocation routines
436  *  called are not going to be atomic context safe
437  **/
i40e_init_arq(struct i40e_hw * hw)438 enum i40e_status_code i40e_init_arq(struct i40e_hw *hw)
439 {
440 	enum i40e_status_code ret_code = I40E_SUCCESS;
441 
442 	if (hw->aq.arq.count > 0) {
443 		/* queue already initialized */
444 		ret_code = I40E_ERR_NOT_READY;
445 		goto init_adminq_exit;
446 	}
447 
448 	/* verify input for valid configuration */
449 	if ((hw->aq.num_arq_entries == 0) ||
450 	    (hw->aq.arq_buf_size == 0)) {
451 		ret_code = I40E_ERR_CONFIG;
452 		goto init_adminq_exit;
453 	}
454 
455 	hw->aq.arq.next_to_use = 0;
456 	hw->aq.arq.next_to_clean = 0;
457 
458 	/* allocate the ring memory */
459 	ret_code = i40e_alloc_adminq_arq_ring(hw);
460 	if (ret_code != I40E_SUCCESS)
461 		goto init_adminq_exit;
462 
463 	/* allocate buffers in the rings */
464 	ret_code = i40e_alloc_arq_bufs(hw);
465 	if (ret_code != I40E_SUCCESS)
466 		goto init_adminq_free_rings;
467 
468 	/* initialize base registers */
469 	ret_code = i40e_config_arq_regs(hw);
470 	if (ret_code != I40E_SUCCESS)
471 		goto init_adminq_free_rings;
472 
473 	/* success! */
474 	hw->aq.arq.count = hw->aq.num_arq_entries;
475 	goto init_adminq_exit;
476 
477 init_adminq_free_rings:
478 	i40e_free_adminq_arq(hw);
479 
480 init_adminq_exit:
481 	return ret_code;
482 }
483 
484 /**
485  *  i40e_shutdown_asq - shutdown the ASQ
486  *  @hw: pointer to the hardware structure
487  *
488  *  The main shutdown routine for the Admin Send Queue
489  **/
i40e_shutdown_asq(struct i40e_hw * hw)490 enum i40e_status_code i40e_shutdown_asq(struct i40e_hw *hw)
491 {
492 	enum i40e_status_code ret_code = I40E_SUCCESS;
493 
494 	i40e_acquire_spinlock(&hw->aq.asq_spinlock);
495 
496 	if (hw->aq.asq.count == 0) {
497 		ret_code = I40E_ERR_NOT_READY;
498 		goto shutdown_asq_out;
499 	}
500 
501 	/* Stop firmware AdminQ processing */
502 	wr32(hw, hw->aq.asq.head, 0);
503 	wr32(hw, hw->aq.asq.tail, 0);
504 	wr32(hw, hw->aq.asq.len, 0);
505 	wr32(hw, hw->aq.asq.bal, 0);
506 	wr32(hw, hw->aq.asq.bah, 0);
507 
508 	hw->aq.asq.count = 0; /* to indicate uninitialized queue */
509 
510 	/* free ring buffers */
511 	i40e_free_asq_bufs(hw);
512 
513 shutdown_asq_out:
514 	i40e_release_spinlock(&hw->aq.asq_spinlock);
515 	return ret_code;
516 }
517 
518 /**
519  *  i40e_shutdown_arq - shutdown ARQ
520  *  @hw: pointer to the hardware structure
521  *
522  *  The main shutdown routine for the Admin Receive Queue
523  **/
i40e_shutdown_arq(struct i40e_hw * hw)524 enum i40e_status_code i40e_shutdown_arq(struct i40e_hw *hw)
525 {
526 	enum i40e_status_code ret_code = I40E_SUCCESS;
527 
528 	i40e_acquire_spinlock(&hw->aq.arq_spinlock);
529 
530 	if (hw->aq.arq.count == 0) {
531 		ret_code = I40E_ERR_NOT_READY;
532 		goto shutdown_arq_out;
533 	}
534 
535 	/* Stop firmware AdminQ processing */
536 	wr32(hw, hw->aq.arq.head, 0);
537 	wr32(hw, hw->aq.arq.tail, 0);
538 	wr32(hw, hw->aq.arq.len, 0);
539 	wr32(hw, hw->aq.arq.bal, 0);
540 	wr32(hw, hw->aq.arq.bah, 0);
541 
542 	hw->aq.arq.count = 0; /* to indicate uninitialized queue */
543 
544 	/* free ring buffers */
545 	i40e_free_arq_bufs(hw);
546 
547 shutdown_arq_out:
548 	i40e_release_spinlock(&hw->aq.arq_spinlock);
549 	return ret_code;
550 }
551 
552 /**
553  *  i40e_resume_aq - resume AQ processing from 0
554  *  @hw: pointer to the hardware structure
555  **/
i40e_resume_aq(struct i40e_hw * hw)556 static void i40e_resume_aq(struct i40e_hw *hw)
557 {
558 	/* Registers are reset after PF reset */
559 	hw->aq.asq.next_to_use = 0;
560 	hw->aq.asq.next_to_clean = 0;
561 
562 	i40e_config_asq_regs(hw);
563 
564 	hw->aq.arq.next_to_use = 0;
565 	hw->aq.arq.next_to_clean = 0;
566 
567 	i40e_config_arq_regs(hw);
568 }
569 
570 /**
571  *  i40e_set_hw_flags - set HW flags
572  *  @hw: pointer to the hardware structure
573  **/
i40e_set_hw_flags(struct i40e_hw * hw)574 static void i40e_set_hw_flags(struct i40e_hw *hw)
575 {
576 	struct i40e_adminq_info *aq = &hw->aq;
577 
578 	hw->flags = 0;
579 
580 	switch (hw->mac.type) {
581 	case I40E_MAC_XL710:
582 		if (aq->api_maj_ver > 1 ||
583 		    (aq->api_maj_ver == 1 &&
584 		     aq->api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_XL710)) {
585 			hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE;
586 			hw->flags |= I40E_HW_FLAG_FW_LLDP_STOPPABLE;
587 			/* The ability to RX (not drop) 802.1ad frames */
588 			hw->flags |= I40E_HW_FLAG_802_1AD_CAPABLE;
589 		}
590 		break;
591 	case I40E_MAC_X722:
592 		hw->flags |= I40E_HW_FLAG_AQ_SRCTL_ACCESS_ENABLE |
593 			     I40E_HW_FLAG_NVM_READ_REQUIRES_LOCK;
594 
595 		if (aq->api_maj_ver > 1 ||
596 		    (aq->api_maj_ver == 1 &&
597 		     aq->api_min_ver >= I40E_MINOR_VER_FW_LLDP_STOPPABLE_X722))
598 			hw->flags |= I40E_HW_FLAG_FW_LLDP_STOPPABLE;
599 
600 		if (aq->api_maj_ver > 1 ||
601 		    (aq->api_maj_ver == 1 &&
602 		     aq->api_min_ver >= I40E_MINOR_VER_GET_LINK_INFO_X722))
603 			hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_CAPABLE;
604 
605 		if (aq->api_maj_ver > 1 ||
606 		    (aq->api_maj_ver == 1 &&
607 		     aq->api_min_ver >= I40E_MINOR_VER_FW_REQUEST_FEC_X722))
608 			hw->flags |= I40E_HW_FLAG_X722_FEC_REQUEST_CAPABLE;
609 
610 		/* fall through */
611 	default:
612 		break;
613 	}
614 
615 	/* Newer versions of firmware require lock when reading the NVM */
616 	if (aq->api_maj_ver > 1 ||
617 	    (aq->api_maj_ver == 1 &&
618 	     aq->api_min_ver >= 5))
619 		hw->flags |= I40E_HW_FLAG_NVM_READ_REQUIRES_LOCK;
620 
621 	if (aq->api_maj_ver > 1 ||
622 	    (aq->api_maj_ver == 1 &&
623 	     aq->api_min_ver >= 8)) {
624 		hw->flags |= I40E_HW_FLAG_FW_LLDP_PERSISTENT;
625 		hw->flags |= I40E_HW_FLAG_DROP_MODE;
626 	}
627 
628 	if (aq->api_maj_ver > 1 ||
629 	    (aq->api_maj_ver == 1 &&
630 	     aq->api_min_ver >= 9))
631 		hw->flags |= I40E_HW_FLAG_AQ_PHY_ACCESS_EXTENDED;
632 }
633 
634 /**
635  *  i40e_init_adminq - main initialization routine for Admin Queue
636  *  @hw: pointer to the hardware structure
637  *
638  *  Prior to calling this function, drivers *MUST* set the following fields
639  *  in the hw->aq structure:
640  *     - hw->aq.num_asq_entries
641  *     - hw->aq.num_arq_entries
642  *     - hw->aq.arq_buf_size
643  *     - hw->aq.asq_buf_size
644  **/
i40e_init_adminq(struct i40e_hw * hw)645 enum i40e_status_code i40e_init_adminq(struct i40e_hw *hw)
646 {
647 	struct i40e_adminq_info *aq = &hw->aq;
648 	enum i40e_status_code ret_code;
649 	u16 cfg_ptr, oem_hi, oem_lo;
650 	u16 eetrack_lo, eetrack_hi;
651 	int retry = 0;
652 
653 	/* verify input for valid configuration */
654 	if (aq->num_arq_entries == 0 ||
655 	    aq->num_asq_entries == 0 ||
656 	    aq->arq_buf_size == 0 ||
657 	    aq->asq_buf_size == 0) {
658 		ret_code = I40E_ERR_CONFIG;
659 		goto init_adminq_exit;
660 	}
661 	i40e_init_spinlock(&aq->asq_spinlock);
662 	i40e_init_spinlock(&aq->arq_spinlock);
663 
664 	/* Set up register offsets */
665 	i40e_adminq_init_regs(hw);
666 
667 	/* setup ASQ command write back timeout */
668 	hw->aq.asq_cmd_timeout = I40E_ASQ_CMD_TIMEOUT;
669 
670 	/* allocate the ASQ */
671 	ret_code = i40e_init_asq(hw);
672 	if (ret_code != I40E_SUCCESS)
673 		goto init_adminq_destroy_spinlocks;
674 
675 	/* allocate the ARQ */
676 	ret_code = i40e_init_arq(hw);
677 	if (ret_code != I40E_SUCCESS)
678 		goto init_adminq_free_asq;
679 
680 	/* VF has no need of firmware */
681 	if (i40e_is_vf(hw))
682 		goto init_adminq_exit;
683 	/* There are some cases where the firmware may not be quite ready
684 	 * for AdminQ operations, so we retry the AdminQ setup a few times
685 	 * if we see timeouts in this first AQ call.
686 	 */
687 	do {
688 		ret_code = i40e_aq_get_firmware_version(hw,
689 							&aq->fw_maj_ver,
690 							&aq->fw_min_ver,
691 							&aq->fw_build,
692 							&aq->api_maj_ver,
693 							&aq->api_min_ver,
694 							NULL);
695 		if (ret_code != I40E_ERR_ADMIN_QUEUE_TIMEOUT)
696 			break;
697 		retry++;
698 		i40e_msec_delay(100);
699 		i40e_resume_aq(hw);
700 	} while (retry < 10);
701 	if (ret_code != I40E_SUCCESS)
702 		goto init_adminq_free_arq;
703 
704 	/*
705 	 * Some features were introduced in different FW API version
706 	 * for different MAC type.
707 	 */
708 	i40e_set_hw_flags(hw);
709 
710 	/* get the NVM version info */
711 	i40e_read_nvm_word(hw, I40E_SR_NVM_DEV_STARTER_VERSION,
712 			   &hw->nvm.version);
713 	i40e_read_nvm_word(hw, I40E_SR_NVM_EETRACK_LO, &eetrack_lo);
714 	i40e_read_nvm_word(hw, I40E_SR_NVM_EETRACK_HI, &eetrack_hi);
715 	hw->nvm.eetrack = (eetrack_hi << 16) | eetrack_lo;
716 	i40e_read_nvm_word(hw, I40E_SR_BOOT_CONFIG_PTR, &cfg_ptr);
717 	i40e_read_nvm_word(hw, (cfg_ptr + I40E_NVM_OEM_VER_OFF),
718 			   &oem_hi);
719 	i40e_read_nvm_word(hw, (cfg_ptr + (I40E_NVM_OEM_VER_OFF + 1)),
720 			   &oem_lo);
721 	hw->nvm.oem_ver = ((u32)oem_hi << 16) | oem_lo;
722 
723 	if (aq->api_maj_ver > I40E_FW_API_VERSION_MAJOR) {
724 		ret_code = I40E_ERR_FIRMWARE_API_VERSION;
725 		goto init_adminq_free_arq;
726 	}
727 
728 	/* pre-emptive resource lock release */
729 	i40e_aq_release_resource(hw, I40E_NVM_RESOURCE_ID, 0, NULL);
730 	hw->nvm_release_on_done = FALSE;
731 	hw->nvmupd_state = I40E_NVMUPD_STATE_INIT;
732 
733 	ret_code = I40E_SUCCESS;
734 
735 	/* success! */
736 	goto init_adminq_exit;
737 
738 init_adminq_free_arq:
739 	i40e_shutdown_arq(hw);
740 init_adminq_free_asq:
741 	i40e_shutdown_asq(hw);
742 init_adminq_destroy_spinlocks:
743 	i40e_destroy_spinlock(&aq->asq_spinlock);
744 	i40e_destroy_spinlock(&aq->arq_spinlock);
745 
746 init_adminq_exit:
747 	return ret_code;
748 }
749 
750 /**
751  *  i40e_shutdown_adminq - shutdown routine for the Admin Queue
752  *  @hw: pointer to the hardware structure
753  **/
i40e_shutdown_adminq(struct i40e_hw * hw)754 enum i40e_status_code i40e_shutdown_adminq(struct i40e_hw *hw)
755 {
756 	enum i40e_status_code ret_code = I40E_SUCCESS;
757 
758 	if (i40e_check_asq_alive(hw))
759 		i40e_aq_queue_shutdown(hw, TRUE);
760 
761 	i40e_shutdown_asq(hw);
762 	i40e_shutdown_arq(hw);
763 	i40e_destroy_spinlock(&hw->aq.asq_spinlock);
764 	i40e_destroy_spinlock(&hw->aq.arq_spinlock);
765 
766 	if (hw->nvm_buff.va)
767 		i40e_free_virt_mem(hw, &hw->nvm_buff);
768 
769 	return ret_code;
770 }
771 
772 /**
773  *  i40e_clean_asq - cleans Admin send queue
774  *  @hw: pointer to the hardware structure
775  *
776  *  returns the number of free desc
777  **/
i40e_clean_asq(struct i40e_hw * hw)778 u16 i40e_clean_asq(struct i40e_hw *hw)
779 {
780 	struct i40e_adminq_ring *asq = &(hw->aq.asq);
781 	struct i40e_asq_cmd_details *details;
782 	u16 ntc = asq->next_to_clean;
783 	struct i40e_aq_desc desc_cb;
784 	struct i40e_aq_desc *desc;
785 
786 	desc = I40E_ADMINQ_DESC(*asq, ntc);
787 	details = I40E_ADMINQ_DETAILS(*asq, ntc);
788 	while (rd32(hw, hw->aq.asq.head) != ntc) {
789 		i40e_debug(hw, I40E_DEBUG_AQ_COMMAND,
790 			   "ntc %d head %d.\n", ntc, rd32(hw, hw->aq.asq.head));
791 
792 		if (details->callback) {
793 			I40E_ADMINQ_CALLBACK cb_func =
794 					(I40E_ADMINQ_CALLBACK)details->callback;
795 			i40e_memcpy(&desc_cb, desc, sizeof(struct i40e_aq_desc),
796 				    I40E_DMA_TO_DMA);
797 			cb_func(hw, &desc_cb);
798 		}
799 		i40e_memset(desc, 0, sizeof(*desc), I40E_DMA_MEM);
800 		i40e_memset(details, 0, sizeof(*details), I40E_NONDMA_MEM);
801 		ntc++;
802 		if (ntc == asq->count)
803 			ntc = 0;
804 		desc = I40E_ADMINQ_DESC(*asq, ntc);
805 		details = I40E_ADMINQ_DETAILS(*asq, ntc);
806 	}
807 
808 	asq->next_to_clean = ntc;
809 
810 	return I40E_DESC_UNUSED(asq);
811 }
812 
813 /**
814  *  i40e_asq_done - check if FW has processed the Admin Send Queue
815  *  @hw: pointer to the hw struct
816  *
817  *  Returns TRUE if the firmware has processed all descriptors on the
818  *  admin send queue. Returns FALSE if there are still requests pending.
819  **/
i40e_asq_done(struct i40e_hw * hw)820 bool i40e_asq_done(struct i40e_hw *hw)
821 {
822 	/* AQ designers suggest use of head for better
823 	 * timing reliability than DD bit
824 	 */
825 	return rd32(hw, hw->aq.asq.head) == hw->aq.asq.next_to_use;
826 
827 }
828 
829 /**
830  *  i40e_asq_send_command - send command to Admin Queue
831  *  @hw: pointer to the hw struct
832  *  @desc: prefilled descriptor describing the command (non DMA mem)
833  *  @buff: buffer to use for indirect commands
834  *  @buff_size: size of buffer for indirect commands
835  *  @cmd_details: pointer to command details structure
836  *
837  *  This is the main send command driver routine for the Admin Queue send
838  *  queue.  It runs the queue, cleans the queue, etc
839  **/
i40e_asq_send_command(struct i40e_hw * hw,struct i40e_aq_desc * desc,void * buff,u16 buff_size,struct i40e_asq_cmd_details * cmd_details)840 enum i40e_status_code i40e_asq_send_command(struct i40e_hw *hw,
841 				struct i40e_aq_desc *desc,
842 				void *buff, /* can be NULL */
843 				u16  buff_size,
844 				struct i40e_asq_cmd_details *cmd_details)
845 {
846 	enum i40e_status_code status = I40E_SUCCESS;
847 	struct i40e_dma_mem *dma_buff = NULL;
848 	struct i40e_asq_cmd_details *details;
849 	struct i40e_aq_desc *desc_on_ring;
850 	bool cmd_completed = FALSE;
851 	u16  retval = 0;
852 	u32  val = 0;
853 
854 	i40e_acquire_spinlock(&hw->aq.asq_spinlock);
855 
856 	hw->aq.asq_last_status = I40E_AQ_RC_OK;
857 
858 	if (hw->aq.asq.count == 0) {
859 		i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
860 			   "AQTX: Admin queue not initialized.\n");
861 		status = I40E_ERR_QUEUE_EMPTY;
862 		goto asq_send_command_error;
863 	}
864 
865 	val = rd32(hw, hw->aq.asq.head);
866 	if (val >= hw->aq.num_asq_entries) {
867 		i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
868 			   "AQTX: head overrun at %d\n", val);
869 		status = I40E_ERR_ADMIN_QUEUE_FULL;
870 		goto asq_send_command_error;
871 	}
872 
873 	details = I40E_ADMINQ_DETAILS(hw->aq.asq, hw->aq.asq.next_to_use);
874 	if (cmd_details) {
875 		i40e_memcpy(details,
876 			    cmd_details,
877 			    sizeof(struct i40e_asq_cmd_details),
878 			    I40E_NONDMA_TO_NONDMA);
879 
880 		/* If the cmd_details are defined copy the cookie.  The
881 		 * CPU_TO_LE32 is not needed here because the data is ignored
882 		 * by the FW, only used by the driver
883 		 */
884 		if (details->cookie) {
885 			desc->cookie_high =
886 				CPU_TO_LE32(I40E_HI_DWORD(details->cookie));
887 			desc->cookie_low =
888 				CPU_TO_LE32(I40E_LO_DWORD(details->cookie));
889 		}
890 	} else {
891 		i40e_memset(details, 0,
892 			    sizeof(struct i40e_asq_cmd_details),
893 			    I40E_NONDMA_MEM);
894 	}
895 
896 	/* clear requested flags and then set additional flags if defined */
897 	desc->flags &= ~CPU_TO_LE16(details->flags_dis);
898 	desc->flags |= CPU_TO_LE16(details->flags_ena);
899 
900 	if (buff_size > hw->aq.asq_buf_size) {
901 		i40e_debug(hw,
902 			   I40E_DEBUG_AQ_MESSAGE,
903 			   "AQTX: Invalid buffer size: %d.\n",
904 			   buff_size);
905 		status = I40E_ERR_INVALID_SIZE;
906 		goto asq_send_command_error;
907 	}
908 
909 	if (details->postpone && !details->async) {
910 		i40e_debug(hw,
911 			   I40E_DEBUG_AQ_MESSAGE,
912 			   "AQTX: Async flag not set along with postpone flag");
913 		status = I40E_ERR_PARAM;
914 		goto asq_send_command_error;
915 	}
916 
917 	/* call clean and check queue available function to reclaim the
918 	 * descriptors that were processed by FW, the function returns the
919 	 * number of desc available
920 	 */
921 	/* the clean function called here could be called in a separate thread
922 	 * in case of asynchronous completions
923 	 */
924 	if (i40e_clean_asq(hw) == 0) {
925 		i40e_debug(hw,
926 			   I40E_DEBUG_AQ_MESSAGE,
927 			   "AQTX: Error queue is full.\n");
928 		status = I40E_ERR_ADMIN_QUEUE_FULL;
929 		goto asq_send_command_error;
930 	}
931 
932 	/* initialize the temp desc pointer with the right desc */
933 	desc_on_ring = I40E_ADMINQ_DESC(hw->aq.asq, hw->aq.asq.next_to_use);
934 
935 	/* if the desc is available copy the temp desc to the right place */
936 	i40e_memcpy(desc_on_ring, desc, sizeof(struct i40e_aq_desc),
937 		    I40E_NONDMA_TO_DMA);
938 
939 	/* if buff is not NULL assume indirect command */
940 	if (buff != NULL) {
941 		dma_buff = &(hw->aq.asq.r.asq_bi[hw->aq.asq.next_to_use]);
942 		/* copy the user buff into the respective DMA buff */
943 		i40e_memcpy(dma_buff->va, buff, buff_size,
944 			    I40E_NONDMA_TO_DMA);
945 		desc_on_ring->datalen = CPU_TO_LE16(buff_size);
946 
947 		/* Update the address values in the desc with the pa value
948 		 * for respective buffer
949 		 */
950 		desc_on_ring->params.external.addr_high =
951 				CPU_TO_LE32(I40E_HI_DWORD(dma_buff->pa));
952 		desc_on_ring->params.external.addr_low =
953 				CPU_TO_LE32(I40E_LO_DWORD(dma_buff->pa));
954 	}
955 
956 	/* bump the tail */
957 	i40e_debug(hw, I40E_DEBUG_AQ_COMMAND, "AQTX: desc and buffer:\n");
958 	i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc_on_ring,
959 		      buff, buff_size);
960 	(hw->aq.asq.next_to_use)++;
961 	if (hw->aq.asq.next_to_use == hw->aq.asq.count)
962 		hw->aq.asq.next_to_use = 0;
963 	if (!details->postpone)
964 		wr32(hw, hw->aq.asq.tail, hw->aq.asq.next_to_use);
965 
966 	/* if cmd_details are not defined or async flag is not set,
967 	 * we need to wait for desc write back
968 	 */
969 	if (!details->async && !details->postpone) {
970 		u32 total_delay = 0;
971 
972 		do {
973 			/* AQ designers suggest use of head for better
974 			 * timing reliability than DD bit
975 			 */
976 			if (i40e_asq_done(hw))
977 				break;
978 			i40e_usec_delay(50);
979 			total_delay += 50;
980 		} while (total_delay < hw->aq.asq_cmd_timeout);
981 	}
982 
983 	/* if ready, copy the desc back to temp */
984 	if (i40e_asq_done(hw)) {
985 		i40e_memcpy(desc, desc_on_ring, sizeof(struct i40e_aq_desc),
986 			    I40E_DMA_TO_NONDMA);
987 		if (buff != NULL)
988 			i40e_memcpy(buff, dma_buff->va, buff_size,
989 				    I40E_DMA_TO_NONDMA);
990 		retval = LE16_TO_CPU(desc->retval);
991 		if (retval != 0) {
992 			i40e_debug(hw,
993 				   I40E_DEBUG_AQ_MESSAGE,
994 				   "AQTX: Command completed with error 0x%X.\n",
995 				   retval);
996 
997 			/* strip off FW internal code */
998 			retval &= 0xff;
999 		}
1000 		cmd_completed = TRUE;
1001 		if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_OK)
1002 			status = I40E_SUCCESS;
1003 		else if ((enum i40e_admin_queue_err)retval == I40E_AQ_RC_EBUSY)
1004 			status = I40E_ERR_NOT_READY;
1005 		else
1006 			status = I40E_ERR_ADMIN_QUEUE_ERROR;
1007 		hw->aq.asq_last_status = (enum i40e_admin_queue_err)retval;
1008 	}
1009 
1010 	i40e_debug(hw, I40E_DEBUG_AQ_COMMAND,
1011 		   "AQTX: desc and buffer writeback:\n");
1012 	i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, buff, buff_size);
1013 
1014 	/* save writeback aq if requested */
1015 	if (details->wb_desc)
1016 		i40e_memcpy(details->wb_desc, desc_on_ring,
1017 			    sizeof(struct i40e_aq_desc), I40E_DMA_TO_NONDMA);
1018 
1019 	/* update the error if time out occurred */
1020 	if ((!cmd_completed) &&
1021 	    (!details->async && !details->postpone)) {
1022 		if (rd32(hw, hw->aq.asq.len) & I40E_GL_ATQLEN_ATQCRIT_MASK) {
1023 			i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
1024 				   "AQTX: AQ Critical error.\n");
1025 			status = I40E_ERR_ADMIN_QUEUE_CRITICAL_ERROR;
1026 		} else {
1027 			i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
1028 				   "AQTX: Writeback timeout.\n");
1029 			status = I40E_ERR_ADMIN_QUEUE_TIMEOUT;
1030 		}
1031 	}
1032 
1033 asq_send_command_error:
1034 	i40e_release_spinlock(&hw->aq.asq_spinlock);
1035 	return status;
1036 }
1037 
1038 /**
1039  *  i40e_fill_default_direct_cmd_desc - AQ descriptor helper function
1040  *  @desc:     pointer to the temp descriptor (non DMA mem)
1041  *  @opcode:   the opcode can be used to decide which flags to turn off or on
1042  *
1043  *  Fill the desc with default values
1044  **/
i40e_fill_default_direct_cmd_desc(struct i40e_aq_desc * desc,u16 opcode)1045 void i40e_fill_default_direct_cmd_desc(struct i40e_aq_desc *desc,
1046 				       u16 opcode)
1047 {
1048 	/* zero out the desc */
1049 	i40e_memset((void *)desc, 0, sizeof(struct i40e_aq_desc),
1050 		    I40E_NONDMA_MEM);
1051 	desc->opcode = CPU_TO_LE16(opcode);
1052 	desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_SI);
1053 }
1054 
1055 /**
1056  *  i40e_clean_arq_element
1057  *  @hw: pointer to the hw struct
1058  *  @e: event info from the receive descriptor, includes any buffers
1059  *  @pending: number of events that could be left to process
1060  *
1061  *  This function cleans one Admin Receive Queue element and returns
1062  *  the contents through e.  It can also return how many events are
1063  *  left to process through 'pending'
1064  **/
i40e_clean_arq_element(struct i40e_hw * hw,struct i40e_arq_event_info * e,u16 * pending)1065 enum i40e_status_code i40e_clean_arq_element(struct i40e_hw *hw,
1066 					     struct i40e_arq_event_info *e,
1067 					     u16 *pending)
1068 {
1069 	enum i40e_status_code ret_code = I40E_SUCCESS;
1070 	u16 ntc = hw->aq.arq.next_to_clean;
1071 	struct i40e_aq_desc *desc;
1072 	struct i40e_dma_mem *bi;
1073 	u16 desc_idx;
1074 	u16 datalen;
1075 	u16 flags;
1076 	u16 ntu;
1077 
1078 	/* pre-clean the event info */
1079 	i40e_memset(&e->desc, 0, sizeof(e->desc), I40E_NONDMA_MEM);
1080 
1081 	/* take the lock before we start messing with the ring */
1082 	i40e_acquire_spinlock(&hw->aq.arq_spinlock);
1083 
1084 	if (hw->aq.arq.count == 0) {
1085 		i40e_debug(hw, I40E_DEBUG_AQ_MESSAGE,
1086 			   "AQRX: Admin queue not initialized.\n");
1087 		ret_code = I40E_ERR_QUEUE_EMPTY;
1088 		goto clean_arq_element_err;
1089 	}
1090 
1091 	/* set next_to_use to head */
1092 	if (!i40e_is_vf(hw))
1093 		ntu = rd32(hw, hw->aq.arq.head) & I40E_PF_ARQH_ARQH_MASK;
1094 	else
1095 		ntu = rd32(hw, hw->aq.arq.head) & I40E_VF_ARQH1_ARQH_MASK;
1096 	if (ntu == ntc) {
1097 		/* nothing to do - shouldn't need to update ring's values */
1098 		ret_code = I40E_ERR_ADMIN_QUEUE_NO_WORK;
1099 		goto clean_arq_element_out;
1100 	}
1101 
1102 	/* now clean the next descriptor */
1103 	desc = I40E_ADMINQ_DESC(hw->aq.arq, ntc);
1104 	desc_idx = ntc;
1105 
1106 	hw->aq.arq_last_status =
1107 		(enum i40e_admin_queue_err)LE16_TO_CPU(desc->retval);
1108 	flags = LE16_TO_CPU(desc->flags);
1109 	if (flags & I40E_AQ_FLAG_ERR) {
1110 		ret_code = I40E_ERR_ADMIN_QUEUE_ERROR;
1111 		i40e_debug(hw,
1112 			   I40E_DEBUG_AQ_MESSAGE,
1113 			   "AQRX: Event received with error 0x%X.\n",
1114 			   hw->aq.arq_last_status);
1115 	}
1116 
1117 	i40e_memcpy(&e->desc, desc, sizeof(struct i40e_aq_desc),
1118 		    I40E_DMA_TO_NONDMA);
1119 	datalen = LE16_TO_CPU(desc->datalen);
1120 	e->msg_len = min(datalen, e->buf_len);
1121 	if (e->msg_buf != NULL && (e->msg_len != 0))
1122 		i40e_memcpy(e->msg_buf,
1123 			    hw->aq.arq.r.arq_bi[desc_idx].va,
1124 			    e->msg_len, I40E_DMA_TO_NONDMA);
1125 
1126 	i40e_debug(hw, I40E_DEBUG_AQ_COMMAND, "AQRX: desc and buffer:\n");
1127 	i40e_debug_aq(hw, I40E_DEBUG_AQ_COMMAND, (void *)desc, e->msg_buf,
1128 		      hw->aq.arq_buf_size);
1129 
1130 	/* Restore the original datalen and buffer address in the desc,
1131 	 * FW updates datalen to indicate the event message
1132 	 * size
1133 	 */
1134 	bi = &hw->aq.arq.r.arq_bi[ntc];
1135 	i40e_memset((void *)desc, 0, sizeof(struct i40e_aq_desc), I40E_DMA_MEM);
1136 
1137 	desc->flags = CPU_TO_LE16(I40E_AQ_FLAG_BUF);
1138 	if (hw->aq.arq_buf_size > I40E_AQ_LARGE_BUF)
1139 		desc->flags |= CPU_TO_LE16(I40E_AQ_FLAG_LB);
1140 	desc->datalen = CPU_TO_LE16((u16)bi->size);
1141 	desc->params.external.addr_high = CPU_TO_LE32(I40E_HI_DWORD(bi->pa));
1142 	desc->params.external.addr_low = CPU_TO_LE32(I40E_LO_DWORD(bi->pa));
1143 
1144 	/* set tail = the last cleaned desc index. */
1145 	wr32(hw, hw->aq.arq.tail, ntc);
1146 	/* ntc is updated to tail + 1 */
1147 	ntc++;
1148 	if (ntc == hw->aq.num_arq_entries)
1149 		ntc = 0;
1150 	hw->aq.arq.next_to_clean = ntc;
1151 	hw->aq.arq.next_to_use = ntu;
1152 
1153 	i40e_nvmupd_check_wait_event(hw, LE16_TO_CPU(e->desc.opcode), &e->desc);
1154 clean_arq_element_out:
1155 	/* Set pending if needed, unlock and return */
1156 	if (pending != NULL)
1157 		*pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc);
1158 clean_arq_element_err:
1159 	i40e_release_spinlock(&hw->aq.arq_spinlock);
1160 
1161 	return ret_code;
1162 }
1163 
1164