Lines Matching refs:channel
88 static int ioat_completion_alloc(ioat_channel_t channel);
89 static void ioat_completion_free(ioat_channel_t channel);
90 static void ioat_channel_start(ioat_channel_t channel);
91 static void ioat_channel_reset(ioat_channel_t channel);
93 int ioat_ring_alloc(ioat_channel_t channel, uint_t desc_cnt);
94 void ioat_ring_free(ioat_channel_t channel);
95 void ioat_ring_seed(ioat_channel_t channel, ioat_chan_dma_desc_t *desc);
96 int ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring,
155 struct ioat_channel_s *channel; in ioat_channel_alloc() local
174 channel = &state->is_channel[chan_num]; in ioat_channel_alloc()
175 channel->ic_inuse = B_TRUE; in ioat_channel_alloc()
176 channel->ic_chan_num = chan_num; in ioat_channel_alloc()
177 channel->ic_ver = state->is_ver; in ioat_channel_alloc()
178 channel->ic_dca_active = B_FALSE; in ioat_channel_alloc()
179 channel->ic_channel_state = IOAT_CHANNEL_OK; in ioat_channel_alloc()
180 channel->ic_dcopy_handle = handle; in ioat_channel_alloc()
186 if (channel->ic_ver == IOAT_CBv2) { in ioat_channel_alloc()
188 (uint16_t *)&channel->ic_regs[IOAT_CHAN_COMP]); in ioat_channel_alloc()
202 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x011C); in ioat_channel_alloc()
206 (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]); in ioat_channel_alloc()
210 "enable\n", estat, channel->ic_chan_num); in ioat_channel_alloc()
213 (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR], estat); in ioat_channel_alloc()
217 e = ioat_ring_alloc(channel, size); in ioat_channel_alloc()
223 e = ioat_completion_alloc(channel); in ioat_channel_alloc()
233 state->is_instance, channel->ic_chan_num); in ioat_channel_alloc()
234 channel->ic_cmd_cache = kmem_cache_create(chanstr, cmd_size, 64, in ioat_channel_alloc()
236 if (channel->ic_cmd_cache == NULL) { in ioat_channel_alloc()
241 ioat_channel_start(channel); in ioat_channel_alloc()
248 info->qc_chan_num = (uint64_t)channel->ic_chan_num; in ioat_channel_alloc()
249 if (channel->ic_ver == IOAT_CBv1) { in ioat_channel_alloc()
259 *chan = channel; in ioat_channel_alloc()
264 ioat_completion_free(channel); in ioat_channel_alloc()
266 ioat_ring_free(channel); in ioat_channel_alloc()
296 ioat_channel_t channel; in ioat_channel_resume() local
302 channel = &state->is_channel[i]; in ioat_channel_resume()
303 ring = channel->ic_ring; in ioat_channel_resume()
305 if (!channel->ic_inuse) { in ioat_channel_resume()
317 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x011C); in ioat_channel_resume()
321 (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]); in ioat_channel_resume()
325 " (%d) enable\n", estat, channel->ic_chan_num); in ioat_channel_resume()
328 (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR], in ioat_channel_resume()
333 bzero(ring->cr_desc, channel->ic_desc_alloc_size); in ioat_channel_resume()
335 if (channel->ic_ver == IOAT_CBv1) { in ioat_channel_resume()
337 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO], in ioat_channel_resume()
340 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI], in ioat_channel_resume()
343 ASSERT(channel->ic_ver == IOAT_CBv2); in ioat_channel_resume()
345 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO], in ioat_channel_resume()
348 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI], in ioat_channel_resume()
353 bzero((void *)channel->ic_cmpl, channel->ic_cmpl_alloc_size); in ioat_channel_resume()
356 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO], in ioat_channel_resume()
357 (uint32_t)(channel->ic_phys_cmpl & 0xffffffff)); in ioat_channel_resume()
359 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI], in ioat_channel_resume()
360 (uint32_t)(channel->ic_phys_cmpl >> 32)); in ioat_channel_resume()
363 ioat_channel_start(channel); in ioat_channel_resume()
390 ioat_channel_t channel = state->is_channel + i; in ioat_channel_quiesce() local
392 if (!channel->ic_inuse) in ioat_channel_quiesce()
397 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], in ioat_channel_quiesce()
400 ioat_channel_reset(channel); in ioat_channel_quiesce()
411 struct ioat_channel_s *channel; in ioat_channel_free() local
418 channel = *chan; in ioat_channel_free()
420 state = channel->ic_state; in ioat_channel_free()
421 chan_num = channel->ic_chan_num; in ioat_channel_free()
425 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], 0x0); in ioat_channel_free()
427 ioat_channel_reset(channel); in ioat_channel_free()
430 kmem_cache_destroy(channel->ic_cmd_cache); in ioat_channel_free()
433 ioat_completion_free(channel); in ioat_channel_free()
434 ioat_ring_free(channel); in ioat_channel_free()
436 channel->ic_inuse = B_FALSE; in ioat_channel_free()
449 ioat_channel_intr(ioat_channel_t channel) in ioat_channel_intr() argument
457 state = channel->ic_state; in ioat_channel_intr()
459 if (channel->ic_ver == IOAT_CBv1) { in ioat_channel_intr()
461 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_STS_LO]); in ioat_channel_intr()
463 ASSERT(channel->ic_ver == IOAT_CBv2); in ioat_channel_intr()
465 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_STS_LO]); in ioat_channel_intr()
471 (uint32_t *)&channel->ic_regs[IOAT_CHAN_ERR]); in ioat_channel_intr()
474 channel->ic_chan_num, status, chanerr); in ioat_channel_intr()
475 channel->ic_channel_state = IOAT_CHANNEL_IN_FAILURE; in ioat_channel_intr()
476 ioat_channel_reset(channel); in ioat_channel_intr()
486 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL]); in ioat_channel_intr()
488 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL], chanctrl); in ioat_channel_intr()
490 (uint16_t *)&channel->ic_regs[IOAT_CHAN_CTL]); in ioat_channel_intr()
493 dcopy_device_channel_notify(channel->ic_dcopy_handle, DCOPY_COMPLETION); in ioat_channel_intr()
501 ioat_channel_start(ioat_channel_t channel) in ioat_channel_start() argument
513 ioat_ring_seed(channel, &desc); in ioat_channel_start()
521 ioat_channel_reset(ioat_channel_t channel) in ioat_channel_reset() argument
525 state = channel->ic_state; in ioat_channel_reset()
528 if (channel->ic_ver == IOAT_CBv1) { in ioat_channel_reset()
530 &channel->ic_regs[IOAT_V1_CHAN_CMD], 0x20); in ioat_channel_reset()
532 ASSERT(channel->ic_ver == IOAT_CBv2); in ioat_channel_reset()
534 &channel->ic_regs[IOAT_V2_CHAN_CMD], 0x20); in ioat_channel_reset()
543 ioat_completion_alloc(ioat_channel_t channel) in ioat_completion_alloc() argument
551 state = channel->ic_state; in ioat_completion_alloc()
558 DDI_DMA_SLEEP, NULL, &channel->ic_cmpl_dma_handle); in ioat_completion_alloc()
562 channel->ic_cmpl_alloc_size = 64; in ioat_completion_alloc()
563 e = ddi_dma_mem_alloc(channel->ic_cmpl_dma_handle, in ioat_completion_alloc()
564 channel->ic_cmpl_alloc_size, &ioat_acc_attr, in ioat_completion_alloc()
566 (caddr_t *)&channel->ic_cmpl, &real_length, in ioat_completion_alloc()
567 &channel->ic_cmpl_handle); in ioat_completion_alloc()
571 bzero((void *)channel->ic_cmpl, channel->ic_cmpl_alloc_size); in ioat_completion_alloc()
572 e = ddi_dma_addr_bind_handle(channel->ic_cmpl_dma_handle, NULL, in ioat_completion_alloc()
573 (caddr_t)channel->ic_cmpl, channel->ic_cmpl_alloc_size, in ioat_completion_alloc()
575 &channel->ic_cmpl_cookie, &cookie_cnt); in ioat_completion_alloc()
580 ASSERT(channel->ic_cmpl_cookie.dmac_size == in ioat_completion_alloc()
581 channel->ic_cmpl_alloc_size); in ioat_completion_alloc()
582 channel->ic_phys_cmpl = channel->ic_cmpl_cookie.dmac_laddress; in ioat_completion_alloc()
586 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO], in ioat_completion_alloc()
587 (uint32_t)(channel->ic_phys_cmpl & 0xffffffff)); in ioat_completion_alloc()
589 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI], in ioat_completion_alloc()
590 (uint32_t)(channel->ic_phys_cmpl >> 32)); in ioat_completion_alloc()
595 ddi_dma_mem_free(&channel->ic_desc_handle); in ioat_completion_alloc()
597 ddi_dma_free_handle(&channel->ic_desc_dma_handle); in ioat_completion_alloc()
607 ioat_completion_free(ioat_channel_t channel) in ioat_completion_free() argument
611 state = channel->ic_state; in ioat_completion_free()
615 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_LO], 0x0); in ioat_completion_free()
617 (uint32_t *)&channel->ic_regs[IOAT_CHAN_CMPL_HI], 0x0); in ioat_completion_free()
620 (void) ddi_dma_unbind_handle(channel->ic_cmpl_dma_handle); in ioat_completion_free()
621 ddi_dma_mem_free(&channel->ic_cmpl_handle); in ioat_completion_free()
622 ddi_dma_free_handle(&channel->ic_cmpl_dma_handle); in ioat_completion_free()
629 ioat_ring_alloc(ioat_channel_t channel, uint_t desc_cnt) in ioat_ring_alloc() argument
638 state = channel->ic_state; in ioat_ring_alloc()
641 channel->ic_ring = ring; in ioat_ring_alloc()
642 ring->cr_chan = channel; in ioat_ring_alloc()
646 channel->ic_state->is_iblock_cookie); in ioat_ring_alloc()
648 channel->ic_state->is_iblock_cookie); in ioat_ring_alloc()
656 DDI_DMA_SLEEP, NULL, &channel->ic_desc_dma_handle); in ioat_ring_alloc()
664 channel->ic_chan_desc_cnt = ((desc_cnt + 1) + 3) & ~0x3; in ioat_ring_alloc()
665 ring->cr_desc_last = channel->ic_chan_desc_cnt - 1; in ioat_ring_alloc()
666 channel->ic_desc_alloc_size = channel->ic_chan_desc_cnt * in ioat_ring_alloc()
668 e = ddi_dma_mem_alloc(channel->ic_desc_dma_handle, in ioat_ring_alloc()
669 channel->ic_desc_alloc_size, &ioat_acc_attr, in ioat_ring_alloc()
671 (caddr_t *)&ring->cr_desc, &real_length, &channel->ic_desc_handle); in ioat_ring_alloc()
675 bzero(ring->cr_desc, channel->ic_desc_alloc_size); in ioat_ring_alloc()
676 e = ddi_dma_addr_bind_handle(channel->ic_desc_dma_handle, NULL, in ioat_ring_alloc()
677 (caddr_t)ring->cr_desc, channel->ic_desc_alloc_size, in ioat_ring_alloc()
679 &channel->ic_desc_cookies, &cookie_cnt); in ioat_ring_alloc()
684 ASSERT(channel->ic_desc_cookies.dmac_size == in ioat_ring_alloc()
685 channel->ic_desc_alloc_size); in ioat_ring_alloc()
686 ring->cr_phys_desc = channel->ic_desc_cookies.dmac_laddress; in ioat_ring_alloc()
689 if (channel->ic_ver == IOAT_CBv1) { in ioat_ring_alloc()
691 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO], in ioat_ring_alloc()
694 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI], in ioat_ring_alloc()
697 ASSERT(channel->ic_ver == IOAT_CBv2); in ioat_ring_alloc()
699 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO], in ioat_ring_alloc()
702 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI], in ioat_ring_alloc()
709 ddi_dma_mem_free(&channel->ic_desc_handle); in ioat_ring_alloc()
711 ddi_dma_free_handle(&channel->ic_desc_dma_handle); in ioat_ring_alloc()
715 kmem_free(channel->ic_ring, sizeof (ioat_channel_ring_t)); in ioat_ring_alloc()
725 ioat_ring_free(ioat_channel_t channel) in ioat_ring_free() argument
730 state = channel->ic_state; in ioat_ring_free()
733 if (channel->ic_ver == IOAT_CBv1) { in ioat_ring_free()
735 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_LO], 0x0); in ioat_ring_free()
737 (uint32_t *)&channel->ic_regs[IOAT_V1_CHAN_ADDR_HI], 0x0); in ioat_ring_free()
739 ASSERT(channel->ic_ver == IOAT_CBv2); in ioat_ring_free()
741 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_LO], 0x0); in ioat_ring_free()
743 (uint32_t *)&channel->ic_regs[IOAT_V2_CHAN_ADDR_HI], 0x0); in ioat_ring_free()
747 (void) ddi_dma_unbind_handle(channel->ic_desc_dma_handle); in ioat_ring_free()
748 ddi_dma_mem_free(&channel->ic_desc_handle); in ioat_ring_free()
749 ddi_dma_free_handle(&channel->ic_desc_dma_handle); in ioat_ring_free()
751 mutex_destroy(&channel->ic_ring->cr_desc_mutex); in ioat_ring_free()
752 mutex_destroy(&channel->ic_ring->cr_cmpl_mutex); in ioat_ring_free()
753 kmem_free(channel->ic_ring, sizeof (ioat_channel_ring_t)); in ioat_ring_free()
763 ioat_ring_seed(ioat_channel_t channel, ioat_chan_dma_desc_t *in_desc) in ioat_ring_seed() argument
771 state = channel->ic_state; in ioat_ring_seed()
772 ring = channel->ic_ring; in ioat_ring_seed()
780 channel->ic_ring->cr_desc[0] = *(ioat_chan_desc_t *)in_desc; in ioat_ring_seed()
785 if (channel->ic_ver == IOAT_CBv1) { in ioat_ring_seed()
788 &channel->ic_regs[IOAT_V1_CHAN_CMD], 0x1); in ioat_ring_seed()
807 (uint16_t *)&channel->ic_regs[IOAT_V2_CHAN_CNT], in ioat_ring_seed()
822 ioat_channel_t channel; in ioat_ring_loop() local
828 channel = ring->cr_chan; in ioat_ring_loop()
829 ASSERT(channel->ic_ver == IOAT_CBv1); in ioat_ring_loop()
836 cmd != NULL && count <= channel->ic_chan_desc_cnt; in ioat_ring_loop()
842 channel->ic_chan_desc_cnt) { in ioat_ring_loop()
847 count += channel->ic_chan_desc_cnt in ioat_ring_loop()
858 if (count >= channel->ic_chan_desc_cnt) { in ioat_ring_loop()
867 (void) ddi_dma_sync(channel->ic_desc_dma_handle, in ioat_ring_loop()
881 ioat_channel_t channel; in ioat_cmd_alloc() local
886 channel = (ioat_channel_t)private; in ioat_cmd_alloc()
897 *cmd = kmem_cache_alloc(channel->ic_cmd_cache, kmflag); in ioat_cmd_alloc()
933 ioat_channel_t channel; in ioat_cmd_free() local
938 channel = (ioat_channel_t)private; in ioat_cmd_free()
948 kmem_cache_free(channel->ic_cmd_cache, cmd); in ioat_cmd_free()
963 ioat_channel_t channel; in ioat_cmd_post() local
977 channel = (ioat_channel_t)private; in ioat_cmd_post()
980 state = channel->ic_state; in ioat_cmd_post()
981 ring = channel->ic_ring; in ioat_cmd_post()
988 (channel->ic_ver != IOAT_CBv1 || in ioat_cmd_post()
1005 if (channel->ic_channel_state == IOAT_CHANNEL_IN_FAILURE) { in ioat_cmd_post()
1011 e = ioat_ring_reserve(channel, ring, cmd); in ioat_cmd_post()
1018 if ((channel->ic_ver == IOAT_CBv2) && in ioat_cmd_post()
1100 if (channel->ic_ver == IOAT_CBv1) { in ioat_cmd_post()
1102 (uint8_t *)&channel->ic_regs[IOAT_V1_CHAN_CMD], in ioat_cmd_post()
1105 ASSERT(channel->ic_ver == IOAT_CBv2); in ioat_cmd_post()
1107 (uint16_t *)&channel->ic_regs[IOAT_V2_CHAN_CNT], in ioat_cmd_post()
1127 ioat_channel_t channel; in ioat_cmd_post_dca() local
1133 channel = ring->cr_chan; in ioat_cmd_post_dca()
1148 if (!channel->ic_dca_active || (channel->ic_dca_current != dca_id)) { in ioat_cmd_post_dca()
1149 channel->ic_dca_active = B_TRUE; in ioat_cmd_post_dca()
1150 channel->ic_dca_current = dca_id; in ioat_cmd_post_dca()
1186 (void) ddi_dma_sync(channel->ic_desc_dma_handle, in ioat_cmd_post_dca()
1197 (void) ddi_dma_sync(channel->ic_desc_dma_handle, next_offset, 64, in ioat_cmd_post_dca()
1202 (void) ddi_dma_sync(channel->ic_desc_dma_handle, prev_offset, 64, in ioat_cmd_post_dca()
1218 ioat_channel_t channel; in ioat_cmd_post_copy() local
1224 channel = ring->cr_chan; in ioat_cmd_post_copy()
1270 (void) ddi_dma_sync(channel->ic_desc_dma_handle, in ioat_cmd_post_copy()
1281 (void) ddi_dma_sync(channel->ic_desc_dma_handle, next_offset, 64, in ioat_cmd_post_copy()
1286 (void) ddi_dma_sync(channel->ic_desc_dma_handle, prev_offset, 64, in ioat_cmd_post_copy()
1299 ioat_channel_t channel; in ioat_cmd_poll() local
1304 channel = (ioat_channel_t)private; in ioat_cmd_poll()
1307 ring = channel->ic_ring; in ioat_cmd_poll()
1322 if ((channel->ic_channel_state == IOAT_CHANNEL_IN_FAILURE) || in ioat_cmd_poll()
1323 IOAT_CMPL_FAILED(channel)) { in ioat_cmd_poll()
1334 (void) ddi_dma_sync(channel->ic_cmpl_dma_handle, 0, 0, in ioat_cmd_poll()
1336 last_cmpl = IOAT_CMPL_INDEX(channel); in ioat_cmd_poll()
1384 ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring, in ioat_ring_reserve() argument
1403 if ((channel->ic_ver == IOAT_CBv2) && in ioat_ring_reserve()
1450 (void) ioat_cmd_poll(channel, cmd); in ioat_ring_reserve()
1474 (void) ioat_cmd_poll(channel, cmd); in ioat_ring_reserve()