Lines Matching refs:ring
96 int ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring,
99 static void ioat_cmd_post_copy(ioat_channel_ring_t *ring, uint64_t src_addr,
101 static void ioat_cmd_post_dca(ioat_channel_ring_t *ring, uint32_t dca_id);
295 ioat_channel_ring_t *ring; in ioat_channel_resume() local
303 ring = channel->ic_ring; in ioat_channel_resume()
333 bzero(ring->cr_desc, channel->ic_desc_alloc_size); in ioat_channel_resume()
338 (uint32_t)(ring->cr_phys_desc & 0xffffffff)); in ioat_channel_resume()
341 (uint32_t)(ring->cr_phys_desc >> 32)); in ioat_channel_resume()
346 (uint32_t)(ring->cr_phys_desc & 0xffffffff)); in ioat_channel_resume()
349 (uint32_t)(ring->cr_phys_desc >> 32)); in ioat_channel_resume()
631 ioat_channel_ring_t *ring; in ioat_ring_alloc() local
640 ring = kmem_zalloc(sizeof (ioat_channel_ring_t), KM_SLEEP); in ioat_ring_alloc()
641 channel->ic_ring = ring; in ioat_ring_alloc()
642 ring->cr_chan = channel; in ioat_ring_alloc()
643 ring->cr_post_cnt = 0; in ioat_ring_alloc()
645 mutex_init(&ring->cr_cmpl_mutex, NULL, MUTEX_DRIVER, in ioat_ring_alloc()
647 mutex_init(&ring->cr_desc_mutex, NULL, MUTEX_DRIVER, in ioat_ring_alloc()
665 ring->cr_desc_last = channel->ic_chan_desc_cnt - 1; in ioat_ring_alloc()
671 (caddr_t *)&ring->cr_desc, &real_length, &channel->ic_desc_handle); in ioat_ring_alloc()
675 bzero(ring->cr_desc, channel->ic_desc_alloc_size); in ioat_ring_alloc()
677 (caddr_t)ring->cr_desc, channel->ic_desc_alloc_size, in ioat_ring_alloc()
686 ring->cr_phys_desc = channel->ic_desc_cookies.dmac_laddress; in ioat_ring_alloc()
692 (uint32_t)(ring->cr_phys_desc & 0xffffffff)); in ioat_ring_alloc()
695 (uint32_t)(ring->cr_phys_desc >> 32)); in ioat_ring_alloc()
700 (uint32_t)(ring->cr_phys_desc & 0xffffffff)); in ioat_ring_alloc()
703 (uint32_t)(ring->cr_phys_desc >> 32)); in ioat_ring_alloc()
713 mutex_destroy(&ring->cr_desc_mutex); in ioat_ring_alloc()
714 mutex_destroy(&ring->cr_cmpl_mutex); in ioat_ring_alloc()
765 ioat_channel_ring_t *ring; in ioat_ring_seed() local
772 ring = channel->ic_ring; in ioat_ring_seed()
775 ring->cr_cmpl_gen = 0x0; in ioat_ring_seed()
776 ring->cr_cmpl_last = 0x0; in ioat_ring_seed()
779 ring->cr_post_cnt++; in ioat_ring_seed()
781 ring->cr_desc_gen = 0; in ioat_ring_seed()
782 ring->cr_desc_prev = 0; in ioat_ring_seed()
783 ring->cr_desc_next = 1; in ioat_ring_seed()
794 ASSERT(ring->cr_chan->ic_ver == IOAT_CBv2); in ioat_ring_seed()
796 &ring->cr_desc[ring->cr_desc_next]; in ioat_ring_seed()
798 &ring->cr_desc[ring->cr_desc_prev]; in ioat_ring_seed()
803 prev->dd_next_desc = ring->cr_phys_desc + in ioat_ring_seed()
804 (ring->cr_desc_next << 6); in ioat_ring_seed()
819 ioat_ring_loop(ioat_channel_ring_t *ring, dcopy_cmd_t cmd) in ioat_ring_loop() argument
828 channel = ring->cr_chan; in ioat_ring_loop()
863 curr = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_prev]; in ioat_ring_loop()
864 curr->dd_next_desc = ring->cr_phys_desc + (currpriv->ip_start << 6); in ioat_ring_loop()
868 ring->cr_desc_prev << 6, 64, DDI_DMA_SYNC_FORDEV); in ioat_ring_loop()
961 ioat_channel_ring_t *ring; in ioat_cmd_post() local
981 ring = channel->ic_ring; in ioat_cmd_post()
994 mutex_enter(&ring->cr_desc_mutex); in ioat_cmd_post()
1000 } else if (mutex_tryenter(&ring->cr_desc_mutex) == 0) { in ioat_cmd_post()
1006 mutex_exit(&ring->cr_desc_mutex); in ioat_cmd_post()
1011 e = ioat_ring_reserve(channel, ring, cmd); in ioat_cmd_post()
1013 mutex_exit(&ring->cr_desc_mutex); in ioat_cmd_post()
1020 ioat_cmd_post_dca(ring, cmd->dp_dca_id); in ioat_cmd_post()
1032 priv->ip_start = ring->cr_desc_next; in ioat_cmd_post()
1074 ioat_cmd_post_copy(ring, src_paddr, dest_paddr, xfer_size, in ioat_cmd_post()
1084 priv->ip_generation = ring->cr_desc_gen_prev; in ioat_cmd_post()
1085 priv->ip_index = ring->cr_desc_prev; in ioat_cmd_post()
1093 e = ioat_ring_loop(ring, cmd); in ioat_cmd_post()
1095 mutex_exit(&ring->cr_desc_mutex); in ioat_cmd_post()
1108 (uint16_t)(ring->cr_post_cnt & 0xFFFF)); in ioat_cmd_post()
1112 mutex_exit(&ring->cr_desc_mutex); in ioat_cmd_post()
1122 ioat_cmd_post_dca(ioat_channel_ring_t *ring, uint32_t dca_id) in ioat_cmd_post_dca() argument
1133 channel = ring->cr_chan; in ioat_cmd_post_dca()
1134 desc = (ioat_chan_dca_desc_t *)&ring->cr_desc[ring->cr_desc_next]; in ioat_cmd_post_dca()
1135 prev = (ioat_chan_dca_desc_t *)&ring->cr_desc[ring->cr_desc_prev]; in ioat_cmd_post_dca()
1138 ring->cr_post_cnt++; in ioat_cmd_post_dca()
1160 prev_offset = ring->cr_desc_prev << 6; in ioat_cmd_post_dca()
1161 next_offset = ring->cr_desc_next << 6; in ioat_cmd_post_dca()
1162 next_desc_phys = ring->cr_phys_desc + next_offset; in ioat_cmd_post_dca()
1165 ring->cr_desc_prev = ring->cr_desc_next; in ioat_cmd_post_dca()
1166 ring->cr_desc_gen_prev = ring->cr_desc_gen; in ioat_cmd_post_dca()
1169 ring->cr_desc_next++; in ioat_cmd_post_dca()
1170 if (ring->cr_desc_next > ring->cr_desc_last) { in ioat_cmd_post_dca()
1171 ring->cr_desc_next = 0; in ioat_cmd_post_dca()
1172 ring->cr_desc_gen++; in ioat_cmd_post_dca()
1179 if (ring->cr_chan->ic_ver == IOAT_CBv2) { in ioat_cmd_post_dca()
1181 &ring->cr_desc[ring->cr_desc_next]; in ioat_cmd_post_dca()
1183 &ring->cr_desc[ring->cr_desc_prev]; in ioat_cmd_post_dca()
1187 ring->cr_desc_next << 6, 64, DDI_DMA_SYNC_FORDEV); in ioat_cmd_post_dca()
1188 prev->dd_next_desc = ring->cr_phys_desc + in ioat_cmd_post_dca()
1189 (ring->cr_desc_next << 6); in ioat_cmd_post_dca()
1212 ioat_cmd_post_copy(ioat_channel_ring_t *ring, uint64_t src_addr, in ioat_cmd_post_copy() argument
1224 channel = ring->cr_chan; in ioat_cmd_post_copy()
1225 desc = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_next]; in ioat_cmd_post_copy()
1226 prev = (ioat_chan_dma_desc_t *)&ring->cr_desc[ring->cr_desc_prev]; in ioat_cmd_post_copy()
1229 ring->cr_post_cnt++; in ioat_cmd_post_copy()
1243 prev_offset = ring->cr_desc_prev << 6; in ioat_cmd_post_copy()
1244 next_offset = ring->cr_desc_next << 6; in ioat_cmd_post_copy()
1245 next_desc_phy = ring->cr_phys_desc + next_offset; in ioat_cmd_post_copy()
1248 ring->cr_desc_prev = ring->cr_desc_next; in ioat_cmd_post_copy()
1249 ring->cr_desc_gen_prev = ring->cr_desc_gen; in ioat_cmd_post_copy()
1252 ring->cr_desc_next++; in ioat_cmd_post_copy()
1253 if (ring->cr_desc_next > ring->cr_desc_last) { in ioat_cmd_post_copy()
1254 ring->cr_desc_next = 0; in ioat_cmd_post_copy()
1255 ring->cr_desc_gen++; in ioat_cmd_post_copy()
1262 if (ring->cr_chan->ic_ver == IOAT_CBv2) { in ioat_cmd_post_copy()
1264 &ring->cr_desc[ring->cr_desc_next]; in ioat_cmd_post_copy()
1266 &ring->cr_desc[ring->cr_desc_prev]; in ioat_cmd_post_copy()
1271 ring->cr_desc_next << 6, 64, DDI_DMA_SYNC_FORDEV); in ioat_cmd_post_copy()
1272 prev->dd_next_desc = ring->cr_phys_desc + in ioat_cmd_post_copy()
1273 (ring->cr_desc_next << 6); in ioat_cmd_post_copy()
1297 ioat_channel_ring_t *ring; in ioat_cmd_poll() local
1307 ring = channel->ic_ring; in ioat_cmd_poll()
1308 ASSERT(ring != NULL); in ioat_cmd_poll()
1311 mutex_enter(&ring->cr_cmpl_mutex); in ioat_cmd_poll()
1317 } else if (mutex_tryenter(&ring->cr_cmpl_mutex) == 0) { in ioat_cmd_poll()
1324 mutex_exit(&ring->cr_cmpl_mutex); in ioat_cmd_poll()
1337 if (last_cmpl != ring->cr_cmpl_last) { in ioat_cmd_poll()
1343 if (last_cmpl < ring->cr_cmpl_last) { in ioat_cmd_poll()
1344 ring->cr_cmpl_gen++; in ioat_cmd_poll()
1346 ring->cr_cmpl_last = last_cmpl; in ioat_cmd_poll()
1347 generation = ring->cr_cmpl_gen; in ioat_cmd_poll()
1350 generation = ring->cr_cmpl_gen; in ioat_cmd_poll()
1353 mutex_exit(&ring->cr_cmpl_mutex); in ioat_cmd_poll()
1384 ioat_ring_reserve(ioat_channel_t channel, ioat_channel_ring_t *ring, in ioat_ring_reserve() argument
1436 desc = ring->cr_desc_next; in ioat_ring_reserve()
1443 if (desc == ring->cr_desc_last) { in ioat_ring_reserve()
1444 if (ring->cr_cmpl_last == 0) { in ioat_ring_reserve()
1452 if (ring->cr_cmpl_last == 0) { in ioat_ring_reserve()
1468 if ((desc + 1) == ring->cr_cmpl_last) { in ioat_ring_reserve()
1476 if ((desc + 1) == ring->cr_cmpl_last) { in ioat_ring_reserve()