Lines Matching refs:cmd
1474 nvme_cmd_t *cmd = kmem_cache_alloc(nvme_cmd_cache, kmflag); in nvme_alloc_cmd() local
1476 if (cmd == NULL) in nvme_alloc_cmd()
1477 return (cmd); in nvme_alloc_cmd()
1479 bzero(cmd, sizeof (nvme_cmd_t)); in nvme_alloc_cmd()
1481 cmd->nc_nvme = nvme; in nvme_alloc_cmd()
1483 mutex_init(&cmd->nc_mutex, NULL, MUTEX_DRIVER, in nvme_alloc_cmd()
1485 cv_init(&cmd->nc_cv, NULL, CV_DRIVER, NULL); in nvme_alloc_cmd()
1487 return (cmd); in nvme_alloc_cmd()
1491 nvme_free_cmd(nvme_cmd_t *cmd) in nvme_free_cmd() argument
1494 if (list_link_active(&cmd->nc_list)) in nvme_free_cmd()
1497 if (cmd->nc_dma) { in nvme_free_cmd()
1498 nvme_free_dma(cmd->nc_dma); in nvme_free_cmd()
1499 cmd->nc_dma = NULL; in nvme_free_cmd()
1502 if (cmd->nc_prp) { in nvme_free_cmd()
1503 kmem_cache_free(cmd->nc_nvme->n_prp_cache, cmd->nc_prp); in nvme_free_cmd()
1504 cmd->nc_prp = NULL; in nvme_free_cmd()
1507 cv_destroy(&cmd->nc_cv); in nvme_free_cmd()
1508 mutex_destroy(&cmd->nc_mutex); in nvme_free_cmd()
1510 kmem_cache_free(nvme_cmd_cache, cmd); in nvme_free_cmd()
1514 nvme_submit_admin_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) in nvme_submit_admin_cmd() argument
1517 nvme_submit_cmd_common(qp, cmd); in nvme_submit_admin_cmd()
1521 nvme_submit_io_cmd(nvme_qpair_t *qp, nvme_cmd_t *cmd) in nvme_submit_io_cmd() argument
1523 if (cmd->nc_nvme->n_dead) { in nvme_submit_io_cmd()
1530 nvme_submit_cmd_common(qp, cmd); in nvme_submit_io_cmd()
1535 nvme_submit_cmd_common(nvme_qpair_t *qp, nvme_cmd_t *cmd) in nvme_submit_cmd_common() argument
1540 cmd->nc_completed = B_FALSE; in nvme_submit_cmd_common()
1550 if (cmd->nc_nvme->n_dead) { in nvme_submit_cmd_common()
1551 taskq_dispatch_ent(qp->nq_cq->ncq_cmd_taskq, cmd->nc_callback, in nvme_submit_cmd_common()
1552 cmd, TQ_NOSLEEP, &cmd->nc_tqent); in nvme_submit_cmd_common()
1566 qp->nq_cmd[qp->nq_next_cmd] = cmd; in nvme_submit_cmd_common()
1570 cmd->nc_sqe.sqe_cid = qp->nq_next_cmd; in nvme_submit_cmd_common()
1571 bcopy(&cmd->nc_sqe, &qp->nq_sq[qp->nq_sqtail], sizeof (nvme_sqe_t)); in nvme_submit_cmd_common()
1578 nvme_put32(cmd->nc_nvme, qp->nq_sqtdbl, tail.r); in nvme_submit_cmd_common()
1586 nvme_cmd_t *cmd; in nvme_unqueue_cmd() local
1591 cmd = qp->nq_cmd[cid]; in nvme_unqueue_cmd()
1603 if (cmd == NULL) in nvme_unqueue_cmd()
1610 ASSERT3P(cmd, !=, NULL); in nvme_unqueue_cmd()
1611 ASSERT3P(cmd->nc_nvme, ==, nvme); in nvme_unqueue_cmd()
1612 ASSERT3S(cmd->nc_sqe.sqe_cid, ==, cid); in nvme_unqueue_cmd()
1614 return (cmd); in nvme_unqueue_cmd()
1626 nvme_cmd_t *cmd; in nvme_get_completed() local
1640 cmd = nvme_unqueue_cmd(nvme, qp, cqe->cqe_cid); in nvme_get_completed()
1650 if (cmd == NULL) { in nvme_get_completed()
1665 ASSERT3U(cmd->nc_sqid, ==, cqe->cqe_sqid); in nvme_get_completed()
1666 bcopy(cqe, &cmd->nc_cqe, sizeof (nvme_cqe_t)); in nvme_get_completed()
1668 return (cmd); in nvme_get_completed()
1678 nvme_cmd_t *cmd; in nvme_process_iocq() local
1688 while ((cmd = nvme_get_completed(nvme, cq)) != NULL) { in nvme_process_iocq()
1689 taskq_dispatch_ent(cq->ncq_cmd_taskq, cmd->nc_callback, cmd, in nvme_process_iocq()
1690 TQ_NOSLEEP, &cmd->nc_tqent); in nvme_process_iocq()
1713 nvme_cmd_t *cmd; in nvme_retrieve_cmd() local
1722 if ((cmd = nvme_get_completed(nvme, cq)) != NULL) { in nvme_retrieve_cmd()
1729 return (cmd); in nvme_retrieve_cmd()
1733 nvme_check_unknown_cmd_status(nvme_cmd_t *cmd) in nvme_check_unknown_cmd_status() argument
1735 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_unknown_cmd_status()
1737 dev_err(cmd->nc_nvme->n_dip, CE_WARN, in nvme_check_unknown_cmd_status()
1739 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, in nvme_check_unknown_cmd_status()
1743 if (cmd->nc_xfer != NULL) in nvme_check_unknown_cmd_status()
1744 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); in nvme_check_unknown_cmd_status()
1746 if (cmd->nc_nvme->n_strict_version) { in nvme_check_unknown_cmd_status()
1747 nvme_ctrl_mark_dead(cmd->nc_nvme, B_FALSE); in nvme_check_unknown_cmd_status()
1754 nvme_check_vendor_cmd_status(nvme_cmd_t *cmd) in nvme_check_vendor_cmd_status() argument
1756 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_vendor_cmd_status()
1758 dev_err(cmd->nc_nvme->n_dip, CE_WARN, in nvme_check_vendor_cmd_status()
1760 "sc = %x, sct = %x, dnr = %d, m = %d", cmd->nc_sqe.sqe_opc, in nvme_check_vendor_cmd_status()
1763 if (!cmd->nc_nvme->n_ignore_unknown_vendor_status) { in nvme_check_vendor_cmd_status()
1764 nvme_ctrl_mark_dead(cmd->nc_nvme, B_FALSE); in nvme_check_vendor_cmd_status()
1771 nvme_check_integrity_cmd_status(nvme_cmd_t *cmd) in nvme_check_integrity_cmd_status() argument
1773 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_integrity_cmd_status()
1779 if (cmd->nc_xfer != NULL) in nvme_check_integrity_cmd_status()
1780 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); in nvme_check_integrity_cmd_status()
1786 if (cmd->nc_xfer != NULL) in nvme_check_integrity_cmd_status()
1787 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); in nvme_check_integrity_cmd_status()
1791 return (nvme_check_unknown_cmd_status(cmd)); in nvme_check_integrity_cmd_status()
1796 nvme_check_generic_cmd_status(nvme_cmd_t *cmd) in nvme_check_generic_cmd_status() argument
1798 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_generic_cmd_status()
1809 if (!cmd->nc_dontpanic) in nvme_check_generic_cmd_status()
1810 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, in nvme_check_generic_cmd_status()
1812 (void *)cmd); in nvme_check_generic_cmd_status()
1817 if (!cmd->nc_dontpanic) in nvme_check_generic_cmd_status()
1818 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, in nvme_check_generic_cmd_status()
1820 (void *)cmd); in nvme_check_generic_cmd_status()
1825 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " in nvme_check_generic_cmd_status()
1826 "cmd ID conflict in cmd %p", (void *)cmd); in nvme_check_generic_cmd_status()
1831 if (!cmd->nc_dontpanic) in nvme_check_generic_cmd_status()
1832 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, in nvme_check_generic_cmd_status()
1834 (void *)cmd); in nvme_check_generic_cmd_status()
1839 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " in nvme_check_generic_cmd_status()
1840 "LBA out of range in cmd %p", (void *)cmd); in nvme_check_generic_cmd_status()
1849 atomic_inc_32(&cmd->nc_nvme->n_data_xfr_err); in nvme_check_generic_cmd_status()
1850 if (cmd->nc_xfer != NULL) in nvme_check_generic_cmd_status()
1851 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); in nvme_check_generic_cmd_status()
1861 atomic_inc_32(&cmd->nc_nvme->n_internal_err); in nvme_check_generic_cmd_status()
1862 if (cmd->nc_xfer != NULL) in nvme_check_generic_cmd_status()
1863 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); in nvme_check_generic_cmd_status()
1872 atomic_inc_32(&cmd->nc_nvme->n_abort_rq_err); in nvme_check_generic_cmd_status()
1877 nvme_ctrl_mark_dead(cmd->nc_nvme, B_FALSE); in nvme_check_generic_cmd_status()
1882 atomic_inc_32(&cmd->nc_nvme->n_abort_sq_del); in nvme_check_generic_cmd_status()
1887 atomic_inc_32(&cmd->nc_nvme->n_nvm_cap_exc); in nvme_check_generic_cmd_status()
1888 if (cmd->nc_xfer != NULL) in nvme_check_generic_cmd_status()
1889 bd_error(cmd->nc_xfer, BD_ERR_MEDIA); in nvme_check_generic_cmd_status()
1894 atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_notrdy); in nvme_check_generic_cmd_status()
1895 if (cmd->nc_xfer != NULL) in nvme_check_generic_cmd_status()
1896 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); in nvme_check_generic_cmd_status()
1901 if (!NVME_VERSION_ATLEAST(&cmd->nc_nvme->n_version, 1, 2)) in nvme_check_generic_cmd_status()
1902 return (nvme_check_unknown_cmd_status(cmd)); in nvme_check_generic_cmd_status()
1903 atomic_inc_32(&cmd->nc_nvme->n_nvm_ns_formatting); in nvme_check_generic_cmd_status()
1904 if (cmd->nc_xfer != NULL) in nvme_check_generic_cmd_status()
1905 bd_error(cmd->nc_xfer, BD_ERR_NTRDY); in nvme_check_generic_cmd_status()
1909 return (nvme_check_unknown_cmd_status(cmd)); in nvme_check_generic_cmd_status()
1914 nvme_check_specific_cmd_status(nvme_cmd_t *cmd) in nvme_check_specific_cmd_status() argument
1916 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_specific_cmd_status()
1921 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE); in nvme_check_specific_cmd_status()
1922 atomic_inc_32(&cmd->nc_nvme->n_inv_cq_err); in nvme_check_specific_cmd_status()
1927 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || in nvme_check_specific_cmd_status()
1928 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_SQUEUE || in nvme_check_specific_cmd_status()
1929 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE || in nvme_check_specific_cmd_status()
1930 cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); in nvme_check_specific_cmd_status()
1931 atomic_inc_32(&cmd->nc_nvme->n_inv_qid_err); in nvme_check_specific_cmd_status()
1936 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_SQUEUE || in nvme_check_specific_cmd_status()
1937 cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); in nvme_check_specific_cmd_status()
1938 atomic_inc_32(&cmd->nc_nvme->n_max_qsz_exc); in nvme_check_specific_cmd_status()
1943 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT); in nvme_check_specific_cmd_status()
1944 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " in nvme_check_specific_cmd_status()
1945 "abort command limit exceeded in cmd %p", (void *)cmd); in nvme_check_specific_cmd_status()
1950 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_ASYNC_EVENT); in nvme_check_specific_cmd_status()
1951 dev_err(cmd->nc_nvme->n_dip, CE_PANIC, "programming error: " in nvme_check_specific_cmd_status()
1953 (void *)cmd); in nvme_check_specific_cmd_status()
1958 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_CREATE_CQUEUE); in nvme_check_specific_cmd_status()
1959 atomic_inc_32(&cmd->nc_nvme->n_inv_int_vect); in nvme_check_specific_cmd_status()
1964 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_GET_LOG_PAGE); in nvme_check_specific_cmd_status()
1965 atomic_inc_32(&cmd->nc_nvme->n_inv_log_page); in nvme_check_specific_cmd_status()
1970 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_FORMAT); in nvme_check_specific_cmd_status()
1971 atomic_inc_32(&cmd->nc_nvme->n_inv_format); in nvme_check_specific_cmd_status()
1972 if (cmd->nc_xfer != NULL) in nvme_check_specific_cmd_status()
1973 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); in nvme_check_specific_cmd_status()
1978 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_DELETE_CQUEUE); in nvme_check_specific_cmd_status()
1979 atomic_inc_32(&cmd->nc_nvme->n_inv_q_del); in nvme_check_specific_cmd_status()
1984 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_DSET_MGMT || in nvme_check_specific_cmd_status()
1985 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || in nvme_check_specific_cmd_status()
1986 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); in nvme_check_specific_cmd_status()
1987 atomic_inc_32(&cmd->nc_nvme->n_cnfl_attr); in nvme_check_specific_cmd_status()
1988 if (cmd->nc_xfer != NULL) in nvme_check_specific_cmd_status()
1989 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); in nvme_check_specific_cmd_status()
1994 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_COMPARE || in nvme_check_specific_cmd_status()
1995 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_READ || in nvme_check_specific_cmd_status()
1996 cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); in nvme_check_specific_cmd_status()
1997 atomic_inc_32(&cmd->nc_nvme->n_inv_prot); in nvme_check_specific_cmd_status()
1998 if (cmd->nc_xfer != NULL) in nvme_check_specific_cmd_status()
1999 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); in nvme_check_specific_cmd_status()
2004 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_NVM_WRITE); in nvme_check_specific_cmd_status()
2005 atomic_inc_32(&cmd->nc_nvme->n_readonly); in nvme_check_specific_cmd_status()
2006 if (cmd->nc_xfer != NULL) in nvme_check_specific_cmd_status()
2007 bd_error(cmd->nc_xfer, BD_ERR_ILLRQ); in nvme_check_specific_cmd_status()
2012 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); in nvme_check_specific_cmd_status()
2017 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); in nvme_check_specific_cmd_status()
2022 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); in nvme_check_specific_cmd_status()
2027 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); in nvme_check_specific_cmd_status()
2032 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); in nvme_check_specific_cmd_status()
2037 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); in nvme_check_specific_cmd_status()
2042 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_ACTIVATE); in nvme_check_specific_cmd_status()
2047 ASSERT(cmd->nc_sqe.sqe_opc == NVME_OPC_FW_IMAGE_LOAD); in nvme_check_specific_cmd_status()
2051 return (nvme_check_unknown_cmd_status(cmd)); in nvme_check_specific_cmd_status()
2056 nvme_check_cmd_status(nvme_cmd_t *cmd) in nvme_check_cmd_status() argument
2058 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_cmd_status()
2064 if (cmd->nc_nvme->n_dead) in nvme_check_cmd_status()
2072 return (nvme_check_generic_cmd_status(cmd)); in nvme_check_cmd_status()
2074 return (nvme_check_specific_cmd_status(cmd)); in nvme_check_cmd_status()
2076 return (nvme_check_integrity_cmd_status(cmd)); in nvme_check_cmd_status()
2078 return (nvme_check_vendor_cmd_status(cmd)); in nvme_check_cmd_status()
2080 return (nvme_check_unknown_cmd_status(cmd)); in nvme_check_cmd_status()
2089 nvme_check_cmd_status_ioctl(nvme_cmd_t *cmd, nvme_ioctl_common_t *ioc) in nvme_check_cmd_status_ioctl() argument
2091 nvme_cqe_t *cqe = &cmd->nc_cqe; in nvme_check_cmd_status_ioctl()
2092 nvme_t *nvme = cmd->nc_nvme; in nvme_check_cmd_status_ioctl()
2103 (void) nvme_check_generic_cmd_status(cmd); in nvme_check_cmd_status_ioctl()
2105 (void) nvme_check_specific_cmd_status(cmd); in nvme_check_cmd_status_ioctl()
2107 (void) nvme_check_integrity_cmd_status(cmd); in nvme_check_cmd_status_ioctl()
2109 (void) nvme_check_vendor_cmd_status(cmd); in nvme_check_cmd_status_ioctl()
2111 (void) nvme_check_unknown_cmd_status(cmd); in nvme_check_cmd_status_ioctl()
2122 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_abort_cmd() local
2131 cmd->nc_sqid = 0; in nvme_abort_cmd()
2132 cmd->nc_sqe.sqe_opc = NVME_OPC_ABORT; in nvme_abort_cmd()
2133 cmd->nc_callback = nvme_wakeup_cmd; in nvme_abort_cmd()
2134 cmd->nc_sqe.sqe_cdw10 = ac.r; in nvme_abort_cmd()
2142 nvme_admin_cmd(cmd, sec); in nvme_abort_cmd()
2145 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_abort_cmd()
2148 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); in nvme_abort_cmd()
2154 cmd->nc_cqe.cqe_dw0 & 1 ? "un" : ""); in nvme_abort_cmd()
2155 if ((cmd->nc_cqe.cqe_dw0 & 1) == 0) in nvme_abort_cmd()
2159 nvme_free_cmd(cmd); in nvme_abort_cmd()
2170 nvme_wait_cmd(nvme_cmd_t *cmd, uint32_t sec) in nvme_wait_cmd() argument
2173 nvme_t *nvme = cmd->nc_nvme; in nvme_wait_cmd()
2177 ASSERT(mutex_owned(&cmd->nc_mutex)); in nvme_wait_cmd()
2179 while (!cmd->nc_completed) { in nvme_wait_cmd()
2180 if (cv_timedwait(&cmd->nc_cv, &cmd->nc_mutex, timeout) == -1) in nvme_wait_cmd()
2184 if (cmd->nc_completed) in nvme_wait_cmd()
2196 "OPC = %x, CFS = %d", cmd->nc_sqe.sqe_cid, cmd->nc_sqid, in nvme_wait_cmd()
2197 cmd->nc_sqe.sqe_opc, csts.b.csts_cfs); in nvme_wait_cmd()
2202 nvme_check_dma_hdl(cmd->nc_dma) || in nvme_wait_cmd()
2203 cmd->nc_sqe.sqe_opc == NVME_OPC_ABORT) { in nvme_wait_cmd()
2204 nvme_ctrl_mark_dead(cmd->nc_nvme, B_FALSE); in nvme_wait_cmd()
2205 } else if (nvme_abort_cmd(cmd, sec) == 0) { in nvme_wait_cmd()
2210 while (!cmd->nc_completed) in nvme_wait_cmd()
2211 cv_wait(&cmd->nc_cv, &cmd->nc_mutex); in nvme_wait_cmd()
2216 qp = nvme->n_ioq[cmd->nc_sqid]; in nvme_wait_cmd()
2219 (void) nvme_unqueue_cmd(nvme, qp, cmd->nc_sqe.sqe_cid); in nvme_wait_cmd()
2227 if (cmd->nc_dma != NULL) { in nvme_wait_cmd()
2229 list_insert_head(&nvme_lost_cmds, cmd); in nvme_wait_cmd()
2237 nvme_cmd_t *cmd = arg; in nvme_wakeup_cmd() local
2239 mutex_enter(&cmd->nc_mutex); in nvme_wakeup_cmd()
2240 cmd->nc_completed = B_TRUE; in nvme_wakeup_cmd()
2241 cv_signal(&cmd->nc_cv); in nvme_wakeup_cmd()
2242 mutex_exit(&cmd->nc_mutex); in nvme_wakeup_cmd()
2248 nvme_cmd_t *cmd = arg; in nvme_async_event_task() local
2249 nvme_t *nvme = cmd->nc_nvme; in nvme_async_event_task()
2271 if (nvme_check_cmd_status(cmd) != 0) { in nvme_async_event_task()
2272 dev_err(cmd->nc_nvme->n_dip, CE_WARN, in nvme_async_event_task()
2274 "sc = 0x%x, dnr = %d, m = %d", cmd->nc_cqe.cqe_sf.sf_sct, in nvme_async_event_task()
2275 cmd->nc_cqe.cqe_sf.sf_sc, cmd->nc_cqe.cqe_sf.sf_dnr, in nvme_async_event_task()
2276 cmd->nc_cqe.cqe_sf.sf_m); in nvme_async_event_task()
2278 if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && in nvme_async_event_task()
2279 cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INTERNAL_ERR) { in nvme_async_event_task()
2280 nvme_ctrl_mark_dead(cmd->nc_nvme, B_FALSE); in nvme_async_event_task()
2283 if (cmd->nc_cqe.cqe_sf.sf_sct == NVME_CQE_SCT_GENERIC && in nvme_async_event_task()
2284 cmd->nc_cqe.cqe_sf.sf_sc == NVME_CQE_SC_GEN_INV_OPC && in nvme_async_event_task()
2285 cmd->nc_cqe.cqe_sf.sf_dnr == 1) { in nvme_async_event_task()
2289 nvme_free_cmd(cmd); in nvme_async_event_task()
2293 event.r = cmd->nc_cqe.cqe_dw0; in nvme_async_event_task()
2296 bzero(&cmd->nc_cqe, sizeof (nvme_cqe_t)); in nvme_async_event_task()
2297 nvme_submit_admin_cmd(nvme->n_adminq, cmd); in nvme_async_event_task()
2298 cmd = NULL; /* cmd can no longer be used after resubmission */ in nvme_async_event_task()
2329 nvme_ctrl_mark_dead(cmd->nc_nvme, B_FALSE); in nvme_async_event_task()
2336 nvme_ctrl_mark_dead(cmd->nc_nvme, B_FALSE); in nvme_async_event_task()
2524 nvme_admin_cmd(nvme_cmd_t *cmd, uint32_t sec) in nvme_admin_cmd() argument
2526 mutex_enter(&cmd->nc_mutex); in nvme_admin_cmd()
2527 nvme_submit_admin_cmd(cmd->nc_nvme->n_adminq, cmd); in nvme_admin_cmd()
2528 nvme_wait_cmd(cmd, sec); in nvme_admin_cmd()
2529 mutex_exit(&cmd->nc_mutex); in nvme_admin_cmd()
2535 nvme_cmd_t *cmd; in nvme_async_event() local
2537 cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_async_event()
2538 cmd->nc_sqid = 0; in nvme_async_event()
2539 cmd->nc_sqe.sqe_opc = NVME_OPC_ASYNC_EVENT; in nvme_async_event()
2540 cmd->nc_callback = nvme_async_event_task; in nvme_async_event()
2541 cmd->nc_dontpanic = B_TRUE; in nvme_async_event()
2543 nvme_submit_admin_cmd(nvme->n_adminq, cmd); in nvme_async_event()
2576 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_format_nvm() local
2583 cmd->nc_sqid = 0; in nvme_format_nvm()
2584 cmd->nc_callback = nvme_wakeup_cmd; in nvme_format_nvm()
2585 cmd->nc_sqe.sqe_nsid = ioc->nif_common.nioc_nsid; in nvme_format_nvm()
2586 cmd->nc_sqe.sqe_opc = NVME_OPC_NVM_FORMAT; in nvme_format_nvm()
2587 cmd->nc_sqe.sqe_cdw10 = format_nvm.r; in nvme_format_nvm()
2600 cmd->nc_dontpanic = B_TRUE; in nvme_format_nvm()
2602 nvme_admin_cmd(cmd, nvme_format_cmd_timeout); in nvme_format_nvm()
2604 if (!nvme_check_cmd_status_ioctl(cmd, &ioc->nif_common) != 0) { in nvme_format_nvm()
2607 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); in nvme_format_nvm()
2614 nvme_free_cmd(cmd); in nvme_format_nvm()
2626 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_get_logpage() local
2638 cmd->nc_sqid = 0; in nvme_get_logpage()
2639 cmd->nc_callback = nvme_wakeup_cmd; in nvme_get_logpage()
2640 cmd->nc_sqe.sqe_opc = NVME_OPC_GET_LOG_PAGE; in nvme_get_logpage()
2641 cmd->nc_sqe.sqe_nsid = log->nigl_common.nioc_nsid; in nvme_get_logpage()
2644 cmd->nc_dontpanic = B_TRUE; in nvme_get_logpage()
2667 cmd->nc_sqe.sqe_cdw10 = dw10.r; in nvme_get_logpage()
2668 cmd->nc_sqe.sqe_cdw11 = dw11.r; in nvme_get_logpage()
2669 cmd->nc_sqe.sqe_cdw12 = offlo; in nvme_get_logpage()
2670 cmd->nc_sqe.sqe_cdw13 = offhi; in nvme_get_logpage()
2671 cmd->nc_sqe.sqe_cdw14 = dw14.r; in nvme_get_logpage()
2674 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { in nvme_get_logpage()
2682 if (nvme_fill_prp(cmd, cmd->nc_dma->nd_dmah) != 0) { in nvme_get_logpage()
2687 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); in nvme_get_logpage()
2689 if (!nvme_check_cmd_status_ioctl(cmd, &log->nigl_common)) { in nvme_get_logpage()
2693 cmd->nc_cqe.cqe_sf.sf_sct, in nvme_get_logpage()
2694 cmd->nc_cqe.cqe_sf.sf_sc); in nvme_get_logpage()
2701 bcopy(cmd->nc_dma->nd_memp, *buf, log->nigl_len); in nvme_get_logpage()
2705 nvme_free_cmd(cmd); in nvme_get_logpage()
2768 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_identify() local
2776 cmd->nc_sqid = 0; in nvme_identify()
2777 cmd->nc_callback = nvme_wakeup_cmd; in nvme_identify()
2778 cmd->nc_sqe.sqe_opc = NVME_OPC_IDENTIFY; in nvme_identify()
2779 cmd->nc_sqe.sqe_nsid = ioc->nid_common.nioc_nsid; in nvme_identify()
2784 cmd->nc_sqe.sqe_cdw10 = dw10.r; in nvme_identify()
2787 &nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) { in nvme_identify()
2795 if (cmd->nc_dma->nd_ncookie > 2) { in nvme_identify()
2804 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_dma->nd_cookie.dmac_laddress; in nvme_identify()
2805 if (cmd->nc_dma->nd_ncookie > 1) { in nvme_identify()
2806 ddi_dma_nextcookie(cmd->nc_dma->nd_dmah, in nvme_identify()
2807 &cmd->nc_dma->nd_cookie); in nvme_identify()
2808 cmd->nc_sqe.sqe_dptr.d_prp[1] = in nvme_identify()
2809 cmd->nc_dma->nd_cookie.dmac_laddress; in nvme_identify()
2813 cmd->nc_dontpanic = B_TRUE; in nvme_identify()
2815 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); in nvme_identify()
2817 if (!nvme_check_cmd_status_ioctl(cmd, &ioc->nid_common)) { in nvme_identify()
2820 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); in nvme_identify()
2826 bcopy(cmd->nc_dma->nd_memp, *buf, NVME_IDENTIFY_BUFSIZE); in nvme_identify()
2830 nvme_free_cmd(cmd); in nvme_identify()
2852 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_set_features() local
2857 cmd->nc_sqid = 0; in nvme_set_features()
2858 cmd->nc_callback = nvme_wakeup_cmd; in nvme_set_features()
2859 cmd->nc_sqe.sqe_opc = NVME_OPC_SET_FEATURES; in nvme_set_features()
2860 cmd->nc_sqe.sqe_cdw10 = feature; in nvme_set_features()
2861 cmd->nc_sqe.sqe_cdw11 = val; in nvme_set_features()
2864 cmd->nc_dontpanic = B_TRUE; in nvme_set_features()
2879 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); in nvme_set_features()
2881 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_set_features()
2884 feature, cmd->nc_cqe.cqe_sf.sf_sct, in nvme_set_features()
2885 cmd->nc_cqe.cqe_sf.sf_sc); in nvme_set_features()
2889 *res = cmd->nc_cqe.cqe_dw0; in nvme_set_features()
2892 nvme_free_cmd(cmd); in nvme_set_features()
2973 nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_create_completion_queue() local
2985 cmd->nc_sqid = 0; in nvme_create_completion_queue()
2986 cmd->nc_callback = nvme_wakeup_cmd; in nvme_create_completion_queue()
2987 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_CQUEUE; in nvme_create_completion_queue()
2988 cmd->nc_sqe.sqe_cdw10 = dw10.r; in nvme_create_completion_queue()
2989 cmd->nc_sqe.sqe_cdw11 = c_dw11.r; in nvme_create_completion_queue()
2990 cmd->nc_sqe.sqe_dptr.d_prp[0] = cq->ncq_dma->nd_cookie.dmac_laddress; in nvme_create_completion_queue()
2992 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); in nvme_create_completion_queue()
2994 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_create_completion_queue()
2997 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); in nvme_create_completion_queue()
3000 nvme_free_cmd(cmd); in nvme_create_completion_queue()
3009 nvme_cmd_t *cmd; in nvme_create_io_qpair() local
3029 cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_create_io_qpair()
3030 cmd->nc_sqid = 0; in nvme_create_io_qpair()
3031 cmd->nc_callback = nvme_wakeup_cmd; in nvme_create_io_qpair()
3032 cmd->nc_sqe.sqe_opc = NVME_OPC_CREATE_SQUEUE; in nvme_create_io_qpair()
3033 cmd->nc_sqe.sqe_cdw10 = dw10.r; in nvme_create_io_qpair()
3034 cmd->nc_sqe.sqe_cdw11 = s_dw11.r; in nvme_create_io_qpair()
3035 cmd->nc_sqe.sqe_dptr.d_prp[0] = qp->nq_sqdma->nd_cookie.dmac_laddress; in nvme_create_io_qpair()
3037 nvme_admin_cmd(cmd, nvme_admin_cmd_timeout); in nvme_create_io_qpair()
3039 if ((ret = nvme_check_cmd_status(cmd)) != 0) { in nvme_create_io_qpair()
3042 cmd->nc_cqe.cqe_sf.sf_sct, cmd->nc_cqe.cqe_sf.sf_sc); in nvme_create_io_qpair()
3045 nvme_free_cmd(cmd); in nvme_create_io_qpair()
4334 nvme_cmd_t *cmd = qp->nq_cmd[j]; in nvme_remove_callback() local
4337 if (cmd == NULL) { in nvme_remove_callback()
4348 u_cmd = nvme_unqueue_cmd(nvme, qp, cmd->nc_sqe.sqe_cid); in nvme_remove_callback()
4350 cmd->nc_callback, cmd, TQ_NOSLEEP, &cmd->nc_tqent); in nvme_remove_callback()
4352 ASSERT3P(u_cmd, ==, cmd); in nvme_remove_callback()
4405 nvme_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) in nvme_attach() argument
4414 if (cmd != DDI_ATTACH) in nvme_attach()
4705 nvme_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) in nvme_detach() argument
4710 if (cmd != DDI_DETACH) in nvme_detach()
4881 nvme_fill_prp(nvme_cmd_t *cmd, ddi_dma_handle_t dma) in nvme_fill_prp() argument
4883 nvme_t *nvme = cmd->nc_nvme; in nvme_fill_prp()
4895 cmd->nc_sqe.sqe_dptr.d_prp[0] = cookie->dmac_laddress; in nvme_fill_prp()
4898 cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; in nvme_fill_prp()
4903 cmd->nc_sqe.sqe_dptr.d_prp[1] = cookie->dmac_laddress; in nvme_fill_prp()
4929 cmd->nc_prp = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP); in nvme_fill_prp()
4930 bzero(cmd->nc_prp->nd_memp, cmd->nc_prp->nd_len); in nvme_fill_prp()
4931 cmd->nc_sqe.sqe_dptr.d_prp[1] = cmd->nc_prp->nd_cookie.dmac_laddress; in nvme_fill_prp()
4933 prp = (uint64_t *)cmd->nc_prp->nd_memp; in nvme_fill_prp()
4940 (void) ddi_dma_sync(cmd->nc_prp->nd_dmah, 0, cmd->nc_prp->nd_len, in nvme_fill_prp()
4956 nvme_fill_ranges(nvme_cmd_t *cmd, bd_xfer_t *xfer, uint64_t blocksize, in nvme_fill_ranges() argument
4961 nvme_t *nvme = cmd->nc_nvme; in nvme_fill_ranges()
4975 cmd->nc_sqe.sqe_cdw10 = (dfl->dfl_num_exts - 1) & 0xff; in nvme_fill_ranges()
4977 cmd->nc_sqe.sqe_cdw11 = NVME_DSET_MGMT_ATTR_DEALLOCATE; in nvme_fill_ranges()
4979 cmd->nc_prp = kmem_cache_alloc(nvme->n_prp_cache, allocflag); in nvme_fill_ranges()
4980 if (cmd->nc_prp == NULL) in nvme_fill_ranges()
4983 bzero(cmd->nc_prp->nd_memp, cmd->nc_prp->nd_len); in nvme_fill_ranges()
4984 ranges = (nvme_range_t *)cmd->nc_prp->nd_memp; in nvme_fill_ranges()
4986 cmd->nc_sqe.sqe_dptr.d_prp[0] = cmd->nc_prp->nd_cookie.dmac_laddress; in nvme_fill_ranges()
4987 cmd->nc_sqe.sqe_dptr.d_prp[1] = 0; in nvme_fill_ranges()
5003 (void) ddi_dma_sync(cmd->nc_prp->nd_dmah, 0, cmd->nc_prp->nd_len, in nvme_fill_ranges()
5013 nvme_cmd_t *cmd; in nvme_create_nvm_cmd() local
5020 cmd = nvme_alloc_cmd(nvme, allocflag); in nvme_create_nvm_cmd()
5022 if (cmd == NULL) in nvme_create_nvm_cmd()
5025 cmd->nc_sqe.sqe_opc = opc; in nvme_create_nvm_cmd()
5026 cmd->nc_callback = nvme_bd_xfer_done; in nvme_create_nvm_cmd()
5027 cmd->nc_xfer = xfer; in nvme_create_nvm_cmd()
5034 cmd->nc_sqe.sqe_nsid = ns->ns_id; in nvme_create_nvm_cmd()
5036 cmd->nc_sqe.sqe_cdw10 = xfer->x_blkno & 0xffffffffu; in nvme_create_nvm_cmd()
5037 cmd->nc_sqe.sqe_cdw11 = (xfer->x_blkno >> 32); in nvme_create_nvm_cmd()
5038 cmd->nc_sqe.sqe_cdw12 = (uint16_t)(xfer->x_nblks - 1); in nvme_create_nvm_cmd()
5040 if (nvme_fill_prp(cmd, xfer->x_dmah) != DDI_SUCCESS) in nvme_create_nvm_cmd()
5045 cmd->nc_sqe.sqe_nsid = ns->ns_id; in nvme_create_nvm_cmd()
5049 cmd->nc_sqe.sqe_nsid = ns->ns_id; in nvme_create_nvm_cmd()
5051 if (nvme_fill_ranges(cmd, xfer, in nvme_create_nvm_cmd()
5060 return (cmd); in nvme_create_nvm_cmd()
5063 nvme_free_cmd(cmd); in nvme_create_nvm_cmd()
5070 nvme_cmd_t *cmd = arg; in nvme_bd_xfer_done() local
5071 bd_xfer_t *xfer = cmd->nc_xfer; in nvme_bd_xfer_done()
5074 error = nvme_check_cmd_status(cmd); in nvme_bd_xfer_done()
5075 nvme_free_cmd(cmd); in nvme_bd_xfer_done()
5206 nvme_cmd_t *cmd; in nvme_bd_cmd() local
5215 cmd = nvme_create_nvm_cmd(ns, opc, xfer); in nvme_bd_cmd()
5216 if (cmd == NULL) in nvme_bd_cmd()
5219 cmd->nc_sqid = xfer->x_qnum + 1; in nvme_bd_cmd()
5220 ASSERT(cmd->nc_sqid <= nvme->n_ioq_count); in nvme_bd_cmd()
5221 ioq = nvme->n_ioq[cmd->nc_sqid]; in nvme_bd_cmd()
5230 ret = nvme_submit_io_cmd(ioq, cmd); in nvme_bd_cmd()
5239 cmd = nvme_retrieve_cmd(nvme, ioq); in nvme_bd_cmd()
5240 if (cmd != NULL) in nvme_bd_cmd()
5241 cmd->nc_callback(cmd); in nvme_bd_cmd()
5954 nvme_cmd_t *cmd; in nvme_ioc_cmd() local
5957 cmd = nvme_alloc_cmd(nvme, KM_SLEEP); in nvme_ioc_cmd()
5958 cmd->nc_sqid = 0; in nvme_ioc_cmd()
5967 cmd->nc_dontpanic = B_TRUE; in nvme_ioc_cmd()
5969 cmd->nc_callback = nvme_wakeup_cmd; in nvme_ioc_cmd()
5970 cmd->nc_sqe = *args->ica_sqe; in nvme_ioc_cmd()
5980 args->ica_dma_flags, &nvme->n_prp_dma_attr, &cmd->nc_dma) != in nvme_ioc_cmd()
5989 if (nvme_fill_prp(cmd, cmd->nc_dma->nd_dmah) != 0) { in nvme_ioc_cmd()
5996 ddi_copyin(args->ica_data, cmd->nc_dma->nd_memp, in nvme_ioc_cmd()
6004 nvme_admin_cmd(cmd, args->ica_timeout); in nvme_ioc_cmd()
6006 if (!nvme_check_cmd_status_ioctl(cmd, ioc)) { in nvme_ioc_cmd()
6011 args->ica_cdw0 = cmd->nc_cqe.cqe_dw0; in nvme_ioc_cmd()
6014 ddi_copyout(cmd->nc_dma->nd_memp, args->ica_data, in nvme_ioc_cmd()
6024 nvme_free_cmd(cmd); in nvme_ioc_cmd()
6562 nvme_passthru_copyin_cmd(const void *buf, nvme_ioctl_passthru_t *cmd, int mode) in nvme_passthru_copyin_cmd() argument
6572 bzero(cmd, sizeof (nvme_ioctl_passthru_t)); in nvme_passthru_copyin_cmd()
6574 cmd->npc_common.nioc_nsid = cmd32.npc_common.nioc_nsid; in nvme_passthru_copyin_cmd()
6575 cmd->npc_opcode = cmd32.npc_opcode; in nvme_passthru_copyin_cmd()
6576 cmd->npc_timeout = cmd32.npc_timeout; in nvme_passthru_copyin_cmd()
6577 cmd->npc_flags = cmd32.npc_flags; in nvme_passthru_copyin_cmd()
6578 cmd->npc_impact = cmd32.npc_impact; in nvme_passthru_copyin_cmd()
6579 cmd->npc_cdw12 = cmd32.npc_cdw12; in nvme_passthru_copyin_cmd()
6580 cmd->npc_cdw13 = cmd32.npc_cdw13; in nvme_passthru_copyin_cmd()
6581 cmd->npc_cdw14 = cmd32.npc_cdw14; in nvme_passthru_copyin_cmd()
6582 cmd->npc_cdw15 = cmd32.npc_cdw15; in nvme_passthru_copyin_cmd()
6583 cmd->npc_buflen = cmd32.npc_buflen; in nvme_passthru_copyin_cmd()
6584 cmd->npc_buf = cmd32.npc_buf; in nvme_passthru_copyin_cmd()
6589 if (ddi_copyin(buf, (void *)cmd, sizeof (nvme_ioctl_passthru_t), in nvme_passthru_copyin_cmd()
6606 nvme_passthru_copyout_cmd(const nvme_ioctl_passthru_t *cmd, void *buf, int mode) in nvme_passthru_copyout_cmd() argument
6615 cmd32.npc_common = cmd->npc_common; in nvme_passthru_copyout_cmd()
6616 cmd32.npc_opcode = cmd->npc_opcode; in nvme_passthru_copyout_cmd()
6617 cmd32.npc_timeout = cmd->npc_timeout; in nvme_passthru_copyout_cmd()
6618 cmd32.npc_flags = cmd->npc_flags; in nvme_passthru_copyout_cmd()
6619 cmd32.npc_impact = cmd->npc_impact; in nvme_passthru_copyout_cmd()
6620 cmd32.npc_cdw0 = cmd->npc_cdw0; in nvme_passthru_copyout_cmd()
6621 cmd32.npc_cdw12 = cmd->npc_cdw12; in nvme_passthru_copyout_cmd()
6622 cmd32.npc_cdw13 = cmd->npc_cdw13; in nvme_passthru_copyout_cmd()
6623 cmd32.npc_cdw14 = cmd->npc_cdw14; in nvme_passthru_copyout_cmd()
6624 cmd32.npc_cdw15 = cmd->npc_cdw15; in nvme_passthru_copyout_cmd()
6625 cmd32.npc_buflen = (size32_t)cmd->npc_buflen; in nvme_passthru_copyout_cmd()
6626 cmd32.npc_buf = (uintptr32_t)cmd->npc_buf; in nvme_passthru_copyout_cmd()
6633 if (ddi_copyout(cmd, buf, sizeof (nvme_ioctl_passthru_t), in nvme_passthru_copyout_cmd()
7019 nvme_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred_p, in nvme_ioctl() argument
7037 if (IS_DEVCTL(cmd)) in nvme_ioctl()
7038 return (ndi_devctl_ioctl(nvme->n_dip, cmd, arg, mode, 0)); in nvme_ioctl()
7040 if (nvme->n_dead && (cmd != NVME_IOC_DETACH && cmd != in nvme_ioctl()
7042 if (IS_NVME_IOC(cmd) == 0) { in nvme_ioctl()
7053 switch (cmd) { in nvme_ioctl()