11c5bc425SAlexey Zaytsev /*
21c5bc425SAlexey Zaytsev * CDDL HEADER START
31c5bc425SAlexey Zaytsev *
41c5bc425SAlexey Zaytsev * The contents of this file are subject to the terms of the
51c5bc425SAlexey Zaytsev * Common Development and Distribution License (the "License").
61c5bc425SAlexey Zaytsev * You may not use this file except in compliance with the License.
71c5bc425SAlexey Zaytsev *
81c5bc425SAlexey Zaytsev * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
91c5bc425SAlexey Zaytsev * or http://www.opensolaris.org/os/licensing.
101c5bc425SAlexey Zaytsev * See the License for the specific language governing permissions
111c5bc425SAlexey Zaytsev * and limitations under the License.
121c5bc425SAlexey Zaytsev *
131c5bc425SAlexey Zaytsev * When distributing Covered Code, include this CDDL HEADER in each
141c5bc425SAlexey Zaytsev * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
151c5bc425SAlexey Zaytsev * If applicable, add the following below this CDDL HEADER, with the
161c5bc425SAlexey Zaytsev * fields enclosed by brackets "[]" replaced with your own identifying
171c5bc425SAlexey Zaytsev * information: Portions Copyright [yyyy] [name of copyright owner]
181c5bc425SAlexey Zaytsev *
191c5bc425SAlexey Zaytsev * CDDL HEADER END
201c5bc425SAlexey Zaytsev */
211c5bc425SAlexey Zaytsev
221c5bc425SAlexey Zaytsev /*
23510a6847SHans Rosenfeld * Copyright (c) 2015, Nexenta Systems, Inc. All rights reserved.
241c5bc425SAlexey Zaytsev * Copyright (c) 2012, Alexey Zaytsev <alexey.zaytsev@gmail.com>
251a5ae140SJason King * Copyright 2020 Joyent Inc.
264d95620bSPaul Winder * Copyright 2019 Western Digital Corporation.
273decf168SPatrick Mooney * Copyright 2020 Oxide Computer Company
28501bc5c0SAndy Fiddaman * Copyright 2022 OmniOS Community Edition (OmniOSce) Association.
291c5bc425SAlexey Zaytsev */
301c5bc425SAlexey Zaytsev
31f8296c60SJoshua M. Clulow /*
32f8296c60SJoshua M. Clulow * VIRTIO BLOCK DRIVER
33f8296c60SJoshua M. Clulow *
34f8296c60SJoshua M. Clulow * This driver provides support for Virtio Block devices. Each driver instance
35f8296c60SJoshua M. Clulow * attaches to a single underlying block device.
36f8296c60SJoshua M. Clulow *
37f8296c60SJoshua M. Clulow * REQUEST CHAIN LAYOUT
38f8296c60SJoshua M. Clulow *
39f8296c60SJoshua M. Clulow * Every request chain sent to the I/O queue has the following structure. Each
40f8296c60SJoshua M. Clulow * box in the diagram represents a descriptor entry (i.e., a DMA cookie) within
41f8296c60SJoshua M. Clulow * the chain:
42f8296c60SJoshua M. Clulow *
43f8296c60SJoshua M. Clulow * +-0-----------------------------------------+
44f8296c60SJoshua M. Clulow * | struct virtio_blk_hdr |-----------------------\
45f8296c60SJoshua M. Clulow * | (written by driver, read by device) | |
46f8296c60SJoshua M. Clulow * +-1-----------------------------------------+ |
47f8296c60SJoshua M. Clulow * | optional data payload |--\ |
48f8296c60SJoshua M. Clulow * | (written by driver for write requests, | | |
49f8296c60SJoshua M. Clulow * | or by device for read requests) | | |
50f8296c60SJoshua M. Clulow * +-2-----------------------------------------+ | |
51f8296c60SJoshua M. Clulow * | ,~` : |-cookies loaned |
52f8296c60SJoshua M. Clulow * |/ : ,~`| | from blkdev |
53f8296c60SJoshua M. Clulow * : / | | |
54f8296c60SJoshua M. Clulow * +-(N - 1)-----------------------------------+ | |
55f8296c60SJoshua M. Clulow * | ... end of data payload. | | |
56f8296c60SJoshua M. Clulow * | | | |
57f8296c60SJoshua M. Clulow * | |--/ |
58f8296c60SJoshua M. Clulow * +-N-----------------------------------------+ |
59f8296c60SJoshua M. Clulow * | status byte | |
60f8296c60SJoshua M. Clulow * | (written by device, read by driver) |--------------------\ |
61f8296c60SJoshua M. Clulow * +-------------------------------------------+ | |
62f8296c60SJoshua M. Clulow * | |
63f8296c60SJoshua M. Clulow * The memory for the header and status bytes (i.e., 0 and N above) | |
64f8296c60SJoshua M. Clulow * is allocated as a single chunk by vioblk_alloc_reqs(): | |
65f8296c60SJoshua M. Clulow * | |
66f8296c60SJoshua M. Clulow * +-------------------------------------------+ | |
67f8296c60SJoshua M. Clulow * | struct virtio_blk_hdr |<----------------------/
68f8296c60SJoshua M. Clulow * +-------------------------------------------+ |
69f8296c60SJoshua M. Clulow * | status byte |<-------------------/
70f8296c60SJoshua M. Clulow * +-------------------------------------------+
71f8296c60SJoshua M. Clulow */
721c5bc425SAlexey Zaytsev
731c5bc425SAlexey Zaytsev #include <sys/modctl.h>
741c5bc425SAlexey Zaytsev #include <sys/blkdev.h>
751c5bc425SAlexey Zaytsev #include <sys/types.h>
761c5bc425SAlexey Zaytsev #include <sys/errno.h>
771c5bc425SAlexey Zaytsev #include <sys/param.h>
781c5bc425SAlexey Zaytsev #include <sys/stropts.h>
791c5bc425SAlexey Zaytsev #include <sys/stream.h>
801c5bc425SAlexey Zaytsev #include <sys/strsubr.h>
811c5bc425SAlexey Zaytsev #include <sys/kmem.h>
821c5bc425SAlexey Zaytsev #include <sys/conf.h>
831c5bc425SAlexey Zaytsev #include <sys/devops.h>
841c5bc425SAlexey Zaytsev #include <sys/ksynch.h>
851c5bc425SAlexey Zaytsev #include <sys/stat.h>
861c5bc425SAlexey Zaytsev #include <sys/modctl.h>
871c5bc425SAlexey Zaytsev #include <sys/debug.h>
881c5bc425SAlexey Zaytsev #include <sys/pci.h>
8994c3dad2SToomas Soome #include <sys/containerof.h>
90f8296c60SJoshua M. Clulow #include <sys/ctype.h>
91f8296c60SJoshua M. Clulow #include <sys/sysmacros.h>
921a5ae140SJason King #include <sys/dkioc_free_util.h>
931c5bc425SAlexey Zaytsev
94f8296c60SJoshua M. Clulow #include "virtio.h"
95f8296c60SJoshua M. Clulow #include "vioblk.h"
961c5bc425SAlexey Zaytsev
97f8296c60SJoshua M. Clulow static void vioblk_get_id(vioblk_t *);
98501bc5c0SAndy Fiddaman static uint_t vioblk_int_handler(caddr_t, caddr_t);
99f8296c60SJoshua M. Clulow static uint_t vioblk_poll(vioblk_t *);
100f8296c60SJoshua M. Clulow static int vioblk_quiesce(dev_info_t *);
101501bc5c0SAndy Fiddaman static int vioblk_read_capacity(vioblk_t *);
102f8296c60SJoshua M. Clulow static int vioblk_attach(dev_info_t *, ddi_attach_cmd_t);
103f8296c60SJoshua M. Clulow static int vioblk_detach(dev_info_t *, ddi_detach_cmd_t);
1041c5bc425SAlexey Zaytsev
105f8296c60SJoshua M. Clulow
106f8296c60SJoshua M. Clulow static struct dev_ops vioblk_dev_ops = {
107f8296c60SJoshua M. Clulow .devo_rev = DEVO_REV,
108f8296c60SJoshua M. Clulow .devo_refcnt = 0,
109f8296c60SJoshua M. Clulow
110f8296c60SJoshua M. Clulow .devo_attach = vioblk_attach,
111f8296c60SJoshua M. Clulow .devo_detach = vioblk_detach,
112f8296c60SJoshua M. Clulow .devo_quiesce = vioblk_quiesce,
113f8296c60SJoshua M. Clulow
114f8296c60SJoshua M. Clulow .devo_getinfo = ddi_no_info,
115f8296c60SJoshua M. Clulow .devo_identify = nulldev,
116f8296c60SJoshua M. Clulow .devo_probe = nulldev,
117f8296c60SJoshua M. Clulow .devo_reset = nodev,
118f8296c60SJoshua M. Clulow .devo_cb_ops = NULL,
119f8296c60SJoshua M. Clulow .devo_bus_ops = NULL,
120f8296c60SJoshua M. Clulow .devo_power = NULL,
1211c5bc425SAlexey Zaytsev };
1221c5bc425SAlexey Zaytsev
123f8296c60SJoshua M. Clulow static struct modldrv vioblk_modldrv = {
124f8296c60SJoshua M. Clulow .drv_modops = &mod_driverops,
125f8296c60SJoshua M. Clulow .drv_linkinfo = "VIRTIO block driver",
126f8296c60SJoshua M. Clulow .drv_dev_ops = &vioblk_dev_ops
1271c5bc425SAlexey Zaytsev };
1281c5bc425SAlexey Zaytsev
129f8296c60SJoshua M. Clulow static struct modlinkage vioblk_modlinkage = {
130f8296c60SJoshua M. Clulow .ml_rev = MODREV_1,
131f8296c60SJoshua M. Clulow .ml_linkage = { &vioblk_modldrv, NULL }
1321c5bc425SAlexey Zaytsev };
1331c5bc425SAlexey Zaytsev
134f8296c60SJoshua M. Clulow /*
135f8296c60SJoshua M. Clulow * DMA attribute template for header and status blocks. We also make a
136f8296c60SJoshua M. Clulow * per-instance copy of this template with negotiated sizes from the device for
137f8296c60SJoshua M. Clulow * blkdev.
138f8296c60SJoshua M. Clulow */
139f8296c60SJoshua M. Clulow static const ddi_dma_attr_t vioblk_dma_attr = {
140f8296c60SJoshua M. Clulow .dma_attr_version = DMA_ATTR_V0,
141f8296c60SJoshua M. Clulow .dma_attr_addr_lo = 0x0000000000000000,
142f8296c60SJoshua M. Clulow .dma_attr_addr_hi = 0xFFFFFFFFFFFFFFFF,
143f8296c60SJoshua M. Clulow .dma_attr_count_max = 0x00000000FFFFFFFF,
144f8296c60SJoshua M. Clulow .dma_attr_align = 1,
145f8296c60SJoshua M. Clulow .dma_attr_burstsizes = 1,
146f8296c60SJoshua M. Clulow .dma_attr_minxfer = 1,
147f8296c60SJoshua M. Clulow .dma_attr_maxxfer = 0x00000000FFFFFFFF,
148f8296c60SJoshua M. Clulow .dma_attr_seg = 0x00000000FFFFFFFF,
149f8296c60SJoshua M. Clulow .dma_attr_sgllen = 1,
150f8296c60SJoshua M. Clulow .dma_attr_granular = 1,
151f8296c60SJoshua M. Clulow .dma_attr_flags = 0
1521c5bc425SAlexey Zaytsev };
1531c5bc425SAlexey Zaytsev
154f8296c60SJoshua M. Clulow static vioblk_req_t *
vioblk_req_alloc(vioblk_t * vib)155f8296c60SJoshua M. Clulow vioblk_req_alloc(vioblk_t *vib)
156f8296c60SJoshua M. Clulow {
157f8296c60SJoshua M. Clulow vioblk_req_t *vbr;
1581c5bc425SAlexey Zaytsev
159f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex));
1601c5bc425SAlexey Zaytsev
161f8296c60SJoshua M. Clulow if ((vbr = list_remove_head(&vib->vib_reqs)) == NULL) {
162f8296c60SJoshua M. Clulow return (NULL);
163f8296c60SJoshua M. Clulow }
164f8296c60SJoshua M. Clulow vib->vib_nreqs_alloc++;
1651c5bc425SAlexey Zaytsev
166f8296c60SJoshua M. Clulow VERIFY0(vbr->vbr_status);
167f8296c60SJoshua M. Clulow vbr->vbr_status |= VIOBLK_REQSTAT_ALLOCATED;
1681c5bc425SAlexey Zaytsev
1693decf168SPatrick Mooney VERIFY3P(vbr->vbr_chain, !=, NULL);
170f8296c60SJoshua M. Clulow VERIFY3P(vbr->vbr_xfer, ==, NULL);
171f8296c60SJoshua M. Clulow VERIFY3S(vbr->vbr_error, ==, 0);
1721c5bc425SAlexey Zaytsev
173f8296c60SJoshua M. Clulow return (vbr);
174f8296c60SJoshua M. Clulow }
1751c5bc425SAlexey Zaytsev
176f8296c60SJoshua M. Clulow static void
vioblk_req_free(vioblk_t * vib,vioblk_req_t * vbr)177f8296c60SJoshua M. Clulow vioblk_req_free(vioblk_t *vib, vioblk_req_t *vbr)
178f8296c60SJoshua M. Clulow {
179f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex));
1801c5bc425SAlexey Zaytsev
181f8296c60SJoshua M. Clulow /*
182f8296c60SJoshua M. Clulow * Check that this request was allocated, then zero the status field to
183f8296c60SJoshua M. Clulow * clear all status bits.
184f8296c60SJoshua M. Clulow */
185f8296c60SJoshua M. Clulow VERIFY(vbr->vbr_status & VIOBLK_REQSTAT_ALLOCATED);
186f8296c60SJoshua M. Clulow vbr->vbr_status = 0;
1871c5bc425SAlexey Zaytsev
188f8296c60SJoshua M. Clulow vbr->vbr_xfer = NULL;
189f8296c60SJoshua M. Clulow vbr->vbr_error = 0;
190f8296c60SJoshua M. Clulow vbr->vbr_type = 0;
1913decf168SPatrick Mooney virtio_chain_clear(vbr->vbr_chain);
1921c5bc425SAlexey Zaytsev
193f8296c60SJoshua M. Clulow list_insert_head(&vib->vib_reqs, vbr);
194f8296c60SJoshua M. Clulow
195f8296c60SJoshua M. Clulow VERIFY3U(vib->vib_nreqs_alloc, >, 0);
196f8296c60SJoshua M. Clulow vib->vib_nreqs_alloc--;
197f8296c60SJoshua M. Clulow }
198f8296c60SJoshua M. Clulow
199f8296c60SJoshua M. Clulow static void
vioblk_complete(vioblk_t * vib,vioblk_req_t * vbr)200f8296c60SJoshua M. Clulow vioblk_complete(vioblk_t *vib, vioblk_req_t *vbr)
2011c5bc425SAlexey Zaytsev {
202f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex));
2031c5bc425SAlexey Zaytsev
204f8296c60SJoshua M. Clulow VERIFY(!(vbr->vbr_status & VIOBLK_REQSTAT_COMPLETE));
205f8296c60SJoshua M. Clulow vbr->vbr_status |= VIOBLK_REQSTAT_COMPLETE;
2061c5bc425SAlexey Zaytsev
207f8296c60SJoshua M. Clulow if (vbr->vbr_type == VIRTIO_BLK_T_FLUSH) {
208f8296c60SJoshua M. Clulow vib->vib_stats->vbs_rw_cacheflush.value.ui64++;
2091c5bc425SAlexey Zaytsev }
2101c5bc425SAlexey Zaytsev
211f8296c60SJoshua M. Clulow if (vbr->vbr_xfer != NULL) {
212f8296c60SJoshua M. Clulow /*
213f8296c60SJoshua M. Clulow * This is a blkdev framework request.
214f8296c60SJoshua M. Clulow */
215f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex);
216f8296c60SJoshua M. Clulow bd_xfer_done(vbr->vbr_xfer, vbr->vbr_error);
217f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex);
218f8296c60SJoshua M. Clulow vbr->vbr_xfer = NULL;
2191c5bc425SAlexey Zaytsev }
220f8296c60SJoshua M. Clulow }
2211c5bc425SAlexey Zaytsev
2223decf168SPatrick Mooney static vioblk_req_t *
vioblk_common_start(vioblk_t * vib,int type,uint64_t sector,boolean_t polled)223f8296c60SJoshua M. Clulow vioblk_common_start(vioblk_t *vib, int type, uint64_t sector,
224f8296c60SJoshua M. Clulow boolean_t polled)
225f8296c60SJoshua M. Clulow {
226f8296c60SJoshua M. Clulow vioblk_req_t *vbr = NULL;
227f8296c60SJoshua M. Clulow
228f8296c60SJoshua M. Clulow if ((vbr = vioblk_req_alloc(vib)) == NULL) {
229f8296c60SJoshua M. Clulow vib->vib_stats->vbs_rw_outofmemory.value.ui64++;
230f8296c60SJoshua M. Clulow return (NULL);
231f8296c60SJoshua M. Clulow }
232f8296c60SJoshua M. Clulow vbr->vbr_type = type;
233f8296c60SJoshua M. Clulow
234f8296c60SJoshua M. Clulow if (polled) {
235f8296c60SJoshua M. Clulow /*
236f8296c60SJoshua M. Clulow * Mark this command as polled so that we can wait on it
237f8296c60SJoshua M. Clulow * ourselves.
238f8296c60SJoshua M. Clulow */
239f8296c60SJoshua M. Clulow vbr->vbr_status |= VIOBLK_REQSTAT_POLLED;
2401c5bc425SAlexey Zaytsev }
2411c5bc425SAlexey Zaytsev
242f8296c60SJoshua M. Clulow struct vioblk_req_hdr vbh;
243f8296c60SJoshua M. Clulow vbh.vbh_type = type;
244f8296c60SJoshua M. Clulow vbh.vbh_ioprio = 0;
245c5c712a8SToomas Soome vbh.vbh_sector = (sector * vib->vib_blk_size) / DEV_BSIZE;
246f8296c60SJoshua M. Clulow bcopy(&vbh, virtio_dma_va(vbr->vbr_dma, 0), sizeof (vbh));
2471c5bc425SAlexey Zaytsev
248f8296c60SJoshua M. Clulow /*
249f8296c60SJoshua M. Clulow * Put the header in the first descriptor. See the block comment at
250f8296c60SJoshua M. Clulow * the top of the file for more details on the chain layout.
251f8296c60SJoshua M. Clulow */
2523decf168SPatrick Mooney if (virtio_chain_append(vbr->vbr_chain,
2533decf168SPatrick Mooney virtio_dma_cookie_pa(vbr->vbr_dma, 0),
254f8296c60SJoshua M. Clulow sizeof (struct vioblk_req_hdr), VIRTIO_DIR_DEVICE_READS) !=
255f8296c60SJoshua M. Clulow DDI_SUCCESS) {
2563decf168SPatrick Mooney vioblk_req_free(vib, vbr);
2573decf168SPatrick Mooney return (NULL);
258f8296c60SJoshua M. Clulow }
259f8296c60SJoshua M. Clulow
2603decf168SPatrick Mooney return (vbr);
2611c5bc425SAlexey Zaytsev }
2621c5bc425SAlexey Zaytsev
2631c5bc425SAlexey Zaytsev static int
vioblk_common_submit(vioblk_t * vib,vioblk_req_t * vbr)2643decf168SPatrick Mooney vioblk_common_submit(vioblk_t *vib, vioblk_req_t *vbr)
2651c5bc425SAlexey Zaytsev {
2663decf168SPatrick Mooney virtio_chain_t *vic = vbr->vbr_chain;
267f8296c60SJoshua M. Clulow int r;
2681c5bc425SAlexey Zaytsev
269f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex));
2701c5bc425SAlexey Zaytsev
271f8296c60SJoshua M. Clulow /*
272f8296c60SJoshua M. Clulow * The device will write the status byte into this last descriptor.
273f8296c60SJoshua M. Clulow * See the block comment at the top of the file for more details on the
274f8296c60SJoshua M. Clulow * chain layout.
275f8296c60SJoshua M. Clulow */
276f8296c60SJoshua M. Clulow if (virtio_chain_append(vic, virtio_dma_cookie_pa(vbr->vbr_dma, 0) +
277f8296c60SJoshua M. Clulow sizeof (struct vioblk_req_hdr), sizeof (uint8_t),
278f8296c60SJoshua M. Clulow VIRTIO_DIR_DEVICE_WRITES) != DDI_SUCCESS) {
2793decf168SPatrick Mooney vioblk_req_free(vib, vbr);
2803decf168SPatrick Mooney return (ENOMEM);
2811c5bc425SAlexey Zaytsev }
2821c5bc425SAlexey Zaytsev
283f8296c60SJoshua M. Clulow virtio_dma_sync(vbr->vbr_dma, DDI_DMA_SYNC_FORDEV);
284f8296c60SJoshua M. Clulow virtio_chain_submit(vic, B_TRUE);
285f8296c60SJoshua M. Clulow
286f8296c60SJoshua M. Clulow if (!(vbr->vbr_status & VIOBLK_REQSTAT_POLLED)) {
287f8296c60SJoshua M. Clulow /*
288f8296c60SJoshua M. Clulow * This is not a polled request. Our request will be freed and
289f8296c60SJoshua M. Clulow * the caller notified later in vioblk_poll().
290f8296c60SJoshua M. Clulow */
291f8296c60SJoshua M. Clulow return (0);
292f8296c60SJoshua M. Clulow }
2931c5bc425SAlexey Zaytsev
294f8296c60SJoshua M. Clulow /*
295f8296c60SJoshua M. Clulow * This is a polled request. We need to block here and wait for the
296f8296c60SJoshua M. Clulow * device to complete request processing.
297f8296c60SJoshua M. Clulow */
298f8296c60SJoshua M. Clulow while (!(vbr->vbr_status & VIOBLK_REQSTAT_POLL_COMPLETE)) {
299f8296c60SJoshua M. Clulow if (ddi_in_panic()) {
300f8296c60SJoshua M. Clulow /*
301f8296c60SJoshua M. Clulow * When panicking, interrupts are disabled. We must
302f8296c60SJoshua M. Clulow * poll the queue manually.
303f8296c60SJoshua M. Clulow */
3041c5bc425SAlexey Zaytsev drv_usecwait(10);
305f8296c60SJoshua M. Clulow (void) vioblk_poll(vib);
306f8296c60SJoshua M. Clulow continue;
3071c5bc425SAlexey Zaytsev }
308f8296c60SJoshua M. Clulow
309f8296c60SJoshua M. Clulow /*
310f8296c60SJoshua M. Clulow * When not panicking, the device will interrupt on command
311f8296c60SJoshua M. Clulow * completion and vioblk_poll() will be called to wake us up.
312f8296c60SJoshua M. Clulow */
313f8296c60SJoshua M. Clulow cv_wait(&vib->vib_cv, &vib->vib_mutex);
3141c5bc425SAlexey Zaytsev }
3151c5bc425SAlexey Zaytsev
316f8296c60SJoshua M. Clulow vioblk_complete(vib, vbr);
317f8296c60SJoshua M. Clulow r = vbr->vbr_error;
318f8296c60SJoshua M. Clulow vioblk_req_free(vib, vbr);
319f8296c60SJoshua M. Clulow return (r);
3201c5bc425SAlexey Zaytsev }
3211c5bc425SAlexey Zaytsev
3221c5bc425SAlexey Zaytsev static int
vioblk_internal(vioblk_t * vib,int type,virtio_dma_t * dma,uint64_t sector,virtio_direction_t dir)323f8296c60SJoshua M. Clulow vioblk_internal(vioblk_t *vib, int type, virtio_dma_t *dma,
324f8296c60SJoshua M. Clulow uint64_t sector, virtio_direction_t dir)
3251c5bc425SAlexey Zaytsev {
326f8296c60SJoshua M. Clulow vioblk_req_t *vbr;
3271c5bc425SAlexey Zaytsev
328f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex));
3291c5bc425SAlexey Zaytsev
330f8296c60SJoshua M. Clulow /*
331f8296c60SJoshua M. Clulow * Allocate a polled request.
332f8296c60SJoshua M. Clulow */
3333decf168SPatrick Mooney if ((vbr = vioblk_common_start(vib, type, sector, B_TRUE)) == NULL) {
334f8296c60SJoshua M. Clulow return (ENOMEM);
335f8296c60SJoshua M. Clulow }
3361c5bc425SAlexey Zaytsev
337f8296c60SJoshua M. Clulow /*
338f8296c60SJoshua M. Clulow * If there is a request payload, it goes between the header and the
339f8296c60SJoshua M. Clulow * status byte. See the block comment at the top of the file for more
340f8296c60SJoshua M. Clulow * detail on the chain layout.
341f8296c60SJoshua M. Clulow */
342f8296c60SJoshua M. Clulow if (dma != NULL) {
3433decf168SPatrick Mooney virtio_chain_t *vic = vbr->vbr_chain;
344f8296c60SJoshua M. Clulow for (uint_t n = 0; n < virtio_dma_ncookies(dma); n++) {
345f8296c60SJoshua M. Clulow if (virtio_chain_append(vic,
346f8296c60SJoshua M. Clulow virtio_dma_cookie_pa(dma, n),
347f8296c60SJoshua M. Clulow virtio_dma_cookie_size(dma, n), dir) !=
348f8296c60SJoshua M. Clulow DDI_SUCCESS) {
3493decf168SPatrick Mooney vioblk_req_free(vib, vbr);
3503decf168SPatrick Mooney return (ENOMEM);
351f8296c60SJoshua M. Clulow }
352f8296c60SJoshua M. Clulow }
3531c5bc425SAlexey Zaytsev }
3541c5bc425SAlexey Zaytsev
3553decf168SPatrick Mooney return (vioblk_common_submit(vib, vbr));
3561c5bc425SAlexey Zaytsev }
3571c5bc425SAlexey Zaytsev
3581a5ae140SJason King static int
vioblk_map_discard(vioblk_t * vib,virtio_chain_t * vic,const bd_xfer_t * xfer)3591a5ae140SJason King vioblk_map_discard(vioblk_t *vib, virtio_chain_t *vic, const bd_xfer_t *xfer)
3601a5ae140SJason King {
3611a5ae140SJason King const dkioc_free_list_t *dfl = xfer->x_dfl;
3621a5ae140SJason King const dkioc_free_list_ext_t *exts = dfl->dfl_exts;
3631a5ae140SJason King virtio_dma_t *dma = NULL;
3641a5ae140SJason King struct vioblk_discard_write_zeroes *wzp = NULL;
3651a5ae140SJason King
3661a5ae140SJason King dma = virtio_dma_alloc(vib->vib_virtio,
3671a5ae140SJason King dfl->dfl_num_exts * sizeof (*wzp), &vioblk_dma_attr,
3681a5ae140SJason King DDI_DMA_CONSISTENT | DDI_DMA_WRITE, KM_SLEEP);
3691a5ae140SJason King if (dma == NULL)
3701a5ae140SJason King return (ENOMEM);
3711a5ae140SJason King
3721a5ae140SJason King wzp = virtio_dma_va(dma, 0);
3731a5ae140SJason King
3741a5ae140SJason King for (uint64_t i = 0; i < dfl->dfl_num_exts; i++, exts++, wzp++) {
3751a5ae140SJason King uint64_t start = dfl->dfl_offset + exts->dfle_start;
3761a5ae140SJason King
3771a5ae140SJason King const struct vioblk_discard_write_zeroes vdwz = {
3781a5ae140SJason King .vdwz_sector = start >> DEV_BSHIFT,
3791a5ae140SJason King .vdwz_num_sectors = exts->dfle_length >> DEV_BSHIFT,
3801a5ae140SJason King .vdwz_flags = 0
3811a5ae140SJason King };
3821a5ae140SJason King
3831a5ae140SJason King bcopy(&vdwz, wzp, sizeof (*wzp));
3841a5ae140SJason King }
3851a5ae140SJason King
3861a5ae140SJason King if (virtio_chain_append(vic,
3871a5ae140SJason King virtio_dma_cookie_pa(dma, 0),
3881a5ae140SJason King virtio_dma_cookie_size(dma, 0),
3891a5ae140SJason King VIRTIO_DIR_DEVICE_READS) != DDI_SUCCESS) {
3901a5ae140SJason King virtio_dma_free(dma);
3911a5ae140SJason King return (ENOMEM);
3921a5ae140SJason King }
3931a5ae140SJason King
3941a5ae140SJason King return (0);
3951a5ae140SJason King }
3961a5ae140SJason King
3971c5bc425SAlexey Zaytsev static int
vioblk_request(vioblk_t * vib,bd_xfer_t * xfer,int type)398f8296c60SJoshua M. Clulow vioblk_request(vioblk_t *vib, bd_xfer_t *xfer, int type)
3991c5bc425SAlexey Zaytsev {
400f8296c60SJoshua M. Clulow vioblk_req_t *vbr = NULL;
401f8296c60SJoshua M. Clulow uint_t total_cookies = 2;
402f8296c60SJoshua M. Clulow boolean_t polled = (xfer->x_flags & BD_XFER_POLL) != 0;
4031c5bc425SAlexey Zaytsev
404f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex));
405f8296c60SJoshua M. Clulow
406f8296c60SJoshua M. Clulow /*
407f8296c60SJoshua M. Clulow * Ensure that this request falls within the advertised size of the
408f8296c60SJoshua M. Clulow * block device. Be careful to avoid overflow.
409f8296c60SJoshua M. Clulow */
410f8296c60SJoshua M. Clulow if (xfer->x_nblks > SIZE_MAX - xfer->x_blkno ||
411f8296c60SJoshua M. Clulow (xfer->x_blkno + xfer->x_nblks) > vib->vib_nblks) {
412f8296c60SJoshua M. Clulow vib->vib_stats->vbs_rw_badoffset.value.ui64++;
413f8296c60SJoshua M. Clulow return (EINVAL);
414f8296c60SJoshua M. Clulow }
4151c5bc425SAlexey Zaytsev
4163decf168SPatrick Mooney if ((vbr = vioblk_common_start(vib, type, xfer->x_blkno, polled)) ==
417f8296c60SJoshua M. Clulow NULL) {
418f8296c60SJoshua M. Clulow return (ENOMEM);
419f8296c60SJoshua M. Clulow }
420f8296c60SJoshua M. Clulow vbr->vbr_xfer = xfer;
421f8296c60SJoshua M. Clulow
422f8296c60SJoshua M. Clulow /*
423f8296c60SJoshua M. Clulow * If there is a request payload, it goes between the header and the
424f8296c60SJoshua M. Clulow * status byte. See the block comment at the top of the file for more
425f8296c60SJoshua M. Clulow * detail on the chain layout.
426f8296c60SJoshua M. Clulow */
427f8296c60SJoshua M. Clulow if ((type == VIRTIO_BLK_T_IN || type == VIRTIO_BLK_T_OUT) &&
428f8296c60SJoshua M. Clulow xfer->x_nblks > 0) {
429f8296c60SJoshua M. Clulow virtio_direction_t dir = (type == VIRTIO_BLK_T_OUT) ?
430f8296c60SJoshua M. Clulow VIRTIO_DIR_DEVICE_READS : VIRTIO_DIR_DEVICE_WRITES;
4313decf168SPatrick Mooney virtio_chain_t *vic = vbr->vbr_chain;
432f8296c60SJoshua M. Clulow
433f8296c60SJoshua M. Clulow for (uint_t n = 0; n < xfer->x_ndmac; n++) {
434f8296c60SJoshua M. Clulow ddi_dma_cookie_t dmac;
435f8296c60SJoshua M. Clulow
436f8296c60SJoshua M. Clulow if (n == 0) {
437f8296c60SJoshua M. Clulow /*
438f8296c60SJoshua M. Clulow * The first cookie is in the blkdev request.
439f8296c60SJoshua M. Clulow */
440f8296c60SJoshua M. Clulow dmac = xfer->x_dmac;
441f8296c60SJoshua M. Clulow } else {
442f8296c60SJoshua M. Clulow ddi_dma_nextcookie(xfer->x_dmah, &dmac);
443f8296c60SJoshua M. Clulow }
444f8296c60SJoshua M. Clulow
445f8296c60SJoshua M. Clulow if (virtio_chain_append(vic, dmac.dmac_laddress,
446f8296c60SJoshua M. Clulow dmac.dmac_size, dir) != DDI_SUCCESS) {
4473decf168SPatrick Mooney vioblk_req_free(vib, vbr);
4483decf168SPatrick Mooney return (ENOMEM);
449f8296c60SJoshua M. Clulow }
4501c5bc425SAlexey Zaytsev }
4511c5bc425SAlexey Zaytsev
452f8296c60SJoshua M. Clulow total_cookies += xfer->x_ndmac;
453f8296c60SJoshua M. Clulow
454f8296c60SJoshua M. Clulow } else if (xfer->x_nblks > 0) {
455f8296c60SJoshua M. Clulow dev_err(vib->vib_dip, CE_PANIC,
456f8296c60SJoshua M. Clulow "request of type %d had payload length of %lu blocks", type,
457f8296c60SJoshua M. Clulow xfer->x_nblks);
4581a5ae140SJason King } else if (type == VIRTIO_BLK_T_DISCARD) {
4593decf168SPatrick Mooney int r = vioblk_map_discard(vib, vbr->vbr_chain, xfer);
4601a5ae140SJason King if (r != 0) {
4613decf168SPatrick Mooney vioblk_req_free(vib, vbr);
4623decf168SPatrick Mooney return (r);
4631a5ae140SJason King }
464f8296c60SJoshua M. Clulow }
465f8296c60SJoshua M. Clulow
466f8296c60SJoshua M. Clulow if (vib->vib_stats->vbs_rw_cookiesmax.value.ui32 < total_cookies) {
467f8296c60SJoshua M. Clulow vib->vib_stats->vbs_rw_cookiesmax.value.ui32 = total_cookies;
4681c5bc425SAlexey Zaytsev }
469f8296c60SJoshua M. Clulow
4703decf168SPatrick Mooney return (vioblk_common_submit(vib, vbr));
4711c5bc425SAlexey Zaytsev }
4721c5bc425SAlexey Zaytsev
4731c5bc425SAlexey Zaytsev static int
vioblk_bd_read(void * arg,bd_xfer_t * xfer)474f8296c60SJoshua M. Clulow vioblk_bd_read(void *arg, bd_xfer_t *xfer)
4751c5bc425SAlexey Zaytsev {
476f8296c60SJoshua M. Clulow vioblk_t *vib = arg;
477f8296c60SJoshua M. Clulow int r;
478f8296c60SJoshua M. Clulow
479f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex);
480f8296c60SJoshua M. Clulow r = vioblk_request(vib, xfer, VIRTIO_BLK_T_IN);
481f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex);
4821c5bc425SAlexey Zaytsev
483f8296c60SJoshua M. Clulow return (r);
484f8296c60SJoshua M. Clulow }
4851c5bc425SAlexey Zaytsev
486f8296c60SJoshua M. Clulow static int
vioblk_bd_write(void * arg,bd_xfer_t * xfer)487f8296c60SJoshua M. Clulow vioblk_bd_write(void *arg, bd_xfer_t *xfer)
488f8296c60SJoshua M. Clulow {
489f8296c60SJoshua M. Clulow vioblk_t *vib = arg;
490f8296c60SJoshua M. Clulow int r;
4911c5bc425SAlexey Zaytsev
492f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex);
493f8296c60SJoshua M. Clulow r = vioblk_request(vib, xfer, VIRTIO_BLK_T_OUT);
494f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex);
4951c5bc425SAlexey Zaytsev
496f8296c60SJoshua M. Clulow return (r);
4971c5bc425SAlexey Zaytsev }
4981c5bc425SAlexey Zaytsev
499f8296c60SJoshua M. Clulow static int
vioblk_bd_flush(void * arg,bd_xfer_t * xfer)500f8296c60SJoshua M. Clulow vioblk_bd_flush(void *arg, bd_xfer_t *xfer)
501f8296c60SJoshua M. Clulow {
502f8296c60SJoshua M. Clulow vioblk_t *vib = arg;
503f8296c60SJoshua M. Clulow int r;
504f8296c60SJoshua M. Clulow
505f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex);
506f8296c60SJoshua M. Clulow if (!virtio_feature_present(vib->vib_virtio, VIRTIO_BLK_F_FLUSH)) {
507f8296c60SJoshua M. Clulow /*
508f8296c60SJoshua M. Clulow * We don't really expect to get here, because if we did not
509f8296c60SJoshua M. Clulow * negotiate the flush feature we would not have installed this
510f8296c60SJoshua M. Clulow * function in the blkdev ops vector.
511f8296c60SJoshua M. Clulow */
512f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex);
513f8296c60SJoshua M. Clulow return (ENOTSUP);
514f8296c60SJoshua M. Clulow }
515f8296c60SJoshua M. Clulow
516f8296c60SJoshua M. Clulow r = vioblk_request(vib, xfer, VIRTIO_BLK_T_FLUSH);
517f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex);
518f8296c60SJoshua M. Clulow
519f8296c60SJoshua M. Clulow return (r);
520f8296c60SJoshua M. Clulow }
5211c5bc425SAlexey Zaytsev
5221c5bc425SAlexey Zaytsev static void
vioblk_bd_driveinfo(void * arg,bd_drive_t * drive)523f8296c60SJoshua M. Clulow vioblk_bd_driveinfo(void *arg, bd_drive_t *drive)
5241c5bc425SAlexey Zaytsev {
525f8296c60SJoshua M. Clulow vioblk_t *vib = arg;
5261c5bc425SAlexey Zaytsev
527f8296c60SJoshua M. Clulow drive->d_qsize = vib->vib_reqs_capacity;
5281c5bc425SAlexey Zaytsev drive->d_removable = B_FALSE;
5291c5bc425SAlexey Zaytsev drive->d_hotpluggable = B_TRUE;
5301c5bc425SAlexey Zaytsev drive->d_target = 0;
5311c5bc425SAlexey Zaytsev drive->d_lun = 0;
532510a6847SHans Rosenfeld
533510a6847SHans Rosenfeld drive->d_vendor = "Virtio";
534510a6847SHans Rosenfeld drive->d_vendor_len = strlen(drive->d_vendor);
535510a6847SHans Rosenfeld
536510a6847SHans Rosenfeld drive->d_product = "Block Device";
537510a6847SHans Rosenfeld drive->d_product_len = strlen(drive->d_product);
538510a6847SHans Rosenfeld
539f8296c60SJoshua M. Clulow drive->d_serial = vib->vib_devid;
540510a6847SHans Rosenfeld drive->d_serial_len = strlen(drive->d_serial);
541510a6847SHans Rosenfeld
542510a6847SHans Rosenfeld drive->d_revision = "0000";
543510a6847SHans Rosenfeld drive->d_revision_len = strlen(drive->d_revision);
5441a5ae140SJason King
5451a5ae140SJason King if (vib->vib_can_discard) {
5461a5ae140SJason King drive->d_free_align = vib->vib_discard_sector_align;
5471a5ae140SJason King drive->d_max_free_seg = vib->vib_max_discard_seg;
5481a5ae140SJason King drive->d_max_free_blks = vib->vib_max_discard_sectors;
5491a5ae140SJason King /*
5501a5ae140SJason King * The virtio 1.1 spec doesn't specify a per segment sector
5511a5ae140SJason King * limit for discards -- only a limit on the total sectors in
5521a5ae140SJason King * a discard request. Therefore, we assume a vioblk device must
5531a5ae140SJason King * be able to accept a single segment of vib_max_discard_sectors
5541a5ae140SJason King * (when it supports discard requests) and use
5551a5ae140SJason King * vib_max_discard_sectors both for the overall limit for
5561a5ae140SJason King * a discard request, but also as the limit for a single
5571a5ae140SJason King * segment. blkdev will ensure we are never called with
5581a5ae140SJason King * a dkioc_free_list_t that violates either limit.
5591a5ae140SJason King */
5601a5ae140SJason King drive->d_max_free_seg_blks = vib->vib_max_discard_sectors;
5611a5ae140SJason King }
5621c5bc425SAlexey Zaytsev }
5631c5bc425SAlexey Zaytsev
5641c5bc425SAlexey Zaytsev static int
vioblk_bd_mediainfo(void * arg,bd_media_t * media)565f8296c60SJoshua M. Clulow vioblk_bd_mediainfo(void *arg, bd_media_t *media)
5661c5bc425SAlexey Zaytsev {
567f8296c60SJoshua M. Clulow vioblk_t *vib = (void *)arg;
5681c5bc425SAlexey Zaytsev
569f8296c60SJoshua M. Clulow /*
570f8296c60SJoshua M. Clulow * The device protocol is specified in terms of 512 byte logical
571f8296c60SJoshua M. Clulow * blocks, regardless of the recommended I/O size which might be
572f8296c60SJoshua M. Clulow * larger.
573f8296c60SJoshua M. Clulow */
574f8296c60SJoshua M. Clulow media->m_nblks = vib->vib_nblks;
575c5c712a8SToomas Soome media->m_blksize = vib->vib_blk_size;
576f8296c60SJoshua M. Clulow
577f8296c60SJoshua M. Clulow media->m_readonly = vib->vib_readonly;
578f8296c60SJoshua M. Clulow media->m_pblksize = vib->vib_pblk_size;
5791c5bc425SAlexey Zaytsev return (0);
5801c5bc425SAlexey Zaytsev }
5811c5bc425SAlexey Zaytsev
582f8296c60SJoshua M. Clulow static void
vioblk_get_id(vioblk_t * vib)583f8296c60SJoshua M. Clulow vioblk_get_id(vioblk_t *vib)
5841c5bc425SAlexey Zaytsev {
585f8296c60SJoshua M. Clulow virtio_dma_t *dma;
586f8296c60SJoshua M. Clulow int r;
5871c5bc425SAlexey Zaytsev
588f8296c60SJoshua M. Clulow if ((dma = virtio_dma_alloc(vib->vib_virtio, VIRTIO_BLK_ID_BYTES,
589f8296c60SJoshua M. Clulow &vioblk_dma_attr, DDI_DMA_CONSISTENT | DDI_DMA_READ,
590f8296c60SJoshua M. Clulow KM_SLEEP)) == NULL) {
591f8296c60SJoshua M. Clulow return;
5921c5bc425SAlexey Zaytsev }
5931c5bc425SAlexey Zaytsev
594f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex);
595f8296c60SJoshua M. Clulow if ((r = vioblk_internal(vib, VIRTIO_BLK_T_GET_ID, dma, 0,
596f8296c60SJoshua M. Clulow VIRTIO_DIR_DEVICE_WRITES)) == 0) {
597f8296c60SJoshua M. Clulow const char *b = virtio_dma_va(dma, 0);
598f8296c60SJoshua M. Clulow uint_t pos = 0;
5991c5bc425SAlexey Zaytsev
600f8296c60SJoshua M. Clulow /*
601f8296c60SJoshua M. Clulow * Save the entire response for debugging purposes.
602f8296c60SJoshua M. Clulow */
603f8296c60SJoshua M. Clulow bcopy(virtio_dma_va(dma, 0), vib->vib_rawid,
604f8296c60SJoshua M. Clulow VIRTIO_BLK_ID_BYTES);
605510a6847SHans Rosenfeld
606f8296c60SJoshua M. Clulow /*
607f8296c60SJoshua M. Clulow * Process the returned ID.
608f8296c60SJoshua M. Clulow */
609f8296c60SJoshua M. Clulow bzero(vib->vib_devid, sizeof (vib->vib_devid));
610f8296c60SJoshua M. Clulow for (uint_t n = 0; n < VIRTIO_BLK_ID_BYTES; n++) {
611f8296c60SJoshua M. Clulow if (isalnum(b[n]) || b[n] == '-' || b[n] == '_') {
612f8296c60SJoshua M. Clulow /*
613f8296c60SJoshua M. Clulow * Accept a subset of printable ASCII
614f8296c60SJoshua M. Clulow * characters.
615f8296c60SJoshua M. Clulow */
616f8296c60SJoshua M. Clulow vib->vib_devid[pos++] = b[n];
617f8296c60SJoshua M. Clulow } else {
618f8296c60SJoshua M. Clulow /*
619f8296c60SJoshua M. Clulow * Stop processing at the first sign of
620f8296c60SJoshua M. Clulow * trouble.
621f8296c60SJoshua M. Clulow */
622f8296c60SJoshua M. Clulow break;
623f8296c60SJoshua M. Clulow }
624f8296c60SJoshua M. Clulow }
625510a6847SHans Rosenfeld
626f8296c60SJoshua M. Clulow vib->vib_devid_fetched = B_TRUE;
6271c5bc425SAlexey Zaytsev }
628f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex);
6291c5bc425SAlexey Zaytsev
630f8296c60SJoshua M. Clulow virtio_dma_free(dma);
6311c5bc425SAlexey Zaytsev }
6321c5bc425SAlexey Zaytsev
6331c5bc425SAlexey Zaytsev static int
vioblk_bd_devid(void * arg,dev_info_t * dip,ddi_devid_t * devid)634f8296c60SJoshua M. Clulow vioblk_bd_devid(void *arg, dev_info_t *dip, ddi_devid_t *devid)
6351c5bc425SAlexey Zaytsev {
636f8296c60SJoshua M. Clulow vioblk_t *vib = arg;
637f8296c60SJoshua M. Clulow size_t len;
638f8296c60SJoshua M. Clulow
639f8296c60SJoshua M. Clulow if ((len = strlen(vib->vib_devid)) == 0) {
640f8296c60SJoshua M. Clulow /*
641f8296c60SJoshua M. Clulow * The device has no ID.
642f8296c60SJoshua M. Clulow */
6431c5bc425SAlexey Zaytsev return (DDI_FAILURE);
6441c5bc425SAlexey Zaytsev }
6451c5bc425SAlexey Zaytsev
646f8296c60SJoshua M. Clulow return (ddi_devid_init(dip, DEVID_ATA_SERIAL, len, vib->vib_devid,
647f8296c60SJoshua M. Clulow devid));
6481c5bc425SAlexey Zaytsev }
6491c5bc425SAlexey Zaytsev
6501a5ae140SJason King static int
vioblk_bd_free_space(void * arg,bd_xfer_t * xfer)6511a5ae140SJason King vioblk_bd_free_space(void *arg, bd_xfer_t *xfer)
6521a5ae140SJason King {
6531a5ae140SJason King vioblk_t *vib = arg;
6541a5ae140SJason King int r = 0;
6551a5ae140SJason King
6561a5ae140SJason King /*
6571a5ae140SJason King * Since vib_can_discard is write once (and set during attach),
6581a5ae140SJason King * we can check if it's enabled without taking the mutex.
6591a5ae140SJason King */
6601a5ae140SJason King if (!vib->vib_can_discard) {
6611a5ae140SJason King return (ENOTSUP);
6621a5ae140SJason King }
6631a5ae140SJason King
6641a5ae140SJason King mutex_enter(&vib->vib_mutex);
6651a5ae140SJason King r = vioblk_request(vib, xfer, VIRTIO_BLK_T_DISCARD);
6661a5ae140SJason King mutex_exit(&vib->vib_mutex);
6671a5ae140SJason King
6681a5ae140SJason King return (r);
6691a5ae140SJason King }
6701a5ae140SJason King
671f8296c60SJoshua M. Clulow /*
672f8296c60SJoshua M. Clulow * As the device completes processing of a request, it returns the chain for
673f8296c60SJoshua M. Clulow * that request to our I/O queue. This routine is called in two contexts:
674f8296c60SJoshua M. Clulow * - from the interrupt handler, in response to notification from the device
675f8296c60SJoshua M. Clulow * - synchronously in line with request processing when panicking
676f8296c60SJoshua M. Clulow */
677f8296c60SJoshua M. Clulow static uint_t
vioblk_poll(vioblk_t * vib)678f8296c60SJoshua M. Clulow vioblk_poll(vioblk_t *vib)
6791c5bc425SAlexey Zaytsev {
680f8296c60SJoshua M. Clulow virtio_chain_t *vic;
681f8296c60SJoshua M. Clulow uint_t count = 0;
682f8296c60SJoshua M. Clulow boolean_t wakeup = B_FALSE;
683f8296c60SJoshua M. Clulow
684f8296c60SJoshua M. Clulow VERIFY(MUTEX_HELD(&vib->vib_mutex));
6851c5bc425SAlexey Zaytsev
686f8296c60SJoshua M. Clulow while ((vic = virtio_queue_poll(vib->vib_vq)) != NULL) {
687f8296c60SJoshua M. Clulow vioblk_req_t *vbr = virtio_chain_data(vic);
688f8296c60SJoshua M. Clulow uint8_t status;
6891c5bc425SAlexey Zaytsev
690f8296c60SJoshua M. Clulow virtio_dma_sync(vbr->vbr_dma, DDI_DMA_SYNC_FORCPU);
691f8296c60SJoshua M. Clulow
692f8296c60SJoshua M. Clulow bcopy(virtio_dma_va(vbr->vbr_dma,
693f8296c60SJoshua M. Clulow sizeof (struct vioblk_req_hdr)), &status, sizeof (status));
6941c5bc425SAlexey Zaytsev
6951c5bc425SAlexey Zaytsev switch (status) {
696f8296c60SJoshua M. Clulow case VIRTIO_BLK_S_OK:
697f8296c60SJoshua M. Clulow vbr->vbr_error = 0;
698f8296c60SJoshua M. Clulow break;
699f8296c60SJoshua M. Clulow case VIRTIO_BLK_S_IOERR:
700f8296c60SJoshua M. Clulow vbr->vbr_error = EIO;
701f8296c60SJoshua M. Clulow vib->vib_stats->vbs_io_errors.value.ui64++;
702f8296c60SJoshua M. Clulow break;
703f8296c60SJoshua M. Clulow case VIRTIO_BLK_S_UNSUPP:
704f8296c60SJoshua M. Clulow vbr->vbr_error = ENOTTY;
705f8296c60SJoshua M. Clulow vib->vib_stats->vbs_unsupp_errors.value.ui64++;
706f8296c60SJoshua M. Clulow break;
707f8296c60SJoshua M. Clulow default:
708f8296c60SJoshua M. Clulow vbr->vbr_error = ENXIO;
709f8296c60SJoshua M. Clulow vib->vib_stats->vbs_nxio_errors.value.ui64++;
710f8296c60SJoshua M. Clulow break;
711f8296c60SJoshua M. Clulow }
712f8296c60SJoshua M. Clulow
713f8296c60SJoshua M. Clulow count++;
714f8296c60SJoshua M. Clulow
715f8296c60SJoshua M. Clulow if (vbr->vbr_status & VIOBLK_REQSTAT_POLLED) {
716f8296c60SJoshua M. Clulow /*
717f8296c60SJoshua M. Clulow * This request must not be freed as it is being held
718f8296c60SJoshua M. Clulow * by a call to vioblk_common_submit().
719f8296c60SJoshua M. Clulow */
720f8296c60SJoshua M. Clulow VERIFY(!(vbr->vbr_status &
721f8296c60SJoshua M. Clulow VIOBLK_REQSTAT_POLL_COMPLETE));
722f8296c60SJoshua M. Clulow vbr->vbr_status |= VIOBLK_REQSTAT_POLL_COMPLETE;
723f8296c60SJoshua M. Clulow wakeup = B_TRUE;
724f8296c60SJoshua M. Clulow continue;
7251c5bc425SAlexey Zaytsev }
7261c5bc425SAlexey Zaytsev
727f8296c60SJoshua M. Clulow vioblk_complete(vib, vbr);
7281c5bc425SAlexey Zaytsev
729f8296c60SJoshua M. Clulow vioblk_req_free(vib, vbr);
7301c5bc425SAlexey Zaytsev }
7311c5bc425SAlexey Zaytsev
732f8296c60SJoshua M. Clulow if (wakeup) {
733f8296c60SJoshua M. Clulow /*
734f8296c60SJoshua M. Clulow * Signal anybody waiting for polled command completion.
735f8296c60SJoshua M. Clulow */
736f8296c60SJoshua M. Clulow cv_broadcast(&vib->vib_cv);
737f8296c60SJoshua M. Clulow }
7381c5bc425SAlexey Zaytsev
739f8296c60SJoshua M. Clulow return (count);
7401c5bc425SAlexey Zaytsev }
7411c5bc425SAlexey Zaytsev
742501bc5c0SAndy Fiddaman static uint_t
vioblk_int_handler(caddr_t arg0,caddr_t arg1 __unused)743501bc5c0SAndy Fiddaman vioblk_int_handler(caddr_t arg0, caddr_t arg1 __unused)
7441c5bc425SAlexey Zaytsev {
745f8296c60SJoshua M. Clulow vioblk_t *vib = (vioblk_t *)arg0;
746f8296c60SJoshua M. Clulow uint_t count;
7471c5bc425SAlexey Zaytsev
748f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex);
749f8296c60SJoshua M. Clulow if ((count = vioblk_poll(vib)) >
750f8296c60SJoshua M. Clulow vib->vib_stats->vbs_intr_queuemax.value.ui32) {
751f8296c60SJoshua M. Clulow vib->vib_stats->vbs_intr_queuemax.value.ui32 = count;
752f8296c60SJoshua M. Clulow }
7531c5bc425SAlexey Zaytsev
754f8296c60SJoshua M. Clulow vib->vib_stats->vbs_intr_total.value.ui64++;
755f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex);
7561c5bc425SAlexey Zaytsev
757f8296c60SJoshua M. Clulow return (DDI_INTR_CLAIMED);
7581c5bc425SAlexey Zaytsev }
7591c5bc425SAlexey Zaytsev
760501bc5c0SAndy Fiddaman static uint_t
vioblk_cfgchange(caddr_t arg0,caddr_t arg1 __unused)761501bc5c0SAndy Fiddaman vioblk_cfgchange(caddr_t arg0, caddr_t arg1 __unused)
762501bc5c0SAndy Fiddaman {
763501bc5c0SAndy Fiddaman vioblk_t *vib = (vioblk_t *)arg0;
764501bc5c0SAndy Fiddaman
765501bc5c0SAndy Fiddaman dev_err(vib->vib_dip, CE_NOTE, "!Configuration changed");
766501bc5c0SAndy Fiddaman
767501bc5c0SAndy Fiddaman mutex_enter(&vib->vib_mutex);
768501bc5c0SAndy Fiddaman
769501bc5c0SAndy Fiddaman /*
770501bc5c0SAndy Fiddaman * The configuration space of the device has changed in some way.
771501bc5c0SAndy Fiddaman * At present, we only re-read the device capacity and trigger
772501bc5c0SAndy Fiddaman * blkdev to check the device state.
773501bc5c0SAndy Fiddaman */
774501bc5c0SAndy Fiddaman
775501bc5c0SAndy Fiddaman if (vioblk_read_capacity(vib) == DDI_FAILURE) {
776501bc5c0SAndy Fiddaman mutex_exit(&vib->vib_mutex);
777501bc5c0SAndy Fiddaman return (DDI_INTR_CLAIMED);
778501bc5c0SAndy Fiddaman }
779501bc5c0SAndy Fiddaman
780501bc5c0SAndy Fiddaman mutex_exit(&vib->vib_mutex);
781501bc5c0SAndy Fiddaman
782501bc5c0SAndy Fiddaman bd_state_change(vib->vib_bd_h);
783501bc5c0SAndy Fiddaman
784501bc5c0SAndy Fiddaman return (DDI_INTR_CLAIMED);
785501bc5c0SAndy Fiddaman }
786501bc5c0SAndy Fiddaman
7871c5bc425SAlexey Zaytsev static void
vioblk_free_reqs(vioblk_t * vib)788f8296c60SJoshua M. Clulow vioblk_free_reqs(vioblk_t *vib)
7891c5bc425SAlexey Zaytsev {
790f8296c60SJoshua M. Clulow VERIFY3U(vib->vib_nreqs_alloc, ==, 0);
7911c5bc425SAlexey Zaytsev
792f8296c60SJoshua M. Clulow for (uint_t i = 0; i < vib->vib_reqs_capacity; i++) {
793f8296c60SJoshua M. Clulow struct vioblk_req *vbr = &vib->vib_reqs_mem[i];
7941c5bc425SAlexey Zaytsev
795f8296c60SJoshua M. Clulow VERIFY(list_link_active(&vbr->vbr_link));
796f8296c60SJoshua M. Clulow list_remove(&vib->vib_reqs, vbr);
7971c5bc425SAlexey Zaytsev
798f8296c60SJoshua M. Clulow VERIFY0(vbr->vbr_status);
7991c5bc425SAlexey Zaytsev
8003decf168SPatrick Mooney if (vbr->vbr_chain != NULL) {
8013decf168SPatrick Mooney virtio_chain_free(vbr->vbr_chain);
8023decf168SPatrick Mooney vbr->vbr_chain = NULL;
8033decf168SPatrick Mooney }
804f8296c60SJoshua M. Clulow if (vbr->vbr_dma != NULL) {
805f8296c60SJoshua M. Clulow virtio_dma_free(vbr->vbr_dma);
806f8296c60SJoshua M. Clulow vbr->vbr_dma = NULL;
807f8296c60SJoshua M. Clulow }
8081c5bc425SAlexey Zaytsev }
809f8296c60SJoshua M. Clulow VERIFY(list_is_empty(&vib->vib_reqs));
8101c5bc425SAlexey Zaytsev
811f8296c60SJoshua M. Clulow if (vib->vib_reqs_mem != NULL) {
812f8296c60SJoshua M. Clulow kmem_free(vib->vib_reqs_mem,
813f8296c60SJoshua M. Clulow sizeof (struct vioblk_req) * vib->vib_reqs_capacity);
814f8296c60SJoshua M. Clulow vib->vib_reqs_mem = NULL;
815f8296c60SJoshua M. Clulow vib->vib_reqs_capacity = 0;
816f8296c60SJoshua M. Clulow }
8171c5bc425SAlexey Zaytsev }
8181c5bc425SAlexey Zaytsev
8191c5bc425SAlexey Zaytsev static int
vioblk_alloc_reqs(vioblk_t * vib)820f8296c60SJoshua M. Clulow vioblk_alloc_reqs(vioblk_t *vib)
8211c5bc425SAlexey Zaytsev {
822f8296c60SJoshua M. Clulow vib->vib_reqs_capacity = MIN(virtio_queue_size(vib->vib_vq),
823f8296c60SJoshua M. Clulow VIRTIO_BLK_REQ_BUFS);
824f8296c60SJoshua M. Clulow vib->vib_reqs_mem = kmem_zalloc(
825f8296c60SJoshua M. Clulow sizeof (struct vioblk_req) * vib->vib_reqs_capacity, KM_SLEEP);
826f8296c60SJoshua M. Clulow vib->vib_nreqs_alloc = 0;
827f8296c60SJoshua M. Clulow
828f8296c60SJoshua M. Clulow for (uint_t i = 0; i < vib->vib_reqs_capacity; i++) {
829f8296c60SJoshua M. Clulow list_insert_tail(&vib->vib_reqs, &vib->vib_reqs_mem[i]);
830f8296c60SJoshua M. Clulow }
8311c5bc425SAlexey Zaytsev
832f8296c60SJoshua M. Clulow for (vioblk_req_t *vbr = list_head(&vib->vib_reqs); vbr != NULL;
833f8296c60SJoshua M. Clulow vbr = list_next(&vib->vib_reqs, vbr)) {
834f8296c60SJoshua M. Clulow if ((vbr->vbr_dma = virtio_dma_alloc(vib->vib_virtio,
8351c5bc425SAlexey Zaytsev sizeof (struct vioblk_req_hdr) + sizeof (uint8_t),
836f8296c60SJoshua M. Clulow &vioblk_dma_attr, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
837f8296c60SJoshua M. Clulow KM_SLEEP)) == NULL) {
838f8296c60SJoshua M. Clulow goto fail;
8391c5bc425SAlexey Zaytsev }
8403decf168SPatrick Mooney vbr->vbr_chain = virtio_chain_alloc(vib->vib_vq, KM_SLEEP);
8413decf168SPatrick Mooney if (vbr->vbr_chain == NULL) {
8423decf168SPatrick Mooney goto fail;
8433decf168SPatrick Mooney }
8443decf168SPatrick Mooney virtio_chain_data_set(vbr->vbr_chain, vbr);
8451c5bc425SAlexey Zaytsev }
8461c5bc425SAlexey Zaytsev
8471c5bc425SAlexey Zaytsev return (0);
8481c5bc425SAlexey Zaytsev
849f8296c60SJoshua M. Clulow fail:
850f8296c60SJoshua M. Clulow vioblk_free_reqs(vib);
8511c5bc425SAlexey Zaytsev return (ENOMEM);
8521c5bc425SAlexey Zaytsev }
8531c5bc425SAlexey Zaytsev
854501bc5c0SAndy Fiddaman static int
vioblk_read_capacity(vioblk_t * vib)855501bc5c0SAndy Fiddaman vioblk_read_capacity(vioblk_t *vib)
856501bc5c0SAndy Fiddaman {
857501bc5c0SAndy Fiddaman virtio_t *vio = vib->vib_virtio;
858501bc5c0SAndy Fiddaman
859501bc5c0SAndy Fiddaman /* The capacity is always available */
860501bc5c0SAndy Fiddaman if ((vib->vib_nblks = virtio_dev_get64(vio,
861501bc5c0SAndy Fiddaman VIRTIO_BLK_CONFIG_CAPACITY)) == UINT64_MAX) {
862501bc5c0SAndy Fiddaman dev_err(vib->vib_dip, CE_WARN, "invalid capacity");
863501bc5c0SAndy Fiddaman return (DDI_FAILURE);
864501bc5c0SAndy Fiddaman }
865501bc5c0SAndy Fiddaman
866501bc5c0SAndy Fiddaman /*
867501bc5c0SAndy Fiddaman * Determine the optimal logical block size recommended by the device.
868501bc5c0SAndy Fiddaman * This size is advisory; the protocol always deals in 512 byte blocks.
869501bc5c0SAndy Fiddaman */
870501bc5c0SAndy Fiddaman vib->vib_blk_size = DEV_BSIZE;
871501bc5c0SAndy Fiddaman if (virtio_feature_present(vio, VIRTIO_BLK_F_BLK_SIZE)) {
872501bc5c0SAndy Fiddaman uint32_t v = virtio_dev_get32(vio, VIRTIO_BLK_CONFIG_BLK_SIZE);
873501bc5c0SAndy Fiddaman
874501bc5c0SAndy Fiddaman if (v != 0 && v != PCI_EINVAL32)
875501bc5c0SAndy Fiddaman vib->vib_blk_size = v;
876501bc5c0SAndy Fiddaman }
877501bc5c0SAndy Fiddaman
878501bc5c0SAndy Fiddaman /*
879501bc5c0SAndy Fiddaman * Device capacity is always in 512-byte units, convert to
880501bc5c0SAndy Fiddaman * native blocks.
881501bc5c0SAndy Fiddaman */
882501bc5c0SAndy Fiddaman vib->vib_nblks = (vib->vib_nblks * DEV_BSIZE) / vib->vib_blk_size;
883501bc5c0SAndy Fiddaman
884501bc5c0SAndy Fiddaman /*
885501bc5c0SAndy Fiddaman * The device may also provide an advisory physical block size.
886501bc5c0SAndy Fiddaman */
887501bc5c0SAndy Fiddaman vib->vib_pblk_size = vib->vib_blk_size;
888501bc5c0SAndy Fiddaman if (virtio_feature_present(vio, VIRTIO_BLK_F_TOPOLOGY)) {
889501bc5c0SAndy Fiddaman uint8_t v = virtio_dev_get8(vio, VIRTIO_BLK_CONFIG_TOPO_PBEXP);
890501bc5c0SAndy Fiddaman
891501bc5c0SAndy Fiddaman if (v != PCI_EINVAL8)
892501bc5c0SAndy Fiddaman vib->vib_pblk_size <<= v;
893501bc5c0SAndy Fiddaman }
894501bc5c0SAndy Fiddaman
895501bc5c0SAndy Fiddaman return (DDI_SUCCESS);
896501bc5c0SAndy Fiddaman }
897501bc5c0SAndy Fiddaman
8981c5bc425SAlexey Zaytsev static int
vioblk_attach(dev_info_t * dip,ddi_attach_cmd_t cmd)899f8296c60SJoshua M. Clulow vioblk_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
9001c5bc425SAlexey Zaytsev {
901f8296c60SJoshua M. Clulow int instance = ddi_get_instance(dip);
902f8296c60SJoshua M. Clulow vioblk_t *vib;
903f8296c60SJoshua M. Clulow virtio_t *vio;
904f8296c60SJoshua M. Clulow boolean_t did_mutex = B_FALSE;
9051c5bc425SAlexey Zaytsev
906f8296c60SJoshua M. Clulow if (cmd != DDI_ATTACH) {
907d48defc5SHans Rosenfeld return (DDI_FAILURE);
908f8296c60SJoshua M. Clulow }
9091c5bc425SAlexey Zaytsev
910f8296c60SJoshua M. Clulow if ((vio = virtio_init(dip, VIRTIO_BLK_WANTED_FEATURES, B_TRUE)) ==
911f8296c60SJoshua M. Clulow NULL) {
912f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "failed to start Virtio init");
913d48defc5SHans Rosenfeld return (DDI_FAILURE);
9141c5bc425SAlexey Zaytsev }
9151c5bc425SAlexey Zaytsev
916f8296c60SJoshua M. Clulow vib = kmem_zalloc(sizeof (*vib), KM_SLEEP);
917f8296c60SJoshua M. Clulow vib->vib_dip = dip;
918f8296c60SJoshua M. Clulow vib->vib_virtio = vio;
919f8296c60SJoshua M. Clulow ddi_set_driver_private(dip, vib);
920f8296c60SJoshua M. Clulow list_create(&vib->vib_reqs, sizeof (vioblk_req_t),
921f8296c60SJoshua M. Clulow offsetof(vioblk_req_t, vbr_link));
9221c5bc425SAlexey Zaytsev
923f8296c60SJoshua M. Clulow /*
924f8296c60SJoshua M. Clulow * Determine how many scatter-gather entries we can use in a single
925f8296c60SJoshua M. Clulow * request.
926f8296c60SJoshua M. Clulow */
927f8296c60SJoshua M. Clulow vib->vib_seg_max = VIRTIO_BLK_DEFAULT_MAX_SEG;
928f8296c60SJoshua M. Clulow if (virtio_feature_present(vio, VIRTIO_BLK_F_SEG_MAX)) {
929f8296c60SJoshua M. Clulow vib->vib_seg_max = virtio_dev_get32(vio,
930f8296c60SJoshua M. Clulow VIRTIO_BLK_CONFIG_SEG_MAX);
9311c5bc425SAlexey Zaytsev
932f8296c60SJoshua M. Clulow if (vib->vib_seg_max == 0 || vib->vib_seg_max == PCI_EINVAL32) {
933f8296c60SJoshua M. Clulow /*
934f8296c60SJoshua M. Clulow * We need to be able to use at least one data segment,
935f8296c60SJoshua M. Clulow * so we'll assume that this device is just poorly
936f8296c60SJoshua M. Clulow * implemented and try for one.
937f8296c60SJoshua M. Clulow */
938f8296c60SJoshua M. Clulow vib->vib_seg_max = 1;
939f8296c60SJoshua M. Clulow }
940f8296c60SJoshua M. Clulow }
9411c5bc425SAlexey Zaytsev
9421a5ae140SJason King if (virtio_feature_present(vio, VIRTIO_BLK_F_DISCARD)) {
9431a5ae140SJason King vib->vib_max_discard_sectors = virtio_dev_get32(vio,
9441a5ae140SJason King VIRTIO_BLK_CONFIG_MAX_DISCARD_SECT);
9451a5ae140SJason King vib->vib_max_discard_seg = virtio_dev_get32(vio,
9461a5ae140SJason King VIRTIO_BLK_CONFIG_MAX_DISCARD_SEG);
9471a5ae140SJason King vib->vib_discard_sector_align = virtio_dev_get32(vio,
9481a5ae140SJason King VIRTIO_BLK_CONFIG_DISCARD_ALIGN);
9491a5ae140SJason King
9501a5ae140SJason King if (vib->vib_max_discard_sectors == 0 ||
9511a5ae140SJason King vib->vib_max_discard_seg == 0 ||
9521a5ae140SJason King vib->vib_discard_sector_align == 0) {
9531a5ae140SJason King vib->vib_can_discard = B_FALSE;
9541a5ae140SJason King
9551a5ae140SJason King /*
9561a5ae140SJason King * The hypervisor shouldn't be giving us bad values.
9571a5ae140SJason King * If it is, it's probably worth notifying the
9581a5ae140SJason King * operator.
9591a5ae140SJason King */
9601a5ae140SJason King dev_err(dip, CE_NOTE,
9611a5ae140SJason King "Host is advertising DISCARD support but with bad"
9621a5ae140SJason King "parameters: max_discard_sectors=%u, "
9631a5ae140SJason King "max_discard_segments=%u, discard_sector_align=%u",
9641a5ae140SJason King vib->vib_max_discard_sectors,
9651a5ae140SJason King vib->vib_max_discard_seg,
9661a5ae140SJason King vib->vib_discard_sector_align);
9671a5ae140SJason King } else {
9681a5ae140SJason King vib->vib_can_discard = B_TRUE;
9691a5ae140SJason King }
9701a5ae140SJason King }
9711a5ae140SJason King
9721c5bc425SAlexey Zaytsev /*
973f8296c60SJoshua M. Clulow * When allocating the request queue, we include two additional
974f8296c60SJoshua M. Clulow * descriptors (beyond those required for request data) to account for
975f8296c60SJoshua M. Clulow * the header and the status byte.
9761c5bc425SAlexey Zaytsev */
977f8296c60SJoshua M. Clulow if ((vib->vib_vq = virtio_queue_alloc(vio, VIRTIO_BLK_VIRTQ_IO, "io",
978f8296c60SJoshua M. Clulow vioblk_int_handler, vib, B_FALSE, vib->vib_seg_max + 2)) == NULL) {
979f8296c60SJoshua M. Clulow goto fail;
980f8296c60SJoshua M. Clulow }
981f8296c60SJoshua M. Clulow
982501bc5c0SAndy Fiddaman virtio_register_cfgchange_handler(vio, vioblk_cfgchange, vib);
983501bc5c0SAndy Fiddaman
984*64439ec0SJoshua M. Clulow if (virtio_init_complete(vio, VIRTIO_ANY_INTR_TYPE) != DDI_SUCCESS) {
985f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "failed to complete Virtio init");
986f8296c60SJoshua M. Clulow goto fail;
987f8296c60SJoshua M. Clulow }
988f8296c60SJoshua M. Clulow
989f8296c60SJoshua M. Clulow cv_init(&vib->vib_cv, NULL, CV_DRIVER, NULL);
990f8296c60SJoshua M. Clulow mutex_init(&vib->vib_mutex, NULL, MUTEX_DRIVER, virtio_intr_pri(vio));
991f8296c60SJoshua M. Clulow did_mutex = B_TRUE;
992f8296c60SJoshua M. Clulow
993f8296c60SJoshua M. Clulow if ((vib->vib_kstat = kstat_create("vioblk", instance,
994f8296c60SJoshua M. Clulow "statistics", "controller", KSTAT_TYPE_NAMED,
9951c5bc425SAlexey Zaytsev sizeof (struct vioblk_stats) / sizeof (kstat_named_t),
996f8296c60SJoshua M. Clulow KSTAT_FLAG_PERSISTENT)) == NULL) {
997f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "kstat_create failed");
998f8296c60SJoshua M. Clulow goto fail;
9991c5bc425SAlexey Zaytsev }
1000f8296c60SJoshua M. Clulow vib->vib_stats = (vioblk_stats_t *)vib->vib_kstat->ks_data;
1001f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_rw_outofmemory,
10021c5bc425SAlexey Zaytsev "total_rw_outofmemory", KSTAT_DATA_UINT64);
1003f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_rw_badoffset,
10041c5bc425SAlexey Zaytsev "total_rw_badoffset", KSTAT_DATA_UINT64);
1005f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_intr_total,
10061c5bc425SAlexey Zaytsev "total_intr", KSTAT_DATA_UINT64);
1007f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_io_errors,
1008f8296c60SJoshua M. Clulow "total_io_errors", KSTAT_DATA_UINT64);
1009f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_unsupp_errors,
1010f8296c60SJoshua M. Clulow "total_unsupp_errors", KSTAT_DATA_UINT64);
1011f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_nxio_errors,
1012f8296c60SJoshua M. Clulow "total_nxio_errors", KSTAT_DATA_UINT64);
1013f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_rw_cacheflush,
10141c5bc425SAlexey Zaytsev "total_rw_cacheflush", KSTAT_DATA_UINT64);
1015f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_rw_cookiesmax,
10161c5bc425SAlexey Zaytsev "max_rw_cookies", KSTAT_DATA_UINT32);
1017f8296c60SJoshua M. Clulow kstat_named_init(&vib->vib_stats->vbs_intr_queuemax,
10181c5bc425SAlexey Zaytsev "max_intr_queue", KSTAT_DATA_UINT32);
1019f8296c60SJoshua M. Clulow kstat_install(vib->vib_kstat);
1020f8296c60SJoshua M. Clulow
1021f8296c60SJoshua M. Clulow vib->vib_readonly = virtio_feature_present(vio, VIRTIO_BLK_F_RO);
10221c5bc425SAlexey Zaytsev
1023501bc5c0SAndy Fiddaman if (vioblk_read_capacity(vib) == DDI_FAILURE)
1024501bc5c0SAndy Fiddaman goto fail;
10251c5bc425SAlexey Zaytsev
1026f8296c60SJoshua M. Clulow /*
1027f8296c60SJoshua M. Clulow * The maximum size for a cookie in a request.
1028f8296c60SJoshua M. Clulow */
1029f8296c60SJoshua M. Clulow vib->vib_seg_size_max = VIRTIO_BLK_DEFAULT_MAX_SIZE;
1030f8296c60SJoshua M. Clulow if (virtio_feature_present(vio, VIRTIO_BLK_F_SIZE_MAX)) {
1031f8296c60SJoshua M. Clulow uint32_t v = virtio_dev_get32(vio, VIRTIO_BLK_CONFIG_SIZE_MAX);
10321c5bc425SAlexey Zaytsev
1033f8296c60SJoshua M. Clulow if (v != 0 && v != PCI_EINVAL32) {
1034f8296c60SJoshua M. Clulow vib->vib_seg_size_max = v;
1035f8296c60SJoshua M. Clulow }
10361c5bc425SAlexey Zaytsev }
10371c5bc425SAlexey Zaytsev
1038f8296c60SJoshua M. Clulow /*
1039f8296c60SJoshua M. Clulow * Set up the DMA attributes for blkdev to use for request data. The
1040f8296c60SJoshua M. Clulow * specification is not extremely clear about whether DMA-related
1041f8296c60SJoshua M. Clulow * parameters include or exclude the header and status descriptors.
1042f8296c60SJoshua M. Clulow * For now, we assume they cover only the request data and not the
1043f8296c60SJoshua M. Clulow * headers.
1044f8296c60SJoshua M. Clulow */
1045f8296c60SJoshua M. Clulow vib->vib_bd_dma_attr = vioblk_dma_attr;
1046f8296c60SJoshua M. Clulow vib->vib_bd_dma_attr.dma_attr_sgllen = vib->vib_seg_max;
1047f8296c60SJoshua M. Clulow vib->vib_bd_dma_attr.dma_attr_count_max = vib->vib_seg_size_max;
1048f8296c60SJoshua M. Clulow vib->vib_bd_dma_attr.dma_attr_maxxfer = vib->vib_seg_max *
1049f8296c60SJoshua M. Clulow vib->vib_seg_size_max;
1050f8296c60SJoshua M. Clulow
1051f8296c60SJoshua M. Clulow if (vioblk_alloc_reqs(vib) != 0) {
1052f8296c60SJoshua M. Clulow goto fail;
10533e0831a9SHans Rosenfeld }
10543e0831a9SHans Rosenfeld
1055f8296c60SJoshua M. Clulow /*
1056f8296c60SJoshua M. Clulow * The blkdev framework does not provide a way to specify that the
1057f8296c60SJoshua M. Clulow * device does not support write cache flushing, except by omitting the
1058f8296c60SJoshua M. Clulow * "o_sync_cache" member from the ops vector. As "bd_alloc_handle()"
1059f8296c60SJoshua M. Clulow * makes a copy of the ops vector, we can safely assemble one on the
1060f8296c60SJoshua M. Clulow * stack based on negotiated features.
10611a5ae140SJason King *
10621a5ae140SJason King * Similarly, the blkdev framework does not provide a way to indicate
10631a5ae140SJason King * if a device supports an TRIM/UNMAP/DISCARD type operation except
10641a5ae140SJason King * by omitting the "o_free_space" member from the ops vector.
1065f8296c60SJoshua M. Clulow */
1066f8296c60SJoshua M. Clulow bd_ops_t vioblk_bd_ops = {
10674d95620bSPaul Winder .o_version = BD_OPS_CURRENT_VERSION,
1068f8296c60SJoshua M. Clulow .o_drive_info = vioblk_bd_driveinfo,
1069f8296c60SJoshua M. Clulow .o_media_info = vioblk_bd_mediainfo,
1070f8296c60SJoshua M. Clulow .o_devid_init = vioblk_bd_devid,
1071f8296c60SJoshua M. Clulow .o_sync_cache = vioblk_bd_flush,
1072f8296c60SJoshua M. Clulow .o_read = vioblk_bd_read,
1073f8296c60SJoshua M. Clulow .o_write = vioblk_bd_write,
10741a5ae140SJason King .o_free_space = vioblk_bd_free_space,
1075f8296c60SJoshua M. Clulow };
1076f8296c60SJoshua M. Clulow if (!virtio_feature_present(vio, VIRTIO_BLK_F_FLUSH)) {
1077f8296c60SJoshua M. Clulow vioblk_bd_ops.o_sync_cache = NULL;
10781c5bc425SAlexey Zaytsev }
10791a5ae140SJason King if (!vib->vib_can_discard) {
10801a5ae140SJason King vioblk_bd_ops.o_free_space = NULL;
10811a5ae140SJason King }
10821c5bc425SAlexey Zaytsev
1083f8296c60SJoshua M. Clulow vib->vib_bd_h = bd_alloc_handle(vib, &vioblk_bd_ops,
1084f8296c60SJoshua M. Clulow &vib->vib_bd_dma_attr, KM_SLEEP);
10851c5bc425SAlexey Zaytsev
1086f8296c60SJoshua M. Clulow /*
1087f8296c60SJoshua M. Clulow * Enable interrupts now so that we can request the device identity.
1088f8296c60SJoshua M. Clulow */
1089f8296c60SJoshua M. Clulow if (virtio_interrupts_enable(vio) != DDI_SUCCESS) {
1090f8296c60SJoshua M. Clulow goto fail;
10911c5bc425SAlexey Zaytsev }
10921c5bc425SAlexey Zaytsev
1093f8296c60SJoshua M. Clulow vioblk_get_id(vib);
10941c5bc425SAlexey Zaytsev
1095f8296c60SJoshua M. Clulow if (bd_attach_handle(dip, vib->vib_bd_h) != DDI_SUCCESS) {
1096f8296c60SJoshua M. Clulow dev_err(dip, CE_WARN, "Failed to attach blkdev");
1097f8296c60SJoshua M. Clulow goto fail;
10981c5bc425SAlexey Zaytsev }
10991c5bc425SAlexey Zaytsev
1100f8296c60SJoshua M. Clulow return (DDI_SUCCESS);
11011c5bc425SAlexey Zaytsev
1102f8296c60SJoshua M. Clulow fail:
1103f8296c60SJoshua M. Clulow if (vib->vib_bd_h != NULL) {
1104f8296c60SJoshua M. Clulow (void) bd_detach_handle(vib->vib_bd_h);
1105f8296c60SJoshua M. Clulow bd_free_handle(vib->vib_bd_h);
11061c5bc425SAlexey Zaytsev }
1107f8296c60SJoshua M. Clulow if (vio != NULL) {
1108f8296c60SJoshua M. Clulow (void) virtio_fini(vio, B_TRUE);
11091c5bc425SAlexey Zaytsev }
1110f8296c60SJoshua M. Clulow if (did_mutex) {
1111f8296c60SJoshua M. Clulow mutex_destroy(&vib->vib_mutex);
1112f8296c60SJoshua M. Clulow cv_destroy(&vib->vib_cv);
11131c5bc425SAlexey Zaytsev }
1114f8296c60SJoshua M. Clulow if (vib->vib_kstat != NULL) {
1115f8296c60SJoshua M. Clulow kstat_delete(vib->vib_kstat);
1116f8296c60SJoshua M. Clulow }
1117f8296c60SJoshua M. Clulow vioblk_free_reqs(vib);
1118f8296c60SJoshua M. Clulow kmem_free(vib, sizeof (*vib));
1119d48defc5SHans Rosenfeld return (DDI_FAILURE);
11201c5bc425SAlexey Zaytsev }
11211c5bc425SAlexey Zaytsev
11221c5bc425SAlexey Zaytsev static int
vioblk_detach(dev_info_t * dip,ddi_detach_cmd_t cmd)1123f8296c60SJoshua M. Clulow vioblk_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
11241c5bc425SAlexey Zaytsev {
1125f8296c60SJoshua M. Clulow vioblk_t *vib = ddi_get_driver_private(dip);
11261c5bc425SAlexey Zaytsev
1127f8296c60SJoshua M. Clulow if (cmd != DDI_DETACH) {
1128f8296c60SJoshua M. Clulow return (DDI_FAILURE);
1129f8296c60SJoshua M. Clulow }
11301c5bc425SAlexey Zaytsev
1131f8296c60SJoshua M. Clulow mutex_enter(&vib->vib_mutex);
1132f8296c60SJoshua M. Clulow if (vib->vib_nreqs_alloc > 0) {
1133f8296c60SJoshua M. Clulow /*
1134f8296c60SJoshua M. Clulow * Cannot detach while there are still outstanding requests.
1135f8296c60SJoshua M. Clulow */
1136f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex);
11371c5bc425SAlexey Zaytsev return (DDI_FAILURE);
1138f8296c60SJoshua M. Clulow }
11391c5bc425SAlexey Zaytsev
1140f8296c60SJoshua M. Clulow if (bd_detach_handle(vib->vib_bd_h) != DDI_SUCCESS) {
1141f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex);
11421c5bc425SAlexey Zaytsev return (DDI_FAILURE);
11431c5bc425SAlexey Zaytsev }
11441c5bc425SAlexey Zaytsev
1145f8296c60SJoshua M. Clulow /*
1146f8296c60SJoshua M. Clulow * Tear down the Virtio framework before freeing the rest of the
1147f8296c60SJoshua M. Clulow * resources. This will ensure the interrupt handlers are no longer
1148f8296c60SJoshua M. Clulow * running.
1149f8296c60SJoshua M. Clulow */
1150f8296c60SJoshua M. Clulow virtio_fini(vib->vib_virtio, B_FALSE);
1151f8296c60SJoshua M. Clulow
1152f8296c60SJoshua M. Clulow vioblk_free_reqs(vib);
1153f8296c60SJoshua M. Clulow kstat_delete(vib->vib_kstat);
1154f8296c60SJoshua M. Clulow
1155f8296c60SJoshua M. Clulow mutex_exit(&vib->vib_mutex);
1156f8296c60SJoshua M. Clulow mutex_destroy(&vib->vib_mutex);
1157f8296c60SJoshua M. Clulow
1158f8296c60SJoshua M. Clulow kmem_free(vib, sizeof (*vib));
11591c5bc425SAlexey Zaytsev
11601c5bc425SAlexey Zaytsev return (DDI_SUCCESS);
11611c5bc425SAlexey Zaytsev }
11621c5bc425SAlexey Zaytsev
11631c5bc425SAlexey Zaytsev static int
vioblk_quiesce(dev_info_t * dip)1164f8296c60SJoshua M. Clulow vioblk_quiesce(dev_info_t *dip)
11651c5bc425SAlexey Zaytsev {
1166f8296c60SJoshua M. Clulow vioblk_t *vib;
11671c5bc425SAlexey Zaytsev
1168f8296c60SJoshua M. Clulow if ((vib = ddi_get_driver_private(dip)) == NULL) {
1169f8296c60SJoshua M. Clulow return (DDI_FAILURE);
1170f8296c60SJoshua M. Clulow }
11711c5bc425SAlexey Zaytsev
1172f8296c60SJoshua M. Clulow return (virtio_quiesce(vib->vib_virtio));
11731c5bc425SAlexey Zaytsev }
11741c5bc425SAlexey Zaytsev
11751c5bc425SAlexey Zaytsev int
_init(void)11761c5bc425SAlexey Zaytsev _init(void)
11771c5bc425SAlexey Zaytsev {
11781c5bc425SAlexey Zaytsev int rv;
11791c5bc425SAlexey Zaytsev
11801c5bc425SAlexey Zaytsev bd_mod_init(&vioblk_dev_ops);
11811c5bc425SAlexey Zaytsev
1182f8296c60SJoshua M. Clulow if ((rv = mod_install(&vioblk_modlinkage)) != 0) {
11831c5bc425SAlexey Zaytsev bd_mod_fini(&vioblk_dev_ops);
11841c5bc425SAlexey Zaytsev }
11851c5bc425SAlexey Zaytsev
11861c5bc425SAlexey Zaytsev return (rv);
11871c5bc425SAlexey Zaytsev }
11881c5bc425SAlexey Zaytsev
11891c5bc425SAlexey Zaytsev int
_fini(void)11901c5bc425SAlexey Zaytsev _fini(void)
11911c5bc425SAlexey Zaytsev {
11921c5bc425SAlexey Zaytsev int rv;
11931c5bc425SAlexey Zaytsev
1194f8296c60SJoshua M. Clulow if ((rv = mod_remove(&vioblk_modlinkage)) == 0) {
11951c5bc425SAlexey Zaytsev bd_mod_fini(&vioblk_dev_ops);
11961c5bc425SAlexey Zaytsev }
11971c5bc425SAlexey Zaytsev
11981c5bc425SAlexey Zaytsev return (rv);
11991c5bc425SAlexey Zaytsev }
12001c5bc425SAlexey Zaytsev
12011c5bc425SAlexey Zaytsev int
_info(struct modinfo * modinfop)12021c5bc425SAlexey Zaytsev _info(struct modinfo *modinfop)
12031c5bc425SAlexey Zaytsev {
1204f8296c60SJoshua M. Clulow return (mod_info(&vioblk_modlinkage, modinfop));
12051c5bc425SAlexey Zaytsev }
1206