1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2016 Jakub Klama <jceel@FreeBSD.org>.
5 * Copyright (c) 2018 Marcelo Araujo <araujo@FreeBSD.org>.
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer
13 * in this position and unchanged.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32
33 #include <sys/param.h>
34 #include <sys/linker_set.h>
35 #include <sys/types.h>
36 #include <sys/uio.h>
37 #include <sys/time.h>
38 #include <sys/queue.h>
39 #include <sys/sbuf.h>
40
41 #include <errno.h>
42 #include <fcntl.h>
43 #include <stdio.h>
44 #include <stdlib.h>
45 #include <stdbool.h>
46 #include <string.h>
47 #include <unistd.h>
48 #include <assert.h>
49 #include <pthread.h>
50 #include <pthread_np.h>
51
52 #include <cam/scsi/scsi_all.h>
53 #include <cam/scsi/scsi_message.h>
54 #include <cam/ctl/ctl.h>
55 #include <cam/ctl/ctl_io.h>
56 #include <cam/ctl/ctl_backend.h>
57 #include <cam/ctl/ctl_ioctl.h>
58 #include <cam/ctl/ctl_util.h>
59 #include <cam/ctl/ctl_scsi_all.h>
60 #include <camlib.h>
61
62 #include "bhyverun.h"
63 #include "config.h"
64 #include "debug.h"
65 #include "pci_emul.h"
66 #include "virtio.h"
67 #include "iov.h"
68
69 #define VTSCSI_RINGSZ 64
70 #define VTSCSI_REQUESTQ 1
71 #define VTSCSI_THR_PER_Q 16
72 #define VTSCSI_MAXQ (VTSCSI_REQUESTQ + 2)
73 #define VTSCSI_MAXSEG 64
74
75 #define VTSCSI_IN_HEADER_LEN(_sc) \
76 (sizeof(struct pci_vtscsi_req_cmd_rd) + _sc->vss_config.cdb_size)
77
78 #define VTSCSI_OUT_HEADER_LEN(_sc) \
79 (sizeof(struct pci_vtscsi_req_cmd_wr) + _sc->vss_config.sense_size)
80
81 #define VIRTIO_SCSI_MAX_CHANNEL 0
82 #define VIRTIO_SCSI_MAX_TARGET 0
83 #define VIRTIO_SCSI_MAX_LUN 16383
84
85 #define VIRTIO_SCSI_F_INOUT (1 << 0)
86 #define VIRTIO_SCSI_F_HOTPLUG (1 << 1)
87 #define VIRTIO_SCSI_F_CHANGE (1 << 2)
88
89 static int pci_vtscsi_debug = 0;
90 #define WPRINTF(msg, params...) PRINTLN("virtio-scsi: " msg, ##params)
91 #define DPRINTF(msg, params...) if (pci_vtscsi_debug) WPRINTF(msg, ##params)
92
93 struct pci_vtscsi_config {
94 uint32_t num_queues;
95 uint32_t seg_max;
96 uint32_t max_sectors;
97 uint32_t cmd_per_lun;
98 uint32_t event_info_size;
99 uint32_t sense_size;
100 uint32_t cdb_size;
101 uint16_t max_channel;
102 uint16_t max_target;
103 uint32_t max_lun;
104 } __attribute__((packed));
105
106 struct pci_vtscsi_queue {
107 struct pci_vtscsi_softc * vsq_sc;
108 struct vqueue_info * vsq_vq;
109 pthread_mutex_t vsq_mtx;
110 pthread_mutex_t vsq_qmtx;
111 pthread_cond_t vsq_cv;
112 STAILQ_HEAD(, pci_vtscsi_request) vsq_requests;
113 LIST_HEAD(, pci_vtscsi_worker) vsq_workers;
114 };
115
116 struct pci_vtscsi_worker {
117 struct pci_vtscsi_queue * vsw_queue;
118 pthread_t vsw_thread;
119 bool vsw_exiting;
120 LIST_ENTRY(pci_vtscsi_worker) vsw_link;
121 };
122
123 struct pci_vtscsi_request {
124 struct pci_vtscsi_queue * vsr_queue;
125 struct iovec vsr_iov_in[VTSCSI_MAXSEG];
126 int vsr_niov_in;
127 struct iovec vsr_iov_out[VTSCSI_MAXSEG];
128 int vsr_niov_out;
129 uint32_t vsr_idx;
130 STAILQ_ENTRY(pci_vtscsi_request) vsr_link;
131 };
132
133 /*
134 * Per-device softc
135 */
136 struct pci_vtscsi_softc {
137 struct virtio_softc vss_vs;
138 struct vqueue_info vss_vq[VTSCSI_MAXQ];
139 struct pci_vtscsi_queue vss_queues[VTSCSI_REQUESTQ];
140 pthread_mutex_t vss_mtx;
141 int vss_iid;
142 int vss_ctl_fd;
143 uint32_t vss_features;
144 struct pci_vtscsi_config vss_config;
145 };
146
147 #define VIRTIO_SCSI_T_TMF 0
148 #define VIRTIO_SCSI_T_TMF_ABORT_TASK 0
149 #define VIRTIO_SCSI_T_TMF_ABORT_TASK_SET 1
150 #define VIRTIO_SCSI_T_TMF_CLEAR_ACA 2
151 #define VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET 3
152 #define VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET 4
153 #define VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET 5
154 #define VIRTIO_SCSI_T_TMF_QUERY_TASK 6
155 #define VIRTIO_SCSI_T_TMF_QUERY_TASK_SET 7
156
157 /* command-specific response values */
158 #define VIRTIO_SCSI_S_FUNCTION_COMPLETE 0
159 #define VIRTIO_SCSI_S_FUNCTION_SUCCEEDED 10
160 #define VIRTIO_SCSI_S_FUNCTION_REJECTED 11
161
162 struct pci_vtscsi_ctrl_tmf {
163 uint32_t type;
164 uint32_t subtype;
165 uint8_t lun[8];
166 uint64_t id;
167 uint8_t response;
168 } __attribute__((packed));
169
170 #define VIRTIO_SCSI_T_AN_QUERY 1
171 #define VIRTIO_SCSI_EVT_ASYNC_OPERATIONAL_CHANGE 2
172 #define VIRTIO_SCSI_EVT_ASYNC_POWER_MGMT 4
173 #define VIRTIO_SCSI_EVT_ASYNC_EXTERNAL_REQUEST 8
174 #define VIRTIO_SCSI_EVT_ASYNC_MEDIA_CHANGE 16
175 #define VIRTIO_SCSI_EVT_ASYNC_MULTI_HOST 32
176 #define VIRTIO_SCSI_EVT_ASYNC_DEVICE_BUSY 64
177
178 struct pci_vtscsi_ctrl_an {
179 uint32_t type;
180 uint8_t lun[8];
181 uint32_t event_requested;
182 uint32_t event_actual;
183 uint8_t response;
184 } __attribute__((packed));
185
186 /* command-specific response values */
187 #define VIRTIO_SCSI_S_OK 0
188 #define VIRTIO_SCSI_S_OVERRUN 1
189 #define VIRTIO_SCSI_S_ABORTED 2
190 #define VIRTIO_SCSI_S_BAD_TARGET 3
191 #define VIRTIO_SCSI_S_RESET 4
192 #define VIRTIO_SCSI_S_BUSY 5
193 #define VIRTIO_SCSI_S_TRANSPORT_FAILURE 6
194 #define VIRTIO_SCSI_S_TARGET_FAILURE 7
195 #define VIRTIO_SCSI_S_NEXUS_FAILURE 8
196 #define VIRTIO_SCSI_S_FAILURE 9
197 #define VIRTIO_SCSI_S_INCORRECT_LUN 12
198
199 /* task_attr */
200 #define VIRTIO_SCSI_S_SIMPLE 0
201 #define VIRTIO_SCSI_S_ORDERED 1
202 #define VIRTIO_SCSI_S_HEAD 2
203 #define VIRTIO_SCSI_S_ACA 3
204
205 struct pci_vtscsi_event {
206 uint32_t event;
207 uint8_t lun[8];
208 uint32_t reason;
209 } __attribute__((packed));
210
211 struct pci_vtscsi_req_cmd_rd {
212 uint8_t lun[8];
213 uint64_t id;
214 uint8_t task_attr;
215 uint8_t prio;
216 uint8_t crn;
217 uint8_t cdb[];
218 } __attribute__((packed));
219
220 struct pci_vtscsi_req_cmd_wr {
221 uint32_t sense_len;
222 uint32_t residual;
223 uint16_t status_qualifier;
224 uint8_t status;
225 uint8_t response;
226 uint8_t sense[];
227 } __attribute__((packed));
228
229 static void *pci_vtscsi_proc(void *);
230 static void pci_vtscsi_reset(void *);
231 static void pci_vtscsi_neg_features(void *, uint64_t);
232 static int pci_vtscsi_cfgread(void *, int, int, uint32_t *);
233 static int pci_vtscsi_cfgwrite(void *, int, int, uint32_t);
234 static inline int pci_vtscsi_get_lun(uint8_t *);
235 static int pci_vtscsi_control_handle(struct pci_vtscsi_softc *, void *, size_t);
236 static int pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *,
237 struct pci_vtscsi_ctrl_tmf *);
238 static int pci_vtscsi_an_handle(struct pci_vtscsi_softc *,
239 struct pci_vtscsi_ctrl_an *);
240 static int pci_vtscsi_request_handle(struct pci_vtscsi_queue *, struct iovec *,
241 int, struct iovec *, int);
242 static void pci_vtscsi_controlq_notify(void *, struct vqueue_info *);
243 static void pci_vtscsi_eventq_notify(void *, struct vqueue_info *);
244 static void pci_vtscsi_requestq_notify(void *, struct vqueue_info *);
245 static int pci_vtscsi_init_queue(struct pci_vtscsi_softc *,
246 struct pci_vtscsi_queue *, int);
247 static int pci_vtscsi_init(struct pci_devinst *, nvlist_t *);
248
249 static struct virtio_consts vtscsi_vi_consts = {
250 .vc_name = "vtscsi",
251 .vc_nvq = VTSCSI_MAXQ,
252 .vc_cfgsize = sizeof(struct pci_vtscsi_config),
253 .vc_reset = pci_vtscsi_reset,
254 .vc_cfgread = pci_vtscsi_cfgread,
255 .vc_cfgwrite = pci_vtscsi_cfgwrite,
256 .vc_apply_features = pci_vtscsi_neg_features,
257 .vc_hv_caps = 0,
258 };
259
260 static void *
pci_vtscsi_proc(void * arg)261 pci_vtscsi_proc(void *arg)
262 {
263 struct pci_vtscsi_worker *worker = (struct pci_vtscsi_worker *)arg;
264 struct pci_vtscsi_queue *q = worker->vsw_queue;
265 struct pci_vtscsi_request *req;
266 int iolen;
267
268 for (;;) {
269 pthread_mutex_lock(&q->vsq_mtx);
270
271 while (STAILQ_EMPTY(&q->vsq_requests)
272 && !worker->vsw_exiting)
273 pthread_cond_wait(&q->vsq_cv, &q->vsq_mtx);
274
275 if (worker->vsw_exiting)
276 break;
277
278 req = STAILQ_FIRST(&q->vsq_requests);
279 STAILQ_REMOVE_HEAD(&q->vsq_requests, vsr_link);
280
281 pthread_mutex_unlock(&q->vsq_mtx);
282 iolen = pci_vtscsi_request_handle(q, req->vsr_iov_in,
283 req->vsr_niov_in, req->vsr_iov_out, req->vsr_niov_out);
284
285 pthread_mutex_lock(&q->vsq_qmtx);
286 vq_relchain(q->vsq_vq, req->vsr_idx, iolen);
287 vq_endchains(q->vsq_vq, 0);
288 pthread_mutex_unlock(&q->vsq_qmtx);
289
290 DPRINTF("request <idx=%d> completed", req->vsr_idx);
291 free(req);
292 }
293
294 pthread_mutex_unlock(&q->vsq_mtx);
295 return (NULL);
296 }
297
298 static void
pci_vtscsi_reset(void * vsc)299 pci_vtscsi_reset(void *vsc)
300 {
301 struct pci_vtscsi_softc *sc;
302
303 sc = vsc;
304
305 DPRINTF("device reset requested");
306 vi_reset_dev(&sc->vss_vs);
307
308 /* initialize config structure */
309 sc->vss_config = (struct pci_vtscsi_config){
310 .num_queues = VTSCSI_REQUESTQ,
311 /* Leave room for the request and the response. */
312 .seg_max = VTSCSI_MAXSEG - 2,
313 .max_sectors = 2,
314 .cmd_per_lun = 1,
315 .event_info_size = sizeof(struct pci_vtscsi_event),
316 .sense_size = 96,
317 .cdb_size = 32,
318 .max_channel = VIRTIO_SCSI_MAX_CHANNEL,
319 .max_target = VIRTIO_SCSI_MAX_TARGET,
320 .max_lun = VIRTIO_SCSI_MAX_LUN
321 };
322 }
323
324 static void
pci_vtscsi_neg_features(void * vsc,uint64_t negotiated_features)325 pci_vtscsi_neg_features(void *vsc, uint64_t negotiated_features)
326 {
327 struct pci_vtscsi_softc *sc = vsc;
328
329 sc->vss_features = negotiated_features;
330 }
331
332 static int
pci_vtscsi_cfgread(void * vsc,int offset,int size,uint32_t * retval)333 pci_vtscsi_cfgread(void *vsc, int offset, int size, uint32_t *retval)
334 {
335 struct pci_vtscsi_softc *sc = vsc;
336 void *ptr;
337
338 ptr = (uint8_t *)&sc->vss_config + offset;
339 memcpy(retval, ptr, size);
340 return (0);
341 }
342
343 static int
pci_vtscsi_cfgwrite(void * vsc __unused,int offset __unused,int size __unused,uint32_t val __unused)344 pci_vtscsi_cfgwrite(void *vsc __unused, int offset __unused, int size __unused,
345 uint32_t val __unused)
346 {
347 return (0);
348 }
349
350 static inline int
pci_vtscsi_get_lun(uint8_t * lun)351 pci_vtscsi_get_lun(uint8_t *lun)
352 {
353
354 return (((lun[2] << 8) | lun[3]) & 0x3fff);
355 }
356
357 static int
pci_vtscsi_control_handle(struct pci_vtscsi_softc * sc,void * buf,size_t bufsize)358 pci_vtscsi_control_handle(struct pci_vtscsi_softc *sc, void *buf,
359 size_t bufsize)
360 {
361 struct pci_vtscsi_ctrl_tmf *tmf;
362 struct pci_vtscsi_ctrl_an *an;
363 uint32_t type;
364
365 if (bufsize < sizeof(uint32_t)) {
366 WPRINTF("ignoring truncated control request");
367 return (0);
368 }
369
370 type = *(uint32_t *)buf;
371
372 if (type == VIRTIO_SCSI_T_TMF) {
373 if (bufsize != sizeof(*tmf)) {
374 WPRINTF("ignoring tmf request with size %zu", bufsize);
375 return (0);
376 }
377 tmf = (struct pci_vtscsi_ctrl_tmf *)buf;
378 return (pci_vtscsi_tmf_handle(sc, tmf));
379 }
380
381 if (type == VIRTIO_SCSI_T_AN_QUERY) {
382 if (bufsize != sizeof(*an)) {
383 WPRINTF("ignoring AN request with size %zu", bufsize);
384 return (0);
385 }
386 an = (struct pci_vtscsi_ctrl_an *)buf;
387 return (pci_vtscsi_an_handle(sc, an));
388 }
389
390 return (0);
391 }
392
393 static int
pci_vtscsi_tmf_handle(struct pci_vtscsi_softc * sc,struct pci_vtscsi_ctrl_tmf * tmf)394 pci_vtscsi_tmf_handle(struct pci_vtscsi_softc *sc,
395 struct pci_vtscsi_ctrl_tmf *tmf)
396 {
397 union ctl_io *io;
398 int err;
399
400 io = ctl_scsi_alloc_io(sc->vss_iid);
401 ctl_scsi_zero_io(io);
402
403 io->io_hdr.io_type = CTL_IO_TASK;
404 io->io_hdr.nexus.initid = sc->vss_iid;
405 io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(tmf->lun);
406 io->taskio.tag_type = CTL_TAG_SIMPLE;
407 io->taskio.tag_num = tmf->id;
408 io->io_hdr.flags |= CTL_FLAG_USER_TAG;
409
410 switch (tmf->subtype) {
411 case VIRTIO_SCSI_T_TMF_ABORT_TASK:
412 io->taskio.task_action = CTL_TASK_ABORT_TASK;
413 break;
414
415 case VIRTIO_SCSI_T_TMF_ABORT_TASK_SET:
416 io->taskio.task_action = CTL_TASK_ABORT_TASK_SET;
417 break;
418
419 case VIRTIO_SCSI_T_TMF_CLEAR_ACA:
420 io->taskio.task_action = CTL_TASK_CLEAR_ACA;
421 break;
422
423 case VIRTIO_SCSI_T_TMF_CLEAR_TASK_SET:
424 io->taskio.task_action = CTL_TASK_CLEAR_TASK_SET;
425 break;
426
427 case VIRTIO_SCSI_T_TMF_I_T_NEXUS_RESET:
428 io->taskio.task_action = CTL_TASK_I_T_NEXUS_RESET;
429 break;
430
431 case VIRTIO_SCSI_T_TMF_LOGICAL_UNIT_RESET:
432 io->taskio.task_action = CTL_TASK_LUN_RESET;
433 break;
434
435 case VIRTIO_SCSI_T_TMF_QUERY_TASK:
436 io->taskio.task_action = CTL_TASK_QUERY_TASK;
437 break;
438
439 case VIRTIO_SCSI_T_TMF_QUERY_TASK_SET:
440 io->taskio.task_action = CTL_TASK_QUERY_TASK_SET;
441 break;
442 }
443
444 if (pci_vtscsi_debug) {
445 struct sbuf *sb = sbuf_new_auto();
446 ctl_io_sbuf(io, sb);
447 sbuf_finish(sb);
448 DPRINTF("%s", sbuf_data(sb));
449 sbuf_delete(sb);
450 }
451
452 err = ioctl(sc->vss_ctl_fd, CTL_IO, io);
453 if (err != 0)
454 WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno));
455
456 tmf->response = io->taskio.task_status;
457 ctl_scsi_free_io(io);
458 return (1);
459 }
460
461 static int
pci_vtscsi_an_handle(struct pci_vtscsi_softc * sc __unused,struct pci_vtscsi_ctrl_an * an __unused)462 pci_vtscsi_an_handle(struct pci_vtscsi_softc *sc __unused,
463 struct pci_vtscsi_ctrl_an *an __unused)
464 {
465 return (0);
466 }
467
468 static int
pci_vtscsi_request_handle(struct pci_vtscsi_queue * q,struct iovec * iov_in,int niov_in,struct iovec * iov_out,int niov_out)469 pci_vtscsi_request_handle(struct pci_vtscsi_queue *q, struct iovec *iov_in,
470 int niov_in, struct iovec *iov_out, int niov_out)
471 {
472 struct pci_vtscsi_softc *sc = q->vsq_sc;
473 struct pci_vtscsi_req_cmd_rd *cmd_rd = NULL;
474 struct pci_vtscsi_req_cmd_wr *cmd_wr;
475 struct iovec data_iov_in[VTSCSI_MAXSEG], data_iov_out[VTSCSI_MAXSEG];
476 union ctl_io *io;
477 int data_niov_in, data_niov_out;
478 void *ext_data_ptr = NULL;
479 uint32_t ext_data_len = 0, ext_sg_entries = 0;
480 int err, nxferred;
481
482 if (count_iov(iov_out, niov_out) < VTSCSI_OUT_HEADER_LEN(sc)) {
483 WPRINTF("ignoring request with insufficient output");
484 return (0);
485 }
486 if (count_iov(iov_in, niov_in) < VTSCSI_IN_HEADER_LEN(sc)) {
487 WPRINTF("ignoring request with incomplete header");
488 return (0);
489 }
490
491 seek_iov(iov_in, niov_in, data_iov_in, &data_niov_in,
492 VTSCSI_IN_HEADER_LEN(sc));
493 seek_iov(iov_out, niov_out, data_iov_out, &data_niov_out,
494 VTSCSI_OUT_HEADER_LEN(sc));
495
496 truncate_iov(iov_in, &niov_in, VTSCSI_IN_HEADER_LEN(sc));
497 truncate_iov(iov_out, &niov_out, VTSCSI_OUT_HEADER_LEN(sc));
498 iov_to_buf(iov_in, niov_in, (void **)&cmd_rd);
499
500 cmd_wr = calloc(1, VTSCSI_OUT_HEADER_LEN(sc));
501 io = ctl_scsi_alloc_io(sc->vss_iid);
502 ctl_scsi_zero_io(io);
503
504 io->io_hdr.nexus.initid = sc->vss_iid;
505 io->io_hdr.nexus.targ_lun = pci_vtscsi_get_lun(cmd_rd->lun);
506
507 io->io_hdr.io_type = CTL_IO_SCSI;
508
509 if (data_niov_in > 0) {
510 ext_data_ptr = (void *)data_iov_in;
511 ext_sg_entries = data_niov_in;
512 ext_data_len = count_iov(data_iov_in, data_niov_in);
513 io->io_hdr.flags |= CTL_FLAG_DATA_OUT;
514 } else if (data_niov_out > 0) {
515 ext_data_ptr = (void *)data_iov_out;
516 ext_sg_entries = data_niov_out;
517 ext_data_len = count_iov(data_iov_out, data_niov_out);
518 io->io_hdr.flags |= CTL_FLAG_DATA_IN;
519 }
520
521 io->scsiio.sense_len = sc->vss_config.sense_size;
522 io->scsiio.tag_num = cmd_rd->id;
523 io->io_hdr.flags |= CTL_FLAG_USER_TAG;
524 switch (cmd_rd->task_attr) {
525 case VIRTIO_SCSI_S_ORDERED:
526 io->scsiio.tag_type = CTL_TAG_ORDERED;
527 break;
528 case VIRTIO_SCSI_S_HEAD:
529 io->scsiio.tag_type = CTL_TAG_HEAD_OF_QUEUE;
530 break;
531 case VIRTIO_SCSI_S_ACA:
532 io->scsiio.tag_type = CTL_TAG_ACA;
533 break;
534 case VIRTIO_SCSI_S_SIMPLE:
535 default:
536 io->scsiio.tag_type = CTL_TAG_SIMPLE;
537 break;
538 }
539 io->scsiio.ext_sg_entries = ext_sg_entries;
540 io->scsiio.ext_data_ptr = ext_data_ptr;
541 io->scsiio.ext_data_len = ext_data_len;
542 io->scsiio.ext_data_filled = 0;
543 io->scsiio.cdb_len = sc->vss_config.cdb_size;
544 memcpy(io->scsiio.cdb, cmd_rd->cdb, sc->vss_config.cdb_size);
545
546 if (pci_vtscsi_debug) {
547 struct sbuf *sb = sbuf_new_auto();
548 ctl_io_sbuf(io, sb);
549 sbuf_finish(sb);
550 DPRINTF("%s", sbuf_data(sb));
551 sbuf_delete(sb);
552 }
553
554 err = ioctl(sc->vss_ctl_fd, CTL_IO, io);
555 if (err != 0) {
556 WPRINTF("CTL_IO: err=%d (%s)", errno, strerror(errno));
557 cmd_wr->response = VIRTIO_SCSI_S_FAILURE;
558 } else {
559 cmd_wr->sense_len = MIN(io->scsiio.sense_len,
560 sc->vss_config.sense_size);
561 cmd_wr->residual = ext_data_len - io->scsiio.ext_data_filled;
562 cmd_wr->status = io->scsiio.scsi_status;
563 cmd_wr->response = VIRTIO_SCSI_S_OK;
564 memcpy(&cmd_wr->sense, &io->scsiio.sense_data,
565 cmd_wr->sense_len);
566 }
567
568 buf_to_iov(cmd_wr, VTSCSI_OUT_HEADER_LEN(sc), iov_out, niov_out, 0);
569 nxferred = VTSCSI_OUT_HEADER_LEN(sc) + io->scsiio.ext_data_filled;
570 free(cmd_rd);
571 free(cmd_wr);
572 ctl_scsi_free_io(io);
573 return (nxferred);
574 }
575
576 static void
pci_vtscsi_controlq_notify(void * vsc,struct vqueue_info * vq)577 pci_vtscsi_controlq_notify(void *vsc, struct vqueue_info *vq)
578 {
579 struct pci_vtscsi_softc *sc;
580 struct iovec iov[VTSCSI_MAXSEG];
581 struct vi_req req;
582 void *buf = NULL;
583 size_t bufsize;
584 int iolen, n;
585
586 sc = vsc;
587
588 while (vq_has_descs(vq)) {
589 n = vq_getchain(vq, iov, VTSCSI_MAXSEG, &req);
590 assert(n >= 1 && n <= VTSCSI_MAXSEG);
591
592 bufsize = iov_to_buf(iov, n, &buf);
593 iolen = pci_vtscsi_control_handle(sc, buf, bufsize);
594 buf_to_iov((uint8_t *)buf + bufsize - iolen, iolen, iov, n,
595 bufsize - iolen);
596
597 /*
598 * Release this chain and handle more
599 */
600 vq_relchain(vq, req.idx, iolen);
601 }
602 vq_endchains(vq, 1); /* Generate interrupt if appropriate. */
603 free(buf);
604 }
605
606 static void
pci_vtscsi_eventq_notify(void * vsc __unused,struct vqueue_info * vq)607 pci_vtscsi_eventq_notify(void *vsc __unused, struct vqueue_info *vq)
608 {
609 vq_kick_disable(vq);
610 }
611
612 static void
pci_vtscsi_requestq_notify(void * vsc,struct vqueue_info * vq)613 pci_vtscsi_requestq_notify(void *vsc, struct vqueue_info *vq)
614 {
615 struct pci_vtscsi_softc *sc;
616 struct pci_vtscsi_queue *q;
617 struct pci_vtscsi_request *req;
618 struct iovec iov[VTSCSI_MAXSEG];
619 struct vi_req vireq;
620 int n;
621
622 sc = vsc;
623 q = &sc->vss_queues[vq->vq_num - 2];
624
625 while (vq_has_descs(vq)) {
626 n = vq_getchain(vq, iov, VTSCSI_MAXSEG, &vireq);
627 assert(n >= 1 && n <= VTSCSI_MAXSEG);
628
629 req = calloc(1, sizeof(struct pci_vtscsi_request));
630 req->vsr_idx = vireq.idx;
631 req->vsr_queue = q;
632 req->vsr_niov_in = vireq.readable;
633 req->vsr_niov_out = vireq.writable;
634 memcpy(req->vsr_iov_in, iov,
635 req->vsr_niov_in * sizeof(struct iovec));
636 memcpy(req->vsr_iov_out, iov + vireq.readable,
637 req->vsr_niov_out * sizeof(struct iovec));
638
639 pthread_mutex_lock(&q->vsq_mtx);
640 STAILQ_INSERT_TAIL(&q->vsq_requests, req, vsr_link);
641 pthread_cond_signal(&q->vsq_cv);
642 pthread_mutex_unlock(&q->vsq_mtx);
643
644 DPRINTF("request <idx=%d> enqueued", vireq.idx);
645 }
646 }
647
648 static int
pci_vtscsi_init_queue(struct pci_vtscsi_softc * sc,struct pci_vtscsi_queue * queue,int num)649 pci_vtscsi_init_queue(struct pci_vtscsi_softc *sc,
650 struct pci_vtscsi_queue *queue, int num)
651 {
652 struct pci_vtscsi_worker *worker;
653 char tname[MAXCOMLEN + 1];
654 int i;
655
656 queue->vsq_sc = sc;
657 queue->vsq_vq = &sc->vss_vq[num + 2];
658
659 pthread_mutex_init(&queue->vsq_mtx, NULL);
660 pthread_mutex_init(&queue->vsq_qmtx, NULL);
661 pthread_cond_init(&queue->vsq_cv, NULL);
662 STAILQ_INIT(&queue->vsq_requests);
663 LIST_INIT(&queue->vsq_workers);
664
665 for (i = 0; i < VTSCSI_THR_PER_Q; i++) {
666 worker = calloc(1, sizeof(struct pci_vtscsi_worker));
667 worker->vsw_queue = queue;
668
669 pthread_create(&worker->vsw_thread, NULL, &pci_vtscsi_proc,
670 (void *)worker);
671
672 snprintf(tname, sizeof(tname), "vtscsi:%d-%d", num, i);
673 pthread_set_name_np(worker->vsw_thread, tname);
674 LIST_INSERT_HEAD(&queue->vsq_workers, worker, vsw_link);
675 }
676
677 return (0);
678 }
679
680 static int
pci_vtscsi_legacy_config(nvlist_t * nvl,const char * opts)681 pci_vtscsi_legacy_config(nvlist_t *nvl, const char *opts)
682 {
683 char *cp, *devname;
684
685 if (opts == NULL)
686 return (0);
687
688 cp = strchr(opts, ',');
689 if (cp == NULL) {
690 set_config_value_node(nvl, "dev", opts);
691 return (0);
692 }
693 devname = strndup(opts, cp - opts);
694 set_config_value_node(nvl, "dev", devname);
695 free(devname);
696 return (pci_parse_legacy_config(nvl, cp + 1));
697 }
698
699 static int
pci_vtscsi_init(struct pci_devinst * pi,nvlist_t * nvl)700 pci_vtscsi_init(struct pci_devinst *pi, nvlist_t *nvl)
701 {
702 struct pci_vtscsi_softc *sc;
703 const char *devname, *value;
704 int i;
705
706 sc = calloc(1, sizeof(struct pci_vtscsi_softc));
707 value = get_config_value_node(nvl, "iid");
708 if (value != NULL)
709 sc->vss_iid = strtoul(value, NULL, 10);
710
711 value = get_config_value_node(nvl, "bootindex");
712 if (value != NULL) {
713 if (pci_emul_add_boot_device(pi, atoi(value))) {
714 EPRINTLN("Invalid bootindex %d", atoi(value));
715 free(sc);
716 return (-1);
717 }
718 }
719
720 devname = get_config_value_node(nvl, "dev");
721 if (devname == NULL)
722 devname = "/dev/cam/ctl";
723 sc->vss_ctl_fd = open(devname, O_RDWR);
724 if (sc->vss_ctl_fd < 0) {
725 WPRINTF("cannot open %s: %s", devname, strerror(errno));
726 free(sc);
727 return (1);
728 }
729
730 pthread_mutex_init(&sc->vss_mtx, NULL);
731
732 vi_softc_linkup(&sc->vss_vs, &vtscsi_vi_consts, sc, pi, sc->vss_vq);
733 sc->vss_vs.vs_mtx = &sc->vss_mtx;
734
735 /* controlq */
736 sc->vss_vq[0].vq_qsize = VTSCSI_RINGSZ;
737 sc->vss_vq[0].vq_notify = pci_vtscsi_controlq_notify;
738
739 /* eventq */
740 sc->vss_vq[1].vq_qsize = VTSCSI_RINGSZ;
741 sc->vss_vq[1].vq_notify = pci_vtscsi_eventq_notify;
742
743 /* request queues */
744 for (i = 2; i < VTSCSI_MAXQ; i++) {
745 sc->vss_vq[i].vq_qsize = VTSCSI_RINGSZ;
746 sc->vss_vq[i].vq_notify = pci_vtscsi_requestq_notify;
747 pci_vtscsi_init_queue(sc, &sc->vss_queues[i - 2], i - 2);
748 }
749
750 /* initialize config space */
751 pci_set_cfgdata16(pi, PCIR_DEVICE, VIRTIO_DEV_SCSI);
752 pci_set_cfgdata16(pi, PCIR_VENDOR, VIRTIO_VENDOR);
753 pci_set_cfgdata8(pi, PCIR_CLASS, PCIC_STORAGE);
754 pci_set_cfgdata16(pi, PCIR_SUBDEV_0, VIRTIO_ID_SCSI);
755 pci_set_cfgdata16(pi, PCIR_SUBVEND_0, VIRTIO_VENDOR);
756
757 if (vi_intr_init(&sc->vss_vs, 1, fbsdrun_virtio_msix()))
758 return (1);
759 vi_set_io_bar(&sc->vss_vs, 0);
760
761 return (0);
762 }
763
764
765 static const struct pci_devemu pci_de_vscsi = {
766 .pe_emu = "virtio-scsi",
767 .pe_init = pci_vtscsi_init,
768 .pe_legacy_config = pci_vtscsi_legacy_config,
769 .pe_barwrite = vi_pci_write,
770 .pe_barread = vi_pci_read
771 };
772 PCI_EMUL_SET(pci_de_vscsi);
773