xref: /illumos-gate/usr/src/uts/common/xen/os/xvdi.c (revision 97869ac5)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * Xen virtual device driver interfaces
31  */
32 
33 /*
34  * todo:
35  * + name space clean up:
36  *	xvdi_* - public xen interfaces, for use by all leaf drivers
37  *	xd_* - public xen data structures
38  *	i_xvdi_* - implementation private functions
39  *	xendev_* - xendev driver interfaces, both internal and in cb_ops/bus_ops
40  * + add mdb dcmds to dump ring status
41  * + implement xvdi_xxx to wrap xenbus_xxx read/write function
42  * + convert (xendev_ring_t *) into xvdi_ring_handle_t
43  */
44 #include <sys/conf.h>
45 #include <sys/param.h>
46 #include <sys/kmem.h>
47 #include <vm/seg_kmem.h>
48 #include <sys/debug.h>
49 #include <sys/modctl.h>
50 #include <sys/autoconf.h>
51 #include <sys/ddi_impldefs.h>
52 #include <sys/ddi_subrdefs.h>
53 #include <sys/ddi.h>
54 #include <sys/sunddi.h>
55 #include <sys/sunndi.h>
56 #include <sys/sunldi.h>
57 #include <sys/fs/dv_node.h>
58 #include <sys/avintr.h>
59 #include <sys/psm.h>
60 #include <sys/spl.h>
61 #include <sys/promif.h>
62 #include <sys/list.h>
63 #include <sys/bootconf.h>
64 #include <sys/bootsvcs.h>
65 #include <sys/bootinfo.h>
66 #include <sys/note.h>
67 #ifdef XPV_HVM_DRIVER
68 #include <sys/xpv_support.h>
69 #include <sys/hypervisor.h>
70 #include <public/grant_table.h>
71 #include <public/xen.h>
72 #include <public/io/xenbus.h>
73 #include <public/io/xs_wire.h>
74 #include <public/event_channel.h>
75 #include <public/io/xenbus.h>
76 #else /* XPV_HVM_DRIVER */
77 #include <sys/hypervisor.h>
78 #include <sys/xen_mmu.h>
79 #include <xen/sys/xenbus_impl.h>
80 #include <sys/evtchn_impl.h>
81 #endif /* XPV_HVM_DRIVER */
82 #include <sys/gnttab.h>
83 #include <xen/sys/xendev.h>
84 #include <vm/hat_i86.h>
85 #include <sys/scsi/generic/inquiry.h>
86 #include <util/sscanf.h>
87 #include <xen/public/io/xs_wire.h>
88 
89 
90 #define	isdigit(ch)	((ch) >= '0' && (ch) <= '9')
91 #define	isxdigit(ch)	(isdigit(ch) || ((ch) >= 'a' && (ch) <= 'f') || \
92 			((ch) >= 'A' && (ch) <= 'F'))
93 
94 static void xvdi_ring_init_sring(xendev_ring_t *);
95 static void xvdi_ring_init_front_ring(xendev_ring_t *, size_t, size_t);
96 #ifndef XPV_HVM_DRIVER
97 static void xvdi_ring_init_back_ring(xendev_ring_t *, size_t, size_t);
98 #endif
99 static void xvdi_reinit_ring(dev_info_t *, grant_ref_t *, xendev_ring_t *);
100 
101 static int i_xvdi_add_watches(dev_info_t *);
102 static void i_xvdi_rem_watches(dev_info_t *);
103 
104 static int i_xvdi_add_watch_oestate(dev_info_t *);
105 static void i_xvdi_rem_watch_oestate(dev_info_t *);
106 static void i_xvdi_oestate_cb(struct xenbus_device *, XenbusState);
107 static void i_xvdi_oestate_handler(void *);
108 
109 static int i_xvdi_add_watch_hpstate(dev_info_t *);
110 static void i_xvdi_rem_watch_hpstate(dev_info_t *);
111 static void i_xvdi_hpstate_cb(struct xenbus_watch *, const char **,
112     unsigned int);
113 static void i_xvdi_hpstate_handler(void *);
114 
115 static int i_xvdi_add_watch_bepath(dev_info_t *);
116 static void i_xvdi_rem_watch_bepath(dev_info_t *);
117 static void i_xvdi_bepath_cb(struct xenbus_watch *, const char **,
118     unsigned in);
119 
120 static void xendev_offline_device(void *);
121 
122 static void i_xvdi_probe_path_cb(struct xenbus_watch *, const char **,
123     unsigned int);
124 static void i_xvdi_probe_path_handler(void *);
125 
126 typedef struct xd_cfg {
127 	xendev_devclass_t devclass;
128 	char *xsdev;
129 	char *xs_path_fe;
130 	char *xs_path_be;
131 	char *node_fe;
132 	char *node_be;
133 	char *device_type;
134 	int xd_ipl;
135 	int flags;
136 } i_xd_cfg_t;
137 
138 #define	XD_DOM_ZERO	0x01	/* dom0 only. */
139 #define	XD_DOM_GUEST	0x02	/* Guest domains (i.e. non-dom0). */
140 #define	XD_DOM_IO	0x04	/* IO domains. */
141 
142 #define	XD_DOM_ALL	(XD_DOM_ZERO | XD_DOM_GUEST)
143 
144 static i_xd_cfg_t xdci[] = {
145 	{ XEN_CONSOLE, NULL, NULL, NULL, "xencons", NULL,
146 	    "console", IPL_CONS, XD_DOM_ALL, },
147 
148 	{ XEN_VNET, "vif", "device/vif", "backend/vif", "xnf", "xnb",
149 	    "network", IPL_VIF, XD_DOM_ALL, },
150 
151 	{ XEN_VBLK, "vbd", "device/vbd", "backend/vbd", "xdf", "xdb",
152 	    "block", IPL_VBD, XD_DOM_ALL, },
153 
154 	{ XEN_XENBUS, NULL, NULL, NULL, "xenbus", NULL,
155 	    NULL, 0, XD_DOM_ALL, },
156 
157 	{ XEN_DOMCAPS, NULL, NULL, NULL, "domcaps", NULL,
158 	    NULL, 0, XD_DOM_ALL, },
159 
160 	{ XEN_BALLOON, NULL, NULL, NULL, "balloon", NULL,
161 	    NULL, 0, XD_DOM_ALL, },
162 
163 	{ XEN_EVTCHN, NULL, NULL, NULL, "evtchn", NULL,
164 	    NULL, 0, XD_DOM_ZERO, },
165 
166 	{ XEN_PRIVCMD, NULL, NULL, NULL, "privcmd", NULL,
167 	    NULL, 0, XD_DOM_ZERO, },
168 };
169 #define	NXDC	(sizeof (xdci) / sizeof (xdci[0]))
170 
171 static void i_xvdi_enum_fe(dev_info_t *, i_xd_cfg_t *);
172 static void i_xvdi_enum_be(dev_info_t *, i_xd_cfg_t *);
173 static void i_xvdi_enum_worker(dev_info_t *, i_xd_cfg_t *, char *);
174 
175 /*
176  * Xen device channel device access and DMA attributes
177  */
178 static ddi_device_acc_attr_t xendev_dc_accattr = {
179 	DDI_DEVICE_ATTR_V0, DDI_NEVERSWAP_ACC, DDI_STRICTORDER_ACC
180 };
181 
182 static ddi_dma_attr_t xendev_dc_dmaattr = {
183 	DMA_ATTR_V0,		/* version of this structure */
184 	0,			/* lowest usable address */
185 	0xffffffffffffffffULL,	/* highest usable address */
186 	0x7fffffff,		/* maximum DMAable byte count */
187 	MMU_PAGESIZE,		/* alignment in bytes */
188 	0x7ff,			/* bitmap of burst sizes */
189 	1,			/* minimum transfer */
190 	0xffffffffU,		/* maximum transfer */
191 	0xffffffffffffffffULL,	/* maximum segment length */
192 	1,			/* maximum number of segments */
193 	1,			/* granularity */
194 	0,			/* flags (reserved) */
195 };
196 
197 static dev_info_t *xendev_dip = NULL;
198 
199 #define	XVDI_DBG_STATE	0x01
200 #define	XVDI_DBG_PROBE	0x02
201 
202 #ifdef DEBUG
203 int i_xvdi_debug = 0;
204 
205 #define	XVDI_DPRINTF(flag, format, ...)			\
206 {							\
207 	if (i_xvdi_debug & (flag))			\
208 		prom_printf((format), __VA_ARGS__);	\
209 }
210 #else
211 #define	XVDI_DPRINTF(flag, format, ...)
212 #endif /* DEBUG */
213 
214 static i_xd_cfg_t *
215 i_xvdi_devclass2cfg(xendev_devclass_t devclass)
216 {
217 	i_xd_cfg_t *xdcp;
218 	int i;
219 
220 	for (i = 0, xdcp = xdci; i < NXDC; i++, xdcp++)
221 		if (xdcp->devclass == devclass)
222 			return (xdcp);
223 
224 	return (NULL);
225 }
226 
227 int
228 xvdi_init_dev(dev_info_t *dip)
229 {
230 	xendev_devclass_t devcls;
231 	int vdevnum;
232 	domid_t domid;
233 	struct xendev_ppd *pdp;
234 	i_xd_cfg_t *xdcp;
235 	boolean_t backend;
236 	char xsnamebuf[TYPICALMAXPATHLEN];
237 	char *xsname;
238 	void *prop_str;
239 	unsigned int prop_len, addr;
240 	char unitaddr[8];
241 
242 	devcls = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
243 	    DDI_PROP_DONTPASS, "devclass", XEN_INVAL);
244 	vdevnum = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
245 	    DDI_PROP_DONTPASS, "vdev", VDEV_NOXS);
246 	domid = (domid_t)ddi_prop_get_int(DDI_DEV_T_ANY, dip,
247 	    DDI_PROP_DONTPASS, "domain", DOMID_SELF);
248 
249 	backend = (domid != DOMID_SELF);
250 	xdcp = i_xvdi_devclass2cfg(devcls);
251 	if (xdcp->device_type != NULL)
252 		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
253 		    "device_type", xdcp->device_type);
254 
255 	pdp = kmem_zalloc(sizeof (*pdp), KM_SLEEP);
256 	pdp->xd_domain = domid;
257 	pdp->xd_vdevnum = vdevnum;
258 	pdp->xd_devclass = devcls;
259 	pdp->xd_evtchn = INVALID_EVTCHN;
260 	mutex_init(&pdp->xd_lk, NULL, MUTEX_DRIVER, NULL);
261 	ddi_set_parent_data(dip, pdp);
262 
263 	/*
264 	 * devices that do not need to interact with xenstore
265 	 */
266 	if (vdevnum == VDEV_NOXS) {
267 		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
268 		    "unit-address", "0");
269 		if (devcls == XEN_CONSOLE)
270 			(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
271 			    "pm-hardware-state", "needs-suspend-resume");
272 		return (DDI_SUCCESS);
273 	}
274 
275 	/*
276 	 * PV devices that need to probe xenstore
277 	 */
278 
279 	(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
280 	    "pm-hardware-state", "needs-suspend-resume");
281 
282 	xsname = xsnamebuf;
283 	if (!backend)
284 		(void) snprintf(xsnamebuf, sizeof (xsnamebuf),
285 		    "%s/%d", xdcp->xs_path_fe, vdevnum);
286 	else
287 		(void) snprintf(xsnamebuf, sizeof (xsnamebuf),
288 		    "%s/%d/%d", xdcp->xs_path_be, domid, vdevnum);
289 	if ((xenbus_read_driver_state(xsname) >= XenbusStateClosing)) {
290 		/* Don't try to init a dev that may be closing */
291 		mutex_destroy(&pdp->xd_lk);
292 		kmem_free(pdp, sizeof (*pdp));
293 		ddi_set_parent_data(dip, NULL);
294 		return (DDI_FAILURE);
295 	}
296 
297 	pdp->xd_xsdev.nodename = i_ddi_strdup(xsname, KM_SLEEP);
298 	pdp->xd_xsdev.devicetype = xdcp->xsdev;
299 	pdp->xd_xsdev.frontend = (backend ? 0 : 1);
300 	pdp->xd_xsdev.data = dip;
301 	pdp->xd_xsdev.otherend_id = (backend ? domid : -1);
302 	if (i_xvdi_add_watches(dip) != DDI_SUCCESS) {
303 		cmn_err(CE_WARN, "xvdi_init_dev: "
304 		    "cannot add watches for %s", xsname);
305 		xvdi_uninit_dev(dip);
306 		return (DDI_FAILURE);
307 	}
308 
309 	if (backend)
310 		return (DDI_SUCCESS);
311 
312 	/*
313 	 * The unit-address for frontend devices is the name of the
314 	 * of the xenstore node containing the device configuration
315 	 * and is contained in the 'vdev' property.
316 	 * VIF devices are named using an incrementing integer.
317 	 * VBD devices are either named using the 16-bit dev_t value
318 	 * for linux 'hd' and 'xvd' devices, or a simple integer value
319 	 * in the range 0..767.  768 is the base value of the linux
320 	 * dev_t namespace, the dev_t value for 'hda'.
321 	 */
322 	(void) snprintf(unitaddr, sizeof (unitaddr), "%d", vdevnum);
323 	(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, "unit-address",
324 	    unitaddr);
325 
326 	switch (devcls) {
327 	case XEN_VNET:
328 		if (xenbus_read(XBT_NULL, xsname, "mac", (void *)&prop_str,
329 		    &prop_len) != 0)
330 			break;
331 		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, "mac",
332 		    prop_str);
333 		kmem_free(prop_str, prop_len);
334 		break;
335 	case XEN_VBLK:
336 		/*
337 		 * cache a copy of the otherend name
338 		 * for ease of observeability
339 		 */
340 		if (xenbus_read(XBT_NULL, pdp->xd_xsdev.otherend, "dev",
341 		    &prop_str, &prop_len) != 0)
342 			break;
343 		(void) ndi_prop_update_string(DDI_DEV_T_NONE, dip,
344 		    "dev-address", prop_str);
345 		kmem_free(prop_str, prop_len);
346 		break;
347 	default:
348 		break;
349 	}
350 
351 	return (DDI_SUCCESS);
352 }
353 
354 void
355 xvdi_uninit_dev(dev_info_t *dip)
356 {
357 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
358 
359 	if (pdp != NULL) {
360 		/* Remove any registered callbacks. */
361 		xvdi_remove_event_handler(dip, NULL);
362 
363 		/* Remove any registered watches. */
364 		i_xvdi_rem_watches(dip);
365 
366 		/* tell other end to close */
367 		if (pdp->xd_xsdev.otherend_id != (domid_t)-1)
368 			(void) xvdi_switch_state(dip, XBT_NULL,
369 			    XenbusStateClosed);
370 
371 		if (pdp->xd_xsdev.nodename != NULL)
372 			kmem_free((char *)(pdp->xd_xsdev.nodename),
373 			    strlen(pdp->xd_xsdev.nodename) + 1);
374 
375 		ddi_set_parent_data(dip, NULL);
376 
377 		mutex_destroy(&pdp->xd_lk);
378 		kmem_free(pdp, sizeof (*pdp));
379 	}
380 }
381 
382 /*
383  * Bind the event channel for this device instance.
384  * Currently we only support one evtchn per device instance.
385  */
386 int
387 xvdi_bind_evtchn(dev_info_t *dip, evtchn_port_t evtchn)
388 {
389 	struct xendev_ppd *pdp;
390 	domid_t oeid;
391 	int r;
392 
393 	pdp = ddi_get_parent_data(dip);
394 	ASSERT(pdp != NULL);
395 	ASSERT(pdp->xd_evtchn == INVALID_EVTCHN);
396 
397 	mutex_enter(&pdp->xd_lk);
398 	if (pdp->xd_devclass == XEN_CONSOLE) {
399 		if (!DOMAIN_IS_INITDOMAIN(xen_info)) {
400 			pdp->xd_evtchn = xen_info->console.domU.evtchn;
401 		} else {
402 			pdp->xd_evtchn = INVALID_EVTCHN;
403 			mutex_exit(&pdp->xd_lk);
404 			return (DDI_SUCCESS);
405 		}
406 	} else {
407 		oeid = pdp->xd_xsdev.otherend_id;
408 		if (oeid == (domid_t)-1) {
409 			mutex_exit(&pdp->xd_lk);
410 			return (DDI_FAILURE);
411 		}
412 
413 		if ((r = xen_bind_interdomain(oeid, evtchn, &pdp->xd_evtchn))) {
414 			xvdi_dev_error(dip, r, "bind event channel");
415 			mutex_exit(&pdp->xd_lk);
416 			return (DDI_FAILURE);
417 		}
418 	}
419 #ifndef XPV_HVM_DRIVER
420 	pdp->xd_ispec.intrspec_vec = ec_bind_evtchn_to_irq(pdp->xd_evtchn);
421 #endif
422 	mutex_exit(&pdp->xd_lk);
423 
424 	return (DDI_SUCCESS);
425 }
426 
427 /*
428  * Allocate an event channel for this device instance.
429  * Currently we only support one evtchn per device instance.
430  */
431 int
432 xvdi_alloc_evtchn(dev_info_t *dip)
433 {
434 	struct xendev_ppd *pdp;
435 	domid_t oeid;
436 	int rv;
437 
438 	pdp = ddi_get_parent_data(dip);
439 	ASSERT(pdp != NULL);
440 	ASSERT(pdp->xd_evtchn == INVALID_EVTCHN);
441 
442 	mutex_enter(&pdp->xd_lk);
443 	if (pdp->xd_devclass == XEN_CONSOLE) {
444 		if (!DOMAIN_IS_INITDOMAIN(xen_info)) {
445 			pdp->xd_evtchn = xen_info->console.domU.evtchn;
446 		} else {
447 			pdp->xd_evtchn = INVALID_EVTCHN;
448 			mutex_exit(&pdp->xd_lk);
449 			return (DDI_SUCCESS);
450 		}
451 	} else {
452 		oeid = pdp->xd_xsdev.otherend_id;
453 		if (oeid == (domid_t)-1) {
454 			mutex_exit(&pdp->xd_lk);
455 			return (DDI_FAILURE);
456 		}
457 
458 		if ((rv = xen_alloc_unbound_evtchn(oeid, &pdp->xd_evtchn))) {
459 			xvdi_dev_error(dip, rv, "bind event channel");
460 			mutex_exit(&pdp->xd_lk);
461 			return (DDI_FAILURE);
462 		}
463 	}
464 #ifndef XPV_HVM_DRIVER
465 	pdp->xd_ispec.intrspec_vec = ec_bind_evtchn_to_irq(pdp->xd_evtchn);
466 #endif
467 	mutex_exit(&pdp->xd_lk);
468 
469 	return (DDI_SUCCESS);
470 }
471 
472 /*
473  * Unbind the event channel for this device instance.
474  * Currently we only support one evtchn per device instance.
475  */
476 void
477 xvdi_free_evtchn(dev_info_t *dip)
478 {
479 	struct xendev_ppd *pdp;
480 
481 	pdp = ddi_get_parent_data(dip);
482 	ASSERT(pdp != NULL);
483 
484 	mutex_enter(&pdp->xd_lk);
485 	if (pdp->xd_evtchn != INVALID_EVTCHN) {
486 #ifndef XPV_HVM_DRIVER
487 		ec_unbind_irq(pdp->xd_ispec.intrspec_vec);
488 		pdp->xd_ispec.intrspec_vec = 0;
489 #endif
490 		pdp->xd_evtchn = INVALID_EVTCHN;
491 	}
492 	mutex_exit(&pdp->xd_lk);
493 }
494 
495 #ifndef XPV_HVM_DRIVER
496 /*
497  * Map an inter-domain communication ring for a virtual device.
498  * This is used by backend drivers.
499  */
500 int
501 xvdi_map_ring(dev_info_t *dip, size_t nentry, size_t entrysize,
502     grant_ref_t gref, xendev_ring_t **ringpp)
503 {
504 	domid_t oeid;
505 	gnttab_map_grant_ref_t mapop;
506 	gnttab_unmap_grant_ref_t unmapop;
507 	caddr_t ringva;
508 	ddi_acc_hdl_t *ap;
509 	ddi_acc_impl_t *iap;
510 	xendev_ring_t *ring;
511 	int err;
512 	char errstr[] = "mapping in ring buffer";
513 
514 	ring = kmem_zalloc(sizeof (xendev_ring_t), KM_SLEEP);
515 	oeid = xvdi_get_oeid(dip);
516 
517 	/* alloc va in backend dom for ring buffer */
518 	ringva = vmem_xalloc(heap_arena, PAGESIZE, PAGESIZE,
519 	    0, 0, 0, 0, VM_SLEEP);
520 
521 	/* map in ring page */
522 	hat_prepare_mapping(kas.a_hat, ringva);
523 	mapop.host_addr = (uint64_t)(uintptr_t)ringva;
524 	mapop.flags = GNTMAP_host_map;
525 	mapop.ref = gref;
526 	mapop.dom = oeid;
527 	err = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, &mapop, 1);
528 	if (err) {
529 		xvdi_fatal_error(dip, err, errstr);
530 		goto errout1;
531 	}
532 
533 	if (mapop.status != 0) {
534 		xvdi_fatal_error(dip, err, errstr);
535 		goto errout2;
536 	}
537 	ring->xr_vaddr = ringva;
538 	ring->xr_grant_hdl = mapop.handle;
539 	ring->xr_gref = gref;
540 
541 	/*
542 	 * init an acc handle and associate it w/ this ring
543 	 * this is only for backend drivers. we get the memory by calling
544 	 * vmem_xalloc(), instead of calling any ddi function, so we have
545 	 * to init an acc handle by ourselves
546 	 */
547 	ring->xr_acc_hdl = impl_acc_hdl_alloc(KM_SLEEP, NULL);
548 	ap = impl_acc_hdl_get(ring->xr_acc_hdl);
549 	ap->ah_vers = VERS_ACCHDL;
550 	ap->ah_dip = dip;
551 	ap->ah_xfermodes = DDI_DMA_CONSISTENT;
552 	ap->ah_acc = xendev_dc_accattr;
553 	iap = (ddi_acc_impl_t *)ap->ah_platform_private;
554 	iap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
555 	impl_acc_hdl_init(ap);
556 	ap->ah_offset = 0;
557 	ap->ah_len = (off_t)PAGESIZE;
558 	ap->ah_addr = ring->xr_vaddr;
559 
560 	/* init backend ring */
561 	xvdi_ring_init_back_ring(ring, nentry, entrysize);
562 
563 	*ringpp = ring;
564 
565 	return (DDI_SUCCESS);
566 
567 errout2:
568 	/* unmap ring page */
569 	unmapop.host_addr = (uint64_t)(uintptr_t)ringva;
570 	unmapop.handle = ring->xr_grant_hdl;
571 	unmapop.dev_bus_addr = NULL;
572 	(void) HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, &unmapop, 1);
573 	hat_release_mapping(kas.a_hat, ringva);
574 errout1:
575 	vmem_xfree(heap_arena, ringva, PAGESIZE);
576 	kmem_free(ring, sizeof (xendev_ring_t));
577 	return (DDI_FAILURE);
578 }
579 
580 /*
581  * Unmap a ring for a virtual device.
582  * This is used by backend drivers.
583  */
584 void
585 xvdi_unmap_ring(xendev_ring_t *ring)
586 {
587 	gnttab_unmap_grant_ref_t unmapop;
588 
589 	ASSERT((ring != NULL) && (ring->xr_vaddr != NULL));
590 
591 	impl_acc_hdl_free(ring->xr_acc_hdl);
592 	unmapop.host_addr = (uint64_t)(uintptr_t)ring->xr_vaddr;
593 	unmapop.handle = ring->xr_grant_hdl;
594 	unmapop.dev_bus_addr = NULL;
595 	(void) HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, &unmapop, 1);
596 	hat_release_mapping(kas.a_hat, ring->xr_vaddr);
597 	vmem_xfree(heap_arena, ring->xr_vaddr, PAGESIZE);
598 	kmem_free(ring, sizeof (xendev_ring_t));
599 }
600 #endif /* XPV_HVM_DRIVER */
601 
602 /*
603  * Re-initialise an inter-domain communications ring for the backend domain.
604  * ring will be re-initialized after re-grant succeed
605  * ring will be freed if fails to re-grant access to backend domain
606  * so, don't keep useful data in the ring
607  * used only in frontend driver
608  */
609 static void
610 xvdi_reinit_ring(dev_info_t *dip, grant_ref_t *gref, xendev_ring_t *ringp)
611 {
612 	paddr_t rpaddr;
613 	maddr_t rmaddr;
614 
615 	ASSERT((ringp != NULL) && (ringp->xr_paddr != 0));
616 	rpaddr = ringp->xr_paddr;
617 
618 	rmaddr = DOMAIN_IS_INITDOMAIN(xen_info) ? rpaddr : pa_to_ma(rpaddr);
619 	gnttab_grant_foreign_access_ref(ringp->xr_gref, xvdi_get_oeid(dip),
620 	    rmaddr >> PAGESHIFT, 0);
621 	*gref = ringp->xr_gref;
622 
623 	/* init frontend ring */
624 	xvdi_ring_init_sring(ringp);
625 	xvdi_ring_init_front_ring(ringp, ringp->xr_sring.fr.nr_ents,
626 	    ringp->xr_entry_size);
627 }
628 
629 /*
630  * allocate Xen inter-domain communications ring for Xen virtual devices
631  * used only in frontend driver
632  * if *ringpp is not NULL, we'll simply re-init it
633  */
634 int
635 xvdi_alloc_ring(dev_info_t *dip, size_t nentry, size_t entrysize,
636     grant_ref_t *gref, xendev_ring_t **ringpp)
637 {
638 	size_t len;
639 	xendev_ring_t *ring;
640 	ddi_dma_cookie_t dma_cookie;
641 	uint_t ncookies;
642 	grant_ref_t ring_gref;
643 	domid_t oeid;
644 	maddr_t rmaddr;
645 
646 	if (*ringpp) {
647 		xvdi_reinit_ring(dip, gref, *ringpp);
648 		return (DDI_SUCCESS);
649 	}
650 
651 	*ringpp = ring = kmem_zalloc(sizeof (xendev_ring_t), KM_SLEEP);
652 	oeid = xvdi_get_oeid(dip);
653 
654 	/*
655 	 * Allocate page for this ring buffer
656 	 */
657 	if (ddi_dma_alloc_handle(dip, &xendev_dc_dmaattr, DDI_DMA_SLEEP,
658 	    0, &ring->xr_dma_hdl) != DDI_SUCCESS)
659 		goto err;
660 
661 	if (ddi_dma_mem_alloc(ring->xr_dma_hdl, PAGESIZE,
662 	    &xendev_dc_accattr, DDI_DMA_CONSISTENT, DDI_DMA_SLEEP, 0,
663 	    &ring->xr_vaddr, &len, &ring->xr_acc_hdl) != DDI_SUCCESS) {
664 		ddi_dma_free_handle(&ring->xr_dma_hdl);
665 		goto err;
666 	}
667 
668 	if (ddi_dma_addr_bind_handle(ring->xr_dma_hdl, NULL,
669 	    ring->xr_vaddr, len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
670 	    DDI_DMA_SLEEP, 0, &dma_cookie, &ncookies) != DDI_DMA_MAPPED) {
671 		ddi_dma_mem_free(&ring->xr_acc_hdl);
672 		ring->xr_vaddr = NULL;
673 		ddi_dma_free_handle(&ring->xr_dma_hdl);
674 		goto err;
675 	}
676 	ASSERT(ncookies == 1);
677 	ring->xr_paddr = dma_cookie.dmac_laddress;
678 	rmaddr = DOMAIN_IS_INITDOMAIN(xen_info) ? ring->xr_paddr :
679 	    pa_to_ma(ring->xr_paddr);
680 
681 	if ((ring_gref = gnttab_grant_foreign_access(oeid,
682 	    rmaddr >> PAGESHIFT, 0)) == (grant_ref_t)-1) {
683 		(void) ddi_dma_unbind_handle(ring->xr_dma_hdl);
684 		ddi_dma_mem_free(&ring->xr_acc_hdl);
685 		ring->xr_vaddr = NULL;
686 		ddi_dma_free_handle(&ring->xr_dma_hdl);
687 		goto err;
688 	}
689 	*gref = ring->xr_gref = ring_gref;
690 
691 	/* init frontend ring */
692 	xvdi_ring_init_sring(ring);
693 	xvdi_ring_init_front_ring(ring, nentry, entrysize);
694 
695 	return (DDI_SUCCESS);
696 
697 err:
698 	kmem_free(ring, sizeof (xendev_ring_t));
699 	return (DDI_FAILURE);
700 }
701 
702 /*
703  * Release ring buffers allocated for Xen devices
704  * used for frontend driver
705  */
706 void
707 xvdi_free_ring(xendev_ring_t *ring)
708 {
709 	ASSERT((ring != NULL) && (ring->xr_vaddr != NULL));
710 
711 	(void) gnttab_end_foreign_access_ref(ring->xr_gref, 0);
712 	(void) ddi_dma_unbind_handle(ring->xr_dma_hdl);
713 	ddi_dma_mem_free(&ring->xr_acc_hdl);
714 	ddi_dma_free_handle(&ring->xr_dma_hdl);
715 	kmem_free(ring, sizeof (xendev_ring_t));
716 }
717 
718 dev_info_t *
719 xvdi_create_dev(dev_info_t *parent, xendev_devclass_t devclass,
720     domid_t dom, int vdev)
721 {
722 	dev_info_t *dip;
723 	boolean_t backend;
724 	i_xd_cfg_t *xdcp;
725 	char xsnamebuf[TYPICALMAXPATHLEN];
726 	char *type, *node = NULL, *xsname = NULL;
727 	unsigned int tlen;
728 	int ret;
729 
730 	ASSERT(DEVI_BUSY_OWNED(parent));
731 
732 	backend = (dom != DOMID_SELF);
733 	xdcp = i_xvdi_devclass2cfg(devclass);
734 	ASSERT(xdcp != NULL);
735 
736 	if (vdev != VDEV_NOXS) {
737 		if (!backend) {
738 			(void) snprintf(xsnamebuf, sizeof (xsnamebuf),
739 			    "%s/%d", xdcp->xs_path_fe, vdev);
740 			xsname = xsnamebuf;
741 			node = xdcp->node_fe;
742 		} else {
743 			(void) snprintf(xsnamebuf, sizeof (xsnamebuf),
744 			    "%s/%d/%d", xdcp->xs_path_be, dom, vdev);
745 			xsname = xsnamebuf;
746 			node = xdcp->node_be;
747 		}
748 	} else {
749 		node = xdcp->node_fe;
750 	}
751 
752 	/* Must have a driver to use. */
753 	if (node == NULL)
754 		return (NULL);
755 
756 	/*
757 	 * We need to check the state of this device before we go
758 	 * further, otherwise we'll end up with a dead loop if
759 	 * anything goes wrong.
760 	 */
761 	if ((xsname != NULL) &&
762 	    (xenbus_read_driver_state(xsname) >= XenbusStateClosing))
763 		return (NULL);
764 
765 	ndi_devi_alloc_sleep(parent, node, DEVI_SID_NODEID, &dip);
766 
767 	/*
768 	 * Driver binding uses the compatible property _before_ the
769 	 * node name, so we set the node name to the 'model' of the
770 	 * device (i.e. 'xnb' or 'xdb') and, if 'type' is present,
771 	 * encode both the model and the type in a compatible property
772 	 * (i.e. 'xnb,netfront' or 'xnb,SUNW_mac').  This allows a
773 	 * driver binding based on the <model,type> pair _before_ a
774 	 * binding based on the node name.
775 	 */
776 	if ((xsname != NULL) &&
777 	    (xenbus_read(XBT_NULL, xsname, "type", (void *)&type, &tlen)
778 	    == 0)) {
779 		size_t clen;
780 		char *c[1];
781 
782 		clen = strlen(node) + strlen(type) + 2;
783 		c[0] = kmem_alloc(clen, KM_SLEEP);
784 		(void) snprintf(c[0], clen, "%s,%s", node, type);
785 
786 		(void) ndi_prop_update_string_array(DDI_DEV_T_NONE,
787 		    dip, "compatible", (char **)c, 1);
788 
789 		kmem_free(c[0], clen);
790 		kmem_free(type, tlen);
791 	}
792 
793 	(void) ndi_prop_update_int(DDI_DEV_T_NONE, dip, "devclass", devclass);
794 	(void) ndi_prop_update_int(DDI_DEV_T_NONE, dip, "domain", dom);
795 	(void) ndi_prop_update_int(DDI_DEV_T_NONE, dip, "vdev", vdev);
796 
797 	if (i_ddi_devi_attached(parent))
798 		ret = ndi_devi_online(dip, 0);
799 	else
800 		ret = ndi_devi_bind_driver(dip, 0);
801 	if (ret != NDI_SUCCESS)
802 		(void) ndi_devi_offline(dip, NDI_DEVI_REMOVE);
803 
804 	return (dip);
805 }
806 
807 /*
808  * xendev_enum_class()
809  */
810 void
811 xendev_enum_class(dev_info_t *parent, xendev_devclass_t devclass)
812 {
813 	boolean_t dom0 = DOMAIN_IS_INITDOMAIN(xen_info);
814 	boolean_t domU = !dom0;
815 	i_xd_cfg_t *xdcp;
816 
817 	xdcp = i_xvdi_devclass2cfg(devclass);
818 	ASSERT(xdcp != NULL);
819 
820 	if (dom0 && !(xdcp->flags & XD_DOM_ZERO))
821 		return;
822 
823 	if (domU && !(xdcp->flags & XD_DOM_GUEST))
824 		return;
825 
826 	if (xdcp->xsdev == NULL) {
827 		int circ;
828 
829 		/*
830 		 * Don't need to probe this kind of device from the
831 		 * store, just create one if it doesn't exist.
832 		 */
833 
834 		ndi_devi_enter(parent, &circ);
835 		if (xvdi_find_dev(parent, devclass, DOMID_SELF, VDEV_NOXS)
836 		    == NULL)
837 			(void) xvdi_create_dev(parent, devclass,
838 			    DOMID_SELF, VDEV_NOXS);
839 		ndi_devi_exit(parent, circ);
840 	} else {
841 		/*
842 		 * Probe this kind of device from the store, both
843 		 * frontend and backend.
844 		 */
845 
846 		i_xvdi_enum_fe(parent, xdcp);
847 		i_xvdi_enum_be(parent, xdcp);
848 	}
849 }
850 
851 /*
852  * xendev_enum_all()
853  */
854 void
855 xendev_enum_all(dev_info_t *parent, boolean_t store_unavailable)
856 {
857 	int i;
858 	i_xd_cfg_t *xdcp;
859 	boolean_t dom0 = DOMAIN_IS_INITDOMAIN(xen_info);
860 
861 	for (i = 0, xdcp = xdci; i < NXDC; i++, xdcp++) {
862 		/*
863 		 * Dom0 relies on watchpoints to create non-soft
864 		 * devices - don't attempt to iterate over the store.
865 		 */
866 		if (dom0 && (xdcp->xsdev != NULL))
867 			continue;
868 
869 		/*
870 		 * If the store is not yet available, don't attempt to
871 		 * iterate.
872 		 */
873 		if (store_unavailable && (xdcp->xsdev != NULL))
874 			continue;
875 
876 		xendev_enum_class(parent, xdcp->devclass);
877 	}
878 }
879 
880 xendev_devclass_t
881 xendev_nodename_to_devclass(char *nodename)
882 {
883 	int i;
884 	i_xd_cfg_t *xdcp;
885 
886 	/*
887 	 * This relies on the convention that variants of a base
888 	 * driver share the same prefix and that there are no drivers
889 	 * which share a common prefix with the name of any other base
890 	 * drivers.
891 	 *
892 	 * So for a base driver 'xnb' (which is the name listed in
893 	 * xdci) the variants all begin with the string 'xnb' (in fact
894 	 * they are 'xnbe', 'xnbo' and 'xnbu') and there are no other
895 	 * base drivers which have the prefix 'xnb'.
896 	 */
897 	ASSERT(nodename != NULL);
898 	for (i = 0, xdcp = xdci; i < NXDC; i++, xdcp++) {
899 		if (((xdcp->node_fe != NULL) &&
900 		    (strncmp(nodename, xdcp->node_fe,
901 		    strlen(xdcp->node_fe)) == 0)) ||
902 		    ((xdcp->node_be != NULL) &&
903 		    (strncmp(nodename, xdcp->node_be,
904 		    strlen(xdcp->node_be)) == 0)))
905 
906 			return (xdcp->devclass);
907 	}
908 	return (XEN_INVAL);
909 }
910 
911 int
912 xendev_devclass_ipl(xendev_devclass_t devclass)
913 {
914 	i_xd_cfg_t *xdcp;
915 
916 	xdcp = i_xvdi_devclass2cfg(devclass);
917 	ASSERT(xdcp != NULL);
918 
919 	return (xdcp->xd_ipl);
920 }
921 
922 /*
923  * Determine if a devinfo instance exists of a particular device
924  * class, domain and xenstore virtual device number.
925  */
926 dev_info_t *
927 xvdi_find_dev(dev_info_t *parent, xendev_devclass_t devclass,
928     domid_t dom, int vdev)
929 {
930 	dev_info_t *dip;
931 
932 	ASSERT(DEVI_BUSY_OWNED(parent));
933 
934 	switch (devclass) {
935 	case XEN_CONSOLE:
936 	case XEN_XENBUS:
937 	case XEN_DOMCAPS:
938 	case XEN_BALLOON:
939 	case XEN_EVTCHN:
940 	case XEN_PRIVCMD:
941 		/* Console and soft devices have no vdev. */
942 		vdev = VDEV_NOXS;
943 		break;
944 	default:
945 		break;
946 	}
947 
948 	for (dip = ddi_get_child(parent); dip != NULL;
949 	    dip = ddi_get_next_sibling(dip)) {
950 		int *vdevnump, *domidp, *devclsp, vdevnum;
951 		uint_t ndomid, nvdevnum, ndevcls;
952 		xendev_devclass_t devcls;
953 		domid_t domid;
954 		struct xendev_ppd *pdp = ddi_get_parent_data(dip);
955 
956 		if (pdp == NULL) {
957 			if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
958 			    DDI_PROP_DONTPASS, "domain", &domidp, &ndomid) !=
959 			    DDI_PROP_SUCCESS)
960 				continue;
961 			ASSERT(ndomid == 1);
962 			domid = (domid_t)*domidp;
963 			ddi_prop_free(domidp);
964 
965 			if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
966 			    DDI_PROP_DONTPASS, "vdev", &vdevnump, &nvdevnum) !=
967 			    DDI_PROP_SUCCESS)
968 				continue;
969 			ASSERT(nvdevnum == 1);
970 			vdevnum = *vdevnump;
971 			ddi_prop_free(vdevnump);
972 
973 			if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
974 			    DDI_PROP_DONTPASS, "devclass", &devclsp,
975 			    &ndevcls) != DDI_PROP_SUCCESS)
976 				continue;
977 			ASSERT(ndevcls == 1);
978 			devcls = (xendev_devclass_t)*devclsp;
979 			ddi_prop_free(devclsp);
980 		} else {
981 			domid = pdp->xd_domain;
982 			vdevnum = pdp->xd_vdevnum;
983 			devcls = pdp->xd_devclass;
984 		}
985 
986 		if ((domid == dom) && (vdevnum == vdev) && (devcls == devclass))
987 			return (dip);
988 	}
989 	return (NULL);
990 }
991 
992 int
993 xvdi_get_evtchn(dev_info_t *xdip)
994 {
995 	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);
996 
997 	ASSERT(pdp != NULL);
998 	return (pdp->xd_evtchn);
999 }
1000 
1001 int
1002 xvdi_get_vdevnum(dev_info_t *xdip)
1003 {
1004 	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);
1005 
1006 	ASSERT(pdp != NULL);
1007 	return (pdp->xd_vdevnum);
1008 }
1009 
1010 char *
1011 xvdi_get_xsname(dev_info_t *xdip)
1012 {
1013 	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);
1014 
1015 	ASSERT(pdp != NULL);
1016 	return ((char *)(pdp->xd_xsdev.nodename));
1017 }
1018 
1019 char *
1020 xvdi_get_oename(dev_info_t *xdip)
1021 {
1022 	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);
1023 
1024 	ASSERT(pdp != NULL);
1025 	if (pdp->xd_devclass == XEN_CONSOLE)
1026 		return (NULL);
1027 	return ((char *)(pdp->xd_xsdev.otherend));
1028 }
1029 
1030 struct xenbus_device *
1031 xvdi_get_xsd(dev_info_t *xdip)
1032 {
1033 	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);
1034 
1035 	ASSERT(pdp != NULL);
1036 	return (&pdp->xd_xsdev);
1037 }
1038 
1039 domid_t
1040 xvdi_get_oeid(dev_info_t *xdip)
1041 {
1042 	struct xendev_ppd *pdp = ddi_get_parent_data(xdip);
1043 
1044 	ASSERT(pdp != NULL);
1045 	if (pdp->xd_devclass == XEN_CONSOLE)
1046 		return ((domid_t)-1);
1047 	return ((domid_t)(pdp->xd_xsdev.otherend_id));
1048 }
1049 
1050 void
1051 xvdi_dev_error(dev_info_t *dip, int errno, char *errstr)
1052 {
1053 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1054 
1055 	ASSERT(pdp != NULL);
1056 	xenbus_dev_error(&pdp->xd_xsdev, errno, errstr);
1057 }
1058 
1059 void
1060 xvdi_fatal_error(dev_info_t *dip, int errno, char *errstr)
1061 {
1062 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1063 
1064 	ASSERT(pdp != NULL);
1065 	xenbus_dev_fatal(&pdp->xd_xsdev, errno, errstr);
1066 }
1067 
1068 static void
1069 i_xvdi_oestate_handler(void *arg)
1070 {
1071 	dev_info_t *dip = arg;
1072 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1073 	XenbusState oestate = pdp->xd_xsdev.otherend_state;
1074 	ddi_eventcookie_t evc;
1075 
1076 	mutex_enter(&pdp->xd_lk);
1077 
1078 	if (pdp->xd_oe_ehid != NULL) {
1079 		/* send notification to driver */
1080 		if (ddi_get_eventcookie(dip, XS_OE_STATE,
1081 		    &evc) == DDI_SUCCESS) {
1082 			mutex_exit(&pdp->xd_lk);
1083 			(void) ndi_post_event(dip, dip, evc, &oestate);
1084 			mutex_enter(&pdp->xd_lk);
1085 		}
1086 	} else {
1087 		/*
1088 		 * take default action, if driver hasn't registered its
1089 		 * event handler yet
1090 		 */
1091 		if (oestate == XenbusStateClosing) {
1092 			(void) xvdi_switch_state(dip, XBT_NULL,
1093 			    XenbusStateClosed);
1094 		} else if (oestate == XenbusStateClosed) {
1095 			(void) xvdi_switch_state(dip, XBT_NULL,
1096 			    XenbusStateClosed);
1097 			(void) xvdi_post_event(dip, XEN_HP_REMOVE);
1098 		}
1099 	}
1100 
1101 	mutex_exit(&pdp->xd_lk);
1102 
1103 	/*
1104 	 * We'll try to remove the devinfo node of this device if the
1105 	 * other end has closed.
1106 	 */
1107 	if (oestate == XenbusStateClosed)
1108 		(void) ddi_taskq_dispatch(DEVI(ddi_get_parent(dip))->devi_taskq,
1109 		    xendev_offline_device, dip, DDI_SLEEP);
1110 }
1111 
1112 static void
1113 i_xvdi_hpstate_handler(void *arg)
1114 {
1115 	dev_info_t *dip = (dev_info_t *)arg;
1116 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1117 	ddi_eventcookie_t evc;
1118 	char *hp_status;
1119 	unsigned int hpl;
1120 
1121 	mutex_enter(&pdp->xd_lk);
1122 	if ((ddi_get_eventcookie(dip, XS_HP_STATE, &evc) == DDI_SUCCESS) &&
1123 	    (xenbus_read(XBT_NULL, pdp->xd_hp_watch.node, "",
1124 	    (void *)&hp_status, &hpl) == 0)) {
1125 
1126 		xendev_hotplug_state_t new_state = Unrecognized;
1127 
1128 		if (strcmp(hp_status, "connected") == 0)
1129 			new_state = Connected;
1130 
1131 		mutex_exit(&pdp->xd_lk);
1132 
1133 		(void) ndi_post_event(dip, dip, evc, &new_state);
1134 		kmem_free(hp_status, hpl);
1135 		return;
1136 	}
1137 	mutex_exit(&pdp->xd_lk);
1138 }
1139 
1140 void
1141 xvdi_notify_oe(dev_info_t *dip)
1142 {
1143 	struct xendev_ppd *pdp;
1144 
1145 	pdp = ddi_get_parent_data(dip);
1146 	ASSERT(pdp->xd_evtchn != INVALID_EVTCHN);
1147 	ec_notify_via_evtchn(pdp->xd_evtchn);
1148 }
1149 
1150 static void
1151 i_xvdi_bepath_cb(struct xenbus_watch *w, const char **vec, unsigned int len)
1152 {
1153 	dev_info_t *dip = (dev_info_t *)w->dev;
1154 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1155 	char *be = NULL;
1156 	unsigned int bel;
1157 
1158 	ASSERT(len > XS_WATCH_PATH);
1159 	ASSERT(vec[XS_WATCH_PATH] != NULL);
1160 
1161 	/*
1162 	 * If the backend is not the same as that we already stored,
1163 	 * re-set our watch for its' state.
1164 	 */
1165 	if ((xenbus_read(XBT_NULL, "", vec[XS_WATCH_PATH], (void *)be, &bel)
1166 	    == 0) && (strcmp(be, pdp->xd_xsdev.otherend) != 0))
1167 		(void) i_xvdi_add_watch_oestate(dip);
1168 
1169 	if (be != NULL) {
1170 		ASSERT(bel > 0);
1171 		kmem_free(be, bel);
1172 	}
1173 }
1174 
1175 static int
1176 i_xvdi_add_watch_oestate(dev_info_t *dip)
1177 {
1178 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1179 
1180 	ASSERT(pdp != NULL);
1181 	ASSERT(pdp->xd_xsdev.nodename != NULL);
1182 	ASSERT(mutex_owned(&pdp->xd_lk));
1183 
1184 	/*
1185 	 * Create taskq for delivering other end state change event to
1186 	 * this device later.
1187 	 *
1188 	 * Set nthreads to 1 to make sure that events can be delivered
1189 	 * in order.
1190 	 *
1191 	 * Note: It is _not_ guaranteed that driver can see every
1192 	 * xenstore change under the path that it is watching. If two
1193 	 * changes happen consecutively in a very short amount of
1194 	 * time, it is likely that the driver will see only the last
1195 	 * one.
1196 	 */
1197 	if (pdp->xd_oe_taskq == NULL)
1198 		if ((pdp->xd_oe_taskq = ddi_taskq_create(dip,
1199 		    "xendev_oe_taskq", 1, TASKQ_DEFAULTPRI, 0)) == NULL)
1200 			return (DDI_FAILURE);
1201 
1202 	/*
1203 	 * Watch for changes to the XenbusState of otherend.
1204 	 */
1205 	pdp->xd_xsdev.otherend_state = XenbusStateUnknown;
1206 	pdp->xd_xsdev.otherend_changed = i_xvdi_oestate_cb;
1207 
1208 	if (talk_to_otherend(&pdp->xd_xsdev) != 0) {
1209 		i_xvdi_rem_watch_oestate(dip);
1210 		return (DDI_FAILURE);
1211 	}
1212 
1213 	return (DDI_SUCCESS);
1214 }
1215 
1216 static void
1217 i_xvdi_rem_watch_oestate(dev_info_t *dip)
1218 {
1219 	struct xendev_ppd *pdp;
1220 	struct xenbus_device *dev;
1221 
1222 	pdp = ddi_get_parent_data(dip);
1223 	ASSERT(pdp != NULL);
1224 	ASSERT(mutex_owned(&pdp->xd_lk));
1225 
1226 	dev = &pdp->xd_xsdev;
1227 
1228 	/* Unwatch for changes to XenbusState of otherend */
1229 	if (dev->otherend_watch.node != NULL) {
1230 		mutex_exit(&pdp->xd_lk);
1231 		unregister_xenbus_watch(&dev->otherend_watch);
1232 		mutex_enter(&pdp->xd_lk);
1233 	}
1234 
1235 	/* make sure no event handler is running */
1236 	if (pdp->xd_oe_taskq != NULL) {
1237 		mutex_exit(&pdp->xd_lk);
1238 		ddi_taskq_destroy(pdp->xd_oe_taskq);
1239 		mutex_enter(&pdp->xd_lk);
1240 		pdp->xd_oe_taskq = NULL;
1241 	}
1242 
1243 	/* clean up */
1244 	dev->otherend_state = XenbusStateUnknown;
1245 	dev->otherend_id = (domid_t)-1;
1246 	if (dev->otherend_watch.node != NULL)
1247 		kmem_free((void *)dev->otherend_watch.node,
1248 		    strlen(dev->otherend_watch.node) + 1);
1249 	dev->otherend_watch.node = NULL;
1250 	if (dev->otherend != NULL)
1251 		kmem_free((void *)dev->otherend, strlen(dev->otherend) + 1);
1252 	dev->otherend = NULL;
1253 }
1254 
1255 static int
1256 i_xvdi_add_watch_hpstate(dev_info_t *dip)
1257 {
1258 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1259 
1260 	ASSERT(pdp != NULL);
1261 	ASSERT(pdp->xd_xsdev.frontend == 0);
1262 	ASSERT(mutex_owned(&pdp->xd_lk));
1263 
1264 	/*
1265 	 * Create taskq for delivering hotplug status change event to
1266 	 * this device later.
1267 	 *
1268 	 * Set nthreads to 1 to make sure that events can be delivered
1269 	 * in order.
1270 	 *
1271 	 * Note: It is _not_ guaranteed that driver can see every
1272 	 * hotplug status change under the path that it is
1273 	 * watching. If two changes happen consecutively in a very
1274 	 * short amount of time, it is likely that the driver only
1275 	 * sees the last one.
1276 	 */
1277 	if (pdp->xd_hp_taskq == NULL)
1278 		if ((pdp->xd_hp_taskq = ddi_taskq_create(dip,
1279 		    "xendev_hp_taskq", 1, TASKQ_DEFAULTPRI, 0)) == NULL)
1280 			return (DDI_FAILURE);
1281 
1282 	if (pdp->xd_hp_watch.node == NULL) {
1283 		size_t len;
1284 		char *path;
1285 
1286 		ASSERT(pdp->xd_xsdev.nodename != NULL);
1287 
1288 		len = strlen(pdp->xd_xsdev.nodename) +
1289 		    strlen("/hotplug-status") + 1;
1290 		path = kmem_alloc(len, KM_SLEEP);
1291 		(void) snprintf(path, len, "%s/hotplug-status",
1292 		    pdp->xd_xsdev.nodename);
1293 
1294 		pdp->xd_hp_watch.node = path;
1295 		pdp->xd_hp_watch.callback = i_xvdi_hpstate_cb;
1296 		pdp->xd_hp_watch.dev = (struct xenbus_device *)dip; /* yuck! */
1297 		if (register_xenbus_watch(&pdp->xd_hp_watch) != 0) {
1298 			i_xvdi_rem_watch_hpstate(dip);
1299 			return (DDI_FAILURE);
1300 		}
1301 	}
1302 
1303 	return (DDI_SUCCESS);
1304 }
1305 
1306 static void
1307 i_xvdi_rem_watch_hpstate(dev_info_t *dip)
1308 {
1309 	struct xendev_ppd *pdp;
1310 	pdp = ddi_get_parent_data(dip);
1311 
1312 	ASSERT(pdp != NULL);
1313 	ASSERT(pdp->xd_xsdev.frontend == 0);
1314 	ASSERT(mutex_owned(&pdp->xd_lk));
1315 
1316 	/* Unwatch for changes to "hotplug-status" node for backend device. */
1317 	if (pdp->xd_hp_watch.node != NULL) {
1318 		mutex_exit(&pdp->xd_lk);
1319 		unregister_xenbus_watch(&pdp->xd_hp_watch);
1320 		mutex_enter(&pdp->xd_lk);
1321 	}
1322 
1323 	/* Make sure no event handler is running. */
1324 	if (pdp->xd_hp_taskq != NULL) {
1325 		mutex_exit(&pdp->xd_lk);
1326 		ddi_taskq_destroy(pdp->xd_hp_taskq);
1327 		mutex_enter(&pdp->xd_lk);
1328 		pdp->xd_hp_taskq = NULL;
1329 	}
1330 
1331 	/* Clean up. */
1332 	if (pdp->xd_hp_watch.node != NULL) {
1333 		kmem_free((void *)pdp->xd_hp_watch.node,
1334 		    strlen(pdp->xd_hp_watch.node) + 1);
1335 		pdp->xd_hp_watch.node = NULL;
1336 	}
1337 }
1338 
1339 static int
1340 i_xvdi_add_watches(dev_info_t *dip)
1341 {
1342 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1343 
1344 	ASSERT(pdp != NULL);
1345 
1346 	mutex_enter(&pdp->xd_lk);
1347 
1348 	if (i_xvdi_add_watch_oestate(dip) != DDI_SUCCESS) {
1349 		mutex_exit(&pdp->xd_lk);
1350 		return (DDI_FAILURE);
1351 	}
1352 
1353 	if (pdp->xd_xsdev.frontend == 1) {
1354 		/*
1355 		 * Frontend devices must watch for the backend path
1356 		 * changing.
1357 		 */
1358 		if (i_xvdi_add_watch_bepath(dip) != DDI_SUCCESS)
1359 			goto unwatch_and_fail;
1360 	} else {
1361 		/*
1362 		 * Backend devices must watch for hotplug events.
1363 		 */
1364 		if (i_xvdi_add_watch_hpstate(dip) != DDI_SUCCESS)
1365 			goto unwatch_and_fail;
1366 	}
1367 
1368 	mutex_exit(&pdp->xd_lk);
1369 
1370 	return (DDI_SUCCESS);
1371 
1372 unwatch_and_fail:
1373 	i_xvdi_rem_watch_oestate(dip);
1374 	mutex_exit(&pdp->xd_lk);
1375 
1376 	return (DDI_FAILURE);
1377 }
1378 
1379 static void
1380 i_xvdi_rem_watches(dev_info_t *dip)
1381 {
1382 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1383 
1384 	ASSERT(pdp != NULL);
1385 
1386 	mutex_enter(&pdp->xd_lk);
1387 
1388 	i_xvdi_rem_watch_oestate(dip);
1389 
1390 	if (pdp->xd_xsdev.frontend == 1)
1391 		i_xvdi_rem_watch_bepath(dip);
1392 	else
1393 		i_xvdi_rem_watch_hpstate(dip);
1394 
1395 	mutex_exit(&pdp->xd_lk);
1396 }
1397 
1398 static int
1399 i_xvdi_add_watch_bepath(dev_info_t *dip)
1400 {
1401 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1402 
1403 	ASSERT(pdp != NULL);
1404 	ASSERT(pdp->xd_xsdev.frontend == 1);
1405 
1406 	/*
1407 	 * Frontend devices need to watch for the backend path changing.
1408 	 */
1409 	if (pdp->xd_bepath_watch.node == NULL) {
1410 		size_t len;
1411 		char *path;
1412 
1413 		ASSERT(pdp->xd_xsdev.nodename != NULL);
1414 
1415 		len = strlen(pdp->xd_xsdev.nodename) + strlen("/backend") + 1;
1416 		path = kmem_alloc(len, KM_SLEEP);
1417 		(void) snprintf(path, len, "%s/backend",
1418 		    pdp->xd_xsdev.nodename);
1419 
1420 		pdp->xd_bepath_watch.node = path;
1421 		pdp->xd_bepath_watch.callback = i_xvdi_bepath_cb;
1422 		pdp->xd_bepath_watch.dev = (struct xenbus_device *)dip;
1423 		if (register_xenbus_watch(&pdp->xd_bepath_watch) != 0) {
1424 			kmem_free(path, len);
1425 			pdp->xd_bepath_watch.node = NULL;
1426 			return (DDI_FAILURE);
1427 		}
1428 	}
1429 
1430 	return (DDI_SUCCESS);
1431 }
1432 
1433 static void
1434 i_xvdi_rem_watch_bepath(dev_info_t *dip)
1435 {
1436 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1437 
1438 	ASSERT(pdp != NULL);
1439 	ASSERT(pdp->xd_xsdev.frontend == 1);
1440 	ASSERT(mutex_owned(&pdp->xd_lk));
1441 
1442 	if (pdp->xd_bepath_watch.node != NULL) {
1443 		mutex_exit(&pdp->xd_lk);
1444 		unregister_xenbus_watch(&pdp->xd_bepath_watch);
1445 		mutex_enter(&pdp->xd_lk);
1446 
1447 		kmem_free((void *)(pdp->xd_bepath_watch.node),
1448 		    strlen(pdp->xd_bepath_watch.node) + 1);
1449 		pdp->xd_bepath_watch.node = NULL;
1450 	}
1451 }
1452 
1453 int
1454 xvdi_switch_state(dev_info_t *dip, xenbus_transaction_t xbt,
1455     XenbusState newState)
1456 {
1457 	int rv;
1458 	struct xendev_ppd *pdp;
1459 
1460 	pdp = ddi_get_parent_data(dip);
1461 	ASSERT(pdp != NULL);
1462 
1463 	XVDI_DPRINTF(XVDI_DBG_STATE,
1464 	    "xvdi_switch_state: dip 0x%p moves to %d",
1465 	    (void *)dip, newState);
1466 
1467 	rv = xenbus_switch_state(&pdp->xd_xsdev, xbt, newState);
1468 	if (rv > 0)
1469 		cmn_err(CE_WARN, "xvdi_switch_state: change state failed");
1470 
1471 	return (rv);
1472 }
1473 
1474 /*
1475  * Notify hotplug script running in userland
1476  */
1477 int
1478 xvdi_post_event(dev_info_t *dip, xendev_hotplug_cmd_t hpc)
1479 {
1480 	struct xendev_ppd *pdp;
1481 	nvlist_t *attr_list = NULL;
1482 	i_xd_cfg_t *xdcp;
1483 	sysevent_id_t eid;
1484 	int err;
1485 	char devname[256]; /* XXPV dme: ? */
1486 
1487 	pdp = ddi_get_parent_data(dip);
1488 	ASSERT(pdp != NULL);
1489 
1490 	xdcp = i_xvdi_devclass2cfg(pdp->xd_devclass);
1491 	ASSERT(xdcp != NULL);
1492 
1493 	(void) snprintf(devname, sizeof (devname) - 1, "%s%d",
1494 	    ddi_driver_name(dip),  ddi_get_instance(dip));
1495 
1496 	err = nvlist_alloc(&attr_list, NV_UNIQUE_NAME, KM_NOSLEEP);
1497 	if (err != DDI_SUCCESS)
1498 		goto failure;
1499 
1500 	err = nvlist_add_int32(attr_list, "domain", pdp->xd_domain);
1501 	if (err != DDI_SUCCESS)
1502 		goto failure;
1503 	err = nvlist_add_int32(attr_list, "vdev", pdp->xd_vdevnum);
1504 	if (err != DDI_SUCCESS)
1505 		goto failure;
1506 	err = nvlist_add_string(attr_list, "devclass", xdcp->xsdev);
1507 	if (err != DDI_SUCCESS)
1508 		goto failure;
1509 	err = nvlist_add_string(attr_list, "device", devname);
1510 	if (err != DDI_SUCCESS)
1511 		goto failure;
1512 	err = nvlist_add_string(attr_list, "fob",
1513 	    ((pdp->xd_xsdev.frontend == 1) ? "frontend" : "backend"));
1514 	if (err != DDI_SUCCESS)
1515 		goto failure;
1516 
1517 	switch (hpc) {
1518 	case XEN_HP_ADD:
1519 		err = ddi_log_sysevent(dip, DDI_VENDOR_SUNW, "EC_xendev",
1520 		    "add", attr_list, &eid, DDI_NOSLEEP);
1521 		break;
1522 	case XEN_HP_REMOVE:
1523 		err = ddi_log_sysevent(dip, DDI_VENDOR_SUNW, "EC_xendev",
1524 		    "remove", attr_list, &eid, DDI_NOSLEEP);
1525 		break;
1526 	default:
1527 		err = DDI_FAILURE;
1528 		goto failure;
1529 	}
1530 
1531 failure:
1532 	if (attr_list != NULL)
1533 		nvlist_free(attr_list);
1534 
1535 	return (err);
1536 }
1537 
1538 /* ARGSUSED */
1539 static void
1540 i_xvdi_probe_path_cb(struct xenbus_watch *w, const char **vec,
1541     unsigned int len)
1542 {
1543 	char *path;
1544 
1545 	if (xendev_dip == NULL)
1546 		xendev_dip = ddi_find_devinfo("xpvd", -1, 0);
1547 
1548 	path = i_ddi_strdup((char *)vec[XS_WATCH_PATH], KM_SLEEP);
1549 
1550 	(void) ddi_taskq_dispatch(DEVI(xendev_dip)->devi_taskq,
1551 	    i_xvdi_probe_path_handler, (void *)path, DDI_SLEEP);
1552 }
1553 
1554 static void
1555 i_xvdi_watch_device(char *path)
1556 {
1557 	struct xenbus_watch *w;
1558 
1559 	ASSERT(path != NULL);
1560 
1561 	w = kmem_zalloc(sizeof (*w), KM_SLEEP);
1562 	w->node = path;
1563 	w->callback = &i_xvdi_probe_path_cb;
1564 	w->dev = NULL;
1565 
1566 	if (register_xenbus_watch(w) != 0) {
1567 		cmn_err(CE_WARN, "i_xvdi_watch_device: "
1568 		    "cannot set watch on %s", path);
1569 		kmem_free(w, sizeof (*w));
1570 		return;
1571 	}
1572 }
1573 
1574 void
1575 xvdi_watch_devices(int newstate)
1576 {
1577 	int devclass;
1578 
1579 	/*
1580 	 * Watch for devices being created in the store.
1581 	 */
1582 	if (newstate == XENSTORE_DOWN)
1583 		return;
1584 	for (devclass = 0; devclass < NXDC; devclass++) {
1585 		if (xdci[devclass].xs_path_fe != NULL)
1586 			i_xvdi_watch_device(xdci[devclass].xs_path_fe);
1587 		if (xdci[devclass].xs_path_be != NULL)
1588 			i_xvdi_watch_device(xdci[devclass].xs_path_be);
1589 	}
1590 }
1591 
1592 /*
1593  * Iterate over the store looking for backend devices to create.
1594  */
1595 static void
1596 i_xvdi_enum_be(dev_info_t *parent, i_xd_cfg_t *xdcp)
1597 {
1598 	char **domains;
1599 	unsigned int ndomains;
1600 	int ldomains, i;
1601 
1602 	if ((domains = xenbus_directory(XBT_NULL, xdcp->xs_path_be, "",
1603 	    &ndomains)) == NULL)
1604 		return;
1605 
1606 	for (i = 0, ldomains = 0; i < ndomains; i++) {
1607 		ldomains += strlen(domains[i]) + 1 + sizeof (char *);
1608 
1609 		i_xvdi_enum_worker(parent, xdcp, domains[i]);
1610 	}
1611 	kmem_free(domains, ldomains);
1612 }
1613 
1614 /*
1615  * Iterate over the store looking for frontend devices to create.
1616  */
1617 static void
1618 i_xvdi_enum_fe(dev_info_t *parent, i_xd_cfg_t *xdcp)
1619 {
1620 	i_xvdi_enum_worker(parent, xdcp, NULL);
1621 }
1622 
1623 static void
1624 i_xvdi_enum_worker(dev_info_t *parent, i_xd_cfg_t *xdcp,
1625     char *domain)
1626 {
1627 	char *path, *domain_path, *ep;
1628 	char **devices;
1629 	unsigned int ndevices;
1630 	int ldevices, j, circ;
1631 	domid_t dom;
1632 	long tmplong;
1633 
1634 	if (domain == NULL) {
1635 		dom = DOMID_SELF;
1636 		path = xdcp->xs_path_fe;
1637 		domain_path = "";
1638 	} else {
1639 		(void) ddi_strtol(domain, &ep, 0, &tmplong);
1640 		dom = tmplong;
1641 		path = xdcp->xs_path_be;
1642 		domain_path = domain;
1643 	}
1644 
1645 	if ((devices = xenbus_directory(XBT_NULL, path, domain_path,
1646 	    &ndevices)) == NULL)
1647 		return;
1648 
1649 	for (j = 0, ldevices = 0; j < ndevices; j++) {
1650 		int vdev;
1651 
1652 		ldevices += strlen(devices[j]) + 1 + sizeof (char *);
1653 		(void) ddi_strtol(devices[j], &ep, 0, &tmplong);
1654 		vdev = tmplong;
1655 
1656 		ndi_devi_enter(parent, &circ);
1657 
1658 		if (xvdi_find_dev(parent, xdcp->devclass, dom, vdev) == NULL)
1659 			(void) xvdi_create_dev(parent, xdcp->devclass,
1660 			    dom, vdev);
1661 
1662 		ndi_devi_exit(parent, circ);
1663 	}
1664 	kmem_free(devices, ldevices);
1665 }
1666 
1667 /*
1668  * Leaf drivers should call this in their detach() routine during suspend.
1669  */
1670 void
1671 xvdi_suspend(dev_info_t *dip)
1672 {
1673 	i_xvdi_rem_watches(dip);
1674 }
1675 
1676 /*
1677  * Leaf drivers should call this in their attach() routine during resume.
1678  */
1679 int
1680 xvdi_resume(dev_info_t *dip)
1681 {
1682 	return (i_xvdi_add_watches(dip));
1683 }
1684 
1685 /*
1686  * Add event handler for the leaf driver
1687  * to handle event triggered by the change in xenstore
1688  */
1689 int
1690 xvdi_add_event_handler(dev_info_t *dip, char *name,
1691     void (*evthandler)(dev_info_t *, ddi_eventcookie_t, void *, void *))
1692 {
1693 	ddi_eventcookie_t ecv;
1694 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
1695 	ddi_callback_id_t *cbid;
1696 
1697 	ASSERT(pdp != NULL);
1698 
1699 	mutex_enter(&pdp->xd_lk);
1700 
1701 	if (strcmp(name, XS_OE_STATE) == 0) {
1702 		ASSERT(pdp->xd_xsdev.otherend != NULL);
1703 
1704 		cbid = &pdp->xd_oe_ehid;
1705 	} else if (strcmp(name, XS_HP_STATE) == 0) {
1706 		if (pdp->xd_xsdev.frontend == 1) {
1707 			mutex_exit(&pdp->xd_lk);
1708 			return (DDI_FAILURE);
1709 		}
1710 
1711 		ASSERT(pdp->xd_hp_watch.node != NULL);
1712 
1713 		cbid = &pdp->xd_hp_ehid;
1714 	} else {
1715 		/* Unsupported watch. */
1716 		mutex_exit(&pdp->xd_lk);
1717 		return (DDI_FAILURE);
1718 	}
1719 
1720 	/*
1721 	 * No event handler provided, take default action to handle
1722 	 * event.
1723 	 */
1724 	if (evthandler == NULL) {
1725 		mutex_exit(&pdp->xd_lk);
1726 		return (DDI_SUCCESS);
1727 	}
1728 
1729 	ASSERT(*cbid == NULL);
1730 
1731 	if (ddi_get_eventcookie(dip, name, &ecv) != DDI_SUCCESS) {
1732 		cmn_err(CE_WARN, "failed to find %s cookie for %s@%s",
1733 		    name, ddi_get_name(dip), ddi_get_name_addr(dip));
1734 		mutex_exit(&pdp->xd_lk);
1735 		return (DDI_FAILURE);
1736 	}
1737 	if (ddi_add_event_handler(dip, ecv, evthandler, NULL, cbid)
1738 	    != DDI_SUCCESS) {
1739 		cmn_err(CE_WARN, "failed to add %s event handler for %s@%s",
1740 		    name, ddi_get_name(dip), ddi_get_name_addr(dip));
1741 		*cbid = NULL;
1742 		mutex_exit(&pdp->xd_lk);
1743 		return (DDI_FAILURE);
1744 	}
1745 
1746 	mutex_exit(&pdp->xd_lk);
1747 
1748 	return (DDI_SUCCESS);
1749 }
1750 
1751 /*
1752  * Remove event handler for the leaf driver and unwatch xenstore
1753  * so, driver will not be notified when xenstore entry changed later
1754  */
1755 void
1756 xvdi_remove_event_handler(dev_info_t *dip, char *name)
1757 {
1758 	struct xendev_ppd *pdp;
1759 	boolean_t rem_oe = B_FALSE, rem_hp = B_FALSE;
1760 	ddi_callback_id_t oeid = NULL, hpid = NULL;
1761 
1762 	pdp = ddi_get_parent_data(dip);
1763 	ASSERT(pdp != NULL);
1764 
1765 	if (name == NULL) {
1766 		rem_oe = B_TRUE;
1767 		rem_hp = B_TRUE;
1768 	} else if (strcmp(name, XS_OE_STATE) == 0) {
1769 		rem_oe = B_TRUE;
1770 	} else if (strcmp(name, XS_HP_STATE) == 0) {
1771 		rem_hp = B_TRUE;
1772 	} else {
1773 		cmn_err(CE_WARN, "event %s not supported, cannot remove", name);
1774 		return;
1775 	}
1776 
1777 	mutex_enter(&pdp->xd_lk);
1778 
1779 	if (rem_oe && (pdp->xd_oe_ehid != NULL)) {
1780 		oeid = pdp->xd_oe_ehid;
1781 		pdp->xd_oe_ehid = NULL;
1782 	}
1783 
1784 	if (rem_hp && (pdp->xd_hp_ehid != NULL)) {
1785 		hpid = pdp->xd_hp_ehid;
1786 		pdp->xd_hp_ehid = NULL;
1787 	}
1788 
1789 	mutex_exit(&pdp->xd_lk);
1790 
1791 	if (oeid != NULL)
1792 		(void) ddi_remove_event_handler(oeid);
1793 	if (hpid != NULL)
1794 		(void) ddi_remove_event_handler(hpid);
1795 }
1796 
1797 
1798 /*
1799  * common ring interfaces
1800  */
1801 
1802 #define	FRONT_RING(_ringp)	(&(_ringp)->xr_sring.fr)
1803 #define	BACK_RING(_ringp)	(&(_ringp)->xr_sring.br)
1804 #define	GET_RING_SIZE(_ringp)	RING_SIZE(FRONT_RING(ringp))
1805 #define	GET_RING_ENTRY_FE(_ringp, _idx)		\
1806 	(FRONT_RING(_ringp)->sring->ring +	\
1807 	(_ringp)->xr_entry_size * ((_idx) & (GET_RING_SIZE(_ringp) - 1)))
1808 #define	GET_RING_ENTRY_BE(_ringp, _idx)		\
1809 	(BACK_RING(_ringp)->sring->ring +	\
1810 	(_ringp)->xr_entry_size * ((_idx) & (GET_RING_SIZE(_ringp) - 1)))
1811 
1812 unsigned int
1813 xvdi_ring_avail_slots(xendev_ring_t *ringp)
1814 {
1815 	comif_ring_fe_t *frp;
1816 	comif_ring_be_t *brp;
1817 
1818 	if (ringp->xr_frontend) {
1819 		frp = FRONT_RING(ringp);
1820 		return (GET_RING_SIZE(ringp) -
1821 		    (frp->req_prod_pvt - frp->rsp_cons));
1822 	} else {
1823 		brp = BACK_RING(ringp);
1824 		return (GET_RING_SIZE(ringp) -
1825 		    (brp->rsp_prod_pvt - brp->req_cons));
1826 	}
1827 }
1828 
1829 int
1830 xvdi_ring_has_unconsumed_requests(xendev_ring_t *ringp)
1831 {
1832 	comif_ring_be_t *brp;
1833 
1834 	ASSERT(!ringp->xr_frontend);
1835 	brp = BACK_RING(ringp);
1836 	return ((brp->req_cons !=
1837 	    ddi_get32(ringp->xr_acc_hdl, &brp->sring->req_prod)) &&
1838 	    ((brp->req_cons - brp->rsp_prod_pvt) != RING_SIZE(brp)));
1839 }
1840 
1841 int
1842 xvdi_ring_has_incomp_request(xendev_ring_t *ringp)
1843 {
1844 	comif_ring_fe_t *frp;
1845 
1846 	ASSERT(ringp->xr_frontend);
1847 	frp = FRONT_RING(ringp);
1848 	return (frp->req_prod_pvt !=
1849 	    ddi_get32(ringp->xr_acc_hdl, &frp->sring->rsp_prod));
1850 }
1851 
1852 int
1853 xvdi_ring_has_unconsumed_responses(xendev_ring_t *ringp)
1854 {
1855 	comif_ring_fe_t *frp;
1856 
1857 	ASSERT(ringp->xr_frontend);
1858 	frp = FRONT_RING(ringp);
1859 	return (frp->rsp_cons !=
1860 	    ddi_get32(ringp->xr_acc_hdl, &frp->sring->rsp_prod));
1861 }
1862 
1863 /* NOTE: req_event will be increased as needed */
1864 void *
1865 xvdi_ring_get_request(xendev_ring_t *ringp)
1866 {
1867 	comif_ring_fe_t *frp;
1868 	comif_ring_be_t *brp;
1869 
1870 	if (ringp->xr_frontend) {
1871 		/* for frontend ring */
1872 		frp = FRONT_RING(ringp);
1873 		if (!RING_FULL(frp))
1874 			return (GET_RING_ENTRY_FE(ringp, frp->req_prod_pvt++));
1875 		else
1876 			return (NULL);
1877 	} else {
1878 		/* for backend ring */
1879 		brp = BACK_RING(ringp);
1880 		/* RING_FINAL_CHECK_FOR_REQUESTS() */
1881 		if (xvdi_ring_has_unconsumed_requests(ringp))
1882 			return (GET_RING_ENTRY_BE(ringp, brp->req_cons++));
1883 		else {
1884 			ddi_put32(ringp->xr_acc_hdl, &brp->sring->req_event,
1885 			    brp->req_cons + 1);
1886 			membar_enter();
1887 			if (xvdi_ring_has_unconsumed_requests(ringp))
1888 				return (GET_RING_ENTRY_BE(ringp,
1889 				    brp->req_cons++));
1890 			else
1891 				return (NULL);
1892 		}
1893 	}
1894 }
1895 
1896 int
1897 xvdi_ring_push_request(xendev_ring_t *ringp)
1898 {
1899 	RING_IDX old, new, reqevt;
1900 	comif_ring_fe_t *frp;
1901 
1902 	/* only frontend should be able to push request */
1903 	ASSERT(ringp->xr_frontend);
1904 
1905 	/* RING_PUSH_REQUEST_AND_CHECK_NOTIFY() */
1906 	frp = FRONT_RING(ringp);
1907 	old = ddi_get32(ringp->xr_acc_hdl, &frp->sring->req_prod);
1908 	new = frp->req_prod_pvt;
1909 	ddi_put32(ringp->xr_acc_hdl, &frp->sring->req_prod, new);
1910 	membar_enter();
1911 	reqevt = ddi_get32(ringp->xr_acc_hdl, &frp->sring->req_event);
1912 	return ((RING_IDX)(new - reqevt) < (RING_IDX)(new - old));
1913 }
1914 
1915 /* NOTE: rsp_event will be increased as needed */
1916 void *
1917 xvdi_ring_get_response(xendev_ring_t *ringp)
1918 {
1919 	comif_ring_fe_t *frp;
1920 	comif_ring_be_t *brp;
1921 
1922 	if (!ringp->xr_frontend) {
1923 		/* for backend ring */
1924 		brp = BACK_RING(ringp);
1925 		return (GET_RING_ENTRY_BE(ringp, brp->rsp_prod_pvt++));
1926 	} else {
1927 		/* for frontend ring */
1928 		frp = FRONT_RING(ringp);
1929 		/* RING_FINAL_CHECK_FOR_RESPONSES() */
1930 		if (xvdi_ring_has_unconsumed_responses(ringp))
1931 			return (GET_RING_ENTRY_FE(ringp, frp->rsp_cons++));
1932 		else {
1933 			ddi_put32(ringp->xr_acc_hdl, &frp->sring->rsp_event,
1934 			    frp->rsp_cons + 1);
1935 			membar_enter();
1936 			if (xvdi_ring_has_unconsumed_responses(ringp))
1937 				return (GET_RING_ENTRY_FE(ringp,
1938 				    frp->rsp_cons++));
1939 			else
1940 				return (NULL);
1941 		}
1942 	}
1943 }
1944 
1945 int
1946 xvdi_ring_push_response(xendev_ring_t *ringp)
1947 {
1948 	RING_IDX old, new, rspevt;
1949 	comif_ring_be_t *brp;
1950 
1951 	/* only backend should be able to push response */
1952 	ASSERT(!ringp->xr_frontend);
1953 
1954 	/* RING_PUSH_RESPONSE_AND_CHECK_NOTIFY() */
1955 	brp = BACK_RING(ringp);
1956 	old = ddi_get32(ringp->xr_acc_hdl, &brp->sring->rsp_prod);
1957 	new = brp->rsp_prod_pvt;
1958 	ddi_put32(ringp->xr_acc_hdl, &brp->sring->rsp_prod, new);
1959 	membar_enter();
1960 	rspevt = ddi_get32(ringp->xr_acc_hdl, &brp->sring->rsp_event);
1961 	return ((RING_IDX)(new - rspevt) < (RING_IDX)(new - old));
1962 }
1963 
1964 static void
1965 xvdi_ring_init_sring(xendev_ring_t *ringp)
1966 {
1967 	ddi_acc_handle_t acchdl;
1968 	comif_sring_t *xsrp;
1969 	int i;
1970 
1971 	xsrp = (comif_sring_t *)ringp->xr_vaddr;
1972 	acchdl = ringp->xr_acc_hdl;
1973 
1974 	/* shared ring initialization */
1975 	ddi_put32(acchdl, &xsrp->req_prod, 0);
1976 	ddi_put32(acchdl, &xsrp->rsp_prod, 0);
1977 	ddi_put32(acchdl, &xsrp->req_event, 1);
1978 	ddi_put32(acchdl, &xsrp->rsp_event, 1);
1979 	for (i = 0; i < sizeof (xsrp->pad); i++)
1980 		ddi_put8(acchdl, xsrp->pad + i, 0);
1981 }
1982 
1983 static void
1984 xvdi_ring_init_front_ring(xendev_ring_t *ringp, size_t nentry, size_t entrysize)
1985 {
1986 	comif_ring_fe_t *xfrp;
1987 
1988 	xfrp = &ringp->xr_sring.fr;
1989 	xfrp->req_prod_pvt = 0;
1990 	xfrp->rsp_cons = 0;
1991 	xfrp->nr_ents = nentry;
1992 	xfrp->sring = (comif_sring_t *)ringp->xr_vaddr;
1993 
1994 	ringp->xr_frontend = 1;
1995 	ringp->xr_entry_size = entrysize;
1996 }
1997 
1998 #ifndef XPV_HVM_DRIVER
1999 static void
2000 xvdi_ring_init_back_ring(xendev_ring_t *ringp, size_t nentry, size_t entrysize)
2001 {
2002 	comif_ring_be_t *xbrp;
2003 
2004 	xbrp = &ringp->xr_sring.br;
2005 	xbrp->rsp_prod_pvt = 0;
2006 	xbrp->req_cons = 0;
2007 	xbrp->nr_ents = nentry;
2008 	xbrp->sring = (comif_sring_t *)ringp->xr_vaddr;
2009 
2010 	ringp->xr_frontend = 0;
2011 	ringp->xr_entry_size = entrysize;
2012 }
2013 #endif /* XPV_HVM_DRIVER */
2014 
2015 static void
2016 xendev_offline_device(void *arg)
2017 {
2018 	dev_info_t *dip = (dev_info_t *)arg;
2019 	char devname[MAXNAMELEN] = {0};
2020 
2021 	/*
2022 	 * This is currently the only chance to delete a devinfo node, which
2023 	 * is _not_ always successful.
2024 	 */
2025 	(void) ddi_deviname(dip, devname);
2026 	(void) devfs_clean(ddi_get_parent(dip), devname + 1, DV_CLEAN_FORCE);
2027 	(void) ndi_devi_offline(dip, NDI_DEVI_REMOVE);
2028 }
2029 
2030 static void
2031 i_xvdi_oestate_cb(struct xenbus_device *dev, XenbusState oestate)
2032 {
2033 	dev_info_t *dip = (dev_info_t *)dev->data;
2034 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
2035 
2036 	/*
2037 	 * Don't trigger two consecutive ndi_devi_offline on the same
2038 	 * dip.
2039 	 */
2040 	if ((oestate == XenbusStateClosed) &&
2041 	    (dev->otherend_state == XenbusStateClosed))
2042 		return;
2043 
2044 	dev->otherend_state = oestate;
2045 	(void) ddi_taskq_dispatch(pdp->xd_oe_taskq,
2046 	    i_xvdi_oestate_handler, (void *)dip, DDI_SLEEP);
2047 }
2048 
2049 /*ARGSUSED*/
2050 static void
2051 i_xvdi_hpstate_cb(struct xenbus_watch *w, const char **vec,
2052     unsigned int len)
2053 {
2054 	dev_info_t *dip = (dev_info_t *)w->dev;
2055 	struct xendev_ppd *pdp = ddi_get_parent_data(dip);
2056 
2057 	(void) ddi_taskq_dispatch(pdp->xd_hp_taskq,
2058 	    i_xvdi_hpstate_handler, (void *)dip, DDI_SLEEP);
2059 }
2060 
2061 static void
2062 i_xvdi_probe_path_handler(void *arg)
2063 {
2064 	dev_info_t *parent;
2065 	char *path = arg, *p = NULL;
2066 	int i, vdev, circ;
2067 	i_xd_cfg_t *xdcp;
2068 	boolean_t frontend;
2069 	domid_t dom;
2070 
2071 	for (i = 0, xdcp = &xdci[0]; i < NXDC; i++, xdcp++) {
2072 
2073 		if ((xdcp->xs_path_fe != NULL) &&
2074 		    (strncmp(path, xdcp->xs_path_fe, strlen(xdcp->xs_path_fe))
2075 		    == 0)) {
2076 
2077 			frontend = B_TRUE;
2078 			p = path + strlen(xdcp->xs_path_fe);
2079 			break;
2080 		}
2081 
2082 		if ((xdcp->xs_path_be != NULL) &&
2083 		    (strncmp(path, xdcp->xs_path_be, strlen(xdcp->xs_path_be))
2084 		    == 0)) {
2085 
2086 			frontend = B_FALSE;
2087 			p = path + strlen(xdcp->xs_path_be);
2088 			break;
2089 		}
2090 
2091 	}
2092 
2093 	if (p == NULL) {
2094 		cmn_err(CE_WARN, "i_xvdi_probe_path_handler: "
2095 		    "unexpected path prefix in %s", path);
2096 		goto done;
2097 	}
2098 
2099 	if (frontend) {
2100 		dom = DOMID_SELF;
2101 		if (sscanf(p, "/%d/", &vdev) != 1) {
2102 			XVDI_DPRINTF(XVDI_DBG_PROBE,
2103 			    "i_xvdi_probe_path_handler: "
2104 			    "cannot parse frontend path %s",
2105 			    path);
2106 			goto done;
2107 		}
2108 	} else {
2109 		if (sscanf(p, "/%d/%d/", &dom, &vdev) != 2) {
2110 			XVDI_DPRINTF(XVDI_DBG_PROBE,
2111 			    "i_xvdi_probe_path_handler: "
2112 			    "cannot parse backend path %s",
2113 			    path);
2114 			goto done;
2115 		}
2116 	}
2117 
2118 	/*
2119 	 * This is an oxymoron, so indicates a bogus configuration we
2120 	 * must check for.
2121 	 */
2122 	if (vdev == VDEV_NOXS) {
2123 		cmn_err(CE_WARN, "i_xvdi_probe_path_handler: "
2124 		    "invalid path %s", path);
2125 		goto done;
2126 	}
2127 
2128 	parent = xendev_dip;
2129 	ASSERT(parent != NULL);
2130 
2131 	ndi_devi_enter(parent, &circ);
2132 
2133 	if (xvdi_find_dev(parent, xdcp->devclass, dom, vdev) == NULL) {
2134 		XVDI_DPRINTF(XVDI_DBG_PROBE,
2135 		    "i_xvdi_probe_path_handler: create for %s", path);
2136 		(void) xvdi_create_dev(parent, xdcp->devclass, dom, vdev);
2137 	} else {
2138 		XVDI_DPRINTF(XVDI_DBG_PROBE,
2139 		    "i_xvdi_probe_path_handler: %s already exists", path);
2140 	}
2141 
2142 	ndi_devi_exit(parent, circ);
2143 
2144 done:
2145 	kmem_free(path, strlen(path) + 1);
2146 }
2147