1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved.
23 *
24 * Copyright 2013 Nexenta Systems, Inc. All rights reserved.
25 * Copyright (c) 2016 Andrey Sokolov
26 * Copyright 2016 Toomas Soome <tsoome@me.com>
27 * Copyright 2019 Joyent, Inc.
28 * Copyright 2019 OmniOS Community Edition (OmniOSce) Association.
29 */
30
31/*
32 * lofi (loopback file) driver - allows you to attach a file to a device,
33 * which can then be accessed through that device. The simple model is that
34 * you tell lofi to open a file, and then use the block device you get as
35 * you would any block device. lofi translates access to the block device
36 * into I/O on the underlying file. This is mostly useful for
37 * mounting images of filesystems.
38 *
39 * lofi is controlled through /dev/lofictl - this is the only device exported
40 * during attach, and is instance number 0. lofiadm communicates with lofi
41 * through ioctls on this device. When a file is attached to lofi, block and
42 * character devices are exported in /dev/lofi and /dev/rlofi. These devices
43 * are identified by lofi instance number, and the instance number is also used
44 * as the name in /dev/lofi.
45 *
46 * Virtual disks, or, labeled lofi, implements virtual disk support to
47 * support partition table and related tools. Such mappings will cause
48 * block and character devices to be exported in /dev/dsk and /dev/rdsk
49 * directories.
50 *
51 * To support virtual disks, the instance number space is divided to two
52 * parts, upper part for instance number and lower part for minor number
53 * space to identify partitions and slices. The virtual disk support is
54 * implemented by stacking cmlb module. For virtual disks, the partition
55 * related ioctl calls are routed to cmlb module. Compression and encryption
56 * is not supported for virtual disks.
57 *
58 * Mapped devices are tracked with state structures handled with
59 * ddi_soft_state(9F) for simplicity.
60 *
61 * A file attached to lofi is opened when attached and not closed until
62 * explicitly detached from lofi. This seems more sensible than deferring
63 * the open until the /dev/lofi device is opened, for a number of reasons.
64 * One is that any failure is likely to be noticed by the person (or script)
65 * running lofiadm. Another is that it would be a security problem if the
66 * file was replaced by another one after being added but before being opened.
67 *
68 * The only hard part about lofi is the ioctls. In order to support things
69 * like 'newfs' on a lofi device, it needs to support certain disk ioctls.
70 * So it has to fake disk geometry and partition information. More may need
71 * to be faked if your favorite utility doesn't work and you think it should
72 * (fdformat doesn't work because it really wants to know the type of floppy
73 * controller to talk to, and that didn't seem easy to fake. Or possibly even
74 * necessary, since we have mkfs_pcfs now).
75 *
76 * Normally, a lofi device cannot be detached if it is open (i.e. busy).  To
77 * support simulation of hotplug events, an optional force flag is provided.
78 * If a lofi device is open when a force detach is requested, then the
79 * underlying file is closed and any subsequent operations return EIO.  When the
80 * device is closed for the last time, it will be cleaned up at that time.  In
81 * addition, the DKIOCSTATE ioctl will return DKIO_DEV_GONE when the device is
82 * detached but not removed.
83 *
84 * If detach was requested and lofi device is not open, we will perform
85 * unmap and remove the lofi instance.
86 *
87 * If the lofi device is open and the li_cleanup is set on ioctl request,
88 * we set ls_cleanup flag to notify the cleanup is requested, and the
89 * last lofi_close will perform the unmapping and this lofi instance will be
90 * removed.
91 *
92 * If the lofi device is open and the li_force is set on ioctl request,
93 * we set ls_cleanup flag to notify the cleanup is requested,
94 * we also set ls_vp_closereq to notify IO tasks to return EIO on new
95 * IO requests and wait in process IO count to become 0, indicating there
96 * are no more IO requests. Since ls_cleanup is set, the last lofi_close
97 * will perform unmap and this lofi instance will be removed.
98 * See also lofi_unmap_file() for details.
99 *
100 * Once ls_cleanup is set for the instance, we do not allow lofi_open()
101 * calls to succeed and can have last lofi_close() to remove the instance.
102 *
103 * Known problems:
104 *
105 *	UFS logging. Mounting a UFS filesystem image "logging"
106 *	works for basic copy testing but wedges during a build of ON through
107 *	that image. Some deadlock in lufs holding the log mutex and then
108 *	getting stuck on a buf. So for now, don't do that.
109 *
110 *	Direct I/O. Since the filesystem data is being cached in the buffer
111 *	cache, _and_ again in the underlying filesystem, it's tempting to
112 *	enable direct I/O on the underlying file. Don't, because that deadlocks.
113 *	I think to fix the cache-twice problem we might need filesystem support.
114 *
115 * Interesting things to do:
116 *
117 *	Allow multiple files for each device. A poor-man's metadisk, basically.
118 *
119 *	Pass-through ioctls on block devices. You can (though it's not
120 *	documented), give lofi a block device as a file name. Then we shouldn't
121 *	need to fake a geometry, however, it may be relevant if you're replacing
122 *	metadisk, or using lofi to get crypto.
123 *	It makes sense to do lofiadm -c aes -a /dev/dsk/c0t0d0s4 /dev/lofi/1
124 *	and then in /etc/vfstab have an entry for /dev/lofi/1 as /export/home.
125 *	In fact this even makes sense if you have lofi "above" metadisk.
126 *
127 * Encryption:
128 *	Each lofi device can have its own symmetric key and cipher.
129 *	They are passed to us by lofiadm(1m) in the correct format for use
130 *	with the misc/kcf crypto_* routines.
131 *
132 *	Each block has its own IV, that is calculated in lofi_blk_mech(), based
133 *	on the "master" key held in the lsp and the block number of the buffer.
134 */
135
136#include <sys/types.h>
137#include <netinet/in.h>
138#include <sys/sysmacros.h>
139#include <sys/uio.h>
140#include <sys/kmem.h>
141#include <sys/cred.h>
142#include <sys/mman.h>
143#include <sys/errno.h>
144#include <sys/aio_req.h>
145#include <sys/stat.h>
146#include <sys/file.h>
147#include <sys/modctl.h>
148#include <sys/conf.h>
149#include <sys/debug.h>
150#include <sys/vnode.h>
151#include <sys/lofi.h>
152#include <sys/lofi_impl.h>	/* for cache structure */
153#include <sys/fcntl.h>
154#include <sys/pathname.h>
155#include <sys/filio.h>
156#include <sys/fdio.h>
157#include <sys/open.h>
158#include <sys/disp.h>
159#include <vm/seg_map.h>
160#include <sys/ddi.h>
161#include <sys/sunddi.h>
162#include <sys/zmod.h>
163#include <sys/id_space.h>
164#include <sys/mkdev.h>
165#include <sys/crypto/common.h>
166#include <sys/crypto/api.h>
167#include <sys/rctl.h>
168#include <sys/vtoc.h>
169#include <sys/scsi/scsi.h>	/* for DTYPE_DIRECT */
170#include <sys/scsi/impl/uscsi.h>
171#include <sys/sysevent/dev.h>
172#include <sys/efi_partition.h>
173#include <sys/note.h>
174#include <LzmaDec.h>
175
176#define	NBLOCKS_PROP_NAME	"Nblocks"
177#define	SIZE_PROP_NAME		"Size"
178#define	ZONE_PROP_NAME		"zone"
179
180#define	SETUP_C_DATA(cd, buf, len)		\
181	(cd).cd_format = CRYPTO_DATA_RAW;	\
182	(cd).cd_offset = 0;			\
183	(cd).cd_miscdata = NULL;		\
184	(cd).cd_length = (len);			\
185	(cd).cd_raw.iov_base = (buf);		\
186	(cd).cd_raw.iov_len = (len);
187
188#define	UIO_CHECK(uio)	\
189	if (((uio)->uio_loffset % DEV_BSIZE) != 0 || \
190	    ((uio)->uio_resid % DEV_BSIZE) != 0) { \
191		return (EINVAL); \
192	}
193
194#define	LOFI_TIMEOUT	120
195
196int lofi_timeout = LOFI_TIMEOUT;
197static void *lofi_statep;
198static kmutex_t lofi_lock;		/* state lock */
199static id_space_t *lofi_id;		/* lofi ID values */
200static list_t lofi_list;
201static zone_key_t lofi_zone_key;
202
203/*
204 * Because lofi_taskq_nthreads limits the actual swamping of the device, the
205 * maxalloc parameter (lofi_taskq_maxalloc) should be tuned conservatively
206 * high.  If we want to be assured that the underlying device is always busy,
207 * we must be sure that the number of bytes enqueued when the number of
208 * enqueued tasks exceeds maxalloc is sufficient to keep the device busy for
209 * the duration of the sleep time in taskq_ent_alloc().  That is, lofi should
210 * set maxalloc to be the maximum throughput (in bytes per second) of the
211 * underlying device divided by the minimum I/O size.  We assume a realistic
212 * maximum throughput of one hundred megabytes per second; we set maxalloc on
213 * the lofi task queue to be 104857600 divided by DEV_BSIZE.
214 */
215static int lofi_taskq_maxalloc = 104857600 / DEV_BSIZE;
216static int lofi_taskq_nthreads = 4;	/* # of taskq threads per device */
217
218const char lofi_crypto_magic[6] = LOFI_CRYPTO_MAGIC;
219
220/*
221 * To avoid decompressing data in a compressed segment multiple times
222 * when accessing small parts of a segment's data, we cache and reuse
223 * the uncompressed segment's data.
224 *
225 * A single cached segment is sufficient to avoid lots of duplicate
226 * segment decompress operations. A small cache size also reduces the
227 * memory footprint.
228 *
229 * lofi_max_comp_cache is the maximum number of decompressed data segments
230 * cached for each compressed lofi image. It can be set to 0 to disable
231 * caching.
232 */
233
234uint32_t lofi_max_comp_cache = 1;
235
236static int gzip_decompress(void *src, size_t srclen, void *dst,
237	size_t *destlen, int level);
238
239static int lzma_decompress(void *src, size_t srclen, void *dst,
240	size_t *dstlen, int level);
241
242lofi_compress_info_t lofi_compress_table[LOFI_COMPRESS_FUNCTIONS] = {
243	{gzip_decompress,	NULL,	6,	"gzip"}, /* default */
244	{gzip_decompress,	NULL,	6,	"gzip-6"},
245	{gzip_decompress,	NULL,	9,	"gzip-9"},
246	{lzma_decompress,	NULL,	0,	"lzma"}
247};
248
249static void lofi_strategy_task(void *);
250static int lofi_tg_rdwr(dev_info_t *, uchar_t, void *, diskaddr_t,
251    size_t, void *);
252static int lofi_tg_getinfo(dev_info_t *, int, void *, void *);
253
254struct cmlb_tg_ops lofi_tg_ops = {
255	TG_DK_OPS_VERSION_1,
256	lofi_tg_rdwr,
257	lofi_tg_getinfo
258};
259
260/*ARGSUSED*/
261static void
262*SzAlloc(void *p, size_t size)
263{
264	return (kmem_alloc(size, KM_SLEEP));
265}
266
267/*ARGSUSED*/
268static void
269SzFree(void *p, void *address, size_t size)
270{
271	kmem_free(address, size);
272}
273
274static ISzAlloc g_Alloc = { SzAlloc, SzFree };
275
276/*
277 * Free data referenced by the linked list of cached uncompressed
278 * segments.
279 */
280static void
281lofi_free_comp_cache(struct lofi_state *lsp)
282{
283	struct lofi_comp_cache *lc;
284
285	while ((lc = list_remove_head(&lsp->ls_comp_cache)) != NULL) {
286		kmem_free(lc->lc_data, lsp->ls_uncomp_seg_sz);
287		kmem_free(lc, sizeof (struct lofi_comp_cache));
288		lsp->ls_comp_cache_count--;
289	}
290	ASSERT(lsp->ls_comp_cache_count == 0);
291}
292
293static int
294is_opened(struct lofi_state *lsp)
295{
296	int i;
297	boolean_t last = B_TRUE;
298
299	ASSERT(MUTEX_HELD(&lofi_lock));
300	for (i = 0; i < LOFI_PART_MAX; i++) {
301		if (lsp->ls_open_lyr[i]) {
302			last = B_FALSE;
303			break;
304		}
305	}
306
307	for (i = 0; last && (i < OTYP_LYR); i++) {
308		if (lsp->ls_open_reg[i]) {
309			last = B_FALSE;
310		}
311	}
312
313	return (!last);
314}
315
316static void
317lofi_set_cleanup(struct lofi_state *lsp)
318{
319	ASSERT(MUTEX_HELD(&lofi_lock));
320
321	lsp->ls_cleanup = B_TRUE;
322
323	/* wake up any threads waiting on dkiocstate */
324	cv_broadcast(&lsp->ls_vp_cv);
325}
326
327static void
328lofi_free_crypto(struct lofi_state *lsp)
329{
330	ASSERT(MUTEX_HELD(&lofi_lock));
331
332	if (lsp->ls_crypto_enabled) {
333		/*
334		 * Clean up the crypto state so that it doesn't hang around
335		 * in memory after we are done with it.
336		 */
337		if (lsp->ls_key.ck_data != NULL) {
338			bzero(lsp->ls_key.ck_data,
339			    CRYPTO_BITS2BYTES(lsp->ls_key.ck_length));
340			kmem_free(lsp->ls_key.ck_data,
341			    CRYPTO_BITS2BYTES(lsp->ls_key.ck_length));
342			lsp->ls_key.ck_data = NULL;
343			lsp->ls_key.ck_length = 0;
344		}
345
346		if (lsp->ls_mech.cm_param != NULL) {
347			kmem_free(lsp->ls_mech.cm_param,
348			    lsp->ls_mech.cm_param_len);
349			lsp->ls_mech.cm_param = NULL;
350			lsp->ls_mech.cm_param_len = 0;
351		}
352
353		if (lsp->ls_iv_mech.cm_param != NULL) {
354			kmem_free(lsp->ls_iv_mech.cm_param,
355			    lsp->ls_iv_mech.cm_param_len);
356			lsp->ls_iv_mech.cm_param = NULL;
357			lsp->ls_iv_mech.cm_param_len = 0;
358		}
359
360		mutex_destroy(&lsp->ls_crypto_lock);
361	}
362}
363
364/* ARGSUSED */
365static int
366lofi_tg_rdwr(dev_info_t *dip, uchar_t cmd, void *bufaddr, diskaddr_t start,
367    size_t length, void *tg_cookie)
368{
369	struct lofi_state *lsp;
370	buf_t	*bp;
371	int	instance;
372	int	rv = 0;
373
374	instance = ddi_get_instance(dip);
375	if (instance == 0)	/* control node does not have disk */
376		return (ENXIO);
377
378	lsp = ddi_get_soft_state(lofi_statep, instance);
379
380	if (lsp == NULL)
381		return (ENXIO);
382
383	if (cmd != TG_READ && cmd != TG_WRITE)
384		return (EINVAL);
385
386	/*
387	 * Make sure the mapping is set up by checking lsp->ls_vp_ready.
388	 */
389	mutex_enter(&lsp->ls_vp_lock);
390	while (lsp->ls_vp_ready == B_FALSE)
391		cv_wait(&lsp->ls_vp_cv, &lsp->ls_vp_lock);
392	mutex_exit(&lsp->ls_vp_lock);
393
394	if (P2PHASE(length, (1U << lsp->ls_lbshift)) != 0) {
395		/* We can only transfer whole blocks at a time! */
396		return (EINVAL);
397	}
398
399	bp = getrbuf(KM_SLEEP);
400
401	if (cmd == TG_READ) {
402		bp->b_flags = B_READ;
403	} else {
404		if (lsp->ls_readonly == B_TRUE) {
405			freerbuf(bp);
406			return (EROFS);
407		}
408		bp->b_flags = B_WRITE;
409	}
410
411	bp->b_un.b_addr = bufaddr;
412	bp->b_bcount = length;
413	bp->b_lblkno = start;
414	bp->b_private = NULL;
415	bp->b_edev = lsp->ls_dev;
416
417	if (lsp->ls_kstat) {
418		mutex_enter(lsp->ls_kstat->ks_lock);
419		kstat_waitq_enter(KSTAT_IO_PTR(lsp->ls_kstat));
420		mutex_exit(lsp->ls_kstat->ks_lock);
421	}
422	(void) taskq_dispatch(lsp->ls_taskq, lofi_strategy_task, bp, KM_SLEEP);
423	(void) biowait(bp);
424
425	rv = geterror(bp);
426	freerbuf(bp);
427	return (rv);
428}
429
430/*
431 * Get device geometry info for cmlb.
432 *
433 * We have mapped disk image as virtual block device and have to report
434 * physical/virtual geometry to cmlb.
435 *
436 * So we have two principal cases:
437 * 1. Uninitialised image without any existing labels,
438 *    for this case we fabricate the data based on mapped image.
439 * 2. Image with existing label information.
440 *    Since we have no information how the image was created (it may be
441 *    dump from some physical device), we need to rely on label information
442 *    from image, or we get "corrupted label" errors.
443 *    NOTE: label can be MBR, MBR+SMI, GPT
444 */
445static int
446lofi_tg_getinfo(dev_info_t *dip, int cmd, void *arg, void *tg_cookie)
447{
448	struct lofi_state *lsp;
449	int instance;
450	int ashift;
451
452	_NOTE(ARGUNUSED(tg_cookie));
453	instance = ddi_get_instance(dip);
454	if (instance == 0)		/* control device has no storage */
455		return (ENXIO);
456
457	lsp = ddi_get_soft_state(lofi_statep, instance);
458
459	if (lsp == NULL)
460		return (ENXIO);
461
462	/*
463	 * Make sure the mapping is set up by checking lsp->ls_vp_ready.
464	 *
465	 * When mapping is created, new lofi instance is created and
466	 * lofi_attach() will call cmlb_attach() as part of the procedure
467	 * to set the mapping up. This chain of events will happen in
468	 * the same thread.
469	 * Since cmlb_attach() will call lofi_tg_getinfo to get
470	 * capacity, we return error on that call if cookie is set,
471	 * otherwise lofi_attach will be stuck as the mapping is not yet
472	 * finalized and lofi is not yet ready.
473	 * Note, such error is not fatal for cmlb, as the label setup
474	 * will be finalized when cmlb_validate() is called.
475	 */
476	mutex_enter(&lsp->ls_vp_lock);
477	if (tg_cookie != NULL && lsp->ls_vp_ready == B_FALSE) {
478		mutex_exit(&lsp->ls_vp_lock);
479		return (ENXIO);
480	}
481	while (lsp->ls_vp_ready == B_FALSE)
482		cv_wait(&lsp->ls_vp_cv, &lsp->ls_vp_lock);
483	mutex_exit(&lsp->ls_vp_lock);
484
485	ashift = lsp->ls_lbshift;
486
487	switch (cmd) {
488	case TG_GETPHYGEOM: {
489		cmlb_geom_t *geomp = arg;
490
491		geomp->g_capacity	=
492		    (lsp->ls_vp_size - lsp->ls_crypto_offset) >> ashift;
493		geomp->g_nsect		= lsp->ls_dkg.dkg_nsect;
494		geomp->g_nhead		= lsp->ls_dkg.dkg_nhead;
495		geomp->g_acyl		= lsp->ls_dkg.dkg_acyl;
496		geomp->g_ncyl		= lsp->ls_dkg.dkg_ncyl;
497		geomp->g_secsize	= (1U << ashift);
498		geomp->g_intrlv		= lsp->ls_dkg.dkg_intrlv;
499		geomp->g_rpm		= lsp->ls_dkg.dkg_rpm;
500		return (0);
501	}
502
503	case TG_GETCAPACITY:
504		*(diskaddr_t *)arg =
505		    (lsp->ls_vp_size - lsp->ls_crypto_offset) >> ashift;
506		return (0);
507
508	case TG_GETBLOCKSIZE:
509		*(uint32_t *)arg = (1U << ashift);
510		return (0);
511
512	case TG_GETATTR: {
513		tg_attribute_t *tgattr = arg;
514
515		tgattr->media_is_writable = !lsp->ls_readonly;
516		tgattr->media_is_solid_state = B_FALSE;
517		tgattr->media_is_rotational = B_FALSE;
518		return (0);
519	}
520
521	default:
522		return (EINVAL);
523	}
524}
525
526static void
527lofi_destroy(struct lofi_state *lsp, cred_t *credp)
528{
529	int id = LOFI_MINOR2ID(getminor(lsp->ls_dev));
530	int i;
531
532	ASSERT(MUTEX_HELD(&lofi_lock));
533
534	/*
535	 * Before we can start to release the other resources,
536	 * make sure we have all tasks completed and taskq removed.
537	 */
538	if (lsp->ls_taskq != NULL) {
539		taskq_destroy(lsp->ls_taskq);
540		lsp->ls_taskq = NULL;
541	}
542
543	list_remove(&lofi_list, lsp);
544
545	lofi_free_crypto(lsp);
546
547	/*
548	 * Free pre-allocated compressed buffers
549	 */
550	if (lsp->ls_comp_bufs != NULL) {
551		for (i = 0; i < lofi_taskq_nthreads; i++) {
552			if (lsp->ls_comp_bufs[i].bufsize > 0)
553				kmem_free(lsp->ls_comp_bufs[i].buf,
554				    lsp->ls_comp_bufs[i].bufsize);
555		}
556		kmem_free(lsp->ls_comp_bufs,
557		    sizeof (struct compbuf) * lofi_taskq_nthreads);
558	}
559
560	if (lsp->ls_vp != NULL) {
561		(void) VOP_PUTPAGE(lsp->ls_vp, 0, 0, B_FREE, credp, NULL);
562		(void) VOP_CLOSE(lsp->ls_vp, lsp->ls_openflag,
563		    1, 0, credp, NULL);
564		VN_RELE(lsp->ls_vp);
565	}
566	if (lsp->ls_stacked_vp != lsp->ls_vp)
567		VN_RELE(lsp->ls_stacked_vp);
568	lsp->ls_vp = lsp->ls_stacked_vp = NULL;
569
570	if (lsp->ls_kstat != NULL) {
571		kstat_delete(lsp->ls_kstat);
572		lsp->ls_kstat = NULL;
573	}
574
575	/*
576	 * Free cached decompressed segment data
577	 */
578	lofi_free_comp_cache(lsp);
579	list_destroy(&lsp->ls_comp_cache);
580
581	if (lsp->ls_uncomp_seg_sz > 0) {
582		kmem_free(lsp->ls_comp_index_data, lsp->ls_comp_index_data_sz);
583		lsp->ls_uncomp_seg_sz = 0;
584	}
585
586	rctl_decr_lofi(lsp->ls_zone.zref_zone, 1);
587	zone_rele_ref(&lsp->ls_zone, ZONE_REF_LOFI);
588
589	mutex_destroy(&lsp->ls_comp_cache_lock);
590	mutex_destroy(&lsp->ls_comp_bufs_lock);
591	mutex_destroy(&lsp->ls_kstat_lock);
592	mutex_destroy(&lsp->ls_vp_lock);
593	cv_destroy(&lsp->ls_vp_cv);
594	lsp->ls_vp_ready = B_FALSE;
595	lsp->ls_vp_closereq = B_FALSE;
596
597	ASSERT(ddi_get_soft_state(lofi_statep, id) == lsp);
598	(void) ndi_devi_offline(lsp->ls_dip, NDI_DEVI_REMOVE);
599	id_free(lofi_id, id);
600}
601
602static void
603lofi_free_dev(struct lofi_state *lsp)
604{
605	ASSERT(MUTEX_HELD(&lofi_lock));
606
607	if (lsp->ls_cmlbhandle != NULL) {
608		cmlb_invalidate(lsp->ls_cmlbhandle, 0);
609		cmlb_detach(lsp->ls_cmlbhandle, 0);
610		cmlb_free_handle(&lsp->ls_cmlbhandle);
611		lsp->ls_cmlbhandle = NULL;
612	}
613	(void) ddi_prop_remove_all(lsp->ls_dip);
614	ddi_remove_minor_node(lsp->ls_dip, NULL);
615}
616
617/*ARGSUSED*/
618static void
619lofi_zone_shutdown(zoneid_t zoneid, void *arg)
620{
621	struct lofi_state *lsp;
622	struct lofi_state *next;
623
624	mutex_enter(&lofi_lock);
625
626	for (lsp = list_head(&lofi_list); lsp != NULL; lsp = next) {
627
628		/* lofi_destroy() frees lsp */
629		next = list_next(&lofi_list, lsp);
630
631		if (lsp->ls_zone.zref_zone->zone_id != zoneid)
632			continue;
633
634		/*
635		 * No in-zone processes are running, but something has this
636		 * open.  It's either a global zone process, or a lofi
637		 * mount.  In either case we set ls_cleanup so the last
638		 * user destroys the device.
639		 */
640		if (is_opened(lsp)) {
641			lofi_set_cleanup(lsp);
642		} else {
643			lofi_free_dev(lsp);
644			lofi_destroy(lsp, kcred);
645		}
646	}
647
648	mutex_exit(&lofi_lock);
649}
650
651/*ARGSUSED*/
652static int
653lofi_open(dev_t *devp, int flag, int otyp, struct cred *credp)
654{
655	int id;
656	minor_t	part;
657	uint64_t mask;
658	diskaddr_t nblks;
659	diskaddr_t lba;
660	boolean_t ndelay;
661
662	struct lofi_state *lsp;
663
664	if (otyp >= OTYPCNT)
665		return (EINVAL);
666
667	ndelay = (flag & (FNDELAY | FNONBLOCK)) ? B_TRUE : B_FALSE;
668
669	/*
670	 * lofiadm -a /dev/lofi/1 gets us here.
671	 */
672	if (mutex_owner(&lofi_lock) == curthread)
673		return (EINVAL);
674
675	mutex_enter(&lofi_lock);
676
677	id = LOFI_MINOR2ID(getminor(*devp));
678	part = LOFI_PART(getminor(*devp));
679	mask = (1U << part);
680
681	/* master control device */
682	if (id == 0) {
683		mutex_exit(&lofi_lock);
684		return (0);
685	}
686
687	/* otherwise, the mapping should already exist */
688	lsp = ddi_get_soft_state(lofi_statep, id);
689	if (lsp == NULL) {
690		mutex_exit(&lofi_lock);
691		return (EINVAL);
692	}
693
694	if (lsp->ls_cleanup == B_TRUE) {
695		mutex_exit(&lofi_lock);
696		return (ENXIO);
697	}
698
699	if (lsp->ls_vp == NULL) {
700		mutex_exit(&lofi_lock);
701		return (ENXIO);
702	}
703
704	if (lsp->ls_readonly && (flag & FWRITE)) {
705		mutex_exit(&lofi_lock);
706		return (EROFS);
707	}
708
709	if ((lsp->ls_open_excl) & (mask)) {
710		mutex_exit(&lofi_lock);
711		return (EBUSY);
712	}
713
714	if (flag & FEXCL) {
715		if (lsp->ls_open_lyr[part]) {
716			mutex_exit(&lofi_lock);
717			return (EBUSY);
718		}
719		for (int i = 0; i < OTYP_LYR; i++) {
720			if (lsp->ls_open_reg[i] & mask) {
721				mutex_exit(&lofi_lock);
722				return (EBUSY);
723			}
724		}
725	}
726
727	if (lsp->ls_cmlbhandle != NULL) {
728		if (cmlb_validate(lsp->ls_cmlbhandle, 0, 0) != 0) {
729			/*
730			 * non-blocking opens are allowed to succeed to
731			 * support format and fdisk to create partitioning.
732			 */
733			if (!ndelay) {
734				mutex_exit(&lofi_lock);
735				return (ENXIO);
736			}
737		} else if (cmlb_partinfo(lsp->ls_cmlbhandle, part, &nblks, &lba,
738		    NULL, NULL, 0) == 0) {
739			if ((!nblks) && ((!ndelay) || (otyp != OTYP_CHR))) {
740				mutex_exit(&lofi_lock);
741				return (ENXIO);
742			}
743		} else if (!ndelay) {
744			mutex_exit(&lofi_lock);
745			return (ENXIO);
746		}
747	}
748
749	if (otyp == OTYP_LYR) {
750		lsp->ls_open_lyr[part]++;
751	} else {
752		lsp->ls_open_reg[otyp] |= mask;
753	}
754	if (flag & FEXCL) {
755		lsp->ls_open_excl |= mask;
756	}
757
758	mutex_exit(&lofi_lock);
759	return (0);
760}
761
762/*ARGSUSED*/
763static int
764lofi_close(dev_t dev, int flag, int otyp, struct cred *credp)
765{
766	minor_t	part;
767	int id;
768	uint64_t mask;
769	struct lofi_state *lsp;
770
771	id = LOFI_MINOR2ID(getminor(dev));
772	part = LOFI_PART(getminor(dev));
773	mask = (1U << part);
774
775	mutex_enter(&lofi_lock);
776	lsp = ddi_get_soft_state(lofi_statep, id);
777	if (lsp == NULL) {
778		mutex_exit(&lofi_lock);
779		return (EINVAL);
780	}
781
782	if (id == 0) {
783		mutex_exit(&lofi_lock);
784		return (0);
785	}
786
787	if (lsp->ls_open_excl & mask)
788		lsp->ls_open_excl &= ~mask;
789
790	if (otyp == OTYP_LYR) {
791		lsp->ls_open_lyr[part]--;
792	} else {
793		lsp->ls_open_reg[otyp] &= ~mask;
794	}
795
796	/*
797	 * If we forcibly closed the underlying device (li_force), or
798	 * asked for cleanup (li_cleanup), finish up if we're the last
799	 * out of the door.
800	 */
801	if (!is_opened(lsp) &&
802	    (lsp->ls_cleanup == B_TRUE || lsp->ls_vp == NULL)) {
803		lofi_free_dev(lsp);
804		lofi_destroy(lsp, credp);
805	}
806
807	mutex_exit(&lofi_lock);
808	return (0);
809}
810
811/*
812 * Sets the mechanism's initialization vector (IV) if one is needed.
813 * The IV is computed from the data block number.  lsp->ls_mech is
814 * altered so that:
815 *	lsp->ls_mech.cm_param_len is set to the IV len.
816 *	lsp->ls_mech.cm_param is set to the IV.
817 */
818static int
819lofi_blk_mech(struct lofi_state *lsp, longlong_t lblkno)
820{
821	int	ret;
822	crypto_data_t cdata;
823	char	*iv;
824	size_t	iv_len;
825	size_t	min;
826	void	*data;
827	size_t	datasz;
828
829	ASSERT(MUTEX_HELD(&lsp->ls_crypto_lock));
830
831	if (lsp == NULL)
832		return (CRYPTO_DEVICE_ERROR);
833
834	/* lsp->ls_mech.cm_param{_len} has already been set for static iv */
835	if (lsp->ls_iv_type == IVM_NONE) {
836		return (CRYPTO_SUCCESS);
837	}
838
839	/*
840	 * if kmem already alloced from previous call and it's the same size
841	 * we need now, just recycle it; allocate new kmem only if we have to
842	 */
843	if (lsp->ls_mech.cm_param == NULL ||
844	    lsp->ls_mech.cm_param_len != lsp->ls_iv_len) {
845		iv_len = lsp->ls_iv_len;
846		iv = kmem_zalloc(iv_len, KM_SLEEP);
847	} else {
848		iv_len = lsp->ls_mech.cm_param_len;
849		iv = lsp->ls_mech.cm_param;
850		bzero(iv, iv_len);
851	}
852
853	switch (lsp->ls_iv_type) {
854	case IVM_ENC_BLKNO:
855		/* iv is not static, lblkno changes each time */
856		data = &lblkno;
857		datasz = sizeof (lblkno);
858		break;
859	default:
860		data = 0;
861		datasz = 0;
862		break;
863	}
864
865	/*
866	 * write blkno into the iv buffer padded on the left in case
867	 * blkno ever grows bigger than its current longlong_t size
868	 * or a variation other than blkno is used for the iv data
869	 */
870	min = MIN(datasz, iv_len);
871	bcopy(data, iv + (iv_len - min), min);
872
873	/* encrypt the data in-place to get the IV */
874	SETUP_C_DATA(cdata, iv, iv_len);
875
876	ret = crypto_encrypt(&lsp->ls_iv_mech, &cdata, &lsp->ls_key,
877	    NULL, NULL, NULL);
878	if (ret != CRYPTO_SUCCESS) {
879		cmn_err(CE_WARN, "failed to create iv for block %lld: (0x%x)",
880		    lblkno, ret);
881		if (lsp->ls_mech.cm_param != iv)
882			kmem_free(iv, iv_len);
883
884		return (ret);
885	}
886
887	/* clean up the iv from the last computation */
888	if (lsp->ls_mech.cm_param != NULL && lsp->ls_mech.cm_param != iv)
889		kmem_free(lsp->ls_mech.cm_param, lsp->ls_mech.cm_param_len);
890
891	lsp->ls_mech.cm_param_len = iv_len;
892	lsp->ls_mech.cm_param = iv;
893
894	return (CRYPTO_SUCCESS);
895}
896
897/*
898 * Performs encryption and decryption of a chunk of data of size "len",
899 * one DEV_BSIZE block at a time.  "len" is assumed to be a multiple of
900 * DEV_BSIZE.
901 */
902static int
903lofi_crypto(struct lofi_state *lsp, struct buf *bp, caddr_t plaintext,
904    caddr_t ciphertext, size_t len, boolean_t op_encrypt)
905{
906	crypto_data_t cdata;
907	crypto_data_t wdata;
908	int ret;
909	longlong_t lblkno = bp->b_lblkno;
910
911	mutex_enter(&lsp->ls_crypto_lock);
912
913	/*
914	 * though we could encrypt/decrypt entire "len" chunk of data, we need
915	 * to break it into DEV_BSIZE pieces to capture blkno incrementing
916	 */
917	SETUP_C_DATA(cdata, plaintext, len);
918	cdata.cd_length = DEV_BSIZE;
919	if (ciphertext != NULL) {		/* not in-place crypto */
920		SETUP_C_DATA(wdata, ciphertext, len);
921		wdata.cd_length = DEV_BSIZE;
922	}
923
924	do {
925		ret = lofi_blk_mech(lsp, lblkno);
926		if (ret != CRYPTO_SUCCESS)
927			continue;
928
929		if (op_encrypt) {
930			ret = crypto_encrypt(&lsp->ls_mech, &cdata,
931			    &lsp->ls_key, NULL,
932			    ((ciphertext != NULL) ? &wdata : NULL), NULL);
933		} else {
934			ret = crypto_decrypt(&lsp->ls_mech, &cdata,
935			    &lsp->ls_key, NULL,
936			    ((ciphertext != NULL) ? &wdata : NULL), NULL);
937		}
938
939		cdata.cd_offset += DEV_BSIZE;
940		if (ciphertext != NULL)
941			wdata.cd_offset += DEV_BSIZE;
942		lblkno++;
943	} while (ret == CRYPTO_SUCCESS && cdata.cd_offset < len);
944
945	mutex_exit(&lsp->ls_crypto_lock);
946
947	if (ret != CRYPTO_SUCCESS) {
948		cmn_err(CE_WARN, "%s failed for block %lld:  (0x%x)",
949		    op_encrypt ? "crypto_encrypt()" : "crypto_decrypt()",
950		    lblkno, ret);
951	}
952
953	return (ret);
954}
955
956#define	RDWR_RAW	1
957#define	RDWR_BCOPY	2
958
959static int
960lofi_rdwr(caddr_t bufaddr, offset_t offset, struct buf *bp,
961    struct lofi_state *lsp, size_t len, int method, caddr_t bcopy_locn)
962{
963	ssize_t resid;
964	int isread;
965	int error;
966
967	/*
968	 * Handles reads/writes for both plain and encrypted lofi
969	 * Note:  offset is already shifted by lsp->ls_crypto_offset
970	 * when it gets here.
971	 */
972
973	isread = bp->b_flags & B_READ;
974	if (isread) {
975		if (method == RDWR_BCOPY) {
976			/* DO NOT update bp->b_resid for bcopy */
977			bcopy(bcopy_locn, bufaddr, len);
978			error = 0;
979		} else {		/* RDWR_RAW */
980			error = vn_rdwr(UIO_READ, lsp->ls_vp, bufaddr, len,
981			    offset, UIO_SYSSPACE, 0, RLIM64_INFINITY, kcred,
982			    &resid);
983			bp->b_resid = resid;
984		}
985		if (lsp->ls_crypto_enabled && error == 0) {
986			if (lofi_crypto(lsp, bp, bufaddr, NULL, len,
987			    B_FALSE) != CRYPTO_SUCCESS) {
988				/*
989				 * XXX: original code didn't set residual
990				 * back to len because no error was expected
991				 * from bcopy() if encryption is not enabled
992				 */
993				if (method != RDWR_BCOPY)
994					bp->b_resid = len;
995				error = EIO;
996			}
997		}
998		return (error);
999	} else {
1000		void *iobuf = bufaddr;
1001
1002		if (lsp->ls_crypto_enabled) {
1003			/* don't do in-place crypto to keep bufaddr intact */
1004			iobuf = kmem_alloc(len, KM_SLEEP);
1005			if (lofi_crypto(lsp, bp, bufaddr, iobuf, len,
1006			    B_TRUE) != CRYPTO_SUCCESS) {
1007				kmem_free(iobuf, len);
1008				if (method != RDWR_BCOPY)
1009					bp->b_resid = len;
1010				return (EIO);
1011			}
1012		}
1013		if (method == RDWR_BCOPY) {
1014			/* DO NOT update bp->b_resid for bcopy */
1015			bcopy(iobuf, bcopy_locn, len);
1016			error = 0;
1017		} else {		/* RDWR_RAW */
1018			error = vn_rdwr(UIO_WRITE, lsp->ls_vp, iobuf, len,
1019			    offset, UIO_SYSSPACE, 0, RLIM64_INFINITY, kcred,
1020			    &resid);
1021			bp->b_resid = resid;
1022		}
1023		if (lsp->ls_crypto_enabled) {
1024			kmem_free(iobuf, len);
1025		}
1026		return (error);
1027	}
1028}
1029
1030static int
1031lofi_mapped_rdwr(caddr_t bufaddr, offset_t offset, struct buf *bp,
1032    struct lofi_state *lsp)
1033{
1034	int error;
1035	offset_t alignedoffset, mapoffset;
1036	size_t	xfersize;
1037	int	isread;
1038	int	smflags;
1039	caddr_t	mapaddr;
1040	size_t	len;
1041	enum seg_rw srw;
1042	int	save_error;
1043
1044	/*
1045	 * Note:  offset is already shifted by lsp->ls_crypto_offset
1046	 * when it gets here.
1047	 */
1048	if (lsp->ls_crypto_enabled)
1049		ASSERT(lsp->ls_vp_comp_size == lsp->ls_vp_size);
1050
1051	/*
1052	 * segmap always gives us an 8K (MAXBSIZE) chunk, aligned on
1053	 * an 8K boundary, but the buf transfer address may not be
1054	 * aligned on more than a 512-byte boundary (we don't enforce
1055	 * that even though we could). This matters since the initial
1056	 * part of the transfer may not start at offset 0 within the
1057	 * segmap'd chunk. So we have to compensate for that with
1058	 * 'mapoffset'. Subsequent chunks always start off at the
1059	 * beginning, and the last is capped by b_resid
1060	 *
1061	 * Visually, where "|" represents page map boundaries:
1062	 *   alignedoffset (mapaddr begins at this segmap boundary)
1063	 *    |   offset (from beginning of file)
1064	 *    |    |	   len
1065	 *    v    v	    v
1066	 * ===|====X========|====...======|========X====|====
1067	 *	   /-------------...---------------/
1068	 *		^ bp->b_bcount/bp->b_resid at start
1069	 *    /----/--------/----...------/--------/
1070	 *	^	^	^   ^		^
1071	 *	|	|	|   |		nth xfersize (<= MAXBSIZE)
1072	 *	|	|	2nd thru n-1st xfersize (= MAXBSIZE)
1073	 *	|	1st xfersize (<= MAXBSIZE)
1074	 *    mapoffset (offset into 1st segmap, non-0 1st time, 0 thereafter)
1075	 *
1076	 * Notes: "alignedoffset" is "offset" rounded down to nearest
1077	 * MAXBSIZE boundary.  "len" is next page boundary of size
1078	 * PAGESIZE after "alignedoffset".
1079	 */
1080	mapoffset = offset & MAXBOFFSET;
1081	alignedoffset = offset - mapoffset;
1082	bp->b_resid = bp->b_bcount;
1083	isread = bp->b_flags & B_READ;
1084	srw = isread ? S_READ : S_WRITE;
1085	do {
1086		xfersize = MIN(lsp->ls_vp_comp_size - offset,
1087		    MIN(MAXBSIZE - mapoffset, bp->b_resid));
1088		len = roundup(mapoffset + xfersize, PAGESIZE);
1089		mapaddr = segmap_getmapflt(segkmap, lsp->ls_vp,
1090		    alignedoffset, MAXBSIZE, 1, srw);
1091		/*
1092		 * Now fault in the pages. This lets us check
1093		 * for errors before we reference mapaddr and
1094		 * try to resolve the fault in bcopy (which would
1095		 * panic instead). And this can easily happen,
1096		 * particularly if you've lofi'd a file over NFS
1097		 * and someone deletes the file on the server.
1098		 */
1099		error = segmap_fault(kas.a_hat, segkmap, mapaddr,
1100		    len, F_SOFTLOCK, srw);
1101		if (error) {
1102			(void) segmap_release(segkmap, mapaddr, 0);
1103			if (FC_CODE(error) == FC_OBJERR)
1104				error = FC_ERRNO(error);
1105			else
1106				error = EIO;
1107			break;
1108		}
1109		/* error may be non-zero for encrypted lofi */
1110		error = lofi_rdwr(bufaddr, 0, bp, lsp, xfersize,
1111		    RDWR_BCOPY, mapaddr + mapoffset);
1112		if (error == 0) {
1113			bp->b_resid -= xfersize;
1114			bufaddr += xfersize;
1115			offset += xfersize;
1116		}
1117		smflags = 0;
1118		if (isread) {
1119			smflags |= SM_FREE;
1120			/*
1121			 * If we're reading an entire page starting
1122			 * at a page boundary, there's a good chance
1123			 * we won't need it again. Put it on the
1124			 * head of the freelist.
1125			 */
1126			if (mapoffset == 0 && xfersize == MAXBSIZE)
1127				smflags |= SM_DONTNEED;
1128		} else {
1129			/*
1130			 * Write back good pages, it is okay to
1131			 * always release asynchronous here as we'll
1132			 * follow with VOP_FSYNC for B_SYNC buffers.
1133			 */
1134			if (error == 0)
1135				smflags |= SM_WRITE | SM_ASYNC;
1136		}
1137		(void) segmap_fault(kas.a_hat, segkmap, mapaddr,
1138		    len, F_SOFTUNLOCK, srw);
1139		save_error = segmap_release(segkmap, mapaddr, smflags);
1140		if (error == 0)
1141			error = save_error;
1142		/* only the first map may start partial */
1143		mapoffset = 0;
1144		alignedoffset += MAXBSIZE;
1145	} while ((error == 0) && (bp->b_resid > 0) &&
1146	    (offset < lsp->ls_vp_comp_size));
1147
1148	return (error);
1149}
1150
1151/*
1152 * Check if segment seg_index is present in the decompressed segment
1153 * data cache.
1154 *
1155 * Returns a pointer to the decompressed segment data cache entry if
1156 * found, and NULL when decompressed data for this segment is not yet
1157 * cached.
1158 */
1159static struct lofi_comp_cache *
1160lofi_find_comp_data(struct lofi_state *lsp, uint64_t seg_index)
1161{
1162	struct lofi_comp_cache *lc;
1163
1164	ASSERT(MUTEX_HELD(&lsp->ls_comp_cache_lock));
1165
1166	for (lc = list_head(&lsp->ls_comp_cache); lc != NULL;
1167	    lc = list_next(&lsp->ls_comp_cache, lc)) {
1168		if (lc->lc_index == seg_index) {
1169			/*
1170			 * Decompressed segment data was found in the
1171			 * cache.
1172			 *
1173			 * The cache uses an LRU replacement strategy;
1174			 * move the entry to head of list.
1175			 */
1176			list_remove(&lsp->ls_comp_cache, lc);
1177			list_insert_head(&lsp->ls_comp_cache, lc);
1178			return (lc);
1179		}
1180	}
1181	return (NULL);
1182}
1183
1184/*
1185 * Add the data for a decompressed segment at segment index
1186 * seg_index to the cache of the decompressed segments.
1187 *
1188 * Returns a pointer to the cache element structure in case
1189 * the data was added to the cache; returns NULL when the data
1190 * wasn't cached.
1191 */
1192static struct lofi_comp_cache *
1193lofi_add_comp_data(struct lofi_state *lsp, uint64_t seg_index,
1194    uchar_t *data)
1195{
1196	struct lofi_comp_cache *lc;
1197
1198	ASSERT(MUTEX_HELD(&lsp->ls_comp_cache_lock));
1199
1200	while (lsp->ls_comp_cache_count > lofi_max_comp_cache) {
1201		lc = list_remove_tail(&lsp->ls_comp_cache);
1202		ASSERT(lc != NULL);
1203		kmem_free(lc->lc_data, lsp->ls_uncomp_seg_sz);
1204		kmem_free(lc, sizeof (struct lofi_comp_cache));
1205		lsp->ls_comp_cache_count--;
1206	}
1207
1208	/*
1209	 * Do not cache when disabled by tunable variable
1210	 */
1211	if (lofi_max_comp_cache == 0)
1212		return (NULL);
1213
1214	/*
1215	 * When the cache has not yet reached the maximum allowed
1216	 * number of segments, allocate a new cache element.
1217	 * Otherwise the cache is full; reuse the last list element
1218	 * (LRU) for caching the decompressed segment data.
1219	 *
1220	 * The cache element for the new decompressed segment data is
1221	 * added to the head of the list.
1222	 */
1223	if (lsp->ls_comp_cache_count < lofi_max_comp_cache) {
1224		lc = kmem_alloc(sizeof (struct lofi_comp_cache), KM_SLEEP);
1225		lc->lc_data = NULL;
1226		list_insert_head(&lsp->ls_comp_cache, lc);
1227		lsp->ls_comp_cache_count++;
1228	} else {
1229		lc = list_remove_tail(&lsp->ls_comp_cache);
1230		if (lc == NULL)
1231			return (NULL);
1232		list_insert_head(&lsp->ls_comp_cache, lc);
1233	}
1234
1235	/*
1236	 * Free old uncompressed segment data when reusing a cache
1237	 * entry.
1238	 */
1239	if (lc->lc_data != NULL)
1240		kmem_free(lc->lc_data, lsp->ls_uncomp_seg_sz);
1241
1242	lc->lc_data = data;
1243	lc->lc_index = seg_index;
1244	return (lc);
1245}
1246
1247
1248/*ARGSUSED*/
1249static int
1250gzip_decompress(void *src, size_t srclen, void *dst,
1251    size_t *dstlen, int level)
1252{
1253	ASSERT(*dstlen >= srclen);
1254
1255	if (z_uncompress(dst, dstlen, src, srclen) != Z_OK)
1256		return (-1);
1257	return (0);
1258}
1259
1260#define	LZMA_HEADER_SIZE	(LZMA_PROPS_SIZE + 8)
1261/*ARGSUSED*/
1262static int
1263lzma_decompress(void *src, size_t srclen, void *dst,
1264    size_t *dstlen, int level)
1265{
1266	size_t insizepure;
1267	void *actual_src;
1268	ELzmaStatus status;
1269
1270	insizepure = srclen - LZMA_HEADER_SIZE;
1271	actual_src = (void *)((Byte *)src + LZMA_HEADER_SIZE);
1272
1273	if (LzmaDecode((Byte *)dst, (size_t *)dstlen,
1274	    (const Byte *)actual_src, &insizepure,
1275	    (const Byte *)src, LZMA_PROPS_SIZE, LZMA_FINISH_ANY, &status,
1276	    &g_Alloc) != SZ_OK) {
1277		return (-1);
1278	}
1279	return (0);
1280}
1281
1282/*
1283 * This is basically what strategy used to be before we found we
1284 * needed task queues.
1285 */
1286static void
1287lofi_strategy_task(void *arg)
1288{
1289	struct buf *bp = (struct buf *)arg;
1290	int error;
1291	int syncflag = 0;
1292	struct lofi_state *lsp;
1293	offset_t offset;
1294	caddr_t	bufaddr;
1295	size_t	len;
1296	size_t	xfersize;
1297	boolean_t bufinited = B_FALSE;
1298
1299	lsp = ddi_get_soft_state(lofi_statep,
1300	    LOFI_MINOR2ID(getminor(bp->b_edev)));
1301
1302	if (lsp == NULL) {
1303		error = ENXIO;
1304		goto errout;
1305	}
1306	if (lsp->ls_kstat) {
1307		mutex_enter(lsp->ls_kstat->ks_lock);
1308		kstat_waitq_to_runq(KSTAT_IO_PTR(lsp->ls_kstat));
1309		mutex_exit(lsp->ls_kstat->ks_lock);
1310	}
1311
1312	mutex_enter(&lsp->ls_vp_lock);
1313	lsp->ls_vp_iocount++;
1314	mutex_exit(&lsp->ls_vp_lock);
1315
1316	bp_mapin(bp);
1317	bufaddr = bp->b_un.b_addr;
1318	offset = (bp->b_lblkno + (diskaddr_t)(uintptr_t)bp->b_private)
1319	    << lsp->ls_lbshift;	/* offset within file */
1320	if (lsp->ls_crypto_enabled) {
1321		/* encrypted data really begins after crypto header */
1322		offset += lsp->ls_crypto_offset;
1323	}
1324	len = bp->b_bcount;
1325	bufinited = B_TRUE;
1326
1327	if (lsp->ls_vp == NULL || lsp->ls_vp_closereq) {
1328		error = EIO;
1329		goto errout;
1330	}
1331
1332	/*
1333	 * If we're writing and the buffer was not B_ASYNC
1334	 * we'll follow up with a VOP_FSYNC() to force any
1335	 * asynchronous I/O to stable storage.
1336	 */
1337	if (!(bp->b_flags & B_READ) && !(bp->b_flags & B_ASYNC))
1338		syncflag = FSYNC;
1339
1340	/*
1341	 * We used to always use vn_rdwr here, but we cannot do that because
1342	 * we might decide to read or write from the the underlying
1343	 * file during this call, which would be a deadlock because
1344	 * we have the rw_lock. So instead we page, unless it's not
1345	 * mapable or it's a character device or it's an encrypted lofi.
1346	 */
1347	if ((lsp->ls_vp->v_flag & VNOMAP) || (lsp->ls_vp->v_type == VCHR) ||
1348	    lsp->ls_crypto_enabled) {
1349		error = lofi_rdwr(bufaddr, offset, bp, lsp, len, RDWR_RAW,
1350		    NULL);
1351	} else if (lsp->ls_uncomp_seg_sz == 0) {
1352		error = lofi_mapped_rdwr(bufaddr, offset, bp, lsp);
1353	} else {
1354		uchar_t *compressed_seg = NULL, *cmpbuf;
1355		uchar_t *uncompressed_seg = NULL;
1356		lofi_compress_info_t *li;
1357		size_t oblkcount;
1358		ulong_t seglen;
1359		uint64_t sblkno, eblkno, cmpbytes;
1360		uint64_t uncompressed_seg_index;
1361		struct lofi_comp_cache *lc;
1362		offset_t sblkoff, eblkoff;
1363		u_offset_t salign, ealign;
1364		u_offset_t sdiff;
1365		uint32_t comp_data_sz;
1366		uint64_t i;
1367		int j;
1368
1369		/*
1370		 * From here on we're dealing primarily with compressed files
1371		 */
1372		ASSERT(!lsp->ls_crypto_enabled);
1373
1374		/*
1375		 * Compressed files can only be read from and
1376		 * not written to
1377		 */
1378		if (!(bp->b_flags & B_READ)) {
1379			bp->b_resid = bp->b_bcount;
1380			error = EROFS;
1381			goto done;
1382		}
1383
1384		ASSERT(lsp->ls_comp_algorithm_index >= 0);
1385		li = &lofi_compress_table[lsp->ls_comp_algorithm_index];
1386		/*
1387		 * Compute starting and ending compressed segment numbers
1388		 * We use only bitwise operations avoiding division and
1389		 * modulus because we enforce the compression segment size
1390		 * to a power of 2
1391		 */
1392		sblkno = offset >> lsp->ls_comp_seg_shift;
1393		sblkoff = offset & (lsp->ls_uncomp_seg_sz - 1);
1394		eblkno = (offset + bp->b_bcount) >> lsp->ls_comp_seg_shift;
1395		eblkoff = (offset + bp->b_bcount) & (lsp->ls_uncomp_seg_sz - 1);
1396
1397		/*
1398		 * Check the decompressed segment cache.
1399		 *
1400		 * The cache is used only when the requested data
1401		 * is within a segment. Requests that cross
1402		 * segment boundaries bypass the cache.
1403		 */
1404		if (sblkno == eblkno ||
1405		    (sblkno + 1 == eblkno && eblkoff == 0)) {
1406			/*
1407			 * Request doesn't cross a segment boundary,
1408			 * now check the cache.
1409			 */
1410			mutex_enter(&lsp->ls_comp_cache_lock);
1411			lc = lofi_find_comp_data(lsp, sblkno);
1412			if (lc != NULL) {
1413				/*
1414				 * We've found the decompressed segment
1415				 * data in the cache; reuse it.
1416				 */
1417				bcopy(lc->lc_data + sblkoff, bufaddr,
1418				    bp->b_bcount);
1419				mutex_exit(&lsp->ls_comp_cache_lock);
1420				bp->b_resid = 0;
1421				error = 0;
1422				goto done;
1423			}
1424			mutex_exit(&lsp->ls_comp_cache_lock);
1425		}
1426
1427		/*
1428		 * Align start offset to block boundary for segmap
1429		 */
1430		salign = lsp->ls_comp_seg_index[sblkno];
1431		sdiff = salign & (DEV_BSIZE - 1);
1432		salign -= sdiff;
1433		if (eblkno >= (lsp->ls_comp_index_sz - 1)) {
1434			/*
1435			 * We're dealing with the last segment of
1436			 * the compressed file -- the size of this
1437			 * segment *may not* be the same as the
1438			 * segment size for the file
1439			 */
1440			eblkoff = (offset + bp->b_bcount) &
1441			    (lsp->ls_uncomp_last_seg_sz - 1);
1442			ealign = lsp->ls_vp_comp_size;
1443		} else {
1444			ealign = lsp->ls_comp_seg_index[eblkno + 1];
1445		}
1446
1447		/*
1448		 * Preserve original request paramaters
1449		 */
1450		oblkcount = bp->b_bcount;
1451
1452		/*
1453		 * Assign the calculated parameters
1454		 */
1455		comp_data_sz = ealign - salign;
1456		bp->b_bcount = comp_data_sz;
1457
1458		/*
1459		 * Buffers to hold compressed segments are pre-allocated
1460		 * on a per-thread basis. Find a pre-allocated buffer
1461		 * that is not currently in use and mark it for use.
1462		 */
1463		mutex_enter(&lsp->ls_comp_bufs_lock);
1464		for (j = 0; j < lofi_taskq_nthreads; j++) {
1465			if (lsp->ls_comp_bufs[j].inuse == 0) {
1466				lsp->ls_comp_bufs[j].inuse = 1;
1467				break;
1468			}
1469		}
1470
1471		mutex_exit(&lsp->ls_comp_bufs_lock);
1472		ASSERT(j < lofi_taskq_nthreads);
1473
1474		/*
1475		 * If the pre-allocated buffer size does not match
1476		 * the size of the I/O request, re-allocate it with
1477		 * the appropriate size
1478		 */
1479		if (lsp->ls_comp_bufs[j].bufsize < bp->b_bcount) {
1480			if (lsp->ls_comp_bufs[j].bufsize > 0)
1481				kmem_free(lsp->ls_comp_bufs[j].buf,
1482				    lsp->ls_comp_bufs[j].bufsize);
1483			lsp->ls_comp_bufs[j].buf = kmem_alloc(bp->b_bcount,
1484			    KM_SLEEP);
1485			lsp->ls_comp_bufs[j].bufsize = bp->b_bcount;
1486		}
1487		compressed_seg = lsp->ls_comp_bufs[j].buf;
1488
1489		/*
1490		 * Map in the calculated number of blocks
1491		 */
1492		error = lofi_mapped_rdwr((caddr_t)compressed_seg, salign,
1493		    bp, lsp);
1494
1495		bp->b_bcount = oblkcount;
1496		bp->b_resid = oblkcount;
1497		if (error != 0)
1498			goto done;
1499
1500		/*
1501		 * decompress compressed blocks start
1502		 */
1503		cmpbuf = compressed_seg + sdiff;
1504		for (i = sblkno; i <= eblkno; i++) {
1505			ASSERT(i < lsp->ls_comp_index_sz - 1);
1506			uchar_t *useg;
1507
1508			/*
1509			 * The last segment is special in that it is
1510			 * most likely not going to be the same
1511			 * (uncompressed) size as the other segments.
1512			 */
1513			if (i == (lsp->ls_comp_index_sz - 2)) {
1514				seglen = lsp->ls_uncomp_last_seg_sz;
1515			} else {
1516				seglen = lsp->ls_uncomp_seg_sz;
1517			}
1518
1519			/*
1520			 * Each of the segment index entries contains
1521			 * the starting block number for that segment.
1522			 * The number of compressed bytes in a segment
1523			 * is thus the difference between the starting
1524			 * block number of this segment and the starting
1525			 * block number of the next segment.
1526			 */
1527			cmpbytes = lsp->ls_comp_seg_index[i + 1] -
1528			    lsp->ls_comp_seg_index[i];
1529
1530			/*
1531			 * The first byte in a compressed segment is a flag
1532			 * that indicates whether this segment is compressed
1533			 * at all.
1534			 *
1535			 * The variable 'useg' is used (instead of
1536			 * uncompressed_seg) in this loop to keep a
1537			 * reference to the uncompressed segment.
1538			 *
1539			 * N.B. If 'useg' is replaced with uncompressed_seg,
1540			 * it leads to memory leaks and heap corruption in
1541			 * corner cases where compressed segments lie
1542			 * adjacent to uncompressed segments.
1543			 */
1544			if (*cmpbuf == UNCOMPRESSED) {
1545				useg = cmpbuf + SEGHDR;
1546			} else {
1547				if (uncompressed_seg == NULL)
1548					uncompressed_seg =
1549					    kmem_alloc(lsp->ls_uncomp_seg_sz,
1550					    KM_SLEEP);
1551				useg = uncompressed_seg;
1552				uncompressed_seg_index = i;
1553
1554				if (li->l_decompress((cmpbuf + SEGHDR),
1555				    (cmpbytes - SEGHDR), uncompressed_seg,
1556				    &seglen, li->l_level) != 0) {
1557					error = EIO;
1558					goto done;
1559				}
1560			}
1561
1562			/*
1563			 * Determine how much uncompressed data we
1564			 * have to copy and copy it
1565			 */
1566			xfersize = lsp->ls_uncomp_seg_sz - sblkoff;
1567			if (i == eblkno)
1568				xfersize -= (lsp->ls_uncomp_seg_sz - eblkoff);
1569
1570			bcopy((useg + sblkoff), bufaddr, xfersize);
1571
1572			cmpbuf += cmpbytes;
1573			bufaddr += xfersize;
1574			bp->b_resid -= xfersize;
1575			sblkoff = 0;
1576
1577			if (bp->b_resid == 0)
1578				break;
1579		} /* decompress compressed blocks ends */
1580
1581		/*
1582		 * Skip to done if there is no uncompressed data to cache
1583		 */
1584		if (uncompressed_seg == NULL)
1585			goto done;
1586
1587		/*
1588		 * Add the data for the last decompressed segment to
1589		 * the cache.
1590		 *
1591		 * In case the uncompressed segment data was added to (and
1592		 * is referenced by) the cache, make sure we don't free it
1593		 * here.
1594		 */
1595		mutex_enter(&lsp->ls_comp_cache_lock);
1596		if ((lc = lofi_add_comp_data(lsp, uncompressed_seg_index,
1597		    uncompressed_seg)) != NULL) {
1598			uncompressed_seg = NULL;
1599		}
1600		mutex_exit(&lsp->ls_comp_cache_lock);
1601
1602done:
1603		if (compressed_seg != NULL) {
1604			mutex_enter(&lsp->ls_comp_bufs_lock);
1605			lsp->ls_comp_bufs[j].inuse = 0;
1606			mutex_exit(&lsp->ls_comp_bufs_lock);
1607		}
1608		if (uncompressed_seg != NULL)
1609			kmem_free(uncompressed_seg, lsp->ls_uncomp_seg_sz);
1610	} /* end of handling compressed files */
1611
1612	if ((error == 0) && (syncflag != 0))
1613		error = VOP_FSYNC(lsp->ls_vp, syncflag, kcred, NULL);
1614
1615errout:
1616	if (bufinited && lsp->ls_kstat) {
1617		size_t n_done = bp->b_bcount - bp->b_resid;
1618		kstat_io_t *kioptr;
1619
1620		mutex_enter(lsp->ls_kstat->ks_lock);
1621		kioptr = KSTAT_IO_PTR(lsp->ls_kstat);
1622		if (bp->b_flags & B_READ) {
1623			kioptr->nread += n_done;
1624			kioptr->reads++;
1625		} else {
1626			kioptr->nwritten += n_done;
1627			kioptr->writes++;
1628		}
1629		kstat_runq_exit(kioptr);
1630		mutex_exit(lsp->ls_kstat->ks_lock);
1631	}
1632
1633	mutex_enter(&lsp->ls_vp_lock);
1634	if (--lsp->ls_vp_iocount == 0)
1635		cv_broadcast(&lsp->ls_vp_cv);
1636	mutex_exit(&lsp->ls_vp_lock);
1637
1638	bioerror(bp, error);
1639	biodone(bp);
1640}
1641
1642static int
1643lofi_strategy(struct buf *bp)
1644{
1645	struct lofi_state *lsp;
1646	offset_t	offset;
1647	minor_t		part;
1648	diskaddr_t	p_lba;
1649	diskaddr_t	p_nblks;
1650	int		shift;
1651
1652	/*
1653	 * We cannot just do I/O here, because the current thread
1654	 * _might_ end up back in here because the underlying filesystem
1655	 * wants a buffer, which eventually gets into bio_recycle and
1656	 * might call into lofi to write out a delayed-write buffer.
1657	 * This is bad if the filesystem above lofi is the same as below.
1658	 *
1659	 * We could come up with a complex strategy using threads to
1660	 * do the I/O asynchronously, or we could use task queues. task
1661	 * queues were incredibly easy so they win.
1662	 */
1663
1664	lsp = ddi_get_soft_state(lofi_statep,
1665	    LOFI_MINOR2ID(getminor(bp->b_edev)));
1666	part = LOFI_PART(getminor(bp->b_edev));
1667
1668	if (lsp == NULL) {
1669		bioerror(bp, ENXIO);
1670		biodone(bp);
1671		return (0);
1672	}
1673
1674	/* Check if we are closing. */
1675	mutex_enter(&lsp->ls_vp_lock);
1676	if (lsp->ls_vp == NULL || lsp->ls_vp_closereq) {
1677		mutex_exit(&lsp->ls_vp_lock);
1678		bioerror(bp, EIO);
1679		biodone(bp);
1680		return (0);
1681	}
1682	mutex_exit(&lsp->ls_vp_lock);
1683
1684	shift = lsp->ls_lbshift;
1685	p_lba = 0;
1686	p_nblks = lsp->ls_vp_size >> shift;
1687
1688	if (lsp->ls_cmlbhandle != NULL) {
1689		if (cmlb_partinfo(lsp->ls_cmlbhandle, part, &p_nblks, &p_lba,
1690		    NULL, NULL, 0)) {
1691			bioerror(bp, ENXIO);
1692			biodone(bp);
1693			return (0);
1694		}
1695	}
1696
1697	/* start block past partition end? */
1698	if (bp->b_lblkno > p_nblks) {
1699		bioerror(bp, ENXIO);
1700		biodone(bp);
1701		return (0);
1702	}
1703
1704	offset = (bp->b_lblkno+p_lba) << shift;	/* offset within file */
1705
1706	mutex_enter(&lsp->ls_vp_lock);
1707	if (lsp->ls_crypto_enabled) {
1708		/* encrypted data really begins after crypto header */
1709		offset += lsp->ls_crypto_offset;
1710	}
1711
1712	/* make sure we will not pass the file or partition size */
1713	if (offset == lsp->ls_vp_size ||
1714	    offset == (((p_lba + p_nblks) << shift) + lsp->ls_crypto_offset)) {
1715		/* EOF */
1716		if ((bp->b_flags & B_READ) != 0) {
1717			bp->b_resid = bp->b_bcount;
1718			bioerror(bp, 0);
1719		} else {
1720			/* writes should fail */
1721			bioerror(bp, ENXIO);
1722		}
1723		biodone(bp);
1724		mutex_exit(&lsp->ls_vp_lock);
1725		return (0);
1726	}
1727	if ((offset > lsp->ls_vp_size) ||
1728	    (offset > (((p_lba + p_nblks) << shift) + lsp->ls_crypto_offset)) ||
1729	    ((offset + bp->b_bcount) > ((p_lba + p_nblks) << shift))) {
1730		bioerror(bp, ENXIO);
1731		biodone(bp);
1732		mutex_exit(&lsp->ls_vp_lock);
1733		return (0);
1734	}
1735
1736	mutex_exit(&lsp->ls_vp_lock);
1737
1738	if (lsp->ls_kstat) {
1739		mutex_enter(lsp->ls_kstat->ks_lock);
1740		kstat_waitq_enter(KSTAT_IO_PTR(lsp->ls_kstat));
1741		mutex_exit(lsp->ls_kstat->ks_lock);
1742	}
1743	bp->b_private = (void *)(uintptr_t)p_lba;	/* partition start */
1744	(void) taskq_dispatch(lsp->ls_taskq, lofi_strategy_task, bp, KM_SLEEP);
1745	return (0);
1746}
1747
1748static int
1749lofi_read(dev_t dev, struct uio *uio, struct cred *credp)
1750{
1751	_NOTE(ARGUNUSED(credp));
1752
1753	if (getminor(dev) == 0)
1754		return (EINVAL);
1755	UIO_CHECK(uio);
1756	return (physio(lofi_strategy, NULL, dev, B_READ, minphys, uio));
1757}
1758
1759static int
1760lofi_write(dev_t dev, struct uio *uio, struct cred *credp)
1761{
1762	_NOTE(ARGUNUSED(credp));
1763
1764	if (getminor(dev) == 0)
1765		return (EINVAL);
1766	UIO_CHECK(uio);
1767	return (physio(lofi_strategy, NULL, dev, B_WRITE, minphys, uio));
1768}
1769
1770static int
1771lofi_urw(struct lofi_state *lsp, uint16_t fmode, diskaddr_t off, size_t size,
1772    intptr_t arg, int flag, cred_t *credp)
1773{
1774	struct uio uio;
1775	iovec_t iov;
1776
1777	/*
1778	 * 1024 * 1024 apes cmlb_tg_max_efi_xfer as a reasonable max.
1779	 */
1780	if (size == 0 || size > 1024 * 1024 ||
1781	    (size % (1 << lsp->ls_lbshift)) != 0)
1782		return (EINVAL);
1783
1784	iov.iov_base = (void *)arg;
1785	iov.iov_len = size;
1786	uio.uio_iov = &iov;
1787	uio.uio_iovcnt = 1;
1788	uio.uio_loffset = off;
1789	uio.uio_segflg = (flag & FKIOCTL) ? UIO_SYSSPACE : UIO_USERSPACE;
1790	uio.uio_llimit = MAXOFFSET_T;
1791	uio.uio_resid = size;
1792	uio.uio_fmode = fmode;
1793	uio.uio_extflg = 0;
1794
1795	return (fmode == FREAD ?
1796	    lofi_read(lsp->ls_dev, &uio, credp) :
1797	    lofi_write(lsp->ls_dev, &uio, credp));
1798}
1799
1800/*ARGSUSED2*/
1801static int
1802lofi_aread(dev_t dev, struct aio_req *aio, struct cred *credp)
1803{
1804	if (getminor(dev) == 0)
1805		return (EINVAL);
1806	UIO_CHECK(aio->aio_uio);
1807	return (aphysio(lofi_strategy, anocancel, dev, B_READ, minphys, aio));
1808}
1809
1810/*ARGSUSED2*/
1811static int
1812lofi_awrite(dev_t dev, struct aio_req *aio, struct cred *credp)
1813{
1814	if (getminor(dev) == 0)
1815		return (EINVAL);
1816	UIO_CHECK(aio->aio_uio);
1817	return (aphysio(lofi_strategy, anocancel, dev, B_WRITE, minphys, aio));
1818}
1819
1820/*ARGSUSED*/
1821static int
1822lofi_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
1823{
1824	struct lofi_state *lsp;
1825	dev_t	dev = (dev_t)arg;
1826	int instance;
1827
1828	instance = LOFI_MINOR2ID(getminor(dev));
1829	switch (infocmd) {
1830	case DDI_INFO_DEVT2DEVINFO:
1831		lsp = ddi_get_soft_state(lofi_statep, instance);
1832		if (lsp == NULL)
1833			return (DDI_FAILURE);
1834		*result = lsp->ls_dip;
1835		return (DDI_SUCCESS);
1836	case DDI_INFO_DEVT2INSTANCE:
1837		*result = (void *) (intptr_t)instance;
1838		return (DDI_SUCCESS);
1839	}
1840	return (DDI_FAILURE);
1841}
1842
1843static int
1844lofi_create_minor_nodes(struct lofi_state *lsp, boolean_t labeled)
1845{
1846	int error = 0;
1847	int instance = ddi_get_instance(lsp->ls_dip);
1848
1849	if (labeled == B_TRUE) {
1850		cmlb_alloc_handle(&lsp->ls_cmlbhandle);
1851		error = cmlb_attach(lsp->ls_dip, &lofi_tg_ops, DTYPE_DIRECT,
1852		    B_FALSE, B_FALSE, DDI_NT_BLOCK_CHAN,
1853		    CMLB_CREATE_P0_MINOR_NODE, lsp->ls_cmlbhandle, (void *)1);
1854
1855		if (error != DDI_SUCCESS) {
1856			cmlb_free_handle(&lsp->ls_cmlbhandle);
1857			lsp->ls_cmlbhandle = NULL;
1858			error = ENXIO;
1859		}
1860	} else {
1861		/* create minor nodes */
1862		error = ddi_create_minor_node(lsp->ls_dip, LOFI_BLOCK_NODE,
1863		    S_IFBLK, LOFI_ID2MINOR(instance), DDI_PSEUDO, 0);
1864		if (error == DDI_SUCCESS) {
1865			error = ddi_create_minor_node(lsp->ls_dip,
1866			    LOFI_CHAR_NODE, S_IFCHR, LOFI_ID2MINOR(instance),
1867			    DDI_PSEUDO, 0);
1868			if (error != DDI_SUCCESS) {
1869				ddi_remove_minor_node(lsp->ls_dip,
1870				    LOFI_BLOCK_NODE);
1871				error = ENXIO;
1872			}
1873		} else
1874			error = ENXIO;
1875	}
1876	return (error);
1877}
1878
1879static int
1880lofi_zone_bind(struct lofi_state *lsp)
1881{
1882	int error = 0;
1883
1884	mutex_enter(&curproc->p_lock);
1885	if ((error = rctl_incr_lofi(curproc, curproc->p_zone, 1)) != 0) {
1886		mutex_exit(&curproc->p_lock);
1887		return (error);
1888	}
1889	mutex_exit(&curproc->p_lock);
1890
1891	if (ddi_prop_update_string(DDI_DEV_T_NONE, lsp->ls_dip, ZONE_PROP_NAME,
1892	    (char *)curproc->p_zone->zone_name) != DDI_PROP_SUCCESS) {
1893		rctl_decr_lofi(curproc->p_zone, 1);
1894		error = EINVAL;
1895	} else {
1896		zone_init_ref(&lsp->ls_zone);
1897		zone_hold_ref(curzone, &lsp->ls_zone, ZONE_REF_LOFI);
1898	}
1899	return (error);
1900}
1901
1902static void
1903lofi_zone_unbind(struct lofi_state *lsp)
1904{
1905	(void) ddi_prop_remove(DDI_DEV_T_NONE, lsp->ls_dip, ZONE_PROP_NAME);
1906	rctl_decr_lofi(curproc->p_zone, 1);
1907	zone_rele_ref(&lsp->ls_zone, ZONE_REF_LOFI);
1908}
1909
1910static int
1911lofi_online_dev(dev_info_t *dip)
1912{
1913	boolean_t labeled;
1914	int	error;
1915	int	instance = ddi_get_instance(dip);
1916	struct lofi_state *lsp;
1917
1918	labeled = B_FALSE;
1919	if (ddi_prop_exists(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, "labeled"))
1920		labeled = B_TRUE;
1921
1922	/* lsp alloc+init, soft state is freed in lofi_detach */
1923	error = ddi_soft_state_zalloc(lofi_statep, instance);
1924	if (error == DDI_FAILURE) {
1925		return (ENOMEM);
1926	}
1927
1928	lsp = ddi_get_soft_state(lofi_statep, instance);
1929	lsp->ls_dip = dip;
1930
1931	if ((error = lofi_zone_bind(lsp)) != 0)
1932		goto err;
1933
1934	cv_init(&lsp->ls_vp_cv, NULL, CV_DRIVER, NULL);
1935	mutex_init(&lsp->ls_comp_cache_lock, NULL, MUTEX_DRIVER, NULL);
1936	mutex_init(&lsp->ls_comp_bufs_lock, NULL, MUTEX_DRIVER, NULL);
1937	mutex_init(&lsp->ls_kstat_lock, NULL, MUTEX_DRIVER, NULL);
1938	mutex_init(&lsp->ls_vp_lock, NULL, MUTEX_DRIVER, NULL);
1939
1940	if ((error = lofi_create_minor_nodes(lsp, labeled)) != 0) {
1941		lofi_zone_unbind(lsp);
1942		goto lerr;
1943	}
1944
1945	/* driver handles kernel-issued IOCTLs */
1946	if (ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
1947	    DDI_KERNEL_IOCTL, NULL, 0) != DDI_PROP_SUCCESS) {
1948		error = DDI_FAILURE;
1949		goto merr;
1950	}
1951
1952	lsp->ls_kstat = kstat_create_zone(LOFI_DRIVER_NAME, instance,
1953	    NULL, "disk", KSTAT_TYPE_IO, 1, 0, getzoneid());
1954	if (lsp->ls_kstat == NULL) {
1955		(void) ddi_prop_remove(DDI_DEV_T_NONE, lsp->ls_dip,
1956		    DDI_KERNEL_IOCTL);
1957		error = ENOMEM;
1958		goto merr;
1959	}
1960
1961	lsp->ls_kstat->ks_lock = &lsp->ls_kstat_lock;
1962	kstat_zone_add(lsp->ls_kstat, GLOBAL_ZONEID);
1963	kstat_install(lsp->ls_kstat);
1964	return (DDI_SUCCESS);
1965merr:
1966	if (lsp->ls_cmlbhandle != NULL) {
1967		cmlb_detach(lsp->ls_cmlbhandle, 0);
1968		cmlb_free_handle(&lsp->ls_cmlbhandle);
1969	}
1970	ddi_remove_minor_node(dip, NULL);
1971	lofi_zone_unbind(lsp);
1972lerr:
1973	mutex_destroy(&lsp->ls_comp_cache_lock);
1974	mutex_destroy(&lsp->ls_comp_bufs_lock);
1975	mutex_destroy(&lsp->ls_kstat_lock);
1976	mutex_destroy(&lsp->ls_vp_lock);
1977	cv_destroy(&lsp->ls_vp_cv);
1978err:
1979	ddi_soft_state_free(lofi_statep, instance);
1980	return (error);
1981}
1982
1983static int
1984lofi_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
1985{
1986	int	rv;
1987	int	instance = ddi_get_instance(dip);
1988	struct lofi_state *lsp;
1989
1990	if (cmd != DDI_ATTACH)
1991		return (DDI_FAILURE);
1992
1993	/*
1994	 * Instance 0 is control instance, attaching control instance
1995	 * will set the lofi up and ready.
1996	 */
1997	if (instance == 0) {
1998		rv = ddi_soft_state_zalloc(lofi_statep, 0);
1999		if (rv == DDI_FAILURE) {
2000			return (DDI_FAILURE);
2001		}
2002		lsp = ddi_get_soft_state(lofi_statep, instance);
2003		rv = ddi_create_minor_node(dip, LOFI_CTL_NODE, S_IFCHR, 0,
2004		    DDI_PSEUDO, 0);
2005		if (rv == DDI_FAILURE) {
2006			ddi_soft_state_free(lofi_statep, 0);
2007			return (DDI_FAILURE);
2008		}
2009		/* driver handles kernel-issued IOCTLs */
2010		if (ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
2011		    DDI_KERNEL_IOCTL, NULL, 0) != DDI_PROP_SUCCESS) {
2012			ddi_remove_minor_node(dip, NULL);
2013			ddi_soft_state_free(lofi_statep, 0);
2014			return (DDI_FAILURE);
2015		}
2016
2017		zone_key_create(&lofi_zone_key, NULL, lofi_zone_shutdown, NULL);
2018
2019		lsp->ls_dip = dip;
2020	} else {
2021		if (lofi_online_dev(dip) == DDI_FAILURE)
2022			return (DDI_FAILURE);
2023	}
2024
2025	ddi_report_dev(dip);
2026	return (DDI_SUCCESS);
2027}
2028
2029static int
2030lofi_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
2031{
2032	struct lofi_state *lsp;
2033	int instance = ddi_get_instance(dip);
2034
2035	if (cmd != DDI_DETACH)
2036		return (DDI_FAILURE);
2037
2038	/*
2039	 * If the instance is not 0, release state.
2040	 * The instance 0 is control device, we can not detach it
2041	 * before other instances are detached.
2042	 */
2043	if (instance != 0) {
2044		lsp = ddi_get_soft_state(lofi_statep, instance);
2045		if (lsp != NULL && lsp->ls_vp_ready == B_FALSE) {
2046			ddi_soft_state_free(lofi_statep, instance);
2047			return (DDI_SUCCESS);
2048		} else
2049			return (DDI_FAILURE);
2050	}
2051	mutex_enter(&lofi_lock);
2052
2053	if (!list_is_empty(&lofi_list)) {
2054		mutex_exit(&lofi_lock);
2055		return (DDI_FAILURE);
2056	}
2057
2058	ddi_remove_minor_node(dip, NULL);
2059	ddi_prop_remove_all(dip);
2060
2061	mutex_exit(&lofi_lock);
2062
2063	if (zone_key_delete(lofi_zone_key) != 0)
2064		cmn_err(CE_WARN, "failed to delete zone key");
2065
2066	ddi_soft_state_free(lofi_statep, 0);
2067
2068	return (DDI_SUCCESS);
2069}
2070
2071/*
2072 * With the addition of encryption, we must be careful that encryption key is
2073 * wiped before kernel's data structures are freed so it cannot accidentally
2074 * slip out to userland through uninitialized data elsewhere.
2075 */
2076static void
2077free_lofi_ioctl(struct lofi_ioctl *klip)
2078{
2079	/* Make sure this encryption key doesn't stick around */
2080	bzero(klip->li_key, sizeof (klip->li_key));
2081	kmem_free(klip, sizeof (struct lofi_ioctl));
2082}
2083
2084/*
2085 * These two functions simplify the rest of the ioctls that need to copyin/out
2086 * the lofi_ioctl structure.
2087 */
2088int
2089copy_in_lofi_ioctl(const struct lofi_ioctl *ulip, struct lofi_ioctl **klipp,
2090    int flag)
2091{
2092	struct lofi_ioctl *klip;
2093	int	error;
2094
2095	klip = *klipp = kmem_alloc(sizeof (struct lofi_ioctl), KM_SLEEP);
2096	error = ddi_copyin(ulip, klip, sizeof (struct lofi_ioctl), flag);
2097	if (error)
2098		goto err;
2099
2100	/* ensure NULL termination */
2101	klip->li_filename[MAXPATHLEN-1] = '\0';
2102	klip->li_devpath[MAXPATHLEN-1] = '\0';
2103	klip->li_algorithm[MAXALGLEN-1] = '\0';
2104	klip->li_cipher[CRYPTO_MAX_MECH_NAME-1] = '\0';
2105	klip->li_iv_cipher[CRYPTO_MAX_MECH_NAME-1] = '\0';
2106
2107	if (klip->li_id > L_MAXMIN32) {
2108		error = EINVAL;
2109		goto err;
2110	}
2111
2112	return (0);
2113
2114err:
2115	free_lofi_ioctl(klip);
2116	return (error);
2117}
2118
2119int
2120copy_out_lofi_ioctl(const struct lofi_ioctl *klip, struct lofi_ioctl *ulip,
2121    int flag)
2122{
2123	int	error;
2124
2125	/*
2126	 * NOTE: Do NOT copy the crypto_key_t "back" to userland.
2127	 * This ensures that an attacker can't trivially find the
2128	 * key for a mapping just by issuing the ioctl.
2129	 *
2130	 * It can still be found by poking around in kmem with mdb(1),
2131	 * but there is no point in making it easy when the info isn't
2132	 * of any use in this direction anyway.
2133	 *
2134	 * Either way we don't actually have the raw key stored in
2135	 * a form that we can get it anyway, since we just used it
2136	 * to create a ctx template and didn't keep "the original".
2137	 */
2138	error = ddi_copyout(klip, ulip, sizeof (struct lofi_ioctl), flag);
2139	if (error)
2140		return (EFAULT);
2141	return (0);
2142}
2143
2144static int
2145lofi_access(struct lofi_state *lsp)
2146{
2147	ASSERT(MUTEX_HELD(&lofi_lock));
2148	if (INGLOBALZONE(curproc) || lsp->ls_zone.zref_zone == curzone)
2149		return (0);
2150	return (EPERM);
2151}
2152
2153/*
2154 * Find the lofi state for the given filename. We compare by vnode to
2155 * allow the global zone visibility into NGZ lofi nodes.
2156 */
2157static int
2158file_to_lofi_nocheck(char *filename, boolean_t readonly,
2159    struct lofi_state **lspp)
2160{
2161	struct lofi_state *lsp;
2162	vnode_t *vp = NULL;
2163	int err = 0;
2164	int rdfiles = 0;
2165
2166	ASSERT(MUTEX_HELD(&lofi_lock));
2167
2168	if ((err = lookupname(filename, UIO_SYSSPACE, FOLLOW,
2169	    NULLVPP, &vp)) != 0)
2170		goto out;
2171
2172	if (vp->v_type == VREG) {
2173		vnode_t *realvp;
2174		if (VOP_REALVP(vp, &realvp, NULL) == 0) {
2175			VN_HOLD(realvp);
2176			VN_RELE(vp);
2177			vp = realvp;
2178		}
2179	}
2180
2181	for (lsp = list_head(&lofi_list); lsp != NULL;
2182	    lsp = list_next(&lofi_list, lsp)) {
2183		if (lsp->ls_vp == vp) {
2184			if (lspp != NULL)
2185				*lspp = lsp;
2186			if (lsp->ls_readonly) {
2187				rdfiles++;
2188				/* Skip if '-r' is specified */
2189				if (readonly)
2190					continue;
2191			}
2192			goto out;
2193		}
2194	}
2195
2196	err = ENOENT;
2197
2198	/*
2199	 * If a filename is given as an argument for lofi_unmap, we shouldn't
2200	 * allow unmap if there are multiple read-only lofi devices associated
2201	 * with this file.
2202	 */
2203	if (lspp != NULL) {
2204		if (rdfiles == 1)
2205			err = 0;
2206		else if (rdfiles > 1)
2207			err = EBUSY;
2208	}
2209
2210out:
2211	if (vp != NULL)
2212		VN_RELE(vp);
2213	return (err);
2214}
2215
2216/*
2217 * Find the minor for the given filename, checking the zone can access
2218 * it.
2219 */
2220static int
2221file_to_lofi(char *filename, boolean_t readonly, struct lofi_state **lspp)
2222{
2223	int err = 0;
2224
2225	ASSERT(MUTEX_HELD(&lofi_lock));
2226
2227	if ((err = file_to_lofi_nocheck(filename, readonly, lspp)) != 0)
2228		return (err);
2229
2230	if ((err = lofi_access(*lspp)) != 0)
2231		return (err);
2232
2233	return (0);
2234}
2235
2236/*
2237 * Fakes up a disk geometry based on the size of the file. This is needed
2238 * to support newfs on traditional lofi device, but also will provide
2239 * geometry hint for cmlb.
2240 */
2241static void
2242fake_disk_geometry(struct lofi_state *lsp)
2243{
2244	u_offset_t dsize = lsp->ls_vp_size - lsp->ls_crypto_offset;
2245
2246	/* dk_geom - see dkio(7I) */
2247	/*
2248	 * dkg_ncyl _could_ be set to one here (one big cylinder with gobs
2249	 * of sectors), but that breaks programs like fdisk which want to
2250	 * partition a disk by cylinder. With one cylinder, you can't create
2251	 * an fdisk partition and put pcfs on it for testing (hard to pick
2252	 * a number between one and one).
2253	 *
2254	 * The cheezy floppy test is an attempt to not have too few cylinders
2255	 * for a small file, or so many on a big file that you waste space
2256	 * for backup superblocks or cylinder group structures.
2257	 */
2258	bzero(&lsp->ls_dkg, sizeof (lsp->ls_dkg));
2259	if (dsize < (2 * 1024 * 1024)) /* floppy? */
2260		lsp->ls_dkg.dkg_ncyl = dsize / (100 * 1024);
2261	else
2262		lsp->ls_dkg.dkg_ncyl = dsize / (300 * 1024);
2263	/* in case file file is < 100k */
2264	if (lsp->ls_dkg.dkg_ncyl == 0)
2265		lsp->ls_dkg.dkg_ncyl = 1;
2266
2267	lsp->ls_dkg.dkg_pcyl = lsp->ls_dkg.dkg_ncyl;
2268	lsp->ls_dkg.dkg_nhead = 1;
2269	lsp->ls_dkg.dkg_rpm = 7200;
2270
2271	lsp->ls_dkg.dkg_nsect = dsize /
2272	    (lsp->ls_dkg.dkg_ncyl << lsp->ls_pbshift);
2273}
2274
2275/*
2276 * build vtoc - see dkio(7I)
2277 *
2278 * Fakes one big partition based on the size of the file. This is needed
2279 * because we allow newfs'ing the traditional lofi device and newfs will
2280 * do several disk ioctls to figure out the geometry and partition information.
2281 * It uses that information to determine the parameters to pass to mkfs.
2282 */
2283static void
2284fake_disk_vtoc(struct lofi_state *lsp, struct vtoc *vt)
2285{
2286	bzero(vt, sizeof (struct vtoc));
2287	vt->v_sanity = VTOC_SANE;
2288	vt->v_version = V_VERSION;
2289	(void) strncpy(vt->v_volume, LOFI_DRIVER_NAME,
2290	    sizeof (vt->v_volume));
2291	vt->v_sectorsz = 1 << lsp->ls_pbshift;
2292	vt->v_nparts = 1;
2293	vt->v_part[0].p_tag = V_UNASSIGNED;
2294
2295	/*
2296	 * A compressed file is read-only, other files can
2297	 * be read-write
2298	 */
2299	if (lsp->ls_uncomp_seg_sz > 0) {
2300		vt->v_part[0].p_flag = V_UNMNT | V_RONLY;
2301	} else {
2302		vt->v_part[0].p_flag = V_UNMNT;
2303	}
2304	vt->v_part[0].p_start = (daddr_t)0;
2305	/*
2306	 * The partition size cannot just be the number of sectors, because
2307	 * that might not end on a cylinder boundary. And if that's the case,
2308	 * newfs/mkfs will print a scary warning. So just figure the size
2309	 * based on the number of cylinders and sectors/cylinder.
2310	 */
2311	vt->v_part[0].p_size = lsp->ls_dkg.dkg_pcyl *
2312	    lsp->ls_dkg.dkg_nsect * lsp->ls_dkg.dkg_nhead;
2313}
2314
2315/*
2316 * build dk_cinfo - see dkio(7I)
2317 */
2318static void
2319fake_disk_info(dev_t dev, struct dk_cinfo *ci)
2320{
2321	bzero(ci, sizeof (struct dk_cinfo));
2322	(void) strlcpy(ci->dki_cname, LOFI_DRIVER_NAME, sizeof (ci->dki_cname));
2323	ci->dki_ctype = DKC_SCSI_CCS;
2324	(void) strlcpy(ci->dki_dname, LOFI_DRIVER_NAME, sizeof (ci->dki_dname));
2325	ci->dki_unit = LOFI_MINOR2ID(getminor(dev));
2326	ci->dki_partition = LOFI_PART(getminor(dev));
2327	/*
2328	 * newfs uses this to set maxcontig. Must not be < 16, or it
2329	 * will be 0 when newfs multiplies it by DEV_BSIZE and divides
2330	 * it by the block size. Then tunefs doesn't work because
2331	 * maxcontig is 0.
2332	 */
2333	ci->dki_maxtransfer = 16;
2334}
2335
2336/*
2337 * map in a compressed file
2338 *
2339 * Read in the header and the index that follows.
2340 *
2341 * The header is as follows -
2342 *
2343 * Signature (name of the compression algorithm)
2344 * Compression segment size (a multiple of 512)
2345 * Number of index entries
2346 * Size of the last block
2347 * The array containing the index entries
2348 *
2349 * The header information is always stored in
2350 * network byte order on disk.
2351 */
2352static int
2353lofi_map_compressed_file(struct lofi_state *lsp, char *buf)
2354{
2355	uint32_t index_sz, header_len, i;
2356	ssize_t	resid;
2357	enum uio_rw rw;
2358	char *tbuf = buf;
2359	int error;
2360
2361	/* The signature has already been read */
2362	tbuf += sizeof (lsp->ls_comp_algorithm);
2363	bcopy(tbuf, &(lsp->ls_uncomp_seg_sz), sizeof (lsp->ls_uncomp_seg_sz));
2364	lsp->ls_uncomp_seg_sz = ntohl(lsp->ls_uncomp_seg_sz);
2365
2366	/*
2367	 * The compressed segment size must be a power of 2
2368	 */
2369	if (lsp->ls_uncomp_seg_sz < DEV_BSIZE ||
2370	    !ISP2(lsp->ls_uncomp_seg_sz))
2371		return (EINVAL);
2372
2373	for (i = 0; !((lsp->ls_uncomp_seg_sz >> i) & 1); i++)
2374		;
2375
2376	lsp->ls_comp_seg_shift = i;
2377
2378	tbuf += sizeof (lsp->ls_uncomp_seg_sz);
2379	bcopy(tbuf, &(lsp->ls_comp_index_sz), sizeof (lsp->ls_comp_index_sz));
2380	lsp->ls_comp_index_sz = ntohl(lsp->ls_comp_index_sz);
2381
2382	tbuf += sizeof (lsp->ls_comp_index_sz);
2383	bcopy(tbuf, &(lsp->ls_uncomp_last_seg_sz),
2384	    sizeof (lsp->ls_uncomp_last_seg_sz));
2385	lsp->ls_uncomp_last_seg_sz = ntohl(lsp->ls_uncomp_last_seg_sz);
2386
2387	/*
2388	 * Compute the total size of the uncompressed data
2389	 * for use in fake_disk_geometry and other calculations.
2390	 * Disk geometry has to be faked with respect to the
2391	 * actual uncompressed data size rather than the
2392	 * compressed file size.
2393	 */
2394	lsp->ls_vp_size =
2395	    (u_offset_t)(lsp->ls_comp_index_sz - 2) * lsp->ls_uncomp_seg_sz
2396	    + lsp->ls_uncomp_last_seg_sz;
2397
2398	/*
2399	 * Index size is rounded up to DEV_BSIZE for ease
2400	 * of segmapping
2401	 */
2402	index_sz = sizeof (*lsp->ls_comp_seg_index) * lsp->ls_comp_index_sz;
2403	header_len = sizeof (lsp->ls_comp_algorithm) +
2404	    sizeof (lsp->ls_uncomp_seg_sz) +
2405	    sizeof (lsp->ls_comp_index_sz) +
2406	    sizeof (lsp->ls_uncomp_last_seg_sz);
2407	lsp->ls_comp_offbase = header_len + index_sz;
2408
2409	index_sz += header_len;
2410	index_sz = roundup(index_sz, DEV_BSIZE);
2411
2412	lsp->ls_comp_index_data = kmem_alloc(index_sz, KM_SLEEP);
2413	lsp->ls_comp_index_data_sz = index_sz;
2414
2415	/*
2416	 * Read in the index -- this has a side-effect
2417	 * of reading in the header as well
2418	 */
2419	rw = UIO_READ;
2420	error = vn_rdwr(rw, lsp->ls_vp, lsp->ls_comp_index_data, index_sz,
2421	    0, UIO_SYSSPACE, 0, RLIM64_INFINITY, kcred, &resid);
2422
2423	if (error != 0)
2424		return (error);
2425
2426	/* Skip the header, this is where the index really begins */
2427	lsp->ls_comp_seg_index =
2428	    /*LINTED*/
2429	    (uint64_t *)(lsp->ls_comp_index_data + header_len);
2430
2431	/*
2432	 * Now recompute offsets in the index to account for
2433	 * the header length
2434	 */
2435	for (i = 0; i < lsp->ls_comp_index_sz; i++) {
2436		lsp->ls_comp_seg_index[i] = lsp->ls_comp_offbase +
2437		    BE_64(lsp->ls_comp_seg_index[i]);
2438	}
2439
2440	return (error);
2441}
2442
2443static int
2444lofi_init_crypto(struct lofi_state *lsp, struct lofi_ioctl *klip)
2445{
2446	struct crypto_meta chead;
2447	char buf[DEV_BSIZE];
2448	ssize_t	resid;
2449	char *marker;
2450	int error;
2451	int ret;
2452	int i;
2453
2454	if (!klip->li_crypto_enabled)
2455		return (0);
2456
2457	/*
2458	 * All current algorithms have a max of 448 bits.
2459	 */
2460	if (klip->li_iv_len > CRYPTO_BITS2BYTES(512))
2461		return (EINVAL);
2462
2463	if (CRYPTO_BITS2BYTES(klip->li_key_len) > sizeof (klip->li_key))
2464		return (EINVAL);
2465
2466	lsp->ls_crypto_enabled = klip->li_crypto_enabled;
2467
2468	mutex_init(&lsp->ls_crypto_lock, NULL, MUTEX_DRIVER, NULL);
2469
2470	lsp->ls_mech.cm_type = crypto_mech2id(klip->li_cipher);
2471	if (lsp->ls_mech.cm_type == CRYPTO_MECH_INVALID) {
2472		cmn_err(CE_WARN, "invalid cipher %s requested for %s",
2473		    klip->li_cipher, klip->li_filename);
2474		return (EINVAL);
2475	}
2476
2477	/* this is just initialization here */
2478	lsp->ls_mech.cm_param = NULL;
2479	lsp->ls_mech.cm_param_len = 0;
2480
2481	lsp->ls_iv_type = klip->li_iv_type;
2482	lsp->ls_iv_mech.cm_type = crypto_mech2id(klip->li_iv_cipher);
2483	if (lsp->ls_iv_mech.cm_type == CRYPTO_MECH_INVALID) {
2484		cmn_err(CE_WARN, "invalid iv cipher %s requested"
2485		    " for %s", klip->li_iv_cipher, klip->li_filename);
2486		return (EINVAL);
2487	}
2488
2489	/* iv mech must itself take a null iv */
2490	lsp->ls_iv_mech.cm_param = NULL;
2491	lsp->ls_iv_mech.cm_param_len = 0;
2492	lsp->ls_iv_len = klip->li_iv_len;
2493
2494	/*
2495	 * Create ctx using li_cipher & the raw li_key after checking
2496	 * that it isn't a weak key.
2497	 */
2498	lsp->ls_key.ck_format = CRYPTO_KEY_RAW;
2499	lsp->ls_key.ck_length = klip->li_key_len;
2500	lsp->ls_key.ck_data = kmem_alloc(
2501	    CRYPTO_BITS2BYTES(lsp->ls_key.ck_length), KM_SLEEP);
2502	bcopy(klip->li_key, lsp->ls_key.ck_data,
2503	    CRYPTO_BITS2BYTES(lsp->ls_key.ck_length));
2504
2505	ret = crypto_key_check(&lsp->ls_mech, &lsp->ls_key);
2506	if (ret != CRYPTO_SUCCESS) {
2507		cmn_err(CE_WARN, "weak key check failed for cipher "
2508		    "%s on file %s (0x%x)", klip->li_cipher,
2509		    klip->li_filename, ret);
2510		return (EINVAL);
2511	}
2512
2513	error = vn_rdwr(UIO_READ, lsp->ls_vp, buf, DEV_BSIZE,
2514	    CRYOFF, UIO_SYSSPACE, 0, RLIM64_INFINITY, kcred, &resid);
2515	if (error != 0)
2516		return (error);
2517
2518	/*
2519	 * This is the case where the header in the lofi image is already
2520	 * initialized to indicate it is encrypted.
2521	 */
2522	if (strncmp(buf, lofi_crypto_magic, sizeof (lofi_crypto_magic)) == 0) {
2523		/*
2524		 * The encryption header information is laid out this way:
2525		 *	6 bytes:	hex "CFLOFI"
2526		 *	2 bytes:	version = 0 ... for now
2527		 *	96 bytes:	reserved1 (not implemented yet)
2528		 *	4 bytes:	data_sector = 2 ... for now
2529		 *	more...		not implemented yet
2530		 */
2531
2532		marker = buf;
2533
2534		/* copy the magic */
2535		bcopy(marker, lsp->ls_crypto.magic,
2536		    sizeof (lsp->ls_crypto.magic));
2537		marker += sizeof (lsp->ls_crypto.magic);
2538
2539		/* read the encryption version number */
2540		bcopy(marker, &(lsp->ls_crypto.version),
2541		    sizeof (lsp->ls_crypto.version));
2542		lsp->ls_crypto.version = ntohs(lsp->ls_crypto.version);
2543		marker += sizeof (lsp->ls_crypto.version);
2544
2545		/* read a chunk of reserved data */
2546		bcopy(marker, lsp->ls_crypto.reserved1,
2547		    sizeof (lsp->ls_crypto.reserved1));
2548		marker += sizeof (lsp->ls_crypto.reserved1);
2549
2550		/* read block number where encrypted data begins */
2551		bcopy(marker, &(lsp->ls_crypto.data_sector),
2552		    sizeof (lsp->ls_crypto.data_sector));
2553		lsp->ls_crypto.data_sector = ntohl(lsp->ls_crypto.data_sector);
2554		marker += sizeof (lsp->ls_crypto.data_sector);
2555
2556		/* and ignore the rest until it is implemented */
2557
2558		lsp->ls_crypto_offset = lsp->ls_crypto.data_sector * DEV_BSIZE;
2559		return (0);
2560	}
2561
2562	/*
2563	 * We've requested encryption, but no magic was found, so it must be
2564	 * a new image.
2565	 */
2566
2567	for (i = 0; i < sizeof (struct crypto_meta); i++) {
2568		if (buf[i] != '\0')
2569			return (EINVAL);
2570	}
2571
2572	marker = buf;
2573	bcopy(lofi_crypto_magic, marker, sizeof (lofi_crypto_magic));
2574	marker += sizeof (lofi_crypto_magic);
2575	chead.version = htons(LOFI_CRYPTO_VERSION);
2576	bcopy(&(chead.version), marker, sizeof (chead.version));
2577	marker += sizeof (chead.version);
2578	marker += sizeof (chead.reserved1);
2579	chead.data_sector = htonl(LOFI_CRYPTO_DATA_SECTOR);
2580	bcopy(&(chead.data_sector), marker, sizeof (chead.data_sector));
2581
2582	/* write the header */
2583	error = vn_rdwr(UIO_WRITE, lsp->ls_vp, buf, DEV_BSIZE,
2584	    CRYOFF, UIO_SYSSPACE, 0, RLIM64_INFINITY, kcred, &resid);
2585	if (error != 0)
2586		return (error);
2587
2588	/* fix things up so it looks like we read this info */
2589	bcopy(lofi_crypto_magic, lsp->ls_crypto.magic,
2590	    sizeof (lofi_crypto_magic));
2591	lsp->ls_crypto.version = LOFI_CRYPTO_VERSION;
2592	lsp->ls_crypto.data_sector = LOFI_CRYPTO_DATA_SECTOR;
2593	lsp->ls_crypto_offset = lsp->ls_crypto.data_sector * DEV_BSIZE;
2594	return (0);
2595}
2596
2597/*
2598 * Check to see if the passed in signature is a valid one.  If it is
2599 * valid, return the index into lofi_compress_table.
2600 *
2601 * Return -1 if it is invalid
2602 */
2603static int
2604lofi_compress_select(const char *signature)
2605{
2606	int i;
2607
2608	for (i = 0; i < LOFI_COMPRESS_FUNCTIONS; i++) {
2609		if (strcmp(lofi_compress_table[i].l_name, signature) == 0)
2610			return (i);
2611	}
2612
2613	return (-1);
2614}
2615
2616static int
2617lofi_init_compress(struct lofi_state *lsp)
2618{
2619	char buf[DEV_BSIZE];
2620	int compress_index;
2621	ssize_t	resid;
2622	int error;
2623
2624	error = vn_rdwr(UIO_READ, lsp->ls_vp, buf, DEV_BSIZE, 0, UIO_SYSSPACE,
2625	    0, RLIM64_INFINITY, kcred, &resid);
2626
2627	if (error != 0)
2628		return (error);
2629
2630	if ((compress_index = lofi_compress_select(buf)) == -1)
2631		return (0);
2632
2633	/* compression and encryption are mutually exclusive */
2634	if (lsp->ls_crypto_enabled)
2635		return (ENOTSUP);
2636
2637	/* initialize compression info for compressed lofi */
2638	lsp->ls_comp_algorithm_index = compress_index;
2639	(void) strlcpy(lsp->ls_comp_algorithm,
2640	    lofi_compress_table[compress_index].l_name,
2641	    sizeof (lsp->ls_comp_algorithm));
2642
2643	/* Finally setup per-thread pre-allocated buffers */
2644	lsp->ls_comp_bufs = kmem_zalloc(lofi_taskq_nthreads *
2645	    sizeof (struct compbuf), KM_SLEEP);
2646
2647	return (lofi_map_compressed_file(lsp, buf));
2648}
2649
2650/*
2651 * Allocate new or proposed id from lofi_id.
2652 *
2653 * Special cases for proposed id:
2654 * 0: not allowed, 0 is id for control device.
2655 * -1: allocate first usable id from lofi_id.
2656 * any other value is proposed value from userland
2657 *
2658 * returns DDI_SUCCESS or errno.
2659 */
2660static int
2661lofi_alloc_id(int *idp)
2662{
2663	int id, error = DDI_SUCCESS;
2664
2665	if (*idp == -1) {
2666		id = id_allocff_nosleep(lofi_id);
2667		if (id == -1) {
2668			error = EAGAIN;
2669			goto err;
2670		}
2671	} else if (*idp == 0) {
2672		error = EINVAL;
2673		goto err;
2674	} else if (*idp > ((1 << (L_BITSMINOR - LOFI_CMLB_SHIFT)) - 1)) {
2675		error = ERANGE;
2676		goto err;
2677	} else {
2678		if (ddi_get_soft_state(lofi_statep, *idp) != NULL) {
2679			error = EEXIST;
2680			goto err;
2681		}
2682
2683		id = id_alloc_specific_nosleep(lofi_id, *idp);
2684		if (id == -1) {
2685			error = EAGAIN;
2686			goto err;
2687		}
2688	}
2689	*idp = id;
2690err:
2691	return (error);
2692}
2693
2694static int
2695lofi_create_dev(struct lofi_ioctl *klip)
2696{
2697	dev_info_t *parent, *child;
2698	struct lofi_state *lsp = NULL;
2699	char namebuf[MAXNAMELEN];
2700	int error, circ;
2701
2702	/* get control device */
2703	lsp = ddi_get_soft_state(lofi_statep, 0);
2704	parent = ddi_get_parent(lsp->ls_dip);
2705
2706	if ((error = lofi_alloc_id((int *)&klip->li_id)))
2707		return (error);
2708
2709	(void) snprintf(namebuf, sizeof (namebuf), LOFI_DRIVER_NAME "@%d",
2710	    klip->li_id);
2711
2712	ndi_devi_enter(parent, &circ);
2713	child = ndi_devi_findchild(parent, namebuf);
2714	ndi_devi_exit(parent, circ);
2715
2716	if (child == NULL) {
2717		child = ddi_add_child(parent, LOFI_DRIVER_NAME,
2718		    (pnode_t)DEVI_SID_NODEID, klip->li_id);
2719		if ((error = ddi_prop_update_int(DDI_DEV_T_NONE, child,
2720		    "instance", klip->li_id)) != DDI_PROP_SUCCESS)
2721			goto err;
2722
2723		if (klip->li_labeled == B_TRUE) {
2724			if ((error = ddi_prop_create(DDI_DEV_T_NONE, child,
2725			    DDI_PROP_CANSLEEP, "labeled", 0, 0))
2726			    != DDI_PROP_SUCCESS)
2727				goto err;
2728		}
2729
2730		if ((error = ndi_devi_online(child, NDI_ONLINE_ATTACH))
2731		    != NDI_SUCCESS)
2732			goto err;
2733	} else {
2734		id_free(lofi_id, klip->li_id);
2735		error = EEXIST;
2736		return (error);
2737	}
2738
2739	goto done;
2740
2741err:
2742	ddi_prop_remove_all(child);
2743	(void) ndi_devi_offline(child, NDI_DEVI_REMOVE);
2744	id_free(lofi_id, klip->li_id);
2745done:
2746
2747	return (error);
2748}
2749
2750static void
2751lofi_create_inquiry(struct lofi_state *lsp, struct scsi_inquiry *inq)
2752{
2753	char *p = NULL;
2754
2755	(void) strlcpy(inq->inq_vid, LOFI_DRIVER_NAME, sizeof (inq->inq_vid));
2756
2757	mutex_enter(&lsp->ls_vp_lock);
2758	if (lsp->ls_vp != NULL)
2759		p = strrchr(lsp->ls_vp->v_path, '/');
2760	if (p != NULL)
2761		(void) strncpy(inq->inq_pid, p + 1, sizeof (inq->inq_pid));
2762	mutex_exit(&lsp->ls_vp_lock);
2763	(void) strlcpy(inq->inq_revision, "1.0", sizeof (inq->inq_revision));
2764}
2765
2766/*
2767 * copy devlink name from event cache
2768 */
2769static void
2770lofi_copy_devpath(struct lofi_ioctl *klip)
2771{
2772	int	error;
2773	char	namebuf[MAXNAMELEN], *str;
2774	clock_t ticks;
2775	nvlist_t *nvl = NULL;
2776
2777	if (klip->li_labeled == B_TRUE)
2778		klip->li_devpath[0] = '\0';
2779	else {
2780		/* no need to wait for messages */
2781		(void) snprintf(klip->li_devpath, sizeof (klip->li_devpath),
2782		    "/dev/" LOFI_CHAR_NAME "/%d", klip->li_id);
2783		return;
2784	}
2785
2786	(void) snprintf(namebuf, sizeof (namebuf), "%d", klip->li_id);
2787
2788	mutex_enter(&lofi_devlink_cache.ln_lock);
2789	for (;;) {
2790		error = nvlist_lookup_nvlist(lofi_devlink_cache.ln_data,
2791		    namebuf, &nvl);
2792
2793		if (error == 0 &&
2794		    nvlist_lookup_string(nvl, DEV_NAME, &str) == 0 &&
2795		    strncmp(str, "/dev/" LOFI_CHAR_NAME,
2796		    sizeof ("/dev/" LOFI_CHAR_NAME) - 1) != 0) {
2797			(void) strlcpy(klip->li_devpath, str,
2798			    sizeof (klip->li_devpath));
2799			break;
2800		}
2801		/*
2802		 * Either there is no data in the cache, or the
2803		 * cache entry still has the wrong device name.
2804		 */
2805		ticks = ddi_get_lbolt() + lofi_timeout * drv_usectohz(1000000);
2806		error = cv_timedwait(&lofi_devlink_cache.ln_cv,
2807		    &lofi_devlink_cache.ln_lock, ticks);
2808		if (error == -1)
2809			break;	/* timeout */
2810	}
2811	mutex_exit(&lofi_devlink_cache.ln_lock);
2812}
2813
2814/*
2815 * map a file to a minor number. Return the minor number.
2816 */
2817static int
2818lofi_map_file(dev_t dev, struct lofi_ioctl *ulip, int pickminor,
2819    int *rvalp, struct cred *credp, int ioctl_flag)
2820{
2821	int	id = -1;
2822	struct lofi_state *lsp = NULL;
2823	struct lofi_ioctl *klip;
2824	int	error;
2825	struct vnode *vp = NULL;
2826	vattr_t	vattr;
2827	int	flag;
2828	char	namebuf[MAXNAMELEN];
2829
2830	error = copy_in_lofi_ioctl(ulip, &klip, ioctl_flag);
2831	if (error != 0)
2832		return (error);
2833
2834	mutex_enter(&lofi_lock);
2835
2836	if (file_to_lofi_nocheck(klip->li_filename, klip->li_readonly,
2837	    NULL) == 0) {
2838		error = EBUSY;
2839		goto err;
2840	}
2841
2842	flag = FREAD | FWRITE | FOFFMAX | FEXCL;
2843	error = vn_open(klip->li_filename, UIO_SYSSPACE, flag, 0, &vp, 0, 0);
2844	if (error) {
2845		/* try read-only */
2846		flag &= ~FWRITE;
2847		error = vn_open(klip->li_filename, UIO_SYSSPACE, flag, 0,
2848		    &vp, 0, 0);
2849		if (error)
2850			goto err;
2851	}
2852
2853	if (!V_ISLOFIABLE(vp->v_type)) {
2854		error = EINVAL;
2855		goto err;
2856	}
2857
2858	vattr.va_mask = AT_SIZE;
2859	error = VOP_GETATTR(vp, &vattr, 0, credp, NULL);
2860	if (error)
2861		goto err;
2862
2863	/* the file needs to be a multiple of the block size */
2864	if ((vattr.va_size % DEV_BSIZE) != 0) {
2865		error = EINVAL;
2866		goto err;
2867	}
2868
2869	if (pickminor) {
2870		klip->li_id = (uint32_t)-1;
2871	}
2872	if ((error = lofi_create_dev(klip)) != 0)
2873		goto err;
2874
2875	id = klip->li_id;
2876	lsp = ddi_get_soft_state(lofi_statep, id);
2877	if (lsp == NULL)
2878		goto err;
2879
2880	/*
2881	 * from this point lofi_destroy() is used to clean up on error
2882	 * make sure the basic data is set
2883	 */
2884	list_insert_tail(&lofi_list, lsp);
2885	lsp->ls_dev = makedevice(getmajor(dev), LOFI_ID2MINOR(id));
2886
2887	list_create(&lsp->ls_comp_cache, sizeof (struct lofi_comp_cache),
2888	    offsetof(struct lofi_comp_cache, lc_list));
2889
2890	/*
2891	 * save open mode so file can be closed properly and vnode counts
2892	 * updated correctly.
2893	 */
2894	lsp->ls_openflag = flag;
2895
2896	lsp->ls_vp = vp;
2897	lsp->ls_stacked_vp = vp;
2898
2899	lsp->ls_vp_size = vattr.va_size;
2900	lsp->ls_vp_comp_size = lsp->ls_vp_size;
2901
2902	/*
2903	 * Try to handle stacked lofs vnodes.
2904	 */
2905	if (vp->v_type == VREG) {
2906		vnode_t *realvp;
2907
2908		if (VOP_REALVP(vp, &realvp, NULL) == 0) {
2909			/*
2910			 * We need to use the realvp for uniqueness
2911			 * checking, but keep the stacked vp for
2912			 * LOFI_GET_FILENAME display.
2913			 */
2914			VN_HOLD(realvp);
2915			lsp->ls_vp = realvp;
2916		}
2917	}
2918
2919	lsp->ls_lbshift = highbit(DEV_BSIZE) - 1;
2920	lsp->ls_pbshift = lsp->ls_lbshift;
2921
2922	lsp->ls_readonly = klip->li_readonly;
2923	lsp->ls_uncomp_seg_sz = 0;
2924	lsp->ls_comp_algorithm[0] = '\0';
2925	lsp->ls_crypto_offset = 0;
2926
2927	(void) snprintf(namebuf, sizeof (namebuf), "%s_taskq_%d",
2928	    LOFI_DRIVER_NAME, id);
2929	lsp->ls_taskq = taskq_create_proc(namebuf, lofi_taskq_nthreads,
2930	    minclsyspri, 1, lofi_taskq_maxalloc, curzone->zone_zsched, 0);
2931
2932	if ((error = lofi_init_crypto(lsp, klip)) != 0)
2933		goto err;
2934
2935	if ((error = lofi_init_compress(lsp)) != 0)
2936		goto err;
2937
2938	fake_disk_geometry(lsp);
2939
2940	/* For unlabeled lofi add Nblocks and Size */
2941	if (klip->li_labeled == B_FALSE) {
2942		error = ddi_prop_update_int64(lsp->ls_dev, lsp->ls_dip,
2943		    SIZE_PROP_NAME, lsp->ls_vp_size - lsp->ls_crypto_offset);
2944		if (error != DDI_PROP_SUCCESS) {
2945			error = EINVAL;
2946			goto err;
2947		}
2948		error = ddi_prop_update_int64(lsp->ls_dev, lsp->ls_dip,
2949		    NBLOCKS_PROP_NAME,
2950		    (lsp->ls_vp_size - lsp->ls_crypto_offset) / DEV_BSIZE);
2951		if (error != DDI_PROP_SUCCESS) {
2952			error = EINVAL;
2953			goto err;
2954		}
2955	}
2956
2957	/*
2958	 * Notify we are ready to rock.
2959	 */
2960	mutex_enter(&lsp->ls_vp_lock);
2961	lsp->ls_vp_ready = B_TRUE;
2962	cv_broadcast(&lsp->ls_vp_cv);
2963	mutex_exit(&lsp->ls_vp_lock);
2964	mutex_exit(&lofi_lock);
2965
2966	lofi_copy_devpath(klip);
2967
2968	if (rvalp)
2969		*rvalp = id;
2970	(void) copy_out_lofi_ioctl(klip, ulip, ioctl_flag);
2971	free_lofi_ioctl(klip);
2972	return (0);
2973
2974err:
2975	if (lsp != NULL) {
2976		lofi_destroy(lsp, credp);
2977	} else {
2978		if (vp != NULL) {
2979			(void) VOP_PUTPAGE(vp, 0, 0, B_FREE, credp, NULL);
2980			(void) VOP_CLOSE(vp, flag, 1, 0, credp, NULL);
2981			VN_RELE(vp);
2982		}
2983	}
2984
2985	mutex_exit(&lofi_lock);
2986	free_lofi_ioctl(klip);
2987	return (error);
2988}
2989
2990/*
2991 * unmap a file.
2992 */
2993static int
2994lofi_unmap_file(struct lofi_ioctl *ulip, int byfilename,
2995    struct cred *credp, int ioctl_flag)
2996{
2997	struct lofi_state *lsp;
2998	struct lofi_ioctl *klip;
2999	char namebuf[MAXNAMELEN];
3000	int err;
3001
3002	err = copy_in_lofi_ioctl(ulip, &klip, ioctl_flag);
3003	if (err != 0)
3004		return (err);
3005
3006	mutex_enter(&lofi_lock);
3007	if (byfilename) {
3008		if ((err = file_to_lofi(klip->li_filename, klip->li_readonly,
3009		    &lsp)) != 0) {
3010			goto done;
3011		}
3012	} else if (klip->li_id == 0) {
3013		err = ENXIO;
3014		goto done;
3015	} else {
3016		lsp = ddi_get_soft_state(lofi_statep, klip->li_id);
3017	}
3018
3019	if (lsp == NULL || lsp->ls_vp == NULL || lofi_access(lsp) != 0) {
3020		err = ENXIO;
3021		goto done;
3022	}
3023
3024	klip->li_id = LOFI_MINOR2ID(getminor(lsp->ls_dev));
3025	(void) snprintf(namebuf, sizeof (namebuf), "%u", klip->li_id);
3026
3027	/*
3028	 * If it's still held open, we'll do one of three things:
3029	 *
3030	 * If no flag is set, just return EBUSY.
3031	 *
3032	 * If the 'cleanup' flag is set, unmap and remove the device when
3033	 * the last user finishes.
3034	 *
3035	 * If the 'force' flag is set, then we forcibly close the underlying
3036	 * file.  Subsequent operations will fail, and the DKIOCSTATE ioctl
3037	 * will return DKIO_DEV_GONE.  When the device is last closed, the
3038	 * device will be cleaned up appropriately.
3039	 *
3040	 * This is complicated by the fact that we may have outstanding
3041	 * dispatched I/Os.  Rather than having a single mutex to serialize all
3042	 * I/O, we keep a count of the number of outstanding I/O requests
3043	 * (ls_vp_iocount), as well as a flag to indicate that no new I/Os
3044	 * should be dispatched (ls_vp_closereq).
3045	 *
3046	 * We set the flag, wait for the number of outstanding I/Os to reach 0,
3047	 * and then close the underlying vnode.
3048	 */
3049	if (is_opened(lsp)) {
3050		if (klip->li_force) {
3051			/* Mark the device for cleanup. */
3052			lofi_set_cleanup(lsp);
3053			mutex_enter(&lsp->ls_vp_lock);
3054			lsp->ls_vp_closereq = B_TRUE;
3055			/* Wake up any threads waiting on dkiocstate. */
3056			cv_broadcast(&lsp->ls_vp_cv);
3057			while (lsp->ls_vp_iocount > 0)
3058				cv_wait(&lsp->ls_vp_cv, &lsp->ls_vp_lock);
3059			mutex_exit(&lsp->ls_vp_lock);
3060		} else if (klip->li_cleanup) {
3061			lofi_set_cleanup(lsp);
3062		} else {
3063			err = EBUSY;
3064		}
3065	} else {
3066		lofi_free_dev(lsp);
3067		lofi_destroy(lsp, credp);
3068	}
3069
3070	/* Remove name from devlink cache */
3071	mutex_enter(&lofi_devlink_cache.ln_lock);
3072	(void) nvlist_remove_all(lofi_devlink_cache.ln_data, namebuf);
3073	cv_broadcast(&lofi_devlink_cache.ln_cv);
3074	mutex_exit(&lofi_devlink_cache.ln_lock);
3075done:
3076	mutex_exit(&lofi_lock);
3077	if (err == 0)
3078		(void) copy_out_lofi_ioctl(klip, ulip, ioctl_flag);
3079	free_lofi_ioctl(klip);
3080	return (err);
3081}
3082
3083/*
3084 * get the filename given the minor number, or the minor number given
3085 * the name.
3086 */
3087/*ARGSUSED*/
3088static int
3089lofi_get_info(dev_t dev, struct lofi_ioctl *ulip, int which,
3090    struct cred *credp, int ioctl_flag)
3091{
3092	struct lofi_ioctl *klip;
3093	struct lofi_state *lsp;
3094	int	error;
3095
3096	error = copy_in_lofi_ioctl(ulip, &klip, ioctl_flag);
3097	if (error != 0)
3098		return (error);
3099
3100	switch (which) {
3101	case LOFI_GET_FILENAME:
3102		if (klip->li_id == 0) {
3103			free_lofi_ioctl(klip);
3104			return (EINVAL);
3105		}
3106
3107		mutex_enter(&lofi_lock);
3108		lsp = ddi_get_soft_state(lofi_statep, klip->li_id);
3109		if (lsp == NULL || lofi_access(lsp) != 0) {
3110			mutex_exit(&lofi_lock);
3111			free_lofi_ioctl(klip);
3112			return (ENXIO);
3113		}
3114
3115		/*
3116		 * This may fail if, for example, we're trying to look
3117		 * up a zoned NFS path from the global zone.
3118		 */
3119		if (vnodetopath(NULL, lsp->ls_stacked_vp, klip->li_filename,
3120		    sizeof (klip->li_filename), CRED()) != 0) {
3121			(void) strlcpy(klip->li_filename, "?",
3122			    sizeof (klip->li_filename));
3123		}
3124
3125		klip->li_readonly = lsp->ls_readonly;
3126		klip->li_labeled = lsp->ls_cmlbhandle != NULL;
3127
3128		(void) strlcpy(klip->li_algorithm, lsp->ls_comp_algorithm,
3129		    sizeof (klip->li_algorithm));
3130		klip->li_crypto_enabled = lsp->ls_crypto_enabled;
3131		mutex_exit(&lofi_lock);
3132
3133		lofi_copy_devpath(klip);
3134		error = copy_out_lofi_ioctl(klip, ulip, ioctl_flag);
3135		free_lofi_ioctl(klip);
3136		return (error);
3137	case LOFI_GET_MINOR:
3138		mutex_enter(&lofi_lock);
3139		error = file_to_lofi(klip->li_filename,
3140		    klip->li_readonly, &lsp);
3141		if (error != 0) {
3142			mutex_exit(&lofi_lock);
3143			free_lofi_ioctl(klip);
3144			return (error);
3145		}
3146		klip->li_id = LOFI_MINOR2ID(getminor(lsp->ls_dev));
3147
3148		klip->li_readonly = lsp->ls_readonly;
3149		klip->li_labeled = lsp->ls_cmlbhandle != NULL;
3150		mutex_exit(&lofi_lock);
3151
3152		lofi_copy_devpath(klip);
3153		error = copy_out_lofi_ioctl(klip, ulip, ioctl_flag);
3154
3155		free_lofi_ioctl(klip);
3156		return (error);
3157	case LOFI_CHECK_COMPRESSED:
3158		mutex_enter(&lofi_lock);
3159		error = file_to_lofi(klip->li_filename,
3160		    klip->li_readonly, &lsp);
3161		if (error != 0) {
3162			mutex_exit(&lofi_lock);
3163			free_lofi_ioctl(klip);
3164			return (error);
3165		}
3166
3167		klip->li_id = LOFI_MINOR2ID(getminor(lsp->ls_dev));
3168		(void) strlcpy(klip->li_algorithm, lsp->ls_comp_algorithm,
3169		    sizeof (klip->li_algorithm));
3170
3171		mutex_exit(&lofi_lock);
3172		error = copy_out_lofi_ioctl(klip, ulip, ioctl_flag);
3173		free_lofi_ioctl(klip);
3174		return (error);
3175	default:
3176		free_lofi_ioctl(klip);
3177		return (EINVAL);
3178	}
3179}
3180
3181static int
3182uscsi_is_inquiry(intptr_t arg, int flag, union scsi_cdb *cdb,
3183    struct uscsi_cmd *uscmd)
3184{
3185	int rval;
3186
3187#ifdef	_MULTI_DATAMODEL
3188	switch (ddi_model_convert_from(flag & FMODELS)) {
3189	case DDI_MODEL_ILP32: {
3190		struct uscsi_cmd32 ucmd32;
3191
3192		if (ddi_copyin((void *)arg, &ucmd32, sizeof (ucmd32), flag)) {
3193			rval = EFAULT;
3194			goto err;
3195		}
3196		uscsi_cmd32touscsi_cmd((&ucmd32), uscmd);
3197		break;
3198	}
3199	case DDI_MODEL_NONE:
3200		if (ddi_copyin((void *)arg, uscmd, sizeof (*uscmd), flag)) {
3201			rval = EFAULT;
3202			goto err;
3203		}
3204		break;
3205	default:
3206		rval = EFAULT;
3207		goto err;
3208	}
3209#else
3210	if (ddi_copyin((void *)arg, uscmd, sizeof (*uscmd), flag)) {
3211		rval = EFAULT;
3212		goto err;
3213	}
3214#endif	/* _MULTI_DATAMODEL */
3215	if (ddi_copyin(uscmd->uscsi_cdb, cdb, uscmd->uscsi_cdblen, flag)) {
3216		rval = EFAULT;
3217		goto err;
3218	}
3219	if (cdb->scc_cmd == SCMD_INQUIRY) {
3220		return (0);
3221	}
3222err:
3223	return (rval);
3224}
3225
3226static int
3227lofi_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *credp,
3228    int *rvalp)
3229{
3230	int error;
3231	enum dkio_state dkstate;
3232	struct lofi_state *lsp;
3233	dk_efi_t user_efi;
3234	int id;
3235
3236	id = LOFI_MINOR2ID(getminor(dev));
3237
3238	/* lofi ioctls only apply to the master device */
3239	if (id == 0) {
3240		struct lofi_ioctl *lip = (struct lofi_ioctl *)arg;
3241
3242		/*
3243		 * the query command only need read-access - i.e., normal
3244		 * users are allowed to do those on the ctl device as
3245		 * long as they can open it read-only.
3246		 */
3247		switch (cmd) {
3248		case LOFI_MAP_FILE:
3249			if ((flag & FWRITE) == 0)
3250				return (EPERM);
3251			return (lofi_map_file(dev, lip, 1, rvalp, credp, flag));
3252		case LOFI_MAP_FILE_MINOR:
3253			if ((flag & FWRITE) == 0)
3254				return (EPERM);
3255			return (lofi_map_file(dev, lip, 0, rvalp, credp, flag));
3256		case LOFI_UNMAP_FILE:
3257			if ((flag & FWRITE) == 0)
3258				return (EPERM);
3259			return (lofi_unmap_file(lip, 1, credp, flag));
3260		case LOFI_UNMAP_FILE_MINOR:
3261			if ((flag & FWRITE) == 0)
3262				return (EPERM);
3263			return (lofi_unmap_file(lip, 0, credp, flag));
3264		case LOFI_GET_FILENAME:
3265			return (lofi_get_info(dev, lip, LOFI_GET_FILENAME,
3266			    credp, flag));
3267		case LOFI_GET_MINOR:
3268			return (lofi_get_info(dev, lip, LOFI_GET_MINOR,
3269			    credp, flag));
3270
3271		/*
3272		 * This API made limited sense when this value was fixed
3273		 * at LOFI_MAX_FILES.  However, its use to iterate
3274		 * across all possible devices in lofiadm means we don't
3275		 * want to return L_MAXMIN, but the highest
3276		 * *allocated* id.
3277		 */
3278		case LOFI_GET_MAXMINOR:
3279			id = 0;
3280
3281			mutex_enter(&lofi_lock);
3282
3283			for (lsp = list_head(&lofi_list); lsp != NULL;
3284			    lsp = list_next(&lofi_list, lsp)) {
3285				int i;
3286				if (lofi_access(lsp) != 0)
3287					continue;
3288
3289				i = ddi_get_instance(lsp->ls_dip);
3290				if (i > id)
3291					id = i;
3292			}
3293
3294			mutex_exit(&lofi_lock);
3295
3296			error = ddi_copyout(&id, &lip->li_id,
3297			    sizeof (id), flag);
3298			if (error)
3299				return (EFAULT);
3300			return (0);
3301
3302		case LOFI_CHECK_COMPRESSED:
3303			return (lofi_get_info(dev, lip, LOFI_CHECK_COMPRESSED,
3304			    credp, flag));
3305		default:
3306			return (EINVAL);
3307		}
3308	}
3309
3310	mutex_enter(&lofi_lock);
3311	lsp = ddi_get_soft_state(lofi_statep, id);
3312	if (lsp == NULL || lsp->ls_cleanup) {
3313		mutex_exit(&lofi_lock);
3314		return (ENXIO);
3315	}
3316	mutex_exit(&lofi_lock);
3317
3318	if (ddi_prop_exists(DDI_DEV_T_ANY, lsp->ls_dip, DDI_PROP_DONTPASS,
3319	    "labeled") == 1) {
3320		error = cmlb_ioctl(lsp->ls_cmlbhandle, dev, cmd, arg, flag,
3321		    credp, rvalp, 0);
3322		if (error != ENOTTY)
3323			return (error);
3324	}
3325
3326	/*
3327	 * We explicitly allow DKIOCSTATE, but all other ioctls should fail with
3328	 * EIO as if the device was no longer present.
3329	 */
3330	if (lsp->ls_vp == NULL && cmd != DKIOCSTATE)
3331		return (EIO);
3332
3333	/* these are for faking out utilities like newfs */
3334	switch (cmd) {
3335	case DKIOCGMEDIAINFO:
3336	case DKIOCGMEDIAINFOEXT: {
3337		struct dk_minfo_ext media_info;
3338		int shift = lsp->ls_lbshift;
3339		int size;
3340
3341		if (cmd == DKIOCGMEDIAINFOEXT) {
3342			media_info.dki_pbsize = 1U << lsp->ls_pbshift;
3343			size = sizeof (struct dk_minfo_ext);
3344		} else {
3345			size = sizeof (struct dk_minfo);
3346		}
3347
3348		media_info.dki_media_type = DK_FIXED_DISK;
3349		media_info.dki_lbsize = 1U << shift;
3350		media_info.dki_capacity =
3351		    (lsp->ls_vp_size - lsp->ls_crypto_offset) >> shift;
3352
3353		if (ddi_copyout(&media_info, (void *)arg, size, flag))
3354			return (EFAULT);
3355		return (0);
3356	}
3357	case DKIOCREMOVABLE: {
3358		int i = 0;
3359		if (ddi_copyout(&i, (caddr_t)arg, sizeof (int), flag))
3360			return (EFAULT);
3361		return (0);
3362	}
3363
3364	case DKIOCGVTOC: {
3365		struct vtoc vt;
3366		fake_disk_vtoc(lsp, &vt);
3367
3368		switch (ddi_model_convert_from(flag & FMODELS)) {
3369		case DDI_MODEL_ILP32: {
3370			struct vtoc32 vtoc32;
3371
3372			vtoctovtoc32(vt, vtoc32);
3373			if (ddi_copyout(&vtoc32, (void *)arg,
3374			    sizeof (struct vtoc32), flag))
3375				return (EFAULT);
3376			break;
3377			}
3378
3379		case DDI_MODEL_NONE:
3380			if (ddi_copyout(&vt, (void *)arg,
3381			    sizeof (struct vtoc), flag))
3382				return (EFAULT);
3383			break;
3384		}
3385		return (0);
3386	}
3387	case DKIOCINFO: {
3388		struct dk_cinfo ci;
3389		fake_disk_info(dev, &ci);
3390		if (ddi_copyout(&ci, (void *)arg, sizeof (ci), flag))
3391			return (EFAULT);
3392		return (0);
3393	}
3394	case DKIOCG_VIRTGEOM:
3395	case DKIOCG_PHYGEOM:
3396	case DKIOCGGEOM:
3397		error = ddi_copyout(&lsp->ls_dkg, (void *)arg,
3398		    sizeof (struct dk_geom), flag);
3399		if (error)
3400			return (EFAULT);
3401		return (0);
3402	case DKIOCSTATE:
3403		/*
3404		 * Normally, lofi devices are always in the INSERTED state.  If
3405		 * a device is forcefully unmapped, then the device transitions
3406		 * to the DKIO_DEV_GONE state.
3407		 */
3408		if (ddi_copyin((void *)arg, &dkstate, sizeof (dkstate),
3409		    flag) != 0)
3410			return (EFAULT);
3411
3412		mutex_enter(&lsp->ls_vp_lock);
3413		while (((dkstate == DKIO_INSERTED && lsp->ls_vp != NULL) ||
3414		    (dkstate == DKIO_DEV_GONE && lsp->ls_vp == NULL)) &&
3415		    !lsp->ls_cleanup) {
3416			/*
3417			 * By virtue of having the device open, we know that
3418			 * 'lsp' will remain valid when we return.
3419			 */
3420			if (!cv_wait_sig(&lsp->ls_vp_cv, &lsp->ls_vp_lock)) {
3421				mutex_exit(&lsp->ls_vp_lock);
3422				return (EINTR);
3423			}
3424		}
3425
3426		dkstate = (!lsp->ls_cleanup && lsp->ls_vp != NULL ?
3427		    DKIO_INSERTED : DKIO_DEV_GONE);
3428		mutex_exit(&lsp->ls_vp_lock);
3429
3430		if (ddi_copyout(&dkstate, (void *)arg,
3431		    sizeof (dkstate), flag) != 0)
3432			return (EFAULT);
3433		return (0);
3434	case USCSICMD: {
3435		struct uscsi_cmd uscmd;
3436		union scsi_cdb cdb;
3437
3438		if (uscsi_is_inquiry(arg, flag, &cdb, &uscmd) == 0) {
3439			struct scsi_inquiry inq = {0};
3440
3441			lofi_create_inquiry(lsp, &inq);
3442			if (ddi_copyout(&inq, uscmd.uscsi_bufaddr,
3443			    uscmd.uscsi_buflen, flag) != 0)
3444				return (EFAULT);
3445			return (0);
3446		} else if (cdb.scc_cmd == SCMD_READ_CAPACITY) {
3447			struct scsi_capacity capacity;
3448
3449			capacity.capacity =
3450			    BE_32((lsp->ls_vp_size - lsp->ls_crypto_offset) >>
3451			    lsp->ls_lbshift);
3452			capacity.lbasize = BE_32(1 << lsp->ls_lbshift);
3453			if (ddi_copyout(&capacity, uscmd.uscsi_bufaddr,
3454			    uscmd.uscsi_buflen, flag) != 0)
3455				return (EFAULT);
3456			return (0);
3457		}
3458
3459		uscmd.uscsi_rqstatus = 0xff;
3460#ifdef	_MULTI_DATAMODEL
3461		switch (ddi_model_convert_from(flag & FMODELS)) {
3462		case DDI_MODEL_ILP32: {
3463			struct uscsi_cmd32 ucmd32;
3464			uscsi_cmdtouscsi_cmd32((&uscmd), (&ucmd32));
3465			if (ddi_copyout(&ucmd32, (void *)arg, sizeof (ucmd32),
3466			    flag) != 0)
3467				return (EFAULT);
3468			break;
3469		}
3470		case DDI_MODEL_NONE:
3471			if (ddi_copyout(&uscmd, (void *)arg, sizeof (uscmd),
3472			    flag) != 0)
3473				return (EFAULT);
3474			break;
3475		default:
3476			return (EFAULT);
3477		}
3478#else
3479		if (ddi_copyout(&uscmd, (void *)arg, sizeof (uscmd), flag) != 0)
3480			return (EFAULT);
3481#endif	/* _MULTI_DATAMODEL */
3482		return (0);
3483	}
3484
3485	case DKIOCGMBOOT:
3486		return (lofi_urw(lsp, FREAD, 0, 1 << lsp->ls_lbshift,
3487		    arg, flag, credp));
3488
3489	case DKIOCSMBOOT:
3490		return (lofi_urw(lsp, FWRITE, 0, 1 << lsp->ls_lbshift,
3491		    arg, flag, credp));
3492
3493	case DKIOCGETEFI:
3494		if (ddi_copyin((void *)arg, &user_efi,
3495		    sizeof (dk_efi_t), flag) != 0)
3496			return (EFAULT);
3497
3498		return (lofi_urw(lsp, FREAD,
3499		    user_efi.dki_lba * (1 << lsp->ls_lbshift),
3500		    user_efi.dki_length, (intptr_t)user_efi.dki_data,
3501		    flag, credp));
3502
3503	case DKIOCSETEFI:
3504		if (ddi_copyin((void *)arg, &user_efi,
3505		    sizeof (dk_efi_t), flag) != 0)
3506			return (EFAULT);
3507
3508		return (lofi_urw(lsp, FWRITE,
3509		    user_efi.dki_lba * (1 << lsp->ls_lbshift),
3510		    user_efi.dki_length, (intptr_t)user_efi.dki_data,
3511		    flag, credp));
3512
3513	default:
3514#ifdef DEBUG
3515		cmn_err(CE_WARN, "lofi_ioctl: %d is not implemented\n", cmd);
3516#endif	/* DEBUG */
3517		return (ENOTTY);
3518	}
3519}
3520
3521static int
3522lofi_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op, int mod_flags,
3523    char *name, caddr_t valuep, int *lengthp)
3524{
3525	struct lofi_state *lsp;
3526	int rc;
3527
3528	lsp = ddi_get_soft_state(lofi_statep, ddi_get_instance(dip));
3529	if (lsp == NULL) {
3530		return (ddi_prop_op(dev, dip, prop_op, mod_flags,
3531		    name, valuep, lengthp));
3532	}
3533
3534	rc = cmlb_prop_op(lsp->ls_cmlbhandle, dev, dip, prop_op, mod_flags,
3535	    name, valuep, lengthp, LOFI_PART(getminor(dev)), NULL);
3536	if (rc == DDI_PROP_SUCCESS)
3537		return (rc);
3538
3539	return (ddi_prop_op(DDI_DEV_T_ANY, dip, prop_op, mod_flags,
3540	    name, valuep, lengthp));
3541}
3542
3543static struct cb_ops lofi_cb_ops = {
3544	lofi_open,		/* open */
3545	lofi_close,		/* close */
3546	lofi_strategy,		/* strategy */
3547	nodev,			/* print */
3548	nodev,			/* dump */
3549	lofi_read,		/* read */
3550	lofi_write,		/* write */
3551	lofi_ioctl,		/* ioctl */
3552	nodev,			/* devmap */
3553	nodev,			/* mmap */
3554	nodev,			/* segmap */
3555	nochpoll,		/* poll */
3556	lofi_prop_op,		/* prop_op */
3557	0,			/* streamtab  */
3558	D_64BIT | D_NEW | D_MP,	/* Driver compatibility flag */
3559	CB_REV,
3560	lofi_aread,
3561	lofi_awrite
3562};
3563
3564static struct dev_ops lofi_ops = {
3565	DEVO_REV,		/* devo_rev, */
3566	0,			/* refcnt  */
3567	lofi_info,		/* info */
3568	nulldev,		/* identify */
3569	nulldev,		/* probe */
3570	lofi_attach,		/* attach */
3571	lofi_detach,		/* detach */
3572	nodev,			/* reset */
3573	&lofi_cb_ops,		/* driver operations */
3574	NULL,			/* no bus operations */
3575	NULL,			/* power */
3576	ddi_quiesce_not_needed,	/* quiesce */
3577};
3578
3579static struct modldrv modldrv = {
3580	&mod_driverops,
3581	"loopback file driver",
3582	&lofi_ops,
3583};
3584
3585static struct modlinkage modlinkage = {
3586	MODREV_1,
3587	&modldrv,
3588	NULL
3589};
3590
3591int
3592_init(void)
3593{
3594	int error;
3595
3596	list_create(&lofi_list, sizeof (struct lofi_state),
3597	    offsetof(struct lofi_state, ls_list));
3598
3599	error = ddi_soft_state_init((void **)&lofi_statep,
3600	    sizeof (struct lofi_state), 0);
3601	if (error) {
3602		list_destroy(&lofi_list);
3603		return (error);
3604	}
3605
3606	/*
3607	 * The minor number is stored as id << LOFI_CMLB_SHIFT as
3608	 * we need to reserve space for cmlb minor numbers.
3609	 * This will leave out 4096 id values on 32bit kernel, which should
3610	 * still suffice.
3611	 */
3612	lofi_id = id_space_create("lofi_id", 1,
3613	    (1 << (L_BITSMINOR - LOFI_CMLB_SHIFT)));
3614
3615	if (lofi_id == NULL) {
3616		ddi_soft_state_fini((void **)&lofi_statep);
3617		list_destroy(&lofi_list);
3618		return (DDI_FAILURE);
3619	}
3620
3621	mutex_init(&lofi_lock, NULL, MUTEX_DRIVER, NULL);
3622
3623	error = mod_install(&modlinkage);
3624
3625	if (error) {
3626		id_space_destroy(lofi_id);
3627		mutex_destroy(&lofi_lock);
3628		ddi_soft_state_fini((void **)&lofi_statep);
3629		list_destroy(&lofi_list);
3630	}
3631
3632	return (error);
3633}
3634
3635int
3636_fini(void)
3637{
3638	int	error;
3639
3640	mutex_enter(&lofi_lock);
3641
3642	if (!list_is_empty(&lofi_list)) {
3643		mutex_exit(&lofi_lock);
3644		return (EBUSY);
3645	}
3646
3647	mutex_exit(&lofi_lock);
3648
3649	error = mod_remove(&modlinkage);
3650	if (error)
3651		return (error);
3652
3653	mutex_destroy(&lofi_lock);
3654	id_space_destroy(lofi_id);
3655	ddi_soft_state_fini((void **)&lofi_statep);
3656	list_destroy(&lofi_list);
3657
3658	return (error);
3659}
3660
3661int
3662_info(struct modinfo *modinfop)
3663{
3664	return (mod_info(&modlinkage, modinfop));
3665}
3666