1fa9e406ahrens/*
2fa9e406ahrens * CDDL HEADER START
3fa9e406ahrens *
4fa9e406ahrens * The contents of this file are subject to the terms of the
5ea8dc4beschrock * Common Development and Distribution License (the "License").
6ea8dc4beschrock * You may not use this file except in compliance with the License.
7fa9e406ahrens *
8fa9e406ahrens * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9fa9e406ahrens * or http://www.opensolaris.org/os/licensing.
10fa9e406ahrens * See the License for the specific language governing permissions
11fa9e406ahrens * and limitations under the License.
12fa9e406ahrens *
13fa9e406ahrens * When distributing Covered Code, include this CDDL HEADER in each
14fa9e406ahrens * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15fa9e406ahrens * If applicable, add the following below this CDDL HEADER, with the
16fa9e406ahrens * fields enclosed by brackets "[]" replaced with your own identifying
17fa9e406ahrens * information: Portions Copyright [yyyy] [name of copyright owner]
18fa9e406ahrens *
19fa9e406ahrens * CDDL HEADER END
20fa9e406ahrens */
21fa9e406ahrens/*
22c99e4bdChris Kirby * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
239a686fbPaul Dagnelie * Copyright (c) 2012, 2015 by Delphix. All rights reserved.
24d8ab6e1Don Brady * Copyright 2020 Joyent, Inc.
25f06dce2Andrew Stormont * Copyright 2017 RackTop Systems.
26fa9e406ahrens */
27fa9e406ahrens
28fa9e406ahrens#include <assert.h>
29c9431faahl#include <fcntl.h>
30fa9e406ahrens#include <poll.h>
31fa9e406ahrens#include <stdio.h>
32fa9e406ahrens#include <stdlib.h>
33c9431faahl#include <string.h>
34c9431faahl#include <zlib.h>
35df15e41Matthew Ahrens#include <libgen.h>
36fa9e406ahrens#include <sys/spa.h>
37c9431faahl#include <sys/stat.h>
38fa9e406ahrens#include <sys/processor.h>
39c9431faahl#include <sys/zfs_context.h>
403b2aab1Matthew Ahrens#include <sys/rrwlock.h>
41c9431faahl#include <sys/zmod.h>
429517395ek#include <sys/utsname.h>
435679c89jv#include <sys/systeminfo.h>
44d8ab6e1Don Brady#include <libzutil.h>
45eb63303Tom Caputi#include <sys/crypto/common.h>
46eb63303Tom Caputi#include <sys/crypto/impl.h>
47eb63303Tom Caputi#include <sys/crypto/api.h>
48eb63303Tom Caputi#include <sys/sha2.h>
49eb63303Tom Caputi#include <crypto/aes/aes_impl.h>
505ad8204nd
51f06dce2Andrew Stormontextern void system_taskq_init(void);
52f06dce2Andrew Stormontextern void system_taskq_fini(void);
53f06dce2Andrew Stormont
54fa9e406ahrens/*
55fa9e406ahrens * Emulation of kernel services in userland.
56fa9e406ahrens */
57fa9e406ahrens
58f06dce2Andrew Stormontpgcnt_t physmem;
59fa9e406ahrensvnode_t *rootdir = (vnode_t *)0xabcd1234;
605679c89jvchar hw_serial[HW_HOSTID_LEN];
61283b846George.Wilsonkmutex_t cpu_lock;
6294dd93aGeorge Wilsonvmem_t *zio_arena = NULL;
639517395ek
64df15e41Matthew Ahrens/* If set, all blocks read will be copied to the specified directory. */
65df15e41Matthew Ahrenschar *vn_dumpdir = NULL;
66df15e41Matthew Ahrens
679517395ekstruct utsname utsname = {
689517395ek	"userland", "libzpool", "1", "1", "na"
699517395ek};
70fa9e406ahrens
71fa9e406ahrens/*
72fa9e406ahrens * =========================================================================
73fa9e406ahrens * vnode operations
74fa9e406ahrens * =========================================================================
75fa9e406ahrens */
76fa9e406ahrens/*
77fa9e406ahrens * Note: for the xxxat() versions of these functions, we assume that the
78fa9e406ahrens * starting vp is always rootdir (which is true for spa_directory.c, the only
79fa9e406ahrens * ZFS consumer of these interfaces).  We assert this is true, and then emulate
80fa9e406ahrens * them by adding '/' in front of the path.
81fa9e406ahrens */
82fa9e406ahrens
83fa9e406ahrens/*ARGSUSED*/
84fa9e406ahrensint
85fa9e406ahrensvn_open(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2, int x3)
86fa9e406ahrens{
87fa9e406ahrens	int fd;
88df15e41Matthew Ahrens	int dump_fd;
89fa9e406ahrens	vnode_t *vp;
90fa9e406ahrens	int old_umask;
91fa9e406ahrens	char realpath[MAXPATHLEN];
92fa9e406ahrens	struct stat64 st;
93fa9e406ahrens
94fa9e406ahrens	/*
95fa9e406ahrens	 * If we're accessing a real disk from userland, we need to use
96fa9e406ahrens	 * the character interface to avoid caching.  This is particularly
97fa9e406ahrens	 * important if we're trying to look at a real in-kernel storage
98fa9e406ahrens	 * pool from userland, e.g. via zdb, because otherwise we won't
99fa9e406ahrens	 * see the changes occurring under the segmap cache.
100fa9e406ahrens	 * On the other hand, the stupid character device returns zero
101fa9e406ahrens	 * for its size.  So -- gag -- we open the block device to get
102fa9e406ahrens	 * its size, and remember it for subsequent VOP_GETATTR().
103fa9e406ahrens	 */
104fa9e406ahrens	if (strncmp(path, "/dev/", 5) == 0) {
105fa9e406ahrens		char *dsk;
106fa9e406ahrens		fd = open64(path, O_RDONLY);
107fa9e406ahrens		if (fd == -1)
108fa9e406ahrens			return (errno);
109fa9e406ahrens		if (fstat64(fd, &st) == -1) {
110fa9e406ahrens			close(fd);
111fa9e406ahrens			return (errno);
112fa9e406ahrens		}
113fa9e406ahrens		close(fd);
114fa9e406ahrens		(void) sprintf(realpath, "%s", path);
115fa9e406ahrens		dsk = strstr(path, "/dsk/");
116fa9e406ahrens		if (dsk != NULL)
117fa9e406ahrens			(void) sprintf(realpath + (dsk - path) + 1, "r%s",
118fa9e406ahrens			    dsk + 1);
119fa9e406ahrens	} else {
120fa9e406ahrens		(void) sprintf(realpath, "%s", path);
121fa9e406ahrens		if (!(flags & FCREAT) && stat64(realpath, &st) == -1)
122fa9e406ahrens			return (errno);
123fa9e406ahrens	}
124fa9e406ahrens
125fa9e406ahrens	if (flags & FCREAT)
126fa9e406ahrens		old_umask = umask(0);
127fa9e406ahrens
128fa9e406ahrens	/*
129fa9e406ahrens	 * The construct 'flags - FREAD' conveniently maps combinations of
130fa9e406ahrens	 * FREAD and FWRITE to the corresponding O_RDONLY, O_WRONLY, and O_RDWR.
131fa9e406ahrens	 */
132fa9e406ahrens	fd = open64(realpath, flags - FREAD, mode);
133fa9e406ahrens
134fa9e406ahrens	if (flags & FCREAT)
135fa9e406ahrens		(void) umask(old_umask);
136fa9e406ahrens
137df15e41Matthew Ahrens	if (vn_dumpdir != NULL) {
138df15e41Matthew Ahrens		char dumppath[MAXPATHLEN];
139df15e41Matthew Ahrens		(void) snprintf(dumppath, sizeof (dumppath),
140df15e41Matthew Ahrens		    "%s/%s", vn_dumpdir, basename(realpath));
141df15e41Matthew Ahrens		dump_fd = open64(dumppath, O_CREAT | O_WRONLY, 0666);
142df15e41Matthew Ahrens		if (dump_fd == -1)
143df15e41Matthew Ahrens			return (errno);
144df15e41Matthew Ahrens	} else {
145df15e41Matthew Ahrens		dump_fd = -1;
146df15e41Matthew Ahrens	}
147df15e41Matthew Ahrens
148fa9e406ahrens	if (fd == -1)
149fa9e406ahrens		return (errno);
150fa9e406ahrens
151fa9e406ahrens	if (fstat64(fd, &st) == -1) {
152fa9e406ahrens		close(fd);
153fa9e406ahrens		return (errno);
154fa9e406ahrens	}
155fa9e406ahrens
156fa9e406ahrens	(void) fcntl(fd, F_SETFD, FD_CLOEXEC);
157fa9e406ahrens
158fa9e406ahrens	*vpp = vp = umem_zalloc(sizeof (vnode_t), UMEM_NOFAIL);
159fa9e406ahrens
160fa9e406ahrens	vp->v_fd = fd;
161fa9e406ahrens	vp->v_size = st.st_size;
162fa9e406ahrens	vp->v_path = spa_strdup(path);
163df15e41Matthew Ahrens	vp->v_dump_fd = dump_fd;
164fa9e406ahrens
165fa9e406ahrens	return (0);
166fa9e406ahrens}
167fa9e406ahrens
168da6c28aamw/*ARGSUSED*/
169fa9e406ahrensint
170fa9e406ahrensvn_openat(char *path, int x1, int flags, int mode, vnode_t **vpp, int x2,
171da6c28aamw    int x3, vnode_t *startvp, int fd)
172fa9e406ahrens{
173fa9e406ahrens	char *realpath = umem_alloc(strlen(path) + 2, UMEM_NOFAIL);
174fa9e406ahrens	int ret;
175fa9e406ahrens
176fa9e406ahrens	ASSERT(startvp == rootdir);
177fa9e406ahrens	(void) sprintf(realpath, "/%s", path);
178fa9e406ahrens
179da6c28aamw	/* fd ignored for now, need if want to simulate nbmand support */
180fa9e406ahrens	ret = vn_open(realpath, x1, flags, mode, vpp, x2, x3);
181fa9e406ahrens
182fa9e406ahrens	umem_free(realpath, strlen(path) + 2);
183fa9e406ahrens
184fa9e406ahrens	return (ret);
185fa9e406ahrens}
186fa9e406ahrens
187fa9e406ahrens/*ARGSUSED*/
188fa9e406ahrensint
189fa9e406ahrensvn_rdwr(int uio, vnode_t *vp, void *addr, ssize_t len, offset_t offset,
1909a686fbPaul Dagnelie    int x1, int x2, rlim64_t x3, void *x4, ssize_t *residp)
191fa9e406ahrens{
192fa9e406ahrens	ssize_t iolen, split;
193fa9e406ahrens
194fa9e406ahrens	if (uio == UIO_READ) {
195fa9e406ahrens		iolen = pread64(vp->v_fd, addr, len, offset);
196df15e41Matthew Ahrens		if (vp->v_dump_fd != -1) {
197df15e41Matthew Ahrens			int status =
198df15e41Matthew Ahrens			    pwrite64(vp->v_dump_fd, addr, iolen, offset);
199df15e41Matthew Ahrens			ASSERT(status != -1);
200df15e41Matthew Ahrens		}
201fa9e406ahrens	} else {
202fa9e406ahrens		/*
203fa9e406ahrens		 * To simulate partial disk writes, we split writes into two
204fa9e406ahrens		 * system calls so that the process can be killed in between.
205fa9e406ahrens		 */
206ad135b5Christopher Siden		int sectors = len >> SPA_MINBLOCKSHIFT;
207ad135b5Christopher Siden		split = (sectors > 0 ? rand() % sectors : 0) <<
208ad135b5Christopher Siden		    SPA_MINBLOCKSHIFT;
209fa9e406ahrens		iolen = pwrite64(vp->v_fd, addr, split, offset);
210fa9e406ahrens		iolen += pwrite64(vp->v_fd, (char *)addr + split,
211fa9e406ahrens		    len - split, offset + split);
212fa9e406ahrens	}
213fa9e406ahrens
214fa9e406ahrens	if (iolen == -1)
215fa9e406ahrens		return (errno);
216fa9e406ahrens	if (residp)
217fa9e406ahrens		*residp = len - iolen;
218fa9e406ahrens	else if (iolen != len)
219fa9e406ahrens		return (EIO);
220fa9e406ahrens	return (0);
221fa9e406ahrens}
222fa9e406ahrens
223fa9e406ahrensvoid
224fa9e406ahrensvn_close(vnode_t *vp)
225fa9e406ahrens{
226fa9e406ahrens	close(vp->v_fd);
227df15e41Matthew Ahrens	if (vp->v_dump_fd != -1)
228df15e41Matthew Ahrens		close(vp->v_dump_fd);
229fa9e406ahrens	spa_strfree(vp->v_path);
230fa9e406ahrens	umem_free(vp, sizeof (vnode_t));
231fa9e406ahrens}
232fa9e406ahrens
233095bcd6George Wilson/*
234095bcd6George Wilson * At a minimum we need to update the size since vdev_reopen()
235095bcd6George Wilson * will no longer call vn_openat().
236095bcd6George Wilson */
237095bcd6George Wilsonint
238095bcd6George Wilsonfop_getattr(vnode_t *vp, vattr_t *vap)
239095bcd6George Wilson{
240095bcd6George Wilson	struct stat64 st;
241095bcd6George Wilson
242095bcd6George Wilson	if (fstat64(vp->v_fd, &st) == -1) {
243095bcd6George Wilson		close(vp->v_fd);
244095bcd6George Wilson		return (errno);
245095bcd6George Wilson	}
246095bcd6George Wilson
247095bcd6George Wilson	vap->va_size = st.st_size;
248095bcd6George Wilson	return (0);
249095bcd6George Wilson}
250095bcd6George Wilson
251fa9e406ahrens#ifdef ZFS_DEBUG
252fa9e406ahrens
253fa9e406ahrens/*
254fa9e406ahrens * =========================================================================
255fa9e406ahrens * Figure out which debugging statements to print
256fa9e406ahrens * =========================================================================
257fa9e406ahrens */
258fa9e406ahrens
259fa9e406ahrensstatic char *dprintf_string;
260fa9e406ahrensstatic int dprintf_print_all;
261fa9e406ahrens
262fa9e406ahrensint
263fa9e406ahrensdprintf_find_string(const char *string)
264fa9e406ahrens{
265fa9e406ahrens	char *tmp_str = dprintf_string;
266fa9e406ahrens	int len = strlen(string);
267fa9e406ahrens
268fa9e406ahrens	/*
269fa9e406ahrens	 * Find out if this is a string we want to print.
270fa9e406ahrens	 * String format: file1.c,function_name1,file2.c,file3.c
271fa9e406ahrens	 */
272fa9e406ahrens
273fa9e406ahrens	while (tmp_str != NULL) {
274fa9e406ahrens		if (strncmp(tmp_str, string, len) == 0 &&
275fa9e406ahrens		    (tmp_str[len] == ',' || tmp_str[len] == '\0'))
276fa9e406ahrens			return (1);
277fa9e406ahrens		tmp_str = strchr(tmp_str, ',');
278fa9e406ahrens		if (tmp_str != NULL)
279fa9e406ahrens			tmp_str++; /* Get rid of , */
280fa9e406ahrens	}
281fa9e406ahrens	return (0);
282fa9e406ahrens}
283fa9e406ahrens
284fa9e406ahrensvoid
285fa9e406ahrensdprintf_setup(int *argc, char **argv)
286fa9e406ahrens{
287fa9e406ahrens	int i, j;
288fa9e406ahrens
289fa9e406ahrens	/*
290fa9e406ahrens	 * Debugging can be specified two ways: by setting the
291fa9e406ahrens	 * environment variable ZFS_DEBUG, or by including a
292fa9e406ahrens	 * "debug=..."  argument on the command line.  The command
293fa9e406ahrens	 * line setting overrides the environment variable.
294fa9e406ahrens	 */
295fa9e406ahrens
296fa9e406ahrens	for (i = 1; i < *argc; i++) {
297fa9e406ahrens		int len = strlen("debug=");
298fa9e406ahrens		/* First look for a command line argument */
299fa9e406ahrens		if (strncmp("debug=", argv[i], len) == 0) {
300fa9e406ahrens			dprintf_string = argv[i] + len;
301fa9e406ahrens			/* Remove from args */
302fa9e406ahrens			for (j = i; j < *argc; j++)
303fa9e406ahrens				argv[j] = argv[j+1];
304fa9e406ahrens			argv[j] = NULL;
305fa9e406ahrens			(*argc)--;
306fa9e406ahrens		}
307fa9e406ahrens	}
308fa9e406ahrens
309fa9e406ahrens	if (dprintf_string == NULL) {
310fa9e406ahrens		/* Look for ZFS_DEBUG environment variable */
311fa9e406ahrens		dprintf_string = getenv("ZFS_DEBUG");
312fa9e406ahrens	}
313fa9e406ahrens
314fa9e406ahrens	/*
315fa9e406ahrens	 * Are we just turning on all debugging?
316fa9e406ahrens	 */
317fa9e406ahrens	if (dprintf_find_string("on"))
318fa9e406ahrens		dprintf_print_all = 1;
3197fa49eaMatthew Ahrens
3207fa49eaMatthew Ahrens	if (dprintf_string != NULL)
3217fa49eaMatthew Ahrens		zfs_flags |= ZFS_DEBUG_DPRINTF;
322fa9e406ahrens}
323fa9e406ahrens
324fa9e406ahrens/*
325fa9e406ahrens * =========================================================================
326fa9e406ahrens * debug printfs
327fa9e406ahrens * =========================================================================
328fa9e406ahrens */
329fa9e406ahrensvoid
330fa9e406ahrens__dprintf(const char *file, const char *func, int line, const char *fmt, ...)
331fa9e406ahrens{
332fa9e406ahrens	const char *newfile;
333fa9e406ahrens	va_list adx;
334fa9e406ahrens
335fa9e406ahrens	/*
336fa9e406ahrens	 * Get rid of annoying "../common/" prefix to filename.
337fa9e406ahrens	 */
338fa9e406ahrens	newfile = strrchr(file, '/');
339fa9e406ahrens	if (newfile != NULL) {
340fa9e406ahrens		newfile = newfile + 1; /* Get rid of leading / */
341fa9e406ahrens	} else {
342fa9e406ahrens		newfile = file;
343fa9e406ahrens	}
344fa9e406ahrens
345fa9e406ahrens	if (dprintf_print_all ||
346fa9e406ahrens	    dprintf_find_string(newfile) ||
347fa9e406ahrens	    dprintf_find_string(func)) {
348fa9e406ahrens		/* Print out just the function name if requested */
349fa9e406ahrens		flockfile(stdout);
350fa9e406ahrens		if (dprintf_find_string("pid"))
351fa9e406ahrens			(void) printf("%d ", getpid());
352fa9e406ahrens		if (dprintf_find_string("tid"))
353fa9e406ahrens			(void) printf("%u ", thr_self());
354fa9e406ahrens		if (dprintf_find_string("cpu"))
355fa9e406ahrens			(void) printf("%u ", getcpuid());
356fa9e406ahrens		if (dprintf_find_string("time"))
357fa9e406ahrens			(void) printf("%llu ", gethrtime());
358fa9e406ahrens		if (dprintf_find_string("long"))
359fa9e406ahrens			(void) printf("%s, line %d: ", newfile, line);
360fa9e406ahrens		(void) printf("%s: ", func);
361fa9e406ahrens		va_start(adx, fmt);
362fa9e406ahrens		(void) vprintf(fmt, adx);
363fa9e406ahrens		va_end(adx);
364fa9e406ahrens		funlockfile(stdout);
365fa9e406ahrens	}
366fa9e406ahrens}
367fa9e406ahrens
368fa9e406ahrens#endif /* ZFS_DEBUG */
369fa9e406ahrens
370fa9e406ahrens/*
371fa9e406ahrens * =========================================================================
372ea8dc4beschrock * kobj interfaces
373ea8dc4beschrock * =========================================================================
374ea8dc4beschrock */
375ea8dc4beschrockstruct _buf *
376ea8dc4beschrockkobj_open_file(char *name)
377ea8dc4beschrock{
378ea8dc4beschrock	struct _buf *file;
379ea8dc4beschrock	vnode_t *vp;
380ea8dc4beschrock
381ea8dc4beschrock	/* set vp as the _fd field of the file */
382da6c28aamw	if (vn_openat(name, UIO_SYSSPACE, FREAD, 0, &vp, 0, 0, rootdir,
383da6c28aamw	    -1) != 0)
384ea8dc4beschrock		return ((void *)-1UL);
385ea8dc4beschrock
386ea8dc4beschrock	file = umem_zalloc(sizeof (struct _buf), UMEM_NOFAIL);
387ea8dc4beschrock	file->_fd = (intptr_t)vp;
388ea8dc4beschrock	return (file);
389ea8dc4beschrock}
390ea8dc4beschrock
391ea8dc4beschrockint
392ea8dc4beschrockkobj_read_file(struct _buf *file, char *buf, unsigned size, unsigned off)
393ea8dc4beschrock{
394ea8dc4beschrock	ssize_t resid;
395ea8dc4beschrock
396ea8dc4beschrock	vn_rdwr(UIO_READ, (vnode_t *)file->_fd, buf, size, (offset_t)off,
397ea8dc4beschrock	    UIO_SYSSPACE, 0, 0, 0, &resid);
398ea8dc4beschrock
399b1b8ab3lling	return (size - resid);
400ea8dc4beschrock}
401ea8dc4beschrock
402ea8dc4beschrockvoid
403ea8dc4beschrockkobj_close_file(struct _buf *file)
404ea8dc4beschrock{
405ea8dc4beschrock	vn_close((vnode_t *)file->_fd);
406ea8dc4beschrock	umem_free(file, sizeof (struct _buf));
407ea8dc4beschrock}
408ea8dc4beschrock
409ea8dc4beschrockint
410b1b8ab3llingkobj_get_filesize(struct _buf *file, uint64_t *size)
411ea8dc4beschrock{
412ea8dc4beschrock	struct stat64 st;
413b1b8ab3lling	vnode_t *vp = (vnode_t *)file->_fd;
414b1b8ab3lling
415ea8dc4beschrock	if (fstat64(vp->v_fd, &st) == -1) {
416ea8dc4beschrock		vn_close(vp);
417ea8dc4beschrock		return (errno);
418ea8dc4beschrock	}
419b1b8ab3lling	*size = st.st_size;
420ea8dc4beschrock	return (0);
421ea8dc4beschrock}
422ea8dc4beschrock
423ea8dc4beschrock/*
424ea8dc4beschrock * =========================================================================
425dd50e0cTony Hutter * misc routines
426dd50e0cTony Hutter * =========================================================================
427dd50e0cTony Hutter */
428dd50e0cTony Hutter
429dd50e0cTony Hutter/*
430dd50e0cTony Hutter * Find lowest one bit set.
431dd50e0cTony Hutter * Returns bit number + 1 of lowest bit that is set, otherwise returns 0.
432dd50e0cTony Hutter * This is basically a reimplementation of ffsll(), which is GNU specific.
433dd50e0cTony Hutter */
434dd50e0cTony Hutterint
435dd50e0cTony Hutterlowbit64(uint64_t i)
436dd50e0cTony Hutter{
437dd50e0cTony Hutter	register int h = 64;
438dd50e0cTony Hutter	if (i == 0)
439dd50e0cTony Hutter		return (0);
440dd50e0cTony Hutter
441dd50e0cTony Hutter	if (i & 0x00000000ffffffffULL)
442dd50e0cTony Hutter		h -= 32;
443dd50e0cTony Hutter	else
444dd50e0cTony Hutter		i >>= 32;
445dd50e0cTony Hutter
446dd50e0cTony Hutter	if (i & 0x0000ffff)
447dd50e0cTony Hutter		h -= 16;
448dd50e0cTony Hutter	else
449dd50e0cTony Hutter		i >>= 16;
450dd50e0cTony Hutter
451dd50e0cTony Hutter	if (i & 0x00ff)
452dd50e0cTony Hutter		h -= 8;
453dd50e0cTony Hutter	else
454dd50e0cTony Hutter		i >>= 8;
455dd50e0cTony Hutter
456dd50e0cTony Hutter	if (i & 0x0f)
457dd50e0cTony Hutter		h -= 4;
458dd50e0cTony Hutter	else
459dd50e0cTony Hutter		i >>= 4;
460dd50e0cTony Hutter
461dd50e0cTony Hutter	if (i & 0x3)
462dd50e0cTony Hutter		h -= 2;
463dd50e0cTony Hutter	else
464dd50e0cTony Hutter		i >>= 2;
465dd50e0cTony Hutter
466dd50e0cTony Hutter	if (i & 0x1)
467dd50e0cTony Hutter		h -= 1;
468dd50e0cTony Hutter
469dd50e0cTony Hutter	return (h);
470dd50e0cTony Hutter}
471dd50e0cTony Hutter
472dd50e0cTony Hutterint
473dd50e0cTony Hutterhighbit64(uint64_t i)
474dd50e0cTony Hutter{
475dd50e0cTony Hutter	int h = 1;
476dd50e0cTony Hutter
477dd50e0cTony Hutter	if (i == 0)
478dd50e0cTony Hutter		return (0);
479dd50e0cTony Hutter	if (i & 0xffffffff00000000ULL) {
480dd50e0cTony Hutter		h += 32; i >>= 32;
481dd50e0cTony Hutter	}
482dd50e0cTony Hutter	if (i & 0xffff0000) {
483dd50e0cTony Hutter		h += 16; i >>= 16;
484dd50e0cTony Hutter	}
485dd50e0cTony Hutter	if (i & 0xff00) {
486dd50e0cTony Hutter		h += 8; i >>= 8;
487dd50e0cTony Hutter	}
488dd50e0cTony Hutter	if (i & 0xf0) {
489dd50e0cTony Hutter		h += 4; i >>= 4;
490dd50e0cTony Hutter	}
491dd50e0cTony Hutter	if (i & 0xc) {
492dd50e0cTony Hutter		h += 2; i >>= 2;
493dd50e0cTony Hutter	}
494dd50e0cTony Hutter	if (i & 0x2) {
495dd50e0cTony Hutter		h += 1;
496dd50e0cTony Hutter	}
497dd50e0cTony Hutter	return (h);
498dd50e0cTony Hutter}
499dd50e0cTony Hutter
500dd50e0cTony Hutter/*
501dd50e0cTony Hutter * =========================================================================
502fa9e406ahrens * kernel emulation setup & teardown
503fa9e406ahrens * =========================================================================
504fa9e406ahrens */
505fa9e406ahrensstatic int
506fa9e406ahrensumem_out_of_memory(void)
507fa9e406ahrens{
508fa9e406ahrens	char errmsg[] = "out of memory -- generating core dump\n";
509fa9e406ahrens
510fa9e406ahrens	write(fileno(stderr), errmsg, sizeof (errmsg));
511fa9e406ahrens	abort();
512fa9e406ahrens	return (0);
513fa9e406ahrens}
514fa9e406ahrens
515fa9e406ahrensvoid
516fa9e406ahrenskernel_init(int mode)
517fa9e406ahrens{
5183b2aab1Matthew Ahrens	extern uint_t rrw_tsd_key;
5193b2aab1Matthew Ahrens
520fa9e406ahrens	umem_nofail_callback(umem_out_of_memory);
521fa9e406ahrens
522fa9e406ahrens	physmem = sysconf(_SC_PHYS_PAGES);
523fa9e406ahrens
524fa9e406ahrens	dprintf("physmem = %llu pages (%.2f GB)\n", physmem,
525fa9e406ahrens	    (double)physmem * sysconf(_SC_PAGE_SIZE) / (1ULL << 30));
526fa9e406ahrens
5273ad6c7fVictor Latushkin	(void) snprintf(hw_serial, sizeof (hw_serial), "%ld",
528e0f1c0aOlaf Faaland	    (mode & FWRITE) ? get_system_hostid() : 0);
5299517395ek
53088b7b0fMatthew Ahrens	system_taskq_init();
53188b7b0fMatthew Ahrens
532283b846George.Wilson	mutex_init(&cpu_lock, NULL, MUTEX_DEFAULT, NULL);
533283b846George.Wilson
534fa9e406ahrens	spa_init(mode);
5353b2aab1Matthew Ahrens
5363b2aab1Matthew Ahrens	tsd_create(&rrw_tsd_key, rrw_tsd_destroy);
537fa9e406ahrens}
538fa9e406ahrens
539fa9e406ahrensvoid
540fa9e406ahrenskernel_fini(void)
541fa9e406ahrens{
542fa9e406ahrens	spa_fini();
54317f17c2bonwick
544d20e665Ricardo M. Correia	system_taskq_fini();
545fa9e406ahrens}
546c9431faahl
5476f79381Pavel Zakharov/* ARGSUSED */
5486f79381Pavel Zakharovuint32_t
5496f79381Pavel Zakharovzone_get_hostid(void *zonep)
5506f79381Pavel Zakharov{
5516f79381Pavel Zakharov	/*
5526f79381Pavel Zakharov	 * We're emulating the system's hostid in userland.
5536f79381Pavel Zakharov	 */
5546f79381Pavel Zakharov	return (strtoul(hw_serial, NULL, 10));
5556f79381Pavel Zakharov}
5566f79381Pavel Zakharov
557c9431faahlint
558c9431faahlz_uncompress(void *dst, size_t *dstlen, const void *src, size_t srclen)
559c9431faahl{
560c9431faahl	int ret;
561c9431faahl	uLongf len = *dstlen;
562c9431faahl
563c9431faahl	if ((ret = uncompress(dst, &len, src, srclen)) == Z_OK)
564c9431faahl		*dstlen = (size_t)len;
565c9431faahl
566c9431faahl	return (ret);
567c9431faahl}
568c9431faahl
569c9431faahlint
570c9431faahlz_compress_level(void *dst, size_t *dstlen, const void *src, size_t srclen,
571c9431faahl    int level)
572c9431faahl{
573c9431faahl	int ret;
574c9431faahl	uLongf len = *dstlen;
575c9431faahl
576c9431faahl	if ((ret = compress2(dst, &len, src, srclen, level)) == Z_OK)
577c9431faahl		*dstlen = (size_t)len;
578c9431faahl
579