1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26/*
27 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
28 * Copyright (c) 2013, 2018 by Delphix. All rights reserved.
29 */
30
31#include <sys/zfs_context.h>
32#include <sys/compress.h>
33#include <sys/spa.h>
34#include <sys/zfeature.h>
35#include <sys/zio.h>
36#include <sys/zio_compress.h>
37
38/*
39 * If nonzero, every 1/X decompression attempts will fail, simulating
40 * an undetected memory error.
41 */
42uint64_t zio_decompress_fail_fraction = 0;
43
44/*
45 * Compression vectors.
46 */
47zio_compress_info_t zio_compress_table[ZIO_COMPRESS_FUNCTIONS] = {
48	{"inherit",		0,	NULL,		NULL},
49	{"on",			0,	NULL,		NULL},
50	{"uncompressed",	0,	NULL,		NULL},
51	{"lzjb",		0,	lzjb_compress,	lzjb_decompress},
52	{"empty",		0,	NULL,		NULL},
53	{"gzip-1",		1,	gzip_compress,	gzip_decompress},
54	{"gzip-2",		2,	gzip_compress,	gzip_decompress},
55	{"gzip-3",		3,	gzip_compress,	gzip_decompress},
56	{"gzip-4",		4,	gzip_compress,	gzip_decompress},
57	{"gzip-5",		5,	gzip_compress,	gzip_decompress},
58	{"gzip-6",		6,	gzip_compress,	gzip_decompress},
59	{"gzip-7",		7,	gzip_compress,	gzip_decompress},
60	{"gzip-8",		8,	gzip_compress,	gzip_decompress},
61	{"gzip-9",		9,	gzip_compress,	gzip_decompress},
62	{"zle",			64,	zle_compress,	zle_decompress},
63	{"lz4",			0,	lz4_compress,	lz4_decompress}
64};
65
66enum zio_compress
67zio_compress_select(spa_t *spa, enum zio_compress child,
68    enum zio_compress parent)
69{
70	enum zio_compress result;
71
72	ASSERT(child < ZIO_COMPRESS_FUNCTIONS);
73	ASSERT(parent < ZIO_COMPRESS_FUNCTIONS);
74	ASSERT(parent != ZIO_COMPRESS_INHERIT);
75
76	result = child;
77	if (result == ZIO_COMPRESS_INHERIT)
78		result = parent;
79
80	if (result == ZIO_COMPRESS_ON) {
81		if (spa_feature_is_active(spa, SPA_FEATURE_LZ4_COMPRESS))
82			result = ZIO_COMPRESS_LZ4_ON_VALUE;
83		else
84			result = ZIO_COMPRESS_LEGACY_ON_VALUE;
85	}
86
87	return (result);
88}
89
90/*ARGSUSED*/
91static int
92zio_compress_zeroed_cb(void *data, size_t len, void *private)
93{
94	uint64_t *end = (uint64_t *)((char *)data + len);
95	for (uint64_t *word = (uint64_t *)data; word < end; word++)
96		if (*word != 0)
97			return (1);
98
99	return (0);
100}
101
102size_t
103zio_compress_data(enum zio_compress c, abd_t *src, void *dst, size_t s_len)
104{
105	size_t c_len, d_len;
106	zio_compress_info_t *ci = &zio_compress_table[c];
107
108	ASSERT((uint_t)c < ZIO_COMPRESS_FUNCTIONS);
109	ASSERT((uint_t)c == ZIO_COMPRESS_EMPTY || ci->ci_compress != NULL);
110
111	/*
112	 * If the data is all zeroes, we don't even need to allocate
113	 * a block for it.  We indicate this by returning zero size.
114	 */
115	if (abd_iterate_func(src, 0, s_len, zio_compress_zeroed_cb, NULL) == 0)
116		return (0);
117
118	if (c == ZIO_COMPRESS_EMPTY)
119		return (s_len);
120
121	/* Compress at least 12.5% */
122	d_len = s_len - (s_len >> 3);
123
124	/* No compression algorithms can read from ABDs directly */
125	void *tmp = abd_borrow_buf_copy(src, s_len);
126	c_len = ci->ci_compress(tmp, dst, s_len, d_len, ci->ci_level);
127	abd_return_buf(src, tmp, s_len);
128
129	if (c_len > d_len)
130		return (s_len);
131
132	ASSERT3U(c_len, <=, d_len);
133	return (c_len);
134}
135
136int
137zio_decompress_data_buf(enum zio_compress c, void *src, void *dst,
138    size_t s_len, size_t d_len)
139{
140	zio_compress_info_t *ci = &zio_compress_table[c];
141	if ((uint_t)c >= ZIO_COMPRESS_FUNCTIONS || ci->ci_decompress == NULL)
142		return (SET_ERROR(EINVAL));
143
144	return (ci->ci_decompress(src, dst, s_len, d_len, ci->ci_level));
145}
146
147void *zio_decompress_failed_buf;
148
149int
150zio_decompress_data(enum zio_compress c, abd_t *src, void *dst,
151    size_t s_len, size_t d_len)
152{
153	void *tmp = abd_borrow_buf_copy(src, s_len);
154	int ret = zio_decompress_data_buf(c, tmp, dst, s_len, d_len);
155
156	/*
157	 * Decompression shouldn't fail, because we've already verified
158	 * the checksum.  However, for extra protection (e.g. against bitflips
159	 * in non-ECC RAM), we handle this error (and test it).
160	 */
161	if (ret != 0) {
162		zio_decompress_failed_buf = kmem_alloc(s_len, KM_SLEEP);
163		bcopy(tmp, zio_decompress_failed_buf, s_len);
164		panic("decompression failed "
165		    "err=%u c=%u buf=%p s_len=%u d_len=%u",
166		    ret, (int)c, zio_decompress_failed_buf,
167		    (int)s_len, (int)d_len);
168	}
169
170	abd_return_buf(src, tmp, s_len);
171
172	if (zio_decompress_fail_fraction != 0 &&
173	    spa_get_random(zio_decompress_fail_fraction) == 0)
174		ret = SET_ERROR(EINVAL);
175
176	return (ret);
177}
178