1/*
2 * CDDL HEADER START
3 *
4 * This file and its contents are supplied under the terms of the
5 * Common Development and Distribution License ("CDDL"), version 1.0.
6 * You may only use this file in accordance with the terms of version
7 * 1.0 of the CDDL.
8 *
9 * A full copy of the text of the CDDL should have accompanied this
10 * source.  A copy of the CDDL is also available via the Internet at
11 * http://www.illumos.org/license/CDDL.
12 *
13 * CDDL HEADER END
14 */
15
16/*
17 * Copyright (c) 2017, Datto, Inc. All rights reserved.
18 */
19
20#include <sys/zio_crypt.h>
21#include <sys/dmu.h>
22#include <sys/dmu_objset.h>
23#include <sys/dnode.h>
24#include <sys/fs/zfs.h>
25#include <sys/zio.h>
26#include <sys/zil.h>
27#include <sys/sha2.h>
28#include <sys/hkdf.h>
29
30/*
31 * This file is responsible for handling all of the details of generating
32 * encryption parameters and performing encryption and authentication.
33 *
34 * BLOCK ENCRYPTION PARAMETERS:
35 * Encryption /Authentication Algorithm Suite (crypt):
36 * The encryption algorithm, mode, and key length we are going to use. We
37 * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit
38 * keys. All authentication is currently done with SHA512-HMAC.
39 *
40 * Plaintext:
41 * The unencrypted data that we want to encrypt.
42 *
43 * Initialization Vector (IV):
44 * An initialization vector for the encryption algorithms. This is used to
45 * "tweak" the encryption algorithms so that two blocks of the same data are
46 * encrypted into different ciphertext outputs, thus obfuscating block patterns.
47 * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is
48 * never reused with the same encryption key. This value is stored unencrypted
49 * and must simply be provided to the decryption function. We use a 96 bit IV
50 * (as recommended by NIST) for all block encryption. For non-dedup blocks we
51 * derive the IV randomly. The first 64 bits of the IV are stored in the second
52 * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of
53 * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits
54 * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count
55 * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of
56 * level 0 blocks is the number of allocated dnodes in that block. The on-disk
57 * format supports at most 2^15 slots per L0 dnode block, because the maximum
58 * block size is 16MB (2^24). In either case, for level 0 blocks this number
59 * will still be smaller than UINT32_MAX so it is safe to store the IV in the
60 * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count
61 * for the dnode code.
62 *
63 * Master key:
64 * This is the most important secret data of an encrypted dataset. It is used
65 * along with the salt to generate that actual encryption keys via HKDF. We
66 * do not use the master key to directly encrypt any data because there are
67 * theoretical limits on how much data can actually be safely encrypted with
68 * any encryption mode. The master key is stored encrypted on disk with the
69 * user's wrapping key. Its length is determined by the encryption algorithm.
70 * For details on how this is stored see the block comment in dsl_crypt.c
71 *
72 * Salt:
73 * Used as an input to the HKDF function, along with the master key. We use a
74 * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt
75 * can be used for encrypting many blocks, so we cache the current salt and the
76 * associated derived key in zio_crypt_t so we do not need to derive it again
77 * needlessly.
78 *
79 * Encryption Key:
80 * A secret binary key, generated from an HKDF function used to encrypt and
81 * decrypt data.
82 *
83 * Message Authenication Code (MAC)
84 * The MAC is an output of authenticated encryption modes such as AES-GCM and
85 * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted
86 * data on disk and return garbage to the application. Effectively, it is a
87 * checksum that can not be reproduced by an attacker. We store the MAC in the
88 * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated
89 * regular checksum of the ciphertext which can be used for scrubbing.
90 *
91 * OBJECT AUTHENTICATION:
92 * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because
93 * they contain some info that always needs to be readable. To prevent this
94 * data from being altered, we authenticate this data using SHA512-HMAC. This
95 * will produce a MAC (similar to the one produced via encryption) which can
96 * be used to verify the object was not modified. HMACs do not require key
97 * rotation or IVs, so we can keep up to the full 3 copies of authenticated
98 * data.
99 *
100 * ZIL ENCRYPTION:
101 * ZIL blocks have their bp written to disk ahead of the associated data, so we
102 * cannot store the MAC there as we normally do. For these blocks the MAC is
103 * stored in the embedded checksum within the zil_chain_t header. The salt and
104 * IV are generated for the block on bp allocation instead of at encryption
105 * time. In addition, ZIL blocks have some pieces that must be left in plaintext
106 * for claiming even though all of the sensitive user data still needs to be
107 * encrypted. The function zio_crypt_init_uios_zil() handles parsing which
108 * pieces of the block need to be encrypted. All data that is not encrypted is
109 * authenticated using the AAD mechanisms that the supported encryption modes
110 * provide for. In order to preserve the semantics of the ZIL for encrypted
111 * datasets, the ZIL is not protected at the objset level as described below.
112 *
113 * DNODE ENCRYPTION:
114 * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left
115 * in plaintext for scrubbing and claiming, but the bonus buffers might contain
116 * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing
117 * which pieces of the block need to be encrypted. For more details about
118 * dnode authentication and encryption, see zio_crypt_init_uios_dnode().
119 *
120 * OBJECT SET AUTHENTICATION:
121 * Up to this point, everything we have encrypted and authenticated has been
122 * at level 0 (or -2 for the ZIL). If we did not do any further work the
123 * on-disk format would be susceptible to attacks that deleted or rearrannged
124 * the order of level 0 blocks. Ideally, the cleanest solution would be to
125 * maintain a tree of authentication MACs going up the bp tree. However, this
126 * presents a problem for raw sends. Send files do not send information about
127 * indirect blocks so there would be no convenient way to transfer the MACs and
128 * they cannot be recalculated on the receive side without the master key which
129 * would defeat one of the purposes of raw sends in the first place. Instead,
130 * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs
131 * from the level below. We also include some portable fields from blk_prop such
132 * as the lsize and compression algorithm to prevent the data from being
133 * misinterpretted.
134 *
135 * At the objset level, we maintain 2 seperate 256 bit MACs in the
136 * objset_phys_t. The first one is "portable" and is the logical root of the
137 * MAC tree maintianed in the metadnode's bps. The second, is "local" and is
138 * used as the root MAC for the user accounting objects, which are also not
139 * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload
140 * of the send file. The useraccounting code ensures that the useraccounting
141 * info is not present upon a receive, so the local MAC can simply be cleared
142 * out at that time. For more info about objset_phys_t authentication, see
143 * zio_crypt_do_objset_hmacs().
144 *
145 * CONSIDERATIONS FOR DEDUP:
146 * In order for dedup to work, blocks that we want to dedup with one another
147 * need to use the same IV and encryption key, so that they will have the same
148 * ciphertext. Normally, one should never reuse an IV with the same encryption
149 * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both
150 * blocks. In this case, however, since we are using the same plaindata as
151 * well all that we end up with is a duplicate of the original ciphertext we
152 * already had. As a result, an attacker with read access to the raw disk will
153 * be able to tell which blocks are the same but this information is given away
154 * by dedup anyway. In order to get the same IVs and encryption keys for
155 * equivalent blocks of data we use an HMAC of the plaindata. We use an HMAC
156 * here so that a reproducible checksum of the plaindata is never available to
157 * the attacker. The HMAC key is kept alongside the master key, encrypted on
158 * disk. The first 64 bits of the HMAC are used in place of the random salt, and
159 * the next 96 bits are used as the IV. As a result of this mechanism, dedup
160 * will only work within a clone family since encrypted dedup requires use of
161 * the same master and HMAC keys.
162 */
163
164/*
165 * After encrypting many blocks with the same key we may start to run up
166 * against the theoretical limits of how much data can securely be encrypted
167 * with a single key using the supported encryption modes. The most obvious
168 * limitation is that our risk of generating 2 equivalent 96 bit IVs increases
169 * the more IVs we generate (which both GCM and CCM modes strictly forbid).
170 * This risk actually grows surprisingly quickly over time according to the
171 * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have
172 * generated n IVs with a cryptographically secure RNG, the approximate
173 * probability p(n) of a collision is given as:
174 *
175 * p(n) ~= e^(-n*(n-1)/(2*(2^96)))
176 *
177 * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html]
178 *
179 * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion
180 * we must not write more than 398,065,730 blocks with the same encryption key.
181 * Therefore, we rotate our keys after 400,000,000 blocks have been written by
182 * generating a new random 64 bit salt for our HKDF encryption key generation
183 * function.
184 */
185#define	ZFS_KEY_MAX_SALT_USES_DEFAULT	400000000
186#define	ZFS_CURRENT_MAX_SALT_USES	\
187	(MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT))
188unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT;
189
190/*
191 * Set to a nonzero value to cause zio_do_crypt_uio() to fail 1/this many
192 * calls, to test decryption error handling code paths.
193 */
194uint64_t zio_decrypt_fail_fraction = 0;
195
196typedef struct blkptr_auth_buf {
197	uint64_t bab_prop;			/* blk_prop - portable mask */
198	uint8_t bab_mac[ZIO_DATA_MAC_LEN];	/* MAC from blk_cksum */
199	uint64_t bab_pad;			/* reserved for future use */
200} blkptr_auth_buf_t;
201
202zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = {
203	{"",			ZC_TYPE_NONE,	0,	"inherit"},
204	{"",			ZC_TYPE_NONE,	0,	"on"},
205	{"",			ZC_TYPE_NONE,	0,	"off"},
206	{SUN_CKM_AES_CCM,	ZC_TYPE_CCM,	16,	"aes-128-ccm"},
207	{SUN_CKM_AES_CCM,	ZC_TYPE_CCM,	24,	"aes-192-ccm"},
208	{SUN_CKM_AES_CCM,	ZC_TYPE_CCM,	32,	"aes-256-ccm"},
209	{SUN_CKM_AES_GCM,	ZC_TYPE_GCM,	16,	"aes-128-gcm"},
210	{SUN_CKM_AES_GCM,	ZC_TYPE_GCM,	24,	"aes-192-gcm"},
211	{SUN_CKM_AES_GCM,	ZC_TYPE_GCM,	32,	"aes-256-gcm"}
212};
213
214void
215zio_crypt_key_destroy(zio_crypt_key_t *key)
216{
217	rw_destroy(&key->zk_salt_lock);
218
219	/* free crypto templates */
220	crypto_destroy_ctx_template(key->zk_current_tmpl);
221	crypto_destroy_ctx_template(key->zk_hmac_tmpl);
222
223	/* zero out sensitive data */
224	bzero(key, sizeof (zio_crypt_key_t));
225}
226
227int
228zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key)
229{
230	int ret;
231	crypto_mechanism_t mech;
232	uint_t keydata_len;
233
234	ASSERT(key != NULL);
235	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
236
237	keydata_len = zio_crypt_table[crypt].ci_keylen;
238	bzero(key, sizeof (zio_crypt_key_t));
239
240	/* fill keydata buffers and salt with random data */
241	ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t));
242	if (ret != 0)
243		goto error;
244
245	ret = random_get_bytes(key->zk_master_keydata, keydata_len);
246	if (ret != 0)
247		goto error;
248
249	ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN);
250	if (ret != 0)
251		goto error;
252
253	ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN);
254	if (ret != 0)
255		goto error;
256
257	/* derive the current key from the master key */
258	ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
259	    key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata,
260	    keydata_len);
261	if (ret != 0)
262		goto error;
263
264	/* initialize keys for the ICP */
265	key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
266	key->zk_current_key.ck_data = key->zk_current_keydata;
267	key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
268
269	key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
270	key->zk_hmac_key.ck_data = &key->zk_hmac_key;
271	key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
272
273	/*
274	 * Initialize the crypto templates. It's ok if this fails because
275	 * this is just an optimization.
276	 */
277	mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
278	ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
279	    &key->zk_current_tmpl, KM_SLEEP);
280	if (ret != CRYPTO_SUCCESS)
281		key->zk_current_tmpl = NULL;
282
283	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
284	ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
285	    &key->zk_hmac_tmpl, KM_SLEEP);
286	if (ret != CRYPTO_SUCCESS)
287		key->zk_hmac_tmpl = NULL;
288
289	key->zk_crypt = crypt;
290	key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION;
291	key->zk_salt_count = 0;
292	rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL);
293
294	return (0);
295
296error:
297	zio_crypt_key_destroy(key);
298	return (ret);
299}
300
301static int
302zio_crypt_key_change_salt(zio_crypt_key_t *key)
303{
304	int ret = 0;
305	uint8_t salt[ZIO_DATA_SALT_LEN];
306	crypto_mechanism_t mech;
307	uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen;
308
309	/* generate a new salt */
310	ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN);
311	if (ret != 0)
312		goto error;
313
314	rw_enter(&key->zk_salt_lock, RW_WRITER);
315
316	/* someone beat us to the salt rotation, just unlock and return */
317	if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES)
318		goto out_unlock;
319
320	/* derive the current key from the master key and the new salt */
321	ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
322	    salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len);
323	if (ret != 0)
324		goto out_unlock;
325
326	/* assign the salt and reset the usage count */
327	bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN);
328	key->zk_salt_count = 0;
329
330	/* destroy the old context template and create the new one */
331	crypto_destroy_ctx_template(key->zk_current_tmpl);
332	ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
333	    &key->zk_current_tmpl, KM_SLEEP);
334	if (ret != CRYPTO_SUCCESS)
335		key->zk_current_tmpl = NULL;
336
337	rw_exit(&key->zk_salt_lock);
338
339	return (0);
340
341out_unlock:
342	rw_exit(&key->zk_salt_lock);
343error:
344	return (ret);
345}
346
347/* See comment above zfs_key_max_salt_uses definition for details */
348int
349zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt)
350{
351	int ret;
352	boolean_t salt_change;
353
354	rw_enter(&key->zk_salt_lock, RW_READER);
355
356	bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN);
357	salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >=
358	    ZFS_CURRENT_MAX_SALT_USES);
359
360	rw_exit(&key->zk_salt_lock);
361
362	if (salt_change) {
363		ret = zio_crypt_key_change_salt(key);
364		if (ret != 0)
365			goto error;
366	}
367
368	return (0);
369
370error:
371	return (ret);
372}
373
374void *failed_decrypt_buf;
375int failed_decrypt_size;
376
377/*
378 * This function handles all encryption and decryption in zfs. When
379 * encrypting it expects puio to reference the plaintext and cuio to
380 * reference the cphertext. cuio must have enough space for the
381 * ciphertext + room for a MAC. datalen should be the length of the
382 * plaintext / ciphertext alone.
383 */
384/* ARGSUSED */
385static int
386zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key,
387    crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen,
388    uio_t *puio, uio_t *cuio, uint8_t *authbuf, uint_t auth_len)
389{
390	int ret;
391	crypto_data_t plaindata, cipherdata;
392	CK_AES_CCM_PARAMS ccmp;
393	CK_AES_GCM_PARAMS gcmp;
394	crypto_mechanism_t mech;
395	zio_crypt_info_t crypt_info;
396	uint_t plain_full_len, maclen;
397
398	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
399	ASSERT3U(key->ck_format, ==, CRYPTO_KEY_RAW);
400
401	/* lookup the encryption info */
402	crypt_info = zio_crypt_table[crypt];
403
404	/* the mac will always be the last iovec_t in the cipher uio */
405	maclen = cuio->uio_iov[cuio->uio_iovcnt - 1].iov_len;
406
407	ASSERT(maclen <= ZIO_DATA_MAC_LEN);
408
409	/* setup encryption mechanism (same as crypt) */
410	mech.cm_type = crypto_mech2id(crypt_info.ci_mechname);
411
412	/*
413	 * Strangely, the ICP requires that plain_full_len must include
414	 * the MAC length when decrypting, even though the UIO does not
415	 * need to have the extra space allocated.
416	 */
417	if (encrypt) {
418		plain_full_len = datalen;
419	} else {
420		plain_full_len = datalen + maclen;
421	}
422
423	/*
424	 * setup encryption params (currently only AES CCM and AES GCM
425	 * are supported)
426	 */
427	if (crypt_info.ci_crypt_type == ZC_TYPE_CCM) {
428		ccmp.ulNonceSize = ZIO_DATA_IV_LEN;
429		ccmp.ulAuthDataSize = auth_len;
430		ccmp.authData = authbuf;
431		ccmp.ulMACSize = maclen;
432		ccmp.nonce = ivbuf;
433		ccmp.ulDataSize = plain_full_len;
434
435		mech.cm_param = (char *)(&ccmp);
436		mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS);
437	} else {
438		gcmp.ulIvLen = ZIO_DATA_IV_LEN;
439		gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN);
440		gcmp.ulAADLen = auth_len;
441		gcmp.pAAD = authbuf;
442		gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen);
443		gcmp.pIv = ivbuf;
444
445		mech.cm_param = (char *)(&gcmp);
446		mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS);
447	}
448
449	/* populate the cipher and plain data structs. */
450	plaindata.cd_format = CRYPTO_DATA_UIO;
451	plaindata.cd_offset = 0;
452	plaindata.cd_uio = puio;
453	plaindata.cd_miscdata = NULL;
454	plaindata.cd_length = plain_full_len;
455
456	cipherdata.cd_format = CRYPTO_DATA_UIO;
457	cipherdata.cd_offset = 0;
458	cipherdata.cd_uio = cuio;
459	cipherdata.cd_miscdata = NULL;
460	cipherdata.cd_length = datalen + maclen;
461
462	/* perform the actual encryption */
463	if (encrypt) {
464		ret = crypto_encrypt(&mech, &plaindata, key, tmpl, &cipherdata,
465		    NULL);
466		if (ret != CRYPTO_SUCCESS) {
467			ret = SET_ERROR(EIO);
468			goto error;
469		}
470	} else {
471		if (zio_decrypt_fail_fraction != 0 &&
472		    spa_get_random(zio_decrypt_fail_fraction) == 0) {
473			ret = CRYPTO_INVALID_MAC;
474		} else {
475			ret = crypto_decrypt(&mech, &cipherdata,
476			    key, tmpl, &plaindata, NULL);
477		}
478		if (ret != CRYPTO_SUCCESS) {
479			ASSERT3U(ret, ==, CRYPTO_INVALID_MAC);
480			ret = SET_ERROR(ECKSUM);
481			goto error;
482		}
483	}
484
485	return (0);
486
487error:
488	return (ret);
489}
490
491int
492zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv,
493    uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out)
494{
495	int ret;
496	uio_t puio, cuio;
497	uint64_t aad[3];
498	iovec_t plain_iovecs[2], cipher_iovecs[3];
499	uint64_t crypt = key->zk_crypt;
500	uint_t enc_len, keydata_len, aad_len;
501
502	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
503	ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW);
504
505	keydata_len = zio_crypt_table[crypt].ci_keylen;
506
507	/* generate iv for wrapping the master and hmac key */
508	ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN);
509	if (ret != 0)
510		goto error;
511
512	/* initialize uio_ts */
513	plain_iovecs[0].iov_base = (char *)key->zk_master_keydata;
514	plain_iovecs[0].iov_len = keydata_len;
515	plain_iovecs[1].iov_base = (char *)key->zk_hmac_keydata;
516	plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
517
518	cipher_iovecs[0].iov_base = (char *)keydata_out;
519	cipher_iovecs[0].iov_len = keydata_len;
520	cipher_iovecs[1].iov_base = (char *)hmac_keydata_out;
521	cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
522	cipher_iovecs[2].iov_base = (char *)mac;
523	cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN;
524
525	/*
526	 * Although we don't support writing to the old format, we do
527	 * support rewrapping the key so that the user can move and
528	 * quarantine datasets on the old format.
529	 */
530	if (key->zk_version == 0) {
531		aad_len = sizeof (uint64_t);
532		aad[0] = LE_64(key->zk_guid);
533	} else {
534		ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
535		aad_len = sizeof (uint64_t) * 3;
536		aad[0] = LE_64(key->zk_guid);
537		aad[1] = LE_64(crypt);
538		aad[2] = LE_64(key->zk_version);
539	}
540
541	enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN;
542	puio.uio_iov = plain_iovecs;
543	puio.uio_iovcnt = 2;
544	puio.uio_segflg = UIO_SYSSPACE;
545	cuio.uio_iov = cipher_iovecs;
546	cuio.uio_iovcnt = 3;
547	cuio.uio_segflg = UIO_SYSSPACE;
548
549	/* encrypt the keys and store the resulting ciphertext and mac */
550	ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len,
551	    &puio, &cuio, (uint8_t *)aad, aad_len);
552	if (ret != 0)
553		goto error;
554
555	return (0);
556
557error:
558	return (ret);
559}
560
561int
562zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version,
563    uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv,
564    uint8_t *mac, zio_crypt_key_t *key)
565{
566	int ret;
567	crypto_mechanism_t mech;
568	uio_t puio, cuio;
569	uint64_t aad[3];
570	iovec_t plain_iovecs[2], cipher_iovecs[3];
571	uint_t enc_len, keydata_len, aad_len;
572
573	ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
574	ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW);
575
576	rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL);
577	keydata_len = zio_crypt_table[crypt].ci_keylen;
578
579	/* initialize uio_ts */
580	plain_iovecs[0].iov_base = (char *)key->zk_master_keydata;
581	plain_iovecs[0].iov_len = keydata_len;
582	plain_iovecs[1].iov_base = (char *)key->zk_hmac_keydata;
583	plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
584
585	cipher_iovecs[0].iov_base = (char *)keydata;
586	cipher_iovecs[0].iov_len = keydata_len;
587	cipher_iovecs[1].iov_base = (char *)hmac_keydata;
588	cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
589	cipher_iovecs[2].iov_base = (char *)mac;
590	cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN;
591
592	if (version == 0) {
593		aad_len = sizeof (uint64_t);
594		aad[0] = LE_64(guid);
595	} else {
596		ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
597		aad_len = sizeof (uint64_t) * 3;
598		aad[0] = LE_64(guid);
599		aad[1] = LE_64(crypt);
600		aad[2] = LE_64(version);
601	}
602
603	enc_len = keydata_len + SHA512_HMAC_KEYLEN;
604	puio.uio_iov = plain_iovecs;
605	puio.uio_segflg = UIO_SYSSPACE;
606	puio.uio_iovcnt = 2;
607	cuio.uio_iov = cipher_iovecs;
608	cuio.uio_iovcnt = 3;
609	cuio.uio_segflg = UIO_SYSSPACE;
610
611	/* decrypt the keys and store the result in the output buffers */
612	ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len,
613	    &puio, &cuio, (uint8_t *)aad, aad_len);
614	if (ret != 0)
615		goto error;
616
617	/* generate a fresh salt */
618	ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN);
619	if (ret != 0)
620		goto error;
621
622	/* derive the current key from the master key */
623	ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
624	    key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata,
625	    keydata_len);
626	if (ret != 0)
627		goto error;
628
629	/* initialize keys for ICP */
630	key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
631	key->zk_current_key.ck_data = key->zk_current_keydata;
632	key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
633
634	key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
635	key->zk_hmac_key.ck_data = key->zk_hmac_keydata;
636	key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
637
638	/*
639	 * Initialize the crypto templates. It's ok if this fails because
640	 * this is just an optimization.
641	 */
642	mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
643	ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
644	    &key->zk_current_tmpl, KM_SLEEP);
645	if (ret != CRYPTO_SUCCESS)
646		key->zk_current_tmpl = NULL;
647
648	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
649	ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
650	    &key->zk_hmac_tmpl, KM_SLEEP);
651	if (ret != CRYPTO_SUCCESS)
652		key->zk_hmac_tmpl = NULL;
653
654	key->zk_crypt = crypt;
655	key->zk_version = version;
656	key->zk_guid = guid;
657	key->zk_salt_count = 0;
658
659	return (0);
660
661error:
662	zio_crypt_key_destroy(key);
663	return (ret);
664}
665
666int
667zio_crypt_generate_iv(uint8_t *ivbuf)
668{
669	int ret;
670
671	/* randomly generate the IV */
672	ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN);
673	if (ret != 0)
674		goto error;
675
676	return (0);
677
678error:
679	bzero(ivbuf, ZIO_DATA_IV_LEN);
680	return (ret);
681}
682
683int
684zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen,
685    uint8_t *digestbuf, uint_t digestlen)
686{
687	int ret;
688	crypto_mechanism_t mech;
689	crypto_data_t in_data, digest_data;
690	uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH];
691
692	ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH);
693
694	/* initialize sha512-hmac mechanism and crypto data */
695	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
696	mech.cm_param = NULL;
697	mech.cm_param_len = 0;
698
699	/* initialize the crypto data */
700	in_data.cd_format = CRYPTO_DATA_RAW;
701	in_data.cd_offset = 0;
702	in_data.cd_length = datalen;
703	in_data.cd_raw.iov_base = (char *)data;
704	in_data.cd_raw.iov_len = in_data.cd_length;
705
706	digest_data.cd_format = CRYPTO_DATA_RAW;
707	digest_data.cd_offset = 0;
708	digest_data.cd_length = SHA512_DIGEST_LENGTH;
709	digest_data.cd_raw.iov_base = (char *)raw_digestbuf;
710	digest_data.cd_raw.iov_len = digest_data.cd_length;
711
712	/* generate the hmac */
713	ret = crypto_mac(&mech, &in_data, &key->zk_hmac_key, key->zk_hmac_tmpl,
714	    &digest_data, NULL);
715	if (ret != CRYPTO_SUCCESS) {
716		ret = SET_ERROR(EIO);
717		goto error;
718	}
719
720	bcopy(raw_digestbuf, digestbuf, digestlen);
721
722	return (0);
723
724error:
725	bzero(digestbuf, digestlen);
726	return (ret);
727}
728
729int
730zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data,
731    uint_t datalen, uint8_t *ivbuf, uint8_t *salt)
732{
733	int ret;
734	uint8_t digestbuf[SHA512_DIGEST_LENGTH];
735
736	ret = zio_crypt_do_hmac(key, data, datalen,
737	    digestbuf, SHA512_DIGEST_LENGTH);
738	if (ret != 0)
739		return (ret);
740
741	bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN);
742	bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN);
743
744	return (0);
745}
746
747/*
748 * The following functions are used to encode and decode encryption parameters
749 * into blkptr_t and zil_header_t. The ICP wants to use these parameters as
750 * byte strings, which normally means that these strings would not need to deal
751 * with byteswapping at all. However, both blkptr_t and zil_header_t may be
752 * byteswapped by lower layers and so we must "undo" that byteswap here upon
753 * decoding and encoding in a non-native byteorder. These functions require
754 * that the byteorder bit is correct before being called.
755 */
756void
757zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv)
758{
759	uint64_t val64;
760	uint32_t val32;
761
762	ASSERT(BP_IS_ENCRYPTED(bp));
763
764	if (!BP_SHOULD_BYTESWAP(bp)) {
765		bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t));
766		bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t));
767		bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t));
768		BP_SET_IV2(bp, val32);
769	} else {
770		bcopy(salt, &val64, sizeof (uint64_t));
771		bp->blk_dva[2].dva_word[0] = BSWAP_64(val64);
772
773		bcopy(iv, &val64, sizeof (uint64_t));
774		bp->blk_dva[2].dva_word[1] = BSWAP_64(val64);
775
776		bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t));
777		BP_SET_IV2(bp, BSWAP_32(val32));
778	}
779}
780
781void
782zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv)
783{
784	uint64_t val64;
785	uint32_t val32;
786
787	ASSERT(BP_IS_PROTECTED(bp));
788
789	/* for convenience, so callers don't need to check */
790	if (BP_IS_AUTHENTICATED(bp)) {
791		bzero(salt, ZIO_DATA_SALT_LEN);
792		bzero(iv, ZIO_DATA_IV_LEN);
793		return;
794	}
795
796	if (!BP_SHOULD_BYTESWAP(bp)) {
797		bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t));
798		bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t));
799
800		val32 = (uint32_t)BP_GET_IV2(bp);
801		bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t));
802	} else {
803		val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]);
804		bcopy(&val64, salt, sizeof (uint64_t));
805
806		val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]);
807		bcopy(&val64, iv, sizeof (uint64_t));
808
809		val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp));
810		bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t));
811	}
812}
813
814void
815zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac)
816{
817	uint64_t val64;
818
819	ASSERT(BP_USES_CRYPT(bp));
820	ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET);
821
822	if (!BP_SHOULD_BYTESWAP(bp)) {
823		bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t));
824		bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3],
825		    sizeof (uint64_t));
826	} else {
827		bcopy(mac, &val64, sizeof (uint64_t));
828		bp->blk_cksum.zc_word[2] = BSWAP_64(val64);
829
830		bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t));
831		bp->blk_cksum.zc_word[3] = BSWAP_64(val64);
832	}
833}
834
835void
836zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac)
837{
838	uint64_t val64;
839
840	ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp));
841
842	/* for convenience, so callers don't need to check */
843	if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
844		bzero(mac, ZIO_DATA_MAC_LEN);
845		return;
846	}
847
848	if (!BP_SHOULD_BYTESWAP(bp)) {
849		bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t));
850		bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t),
851		    sizeof (uint64_t));
852	} else {
853		val64 = BSWAP_64(bp->blk_cksum.zc_word[2]);
854		bcopy(&val64, mac, sizeof (uint64_t));
855
856		val64 = BSWAP_64(bp->blk_cksum.zc_word[3]);
857		bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t));
858	}
859}
860
861void
862zio_crypt_encode_mac_zil(void *data, uint8_t *mac)
863{
864	zil_chain_t *zilc = data;
865
866	bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t));
867	bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3],
868	    sizeof (uint64_t));
869}
870
871void
872zio_crypt_decode_mac_zil(const void *data, uint8_t *mac)
873{
874	/*
875	 * The ZIL MAC is embedded in the block it protects, which will
876	 * not have been byteswapped by the time this function has been called.
877	 * As a result, we don't need to worry about byteswapping the MAC.
878	 */
879	const zil_chain_t *zilc = data;
880
881	bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t));
882	bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t),
883	    sizeof (uint64_t));
884}
885
886/*
887 * This routine takes a block of dnodes (src_abd) and copies only the bonus
888 * buffers to the same offsets in the dst buffer. datalen should be the size
889 * of both the src_abd and the dst buffer (not just the length of the bonus
890 * buffers).
891 */
892void
893zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen)
894{
895	uint_t i, max_dnp = datalen >> DNODE_SHIFT;
896	uint8_t *src;
897	dnode_phys_t *dnp, *sdnp, *ddnp;
898
899	src = abd_borrow_buf_copy(src_abd, datalen);
900
901	sdnp = (dnode_phys_t *)src;
902	ddnp = (dnode_phys_t *)dst;
903
904	for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
905		dnp = &sdnp[i];
906		if (dnp->dn_type != DMU_OT_NONE &&
907		    DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) &&
908		    dnp->dn_bonuslen != 0) {
909			bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]),
910			    DN_MAX_BONUS_LEN(dnp));
911		}
912	}
913
914	abd_return_buf(src_abd, src, datalen);
915}
916
917/*
918 * This function decides what fields from blk_prop are included in
919 * the on-disk various MAC algorithms.
920 */
921static void
922zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version)
923{
924	/*
925	 * Version 0 did not properly zero out all non-portable fields
926	 * as it should have done. We maintain this code so that we can
927	 * do read-only imports of pools on this version.
928	 */
929	if (version == 0) {
930		BP_SET_DEDUP(bp, 0);
931		BP_SET_CHECKSUM(bp, 0);
932		BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE);
933		return;
934	}
935
936	ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
937
938	/*
939	 * The hole_birth feature might set these fields even if this bp
940	 * is a hole. We zero them out here to guarantee that raw sends
941	 * will function with or without the feature.
942	 */
943	if (BP_IS_HOLE(bp)) {
944		bp->blk_prop = 0ULL;
945		return;
946	}
947
948	/*
949	 * At L0 we want to verify these fields to ensure that data blocks
950	 * can not be reinterpretted. For instance, we do not want an attacker
951	 * to trick us into returning raw lz4 compressed data to the user
952	 * by modifying the compression bits. At higher levels, we cannot
953	 * enforce this policy since raw sends do not convey any information
954	 * about indirect blocks, so these values might be different on the
955	 * receive side. Fortunately, this does not open any new attack
956	 * vectors, since any alterations that can be made to a higher level
957	 * bp must still verify the correct order of the layer below it.
958	 */
959	if (BP_GET_LEVEL(bp) != 0) {
960		BP_SET_BYTEORDER(bp, 0);
961		BP_SET_COMPRESS(bp, 0);
962
963		/*
964		 * psize cannot be set to zero or it will trigger
965		 * asserts, but the value doesn't really matter as
966		 * long as it is constant.
967		 */
968		BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE);
969	}
970
971	BP_SET_DEDUP(bp, 0);
972	BP_SET_CHECKSUM(bp, 0);
973}
974
975static void
976zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp,
977    blkptr_auth_buf_t *bab, uint_t *bab_len)
978{
979	blkptr_t tmpbp = *bp;
980
981	if (should_bswap)
982		byteswap_uint64_array(&tmpbp, sizeof (blkptr_t));
983
984	ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp));
985	ASSERT0(BP_IS_EMBEDDED(&tmpbp));
986
987	zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac);
988
989	/*
990	 * We always MAC blk_prop in LE to ensure portability. This
991	 * must be done after decoding the mac, since the endianness
992	 * will get zero'd out here.
993	 */
994	zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version);
995	bab->bab_prop = LE_64(tmpbp.blk_prop);
996	bab->bab_pad = 0ULL;
997
998	/* version 0 did not include the padding */
999	*bab_len = sizeof (blkptr_auth_buf_t);
1000	if (version == 0)
1001		*bab_len -= sizeof (uint64_t);
1002}
1003
1004static int
1005zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version,
1006    boolean_t should_bswap, blkptr_t *bp)
1007{
1008	int ret;
1009	uint_t bab_len;
1010	blkptr_auth_buf_t bab;
1011	crypto_data_t cd;
1012
1013	zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1014	cd.cd_format = CRYPTO_DATA_RAW;
1015	cd.cd_offset = 0;
1016	cd.cd_length = bab_len;
1017	cd.cd_raw.iov_base = (char *)&bab;
1018	cd.cd_raw.iov_len = cd.cd_length;
1019
1020	ret = crypto_mac_update(ctx, &cd, NULL);
1021	if (ret != CRYPTO_SUCCESS) {
1022		ret = SET_ERROR(EIO);
1023		goto error;
1024	}
1025
1026	return (0);
1027
1028error:
1029	return (ret);
1030}
1031
1032static void
1033zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version,
1034    boolean_t should_bswap, blkptr_t *bp)
1035{
1036	uint_t bab_len;
1037	blkptr_auth_buf_t bab;
1038
1039	zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1040	SHA2Update(ctx, &bab, bab_len);
1041}
1042
1043static void
1044zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version,
1045    boolean_t should_bswap, blkptr_t *bp)
1046{
1047	uint_t bab_len;
1048	blkptr_auth_buf_t bab;
1049
1050	zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1051	bcopy(&bab, *aadp, bab_len);
1052	*aadp += bab_len;
1053	*aad_len += bab_len;
1054}
1055
1056static int
1057zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version,
1058    boolean_t should_bswap, dnode_phys_t *dnp)
1059{
1060	int ret, i;
1061	dnode_phys_t *adnp;
1062	boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER);
1063	crypto_data_t cd;
1064	uint8_t tmp_dncore[offsetof(dnode_phys_t, dn_blkptr)];
1065
1066	cd.cd_format = CRYPTO_DATA_RAW;
1067	cd.cd_offset = 0;
1068
1069	/* authenticate the core dnode (masking out non-portable bits) */
1070	bcopy(dnp, tmp_dncore, sizeof (tmp_dncore));
1071	adnp = (dnode_phys_t *)tmp_dncore;
1072	if (le_bswap) {
1073		adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec);
1074		adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen);
1075		adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid);
1076		adnp->dn_used = BSWAP_64(adnp->dn_used);
1077	}
1078	adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK;
1079	adnp->dn_used = 0;
1080
1081	cd.cd_length = sizeof (tmp_dncore);
1082	cd.cd_raw.iov_base = (char *)adnp;
1083	cd.cd_raw.iov_len = cd.cd_length;
1084
1085	ret = crypto_mac_update(ctx, &cd, NULL);
1086	if (ret != CRYPTO_SUCCESS) {
1087		ret = SET_ERROR(EIO);
1088		goto error;
1089	}
1090
1091	for (i = 0; i < dnp->dn_nblkptr; i++) {
1092		ret = zio_crypt_bp_do_hmac_updates(ctx, version,
1093		    should_bswap, &dnp->dn_blkptr[i]);
1094		if (ret != 0)
1095			goto error;
1096	}
1097
1098	if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1099		ret = zio_crypt_bp_do_hmac_updates(ctx, version,
1100		    should_bswap, DN_SPILL_BLKPTR(dnp));
1101		if (ret != 0)
1102			goto error;
1103	}
1104
1105	return (0);
1106
1107error:
1108	return (ret);
1109}
1110
1111/*
1112 * objset_phys_t blocks introduce a number of exceptions to the normal
1113 * authentication process. objset_phys_t's contain 2 seperate HMACS for
1114 * protecting the integrity of their data. The portable_mac protects the
1115 * the metadnode. This MAC can be sent with a raw send and protects against
1116 * reordering of data within the metadnode. The local_mac protects the user
1117 * accounting objects which are not sent from one system to another.
1118 *
1119 * In addition, objset blocks are the only blocks that can be modified and
1120 * written to disk without the key loaded under certain circumstances. During
1121 * zil_claim() we need to be able to update the zil_header_t to complete
1122 * claiming log blocks and during raw receives we need to write out the
1123 * portable_mac from the send file. Both of these actions are possible
1124 * because these fields are not protected by either MAC so neither one will
1125 * need to modify the MACs without the key. However, when the modified blocks
1126 * are written out they will be byteswapped into the host machine's native
1127 * endianness which will modify fields protected by the MAC. As a result, MAC
1128 * calculation for objset blocks works slightly differently from other block
1129 * types. Where other block types MAC the data in whatever endianness is
1130 * written to disk, objset blocks always MAC little endian version of their
1131 * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP()
1132 * and le_bswap indicates whether a byteswap is needed to get this block
1133 * into little endian format.
1134 */
1135/* ARGSUSED */
1136int
1137zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
1138    boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac)
1139{
1140	int ret;
1141	crypto_mechanism_t mech;
1142	crypto_context_t ctx;
1143	crypto_data_t cd;
1144	objset_phys_t *osp = data;
1145	uint64_t intval;
1146	boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER);
1147	uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH];
1148	uint8_t raw_local_mac[SHA512_DIGEST_LENGTH];
1149
1150	/* initialize HMAC mechanism */
1151	mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
1152	mech.cm_param = NULL;
1153	mech.cm_param_len = 0;
1154
1155	cd.cd_format = CRYPTO_DATA_RAW;
1156	cd.cd_offset = 0;
1157
1158	/* calculate the portable MAC from the portable fields and metadnode */
1159	ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL);
1160	if (ret != CRYPTO_SUCCESS) {
1161		ret = SET_ERROR(EIO);
1162		goto error;
1163	}
1164
1165	/* add in the os_type */
1166	intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type);
1167	cd.cd_length = sizeof (uint64_t);
1168	cd.cd_raw.iov_base = (char *)&intval;
1169	cd.cd_raw.iov_len = cd.cd_length;
1170
1171	ret = crypto_mac_update(ctx, &cd, NULL);
1172	if (ret != CRYPTO_SUCCESS) {
1173		ret = SET_ERROR(EIO);
1174		goto error;
1175	}
1176
1177	/* add in the portable os_flags */
1178	intval = osp->os_flags;
1179	if (should_bswap)
1180		intval = BSWAP_64(intval);
1181	intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK;
1182	/* CONSTCOND */
1183	if (!ZFS_HOST_BYTEORDER)
1184		intval = BSWAP_64(intval);
1185
1186	cd.cd_length = sizeof (uint64_t);
1187	cd.cd_raw.iov_base = (char *)&intval;
1188	cd.cd_raw.iov_len = cd.cd_length;
1189
1190	ret = crypto_mac_update(ctx, &cd, NULL);
1191	if (ret != CRYPTO_SUCCESS) {
1192		ret = SET_ERROR(EIO);
1193		goto error;
1194	}
1195
1196	/* add in fields from the metadnode */
1197	ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1198	    should_bswap, &osp->os_meta_dnode);
1199	if (ret)
1200		goto error;
1201
1202	/* store the final digest in a temporary buffer and copy what we need */
1203	cd.cd_length = SHA512_DIGEST_LENGTH;
1204	cd.cd_raw.iov_base = (char *)raw_portable_mac;
1205	cd.cd_raw.iov_len = cd.cd_length;
1206
1207	ret = crypto_mac_final(ctx, &cd, NULL);
1208	if (ret != CRYPTO_SUCCESS) {
1209		ret = SET_ERROR(EIO);
1210		goto error;
1211	}
1212
1213	bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN);
1214
1215	/*
1216	 * The local MAC protects the user, group and project accounting.
1217	 * If these objects are not present, the local MAC is zeroed out.
1218	 */
1219	if ((datalen >= OBJSET_PHYS_SIZE_V3 &&
1220	    osp->os_userused_dnode.dn_type == DMU_OT_NONE &&
1221	    osp->os_groupused_dnode.dn_type == DMU_OT_NONE &&
1222	    osp->os_projectused_dnode.dn_type == DMU_OT_NONE) ||
1223	    (datalen >= OBJSET_PHYS_SIZE_V2 &&
1224	    osp->os_userused_dnode.dn_type == DMU_OT_NONE &&
1225	    osp->os_groupused_dnode.dn_type == DMU_OT_NONE) ||
1226	    (datalen <= OBJSET_PHYS_SIZE_V1)) {
1227		bzero(local_mac, ZIO_OBJSET_MAC_LEN);
1228		return (0);
1229	}
1230
1231	/* calculate the local MAC from the userused and groupused dnodes */
1232	ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL);
1233	if (ret != CRYPTO_SUCCESS) {
1234		ret = SET_ERROR(EIO);
1235		goto error;
1236	}
1237
1238	/* add in the non-portable os_flags */
1239	intval = osp->os_flags;
1240	if (should_bswap)
1241		intval = BSWAP_64(intval);
1242	intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK;
1243	/* CONSTCOND */
1244	if (!ZFS_HOST_BYTEORDER)
1245		intval = BSWAP_64(intval);
1246
1247	cd.cd_length = sizeof (uint64_t);
1248	cd.cd_raw.iov_base = (char *)&intval;
1249	cd.cd_raw.iov_len = cd.cd_length;
1250
1251	ret = crypto_mac_update(ctx, &cd, NULL);
1252	if (ret != CRYPTO_SUCCESS) {
1253		ret = SET_ERROR(EIO);
1254		goto error;
1255	}
1256
1257	/* add in fields from the user accounting dnodes */
1258	ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1259	    should_bswap, &osp->os_userused_dnode);
1260	if (ret)
1261		goto error;
1262
1263	ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1264	    should_bswap, &osp->os_groupused_dnode);
1265	if (ret)
1266		goto error;
1267
1268	/* store the final digest in a temporary buffer and copy what we need */
1269	cd.cd_length = SHA512_DIGEST_LENGTH;
1270	cd.cd_raw.iov_base = (char *)raw_local_mac;
1271	cd.cd_raw.iov_len = cd.cd_length;
1272
1273	ret = crypto_mac_final(ctx, &cd, NULL);
1274	if (ret != CRYPTO_SUCCESS) {
1275		ret = SET_ERROR(EIO);
1276		goto error;
1277	}
1278
1279	bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN);
1280
1281	return (0);
1282
1283error:
1284	bzero(portable_mac, ZIO_OBJSET_MAC_LEN);
1285	bzero(local_mac, ZIO_OBJSET_MAC_LEN);
1286	return (ret);
1287}
1288
1289static void
1290zio_crypt_destroy_uio(uio_t *uio)
1291{
1292	if (uio->uio_iov)
1293		kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t));
1294}
1295
1296/*
1297 * This function parses an uncompressed indirect block and returns a checksum
1298 * of all the portable fields from all of the contained bps. The portable
1299 * fields are the MAC and all of the fields from blk_prop except for the dedup,
1300 * checksum, and psize bits. For an explanation of the purpose of this, see
1301 * the comment block on object set authentication.
1302 */
1303static int
1304zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf,
1305    uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum)
1306{
1307	blkptr_t *bp;
1308	int i, epb = datalen >> SPA_BLKPTRSHIFT;
1309	SHA2_CTX ctx;
1310	uint8_t digestbuf[SHA512_DIGEST_LENGTH];
1311
1312	/* checksum all of the MACs from the layer below */
1313	SHA2Init(SHA512, &ctx);
1314	for (i = 0, bp = buf; i < epb; i++, bp++) {
1315		zio_crypt_bp_do_indrect_checksum_updates(&ctx, version,
1316		    byteswap, bp);
1317	}
1318	SHA2Final(digestbuf, &ctx);
1319
1320	if (generate) {
1321		bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN);
1322		return (0);
1323	}
1324
1325	if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0)
1326		return (SET_ERROR(ECKSUM));
1327
1328	return (0);
1329}
1330
1331int
1332zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf,
1333    uint_t datalen, boolean_t byteswap, uint8_t *cksum)
1334{
1335	int ret;
1336
1337	/*
1338	 * Unfortunately, callers of this function will not always have
1339	 * easy access to the on-disk format version. This info is
1340	 * normally found in the DSL Crypto Key, but the checksum-of-MACs
1341	 * is expected to be verifiable even when the key isn't loaded.
1342	 * Here, instead of doing a ZAP lookup for the version for each
1343	 * zio, we simply try both existing formats.
1344	 */
1345	ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf,
1346	    datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum);
1347	if (ret == ECKSUM) {
1348		ASSERT(!generate);
1349		ret = zio_crypt_do_indirect_mac_checksum_impl(generate,
1350		    buf, datalen, 0, byteswap, cksum);
1351	}
1352
1353	return (ret);
1354}
1355
1356int
1357zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd,
1358    uint_t datalen, boolean_t byteswap, uint8_t *cksum)
1359{
1360	int ret;
1361	void *buf;
1362
1363	buf = abd_borrow_buf_copy(abd, datalen);
1364	ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen,
1365	    byteswap, cksum);
1366	abd_return_buf(abd, buf, datalen);
1367
1368	return (ret);
1369}
1370
1371/*
1372 * Special case handling routine for encrypting / decrypting ZIL blocks.
1373 * We do not check for the older ZIL chain because the encryption feature
1374 * was not available before the newer ZIL chain was introduced. The goal
1375 * here is to encrypt everything except the blkptr_t of a lr_write_t and
1376 * the zil_chain_t header. Everything that is not encrypted is authenticated.
1377 */
1378
1379/* ARGSUSED */
1380static int
1381zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf,
1382    uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, uio_t *puio,
1383    uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len,
1384    boolean_t *no_crypt)
1385{
1386	int ret;
1387	uint64_t txtype, lr_len;
1388	uint_t nr_src, nr_dst, crypt_len;
1389	uint_t aad_len = 0, nr_iovecs = 0, total_len = 0;
1390	iovec_t *src_iovecs = NULL, *dst_iovecs = NULL;
1391	uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp;
1392	zil_chain_t *zilc;
1393	lr_t *lr;
1394	uint8_t *aadbuf = zio_buf_alloc(datalen);
1395
1396	/* cipherbuf always needs an extra iovec for the MAC */
1397	if (encrypt) {
1398		src = plainbuf;
1399		dst = cipherbuf;
1400		nr_src = 0;
1401		nr_dst = 1;
1402	} else {
1403		src = cipherbuf;
1404		dst = plainbuf;
1405		nr_src = 1;
1406		nr_dst = 0;
1407	}
1408
1409	/* find the start and end record of the log block */
1410	zilc = (zil_chain_t *)src;
1411	slrp = src + sizeof (zil_chain_t);
1412	aadp = aadbuf;
1413	blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused);
1414
1415	/* calculate the number of encrypted iovecs we will need */
1416	for (; slrp < blkend; slrp += lr_len) {
1417		lr = (lr_t *)slrp;
1418
1419		if (!byteswap) {
1420			txtype = lr->lrc_txtype;
1421			lr_len = lr->lrc_reclen;
1422		} else {
1423			txtype = BSWAP_64(lr->lrc_txtype);
1424			lr_len = BSWAP_64(lr->lrc_reclen);
1425		}
1426
1427		nr_iovecs++;
1428		if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t))
1429			nr_iovecs++;
1430	}
1431
1432	nr_src += nr_iovecs;
1433	nr_dst += nr_iovecs;
1434
1435	/* allocate the iovec arrays */
1436	if (nr_src != 0) {
1437		src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
1438		if (src_iovecs == NULL) {
1439			ret = SET_ERROR(ENOMEM);
1440			goto error;
1441		}
1442	}
1443
1444	if (nr_dst != 0) {
1445		dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
1446		if (dst_iovecs == NULL) {
1447			ret = SET_ERROR(ENOMEM);
1448			goto error;
1449		}
1450	}
1451
1452	/*
1453	 * Copy the plain zil header over and authenticate everything except
1454	 * the checksum that will store our MAC. If we are writing the data
1455	 * the embedded checksum will not have been calculated yet, so we don't
1456	 * authenticate that.
1457	 */
1458	bcopy(src, dst, sizeof (zil_chain_t));
1459	bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t));
1460	aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t);
1461	aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t);
1462
1463	/* loop over records again, filling in iovecs */
1464	nr_iovecs = 0;
1465	slrp = src + sizeof (zil_chain_t);
1466	dlrp = dst + sizeof (zil_chain_t);
1467
1468	for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) {
1469		lr = (lr_t *)slrp;
1470
1471		if (!byteswap) {
1472			txtype = lr->lrc_txtype;
1473			lr_len = lr->lrc_reclen;
1474		} else {
1475			txtype = BSWAP_64(lr->lrc_txtype);
1476			lr_len = BSWAP_64(lr->lrc_reclen);
1477		}
1478
1479		/* copy the common lr_t */
1480		bcopy(slrp, dlrp, sizeof (lr_t));
1481		bcopy(slrp, aadp, sizeof (lr_t));
1482		aadp += sizeof (lr_t);
1483		aad_len += sizeof (lr_t);
1484
1485		ASSERT3P(src_iovecs, !=, NULL);
1486		ASSERT3P(dst_iovecs, !=, NULL);
1487
1488		/*
1489		 * If this is a TX_WRITE record we want to encrypt everything
1490		 * except the bp if exists. If the bp does exist we want to
1491		 * authenticate it.
1492		 */
1493		if (txtype == TX_WRITE) {
1494			crypt_len = sizeof (lr_write_t) -
1495			    sizeof (lr_t) - sizeof (blkptr_t);
1496			src_iovecs[nr_iovecs].iov_base = (char *)slrp +
1497			    sizeof (lr_t);
1498			src_iovecs[nr_iovecs].iov_len = crypt_len;
1499			dst_iovecs[nr_iovecs].iov_base = (char *)dlrp +
1500			    sizeof (lr_t);
1501			dst_iovecs[nr_iovecs].iov_len = crypt_len;
1502
1503			/* copy the bp now since it will not be encrypted */
1504			bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1505			    dlrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1506			    sizeof (blkptr_t));
1507			bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1508			    aadp, sizeof (blkptr_t));
1509			aadp += sizeof (blkptr_t);
1510			aad_len += sizeof (blkptr_t);
1511			nr_iovecs++;
1512			total_len += crypt_len;
1513
1514			if (lr_len != sizeof (lr_write_t)) {
1515				crypt_len = lr_len - sizeof (lr_write_t);
1516				src_iovecs[nr_iovecs].iov_base = (char *)
1517				    slrp + sizeof (lr_write_t);
1518				src_iovecs[nr_iovecs].iov_len = crypt_len;
1519				dst_iovecs[nr_iovecs].iov_base = (char *)
1520				    dlrp + sizeof (lr_write_t);
1521				dst_iovecs[nr_iovecs].iov_len = crypt_len;
1522				nr_iovecs++;
1523				total_len += crypt_len;
1524			}
1525		} else {
1526			crypt_len = lr_len - sizeof (lr_t);
1527			src_iovecs[nr_iovecs].iov_base = (char *)slrp +
1528			    sizeof (lr_t);
1529			src_iovecs[nr_iovecs].iov_len = crypt_len;
1530			dst_iovecs[nr_iovecs].iov_base = (char *)dlrp +
1531			    sizeof (lr_t);
1532			dst_iovecs[nr_iovecs].iov_len = crypt_len;
1533			nr_iovecs++;
1534			total_len += crypt_len;
1535		}
1536	}
1537
1538	*no_crypt = (nr_iovecs == 0);
1539	*enc_len = total_len;
1540	*authbuf = aadbuf;
1541	*auth_len = aad_len;
1542
1543	if (encrypt) {
1544		puio->uio_iov = src_iovecs;
1545		puio->uio_iovcnt = nr_src;
1546		cuio->uio_iov = dst_iovecs;
1547		cuio->uio_iovcnt = nr_dst;
1548	} else {
1549		puio->uio_iov = dst_iovecs;
1550		puio->uio_iovcnt = nr_dst;
1551		cuio->uio_iov = src_iovecs;
1552		cuio->uio_iovcnt = nr_src;
1553	}
1554
1555	return (0);
1556
1557error:
1558	zio_buf_free(aadbuf, datalen);
1559	if (src_iovecs != NULL)
1560		kmem_free(src_iovecs, nr_src * sizeof (iovec_t));
1561	if (dst_iovecs != NULL)
1562		kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t));
1563
1564	*enc_len = 0;
1565	*authbuf = NULL;
1566	*auth_len = 0;
1567	*no_crypt = B_FALSE;
1568	puio->uio_iov = NULL;
1569	puio->uio_iovcnt = 0;
1570	cuio->uio_iov = NULL;
1571	cuio->uio_iovcnt = 0;
1572	return (ret);
1573}
1574
1575/*
1576 * Special case handling routine for encrypting / decrypting dnode blocks.
1577 */
1578static int
1579zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version,
1580    uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap,
1581    uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf,
1582    uint_t *auth_len, boolean_t *no_crypt)
1583{
1584	int ret;
1585	uint_t nr_src, nr_dst, crypt_len;
1586	uint_t aad_len = 0, nr_iovecs = 0, total_len = 0;
1587	uint_t i, j, max_dnp = datalen >> DNODE_SHIFT;
1588	iovec_t *src_iovecs = NULL, *dst_iovecs = NULL;
1589	uint8_t *src, *dst, *aadp;
1590	dnode_phys_t *dnp, *adnp, *sdnp, *ddnp;
1591	uint8_t *aadbuf = zio_buf_alloc(datalen);
1592
1593	if (encrypt) {
1594		src = plainbuf;
1595		dst = cipherbuf;
1596		nr_src = 0;
1597		nr_dst = 1;
1598	} else {
1599		src = cipherbuf;
1600		dst = plainbuf;
1601		nr_src = 1;
1602		nr_dst = 0;
1603	}
1604
1605	sdnp = (dnode_phys_t *)src;
1606	ddnp = (dnode_phys_t *)dst;
1607	aadp = aadbuf;
1608
1609	/*
1610	 * Count the number of iovecs we will need to do the encryption by
1611	 * counting the number of bonus buffers that need to be encrypted.
1612	 */
1613	for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
1614		/*
1615		 * This block may still be byteswapped. However, all of the
1616		 * values we use are either uint8_t's (for which byteswapping
1617		 * is a noop) or a * != 0 check, which will work regardless
1618		 * of whether or not we byteswap.
1619		 */
1620		if (sdnp[i].dn_type != DMU_OT_NONE &&
1621		    DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) &&
1622		    sdnp[i].dn_bonuslen != 0) {
1623			nr_iovecs++;
1624		}
1625	}
1626
1627	nr_src += nr_iovecs;
1628	nr_dst += nr_iovecs;
1629
1630	if (nr_src != 0) {
1631		src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
1632		if (src_iovecs == NULL) {
1633			ret = SET_ERROR(ENOMEM);
1634			goto error;
1635		}
1636	}
1637
1638	if (nr_dst != 0) {
1639		dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
1640		if (dst_iovecs == NULL) {
1641			ret = SET_ERROR(ENOMEM);
1642			goto error;
1643		}
1644	}
1645
1646	nr_iovecs = 0;
1647
1648	/*
1649	 * Iterate through the dnodes again, this time filling in the uios
1650	 * we allocated earlier. We also concatenate any data we want to
1651	 * authenticate onto aadbuf.
1652	 */
1653	for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
1654		dnp = &sdnp[i];
1655		/* copy over the core fields and blkptrs (kept as plaintext) */
1656		bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp);
1657		if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1658			bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]),
1659			    sizeof (blkptr_t));
1660		}
1661
1662		/*
1663		 * Handle authenticated data. We authenticate everything in
1664		 * the dnode that can be brought over when we do a raw send.
1665		 * This includes all of the core fields as well as the MACs
1666		 * stored in the bp checksums and all of the portable bits
1667		 * from blk_prop. We include the dnode padding here in case it
1668		 * ever gets used in the future. Some dn_flags and dn_used are
1669		 * not portable so we mask those out values out of the
1670		 * authenticated data.
1671		 */
1672		crypt_len = offsetof(dnode_phys_t, dn_blkptr);
1673		bcopy(dnp, aadp, crypt_len);
1674		adnp = (dnode_phys_t *)aadp;
1675		adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK;
1676		adnp->dn_used = 0;
1677		aadp += crypt_len;
1678		aad_len += crypt_len;
1679
1680		for (j = 0; j < dnp->dn_nblkptr; j++) {
1681			zio_crypt_bp_do_aad_updates(&aadp, &aad_len,
1682			    version, byteswap, &dnp->dn_blkptr[j]);
1683		}
1684
1685		if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1686			zio_crypt_bp_do_aad_updates(&aadp, &aad_len,
1687			    version, byteswap, DN_SPILL_BLKPTR(dnp));
1688		}
1689
1690		/*
1691		 * If this bonus buffer needs to be encrypted, we prepare an
1692		 * iovec_t. The encryption / decryption functions will fill
1693		 * this in for us with the encrypted or decrypted data.
1694		 * Otherwise we add the bonus buffer to the authenticated
1695		 * data buffer and copy it over to the destination. The
1696		 * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that
1697		 * we can guarantee alignment with the AES block size
1698		 * (128 bits).
1699		 */
1700		crypt_len = DN_MAX_BONUS_LEN(dnp);
1701		if (dnp->dn_type != DMU_OT_NONE &&
1702		    DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) &&
1703		    dnp->dn_bonuslen != 0) {
1704			ASSERT3U(nr_iovecs, <, nr_src);
1705			ASSERT3U(nr_iovecs, <, nr_dst);
1706			ASSERT3P(src_iovecs, !=, NULL);
1707			ASSERT3P(dst_iovecs, !=, NULL);
1708			src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp);
1709			src_iovecs[nr_iovecs].iov_len = crypt_len;
1710			dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]);
1711			dst_iovecs[nr_iovecs].iov_len = crypt_len;
1712
1713			nr_iovecs++;
1714			total_len += crypt_len;
1715		} else {
1716			bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len);
1717			bcopy(DN_BONUS(dnp), aadp, crypt_len);
1718			aadp += crypt_len;
1719			aad_len += crypt_len;
1720		}
1721	}
1722
1723	*no_crypt = (nr_iovecs == 0);
1724	*enc_len = total_len;
1725	*authbuf = aadbuf;
1726	*auth_len = aad_len;
1727
1728	if (encrypt) {
1729		puio->uio_iov = src_iovecs;
1730		puio->uio_iovcnt = nr_src;
1731		cuio->uio_iov = dst_iovecs;
1732		cuio->uio_iovcnt = nr_dst;
1733	} else {
1734		puio->uio_iov = dst_iovecs;
1735		puio->uio_iovcnt = nr_dst;
1736		cuio->uio_iov = src_iovecs;
1737		cuio->uio_iovcnt = nr_src;
1738	}
1739
1740	return (0);
1741
1742error:
1743	zio_buf_free(aadbuf, datalen);
1744	if (src_iovecs != NULL)
1745		kmem_free(src_iovecs, nr_src * sizeof (iovec_t));
1746	if (dst_iovecs != NULL)
1747		kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t));
1748
1749	*enc_len = 0;
1750	*authbuf = NULL;
1751	*auth_len = 0;
1752	*no_crypt = B_FALSE;
1753	puio->uio_iov = NULL;
1754	puio->uio_iovcnt = 0;
1755	cuio->uio_iov = NULL;
1756	cuio->uio_iovcnt = 0;
1757	return (ret);
1758}
1759
1760/* ARGSUSED */
1761static int
1762zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf,
1763    uint8_t *cipherbuf, uint_t datalen, uio_t *puio, uio_t *cuio,
1764    uint_t *enc_len)
1765{
1766	int ret;
1767	uint_t nr_plain = 1, nr_cipher = 2;
1768	iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL;
1769
1770	/* allocate the iovecs for the plain and cipher data */
1771	plain_iovecs = kmem_alloc(nr_plain * sizeof (iovec_t),
1772	    KM_SLEEP);
1773	if (!plain_iovecs) {
1774		ret = SET_ERROR(ENOMEM);
1775		goto error;
1776	}
1777
1778	cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t),
1779	    KM_SLEEP);
1780	if (!cipher_iovecs) {
1781		ret = SET_ERROR(ENOMEM);
1782		goto error;
1783	}
1784
1785	plain_iovecs[0].iov_base = (void *)plainbuf;
1786	plain_iovecs[0].iov_len = datalen;
1787	cipher_iovecs[0].iov_base = (void *)cipherbuf;
1788	cipher_iovecs[0].iov_len = datalen;
1789
1790	*enc_len = datalen;
1791	puio->uio_iov = plain_iovecs;
1792	puio->uio_iovcnt = nr_plain;
1793	cuio->uio_iov = cipher_iovecs;
1794	cuio->uio_iovcnt = nr_cipher;
1795
1796	return (0);
1797
1798error:
1799	if (plain_iovecs != NULL)
1800		kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t));
1801	if (cipher_iovecs != NULL)
1802		kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t));
1803
1804	*enc_len = 0;
1805	puio->uio_iov = NULL;
1806	puio->uio_iovcnt = 0;
1807	cuio->uio_iov = NULL;
1808	cuio->uio_iovcnt = 0;
1809	return (ret);
1810}
1811
1812/*
1813 * This function builds up the plaintext (puio) and ciphertext (cuio) uios so
1814 * that they can be used for encryption and decryption by zio_do_crypt_uio().
1815 * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks
1816 * requiring special handling to parse out pieces that are to be encrypted. The
1817 * authbuf is used by these special cases to store additional authenticated
1818 * data (AAD) for the encryption modes.
1819 */
1820/* ARGSUSED */
1821static int
1822zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot,
1823    uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap,
1824    uint8_t *mac, uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf,
1825    uint_t *auth_len, boolean_t *no_crypt)
1826{
1827	int ret;
1828	iovec_t *mac_iov;
1829
1830	ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE);
1831
1832	/* route to handler */
1833	switch (ot) {
1834	case DMU_OT_INTENT_LOG:
1835		ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf,
1836		    datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len,
1837		    no_crypt);
1838		break;
1839	case DMU_OT_DNODE:
1840		ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf,
1841		    cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf,
1842		    auth_len, no_crypt);
1843		break;
1844	default:
1845		ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf,
1846		    datalen, puio, cuio, enc_len);
1847		*authbuf = NULL;
1848		*auth_len = 0;
1849		*no_crypt = B_FALSE;
1850		break;
1851	}
1852
1853	if (ret != 0)
1854		goto error;
1855
1856	/* populate the uios */
1857	puio->uio_segflg = UIO_SYSSPACE;
1858	cuio->uio_segflg = UIO_SYSSPACE;
1859
1860	mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]);
1861	mac_iov->iov_base = (void *)mac;
1862	mac_iov->iov_len = ZIO_DATA_MAC_LEN;
1863
1864	return (0);
1865
1866error:
1867	return (ret);
1868}
1869
1870/*
1871 * Primary encryption / decryption entrypoint for zio data.
1872 */
1873int
1874zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key,
1875    dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
1876    uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf,
1877    boolean_t *no_crypt)
1878{
1879	int ret;
1880	boolean_t locked = B_FALSE;
1881	uint64_t crypt = key->zk_crypt;
1882	uint_t keydata_len = zio_crypt_table[crypt].ci_keylen;
1883	uint_t enc_len, auth_len;
1884	uio_t puio, cuio;
1885	uint8_t enc_keydata[MASTER_KEY_MAX_LEN];
1886	crypto_key_t tmp_ckey, *ckey = NULL;
1887	crypto_ctx_template_t tmpl;
1888	uint8_t *authbuf = NULL;
1889
1890	bzero(&puio, sizeof (uio_t));
1891	bzero(&cuio, sizeof (uio_t));
1892
1893	/* create uios for encryption */
1894	ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf,
1895	    cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len,
1896	    &authbuf, &auth_len, no_crypt);
1897	if (ret != 0)
1898		return (ret);
1899
1900	/*
1901	 * If the needed key is the current one, just use it. Otherwise we
1902	 * need to generate a temporary one from the given salt + master key.
1903	 * If we are encrypting, we must return a copy of the current salt
1904	 * so that it can be stored in the blkptr_t.
1905	 */
1906	rw_enter(&key->zk_salt_lock, RW_READER);
1907	locked = B_TRUE;
1908
1909	if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) {
1910		ckey = &key->zk_current_key;
1911		tmpl = key->zk_current_tmpl;
1912	} else {
1913		rw_exit(&key->zk_salt_lock);
1914		locked = B_FALSE;
1915
1916		ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
1917		    salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len);
1918		if (ret != 0)
1919			goto error;
1920
1921		tmp_ckey.ck_format = CRYPTO_KEY_RAW;
1922		tmp_ckey.ck_data = enc_keydata;
1923		tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len);
1924
1925		ckey = &tmp_ckey;
1926		tmpl = NULL;
1927	}
1928
1929	/* perform the encryption / decryption */
1930	ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len,
1931	    &puio, &cuio, authbuf, auth_len);
1932	if (ret != 0)
1933		goto error;
1934
1935	if (locked) {
1936		rw_exit(&key->zk_salt_lock);
1937		locked = B_FALSE;
1938	}
1939
1940	if (authbuf != NULL)
1941		zio_buf_free(authbuf, datalen);
1942	if (ckey == &tmp_ckey)
1943		bzero(enc_keydata, keydata_len);
1944	zio_crypt_destroy_uio(&puio);
1945	zio_crypt_destroy_uio(&cuio);
1946
1947	return (0);
1948
1949error:
1950	if (!encrypt) {
1951		if (failed_decrypt_buf != NULL)
1952			kmem_free(failed_decrypt_buf, failed_decrypt_size);
1953		failed_decrypt_buf = kmem_alloc(datalen, KM_SLEEP);
1954		failed_decrypt_size = datalen;
1955		bcopy(cipherbuf, failed_decrypt_buf, datalen);
1956	}
1957	if (locked)
1958		rw_exit(&key->zk_salt_lock);
1959	if (authbuf != NULL)
1960		zio_buf_free(authbuf, datalen);
1961	if (ckey == &tmp_ckey)
1962		bzero(enc_keydata, keydata_len);
1963	zio_crypt_destroy_uio(&puio);
1964	zio_crypt_destroy_uio(&cuio);
1965
1966	return (ret);
1967}
1968
1969/*
1970 * Simple wrapper around zio_do_crypt_data() to work with abd's instead of
1971 * linear buffers.
1972 */
1973int
1974zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot,
1975    boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac,
1976    uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt)
1977{
1978	int ret;
1979	void *ptmp, *ctmp;
1980
1981	if (encrypt) {
1982		ptmp = abd_borrow_buf_copy(pabd, datalen);
1983		ctmp = abd_borrow_buf(cabd, datalen);
1984	} else {
1985		ptmp = abd_borrow_buf(pabd, datalen);
1986		ctmp = abd_borrow_buf_copy(cabd, datalen);
1987	}
1988
1989	ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac,
1990	    datalen, ptmp, ctmp, no_crypt);
1991	if (ret != 0)
1992		goto error;
1993
1994	if (encrypt) {
1995		abd_return_buf(pabd, ptmp, datalen);
1996		abd_return_buf_copy(cabd, ctmp, datalen);
1997	} else {
1998		abd_return_buf_copy(pabd, ptmp, datalen);
1999		abd_return_buf(cabd, ctmp, datalen);
2000	}
2001
2002	return (0);
2003
2004error:
2005	if (encrypt) {
2006		abd_return_buf(pabd, ptmp, datalen);
2007		abd_return_buf_copy(cabd, ctmp, datalen);
2008	} else {
2009		abd_return_buf_copy(pabd, ptmp, datalen);
2010		abd_return_buf(cabd, ctmp, datalen);
2011	}
2012
2013	return (ret);
2014}
2015