1eb633035STom Caputi /*
2eb633035STom Caputi * CDDL HEADER START
3eb633035STom Caputi *
4eb633035STom Caputi * This file and its contents are supplied under the terms of the
5eb633035STom Caputi * Common Development and Distribution License ("CDDL"), version 1.0.
6eb633035STom Caputi * You may only use this file in accordance with the terms of version
7eb633035STom Caputi * 1.0 of the CDDL.
8eb633035STom Caputi *
9eb633035STom Caputi * A full copy of the text of the CDDL should have accompanied this
10eb633035STom Caputi * source. A copy of the CDDL is also available via the Internet at
11eb633035STom Caputi * http://www.illumos.org/license/CDDL.
12eb633035STom Caputi *
13eb633035STom Caputi * CDDL HEADER END
14eb633035STom Caputi */
15eb633035STom Caputi
16eb633035STom Caputi /*
17eb633035STom Caputi * Copyright (c) 2017, Datto, Inc. All rights reserved.
18eb633035STom Caputi */
19eb633035STom Caputi
20eb633035STom Caputi #include <sys/zio_crypt.h>
21eb633035STom Caputi #include <sys/dmu.h>
22eb633035STom Caputi #include <sys/dmu_objset.h>
23eb633035STom Caputi #include <sys/dnode.h>
24eb633035STom Caputi #include <sys/fs/zfs.h>
25eb633035STom Caputi #include <sys/zio.h>
26eb633035STom Caputi #include <sys/zil.h>
27eb633035STom Caputi #include <sys/sha2.h>
28eb633035STom Caputi #include <sys/hkdf.h>
29eb633035STom Caputi
30eb633035STom Caputi /*
31eb633035STom Caputi * This file is responsible for handling all of the details of generating
32eb633035STom Caputi * encryption parameters and performing encryption and authentication.
33eb633035STom Caputi *
34eb633035STom Caputi * BLOCK ENCRYPTION PARAMETERS:
35eb633035STom Caputi * Encryption /Authentication Algorithm Suite (crypt):
36eb633035STom Caputi * The encryption algorithm, mode, and key length we are going to use. We
37eb633035STom Caputi * currently support AES in either GCM or CCM modes with 128, 192, and 256 bit
38eb633035STom Caputi * keys. All authentication is currently done with SHA512-HMAC.
39eb633035STom Caputi *
40eb633035STom Caputi * Plaintext:
41eb633035STom Caputi * The unencrypted data that we want to encrypt.
42eb633035STom Caputi *
43eb633035STom Caputi * Initialization Vector (IV):
44eb633035STom Caputi * An initialization vector for the encryption algorithms. This is used to
45eb633035STom Caputi * "tweak" the encryption algorithms so that two blocks of the same data are
46eb633035STom Caputi * encrypted into different ciphertext outputs, thus obfuscating block patterns.
47eb633035STom Caputi * The supported encryption modes (AES-GCM and AES-CCM) require that an IV is
48eb633035STom Caputi * never reused with the same encryption key. This value is stored unencrypted
49eb633035STom Caputi * and must simply be provided to the decryption function. We use a 96 bit IV
50eb633035STom Caputi * (as recommended by NIST) for all block encryption. For non-dedup blocks we
51eb633035STom Caputi * derive the IV randomly. The first 64 bits of the IV are stored in the second
52eb633035STom Caputi * word of DVA[2] and the remaining 32 bits are stored in the upper 32 bits of
53eb633035STom Caputi * blk_fill. This is safe because encrypted blocks can't use the upper 32 bits
54eb633035STom Caputi * of blk_fill. We only encrypt level 0 blocks, which normally have a fill count
55eb633035STom Caputi * of 1. The only exception is for DMU_OT_DNODE objects, where the fill count of
56eb633035STom Caputi * level 0 blocks is the number of allocated dnodes in that block. The on-disk
57eb633035STom Caputi * format supports at most 2^15 slots per L0 dnode block, because the maximum
58eb633035STom Caputi * block size is 16MB (2^24). In either case, for level 0 blocks this number
59eb633035STom Caputi * will still be smaller than UINT32_MAX so it is safe to store the IV in the
60eb633035STom Caputi * top 32 bits of blk_fill, while leaving the bottom 32 bits of the fill count
61eb633035STom Caputi * for the dnode code.
62eb633035STom Caputi *
63eb633035STom Caputi * Master key:
64eb633035STom Caputi * This is the most important secret data of an encrypted dataset. It is used
65eb633035STom Caputi * along with the salt to generate that actual encryption keys via HKDF. We
66eb633035STom Caputi * do not use the master key to directly encrypt any data because there are
67eb633035STom Caputi * theoretical limits on how much data can actually be safely encrypted with
68eb633035STom Caputi * any encryption mode. The master key is stored encrypted on disk with the
69eb633035STom Caputi * user's wrapping key. Its length is determined by the encryption algorithm.
70eb633035STom Caputi * For details on how this is stored see the block comment in dsl_crypt.c
71eb633035STom Caputi *
72eb633035STom Caputi * Salt:
73eb633035STom Caputi * Used as an input to the HKDF function, along with the master key. We use a
74eb633035STom Caputi * 64 bit salt, stored unencrypted in the first word of DVA[2]. Any given salt
75eb633035STom Caputi * can be used for encrypting many blocks, so we cache the current salt and the
76eb633035STom Caputi * associated derived key in zio_crypt_t so we do not need to derive it again
77eb633035STom Caputi * needlessly.
78eb633035STom Caputi *
79eb633035STom Caputi * Encryption Key:
80eb633035STom Caputi * A secret binary key, generated from an HKDF function used to encrypt and
81eb633035STom Caputi * decrypt data.
82eb633035STom Caputi *
83eb633035STom Caputi * Message Authenication Code (MAC)
84eb633035STom Caputi * The MAC is an output of authenticated encryption modes such as AES-GCM and
85eb633035STom Caputi * AES-CCM. Its purpose is to ensure that an attacker cannot modify encrypted
86eb633035STom Caputi * data on disk and return garbage to the application. Effectively, it is a
87eb633035STom Caputi * checksum that can not be reproduced by an attacker. We store the MAC in the
88eb633035STom Caputi * second 128 bits of blk_cksum, leaving the first 128 bits for a truncated
89eb633035STom Caputi * regular checksum of the ciphertext which can be used for scrubbing.
90eb633035STom Caputi *
91eb633035STom Caputi * OBJECT AUTHENTICATION:
92eb633035STom Caputi * Some object types, such as DMU_OT_MASTER_NODE cannot be encrypted because
93eb633035STom Caputi * they contain some info that always needs to be readable. To prevent this
94eb633035STom Caputi * data from being altered, we authenticate this data using SHA512-HMAC. This
95eb633035STom Caputi * will produce a MAC (similar to the one produced via encryption) which can
96eb633035STom Caputi * be used to verify the object was not modified. HMACs do not require key
97eb633035STom Caputi * rotation or IVs, so we can keep up to the full 3 copies of authenticated
98eb633035STom Caputi * data.
99eb633035STom Caputi *
100eb633035STom Caputi * ZIL ENCRYPTION:
101eb633035STom Caputi * ZIL blocks have their bp written to disk ahead of the associated data, so we
102eb633035STom Caputi * cannot store the MAC there as we normally do. For these blocks the MAC is
103eb633035STom Caputi * stored in the embedded checksum within the zil_chain_t header. The salt and
104eb633035STom Caputi * IV are generated for the block on bp allocation instead of at encryption
105eb633035STom Caputi * time. In addition, ZIL blocks have some pieces that must be left in plaintext
106eb633035STom Caputi * for claiming even though all of the sensitive user data still needs to be
107eb633035STom Caputi * encrypted. The function zio_crypt_init_uios_zil() handles parsing which
108eb633035STom Caputi * pieces of the block need to be encrypted. All data that is not encrypted is
109eb633035STom Caputi * authenticated using the AAD mechanisms that the supported encryption modes
110eb633035STom Caputi * provide for. In order to preserve the semantics of the ZIL for encrypted
111eb633035STom Caputi * datasets, the ZIL is not protected at the objset level as described below.
112eb633035STom Caputi *
113eb633035STom Caputi * DNODE ENCRYPTION:
114eb633035STom Caputi * Similarly to ZIL blocks, the core part of each dnode_phys_t needs to be left
115eb633035STom Caputi * in plaintext for scrubbing and claiming, but the bonus buffers might contain
116eb633035STom Caputi * sensitive user data. The function zio_crypt_init_uios_dnode() handles parsing
117eb633035STom Caputi * which pieces of the block need to be encrypted. For more details about
118eb633035STom Caputi * dnode authentication and encryption, see zio_crypt_init_uios_dnode().
119eb633035STom Caputi *
120eb633035STom Caputi * OBJECT SET AUTHENTICATION:
121eb633035STom Caputi * Up to this point, everything we have encrypted and authenticated has been
122eb633035STom Caputi * at level 0 (or -2 for the ZIL). If we did not do any further work the
123eb633035STom Caputi * on-disk format would be susceptible to attacks that deleted or rearrannged
124eb633035STom Caputi * the order of level 0 blocks. Ideally, the cleanest solution would be to
125eb633035STom Caputi * maintain a tree of authentication MACs going up the bp tree. However, this
126eb633035STom Caputi * presents a problem for raw sends. Send files do not send information about
127eb633035STom Caputi * indirect blocks so there would be no convenient way to transfer the MACs and
128eb633035STom Caputi * they cannot be recalculated on the receive side without the master key which
129eb633035STom Caputi * would defeat one of the purposes of raw sends in the first place. Instead,
130eb633035STom Caputi * for the indirect levels of the bp tree, we use a regular SHA512 of the MACs
131eb633035STom Caputi * from the level below. We also include some portable fields from blk_prop such
132eb633035STom Caputi * as the lsize and compression algorithm to prevent the data from being
133eb633035STom Caputi * misinterpretted.
134eb633035STom Caputi *
135eb633035STom Caputi * At the objset level, we maintain 2 seperate 256 bit MACs in the
136eb633035STom Caputi * objset_phys_t. The first one is "portable" and is the logical root of the
137eb633035STom Caputi * MAC tree maintianed in the metadnode's bps. The second, is "local" and is
138eb633035STom Caputi * used as the root MAC for the user accounting objects, which are also not
139eb633035STom Caputi * transferred via "zfs send". The portable MAC is sent in the DRR_BEGIN payload
140eb633035STom Caputi * of the send file. The useraccounting code ensures that the useraccounting
141eb633035STom Caputi * info is not present upon a receive, so the local MAC can simply be cleared
142eb633035STom Caputi * out at that time. For more info about objset_phys_t authentication, see
143eb633035STom Caputi * zio_crypt_do_objset_hmacs().
144eb633035STom Caputi *
145eb633035STom Caputi * CONSIDERATIONS FOR DEDUP:
146eb633035STom Caputi * In order for dedup to work, blocks that we want to dedup with one another
147eb633035STom Caputi * need to use the same IV and encryption key, so that they will have the same
148eb633035STom Caputi * ciphertext. Normally, one should never reuse an IV with the same encryption
149eb633035STom Caputi * key or else AES-GCM and AES-CCM can both actually leak the plaintext of both
150eb633035STom Caputi * blocks. In this case, however, since we are using the same plaindata as
151eb633035STom Caputi * well all that we end up with is a duplicate of the original ciphertext we
152eb633035STom Caputi * already had. As a result, an attacker with read access to the raw disk will
153eb633035STom Caputi * be able to tell which blocks are the same but this information is given away
154eb633035STom Caputi * by dedup anyway. In order to get the same IVs and encryption keys for
155eb633035STom Caputi * equivalent blocks of data we use an HMAC of the plaindata. We use an HMAC
156eb633035STom Caputi * here so that a reproducible checksum of the plaindata is never available to
157eb633035STom Caputi * the attacker. The HMAC key is kept alongside the master key, encrypted on
158eb633035STom Caputi * disk. The first 64 bits of the HMAC are used in place of the random salt, and
159eb633035STom Caputi * the next 96 bits are used as the IV. As a result of this mechanism, dedup
160eb633035STom Caputi * will only work within a clone family since encrypted dedup requires use of
161eb633035STom Caputi * the same master and HMAC keys.
162eb633035STom Caputi */
163eb633035STom Caputi
164eb633035STom Caputi /*
165eb633035STom Caputi * After encrypting many blocks with the same key we may start to run up
166eb633035STom Caputi * against the theoretical limits of how much data can securely be encrypted
167eb633035STom Caputi * with a single key using the supported encryption modes. The most obvious
168eb633035STom Caputi * limitation is that our risk of generating 2 equivalent 96 bit IVs increases
169eb633035STom Caputi * the more IVs we generate (which both GCM and CCM modes strictly forbid).
170eb633035STom Caputi * This risk actually grows surprisingly quickly over time according to the
171eb633035STom Caputi * Birthday Problem. With a total IV space of 2^(96 bits), and assuming we have
172eb633035STom Caputi * generated n IVs with a cryptographically secure RNG, the approximate
173eb633035STom Caputi * probability p(n) of a collision is given as:
174eb633035STom Caputi *
175eb633035STom Caputi * p(n) ~= e^(-n*(n-1)/(2*(2^96)))
176eb633035STom Caputi *
177eb633035STom Caputi * [http://www.math.cornell.edu/~mec/2008-2009/TianyiZheng/Birthday.html]
178eb633035STom Caputi *
179eb633035STom Caputi * Assuming that we want to ensure that p(n) never goes over 1 / 1 trillion
180eb633035STom Caputi * we must not write more than 398,065,730 blocks with the same encryption key.
181eb633035STom Caputi * Therefore, we rotate our keys after 400,000,000 blocks have been written by
182eb633035STom Caputi * generating a new random 64 bit salt for our HKDF encryption key generation
183eb633035STom Caputi * function.
184eb633035STom Caputi */
185eb633035STom Caputi #define ZFS_KEY_MAX_SALT_USES_DEFAULT 400000000
186eb633035STom Caputi #define ZFS_CURRENT_MAX_SALT_USES \
187eb633035STom Caputi (MIN(zfs_key_max_salt_uses, ZFS_KEY_MAX_SALT_USES_DEFAULT))
188eb633035STom Caputi unsigned long zfs_key_max_salt_uses = ZFS_KEY_MAX_SALT_USES_DEFAULT;
189eb633035STom Caputi
190eb633035STom Caputi /*
191eb633035STom Caputi * Set to a nonzero value to cause zio_do_crypt_uio() to fail 1/this many
192eb633035STom Caputi * calls, to test decryption error handling code paths.
193eb633035STom Caputi */
194eb633035STom Caputi uint64_t zio_decrypt_fail_fraction = 0;
195eb633035STom Caputi
196eb633035STom Caputi typedef struct blkptr_auth_buf {
197eb633035STom Caputi uint64_t bab_prop; /* blk_prop - portable mask */
198eb633035STom Caputi uint8_t bab_mac[ZIO_DATA_MAC_LEN]; /* MAC from blk_cksum */
199eb633035STom Caputi uint64_t bab_pad; /* reserved for future use */
200eb633035STom Caputi } blkptr_auth_buf_t;
201eb633035STom Caputi
202eb633035STom Caputi zio_crypt_info_t zio_crypt_table[ZIO_CRYPT_FUNCTIONS] = {
203eb633035STom Caputi {"", ZC_TYPE_NONE, 0, "inherit"},
204eb633035STom Caputi {"", ZC_TYPE_NONE, 0, "on"},
205eb633035STom Caputi {"", ZC_TYPE_NONE, 0, "off"},
206eb633035STom Caputi {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 16, "aes-128-ccm"},
207eb633035STom Caputi {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 24, "aes-192-ccm"},
208eb633035STom Caputi {SUN_CKM_AES_CCM, ZC_TYPE_CCM, 32, "aes-256-ccm"},
209eb633035STom Caputi {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 16, "aes-128-gcm"},
210eb633035STom Caputi {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 24, "aes-192-gcm"},
211eb633035STom Caputi {SUN_CKM_AES_GCM, ZC_TYPE_GCM, 32, "aes-256-gcm"}
212eb633035STom Caputi };
213eb633035STom Caputi
214eb633035STom Caputi void
zio_crypt_key_destroy(zio_crypt_key_t * key)215eb633035STom Caputi zio_crypt_key_destroy(zio_crypt_key_t *key)
216eb633035STom Caputi {
217eb633035STom Caputi rw_destroy(&key->zk_salt_lock);
218eb633035STom Caputi
219eb633035STom Caputi /* free crypto templates */
220eb633035STom Caputi crypto_destroy_ctx_template(key->zk_current_tmpl);
221eb633035STom Caputi crypto_destroy_ctx_template(key->zk_hmac_tmpl);
222eb633035STom Caputi
223eb633035STom Caputi /* zero out sensitive data */
224eb633035STom Caputi bzero(key, sizeof (zio_crypt_key_t));
225eb633035STom Caputi }
226eb633035STom Caputi
227eb633035STom Caputi int
zio_crypt_key_init(uint64_t crypt,zio_crypt_key_t * key)228eb633035STom Caputi zio_crypt_key_init(uint64_t crypt, zio_crypt_key_t *key)
229eb633035STom Caputi {
230eb633035STom Caputi int ret;
231eb633035STom Caputi crypto_mechanism_t mech;
232eb633035STom Caputi uint_t keydata_len;
233eb633035STom Caputi
234eb633035STom Caputi ASSERT(key != NULL);
235eb633035STom Caputi ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
236eb633035STom Caputi
237eb633035STom Caputi keydata_len = zio_crypt_table[crypt].ci_keylen;
238eb633035STom Caputi bzero(key, sizeof (zio_crypt_key_t));
239eb633035STom Caputi
240eb633035STom Caputi /* fill keydata buffers and salt with random data */
241eb633035STom Caputi ret = random_get_bytes((uint8_t *)&key->zk_guid, sizeof (uint64_t));
242eb633035STom Caputi if (ret != 0)
243eb633035STom Caputi goto error;
244eb633035STom Caputi
245eb633035STom Caputi ret = random_get_bytes(key->zk_master_keydata, keydata_len);
246eb633035STom Caputi if (ret != 0)
247eb633035STom Caputi goto error;
248eb633035STom Caputi
249eb633035STom Caputi ret = random_get_bytes(key->zk_hmac_keydata, SHA512_HMAC_KEYLEN);
250eb633035STom Caputi if (ret != 0)
251eb633035STom Caputi goto error;
252eb633035STom Caputi
253eb633035STom Caputi ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN);
254eb633035STom Caputi if (ret != 0)
255eb633035STom Caputi goto error;
256eb633035STom Caputi
257eb633035STom Caputi /* derive the current key from the master key */
258eb633035STom Caputi ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
259eb633035STom Caputi key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata,
260eb633035STom Caputi keydata_len);
261eb633035STom Caputi if (ret != 0)
262eb633035STom Caputi goto error;
263eb633035STom Caputi
264eb633035STom Caputi /* initialize keys for the ICP */
265eb633035STom Caputi key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
266eb633035STom Caputi key->zk_current_key.ck_data = key->zk_current_keydata;
267eb633035STom Caputi key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
268eb633035STom Caputi
269eb633035STom Caputi key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
270eb633035STom Caputi key->zk_hmac_key.ck_data = &key->zk_hmac_key;
271eb633035STom Caputi key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
272eb633035STom Caputi
273eb633035STom Caputi /*
274eb633035STom Caputi * Initialize the crypto templates. It's ok if this fails because
275eb633035STom Caputi * this is just an optimization.
276eb633035STom Caputi */
277eb633035STom Caputi mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
278eb633035STom Caputi ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
279eb633035STom Caputi &key->zk_current_tmpl, KM_SLEEP);
280eb633035STom Caputi if (ret != CRYPTO_SUCCESS)
281eb633035STom Caputi key->zk_current_tmpl = NULL;
282eb633035STom Caputi
283eb633035STom Caputi mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
284eb633035STom Caputi ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
285eb633035STom Caputi &key->zk_hmac_tmpl, KM_SLEEP);
286eb633035STom Caputi if (ret != CRYPTO_SUCCESS)
287eb633035STom Caputi key->zk_hmac_tmpl = NULL;
288eb633035STom Caputi
289eb633035STom Caputi key->zk_crypt = crypt;
290eb633035STom Caputi key->zk_version = ZIO_CRYPT_KEY_CURRENT_VERSION;
291eb633035STom Caputi key->zk_salt_count = 0;
292eb633035STom Caputi rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL);
293eb633035STom Caputi
294eb633035STom Caputi return (0);
295eb633035STom Caputi
296eb633035STom Caputi error:
297eb633035STom Caputi zio_crypt_key_destroy(key);
298eb633035STom Caputi return (ret);
299eb633035STom Caputi }
300eb633035STom Caputi
301eb633035STom Caputi static int
zio_crypt_key_change_salt(zio_crypt_key_t * key)302eb633035STom Caputi zio_crypt_key_change_salt(zio_crypt_key_t *key)
303eb633035STom Caputi {
304eb633035STom Caputi int ret = 0;
305eb633035STom Caputi uint8_t salt[ZIO_DATA_SALT_LEN];
306eb633035STom Caputi crypto_mechanism_t mech;
307eb633035STom Caputi uint_t keydata_len = zio_crypt_table[key->zk_crypt].ci_keylen;
308eb633035STom Caputi
309eb633035STom Caputi /* generate a new salt */
310eb633035STom Caputi ret = random_get_bytes(salt, ZIO_DATA_SALT_LEN);
311eb633035STom Caputi if (ret != 0)
312eb633035STom Caputi goto error;
313eb633035STom Caputi
314eb633035STom Caputi rw_enter(&key->zk_salt_lock, RW_WRITER);
315eb633035STom Caputi
316eb633035STom Caputi /* someone beat us to the salt rotation, just unlock and return */
317eb633035STom Caputi if (key->zk_salt_count < ZFS_CURRENT_MAX_SALT_USES)
318eb633035STom Caputi goto out_unlock;
319eb633035STom Caputi
320eb633035STom Caputi /* derive the current key from the master key and the new salt */
321eb633035STom Caputi ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
322eb633035STom Caputi salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata, keydata_len);
323eb633035STom Caputi if (ret != 0)
324eb633035STom Caputi goto out_unlock;
325eb633035STom Caputi
326eb633035STom Caputi /* assign the salt and reset the usage count */
327eb633035STom Caputi bcopy(salt, key->zk_salt, ZIO_DATA_SALT_LEN);
328eb633035STom Caputi key->zk_salt_count = 0;
329eb633035STom Caputi
330eb633035STom Caputi /* destroy the old context template and create the new one */
331eb633035STom Caputi crypto_destroy_ctx_template(key->zk_current_tmpl);
332eb633035STom Caputi ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
333eb633035STom Caputi &key->zk_current_tmpl, KM_SLEEP);
334eb633035STom Caputi if (ret != CRYPTO_SUCCESS)
335eb633035STom Caputi key->zk_current_tmpl = NULL;
336eb633035STom Caputi
337eb633035STom Caputi rw_exit(&key->zk_salt_lock);
338eb633035STom Caputi
339eb633035STom Caputi return (0);
340eb633035STom Caputi
341eb633035STom Caputi out_unlock:
342eb633035STom Caputi rw_exit(&key->zk_salt_lock);
343eb633035STom Caputi error:
344eb633035STom Caputi return (ret);
345eb633035STom Caputi }
346eb633035STom Caputi
347eb633035STom Caputi /* See comment above zfs_key_max_salt_uses definition for details */
348eb633035STom Caputi int
zio_crypt_key_get_salt(zio_crypt_key_t * key,uint8_t * salt)349eb633035STom Caputi zio_crypt_key_get_salt(zio_crypt_key_t *key, uint8_t *salt)
350eb633035STom Caputi {
351eb633035STom Caputi int ret;
352eb633035STom Caputi boolean_t salt_change;
353eb633035STom Caputi
354eb633035STom Caputi rw_enter(&key->zk_salt_lock, RW_READER);
355eb633035STom Caputi
356eb633035STom Caputi bcopy(key->zk_salt, salt, ZIO_DATA_SALT_LEN);
357eb633035STom Caputi salt_change = (atomic_inc_64_nv(&key->zk_salt_count) >=
358eb633035STom Caputi ZFS_CURRENT_MAX_SALT_USES);
359eb633035STom Caputi
360eb633035STom Caputi rw_exit(&key->zk_salt_lock);
361eb633035STom Caputi
362eb633035STom Caputi if (salt_change) {
363eb633035STom Caputi ret = zio_crypt_key_change_salt(key);
364eb633035STom Caputi if (ret != 0)
365eb633035STom Caputi goto error;
366eb633035STom Caputi }
367eb633035STom Caputi
368eb633035STom Caputi return (0);
369eb633035STom Caputi
370eb633035STom Caputi error:
371eb633035STom Caputi return (ret);
372eb633035STom Caputi }
373eb633035STom Caputi
374eb633035STom Caputi void *failed_decrypt_buf;
375eb633035STom Caputi int failed_decrypt_size;
376eb633035STom Caputi
377eb633035STom Caputi /*
378eb633035STom Caputi * This function handles all encryption and decryption in zfs. When
379eb633035STom Caputi * encrypting it expects puio to reference the plaintext and cuio to
380eb633035STom Caputi * reference the cphertext. cuio must have enough space for the
381eb633035STom Caputi * ciphertext + room for a MAC. datalen should be the length of the
382eb633035STom Caputi * plaintext / ciphertext alone.
383eb633035STom Caputi */
384eb633035STom Caputi /* ARGSUSED */
385eb633035STom Caputi static int
zio_do_crypt_uio(boolean_t encrypt,uint64_t crypt,crypto_key_t * key,crypto_ctx_template_t tmpl,uint8_t * ivbuf,uint_t datalen,uio_t * puio,uio_t * cuio,uint8_t * authbuf,uint_t auth_len)386eb633035STom Caputi zio_do_crypt_uio(boolean_t encrypt, uint64_t crypt, crypto_key_t *key,
387eb633035STom Caputi crypto_ctx_template_t tmpl, uint8_t *ivbuf, uint_t datalen,
388eb633035STom Caputi uio_t *puio, uio_t *cuio, uint8_t *authbuf, uint_t auth_len)
389eb633035STom Caputi {
390eb633035STom Caputi int ret;
391eb633035STom Caputi crypto_data_t plaindata, cipherdata;
392eb633035STom Caputi CK_AES_CCM_PARAMS ccmp;
393eb633035STom Caputi CK_AES_GCM_PARAMS gcmp;
394eb633035STom Caputi crypto_mechanism_t mech;
395eb633035STom Caputi zio_crypt_info_t crypt_info;
396eb633035STom Caputi uint_t plain_full_len, maclen;
397eb633035STom Caputi
398eb633035STom Caputi ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
399eb633035STom Caputi ASSERT3U(key->ck_format, ==, CRYPTO_KEY_RAW);
400eb633035STom Caputi
401eb633035STom Caputi /* lookup the encryption info */
402eb633035STom Caputi crypt_info = zio_crypt_table[crypt];
403eb633035STom Caputi
404eb633035STom Caputi /* the mac will always be the last iovec_t in the cipher uio */
405eb633035STom Caputi maclen = cuio->uio_iov[cuio->uio_iovcnt - 1].iov_len;
406eb633035STom Caputi
407eb633035STom Caputi ASSERT(maclen <= ZIO_DATA_MAC_LEN);
408eb633035STom Caputi
409eb633035STom Caputi /* setup encryption mechanism (same as crypt) */
410eb633035STom Caputi mech.cm_type = crypto_mech2id(crypt_info.ci_mechname);
411eb633035STom Caputi
412eb633035STom Caputi /*
413eb633035STom Caputi * Strangely, the ICP requires that plain_full_len must include
414eb633035STom Caputi * the MAC length when decrypting, even though the UIO does not
415eb633035STom Caputi * need to have the extra space allocated.
416eb633035STom Caputi */
417eb633035STom Caputi if (encrypt) {
418eb633035STom Caputi plain_full_len = datalen;
419eb633035STom Caputi } else {
420eb633035STom Caputi plain_full_len = datalen + maclen;
421eb633035STom Caputi }
422eb633035STom Caputi
423eb633035STom Caputi /*
424eb633035STom Caputi * setup encryption params (currently only AES CCM and AES GCM
425eb633035STom Caputi * are supported)
426eb633035STom Caputi */
427eb633035STom Caputi if (crypt_info.ci_crypt_type == ZC_TYPE_CCM) {
428eb633035STom Caputi ccmp.ulNonceSize = ZIO_DATA_IV_LEN;
429eb633035STom Caputi ccmp.ulAuthDataSize = auth_len;
430eb633035STom Caputi ccmp.authData = authbuf;
431eb633035STom Caputi ccmp.ulMACSize = maclen;
432eb633035STom Caputi ccmp.nonce = ivbuf;
433eb633035STom Caputi ccmp.ulDataSize = plain_full_len;
434eb633035STom Caputi
435eb633035STom Caputi mech.cm_param = (char *)(&ccmp);
436eb633035STom Caputi mech.cm_param_len = sizeof (CK_AES_CCM_PARAMS);
437eb633035STom Caputi } else {
438eb633035STom Caputi gcmp.ulIvLen = ZIO_DATA_IV_LEN;
439eb633035STom Caputi gcmp.ulIvBits = CRYPTO_BYTES2BITS(ZIO_DATA_IV_LEN);
440eb633035STom Caputi gcmp.ulAADLen = auth_len;
441eb633035STom Caputi gcmp.pAAD = authbuf;
442eb633035STom Caputi gcmp.ulTagBits = CRYPTO_BYTES2BITS(maclen);
443eb633035STom Caputi gcmp.pIv = ivbuf;
444eb633035STom Caputi
445eb633035STom Caputi mech.cm_param = (char *)(&gcmp);
446eb633035STom Caputi mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS);
447eb633035STom Caputi }
448eb633035STom Caputi
449eb633035STom Caputi /* populate the cipher and plain data structs. */
450eb633035STom Caputi plaindata.cd_format = CRYPTO_DATA_UIO;
451eb633035STom Caputi plaindata.cd_offset = 0;
452eb633035STom Caputi plaindata.cd_uio = puio;
453eb633035STom Caputi plaindata.cd_miscdata = NULL;
454eb633035STom Caputi plaindata.cd_length = plain_full_len;
455eb633035STom Caputi
456eb633035STom Caputi cipherdata.cd_format = CRYPTO_DATA_UIO;
457eb633035STom Caputi cipherdata.cd_offset = 0;
458eb633035STom Caputi cipherdata.cd_uio = cuio;
459eb633035STom Caputi cipherdata.cd_miscdata = NULL;
460eb633035STom Caputi cipherdata.cd_length = datalen + maclen;
461eb633035STom Caputi
462eb633035STom Caputi /* perform the actual encryption */
463eb633035STom Caputi if (encrypt) {
464eb633035STom Caputi ret = crypto_encrypt(&mech, &plaindata, key, tmpl, &cipherdata,
465eb633035STom Caputi NULL);
466eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
467eb633035STom Caputi ret = SET_ERROR(EIO);
468eb633035STom Caputi goto error;
469eb633035STom Caputi }
470eb633035STom Caputi } else {
471eb633035STom Caputi if (zio_decrypt_fail_fraction != 0 &&
472eb633035STom Caputi spa_get_random(zio_decrypt_fail_fraction) == 0) {
473eb633035STom Caputi ret = CRYPTO_INVALID_MAC;
474eb633035STom Caputi } else {
475eb633035STom Caputi ret = crypto_decrypt(&mech, &cipherdata,
476eb633035STom Caputi key, tmpl, &plaindata, NULL);
477eb633035STom Caputi }
478eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
479eb633035STom Caputi ASSERT3U(ret, ==, CRYPTO_INVALID_MAC);
480eb633035STom Caputi ret = SET_ERROR(ECKSUM);
481eb633035STom Caputi goto error;
482eb633035STom Caputi }
483eb633035STom Caputi }
484eb633035STom Caputi
485eb633035STom Caputi return (0);
486eb633035STom Caputi
487eb633035STom Caputi error:
488eb633035STom Caputi return (ret);
489eb633035STom Caputi }
490eb633035STom Caputi
491eb633035STom Caputi int
zio_crypt_key_wrap(crypto_key_t * cwkey,zio_crypt_key_t * key,uint8_t * iv,uint8_t * mac,uint8_t * keydata_out,uint8_t * hmac_keydata_out)492eb633035STom Caputi zio_crypt_key_wrap(crypto_key_t *cwkey, zio_crypt_key_t *key, uint8_t *iv,
493eb633035STom Caputi uint8_t *mac, uint8_t *keydata_out, uint8_t *hmac_keydata_out)
494eb633035STom Caputi {
495eb633035STom Caputi int ret;
496eb633035STom Caputi uio_t puio, cuio;
497eb633035STom Caputi uint64_t aad[3];
498eb633035STom Caputi iovec_t plain_iovecs[2], cipher_iovecs[3];
499eb633035STom Caputi uint64_t crypt = key->zk_crypt;
500eb633035STom Caputi uint_t enc_len, keydata_len, aad_len;
501eb633035STom Caputi
502eb633035STom Caputi ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
503eb633035STom Caputi ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW);
504eb633035STom Caputi
505eb633035STom Caputi keydata_len = zio_crypt_table[crypt].ci_keylen;
506eb633035STom Caputi
507eb633035STom Caputi /* generate iv for wrapping the master and hmac key */
508eb633035STom Caputi ret = random_get_pseudo_bytes(iv, WRAPPING_IV_LEN);
509eb633035STom Caputi if (ret != 0)
510eb633035STom Caputi goto error;
511eb633035STom Caputi
512eb633035STom Caputi /* initialize uio_ts */
513eb633035STom Caputi plain_iovecs[0].iov_base = (char *)key->zk_master_keydata;
514eb633035STom Caputi plain_iovecs[0].iov_len = keydata_len;
515eb633035STom Caputi plain_iovecs[1].iov_base = (char *)key->zk_hmac_keydata;
516eb633035STom Caputi plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
517eb633035STom Caputi
518eb633035STom Caputi cipher_iovecs[0].iov_base = (char *)keydata_out;
519eb633035STom Caputi cipher_iovecs[0].iov_len = keydata_len;
520eb633035STom Caputi cipher_iovecs[1].iov_base = (char *)hmac_keydata_out;
521eb633035STom Caputi cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
522eb633035STom Caputi cipher_iovecs[2].iov_base = (char *)mac;
523eb633035STom Caputi cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN;
524eb633035STom Caputi
525eb633035STom Caputi /*
526eb633035STom Caputi * Although we don't support writing to the old format, we do
527eb633035STom Caputi * support rewrapping the key so that the user can move and
528eb633035STom Caputi * quarantine datasets on the old format.
529eb633035STom Caputi */
530eb633035STom Caputi if (key->zk_version == 0) {
531eb633035STom Caputi aad_len = sizeof (uint64_t);
532eb633035STom Caputi aad[0] = LE_64(key->zk_guid);
533eb633035STom Caputi } else {
534eb633035STom Caputi ASSERT3U(key->zk_version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
535eb633035STom Caputi aad_len = sizeof (uint64_t) * 3;
536eb633035STom Caputi aad[0] = LE_64(key->zk_guid);
537eb633035STom Caputi aad[1] = LE_64(crypt);
538eb633035STom Caputi aad[2] = LE_64(key->zk_version);
539eb633035STom Caputi }
540eb633035STom Caputi
541eb633035STom Caputi enc_len = zio_crypt_table[crypt].ci_keylen + SHA512_HMAC_KEYLEN;
542eb633035STom Caputi puio.uio_iov = plain_iovecs;
543eb633035STom Caputi puio.uio_iovcnt = 2;
544eb633035STom Caputi puio.uio_segflg = UIO_SYSSPACE;
545eb633035STom Caputi cuio.uio_iov = cipher_iovecs;
546eb633035STom Caputi cuio.uio_iovcnt = 3;
547eb633035STom Caputi cuio.uio_segflg = UIO_SYSSPACE;
548eb633035STom Caputi
549eb633035STom Caputi /* encrypt the keys and store the resulting ciphertext and mac */
550eb633035STom Caputi ret = zio_do_crypt_uio(B_TRUE, crypt, cwkey, NULL, iv, enc_len,
551eb633035STom Caputi &puio, &cuio, (uint8_t *)aad, aad_len);
552eb633035STom Caputi if (ret != 0)
553eb633035STom Caputi goto error;
554eb633035STom Caputi
555eb633035STom Caputi return (0);
556eb633035STom Caputi
557eb633035STom Caputi error:
558eb633035STom Caputi return (ret);
559eb633035STom Caputi }
560eb633035STom Caputi
561eb633035STom Caputi int
zio_crypt_key_unwrap(crypto_key_t * cwkey,uint64_t crypt,uint64_t version,uint64_t guid,uint8_t * keydata,uint8_t * hmac_keydata,uint8_t * iv,uint8_t * mac,zio_crypt_key_t * key)562eb633035STom Caputi zio_crypt_key_unwrap(crypto_key_t *cwkey, uint64_t crypt, uint64_t version,
563eb633035STom Caputi uint64_t guid, uint8_t *keydata, uint8_t *hmac_keydata, uint8_t *iv,
564eb633035STom Caputi uint8_t *mac, zio_crypt_key_t *key)
565eb633035STom Caputi {
566eb633035STom Caputi int ret;
567eb633035STom Caputi crypto_mechanism_t mech;
568eb633035STom Caputi uio_t puio, cuio;
569eb633035STom Caputi uint64_t aad[3];
570eb633035STom Caputi iovec_t plain_iovecs[2], cipher_iovecs[3];
571eb633035STom Caputi uint_t enc_len, keydata_len, aad_len;
572eb633035STom Caputi
573eb633035STom Caputi ASSERT3U(crypt, <, ZIO_CRYPT_FUNCTIONS);
574eb633035STom Caputi ASSERT3U(cwkey->ck_format, ==, CRYPTO_KEY_RAW);
575eb633035STom Caputi
576eb633035STom Caputi rw_init(&key->zk_salt_lock, NULL, RW_DEFAULT, NULL);
577eb633035STom Caputi keydata_len = zio_crypt_table[crypt].ci_keylen;
578eb633035STom Caputi
579eb633035STom Caputi /* initialize uio_ts */
580eb633035STom Caputi plain_iovecs[0].iov_base = (char *)key->zk_master_keydata;
581eb633035STom Caputi plain_iovecs[0].iov_len = keydata_len;
582eb633035STom Caputi plain_iovecs[1].iov_base = (char *)key->zk_hmac_keydata;
583eb633035STom Caputi plain_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
584eb633035STom Caputi
585eb633035STom Caputi cipher_iovecs[0].iov_base = (char *)keydata;
586eb633035STom Caputi cipher_iovecs[0].iov_len = keydata_len;
587eb633035STom Caputi cipher_iovecs[1].iov_base = (char *)hmac_keydata;
588eb633035STom Caputi cipher_iovecs[1].iov_len = SHA512_HMAC_KEYLEN;
589eb633035STom Caputi cipher_iovecs[2].iov_base = (char *)mac;
590eb633035STom Caputi cipher_iovecs[2].iov_len = WRAPPING_MAC_LEN;
591eb633035STom Caputi
592eb633035STom Caputi if (version == 0) {
593eb633035STom Caputi aad_len = sizeof (uint64_t);
594eb633035STom Caputi aad[0] = LE_64(guid);
595eb633035STom Caputi } else {
596eb633035STom Caputi ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
597eb633035STom Caputi aad_len = sizeof (uint64_t) * 3;
598eb633035STom Caputi aad[0] = LE_64(guid);
599eb633035STom Caputi aad[1] = LE_64(crypt);
600eb633035STom Caputi aad[2] = LE_64(version);
601eb633035STom Caputi }
602eb633035STom Caputi
603eb633035STom Caputi enc_len = keydata_len + SHA512_HMAC_KEYLEN;
604eb633035STom Caputi puio.uio_iov = plain_iovecs;
605eb633035STom Caputi puio.uio_segflg = UIO_SYSSPACE;
606eb633035STom Caputi puio.uio_iovcnt = 2;
607eb633035STom Caputi cuio.uio_iov = cipher_iovecs;
608eb633035STom Caputi cuio.uio_iovcnt = 3;
609eb633035STom Caputi cuio.uio_segflg = UIO_SYSSPACE;
610eb633035STom Caputi
611eb633035STom Caputi /* decrypt the keys and store the result in the output buffers */
612eb633035STom Caputi ret = zio_do_crypt_uio(B_FALSE, crypt, cwkey, NULL, iv, enc_len,
613eb633035STom Caputi &puio, &cuio, (uint8_t *)aad, aad_len);
614eb633035STom Caputi if (ret != 0)
615eb633035STom Caputi goto error;
616eb633035STom Caputi
617eb633035STom Caputi /* generate a fresh salt */
618eb633035STom Caputi ret = random_get_bytes(key->zk_salt, ZIO_DATA_SALT_LEN);
619eb633035STom Caputi if (ret != 0)
620eb633035STom Caputi goto error;
621eb633035STom Caputi
622eb633035STom Caputi /* derive the current key from the master key */
623eb633035STom Caputi ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
624eb633035STom Caputi key->zk_salt, ZIO_DATA_SALT_LEN, key->zk_current_keydata,
625eb633035STom Caputi keydata_len);
626eb633035STom Caputi if (ret != 0)
627eb633035STom Caputi goto error;
628eb633035STom Caputi
629eb633035STom Caputi /* initialize keys for ICP */
630eb633035STom Caputi key->zk_current_key.ck_format = CRYPTO_KEY_RAW;
631eb633035STom Caputi key->zk_current_key.ck_data = key->zk_current_keydata;
632eb633035STom Caputi key->zk_current_key.ck_length = CRYPTO_BYTES2BITS(keydata_len);
633eb633035STom Caputi
634eb633035STom Caputi key->zk_hmac_key.ck_format = CRYPTO_KEY_RAW;
635eb633035STom Caputi key->zk_hmac_key.ck_data = key->zk_hmac_keydata;
636eb633035STom Caputi key->zk_hmac_key.ck_length = CRYPTO_BYTES2BITS(SHA512_HMAC_KEYLEN);
637eb633035STom Caputi
638eb633035STom Caputi /*
639eb633035STom Caputi * Initialize the crypto templates. It's ok if this fails because
640eb633035STom Caputi * this is just an optimization.
641eb633035STom Caputi */
642eb633035STom Caputi mech.cm_type = crypto_mech2id(zio_crypt_table[crypt].ci_mechname);
643eb633035STom Caputi ret = crypto_create_ctx_template(&mech, &key->zk_current_key,
644eb633035STom Caputi &key->zk_current_tmpl, KM_SLEEP);
645eb633035STom Caputi if (ret != CRYPTO_SUCCESS)
646eb633035STom Caputi key->zk_current_tmpl = NULL;
647eb633035STom Caputi
648eb633035STom Caputi mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
649eb633035STom Caputi ret = crypto_create_ctx_template(&mech, &key->zk_hmac_key,
650eb633035STom Caputi &key->zk_hmac_tmpl, KM_SLEEP);
651eb633035STom Caputi if (ret != CRYPTO_SUCCESS)
652eb633035STom Caputi key->zk_hmac_tmpl = NULL;
653eb633035STom Caputi
654eb633035STom Caputi key->zk_crypt = crypt;
655eb633035STom Caputi key->zk_version = version;
656eb633035STom Caputi key->zk_guid = guid;
657eb633035STom Caputi key->zk_salt_count = 0;
658eb633035STom Caputi
659eb633035STom Caputi return (0);
660eb633035STom Caputi
661eb633035STom Caputi error:
662eb633035STom Caputi zio_crypt_key_destroy(key);
663eb633035STom Caputi return (ret);
664eb633035STom Caputi }
665eb633035STom Caputi
666eb633035STom Caputi int
zio_crypt_generate_iv(uint8_t * ivbuf)667eb633035STom Caputi zio_crypt_generate_iv(uint8_t *ivbuf)
668eb633035STom Caputi {
669eb633035STom Caputi int ret;
670eb633035STom Caputi
671eb633035STom Caputi /* randomly generate the IV */
672eb633035STom Caputi ret = random_get_pseudo_bytes(ivbuf, ZIO_DATA_IV_LEN);
673eb633035STom Caputi if (ret != 0)
674eb633035STom Caputi goto error;
675eb633035STom Caputi
676eb633035STom Caputi return (0);
677eb633035STom Caputi
678eb633035STom Caputi error:
679eb633035STom Caputi bzero(ivbuf, ZIO_DATA_IV_LEN);
680eb633035STom Caputi return (ret);
681eb633035STom Caputi }
682eb633035STom Caputi
683eb633035STom Caputi int
zio_crypt_do_hmac(zio_crypt_key_t * key,uint8_t * data,uint_t datalen,uint8_t * digestbuf,uint_t digestlen)684eb633035STom Caputi zio_crypt_do_hmac(zio_crypt_key_t *key, uint8_t *data, uint_t datalen,
685eb633035STom Caputi uint8_t *digestbuf, uint_t digestlen)
686eb633035STom Caputi {
687eb633035STom Caputi int ret;
688eb633035STom Caputi crypto_mechanism_t mech;
689eb633035STom Caputi crypto_data_t in_data, digest_data;
690eb633035STom Caputi uint8_t raw_digestbuf[SHA512_DIGEST_LENGTH];
691eb633035STom Caputi
692eb633035STom Caputi ASSERT3U(digestlen, <=, SHA512_DIGEST_LENGTH);
693eb633035STom Caputi
694eb633035STom Caputi /* initialize sha512-hmac mechanism and crypto data */
695eb633035STom Caputi mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
696eb633035STom Caputi mech.cm_param = NULL;
697eb633035STom Caputi mech.cm_param_len = 0;
698eb633035STom Caputi
699eb633035STom Caputi /* initialize the crypto data */
700eb633035STom Caputi in_data.cd_format = CRYPTO_DATA_RAW;
701eb633035STom Caputi in_data.cd_offset = 0;
702eb633035STom Caputi in_data.cd_length = datalen;
703eb633035STom Caputi in_data.cd_raw.iov_base = (char *)data;
704eb633035STom Caputi in_data.cd_raw.iov_len = in_data.cd_length;
705eb633035STom Caputi
706eb633035STom Caputi digest_data.cd_format = CRYPTO_DATA_RAW;
707eb633035STom Caputi digest_data.cd_offset = 0;
708eb633035STom Caputi digest_data.cd_length = SHA512_DIGEST_LENGTH;
709eb633035STom Caputi digest_data.cd_raw.iov_base = (char *)raw_digestbuf;
710eb633035STom Caputi digest_data.cd_raw.iov_len = digest_data.cd_length;
711eb633035STom Caputi
712eb633035STom Caputi /* generate the hmac */
713eb633035STom Caputi ret = crypto_mac(&mech, &in_data, &key->zk_hmac_key, key->zk_hmac_tmpl,
714eb633035STom Caputi &digest_data, NULL);
715eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
716eb633035STom Caputi ret = SET_ERROR(EIO);
717eb633035STom Caputi goto error;
718eb633035STom Caputi }
719eb633035STom Caputi
720eb633035STom Caputi bcopy(raw_digestbuf, digestbuf, digestlen);
721eb633035STom Caputi
722eb633035STom Caputi return (0);
723eb633035STom Caputi
724eb633035STom Caputi error:
725eb633035STom Caputi bzero(digestbuf, digestlen);
726eb633035STom Caputi return (ret);
727eb633035STom Caputi }
728eb633035STom Caputi
729eb633035STom Caputi int
zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t * key,uint8_t * data,uint_t datalen,uint8_t * ivbuf,uint8_t * salt)730eb633035STom Caputi zio_crypt_generate_iv_salt_dedup(zio_crypt_key_t *key, uint8_t *data,
731eb633035STom Caputi uint_t datalen, uint8_t *ivbuf, uint8_t *salt)
732eb633035STom Caputi {
733eb633035STom Caputi int ret;
734eb633035STom Caputi uint8_t digestbuf[SHA512_DIGEST_LENGTH];
735eb633035STom Caputi
736eb633035STom Caputi ret = zio_crypt_do_hmac(key, data, datalen,
737eb633035STom Caputi digestbuf, SHA512_DIGEST_LENGTH);
738eb633035STom Caputi if (ret != 0)
739eb633035STom Caputi return (ret);
740eb633035STom Caputi
741eb633035STom Caputi bcopy(digestbuf, salt, ZIO_DATA_SALT_LEN);
742eb633035STom Caputi bcopy(digestbuf + ZIO_DATA_SALT_LEN, ivbuf, ZIO_DATA_IV_LEN);
743eb633035STom Caputi
744eb633035STom Caputi return (0);
745eb633035STom Caputi }
746eb633035STom Caputi
747eb633035STom Caputi /*
748eb633035STom Caputi * The following functions are used to encode and decode encryption parameters
749eb633035STom Caputi * into blkptr_t and zil_header_t. The ICP wants to use these parameters as
750eb633035STom Caputi * byte strings, which normally means that these strings would not need to deal
751eb633035STom Caputi * with byteswapping at all. However, both blkptr_t and zil_header_t may be
752eb633035STom Caputi * byteswapped by lower layers and so we must "undo" that byteswap here upon
753eb633035STom Caputi * decoding and encoding in a non-native byteorder. These functions require
754eb633035STom Caputi * that the byteorder bit is correct before being called.
755eb633035STom Caputi */
756eb633035STom Caputi void
zio_crypt_encode_params_bp(blkptr_t * bp,uint8_t * salt,uint8_t * iv)757eb633035STom Caputi zio_crypt_encode_params_bp(blkptr_t *bp, uint8_t *salt, uint8_t *iv)
758eb633035STom Caputi {
759eb633035STom Caputi uint64_t val64;
760eb633035STom Caputi uint32_t val32;
761eb633035STom Caputi
762eb633035STom Caputi ASSERT(BP_IS_ENCRYPTED(bp));
763eb633035STom Caputi
764eb633035STom Caputi if (!BP_SHOULD_BYTESWAP(bp)) {
765eb633035STom Caputi bcopy(salt, &bp->blk_dva[2].dva_word[0], sizeof (uint64_t));
766eb633035STom Caputi bcopy(iv, &bp->blk_dva[2].dva_word[1], sizeof (uint64_t));
767eb633035STom Caputi bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t));
768eb633035STom Caputi BP_SET_IV2(bp, val32);
769eb633035STom Caputi } else {
770eb633035STom Caputi bcopy(salt, &val64, sizeof (uint64_t));
771eb633035STom Caputi bp->blk_dva[2].dva_word[0] = BSWAP_64(val64);
772eb633035STom Caputi
773eb633035STom Caputi bcopy(iv, &val64, sizeof (uint64_t));
774eb633035STom Caputi bp->blk_dva[2].dva_word[1] = BSWAP_64(val64);
775eb633035STom Caputi
776eb633035STom Caputi bcopy(iv + sizeof (uint64_t), &val32, sizeof (uint32_t));
777eb633035STom Caputi BP_SET_IV2(bp, BSWAP_32(val32));
778eb633035STom Caputi }
779eb633035STom Caputi }
780eb633035STom Caputi
781eb633035STom Caputi void
zio_crypt_decode_params_bp(const blkptr_t * bp,uint8_t * salt,uint8_t * iv)782eb633035STom Caputi zio_crypt_decode_params_bp(const blkptr_t *bp, uint8_t *salt, uint8_t *iv)
783eb633035STom Caputi {
784eb633035STom Caputi uint64_t val64;
785eb633035STom Caputi uint32_t val32;
786eb633035STom Caputi
787eb633035STom Caputi ASSERT(BP_IS_PROTECTED(bp));
788eb633035STom Caputi
789eb633035STom Caputi /* for convenience, so callers don't need to check */
790eb633035STom Caputi if (BP_IS_AUTHENTICATED(bp)) {
791eb633035STom Caputi bzero(salt, ZIO_DATA_SALT_LEN);
792eb633035STom Caputi bzero(iv, ZIO_DATA_IV_LEN);
793eb633035STom Caputi return;
794eb633035STom Caputi }
795eb633035STom Caputi
796eb633035STom Caputi if (!BP_SHOULD_BYTESWAP(bp)) {
797eb633035STom Caputi bcopy(&bp->blk_dva[2].dva_word[0], salt, sizeof (uint64_t));
798eb633035STom Caputi bcopy(&bp->blk_dva[2].dva_word[1], iv, sizeof (uint64_t));
799eb633035STom Caputi
800eb633035STom Caputi val32 = (uint32_t)BP_GET_IV2(bp);
801eb633035STom Caputi bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t));
802eb633035STom Caputi } else {
803eb633035STom Caputi val64 = BSWAP_64(bp->blk_dva[2].dva_word[0]);
804eb633035STom Caputi bcopy(&val64, salt, sizeof (uint64_t));
805eb633035STom Caputi
806eb633035STom Caputi val64 = BSWAP_64(bp->blk_dva[2].dva_word[1]);
807eb633035STom Caputi bcopy(&val64, iv, sizeof (uint64_t));
808eb633035STom Caputi
809eb633035STom Caputi val32 = BSWAP_32((uint32_t)BP_GET_IV2(bp));
810eb633035STom Caputi bcopy(&val32, iv + sizeof (uint64_t), sizeof (uint32_t));
811eb633035STom Caputi }
812eb633035STom Caputi }
813eb633035STom Caputi
814eb633035STom Caputi void
zio_crypt_encode_mac_bp(blkptr_t * bp,uint8_t * mac)815eb633035STom Caputi zio_crypt_encode_mac_bp(blkptr_t *bp, uint8_t *mac)
816eb633035STom Caputi {
817eb633035STom Caputi uint64_t val64;
818eb633035STom Caputi
819eb633035STom Caputi ASSERT(BP_USES_CRYPT(bp));
820eb633035STom Caputi ASSERT3U(BP_GET_TYPE(bp), !=, DMU_OT_OBJSET);
821eb633035STom Caputi
822eb633035STom Caputi if (!BP_SHOULD_BYTESWAP(bp)) {
823eb633035STom Caputi bcopy(mac, &bp->blk_cksum.zc_word[2], sizeof (uint64_t));
824eb633035STom Caputi bcopy(mac + sizeof (uint64_t), &bp->blk_cksum.zc_word[3],
825eb633035STom Caputi sizeof (uint64_t));
826eb633035STom Caputi } else {
827eb633035STom Caputi bcopy(mac, &val64, sizeof (uint64_t));
828eb633035STom Caputi bp->blk_cksum.zc_word[2] = BSWAP_64(val64);
829eb633035STom Caputi
830eb633035STom Caputi bcopy(mac + sizeof (uint64_t), &val64, sizeof (uint64_t));
831eb633035STom Caputi bp->blk_cksum.zc_word[3] = BSWAP_64(val64);
832eb633035STom Caputi }
833eb633035STom Caputi }
834eb633035STom Caputi
835eb633035STom Caputi void
zio_crypt_decode_mac_bp(const blkptr_t * bp,uint8_t * mac)836eb633035STom Caputi zio_crypt_decode_mac_bp(const blkptr_t *bp, uint8_t *mac)
837eb633035STom Caputi {
838eb633035STom Caputi uint64_t val64;
839eb633035STom Caputi
840eb633035STom Caputi ASSERT(BP_USES_CRYPT(bp) || BP_IS_HOLE(bp));
841eb633035STom Caputi
842eb633035STom Caputi /* for convenience, so callers don't need to check */
843eb633035STom Caputi if (BP_GET_TYPE(bp) == DMU_OT_OBJSET) {
844eb633035STom Caputi bzero(mac, ZIO_DATA_MAC_LEN);
845eb633035STom Caputi return;
846eb633035STom Caputi }
847eb633035STom Caputi
848eb633035STom Caputi if (!BP_SHOULD_BYTESWAP(bp)) {
849eb633035STom Caputi bcopy(&bp->blk_cksum.zc_word[2], mac, sizeof (uint64_t));
850eb633035STom Caputi bcopy(&bp->blk_cksum.zc_word[3], mac + sizeof (uint64_t),
851eb633035STom Caputi sizeof (uint64_t));
852eb633035STom Caputi } else {
853eb633035STom Caputi val64 = BSWAP_64(bp->blk_cksum.zc_word[2]);
854eb633035STom Caputi bcopy(&val64, mac, sizeof (uint64_t));
855eb633035STom Caputi
856eb633035STom Caputi val64 = BSWAP_64(bp->blk_cksum.zc_word[3]);
857eb633035STom Caputi bcopy(&val64, mac + sizeof (uint64_t), sizeof (uint64_t));
858eb633035STom Caputi }
859eb633035STom Caputi }
860eb633035STom Caputi
861eb633035STom Caputi void
zio_crypt_encode_mac_zil(void * data,uint8_t * mac)862eb633035STom Caputi zio_crypt_encode_mac_zil(void *data, uint8_t *mac)
863eb633035STom Caputi {
864eb633035STom Caputi zil_chain_t *zilc = data;
865eb633035STom Caputi
866eb633035STom Caputi bcopy(mac, &zilc->zc_eck.zec_cksum.zc_word[2], sizeof (uint64_t));
867eb633035STom Caputi bcopy(mac + sizeof (uint64_t), &zilc->zc_eck.zec_cksum.zc_word[3],
868eb633035STom Caputi sizeof (uint64_t));
869eb633035STom Caputi }
870eb633035STom Caputi
871eb633035STom Caputi void
zio_crypt_decode_mac_zil(const void * data,uint8_t * mac)872eb633035STom Caputi zio_crypt_decode_mac_zil(const void *data, uint8_t *mac)
873eb633035STom Caputi {
874eb633035STom Caputi /*
875eb633035STom Caputi * The ZIL MAC is embedded in the block it protects, which will
876eb633035STom Caputi * not have been byteswapped by the time this function has been called.
877eb633035STom Caputi * As a result, we don't need to worry about byteswapping the MAC.
878eb633035STom Caputi */
879eb633035STom Caputi const zil_chain_t *zilc = data;
880eb633035STom Caputi
881eb633035STom Caputi bcopy(&zilc->zc_eck.zec_cksum.zc_word[2], mac, sizeof (uint64_t));
882eb633035STom Caputi bcopy(&zilc->zc_eck.zec_cksum.zc_word[3], mac + sizeof (uint64_t),
883eb633035STom Caputi sizeof (uint64_t));
884eb633035STom Caputi }
885eb633035STom Caputi
886eb633035STom Caputi /*
887eb633035STom Caputi * This routine takes a block of dnodes (src_abd) and copies only the bonus
888eb633035STom Caputi * buffers to the same offsets in the dst buffer. datalen should be the size
889eb633035STom Caputi * of both the src_abd and the dst buffer (not just the length of the bonus
890eb633035STom Caputi * buffers).
891eb633035STom Caputi */
892eb633035STom Caputi void
zio_crypt_copy_dnode_bonus(abd_t * src_abd,uint8_t * dst,uint_t datalen)893eb633035STom Caputi zio_crypt_copy_dnode_bonus(abd_t *src_abd, uint8_t *dst, uint_t datalen)
894eb633035STom Caputi {
895eb633035STom Caputi uint_t i, max_dnp = datalen >> DNODE_SHIFT;
896eb633035STom Caputi uint8_t *src;
897eb633035STom Caputi dnode_phys_t *dnp, *sdnp, *ddnp;
898eb633035STom Caputi
899eb633035STom Caputi src = abd_borrow_buf_copy(src_abd, datalen);
900eb633035STom Caputi
901eb633035STom Caputi sdnp = (dnode_phys_t *)src;
902eb633035STom Caputi ddnp = (dnode_phys_t *)dst;
903eb633035STom Caputi
904eb633035STom Caputi for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
905eb633035STom Caputi dnp = &sdnp[i];
906eb633035STom Caputi if (dnp->dn_type != DMU_OT_NONE &&
907eb633035STom Caputi DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) &&
908eb633035STom Caputi dnp->dn_bonuslen != 0) {
909eb633035STom Caputi bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]),
910eb633035STom Caputi DN_MAX_BONUS_LEN(dnp));
911eb633035STom Caputi }
912eb633035STom Caputi }
913eb633035STom Caputi
914eb633035STom Caputi abd_return_buf(src_abd, src, datalen);
915eb633035STom Caputi }
916eb633035STom Caputi
917eb633035STom Caputi /*
918eb633035STom Caputi * This function decides what fields from blk_prop are included in
919eb633035STom Caputi * the on-disk various MAC algorithms.
920eb633035STom Caputi */
921eb633035STom Caputi static void
zio_crypt_bp_zero_nonportable_blkprop(blkptr_t * bp,uint64_t version)922eb633035STom Caputi zio_crypt_bp_zero_nonportable_blkprop(blkptr_t *bp, uint64_t version)
923eb633035STom Caputi {
924eb633035STom Caputi /*
925eb633035STom Caputi * Version 0 did not properly zero out all non-portable fields
926eb633035STom Caputi * as it should have done. We maintain this code so that we can
927eb633035STom Caputi * do read-only imports of pools on this version.
928eb633035STom Caputi */
929eb633035STom Caputi if (version == 0) {
930eb633035STom Caputi BP_SET_DEDUP(bp, 0);
931eb633035STom Caputi BP_SET_CHECKSUM(bp, 0);
932eb633035STom Caputi BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE);
933eb633035STom Caputi return;
934eb633035STom Caputi }
935eb633035STom Caputi
936eb633035STom Caputi ASSERT3U(version, ==, ZIO_CRYPT_KEY_CURRENT_VERSION);
937eb633035STom Caputi
938eb633035STom Caputi /*
939eb633035STom Caputi * The hole_birth feature might set these fields even if this bp
940eb633035STom Caputi * is a hole. We zero them out here to guarantee that raw sends
941eb633035STom Caputi * will function with or without the feature.
942eb633035STom Caputi */
943eb633035STom Caputi if (BP_IS_HOLE(bp)) {
944eb633035STom Caputi bp->blk_prop = 0ULL;
945eb633035STom Caputi return;
946eb633035STom Caputi }
947eb633035STom Caputi
948eb633035STom Caputi /*
949eb633035STom Caputi * At L0 we want to verify these fields to ensure that data blocks
950eb633035STom Caputi * can not be reinterpretted. For instance, we do not want an attacker
951eb633035STom Caputi * to trick us into returning raw lz4 compressed data to the user
952eb633035STom Caputi * by modifying the compression bits. At higher levels, we cannot
953eb633035STom Caputi * enforce this policy since raw sends do not convey any information
954eb633035STom Caputi * about indirect blocks, so these values might be different on the
955eb633035STom Caputi * receive side. Fortunately, this does not open any new attack
956eb633035STom Caputi * vectors, since any alterations that can be made to a higher level
957eb633035STom Caputi * bp must still verify the correct order of the layer below it.
958eb633035STom Caputi */
959eb633035STom Caputi if (BP_GET_LEVEL(bp) != 0) {
960eb633035STom Caputi BP_SET_BYTEORDER(bp, 0);
961eb633035STom Caputi BP_SET_COMPRESS(bp, 0);
962eb633035STom Caputi
963eb633035STom Caputi /*
964eb633035STom Caputi * psize cannot be set to zero or it will trigger
965eb633035STom Caputi * asserts, but the value doesn't really matter as
966eb633035STom Caputi * long as it is constant.
967eb633035STom Caputi */
968eb633035STom Caputi BP_SET_PSIZE(bp, SPA_MINBLOCKSIZE);
969eb633035STom Caputi }
970eb633035STom Caputi
971eb633035STom Caputi BP_SET_DEDUP(bp, 0);
972eb633035STom Caputi BP_SET_CHECKSUM(bp, 0);
973eb633035STom Caputi }
974eb633035STom Caputi
975eb633035STom Caputi static void
zio_crypt_bp_auth_init(uint64_t version,boolean_t should_bswap,blkptr_t * bp,blkptr_auth_buf_t * bab,uint_t * bab_len)976eb633035STom Caputi zio_crypt_bp_auth_init(uint64_t version, boolean_t should_bswap, blkptr_t *bp,
977eb633035STom Caputi blkptr_auth_buf_t *bab, uint_t *bab_len)
978eb633035STom Caputi {
979eb633035STom Caputi blkptr_t tmpbp = *bp;
980eb633035STom Caputi
981eb633035STom Caputi if (should_bswap)
982eb633035STom Caputi byteswap_uint64_array(&tmpbp, sizeof (blkptr_t));
983eb633035STom Caputi
984eb633035STom Caputi ASSERT(BP_USES_CRYPT(&tmpbp) || BP_IS_HOLE(&tmpbp));
985eb633035STom Caputi ASSERT0(BP_IS_EMBEDDED(&tmpbp));
986eb633035STom Caputi
987eb633035STom Caputi zio_crypt_decode_mac_bp(&tmpbp, bab->bab_mac);
988eb633035STom Caputi
989eb633035STom Caputi /*
990eb633035STom Caputi * We always MAC blk_prop in LE to ensure portability. This
991eb633035STom Caputi * must be done after decoding the mac, since the endianness
992eb633035STom Caputi * will get zero'd out here.
993eb633035STom Caputi */
994eb633035STom Caputi zio_crypt_bp_zero_nonportable_blkprop(&tmpbp, version);
995eb633035STom Caputi bab->bab_prop = LE_64(tmpbp.blk_prop);
996eb633035STom Caputi bab->bab_pad = 0ULL;
997eb633035STom Caputi
998eb633035STom Caputi /* version 0 did not include the padding */
999eb633035STom Caputi *bab_len = sizeof (blkptr_auth_buf_t);
1000eb633035STom Caputi if (version == 0)
1001eb633035STom Caputi *bab_len -= sizeof (uint64_t);
1002eb633035STom Caputi }
1003eb633035STom Caputi
1004eb633035STom Caputi static int
zio_crypt_bp_do_hmac_updates(crypto_context_t ctx,uint64_t version,boolean_t should_bswap,blkptr_t * bp)1005eb633035STom Caputi zio_crypt_bp_do_hmac_updates(crypto_context_t ctx, uint64_t version,
1006eb633035STom Caputi boolean_t should_bswap, blkptr_t *bp)
1007eb633035STom Caputi {
1008eb633035STom Caputi int ret;
1009eb633035STom Caputi uint_t bab_len;
1010eb633035STom Caputi blkptr_auth_buf_t bab;
1011eb633035STom Caputi crypto_data_t cd;
1012eb633035STom Caputi
1013eb633035STom Caputi zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1014eb633035STom Caputi cd.cd_format = CRYPTO_DATA_RAW;
1015eb633035STom Caputi cd.cd_offset = 0;
1016eb633035STom Caputi cd.cd_length = bab_len;
1017eb633035STom Caputi cd.cd_raw.iov_base = (char *)&bab;
1018eb633035STom Caputi cd.cd_raw.iov_len = cd.cd_length;
1019eb633035STom Caputi
1020eb633035STom Caputi ret = crypto_mac_update(ctx, &cd, NULL);
1021eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
1022eb633035STom Caputi ret = SET_ERROR(EIO);
1023eb633035STom Caputi goto error;
1024eb633035STom Caputi }
1025eb633035STom Caputi
1026eb633035STom Caputi return (0);
1027eb633035STom Caputi
1028eb633035STom Caputi error:
1029eb633035STom Caputi return (ret);
1030eb633035STom Caputi }
1031eb633035STom Caputi
1032eb633035STom Caputi static void
zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX * ctx,uint64_t version,boolean_t should_bswap,blkptr_t * bp)1033eb633035STom Caputi zio_crypt_bp_do_indrect_checksum_updates(SHA2_CTX *ctx, uint64_t version,
1034eb633035STom Caputi boolean_t should_bswap, blkptr_t *bp)
1035eb633035STom Caputi {
1036eb633035STom Caputi uint_t bab_len;
1037eb633035STom Caputi blkptr_auth_buf_t bab;
1038eb633035STom Caputi
1039eb633035STom Caputi zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1040eb633035STom Caputi SHA2Update(ctx, &bab, bab_len);
1041eb633035STom Caputi }
1042eb633035STom Caputi
1043eb633035STom Caputi static void
zio_crypt_bp_do_aad_updates(uint8_t ** aadp,uint_t * aad_len,uint64_t version,boolean_t should_bswap,blkptr_t * bp)1044eb633035STom Caputi zio_crypt_bp_do_aad_updates(uint8_t **aadp, uint_t *aad_len, uint64_t version,
1045eb633035STom Caputi boolean_t should_bswap, blkptr_t *bp)
1046eb633035STom Caputi {
1047eb633035STom Caputi uint_t bab_len;
1048eb633035STom Caputi blkptr_auth_buf_t bab;
1049eb633035STom Caputi
1050eb633035STom Caputi zio_crypt_bp_auth_init(version, should_bswap, bp, &bab, &bab_len);
1051eb633035STom Caputi bcopy(&bab, *aadp, bab_len);
1052eb633035STom Caputi *aadp += bab_len;
1053eb633035STom Caputi *aad_len += bab_len;
1054eb633035STom Caputi }
1055eb633035STom Caputi
1056eb633035STom Caputi static int
zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx,uint64_t version,boolean_t should_bswap,dnode_phys_t * dnp)1057eb633035STom Caputi zio_crypt_do_dnode_hmac_updates(crypto_context_t ctx, uint64_t version,
1058eb633035STom Caputi boolean_t should_bswap, dnode_phys_t *dnp)
1059eb633035STom Caputi {
1060eb633035STom Caputi int ret, i;
1061eb633035STom Caputi dnode_phys_t *adnp;
1062eb633035STom Caputi boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER);
1063eb633035STom Caputi crypto_data_t cd;
1064*0d48c3fdSToomas Soome uint8_t tmp_dncore[sizeof (dnode_phys_t)];
1065eb633035STom Caputi
1066*0d48c3fdSToomas Soome adnp = (dnode_phys_t *)tmp_dncore;
1067eb633035STom Caputi cd.cd_format = CRYPTO_DATA_RAW;
1068eb633035STom Caputi cd.cd_offset = 0;
1069*0d48c3fdSToomas Soome cd.cd_length = offsetof(dnode_phys_t, dn_blkptr);
1070*0d48c3fdSToomas Soome cd.cd_raw.iov_base = (char *)adnp;
1071*0d48c3fdSToomas Soome cd.cd_raw.iov_len = cd.cd_length;
1072eb633035STom Caputi
1073eb633035STom Caputi /* authenticate the core dnode (masking out non-portable bits) */
1074*0d48c3fdSToomas Soome bcopy(dnp, tmp_dncore, cd.cd_length);
1075eb633035STom Caputi if (le_bswap) {
1076eb633035STom Caputi adnp->dn_datablkszsec = BSWAP_16(adnp->dn_datablkszsec);
1077eb633035STom Caputi adnp->dn_bonuslen = BSWAP_16(adnp->dn_bonuslen);
1078eb633035STom Caputi adnp->dn_maxblkid = BSWAP_64(adnp->dn_maxblkid);
1079eb633035STom Caputi adnp->dn_used = BSWAP_64(adnp->dn_used);
1080eb633035STom Caputi }
1081eb633035STom Caputi adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK;
1082eb633035STom Caputi adnp->dn_used = 0;
1083eb633035STom Caputi
1084eb633035STom Caputi ret = crypto_mac_update(ctx, &cd, NULL);
1085eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
1086eb633035STom Caputi ret = SET_ERROR(EIO);
1087eb633035STom Caputi goto error;
1088eb633035STom Caputi }
1089eb633035STom Caputi
1090eb633035STom Caputi for (i = 0; i < dnp->dn_nblkptr; i++) {
1091eb633035STom Caputi ret = zio_crypt_bp_do_hmac_updates(ctx, version,
1092eb633035STom Caputi should_bswap, &dnp->dn_blkptr[i]);
1093eb633035STom Caputi if (ret != 0)
1094eb633035STom Caputi goto error;
1095eb633035STom Caputi }
1096eb633035STom Caputi
1097eb633035STom Caputi if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1098eb633035STom Caputi ret = zio_crypt_bp_do_hmac_updates(ctx, version,
1099eb633035STom Caputi should_bswap, DN_SPILL_BLKPTR(dnp));
1100eb633035STom Caputi if (ret != 0)
1101eb633035STom Caputi goto error;
1102eb633035STom Caputi }
1103eb633035STom Caputi
1104eb633035STom Caputi return (0);
1105eb633035STom Caputi
1106eb633035STom Caputi error:
1107eb633035STom Caputi return (ret);
1108eb633035STom Caputi }
1109eb633035STom Caputi
1110eb633035STom Caputi /*
1111eb633035STom Caputi * objset_phys_t blocks introduce a number of exceptions to the normal
1112eb633035STom Caputi * authentication process. objset_phys_t's contain 2 seperate HMACS for
1113eb633035STom Caputi * protecting the integrity of their data. The portable_mac protects the
1114eb633035STom Caputi * the metadnode. This MAC can be sent with a raw send and protects against
1115eb633035STom Caputi * reordering of data within the metadnode. The local_mac protects the user
1116eb633035STom Caputi * accounting objects which are not sent from one system to another.
1117eb633035STom Caputi *
1118eb633035STom Caputi * In addition, objset blocks are the only blocks that can be modified and
1119eb633035STom Caputi * written to disk without the key loaded under certain circumstances. During
1120eb633035STom Caputi * zil_claim() we need to be able to update the zil_header_t to complete
1121eb633035STom Caputi * claiming log blocks and during raw receives we need to write out the
1122eb633035STom Caputi * portable_mac from the send file. Both of these actions are possible
1123eb633035STom Caputi * because these fields are not protected by either MAC so neither one will
1124eb633035STom Caputi * need to modify the MACs without the key. However, when the modified blocks
1125eb633035STom Caputi * are written out they will be byteswapped into the host machine's native
1126eb633035STom Caputi * endianness which will modify fields protected by the MAC. As a result, MAC
1127eb633035STom Caputi * calculation for objset blocks works slightly differently from other block
1128eb633035STom Caputi * types. Where other block types MAC the data in whatever endianness is
1129eb633035STom Caputi * written to disk, objset blocks always MAC little endian version of their
1130eb633035STom Caputi * values. In the code, should_bswap is the value from BP_SHOULD_BYTESWAP()
1131eb633035STom Caputi * and le_bswap indicates whether a byteswap is needed to get this block
1132eb633035STom Caputi * into little endian format.
1133eb633035STom Caputi */
1134eb633035STom Caputi /* ARGSUSED */
1135eb633035STom Caputi int
zio_crypt_do_objset_hmacs(zio_crypt_key_t * key,void * data,uint_t datalen,boolean_t should_bswap,uint8_t * portable_mac,uint8_t * local_mac)1136eb633035STom Caputi zio_crypt_do_objset_hmacs(zio_crypt_key_t *key, void *data, uint_t datalen,
1137eb633035STom Caputi boolean_t should_bswap, uint8_t *portable_mac, uint8_t *local_mac)
1138eb633035STom Caputi {
1139eb633035STom Caputi int ret;
1140eb633035STom Caputi crypto_mechanism_t mech;
1141eb633035STom Caputi crypto_context_t ctx;
1142eb633035STom Caputi crypto_data_t cd;
1143eb633035STom Caputi objset_phys_t *osp = data;
1144eb633035STom Caputi uint64_t intval;
1145eb633035STom Caputi boolean_t le_bswap = (should_bswap == ZFS_HOST_BYTEORDER);
1146eb633035STom Caputi uint8_t raw_portable_mac[SHA512_DIGEST_LENGTH];
1147eb633035STom Caputi uint8_t raw_local_mac[SHA512_DIGEST_LENGTH];
1148eb633035STom Caputi
1149eb633035STom Caputi /* initialize HMAC mechanism */
1150eb633035STom Caputi mech.cm_type = crypto_mech2id(SUN_CKM_SHA512_HMAC);
1151eb633035STom Caputi mech.cm_param = NULL;
1152eb633035STom Caputi mech.cm_param_len = 0;
1153eb633035STom Caputi
1154eb633035STom Caputi cd.cd_format = CRYPTO_DATA_RAW;
1155eb633035STom Caputi cd.cd_offset = 0;
1156eb633035STom Caputi
1157eb633035STom Caputi /* calculate the portable MAC from the portable fields and metadnode */
1158eb633035STom Caputi ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL);
1159eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
1160eb633035STom Caputi ret = SET_ERROR(EIO);
1161eb633035STom Caputi goto error;
1162eb633035STom Caputi }
1163eb633035STom Caputi
1164eb633035STom Caputi /* add in the os_type */
1165eb633035STom Caputi intval = (le_bswap) ? osp->os_type : BSWAP_64(osp->os_type);
1166eb633035STom Caputi cd.cd_length = sizeof (uint64_t);
1167eb633035STom Caputi cd.cd_raw.iov_base = (char *)&intval;
1168eb633035STom Caputi cd.cd_raw.iov_len = cd.cd_length;
1169eb633035STom Caputi
1170eb633035STom Caputi ret = crypto_mac_update(ctx, &cd, NULL);
1171eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
1172eb633035STom Caputi ret = SET_ERROR(EIO);
1173eb633035STom Caputi goto error;
1174eb633035STom Caputi }
1175eb633035STom Caputi
1176eb633035STom Caputi /* add in the portable os_flags */
1177eb633035STom Caputi intval = osp->os_flags;
1178eb633035STom Caputi if (should_bswap)
1179eb633035STom Caputi intval = BSWAP_64(intval);
1180eb633035STom Caputi intval &= OBJSET_CRYPT_PORTABLE_FLAGS_MASK;
1181eb633035STom Caputi /* CONSTCOND */
1182eb633035STom Caputi if (!ZFS_HOST_BYTEORDER)
1183eb633035STom Caputi intval = BSWAP_64(intval);
1184eb633035STom Caputi
1185eb633035STom Caputi cd.cd_length = sizeof (uint64_t);
1186eb633035STom Caputi cd.cd_raw.iov_base = (char *)&intval;
1187eb633035STom Caputi cd.cd_raw.iov_len = cd.cd_length;
1188eb633035STom Caputi
1189eb633035STom Caputi ret = crypto_mac_update(ctx, &cd, NULL);
1190eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
1191eb633035STom Caputi ret = SET_ERROR(EIO);
1192eb633035STom Caputi goto error;
1193eb633035STom Caputi }
1194eb633035STom Caputi
1195eb633035STom Caputi /* add in fields from the metadnode */
1196eb633035STom Caputi ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1197eb633035STom Caputi should_bswap, &osp->os_meta_dnode);
1198eb633035STom Caputi if (ret)
1199eb633035STom Caputi goto error;
1200eb633035STom Caputi
1201eb633035STom Caputi /* store the final digest in a temporary buffer and copy what we need */
1202eb633035STom Caputi cd.cd_length = SHA512_DIGEST_LENGTH;
1203eb633035STom Caputi cd.cd_raw.iov_base = (char *)raw_portable_mac;
1204eb633035STom Caputi cd.cd_raw.iov_len = cd.cd_length;
1205eb633035STom Caputi
1206eb633035STom Caputi ret = crypto_mac_final(ctx, &cd, NULL);
1207eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
1208eb633035STom Caputi ret = SET_ERROR(EIO);
1209eb633035STom Caputi goto error;
1210eb633035STom Caputi }
1211eb633035STom Caputi
1212eb633035STom Caputi bcopy(raw_portable_mac, portable_mac, ZIO_OBJSET_MAC_LEN);
1213eb633035STom Caputi
1214eb633035STom Caputi /*
1215f67950b2SNasf-Fan * The local MAC protects the user, group and project accounting.
1216f67950b2SNasf-Fan * If these objects are not present, the local MAC is zeroed out.
1217eb633035STom Caputi */
1218f67950b2SNasf-Fan if ((datalen >= OBJSET_PHYS_SIZE_V3 &&
1219f67950b2SNasf-Fan osp->os_userused_dnode.dn_type == DMU_OT_NONE &&
1220f67950b2SNasf-Fan osp->os_groupused_dnode.dn_type == DMU_OT_NONE &&
1221f67950b2SNasf-Fan osp->os_projectused_dnode.dn_type == DMU_OT_NONE) ||
1222f67950b2SNasf-Fan (datalen >= OBJSET_PHYS_SIZE_V2 &&
1223f67950b2SNasf-Fan osp->os_userused_dnode.dn_type == DMU_OT_NONE &&
1224eb633035STom Caputi osp->os_groupused_dnode.dn_type == DMU_OT_NONE) ||
1225f67950b2SNasf-Fan (datalen <= OBJSET_PHYS_SIZE_V1)) {
1226eb633035STom Caputi bzero(local_mac, ZIO_OBJSET_MAC_LEN);
1227eb633035STom Caputi return (0);
1228eb633035STom Caputi }
1229eb633035STom Caputi
1230eb633035STom Caputi /* calculate the local MAC from the userused and groupused dnodes */
1231eb633035STom Caputi ret = crypto_mac_init(&mech, &key->zk_hmac_key, NULL, &ctx, NULL);
1232eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
1233eb633035STom Caputi ret = SET_ERROR(EIO);
1234eb633035STom Caputi goto error;
1235eb633035STom Caputi }
1236eb633035STom Caputi
1237eb633035STom Caputi /* add in the non-portable os_flags */
1238eb633035STom Caputi intval = osp->os_flags;
1239eb633035STom Caputi if (should_bswap)
1240eb633035STom Caputi intval = BSWAP_64(intval);
1241eb633035STom Caputi intval &= ~OBJSET_CRYPT_PORTABLE_FLAGS_MASK;
1242eb633035STom Caputi /* CONSTCOND */
1243eb633035STom Caputi if (!ZFS_HOST_BYTEORDER)
1244eb633035STom Caputi intval = BSWAP_64(intval);
1245eb633035STom Caputi
1246eb633035STom Caputi cd.cd_length = sizeof (uint64_t);
1247eb633035STom Caputi cd.cd_raw.iov_base = (char *)&intval;
1248eb633035STom Caputi cd.cd_raw.iov_len = cd.cd_length;
1249eb633035STom Caputi
1250eb633035STom Caputi ret = crypto_mac_update(ctx, &cd, NULL);
1251eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
1252eb633035STom Caputi ret = SET_ERROR(EIO);
1253eb633035STom Caputi goto error;
1254eb633035STom Caputi }
1255eb633035STom Caputi
1256eb633035STom Caputi /* add in fields from the user accounting dnodes */
1257eb633035STom Caputi ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1258eb633035STom Caputi should_bswap, &osp->os_userused_dnode);
1259eb633035STom Caputi if (ret)
1260eb633035STom Caputi goto error;
1261eb633035STom Caputi
1262eb633035STom Caputi ret = zio_crypt_do_dnode_hmac_updates(ctx, key->zk_version,
1263eb633035STom Caputi should_bswap, &osp->os_groupused_dnode);
1264eb633035STom Caputi if (ret)
1265eb633035STom Caputi goto error;
1266eb633035STom Caputi
1267eb633035STom Caputi /* store the final digest in a temporary buffer and copy what we need */
1268eb633035STom Caputi cd.cd_length = SHA512_DIGEST_LENGTH;
1269eb633035STom Caputi cd.cd_raw.iov_base = (char *)raw_local_mac;
1270eb633035STom Caputi cd.cd_raw.iov_len = cd.cd_length;
1271eb633035STom Caputi
1272eb633035STom Caputi ret = crypto_mac_final(ctx, &cd, NULL);
1273eb633035STom Caputi if (ret != CRYPTO_SUCCESS) {
1274eb633035STom Caputi ret = SET_ERROR(EIO);
1275eb633035STom Caputi goto error;
1276eb633035STom Caputi }
1277eb633035STom Caputi
1278eb633035STom Caputi bcopy(raw_local_mac, local_mac, ZIO_OBJSET_MAC_LEN);
1279eb633035STom Caputi
1280eb633035STom Caputi return (0);
1281eb633035STom Caputi
1282eb633035STom Caputi error:
1283eb633035STom Caputi bzero(portable_mac, ZIO_OBJSET_MAC_LEN);
1284eb633035STom Caputi bzero(local_mac, ZIO_OBJSET_MAC_LEN);
1285eb633035STom Caputi return (ret);
1286eb633035STom Caputi }
1287eb633035STom Caputi
1288eb633035STom Caputi static void
zio_crypt_destroy_uio(uio_t * uio)1289eb633035STom Caputi zio_crypt_destroy_uio(uio_t *uio)
1290eb633035STom Caputi {
1291eb633035STom Caputi if (uio->uio_iov)
1292eb633035STom Caputi kmem_free(uio->uio_iov, uio->uio_iovcnt * sizeof (iovec_t));
1293eb633035STom Caputi }
1294eb633035STom Caputi
1295eb633035STom Caputi /*
1296eb633035STom Caputi * This function parses an uncompressed indirect block and returns a checksum
1297eb633035STom Caputi * of all the portable fields from all of the contained bps. The portable
1298eb633035STom Caputi * fields are the MAC and all of the fields from blk_prop except for the dedup,
1299eb633035STom Caputi * checksum, and psize bits. For an explanation of the purpose of this, see
1300eb633035STom Caputi * the comment block on object set authentication.
1301eb633035STom Caputi */
1302eb633035STom Caputi static int
zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate,void * buf,uint_t datalen,uint64_t version,boolean_t byteswap,uint8_t * cksum)1303eb633035STom Caputi zio_crypt_do_indirect_mac_checksum_impl(boolean_t generate, void *buf,
1304eb633035STom Caputi uint_t datalen, uint64_t version, boolean_t byteswap, uint8_t *cksum)
1305eb633035STom Caputi {
1306eb633035STom Caputi blkptr_t *bp;
1307eb633035STom Caputi int i, epb = datalen >> SPA_BLKPTRSHIFT;
1308eb633035STom Caputi SHA2_CTX ctx;
1309eb633035STom Caputi uint8_t digestbuf[SHA512_DIGEST_LENGTH];
1310eb633035STom Caputi
1311eb633035STom Caputi /* checksum all of the MACs from the layer below */
1312eb633035STom Caputi SHA2Init(SHA512, &ctx);
1313eb633035STom Caputi for (i = 0, bp = buf; i < epb; i++, bp++) {
1314eb633035STom Caputi zio_crypt_bp_do_indrect_checksum_updates(&ctx, version,
1315eb633035STom Caputi byteswap, bp);
1316eb633035STom Caputi }
1317eb633035STom Caputi SHA2Final(digestbuf, &ctx);
1318eb633035STom Caputi
1319eb633035STom Caputi if (generate) {
1320eb633035STom Caputi bcopy(digestbuf, cksum, ZIO_DATA_MAC_LEN);
1321eb633035STom Caputi return (0);
1322eb633035STom Caputi }
1323eb633035STom Caputi
1324eb633035STom Caputi if (bcmp(digestbuf, cksum, ZIO_DATA_MAC_LEN) != 0)
1325eb633035STom Caputi return (SET_ERROR(ECKSUM));
1326eb633035STom Caputi
1327eb633035STom Caputi return (0);
1328eb633035STom Caputi }
1329eb633035STom Caputi
1330eb633035STom Caputi int
zio_crypt_do_indirect_mac_checksum(boolean_t generate,void * buf,uint_t datalen,boolean_t byteswap,uint8_t * cksum)1331eb633035STom Caputi zio_crypt_do_indirect_mac_checksum(boolean_t generate, void *buf,
1332eb633035STom Caputi uint_t datalen, boolean_t byteswap, uint8_t *cksum)
1333eb633035STom Caputi {
1334eb633035STom Caputi int ret;
1335eb633035STom Caputi
1336eb633035STom Caputi /*
1337eb633035STom Caputi * Unfortunately, callers of this function will not always have
1338eb633035STom Caputi * easy access to the on-disk format version. This info is
1339eb633035STom Caputi * normally found in the DSL Crypto Key, but the checksum-of-MACs
1340eb633035STom Caputi * is expected to be verifiable even when the key isn't loaded.
1341eb633035STom Caputi * Here, instead of doing a ZAP lookup for the version for each
1342eb633035STom Caputi * zio, we simply try both existing formats.
1343eb633035STom Caputi */
1344eb633035STom Caputi ret = zio_crypt_do_indirect_mac_checksum_impl(generate, buf,
1345eb633035STom Caputi datalen, ZIO_CRYPT_KEY_CURRENT_VERSION, byteswap, cksum);
1346eb633035STom Caputi if (ret == ECKSUM) {
1347eb633035STom Caputi ASSERT(!generate);
1348eb633035STom Caputi ret = zio_crypt_do_indirect_mac_checksum_impl(generate,
1349eb633035STom Caputi buf, datalen, 0, byteswap, cksum);
1350eb633035STom Caputi }
1351eb633035STom Caputi
1352eb633035STom Caputi return (ret);
1353eb633035STom Caputi }
1354eb633035STom Caputi
1355eb633035STom Caputi int
zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate,abd_t * abd,uint_t datalen,boolean_t byteswap,uint8_t * cksum)1356eb633035STom Caputi zio_crypt_do_indirect_mac_checksum_abd(boolean_t generate, abd_t *abd,
1357eb633035STom Caputi uint_t datalen, boolean_t byteswap, uint8_t *cksum)
1358eb633035STom Caputi {
1359eb633035STom Caputi int ret;
1360eb633035STom Caputi void *buf;
1361eb633035STom Caputi
1362eb633035STom Caputi buf = abd_borrow_buf_copy(abd, datalen);
1363eb633035STom Caputi ret = zio_crypt_do_indirect_mac_checksum(generate, buf, datalen,
1364eb633035STom Caputi byteswap, cksum);
1365eb633035STom Caputi abd_return_buf(abd, buf, datalen);
1366eb633035STom Caputi
1367eb633035STom Caputi return (ret);
1368eb633035STom Caputi }
1369eb633035STom Caputi
1370eb633035STom Caputi /*
1371eb633035STom Caputi * Special case handling routine for encrypting / decrypting ZIL blocks.
1372eb633035STom Caputi * We do not check for the older ZIL chain because the encryption feature
1373eb633035STom Caputi * was not available before the newer ZIL chain was introduced. The goal
1374eb633035STom Caputi * here is to encrypt everything except the blkptr_t of a lr_write_t and
1375eb633035STom Caputi * the zil_chain_t header. Everything that is not encrypted is authenticated.
1376eb633035STom Caputi */
1377eb633035STom Caputi
1378eb633035STom Caputi /* ARGSUSED */
1379eb633035STom Caputi static int
zio_crypt_init_uios_zil(boolean_t encrypt,uint8_t * plainbuf,uint8_t * cipherbuf,uint_t datalen,boolean_t byteswap,uio_t * puio,uio_t * cuio,uint_t * enc_len,uint8_t ** authbuf,uint_t * auth_len,boolean_t * no_crypt)1380eb633035STom Caputi zio_crypt_init_uios_zil(boolean_t encrypt, uint8_t *plainbuf,
1381eb633035STom Caputi uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap, uio_t *puio,
1382eb633035STom Caputi uio_t *cuio, uint_t *enc_len, uint8_t **authbuf, uint_t *auth_len,
1383eb633035STom Caputi boolean_t *no_crypt)
1384eb633035STom Caputi {
1385eb633035STom Caputi int ret;
1386eb633035STom Caputi uint64_t txtype, lr_len;
1387eb633035STom Caputi uint_t nr_src, nr_dst, crypt_len;
1388eb633035STom Caputi uint_t aad_len = 0, nr_iovecs = 0, total_len = 0;
1389eb633035STom Caputi iovec_t *src_iovecs = NULL, *dst_iovecs = NULL;
1390eb633035STom Caputi uint8_t *src, *dst, *slrp, *dlrp, *blkend, *aadp;
1391eb633035STom Caputi zil_chain_t *zilc;
1392eb633035STom Caputi lr_t *lr;
1393eb633035STom Caputi uint8_t *aadbuf = zio_buf_alloc(datalen);
1394eb633035STom Caputi
1395eb633035STom Caputi /* cipherbuf always needs an extra iovec for the MAC */
1396eb633035STom Caputi if (encrypt) {
1397eb633035STom Caputi src = plainbuf;
1398eb633035STom Caputi dst = cipherbuf;
1399eb633035STom Caputi nr_src = 0;
1400eb633035STom Caputi nr_dst = 1;
1401eb633035STom Caputi } else {
1402eb633035STom Caputi src = cipherbuf;
1403eb633035STom Caputi dst = plainbuf;
1404eb633035STom Caputi nr_src = 1;
1405eb633035STom Caputi nr_dst = 0;
1406eb633035STom Caputi }
1407eb633035STom Caputi
1408eb633035STom Caputi /* find the start and end record of the log block */
1409eb633035STom Caputi zilc = (zil_chain_t *)src;
1410eb633035STom Caputi slrp = src + sizeof (zil_chain_t);
1411eb633035STom Caputi aadp = aadbuf;
1412eb633035STom Caputi blkend = src + ((byteswap) ? BSWAP_64(zilc->zc_nused) : zilc->zc_nused);
1413eb633035STom Caputi
1414eb633035STom Caputi /* calculate the number of encrypted iovecs we will need */
1415eb633035STom Caputi for (; slrp < blkend; slrp += lr_len) {
1416eb633035STom Caputi lr = (lr_t *)slrp;
1417eb633035STom Caputi
1418eb633035STom Caputi if (!byteswap) {
1419eb633035STom Caputi txtype = lr->lrc_txtype;
1420eb633035STom Caputi lr_len = lr->lrc_reclen;
1421eb633035STom Caputi } else {
1422eb633035STom Caputi txtype = BSWAP_64(lr->lrc_txtype);
1423eb633035STom Caputi lr_len = BSWAP_64(lr->lrc_reclen);
1424eb633035STom Caputi }
1425eb633035STom Caputi
1426eb633035STom Caputi nr_iovecs++;
1427eb633035STom Caputi if (txtype == TX_WRITE && lr_len != sizeof (lr_write_t))
1428eb633035STom Caputi nr_iovecs++;
1429eb633035STom Caputi }
1430eb633035STom Caputi
1431eb633035STom Caputi nr_src += nr_iovecs;
1432eb633035STom Caputi nr_dst += nr_iovecs;
1433eb633035STom Caputi
1434eb633035STom Caputi /* allocate the iovec arrays */
1435eb633035STom Caputi if (nr_src != 0) {
1436eb633035STom Caputi src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
1437eb633035STom Caputi if (src_iovecs == NULL) {
1438eb633035STom Caputi ret = SET_ERROR(ENOMEM);
1439eb633035STom Caputi goto error;
1440eb633035STom Caputi }
1441eb633035STom Caputi }
1442eb633035STom Caputi
1443eb633035STom Caputi if (nr_dst != 0) {
1444eb633035STom Caputi dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
1445eb633035STom Caputi if (dst_iovecs == NULL) {
1446eb633035STom Caputi ret = SET_ERROR(ENOMEM);
1447eb633035STom Caputi goto error;
1448eb633035STom Caputi }
1449eb633035STom Caputi }
1450eb633035STom Caputi
1451eb633035STom Caputi /*
1452eb633035STom Caputi * Copy the plain zil header over and authenticate everything except
1453eb633035STom Caputi * the checksum that will store our MAC. If we are writing the data
1454eb633035STom Caputi * the embedded checksum will not have been calculated yet, so we don't
1455eb633035STom Caputi * authenticate that.
1456eb633035STom Caputi */
1457eb633035STom Caputi bcopy(src, dst, sizeof (zil_chain_t));
1458eb633035STom Caputi bcopy(src, aadp, sizeof (zil_chain_t) - sizeof (zio_eck_t));
1459eb633035STom Caputi aadp += sizeof (zil_chain_t) - sizeof (zio_eck_t);
1460eb633035STom Caputi aad_len += sizeof (zil_chain_t) - sizeof (zio_eck_t);
1461eb633035STom Caputi
1462eb633035STom Caputi /* loop over records again, filling in iovecs */
1463eb633035STom Caputi nr_iovecs = 0;
1464eb633035STom Caputi slrp = src + sizeof (zil_chain_t);
1465eb633035STom Caputi dlrp = dst + sizeof (zil_chain_t);
1466eb633035STom Caputi
1467eb633035STom Caputi for (; slrp < blkend; slrp += lr_len, dlrp += lr_len) {
1468eb633035STom Caputi lr = (lr_t *)slrp;
1469eb633035STom Caputi
1470eb633035STom Caputi if (!byteswap) {
1471eb633035STom Caputi txtype = lr->lrc_txtype;
1472eb633035STom Caputi lr_len = lr->lrc_reclen;
1473eb633035STom Caputi } else {
1474eb633035STom Caputi txtype = BSWAP_64(lr->lrc_txtype);
1475eb633035STom Caputi lr_len = BSWAP_64(lr->lrc_reclen);
1476eb633035STom Caputi }
1477eb633035STom Caputi
1478eb633035STom Caputi /* copy the common lr_t */
1479eb633035STom Caputi bcopy(slrp, dlrp, sizeof (lr_t));
1480eb633035STom Caputi bcopy(slrp, aadp, sizeof (lr_t));
1481eb633035STom Caputi aadp += sizeof (lr_t);
1482eb633035STom Caputi aad_len += sizeof (lr_t);
1483eb633035STom Caputi
1484eb633035STom Caputi ASSERT3P(src_iovecs, !=, NULL);
1485eb633035STom Caputi ASSERT3P(dst_iovecs, !=, NULL);
1486eb633035STom Caputi
1487eb633035STom Caputi /*
1488eb633035STom Caputi * If this is a TX_WRITE record we want to encrypt everything
1489eb633035STom Caputi * except the bp if exists. If the bp does exist we want to
1490eb633035STom Caputi * authenticate it.
1491eb633035STom Caputi */
1492eb633035STom Caputi if (txtype == TX_WRITE) {
1493eb633035STom Caputi crypt_len = sizeof (lr_write_t) -
1494eb633035STom Caputi sizeof (lr_t) - sizeof (blkptr_t);
1495eb633035STom Caputi src_iovecs[nr_iovecs].iov_base = (char *)slrp +
1496eb633035STom Caputi sizeof (lr_t);
1497eb633035STom Caputi src_iovecs[nr_iovecs].iov_len = crypt_len;
1498eb633035STom Caputi dst_iovecs[nr_iovecs].iov_base = (char *)dlrp +
1499eb633035STom Caputi sizeof (lr_t);
1500eb633035STom Caputi dst_iovecs[nr_iovecs].iov_len = crypt_len;
1501eb633035STom Caputi
1502eb633035STom Caputi /* copy the bp now since it will not be encrypted */
1503eb633035STom Caputi bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1504eb633035STom Caputi dlrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1505eb633035STom Caputi sizeof (blkptr_t));
1506eb633035STom Caputi bcopy(slrp + sizeof (lr_write_t) - sizeof (blkptr_t),
1507eb633035STom Caputi aadp, sizeof (blkptr_t));
1508eb633035STom Caputi aadp += sizeof (blkptr_t);
1509eb633035STom Caputi aad_len += sizeof (blkptr_t);
1510eb633035STom Caputi nr_iovecs++;
1511eb633035STom Caputi total_len += crypt_len;
1512eb633035STom Caputi
1513eb633035STom Caputi if (lr_len != sizeof (lr_write_t)) {
1514eb633035STom Caputi crypt_len = lr_len - sizeof (lr_write_t);
1515eb633035STom Caputi src_iovecs[nr_iovecs].iov_base = (char *)
1516eb633035STom Caputi slrp + sizeof (lr_write_t);
1517eb633035STom Caputi src_iovecs[nr_iovecs].iov_len = crypt_len;
1518eb633035STom Caputi dst_iovecs[nr_iovecs].iov_base = (char *)
1519eb633035STom Caputi dlrp + sizeof (lr_write_t);
1520eb633035STom Caputi dst_iovecs[nr_iovecs].iov_len = crypt_len;
1521eb633035STom Caputi nr_iovecs++;
1522eb633035STom Caputi total_len += crypt_len;
1523eb633035STom Caputi }
1524eb633035STom Caputi } else {
1525eb633035STom Caputi crypt_len = lr_len - sizeof (lr_t);
1526eb633035STom Caputi src_iovecs[nr_iovecs].iov_base = (char *)slrp +
1527eb633035STom Caputi sizeof (lr_t);
1528eb633035STom Caputi src_iovecs[nr_iovecs].iov_len = crypt_len;
1529eb633035STom Caputi dst_iovecs[nr_iovecs].iov_base = (char *)dlrp +
1530eb633035STom Caputi sizeof (lr_t);
1531eb633035STom Caputi dst_iovecs[nr_iovecs].iov_len = crypt_len;
1532eb633035STom Caputi nr_iovecs++;
1533eb633035STom Caputi total_len += crypt_len;
1534eb633035STom Caputi }
1535eb633035STom Caputi }
1536eb633035STom Caputi
1537eb633035STom Caputi *no_crypt = (nr_iovecs == 0);
1538eb633035STom Caputi *enc_len = total_len;
1539eb633035STom Caputi *authbuf = aadbuf;
1540eb633035STom Caputi *auth_len = aad_len;
1541eb633035STom Caputi
1542eb633035STom Caputi if (encrypt) {
1543eb633035STom Caputi puio->uio_iov = src_iovecs;
1544eb633035STom Caputi puio->uio_iovcnt = nr_src;
1545eb633035STom Caputi cuio->uio_iov = dst_iovecs;
1546eb633035STom Caputi cuio->uio_iovcnt = nr_dst;
1547eb633035STom Caputi } else {
1548eb633035STom Caputi puio->uio_iov = dst_iovecs;
1549eb633035STom Caputi puio->uio_iovcnt = nr_dst;
1550eb633035STom Caputi cuio->uio_iov = src_iovecs;
1551eb633035STom Caputi cuio->uio_iovcnt = nr_src;
1552eb633035STom Caputi }
1553eb633035STom Caputi
1554eb633035STom Caputi return (0);
1555eb633035STom Caputi
1556eb633035STom Caputi error:
1557eb633035STom Caputi zio_buf_free(aadbuf, datalen);
1558eb633035STom Caputi if (src_iovecs != NULL)
1559eb633035STom Caputi kmem_free(src_iovecs, nr_src * sizeof (iovec_t));
1560eb633035STom Caputi if (dst_iovecs != NULL)
1561eb633035STom Caputi kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t));
1562eb633035STom Caputi
1563eb633035STom Caputi *enc_len = 0;
1564eb633035STom Caputi *authbuf = NULL;
1565eb633035STom Caputi *auth_len = 0;
1566eb633035STom Caputi *no_crypt = B_FALSE;
1567eb633035STom Caputi puio->uio_iov = NULL;
1568eb633035STom Caputi puio->uio_iovcnt = 0;
1569eb633035STom Caputi cuio->uio_iov = NULL;
1570eb633035STom Caputi cuio->uio_iovcnt = 0;
1571eb633035STom Caputi return (ret);
1572eb633035STom Caputi }
1573eb633035STom Caputi
1574eb633035STom Caputi /*
1575eb633035STom Caputi * Special case handling routine for encrypting / decrypting dnode blocks.
1576eb633035STom Caputi */
1577eb633035STom Caputi static int
zio_crypt_init_uios_dnode(boolean_t encrypt,uint64_t version,uint8_t * plainbuf,uint8_t * cipherbuf,uint_t datalen,boolean_t byteswap,uio_t * puio,uio_t * cuio,uint_t * enc_len,uint8_t ** authbuf,uint_t * auth_len,boolean_t * no_crypt)1578eb633035STom Caputi zio_crypt_init_uios_dnode(boolean_t encrypt, uint64_t version,
1579eb633035STom Caputi uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap,
1580eb633035STom Caputi uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf,
1581eb633035STom Caputi uint_t *auth_len, boolean_t *no_crypt)
1582eb633035STom Caputi {
1583eb633035STom Caputi int ret;
1584eb633035STom Caputi uint_t nr_src, nr_dst, crypt_len;
1585eb633035STom Caputi uint_t aad_len = 0, nr_iovecs = 0, total_len = 0;
1586eb633035STom Caputi uint_t i, j, max_dnp = datalen >> DNODE_SHIFT;
1587eb633035STom Caputi iovec_t *src_iovecs = NULL, *dst_iovecs = NULL;
1588eb633035STom Caputi uint8_t *src, *dst, *aadp;
1589eb633035STom Caputi dnode_phys_t *dnp, *adnp, *sdnp, *ddnp;
1590eb633035STom Caputi uint8_t *aadbuf = zio_buf_alloc(datalen);
1591eb633035STom Caputi
1592eb633035STom Caputi if (encrypt) {
1593eb633035STom Caputi src = plainbuf;
1594eb633035STom Caputi dst = cipherbuf;
1595eb633035STom Caputi nr_src = 0;
1596eb633035STom Caputi nr_dst = 1;
1597eb633035STom Caputi } else {
1598eb633035STom Caputi src = cipherbuf;
1599eb633035STom Caputi dst = plainbuf;
1600eb633035STom Caputi nr_src = 1;
1601eb633035STom Caputi nr_dst = 0;
1602eb633035STom Caputi }
1603eb633035STom Caputi
1604eb633035STom Caputi sdnp = (dnode_phys_t *)src;
1605eb633035STom Caputi ddnp = (dnode_phys_t *)dst;
1606eb633035STom Caputi aadp = aadbuf;
1607eb633035STom Caputi
1608eb633035STom Caputi /*
1609eb633035STom Caputi * Count the number of iovecs we will need to do the encryption by
1610eb633035STom Caputi * counting the number of bonus buffers that need to be encrypted.
1611eb633035STom Caputi */
1612eb633035STom Caputi for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
1613eb633035STom Caputi /*
1614eb633035STom Caputi * This block may still be byteswapped. However, all of the
1615eb633035STom Caputi * values we use are either uint8_t's (for which byteswapping
1616eb633035STom Caputi * is a noop) or a * != 0 check, which will work regardless
1617eb633035STom Caputi * of whether or not we byteswap.
1618eb633035STom Caputi */
1619eb633035STom Caputi if (sdnp[i].dn_type != DMU_OT_NONE &&
1620eb633035STom Caputi DMU_OT_IS_ENCRYPTED(sdnp[i].dn_bonustype) &&
1621eb633035STom Caputi sdnp[i].dn_bonuslen != 0) {
1622eb633035STom Caputi nr_iovecs++;
1623eb633035STom Caputi }
1624eb633035STom Caputi }
1625eb633035STom Caputi
1626eb633035STom Caputi nr_src += nr_iovecs;
1627eb633035STom Caputi nr_dst += nr_iovecs;
1628eb633035STom Caputi
1629eb633035STom Caputi if (nr_src != 0) {
1630eb633035STom Caputi src_iovecs = kmem_alloc(nr_src * sizeof (iovec_t), KM_SLEEP);
1631eb633035STom Caputi if (src_iovecs == NULL) {
1632eb633035STom Caputi ret = SET_ERROR(ENOMEM);
1633eb633035STom Caputi goto error;
1634eb633035STom Caputi }
1635eb633035STom Caputi }
1636eb633035STom Caputi
1637eb633035STom Caputi if (nr_dst != 0) {
1638eb633035STom Caputi dst_iovecs = kmem_alloc(nr_dst * sizeof (iovec_t), KM_SLEEP);
1639eb633035STom Caputi if (dst_iovecs == NULL) {
1640eb633035STom Caputi ret = SET_ERROR(ENOMEM);
1641eb633035STom Caputi goto error;
1642eb633035STom Caputi }
1643eb633035STom Caputi }
1644eb633035STom Caputi
1645eb633035STom Caputi nr_iovecs = 0;
1646eb633035STom Caputi
1647eb633035STom Caputi /*
1648eb633035STom Caputi * Iterate through the dnodes again, this time filling in the uios
1649eb633035STom Caputi * we allocated earlier. We also concatenate any data we want to
1650eb633035STom Caputi * authenticate onto aadbuf.
1651eb633035STom Caputi */
1652eb633035STom Caputi for (i = 0; i < max_dnp; i += sdnp[i].dn_extra_slots + 1) {
1653eb633035STom Caputi dnp = &sdnp[i];
1654eb633035STom Caputi /* copy over the core fields and blkptrs (kept as plaintext) */
1655eb633035STom Caputi bcopy(dnp, &ddnp[i], (uint8_t *)DN_BONUS(dnp) - (uint8_t *)dnp);
1656eb633035STom Caputi if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1657eb633035STom Caputi bcopy(DN_SPILL_BLKPTR(dnp), DN_SPILL_BLKPTR(&ddnp[i]),
1658eb633035STom Caputi sizeof (blkptr_t));
1659eb633035STom Caputi }
1660eb633035STom Caputi
1661eb633035STom Caputi /*
1662eb633035STom Caputi * Handle authenticated data. We authenticate everything in
1663eb633035STom Caputi * the dnode that can be brought over when we do a raw send.
1664eb633035STom Caputi * This includes all of the core fields as well as the MACs
1665eb633035STom Caputi * stored in the bp checksums and all of the portable bits
1666eb633035STom Caputi * from blk_prop. We include the dnode padding here in case it
1667eb633035STom Caputi * ever gets used in the future. Some dn_flags and dn_used are
1668eb633035STom Caputi * not portable so we mask those out values out of the
1669eb633035STom Caputi * authenticated data.
1670eb633035STom Caputi */
1671eb633035STom Caputi crypt_len = offsetof(dnode_phys_t, dn_blkptr);
1672eb633035STom Caputi bcopy(dnp, aadp, crypt_len);
1673eb633035STom Caputi adnp = (dnode_phys_t *)aadp;
1674eb633035STom Caputi adnp->dn_flags &= DNODE_CRYPT_PORTABLE_FLAGS_MASK;
1675eb633035STom Caputi adnp->dn_used = 0;
1676eb633035STom Caputi aadp += crypt_len;
1677eb633035STom Caputi aad_len += crypt_len;
1678eb633035STom Caputi
1679eb633035STom Caputi for (j = 0; j < dnp->dn_nblkptr; j++) {
1680eb633035STom Caputi zio_crypt_bp_do_aad_updates(&aadp, &aad_len,
1681eb633035STom Caputi version, byteswap, &dnp->dn_blkptr[j]);
1682eb633035STom Caputi }
1683eb633035STom Caputi
1684eb633035STom Caputi if (dnp->dn_flags & DNODE_FLAG_SPILL_BLKPTR) {
1685eb633035STom Caputi zio_crypt_bp_do_aad_updates(&aadp, &aad_len,
1686eb633035STom Caputi version, byteswap, DN_SPILL_BLKPTR(dnp));
1687eb633035STom Caputi }
1688eb633035STom Caputi
1689eb633035STom Caputi /*
1690eb633035STom Caputi * If this bonus buffer needs to be encrypted, we prepare an
1691eb633035STom Caputi * iovec_t. The encryption / decryption functions will fill
1692eb633035STom Caputi * this in for us with the encrypted or decrypted data.
1693eb633035STom Caputi * Otherwise we add the bonus buffer to the authenticated
1694eb633035STom Caputi * data buffer and copy it over to the destination. The
1695eb633035STom Caputi * encrypted iovec extends to DN_MAX_BONUS_LEN(dnp) so that
1696eb633035STom Caputi * we can guarantee alignment with the AES block size
1697eb633035STom Caputi * (128 bits).
1698eb633035STom Caputi */
1699eb633035STom Caputi crypt_len = DN_MAX_BONUS_LEN(dnp);
1700eb633035STom Caputi if (dnp->dn_type != DMU_OT_NONE &&
1701eb633035STom Caputi DMU_OT_IS_ENCRYPTED(dnp->dn_bonustype) &&
1702eb633035STom Caputi dnp->dn_bonuslen != 0) {
1703eb633035STom Caputi ASSERT3U(nr_iovecs, <, nr_src);
1704eb633035STom Caputi ASSERT3U(nr_iovecs, <, nr_dst);
1705eb633035STom Caputi ASSERT3P(src_iovecs, !=, NULL);
1706eb633035STom Caputi ASSERT3P(dst_iovecs, !=, NULL);
1707eb633035STom Caputi src_iovecs[nr_iovecs].iov_base = DN_BONUS(dnp);
1708eb633035STom Caputi src_iovecs[nr_iovecs].iov_len = crypt_len;
1709eb633035STom Caputi dst_iovecs[nr_iovecs].iov_base = DN_BONUS(&ddnp[i]);
1710eb633035STom Caputi dst_iovecs[nr_iovecs].iov_len = crypt_len;
1711eb633035STom Caputi
1712eb633035STom Caputi nr_iovecs++;
1713eb633035STom Caputi total_len += crypt_len;
1714eb633035STom Caputi } else {
1715eb633035STom Caputi bcopy(DN_BONUS(dnp), DN_BONUS(&ddnp[i]), crypt_len);
1716eb633035STom Caputi bcopy(DN_BONUS(dnp), aadp, crypt_len);
1717eb633035STom Caputi aadp += crypt_len;
1718eb633035STom Caputi aad_len += crypt_len;
1719eb633035STom Caputi }
1720eb633035STom Caputi }
1721eb633035STom Caputi
1722eb633035STom Caputi *no_crypt = (nr_iovecs == 0);
1723eb633035STom Caputi *enc_len = total_len;
1724eb633035STom Caputi *authbuf = aadbuf;
1725eb633035STom Caputi *auth_len = aad_len;
1726eb633035STom Caputi
1727eb633035STom Caputi if (encrypt) {
1728eb633035STom Caputi puio->uio_iov = src_iovecs;
1729eb633035STom Caputi puio->uio_iovcnt = nr_src;
1730eb633035STom Caputi cuio->uio_iov = dst_iovecs;
1731eb633035STom Caputi cuio->uio_iovcnt = nr_dst;
1732eb633035STom Caputi } else {
1733eb633035STom Caputi puio->uio_iov = dst_iovecs;
1734eb633035STom Caputi puio->uio_iovcnt = nr_dst;
1735eb633035STom Caputi cuio->uio_iov = src_iovecs;
1736eb633035STom Caputi cuio->uio_iovcnt = nr_src;
1737eb633035STom Caputi }
1738eb633035STom Caputi
1739eb633035STom Caputi return (0);
1740eb633035STom Caputi
1741eb633035STom Caputi error:
1742eb633035STom Caputi zio_buf_free(aadbuf, datalen);
1743eb633035STom Caputi if (src_iovecs != NULL)
1744eb633035STom Caputi kmem_free(src_iovecs, nr_src * sizeof (iovec_t));
1745eb633035STom Caputi if (dst_iovecs != NULL)
1746eb633035STom Caputi kmem_free(dst_iovecs, nr_dst * sizeof (iovec_t));
1747eb633035STom Caputi
1748eb633035STom Caputi *enc_len = 0;
1749eb633035STom Caputi *authbuf = NULL;
1750eb633035STom Caputi *auth_len = 0;
1751eb633035STom Caputi *no_crypt = B_FALSE;
1752eb633035STom Caputi puio->uio_iov = NULL;
1753eb633035STom Caputi puio->uio_iovcnt = 0;
1754eb633035STom Caputi cuio->uio_iov = NULL;
1755eb633035STom Caputi cuio->uio_iovcnt = 0;
1756eb633035STom Caputi return (ret);
1757eb633035STom Caputi }
1758eb633035STom Caputi
1759eb633035STom Caputi /* ARGSUSED */
1760eb633035STom Caputi static int
zio_crypt_init_uios_normal(boolean_t encrypt,uint8_t * plainbuf,uint8_t * cipherbuf,uint_t datalen,uio_t * puio,uio_t * cuio,uint_t * enc_len)1761eb633035STom Caputi zio_crypt_init_uios_normal(boolean_t encrypt, uint8_t *plainbuf,
1762eb633035STom Caputi uint8_t *cipherbuf, uint_t datalen, uio_t *puio, uio_t *cuio,
1763eb633035STom Caputi uint_t *enc_len)
1764eb633035STom Caputi {
1765eb633035STom Caputi int ret;
1766eb633035STom Caputi uint_t nr_plain = 1, nr_cipher = 2;
1767eb633035STom Caputi iovec_t *plain_iovecs = NULL, *cipher_iovecs = NULL;
1768eb633035STom Caputi
1769eb633035STom Caputi /* allocate the iovecs for the plain and cipher data */
1770eb633035STom Caputi plain_iovecs = kmem_alloc(nr_plain * sizeof (iovec_t),
1771eb633035STom Caputi KM_SLEEP);
1772eb633035STom Caputi if (!plain_iovecs) {
1773eb633035STom Caputi ret = SET_ERROR(ENOMEM);
1774eb633035STom Caputi goto error;
1775eb633035STom Caputi }
1776eb633035STom Caputi
1777eb633035STom Caputi cipher_iovecs = kmem_alloc(nr_cipher * sizeof (iovec_t),
1778eb633035STom Caputi KM_SLEEP);
1779eb633035STom Caputi if (!cipher_iovecs) {
1780eb633035STom Caputi ret = SET_ERROR(ENOMEM);
1781eb633035STom Caputi goto error;
1782eb633035STom Caputi }
1783eb633035STom Caputi
1784eb633035STom Caputi plain_iovecs[0].iov_base = (void *)plainbuf;
1785eb633035STom Caputi plain_iovecs[0].iov_len = datalen;
1786eb633035STom Caputi cipher_iovecs[0].iov_base = (void *)cipherbuf;
1787eb633035STom Caputi cipher_iovecs[0].iov_len = datalen;
1788eb633035STom Caputi
1789eb633035STom Caputi *enc_len = datalen;
1790eb633035STom Caputi puio->uio_iov = plain_iovecs;
1791eb633035STom Caputi puio->uio_iovcnt = nr_plain;
1792eb633035STom Caputi cuio->uio_iov = cipher_iovecs;
1793eb633035STom Caputi cuio->uio_iovcnt = nr_cipher;
1794eb633035STom Caputi
1795eb633035STom Caputi return (0);
1796eb633035STom Caputi
1797eb633035STom Caputi error:
1798eb633035STom Caputi if (plain_iovecs != NULL)
1799eb633035STom Caputi kmem_free(plain_iovecs, nr_plain * sizeof (iovec_t));
1800eb633035STom Caputi if (cipher_iovecs != NULL)
1801eb633035STom Caputi kmem_free(cipher_iovecs, nr_cipher * sizeof (iovec_t));
1802eb633035STom Caputi
1803eb633035STom Caputi *enc_len = 0;
1804eb633035STom Caputi puio->uio_iov = NULL;
1805eb633035STom Caputi puio->uio_iovcnt = 0;
1806eb633035STom Caputi cuio->uio_iov = NULL;
1807eb633035STom Caputi cuio->uio_iovcnt = 0;
1808eb633035STom Caputi return (ret);
1809eb633035STom Caputi }
1810eb633035STom Caputi
1811eb633035STom Caputi /*
1812eb633035STom Caputi * This function builds up the plaintext (puio) and ciphertext (cuio) uios so
1813eb633035STom Caputi * that they can be used for encryption and decryption by zio_do_crypt_uio().
1814eb633035STom Caputi * Most blocks will use zio_crypt_init_uios_normal(), with ZIL and dnode blocks
1815eb633035STom Caputi * requiring special handling to parse out pieces that are to be encrypted. The
1816eb633035STom Caputi * authbuf is used by these special cases to store additional authenticated
1817eb633035STom Caputi * data (AAD) for the encryption modes.
1818eb633035STom Caputi */
1819eb633035STom Caputi /* ARGSUSED */
1820eb633035STom Caputi static int
zio_crypt_init_uios(boolean_t encrypt,uint64_t version,dmu_object_type_t ot,uint8_t * plainbuf,uint8_t * cipherbuf,uint_t datalen,boolean_t byteswap,uint8_t * mac,uio_t * puio,uio_t * cuio,uint_t * enc_len,uint8_t ** authbuf,uint_t * auth_len,boolean_t * no_crypt)1821eb633035STom Caputi zio_crypt_init_uios(boolean_t encrypt, uint64_t version, dmu_object_type_t ot,
1822eb633035STom Caputi uint8_t *plainbuf, uint8_t *cipherbuf, uint_t datalen, boolean_t byteswap,
1823eb633035STom Caputi uint8_t *mac, uio_t *puio, uio_t *cuio, uint_t *enc_len, uint8_t **authbuf,
1824eb633035STom Caputi uint_t *auth_len, boolean_t *no_crypt)
1825eb633035STom Caputi {
1826eb633035STom Caputi int ret;
1827eb633035STom Caputi iovec_t *mac_iov;
1828eb633035STom Caputi
1829eb633035STom Caputi ASSERT(DMU_OT_IS_ENCRYPTED(ot) || ot == DMU_OT_NONE);
1830eb633035STom Caputi
1831eb633035STom Caputi /* route to handler */
1832eb633035STom Caputi switch (ot) {
1833eb633035STom Caputi case DMU_OT_INTENT_LOG:
1834eb633035STom Caputi ret = zio_crypt_init_uios_zil(encrypt, plainbuf, cipherbuf,
1835eb633035STom Caputi datalen, byteswap, puio, cuio, enc_len, authbuf, auth_len,
1836eb633035STom Caputi no_crypt);
1837eb633035STom Caputi break;
1838eb633035STom Caputi case DMU_OT_DNODE:
1839eb633035STom Caputi ret = zio_crypt_init_uios_dnode(encrypt, version, plainbuf,
1840eb633035STom Caputi cipherbuf, datalen, byteswap, puio, cuio, enc_len, authbuf,
1841eb633035STom Caputi auth_len, no_crypt);
1842eb633035STom Caputi break;
1843eb633035STom Caputi default:
1844eb633035STom Caputi ret = zio_crypt_init_uios_normal(encrypt, plainbuf, cipherbuf,
1845eb633035STom Caputi datalen, puio, cuio, enc_len);
1846eb633035STom Caputi *authbuf = NULL;
1847eb633035STom Caputi *auth_len = 0;
1848eb633035STom Caputi *no_crypt = B_FALSE;
1849eb633035STom Caputi break;
1850eb633035STom Caputi }
1851eb633035STom Caputi
1852eb633035STom Caputi if (ret != 0)
1853eb633035STom Caputi goto error;
1854eb633035STom Caputi
1855eb633035STom Caputi /* populate the uios */
1856eb633035STom Caputi puio->uio_segflg = UIO_SYSSPACE;
1857eb633035STom Caputi cuio->uio_segflg = UIO_SYSSPACE;
1858eb633035STom Caputi
1859eb633035STom Caputi mac_iov = ((iovec_t *)&cuio->uio_iov[cuio->uio_iovcnt - 1]);
1860eb633035STom Caputi mac_iov->iov_base = (void *)mac;
1861eb633035STom Caputi mac_iov->iov_len = ZIO_DATA_MAC_LEN;
1862eb633035STom Caputi
1863eb633035STom Caputi return (0);
1864eb633035STom Caputi
1865eb633035STom Caputi error:
1866eb633035STom Caputi return (ret);
1867eb633035STom Caputi }
1868eb633035STom Caputi
1869eb633035STom Caputi /*
1870eb633035STom Caputi * Primary encryption / decryption entrypoint for zio data.
1871eb633035STom Caputi */
1872eb633035STom Caputi int
zio_do_crypt_data(boolean_t encrypt,zio_crypt_key_t * key,dmu_object_type_t ot,boolean_t byteswap,uint8_t * salt,uint8_t * iv,uint8_t * mac,uint_t datalen,uint8_t * plainbuf,uint8_t * cipherbuf,boolean_t * no_crypt)1873eb633035STom Caputi zio_do_crypt_data(boolean_t encrypt, zio_crypt_key_t *key,
1874eb633035STom Caputi dmu_object_type_t ot, boolean_t byteswap, uint8_t *salt, uint8_t *iv,
1875eb633035STom Caputi uint8_t *mac, uint_t datalen, uint8_t *plainbuf, uint8_t *cipherbuf,
1876eb633035STom Caputi boolean_t *no_crypt)
1877eb633035STom Caputi {
1878eb633035STom Caputi int ret;
1879eb633035STom Caputi boolean_t locked = B_FALSE;
1880eb633035STom Caputi uint64_t crypt = key->zk_crypt;
1881eb633035STom Caputi uint_t keydata_len = zio_crypt_table[crypt].ci_keylen;
1882eb633035STom Caputi uint_t enc_len, auth_len;
1883eb633035STom Caputi uio_t puio, cuio;
1884eb633035STom Caputi uint8_t enc_keydata[MASTER_KEY_MAX_LEN];
1885eb633035STom Caputi crypto_key_t tmp_ckey, *ckey = NULL;
1886eb633035STom Caputi crypto_ctx_template_t tmpl;
1887eb633035STom Caputi uint8_t *authbuf = NULL;
1888eb633035STom Caputi
1889eb633035STom Caputi bzero(&puio, sizeof (uio_t));
1890eb633035STom Caputi bzero(&cuio, sizeof (uio_t));
1891eb633035STom Caputi
1892eb633035STom Caputi /* create uios for encryption */
1893eb633035STom Caputi ret = zio_crypt_init_uios(encrypt, key->zk_version, ot, plainbuf,
1894eb633035STom Caputi cipherbuf, datalen, byteswap, mac, &puio, &cuio, &enc_len,
1895eb633035STom Caputi &authbuf, &auth_len, no_crypt);
1896eb633035STom Caputi if (ret != 0)
1897eb633035STom Caputi return (ret);
1898eb633035STom Caputi
1899eb633035STom Caputi /*
1900eb633035STom Caputi * If the needed key is the current one, just use it. Otherwise we
1901eb633035STom Caputi * need to generate a temporary one from the given salt + master key.
1902eb633035STom Caputi * If we are encrypting, we must return a copy of the current salt
1903eb633035STom Caputi * so that it can be stored in the blkptr_t.
1904eb633035STom Caputi */
1905eb633035STom Caputi rw_enter(&key->zk_salt_lock, RW_READER);
1906eb633035STom Caputi locked = B_TRUE;
1907eb633035STom Caputi
1908eb633035STom Caputi if (bcmp(salt, key->zk_salt, ZIO_DATA_SALT_LEN) == 0) {
1909eb633035STom Caputi ckey = &key->zk_current_key;
1910eb633035STom Caputi tmpl = key->zk_current_tmpl;
1911eb633035STom Caputi } else {
1912eb633035STom Caputi rw_exit(&key->zk_salt_lock);
1913eb633035STom Caputi locked = B_FALSE;
1914eb633035STom Caputi
1915eb633035STom Caputi ret = hkdf_sha512(key->zk_master_keydata, keydata_len, NULL, 0,
1916eb633035STom Caputi salt, ZIO_DATA_SALT_LEN, enc_keydata, keydata_len);
1917eb633035STom Caputi if (ret != 0)
1918eb633035STom Caputi goto error;
1919eb633035STom Caputi
1920eb633035STom Caputi tmp_ckey.ck_format = CRYPTO_KEY_RAW;
1921eb633035STom Caputi tmp_ckey.ck_data = enc_keydata;
1922eb633035STom Caputi tmp_ckey.ck_length = CRYPTO_BYTES2BITS(keydata_len);
1923eb633035STom Caputi
1924eb633035STom Caputi ckey = &tmp_ckey;
1925eb633035STom Caputi tmpl = NULL;
1926eb633035STom Caputi }
1927eb633035STom Caputi
1928eb633035STom Caputi /* perform the encryption / decryption */
1929eb633035STom Caputi ret = zio_do_crypt_uio(encrypt, key->zk_crypt, ckey, tmpl, iv, enc_len,
1930eb633035STom Caputi &puio, &cuio, authbuf, auth_len);
1931eb633035STom Caputi if (ret != 0)
1932eb633035STom Caputi goto error;
1933eb633035STom Caputi
1934eb633035STom Caputi if (locked) {
1935eb633035STom Caputi rw_exit(&key->zk_salt_lock);
1936eb633035STom Caputi locked = B_FALSE;
1937eb633035STom Caputi }
1938eb633035STom Caputi
1939eb633035STom Caputi if (authbuf != NULL)
1940eb633035STom Caputi zio_buf_free(authbuf, datalen);
1941eb633035STom Caputi if (ckey == &tmp_ckey)
1942eb633035STom Caputi bzero(enc_keydata, keydata_len);
1943eb633035STom Caputi zio_crypt_destroy_uio(&puio);
1944eb633035STom Caputi zio_crypt_destroy_uio(&cuio);
1945eb633035STom Caputi
1946eb633035STom Caputi return (0);
1947eb633035STom Caputi
1948eb633035STom Caputi error:
1949eb633035STom Caputi if (!encrypt) {
1950eb633035STom Caputi if (failed_decrypt_buf != NULL)
1951eb633035STom Caputi kmem_free(failed_decrypt_buf, failed_decrypt_size);
1952eb633035STom Caputi failed_decrypt_buf = kmem_alloc(datalen, KM_SLEEP);
1953eb633035STom Caputi failed_decrypt_size = datalen;
1954eb633035STom Caputi bcopy(cipherbuf, failed_decrypt_buf, datalen);
1955eb633035STom Caputi }
1956eb633035STom Caputi if (locked)
1957eb633035STom Caputi rw_exit(&key->zk_salt_lock);
1958eb633035STom Caputi if (authbuf != NULL)
1959eb633035STom Caputi zio_buf_free(authbuf, datalen);
1960eb633035STom Caputi if (ckey == &tmp_ckey)
1961eb633035STom Caputi bzero(enc_keydata, keydata_len);
1962eb633035STom Caputi zio_crypt_destroy_uio(&puio);
1963eb633035STom Caputi zio_crypt_destroy_uio(&cuio);
1964eb633035STom Caputi
1965eb633035STom Caputi return (ret);
1966eb633035STom Caputi }
1967eb633035STom Caputi
1968eb633035STom Caputi /*
1969eb633035STom Caputi * Simple wrapper around zio_do_crypt_data() to work with abd's instead of
1970eb633035STom Caputi * linear buffers.
1971eb633035STom Caputi */
1972eb633035STom Caputi int
zio_do_crypt_abd(boolean_t encrypt,zio_crypt_key_t * key,dmu_object_type_t ot,boolean_t byteswap,uint8_t * salt,uint8_t * iv,uint8_t * mac,uint_t datalen,abd_t * pabd,abd_t * cabd,boolean_t * no_crypt)1973eb633035STom Caputi zio_do_crypt_abd(boolean_t encrypt, zio_crypt_key_t *key, dmu_object_type_t ot,
1974eb633035STom Caputi boolean_t byteswap, uint8_t *salt, uint8_t *iv, uint8_t *mac,
1975eb633035STom Caputi uint_t datalen, abd_t *pabd, abd_t *cabd, boolean_t *no_crypt)
1976eb633035STom Caputi {
1977eb633035STom Caputi int ret;
1978eb633035STom Caputi void *ptmp, *ctmp;
1979eb633035STom Caputi
1980eb633035STom Caputi if (encrypt) {
1981eb633035STom Caputi ptmp = abd_borrow_buf_copy(pabd, datalen);
1982eb633035STom Caputi ctmp = abd_borrow_buf(cabd, datalen);
1983eb633035STom Caputi } else {
1984eb633035STom Caputi ptmp = abd_borrow_buf(pabd, datalen);
1985eb633035STom Caputi ctmp = abd_borrow_buf_copy(cabd, datalen);
1986eb633035STom Caputi }
1987eb633035STom Caputi
1988eb633035STom Caputi ret = zio_do_crypt_data(encrypt, key, ot, byteswap, salt, iv, mac,
1989eb633035STom Caputi datalen, ptmp, ctmp, no_crypt);
1990eb633035STom Caputi if (ret != 0)
1991eb633035STom Caputi goto error;
1992eb633035STom Caputi
1993eb633035STom Caputi if (encrypt) {
1994eb633035STom Caputi abd_return_buf(pabd, ptmp, datalen);
1995eb633035STom Caputi abd_return_buf_copy(cabd, ctmp, datalen);
1996eb633035STom Caputi } else {
1997eb633035STom Caputi abd_return_buf_copy(pabd, ptmp, datalen);
1998eb633035STom Caputi abd_return_buf(cabd, ctmp, datalen);
1999eb633035STom Caputi }
2000eb633035STom Caputi
2001eb633035STom Caputi return (0);
2002eb633035STom Caputi
2003eb633035STom Caputi error:
2004eb633035STom Caputi if (encrypt) {
2005eb633035STom Caputi abd_return_buf(pabd, ptmp, datalen);
2006eb633035STom Caputi abd_return_buf_copy(cabd, ctmp, datalen);
2007eb633035STom Caputi } else {
2008eb633035STom Caputi abd_return_buf_copy(pabd, ptmp, datalen);
2009eb633035STom Caputi abd_return_buf(cabd, ctmp, datalen);
2010eb633035STom Caputi }
2011eb633035STom Caputi
2012eb633035STom Caputi return (ret);
2013eb633035STom Caputi }
2014