1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2017 Nexenta Systems, Inc.  All rights reserved.
24 * Copyright 2019 Joyent, Inc.
25 */
26
27/*
28 * AES provider for the Kernel Cryptographic Framework (KCF)
29 */
30
31#include <sys/types.h>
32#include <sys/systm.h>
33#include <sys/modctl.h>
34#include <sys/cmn_err.h>
35#include <sys/ddi.h>
36#include <sys/crypto/common.h>
37#include <sys/crypto/impl.h>
38#include <sys/crypto/spi.h>
39#include <sys/sysmacros.h>
40#include <sys/strsun.h>
41#include <modes/modes.h>
42#define	_AES_IMPL
43#include <aes/aes_impl.h>
44
45extern struct mod_ops mod_cryptoops;
46
47/*
48 * Module linkage information for the kernel.
49 */
50static struct modlcrypto modlcrypto = {
51	&mod_cryptoops,
52	"AES Kernel SW Provider"
53};
54
55static struct modlinkage modlinkage = {
56	MODREV_1,
57	(void *)&modlcrypto,
58	NULL
59};
60
61/*
62 * Mechanism info structure passed to KCF during registration.
63 */
64static crypto_mech_info_t aes_mech_info_tab[] = {
65	/* AES_ECB */
66	{SUN_CKM_AES_ECB, AES_ECB_MECH_INFO_TYPE,
67	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
68	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC,
69	    AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
70	/* AES_CBC */
71	{SUN_CKM_AES_CBC, AES_CBC_MECH_INFO_TYPE,
72	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
73	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC,
74	    AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
75	/* AES_CMAC */
76	{SUN_CKM_AES_CMAC, AES_CMAC_MECH_INFO_TYPE,
77	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
78	    CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC,
79	    AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
80	/* AES_CTR */
81	{SUN_CKM_AES_CTR, AES_CTR_MECH_INFO_TYPE,
82	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
83	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC,
84	    AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
85	/* AES_CCM */
86	{SUN_CKM_AES_CCM, AES_CCM_MECH_INFO_TYPE,
87	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
88	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC,
89	    AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
90	/* AES_GCM */
91	{SUN_CKM_AES_GCM, AES_GCM_MECH_INFO_TYPE,
92	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
93	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC,
94	    AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES},
95	/* AES_GMAC */
96	{SUN_CKM_AES_GMAC, AES_GMAC_MECH_INFO_TYPE,
97	    CRYPTO_FG_ENCRYPT | CRYPTO_FG_ENCRYPT_ATOMIC |
98	    CRYPTO_FG_DECRYPT | CRYPTO_FG_DECRYPT_ATOMIC |
99	    CRYPTO_FG_MAC | CRYPTO_FG_MAC_ATOMIC |
100	    CRYPTO_FG_SIGN | CRYPTO_FG_SIGN_ATOMIC |
101	    CRYPTO_FG_VERIFY | CRYPTO_FG_VERIFY_ATOMIC,
102	    AES_MIN_KEY_BYTES, AES_MAX_KEY_BYTES, CRYPTO_KEYSIZE_UNIT_IN_BYTES}
103};
104
105/* operations are in-place if the output buffer is NULL */
106#define	AES_ARG_INPLACE(input, output)				\
107	if ((output) == NULL)					\
108		(output) = (input);
109
110static void aes_provider_status(crypto_provider_handle_t, uint_t *);
111
112static crypto_control_ops_t aes_control_ops = {
113	aes_provider_status
114};
115
116static int aes_encrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
117    crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
118static int aes_decrypt_init(crypto_ctx_t *, crypto_mechanism_t *,
119    crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
120static int aes_common_init(crypto_ctx_t *, crypto_mechanism_t *,
121    crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t, boolean_t);
122static int aes_common_init_ctx(aes_ctx_t *, crypto_spi_ctx_template_t *,
123    crypto_mechanism_t *, crypto_key_t *, int, boolean_t);
124static int aes_encrypt_final(crypto_ctx_t *, crypto_data_t *,
125    crypto_req_handle_t);
126static int aes_decrypt_final(crypto_ctx_t *, crypto_data_t *,
127    crypto_req_handle_t);
128
129static int aes_encrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
130    crypto_req_handle_t);
131static int aes_encrypt_update(crypto_ctx_t *, crypto_data_t *,
132    crypto_data_t *, crypto_req_handle_t);
133static int aes_encrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
134    crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
135    crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
136
137static int aes_decrypt(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
138    crypto_req_handle_t);
139static int aes_decrypt_update(crypto_ctx_t *, crypto_data_t *,
140    crypto_data_t *, crypto_req_handle_t);
141static int aes_decrypt_atomic(crypto_provider_handle_t, crypto_session_id_t,
142    crypto_mechanism_t *, crypto_key_t *, crypto_data_t *,
143    crypto_data_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
144
145static crypto_cipher_ops_t aes_cipher_ops = {
146	aes_encrypt_init,
147	aes_encrypt,
148	aes_encrypt_update,
149	aes_encrypt_final,
150	aes_encrypt_atomic,
151	aes_decrypt_init,
152	aes_decrypt,
153	aes_decrypt_update,
154	aes_decrypt_final,
155	aes_decrypt_atomic
156};
157
158static int aes_mac_init(crypto_ctx_t *, crypto_mechanism_t *,
159    crypto_key_t *, crypto_spi_ctx_template_t, crypto_req_handle_t);
160static int aes_mac(crypto_ctx_t *, crypto_data_t *, crypto_data_t *,
161    crypto_req_handle_t);
162static int aes_mac_update(crypto_ctx_t *, crypto_data_t *,
163    crypto_req_handle_t);
164static int aes_mac_final(crypto_ctx_t *, crypto_data_t *,
165    crypto_req_handle_t);
166static int aes_mac_atomic(crypto_provider_handle_t, crypto_session_id_t,
167    crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
168    crypto_spi_ctx_template_t, crypto_req_handle_t);
169static int aes_mac_verify_atomic(crypto_provider_handle_t, crypto_session_id_t,
170    crypto_mechanism_t *, crypto_key_t *, crypto_data_t *, crypto_data_t *,
171    crypto_spi_ctx_template_t, crypto_req_handle_t);
172
173static crypto_mac_ops_t aes_mac_ops = {
174	aes_mac_init,
175	aes_mac,
176	aes_mac_update,
177	aes_mac_final,
178	aes_mac_atomic,
179	aes_mac_verify_atomic
180};
181
182static int aes_create_ctx_template(crypto_provider_handle_t,
183    crypto_mechanism_t *, crypto_key_t *, crypto_spi_ctx_template_t *,
184    size_t *, crypto_req_handle_t);
185static int aes_free_context(crypto_ctx_t *);
186
187static crypto_ctx_ops_t aes_ctx_ops = {
188	aes_create_ctx_template,
189	aes_free_context
190};
191
192static crypto_ops_t aes_crypto_ops = {
193	&aes_control_ops,
194	NULL,
195	&aes_cipher_ops,
196	&aes_mac_ops,
197	NULL,
198	NULL,
199	NULL,
200	NULL,
201	NULL,
202	NULL,
203	NULL,
204	NULL,
205	NULL,
206	&aes_ctx_ops,
207	NULL,
208	NULL,
209	NULL,
210};
211
212static crypto_provider_info_t aes_prov_info = {
213	CRYPTO_SPI_VERSION_4,
214	"AES Software Provider",
215	CRYPTO_SW_PROVIDER,
216	{&modlinkage},
217	NULL,
218	&aes_crypto_ops,
219	sizeof (aes_mech_info_tab)/sizeof (crypto_mech_info_t),
220	aes_mech_info_tab
221};
222
223static crypto_kcf_provider_handle_t aes_prov_handle = 0;
224static crypto_data_t null_crypto_data = { CRYPTO_DATA_RAW };
225
226int
227_init(void)
228{
229	int ret;
230
231	if ((ret = mod_install(&modlinkage)) != 0)
232		return (ret);
233
234	/* Register with KCF.  If the registration fails, remove the module. */
235	if (crypto_register_provider(&aes_prov_info, &aes_prov_handle)) {
236		(void) mod_remove(&modlinkage);
237		return (EACCES);
238	}
239
240	return (0);
241}
242
243int
244_fini(void)
245{
246	/* Unregister from KCF if module is registered */
247	if (aes_prov_handle != 0) {
248		if (crypto_unregister_provider(aes_prov_handle))
249			return (EBUSY);
250
251		aes_prov_handle = 0;
252	}
253
254	return (mod_remove(&modlinkage));
255}
256
257int
258_info(struct modinfo *modinfop)
259{
260	return (mod_info(&modlinkage, modinfop));
261}
262
263
264static int
265aes_check_mech_param(crypto_mechanism_t *mechanism, aes_ctx_t **ctx, int kmflag)
266{
267	void *p = NULL;
268	boolean_t param_required = B_TRUE;
269	size_t param_len;
270	void *(*alloc_fun)(int);
271	int rv = CRYPTO_SUCCESS;
272
273	switch (mechanism->cm_type) {
274	case AES_ECB_MECH_INFO_TYPE:
275		param_required = B_FALSE;
276		alloc_fun = ecb_alloc_ctx;
277		break;
278	case AES_CBC_MECH_INFO_TYPE:
279		param_len = AES_BLOCK_LEN;
280		alloc_fun = cbc_alloc_ctx;
281		break;
282	case AES_CMAC_MECH_INFO_TYPE:
283		param_required = B_FALSE;
284		alloc_fun = cmac_alloc_ctx;
285		break;
286	case AES_CTR_MECH_INFO_TYPE:
287		param_len = sizeof (CK_AES_CTR_PARAMS);
288		alloc_fun = ctr_alloc_ctx;
289		break;
290	case AES_CCM_MECH_INFO_TYPE:
291		param_len = sizeof (CK_AES_CCM_PARAMS);
292		alloc_fun = ccm_alloc_ctx;
293		break;
294	case AES_GCM_MECH_INFO_TYPE:
295		param_len = sizeof (CK_AES_GCM_PARAMS);
296		alloc_fun = gcm_alloc_ctx;
297		break;
298	case AES_GMAC_MECH_INFO_TYPE:
299		param_len = sizeof (CK_AES_GMAC_PARAMS);
300		alloc_fun = gmac_alloc_ctx;
301		break;
302	default:
303		rv = CRYPTO_MECHANISM_INVALID;
304		return (rv);
305	}
306	if (param_required && mechanism->cm_param != NULL &&
307	    mechanism->cm_param_len != param_len) {
308		rv = CRYPTO_MECHANISM_PARAM_INVALID;
309	}
310	if (ctx != NULL) {
311		p = (alloc_fun)(kmflag);
312		*ctx = p;
313	}
314	return (rv);
315}
316
317/*
318 * Initialize key schedules for AES
319 */
320static int
321init_keysched(crypto_key_t *key, void *newbie)
322{
323	/*
324	 * Only keys by value are supported by this module.
325	 */
326	switch (key->ck_format) {
327	case CRYPTO_KEY_RAW:
328		if (key->ck_length < AES_MINBITS ||
329		    key->ck_length > AES_MAXBITS) {
330			return (CRYPTO_KEY_SIZE_RANGE);
331		}
332
333		/* key length must be either 128, 192, or 256 */
334		if ((key->ck_length & 63) != 0)
335			return (CRYPTO_KEY_SIZE_RANGE);
336		break;
337	default:
338		return (CRYPTO_KEY_TYPE_INCONSISTENT);
339	}
340
341	aes_init_keysched(key->ck_data, key->ck_length, newbie);
342	return (CRYPTO_SUCCESS);
343}
344
345/*
346 * KCF software provider control entry points.
347 */
348/* ARGSUSED */
349static void
350aes_provider_status(crypto_provider_handle_t provider, uint_t *status)
351{
352	*status = CRYPTO_PROVIDER_READY;
353}
354
355static int
356aes_encrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
357    crypto_key_t *key, crypto_spi_ctx_template_t template,
358    crypto_req_handle_t req)
359{
360	return (aes_common_init(ctx, mechanism, key, template, req, B_TRUE));
361}
362
363static int
364aes_decrypt_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
365    crypto_key_t *key, crypto_spi_ctx_template_t template,
366    crypto_req_handle_t req)
367{
368	return (aes_common_init(ctx, mechanism, key, template, req, B_FALSE));
369}
370
371
372
373/*
374 * KCF software provider encrypt entry points.
375 */
376static int
377aes_common_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
378    crypto_key_t *key, crypto_spi_ctx_template_t template,
379    crypto_req_handle_t req, boolean_t is_encrypt_init)
380{
381	aes_ctx_t *aes_ctx;
382	int rv;
383	int kmflag;
384
385	/*
386	 * Only keys by value are supported by this module.
387	 */
388	if (key->ck_format != CRYPTO_KEY_RAW) {
389		return (CRYPTO_KEY_TYPE_INCONSISTENT);
390	}
391
392	kmflag = crypto_kmflag(req);
393	if ((rv = aes_check_mech_param(mechanism, &aes_ctx, kmflag))
394	    != CRYPTO_SUCCESS)
395		return (rv);
396
397	rv = aes_common_init_ctx(aes_ctx, template, mechanism, key, kmflag,
398	    is_encrypt_init);
399	if (rv != CRYPTO_SUCCESS) {
400		crypto_free_mode_ctx(aes_ctx);
401		return (rv);
402	}
403
404	ctx->cc_provider_private = aes_ctx;
405
406	return (CRYPTO_SUCCESS);
407}
408
409static int
410aes_encrypt(crypto_ctx_t *ctx, crypto_data_t *plaintext,
411    crypto_data_t *ciphertext, crypto_req_handle_t req)
412{
413	int ret = CRYPTO_FAILED;
414
415	aes_ctx_t *aes_ctx;
416	size_t saved_length, saved_offset, length_needed;
417
418	ASSERT(ctx->cc_provider_private != NULL);
419	aes_ctx = ctx->cc_provider_private;
420
421	/*
422	 * For block ciphers, plaintext must be a multiple of AES block size.
423	 * This test is only valid for ciphers whose blocksize is a power of 2.
424	 */
425	if (((aes_ctx->ac_flags & (CMAC_MODE|CTR_MODE|CCM_MODE|
426	    GCM_MODE|GMAC_MODE)) == 0) &&
427	    (plaintext->cd_length & (AES_BLOCK_LEN - 1)) != 0)
428		return (CRYPTO_DATA_LEN_RANGE);
429
430	AES_ARG_INPLACE(plaintext, ciphertext);
431
432	/*
433	 * We need to just return the length needed to store the output.
434	 * We should not destroy the context for the following case.
435	 */
436	switch (aes_ctx->ac_flags & (CMAC_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) {
437	case CCM_MODE:
438		length_needed = plaintext->cd_length + aes_ctx->ac_mac_len;
439		break;
440	case GCM_MODE:
441		length_needed = plaintext->cd_length + aes_ctx->ac_tag_len;
442		break;
443	case CMAC_MODE:
444		length_needed = AES_BLOCK_LEN;
445		break;
446	case GMAC_MODE:
447		if (plaintext->cd_length != 0)
448			return (CRYPTO_ARGUMENTS_BAD);
449
450		length_needed = aes_ctx->ac_tag_len;
451		break;
452	default:
453		length_needed = plaintext->cd_length;
454	}
455
456	if (ciphertext->cd_length < length_needed) {
457		ciphertext->cd_length = length_needed;
458		return (CRYPTO_BUFFER_TOO_SMALL);
459	}
460
461	saved_length = ciphertext->cd_length;
462	saved_offset = ciphertext->cd_offset;
463
464	/*
465	 * Do an update on the specified input data.
466	 */
467	ret = aes_encrypt_update(ctx, plaintext, ciphertext, req);
468	if (ret != CRYPTO_SUCCESS) {
469		return (ret);
470	}
471
472	/*
473	 * For CCM mode, aes_ccm_encrypt_final() will take care of any
474	 * left-over unprocessed data, and compute the MAC
475	 */
476	if (aes_ctx->ac_flags & CCM_MODE) {
477		/*
478		 * ccm_encrypt_final() will compute the MAC and append
479		 * it to existing ciphertext. So, need to adjust the left over
480		 * length value accordingly
481		 */
482
483		/* order of following 2 lines MUST not be reversed */
484		ciphertext->cd_offset = ciphertext->cd_length;
485		ciphertext->cd_length = saved_length - ciphertext->cd_length;
486		ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, ciphertext,
487		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
488		if (ret != CRYPTO_SUCCESS) {
489			return (ret);
490		}
491
492		if (plaintext != ciphertext) {
493			ciphertext->cd_length =
494			    ciphertext->cd_offset - saved_offset;
495		}
496		ciphertext->cd_offset = saved_offset;
497	} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
498		/*
499		 * gcm_encrypt_final() will compute the MAC and append
500		 * it to existing ciphertext. So, need to adjust the left over
501		 * length value accordingly
502		 */
503
504		/* order of following 2 lines MUST not be reversed */
505		ciphertext->cd_offset = ciphertext->cd_length;
506		ciphertext->cd_length = saved_length - ciphertext->cd_length;
507		ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, ciphertext,
508		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
509		    aes_xor_block);
510		if (ret != CRYPTO_SUCCESS) {
511			return (ret);
512		}
513
514		if (plaintext != ciphertext) {
515			ciphertext->cd_length =
516			    ciphertext->cd_offset - saved_offset;
517		}
518		ciphertext->cd_offset = saved_offset;
519	} else if (aes_ctx->ac_flags & CMAC_MODE) {
520		/* cmac_update doesn't store data */
521		ciphertext->cd_length = saved_length;
522		ret = cmac_mode_final((cbc_ctx_t *)aes_ctx, ciphertext,
523		    aes_encrypt_block, aes_xor_block);
524		aes_ctx->ac_remainder_len = 0;
525	}
526
527	ASSERT(aes_ctx->ac_remainder_len == 0);
528	(void) aes_free_context(ctx);
529
530	return (ret);
531}
532
533
534static int
535aes_decrypt(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
536    crypto_data_t *plaintext, crypto_req_handle_t req)
537{
538	int ret = CRYPTO_FAILED;
539
540	aes_ctx_t *aes_ctx;
541	off_t saved_offset;
542	size_t saved_length, length_needed;
543
544	ASSERT(ctx->cc_provider_private != NULL);
545	aes_ctx = ctx->cc_provider_private;
546
547	/*
548	 * For block ciphers, plaintext must be a multiple of AES block size.
549	 * This test is only valid for ciphers whose blocksize is a power of 2.
550	 */
551	if (((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE))
552	    == 0) && (ciphertext->cd_length & (AES_BLOCK_LEN - 1)) != 0) {
553		return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
554	}
555
556	AES_ARG_INPLACE(ciphertext, plaintext);
557
558	/*
559	 * Return length needed to store the output.
560	 * Do not destroy context when plaintext buffer is too small.
561	 *
562	 * CCM:  plaintext is MAC len smaller than cipher text
563	 * GCM:  plaintext is TAG len smaller than cipher text
564	 * GMAC: plaintext length must be zero
565	 */
566	switch (aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) {
567	case CCM_MODE:
568		length_needed = aes_ctx->ac_processed_data_len;
569		break;
570	case GCM_MODE:
571		length_needed = ciphertext->cd_length - aes_ctx->ac_tag_len;
572		break;
573	case GMAC_MODE:
574		if (plaintext->cd_length != 0)
575			return (CRYPTO_ARGUMENTS_BAD);
576
577		length_needed = 0;
578		break;
579	default:
580		length_needed = ciphertext->cd_length;
581	}
582
583	if (plaintext->cd_length < length_needed) {
584		plaintext->cd_length = length_needed;
585		return (CRYPTO_BUFFER_TOO_SMALL);
586	}
587
588	saved_offset = plaintext->cd_offset;
589	saved_length = plaintext->cd_length;
590
591	/*
592	 * Do an update on the specified input data.
593	 */
594	ret = aes_decrypt_update(ctx, ciphertext, plaintext, req);
595	if (ret != CRYPTO_SUCCESS) {
596		goto cleanup;
597	}
598
599	if (aes_ctx->ac_flags & CCM_MODE) {
600		ASSERT(aes_ctx->ac_processed_data_len == aes_ctx->ac_data_len);
601		ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len);
602
603		/* order of following 2 lines MUST not be reversed */
604		plaintext->cd_offset = plaintext->cd_length;
605		plaintext->cd_length = saved_length - plaintext->cd_length;
606
607		ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, plaintext,
608		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
609		    aes_xor_block);
610		if (ret == CRYPTO_SUCCESS) {
611			if (plaintext != ciphertext) {
612				plaintext->cd_length =
613				    plaintext->cd_offset - saved_offset;
614			}
615		} else {
616			plaintext->cd_length = saved_length;
617		}
618
619		plaintext->cd_offset = saved_offset;
620	} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
621		/* order of following 2 lines MUST not be reversed */
622		plaintext->cd_offset = plaintext->cd_length;
623		plaintext->cd_length = saved_length - plaintext->cd_length;
624
625		ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, plaintext,
626		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
627		if (ret == CRYPTO_SUCCESS) {
628			if (plaintext != ciphertext) {
629				plaintext->cd_length =
630				    plaintext->cd_offset - saved_offset;
631			}
632		} else {
633			plaintext->cd_length = saved_length;
634		}
635
636		plaintext->cd_offset = saved_offset;
637	}
638
639	ASSERT(aes_ctx->ac_remainder_len == 0);
640
641cleanup:
642	(void) aes_free_context(ctx);
643
644	return (ret);
645}
646
647
648/* ARGSUSED */
649static int
650aes_encrypt_update(crypto_ctx_t *ctx, crypto_data_t *plaintext,
651    crypto_data_t *ciphertext, crypto_req_handle_t req)
652{
653	off_t saved_offset;
654	size_t saved_length, out_len;
655	int ret = CRYPTO_SUCCESS;
656	aes_ctx_t *aes_ctx;
657
658	ASSERT(ctx->cc_provider_private != NULL);
659	aes_ctx = ctx->cc_provider_private;
660
661	AES_ARG_INPLACE(plaintext, ciphertext);
662
663	/*
664	 * CTR mode does not accumulate plaintext across xx_update() calls --
665	 * it always outputs the same number of bytes as the input (so
666	 * ac_remainder_len is always 0).  Other modes _do_ accumulate
667	 * plaintext, and output only full blocks. For non-CTR modes, adjust
668	 * the output size to reflect this.
669	 */
670	out_len = plaintext->cd_length + aes_ctx->ac_remainder_len;
671	if ((aes_ctx->ac_flags & CTR_MODE) == 0)
672		out_len &= ~(AES_BLOCK_LEN - 1);
673
674	/*
675	 * return length needed to store the output.
676	 * CMAC stores its output in a local buffer until *_final.
677	 */
678	if ((aes_ctx->ac_flags & CMAC_MODE) == 0 &&
679	    ciphertext->cd_length < out_len) {
680		ciphertext->cd_length = out_len;
681		return (CRYPTO_BUFFER_TOO_SMALL);
682	}
683
684	saved_offset = ciphertext->cd_offset;
685	saved_length = ciphertext->cd_length;
686
687	/*
688	 * Do the AES update on the specified input data.
689	 */
690	switch (plaintext->cd_format) {
691	case CRYPTO_DATA_RAW:
692		ret = crypto_update_iov(ctx->cc_provider_private,
693		    plaintext, ciphertext, aes_encrypt_contiguous_blocks,
694		    aes_copy_block64);
695		break;
696	case CRYPTO_DATA_UIO:
697		ret = crypto_update_uio(ctx->cc_provider_private,
698		    plaintext, ciphertext, aes_encrypt_contiguous_blocks,
699		    aes_copy_block64);
700		break;
701	case CRYPTO_DATA_MBLK:
702		ret = crypto_update_mp(ctx->cc_provider_private,
703		    plaintext, ciphertext, aes_encrypt_contiguous_blocks,
704		    aes_copy_block64);
705		break;
706	default:
707		ret = CRYPTO_ARGUMENTS_BAD;
708	}
709
710	if (ret == CRYPTO_SUCCESS) {
711		if (plaintext != ciphertext) {
712			ciphertext->cd_length =
713			    ciphertext->cd_offset - saved_offset;
714		}
715	} else {
716		ciphertext->cd_length = saved_length;
717	}
718	ciphertext->cd_offset = saved_offset;
719
720	return (ret);
721}
722
723
724static int
725aes_decrypt_update(crypto_ctx_t *ctx, crypto_data_t *ciphertext,
726    crypto_data_t *plaintext, crypto_req_handle_t req)
727{
728	off_t saved_offset;
729	size_t saved_length, out_len;
730	int ret = CRYPTO_SUCCESS;
731	aes_ctx_t *aes_ctx;
732
733	ASSERT(ctx->cc_provider_private != NULL);
734	aes_ctx = ctx->cc_provider_private;
735
736	AES_ARG_INPLACE(ciphertext, plaintext);
737
738	/*
739	 * Adjust the number of bytes that will hold the plaintext (out_len).
740	 * CCM, GCM, and GMAC mechanisms never return plaintext for update
741	 * operations, so we set out_len to 0 for those.
742	 *
743	 * CTR mode does not accumulate any ciphertext across xx_decrypt
744	 * calls, and always outputs as many bytes of plaintext as
745	 * ciphertext.
746	 *
747	 * The remaining mechanisms output full blocks of plaintext, so
748	 * we round out_len down to the closest multiple of AES_BLOCK_LEN.
749	 */
750	out_len = aes_ctx->ac_remainder_len + ciphertext->cd_length;
751	if ((aes_ctx->ac_flags & (CCM_MODE|GCM_MODE|GMAC_MODE)) != 0) {
752		out_len = 0;
753	} else if ((aes_ctx->ac_flags & CTR_MODE) == 0) {
754		out_len &= ~(AES_BLOCK_LEN - 1);
755	}
756
757	/* return length needed to store the output */
758	if (plaintext->cd_length < out_len) {
759		plaintext->cd_length = out_len;
760		return (CRYPTO_BUFFER_TOO_SMALL);
761	}
762
763	saved_offset = plaintext->cd_offset;
764	saved_length = plaintext->cd_length;
765
766	if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE))
767		gcm_set_kmflag((gcm_ctx_t *)aes_ctx, crypto_kmflag(req));
768
769	/*
770	 * Do the AES update on the specified input data.
771	 */
772	switch (ciphertext->cd_format) {
773	case CRYPTO_DATA_RAW:
774		ret = crypto_update_iov(ctx->cc_provider_private,
775		    ciphertext, plaintext, aes_decrypt_contiguous_blocks,
776		    aes_copy_block64);
777		break;
778	case CRYPTO_DATA_UIO:
779		ret = crypto_update_uio(ctx->cc_provider_private,
780		    ciphertext, plaintext, aes_decrypt_contiguous_blocks,
781		    aes_copy_block64);
782		break;
783	case CRYPTO_DATA_MBLK:
784		ret = crypto_update_mp(ctx->cc_provider_private,
785		    ciphertext, plaintext, aes_decrypt_contiguous_blocks,
786		    aes_copy_block64);
787		break;
788	default:
789		ret = CRYPTO_ARGUMENTS_BAD;
790	}
791
792	if (ret == CRYPTO_SUCCESS) {
793		if (ciphertext != plaintext)
794			plaintext->cd_length =
795			    plaintext->cd_offset - saved_offset;
796	} else {
797		plaintext->cd_length = saved_length;
798	}
799	plaintext->cd_offset = saved_offset;
800
801
802	return (ret);
803}
804
805/* ARGSUSED */
806static int
807aes_encrypt_final(crypto_ctx_t *ctx, crypto_data_t *data,
808    crypto_req_handle_t req)
809{
810	aes_ctx_t *aes_ctx;
811	int ret;
812
813	ASSERT(ctx->cc_provider_private != NULL);
814	aes_ctx = ctx->cc_provider_private;
815
816	if (data->cd_format != CRYPTO_DATA_RAW &&
817	    data->cd_format != CRYPTO_DATA_UIO &&
818	    data->cd_format != CRYPTO_DATA_MBLK) {
819		return (CRYPTO_ARGUMENTS_BAD);
820	}
821
822	if (aes_ctx->ac_flags & CCM_MODE) {
823		ret = ccm_encrypt_final((ccm_ctx_t *)aes_ctx, data,
824		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
825		if (ret != CRYPTO_SUCCESS) {
826			return (ret);
827		}
828	} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
829		size_t saved_offset = data->cd_offset;
830
831		ret = gcm_encrypt_final((gcm_ctx_t *)aes_ctx, data,
832		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
833		    aes_xor_block);
834		if (ret != CRYPTO_SUCCESS) {
835			return (ret);
836		}
837		data->cd_length = data->cd_offset - saved_offset;
838		data->cd_offset = saved_offset;
839	} else if (aes_ctx->ac_flags & CMAC_MODE) {
840		ret = cmac_mode_final((cbc_ctx_t *)aes_ctx, data,
841		    aes_encrypt_block, aes_xor_block);
842		if (ret != CRYPTO_SUCCESS)
843			return (ret);
844		data->cd_length = AES_BLOCK_LEN;
845	} else if ((aes_ctx->ac_flags & CTR_MODE) == 0) {
846		/*
847		 * There must be no unprocessed plaintext.
848		 * This happens if the length of the last data is
849		 * not a multiple of the AES block length.
850		 */
851		if (aes_ctx->ac_remainder_len > 0) {
852			return (CRYPTO_DATA_LEN_RANGE);
853		}
854		data->cd_length = 0;
855	}
856
857	(void) aes_free_context(ctx);
858
859	return (CRYPTO_SUCCESS);
860}
861
862/* ARGSUSED */
863static int
864aes_decrypt_final(crypto_ctx_t *ctx, crypto_data_t *data,
865    crypto_req_handle_t req)
866{
867	aes_ctx_t *aes_ctx;
868	int ret;
869	off_t saved_offset;
870	size_t saved_length;
871
872	ASSERT(ctx->cc_provider_private != NULL);
873	aes_ctx = ctx->cc_provider_private;
874
875	if (data->cd_format != CRYPTO_DATA_RAW &&
876	    data->cd_format != CRYPTO_DATA_UIO &&
877	    data->cd_format != CRYPTO_DATA_MBLK) {
878		return (CRYPTO_ARGUMENTS_BAD);
879	}
880
881	/*
882	 * There must be no unprocessed ciphertext.
883	 * This happens if the length of the last ciphertext is
884	 * not a multiple of the AES block length.
885	 *
886	 * For CTR mode, ac_remainder_len is always zero (we never
887	 * accumulate ciphertext across update calls with CTR mode).
888	 */
889	if (aes_ctx->ac_remainder_len > 0 &&
890	    (aes_ctx->ac_flags & CTR_MODE) == 0) {
891		return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
892	}
893
894	if (aes_ctx->ac_flags & CCM_MODE) {
895		/*
896		 * This is where all the plaintext is returned, make sure
897		 * the plaintext buffer is big enough
898		 */
899		size_t pt_len = aes_ctx->ac_data_len;
900		if (data->cd_length < pt_len) {
901			data->cd_length = pt_len;
902			return (CRYPTO_BUFFER_TOO_SMALL);
903		}
904
905		ASSERT(aes_ctx->ac_processed_data_len == pt_len);
906		ASSERT(aes_ctx->ac_processed_mac_len == aes_ctx->ac_mac_len);
907		saved_offset = data->cd_offset;
908		saved_length = data->cd_length;
909		ret = ccm_decrypt_final((ccm_ctx_t *)aes_ctx, data,
910		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
911		    aes_xor_block);
912		if (ret == CRYPTO_SUCCESS) {
913			data->cd_length = data->cd_offset - saved_offset;
914		} else {
915			data->cd_length = saved_length;
916		}
917
918		data->cd_offset = saved_offset;
919		if (ret != CRYPTO_SUCCESS) {
920			return (ret);
921		}
922	} else if (aes_ctx->ac_flags & (GCM_MODE|GMAC_MODE)) {
923		/*
924		 * This is where all the plaintext is returned, make sure
925		 * the plaintext buffer is big enough
926		 */
927		gcm_ctx_t *ctx = (gcm_ctx_t *)aes_ctx;
928		size_t pt_len = ctx->gcm_processed_data_len - ctx->gcm_tag_len;
929
930		if (data->cd_length < pt_len) {
931			data->cd_length = pt_len;
932			return (CRYPTO_BUFFER_TOO_SMALL);
933		}
934
935		saved_offset = data->cd_offset;
936		saved_length = data->cd_length;
937		ret = gcm_decrypt_final((gcm_ctx_t *)aes_ctx, data,
938		    AES_BLOCK_LEN, aes_encrypt_block, aes_xor_block);
939		if (ret == CRYPTO_SUCCESS) {
940			data->cd_length = data->cd_offset - saved_offset;
941		} else {
942			data->cd_length = saved_length;
943		}
944
945		data->cd_offset = saved_offset;
946		if (ret != CRYPTO_SUCCESS) {
947			return (ret);
948		}
949	}
950
951
952	if ((aes_ctx->ac_flags & (CTR_MODE|CCM_MODE|GCM_MODE|GMAC_MODE)) == 0) {
953		data->cd_length = 0;
954	}
955
956	(void) aes_free_context(ctx);
957
958	return (CRYPTO_SUCCESS);
959}
960
961/* ARGSUSED */
962static int
963aes_encrypt_atomic(crypto_provider_handle_t provider,
964    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
965    crypto_key_t *key, crypto_data_t *plaintext, crypto_data_t *ciphertext,
966    crypto_spi_ctx_template_t template, crypto_req_handle_t req)
967{
968	aes_ctx_t aes_ctx;	/* on the stack */
969	off_t saved_offset;
970	size_t saved_length;
971	size_t length_needed;
972	int ret;
973
974	AES_ARG_INPLACE(plaintext, ciphertext);
975
976	/*
977	 * CTR, CCM, CMAC, GCM, and GMAC modes do not require that plaintext
978	 * be a multiple of AES block size.
979	 */
980	switch (mechanism->cm_type) {
981	case AES_CTR_MECH_INFO_TYPE:
982	case AES_CCM_MECH_INFO_TYPE:
983	case AES_GCM_MECH_INFO_TYPE:
984	case AES_GMAC_MECH_INFO_TYPE:
985	case AES_CMAC_MECH_INFO_TYPE:
986		break;
987	default:
988		if ((plaintext->cd_length & (AES_BLOCK_LEN - 1)) != 0)
989			return (CRYPTO_DATA_LEN_RANGE);
990	}
991
992	if ((ret = aes_check_mech_param(mechanism, NULL, 0)) != CRYPTO_SUCCESS)
993		return (ret);
994
995	bzero(&aes_ctx, sizeof (aes_ctx_t));
996
997	ret = aes_common_init_ctx(&aes_ctx, template, mechanism, key,
998	    crypto_kmflag(req), B_TRUE);
999	if (ret != CRYPTO_SUCCESS)
1000		return (ret);
1001
1002	switch (mechanism->cm_type) {
1003	case AES_CCM_MECH_INFO_TYPE:
1004		length_needed = plaintext->cd_length + aes_ctx.ac_mac_len;
1005		break;
1006	case AES_GMAC_MECH_INFO_TYPE:
1007		if (plaintext->cd_length != 0)
1008			return (CRYPTO_ARGUMENTS_BAD);
1009		/* FALLTHRU */
1010	case AES_GCM_MECH_INFO_TYPE:
1011		length_needed = plaintext->cd_length + aes_ctx.ac_tag_len;
1012		break;
1013	case AES_CMAC_MECH_INFO_TYPE:
1014		length_needed = AES_BLOCK_LEN;
1015		break;
1016	default:
1017		length_needed = plaintext->cd_length;
1018	}
1019
1020	/* return size of buffer needed to store output */
1021	if (ciphertext->cd_length < length_needed) {
1022		ciphertext->cd_length = length_needed;
1023		ret = CRYPTO_BUFFER_TOO_SMALL;
1024		goto out;
1025	}
1026
1027	saved_offset = ciphertext->cd_offset;
1028	saved_length = ciphertext->cd_length;
1029
1030	/*
1031	 * Do an update on the specified input data.
1032	 */
1033	switch (plaintext->cd_format) {
1034	case CRYPTO_DATA_RAW:
1035		ret = crypto_update_iov(&aes_ctx, plaintext, ciphertext,
1036		    aes_encrypt_contiguous_blocks, aes_copy_block64);
1037		break;
1038	case CRYPTO_DATA_UIO:
1039		ret = crypto_update_uio(&aes_ctx, plaintext, ciphertext,
1040		    aes_encrypt_contiguous_blocks, aes_copy_block64);
1041		break;
1042	case CRYPTO_DATA_MBLK:
1043		ret = crypto_update_mp(&aes_ctx, plaintext, ciphertext,
1044		    aes_encrypt_contiguous_blocks, aes_copy_block64);
1045		break;
1046	default:
1047		ret = CRYPTO_ARGUMENTS_BAD;
1048	}
1049
1050	if (ret == CRYPTO_SUCCESS) {
1051		switch (mechanism->cm_type) {
1052		case AES_CCM_MECH_INFO_TYPE:
1053			ret = ccm_encrypt_final((ccm_ctx_t *)&aes_ctx,
1054			    ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
1055			    aes_xor_block);
1056			if (ret != CRYPTO_SUCCESS)
1057				goto out;
1058			ASSERT3U(aes_ctx.ac_remainder_len, ==, 0);
1059			break;
1060		case AES_GCM_MECH_INFO_TYPE:
1061		case AES_GMAC_MECH_INFO_TYPE:
1062			ret = gcm_encrypt_final((gcm_ctx_t *)&aes_ctx,
1063			    ciphertext, AES_BLOCK_LEN, aes_encrypt_block,
1064			    aes_copy_block, aes_xor_block);
1065			if (ret != CRYPTO_SUCCESS)
1066				goto out;
1067			ASSERT3U(aes_ctx.ac_remainder_len, ==, 0);
1068			break;
1069		case AES_CTR_MECH_INFO_TYPE:
1070			/*
1071			 * Note that this use of the ASSERT3U has a slightly
1072			 * different meaning than the other uses in the
1073			 * switch statement. The other uses are to ensure
1074			 * no unprocessed plaintext remains after encryption
1075			 * (and that the input plaintext was an exact multiple
1076			 * of AES_BLOCK_LEN).
1077			 *
1078			 * For CTR mode, it is ensuring that no input
1079			 * plaintext was ever segmented and buffered during
1080			 * processing (since it's a stream cipher).
1081			 */
1082			ASSERT3U(aes_ctx.ac_remainder_len, ==, 0);
1083			break;
1084		case AES_CMAC_MECH_INFO_TYPE:
1085			ret = cmac_mode_final((cbc_ctx_t *)&aes_ctx,
1086			    ciphertext, aes_encrypt_block,
1087			    aes_xor_block);
1088			if (ret != CRYPTO_SUCCESS)
1089				goto out;
1090			break;
1091		default:
1092			ASSERT3U(aes_ctx.ac_remainder_len, ==, 0);
1093			break;
1094		}
1095
1096		if (plaintext != ciphertext) {
1097			ciphertext->cd_length =
1098			    ciphertext->cd_offset - saved_offset;
1099		}
1100	} else {
1101		ciphertext->cd_length = saved_length;
1102	}
1103	ciphertext->cd_offset = saved_offset;
1104
1105out:
1106	if (aes_ctx.ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) {
1107		bzero(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
1108		kmem_free(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
1109	}
1110
1111	return (ret);
1112}
1113
1114/* ARGSUSED */
1115static int
1116aes_decrypt_atomic(crypto_provider_handle_t provider,
1117    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1118    crypto_key_t *key, crypto_data_t *ciphertext, crypto_data_t *plaintext,
1119    crypto_spi_ctx_template_t template, crypto_req_handle_t req)
1120{
1121	aes_ctx_t aes_ctx;	/* on the stack */
1122	off_t saved_offset;
1123	size_t saved_length;
1124	size_t length_needed;
1125	int ret;
1126
1127	AES_ARG_INPLACE(ciphertext, plaintext);
1128
1129	/*
1130	 * CCM, GCM, CTR, and GMAC modes do not require that ciphertext
1131	 * be a multiple of AES block size.
1132	 */
1133	switch (mechanism->cm_type) {
1134	case AES_CTR_MECH_INFO_TYPE:
1135	case AES_CCM_MECH_INFO_TYPE:
1136	case AES_GCM_MECH_INFO_TYPE:
1137	case AES_GMAC_MECH_INFO_TYPE:
1138		break;
1139	default:
1140		if ((ciphertext->cd_length & (AES_BLOCK_LEN - 1)) != 0)
1141			return (CRYPTO_ENCRYPTED_DATA_LEN_RANGE);
1142	}
1143
1144	if ((ret = aes_check_mech_param(mechanism, NULL, 0)) != CRYPTO_SUCCESS)
1145		return (ret);
1146
1147	bzero(&aes_ctx, sizeof (aes_ctx_t));
1148
1149	ret = aes_common_init_ctx(&aes_ctx, template, mechanism, key,
1150	    crypto_kmflag(req), B_FALSE);
1151	if (ret != CRYPTO_SUCCESS)
1152		return (ret);
1153
1154	switch (mechanism->cm_type) {
1155	case AES_CCM_MECH_INFO_TYPE:
1156		length_needed = aes_ctx.ac_data_len;
1157		break;
1158	case AES_GCM_MECH_INFO_TYPE:
1159		length_needed = ciphertext->cd_length - aes_ctx.ac_tag_len;
1160		break;
1161	case AES_GMAC_MECH_INFO_TYPE:
1162		if (plaintext->cd_length != 0)
1163			return (CRYPTO_ARGUMENTS_BAD);
1164		length_needed = 0;
1165		break;
1166	default:
1167		length_needed = ciphertext->cd_length;
1168	}
1169
1170	/* return size of buffer needed to store output */
1171	if (plaintext->cd_length < length_needed) {
1172		plaintext->cd_length = length_needed;
1173		ret = CRYPTO_BUFFER_TOO_SMALL;
1174		goto out;
1175	}
1176
1177	saved_offset = plaintext->cd_offset;
1178	saved_length = plaintext->cd_length;
1179
1180	if (mechanism->cm_type == AES_GCM_MECH_INFO_TYPE ||
1181	    mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE)
1182		gcm_set_kmflag((gcm_ctx_t *)&aes_ctx, crypto_kmflag(req));
1183
1184	/*
1185	 * Do an update on the specified input data.
1186	 */
1187	switch (ciphertext->cd_format) {
1188	case CRYPTO_DATA_RAW:
1189		ret = crypto_update_iov(&aes_ctx, ciphertext, plaintext,
1190		    aes_decrypt_contiguous_blocks, aes_copy_block64);
1191		break;
1192	case CRYPTO_DATA_UIO:
1193		ret = crypto_update_uio(&aes_ctx, ciphertext, plaintext,
1194		    aes_decrypt_contiguous_blocks, aes_copy_block64);
1195		break;
1196	case CRYPTO_DATA_MBLK:
1197		ret = crypto_update_mp(&aes_ctx, ciphertext, plaintext,
1198		    aes_decrypt_contiguous_blocks, aes_copy_block64);
1199		break;
1200	default:
1201		ret = CRYPTO_ARGUMENTS_BAD;
1202	}
1203
1204	if (ret == CRYPTO_SUCCESS) {
1205		switch (mechanism->cm_type) {
1206		case AES_CCM_MECH_INFO_TYPE:
1207			ASSERT(aes_ctx.ac_processed_data_len
1208			    == aes_ctx.ac_data_len);
1209			ASSERT(aes_ctx.ac_processed_mac_len
1210			    == aes_ctx.ac_mac_len);
1211			ret = ccm_decrypt_final((ccm_ctx_t *)&aes_ctx,
1212			    plaintext, AES_BLOCK_LEN, aes_encrypt_block,
1213			    aes_copy_block, aes_xor_block);
1214			ASSERT3U(aes_ctx.ac_remainder_len, ==, 0);
1215			if ((ret == CRYPTO_SUCCESS) &&
1216			    (ciphertext != plaintext)) {
1217				plaintext->cd_length =
1218				    plaintext->cd_offset - saved_offset;
1219			} else {
1220				plaintext->cd_length = saved_length;
1221			}
1222			break;
1223		case AES_GCM_MECH_INFO_TYPE:
1224		case AES_GMAC_MECH_INFO_TYPE:
1225			ret = gcm_decrypt_final((gcm_ctx_t *)&aes_ctx,
1226			    plaintext, AES_BLOCK_LEN, aes_encrypt_block,
1227			    aes_xor_block);
1228			ASSERT3U(aes_ctx.ac_remainder_len, ==, 0);
1229			if ((ret == CRYPTO_SUCCESS) &&
1230			    (ciphertext != plaintext)) {
1231				plaintext->cd_length =
1232				    plaintext->cd_offset - saved_offset;
1233			} else {
1234				plaintext->cd_length = saved_length;
1235			}
1236			break;
1237		case AES_CTR_MECH_INFO_TYPE:
1238			if (ciphertext != plaintext) {
1239				plaintext->cd_length =
1240				    plaintext->cd_offset - saved_offset;
1241			}
1242			break;
1243		default:
1244			ASSERT3U(aes_ctx.ac_remainder_len, ==, 0);
1245			if (ciphertext != plaintext) {
1246				plaintext->cd_length =
1247				    plaintext->cd_offset - saved_offset;
1248			}
1249			break;
1250		}
1251	} else {
1252		plaintext->cd_length = saved_length;
1253	}
1254	plaintext->cd_offset = saved_offset;
1255
1256out:
1257	if (aes_ctx.ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) {
1258		bzero(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
1259		kmem_free(aes_ctx.ac_keysched, aes_ctx.ac_keysched_len);
1260	}
1261
1262	if (aes_ctx.ac_flags & CCM_MODE) {
1263		if (aes_ctx.ac_pt_buf != NULL) {
1264			kmem_free(aes_ctx.ac_pt_buf, aes_ctx.ac_data_len);
1265		}
1266	} else if (aes_ctx.ac_flags & (GCM_MODE|GMAC_MODE)) {
1267		if (((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf != NULL) {
1268			kmem_free(((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf,
1269			    ((gcm_ctx_t *)&aes_ctx)->gcm_pt_buf_len);
1270		}
1271	}
1272
1273	return (ret);
1274}
1275
1276/*
1277 * KCF software provider context template entry points.
1278 */
1279/* ARGSUSED */
1280static int
1281aes_create_ctx_template(crypto_provider_handle_t provider,
1282    crypto_mechanism_t *mechanism, crypto_key_t *key,
1283    crypto_spi_ctx_template_t *tmpl, size_t *tmpl_size, crypto_req_handle_t req)
1284{
1285	void *keysched;
1286	size_t size;
1287	int rv;
1288
1289	if (mechanism->cm_type != AES_ECB_MECH_INFO_TYPE &&
1290	    mechanism->cm_type != AES_CBC_MECH_INFO_TYPE &&
1291	    mechanism->cm_type != AES_CMAC_MECH_INFO_TYPE &&
1292	    mechanism->cm_type != AES_CTR_MECH_INFO_TYPE &&
1293	    mechanism->cm_type != AES_CCM_MECH_INFO_TYPE &&
1294	    mechanism->cm_type != AES_GCM_MECH_INFO_TYPE &&
1295	    mechanism->cm_type != AES_GMAC_MECH_INFO_TYPE)
1296		return (CRYPTO_MECHANISM_INVALID);
1297
1298	if ((keysched = aes_alloc_keysched(&size,
1299	    crypto_kmflag(req))) == NULL) {
1300		return (CRYPTO_HOST_MEMORY);
1301	}
1302
1303	/*
1304	 * Initialize key schedule.  Key length information is stored
1305	 * in the key.
1306	 */
1307	if ((rv = init_keysched(key, keysched)) != CRYPTO_SUCCESS) {
1308		bzero(keysched, size);
1309		kmem_free(keysched, size);
1310		return (rv);
1311	}
1312
1313	*tmpl = keysched;
1314	*tmpl_size = size;
1315
1316	return (CRYPTO_SUCCESS);
1317}
1318
1319
1320static int
1321aes_free_context(crypto_ctx_t *ctx)
1322{
1323	aes_ctx_t *aes_ctx = ctx->cc_provider_private;
1324
1325	if (aes_ctx != NULL) {
1326		if (aes_ctx->ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) {
1327			ASSERT(aes_ctx->ac_keysched_len != 0);
1328			bzero(aes_ctx->ac_keysched, aes_ctx->ac_keysched_len);
1329			kmem_free(aes_ctx->ac_keysched,
1330			    aes_ctx->ac_keysched_len);
1331		}
1332		crypto_free_mode_ctx(aes_ctx);
1333		ctx->cc_provider_private = NULL;
1334	}
1335
1336	return (CRYPTO_SUCCESS);
1337}
1338
1339
1340static int
1341aes_common_init_ctx(aes_ctx_t *aes_ctx, crypto_spi_ctx_template_t *template,
1342    crypto_mechanism_t *mechanism, crypto_key_t *key, int kmflag,
1343    boolean_t is_encrypt_init)
1344{
1345	int rv = CRYPTO_SUCCESS;
1346	void *keysched;
1347	size_t size;
1348
1349	if (template == NULL) {
1350		if ((keysched = aes_alloc_keysched(&size, kmflag)) == NULL)
1351			return (CRYPTO_HOST_MEMORY);
1352		/*
1353		 * Initialize key schedule.
1354		 * Key length is stored in the key.
1355		 */
1356		if ((rv = init_keysched(key, keysched)) != CRYPTO_SUCCESS) {
1357			kmem_free(keysched, size);
1358			return (rv);
1359		}
1360
1361		aes_ctx->ac_flags |= PROVIDER_OWNS_KEY_SCHEDULE;
1362		aes_ctx->ac_keysched_len = size;
1363	} else {
1364		keysched = template;
1365	}
1366	aes_ctx->ac_keysched = keysched;
1367
1368	switch (mechanism->cm_type) {
1369	case AES_CBC_MECH_INFO_TYPE:
1370		rv = cbc_init_ctx((cbc_ctx_t *)aes_ctx, mechanism->cm_param,
1371		    mechanism->cm_param_len, AES_BLOCK_LEN, aes_copy_block64);
1372		break;
1373	case AES_CMAC_MECH_INFO_TYPE:
1374		rv = cmac_init_ctx((cbc_ctx_t *)aes_ctx, AES_BLOCK_LEN);
1375		break;
1376	case AES_CTR_MECH_INFO_TYPE: {
1377		CK_AES_CTR_PARAMS *pp;
1378
1379		if (mechanism->cm_param == NULL ||
1380		    mechanism->cm_param_len != sizeof (CK_AES_CTR_PARAMS)) {
1381			return (CRYPTO_MECHANISM_PARAM_INVALID);
1382		}
1383		pp = (CK_AES_CTR_PARAMS *)(void *)mechanism->cm_param;
1384		rv = ctr_init_ctx((ctr_ctx_t *)aes_ctx, pp->ulCounterBits,
1385		    pp->cb, aes_encrypt_block, aes_copy_block);
1386		break;
1387	}
1388	case AES_CCM_MECH_INFO_TYPE:
1389		if (mechanism->cm_param == NULL ||
1390		    mechanism->cm_param_len != sizeof (CK_AES_CCM_PARAMS)) {
1391			return (CRYPTO_MECHANISM_PARAM_INVALID);
1392		}
1393		rv = ccm_init_ctx((ccm_ctx_t *)aes_ctx, mechanism->cm_param,
1394		    kmflag, is_encrypt_init, AES_BLOCK_LEN, aes_encrypt_block,
1395		    aes_xor_block);
1396		break;
1397	case AES_GCM_MECH_INFO_TYPE:
1398		if (mechanism->cm_param == NULL ||
1399		    mechanism->cm_param_len != sizeof (CK_AES_GCM_PARAMS)) {
1400			return (CRYPTO_MECHANISM_PARAM_INVALID);
1401		}
1402		rv = gcm_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
1403		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
1404		    aes_xor_block);
1405		break;
1406	case AES_GMAC_MECH_INFO_TYPE:
1407		if (mechanism->cm_param == NULL ||
1408		    mechanism->cm_param_len != sizeof (CK_AES_GMAC_PARAMS)) {
1409			return (CRYPTO_MECHANISM_PARAM_INVALID);
1410		}
1411		rv = gmac_init_ctx((gcm_ctx_t *)aes_ctx, mechanism->cm_param,
1412		    AES_BLOCK_LEN, aes_encrypt_block, aes_copy_block,
1413		    aes_xor_block);
1414		break;
1415	case AES_ECB_MECH_INFO_TYPE:
1416		aes_ctx->ac_flags |= ECB_MODE;
1417	}
1418
1419	if (rv != CRYPTO_SUCCESS) {
1420		if (aes_ctx->ac_flags & PROVIDER_OWNS_KEY_SCHEDULE) {
1421			bzero(keysched, size);
1422			kmem_free(keysched, size);
1423		}
1424	}
1425
1426	return (rv);
1427}
1428
1429static int
1430process_gmac_mech(crypto_mechanism_t *mech, crypto_data_t *data,
1431    CK_AES_GCM_PARAMS *gcm_params)
1432{
1433	/* LINTED: pointer alignment */
1434	CK_AES_GMAC_PARAMS *params = (CK_AES_GMAC_PARAMS *)mech->cm_param;
1435
1436	if (mech->cm_type != AES_GMAC_MECH_INFO_TYPE)
1437		return (CRYPTO_MECHANISM_INVALID);
1438
1439	if (mech->cm_param_len != sizeof (CK_AES_GMAC_PARAMS))
1440		return (CRYPTO_MECHANISM_PARAM_INVALID);
1441
1442	if (params->pIv == NULL)
1443		return (CRYPTO_MECHANISM_PARAM_INVALID);
1444
1445	gcm_params->pIv = params->pIv;
1446	gcm_params->ulIvLen = AES_GMAC_IV_LEN;
1447	gcm_params->ulTagBits = AES_GMAC_TAG_BITS;
1448
1449	if (data == NULL)
1450		return (CRYPTO_SUCCESS);
1451
1452	if (data->cd_format != CRYPTO_DATA_RAW)
1453		return (CRYPTO_ARGUMENTS_BAD);
1454
1455	gcm_params->pAAD = (uchar_t *)data->cd_raw.iov_base;
1456	gcm_params->ulAADLen = data->cd_length;
1457	return (CRYPTO_SUCCESS);
1458}
1459
1460static int
1461aes_mac_init(crypto_ctx_t *ctx, crypto_mechanism_t *mechanism,
1462    crypto_key_t *key, crypto_spi_ctx_template_t template,
1463    crypto_req_handle_t req)
1464{
1465	return (aes_encrypt_init(ctx, mechanism,
1466	    key, template, req));
1467}
1468
1469static int
1470aes_mac(crypto_ctx_t *ctx, crypto_data_t *plaintext, crypto_data_t *ciphertext,
1471    crypto_req_handle_t req)
1472{
1473	return (aes_encrypt(ctx, plaintext, ciphertext, req));
1474}
1475
1476static int
1477aes_mac_update(crypto_ctx_t *ctx, crypto_data_t *data,
1478    crypto_req_handle_t req)
1479{
1480	crypto_data_t out;
1481	uint8_t block[AES_BLOCK_LEN];
1482	out.cd_format = CRYPTO_DATA_RAW;
1483	out.cd_offset = 0;
1484	out.cd_length = sizeof (block);
1485	out.cd_miscdata = NULL;
1486	out.cd_raw.iov_base = (void *)block;
1487	out.cd_raw.iov_len = sizeof (block);
1488
1489	return (aes_encrypt_update(ctx, data, &out, req));
1490}
1491
1492static int
1493aes_mac_final(crypto_ctx_t *ctx, crypto_data_t *mac, crypto_req_handle_t req)
1494{
1495	return (aes_encrypt_final(ctx, mac, req));
1496}
1497
1498static int
1499aes_mac_atomic(crypto_provider_handle_t provider,
1500    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1501    crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac,
1502    crypto_spi_ctx_template_t template, crypto_req_handle_t req)
1503{
1504	CK_AES_GCM_PARAMS gcm_params;
1505	crypto_mechanism_t gcm_mech;
1506	int rv;
1507
1508	if (mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
1509		if ((rv = process_gmac_mech(mechanism, data, &gcm_params))
1510		    != CRYPTO_SUCCESS)
1511			return (rv);
1512
1513		gcm_mech.cm_type = AES_GCM_MECH_INFO_TYPE;
1514		gcm_mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS);
1515		gcm_mech.cm_param = (char *)&gcm_params;
1516
1517		return (aes_encrypt_atomic(provider, session_id, &gcm_mech,
1518		    key, &null_crypto_data, mac, template, req));
1519	}
1520	/* CMAC */
1521	return (aes_encrypt_atomic(provider, session_id, mechanism,
1522	    key, data, mac, template, req));
1523}
1524
1525static int
1526aes_mac_verify_atomic(crypto_provider_handle_t provider,
1527    crypto_session_id_t session_id, crypto_mechanism_t *mechanism,
1528    crypto_key_t *key, crypto_data_t *data, crypto_data_t *mac,
1529    crypto_spi_ctx_template_t template, crypto_req_handle_t req)
1530{
1531	CK_AES_GCM_PARAMS gcm_params;
1532	crypto_mechanism_t gcm_mech;
1533	crypto_data_t data_mac;
1534	char buf[AES_BLOCK_LEN];
1535	int rv;
1536
1537	if (mechanism->cm_type == AES_GMAC_MECH_INFO_TYPE) {
1538		if ((rv = process_gmac_mech(mechanism, data, &gcm_params))
1539		    != CRYPTO_SUCCESS)
1540			return (rv);
1541
1542		gcm_mech.cm_type = AES_GCM_MECH_INFO_TYPE;
1543		gcm_mech.cm_param_len = sizeof (CK_AES_GCM_PARAMS);
1544		gcm_mech.cm_param = (char *)&gcm_params;
1545
1546		return (aes_decrypt_atomic(provider, session_id, &gcm_mech,
1547		    key, mac, &null_crypto_data, template, req));
1548	}
1549
1550	/* CMAC */
1551
1552	data_mac.cd_format = CRYPTO_DATA_RAW;
1553	data_mac.cd_offset = 0;
1554	data_mac.cd_length = AES_BLOCK_LEN;
1555	data_mac.cd_miscdata = NULL;
1556	data_mac.cd_raw.iov_base = (void *) buf;
1557	data_mac.cd_raw.iov_len = AES_BLOCK_LEN;
1558
1559	rv = aes_encrypt_atomic(provider, session_id, &gcm_mech,
1560	    key, data, &data_mac, template, req);
1561
1562	if (rv != CRYPTO_SUCCESS)
1563		return (rv);
1564
1565	/* should use get_input_data for mac? */
1566	if (bcmp(buf, mac->cd_raw.iov_base + mac->cd_offset,
1567	    AES_BLOCK_LEN) != 0)
1568		return (CRYPTO_INVALID_MAC);
1569
1570	return (CRYPTO_SUCCESS);
1571}
1572