1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26#include <pthread.h>
27#include <stdlib.h>
28#include <string.h>
29#include <strings.h>
30#include <sys/types.h>
31#include <security/cryptoki.h>
32#include <cryptoutil.h>
33#include "softGlobal.h"
34#include "softSession.h"
35#include "softObject.h"
36#include "softOps.h"
37#include "softRSA.h"
38#include "softMAC.h"
39#include "softCrypt.h"
40
41CK_RV
42soft_rsa_encrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
43    CK_BYTE_PTR out, int realpublic)
44{
45
46	CK_RV rv = CKR_OK;
47
48	uchar_t expo[MAX_KEY_ATTR_BUFLEN];
49	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
50	uint32_t expo_len = sizeof (expo);
51	uint32_t modulus_len = sizeof (modulus);
52	RSAbytekey k;
53
54	if (realpublic) {
55		rv = soft_get_public_value(key, CKA_PUBLIC_EXPONENT, expo,
56		    &expo_len);
57		if (rv != CKR_OK) {
58			goto clean1;
59		}
60	} else {
61		rv = soft_get_private_value(key, CKA_PRIVATE_EXPONENT, expo,
62		    &expo_len);
63		if (rv != CKR_OK) {
64			goto clean1;
65		}
66	}
67
68	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
69	if (rv != CKR_OK) {
70		goto clean1;
71	}
72
73	k.modulus = modulus;
74	k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
75	k.pubexpo = expo;
76	k.pubexpo_bytes = expo_len;
77	k.rfunc = NULL;
78
79	rv = rsa_encrypt(&k, in, in_len, out);
80
81clean1:
82
83	return (rv);
84}
85
86
87CK_RV
88soft_rsa_decrypt(soft_object_t *key, CK_BYTE_PTR in, uint32_t in_len,
89    CK_BYTE_PTR out)
90{
91
92	CK_RV rv = CKR_OK;
93
94	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
95	uchar_t prime1[MAX_KEY_ATTR_BUFLEN];
96	uchar_t prime2[MAX_KEY_ATTR_BUFLEN];
97	uchar_t expo1[MAX_KEY_ATTR_BUFLEN];
98	uchar_t expo2[MAX_KEY_ATTR_BUFLEN];
99	uchar_t coef[MAX_KEY_ATTR_BUFLEN];
100	uint32_t modulus_len = sizeof (modulus);
101	uint32_t prime1_len = sizeof (prime1);
102	uint32_t prime2_len = sizeof (prime2);
103	uint32_t expo1_len = sizeof (expo1);
104	uint32_t expo2_len = sizeof (expo2);
105	uint32_t coef_len = sizeof (coef);
106	RSAbytekey k;
107
108	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
109	if (rv != CKR_OK) {
110		goto clean1;
111	}
112
113	rv = soft_get_private_value(key, CKA_PRIME_1, prime1, &prime1_len);
114
115	if ((prime1_len == 0) && (rv == CKR_OK)) {
116		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
117		goto clean1;
118	} else {
119		if (rv != CKR_OK)
120			goto clean1;
121	}
122
123	rv = soft_get_private_value(key, CKA_PRIME_2, prime2, &prime2_len);
124
125	if ((prime2_len == 0) && (rv == CKR_OK)) {
126		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
127		goto clean1;
128	} else {
129		if (rv != CKR_OK)
130			goto clean1;
131	}
132
133	rv = soft_get_private_value(key, CKA_EXPONENT_1, expo1, &expo1_len);
134
135	if ((expo1_len == 0) && (rv == CKR_OK)) {
136		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
137		goto clean1;
138	} else {
139		if (rv != CKR_OK)
140			goto clean1;
141	}
142
143	rv = soft_get_private_value(key, CKA_EXPONENT_2, expo2, &expo2_len);
144
145	if ((expo2_len == 0) && (rv == CKR_OK)) {
146		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
147		goto clean1;
148	} else {
149		if (rv != CKR_OK)
150			goto clean1;
151	}
152
153	rv = soft_get_private_value(key, CKA_COEFFICIENT, coef, &coef_len);
154
155	if ((coef_len == 0) && (rv == CKR_OK)) {
156		rv = soft_rsa_encrypt(key, in, in_len, out, 0);
157		goto clean1;
158	} else {
159		if (rv != CKR_OK)
160			goto clean1;
161	}
162
163	k.modulus = modulus;
164	k.modulus_bits = CRYPTO_BYTES2BITS(modulus_len);
165	k.prime1 = prime1;
166	k.prime1_bytes = prime1_len;
167	k.prime2 = prime2;
168	k.prime2_bytes = prime2_len;
169	k.expo1 = expo1;
170	k.expo1_bytes = expo1_len;
171	k.expo2 = expo2;
172	k.expo2_bytes = expo2_len;
173	k.coeff = coef;
174	k.coeff_bytes = coef_len;
175	k.rfunc = NULL;
176
177	rv = rsa_decrypt(&k, in, in_len, out);
178
179clean1:
180
181	return (rv);
182}
183
184/*
185 * Allocate a RSA context for the active encryption or decryption operation.
186 * This function is called without the session lock held.
187 */
188CK_RV
189soft_rsa_crypt_init_common(soft_session_t *session_p,
190    CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
191    boolean_t encrypt)
192{
193
194	soft_rsa_ctx_t *rsa_ctx;
195	soft_object_t *tmp_key = NULL;
196	CK_RV rv;
197
198	rsa_ctx = calloc(1, sizeof (soft_rsa_ctx_t));
199	if (rsa_ctx == NULL) {
200		return (CKR_HOST_MEMORY);
201	}
202
203	/*
204	 * Make a copy of the encryption or decryption key, and save it
205	 * in the RSA crypto context since it will be used later for
206	 * encryption/decryption. We don't want to hold any object reference
207	 * on this original key while doing encryption/decryption.
208	 */
209	(void) pthread_mutex_lock(&key_p->object_mutex);
210	rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
211	    NULL);
212
213	if ((rv != CKR_OK) || (tmp_key == NULL)) {
214		/* Most likely we ran out of space. */
215		(void) pthread_mutex_unlock(&key_p->object_mutex);
216		free(rsa_ctx);
217		return (rv);
218	}
219
220	/* No need to hold the lock on the old object. */
221	(void) pthread_mutex_unlock(&key_p->object_mutex);
222	rsa_ctx->key = tmp_key;
223
224	(void) pthread_mutex_lock(&session_p->session_mutex);
225	if (encrypt) {
226		/* Called by C_EncryptInit. */
227		session_p->encrypt.context = rsa_ctx;
228		session_p->encrypt.mech.mechanism = pMechanism->mechanism;
229	} else {
230		/* Called by C_DecryptInit. */
231		session_p->decrypt.context = rsa_ctx;
232		session_p->decrypt.mech.mechanism = pMechanism->mechanism;
233	}
234	(void) pthread_mutex_unlock(&session_p->session_mutex);
235
236	return (CKR_OK);
237}
238
239CK_RV
240soft_rsa_encrypt_common(soft_session_t *session_p, CK_BYTE_PTR pData,
241    CK_ULONG ulDataLen, CK_BYTE_PTR pEncrypted,
242    CK_ULONG_PTR pulEncryptedLen, CK_MECHANISM_TYPE mechanism)
243{
244
245	soft_rsa_ctx_t *rsa_ctx = session_p->encrypt.context;
246	soft_object_t *key = rsa_ctx->key;
247	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
248	uint32_t modulus_len = sizeof (modulus);
249	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
250	CK_BYTE	cipher_data[MAX_RSA_KEYLENGTH_IN_BYTES];
251	CK_RV rv = CKR_OK;
252
253	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
254	if (rv != CKR_OK) {
255		goto clean_exit;
256	}
257
258	if (pEncrypted == NULL) {
259		/*
260		 * Application asks for the length of the output buffer
261		 * to hold the ciphertext.
262		 */
263		*pulEncryptedLen = modulus_len;
264		rv = CKR_OK;
265		goto clean1;
266	}
267
268	if (mechanism == CKM_RSA_PKCS) {
269		/*
270		 * Input data length needs to be <=
271		 * modulus length-MIN_PKCS1_PADLEN.
272		 */
273		if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
274			*pulEncryptedLen = modulus_len;
275			rv = CKR_DATA_LEN_RANGE;
276			goto clean_exit;
277		}
278	} else {
279		/* Input data length needs to be <= modulus length. */
280		if (ulDataLen > (CK_ULONG)modulus_len) {
281			*pulEncryptedLen = modulus_len;
282			rv = CKR_DATA_LEN_RANGE;
283			goto clean_exit;
284		}
285	}
286
287	/* Is the application-supplied buffer large enough? */
288	if (*pulEncryptedLen < (CK_ULONG)modulus_len) {
289		*pulEncryptedLen = modulus_len;
290		rv = CKR_BUFFER_TOO_SMALL;
291		goto clean1;
292	}
293
294	if (mechanism == CKM_RSA_PKCS) {
295		/*
296		 * Add PKCS padding to the input data to format a block
297		 * type "02" encryption block.
298		 */
299		rv = pkcs1_encode(PKCS1_ENCRYPT, pData, ulDataLen, plain_data,
300		    modulus_len);
301
302		if (rv != CKR_OK)
303			goto clean_exit;
304	} else {
305		/* Pad zeros for the leading bytes of the input data. */
306		(void) memset(plain_data, 0x0, modulus_len - ulDataLen);
307		(void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
308		    ulDataLen);
309	}
310
311	rv = soft_rsa_encrypt(key, plain_data, modulus_len, cipher_data, 1);
312	if (rv == CKR_OK) {
313		(void) memcpy(pEncrypted, cipher_data, modulus_len);
314		*pulEncryptedLen = modulus_len;
315	}
316
317clean_exit:
318	(void) pthread_mutex_lock(&session_p->session_mutex);
319	free(session_p->encrypt.context);
320	session_p->encrypt.context = NULL;
321	(void) pthread_mutex_unlock(&session_p->session_mutex);
322	soft_cleanup_object(key);
323	free(key);
324clean1:
325	return (rv);
326}
327
328
329CK_RV
330soft_rsa_decrypt_common(soft_session_t *session_p, CK_BYTE_PTR pEncrypted,
331    CK_ULONG ulEncryptedLen, CK_BYTE_PTR pData,
332    CK_ULONG_PTR pulDataLen, CK_MECHANISM_TYPE mechanism)
333{
334
335	soft_rsa_ctx_t *rsa_ctx = session_p->decrypt.context;
336	soft_object_t *key = rsa_ctx->key;
337	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
338	uint32_t modulus_len = sizeof (modulus);
339	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
340	CK_RV rv = CKR_OK;
341
342	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
343	if (rv != CKR_OK) {
344		goto clean_exit;
345	}
346
347	if (ulEncryptedLen != (CK_ULONG)modulus_len) {
348		rv = CKR_ENCRYPTED_DATA_LEN_RANGE;
349		goto clean_exit;
350	}
351
352	if (pData == NULL) {
353		/*
354		 * Application asks for the length of the output buffer
355		 * to hold the recovered data.
356		 */
357		*pulDataLen = modulus_len;
358		rv = CKR_OK;
359		goto clean1;
360	}
361
362	if (mechanism == CKM_RSA_X_509) {
363		if (*pulDataLen < (CK_ULONG)modulus_len) {
364			*pulDataLen = modulus_len;
365			rv = CKR_BUFFER_TOO_SMALL;
366			goto clean1;
367		}
368	}
369
370	rv = soft_rsa_decrypt(key, pEncrypted, modulus_len, plain_data);
371	if (rv != CKR_OK) {
372		goto clean_exit;
373	}
374
375	if (mechanism == CKM_RSA_PKCS) {
376		size_t plain_len = modulus_len;
377		size_t num_padding;
378
379		/* Strip off the PKCS block formatting data. */
380		rv = pkcs1_decode(PKCS1_DECRYPT, plain_data, &plain_len);
381		if (rv != CKR_OK)
382			goto clean_exit;
383
384		num_padding = modulus_len - plain_len;
385		if (ulEncryptedLen - num_padding > *pulDataLen) {
386			*pulDataLen = plain_len;
387			rv = CKR_BUFFER_TOO_SMALL;
388			goto clean1;
389		}
390
391		(void) memcpy(pData, &plain_data[num_padding], plain_len);
392		*pulDataLen = plain_len;
393	} else {
394		(void) memcpy(pData, plain_data, modulus_len);
395		*pulDataLen = modulus_len;
396	}
397
398clean_exit:
399	(void) pthread_mutex_lock(&session_p->session_mutex);
400	free(session_p->decrypt.context);
401	session_p->decrypt.context = NULL;
402	(void) pthread_mutex_unlock(&session_p->session_mutex);
403	soft_cleanup_object(key);
404	free(key);
405
406clean1:
407	return (rv);
408}
409
410/*
411 * Allocate a RSA context for the active sign or verify operation.
412 * This function is called without the session lock held.
413 */
414CK_RV
415soft_rsa_sign_verify_init_common(soft_session_t *session_p,
416    CK_MECHANISM_PTR pMechanism, soft_object_t *key_p,
417    boolean_t sign)
418{
419	CK_RV rv = CKR_OK;
420	soft_rsa_ctx_t *rsa_ctx;
421	CK_MECHANISM digest_mech;
422	soft_object_t *tmp_key = NULL;
423
424	if (sign) {
425		if ((key_p->class != CKO_PRIVATE_KEY) ||
426		    (key_p->key_type != CKK_RSA))
427			return (CKR_KEY_TYPE_INCONSISTENT);
428	} else {
429		if ((key_p->class != CKO_PUBLIC_KEY) ||
430		    (key_p->key_type != CKK_RSA))
431			return (CKR_KEY_TYPE_INCONSISTENT);
432	}
433
434	switch (pMechanism->mechanism) {
435	case CKM_MD5_RSA_PKCS:
436		digest_mech.mechanism = CKM_MD5;
437		rv = soft_digest_init_internal(session_p, &digest_mech);
438		if (rv != CKR_OK)
439			return (rv);
440		break;
441
442	case CKM_SHA1_RSA_PKCS:
443		digest_mech.mechanism = CKM_SHA_1;
444		digest_mech.pParameter = pMechanism->pParameter;
445		digest_mech.ulParameterLen = pMechanism->ulParameterLen;
446		rv = soft_digest_init_internal(session_p, &digest_mech);
447		if (rv != CKR_OK)
448			return (rv);
449		break;
450
451	case CKM_SHA256_RSA_PKCS:
452		digest_mech.mechanism = CKM_SHA256;
453		rv = soft_digest_init_internal(session_p, &digest_mech);
454		if (rv != CKR_OK)
455			return (rv);
456		break;
457
458	case CKM_SHA384_RSA_PKCS:
459		digest_mech.mechanism = CKM_SHA384;
460		rv = soft_digest_init_internal(session_p, &digest_mech);
461		if (rv != CKR_OK)
462			return (rv);
463		break;
464
465	case CKM_SHA512_RSA_PKCS:
466		digest_mech.mechanism = CKM_SHA512;
467		rv = soft_digest_init_internal(session_p, &digest_mech);
468		if (rv != CKR_OK)
469			return (rv);
470		break;
471	}
472
473	rsa_ctx = malloc(sizeof (soft_rsa_ctx_t));
474
475	if (rsa_ctx == NULL) {
476		rv = CKR_HOST_MEMORY;
477		goto clean_exit;
478	}
479
480	(void) pthread_mutex_lock(&key_p->object_mutex);
481	rv = soft_copy_object(key_p, &tmp_key, SOFT_COPY_OBJ_ORIG_SH,
482	    NULL);
483
484	if ((rv != CKR_OK) || (tmp_key == NULL)) {
485		/* Most likely we ran out of space. */
486		(void) pthread_mutex_unlock(&key_p->object_mutex);
487		free(rsa_ctx);
488		goto clean_exit;
489	}
490
491	/* No need to hold the lock on the old object. */
492	(void) pthread_mutex_unlock(&key_p->object_mutex);
493	rsa_ctx->key = tmp_key;
494
495	(void) pthread_mutex_lock(&session_p->session_mutex);
496
497	if (sign) {
498		session_p->sign.context = rsa_ctx;
499		session_p->sign.mech.mechanism = pMechanism->mechanism;
500	} else {
501		session_p->verify.context = rsa_ctx;
502		session_p->verify.mech.mechanism = pMechanism->mechanism;
503	}
504
505	(void) pthread_mutex_unlock(&session_p->session_mutex);
506
507	return (CKR_OK);
508
509clean_exit:
510	(void) pthread_mutex_lock(&session_p->session_mutex);
511	if (session_p->digest.context != NULL) {
512		free(session_p->digest.context);
513		session_p->digest.context = NULL;
514		session_p->digest.flags = 0;
515	}
516	(void) pthread_mutex_unlock(&session_p->session_mutex);
517	return (rv);
518
519}
520
521
522CK_RV
523soft_rsa_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
524    CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
525    CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism)
526{
527
528	CK_RV rv = CKR_OK;
529	soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
530	soft_object_t *key = rsa_ctx->key;
531	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
532	uint32_t modulus_len = sizeof (modulus);
533	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
534	CK_BYTE	signed_data[MAX_RSA_KEYLENGTH_IN_BYTES];
535
536	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
537	if (rv != CKR_OK) {
538		goto clean_exit;
539	}
540
541	if (pSigned == NULL) {
542		/* Application asks for the length of the output buffer. */
543		*pulSignedLen = modulus_len;
544		rv = CKR_OK;
545		goto clean1;
546	}
547
548	switch (mechanism) {
549
550	case CKM_RSA_PKCS:
551
552		/*
553		 * Input data length needs to be <=
554		 * modulus length-MIN_PKCS1_PADLEN.
555		 */
556		if (ulDataLen > ((CK_ULONG)modulus_len - MIN_PKCS1_PADLEN)) {
557			*pulSignedLen = modulus_len;
558			rv = CKR_DATA_LEN_RANGE;
559			goto clean_exit;
560		}
561		break;
562
563	case CKM_RSA_X_509:
564
565		/* Input data length needs to be <= modulus length. */
566		if (ulDataLen > (CK_ULONG)modulus_len) {
567			*pulSignedLen = modulus_len;
568			rv = CKR_DATA_LEN_RANGE;
569			goto clean_exit;
570		}
571		break;
572	}
573
574	/* Is the application-supplied buffer large enough? */
575	if (*pulSignedLen < (CK_ULONG)modulus_len) {
576		*pulSignedLen = modulus_len;
577		rv = CKR_BUFFER_TOO_SMALL;
578		goto clean1;
579	}
580
581	switch (mechanism) {
582
583	case CKM_RSA_PKCS:
584	case CKM_MD5_RSA_PKCS:
585	case CKM_SHA1_RSA_PKCS:
586	case CKM_SHA256_RSA_PKCS:
587	case CKM_SHA384_RSA_PKCS:
588	case CKM_SHA512_RSA_PKCS:
589		/*
590		 * Add PKCS padding to the input data to format a block
591		 * type "01" encryption block.
592		 */
593		rv = pkcs1_encode(PKCS1_SIGN, pData, ulDataLen, plain_data,
594		    modulus_len);
595
596		if (rv != CKR_OK) {
597			goto clean_exit;
598		}
599		break;
600
601	case CKM_RSA_X_509:
602
603		/* Pad zeros for the leading bytes of the input data. */
604		(void) memset(plain_data, 0x0, modulus_len - ulDataLen);
605		(void) memcpy(&plain_data[modulus_len - ulDataLen], pData,
606		    ulDataLen);
607		break;
608	}
609
610	/*
611	 * Perform RSA encryption with the signer's RSA private key
612	 * for signature process.
613	 */
614	rv = soft_rsa_decrypt(key, plain_data, modulus_len, signed_data);
615
616	if (rv == CKR_OK) {
617		(void) memcpy(pSigned, signed_data, modulus_len);
618		*pulSignedLen = modulus_len;
619	}
620
621clean_exit:
622	(void) pthread_mutex_lock(&session_p->session_mutex);
623	free(session_p->sign.context);
624	session_p->sign.context = NULL;
625	if (session_p->digest.context != NULL) {
626		free(session_p->digest.context);
627		session_p->digest.context = NULL;
628		session_p->digest.flags = 0;
629	}
630	(void) pthread_mutex_unlock(&session_p->session_mutex);
631	soft_cleanup_object(key);
632	free(key);
633
634clean1:
635	return (rv);
636}
637
638
639CK_RV
640soft_rsa_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
641    CK_ULONG ulDataLen, CK_BYTE_PTR pSignature,
642    CK_ULONG ulSignatureLen, CK_MECHANISM_TYPE mechanism)
643{
644
645	CK_RV rv = CKR_OK;
646	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
647	soft_object_t *key = rsa_ctx->key;
648	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
649	uint32_t modulus_len = sizeof (modulus);
650	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
651
652	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
653	if (rv != CKR_OK) {
654		goto clean_exit;
655	}
656
657	if (ulDataLen == 0) {
658		rv = CKR_DATA_LEN_RANGE;
659		goto clean_exit;
660	}
661
662	if (ulSignatureLen != (CK_ULONG)modulus_len) {
663		rv = CKR_SIGNATURE_LEN_RANGE;
664		goto clean_exit;
665	}
666
667	/*
668	 * Perform RSA decryption with the signer's RSA public key
669	 * for verification process.
670	 */
671	rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
672	if (rv == CKR_OK) {
673		switch (mechanism) {
674
675		case CKM_RSA_PKCS:
676		case CKM_MD5_RSA_PKCS:
677		case CKM_SHA1_RSA_PKCS:
678		case CKM_SHA256_RSA_PKCS:
679		case CKM_SHA384_RSA_PKCS:
680		case CKM_SHA512_RSA_PKCS:
681		{
682			/*
683			 * Strip off the encoded padding bytes in front of the
684			 * recovered data, then compare the recovered data with
685			 * the original data.
686			 */
687			size_t data_len = modulus_len;
688
689			rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
690			if (rv != CKR_OK) {
691				goto clean_exit;
692			}
693
694			if ((CK_ULONG)data_len != ulDataLen) {
695				rv = CKR_DATA_LEN_RANGE;
696				goto clean_exit;
697			} else if (memcmp(pData,
698			    &plain_data[modulus_len - data_len],
699			    ulDataLen) != 0) {
700				rv = CKR_SIGNATURE_INVALID;
701				goto clean_exit;
702			}
703			break;
704		}
705
706		case CKM_RSA_X_509:
707			/*
708			 * Strip off the encoded padding bytes in front of the
709			 * recovered plain_data, then compare the input data
710			 * with the recovered data.
711			 */
712			if (memcmp(pData,
713			    plain_data + ulSignatureLen - ulDataLen,
714			    ulDataLen) != 0) {
715				rv = CKR_SIGNATURE_INVALID;
716				goto clean_exit;
717			}
718			break;
719		}
720	}
721
722	if (rv == CKR_DATA_LEN_RANGE) {
723		if ((mechanism == CKM_MD5_RSA_PKCS) ||
724		    (mechanism == CKM_SHA1_RSA_PKCS) ||
725		    (mechanism == CKM_SHA256_RSA_PKCS) ||
726		    (mechanism == CKM_SHA384_RSA_PKCS) ||
727		    (mechanism == CKM_SHA512_RSA_PKCS))
728			rv = CKR_SIGNATURE_INVALID;
729	}
730
731clean_exit:
732	(void) pthread_mutex_lock(&session_p->session_mutex);
733	free(session_p->verify.context);
734	session_p->verify.context = NULL;
735	if (session_p->digest.context != NULL) {
736		free(session_p->digest.context);
737		session_p->digest.context = NULL;
738		session_p->digest.flags = 0;
739	}
740	(void) pthread_mutex_unlock(&session_p->session_mutex);
741	soft_cleanup_object(key);
742	free(key);
743	return (rv);
744}
745
746CK_RV
747soft_genRSAkey_set_attribute(soft_object_t *key, CK_ATTRIBUTE_TYPE type,
748    uchar_t *buf, uint32_t buflen, boolean_t public)
749{
750	CK_RV rv = CKR_OK;
751	biginteger_t *dst = NULL;
752	biginteger_t src;
753
754	switch (type) {
755
756	case CKA_MODULUS:
757
758		if (public)
759			dst = OBJ_PUB_RSA_MOD(key);
760		else
761			dst = OBJ_PRI_RSA_MOD(key);
762		break;
763
764	case CKA_PUBLIC_EXPONENT:
765
766		if (public)
767			dst = OBJ_PUB_RSA_PUBEXPO(key);
768		else
769			dst = OBJ_PRI_RSA_PUBEXPO(key);
770		break;
771
772	case CKA_PRIVATE_EXPONENT:
773
774		dst = OBJ_PRI_RSA_PRIEXPO(key);
775		break;
776
777	case CKA_PRIME_1:
778
779		dst = OBJ_PRI_RSA_PRIME1(key);
780		break;
781
782	case CKA_PRIME_2:
783
784		dst = OBJ_PRI_RSA_PRIME2(key);
785		break;
786
787	case CKA_EXPONENT_1:
788
789		dst = OBJ_PRI_RSA_EXPO1(key);
790		break;
791
792	case CKA_EXPONENT_2:
793
794		dst = OBJ_PRI_RSA_EXPO2(key);
795		break;
796
797	case CKA_COEFFICIENT:
798
799		dst = OBJ_PRI_RSA_COEF(key);
800		break;
801	}
802
803	/* Note: no explanation found for why this is needed */
804	while (buf[0] == 0) {	/* remove proceeding 0x00 */
805		buf++;
806		buflen--;
807	}
808
809	if ((rv = dup_bigint_attr(&src, buf, buflen)) != CKR_OK)
810		goto cleanexit;
811
812	/* Copy the attribute in the key object. */
813	copy_bigint_attr(&src, dst);
814
815cleanexit:
816	return (rv);
817
818}
819
820
821CK_RV
822soft_rsa_genkey_pair(soft_object_t *pubkey, soft_object_t *prikey)
823{
824	CK_RV rv = CKR_OK;
825	CK_ATTRIBUTE template;
826	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
827	uint32_t modulus_len;
828	uchar_t pub_expo[MAX_KEY_ATTR_BUFLEN];
829	uint32_t pub_expo_len = sizeof (pub_expo);
830	uchar_t private_exponent[MAX_KEY_ATTR_BUFLEN];
831	uint32_t private_exponent_len = sizeof (private_exponent);
832	uchar_t prime1[MAX_KEY_ATTR_BUFLEN];
833	uint32_t prime1_len = sizeof (prime1);
834	uchar_t prime2[MAX_KEY_ATTR_BUFLEN];
835	uint32_t prime2_len = sizeof (prime2);
836	uchar_t exponent1[MAX_KEY_ATTR_BUFLEN];
837	uint32_t exponent1_len = sizeof (exponent1);
838	uchar_t exponent2[MAX_KEY_ATTR_BUFLEN];
839	uint32_t exponent2_len = sizeof (exponent2);
840	uchar_t coefficient[MAX_KEY_ATTR_BUFLEN];
841	uint32_t coefficient_len = sizeof (coefficient);
842	RSAbytekey k;
843
844	if ((pubkey == NULL) || (prikey == NULL)) {
845		return (CKR_ARGUMENTS_BAD);
846	}
847
848	template.pValue = malloc(sizeof (CK_ULONG));
849	if (template.pValue == NULL) {
850		return (CKR_HOST_MEMORY);
851	}
852	template.ulValueLen = sizeof (CK_ULONG);
853
854	rv = get_ulong_attr_from_object(OBJ_PUB_RSA_MOD_BITS(pubkey),
855	    &template);
856	if (rv != CKR_OK) {
857		free(template.pValue);
858		goto clean0;
859	}
860
861#ifdef	__sparcv9
862	/* LINTED */
863	modulus_len = (uint32_t)(*((CK_ULONG *)(template.pValue)));
864#else	/* !__sparcv9 */
865	modulus_len = *((CK_ULONG *)(template.pValue));
866#endif	/* __sparcv9 */
867
868	free(template.pValue);
869
870	rv = soft_get_public_value(pubkey, CKA_PUBLIC_EXPONENT, pub_expo,
871	    &pub_expo_len);
872	if (rv != CKR_OK) {
873		goto clean0;
874	}
875
876	/* Inputs to RSA key pair generation */
877	k.modulus_bits = modulus_len;		/* save modulus len in bits  */
878	modulus_len = CRYPTO_BITS2BYTES(modulus_len);	/* convert to bytes */
879	k.modulus = modulus;
880	k.pubexpo = pub_expo;
881	k.pubexpo_bytes = pub_expo_len;
882	k.rfunc = (IS_TOKEN_OBJECT(pubkey) || IS_TOKEN_OBJECT(prikey)) ?
883	    pkcs11_get_random : pkcs11_get_urandom;
884
885	/* Outputs from RSA key pair generation */
886	k.privexpo = private_exponent;
887	k.privexpo_bytes = private_exponent_len;
888	k.prime1 = prime1;
889	k.prime1_bytes = prime1_len;
890	k.prime2 = prime2;
891	k.prime2_bytes = prime2_len;
892	k.expo1 = exponent1;
893	k.expo1_bytes = exponent1_len;
894	k.expo2 = exponent2;
895	k.expo2_bytes = exponent2_len;
896	k.coeff = coefficient;
897	k.coeff_bytes = coefficient_len;
898
899	rv = rsa_genkey_pair(&k);
900
901	if (rv != CKR_OK) {
902		goto clean0;
903	}
904
905	/*
906	 * Add modulus in public template, and add all eight key fields
907	 * in private template.
908	 */
909	if ((rv = soft_genRSAkey_set_attribute(pubkey, CKA_MODULUS,
910	    modulus, CRYPTO_BITS2BYTES(k.modulus_bits), B_TRUE)) != CKR_OK) {
911		goto clean0;
912	}
913
914	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_MODULUS,
915	    modulus, CRYPTO_BITS2BYTES(k.modulus_bits), B_FALSE)) != CKR_OK) {
916		goto clean0;
917	}
918
919	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIVATE_EXPONENT,
920	    private_exponent, k.privexpo_bytes, B_FALSE)) != CKR_OK) {
921		goto clean0;
922	}
923
924	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PUBLIC_EXPONENT,
925	    pub_expo, k.pubexpo_bytes, B_FALSE)) != CKR_OK) {
926		goto clean0;
927	}
928
929	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIME_1,
930	    prime1, k.prime1_bytes, B_FALSE)) != CKR_OK) {
931		goto clean0;
932	}
933
934	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_PRIME_2,
935	    prime2, k.prime2_bytes, B_FALSE)) != CKR_OK) {
936		goto clean0;
937	}
938
939	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_EXPONENT_1,
940	    exponent1, k.expo1_bytes, B_FALSE)) != CKR_OK) {
941		goto clean0;
942	}
943
944	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_EXPONENT_2,
945	    exponent2, k.expo2_bytes, B_FALSE)) != CKR_OK) {
946		goto clean0;
947	}
948
949	if ((rv = soft_genRSAkey_set_attribute(prikey, CKA_COEFFICIENT,
950	    coefficient, k.coeff_bytes, B_FALSE)) != CKR_OK) {
951		goto clean0;
952	}
953
954clean0:
955	return (rv);
956}
957
958
959CK_ULONG
960get_rsa_sha1_prefix(CK_MECHANISM_PTR mech, CK_BYTE_PTR *prefix) {
961	if (mech->pParameter == NULL) {
962		*prefix = (CK_BYTE *)SHA1_DER_PREFIX;
963		return (SHA1_DER_PREFIX_Len);
964	}
965
966	*prefix = (CK_BYTE *)SHA1_DER_PREFIX_OID;
967	return (SHA1_DER_PREFIX_OID_Len);
968}
969
970CK_RV
971soft_rsa_digest_sign_common(soft_session_t *session_p, CK_BYTE_PTR pData,
972    CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
973    CK_ULONG_PTR pulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
974{
975
976	CK_RV rv = CKR_OK;
977	CK_BYTE hash[SHA512_DIGEST_LENGTH];  /* space enough for all mechs */
978	CK_ULONG hash_len = SHA512_DIGEST_LENGTH;
979	/* space enough for all mechs */
980	CK_BYTE der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
981	CK_ULONG der_data_len;
982	soft_rsa_ctx_t *rsa_ctx = session_p->sign.context;
983	soft_object_t *key = rsa_ctx->key;
984	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
985	uint32_t modulus_len = sizeof (modulus);
986	CK_ULONG der_len;
987	CK_BYTE_PTR der_prefix;
988
989	rv = soft_get_private_value(key, CKA_MODULUS, modulus, &modulus_len);
990	if (rv != CKR_OK) {
991		(void) pthread_mutex_lock(&session_p->session_mutex);
992		free(session_p->digest.context);
993		session_p->digest.context = NULL;
994		session_p->digest.flags = 0;
995		(void) pthread_mutex_unlock(&session_p->session_mutex);
996		soft_cleanup_object(key);
997		free(key);
998		goto clean1;
999	}
1000
1001	/* Check arguments before performing message digest. */
1002	if (pSigned == NULL) {
1003		/* Application asks for the length of the output buffer. */
1004		*pulSignedLen = modulus_len;
1005		rv = CKR_OK;
1006		goto clean1;
1007	}
1008
1009	/* Is the application-supplied buffer large enough? */
1010	if (*pulSignedLen < (CK_ULONG)modulus_len) {
1011		*pulSignedLen = modulus_len;
1012		rv = CKR_BUFFER_TOO_SMALL;
1013		goto clean1;
1014	}
1015
1016	if (Final) {
1017		rv = soft_digest_final(session_p, hash, &hash_len);
1018	} else {
1019		rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
1020	}
1021
1022	if (rv != CKR_OK) {
1023		/* free the signature key */
1024		soft_cleanup_object(key);
1025		free(key);
1026		goto clean_exit;
1027	}
1028
1029	/*
1030	 * Prepare the DER encoding of the DigestInfo value by setting it to:
1031	 *	<MECH>_DER_PREFIX || H
1032	 *
1033	 * See rsa_impl.c for more details.
1034	 */
1035	switch (session_p->digest.mech.mechanism) {
1036	case CKM_MD5:
1037		(void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
1038		(void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
1039		der_data_len = MD5_DER_PREFIX_Len + hash_len;
1040		break;
1041	case CKM_SHA_1:
1042		der_len = get_rsa_sha1_prefix(&(session_p->digest.mech),
1043		    &der_prefix);
1044		(void) memcpy(der_data, der_prefix, der_len);
1045		(void) memcpy(der_data + der_len, hash, hash_len);
1046		der_data_len = der_len + hash_len;
1047		break;
1048	case CKM_SHA256:
1049		(void) memcpy(der_data, SHA256_DER_PREFIX,
1050		    SHA2_DER_PREFIX_Len);
1051		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1052		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1053		break;
1054	case CKM_SHA384:
1055		(void) memcpy(der_data, SHA384_DER_PREFIX,
1056		    SHA2_DER_PREFIX_Len);
1057		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1058		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1059		break;
1060	case CKM_SHA512:
1061		(void) memcpy(der_data, SHA512_DER_PREFIX,
1062		    SHA2_DER_PREFIX_Len);
1063		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1064		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1065		break;
1066	}
1067
1068	/*
1069	 * Now, we are ready to sign the DER_ENCODED data
1070	 * soft_rsa_sign_common() will free the signature key.
1071	 */
1072	rv = soft_rsa_sign_common(session_p, der_data, der_data_len,
1073	    pSigned, pulSignedLen, mechanism);
1074
1075clean_exit:
1076	(void) pthread_mutex_lock(&session_p->session_mutex);
1077	/* soft_digest_common() has freed the digest context */
1078	session_p->digest.flags = 0;
1079	(void) pthread_mutex_unlock(&session_p->session_mutex);
1080
1081clean1:
1082	return (rv);
1083}
1084
1085
1086CK_RV
1087soft_rsa_digest_verify_common(soft_session_t *session_p, CK_BYTE_PTR pData,
1088    CK_ULONG ulDataLen, CK_BYTE_PTR pSigned,
1089    CK_ULONG ulSignedLen, CK_MECHANISM_TYPE mechanism, boolean_t Final)
1090{
1091
1092	CK_RV rv = CKR_OK;
1093	CK_BYTE hash[SHA512_DIGEST_LENGTH];  /* space for all mechs */
1094	CK_ULONG hash_len = SHA512_DIGEST_LENGTH;
1095	CK_BYTE der_data[SHA512_DIGEST_LENGTH + SHA2_DER_PREFIX_Len];
1096	CK_ULONG der_data_len;
1097	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
1098	soft_object_t *key = rsa_ctx->key;
1099	CK_ULONG der_len;
1100	CK_BYTE_PTR der_prefix;
1101
1102	if (Final) {
1103		rv = soft_digest_final(session_p, hash, &hash_len);
1104	} else {
1105		rv = soft_digest(session_p, pData, ulDataLen, hash, &hash_len);
1106	}
1107
1108	if (rv != CKR_OK) {
1109		/* free the verification key */
1110		soft_cleanup_object(key);
1111		free(key);
1112		goto clean_exit;
1113	}
1114
1115	/*
1116	 * Prepare the DER encoding of the DigestInfo value as follows:
1117	 * MD5:		MD5_DER_PREFIX || H
1118	 * SHA-1:	SHA1_DER_PREFIX || H
1119	 * SHA2:	SHA2_DER_PREFIX || H
1120	 *
1121	 * See rsa_impl.c for more details.
1122	 */
1123	switch (session_p->digest.mech.mechanism) {
1124	case CKM_MD5:
1125		(void) memcpy(der_data, MD5_DER_PREFIX, MD5_DER_PREFIX_Len);
1126		(void) memcpy(der_data + MD5_DER_PREFIX_Len, hash, hash_len);
1127		der_data_len = MD5_DER_PREFIX_Len + hash_len;
1128		break;
1129	case CKM_SHA_1:
1130		der_len = get_rsa_sha1_prefix(&(session_p->digest.mech),
1131		    &der_prefix);
1132		(void) memcpy(der_data, der_prefix, der_len);
1133		(void) memcpy(der_data + der_len, hash, hash_len);
1134		der_data_len = der_len + hash_len;
1135		break;
1136	case CKM_SHA256:
1137		(void) memcpy(der_data, SHA256_DER_PREFIX,
1138		    SHA2_DER_PREFIX_Len);
1139		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1140		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1141		break;
1142	case CKM_SHA384:
1143		(void) memcpy(der_data, SHA384_DER_PREFIX,
1144		    SHA2_DER_PREFIX_Len);
1145		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1146		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1147		break;
1148	case CKM_SHA512:
1149		(void) memcpy(der_data, SHA512_DER_PREFIX,
1150		    SHA2_DER_PREFIX_Len);
1151		(void) memcpy(der_data + SHA2_DER_PREFIX_Len, hash, hash_len);
1152		der_data_len = SHA2_DER_PREFIX_Len + hash_len;
1153		break;
1154	}
1155
1156	/*
1157	 * Now, we are ready to verify the DER_ENCODED data using signature.
1158	 * soft_rsa_verify_common() will free the verification key.
1159	 */
1160	rv = soft_rsa_verify_common(session_p, der_data, der_data_len,
1161	    pSigned, ulSignedLen, mechanism);
1162
1163clean_exit:
1164	(void) pthread_mutex_lock(&session_p->session_mutex);
1165	/* soft_digest_common() has freed the digest context */
1166	session_p->digest.flags = 0;
1167	(void) pthread_mutex_unlock(&session_p->session_mutex);
1168
1169	return (rv);
1170
1171}
1172
1173
1174CK_RV
1175soft_rsa_verify_recover(soft_session_t *session_p, CK_BYTE_PTR pSignature,
1176    CK_ULONG ulSignatureLen, CK_BYTE_PTR pData, CK_ULONG_PTR pulDataLen)
1177{
1178
1179	CK_RV rv = CKR_OK;
1180	soft_rsa_ctx_t *rsa_ctx = session_p->verify.context;
1181	CK_MECHANISM_TYPE mechanism = session_p->verify.mech.mechanism;
1182	soft_object_t *key = rsa_ctx->key;
1183	uchar_t modulus[MAX_KEY_ATTR_BUFLEN];
1184	uint32_t modulus_len = sizeof (modulus);
1185	CK_BYTE	plain_data[MAX_RSA_KEYLENGTH_IN_BYTES];
1186
1187	rv = soft_get_public_value(key, CKA_MODULUS, modulus, &modulus_len);
1188	if (rv != CKR_OK) {
1189		goto clean_exit;
1190	}
1191
1192	if (ulSignatureLen != (CK_ULONG)modulus_len) {
1193		rv = CKR_SIGNATURE_LEN_RANGE;
1194		goto clean_exit;
1195	}
1196
1197	/*
1198	 * Perform RSA decryption with the signer's RSA public key
1199	 * for verification process.
1200	 */
1201	rv = soft_rsa_encrypt(key, pSignature, modulus_len, plain_data, 1);
1202	if (rv == CKR_OK) {
1203		switch (mechanism) {
1204
1205		case CKM_RSA_PKCS:
1206		{
1207			/*
1208			 * Strip off the encoded padding bytes in front of the
1209			 * recovered data.
1210			 */
1211			size_t data_len = modulus_len;
1212
1213			rv = pkcs1_decode(PKCS1_VERIFY, plain_data, &data_len);
1214			if (rv != CKR_OK) {
1215				goto clean_exit;
1216			}
1217
1218			/*
1219			 * If application asks for the length of the output
1220			 * buffer?
1221			 */
1222			if (pData == NULL) {
1223				*pulDataLen = data_len;
1224				rv = CKR_OK;
1225				goto clean1;
1226			}
1227
1228			/* Is the application-supplied buffer large enough? */
1229			if (*pulDataLen < (CK_ULONG)data_len) {
1230				*pulDataLen = data_len;
1231				rv = CKR_BUFFER_TOO_SMALL;
1232				goto clean1;
1233			}
1234
1235			(void) memcpy(pData,
1236			    &plain_data[modulus_len - data_len], data_len);
1237			*pulDataLen = data_len;
1238
1239			break;
1240		}
1241
1242		case CKM_RSA_X_509:
1243			/*
1244			 * If application asks for the length of the output
1245			 * buffer?
1246			 */
1247			if (pData == NULL) {
1248				*pulDataLen = modulus_len;
1249				rv = CKR_OK;
1250				goto clean1;
1251			}
1252
1253			/* Is the application-supplied buffer large enough? */
1254			if (*pulDataLen < (CK_ULONG)modulus_len) {
1255				*pulDataLen = modulus_len;
1256				rv = CKR_BUFFER_TOO_SMALL;
1257				goto clean1;
1258			}
1259
1260			(void) memcpy(pData, plain_data, modulus_len);
1261			*pulDataLen = modulus_len;
1262
1263			break;
1264		}
1265	}
1266
1267clean_exit:
1268	(void) pthread_mutex_lock(&session_p->session_mutex);
1269	free(session_p->verify.context);
1270	session_p->verify.context = NULL;
1271	(void) pthread_mutex_unlock(&session_p->session_mutex);
1272	soft_cleanup_object(key);
1273	free(key);
1274
1275clean1:
1276	return (rv);
1277}
1278