xref: /illumos-gate/usr/src/lib/gss_mechs/mech_krb5/crypto/keyhash_provider/k5_md5des.c (revision 7c478bd95313f5f23a4c958a745db2134aa03244)
1 /*
2  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
3  * Use is subject to license terms.
4  */
5 
6 #pragma ident	"%Z%%M%	%I%	%E% SMI"
7 
8 /*
9  * Copyright (C) 1998 by the FundsXpress, INC.
10  *
11  * All rights reserved.
12  *
13  * Export of this software from the United States of America may require
14  * a specific license from the United States Government.  It is the
15  * responsibility of any person or organization contemplating export to
16  * obtain such a license before exporting.
17  *
18  * WITHIN THAT CONSTRAINT, permission to use, copy, modify, and
19  * distribute this software and its documentation for any purpose and
20  * without fee is hereby granted, provided that the above copyright
21  * notice appear in all copies and that both that copyright notice and
22  * this permission notice appear in supporting documentation, and that
23  * the name of FundsXpress. not be used in advertising or publicity pertaining
24  * to distribution of the software without specific, written prior
25  * permission.  FundsXpress makes no representations about the suitability of
26  * this software for any purpose.  It is provided "as is" without express
27  * or implied warranty.
28  *
29  * THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
30  * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
31  * WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
32  */
33 
34 #include <k5-int.h>
35 #include <des_int.h>
36 #include <keyhash_provider.h>
37 
38 #define CONFLENGTH 8
39 
40 /* Force acceptance of krb5-beta5 md5des checksum for now. */
41 #define KRB5_MD5DES_BETA5_COMPAT
42 
43 static const mit_des_cblock mit_des_zeroblock[8] = {
44 	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
45 	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
46 	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
47 	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
48 	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
49 	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
50 	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
51 	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00} };
52 
53 static void
54 k5_md5des_hash_size(size_t *output)
55 {
56     *output = CONFLENGTH + MD5_CKSUM_LENGTH;
57 }
58 
59 /* des-cbc(xorkey, conf | rsa-md5(conf | data)) */
60 
61 /* this could be done in terms of the md5 and des providers, but
62    that's less efficient, and there's no need for this to be generic */
63 
64 /*ARGSUSED*/
65 static krb5_error_code
66 k5_md5des_hash(krb5_context context,
67 	krb5_const krb5_keyblock *key,
68 	krb5_keyusage usage,
69 	krb5_const krb5_data *ivec,
70 	krb5_const krb5_data *input, krb5_data *output)
71 {
72     krb5_error_code ret = 0;
73     krb5_data data;
74     unsigned char conf[CONFLENGTH];
75     krb5_keyblock xorkey;
76     int i;
77     CK_MECHANISM mechanism;
78     CK_RV rv;
79     CK_ULONG hashlen = MD5_CKSUM_LENGTH;
80 
81     if (key->length != 8)
82 	return(KRB5_BAD_KEYSIZE);
83     if (ivec)
84 	return(KRB5_CRYPTO_INTERNAL);
85     if (output->length != (CONFLENGTH+MD5_CKSUM_LENGTH))
86 	return(KRB5_CRYPTO_INTERNAL);
87 
88     /* create the confouder */
89 
90     data.length = CONFLENGTH;
91     data.data = (char *) conf;
92     if ((ret = krb5_c_random_make_octets(context, &data)))
93 	return(ret);
94 
95     xorkey.magic = key->magic;
96     xorkey.enctype = key->enctype;
97     xorkey.length = key->length;
98     xorkey.contents = (krb5_octet *)malloc(key->length);
99     if (xorkey.contents == NULL)
100 	return(KRB5_CRYPTO_INTERNAL);
101 
102     (void) memcpy(xorkey.contents, key->contents, xorkey.length);
103 
104     for (i=0; i<xorkey.length; i++)
105 	xorkey.contents[i] ^= 0xf0;
106 
107     if (!mit_des_check_key_parity(xorkey.contents)) {
108 	ret = KRB5DES_BAD_KEYPAR;
109 	goto cleanup;
110     }
111 
112     if (mit_des_is_weak_key(xorkey.contents)) {
113 	ret = KRB5DES_WEAK_KEY;
114 	goto cleanup;
115     }
116 
117     /* hash the confounder, then the input data */
118     mechanism.mechanism = CKM_MD5;
119     mechanism.pParameter = NULL_PTR;
120     mechanism.ulParameterLen = 0;
121 
122     if ((rv = C_DigestInit(krb_ctx_hSession(context), &mechanism)) != CKR_OK) {
123 	KRB5_LOG(KRB5_ERR, "C_DigestInit failed in k5_md5des_hash: "
124 	"rv = 0x%x.", rv);
125 	ret = PKCS_ERR;
126 	goto cleanup;
127     }
128 
129     if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
130 	(CK_BYTE_PTR)conf, (CK_ULONG)sizeof(conf))) != CKR_OK) {
131 	KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_hash: "
132 	    "rv = 0x%x", rv);
133 	ret = PKCS_ERR;
134 	goto cleanup;
135     }
136 
137     if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
138 	(CK_BYTE_PTR)input->data, (CK_ULONG)input->length)) != CKR_OK) {
139 	KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_hash: "
140 	    "rv = 0x%x", rv);
141 	return(PKCS_ERR);
142     }
143 
144     if ((rv = C_DigestFinal(krb_ctx_hSession(context),
145 	(CK_BYTE_PTR)(output->data + CONFLENGTH),
146 	(CK_ULONG_PTR)&hashlen)) != CKR_OK) {
147 	KRB5_LOG(KRB5_ERR, "C_DigestFinal failed in k5_md5des_hash: "
148 	    "rv = 0x%x", rv);
149 	ret = PKCS_ERR;
150 	goto cleanup;
151     }
152 
153     /* construct the buffer to be encrypted */
154 
155     (void) memcpy(output->data, conf, CONFLENGTH);
156 
157     /* encrypt it, in place.  this has a return value, but it's
158        always zero.  */
159 
160     ret = mit_des_cbc_encrypt(context,
161 	(krb5_pointer) output->data,
162 	(krb5_pointer) output->data, output->length,
163 	&xorkey, (unsigned char*) mit_des_zeroblock, 1);
164 
165 cleanup:
166     free(xorkey.contents);
167     return(ret);
168 }
169 
170 /*ARGSUSED*/
171 static krb5_error_code
172 k5_md5des_verify(krb5_context context,
173 	krb5_const krb5_keyblock *key,
174 	krb5_keyusage usage,
175 	krb5_const krb5_data *ivec,
176 	krb5_const krb5_data *input,
177 	krb5_const krb5_data *hash,
178 	krb5_boolean *valid)
179 {
180     krb5_error_code ret = 0;
181     unsigned char plaintext[CONFLENGTH+MD5_CKSUM_LENGTH];
182     unsigned char digest[MD5_CKSUM_LENGTH];
183     krb5_keyblock xorkey;
184     int i;
185     int compathash = 0;
186     CK_MECHANISM mechanism;
187     CK_RV rv;
188     CK_ULONG hashlen = MD5_CKSUM_LENGTH;
189 
190     if (key->length != 8)
191 	return(KRB5_BAD_KEYSIZE);
192     if (ivec)
193 	return(KRB5_CRYPTO_INTERNAL);
194     if (hash->length != (CONFLENGTH + MD5_CKSUM_LENGTH)) {
195 #ifdef KRB5_MD5DES_BETA5_COMPAT
196 	if (hash->length != MD5_CKSUM_LENGTH)
197 	    return(KRB5_CRYPTO_INTERNAL);
198 	else
199 	    compathash = 1;
200 #else
201 	return(KRB5_CRYPTO_INTERNAL);
202 #endif
203     }
204 
205     /* create and the encryption key */
206     xorkey.magic = key->magic;
207     xorkey.enctype = key->enctype;
208     xorkey.length = key->length;
209     xorkey.contents = (krb5_octet *)malloc(key->length);
210     if (xorkey.contents == NULL)
211 	return(KRB5_CRYPTO_INTERNAL);
212 
213     (void) memcpy(xorkey.contents, key->contents, xorkey.length);
214     if (!compathash) {
215         for (i=0; i<xorkey.length; i++)
216 	    xorkey.contents[i] ^= 0xf0;
217     }
218 
219     if (!mit_des_check_key_parity(xorkey.contents)) {
220 	ret = KRB5DES_BAD_KEYPAR;
221 	goto cleanup;
222     }
223 
224     if (mit_des_is_weak_key(xorkey.contents)) {
225 	ret = KRB5DES_WEAK_KEY;
226 	goto cleanup;
227     }
228 
229     /* decrypt it.  this has a return value, but it's always zero.  */
230     if (!compathash) {
231 	ret = mit_des_cbc_encrypt(context,
232 		(krb5_pointer) hash->data,
233 		(krb5_pointer) plaintext, hash->length,
234 		&xorkey, (unsigned char*) mit_des_zeroblock, 0);
235     } else {
236 	ret = mit_des_cbc_encrypt(context,
237 		(krb5_pointer) hash->data,
238 		(krb5_pointer) plaintext, hash->length,
239 		&xorkey, xorkey.contents, 0);
240     }
241     if (ret) goto cleanup;
242 
243     /* hash the confounder, then the input data */
244     mechanism.mechanism = CKM_MD5;
245     mechanism.pParameter = NULL_PTR;
246     mechanism.ulParameterLen = 0;
247 
248     if ((rv = C_DigestInit(krb_ctx_hSession(context), &mechanism)) != CKR_OK) {
249 	KRB5_LOG(KRB5_ERR, "C_DigestInit failed in k5_md5des_verify: "
250 	"rv = 0x%x.", rv);
251 	ret = PKCS_ERR;
252 	goto cleanup;
253     }
254 
255     if (!compathash) {
256 	if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
257 	    (CK_BYTE_PTR)plaintext, (CK_ULONG)CONFLENGTH)) != CKR_OK) {
258 	    KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_verify: "
259 		"rv = 0x%x", rv);
260 	    ret = PKCS_ERR;
261 	    goto cleanup;
262 	}
263     }
264     if ((rv = C_DigestUpdate(krb_ctx_hSession(context),
265 	(CK_BYTE_PTR)input->data, (CK_ULONG)input->length)) != CKR_OK) {
266 	KRB5_LOG(KRB5_ERR, "C_DigestUpdate failed in k5_md5des_verify: "
267 	    "rv = 0x%x", rv);
268 	ret = PKCS_ERR;
269 	goto cleanup;
270     }
271     if ((rv = C_DigestFinal(krb_ctx_hSession(context),
272 	(CK_BYTE_PTR)digest, (CK_ULONG_PTR)&hashlen)) != CKR_OK) {
273 	KRB5_LOG(KRB5_ERR, "C_DigestFinal failed in k5_md5des_verify: "
274 	    "rv = 0x%x", rv);
275 	ret = PKCS_ERR;
276 	goto cleanup;
277     }
278 
279     /* compare the decrypted hash to the computed one */
280 
281     if (!compathash) {
282 	*valid = (memcmp(plaintext+CONFLENGTH, digest, sizeof(digest)) == 0);
283     } else {
284 	*valid = (memcmp(plaintext, digest, sizeof(digest)) == 0);
285     }
286     (void) memset(plaintext, 0, sizeof(plaintext));
287 
288 cleanup:
289     free(xorkey.contents);
290     return(ret);
291 }
292 
293 const struct krb5_keyhash_provider krb5_keyhash_md5des = {
294     k5_md5des_hash_size,
295     k5_md5des_hash,
296     k5_md5des_verify
297 };
298