xref: /illumos-gate/usr/src/lib/libnsl/key/xcrypt.c (revision 7c478bd9)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2004 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 /* All Rights Reserved */
29 
30 /*
31  * Portions of this source code were derived from Berkeley 4.3 BSD
32  * under license from the Regents of the University of California.
33  */
34 
35 #pragma ident	"%Z%%M%	%I%	%E% SMI"
36 
37 /*
38  * xcrypt.c: Hex encryption/decryption and utility routines
39  */
40 
41 #include <stdio.h>
42 #include <stdlib.h>
43 #include <sys/types.h>
44 #include <rpc/trace.h>
45 #include <rpc/rpc.h>
46 #include <rpc/key_prot.h>   /* for KEYCHECKSUMSIZE */
47 #include <rpc/des_crypt.h>
48 #include <string.h>
49 #include <rpcsvc/nis_dhext.h>
50 #include <md5.h>
51 
52 #define	MD5HEXSIZE	32
53 
54 extern int bin2hex(int len, unsigned char *binnum, char *hexnum);
55 extern int hex2bin(int len, char *hexnum, char *binnum);
56 static char hex[];	/* forward */
57 static char hexval();
58 
59 int passwd2des(char *, char *);
60 static int weak_DES_key(des_block);
61 
62 /* EXPORT DELETE START */
63 /*
64  * For export control reasons, we want to limit the maximum size of
65  * data that can be encrypted or decrypted.  We limit this to 1024
66  * bits of key data, which amounts to 128 bytes.
67  *
68  * For the extended DH project, we have increased it to
69  * 144 bytes (128key + 16checksum) to accomadate all the 128 bytes
70  * being used by the new 1024bit keys plus 16 bytes MD5 checksum.
71  * We discussed this with Sun's export control office and lawyers
72  * and we have reason to believe this is ok for export.
73  */
74 #define	MAX_KEY_CRYPT_LEN	144
75 /* EXPORT DELETE END */
76 
77 /*
78  * Encrypt a secret key given passwd
79  * The secret key is passed and returned in hex notation.
80  * Its length must be a multiple of 16 hex digits (64 bits).
81  */
82 int
83 xencrypt(secret, passwd)
84 	char *secret;
85 	char *passwd;
86 {
87 /* EXPORT DELETE START */
88 	char key[8];
89 	char ivec[8];
90 	char *buf;
91 	int err;
92 	int len;
93 
94 	trace1(TR_xencrypt, 0);
95 	len = (int)strlen(secret) / 2;
96 	if (len > MAX_KEY_CRYPT_LEN) {
97 		trace1(TR_xencrypt, 1);
98 		return (0);
99 	}
100 	buf = malloc((unsigned)len);
101 	(void) hex2bin(len, secret, buf);
102 	(void) passwd2des(passwd, key);
103 	(void) memset(ivec, 0, 8);
104 
105 	err = cbc_crypt(key, buf, len, DES_ENCRYPT | DES_HW, ivec);
106 	if (DES_FAILED(err)) {
107 		free(buf);
108 		trace1(TR_xencrypt, 1);
109 		return (0);
110 	}
111 	(void) bin2hex(len, (unsigned char *) buf, secret);
112 	free(buf);
113 	trace1(TR_xencrypt, 1);
114 	return (1);
115 #if 0
116 /* EXPORT DELETE END */
117 	trace1(TR_xencrypt, 1);
118 	return (0);
119 /* EXPORT DELETE START */
120 #endif
121 /* EXPORT DELETE END */
122 }
123 
124 /*
125  * Decrypt secret key using passwd
126  * The secret key is passed and returned in hex notation.
127  * Once again, the length is a multiple of 16 hex digits
128  */
129 int
130 xdecrypt(secret, passwd)
131 	char *secret;
132 	char *passwd;
133 {
134 /* EXPORT DELETE START */
135 	char key[8];
136 	char ivec[8];
137 	char *buf;
138 	int err;
139 	int len;
140 
141 	trace1(TR_xdecrypt, 0);
142 	len = (int)strlen(secret) / 2;
143 	if (len > MAX_KEY_CRYPT_LEN) {
144 		trace1(TR_xdecrypt, 1);
145 		return (0);
146 	}
147 	buf = malloc((unsigned)len);
148 
149 	(void) hex2bin(len, secret, buf);
150 	(void) passwd2des(passwd, key);
151 	(void) memset(ivec, 0, 8);
152 
153 	err = cbc_crypt(key, buf, len, DES_DECRYPT | DES_HW, ivec);
154 	if (DES_FAILED(err)) {
155 		free(buf);
156 		trace1(TR_xdecrypt, 1);
157 		return (0);
158 	}
159 	(void) bin2hex(len, (unsigned char *) buf, secret);
160 	free(buf);
161 	trace1(TR_xdecrypt, 1);
162 	return (1);
163 #if 0
164 /* EXPORT DELETE END */
165 	trace1(TR_xdecrypt, 1);
166 	return (0);
167 /* EXPORT DELETE START */
168 #endif
169 /* EXPORT DELETE END */
170 }
171 
172 /*
173  * Turn password into DES key
174  */
175 int
176 passwd2des(pw, key)
177 	char *pw;
178 	char *key;
179 {
180 	int i;
181 
182 	trace1(TR_passwd2des, 0);
183 	(void) memset(key, 0, 8);
184 	for (i = 0; *pw; i = (i+1) % 8) {
185 		key[i] ^= *pw++ << 1;
186 	}
187 	des_setparity(key);
188 	trace1(TR_passwd2des, 1);
189 	return (1);
190 }
191 
192 
193 /*
194  * Hex to binary conversion
195  */
196 int
197 hex2bin(len, hexnum, binnum)
198 	int len;
199 	char *hexnum;
200 	char *binnum;
201 {
202 	int i;
203 
204 	trace2(TR_hex2bin, 0, len);
205 	for (i = 0; i < len; i++) {
206 		*binnum++ = 16 * hexval(hexnum[2 * i]) +
207 					hexval(hexnum[2 * i + 1]);
208 	}
209 	trace1(TR_hex2bin, 1);
210 	return (1);
211 }
212 
213 /*
214  * Binary to hex conversion
215  */
216 int
217 bin2hex(len, binnum, hexnum)
218 	int len;
219 	unsigned char *binnum;
220 	char *hexnum;
221 {
222 	int i;
223 	unsigned val;
224 
225 	trace2(TR_bin2hex, 0, len);
226 	for (i = 0; i < len; i++) {
227 		val = binnum[i];
228 		hexnum[i*2] = hex[val >> 4];
229 		hexnum[i*2+1] = hex[val & 0xf];
230 	}
231 	hexnum[len*2] = 0;
232 	trace1(TR_bin2hex, 1);
233 	return (1);
234 }
235 
236 static char hex[16] = {
237 	'0', '1', '2', '3', '4', '5', '6', '7',
238 	'8', '9', 'a', 'b', 'c', 'd', 'e', 'f',
239 };
240 
241 static char
242 hexval(c)
243 	char c;
244 {
245 	trace1(TR_hexval, 0);
246 	if (c >= '0' && c <= '9') {
247 		trace1(TR_hexval, 1);
248 		return (c - '0');
249 	} else if (c >= 'a' && c <= 'z') {
250 		trace1(TR_hexval, 1);
251 		return (c - 'a' + 10);
252 	} else if (c >= 'A' && c <= 'Z') {
253 		trace1(TR_hexval, 1);
254 		return (c - 'A' + 10);
255 	} else {
256 		trace1(TR_hexval, 1);
257 		return (-1);
258 	}
259 }
260 
261 /*
262  * Generic key length/algorithm version of xencrypt().
263  *
264  * Encrypt a secret key given passwd.
265  * The secret key is passed in hex notation.
266  * Arg encrypted_secret will be set to point to the encrypted
267  * secret key (NUL term, hex notation).
268  *
269  * Its length must be a multiple of 16 hex digits (64 bits).
270  *
271  * For 192-0 (AUTH_DES), then encrypt using the same method as xencrypt().
272  *
273  * If arg do_chksum is TRUE, append the checksum before the encrypt.
274  * For 192-0, the checksum is done the same as in xencrypt().  For
275  * bigger keys, MD5 is used.
276  *
277  * Arg netname can be NULL for 192-0.
278  */
279 int
280 xencrypt_g(
281 	char *secret,			/* in  */
282 	keylen_t keylen,		/* in  */
283 	algtype_t algtype,		/* in  */
284 	const char *passwd,		/* in  */
285 	const char netname[],  		/* in  */
286 	char **encrypted_secret,	/* out */
287 	bool_t do_chksum)		/* in  */
288 {
289 /* EXPORT DELETE START */
290 	des_block key;
291 	char ivec[8];
292 	char *binkeybuf;
293 	int err;
294 	const int classic_des = keylen == 192 && algtype == 0;
295 	const int hexkeybytes = BITS2NIBBLES(keylen);
296 	const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
297 	const int binkeybytes = do_chksum ? keylen/8 + keychecksumsize/2 :
298 		keylen/8;
299 	const int bufsize = do_chksum ? hexkeybytes + keychecksumsize + 1 :
300 		hexkeybytes + 1;
301 	char *hexkeybuf;
302 
303 	trace1(TR_xencrypt_g, 0);
304 
305 	if (!secret || !keylen || !passwd || !encrypted_secret) {
306 		trace1(TR_xencrypt_g, 1);
307 		return (0);
308 	}
309 
310 	if ((hexkeybuf = malloc(bufsize)) == 0)
311 		return (0);
312 
313 	(void) memcpy(hexkeybuf, secret, hexkeybytes);
314 	if (do_chksum)
315 		if (classic_des) {
316 			(void) memcpy(hexkeybuf + hexkeybytes, secret,
317 					keychecksumsize);
318 		} else {
319 			MD5_CTX md5_ctx;
320 			char md5hexbuf[MD5HEXSIZE + 1] = {0};
321 			uint8_t digest[MD5HEXSIZE/2];
322 
323 			MD5Init(&md5_ctx);
324 			MD5Update(&md5_ctx, (unsigned char *)hexkeybuf,
325 					hexkeybytes);
326 			MD5Final(digest, &md5_ctx);
327 
328 			/* convert md5 binary digest to hex */
329 			(void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
330 
331 			/* append the hex md5 string to the end of the key */
332 			(void) memcpy(hexkeybuf + hexkeybytes,
333 					(void *)md5hexbuf, MD5HEXSIZE);
334 		}
335 	hexkeybuf[bufsize - 1] = 0;
336 
337 	if (binkeybytes > MAX_KEY_CRYPT_LEN) {
338 		free(hexkeybuf);
339 		trace1(TR_xencrypt_g, 1);
340 		return (0);
341 	}
342 	if ((binkeybuf = malloc((unsigned)binkeybytes)) == 0) {
343 		free(hexkeybuf);
344 		return (0);
345 	}
346 
347 	(void) hex2bin(binkeybytes, hexkeybuf, binkeybuf);
348 	if (classic_des)
349 		(void) passwd2des((char *)passwd, key.c);
350 	else
351 		if (netname)
352 			(void) passwd2des_g(passwd, netname,
353 					(int)strlen(netname), &key, FALSE);
354 		else {
355 			free(hexkeybuf);
356 			return (0);
357 		}
358 
359 	(void) memset(ivec, 0, 8);
360 
361 	err = cbc_crypt(key.c, binkeybuf, binkeybytes, DES_ENCRYPT | DES_HW,
362 			ivec);
363 	if (DES_FAILED(err)) {
364 		free(hexkeybuf);
365 		free(binkeybuf);
366 		trace1(TR_xencrypt_g, 1);
367 		return (0);
368 	}
369 	(void) bin2hex(binkeybytes, (unsigned char *) binkeybuf, hexkeybuf);
370 	free(binkeybuf);
371 	*encrypted_secret = hexkeybuf;
372 	trace1(TR_xencrypt_g, 1);
373 	return (1);
374 #if 0
375 /* EXPORT DELETE END */
376 	trace1(TR_xencrypt_g, 1);
377 	return (0);
378 /* EXPORT DELETE START */
379 #endif
380 /* EXPORT DELETE END */
381 }
382 
383 /*
384  * Generic key len and alg type for version of xdecrypt.
385  *
386  * Decrypt secret key using passwd.  The decrypted secret key
387  * *overwrites* the supplied encrypted secret key.
388  * The secret key is passed and returned in hex notation.
389  * Once again, the length is a multiple of 16 hex digits.
390  *
391  * If 'do_chksum' is TRUE, the 'secret' buffer is assumed to contain
392  * a checksum calculated by a call to xencrypt_g().
393  *
394  * If keylen is 192 and algtype is 0, then decrypt the same way
395  * as xdecrypt().
396  *
397  * Arg netname can be NULL for 192-0.
398  */
399 int
400 xdecrypt_g(
401 	char *secret,		/* out  */
402 	int keylen,		/* in  */
403 	int algtype,		/* in  */
404 	const char *passwd,	/* in  */
405 	const char netname[],	/* in  */
406 	bool_t do_chksum)	/* in  */
407 {
408 /* EXPORT DELETE START */
409 	des_block key;
410 	char ivec[8];
411 	char *buf;
412 	int err;
413 	int len;
414 	const int classic_des = keylen == 192 && algtype == 0;
415 	const int hexkeybytes = BITS2NIBBLES(keylen);
416 	const int keychecksumsize = classic_des ? KEYCHECKSUMSIZE : MD5HEXSIZE;
417 
418 	trace1(TR_xdecrypt_g, 0);
419 	len = (int)strlen(secret) / 2;
420 	if (len > MAX_KEY_CRYPT_LEN) {
421 		trace1(TR_xdecrypt_g, 1);
422 		return (0);
423 	}
424 	if ((buf = malloc((unsigned)len)) == 0)
425 		return (0);
426 
427 	(void) hex2bin(len, secret, buf);
428 	if (classic_des)
429 		(void) passwd2des((char *)passwd, key.c);
430 	else
431 		if (netname)
432 			(void) passwd2des_g(passwd, netname,
433 					(int)strlen(netname), &key, FALSE);
434 		else {
435 			free(buf);
436 			trace1(TR_xdecrypt_g, 1);
437 			return (0);
438 		}
439 	(void) memset(ivec, 0, 8);
440 
441 	err = cbc_crypt(key.c, buf, len, DES_DECRYPT | DES_HW, ivec);
442 	if (DES_FAILED(err)) {
443 		free(buf);
444 		trace1(TR_xdecrypt_g, 1);
445 		return (0);
446 	}
447 	(void) bin2hex(len, (unsigned char *) buf, secret);
448 	free(buf);
449 
450 	if (do_chksum)
451 		if (classic_des) {
452 			if (memcmp(secret, &(secret[hexkeybytes]),
453 					keychecksumsize) != 0) {
454 				secret[0] = 0;
455 				trace1(TR_xdecrypt_g, 1);
456 				return (0);
457 			}
458 		} else {
459 			MD5_CTX md5_ctx;
460 			char md5hexbuf[MD5HEXSIZE + 1] = {0};
461 			uint8_t digest[MD5HEXSIZE/2];
462 
463 			MD5Init(&md5_ctx);
464 			MD5Update(&md5_ctx, (unsigned char *)secret,
465 					hexkeybytes);
466 			MD5Final(digest, &md5_ctx);
467 
468 			/* convert md5 binary digest to hex */
469 			(void) bin2hex(MD5HEXSIZE/2, digest, md5hexbuf);
470 
471 			/* does the digest match the appended one? */
472 			if (memcmp(&(secret[hexkeybytes]),
473 					(void *)md5hexbuf, MD5HEXSIZE) != 0) {
474 				secret[0] = 0;
475 				trace1(TR_xdecrypt_g, 1);
476 				return (0);
477 			}
478 		}
479 
480 	secret[hexkeybytes] = '\0';
481 
482 	trace1(TR_xdecrypt_g, 1);
483 	return (1);
484 #if 0
485 /* EXPORT DELETE END */
486 	trace1(TR_xdecrypt_g, 1);
487 	return (0);
488 /* EXPORT DELETE START */
489 #endif
490 /* EXPORT DELETE END */
491 }
492 
493 
494 /*
495  * Modified version of passwd2des(). passwd2des_g() uses the Kerberos
496  * RFC 1510 algorithm to generate a DES key from a user password
497  * and mix-in string. The mix-in is expected to be the netname.
498  * This function to be used only for extended Diffie-Hellman keys.
499  *
500  * If altarg is TRUE, reverse the concat of passwd and mix-in.
501  */
502 int
503 passwd2des_g(
504 	const char *pw,
505 	const char *mixin,
506 	int len,
507 	des_block *key, /* out */
508 	bool_t altalg)
509 {
510 
511 	int  i, j, incr = 1;
512 	des_block ivec, tkey;
513 	char *text;
514 	int  plen, tlen;
515 
516 	trace1(TR_passwd2des, 0);
517 	(void) memset(tkey.c, 0, 8);
518 	(void) memset(ivec.c, 0, 8);
519 
520 
521 /*
522  * Concatentate the password and the mix-in string, fan-fold and XOR them
523  * to the required eight byte initial DES key. Since passwords can be
524  * expected to use mostly seven bit ASCII, left shift the password one
525  * bit in order to preserve as much key space as possible.
526  */
527 
528 #define	KEYLEN sizeof (tkey.c)
529 	plen = strlen(pw);
530 	tlen = ((plen + len + (KEYLEN-1))/KEYLEN)*KEYLEN;
531 	if ((text = (char *)malloc(tlen)) == NULL) {
532 		return (0);
533 	}
534 
535 	(void) memset(text, 0, tlen);
536 
537 	if (! altalg) {
538 
539 /*
540  * Concatenate the password and the mix-in string, fan-fold and XOR them
541  * to the required eight byte initial DES key. Since passwords can be
542  * expected to use mostly seven bit ASCII, left shift the password one
543  * bit in order to preserve as much key space as possible.
544  */
545 		(void) memcpy(text, pw, plen);
546 		(void) memcpy(&text[plen], mixin, len);
547 
548 		for (i = 0, j = 0; pw[j]; j++) {
549 			tkey.c[i] ^= pw[j] << 1;
550 			i += incr;
551 			if (i == 8) {
552 				i = 7;
553 				incr = -incr;
554 			} else if (i == -1) {
555 				i = 0;
556 				incr = -incr;
557 			}
558 		}
559 
560 		for (j = 0; j < len; j++) {
561 			tkey.c[i] ^= mixin[j];
562 			i += incr;
563 			if (i == 8) {
564 				i = 7;
565 				incr = -incr;
566 			} else if (i == -1) {
567 				i = 0;
568 				incr = -incr;
569 			}
570 		}
571 	} else {  /* use alternative algorithm */
572 		(void) memcpy(text, mixin, len);
573 		(void) memcpy(&text[len], pw, plen);
574 
575 		for (i = 0, j = 0; j < len; j++) {
576 			tkey.c[i] ^= mixin[j];
577 			i += incr;
578 			if (i == 8) {
579 				i = 7;
580 				incr = -incr;
581 			} else if (i == -1) {
582 				i = 0;
583 				incr = -incr;
584 			}
585 		}
586 
587 		for (j = 0; pw[j]; j++) {
588 			tkey.c[i] ^= pw[j] << 1;
589 			i += incr;
590 			if (i == 8) {
591 				i = 7;
592 				incr = -incr;
593 			} else if (i == -1) {
594 				i = 0;
595 				incr = -incr;
596 			}
597 		}
598 	}
599 	des_setparity_g(&tkey);
600 
601 	/*
602 	 * Use the temporary key to produce a DES CBC checksum for the text
603 	 * string; cbc_crypt returns the checksum in the ivec.
604 	 */
605 	(void) cbc_crypt(tkey.c, text, tlen, DES_ENCRYPT|DES_HW, ivec.c);
606 	des_setparity_g(&ivec);
607 	free(text);
608 
609 	if (weak_DES_key(ivec)) {
610 		ivec.c[7] ^= 0xf0;
611 		/*
612 		 *  XORing with 0xf0 preserves parity, so no need to check
613 		 *  that again.
614 		 */
615 	}
616 
617 	(void) memcpy((*key).c, ivec.c, sizeof (ivec.c));
618 
619 	trace1(TR_passwd2des_g, 1);
620 	return (1);
621 
622 }
623 
624 struct DESkey {
625 	uint32_t h1;
626 	uint32_t h2;
627 };
628 
629 /*
630  * Weak and semiweak keys from "Applied Cryptography", second edition,
631  * by Bruce Schneier, Wiley 1996.
632  */
633 static struct DESkey weakDESkeys[] = {
634 	/* Weak keys */
635 	{0x01010101, 0x01010101},
636 	{0x1f1f1f1f, 0x1f1f1f1f},
637 	{0xe0e0e0e0, 0xe0e0e0e0},
638 	{0xfefefefe, 0xfefefefe},
639 	/* Semiweak keys */
640 	{0x01fe01fe, 0x01fe01fe},
641 	{0x1fe01fe0, 0x0ef10ef1},
642 	{0x01e001e0, 0x01f101f1},
643 	{0x1ffe1ffe, 0x0efe0efe},
644 	{0x011f011f, 0x010e010e},
645 	{0xe0fee0fe, 0xf1fef1fe},
646 	{0xfe01fe01, 0xfe01fe01},
647 	{0xe01fe01f, 0xf10ef10e},
648 	{0xe001e001, 0xf101f101},
649 	{0xfe1ffe1f, 0xfe0efe0e},
650 	{0x1f011f01, 0x0e010e01},
651 	{0xfee0fee0, 0xfef1fef1}
652 };
653 
654 static int
655 weak_DES_key(des_block db)
656 {
657 	int i;
658 
659 	trace1(TR_weak_DES_key, 0);
660 	for (i = 0; i < sizeof (weakDESkeys)/sizeof (struct DESkey); i++) {
661 		if (weakDESkeys[i].h1 == db.key.high &&
662 			weakDESkeys[i].h2 == db.key.low) {
663 			trace1(TR_weak_DES_key, 1);
664 			return (1);
665 		}
666 	}
667 
668 	trace1(TR_weak_DES_key, 1);
669 	return (0);
670 }
671