xref: /illumos-gate/usr/src/lib/libnsl/key/gen_dhkeys.c (revision 7c478bd9)
1*7c478bd9Sstevel@tonic-gate /*
2*7c478bd9Sstevel@tonic-gate  * CDDL HEADER START
3*7c478bd9Sstevel@tonic-gate  *
4*7c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5*7c478bd9Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
6*7c478bd9Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
7*7c478bd9Sstevel@tonic-gate  * with the License.
8*7c478bd9Sstevel@tonic-gate  *
9*7c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10*7c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
11*7c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
12*7c478bd9Sstevel@tonic-gate  * and limitations under the License.
13*7c478bd9Sstevel@tonic-gate  *
14*7c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
15*7c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16*7c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
17*7c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
18*7c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
19*7c478bd9Sstevel@tonic-gate  *
20*7c478bd9Sstevel@tonic-gate  * CDDL HEADER END
21*7c478bd9Sstevel@tonic-gate  */
22*7c478bd9Sstevel@tonic-gate /*
23*7c478bd9Sstevel@tonic-gate  * Copyright 2003 Sun Microsystems, Inc.  All rights reserved.
24*7c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
25*7c478bd9Sstevel@tonic-gate  */
26*7c478bd9Sstevel@tonic-gate 
27*7c478bd9Sstevel@tonic-gate /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
28*7c478bd9Sstevel@tonic-gate /* All Rights Reserved */
29*7c478bd9Sstevel@tonic-gate 
30*7c478bd9Sstevel@tonic-gate /*
31*7c478bd9Sstevel@tonic-gate  * Portions of this source code were derived from Berkeley 4.3 BSD
32*7c478bd9Sstevel@tonic-gate  * under license from the Regents of the University of California.
33*7c478bd9Sstevel@tonic-gate  */
34*7c478bd9Sstevel@tonic-gate 
35*7c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
36*7c478bd9Sstevel@tonic-gate 
37*7c478bd9Sstevel@tonic-gate #include "mt.h"
38*7c478bd9Sstevel@tonic-gate #include <stdlib.h>
39*7c478bd9Sstevel@tonic-gate #include <mp.h>
40*7c478bd9Sstevel@tonic-gate #include <rpc/key_prot.h>
41*7c478bd9Sstevel@tonic-gate #include <rpcsvc/nis_dhext.h>
42*7c478bd9Sstevel@tonic-gate #include <thread.h>
43*7c478bd9Sstevel@tonic-gate 
44*7c478bd9Sstevel@tonic-gate extern long	random();
45*7c478bd9Sstevel@tonic-gate extern void	_mp_move(MINT *, MINT *);
46*7c478bd9Sstevel@tonic-gate extern void	des_setparity(char *);
47*7c478bd9Sstevel@tonic-gate static void	adjust();
48*7c478bd9Sstevel@tonic-gate void		__gen_dhkeys();
49*7c478bd9Sstevel@tonic-gate 
50*7c478bd9Sstevel@tonic-gate static MINT	*MODULUS_192_0;
51*7c478bd9Sstevel@tonic-gate static mutex_t	mod_192_0_lck = DEFAULTMUTEX;
52*7c478bd9Sstevel@tonic-gate static bool_t	first_time = TRUE;
53*7c478bd9Sstevel@tonic-gate 
54*7c478bd9Sstevel@tonic-gate /*
55*7c478bd9Sstevel@tonic-gate  * symbol names for the entry points into the Diffie-Hellman
56*7c478bd9Sstevel@tonic-gate  * GSS mech backend routines
57*7c478bd9Sstevel@tonic-gate  */
58*7c478bd9Sstevel@tonic-gate static char	dl_gen_funcname[] = "__dl_gen_dhkeys";
59*7c478bd9Sstevel@tonic-gate static char	dl_gen_common_funcname[] = "__dl_gen_common_dhkeys";
60*7c478bd9Sstevel@tonic-gate 
61*7c478bd9Sstevel@tonic-gate /*
62*7c478bd9Sstevel@tonic-gate  * Generate a seed
63*7c478bd9Sstevel@tonic-gate  */
64*7c478bd9Sstevel@tonic-gate static void
65*7c478bd9Sstevel@tonic-gate getseed(seed, seedsize, pass)
66*7c478bd9Sstevel@tonic-gate char *seed;
67*7c478bd9Sstevel@tonic-gate int seedsize;
68*7c478bd9Sstevel@tonic-gate unsigned char *pass;
69*7c478bd9Sstevel@tonic-gate {
70*7c478bd9Sstevel@tonic-gate 	int i;
71*7c478bd9Sstevel@tonic-gate 	int rseed;
72*7c478bd9Sstevel@tonic-gate 	struct timeval tv;
73*7c478bd9Sstevel@tonic-gate 
74*7c478bd9Sstevel@tonic-gate 	(void) gettimeofday(&tv, (struct timezone *)NULL);
75*7c478bd9Sstevel@tonic-gate 	rseed = tv.tv_sec + tv.tv_usec;
76*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < 8; i++) {
77*7c478bd9Sstevel@tonic-gate 		rseed ^= (rseed << 8) | pass[i];
78*7c478bd9Sstevel@tonic-gate 	}
79*7c478bd9Sstevel@tonic-gate 	(void) srandom(rseed);
80*7c478bd9Sstevel@tonic-gate 
81*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < seedsize; i++) {
82*7c478bd9Sstevel@tonic-gate 		seed[i] = (random() & 0xff) ^ pass[i % 8];
83*7c478bd9Sstevel@tonic-gate 	}
84*7c478bd9Sstevel@tonic-gate }
85*7c478bd9Sstevel@tonic-gate 
86*7c478bd9Sstevel@tonic-gate /*
87*7c478bd9Sstevel@tonic-gate  * Adjust the input key so that it is 0-filled on the left
88*7c478bd9Sstevel@tonic-gate  */
89*7c478bd9Sstevel@tonic-gate static void
90*7c478bd9Sstevel@tonic-gate adjust(keyout, keyin)
91*7c478bd9Sstevel@tonic-gate char keyout[HEXKEYBYTES + 1];
92*7c478bd9Sstevel@tonic-gate char *keyin;
93*7c478bd9Sstevel@tonic-gate {
94*7c478bd9Sstevel@tonic-gate 	char *p;
95*7c478bd9Sstevel@tonic-gate 	char *s;
96*7c478bd9Sstevel@tonic-gate 
97*7c478bd9Sstevel@tonic-gate 	for (p = keyin; *p; p++)
98*7c478bd9Sstevel@tonic-gate 		;
99*7c478bd9Sstevel@tonic-gate 	for (s = keyout + HEXKEYBYTES; p >= keyin; p--, s--) {
100*7c478bd9Sstevel@tonic-gate 		*s = *p;
101*7c478bd9Sstevel@tonic-gate 	}
102*7c478bd9Sstevel@tonic-gate 	while (s >= keyout) {
103*7c478bd9Sstevel@tonic-gate 		*s-- = '0';
104*7c478bd9Sstevel@tonic-gate 	}
105*7c478bd9Sstevel@tonic-gate }
106*7c478bd9Sstevel@tonic-gate 
107*7c478bd9Sstevel@tonic-gate /*
108*7c478bd9Sstevel@tonic-gate  * generate a Diffie-Hellman key-pair based on the given password.
109*7c478bd9Sstevel@tonic-gate  * public and secret are buffers of size HEXKEYBYTES + 1.
110*7c478bd9Sstevel@tonic-gate  */
111*7c478bd9Sstevel@tonic-gate void
112*7c478bd9Sstevel@tonic-gate __gen_dhkeys(public, secret, pass)
113*7c478bd9Sstevel@tonic-gate char *public;
114*7c478bd9Sstevel@tonic-gate char *secret;
115*7c478bd9Sstevel@tonic-gate char *pass;
116*7c478bd9Sstevel@tonic-gate {
117*7c478bd9Sstevel@tonic-gate 	int i;
118*7c478bd9Sstevel@tonic-gate 
119*7c478bd9Sstevel@tonic-gate #define	BASEBITS	(8 * sizeof (short) - 1)
120*7c478bd9Sstevel@tonic-gate #define	BASE		(1 << BASEBITS)
121*7c478bd9Sstevel@tonic-gate 
122*7c478bd9Sstevel@tonic-gate 	MINT *pk = mp_itom(0);
123*7c478bd9Sstevel@tonic-gate 	MINT *sk = mp_itom(0);
124*7c478bd9Sstevel@tonic-gate 	MINT *tmp;
125*7c478bd9Sstevel@tonic-gate 	MINT *base = mp_itom(BASE/2);	/* BASE won't fit in a short */
126*7c478bd9Sstevel@tonic-gate 	MINT *root = mp_itom(PROOT);
127*7c478bd9Sstevel@tonic-gate 	MINT *modulus = mp_xtom(HEXMODULUS);
128*7c478bd9Sstevel@tonic-gate 	unsigned short r;
129*7c478bd9Sstevel@tonic-gate 	unsigned short seed[KEYSIZE/BASEBITS + 1];
130*7c478bd9Sstevel@tonic-gate 	char *xkey;
131*7c478bd9Sstevel@tonic-gate 
132*7c478bd9Sstevel@tonic-gate 	/* multiply base by 2 to get BASE */
133*7c478bd9Sstevel@tonic-gate 	tmp = mp_itom(2);
134*7c478bd9Sstevel@tonic-gate 	mp_mult(base, tmp, base);
135*7c478bd9Sstevel@tonic-gate 	mp_mfree(tmp);
136*7c478bd9Sstevel@tonic-gate 
137*7c478bd9Sstevel@tonic-gate 	getseed((char *)seed, (int)sizeof (seed), (uchar_t *)pass);
138*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < KEYSIZE/BASEBITS + 1; i++) {
139*7c478bd9Sstevel@tonic-gate 		r = seed[i] % ((unsigned short)BASE);
140*7c478bd9Sstevel@tonic-gate 		tmp = mp_itom(r);
141*7c478bd9Sstevel@tonic-gate 		mp_mult(sk, base, sk);
142*7c478bd9Sstevel@tonic-gate 		mp_madd(sk, tmp, sk);
143*7c478bd9Sstevel@tonic-gate 		mp_mfree(tmp);
144*7c478bd9Sstevel@tonic-gate 	}
145*7c478bd9Sstevel@tonic-gate 	tmp = mp_itom(0);
146*7c478bd9Sstevel@tonic-gate 	mp_mdiv(sk, modulus, tmp, sk);
147*7c478bd9Sstevel@tonic-gate 	mp_mfree(tmp);
148*7c478bd9Sstevel@tonic-gate 	mp_pow(root, sk, modulus, pk);
149*7c478bd9Sstevel@tonic-gate 	xkey = mp_mtox(sk);
150*7c478bd9Sstevel@tonic-gate 	(void) adjust(secret, xkey);
151*7c478bd9Sstevel@tonic-gate 	xkey = mp_mtox(pk);
152*7c478bd9Sstevel@tonic-gate 	(void) adjust(public, xkey);
153*7c478bd9Sstevel@tonic-gate 	mp_mfree(sk);
154*7c478bd9Sstevel@tonic-gate 	mp_mfree(base);
155*7c478bd9Sstevel@tonic-gate 	mp_mfree(pk);
156*7c478bd9Sstevel@tonic-gate 	mp_mfree(root);
157*7c478bd9Sstevel@tonic-gate 	mp_mfree(modulus);
158*7c478bd9Sstevel@tonic-gate }
159*7c478bd9Sstevel@tonic-gate 
160*7c478bd9Sstevel@tonic-gate 
161*7c478bd9Sstevel@tonic-gate /*
162*7c478bd9Sstevel@tonic-gate  * Generic key size Diffie-Hellman key pair generation routine.  For classic
163*7c478bd9Sstevel@tonic-gate  * AUTH_DES, just call the current routine to handle it.  Else, call the
164*7c478bd9Sstevel@tonic-gate  * one in the appro GSS mech backend.
165*7c478bd9Sstevel@tonic-gate  *
166*7c478bd9Sstevel@tonic-gate  */
167*7c478bd9Sstevel@tonic-gate int
168*7c478bd9Sstevel@tonic-gate __gen_dhkeys_g(char *pkey,	/* out */
169*7c478bd9Sstevel@tonic-gate 	char *skey,		/* out */
170*7c478bd9Sstevel@tonic-gate 	keylen_t keylen,	/* in  */
171*7c478bd9Sstevel@tonic-gate 	algtype_t algtype,	/* in  */
172*7c478bd9Sstevel@tonic-gate 	char *pass)		/* in  */
173*7c478bd9Sstevel@tonic-gate {
174*7c478bd9Sstevel@tonic-gate 	const int classic_des = keylen == 192 && algtype == 0;
175*7c478bd9Sstevel@tonic-gate 
176*7c478bd9Sstevel@tonic-gate 	if (! pkey || ! skey || ! pass)
177*7c478bd9Sstevel@tonic-gate 		return (0);
178*7c478bd9Sstevel@tonic-gate 
179*7c478bd9Sstevel@tonic-gate 	if (classic_des) {
180*7c478bd9Sstevel@tonic-gate 		__gen_dhkeys(pkey, skey, pass);
181*7c478bd9Sstevel@tonic-gate 		return (1);
182*7c478bd9Sstevel@tonic-gate 	} else {
183*7c478bd9Sstevel@tonic-gate 		int (*dlfp)(); /* func ptr to dynamic loaded lib */
184*7c478bd9Sstevel@tonic-gate 
185*7c478bd9Sstevel@tonic-gate 		if (dlfp = (int (*)())__nis_get_mechanism_symbol(keylen,
186*7c478bd9Sstevel@tonic-gate 							algtype,
187*7c478bd9Sstevel@tonic-gate 							dl_gen_funcname)) {
188*7c478bd9Sstevel@tonic-gate 			(*dlfp)(pkey, skey, pass); /* void */
189*7c478bd9Sstevel@tonic-gate 			return (1);
190*7c478bd9Sstevel@tonic-gate 		}
191*7c478bd9Sstevel@tonic-gate 	}
192*7c478bd9Sstevel@tonic-gate 
193*7c478bd9Sstevel@tonic-gate 	return (0);
194*7c478bd9Sstevel@tonic-gate }
195*7c478bd9Sstevel@tonic-gate 
196*7c478bd9Sstevel@tonic-gate 
197*7c478bd9Sstevel@tonic-gate /*
198*7c478bd9Sstevel@tonic-gate  * Choose middle 64 bits of the common key to use as our des key, possibly
199*7c478bd9Sstevel@tonic-gate  * overwriting the lower order bits by setting parity.
200*7c478bd9Sstevel@tonic-gate  *
201*7c478bd9Sstevel@tonic-gate  * (copied/moved) from keyserv's setkey.c for the DH extensions.
202*7c478bd9Sstevel@tonic-gate  */
203*7c478bd9Sstevel@tonic-gate int
204*7c478bd9Sstevel@tonic-gate __extractdeskey(ck, deskey)
205*7c478bd9Sstevel@tonic-gate 	MINT *ck;
206*7c478bd9Sstevel@tonic-gate 	des_block *deskey;
207*7c478bd9Sstevel@tonic-gate {
208*7c478bd9Sstevel@tonic-gate 	MINT *a;
209*7c478bd9Sstevel@tonic-gate 	short r;
210*7c478bd9Sstevel@tonic-gate 	int i;
211*7c478bd9Sstevel@tonic-gate 	short base = (1 << 8);
212*7c478bd9Sstevel@tonic-gate 	char *k;
213*7c478bd9Sstevel@tonic-gate 
214*7c478bd9Sstevel@tonic-gate 	a = mp_itom(0);
215*7c478bd9Sstevel@tonic-gate 	_mp_move(ck, a);
216*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < ((KEYSIZE - 64) / 2) / 8; i++) {
217*7c478bd9Sstevel@tonic-gate 		mp_sdiv(a, base, a, &r);
218*7c478bd9Sstevel@tonic-gate 	}
219*7c478bd9Sstevel@tonic-gate 	k = deskey->c;
220*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < 8; i++) {
221*7c478bd9Sstevel@tonic-gate 		mp_sdiv(a, base, a, &r);
222*7c478bd9Sstevel@tonic-gate 		*k++ = r;
223*7c478bd9Sstevel@tonic-gate 	}
224*7c478bd9Sstevel@tonic-gate 	mp_mfree(a);
225*7c478bd9Sstevel@tonic-gate 	des_setparity((char *)deskey);
226*7c478bd9Sstevel@tonic-gate 	return (0);
227*7c478bd9Sstevel@tonic-gate }
228*7c478bd9Sstevel@tonic-gate 
229*7c478bd9Sstevel@tonic-gate 
230*7c478bd9Sstevel@tonic-gate /*
231*7c478bd9Sstevel@tonic-gate  * Set the modulus for all our 192bit (algtype=0) Diffie-Hellman operations
232*7c478bd9Sstevel@tonic-gate  */
233*7c478bd9Sstevel@tonic-gate static void
234*7c478bd9Sstevel@tonic-gate setmodulus_192_0(void)
235*7c478bd9Sstevel@tonic-gate {
236*7c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&mod_192_0_lck);
237*7c478bd9Sstevel@tonic-gate 	if (first_time) {
238*7c478bd9Sstevel@tonic-gate 		first_time = FALSE;
239*7c478bd9Sstevel@tonic-gate 		MODULUS_192_0 = mp_xtom(HEXMODULUS);
240*7c478bd9Sstevel@tonic-gate 	}
241*7c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&mod_192_0_lck);
242*7c478bd9Sstevel@tonic-gate }
243*7c478bd9Sstevel@tonic-gate 
244*7c478bd9Sstevel@tonic-gate /*
245*7c478bd9Sstevel@tonic-gate  * Generic key size Diffie-Hellman common key generation routine.
246*7c478bd9Sstevel@tonic-gate  * For classic AUTH_DES, do it inline like it's already done in several
247*7c478bd9Sstevel@tonic-gate  * places (keyserv being one place).  For new long key sizes,
248*7c478bd9Sstevel@tonic-gate  * call the appro GSS mech backend routine.
249*7c478bd9Sstevel@tonic-gate  *
250*7c478bd9Sstevel@tonic-gate  * Arg 'keynum' is the size of the 'deskeys' array.  It should be a 1
251*7c478bd9Sstevel@tonic-gate  * classic AUTH_DES and a 3 for new long DH keys.
252*7c478bd9Sstevel@tonic-gate  *
253*7c478bd9Sstevel@tonic-gate  * Returns 1 on success and 0 on err.
254*7c478bd9Sstevel@tonic-gate  */
255*7c478bd9Sstevel@tonic-gate int
256*7c478bd9Sstevel@tonic-gate __gen_common_dhkeys_g(char *xpublic,	/* in  */
257*7c478bd9Sstevel@tonic-gate 		char *xsecret,		/* in  */
258*7c478bd9Sstevel@tonic-gate 		keylen_t keylen,	/* in  */
259*7c478bd9Sstevel@tonic-gate 		algtype_t algtype,	/* in  */
260*7c478bd9Sstevel@tonic-gate 		des_block deskeys[],	/* out */
261*7c478bd9Sstevel@tonic-gate 		keynum_t keynum)	/* in  */
262*7c478bd9Sstevel@tonic-gate {
263*7c478bd9Sstevel@tonic-gate 	const int classic_des = keylen == 192 && algtype == 0;
264*7c478bd9Sstevel@tonic-gate 
265*7c478bd9Sstevel@tonic-gate 	if (! xpublic || ! xsecret || ! deskeys)
266*7c478bd9Sstevel@tonic-gate 		return (0);
267*7c478bd9Sstevel@tonic-gate 
268*7c478bd9Sstevel@tonic-gate 	if (classic_des) {
269*7c478bd9Sstevel@tonic-gate 		MINT *common;
270*7c478bd9Sstevel@tonic-gate 		MINT *public;
271*7c478bd9Sstevel@tonic-gate 		MINT *secret;
272*7c478bd9Sstevel@tonic-gate 
273*7c478bd9Sstevel@tonic-gate 		setmodulus_192_0();
274*7c478bd9Sstevel@tonic-gate 
275*7c478bd9Sstevel@tonic-gate 		public = mp_xtom(xpublic);
276*7c478bd9Sstevel@tonic-gate 		secret = mp_xtom(xsecret);
277*7c478bd9Sstevel@tonic-gate 		common = mp_itom(0);
278*7c478bd9Sstevel@tonic-gate 		mp_pow(public, secret, MODULUS_192_0, common);
279*7c478bd9Sstevel@tonic-gate 		(void) __extractdeskey(common, &deskeys[0]);
280*7c478bd9Sstevel@tonic-gate 		return (1);
281*7c478bd9Sstevel@tonic-gate 	} else {
282*7c478bd9Sstevel@tonic-gate 		int (*dlfp)(); /* func ptr to dynamically loaded lib */
283*7c478bd9Sstevel@tonic-gate 
284*7c478bd9Sstevel@tonic-gate 		if (dlfp = (int (*)())__nis_get_mechanism_symbol(keylen,
285*7c478bd9Sstevel@tonic-gate 						algtype,
286*7c478bd9Sstevel@tonic-gate 						dl_gen_common_funcname)) {
287*7c478bd9Sstevel@tonic-gate 			/* function called will have void return value */
288*7c478bd9Sstevel@tonic-gate 			(*dlfp)(xpublic, xsecret, deskeys, keynum);
289*7c478bd9Sstevel@tonic-gate 			return (1);
290*7c478bd9Sstevel@tonic-gate 		}
291*7c478bd9Sstevel@tonic-gate 	}
292*7c478bd9Sstevel@tonic-gate 
293*7c478bd9Sstevel@tonic-gate 	return (0);
294*7c478bd9Sstevel@tonic-gate }
295