xref: /illumos-gate/usr/src/lib/libnsl/key/gen_dhkeys.c (revision 1da57d55)
1*7c478bd9Sstevel@tonic-gate /*
2*7c478bd9Sstevel@tonic-gate  * CDDL HEADER START
3*7c478bd9Sstevel@tonic-gate  *
4*7c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5*7c478bd9Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
6*7c478bd9Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
7*7c478bd9Sstevel@tonic-gate  * with the License.
8*7c478bd9Sstevel@tonic-gate  *
9*7c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10*7c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
11*7c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
12*7c478bd9Sstevel@tonic-gate  * and limitations under the License.
13*7c478bd9Sstevel@tonic-gate  *
14*7c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
15*7c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16*7c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
17*7c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
18*7c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
19*7c478bd9Sstevel@tonic-gate  *
20*7c478bd9Sstevel@tonic-gate  * CDDL HEADER END
21*7c478bd9Sstevel@tonic-gate  */
22*7c478bd9Sstevel@tonic-gate /*
23*7c478bd9Sstevel@tonic-gate  * Copyright 2003 Sun Microsystems, Inc.  All rights reserved.
24*7c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
25*7c478bd9Sstevel@tonic-gate  */
26*7c478bd9Sstevel@tonic-gate 
27*7c478bd9Sstevel@tonic-gate /* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
28*7c478bd9Sstevel@tonic-gate /* All Rights Reserved */
29*7c478bd9Sstevel@tonic-gate 
30*7c478bd9Sstevel@tonic-gate /*
31*7c478bd9Sstevel@tonic-gate  * Portions of this source code were derived from Berkeley 4.3 BSD
32*7c478bd9Sstevel@tonic-gate  * under license from the Regents of the University of California.
33*7c478bd9Sstevel@tonic-gate  */
34*7c478bd9Sstevel@tonic-gate 
35*7c478bd9Sstevel@tonic-gate #include "mt.h"
36*7c478bd9Sstevel@tonic-gate #include <stdlib.h>
37*7c478bd9Sstevel@tonic-gate #include <mp.h>
38*7c478bd9Sstevel@tonic-gate #include <rpc/key_prot.h>
39*7c478bd9Sstevel@tonic-gate #include <rpcsvc/nis_dhext.h>
40*7c478bd9Sstevel@tonic-gate #include <thread.h>
41*7c478bd9Sstevel@tonic-gate 
42*7c478bd9Sstevel@tonic-gate extern long	random();
43*7c478bd9Sstevel@tonic-gate extern void	_mp_move(MINT *, MINT *);
44*7c478bd9Sstevel@tonic-gate extern void	des_setparity(char *);
45*7c478bd9Sstevel@tonic-gate static void	adjust();
46*7c478bd9Sstevel@tonic-gate void		__gen_dhkeys();
47*7c478bd9Sstevel@tonic-gate 
48*7c478bd9Sstevel@tonic-gate static MINT	*MODULUS_192_0;
49*7c478bd9Sstevel@tonic-gate static mutex_t	mod_192_0_lck = DEFAULTMUTEX;
50*7c478bd9Sstevel@tonic-gate static bool_t	first_time = TRUE;
51*7c478bd9Sstevel@tonic-gate 
52*7c478bd9Sstevel@tonic-gate /*
53*7c478bd9Sstevel@tonic-gate  * symbol names for the entry points into the Diffie-Hellman
54*7c478bd9Sstevel@tonic-gate  * GSS mech backend routines
55*7c478bd9Sstevel@tonic-gate  */
56*7c478bd9Sstevel@tonic-gate static char	dl_gen_funcname[] = "__dl_gen_dhkeys";
57*7c478bd9Sstevel@tonic-gate static char	dl_gen_common_funcname[] = "__dl_gen_common_dhkeys";
58*7c478bd9Sstevel@tonic-gate 
59*7c478bd9Sstevel@tonic-gate /*
60*7c478bd9Sstevel@tonic-gate  * Generate a seed
61*7c478bd9Sstevel@tonic-gate  */
62*7c478bd9Sstevel@tonic-gate static void
getseed(seed,seedsize,pass)63*7c478bd9Sstevel@tonic-gate getseed(seed, seedsize, pass)
64*7c478bd9Sstevel@tonic-gate char *seed;
65*7c478bd9Sstevel@tonic-gate int seedsize;
66*7c478bd9Sstevel@tonic-gate unsigned char *pass;
67*7c478bd9Sstevel@tonic-gate {
68*7c478bd9Sstevel@tonic-gate 	int i;
69*7c478bd9Sstevel@tonic-gate 	int rseed;
70*7c478bd9Sstevel@tonic-gate 	struct timeval tv;
71*7c478bd9Sstevel@tonic-gate 
72*7c478bd9Sstevel@tonic-gate 	(void) gettimeofday(&tv, (struct timezone *)NULL);
73*7c478bd9Sstevel@tonic-gate 	rseed = tv.tv_sec + tv.tv_usec;
74*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < 8; i++) {
75*7c478bd9Sstevel@tonic-gate 		rseed ^= (rseed << 8) | pass[i];
76*7c478bd9Sstevel@tonic-gate 	}
77*7c478bd9Sstevel@tonic-gate 	(void) srandom(rseed);
78*7c478bd9Sstevel@tonic-gate 
79*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < seedsize; i++) {
80*7c478bd9Sstevel@tonic-gate 		seed[i] = (random() & 0xff) ^ pass[i % 8];
81*7c478bd9Sstevel@tonic-gate 	}
82*7c478bd9Sstevel@tonic-gate }
83*7c478bd9Sstevel@tonic-gate 
84*7c478bd9Sstevel@tonic-gate /*
85*7c478bd9Sstevel@tonic-gate  * Adjust the input key so that it is 0-filled on the left
86*7c478bd9Sstevel@tonic-gate  */
87*7c478bd9Sstevel@tonic-gate static void
adjust(keyout,keyin)88*7c478bd9Sstevel@tonic-gate adjust(keyout, keyin)
89*7c478bd9Sstevel@tonic-gate char keyout[HEXKEYBYTES + 1];
90*7c478bd9Sstevel@tonic-gate char *keyin;
91*7c478bd9Sstevel@tonic-gate {
92*7c478bd9Sstevel@tonic-gate 	char *p;
93*7c478bd9Sstevel@tonic-gate 	char *s;
94*7c478bd9Sstevel@tonic-gate 
95*7c478bd9Sstevel@tonic-gate 	for (p = keyin; *p; p++)
96*7c478bd9Sstevel@tonic-gate 		;
97*7c478bd9Sstevel@tonic-gate 	for (s = keyout + HEXKEYBYTES; p >= keyin; p--, s--) {
98*7c478bd9Sstevel@tonic-gate 		*s = *p;
99*7c478bd9Sstevel@tonic-gate 	}
100*7c478bd9Sstevel@tonic-gate 	while (s >= keyout) {
101*7c478bd9Sstevel@tonic-gate 		*s-- = '0';
102*7c478bd9Sstevel@tonic-gate 	}
103*7c478bd9Sstevel@tonic-gate }
104*7c478bd9Sstevel@tonic-gate 
105*7c478bd9Sstevel@tonic-gate /*
106*7c478bd9Sstevel@tonic-gate  * generate a Diffie-Hellman key-pair based on the given password.
107*7c478bd9Sstevel@tonic-gate  * public and secret are buffers of size HEXKEYBYTES + 1.
108*7c478bd9Sstevel@tonic-gate  */
109*7c478bd9Sstevel@tonic-gate void
__gen_dhkeys(public,secret,pass)110*7c478bd9Sstevel@tonic-gate __gen_dhkeys(public, secret, pass)
111*7c478bd9Sstevel@tonic-gate char *public;
112*7c478bd9Sstevel@tonic-gate char *secret;
113*7c478bd9Sstevel@tonic-gate char *pass;
114*7c478bd9Sstevel@tonic-gate {
115*7c478bd9Sstevel@tonic-gate 	int i;
116*7c478bd9Sstevel@tonic-gate 
117*7c478bd9Sstevel@tonic-gate #define	BASEBITS	(8 * sizeof (short) - 1)
118*7c478bd9Sstevel@tonic-gate #define	BASE		(1 << BASEBITS)
119*7c478bd9Sstevel@tonic-gate 
120*7c478bd9Sstevel@tonic-gate 	MINT *pk = mp_itom(0);
121*7c478bd9Sstevel@tonic-gate 	MINT *sk = mp_itom(0);
122*7c478bd9Sstevel@tonic-gate 	MINT *tmp;
123*7c478bd9Sstevel@tonic-gate 	MINT *base = mp_itom(BASE/2);	/* BASE won't fit in a short */
124*7c478bd9Sstevel@tonic-gate 	MINT *root = mp_itom(PROOT);
125*7c478bd9Sstevel@tonic-gate 	MINT *modulus = mp_xtom(HEXMODULUS);
126*7c478bd9Sstevel@tonic-gate 	unsigned short r;
127*7c478bd9Sstevel@tonic-gate 	unsigned short seed[KEYSIZE/BASEBITS + 1];
128*7c478bd9Sstevel@tonic-gate 	char *xkey;
129*7c478bd9Sstevel@tonic-gate 
130*7c478bd9Sstevel@tonic-gate 	/* multiply base by 2 to get BASE */
131*7c478bd9Sstevel@tonic-gate 	tmp = mp_itom(2);
132*7c478bd9Sstevel@tonic-gate 	mp_mult(base, tmp, base);
133*7c478bd9Sstevel@tonic-gate 	mp_mfree(tmp);
134*7c478bd9Sstevel@tonic-gate 
135*7c478bd9Sstevel@tonic-gate 	getseed((char *)seed, (int)sizeof (seed), (uchar_t *)pass);
136*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < KEYSIZE/BASEBITS + 1; i++) {
137*7c478bd9Sstevel@tonic-gate 		r = seed[i] % ((unsigned short)BASE);
138*7c478bd9Sstevel@tonic-gate 		tmp = mp_itom(r);
139*7c478bd9Sstevel@tonic-gate 		mp_mult(sk, base, sk);
140*7c478bd9Sstevel@tonic-gate 		mp_madd(sk, tmp, sk);
141*7c478bd9Sstevel@tonic-gate 		mp_mfree(tmp);
142*7c478bd9Sstevel@tonic-gate 	}
143*7c478bd9Sstevel@tonic-gate 	tmp = mp_itom(0);
144*7c478bd9Sstevel@tonic-gate 	mp_mdiv(sk, modulus, tmp, sk);
145*7c478bd9Sstevel@tonic-gate 	mp_mfree(tmp);
146*7c478bd9Sstevel@tonic-gate 	mp_pow(root, sk, modulus, pk);
147*7c478bd9Sstevel@tonic-gate 	xkey = mp_mtox(sk);
148*7c478bd9Sstevel@tonic-gate 	(void) adjust(secret, xkey);
149*7c478bd9Sstevel@tonic-gate 	xkey = mp_mtox(pk);
150*7c478bd9Sstevel@tonic-gate 	(void) adjust(public, xkey);
151*7c478bd9Sstevel@tonic-gate 	mp_mfree(sk);
152*7c478bd9Sstevel@tonic-gate 	mp_mfree(base);
153*7c478bd9Sstevel@tonic-gate 	mp_mfree(pk);
154*7c478bd9Sstevel@tonic-gate 	mp_mfree(root);
155*7c478bd9Sstevel@tonic-gate 	mp_mfree(modulus);
156*7c478bd9Sstevel@tonic-gate }
157*7c478bd9Sstevel@tonic-gate 
158*7c478bd9Sstevel@tonic-gate 
159*7c478bd9Sstevel@tonic-gate /*
160*7c478bd9Sstevel@tonic-gate  * Generic key size Diffie-Hellman key pair generation routine.  For classic
161*7c478bd9Sstevel@tonic-gate  * AUTH_DES, just call the current routine to handle it.  Else, call the
162*7c478bd9Sstevel@tonic-gate  * one in the appro GSS mech backend.
163*7c478bd9Sstevel@tonic-gate  *
164*7c478bd9Sstevel@tonic-gate  */
165*7c478bd9Sstevel@tonic-gate int
__gen_dhkeys_g(char * pkey,char * skey,keylen_t keylen,algtype_t algtype,char * pass)166*7c478bd9Sstevel@tonic-gate __gen_dhkeys_g(char *pkey,	/* out */
167*7c478bd9Sstevel@tonic-gate 	char *skey,		/* out */
168*7c478bd9Sstevel@tonic-gate 	keylen_t keylen,	/* in  */
169*7c478bd9Sstevel@tonic-gate 	algtype_t algtype,	/* in  */
170*7c478bd9Sstevel@tonic-gate 	char *pass)		/* in  */
171*7c478bd9Sstevel@tonic-gate {
172*7c478bd9Sstevel@tonic-gate 	const int classic_des = keylen == 192 && algtype == 0;
173*7c478bd9Sstevel@tonic-gate 
174*7c478bd9Sstevel@tonic-gate 	if (! pkey || ! skey || ! pass)
175*7c478bd9Sstevel@tonic-gate 		return (0);
176*7c478bd9Sstevel@tonic-gate 
177*7c478bd9Sstevel@tonic-gate 	if (classic_des) {
178*7c478bd9Sstevel@tonic-gate 		__gen_dhkeys(pkey, skey, pass);
179*7c478bd9Sstevel@tonic-gate 		return (1);
180*7c478bd9Sstevel@tonic-gate 	} else {
181*7c478bd9Sstevel@tonic-gate 		int (*dlfp)(); /* func ptr to dynamic loaded lib */
182*7c478bd9Sstevel@tonic-gate 
183*7c478bd9Sstevel@tonic-gate 		if (dlfp = (int (*)())__nis_get_mechanism_symbol(keylen,
184*7c478bd9Sstevel@tonic-gate 							algtype,
185*7c478bd9Sstevel@tonic-gate 							dl_gen_funcname)) {
186*7c478bd9Sstevel@tonic-gate 			(*dlfp)(pkey, skey, pass); /* void */
187*7c478bd9Sstevel@tonic-gate 			return (1);
188*7c478bd9Sstevel@tonic-gate 		}
189*7c478bd9Sstevel@tonic-gate 	}
190*7c478bd9Sstevel@tonic-gate 
191*7c478bd9Sstevel@tonic-gate 	return (0);
192*7c478bd9Sstevel@tonic-gate }
193*7c478bd9Sstevel@tonic-gate 
194*7c478bd9Sstevel@tonic-gate 
195*7c478bd9Sstevel@tonic-gate /*
196*7c478bd9Sstevel@tonic-gate  * Choose middle 64 bits of the common key to use as our des key, possibly
197*7c478bd9Sstevel@tonic-gate  * overwriting the lower order bits by setting parity.
198*7c478bd9Sstevel@tonic-gate  *
199*7c478bd9Sstevel@tonic-gate  * (copied/moved) from keyserv's setkey.c for the DH extensions.
200*7c478bd9Sstevel@tonic-gate  */
201*7c478bd9Sstevel@tonic-gate int
__extractdeskey(ck,deskey)202*7c478bd9Sstevel@tonic-gate __extractdeskey(ck, deskey)
203*7c478bd9Sstevel@tonic-gate 	MINT *ck;
204*7c478bd9Sstevel@tonic-gate 	des_block *deskey;
205*7c478bd9Sstevel@tonic-gate {
206*7c478bd9Sstevel@tonic-gate 	MINT *a;
207*7c478bd9Sstevel@tonic-gate 	short r;
208*7c478bd9Sstevel@tonic-gate 	int i;
209*7c478bd9Sstevel@tonic-gate 	short base = (1 << 8);
210*7c478bd9Sstevel@tonic-gate 	char *k;
211*7c478bd9Sstevel@tonic-gate 
212*7c478bd9Sstevel@tonic-gate 	a = mp_itom(0);
213*7c478bd9Sstevel@tonic-gate 	_mp_move(ck, a);
214*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < ((KEYSIZE - 64) / 2) / 8; i++) {
215*7c478bd9Sstevel@tonic-gate 		mp_sdiv(a, base, a, &r);
216*7c478bd9Sstevel@tonic-gate 	}
217*7c478bd9Sstevel@tonic-gate 	k = deskey->c;
218*7c478bd9Sstevel@tonic-gate 	for (i = 0; i < 8; i++) {
219*7c478bd9Sstevel@tonic-gate 		mp_sdiv(a, base, a, &r);
220*7c478bd9Sstevel@tonic-gate 		*k++ = r;
221*7c478bd9Sstevel@tonic-gate 	}
222*7c478bd9Sstevel@tonic-gate 	mp_mfree(a);
223*7c478bd9Sstevel@tonic-gate 	des_setparity((char *)deskey);
224*7c478bd9Sstevel@tonic-gate 	return (0);
225*7c478bd9Sstevel@tonic-gate }
226*7c478bd9Sstevel@tonic-gate 
227*7c478bd9Sstevel@tonic-gate 
228*7c478bd9Sstevel@tonic-gate /*
229*7c478bd9Sstevel@tonic-gate  * Set the modulus for all our 192bit (algtype=0) Diffie-Hellman operations
230*7c478bd9Sstevel@tonic-gate  */
231*7c478bd9Sstevel@tonic-gate static void
setmodulus_192_0(void)232*7c478bd9Sstevel@tonic-gate setmodulus_192_0(void)
233*7c478bd9Sstevel@tonic-gate {
234*7c478bd9Sstevel@tonic-gate 	(void) mutex_lock(&mod_192_0_lck);
235*7c478bd9Sstevel@tonic-gate 	if (first_time) {
236*7c478bd9Sstevel@tonic-gate 		first_time = FALSE;
237*7c478bd9Sstevel@tonic-gate 		MODULUS_192_0 = mp_xtom(HEXMODULUS);
238*7c478bd9Sstevel@tonic-gate 	}
239*7c478bd9Sstevel@tonic-gate 	(void) mutex_unlock(&mod_192_0_lck);
240*7c478bd9Sstevel@tonic-gate }
241*7c478bd9Sstevel@tonic-gate 
242*7c478bd9Sstevel@tonic-gate /*
243*7c478bd9Sstevel@tonic-gate  * Generic key size Diffie-Hellman common key generation routine.
244*7c478bd9Sstevel@tonic-gate  * For classic AUTH_DES, do it inline like it's already done in several
245*7c478bd9Sstevel@tonic-gate  * places (keyserv being one place).  For new long key sizes,
246*7c478bd9Sstevel@tonic-gate  * call the appro GSS mech backend routine.
247*7c478bd9Sstevel@tonic-gate  *
248*7c478bd9Sstevel@tonic-gate  * Arg 'keynum' is the size of the 'deskeys' array.  It should be a 1
249*7c478bd9Sstevel@tonic-gate  * classic AUTH_DES and a 3 for new long DH keys.
250*7c478bd9Sstevel@tonic-gate  *
251*7c478bd9Sstevel@tonic-gate  * Returns 1 on success and 0 on err.
252*7c478bd9Sstevel@tonic-gate  */
253*7c478bd9Sstevel@tonic-gate int
__gen_common_dhkeys_g(char * xpublic,char * xsecret,keylen_t keylen,algtype_t algtype,des_block deskeys[],keynum_t keynum)254*7c478bd9Sstevel@tonic-gate __gen_common_dhkeys_g(char *xpublic,	/* in  */
255*7c478bd9Sstevel@tonic-gate 		char *xsecret,		/* in  */
256*7c478bd9Sstevel@tonic-gate 		keylen_t keylen,	/* in  */
257*7c478bd9Sstevel@tonic-gate 		algtype_t algtype,	/* in  */
258*7c478bd9Sstevel@tonic-gate 		des_block deskeys[],	/* out */
259*7c478bd9Sstevel@tonic-gate 		keynum_t keynum)	/* in  */
260*7c478bd9Sstevel@tonic-gate {
261*7c478bd9Sstevel@tonic-gate 	const int classic_des = keylen == 192 && algtype == 0;
262*7c478bd9Sstevel@tonic-gate 
263*7c478bd9Sstevel@tonic-gate 	if (! xpublic || ! xsecret || ! deskeys)
264*7c478bd9Sstevel@tonic-gate 		return (0);
265*7c478bd9Sstevel@tonic-gate 
266*7c478bd9Sstevel@tonic-gate 	if (classic_des) {
267*7c478bd9Sstevel@tonic-gate 		MINT *common;
268*7c478bd9Sstevel@tonic-gate 		MINT *public;
269*7c478bd9Sstevel@tonic-gate 		MINT *secret;
270*7c478bd9Sstevel@tonic-gate 
271*7c478bd9Sstevel@tonic-gate 		setmodulus_192_0();
272*7c478bd9Sstevel@tonic-gate 
273*7c478bd9Sstevel@tonic-gate 		public = mp_xtom(xpublic);
274*7c478bd9Sstevel@tonic-gate 		secret = mp_xtom(xsecret);
275*7c478bd9Sstevel@tonic-gate 		common = mp_itom(0);
276*7c478bd9Sstevel@tonic-gate 		mp_pow(public, secret, MODULUS_192_0, common);
277*7c478bd9Sstevel@tonic-gate 		(void) __extractdeskey(common, &deskeys[0]);
278*7c478bd9Sstevel@tonic-gate 		return (1);
279*7c478bd9Sstevel@tonic-gate 	} else {
280*7c478bd9Sstevel@tonic-gate 		int (*dlfp)(); /* func ptr to dynamically loaded lib */
281*7c478bd9Sstevel@tonic-gate 
282*7c478bd9Sstevel@tonic-gate 		if (dlfp = (int (*)())__nis_get_mechanism_symbol(keylen,
283*7c478bd9Sstevel@tonic-gate 						algtype,
284*7c478bd9Sstevel@tonic-gate 						dl_gen_common_funcname)) {
285*7c478bd9Sstevel@tonic-gate 			/* function called will have void return value */
286*7c478bd9Sstevel@tonic-gate 			(*dlfp)(xpublic, xsecret, deskeys, keynum);
287*7c478bd9Sstevel@tonic-gate 			return (1);
288*7c478bd9Sstevel@tonic-gate 		}
289*7c478bd9Sstevel@tonic-gate 	}
290*7c478bd9Sstevel@tonic-gate 
291*7c478bd9Sstevel@tonic-gate 	return (0);
292*7c478bd9Sstevel@tonic-gate }
293