1*7c478bd9Sstevel@tonic-gate /*
2*7c478bd9Sstevel@tonic-gate  * CDDL HEADER START
3*7c478bd9Sstevel@tonic-gate  *
4*7c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5*7c478bd9Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
6*7c478bd9Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
7*7c478bd9Sstevel@tonic-gate  * with the License.
8*7c478bd9Sstevel@tonic-gate  *
9*7c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10*7c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
11*7c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
12*7c478bd9Sstevel@tonic-gate  * and limitations under the License.
13*7c478bd9Sstevel@tonic-gate  *
14*7c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
15*7c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16*7c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
17*7c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
18*7c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
19*7c478bd9Sstevel@tonic-gate  *
20*7c478bd9Sstevel@tonic-gate  * CDDL HEADER END
21*7c478bd9Sstevel@tonic-gate  */
22*7c478bd9Sstevel@tonic-gate /*
23*7c478bd9Sstevel@tonic-gate  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24*7c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
25*7c478bd9Sstevel@tonic-gate  */
26*7c478bd9Sstevel@tonic-gate 
27*7c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
28*7c478bd9Sstevel@tonic-gate 
29*7c478bd9Sstevel@tonic-gate /*
30*7c478bd9Sstevel@tonic-gate  * Copyright (c) 1998-1999 Innosoft International, Inc.  All Rights Reserved.
31*7c478bd9Sstevel@tonic-gate  *
32*7c478bd9Sstevel@tonic-gate  * Copyright (c) 1996-1997 Critical Angle Inc. All Rights Reserved.
33*7c478bd9Sstevel@tonic-gate  */
34*7c478bd9Sstevel@tonic-gate 
35*7c478bd9Sstevel@tonic-gate #include <stdio.h>
36*7c478bd9Sstevel@tonic-gate #include <string.h>
37*7c478bd9Sstevel@tonic-gate #include <stdlib.h>
38*7c478bd9Sstevel@tonic-gate #include <unistd.h>
39*7c478bd9Sstevel@tonic-gate #include <fcntl.h>
40*7c478bd9Sstevel@tonic-gate #include <ctype.h>
41*7c478bd9Sstevel@tonic-gate #include <md5.h>
42*7c478bd9Sstevel@tonic-gate #include <sys/time.h>
43*7c478bd9Sstevel@tonic-gate 
44*7c478bd9Sstevel@tonic-gate #include "lber.h"
45*7c478bd9Sstevel@tonic-gate #include "ldap.h"
46*7c478bd9Sstevel@tonic-gate #include "ldap-int.h"
47*7c478bd9Sstevel@tonic-gate 
48*7c478bd9Sstevel@tonic-gate /*
49*7c478bd9Sstevel@tonic-gate  * DIGEST-MD5 SASL Mechanism
50*7c478bd9Sstevel@tonic-gate  */
51*7c478bd9Sstevel@tonic-gate 
52*7c478bd9Sstevel@tonic-gate /* use this instead of "const unsigned char" to eliminate compiler warnings */
53*7c478bd9Sstevel@tonic-gate typedef /* const */ unsigned char CONST_UCHAR;
54*7c478bd9Sstevel@tonic-gate 
55*7c478bd9Sstevel@tonic-gate /* size of a digest result */
56*7c478bd9Sstevel@tonic-gate #define	DIGEST_SIZE	 16
57*7c478bd9Sstevel@tonic-gate 
58*7c478bd9Sstevel@tonic-gate /* size of a digest hex string */
59*7c478bd9Sstevel@tonic-gate #define	DIGEST_HEX_SIZE (DIGEST_SIZE * 2 + 1)
60*7c478bd9Sstevel@tonic-gate 
61*7c478bd9Sstevel@tonic-gate /*
62*7c478bd9Sstevel@tonic-gate  * extra bytes which a client response needs in addition to size of
63*7c478bd9Sstevel@tonic-gate  * server challenge */
64*7c478bd9Sstevel@tonic-gate #define	DIGEST_CLIENT_EXTRA (DIGEST_HEX_SIZE + 128)
65*7c478bd9Sstevel@tonic-gate 
66*7c478bd9Sstevel@tonic-gate /* erase a digest_attrs_t structure */
67*7c478bd9Sstevel@tonic-gate #define	digest_clear(attrs) memset((attrs), 0, sizeof (digest_attrs_t))
68*7c478bd9Sstevel@tonic-gate 
69*7c478bd9Sstevel@tonic-gate /*
70*7c478bd9Sstevel@tonic-gate  * broken-out digest attributes (with quotes removed)
71*7c478bd9Sstevel@tonic-gate  *  probably not NUL terminated.
72*7c478bd9Sstevel@tonic-gate  */
73*7c478bd9Sstevel@tonic-gate typedef struct {
74*7c478bd9Sstevel@tonic-gate 	const char *realm, *nonce, *cnonce, *qop, *user, *resp, *dom;
75*7c478bd9Sstevel@tonic-gate 	const char *max, *stale, *ncount, *uri, *charset;
76*7c478bd9Sstevel@tonic-gate 	int rlen, nlen, clen, qlen, ulen, resplen, dlen;
77*7c478bd9Sstevel@tonic-gate 	int mlen, slen, nclen, urilen, charsetlen;
78*7c478bd9Sstevel@tonic-gate 	char ncbuf[9];
79*7c478bd9Sstevel@tonic-gate } digest_attrs_t;
80*7c478bd9Sstevel@tonic-gate 
81*7c478bd9Sstevel@tonic-gate static const char hextab[] = "0123456789abcdef";
82*7c478bd9Sstevel@tonic-gate static CONST_UCHAR colon[] = ":";
83*7c478bd9Sstevel@tonic-gate 
84*7c478bd9Sstevel@tonic-gate /*
85*7c478bd9Sstevel@tonic-gate  * Make a nonce (NUL terminated)
86*7c478bd9Sstevel@tonic-gate  *  buf	-- buffer for result
87*7c478bd9Sstevel@tonic-gate  *  maxlen -- max length of result
88*7c478bd9Sstevel@tonic-gate  * returns final length or -1 on error
89*7c478bd9Sstevel@tonic-gate  */
90*7c478bd9Sstevel@tonic-gate static int
digest_nonce(char * buf,int maxlen)91*7c478bd9Sstevel@tonic-gate digest_nonce(char *buf, int maxlen)
92*7c478bd9Sstevel@tonic-gate {
93*7c478bd9Sstevel@tonic-gate 	/*
94*7c478bd9Sstevel@tonic-gate 	 * it shouldn't matter too much if two threads step on this counter
95*7c478bd9Sstevel@tonic-gate 	 * at the same time, but mutexing it wouldn't hurt
96*7c478bd9Sstevel@tonic-gate 	 */
97*7c478bd9Sstevel@tonic-gate 	static int counter;
98*7c478bd9Sstevel@tonic-gate 	char *dst;
99*7c478bd9Sstevel@tonic-gate 	int len;
100*7c478bd9Sstevel@tonic-gate 	struct chal_info {
101*7c478bd9Sstevel@tonic-gate 		time_t mytime;
102*7c478bd9Sstevel@tonic-gate 		unsigned char digest[16];
103*7c478bd9Sstevel@tonic-gate 	} cinfo;
104*7c478bd9Sstevel@tonic-gate 	MD5_CTX ctx;
105*7c478bd9Sstevel@tonic-gate 	long r;
106*7c478bd9Sstevel@tonic-gate 	static int set_rand = 0;
107*7c478bd9Sstevel@tonic-gate 	unsigned char *p;
108*7c478bd9Sstevel@tonic-gate 	int j;
109*7c478bd9Sstevel@tonic-gate 	int fd;
110*7c478bd9Sstevel@tonic-gate 	int got_random;
111*7c478bd9Sstevel@tonic-gate 
112*7c478bd9Sstevel@tonic-gate 	/* initialize challenge */
113*7c478bd9Sstevel@tonic-gate 	if (maxlen < 2 * sizeof (cinfo))
114*7c478bd9Sstevel@tonic-gate 		return (-1);
115*7c478bd9Sstevel@tonic-gate 	dst = buf;
116*7c478bd9Sstevel@tonic-gate 
117*7c478bd9Sstevel@tonic-gate 	/* get a timestamp */
118*7c478bd9Sstevel@tonic-gate 	time(&cinfo.mytime);
119*7c478bd9Sstevel@tonic-gate 
120*7c478bd9Sstevel@tonic-gate 	/* get some randomness */
121*7c478bd9Sstevel@tonic-gate 
122*7c478bd9Sstevel@tonic-gate 	got_random = 0;
123*7c478bd9Sstevel@tonic-gate 	fd = open("/dev/urandom", O_RDONLY);
124*7c478bd9Sstevel@tonic-gate 	if (fd != -1) {
125*7c478bd9Sstevel@tonic-gate 	    got_random =
126*7c478bd9Sstevel@tonic-gate 		(read(fd, &r, sizeof (r)) == sizeof (r));
127*7c478bd9Sstevel@tonic-gate 	    close(fd);
128*7c478bd9Sstevel@tonic-gate 	}
129*7c478bd9Sstevel@tonic-gate 
130*7c478bd9Sstevel@tonic-gate 	if (!got_random) {
131*7c478bd9Sstevel@tonic-gate 	    if (set_rand == 0) {
132*7c478bd9Sstevel@tonic-gate 		struct timeval tv;
133*7c478bd9Sstevel@tonic-gate 
134*7c478bd9Sstevel@tonic-gate 		r = cinfo.mytime - (getpid() *65536) + (random() & 0xffff);
135*7c478bd9Sstevel@tonic-gate 
136*7c478bd9Sstevel@tonic-gate 		gettimeofday(&tv, NULL);
137*7c478bd9Sstevel@tonic-gate 		r ^= tv.tv_usec;
138*7c478bd9Sstevel@tonic-gate 		r ^= gethostid();
139*7c478bd9Sstevel@tonic-gate 
140*7c478bd9Sstevel@tonic-gate 		srandom(r);
141*7c478bd9Sstevel@tonic-gate 		set_rand = 1;
142*7c478bd9Sstevel@tonic-gate 	    }
143*7c478bd9Sstevel@tonic-gate 
144*7c478bd9Sstevel@tonic-gate 	    r = random();
145*7c478bd9Sstevel@tonic-gate 	}
146*7c478bd9Sstevel@tonic-gate 
147*7c478bd9Sstevel@tonic-gate 	MD5Init(&ctx);
148*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, (unsigned char *) &r, sizeof (r));
149*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, (unsigned char *) &counter, sizeof (counter));
150*7c478bd9Sstevel@tonic-gate 	++counter;
151*7c478bd9Sstevel@tonic-gate 	MD5Final(cinfo.digest, &ctx);
152*7c478bd9Sstevel@tonic-gate 
153*7c478bd9Sstevel@tonic-gate 	/* compute hex for result */
154*7c478bd9Sstevel@tonic-gate 	for (j = 0, p = (unsigned char *)&cinfo; j < sizeof (cinfo); ++j) {
155*7c478bd9Sstevel@tonic-gate 		dst[j * 2]	= hextab[p[j] >> 4];
156*7c478bd9Sstevel@tonic-gate 		dst[j * 2 + 1]	= hextab[p[j] & 0xf];
157*7c478bd9Sstevel@tonic-gate 	}
158*7c478bd9Sstevel@tonic-gate 
159*7c478bd9Sstevel@tonic-gate 	/* take the entire time_t, plus at least 6 bytes of MD5 output */
160*7c478bd9Sstevel@tonic-gate 	len = ((sizeof (time_t) + 6) * 2);
161*7c478bd9Sstevel@tonic-gate 	dst += len;
162*7c478bd9Sstevel@tonic-gate 	maxlen -= len;
163*7c478bd9Sstevel@tonic-gate 
164*7c478bd9Sstevel@tonic-gate 	*dst = '\0';
165*7c478bd9Sstevel@tonic-gate 
166*7c478bd9Sstevel@tonic-gate 	return (dst - buf);
167*7c478bd9Sstevel@tonic-gate }
168*7c478bd9Sstevel@tonic-gate 
169*7c478bd9Sstevel@tonic-gate /*
170*7c478bd9Sstevel@tonic-gate  * if the string is entirely in the 8859-1 subset of UTF-8, then translate
171*7c478bd9Sstevel@tonic-gate  * to 8859-1 prior to MD5
172*7c478bd9Sstevel@tonic-gate  */
173*7c478bd9Sstevel@tonic-gate static void
MD5_UTF8_8859_1(MD5_CTX * ctx,CONST_UCHAR * base,int len)174*7c478bd9Sstevel@tonic-gate MD5_UTF8_8859_1(MD5_CTX *ctx, CONST_UCHAR *base, int len)
175*7c478bd9Sstevel@tonic-gate {
176*7c478bd9Sstevel@tonic-gate 	CONST_UCHAR *scan, *end;
177*7c478bd9Sstevel@tonic-gate 	unsigned char cbuf;
178*7c478bd9Sstevel@tonic-gate 
179*7c478bd9Sstevel@tonic-gate 	end = base + len;
180*7c478bd9Sstevel@tonic-gate 	for (scan = base; scan < end; ++scan) {
181*7c478bd9Sstevel@tonic-gate 		if (*scan > 0xC3) break; /* abort if outside 8859-1 */
182*7c478bd9Sstevel@tonic-gate 		if (*scan >= 0xC0 && *scan <= 0xC3) {
183*7c478bd9Sstevel@tonic-gate 		    if (++scan == end || *scan < 0x80 || *scan > 0xBF) break;
184*7c478bd9Sstevel@tonic-gate 		}
185*7c478bd9Sstevel@tonic-gate 	}
186*7c478bd9Sstevel@tonic-gate 	/* if we found a character outside 8859-1, don't alter string */
187*7c478bd9Sstevel@tonic-gate 	if (scan < end) {
188*7c478bd9Sstevel@tonic-gate 		MD5Update(ctx, base, len);
189*7c478bd9Sstevel@tonic-gate 		return;
190*7c478bd9Sstevel@tonic-gate 	}
191*7c478bd9Sstevel@tonic-gate 
192*7c478bd9Sstevel@tonic-gate 	/* convert to 8859-1 prior to applying hash */
193*7c478bd9Sstevel@tonic-gate 	do {
194*7c478bd9Sstevel@tonic-gate 		for (scan = base; scan < end && *scan < 0xC0; ++scan)
195*7c478bd9Sstevel@tonic-gate 			;
196*7c478bd9Sstevel@tonic-gate 		if (scan != base) MD5Update(ctx, base, scan - base);
197*7c478bd9Sstevel@tonic-gate 		if (scan + 1 >= end) break;
198*7c478bd9Sstevel@tonic-gate 		cbuf = ((scan[0] & 0x3) << 6) | (scan[1] & 0x3f);
199*7c478bd9Sstevel@tonic-gate 		MD5Update(ctx, &cbuf, 1);
200*7c478bd9Sstevel@tonic-gate 		base = scan + 2;
201*7c478bd9Sstevel@tonic-gate 	} while (base < end);
202*7c478bd9Sstevel@tonic-gate }
203*7c478bd9Sstevel@tonic-gate 
204*7c478bd9Sstevel@tonic-gate /*
205*7c478bd9Sstevel@tonic-gate  * Compute MD5( "<user>:<realm>:<pass>" )
206*7c478bd9Sstevel@tonic-gate  *  if use8859_1 is non-zero, then user/realm is 8859-1 charset
207*7c478bd9Sstevel@tonic-gate  *  if supplied lengths are 0, strlen() is used
208*7c478bd9Sstevel@tonic-gate  *  places result in hash_pass (of size DIGEST_SIZE) and returns it.
209*7c478bd9Sstevel@tonic-gate  */
210*7c478bd9Sstevel@tonic-gate static unsigned char *
digest_hash_pass(const char * user,int ulen,const char * realm,int rlen,const char * pass,int passlen,int use8859_1,unsigned char * hash_pass)211*7c478bd9Sstevel@tonic-gate digest_hash_pass(const char *user, int ulen, const char *realm, int rlen,
212*7c478bd9Sstevel@tonic-gate 		const char *pass, int passlen, int use8859_1,
213*7c478bd9Sstevel@tonic-gate 		unsigned char *hash_pass)
214*7c478bd9Sstevel@tonic-gate {
215*7c478bd9Sstevel@tonic-gate 	MD5_CTX ctx;
216*7c478bd9Sstevel@tonic-gate 
217*7c478bd9Sstevel@tonic-gate 	MD5Init(&ctx);
218*7c478bd9Sstevel@tonic-gate 	if (ulen == 0) ulen = strlen(user);
219*7c478bd9Sstevel@tonic-gate 	if (use8859_1) {
220*7c478bd9Sstevel@tonic-gate 		MD5Update(&ctx, (CONST_UCHAR *) user, ulen);
221*7c478bd9Sstevel@tonic-gate 	} else {
222*7c478bd9Sstevel@tonic-gate 		MD5_UTF8_8859_1(&ctx, (CONST_UCHAR *) user, ulen);
223*7c478bd9Sstevel@tonic-gate 	}
224*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, colon, 1);
225*7c478bd9Sstevel@tonic-gate 	if (rlen == 0) rlen = strlen(realm);
226*7c478bd9Sstevel@tonic-gate 	if (use8859_1) {
227*7c478bd9Sstevel@tonic-gate 		MD5Update(&ctx, (CONST_UCHAR *) realm, rlen);
228*7c478bd9Sstevel@tonic-gate 	} else {
229*7c478bd9Sstevel@tonic-gate 		MD5_UTF8_8859_1(&ctx, (CONST_UCHAR *) realm, rlen);
230*7c478bd9Sstevel@tonic-gate 	}
231*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, colon, 1);
232*7c478bd9Sstevel@tonic-gate 	if (passlen == 0) passlen = strlen(pass);
233*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, (CONST_UCHAR *) pass, passlen);
234*7c478bd9Sstevel@tonic-gate 	MD5Final(hash_pass, &ctx);
235*7c478bd9Sstevel@tonic-gate 
236*7c478bd9Sstevel@tonic-gate 	return (hash_pass);
237*7c478bd9Sstevel@tonic-gate }
238*7c478bd9Sstevel@tonic-gate 
239*7c478bd9Sstevel@tonic-gate /*
240*7c478bd9Sstevel@tonic-gate  * Compute MD5("<hash_pass>:<nonce>:<cnonce>")
241*7c478bd9Sstevel@tonic-gate  * places result in hash_a1 and returns hash_a1
242*7c478bd9Sstevel@tonic-gate  * note that hash_pass and hash_a1 may be the same
243*7c478bd9Sstevel@tonic-gate  */
244*7c478bd9Sstevel@tonic-gate static unsigned char *
digest_hash_a1(const digest_attrs_t * attr,CONST_UCHAR * hash_pass,unsigned char * hash_a1)245*7c478bd9Sstevel@tonic-gate digest_hash_a1(const digest_attrs_t *attr, CONST_UCHAR *hash_pass,
246*7c478bd9Sstevel@tonic-gate 		unsigned char *hash_a1)
247*7c478bd9Sstevel@tonic-gate {
248*7c478bd9Sstevel@tonic-gate 	MD5_CTX ctx;
249*7c478bd9Sstevel@tonic-gate 
250*7c478bd9Sstevel@tonic-gate 	MD5Init(&ctx);
251*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, hash_pass, DIGEST_SIZE);
252*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, colon, 1);
253*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, (CONST_UCHAR *) attr->nonce, attr->nlen);
254*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, colon, 1);
255*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, (CONST_UCHAR *) attr->cnonce, attr->clen);
256*7c478bd9Sstevel@tonic-gate 	MD5Final(hash_a1, &ctx);
257*7c478bd9Sstevel@tonic-gate 
258*7c478bd9Sstevel@tonic-gate 	return (hash_a1);
259*7c478bd9Sstevel@tonic-gate }
260*7c478bd9Sstevel@tonic-gate 
261*7c478bd9Sstevel@tonic-gate /*
262*7c478bd9Sstevel@tonic-gate  * calculate hash response for digest auth.
263*7c478bd9Sstevel@tonic-gate  *  outresp must be buffer of at least DIGEST_HEX_SIZE
264*7c478bd9Sstevel@tonic-gate  *  outresp and hex_int may be the same
265*7c478bd9Sstevel@tonic-gate  *  method may be NULL if mlen is 0
266*7c478bd9Sstevel@tonic-gate  */
267*7c478bd9Sstevel@tonic-gate static void
digest_calc_resp(const digest_attrs_t * attr,CONST_UCHAR * hash_a1,const char * method,int mlen,CONST_UCHAR * hex_int,char * outresp)268*7c478bd9Sstevel@tonic-gate digest_calc_resp(const digest_attrs_t *attr,
269*7c478bd9Sstevel@tonic-gate 		CONST_UCHAR *hash_a1, const char *method, int mlen,
270*7c478bd9Sstevel@tonic-gate 		CONST_UCHAR *hex_int, char *outresp)
271*7c478bd9Sstevel@tonic-gate {
272*7c478bd9Sstevel@tonic-gate 	static CONST_UCHAR defncount[] = ":00000001:";
273*7c478bd9Sstevel@tonic-gate 	static CONST_UCHAR empty_hex_int[] =
274*7c478bd9Sstevel@tonic-gate 			"00000000000000000000000000000000";
275*7c478bd9Sstevel@tonic-gate 	MD5_CTX ctx;
276*7c478bd9Sstevel@tonic-gate 	unsigned char resp[DIGEST_SIZE];
277*7c478bd9Sstevel@tonic-gate 	unsigned char *hex_a1 = (unsigned char *) outresp;
278*7c478bd9Sstevel@tonic-gate 	unsigned char *hex_a2 = (unsigned char *) outresp;
279*7c478bd9Sstevel@tonic-gate 	unsigned j;
280*7c478bd9Sstevel@tonic-gate 
281*7c478bd9Sstevel@tonic-gate 	/* compute hash of A2 and put in resp */
282*7c478bd9Sstevel@tonic-gate 	MD5Init(&ctx);
283*7c478bd9Sstevel@tonic-gate 	if (mlen == 0 && method != NULL) mlen = strlen(method);
284*7c478bd9Sstevel@tonic-gate 	if (mlen) MD5Update(&ctx, (CONST_UCHAR *) method, mlen);
285*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, colon, 1);
286*7c478bd9Sstevel@tonic-gate 	if (attr->urilen != 0) {
287*7c478bd9Sstevel@tonic-gate 		MD5Update(&ctx, (CONST_UCHAR *) attr->uri, attr->urilen);
288*7c478bd9Sstevel@tonic-gate 	}
289*7c478bd9Sstevel@tonic-gate 	if (attr->qlen != 4 || strncasecmp(attr->qop, "auth", 4) != 0) {
290*7c478bd9Sstevel@tonic-gate 		MD5Update(&ctx, colon, 1);
291*7c478bd9Sstevel@tonic-gate 	if (hex_int == NULL) hex_int = empty_hex_int;
292*7c478bd9Sstevel@tonic-gate 		MD5Update(&ctx, hex_int, DIGEST_SIZE * 2);
293*7c478bd9Sstevel@tonic-gate 	}
294*7c478bd9Sstevel@tonic-gate 	MD5Final(resp, &ctx);
295*7c478bd9Sstevel@tonic-gate 
296*7c478bd9Sstevel@tonic-gate 	/* compute hex_a1 from hash_a1 */
297*7c478bd9Sstevel@tonic-gate 	for (j = 0; j < DIGEST_SIZE; ++j) {
298*7c478bd9Sstevel@tonic-gate 		hex_a1[j * 2]	 = hextab[hash_a1[j] >> 4];
299*7c478bd9Sstevel@tonic-gate 		hex_a1[j * 2 + 1] = hextab[hash_a1[j] & 0xf];
300*7c478bd9Sstevel@tonic-gate 	}
301*7c478bd9Sstevel@tonic-gate 
302*7c478bd9Sstevel@tonic-gate 	/* compute response */
303*7c478bd9Sstevel@tonic-gate 	MD5Init(&ctx);
304*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, hex_a1, DIGEST_SIZE * 2);
305*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, colon, 1);
306*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, (CONST_UCHAR *) attr->nonce, attr->nlen);
307*7c478bd9Sstevel@tonic-gate 	if (attr->ncount != NULL) {
308*7c478bd9Sstevel@tonic-gate 		MD5Update(&ctx, colon, 1);
309*7c478bd9Sstevel@tonic-gate 		MD5Update(&ctx, (CONST_UCHAR *) attr->ncount, attr->nclen);
310*7c478bd9Sstevel@tonic-gate 		MD5Update(&ctx, colon, 1);
311*7c478bd9Sstevel@tonic-gate 	} else {
312*7c478bd9Sstevel@tonic-gate 		MD5Update(&ctx, defncount, sizeof (defncount) - 1);
313*7c478bd9Sstevel@tonic-gate 	}
314*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, (CONST_UCHAR *) attr->cnonce, attr->clen);
315*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, colon, 1);
316*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, (CONST_UCHAR *) attr->qop, attr->qlen);
317*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, colon, 1);
318*7c478bd9Sstevel@tonic-gate 
319*7c478bd9Sstevel@tonic-gate 	/* compute hex_a2 from hash_a2 */
320*7c478bd9Sstevel@tonic-gate 	for (j = 0; j < DIGEST_SIZE; ++j) {
321*7c478bd9Sstevel@tonic-gate 		hex_a2[j * 2]	 = hextab[resp[j] >> 4];
322*7c478bd9Sstevel@tonic-gate 		hex_a2[j * 2 + 1] = hextab[resp[j] & 0xf];
323*7c478bd9Sstevel@tonic-gate 	}
324*7c478bd9Sstevel@tonic-gate 	MD5Update(&ctx, hex_a2, DIGEST_SIZE * 2);
325*7c478bd9Sstevel@tonic-gate 	MD5Final(resp, &ctx);
326*7c478bd9Sstevel@tonic-gate 
327*7c478bd9Sstevel@tonic-gate 	/* generate hex output */
328*7c478bd9Sstevel@tonic-gate 	for (j = 0; j < DIGEST_SIZE; ++j) {
329*7c478bd9Sstevel@tonic-gate 		outresp[j * 2]	 = hextab[resp[j] >> 4];
330*7c478bd9Sstevel@tonic-gate 		outresp[j * 2 + 1] = hextab[resp[j] & 0xf];
331*7c478bd9Sstevel@tonic-gate 	}
332*7c478bd9Sstevel@tonic-gate 	outresp[DIGEST_SIZE * 2] = '\0';
333*7c478bd9Sstevel@tonic-gate 	memset(resp, 0, sizeof (resp));
334*7c478bd9Sstevel@tonic-gate }
335*7c478bd9Sstevel@tonic-gate 
336*7c478bd9Sstevel@tonic-gate /*
337*7c478bd9Sstevel@tonic-gate  * generate the client response from attributes
338*7c478bd9Sstevel@tonic-gate  *  either one of hash_pass and hash_a1 may be NULL
339*7c478bd9Sstevel@tonic-gate  *  hash_a1 is used on re-authentication and takes precedence over hash_pass
340*7c478bd9Sstevel@tonic-gate  */
341*7c478bd9Sstevel@tonic-gate static int
digest_client_resp(const char * method,int mlen,CONST_UCHAR * hash_pass,CONST_UCHAR * hash_a1,digest_attrs_t * attr,char * outbuf,int maxout,int * plen)342*7c478bd9Sstevel@tonic-gate digest_client_resp(const char *method, int mlen,
343*7c478bd9Sstevel@tonic-gate 		CONST_UCHAR *hash_pass, CONST_UCHAR *hash_a1,
344*7c478bd9Sstevel@tonic-gate 		digest_attrs_t *attr, /* in/out attributes */
345*7c478bd9Sstevel@tonic-gate 		char *outbuf, int maxout, int *plen)
346*7c478bd9Sstevel@tonic-gate {
347*7c478bd9Sstevel@tonic-gate #define	prefixsize (sizeof (prefix) - 4 * 4 - 1)
348*7c478bd9Sstevel@tonic-gate #define	suffixsize (sizeof (rstr) + sizeof (qstr) - 1 + DIGEST_SIZE * 2)
349*7c478bd9Sstevel@tonic-gate 	static const char prefix[] =
350*7c478bd9Sstevel@tonic-gate 	"username=\"%.*s\",realm=\"%.*s\",nonce=\"%.*s\",nc=%.*s,cnonce=\"";
351*7c478bd9Sstevel@tonic-gate 	static const char rstr[] = "\",response=";
352*7c478bd9Sstevel@tonic-gate 	static const char qstr[] = ",qop=auth";
353*7c478bd9Sstevel@tonic-gate 	static const char chstr[] = "charset=";
354*7c478bd9Sstevel@tonic-gate 	char *scan;
355*7c478bd9Sstevel@tonic-gate 	int len;
356*7c478bd9Sstevel@tonic-gate 	char hexbuf[DIGEST_HEX_SIZE];
357*7c478bd9Sstevel@tonic-gate 	unsigned char hashbuf[DIGEST_SIZE];
358*7c478bd9Sstevel@tonic-gate 
359*7c478bd9Sstevel@tonic-gate 	/* make sure we have mandatory attributes */
360*7c478bd9Sstevel@tonic-gate 	if (attr->nonce == NULL || attr->nlen == 0 ||
361*7c478bd9Sstevel@tonic-gate 	    attr->realm == NULL || attr->rlen == 0 ||
362*7c478bd9Sstevel@tonic-gate 	    attr->qop == NULL || attr->qlen == 0 ||
363*7c478bd9Sstevel@tonic-gate 	    (attr->nclen != 0 && attr->nclen != 8)) {
364*7c478bd9Sstevel@tonic-gate 		return (-5);
365*7c478bd9Sstevel@tonic-gate 	}
366*7c478bd9Sstevel@tonic-gate 	if (mlen != 0 && method == NULL)
367*7c478bd9Sstevel@tonic-gate 		return (-7);
368*7c478bd9Sstevel@tonic-gate 
369*7c478bd9Sstevel@tonic-gate 	/* initialize ncount */
370*7c478bd9Sstevel@tonic-gate 	if (attr->ncount == NULL) {
371*7c478bd9Sstevel@tonic-gate 		strcpy(attr->ncbuf, "00000001");
372*7c478bd9Sstevel@tonic-gate 		attr->ncount = attr->ncbuf;
373*7c478bd9Sstevel@tonic-gate 		attr->nclen = 8;
374*7c478bd9Sstevel@tonic-gate 	} else if (attr->ncount == attr->ncbuf) {
375*7c478bd9Sstevel@tonic-gate 		/* increment ncount */
376*7c478bd9Sstevel@tonic-gate 		scan = attr->ncbuf + 7;
377*7c478bd9Sstevel@tonic-gate 		while (scan >= attr->ncbuf) {
378*7c478bd9Sstevel@tonic-gate 			if (*scan == '9') {
379*7c478bd9Sstevel@tonic-gate 				*scan = 'a';
380*7c478bd9Sstevel@tonic-gate 				break;
381*7c478bd9Sstevel@tonic-gate 			} else if (*scan != 'f') {
382*7c478bd9Sstevel@tonic-gate 				++*scan;
383*7c478bd9Sstevel@tonic-gate 				break;
384*7c478bd9Sstevel@tonic-gate 			}
385*7c478bd9Sstevel@tonic-gate 			*scan = '0';
386*7c478bd9Sstevel@tonic-gate 			--scan;
387*7c478bd9Sstevel@tonic-gate 		}
388*7c478bd9Sstevel@tonic-gate 	}
389*7c478bd9Sstevel@tonic-gate 
390*7c478bd9Sstevel@tonic-gate 	/* sanity check length */
391*7c478bd9Sstevel@tonic-gate 	len = prefixsize + attr->ulen + attr->rlen + attr->nlen + attr->nclen;
392*7c478bd9Sstevel@tonic-gate 	if (attr->charsetlen > 0) {
393*7c478bd9Sstevel@tonic-gate 		/* includes 1 for a comma */
394*7c478bd9Sstevel@tonic-gate 		len += sizeof (chstr) + attr->charsetlen;
395*7c478bd9Sstevel@tonic-gate 	}
396*7c478bd9Sstevel@tonic-gate 	if (len + suffixsize >= maxout)
397*7c478bd9Sstevel@tonic-gate 		return (-3);
398*7c478bd9Sstevel@tonic-gate 
399*7c478bd9Sstevel@tonic-gate 	scan = outbuf;
400*7c478bd9Sstevel@tonic-gate 
401*7c478bd9Sstevel@tonic-gate 	/* charset */
402*7c478bd9Sstevel@tonic-gate 	if (attr->charsetlen > 0 && attr->charset != NULL) {
403*7c478bd9Sstevel@tonic-gate 		memcpy(scan, chstr, sizeof (chstr) - 1);
404*7c478bd9Sstevel@tonic-gate 		scan += sizeof (chstr) - 1;
405*7c478bd9Sstevel@tonic-gate 		memcpy(scan, attr->charset, attr->charsetlen);
406*7c478bd9Sstevel@tonic-gate 		scan += attr->charsetlen;
407*7c478bd9Sstevel@tonic-gate 		*scan++ = ',';
408*7c478bd9Sstevel@tonic-gate 	}
409*7c478bd9Sstevel@tonic-gate 
410*7c478bd9Sstevel@tonic-gate 	/* generate string up to the client nonce */
411*7c478bd9Sstevel@tonic-gate 	sprintf(scan, prefix, attr->ulen, attr->user,
412*7c478bd9Sstevel@tonic-gate 		attr->rlen, attr->realm, attr->nlen, attr->nonce,
413*7c478bd9Sstevel@tonic-gate 		attr->nclen, attr->ncount);
414*7c478bd9Sstevel@tonic-gate 	scan = outbuf + len;
415*7c478bd9Sstevel@tonic-gate 
416*7c478bd9Sstevel@tonic-gate 	/* generate client nonce */
417*7c478bd9Sstevel@tonic-gate 	len = digest_nonce(scan, maxout - (scan - outbuf));
418*7c478bd9Sstevel@tonic-gate 	if (len < 0)
419*7c478bd9Sstevel@tonic-gate 		return (len);
420*7c478bd9Sstevel@tonic-gate 	attr->cnonce = scan;
421*7c478bd9Sstevel@tonic-gate 	attr->clen = len;
422*7c478bd9Sstevel@tonic-gate 	scan += len;
423*7c478bd9Sstevel@tonic-gate 	if (scan - outbuf + suffixsize > maxout)
424*7c478bd9Sstevel@tonic-gate 		return (-3);
425*7c478bd9Sstevel@tonic-gate 
426*7c478bd9Sstevel@tonic-gate 	/* compute response */
427*7c478bd9Sstevel@tonic-gate 	if (hash_a1 == NULL) {
428*7c478bd9Sstevel@tonic-gate 		if (hash_pass == NULL)
429*7c478bd9Sstevel@tonic-gate 			return (-7);
430*7c478bd9Sstevel@tonic-gate 		hash_a1 = digest_hash_a1(attr, hash_pass, hashbuf);
431*7c478bd9Sstevel@tonic-gate 	}
432*7c478bd9Sstevel@tonic-gate 	digest_calc_resp(attr, hash_a1, method, mlen, NULL, hexbuf);
433*7c478bd9Sstevel@tonic-gate 
434*7c478bd9Sstevel@tonic-gate 	/* finish it */
435*7c478bd9Sstevel@tonic-gate 	memcpy(scan, rstr, sizeof (rstr) - 1);
436*7c478bd9Sstevel@tonic-gate 	scan += sizeof (rstr) - 1;
437*7c478bd9Sstevel@tonic-gate 	memcpy(scan, hexbuf, DIGEST_SIZE * 2);
438*7c478bd9Sstevel@tonic-gate 	attr->resp = scan;
439*7c478bd9Sstevel@tonic-gate 	attr->resplen = DIGEST_SIZE * 2;
440*7c478bd9Sstevel@tonic-gate 	scan += DIGEST_SIZE * 2;
441*7c478bd9Sstevel@tonic-gate 	memcpy(scan, qstr, sizeof (qstr));
442*7c478bd9Sstevel@tonic-gate 
443*7c478bd9Sstevel@tonic-gate 	/* set final length */
444*7c478bd9Sstevel@tonic-gate 	if (plen != NULL) *plen = scan - outbuf + sizeof (qstr) - 1;
445*7c478bd9Sstevel@tonic-gate 
446*7c478bd9Sstevel@tonic-gate 	return (0);
447*7c478bd9Sstevel@tonic-gate }
448*7c478bd9Sstevel@tonic-gate 
449*7c478bd9Sstevel@tonic-gate #define	lstreqcase(conststr, val, len) ((len) == sizeof (conststr) - 1 && \
450*7c478bd9Sstevel@tonic-gate 		strncasecmp((conststr), (val), sizeof (conststr) - 1) == 0)
451*7c478bd9Sstevel@tonic-gate 
452*7c478bd9Sstevel@tonic-gate /* parse a digest auth string */
453*7c478bd9Sstevel@tonic-gate static int
digest_parse(const char * str,int len,digest_attrs_t * attr_out)454*7c478bd9Sstevel@tonic-gate digest_parse(const char *str, int len, digest_attrs_t *attr_out)
455*7c478bd9Sstevel@tonic-gate {
456*7c478bd9Sstevel@tonic-gate 	static const char rstr[] = "realm";
457*7c478bd9Sstevel@tonic-gate 	static const char nstr[] = "nonce";
458*7c478bd9Sstevel@tonic-gate 	static const char cstr[] = "cnonce";
459*7c478bd9Sstevel@tonic-gate 	static const char qstr[] = "qop";
460*7c478bd9Sstevel@tonic-gate 	static const char ustr[] = "username";
461*7c478bd9Sstevel@tonic-gate 	static const char respstr[] = "response";
462*7c478bd9Sstevel@tonic-gate 	static const char dstr[] = "domain";
463*7c478bd9Sstevel@tonic-gate 	static const char mstr[] = "maxbuf";
464*7c478bd9Sstevel@tonic-gate 	static const char sstr[] = "stale";
465*7c478bd9Sstevel@tonic-gate 	static const char ncstr[] = "nc";
466*7c478bd9Sstevel@tonic-gate 	static const char uristr[] = "digest-uri";
467*7c478bd9Sstevel@tonic-gate 	static const char charsetstr[] = "charset";
468*7c478bd9Sstevel@tonic-gate 	const char *scan, *attr, *val, *end;
469*7c478bd9Sstevel@tonic-gate 	int alen, vlen;
470*7c478bd9Sstevel@tonic-gate 
471*7c478bd9Sstevel@tonic-gate 	if (len == 0) len = strlen(str);
472*7c478bd9Sstevel@tonic-gate 	scan = str;
473*7c478bd9Sstevel@tonic-gate 	end = str + len;
474*7c478bd9Sstevel@tonic-gate 	for (;;) {
475*7c478bd9Sstevel@tonic-gate 		/* skip over commas */
476*7c478bd9Sstevel@tonic-gate 		while (scan < end && (*scan == ',' || isspace(*scan))) ++scan;
477*7c478bd9Sstevel@tonic-gate 		/* parse attribute */
478*7c478bd9Sstevel@tonic-gate 		attr = scan;
479*7c478bd9Sstevel@tonic-gate 		while (scan < end && *scan != '=') ++scan;
480*7c478bd9Sstevel@tonic-gate 		alen = scan - attr;
481*7c478bd9Sstevel@tonic-gate 		if (!alen || scan == end || scan + 1 == end) {
482*7c478bd9Sstevel@tonic-gate 			return (-5);
483*7c478bd9Sstevel@tonic-gate 		}
484*7c478bd9Sstevel@tonic-gate 
485*7c478bd9Sstevel@tonic-gate 		/* parse value */
486*7c478bd9Sstevel@tonic-gate 		if (scan[1] == '"') {
487*7c478bd9Sstevel@tonic-gate 			scan += 2;
488*7c478bd9Sstevel@tonic-gate 			val = scan;
489*7c478bd9Sstevel@tonic-gate 			while (scan < end && *scan != '"') {
490*7c478bd9Sstevel@tonic-gate 				/* skip over "\" quoting, but don't remove it */
491*7c478bd9Sstevel@tonic-gate 				if (*scan == '\\') {
492*7c478bd9Sstevel@tonic-gate 					if (scan + 1 == end)
493*7c478bd9Sstevel@tonic-gate 						return (-5);
494*7c478bd9Sstevel@tonic-gate 					scan += 2;
495*7c478bd9Sstevel@tonic-gate 				} else {
496*7c478bd9Sstevel@tonic-gate 					++scan;
497*7c478bd9Sstevel@tonic-gate 				}
498*7c478bd9Sstevel@tonic-gate 			}
499*7c478bd9Sstevel@tonic-gate 			vlen = scan - val;
500*7c478bd9Sstevel@tonic-gate 			if (*scan != '"')
501*7c478bd9Sstevel@tonic-gate 				return (-5);
502*7c478bd9Sstevel@tonic-gate 			++scan;
503*7c478bd9Sstevel@tonic-gate 		} else {
504*7c478bd9Sstevel@tonic-gate 			++scan;
505*7c478bd9Sstevel@tonic-gate 			val = scan;
506*7c478bd9Sstevel@tonic-gate 			while (scan < end && *scan != ',') ++scan;
507*7c478bd9Sstevel@tonic-gate 			vlen = scan - val;
508*7c478bd9Sstevel@tonic-gate 		}
509*7c478bd9Sstevel@tonic-gate 		if (!vlen)
510*7c478bd9Sstevel@tonic-gate 			return (-5);
511*7c478bd9Sstevel@tonic-gate 
512*7c478bd9Sstevel@tonic-gate 		/* lookup the attribute */
513*7c478bd9Sstevel@tonic-gate 		switch (*attr) {
514*7c478bd9Sstevel@tonic-gate 		    case 'c':
515*7c478bd9Sstevel@tonic-gate 		    case 'C':
516*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(cstr, attr, alen)) {
517*7c478bd9Sstevel@tonic-gate 				attr_out->cnonce = val;
518*7c478bd9Sstevel@tonic-gate 				attr_out->clen = vlen;
519*7c478bd9Sstevel@tonic-gate 			}
520*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(charsetstr, attr, alen)) {
521*7c478bd9Sstevel@tonic-gate 				attr_out->charset = val;
522*7c478bd9Sstevel@tonic-gate 				attr_out->charsetlen = vlen;
523*7c478bd9Sstevel@tonic-gate 			}
524*7c478bd9Sstevel@tonic-gate 			break;
525*7c478bd9Sstevel@tonic-gate 		    case 'd':
526*7c478bd9Sstevel@tonic-gate 		    case 'D':
527*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(dstr, attr, alen)) {
528*7c478bd9Sstevel@tonic-gate 				attr_out->dom = val;
529*7c478bd9Sstevel@tonic-gate 				attr_out->dlen = vlen;
530*7c478bd9Sstevel@tonic-gate 			}
531*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(uristr, attr, alen)) {
532*7c478bd9Sstevel@tonic-gate 				attr_out->uri = val;
533*7c478bd9Sstevel@tonic-gate 				attr_out->urilen = vlen;
534*7c478bd9Sstevel@tonic-gate 			}
535*7c478bd9Sstevel@tonic-gate 			break;
536*7c478bd9Sstevel@tonic-gate 		    case 'm':
537*7c478bd9Sstevel@tonic-gate 		    case 'M':
538*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(mstr, attr, alen)) {
539*7c478bd9Sstevel@tonic-gate 				attr_out->max = val;
540*7c478bd9Sstevel@tonic-gate 				attr_out->mlen = vlen;
541*7c478bd9Sstevel@tonic-gate 			}
542*7c478bd9Sstevel@tonic-gate 			break;
543*7c478bd9Sstevel@tonic-gate 		    case 'n':
544*7c478bd9Sstevel@tonic-gate 		    case 'N':
545*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(nstr, attr, alen)) {
546*7c478bd9Sstevel@tonic-gate 				attr_out->nonce = val;
547*7c478bd9Sstevel@tonic-gate 				attr_out->nlen = vlen;
548*7c478bd9Sstevel@tonic-gate 			}
549*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(ncstr, attr, alen)) {
550*7c478bd9Sstevel@tonic-gate 				attr_out->ncount = val;
551*7c478bd9Sstevel@tonic-gate 				attr_out->nclen = vlen;
552*7c478bd9Sstevel@tonic-gate 			}
553*7c478bd9Sstevel@tonic-gate 			break;
554*7c478bd9Sstevel@tonic-gate 		    case 'q':
555*7c478bd9Sstevel@tonic-gate 		    case 'Q':
556*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(qstr, attr, alen)) {
557*7c478bd9Sstevel@tonic-gate 				attr_out->qop = val;
558*7c478bd9Sstevel@tonic-gate 				attr_out->qlen = vlen;
559*7c478bd9Sstevel@tonic-gate 			}
560*7c478bd9Sstevel@tonic-gate 			break;
561*7c478bd9Sstevel@tonic-gate 		    case 'r':
562*7c478bd9Sstevel@tonic-gate 		    case 'R':
563*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(rstr, attr, alen)) {
564*7c478bd9Sstevel@tonic-gate 				attr_out->realm = val;
565*7c478bd9Sstevel@tonic-gate 				attr_out->rlen = vlen;
566*7c478bd9Sstevel@tonic-gate 			}
567*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(respstr, attr, alen)) {
568*7c478bd9Sstevel@tonic-gate 				attr_out->resp = val;
569*7c478bd9Sstevel@tonic-gate 				attr_out->resplen = vlen;
570*7c478bd9Sstevel@tonic-gate 			}
571*7c478bd9Sstevel@tonic-gate 			break;
572*7c478bd9Sstevel@tonic-gate 		    case 's':
573*7c478bd9Sstevel@tonic-gate 		    case 'S':
574*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(sstr, attr, alen)) {
575*7c478bd9Sstevel@tonic-gate 				attr_out->stale = val;
576*7c478bd9Sstevel@tonic-gate 				attr_out->slen = vlen;
577*7c478bd9Sstevel@tonic-gate 			}
578*7c478bd9Sstevel@tonic-gate 			break;
579*7c478bd9Sstevel@tonic-gate 		    case 'u':
580*7c478bd9Sstevel@tonic-gate 		    case 'U':
581*7c478bd9Sstevel@tonic-gate 			if (lstreqcase(ustr, attr, alen)) {
582*7c478bd9Sstevel@tonic-gate 				attr_out->user = val;
583*7c478bd9Sstevel@tonic-gate 				attr_out->ulen = vlen;
584*7c478bd9Sstevel@tonic-gate 			}
585*7c478bd9Sstevel@tonic-gate 			break;
586*7c478bd9Sstevel@tonic-gate 		}
587*7c478bd9Sstevel@tonic-gate 
588*7c478bd9Sstevel@tonic-gate 		/* we should be at the end of the string or a comma */
589*7c478bd9Sstevel@tonic-gate 		if (scan == end) break;
590*7c478bd9Sstevel@tonic-gate 		if (*scan != ',')
591*7c478bd9Sstevel@tonic-gate 			return (-5);
592*7c478bd9Sstevel@tonic-gate 	}
593*7c478bd9Sstevel@tonic-gate 
594*7c478bd9Sstevel@tonic-gate 	return (0);
595*7c478bd9Sstevel@tonic-gate }
596*7c478bd9Sstevel@tonic-gate 
ldap_digest_md5_encode(const char * challenge,const char * username,const char * passwd,char ** digest)597*7c478bd9Sstevel@tonic-gate static int ldap_digest_md5_encode(
598*7c478bd9Sstevel@tonic-gate 	const char *challenge,
599*7c478bd9Sstevel@tonic-gate 	const char *username,
600*7c478bd9Sstevel@tonic-gate 	const char *passwd,
601*7c478bd9Sstevel@tonic-gate 	char **digest
602*7c478bd9Sstevel@tonic-gate )
603*7c478bd9Sstevel@tonic-gate {
604*7c478bd9Sstevel@tonic-gate 	unsigned char hash_pass[DIGEST_SIZE];
605*7c478bd9Sstevel@tonic-gate 	digest_attrs_t attrs;
606*7c478bd9Sstevel@tonic-gate 	char *outbuf;
607*7c478bd9Sstevel@tonic-gate 	int outlen;
608*7c478bd9Sstevel@tonic-gate 	int ret;
609*7c478bd9Sstevel@tonic-gate 
610*7c478bd9Sstevel@tonic-gate 	/* validate args */
611*7c478bd9Sstevel@tonic-gate 	if (challenge == NULL || username == NULL || passwd == NULL) {
612*7c478bd9Sstevel@tonic-gate 		return (LDAP_PARAM_ERROR);
613*7c478bd9Sstevel@tonic-gate 	}
614*7c478bd9Sstevel@tonic-gate 
615*7c478bd9Sstevel@tonic-gate 	/* parse the challenge */
616*7c478bd9Sstevel@tonic-gate 	digest_clear(&attrs);
617*7c478bd9Sstevel@tonic-gate 	ret = digest_parse(challenge, 0, &attrs);
618*7c478bd9Sstevel@tonic-gate 	if (ret != 0)
619*7c478bd9Sstevel@tonic-gate 		return (LDAP_DECODING_ERROR);
620*7c478bd9Sstevel@tonic-gate 
621*7c478bd9Sstevel@tonic-gate 	/* server MUST specify support for charset=utf-8 */
622*7c478bd9Sstevel@tonic-gate 	if (attrs.charsetlen != 5 ||
623*7c478bd9Sstevel@tonic-gate 	    strncasecmp(attrs.charset, "utf-8", 5) != 0) {
624*7c478bd9Sstevel@tonic-gate 		LDAPDebug(LDAP_DEBUG_TRACE,
625*7c478bd9Sstevel@tonic-gate 			"server did not specify charset=utf-8\n",
626*7c478bd9Sstevel@tonic-gate 			0, 0, 0);
627*7c478bd9Sstevel@tonic-gate 		return (LDAP_NOT_SUPPORTED);
628*7c478bd9Sstevel@tonic-gate 	}
629*7c478bd9Sstevel@tonic-gate 
630*7c478bd9Sstevel@tonic-gate 	/* set up digest attributes */
631*7c478bd9Sstevel@tonic-gate 	attrs.user = username;
632*7c478bd9Sstevel@tonic-gate 	attrs.ulen = strlen(attrs.user);
633*7c478bd9Sstevel@tonic-gate 
634*7c478bd9Sstevel@tonic-gate 	/* allocate the output buffer */
635*7c478bd9Sstevel@tonic-gate 	outlen = strlen(challenge) + DIGEST_CLIENT_EXTRA + 1;
636*7c478bd9Sstevel@tonic-gate 	outbuf = (char *)malloc(outlen);
637*7c478bd9Sstevel@tonic-gate 	if (outbuf == NULL)
638*7c478bd9Sstevel@tonic-gate 		return (LDAP_NO_MEMORY);
639*7c478bd9Sstevel@tonic-gate 
640*7c478bd9Sstevel@tonic-gate 	/* hash the password */
641*7c478bd9Sstevel@tonic-gate 	digest_hash_pass(username, 0, attrs.realm, attrs.rlen,
642*7c478bd9Sstevel@tonic-gate 				passwd, 0, 0, hash_pass),
643*7c478bd9Sstevel@tonic-gate 
644*7c478bd9Sstevel@tonic-gate 	/* create the response */
645*7c478bd9Sstevel@tonic-gate 	ret = digest_client_resp("AUTHENTICATE", 12, hash_pass, NULL,
646*7c478bd9Sstevel@tonic-gate 			&attrs, outbuf, outlen, &outlen);
647*7c478bd9Sstevel@tonic-gate 	memset(hash_pass, 0, DIGEST_SIZE);
648*7c478bd9Sstevel@tonic-gate 	if (ret != 0) {
649*7c478bd9Sstevel@tonic-gate 		free(outbuf);
650*7c478bd9Sstevel@tonic-gate 		return (LDAP_DECODING_ERROR);
651*7c478bd9Sstevel@tonic-gate 	}
652*7c478bd9Sstevel@tonic-gate 
653*7c478bd9Sstevel@tonic-gate 	/* null terminate the response */
654*7c478bd9Sstevel@tonic-gate 	*(outbuf+outlen) = '\0';
655*7c478bd9Sstevel@tonic-gate 
656*7c478bd9Sstevel@tonic-gate 	*digest = outbuf;
657*7c478bd9Sstevel@tonic-gate 	return (LDAP_SUCCESS);
658*7c478bd9Sstevel@tonic-gate }
659*7c478bd9Sstevel@tonic-gate 
ldap_x_sasl_digest_md5_bind_s(LDAP * ld,char * user_name,struct berval * cred,LDAPControl ** serverctrls,LDAPControl ** clientctrls)660*7c478bd9Sstevel@tonic-gate int ldap_x_sasl_digest_md5_bind_s(
661*7c478bd9Sstevel@tonic-gate 	LDAP *ld,
662*7c478bd9Sstevel@tonic-gate 	char *user_name,
663*7c478bd9Sstevel@tonic-gate 	struct berval *cred,
664*7c478bd9Sstevel@tonic-gate 	LDAPControl **serverctrls,
665*7c478bd9Sstevel@tonic-gate 	LDAPControl **clientctrls)
666*7c478bd9Sstevel@tonic-gate {
667*7c478bd9Sstevel@tonic-gate 	struct berval	*challenge = NULL;
668*7c478bd9Sstevel@tonic-gate 	int		errnum;
669*7c478bd9Sstevel@tonic-gate 	char		*digest = NULL;
670*7c478bd9Sstevel@tonic-gate 	struct berval	resp;
671*7c478bd9Sstevel@tonic-gate 
672*7c478bd9Sstevel@tonic-gate 	LDAPDebug(LDAP_DEBUG_TRACE, "ldap_x_sasl_digest_md5_bind_s\n", 0, 0, 0);
673*7c478bd9Sstevel@tonic-gate 
674*7c478bd9Sstevel@tonic-gate 	/* Add debug */
675*7c478bd9Sstevel@tonic-gate 	if (ld == NULL || user_name == NULL || cred == NULL ||
676*7c478bd9Sstevel@tonic-gate 	    cred->bv_val == NULL)
677*7c478bd9Sstevel@tonic-gate 		return (LDAP_PARAM_ERROR);
678*7c478bd9Sstevel@tonic-gate 
679*7c478bd9Sstevel@tonic-gate 	if (ld->ld_version < LDAP_VERSION3)
680*7c478bd9Sstevel@tonic-gate 		return (LDAP_PARAM_ERROR);
681*7c478bd9Sstevel@tonic-gate 
682*7c478bd9Sstevel@tonic-gate 	errnum = ldap_sasl_bind_s(ld, NULL, LDAP_SASL_DIGEST_MD5,
683*7c478bd9Sstevel@tonic-gate 		NULL, serverctrls, clientctrls, &challenge);
684*7c478bd9Sstevel@tonic-gate 
685*7c478bd9Sstevel@tonic-gate 	if (errnum == LDAP_SASL_BIND_IN_PROGRESS) {
686*7c478bd9Sstevel@tonic-gate 		if (challenge != NULL) {
687*7c478bd9Sstevel@tonic-gate 			LDAPDebug(LDAP_DEBUG_TRACE,
688*7c478bd9Sstevel@tonic-gate 				"SASL challenge: %s\n",
689*7c478bd9Sstevel@tonic-gate 				challenge->bv_val, 0, 0);
690*7c478bd9Sstevel@tonic-gate 			errnum = ldap_digest_md5_encode(challenge->bv_val,
691*7c478bd9Sstevel@tonic-gate 				user_name, cred->bv_val, &digest);
692*7c478bd9Sstevel@tonic-gate 			ber_bvfree(challenge);
693*7c478bd9Sstevel@tonic-gate 			challenge = NULL;
694*7c478bd9Sstevel@tonic-gate 			if (errnum == LDAP_SUCCESS) {
695*7c478bd9Sstevel@tonic-gate 				resp.bv_val = digest;
696*7c478bd9Sstevel@tonic-gate 				resp.bv_len = strlen(digest);
697*7c478bd9Sstevel@tonic-gate 				LDAPDebug(LDAP_DEBUG_TRACE,
698*7c478bd9Sstevel@tonic-gate 					"SASL reply: %s\n",
699*7c478bd9Sstevel@tonic-gate 					digest, 0, 0);
700*7c478bd9Sstevel@tonic-gate 				errnum = ldap_sasl_bind_s(ld, NULL,
701*7c478bd9Sstevel@tonic-gate 					LDAP_SASL_DIGEST_MD5, &resp,
702*7c478bd9Sstevel@tonic-gate 					serverctrls, clientctrls, &challenge);
703*7c478bd9Sstevel@tonic-gate 				free(digest);
704*7c478bd9Sstevel@tonic-gate 			}
705*7c478bd9Sstevel@tonic-gate 			if (challenge != NULL)
706*7c478bd9Sstevel@tonic-gate 				ber_bvfree(challenge);
707*7c478bd9Sstevel@tonic-gate 		} else {
708*7c478bd9Sstevel@tonic-gate 			errnum = LDAP_NO_MEMORY; /* TO DO: What val? */
709*7c478bd9Sstevel@tonic-gate 		}
710*7c478bd9Sstevel@tonic-gate 	}
711*7c478bd9Sstevel@tonic-gate 
712*7c478bd9Sstevel@tonic-gate 	LDAP_MUTEX_LOCK(ld, LDAP_ERR_LOCK);
713*7c478bd9Sstevel@tonic-gate 	ld->ld_errno = errnum;
714*7c478bd9Sstevel@tonic-gate 	LDAP_MUTEX_UNLOCK(ld, LDAP_ERR_LOCK);
715*7c478bd9Sstevel@tonic-gate 	return (errnum);
716*7c478bd9Sstevel@tonic-gate }
717*7c478bd9Sstevel@tonic-gate 
718*7c478bd9Sstevel@tonic-gate static int
sasl_digest_md5_bind_1(LDAP * ld,char * user_name,LDAPControl ** serverctrls,LDAPControl ** clientctrls,int * msgidp)719*7c478bd9Sstevel@tonic-gate sasl_digest_md5_bind_1(
720*7c478bd9Sstevel@tonic-gate 	LDAP *ld,
721*7c478bd9Sstevel@tonic-gate 	char *user_name,
722*7c478bd9Sstevel@tonic-gate 	LDAPControl **serverctrls,
723*7c478bd9Sstevel@tonic-gate 	LDAPControl **clientctrls,
724*7c478bd9Sstevel@tonic-gate 	int *msgidp)
725*7c478bd9Sstevel@tonic-gate {
726*7c478bd9Sstevel@tonic-gate 	if (ld == NULL || user_name == NULL || msgidp == NULL)
727*7c478bd9Sstevel@tonic-gate 		return (LDAP_PARAM_ERROR);
728*7c478bd9Sstevel@tonic-gate 
729*7c478bd9Sstevel@tonic-gate 	if (ld->ld_version < LDAP_VERSION3)
730*7c478bd9Sstevel@tonic-gate 		return (LDAP_PARAM_ERROR);
731*7c478bd9Sstevel@tonic-gate 
732*7c478bd9Sstevel@tonic-gate 	return (ldap_sasl_bind(ld, NULL, LDAP_SASL_DIGEST_MD5,
733*7c478bd9Sstevel@tonic-gate 		NULL, serverctrls, clientctrls, msgidp));
734*7c478bd9Sstevel@tonic-gate }
735*7c478bd9Sstevel@tonic-gate 
736*7c478bd9Sstevel@tonic-gate static int
sasl_digest_md5_bind_2(LDAP * ld,char * user_name,struct berval * cred,LDAPControl ** serverctrls,LDAPControl ** clientctrls,LDAPMessage * result,int * msgidp)737*7c478bd9Sstevel@tonic-gate sasl_digest_md5_bind_2(
738*7c478bd9Sstevel@tonic-gate 	LDAP *ld,
739*7c478bd9Sstevel@tonic-gate 	char *user_name,
740*7c478bd9Sstevel@tonic-gate 	struct berval *cred,
741*7c478bd9Sstevel@tonic-gate 	LDAPControl **serverctrls,
742*7c478bd9Sstevel@tonic-gate 	LDAPControl **clientctrls,
743*7c478bd9Sstevel@tonic-gate 	LDAPMessage *result,
744*7c478bd9Sstevel@tonic-gate 	int *msgidp)
745*7c478bd9Sstevel@tonic-gate {
746*7c478bd9Sstevel@tonic-gate 	struct berval	*challenge = NULL;
747*7c478bd9Sstevel@tonic-gate 	struct berval	resp;
748*7c478bd9Sstevel@tonic-gate 	int		errnum;
749*7c478bd9Sstevel@tonic-gate 	char		*digest = NULL;
750*7c478bd9Sstevel@tonic-gate 	int		err;
751*7c478bd9Sstevel@tonic-gate 
752*7c478bd9Sstevel@tonic-gate 	if (ld == NULL || user_name == NULL || cred == NULL ||
753*7c478bd9Sstevel@tonic-gate 	    cred->bv_val == NULL || result == NULL || msgidp == NULL)
754*7c478bd9Sstevel@tonic-gate 		return (LDAP_PARAM_ERROR);
755*7c478bd9Sstevel@tonic-gate 
756*7c478bd9Sstevel@tonic-gate 	if (ld->ld_version < LDAP_VERSION3)
757*7c478bd9Sstevel@tonic-gate 		return (LDAP_PARAM_ERROR);
758*7c478bd9Sstevel@tonic-gate 
759*7c478bd9Sstevel@tonic-gate 	err = ldap_result2error(ld, result, 0);
760*7c478bd9Sstevel@tonic-gate 	if (err != LDAP_SASL_BIND_IN_PROGRESS)
761*7c478bd9Sstevel@tonic-gate 		return (err);
762*7c478bd9Sstevel@tonic-gate 
763*7c478bd9Sstevel@tonic-gate 	if ((err = ldap_parse_sasl_bind_result(ld, result, &challenge, 0))
764*7c478bd9Sstevel@tonic-gate 			!= LDAP_SUCCESS)
765*7c478bd9Sstevel@tonic-gate 		return (err);
766*7c478bd9Sstevel@tonic-gate 	if (challenge == NULL)
767*7c478bd9Sstevel@tonic-gate 		return (LDAP_NO_MEMORY);
768*7c478bd9Sstevel@tonic-gate 
769*7c478bd9Sstevel@tonic-gate 	err = ldap_digest_md5_encode(challenge->bv_val,
770*7c478bd9Sstevel@tonic-gate 			user_name, cred->bv_val, &digest);
771*7c478bd9Sstevel@tonic-gate 	ber_bvfree(challenge);
772*7c478bd9Sstevel@tonic-gate 
773*7c478bd9Sstevel@tonic-gate 	if (err == LDAP_SUCCESS) {
774*7c478bd9Sstevel@tonic-gate 		resp.bv_val = digest;
775*7c478bd9Sstevel@tonic-gate 		resp.bv_len = strlen(digest);
776*7c478bd9Sstevel@tonic-gate 		LDAPDebug(LDAP_DEBUG_TRACE, "SASL reply: %s\n",
777*7c478bd9Sstevel@tonic-gate 			digest, 0, 0);
778*7c478bd9Sstevel@tonic-gate 		err = ldap_sasl_bind(ld, NULL, LDAP_SASL_DIGEST_MD5,
779*7c478bd9Sstevel@tonic-gate 			&resp, serverctrls, clientctrls, msgidp);
780*7c478bd9Sstevel@tonic-gate 		free(digest);
781*7c478bd9Sstevel@tonic-gate 	}
782*7c478bd9Sstevel@tonic-gate 	return (err);
783*7c478bd9Sstevel@tonic-gate }
784*7c478bd9Sstevel@tonic-gate 
ldap_x_sasl_digest_md5_bind(LDAP * ld,char * user_name,struct berval * cred,LDAPControl ** serverctrls,LDAPControl ** clientctrls,struct timeval * timeout,LDAPMessage ** result)785*7c478bd9Sstevel@tonic-gate int ldap_x_sasl_digest_md5_bind(
786*7c478bd9Sstevel@tonic-gate 	LDAP *ld,
787*7c478bd9Sstevel@tonic-gate 	char *user_name,
788*7c478bd9Sstevel@tonic-gate 	struct berval *cred,
789*7c478bd9Sstevel@tonic-gate 	LDAPControl **serverctrls,
790*7c478bd9Sstevel@tonic-gate 	LDAPControl **clientctrls,
791*7c478bd9Sstevel@tonic-gate 	struct timeval *timeout,
792*7c478bd9Sstevel@tonic-gate 	LDAPMessage **result)
793*7c478bd9Sstevel@tonic-gate {
794*7c478bd9Sstevel@tonic-gate 	LDAPMessage	*res = NULL;
795*7c478bd9Sstevel@tonic-gate 	int		msgid;
796*7c478bd9Sstevel@tonic-gate 	int		rc;
797*7c478bd9Sstevel@tonic-gate 
798*7c478bd9Sstevel@tonic-gate 	if (ld == NULL || user_name == NULL || cred == NULL ||
799*7c478bd9Sstevel@tonic-gate 		result == NULL)
800*7c478bd9Sstevel@tonic-gate 		return (LDAP_PARAM_ERROR);
801*7c478bd9Sstevel@tonic-gate 
802*7c478bd9Sstevel@tonic-gate 	if (ld->ld_version < LDAP_VERSION3)
803*7c478bd9Sstevel@tonic-gate 		return (LDAP_PARAM_ERROR);
804*7c478bd9Sstevel@tonic-gate 
805*7c478bd9Sstevel@tonic-gate 	*result = NULL;
806*7c478bd9Sstevel@tonic-gate 
807*7c478bd9Sstevel@tonic-gate 	rc = sasl_digest_md5_bind_1(ld, user_name,
808*7c478bd9Sstevel@tonic-gate 		serverctrls, clientctrls, &msgid);
809*7c478bd9Sstevel@tonic-gate 	if (rc != LDAP_SUCCESS)
810*7c478bd9Sstevel@tonic-gate 		return (rc);
811*7c478bd9Sstevel@tonic-gate 
812*7c478bd9Sstevel@tonic-gate 	rc = ldap_result(ld, msgid, 1, timeout, &res);
813*7c478bd9Sstevel@tonic-gate 	if (rc == -1) {
814*7c478bd9Sstevel@tonic-gate 		if (res != NULL)
815*7c478bd9Sstevel@tonic-gate 			ldap_msgfree(res);
816*7c478bd9Sstevel@tonic-gate 		return (ldap_get_lderrno(ld, NULL, NULL));
817*7c478bd9Sstevel@tonic-gate 	}
818*7c478bd9Sstevel@tonic-gate 	rc = ldap_result2error(ld, res, 0);
819*7c478bd9Sstevel@tonic-gate 	if (rc != LDAP_SASL_BIND_IN_PROGRESS) {
820*7c478bd9Sstevel@tonic-gate 		*result = res;
821*7c478bd9Sstevel@tonic-gate 		return (rc);
822*7c478bd9Sstevel@tonic-gate 	}
823*7c478bd9Sstevel@tonic-gate 
824*7c478bd9Sstevel@tonic-gate 	rc = sasl_digest_md5_bind_2(ld, user_name, cred,
825*7c478bd9Sstevel@tonic-gate 		serverctrls, clientctrls, res, &msgid);
826*7c478bd9Sstevel@tonic-gate 	ldap_msgfree(res);
827*7c478bd9Sstevel@tonic-gate 	res = NULL;
828*7c478bd9Sstevel@tonic-gate 
829*7c478bd9Sstevel@tonic-gate 	if (rc != LDAP_SUCCESS)
830*7c478bd9Sstevel@tonic-gate 		return (rc);
831*7c478bd9Sstevel@tonic-gate 
832*7c478bd9Sstevel@tonic-gate 	rc = ldap_result(ld, msgid, 1, timeout, &res);
833*7c478bd9Sstevel@tonic-gate 	if (rc == -1) {
834*7c478bd9Sstevel@tonic-gate 		if (res != NULL)
835*7c478bd9Sstevel@tonic-gate 			ldap_msgfree(res);
836*7c478bd9Sstevel@tonic-gate 		return (ldap_get_lderrno(ld, NULL, NULL));
837*7c478bd9Sstevel@tonic-gate 	}
838*7c478bd9Sstevel@tonic-gate 	*result = res;
839*7c478bd9Sstevel@tonic-gate 	rc = ldap_result2error(ld, res, 0);
840*7c478bd9Sstevel@tonic-gate 	return (rc);
841*7c478bd9Sstevel@tonic-gate }
842