xref: /illumos-gate/usr/src/lib/smbsrv/libsmb/common/smb_auth.c (revision c8ec8eea9849cac239663c46be8a7f5d2ba7ca00)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"@(#)smb_auth.c	1.5	08/07/08 SMI"
27 
28 #include <strings.h>
29 #include <stdlib.h>
30 #include <smbsrv/codepage.h>
31 #include <smbsrv/oem.h>
32 #include <smbsrv/ctype.h>
33 #include <smbsrv/crypt.h>
34 #include <smbsrv/libsmb.h>
35 
36 extern void randomize(char *data, unsigned len);
37 static uint64_t unix_micro_to_nt_time(struct timeval *unix_time);
38 
39 /*
40  * smb_auth_qnd_unicode
41  *
42  * Quick and dirty unicode conversion!
43  * Returns the length of dst in bytes.
44  */
45 int
46 smb_auth_qnd_unicode(mts_wchar_t *dst, char *src, int length)
47 {
48 	int i;
49 
50 	unsigned int cpid = oem_get_telnet_cpid();
51 	unsigned int count;
52 	mts_wchar_t new_char;
53 
54 	if ((count = oemstounicodes(dst, src, length, cpid)) == 0) {
55 		for (i = 0; i < length; ++i) {
56 			new_char = (mts_wchar_t)src[i] & 0xff;
57 			dst[i] = LE_IN16(&new_char);
58 		}
59 		dst[i] = 0;
60 		count = length;
61 	}
62 
63 	return (count * sizeof (mts_wchar_t));
64 }
65 
66 /*
67  * smb_auth_lmupr
68  *
69  * Converts the given LM password to all uppercase.
70  * The standard strupr cannot
71  * be used here because lm_pwd doesn't have to be
72  * nul terminated.
73  */
74 static void
75 smb_auth_lmupr(unsigned char *lm_pwd)
76 {
77 	unsigned char *p = lm_pwd;
78 	int i;
79 
80 	for (i = 0; (*p) && (i < SMBAUTH_LM_PWD_SZ); i++) {
81 		if (mts_isascii(*p)) {
82 			*p = codepage_toupper(*p);
83 			p++;
84 		}
85 	}
86 }
87 
88 /*
89  * smb_auth_lm_hash
90  *
91  * Source: Implementing CIFS (Chris Hertel)
92  *
93  * 1. The password, as entered by user, is either padded with nulls
94  *	  or trimmed to 14 bytes.
95  *    . Note that the 14-byte result string is not handled as a
96  *	    nul-terminated string.
97  *	  . The given password is OEM not Unicode
98  *
99  * 2. The 14-byte password is converted to all uppercase
100  *
101  * 3. The result is used as key to encrypt the KGS magic string to
102  *    make a 16-byte hash.
103  */
104 int
105 smb_auth_lm_hash(char *password, unsigned char *lm_hash)
106 {
107 	unsigned char lm_pwd[SMBAUTH_LM_PWD_SZ];
108 
109 	bzero((void *)lm_pwd, SMBAUTH_LM_PWD_SZ);
110 	(void) strncpy((char *)lm_pwd, password, SMBAUTH_LM_PWD_SZ);
111 	smb_auth_lmupr(lm_pwd);
112 
113 	return (smb_auth_DES(lm_hash, SMBAUTH_HASH_SZ, lm_pwd,
114 	    SMBAUTH_LM_PWD_SZ, (unsigned char *)SMBAUTH_LM_MAGIC_STR,
115 	    sizeof (SMBAUTH_LM_MAGIC_STR)));
116 }
117 
118 /*
119  * smb_auth_lm_response
120  *
121  * Create a LM response from the given LM hash and challenge.
122  *
123  * Returns SMBAUTH_FAILURE if any problems occur, SMBAUTH_SUCCESS if
124  * all goes well.
125  */
126 static int
127 smb_auth_lm_response(unsigned char *hash,
128     unsigned char *challenge, int clen,
129     unsigned char *lm_rsp)
130 {
131 	unsigned char S21[21];
132 
133 	/*
134 	 * 14-byte LM Hash should be padded with 5 nul bytes to create
135 	 * a 21-byte string to be used in producing LM response
136 	 */
137 	bzero(&S21[SMBAUTH_HASH_SZ], 5);
138 	bcopy(hash, S21, SMBAUTH_HASH_SZ);
139 
140 	/* padded LM Hash -> LM Response */
141 	return (smb_auth_DES(lm_rsp, SMBAUTH_LM_RESP_SZ, S21, 21,
142 	    challenge, clen));
143 }
144 
145 /*
146  * smb_auth_ntlm_hash
147  *
148  * Make NTLM Hash (using MD4) from the given password.
149  * The result will contain a 16-byte NTLM hash.
150  */
151 int
152 smb_auth_ntlm_hash(char *password, unsigned char *hash)
153 {
154 	mts_wchar_t *unicode_password;
155 	int length;
156 	int rc;
157 
158 	if (password == NULL || hash == NULL)
159 		return (SMBAUTH_FAILURE);
160 
161 	length = strlen(password);
162 	unicode_password = (mts_wchar_t *)
163 	    malloc((length + 1) * sizeof (mts_wchar_t));
164 
165 	if (unicode_password == NULL)
166 		return (SMBAUTH_FAILURE);
167 
168 	length = smb_auth_qnd_unicode(unicode_password, password, length);
169 	rc = smb_auth_md4(hash, (unsigned char *)unicode_password, length);
170 
171 	free(unicode_password);
172 	return (rc);
173 }
174 
175 /*
176  * smb_auth_ntlm_response
177  *
178  * Make LM/NTLM response from the given LM/NTLM Hash and given
179  * challenge.
180  */
181 static int
182 smb_auth_ntlm_response(unsigned char *hash,
183     unsigned char *challenge, int clen,
184     unsigned char *ntlm_rsp)
185 {
186 	unsigned char S21[21];
187 
188 	bcopy(hash, S21, SMBAUTH_HASH_SZ);
189 	bzero(&S21[SMBAUTH_HASH_SZ], 5);
190 	if (smb_auth_DES((unsigned char *)ntlm_rsp, SMBAUTH_LM_RESP_SZ,
191 	    S21, 21, challenge, clen) == SMBAUTH_FAILURE)
192 		return (0);
193 	return (SMBAUTH_LM_RESP_SZ);
194 }
195 
196 /*
197  * smb_auth_gen_data_blob
198  *
199  * Fill the NTLMv2 data blob structure with information as described in
200  * "Implementing CIFS, The Common Internet File System". (pg. 282)
201  */
202 static void
203 smb_auth_gen_data_blob(smb_auth_data_blob_t *blob, char *ntdomain)
204 {
205 	struct timeval now;
206 
207 	(void) memset(blob->ndb_signature, 1, 2);
208 	(void) memset(&blob->ndb_signature[2], 0, 2);
209 	(void) memset(blob->ndb_reserved, 0, sizeof (blob->ndb_reserved));
210 
211 	(void) gettimeofday(&now, 0);
212 	blob->ndb_timestamp = unix_micro_to_nt_time(&now);
213 	randomize((char *)blob->ndb_clnt_challenge,
214 	    SMBAUTH_V2_CLNT_CHALLENGE_SZ);
215 	(void) memset(blob->ndb_unknown, 0, sizeof (blob->ndb_unknown));
216 	blob->ndb_names[0].nne_len = smb_auth_qnd_unicode(
217 	    blob->ndb_names[0].nne_name, ntdomain, strlen(ntdomain));
218 	blob->ndb_names[0].nne_type = SMBAUTH_NAME_TYPE_DOMAIN_NETBIOS;
219 	blob->ndb_names[1].nne_len = 0;
220 	blob->ndb_names[1].nne_type = SMBAUTH_NAME_TYPE_LIST_END;
221 	*blob->ndb_names[1].nne_name = 0;
222 	(void) memset(blob->ndb_unknown2, 0, sizeof (blob->ndb_unknown2));
223 }
224 
225 /*
226  * smb_auth_memcpy
227  *
228  * It increments the pointer to the destination buffer for the easy of
229  * concatenation.
230  */
231 static void
232 smb_auth_memcpy(unsigned char **dstbuf,
233 	unsigned char *srcbuf,
234 	int srcbuf_len)
235 {
236 	(void) memcpy(*dstbuf, srcbuf, srcbuf_len);
237 	*dstbuf += srcbuf_len;
238 }
239 
240 /*
241  * smb_auth_blob_to_string
242  *
243  * Prepare the data blob string which will be used in NTLMv2 response
244  * generation.
245  *
246  * Assumption: Caller must allocate big enough buffer to prevent buffer
247  * overrun.
248  *
249  * Returns the len of the data blob string.
250  */
251 static int
252 smb_auth_blob_to_string(smb_auth_data_blob_t *blob, unsigned char *data_blob)
253 {
254 	unsigned char *bufp = data_blob;
255 
256 	smb_auth_memcpy(&bufp, blob->ndb_signature,
257 	    sizeof (blob->ndb_signature));
258 	smb_auth_memcpy(&bufp, blob->ndb_reserved,
259 	    sizeof (blob->ndb_reserved));
260 	smb_auth_memcpy(&bufp, (unsigned char *)&blob->ndb_timestamp,
261 	    sizeof (blob->ndb_timestamp));
262 	smb_auth_memcpy(&bufp, blob->ndb_clnt_challenge,
263 	    SMBAUTH_V2_CLNT_CHALLENGE_SZ);
264 	smb_auth_memcpy(&bufp, blob->ndb_unknown, sizeof (blob->ndb_unknown));
265 	smb_auth_memcpy(&bufp, (unsigned char *)&blob->ndb_names[0].nne_type,
266 	    sizeof (blob->ndb_names[0].nne_type));
267 	smb_auth_memcpy(&bufp, (unsigned char *)&blob->ndb_names[0].nne_len,
268 	    sizeof (blob->ndb_names[0].nne_len));
269 	smb_auth_memcpy(&bufp, (unsigned char *)blob->ndb_names[0].nne_name,
270 	    blob->ndb_names[0].nne_len);
271 	smb_auth_memcpy(&bufp, (unsigned char *)&blob->ndb_names[1].nne_type,
272 	    sizeof (blob->ndb_names[1].nne_type));
273 	smb_auth_memcpy(&bufp, (unsigned char *)&blob->ndb_names[1].nne_len,
274 	    sizeof (blob->ndb_names[1].nne_len));
275 	smb_auth_memcpy(&bufp, blob->ndb_unknown2, sizeof (blob->ndb_unknown2));
276 
277 	/*LINTED E_PTRDIFF_OVERFLOW*/
278 	return (bufp - data_blob);
279 }
280 
281 /*
282  * smb_auth_ntlmv2_hash
283  *
284  * The NTLM v2 hash will be created from the given NTLM hash, username,
285  * and the NETBIOS name of the domain.
286  *
287  * The NTLMv2 hash will be returned via the ntlmv2_hash parameter which
288  * will be used in the calculation of the NTLMv2 and LMv2 responses.
289  */
290 int
291 smb_auth_ntlmv2_hash(unsigned char *ntlm_hash,
292     char *username,
293     char *ntdomain,
294     unsigned char *ntlmv2_hash)
295 {
296 	mts_wchar_t *data;
297 	int data_len;
298 	unsigned char *buf;
299 	int rc;
300 
301 	if (username == NULL || ntdomain == NULL)
302 		return (SMBAUTH_FAILURE);
303 
304 	(void) utf8_strupr(username);
305 
306 	data_len = strlen(username) + strlen(ntdomain);
307 	buf = (unsigned char *)malloc((data_len + 1) * sizeof (char));
308 	if (buf == NULL)
309 		return (SMBAUTH_FAILURE);
310 
311 	(void) snprintf((char *)buf, data_len + 1, "%s%s", username, ntdomain);
312 	data = (mts_wchar_t *)malloc((data_len + 1) * sizeof (mts_wchar_t));
313 	if (data == NULL) {
314 		free(buf);
315 		return (SMBAUTH_FAILURE);
316 	}
317 
318 	data_len = smb_auth_qnd_unicode(data, (char *)buf, data_len);
319 	rc = SMBAUTH_HMACT64((unsigned char *)data, data_len, ntlm_hash,
320 	    SMBAUTH_HASH_SZ, ntlmv2_hash);
321 
322 	free(buf);
323 	free(data);
324 	return (rc);
325 }
326 
327 /*
328  * smb_auth_v2_response
329  *
330  * Caculates either the LMv2 or NTLMv2 response.
331  *
332  * Same algorithm is used for calculating both LMv2 or NTLMv2 responses.
333  * This routine will return NTLMv2 response if the data blob information
334  * is passed in as the clnt_data. Otherwise, it will return LMv2 response
335  * with the 8-byte client challenge(a.k.a blip) as the clnt_data.
336  *
337  * (LM/NTLM)v2 response is the hmac-md5 hash of the specified data
338  * (server challenge + NTLMv2 data blob or LMv2 client challenge)
339  * using the NTLMv2 hash as the key.
340  *
341  * Returns the size of the corresponding v2 response upon success.
342  * Otherwise, returns -1 on error.
343  */
344 static int
345 smb_auth_v2_response(
346 	unsigned char *hash,
347 	unsigned char *srv_challenge, int slen,
348 	unsigned char *clnt_data, int clen,
349 	unsigned char *v2_rsp)
350 {
351 	unsigned char *hmac_data;
352 
353 	hmac_data = (unsigned char *)malloc((slen + clen) * sizeof (char));
354 	if (!hmac_data) {
355 		return (-1);
356 	}
357 
358 	(void) memcpy(hmac_data, srv_challenge, slen);
359 	(void) memcpy(&hmac_data[slen], clnt_data, clen);
360 	if (SMBAUTH_HMACT64(hmac_data, slen + clen, (unsigned char *)hash,
361 	    SMBAUTH_HASH_SZ, (unsigned char *)v2_rsp) != SMBAUTH_SUCCESS)
362 		return (-1);
363 	(void) memcpy(&v2_rsp[SMBAUTH_HASH_SZ], clnt_data, clen);
364 
365 	free(hmac_data);
366 	return (SMBAUTH_HASH_SZ + clen);
367 }
368 
369 /*
370  * smb_auth_set_info
371  *
372  * Fill the smb_auth_info instance with either NTLM or NTLMv2 related
373  * authentication information based on the LMCompatibilityLevel.
374  *
375  * If the LMCompatibilityLevel equals 2, the SMB Redirector will perform
376  * NTLM challenge/response authentication which requires the NTLM hash and
377  * NTLM response.
378  *
379  * If the LMCompatibilityLevel is 3 or above, the SMB Redirector will
380  * perfrom NTLMv2 challenge/response authenticatoin which requires the
381  * NTLM hash, NTLMv2 hash, NTLMv2 response and LMv2 response.
382  *
383  * Returns -1 on error. Otherwise, returns 0 upon success.
384  */
385 int
386 smb_auth_set_info(char *username,
387 	char *password,
388 	unsigned char *ntlm_hash,
389 	char *domain,
390 	unsigned char *srv_challenge_key,
391 	int srv_challenge_len,
392 	int lmcomp_lvl,
393 	smb_auth_info_t *auth)
394 {
395 	unsigned short blob_len;
396 	unsigned char blob_buf[SMBAUTH_BLOB_MAXLEN];
397 	int rc;
398 	char *uppercase_dom;
399 
400 	auth->lmcompatibility_lvl = lmcomp_lvl;
401 	if (lmcomp_lvl == 2) {
402 		auth->ci_len = 0;
403 		*auth->ci = 0;
404 		if (!ntlm_hash) {
405 			if (smb_auth_ntlm_hash(password, auth->hash) !=
406 			    SMBAUTH_SUCCESS)
407 				return (-1);
408 		} else {
409 			(void) memcpy(auth->hash, ntlm_hash, SMBAUTH_HASH_SZ);
410 		}
411 
412 		auth->cs_len = smb_auth_ntlm_response(auth->hash,
413 		    srv_challenge_key, srv_challenge_len, auth->cs);
414 	} else {
415 		if (!ntlm_hash) {
416 			if (smb_auth_ntlm_hash(password, auth->hash) !=
417 			    SMBAUTH_SUCCESS)
418 				return (-1);
419 		} else {
420 			(void) memcpy(auth->hash, ntlm_hash, SMBAUTH_HASH_SZ);
421 		}
422 
423 		if (!domain)
424 			return (-1);
425 
426 		if ((uppercase_dom = strdup(domain)) == NULL)
427 			return (-1);
428 
429 		(void) utf8_strupr(uppercase_dom);
430 
431 		if (smb_auth_ntlmv2_hash(auth->hash, username,
432 		    uppercase_dom, auth->hash_v2) != SMBAUTH_SUCCESS) {
433 			free(uppercase_dom);
434 			return (-1);
435 		}
436 
437 		/* generate data blob */
438 		smb_auth_gen_data_blob(&auth->data_blob, uppercase_dom);
439 		free(uppercase_dom);
440 		blob_len = smb_auth_blob_to_string(&auth->data_blob, blob_buf);
441 
442 		/* generate NTLMv2 response */
443 		rc = smb_auth_v2_response(auth->hash_v2, srv_challenge_key,
444 		    srv_challenge_len, blob_buf, blob_len, auth->cs);
445 
446 		if (rc < 0)
447 			return (-1);
448 
449 		auth->cs_len = rc;
450 
451 		/* generate LMv2 response */
452 		rc = smb_auth_v2_response(auth->hash_v2, srv_challenge_key,
453 		    srv_challenge_len, auth->data_blob.ndb_clnt_challenge,
454 		    SMBAUTH_V2_CLNT_CHALLENGE_SZ, auth->ci);
455 
456 		if (rc < 0)
457 			return (-1);
458 
459 		auth->ci_len = rc;
460 	}
461 
462 	return (0);
463 }
464 
465 /*
466  * smb_auth_gen_session_key
467  *
468  * Generate the NTLM user session key if LMCompatibilityLevel is 2 or
469  * NTLMv2 user session key if LMCompatibilityLevel is 3 or above.
470  *
471  * NTLM_Session_Key = MD4(NTLM_Hash);
472  *
473  * NTLMv2_Session_Key = HMAC_MD5(NTLMv2Hash, 16, NTLMv2_HMAC, 16)
474  *
475  * Prior to calling this function, the auth instance should be set
476  * via smb_auth_set_info().
477  *
478  * Returns the appropriate session key.
479  */
480 int
481 smb_auth_gen_session_key(smb_auth_info_t *auth, unsigned char *session_key)
482 {
483 	int rc;
484 
485 	if (auth->lmcompatibility_lvl == 2)
486 		rc = smb_auth_md4(session_key, auth->hash, SMBAUTH_HASH_SZ);
487 	else
488 		rc = SMBAUTH_HMACT64((unsigned char *)auth->cs,
489 		    SMBAUTH_HASH_SZ, (unsigned char *)auth->hash_v2,
490 		    SMBAUTH_SESSION_KEY_SZ, session_key);
491 
492 	return (rc);
493 }
494 
495 /* 100's of ns between 1/1/1970 and 1/1/1601 */
496 #define	NT_TIME_BIAS    (134774LL * 24LL * 60LL * 60LL * 10000000LL)
497 
498 static uint64_t
499 unix_micro_to_nt_time(struct timeval *unix_time)
500 {
501 	uint64_t nt_time;
502 
503 	nt_time = unix_time->tv_sec;
504 	nt_time *= 10000000;  /* seconds to 100ns */
505 	nt_time += unix_time->tv_usec * 10;
506 	return (nt_time + NT_TIME_BIAS);
507 }
508 
509 static boolean_t
510 smb_lm_password_ok(
511     unsigned char *challenge,
512     uint32_t clen,
513     unsigned char *lm_hash,
514     unsigned char *passwd)
515 {
516 	unsigned char lm_resp[SMBAUTH_LM_RESP_SZ];
517 	int rc;
518 
519 	rc = smb_auth_lm_response(lm_hash, challenge, clen, lm_resp);
520 	if (rc != SMBAUTH_SUCCESS)
521 		return (B_FALSE);
522 
523 	return (bcmp(lm_resp, passwd, SMBAUTH_LM_RESP_SZ) == 0);
524 }
525 
526 static boolean_t
527 smb_ntlm_password_ok(
528     unsigned char *challenge,
529     uint32_t clen,
530     unsigned char *ntlm_hash,
531     unsigned char *passwd,
532     unsigned char *session_key)
533 {
534 	unsigned char ntlm_resp[SMBAUTH_LM_RESP_SZ];
535 	int rc;
536 	boolean_t ok;
537 
538 	rc = smb_auth_ntlm_response(ntlm_hash, challenge, clen, ntlm_resp);
539 	if (rc != SMBAUTH_LM_RESP_SZ)
540 		return (B_FALSE);
541 
542 	ok = (bcmp(ntlm_resp, passwd, SMBAUTH_LM_RESP_SZ) == 0);
543 	if (ok && (session_key)) {
544 		rc = smb_auth_md4(session_key, ntlm_hash, SMBAUTH_HASH_SZ);
545 		if (rc != SMBAUTH_SUCCESS)
546 			ok = B_FALSE;
547 	}
548 	return (ok);
549 }
550 
551 static boolean_t
552 smb_ntlmv2_password_ok(
553     unsigned char *challenge,
554     uint32_t clen,
555     unsigned char *ntlm_hash,
556     unsigned char *passwd,
557     int pwdlen,
558     char *domain,
559     char *username,
560     uchar_t *session_key)
561 {
562 	unsigned char *clnt_blob;
563 	int clnt_blob_len;
564 	unsigned char ntlmv2_hash[SMBAUTH_HASH_SZ];
565 	unsigned char *ntlmv2_resp;
566 	boolean_t ok = B_FALSE;
567 	char *dest[3];
568 	int i;
569 	int rc;
570 
571 	clnt_blob_len = pwdlen - SMBAUTH_HASH_SZ;
572 	clnt_blob = &passwd[SMBAUTH_HASH_SZ];
573 	dest[0] = domain;
574 	if ((dest[1] = strdup(domain)) == NULL)
575 		return (B_FALSE);
576 	(void) utf8_strupr(dest[1]);
577 	dest[2] = "";
578 
579 	/*
580 	 * 15.5.2 The NTLMv2 Password Hash, pg. 279, of the "Implementing CIFS"
581 	 *
582 	 * The NTLMv2 Hash is created from:
583 	 * - NTLM hash
584 	 * - user's username, and
585 	 * - the name of the logon destination(i.e. the NetBIOS name of either
586 	 *   the SMB server or NT Domain against which the user is trying to
587 	 *   authenticate.
588 	 *
589 	 * Experiments show this is not exactly the case.
590 	 * For Windows Server 2003, the domain name needs to be included and
591 	 * converted to uppercase. For Vista, the domain name needs to be
592 	 * included also, but leave the case alone.  And in some cases it needs
593 	 * to be empty. All three variants are tried here.
594 	 */
595 
596 	ntlmv2_resp = (unsigned char *)malloc(SMBAUTH_HASH_SZ + clnt_blob_len);
597 	if (ntlmv2_resp == NULL) {
598 		free(dest[1]);
599 		return (B_FALSE);
600 	}
601 
602 	for (i = 0; i < (sizeof (dest) / sizeof (char *)); i++) {
603 		if (smb_auth_ntlmv2_hash(ntlm_hash, username, dest[i],
604 		    ntlmv2_hash) != SMBAUTH_SUCCESS)
605 			break;
606 
607 		if (smb_auth_v2_response(ntlmv2_hash, challenge,
608 		    clen, clnt_blob, clnt_blob_len, ntlmv2_resp) < 0)
609 			break;
610 
611 		ok = (bcmp(passwd, ntlmv2_resp, pwdlen) == 0);
612 		if (ok && session_key) {
613 			rc = SMBAUTH_HMACT64(ntlmv2_resp,
614 			    SMBAUTH_HASH_SZ, ntlmv2_hash,
615 			    SMBAUTH_SESSION_KEY_SZ, session_key);
616 			if (rc != SMBAUTH_SUCCESS) {
617 				ok = B_FALSE;
618 			}
619 			break;
620 		}
621 	}
622 
623 	free(dest[1]);
624 	free(ntlmv2_resp);
625 	return (ok);
626 }
627 
628 static boolean_t
629 smb_lmv2_password_ok(
630     unsigned char *challenge,
631     uint32_t clen,
632     unsigned char *ntlm_hash,
633     unsigned char *passwd,
634     char *domain,
635     char *username)
636 {
637 	unsigned char *clnt_challenge;
638 	unsigned char ntlmv2_hash[SMBAUTH_HASH_SZ];
639 	unsigned char lmv2_resp[SMBAUTH_LM_RESP_SZ];
640 	boolean_t ok = B_FALSE;
641 	char *dest[3];
642 	int i;
643 
644 	clnt_challenge = &passwd[SMBAUTH_HASH_SZ];
645 	dest[0] = domain;
646 	if ((dest[1] = strdup(domain)) == NULL)
647 		return (B_FALSE);
648 	(void) utf8_strupr(dest[1]);
649 	dest[2] = "";
650 
651 	/*
652 	 * 15.5.2 The NTLMv2 Password Hash, pg. 279, of the "Implementing CIFS"
653 	 *
654 	 * The NTLMv2 Hash is created from:
655 	 * - NTLM hash
656 	 * - user's username, and
657 	 * - the name of the logon destination(i.e. the NetBIOS name of either
658 	 *   the SMB server or NT Domain against which the suer is trying to
659 	 *   authenticate.
660 	 *
661 	 * Experiments show this is not exactly the case.
662 	 * For Windows Server 2003, the domain name needs to be included and
663 	 * converted to uppercase. For Vista, the domain name needs to be
664 	 * included also, but leave the case alone.  And in some cases it needs
665 	 * to be empty. All three variants are tried here.
666 	 */
667 
668 	for (i = 0; i < (sizeof (dest) / sizeof (char *)); i++) {
669 		if (smb_auth_ntlmv2_hash(ntlm_hash, username, dest[i],
670 		    ntlmv2_hash) != SMBAUTH_SUCCESS)
671 			break;
672 
673 		if (smb_auth_v2_response(ntlmv2_hash, challenge,
674 		    clen, clnt_challenge, SMBAUTH_V2_CLNT_CHALLENGE_SZ,
675 		    lmv2_resp) < 0)
676 			break;
677 
678 		ok = (bcmp(passwd, lmv2_resp, SMBAUTH_LM_RESP_SZ) == 0);
679 		if (ok)
680 			break;
681 	}
682 
683 	free(dest[1]);
684 	return (ok);
685 }
686 
687 /*
688  * smb_auth_validate_lm
689  *
690  * Validates given LM/LMv2 client response, passed in passwd arg, against
691  * stored user's password, passed in smbpw
692  *
693  * If LM level <=3 server accepts LM responses, otherwise LMv2
694  */
695 boolean_t
696 smb_auth_validate_lm(
697     unsigned char *challenge,
698     uint32_t clen,
699     smb_passwd_t *smbpw,
700     unsigned char *passwd,
701     int pwdlen,
702     char *domain,
703     char *username)
704 {
705 	boolean_t ok = B_FALSE;
706 	int64_t lmlevel;
707 
708 	if (pwdlen != SMBAUTH_LM_RESP_SZ)
709 		return (B_FALSE);
710 
711 	if (smb_config_getnum(SMB_CI_LM_LEVEL, &lmlevel) != SMBD_SMF_OK)
712 		return (B_FALSE);
713 
714 	if (lmlevel <= 3) {
715 		ok = smb_lm_password_ok(challenge, clen, smbpw->pw_lmhash,
716 		    passwd);
717 	}
718 
719 	if (!ok)
720 		ok = smb_lmv2_password_ok(challenge, clen, smbpw->pw_nthash,
721 		    passwd, domain, username);
722 
723 	return (ok);
724 }
725 
726 /*
727  * smb_auth_validate_nt
728  *
729  * Validates given NTLM/NTLMv2 client response, passed in passwd arg, against
730  * stored user's password, passed in smbpw
731  *
732  * If LM level <=4 server accepts NTLM/NTLMv2 responses, otherwise only NTLMv2
733  */
734 boolean_t
735 smb_auth_validate_nt(
736     unsigned char *challenge,
737     uint32_t clen,
738     smb_passwd_t *smbpw,
739     unsigned char *passwd,
740     int pwdlen,
741     char *domain,
742     char *username,
743     uchar_t *session_key)
744 {
745 	int64_t lmlevel;
746 	boolean_t ok;
747 
748 	if (smb_config_getnum(SMB_CI_LM_LEVEL, &lmlevel) != SMBD_SMF_OK)
749 		return (B_FALSE);
750 
751 	if ((lmlevel == 5) && (pwdlen <= SMBAUTH_LM_RESP_SZ))
752 		return (B_FALSE);
753 
754 	if (pwdlen > SMBAUTH_LM_RESP_SZ)
755 		ok = smb_ntlmv2_password_ok(challenge, clen,
756 		    smbpw->pw_nthash, passwd, pwdlen,
757 		    domain, username, session_key);
758 	else
759 		ok = smb_ntlm_password_ok(challenge, clen,
760 		    smbpw->pw_nthash, passwd, session_key);
761 
762 	return (ok);
763 }
764