1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2012 Nexenta Systems, Inc.  All rights reserved.
26 */
27
28#include <sys/md4.h>
29#include <sys/types.h>
30#include <string.h>
31#include <security/cryptoki.h>
32#include <security/pkcs11.h>
33#include <cryptoutil.h>
34#include <smbsrv/libsmb.h>
35
36static void smb_initlmkey(unsigned char *keyin, unsigned char *keyout);
37
38/*
39 * randomize
40 *
41 * Randomize the contents of the specified buffer.
42 */
43void
44randomize(char *data, unsigned len)
45{
46	char *p = data;
47
48	if (pkcs11_get_random(data, len) == 0)
49		return;
50
51	/*
52	 * Implement a "fall back", because current callers
53	 * don't expect an error from this.  In practice,
54	 * we never use this fall back.
55	 */
56	while (len--) {
57		*p++ = (random() >> 24);
58	}
59}
60
61/*
62 * smb_auth_md4
63 *
64 * Compute an MD4 digest.
65 */
66int
67smb_auth_md4(unsigned char *result, unsigned char *input, int length)
68{
69	MD4_CTX md4_context;
70
71	MD4Init(&md4_context);
72	MD4Update(&md4_context, input, length);
73	MD4Final(result, &md4_context);
74	return (SMBAUTH_SUCCESS);
75}
76
77int
78smb_auth_hmac_md5(unsigned char *data,
79	int data_len,
80	unsigned char *key,
81	int key_len,
82	unsigned char *digest)
83{
84	CK_RV rv;
85	CK_MECHANISM mechanism;
86	CK_OBJECT_HANDLE hKey;
87	CK_SESSION_HANDLE hSession;
88	CK_ULONG diglen = MD_DIGEST_LEN;
89
90	mechanism.mechanism = CKM_MD5_HMAC;
91	mechanism.pParameter = 0;
92	mechanism.ulParameterLen = 0;
93	rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
94	if (rv != CKR_OK) {
95		return (SMBAUTH_FAILURE);
96	}
97
98	rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
99	    key, key_len, &hKey);
100	if (rv != CKR_OK) {
101		(void) C_CloseSession(hSession);
102		return (SMBAUTH_FAILURE);
103	}
104
105	/* Initialize the digest operation in the session */
106	rv = C_SignInit(hSession, &mechanism, hKey);
107	if (rv != CKR_OK) {
108		(void) C_DestroyObject(hSession, hKey);
109		(void) C_CloseSession(hSession);
110		return (SMBAUTH_FAILURE);
111	}
112	rv = C_SignUpdate(hSession, (CK_BYTE_PTR)data, data_len);
113	if (rv != CKR_OK) {
114		(void) C_DestroyObject(hSession, hKey);
115		(void) C_CloseSession(hSession);
116		return (SMBAUTH_FAILURE);
117	}
118	rv = C_SignFinal(hSession, (CK_BYTE_PTR)digest, &diglen);
119	if (rv != CKR_OK) {
120		(void) C_DestroyObject(hSession, hKey);
121		(void) C_CloseSession(hSession);
122		return (SMBAUTH_FAILURE);
123	}
124	(void) C_DestroyObject(hSession, hKey);
125	(void) C_CloseSession(hSession);
126	if (diglen != MD_DIGEST_LEN) {
127		return (SMBAUTH_FAILURE);
128	}
129	return (SMBAUTH_SUCCESS);
130}
131
132int
133smb_auth_DES(unsigned char *Result, int ResultLen,
134    unsigned char *Key, int KeyLen,
135    unsigned char *Data, int DataLen)
136{
137	CK_RV rv;
138	CK_MECHANISM mechanism;
139	CK_OBJECT_HANDLE hKey;
140	CK_SESSION_HANDLE hSession;
141	CK_ULONG ciphertext_len;
142	uchar_t des_key[8];
143	int error = 0;
144	int K, D;
145	int k, d;
146
147	/*
148	 * Calculate proper number of iterations.
149	 * Known call cases include:
150	 *   ResultLen=16, KeyLen=14, DataLen=8
151	 *   ResultLen=24, KeyLen=21, DataLen=8
152	 *   ResultLen=16, KeyLen=14, DataLen=16
153	 */
154	K = KeyLen / 7;
155	D = DataLen / 8;
156	if ((KeyLen % 7) || (DataLen % 8))
157		return (EINVAL);
158	if (K == 0 || D == 0)
159		return (EINVAL);
160	if (ResultLen < (K * 8))
161		return (EINVAL);
162
163	/*
164	 * Use SUNW convenience function to initialize the cryptoki
165	 * library, and open a session with a slot that supports
166	 * the mechanism we plan on using.
167	 */
168	mechanism.mechanism = CKM_DES_ECB;
169	mechanism.pParameter = NULL;
170	mechanism.ulParameterLen = 0;
171	rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
172	if (rv != CKR_OK) {
173		return (SMBAUTH_FAILURE);
174	}
175
176	for (d = k = 0; k < K; k++, d++) {
177		/* Cycle the input again, as necessary. */
178		if (d == D)
179			d = 0;
180		smb_initlmkey(&Key[k * 7], des_key);
181		rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
182		    des_key, 8, &hKey);
183		if (rv != CKR_OK) {
184			error = 1;
185			goto exit_session;
186		}
187		/* Initialize the encryption operation in the session */
188		rv = C_EncryptInit(hSession, &mechanism, hKey);
189		if (rv != CKR_OK) {
190			error = 1;
191			goto exit_encrypt;
192		}
193		ciphertext_len = 8;
194
195		/* Read in the data and encrypt this portion */
196		rv = C_EncryptUpdate(hSession,
197		    (CK_BYTE_PTR)Data + (d * 8), 8,
198		    (CK_BYTE_PTR)Result + (k * 8),
199		    &ciphertext_len);
200		if (rv != CKR_OK) {
201			error = 1;
202			goto exit_encrypt;
203		}
204
205		(void) C_DestroyObject(hSession, hKey);
206	}
207	goto exit_session;
208
209exit_encrypt:
210	(void) C_DestroyObject(hSession, hKey);
211exit_session:
212	(void) C_CloseSession(hSession);
213
214	if (error)
215		return (SMBAUTH_FAILURE);
216
217	return (SMBAUTH_SUCCESS);
218}
219
220/*
221 * See "Netlogon Credential Computation" section of MS-NRPC document.
222 */
223static void
224smb_initlmkey(unsigned char *keyin, unsigned char *keyout)
225{
226	int i;
227
228	keyout[0] = keyin[0] >> 0x01;
229	keyout[1] = ((keyin[0] & 0x01) << 6) | (keyin[1] >> 2);
230	keyout[2] = ((keyin[1] & 0x03) << 5) | (keyin[2] >> 3);
231	keyout[3] = ((keyin[2] & 0x07) << 4) | (keyin[3] >> 4);
232	keyout[4] = ((keyin[3] & 0x0f) << 3) | (keyin[4] >> 5);
233	keyout[5] = ((keyin[4] & 0x1f) << 2) | (keyin[5] >> 6);
234	keyout[6] = ((keyin[5] & 0x3f) << 1) | (keyin[6] >> 7);
235	keyout[7] = keyin[6] & 0x7f;
236
237	for (i = 0; i < 8; i++)
238		keyout[i] = (keyout[i] << 1) & 0xfe;
239}
240
241/*
242 * CKM_RC4
243 */
244int
245smb_auth_RC4(uchar_t *Result, int ResultLen,
246	uchar_t *Key, int KeyLen,
247	uchar_t *Data, int DataLen)
248{
249	CK_RV rv;
250	CK_MECHANISM mechanism;
251	CK_OBJECT_HANDLE hKey;
252	CK_SESSION_HANDLE hSession;
253	CK_ULONG ciphertext_len;
254	int error = SMBAUTH_FAILURE;
255
256	/*
257	 * Use SUNW convenience function to initialize the cryptoki
258	 * library, and open a session with a slot that supports
259	 * the mechanism we plan on using.
260	 */
261	mechanism.mechanism = CKM_RC4;
262	mechanism.pParameter = NULL;
263	mechanism.ulParameterLen = 0;
264	rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
265	if (rv != CKR_OK) {
266		return (SMBAUTH_FAILURE);
267	}
268
269	rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
270	    Key, KeyLen, &hKey);
271	if (rv != CKR_OK)
272		goto exit_session;
273
274	/* Initialize the encryption operation in the session */
275	rv = C_EncryptInit(hSession, &mechanism, hKey);
276	if (rv != CKR_OK)
277		goto exit_encrypt;
278
279	ciphertext_len = ResultLen;
280	rv = C_EncryptUpdate(hSession,
281	    (CK_BYTE_PTR)Data, DataLen,
282	    (CK_BYTE_PTR)Result, &ciphertext_len);
283	if (rv == CKR_OK)
284		error = 0;
285
286exit_encrypt:
287	(void) C_DestroyObject(hSession, hKey);
288exit_session:
289	(void) C_CloseSession(hSession);
290
291	return (error);
292}
293