1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
26 */
27
28 #include <sys/md4.h>
29 #include <sys/types.h>
30 #include <string.h>
31 #include <security/cryptoki.h>
32 #include <security/pkcs11.h>
33 #include <cryptoutil.h>
34 #include <smbsrv/libsmb.h>
35
36 static void smb_initlmkey(unsigned char *keyin, unsigned char *keyout);
37
38 /*
39 * randomize
40 *
41 * Randomize the contents of the specified buffer.
42 */
43 void
randomize(char * data,unsigned len)44 randomize(char *data, unsigned len)
45 {
46 char *p = data;
47
48 if (pkcs11_get_random(data, len) == 0)
49 return;
50
51 /*
52 * Implement a "fall back", because current callers
53 * don't expect an error from this. In practice,
54 * we never use this fall back.
55 */
56 while (len--) {
57 *p++ = (random() >> 24);
58 }
59 }
60
61 /*
62 * smb_auth_md4
63 *
64 * Compute an MD4 digest.
65 */
66 int
smb_auth_md4(unsigned char * result,unsigned char * input,int length)67 smb_auth_md4(unsigned char *result, unsigned char *input, int length)
68 {
69 MD4_CTX md4_context;
70
71 MD4Init(&md4_context);
72 MD4Update(&md4_context, input, length);
73 MD4Final(result, &md4_context);
74 return (SMBAUTH_SUCCESS);
75 }
76
77 int
smb_auth_hmac_md5(unsigned char * data,int data_len,unsigned char * key,int key_len,unsigned char * digest)78 smb_auth_hmac_md5(unsigned char *data,
79 int data_len,
80 unsigned char *key,
81 int key_len,
82 unsigned char *digest)
83 {
84 CK_RV rv;
85 CK_MECHANISM mechanism;
86 CK_OBJECT_HANDLE hKey;
87 CK_SESSION_HANDLE hSession;
88 CK_ULONG diglen = MD_DIGEST_LEN;
89
90 mechanism.mechanism = CKM_MD5_HMAC;
91 mechanism.pParameter = 0;
92 mechanism.ulParameterLen = 0;
93 rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
94 if (rv != CKR_OK) {
95 return (SMBAUTH_FAILURE);
96 }
97
98 rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
99 key, key_len, &hKey);
100 if (rv != CKR_OK) {
101 (void) C_CloseSession(hSession);
102 return (SMBAUTH_FAILURE);
103 }
104
105 /* Initialize the digest operation in the session */
106 rv = C_SignInit(hSession, &mechanism, hKey);
107 if (rv != CKR_OK) {
108 (void) C_DestroyObject(hSession, hKey);
109 (void) C_CloseSession(hSession);
110 return (SMBAUTH_FAILURE);
111 }
112 rv = C_SignUpdate(hSession, (CK_BYTE_PTR)data, data_len);
113 if (rv != CKR_OK) {
114 (void) C_DestroyObject(hSession, hKey);
115 (void) C_CloseSession(hSession);
116 return (SMBAUTH_FAILURE);
117 }
118 rv = C_SignFinal(hSession, (CK_BYTE_PTR)digest, &diglen);
119 if (rv != CKR_OK) {
120 (void) C_DestroyObject(hSession, hKey);
121 (void) C_CloseSession(hSession);
122 return (SMBAUTH_FAILURE);
123 }
124 (void) C_DestroyObject(hSession, hKey);
125 (void) C_CloseSession(hSession);
126 if (diglen != MD_DIGEST_LEN) {
127 return (SMBAUTH_FAILURE);
128 }
129 return (SMBAUTH_SUCCESS);
130 }
131
132 int
smb_auth_DES(unsigned char * Result,int ResultLen,unsigned char * Key,int KeyLen,unsigned char * Data,int DataLen)133 smb_auth_DES(unsigned char *Result, int ResultLen,
134 unsigned char *Key, int KeyLen,
135 unsigned char *Data, int DataLen)
136 {
137 CK_RV rv;
138 CK_MECHANISM mechanism;
139 CK_OBJECT_HANDLE hKey;
140 CK_SESSION_HANDLE hSession;
141 CK_ULONG ciphertext_len;
142 uchar_t des_key[8];
143 int error = 0;
144 int K, D;
145 int k, d;
146
147 /*
148 * Calculate proper number of iterations.
149 * Known call cases include:
150 * ResultLen=16, KeyLen=14, DataLen=8
151 * ResultLen=24, KeyLen=21, DataLen=8
152 * ResultLen=16, KeyLen=14, DataLen=16
153 */
154 K = KeyLen / 7;
155 D = DataLen / 8;
156 if ((KeyLen % 7) || (DataLen % 8))
157 return (EINVAL);
158 if (K == 0 || D == 0)
159 return (EINVAL);
160 if (ResultLen < (K * 8))
161 return (EINVAL);
162
163 /*
164 * Use SUNW convenience function to initialize the cryptoki
165 * library, and open a session with a slot that supports
166 * the mechanism we plan on using.
167 */
168 mechanism.mechanism = CKM_DES_ECB;
169 mechanism.pParameter = NULL;
170 mechanism.ulParameterLen = 0;
171 rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
172 if (rv != CKR_OK) {
173 return (SMBAUTH_FAILURE);
174 }
175
176 for (d = k = 0; k < K; k++, d++) {
177 /* Cycle the input again, as necessary. */
178 if (d == D)
179 d = 0;
180 smb_initlmkey(&Key[k * 7], des_key);
181 rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
182 des_key, 8, &hKey);
183 if (rv != CKR_OK) {
184 error = 1;
185 goto exit_session;
186 }
187 /* Initialize the encryption operation in the session */
188 rv = C_EncryptInit(hSession, &mechanism, hKey);
189 if (rv != CKR_OK) {
190 error = 1;
191 goto exit_encrypt;
192 }
193 ciphertext_len = 8;
194
195 /* Read in the data and encrypt this portion */
196 rv = C_EncryptUpdate(hSession,
197 (CK_BYTE_PTR)Data + (d * 8), 8,
198 (CK_BYTE_PTR)Result + (k * 8),
199 &ciphertext_len);
200 if (rv != CKR_OK) {
201 error = 1;
202 goto exit_encrypt;
203 }
204
205 (void) C_DestroyObject(hSession, hKey);
206 }
207 goto exit_session;
208
209 exit_encrypt:
210 (void) C_DestroyObject(hSession, hKey);
211 exit_session:
212 (void) C_CloseSession(hSession);
213
214 if (error)
215 return (SMBAUTH_FAILURE);
216
217 return (SMBAUTH_SUCCESS);
218 }
219
220 /*
221 * See "Netlogon Credential Computation" section of MS-NRPC document.
222 */
223 static void
smb_initlmkey(unsigned char * keyin,unsigned char * keyout)224 smb_initlmkey(unsigned char *keyin, unsigned char *keyout)
225 {
226 int i;
227
228 keyout[0] = keyin[0] >> 0x01;
229 keyout[1] = ((keyin[0] & 0x01) << 6) | (keyin[1] >> 2);
230 keyout[2] = ((keyin[1] & 0x03) << 5) | (keyin[2] >> 3);
231 keyout[3] = ((keyin[2] & 0x07) << 4) | (keyin[3] >> 4);
232 keyout[4] = ((keyin[3] & 0x0f) << 3) | (keyin[4] >> 5);
233 keyout[5] = ((keyin[4] & 0x1f) << 2) | (keyin[5] >> 6);
234 keyout[6] = ((keyin[5] & 0x3f) << 1) | (keyin[6] >> 7);
235 keyout[7] = keyin[6] & 0x7f;
236
237 for (i = 0; i < 8; i++)
238 keyout[i] = (keyout[i] << 1) & 0xfe;
239 }
240
241 /*
242 * CKM_RC4
243 */
244 int
smb_auth_RC4(uchar_t * Result,int ResultLen,uchar_t * Key,int KeyLen,uchar_t * Data,int DataLen)245 smb_auth_RC4(uchar_t *Result, int ResultLen,
246 uchar_t *Key, int KeyLen,
247 uchar_t *Data, int DataLen)
248 {
249 CK_RV rv;
250 CK_MECHANISM mechanism;
251 CK_OBJECT_HANDLE hKey;
252 CK_SESSION_HANDLE hSession;
253 CK_ULONG ciphertext_len;
254 int error = SMBAUTH_FAILURE;
255
256 /*
257 * Use SUNW convenience function to initialize the cryptoki
258 * library, and open a session with a slot that supports
259 * the mechanism we plan on using.
260 */
261 mechanism.mechanism = CKM_RC4;
262 mechanism.pParameter = NULL;
263 mechanism.ulParameterLen = 0;
264 rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
265 if (rv != CKR_OK) {
266 return (SMBAUTH_FAILURE);
267 }
268
269 rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
270 Key, KeyLen, &hKey);
271 if (rv != CKR_OK)
272 goto exit_session;
273
274 /* Initialize the encryption operation in the session */
275 rv = C_EncryptInit(hSession, &mechanism, hKey);
276 if (rv != CKR_OK)
277 goto exit_encrypt;
278
279 ciphertext_len = ResultLen;
280 rv = C_EncryptUpdate(hSession,
281 (CK_BYTE_PTR)Data, DataLen,
282 (CK_BYTE_PTR)Result, &ciphertext_len);
283 if (rv == CKR_OK)
284 error = 0;
285
286 exit_encrypt:
287 (void) C_DestroyObject(hSession, hKey);
288 exit_session:
289 (void) C_CloseSession(hSession);
290
291 return (error);
292 }
293