1da6c28aaSamw /*
2da6c28aaSamw * CDDL HEADER START
3da6c28aaSamw *
4da6c28aaSamw * The contents of this file are subject to the terms of the
5da6c28aaSamw * Common Development and Distribution License (the "License").
6da6c28aaSamw * You may not use this file except in compliance with the License.
7da6c28aaSamw *
8da6c28aaSamw * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9da6c28aaSamw * or http://www.opensolaris.org/os/licensing.
10da6c28aaSamw * See the License for the specific language governing permissions
11da6c28aaSamw * and limitations under the License.
12da6c28aaSamw *
13da6c28aaSamw * When distributing Covered Code, include this CDDL HEADER in each
14da6c28aaSamw * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15da6c28aaSamw * If applicable, add the following below this CDDL HEADER, with the
16da6c28aaSamw * fields enclosed by brackets "[]" replaced with your own identifying
17da6c28aaSamw * information: Portions Copyright [yyyy] [name of copyright owner]
18da6c28aaSamw *
19da6c28aaSamw * CDDL HEADER END
20da6c28aaSamw */
21da6c28aaSamw
22da6c28aaSamw /*
233db3f65cSamw * Copyright 2008 Sun Microsystems, Inc. All rights reserved.
24da6c28aaSamw * Use is subject to license terms.
25*1ed6b69aSGordon Ross * Copyright 2012 Nexenta Systems, Inc. All rights reserved.
26da6c28aaSamw */
27da6c28aaSamw
28da6c28aaSamw #include <sys/md4.h>
29da6c28aaSamw #include <sys/types.h>
30da6c28aaSamw #include <string.h>
31da6c28aaSamw #include <security/cryptoki.h>
32da6c28aaSamw #include <security/pkcs11.h>
33da6c28aaSamw #include <cryptoutil.h>
34da6c28aaSamw #include <smbsrv/libsmb.h>
35da6c28aaSamw
363db3f65cSamw static void smb_initlmkey(unsigned char *keyin, unsigned char *keyout);
37da6c28aaSamw
38*1ed6b69aSGordon Ross /*
39*1ed6b69aSGordon Ross * randomize
40*1ed6b69aSGordon Ross *
41*1ed6b69aSGordon Ross * Randomize the contents of the specified buffer.
42*1ed6b69aSGordon Ross */
43*1ed6b69aSGordon Ross void
randomize(char * data,unsigned len)44*1ed6b69aSGordon Ross randomize(char *data, unsigned len)
45*1ed6b69aSGordon Ross {
46*1ed6b69aSGordon Ross char *p = data;
47*1ed6b69aSGordon Ross
48*1ed6b69aSGordon Ross if (pkcs11_get_random(data, len) == 0)
49*1ed6b69aSGordon Ross return;
50*1ed6b69aSGordon Ross
51*1ed6b69aSGordon Ross /*
52*1ed6b69aSGordon Ross * Implement a "fall back", because current callers
53*1ed6b69aSGordon Ross * don't expect an error from this. In practice,
54*1ed6b69aSGordon Ross * we never use this fall back.
55*1ed6b69aSGordon Ross */
56*1ed6b69aSGordon Ross while (len--) {
57*1ed6b69aSGordon Ross *p++ = (random() >> 24);
58*1ed6b69aSGordon Ross }
59*1ed6b69aSGordon Ross }
60*1ed6b69aSGordon Ross
61da6c28aaSamw /*
62da6c28aaSamw * smb_auth_md4
63da6c28aaSamw *
64da6c28aaSamw * Compute an MD4 digest.
65da6c28aaSamw */
66da6c28aaSamw int
smb_auth_md4(unsigned char * result,unsigned char * input,int length)67da6c28aaSamw smb_auth_md4(unsigned char *result, unsigned char *input, int length)
68da6c28aaSamw {
69da6c28aaSamw MD4_CTX md4_context;
70da6c28aaSamw
71da6c28aaSamw MD4Init(&md4_context);
72da6c28aaSamw MD4Update(&md4_context, input, length);
73da6c28aaSamw MD4Final(result, &md4_context);
74da6c28aaSamw return (SMBAUTH_SUCCESS);
75da6c28aaSamw }
76da6c28aaSamw
77da6c28aaSamw int
smb_auth_hmac_md5(unsigned char * data,int data_len,unsigned char * key,int key_len,unsigned char * digest)78da6c28aaSamw smb_auth_hmac_md5(unsigned char *data,
79da6c28aaSamw int data_len,
80da6c28aaSamw unsigned char *key,
81da6c28aaSamw int key_len,
82da6c28aaSamw unsigned char *digest)
83da6c28aaSamw {
84da6c28aaSamw CK_RV rv;
85da6c28aaSamw CK_MECHANISM mechanism;
86da6c28aaSamw CK_OBJECT_HANDLE hKey;
87da6c28aaSamw CK_SESSION_HANDLE hSession;
883db3f65cSamw CK_ULONG diglen = MD_DIGEST_LEN;
89da6c28aaSamw
90da6c28aaSamw mechanism.mechanism = CKM_MD5_HMAC;
91da6c28aaSamw mechanism.pParameter = 0;
92da6c28aaSamw mechanism.ulParameterLen = 0;
93da6c28aaSamw rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
94da6c28aaSamw if (rv != CKR_OK) {
95da6c28aaSamw return (SMBAUTH_FAILURE);
96da6c28aaSamw }
97da6c28aaSamw
98da6c28aaSamw rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
99da6c28aaSamw key, key_len, &hKey);
100da6c28aaSamw if (rv != CKR_OK) {
101da6c28aaSamw (void) C_CloseSession(hSession);
102da6c28aaSamw return (SMBAUTH_FAILURE);
103da6c28aaSamw }
104da6c28aaSamw
105da6c28aaSamw /* Initialize the digest operation in the session */
106da6c28aaSamw rv = C_SignInit(hSession, &mechanism, hKey);
107da6c28aaSamw if (rv != CKR_OK) {
108da6c28aaSamw (void) C_DestroyObject(hSession, hKey);
109da6c28aaSamw (void) C_CloseSession(hSession);
110da6c28aaSamw return (SMBAUTH_FAILURE);
111da6c28aaSamw }
112da6c28aaSamw rv = C_SignUpdate(hSession, (CK_BYTE_PTR)data, data_len);
113da6c28aaSamw if (rv != CKR_OK) {
114da6c28aaSamw (void) C_DestroyObject(hSession, hKey);
115da6c28aaSamw (void) C_CloseSession(hSession);
116da6c28aaSamw return (SMBAUTH_FAILURE);
117da6c28aaSamw }
118da6c28aaSamw rv = C_SignFinal(hSession, (CK_BYTE_PTR)digest, &diglen);
119da6c28aaSamw if (rv != CKR_OK) {
120da6c28aaSamw (void) C_DestroyObject(hSession, hKey);
121da6c28aaSamw (void) C_CloseSession(hSession);
122da6c28aaSamw return (SMBAUTH_FAILURE);
123da6c28aaSamw }
124da6c28aaSamw (void) C_DestroyObject(hSession, hKey);
125da6c28aaSamw (void) C_CloseSession(hSession);
126da6c28aaSamw if (diglen != MD_DIGEST_LEN) {
127da6c28aaSamw return (SMBAUTH_FAILURE);
128da6c28aaSamw }
129da6c28aaSamw return (SMBAUTH_SUCCESS);
130da6c28aaSamw }
131da6c28aaSamw
132da6c28aaSamw int
smb_auth_DES(unsigned char * Result,int ResultLen,unsigned char * Key,int KeyLen,unsigned char * Data,int DataLen)133da6c28aaSamw smb_auth_DES(unsigned char *Result, int ResultLen,
134da6c28aaSamw unsigned char *Key, int KeyLen,
135da6c28aaSamw unsigned char *Data, int DataLen)
136da6c28aaSamw {
137da6c28aaSamw CK_RV rv;
138da6c28aaSamw CK_MECHANISM mechanism;
139da6c28aaSamw CK_OBJECT_HANDLE hKey;
140da6c28aaSamw CK_SESSION_HANDLE hSession;
141da6c28aaSamw CK_ULONG ciphertext_len;
142da6c28aaSamw uchar_t des_key[8];
143da6c28aaSamw int error = 0;
144da6c28aaSamw int K, D;
145da6c28aaSamw int k, d;
146da6c28aaSamw
147*1ed6b69aSGordon Ross /*
148*1ed6b69aSGordon Ross * Calculate proper number of iterations.
149*1ed6b69aSGordon Ross * Known call cases include:
150*1ed6b69aSGordon Ross * ResultLen=16, KeyLen=14, DataLen=8
151*1ed6b69aSGordon Ross * ResultLen=24, KeyLen=21, DataLen=8
152*1ed6b69aSGordon Ross * ResultLen=16, KeyLen=14, DataLen=16
153*1ed6b69aSGordon Ross */
154da6c28aaSamw K = KeyLen / 7;
155da6c28aaSamw D = DataLen / 8;
156*1ed6b69aSGordon Ross if ((KeyLen % 7) || (DataLen % 8))
157*1ed6b69aSGordon Ross return (EINVAL);
158*1ed6b69aSGordon Ross if (K == 0 || D == 0)
159*1ed6b69aSGordon Ross return (EINVAL);
160*1ed6b69aSGordon Ross if (ResultLen < (K * 8))
161*1ed6b69aSGordon Ross return (EINVAL);
162da6c28aaSamw
163da6c28aaSamw /*
164da6c28aaSamw * Use SUNW convenience function to initialize the cryptoki
165da6c28aaSamw * library, and open a session with a slot that supports
166da6c28aaSamw * the mechanism we plan on using.
167da6c28aaSamw */
168da6c28aaSamw mechanism.mechanism = CKM_DES_ECB;
169da6c28aaSamw mechanism.pParameter = NULL;
170da6c28aaSamw mechanism.ulParameterLen = 0;
171da6c28aaSamw rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
172da6c28aaSamw if (rv != CKR_OK) {
173da6c28aaSamw return (SMBAUTH_FAILURE);
174da6c28aaSamw }
175da6c28aaSamw
176*1ed6b69aSGordon Ross for (d = k = 0; k < K; k++, d++) {
177*1ed6b69aSGordon Ross /* Cycle the input again, as necessary. */
178*1ed6b69aSGordon Ross if (d == D)
179*1ed6b69aSGordon Ross d = 0;
1803db3f65cSamw smb_initlmkey(&Key[k * 7], des_key);
181da6c28aaSamw rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
182da6c28aaSamw des_key, 8, &hKey);
183da6c28aaSamw if (rv != CKR_OK) {
184da6c28aaSamw error = 1;
185da6c28aaSamw goto exit_session;
186da6c28aaSamw }
187da6c28aaSamw /* Initialize the encryption operation in the session */
188da6c28aaSamw rv = C_EncryptInit(hSession, &mechanism, hKey);
189da6c28aaSamw if (rv != CKR_OK) {
190da6c28aaSamw error = 1;
191da6c28aaSamw goto exit_encrypt;
192da6c28aaSamw }
193*1ed6b69aSGordon Ross ciphertext_len = 8;
194*1ed6b69aSGordon Ross
195*1ed6b69aSGordon Ross /* Read in the data and encrypt this portion */
196*1ed6b69aSGordon Ross rv = C_EncryptUpdate(hSession,
197*1ed6b69aSGordon Ross (CK_BYTE_PTR)Data + (d * 8), 8,
198*1ed6b69aSGordon Ross (CK_BYTE_PTR)Result + (k * 8),
199*1ed6b69aSGordon Ross &ciphertext_len);
200*1ed6b69aSGordon Ross if (rv != CKR_OK) {
201*1ed6b69aSGordon Ross error = 1;
202*1ed6b69aSGordon Ross goto exit_encrypt;
203da6c28aaSamw }
204*1ed6b69aSGordon Ross
205da6c28aaSamw (void) C_DestroyObject(hSession, hKey);
206da6c28aaSamw }
207da6c28aaSamw goto exit_session;
208da6c28aaSamw
209da6c28aaSamw exit_encrypt:
210da6c28aaSamw (void) C_DestroyObject(hSession, hKey);
211da6c28aaSamw exit_session:
212da6c28aaSamw (void) C_CloseSession(hSession);
213da6c28aaSamw
214da6c28aaSamw if (error)
215da6c28aaSamw return (SMBAUTH_FAILURE);
216da6c28aaSamw
217da6c28aaSamw return (SMBAUTH_SUCCESS);
218da6c28aaSamw }
219da6c28aaSamw
220da6c28aaSamw /*
2213db3f65cSamw * See "Netlogon Credential Computation" section of MS-NRPC document.
222da6c28aaSamw */
223da6c28aaSamw static void
smb_initlmkey(unsigned char * keyin,unsigned char * keyout)2243db3f65cSamw smb_initlmkey(unsigned char *keyin, unsigned char *keyout)
225da6c28aaSamw {
2263db3f65cSamw int i;
2273db3f65cSamw
2283db3f65cSamw keyout[0] = keyin[0] >> 0x01;
2293db3f65cSamw keyout[1] = ((keyin[0] & 0x01) << 6) | (keyin[1] >> 2);
2303db3f65cSamw keyout[2] = ((keyin[1] & 0x03) << 5) | (keyin[2] >> 3);
2313db3f65cSamw keyout[3] = ((keyin[2] & 0x07) << 4) | (keyin[3] >> 4);
2323db3f65cSamw keyout[4] = ((keyin[3] & 0x0f) << 3) | (keyin[4] >> 5);
2333db3f65cSamw keyout[5] = ((keyin[4] & 0x1f) << 2) | (keyin[5] >> 6);
2343db3f65cSamw keyout[6] = ((keyin[5] & 0x3f) << 1) | (keyin[6] >> 7);
2353db3f65cSamw keyout[7] = keyin[6] & 0x7f;
2363db3f65cSamw
2373db3f65cSamw for (i = 0; i < 8; i++)
2383db3f65cSamw keyout[i] = (keyout[i] << 1) & 0xfe;
239da6c28aaSamw }
240*1ed6b69aSGordon Ross
241*1ed6b69aSGordon Ross /*
242*1ed6b69aSGordon Ross * CKM_RC4
243*1ed6b69aSGordon Ross */
244*1ed6b69aSGordon Ross int
smb_auth_RC4(uchar_t * Result,int ResultLen,uchar_t * Key,int KeyLen,uchar_t * Data,int DataLen)245*1ed6b69aSGordon Ross smb_auth_RC4(uchar_t *Result, int ResultLen,
246*1ed6b69aSGordon Ross uchar_t *Key, int KeyLen,
247*1ed6b69aSGordon Ross uchar_t *Data, int DataLen)
248*1ed6b69aSGordon Ross {
249*1ed6b69aSGordon Ross CK_RV rv;
250*1ed6b69aSGordon Ross CK_MECHANISM mechanism;
251*1ed6b69aSGordon Ross CK_OBJECT_HANDLE hKey;
252*1ed6b69aSGordon Ross CK_SESSION_HANDLE hSession;
253*1ed6b69aSGordon Ross CK_ULONG ciphertext_len;
254*1ed6b69aSGordon Ross int error = SMBAUTH_FAILURE;
255*1ed6b69aSGordon Ross
256*1ed6b69aSGordon Ross /*
257*1ed6b69aSGordon Ross * Use SUNW convenience function to initialize the cryptoki
258*1ed6b69aSGordon Ross * library, and open a session with a slot that supports
259*1ed6b69aSGordon Ross * the mechanism we plan on using.
260*1ed6b69aSGordon Ross */
261*1ed6b69aSGordon Ross mechanism.mechanism = CKM_RC4;
262*1ed6b69aSGordon Ross mechanism.pParameter = NULL;
263*1ed6b69aSGordon Ross mechanism.ulParameterLen = 0;
264*1ed6b69aSGordon Ross rv = SUNW_C_GetMechSession(mechanism.mechanism, &hSession);
265*1ed6b69aSGordon Ross if (rv != CKR_OK) {
266*1ed6b69aSGordon Ross return (SMBAUTH_FAILURE);
267*1ed6b69aSGordon Ross }
268*1ed6b69aSGordon Ross
269*1ed6b69aSGordon Ross rv = SUNW_C_KeyToObject(hSession, mechanism.mechanism,
270*1ed6b69aSGordon Ross Key, KeyLen, &hKey);
271*1ed6b69aSGordon Ross if (rv != CKR_OK)
272*1ed6b69aSGordon Ross goto exit_session;
273*1ed6b69aSGordon Ross
274*1ed6b69aSGordon Ross /* Initialize the encryption operation in the session */
275*1ed6b69aSGordon Ross rv = C_EncryptInit(hSession, &mechanism, hKey);
276*1ed6b69aSGordon Ross if (rv != CKR_OK)
277*1ed6b69aSGordon Ross goto exit_encrypt;
278*1ed6b69aSGordon Ross
279*1ed6b69aSGordon Ross ciphertext_len = ResultLen;
280*1ed6b69aSGordon Ross rv = C_EncryptUpdate(hSession,
281*1ed6b69aSGordon Ross (CK_BYTE_PTR)Data, DataLen,
282*1ed6b69aSGordon Ross (CK_BYTE_PTR)Result, &ciphertext_len);
283*1ed6b69aSGordon Ross if (rv == CKR_OK)
284*1ed6b69aSGordon Ross error = 0;
285*1ed6b69aSGordon Ross
286*1ed6b69aSGordon Ross exit_encrypt:
287*1ed6b69aSGordon Ross (void) C_DestroyObject(hSession, hKey);
288*1ed6b69aSGordon Ross exit_session:
289*1ed6b69aSGordon Ross (void) C_CloseSession(hSession);
290*1ed6b69aSGordon Ross
291*1ed6b69aSGordon Ross return (error);
292*1ed6b69aSGordon Ross }
293