xref: /illumos-gate/usr/src/lib/libsasl/lib/md5.c (revision 7c478bd9)
1 /*
2  * Copyright 2003 Sun Microsystems, Inc.  All rights reserved.
3  * Use is subject to license terms.
4  */
5 #pragma ident	"%Z%%M%	%I%	%E% SMI"
6 
7 /* MD5C.C - RSA Data Security, Inc., MD5 message-digest algorithm
8  */
9 
10 /* Function names changed to avoid namespace collisions: Rob Siemborski */
11 
12 /* Copyright (C) 1991-2, RSA Data Security, Inc. Created 1991. All
13 rights reserved.
14 
15 License to copy and use this software is granted provided that it
16 is identified as the "RSA Data Security, Inc. MD5 Message-Digest
17 Algorithm" in all material mentioning or referencing this software
18 or this function.
19 
20 License is also granted to make and use derivative works provided
21 that such works are identified as "derived from the RSA Data
22 Security, Inc. MD5 Message-Digest Algorithm" in all material
23 mentioning or referencing the derived work.
24 
25 RSA Data Security, Inc. makes no representations concerning either
26 the merchantability of this software or the suitability of this
27 software for any particular purpose. It is provided "as is"
28 without express or implied warranty of any kind.
29 
30 These notices must be retained in any copies of any part of this
31 documentation and/or software.
32 */
33 
34 #include <config.h>
35 #include "md5global.h"
36 #ifdef _HAVE_LIB_MD5
37 /*
38  * If libmd5 is available, we will use it.
39  * sasl_hmac_md5* functions are still needed.
40  */
41 #include "md5_private.h"
42 #else
43 #include "md5.h"
44 #endif /* _HAVE_LIB_MD5 */
45 #include "hmac-md5.h"
46 
47 #ifndef WIN32
48 #include <arpa/inet.h>
49 #endif
50 
51 /* Constants for MD5Transform routine.
52 */
53 
54 #define S11 7
55 #define S12 12
56 #define S13 17
57 #define S14 22
58 #define S21 5
59 #define S22 9
60 #define S23 14
61 #define S24 20
62 #define S31 4
63 #define S32 11
64 #define S33 16
65 #define S34 23
66 #define S41 6
67 #define S42 10
68 #define S43 15
69 #define S44 21
70 
71 #ifdef _HAVE_LIB_MD5
72 #define MD5_memcpy(s1, s2, n) memcpy(s1, s2, n)
73 #define MD5_memset(s1, c, n) memset(s1, c, n)
74 #else
75 static void MD5Transform PROTO_LIST ((UINT4 [4], unsigned char [64]));
76 static void Encode PROTO_LIST
77        ((unsigned char *, UINT4 *, unsigned int));
78 static void Decode PROTO_LIST
79        ((UINT4 *, unsigned char *, unsigned int));
80 static void MD5_memcpy PROTO_LIST ((POINTER, POINTER, unsigned int));
81 static void MD5_memset PROTO_LIST ((POINTER, int, unsigned int));
82 
83 static unsigned char PADDING[64] = {
84        0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
85        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
86 };
87 #endif /* _HAVE_LIB_MD5 */
88 
89 /* F, G, H and I are basic MD5 functions.
90 
91         */
92 #ifdef I
93 /* This might be defined via NANA */
94 #undef I
95 #endif
96 
97 #define F(x, y, z) (((x) & (y)) | ((~x) & (z)))
98 #define G(x, y, z) (((x) & (z)) | ((y) & (~z)))
99 #define H(x, y, z) ((x) ^ (y) ^ (z))
100 #define I(x, y, z) ((y) ^ ((x) | (~z)))
101 
102 /* ROTATE_LEFT rotates x left n bits.
103 
104         */
105 
106 #define ROTATE_LEFT(x, n) (((x) << (n)) | ((x) >> (32-(n))))
107 
108 /* FF, GG, HH, and II transformations for rounds 1, 2, 3, and 4.
109 Rotation is separate from addition to prevent recomputation.
110 */
111 
112 #define FF(a, b, c, d, x, s, ac) { (a) += F ((b), (c), (d)) + (x) + (UINT4)(ac); (a) = ROTATE_LEFT ((a), (s));        (a) += (b);        }
113 #define GG(a, b, c, d, x, s, ac) {        (a) += G ((b), (c), (d)) + (x) + (UINT4)(ac);        (a) = ROTATE_LEFT ((a), (s));        (a) += (b);         }
114 #define HH(a, b, c, d, x, s, ac) {        (a) += H ((b), (c), (d)) + (x) + (UINT4)(ac);        (a) = ROTATE_LEFT ((a), (s));        (a) += (b);        }
115 #define II(a, b, c, d, x, s, ac) {        (a) += I ((b), (c), (d)) + (x) + (UINT4)(ac);        (a) = ROTATE_LEFT ((a), (s));        (a) += (b);        }
116 
117 /* MD5 initialization. Begins an MD5 operation, writing a new context.
118 */
119 
120 #ifndef _HAVE_LIB_MD5
_sasl_MD5Init(context)121 void _sasl_MD5Init (context)
122 MD5_CTX *context; /* context */
123 {
124        context->count[0] = context->count[1] = 0;
125 
126        /* Load magic initialization constants. */
127        context->state[0] = 0x67452301;
128        context->state[1] = 0xefcdab89;
129        context->state[2] = 0x98badcfe;
130        context->state[3] = 0x10325476;
131 }
132 
133 /* MD5 block update operation. Continues an MD5 message-digest
134        operation, processing another message block, and updating the context.
135 */
136 
_sasl_MD5Update(context,input,inputLen)137 void _sasl_MD5Update (context, input, inputLen)
138 MD5_CTX *context; /* context */
139 unsigned char *input; /* input block */
140 unsigned int inputLen; /* length of input block */
141 {
142        unsigned int i, index, partLen;
143 
144          /* Compute number of bytes mod 64 */
145          index = (unsigned int)((context->count[0] >> 3) & 0x3F);
146 
147          /* Update number of bits */
148          if ((context->count[0] += ((UINT4)inputLen << 3))
149           < ((UINT4)inputLen << 3))
150         context->count[1]++;
151          context->count[1] += ((UINT4)inputLen >> 29);
152 
153        partLen = 64 - index;
154 
155          /* Transform as many times as possible.
156 
157 */
158        if (inputLen >= partLen) {
159        MD5_memcpy
160        ((POINTER)&context->buffer[index], (POINTER)input, partLen); MD5Transform
161        (context->state, context->buffer);
162 
163        for (i = partLen; i + 63 < inputLen; i += 64)
164        MD5Transform (context->state, &input[i]);
165 
166        index = 0;
167        }
168        else
169        i = 0;
170 
171          /* Buffer remaining input */
172          MD5_memcpy
173         ((POINTER)&context->buffer[index], (POINTER)&input[i],
174          inputLen-i);
175 
176 }
177 
178 /* MD5 finalization. Ends an MD5 message-digest operation, writing the
179        the message digest and zeroizing the context.
180 */
181 
_sasl_MD5Final(digest,context)182 void _sasl_MD5Final (digest, context)
183 unsigned char digest[16]; /* message digest */
184 MD5_CTX *context; /* context */
185 {
186        unsigned char bits[8];
187        unsigned int index, padLen;
188 
189          /* Save number of bits */
190          Encode (bits, context->count, 8);
191 
192          /* Pad out to 56 mod 64. */
193 	 index = (unsigned int)((context->count[0] >> 3) & 0x3f);
194 	 padLen = (index < 56) ? (56 - index) : (120 - index);
195 	 _sasl_MD5Update (context, PADDING, padLen);
196 
197          /* Append length (before padding) */
198          _sasl_MD5Update (context, bits, 8);
199 
200          /* Store state in digest */
201          Encode (digest, context->state, 16);
202 
203          /* Zeroize sensitive information. */
204        MD5_memset ((POINTER)context, 0, sizeof (*context));
205 }
206 
207 /* MD5 basic transformation. Transforms state based on block. */
208 
MD5Transform(state,block)209 static void MD5Transform (state, block)
210 UINT4 state[4];
211 unsigned char block[64];
212 {
213        UINT4 a = state[0], b = state[1], c = state[2], d = state[3], x[16];
214 
215        Decode (x, block, 64);
216 
217          /* Round 1 */
218          FF (a, b, c, d, x[ 0], S11, 0xd76aa478); /* 1 */
219          FF (d, a, b, c, x[ 1], S12, 0xe8c7b756); /* 2 */
220          FF (c, d, a, b, x[ 2], S13, 0x242070db); /* 3 */
221          FF (b, c, d, a, x[ 3], S14, 0xc1bdceee); /* 4 */
222          FF (a, b, c, d, x[ 4], S11, 0xf57c0faf); /* 5 */
223          FF (d, a, b, c, x[ 5], S12, 0x4787c62a); /* 6 */
224          FF (c, d, a, b, x[ 6], S13, 0xa8304613); /* 7 */
225          FF (b, c, d, a, x[ 7], S14, 0xfd469501); /* 8 */
226          FF (a, b, c, d, x[ 8], S11, 0x698098d8); /* 9 */
227          FF (d, a, b, c, x[ 9], S12, 0x8b44f7af); /* 10 */
228          FF (c, d, a, b, x[10], S13, 0xffff5bb1); /* 11 */
229          FF (b, c, d, a, x[11], S14, 0x895cd7be); /* 12 */
230          FF (a, b, c, d, x[12], S11, 0x6b901122); /* 13 */
231          FF (d, a, b, c, x[13], S12, 0xfd987193); /* 14 */
232          FF (c, d, a, b, x[14], S13, 0xa679438e); /* 15 */
233          FF (b, c, d, a, x[15], S14, 0x49b40821); /* 16 */
234 
235         /* Round 2 */
236          GG (a, b, c, d, x[ 1], S21, 0xf61e2562); /* 17 */
237          GG (d, a, b, c, x[ 6], S22, 0xc040b340); /* 18 */
238          GG (c, d, a, b, x[11], S23, 0x265e5a51); /* 19 */
239          GG (b, c, d, a, x[ 0], S24, 0xe9b6c7aa); /* 20 */
240          GG (a, b, c, d, x[ 5], S21, 0xd62f105d); /* 21 */
241          GG (d, a, b, c, x[10], S22,  0x2441453); /* 22 */
242          GG (c, d, a, b, x[15], S23, 0xd8a1e681); /* 23 */
243          GG (b, c, d, a, x[ 4], S24, 0xe7d3fbc8); /* 24 */
244          GG (a, b, c, d, x[ 9], S21, 0x21e1cde6); /* 25 */
245          GG (d, a, b, c, x[14], S22, 0xc33707d6); /* 26 */
246          GG (c, d, a, b, x[ 3], S23, 0xf4d50d87); /* 27 */
247 	 GG (b, c, d, a, x[ 8], S24, 0x455a14ed); /* 28 */
248 	 GG (a, b, c, d, x[13], S21, 0xa9e3e905); /* 29 */
249 	 GG (d, a, b, c, x[ 2], S22, 0xfcefa3f8); /* 30 */
250 	 GG (c, d, a, b, x[ 7], S23, 0x676f02d9); /* 31 */
251 	 GG (b, c, d, a, x[12], S24, 0x8d2a4c8a); /* 32 */
252 
253          /* Round 3 */
254          HH (a, b, c, d, x[ 5], S31, 0xfffa3942); /* 33 */
255          HH (d, a, b, c, x[ 8], S32, 0x8771f681); /* 34 */
256          HH (c, d, a, b, x[11], S33, 0x6d9d6122); /* 35 */
257          HH (b, c, d, a, x[14], S34, 0xfde5380c); /* 36 */
258          HH (a, b, c, d, x[ 1], S31, 0xa4beea44); /* 37 */
259          HH (d, a, b, c, x[ 4], S32, 0x4bdecfa9); /* 38 */
260          HH (c, d, a, b, x[ 7], S33, 0xf6bb4b60); /* 39 */
261          HH (b, c, d, a, x[10], S34, 0xbebfbc70); /* 40 */
262          HH (a, b, c, d, x[13], S31, 0x289b7ec6); /* 41 */
263          HH (d, a, b, c, x[ 0], S32, 0xeaa127fa); /* 42 */
264          HH (c, d, a, b, x[ 3], S33, 0xd4ef3085); /* 43 */
265          HH (b, c, d, a, x[ 6], S34,  0x4881d05); /* 44 */
266          HH (a, b, c, d, x[ 9], S31, 0xd9d4d039); /* 45 */
267          HH (d, a, b, c, x[12], S32, 0xe6db99e5); /* 46 */
268          HH (c, d, a, b, x[15], S33, 0x1fa27cf8); /* 47 */
269          HH (b, c, d, a, x[ 2], S34, 0xc4ac5665); /* 48 */
270 
271          /* Round 4 */
272          II (a, b, c, d, x[ 0], S41, 0xf4292244); /* 49 */
273          II (d, a, b, c, x[ 7], S42, 0x432aff97); /* 50 */
274          II (c, d, a, b, x[14], S43, 0xab9423a7); /* 51 */
275          II (b, c, d, a, x[ 5], S44, 0xfc93a039); /* 52 */
276          II (a, b, c, d, x[12], S41, 0x655b59c3); /* 53 */
277          II (d, a, b, c, x[ 3], S42, 0x8f0ccc92); /* 54 */
278          II (c, d, a, b, x[10], S43, 0xffeff47d); /* 55 */
279          II (b, c, d, a, x[ 1], S44, 0x85845dd1); /* 56 */
280          II (a, b, c, d, x[ 8], S41, 0x6fa87e4f); /* 57 */
281          II (d, a, b, c, x[15], S42, 0xfe2ce6e0); /* 58 */
282          II (c, d, a, b, x[ 6], S43, 0xa3014314); /* 59 */
283          II (b, c, d, a, x[13], S44, 0x4e0811a1); /* 60 */
284          II (a, b, c, d, x[ 4], S41, 0xf7537e82); /* 61 */
285          II (d, a, b, c, x[11], S42, 0xbd3af235); /* 62 */
286          II (c, d, a, b, x[ 2], S43, 0x2ad7d2bb); /* 63 */
287          II (b, c, d, a, x[ 9], S44, 0xeb86d391); /* 64 */
288 
289        state[0] += a;
290        state[1] += b;
291        state[2] += c;
292        state[3] += d;
293 
294          /* Zeroize sensitive information.
295 	 */
296        MD5_memset ((POINTER)x, 0, sizeof (x));
297 }
298 
299 /* Encodes input (UINT4) into output (unsigned char). Assumes len is
300        a multiple of 4.
301 
302         */
303 
Encode(output,input,len)304 static void Encode (output, input, len)
305 unsigned char *output;
306 UINT4 *input;
307 unsigned int len;
308 {
309        unsigned int i, j;
310 
311        for (i = 0, j = 0; j < len; i++, j += 4) {
312        output[j] = (unsigned char)(input[i] & 0xff);
313        output[j+1] = (unsigned char)((input[i] >> 8) & 0xff);
314        output[j+2] = (unsigned char)((input[i] >> 16) & 0xff);
315        output[j+3] = (unsigned char)((input[i] >> 24) & 0xff);
316        }
317 }
318 
319 /* Decodes input (unsigned char) into output (UINT4). Assumes len is
320        a multiple of 4.
321 
322         */
323 
Decode(output,input,len)324 static void Decode (output, input, len)
325 UINT4 *output;
326 unsigned char *input;
327 unsigned int len;
328 {
329        unsigned int i, j;
330 
331        for (i = 0, j = 0; j < len; i++, j += 4)
332        output[i] = ((UINT4)input[j]) | (((UINT4)input[j+1]) << 8) | (((UINT4)input[j+2]) << 16)
333        | (((UINT4)input[j+3]) << 24);
334 }
335 
336 /* Note: Replace "for loop" with standard memcpy if possible.
337 
338         */
339 
MD5_memcpy(output,input,len)340 static void MD5_memcpy (output, input, len)
341 POINTER output;
342 POINTER input;
343 unsigned int len;
344 {
345        unsigned int i;
346 
347        for (i = 0; i < len; i++)
348 	      output[i] = input[i];
349 }
350 
351 /* Note: Replace "for loop" with standard memset if possible.
352 */
353 
MD5_memset(output,value,len)354 static void MD5_memset (output, value, len)
355 POINTER output;
356 int value;
357 unsigned int len;
358 {
359        unsigned int i;
360 
361        for (i = 0; i < len; i++)
362        ((char *)output)[i] = (char)value;
363 }
364 #endif /* !_HAVE_LIB_MD5 */
365 
_sasl_hmac_md5_init(HMAC_MD5_CTX * hmac,const unsigned char * key,int key_len)366 void _sasl_hmac_md5_init(HMAC_MD5_CTX *hmac,
367 			 const unsigned char *key,
368 			 int key_len)
369 {
370   unsigned char k_ipad[65];    /* inner padding -
371 				* key XORd with ipad
372 				*/
373   unsigned char k_opad[65];    /* outer padding -
374 				* key XORd with opad
375 				*/
376   unsigned char tk[16];
377   int i;
378   /* if key is longer than 64 bytes reset it to key=MD5(key) */
379   if (key_len > 64) {
380 
381     MD5_CTX      tctx;
382 
383     _sasl_MD5Init(&tctx);
384     _sasl_MD5Update(&tctx, key, key_len);
385     _sasl_MD5Final(tk, &tctx);
386 
387     key = tk;
388     key_len = 16;
389   }
390 
391   /*
392    * the HMAC_MD5 transform looks like:
393    *
394    * MD5(K XOR opad, MD5(K XOR ipad, text))
395    *
396    * where K is an n byte key
397    * ipad is the byte 0x36 repeated 64 times
398    * opad is the byte 0x5c repeated 64 times
399    * and text is the data being protected
400    */
401 
402   /* start out by storing key in pads */
403   MD5_memset(k_ipad, '\0', sizeof k_ipad);
404   MD5_memset(k_opad, '\0', sizeof k_opad);
405   MD5_memcpy( k_ipad, key, key_len);
406   MD5_memcpy( k_opad, key, key_len);
407 
408   /* XOR key with ipad and opad values */
409   for (i=0; i<64; i++) {
410     k_ipad[i] ^= 0x36;
411     k_opad[i] ^= 0x5c;
412   }
413 
414   _sasl_MD5Init(&hmac->ictx);                   /* init inner context */
415   _sasl_MD5Update(&hmac->ictx, k_ipad, 64);     /* apply inner pad */
416 
417   _sasl_MD5Init(&hmac->octx);                   /* init outer context */
418   _sasl_MD5Update(&hmac->octx, k_opad, 64);     /* apply outer pad */
419 
420   /* scrub the pads and key context (if used) */
421   MD5_memset(&k_ipad, 0, sizeof(k_ipad));
422   MD5_memset(&k_opad, 0, sizeof(k_opad));
423   MD5_memset(&tk, 0, sizeof(tk));
424 
425   /* and we're done. */
426 }
427 
428 /* The precalc and import routines here rely on the fact that we pad
429  * the key out to 64 bytes and use that to initialize the md5
430  * contexts, and that updating an md5 context with 64 bytes of data
431  * leaves nothing left over; all of the interesting state is contained
432  * in the state field, and none of it is left over in the count and
433  * buffer fields.  So all we have to do is save the state field; we
434  * can zero the others when we reload it.  Which is why the decision
435  * was made to pad the key out to 64 bytes in the first place. */
_sasl_hmac_md5_precalc(HMAC_MD5_STATE * state,const unsigned char * key,int key_len)436 void _sasl_hmac_md5_precalc(HMAC_MD5_STATE *state,
437 			    const unsigned char *key,
438 			    int key_len)
439 {
440   HMAC_MD5_CTX hmac;
441   unsigned lupe;
442 
443   _sasl_hmac_md5_init(&hmac, key, key_len);
444   for (lupe = 0; lupe < 4; lupe++) {
445     state->istate[lupe] = htonl(hmac.ictx.state[lupe]);
446     state->ostate[lupe] = htonl(hmac.octx.state[lupe]);
447   }
448   MD5_memset(&hmac, 0, sizeof(hmac));
449 }
450 
451 
_sasl_hmac_md5_import(HMAC_MD5_CTX * hmac,HMAC_MD5_STATE * state)452 void _sasl_hmac_md5_import(HMAC_MD5_CTX *hmac,
453 		     HMAC_MD5_STATE *state)
454 {
455   unsigned lupe;
456   MD5_memset(hmac, 0, sizeof(HMAC_MD5_CTX));
457   for (lupe = 0; lupe < 4; lupe++) {
458     hmac->ictx.state[lupe] = ntohl(state->istate[lupe]);
459     hmac->octx.state[lupe] = ntohl(state->ostate[lupe]);
460   }
461   /* Init the counts to account for our having applied
462    * 64 bytes of key; this works out to 0x200 (64 << 3; see
463    * MD5Update above...) */
464   hmac->ictx.count[0] = hmac->octx.count[0] = 0x200;
465 }
466 
_sasl_hmac_md5_final(unsigned char digest[HMAC_MD5_SIZE],HMAC_MD5_CTX * hmac)467 void _sasl_hmac_md5_final(unsigned char digest[HMAC_MD5_SIZE],
468 			  HMAC_MD5_CTX *hmac)
469 {
470   _sasl_MD5Final(digest, &hmac->ictx);  /* Finalize inner md5 */
471   _sasl_MD5Update(&hmac->octx, digest, 16); /* Update outer ctx */
472   _sasl_MD5Final(digest, &hmac->octx); /* Finalize outer md5 */
473 }
474 
475 
_sasl_hmac_md5(text,text_len,key,key_len,digest)476 void _sasl_hmac_md5(text, text_len, key, key_len, digest)
477 const unsigned char* text; /* pointer to data stream */
478 int text_len; /* length of data stream */
479 const unsigned char* key; /* pointer to authentication key */
480 int key_len; /* length of authentication key */
481 unsigned char *digest; /* caller digest to be filled in */
482 {
483   MD5_CTX context;
484 
485   unsigned char k_ipad[65];    /* inner padding -
486 				* key XORd with ipad
487 				*/
488   unsigned char k_opad[65];    /* outer padding -
489 				* key XORd with opad
490 				*/
491   unsigned char tk[16];
492   int i;
493   /* if key is longer than 64 bytes reset it to key=MD5(key) */
494   if (key_len > 64) {
495 
496     MD5_CTX      tctx;
497 
498     _sasl_MD5Init(&tctx);
499     _sasl_MD5Update(&tctx, key, key_len);
500     _sasl_MD5Final(tk, &tctx);
501 
502     key = tk;
503     key_len = 16;
504   }
505 
506   /*
507    * the HMAC_MD5 transform looks like:
508    *
509    * MD5(K XOR opad, MD5(K XOR ipad, text))
510    *
511    * where K is an n byte key
512    * ipad is the byte 0x36 repeated 64 times
513    * opad is the byte 0x5c repeated 64 times
514    * and text is the data being protected
515    */
516 
517   /* start out by storing key in pads */
518   MD5_memset(k_ipad, '\0', sizeof k_ipad);
519   MD5_memset(k_opad, '\0', sizeof k_opad);
520   MD5_memcpy( k_ipad, key, key_len);
521   MD5_memcpy( k_opad, key, key_len);
522 
523   /* XOR key with ipad and opad values */
524   for (i=0; i<64; i++) {
525     k_ipad[i] ^= 0x36;
526     k_opad[i] ^= 0x5c;
527   }
528   /*
529    * perform inner MD5
530    */
531 
532   _sasl_MD5Init(&context);                   /* init context for 1st
533 					       * pass */
534   _sasl_MD5Update(&context, k_ipad, 64);      /* start with inner pad */
535   _sasl_MD5Update(&context, text, text_len); /* then text of datagram */
536   _sasl_MD5Final(digest, &context);          /* finish up 1st pass */
537 
538   /*
539    * perform outer MD5
540    */
541   _sasl_MD5Init(&context);                   /* init context for 2nd
542 					* pass */
543   _sasl_MD5Update(&context, k_opad, 64);     /* start with outer pad */
544   _sasl_MD5Update(&context, digest, 16);     /* then results of 1st
545 					* hash */
546   _sasl_MD5Final(digest, &context);          /* finish up 2nd pass */
547 
548 }
549