xref: /illumos-gate/usr/src/uts/common/gssapi/mechs/krb5/crypto/arcfour/k5_arcfour.c (revision 7c478bd95313f5f23a4c958a745db2134aa0324)
1*7c478bd9Sstevel@tonic-gate /*
2*7c478bd9Sstevel@tonic-gate  * Copyright 2003 Sun Microsystems, Inc.  All rights reserved.
3*7c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
4*7c478bd9Sstevel@tonic-gate  */
5*7c478bd9Sstevel@tonic-gate 
6*7c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
7*7c478bd9Sstevel@tonic-gate 
8*7c478bd9Sstevel@tonic-gate /*
9*7c478bd9Sstevel@tonic-gate 
10*7c478bd9Sstevel@tonic-gate ARCFOUR cipher (based on a cipher posted on the Usenet in Spring-95).
11*7c478bd9Sstevel@tonic-gate This cipher is widely believed and has been tested to be equivalent
12*7c478bd9Sstevel@tonic-gate with the RC4 cipher from RSA Data Security, Inc.  (RC4 is a trademark
13*7c478bd9Sstevel@tonic-gate of RSA Data Security)
14*7c478bd9Sstevel@tonic-gate 
15*7c478bd9Sstevel@tonic-gate */
16*7c478bd9Sstevel@tonic-gate #include <k5-int.h>
17*7c478bd9Sstevel@tonic-gate #include <arcfour.h>
18*7c478bd9Sstevel@tonic-gate 
19*7c478bd9Sstevel@tonic-gate /* salt string used  for exportable ARCFOUR */
20*7c478bd9Sstevel@tonic-gate static const  char *l40 = "fortybits";
21*7c478bd9Sstevel@tonic-gate 
22*7c478bd9Sstevel@tonic-gate void
23*7c478bd9Sstevel@tonic-gate krb5_arcfour_encrypt_length(enc, hash, inputlen, length)
24*7c478bd9Sstevel@tonic-gate      const struct krb5_enc_provider *enc;
25*7c478bd9Sstevel@tonic-gate      const struct krb5_hash_provider *hash;
26*7c478bd9Sstevel@tonic-gate      size_t inputlen;
27*7c478bd9Sstevel@tonic-gate      size_t *length;
28*7c478bd9Sstevel@tonic-gate {
29*7c478bd9Sstevel@tonic-gate   size_t blocksize, hashsize;
30*7c478bd9Sstevel@tonic-gate 
31*7c478bd9Sstevel@tonic-gate   (*(enc->block_size))(&blocksize);
32*7c478bd9Sstevel@tonic-gate   (*(hash->hash_size))(&hashsize);
33*7c478bd9Sstevel@tonic-gate 
34*7c478bd9Sstevel@tonic-gate   /* checksum + (confounder + inputlen, in even blocksize) */
35*7c478bd9Sstevel@tonic-gate   *length = hashsize + krb5_roundup(8 + inputlen, blocksize);
36*7c478bd9Sstevel@tonic-gate }
37*7c478bd9Sstevel@tonic-gate 
38*7c478bd9Sstevel@tonic-gate krb5_keyusage
39*7c478bd9Sstevel@tonic-gate krb5int_arcfour_translate_usage(krb5_keyusage usage)
40*7c478bd9Sstevel@tonic-gate {
41*7c478bd9Sstevel@tonic-gate   switch (usage) {
42*7c478bd9Sstevel@tonic-gate   case 1:			/* AS-REQ PA-ENC-TIMESTAMP padata timestamp,  */
43*7c478bd9Sstevel@tonic-gate     return 1;
44*7c478bd9Sstevel@tonic-gate   case 2:			/* ticket from kdc */
45*7c478bd9Sstevel@tonic-gate     return 2;
46*7c478bd9Sstevel@tonic-gate   case 3:			/* as-rep encrypted part */
47*7c478bd9Sstevel@tonic-gate     return 8;
48*7c478bd9Sstevel@tonic-gate   case 4:			/* tgs-req authz data */
49*7c478bd9Sstevel@tonic-gate     return 4;
50*7c478bd9Sstevel@tonic-gate   case 5:			/* tgs-req authz data in subkey */
51*7c478bd9Sstevel@tonic-gate     return 5;
52*7c478bd9Sstevel@tonic-gate   case 6:			/* tgs-req authenticator cksum */
53*7c478bd9Sstevel@tonic-gate     return 6;
54*7c478bd9Sstevel@tonic-gate   case 7:			/* tgs-req authenticator */
55*7c478bd9Sstevel@tonic-gate     return 7;
56*7c478bd9Sstevel@tonic-gate   case 8:
57*7c478bd9Sstevel@tonic-gate     return 8;
58*7c478bd9Sstevel@tonic-gate   case 9:			/* tgs-rep encrypted with subkey */
59*7c478bd9Sstevel@tonic-gate     return 8;
60*7c478bd9Sstevel@tonic-gate   case 10:			/* ap-rep authentication cksum */
61*7c478bd9Sstevel@tonic-gate     return 10;			/* xxx  Microsoft never uses this*/
62*7c478bd9Sstevel@tonic-gate   case 11:			/* app-req authenticator */
63*7c478bd9Sstevel@tonic-gate     return 11;
64*7c478bd9Sstevel@tonic-gate   case 12:			/* app-rep encrypted part */
65*7c478bd9Sstevel@tonic-gate     return 12;
66*7c478bd9Sstevel@tonic-gate   case 23: /* sign wrap token*/
67*7c478bd9Sstevel@tonic-gate     return 13;
68*7c478bd9Sstevel@tonic-gate   default:
69*7c478bd9Sstevel@tonic-gate       return usage;
70*7c478bd9Sstevel@tonic-gate }
71*7c478bd9Sstevel@tonic-gate }
72*7c478bd9Sstevel@tonic-gate 
73*7c478bd9Sstevel@tonic-gate krb5_error_code
74*7c478bd9Sstevel@tonic-gate krb5_arcfour_encrypt(context, enc, hash, key, usage, ivec, input, output)
75*7c478bd9Sstevel@tonic-gate      krb5_context context;
76*7c478bd9Sstevel@tonic-gate      const struct krb5_enc_provider *enc;
77*7c478bd9Sstevel@tonic-gate      const struct krb5_hash_provider *hash;
78*7c478bd9Sstevel@tonic-gate      const krb5_keyblock *key;
79*7c478bd9Sstevel@tonic-gate      krb5_keyusage usage;
80*7c478bd9Sstevel@tonic-gate      const krb5_data *ivec;
81*7c478bd9Sstevel@tonic-gate      const krb5_data *input;
82*7c478bd9Sstevel@tonic-gate      krb5_data *output;
83*7c478bd9Sstevel@tonic-gate {
84*7c478bd9Sstevel@tonic-gate   krb5_keyblock k1, k2, k3;
85*7c478bd9Sstevel@tonic-gate   krb5_keyblock *kptr;
86*7c478bd9Sstevel@tonic-gate   krb5_data d1, d2, d3, salt, plaintext, checksum, ciphertext, confounder;
87*7c478bd9Sstevel@tonic-gate   krb5_keyusage ms_usage;
88*7c478bd9Sstevel@tonic-gate   size_t keylength, keybytes, blocksize, hashsize;
89*7c478bd9Sstevel@tonic-gate   krb5_error_code ret = 0;
90*7c478bd9Sstevel@tonic-gate 
91*7c478bd9Sstevel@tonic-gate   (*(enc->block_size))(&blocksize);
92*7c478bd9Sstevel@tonic-gate   (*(enc->keysize))(&keybytes, &keylength);
93*7c478bd9Sstevel@tonic-gate   (*(hash->hash_size))(&hashsize);
94*7c478bd9Sstevel@tonic-gate 
95*7c478bd9Sstevel@tonic-gate   bzero(&d2, sizeof(krb5_data));
96*7c478bd9Sstevel@tonic-gate   bzero(&k2, sizeof(krb5_keyblock));
97*7c478bd9Sstevel@tonic-gate   /*
98*7c478bd9Sstevel@tonic-gate    * d1 is the contents buffer for key k1.
99*7c478bd9Sstevel@tonic-gate    * k1  = HMAC(input_key, salt)
100*7c478bd9Sstevel@tonic-gate    */
101*7c478bd9Sstevel@tonic-gate   d1.length=keybytes;
102*7c478bd9Sstevel@tonic-gate   d1.data=MALLOC(d1.length);
103*7c478bd9Sstevel@tonic-gate   if (d1.data == NULL)
104*7c478bd9Sstevel@tonic-gate     return (ENOMEM);
105*7c478bd9Sstevel@tonic-gate   bcopy(key, &k1, sizeof (krb5_keyblock));
106*7c478bd9Sstevel@tonic-gate   k1.length=d1.length;
107*7c478bd9Sstevel@tonic-gate   k1.contents= (void *) d1.data;
108*7c478bd9Sstevel@tonic-gate 
109*7c478bd9Sstevel@tonic-gate   /*
110*7c478bd9Sstevel@tonic-gate    * d2 is the contents of key 'k2', which is used to generate the
111*7c478bd9Sstevel@tonic-gate    * checksum field.  'd2' == 'd1' when not using the exportable
112*7c478bd9Sstevel@tonic-gate    * enctype.  This is only needed when using the exportable
113*7c478bd9Sstevel@tonic-gate    * enctype.
114*7c478bd9Sstevel@tonic-gate    */
115*7c478bd9Sstevel@tonic-gate   if (key->enctype==ENCTYPE_ARCFOUR_HMAC_EXP) {
116*7c478bd9Sstevel@tonic-gate 	d2.length=keybytes;
117*7c478bd9Sstevel@tonic-gate 	d2.data=MALLOC(d2.length);
118*7c478bd9Sstevel@tonic-gate 	if (d2.data == NULL) {
119*7c478bd9Sstevel@tonic-gate 		FREE(d1.data, d1.length);
120*7c478bd9Sstevel@tonic-gate 		return (ENOMEM);
121*7c478bd9Sstevel@tonic-gate 	}
122*7c478bd9Sstevel@tonic-gate 	bcopy(key, &k2, sizeof (krb5_keyblock));
123*7c478bd9Sstevel@tonic-gate 	k2.length=d2.length;
124*7c478bd9Sstevel@tonic-gate 	k2.contents=(void *) d2.data;
125*7c478bd9Sstevel@tonic-gate   }
126*7c478bd9Sstevel@tonic-gate 
127*7c478bd9Sstevel@tonic-gate   /*
128*7c478bd9Sstevel@tonic-gate    * d3 will hold the contents of the final key used for the
129*7c478bd9Sstevel@tonic-gate    * encryption step.  'k3' is the key structure that has 'd3'
130*7c478bd9Sstevel@tonic-gate    * as its 'contents' field.
131*7c478bd9Sstevel@tonic-gate    * k3 = HMAC(k1, checksum)
132*7c478bd9Sstevel@tonic-gate    */
133*7c478bd9Sstevel@tonic-gate   d3.length=keybytes;
134*7c478bd9Sstevel@tonic-gate   d3.data=MALLOC(d3.length);
135*7c478bd9Sstevel@tonic-gate   if (d3.data == NULL) {
136*7c478bd9Sstevel@tonic-gate     FREE(d1.data, d1.length);
137*7c478bd9Sstevel@tonic-gate     if (d2.data)
138*7c478bd9Sstevel@tonic-gate 	FREE(d2.data, d2.length);
139*7c478bd9Sstevel@tonic-gate     return (ENOMEM);
140*7c478bd9Sstevel@tonic-gate   }
141*7c478bd9Sstevel@tonic-gate   bcopy(key, &k3, sizeof (krb5_keyblock));
142*7c478bd9Sstevel@tonic-gate   k3.length=d3.length;
143*7c478bd9Sstevel@tonic-gate   k3.contents= (void *) d3.data;
144*7c478bd9Sstevel@tonic-gate 
145*7c478bd9Sstevel@tonic-gate   salt.length=14;
146*7c478bd9Sstevel@tonic-gate   salt.data=MALLOC(salt.length);
147*7c478bd9Sstevel@tonic-gate 
148*7c478bd9Sstevel@tonic-gate   if (salt.data == NULL) {
149*7c478bd9Sstevel@tonic-gate     FREE(d1.data, d1.length);
150*7c478bd9Sstevel@tonic-gate     if (d2.data)
151*7c478bd9Sstevel@tonic-gate 	FREE(d2.data, d2.length);
152*7c478bd9Sstevel@tonic-gate     FREE(d3.data, d3.length);
153*7c478bd9Sstevel@tonic-gate     return (ENOMEM);
154*7c478bd9Sstevel@tonic-gate   }
155*7c478bd9Sstevel@tonic-gate 
156*7c478bd9Sstevel@tonic-gate   /* is "input" already blocksize aligned?  if it is, then we need this
157*7c478bd9Sstevel@tonic-gate      step, otherwise we do not */
158*7c478bd9Sstevel@tonic-gate   plaintext.length=krb5_roundup(input->length+CONFOUNDERLENGTH,blocksize);
159*7c478bd9Sstevel@tonic-gate   plaintext.data=MALLOC(plaintext.length);
160*7c478bd9Sstevel@tonic-gate 
161*7c478bd9Sstevel@tonic-gate   if (plaintext.data == NULL) {
162*7c478bd9Sstevel@tonic-gate     FREE(d1.data, d1.length);
163*7c478bd9Sstevel@tonic-gate     if (d2.data)
164*7c478bd9Sstevel@tonic-gate 	FREE(d2.data, d2.length);
165*7c478bd9Sstevel@tonic-gate     FREE(d3.data, d3.length);
166*7c478bd9Sstevel@tonic-gate     FREE(salt.data, salt.length);
167*7c478bd9Sstevel@tonic-gate     return(ENOMEM);
168*7c478bd9Sstevel@tonic-gate   }
169*7c478bd9Sstevel@tonic-gate   bzero(plaintext.data, plaintext.length);
170*7c478bd9Sstevel@tonic-gate 
171*7c478bd9Sstevel@tonic-gate   /* setup convienient pointers into the allocated data */
172*7c478bd9Sstevel@tonic-gate   checksum.length=hashsize;
173*7c478bd9Sstevel@tonic-gate   checksum.data=output->data;
174*7c478bd9Sstevel@tonic-gate 
175*7c478bd9Sstevel@tonic-gate   ciphertext.length=krb5_roundup(input->length+CONFOUNDERLENGTH,blocksize);
176*7c478bd9Sstevel@tonic-gate   ciphertext.data=output->data+hashsize;
177*7c478bd9Sstevel@tonic-gate 
178*7c478bd9Sstevel@tonic-gate   confounder.length=CONFOUNDERLENGTH;
179*7c478bd9Sstevel@tonic-gate   confounder.data=plaintext.data;
180*7c478bd9Sstevel@tonic-gate 
181*7c478bd9Sstevel@tonic-gate   output->length = plaintext.length+hashsize;
182*7c478bd9Sstevel@tonic-gate 
183*7c478bd9Sstevel@tonic-gate   /* begin the encryption, computer K1 */
184*7c478bd9Sstevel@tonic-gate   ms_usage=krb5int_arcfour_translate_usage(usage);
185*7c478bd9Sstevel@tonic-gate   if (key->enctype == ENCTYPE_ARCFOUR_HMAC_EXP) {
186*7c478bd9Sstevel@tonic-gate     (void) strncpy(salt.data, l40, salt.length);
187*7c478bd9Sstevel@tonic-gate     salt.data[10]=ms_usage & 0xff;
188*7c478bd9Sstevel@tonic-gate     salt.data[11]=(ms_usage >> 8) & 0xff;
189*7c478bd9Sstevel@tonic-gate     salt.data[12]=(ms_usage >> 16) & 0xff;
190*7c478bd9Sstevel@tonic-gate     salt.data[13]=(ms_usage >> 24) & 0xff;
191*7c478bd9Sstevel@tonic-gate   } else {
192*7c478bd9Sstevel@tonic-gate     salt.length=4;
193*7c478bd9Sstevel@tonic-gate     salt.data[0]=ms_usage & 0xff;
194*7c478bd9Sstevel@tonic-gate     salt.data[1]=(ms_usage >> 8) & 0xff;
195*7c478bd9Sstevel@tonic-gate     salt.data[2]=(ms_usage >> 16) & 0xff;
196*7c478bd9Sstevel@tonic-gate     salt.data[3]=(ms_usage >> 24) & 0xff;
197*7c478bd9Sstevel@tonic-gate   }
198*7c478bd9Sstevel@tonic-gate 
199*7c478bd9Sstevel@tonic-gate #ifdef _KERNEL
200*7c478bd9Sstevel@tonic-gate   ret = krb5_hmac(context, key, &salt, &d1);
201*7c478bd9Sstevel@tonic-gate #else
202*7c478bd9Sstevel@tonic-gate   ret = krb5_hmac(context, hash, key, 1, &salt, &d1);
203*7c478bd9Sstevel@tonic-gate #endif /* _KERNEL */
204*7c478bd9Sstevel@tonic-gate   if (ret != 0)
205*7c478bd9Sstevel@tonic-gate 	goto cleanup;
206*7c478bd9Sstevel@tonic-gate 
207*7c478bd9Sstevel@tonic-gate   if (key->enctype==ENCTYPE_ARCFOUR_HMAC_EXP) {
208*7c478bd9Sstevel@tonic-gate     bcopy(k1.contents, k2.contents, k2.length);
209*7c478bd9Sstevel@tonic-gate     (void) memset(k1.contents+7, 0xab, 9);
210*7c478bd9Sstevel@tonic-gate     kptr = &k2;
211*7c478bd9Sstevel@tonic-gate   } else {
212*7c478bd9Sstevel@tonic-gate     kptr = &k1;
213*7c478bd9Sstevel@tonic-gate   }
214*7c478bd9Sstevel@tonic-gate 
215*7c478bd9Sstevel@tonic-gate   /* create a confounder block */
216*7c478bd9Sstevel@tonic-gate   ret=krb5_c_random_make_octets(context, &confounder);
217*7c478bd9Sstevel@tonic-gate   bcopy(input->data, plaintext.data+confounder.length, input->length);
218*7c478bd9Sstevel@tonic-gate   if (ret)
219*7c478bd9Sstevel@tonic-gate     goto cleanup;
220*7c478bd9Sstevel@tonic-gate 
221*7c478bd9Sstevel@tonic-gate   /*
222*7c478bd9Sstevel@tonic-gate    * Compute the HMAC checksum field.
223*7c478bd9Sstevel@tonic-gate    * checksum = HMAC(k1/k2, plaintext);
224*7c478bd9Sstevel@tonic-gate    *    k2 used when key->enctype==ENCTYPE_ARCFOUR_HMAC_EXP
225*7c478bd9Sstevel@tonic-gate    */
226*7c478bd9Sstevel@tonic-gate #ifdef _KERNEL
227*7c478bd9Sstevel@tonic-gate   ret = krb5_hmac(context, kptr, &plaintext, &checksum);
228*7c478bd9Sstevel@tonic-gate #else
229*7c478bd9Sstevel@tonic-gate   ret = krb5_hmac(context, hash, kptr, 1, &plaintext, &checksum);
230*7c478bd9Sstevel@tonic-gate #endif /* _KERNEL */
231*7c478bd9Sstevel@tonic-gate   if (ret)
232*7c478bd9Sstevel@tonic-gate     goto cleanup;
233*7c478bd9Sstevel@tonic-gate 
234*7c478bd9Sstevel@tonic-gate   /*
235*7c478bd9Sstevel@tonic-gate    * The final encryption key is the HMAC of the checksum
236*7c478bd9Sstevel@tonic-gate    * using k1
237*7c478bd9Sstevel@tonic-gate    *
238*7c478bd9Sstevel@tonic-gate    * k3 = HMAC(k1, checksum);
239*7c478bd9Sstevel@tonic-gate    *  == or (in other terms) ==
240*7c478bd9Sstevel@tonic-gate    * k3 = HMAC((HMAC(input_key,salt), HMAC(k1, plaintext));
241*7c478bd9Sstevel@tonic-gate    */
242*7c478bd9Sstevel@tonic-gate #ifdef _KERNEL
243*7c478bd9Sstevel@tonic-gate   ret = krb5_hmac(context, &k1, &checksum, &d3);
244*7c478bd9Sstevel@tonic-gate #else
245*7c478bd9Sstevel@tonic-gate   ret = krb5_hmac(context, hash, &k1, 1,  &checksum, &d3);
246*7c478bd9Sstevel@tonic-gate #endif /* _KERNEL */
247*7c478bd9Sstevel@tonic-gate   if (ret)
248*7c478bd9Sstevel@tonic-gate     goto cleanup;
249*7c478bd9Sstevel@tonic-gate 
250*7c478bd9Sstevel@tonic-gate   ret = (*(enc->encrypt))(context, &k3, ivec, &plaintext, &ciphertext);
251*7c478bd9Sstevel@tonic-gate 
252*7c478bd9Sstevel@tonic-gate  cleanup:
253*7c478bd9Sstevel@tonic-gate   bzero(d1.data, d1.length);
254*7c478bd9Sstevel@tonic-gate   if (d2.data) {
255*7c478bd9Sstevel@tonic-gate 	bzero(d2.data, d2.length);
256*7c478bd9Sstevel@tonic-gate 	FREE(d2.data, d2.length);
257*7c478bd9Sstevel@tonic-gate   }
258*7c478bd9Sstevel@tonic-gate   bzero(d3.data, d3.length);
259*7c478bd9Sstevel@tonic-gate   bzero(salt.data, salt.length);
260*7c478bd9Sstevel@tonic-gate   bzero(plaintext.data, plaintext.length);
261*7c478bd9Sstevel@tonic-gate 
262*7c478bd9Sstevel@tonic-gate   FREE(d1.data, d1.length);
263*7c478bd9Sstevel@tonic-gate   FREE(d3.data, d3.length);
264*7c478bd9Sstevel@tonic-gate   FREE(salt.data, salt.length);
265*7c478bd9Sstevel@tonic-gate   FREE(plaintext.data, plaintext.length);
266*7c478bd9Sstevel@tonic-gate   return (ret);
267*7c478bd9Sstevel@tonic-gate }
268*7c478bd9Sstevel@tonic-gate 
269*7c478bd9Sstevel@tonic-gate /* This is the arcfour-hmac decryption routine */
270*7c478bd9Sstevel@tonic-gate krb5_error_code
271*7c478bd9Sstevel@tonic-gate krb5_arcfour_decrypt(context, enc, hash, key, usage, ivec, input, output)
272*7c478bd9Sstevel@tonic-gate      krb5_context context;
273*7c478bd9Sstevel@tonic-gate      const struct krb5_enc_provider *enc;
274*7c478bd9Sstevel@tonic-gate      const struct krb5_hash_provider *hash;
275*7c478bd9Sstevel@tonic-gate      const krb5_keyblock *key;
276*7c478bd9Sstevel@tonic-gate      krb5_keyusage usage;
277*7c478bd9Sstevel@tonic-gate      const krb5_data *ivec;
278*7c478bd9Sstevel@tonic-gate      const krb5_data *input;
279*7c478bd9Sstevel@tonic-gate      krb5_data *output;
280*7c478bd9Sstevel@tonic-gate {
281*7c478bd9Sstevel@tonic-gate   krb5_keyblock k1,k2,k3, *kptr;
282*7c478bd9Sstevel@tonic-gate   krb5_data d1,d2,d3,salt,ciphertext,plaintext,checksum;
283*7c478bd9Sstevel@tonic-gate   krb5_keyusage ms_usage;
284*7c478bd9Sstevel@tonic-gate   size_t keybytes, keylength, hashsize, blocksize;
285*7c478bd9Sstevel@tonic-gate   krb5_error_code ret;
286*7c478bd9Sstevel@tonic-gate 
287*7c478bd9Sstevel@tonic-gate   (*(enc->block_size))(&blocksize);
288*7c478bd9Sstevel@tonic-gate   (*(enc->keysize))(&keybytes, &keylength);
289*7c478bd9Sstevel@tonic-gate   (*(hash->hash_size))(&hashsize);
290*7c478bd9Sstevel@tonic-gate 
291*7c478bd9Sstevel@tonic-gate   bzero(&d2, sizeof(krb5_data));
292*7c478bd9Sstevel@tonic-gate   bzero(&k2, sizeof(krb5_keyblock));
293*7c478bd9Sstevel@tonic-gate   /*
294*7c478bd9Sstevel@tonic-gate    * d1 is the contents buffer for key k1.
295*7c478bd9Sstevel@tonic-gate    * k1  = HMAC(input_key, salt)
296*7c478bd9Sstevel@tonic-gate    */
297*7c478bd9Sstevel@tonic-gate   d1.length=keybytes;
298*7c478bd9Sstevel@tonic-gate   d1.data=MALLOC(d1.length);
299*7c478bd9Sstevel@tonic-gate   if (d1.data == NULL)
300*7c478bd9Sstevel@tonic-gate     return (ENOMEM);
301*7c478bd9Sstevel@tonic-gate   (void) bcopy(key, &k1, sizeof (krb5_keyblock));
302*7c478bd9Sstevel@tonic-gate   k1.length=d1.length;
303*7c478bd9Sstevel@tonic-gate   k1.contents= (void *) d1.data;
304*7c478bd9Sstevel@tonic-gate 
305*7c478bd9Sstevel@tonic-gate   /*
306*7c478bd9Sstevel@tonic-gate    * d2 is the contents of key 'k2', which is used to generate the
307*7c478bd9Sstevel@tonic-gate    * checksum field.  'd2' == 'd1' when not using the exportable
308*7c478bd9Sstevel@tonic-gate    * enctype.  This is only needed when using the exportable
309*7c478bd9Sstevel@tonic-gate    * enctype.
310*7c478bd9Sstevel@tonic-gate    */
311*7c478bd9Sstevel@tonic-gate   if (key->enctype==ENCTYPE_ARCFOUR_HMAC_EXP) {
312*7c478bd9Sstevel@tonic-gate 	d2.length=keybytes;
313*7c478bd9Sstevel@tonic-gate 	d2.data=MALLOC(d2.length);
314*7c478bd9Sstevel@tonic-gate 	if (d2.data == NULL) {
315*7c478bd9Sstevel@tonic-gate 		FREE(d1.data, d1.length);
316*7c478bd9Sstevel@tonic-gate 		return (ENOMEM);
317*7c478bd9Sstevel@tonic-gate 	}
318*7c478bd9Sstevel@tonic-gate 	(void) bcopy(key, &k2, sizeof(krb5_keyblock));
319*7c478bd9Sstevel@tonic-gate 	k2.length=d2.length;
320*7c478bd9Sstevel@tonic-gate 	k2.contents= (void *) d2.data;
321*7c478bd9Sstevel@tonic-gate   }
322*7c478bd9Sstevel@tonic-gate 
323*7c478bd9Sstevel@tonic-gate   /*
324*7c478bd9Sstevel@tonic-gate    * d3 will hold the contents of the final key used for the
325*7c478bd9Sstevel@tonic-gate    * encryption step.  'k3' is the key structure that has 'd3'
326*7c478bd9Sstevel@tonic-gate    * as its 'contents' field.
327*7c478bd9Sstevel@tonic-gate    * k3 = HMAC(k1, checksum)
328*7c478bd9Sstevel@tonic-gate    */
329*7c478bd9Sstevel@tonic-gate   d3.length=keybytes;
330*7c478bd9Sstevel@tonic-gate   d3.data=MALLOC(d3.length);
331*7c478bd9Sstevel@tonic-gate   if  (d3.data == NULL) {
332*7c478bd9Sstevel@tonic-gate     FREE(d1.data, d1.length);
333*7c478bd9Sstevel@tonic-gate     if (d2.data)
334*7c478bd9Sstevel@tonic-gate 	FREE(d2.data, d2.length);
335*7c478bd9Sstevel@tonic-gate     return (ENOMEM);
336*7c478bd9Sstevel@tonic-gate   }
337*7c478bd9Sstevel@tonic-gate   bcopy(key, &k3, sizeof(krb5_keyblock));
338*7c478bd9Sstevel@tonic-gate   k3.length=d3.length;
339*7c478bd9Sstevel@tonic-gate   k3.contents= (void *) d3.data;
340*7c478bd9Sstevel@tonic-gate 
341*7c478bd9Sstevel@tonic-gate   salt.length=14;
342*7c478bd9Sstevel@tonic-gate   salt.data=MALLOC(salt.length);
343*7c478bd9Sstevel@tonic-gate   if(salt.data==NULL) {
344*7c478bd9Sstevel@tonic-gate     FREE(d1.data, d1.length);
345*7c478bd9Sstevel@tonic-gate     if (d2.data)
346*7c478bd9Sstevel@tonic-gate 	FREE(d2.data, d2.length);
347*7c478bd9Sstevel@tonic-gate     FREE(d3.data, d3.length);
348*7c478bd9Sstevel@tonic-gate     return (ENOMEM);
349*7c478bd9Sstevel@tonic-gate   }
350*7c478bd9Sstevel@tonic-gate 
351*7c478bd9Sstevel@tonic-gate   ciphertext.length=input->length-hashsize;
352*7c478bd9Sstevel@tonic-gate   ciphertext.data=input->data+hashsize;
353*7c478bd9Sstevel@tonic-gate 
354*7c478bd9Sstevel@tonic-gate   plaintext.length=ciphertext.length;
355*7c478bd9Sstevel@tonic-gate   plaintext.data=MALLOC(plaintext.length);
356*7c478bd9Sstevel@tonic-gate   if (plaintext.data == NULL) {
357*7c478bd9Sstevel@tonic-gate     FREE(d1.data, d1.length);
358*7c478bd9Sstevel@tonic-gate     if (d2.data)
359*7c478bd9Sstevel@tonic-gate 	FREE(d2.data, d2.length);
360*7c478bd9Sstevel@tonic-gate     FREE(d3.data, d3.length);
361*7c478bd9Sstevel@tonic-gate     FREE(salt.data, salt.length);
362*7c478bd9Sstevel@tonic-gate     return (ENOMEM);
363*7c478bd9Sstevel@tonic-gate   }
364*7c478bd9Sstevel@tonic-gate 
365*7c478bd9Sstevel@tonic-gate   checksum.length=hashsize;
366*7c478bd9Sstevel@tonic-gate   checksum.data=input->data;
367*7c478bd9Sstevel@tonic-gate 
368*7c478bd9Sstevel@tonic-gate   /* compute the salt */
369*7c478bd9Sstevel@tonic-gate   ms_usage=krb5int_arcfour_translate_usage(usage);
370*7c478bd9Sstevel@tonic-gate   if (key->enctype == ENCTYPE_ARCFOUR_HMAC_EXP) {
371*7c478bd9Sstevel@tonic-gate     (void) strncpy(salt.data, l40, salt.length);
372*7c478bd9Sstevel@tonic-gate     salt.data[10]=ms_usage & 0xff;
373*7c478bd9Sstevel@tonic-gate     salt.data[11]=(ms_usage>>8) & 0xff;
374*7c478bd9Sstevel@tonic-gate     salt.data[12]=(ms_usage>>16) & 0xff;
375*7c478bd9Sstevel@tonic-gate     salt.data[13]=(ms_usage>>24) & 0xff;
376*7c478bd9Sstevel@tonic-gate   } else {
377*7c478bd9Sstevel@tonic-gate     salt.length=4;
378*7c478bd9Sstevel@tonic-gate     salt.data[0]=ms_usage & 0xff;
379*7c478bd9Sstevel@tonic-gate     salt.data[1]=(ms_usage>>8) & 0xff;
380*7c478bd9Sstevel@tonic-gate     salt.data[2]=(ms_usage>>16) & 0xff;
381*7c478bd9Sstevel@tonic-gate     salt.data[3]=(ms_usage>>24) & 0xff;
382*7c478bd9Sstevel@tonic-gate   }
383*7c478bd9Sstevel@tonic-gate 
384*7c478bd9Sstevel@tonic-gate #ifdef _KERNEL
385*7c478bd9Sstevel@tonic-gate   ret=krb5_hmac(context, key, &salt, &d1);
386*7c478bd9Sstevel@tonic-gate #else
387*7c478bd9Sstevel@tonic-gate   ret=krb5_hmac(context, hash, key, 1, &salt, &d1);
388*7c478bd9Sstevel@tonic-gate #endif /* _KERNEL */
389*7c478bd9Sstevel@tonic-gate   if (ret)
390*7c478bd9Sstevel@tonic-gate     goto cleanup;
391*7c478bd9Sstevel@tonic-gate 
392*7c478bd9Sstevel@tonic-gate   if (key->enctype == ENCTYPE_ARCFOUR_HMAC_EXP) {
393*7c478bd9Sstevel@tonic-gate     bcopy(k1.contents, k2.contents, d1.length);
394*7c478bd9Sstevel@tonic-gate     (void) memset(k1.contents+7, 0xab, 9);
395*7c478bd9Sstevel@tonic-gate     kptr = &k2;
396*7c478bd9Sstevel@tonic-gate   } else {
397*7c478bd9Sstevel@tonic-gate     kptr = &k1;
398*7c478bd9Sstevel@tonic-gate   }
399*7c478bd9Sstevel@tonic-gate 
400*7c478bd9Sstevel@tonic-gate #ifdef _KERNEL
401*7c478bd9Sstevel@tonic-gate   ret = krb5_hmac(context, &k1, &checksum, &d3);
402*7c478bd9Sstevel@tonic-gate #else
403*7c478bd9Sstevel@tonic-gate   ret = krb5_hmac(context, hash, &k1, 1, &checksum, &d3);
404*7c478bd9Sstevel@tonic-gate #endif /* _KERNEL */
405*7c478bd9Sstevel@tonic-gate 
406*7c478bd9Sstevel@tonic-gate   if (ret)
407*7c478bd9Sstevel@tonic-gate     goto cleanup;
408*7c478bd9Sstevel@tonic-gate 
409*7c478bd9Sstevel@tonic-gate   ret=(*(enc->decrypt))(context, &k3, ivec, &ciphertext, &plaintext);
410*7c478bd9Sstevel@tonic-gate   if (ret)
411*7c478bd9Sstevel@tonic-gate     goto cleanup;
412*7c478bd9Sstevel@tonic-gate 
413*7c478bd9Sstevel@tonic-gate #ifdef _KERNEL
414*7c478bd9Sstevel@tonic-gate   ret = krb5_hmac(context, kptr, &plaintext, &d1);
415*7c478bd9Sstevel@tonic-gate #else
416*7c478bd9Sstevel@tonic-gate   ret = krb5_hmac(context, hash, kptr, 1, &plaintext, &d1);
417*7c478bd9Sstevel@tonic-gate #endif /* _KERNEL */
418*7c478bd9Sstevel@tonic-gate 
419*7c478bd9Sstevel@tonic-gate   if (ret)
420*7c478bd9Sstevel@tonic-gate     goto cleanup;
421*7c478bd9Sstevel@tonic-gate 
422*7c478bd9Sstevel@tonic-gate   if (bcmp(checksum.data, d1.data, hashsize) != 0) {
423*7c478bd9Sstevel@tonic-gate     ret=KRB5KRB_AP_ERR_BAD_INTEGRITY;
424*7c478bd9Sstevel@tonic-gate     goto cleanup;
425*7c478bd9Sstevel@tonic-gate   }
426*7c478bd9Sstevel@tonic-gate 
427*7c478bd9Sstevel@tonic-gate   bcopy(plaintext.data+CONFOUNDERLENGTH, output->data,
428*7c478bd9Sstevel@tonic-gate 	 (plaintext.length-CONFOUNDERLENGTH));
429*7c478bd9Sstevel@tonic-gate   output->length=plaintext.length-CONFOUNDERLENGTH;
430*7c478bd9Sstevel@tonic-gate 
431*7c478bd9Sstevel@tonic-gate  cleanup:
432*7c478bd9Sstevel@tonic-gate   bzero(d1.data, d1.length);
433*7c478bd9Sstevel@tonic-gate   if (d2.data) {
434*7c478bd9Sstevel@tonic-gate 	bzero(d2.data, d2.length);
435*7c478bd9Sstevel@tonic-gate 	FREE(d2.data, d2.length);
436*7c478bd9Sstevel@tonic-gate   }
437*7c478bd9Sstevel@tonic-gate   bzero(d3.data, d2.length);
438*7c478bd9Sstevel@tonic-gate   bzero(salt.data, salt.length);
439*7c478bd9Sstevel@tonic-gate   bzero(plaintext.data, plaintext.length);
440*7c478bd9Sstevel@tonic-gate 
441*7c478bd9Sstevel@tonic-gate   FREE(d1.data, d1.length);
442*7c478bd9Sstevel@tonic-gate   FREE(d3.data, d3.length);
443*7c478bd9Sstevel@tonic-gate   FREE(salt.data, salt.length);
444*7c478bd9Sstevel@tonic-gate   FREE(plaintext.data, plaintext.length);
445*7c478bd9Sstevel@tonic-gate 
446*7c478bd9Sstevel@tonic-gate   return (ret);
447*7c478bd9Sstevel@tonic-gate }
448*7c478bd9Sstevel@tonic-gate 
449