xref: /illumos-gate/usr/src/lib/smbsrv/libsmb/common/smb_util.c (revision b811a51ae52cf3dd141a03c928fdcfbd6160c008)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <ctype.h>
27 #include <stdio.h>
28 #include <string.h>
29 #include <stdlib.h>
30 #include <pthread.h>
31 #include <sys/varargs.h>
32 #include <sys/types.h>
33 #include <smbsrv/string.h>
34 #include <smbsrv/libsmb.h>
35 
36 #define	C2H(c)		"0123456789ABCDEF"[(c)]
37 #define	H2C(c)    (((c) >= '0' && (c) <= '9') ? ((c) - '0') :     \
38 	((c) >= 'a' && (c) <= 'f') ? ((c) - 'a' + 10) :         \
39 	((c) >= 'A' && (c) <= 'F') ? ((c) - 'A' + 10) :         \
40 	'\0')
41 #define	DEFAULT_SBOX_SIZE		256
42 
43 /*
44  *
45  * hexdump
46  *
47  * Simple hex dump display function. Displays nbytes of buffer in hex and
48  * printable format. Non-printing characters are shown as '.'. It is safe
49  * to pass a null pointer. Each line begins with the offset. If nbytes is
50  * 0, the line will be blank except for the offset. Example output:
51  *
52  * 00000000  54 68 69 73 20 69 73 20 61 20 70 72 6F 67 72 61  This is a progra
53  * 00000010  6D 20 74 65 73 74 2E 00                          m test..
54  *
55  */
56 void
57 hexdump_offset(unsigned char *buffer, int nbytes, unsigned long *start)
58 {
59 	static char *hex = "0123456789ABCDEF";
60 	int i, count;
61 	int offset;
62 	unsigned char *p;
63 	char ascbuf[64];
64 	char hexbuf[64];
65 	char *ap = ascbuf;
66 	char *hp = hexbuf;
67 
68 	if ((p = buffer) == NULL)
69 		return;
70 
71 	offset = *start;
72 
73 	*ap = '\0';
74 	*hp = '\0';
75 	count = 0;
76 
77 	for (i = 0; i < nbytes; ++i) {
78 		if (i && (i % 16) == 0) {
79 			smb_tracef("%06X %s  %s", offset, hexbuf, ascbuf);
80 			ap = ascbuf;
81 			hp = hexbuf;
82 			count = 0;
83 			offset += 16;
84 		}
85 
86 		ap += sprintf(ap, "%c",
87 		    (*p >= 0x20 && *p < 0x7F) ? *p : '.');
88 		hp += sprintf(hp, " %c%c",
89 		    hex[(*p >> 4) & 0x0F], hex[(*p & 0x0F)]);
90 		++p;
91 		++count;
92 	}
93 
94 	if (count) {
95 		smb_tracef("%06X %-48s  %s", offset, hexbuf, ascbuf);
96 		offset += count;
97 	}
98 
99 	*start = offset;
100 }
101 
102 void
103 hexdump(unsigned char *buffer, int nbytes)
104 {
105 	unsigned long start = 0;
106 
107 	hexdump_offset(buffer, nbytes, &start);
108 }
109 
110 /*
111  * bintohex
112  *
113  * Converts the given binary data (srcbuf) to
114  * its equivalent hex chars (hexbuf).
115  *
116  * hexlen should be at least twice as srclen.
117  * if hexbuf is not big enough returns 0.
118  * otherwise returns number of valid chars in
119  * hexbuf which is srclen * 2.
120  */
121 size_t
122 bintohex(const char *srcbuf, size_t srclen,
123     char *hexbuf, size_t hexlen)
124 {
125 	size_t outlen;
126 	char c;
127 
128 	outlen = srclen << 1;
129 
130 	if (hexlen < outlen)
131 		return (0);
132 
133 	while (srclen-- > 0) {
134 		c = *srcbuf++;
135 		*hexbuf++ = C2H(c & 0xF);
136 		*hexbuf++ = C2H((c >> 4) & 0xF);
137 	}
138 
139 	return (outlen);
140 }
141 
142 /*
143  * hextobin
144  *
145  * Converts hex to binary.
146  *
147  * Assuming hexbuf only contains hex digits (chars)
148  * this function convert every two bytes of hexbuf
149  * to one byte and put it in dstbuf.
150  *
151  * hexlen should be an even number.
152  * dstlen should be at least half of hexlen.
153  *
154  * Returns 0 if sizes are not correct, otherwise
155  * returns the number of converted bytes in dstbuf
156  * which is half of hexlen.
157  */
158 size_t
159 hextobin(const char *hexbuf, size_t hexlen,
160     char *dstbuf, size_t dstlen)
161 {
162 	size_t outlen;
163 
164 	if ((hexlen % 2) != 0)
165 		return (0);
166 
167 	outlen = hexlen >> 1;
168 	if (dstlen < outlen)
169 		return (0);
170 
171 	while (hexlen > 0) {
172 		*dstbuf = H2C(*hexbuf) & 0x0F;
173 		hexbuf++;
174 		*dstbuf++ |= (H2C(*hexbuf) << 4) & 0xF0;
175 		hexbuf++;
176 
177 		hexlen -= 2;
178 	}
179 
180 	return (outlen);
181 }
182 
183 /*
184  * trim_whitespace
185  *
186  * Trim leading and trailing whitespace chars (as defined by isspace)
187  * from a buffer. Example; if the input buffer contained "  text  ",
188  * it will contain "text", when we return. We assume that the buffer
189  * contains a null terminated string. A pointer to the buffer is
190  * returned.
191  */
192 char *
193 trim_whitespace(char *buf)
194 {
195 	char *p = buf;
196 	char *q = buf;
197 
198 	if (buf == NULL)
199 		return (NULL);
200 
201 	while (*p && isspace(*p))
202 		++p;
203 
204 	while ((*q = *p++) != 0)
205 		++q;
206 
207 	if (q != buf) {
208 		while ((--q, isspace(*q)) != 0)
209 			*q = '\0';
210 	}
211 
212 	return (buf);
213 }
214 
215 /*
216  * randomize
217  *
218  * Randomize the contents of the specified buffer.
219  */
220 void
221 randomize(char *data, unsigned len)
222 {
223 	unsigned dwlen = len / 4;
224 	unsigned remlen = len % 4;
225 	unsigned tmp;
226 	unsigned i; /*LINTED E_BAD_PTR_CAST_ALIGN*/
227 	unsigned *p = (unsigned *)data;
228 
229 	for (i = 0; i < dwlen; ++i)
230 		*p++ = random();
231 
232 	if (remlen) {
233 		tmp = random();
234 		(void) memcpy(p, &tmp, remlen);
235 	}
236 }
237 
238 /*
239  * This is the hash mechanism used to encrypt passwords for commands like
240  * SamrSetUserInformation. It uses a 256 byte s-box.
241  */
242 void
243 rand_hash(
244     unsigned char *data,
245     size_t datalen,
246     unsigned char *key,
247     size_t keylen)
248 {
249 	unsigned char sbox[DEFAULT_SBOX_SIZE];
250 	unsigned char tmp;
251 	unsigned char index_i = 0;
252 	unsigned char index_j = 0;
253 	unsigned char j = 0;
254 	int i;
255 
256 	for (i = 0; i < DEFAULT_SBOX_SIZE; ++i)
257 		sbox[i] = (unsigned char)i;
258 
259 	for (i = 0; i < DEFAULT_SBOX_SIZE; ++i) {
260 		j += (sbox[i] + key[i % keylen]);
261 
262 		tmp = sbox[i];
263 		sbox[i] = sbox[j];
264 		sbox[j] = tmp;
265 	}
266 
267 	for (i = 0; i < datalen; ++i) {
268 		index_i++;
269 		index_j += sbox[index_i];
270 
271 		tmp = sbox[index_i];
272 		sbox[index_i] = sbox[index_j];
273 		sbox[index_j] = tmp;
274 
275 		tmp = sbox[index_i] + sbox[index_j];
276 		data[i] = data[i] ^ sbox[tmp];
277 	}
278 }
279