xref: /illumos-gate/usr/src/cmd/iconv/charmap.c (revision 48edc7cf)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
14  */
15 
16 /*
17  * CHARMAP file handling for iconv.
18  */
19 
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <string.h>
23 #include <errno.h>
24 #include <limits.h>
25 #include <unistd.h>
26 #include <alloca.h>
27 #include <sys/avl.h>
28 #include <stddef.h>
29 #include <unistd.h>
30 #include "charmap.h"
31 #include "parser.tab.h"
32 #include <assert.h>
33 
34 enum cmap_pass cmap_pass;
35 static avl_tree_t	cmap_sym;
36 static avl_tree_t	cmap_mbs;
37 
38 typedef struct charmap {
39 	const char *cm_name;
40 	struct charmap *cm_alias_of;
41 	avl_node_t cm_avl_sym;
42 	avl_node_t cm_avl_mbs;
43 	int cm_warned;
44 	int cm_frmbs_len;
45 	int cm_tombs_len;
46 	char cm_frmbs[MB_LEN_MAX + 1]; /* input */
47 	char cm_tombs[MB_LEN_MAX + 1]; /* output */
48 } charmap_t;
49 
50 static void add_charmap_impl_fr(char *sym, char *mbs, int mbs_len, int nodups);
51 static void add_charmap_impl_to(char *sym, char *mbs, int mbs_len, int nodups);
52 
53 /*
54  * Array of POSIX specific portable characters.
55  */
56 static const struct {
57 	char	*name;
58 	int	ch;
59 } portable_chars[] = {
60 	{ "NUL",		 '\0' },
61 	{ "alert",		'\a' },
62 	{ "backspace",		'\b' },
63 	{ "tab",		'\t' },
64 	{ "carriage-return",	'\r' },
65 	{ "newline",		'\n' },
66 	{ "vertical-tab",	'\v' },
67 	{ "form-feed",		'\f' },
68 	{ "space",		' ' },
69 	{ "exclamation-mark",	'!' },
70 	{ "quotation-mark",	'"' },
71 	{ "number-sign",	'#' },
72 	{ "dollar-sign",	'$' },
73 	{ "percent-sign",	'%' },
74 	{ "ampersand",		'&' },
75 	{ "apostrophe",		'\'' },
76 	{ "left-parenthesis",	'(' },
77 	{ "right-parenthesis",	'(' },
78 	{ "asterisk",		'*' },
79 	{ "plus-sign",		'+' },
80 	{ "comma",		 ','},
81 	{ "hyphen-minus",	'-' },
82 	{ "hyphen",		'-' },
83 	{ "full-stop",		'.' },
84 	{ "period",		'.' },
85 	{ "slash",		'/' },
86 	{ "solidus",		'/' },
87 	{ "zero",		'0' },
88 	{ "one",		'1' },
89 	{ "two",		'2' },
90 	{ "three",		'3' },
91 	{ "four",		'4' },
92 	{ "five",		'5' },
93 	{ "six",		'6' },
94 	{ "seven",		'7' },
95 	{ "eight",		'8' },
96 	{ "nine",		'9' },
97 	{ "colon",		':' },
98 	{ "semicolon",		';' },
99 	{ "less-than-sign",	'<' },
100 	{ "equals-sign",	'=' },
101 	{ "greater-than-sign",	'>' },
102 	{ "question-mark",	'?' },
103 	{ "commercial-at",	'@' },
104 	{ "left-square-bracket", '[' },
105 	{ "backslash",		'\\' },
106 	{ "reverse-solidus",	'\\' },
107 	{ "right-square-bracket", ']' },
108 	{ "circumflex",		'^' },
109 	{ "circumflex-accent",	'^' },
110 	{ "low-line",		'_' },
111 	{ "underscore",		'_' },
112 	{ "grave-accent",	'`' },
113 	{ "left-brace",		'{' },
114 	{ "left-curly-bracket",	'{' },
115 	{ "vertical-line",	'|' },
116 	{ "right-brace",	'}' },
117 	{ "right-curly-bracket", '}' },
118 	{ "tilde",		'~' },
119 	{ "A", 'A' },
120 	{ "B", 'B' },
121 	{ "C", 'C' },
122 	{ "D", 'D' },
123 	{ "E", 'E' },
124 	{ "F", 'F' },
125 	{ "G", 'G' },
126 	{ "H", 'H' },
127 	{ "I", 'I' },
128 	{ "J", 'J' },
129 	{ "K", 'K' },
130 	{ "L", 'L' },
131 	{ "M", 'M' },
132 	{ "N", 'N' },
133 	{ "O", 'O' },
134 	{ "P", 'P' },
135 	{ "Q", 'Q' },
136 	{ "R", 'R' },
137 	{ "S", 'S' },
138 	{ "T", 'T' },
139 	{ "U", 'U' },
140 	{ "V", 'V' },
141 	{ "W", 'W' },
142 	{ "X", 'X' },
143 	{ "Y", 'Y' },
144 	{ "Z", 'Z' },
145 	{ "a", 'a' },
146 	{ "b", 'b' },
147 	{ "c", 'c' },
148 	{ "d", 'd' },
149 	{ "e", 'e' },
150 	{ "f", 'f' },
151 	{ "g", 'g' },
152 	{ "h", 'h' },
153 	{ "i", 'i' },
154 	{ "j", 'j' },
155 	{ "k", 'k' },
156 	{ "l", 'l' },
157 	{ "m", 'm' },
158 	{ "n", 'n' },
159 	{ "o", 'o' },
160 	{ "p", 'p' },
161 	{ "q", 'q' },
162 	{ "r", 'r' },
163 	{ "s", 's' },
164 	{ "t", 't' },
165 	{ "u", 'u' },
166 	{ "v", 'v' },
167 	{ "w", 'w' },
168 	{ "x", 'x' },
169 	{ "y", 'y' },
170 	{ "z", 'z' },
171 	{ NULL, 0 }
172 };
173 
174 static int
cmap_compare_sym(const void * n1,const void * n2)175 cmap_compare_sym(const void *n1, const void *n2)
176 {
177 	const charmap_t *c1 = n1;
178 	const charmap_t *c2 = n2;
179 	int rv;
180 
181 	rv = strcmp(c1->cm_name, c2->cm_name);
182 	return ((rv < 0) ? -1 : (rv > 0) ? 1 : 0);
183 }
184 
185 /*
186  * In order for partial match searches to work,
187  * we need these sorted by mbs contents.
188  */
189 static int
cmap_compare_mbs(const void * n1,const void * n2)190 cmap_compare_mbs(const void *n1, const void *n2)
191 {
192 	const charmap_t *c1 = n1;
193 	const charmap_t *c2 = n2;
194 	int len, rv;
195 
196 	len = c1->cm_frmbs_len;
197 	if (len < c2->cm_frmbs_len)
198 		len = c2->cm_frmbs_len;
199 	rv = memcmp(c1->cm_frmbs, c2->cm_frmbs, len);
200 	if (rv < 0)
201 		return (-1);
202 	if (rv > 0)
203 		return (1);
204 	/* they match through length */
205 	if (c1->cm_frmbs_len < c2->cm_frmbs_len)
206 		return (-1);
207 	if (c2->cm_frmbs_len < c1->cm_frmbs_len)
208 		return (1);
209 	return (0);
210 }
211 
212 void
charmap_init(char * to_map,char * from_map)213 charmap_init(char *to_map, char *from_map)
214 {
215 	avl_create(&cmap_sym, cmap_compare_sym, sizeof (charmap_t),
216 	    offsetof(charmap_t, cm_avl_sym));
217 
218 	avl_create(&cmap_mbs, cmap_compare_mbs, sizeof (charmap_t),
219 	    offsetof(charmap_t, cm_avl_mbs));
220 
221 	cmap_pass = CMAP_PASS_FROM;
222 	reset_scanner(from_map);
223 	(void) yyparse();
224 	add_charmap_posix();
225 
226 	cmap_pass = CMAP_PASS_TO;
227 	reset_scanner(to_map);
228 	(void) yyparse();
229 }
230 
231 void
charmap_dump()232 charmap_dump()
233 {
234 	charmap_t *cm;
235 	int i;
236 
237 	cm = avl_first(&cmap_mbs);
238 	while (cm != NULL) {
239 		(void) printf("name=\"%s\"\n", cm->cm_name);
240 
241 		(void) printf("\timbs=\"");
242 		for (i = 0; i < cm->cm_frmbs_len; i++)
243 			(void) printf("\\x%02x", cm->cm_frmbs[i] & 0xFF);
244 		(void) printf("\"\n");
245 
246 		(void) printf("\tombs=\"");
247 		for (i = 0; i < cm->cm_tombs_len; i++)
248 			(void) printf("\\x%02x", cm->cm_tombs[i] & 0xFF);
249 		(void) printf("\"\n");
250 
251 		cm = AVL_NEXT(&cmap_mbs, cm);
252 	}
253 }
254 
255 /*
256  * We parse two charmap files:  First the "from" map, where we build
257  * cmap_mbs and cmap_sym which we'll later use to translate the input
258  * stream (mbs encodings) to symbols.  Second, we parse the "to" map,
259  * where we fill in the tombs members of entries in cmap_sym, (which
260  * must alread exist) used later to write the output encoding.
261  */
262 static void
add_charmap_impl(char * sym,char * mbs,int mbs_len,int nodups)263 add_charmap_impl(char *sym, char *mbs, int mbs_len, int nodups)
264 {
265 
266 	/*
267 	 * While parsing both the "from" and "to" cmaps,
268 	 * require both the symbol and encoding.
269 	 */
270 	if (sym == NULL || mbs == NULL) {
271 		errf(_("invalid charmap entry"));
272 		return;
273 	}
274 
275 	switch (cmap_pass) {
276 	case CMAP_PASS_FROM:
277 		add_charmap_impl_fr(sym, mbs, mbs_len, nodups);
278 		break;
279 	case CMAP_PASS_TO:
280 		add_charmap_impl_to(sym, mbs, mbs_len, nodups);
281 		break;
282 	default:
283 		abort();
284 		break;
285 	}
286 }
287 
288 static void
add_charmap_impl_fr(char * sym,char * mbs,int mbs_len,int nodups)289 add_charmap_impl_fr(char *sym, char *mbs, int mbs_len, int nodups)
290 {
291 	charmap_t	*m, *n, *s;
292 	avl_index_t	where_sym, where_mbs;
293 
294 	if ((n = calloc(1, sizeof (*n))) == NULL) {
295 		errf(_("out of memory"));
296 		return;
297 	}
298 	n->cm_name = sym;
299 
300 	assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
301 	(void) memcpy(n->cm_frmbs, mbs, mbs_len);
302 	n->cm_frmbs_len = mbs_len;
303 
304 	m = avl_find(&cmap_mbs, n, &where_mbs);
305 	s = avl_find(&cmap_sym, n, &where_sym);
306 
307 	/*
308 	 * If we found the symbol, this is a dup.
309 	 */
310 	if (s != NULL) {
311 		if (nodups) {
312 			warn(_("%s: duplicate character symbol"), sym);
313 		}
314 		free(n);
315 		return;
316 	}
317 
318 	/*
319 	 * If we found the mbs, the new one is an alias,
320 	 * which we'll add _only_ to the symbol AVL.
321 	 */
322 	if (m != NULL) {
323 		/* The new one is an alias of the original. */
324 		n->cm_alias_of = m;
325 		avl_insert(&cmap_sym, n, where_sym);
326 		return;
327 	}
328 
329 	avl_insert(&cmap_sym, n, where_sym);
330 	avl_insert(&cmap_mbs, n, where_mbs);
331 }
332 
333 static void
add_charmap_impl_to(char * sym,char * mbs,int mbs_len,int nodups)334 add_charmap_impl_to(char *sym, char *mbs, int mbs_len, int nodups)
335 {
336 	charmap_t	srch = {0};
337 	charmap_t	*m;
338 
339 	assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
340 
341 	srch.cm_name = sym;
342 
343 	m = avl_find(&cmap_sym, &srch, NULL);
344 	if (m == NULL) {
345 		if (sflag == 0)
346 			warn(_("%s: symbol not found"), sym);
347 		return;
348 	}
349 	if (m->cm_alias_of != NULL) {
350 		m = m->cm_alias_of;
351 
352 		/* don't warn for dups with aliases */
353 		if (m->cm_tombs_len != 0)
354 			return;
355 	}
356 
357 	if (m->cm_tombs_len != 0) {
358 		if (nodups) {
359 			warn(_("%s: duplicate encoding for"), sym);
360 		}
361 		return;
362 	}
363 
364 	(void) memcpy(m->cm_tombs, mbs, mbs_len);
365 	m->cm_tombs_len = mbs_len;
366 }
367 
368 void
add_charmap(char * sym,char * mbs)369 add_charmap(char *sym, char *mbs)
370 {
371 	/* mbs[0] is the length */
372 	int mbs_len = *mbs++;
373 	assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
374 	add_charmap_impl(sym, mbs, mbs_len, 1);
375 }
376 
377 
378 /*
379  * This is called by the parser with start/end symbol strings (ssym, esym),
380  * which are allocated in the scanner (T_SYMBOL) and free'd here.
381  */
382 void
add_charmap_range(char * ssym,char * esym,char * mbs)383 add_charmap_range(char *ssym, char *esym, char *mbs)
384 {
385 	int	ls, le;
386 	int	si;
387 	int	sn, en;
388 	int	i;
389 	int	mbs_len;
390 	char	tmbs[MB_LEN_MAX+1];
391 	char	*mb_last;
392 
393 	static const char *digits = "0123456789";
394 
395 	/* mbs[0] is the length */
396 	mbs_len = *mbs++;
397 	assert(0 < mbs_len && mbs_len <= MB_LEN_MAX);
398 	(void) memcpy(tmbs, mbs, mbs_len);
399 	mb_last = tmbs + mbs_len - 1;
400 
401 	ls = strlen(ssym);
402 	le = strlen(esym);
403 
404 	if (((si = strcspn(ssym, digits)) == 0) || (si == ls) ||
405 	    (strncmp(ssym, esym, si) != 0) ||
406 	    (strspn(ssym + si, digits) != (ls - si)) ||
407 	    (strspn(esym + si, digits) != (le - si)) ||
408 	    ((sn = atoi(ssym + si)) > ((en = atoi(esym + si))))) {
409 		errf(_("malformed charmap range"));
410 		return;
411 	}
412 
413 	ssym[si] = 0;
414 	for (i = sn; i <= en; i++) {
415 		char *nn;
416 		(void) asprintf(&nn, "%s%0*u", ssym, ls - si, i);
417 		if (nn == NULL) {
418 			errf(_("out of memory"));
419 			return;
420 		}
421 
422 		add_charmap_impl(nn, tmbs, mbs_len, 1);
423 		(*mb_last)++;
424 	}
425 	free(ssym);
426 	free(esym);
427 }
428 
429 void
add_charmap_char(char * name,int c)430 add_charmap_char(char *name, int c)
431 {
432 	char mbs[MB_LEN_MAX+1];
433 
434 	mbs[0] = c;
435 	mbs[1] = '\0';
436 	add_charmap_impl(name, mbs, 1, 0);
437 }
438 
439 /*
440  * POSIX insists that certain entries be present, even when not in the
441  * orginal charmap file.
442  */
443 void
add_charmap_posix(void)444 add_charmap_posix(void)
445 {
446 	int	i;
447 
448 	for (i = 0; portable_chars[i].name; i++) {
449 		add_charmap_char(portable_chars[i].name, portable_chars[i].ch);
450 	}
451 }
452 
453 /*
454  * This is called with a buffer of (typically) MB_LEN_MAX bytes,
455  * which is potentially a multi-byte symbol, but often contains
456  * extra bytes. Find and return the longest match in the charmap.
457  */
458 static charmap_t *
find_mbs(const char * mbs,int len)459 find_mbs(const char *mbs, int len)
460 {
461 	charmap_t srch = {0};
462 	charmap_t *cm = NULL;
463 
464 	while (len > 0) {
465 		(void) memcpy(srch.cm_frmbs, mbs, len);
466 		srch.cm_frmbs_len = len;
467 		cm = avl_find(&cmap_mbs, &srch, NULL);
468 		if (cm != NULL)
469 			break;
470 		len--;
471 	}
472 
473 	return (cm);
474 }
475 
476 /*
477  * Return true if this sequence matches the initial part
478  * of any sequence known in this charmap.
479  */
480 static boolean_t
find_mbs_partial(const char * mbs,int len)481 find_mbs_partial(const char *mbs, int len)
482 {
483 	charmap_t srch = {0};
484 	charmap_t *cm;
485 	avl_index_t where;
486 
487 	(void) memcpy(srch.cm_frmbs, mbs, len);
488 	srch.cm_frmbs_len = len;
489 	cm = avl_find(&cmap_mbs, &srch, &where);
490 	if (cm != NULL) {
491 		/* full match - not expected, but OK */
492 		return (B_TRUE);
493 	}
494 	cm = avl_nearest(&cmap_mbs, where, AVL_AFTER);
495 	if (cm != NULL && 0 == memcmp(cm->cm_frmbs, mbs, len))
496 		return (B_TRUE);
497 
498 	return (B_FALSE);
499 }
500 
501 /*
502  * Do like iconv(3), but with charmaps.
503  */
504 size_t
cm_iconv(const char ** iptr,size_t * ileft,char ** optr,size_t * oleft)505 cm_iconv(const char **iptr, size_t *ileft, char **optr, size_t *oleft)
506 {
507 	charmap_t *cm;
508 	int mbs_len;
509 
510 	/* Ignore state reset requests. */
511 	if (iptr == NULL || *iptr == NULL)
512 		return (0);
513 
514 	if (*oleft < MB_LEN_MAX) {
515 		errno = E2BIG;
516 		return ((size_t)-1);
517 	}
518 
519 	while (*ileft > 0 && *oleft >= MB_LEN_MAX) {
520 		mbs_len = MB_LEN_MAX;
521 		if (mbs_len > *ileft)
522 			mbs_len = *ileft;
523 		cm = find_mbs(*iptr, mbs_len);
524 		if (cm == NULL) {
525 			if (mbs_len < MB_LEN_MAX &&
526 			    find_mbs_partial(*iptr, mbs_len)) {
527 				/* incomplete sequence */
528 				errno = EINVAL;
529 			} else {
530 				errno = EILSEQ;
531 			}
532 			return ((size_t)-1);
533 		}
534 		assert(cm->cm_frmbs_len > 0);
535 		if (cm->cm_tombs_len == 0) {
536 			if (sflag == 0 && cm->cm_warned == 0) {
537 				cm->cm_warned = 1;
538 				warn(_("To-map does not encode <%s>\n"),
539 				    cm->cm_name);
540 			}
541 			if (cflag == 0) {
542 				errno = EILSEQ;
543 				return ((size_t)-1);
544 			}
545 			/* just skip this input seq. */
546 			*iptr  += cm->cm_frmbs_len;
547 			*ileft -= cm->cm_frmbs_len;
548 			continue;
549 		}
550 
551 		*iptr  += cm->cm_frmbs_len;
552 		*ileft -= cm->cm_frmbs_len;
553 		(void) memcpy(*optr, cm->cm_tombs, cm->cm_tombs_len);
554 		*optr  += cm->cm_tombs_len;
555 		*oleft -= cm->cm_tombs_len;
556 	}
557 
558 	return (0);
559 }
560