xref: /illumos-gate/usr/src/cmd/localedef/wide.c (revision 7262c8a6)
1 /*
2  * This file and its contents are supplied under the terms of the
3  * Common Development and Distribution License ("CDDL"), version 1.0.
4  * You may only use this file in accordance with the terms of version
5  * 1.0 of the CDDL.
6  *
7  * A full copy of the text of the CDDL should have accompanied this
8  * source.  A copy of the CDDL is also available via the Internet at
9  * http://www.illumos.org/license/CDDL.
10  */
11 
12 /*
13  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
14  * Copyright 2012 Garrett D'Amore <garrett@damore.org>  All rights reserved.
15  */
16 
17 /*
18  * The functions in this file convert from the standard multibyte forms
19  * to the wide character forms used internally by libc.  Unfortunately,
20  * this approach means that we need a method for each and every encoding.
21  */
22 
23 #include <stdlib.h>
24 #include <wchar.h>
25 #include <string.h>
26 #include <note.h>
27 #include <sys/types.h>
28 #include "localedef.h"
29 
30 static int towide_none(wchar_t *, const char *, unsigned);
31 static int towide_utf8(wchar_t *, const char *, unsigned);
32 static int towide_big5(wchar_t *, const char *, unsigned);
33 static int towide_gbk(wchar_t *, const char *, unsigned);
34 static int towide_gb2312(wchar_t *, const char *, unsigned);
35 static int towide_gb18030(wchar_t *, const char *, unsigned);
36 static int towide_mskanji(wchar_t *, const char *, unsigned);
37 static int towide_euccn(wchar_t *, const char *, unsigned);
38 static int towide_eucjp(wchar_t *, const char *, unsigned);
39 static int towide_euckr(wchar_t *, const char *, unsigned);
40 static int towide_euctw(wchar_t *, const char *, unsigned);
41 
42 static int tomb_none(char *, wchar_t);
43 static int tomb_utf8(char *, wchar_t);
44 static int tomb_mbs(char *, wchar_t);
45 
46 static int (*_towide)(wchar_t *, const char *, unsigned) = towide_none;
47 static int (*_tomb)(char *, wchar_t) = tomb_none;
48 static const char *_encoding = "NONE";
49 static int _nbits = 7;
50 
51 /*
52  * Table of supported encodings.  We only bother to list the multibyte
53  * encodings here, because single byte locales are handed by "NONE".
54  */
55 static struct {
56 	const char *name;
57 	/* the name that the underlying libc implemenation uses */
58 	const char *cname;
59 	/* the maximum number of bits required for priorities */
60 	int nbits;
61 	int (*towide)(wchar_t *, const char *, unsigned);
62 	int (*tomb)(char *, wchar_t);
63 } mb_encodings[] = {
64 	/*
65 	 * UTF8 values max out at 0x1fffff (although in theory there could
66 	 * be later extensions, but it won't happen.)  This means we only need
67 	 * 21 bits to be able to encode the entire range of priorities.
68 	 */
69 	{ "UTF-8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
70 	{ "UTF8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
71 	{ "utf8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
72 	{ "utf-8",	"UTF-8",	21, towide_utf8, tomb_utf8 },
73 
74 	{ "EUC-CN",	"EUC-CN",	16, towide_euccn, tomb_mbs },
75 	{ "eucCN",	"EUC-CN",	16, towide_euccn, tomb_mbs },
76 	/*
77 	 * Becuase the 3-byte form of EUC-JP use the same leading byte,
78 	 * only 17 bits required to provide unique priorities.  (The low
79 	 * bit of that first byte is set.)  By setting this value low,
80 	 * we can get by with only 3 bytes in the strxfrm expansion.
81 	 */
82 	{ "EUC-JP",	"EUC-JP",	17, towide_eucjp, tomb_mbs },
83 	{ "eucJP",	"EUC-JP",	17, towide_eucjp, tomb_mbs },
84 
85 	{ "EUC-KR",	"EUC-KR",	16, towide_euckr, tomb_mbs },
86 	{ "eucKR",	"EUC-KR",	16, towide_euckr, tomb_mbs },
87 	/*
88 	 * EUC-TW uses 2 bytes most of the time, but 4 bytes if the
89 	 * high order byte is 0x8E.  However, with 4 byte encodings,
90 	 * the third byte will be A0-B0.  So we only need to consider
91 	 * the lower order 24 bits for collation.
92 	 */
93 	{ "EUC-TW",	"EUC-TW",	24, towide_euctw, tomb_mbs },
94 	{ "eucTW",	"EUC-TW",	24, towide_euctw, tomb_mbs },
95 
96 	{ "MS_Kanji",	"MSKanji",	16, towide_mskanji, tomb_mbs },
97 	{ "MSKanji",	"MSKanji",	16, towide_mskanji, tomb_mbs },
98 	{ "PCK",	"MSKanji",	16, towide_mskanji, tomb_mbs },
99 	{ "SJIS",	"MSKanji",	16, towide_mskanji, tomb_mbs },
100 	{ "Shift_JIS",	"MSKanji",	16, towide_mskanji, tomb_mbs },
101 
102 	{ "BIG5",	"BIG5",		16, towide_big5, tomb_mbs },
103 	{ "big5",	"BIG5",		16, towide_big5, tomb_mbs },
104 	{ "Big5",	"BIG5",		16, towide_big5, tomb_mbs },
105 
106 	{ "GBK",	"GBK",		16, towide_gbk,	tomb_mbs },
107 
108 	/*
109 	 * GB18030 can get away with just 31 bits.  This is because the
110 	 * high order bit is always set for 4 byte values, and the
111 	 * at least one of the other bits in that 4 byte value will
112 	 * be non-zero.
113 	 */
114 	{ "GB18030",	"GB18030",	31, towide_gb18030, tomb_mbs },
115 
116 	/*
117 	 * This should probably be an aliase for euc-cn, or vice versa.
118 	 */
119 	{ "GB2312",	"GB2312",	16, towide_gb2312, tomb_mbs },
120 
121 	{ NULL, NULL },
122 };
123 
124 static char *
show_mb(const char * mb)125 show_mb(const char *mb)
126 {
127 	static char buf[64];
128 
129 	/* ASCII stuff we just print */
130 	if (isascii(*mb) && isgraph(*mb)) {
131 		buf[0] = *mb;
132 		buf[1] = 0;
133 		return (buf);
134 	}
135 	buf[0] = 0;
136 	while (*mb != 0) {
137 		char scr[8];
138 		(void) snprintf(scr, sizeof (scr), "\\x%02x", *mb);
139 		(void) strlcat(buf, scr, sizeof (buf));
140 		mb++;
141 	}
142 	return (buf);
143 }
144 
145 static char	*widemsg;
146 
147 void
werr(const char * fmt,...)148 werr(const char *fmt, ...)
149 {
150 	char	*msg;
151 
152 	va_list	va;
153 	va_start(va, fmt);
154 	(void) vasprintf(&msg, fmt, va);
155 	va_end(va);
156 
157 	free(widemsg);
158 	widemsg = msg;
159 }
160 
161 /*
162  * This is used for 8-bit encodings.
163  */
164 int
towide_none(wchar_t * c,const char * mb,unsigned n)165 towide_none(wchar_t *c, const char *mb, unsigned n)
166 {
167 	_NOTE(ARGUNUSED(n));
168 
169 	if (mb_cur_max != 1) {
170 		werr("invalid or unsupported multibyte locale");
171 		return (-1);
172 	}
173 	*c = (uint8_t)*mb;
174 	return (1);
175 }
176 
177 int
tomb_none(char * mb,wchar_t wc)178 tomb_none(char *mb, wchar_t wc)
179 {
180 	if (mb_cur_max != 1) {
181 		werr("invalid or unsupported multibyte locale");
182 		return (-1);
183 	}
184 	*(uint8_t *)mb = (wc & 0xff);
185 	mb[1] = 0;
186 	return (1);
187 }
188 
189 /*
190  * UTF-8 stores wide characters in UTF-32 form.
191  */
192 int
towide_utf8(wchar_t * wc,const char * mb,unsigned n)193 towide_utf8(wchar_t *wc, const char *mb, unsigned n)
194 {
195 	wchar_t	c;
196 	int	nb;
197 	int	lv;	/* lowest legal value */
198 	int	i;
199 	const uint8_t *s = (const uint8_t *)mb;
200 
201 	c = *s;
202 
203 	if ((c & 0x80) == 0) {
204 		/* 7-bit ASCII */
205 		*wc = c;
206 		return (1);
207 	} else if ((c & 0xe0) == 0xc0) {
208 		/* u80-u7ff - two bytes encoded */
209 		nb = 2;
210 		lv = 0x80;
211 		c &= ~0xe0;
212 	} else if ((c & 0xf0) == 0xe0) {
213 		/* u800-uffff - three bytes encoded */
214 		nb = 3;
215 		lv = 0x800;
216 		c &= ~0xf0;
217 	} else if ((c & 0xf8) == 0xf0) {
218 		/* u1000-u1fffff - four bytes encoded */
219 		nb = 4;
220 		lv = 0x1000;
221 		c &= ~0xf8;
222 	} else {
223 		/* 5 and 6 byte encodings are not legal unicode */
224 		werr("utf8 encoding too large (%s)", show_mb(mb));
225 		return (-1);
226 	}
227 	if (nb > n) {
228 		werr("incomplete utf8 sequence (%s)", show_mb(mb));
229 		return (-1);
230 	}
231 
232 	for (i = 1; i < nb; i++) {
233 		if (((s[i]) & 0xc0) != 0x80) {
234 			werr("illegal utf8 byte (%x)", s[i]);
235 			return (-1);
236 		}
237 		c <<= 6;
238 		c |= (s[i] & 0x3f);
239 	}
240 
241 	if (c < lv) {
242 		werr("illegal redundant utf8 encoding (%s)", show_mb(mb));
243 		return (-1);
244 	}
245 	*wc = c;
246 	return (nb);
247 }
248 
249 int
tomb_utf8(char * mb,wchar_t wc)250 tomb_utf8(char *mb, wchar_t wc)
251 {
252 	uint8_t *s = (uint8_t *)mb;
253 	uint8_t msk;
254 	int cnt;
255 	int i;
256 
257 	if (wc <= 0x7f) {
258 		s[0] = wc & 0x7f;
259 		s[1] = 0;
260 		return (1);
261 	}
262 	if (wc <= 0x7ff) {
263 		cnt = 2;
264 		msk = 0xc0;
265 	} else if (wc <= 0xffff) {
266 		cnt = 3;
267 		msk = 0xe0;
268 	} else if (wc <= 0x1fffff) {
269 		cnt = 4;
270 		msk = 0xf0;
271 	} else {
272 		werr("illegal uf8 char (%x)", wc);
273 		return (-1);
274 	}
275 	for (i = cnt - 1; i; i--) {
276 		s[i] = (wc & 0x3f) | 0x80;
277 		wc >>= 6;
278 	}
279 	s[0] = (msk) | wc;
280 	s[cnt] = 0;
281 	return (cnt);
282 }
283 
284 /*
285  * Several encodings share a simplistic dual byte encoding.  In these
286  * forms, they all indicate that a two byte sequence is to be used if
287  * the first byte has its high bit set.  They all store this simple
288  * encoding as a 16-bit value, although a great many of the possible
289  * code points are not used in most character sets.  This gives a possible
290  * set of just over 32,000 valid code points.
291  *
292  * 0x00 - 0x7f		- 1 byte encoding
293  * 0x80 - 0x7fff	- illegal
294  * 0x8000 - 0xffff	- 2 byte encoding
295  */
296 static int
towide_dbcs(wchar_t * wc,const char * mb,unsigned n)297 towide_dbcs(wchar_t *wc, const char *mb, unsigned n)
298 {
299 	wchar_t	c;
300 
301 	c = *(uint8_t *)mb;
302 
303 	if ((c & 0x80) == 0) {
304 		/* 7-bit */
305 		*wc = c;
306 		return (1);
307 	}
308 	if (n < 2) {
309 		werr("incomplete character sequence (%s)", show_mb(mb));
310 		return (-1);
311 	}
312 
313 	/* Store both bytes as a single 16-bit wide. */
314 	c <<= 8;
315 	c |= (uint8_t)(mb[1]);
316 	*wc = c;
317 	return (2);
318 }
319 
320 /*
321  * Most multibyte locales just convert the wide character to the multibyte
322  * form by stripping leading null bytes, and writing the 32-bit quantity
323  * in big-endian order.
324  */
325 int
tomb_mbs(char * mb,wchar_t wc)326 tomb_mbs(char *mb, wchar_t wc)
327 {
328 	uint8_t *s = (uint8_t *)mb;
329 	int 	n = 0, c;
330 
331 	if ((wc & 0xff000000U) != 0) {
332 		n = 4;
333 	} else if ((wc & 0x00ff0000U) != 0) {
334 		n = 3;
335 	} else if ((wc & 0x0000ff00U) != 0) {
336 		n = 2;
337 	} else {
338 		n = 1;
339 	}
340 	c = n;
341 	while (n) {
342 		n--;
343 		s[n] = wc & 0xff;
344 		wc >>= 8;
345 	}
346 	/* ensure null termination */
347 	s[c] = 0;
348 	return (c);
349 }
350 
351 
352 /*
353  * big5 is a simple dual byte character set.
354  */
355 int
towide_big5(wchar_t * wc,const char * mb,unsigned n)356 towide_big5(wchar_t *wc, const char *mb, unsigned n)
357 {
358 	return (towide_dbcs(wc, mb, n));
359 }
360 
361 /*
362  * GBK encodes wides in the same way that big5 does, the high order
363  * bit of the first byte indicates a double byte character.
364  */
365 int
towide_gbk(wchar_t * wc,const char * mb,unsigned n)366 towide_gbk(wchar_t *wc, const char *mb, unsigned n)
367 {
368 	return (towide_dbcs(wc, mb, n));
369 }
370 
371 /*
372  * GB2312 is another DBCS.  Its cleaner than others in that the second
373  * byte does not encode ASCII, but it supports characters.
374  */
375 int
towide_gb2312(wchar_t * wc,const char * mb,unsigned n)376 towide_gb2312(wchar_t *wc, const char *mb, unsigned n)
377 {
378 	return (towide_dbcs(wc, mb, n));
379 }
380 
381 /*
382  * GB18030.  This encodes as 8, 16, or 32-bits.
383  * 7-bit values are in 1 byte,  4 byte sequences are used when
384  * the second byte encodes 0x30-39 and all other sequences are 2 bytes.
385  */
386 int
towide_gb18030(wchar_t * wc,const char * mb,unsigned n)387 towide_gb18030(wchar_t *wc, const char *mb, unsigned n)
388 {
389 	wchar_t	c;
390 
391 	c = *(uint8_t *)mb;
392 
393 	if ((c & 0x80) == 0) {
394 		/* 7-bit */
395 		*wc = c;
396 		return (1);
397 	}
398 	if (n < 2) {
399 		werr("incomplete character sequence (%s)", show_mb(mb));
400 		return (-1);
401 	}
402 
403 	/* pull in the second byte */
404 	c <<= 8;
405 	c |= (uint8_t)(mb[1]);
406 
407 	if (((c & 0xff) >= 0x30) && ((c & 0xff) <= 0x39)) {
408 		if (n < 4) {
409 			werr("incomplete 4-byte character sequence (%s)",
410 			    show_mb(mb));
411 			return (-1);
412 		}
413 		c <<= 8;
414 		c |= (uint8_t)(mb[2]);
415 		c <<= 8;
416 		c |= (uint8_t)(mb[3]);
417 		*wc = c;
418 		return (4);
419 	}
420 
421 	*wc = c;
422 	return (2);
423 }
424 
425 /*
426  * MS-Kanji (aka SJIS) is almost a clean DBCS like the others, but it
427  * also has a range of single byte characters above 0x80.  (0xa1-0xdf).
428  */
429 int
towide_mskanji(wchar_t * wc,const char * mb,unsigned n)430 towide_mskanji(wchar_t *wc, const char *mb, unsigned n)
431 {
432 	wchar_t	c;
433 
434 	c = *(uint8_t *)mb;
435 
436 	if ((c < 0x80) || ((c > 0xa0) && (c < 0xe0))) {
437 		/* 7-bit */
438 		*wc = c;
439 		return (1);
440 	}
441 
442 	if (n < 2) {
443 		werr("incomplete character sequence (%s)", show_mb(mb));
444 		return (-1);
445 	}
446 
447 	/* Store both bytes as a single 16-bit wide. */
448 	c <<= 8;
449 	c |= (uint8_t)(mb[1]);
450 	*wc = c;
451 	return (2);
452 }
453 
454 /*
455  * EUC forms.  EUC encodings are "variable".  FreeBSD carries some additional
456  * variable data to encode these, but we're going to treat each as independent
457  * instead.  Its the only way we can sensibly move forward.
458  *
459  * Note that the way in which the different EUC forms vary is how wide
460  * CS2 and CS3 are and what the first byte of them is.
461  */
462 static int
towide_euc_impl(wchar_t * wc,const char * mb,unsigned n,uint8_t cs2,uint8_t cs2width,uint8_t cs3,uint8_t cs3width)463 towide_euc_impl(wchar_t *wc, const char *mb, unsigned n,
464     uint8_t cs2, uint8_t cs2width, uint8_t cs3, uint8_t cs3width)
465 {
466 	int i;
467 	int width = 2;
468 	wchar_t	c;
469 
470 	c = *(uint8_t *)mb;
471 
472 	/*
473 	 * All variations of EUC encode 7-bit ASCII as one byte, and use
474 	 * additional bytes for more than that.
475 	 */
476 	if ((c & 0x80) == 0) {
477 		/* 7-bit */
478 		*wc = c;
479 		return (1);
480 	}
481 
482 	/*
483 	 * All EUC variants reserve 0xa1-0xff to identify CS1, which
484 	 * is always two bytes wide.  Note that unused CS will be zero,
485 	 * and that cannot be true because we know that the high order
486 	 * bit must be set.
487 	 */
488 	if (c >= 0xa1) {
489 		width = 2;
490 	} else if (c == cs2) {
491 		width = cs2width;
492 	} else if (c == cs3) {
493 		width = cs3width;
494 	}
495 
496 	if (n < width) {
497 		werr("incomplete character sequence (%s)", show_mb(mb));
498 		return (-1);
499 	}
500 
501 	for (i = 1; i < width; i++) {
502 		/* pull in the next byte */
503 		c <<= 8;
504 		c |= (uint8_t)(mb[i]);
505 	}
506 
507 	*wc = c;
508 	return (width);
509 }
510 
511 /*
512  * EUC-CN encodes as follows:
513  *
514  * Code set 0 (ASCII):				0x21-0x7E
515  * Code set 1 (CNS 11643-1992 Plane 1):		0xA1A1-0xFEFE
516  * Code set 2:					unused
517  * Code set 3:					unused
518  */
519 int
towide_euccn(wchar_t * wc,const char * mb,unsigned n)520 towide_euccn(wchar_t *wc, const char *mb, unsigned n)
521 {
522 	return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
523 }
524 
525 /*
526  * EUC-JP encodes as follows:
527  *
528  * Code set 0 (ASCII or JIS X 0201-1976 Roman):	0x21-0x7E
529  * Code set 1 (JIS X 0208):			0xA1A1-0xFEFE
530  * Code set 2 (half-width katakana):		0x8EA1-0x8EDF
531  * Code set 3 (JIS X 0212-1990):		0x8FA1A1-0x8FFEFE
532  */
533 int
towide_eucjp(wchar_t * wc,const char * mb,unsigned n)534 towide_eucjp(wchar_t *wc, const char *mb, unsigned n)
535 {
536 	return (towide_euc_impl(wc, mb, n, 0x8e, 2, 0x8f, 3));
537 }
538 
539 /*
540  * EUC-KR encodes as follows:
541  *
542  * Code set 0 (ASCII or KS C 5636-1993):	0x21-0x7E
543  * Code set 1 (KS C 5601-1992):			0xA1A1-0xFEFE
544  * Code set 2:					unused
545  * Code set 3:					unused
546  */
547 int
towide_euckr(wchar_t * wc,const char * mb,unsigned n)548 towide_euckr(wchar_t *wc, const char *mb, unsigned n)
549 {
550 	return (towide_euc_impl(wc, mb, n, 0, 0, 0, 0));
551 }
552 
553 /*
554  * EUC-TW encodes as follows:
555  *
556  * Code set 0 (ASCII):				0x21-0x7E
557  * Code set 1 (CNS 11643-1992 Plane 1):		0xA1A1-0xFEFE
558  * Code set 2 (CNS 11643-1992 Planes 1-16):	0x8EA1A1A1-0x8EB0FEFE
559  * Code set 3:					unused
560  */
561 int
towide_euctw(wchar_t * wc,const char * mb,unsigned n)562 towide_euctw(wchar_t *wc, const char *mb, unsigned n)
563 {
564 	return (towide_euc_impl(wc, mb, n, 0x8e, 4, 0, 0));
565 }
566 
567 /*
568  * Public entry points.
569  */
570 
571 int
to_wide(wchar_t * wc,const char * mb)572 to_wide(wchar_t *wc, const char *mb)
573 {
574 	/* this won't fail hard */
575 	return (_towide(wc, mb, strlen(mb)));
576 }
577 
578 int
to_mb(char * mb,wchar_t wc)579 to_mb(char *mb, wchar_t wc)
580 {
581 	int	rv;
582 
583 	if ((rv = _tomb(mb, wc)) < 0) {
584 		errf(widemsg);
585 		free(widemsg);
586 		widemsg = NULL;
587 	}
588 	return (rv);
589 }
590 
591 char *
to_mb_string(const wchar_t * wcs)592 to_mb_string(const wchar_t *wcs)
593 {
594 	char	*mbs;
595 	char	*ptr;
596 	int	len;
597 
598 	mbs = malloc((wcslen(wcs) * mb_cur_max) + 1);
599 	if (mbs == NULL) {
600 		errf("out of memory");
601 		return (NULL);
602 	}
603 	ptr = mbs;
604 	while (*wcs) {
605 		if ((len = to_mb(ptr, *wcs)) < 0) {
606 			INTERR;
607 			free(mbs);
608 			return (NULL);
609 		}
610 		wcs++;
611 		ptr += len;
612 	}
613 	*ptr = 0;
614 	return (mbs);
615 }
616 
617 void
set_wide_encoding(const char * encoding)618 set_wide_encoding(const char *encoding)
619 {
620 	int i;
621 
622 	_towide = towide_none;
623 	_tomb = tomb_none;
624 	_encoding = "NONE";
625 	_nbits = 8;
626 
627 	for (i = 0; mb_encodings[i].name; i++) {
628 		if (strcasecmp(encoding, mb_encodings[i].name) == 0) {
629 			_towide = mb_encodings[i].towide;
630 			_tomb = mb_encodings[i].tomb;
631 			_encoding = mb_encodings[i].cname;
632 			_nbits = mb_encodings[i].nbits;
633 			break;
634 		}
635 	}
636 }
637 
638 const char *
get_wide_encoding(void)639 get_wide_encoding(void)
640 {
641 	return (_encoding);
642 }
643 
644 int
max_wide(void)645 max_wide(void)
646 {
647 	return ((int)((1U << _nbits) - 1));
648 }
649