xref: /illumos-gate/usr/src/cmd/iconv/scanner.c (revision 48edc7cf)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
 */

/*
 * This file contains the "scanner", which tokenizes charmap files
 * for iconv for processing by the higher level grammar processor.
 */

#include <stdio.h>
#include <stdlib.h>
#include <ctype.h>
#include <limits.h>
#include <string.h>
#include <widec.h>
#include <sys/types.h>
#include <assert.h>
#include "charmap.h"
#include "parser.tab.h"

int			com_char = '#';
int			esc_char = '\\';
int			mb_cur_min = 1;
int			mb_cur_max = MB_LEN_MAX;
int			lineno = 1;
int			warnings = 0;
static int		nextline;
static FILE		*input = stdin;
static const char	*filename = "<stdin>";
static int		instring = 0;
static int		escaped = 0;

/*
 * Token space ... grows on demand.
 */
static char *token = NULL;
static int tokidx;
static int toksz = 0;
static int hadtok = 0;

/*
 * The last keyword seen.  This is useful to trigger the special lexer rules
 * for "copy" and also collating symbols and elements.
 */
int	last_kw = 0;
static int	category = T_END;

static struct token {
	int id;
	const char *name;
} keywords[] = {
	{ T_COM_CHAR,		"comment_char" },
	{ T_ESC_CHAR,		"escape_char" },
	{ T_END,		"END" },

	/*
	 * These are keywords used in the charmap file.  Note that
	 * Solaris orginally used angle brackets to wrap some of them,
	 * but we removed that to simplify our parser.  The first of these
	 * items are "global items."
	 */
	{ T_CHARMAP,		"CHARMAP" },
	{ T_WIDTH,		"WIDTH" },
	{ T_WIDTH_DEFAULT,	"WIDTH_DEFAULT" },

	{ -1, NULL },
};

/*
 * These special words are only used in a charmap file, enclosed in <>.
 */
static struct token symwords[] = {
	{ T_COM_CHAR,		"comment_char" },
	{ T_ESC_CHAR,		"escape_char" },
	{ T_CODE_SET,		"code_set_name" },
	{ T_MB_CUR_MAX,		"mb_cur_max" },
	{ T_MB_CUR_MIN,		"mb_cur_min" },
	{ -1, NULL },
};

static int categories[] = {
	T_CHARMAP,
	0
};

void
reset_scanner(const char *fname)
{
	if (fname == NULL) {
		filename = "<stdin>";
		input = stdin;
	} else {
		if (input != stdin)
			(void) fclose(input);
		if ((input = fopen(fname, "r")) == NULL) {
			perror(fname);
			exit(1);
		}
		filename = fname;
	}
	com_char = '#';
	esc_char = '\\';
	instring = 0;
	escaped = 0;
	lineno = 1;
	nextline = 1;
	tokidx = 0;
	last_kw = 0;
	category = T_END;
}

#define	hex(x)	\
	(isdigit(x) ? (x - '0') : ((islower(x) ? (x - 'a') : (x - 'A')) + 10))
#define	isodigit(x)	((x >= '0') && (x <= '7'))

static int
scanc(void)
{
	int	c;

	c = getc(input);
	lineno = nextline;
	if (c == '\n') {
		nextline++;
	}
	return (c);
}

static void
unscanc(int c)
{
	if (c == '\n') {
		nextline--;
	}
	if (ungetc(c, input) < 0) {
		yyerror(_("ungetc failed"));
	}
}

static int
scan_hex_byte(void)
{
	int	c1, c2;
	int	v;

	c1 = scanc();
	if (!isxdigit(c1)) {
		yyerror(_("malformed hex digit"));
		return (0);
	}
	c2 = scanc();
	if (!isxdigit(c2)) {
		yyerror(_("malformed hex digit"));
		return (0);
	}
	v = ((hex(c1) << 4) | hex(c2));
	return (v);
}

static int
scan_dec_byte(void)
{
	int	c1, c2, c3;
	int	b;

	c1 = scanc();
	if (!isdigit(c1)) {
		yyerror(_("malformed decimal digit"));
		return (0);
	}
	b = c1 - '0';
	c2 = scanc();
	if (!isdigit(c2)) {
		yyerror(_("malformed decimal digit"));
		return (0);
	}
	b *= 10;
	b += (c2 - '0');
	c3 = scanc();
	if (!isdigit(c3)) {
		unscanc(c3);
	} else {
		b *= 10;
		b += (c3 - '0');
	}
	return (b);
}

static int
scan_oct_byte(void)
{
	int c1, c2, c3;
	int	b;

	b = 0;

	c1 = scanc();
	if (!isodigit(c1)) {
		yyerror(_("malformed octal digit"));
		return (0);
	}
	b = c1 - '0';
	c2 = scanc();
	if (!isodigit(c2)) {
		yyerror(_("malformed octal digit"));
		return (0);
	}
	b *= 8;
	b += (c2 - '0');
	c3 = scanc();
	if (!isodigit(c3)) {
		unscanc(c3);
	} else {
		b *= 8;
		b += (c3 - '0');
	}
	return (b);
}

void
add_tok(int c)
{
	if ((tokidx + 1) >= toksz) {
		toksz += 64;
		if ((token = realloc(token, toksz)) == NULL) {
			yyerror(_("out of memory"));
			tokidx = 0;
			toksz = 0;
			return;
		}
	}

	token[tokidx++] = (char)c;
	token[tokidx] = 0;
}

static int
get_byte(void)
{
	int	c;

	if ((c = scanc()) != esc_char) {
		unscanc(c);
		return (EOF);
	}
	c = scanc();

	switch (c) {
	case 'd':
	case 'D':
		return (scan_dec_byte());
	case 'x':
	case 'X':
		return (scan_hex_byte());
	case '0':
	case '1':
	case '2':
	case '3':
	case '4':
	case '5':
	case '6':
	case '7':
		/* put the character back so we can get it */
		unscanc(c);
		return (scan_oct_byte());
	default:
		unscanc(c);
		unscanc(esc_char);
		return (EOF);
	}
}

int
get_escaped(int c)
{
	switch (c) {
	case 'n':
		return ('\n');
	case 'r':
		return ('\r');
	case 't':
		return ('\t');
	case 'f':
		return ('\f');
	case 'v':
		return ('\v');
	case 'b':
		return ('\b');
	case 'a':
		return ('\a');
	default:
		return (c);
	}
}

int
get_wide(void)
{
	/* NB: yylval.mbs[0] is the length */
	char *mbs = &yylval.mbs[1];
	int mbi = 0;
	int c;

	mbs[mbi] = 0;
	if (mb_cur_max > MB_LEN_MAX) {
		yyerror(_("max multibyte character size too big"));
		return (T_NULL);
	}
	for (;;) {
		if ((c = get_byte()) == EOF)
			break;
		if (mbi == mb_cur_max) {
			unscanc(c);
			yyerror(_("length > mb_cur_max"));
			return (T_NULL);
		}
		mbs[mbi++] = c;
		mbs[mbi] = 0;
	}

	/* result in yylval.mbs */
	mbs[-1] = mbi;
	return (T_CHAR);
}

int
get_symbol(void)
{
	int	c;

	while ((c = scanc()) != EOF) {
		if (escaped) {
			escaped = 0;
			if (c == '\n')
				continue;
			add_tok(get_escaped(c));
			continue;
		}
		if (c == esc_char) {
			escaped = 1;
			continue;
		}
		if (c == '\n') {	/* well that's strange! */
			yyerror(_("unterminated symbolic name"));
			continue;
		}
		if (c == '>') {		/* end of symbol */

			/*
			 * This restarts the token from the beginning
			 * the next time we scan a character.  (This
			 * token is complete.)
			 */

			if (token == NULL) {
				yyerror(_("missing symbolic name"));
				return (T_NULL);
			}
			tokidx = 0;

			/*
			 * A few symbols are handled as keywords outside
			 * of the normal categories.
			 */
			if (category == T_END) {
				int i;
				for (i = 0; symwords[i].name != 0; i++) {
					if (strcmp(token, symwords[i].name) ==
					    0) {
						last_kw = symwords[i].id;
						return (last_kw);
					}
				}
			}
			/* its an undefined symbol */
			yylval.token = strdup(token);
			if (yylval.token == NULL) {
				perror("malloc");
				exit(1);
			}
			token = NULL;
			toksz = 0;
			tokidx = 0;
			return (T_SYMBOL);
		}
		add_tok(c);
	}

	yyerror(_("unterminated symbolic name"));
	return (EOF);
}


static int
consume_token(void)
{
	int	len = tokidx;
	int	i;

	tokidx = 0;
	if (token == NULL)
		return (T_NULL);

	/*
	 * this one is special, because we don't want it to alter the
	 * last_kw field.
	 */
	if (strcmp(token, "...") == 0) {
		return (T_ELLIPSIS);
	}

	/* search for reserved words first */
	for (i = 0; keywords[i].name; i++) {
		int j;
		if (strcmp(keywords[i].name, token) != 0) {
			continue;
		}

		last_kw = keywords[i].id;

		/* clear the top level category if we're done with it */
		if (last_kw == T_END) {
			category = T_END;
		}

		/* set the top level category if we're changing */
		for (j = 0; categories[j]; j++) {
			if (categories[j] != last_kw)
				continue;
			category = last_kw;
		}

		return (keywords[i].id);
	}

	/* maybe its a numeric constant? */
	if (isdigit(*token) || (*token == '-' && isdigit(token[1]))) {
		char *eptr;
		yylval.num = strtol(token, &eptr, 10);
		if (*eptr != 0)
			yyerror(_("malformed number"));
		return (T_NUMBER);
	}

	/*
	 * A single lone character is treated as a character literal.
	 * To avoid duplication of effort, we stick in the charmap.
	 */
	if (len == 1) {
		yylval.mbs[0] = 1; /* length */
		yylval.mbs[1] = token[0];
		yylval.mbs[2] = '\0';
		return (T_CHAR);
	}

	/* anything else is treated as a symbolic name */
	yylval.token = strdup(token);
	token = NULL;
	toksz = 0;
	tokidx = 0;
	return (T_NAME);
}

void
scan_to_eol(void)
{
	int	c;
	while ((c = scanc()) != '\n') {
		if (c == EOF) {
			/* end of file without newline! */
			errf(_("missing newline"));
			return;
		}
	}
	assert(c == '\n');
}

int
yylex(void)
{
	int		c;

	while ((c = scanc()) != EOF) {

		/* special handling for quoted string */
		if (instring) {
			if (escaped) {
				escaped = 0;

				/* if newline, just eat and forget it */
				if (c == '\n')
					continue;

				if (strchr("xXd01234567", c)) {
					unscanc(c);
					unscanc(esc_char);
					return (get_wide());
				}
				yylval.mbs[0] = 1; /* length */
				yylval.mbs[1] = get_escaped(c);
				yylval.mbs[2] = '\0';
				return (T_CHAR);
			}
			if (c == esc_char) {
				escaped = 1;
				continue;
			}
			switch (c) {
			case '<':
				return (get_symbol());
			case '>':
				/* oops! should generate syntax error  */
				return (T_GT);
			case '"':
				instring = 0;
				return (T_QUOTE);
			default:
				yylval.mbs[0] = 1; /* length */
				yylval.mbs[1] = c;
				yylval.mbs[2] = '\0';
				return (T_CHAR);
			}
		}

		/* escaped characters first */
		if (escaped) {
			escaped = 0;
			if (c == '\n') {
				/* eat the newline */
				continue;
			}
			hadtok = 1;
			if (tokidx) {
				/* an escape mid-token is nonsense */
				return (T_NULL);
			}

			/* numeric escapes are treated as wide characters */
			if (strchr("xXd01234567", c)) {
				unscanc(c);
				unscanc(esc_char);
				return (get_wide());
			}

			add_tok(get_escaped(c));
			continue;
		}

		/* if it is the escape charter itself note it */
		if (c == esc_char) {
			escaped = 1;
			continue;
		}

		/* remove from the comment char to end of line */
		if (c == com_char) {
			while (c != '\n') {
				if ((c = scanc()) == EOF) {
					/* end of file without newline! */
					return (EOF);
				}
			}
			assert(c == '\n');
			if (!hadtok) {
				/*
				 * If there were no tokens on this line,
				 * then just pretend it didn't exist at all.
				 */
				continue;
			}
			hadtok = 0;
			return (T_NL);
		}

		if (strchr(" \t\n;()<>,\"", c) && (tokidx != 0)) {
			/*
			 * These are all token delimiters.  If there
			 * is a token already in progress, we need to
			 * process it.
			 */
			unscanc(c);
			return (consume_token());
		}

		switch (c) {
		case '\n':
			if (!hadtok) {
				/*
				 * If the line was completely devoid of tokens,
				 * then just ignore it.
				 */
				continue;
			}
			/* we're starting a new line, reset the token state */
			hadtok = 0;
			return (T_NL);
		case ',':
			hadtok = 1;
			return (T_COMMA);
		case ';':
			hadtok = 1;
			return (T_SEMI);
		case '(':
			hadtok = 1;
			return (T_LPAREN);
		case ')':
			hadtok = 1;
			return (T_RPAREN);
		case '>':
			hadtok = 1;
			return (T_GT);
		case '<':
			/* symbol start! */
			hadtok = 1;
			return (get_symbol());
		case ' ':
		case '\t':
			/* whitespace, just ignore it */
			continue;
		case '"':
			hadtok = 1;
			instring = 1;
			return (T_QUOTE);
		default:
			hadtok = 1;
			add_tok(c);
			continue;
		}
	}
	return (EOF);
}

void
yyerror(const char *msg)
{
	(void) fprintf(stderr, _("%s: %d: error: %s\n"),
	    filename, lineno, msg);
	exit(1);
}

void
errf(const char *fmt, ...)
{
	char	*msg;

	va_list	va;
	va_start(va, fmt);
	(void) vasprintf(&msg, fmt, va);
	va_end(va);

	(void) fprintf(stderr, _("%s: %d: error: %s\n"),
	    filename, lineno, msg);
	free(msg);
	exit(1);
}

void
warn(const char *fmt, ...)
{
	char	*msg;

	va_list	va;
	va_start(va, fmt);
	(void) vasprintf(&msg, fmt, va);
	va_end(va);

	(void) fprintf(stderr, _("%s: %d: warning: %s\n"),
	    filename, lineno, msg);
	free(msg);
	warnings++;
}