1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  *
25  * esclex.c -- lexer for esc
26  *
27  * this module provides lexical analysis and error handling routine
28  * expected by the yacc-generated parser (i.e. yylex() and yyerror()).
29  * it also does lots of tracking of things like filenames, line numbers,
30  * and what tokens are seen on a line up to the point where a syntax error
31  * was found.  this module also arranges for the input source files to
32  * be run through cpp.
33  */
34 
35 #pragma ident	"%Z%%M%	%I%	%E% SMI"
36 
37 #include <stdio.h>
38 #include <ctype.h>
39 #include <string.h>
40 #include <stdlib.h>
41 #include <unistd.h>
42 #include <time.h>
43 #include <errno.h>
44 #include "out.h"
45 #include "alloc.h"
46 #include "stats.h"
47 #include "stable.h"
48 #include "lut.h"
49 #include "literals.h"
50 #include "tree.h"
51 #include "esclex.h"
52 #include "eftread.h"
53 #include "check.h"
54 #include "y.tab.h"
55 
56 /* ridiculously long token buffer -- disallow any token longer than this */
57 #define	MAXTOK	8192
58 static char Tok[MAXTOK];
59 
60 /* some misc stats we keep on the lexer & parser */
61 static struct stats *Tokcount;
62 static struct stats *Lexelapse;
63 struct stats *Filecount;
64 struct filestats {
65 	struct filestats *next;
66 	struct stats *stats;
67 	struct stats *idstats;
68 } *Fstats;
69 
70 static int Errcount;
71 
72 /* input file state */
73 static char **Files;
74 static const char *Fileopened;
75 static FILE *Fp;
76 static int Line;
77 static const char *File;
78 static const char *Cpp = "/usr/bin/cpp";
79 #ifdef	ESC
80 static const char *Cppargs;
81 static const char *Cppstdargs = "-undef -Y.";
82 #endif	/* ESC */
83 
84 /* for debugging */
85 static int Lexecho;	/* echo tokens as we read them */
86 
87 /* forward declarations of our internal routines */
88 static int record(int tok, const char *s);
89 static void dumpline(int flags);
90 static void doident();
91 static void dopragma(const char *tok);
92 
93 /*
94  * table of reserved words.  this table is only used by lex_init()
95  * to intialize the Rwords lookup table.
96  */
97 static const struct {
98 	const char *word;
99 	const int val;
100 } Rwords[] = {
101 	{ "asru", ASRU },
102 	{ "count", COUNT },
103 	{ "div", DIV },
104 	{ "engine", ENGINE },
105 	{ "event", EVENT },
106 	{ "fru", FRU },
107 	{ "if", IF },
108 	{ "mask", MASK },
109 	{ "prop", PROP },
110 	{ "config", CONFIG },
111 	/*
112 	 * PATHFUNC indicates functions that operate only on paths
113 	 * and quotes
114 	 */
115 	{ "is_connected", PATHFUNC },
116 	{ "is_under", PATHFUNC },
117 	{ "is_on", PATHFUNC },
118 	{ "is_present", PATHFUNC },
119 	{ "is_type", PATHFUNC },
120 	{ "confprop", PATHFUNC },
121 	{ "confprop_defined", PATHFUNC },
122 };
123 
124 /*
125  * Rwordslut is a lookup table of reserved words.  lhs is the word
126  * (in the string table) and the rhs is the token value returned
127  * by the yylex() for that word.
128  */
129 static struct lut *Rwordslut;
130 
131 static const struct {
132 	const char *suffix;
133 	const unsigned long long nsec;
134 } Timesuffix[] = {
135 	{ "nanosecond",		1ULL },
136 	{ "nanoseconds",	1ULL },
137 	{ "nsec",		1ULL },
138 	{ "nsecs",		1ULL },
139 	{ "ns",			1ULL },
140 	{ "microsecond",	1000ULL },
141 	{ "microseconds",	1000ULL },
142 	{ "usec",		1000ULL },
143 	{ "usecs",		1000ULL },
144 	{ "us",			1000ULL },
145 	{ "millisecond",	1000000ULL },
146 	{ "milliseconds",	1000000ULL },
147 	{ "msec",		1000000ULL },
148 	{ "msecs",		1000000ULL },
149 	{ "ms",			1000000ULL },
150 	{ "second",		1000000000ULL },
151 	{ "seconds",		1000000000ULL },
152 	{ "s",			1000000000ULL },
153 	{ "minute",		1000000000ULL * 60 },
154 	{ "minutes",		1000000000ULL * 60 },
155 	{ "min",		1000000000ULL * 60 },
156 	{ "mins",		1000000000ULL * 60 },
157 	{ "m",			1000000000ULL * 60 },
158 	{ "hour",		1000000000ULL * 60 * 60 },
159 	{ "hours",		1000000000ULL * 60 * 60 },
160 	{ "hr",			1000000000ULL * 60 * 60 },
161 	{ "hrs",		1000000000ULL * 60 * 60 },
162 	{ "h",			1000000000ULL * 60 * 60 },
163 	{ "day",		1000000000ULL * 60 * 60 * 24 },
164 	{ "days",		1000000000ULL * 60 * 60 * 24 },
165 	{ "d",			1000000000ULL * 60 * 60 * 24 },
166 	{ "week",		1000000000ULL * 60 * 60 * 24 * 7 },
167 	{ "weeks",		1000000000ULL * 60 * 60 * 24 * 7 },
168 	{ "wk",			1000000000ULL * 60 * 60 * 24 * 7 },
169 	{ "wks",		1000000000ULL * 60 * 60 * 24 * 7 },
170 	{ "month",		1000000000ULL * 60 * 60 * 24 * 30 },
171 	{ "months",		1000000000ULL * 60 * 60 * 24 * 30 },
172 	{ "year",		1000000000ULL * 60 * 60 * 24 * 365 },
173 	{ "years",		1000000000ULL * 60 * 60 * 24 * 365 },
174 	{ "yr",			1000000000ULL * 60 * 60 * 24 * 365 },
175 	{ "yrs",		1000000000ULL * 60 * 60 * 24 * 365 },
176 };
177 
178 /*
179  * some wrappers around the general lut functions to provide type checking...
180  */
181 
182 static struct lut *
183 lex_s2i_lut_add(struct lut *root, const char *s, intptr_t i)
184 {
185 	return (lut_add(root, (void *)s, (void *)i, NULL));
186 }
187 
188 static int
189 lex_s2i_lut_lookup(struct lut *root, const char *s)
190 {
191 	return ((intptr_t)lut_lookup(root, (void *)s, NULL));
192 }
193 
194 static struct lut *
195 lex_s2ullp_lut_add(struct lut *root, const char *s,
196     const unsigned long long *ullp)
197 {
198 	return (lut_add(root, (void *)s, (void *)ullp, NULL));
199 }
200 
201 const unsigned long long *
202 lex_s2ullp_lut_lookup(struct lut *root, const char *s)
203 {
204 	return ((unsigned long long *)lut_lookup(root, (void *)s, NULL));
205 }
206 
207 /*
208  * lex_init -- initialize the lexer with appropriate filenames & debug flags
209  */
210 
211 /*ARGSUSED*/
212 void
213 lex_init(char **av, const char *cppargs, int lexecho)
214 {
215 	int i;
216 #ifdef	ESC
217 	const char *ptr;
218 #endif	/* ESC */
219 
220 	Lexecho = lexecho;
221 	Tokcount = stats_new_counter("lex.tokens", "total tokens in", 1);
222 	Filecount = stats_new_counter("lex.files", "total files read", 0);
223 	Lexelapse = stats_new_elapse("lex.time", "elapsed lex/parse time", 1);
224 
225 #ifdef	ESC
226 	Cppargs = cppargs;
227 
228 	/* allow user to tell us where cpp is if it is some weird place */
229 	if (ptr = getenv("_ESC_CPP"))
230 		Cpp = ptr;
231 
232 	/* and in case it takes some special stdargs */
233 	if (ptr = getenv("_ESC_CPP_STDARGS"))
234 		Cppstdargs = ptr;
235 
236 	/* verify we can find cpp */
237 	if (access(Cpp, X_OK) < 0) {
238 		Cpp = "/usr/lib/cpp";
239 		if (access(Cpp, X_OK) < 0)
240 			out(O_DIE, "can't locate cpp");
241 	}
242 #endif	/* ESC */
243 
244 	Files = av;
245 
246 	/* verify we can find all the input files */
247 	while (*av) {
248 		if (strlen(*av) >= MAXTOK - strlen(Cpp) - 3)
249 			out(O_DIE, "filename too long: %.100s...", *av);
250 		if (access(*av, R_OK) < 0)
251 			out(O_DIE|O_SYS, "%s", *av);
252 		av++;
253 		stats_counter_bump(Filecount);
254 	}
255 
256 	/* put reserved words into the string table & a lookup table */
257 	for (i = 0; i < sizeof (Rwords) / sizeof (*Rwords); i++)
258 		Rwordslut = lex_s2i_lut_add(Rwordslut,
259 		    stable(Rwords[i].word), Rwords[i].val);
260 
261 	/* initialize table of timeval suffixes */
262 	for (i = 0; i < sizeof (Timesuffix) / sizeof (*Timesuffix); i++) {
263 		Timesuffixlut = lex_s2ullp_lut_add(Timesuffixlut,
264 		    stable(Timesuffix[i].suffix), &Timesuffix[i].nsec);
265 	}
266 
267 	/* record start time */
268 	stats_elapse_start(Lexelapse);
269 }
270 
271 void
272 closefile(void)
273 {
274 	if (Fp != NULL) {
275 #ifdef	ESC
276 		if (pclose(Fp) > 0)
277 			out(O_DIE, "cpp errors while reading \"%s\", "
278 			    "bailing out.", Fileopened);
279 #else
280 		(void) fclose(Fp);
281 #endif	/* ESC */
282 	}
283 	Fp = NULL;
284 }
285 
286 /*
287  * yylex -- the lexer, called yylex() because that's what yacc wants
288  */
289 
290 int
291 yylex()
292 {
293 	int c;
294 	int nextc;
295 	char *ptr = Tok;
296 	char *eptr = &Tok[MAXTOK];
297 	const char *cptr;
298 	int startline;
299 	int val;
300 	static int bol = 1;	/* true if we're at beginning of line */
301 
302 	for (;;) {
303 		while (Fp == NULL) {
304 			char ibuf[80];
305 
306 			if (*Files == NULL)
307 				return (record(EOF, NULL));
308 			Fileopened = stable(*Files++);
309 #ifdef	ESC
310 			sprintf(Tok, "%s %s %s %s",
311 			    Cpp, Cppstdargs, Cppargs, Fileopened);
312 			if ((Fp = popen(Tok, "r")) == NULL)
313 				out(O_DIE|O_SYS, "%s", Tok);
314 #else
315 			Fp = eftread_fopen(Fileopened, ibuf, sizeof (ibuf));
316 #endif	/* ESC */
317 			Line = 1;
318 			bol = 1;
319 
320 			/* add name to stats for visibility */
321 			if (Fp != NULL) {
322 				static int fnum;
323 				char nbuf[100];
324 				struct filestats *nfs = MALLOC(sizeof (*nfs));
325 
326 				(void) sprintf(nbuf, "lex.file%d", fnum);
327 				nfs->stats = stats_new_string(nbuf, "", 0);
328 				stats_string_set(nfs->stats, Fileopened);
329 
330 				if (ibuf[0] != '\0') {
331 					(void) sprintf(nbuf, "lex.file%d-ident",
332 					    fnum);
333 					nfs->idstats =
334 					    stats_new_string(nbuf, "", 0);
335 					stats_string_set(nfs->idstats, ibuf);
336 				} else {
337 					nfs->idstats = NULL;
338 				}
339 
340 				nfs->next = Fstats;
341 				Fstats = nfs;
342 				fnum++;
343 			}
344 		}
345 
346 		switch (c = getc(Fp)) {
347 		case '#':
348 			/* enforce that we're at beginning of line */
349 			if (!bol)
350 				return (record(c, NULL));
351 
352 			while ((c = getc(Fp)) != EOF &&
353 			    (c == ' ' || c == '\t'))
354 				;
355 			if (!isdigit(c)) {
356 				/*
357 				 * three cases here:
358 				 *	#pragma
359 				 *	#ident
360 				 *	#something-we-don't-understand
361 				 * anything we don't expect we just ignore.
362 				 */
363 				*ptr++ = c;
364 				while ((c = getc(Fp)) != EOF && isalnum(c))
365 					if (ptr < eptr - 1)
366 						*ptr++ = c;
367 				*ptr++ = '\0';
368 				if (strcmp(Tok, "pragma") == 0) {
369 					/* skip white space */
370 					while ((c = getc(Fp)) != EOF &&
371 					    (c == ' ' || c == '\t'))
372 						;
373 
374 					if (c == EOF || c == '\n')
375 						outfl(O_DIE, File, Line,
376 						    "bad #pragma");
377 
378 					/* pull in next token */
379 					ptr = Tok;
380 					*ptr++ = c;
381 					while ((c = getc(Fp)) != EOF &&
382 					    !isspace(c))
383 						if (ptr < eptr - 1)
384 							*ptr++ = c;
385 					*ptr++ = '\0';
386 					(void) ungetc(c, Fp);
387 
388 					dopragma(Tok);
389 				} else if (strcmp(Tok, "ident") == 0)
390 					doident();
391 			} else {
392 				/* handle file & line info from cpp */
393 				Line = 0;
394 				do {
395 					if (!isdigit(c))
396 						break;
397 					Line = Line * 10 + c - '0';
398 				} while ((c = getc(Fp)) != EOF);
399 				Line--;		/* newline will increment it */
400 				while (c != EOF && isspace(c))
401 					c = getc(Fp);
402 				if (c != '"')
403 					outfl(O_DIE, File, Line,
404 					    "bad # statement (file name)");
405 				while ((c = getc(Fp)) != EOF && c != '"')
406 					if (ptr < eptr - 1)
407 						*ptr++ = c;
408 				*ptr++ = '\0';
409 				if (c != '"')
410 					outfl(O_DIE, File, Line,
411 					    "bad # statement (quotes)");
412 				File = stable(Tok);
413 			}
414 			/* skip the rest of the cpp line */
415 			while ((c = getc(Fp)) != EOF && c != '\n' && c != '\r')
416 				;
417 			if (c == EOF)
418 				return (record(c, NULL));
419 			else
420 				(void) ungetc(c, Fp);
421 			ptr = Tok;
422 			break;
423 
424 		case EOF:
425 			closefile();
426 			continue;
427 
428 		case '\n':
429 			Line++;
430 			bol = 1;
431 			break;
432 
433 		case '\r':
434 		case ' ':
435 		case '\t':
436 			bol = 0;
437 			break;
438 
439 		case '/':
440 			bol = 0;
441 			/* comment handling */
442 			if ((nextc = getc(Fp)) == EOF)
443 				outfl(O_DIE, File, Line, "unexpected EOF");
444 			else if (nextc == '*') {
445 				startline = Line;
446 				while ((c = getc(Fp)) != EOF) {
447 					if (c == '\n')
448 						Line++;
449 					else if (c == '*' &&
450 					    (((c = getc(Fp)) == EOF) ||
451 					    (c == '/')))
452 						break;
453 				}
454 				if (c == EOF) {
455 					outfl(O_DIE, File, Line,
456 					    "end of comment not seen "
457 					    "(started on line %d)",
458 					    startline);
459 				}
460 			} else {
461 				/* wasn't a comment, return the '/' token */
462 				(void) ungetc(nextc, Fp);
463 				return (record(c, NULL));
464 			}
465 			break;
466 
467 		case '"': {
468 			int prevc;
469 
470 			bol = 0;
471 			prevc = '\0';
472 			/* quoted string handling */
473 			startline = Line;
474 			for (;;) {
475 				c = getc(Fp);
476 				if (c == EOF)
477 					outfl(O_DIE, File, Line,
478 					    "end of string not seen "
479 					    "(started on line %d)",
480 					    startline);
481 				else if (c == '\n')
482 					Line++;
483 				else if (c == '"' && prevc != '\\')
484 					break;
485 				else if (ptr < eptr)
486 					*ptr++ = c;
487 				prevc = c;
488 			}
489 			if (ptr >= eptr)
490 				out(O_DIE, File, Line, "string too long");
491 			*ptr++ = '\0';
492 			return (record(QUOTE, stable(Tok)));
493 		}
494 		case '&':
495 			bol = 0;
496 			/* && */
497 			if ((nextc = getc(Fp)) == '&')
498 				return (record(AND, NULL));
499 			else {
500 				(void) ungetc(nextc, Fp);
501 				return (record(c, NULL));
502 			}
503 			/*NOTREACHED*/
504 			break;
505 
506 		case '|':
507 			bol = 0;
508 			/* || */
509 			if ((nextc = getc(Fp)) == '|')
510 				return (record(OR, NULL));
511 			else {
512 				(void) ungetc(nextc, Fp);
513 				return (record(c, NULL));
514 			}
515 			/*NOTREACHED*/
516 			break;
517 
518 		case '!':
519 			bol = 0;
520 			/* ! or != */
521 			if ((nextc = getc(Fp)) == '=')
522 				return (record(NE, NULL));
523 			else {
524 				(void) ungetc(nextc, Fp);
525 				return (record(c, NULL));
526 			}
527 			/*NOTREACHED*/
528 			break;
529 
530 		case '=':
531 			bol = 0;
532 			/* == */
533 			if ((nextc = getc(Fp)) == '=')
534 				return (record(EQ, NULL));
535 			else {
536 				(void) ungetc(nextc, Fp);
537 				return (record(c, NULL));
538 			}
539 			/*NOTREACHED*/
540 			break;
541 
542 		case '-':
543 			bol = 0;
544 			/* -> */
545 			if ((nextc = getc(Fp)) == '>')
546 				return (record(ARROW, stable(Tok)));
547 			else {
548 				(void) ungetc(nextc, Fp);
549 				return (record(c, NULL));
550 			}
551 			/*NOTREACHED*/
552 			break;
553 
554 		case '<':
555 			bol = 0;
556 			if ((nextc = getc(Fp)) == '=')
557 				/* <= */
558 				return (record(LE, NULL));
559 			else if (nextc == '<')
560 				/* << */
561 				return (record(LSHIFT, NULL));
562 			else {
563 				(void) ungetc(nextc, Fp);
564 				return (record(c, NULL));
565 			}
566 			/*NOTREACHED*/
567 			break;
568 
569 		case '>':
570 			bol = 0;
571 			if ((nextc = getc(Fp)) == '=')
572 				/* >= */
573 				return (record(GE, NULL));
574 			else if (nextc == '>')
575 				/* >> */
576 				return (record(RSHIFT, NULL));
577 			else {
578 				(void) ungetc(nextc, Fp);
579 				return (record(c, NULL));
580 			}
581 			/*NOTREACHED*/
582 			break;
583 
584 		default:
585 			bol = 0;
586 			if (isdigit(c)) {
587 				int base;
588 
589 				/* collect rest of number */
590 				if (c == '0') {
591 					*ptr++ = c;
592 					if ((c = getc(Fp)) == EOF) {
593 						*ptr++ = '\0';
594 						return (record(NUMBER,
595 						    stable(Tok)));
596 					} else if (c == 'x' || c == 'X') {
597 						*ptr++ = c;
598 						base = 16;
599 					} else {
600 						(void) ungetc(c, Fp);
601 						base = 8;
602 					}
603 				} else {
604 					*ptr++ = c;
605 					base = 10;
606 				}
607 				while ((c = getc(Fp)) != EOF) {
608 					if (ptr >= eptr)
609 						out(O_DIE, File, Line,
610 						    "number too long");
611 
612 					switch (base) {
613 					case 16:
614 						if (c >= 'a' && c <= 'f' ||
615 						    c >= 'A' && c <= 'F') {
616 							*ptr++ = c;
617 							continue;
618 						}
619 						/*FALLTHRU*/
620 					case 10:
621 						if (c >= '8' && c <= '9') {
622 							*ptr++ = c;
623 							continue;
624 						}
625 						/*FALLTHRU*/
626 					case 8:
627 						if (c >= '0' && c <= '7') {
628 							*ptr++ = c;
629 							continue;
630 						}
631 						/* not valid for this base */
632 						*ptr++ = '\0';
633 						(void) ungetc(c, Fp);
634 						return (record(NUMBER,
635 						    stable(Tok)));
636 					}
637 				}
638 				*ptr++ = '\0';
639 				return (record(NUMBER, stable(Tok)));
640 			} else if (isalpha(c)) {
641 				/* collect identifier */
642 				*ptr++ = c;
643 				for (;;) {
644 					c = getc(Fp);
645 					if ((isalnum(c) || c == '_') &&
646 					    ptr < eptr)
647 						*ptr++ = c;
648 					else {
649 						(void) ungetc(c, Fp);
650 						break;
651 					}
652 				}
653 				if (ptr >= eptr)
654 					out(O_DIE, File, Line,
655 					    "identifier too long");
656 				*ptr++ = '\0';
657 				cptr = stable(Tok);
658 				if (val = lex_s2i_lut_lookup(Rwordslut, cptr)) {
659 					return (record(val, cptr));
660 				}
661 				return (record(ID, cptr));
662 			} else
663 				return (record(c, NULL));
664 		}
665 		/*NOTREACHED*/
666 	}
667 }
668 
669 /*
670  * the record()/dumpline() routines are used to track & report
671  * the list of tokens seen on a given line.  this is used in two ways.
672  * first, syntax errors found by the parser are reported by us (via
673  * yyerror()) and we tack on the tokens processed so far on the current
674  * line to help indicate exactly where the error is.  second, if "lexecho"
675  * debugging is turned on, these routines provide it.
676  */
677 #define	MAXRECORD 1000
678 static int Recordedline;
679 static struct {
680 	int tok;
681 	const char *s;
682 } Recorded[MAXRECORD];
683 static int Recordnext;
684 
685 static int
686 record(int tok, const char *s)
687 {
688 	stats_counter_bump(Tokcount);
689 	if (Line != Recordedline) {
690 		/* starting new line, dump out the previous line */
691 		if (Lexecho && Recordedline) {
692 			outfl(O_NONL, File, Recordedline, "lex: ");
693 			dumpline(O_OK);
694 		}
695 		Recordedline = Line;
696 		Recordnext = 0;
697 	}
698 	if (Recordnext >= MAXRECORD)
699 		outfl(O_DIE, File, Line, "line too long, bailing out");
700 	Recorded[Recordnext].tok = tok;
701 	Recorded[Recordnext++].s = s;
702 
703 	yylval.tok.s = s;
704 	yylval.tok.file = File;
705 	yylval.tok.line = Line;
706 	return (tok);
707 }
708 
709 /*ARGSUSED*/
710 static void
711 dumpline(int flags)
712 {
713 	int i;
714 
715 	for (i = 0; i < Recordnext; i++)
716 		if (Recorded[i].s && Recorded[i].tok != ARROW)
717 			switch (Recorded[i].tok) {
718 			case T_QUOTE:
719 				out(flags|O_NONL, " \"%s\"",
720 				    Recorded[i].s);
721 				break;
722 
723 			default:
724 				out(flags|O_NONL, " %s",
725 				    Recorded[i].s);
726 				break;
727 			}
728 		else
729 			switch (Recorded[i].tok) {
730 			case EOF:
731 				out(flags|O_NONL, " EOF");
732 				break;
733 			case ARROW:
734 				out(flags|O_NONL, " ->%s",
735 				    Recorded[i].s);
736 				break;
737 			case EQ:
738 				out(flags|O_NONL, " ==");
739 				break;
740 			case NE:
741 				out(flags|O_NONL, " !=");
742 				break;
743 			case OR:
744 				out(flags|O_NONL, " ||");
745 				break;
746 			case AND:
747 				out(flags|O_NONL, " &&");
748 				break;
749 			case LE:
750 				out(flags|O_NONL, " <=");
751 				break;
752 			case GE:
753 				out(flags|O_NONL, " >=");
754 				break;
755 			case LSHIFT:
756 				out(flags|O_NONL, " <<");
757 				break;
758 			case RSHIFT:
759 				out(flags|O_NONL, " >>");
760 				break;
761 			default:
762 				if (isprint(Recorded[i].tok))
763 					out(flags|O_NONL, " %c",
764 					    Recorded[i].tok);
765 				else
766 					out(flags|O_NONL, " '\\%03o'",
767 					    Recorded[i].tok);
768 				break;
769 			}
770 	out(flags, NULL);
771 }
772 
773 /*
774  * yyerror -- report a pareser error, called yyerror because yacc wants it
775  */
776 
777 void
778 yyerror(const char *s)
779 {
780 	Errcount++;
781 	outfl(O_ERR|O_NONL, File, Line, "%s, tokens: ", s);
782 	dumpline(O_ERR);
783 }
784 
785 /*
786  * doident -- handle "#pragma ident" directives
787  */
788 static void
789 doident()
790 {
791 	int c;
792 	char *ptr = Tok;
793 	char *eptr = &Tok[MAXTOK];
794 
795 	/* skip white space and quotes */
796 	while ((c = getc(Fp)) != EOF &&
797 	    (c == ' ' || c == '\t' || c == '"'))
798 		;
799 
800 	if (c == EOF || c == '\n')
801 		outfl(O_DIE, File, Line, "bad ident");
802 
803 	/* pull in next token */
804 	ptr = Tok;
805 	*ptr++ = c;
806 	while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
807 		if (ptr < eptr - 1)
808 			*ptr++ = c;
809 	*ptr++ = '\0';
810 	if (c != '\n') {
811 		/* skip to end of line (including close quote, if any) */
812 		while ((c = getc(Fp)) != EOF && c != '\n')
813 			;
814 	}
815 	(void) ungetc(c, Fp);
816 	Ident = lut_add(Ident, (void *)stable(Tok), (void *)0, NULL);
817 
818 	outfl(O_VERB, File, Line, "pragma set: ident \"%s\"", Tok);
819 }
820 
821 /*
822  * dodictionary -- handle "#pragma dictionary" directives
823  */
824 static void
825 dodictionary()
826 {
827 	int c;
828 	char *ptr = Tok;
829 	char *eptr = &Tok[MAXTOK];
830 
831 	/* skip white space and quotes */
832 	while ((c = getc(Fp)) != EOF &&
833 	    (c == ' ' || c == '\t' || c == '"'))
834 		;
835 
836 	if (c == EOF || c == '\n')
837 		outfl(O_DIE, File, Line, "bad dictionary");
838 
839 	/* pull in next token */
840 	ptr = Tok;
841 	*ptr++ = c;
842 	while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
843 		if (ptr < eptr - 1)
844 			*ptr++ = c;
845 	*ptr++ = '\0';
846 	if (c != '\n') {
847 		/* skip to end of line (including close quote, if any) */
848 		while ((c = getc(Fp)) != EOF && c != '\n')
849 			;
850 	}
851 	(void) ungetc(c, Fp);
852 	Dicts = lut_add(Dicts, (void *)stable(Tok), (void *)0, NULL);
853 
854 	outfl(O_VERB, File, Line, "pragma set: dictionary \"%s\"", Tok);
855 }
856 
857 /*
858  * doallow_cycles -- handle "#pragma allow_cycles" directives
859  */
860 static void
861 doallow_cycles()
862 {
863 	int c;
864 	char *ptr = Tok;
865 	char *eptr = &Tok[MAXTOK];
866 	unsigned long long newlevel;
867 
868 	/*
869 	 * by default the compiler does not allow cycles or loops
870 	 * in propagations.  when cycles are encountered, the
871 	 * compiler prints out an error message.
872 	 *
873 	 *   "#pragma allow_cycles" and
874 	 *   "#pragma allow_cycles 0"
875 	 * allow cycles, but any such cycle will produce a warning
876 	 * message.
877 	 *
878 	 *   "#pragma allow_cycles N"
879 	 * with N > 0 will allow cycles and not produce any
880 	 * warning messages.
881 	 */
882 
883 	/* skip white space and quotes */
884 	while ((c = getc(Fp)) != EOF &&
885 	    (c == ' ' || c == '\t' || c == '"'))
886 		;
887 
888 	if (c == EOF || c == '\n')
889 		newlevel = 0ULL;
890 	else {
891 
892 		/* pull in next token */
893 		ptr = Tok;
894 		*ptr++ = c;
895 		while ((c = getc(Fp)) != EOF && c != '"' && c != '\n')
896 			if (ptr < eptr - 1)
897 				*ptr++ = c;
898 		*ptr++ = '\0';
899 		if (c != '\n') {
900 			/* skip to end of line */
901 			while ((c = getc(Fp)) != EOF && c != '\n')
902 				;
903 		}
904 		newlevel = strtoll(Tok, NULL, 0);
905 	}
906 	(void) ungetc(c, Fp);
907 
908 	(void) check_cycle_level(newlevel);
909 	outfl(O_VERB, File, Line,
910 	    "pragma set: allow_cycles (%s)",
911 	    newlevel ? "no warnings" : "with warnings");
912 }
913 
914 /*
915  * dopragma -- handle #pragma directives
916  */
917 static void
918 dopragma(const char *tok)
919 {
920 	if (strcmp(tok, "ident") == 0)
921 		doident();
922 	else if (strcmp(tok, "dictionary") == 0)
923 		dodictionary();
924 	else if (strcmp(tok, "new_errors_only") == 0) {
925 		if (Pragma_new_errors_only++ == 0)
926 			outfl(O_VERB, File, Line,
927 			    "pragma set: new_errors_only");
928 	} else if (strcmp(tok, "trust_ereports") == 0) {
929 		if (Pragma_trust_ereports++ == 0)
930 			outfl(O_VERB, File, Line,
931 			    "pragma set: trust_ereports");
932 	} else if (strcmp(tok, "allow_cycles") == 0)
933 		doallow_cycles();
934 	else
935 		outfl(O_VERB, File, Line,
936 		    "unknown pragma ignored: \"%s\"", tok);
937 }
938 
939 /*
940  * lex_fini -- finalize the lexer
941  */
942 
943 int
944 lex_fini(void)
945 {
946 	stats_elapse_stop(Lexelapse);
947 	closefile();
948 	if (Lexecho) {
949 		outfl(O_OK, File, Line, "lex: ");
950 		dumpline(O_OK);
951 	}
952 	return (Errcount);
953 }
954 
955 void
956 lex_free(void)
957 {
958 	struct filestats *nfstats = Fstats;
959 
960 	/*
961 	 * Free up memory consumed by the lexer
962 	 */
963 	stats_delete(Tokcount);
964 	stats_delete(Filecount);
965 	stats_delete(Lexelapse);
966 	while (nfstats != NULL) {
967 		Fstats = nfstats->next;
968 		stats_delete(nfstats->stats);
969 		if (nfstats->idstats != NULL)
970 			stats_delete(nfstats->idstats);
971 		FREE(nfstats);
972 		nfstats = Fstats;
973 	}
974 	lut_free(Timesuffixlut, NULL, NULL);
975 	lut_free(Rwordslut, NULL, NULL);
976 	lut_free(Ident, NULL, NULL);
977 	lut_free(Dicts, NULL, NULL);
978 }
979