1/*
2 * Copyright (c) Ian F. Darwin 1986-1995.
3 * Software written by Ian F. Darwin and others;
4 * maintained 1995-present by Christos Zoulas and others.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice immediately at the beginning of the file, without modification,
11 *    this list of conditions, and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28/*
29 * apprentice - make one pass through /etc/magic, learning its secrets.
30 */
31
32#include "file.h"
33
34#ifndef	lint
35FILE_RCSID("@(#)$File: apprentice.c,v 1.284 2019/06/29 22:31:04 christos Exp $")
36#endif	/* lint */
37
38#include "magic.h"
39#include <stdlib.h>
40#ifdef HAVE_UNISTD_H
41#include <unistd.h>
42#endif
43#include <stddef.h>
44#include <string.h>
45#include <assert.h>
46#include <ctype.h>
47#include <fcntl.h>
48#ifdef QUICK
49#include <sys/mman.h>
50#endif
51#include <dirent.h>
52#include <limits.h>
53
54
55#define	EATAB {while (isascii(CAST(unsigned char, *l)) && \
56		      isspace(CAST(unsigned char, *l)))  ++l;}
57#define LOWCASE(l) (isupper(CAST(unsigned char, l)) ? \
58			tolower(CAST(unsigned char, l)) : (l))
59/*
60 * Work around a bug in headers on Digital Unix.
61 * At least confirmed for: OSF1 V4.0 878
62 */
63#if defined(__osf__) && defined(__DECC)
64#ifdef MAP_FAILED
65#undef MAP_FAILED
66#endif
67#endif
68
69#ifndef MAP_FAILED
70#define MAP_FAILED (void *) -1
71#endif
72
73#ifndef MAP_FILE
74#define MAP_FILE 0
75#endif
76
77#define ALLOC_CHUNK	CAST(size_t, 10)
78#define ALLOC_INCR	CAST(size_t, 200)
79
80#define MAP_TYPE_USER	0
81#define MAP_TYPE_MALLOC	1
82#define MAP_TYPE_MMAP	2
83
84struct magic_entry {
85	struct magic *mp;
86	uint32_t cont_count;
87	uint32_t max_count;
88};
89
90struct magic_entry_set {
91	struct magic_entry *me;
92	uint32_t count;
93	uint32_t max;
94};
95
96struct magic_map {
97	void *p;
98	size_t len;
99	int type;
100	struct magic *magic[MAGIC_SETS];
101	uint32_t nmagic[MAGIC_SETS];
102};
103
104int file_formats[FILE_NAMES_SIZE];
105const size_t file_nformats = FILE_NAMES_SIZE;
106const char *file_names[FILE_NAMES_SIZE];
107const size_t file_nnames = FILE_NAMES_SIZE;
108
109private int getvalue(struct magic_set *ms, struct magic *, const char **, int);
110private int hextoint(int);
111private const char *getstr(struct magic_set *, struct magic *, const char *,
112    int);
113private int parse(struct magic_set *, struct magic_entry *, const char *,
114    size_t, int);
115private void eatsize(const char **);
116private int apprentice_1(struct magic_set *, const char *, int);
117private size_t apprentice_magic_strength(const struct magic *);
118private int apprentice_sort(const void *, const void *);
119private void apprentice_list(struct mlist *, int );
120private struct magic_map *apprentice_load(struct magic_set *,
121    const char *, int);
122private struct mlist *mlist_alloc(void);
123private void mlist_free(struct mlist *);
124private void byteswap(struct magic *, uint32_t);
125private void bs1(struct magic *);
126private uint16_t swap2(uint16_t);
127private uint32_t swap4(uint32_t);
128private uint64_t swap8(uint64_t);
129private char *mkdbname(struct magic_set *, const char *, int);
130private struct magic_map *apprentice_buf(struct magic_set *, struct magic *,
131    size_t);
132private struct magic_map *apprentice_map(struct magic_set *, const char *);
133private int check_buffer(struct magic_set *, struct magic_map *, const char *);
134private void apprentice_unmap(struct magic_map *);
135private int apprentice_compile(struct magic_set *, struct magic_map *,
136    const char *);
137private int check_format_type(const char *, int, const char **);
138private int check_format(struct magic_set *, struct magic *);
139private int get_op(char);
140private int parse_mime(struct magic_set *, struct magic_entry *, const char *);
141private int parse_strength(struct magic_set *, struct magic_entry *, const char *);
142private int parse_apple(struct magic_set *, struct magic_entry *, const char *);
143private int parse_ext(struct magic_set *, struct magic_entry *, const char *);
144
145
146private size_t magicsize = sizeof(struct magic);
147
148private const char usg_hdr[] = "cont\toffset\ttype\topcode\tmask\tvalue\tdesc";
149
150private struct {
151	const char *name;
152	size_t len;
153	int (*fun)(struct magic_set *, struct magic_entry *, const char *);
154} bang[] = {
155#define	DECLARE_FIELD(name) { # name, sizeof(# name) - 1, parse_ ## name }
156	DECLARE_FIELD(mime),
157	DECLARE_FIELD(apple),
158	DECLARE_FIELD(ext),
159	DECLARE_FIELD(strength),
160#undef	DECLARE_FIELD
161	{ NULL, 0, NULL }
162};
163
164#ifdef COMPILE_ONLY
165
166int main(int, char *[]);
167
168int
169main(int argc, char *argv[])
170{
171	int ret;
172	struct magic_set *ms;
173	char *progname;
174
175	if ((progname = strrchr(argv[0], '/')) != NULL)
176		progname++;
177	else
178		progname = argv[0];
179
180	if (argc != 2) {
181		(void)fprintf(stderr, "Usage: %s file\n", progname);
182		return 1;
183	}
184
185	if ((ms = magic_open(MAGIC_CHECK)) == NULL) {
186		(void)fprintf(stderr, "%s: %s\n", progname, strerror(errno));
187		return 1;
188	}
189	ret = magic_compile(ms, argv[1]) == -1 ? 1 : 0;
190	if (ret == 1)
191		(void)fprintf(stderr, "%s: %s\n", progname, magic_error(ms));
192	magic_close(ms);
193	return ret;
194}
195#endif /* COMPILE_ONLY */
196
197struct type_tbl_s {
198	const char name[16];
199	const size_t len;
200	const int type;
201	const int format;
202};
203
204/*
205 * XXX - the actual Single UNIX Specification says that "long" means "long",
206 * as in the C data type, but we treat it as meaning "4-byte integer".
207 * Given that the OS X version of file 5.04 did the same, I guess that passes
208 * the actual test; having "long" be dependent on how big a "long" is on
209 * the machine running "file" is silly.
210 */
211static const struct type_tbl_s type_tbl[] = {
212# define XX(s)		s, (sizeof(s) - 1)
213# define XX_NULL	"", 0
214	{ XX("invalid"),	FILE_INVALID,		FILE_FMT_NONE },
215	{ XX("byte"),		FILE_BYTE,		FILE_FMT_NUM },
216	{ XX("short"),		FILE_SHORT,		FILE_FMT_NUM },
217	{ XX("default"),	FILE_DEFAULT,		FILE_FMT_NONE },
218	{ XX("long"),		FILE_LONG,		FILE_FMT_NUM },
219	{ XX("string"),		FILE_STRING,		FILE_FMT_STR },
220	{ XX("date"),		FILE_DATE,		FILE_FMT_STR },
221	{ XX("beshort"),	FILE_BESHORT,		FILE_FMT_NUM },
222	{ XX("belong"),		FILE_BELONG,		FILE_FMT_NUM },
223	{ XX("bedate"),		FILE_BEDATE,		FILE_FMT_STR },
224	{ XX("leshort"),	FILE_LESHORT,		FILE_FMT_NUM },
225	{ XX("lelong"),		FILE_LELONG,		FILE_FMT_NUM },
226	{ XX("ledate"),		FILE_LEDATE,		FILE_FMT_STR },
227	{ XX("pstring"),	FILE_PSTRING,		FILE_FMT_STR },
228	{ XX("ldate"),		FILE_LDATE,		FILE_FMT_STR },
229	{ XX("beldate"),	FILE_BELDATE,		FILE_FMT_STR },
230	{ XX("leldate"),	FILE_LELDATE,		FILE_FMT_STR },
231	{ XX("regex"),		FILE_REGEX,		FILE_FMT_STR },
232	{ XX("bestring16"),	FILE_BESTRING16,	FILE_FMT_STR },
233	{ XX("lestring16"),	FILE_LESTRING16,	FILE_FMT_STR },
234	{ XX("search"),		FILE_SEARCH,		FILE_FMT_STR },
235	{ XX("medate"),		FILE_MEDATE,		FILE_FMT_STR },
236	{ XX("meldate"),	FILE_MELDATE,		FILE_FMT_STR },
237	{ XX("melong"),		FILE_MELONG,		FILE_FMT_NUM },
238	{ XX("quad"),		FILE_QUAD,		FILE_FMT_QUAD },
239	{ XX("lequad"),		FILE_LEQUAD,		FILE_FMT_QUAD },
240	{ XX("bequad"),		FILE_BEQUAD,		FILE_FMT_QUAD },
241	{ XX("qdate"),		FILE_QDATE,		FILE_FMT_STR },
242	{ XX("leqdate"),	FILE_LEQDATE,		FILE_FMT_STR },
243	{ XX("beqdate"),	FILE_BEQDATE,		FILE_FMT_STR },
244	{ XX("qldate"),		FILE_QLDATE,		FILE_FMT_STR },
245	{ XX("leqldate"),	FILE_LEQLDATE,		FILE_FMT_STR },
246	{ XX("beqldate"),	FILE_BEQLDATE,		FILE_FMT_STR },
247	{ XX("float"),		FILE_FLOAT,		FILE_FMT_FLOAT },
248	{ XX("befloat"),	FILE_BEFLOAT,		FILE_FMT_FLOAT },
249	{ XX("lefloat"),	FILE_LEFLOAT,		FILE_FMT_FLOAT },
250	{ XX("double"),		FILE_DOUBLE,		FILE_FMT_DOUBLE },
251	{ XX("bedouble"),	FILE_BEDOUBLE,		FILE_FMT_DOUBLE },
252	{ XX("ledouble"),	FILE_LEDOUBLE,		FILE_FMT_DOUBLE },
253	{ XX("leid3"),		FILE_LEID3,		FILE_FMT_NUM },
254	{ XX("beid3"),		FILE_BEID3,		FILE_FMT_NUM },
255	{ XX("indirect"),	FILE_INDIRECT,		FILE_FMT_NUM },
256	{ XX("qwdate"),		FILE_QWDATE,		FILE_FMT_STR },
257	{ XX("leqwdate"),	FILE_LEQWDATE,		FILE_FMT_STR },
258	{ XX("beqwdate"),	FILE_BEQWDATE,		FILE_FMT_STR },
259	{ XX("name"),		FILE_NAME,		FILE_FMT_NONE },
260	{ XX("use"),		FILE_USE,		FILE_FMT_NONE },
261	{ XX("clear"),		FILE_CLEAR,		FILE_FMT_NONE },
262	{ XX("der"),		FILE_DER,		FILE_FMT_STR },
263	{ XX_NULL,		FILE_INVALID,		FILE_FMT_NONE },
264};
265
266/*
267 * These are not types, and cannot be preceded by "u" to make them
268 * unsigned.
269 */
270static const struct type_tbl_s special_tbl[] = {
271	{ XX("der"),		FILE_DER,		FILE_FMT_STR },
272	{ XX("name"),		FILE_NAME,		FILE_FMT_STR },
273	{ XX("use"),		FILE_USE,		FILE_FMT_STR },
274	{ XX_NULL,		FILE_INVALID,		FILE_FMT_NONE },
275};
276# undef XX
277# undef XX_NULL
278
279private int
280get_type(const struct type_tbl_s *tbl, const char *l, const char **t)
281{
282	const struct type_tbl_s *p;
283
284	for (p = tbl; p->len; p++) {
285		if (strncmp(l, p->name, p->len) == 0) {
286			if (t)
287				*t = l + p->len;
288			break;
289		}
290	}
291	return p->type;
292}
293
294private off_t
295maxoff_t(void) {
296	if (/*CONSTCOND*/sizeof(off_t) == sizeof(int))
297		return CAST(off_t, INT_MAX);
298	if (/*CONSTCOND*/sizeof(off_t) == sizeof(long))
299		return CAST(off_t, LONG_MAX);
300	return 0x7fffffff;
301}
302
303private int
304get_standard_integer_type(const char *l, const char **t)
305{
306	int type;
307
308	if (isalpha(CAST(unsigned char, l[1]))) {
309		switch (l[1]) {
310		case 'C':
311			/* "dC" and "uC" */
312			type = FILE_BYTE;
313			break;
314		case 'S':
315			/* "dS" and "uS" */
316			type = FILE_SHORT;
317			break;
318		case 'I':
319		case 'L':
320			/*
321			 * "dI", "dL", "uI", and "uL".
322			 *
323			 * XXX - the actual Single UNIX Specification says
324			 * that "L" means "long", as in the C data type,
325			 * but we treat it as meaning "4-byte integer".
326			 * Given that the OS X version of file 5.04 did
327			 * the same, I guess that passes the actual SUS
328			 * validation suite; having "dL" be dependent on
329			 * how big a "long" is on the machine running
330			 * "file" is silly.
331			 */
332			type = FILE_LONG;
333			break;
334		case 'Q':
335			/* "dQ" and "uQ" */
336			type = FILE_QUAD;
337			break;
338		default:
339			/* "d{anything else}", "u{anything else}" */
340			return FILE_INVALID;
341		}
342		l += 2;
343	} else if (isdigit(CAST(unsigned char, l[1]))) {
344		/*
345		 * "d{num}" and "u{num}"; we only support {num} values
346		 * of 1, 2, 4, and 8 - the Single UNIX Specification
347		 * doesn't say anything about whether arbitrary
348		 * values should be supported, but both the Solaris 10
349		 * and OS X Mountain Lion versions of file passed the
350		 * Single UNIX Specification validation suite, and
351		 * neither of them support values bigger than 8 or
352		 * non-power-of-2 values.
353		 */
354		if (isdigit(CAST(unsigned char, l[2]))) {
355			/* Multi-digit, so > 9 */
356			return FILE_INVALID;
357		}
358		switch (l[1]) {
359		case '1':
360			type = FILE_BYTE;
361			break;
362		case '2':
363			type = FILE_SHORT;
364			break;
365		case '4':
366			type = FILE_LONG;
367			break;
368		case '8':
369			type = FILE_QUAD;
370			break;
371		default:
372			/* XXX - what about 3, 5, 6, or 7? */
373			return FILE_INVALID;
374		}
375		l += 2;
376	} else {
377		/*
378		 * "d" or "u" by itself.
379		 */
380		type = FILE_LONG;
381		++l;
382	}
383	if (t)
384		*t = l;
385	return type;
386}
387
388private void
389init_file_tables(void)
390{
391	static int done = 0;
392	const struct type_tbl_s *p;
393
394	if (done)
395		return;
396	done++;
397
398	for (p = type_tbl; p->len; p++) {
399		assert(p->type < FILE_NAMES_SIZE);
400		file_names[p->type] = p->name;
401		file_formats[p->type] = p->format;
402	}
403	assert(p - type_tbl == FILE_NAMES_SIZE);
404}
405
406private int
407add_mlist(struct mlist *mlp, struct magic_map *map, size_t idx)
408{
409	struct mlist *ml;
410
411	mlp->map = NULL;
412	if ((ml = CAST(struct mlist *, malloc(sizeof(*ml)))) == NULL)
413		return -1;
414
415	ml->map = idx == 0 ? map : NULL;
416	ml->magic = map->magic[idx];
417	ml->nmagic = map->nmagic[idx];
418
419	mlp->prev->next = ml;
420	ml->prev = mlp->prev;
421	ml->next = mlp;
422	mlp->prev = ml;
423	return 0;
424}
425
426/*
427 * Handle one file or directory.
428 */
429private int
430apprentice_1(struct magic_set *ms, const char *fn, int action)
431{
432	struct magic_map *map;
433#ifndef COMPILE_ONLY
434	struct mlist *ml;
435	size_t i;
436#endif
437
438	if (magicsize != FILE_MAGICSIZE) {
439		file_error(ms, 0, "magic element size %lu != %lu",
440		    CAST(unsigned long, sizeof(*map->magic[0])),
441		    CAST(unsigned long, FILE_MAGICSIZE));
442		return -1;
443	}
444
445	if (action == FILE_COMPILE) {
446		map = apprentice_load(ms, fn, action);
447		if (map == NULL)
448			return -1;
449		return apprentice_compile(ms, map, fn);
450	}
451
452#ifndef COMPILE_ONLY
453	map = apprentice_map(ms, fn);
454	if (map == RCAST(struct magic_map *, -1))
455		return -1;
456	if (map == NULL) {
457		if (ms->flags & MAGIC_CHECK)
458			file_magwarn(ms, "using regular magic file `%s'", fn);
459		map = apprentice_load(ms, fn, action);
460		if (map == NULL)
461			return -1;
462	}
463
464	for (i = 0; i < MAGIC_SETS; i++) {
465		if (add_mlist(ms->mlist[i], map, i) == -1) {
466			file_oomem(ms, sizeof(*ml));
467			return -1;
468		}
469	}
470
471	if (action == FILE_LIST) {
472		for (i = 0; i < MAGIC_SETS; i++) {
473			printf("Set %" SIZE_T_FORMAT "u:\nBinary patterns:\n",
474			    i);
475			apprentice_list(ms->mlist[i], BINTEST);
476			printf("Text patterns:\n");
477			apprentice_list(ms->mlist[i], TEXTTEST);
478		}
479	}
480	return 0;
481#else
482	return 0;
483#endif /* COMPILE_ONLY */
484}
485
486protected void
487file_ms_free(struct magic_set *ms)
488{
489	size_t i;
490	if (ms == NULL)
491		return;
492	for (i = 0; i < MAGIC_SETS; i++)
493		mlist_free(ms->mlist[i]);
494	free(ms->o.pbuf);
495	free(ms->o.buf);
496	free(ms->c.li);
497	free(ms);
498}
499
500protected struct magic_set *
501file_ms_alloc(int flags)
502{
503	struct magic_set *ms;
504	size_t i, len;
505
506	if ((ms = CAST(struct magic_set *, calloc(CAST(size_t, 1u),
507	    sizeof(struct magic_set)))) == NULL)
508		return NULL;
509
510	if (magic_setflags(ms, flags) == -1) {
511		errno = EINVAL;
512		goto free;
513	}
514
515	ms->o.buf = ms->o.pbuf = NULL;
516	len = (ms->c.len = 10) * sizeof(*ms->c.li);
517
518	if ((ms->c.li = CAST(struct level_info *, malloc(len))) == NULL)
519		goto free;
520
521	ms->event_flags = 0;
522	ms->error = -1;
523	for (i = 0; i < MAGIC_SETS; i++)
524		ms->mlist[i] = NULL;
525	ms->file = "unknown";
526	ms->line = 0;
527	ms->indir_max = FILE_INDIR_MAX;
528	ms->name_max = FILE_NAME_MAX;
529	ms->elf_shnum_max = FILE_ELF_SHNUM_MAX;
530	ms->elf_phnum_max = FILE_ELF_PHNUM_MAX;
531	ms->elf_notes_max = FILE_ELF_NOTES_MAX;
532	ms->regex_max = FILE_REGEX_MAX;
533	ms->bytes_max = FILE_BYTES_MAX;
534	return ms;
535free:
536	free(ms);
537	return NULL;
538}
539
540private void
541apprentice_unmap(struct magic_map *map)
542{
543	size_t i;
544	if (map == NULL)
545		return;
546
547	switch (map->type) {
548	case MAP_TYPE_USER:
549		break;
550	case MAP_TYPE_MALLOC:
551		for (i = 0; i < MAGIC_SETS; i++) {
552			void *b = map->magic[i];
553			void *p = map->p;
554			if (CAST(char *, b) >= CAST(char *, p) &&
555			    CAST(char *, b) <= CAST(char *, p) + map->len)
556				continue;
557			free(map->magic[i]);
558		}
559		free(map->p);
560		break;
561#ifdef QUICK
562	case MAP_TYPE_MMAP:
563		if (map->p && map->p != MAP_FAILED)
564			(void)munmap(map->p, map->len);
565		break;
566#endif
567	default:
568		abort();
569	}
570	free(map);
571}
572
573private struct mlist *
574mlist_alloc(void)
575{
576	struct mlist *mlist;
577	if ((mlist = CAST(struct mlist *, calloc(1, sizeof(*mlist)))) == NULL) {
578		return NULL;
579	}
580	mlist->next = mlist->prev = mlist;
581	return mlist;
582}
583
584private void
585mlist_free_one(struct mlist *ml)
586{
587	if (ml->map)
588		apprentice_unmap(CAST(struct magic_map *, ml->map));
589	free(ml);
590}
591
592private void
593mlist_free(struct mlist *mlist)
594{
595	struct mlist *ml, *next;
596
597	if (mlist == NULL)
598		return;
599
600	for (ml = mlist->next; ml != mlist; ml = next) {
601		next = ml->next;
602		mlist_free_one(ml);
603	}
604	mlist_free_one(mlist);
605}
606
607#ifndef COMPILE_ONLY
608/* void **bufs: an array of compiled magic files */
609protected int
610buffer_apprentice(struct magic_set *ms, struct magic **bufs,
611    size_t *sizes, size_t nbufs)
612{
613	size_t i, j;
614	struct mlist *ml;
615	struct magic_map *map;
616
617	if (nbufs == 0)
618		return -1;
619
620	(void)file_reset(ms, 0);
621
622	init_file_tables();
623
624	for (i = 0; i < MAGIC_SETS; i++) {
625		mlist_free(ms->mlist[i]);
626		if ((ms->mlist[i] = mlist_alloc()) == NULL) {
627			file_oomem(ms, sizeof(*ms->mlist[i]));
628			goto fail;
629		}
630	}
631
632	for (i = 0; i < nbufs; i++) {
633		map = apprentice_buf(ms, bufs[i], sizes[i]);
634		if (map == NULL)
635			goto fail;
636
637		for (j = 0; j < MAGIC_SETS; j++) {
638			if (add_mlist(ms->mlist[j], map, j) == -1) {
639				file_oomem(ms, sizeof(*ml));
640				goto fail;
641			}
642		}
643	}
644
645	return 0;
646fail:
647	for (i = 0; i < MAGIC_SETS; i++) {
648		mlist_free(ms->mlist[i]);
649		ms->mlist[i] = NULL;
650	}
651	return -1;
652}
653#endif
654
655/* const char *fn: list of magic files and directories */
656protected int
657file_apprentice(struct magic_set *ms, const char *fn, int action)
658{
659	char *p, *mfn;
660	int fileerr, errs = -1;
661	size_t i;
662
663	(void)file_reset(ms, 0);
664
665	if ((fn = magic_getpath(fn, action)) == NULL)
666		return -1;
667
668	init_file_tables();
669
670	if ((mfn = strdup(fn)) == NULL) {
671		file_oomem(ms, strlen(fn));
672		return -1;
673	}
674
675	for (i = 0; i < MAGIC_SETS; i++) {
676		mlist_free(ms->mlist[i]);
677		if ((ms->mlist[i] = mlist_alloc()) == NULL) {
678			file_oomem(ms, sizeof(*ms->mlist[i]));
679			while (i-- > 0) {
680				mlist_free(ms->mlist[i]);
681				ms->mlist[i] = NULL;
682			}
683			free(mfn);
684			return -1;
685		}
686	}
687	fn = mfn;
688
689	while (fn) {
690		p = strchr(fn, PATHSEP);
691		if (p)
692			*p++ = '\0';
693		if (*fn == '\0')
694			break;
695		fileerr = apprentice_1(ms, fn, action);
696		errs = MAX(errs, fileerr);
697		fn = p;
698	}
699
700	free(mfn);
701
702	if (errs == -1) {
703		for (i = 0; i < MAGIC_SETS; i++) {
704			mlist_free(ms->mlist[i]);
705			ms->mlist[i] = NULL;
706		}
707		file_error(ms, 0, "could not find any valid magic files!");
708		return -1;
709	}
710
711#if 0
712	/*
713	 * Always leave the database loaded
714	 */
715	if (action == FILE_LOAD)
716		return 0;
717
718	for (i = 0; i < MAGIC_SETS; i++) {
719		mlist_free(ms->mlist[i]);
720		ms->mlist[i] = NULL;
721	}
722#endif
723
724	switch (action) {
725	case FILE_LOAD:
726	case FILE_COMPILE:
727	case FILE_CHECK:
728	case FILE_LIST:
729		return 0;
730	default:
731		file_error(ms, 0, "Invalid action %d", action);
732		return -1;
733	}
734}
735
736/*
737 * Compute the real length of a magic expression, for the purposes
738 * of determining how "strong" a magic expression is (approximating
739 * how specific its matches are):
740 *	- magic characters count 0 unless escaped.
741 *	- [] expressions count 1
742 *	- {} expressions count 0
743 *	- regular characters or escaped magic characters count 1
744 *	- 0 length expressions count as one
745 */
746private size_t
747nonmagic(const char *str)
748{
749	const char *p;
750	size_t rv = 0;
751
752	for (p = str; *p; p++)
753		switch (*p) {
754		case '\\':	/* Escaped anything counts 1 */
755			if (!*++p)
756				p--;
757			rv++;
758			continue;
759		case '?':	/* Magic characters count 0 */
760		case '*':
761		case '.':
762		case '+':
763		case '^':
764		case '$':
765			continue;
766		case '[':	/* Bracketed expressions count 1 the ']' */
767			while (*p && *p != ']')
768				p++;
769			p--;
770			continue;
771		case '{':	/* Braced expressions count 0 */
772			while (*p && *p != '}')
773				p++;
774			if (!*p)
775				p--;
776			continue;
777		default:	/* Anything else counts 1 */
778			rv++;
779			continue;
780		}
781
782	return rv == 0 ? 1 : rv;	/* Return at least 1 */
783}
784
785
786private size_t
787typesize(int type)
788{
789	switch (type) {
790	case FILE_BYTE:
791		return 1;
792
793	case FILE_SHORT:
794	case FILE_LESHORT:
795	case FILE_BESHORT:
796		return 2;
797
798	case FILE_LONG:
799	case FILE_LELONG:
800	case FILE_BELONG:
801	case FILE_MELONG:
802		return 4;
803
804	case FILE_DATE:
805	case FILE_LEDATE:
806	case FILE_BEDATE:
807	case FILE_MEDATE:
808	case FILE_LDATE:
809	case FILE_LELDATE:
810	case FILE_BELDATE:
811	case FILE_MELDATE:
812	case FILE_FLOAT:
813	case FILE_BEFLOAT:
814	case FILE_LEFLOAT:
815		return 4;
816
817	case FILE_QUAD:
818	case FILE_BEQUAD:
819	case FILE_LEQUAD:
820	case FILE_QDATE:
821	case FILE_LEQDATE:
822	case FILE_BEQDATE:
823	case FILE_QLDATE:
824	case FILE_LEQLDATE:
825	case FILE_BEQLDATE:
826	case FILE_QWDATE:
827	case FILE_LEQWDATE:
828	case FILE_BEQWDATE:
829	case FILE_DOUBLE:
830	case FILE_BEDOUBLE:
831	case FILE_LEDOUBLE:
832		return 8;
833	default:
834		return CAST(size_t, ~0);
835	}
836}
837
838/*
839 * Get weight of this magic entry, for sorting purposes.
840 */
841private size_t
842apprentice_magic_strength(const struct magic *m)
843{
844#define MULT 10U
845	size_t ts, v;
846	ssize_t val = 2 * MULT;	/* baseline strength */
847
848	switch (m->type) {
849	case FILE_DEFAULT:	/* make sure this sorts last */
850		if (m->factor_op != FILE_FACTOR_OP_NONE)
851			abort();
852		return 0;
853
854	case FILE_BYTE:
855	case FILE_SHORT:
856	case FILE_LESHORT:
857	case FILE_BESHORT:
858	case FILE_LONG:
859	case FILE_LELONG:
860	case FILE_BELONG:
861	case FILE_MELONG:
862	case FILE_DATE:
863	case FILE_LEDATE:
864	case FILE_BEDATE:
865	case FILE_MEDATE:
866	case FILE_LDATE:
867	case FILE_LELDATE:
868	case FILE_BELDATE:
869	case FILE_MELDATE:
870	case FILE_FLOAT:
871	case FILE_BEFLOAT:
872	case FILE_LEFLOAT:
873	case FILE_QUAD:
874	case FILE_BEQUAD:
875	case FILE_LEQUAD:
876	case FILE_QDATE:
877	case FILE_LEQDATE:
878	case FILE_BEQDATE:
879	case FILE_QLDATE:
880	case FILE_LEQLDATE:
881	case FILE_BEQLDATE:
882	case FILE_QWDATE:
883	case FILE_LEQWDATE:
884	case FILE_BEQWDATE:
885	case FILE_DOUBLE:
886	case FILE_BEDOUBLE:
887	case FILE_LEDOUBLE:
888		ts = typesize(m->type);
889		if (ts == CAST(size_t, ~0))
890			abort();
891		val += ts * MULT;
892		break;
893
894	case FILE_PSTRING:
895	case FILE_STRING:
896		val += m->vallen * MULT;
897		break;
898
899	case FILE_BESTRING16:
900	case FILE_LESTRING16:
901		val += m->vallen * MULT / 2;
902		break;
903
904	case FILE_SEARCH:
905		if (m->vallen == 0)
906			break;
907		val += m->vallen * MAX(MULT / m->vallen, 1);
908		break;
909
910	case FILE_REGEX:
911		v = nonmagic(m->value.s);
912		val += v * MAX(MULT / v, 1);
913		break;
914
915	case FILE_INDIRECT:
916	case FILE_NAME:
917	case FILE_USE:
918		break;
919
920	case FILE_DER:
921		val += MULT;
922		break;
923
924	default:
925		(void)fprintf(stderr, "Bad type %d\n", m->type);
926		abort();
927	}
928
929	switch (m->reln) {
930	case 'x':	/* matches anything penalize */
931	case '!':       /* matches almost anything penalize */
932		val = 0;
933		break;
934
935	case '=':	/* Exact match, prefer */
936		val += MULT;
937		break;
938
939	case '>':
940	case '<':	/* comparison match reduce strength */
941		val -= 2 * MULT;
942		break;
943
944	case '^':
945	case '&':	/* masking bits, we could count them too */
946		val -= MULT;
947		break;
948
949	default:
950		(void)fprintf(stderr, "Bad relation %c\n", m->reln);
951		abort();
952	}
953
954	switch (m->factor_op) {
955	case FILE_FACTOR_OP_NONE:
956		break;
957	case FILE_FACTOR_OP_PLUS:
958		val += m->factor;
959		break;
960	case FILE_FACTOR_OP_MINUS:
961		val -= m->factor;
962		break;
963	case FILE_FACTOR_OP_TIMES:
964		val *= m->factor;
965		break;
966	case FILE_FACTOR_OP_DIV:
967		val /= m->factor;
968		break;
969	default:
970		abort();
971	}
972
973	if (val <= 0)	/* ensure we only return 0 for FILE_DEFAULT */
974		val = 1;
975
976	/*
977	 * Magic entries with no description get a bonus because they depend
978	 * on subsequent magic entries to print something.
979	 */
980	if (m->desc[0] == '\0')
981		val++;
982	return val;
983}
984
985/*
986 * Sort callback for sorting entries by "strength" (basically length)
987 */
988private int
989apprentice_sort(const void *a, const void *b)
990{
991	const struct magic_entry *ma = CAST(const struct magic_entry *, a);
992	const struct magic_entry *mb = CAST(const struct magic_entry *, b);
993	size_t sa = apprentice_magic_strength(ma->mp);
994	size_t sb = apprentice_magic_strength(mb->mp);
995	if (sa == sb)
996		return 0;
997	else if (sa > sb)
998		return -1;
999	else
1000		return 1;
1001}
1002
1003/*
1004 * Shows sorted patterns list in the order which is used for the matching
1005 */
1006private void
1007apprentice_list(struct mlist *mlist, int mode)
1008{
1009	uint32_t magindex = 0;
1010	struct mlist *ml;
1011	for (ml = mlist->next; ml != mlist; ml = ml->next) {
1012		for (magindex = 0; magindex < ml->nmagic; magindex++) {
1013			struct magic *m = &ml->magic[magindex];
1014			if ((m->flag & mode) != mode) {
1015				/* Skip sub-tests */
1016				while (magindex + 1 < ml->nmagic &&
1017				       ml->magic[magindex + 1].cont_level != 0)
1018					++magindex;
1019				continue; /* Skip to next top-level test*/
1020			}
1021
1022			/*
1023			 * Try to iterate over the tree until we find item with
1024			 * description/mimetype.
1025			 */
1026			while (magindex + 1 < ml->nmagic &&
1027			       ml->magic[magindex + 1].cont_level != 0 &&
1028			       *ml->magic[magindex].desc == '\0' &&
1029			       *ml->magic[magindex].mimetype == '\0')
1030				magindex++;
1031
1032			printf("Strength = %3" SIZE_T_FORMAT "u@%u: %s [%s]\n",
1033			    apprentice_magic_strength(m),
1034			    ml->magic[magindex].lineno,
1035			    ml->magic[magindex].desc,
1036			    ml->magic[magindex].mimetype);
1037		}
1038	}
1039}
1040
1041private void
1042set_test_type(struct magic *mstart, struct magic *m)
1043{
1044	switch (m->type) {
1045	case FILE_BYTE:
1046	case FILE_SHORT:
1047	case FILE_LONG:
1048	case FILE_DATE:
1049	case FILE_BESHORT:
1050	case FILE_BELONG:
1051	case FILE_BEDATE:
1052	case FILE_LESHORT:
1053	case FILE_LELONG:
1054	case FILE_LEDATE:
1055	case FILE_LDATE:
1056	case FILE_BELDATE:
1057	case FILE_LELDATE:
1058	case FILE_MEDATE:
1059	case FILE_MELDATE:
1060	case FILE_MELONG:
1061	case FILE_QUAD:
1062	case FILE_LEQUAD:
1063	case FILE_BEQUAD:
1064	case FILE_QDATE:
1065	case FILE_LEQDATE:
1066	case FILE_BEQDATE:
1067	case FILE_QLDATE:
1068	case FILE_LEQLDATE:
1069	case FILE_BEQLDATE:
1070	case FILE_QWDATE:
1071	case FILE_LEQWDATE:
1072	case FILE_BEQWDATE:
1073	case FILE_FLOAT:
1074	case FILE_BEFLOAT:
1075	case FILE_LEFLOAT:
1076	case FILE_DOUBLE:
1077	case FILE_BEDOUBLE:
1078	case FILE_LEDOUBLE:
1079	case FILE_DER:
1080		mstart->flag |= BINTEST;
1081		break;
1082	case FILE_STRING:
1083	case FILE_PSTRING:
1084	case FILE_BESTRING16:
1085	case FILE_LESTRING16:
1086		/* Allow text overrides */
1087		if (mstart->str_flags & STRING_TEXTTEST)
1088			mstart->flag |= TEXTTEST;
1089		else
1090			mstart->flag |= BINTEST;
1091		break;
1092	case FILE_REGEX:
1093	case FILE_SEARCH:
1094		/* Check for override */
1095		if (mstart->str_flags & STRING_BINTEST)
1096			mstart->flag |= BINTEST;
1097		if (mstart->str_flags & STRING_TEXTTEST)
1098			mstart->flag |= TEXTTEST;
1099
1100		if (mstart->flag & (TEXTTEST|BINTEST))
1101			break;
1102
1103		/* binary test if pattern is not text */
1104		if (file_looks_utf8(m->value.us, CAST(size_t, m->vallen), NULL,
1105		    NULL) <= 0)
1106			mstart->flag |= BINTEST;
1107		else
1108			mstart->flag |= TEXTTEST;
1109		break;
1110	case FILE_DEFAULT:
1111		/* can't deduce anything; we shouldn't see this at the
1112		   top level anyway */
1113		break;
1114	case FILE_INVALID:
1115	default:
1116		/* invalid search type, but no need to complain here */
1117		break;
1118	}
1119}
1120
1121private int
1122addentry(struct magic_set *ms, struct magic_entry *me,
1123   struct magic_entry_set *mset)
1124{
1125	size_t i = me->mp->type == FILE_NAME ? 1 : 0;
1126	if (mset[i].count == mset[i].max) {
1127		struct magic_entry *mp;
1128
1129		mset[i].max += ALLOC_INCR;
1130		if ((mp = CAST(struct magic_entry *,
1131		    realloc(mset[i].me, sizeof(*mp) * mset[i].max))) ==
1132		    NULL) {
1133			file_oomem(ms, sizeof(*mp) * mset[i].max);
1134			return -1;
1135		}
1136		(void)memset(&mp[mset[i].count], 0, sizeof(*mp) *
1137		    ALLOC_INCR);
1138		mset[i].me = mp;
1139	}
1140	mset[i].me[mset[i].count++] = *me;
1141	memset(me, 0, sizeof(*me));
1142	return 0;
1143}
1144
1145/*
1146 * Load and parse one file.
1147 */
1148private void
1149load_1(struct magic_set *ms, int action, const char *fn, int *errs,
1150   struct magic_entry_set *mset)
1151{
1152	size_t lineno = 0, llen = 0;
1153	char *line = NULL;
1154	ssize_t len;
1155	struct magic_entry me;
1156
1157	FILE *f = fopen(ms->file = fn, "r");
1158	if (f == NULL) {
1159		if (errno != ENOENT)
1160			file_error(ms, errno, "cannot read magic file `%s'",
1161				   fn);
1162		(*errs)++;
1163		return;
1164	}
1165
1166	memset(&me, 0, sizeof(me));
1167	/* read and parse this file */
1168	for (ms->line = 1; (len = getline(&line, &llen, f)) != -1;
1169	    ms->line++) {
1170		if (len == 0) /* null line, garbage, etc */
1171			continue;
1172		if (line[len - 1] == '\n') {
1173			lineno++;
1174			line[len - 1] = '\0'; /* delete newline */
1175		}
1176		switch (line[0]) {
1177		case '\0':	/* empty, do not parse */
1178		case '#':	/* comment, do not parse */
1179			continue;
1180		case '!':
1181			if (line[1] == ':') {
1182				size_t i;
1183
1184				for (i = 0; bang[i].name != NULL; i++) {
1185					if (CAST(size_t, len - 2) > bang[i].len &&
1186					    memcmp(bang[i].name, line + 2,
1187					    bang[i].len) == 0)
1188						break;
1189				}
1190				if (bang[i].name == NULL) {
1191					file_error(ms, 0,
1192					    "Unknown !: entry `%s'", line);
1193					(*errs)++;
1194					continue;
1195				}
1196				if (me.mp == NULL) {
1197					file_error(ms, 0,
1198					    "No current entry for :!%s type",
1199						bang[i].name);
1200					(*errs)++;
1201					continue;
1202				}
1203				if ((*bang[i].fun)(ms, &me,
1204				    line + bang[i].len + 2) != 0) {
1205					(*errs)++;
1206					continue;
1207				}
1208				continue;
1209			}
1210			/*FALLTHROUGH*/
1211		default:
1212		again:
1213			switch (parse(ms, &me, line, lineno, action)) {
1214			case 0:
1215				continue;
1216			case 1:
1217				(void)addentry(ms, &me, mset);
1218				goto again;
1219			default:
1220				(*errs)++;
1221				break;
1222			}
1223		}
1224	}
1225	if (me.mp)
1226		(void)addentry(ms, &me, mset);
1227	free(line);
1228	(void)fclose(f);
1229}
1230
1231/*
1232 * parse a file or directory of files
1233 * const char *fn: name of magic file or directory
1234 */
1235private int
1236cmpstrp(const void *p1, const void *p2)
1237{
1238        return strcmp(*RCAST(char *const *, p1), *RCAST(char *const *, p2));
1239}
1240
1241
1242private uint32_t
1243set_text_binary(struct magic_set *ms, struct magic_entry *me, uint32_t nme,
1244    uint32_t starttest)
1245{
1246	static const char text[] = "text";
1247	static const char binary[] = "binary";
1248	static const size_t len = sizeof(text);
1249
1250	uint32_t i = starttest;
1251
1252	do {
1253		set_test_type(me[starttest].mp, me[i].mp);
1254		if ((ms->flags & MAGIC_DEBUG) == 0)
1255			continue;
1256		(void)fprintf(stderr, "%s%s%s: %s\n",
1257		    me[i].mp->mimetype,
1258		    me[i].mp->mimetype[0] == '\0' ? "" : "; ",
1259		    me[i].mp->desc[0] ? me[i].mp->desc : "(no description)",
1260		    me[i].mp->flag & BINTEST ? binary : text);
1261		if (me[i].mp->flag & BINTEST) {
1262			char *p = strstr(me[i].mp->desc, text);
1263			if (p && (p == me[i].mp->desc ||
1264			    isspace(CAST(unsigned char, p[-1]))) &&
1265			    (p + len - me[i].mp->desc == MAXstring
1266			    || (p[len] == '\0' ||
1267			    isspace(CAST(unsigned char, p[len])))))
1268				(void)fprintf(stderr, "*** Possible "
1269				    "binary test for text type\n");
1270		}
1271	} while (++i < nme && me[i].mp->cont_level != 0);
1272	return i;
1273}
1274
1275private void
1276set_last_default(struct magic_set *ms, struct magic_entry *me, uint32_t nme)
1277{
1278	uint32_t i;
1279	for (i = 0; i < nme; i++) {
1280		if (me[i].mp->cont_level == 0 &&
1281		    me[i].mp->type == FILE_DEFAULT) {
1282			while (++i < nme)
1283				if (me[i].mp->cont_level == 0)
1284					break;
1285			if (i != nme) {
1286				/* XXX - Ugh! */
1287				ms->line = me[i].mp->lineno;
1288				file_magwarn(ms,
1289				    "level 0 \"default\" did not sort last");
1290			}
1291			return;
1292		}
1293	}
1294}
1295
1296private int
1297coalesce_entries(struct magic_set *ms, struct magic_entry *me, uint32_t nme,
1298    struct magic **ma, uint32_t *nma)
1299{
1300	uint32_t i, mentrycount = 0;
1301	size_t slen;
1302
1303	for (i = 0; i < nme; i++)
1304		mentrycount += me[i].cont_count;
1305
1306	slen = sizeof(**ma) * mentrycount;
1307	if ((*ma = CAST(struct magic *, malloc(slen))) == NULL) {
1308		file_oomem(ms, slen);
1309		return -1;
1310	}
1311
1312	mentrycount = 0;
1313	for (i = 0; i < nme; i++) {
1314		(void)memcpy(*ma + mentrycount, me[i].mp,
1315		    me[i].cont_count * sizeof(**ma));
1316		mentrycount += me[i].cont_count;
1317	}
1318	*nma = mentrycount;
1319	return 0;
1320}
1321
1322private void
1323magic_entry_free(struct magic_entry *me, uint32_t nme)
1324{
1325	uint32_t i;
1326	if (me == NULL)
1327		return;
1328	for (i = 0; i < nme; i++)
1329		free(me[i].mp);
1330	free(me);
1331}
1332
1333private struct magic_map *
1334apprentice_load(struct magic_set *ms, const char *fn, int action)
1335{
1336	int errs = 0;
1337	uint32_t i, j;
1338	size_t files = 0, maxfiles = 0;
1339	char **filearr = NULL, *mfn;
1340	struct stat st;
1341	struct magic_map *map;
1342	struct magic_entry_set mset[MAGIC_SETS];
1343	DIR *dir;
1344	struct dirent *d;
1345
1346	memset(mset, 0, sizeof(mset));
1347	ms->flags |= MAGIC_CHECK;	/* Enable checks for parsed files */
1348
1349
1350	if ((map = CAST(struct magic_map *, calloc(1, sizeof(*map)))) == NULL)
1351	{
1352		file_oomem(ms, sizeof(*map));
1353		return NULL;
1354	}
1355	map->type = MAP_TYPE_MALLOC;
1356
1357	/* print silly verbose header for USG compat. */
1358	if (action == FILE_CHECK)
1359		(void)fprintf(stderr, "%s\n", usg_hdr);
1360
1361	/* load directory or file */
1362	if (stat(fn, &st) == 0 && S_ISDIR(st.st_mode)) {
1363		dir = opendir(fn);
1364		if (!dir) {
1365			errs++;
1366			goto out;
1367		}
1368		while ((d = readdir(dir)) != NULL) {
1369			if (d->d_name[0] == '.')
1370				continue;
1371			if (asprintf(&mfn, "%s/%s", fn, d->d_name) < 0) {
1372				file_oomem(ms,
1373				    strlen(fn) + strlen(d->d_name) + 2);
1374				errs++;
1375				closedir(dir);
1376				goto out;
1377			}
1378			if (stat(mfn, &st) == -1 || !S_ISREG(st.st_mode)) {
1379				free(mfn);
1380				continue;
1381			}
1382			if (files >= maxfiles) {
1383				size_t mlen;
1384				maxfiles = (maxfiles + 1) * 2;
1385				mlen = maxfiles * sizeof(*filearr);
1386				if ((filearr = CAST(char **,
1387				    realloc(filearr, mlen))) == NULL) {
1388					file_oomem(ms, mlen);
1389					free(mfn);
1390					closedir(dir);
1391					errs++;
1392					goto out;
1393				}
1394			}
1395			filearr[files++] = mfn;
1396		}
1397		closedir(dir);
1398		if (filearr) {
1399			qsort(filearr, files, sizeof(*filearr), cmpstrp);
1400			for (i = 0; i < files; i++) {
1401				load_1(ms, action, filearr[i], &errs, mset);
1402				free(filearr[i]);
1403			}
1404			free(filearr);
1405		}
1406	} else
1407		load_1(ms, action, fn, &errs, mset);
1408	if (errs)
1409		goto out;
1410
1411	for (j = 0; j < MAGIC_SETS; j++) {
1412		/* Set types of tests */
1413		for (i = 0; i < mset[j].count; ) {
1414			if (mset[j].me[i].mp->cont_level != 0) {
1415				i++;
1416				continue;
1417			}
1418			i = set_text_binary(ms, mset[j].me, mset[j].count, i);
1419		}
1420		if (mset[j].me)
1421			qsort(mset[j].me, mset[j].count, sizeof(*mset[j].me),
1422			    apprentice_sort);
1423
1424		/*
1425		 * Make sure that any level 0 "default" line is last
1426		 * (if one exists).
1427		 */
1428		set_last_default(ms, mset[j].me, mset[j].count);
1429
1430		/* coalesce per file arrays into a single one */
1431		if (coalesce_entries(ms, mset[j].me, mset[j].count,
1432		    &map->magic[j], &map->nmagic[j]) == -1) {
1433			errs++;
1434			goto out;
1435		}
1436	}
1437
1438out:
1439	for (j = 0; j < MAGIC_SETS; j++)
1440		magic_entry_free(mset[j].me, mset[j].count);
1441
1442	if (errs) {
1443		apprentice_unmap(map);
1444		return NULL;
1445	}
1446	return map;
1447}
1448
1449/*
1450 * extend the sign bit if the comparison is to be signed
1451 */
1452protected uint64_t
1453file_signextend(struct magic_set *ms, struct magic *m, uint64_t v)
1454{
1455	if (!(m->flag & UNSIGNED)) {
1456		switch(m->type) {
1457		/*
1458		 * Do not remove the casts below.  They are
1459		 * vital.  When later compared with the data,
1460		 * the sign extension must have happened.
1461		 */
1462		case FILE_BYTE:
1463			v = CAST(signed char,  v);
1464			break;
1465		case FILE_SHORT:
1466		case FILE_BESHORT:
1467		case FILE_LESHORT:
1468			v = CAST(short, v);
1469			break;
1470		case FILE_DATE:
1471		case FILE_BEDATE:
1472		case FILE_LEDATE:
1473		case FILE_MEDATE:
1474		case FILE_LDATE:
1475		case FILE_BELDATE:
1476		case FILE_LELDATE:
1477		case FILE_MELDATE:
1478		case FILE_LONG:
1479		case FILE_BELONG:
1480		case FILE_LELONG:
1481		case FILE_MELONG:
1482		case FILE_FLOAT:
1483		case FILE_BEFLOAT:
1484		case FILE_LEFLOAT:
1485			v = CAST(int32_t, v);
1486			break;
1487		case FILE_QUAD:
1488		case FILE_BEQUAD:
1489		case FILE_LEQUAD:
1490		case FILE_QDATE:
1491		case FILE_QLDATE:
1492		case FILE_QWDATE:
1493		case FILE_BEQDATE:
1494		case FILE_BEQLDATE:
1495		case FILE_BEQWDATE:
1496		case FILE_LEQDATE:
1497		case FILE_LEQLDATE:
1498		case FILE_LEQWDATE:
1499		case FILE_DOUBLE:
1500		case FILE_BEDOUBLE:
1501		case FILE_LEDOUBLE:
1502			v = CAST(int64_t, v);
1503			break;
1504		case FILE_STRING:
1505		case FILE_PSTRING:
1506		case FILE_BESTRING16:
1507		case FILE_LESTRING16:
1508		case FILE_REGEX:
1509		case FILE_SEARCH:
1510		case FILE_DEFAULT:
1511		case FILE_INDIRECT:
1512		case FILE_NAME:
1513		case FILE_USE:
1514		case FILE_CLEAR:
1515		case FILE_DER:
1516			break;
1517		default:
1518			if (ms->flags & MAGIC_CHECK)
1519			    file_magwarn(ms, "cannot happen: m->type=%d\n",
1520				    m->type);
1521			return ~0U;
1522		}
1523	}
1524	return v;
1525}
1526
1527private int
1528string_modifier_check(struct magic_set *ms, struct magic *m)
1529{
1530	if ((ms->flags & MAGIC_CHECK) == 0)
1531		return 0;
1532
1533	if ((m->type != FILE_REGEX || (m->str_flags & REGEX_LINE_COUNT) == 0) &&
1534	    (m->type != FILE_PSTRING && (m->str_flags & PSTRING_LEN) != 0)) {
1535		file_magwarn(ms,
1536		    "'/BHhLl' modifiers are only allowed for pascal strings\n");
1537		return -1;
1538	}
1539	switch (m->type) {
1540	case FILE_BESTRING16:
1541	case FILE_LESTRING16:
1542		if (m->str_flags != 0) {
1543			file_magwarn(ms,
1544			    "no modifiers allowed for 16-bit strings\n");
1545			return -1;
1546		}
1547		break;
1548	case FILE_STRING:
1549	case FILE_PSTRING:
1550		if ((m->str_flags & REGEX_OFFSET_START) != 0) {
1551			file_magwarn(ms,
1552			    "'/%c' only allowed on regex and search\n",
1553			    CHAR_REGEX_OFFSET_START);
1554			return -1;
1555		}
1556		break;
1557	case FILE_SEARCH:
1558		if (m->str_range == 0) {
1559			file_magwarn(ms,
1560			    "missing range; defaulting to %d\n",
1561                            STRING_DEFAULT_RANGE);
1562			m->str_range = STRING_DEFAULT_RANGE;
1563			return -1;
1564		}
1565		break;
1566	case FILE_REGEX:
1567		if ((m->str_flags & STRING_COMPACT_WHITESPACE) != 0) {
1568			file_magwarn(ms, "'/%c' not allowed on regex\n",
1569			    CHAR_COMPACT_WHITESPACE);
1570			return -1;
1571		}
1572		if ((m->str_flags & STRING_COMPACT_OPTIONAL_WHITESPACE) != 0) {
1573			file_magwarn(ms, "'/%c' not allowed on regex\n",
1574			    CHAR_COMPACT_OPTIONAL_WHITESPACE);
1575			return -1;
1576		}
1577		break;
1578	default:
1579		file_magwarn(ms, "coding error: m->type=%d\n",
1580		    m->type);
1581		return -1;
1582	}
1583	return 0;
1584}
1585
1586private int
1587get_op(char c)
1588{
1589	switch (c) {
1590	case '&':
1591		return FILE_OPAND;
1592	case '|':
1593		return FILE_OPOR;
1594	case '^':
1595		return FILE_OPXOR;
1596	case '+':
1597		return FILE_OPADD;
1598	case '-':
1599		return FILE_OPMINUS;
1600	case '*':
1601		return FILE_OPMULTIPLY;
1602	case '/':
1603		return FILE_OPDIVIDE;
1604	case '%':
1605		return FILE_OPMODULO;
1606	default:
1607		return -1;
1608	}
1609}
1610
1611#ifdef ENABLE_CONDITIONALS
1612private int
1613get_cond(const char *l, const char **t)
1614{
1615	static const struct cond_tbl_s {
1616		char name[8];
1617		size_t len;
1618		int cond;
1619	} cond_tbl[] = {
1620		{ "if",		2,	COND_IF },
1621		{ "elif",	4,	COND_ELIF },
1622		{ "else",	4,	COND_ELSE },
1623		{ "",		0,	COND_NONE },
1624	};
1625	const struct cond_tbl_s *p;
1626
1627	for (p = cond_tbl; p->len; p++) {
1628		if (strncmp(l, p->name, p->len) == 0 &&
1629		    isspace(CAST(unsigned char, l[p->len]))) {
1630			if (t)
1631				*t = l + p->len;
1632			break;
1633		}
1634	}
1635	return p->cond;
1636}
1637
1638private int
1639check_cond(struct magic_set *ms, int cond, uint32_t cont_level)
1640{
1641	int last_cond;
1642	last_cond = ms->c.li[cont_level].last_cond;
1643
1644	switch (cond) {
1645	case COND_IF:
1646		if (last_cond != COND_NONE && last_cond != COND_ELIF) {
1647			if (ms->flags & MAGIC_CHECK)
1648				file_magwarn(ms, "syntax error: `if'");
1649			return -1;
1650		}
1651		last_cond = COND_IF;
1652		break;
1653
1654	case COND_ELIF:
1655		if (last_cond != COND_IF && last_cond != COND_ELIF) {
1656			if (ms->flags & MAGIC_CHECK)
1657				file_magwarn(ms, "syntax error: `elif'");
1658			return -1;
1659		}
1660		last_cond = COND_ELIF;
1661		break;
1662
1663	case COND_ELSE:
1664		if (last_cond != COND_IF && last_cond != COND_ELIF) {
1665			if (ms->flags & MAGIC_CHECK)
1666				file_magwarn(ms, "syntax error: `else'");
1667			return -1;
1668		}
1669		last_cond = COND_NONE;
1670		break;
1671
1672	case COND_NONE:
1673		last_cond = COND_NONE;
1674		break;
1675	}
1676
1677	ms->c.li[cont_level].last_cond = last_cond;
1678	return 0;
1679}
1680#endif /* ENABLE_CONDITIONALS */
1681
1682private int
1683parse_indirect_modifier(struct magic_set *ms, struct magic *m, const char **lp)
1684{
1685	const char *l = *lp;
1686
1687	while (!isspace(CAST(unsigned char, *++l)))
1688		switch (*l) {
1689		case CHAR_INDIRECT_RELATIVE:
1690			m->str_flags |= INDIRECT_RELATIVE;
1691			break;
1692		default:
1693			if (ms->flags & MAGIC_CHECK)
1694				file_magwarn(ms, "indirect modifier `%c' "
1695					"invalid", *l);
1696			*lp = l;
1697			return -1;
1698		}
1699	*lp = l;
1700	return 0;
1701}
1702
1703private void
1704parse_op_modifier(struct magic_set *ms, struct magic *m, const char **lp,
1705    int op)
1706{
1707	const char *l = *lp;
1708	char *t;
1709	uint64_t val;
1710
1711	++l;
1712	m->mask_op |= op;
1713	val = CAST(uint64_t, strtoull(l, &t, 0));
1714	l = t;
1715	m->num_mask = file_signextend(ms, m, val);
1716	eatsize(&l);
1717	*lp = l;
1718}
1719
1720private int
1721parse_string_modifier(struct magic_set *ms, struct magic *m, const char **lp)
1722{
1723	const char *l = *lp;
1724	char *t;
1725	int have_range = 0;
1726
1727	while (!isspace(CAST(unsigned char, *++l))) {
1728		switch (*l) {
1729		case '0':  case '1':  case '2':
1730		case '3':  case '4':  case '5':
1731		case '6':  case '7':  case '8':
1732		case '9':
1733			if (have_range && (ms->flags & MAGIC_CHECK))
1734				file_magwarn(ms, "multiple ranges");
1735			have_range = 1;
1736			m->str_range = CAST(uint32_t, strtoul(l, &t, 0));
1737			if (m->str_range == 0)
1738				file_magwarn(ms, "zero range");
1739			l = t - 1;
1740			break;
1741		case CHAR_COMPACT_WHITESPACE:
1742			m->str_flags |= STRING_COMPACT_WHITESPACE;
1743			break;
1744		case CHAR_COMPACT_OPTIONAL_WHITESPACE:
1745			m->str_flags |= STRING_COMPACT_OPTIONAL_WHITESPACE;
1746			break;
1747		case CHAR_IGNORE_LOWERCASE:
1748			m->str_flags |= STRING_IGNORE_LOWERCASE;
1749			break;
1750		case CHAR_IGNORE_UPPERCASE:
1751			m->str_flags |= STRING_IGNORE_UPPERCASE;
1752			break;
1753		case CHAR_REGEX_OFFSET_START:
1754			m->str_flags |= REGEX_OFFSET_START;
1755			break;
1756		case CHAR_BINTEST:
1757			m->str_flags |= STRING_BINTEST;
1758			break;
1759		case CHAR_TEXTTEST:
1760			m->str_flags |= STRING_TEXTTEST;
1761			break;
1762		case CHAR_TRIM:
1763			m->str_flags |= STRING_TRIM;
1764			break;
1765		case CHAR_PSTRING_1_LE:
1766#define SET_LENGTH(a) m->str_flags = (m->str_flags & ~PSTRING_LEN) | (a)
1767			if (m->type != FILE_PSTRING)
1768				goto bad;
1769			SET_LENGTH(PSTRING_1_LE);
1770			break;
1771		case CHAR_PSTRING_2_BE:
1772			if (m->type != FILE_PSTRING)
1773				goto bad;
1774			SET_LENGTH(PSTRING_2_BE);
1775			break;
1776		case CHAR_PSTRING_2_LE:
1777			if (m->type != FILE_PSTRING)
1778				goto bad;
1779			SET_LENGTH(PSTRING_2_LE);
1780			break;
1781		case CHAR_PSTRING_4_BE:
1782			if (m->type != FILE_PSTRING)
1783				goto bad;
1784			SET_LENGTH(PSTRING_4_BE);
1785			break;
1786		case CHAR_PSTRING_4_LE:
1787			switch (m->type) {
1788			case FILE_PSTRING:
1789			case FILE_REGEX:
1790				break;
1791			default:
1792				goto bad;
1793			}
1794			SET_LENGTH(PSTRING_4_LE);
1795			break;
1796		case CHAR_PSTRING_LENGTH_INCLUDES_ITSELF:
1797			if (m->type != FILE_PSTRING)
1798				goto bad;
1799			m->str_flags |= PSTRING_LENGTH_INCLUDES_ITSELF;
1800			break;
1801		default:
1802		bad:
1803			if (ms->flags & MAGIC_CHECK)
1804				file_magwarn(ms, "string modifier `%c' "
1805					"invalid", *l);
1806			goto out;
1807		}
1808		/* allow multiple '/' for readability */
1809		if (l[1] == '/' && !isspace(CAST(unsigned char, l[2])))
1810			l++;
1811	}
1812	if (string_modifier_check(ms, m) == -1)
1813		goto out;
1814	*lp = l;
1815	return 0;
1816out:
1817	*lp = l;
1818	return -1;
1819}
1820
1821/*
1822 * parse one line from magic file, put into magic[index++] if valid
1823 */
1824private int
1825parse(struct magic_set *ms, struct magic_entry *me, const char *line,
1826    size_t lineno, int action)
1827{
1828#ifdef ENABLE_CONDITIONALS
1829	static uint32_t last_cont_level = 0;
1830#endif
1831	size_t i;
1832	struct magic *m;
1833	const char *l = line;
1834	char *t;
1835	int op;
1836	uint32_t cont_level;
1837	int32_t diff;
1838
1839	cont_level = 0;
1840
1841	/*
1842	 * Parse the offset.
1843	 */
1844	while (*l == '>') {
1845		++l;		/* step over */
1846		cont_level++;
1847	}
1848#ifdef ENABLE_CONDITIONALS
1849	if (cont_level == 0 || cont_level > last_cont_level)
1850		if (file_check_mem(ms, cont_level) == -1)
1851			return -1;
1852	last_cont_level = cont_level;
1853#endif
1854	if (cont_level != 0) {
1855		if (me->mp == NULL) {
1856			file_magerror(ms, "No current entry for continuation");
1857			return -1;
1858		}
1859		if (me->cont_count == 0) {
1860			file_magerror(ms, "Continuations present with 0 count");
1861			return -1;
1862		}
1863		m = &me->mp[me->cont_count - 1];
1864		diff = CAST(int32_t, cont_level) - CAST(int32_t, m->cont_level);
1865		if (diff > 1)
1866			file_magwarn(ms, "New continuation level %u is more "
1867			    "than one larger than current level %u", cont_level,
1868			    m->cont_level);
1869		if (me->cont_count == me->max_count) {
1870			struct magic *nm;
1871			size_t cnt = me->max_count + ALLOC_CHUNK;
1872			if ((nm = CAST(struct magic *, realloc(me->mp,
1873			    sizeof(*nm) * cnt))) == NULL) {
1874				file_oomem(ms, sizeof(*nm) * cnt);
1875				return -1;
1876			}
1877			me->mp = nm;
1878			me->max_count = CAST(uint32_t, cnt);
1879		}
1880		m = &me->mp[me->cont_count++];
1881		(void)memset(m, 0, sizeof(*m));
1882		m->cont_level = cont_level;
1883	} else {
1884		static const size_t len = sizeof(*m) * ALLOC_CHUNK;
1885		if (me->mp != NULL)
1886			return 1;
1887		if ((m = CAST(struct magic *, malloc(len))) == NULL) {
1888			file_oomem(ms, len);
1889			return -1;
1890		}
1891		me->mp = m;
1892		me->max_count = ALLOC_CHUNK;
1893		(void)memset(m, 0, sizeof(*m));
1894		m->factor_op = FILE_FACTOR_OP_NONE;
1895		m->cont_level = 0;
1896		me->cont_count = 1;
1897	}
1898	m->lineno = CAST(uint32_t, lineno);
1899
1900	if (*l == '&') {  /* m->cont_level == 0 checked below. */
1901                ++l;            /* step over */
1902                m->flag |= OFFADD;
1903        }
1904	if (*l == '(') {
1905		++l;		/* step over */
1906		m->flag |= INDIR;
1907		if (m->flag & OFFADD)
1908			m->flag = (m->flag & ~OFFADD) | INDIROFFADD;
1909
1910		if (*l == '&') {  /* m->cont_level == 0 checked below */
1911			++l;            /* step over */
1912			m->flag |= OFFADD;
1913		}
1914	}
1915	/* Indirect offsets are not valid at level 0. */
1916	if (m->cont_level == 0 && (m->flag & (OFFADD | INDIROFFADD))) {
1917		if (ms->flags & MAGIC_CHECK)
1918			file_magwarn(ms, "relative offset at level 0");
1919		return -1;
1920	}
1921
1922	/* get offset, then skip over it */
1923	m->offset = CAST(int32_t, strtol(l, &t, 0));
1924        if (l == t) {
1925		if (ms->flags & MAGIC_CHECK)
1926			file_magwarn(ms, "offset `%s' invalid", l);
1927		return -1;
1928	}
1929
1930        l = t;
1931
1932	if (m->flag & INDIR) {
1933		m->in_type = FILE_LONG;
1934		m->in_offset = 0;
1935		m->in_op = 0;
1936		/*
1937		 * read [.,lbs][+-]nnnnn)
1938		 */
1939		if (*l == '.' || *l == ',') {
1940			if (*l == ',')
1941				m->in_op |= FILE_OPSIGNED;
1942			l++;
1943			switch (*l) {
1944			case 'l':
1945				m->in_type = FILE_LELONG;
1946				break;
1947			case 'L':
1948				m->in_type = FILE_BELONG;
1949				break;
1950			case 'm':
1951				m->in_type = FILE_MELONG;
1952				break;
1953			case 'h':
1954			case 's':
1955				m->in_type = FILE_LESHORT;
1956				break;
1957			case 'H':
1958			case 'S':
1959				m->in_type = FILE_BESHORT;
1960				break;
1961			case 'c':
1962			case 'b':
1963			case 'C':
1964			case 'B':
1965				m->in_type = FILE_BYTE;
1966				break;
1967			case 'e':
1968			case 'f':
1969			case 'g':
1970				m->in_type = FILE_LEDOUBLE;
1971				break;
1972			case 'E':
1973			case 'F':
1974			case 'G':
1975				m->in_type = FILE_BEDOUBLE;
1976				break;
1977			case 'i':
1978				m->in_type = FILE_LEID3;
1979				break;
1980			case 'I':
1981				m->in_type = FILE_BEID3;
1982				break;
1983			case 'q':
1984				m->in_type = FILE_LEQUAD;
1985				break;
1986			case 'Q':
1987				m->in_type = FILE_BEQUAD;
1988				break;
1989			default:
1990				if (ms->flags & MAGIC_CHECK)
1991					file_magwarn(ms,
1992					    "indirect offset type `%c' invalid",
1993					    *l);
1994				return -1;
1995			}
1996			l++;
1997		}
1998
1999		if (*l == '~') {
2000			m->in_op |= FILE_OPINVERSE;
2001			l++;
2002		}
2003		if ((op = get_op(*l)) != -1) {
2004			m->in_op |= op;
2005			l++;
2006		}
2007		if (*l == '(') {
2008			m->in_op |= FILE_OPINDIRECT;
2009			l++;
2010		}
2011		if (isdigit(CAST(unsigned char, *l)) || *l == '-') {
2012			m->in_offset = CAST(int32_t, strtol(l, &t, 0));
2013			if (l == t) {
2014				if (ms->flags & MAGIC_CHECK)
2015					file_magwarn(ms,
2016					    "in_offset `%s' invalid", l);
2017				return -1;
2018			}
2019			l = t;
2020		}
2021		if (*l++ != ')' ||
2022		    ((m->in_op & FILE_OPINDIRECT) && *l++ != ')')) {
2023			if (ms->flags & MAGIC_CHECK)
2024				file_magwarn(ms,
2025				    "missing ')' in indirect offset");
2026			return -1;
2027		}
2028	}
2029	EATAB;
2030
2031#ifdef ENABLE_CONDITIONALS
2032	m->cond = get_cond(l, &l);
2033	if (check_cond(ms, m->cond, cont_level) == -1)
2034		return -1;
2035
2036	EATAB;
2037#endif
2038
2039	/*
2040	 * Parse the type.
2041	 */
2042	if (*l == 'u') {
2043		/*
2044		 * Try it as a keyword type prefixed by "u"; match what
2045		 * follows the "u".  If that fails, try it as an SUS
2046		 * integer type.
2047		 */
2048		m->type = get_type(type_tbl, l + 1, &l);
2049		if (m->type == FILE_INVALID) {
2050			/*
2051			 * Not a keyword type; parse it as an SUS type,
2052			 * 'u' possibly followed by a number or C/S/L.
2053			 */
2054			m->type = get_standard_integer_type(l, &l);
2055		}
2056		/* It's unsigned. */
2057		if (m->type != FILE_INVALID)
2058			m->flag |= UNSIGNED;
2059	} else {
2060		/*
2061		 * Try it as a keyword type.  If that fails, try it as
2062		 * an SUS integer type if it begins with "d" or as an
2063		 * SUS string type if it begins with "s".  In any case,
2064		 * it's not unsigned.
2065		 */
2066		m->type = get_type(type_tbl, l, &l);
2067		if (m->type == FILE_INVALID) {
2068			/*
2069			 * Not a keyword type; parse it as an SUS type,
2070			 * either 'd' possibly followed by a number or
2071			 * C/S/L, or just 's'.
2072			 */
2073			if (*l == 'd')
2074				m->type = get_standard_integer_type(l, &l);
2075			else if (*l == 's'
2076			    && !isalpha(CAST(unsigned char, l[1]))) {
2077				m->type = FILE_STRING;
2078				++l;
2079			}
2080		}
2081	}
2082
2083	if (m->type == FILE_INVALID) {
2084		/* Not found - try it as a special keyword. */
2085		m->type = get_type(special_tbl, l, &l);
2086	}
2087
2088	if (m->type == FILE_INVALID) {
2089		if (ms->flags & MAGIC_CHECK)
2090			file_magwarn(ms, "type `%s' invalid", l);
2091		return -1;
2092	}
2093
2094	/* New-style anding: "0 byte&0x80 =0x80 dynamically linked" */
2095	/* New and improved: ~ & | ^ + - * / % -- exciting, isn't it? */
2096
2097	m->mask_op = 0;
2098	if (*l == '~') {
2099		if (!IS_STRING(m->type))
2100			m->mask_op |= FILE_OPINVERSE;
2101		else if (ms->flags & MAGIC_CHECK)
2102			file_magwarn(ms, "'~' invalid for string types");
2103		++l;
2104	}
2105	m->str_range = 0;
2106	m->str_flags = m->type == FILE_PSTRING ? PSTRING_1_LE : 0;
2107	if ((op = get_op(*l)) != -1) {
2108		if (IS_STRING(m->type)) {
2109			int r;
2110
2111			if (op != FILE_OPDIVIDE) {
2112				if (ms->flags & MAGIC_CHECK)
2113					file_magwarn(ms,
2114					    "invalid string/indirect op: "
2115					    "`%c'", *t);
2116				return -1;
2117			}
2118
2119			if (m->type == FILE_INDIRECT)
2120				r = parse_indirect_modifier(ms, m, &l);
2121			else
2122				r = parse_string_modifier(ms, m, &l);
2123			if (r == -1)
2124				return -1;
2125		} else
2126			parse_op_modifier(ms, m, &l, op);
2127	}
2128
2129	/*
2130	 * We used to set mask to all 1's here, instead let's just not do
2131	 * anything if mask = 0 (unless you have a better idea)
2132	 */
2133	EATAB;
2134
2135	switch (*l) {
2136	case '>':
2137	case '<':
2138  		m->reln = *l;
2139  		++l;
2140		if (*l == '=') {
2141			if (ms->flags & MAGIC_CHECK) {
2142				file_magwarn(ms, "%c= not supported",
2143				    m->reln);
2144				return -1;
2145			}
2146		   ++l;
2147		}
2148		break;
2149	/* Old-style anding: "0 byte &0x80 dynamically linked" */
2150	case '&':
2151	case '^':
2152	case '=':
2153  		m->reln = *l;
2154  		++l;
2155		if (*l == '=') {
2156		   /* HP compat: ignore &= etc. */
2157		   ++l;
2158		}
2159		break;
2160	case '!':
2161		m->reln = *l;
2162		++l;
2163		break;
2164	default:
2165  		m->reln = '=';	/* the default relation */
2166		if (*l == 'x' && ((isascii(CAST(unsigned char, l[1])) &&
2167		    isspace(CAST(unsigned char, l[1]))) || !l[1])) {
2168			m->reln = *l;
2169			++l;
2170		}
2171		break;
2172	}
2173	/*
2174	 * Grab the value part, except for an 'x' reln.
2175	 */
2176	if (m->reln != 'x' && getvalue(ms, m, &l, action))
2177		return -1;
2178
2179	/*
2180	 * TODO finish this macro and start using it!
2181	 * #define offsetcheck {if (offset > ms->bytes_max -1)
2182	 *	magwarn("offset too big"); }
2183	 */
2184
2185	/*
2186	 * Now get last part - the description
2187	 */
2188	EATAB;
2189	if (l[0] == '\b') {
2190		++l;
2191		m->flag |= NOSPACE;
2192	} else if ((l[0] == '\\') && (l[1] == 'b')) {
2193		++l;
2194		++l;
2195		m->flag |= NOSPACE;
2196	}
2197	for (i = 0; (m->desc[i++] = *l++) != '\0' && i < sizeof(m->desc); )
2198		continue;
2199	if (i == sizeof(m->desc)) {
2200		m->desc[sizeof(m->desc) - 1] = '\0';
2201		if (ms->flags & MAGIC_CHECK)
2202			file_magwarn(ms, "description `%s' truncated", m->desc);
2203	}
2204
2205        /*
2206	 * We only do this check while compiling, or if any of the magic
2207	 * files were not compiled.
2208         */
2209        if (ms->flags & MAGIC_CHECK) {
2210		if (check_format(ms, m) == -1)
2211			return -1;
2212	}
2213#ifndef COMPILE_ONLY
2214	if (action == FILE_CHECK) {
2215		file_mdump(m);
2216	}
2217#endif
2218	m->mimetype[0] = '\0';		/* initialise MIME type to none */
2219	return 0;
2220}
2221
2222/*
2223 * parse a STRENGTH annotation line from magic file, put into magic[index - 1]
2224 * if valid
2225 */
2226private int
2227parse_strength(struct magic_set *ms, struct magic_entry *me, const char *line)
2228{
2229	const char *l = line;
2230	char *el;
2231	unsigned long factor;
2232	struct magic *m = &me->mp[0];
2233
2234	if (m->factor_op != FILE_FACTOR_OP_NONE) {
2235		file_magwarn(ms,
2236		    "Current entry already has a strength type: %c %d",
2237		    m->factor_op, m->factor);
2238		return -1;
2239	}
2240	if (m->type == FILE_NAME) {
2241		file_magwarn(ms, "%s: Strength setting is not supported in "
2242		    "\"name\" magic entries", m->value.s);
2243		return -1;
2244	}
2245	EATAB;
2246	switch (*l) {
2247	case FILE_FACTOR_OP_NONE:
2248	case FILE_FACTOR_OP_PLUS:
2249	case FILE_FACTOR_OP_MINUS:
2250	case FILE_FACTOR_OP_TIMES:
2251	case FILE_FACTOR_OP_DIV:
2252		m->factor_op = *l++;
2253		break;
2254	default:
2255		file_magwarn(ms, "Unknown factor op `%c'", *l);
2256		return -1;
2257	}
2258	EATAB;
2259	factor = strtoul(l, &el, 0);
2260	if (factor > 255) {
2261		file_magwarn(ms, "Too large factor `%lu'", factor);
2262		goto out;
2263	}
2264	if (*el && !isspace(CAST(unsigned char, *el))) {
2265		file_magwarn(ms, "Bad factor `%s'", l);
2266		goto out;
2267	}
2268	m->factor = CAST(uint8_t, factor);
2269	if (m->factor == 0 && m->factor_op == FILE_FACTOR_OP_DIV) {
2270		file_magwarn(ms, "Cannot have factor op `%c' and factor %u",
2271		    m->factor_op, m->factor);
2272		goto out;
2273	}
2274	return 0;
2275out:
2276	m->factor_op = FILE_FACTOR_OP_NONE;
2277	m->factor = 0;
2278	return -1;
2279}
2280
2281private int
2282goodchar(unsigned char x, const char *extra)
2283{
2284	return (isascii(x) && isalnum(x)) || strchr(extra, x);
2285}
2286
2287private int
2288parse_extra(struct magic_set *ms, struct magic_entry *me, const char *line,
2289    off_t off, size_t len, const char *name, const char *extra, int nt)
2290{
2291	size_t i;
2292	const char *l = line;
2293	struct magic *m = &me->mp[me->cont_count == 0 ? 0 : me->cont_count - 1];
2294	char *buf = CAST(char *, CAST(void *, m)) + off;
2295
2296	if (buf[0] != '\0') {
2297		len = nt ? strlen(buf) : len;
2298		file_magwarn(ms, "Current entry already has a %s type "
2299		    "`%.*s', new type `%s'", name, CAST(int, len), buf, l);
2300		return -1;
2301	}
2302
2303	if (*m->desc == '\0') {
2304		file_magwarn(ms, "Current entry does not yet have a "
2305		    "description for adding a %s type", name);
2306		return -1;
2307	}
2308
2309	EATAB;
2310	for (i = 0; *l && i < len && goodchar(*l, extra); buf[i++] = *l++)
2311		continue;
2312
2313	if (i == len && *l) {
2314		if (nt)
2315			buf[len - 1] = '\0';
2316		if (ms->flags & MAGIC_CHECK)
2317			file_magwarn(ms, "%s type `%s' truncated %"
2318			    SIZE_T_FORMAT "u", name, line, i);
2319	} else {
2320		if (!isspace(CAST(unsigned char, *l)) && !goodchar(*l, extra))
2321			file_magwarn(ms, "%s type `%s' has bad char '%c'",
2322			    name, line, *l);
2323		if (nt)
2324			buf[i] = '\0';
2325	}
2326
2327	if (i > 0)
2328		return 0;
2329
2330	file_magerror(ms, "Bad magic entry '%s'", line);
2331	return -1;
2332}
2333
2334/*
2335 * Parse an Apple CREATOR/TYPE annotation from magic file and put it into
2336 * magic[index - 1]
2337 */
2338private int
2339parse_apple(struct magic_set *ms, struct magic_entry *me, const char *line)
2340{
2341	struct magic *m = &me->mp[0];
2342
2343	return parse_extra(ms, me, line,
2344	    CAST(off_t, offsetof(struct magic, apple)),
2345	    sizeof(m->apple), "APPLE", "!+-./?", 0);
2346}
2347
2348/*
2349 * Parse a comma-separated list of extensions
2350 */
2351private int
2352parse_ext(struct magic_set *ms, struct magic_entry *me, const char *line)
2353{
2354	struct magic *m = &me->mp[0];
2355
2356	return parse_extra(ms, me, line,
2357	    CAST(off_t, offsetof(struct magic, ext)),
2358	    sizeof(m->ext), "EXTENSION", ",!+-/@?_$", 0);
2359}
2360
2361/*
2362 * parse a MIME annotation line from magic file, put into magic[index - 1]
2363 * if valid
2364 */
2365private int
2366parse_mime(struct magic_set *ms, struct magic_entry *me, const char *line)
2367{
2368	struct magic *m = &me->mp[0];
2369
2370	return parse_extra(ms, me, line,
2371	    CAST(off_t, offsetof(struct magic, mimetype)),
2372	    sizeof(m->mimetype), "MIME", "+-/.$?:{}", 1);
2373}
2374
2375private int
2376check_format_type(const char *ptr, int type, const char **estr)
2377{
2378	int quad = 0, h;
2379	size_t len, cnt;
2380	if (*ptr == '\0') {
2381		/* Missing format string; bad */
2382		*estr = "missing format spec";
2383		return -1;
2384	}
2385
2386	switch (file_formats[type]) {
2387	case FILE_FMT_QUAD:
2388		quad = 1;
2389		/*FALLTHROUGH*/
2390	case FILE_FMT_NUM:
2391		if (quad == 0) {
2392			switch (type) {
2393			case FILE_BYTE:
2394				h = 2;
2395				break;
2396			case FILE_SHORT:
2397			case FILE_BESHORT:
2398			case FILE_LESHORT:
2399				h = 1;
2400				break;
2401			case FILE_LONG:
2402			case FILE_BELONG:
2403			case FILE_LELONG:
2404			case FILE_MELONG:
2405			case FILE_LEID3:
2406			case FILE_BEID3:
2407			case FILE_INDIRECT:
2408				h = 0;
2409				break;
2410			default:
2411				abort();
2412			}
2413		} else
2414			h = 0;
2415		if (*ptr == '-')
2416			ptr++;
2417		if (*ptr == '.')
2418			ptr++;
2419		if (*ptr == '#')
2420			ptr++;
2421#define CHECKLEN() do { \
2422	for (len = cnt = 0; isdigit(CAST(unsigned char, *ptr)); ptr++, cnt++) \
2423		len = len * 10 + (*ptr - '0'); \
2424	if (cnt > 5 || len > 1024) \
2425		goto toolong; \
2426} while (/*CONSTCOND*/0)
2427
2428		CHECKLEN();
2429		if (*ptr == '.')
2430			ptr++;
2431		CHECKLEN();
2432		if (quad) {
2433			if (*ptr++ != 'l')
2434				goto invalid;
2435			if (*ptr++ != 'l')
2436				goto invalid;
2437		}
2438
2439		switch (*ptr++) {
2440#ifdef STRICT_FORMAT 	/* "long" formats are int formats for us */
2441		/* so don't accept the 'l' modifier */
2442		case 'l':
2443			switch (*ptr++) {
2444			case 'i':
2445			case 'd':
2446			case 'u':
2447			case 'o':
2448			case 'x':
2449			case 'X':
2450				if (h == 0)
2451					return 0;
2452				/*FALLTHROUGH*/
2453			default:
2454				goto invalid;
2455			}
2456
2457		/*
2458		 * Don't accept h and hh modifiers. They make writing
2459		 * magic entries more complicated, for very little benefit
2460		 */
2461		case 'h':
2462			if (h-- <= 0)
2463				goto invalid;
2464			switch (*ptr++) {
2465			case 'h':
2466				if (h-- <= 0)
2467					goto invalid;
2468				switch (*ptr++) {
2469				case 'i':
2470				case 'd':
2471				case 'u':
2472				case 'o':
2473				case 'x':
2474				case 'X':
2475					return 0;
2476				default:
2477					goto invalid;
2478				}
2479			case 'i':
2480			case 'd':
2481			case 'u':
2482			case 'o':
2483			case 'x':
2484			case 'X':
2485				if (h == 0)
2486					return 0;
2487				/*FALLTHROUGH*/
2488			default:
2489				goto invalid;
2490			}
2491#endif
2492		case 'c':
2493			if (h == 2)
2494				return 0;
2495			goto invalid;
2496		case 'i':
2497		case 'd':
2498		case 'u':
2499		case 'o':
2500		case 'x':
2501		case 'X':
2502#ifdef STRICT_FORMAT
2503			if (h == 0)
2504				return 0;
2505			/*FALLTHROUGH*/
2506#else
2507			return 0;
2508#endif
2509		default:
2510			goto invalid;
2511		}
2512
2513	case FILE_FMT_FLOAT:
2514	case FILE_FMT_DOUBLE:
2515		if (*ptr == '-')
2516			ptr++;
2517		if (*ptr == '.')
2518			ptr++;
2519		CHECKLEN();
2520		if (*ptr == '.')
2521			ptr++;
2522		CHECKLEN();
2523		switch (*ptr++) {
2524		case 'e':
2525		case 'E':
2526		case 'f':
2527		case 'F':
2528		case 'g':
2529		case 'G':
2530			return 0;
2531
2532		default:
2533			goto invalid;
2534		}
2535
2536
2537	case FILE_FMT_STR:
2538		if (*ptr == '-')
2539			ptr++;
2540		while (isdigit(CAST(unsigned char, *ptr)))
2541			ptr++;
2542		if (*ptr == '.') {
2543			ptr++;
2544			while (isdigit(CAST(unsigned char , *ptr)))
2545				ptr++;
2546		}
2547
2548		switch (*ptr++) {
2549		case 's':
2550			return 0;
2551		default:
2552			goto invalid;
2553		}
2554
2555	default:
2556		/* internal error */
2557		abort();
2558	}
2559invalid:
2560	*estr = "not valid";
2561toolong:
2562	*estr = "too long";
2563	return -1;
2564}
2565
2566/*
2567 * Check that the optional printf format in description matches
2568 * the type of the magic.
2569 */
2570private int
2571check_format(struct magic_set *ms, struct magic *m)
2572{
2573	char *ptr;
2574	const char *estr;
2575
2576	for (ptr = m->desc; *ptr; ptr++)
2577		if (*ptr == '%')
2578			break;
2579	if (*ptr == '\0') {
2580		/* No format string; ok */
2581		return 1;
2582	}
2583
2584	assert(file_nformats == file_nnames);
2585
2586	if (m->type >= file_nformats) {
2587		file_magwarn(ms, "Internal error inconsistency between "
2588		    "m->type and format strings");
2589		return -1;
2590	}
2591	if (file_formats[m->type] == FILE_FMT_NONE) {
2592		file_magwarn(ms, "No format string for `%s' with description "
2593		    "`%s'", m->desc, file_names[m->type]);
2594		return -1;
2595	}
2596
2597	ptr++;
2598	if (check_format_type(ptr, m->type, &estr) == -1) {
2599		/*
2600		 * TODO: this error message is unhelpful if the format
2601		 * string is not one character long
2602		 */
2603		file_magwarn(ms, "Printf format is %s for type "
2604		    "`%s' in description `%s'", estr,
2605		    file_names[m->type], m->desc);
2606		return -1;
2607	}
2608
2609	for (; *ptr; ptr++) {
2610		if (*ptr == '%') {
2611			file_magwarn(ms,
2612			    "Too many format strings (should have at most one) "
2613			    "for `%s' with description `%s'",
2614			    file_names[m->type], m->desc);
2615			return -1;
2616		}
2617	}
2618	return 0;
2619}
2620
2621/*
2622 * Read a numeric value from a pointer, into the value union of a magic
2623 * pointer, according to the magic type.  Update the string pointer to point
2624 * just after the number read.  Return 0 for success, non-zero for failure.
2625 */
2626private int
2627getvalue(struct magic_set *ms, struct magic *m, const char **p, int action)
2628{
2629	char *ep;
2630	uint64_t ull;
2631
2632	switch (m->type) {
2633	case FILE_BESTRING16:
2634	case FILE_LESTRING16:
2635	case FILE_STRING:
2636	case FILE_PSTRING:
2637	case FILE_REGEX:
2638	case FILE_SEARCH:
2639	case FILE_NAME:
2640	case FILE_USE:
2641	case FILE_DER:
2642		*p = getstr(ms, m, *p, action == FILE_COMPILE);
2643		if (*p == NULL) {
2644			if (ms->flags & MAGIC_CHECK)
2645				file_magwarn(ms, "cannot get string from `%s'",
2646				    m->value.s);
2647			return -1;
2648		}
2649		if (m->type == FILE_REGEX) {
2650			file_regex_t rx;
2651			int rc = file_regcomp(&rx, m->value.s, REG_EXTENDED);
2652			if (rc) {
2653				if (ms->flags & MAGIC_CHECK)
2654					file_regerror(&rx, rc, ms);
2655			}
2656			file_regfree(&rx);
2657			return rc ? -1 : 0;
2658		}
2659		return 0;
2660	default:
2661		if (m->reln == 'x')
2662			return 0;
2663		break;
2664	}
2665
2666	switch (m->type) {
2667	case FILE_FLOAT:
2668	case FILE_BEFLOAT:
2669	case FILE_LEFLOAT:
2670		errno = 0;
2671#ifdef HAVE_STRTOF
2672		m->value.f = strtof(*p, &ep);
2673#else
2674		m->value.f = (float)strtod(*p, &ep);
2675#endif
2676		if (errno == 0)
2677			*p = ep;
2678		return 0;
2679	case FILE_DOUBLE:
2680	case FILE_BEDOUBLE:
2681	case FILE_LEDOUBLE:
2682		errno = 0;
2683		m->value.d = strtod(*p, &ep);
2684		if (errno == 0)
2685			*p = ep;
2686		return 0;
2687	default:
2688		errno = 0;
2689		ull = CAST(uint64_t, strtoull(*p, &ep, 0));
2690		m->value.q = file_signextend(ms, m, ull);
2691		if (*p == ep) {
2692			file_magwarn(ms, "Unparseable number `%s'", *p);
2693		} else {
2694			size_t ts = typesize(m->type);
2695			uint64_t x;
2696			const char *q;
2697
2698			if (ts == CAST(size_t, ~0)) {
2699				file_magwarn(ms,
2700				    "Expected numeric type got `%s'",
2701				    type_tbl[m->type].name);
2702			}
2703			for (q = *p; isspace(CAST(unsigned char, *q)); q++)
2704				continue;
2705			if (*q == '-')
2706				ull = -CAST(int64_t, ull);
2707			switch (ts) {
2708			case 1:
2709				x = CAST(uint64_t, ull & ~0xffULL);
2710				break;
2711			case 2:
2712				x = CAST(uint64_t, ull & ~0xffffULL);
2713				break;
2714			case 4:
2715				x = CAST(uint64_t, ull & ~0xffffffffULL);
2716				break;
2717			case 8:
2718				x = 0;
2719				break;
2720			default:
2721				abort();
2722			}
2723			if (x) {
2724				file_magwarn(ms, "Overflow for numeric"
2725				    " type `%s' value %#" PRIx64,
2726				    type_tbl[m->type].name, ull);
2727			}
2728		}
2729		if (errno == 0) {
2730			*p = ep;
2731			eatsize(p);
2732		}
2733		return 0;
2734	}
2735}
2736
2737/*
2738 * Convert a string containing C character escapes.  Stop at an unescaped
2739 * space or tab.
2740 * Copy the converted version to "m->value.s", and the length in m->vallen.
2741 * Return updated scan pointer as function result. Warn if set.
2742 */
2743private const char *
2744getstr(struct magic_set *ms, struct magic *m, const char *s, int warn)
2745{
2746	const char *origs = s;
2747	char	*p = m->value.s;
2748	size_t  plen = sizeof(m->value.s);
2749	char 	*origp = p;
2750	char	*pmax = p + plen - 1;
2751	int	c;
2752	int	val;
2753
2754	while ((c = *s++) != '\0') {
2755		if (isspace(CAST(unsigned char, c)))
2756			break;
2757		if (p >= pmax) {
2758			file_error(ms, 0, "string too long: `%s'", origs);
2759			return NULL;
2760		}
2761		if (c == '\\') {
2762			switch(c = *s++) {
2763
2764			case '\0':
2765				if (warn)
2766					file_magwarn(ms, "incomplete escape");
2767				s--;
2768				goto out;
2769
2770			case '\t':
2771				if (warn) {
2772					file_magwarn(ms,
2773					    "escaped tab found, use \\t instead");
2774					warn = 0;	/* already did */
2775				}
2776				/*FALLTHROUGH*/
2777			default:
2778				if (warn) {
2779					if (isprint(CAST(unsigned char, c))) {
2780						/* Allow escaping of
2781						 * ``relations'' */
2782						if (strchr("<>&^=!", c) == NULL
2783						    && (m->type != FILE_REGEX ||
2784						    strchr("[]().*?^$|{}", c)
2785						    == NULL)) {
2786							file_magwarn(ms, "no "
2787							    "need to escape "
2788							    "`%c'", c);
2789						}
2790					} else {
2791						file_magwarn(ms,
2792						    "unknown escape sequence: "
2793						    "\\%03o", c);
2794					}
2795				}
2796				/*FALLTHROUGH*/
2797			/* space, perhaps force people to use \040? */
2798			case ' ':
2799#if 0
2800			/*
2801			 * Other things people escape, but shouldn't need to,
2802			 * so we disallow them
2803			 */
2804			case '\'':
2805			case '"':
2806			case '?':
2807#endif
2808			/* Relations */
2809			case '>':
2810			case '<':
2811			case '&':
2812			case '^':
2813			case '=':
2814			case '!':
2815			/* and baskslash itself */
2816			case '\\':
2817				*p++ = CAST(char, c);
2818				break;
2819
2820			case 'a':
2821				*p++ = '\a';
2822				break;
2823
2824			case 'b':
2825				*p++ = '\b';
2826				break;
2827
2828			case 'f':
2829				*p++ = '\f';
2830				break;
2831
2832			case 'n':
2833				*p++ = '\n';
2834				break;
2835
2836			case 'r':
2837				*p++ = '\r';
2838				break;
2839
2840			case 't':
2841				*p++ = '\t';
2842				break;
2843
2844			case 'v':
2845				*p++ = '\v';
2846				break;
2847
2848			/* \ and up to 3 octal digits */
2849			case '0':
2850			case '1':
2851			case '2':
2852			case '3':
2853			case '4':
2854			case '5':
2855			case '6':
2856			case '7':
2857				val = c - '0';
2858				c = *s++;  /* try for 2 */
2859				if (c >= '0' && c <= '7') {
2860					val = (val << 3) | (c - '0');
2861					c = *s++;  /* try for 3 */
2862					if (c >= '0' && c <= '7')
2863						val = (val << 3) | (c-'0');
2864					else
2865						--s;
2866				}
2867				else
2868					--s;
2869				*p++ = CAST(char, val);
2870				break;
2871
2872			/* \x and up to 2 hex digits */
2873			case 'x':
2874				val = 'x';	/* Default if no digits */
2875				c = hextoint(*s++);	/* Get next char */
2876				if (c >= 0) {
2877					val = c;
2878					c = hextoint(*s++);
2879					if (c >= 0)
2880						val = (val << 4) + c;
2881					else
2882						--s;
2883				} else
2884					--s;
2885				*p++ = CAST(char, val);
2886				break;
2887			}
2888		} else
2889			*p++ = CAST(char, c);
2890	}
2891	--s;
2892out:
2893	*p = '\0';
2894	m->vallen = CAST(unsigned char, (p - origp));
2895	if (m->type == FILE_PSTRING)
2896		m->vallen += CAST(unsigned char, file_pstring_length_size(m));
2897	return s;
2898}
2899
2900
2901/* Single hex char to int; -1 if not a hex char. */
2902private int
2903hextoint(int c)
2904{
2905	if (!isascii(CAST(unsigned char, c)))
2906		return -1;
2907	if (isdigit(CAST(unsigned char, c)))
2908		return c - '0';
2909	if ((c >= 'a') && (c <= 'f'))
2910		return c + 10 - 'a';
2911	if (( c>= 'A') && (c <= 'F'))
2912		return c + 10 - 'A';
2913	return -1;
2914}
2915
2916
2917/*
2918 * Print a string containing C character escapes.
2919 */
2920protected void
2921file_showstr(FILE *fp, const char *s, size_t len)
2922{
2923	char	c;
2924
2925	for (;;) {
2926		if (len == ~0U) {
2927			c = *s++;
2928			if (c == '\0')
2929				break;
2930		}
2931		else  {
2932			if (len-- == 0)
2933				break;
2934			c = *s++;
2935		}
2936		if (c >= 040 && c <= 0176)	/* TODO isprint && !iscntrl */
2937			(void) fputc(c, fp);
2938		else {
2939			(void) fputc('\\', fp);
2940			switch (c) {
2941			case '\a':
2942				(void) fputc('a', fp);
2943				break;
2944
2945			case '\b':
2946				(void) fputc('b', fp);
2947				break;
2948
2949			case '\f':
2950				(void) fputc('f', fp);
2951				break;
2952
2953			case '\n':
2954				(void) fputc('n', fp);
2955				break;
2956
2957			case '\r':
2958				(void) fputc('r', fp);
2959				break;
2960
2961			case '\t':
2962				(void) fputc('t', fp);
2963				break;
2964
2965			case '\v':
2966				(void) fputc('v', fp);
2967				break;
2968
2969			default:
2970				(void) fprintf(fp, "%.3o", c & 0377);
2971				break;
2972			}
2973		}
2974	}
2975}
2976
2977/*
2978 * eatsize(): Eat the size spec from a number [eg. 10UL]
2979 */
2980private void
2981eatsize(const char **p)
2982{
2983	const char *l = *p;
2984
2985	if (LOWCASE(*l) == 'u')
2986		l++;
2987
2988	switch (LOWCASE(*l)) {
2989	case 'l':    /* long */
2990	case 's':    /* short */
2991	case 'h':    /* short */
2992	case 'b':    /* char/byte */
2993	case 'c':    /* char/byte */
2994		l++;
2995		/*FALLTHROUGH*/
2996	default:
2997		break;
2998	}
2999
3000	*p = l;
3001}
3002
3003/*
3004 * handle a buffer containing a compiled file.
3005 */
3006private struct magic_map *
3007apprentice_buf(struct magic_set *ms, struct magic *buf, size_t len)
3008{
3009	struct magic_map *map;
3010
3011	if ((map = CAST(struct magic_map *, calloc(1, sizeof(*map)))) == NULL) {
3012		file_oomem(ms, sizeof(*map));
3013		return NULL;
3014	}
3015	map->len = len;
3016	map->p = buf;
3017	map->type = MAP_TYPE_USER;
3018	if (check_buffer(ms, map, "buffer") != 0) {
3019		apprentice_unmap(map);
3020		return NULL;
3021	}
3022	return map;
3023}
3024
3025/*
3026 * handle a compiled file.
3027 */
3028
3029private struct magic_map *
3030apprentice_map(struct magic_set *ms, const char *fn)
3031{
3032	int fd;
3033	struct stat st;
3034	char *dbname = NULL;
3035	struct magic_map *map;
3036	struct magic_map *rv = NULL;
3037
3038	fd = -1;
3039	if ((map = CAST(struct magic_map *, calloc(1, sizeof(*map)))) == NULL) {
3040		file_oomem(ms, sizeof(*map));
3041		goto error;
3042	}
3043	map->type = MAP_TYPE_USER;	/* unspecified */
3044
3045	dbname = mkdbname(ms, fn, 0);
3046	if (dbname == NULL)
3047		goto error;
3048
3049	if ((fd = open(dbname, O_RDONLY|O_BINARY)) == -1)
3050		goto error;
3051
3052	if (fstat(fd, &st) == -1) {
3053		file_error(ms, errno, "cannot stat `%s'", dbname);
3054		goto error;
3055	}
3056	if (st.st_size < 8 || st.st_size > maxoff_t()) {
3057		file_error(ms, 0, "file `%s' is too %s", dbname,
3058		    st.st_size < 8 ? "small" : "large");
3059		goto error;
3060	}
3061
3062	map->len = CAST(size_t, st.st_size);
3063#ifdef QUICK
3064	map->type = MAP_TYPE_MMAP;
3065	if ((map->p = mmap(0, CAST(size_t, st.st_size), PROT_READ|PROT_WRITE,
3066	    MAP_PRIVATE|MAP_FILE, fd, CAST(off_t, 0))) == MAP_FAILED) {
3067		file_error(ms, errno, "cannot map `%s'", dbname);
3068		goto error;
3069	}
3070#else
3071	map->type = MAP_TYPE_MALLOC;
3072	if ((map->p = CAST(void *, malloc(map->len))) == NULL) {
3073		file_oomem(ms, map->len);
3074		goto error;
3075	}
3076	if (read(fd, map->p, map->len) != (ssize_t)map->len) {
3077		file_badread(ms);
3078		goto error;
3079	}
3080#define RET	1
3081#endif
3082	(void)close(fd);
3083	fd = -1;
3084
3085	if (check_buffer(ms, map, dbname) != 0) {
3086		rv = RCAST(struct magic_map *, -1);
3087		goto error;
3088	}
3089#ifdef QUICK
3090	if (mprotect(map->p, CAST(size_t, st.st_size), PROT_READ) == -1) {
3091		file_error(ms, errno, "cannot mprotect `%s'", dbname);
3092		goto error;
3093	}
3094#endif
3095
3096	free(dbname);
3097	return map;
3098
3099error:
3100	if (fd != -1)
3101		(void)close(fd);
3102	apprentice_unmap(map);
3103	free(dbname);
3104	return rv;
3105}
3106
3107private int
3108check_buffer(struct magic_set *ms, struct magic_map *map, const char *dbname)
3109{
3110	uint32_t *ptr;
3111	uint32_t entries, nentries;
3112	uint32_t version;
3113	int i, needsbyteswap;
3114
3115	ptr = CAST(uint32_t *, map->p);
3116	if (*ptr != MAGICNO) {
3117		if (swap4(*ptr) != MAGICNO) {
3118			file_error(ms, 0, "bad magic in `%s'", dbname);
3119			return -1;
3120		}
3121		needsbyteswap = 1;
3122	} else
3123		needsbyteswap = 0;
3124	if (needsbyteswap)
3125		version = swap4(ptr[1]);
3126	else
3127		version = ptr[1];
3128	if (version != VERSIONNO) {
3129		file_error(ms, 0, "File %s supports only version %d magic "
3130		    "files. `%s' is version %d", VERSION,
3131		    VERSIONNO, dbname, version);
3132		return -1;
3133	}
3134	entries = CAST(uint32_t, map->len / sizeof(struct magic));
3135	if ((entries * sizeof(struct magic)) != map->len) {
3136		file_error(ms, 0, "Size of `%s' %" SIZE_T_FORMAT "u is not "
3137		    "a multiple of %" SIZE_T_FORMAT "u",
3138		    dbname, map->len, sizeof(struct magic));
3139		return -1;
3140	}
3141	map->magic[0] = CAST(struct magic *, map->p) + 1;
3142	nentries = 0;
3143	for (i = 0; i < MAGIC_SETS; i++) {
3144		if (needsbyteswap)
3145			map->nmagic[i] = swap4(ptr[i + 2]);
3146		else
3147			map->nmagic[i] = ptr[i + 2];
3148		if (i != MAGIC_SETS - 1)
3149			map->magic[i + 1] = map->magic[i] + map->nmagic[i];
3150		nentries += map->nmagic[i];
3151	}
3152	if (entries != nentries + 1) {
3153		file_error(ms, 0, "Inconsistent entries in `%s' %u != %u",
3154		    dbname, entries, nentries + 1);
3155		return -1;
3156	}
3157	if (needsbyteswap)
3158		for (i = 0; i < MAGIC_SETS; i++)
3159			byteswap(map->magic[i], map->nmagic[i]);
3160	return 0;
3161}
3162
3163/*
3164 * handle an mmaped file.
3165 */
3166private int
3167apprentice_compile(struct magic_set *ms, struct magic_map *map, const char *fn)
3168{
3169	static const size_t nm = sizeof(*map->nmagic) * MAGIC_SETS;
3170	static const size_t m = sizeof(**map->magic);
3171	int fd = -1;
3172	size_t len;
3173	char *dbname;
3174	int rv = -1;
3175	uint32_t i;
3176	union {
3177		struct magic m;
3178		uint32_t h[2 + MAGIC_SETS];
3179	} hdr;
3180
3181	dbname = mkdbname(ms, fn, 1);
3182
3183	if (dbname == NULL)
3184		goto out;
3185
3186	if ((fd = open(dbname, O_WRONLY|O_CREAT|O_TRUNC|O_BINARY, 0644)) == -1)
3187	{
3188		file_error(ms, errno, "cannot open `%s'", dbname);
3189		goto out;
3190	}
3191	memset(&hdr, 0, sizeof(hdr));
3192	hdr.h[0] = MAGICNO;
3193	hdr.h[1] = VERSIONNO;
3194	memcpy(hdr.h + 2, map->nmagic, nm);
3195
3196	if (write(fd, &hdr, sizeof(hdr)) != CAST(ssize_t, sizeof(hdr))) {
3197		file_error(ms, errno, "error writing `%s'", dbname);
3198		goto out2;
3199	}
3200
3201	for (i = 0; i < MAGIC_SETS; i++) {
3202		len = m * map->nmagic[i];
3203		if (write(fd, map->magic[i], len) != CAST(ssize_t, len)) {
3204			file_error(ms, errno, "error writing `%s'", dbname);
3205			goto out2;
3206		}
3207	}
3208
3209	rv = 0;
3210out2:
3211	if (fd != -1)
3212		(void)close(fd);
3213out:
3214	apprentice_unmap(map);
3215	free(dbname);
3216	return rv;
3217}
3218
3219private const char ext[] = ".mgc";
3220/*
3221 * make a dbname
3222 */
3223private char *
3224mkdbname(struct magic_set *ms, const char *fn, int strip)
3225{
3226	const char *p, *q;
3227	char *buf;
3228
3229	if (strip) {
3230		if ((p = strrchr(fn, '/')) != NULL)
3231			fn = ++p;
3232	}
3233
3234	for (q = fn; *q; q++)
3235		continue;
3236	/* Look for .mgc */
3237	for (p = ext + sizeof(ext) - 1; p >= ext && q >= fn; p--, q--)
3238		if (*p != *q)
3239			break;
3240
3241	/* Did not find .mgc, restore q */
3242	if (p >= ext)
3243		while (*q)
3244			q++;
3245
3246	q++;
3247	/* Compatibility with old code that looked in .mime */
3248	if (ms->flags & MAGIC_MIME) {
3249		if (asprintf(&buf, "%.*s.mime%s", CAST(int, q - fn), fn, ext)
3250		    < 0)
3251			return NULL;
3252		if (access(buf, R_OK) != -1) {
3253			ms->flags &= MAGIC_MIME_TYPE;
3254			return buf;
3255		}
3256		free(buf);
3257	}
3258	if (asprintf(&buf, "%.*s%s", CAST(int, q - fn), fn, ext) < 0)
3259		return NULL;
3260
3261	/* Compatibility with old code that looked in .mime */
3262	if (strstr(fn, ".mime") != NULL)
3263		ms->flags &= MAGIC_MIME_TYPE;
3264	return buf;
3265}
3266
3267/*
3268 * Byteswap an mmap'ed file if needed
3269 */
3270private void
3271byteswap(struct magic *magic, uint32_t nmagic)
3272{
3273	uint32_t i;
3274	for (i = 0; i < nmagic; i++)
3275		bs1(&magic[i]);
3276}
3277
3278/*
3279 * swap a short
3280 */
3281private uint16_t
3282swap2(uint16_t sv)
3283{
3284	uint16_t rv;
3285	uint8_t *s = RCAST(uint8_t *, RCAST(void *, &sv));
3286	uint8_t *d = RCAST(uint8_t *, RCAST(void *, &rv));
3287	d[0] = s[1];
3288	d[1] = s[0];
3289	return rv;
3290}
3291
3292/*
3293 * swap an int
3294 */
3295private uint32_t
3296swap4(uint32_t sv)
3297{
3298	uint32_t rv;
3299	uint8_t *s = RCAST(uint8_t *, RCAST(void *, &sv));
3300	uint8_t *d = RCAST(uint8_t *, RCAST(void *, &rv));
3301	d[0] = s[3];
3302	d[1] = s[2];
3303	d[2] = s[1];
3304	d[3] = s[0];
3305	return rv;
3306}
3307
3308/*
3309 * swap a quad
3310 */
3311private uint64_t
3312swap8(uint64_t sv)
3313{
3314	uint64_t rv;
3315	uint8_t *s = RCAST(uint8_t *, RCAST(void *, &sv));
3316	uint8_t *d = RCAST(uint8_t *, RCAST(void *, &rv));
3317#if 0
3318	d[0] = s[3];
3319	d[1] = s[2];
3320	d[2] = s[1];
3321	d[3] = s[0];
3322	d[4] = s[7];
3323	d[5] = s[6];
3324	d[6] = s[5];
3325	d[7] = s[4];
3326#else
3327	d[0] = s[7];
3328	d[1] = s[6];
3329	d[2] = s[5];
3330	d[3] = s[4];
3331	d[4] = s[3];
3332	d[5] = s[2];
3333	d[6] = s[1];
3334	d[7] = s[0];
3335#endif
3336	return rv;
3337}
3338
3339/*
3340 * byteswap a single magic entry
3341 */
3342private void
3343bs1(struct magic *m)
3344{
3345	m->cont_level = swap2(m->cont_level);
3346	m->offset = swap4(CAST(uint32_t, m->offset));
3347	m->in_offset = swap4(CAST(uint32_t, m->in_offset));
3348	m->lineno = swap4(CAST(uint32_t, m->lineno));
3349	if (IS_STRING(m->type)) {
3350		m->str_range = swap4(m->str_range);
3351		m->str_flags = swap4(m->str_flags);
3352	}
3353	else {
3354		m->value.q = swap8(m->value.q);
3355		m->num_mask = swap8(m->num_mask);
3356	}
3357}
3358
3359protected size_t
3360file_pstring_length_size(const struct magic *m)
3361{
3362	switch (m->str_flags & PSTRING_LEN) {
3363	case PSTRING_1_LE:
3364		return 1;
3365	case PSTRING_2_LE:
3366	case PSTRING_2_BE:
3367		return 2;
3368	case PSTRING_4_LE:
3369	case PSTRING_4_BE:
3370		return 4;
3371	default:
3372		abort();	/* Impossible */
3373		return 1;
3374	}
3375}
3376protected size_t
3377file_pstring_get_length(const struct magic *m, const char *ss)
3378{
3379	size_t len = 0;
3380	const unsigned char *s = RCAST(const unsigned char *, ss);
3381	unsigned int s3, s2, s1, s0;
3382
3383	switch (m->str_flags & PSTRING_LEN) {
3384	case PSTRING_1_LE:
3385		len = *s;
3386		break;
3387	case PSTRING_2_LE:
3388		s0 = s[0];
3389		s1 = s[1];
3390		len = (s1 << 8) | s0;
3391		break;
3392	case PSTRING_2_BE:
3393		s0 = s[0];
3394		s1 = s[1];
3395		len = (s0 << 8) | s1;
3396		break;
3397	case PSTRING_4_LE:
3398		s0 = s[0];
3399		s1 = s[1];
3400		s2 = s[2];
3401		s3 = s[3];
3402		len = (s3 << 24) | (s2 << 16) | (s1 << 8) | s0;
3403		break;
3404	case PSTRING_4_BE:
3405		s0 = s[0];
3406		s1 = s[1];
3407		s2 = s[2];
3408		s3 = s[3];
3409		len = (s0 << 24) | (s1 << 16) | (s2 << 8) | s3;
3410		break;
3411	default:
3412		abort();	/* Impossible */
3413	}
3414
3415	if (m->str_flags & PSTRING_LENGTH_INCLUDES_ITSELF)
3416		len -= file_pstring_length_size(m);
3417
3418	return len;
3419}
3420
3421protected int
3422file_magicfind(struct magic_set *ms, const char *name, struct mlist *v)
3423{
3424	uint32_t i, j;
3425	struct mlist *mlist, *ml;
3426
3427	mlist = ms->mlist[1];
3428
3429	for (ml = mlist->next; ml != mlist; ml = ml->next) {
3430		struct magic *ma = ml->magic;
3431		uint32_t nma = ml->nmagic;
3432		for (i = 0; i < nma; i++) {
3433			if (ma[i].type != FILE_NAME)
3434				continue;
3435			if (strcmp(ma[i].value.s, name) == 0) {
3436				v->magic = &ma[i];
3437				for (j = i + 1; j < nma; j++)
3438				    if (ma[j].cont_level == 0)
3439					    break;
3440				v->nmagic = j - i;
3441				return 0;
3442			}
3443		}
3444	}
3445	return -1;
3446}
3447