softmagic.c revision 34650d7da670a6653a5b0546b0656d1772779fd2
1/*
2 * Copyright (c) Ian F. Darwin 1986-1995.
3 * Software written by Ian F. Darwin and others;
4 * maintained 1995-present by Christos Zoulas and others.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice immediately at the beginning of the file, without modification,
11 *    this list of conditions, and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
20 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28/*
29 * softmagic - interpret variable magic from MAGIC
30 */
31
32#include "file.h"
33
34#ifndef	lint
35FILE_RCSID("@(#)$File: softmagic.c,v 1.218 2015/09/11 17:24:09 christos Exp $")
36#endif	/* lint */
37
38#include "magic.h"
39#include <assert.h>
40#include <string.h>
41#include <ctype.h>
42#include <stdlib.h>
43#include <time.h>
44
45private int match(struct magic_set *, struct magic *, uint32_t,
46    const unsigned char *, size_t, size_t, int, int, int, uint16_t,
47    uint16_t *, int *, int *, int *);
48private int mget(struct magic_set *, const unsigned char *,
49    struct magic *, size_t, size_t, unsigned int, int, int, int, uint16_t,
50    uint16_t *, int *, int *, int *);
51private int magiccheck(struct magic_set *, struct magic *);
52private int32_t mprint(struct magic_set *, struct magic *);
53private int32_t moffset(struct magic_set *, struct magic *);
54private void mdebug(uint32_t, const char *, size_t);
55private int mcopy(struct magic_set *, union VALUETYPE *, int, int,
56    const unsigned char *, uint32_t, size_t, struct magic *);
57private int mconvert(struct magic_set *, struct magic *, int);
58private int print_sep(struct magic_set *, int);
59private int handle_annotation(struct magic_set *, struct magic *);
60private void cvt_8(union VALUETYPE *, const struct magic *);
61private void cvt_16(union VALUETYPE *, const struct magic *);
62private void cvt_32(union VALUETYPE *, const struct magic *);
63private void cvt_64(union VALUETYPE *, const struct magic *);
64
65#define OFFSET_OOB(n, o, i)	((n) < (o) || (i) > ((n) - (o)))
66#define BE64(p) (((uint64_t)(p)->hq[0]<<56)|((uint64_t)(p)->hq[1]<<48)| \
67    ((uint64_t)(p)->hq[2]<<40)|((uint64_t)(p)->hq[3]<<32)| \
68    ((uint64_t)(p)->hq[4]<<24)|((uint64_t)(p)->hq[5]<<16)| \
69    ((uint64_t)(p)->hq[6]<<8)|((uint64_t)(p)->hq[7]))
70#define LE64(p) (((uint64_t)(p)->hq[7]<<56)|((uint64_t)(p)->hq[6]<<48)| \
71    ((uint64_t)(p)->hq[5]<<40)|((uint64_t)(p)->hq[4]<<32)| \
72    ((uint64_t)(p)->hq[3]<<24)|((uint64_t)(p)->hq[2]<<16)| \
73    ((uint64_t)(p)->hq[1]<<8)|((uint64_t)(p)->hq[0]))
74#define LE32(p) (((uint32_t)(p)->hl[3]<<24)|((uint32_t)(p)->hl[2]<<16)| \
75     ((uint32_t)(p)->hl[1]<<8)|((uint32_t)(p)->hl[0]))
76#define BE32(p) (((uint32_t)(p)->hl[0]<<24)|((uint32_t)(p)->hl[1]<<16)| \
77     ((uint32_t)(p)->hl[2]<<8)|((uint32_t)(p)->hl[3]))
78#define ME32(p) (((uint32_t)(p)->hl[1]<<24)|((uint32_t)(p)->hl[0]<<16)| \
79     ((uint32_t)(p)->hl[3]<<8)|((uint32_t)(p)->hl[2]))
80#define BE16(p) (((uint16_t)(p)->hs[0]<<8)|((uint16_t)(p)->hs[1]))
81#define LE16(p) (((uint16_t)(p)->hs[1]<<8)|((uint16_t)(p)->hs[0]))
82
83/*
84 * softmagic - lookup one file in parsed, in-memory copy of database
85 * Passed the name and FILE * of one file to be typed.
86 */
87/*ARGSUSED1*/		/* nbytes passed for regularity, maybe need later */
88protected int
89file_softmagic(struct magic_set *ms, const unsigned char *buf, size_t nbytes,
90    uint16_t indir_level, uint16_t *name_count, int mode, int text)
91{
92	struct mlist *ml;
93	int rv, printed_something = 0, need_separator = 0;
94	uint16_t nc;
95
96	if (name_count == NULL) {
97		nc = 0;
98		name_count = &nc;
99	}
100
101	for (ml = ms->mlist[0]->next; ml != ms->mlist[0]; ml = ml->next)
102		if ((rv = match(ms, ml->magic, ml->nmagic, buf, nbytes, 0, mode,
103		    text, 0, indir_level, name_count,
104		    &printed_something, &need_separator, NULL)) != 0)
105			return rv;
106
107	return 0;
108}
109
110#define FILE_FMTDEBUG
111#ifdef FILE_FMTDEBUG
112#define F(a, b, c) file_fmtcheck((a), (b), (c), __FILE__, __LINE__)
113
114private const char * __attribute__((__format_arg__(3)))
115file_fmtcheck(struct magic_set *ms, const struct magic *m, const char *def,
116	const char *file, size_t line)
117{
118	const char *ptr = fmtcheck(m->desc, def);
119	if (ptr == def)
120		file_magerror(ms,
121		    "%s, %" SIZE_T_FORMAT "u: format `%s' does not match"
122		    " with `%s'", file, line, m->desc, def);
123	return ptr;
124}
125#else
126#define F(a, b, c) fmtcheck((b)->desc, (c))
127#endif
128
129/*
130 * Go through the whole list, stopping if you find a match.  Process all
131 * the continuations of that match before returning.
132 *
133 * We support multi-level continuations:
134 *
135 *	At any time when processing a successful top-level match, there is a
136 *	current continuation level; it represents the level of the last
137 *	successfully matched continuation.
138 *
139 *	Continuations above that level are skipped as, if we see one, it
140 *	means that the continuation that controls them - i.e, the
141 *	lower-level continuation preceding them - failed to match.
142 *
143 *	Continuations below that level are processed as, if we see one,
144 *	it means we've finished processing or skipping higher-level
145 *	continuations under the control of a successful or unsuccessful
146 *	lower-level continuation, and are now seeing the next lower-level
147 *	continuation and should process it.  The current continuation
148 *	level reverts to the level of the one we're seeing.
149 *
150 *	Continuations at the current level are processed as, if we see
151 *	one, there's no lower-level continuation that may have failed.
152 *
153 *	If a continuation matches, we bump the current continuation level
154 *	so that higher-level continuations are processed.
155 */
156private int
157match(struct magic_set *ms, struct magic *magic, uint32_t nmagic,
158    const unsigned char *s, size_t nbytes, size_t offset, int mode, int text,
159    int flip, uint16_t indir_level, uint16_t *name_count,
160    int *printed_something, int *need_separator, int *returnval)
161{
162	uint32_t magindex = 0;
163	unsigned int cont_level = 0;
164	int returnvalv = 0, e; /* if a match is found it is set to 1*/
165	int firstline = 1; /* a flag to print X\n  X\n- X */
166	int print = (ms->flags & MAGIC_NODESC) == 0;
167
168	if (returnval == NULL)
169		returnval = &returnvalv;
170
171	if (file_check_mem(ms, cont_level) == -1)
172		return -1;
173
174	for (magindex = 0; magindex < nmagic; magindex++) {
175		int flush = 0;
176		struct magic *m = &magic[magindex];
177
178		if (m->type != FILE_NAME)
179		if ((IS_STRING(m->type) &&
180#define FLT (STRING_BINTEST | STRING_TEXTTEST)
181		     ((text && (m->str_flags & FLT) == STRING_BINTEST) ||
182		      (!text && (m->str_flags & FLT) == STRING_TEXTTEST))) ||
183		    (m->flag & mode) != mode) {
184			/* Skip sub-tests */
185			while (magindex + 1 < nmagic &&
186                               magic[magindex + 1].cont_level != 0 &&
187			       ++magindex)
188				continue;
189			continue; /* Skip to next top-level test*/
190		}
191
192		ms->offset = m->offset;
193		ms->line = m->lineno;
194
195		/* if main entry matches, print it... */
196		switch (mget(ms, s, m, nbytes, offset, cont_level, mode, text,
197		    flip, indir_level, name_count,
198		    printed_something, need_separator, returnval)) {
199		case -1:
200			return -1;
201		case 0:
202			flush = m->reln != '!';
203			break;
204		default:
205			if (m->type == FILE_INDIRECT)
206				*returnval = 1;
207
208			switch (magiccheck(ms, m)) {
209			case -1:
210				return -1;
211			case 0:
212				flush++;
213				break;
214			default:
215				flush = 0;
216				break;
217			}
218			break;
219		}
220		if (flush) {
221			/*
222			 * main entry didn't match,
223			 * flush its continuations
224			 */
225			while (magindex < nmagic - 1 &&
226			    magic[magindex + 1].cont_level != 0)
227				magindex++;
228			continue;
229		}
230
231		if ((e = handle_annotation(ms, m)) != 0) {
232			*need_separator = 1;
233			*printed_something = 1;
234			*returnval = 1;
235			return e;
236		}
237		/*
238		 * If we are going to print something, we'll need to print
239		 * a blank before we print something else.
240		 */
241		if (*m->desc) {
242			*need_separator = 1;
243			*printed_something = 1;
244			if (print_sep(ms, firstline) == -1)
245				return -1;
246		}
247
248
249		if (print && mprint(ms, m) == -1)
250			return -1;
251
252		ms->c.li[cont_level].off = moffset(ms, m);
253
254		/* and any continuations that match */
255		if (file_check_mem(ms, ++cont_level) == -1)
256			return -1;
257
258		while (magindex + 1 < nmagic &&
259		    magic[magindex + 1].cont_level != 0) {
260			m = &magic[++magindex];
261			ms->line = m->lineno; /* for messages */
262
263			if (cont_level < m->cont_level)
264				continue;
265			if (cont_level > m->cont_level) {
266				/*
267				 * We're at the end of the level
268				 * "cont_level" continuations.
269				 */
270				cont_level = m->cont_level;
271			}
272			ms->offset = m->offset;
273			if (m->flag & OFFADD) {
274				ms->offset +=
275				    ms->c.li[cont_level - 1].off;
276			}
277
278#ifdef ENABLE_CONDITIONALS
279			if (m->cond == COND_ELSE ||
280			    m->cond == COND_ELIF) {
281				if (ms->c.li[cont_level].last_match == 1)
282					continue;
283			}
284#endif
285			switch (mget(ms, s, m, nbytes, offset, cont_level, mode,
286			    text, flip, indir_level, name_count,
287			    printed_something, need_separator, returnval)) {
288			case -1:
289				return -1;
290			case 0:
291				if (m->reln != '!')
292					continue;
293				flush = 1;
294				break;
295			default:
296				if (m->type == FILE_INDIRECT)
297					*returnval = 1;
298				flush = 0;
299				break;
300			}
301
302			switch (flush ? 1 : magiccheck(ms, m)) {
303			case -1:
304				return -1;
305			case 0:
306#ifdef ENABLE_CONDITIONALS
307				ms->c.li[cont_level].last_match = 0;
308#endif
309				break;
310			default:
311#ifdef ENABLE_CONDITIONALS
312				ms->c.li[cont_level].last_match = 1;
313#endif
314				if (m->type == FILE_CLEAR)
315					ms->c.li[cont_level].got_match = 0;
316				else if (ms->c.li[cont_level].got_match) {
317					if (m->type == FILE_DEFAULT)
318						break;
319				} else
320					ms->c.li[cont_level].got_match = 1;
321				if ((e = handle_annotation(ms, m)) != 0) {
322					*need_separator = 1;
323					*printed_something = 1;
324					*returnval = 1;
325					return e;
326				}
327				/*
328				 * If we are going to print something,
329				 * make sure that we have a separator first.
330				 */
331				if (*m->desc) {
332					if (!*printed_something) {
333						*printed_something = 1;
334						if (print_sep(ms, firstline)
335						    == -1)
336							return -1;
337					}
338				}
339				/*
340				 * This continuation matched.  Print
341				 * its message, with a blank before it
342				 * if the previous item printed and
343				 * this item isn't empty.
344				 */
345				/* space if previous printed */
346				if (*need_separator
347				    && ((m->flag & NOSPACE) == 0)
348				    && *m->desc) {
349					if (print &&
350					    file_printf(ms, " ") == -1)
351						return -1;
352					*need_separator = 0;
353				}
354				if (print && mprint(ms, m) == -1)
355					return -1;
356
357				ms->c.li[cont_level].off = moffset(ms, m);
358
359				if (*m->desc)
360					*need_separator = 1;
361
362				/*
363				 * If we see any continuations
364				 * at a higher level,
365				 * process them.
366				 */
367				if (file_check_mem(ms, ++cont_level) == -1)
368					return -1;
369				break;
370			}
371		}
372		if (*printed_something) {
373			firstline = 0;
374			if (print)
375				*returnval = 1;
376		}
377		if ((ms->flags & MAGIC_CONTINUE) == 0 && *printed_something) {
378			return *returnval; /* don't keep searching */
379		}
380		cont_level = 0;
381	}
382	return *returnval;  /* This is hit if -k is set or there is no match */
383}
384
385private int
386check_fmt(struct magic_set *ms, struct magic *m)
387{
388	file_regex_t rx;
389	int rc, rv = -1;
390
391	if (strchr(m->desc, '%') == NULL)
392		return 0;
393
394	rc = file_regcomp(&rx, "%[-0-9\\.]*s", REG_EXTENDED|REG_NOSUB);
395	if (rc) {
396		file_regerror(&rx, rc, ms);
397	} else {
398		rc = file_regexec(&rx, m->desc, 0, 0, 0);
399		rv = !rc;
400	}
401	file_regfree(&rx);
402	return rv;
403}
404
405#ifndef HAVE_STRNDUP
406char * strndup(const char *, size_t);
407
408char *
409strndup(const char *str, size_t n)
410{
411	size_t len;
412	char *copy;
413
414	for (len = 0; len < n && str[len]; len++)
415		continue;
416	if ((copy = malloc(len + 1)) == NULL)
417		return NULL;
418	(void)memcpy(copy, str, len);
419	copy[len] = '\0';
420	return copy;
421}
422#endif /* HAVE_STRNDUP */
423
424private int32_t
425mprint(struct magic_set *ms, struct magic *m)
426{
427	uint64_t v;
428	float vf;
429	double vd;
430	int64_t t = 0;
431 	char buf[128], tbuf[26], sbuf[512];
432	union VALUETYPE *p = &ms->ms_value;
433
434  	switch (m->type) {
435  	case FILE_BYTE:
436		v = file_signextend(ms, m, (uint64_t)p->b);
437		switch (check_fmt(ms, m)) {
438		case -1:
439			return -1;
440		case 1:
441			(void)snprintf(buf, sizeof(buf), "%d",
442			    (unsigned char)v);
443			if (file_printf(ms, F(ms, m, "%s"), buf) == -1)
444				return -1;
445			break;
446		default:
447			if (file_printf(ms, F(ms, m, "%d"),
448			    (unsigned char) v) == -1)
449				return -1;
450			break;
451		}
452		t = ms->offset + sizeof(char);
453		break;
454
455  	case FILE_SHORT:
456  	case FILE_BESHORT:
457  	case FILE_LESHORT:
458		v = file_signextend(ms, m, (uint64_t)p->h);
459		switch (check_fmt(ms, m)) {
460		case -1:
461			return -1;
462		case 1:
463			(void)snprintf(buf, sizeof(buf), "%u",
464			    (unsigned short)v);
465			if (file_printf(ms, F(ms, m, "%s"), buf) == -1)
466				return -1;
467			break;
468		default:
469			if (file_printf(ms, F(ms, m, "%u"),
470			    (unsigned short) v) == -1)
471				return -1;
472			break;
473		}
474		t = ms->offset + sizeof(short);
475		break;
476
477  	case FILE_LONG:
478  	case FILE_BELONG:
479  	case FILE_LELONG:
480  	case FILE_MELONG:
481		v = file_signextend(ms, m, (uint64_t)p->l);
482		switch (check_fmt(ms, m)) {
483		case -1:
484			return -1;
485		case 1:
486			(void)snprintf(buf, sizeof(buf), "%u", (uint32_t) v);
487			if (file_printf(ms, F(ms, m, "%s"), buf) == -1)
488				return -1;
489			break;
490		default:
491			if (file_printf(ms, F(ms, m, "%u"), (uint32_t) v) == -1)
492				return -1;
493			break;
494		}
495		t = ms->offset + sizeof(int32_t);
496  		break;
497
498  	case FILE_QUAD:
499  	case FILE_BEQUAD:
500  	case FILE_LEQUAD:
501		v = file_signextend(ms, m, p->q);
502		switch (check_fmt(ms, m)) {
503		case -1:
504			return -1;
505		case 1:
506			(void)snprintf(buf, sizeof(buf), "%" INT64_T_FORMAT "u",
507			    (unsigned long long)v);
508			if (file_printf(ms, F(ms, m, "%s"), buf) == -1)
509				return -1;
510			break;
511		default:
512			if (file_printf(ms, F(ms, m, "%" INT64_T_FORMAT "u"),
513			    (unsigned long long) v) == -1)
514				return -1;
515			break;
516		}
517		t = ms->offset + sizeof(int64_t);
518  		break;
519
520  	case FILE_STRING:
521  	case FILE_PSTRING:
522  	case FILE_BESTRING16:
523  	case FILE_LESTRING16:
524		if (m->reln == '=' || m->reln == '!') {
525			if (file_printf(ms, F(ms, m, "%s"),
526			    file_printable(sbuf, sizeof(sbuf), m->value.s))
527			    == -1)
528				return -1;
529			t = ms->offset + m->vallen;
530		}
531		else {
532			char *str = p->s;
533
534			/* compute t before we mangle the string? */
535			t = ms->offset + strlen(str);
536
537			if (*m->value.s == '\0')
538				str[strcspn(str, "\r\n")] = '\0';
539
540			if (m->str_flags & STRING_TRIM) {
541				char *last;
542				while (isspace((unsigned char)*str))
543					str++;
544				last = str;
545				while (*last)
546					last++;
547				--last;
548				while (isspace((unsigned char)*last))
549					last--;
550				*++last = '\0';
551			}
552
553			if (file_printf(ms, F(ms, m, "%s"),
554			    file_printable(sbuf, sizeof(sbuf), str)) == -1)
555				return -1;
556
557			if (m->type == FILE_PSTRING)
558				t += file_pstring_length_size(m);
559		}
560		break;
561
562	case FILE_DATE:
563	case FILE_BEDATE:
564	case FILE_LEDATE:
565	case FILE_MEDATE:
566		if (file_printf(ms, F(ms, m, "%s"),
567		    file_fmttime(p->l, 0, tbuf)) == -1)
568			return -1;
569		t = ms->offset + sizeof(uint32_t);
570		break;
571
572	case FILE_LDATE:
573	case FILE_BELDATE:
574	case FILE_LELDATE:
575	case FILE_MELDATE:
576		if (file_printf(ms, F(ms, m, "%s"),
577		    file_fmttime(p->l, FILE_T_LOCAL, tbuf)) == -1)
578			return -1;
579		t = ms->offset + sizeof(uint32_t);
580		break;
581
582	case FILE_QDATE:
583	case FILE_BEQDATE:
584	case FILE_LEQDATE:
585		if (file_printf(ms, F(ms, m, "%s"),
586		    file_fmttime(p->q, 0, tbuf)) == -1)
587			return -1;
588		t = ms->offset + sizeof(uint64_t);
589		break;
590
591	case FILE_QLDATE:
592	case FILE_BEQLDATE:
593	case FILE_LEQLDATE:
594		if (file_printf(ms, F(ms, m, "%s"),
595		    file_fmttime(p->q, FILE_T_LOCAL, tbuf)) == -1)
596			return -1;
597		t = ms->offset + sizeof(uint64_t);
598		break;
599
600	case FILE_QWDATE:
601	case FILE_BEQWDATE:
602	case FILE_LEQWDATE:
603		if (file_printf(ms, F(ms, m, "%s"),
604		    file_fmttime(p->q, FILE_T_WINDOWS, tbuf)) == -1)
605			return -1;
606		t = ms->offset + sizeof(uint64_t);
607		break;
608
609	case FILE_FLOAT:
610	case FILE_BEFLOAT:
611	case FILE_LEFLOAT:
612		vf = p->f;
613		switch (check_fmt(ms, m)) {
614		case -1:
615			return -1;
616		case 1:
617			(void)snprintf(buf, sizeof(buf), "%g", vf);
618			if (file_printf(ms, F(ms, m, "%s"), buf) == -1)
619				return -1;
620			break;
621		default:
622			if (file_printf(ms, F(ms, m, "%g"), vf) == -1)
623				return -1;
624			break;
625		}
626		t = ms->offset + sizeof(float);
627  		break;
628
629	case FILE_DOUBLE:
630	case FILE_BEDOUBLE:
631	case FILE_LEDOUBLE:
632		vd = p->d;
633		switch (check_fmt(ms, m)) {
634		case -1:
635			return -1;
636		case 1:
637			(void)snprintf(buf, sizeof(buf), "%g", vd);
638			if (file_printf(ms, F(ms, m, "%s"), buf) == -1)
639				return -1;
640			break;
641		default:
642			if (file_printf(ms, F(ms, m, "%g"), vd) == -1)
643				return -1;
644			break;
645		}
646		t = ms->offset + sizeof(double);
647  		break;
648
649	case FILE_SEARCH:
650	case FILE_REGEX: {
651		char *cp;
652		int rval;
653
654		cp = strndup((const char *)ms->search.s, ms->search.rm_len);
655		if (cp == NULL) {
656			file_oomem(ms, ms->search.rm_len);
657			return -1;
658		}
659		rval = file_printf(ms, F(ms, m, "%s"),
660		    file_printable(sbuf, sizeof(sbuf), cp));
661		free(cp);
662
663		if (rval == -1)
664			return -1;
665
666		if ((m->str_flags & REGEX_OFFSET_START))
667			t = ms->search.offset;
668		else
669			t = ms->search.offset + ms->search.rm_len;
670		break;
671	}
672
673	case FILE_DEFAULT:
674	case FILE_CLEAR:
675	  	if (file_printf(ms, "%s", m->desc) == -1)
676			return -1;
677		t = ms->offset;
678		break;
679
680	case FILE_INDIRECT:
681	case FILE_USE:
682	case FILE_NAME:
683		t = ms->offset;
684		break;
685
686	default:
687		file_magerror(ms, "invalid m->type (%d) in mprint()", m->type);
688		return -1;
689	}
690	return (int32_t)t;
691}
692
693private int32_t
694moffset(struct magic_set *ms, struct magic *m)
695{
696  	switch (m->type) {
697  	case FILE_BYTE:
698		return CAST(int32_t, (ms->offset + sizeof(char)));
699
700  	case FILE_SHORT:
701  	case FILE_BESHORT:
702  	case FILE_LESHORT:
703		return CAST(int32_t, (ms->offset + sizeof(short)));
704
705  	case FILE_LONG:
706  	case FILE_BELONG:
707  	case FILE_LELONG:
708  	case FILE_MELONG:
709		return CAST(int32_t, (ms->offset + sizeof(int32_t)));
710
711  	case FILE_QUAD:
712  	case FILE_BEQUAD:
713  	case FILE_LEQUAD:
714		return CAST(int32_t, (ms->offset + sizeof(int64_t)));
715
716  	case FILE_STRING:
717  	case FILE_PSTRING:
718  	case FILE_BESTRING16:
719  	case FILE_LESTRING16:
720		if (m->reln == '=' || m->reln == '!')
721			return ms->offset + m->vallen;
722		else {
723			union VALUETYPE *p = &ms->ms_value;
724			uint32_t t;
725
726			if (*m->value.s == '\0')
727				p->s[strcspn(p->s, "\r\n")] = '\0';
728			t = CAST(uint32_t, (ms->offset + strlen(p->s)));
729			if (m->type == FILE_PSTRING)
730				t += (uint32_t)file_pstring_length_size(m);
731			return t;
732		}
733
734	case FILE_DATE:
735	case FILE_BEDATE:
736	case FILE_LEDATE:
737	case FILE_MEDATE:
738		return CAST(int32_t, (ms->offset + sizeof(uint32_t)));
739
740	case FILE_LDATE:
741	case FILE_BELDATE:
742	case FILE_LELDATE:
743	case FILE_MELDATE:
744		return CAST(int32_t, (ms->offset + sizeof(uint32_t)));
745
746	case FILE_QDATE:
747	case FILE_BEQDATE:
748	case FILE_LEQDATE:
749		return CAST(int32_t, (ms->offset + sizeof(uint64_t)));
750
751	case FILE_QLDATE:
752	case FILE_BEQLDATE:
753	case FILE_LEQLDATE:
754		return CAST(int32_t, (ms->offset + sizeof(uint64_t)));
755
756  	case FILE_FLOAT:
757  	case FILE_BEFLOAT:
758  	case FILE_LEFLOAT:
759		return CAST(int32_t, (ms->offset + sizeof(float)));
760
761  	case FILE_DOUBLE:
762  	case FILE_BEDOUBLE:
763  	case FILE_LEDOUBLE:
764		return CAST(int32_t, (ms->offset + sizeof(double)));
765
766	case FILE_REGEX:
767		if ((m->str_flags & REGEX_OFFSET_START) != 0)
768			return CAST(int32_t, ms->search.offset);
769		else
770			return CAST(int32_t, (ms->search.offset +
771			    ms->search.rm_len));
772
773	case FILE_SEARCH:
774		if ((m->str_flags & REGEX_OFFSET_START) != 0)
775			return CAST(int32_t, ms->search.offset);
776		else
777			return CAST(int32_t, (ms->search.offset + m->vallen));
778
779	case FILE_CLEAR:
780	case FILE_DEFAULT:
781	case FILE_INDIRECT:
782		return ms->offset;
783
784	default:
785		return 0;
786	}
787}
788
789private int
790cvt_flip(int type, int flip)
791{
792	if (flip == 0)
793		return type;
794	switch (type) {
795	case FILE_BESHORT:
796		return FILE_LESHORT;
797	case FILE_BELONG:
798		return FILE_LELONG;
799	case FILE_BEDATE:
800		return FILE_LEDATE;
801	case FILE_BELDATE:
802		return FILE_LELDATE;
803	case FILE_BEQUAD:
804		return FILE_LEQUAD;
805	case FILE_BEQDATE:
806		return FILE_LEQDATE;
807	case FILE_BEQLDATE:
808		return FILE_LEQLDATE;
809	case FILE_BEQWDATE:
810		return FILE_LEQWDATE;
811	case FILE_LESHORT:
812		return FILE_BESHORT;
813	case FILE_LELONG:
814		return FILE_BELONG;
815	case FILE_LEDATE:
816		return FILE_BEDATE;
817	case FILE_LELDATE:
818		return FILE_BELDATE;
819	case FILE_LEQUAD:
820		return FILE_BEQUAD;
821	case FILE_LEQDATE:
822		return FILE_BEQDATE;
823	case FILE_LEQLDATE:
824		return FILE_BEQLDATE;
825	case FILE_LEQWDATE:
826		return FILE_BEQWDATE;
827	case FILE_BEFLOAT:
828		return FILE_LEFLOAT;
829	case FILE_LEFLOAT:
830		return FILE_BEFLOAT;
831	case FILE_BEDOUBLE:
832		return FILE_LEDOUBLE;
833	case FILE_LEDOUBLE:
834		return FILE_BEDOUBLE;
835	default:
836		return type;
837	}
838}
839#define DO_CVT(fld, cast) \
840	if (m->num_mask) \
841		switch (m->mask_op & FILE_OPS_MASK) { \
842		case FILE_OPAND: \
843			p->fld &= cast m->num_mask; \
844			break; \
845		case FILE_OPOR: \
846			p->fld |= cast m->num_mask; \
847			break; \
848		case FILE_OPXOR: \
849			p->fld ^= cast m->num_mask; \
850			break; \
851		case FILE_OPADD: \
852			p->fld += cast m->num_mask; \
853			break; \
854		case FILE_OPMINUS: \
855			p->fld -= cast m->num_mask; \
856			break; \
857		case FILE_OPMULTIPLY: \
858			p->fld *= cast m->num_mask; \
859			break; \
860		case FILE_OPDIVIDE: \
861			p->fld /= cast m->num_mask; \
862			break; \
863		case FILE_OPMODULO: \
864			p->fld %= cast m->num_mask; \
865			break; \
866		} \
867	if (m->mask_op & FILE_OPINVERSE) \
868		p->fld = ~p->fld \
869
870private void
871cvt_8(union VALUETYPE *p, const struct magic *m)
872{
873	DO_CVT(b, (uint8_t));
874}
875
876private void
877cvt_16(union VALUETYPE *p, const struct magic *m)
878{
879	DO_CVT(h, (uint16_t));
880}
881
882private void
883cvt_32(union VALUETYPE *p, const struct magic *m)
884{
885	DO_CVT(l, (uint32_t));
886}
887
888private void
889cvt_64(union VALUETYPE *p, const struct magic *m)
890{
891	DO_CVT(q, (uint64_t));
892}
893
894#define DO_CVT2(fld, cast) \
895	if (m->num_mask) \
896		switch (m->mask_op & FILE_OPS_MASK) { \
897		case FILE_OPADD: \
898			p->fld += cast m->num_mask; \
899			break; \
900		case FILE_OPMINUS: \
901			p->fld -= cast m->num_mask; \
902			break; \
903		case FILE_OPMULTIPLY: \
904			p->fld *= cast m->num_mask; \
905			break; \
906		case FILE_OPDIVIDE: \
907			p->fld /= cast m->num_mask; \
908			break; \
909		} \
910
911private void
912cvt_float(union VALUETYPE *p, const struct magic *m)
913{
914	DO_CVT2(f, (float));
915}
916
917private void
918cvt_double(union VALUETYPE *p, const struct magic *m)
919{
920	DO_CVT2(d, (double));
921}
922
923/*
924 * Convert the byte order of the data we are looking at
925 * While we're here, let's apply the mask operation
926 * (unless you have a better idea)
927 */
928private int
929mconvert(struct magic_set *ms, struct magic *m, int flip)
930{
931	union VALUETYPE *p = &ms->ms_value;
932	uint8_t type;
933
934	switch (type = cvt_flip(m->type, flip)) {
935	case FILE_BYTE:
936		cvt_8(p, m);
937		return 1;
938	case FILE_SHORT:
939		cvt_16(p, m);
940		return 1;
941	case FILE_LONG:
942	case FILE_DATE:
943	case FILE_LDATE:
944		cvt_32(p, m);
945		return 1;
946	case FILE_QUAD:
947	case FILE_QDATE:
948	case FILE_QLDATE:
949	case FILE_QWDATE:
950		cvt_64(p, m);
951		return 1;
952	case FILE_STRING:
953	case FILE_BESTRING16:
954	case FILE_LESTRING16: {
955		/* Null terminate and eat *trailing* return */
956		p->s[sizeof(p->s) - 1] = '\0';
957		return 1;
958	}
959	case FILE_PSTRING: {
960		size_t sz = file_pstring_length_size(m);
961		char *ptr1 = p->s, *ptr2 = ptr1 + sz;
962		size_t len = file_pstring_get_length(m, ptr1);
963		sz = sizeof(p->s) - sz; /* maximum length of string */
964		if (len >= sz) {
965			/*
966			 * The size of the pascal string length (sz)
967			 * is 1, 2, or 4. We need at least 1 byte for NUL
968			 * termination, but we've already truncated the
969			 * string by p->s, so we need to deduct sz.
970			 * Because we can use one of the bytes of the length
971			 * after we shifted as NUL termination.
972			 */
973			len = sz;
974		}
975		while (len--)
976			*ptr1++ = *ptr2++;
977		*ptr1 = '\0';
978		return 1;
979	}
980	case FILE_BESHORT:
981		p->h = (short)BE16(p);
982		cvt_16(p, m);
983		return 1;
984	case FILE_BELONG:
985	case FILE_BEDATE:
986	case FILE_BELDATE:
987		p->l = (int32_t)BE32(p);
988		cvt_32(p, m);
989		return 1;
990	case FILE_BEQUAD:
991	case FILE_BEQDATE:
992	case FILE_BEQLDATE:
993	case FILE_BEQWDATE:
994		p->q = (uint64_t)BE64(p);
995		cvt_64(p, m);
996		return 1;
997	case FILE_LESHORT:
998		p->h = (short)LE16(p);
999		cvt_16(p, m);
1000		return 1;
1001	case FILE_LELONG:
1002	case FILE_LEDATE:
1003	case FILE_LELDATE:
1004		p->l = (int32_t)LE32(p);
1005		cvt_32(p, m);
1006		return 1;
1007	case FILE_LEQUAD:
1008	case FILE_LEQDATE:
1009	case FILE_LEQLDATE:
1010	case FILE_LEQWDATE:
1011		p->q = (uint64_t)LE64(p);
1012		cvt_64(p, m);
1013		return 1;
1014	case FILE_MELONG:
1015	case FILE_MEDATE:
1016	case FILE_MELDATE:
1017		p->l = (int32_t)ME32(p);
1018		cvt_32(p, m);
1019		return 1;
1020	case FILE_FLOAT:
1021		cvt_float(p, m);
1022		return 1;
1023	case FILE_BEFLOAT:
1024		p->l = BE32(p);
1025		cvt_float(p, m);
1026		return 1;
1027	case FILE_LEFLOAT:
1028		p->l = LE32(p);
1029		cvt_float(p, m);
1030		return 1;
1031	case FILE_DOUBLE:
1032		cvt_double(p, m);
1033		return 1;
1034	case FILE_BEDOUBLE:
1035		p->q = BE64(p);
1036		cvt_double(p, m);
1037		return 1;
1038	case FILE_LEDOUBLE:
1039		p->q = LE64(p);
1040		cvt_double(p, m);
1041		return 1;
1042	case FILE_REGEX:
1043	case FILE_SEARCH:
1044	case FILE_DEFAULT:
1045	case FILE_CLEAR:
1046	case FILE_NAME:
1047	case FILE_USE:
1048		return 1;
1049	default:
1050		file_magerror(ms, "invalid type %d in mconvert()", m->type);
1051		return 0;
1052	}
1053}
1054
1055
1056private void
1057mdebug(uint32_t offset, const char *str, size_t len)
1058{
1059	(void) fprintf(stderr, "mget/%" SIZE_T_FORMAT "u @%d: ", len, offset);
1060	file_showstr(stderr, str, len);
1061	(void) fputc('\n', stderr);
1062	(void) fputc('\n', stderr);
1063}
1064
1065private int
1066mcopy(struct magic_set *ms, union VALUETYPE *p, int type, int indir,
1067    const unsigned char *s, uint32_t offset, size_t nbytes, struct magic *m)
1068{
1069	/*
1070	 * Note: FILE_SEARCH and FILE_REGEX do not actually copy
1071	 * anything, but setup pointers into the source
1072	 */
1073	if (indir == 0) {
1074		switch (type) {
1075		case FILE_SEARCH:
1076			ms->search.s = RCAST(const char *, s) + offset;
1077			ms->search.s_len = nbytes - offset;
1078			ms->search.offset = offset;
1079			return 0;
1080
1081		case FILE_REGEX: {
1082			const char *b;
1083			const char *c;
1084			const char *last;	/* end of search region */
1085			const char *buf;	/* start of search region */
1086			const char *end;
1087			size_t lines, linecnt, bytecnt;
1088
1089			if (s == NULL) {
1090				ms->search.s_len = 0;
1091				ms->search.s = NULL;
1092				return 0;
1093			}
1094
1095			if (m->str_flags & REGEX_LINE_COUNT) {
1096				linecnt = m->str_range;
1097				bytecnt = linecnt * 80;
1098			} else {
1099				linecnt = 0;
1100				bytecnt = m->str_range;
1101			}
1102
1103			if (bytecnt == 0 || bytecnt > nbytes - offset)
1104				bytecnt = nbytes - offset;
1105			if (bytecnt > ms->regex_max)
1106				bytecnt = ms->regex_max;
1107
1108			buf = RCAST(const char *, s) + offset;
1109			end = last = RCAST(const char *, s) + bytecnt + offset;
1110			/* mget() guarantees buf <= last */
1111			for (lines = linecnt, b = buf; lines && b < end &&
1112			     ((b = CAST(const char *,
1113				 memchr(c = b, '\n', CAST(size_t, (end - b)))))
1114			     || (b = CAST(const char *,
1115				 memchr(c, '\r', CAST(size_t, (end - c))))));
1116			     lines--, b++) {
1117				last = b;
1118				if (b[0] == '\r' && b[1] == '\n')
1119					b++;
1120			}
1121			if (lines)
1122				last = RCAST(const char *, s) + bytecnt;
1123
1124			ms->search.s = buf;
1125			ms->search.s_len = last - buf;
1126			ms->search.offset = offset;
1127			ms->search.rm_len = 0;
1128			return 0;
1129		}
1130		case FILE_BESTRING16:
1131		case FILE_LESTRING16: {
1132			const unsigned char *src = s + offset;
1133			const unsigned char *esrc = s + nbytes;
1134			char *dst = p->s;
1135			char *edst = &p->s[sizeof(p->s) - 1];
1136
1137			if (type == FILE_BESTRING16)
1138				src++;
1139
1140			/* check that offset is within range */
1141			if (offset >= nbytes)
1142				break;
1143			for (/*EMPTY*/; src < esrc; src += 2, dst++) {
1144				if (dst < edst)
1145					*dst = *src;
1146				else
1147					break;
1148				if (*dst == '\0') {
1149					if (type == FILE_BESTRING16 ?
1150					    *(src - 1) != '\0' :
1151					    *(src + 1) != '\0')
1152						*dst = ' ';
1153				}
1154			}
1155			*edst = '\0';
1156			return 0;
1157		}
1158		case FILE_STRING:	/* XXX - these two should not need */
1159		case FILE_PSTRING:	/* to copy anything, but do anyway. */
1160		default:
1161			break;
1162		}
1163	}
1164
1165	if (offset >= nbytes) {
1166		(void)memset(p, '\0', sizeof(*p));
1167		return 0;
1168	}
1169	if (nbytes - offset < sizeof(*p))
1170		nbytes = nbytes - offset;
1171	else
1172		nbytes = sizeof(*p);
1173
1174	(void)memcpy(p, s + offset, nbytes);
1175
1176	/*
1177	 * the usefulness of padding with zeroes eludes me, it
1178	 * might even cause problems
1179	 */
1180	if (nbytes < sizeof(*p))
1181		(void)memset(((char *)(void *)p) + nbytes, '\0',
1182		    sizeof(*p) - nbytes);
1183	return 0;
1184}
1185
1186private int
1187mget(struct magic_set *ms, const unsigned char *s, struct magic *m,
1188    size_t nbytes, size_t o, unsigned int cont_level, int mode, int text,
1189    int flip, uint16_t indir_level, uint16_t *name_count,
1190    int *printed_something, int *need_separator, int *returnval)
1191{
1192	uint32_t offset = ms->offset;
1193	uint32_t lhs;
1194	file_pushbuf_t *pb;
1195	int rv, oneed_separator, in_type;
1196	char *rbuf;
1197	union VALUETYPE *p = &ms->ms_value;
1198	struct mlist ml;
1199
1200	if (indir_level >= ms->indir_max) {
1201		file_error(ms, 0, "indirect recursion nesting (%hu) exceeded",
1202		    indir_level);
1203		return -1;
1204	}
1205
1206	if (*name_count >= ms->name_max) {
1207		file_error(ms, 0, "name use count (%hu) exceeded",
1208		    *name_count);
1209		return -1;
1210	}
1211
1212	if (mcopy(ms, p, m->type, m->flag & INDIR, s, (uint32_t)(offset + o),
1213	    (uint32_t)nbytes, m) == -1)
1214		return -1;
1215
1216	if ((ms->flags & MAGIC_DEBUG) != 0) {
1217		fprintf(stderr, "mget(type=%d, flag=%x, offset=%u, o=%"
1218		    SIZE_T_FORMAT "u, " "nbytes=%" SIZE_T_FORMAT
1219		    "u, il=%hu, nc=%hu)\n",
1220		    m->type, m->flag, offset, o, nbytes,
1221		    indir_level, *name_count);
1222		mdebug(offset, (char *)(void *)p, sizeof(union VALUETYPE));
1223#ifndef COMPILE_ONLY
1224		file_mdump(m);
1225#endif
1226	}
1227
1228	if (m->flag & INDIR) {
1229		int off = m->in_offset;
1230		if (m->in_op & FILE_OPINDIRECT) {
1231			const union VALUETYPE *q = CAST(const union VALUETYPE *,
1232			    ((const void *)(s + offset + off)));
1233			switch (cvt_flip(m->in_type, flip)) {
1234			case FILE_BYTE:
1235				off = q->b;
1236				break;
1237			case FILE_SHORT:
1238				off = q->h;
1239				break;
1240			case FILE_BESHORT:
1241				off = (short)BE16(q);
1242				break;
1243			case FILE_LESHORT:
1244				off = (short)LE16(q);
1245				break;
1246			case FILE_LONG:
1247				off = q->l;
1248				break;
1249			case FILE_BELONG:
1250			case FILE_BEID3:
1251				off = (int32_t)BE32(q);
1252				break;
1253			case FILE_LEID3:
1254			case FILE_LELONG:
1255				off = (int32_t)LE32(q);
1256				break;
1257			case FILE_MELONG:
1258				off = (int32_t)ME32(q);
1259				break;
1260			}
1261			if ((ms->flags & MAGIC_DEBUG) != 0)
1262				fprintf(stderr, "indirect offs=%u\n", off);
1263		}
1264		switch (in_type = cvt_flip(m->in_type, flip)) {
1265		case FILE_BYTE:
1266			if (OFFSET_OOB(nbytes, offset, 1))
1267				return 0;
1268			if (off) {
1269				switch (m->in_op & FILE_OPS_MASK) {
1270				case FILE_OPAND:
1271					offset = p->b & off;
1272					break;
1273				case FILE_OPOR:
1274					offset = p->b | off;
1275					break;
1276				case FILE_OPXOR:
1277					offset = p->b ^ off;
1278					break;
1279				case FILE_OPADD:
1280					offset = p->b + off;
1281					break;
1282				case FILE_OPMINUS:
1283					offset = p->b - off;
1284					break;
1285				case FILE_OPMULTIPLY:
1286					offset = p->b * off;
1287					break;
1288				case FILE_OPDIVIDE:
1289					offset = p->b / off;
1290					break;
1291				case FILE_OPMODULO:
1292					offset = p->b % off;
1293					break;
1294				}
1295			} else
1296				offset = p->b;
1297			if (m->in_op & FILE_OPINVERSE)
1298				offset = ~offset;
1299			break;
1300		case FILE_BESHORT:
1301			if (OFFSET_OOB(nbytes, offset, 2))
1302				return 0;
1303			lhs = (p->hs[0] << 8) | p->hs[1];
1304			if (off) {
1305				switch (m->in_op & FILE_OPS_MASK) {
1306				case FILE_OPAND:
1307					offset = lhs & off;
1308					break;
1309				case FILE_OPOR:
1310					offset = lhs | off;
1311					break;
1312				case FILE_OPXOR:
1313					offset = lhs ^ off;
1314					break;
1315				case FILE_OPADD:
1316					offset = lhs + off;
1317					break;
1318				case FILE_OPMINUS:
1319					offset = lhs - off;
1320					break;
1321				case FILE_OPMULTIPLY:
1322					offset = lhs * off;
1323					break;
1324				case FILE_OPDIVIDE:
1325					offset = lhs / off;
1326					break;
1327				case FILE_OPMODULO:
1328					offset = lhs % off;
1329					break;
1330				}
1331			} else
1332				offset = lhs;
1333			if (m->in_op & FILE_OPINVERSE)
1334				offset = ~offset;
1335			break;
1336		case FILE_LESHORT:
1337			if (OFFSET_OOB(nbytes, offset, 2))
1338				return 0;
1339			lhs = (p->hs[1] << 8) | p->hs[0];
1340			if (off) {
1341				switch (m->in_op & FILE_OPS_MASK) {
1342				case FILE_OPAND:
1343					offset = lhs & off;
1344					break;
1345				case FILE_OPOR:
1346					offset = lhs | off;
1347					break;
1348				case FILE_OPXOR:
1349					offset = lhs ^ off;
1350					break;
1351				case FILE_OPADD:
1352					offset = lhs + off;
1353					break;
1354				case FILE_OPMINUS:
1355					offset = lhs - off;
1356					break;
1357				case FILE_OPMULTIPLY:
1358					offset = lhs * off;
1359					break;
1360				case FILE_OPDIVIDE:
1361					offset = lhs / off;
1362					break;
1363				case FILE_OPMODULO:
1364					offset = lhs % off;
1365					break;
1366				}
1367			} else
1368				offset = lhs;
1369			if (m->in_op & FILE_OPINVERSE)
1370				offset = ~offset;
1371			break;
1372		case FILE_SHORT:
1373			if (OFFSET_OOB(nbytes, offset, 2))
1374				return 0;
1375			if (off) {
1376				switch (m->in_op & FILE_OPS_MASK) {
1377				case FILE_OPAND:
1378					offset = p->h & off;
1379					break;
1380				case FILE_OPOR:
1381					offset = p->h | off;
1382					break;
1383				case FILE_OPXOR:
1384					offset = p->h ^ off;
1385					break;
1386				case FILE_OPADD:
1387					offset = p->h + off;
1388					break;
1389				case FILE_OPMINUS:
1390					offset = p->h - off;
1391					break;
1392				case FILE_OPMULTIPLY:
1393					offset = p->h * off;
1394					break;
1395				case FILE_OPDIVIDE:
1396					offset = p->h / off;
1397					break;
1398				case FILE_OPMODULO:
1399					offset = p->h % off;
1400					break;
1401				}
1402			}
1403			else
1404				offset = p->h;
1405			if (m->in_op & FILE_OPINVERSE)
1406				offset = ~offset;
1407			break;
1408		case FILE_BELONG:
1409		case FILE_BEID3:
1410			if (OFFSET_OOB(nbytes, offset, 4))
1411				return 0;
1412			lhs = BE32(p);
1413			if (off) {
1414				switch (m->in_op & FILE_OPS_MASK) {
1415				case FILE_OPAND:
1416					offset = lhs & off;
1417					break;
1418				case FILE_OPOR:
1419					offset = lhs | off;
1420					break;
1421				case FILE_OPXOR:
1422					offset = lhs ^ off;
1423					break;
1424				case FILE_OPADD:
1425					offset = lhs + off;
1426					break;
1427				case FILE_OPMINUS:
1428					offset = lhs - off;
1429					break;
1430				case FILE_OPMULTIPLY:
1431					offset = lhs * off;
1432					break;
1433				case FILE_OPDIVIDE:
1434					offset = lhs / off;
1435					break;
1436				case FILE_OPMODULO:
1437					offset = lhs % off;
1438					break;
1439				}
1440			} else
1441				offset = lhs;
1442			if (m->in_op & FILE_OPINVERSE)
1443				offset = ~offset;
1444			break;
1445		case FILE_LELONG:
1446		case FILE_LEID3:
1447			if (OFFSET_OOB(nbytes, offset, 4))
1448				return 0;
1449			lhs = LE32(p);
1450			if (off) {
1451				switch (m->in_op & FILE_OPS_MASK) {
1452				case FILE_OPAND:
1453					offset = lhs & off;
1454					break;
1455				case FILE_OPOR:
1456					offset = lhs | off;
1457					break;
1458				case FILE_OPXOR:
1459					offset = lhs ^ off;
1460					break;
1461				case FILE_OPADD:
1462					offset = lhs + off;
1463					break;
1464				case FILE_OPMINUS:
1465					offset = lhs - off;
1466					break;
1467				case FILE_OPMULTIPLY:
1468					offset = lhs * off;
1469					break;
1470				case FILE_OPDIVIDE:
1471					offset = lhs / off;
1472					break;
1473				case FILE_OPMODULO:
1474					offset = lhs % off;
1475					break;
1476				}
1477			} else
1478				offset = lhs;
1479			if (m->in_op & FILE_OPINVERSE)
1480				offset = ~offset;
1481			break;
1482		case FILE_MELONG:
1483			if (OFFSET_OOB(nbytes, offset, 4))
1484				return 0;
1485			lhs = ME32(p);
1486			if (off) {
1487				switch (m->in_op & FILE_OPS_MASK) {
1488				case FILE_OPAND:
1489					offset = lhs & off;
1490					break;
1491				case FILE_OPOR:
1492					offset = lhs | off;
1493					break;
1494				case FILE_OPXOR:
1495					offset = lhs ^ off;
1496					break;
1497				case FILE_OPADD:
1498					offset = lhs + off;
1499					break;
1500				case FILE_OPMINUS:
1501					offset = lhs - off;
1502					break;
1503				case FILE_OPMULTIPLY:
1504					offset = lhs * off;
1505					break;
1506				case FILE_OPDIVIDE:
1507					offset = lhs / off;
1508					break;
1509				case FILE_OPMODULO:
1510					offset = lhs % off;
1511					break;
1512				}
1513			} else
1514				offset = lhs;
1515			if (m->in_op & FILE_OPINVERSE)
1516				offset = ~offset;
1517			break;
1518		case FILE_LONG:
1519			if (OFFSET_OOB(nbytes, offset, 4))
1520				return 0;
1521			if (off) {
1522				switch (m->in_op & FILE_OPS_MASK) {
1523				case FILE_OPAND:
1524					offset = p->l & off;
1525					break;
1526				case FILE_OPOR:
1527					offset = p->l | off;
1528					break;
1529				case FILE_OPXOR:
1530					offset = p->l ^ off;
1531					break;
1532				case FILE_OPADD:
1533					offset = p->l + off;
1534					break;
1535				case FILE_OPMINUS:
1536					offset = p->l - off;
1537					break;
1538				case FILE_OPMULTIPLY:
1539					offset = p->l * off;
1540					break;
1541				case FILE_OPDIVIDE:
1542					offset = p->l / off;
1543					break;
1544				case FILE_OPMODULO:
1545					offset = p->l % off;
1546					break;
1547				}
1548			} else
1549				offset = p->l;
1550			if (m->in_op & FILE_OPINVERSE)
1551				offset = ~offset;
1552			break;
1553		default:
1554			break;
1555		}
1556
1557		switch (in_type) {
1558		case FILE_LEID3:
1559		case FILE_BEID3:
1560			offset = ((((offset >>  0) & 0x7f) <<  0) |
1561				  (((offset >>  8) & 0x7f) <<  7) |
1562				  (((offset >> 16) & 0x7f) << 14) |
1563				  (((offset >> 24) & 0x7f) << 21));
1564			if ((ms->flags & MAGIC_DEBUG) != 0)
1565				fprintf(stderr, "id3 offs=%u\n", offset);
1566			break;
1567		default:
1568			break;
1569		}
1570
1571		if (m->flag & INDIROFFADD) {
1572			offset += ms->c.li[cont_level-1].off;
1573			if (offset == 0) {
1574				if ((ms->flags & MAGIC_DEBUG) != 0)
1575					fprintf(stderr,
1576					    "indirect *zero* offset\n");
1577				return 0;
1578			}
1579			if ((ms->flags & MAGIC_DEBUG) != 0)
1580				fprintf(stderr, "indirect +offs=%u\n", offset);
1581		}
1582		if (mcopy(ms, p, m->type, 0, s, offset, nbytes, m) == -1)
1583			return -1;
1584		ms->offset = offset;
1585
1586		if ((ms->flags & MAGIC_DEBUG) != 0) {
1587			mdebug(offset, (char *)(void *)p,
1588			    sizeof(union VALUETYPE));
1589#ifndef COMPILE_ONLY
1590			file_mdump(m);
1591#endif
1592		}
1593	}
1594
1595	/* Verify we have enough data to match magic type */
1596	switch (m->type) {
1597	case FILE_BYTE:
1598		if (OFFSET_OOB(nbytes, offset, 1))
1599			return 0;
1600		break;
1601
1602	case FILE_SHORT:
1603	case FILE_BESHORT:
1604	case FILE_LESHORT:
1605		if (OFFSET_OOB(nbytes, offset, 2))
1606			return 0;
1607		break;
1608
1609	case FILE_LONG:
1610	case FILE_BELONG:
1611	case FILE_LELONG:
1612	case FILE_MELONG:
1613	case FILE_DATE:
1614	case FILE_BEDATE:
1615	case FILE_LEDATE:
1616	case FILE_MEDATE:
1617	case FILE_LDATE:
1618	case FILE_BELDATE:
1619	case FILE_LELDATE:
1620	case FILE_MELDATE:
1621	case FILE_FLOAT:
1622	case FILE_BEFLOAT:
1623	case FILE_LEFLOAT:
1624		if (OFFSET_OOB(nbytes, offset, 4))
1625			return 0;
1626		break;
1627
1628	case FILE_DOUBLE:
1629	case FILE_BEDOUBLE:
1630	case FILE_LEDOUBLE:
1631		if (OFFSET_OOB(nbytes, offset, 8))
1632			return 0;
1633		break;
1634
1635	case FILE_STRING:
1636	case FILE_PSTRING:
1637	case FILE_SEARCH:
1638		if (OFFSET_OOB(nbytes, offset, m->vallen))
1639			return 0;
1640		break;
1641
1642	case FILE_REGEX:
1643		if (nbytes < offset)
1644			return 0;
1645		break;
1646
1647	case FILE_INDIRECT:
1648		if (m->str_flags & INDIRECT_RELATIVE)
1649			offset += CAST(uint32_t, o);
1650		if (offset == 0)
1651			return 0;
1652
1653		if (nbytes < offset)
1654			return 0;
1655
1656		if ((pb = file_push_buffer(ms)) == NULL)
1657			return -1;
1658
1659		rv = file_softmagic(ms, s + offset, nbytes - offset,
1660		    indir_level + 1, name_count, BINTEST, text);
1661
1662		if ((ms->flags & MAGIC_DEBUG) != 0)
1663			fprintf(stderr, "indirect @offs=%u[%d]\n", offset, rv);
1664
1665		rbuf = file_pop_buffer(ms, pb);
1666		if (rbuf == NULL && ms->event_flags & EVENT_HAD_ERR)
1667			return -1;
1668
1669		if (rv == 1) {
1670			if ((ms->flags & MAGIC_NODESC) == 0 &&
1671			    file_printf(ms, F(ms, m, "%u"), offset) == -1) {
1672				free(rbuf);
1673				return -1;
1674			}
1675			if (file_printf(ms, "%s", rbuf) == -1) {
1676				free(rbuf);
1677				return -1;
1678			}
1679		}
1680		free(rbuf);
1681		return rv;
1682
1683	case FILE_USE:
1684		if (nbytes < offset)
1685			return 0;
1686		rbuf = m->value.s;
1687		if (*rbuf == '^') {
1688			rbuf++;
1689			flip = !flip;
1690		}
1691		if (file_magicfind(ms, rbuf, &ml) == -1) {
1692			file_error(ms, 0, "cannot find entry `%s'", rbuf);
1693			return -1;
1694		}
1695		(*name_count)++;
1696		oneed_separator = *need_separator;
1697		if (m->flag & NOSPACE)
1698			*need_separator = 0;
1699		rv = match(ms, ml.magic, ml.nmagic, s, nbytes, offset + o,
1700		    mode, text, flip, indir_level, name_count,
1701		    printed_something, need_separator, returnval);
1702		if (rv != 1)
1703		    *need_separator = oneed_separator;
1704		return 1;
1705
1706	case FILE_NAME:
1707		if (ms->flags & MAGIC_NODESC)
1708			return 1;
1709		if (file_printf(ms, "%s", m->desc) == -1)
1710			return -1;
1711		return 1;
1712	case FILE_DEFAULT:	/* nothing to check */
1713	case FILE_CLEAR:
1714	default:
1715		break;
1716	}
1717	if (!mconvert(ms, m, flip))
1718		return 0;
1719	return 1;
1720}
1721
1722private uint64_t
1723file_strncmp(const char *s1, const char *s2, size_t len, uint32_t flags)
1724{
1725	/*
1726	 * Convert the source args to unsigned here so that (1) the
1727	 * compare will be unsigned as it is in strncmp() and (2) so
1728	 * the ctype functions will work correctly without extra
1729	 * casting.
1730	 */
1731	const unsigned char *a = (const unsigned char *)s1;
1732	const unsigned char *b = (const unsigned char *)s2;
1733	uint64_t v;
1734
1735	/*
1736	 * What we want here is v = strncmp(s1, s2, len),
1737	 * but ignoring any nulls.
1738	 */
1739	v = 0;
1740	if (0L == flags) { /* normal string: do it fast */
1741		while (len-- > 0)
1742			if ((v = *b++ - *a++) != '\0')
1743				break;
1744	}
1745	else { /* combine the others */
1746		while (len-- > 0) {
1747			if ((flags & STRING_IGNORE_LOWERCASE) &&
1748			    islower(*a)) {
1749				if ((v = tolower(*b++) - *a++) != '\0')
1750					break;
1751			}
1752			else if ((flags & STRING_IGNORE_UPPERCASE) &&
1753			    isupper(*a)) {
1754				if ((v = toupper(*b++) - *a++) != '\0')
1755					break;
1756			}
1757			else if ((flags & STRING_COMPACT_WHITESPACE) &&
1758			    isspace(*a)) {
1759				a++;
1760				if (isspace(*b++)) {
1761					if (!isspace(*a))
1762						while (isspace(*b))
1763							b++;
1764				}
1765				else {
1766					v = 1;
1767					break;
1768				}
1769			}
1770			else if ((flags & STRING_COMPACT_OPTIONAL_WHITESPACE) &&
1771			    isspace(*a)) {
1772				a++;
1773				while (isspace(*b))
1774					b++;
1775			}
1776			else {
1777				if ((v = *b++ - *a++) != '\0')
1778					break;
1779			}
1780		}
1781	}
1782	return v;
1783}
1784
1785private uint64_t
1786file_strncmp16(const char *a, const char *b, size_t len, uint32_t flags)
1787{
1788	/*
1789	 * XXX - The 16-bit string compare probably needs to be done
1790	 * differently, especially if the flags are to be supported.
1791	 * At the moment, I am unsure.
1792	 */
1793	flags = 0;
1794	return file_strncmp(a, b, len, flags);
1795}
1796
1797private int
1798magiccheck(struct magic_set *ms, struct magic *m)
1799{
1800	uint64_t l = m->value.q;
1801	uint64_t v;
1802	float fl, fv;
1803	double dl, dv;
1804	int matched;
1805	union VALUETYPE *p = &ms->ms_value;
1806
1807	switch (m->type) {
1808	case FILE_BYTE:
1809		v = p->b;
1810		break;
1811
1812	case FILE_SHORT:
1813	case FILE_BESHORT:
1814	case FILE_LESHORT:
1815		v = p->h;
1816		break;
1817
1818	case FILE_LONG:
1819	case FILE_BELONG:
1820	case FILE_LELONG:
1821	case FILE_MELONG:
1822	case FILE_DATE:
1823	case FILE_BEDATE:
1824	case FILE_LEDATE:
1825	case FILE_MEDATE:
1826	case FILE_LDATE:
1827	case FILE_BELDATE:
1828	case FILE_LELDATE:
1829	case FILE_MELDATE:
1830		v = p->l;
1831		break;
1832
1833	case FILE_QUAD:
1834	case FILE_LEQUAD:
1835	case FILE_BEQUAD:
1836	case FILE_QDATE:
1837	case FILE_BEQDATE:
1838	case FILE_LEQDATE:
1839	case FILE_QLDATE:
1840	case FILE_BEQLDATE:
1841	case FILE_LEQLDATE:
1842	case FILE_QWDATE:
1843	case FILE_BEQWDATE:
1844	case FILE_LEQWDATE:
1845		v = p->q;
1846		break;
1847
1848	case FILE_FLOAT:
1849	case FILE_BEFLOAT:
1850	case FILE_LEFLOAT:
1851		fl = m->value.f;
1852		fv = p->f;
1853		switch (m->reln) {
1854		case 'x':
1855			matched = 1;
1856			break;
1857
1858		case '!':
1859			matched = fv != fl;
1860			break;
1861
1862		case '=':
1863			matched = fv == fl;
1864			break;
1865
1866		case '>':
1867			matched = fv > fl;
1868			break;
1869
1870		case '<':
1871			matched = fv < fl;
1872			break;
1873
1874		default:
1875			file_magerror(ms, "cannot happen with float: invalid relation `%c'",
1876			    m->reln);
1877			return -1;
1878		}
1879		return matched;
1880
1881	case FILE_DOUBLE:
1882	case FILE_BEDOUBLE:
1883	case FILE_LEDOUBLE:
1884		dl = m->value.d;
1885		dv = p->d;
1886		switch (m->reln) {
1887		case 'x':
1888			matched = 1;
1889			break;
1890
1891		case '!':
1892			matched = dv != dl;
1893			break;
1894
1895		case '=':
1896			matched = dv == dl;
1897			break;
1898
1899		case '>':
1900			matched = dv > dl;
1901			break;
1902
1903		case '<':
1904			matched = dv < dl;
1905			break;
1906
1907		default:
1908			file_magerror(ms, "cannot happen with double: invalid relation `%c'", m->reln);
1909			return -1;
1910		}
1911		return matched;
1912
1913	case FILE_DEFAULT:
1914	case FILE_CLEAR:
1915		l = 0;
1916		v = 0;
1917		break;
1918
1919	case FILE_STRING:
1920	case FILE_PSTRING:
1921		l = 0;
1922		v = file_strncmp(m->value.s, p->s, (size_t)m->vallen, m->str_flags);
1923		break;
1924
1925	case FILE_BESTRING16:
1926	case FILE_LESTRING16:
1927		l = 0;
1928		v = file_strncmp16(m->value.s, p->s, (size_t)m->vallen, m->str_flags);
1929		break;
1930
1931	case FILE_SEARCH: { /* search ms->search.s for the string m->value.s */
1932		size_t slen;
1933		size_t idx;
1934
1935		if (ms->search.s == NULL)
1936			return 0;
1937
1938		slen = MIN(m->vallen, sizeof(m->value.s));
1939		l = 0;
1940		v = 0;
1941
1942		for (idx = 0; m->str_range == 0 || idx < m->str_range; idx++) {
1943			if (slen + idx > ms->search.s_len)
1944				break;
1945
1946			v = file_strncmp(m->value.s, ms->search.s + idx, slen,
1947			    m->str_flags);
1948			if (v == 0) {	/* found match */
1949				ms->search.offset += idx;
1950				ms->search.rm_len = m->str_range - idx;
1951				break;
1952			}
1953		}
1954		break;
1955	}
1956	case FILE_REGEX: {
1957		int rc;
1958		file_regex_t rx;
1959		const char *search;
1960
1961		if (ms->search.s == NULL)
1962			return 0;
1963
1964		l = 0;
1965		rc = file_regcomp(&rx, m->value.s,
1966		    REG_EXTENDED|REG_NEWLINE|
1967		    ((m->str_flags & STRING_IGNORE_CASE) ? REG_ICASE : 0));
1968		if (rc) {
1969			file_regerror(&rx, rc, ms);
1970			v = (uint64_t)-1;
1971		} else {
1972			regmatch_t pmatch[1];
1973			size_t slen = ms->search.s_len;
1974#ifndef REG_STARTEND
1975#define	REG_STARTEND	0
1976			char *copy;
1977			if (slen != 0) {
1978			    copy = malloc(slen);
1979			    if (copy == NULL)  {
1980				file_error(ms, errno,
1981				    "can't allocate %" SIZE_T_FORMAT "u bytes",
1982				    slen);
1983				return -1;
1984			    }
1985			    memcpy(copy, ms->search.s, slen);
1986			    copy[--slen] = '\0';
1987			    search = copy;
1988			} else {
1989			    search = ms->search.s;
1990			    copy = NULL;
1991			}
1992#else
1993			search = ms->search.s;
1994			pmatch[0].rm_so = 0;
1995			pmatch[0].rm_eo = slen;
1996#endif
1997			rc = file_regexec(&rx, (const char *)search,
1998			    1, pmatch, REG_STARTEND);
1999#if REG_STARTEND == 0
2000			free(copy);
2001#endif
2002			switch (rc) {
2003			case 0:
2004				ms->search.s += (int)pmatch[0].rm_so;
2005				ms->search.offset += (size_t)pmatch[0].rm_so;
2006				ms->search.rm_len =
2007				    (size_t)(pmatch[0].rm_eo - pmatch[0].rm_so);
2008				v = 0;
2009				break;
2010
2011			case REG_NOMATCH:
2012				v = 1;
2013				break;
2014
2015			default:
2016				file_regerror(&rx, rc, ms);
2017				v = (uint64_t)-1;
2018				break;
2019			}
2020		}
2021		file_regfree(&rx);
2022		if (v == (uint64_t)-1)
2023			return -1;
2024		break;
2025	}
2026	case FILE_INDIRECT:
2027	case FILE_USE:
2028	case FILE_NAME:
2029		return 1;
2030	default:
2031		file_magerror(ms, "invalid type %d in magiccheck()", m->type);
2032		return -1;
2033	}
2034
2035	v = file_signextend(ms, m, v);
2036
2037	switch (m->reln) {
2038	case 'x':
2039		if ((ms->flags & MAGIC_DEBUG) != 0)
2040			(void) fprintf(stderr, "%" INT64_T_FORMAT
2041			    "u == *any* = 1\n", (unsigned long long)v);
2042		matched = 1;
2043		break;
2044
2045	case '!':
2046		matched = v != l;
2047		if ((ms->flags & MAGIC_DEBUG) != 0)
2048			(void) fprintf(stderr, "%" INT64_T_FORMAT "u != %"
2049			    INT64_T_FORMAT "u = %d\n", (unsigned long long)v,
2050			    (unsigned long long)l, matched);
2051		break;
2052
2053	case '=':
2054		matched = v == l;
2055		if ((ms->flags & MAGIC_DEBUG) != 0)
2056			(void) fprintf(stderr, "%" INT64_T_FORMAT "u == %"
2057			    INT64_T_FORMAT "u = %d\n", (unsigned long long)v,
2058			    (unsigned long long)l, matched);
2059		break;
2060
2061	case '>':
2062		if (m->flag & UNSIGNED) {
2063			matched = v > l;
2064			if ((ms->flags & MAGIC_DEBUG) != 0)
2065				(void) fprintf(stderr, "%" INT64_T_FORMAT
2066				    "u > %" INT64_T_FORMAT "u = %d\n",
2067				    (unsigned long long)v,
2068				    (unsigned long long)l, matched);
2069		}
2070		else {
2071			matched = (int64_t) v > (int64_t) l;
2072			if ((ms->flags & MAGIC_DEBUG) != 0)
2073				(void) fprintf(stderr, "%" INT64_T_FORMAT
2074				    "d > %" INT64_T_FORMAT "d = %d\n",
2075				    (long long)v, (long long)l, matched);
2076		}
2077		break;
2078
2079	case '<':
2080		if (m->flag & UNSIGNED) {
2081			matched = v < l;
2082			if ((ms->flags & MAGIC_DEBUG) != 0)
2083				(void) fprintf(stderr, "%" INT64_T_FORMAT
2084				    "u < %" INT64_T_FORMAT "u = %d\n",
2085				    (unsigned long long)v,
2086				    (unsigned long long)l, matched);
2087		}
2088		else {
2089			matched = (int64_t) v < (int64_t) l;
2090			if ((ms->flags & MAGIC_DEBUG) != 0)
2091				(void) fprintf(stderr, "%" INT64_T_FORMAT
2092				    "d < %" INT64_T_FORMAT "d = %d\n",
2093				     (long long)v, (long long)l, matched);
2094		}
2095		break;
2096
2097	case '&':
2098		matched = (v & l) == l;
2099		if ((ms->flags & MAGIC_DEBUG) != 0)
2100			(void) fprintf(stderr, "((%" INT64_T_FORMAT "x & %"
2101			    INT64_T_FORMAT "x) == %" INT64_T_FORMAT
2102			    "x) = %d\n", (unsigned long long)v,
2103			    (unsigned long long)l, (unsigned long long)l,
2104			    matched);
2105		break;
2106
2107	case '^':
2108		matched = (v & l) != l;
2109		if ((ms->flags & MAGIC_DEBUG) != 0)
2110			(void) fprintf(stderr, "((%" INT64_T_FORMAT "x & %"
2111			    INT64_T_FORMAT "x) != %" INT64_T_FORMAT
2112			    "x) = %d\n", (unsigned long long)v,
2113			    (unsigned long long)l, (unsigned long long)l,
2114			    matched);
2115		break;
2116
2117	default:
2118		file_magerror(ms, "cannot happen: invalid relation `%c'",
2119		    m->reln);
2120		return -1;
2121	}
2122
2123	return matched;
2124}
2125
2126private int
2127handle_annotation(struct magic_set *ms, struct magic *m)
2128{
2129	if (ms->flags & MAGIC_APPLE) {
2130		if (file_printf(ms, "%.8s", m->apple) == -1)
2131			return -1;
2132		return 1;
2133	}
2134	if (ms->flags & MAGIC_EXTENSION) {
2135		if (file_printf(ms, "%s", m->ext) == -1)
2136			return -1;
2137		return 1;
2138	}
2139	if ((ms->flags & MAGIC_MIME_TYPE) && m->mimetype[0]) {
2140		if (file_printf(ms, "%s", m->mimetype) == -1)
2141			return -1;
2142		return 1;
2143	}
2144	return 0;
2145}
2146
2147private int
2148print_sep(struct magic_set *ms, int firstline)
2149{
2150	if (ms->flags & MAGIC_NODESC)
2151		return 0;
2152	if (firstline)
2153		return 0;
2154	/*
2155	 * we found another match
2156	 * put a newline and '-' to do some simple formatting
2157	 */
2158	return file_printf(ms, "\n- ");
2159}
2160