/* * CDDL HEADER START * * The contents of this file are subject to the terms of the * Common Development and Distribution License (the "License"). * You may not use this file except in compliance with the License. * * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE * or http://www.opensolaris.org/os/licensing. * See the License for the specific language governing permissions * and limitations under the License. * * When distributing Covered Code, include this CDDL HEADER in each * file and include the License file at usr/src/OPENSOLARIS.LICENSE. * If applicable, add the following below this CDDL HEADER, with the * fields enclosed by brackets "[]" replaced with your own identifying * information: Portions Copyright [yyyy] [name of copyright owner] * * CDDL HEADER END */ /* * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. * Copyright 2012 Nexenta Systems, Inc. All rights reserved. * Copyright (c) 2013, Joyent, Inc. All rights reserved. * Copyright 2024 Oxide Computer Co. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #define FMDUMP_EXIT_SUCCESS 0 #define FMDUMP_EXIT_FATAL 1 #define FMDUMP_EXIT_USAGE 2 #define FMDUMP_EXIT_ERROR 3 const char *g_pname; ulong_t g_errs; ulong_t g_recs; char *g_root; struct topo_hdl *g_thp; fmd_msg_hdl_t *g_msg; /*PRINTFLIKE2*/ void fmdump_printf(FILE *fp, const char *format, ...) { va_list ap; va_start(ap, format); if (vfprintf(fp, format, ap) < 0) { (void) fprintf(stderr, "%s: failed to print record: %s\n", g_pname, strerror(errno)); g_errs++; } va_end(ap); } void fmdump_vwarn(const char *format, va_list ap) { int err = errno; (void) fprintf(stderr, "%s: warning: ", g_pname); (void) vfprintf(stderr, format, ap); if (strchr(format, '\n') == NULL) (void) fprintf(stderr, ": %s\n", strerror(err)); g_errs++; } /*PRINTFLIKE1*/ void fmdump_warn(const char *format, ...) { va_list ap; va_start(ap, format); fmdump_vwarn(format, ap); va_end(ap); } static void fmdump_exit(int err, int exitcode, const char *format, va_list ap) { (void) fprintf(stderr, "%s: ", g_pname); (void) vfprintf(stderr, format, ap); if (strchr(format, '\n') == NULL) (void) fprintf(stderr, ": %s\n", strerror(err)); exit(exitcode); } /*PRINTFLIKE1*/ static void fmdump_fatal(const char *format, ...) { int err = errno; va_list ap; va_start(ap, format); fmdump_exit(err, FMDUMP_EXIT_FATAL, format, ap); va_end(ap); } /*PRINTFLIKE1*/ static void fmdump_usage(const char *format, ...) { int err = errno; va_list ap; va_start(ap, format); fmdump_exit(err, FMDUMP_EXIT_USAGE, format, ap); va_end(ap); } char * fmdump_date(char *buf, size_t len, const fmd_log_record_t *rp) { if (rp->rec_sec > LONG_MAX) { fmdump_warn("record time is too large for 32-bit utility\n"); (void) snprintf(buf, len, "0x%llx", rp->rec_sec); } else { time_t tod = (time_t)rp->rec_sec; time_t now = time(NULL); if (tod > now+60 || tod < now - 6L*30L*24L*60L*60L) { /* 6 months ago */ (void) strftime(buf, len, "%b %d %Y %T", localtime(&tod)); } else { size_t sz; sz = strftime(buf, len, "%b %d %T", localtime(&tod)); (void) snprintf(buf + sz, len - sz, ".%4.4llu", rp->rec_nsec / (NANOSEC / 10000)); } } return (buf); } char * fmdump_year(char *buf, size_t len, const fmd_log_record_t *rp) { #ifdef _ILP32 if (rp->rec_sec > LONG_MAX) { fmdump_warn("record time is too large for 32-bit utility\n"); (void) snprintf(buf, len, "0x%llx", rp->rec_sec); } else { #endif time_t tod = (time_t)rp->rec_sec; (void) strftime(buf, len, "%b %d %Y %T", localtime(&tod)); #ifdef _ILP32 } #endif return (buf); } /* BEGIN CSTYLED */ static const char *synopsis = "Usage: %s [[-e | -i | -I | -u] | -A ] [-f] [-aHmvVp] [-c class] [-R root]\n" "\t [-t time] [-T time] [-u uuid] [-n name[.name]*[=value]]\n" "\t [-N name[.name]*[=value][;name[.name]*[=value]]*] " "[file]...\n " "Log selection: [-e | -i | -I] or one [file]; default is the fault log\n" "\t-e display error log content\n" "\t-i display infolog content\n" "\t-I display the high-value-infolog content\n" "\t-R set root directory for pathname expansions\n " "Command behaviour:\n" "\t-A Aggregate specified [file]s or, if no [file], all known logs\n" "\t-H display the log's header attributes instead of contents\n" "\t-f follow growth of log file by waiting for additional data\n " "Output options:\n" "\t-j Used with -V: emit JSON-formatted output\n" "\t-m display human-readable messages (only for fault logs)\n" "\t-p Used with -V: apply some output prettification\n" "\t-v set verbose mode: display additional event detail\n" "\t-V set very verbose mode: display complete event contents\n " "Selection filters:\n" "\t-a select all events, including normally silent events\n" "\t-c select events that match the specified class\n" "\t-n select events containing named nvpair (with matching value)\n" "\t-N select events matching multiple property names (or nvpairs)\n" "\t-t select events that occurred after the specified time\n" "\t-T select events that occurred before the specified time\n" "\t-u select events that match the specified diagnosis uuid\n"; /* END CSTYLED */ static int usage(FILE *fp) { (void) fprintf(fp, synopsis, g_pname); return (FMDUMP_EXIT_USAGE); } /*ARGSUSED*/ static int error(fmd_log_t *lp, void *private) { fmdump_warn("skipping record: %s\n", fmd_log_errmsg(lp, fmd_log_errno(lp))); return (0); } /* * Yet another disgusting argument parsing function (TM). We attempt to parse * a time argument in a variety of strptime(3C) formats, in which case it is * interpreted as a local time and is converted to a timeval using mktime(3C). * If those formats fail, we look to see if the time is a decimal integer * followed by one of our magic suffixes, in which case the time is interpreted * as a time delta *before* the current time-of-day (i.e. "1h" = "1 hour ago"). */ static struct timeval * gettimeopt(const char *arg) { const struct { const char *name; hrtime_t mul; } suffix[] = { { "ns", NANOSEC / NANOSEC }, { "nsec", NANOSEC / NANOSEC }, { "us", NANOSEC / MICROSEC }, { "usec", NANOSEC / MICROSEC }, { "ms", NANOSEC / MILLISEC }, { "msec", NANOSEC / MILLISEC }, { "s", NANOSEC / SEC }, { "sec", NANOSEC / SEC }, { "m", NANOSEC * (hrtime_t)60 }, { "min", NANOSEC * (hrtime_t)60 }, { "h", NANOSEC * (hrtime_t)(60 * 60) }, { "hour", NANOSEC * (hrtime_t)(60 * 60) }, { "d", NANOSEC * (hrtime_t)(24 * 60 * 60) }, { "day", NANOSEC * (hrtime_t)(24 * 60 * 60) }, { NULL } }; struct timeval *tvp = malloc(sizeof (struct timeval)); struct timeval tod; struct tm tm; char *p; if (tvp == NULL) fmdump_fatal("failed to allocate memory"); if (gettimeofday(&tod, NULL) != 0) fmdump_fatal("failed to get tod"); /* * First try a variety of strptime() calls. If these all fail, we'll * try parsing an integer followed by one of our suffix[] strings. */ if ((p = strptime(arg, "%m/%d/%Y %H:%M:%S", &tm)) == NULL && (p = strptime(arg, "%m/%d/%y %H:%M:%S", &tm)) == NULL && (p = strptime(arg, "%m/%d/%Y %H:%M", &tm)) == NULL && (p = strptime(arg, "%m/%d/%y %H:%M", &tm)) == NULL && (p = strptime(arg, "%m/%d/%Y", &tm)) == NULL && (p = strptime(arg, "%m/%d/%y", &tm)) == NULL && (p = strptime(arg, "%Y-%m-%dT%H:%M:%S", &tm)) == NULL && (p = strptime(arg, "%y-%m-%dT%H:%M:%S", &tm)) == NULL && (p = strptime(arg, "%Y-%m-%dT%H:%M", &tm)) == NULL && (p = strptime(arg, "%y-%m-%dT%H:%M", &tm)) == NULL && (p = strptime(arg, "%Y-%m-%d", &tm)) == NULL && (p = strptime(arg, "%y-%m-%d", &tm)) == NULL && (p = strptime(arg, "%d%b%Y %H:%M:%S", &tm)) == NULL && (p = strptime(arg, "%d%b%y %H:%M:%S", &tm)) == NULL && (p = strptime(arg, "%d%b%Y %H:%M", &tm)) == NULL && (p = strptime(arg, "%d%b%y %H:%M", &tm)) == NULL && (p = strptime(arg, "%d%b%Y", &tm)) == NULL && (p = strptime(arg, "%d%b%y", &tm)) == NULL && (p = strptime(arg, "%b %d %H:%M:%S", &tm)) == NULL && (p = strptime(arg, "%b %d %H:%M:%S", &tm)) == NULL && (p = strptime(arg, "%H:%M:%S", &tm)) == NULL && (p = strptime(arg, "%H:%M", &tm)) == NULL) { hrtime_t nsec; int i; errno = 0; nsec = strtol(arg, (char **)&p, 10); if (errno != 0 || nsec == 0 || p == arg || *p == '\0') fmdump_usage("illegal time format -- %s\n", arg); for (i = 0; suffix[i].name != NULL; i++) { if (strcasecmp(suffix[i].name, p) == 0) { nsec *= suffix[i].mul; break; } } if (suffix[i].name == NULL) fmdump_usage("illegal time format -- %s\n", arg); tvp->tv_sec = nsec / NANOSEC; tvp->tv_usec = (nsec % NANOSEC) / (NANOSEC / MICROSEC); if (tvp->tv_sec > tod.tv_sec) fmdump_usage("time delta precedes UTC time origin " "-- %s\n", arg); tvp->tv_sec = tod.tv_sec - tvp->tv_sec; } else if (*p == '\0' || *p == '.') { /* * If tm_year is zero, we matched [%b %d] %H:%M[:%S]; use * the result of localtime(&tod.tv_sec) to fill in the rest. */ if (tm.tm_year == 0) { int h = tm.tm_hour; int m = tm.tm_min; int s = tm.tm_sec; int b = tm.tm_mon; int d = tm.tm_mday; bcopy(localtime(&tod.tv_sec), &tm, sizeof (tm)); tm.tm_isdst = 0; /* see strptime(3C) and below */ if (d > 0) { tm.tm_mon = b; tm.tm_mday = d; } tm.tm_hour = h; tm.tm_min = m; tm.tm_sec = s; } errno = 0; tvp->tv_sec = mktime(&tm); tvp->tv_usec = 0; if (tvp->tv_sec == -1L && errno != 0) fmdump_fatal("failed to compose time %s", arg); /* * If our mktime() set tm_isdst, adjust the result for DST by * subtracting the offset between the main and alternate zones. */ if (tm.tm_isdst) tvp->tv_sec -= timezone - altzone; if (p[0] == '.') { arg = p; errno = 0; tvp->tv_usec = (suseconds_t)(strtod(arg, &p) * (double)MICROSEC); if (errno != 0 || p == arg || *p != '\0') fmdump_usage("illegal time suffix -- .%s\n", arg); } } else { fmdump_usage("unexpected suffix after time %s -- %s\n", arg, p); } return (tvp); } /* * If the -u option is specified in combination with the -e option, we iterate * over each record in the fault log with a matching UUID finding xrefs to the * error log, and then use this function to iterate over every xref'd record. */ int xref_iter(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg) { const fmd_log_record_t *xrp = rp->rec_xrefs; fmdump_arg_t *dap = arg; int i, rv = 0; for (i = 0; rv == 0 && i < rp->rec_nrefs; i++, xrp++) { if (fmd_log_filter(lp, dap->da_fc, dap->da_fv, xrp)) rv = dap->da_fmt->do_func(lp, xrp, dap->da_fp); } return (rv); } int xoff_iter(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg) { fmdump_lyr_t *dyp = arg; fmdump_printf(dyp->dy_fp, "%16llx ", (u_longlong_t)rp->rec_off); return (dyp->dy_func(lp, rp, dyp->dy_arg)); } /* * Initialize fmd_log_filter_nvarg_t from -n name=value argument string. */ static fmd_log_filter_nvarg_t * setupnamevalue(char *namevalue) { fmd_log_filter_nvarg_t *argt; char *value; regex_t *value_regex = NULL; char errstr[128]; int rv; if ((value = strchr(namevalue, '=')) == NULL) { value_regex = NULL; } else { *value++ = '\0'; /* separate name and value string */ /* * Skip white space before value to facilitate direct * cut/paste from previous fmdump output. */ while (isspace(*value)) value++; if ((value_regex = malloc(sizeof (regex_t))) == NULL) fmdump_fatal("failed to allocate memory"); /* compile regular expression for possible string match */ if ((rv = regcomp(value_regex, value, REG_NOSUB|REG_NEWLINE)) != 0) { (void) regerror(rv, value_regex, errstr, sizeof (errstr)); free(value_regex); fmdump_usage("unexpected regular expression in " "%s: %s\n", value, errstr); } } if ((argt = calloc(1, sizeof (fmd_log_filter_nvarg_t))) == NULL) fmdump_fatal("failed to allocate memory"); argt->nvarg_name = namevalue; /* now just name */ argt->nvarg_value = value; argt->nvarg_value_regex = value_regex; return (argt); } /* * As for setupnamevalue() above, create our chain of filter arguments for -N * [name[=value][;name[=value]]*. This would be simple except for the problems * of escaping something in a string. To accommodate the use of the ; within * the chain, we allow it to be escaped. One might imagine that the backslash * character should be used to escape it, but that opens Pandora's box because * the value portion of each entry (if present) is allowed to be a regex. The * treatment of backslashes within regexes is not something we want to replicate * here, which would be necessary if we wanted to allow escaping the ; with a * backslash. Specifically, consider how we treat the sequence of characters * '\\;x' (two backslash characters followed by a semicolon and then some other * character x). In the name portion of the entry, this would be a backslash * followed by an escaped semicolon, so that we would treat this as '\;' and * include x and subsequent characters in this entry. In the value portion (if * present), we would have to treat it as a pair of backslashes followed by the * terminating ; and the next entry would begin with 'x'... except that we * might be inside [] where the backslash is not special, and so on. * * Let's not do that. Instead, we allow the user to 'escape' the ; by repeating * it, and we interpret that before any regex interpretation is done. Therefore * *every* pair of consecutive semicolons, regardless of where it appears, is * replaced by a literal semicolon. This allows the semicolon to appear any * number of times in either the name or, if present, the value, including as * part of a regex (see regexp(7)), simply by doubling it. A non-doubled * semicolon always terminates the entry. This now creates one more problem: * whether to treat ';;;' as a literal semicolon followed by the entry * terminator, or the entry terminator followed by a literal semicolon to start * the next entry. Here we have to cheat a little: it's clear from the FMD PRM * (especially chapter 10 as well as the schema for module properties, buffers, * statistics, and other entities) that the event member namespace is intended * to exclude both the semicolon and whitespace. A value, or a regex intended * to match values, might well include anything. Therefore, a semicolon at the * beginning of an entry is unlikely to be useful, while one at the end of an * entry may well be intentional. We'll allow either or both when unambiguous, * but a sequence containing an odd number of consecutive ';' characters will be * interpreted as half that number of literal semicolons (rounded down) followed * by the terminator. If the user wishes to begin an event property name with a * semicolon, it needs to be the first property in the chain. Chains with * multiple properties whose names begin with a literal semicolon are not * supported. Again, this almost certainly can never matter as no event should * ever have a property whose name contains a semicolon. * * We choose the semicolon because the comma is very likely to be present in * some property values on which the user may want to filter, especially the * name of device paths. The semicolon may itself appear in values, especially * if the property is a URI, though it is likely much less common. We have to * pick something. If this proves unwieldy or insufficiently expressive, it * will need to be replaced by a full-on logical expression parser with * first-class support for internal quoting, escaping, and regexes. One might * be better off dumping JSON and importing it into a SQL database if that level * of complexity is required. */ static fmd_log_filter_nvarg_t * setupnamevalue_multi(char *chainstr) { fmd_log_filter_nvarg_t *argchain = NULL; size_t rem = strlen(chainstr) + 1; fmd_log_filter_nvarg_t *argt; /* * Here, rem holds the number of characters remaining that we are * permitted to examine, including the terminating NUL. If the first * entry begins with a single semicolon, it is considered empty and * ignored. Similarly, a trailing semicolon is optional and ignored if * present. We won't create empty filter entries for any input. */ for (char *nv = chainstr; rem > 0; ++chainstr, --rem) { switch (*chainstr) { case ';': ASSERT(rem > 1); /* * Check for double-semicolon. If found, * de-duplicate it and advance past, then continue the * loop: we can't be done yet. */ if (chainstr[1] == ';') { ASSERT(rem > 2); --rem; (void) memmove(chainstr, chainstr + 1, rem); break; } *chainstr = '\0'; /*FALLTHROUGH*/ case '\0': if (chainstr != nv) { argt = setupnamevalue(nv); argt->nvarg_next = argchain; argchain = argt; } nv = chainstr + 1; /*FALLTHROUGH*/ default: ASSERT(rem > 0); } } return (argchain); } /* * If the -a option is not present, filter out fault records that correspond * to events that the producer requested not be messaged for administrators. */ /*ARGSUSED*/ int log_filter_silent(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg) { int opt_A = (arg != NULL); boolean_t msg; char *class; /* * If -A was used then apply this filter only to events of list class */ if (opt_A) { if (nvlist_lookup_string(rp->rec_nvl, FM_CLASS, &class) != 0 || strncmp(class, FM_LIST_EVENT ".", sizeof (FM_LIST_EVENT)) != 0) return (1); } return (nvlist_lookup_boolean_value(rp->rec_nvl, FM_SUSPECT_MESSAGE, &msg) != 0 || msg != 0); } struct loglink { char *path; long suffix; struct loglink *next; }; static void addlink(struct loglink **llp, char *dirname, char *logname, long suffix) { struct loglink *newp; size_t len; char *str; newp = malloc(sizeof (struct loglink)); len = strlen(dirname) + strlen(logname) + 2; str = malloc(len); if (newp == NULL || str == NULL) fmdump_fatal("failed to allocate memory"); (void) snprintf(str, len, "%s/%s", dirname, logname); newp->path = str; newp->suffix = suffix; while (*llp != NULL && suffix < (*llp)->suffix) llp = &(*llp)->next; newp->next = *llp; *llp = newp; } /* * Find and return all the rotated logs. */ static struct loglink * get_rotated_logs(char *logpath) { char dirname[PATH_MAX], *logname, *endptr; DIR *dirp; struct dirent *dp; long len, suffix; struct loglink *head = NULL; (void) strlcpy(dirname, logpath, sizeof (dirname)); logname = strrchr(dirname, '/'); *logname++ = '\0'; len = strlen(logname); if ((dirp = opendir(dirname)) == NULL) { fmdump_warn("failed to opendir `%s'", dirname); g_errs++; return (NULL); } while ((dp = readdir(dirp)) != NULL) { /* * Search the log directory for logs named ".0", * ".1", etc and add to the link in the * reverse numeric order. */ if (strlen(dp->d_name) < len + 2 || strncmp(dp->d_name, logname, len) != 0 || dp->d_name[len] != '.') continue; /* * "*.0-" file normally should not be seen. It may * exist when user manually run 'fmadm rotate'. * In such case, we put it at the end of the list so * it'll be dumped after all the rotated logs, before * the current one. */ if (strcmp(dp->d_name + len + 1, "0-") == 0) addlink(&head, dirname, dp->d_name, -1); else if ((suffix = strtol(dp->d_name + len + 1, &endptr, 10)) >= 0 && *endptr == '\0') addlink(&head, dirname, dp->d_name, suffix); } (void) closedir(dirp); return (head); } /* * Aggregate log files. If ifiles is not NULL then one or more files * were listed on the command line, and we will merge just those files. * Otherwise we will merge all known log file types, and include the * rotated logs for each type (you can suppress the inclusion of * some logtypes through use of FMDUMP_AGGREGATE_IGNORE in the process * environment, setting it to a comma-separated list of log labels and/or * log filenames to ignore). * * We will not attempt to perform a chronological sort across all log records * of all files. Indeed, we won't even sort individual log files - * we will not re-order events differently to how they appeared in their * original log file. This is because log files are already inherently * ordered by the order in which fmd receives and processes events. * So we determine the output order by comparing the "next" record * off the top of each log file. * * We will construct a number of log record source "pipelines". As above, * the next record to render in the overall output is that from the * pipeline with the oldest event. * * For the case that input logfiles were listed on the command line, each * pipeline will process exactly one of those logfiles. Distinct pipelines * may process logfiles of the same "type" - eg if two "error" logs and * one "fault" logs are specified then there'll be two pipelines producing * events from "error" logs. * * If we are merging all known log types then we will construct exactly * one pipeline for each known log type - one for error, one for fault, etc. * Each pipeline will process first the rotated logs of that type and then * move on to the current log of that type. * * The output from all pipelines flows into a serializer which selects * the next record once all pipelines have asserted their output state. * The output state of a pipeline is one of: * * - record available: the next record from this pipeline is available * for comparison and consumption * * - done: this pipeline will produce no more records * * - polling: this pipeline is polling for new records and will * make them available as output if/when any are observed * * - processing: output state will be updated shortly * * A pipeline iterates over each file queued to it using fmd_log_xiter. * We do this in a separate thread for each pipeline. The callback on * each iteration must update the serializer to let it know that * a new record is available. In the serializer thread we decide whether * we have all records expected have arrived and it is time to choose * the next output record. */ /* * A pipeline descriptor. The pl_cv condition variable is used together * with pl_lock for initial synchronisation, and thereafter with the * lock for the serializer for pausing and continuing this pipeline. */ struct fmdump_pipeline { pthread_mutex_t pl_lock; /* used only in pipeline startup */ int pl_started; /* sync with main thread on startup */ pthread_t pl_thr; /* our processing thread */ pthread_cond_t pl_cv; /* see above */ struct loglink *pl_rotated; /* rotated logs to process first */ char *pl_logpath; /* target path to process */ char *pl_processing; /* path currently being processed */ struct fmdump_srlzer *pl_srlzer; /* link to serializer */ int pl_srlzeridx; /* serializer index for this pipeline */ const fmdump_ops_t *pl_ops; /* ops for the log type we're given */ int pl_fmt; /* FMDUMP_{SHORT,VERB1,VERB2,PRETTY} */ boolean_t pl_follow; /* go into poll mode at log end */ fmdump_arg_t pl_arg; /* arguments */ }; enum fmdump_pipestate { FMDUMP_PIPE_PROCESSING = 0x1000, FMDUMP_PIPE_RECORDAVAIL, FMDUMP_PIPE_POLLING, FMDUMP_PIPE_DONE }; /* * Each pipeline has an associated output slot in the serializer. This * must be updated with the serializer locked. After update evaluate * whether there are enough slots decided that we should select a * record to output. */ struct fmdump_srlzer_slot { enum fmdump_pipestate ss_state; uint64_t ss_sec; uint64_t ss_nsec; }; /* * All pipelines are linked to a single serializer. The serializer * structure must be updated under the ds_lock; this mutex is also * paired with the pl_cv of individual pipelines (one mutex, many condvars) * in pausing and continuing individual pipelines. */ struct fmdump_srlzer { struct fmdump_pipeline *ds_pipearr; /* pipeline array */ pthread_mutex_t ds_lock; /* see above */ uint32_t ds_pipecnt; /* number of pipelines */ uint32_t ds_pollcnt; /* pipelines in poll mode */ uint32_t ds_nrecordavail; /* pipelines with a record */ uint32_t ds_ndone; /* completed pipelines */ struct fmdump_srlzer_slot *ds_slot; /* slot array */ }; /* * All known log types. When aggregation is requested an no file list * is provided we will process the logs identified here (if lt_enabled * is true and not over-ridden by environment settings). We also * use this in determining the appropriate ops structure for each distinct * label. */ static struct fmdump_logtype { const char *lt_label; /* label from log header */ boolean_t lt_enabled; /* include in merge? */ const char *lt_logname; /* var/fm/fmd/%s */ const fmdump_ops_t *lt_ops; } logtypes[] = { { "error", B_TRUE, "errlog", &fmdump_err_ops }, { "fault", B_TRUE, "fltlog", &fmdump_flt_ops }, { "info", B_TRUE, "infolog", &fmdump_info_ops }, { "info", B_TRUE, "infolog_hival", &fmdump_info_ops }, { "asru", B_FALSE, /* not included unless in file list */ NULL, &fmdump_asru_ops /* but we need ops when it is */ } }; /* * Disable logtypes per environment setting. Does not apply when a list * of logs is provided on the command line. */ static void do_disables(void) { char *env = getenv("FMDUMP_AGGREGATE_IGNORE"); char *dup, *start, *tofree; int i; if (env == NULL) return; tofree = dup = strdup(env); while (dup != NULL) { start = strsep(&dup, ","); for (i = 0; i < sizeof (logtypes) / sizeof (logtypes[0]); i++) { if (logtypes[i].lt_logname == NULL) continue; if (strcmp(start, logtypes[i].lt_label) == 0 || strcmp(start, logtypes[i].lt_logname) == 0) { logtypes[i].lt_enabled = B_FALSE; } } } free(tofree); } static void srlzer_enter(struct fmdump_pipeline *pl) { struct fmdump_srlzer *srlzer = pl->pl_srlzer; (void) pthread_mutex_lock(&srlzer->ds_lock); } static void srlzer_exit(struct fmdump_pipeline *pl) { struct fmdump_srlzer *srlzer = pl->pl_srlzer; ASSERT(MUTEX_HELD(&srlzer->ds_lock)); (void) pthread_mutex_unlock(&srlzer->ds_lock); } static struct fmdump_pipeline * srlzer_choose(struct fmdump_srlzer *srlzer) { struct fmdump_srlzer_slot *slot, *oldest; int oldestidx = -1; int first = 1; int i; ASSERT(MUTEX_HELD(&srlzer->ds_lock)); for (i = 0, slot = &srlzer->ds_slot[0]; i < srlzer->ds_pipecnt; i++, slot++) { if (slot->ss_state != FMDUMP_PIPE_RECORDAVAIL) continue; if (first) { oldest = slot; oldestidx = i; first = 0; continue; } if (slot->ss_sec < oldest->ss_sec || slot->ss_sec == oldest->ss_sec && slot->ss_nsec < oldest->ss_nsec) { oldest = slot; oldestidx = i; } } return (oldestidx >= 0 ? &srlzer->ds_pipearr[oldestidx] : NULL); } static void pipeline_stall(struct fmdump_pipeline *pl) { struct fmdump_srlzer *srlzer = pl->pl_srlzer; ASSERT(MUTEX_HELD(&srlzer->ds_lock)); (void) pthread_cond_wait(&pl->pl_cv, &srlzer->ds_lock); } static void pipeline_continue(struct fmdump_pipeline *pl) { struct fmdump_srlzer *srlzer = pl->pl_srlzer; ASSERT(MUTEX_HELD(&srlzer->ds_lock)); (void) pthread_cond_signal(&srlzer->ds_pipearr[pl->pl_srlzeridx].pl_cv); } /* * Called on each pipeline record iteration to make a new record * available for input to the serializer. Returns 0 to indicate that * the caller must stall the pipeline, or 1 to indicate that the * caller should go ahead and render their record. If this record * addition fills the serializer then choose a pipeline that must * render output. */ static int pipeline_output(struct fmdump_pipeline *pl, const fmd_log_record_t *rp) { struct fmdump_srlzer *srlzer = pl->pl_srlzer; struct fmdump_srlzer_slot *slot; struct fmdump_pipeline *wpl; int thisidx = pl->pl_srlzeridx; ASSERT(MUTEX_HELD(&srlzer->ds_lock)); slot = &srlzer->ds_slot[thisidx]; slot->ss_state = FMDUMP_PIPE_RECORDAVAIL; slot->ss_sec = rp->rec_sec; slot->ss_nsec = rp->rec_nsec; srlzer->ds_nrecordavail++; /* * Once all pipelines are polling we just render in arrival order. */ if (srlzer->ds_pollcnt == srlzer->ds_pipecnt) return (1); /* * If not all pipelines have asserted an output yet then the * caller must block. */ if (srlzer->ds_nrecordavail + srlzer->ds_ndone + srlzer->ds_pollcnt < srlzer->ds_pipecnt) return (0); /* * Right so it's time to turn the crank by choosing which of the * filled line of slots should produce output. If it is the slot * for our caller then return their index to them, otherwise return * -1 to the caller to make them block and cv_signal the winner. */ wpl = srlzer_choose(srlzer); ASSERT(wpl != NULL); if (wpl == pl) return (1); /* Wake the oldest, and return 0 to put the caller to sleep */ pipeline_continue(wpl); return (0); } static void pipeline_mark_consumed(struct fmdump_pipeline *pl) { struct fmdump_srlzer *srlzer = pl->pl_srlzer; ASSERT(MUTEX_HELD(&srlzer->ds_lock)); srlzer->ds_slot[pl->pl_srlzeridx].ss_state = FMDUMP_PIPE_PROCESSING; srlzer->ds_nrecordavail--; } static void pipeline_done(struct fmdump_pipeline *pl) { struct fmdump_srlzer *srlzer = pl->pl_srlzer; struct fmdump_pipeline *wpl; srlzer_enter(pl); srlzer->ds_slot[pl->pl_srlzeridx].ss_state = FMDUMP_PIPE_DONE; srlzer->ds_ndone++; wpl = srlzer_choose(srlzer); if (wpl != NULL) pipeline_continue(wpl); srlzer_exit(pl); } static void pipeline_pollmode(struct fmdump_pipeline *pl) { struct fmdump_srlzer *srlzer = pl->pl_srlzer; struct fmdump_pipeline *wpl; if (srlzer->ds_slot[pl->pl_srlzeridx].ss_state == FMDUMP_PIPE_POLLING) return; srlzer_enter(pl); srlzer->ds_slot[pl->pl_srlzeridx].ss_state = FMDUMP_PIPE_POLLING; if (++srlzer->ds_pollcnt + srlzer->ds_nrecordavail == srlzer->ds_pipecnt && (wpl = srlzer_choose(srlzer)) != NULL) pipeline_continue(wpl); srlzer_exit(pl); } static int pipeline_err(fmd_log_t *lp, void *arg) { struct fmdump_pipeline *pl = (struct fmdump_pipeline *)arg; fmdump_warn("skipping record in %s: %s\n", pl->pl_processing, fmd_log_errmsg(lp, fmd_log_errno(lp))); g_errs++; return (0); } static int pipeline_cb(fmd_log_t *lp, const fmd_log_record_t *rp, void *arg) { struct fmdump_pipeline *pl = (struct fmdump_pipeline *)arg; int rc; fmd_log_rec_f *func = pl->pl_arg.da_fmt->do_func; srlzer_enter(pl); if (!pipeline_output(pl, rp)) pipeline_stall(pl); rc = func(lp, rp, pl->pl_arg.da_fp); pipeline_mark_consumed(pl); srlzer_exit(pl); return (rc); } static void pipeline_process(struct fmdump_pipeline *pl, char *logpath, boolean_t follow) { fmd_log_header_t log; fmd_log_t *lp; int err; int i; pl->pl_processing = logpath; if ((lp = fmd_log_open(FMD_LOG_VERSION, logpath, &err)) == NULL) { fmdump_warn("failed to open %s: %s\n", logpath, fmd_log_errmsg(NULL, err)); g_errs++; return; } fmd_log_header(lp, &log); for (i = 0; i < sizeof (logtypes) / sizeof (logtypes[0]); i++) { if (strcmp(log.log_label, logtypes[i].lt_label) == 0) { pl->pl_ops = logtypes[i].lt_ops; pl->pl_arg.da_fmt = &pl->pl_ops->do_formats[pl->pl_fmt]; break; } } if (pl->pl_ops == NULL) { fmdump_warn("unknown log type %s for %s\n", log.log_label, logpath); g_errs++; return; } do { if (fmd_log_xiter(lp, FMD_LOG_XITER_REFS, pl->pl_arg.da_fc, pl->pl_arg.da_fv, pipeline_cb, pipeline_err, (void *)pl, NULL) != 0) { fmdump_warn("failed to dump %s: %s\n", logpath, fmd_log_errmsg(lp, fmd_log_errno(lp))); g_errs++; fmd_log_close(lp); return; } if (follow) { pipeline_pollmode(pl); (void) sleep(1); } } while (follow); fmd_log_close(lp); } static void * pipeline_thr(void *arg) { struct fmdump_pipeline *pl = (struct fmdump_pipeline *)arg; struct loglink *ll; (void) pthread_mutex_lock(&pl->pl_lock); pl->pl_started = 1; (void) pthread_mutex_unlock(&pl->pl_lock); (void) pthread_cond_signal(&pl->pl_cv); for (ll = pl->pl_rotated; ll != NULL; ll = ll->next) pipeline_process(pl, ll->path, B_FALSE); pipeline_process(pl, pl->pl_logpath, pl->pl_follow); pipeline_done(pl); return (NULL); } static int aggregate(char **ifiles, int n_ifiles, int opt_f, fmd_log_filter_t *fv, uint_t fc, int opt_v, int opt_V, int opt_p, int opt_j) { struct fmdump_pipeline *pipeline, *pl; struct fmdump_srlzer srlzer; uint32_t npipe; int fmt; int i; if (ifiles != NULL) { npipe = n_ifiles; pipeline = calloc(npipe, sizeof (struct fmdump_pipeline)); if (!pipeline) fmdump_fatal("failed to allocate memory"); for (i = 0; i < n_ifiles; i++) pipeline[i].pl_logpath = ifiles[i]; } else { pipeline = calloc(sizeof (logtypes) / sizeof (logtypes[0]), sizeof (struct fmdump_pipeline)); if (!pipeline) fmdump_fatal("failed to allocate memory"); do_disables(); npipe = 0; for (i = 0; i < sizeof (logtypes) / sizeof (logtypes[0]); i++) { struct fmdump_logtype *ltp = &logtypes[i]; char *logpath; if (ltp->lt_enabled == B_FALSE) continue; if ((logpath = malloc(PATH_MAX)) == NULL) fmdump_fatal("failed to allocate memory"); (void) snprintf(logpath, PATH_MAX, "%s/var/fm/fmd/%s", g_root ? g_root : "", ltp->lt_logname); pipeline[npipe].pl_rotated = get_rotated_logs(logpath); pipeline[npipe++].pl_logpath = logpath; } } if (opt_V) fmt = opt_p ? FMDUMP_PRETTY : opt_j ? FMDUMP_JSON : FMDUMP_VERB2; else if (opt_v) fmt = FMDUMP_VERB1; else fmt = FMDUMP_SHORT; bzero(&srlzer, sizeof (srlzer)); srlzer.ds_pipearr = pipeline; srlzer.ds_pipecnt = npipe; srlzer.ds_slot = calloc(npipe, sizeof (struct fmdump_srlzer_slot)); if (!srlzer.ds_slot) fmdump_fatal("failed to allocate memory"); (void) pthread_mutex_init(&srlzer.ds_lock, NULL); for (i = 0, pl = &pipeline[0]; i < npipe; i++, pl++) { (void) pthread_mutex_init(&pl->pl_lock, NULL); (void) pthread_cond_init(&pl->pl_cv, NULL); srlzer.ds_slot[i].ss_state = FMDUMP_PIPE_PROCESSING; pl->pl_srlzer = &srlzer; pl->pl_srlzeridx = i; pl->pl_follow = opt_f ? B_TRUE : B_FALSE; pl->pl_fmt = fmt; pl->pl_arg.da_fv = fv; pl->pl_arg.da_fc = fc; pl->pl_arg.da_fp = stdout; (void) pthread_mutex_lock(&pl->pl_lock); if (pthread_create(&pl->pl_thr, NULL, pipeline_thr, (void *)pl) != 0) fmdump_fatal("pthread_create for pipeline %d failed", i); } for (i = 0, pl = &pipeline[0]; i < npipe; i++, pl++) { while (!pl->pl_started) (void) pthread_cond_wait(&pl->pl_cv, &pl->pl_lock); (void) pthread_mutex_unlock(&pl->pl_lock); } for (i = 0, pl = &pipeline[0]; i < npipe; i++, pl++) (void) pthread_join(pl->pl_thr, NULL); if (ifiles == NULL) { for (i = 0; i < npipe; i++) free(pipeline[i].pl_logpath); } free(srlzer.ds_slot); free(pipeline); return (FMDUMP_EXIT_SUCCESS); } static void cleanup(char **ifiles, int n_ifiles) { int i; if (ifiles == NULL) return; for (i = 0; i < n_ifiles; i++) { if (ifiles[i] != NULL) { free(ifiles[i]); ifiles[i] = NULL; } } free(ifiles); } int main(int argc, char *argv[]) { int opt_a = 0, opt_e = 0, opt_f = 0, opt_H = 0, opt_m = 0, opt_p = 0; int opt_u = 0, opt_v = 0, opt_V = 0, opt_j = 0; int opt_i = 0, opt_I = 0; int opt_A = 0; char **ifiles = NULL; char *ifile = NULL; int n_ifiles; int ifileidx = 0; int iflags = 0; fmdump_arg_t arg; fmdump_lyr_t lyr; const fmdump_ops_t *ops; fmd_log_filter_t *filtv; uint_t filtc; fmd_log_filter_t *errfv, *fltfv, *allfv; uint_t errfc = 0, fltfc = 0, allfc = 0; fmd_log_header_t log; fmd_log_rec_f *func; void *farg; fmd_log_t *lp; int c, err; off64_t off = 0; ulong_t recs; struct loglink *rotated_logs = NULL, *llp; g_pname = argv[0]; errfv = alloca(sizeof (fmd_log_filter_t) * argc); fltfv = alloca(sizeof (fmd_log_filter_t) * argc); allfv = alloca(sizeof (fmd_log_filter_t) * argc); while (optind < argc) { while ((c = getopt(argc, argv, "Aac:efHiIjmN:n:O:pR:t:T:u:vV")) != EOF) { switch (c) { case 'A': opt_A++; break; case 'a': opt_a++; break; case 'c': errfv[errfc].filt_func = fmd_log_filter_class; errfv[errfc].filt_arg = optarg; allfv[allfc++] = errfv[errfc++]; break; case 'e': if (opt_i) return (usage(stderr)); opt_e++; break; case 'f': opt_f++; break; case 'H': opt_H++; break; case 'i': if (opt_e || opt_I) return (usage(stderr)); opt_i++; break; case 'I': if (opt_e || opt_i) return (usage(stderr)); opt_I++; break; case 'j': if (opt_p) return (usage(stderr)); opt_j++; break; case 'm': opt_m++; break; case 'N': fltfv[fltfc].filt_func = fmd_log_filter_nv_multi; fltfv[fltfc].filt_arg = setupnamevalue_multi(optarg); allfv[allfc++] = fltfv[fltfc++]; break; case 'n': fltfv[fltfc].filt_func = fmd_log_filter_nv; fltfv[fltfc].filt_arg = setupnamevalue(optarg); allfv[allfc++] = fltfv[fltfc++]; break; case 'O': { char *p; errno = 0; off = strtoull(optarg, &p, 16); if (errno != 0 || p == optarg || *p != '\0') { fmdump_usage( "illegal offset format -- %s\n", optarg); } iflags |= FMD_LOG_XITER_OFFS; break; } case 'p': if (opt_j) return (usage(stderr)); opt_p++; break; case 'R': g_root = optarg; break; case 't': errfv[errfc].filt_func = fmd_log_filter_after; errfv[errfc].filt_arg = gettimeopt(optarg); allfv[allfc++] = errfv[errfc++]; break; case 'T': errfv[errfc].filt_func = fmd_log_filter_before; errfv[errfc].filt_arg = gettimeopt(optarg); allfv[allfc++] = errfv[errfc++]; break; case 'u': fltfv[fltfc].filt_func = fmd_log_filter_uuid; fltfv[fltfc].filt_arg = optarg; allfv[allfc++] = fltfv[fltfc++]; opt_u++; opt_a++; /* -u implies -a */ break; case 'v': opt_v++; break; case 'V': opt_V++; break; default: return (usage(stderr)); } } if (opt_A && (opt_e || opt_i || opt_I || opt_m || opt_u)) fmdump_usage("-A excludes all of " "-e, -i, -I, -m and -u\n"); if (optind < argc) { char *dest; if (ifiles == NULL) { n_ifiles = argc - optind; ifiles = calloc(n_ifiles, sizeof (char *)); if (ifiles == NULL) { fmdump_fatal( "failed to allocate memory for " "%d input file%s", n_ifiles, n_ifiles > 1 ? "s" : ""); } } if (ifileidx > 0 && !opt_A) fmdump_usage("illegal argument -- %s\n", argv[optind]); ASSERT(ifileidx < n_ifiles); if ((dest = malloc(PATH_MAX)) == NULL) fmdump_fatal("failed to allocate memory"); (void) strlcpy(dest, argv[optind++], PATH_MAX); ifiles[ifileidx++] = dest; } } /* * It's possible that file arguments were interleaved with options and * option arguments, in which case we allocated space for more file * arguments that we actually got. Adjust as required so that we don't * reference invalid entries. */ n_ifiles = ifileidx; if (opt_A) { int rc; if (!opt_a) { fltfv[fltfc].filt_func = log_filter_silent; fltfv[fltfc].filt_arg = (void *)1; allfv[allfc++] = fltfv[fltfc++]; } rc = aggregate(ifiles, n_ifiles, opt_f, allfv, allfc, opt_v, opt_V, opt_p, opt_j); cleanup(ifiles, n_ifiles); return (rc); } else { if (ifiles == NULL) { if ((ifile = calloc(1, PATH_MAX)) == NULL) fmdump_fatal("failed to allocate memory"); } else { ifile = ifiles[0]; } } if (*ifile == '\0') { const char *pfx, *sfx; if (opt_u || (!opt_e && !opt_i && !opt_I)) { pfx = "flt"; sfx = ""; } else { if (opt_e) { pfx = "err"; sfx = ""; } else { pfx = "info"; sfx = opt_I ? "_hival" : ""; } } (void) snprintf(ifile, PATH_MAX, "%s/var/fm/fmd/%slog%s", g_root ? g_root : "", pfx, sfx); /* * logadm may rotate the logs. When no input file is specified, * we try to dump all the rotated logs as well in the right * order. */ if (!opt_H && off == 0) rotated_logs = get_rotated_logs(ifile); } else if (g_root != NULL) { fmdump_usage("-R option is not appropriate " "when file operand is present\n"); } if ((g_msg = fmd_msg_init(g_root, FMD_MSG_VERSION)) == NULL) fmdump_fatal("failed to initialize libfmd_msg"); if ((lp = fmd_log_open(FMD_LOG_VERSION, ifile, &err)) == NULL) { fmdump_fatal("failed to open %s: %s\n", ifile, fmd_log_errmsg(NULL, err)); } if (opt_H) { fmd_log_header(lp, &log); (void) printf("EXD_CREATOR = %s\n", log.log_creator); (void) printf("EXD_HOSTNAME = %s\n", log.log_hostname); (void) printf("EXD_FMA_LABEL = %s\n", log.log_label); (void) printf("EXD_FMA_VERSION = %s\n", log.log_version); (void) printf("EXD_FMA_OSREL = %s\n", log.log_osrelease); (void) printf("EXD_FMA_OSVER = %s\n", log.log_osversion); (void) printf("EXD_FMA_PLAT = %s\n", log.log_platform); (void) printf("EXD_FMA_UUID = %s\n", log.log_uuid); return (FMDUMP_EXIT_SUCCESS); } if (off != 0 && fmd_log_seek(lp, off) != 0) { fmdump_fatal("failed to seek %s: %s\n", ifile, fmd_log_errmsg(lp, fmd_log_errno(lp))); } if (opt_e && opt_u) ops = &fmdump_err_ops; else if (strcmp(fmd_log_label(lp), fmdump_flt_ops.do_label) == 0) ops = &fmdump_flt_ops; else if (strcmp(fmd_log_label(lp), fmdump_asru_ops.do_label) == 0) ops = &fmdump_asru_ops; else if (strcmp(fmd_log_label(lp), fmdump_info_ops.do_label) == 0) ops = &fmdump_info_ops; else ops = &fmdump_err_ops; if (!opt_a && ops == &fmdump_flt_ops) { fltfv[fltfc].filt_func = log_filter_silent; fltfv[fltfc].filt_arg = NULL; allfv[allfc++] = fltfv[fltfc++]; } if (opt_V) { arg.da_fmt = &ops->do_formats[opt_p ? FMDUMP_PRETTY : opt_j ? FMDUMP_JSON : FMDUMP_VERB2]; iflags |= FMD_LOG_XITER_REFS; } else if (opt_v) { arg.da_fmt = &ops->do_formats[FMDUMP_VERB1]; } else if (opt_m) { arg.da_fmt = &ops->do_formats[FMDUMP_MSG]; } else arg.da_fmt = &ops->do_formats[FMDUMP_SHORT]; if (opt_m && arg.da_fmt->do_func == NULL) { fmdump_usage("-m mode is not supported for " "log of type %s: %s\n", fmd_log_label(lp), ifile); } arg.da_fv = errfv; arg.da_fc = errfc; arg.da_fp = stdout; if (iflags & FMD_LOG_XITER_OFFS) fmdump_printf(arg.da_fp, "%16s ", "OFFSET"); if (arg.da_fmt->do_hdr && !(opt_V && ops == &fmdump_flt_ops)) fmdump_printf(arg.da_fp, "%s\n", arg.da_fmt->do_hdr); if (opt_e && opt_u) { iflags |= FMD_LOG_XITER_REFS; func = xref_iter; farg = &arg; filtc = fltfc; filtv = fltfv; } else { func = arg.da_fmt->do_func; farg = arg.da_fp; filtc = allfc; filtv = allfv; } if (iflags & FMD_LOG_XITER_OFFS) { lyr.dy_func = func; lyr.dy_arg = farg; lyr.dy_fp = arg.da_fp; func = xoff_iter; farg = &lyr; } for (llp = rotated_logs; llp != NULL; llp = llp->next) { fmd_log_t *rlp; if ((rlp = fmd_log_open(FMD_LOG_VERSION, llp->path, &err)) == NULL) { fmdump_warn("failed to open %s: %s\n", llp->path, fmd_log_errmsg(NULL, err)); g_errs++; continue; } recs = 0; if (fmd_log_xiter(rlp, iflags, filtc, filtv, func, error, farg, &recs) != 0) { fmdump_warn("failed to dump %s: %s\n", llp->path, fmd_log_errmsg(rlp, fmd_log_errno(rlp))); g_errs++; } g_recs += recs; fmd_log_close(rlp); } do { recs = 0; if (fmd_log_xiter(lp, iflags, filtc, filtv, func, error, farg, &recs) != 0) { fmdump_warn("failed to dump %s: %s\n", ifile, fmd_log_errmsg(lp, fmd_log_errno(lp))); g_errs++; } g_recs += recs; if (opt_f) (void) sleep(1); } while (opt_f); if (!opt_f && g_recs == 0 && isatty(STDOUT_FILENO)) fmdump_warn("%s is empty\n", ifile); if (g_thp != NULL) topo_close(g_thp); fmd_log_close(lp); fmd_msg_fini(g_msg); if (ifiles == NULL) free(ifile); else cleanup(ifiles, n_ifiles); return (g_errs ? FMDUMP_EXIT_ERROR : FMDUMP_EXIT_SUCCESS); }