1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25
26/*
27 * Copyright (c) 2017, Joyent, Inc. All rights reserved.
28 * Copyright (c) 2012 by Delphix. All rights reserved.
29 */
30
31#include <stdlib.h>
32#include <strings.h>
33#include <errno.h>
34#include <unistd.h>
35#include <limits.h>
36#include <assert.h>
37#include <ctype.h>
38#include <alloca.h>
39#include <dt_impl.h>
40#include <dt_pq.h>
41
42#define	DT_MASK_LO 0x00000000FFFFFFFFULL
43
44/*
45 * We declare this here because (1) we need it and (2) we want to avoid a
46 * dependency on libm in libdtrace.
47 */
48static long double
49dt_fabsl(long double x)
50{
51	if (x < 0)
52		return (-x);
53
54	return (x);
55}
56
57static int
58dt_ndigits(long long val)
59{
60	int rval = 1;
61	long long cmp = 10;
62
63	if (val < 0) {
64		val = val == INT64_MIN ? INT64_MAX : -val;
65		rval++;
66	}
67
68	while (val > cmp && cmp > 0) {
69		rval++;
70		cmp *= 10;
71	}
72
73	return (rval < 4 ? 4 : rval);
74}
75
76/*
77 * 128-bit arithmetic functions needed to support the stddev() aggregating
78 * action.
79 */
80static int
81dt_gt_128(uint64_t *a, uint64_t *b)
82{
83	return (a[1] > b[1] || (a[1] == b[1] && a[0] > b[0]));
84}
85
86static int
87dt_ge_128(uint64_t *a, uint64_t *b)
88{
89	return (a[1] > b[1] || (a[1] == b[1] && a[0] >= b[0]));
90}
91
92static int
93dt_le_128(uint64_t *a, uint64_t *b)
94{
95	return (a[1] < b[1] || (a[1] == b[1] && a[0] <= b[0]));
96}
97
98/*
99 * Shift the 128-bit value in a by b. If b is positive, shift left.
100 * If b is negative, shift right.
101 */
102static void
103dt_shift_128(uint64_t *a, int b)
104{
105	uint64_t mask;
106
107	if (b == 0)
108		return;
109
110	if (b < 0) {
111		b = -b;
112		if (b >= 64) {
113			a[0] = a[1] >> (b - 64);
114			a[1] = 0;
115		} else {
116			a[0] >>= b;
117			mask = 1LL << (64 - b);
118			mask -= 1;
119			a[0] |= ((a[1] & mask) << (64 - b));
120			a[1] >>= b;
121		}
122	} else {
123		if (b >= 64) {
124			a[1] = a[0] << (b - 64);
125			a[0] = 0;
126		} else {
127			a[1] <<= b;
128			mask = a[0] >> (64 - b);
129			a[1] |= mask;
130			a[0] <<= b;
131		}
132	}
133}
134
135static int
136dt_nbits_128(uint64_t *a)
137{
138	int nbits = 0;
139	uint64_t tmp[2];
140	uint64_t zero[2] = { 0, 0 };
141
142	tmp[0] = a[0];
143	tmp[1] = a[1];
144
145	dt_shift_128(tmp, -1);
146	while (dt_gt_128(tmp, zero)) {
147		dt_shift_128(tmp, -1);
148		nbits++;
149	}
150
151	return (nbits);
152}
153
154static void
155dt_subtract_128(uint64_t *minuend, uint64_t *subtrahend, uint64_t *difference)
156{
157	uint64_t result[2];
158
159	result[0] = minuend[0] - subtrahend[0];
160	result[1] = minuend[1] - subtrahend[1] -
161	    (minuend[0] < subtrahend[0] ? 1 : 0);
162
163	difference[0] = result[0];
164	difference[1] = result[1];
165}
166
167static void
168dt_add_128(uint64_t *addend1, uint64_t *addend2, uint64_t *sum)
169{
170	uint64_t result[2];
171
172	result[0] = addend1[0] + addend2[0];
173	result[1] = addend1[1] + addend2[1] +
174	    (result[0] < addend1[0] || result[0] < addend2[0] ? 1 : 0);
175
176	sum[0] = result[0];
177	sum[1] = result[1];
178}
179
180/*
181 * The basic idea is to break the 2 64-bit values into 4 32-bit values,
182 * use native multiplication on those, and then re-combine into the
183 * resulting 128-bit value.
184 *
185 * (hi1 << 32 + lo1) * (hi2 << 32 + lo2) =
186 *     hi1 * hi2 << 64 +
187 *     hi1 * lo2 << 32 +
188 *     hi2 * lo1 << 32 +
189 *     lo1 * lo2
190 */
191static void
192dt_multiply_128(uint64_t factor1, uint64_t factor2, uint64_t *product)
193{
194	uint64_t hi1, hi2, lo1, lo2;
195	uint64_t tmp[2];
196
197	hi1 = factor1 >> 32;
198	hi2 = factor2 >> 32;
199
200	lo1 = factor1 & DT_MASK_LO;
201	lo2 = factor2 & DT_MASK_LO;
202
203	product[0] = lo1 * lo2;
204	product[1] = hi1 * hi2;
205
206	tmp[0] = hi1 * lo2;
207	tmp[1] = 0;
208	dt_shift_128(tmp, 32);
209	dt_add_128(product, tmp, product);
210
211	tmp[0] = hi2 * lo1;
212	tmp[1] = 0;
213	dt_shift_128(tmp, 32);
214	dt_add_128(product, tmp, product);
215}
216
217/*
218 * This is long-hand division.
219 *
220 * We initialize subtrahend by shifting divisor left as far as possible. We
221 * loop, comparing subtrahend to dividend:  if subtrahend is smaller, we
222 * subtract and set the appropriate bit in the result.  We then shift
223 * subtrahend right by one bit for the next comparison.
224 */
225static void
226dt_divide_128(uint64_t *dividend, uint64_t divisor, uint64_t *quotient)
227{
228	uint64_t result[2] = { 0, 0 };
229	uint64_t remainder[2];
230	uint64_t subtrahend[2];
231	uint64_t divisor_128[2];
232	uint64_t mask[2] = { 1, 0 };
233	int log = 0;
234
235	assert(divisor != 0);
236
237	divisor_128[0] = divisor;
238	divisor_128[1] = 0;
239
240	remainder[0] = dividend[0];
241	remainder[1] = dividend[1];
242
243	subtrahend[0] = divisor;
244	subtrahend[1] = 0;
245
246	while (divisor > 0) {
247		log++;
248		divisor >>= 1;
249	}
250
251	dt_shift_128(subtrahend, 128 - log);
252	dt_shift_128(mask, 128 - log);
253
254	while (dt_ge_128(remainder, divisor_128)) {
255		if (dt_ge_128(remainder, subtrahend)) {
256			dt_subtract_128(remainder, subtrahend, remainder);
257			result[0] |= mask[0];
258			result[1] |= mask[1];
259		}
260
261		dt_shift_128(subtrahend, -1);
262		dt_shift_128(mask, -1);
263	}
264
265	quotient[0] = result[0];
266	quotient[1] = result[1];
267}
268
269/*
270 * This is the long-hand method of calculating a square root.
271 * The algorithm is as follows:
272 *
273 * 1. Group the digits by 2 from the right.
274 * 2. Over the leftmost group, find the largest single-digit number
275 *    whose square is less than that group.
276 * 3. Subtract the result of the previous step (2 or 4, depending) and
277 *    bring down the next two-digit group.
278 * 4. For the result R we have so far, find the largest single-digit number
279 *    x such that 2 * R * 10 * x + x^2 is less than the result from step 3.
280 *    (Note that this is doubling R and performing a decimal left-shift by 1
281 *    and searching for the appropriate decimal to fill the one's place.)
282 *    The value x is the next digit in the square root.
283 * Repeat steps 3 and 4 until the desired precision is reached.  (We're
284 * dealing with integers, so the above is sufficient.)
285 *
286 * In decimal, the square root of 582,734 would be calculated as so:
287 *
288 *     __7__6__3
289 *    | 58 27 34
290 *     -49       (7^2 == 49 => 7 is the first digit in the square root)
291 *      --
292 *       9 27    (Subtract and bring down the next group.)
293 * 146   8 76    (2 * 7 * 10 * 6 + 6^2 == 876 => 6 is the next digit in
294 *      -----     the square root)
295 *         51 34 (Subtract and bring down the next group.)
296 * 1523    45 69 (2 * 76 * 10 * 3 + 3^2 == 4569 => 3 is the next digit in
297 *         -----  the square root)
298 *          5 65 (remainder)
299 *
300 * The above algorithm applies similarly in binary, but note that the
301 * only possible non-zero value for x in step 4 is 1, so step 4 becomes a
302 * simple decision: is 2 * R * 2 * 1 + 1^2 (aka R << 2 + 1) less than the
303 * preceding difference?
304 *
305 * In binary, the square root of 11011011 would be calculated as so:
306 *
307 *     __1__1__1__0
308 *    | 11 01 10 11
309 *      01          (0 << 2 + 1 == 1 < 11 => this bit is 1)
310 *      --
311 *      10 01 10 11
312 * 101   1 01       (1 << 2 + 1 == 101 < 1001 => next bit is 1)
313 *      -----
314 *       1 00 10 11
315 * 1101    11 01    (11 << 2 + 1 == 1101 < 10010 => next bit is 1)
316 *       -------
317 *          1 01 11
318 * 11101    1 11 01 (111 << 2 + 1 == 11101 > 10111 => last bit is 0)
319 *
320 */
321static uint64_t
322dt_sqrt_128(uint64_t *square)
323{
324	uint64_t result[2] = { 0, 0 };
325	uint64_t diff[2] = { 0, 0 };
326	uint64_t one[2] = { 1, 0 };
327	uint64_t next_pair[2];
328	uint64_t next_try[2];
329	uint64_t bit_pairs, pair_shift;
330	int i;
331
332	bit_pairs = dt_nbits_128(square) / 2;
333	pair_shift = bit_pairs * 2;
334
335	for (i = 0; i <= bit_pairs; i++) {
336		/*
337		 * Bring down the next pair of bits.
338		 */
339		next_pair[0] = square[0];
340		next_pair[1] = square[1];
341		dt_shift_128(next_pair, -pair_shift);
342		next_pair[0] &= 0x3;
343		next_pair[1] = 0;
344
345		dt_shift_128(diff, 2);
346		dt_add_128(diff, next_pair, diff);
347
348		/*
349		 * next_try = R << 2 + 1
350		 */
351		next_try[0] = result[0];
352		next_try[1] = result[1];
353		dt_shift_128(next_try, 2);
354		dt_add_128(next_try, one, next_try);
355
356		if (dt_le_128(next_try, diff)) {
357			dt_subtract_128(diff, next_try, diff);
358			dt_shift_128(result, 1);
359			dt_add_128(result, one, result);
360		} else {
361			dt_shift_128(result, 1);
362		}
363
364		pair_shift -= 2;
365	}
366
367	assert(result[1] == 0);
368
369	return (result[0]);
370}
371
372uint64_t
373dt_stddev(uint64_t *data, uint64_t normal)
374{
375	uint64_t avg_of_squares[2];
376	uint64_t square_of_avg[2];
377	int64_t norm_avg;
378	uint64_t diff[2];
379
380	/*
381	 * The standard approximation for standard deviation is
382	 * sqrt(average(x**2) - average(x)**2), i.e. the square root
383	 * of the average of the squares minus the square of the average.
384	 * When normalizing, we should divide the sum of x**2 by normal**2.
385	 */
386	dt_divide_128(data + 2, normal, avg_of_squares);
387	dt_divide_128(avg_of_squares, normal, avg_of_squares);
388	dt_divide_128(avg_of_squares, data[0], avg_of_squares);
389
390	norm_avg = (int64_t)data[1] / (int64_t)normal / (int64_t)data[0];
391
392	if (norm_avg < 0)
393		norm_avg = -norm_avg;
394
395	dt_multiply_128((uint64_t)norm_avg, (uint64_t)norm_avg, square_of_avg);
396
397	dt_subtract_128(avg_of_squares, square_of_avg, diff);
398
399	return (dt_sqrt_128(diff));
400}
401
402static int
403dt_flowindent(dtrace_hdl_t *dtp, dtrace_probedata_t *data, dtrace_epid_t last,
404    dtrace_bufdesc_t *buf, size_t offs)
405{
406	dtrace_probedesc_t *pd = data->dtpda_pdesc, *npd;
407	dtrace_eprobedesc_t *epd = data->dtpda_edesc, *nepd;
408	char *p = pd->dtpd_provider, *n = pd->dtpd_name, *sub;
409	dtrace_flowkind_t flow = DTRACEFLOW_NONE;
410	const char *str = NULL;
411	static const char *e_str[2] = { " -> ", " => " };
412	static const char *r_str[2] = { " <- ", " <= " };
413	static const char *ent = "entry", *ret = "return";
414	static int entlen = 0, retlen = 0;
415	dtrace_epid_t next, id = epd->dtepd_epid;
416	int rval;
417
418	if (entlen == 0) {
419		assert(retlen == 0);
420		entlen = strlen(ent);
421		retlen = strlen(ret);
422	}
423
424	/*
425	 * If the name of the probe is "entry" or ends with "-entry", we
426	 * treat it as an entry; if it is "return" or ends with "-return",
427	 * we treat it as a return.  (This allows application-provided probes
428	 * like "method-entry" or "function-entry" to participate in flow
429	 * indentation -- without accidentally misinterpreting popular probe
430	 * names like "carpentry", "gentry" or "Coventry".)
431	 */
432	if ((sub = strstr(n, ent)) != NULL && sub[entlen] == '\0' &&
433	    (sub == n || sub[-1] == '-')) {
434		flow = DTRACEFLOW_ENTRY;
435		str = e_str[strcmp(p, "syscall") == 0];
436	} else if ((sub = strstr(n, ret)) != NULL && sub[retlen] == '\0' &&
437	    (sub == n || sub[-1] == '-')) {
438		flow = DTRACEFLOW_RETURN;
439		str = r_str[strcmp(p, "syscall") == 0];
440	}
441
442	/*
443	 * If we're going to indent this, we need to check the ID of our last
444	 * call.  If we're looking at the same probe ID but a different EPID,
445	 * we _don't_ want to indent.  (Yes, there are some minor holes in
446	 * this scheme -- it's a heuristic.)
447	 */
448	if (flow == DTRACEFLOW_ENTRY) {
449		if ((last != DTRACE_EPIDNONE && id != last &&
450		    pd->dtpd_id == dtp->dt_pdesc[last]->dtpd_id))
451			flow = DTRACEFLOW_NONE;
452	}
453
454	/*
455	 * If we're going to unindent this, it's more difficult to see if
456	 * we don't actually want to unindent it -- we need to look at the
457	 * _next_ EPID.
458	 */
459	if (flow == DTRACEFLOW_RETURN) {
460		offs += epd->dtepd_size;
461
462		do {
463			if (offs >= buf->dtbd_size)
464				goto out;
465
466			next = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
467
468			if (next == DTRACE_EPIDNONE)
469				offs += sizeof (id);
470		} while (next == DTRACE_EPIDNONE);
471
472		if ((rval = dt_epid_lookup(dtp, next, &nepd, &npd)) != 0)
473			return (rval);
474
475		if (next != id && npd->dtpd_id == pd->dtpd_id)
476			flow = DTRACEFLOW_NONE;
477	}
478
479out:
480	if (flow == DTRACEFLOW_ENTRY || flow == DTRACEFLOW_RETURN) {
481		data->dtpda_prefix = str;
482	} else {
483		data->dtpda_prefix = "| ";
484	}
485
486	if (flow == DTRACEFLOW_RETURN && data->dtpda_indent > 0)
487		data->dtpda_indent -= 2;
488
489	data->dtpda_flow = flow;
490
491	return (0);
492}
493
494static int
495dt_nullprobe()
496{
497	return (DTRACE_CONSUME_THIS);
498}
499
500static int
501dt_nullrec()
502{
503	return (DTRACE_CONSUME_NEXT);
504}
505
506static void
507dt_quantize_total(dtrace_hdl_t *dtp, int64_t datum, long double *total)
508{
509	long double val = dt_fabsl((long double)datum);
510
511	if (dtp->dt_options[DTRACEOPT_AGGZOOM] == DTRACEOPT_UNSET) {
512		*total += val;
513		return;
514	}
515
516	/*
517	 * If we're zooming in on an aggregation, we want the height of the
518	 * highest value to be approximately 95% of total bar height -- so we
519	 * adjust up by the reciprocal of DTRACE_AGGZOOM_MAX when comparing to
520	 * our highest value.
521	 */
522	val *= 1 / DTRACE_AGGZOOM_MAX;
523
524	if (*total < val)
525		*total = val;
526}
527
528static int
529dt_print_quanthdr(dtrace_hdl_t *dtp, FILE *fp, int width)
530{
531	return (dt_printf(dtp, fp, "\n%*s %41s %-9s\n",
532	    width ? width : 16, width ? "key" : "value",
533	    "------------- Distribution -------------", "count"));
534}
535
536static int
537dt_print_quanthdr_packed(dtrace_hdl_t *dtp, FILE *fp, int width,
538    const dtrace_aggdata_t *aggdata, dtrace_actkind_t action)
539{
540	int min = aggdata->dtada_minbin, max = aggdata->dtada_maxbin;
541	int minwidth, maxwidth, i;
542
543	assert(action == DTRACEAGG_QUANTIZE || action == DTRACEAGG_LQUANTIZE);
544
545	if (action == DTRACEAGG_QUANTIZE) {
546		if (min != 0 && min != DTRACE_QUANTIZE_ZEROBUCKET)
547			min--;
548
549		if (max < DTRACE_QUANTIZE_NBUCKETS - 1)
550			max++;
551
552		minwidth = dt_ndigits(DTRACE_QUANTIZE_BUCKETVAL(min));
553		maxwidth = dt_ndigits(DTRACE_QUANTIZE_BUCKETVAL(max));
554	} else {
555		maxwidth = 8;
556		minwidth = maxwidth - 1;
557		max++;
558	}
559
560	if (dt_printf(dtp, fp, "\n%*s %*s .",
561	    width, width > 0 ? "key" : "", minwidth, "min") < 0)
562		return (-1);
563
564	for (i = min; i <= max; i++) {
565		if (dt_printf(dtp, fp, "-") < 0)
566			return (-1);
567	}
568
569	return (dt_printf(dtp, fp, ". %*s | count\n", -maxwidth, "max"));
570}
571
572/*
573 * We use a subset of the Unicode Block Elements (U+2588 through U+258F,
574 * inclusive) to represent aggregations via UTF-8 -- which are expressed via
575 * 3-byte UTF-8 sequences.
576 */
577#define	DTRACE_AGGUTF8_FULL	0x2588
578#define	DTRACE_AGGUTF8_BASE	0x258f
579#define	DTRACE_AGGUTF8_LEVELS	8
580
581#define	DTRACE_AGGUTF8_BYTE0(val)	(0xe0 | ((val) >> 12))
582#define	DTRACE_AGGUTF8_BYTE1(val)	(0x80 | (((val) >> 6) & 0x3f))
583#define	DTRACE_AGGUTF8_BYTE2(val)	(0x80 | ((val) & 0x3f))
584
585static int
586dt_print_quantline_utf8(dtrace_hdl_t *dtp, FILE *fp, int64_t val,
587    uint64_t normal, long double total)
588{
589	uint_t len = 40, i, whole, partial;
590	long double f = (dt_fabsl((long double)val) * len) / total;
591	const char *spaces = "                                        ";
592
593	whole = (uint_t)f;
594	partial = (uint_t)((f - (long double)(uint_t)f) *
595	    (long double)DTRACE_AGGUTF8_LEVELS);
596
597	if (dt_printf(dtp, fp, "|") < 0)
598		return (-1);
599
600	for (i = 0; i < whole; i++) {
601		if (dt_printf(dtp, fp, "%c%c%c",
602		    DTRACE_AGGUTF8_BYTE0(DTRACE_AGGUTF8_FULL),
603		    DTRACE_AGGUTF8_BYTE1(DTRACE_AGGUTF8_FULL),
604		    DTRACE_AGGUTF8_BYTE2(DTRACE_AGGUTF8_FULL)) < 0)
605			return (-1);
606	}
607
608	if (partial != 0) {
609		partial = DTRACE_AGGUTF8_BASE - (partial - 1);
610
611		if (dt_printf(dtp, fp, "%c%c%c",
612		    DTRACE_AGGUTF8_BYTE0(partial),
613		    DTRACE_AGGUTF8_BYTE1(partial),
614		    DTRACE_AGGUTF8_BYTE2(partial)) < 0)
615			return (-1);
616
617		i++;
618	}
619
620	return (dt_printf(dtp, fp, "%s %-9lld\n", spaces + i,
621	    (long long)val / normal));
622}
623
624static int
625dt_print_quantline(dtrace_hdl_t *dtp, FILE *fp, int64_t val,
626    uint64_t normal, long double total, char positives, char negatives)
627{
628	long double f;
629	uint_t depth, len = 40;
630
631	const char *ats = "@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@";
632	const char *spaces = "                                        ";
633
634	assert(strlen(ats) == len && strlen(spaces) == len);
635	assert(!(total == 0 && (positives || negatives)));
636	assert(!(val < 0 && !negatives));
637	assert(!(val > 0 && !positives));
638	assert(!(val != 0 && total == 0));
639
640	if (!negatives) {
641		if (positives) {
642			if (dtp->dt_encoding == DT_ENCODING_UTF8) {
643				return (dt_print_quantline_utf8(dtp, fp, val,
644				    normal, total));
645			}
646
647			f = (dt_fabsl((long double)val) * len) / total;
648			depth = (uint_t)(f + 0.5);
649		} else {
650			depth = 0;
651		}
652
653		return (dt_printf(dtp, fp, "|%s%s %-9lld\n", ats + len - depth,
654		    spaces + depth, (long long)val / normal));
655	}
656
657	if (!positives) {
658		f = (dt_fabsl((long double)val) * len) / total;
659		depth = (uint_t)(f + 0.5);
660
661		return (dt_printf(dtp, fp, "%s%s| %-9lld\n", spaces + depth,
662		    ats + len - depth, (long long)val / normal));
663	}
664
665	/*
666	 * If we're here, we have both positive and negative bucket values.
667	 * To express this graphically, we're going to generate both positive
668	 * and negative bars separated by a centerline.  These bars are half
669	 * the size of normal quantize()/lquantize() bars, so we divide the
670	 * length in half before calculating the bar length.
671	 */
672	len /= 2;
673	ats = &ats[len];
674	spaces = &spaces[len];
675
676	f = (dt_fabsl((long double)val) * len) / total;
677	depth = (uint_t)(f + 0.5);
678
679	if (val <= 0) {
680		return (dt_printf(dtp, fp, "%s%s|%*s %-9lld\n", spaces + depth,
681		    ats + len - depth, len, "", (long long)val / normal));
682	} else {
683		return (dt_printf(dtp, fp, "%20s|%s%s %-9lld\n", "",
684		    ats + len - depth, spaces + depth,
685		    (long long)val / normal));
686	}
687}
688
689/*
690 * As with UTF-8 printing of aggregations, we use a subset of the Unicode
691 * Block Elements (U+2581 through U+2588, inclusive) to represent our packed
692 * aggregation.
693 */
694#define	DTRACE_AGGPACK_BASE	0x2581
695#define	DTRACE_AGGPACK_LEVELS	8
696
697static int
698dt_print_packed(dtrace_hdl_t *dtp, FILE *fp,
699    long double datum, long double total)
700{
701	static boolean_t utf8_checked = B_FALSE;
702	static boolean_t utf8;
703	char *ascii = "__xxxxXX";
704	char *neg = "vvvvVV";
705	unsigned int len;
706	long double val;
707
708	if (!utf8_checked) {
709		char *term;
710
711		/*
712		 * We want to determine if we can reasonably emit UTF-8 for our
713		 * packed aggregation.  To do this, we will check for terminals
714		 * that are known to be primitive to emit UTF-8 on these.
715		 */
716		utf8_checked = B_TRUE;
717
718		if (dtp->dt_encoding == DT_ENCODING_ASCII) {
719			utf8 = B_FALSE;
720		} else if (dtp->dt_encoding == DT_ENCODING_UTF8) {
721			utf8 = B_TRUE;
722		} else if ((term = getenv("TERM")) != NULL &&
723		    (strcmp(term, "sun") == 0 ||
724		    strcmp(term, "sun-color") == 0) ||
725		    strcmp(term, "dumb") == 0) {
726			utf8 = B_FALSE;
727		} else {
728			utf8 = B_TRUE;
729		}
730	}
731
732	if (datum == 0)
733		return (dt_printf(dtp, fp, " "));
734
735	if (datum < 0) {
736		len = strlen(neg);
737		val = dt_fabsl(datum * (len - 1)) / total;
738		return (dt_printf(dtp, fp, "%c", neg[(uint_t)(val + 0.5)]));
739	}
740
741	if (utf8) {
742		int block = DTRACE_AGGPACK_BASE + (unsigned int)(((datum *
743		    (DTRACE_AGGPACK_LEVELS - 1)) / total) + 0.5);
744
745		return (dt_printf(dtp, fp, "%c%c%c",
746		    DTRACE_AGGUTF8_BYTE0(block),
747		    DTRACE_AGGUTF8_BYTE1(block),
748		    DTRACE_AGGUTF8_BYTE2(block)));
749	}
750
751	len = strlen(ascii);
752	val = (datum * (len - 1)) / total;
753	return (dt_printf(dtp, fp, "%c", ascii[(uint_t)(val + 0.5)]));
754}
755
756int
757dt_print_quantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
758    size_t size, uint64_t normal)
759{
760	const int64_t *data = addr;
761	int i, first_bin = 0, last_bin = DTRACE_QUANTIZE_NBUCKETS - 1;
762	long double total = 0;
763	char positives = 0, negatives = 0;
764
765	if (size != DTRACE_QUANTIZE_NBUCKETS * sizeof (uint64_t))
766		return (dt_set_errno(dtp, EDT_DMISMATCH));
767
768	while (first_bin < DTRACE_QUANTIZE_NBUCKETS - 1 && data[first_bin] == 0)
769		first_bin++;
770
771	if (first_bin == DTRACE_QUANTIZE_NBUCKETS - 1) {
772		/*
773		 * There isn't any data.  This is possible if the aggregation
774		 * has been clear()'d or if negative increment values have been
775		 * used.  Regardless, we'll print the buckets around 0.
776		 */
777		first_bin = DTRACE_QUANTIZE_ZEROBUCKET - 1;
778		last_bin = DTRACE_QUANTIZE_ZEROBUCKET + 1;
779	} else {
780		if (first_bin > 0)
781			first_bin--;
782
783		while (last_bin > 0 && data[last_bin] == 0)
784			last_bin--;
785
786		if (last_bin < DTRACE_QUANTIZE_NBUCKETS - 1)
787			last_bin++;
788	}
789
790	for (i = first_bin; i <= last_bin; i++) {
791		positives |= (data[i] > 0);
792		negatives |= (data[i] < 0);
793		dt_quantize_total(dtp, data[i], &total);
794	}
795
796	if (dt_print_quanthdr(dtp, fp, 0) < 0)
797		return (-1);
798
799	for (i = first_bin; i <= last_bin; i++) {
800		if (dt_printf(dtp, fp, "%16lld ",
801		    (long long)DTRACE_QUANTIZE_BUCKETVAL(i)) < 0)
802			return (-1);
803
804		if (dt_print_quantline(dtp, fp, data[i], normal, total,
805		    positives, negatives) < 0)
806			return (-1);
807	}
808
809	return (0);
810}
811
812int
813dt_print_quantize_packed(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
814    size_t size, const dtrace_aggdata_t *aggdata)
815{
816	const int64_t *data = addr;
817	long double total = 0, count = 0;
818	int min = aggdata->dtada_minbin, max = aggdata->dtada_maxbin, i;
819	int64_t minval, maxval;
820
821	if (size != DTRACE_QUANTIZE_NBUCKETS * sizeof (uint64_t))
822		return (dt_set_errno(dtp, EDT_DMISMATCH));
823
824	if (min != 0 && min != DTRACE_QUANTIZE_ZEROBUCKET)
825		min--;
826
827	if (max < DTRACE_QUANTIZE_NBUCKETS - 1)
828		max++;
829
830	minval = DTRACE_QUANTIZE_BUCKETVAL(min);
831	maxval = DTRACE_QUANTIZE_BUCKETVAL(max);
832
833	if (dt_printf(dtp, fp, " %*lld :", dt_ndigits(minval),
834	    (long long)minval) < 0)
835		return (-1);
836
837	for (i = min; i <= max; i++) {
838		dt_quantize_total(dtp, data[i], &total);
839		count += data[i];
840	}
841
842	for (i = min; i <= max; i++) {
843		if (dt_print_packed(dtp, fp, data[i], total) < 0)
844			return (-1);
845	}
846
847	if (dt_printf(dtp, fp, ": %*lld | %lld\n",
848	    -dt_ndigits(maxval), (long long)maxval, (long long)count) < 0)
849		return (-1);
850
851	return (0);
852}
853
854int
855dt_print_lquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
856    size_t size, uint64_t normal)
857{
858	const int64_t *data = addr;
859	int i, first_bin, last_bin, base;
860	uint64_t arg;
861	long double total = 0;
862	uint16_t step, levels;
863	char positives = 0, negatives = 0;
864
865	if (size < sizeof (uint64_t))
866		return (dt_set_errno(dtp, EDT_DMISMATCH));
867
868	arg = *data++;
869	size -= sizeof (uint64_t);
870
871	base = DTRACE_LQUANTIZE_BASE(arg);
872	step = DTRACE_LQUANTIZE_STEP(arg);
873	levels = DTRACE_LQUANTIZE_LEVELS(arg);
874
875	first_bin = 0;
876	last_bin = levels + 1;
877
878	if (size != sizeof (uint64_t) * (levels + 2))
879		return (dt_set_errno(dtp, EDT_DMISMATCH));
880
881	while (first_bin <= levels + 1 && data[first_bin] == 0)
882		first_bin++;
883
884	if (first_bin > levels + 1) {
885		first_bin = 0;
886		last_bin = 2;
887	} else {
888		if (first_bin > 0)
889			first_bin--;
890
891		while (last_bin > 0 && data[last_bin] == 0)
892			last_bin--;
893
894		if (last_bin < levels + 1)
895			last_bin++;
896	}
897
898	for (i = first_bin; i <= last_bin; i++) {
899		positives |= (data[i] > 0);
900		negatives |= (data[i] < 0);
901		dt_quantize_total(dtp, data[i], &total);
902	}
903
904	if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
905	    "------------- Distribution -------------", "count") < 0)
906		return (-1);
907
908	for (i = first_bin; i <= last_bin; i++) {
909		char c[32];
910		int err;
911
912		if (i == 0) {
913			(void) snprintf(c, sizeof (c), "< %d", base);
914			err = dt_printf(dtp, fp, "%16s ", c);
915		} else if (i == levels + 1) {
916			(void) snprintf(c, sizeof (c), ">= %d",
917			    base + (levels * step));
918			err = dt_printf(dtp, fp, "%16s ", c);
919		} else {
920			err = dt_printf(dtp, fp, "%16d ",
921			    base + (i - 1) * step);
922		}
923
924		if (err < 0 || dt_print_quantline(dtp, fp, data[i], normal,
925		    total, positives, negatives) < 0)
926			return (-1);
927	}
928
929	return (0);
930}
931
932/*ARGSUSED*/
933int
934dt_print_lquantize_packed(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
935    size_t size, const dtrace_aggdata_t *aggdata)
936{
937	const int64_t *data = addr;
938	long double total = 0, count = 0;
939	int min, max, base, err;
940	uint64_t arg;
941	uint16_t step, levels;
942	char c[32];
943	unsigned int i;
944
945	if (size < sizeof (uint64_t))
946		return (dt_set_errno(dtp, EDT_DMISMATCH));
947
948	arg = *data++;
949	size -= sizeof (uint64_t);
950
951	base = DTRACE_LQUANTIZE_BASE(arg);
952	step = DTRACE_LQUANTIZE_STEP(arg);
953	levels = DTRACE_LQUANTIZE_LEVELS(arg);
954
955	if (size != sizeof (uint64_t) * (levels + 2))
956		return (dt_set_errno(dtp, EDT_DMISMATCH));
957
958	min = 0;
959	max = levels + 1;
960
961	if (min == 0) {
962		(void) snprintf(c, sizeof (c), "< %d", base);
963		err = dt_printf(dtp, fp, "%8s :", c);
964	} else {
965		err = dt_printf(dtp, fp, "%8d :", base + (min - 1) * step);
966	}
967
968	if (err < 0)
969		return (-1);
970
971	for (i = min; i <= max; i++) {
972		dt_quantize_total(dtp, data[i], &total);
973		count += data[i];
974	}
975
976	for (i = min; i <= max; i++) {
977		if (dt_print_packed(dtp, fp, data[i], total) < 0)
978			return (-1);
979	}
980
981	(void) snprintf(c, sizeof (c), ">= %d", base + (levels * step));
982	return (dt_printf(dtp, fp, ": %-8s | %lld\n", c, (long long)count));
983}
984
985int
986dt_print_llquantize(dtrace_hdl_t *dtp, FILE *fp, const void *addr,
987    size_t size, uint64_t normal)
988{
989	int i, first_bin, last_bin, bin = 1, order, levels;
990	uint16_t factor, low, high, nsteps;
991	const int64_t *data = addr;
992	int64_t value = 1, next, step;
993	char positives = 0, negatives = 0;
994	long double total = 0;
995	uint64_t arg;
996	char c[32];
997
998	if (size < sizeof (uint64_t))
999		return (dt_set_errno(dtp, EDT_DMISMATCH));
1000
1001	arg = *data++;
1002	size -= sizeof (uint64_t);
1003
1004	factor = DTRACE_LLQUANTIZE_FACTOR(arg);
1005	low = DTRACE_LLQUANTIZE_LOW(arg);
1006	high = DTRACE_LLQUANTIZE_HIGH(arg);
1007	nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
1008
1009	/*
1010	 * We don't expect to be handed invalid llquantize() parameters here,
1011	 * but sanity check them (to a degree) nonetheless.
1012	 */
1013	if (size > INT32_MAX || factor < 2 || low >= high ||
1014	    nsteps == 0 || factor > nsteps)
1015		return (dt_set_errno(dtp, EDT_DMISMATCH));
1016
1017	levels = (int)size / sizeof (uint64_t);
1018
1019	first_bin = 0;
1020	last_bin = levels - 1;
1021
1022	while (first_bin < levels && data[first_bin] == 0)
1023		first_bin++;
1024
1025	if (first_bin == levels) {
1026		first_bin = 0;
1027		last_bin = 1;
1028	} else {
1029		if (first_bin > 0)
1030			first_bin--;
1031
1032		while (last_bin > 0 && data[last_bin] == 0)
1033			last_bin--;
1034
1035		if (last_bin < levels - 1)
1036			last_bin++;
1037	}
1038
1039	for (i = first_bin; i <= last_bin; i++) {
1040		positives |= (data[i] > 0);
1041		negatives |= (data[i] < 0);
1042		dt_quantize_total(dtp, data[i], &total);
1043	}
1044
1045	if (dt_printf(dtp, fp, "\n%16s %41s %-9s\n", "value",
1046	    "------------- Distribution -------------", "count") < 0)
1047		return (-1);
1048
1049	for (order = 0; order < low; order++)
1050		value *= factor;
1051
1052	next = value * factor;
1053	step = next > nsteps ? next / nsteps : 1;
1054
1055	if (first_bin == 0) {
1056		(void) snprintf(c, sizeof (c), "< %lld", value);
1057
1058		if (dt_printf(dtp, fp, "%16s ", c) < 0)
1059			return (-1);
1060
1061		if (dt_print_quantline(dtp, fp, data[0], normal,
1062		    total, positives, negatives) < 0)
1063			return (-1);
1064	}
1065
1066	while (order <= high) {
1067		if (bin >= first_bin && bin <= last_bin) {
1068			if (dt_printf(dtp, fp, "%16lld ", (long long)value) < 0)
1069				return (-1);
1070
1071			if (dt_print_quantline(dtp, fp, data[bin],
1072			    normal, total, positives, negatives) < 0)
1073				return (-1);
1074		}
1075
1076		assert(value < next);
1077		bin++;
1078
1079		if ((value += step) != next)
1080			continue;
1081
1082		next = value * factor;
1083		step = next > nsteps ? next / nsteps : 1;
1084		order++;
1085	}
1086
1087	if (last_bin < bin)
1088		return (0);
1089
1090	assert(last_bin == bin);
1091	(void) snprintf(c, sizeof (c), ">= %lld", value);
1092
1093	if (dt_printf(dtp, fp, "%16s ", c) < 0)
1094		return (-1);
1095
1096	return (dt_print_quantline(dtp, fp, data[bin], normal,
1097	    total, positives, negatives));
1098}
1099
1100/*ARGSUSED*/
1101static int
1102dt_print_average(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
1103    size_t size, uint64_t normal)
1104{
1105	/* LINTED - alignment */
1106	int64_t *data = (int64_t *)addr;
1107
1108	return (dt_printf(dtp, fp, " %16lld", data[0] ?
1109	    (long long)(data[1] / (int64_t)normal / data[0]) : 0));
1110}
1111
1112/*ARGSUSED*/
1113static int
1114dt_print_stddev(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
1115    size_t size, uint64_t normal)
1116{
1117	/* LINTED - alignment */
1118	uint64_t *data = (uint64_t *)addr;
1119
1120	return (dt_printf(dtp, fp, " %16llu", data[0] ?
1121	    (unsigned long long) dt_stddev(data, normal) : 0));
1122}
1123
1124/*ARGSUSED*/
1125static int
1126dt_print_bytes(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr,
1127    size_t nbytes, int width, int quiet, int forceraw)
1128{
1129	/*
1130	 * If the byte stream is a series of printable characters, followed by
1131	 * a terminating byte, we print it out as a string.  Otherwise, we
1132	 * assume that it's something else and just print the bytes.
1133	 */
1134	int i, j, margin = 5;
1135	char *c = (char *)addr;
1136
1137	if (nbytes == 0)
1138		return (0);
1139
1140	if (forceraw)
1141		goto raw;
1142
1143	if (dtp->dt_options[DTRACEOPT_RAWBYTES] != DTRACEOPT_UNSET)
1144		goto raw;
1145
1146	for (i = 0; i < nbytes; i++) {
1147		/*
1148		 * We define a "printable character" to be one for which
1149		 * isprint(3C) returns non-zero, isspace(3C) returns non-zero,
1150		 * or a character which is either backspace or the bell.
1151		 * Backspace and the bell are regrettably special because
1152		 * they fail the first two tests -- and yet they are entirely
1153		 * printable.  These are the only two control characters that
1154		 * have meaning for the terminal and for which isprint(3C) and
1155		 * isspace(3C) return 0.
1156		 */
1157		if (isprint(c[i]) || isspace(c[i]) ||
1158		    c[i] == '\b' || c[i] == '\a')
1159			continue;
1160
1161		if (c[i] == '\0' && i > 0) {
1162			/*
1163			 * This looks like it might be a string.  Before we
1164			 * assume that it is indeed a string, check the
1165			 * remainder of the byte range; if it contains
1166			 * additional non-nul characters, we'll assume that
1167			 * it's a binary stream that just happens to look like
1168			 * a string, and we'll print out the individual bytes.
1169			 */
1170			for (j = i + 1; j < nbytes; j++) {
1171				if (c[j] != '\0')
1172					break;
1173			}
1174
1175			if (j != nbytes)
1176				break;
1177
1178			if (quiet) {
1179				return (dt_printf(dtp, fp, "%s", c));
1180			} else {
1181				return (dt_printf(dtp, fp, " %s%*s",
1182				    width < 0 ? " " : "", width, c));
1183			}
1184		}
1185
1186		break;
1187	}
1188
1189	if (i == nbytes) {
1190		/*
1191		 * The byte range is all printable characters, but there is
1192		 * no trailing nul byte.  We'll assume that it's a string and
1193		 * print it as such.
1194		 */
1195		char *s = alloca(nbytes + 1);
1196		bcopy(c, s, nbytes);
1197		s[nbytes] = '\0';
1198		return (dt_printf(dtp, fp, "  %-*s", width, s));
1199	}
1200
1201raw:
1202	if (dt_printf(dtp, fp, "\n%*s      ", margin, "") < 0)
1203		return (-1);
1204
1205	for (i = 0; i < 16; i++)
1206		if (dt_printf(dtp, fp, "  %c", "0123456789abcdef"[i]) < 0)
1207			return (-1);
1208
1209	if (dt_printf(dtp, fp, "  0123456789abcdef\n") < 0)
1210		return (-1);
1211
1212
1213	for (i = 0; i < nbytes; i += 16) {
1214		if (dt_printf(dtp, fp, "%*s%5x:", margin, "", i) < 0)
1215			return (-1);
1216
1217		for (j = i; j < i + 16 && j < nbytes; j++) {
1218			if (dt_printf(dtp, fp, " %02x", (uchar_t)c[j]) < 0)
1219				return (-1);
1220		}
1221
1222		while (j++ % 16) {
1223			if (dt_printf(dtp, fp, "   ") < 0)
1224				return (-1);
1225		}
1226
1227		if (dt_printf(dtp, fp, "  ") < 0)
1228			return (-1);
1229
1230		for (j = i; j < i + 16 && j < nbytes; j++) {
1231			if (dt_printf(dtp, fp, "%c",
1232			    c[j] < ' ' || c[j] > '~' ? '.' : c[j]) < 0)
1233				return (-1);
1234		}
1235
1236		if (dt_printf(dtp, fp, "\n") < 0)
1237			return (-1);
1238	}
1239
1240	return (0);
1241}
1242
1243int
1244dt_print_stack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
1245    caddr_t addr, int depth, int size)
1246{
1247	dtrace_syminfo_t dts;
1248	GElf_Sym sym;
1249	int i, indent;
1250	char c[PATH_MAX * 2];
1251	uint64_t pc;
1252
1253	if (dt_printf(dtp, fp, "\n") < 0)
1254		return (-1);
1255
1256	if (format == NULL)
1257		format = "%s";
1258
1259	if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
1260		indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
1261	else
1262		indent = _dtrace_stkindent;
1263
1264	for (i = 0; i < depth; i++) {
1265		switch (size) {
1266		case sizeof (uint32_t):
1267			/* LINTED - alignment */
1268			pc = *((uint32_t *)addr);
1269			break;
1270
1271		case sizeof (uint64_t):
1272			/* LINTED - alignment */
1273			pc = *((uint64_t *)addr);
1274			break;
1275
1276		default:
1277			return (dt_set_errno(dtp, EDT_BADSTACKPC));
1278		}
1279
1280		if (pc == 0)
1281			break;
1282
1283		addr += size;
1284
1285		if (dt_printf(dtp, fp, "%*s", indent, "") < 0)
1286			return (-1);
1287
1288		if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
1289			if (pc > sym.st_value) {
1290				(void) snprintf(c, sizeof (c), "%s`%s+0x%llx",
1291				    dts.dts_object, dts.dts_name,
1292				    pc - sym.st_value);
1293			} else {
1294				(void) snprintf(c, sizeof (c), "%s`%s",
1295				    dts.dts_object, dts.dts_name);
1296			}
1297		} else {
1298			/*
1299			 * We'll repeat the lookup, but this time we'll specify
1300			 * a NULL GElf_Sym -- indicating that we're only
1301			 * interested in the containing module.
1302			 */
1303			if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1304				(void) snprintf(c, sizeof (c), "%s`0x%llx",
1305				    dts.dts_object, pc);
1306			} else {
1307				(void) snprintf(c, sizeof (c), "0x%llx", pc);
1308			}
1309		}
1310
1311		if (dt_printf(dtp, fp, format, c) < 0)
1312			return (-1);
1313
1314		if (dt_printf(dtp, fp, "\n") < 0)
1315			return (-1);
1316	}
1317
1318	return (0);
1319}
1320
1321int
1322dt_print_ustack(dtrace_hdl_t *dtp, FILE *fp, const char *format,
1323    caddr_t addr, uint64_t arg)
1324{
1325	/* LINTED - alignment */
1326	uint64_t *pc = (uint64_t *)addr;
1327	uint32_t depth = DTRACE_USTACK_NFRAMES(arg);
1328	uint32_t strsize = DTRACE_USTACK_STRSIZE(arg);
1329	const char *strbase = addr + (depth + 1) * sizeof (uint64_t);
1330	const char *str = strsize ? strbase : NULL;
1331	int err = 0;
1332
1333	char name[PATH_MAX], objname[PATH_MAX], c[PATH_MAX * 2];
1334	struct ps_prochandle *P;
1335	GElf_Sym sym;
1336	int i, indent;
1337	pid_t pid;
1338
1339	if (depth == 0)
1340		return (0);
1341
1342	pid = (pid_t)*pc++;
1343
1344	if (dt_printf(dtp, fp, "\n") < 0)
1345		return (-1);
1346
1347	if (format == NULL)
1348		format = "%s";
1349
1350	if (dtp->dt_options[DTRACEOPT_STACKINDENT] != DTRACEOPT_UNSET)
1351		indent = (int)dtp->dt_options[DTRACEOPT_STACKINDENT];
1352	else
1353		indent = _dtrace_stkindent;
1354
1355	/*
1356	 * Ultimately, we need to add an entry point in the library vector for
1357	 * determining <symbol, offset> from <pid, address>.  For now, if
1358	 * this is a vector open, we just print the raw address or string.
1359	 */
1360	if (dtp->dt_vector == NULL)
1361		P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
1362	else
1363		P = NULL;
1364
1365	if (P != NULL)
1366		dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
1367
1368	for (i = 0; i < depth && pc[i] != 0; i++) {
1369		const prmap_t *map;
1370
1371		if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
1372			break;
1373
1374		if (P != NULL && Plookup_by_addr(P, pc[i],
1375		    name, sizeof (name), &sym) == 0) {
1376			(void) Pobjname(P, pc[i], objname, sizeof (objname));
1377
1378			if (pc[i] > sym.st_value) {
1379				(void) snprintf(c, sizeof (c),
1380				    "%s`%s+0x%llx", dt_basename(objname), name,
1381				    (u_longlong_t)(pc[i] - sym.st_value));
1382			} else {
1383				(void) snprintf(c, sizeof (c),
1384				    "%s`%s", dt_basename(objname), name);
1385			}
1386		} else if (str != NULL && str[0] != '\0' && str[0] != '@' &&
1387		    (P == NULL || (map = Paddr_to_map(P, pc[i])) == NULL ||
1388		    map->pr_mflags & MA_WRITE)) {
1389			/*
1390			 * If the current string pointer in the string table
1391			 * does not point to an empty string _and_ the program
1392			 * counter falls in a writable region, we'll use the
1393			 * string from the string table instead of the raw
1394			 * address.  This last condition is necessary because
1395			 * some (broken) ustack helpers will return a string
1396			 * even for a program counter that they can't
1397			 * identify.  If we have a string for a program
1398			 * counter that falls in a segment that isn't
1399			 * writable, we assume that we have fallen into this
1400			 * case and we refuse to use the string.  Finally,
1401			 * note that if we could not grab the process (e.g.,
1402			 * because it exited), the information from the helper
1403			 * is better than nothing.
1404			 */
1405			(void) snprintf(c, sizeof (c), "%s", str);
1406		} else {
1407			if (P != NULL && Pobjname(P, pc[i], objname,
1408			    sizeof (objname)) != NULL) {
1409				(void) snprintf(c, sizeof (c), "%s`0x%llx",
1410				    dt_basename(objname), (u_longlong_t)pc[i]);
1411			} else {
1412				(void) snprintf(c, sizeof (c), "0x%llx",
1413				    (u_longlong_t)pc[i]);
1414			}
1415		}
1416
1417		if ((err = dt_printf(dtp, fp, format, c)) < 0)
1418			break;
1419
1420		if ((err = dt_printf(dtp, fp, "\n")) < 0)
1421			break;
1422
1423		if (str != NULL && str[0] == '@') {
1424			/*
1425			 * If the first character of the string is an "at" sign,
1426			 * then the string is inferred to be an annotation --
1427			 * and it is printed out beneath the frame and offset
1428			 * with brackets.
1429			 */
1430			if ((err = dt_printf(dtp, fp, "%*s", indent, "")) < 0)
1431				break;
1432
1433			(void) snprintf(c, sizeof (c), "  [ %s ]", &str[1]);
1434
1435			if ((err = dt_printf(dtp, fp, format, c)) < 0)
1436				break;
1437
1438			if ((err = dt_printf(dtp, fp, "\n")) < 0)
1439				break;
1440		}
1441
1442		if (str != NULL) {
1443			str += strlen(str) + 1;
1444			if (str - strbase >= strsize)
1445				str = NULL;
1446		}
1447	}
1448
1449	if (P != NULL) {
1450		dt_proc_unlock(dtp, P);
1451		dt_proc_release(dtp, P);
1452	}
1453
1454	return (err);
1455}
1456
1457static int
1458dt_print_usym(dtrace_hdl_t *dtp, FILE *fp, caddr_t addr, dtrace_actkind_t act)
1459{
1460	/* LINTED - alignment */
1461	uint64_t pid = ((uint64_t *)addr)[0];
1462	/* LINTED - alignment */
1463	uint64_t pc = ((uint64_t *)addr)[1];
1464	const char *format = "  %-50s";
1465	char *s;
1466	int n, len = 256;
1467
1468	if (act == DTRACEACT_USYM && dtp->dt_vector == NULL) {
1469		struct ps_prochandle *P;
1470
1471		if ((P = dt_proc_grab(dtp, pid,
1472		    PGRAB_RDONLY | PGRAB_FORCE, 0)) != NULL) {
1473			GElf_Sym sym;
1474
1475			dt_proc_lock(dtp, P);
1476
1477			if (Plookup_by_addr(P, pc, NULL, 0, &sym) == 0)
1478				pc = sym.st_value;
1479
1480			dt_proc_unlock(dtp, P);
1481			dt_proc_release(dtp, P);
1482		}
1483	}
1484
1485	do {
1486		n = len;
1487		s = alloca(n);
1488	} while ((len = dtrace_uaddr2str(dtp, pid, pc, s, n)) > n);
1489
1490	return (dt_printf(dtp, fp, format, s));
1491}
1492
1493int
1494dt_print_umod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1495{
1496	/* LINTED - alignment */
1497	uint64_t pid = ((uint64_t *)addr)[0];
1498	/* LINTED - alignment */
1499	uint64_t pc = ((uint64_t *)addr)[1];
1500	int err = 0;
1501
1502	char objname[PATH_MAX], c[PATH_MAX * 2];
1503	struct ps_prochandle *P;
1504
1505	if (format == NULL)
1506		format = "  %-50s";
1507
1508	/*
1509	 * See the comment in dt_print_ustack() for the rationale for
1510	 * printing raw addresses in the vectored case.
1511	 */
1512	if (dtp->dt_vector == NULL)
1513		P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0);
1514	else
1515		P = NULL;
1516
1517	if (P != NULL)
1518		dt_proc_lock(dtp, P); /* lock handle while we perform lookups */
1519
1520	if (P != NULL && Pobjname(P, pc, objname, sizeof (objname)) != NULL) {
1521		(void) snprintf(c, sizeof (c), "%s", dt_basename(objname));
1522	} else {
1523		(void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1524	}
1525
1526	err = dt_printf(dtp, fp, format, c);
1527
1528	if (P != NULL) {
1529		dt_proc_unlock(dtp, P);
1530		dt_proc_release(dtp, P);
1531	}
1532
1533	return (err);
1534}
1535
1536static int
1537dt_print_sym(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1538{
1539	/* LINTED - alignment */
1540	uint64_t pc = *((uint64_t *)addr);
1541	dtrace_syminfo_t dts;
1542	GElf_Sym sym;
1543	char c[PATH_MAX * 2];
1544
1545	if (format == NULL)
1546		format = "  %-50s";
1547
1548	if (dtrace_lookup_by_addr(dtp, pc, &sym, &dts) == 0) {
1549		(void) snprintf(c, sizeof (c), "%s`%s",
1550		    dts.dts_object, dts.dts_name);
1551	} else {
1552		/*
1553		 * We'll repeat the lookup, but this time we'll specify a
1554		 * NULL GElf_Sym -- indicating that we're only interested in
1555		 * the containing module.
1556		 */
1557		if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1558			(void) snprintf(c, sizeof (c), "%s`0x%llx",
1559			    dts.dts_object, (u_longlong_t)pc);
1560		} else {
1561			(void) snprintf(c, sizeof (c), "0x%llx",
1562			    (u_longlong_t)pc);
1563		}
1564	}
1565
1566	if (dt_printf(dtp, fp, format, c) < 0)
1567		return (-1);
1568
1569	return (0);
1570}
1571
1572int
1573dt_print_mod(dtrace_hdl_t *dtp, FILE *fp, const char *format, caddr_t addr)
1574{
1575	/* LINTED - alignment */
1576	uint64_t pc = *((uint64_t *)addr);
1577	dtrace_syminfo_t dts;
1578	char c[PATH_MAX * 2];
1579
1580	if (format == NULL)
1581		format = "  %-50s";
1582
1583	if (dtrace_lookup_by_addr(dtp, pc, NULL, &dts) == 0) {
1584		(void) snprintf(c, sizeof (c), "%s", dts.dts_object);
1585	} else {
1586		(void) snprintf(c, sizeof (c), "0x%llx", (u_longlong_t)pc);
1587	}
1588
1589	if (dt_printf(dtp, fp, format, c) < 0)
1590		return (-1);
1591
1592	return (0);
1593}
1594
1595typedef struct dt_normal {
1596	dtrace_aggvarid_t dtnd_id;
1597	uint64_t dtnd_normal;
1598} dt_normal_t;
1599
1600static int
1601dt_normalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1602{
1603	dt_normal_t *normal = arg;
1604	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1605	dtrace_aggvarid_t id = normal->dtnd_id;
1606
1607	if (agg->dtagd_nrecs == 0)
1608		return (DTRACE_AGGWALK_NEXT);
1609
1610	if (agg->dtagd_varid != id)
1611		return (DTRACE_AGGWALK_NEXT);
1612
1613	((dtrace_aggdata_t *)aggdata)->dtada_normal = normal->dtnd_normal;
1614	return (DTRACE_AGGWALK_NORMALIZE);
1615}
1616
1617static int
1618dt_normalize(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
1619{
1620	dt_normal_t normal;
1621	caddr_t addr;
1622
1623	/*
1624	 * We (should) have two records:  the aggregation ID followed by the
1625	 * normalization value.
1626	 */
1627	addr = base + rec->dtrd_offset;
1628
1629	if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
1630		return (dt_set_errno(dtp, EDT_BADNORMAL));
1631
1632	/* LINTED - alignment */
1633	normal.dtnd_id = *((dtrace_aggvarid_t *)addr);
1634	rec++;
1635
1636	if (rec->dtrd_action != DTRACEACT_LIBACT)
1637		return (dt_set_errno(dtp, EDT_BADNORMAL));
1638
1639	if (rec->dtrd_arg != DT_ACT_NORMALIZE)
1640		return (dt_set_errno(dtp, EDT_BADNORMAL));
1641
1642	addr = base + rec->dtrd_offset;
1643
1644	switch (rec->dtrd_size) {
1645	case sizeof (uint64_t):
1646		/* LINTED - alignment */
1647		normal.dtnd_normal = *((uint64_t *)addr);
1648		break;
1649	case sizeof (uint32_t):
1650		/* LINTED - alignment */
1651		normal.dtnd_normal = *((uint32_t *)addr);
1652		break;
1653	case sizeof (uint16_t):
1654		/* LINTED - alignment */
1655		normal.dtnd_normal = *((uint16_t *)addr);
1656		break;
1657	case sizeof (uint8_t):
1658		normal.dtnd_normal = *((uint8_t *)addr);
1659		break;
1660	default:
1661		return (dt_set_errno(dtp, EDT_BADNORMAL));
1662	}
1663
1664	(void) dtrace_aggregate_walk(dtp, dt_normalize_agg, &normal);
1665
1666	return (0);
1667}
1668
1669static int
1670dt_denormalize_agg(const dtrace_aggdata_t *aggdata, void *arg)
1671{
1672	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1673	dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
1674
1675	if (agg->dtagd_nrecs == 0)
1676		return (DTRACE_AGGWALK_NEXT);
1677
1678	if (agg->dtagd_varid != id)
1679		return (DTRACE_AGGWALK_NEXT);
1680
1681	return (DTRACE_AGGWALK_DENORMALIZE);
1682}
1683
1684static int
1685dt_clear_agg(const dtrace_aggdata_t *aggdata, void *arg)
1686{
1687	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1688	dtrace_aggvarid_t id = *((dtrace_aggvarid_t *)arg);
1689
1690	if (agg->dtagd_nrecs == 0)
1691		return (DTRACE_AGGWALK_NEXT);
1692
1693	if (agg->dtagd_varid != id)
1694		return (DTRACE_AGGWALK_NEXT);
1695
1696	return (DTRACE_AGGWALK_CLEAR);
1697}
1698
1699typedef struct dt_trunc {
1700	dtrace_aggvarid_t dttd_id;
1701	uint64_t dttd_remaining;
1702} dt_trunc_t;
1703
1704static int
1705dt_trunc_agg(const dtrace_aggdata_t *aggdata, void *arg)
1706{
1707	dt_trunc_t *trunc = arg;
1708	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1709	dtrace_aggvarid_t id = trunc->dttd_id;
1710
1711	if (agg->dtagd_nrecs == 0)
1712		return (DTRACE_AGGWALK_NEXT);
1713
1714	if (agg->dtagd_varid != id)
1715		return (DTRACE_AGGWALK_NEXT);
1716
1717	if (trunc->dttd_remaining == 0)
1718		return (DTRACE_AGGWALK_REMOVE);
1719
1720	trunc->dttd_remaining--;
1721	return (DTRACE_AGGWALK_NEXT);
1722}
1723
1724static int
1725dt_trunc(dtrace_hdl_t *dtp, caddr_t base, dtrace_recdesc_t *rec)
1726{
1727	dt_trunc_t trunc;
1728	caddr_t addr;
1729	int64_t remaining;
1730	int (*func)(dtrace_hdl_t *, dtrace_aggregate_f *, void *);
1731
1732	/*
1733	 * We (should) have two records:  the aggregation ID followed by the
1734	 * number of aggregation entries after which the aggregation is to be
1735	 * truncated.
1736	 */
1737	addr = base + rec->dtrd_offset;
1738
1739	if (rec->dtrd_size != sizeof (dtrace_aggvarid_t))
1740		return (dt_set_errno(dtp, EDT_BADTRUNC));
1741
1742	/* LINTED - alignment */
1743	trunc.dttd_id = *((dtrace_aggvarid_t *)addr);
1744	rec++;
1745
1746	if (rec->dtrd_action != DTRACEACT_LIBACT)
1747		return (dt_set_errno(dtp, EDT_BADTRUNC));
1748
1749	if (rec->dtrd_arg != DT_ACT_TRUNC)
1750		return (dt_set_errno(dtp, EDT_BADTRUNC));
1751
1752	addr = base + rec->dtrd_offset;
1753
1754	switch (rec->dtrd_size) {
1755	case sizeof (uint64_t):
1756		/* LINTED - alignment */
1757		remaining = *((int64_t *)addr);
1758		break;
1759	case sizeof (uint32_t):
1760		/* LINTED - alignment */
1761		remaining = *((int32_t *)addr);
1762		break;
1763	case sizeof (uint16_t):
1764		/* LINTED - alignment */
1765		remaining = *((int16_t *)addr);
1766		break;
1767	case sizeof (uint8_t):
1768		remaining = *((int8_t *)addr);
1769		break;
1770	default:
1771		return (dt_set_errno(dtp, EDT_BADNORMAL));
1772	}
1773
1774	if (remaining < 0) {
1775		func = dtrace_aggregate_walk_valsorted;
1776		remaining = -remaining;
1777	} else {
1778		func = dtrace_aggregate_walk_valrevsorted;
1779	}
1780
1781	assert(remaining >= 0);
1782	trunc.dttd_remaining = remaining;
1783
1784	(void) func(dtp, dt_trunc_agg, &trunc);
1785
1786	return (0);
1787}
1788
1789static int
1790dt_print_datum(dtrace_hdl_t *dtp, FILE *fp, dtrace_recdesc_t *rec,
1791    caddr_t addr, size_t size, const dtrace_aggdata_t *aggdata,
1792    uint64_t normal, dt_print_aggdata_t *pd)
1793{
1794	int err, width;
1795	dtrace_actkind_t act = rec->dtrd_action;
1796	boolean_t packed = pd->dtpa_agghist || pd->dtpa_aggpack;
1797	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1798
1799	static struct {
1800		size_t size;
1801		int width;
1802		int packedwidth;
1803	} *fmt, fmttab[] = {
1804		{ sizeof (uint8_t),	3,	3 },
1805		{ sizeof (uint16_t),	5,	5 },
1806		{ sizeof (uint32_t),	8,	8 },
1807		{ sizeof (uint64_t),	16,	16 },
1808		{ 0,			-50,	16 }
1809	};
1810
1811	if (packed && pd->dtpa_agghisthdr != agg->dtagd_varid) {
1812		dtrace_recdesc_t *r;
1813
1814		width = 0;
1815
1816		/*
1817		 * To print our quantization header for either an agghist or
1818		 * aggpack aggregation, we need to iterate through all of our
1819		 * of our records to determine their width.
1820		 */
1821		for (r = rec; !DTRACEACT_ISAGG(r->dtrd_action); r++) {
1822			for (fmt = fmttab; fmt->size &&
1823			    fmt->size != r->dtrd_size; fmt++)
1824				continue;
1825
1826			width += fmt->packedwidth + 1;
1827		}
1828
1829		if (pd->dtpa_agghist) {
1830			if (dt_print_quanthdr(dtp, fp, width) < 0)
1831				return (-1);
1832		} else {
1833			if (dt_print_quanthdr_packed(dtp, fp,
1834			    width, aggdata, r->dtrd_action) < 0)
1835				return (-1);
1836		}
1837
1838		pd->dtpa_agghisthdr = agg->dtagd_varid;
1839	}
1840
1841	if (pd->dtpa_agghist && DTRACEACT_ISAGG(act)) {
1842		char positives = aggdata->dtada_flags & DTRACE_A_HASPOSITIVES;
1843		char negatives = aggdata->dtada_flags & DTRACE_A_HASNEGATIVES;
1844		int64_t val;
1845
1846		assert(act == DTRACEAGG_SUM || act == DTRACEAGG_COUNT);
1847		val = (long long)*((uint64_t *)addr);
1848
1849		if (dt_printf(dtp, fp, " ") < 0)
1850			return (-1);
1851
1852		return (dt_print_quantline(dtp, fp, val, normal,
1853		    aggdata->dtada_total, positives, negatives));
1854	}
1855
1856	if (pd->dtpa_aggpack && DTRACEACT_ISAGG(act)) {
1857		switch (act) {
1858		case DTRACEAGG_QUANTIZE:
1859			return (dt_print_quantize_packed(dtp,
1860			    fp, addr, size, aggdata));
1861		case DTRACEAGG_LQUANTIZE:
1862			return (dt_print_lquantize_packed(dtp,
1863			    fp, addr, size, aggdata));
1864		default:
1865			break;
1866		}
1867	}
1868
1869	switch (act) {
1870	case DTRACEACT_STACK:
1871		return (dt_print_stack(dtp, fp, NULL, addr,
1872		    rec->dtrd_arg, rec->dtrd_size / rec->dtrd_arg));
1873
1874	case DTRACEACT_USTACK:
1875	case DTRACEACT_JSTACK:
1876		return (dt_print_ustack(dtp, fp, NULL, addr, rec->dtrd_arg));
1877
1878	case DTRACEACT_USYM:
1879	case DTRACEACT_UADDR:
1880		return (dt_print_usym(dtp, fp, addr, act));
1881
1882	case DTRACEACT_UMOD:
1883		return (dt_print_umod(dtp, fp, NULL, addr));
1884
1885	case DTRACEACT_SYM:
1886		return (dt_print_sym(dtp, fp, NULL, addr));
1887
1888	case DTRACEACT_MOD:
1889		return (dt_print_mod(dtp, fp, NULL, addr));
1890
1891	case DTRACEAGG_QUANTIZE:
1892		return (dt_print_quantize(dtp, fp, addr, size, normal));
1893
1894	case DTRACEAGG_LQUANTIZE:
1895		return (dt_print_lquantize(dtp, fp, addr, size, normal));
1896
1897	case DTRACEAGG_LLQUANTIZE:
1898		return (dt_print_llquantize(dtp, fp, addr, size, normal));
1899
1900	case DTRACEAGG_AVG:
1901		return (dt_print_average(dtp, fp, addr, size, normal));
1902
1903	case DTRACEAGG_STDDEV:
1904		return (dt_print_stddev(dtp, fp, addr, size, normal));
1905
1906	default:
1907		break;
1908	}
1909
1910	for (fmt = fmttab; fmt->size && fmt->size != size; fmt++)
1911		continue;
1912
1913	width = packed ? fmt->packedwidth : fmt->width;
1914
1915	switch (size) {
1916	case sizeof (uint64_t):
1917		err = dt_printf(dtp, fp, " %*lld", width,
1918		    /* LINTED - alignment */
1919		    (long long)*((uint64_t *)addr) / normal);
1920		break;
1921	case sizeof (uint32_t):
1922		/* LINTED - alignment */
1923		err = dt_printf(dtp, fp, " %*d", width, *((uint32_t *)addr) /
1924		    (uint32_t)normal);
1925		break;
1926	case sizeof (uint16_t):
1927		/* LINTED - alignment */
1928		err = dt_printf(dtp, fp, " %*d", width, *((uint16_t *)addr) /
1929		    (uint32_t)normal);
1930		break;
1931	case sizeof (uint8_t):
1932		err = dt_printf(dtp, fp, " %*d", width, *((uint8_t *)addr) /
1933		    (uint32_t)normal);
1934		break;
1935	default:
1936		err = dt_print_bytes(dtp, fp, addr, size, width, 0, 0);
1937		break;
1938	}
1939
1940	return (err);
1941}
1942
1943int
1944dt_print_aggs(const dtrace_aggdata_t **aggsdata, int naggvars, void *arg)
1945{
1946	int i, aggact = 0;
1947	dt_print_aggdata_t *pd = arg;
1948	const dtrace_aggdata_t *aggdata = aggsdata[0];
1949	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
1950	FILE *fp = pd->dtpa_fp;
1951	dtrace_hdl_t *dtp = pd->dtpa_dtp;
1952	dtrace_recdesc_t *rec;
1953	dtrace_actkind_t act;
1954	caddr_t addr;
1955	size_t size;
1956
1957	pd->dtpa_agghist = (aggdata->dtada_flags & DTRACE_A_TOTAL);
1958	pd->dtpa_aggpack = (aggdata->dtada_flags & DTRACE_A_MINMAXBIN);
1959
1960	/*
1961	 * Iterate over each record description in the key, printing the traced
1962	 * data, skipping the first datum (the tuple member created by the
1963	 * compiler).
1964	 */
1965	for (i = 1; i < agg->dtagd_nrecs; i++) {
1966		rec = &agg->dtagd_rec[i];
1967		act = rec->dtrd_action;
1968		addr = aggdata->dtada_data + rec->dtrd_offset;
1969		size = rec->dtrd_size;
1970
1971		if (DTRACEACT_ISAGG(act)) {
1972			aggact = i;
1973			break;
1974		}
1975
1976		if (dt_print_datum(dtp, fp, rec, addr,
1977		    size, aggdata, 1, pd) < 0)
1978			return (-1);
1979
1980		if (dt_buffered_flush(dtp, NULL, rec, aggdata,
1981		    DTRACE_BUFDATA_AGGKEY) < 0)
1982			return (-1);
1983	}
1984
1985	assert(aggact != 0);
1986
1987	for (i = (naggvars == 1 ? 0 : 1); i < naggvars; i++) {
1988		uint64_t normal;
1989
1990		aggdata = aggsdata[i];
1991		agg = aggdata->dtada_desc;
1992		rec = &agg->dtagd_rec[aggact];
1993		act = rec->dtrd_action;
1994		addr = aggdata->dtada_data + rec->dtrd_offset;
1995		size = rec->dtrd_size;
1996
1997		assert(DTRACEACT_ISAGG(act));
1998		normal = aggdata->dtada_normal;
1999
2000		if (dt_print_datum(dtp, fp, rec, addr,
2001		    size, aggdata, normal, pd) < 0)
2002			return (-1);
2003
2004		if (dt_buffered_flush(dtp, NULL, rec, aggdata,
2005		    DTRACE_BUFDATA_AGGVAL) < 0)
2006			return (-1);
2007
2008		if (!pd->dtpa_allunprint)
2009			agg->dtagd_flags |= DTRACE_AGD_PRINTED;
2010	}
2011
2012	if (!pd->dtpa_agghist && !pd->dtpa_aggpack) {
2013		if (dt_printf(dtp, fp, "\n") < 0)
2014			return (-1);
2015	}
2016
2017	if (dt_buffered_flush(dtp, NULL, NULL, aggdata,
2018	    DTRACE_BUFDATA_AGGFORMAT | DTRACE_BUFDATA_AGGLAST) < 0)
2019		return (-1);
2020
2021	return (0);
2022}
2023
2024int
2025dt_print_agg(const dtrace_aggdata_t *aggdata, void *arg)
2026{
2027	dt_print_aggdata_t *pd = arg;
2028	dtrace_aggdesc_t *agg = aggdata->dtada_desc;
2029	dtrace_aggvarid_t aggvarid = pd->dtpa_id;
2030
2031	if (pd->dtpa_allunprint) {
2032		if (agg->dtagd_flags & DTRACE_AGD_PRINTED)
2033			return (0);
2034	} else {
2035		/*
2036		 * If we're not printing all unprinted aggregations, then the
2037		 * aggregation variable ID denotes a specific aggregation
2038		 * variable that we should print -- skip any other aggregations
2039		 * that we encounter.
2040		 */
2041		if (agg->dtagd_nrecs == 0)
2042			return (0);
2043
2044		if (aggvarid != agg->dtagd_varid)
2045			return (0);
2046	}
2047
2048	return (dt_print_aggs(&aggdata, 1, arg));
2049}
2050
2051int
2052dt_setopt(dtrace_hdl_t *dtp, const dtrace_probedata_t *data,
2053    const char *option, const char *value)
2054{
2055	int len, rval;
2056	char *msg;
2057	const char *errstr;
2058	dtrace_setoptdata_t optdata;
2059
2060	bzero(&optdata, sizeof (optdata));
2061	(void) dtrace_getopt(dtp, option, &optdata.dtsda_oldval);
2062
2063	if (dtrace_setopt(dtp, option, value) == 0) {
2064		(void) dtrace_getopt(dtp, option, &optdata.dtsda_newval);
2065		optdata.dtsda_probe = data;
2066		optdata.dtsda_option = option;
2067		optdata.dtsda_handle = dtp;
2068
2069		if ((rval = dt_handle_setopt(dtp, &optdata)) != 0)
2070			return (rval);
2071
2072		return (0);
2073	}
2074
2075	errstr = dtrace_errmsg(dtp, dtrace_errno(dtp));
2076	len = strlen(option) + strlen(value) + strlen(errstr) + 80;
2077	msg = alloca(len);
2078
2079	(void) snprintf(msg, len, "couldn't set option \"%s\" to \"%s\": %s\n",
2080	    option, value, errstr);
2081
2082	if ((rval = dt_handle_liberr(dtp, data, msg)) == 0)
2083		return (0);
2084
2085	return (rval);
2086}
2087
2088static int
2089dt_consume_cpu(dtrace_hdl_t *dtp, FILE *fp, int cpu,
2090    dtrace_bufdesc_t *buf, boolean_t just_one,
2091    dtrace_consume_probe_f *efunc, dtrace_consume_rec_f *rfunc, void *arg)
2092{
2093	dtrace_epid_t id;
2094	size_t offs;
2095	int flow = (dtp->dt_options[DTRACEOPT_FLOWINDENT] != DTRACEOPT_UNSET);
2096	int quiet = (dtp->dt_options[DTRACEOPT_QUIET] != DTRACEOPT_UNSET);
2097	int rval, i, n;
2098	uint64_t tracememsize = 0;
2099	dtrace_probedata_t data;
2100	uint64_t drops;
2101
2102	bzero(&data, sizeof (data));
2103	data.dtpda_handle = dtp;
2104	data.dtpda_cpu = cpu;
2105	data.dtpda_flow = dtp->dt_flow;
2106	data.dtpda_indent = dtp->dt_indent;
2107	data.dtpda_prefix = dtp->dt_prefix;
2108
2109	for (offs = buf->dtbd_oldest; offs < buf->dtbd_size; ) {
2110		dtrace_eprobedesc_t *epd;
2111
2112		/*
2113		 * We're guaranteed to have an ID.
2114		 */
2115		id = *(uint32_t *)((uintptr_t)buf->dtbd_data + offs);
2116
2117		if (id == DTRACE_EPIDNONE) {
2118			/*
2119			 * This is filler to assure proper alignment of the
2120			 * next record; we simply ignore it.
2121			 */
2122			offs += sizeof (id);
2123			continue;
2124		}
2125
2126		if ((rval = dt_epid_lookup(dtp, id, &data.dtpda_edesc,
2127		    &data.dtpda_pdesc)) != 0)
2128			return (rval);
2129
2130		epd = data.dtpda_edesc;
2131		data.dtpda_data = buf->dtbd_data + offs;
2132
2133		if (data.dtpda_edesc->dtepd_uarg != DT_ECB_DEFAULT) {
2134			rval = dt_handle(dtp, &data);
2135
2136			if (rval == DTRACE_CONSUME_NEXT)
2137				goto nextepid;
2138
2139			if (rval == DTRACE_CONSUME_ERROR)
2140				return (-1);
2141		}
2142
2143		if (flow)
2144			(void) dt_flowindent(dtp, &data, dtp->dt_last_epid,
2145			    buf, offs);
2146
2147		rval = (*efunc)(&data, arg);
2148
2149		if (flow) {
2150			if (data.dtpda_flow == DTRACEFLOW_ENTRY)
2151				data.dtpda_indent += 2;
2152		}
2153
2154		if (rval == DTRACE_CONSUME_NEXT)
2155			goto nextepid;
2156
2157		if (rval == DTRACE_CONSUME_ABORT)
2158			return (dt_set_errno(dtp, EDT_DIRABORT));
2159
2160		if (rval != DTRACE_CONSUME_THIS)
2161			return (dt_set_errno(dtp, EDT_BADRVAL));
2162
2163		for (i = 0; i < epd->dtepd_nrecs; i++) {
2164			caddr_t addr;
2165			dtrace_recdesc_t *rec = &epd->dtepd_rec[i];
2166			dtrace_actkind_t act = rec->dtrd_action;
2167
2168			data.dtpda_data = buf->dtbd_data + offs +
2169			    rec->dtrd_offset;
2170			addr = data.dtpda_data;
2171
2172			if (act == DTRACEACT_LIBACT) {
2173				uint64_t arg = rec->dtrd_arg;
2174				dtrace_aggvarid_t id;
2175
2176				switch (arg) {
2177				case DT_ACT_CLEAR:
2178					/* LINTED - alignment */
2179					id = *((dtrace_aggvarid_t *)addr);
2180					(void) dtrace_aggregate_walk(dtp,
2181					    dt_clear_agg, &id);
2182					continue;
2183
2184				case DT_ACT_DENORMALIZE:
2185					/* LINTED - alignment */
2186					id = *((dtrace_aggvarid_t *)addr);
2187					(void) dtrace_aggregate_walk(dtp,
2188					    dt_denormalize_agg, &id);
2189					continue;
2190
2191				case DT_ACT_FTRUNCATE:
2192					if (fp == NULL)
2193						continue;
2194
2195					(void) fflush(fp);
2196					(void) ftruncate(fileno(fp), 0);
2197					(void) fseeko(fp, 0, SEEK_SET);
2198					continue;
2199
2200				case DT_ACT_NORMALIZE:
2201					if (i == epd->dtepd_nrecs - 1)
2202						return (dt_set_errno(dtp,
2203						    EDT_BADNORMAL));
2204
2205					if (dt_normalize(dtp,
2206					    buf->dtbd_data + offs, rec) != 0)
2207						return (-1);
2208
2209					i++;
2210					continue;
2211
2212				case DT_ACT_SETOPT: {
2213					uint64_t *opts = dtp->dt_options;
2214					dtrace_recdesc_t *valrec;
2215					uint32_t valsize;
2216					caddr_t val;
2217					int rv;
2218
2219					if (i == epd->dtepd_nrecs - 1) {
2220						return (dt_set_errno(dtp,
2221						    EDT_BADSETOPT));
2222					}
2223
2224					valrec = &epd->dtepd_rec[++i];
2225					valsize = valrec->dtrd_size;
2226
2227					if (valrec->dtrd_action != act ||
2228					    valrec->dtrd_arg != arg) {
2229						return (dt_set_errno(dtp,
2230						    EDT_BADSETOPT));
2231					}
2232
2233					if (valsize > sizeof (uint64_t)) {
2234						val = buf->dtbd_data + offs +
2235						    valrec->dtrd_offset;
2236					} else {
2237						val = "1";
2238					}
2239
2240					rv = dt_setopt(dtp, &data, addr, val);
2241
2242					if (rv != 0)
2243						return (-1);
2244
2245					flow = (opts[DTRACEOPT_FLOWINDENT] !=
2246					    DTRACEOPT_UNSET);
2247					quiet = (opts[DTRACEOPT_QUIET] !=
2248					    DTRACEOPT_UNSET);
2249
2250					continue;
2251				}
2252
2253				case DT_ACT_TRUNC:
2254					if (i == epd->dtepd_nrecs - 1)
2255						return (dt_set_errno(dtp,
2256						    EDT_BADTRUNC));
2257
2258					if (dt_trunc(dtp,
2259					    buf->dtbd_data + offs, rec) != 0)
2260						return (-1);
2261
2262					i++;
2263					continue;
2264
2265				default:
2266					continue;
2267				}
2268			}
2269
2270			if (act == DTRACEACT_TRACEMEM_DYNSIZE &&
2271			    rec->dtrd_size == sizeof (uint64_t)) {
2272				/* LINTED - alignment */
2273				tracememsize = *((unsigned long long *)addr);
2274				continue;
2275			}
2276
2277			rval = (*rfunc)(&data, rec, arg);
2278
2279			if (rval == DTRACE_CONSUME_NEXT)
2280				continue;
2281
2282			if (rval == DTRACE_CONSUME_ABORT)
2283				return (dt_set_errno(dtp, EDT_DIRABORT));
2284
2285			if (rval != DTRACE_CONSUME_THIS)
2286				return (dt_set_errno(dtp, EDT_BADRVAL));
2287
2288			if (act == DTRACEACT_STACK) {
2289				int depth = rec->dtrd_arg;
2290
2291				if (dt_print_stack(dtp, fp, NULL, addr, depth,
2292				    rec->dtrd_size / depth) < 0)
2293					return (-1);
2294				goto nextrec;
2295			}
2296
2297			if (act == DTRACEACT_USTACK ||
2298			    act == DTRACEACT_JSTACK) {
2299				if (dt_print_ustack(dtp, fp, NULL,
2300				    addr, rec->dtrd_arg) < 0)
2301					return (-1);
2302				goto nextrec;
2303			}
2304
2305			if (act == DTRACEACT_SYM) {
2306				if (dt_print_sym(dtp, fp, NULL, addr) < 0)
2307					return (-1);
2308				goto nextrec;
2309			}
2310
2311			if (act == DTRACEACT_MOD) {
2312				if (dt_print_mod(dtp, fp, NULL, addr) < 0)
2313					return (-1);
2314				goto nextrec;
2315			}
2316
2317			if (act == DTRACEACT_USYM || act == DTRACEACT_UADDR) {
2318				if (dt_print_usym(dtp, fp, addr, act) < 0)
2319					return (-1);
2320				goto nextrec;
2321			}
2322
2323			if (act == DTRACEACT_UMOD) {
2324				if (dt_print_umod(dtp, fp, NULL, addr) < 0)
2325					return (-1);
2326				goto nextrec;
2327			}
2328
2329			if (DTRACEACT_ISPRINTFLIKE(act)) {
2330				void *fmtdata;
2331				int (*func)(dtrace_hdl_t *, FILE *, void *,
2332				    const dtrace_probedata_t *,
2333				    const dtrace_recdesc_t *, uint_t,
2334				    const void *buf, size_t);
2335
2336				if ((fmtdata = dt_format_lookup(dtp,
2337				    rec->dtrd_format)) == NULL)
2338					goto nofmt;
2339
2340				switch (act) {
2341				case DTRACEACT_PRINTF:
2342					func = dtrace_fprintf;
2343					break;
2344				case DTRACEACT_PRINTA:
2345					func = dtrace_fprinta;
2346					break;
2347				case DTRACEACT_SYSTEM:
2348					func = dtrace_system;
2349					break;
2350				case DTRACEACT_FREOPEN:
2351					func = dtrace_freopen;
2352					break;
2353				}
2354
2355				n = (*func)(dtp, fp, fmtdata, &data,
2356				    rec, epd->dtepd_nrecs - i,
2357				    (uchar_t *)buf->dtbd_data + offs,
2358				    buf->dtbd_size - offs);
2359
2360				if (n < 0)
2361					return (-1); /* errno is set for us */
2362
2363				if (n > 0)
2364					i += n - 1;
2365				goto nextrec;
2366			}
2367
2368			/*
2369			 * If this is a DIF expression, and the record has a
2370			 * format set, this indicates we have a CTF type name
2371			 * associated with the data and we should try to print
2372			 * it out by type.
2373			 */
2374			if (act == DTRACEACT_DIFEXPR) {
2375				const char *strdata = dt_strdata_lookup(dtp,
2376				    rec->dtrd_format);
2377				if (strdata != NULL) {
2378					n = dtrace_print(dtp, fp, strdata,
2379					    addr, rec->dtrd_size);
2380
2381					/*
2382					 * dtrace_print() will return -1 on
2383					 * error, or return the number of bytes
2384					 * consumed.  It will return 0 if the
2385					 * type couldn't be determined, and we
2386					 * should fall through to the normal
2387					 * trace method.
2388					 */
2389					if (n < 0)
2390						return (-1);
2391
2392					if (n > 0)
2393						goto nextrec;
2394				}
2395			}
2396
2397nofmt:
2398			if (act == DTRACEACT_PRINTA) {
2399				dt_print_aggdata_t pd;
2400				dtrace_aggvarid_t *aggvars;
2401				int j, naggvars = 0;
2402				size_t size = ((epd->dtepd_nrecs - i) *
2403				    sizeof (dtrace_aggvarid_t));
2404
2405				if ((aggvars = dt_alloc(dtp, size)) == NULL)
2406					return (-1);
2407
2408				/*
2409				 * This might be a printa() with multiple
2410				 * aggregation variables.  We need to scan
2411				 * forward through the records until we find
2412				 * a record from a different statement.
2413				 */
2414				for (j = i; j < epd->dtepd_nrecs; j++) {
2415					dtrace_recdesc_t *nrec;
2416					caddr_t naddr;
2417
2418					nrec = &epd->dtepd_rec[j];
2419
2420					if (nrec->dtrd_uarg != rec->dtrd_uarg)
2421						break;
2422
2423					if (nrec->dtrd_action != act) {
2424						return (dt_set_errno(dtp,
2425						    EDT_BADAGG));
2426					}
2427
2428					naddr = buf->dtbd_data + offs +
2429					    nrec->dtrd_offset;
2430
2431					aggvars[naggvars++] =
2432					    /* LINTED - alignment */
2433					    *((dtrace_aggvarid_t *)naddr);
2434				}
2435
2436				i = j - 1;
2437				bzero(&pd, sizeof (pd));
2438				pd.dtpa_dtp = dtp;
2439				pd.dtpa_fp = fp;
2440
2441				assert(naggvars >= 1);
2442
2443				if (naggvars == 1) {
2444					pd.dtpa_id = aggvars[0];
2445					dt_free(dtp, aggvars);
2446
2447					if (dt_printf(dtp, fp, "\n") < 0 ||
2448					    dtrace_aggregate_walk_sorted(dtp,
2449					    dt_print_agg, &pd) < 0)
2450						return (-1);
2451					goto nextrec;
2452				}
2453
2454				if (dt_printf(dtp, fp, "\n") < 0 ||
2455				    dtrace_aggregate_walk_joined(dtp, aggvars,
2456				    naggvars, dt_print_aggs, &pd) < 0) {
2457					dt_free(dtp, aggvars);
2458					return (-1);
2459				}
2460
2461				dt_free(dtp, aggvars);
2462				goto nextrec;
2463			}
2464
2465			if (act == DTRACEACT_TRACEMEM) {
2466				if (tracememsize == 0 ||
2467				    tracememsize > rec->dtrd_size) {
2468					tracememsize = rec->dtrd_size;
2469				}
2470
2471				n = dt_print_bytes(dtp, fp, addr,
2472				    tracememsize, -33, quiet, 1);
2473
2474				tracememsize = 0;
2475
2476				if (n < 0)
2477					return (-1);
2478
2479				goto nextrec;
2480			}
2481
2482			switch (rec->dtrd_size) {
2483			case sizeof (uint64_t):
2484				n = dt_printf(dtp, fp,
2485				    quiet ? "%lld" : " %16lld",
2486				    /* LINTED - alignment */
2487				    *((unsigned long long *)addr));
2488				break;
2489			case sizeof (uint32_t):
2490				n = dt_printf(dtp, fp, quiet ? "%d" : " %8d",
2491				    /* LINTED - alignment */
2492				    *((uint32_t *)addr));
2493				break;
2494			case sizeof (uint16_t):
2495				n = dt_printf(dtp, fp, quiet ? "%d" : " %5d",
2496				    /* LINTED - alignment */
2497				    *((uint16_t *)addr));
2498				break;
2499			case sizeof (uint8_t):
2500				n = dt_printf(dtp, fp, quiet ? "%d" : " %3d",
2501				    *((uint8_t *)addr));
2502				break;
2503			default:
2504				n = dt_print_bytes(dtp, fp, addr,
2505				    rec->dtrd_size, -33, quiet, 0);
2506				break;
2507			}
2508
2509			if (n < 0)
2510				return (-1); /* errno is set for us */
2511
2512nextrec:
2513			if (dt_buffered_flush(dtp, &data, rec, NULL, 0) < 0)
2514				return (-1); /* errno is set for us */
2515		}
2516
2517		/*
2518		 * Call the record callback with a NULL record to indicate
2519		 * that we're done processing this EPID.
2520		 */
2521		rval = (*rfunc)(&data, NULL, arg);
2522nextepid:
2523		offs += epd->dtepd_size;
2524		dtp->dt_last_epid = id;
2525		if (just_one) {
2526			buf->dtbd_oldest = offs;
2527			break;
2528		}
2529	}
2530
2531	dtp->dt_flow = data.dtpda_flow;
2532	dtp->dt_indent = data.dtpda_indent;
2533	dtp->dt_prefix = data.dtpda_prefix;
2534
2535	if ((drops = buf->dtbd_drops) == 0)
2536		return (0);
2537
2538	/*
2539	 * Explicitly zero the drops to prevent us from processing them again.
2540	 */
2541	buf->dtbd_drops = 0;
2542
2543	return (dt_handle_cpudrop(dtp, cpu, DTRACEDROP_PRINCIPAL, drops));
2544}
2545
2546/*
2547 * Reduce memory usage by shrinking the buffer if it's no more than half full.
2548 * Note, we need to preserve the alignment of the data at dtbd_oldest, which is
2549 * only 4-byte aligned.
2550 */
2551static void
2552dt_realloc_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf, int cursize)
2553{
2554	uint64_t used = buf->dtbd_size - buf->dtbd_oldest;
2555	if (used < cursize / 2) {
2556		int misalign = buf->dtbd_oldest & (sizeof (uint64_t) - 1);
2557		char *newdata = dt_alloc(dtp, used + misalign);
2558		if (newdata == NULL)
2559			return;
2560		bzero(newdata, misalign);
2561		bcopy(buf->dtbd_data + buf->dtbd_oldest,
2562		    newdata + misalign, used);
2563		dt_free(dtp, buf->dtbd_data);
2564		buf->dtbd_oldest = misalign;
2565		buf->dtbd_size = used + misalign;
2566		buf->dtbd_data = newdata;
2567	}
2568}
2569
2570/*
2571 * If the ring buffer has wrapped, the data is not in order.  Rearrange it
2572 * so that it is.  Note, we need to preserve the alignment of the data at
2573 * dtbd_oldest, which is only 4-byte aligned.
2574 */
2575static int
2576dt_unring_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf)
2577{
2578	int misalign;
2579	char *newdata, *ndp;
2580
2581	if (buf->dtbd_oldest == 0)
2582		return (0);
2583
2584	misalign = buf->dtbd_oldest & (sizeof (uint64_t) - 1);
2585	newdata = ndp = dt_alloc(dtp, buf->dtbd_size + misalign);
2586
2587	if (newdata == NULL)
2588		return (-1);
2589
2590	assert(0 == (buf->dtbd_size & (sizeof (uint64_t) - 1)));
2591
2592	bzero(ndp, misalign);
2593	ndp += misalign;
2594
2595	bcopy(buf->dtbd_data + buf->dtbd_oldest, ndp,
2596	    buf->dtbd_size - buf->dtbd_oldest);
2597	ndp += buf->dtbd_size - buf->dtbd_oldest;
2598
2599	bcopy(buf->dtbd_data, ndp, buf->dtbd_oldest);
2600
2601	dt_free(dtp, buf->dtbd_data);
2602	buf->dtbd_oldest = 0;
2603	buf->dtbd_data = newdata;
2604	buf->dtbd_size += misalign;
2605
2606	return (0);
2607}
2608
2609static void
2610dt_put_buf(dtrace_hdl_t *dtp, dtrace_bufdesc_t *buf)
2611{
2612	dt_free(dtp, buf->dtbd_data);
2613	dt_free(dtp, buf);
2614}
2615
2616/*
2617 * Returns 0 on success, in which case *cbp will be filled in if we retrieved
2618 * data, or NULL if there is no data for this CPU.
2619 * Returns -1 on failure and sets dt_errno.
2620 */
2621static int
2622dt_get_buf(dtrace_hdl_t *dtp, int cpu, dtrace_bufdesc_t **bufp)
2623{
2624	dtrace_optval_t size;
2625	dtrace_bufdesc_t *buf = dt_zalloc(dtp, sizeof (*buf));
2626	int error;
2627
2628	if (buf == NULL)
2629		return (-1);
2630
2631	(void) dtrace_getopt(dtp, "bufsize", &size);
2632	buf->dtbd_data = dt_alloc(dtp, size);
2633	if (buf->dtbd_data == NULL) {
2634		dt_free(dtp, buf);
2635		return (-1);
2636	}
2637	buf->dtbd_size = size;
2638	buf->dtbd_cpu = cpu;
2639
2640	if (dt_ioctl(dtp, DTRACEIOC_BUFSNAP, buf) == -1) {
2641		dt_put_buf(dtp, buf);
2642		/*
2643		 * If we failed with ENOENT, it may be because the
2644		 * CPU was unconfigured -- this is okay.  Any other
2645		 * error, however, is unexpected.
2646		 */
2647		if (errno == ENOENT) {
2648			*bufp = NULL;
2649			return (0);
2650		}
2651
2652		return (dt_set_errno(dtp, errno));
2653	}
2654
2655	error = dt_unring_buf(dtp, buf);
2656	if (error != 0) {
2657		dt_put_buf(dtp, buf);
2658		return (error);
2659	}
2660	dt_realloc_buf(dtp, buf, size);
2661
2662	*bufp = buf;
2663	return (0);
2664}
2665
2666typedef struct dt_begin {
2667	dtrace_consume_probe_f *dtbgn_probefunc;
2668	dtrace_consume_rec_f *dtbgn_recfunc;
2669	void *dtbgn_arg;
2670	dtrace_handle_err_f *dtbgn_errhdlr;
2671	void *dtbgn_errarg;
2672	int dtbgn_beginonly;
2673} dt_begin_t;
2674
2675static int
2676dt_consume_begin_probe(const dtrace_probedata_t *data, void *arg)
2677{
2678	dt_begin_t *begin = arg;
2679	dtrace_probedesc_t *pd = data->dtpda_pdesc;
2680
2681	int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
2682	int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
2683
2684	if (begin->dtbgn_beginonly) {
2685		if (!(r1 && r2))
2686			return (DTRACE_CONSUME_NEXT);
2687	} else {
2688		if (r1 && r2)
2689			return (DTRACE_CONSUME_NEXT);
2690	}
2691
2692	/*
2693	 * We have a record that we're interested in.  Now call the underlying
2694	 * probe function...
2695	 */
2696	return (begin->dtbgn_probefunc(data, begin->dtbgn_arg));
2697}
2698
2699static int
2700dt_consume_begin_record(const dtrace_probedata_t *data,
2701    const dtrace_recdesc_t *rec, void *arg)
2702{
2703	dt_begin_t *begin = arg;
2704
2705	return (begin->dtbgn_recfunc(data, rec, begin->dtbgn_arg));
2706}
2707
2708static int
2709dt_consume_begin_error(const dtrace_errdata_t *data, void *arg)
2710{
2711	dt_begin_t *begin = (dt_begin_t *)arg;
2712	dtrace_probedesc_t *pd = data->dteda_pdesc;
2713
2714	int r1 = (strcmp(pd->dtpd_provider, "dtrace") == 0);
2715	int r2 = (strcmp(pd->dtpd_name, "BEGIN") == 0);
2716
2717	if (begin->dtbgn_beginonly) {
2718		if (!(r1 && r2))
2719			return (DTRACE_HANDLE_OK);
2720	} else {
2721		if (r1 && r2)
2722			return (DTRACE_HANDLE_OK);
2723	}
2724
2725	return (begin->dtbgn_errhdlr(data, begin->dtbgn_errarg));
2726}
2727
2728static int
2729dt_consume_begin(dtrace_hdl_t *dtp, FILE *fp,
2730    dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
2731{
2732	/*
2733	 * There's this idea that the BEGIN probe should be processed before
2734	 * everything else, and that the END probe should be processed after
2735	 * anything else.  In the common case, this is pretty easy to deal
2736	 * with.  However, a situation may arise where the BEGIN enabling and
2737	 * END enabling are on the same CPU, and some enabling in the middle
2738	 * occurred on a different CPU.  To deal with this (blech!) we need to
2739	 * consume the BEGIN buffer up until the end of the BEGIN probe, and
2740	 * then set it aside.  We will then process every other CPU, and then
2741	 * we'll return to the BEGIN CPU and process the rest of the data
2742	 * (which will inevitably include the END probe, if any).  Making this
2743	 * even more complicated (!) is the library's ERROR enabling.  Because
2744	 * this enabling is processed before we even get into the consume call
2745	 * back, any ERROR firing would result in the library's ERROR enabling
2746	 * being processed twice -- once in our first pass (for BEGIN probes),
2747	 * and again in our second pass (for everything but BEGIN probes).  To
2748	 * deal with this, we interpose on the ERROR handler to assure that we
2749	 * only process ERROR enablings induced by BEGIN enablings in the
2750	 * first pass, and that we only process ERROR enablings _not_ induced
2751	 * by BEGIN enablings in the second pass.
2752	 */
2753
2754	dt_begin_t begin;
2755	processorid_t cpu = dtp->dt_beganon;
2756	int rval, i;
2757	static int max_ncpus;
2758	dtrace_bufdesc_t *buf;
2759
2760	dtp->dt_beganon = -1;
2761
2762	if (dt_get_buf(dtp, cpu, &buf) != 0)
2763		return (-1);
2764	if (buf == NULL)
2765		return (0);
2766
2767	if (!dtp->dt_stopped || buf->dtbd_cpu != dtp->dt_endedon) {
2768		/*
2769		 * This is the simple case.  We're either not stopped, or if
2770		 * we are, we actually processed any END probes on another
2771		 * CPU.  We can simply consume this buffer and return.
2772		 */
2773		rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
2774		    pf, rf, arg);
2775		dt_put_buf(dtp, buf);
2776		return (rval);
2777	}
2778
2779	begin.dtbgn_probefunc = pf;
2780	begin.dtbgn_recfunc = rf;
2781	begin.dtbgn_arg = arg;
2782	begin.dtbgn_beginonly = 1;
2783
2784	/*
2785	 * We need to interpose on the ERROR handler to be sure that we
2786	 * only process ERRORs induced by BEGIN.
2787	 */
2788	begin.dtbgn_errhdlr = dtp->dt_errhdlr;
2789	begin.dtbgn_errarg = dtp->dt_errarg;
2790	dtp->dt_errhdlr = dt_consume_begin_error;
2791	dtp->dt_errarg = &begin;
2792
2793	rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
2794	    dt_consume_begin_probe, dt_consume_begin_record, &begin);
2795
2796	dtp->dt_errhdlr = begin.dtbgn_errhdlr;
2797	dtp->dt_errarg = begin.dtbgn_errarg;
2798
2799	if (rval != 0) {
2800		dt_put_buf(dtp, buf);
2801		return (rval);
2802	}
2803
2804	if (max_ncpus == 0)
2805		max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
2806
2807	for (i = 0; i < max_ncpus; i++) {
2808		dtrace_bufdesc_t *nbuf;
2809		if (i == cpu)
2810			continue;
2811
2812		if (dt_get_buf(dtp, i, &nbuf) != 0) {
2813			dt_put_buf(dtp, buf);
2814			return (-1);
2815		}
2816		if (nbuf == NULL)
2817			continue;
2818
2819		rval = dt_consume_cpu(dtp, fp, i, nbuf, B_FALSE,
2820		    pf, rf, arg);
2821		dt_put_buf(dtp, nbuf);
2822		if (rval != 0) {
2823			dt_put_buf(dtp, buf);
2824			return (rval);
2825		}
2826	}
2827
2828	/*
2829	 * Okay -- we're done with the other buffers.  Now we want to
2830	 * reconsume the first buffer -- but this time we're looking for
2831	 * everything _but_ BEGIN.  And of course, in order to only consume
2832	 * those ERRORs _not_ associated with BEGIN, we need to reinstall our
2833	 * ERROR interposition function...
2834	 */
2835	begin.dtbgn_beginonly = 0;
2836
2837	assert(begin.dtbgn_errhdlr == dtp->dt_errhdlr);
2838	assert(begin.dtbgn_errarg == dtp->dt_errarg);
2839	dtp->dt_errhdlr = dt_consume_begin_error;
2840	dtp->dt_errarg = &begin;
2841
2842	rval = dt_consume_cpu(dtp, fp, cpu, buf, B_FALSE,
2843	    dt_consume_begin_probe, dt_consume_begin_record, &begin);
2844
2845	dtp->dt_errhdlr = begin.dtbgn_errhdlr;
2846	dtp->dt_errarg = begin.dtbgn_errarg;
2847
2848	return (rval);
2849}
2850
2851/* ARGSUSED */
2852static uint64_t
2853dt_buf_oldest(void *elem, void *arg)
2854{
2855	dtrace_bufdesc_t *buf = elem;
2856	size_t offs = buf->dtbd_oldest;
2857
2858	while (offs < buf->dtbd_size) {
2859		dtrace_rechdr_t *dtrh =
2860		    /* LINTED - alignment */
2861		    (dtrace_rechdr_t *)(buf->dtbd_data + offs);
2862		if (dtrh->dtrh_epid == DTRACE_EPIDNONE) {
2863			offs += sizeof (dtrace_epid_t);
2864		} else {
2865			return (DTRACE_RECORD_LOAD_TIMESTAMP(dtrh));
2866		}
2867	}
2868
2869	/* There are no records left; use the time the buffer was retrieved. */
2870	return (buf->dtbd_timestamp);
2871}
2872
2873int
2874dtrace_consume(dtrace_hdl_t *dtp, FILE *fp,
2875    dtrace_consume_probe_f *pf, dtrace_consume_rec_f *rf, void *arg)
2876{
2877	dtrace_optval_t size;
2878	static int max_ncpus;
2879	int i, rval;
2880	dtrace_optval_t interval = dtp->dt_options[DTRACEOPT_SWITCHRATE];
2881	hrtime_t now = gethrtime();
2882
2883	if (dtp->dt_lastswitch != 0) {
2884		if (now - dtp->dt_lastswitch < interval)
2885			return (0);
2886
2887		dtp->dt_lastswitch += interval;
2888	} else {
2889		dtp->dt_lastswitch = now;
2890	}
2891
2892	if (!dtp->dt_active)
2893		return (dt_set_errno(dtp, EINVAL));
2894
2895	if (max_ncpus == 0)
2896		max_ncpus = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
2897
2898	if (pf == NULL)
2899		pf = (dtrace_consume_probe_f *)dt_nullprobe;
2900
2901	if (rf == NULL)
2902		rf = (dtrace_consume_rec_f *)dt_nullrec;
2903
2904	if (dtp->dt_options[DTRACEOPT_TEMPORAL] == DTRACEOPT_UNSET) {
2905		/*
2906		 * The output will not be in the order it was traced.  Rather,
2907		 * we will consume all of the data from each CPU's buffer in
2908		 * turn.  We apply special handling for the records from BEGIN
2909		 * and END probes so that they are consumed first and last,
2910		 * respectively.
2911		 *
2912		 * If we have just begun, we want to first process the CPU that
2913		 * executed the BEGIN probe (if any).
2914		 */
2915		if (dtp->dt_active && dtp->dt_beganon != -1 &&
2916		    (rval = dt_consume_begin(dtp, fp, pf, rf, arg)) != 0)
2917			return (rval);
2918
2919		for (i = 0; i < max_ncpus; i++) {
2920			dtrace_bufdesc_t *buf;
2921
2922			/*
2923			 * If we have stopped, we want to process the CPU on
2924			 * which the END probe was processed only _after_ we
2925			 * have processed everything else.
2926			 */
2927			if (dtp->dt_stopped && (i == dtp->dt_endedon))
2928				continue;
2929
2930			if (dt_get_buf(dtp, i, &buf) != 0)
2931				return (-1);
2932			if (buf == NULL)
2933				continue;
2934
2935			dtp->dt_flow = 0;
2936			dtp->dt_indent = 0;
2937			dtp->dt_prefix = NULL;
2938			rval = dt_consume_cpu(dtp, fp, i,
2939			    buf, B_FALSE, pf, rf, arg);
2940			dt_put_buf(dtp, buf);
2941			if (rval != 0)
2942				return (rval);
2943		}
2944		if (dtp->dt_stopped) {
2945			dtrace_bufdesc_t *buf;
2946
2947			if (dt_get_buf(dtp, dtp->dt_endedon, &buf) != 0)
2948				return (-1);
2949			if (buf == NULL)
2950				return (0);
2951
2952			rval = dt_consume_cpu(dtp, fp, dtp->dt_endedon,
2953			    buf, B_FALSE, pf, rf, arg);
2954			dt_put_buf(dtp, buf);
2955			return (rval);
2956		}
2957	} else {
2958		/*
2959		 * The output will be in the order it was traced (or for
2960		 * speculations, when it was committed).  We retrieve a buffer
2961		 * from each CPU and put it into a priority queue, which sorts
2962		 * based on the first entry in the buffer.  This is sufficient
2963		 * because entries within a buffer are already sorted.
2964		 *
2965		 * We then consume records one at a time, always consuming the
2966		 * oldest record, as determined by the priority queue.  When
2967		 * we reach the end of the time covered by these buffers,
2968		 * we need to stop and retrieve more records on the next pass.
2969		 * The kernel tells us the time covered by each buffer, in
2970		 * dtbd_timestamp.  The first buffer's timestamp tells us the
2971		 * time covered by all buffers, as subsequently retrieved
2972		 * buffers will cover to a more recent time.
2973		 */
2974
2975		uint64_t *drops = alloca(max_ncpus * sizeof (uint64_t));
2976		uint64_t first_timestamp = 0;
2977		uint_t cookie = 0;
2978		dtrace_bufdesc_t *buf;
2979
2980		bzero(drops, max_ncpus * sizeof (uint64_t));
2981
2982		if (dtp->dt_bufq == NULL) {
2983			dtp->dt_bufq = dt_pq_init(dtp, max_ncpus * 2,
2984			    dt_buf_oldest, NULL);
2985			if (dtp->dt_bufq == NULL) /* ENOMEM */
2986				return (-1);
2987		}
2988
2989		/* Retrieve data from each CPU. */
2990		(void) dtrace_getopt(dtp, "bufsize", &size);
2991		for (i = 0; i < max_ncpus; i++) {
2992			dtrace_bufdesc_t *buf;
2993
2994			if (dt_get_buf(dtp, i, &buf) != 0)
2995				return (-1);
2996			if (buf != NULL) {
2997				if (first_timestamp == 0)
2998					first_timestamp = buf->dtbd_timestamp;
2999				assert(buf->dtbd_timestamp >= first_timestamp);
3000
3001				dt_pq_insert(dtp->dt_bufq, buf);
3002				drops[i] = buf->dtbd_drops;
3003				buf->dtbd_drops = 0;
3004			}
3005		}
3006
3007		/* Consume records. */
3008		for (;;) {
3009			dtrace_bufdesc_t *buf = dt_pq_pop(dtp->dt_bufq);
3010			uint64_t timestamp;
3011
3012			if (buf == NULL)
3013				break;
3014
3015			timestamp = dt_buf_oldest(buf, dtp);
3016			if (timestamp == buf->dtbd_timestamp) {
3017				/*
3018				 * We've reached the end of the time covered
3019				 * by this buffer.  If this is the oldest
3020				 * buffer, we must do another pass
3021				 * to retrieve more data.
3022				 */
3023				dt_put_buf(dtp, buf);
3024				if (timestamp == first_timestamp &&
3025				    !dtp->dt_stopped)
3026					break;
3027				continue;
3028			}
3029			assert(timestamp >= dtp->dt_last_timestamp);
3030			dtp->dt_last_timestamp = timestamp;
3031
3032			if ((rval = dt_consume_cpu(dtp, fp,
3033			    buf->dtbd_cpu, buf, B_TRUE, pf, rf, arg)) != 0)
3034				return (rval);
3035			dt_pq_insert(dtp->dt_bufq, buf);
3036		}
3037
3038		/* Consume drops. */
3039		for (i = 0; i < max_ncpus; i++) {
3040			if (drops[i] != 0) {
3041				int error = dt_handle_cpudrop(dtp, i,
3042				    DTRACEDROP_PRINCIPAL, drops[i]);
3043				if (error != 0)
3044					return (error);
3045			}
3046		}
3047
3048		/*
3049		 * Reduce memory usage by re-allocating smaller buffers
3050		 * for the "remnants".
3051		 */
3052		while (buf = dt_pq_walk(dtp->dt_bufq, &cookie))
3053			dt_realloc_buf(dtp, buf, buf->dtbd_size);
3054	}
3055
3056	return (0);
3057}
3058