17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5127bbe13Stomee  * Common Development and Distribution License (the "License").
6127bbe13Stomee  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
21127bbe13Stomee 
227c478bd9Sstevel@tonic-gate /*
236e0bee74Sjhaslam  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
272b6389efSBryan Cantrill /*
28*0ddc0ebbSBryan Cantrill  * Copyright (c) 2016, Joyent, Inc. All rights reserved.
29e5803b76SAdam H. Leventhal  * Copyright (c) 2012 by Delphix. All rights reserved.
302b6389efSBryan Cantrill  */
317c478bd9Sstevel@tonic-gate 
327c478bd9Sstevel@tonic-gate #include <stdlib.h>
337c478bd9Sstevel@tonic-gate #include <strings.h>
347c478bd9Sstevel@tonic-gate #include <errno.h>
357c478bd9Sstevel@tonic-gate #include <unistd.h>
367c478bd9Sstevel@tonic-gate #include <dt_impl.h>
377c478bd9Sstevel@tonic-gate #include <assert.h>
3830ef842dSbmc #include <alloca.h>
3930ef842dSbmc #include <limits.h>
407c478bd9Sstevel@tonic-gate 
417c478bd9Sstevel@tonic-gate #define	DTRACE_AHASHSIZE	32779		/* big 'ol prime */
427c478bd9Sstevel@tonic-gate 
4330ef842dSbmc /*
4430ef842dSbmc  * Because qsort(3C) does not allow an argument to be passed to a comparison
4530ef842dSbmc  * function, the variables that affect comparison must regrettably be global;
4630ef842dSbmc  * they are protected by a global static lock, dt_qsort_lock.
4730ef842dSbmc  */
4830ef842dSbmc static pthread_mutex_t dt_qsort_lock = PTHREAD_MUTEX_INITIALIZER;
4930ef842dSbmc 
5030ef842dSbmc static int dt_revsort;
5130ef842dSbmc static int dt_keysort;
5230ef842dSbmc static int dt_keypos;
5330ef842dSbmc 
5430ef842dSbmc #define	DT_LESSTHAN	(dt_revsort == 0 ? -1 : 1)
5530ef842dSbmc #define	DT_GREATERTHAN	(dt_revsort == 0 ? 1 : -1)
5630ef842dSbmc 
577c478bd9Sstevel@tonic-gate static void
dt_aggregate_count(int64_t * existing,int64_t * new,size_t size)58187eccf8Sbmc dt_aggregate_count(int64_t *existing, int64_t *new, size_t size)
597c478bd9Sstevel@tonic-gate {
607c478bd9Sstevel@tonic-gate 	int i;
617c478bd9Sstevel@tonic-gate 
62187eccf8Sbmc 	for (i = 0; i < size / sizeof (int64_t); i++)
637c478bd9Sstevel@tonic-gate 		existing[i] = existing[i] + new[i];
647c478bd9Sstevel@tonic-gate }
657c478bd9Sstevel@tonic-gate 
667c478bd9Sstevel@tonic-gate static int
dt_aggregate_countcmp(int64_t * lhs,int64_t * rhs)67187eccf8Sbmc dt_aggregate_countcmp(int64_t *lhs, int64_t *rhs)
687c478bd9Sstevel@tonic-gate {
69187eccf8Sbmc 	int64_t lvar = *lhs;
70187eccf8Sbmc 	int64_t rvar = *rhs;
717c478bd9Sstevel@tonic-gate 
727c478bd9Sstevel@tonic-gate 	if (lvar < rvar)
7330ef842dSbmc 		return (DT_LESSTHAN);
7430ef842dSbmc 
7530ef842dSbmc 	if (lvar > rvar)
7630ef842dSbmc 		return (DT_GREATERTHAN);
777c478bd9Sstevel@tonic-gate 
787c478bd9Sstevel@tonic-gate 	return (0);
797c478bd9Sstevel@tonic-gate }
807c478bd9Sstevel@tonic-gate 
817c478bd9Sstevel@tonic-gate /*ARGSUSED*/
827c478bd9Sstevel@tonic-gate static void
dt_aggregate_min(int64_t * existing,int64_t * new,size_t size)83187eccf8Sbmc dt_aggregate_min(int64_t *existing, int64_t *new, size_t size)
847c478bd9Sstevel@tonic-gate {
857c478bd9Sstevel@tonic-gate 	if (*new < *existing)
867c478bd9Sstevel@tonic-gate 		*existing = *new;
877c478bd9Sstevel@tonic-gate }
887c478bd9Sstevel@tonic-gate 
897c478bd9Sstevel@tonic-gate /*ARGSUSED*/
907c478bd9Sstevel@tonic-gate static void
dt_aggregate_max(int64_t * existing,int64_t * new,size_t size)91187eccf8Sbmc dt_aggregate_max(int64_t *existing, int64_t *new, size_t size)
927c478bd9Sstevel@tonic-gate {
937c478bd9Sstevel@tonic-gate 	if (*new > *existing)
947c478bd9Sstevel@tonic-gate 		*existing = *new;
957c478bd9Sstevel@tonic-gate }
967c478bd9Sstevel@tonic-gate 
977c478bd9Sstevel@tonic-gate static int
dt_aggregate_averagecmp(int64_t * lhs,int64_t * rhs)98187eccf8Sbmc dt_aggregate_averagecmp(int64_t *lhs, int64_t *rhs)
997c478bd9Sstevel@tonic-gate {
100187eccf8Sbmc 	int64_t lavg = lhs[0] ? (lhs[1] / lhs[0]) : 0;
101187eccf8Sbmc 	int64_t ravg = rhs[0] ? (rhs[1] / rhs[0]) : 0;
1027c478bd9Sstevel@tonic-gate 
1037c478bd9Sstevel@tonic-gate 	if (lavg < ravg)
10430ef842dSbmc 		return (DT_LESSTHAN);
10530ef842dSbmc 
10630ef842dSbmc 	if (lavg > ravg)
10730ef842dSbmc 		return (DT_GREATERTHAN);
1087c478bd9Sstevel@tonic-gate 
1097c478bd9Sstevel@tonic-gate 	return (0);
1107c478bd9Sstevel@tonic-gate }
1117c478bd9Sstevel@tonic-gate 
1126e0bee74Sjhaslam static int
dt_aggregate_stddevcmp(int64_t * lhs,int64_t * rhs)1136e0bee74Sjhaslam dt_aggregate_stddevcmp(int64_t *lhs, int64_t *rhs)
1146e0bee74Sjhaslam {
1156e0bee74Sjhaslam 	uint64_t lsd = dt_stddev((uint64_t *)lhs, 1);
1166e0bee74Sjhaslam 	uint64_t rsd = dt_stddev((uint64_t *)rhs, 1);
1176e0bee74Sjhaslam 
1186e0bee74Sjhaslam 	if (lsd < rsd)
1196e0bee74Sjhaslam 		return (DT_LESSTHAN);
1206e0bee74Sjhaslam 
1216e0bee74Sjhaslam 	if (lsd > rsd)
1226e0bee74Sjhaslam 		return (DT_GREATERTHAN);
1236e0bee74Sjhaslam 
1246e0bee74Sjhaslam 	return (0);
1256e0bee74Sjhaslam }
1266e0bee74Sjhaslam 
1277c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1287c478bd9Sstevel@tonic-gate static void
dt_aggregate_lquantize(int64_t * existing,int64_t * new,size_t size)129187eccf8Sbmc dt_aggregate_lquantize(int64_t *existing, int64_t *new, size_t size)
1307c478bd9Sstevel@tonic-gate {
131187eccf8Sbmc 	int64_t arg = *existing++;
1327c478bd9Sstevel@tonic-gate 	uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
1337c478bd9Sstevel@tonic-gate 	int i;
1347c478bd9Sstevel@tonic-gate 
1357c478bd9Sstevel@tonic-gate 	for (i = 0; i <= levels + 1; i++)
1367c478bd9Sstevel@tonic-gate 		existing[i] = existing[i] + new[i + 1];
1377c478bd9Sstevel@tonic-gate }
1387c478bd9Sstevel@tonic-gate 
139a1b5e537Sbmc static long double
dt_aggregate_lquantizedsum(int64_t * lquanta)140187eccf8Sbmc dt_aggregate_lquantizedsum(int64_t *lquanta)
1417c478bd9Sstevel@tonic-gate {
142187eccf8Sbmc 	int64_t arg = *lquanta++;
1437c478bd9Sstevel@tonic-gate 	int32_t base = DTRACE_LQUANTIZE_BASE(arg);
1447c478bd9Sstevel@tonic-gate 	uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
1457c478bd9Sstevel@tonic-gate 	uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg), i;
146a1b5e537Sbmc 	long double total = (long double)lquanta[0] * (long double)(base - 1);
1477c478bd9Sstevel@tonic-gate 
1487c478bd9Sstevel@tonic-gate 	for (i = 0; i < levels; base += step, i++)
149a1b5e537Sbmc 		total += (long double)lquanta[i + 1] * (long double)base;
1507c478bd9Sstevel@tonic-gate 
151a1b5e537Sbmc 	return (total + (long double)lquanta[levels + 1] *
152a1b5e537Sbmc 	    (long double)(base + 1));
1537c478bd9Sstevel@tonic-gate }
1547c478bd9Sstevel@tonic-gate 
155187eccf8Sbmc static int64_t
dt_aggregate_lquantizedzero(int64_t * lquanta)156187eccf8Sbmc dt_aggregate_lquantizedzero(int64_t *lquanta)
157187eccf8Sbmc {
158187eccf8Sbmc 	int64_t arg = *lquanta++;
159187eccf8Sbmc 	int32_t base = DTRACE_LQUANTIZE_BASE(arg);
160187eccf8Sbmc 	uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
161187eccf8Sbmc 	uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg), i;
162187eccf8Sbmc 
163187eccf8Sbmc 	if (base - 1 == 0)
164187eccf8Sbmc 		return (lquanta[0]);
165187eccf8Sbmc 
166187eccf8Sbmc 	for (i = 0; i < levels; base += step, i++) {
167187eccf8Sbmc 		if (base != 0)
168187eccf8Sbmc 			continue;
169187eccf8Sbmc 
170187eccf8Sbmc 		return (lquanta[i + 1]);
171187eccf8Sbmc 	}
172187eccf8Sbmc 
173187eccf8Sbmc 	if (base + 1 == 0)
174187eccf8Sbmc 		return (lquanta[levels + 1]);
175187eccf8Sbmc 
176187eccf8Sbmc 	return (0);
177187eccf8Sbmc }
178187eccf8Sbmc 
1797c478bd9Sstevel@tonic-gate static int
dt_aggregate_lquantizedcmp(int64_t * lhs,int64_t * rhs)180187eccf8Sbmc dt_aggregate_lquantizedcmp(int64_t *lhs, int64_t *rhs)
1817c478bd9Sstevel@tonic-gate {
182a1b5e537Sbmc 	long double lsum = dt_aggregate_lquantizedsum(lhs);
183a1b5e537Sbmc 	long double rsum = dt_aggregate_lquantizedsum(rhs);
184187eccf8Sbmc 	int64_t lzero, rzero;
1857c478bd9Sstevel@tonic-gate 
1867c478bd9Sstevel@tonic-gate 	if (lsum < rsum)
18730ef842dSbmc 		return (DT_LESSTHAN);
18830ef842dSbmc 
18930ef842dSbmc 	if (lsum > rsum)
19030ef842dSbmc 		return (DT_GREATERTHAN);
1917c478bd9Sstevel@tonic-gate 
192187eccf8Sbmc 	/*
193187eccf8Sbmc 	 * If they're both equal, then we will compare based on the weights at
194187eccf8Sbmc 	 * zero.  If the weights at zero are equal (or if zero is not within
195187eccf8Sbmc 	 * the range of the linear quantization), then this will be judged a
196187eccf8Sbmc 	 * tie and will be resolved based on the key comparison.
197187eccf8Sbmc 	 */
198187eccf8Sbmc 	lzero = dt_aggregate_lquantizedzero(lhs);
199187eccf8Sbmc 	rzero = dt_aggregate_lquantizedzero(rhs);
200187eccf8Sbmc 
201187eccf8Sbmc 	if (lzero < rzero)
20230ef842dSbmc 		return (DT_LESSTHAN);
20330ef842dSbmc 
20430ef842dSbmc 	if (lzero > rzero)
20530ef842dSbmc 		return (DT_GREATERTHAN);
206187eccf8Sbmc 
2077c478bd9Sstevel@tonic-gate 	return (0);
2087c478bd9Sstevel@tonic-gate }
2097c478bd9Sstevel@tonic-gate 
2102b6389efSBryan Cantrill static void
dt_aggregate_llquantize(int64_t * existing,int64_t * new,size_t size)2112b6389efSBryan Cantrill dt_aggregate_llquantize(int64_t *existing, int64_t *new, size_t size)
2122b6389efSBryan Cantrill {
2132b6389efSBryan Cantrill 	int i;
2142b6389efSBryan Cantrill 
2152b6389efSBryan Cantrill 	for (i = 1; i < size / sizeof (int64_t); i++)
2162b6389efSBryan Cantrill 		existing[i] = existing[i] + new[i];
2172b6389efSBryan Cantrill }
2182b6389efSBryan Cantrill 
2192b6389efSBryan Cantrill static long double
dt_aggregate_llquantizedsum(int64_t * llquanta)2202b6389efSBryan Cantrill dt_aggregate_llquantizedsum(int64_t *llquanta)
2212b6389efSBryan Cantrill {
2222b6389efSBryan Cantrill 	int64_t arg = *llquanta++;
2232b6389efSBryan Cantrill 	uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg);
2242b6389efSBryan Cantrill 	uint16_t low = DTRACE_LLQUANTIZE_LOW(arg);
2252b6389efSBryan Cantrill 	uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg);
2262b6389efSBryan Cantrill 	uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
2272b6389efSBryan Cantrill 	int bin = 0, order;
2282b6389efSBryan Cantrill 	int64_t value = 1, next, step;
2292b6389efSBryan Cantrill 	long double total;
2302b6389efSBryan Cantrill 
2312b6389efSBryan Cantrill 	assert(nsteps >= factor);
2322b6389efSBryan Cantrill 	assert(nsteps % factor == 0);
2332b6389efSBryan Cantrill 
2342b6389efSBryan Cantrill 	for (order = 0; order < low; order++)
2352b6389efSBryan Cantrill 		value *= factor;
2362b6389efSBryan Cantrill 
2372b6389efSBryan Cantrill 	total = (long double)llquanta[bin++] * (long double)(value - 1);
2382b6389efSBryan Cantrill 
2392b6389efSBryan Cantrill 	next = value * factor;
2402b6389efSBryan Cantrill 	step = next > nsteps ? next / nsteps : 1;
2412b6389efSBryan Cantrill 
2422b6389efSBryan Cantrill 	while (order <= high) {
2432b6389efSBryan Cantrill 		assert(value < next);
2442b6389efSBryan Cantrill 		total += (long double)llquanta[bin++] * (long double)(value);
2452b6389efSBryan Cantrill 
2462b6389efSBryan Cantrill 		if ((value += step) != next)
2472b6389efSBryan Cantrill 			continue;
2482b6389efSBryan Cantrill 
2492b6389efSBryan Cantrill 		next = value * factor;
2502b6389efSBryan Cantrill 		step = next > nsteps ? next / nsteps : 1;
2512b6389efSBryan Cantrill 		order++;
2522b6389efSBryan Cantrill 	}
2532b6389efSBryan Cantrill 
2542b6389efSBryan Cantrill 	return (total + (long double)llquanta[bin] * (long double)value);
2552b6389efSBryan Cantrill }
2562b6389efSBryan Cantrill 
2572b6389efSBryan Cantrill static int
dt_aggregate_llquantizedcmp(int64_t * lhs,int64_t * rhs)2582b6389efSBryan Cantrill dt_aggregate_llquantizedcmp(int64_t *lhs, int64_t *rhs)
2592b6389efSBryan Cantrill {
2602b6389efSBryan Cantrill 	long double lsum = dt_aggregate_llquantizedsum(lhs);
2612b6389efSBryan Cantrill 	long double rsum = dt_aggregate_llquantizedsum(rhs);
2622b6389efSBryan Cantrill 	int64_t lzero, rzero;
2632b6389efSBryan Cantrill 
2642b6389efSBryan Cantrill 	if (lsum < rsum)
2652b6389efSBryan Cantrill 		return (DT_LESSTHAN);
2662b6389efSBryan Cantrill 
2672b6389efSBryan Cantrill 	if (lsum > rsum)
2682b6389efSBryan Cantrill 		return (DT_GREATERTHAN);
2692b6389efSBryan Cantrill 
2702b6389efSBryan Cantrill 	/*
2712b6389efSBryan Cantrill 	 * If they're both equal, then we will compare based on the weights at
2722b6389efSBryan Cantrill 	 * zero.  If the weights at zero are equal, then this will be judged a
2732b6389efSBryan Cantrill 	 * tie and will be resolved based on the key comparison.
2742b6389efSBryan Cantrill 	 */
2752b6389efSBryan Cantrill 	lzero = lhs[1];
2762b6389efSBryan Cantrill 	rzero = rhs[1];
2772b6389efSBryan Cantrill 
2782b6389efSBryan Cantrill 	if (lzero < rzero)
2792b6389efSBryan Cantrill 		return (DT_LESSTHAN);
2802b6389efSBryan Cantrill 
2812b6389efSBryan Cantrill 	if (lzero > rzero)
2822b6389efSBryan Cantrill 		return (DT_GREATERTHAN);
2832b6389efSBryan Cantrill 
2842b6389efSBryan Cantrill 	return (0);
2852b6389efSBryan Cantrill }
2862b6389efSBryan Cantrill 
2877c478bd9Sstevel@tonic-gate static int
dt_aggregate_quantizedcmp(int64_t * lhs,int64_t * rhs)288187eccf8Sbmc dt_aggregate_quantizedcmp(int64_t *lhs, int64_t *rhs)
2897c478bd9Sstevel@tonic-gate {
2907c478bd9Sstevel@tonic-gate 	int nbuckets = DTRACE_QUANTIZE_NBUCKETS, i;
291a1b5e537Sbmc 	long double ltotal = 0, rtotal = 0;
292187eccf8Sbmc 	int64_t lzero, rzero;
2937c478bd9Sstevel@tonic-gate 
2947c478bd9Sstevel@tonic-gate 	for (i = 0; i < nbuckets; i++) {
2957c478bd9Sstevel@tonic-gate 		int64_t bucketval = DTRACE_QUANTIZE_BUCKETVAL(i);
2967c478bd9Sstevel@tonic-gate 
297187eccf8Sbmc 		if (bucketval == 0) {
298187eccf8Sbmc 			lzero = lhs[i];
299187eccf8Sbmc 			rzero = rhs[i];
300187eccf8Sbmc 		}
301187eccf8Sbmc 
302a1b5e537Sbmc 		ltotal += (long double)bucketval * (long double)lhs[i];
303a1b5e537Sbmc 		rtotal += (long double)bucketval * (long double)rhs[i];
3047c478bd9Sstevel@tonic-gate 	}
3057c478bd9Sstevel@tonic-gate 
3067c478bd9Sstevel@tonic-gate 	if (ltotal < rtotal)
30730ef842dSbmc 		return (DT_LESSTHAN);
30830ef842dSbmc 
30930ef842dSbmc 	if (ltotal > rtotal)
31030ef842dSbmc 		return (DT_GREATERTHAN);
3117c478bd9Sstevel@tonic-gate 
312187eccf8Sbmc 	/*
313187eccf8Sbmc 	 * If they're both equal, then we will compare based on the weights at
314187eccf8Sbmc 	 * zero.  If the weights at zero are equal, then this will be judged a
315187eccf8Sbmc 	 * tie and will be resolved based on the key comparison.
316187eccf8Sbmc 	 */
317187eccf8Sbmc 	if (lzero < rzero)
31830ef842dSbmc 		return (DT_LESSTHAN);
31930ef842dSbmc 
32030ef842dSbmc 	if (lzero > rzero)
32130ef842dSbmc 		return (DT_GREATERTHAN);
322187eccf8Sbmc 
3237c478bd9Sstevel@tonic-gate 	return (0);
3247c478bd9Sstevel@tonic-gate }
3257c478bd9Sstevel@tonic-gate 
326a1b5e537Sbmc static void
dt_aggregate_usym(dtrace_hdl_t * dtp,uint64_t * data)327a1b5e537Sbmc dt_aggregate_usym(dtrace_hdl_t *dtp, uint64_t *data)
328a1b5e537Sbmc {
329a1b5e537Sbmc 	uint64_t pid = data[0];
330a1b5e537Sbmc 	uint64_t *pc = &data[1];
331a1b5e537Sbmc 	struct ps_prochandle *P;
332a1b5e537Sbmc 	GElf_Sym sym;
333a1b5e537Sbmc 
334a1b5e537Sbmc 	if (dtp->dt_vector != NULL)
335a1b5e537Sbmc 		return;
336a1b5e537Sbmc 
337a1b5e537Sbmc 	if ((P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0)) == NULL)
338a1b5e537Sbmc 		return;
339a1b5e537Sbmc 
340a1b5e537Sbmc 	dt_proc_lock(dtp, P);
341a1b5e537Sbmc 
342a1b5e537Sbmc 	if (Plookup_by_addr(P, *pc, NULL, 0, &sym) == 0)
343a1b5e537Sbmc 		*pc = sym.st_value;
344a1b5e537Sbmc 
345a1b5e537Sbmc 	dt_proc_unlock(dtp, P);
346a1b5e537Sbmc 	dt_proc_release(dtp, P);
347a1b5e537Sbmc }
348a1b5e537Sbmc 
349a1b5e537Sbmc static void
dt_aggregate_umod(dtrace_hdl_t * dtp,uint64_t * data)350a1b5e537Sbmc dt_aggregate_umod(dtrace_hdl_t *dtp, uint64_t *data)
351a1b5e537Sbmc {
352a1b5e537Sbmc 	uint64_t pid = data[0];
353a1b5e537Sbmc 	uint64_t *pc = &data[1];
354a1b5e537Sbmc 	struct ps_prochandle *P;
355a1b5e537Sbmc 	const prmap_t *map;
356a1b5e537Sbmc 
357a1b5e537Sbmc 	if (dtp->dt_vector != NULL)
358a1b5e537Sbmc 		return;
359a1b5e537Sbmc 
360a1b5e537Sbmc 	if ((P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0)) == NULL)
361a1b5e537Sbmc 		return;
362a1b5e537Sbmc 
363a1b5e537Sbmc 	dt_proc_lock(dtp, P);
364a1b5e537Sbmc 
365a1b5e537Sbmc 	if ((map = Paddr_to_map(P, *pc)) != NULL)
366a1b5e537Sbmc 		*pc = map->pr_vaddr;
367a1b5e537Sbmc 
368a1b5e537Sbmc 	dt_proc_unlock(dtp, P);
369a1b5e537Sbmc 	dt_proc_release(dtp, P);
370a1b5e537Sbmc }
371a1b5e537Sbmc 
372a1b5e537Sbmc static void
dt_aggregate_sym(dtrace_hdl_t * dtp,uint64_t * data)373a1b5e537Sbmc dt_aggregate_sym(dtrace_hdl_t *dtp, uint64_t *data)
374a1b5e537Sbmc {
375a1b5e537Sbmc 	GElf_Sym sym;
376a1b5e537Sbmc 	uint64_t *pc = data;
377a1b5e537Sbmc 
378a1b5e537Sbmc 	if (dtrace_lookup_by_addr(dtp, *pc, &sym, NULL) == 0)
379a1b5e537Sbmc 		*pc = sym.st_value;
380a1b5e537Sbmc }
381a1b5e537Sbmc 
382a1b5e537Sbmc static void
dt_aggregate_mod(dtrace_hdl_t * dtp,uint64_t * data)383a1b5e537Sbmc dt_aggregate_mod(dtrace_hdl_t *dtp, uint64_t *data)
384a1b5e537Sbmc {
385a1b5e537Sbmc 	uint64_t *pc = data;
386a1b5e537Sbmc 	dt_module_t *dmp;
387a1b5e537Sbmc 
388a1b5e537Sbmc 	if (dtp->dt_vector != NULL) {
389a1b5e537Sbmc 		/*
390a1b5e537Sbmc 		 * We don't have a way of just getting the module for a
391a1b5e537Sbmc 		 * vectored open, and it doesn't seem to be worth defining
392a1b5e537Sbmc 		 * one.  This means that use of mod() won't get true
393a1b5e537Sbmc 		 * aggregation in the postmortem case (some modules may
394a1b5e537Sbmc 		 * appear more than once in aggregation output).  It seems
395a1b5e537Sbmc 		 * unlikely that anyone will ever notice or care...
396a1b5e537Sbmc 		 */
397a1b5e537Sbmc 		return;
398a1b5e537Sbmc 	}
399a1b5e537Sbmc 
400a1b5e537Sbmc 	for (dmp = dt_list_next(&dtp->dt_modlist); dmp != NULL;
401a1b5e537Sbmc 	    dmp = dt_list_next(dmp)) {
402a1b5e537Sbmc 		if (*pc - dmp->dm_text_va < dmp->dm_text_size) {
403a1b5e537Sbmc 			*pc = dmp->dm_text_va;
404a1b5e537Sbmc 			return;
405a1b5e537Sbmc 		}
406a1b5e537Sbmc 	}
407a1b5e537Sbmc }
408a1b5e537Sbmc 
40930ef842dSbmc static dtrace_aggvarid_t
dt_aggregate_aggvarid(dt_ahashent_t * ent)41030ef842dSbmc dt_aggregate_aggvarid(dt_ahashent_t *ent)
41130ef842dSbmc {
41230ef842dSbmc 	dtrace_aggdesc_t *agg = ent->dtahe_data.dtada_desc;
41330ef842dSbmc 	caddr_t data = ent->dtahe_data.dtada_data;
41430ef842dSbmc 	dtrace_recdesc_t *rec = agg->dtagd_rec;
41530ef842dSbmc 
41630ef842dSbmc 	/*
41730ef842dSbmc 	 * First, we'll check the variable ID in the aggdesc.  If it's valid,
41830ef842dSbmc 	 * we'll return it.  If not, we'll use the compiler-generated ID
41930ef842dSbmc 	 * present as the first record.
42030ef842dSbmc 	 */
42130ef842dSbmc 	if (agg->dtagd_varid != DTRACE_AGGVARIDNONE)
42230ef842dSbmc 		return (agg->dtagd_varid);
42330ef842dSbmc 
42430ef842dSbmc 	agg->dtagd_varid = *((dtrace_aggvarid_t *)(uintptr_t)(data +
42530ef842dSbmc 	    rec->dtrd_offset));
42630ef842dSbmc 
42730ef842dSbmc 	return (agg->dtagd_varid);
42830ef842dSbmc }
42930ef842dSbmc 
43030ef842dSbmc 
4317c478bd9Sstevel@tonic-gate static int
dt_aggregate_snap_cpu(dtrace_hdl_t * dtp,processorid_t cpu)4327c478bd9Sstevel@tonic-gate dt_aggregate_snap_cpu(dtrace_hdl_t *dtp, processorid_t cpu)
4337c478bd9Sstevel@tonic-gate {
4347c478bd9Sstevel@tonic-gate 	dtrace_epid_t id;
4357c478bd9Sstevel@tonic-gate 	uint64_t hashval;
4367c478bd9Sstevel@tonic-gate 	size_t offs, roffs, size, ndx;
4377c478bd9Sstevel@tonic-gate 	int i, j, rval;
4387c478bd9Sstevel@tonic-gate 	caddr_t addr, data;
4397c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *rec;
4407c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
4417c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *agg;
4427c478bd9Sstevel@tonic-gate 	dt_ahash_t *hash = &agp->dtat_hash;
4437c478bd9Sstevel@tonic-gate 	dt_ahashent_t *h;
4447c478bd9Sstevel@tonic-gate 	dtrace_bufdesc_t b = agp->dtat_buf, *buf = &b;
4457c478bd9Sstevel@tonic-gate 	dtrace_aggdata_t *aggdata;
4467c478bd9Sstevel@tonic-gate 	int flags = agp->dtat_flags;
4477c478bd9Sstevel@tonic-gate 
4487c478bd9Sstevel@tonic-gate 	buf->dtbd_cpu = cpu;
4497c478bd9Sstevel@tonic-gate 
4507c478bd9Sstevel@tonic-gate 	if (dt_ioctl(dtp, DTRACEIOC_AGGSNAP, buf) == -1) {
4517c478bd9Sstevel@tonic-gate 		if (errno == ENOENT) {
4527c478bd9Sstevel@tonic-gate 			/*
4537c478bd9Sstevel@tonic-gate 			 * If that failed with ENOENT, it may be because the
4547c478bd9Sstevel@tonic-gate 			 * CPU was unconfigured.  This is okay; we'll just
4557c478bd9Sstevel@tonic-gate 			 * do nothing but return success.
4567c478bd9Sstevel@tonic-gate 			 */
4577c478bd9Sstevel@tonic-gate 			return (0);
4587c478bd9Sstevel@tonic-gate 		}
4597c478bd9Sstevel@tonic-gate 
4607c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, errno));
4617c478bd9Sstevel@tonic-gate 	}
4627c478bd9Sstevel@tonic-gate 
4637c478bd9Sstevel@tonic-gate 	if (buf->dtbd_drops != 0) {
4647c478bd9Sstevel@tonic-gate 		if (dt_handle_cpudrop(dtp, cpu,
4657c478bd9Sstevel@tonic-gate 		    DTRACEDROP_AGGREGATION, buf->dtbd_drops) == -1)
4667c478bd9Sstevel@tonic-gate 			return (-1);
4677c478bd9Sstevel@tonic-gate 	}
4687c478bd9Sstevel@tonic-gate 
4697c478bd9Sstevel@tonic-gate 	if (buf->dtbd_size == 0)
4707c478bd9Sstevel@tonic-gate 		return (0);
4717c478bd9Sstevel@tonic-gate 
4727c478bd9Sstevel@tonic-gate 	if (hash->dtah_hash == NULL) {
4737c478bd9Sstevel@tonic-gate 		size_t size;
4747c478bd9Sstevel@tonic-gate 
4757c478bd9Sstevel@tonic-gate 		hash->dtah_size = DTRACE_AHASHSIZE;
4767c478bd9Sstevel@tonic-gate 		size = hash->dtah_size * sizeof (dt_ahashent_t *);
4777c478bd9Sstevel@tonic-gate 
4787c478bd9Sstevel@tonic-gate 		if ((hash->dtah_hash = malloc(size)) == NULL)
4797c478bd9Sstevel@tonic-gate 			return (dt_set_errno(dtp, EDT_NOMEM));
4807c478bd9Sstevel@tonic-gate 
4817c478bd9Sstevel@tonic-gate 		bzero(hash->dtah_hash, size);
4827c478bd9Sstevel@tonic-gate 	}
4837c478bd9Sstevel@tonic-gate 
4847c478bd9Sstevel@tonic-gate 	for (offs = 0; offs < buf->dtbd_size; ) {
4857c478bd9Sstevel@tonic-gate 		/*
4867c478bd9Sstevel@tonic-gate 		 * We're guaranteed to have an ID.
4877c478bd9Sstevel@tonic-gate 		 */
4887c478bd9Sstevel@tonic-gate 		id = *((dtrace_epid_t *)((uintptr_t)buf->dtbd_data +
4897c478bd9Sstevel@tonic-gate 		    (uintptr_t)offs));
4907c478bd9Sstevel@tonic-gate 
4917c478bd9Sstevel@tonic-gate 		if (id == DTRACE_AGGIDNONE) {
4927c478bd9Sstevel@tonic-gate 			/*
4937c478bd9Sstevel@tonic-gate 			 * This is filler to assure proper alignment of the
4947c478bd9Sstevel@tonic-gate 			 * next record; we simply ignore it.
4957c478bd9Sstevel@tonic-gate 			 */
4967c478bd9Sstevel@tonic-gate 			offs += sizeof (id);
4977c478bd9Sstevel@tonic-gate 			continue;
4987c478bd9Sstevel@tonic-gate 		}
4997c478bd9Sstevel@tonic-gate 
5007c478bd9Sstevel@tonic-gate 		if ((rval = dt_aggid_lookup(dtp, id, &agg)) != 0)
5017c478bd9Sstevel@tonic-gate 			return (rval);
5027c478bd9Sstevel@tonic-gate 
5037c478bd9Sstevel@tonic-gate 		addr = buf->dtbd_data + offs;
5047c478bd9Sstevel@tonic-gate 		size = agg->dtagd_size;
5057c478bd9Sstevel@tonic-gate 		hashval = 0;
5067c478bd9Sstevel@tonic-gate 
5077c478bd9Sstevel@tonic-gate 		for (j = 0; j < agg->dtagd_nrecs - 1; j++) {
5087c478bd9Sstevel@tonic-gate 			rec = &agg->dtagd_rec[j];
5097c478bd9Sstevel@tonic-gate 			roffs = rec->dtrd_offset;
5107c478bd9Sstevel@tonic-gate 
511a1b5e537Sbmc 			switch (rec->dtrd_action) {
512a1b5e537Sbmc 			case DTRACEACT_USYM:
513a1b5e537Sbmc 				dt_aggregate_usym(dtp,
514a1b5e537Sbmc 				    /* LINTED - alignment */
515a1b5e537Sbmc 				    (uint64_t *)&addr[roffs]);
516a1b5e537Sbmc 				break;
517a1b5e537Sbmc 
518a1b5e537Sbmc 			case DTRACEACT_UMOD:
519a1b5e537Sbmc 				dt_aggregate_umod(dtp,
520a1b5e537Sbmc 				    /* LINTED - alignment */
521a1b5e537Sbmc 				    (uint64_t *)&addr[roffs]);
522a1b5e537Sbmc 				break;
523a1b5e537Sbmc 
524a1b5e537Sbmc 			case DTRACEACT_SYM:
525a1b5e537Sbmc 				/* LINTED - alignment */
526a1b5e537Sbmc 				dt_aggregate_sym(dtp, (uint64_t *)&addr[roffs]);
527a1b5e537Sbmc 				break;
528a1b5e537Sbmc 
529a1b5e537Sbmc 			case DTRACEACT_MOD:
530a1b5e537Sbmc 				/* LINTED - alignment */
531a1b5e537Sbmc 				dt_aggregate_mod(dtp, (uint64_t *)&addr[roffs]);
532a1b5e537Sbmc 				break;
533a1b5e537Sbmc 
534a1b5e537Sbmc 			default:
535a1b5e537Sbmc 				break;
536a1b5e537Sbmc 			}
537a1b5e537Sbmc 
5387c478bd9Sstevel@tonic-gate 			for (i = 0; i < rec->dtrd_size; i++)
5397c478bd9Sstevel@tonic-gate 				hashval += addr[roffs + i];
5407c478bd9Sstevel@tonic-gate 		}
5417c478bd9Sstevel@tonic-gate 
5427c478bd9Sstevel@tonic-gate 		ndx = hashval % hash->dtah_size;
5437c478bd9Sstevel@tonic-gate 
5447c478bd9Sstevel@tonic-gate 		for (h = hash->dtah_hash[ndx]; h != NULL; h = h->dtahe_next) {
5457c478bd9Sstevel@tonic-gate 			if (h->dtahe_hashval != hashval)
5467c478bd9Sstevel@tonic-gate 				continue;
5477c478bd9Sstevel@tonic-gate 
5487c478bd9Sstevel@tonic-gate 			if (h->dtahe_size != size)
5497c478bd9Sstevel@tonic-gate 				continue;
5507c478bd9Sstevel@tonic-gate 
5517c478bd9Sstevel@tonic-gate 			aggdata = &h->dtahe_data;
5527c478bd9Sstevel@tonic-gate 			data = aggdata->dtada_data;
5537c478bd9Sstevel@tonic-gate 
5547c478bd9Sstevel@tonic-gate 			for (j = 0; j < agg->dtagd_nrecs - 1; j++) {
5557c478bd9Sstevel@tonic-gate 				rec = &agg->dtagd_rec[j];
5567c478bd9Sstevel@tonic-gate 				roffs = rec->dtrd_offset;
5577c478bd9Sstevel@tonic-gate 
5587c478bd9Sstevel@tonic-gate 				for (i = 0; i < rec->dtrd_size; i++)
5597c478bd9Sstevel@tonic-gate 					if (addr[roffs + i] != data[roffs + i])
5607c478bd9Sstevel@tonic-gate 						goto hashnext;
5617c478bd9Sstevel@tonic-gate 			}
5627c478bd9Sstevel@tonic-gate 
5637c478bd9Sstevel@tonic-gate 			/*
5647c478bd9Sstevel@tonic-gate 			 * We found it.  Now we need to apply the aggregating
5657c478bd9Sstevel@tonic-gate 			 * action on the data here.
5667c478bd9Sstevel@tonic-gate 			 */
5677c478bd9Sstevel@tonic-gate 			rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
5687c478bd9Sstevel@tonic-gate 			roffs = rec->dtrd_offset;
5697c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
570187eccf8Sbmc 			h->dtahe_aggregate((int64_t *)&data[roffs],
5717c478bd9Sstevel@tonic-gate 			    /* LINTED - alignment */
572187eccf8Sbmc 			    (int64_t *)&addr[roffs], rec->dtrd_size);
5737c478bd9Sstevel@tonic-gate 
5747c478bd9Sstevel@tonic-gate 			/*
5757c478bd9Sstevel@tonic-gate 			 * If we're keeping per CPU data, apply the aggregating
5767c478bd9Sstevel@tonic-gate 			 * action there as well.
5777c478bd9Sstevel@tonic-gate 			 */
5787c478bd9Sstevel@tonic-gate 			if (aggdata->dtada_percpu != NULL) {
5797c478bd9Sstevel@tonic-gate 				data = aggdata->dtada_percpu[cpu];
5807c478bd9Sstevel@tonic-gate 
5817c478bd9Sstevel@tonic-gate 				/* LINTED - alignment */
582187eccf8Sbmc 				h->dtahe_aggregate((int64_t *)data,
5837c478bd9Sstevel@tonic-gate 				    /* LINTED - alignment */
584187eccf8Sbmc 				    (int64_t *)&addr[roffs], rec->dtrd_size);
5857c478bd9Sstevel@tonic-gate 			}
5867c478bd9Sstevel@tonic-gate 
5877c478bd9Sstevel@tonic-gate 			goto bufnext;
5887c478bd9Sstevel@tonic-gate hashnext:
5897c478bd9Sstevel@tonic-gate 			continue;
5907c478bd9Sstevel@tonic-gate 		}
5917c478bd9Sstevel@tonic-gate 
5927c478bd9Sstevel@tonic-gate 		/*
5937c478bd9Sstevel@tonic-gate 		 * If we're here, we couldn't find an entry for this record.
5947c478bd9Sstevel@tonic-gate 		 */
5957c478bd9Sstevel@tonic-gate 		if ((h = malloc(sizeof (dt_ahashent_t))) == NULL)
5967c478bd9Sstevel@tonic-gate 			return (dt_set_errno(dtp, EDT_NOMEM));
5977c478bd9Sstevel@tonic-gate 		bzero(h, sizeof (dt_ahashent_t));
5987c478bd9Sstevel@tonic-gate 		aggdata = &h->dtahe_data;
5997c478bd9Sstevel@tonic-gate 
6007c478bd9Sstevel@tonic-gate 		if ((aggdata->dtada_data = malloc(size)) == NULL) {
6017c478bd9Sstevel@tonic-gate 			free(h);
6027c478bd9Sstevel@tonic-gate 			return (dt_set_errno(dtp, EDT_NOMEM));
6037c478bd9Sstevel@tonic-gate 		}
6047c478bd9Sstevel@tonic-gate 
6057c478bd9Sstevel@tonic-gate 		bcopy(addr, aggdata->dtada_data, size);
6067c478bd9Sstevel@tonic-gate 		aggdata->dtada_size = size;
6077c478bd9Sstevel@tonic-gate 		aggdata->dtada_desc = agg;
6087c478bd9Sstevel@tonic-gate 		aggdata->dtada_handle = dtp;
6097c478bd9Sstevel@tonic-gate 		(void) dt_epid_lookup(dtp, agg->dtagd_epid,
6107c478bd9Sstevel@tonic-gate 		    &aggdata->dtada_edesc, &aggdata->dtada_pdesc);
6117c478bd9Sstevel@tonic-gate 		aggdata->dtada_normal = 1;
6127c478bd9Sstevel@tonic-gate 
6137c478bd9Sstevel@tonic-gate 		h->dtahe_hashval = hashval;
6147c478bd9Sstevel@tonic-gate 		h->dtahe_size = size;
61530ef842dSbmc 		(void) dt_aggregate_aggvarid(h);
6167c478bd9Sstevel@tonic-gate 
6177c478bd9Sstevel@tonic-gate 		rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
6187c478bd9Sstevel@tonic-gate 
6197c478bd9Sstevel@tonic-gate 		if (flags & DTRACE_A_PERCPU) {
6207c478bd9Sstevel@tonic-gate 			int max_cpus = agp->dtat_maxcpu;
6217c478bd9Sstevel@tonic-gate 			caddr_t *percpu = malloc(max_cpus * sizeof (caddr_t));
6227c478bd9Sstevel@tonic-gate 
6237c478bd9Sstevel@tonic-gate 			if (percpu == NULL) {
6247c478bd9Sstevel@tonic-gate 				free(aggdata->dtada_data);
6257c478bd9Sstevel@tonic-gate 				free(h);
6267c478bd9Sstevel@tonic-gate 				return (dt_set_errno(dtp, EDT_NOMEM));
6277c478bd9Sstevel@tonic-gate 			}
6287c478bd9Sstevel@tonic-gate 
6297c478bd9Sstevel@tonic-gate 			for (j = 0; j < max_cpus; j++) {
6307c478bd9Sstevel@tonic-gate 				percpu[j] = malloc(rec->dtrd_size);
6317c478bd9Sstevel@tonic-gate 
6327c478bd9Sstevel@tonic-gate 				if (percpu[j] == NULL) {
6337c478bd9Sstevel@tonic-gate 					while (--j >= 0)
6347c478bd9Sstevel@tonic-gate 						free(percpu[j]);
6357c478bd9Sstevel@tonic-gate 
6367c478bd9Sstevel@tonic-gate 					free(aggdata->dtada_data);
6377c478bd9Sstevel@tonic-gate 					free(h);
6387c478bd9Sstevel@tonic-gate 					return (dt_set_errno(dtp, EDT_NOMEM));
6397c478bd9Sstevel@tonic-gate 				}
6407c478bd9Sstevel@tonic-gate 
6417c478bd9Sstevel@tonic-gate 				if (j == cpu) {
6427c478bd9Sstevel@tonic-gate 					bcopy(&addr[rec->dtrd_offset],
6437c478bd9Sstevel@tonic-gate 					    percpu[j], rec->dtrd_size);
6447c478bd9Sstevel@tonic-gate 				} else {
6457c478bd9Sstevel@tonic-gate 					bzero(percpu[j], rec->dtrd_size);
6467c478bd9Sstevel@tonic-gate 				}
6477c478bd9Sstevel@tonic-gate 			}
6487c478bd9Sstevel@tonic-gate 
6497c478bd9Sstevel@tonic-gate 			aggdata->dtada_percpu = percpu;
6507c478bd9Sstevel@tonic-gate 		}
6517c478bd9Sstevel@tonic-gate 
6527c478bd9Sstevel@tonic-gate 		switch (rec->dtrd_action) {
6537c478bd9Sstevel@tonic-gate 		case DTRACEAGG_MIN:
6547c478bd9Sstevel@tonic-gate 			h->dtahe_aggregate = dt_aggregate_min;
6557c478bd9Sstevel@tonic-gate 			break;
6567c478bd9Sstevel@tonic-gate 
6577c478bd9Sstevel@tonic-gate 		case DTRACEAGG_MAX:
6587c478bd9Sstevel@tonic-gate 			h->dtahe_aggregate = dt_aggregate_max;
6597c478bd9Sstevel@tonic-gate 			break;
6607c478bd9Sstevel@tonic-gate 
6617c478bd9Sstevel@tonic-gate 		case DTRACEAGG_LQUANTIZE:
6627c478bd9Sstevel@tonic-gate 			h->dtahe_aggregate = dt_aggregate_lquantize;
6637c478bd9Sstevel@tonic-gate 			break;
6647c478bd9Sstevel@tonic-gate 
6652b6389efSBryan Cantrill 		case DTRACEAGG_LLQUANTIZE:
6662b6389efSBryan Cantrill 			h->dtahe_aggregate = dt_aggregate_llquantize;
6672b6389efSBryan Cantrill 			break;
6682b6389efSBryan Cantrill 
6697c478bd9Sstevel@tonic-gate 		case DTRACEAGG_COUNT:
6707c478bd9Sstevel@tonic-gate 		case DTRACEAGG_SUM:
6717c478bd9Sstevel@tonic-gate 		case DTRACEAGG_AVG:
6726e0bee74Sjhaslam 		case DTRACEAGG_STDDEV:
6737c478bd9Sstevel@tonic-gate 		case DTRACEAGG_QUANTIZE:
6747c478bd9Sstevel@tonic-gate 			h->dtahe_aggregate = dt_aggregate_count;
6757c478bd9Sstevel@tonic-gate 			break;
6767c478bd9Sstevel@tonic-gate 
6777c478bd9Sstevel@tonic-gate 		default:
6787c478bd9Sstevel@tonic-gate 			return (dt_set_errno(dtp, EDT_BADAGG));
6797c478bd9Sstevel@tonic-gate 		}
6807c478bd9Sstevel@tonic-gate 
6817c478bd9Sstevel@tonic-gate 		if (hash->dtah_hash[ndx] != NULL)
6827c478bd9Sstevel@tonic-gate 			hash->dtah_hash[ndx]->dtahe_prev = h;
6837c478bd9Sstevel@tonic-gate 
6847c478bd9Sstevel@tonic-gate 		h->dtahe_next = hash->dtah_hash[ndx];
6857c478bd9Sstevel@tonic-gate 		hash->dtah_hash[ndx] = h;
6867c478bd9Sstevel@tonic-gate 
6877c478bd9Sstevel@tonic-gate 		if (hash->dtah_all != NULL)
6887c478bd9Sstevel@tonic-gate 			hash->dtah_all->dtahe_prevall = h;
6897c478bd9Sstevel@tonic-gate 
6907c478bd9Sstevel@tonic-gate 		h->dtahe_nextall = hash->dtah_all;
6917c478bd9Sstevel@tonic-gate 		hash->dtah_all = h;
6927c478bd9Sstevel@tonic-gate bufnext:
6937c478bd9Sstevel@tonic-gate 		offs += agg->dtagd_size;
6947c478bd9Sstevel@tonic-gate 	}
6957c478bd9Sstevel@tonic-gate 
6967c478bd9Sstevel@tonic-gate 	return (0);
6977c478bd9Sstevel@tonic-gate }
6987c478bd9Sstevel@tonic-gate 
6997c478bd9Sstevel@tonic-gate int
dtrace_aggregate_snap(dtrace_hdl_t * dtp)7007c478bd9Sstevel@tonic-gate dtrace_aggregate_snap(dtrace_hdl_t *dtp)
7017c478bd9Sstevel@tonic-gate {
7027c478bd9Sstevel@tonic-gate 	int i, rval;
7037c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
7047c478bd9Sstevel@tonic-gate 	hrtime_t now = gethrtime();
7057c478bd9Sstevel@tonic-gate 	dtrace_optval_t interval = dtp->dt_options[DTRACEOPT_AGGRATE];
7067c478bd9Sstevel@tonic-gate 
7077c478bd9Sstevel@tonic-gate 	if (dtp->dt_lastagg != 0) {
7087c478bd9Sstevel@tonic-gate 		if (now - dtp->dt_lastagg < interval)
7097c478bd9Sstevel@tonic-gate 			return (0);
7107c478bd9Sstevel@tonic-gate 
7117c478bd9Sstevel@tonic-gate 		dtp->dt_lastagg += interval;
7127c478bd9Sstevel@tonic-gate 	} else {
7137c478bd9Sstevel@tonic-gate 		dtp->dt_lastagg = now;
7147c478bd9Sstevel@tonic-gate 	}
7157c478bd9Sstevel@tonic-gate 
7167c478bd9Sstevel@tonic-gate 	if (!dtp->dt_active)
7177c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, EINVAL));
7187c478bd9Sstevel@tonic-gate 
7197c478bd9Sstevel@tonic-gate 	if (agp->dtat_buf.dtbd_size == 0)
7207c478bd9Sstevel@tonic-gate 		return (0);
7217c478bd9Sstevel@tonic-gate 
7227c478bd9Sstevel@tonic-gate 	for (i = 0; i < agp->dtat_ncpus; i++) {
7237c478bd9Sstevel@tonic-gate 		if (rval = dt_aggregate_snap_cpu(dtp, agp->dtat_cpus[i]))
7247c478bd9Sstevel@tonic-gate 			return (rval);
7257c478bd9Sstevel@tonic-gate 	}
7267c478bd9Sstevel@tonic-gate 
7277c478bd9Sstevel@tonic-gate 	return (0);
7287c478bd9Sstevel@tonic-gate }
7297c478bd9Sstevel@tonic-gate 
7307c478bd9Sstevel@tonic-gate static int
dt_aggregate_hashcmp(const void * lhs,const void * rhs)7317c478bd9Sstevel@tonic-gate dt_aggregate_hashcmp(const void *lhs, const void *rhs)
7327c478bd9Sstevel@tonic-gate {
7337c478bd9Sstevel@tonic-gate 	dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
7347c478bd9Sstevel@tonic-gate 	dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
7357c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
7367c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
7377c478bd9Sstevel@tonic-gate 
7387c478bd9Sstevel@tonic-gate 	if (lagg->dtagd_nrecs < ragg->dtagd_nrecs)
73930ef842dSbmc 		return (DT_LESSTHAN);
7407c478bd9Sstevel@tonic-gate 
7417c478bd9Sstevel@tonic-gate 	if (lagg->dtagd_nrecs > ragg->dtagd_nrecs)
74230ef842dSbmc 		return (DT_GREATERTHAN);
7437c478bd9Sstevel@tonic-gate 
7447c478bd9Sstevel@tonic-gate 	return (0);
7457c478bd9Sstevel@tonic-gate }
7467c478bd9Sstevel@tonic-gate 
7477c478bd9Sstevel@tonic-gate static int
dt_aggregate_varcmp(const void * lhs,const void * rhs)7487c478bd9Sstevel@tonic-gate dt_aggregate_varcmp(const void *lhs, const void *rhs)
7497c478bd9Sstevel@tonic-gate {
7507c478bd9Sstevel@tonic-gate 	dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
7517c478bd9Sstevel@tonic-gate 	dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
75230ef842dSbmc 	dtrace_aggvarid_t lid, rid;
7537c478bd9Sstevel@tonic-gate 
75430ef842dSbmc 	lid = dt_aggregate_aggvarid(lh);
75530ef842dSbmc 	rid = dt_aggregate_aggvarid(rh);
7567c478bd9Sstevel@tonic-gate 
7577c478bd9Sstevel@tonic-gate 	if (lid < rid)
75830ef842dSbmc 		return (DT_LESSTHAN);
7597c478bd9Sstevel@tonic-gate 
7607c478bd9Sstevel@tonic-gate 	if (lid > rid)
76130ef842dSbmc 		return (DT_GREATERTHAN);
7627c478bd9Sstevel@tonic-gate 
7637c478bd9Sstevel@tonic-gate 	return (0);
7647c478bd9Sstevel@tonic-gate }
7657c478bd9Sstevel@tonic-gate 
7667c478bd9Sstevel@tonic-gate static int
dt_aggregate_keycmp(const void * lhs,const void * rhs)7677c478bd9Sstevel@tonic-gate dt_aggregate_keycmp(const void *lhs, const void *rhs)
7687c478bd9Sstevel@tonic-gate {
7697c478bd9Sstevel@tonic-gate 	dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
7707c478bd9Sstevel@tonic-gate 	dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
7717c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
7727c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
7737c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *lrec, *rrec;
7747c478bd9Sstevel@tonic-gate 	char *ldata, *rdata;
77530ef842dSbmc 	int rval, i, j, keypos, nrecs;
7767c478bd9Sstevel@tonic-gate 
7777c478bd9Sstevel@tonic-gate 	if ((rval = dt_aggregate_hashcmp(lhs, rhs)) != 0)
7787c478bd9Sstevel@tonic-gate 		return (rval);
7797c478bd9Sstevel@tonic-gate 
78030ef842dSbmc 	nrecs = lagg->dtagd_nrecs - 1;
78130ef842dSbmc 	assert(nrecs == ragg->dtagd_nrecs - 1);
78230ef842dSbmc 
78330ef842dSbmc 	keypos = dt_keypos + 1 >= nrecs ? 0 : dt_keypos;
78430ef842dSbmc 
78530ef842dSbmc 	for (i = 1; i < nrecs; i++) {
7867c478bd9Sstevel@tonic-gate 		uint64_t lval, rval;
78730ef842dSbmc 		int ndx = i + keypos;
7887c478bd9Sstevel@tonic-gate 
78930ef842dSbmc 		if (ndx >= nrecs)
79030ef842dSbmc 			ndx = ndx - nrecs + 1;
79130ef842dSbmc 
79230ef842dSbmc 		lrec = &lagg->dtagd_rec[ndx];
79330ef842dSbmc 		rrec = &ragg->dtagd_rec[ndx];
7947c478bd9Sstevel@tonic-gate 
7957c478bd9Sstevel@tonic-gate 		ldata = lh->dtahe_data.dtada_data + lrec->dtrd_offset;
7967c478bd9Sstevel@tonic-gate 		rdata = rh->dtahe_data.dtada_data + rrec->dtrd_offset;
7977c478bd9Sstevel@tonic-gate 
7987c478bd9Sstevel@tonic-gate 		if (lrec->dtrd_size < rrec->dtrd_size)
79930ef842dSbmc 			return (DT_LESSTHAN);
8007c478bd9Sstevel@tonic-gate 
8017c478bd9Sstevel@tonic-gate 		if (lrec->dtrd_size > rrec->dtrd_size)
80230ef842dSbmc 			return (DT_GREATERTHAN);
8037c478bd9Sstevel@tonic-gate 
8047c478bd9Sstevel@tonic-gate 		switch (lrec->dtrd_size) {
8057c478bd9Sstevel@tonic-gate 		case sizeof (uint64_t):
8067c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
8077c478bd9Sstevel@tonic-gate 			lval = *((uint64_t *)ldata);
8087c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
8097c478bd9Sstevel@tonic-gate 			rval = *((uint64_t *)rdata);
8107c478bd9Sstevel@tonic-gate 			break;
8117c478bd9Sstevel@tonic-gate 
8127c478bd9Sstevel@tonic-gate 		case sizeof (uint32_t):
8137c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
8147c478bd9Sstevel@tonic-gate 			lval = *((uint32_t *)ldata);
8157c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
8167c478bd9Sstevel@tonic-gate 			rval = *((uint32_t *)rdata);
8177c478bd9Sstevel@tonic-gate 			break;
8187c478bd9Sstevel@tonic-gate 
8197c478bd9Sstevel@tonic-gate 		case sizeof (uint16_t):
8207c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
8217c478bd9Sstevel@tonic-gate 			lval = *((uint16_t *)ldata);
8227c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
8237c478bd9Sstevel@tonic-gate 			rval = *((uint16_t *)rdata);
8247c478bd9Sstevel@tonic-gate 			break;
8257c478bd9Sstevel@tonic-gate 
8267c478bd9Sstevel@tonic-gate 		case sizeof (uint8_t):
8277c478bd9Sstevel@tonic-gate 			lval = *((uint8_t *)ldata);
8287c478bd9Sstevel@tonic-gate 			rval = *((uint8_t *)rdata);
8297c478bd9Sstevel@tonic-gate 			break;
8307c478bd9Sstevel@tonic-gate 
8317c478bd9Sstevel@tonic-gate 		default:
832127bbe13Stomee 			switch (lrec->dtrd_action) {
833127bbe13Stomee 			case DTRACEACT_UMOD:
834127bbe13Stomee 			case DTRACEACT_UADDR:
835127bbe13Stomee 			case DTRACEACT_USYM:
836127bbe13Stomee 				for (j = 0; j < 2; j++) {
837127bbe13Stomee 					/* LINTED - alignment */
838127bbe13Stomee 					lval = ((uint64_t *)ldata)[j];
839127bbe13Stomee 					/* LINTED - alignment */
840127bbe13Stomee 					rval = ((uint64_t *)rdata)[j];
841127bbe13Stomee 
842127bbe13Stomee 					if (lval < rval)
843127bbe13Stomee 						return (DT_LESSTHAN);
844127bbe13Stomee 
845127bbe13Stomee 					if (lval > rval)
846127bbe13Stomee 						return (DT_GREATERTHAN);
847127bbe13Stomee 				}
8487c478bd9Sstevel@tonic-gate 
849127bbe13Stomee 				break;
8507c478bd9Sstevel@tonic-gate 
851127bbe13Stomee 			default:
852127bbe13Stomee 				for (j = 0; j < lrec->dtrd_size; j++) {
853127bbe13Stomee 					lval = ((uint8_t *)ldata)[j];
854127bbe13Stomee 					rval = ((uint8_t *)rdata)[j];
85530ef842dSbmc 
856127bbe13Stomee 					if (lval < rval)
857127bbe13Stomee 						return (DT_LESSTHAN);
858127bbe13Stomee 
859127bbe13Stomee 					if (lval > rval)
860127bbe13Stomee 						return (DT_GREATERTHAN);
861127bbe13Stomee 				}
8627c478bd9Sstevel@tonic-gate 			}
8637c478bd9Sstevel@tonic-gate 
8647c478bd9Sstevel@tonic-gate 			continue;
8657c478bd9Sstevel@tonic-gate 		}
8667c478bd9Sstevel@tonic-gate 
8677c478bd9Sstevel@tonic-gate 		if (lval < rval)
86830ef842dSbmc 			return (DT_LESSTHAN);
8697c478bd9Sstevel@tonic-gate 
8707c478bd9Sstevel@tonic-gate 		if (lval > rval)
87130ef842dSbmc 			return (DT_GREATERTHAN);
8727c478bd9Sstevel@tonic-gate 	}
8737c478bd9Sstevel@tonic-gate 
8747c478bd9Sstevel@tonic-gate 	return (0);
8757c478bd9Sstevel@tonic-gate }
8767c478bd9Sstevel@tonic-gate 
8777c478bd9Sstevel@tonic-gate static int
dt_aggregate_valcmp(const void * lhs,const void * rhs)8787c478bd9Sstevel@tonic-gate dt_aggregate_valcmp(const void *lhs, const void *rhs)
8797c478bd9Sstevel@tonic-gate {
8807c478bd9Sstevel@tonic-gate 	dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
8817c478bd9Sstevel@tonic-gate 	dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
8827c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
8837c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
8847c478bd9Sstevel@tonic-gate 	caddr_t ldata = lh->dtahe_data.dtada_data;
8857c478bd9Sstevel@tonic-gate 	caddr_t rdata = rh->dtahe_data.dtada_data;
8867c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *lrec, *rrec;
887187eccf8Sbmc 	int64_t *laddr, *raddr;
888e5803b76SAdam H. Leventhal 	int rval;
8897c478bd9Sstevel@tonic-gate 
890e5803b76SAdam H. Leventhal 	assert(lagg->dtagd_nrecs == ragg->dtagd_nrecs);
8917c478bd9Sstevel@tonic-gate 
892e5803b76SAdam H. Leventhal 	lrec = &lagg->dtagd_rec[lagg->dtagd_nrecs - 1];
893e5803b76SAdam H. Leventhal 	rrec = &ragg->dtagd_rec[ragg->dtagd_nrecs - 1];
8947c478bd9Sstevel@tonic-gate 
895e5803b76SAdam H. Leventhal 	assert(lrec->dtrd_action == rrec->dtrd_action);
8967c478bd9Sstevel@tonic-gate 
897187eccf8Sbmc 	laddr = (int64_t *)(uintptr_t)(ldata + lrec->dtrd_offset);
898187eccf8Sbmc 	raddr = (int64_t *)(uintptr_t)(rdata + rrec->dtrd_offset);
8997c478bd9Sstevel@tonic-gate 
9007c478bd9Sstevel@tonic-gate 	switch (lrec->dtrd_action) {
9017c478bd9Sstevel@tonic-gate 	case DTRACEAGG_AVG:
9027c478bd9Sstevel@tonic-gate 		rval = dt_aggregate_averagecmp(laddr, raddr);
9037c478bd9Sstevel@tonic-gate 		break;
9047c478bd9Sstevel@tonic-gate 
9056e0bee74Sjhaslam 	case DTRACEAGG_STDDEV:
9066e0bee74Sjhaslam 		rval = dt_aggregate_stddevcmp(laddr, raddr);
9076e0bee74Sjhaslam 		break;
9086e0bee74Sjhaslam 
9097c478bd9Sstevel@tonic-gate 	case DTRACEAGG_QUANTIZE:
9107c478bd9Sstevel@tonic-gate 		rval = dt_aggregate_quantizedcmp(laddr, raddr);
9117c478bd9Sstevel@tonic-gate 		break;
9127c478bd9Sstevel@tonic-gate 
9137c478bd9Sstevel@tonic-gate 	case DTRACEAGG_LQUANTIZE:
9147c478bd9Sstevel@tonic-gate 		rval = dt_aggregate_lquantizedcmp(laddr, raddr);
9157c478bd9Sstevel@tonic-gate 		break;
9167c478bd9Sstevel@tonic-gate 
9172b6389efSBryan Cantrill 	case DTRACEAGG_LLQUANTIZE:
9182b6389efSBryan Cantrill 		rval = dt_aggregate_llquantizedcmp(laddr, raddr);
9192b6389efSBryan Cantrill 		break;
9202b6389efSBryan Cantrill 
9217c478bd9Sstevel@tonic-gate 	case DTRACEAGG_COUNT:
9227c478bd9Sstevel@tonic-gate 	case DTRACEAGG_SUM:
9237c478bd9Sstevel@tonic-gate 	case DTRACEAGG_MIN:
9247c478bd9Sstevel@tonic-gate 	case DTRACEAGG_MAX:
9257c478bd9Sstevel@tonic-gate 		rval = dt_aggregate_countcmp(laddr, raddr);
9267c478bd9Sstevel@tonic-gate 		break;
9277c478bd9Sstevel@tonic-gate 
9287c478bd9Sstevel@tonic-gate 	default:
9297c478bd9Sstevel@tonic-gate 		assert(0);
9307c478bd9Sstevel@tonic-gate 	}
9317c478bd9Sstevel@tonic-gate 
93230ef842dSbmc 	return (rval);
93330ef842dSbmc }
93430ef842dSbmc 
93530ef842dSbmc static int
dt_aggregate_valkeycmp(const void * lhs,const void * rhs)93630ef842dSbmc dt_aggregate_valkeycmp(const void *lhs, const void *rhs)
93730ef842dSbmc {
93830ef842dSbmc 	int rval;
93930ef842dSbmc 
94030ef842dSbmc 	if ((rval = dt_aggregate_valcmp(lhs, rhs)) != 0)
9417c478bd9Sstevel@tonic-gate 		return (rval);
9427c478bd9Sstevel@tonic-gate 
9437c478bd9Sstevel@tonic-gate 	/*
9447c478bd9Sstevel@tonic-gate 	 * If we're here, the values for the two aggregation elements are
9457c478bd9Sstevel@tonic-gate 	 * equal.  We already know that the key layout is the same for the two
9467c478bd9Sstevel@tonic-gate 	 * elements; we must now compare the keys themselves as a tie-breaker.
9477c478bd9Sstevel@tonic-gate 	 */
9487c478bd9Sstevel@tonic-gate 	return (dt_aggregate_keycmp(lhs, rhs));
9497c478bd9Sstevel@tonic-gate }
9507c478bd9Sstevel@tonic-gate 
9517c478bd9Sstevel@tonic-gate static int
dt_aggregate_keyvarcmp(const void * lhs,const void * rhs)9527c478bd9Sstevel@tonic-gate dt_aggregate_keyvarcmp(const void *lhs, const void *rhs)
9537c478bd9Sstevel@tonic-gate {
9547c478bd9Sstevel@tonic-gate 	int rval;
9557c478bd9Sstevel@tonic-gate 
9567c478bd9Sstevel@tonic-gate 	if ((rval = dt_aggregate_keycmp(lhs, rhs)) != 0)
9577c478bd9Sstevel@tonic-gate 		return (rval);
9587c478bd9Sstevel@tonic-gate 
9597c478bd9Sstevel@tonic-gate 	return (dt_aggregate_varcmp(lhs, rhs));
9607c478bd9Sstevel@tonic-gate }
9617c478bd9Sstevel@tonic-gate 
9627c478bd9Sstevel@tonic-gate static int
dt_aggregate_varkeycmp(const void * lhs,const void * rhs)9637c478bd9Sstevel@tonic-gate dt_aggregate_varkeycmp(const void *lhs, const void *rhs)
9647c478bd9Sstevel@tonic-gate {
9657c478bd9Sstevel@tonic-gate 	int rval;
9667c478bd9Sstevel@tonic-gate 
9677c478bd9Sstevel@tonic-gate 	if ((rval = dt_aggregate_varcmp(lhs, rhs)) != 0)
9687c478bd9Sstevel@tonic-gate 		return (rval);
9697c478bd9Sstevel@tonic-gate 
9707c478bd9Sstevel@tonic-gate 	return (dt_aggregate_keycmp(lhs, rhs));
9717c478bd9Sstevel@tonic-gate }
9727c478bd9Sstevel@tonic-gate 
9737c478bd9Sstevel@tonic-gate static int
dt_aggregate_valvarcmp(const void * lhs,const void * rhs)9747c478bd9Sstevel@tonic-gate dt_aggregate_valvarcmp(const void *lhs, const void *rhs)
9757c478bd9Sstevel@tonic-gate {
9767c478bd9Sstevel@tonic-gate 	int rval;
9777c478bd9Sstevel@tonic-gate 
97830ef842dSbmc 	if ((rval = dt_aggregate_valkeycmp(lhs, rhs)) != 0)
9797c478bd9Sstevel@tonic-gate 		return (rval);
9807c478bd9Sstevel@tonic-gate 
9817c478bd9Sstevel@tonic-gate 	return (dt_aggregate_varcmp(lhs, rhs));
9827c478bd9Sstevel@tonic-gate }
9837c478bd9Sstevel@tonic-gate 
9847c478bd9Sstevel@tonic-gate static int
dt_aggregate_varvalcmp(const void * lhs,const void * rhs)9857c478bd9Sstevel@tonic-gate dt_aggregate_varvalcmp(const void *lhs, const void *rhs)
9867c478bd9Sstevel@tonic-gate {
9877c478bd9Sstevel@tonic-gate 	int rval;
9887c478bd9Sstevel@tonic-gate 
9897c478bd9Sstevel@tonic-gate 	if ((rval = dt_aggregate_varcmp(lhs, rhs)) != 0)
9907c478bd9Sstevel@tonic-gate 		return (rval);
9917c478bd9Sstevel@tonic-gate 
99230ef842dSbmc 	return (dt_aggregate_valkeycmp(lhs, rhs));
9937c478bd9Sstevel@tonic-gate }
9947c478bd9Sstevel@tonic-gate 
9957c478bd9Sstevel@tonic-gate static int
dt_aggregate_keyvarrevcmp(const void * lhs,const void * rhs)9967c478bd9Sstevel@tonic-gate dt_aggregate_keyvarrevcmp(const void *lhs, const void *rhs)
9977c478bd9Sstevel@tonic-gate {
9987c478bd9Sstevel@tonic-gate 	return (dt_aggregate_keyvarcmp(rhs, lhs));
9997c478bd9Sstevel@tonic-gate }
10007c478bd9Sstevel@tonic-gate 
10017c478bd9Sstevel@tonic-gate static int
dt_aggregate_varkeyrevcmp(const void * lhs,const void * rhs)10027c478bd9Sstevel@tonic-gate dt_aggregate_varkeyrevcmp(const void *lhs, const void *rhs)
10037c478bd9Sstevel@tonic-gate {
10047c478bd9Sstevel@tonic-gate 	return (dt_aggregate_varkeycmp(rhs, lhs));
10057c478bd9Sstevel@tonic-gate }
10067c478bd9Sstevel@tonic-gate 
10077c478bd9Sstevel@tonic-gate static int
dt_aggregate_valvarrevcmp(const void * lhs,const void * rhs)10087c478bd9Sstevel@tonic-gate dt_aggregate_valvarrevcmp(const void *lhs, const void *rhs)
10097c478bd9Sstevel@tonic-gate {
10107c478bd9Sstevel@tonic-gate 	return (dt_aggregate_valvarcmp(rhs, lhs));
10117c478bd9Sstevel@tonic-gate }
10127c478bd9Sstevel@tonic-gate 
10137c478bd9Sstevel@tonic-gate static int
dt_aggregate_varvalrevcmp(const void * lhs,const void * rhs)10147c478bd9Sstevel@tonic-gate dt_aggregate_varvalrevcmp(const void *lhs, const void *rhs)
10157c478bd9Sstevel@tonic-gate {
10167c478bd9Sstevel@tonic-gate 	return (dt_aggregate_varvalcmp(rhs, lhs));
10177c478bd9Sstevel@tonic-gate }
10187c478bd9Sstevel@tonic-gate 
101930ef842dSbmc static int
dt_aggregate_bundlecmp(const void * lhs,const void * rhs)102030ef842dSbmc dt_aggregate_bundlecmp(const void *lhs, const void *rhs)
102130ef842dSbmc {
102230ef842dSbmc 	dt_ahashent_t **lh = *((dt_ahashent_t ***)lhs);
102330ef842dSbmc 	dt_ahashent_t **rh = *((dt_ahashent_t ***)rhs);
102430ef842dSbmc 	int i, rval;
102530ef842dSbmc 
102630ef842dSbmc 	if (dt_keysort) {
102730ef842dSbmc 		/*
102830ef842dSbmc 		 * If we're sorting on keys, we need to scan until we find the
102930ef842dSbmc 		 * last entry -- that's the representative key.  (The order of
103030ef842dSbmc 		 * the bundle is values followed by key to accommodate the
103130ef842dSbmc 		 * default behavior of sorting by value.)  If the keys are
103230ef842dSbmc 		 * equal, we'll fall into the value comparison loop, below.
103330ef842dSbmc 		 */
103430ef842dSbmc 		for (i = 0; lh[i + 1] != NULL; i++)
103530ef842dSbmc 			continue;
103630ef842dSbmc 
103730ef842dSbmc 		assert(i != 0);
103830ef842dSbmc 		assert(rh[i + 1] == NULL);
103930ef842dSbmc 
104030ef842dSbmc 		if ((rval = dt_aggregate_keycmp(&lh[i], &rh[i])) != 0)
104130ef842dSbmc 			return (rval);
104230ef842dSbmc 	}
104330ef842dSbmc 
104430ef842dSbmc 	for (i = 0; ; i++) {
104530ef842dSbmc 		if (lh[i + 1] == NULL) {
104630ef842dSbmc 			/*
104730ef842dSbmc 			 * All of the values are equal; if we're sorting on
104830ef842dSbmc 			 * keys, then we're only here because the keys were
104930ef842dSbmc 			 * found to be equal and these records are therefore
105030ef842dSbmc 			 * equal.  If we're not sorting on keys, we'll use the
105130ef842dSbmc 			 * key comparison from the representative key as the
105230ef842dSbmc 			 * tie-breaker.
105330ef842dSbmc 			 */
105430ef842dSbmc 			if (dt_keysort)
105530ef842dSbmc 				return (0);
105630ef842dSbmc 
105730ef842dSbmc 			assert(i != 0);
105830ef842dSbmc 			assert(rh[i + 1] == NULL);
105930ef842dSbmc 			return (dt_aggregate_keycmp(&lh[i], &rh[i]));
106030ef842dSbmc 		} else {
106130ef842dSbmc 			if ((rval = dt_aggregate_valcmp(&lh[i], &rh[i])) != 0)
106230ef842dSbmc 				return (rval);
106330ef842dSbmc 		}
106430ef842dSbmc 	}
106530ef842dSbmc }
106630ef842dSbmc 
10677c478bd9Sstevel@tonic-gate int
dt_aggregate_go(dtrace_hdl_t * dtp)10687c478bd9Sstevel@tonic-gate dt_aggregate_go(dtrace_hdl_t *dtp)
10697c478bd9Sstevel@tonic-gate {
10707c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
10717c478bd9Sstevel@tonic-gate 	dtrace_optval_t size, cpu;
10727c478bd9Sstevel@tonic-gate 	dtrace_bufdesc_t *buf = &agp->dtat_buf;
10737c478bd9Sstevel@tonic-gate 	int rval, i;
10747c478bd9Sstevel@tonic-gate 
10757c478bd9Sstevel@tonic-gate 	assert(agp->dtat_maxcpu == 0);
10767c478bd9Sstevel@tonic-gate 	assert(agp->dtat_ncpu == 0);
10777c478bd9Sstevel@tonic-gate 	assert(agp->dtat_cpus == NULL);
10787c478bd9Sstevel@tonic-gate 
10797c478bd9Sstevel@tonic-gate 	agp->dtat_maxcpu = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
10807c478bd9Sstevel@tonic-gate 	agp->dtat_ncpu = dt_sysconf(dtp, _SC_NPROCESSORS_MAX);
10817c478bd9Sstevel@tonic-gate 	agp->dtat_cpus = malloc(agp->dtat_ncpu * sizeof (processorid_t));
10827c478bd9Sstevel@tonic-gate 
10837c478bd9Sstevel@tonic-gate 	if (agp->dtat_cpus == NULL)
10847c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, EDT_NOMEM));
10857c478bd9Sstevel@tonic-gate 
10867c478bd9Sstevel@tonic-gate 	/*
10877c478bd9Sstevel@tonic-gate 	 * Use the aggregation buffer size as reloaded from the kernel.
10887c478bd9Sstevel@tonic-gate 	 */
10897c478bd9Sstevel@tonic-gate 	size = dtp->dt_options[DTRACEOPT_AGGSIZE];
10907c478bd9Sstevel@tonic-gate 
10917c478bd9Sstevel@tonic-gate 	rval = dtrace_getopt(dtp, "aggsize", &size);
10927c478bd9Sstevel@tonic-gate 	assert(rval == 0);
10937c478bd9Sstevel@tonic-gate 
10947c478bd9Sstevel@tonic-gate 	if (size == 0 || size == DTRACEOPT_UNSET)
10957c478bd9Sstevel@tonic-gate 		return (0);
10967c478bd9Sstevel@tonic-gate 
10977c478bd9Sstevel@tonic-gate 	buf = &agp->dtat_buf;
10987c478bd9Sstevel@tonic-gate 	buf->dtbd_size = size;
10997c478bd9Sstevel@tonic-gate 
11007c478bd9Sstevel@tonic-gate 	if ((buf->dtbd_data = malloc(buf->dtbd_size)) == NULL)
11017c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, EDT_NOMEM));
11027c478bd9Sstevel@tonic-gate 
11037c478bd9Sstevel@tonic-gate 	/*
11047c478bd9Sstevel@tonic-gate 	 * Now query for the CPUs enabled.
11057c478bd9Sstevel@tonic-gate 	 */
11067c478bd9Sstevel@tonic-gate 	rval = dtrace_getopt(dtp, "cpu", &cpu);
11077c478bd9Sstevel@tonic-gate 	assert(rval == 0 && cpu != DTRACEOPT_UNSET);
11087c478bd9Sstevel@tonic-gate 
11097c478bd9Sstevel@tonic-gate 	if (cpu != DTRACE_CPUALL) {
11107c478bd9Sstevel@tonic-gate 		assert(cpu < agp->dtat_ncpu);
11117c478bd9Sstevel@tonic-gate 		agp->dtat_cpus[agp->dtat_ncpus++] = (processorid_t)cpu;
11127c478bd9Sstevel@tonic-gate 
11137c478bd9Sstevel@tonic-gate 		return (0);
11147c478bd9Sstevel@tonic-gate 	}
11157c478bd9Sstevel@tonic-gate 
11167c478bd9Sstevel@tonic-gate 	agp->dtat_ncpus = 0;
11177c478bd9Sstevel@tonic-gate 	for (i = 0; i < agp->dtat_maxcpu; i++) {
11187c478bd9Sstevel@tonic-gate 		if (dt_status(dtp, i) == -1)
11197c478bd9Sstevel@tonic-gate 			continue;
11207c478bd9Sstevel@tonic-gate 
11217c478bd9Sstevel@tonic-gate 		agp->dtat_cpus[agp->dtat_ncpus++] = i;
11227c478bd9Sstevel@tonic-gate 	}
11237c478bd9Sstevel@tonic-gate 
11247c478bd9Sstevel@tonic-gate 	return (0);
11257c478bd9Sstevel@tonic-gate }
11267c478bd9Sstevel@tonic-gate 
11277c478bd9Sstevel@tonic-gate static int
dt_aggwalk_rval(dtrace_hdl_t * dtp,dt_ahashent_t * h,int rval)11287c478bd9Sstevel@tonic-gate dt_aggwalk_rval(dtrace_hdl_t *dtp, dt_ahashent_t *h, int rval)
11297c478bd9Sstevel@tonic-gate {
11307c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
11317c478bd9Sstevel@tonic-gate 	dtrace_aggdata_t *data;
11327c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *aggdesc;
11337c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *rec;
11347c478bd9Sstevel@tonic-gate 	int i;
11357c478bd9Sstevel@tonic-gate 
11367c478bd9Sstevel@tonic-gate 	switch (rval) {
11377c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_NEXT:
11387c478bd9Sstevel@tonic-gate 		break;
11397c478bd9Sstevel@tonic-gate 
11407c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_CLEAR: {
11417c478bd9Sstevel@tonic-gate 		uint32_t size, offs = 0;
11427c478bd9Sstevel@tonic-gate 
11437c478bd9Sstevel@tonic-gate 		aggdesc = h->dtahe_data.dtada_desc;
11447c478bd9Sstevel@tonic-gate 		rec = &aggdesc->dtagd_rec[aggdesc->dtagd_nrecs - 1];
11457c478bd9Sstevel@tonic-gate 		size = rec->dtrd_size;
11467c478bd9Sstevel@tonic-gate 		data = &h->dtahe_data;
11477c478bd9Sstevel@tonic-gate 
1148*0ddc0ebbSBryan Cantrill 		if (rec->dtrd_action == DTRACEAGG_LQUANTIZE ||
1149*0ddc0ebbSBryan Cantrill 		    rec->dtrd_action == DTRACEAGG_LLQUANTIZE) {
1150*0ddc0ebbSBryan Cantrill 			/*
1151*0ddc0ebbSBryan Cantrill 			 * For lquantize() and llquantize(), we want to be
1152*0ddc0ebbSBryan Cantrill 			 * sure to not zero the aggregation parameters; step
1153*0ddc0ebbSBryan Cantrill 			 * over them and adjust our size accordingly.
1154*0ddc0ebbSBryan Cantrill 			 */
11557c478bd9Sstevel@tonic-gate 			offs = sizeof (uint64_t);
11567c478bd9Sstevel@tonic-gate 			size -= sizeof (uint64_t);
11577c478bd9Sstevel@tonic-gate 		}
11587c478bd9Sstevel@tonic-gate 
11597c478bd9Sstevel@tonic-gate 		bzero(&data->dtada_data[rec->dtrd_offset] + offs, size);
11607c478bd9Sstevel@tonic-gate 
11617c478bd9Sstevel@tonic-gate 		if (data->dtada_percpu == NULL)
11627c478bd9Sstevel@tonic-gate 			break;
11637c478bd9Sstevel@tonic-gate 
11647c478bd9Sstevel@tonic-gate 		for (i = 0; i < dtp->dt_aggregate.dtat_maxcpu; i++)
11657c478bd9Sstevel@tonic-gate 			bzero(data->dtada_percpu[i] + offs, size);
11667c478bd9Sstevel@tonic-gate 		break;
11677c478bd9Sstevel@tonic-gate 	}
11687c478bd9Sstevel@tonic-gate 
11697c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_ERROR:
11707c478bd9Sstevel@tonic-gate 		/*
11717c478bd9Sstevel@tonic-gate 		 * We assume that errno is already set in this case.
11727c478bd9Sstevel@tonic-gate 		 */
11737c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, errno));
11747c478bd9Sstevel@tonic-gate 
11757c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_ABORT:
11767c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, EDT_DIRABORT));
11777c478bd9Sstevel@tonic-gate 
11787c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_DENORMALIZE:
11797c478bd9Sstevel@tonic-gate 		h->dtahe_data.dtada_normal = 1;
11807c478bd9Sstevel@tonic-gate 		return (0);
11817c478bd9Sstevel@tonic-gate 
11827c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_NORMALIZE:
11837c478bd9Sstevel@tonic-gate 		if (h->dtahe_data.dtada_normal == 0) {
11847c478bd9Sstevel@tonic-gate 			h->dtahe_data.dtada_normal = 1;
11857c478bd9Sstevel@tonic-gate 			return (dt_set_errno(dtp, EDT_BADRVAL));
11867c478bd9Sstevel@tonic-gate 		}
11877c478bd9Sstevel@tonic-gate 
11887c478bd9Sstevel@tonic-gate 		return (0);
11897c478bd9Sstevel@tonic-gate 
11907c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_REMOVE: {
11917c478bd9Sstevel@tonic-gate 		dtrace_aggdata_t *aggdata = &h->dtahe_data;
11927c478bd9Sstevel@tonic-gate 		int i, max_cpus = agp->dtat_maxcpu;
11937c478bd9Sstevel@tonic-gate 
11947c478bd9Sstevel@tonic-gate 		/*
11957c478bd9Sstevel@tonic-gate 		 * First, remove this hash entry from its hash chain.
11967c478bd9Sstevel@tonic-gate 		 */
11977c478bd9Sstevel@tonic-gate 		if (h->dtahe_prev != NULL) {
11987c478bd9Sstevel@tonic-gate 			h->dtahe_prev->dtahe_next = h->dtahe_next;
11997c478bd9Sstevel@tonic-gate 		} else {
12007c478bd9Sstevel@tonic-gate 			dt_ahash_t *hash = &agp->dtat_hash;
12017c478bd9Sstevel@tonic-gate 			size_t ndx = h->dtahe_hashval % hash->dtah_size;
12027c478bd9Sstevel@tonic-gate 
12037c478bd9Sstevel@tonic-gate 			assert(hash->dtah_hash[ndx] == h);
12047c478bd9Sstevel@tonic-gate 			hash->dtah_hash[ndx] = h->dtahe_next;
12057c478bd9Sstevel@tonic-gate 		}
12067c478bd9Sstevel@tonic-gate 
12077c478bd9Sstevel@tonic-gate 		if (h->dtahe_next != NULL)
12087c478bd9Sstevel@tonic-gate 			h->dtahe_next->dtahe_prev = h->dtahe_prev;
12097c478bd9Sstevel@tonic-gate 
12107c478bd9Sstevel@tonic-gate 		/*
12117c478bd9Sstevel@tonic-gate 		 * Now remove it from the list of all hash entries.
12127c478bd9Sstevel@tonic-gate 		 */
12137c478bd9Sstevel@tonic-gate 		if (h->dtahe_prevall != NULL) {
12147c478bd9Sstevel@tonic-gate 			h->dtahe_prevall->dtahe_nextall = h->dtahe_nextall;
12157c478bd9Sstevel@tonic-gate 		} else {
12167c478bd9Sstevel@tonic-gate 			dt_ahash_t *hash = &agp->dtat_hash;
12177c478bd9Sstevel@tonic-gate 
12187c478bd9Sstevel@tonic-gate 			assert(hash->dtah_all == h);
12197c478bd9Sstevel@tonic-gate 			hash->dtah_all = h->dtahe_nextall;
12207c478bd9Sstevel@tonic-gate 		}
12217c478bd9Sstevel@tonic-gate 
12227c478bd9Sstevel@tonic-gate 		if (h->dtahe_nextall != NULL)
12237c478bd9Sstevel@tonic-gate 			h->dtahe_nextall->dtahe_prevall = h->dtahe_prevall;
12247c478bd9Sstevel@tonic-gate 
12257c478bd9Sstevel@tonic-gate 		/*
12267c478bd9Sstevel@tonic-gate 		 * We're unlinked.  We can safely destroy the data.
12277c478bd9Sstevel@tonic-gate 		 */
12287c478bd9Sstevel@tonic-gate 		if (aggdata->dtada_percpu != NULL) {
12297c478bd9Sstevel@tonic-gate 			for (i = 0; i < max_cpus; i++)
12307c478bd9Sstevel@tonic-gate 				free(aggdata->dtada_percpu[i]);
12317c478bd9Sstevel@tonic-gate 			free(aggdata->dtada_percpu);
12327c478bd9Sstevel@tonic-gate 		}
12337c478bd9Sstevel@tonic-gate 
12347c478bd9Sstevel@tonic-gate 		free(aggdata->dtada_data);
12357c478bd9Sstevel@tonic-gate 		free(h);
12367c478bd9Sstevel@tonic-gate 
12377c478bd9Sstevel@tonic-gate 		return (0);
12387c478bd9Sstevel@tonic-gate 	}
12397c478bd9Sstevel@tonic-gate 
12407c478bd9Sstevel@tonic-gate 	default:
12417c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, EDT_BADRVAL));
12427c478bd9Sstevel@tonic-gate 	}
12437c478bd9Sstevel@tonic-gate 
12447c478bd9Sstevel@tonic-gate 	return (0);
12457c478bd9Sstevel@tonic-gate }
12467c478bd9Sstevel@tonic-gate 
124730ef842dSbmc void
dt_aggregate_qsort(dtrace_hdl_t * dtp,void * base,size_t nel,size_t width,int (* compar)(const void *,const void *))124830ef842dSbmc dt_aggregate_qsort(dtrace_hdl_t *dtp, void *base, size_t nel, size_t width,
124930ef842dSbmc     int (*compar)(const void *, const void *))
125030ef842dSbmc {
125130ef842dSbmc 	int rev = dt_revsort, key = dt_keysort, keypos = dt_keypos;
125230ef842dSbmc 	dtrace_optval_t keyposopt = dtp->dt_options[DTRACEOPT_AGGSORTKEYPOS];
125330ef842dSbmc 
125430ef842dSbmc 	dt_revsort = (dtp->dt_options[DTRACEOPT_AGGSORTREV] != DTRACEOPT_UNSET);
125530ef842dSbmc 	dt_keysort = (dtp->dt_options[DTRACEOPT_AGGSORTKEY] != DTRACEOPT_UNSET);
125630ef842dSbmc 
125730ef842dSbmc 	if (keyposopt != DTRACEOPT_UNSET && keyposopt <= INT_MAX) {
125830ef842dSbmc 		dt_keypos = (int)keyposopt;
125930ef842dSbmc 	} else {
126030ef842dSbmc 		dt_keypos = 0;
126130ef842dSbmc 	}
126230ef842dSbmc 
126330ef842dSbmc 	if (compar == NULL) {
126430ef842dSbmc 		if (!dt_keysort) {
126530ef842dSbmc 			compar = dt_aggregate_varvalcmp;
126630ef842dSbmc 		} else {
126730ef842dSbmc 			compar = dt_aggregate_varkeycmp;
126830ef842dSbmc 		}
126930ef842dSbmc 	}
127030ef842dSbmc 
127130ef842dSbmc 	qsort(base, nel, width, compar);
127230ef842dSbmc 
127330ef842dSbmc 	dt_revsort = rev;
127430ef842dSbmc 	dt_keysort = key;
127530ef842dSbmc 	dt_keypos = keypos;
127630ef842dSbmc }
127730ef842dSbmc 
12787c478bd9Sstevel@tonic-gate int
dtrace_aggregate_walk(dtrace_hdl_t * dtp,dtrace_aggregate_f * func,void * arg)12797c478bd9Sstevel@tonic-gate dtrace_aggregate_walk(dtrace_hdl_t *dtp, dtrace_aggregate_f *func, void *arg)
12807c478bd9Sstevel@tonic-gate {
12817c478bd9Sstevel@tonic-gate 	dt_ahashent_t *h, *next;
12827c478bd9Sstevel@tonic-gate 	dt_ahash_t *hash = &dtp->dt_aggregate.dtat_hash;
12837c478bd9Sstevel@tonic-gate 
12847c478bd9Sstevel@tonic-gate 	for (h = hash->dtah_all; h != NULL; h = next) {
12857c478bd9Sstevel@tonic-gate 		/*
12867c478bd9Sstevel@tonic-gate 		 * dt_aggwalk_rval() can potentially remove the current hash
12877c478bd9Sstevel@tonic-gate 		 * entry; we need to load the next hash entry before calling
12887c478bd9Sstevel@tonic-gate 		 * into it.
12897c478bd9Sstevel@tonic-gate 		 */
12907c478bd9Sstevel@tonic-gate 		next = h->dtahe_nextall;
12917c478bd9Sstevel@tonic-gate 
12927c478bd9Sstevel@tonic-gate 		if (dt_aggwalk_rval(dtp, h, func(&h->dtahe_data, arg)) == -1)
12937c478bd9Sstevel@tonic-gate 			return (-1);
12947c478bd9Sstevel@tonic-gate 	}
12957c478bd9Sstevel@tonic-gate 
12967c478bd9Sstevel@tonic-gate 	return (0);
12977c478bd9Sstevel@tonic-gate }
12987c478bd9Sstevel@tonic-gate 
129933093f5bSBryan Cantrill static int
dt_aggregate_total(dtrace_hdl_t * dtp,boolean_t clear)130033093f5bSBryan Cantrill dt_aggregate_total(dtrace_hdl_t *dtp, boolean_t clear)
130133093f5bSBryan Cantrill {
130233093f5bSBryan Cantrill 	dt_ahashent_t *h;
130333093f5bSBryan Cantrill 	dtrace_aggdata_t **total;
130433093f5bSBryan Cantrill 	dtrace_aggid_t max = DTRACE_AGGVARIDNONE, id;
130533093f5bSBryan Cantrill 	dt_aggregate_t *agp = &dtp->dt_aggregate;
130633093f5bSBryan Cantrill 	dt_ahash_t *hash = &agp->dtat_hash;
130733093f5bSBryan Cantrill 	uint32_t tflags;
130833093f5bSBryan Cantrill 
130933093f5bSBryan Cantrill 	tflags = DTRACE_A_TOTAL | DTRACE_A_HASNEGATIVES | DTRACE_A_HASPOSITIVES;
131033093f5bSBryan Cantrill 
131133093f5bSBryan Cantrill 	/*
131233093f5bSBryan Cantrill 	 * If we need to deliver per-aggregation totals, we're going to take
131333093f5bSBryan Cantrill 	 * three passes over the aggregate:  one to clear everything out and
131433093f5bSBryan Cantrill 	 * determine our maximum aggregation ID, one to actually total
131533093f5bSBryan Cantrill 	 * everything up, and a final pass to assign the totals to the
131633093f5bSBryan Cantrill 	 * individual elements.
131733093f5bSBryan Cantrill 	 */
131833093f5bSBryan Cantrill 	for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
131933093f5bSBryan Cantrill 		dtrace_aggdata_t *aggdata = &h->dtahe_data;
132033093f5bSBryan Cantrill 
132133093f5bSBryan Cantrill 		if ((id = dt_aggregate_aggvarid(h)) > max)
132233093f5bSBryan Cantrill 			max = id;
132333093f5bSBryan Cantrill 
132433093f5bSBryan Cantrill 		aggdata->dtada_total = 0;
132533093f5bSBryan Cantrill 		aggdata->dtada_flags &= ~tflags;
132633093f5bSBryan Cantrill 	}
132733093f5bSBryan Cantrill 
132833093f5bSBryan Cantrill 	if (clear || max == DTRACE_AGGVARIDNONE)
132933093f5bSBryan Cantrill 		return (0);
133033093f5bSBryan Cantrill 
133133093f5bSBryan Cantrill 	total = dt_zalloc(dtp, (max + 1) * sizeof (dtrace_aggdata_t *));
133233093f5bSBryan Cantrill 
133333093f5bSBryan Cantrill 	if (total == NULL)
133433093f5bSBryan Cantrill 		return (-1);
133533093f5bSBryan Cantrill 
133633093f5bSBryan Cantrill 	for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
133733093f5bSBryan Cantrill 		dtrace_aggdata_t *aggdata = &h->dtahe_data;
133833093f5bSBryan Cantrill 		dtrace_aggdesc_t *agg = aggdata->dtada_desc;
133933093f5bSBryan Cantrill 		dtrace_recdesc_t *rec;
134033093f5bSBryan Cantrill 		caddr_t data;
134133093f5bSBryan Cantrill 		int64_t val, *addr;
134233093f5bSBryan Cantrill 
134333093f5bSBryan Cantrill 		rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
134433093f5bSBryan Cantrill 		data = aggdata->dtada_data;
134533093f5bSBryan Cantrill 		addr = (int64_t *)(uintptr_t)(data + rec->dtrd_offset);
134633093f5bSBryan Cantrill 
134733093f5bSBryan Cantrill 		switch (rec->dtrd_action) {
134833093f5bSBryan Cantrill 		case DTRACEAGG_STDDEV:
134933093f5bSBryan Cantrill 			val = dt_stddev((uint64_t *)addr, 1);
135033093f5bSBryan Cantrill 			break;
135133093f5bSBryan Cantrill 
135233093f5bSBryan Cantrill 		case DTRACEAGG_SUM:
135333093f5bSBryan Cantrill 		case DTRACEAGG_COUNT:
135433093f5bSBryan Cantrill 			val = *addr;
135533093f5bSBryan Cantrill 			break;
135633093f5bSBryan Cantrill 
135733093f5bSBryan Cantrill 		case DTRACEAGG_AVG:
135833093f5bSBryan Cantrill 			val = addr[0] ? (addr[1] / addr[0]) : 0;
135933093f5bSBryan Cantrill 			break;
136033093f5bSBryan Cantrill 
136133093f5bSBryan Cantrill 		default:
136233093f5bSBryan Cantrill 			continue;
136333093f5bSBryan Cantrill 		}
136433093f5bSBryan Cantrill 
136533093f5bSBryan Cantrill 		if (total[agg->dtagd_varid] == NULL) {
136633093f5bSBryan Cantrill 			total[agg->dtagd_varid] = aggdata;
136733093f5bSBryan Cantrill 			aggdata->dtada_flags |= DTRACE_A_TOTAL;
136833093f5bSBryan Cantrill 		} else {
136933093f5bSBryan Cantrill 			aggdata = total[agg->dtagd_varid];
137033093f5bSBryan Cantrill 		}
137133093f5bSBryan Cantrill 
137233093f5bSBryan Cantrill 		if (val > 0)
137333093f5bSBryan Cantrill 			aggdata->dtada_flags |= DTRACE_A_HASPOSITIVES;
137433093f5bSBryan Cantrill 
137533093f5bSBryan Cantrill 		if (val < 0) {
137633093f5bSBryan Cantrill 			aggdata->dtada_flags |= DTRACE_A_HASNEGATIVES;
137733093f5bSBryan Cantrill 			val = -val;
137833093f5bSBryan Cantrill 		}
137933093f5bSBryan Cantrill 
138033093f5bSBryan Cantrill 		if (dtp->dt_options[DTRACEOPT_AGGZOOM] != DTRACEOPT_UNSET) {
138133093f5bSBryan Cantrill 			val = (int64_t)((long double)val *
138233093f5bSBryan Cantrill 			    (1 / DTRACE_AGGZOOM_MAX));
138333093f5bSBryan Cantrill 
138433093f5bSBryan Cantrill 			if (val > aggdata->dtada_total)
138533093f5bSBryan Cantrill 				aggdata->dtada_total = val;
138633093f5bSBryan Cantrill 		} else {
138733093f5bSBryan Cantrill 			aggdata->dtada_total += val;
138833093f5bSBryan Cantrill 		}
138933093f5bSBryan Cantrill 	}
139033093f5bSBryan Cantrill 
139133093f5bSBryan Cantrill 	/*
139233093f5bSBryan Cantrill 	 * And now one final pass to set everyone's total.
139333093f5bSBryan Cantrill 	 */
139433093f5bSBryan Cantrill 	for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
139533093f5bSBryan Cantrill 		dtrace_aggdata_t *aggdata = &h->dtahe_data, *t;
139633093f5bSBryan Cantrill 		dtrace_aggdesc_t *agg = aggdata->dtada_desc;
139733093f5bSBryan Cantrill 
139833093f5bSBryan Cantrill 		if ((t = total[agg->dtagd_varid]) == NULL || aggdata == t)
139933093f5bSBryan Cantrill 			continue;
140033093f5bSBryan Cantrill 
140133093f5bSBryan Cantrill 		aggdata->dtada_total = t->dtada_total;
140233093f5bSBryan Cantrill 		aggdata->dtada_flags |= (t->dtada_flags & tflags);
140333093f5bSBryan Cantrill 	}
140433093f5bSBryan Cantrill 
140533093f5bSBryan Cantrill 	dt_free(dtp, total);
140633093f5bSBryan Cantrill 
140733093f5bSBryan Cantrill 	return (0);
140833093f5bSBryan Cantrill }
140933093f5bSBryan Cantrill 
141033093f5bSBryan Cantrill static int
dt_aggregate_minmaxbin(dtrace_hdl_t * dtp,boolean_t clear)141133093f5bSBryan Cantrill dt_aggregate_minmaxbin(dtrace_hdl_t *dtp, boolean_t clear)
141233093f5bSBryan Cantrill {
141333093f5bSBryan Cantrill 	dt_ahashent_t *h;
141433093f5bSBryan Cantrill 	dtrace_aggdata_t **minmax;
141533093f5bSBryan Cantrill 	dtrace_aggid_t max = DTRACE_AGGVARIDNONE, id;
141633093f5bSBryan Cantrill 	dt_aggregate_t *agp = &dtp->dt_aggregate;
141733093f5bSBryan Cantrill 	dt_ahash_t *hash = &agp->dtat_hash;
141833093f5bSBryan Cantrill 
141933093f5bSBryan Cantrill 	for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
142033093f5bSBryan Cantrill 		dtrace_aggdata_t *aggdata = &h->dtahe_data;
142133093f5bSBryan Cantrill 
142233093f5bSBryan Cantrill 		if ((id = dt_aggregate_aggvarid(h)) > max)
142333093f5bSBryan Cantrill 			max = id;
142433093f5bSBryan Cantrill 
142533093f5bSBryan Cantrill 		aggdata->dtada_minbin = 0;
142633093f5bSBryan Cantrill 		aggdata->dtada_maxbin = 0;
142733093f5bSBryan Cantrill 		aggdata->dtada_flags &= ~DTRACE_A_MINMAXBIN;
142833093f5bSBryan Cantrill 	}
142933093f5bSBryan Cantrill 
143033093f5bSBryan Cantrill 	if (clear || max == DTRACE_AGGVARIDNONE)
143133093f5bSBryan Cantrill 		return (0);
143233093f5bSBryan Cantrill 
143333093f5bSBryan Cantrill 	minmax = dt_zalloc(dtp, (max + 1) * sizeof (dtrace_aggdata_t *));
143433093f5bSBryan Cantrill 
143533093f5bSBryan Cantrill 	if (minmax == NULL)
143633093f5bSBryan Cantrill 		return (-1);
143733093f5bSBryan Cantrill 
143833093f5bSBryan Cantrill 	for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
143933093f5bSBryan Cantrill 		dtrace_aggdata_t *aggdata = &h->dtahe_data;
144033093f5bSBryan Cantrill 		dtrace_aggdesc_t *agg = aggdata->dtada_desc;
144133093f5bSBryan Cantrill 		dtrace_recdesc_t *rec;
144233093f5bSBryan Cantrill 		caddr_t data;
144333093f5bSBryan Cantrill 		int64_t *addr;
144433093f5bSBryan Cantrill 		int minbin = -1, maxbin = -1, i;
144533093f5bSBryan Cantrill 		int start = 0, size;
144633093f5bSBryan Cantrill 
144733093f5bSBryan Cantrill 		rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
144833093f5bSBryan Cantrill 		size = rec->dtrd_size / sizeof (int64_t);
144933093f5bSBryan Cantrill 		data = aggdata->dtada_data;
145033093f5bSBryan Cantrill 		addr = (int64_t *)(uintptr_t)(data + rec->dtrd_offset);
145133093f5bSBryan Cantrill 
145233093f5bSBryan Cantrill 		switch (rec->dtrd_action) {
145333093f5bSBryan Cantrill 		case DTRACEAGG_LQUANTIZE:
145433093f5bSBryan Cantrill 			/*
145533093f5bSBryan Cantrill 			 * For lquantize(), we always display the entire range
145633093f5bSBryan Cantrill 			 * of the aggregation when aggpack is set.
145733093f5bSBryan Cantrill 			 */
145833093f5bSBryan Cantrill 			start = 1;
145933093f5bSBryan Cantrill 			minbin = start;
146033093f5bSBryan Cantrill 			maxbin = size - 1 - start;
146133093f5bSBryan Cantrill 			break;
146233093f5bSBryan Cantrill 
146333093f5bSBryan Cantrill 		case DTRACEAGG_QUANTIZE:
146433093f5bSBryan Cantrill 			for (i = start; i < size; i++) {
146533093f5bSBryan Cantrill 				if (!addr[i])
146633093f5bSBryan Cantrill 					continue;
146733093f5bSBryan Cantrill 
146833093f5bSBryan Cantrill 				if (minbin == -1)
146933093f5bSBryan Cantrill 					minbin = i - start;
147033093f5bSBryan Cantrill 
147133093f5bSBryan Cantrill 				maxbin = i - start;
147233093f5bSBryan Cantrill 			}
147333093f5bSBryan Cantrill 
147433093f5bSBryan Cantrill 			if (minbin == -1) {
147533093f5bSBryan Cantrill 				/*
147633093f5bSBryan Cantrill 				 * If we have no data (e.g., due to a clear()
147733093f5bSBryan Cantrill 				 * or negative increments), we'll use the
147833093f5bSBryan Cantrill 				 * zero bucket as both our min and max.
147933093f5bSBryan Cantrill 				 */
148033093f5bSBryan Cantrill 				minbin = maxbin = DTRACE_QUANTIZE_ZEROBUCKET;
148133093f5bSBryan Cantrill 			}
148233093f5bSBryan Cantrill 
148333093f5bSBryan Cantrill 			break;
148433093f5bSBryan Cantrill 
148533093f5bSBryan Cantrill 		default:
148633093f5bSBryan Cantrill 			continue;
148733093f5bSBryan Cantrill 		}
148833093f5bSBryan Cantrill 
148933093f5bSBryan Cantrill 		if (minmax[agg->dtagd_varid] == NULL) {
149033093f5bSBryan Cantrill 			minmax[agg->dtagd_varid] = aggdata;
149133093f5bSBryan Cantrill 			aggdata->dtada_flags |= DTRACE_A_MINMAXBIN;
149233093f5bSBryan Cantrill 			aggdata->dtada_minbin = minbin;
149333093f5bSBryan Cantrill 			aggdata->dtada_maxbin = maxbin;
149433093f5bSBryan Cantrill 			continue;
149533093f5bSBryan Cantrill 		}
149633093f5bSBryan Cantrill 
149733093f5bSBryan Cantrill 		if (minbin < minmax[agg->dtagd_varid]->dtada_minbin)
149833093f5bSBryan Cantrill 			minmax[agg->dtagd_varid]->dtada_minbin = minbin;
149933093f5bSBryan Cantrill 
150033093f5bSBryan Cantrill 		if (maxbin > minmax[agg->dtagd_varid]->dtada_maxbin)
150133093f5bSBryan Cantrill 			minmax[agg->dtagd_varid]->dtada_maxbin = maxbin;
150233093f5bSBryan Cantrill 	}
150333093f5bSBryan Cantrill 
150433093f5bSBryan Cantrill 	/*
150533093f5bSBryan Cantrill 	 * And now one final pass to set everyone's minbin and maxbin.
150633093f5bSBryan Cantrill 	 */
150733093f5bSBryan Cantrill 	for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
150833093f5bSBryan Cantrill 		dtrace_aggdata_t *aggdata = &h->dtahe_data, *mm;
150933093f5bSBryan Cantrill 		dtrace_aggdesc_t *agg = aggdata->dtada_desc;
151033093f5bSBryan Cantrill 
151133093f5bSBryan Cantrill 		if ((mm = minmax[agg->dtagd_varid]) == NULL || aggdata == mm)
151233093f5bSBryan Cantrill 			continue;
151333093f5bSBryan Cantrill 
151433093f5bSBryan Cantrill 		aggdata->dtada_minbin = mm->dtada_minbin;
151533093f5bSBryan Cantrill 		aggdata->dtada_maxbin = mm->dtada_maxbin;
151633093f5bSBryan Cantrill 		aggdata->dtada_flags |= DTRACE_A_MINMAXBIN;
151733093f5bSBryan Cantrill 	}
151833093f5bSBryan Cantrill 
151933093f5bSBryan Cantrill 	dt_free(dtp, minmax);
152033093f5bSBryan Cantrill 
152133093f5bSBryan Cantrill 	return (0);
152233093f5bSBryan Cantrill }
152333093f5bSBryan Cantrill 
15247c478bd9Sstevel@tonic-gate static int
dt_aggregate_walk_sorted(dtrace_hdl_t * dtp,dtrace_aggregate_f * func,void * arg,int (* sfunc)(const void *,const void *))15257c478bd9Sstevel@tonic-gate dt_aggregate_walk_sorted(dtrace_hdl_t *dtp,
15267c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg,
15277c478bd9Sstevel@tonic-gate     int (*sfunc)(const void *, const void *))
15287c478bd9Sstevel@tonic-gate {
15297c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
15307c478bd9Sstevel@tonic-gate 	dt_ahashent_t *h, **sorted;
15317c478bd9Sstevel@tonic-gate 	dt_ahash_t *hash = &agp->dtat_hash;
15327c478bd9Sstevel@tonic-gate 	size_t i, nentries = 0;
153333093f5bSBryan Cantrill 	int rval = -1;
153433093f5bSBryan Cantrill 
153533093f5bSBryan Cantrill 	agp->dtat_flags &= ~(DTRACE_A_TOTAL | DTRACE_A_MINMAXBIN);
153633093f5bSBryan Cantrill 
153733093f5bSBryan Cantrill 	if (dtp->dt_options[DTRACEOPT_AGGHIST] != DTRACEOPT_UNSET) {
153833093f5bSBryan Cantrill 		agp->dtat_flags |= DTRACE_A_TOTAL;
153933093f5bSBryan Cantrill 
154033093f5bSBryan Cantrill 		if (dt_aggregate_total(dtp, B_FALSE) != 0)
154133093f5bSBryan Cantrill 			return (-1);
154233093f5bSBryan Cantrill 	}
154333093f5bSBryan Cantrill 
154433093f5bSBryan Cantrill 	if (dtp->dt_options[DTRACEOPT_AGGPACK] != DTRACEOPT_UNSET) {
154533093f5bSBryan Cantrill 		agp->dtat_flags |= DTRACE_A_MINMAXBIN;
154633093f5bSBryan Cantrill 
154733093f5bSBryan Cantrill 		if (dt_aggregate_minmaxbin(dtp, B_FALSE) != 0)
154833093f5bSBryan Cantrill 			return (-1);
154933093f5bSBryan Cantrill 	}
15507c478bd9Sstevel@tonic-gate 
15517c478bd9Sstevel@tonic-gate 	for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall)
15527c478bd9Sstevel@tonic-gate 		nentries++;
15537c478bd9Sstevel@tonic-gate 
155430ef842dSbmc 	sorted = dt_alloc(dtp, nentries * sizeof (dt_ahashent_t *));
15557c478bd9Sstevel@tonic-gate 
15567c478bd9Sstevel@tonic-gate 	if (sorted == NULL)
155733093f5bSBryan Cantrill 		goto out;
15587c478bd9Sstevel@tonic-gate 
15597c478bd9Sstevel@tonic-gate 	for (h = hash->dtah_all, i = 0; h != NULL; h = h->dtahe_nextall)
15607c478bd9Sstevel@tonic-gate 		sorted[i++] = h;
15617c478bd9Sstevel@tonic-gate 
156230ef842dSbmc 	(void) pthread_mutex_lock(&dt_qsort_lock);
156330ef842dSbmc 
156430ef842dSbmc 	if (sfunc == NULL) {
156530ef842dSbmc 		dt_aggregate_qsort(dtp, sorted, nentries,
156630ef842dSbmc 		    sizeof (dt_ahashent_t *), NULL);
156730ef842dSbmc 	} else {
156830ef842dSbmc 		/*
156930ef842dSbmc 		 * If we've been explicitly passed a sorting function,
157030ef842dSbmc 		 * we'll use that -- ignoring the values of the "aggsortrev",
157130ef842dSbmc 		 * "aggsortkey" and "aggsortkeypos" options.
157230ef842dSbmc 		 */
157330ef842dSbmc 		qsort(sorted, nentries, sizeof (dt_ahashent_t *), sfunc);
157430ef842dSbmc 	}
157530ef842dSbmc 
157630ef842dSbmc 	(void) pthread_mutex_unlock(&dt_qsort_lock);
15777c478bd9Sstevel@tonic-gate 
15787c478bd9Sstevel@tonic-gate 	for (i = 0; i < nentries; i++) {
15797c478bd9Sstevel@tonic-gate 		h = sorted[i];
15807c478bd9Sstevel@tonic-gate 
158133093f5bSBryan Cantrill 		if (dt_aggwalk_rval(dtp, h, func(&h->dtahe_data, arg)) == -1)
158233093f5bSBryan Cantrill 			goto out;
15837c478bd9Sstevel@tonic-gate 	}
15847c478bd9Sstevel@tonic-gate 
158533093f5bSBryan Cantrill 	rval = 0;
158633093f5bSBryan Cantrill out:
158733093f5bSBryan Cantrill 	if (agp->dtat_flags & DTRACE_A_TOTAL)
158833093f5bSBryan Cantrill 		(void) dt_aggregate_total(dtp, B_TRUE);
158933093f5bSBryan Cantrill 
159033093f5bSBryan Cantrill 	if (agp->dtat_flags & DTRACE_A_MINMAXBIN)
159133093f5bSBryan Cantrill 		(void) dt_aggregate_minmaxbin(dtp, B_TRUE);
159233093f5bSBryan Cantrill 
159330ef842dSbmc 	dt_free(dtp, sorted);
159433093f5bSBryan Cantrill 	return (rval);
15957c478bd9Sstevel@tonic-gate }
15967c478bd9Sstevel@tonic-gate 
159730ef842dSbmc int
dtrace_aggregate_walk_sorted(dtrace_hdl_t * dtp,dtrace_aggregate_f * func,void * arg)159830ef842dSbmc dtrace_aggregate_walk_sorted(dtrace_hdl_t *dtp,
159930ef842dSbmc     dtrace_aggregate_f *func, void *arg)
160030ef842dSbmc {
160130ef842dSbmc 	return (dt_aggregate_walk_sorted(dtp, func, arg, NULL));
160230ef842dSbmc }
160330ef842dSbmc 
16047c478bd9Sstevel@tonic-gate int
dtrace_aggregate_walk_keysorted(dtrace_hdl_t * dtp,dtrace_aggregate_f * func,void * arg)16057c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_keysorted(dtrace_hdl_t *dtp,
16067c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
16077c478bd9Sstevel@tonic-gate {
16087c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
16097c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_varkeycmp));
16107c478bd9Sstevel@tonic-gate }
16117c478bd9Sstevel@tonic-gate 
16127c478bd9Sstevel@tonic-gate int
dtrace_aggregate_walk_valsorted(dtrace_hdl_t * dtp,dtrace_aggregate_f * func,void * arg)16137c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_valsorted(dtrace_hdl_t *dtp,
16147c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
16157c478bd9Sstevel@tonic-gate {
16167c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
16177c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_varvalcmp));
16187c478bd9Sstevel@tonic-gate }
16197c478bd9Sstevel@tonic-gate 
16207c478bd9Sstevel@tonic-gate int
dtrace_aggregate_walk_keyvarsorted(dtrace_hdl_t * dtp,dtrace_aggregate_f * func,void * arg)16217c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_keyvarsorted(dtrace_hdl_t *dtp,
16227c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
16237c478bd9Sstevel@tonic-gate {
16247c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
16257c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_keyvarcmp));
16267c478bd9Sstevel@tonic-gate }
16277c478bd9Sstevel@tonic-gate 
16287c478bd9Sstevel@tonic-gate int
dtrace_aggregate_walk_valvarsorted(dtrace_hdl_t * dtp,dtrace_aggregate_f * func,void * arg)16297c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_valvarsorted(dtrace_hdl_t *dtp,
16307c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
16317c478bd9Sstevel@tonic-gate {
16327c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
16337c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_valvarcmp));
16347c478bd9Sstevel@tonic-gate }
16357c478bd9Sstevel@tonic-gate 
16367c478bd9Sstevel@tonic-gate int
dtrace_aggregate_walk_keyrevsorted(dtrace_hdl_t * dtp,dtrace_aggregate_f * func,void * arg)16377c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_keyrevsorted(dtrace_hdl_t *dtp,
16387c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
16397c478bd9Sstevel@tonic-gate {
16407c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
16417c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_varkeyrevcmp));
16427c478bd9Sstevel@tonic-gate }
16437c478bd9Sstevel@tonic-gate 
16447c478bd9Sstevel@tonic-gate int
dtrace_aggregate_walk_valrevsorted(dtrace_hdl_t * dtp,dtrace_aggregate_f * func,void * arg)16457c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_valrevsorted(dtrace_hdl_t *dtp,
16467c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
16477c478bd9Sstevel@tonic-gate {
16487c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
16497c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_varvalrevcmp));
16507c478bd9Sstevel@tonic-gate }
16517c478bd9Sstevel@tonic-gate 
16527c478bd9Sstevel@tonic-gate int
dtrace_aggregate_walk_keyvarrevsorted(dtrace_hdl_t * dtp,dtrace_aggregate_f * func,void * arg)16537c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_keyvarrevsorted(dtrace_hdl_t *dtp,
16547c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
16557c478bd9Sstevel@tonic-gate {
16567c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
16577c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_keyvarrevcmp));
16587c478bd9Sstevel@tonic-gate }
16597c478bd9Sstevel@tonic-gate 
16607c478bd9Sstevel@tonic-gate int
dtrace_aggregate_walk_valvarrevsorted(dtrace_hdl_t * dtp,dtrace_aggregate_f * func,void * arg)16617c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_valvarrevsorted(dtrace_hdl_t *dtp,
16627c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
16637c478bd9Sstevel@tonic-gate {
16647c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
16657c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_valvarrevcmp));
16667c478bd9Sstevel@tonic-gate }
16677c478bd9Sstevel@tonic-gate 
166830ef842dSbmc int
dtrace_aggregate_walk_joined(dtrace_hdl_t * dtp,dtrace_aggvarid_t * aggvars,int naggvars,dtrace_aggregate_walk_joined_f * func,void * arg)166930ef842dSbmc dtrace_aggregate_walk_joined(dtrace_hdl_t *dtp, dtrace_aggvarid_t *aggvars,
167030ef842dSbmc     int naggvars, dtrace_aggregate_walk_joined_f *func, void *arg)
167130ef842dSbmc {
167230ef842dSbmc 	dt_aggregate_t *agp = &dtp->dt_aggregate;
167330ef842dSbmc 	dt_ahashent_t *h, **sorted = NULL, ***bundle, **nbundle;
167430ef842dSbmc 	const dtrace_aggdata_t **data;
167530ef842dSbmc 	dt_ahashent_t *zaggdata = NULL;
167630ef842dSbmc 	dt_ahash_t *hash = &agp->dtat_hash;
167730ef842dSbmc 	size_t nentries = 0, nbundles = 0, start, zsize = 0, bundlesize;
167830ef842dSbmc 	dtrace_aggvarid_t max = 0, aggvar;
167930ef842dSbmc 	int rval = -1, *map, *remap = NULL;
168030ef842dSbmc 	int i, j;
168130ef842dSbmc 	dtrace_optval_t sortpos = dtp->dt_options[DTRACEOPT_AGGSORTPOS];
168230ef842dSbmc 
168330ef842dSbmc 	/*
168430ef842dSbmc 	 * If the sorting position is greater than the number of aggregation
168530ef842dSbmc 	 * variable IDs, we silently set it to 0.
168630ef842dSbmc 	 */
168730ef842dSbmc 	if (sortpos == DTRACEOPT_UNSET || sortpos >= naggvars)
168830ef842dSbmc 		sortpos = 0;
168930ef842dSbmc 
169030ef842dSbmc 	/*
169130ef842dSbmc 	 * First we need to translate the specified aggregation variable IDs
169230ef842dSbmc 	 * into a linear map that will allow us to translate an aggregation
169330ef842dSbmc 	 * variable ID into its position in the specified aggvars.
169430ef842dSbmc 	 */
169530ef842dSbmc 	for (i = 0; i < naggvars; i++) {
169630ef842dSbmc 		if (aggvars[i] == DTRACE_AGGVARIDNONE || aggvars[i] < 0)
169730ef842dSbmc 			return (dt_set_errno(dtp, EDT_BADAGGVAR));
169830ef842dSbmc 
169930ef842dSbmc 		if (aggvars[i] > max)
170030ef842dSbmc 			max = aggvars[i];
170130ef842dSbmc 	}
170230ef842dSbmc 
170330ef842dSbmc 	if ((map = dt_zalloc(dtp, (max + 1) * sizeof (int))) == NULL)
170430ef842dSbmc 		return (-1);
170530ef842dSbmc 
170630ef842dSbmc 	zaggdata = dt_zalloc(dtp, naggvars * sizeof (dt_ahashent_t));
170730ef842dSbmc 
170830ef842dSbmc 	if (zaggdata == NULL)
170930ef842dSbmc 		goto out;
171030ef842dSbmc 
171130ef842dSbmc 	for (i = 0; i < naggvars; i++) {
171230ef842dSbmc 		int ndx = i + sortpos;
171330ef842dSbmc 
171430ef842dSbmc 		if (ndx >= naggvars)
171530ef842dSbmc 			ndx -= naggvars;
171630ef842dSbmc 
171730ef842dSbmc 		aggvar = aggvars[ndx];
171830ef842dSbmc 		assert(aggvar <= max);
171930ef842dSbmc 
172030ef842dSbmc 		if (map[aggvar]) {
172130ef842dSbmc 			/*
172230ef842dSbmc 			 * We have an aggregation variable that is present
172330ef842dSbmc 			 * more than once in the array of aggregation
172430ef842dSbmc 			 * variables.  While it's unclear why one might want
172530ef842dSbmc 			 * to do this, it's legal.  To support this construct,
172630ef842dSbmc 			 * we will allocate a remap that will indicate the
172730ef842dSbmc 			 * position from which this aggregation variable
172830ef842dSbmc 			 * should be pulled.  (That is, where the remap will
172930ef842dSbmc 			 * map from one position to another.)
173030ef842dSbmc 			 */
173130ef842dSbmc 			if (remap == NULL) {
173230ef842dSbmc 				remap = dt_zalloc(dtp, naggvars * sizeof (int));
173330ef842dSbmc 
173430ef842dSbmc 				if (remap == NULL)
173530ef842dSbmc 					goto out;
173630ef842dSbmc 			}
173730ef842dSbmc 
173830ef842dSbmc 			/*
173930ef842dSbmc 			 * Given that the variable is already present, assert
174030ef842dSbmc 			 * that following through the mapping and adjusting
174130ef842dSbmc 			 * for the sort position yields the same aggregation
174230ef842dSbmc 			 * variable ID.
174330ef842dSbmc 			 */
174430ef842dSbmc 			assert(aggvars[(map[aggvar] - 1 + sortpos) %
174530ef842dSbmc 			    naggvars] == aggvars[ndx]);
174630ef842dSbmc 
174730ef842dSbmc 			remap[i] = map[aggvar];
174830ef842dSbmc 			continue;
174930ef842dSbmc 		}
175030ef842dSbmc 
175130ef842dSbmc 		map[aggvar] = i + 1;
175230ef842dSbmc 	}
175330ef842dSbmc 
175430ef842dSbmc 	/*
175530ef842dSbmc 	 * We need to take two passes over the data to size our allocation, so
175630ef842dSbmc 	 * we'll use the first pass to also fill in the zero-filled data to be
175730ef842dSbmc 	 * used to properly format a zero-valued aggregation.
175830ef842dSbmc 	 */
175930ef842dSbmc 	for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
176030ef842dSbmc 		dtrace_aggvarid_t id;
176130ef842dSbmc 		int ndx;
176230ef842dSbmc 
176330ef842dSbmc 		if ((id = dt_aggregate_aggvarid(h)) > max || !(ndx = map[id]))
176430ef842dSbmc 			continue;
176530ef842dSbmc 
176630ef842dSbmc 		if (zaggdata[ndx - 1].dtahe_size == 0) {
176730ef842dSbmc 			zaggdata[ndx - 1].dtahe_size = h->dtahe_size;
176830ef842dSbmc 			zaggdata[ndx - 1].dtahe_data = h->dtahe_data;
176930ef842dSbmc 		}
177030ef842dSbmc 
177130ef842dSbmc 		nentries++;
177230ef842dSbmc 	}
177330ef842dSbmc 
177430ef842dSbmc 	if (nentries == 0) {
177530ef842dSbmc 		/*
177630ef842dSbmc 		 * We couldn't find any entries; there is nothing else to do.
177730ef842dSbmc 		 */
177830ef842dSbmc 		rval = 0;
177930ef842dSbmc 		goto out;
178030ef842dSbmc 	}
178130ef842dSbmc 
178230ef842dSbmc 	/*
178330ef842dSbmc 	 * Before we sort the data, we're going to look for any holes in our
178430ef842dSbmc 	 * zero-filled data.  This will occur if an aggregation variable that
178530ef842dSbmc 	 * we are being asked to print has not yet been assigned the result of
178630ef842dSbmc 	 * any aggregating action for _any_ tuple.  The issue becomes that we
178730ef842dSbmc 	 * would like a zero value to be printed for all columns for this
178830ef842dSbmc 	 * aggregation, but without any record description, we don't know the
178930ef842dSbmc 	 * aggregating action that corresponds to the aggregation variable.  To
179030ef842dSbmc 	 * try to find a match, we're simply going to lookup aggregation IDs
179130ef842dSbmc 	 * (which are guaranteed to be contiguous and to start from 1), looking
179230ef842dSbmc 	 * for the specified aggregation variable ID.  If we find a match,
179330ef842dSbmc 	 * we'll use that.  If we iterate over all aggregation IDs and don't
179430ef842dSbmc 	 * find a match, then we must be an anonymous enabling.  (Anonymous
179530ef842dSbmc 	 * enablings can't currently derive either aggregation variable IDs or
179630ef842dSbmc 	 * aggregation variable names given only an aggregation ID.)  In this
179730ef842dSbmc 	 * obscure case (anonymous enabling, multiple aggregation printa() with
179830ef842dSbmc 	 * some aggregations not represented for any tuple), our defined
179930ef842dSbmc 	 * behavior is that the zero will be printed in the format of the first
180030ef842dSbmc 	 * aggregation variable that contains any non-zero value.
180130ef842dSbmc 	 */
180230ef842dSbmc 	for (i = 0; i < naggvars; i++) {
180330ef842dSbmc 		if (zaggdata[i].dtahe_size == 0) {
180430ef842dSbmc 			dtrace_aggvarid_t aggvar;
180530ef842dSbmc 
180630ef842dSbmc 			aggvar = aggvars[(i - sortpos + naggvars) % naggvars];
180730ef842dSbmc 			assert(zaggdata[i].dtahe_data.dtada_data == NULL);
180830ef842dSbmc 
180930ef842dSbmc 			for (j = DTRACE_AGGIDNONE + 1; ; j++) {
181030ef842dSbmc 				dtrace_aggdesc_t *agg;
181130ef842dSbmc 				dtrace_aggdata_t *aggdata;
181230ef842dSbmc 
181330ef842dSbmc 				if (dt_aggid_lookup(dtp, j, &agg) != 0)
181430ef842dSbmc 					break;
181530ef842dSbmc 
181630ef842dSbmc 				if (agg->dtagd_varid != aggvar)
181730ef842dSbmc 					continue;
181830ef842dSbmc 
181930ef842dSbmc 				/*
182030ef842dSbmc 				 * We have our description -- now we need to
182130ef842dSbmc 				 * cons up the zaggdata entry for it.
182230ef842dSbmc 				 */
182330ef842dSbmc 				aggdata = &zaggdata[i].dtahe_data;
182430ef842dSbmc 				aggdata->dtada_size = agg->dtagd_size;
182530ef842dSbmc 				aggdata->dtada_desc = agg;
182630ef842dSbmc 				aggdata->dtada_handle = dtp;
182730ef842dSbmc 				(void) dt_epid_lookup(dtp, agg->dtagd_epid,
182830ef842dSbmc 				    &aggdata->dtada_edesc,
182930ef842dSbmc 				    &aggdata->dtada_pdesc);
183030ef842dSbmc 				aggdata->dtada_normal = 1;
183130ef842dSbmc 				zaggdata[i].dtahe_hashval = 0;
183230ef842dSbmc 				zaggdata[i].dtahe_size = agg->dtagd_size;
183330ef842dSbmc 				break;
183430ef842dSbmc 			}
183530ef842dSbmc 
183630ef842dSbmc 			if (zaggdata[i].dtahe_size == 0) {
183730ef842dSbmc 				caddr_t data;
183830ef842dSbmc 
183930ef842dSbmc 				/*
184030ef842dSbmc 				 * We couldn't find this aggregation, meaning
184130ef842dSbmc 				 * that we have never seen it before for any
184230ef842dSbmc 				 * tuple _and_ this is an anonymous enabling.
184330ef842dSbmc 				 * That is, we're in the obscure case outlined
184430ef842dSbmc 				 * above.  In this case, our defined behavior
184530ef842dSbmc 				 * is to format the data in the format of the
184630ef842dSbmc 				 * first non-zero aggregation -- of which, of
184730ef842dSbmc 				 * course, we know there to be at least one
184830ef842dSbmc 				 * (or nentries would have been zero).
184930ef842dSbmc 				 */
185030ef842dSbmc 				for (j = 0; j < naggvars; j++) {
185130ef842dSbmc 					if (zaggdata[j].dtahe_size != 0)
185230ef842dSbmc 						break;
185330ef842dSbmc 				}
185430ef842dSbmc 
185530ef842dSbmc 				assert(j < naggvars);
185630ef842dSbmc 				zaggdata[i] = zaggdata[j];
185730ef842dSbmc 
185830ef842dSbmc 				data = zaggdata[i].dtahe_data.dtada_data;
185930ef842dSbmc 				assert(data != NULL);
186030ef842dSbmc 			}
186130ef842dSbmc 		}
186230ef842dSbmc 	}
186330ef842dSbmc 
186430ef842dSbmc 	/*
186530ef842dSbmc 	 * Now we need to allocate our zero-filled data for use for
186630ef842dSbmc 	 * aggregations that don't have a value corresponding to a given key.
186730ef842dSbmc 	 */
186830ef842dSbmc 	for (i = 0; i < naggvars; i++) {
186930ef842dSbmc 		dtrace_aggdata_t *aggdata = &zaggdata[i].dtahe_data;
187030ef842dSbmc 		dtrace_aggdesc_t *aggdesc = aggdata->dtada_desc;
187130ef842dSbmc 		dtrace_recdesc_t *rec;
187230ef842dSbmc 		uint64_t larg;
187330ef842dSbmc 		caddr_t zdata;
187430ef842dSbmc 
187530ef842dSbmc 		zsize = zaggdata[i].dtahe_size;
187630ef842dSbmc 		assert(zsize != 0);
187730ef842dSbmc 
187830ef842dSbmc 		if ((zdata = dt_zalloc(dtp, zsize)) == NULL) {
187930ef842dSbmc 			/*
188030ef842dSbmc 			 * If we failed to allocated some zero-filled data, we
188130ef842dSbmc 			 * need to zero out the remaining dtada_data pointers
188230ef842dSbmc 			 * to prevent the wrong data from being freed below.
188330ef842dSbmc 			 */
188430ef842dSbmc 			for (j = i; j < naggvars; j++)
188530ef842dSbmc 				zaggdata[j].dtahe_data.dtada_data = NULL;
188630ef842dSbmc 			goto out;
188730ef842dSbmc 		}
188830ef842dSbmc 
188930ef842dSbmc 		aggvar = aggvars[(i - sortpos + naggvars) % naggvars];
189030ef842dSbmc 
189130ef842dSbmc 		/*
189230ef842dSbmc 		 * First, the easy bit.  To maintain compatibility with
189330ef842dSbmc 		 * consumers that pull the compiler-generated ID out of the
189430ef842dSbmc 		 * data, we put that ID at the top of the zero-filled data.
189530ef842dSbmc 		 */
189630ef842dSbmc 		rec = &aggdesc->dtagd_rec[0];
189730ef842dSbmc 		/* LINTED - alignment */
189830ef842dSbmc 		*((dtrace_aggvarid_t *)(zdata + rec->dtrd_offset)) = aggvar;
189930ef842dSbmc 
190030ef842dSbmc 		rec = &aggdesc->dtagd_rec[aggdesc->dtagd_nrecs - 1];
190130ef842dSbmc 
190230ef842dSbmc 		/*
1903*0ddc0ebbSBryan Cantrill 		 * Now for the more complicated part.  For the lquantize() and
1904*0ddc0ebbSBryan Cantrill 		 * llquantize() aggregating actions, zero-filled data is not
1905*0ddc0ebbSBryan Cantrill 		 * equivalent to an empty record:  we must also get the
1906*0ddc0ebbSBryan Cantrill 		 * parameters for the lquantize()/llquantize().
190730ef842dSbmc 		 */
1908*0ddc0ebbSBryan Cantrill 		if (rec->dtrd_action == DTRACEAGG_LQUANTIZE ||
1909*0ddc0ebbSBryan Cantrill 		    rec->dtrd_action == DTRACEAGG_LLQUANTIZE) {
191030ef842dSbmc 			if (aggdata->dtada_data != NULL) {
191130ef842dSbmc 				/*
191230ef842dSbmc 				 * The easier case here is if we actually have
191330ef842dSbmc 				 * some prototype data -- in which case we
191430ef842dSbmc 				 * manually dig it out of the aggregation
191530ef842dSbmc 				 * record.
191630ef842dSbmc 				 */
191730ef842dSbmc 				/* LINTED - alignment */
191830ef842dSbmc 				larg = *((uint64_t *)(aggdata->dtada_data +
191930ef842dSbmc 				    rec->dtrd_offset));
192030ef842dSbmc 			} else {
192130ef842dSbmc 				/*
192230ef842dSbmc 				 * We don't have any prototype data.  As a
192330ef842dSbmc 				 * result, we know that we _do_ have the
192430ef842dSbmc 				 * compiler-generated information.  (If this
192530ef842dSbmc 				 * were an anonymous enabling, all of our
192630ef842dSbmc 				 * zero-filled data would have prototype data
192730ef842dSbmc 				 * -- either directly or indirectly.) So as
192830ef842dSbmc 				 * gross as it is, we'll grovel around in the
192930ef842dSbmc 				 * compiler-generated information to find the
1930*0ddc0ebbSBryan Cantrill 				 * lquantize()/llquantize() parameters.
193130ef842dSbmc 				 */
193230ef842dSbmc 				dtrace_stmtdesc_t *sdp;
193330ef842dSbmc 				dt_ident_t *aid;
193430ef842dSbmc 				dt_idsig_t *isp;
193530ef842dSbmc 
193630ef842dSbmc 				sdp = (dtrace_stmtdesc_t *)(uintptr_t)
193730ef842dSbmc 				    aggdesc->dtagd_rec[0].dtrd_uarg;
193830ef842dSbmc 				aid = sdp->dtsd_aggdata;
193930ef842dSbmc 				isp = (dt_idsig_t *)aid->di_data;
194030ef842dSbmc 				assert(isp->dis_auxinfo != 0);
194130ef842dSbmc 				larg = isp->dis_auxinfo;
194230ef842dSbmc 			}
194330ef842dSbmc 
194430ef842dSbmc 			/* LINTED - alignment */
194530ef842dSbmc 			*((uint64_t *)(zdata + rec->dtrd_offset)) = larg;
194630ef842dSbmc 		}
194730ef842dSbmc 
194830ef842dSbmc 		aggdata->dtada_data = zdata;
194930ef842dSbmc 	}
195030ef842dSbmc 
195130ef842dSbmc 	/*
195230ef842dSbmc 	 * Now that we've dealt with setting up our zero-filled data, we can
195330ef842dSbmc 	 * allocate our sorted array, and take another pass over the data to
195430ef842dSbmc 	 * fill it.
195530ef842dSbmc 	 */
195630ef842dSbmc 	sorted = dt_alloc(dtp, nentries * sizeof (dt_ahashent_t *));
195730ef842dSbmc 
195830ef842dSbmc 	if (sorted == NULL)
195930ef842dSbmc 		goto out;
196030ef842dSbmc 
196130ef842dSbmc 	for (h = hash->dtah_all, i = 0; h != NULL; h = h->dtahe_nextall) {
196230ef842dSbmc 		dtrace_aggvarid_t id;
196330ef842dSbmc 
196430ef842dSbmc 		if ((id = dt_aggregate_aggvarid(h)) > max || !map[id])
196530ef842dSbmc 			continue;
196630ef842dSbmc 
196730ef842dSbmc 		sorted[i++] = h;
196830ef842dSbmc 	}
196930ef842dSbmc 
197030ef842dSbmc 	assert(i == nentries);
197130ef842dSbmc 
197230ef842dSbmc 	/*
197330ef842dSbmc 	 * We've loaded our array; now we need to sort by value to allow us
197430ef842dSbmc 	 * to create bundles of like value.  We're going to acquire the
197530ef842dSbmc 	 * dt_qsort_lock here, and hold it across all of our subsequent
197630ef842dSbmc 	 * comparison and sorting.
197730ef842dSbmc 	 */
197830ef842dSbmc 	(void) pthread_mutex_lock(&dt_qsort_lock);
197930ef842dSbmc 
198030ef842dSbmc 	qsort(sorted, nentries, sizeof (dt_ahashent_t *),
198130ef842dSbmc 	    dt_aggregate_keyvarcmp);
198230ef842dSbmc 
198330ef842dSbmc 	/*
198430ef842dSbmc 	 * Now we need to go through and create bundles.  Because the number
198530ef842dSbmc 	 * of bundles is bounded by the size of the sorted array, we're going
198630ef842dSbmc 	 * to reuse the underlying storage.  And note that "bundle" is an
198730ef842dSbmc 	 * array of pointers to arrays of pointers to dt_ahashent_t -- making
198830ef842dSbmc 	 * its type (regrettably) "dt_ahashent_t ***".  (Regrettable because
198930ef842dSbmc 	 * '*' -- like '_' and 'X' -- should never appear in triplicate in
199030ef842dSbmc 	 * an ideal world.)
199130ef842dSbmc 	 */
199230ef842dSbmc 	bundle = (dt_ahashent_t ***)sorted;
199330ef842dSbmc 
199430ef842dSbmc 	for (i = 1, start = 0; i <= nentries; i++) {
199530ef842dSbmc 		if (i < nentries &&
199630ef842dSbmc 		    dt_aggregate_keycmp(&sorted[i], &sorted[i - 1]) == 0)
199730ef842dSbmc 			continue;
199830ef842dSbmc 
199930ef842dSbmc 		/*
200030ef842dSbmc 		 * We have a bundle boundary.  Everything from start to
200130ef842dSbmc 		 * (i - 1) belongs in one bundle.
200230ef842dSbmc 		 */
200330ef842dSbmc 		assert(i - start <= naggvars);
200430ef842dSbmc 		bundlesize = (naggvars + 2) * sizeof (dt_ahashent_t *);
200530ef842dSbmc 
200630ef842dSbmc 		if ((nbundle = dt_zalloc(dtp, bundlesize)) == NULL) {
200730ef842dSbmc 			(void) pthread_mutex_unlock(&dt_qsort_lock);
200830ef842dSbmc 			goto out;
200930ef842dSbmc 		}
201030ef842dSbmc 
201130ef842dSbmc 		for (j = start; j < i; j++) {
201230ef842dSbmc 			dtrace_aggvarid_t id = dt_aggregate_aggvarid(sorted[j]);
201330ef842dSbmc 
201430ef842dSbmc 			assert(id <= max);
201530ef842dSbmc 			assert(map[id] != 0);
201630ef842dSbmc 			assert(map[id] - 1 < naggvars);
201730ef842dSbmc 			assert(nbundle[map[id] - 1] == NULL);
201830ef842dSbmc 			nbundle[map[id] - 1] = sorted[j];
201930ef842dSbmc 
202030ef842dSbmc 			if (nbundle[naggvars] == NULL)
202130ef842dSbmc 				nbundle[naggvars] = sorted[j];
202230ef842dSbmc 		}
202330ef842dSbmc 
202430ef842dSbmc 		for (j = 0; j < naggvars; j++) {
202530ef842dSbmc 			if (nbundle[j] != NULL)
202630ef842dSbmc 				continue;
202730ef842dSbmc 
202830ef842dSbmc 			/*
202930ef842dSbmc 			 * Before we assume that this aggregation variable
203030ef842dSbmc 			 * isn't present (and fall back to using the
203130ef842dSbmc 			 * zero-filled data allocated earlier), check the
203230ef842dSbmc 			 * remap.  If we have a remapping, we'll drop it in
203330ef842dSbmc 			 * here.  Note that we might be remapping an
203430ef842dSbmc 			 * aggregation variable that isn't present for this
203530ef842dSbmc 			 * key; in this case, the aggregation data that we
203630ef842dSbmc 			 * copy will point to the zeroed data.
203730ef842dSbmc 			 */
203830ef842dSbmc 			if (remap != NULL && remap[j]) {
203930ef842dSbmc 				assert(remap[j] - 1 < j);
204030ef842dSbmc 				assert(nbundle[remap[j] - 1] != NULL);
204130ef842dSbmc 				nbundle[j] = nbundle[remap[j] - 1];
204230ef842dSbmc 			} else {
204330ef842dSbmc 				nbundle[j] = &zaggdata[j];
204430ef842dSbmc 			}
204530ef842dSbmc 		}
204630ef842dSbmc 
204730ef842dSbmc 		bundle[nbundles++] = nbundle;
204830ef842dSbmc 		start = i;
204930ef842dSbmc 	}
205030ef842dSbmc 
205130ef842dSbmc 	/*
205230ef842dSbmc 	 * Now we need to re-sort based on the first value.
205330ef842dSbmc 	 */
205430ef842dSbmc 	dt_aggregate_qsort(dtp, bundle, nbundles, sizeof (dt_ahashent_t **),
205530ef842dSbmc 	    dt_aggregate_bundlecmp);
205630ef842dSbmc 
205730ef842dSbmc 	(void) pthread_mutex_unlock(&dt_qsort_lock);
205830ef842dSbmc 
205930ef842dSbmc 	/*
206030ef842dSbmc 	 * We're done!  Now we just need to go back over the sorted bundles,
206130ef842dSbmc 	 * calling the function.
206230ef842dSbmc 	 */
206330ef842dSbmc 	data = alloca((naggvars + 1) * sizeof (dtrace_aggdata_t *));
206430ef842dSbmc 
206530ef842dSbmc 	for (i = 0; i < nbundles; i++) {
206630ef842dSbmc 		for (j = 0; j < naggvars; j++)
206730ef842dSbmc 			data[j + 1] = NULL;
206830ef842dSbmc 
206930ef842dSbmc 		for (j = 0; j < naggvars; j++) {
207030ef842dSbmc 			int ndx = j - sortpos;
207130ef842dSbmc 
207230ef842dSbmc 			if (ndx < 0)
207330ef842dSbmc 				ndx += naggvars;
207430ef842dSbmc 
207530ef842dSbmc 			assert(bundle[i][ndx] != NULL);
207630ef842dSbmc 			data[j + 1] = &bundle[i][ndx]->dtahe_data;
207730ef842dSbmc 		}
207830ef842dSbmc 
207930ef842dSbmc 		for (j = 0; j < naggvars; j++)
208030ef842dSbmc 			assert(data[j + 1] != NULL);
208130ef842dSbmc 
208230ef842dSbmc 		/*
208330ef842dSbmc 		 * The representative key is the last element in the bundle.
208430ef842dSbmc 		 * Assert that we have one, and then set it to be the first
208530ef842dSbmc 		 * element of data.
208630ef842dSbmc 		 */
208730ef842dSbmc 		assert(bundle[i][j] != NULL);
208830ef842dSbmc 		data[0] = &bundle[i][j]->dtahe_data;
208930ef842dSbmc 
209030ef842dSbmc 		if ((rval = func(data, naggvars + 1, arg)) == -1)
209130ef842dSbmc 			goto out;
209230ef842dSbmc 	}
209330ef842dSbmc 
209430ef842dSbmc 	rval = 0;
209530ef842dSbmc out:
209630ef842dSbmc 	for (i = 0; i < nbundles; i++)
209730ef842dSbmc 		dt_free(dtp, bundle[i]);
209830ef842dSbmc 
209930ef842dSbmc 	if (zaggdata != NULL) {
210030ef842dSbmc 		for (i = 0; i < naggvars; i++)
210130ef842dSbmc 			dt_free(dtp, zaggdata[i].dtahe_data.dtada_data);
210230ef842dSbmc 	}
210330ef842dSbmc 
210430ef842dSbmc 	dt_free(dtp, zaggdata);
210530ef842dSbmc 	dt_free(dtp, sorted);
210630ef842dSbmc 	dt_free(dtp, remap);
210730ef842dSbmc 	dt_free(dtp, map);
210830ef842dSbmc 
210930ef842dSbmc 	return (rval);
211030ef842dSbmc }
211130ef842dSbmc 
21127c478bd9Sstevel@tonic-gate int
dtrace_aggregate_print(dtrace_hdl_t * dtp,FILE * fp,dtrace_aggregate_walk_f * func)21137c478bd9Sstevel@tonic-gate dtrace_aggregate_print(dtrace_hdl_t *dtp, FILE *fp,
21147c478bd9Sstevel@tonic-gate     dtrace_aggregate_walk_f *func)
21157c478bd9Sstevel@tonic-gate {
21167c478bd9Sstevel@tonic-gate 	dt_print_aggdata_t pd;
21177c478bd9Sstevel@tonic-gate 
211833093f5bSBryan Cantrill 	bzero(&pd, sizeof (pd));
211933093f5bSBryan Cantrill 
21207c478bd9Sstevel@tonic-gate 	pd.dtpa_dtp = dtp;
21217c478bd9Sstevel@tonic-gate 	pd.dtpa_fp = fp;
21227c478bd9Sstevel@tonic-gate 	pd.dtpa_allunprint = 1;
21237c478bd9Sstevel@tonic-gate 
21247c478bd9Sstevel@tonic-gate 	if (func == NULL)
212530ef842dSbmc 		func = dtrace_aggregate_walk_sorted;
21267c478bd9Sstevel@tonic-gate 
21277c478bd9Sstevel@tonic-gate 	if ((*func)(dtp, dt_print_agg, &pd) == -1)
21287c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, dtp->dt_errno));
21297c478bd9Sstevel@tonic-gate 
21307c478bd9Sstevel@tonic-gate 	return (0);
21317c478bd9Sstevel@tonic-gate }
21327c478bd9Sstevel@tonic-gate 
21337c478bd9Sstevel@tonic-gate void
dtrace_aggregate_clear(dtrace_hdl_t * dtp)21347c478bd9Sstevel@tonic-gate dtrace_aggregate_clear(dtrace_hdl_t *dtp)
21357c478bd9Sstevel@tonic-gate {
21367c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
21377c478bd9Sstevel@tonic-gate 	dt_ahash_t *hash = &agp->dtat_hash;
21387c478bd9Sstevel@tonic-gate 	dt_ahashent_t *h;
21397c478bd9Sstevel@tonic-gate 	dtrace_aggdata_t *data;
21407c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *aggdesc;
21417c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *rec;
21427c478bd9Sstevel@tonic-gate 	int i, max_cpus = agp->dtat_maxcpu;
21437c478bd9Sstevel@tonic-gate 
21447c478bd9Sstevel@tonic-gate 	for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
21457c478bd9Sstevel@tonic-gate 		aggdesc = h->dtahe_data.dtada_desc;
21467c478bd9Sstevel@tonic-gate 		rec = &aggdesc->dtagd_rec[aggdesc->dtagd_nrecs - 1];
21477c478bd9Sstevel@tonic-gate 		data = &h->dtahe_data;
21487c478bd9Sstevel@tonic-gate 
21497c478bd9Sstevel@tonic-gate 		bzero(&data->dtada_data[rec->dtrd_offset], rec->dtrd_size);
21507c478bd9Sstevel@tonic-gate 
21517c478bd9Sstevel@tonic-gate 		if (data->dtada_percpu == NULL)
21527c478bd9Sstevel@tonic-gate 			continue;
21537c478bd9Sstevel@tonic-gate 
21547c478bd9Sstevel@tonic-gate 		for (i = 0; i < max_cpus; i++)
21557c478bd9Sstevel@tonic-gate 			bzero(data->dtada_percpu[i], rec->dtrd_size);
21567c478bd9Sstevel@tonic-gate 	}
21577c478bd9Sstevel@tonic-gate }
21587c478bd9Sstevel@tonic-gate 
21597c478bd9Sstevel@tonic-gate void
dt_aggregate_destroy(dtrace_hdl_t * dtp)21607c478bd9Sstevel@tonic-gate dt_aggregate_destroy(dtrace_hdl_t *dtp)
21617c478bd9Sstevel@tonic-gate {
21627c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
21637c478bd9Sstevel@tonic-gate 	dt_ahash_t *hash = &agp->dtat_hash;
21647c478bd9Sstevel@tonic-gate 	dt_ahashent_t *h, *next;
21657c478bd9Sstevel@tonic-gate 	dtrace_aggdata_t *aggdata;
21667c478bd9Sstevel@tonic-gate 	int i, max_cpus = agp->dtat_maxcpu;
21677c478bd9Sstevel@tonic-gate 
21687c478bd9Sstevel@tonic-gate 	if (hash->dtah_hash == NULL) {
21697c478bd9Sstevel@tonic-gate 		assert(hash->dtah_all == NULL);
21707c478bd9Sstevel@tonic-gate 	} else {
21717c478bd9Sstevel@tonic-gate 		free(hash->dtah_hash);
21727c478bd9Sstevel@tonic-gate 
21737c478bd9Sstevel@tonic-gate 		for (h = hash->dtah_all; h != NULL; h = next) {
21747c478bd9Sstevel@tonic-gate 			next = h->dtahe_nextall;
21757c478bd9Sstevel@tonic-gate 
21767c478bd9Sstevel@tonic-gate 			aggdata = &h->dtahe_data;
21777c478bd9Sstevel@tonic-gate 
21787c478bd9Sstevel@tonic-gate 			if (aggdata->dtada_percpu != NULL) {
21797c478bd9Sstevel@tonic-gate 				for (i = 0; i < max_cpus; i++)
21807c478bd9Sstevel@tonic-gate 					free(aggdata->dtada_percpu[i]);
21817c478bd9Sstevel@tonic-gate 				free(aggdata->dtada_percpu);
21827c478bd9Sstevel@tonic-gate 			}
21837c478bd9Sstevel@tonic-gate 
21847c478bd9Sstevel@tonic-gate 			free(aggdata->dtada_data);
21857c478bd9Sstevel@tonic-gate 			free(h);
21867c478bd9Sstevel@tonic-gate 		}
21877c478bd9Sstevel@tonic-gate 
21887c478bd9Sstevel@tonic-gate 		hash->dtah_hash = NULL;
21897c478bd9Sstevel@tonic-gate 		hash->dtah_all = NULL;
21907c478bd9Sstevel@tonic-gate 		hash->dtah_size = 0;
21917c478bd9Sstevel@tonic-gate 	}
21927c478bd9Sstevel@tonic-gate 
21937c478bd9Sstevel@tonic-gate 	free(agp->dtat_buf.dtbd_data);
21947c478bd9Sstevel@tonic-gate 	free(agp->dtat_cpus);
21957c478bd9Sstevel@tonic-gate }
2196