17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5127bbe13Stomee  * Common Development and Distribution License (the "License").
6127bbe13Stomee  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
21127bbe13Stomee 
227c478bd9Sstevel@tonic-gate /*
236e0bee74Sjhaslam  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
272b6389efSBryan Cantrill /*
28*0ddc0ebbSBryan Cantrill  * Copyright (c) 2016, Joyent, Inc. All rights reserved.
29e5803b76SAdam H. Leventhal  * Copyright (c) 2012 by Delphix. All rights reserved.
302b6389efSBryan Cantrill  */
317c478bd9Sstevel@tonic-gate 
327c478bd9Sstevel@tonic-gate #include <stdlib.h>
337c478bd9Sstevel@tonic-gate #include <strings.h>
347c478bd9Sstevel@tonic-gate #include <errno.h>
357c478bd9Sstevel@tonic-gate #include <unistd.h>
367c478bd9Sstevel@tonic-gate #include <dt_impl.h>
377c478bd9Sstevel@tonic-gate #include <assert.h>
3830ef842dSbmc #include <alloca.h>
3930ef842dSbmc #include <limits.h>
407c478bd9Sstevel@tonic-gate 
417c478bd9Sstevel@tonic-gate #define	DTRACE_AHASHSIZE	32779		/* big 'ol prime */
427c478bd9Sstevel@tonic-gate 
4330ef842dSbmc /*
4430ef842dSbmc  * Because qsort(3C) does not allow an argument to be passed to a comparison
4530ef842dSbmc  * function, the variables that affect comparison must regrettably be global;
4630ef842dSbmc  * they are protected by a global static lock, dt_qsort_lock.
4730ef842dSbmc  */
4830ef842dSbmc static pthread_mutex_t dt_qsort_lock = PTHREAD_MUTEX_INITIALIZER;
4930ef842dSbmc 
5030ef842dSbmc static int dt_revsort;
5130ef842dSbmc static int dt_keysort;
5230ef842dSbmc static int dt_keypos;
5330ef842dSbmc 
5430ef842dSbmc #define	DT_LESSTHAN	(dt_revsort == 0 ? -1 : 1)
5530ef842dSbmc #define	DT_GREATERTHAN	(dt_revsort == 0 ? 1 : -1)
5630ef842dSbmc 
577c478bd9Sstevel@tonic-gate static void
dt_aggregate_count(int64_t * existing,int64_t * new,size_t size)58187eccf8Sbmc dt_aggregate_count(int64_t *existing, int64_t *new, size_t size)
597c478bd9Sstevel@tonic-gate {
607c478bd9Sstevel@tonic-gate 	int i;
617c478bd9Sstevel@tonic-gate 
62187eccf8Sbmc 	for (i = 0; i < size / sizeof (int64_t); i++)
637c478bd9Sstevel@tonic-gate 		existing[i] = existing[i] + new[i];
647c478bd9Sstevel@tonic-gate }
657c478bd9Sstevel@tonic-gate 
667c478bd9Sstevel@tonic-gate static int
dt_aggregate_countcmp(int64_t * lhs,int64_t * rhs)67187eccf8Sbmc dt_aggregate_countcmp(int64_t *lhs, int64_t *rhs)
687c478bd9Sstevel@tonic-gate {
69187eccf8Sbmc 	int64_t lvar = *lhs;
70187eccf8Sbmc 	int64_t rvar = *rhs;
717c478bd9Sstevel@tonic-gate 
727c478bd9Sstevel@tonic-gate 	if (lvar < rvar)
7330ef842dSbmc 		return (DT_LESSTHAN);
7430ef842dSbmc 
7530ef842dSbmc 	if (lvar > rvar)
7630ef842dSbmc 		return (DT_GREATERTHAN);
777c478bd9Sstevel@tonic-gate 
787c478bd9Sstevel@tonic-gate 	return (0);
797c478bd9Sstevel@tonic-gate }
807c478bd9Sstevel@tonic-gate 
817c478bd9Sstevel@tonic-gate /*ARGSUSED*/
827c478bd9Sstevel@tonic-gate static void
dt_aggregate_min(int64_t * existing,int64_t * new,size_t size)83187eccf8Sbmc dt_aggregate_min(int64_t *existing, int64_t *new, size_t size)
847c478bd9Sstevel@tonic-gate {
857c478bd9Sstevel@tonic-gate 	if (*new < *existing)
867c478bd9Sstevel@tonic-gate 		*existing = *new;
877c478bd9Sstevel@tonic-gate }
887c478bd9Sstevel@tonic-gate 
897c478bd9Sstevel@tonic-gate /*ARGSUSED*/
907c478bd9Sstevel@tonic-gate static void
dt_aggregate_max(int64_t * existing,int64_t * new,size_t size)91187eccf8Sbmc dt_aggregate_max(int64_t *existing, int64_t *new, size_t size)
927c478bd9Sstevel@tonic-gate {
937c478bd9Sstevel@tonic-gate 	if (*new > *existing)
947c478bd9Sstevel@tonic-gate 		*existing = *new;
957c478bd9Sstevel@tonic-gate }
967c478bd9Sstevel@tonic-gate 
977c478bd9Sstevel@tonic-gate static int
dt_aggregate_averagecmp(int64_t * lhs,int64_t * rhs)98187eccf8Sbmc dt_aggregate_averagecmp(int64_t *lhs, int64_t *rhs)
997c478bd9Sstevel@tonic-gate {
100187eccf8Sbmc 	int64_t lavg = lhs[0] ? (lhs[1] / lhs[0]) : 0;
101187eccf8Sbmc 	int64_t ravg = rhs[0] ? (rhs[1] / rhs[0]) : 0;
1027c478bd9Sstevel@tonic-gate 
1037c478bd9Sstevel@tonic-gate 	if (lavg < ravg)
10430ef842dSbmc 		return (DT_LESSTHAN);
10530ef842dSbmc 
10630ef842dSbmc 	if (lavg > ravg)
10730ef842dSbmc 		return (DT_GREATERTHAN);
1087c478bd9Sstevel@tonic-gate 
1097c478bd9Sstevel@tonic-gate 	return (0);
1107c478bd9Sstevel@tonic-gate }
1117c478bd9Sstevel@tonic-gate 
1126e0bee74Sjhaslam static int
dt_aggregate_stddevcmp(int64_t * lhs,int64_t * rhs)1136e0bee74Sjhaslam dt_aggregate_stddevcmp(int64_t *lhs, int64_t *rhs)
1146e0bee74Sjhaslam {
1156e0bee74Sjhaslam 	uint64_t lsd = dt_stddev((uint64_t *)lhs, 1);
1166e0bee74Sjhaslam 	uint64_t rsd = dt_stddev((uint64_t *)rhs, 1);
1176e0bee74Sjhaslam 
1186e0bee74Sjhaslam 	if (lsd < rsd)
1196e0bee74Sjhaslam 		return (DT_LESSTHAN);
1206e0bee74Sjhaslam 
1216e0bee74Sjhaslam 	if (lsd > rsd)
1226e0bee74Sjhaslam 		return (DT_GREATERTHAN);
1236e0bee74Sjhaslam 
1246e0bee74Sjhaslam 	return (0);
1256e0bee74Sjhaslam }
1266e0bee74Sjhaslam 
1277c478bd9Sstevel@tonic-gate /*ARGSUSED*/
1287c478bd9Sstevel@tonic-gate static void
dt_aggregate_lquantize(int64_t * existing,int64_t * new,size_t size)129187eccf8Sbmc dt_aggregate_lquantize(int64_t *existing, int64_t *new, size_t size)
1307c478bd9Sstevel@tonic-gate {
131187eccf8Sbmc 	int64_t arg = *existing++;
1327c478bd9Sstevel@tonic-gate 	uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
1337c478bd9Sstevel@tonic-gate 	int i;
1347c478bd9Sstevel@tonic-gate 
1357c478bd9Sstevel@tonic-gate 	for (i = 0; i <= levels + 1; i++)
1367c478bd9Sstevel@tonic-gate 		existing[i] = existing[i] + new[i + 1];
1377c478bd9Sstevel@tonic-gate }
1387c478bd9Sstevel@tonic-gate 
139a1b5e537Sbmc static long double
dt_aggregate_lquantizedsum(int64_t * lquanta)140187eccf8Sbmc dt_aggregate_lquantizedsum(int64_t *lquanta)
1417c478bd9Sstevel@tonic-gate {
142187eccf8Sbmc 	int64_t arg = *lquanta++;
1437c478bd9Sstevel@tonic-gate 	int32_t base = DTRACE_LQUANTIZE_BASE(arg);
1447c478bd9Sstevel@tonic-gate 	uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
1457c478bd9Sstevel@tonic-gate 	uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg), i;
146a1b5e537Sbmc 	long double total = (long double)lquanta[0] * (long double)(base - 1);
1477c478bd9Sstevel@tonic-gate 
1487c478bd9Sstevel@tonic-gate 	for (i = 0; i < levels; base += step, i++)
149a1b5e537Sbmc 		total += (long double)lquanta[i + 1] * (long double)base;
1507c478bd9Sstevel@tonic-gate 
151a1b5e537Sbmc 	return (total + (long double)lquanta[levels + 1] *
152a1b5e537Sbmc 	    (long double)(base + 1));
1537c478bd9Sstevel@tonic-gate }
1547c478bd9Sstevel@tonic-gate 
155187eccf8Sbmc static int64_t
dt_aggregate_lquantizedzero(int64_t * lquanta)156187eccf8Sbmc dt_aggregate_lquantizedzero(int64_t *lquanta)
157187eccf8Sbmc {
158187eccf8Sbmc 	int64_t arg = *lquanta++;
159187eccf8Sbmc 	int32_t base = DTRACE_LQUANTIZE_BASE(arg);
160187eccf8Sbmc 	uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
161187eccf8Sbmc 	uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg), i;
162187eccf8Sbmc 
163187eccf8Sbmc 	if (base - 1 == 0)
164187eccf8Sbmc 		return (lquanta[0]);
165187eccf8Sbmc 
166187eccf8Sbmc 	for (i = 0; i < levels; base += step, i++) {
167187eccf8Sbmc 		if (base != 0)
168187eccf8Sbmc 			continue;
169187eccf8Sbmc 
170187eccf8Sbmc 		return (lquanta[i + 1]);
171187eccf8Sbmc 	}
172187eccf8Sbmc 
173187eccf8Sbmc 	if (base + 1 == 0)
174187eccf8Sbmc 		return (lquanta[levels + 1]);
175187eccf8Sbmc 
176187eccf8Sbmc 	return (0);
177187eccf8Sbmc }
178187eccf8Sbmc 
1797c478bd9Sstevel@tonic-gate static int
dt_aggregate_lquantizedcmp(int64_t * lhs,int64_t * rhs)180187eccf8Sbmc dt_aggregate_lquantizedcmp(int64_t *lhs, int64_t *rhs)
1817c478bd9Sstevel@tonic-gate {
182a1b5e537Sbmc 	long double lsum = dt_aggregate_lquantizedsum(lhs);
183a1b5e537Sbmc 	long double rsum = dt_aggregate_lquantizedsum(rhs);
184187eccf8Sbmc 	int64_t lzero, rzero;
1857c478bd9Sstevel@tonic-gate 
1867c478bd9Sstevel@tonic-gate 	if (lsum < rsum)
18730ef842dSbmc 		return (DT_LESSTHAN);
18830ef842dSbmc 
18930ef842dSbmc 	if (lsum > rsum)
19030ef842dSbmc 		return (DT_GREATERTHAN);
1917c478bd9Sstevel@tonic-gate 
192187eccf8Sbmc 	/*
193187eccf8Sbmc 	 * If they're both equal, then we will compare based on the weights at
194187eccf8Sbmc 	 * zero.  If the weights at zero are equal (or if zero is not within
195187eccf8Sbmc 	 * the range of the linear quantization), then this will be judged a
196187eccf8Sbmc 	 * tie and will be resolved based on the key comparison.
197187eccf8Sbmc 	 */
198187eccf8Sbmc 	lzero = dt_aggregate_lquantizedzero(lhs);
199187eccf8Sbmc 	rzero = dt_aggregate_lquantizedzero(rhs);
200187eccf8Sbmc 
201187eccf8Sbmc 	if (lzero < rzero)
20230ef842dSbmc 		return (DT_LESSTHAN);
20330ef842dSbmc 
20430ef842dSbmc 	if (lzero > rzero)
20530ef842dSbmc 		return (DT_GREATERTHAN);
206187eccf8Sbmc 
2077c478bd9Sstevel@tonic-gate 	return (0);
2087c478bd9Sstevel@tonic-gate }
2097c478bd9Sstevel@tonic-gate 
2102b6389efSBryan Cantrill static void
dt_aggregate_llquantize(int64_t * existing,int64_t * new,size_t size)2112b6389efSBryan Cantrill dt_aggregate_llquantize(int64_t *existing, int64_t *new, size_t size)
2122b6389efSBryan Cantrill {
2132b6389efSBryan Cantrill 	int i;
2142b6389efSBryan Cantrill 
2152b6389efSBryan Cantrill 	for (i = 1; i < size / sizeof (int64_t); i++)
2162b6389efSBryan Cantrill 		existing[i] = existing[i] + new[i];
2172b6389efSBryan Cantrill }
2182b6389efSBryan Cantrill 
2192b6389efSBryan Cantrill static long double
dt_aggregate_llquantizedsum(int64_t * llquanta)2202b6389efSBryan Cantrill dt_aggregate_llquantizedsum(int64_t *llquanta)
2212b6389efSBryan Cantrill {
2222b6389efSBryan Cantrill 	int64_t arg = *llquanta++;
2232b6389efSBryan Cantrill 	uint16_t factor = DTRACE_LLQUANTIZE_FACTOR(arg);
2242b6389efSBryan Cantrill 	uint16_t low = DTRACE_LLQUANTIZE_LOW(arg);
2252b6389efSBryan Cantrill 	uint16_t high = DTRACE_LLQUANTIZE_HIGH(arg);
2262b6389efSBryan Cantrill 	uint16_t nsteps = DTRACE_LLQUANTIZE_NSTEP(arg);
2272b6389efSBryan Cantrill 	int bin = 0, order;
2282b6389efSBryan Cantrill 	int64_t value = 1, next, step;
2292b6389efSBryan Cantrill 	long double total;
2302b6389efSBryan Cantrill 
2312b6389efSBryan Cantrill 	assert(nsteps >= factor);
2322b6389efSBryan Cantrill 	assert(nsteps % factor == 0);
2332b6389efSBryan Cantrill 
2342b6389efSBryan Cantrill 	for (order = 0; order < low; order++)
2352b6389efSBryan Cantrill 		value *= factor;
2362b6389efSBryan Cantrill 
2372b6389efSBryan Cantrill 	total = (long double)llquanta[bin++] * (long double)(value - 1);
2382b6389efSBryan Cantrill 
2392b6389efSBryan Cantrill 	next = value * factor;
2402b6389efSBryan Cantrill 	step = next > nsteps ? next / nsteps : 1;
2412b6389efSBryan Cantrill 
2422b6389efSBryan Cantrill 	while (order <= high) {
2432b6389efSBryan Cantrill 		assert(value < next);
2442b6389efSBryan Cantrill 		total += (long double)llquanta[bin++] * (long double)(value);
2452b6389efSBryan Cantrill 
2462b6389efSBryan Cantrill 		if ((value += step) != next)
2472b6389efSBryan Cantrill 			continue;
2482b6389efSBryan Cantrill 
2492b6389efSBryan Cantrill 		next = value * factor;
2502b6389efSBryan Cantrill 		step = next > nsteps ? next / nsteps : 1;
2512b6389efSBryan Cantrill 		order++;
2522b6389efSBryan Cantrill 	}
2532b6389efSBryan Cantrill 
2542b6389efSBryan Cantrill 	return (total + (long double)llquanta[bin] * (long double)value);
2552b6389efSBryan Cantrill }
2562b6389efSBryan Cantrill 
2572b6389efSBryan Cantrill static int
dt_aggregate_llquantizedcmp(int64_t * lhs,int64_t * rhs)2582b6389efSBryan Cantrill dt_aggregate_llquantizedcmp(int64_t *lhs, int64_t *rhs)
2592b6389efSBryan Cantrill {
2602b6389efSBryan Cantrill 	long double lsum = dt_aggregate_llquantizedsum(lhs);
2612b6389efSBryan Cantrill 	long double rsum = dt_aggregate_llquantizedsum(rhs);
2622b6389efSBryan Cantrill 	int64_t lzero, rzero;
2632b6389efSBryan Cantrill 
2642b6389efSBryan Cantrill 	if (lsum < rsum)
2652b6389efSBryan Cantrill 		return (DT_LESSTHAN);
2662b6389efSBryan Cantrill 
2672b6389efSBryan Cantrill 	if (lsum > rsum)
2682b6389efSBryan Cantrill 		return (DT_GREATERTHAN);
2692b6389efSBryan Cantrill 
2702b6389efSBryan Cantrill 	/*
2712b6389efSBryan Cantrill 	 * If they're both equal, then we will compare based on the weights at
2722b6389efSBryan Cantrill 	 * zero.  If the weights at zero are equal, then this will be judged a
2732b6389efSBryan Cantrill 	 * tie and will be resolved based on the key comparison.
2742b6389efSBryan Cantrill 	 */
2752b6389efSBryan Cantrill 	lzero = lhs[1];
2762b6389efSBryan Cantrill 	rzero = rhs[1];
2772b6389efSBryan Cantrill 
2782b6389efSBryan Cantrill 	if (lzero < rzero)
2792b6389efSBryan Cantrill 		return (DT_LESSTHAN);
2802b6389efSBryan Cantrill 
2812b6389efSBryan Cantrill 	if (lzero > rzero)
2822b6389efSBryan Cantrill 		return (DT_GREATERTHAN);
2832b6389efSBryan Cantrill 
2842b6389efSBryan Cantrill 	return (0);
2852b6389efSBryan Cantrill }
2862b6389efSBryan Cantrill 
2877c478bd9Sstevel@tonic-gate static int
dt_aggregate_quantizedcmp(int64_t * lhs,int64_t * rhs)288187eccf8Sbmc dt_aggregate_quantizedcmp(int64_t *lhs, int64_t *rhs)
2897c478bd9Sstevel@tonic-gate {
2907c478bd9Sstevel@tonic-gate 	int nbuckets = DTRACE_QUANTIZE_NBUCKETS, i;
291a1b5e537Sbmc 	long double ltotal = 0, rtotal = 0;
292187eccf8Sbmc 	int64_t lzero, rzero;
2937c478bd9Sstevel@tonic-gate 
2947c478bd9Sstevel@tonic-gate 	for (i = 0; i < nbuckets; i++) {
2957c478bd9Sstevel@tonic-gate 		int64_t bucketval = DTRACE_QUANTIZE_BUCKETVAL(i);
2967c478bd9Sstevel@tonic-gate 
297187eccf8Sbmc 		if (bucketval == 0) {
298187eccf8Sbmc 			lzero = lhs[i];
299187eccf8Sbmc 			rzero = rhs[i];
300187eccf8Sbmc 		}
301187eccf8Sbmc 
302a1b5e537Sbmc 		ltotal += (long double)bucketval * (long double)lhs[i];
303a1b5e537Sbmc 		rtotal += (long double)bucketval * (long double)rhs[i];
3047c478bd9Sstevel@tonic-gate 	}
3057c478bd9Sstevel@tonic-gate 
3067c478bd9Sstevel@tonic-gate 	if (ltotal < rtotal)
30730ef842dSbmc 		return (DT_LESSTHAN);
30830ef842dSbmc 
30930ef842dSbmc 	if (ltotal > rtotal)
31030ef842dSbmc 		return (DT_GREATERTHAN);
3117c478bd9Sstevel@tonic-gate 
312187eccf8Sbmc 	/*
313187eccf8Sbmc 	 * If they're both equal, then we will compare based on the weights at
314187eccf8Sbmc 	 * zero.  If the weights at zero are equal, then this will be judged a
315187eccf8Sbmc 	 * tie and will be resolved based on the key comparison.
316187eccf8Sbmc 	 */
317187eccf8Sbmc 	if (lzero < rzero)
31830ef842dSbmc 		return (DT_LESSTHAN);
31930ef842dSbmc 
32030ef842dSbmc 	if (lzero > rzero)
32130ef842dSbmc 		return (DT_GREATERTHAN);
322187eccf8Sbmc 
3237c478bd9Sstevel@tonic-gate 	return (0);
3247c478bd9Sstevel@tonic-gate }
3257c478bd9Sstevel@tonic-gate 
326a1b5e537Sbmc static void
dt_aggregate_usym(dtrace_hdl_t * dtp,uint64_t * data)327a1b5e537Sbmc dt_aggregate_usym(dtrace_hdl_t *dtp, uint64_t *data)
328a1b5e537Sbmc {
329a1b5e537Sbmc 	uint64_t pid = data[0];
330a1b5e537Sbmc 	uint64_t *pc = &data[1];
331a1b5e537Sbmc 	struct ps_prochandle *P;
332a1b5e537Sbmc 	GElf_Sym sym;
333a1b5e537Sbmc 
334a1b5e537Sbmc 	if (dtp->dt_vector != NULL)
335a1b5e537Sbmc 		return;
336a1b5e537Sbmc 
337a1b5e537Sbmc 	if ((P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0)) == NULL)
338a1b5e537Sbmc 		return;
339a1b5e537Sbmc 
340a1b5e537Sbmc 	dt_proc_lock(dtp, P);
341a1b5e537Sbmc 
342a1b5e537Sbmc 	if (Plookup_by_addr(P, *pc, NULL, 0, &sym) == 0)
343a1b5e537Sbmc 		*pc = sym.st_value;
344a1b5e537Sbmc 
345a1b5e537Sbmc 	dt_proc_unlock(dtp, P);
346a1b5e537Sbmc 	dt_proc_release(dtp, P);
347a1b5e537Sbmc }
348a1b5e537Sbmc 
349a1b5e537Sbmc static void
dt_aggregate_umod(dtrace_hdl_t * dtp,uint64_t * data)350a1b5e537Sbmc dt_aggregate_umod(dtrace_hdl_t *dtp, uint64_t *data)
351a1b5e537Sbmc {
352a1b5e537Sbmc 	uint64_t pid = data[0];
353a1b5e537Sbmc 	uint64_t *pc = &data[1];
354a1b5e537Sbmc 	struct ps_prochandle *P;
355a1b5e537Sbmc 	const prmap_t *map;
356a1b5e537Sbmc 
357a1b5e537Sbmc 	if (dtp->dt_vector != NULL)
358a1b5e537Sbmc 		return;
359a1b5e537Sbmc 
360a1b5e537Sbmc 	if ((P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0)) == NULL)
361a1b5e537Sbmc 		return;
362a1b5e537Sbmc 
363a1b5e537Sbmc 	dt_proc_lock(dtp, P);
364a1b5e537Sbmc 
365a1b5e537Sbmc 	if ((map = Paddr_to_map(P, *pc)) != NULL)
366a1b5e537Sbmc 		*pc = map->pr_vaddr;
367a1b5e537Sbmc 
368a1b5e537Sbmc 	dt_proc_unlock(dtp, P);
369a1b5e537Sbmc 	dt_proc_release(dtp, P);
370a1b5e537Sbmc }
371a1b5e537Sbmc 
372a1b5e537Sbmc static void
dt_aggregate_sym(dtrace_hdl_t * dtp,uint64_t * data)373a1b5e537Sbmc dt_aggregate_sym(dtrace_hdl_t *dtp, uint64_t *data)
374a1b5e537Sbmc {
375a1b5e537Sbmc 	GElf_Sym sym;
376a1b5e537Sbmc 	uint64_t *pc = data;
377a1b5e537Sbmc 
378a1b5e537Sbmc 	if (dtrace_lookup_by_addr(dtp, *pc, &sym, NULL) == 0)
379a1b5e537Sbmc 		*pc = sym.st_value;
380a1b5e537Sbmc }
381a1b5e537Sbmc 
382a1b5e537Sbmc static void
dt_aggregate_mod(dtrace_hdl_t * dtp,uint64_t * data)383a1b5e537Sbmc dt_aggregate_mod(dtrace_hdl_t *dtp, uint64_t *data)
384a1b5e537Sbmc {
385a1b5e537Sbmc 	uint64_t *pc = data;
386a1b5e537Sbmc 	dt_module_t *dmp;
387a1b5e537Sbmc 
388a1b5e537Sbmc 	if (dtp->dt_vector != NULL) {
389a1b5e537Sbmc 		/*
390a1b5e537Sbmc 		 * We don't have a way of just getting the module for a
391a1b5e537Sbmc 		 * vectored open, and it doesn't seem to be worth defining
392a1b5e537Sbmc 		 * one.  This means that use of mod() won't get true
393a1b5e537Sbmc 		 * aggregation in the postmortem case (some modules may
394a1b5e537Sbmc 		 * appear more than once in aggregation output).  It seems
395a1b5e537Sbmc 		 * unlikely that anyone will ever notice or care...
396a1b5e537Sbmc 		 */
397a1b5e537Sbmc 		return;
398a1b5e537Sbmc 	}
399a1b5e537Sbmc 
400a1b5e537Sbmc 	for (dmp = dt_list_next(&dtp->dt_modlist); dmp != NULL;
401a1b5e537Sbmc 	    dmp = dt_list_next(dmp)) {
402a1b5e537Sbmc 		if (*pc - dmp->dm_text_va < dmp->dm_text_size) {
403a1b5e537Sbmc 			*pc = dmp->dm_text_va;
404a1b5e537Sbmc 			return;
405a1b5e537Sbmc 		}
406a1b5e537Sbmc 	}
407a1b5e537Sbmc }
408a1b5e537Sbmc 
40930ef842dSbmc static dtrace_aggvarid_t
dt_aggregate_aggvarid(dt_ahashent_t * ent)41030ef842dSbmc dt_aggregate_aggvarid(dt_ahashent_t *ent)
41130ef842dSbmc {
41230ef842dSbmc 	dtrace_aggdesc_t *agg = ent->dtahe_data.dtada_desc;
41330ef842dSbmc 	caddr_t data = ent->dtahe_data.dtada_data;
41430ef842dSbmc 	dtrace_recdesc_t *rec = agg->dtagd_rec;
41530ef842dSbmc 
41630ef842dSbmc 	/*
41730ef842dSbmc 	 * First, we'll check the variable ID in the aggdesc.  If it's valid,
41830ef842dSbmc 	 * we'll return it.  If not, we'll use the compiler-generated ID
41930ef842dSbmc 	 * present as the first record.
42030ef842dSbmc 	 */
42130ef842dSbmc 	if (agg->dtagd_varid != DTRACE_AGGVARIDNONE)
42230ef842dSbmc 		return (agg->dtagd_varid);
42330ef842dSbmc 
42430ef842dSbmc 	agg->dtagd_varid = *((dtrace_aggvarid_t *)(uintptr_t)(data +
42530ef842dSbmc 	    rec->dtrd_offset));
42630ef842dSbmc 
42730ef842dSbmc 	return (agg->dtagd_varid);
42830ef842dSbmc }
42930ef842dSbmc 
43030ef842dSbmc 
4317c478bd9Sstevel@tonic-gate static int
dt_aggregate_snap_cpu(dtrace_hdl_t * dtp,processorid_t cpu)4327c478bd9Sstevel@tonic-gate dt_aggregate_snap_cpu(dtrace_hdl_t *dtp, processorid_t cpu)
4337c478bd9Sstevel@tonic-gate {
4347c478bd9Sstevel@tonic-gate 	dtrace_epid_t id;
4357c478bd9Sstevel@tonic-gate 	uint64_t hashval;
4367c478bd9Sstevel@tonic-gate 	size_t offs, roffs, size, ndx;
4377c478bd9Sstevel@tonic-gate 	int i, j, rval;
4387c478bd9Sstevel@tonic-gate 	caddr_t addr, data;
4397c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *rec;
4407c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
4417c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *agg;
4427c478bd9Sstevel@tonic-gate 	dt_ahash_t *hash = &agp->dtat_hash;
4437c478bd9Sstevel@tonic-gate 	dt_ahashent_t *h;
4447c478bd9Sstevel@tonic-gate 	dtrace_bufdesc_t b = agp->dtat_buf, *buf = &b;
4457c478bd9Sstevel@tonic-gate 	dtrace_aggdata_t *aggdata;
4467c478bd9Sstevel@tonic-gate 	int flags = agp->dtat_flags;
4477c478bd9Sstevel@tonic-gate 
4487c478bd9Sstevel@tonic-gate 	buf->dtbd_cpu = cpu;
4497c478bd9Sstevel@tonic-gate 
4507c478bd9Sstevel@tonic-gate 	if (dt_ioctl(dtp, DTRACEIOC_AGGSNAP, buf) == -1) {
4517c478bd9Sstevel@tonic-gate 		if (errno == ENOENT) {
4527c478bd9Sstevel@tonic-gate 			/*
4537c478bd9Sstevel@tonic-gate 			 * If that failed with ENOENT, it may be because the
4547c478bd9Sstevel@tonic-gate 			 * CPU was unconfigured.  This is okay; we'll just
4557c478bd9Sstevel@tonic-gate 			 * do nothing but return success.
4567c478bd9Sstevel@tonic-gate 			 */
4577c478bd9Sstevel@tonic-gate 			return (0);
4587c478bd9Sstevel@tonic-gate 		}
4597c478bd9Sstevel@tonic-gate 
4607c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, errno));
4617c478bd9Sstevel@tonic-gate 	}
4627c478bd9Sstevel@tonic-gate 
4637c478bd9Sstevel@tonic-gate 	if (buf->dtbd_drops != 0) {
4647c478bd9Sstevel@tonic-gate 		if (dt_handle_cpudrop(dtp, cpu,
4657c478bd9Sstevel@tonic-gate 		    DTRACEDROP_AGGREGATION, buf->dtbd_drops) == -1)
4667c478bd9Sstevel@tonic-gate 			return (-1);
4677c478bd9Sstevel@tonic-gate 	}
4687c478bd9Sstevel@tonic-gate 
4697c478bd9Sstevel@tonic-gate 	if (buf->dtbd_size == 0)
4707c478bd9Sstevel@tonic-gate 		return (0);
4717c478bd9Sstevel@tonic-gate 
4727c478bd9Sstevel@tonic-gate 	if (hash->dtah_hash == NULL) {
4737c478bd9Sstevel@tonic-gate 		size_t size;
4747c478bd9Sstevel@tonic-gate 
4757c478bd9Sstevel@tonic-gate 		hash->dtah_size = DTRACE_AHASHSIZE;
4767c478bd9Sstevel@tonic-gate 		size = hash->dtah_size * sizeof (dt_ahashent_t *);
4777c478bd9Sstevel@tonic-gate 
4787c478bd9Sstevel@tonic-gate 		if ((hash->dtah_hash = malloc(size)) == NULL)
4797c478bd9Sstevel@tonic-gate 			return (dt_set_errno(dtp, EDT_NOMEM));
4807c478bd9Sstevel@tonic-gate 
4817c478bd9Sstevel@tonic-gate 		bzero(hash->dtah_hash, size);
4827c478bd9Sstevel@tonic-gate 	}
4837c478bd9Sstevel@tonic-gate 
4847c478bd9Sstevel@tonic-gate 	for (offs = 0; offs < buf->dtbd_size; ) {
4857c478bd9Sstevel@tonic-gate 		/*
4867c478bd9Sstevel@tonic-gate 		 * We're guaranteed to have an ID.
4877c478bd9Sstevel@tonic-gate 		 */
4887c478bd9Sstevel@tonic-gate 		id = *((dtrace_epid_t *)((uintptr_t)buf->dtbd_data +
4897c478bd9Sstevel@tonic-gate 		    (uintptr_t)offs));
4907c478bd9Sstevel@tonic-gate 
4917c478bd9Sstevel@tonic-gate 		if (id == DTRACE_AGGIDNONE) {
4927c478bd9Sstevel@tonic-gate 			/*
4937c478bd9Sstevel@tonic-gate 			 * This is filler to assure proper alignment of the
4947c478bd9Sstevel@tonic-gate 			 * next record; we simply ignore it.
4957c478bd9Sstevel@tonic-gate 			 */
4967c478bd9Sstevel@tonic-gate 			offs += sizeof (id);
4977c478bd9Sstevel@tonic-gate 			continue;
4987c478bd9Sstevel@tonic-gate 		}
4997c478bd9Sstevel@tonic-gate 
5007c478bd9Sstevel@tonic-gate 		if ((rval = dt_aggid_lookup(dtp, id, &agg)) != 0)
5017c478bd9Sstevel@tonic-gate 			return (rval);
5027c478bd9Sstevel@tonic-gate 
5037c478bd9Sstevel@tonic-gate 		addr = buf->dtbd_data + offs;
5047c478bd9Sstevel@tonic-gate 		size = agg->dtagd_size;
5057c478bd9Sstevel@tonic-gate 		hashval = 0;
5067c478bd9Sstevel@tonic-gate 
5077c478bd9Sstevel@tonic-gate 		for (j = 0; j < agg->dtagd_nrecs - 1; j++) {
5087c478bd9Sstevel@tonic-gate 			rec = &agg->dtagd_rec[j];
5097c478bd9Sstevel@tonic-gate 			roffs = rec->dtrd_offset;
5107c478bd9Sstevel@tonic-gate 
511a1b5e537Sbmc 			switch (rec->dtrd_action) {
512a1b5e537Sbmc 			case DTRACEACT_USYM:
513a1b5e537Sbmc 				dt_aggregate_usym(dtp,
514a1b5e537Sbmc 				    /* LINTED - alignment */
515a1b5e537Sbmc 				    (uint64_t *)&addr[roffs]);
516a1b5e537Sbmc 				break;
517a1b5e537Sbmc 
518a1b5e537Sbmc 			case DTRACEACT_UMOD:
519a1b5e537Sbmc 				dt_aggregate_umod(dtp,
520a1b5e537Sbmc 				    /* LINTED - alignment */
521a1b5e537Sbmc 				    (uint64_t *)&addr[roffs]);
522a1b5e537Sbmc 				break;
523a1b5e537Sbmc 
524a1b5e537Sbmc 			case DTRACEACT_SYM:
525a1b5e537Sbmc 				/* LINTED - alignment */
526a1b5e537Sbmc 				dt_aggregate_sym(dtp, (uint64_t *)&addr[roffs]);
527a1b5e537Sbmc 				break;
528a1b5e537Sbmc 
529a1b5e537Sbmc 			case DTRACEACT_MOD:
530a1b5e537Sbmc 				/* LINTED - alignment */
531a1b5e537Sbmc 				dt_aggregate_mod(dtp, (uint64_t *)&addr[roffs]);
532a1b5e537Sbmc 				break;
533a1b5e537Sbmc 
534a1b5e537Sbmc 			default:
535a1b5e537Sbmc 				break;
536a1b5e537Sbmc 			}
537a1b5e537Sbmc 
5387c478bd9Sstevel@tonic-gate 			for (i = 0; i < rec->dtrd_size; i++)
5397c478bd9Sstevel@tonic-gate 				hashval += addr[roffs + i];
5407c478bd9Sstevel@tonic-gate 		}
5417c478bd9Sstevel@tonic-gate 
5427c478bd9Sstevel@tonic-gate 		ndx = hashval % hash->dtah_size;
5437c478bd9Sstevel@tonic-gate 
5447c478bd9Sstevel@tonic-gate 		for (h = hash->dtah_hash[ndx]; h != NULL; h = h->dtahe_next) {
5457c478bd9Sstevel@tonic-gate 			if (h->dtahe_hashval != hashval)
5467c478bd9Sstevel@tonic-gate 				continue;
5477c478bd9Sstevel@tonic-gate 
5487c478bd9Sstevel@tonic-gate 			if (h->dtahe_size != size)
5497c478bd9Sstevel@tonic-gate 				continue;
5507c478bd9Sstevel@tonic-gate 
5517c478bd9Sstevel@tonic-gate 			aggdata = &h->dtahe_data;
5527c478bd9Sstevel@tonic-gate 			data = aggdata->dtada_data;
5537c478bd9Sstevel@tonic-gate 
5547c478bd9Sstevel@tonic-gate 			for (j = 0; j < agg->dtagd_nrecs - 1; j++) {
5557c478bd9Sstevel@tonic-gate 				rec = &agg->dtagd_rec[j];
5567c478bd9Sstevel@tonic-gate 				roffs = rec->dtrd_offset;
5577c478bd9Sstevel@tonic-gate 
5587c478bd9Sstevel@tonic-gate 				for (i = 0; i < rec->dtrd_size; i++)
5597c478bd9Sstevel@tonic-gate 					if (addr[roffs + i] != data[roffs + i])
5607c478bd9Sstevel@tonic-gate 						goto hashnext;
5617c478bd9Sstevel@tonic-gate 			}
5627c478bd9Sstevel@tonic-gate 
5637c478bd9Sstevel@tonic-gate 			/*
5647c478bd9Sstevel@tonic-gate 			 * We found it.  Now we need to apply the aggregating
5657c478bd9Sstevel@tonic-gate 			 * action on the data here.
5667c478bd9Sstevel@tonic-gate 			 */
5677c478bd9Sstevel@tonic-gate 			rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
5687c478bd9Sstevel@tonic-gate 			roffs = rec->dtrd_offset;
5697c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
570187eccf8Sbmc 			h->dtahe_aggregate((int64_t *)&data[roffs],
5717c478bd9Sstevel@tonic-gate 			    /* LINTED - alignment */
572187eccf8Sbmc 			    (int64_t *)&addr[roffs], rec->dtrd_size);
5737c478bd9Sstevel@tonic-gate 
5747c478bd9Sstevel@tonic-gate 			/*
5757c478bd9Sstevel@tonic-gate 			 * If we're keeping per CPU data, apply the aggregating
5767c478bd9Sstevel@tonic-gate 			 * action there as well.
5777c478bd9Sstevel@tonic-gate 			 */
5787c478bd9Sstevel@tonic-gate 			if (aggdata->dtada_percpu != NULL) {
5797c478bd9Sstevel@tonic-gate 				data = aggdata->dtada_percpu[cpu];
5807c478bd9Sstevel@tonic-gate 
5817c478bd9Sstevel@tonic-gate 				/* LINTED - alignment */
582187eccf8Sbmc 				h->dtahe_aggregate((int64_t *)data,
5837c478bd9Sstevel@tonic-gate 				    /* LINTED - alignment */
584187eccf8Sbmc 				    (int64_t *)&addr[roffs], rec->dtrd_size);
5857c478bd9Sstevel@tonic-gate 			}
5867c478bd9Sstevel@tonic-gate 
5877c478bd9Sstevel@tonic-gate 			goto bufnext;
5887c478bd9Sstevel@tonic-gate hashnext:
5897c478bd9Sstevel@tonic-gate 			continue;
5907c478bd9Sstevel@tonic-gate 		}
5917c478bd9Sstevel@tonic-gate 
5927c478bd9Sstevel@tonic-gate 		/*
5937c478bd9