17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
57c478bd9Sstevel@tonic-gate  * Common Development and Distribution License, Version 1.0 only
67c478bd9Sstevel@tonic-gate  * (the "License").  You may not use this file except in compliance
77c478bd9Sstevel@tonic-gate  * with the License.
87c478bd9Sstevel@tonic-gate  *
97c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
107c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
117c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
127c478bd9Sstevel@tonic-gate  * and limitations under the License.
137c478bd9Sstevel@tonic-gate  *
147c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
157c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
167c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
177c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
187c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
197c478bd9Sstevel@tonic-gate  *
207c478bd9Sstevel@tonic-gate  * CDDL HEADER END
217c478bd9Sstevel@tonic-gate  */
227c478bd9Sstevel@tonic-gate /*
237c478bd9Sstevel@tonic-gate  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
277c478bd9Sstevel@tonic-gate #pragma ident	"%Z%%M%	%I%	%E% SMI"
287c478bd9Sstevel@tonic-gate 
297c478bd9Sstevel@tonic-gate #include <stdlib.h>
307c478bd9Sstevel@tonic-gate #include <strings.h>
317c478bd9Sstevel@tonic-gate #include <errno.h>
327c478bd9Sstevel@tonic-gate #include <unistd.h>
337c478bd9Sstevel@tonic-gate #include <dt_impl.h>
347c478bd9Sstevel@tonic-gate #include <assert.h>
357c478bd9Sstevel@tonic-gate 
367c478bd9Sstevel@tonic-gate #define	DTRACE_AHASHSIZE	32779		/* big 'ol prime */
377c478bd9Sstevel@tonic-gate 
387c478bd9Sstevel@tonic-gate static void
397c478bd9Sstevel@tonic-gate dt_aggregate_count(uint64_t *existing, uint64_t *new, size_t size)
407c478bd9Sstevel@tonic-gate {
417c478bd9Sstevel@tonic-gate 	int i;
427c478bd9Sstevel@tonic-gate 
437c478bd9Sstevel@tonic-gate 	for (i = 0; i < size / sizeof (uint64_t); i++)
447c478bd9Sstevel@tonic-gate 		existing[i] = existing[i] + new[i];
457c478bd9Sstevel@tonic-gate }
467c478bd9Sstevel@tonic-gate 
477c478bd9Sstevel@tonic-gate static int
487c478bd9Sstevel@tonic-gate dt_aggregate_countcmp(uint64_t *lhs, uint64_t *rhs)
497c478bd9Sstevel@tonic-gate {
507c478bd9Sstevel@tonic-gate 	uint64_t lvar = *lhs;
517c478bd9Sstevel@tonic-gate 	uint64_t rvar = *rhs;
527c478bd9Sstevel@tonic-gate 
537c478bd9Sstevel@tonic-gate 	if (lvar > rvar)
547c478bd9Sstevel@tonic-gate 		return (1);
557c478bd9Sstevel@tonic-gate 
567c478bd9Sstevel@tonic-gate 	if (lvar < rvar)
577c478bd9Sstevel@tonic-gate 		return (-1);
587c478bd9Sstevel@tonic-gate 
597c478bd9Sstevel@tonic-gate 	return (0);
607c478bd9Sstevel@tonic-gate }
617c478bd9Sstevel@tonic-gate 
627c478bd9Sstevel@tonic-gate /*ARGSUSED*/
637c478bd9Sstevel@tonic-gate static void
647c478bd9Sstevel@tonic-gate dt_aggregate_min(uint64_t *existing, uint64_t *new, size_t size)
657c478bd9Sstevel@tonic-gate {
667c478bd9Sstevel@tonic-gate 	if (*new < *existing)
677c478bd9Sstevel@tonic-gate 		*existing = *new;
687c478bd9Sstevel@tonic-gate }
697c478bd9Sstevel@tonic-gate 
707c478bd9Sstevel@tonic-gate /*ARGSUSED*/
717c478bd9Sstevel@tonic-gate static void
727c478bd9Sstevel@tonic-gate dt_aggregate_max(uint64_t *existing, uint64_t *new, size_t size)
737c478bd9Sstevel@tonic-gate {
747c478bd9Sstevel@tonic-gate 	if (*new > *existing)
757c478bd9Sstevel@tonic-gate 		*existing = *new;
767c478bd9Sstevel@tonic-gate }
777c478bd9Sstevel@tonic-gate 
787c478bd9Sstevel@tonic-gate static int
797c478bd9Sstevel@tonic-gate dt_aggregate_averagecmp(uint64_t *lhs, uint64_t *rhs)
807c478bd9Sstevel@tonic-gate {
817c478bd9Sstevel@tonic-gate 	uint64_t lavg = lhs[0] ? (lhs[1] / lhs[0]) : 0;
827c478bd9Sstevel@tonic-gate 	uint64_t ravg = rhs[0] ? (rhs[1] / rhs[0]) : 0;
837c478bd9Sstevel@tonic-gate 
847c478bd9Sstevel@tonic-gate 	if (lavg > ravg)
857c478bd9Sstevel@tonic-gate 		return (1);
867c478bd9Sstevel@tonic-gate 
877c478bd9Sstevel@tonic-gate 	if (lavg < ravg)
887c478bd9Sstevel@tonic-gate 		return (-1);
897c478bd9Sstevel@tonic-gate 
907c478bd9Sstevel@tonic-gate 	return (0);
917c478bd9Sstevel@tonic-gate }
927c478bd9Sstevel@tonic-gate 
937c478bd9Sstevel@tonic-gate /*ARGSUSED*/
947c478bd9Sstevel@tonic-gate static void
957c478bd9Sstevel@tonic-gate dt_aggregate_lquantize(uint64_t *existing, uint64_t *new, size_t size)
967c478bd9Sstevel@tonic-gate {
977c478bd9Sstevel@tonic-gate 	uint64_t arg = *existing++;
987c478bd9Sstevel@tonic-gate 	uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
997c478bd9Sstevel@tonic-gate 	int i;
1007c478bd9Sstevel@tonic-gate 
1017c478bd9Sstevel@tonic-gate 	for (i = 0; i <= levels + 1; i++)
1027c478bd9Sstevel@tonic-gate 		existing[i] = existing[i] + new[i + 1];
1037c478bd9Sstevel@tonic-gate }
1047c478bd9Sstevel@tonic-gate 
105*a1b5e537Sbmc static long double
1067c478bd9Sstevel@tonic-gate dt_aggregate_lquantizedsum(uint64_t *lquanta)
1077c478bd9Sstevel@tonic-gate {
1087c478bd9Sstevel@tonic-gate 	uint64_t arg = *lquanta++;
1097c478bd9Sstevel@tonic-gate 	int32_t base = DTRACE_LQUANTIZE_BASE(arg);
1107c478bd9Sstevel@tonic-gate 	uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
1117c478bd9Sstevel@tonic-gate 	uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg), i;
112*a1b5e537Sbmc 	long double total = (long double)lquanta[0] * (long double)(base - 1);
1137c478bd9Sstevel@tonic-gate 
1147c478bd9Sstevel@tonic-gate 	for (i = 0; i < levels; base += step, i++)
115*a1b5e537Sbmc 		total += (long double)lquanta[i + 1] * (long double)base;
1167c478bd9Sstevel@tonic-gate 
117*a1b5e537Sbmc 	return (total + (long double)lquanta[levels + 1] *
118*a1b5e537Sbmc 	    (long double)(base + 1));
1197c478bd9Sstevel@tonic-gate }
1207c478bd9Sstevel@tonic-gate 
1217c478bd9Sstevel@tonic-gate static int
1227c478bd9Sstevel@tonic-gate dt_aggregate_lquantizedcmp(uint64_t *lhs, uint64_t *rhs)
1237c478bd9Sstevel@tonic-gate {
124*a1b5e537Sbmc 	long double lsum = dt_aggregate_lquantizedsum(lhs);
125*a1b5e537Sbmc 	long double rsum = dt_aggregate_lquantizedsum(rhs);
1267c478bd9Sstevel@tonic-gate 
1277c478bd9Sstevel@tonic-gate 	if (lsum > rsum)
1287c478bd9Sstevel@tonic-gate 		return (1);
1297c478bd9Sstevel@tonic-gate 
1307c478bd9Sstevel@tonic-gate 	if (lsum < rsum)
1317c478bd9Sstevel@tonic-gate 		return (-1);
1327c478bd9Sstevel@tonic-gate 
1337c478bd9Sstevel@tonic-gate 	return (0);
1347c478bd9Sstevel@tonic-gate }
1357c478bd9Sstevel@tonic-gate 
1367c478bd9Sstevel@tonic-gate static int
1377c478bd9Sstevel@tonic-gate dt_aggregate_quantizedcmp(uint64_t *lhs, uint64_t *rhs)
1387c478bd9Sstevel@tonic-gate {
1397c478bd9Sstevel@tonic-gate 	int nbuckets = DTRACE_QUANTIZE_NBUCKETS, i;
140*a1b5e537Sbmc 	long double ltotal = 0, rtotal = 0;
1417c478bd9Sstevel@tonic-gate 
1427c478bd9Sstevel@tonic-gate 	for (i = 0; i < nbuckets; i++) {
1437c478bd9Sstevel@tonic-gate 		int64_t bucketval = DTRACE_QUANTIZE_BUCKETVAL(i);
1447c478bd9Sstevel@tonic-gate 
145*a1b5e537Sbmc 		ltotal += (long double)bucketval * (long double)lhs[i];
146*a1b5e537Sbmc 		rtotal += (long double)bucketval * (long double)rhs[i];
1477c478bd9Sstevel@tonic-gate 	}
1487c478bd9Sstevel@tonic-gate 
1497c478bd9Sstevel@tonic-gate 	if (ltotal > rtotal)
1507c478bd9Sstevel@tonic-gate 		return (1);
1517c478bd9Sstevel@tonic-gate 
1527c478bd9Sstevel@tonic-gate 	if (ltotal < rtotal)
1537c478bd9Sstevel@tonic-gate 		return (-1);
1547c478bd9Sstevel@tonic-gate 
1557c478bd9Sstevel@tonic-gate 	return (0);
1567c478bd9Sstevel@tonic-gate }
1577c478bd9Sstevel@tonic-gate 
158*a1b5e537Sbmc static void
159*a1b5e537Sbmc dt_aggregate_usym(dtrace_hdl_t *dtp, uint64_t *data)
160*a1b5e537Sbmc {
161*a1b5e537Sbmc 	uint64_t pid = data[0];
162*a1b5e537Sbmc 	uint64_t *pc = &data[1];
163*a1b5e537Sbmc 	struct ps_prochandle *P;
164*a1b5e537Sbmc 	GElf_Sym sym;
165*a1b5e537Sbmc 
166*a1b5e537Sbmc 	if (dtp->dt_vector != NULL)
167*a1b5e537Sbmc 		return;
168*a1b5e537Sbmc 
169*a1b5e537Sbmc 	if ((P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0)) == NULL)
170*a1b5e537Sbmc 		return;
171*a1b5e537Sbmc 
172*a1b5e537Sbmc 	dt_proc_lock(dtp, P);
173*a1b5e537Sbmc 
174*a1b5e537Sbmc 	if (Plookup_by_addr(P, *pc, NULL, 0, &sym) == 0)
175*a1b5e537Sbmc 		*pc = sym.st_value;
176*a1b5e537Sbmc 
177*a1b5e537Sbmc 	dt_proc_unlock(dtp, P);
178*a1b5e537Sbmc 	dt_proc_release(dtp, P);
179*a1b5e537Sbmc }
180*a1b5e537Sbmc 
181*a1b5e537Sbmc static void
182*a1b5e537Sbmc dt_aggregate_umod(dtrace_hdl_t *dtp, uint64_t *data)
183*a1b5e537Sbmc {
184*a1b5e537Sbmc 	uint64_t pid = data[0];
185*a1b5e537Sbmc 	uint64_t *pc = &data[1];
186*a1b5e537Sbmc 	struct ps_prochandle *P;
187*a1b5e537Sbmc 	const prmap_t *map;
188*a1b5e537Sbmc 
189*a1b5e537Sbmc 	if (dtp->dt_vector != NULL)
190*a1b5e537Sbmc 		return;
191*a1b5e537Sbmc 
192*a1b5e537Sbmc 	if ((P = dt_proc_grab(dtp, pid, PGRAB_RDONLY | PGRAB_FORCE, 0)) == NULL)
193*a1b5e537Sbmc 		return;
194*a1b5e537Sbmc 
195*a1b5e537Sbmc 	dt_proc_lock(dtp, P);
196*a1b5e537Sbmc 
197*a1b5e537Sbmc 	if ((map = Paddr_to_map(P, *pc)) != NULL)
198*a1b5e537Sbmc 		*pc = map->pr_vaddr;
199*a1b5e537Sbmc 
200*a1b5e537Sbmc 	dt_proc_unlock(dtp, P);
201*a1b5e537Sbmc 	dt_proc_release(dtp, P);
202*a1b5e537Sbmc }
203*a1b5e537Sbmc 
204*a1b5e537Sbmc static void
205*a1b5e537Sbmc dt_aggregate_sym(dtrace_hdl_t *dtp, uint64_t *data)
206*a1b5e537Sbmc {
207*a1b5e537Sbmc 	GElf_Sym sym;
208*a1b5e537Sbmc 	uint64_t *pc = data;
209*a1b5e537Sbmc 
210*a1b5e537Sbmc 	if (dtrace_lookup_by_addr(dtp, *pc, &sym, NULL) == 0)
211*a1b5e537Sbmc 		*pc = sym.st_value;
212*a1b5e537Sbmc }
213*a1b5e537Sbmc 
214*a1b5e537Sbmc static void
215*a1b5e537Sbmc dt_aggregate_mod(dtrace_hdl_t *dtp, uint64_t *data)
216*a1b5e537Sbmc {
217*a1b5e537Sbmc 	uint64_t *pc = data;
218*a1b5e537Sbmc 	dt_module_t *dmp;
219*a1b5e537Sbmc 
220*a1b5e537Sbmc 	if (dtp->dt_vector != NULL) {
221*a1b5e537Sbmc 		/*
222*a1b5e537Sbmc 		 * We don't have a way of just getting the module for a
223*a1b5e537Sbmc 		 * vectored open, and it doesn't seem to be worth defining
224*a1b5e537Sbmc 		 * one.  This means that use of mod() won't get true
225*a1b5e537Sbmc 		 * aggregation in the postmortem case (some modules may
226*a1b5e537Sbmc 		 * appear more than once in aggregation output).  It seems
227*a1b5e537Sbmc 		 * unlikely that anyone will ever notice or care...
228*a1b5e537Sbmc 		 */
229*a1b5e537Sbmc 		return;
230*a1b5e537Sbmc 	}
231*a1b5e537Sbmc 
232*a1b5e537Sbmc 	for (dmp = dt_list_next(&dtp->dt_modlist); dmp != NULL;
233*a1b5e537Sbmc 	    dmp = dt_list_next(dmp)) {
234*a1b5e537Sbmc 		if (*pc - dmp->dm_text_va < dmp->dm_text_size) {
235*a1b5e537Sbmc 			*pc = dmp->dm_text_va;
236*a1b5e537Sbmc 			return;
237*a1b5e537Sbmc 		}
238*a1b5e537Sbmc 	}
239*a1b5e537Sbmc }
240*a1b5e537Sbmc 
2417c478bd9Sstevel@tonic-gate static int
2427c478bd9Sstevel@tonic-gate dt_aggregate_snap_cpu(dtrace_hdl_t *dtp, processorid_t cpu)
2437c478bd9Sstevel@tonic-gate {
2447c478bd9Sstevel@tonic-gate 	dtrace_epid_t id;
2457c478bd9Sstevel@tonic-gate 	uint64_t hashval;
2467c478bd9Sstevel@tonic-gate 	size_t offs, roffs, size, ndx;
2477c478bd9Sstevel@tonic-gate 	int i, j, rval;
2487c478bd9Sstevel@tonic-gate 	caddr_t addr, data;
2497c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *rec;
2507c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
2517c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *agg;
2527c478bd9Sstevel@tonic-gate 	dt_ahash_t *hash = &agp->dtat_hash;
2537c478bd9Sstevel@tonic-gate 	dt_ahashent_t *h;
2547c478bd9Sstevel@tonic-gate 	dtrace_bufdesc_t b = agp->dtat_buf, *buf = &b;
2557c478bd9Sstevel@tonic-gate 	dtrace_aggdata_t *aggdata;
2567c478bd9Sstevel@tonic-gate 	int flags = agp->dtat_flags;
2577c478bd9Sstevel@tonic-gate 
2587c478bd9Sstevel@tonic-gate 	buf->dtbd_cpu = cpu;
2597c478bd9Sstevel@tonic-gate 
2607c478bd9Sstevel@tonic-gate 	if (dt_ioctl(dtp, DTRACEIOC_AGGSNAP, buf) == -1) {
2617c478bd9Sstevel@tonic-gate 		if (errno == ENOENT) {
2627c478bd9Sstevel@tonic-gate 			/*
2637c478bd9Sstevel@tonic-gate 			 * If that failed with ENOENT, it may be because the
2647c478bd9Sstevel@tonic-gate 			 * CPU was unconfigured.  This is okay; we'll just
2657c478bd9Sstevel@tonic-gate 			 * do nothing but return success.
2667c478bd9Sstevel@tonic-gate 			 */
2677c478bd9Sstevel@tonic-gate 			return (0);
2687c478bd9Sstevel@tonic-gate 		}
2697c478bd9Sstevel@tonic-gate 
2707c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, errno));
2717c478bd9Sstevel@tonic-gate 	}
2727c478bd9Sstevel@tonic-gate 
2737c478bd9Sstevel@tonic-gate 	if (buf->dtbd_drops != 0) {
2747c478bd9Sstevel@tonic-gate 		if (dt_handle_cpudrop(dtp, cpu,
2757c478bd9Sstevel@tonic-gate 		    DTRACEDROP_AGGREGATION, buf->dtbd_drops) == -1)
2767c478bd9Sstevel@tonic-gate 			return (-1);
2777c478bd9Sstevel@tonic-gate 	}
2787c478bd9Sstevel@tonic-gate 
2797c478bd9Sstevel@tonic-gate 	if (buf->dtbd_size == 0)
2807c478bd9Sstevel@tonic-gate 		return (0);
2817c478bd9Sstevel@tonic-gate 
2827c478bd9Sstevel@tonic-gate 	if (hash->dtah_hash == NULL) {
2837c478bd9Sstevel@tonic-gate 		size_t size;
2847c478bd9Sstevel@tonic-gate 
2857c478bd9Sstevel@tonic-gate 		hash->dtah_size = DTRACE_AHASHSIZE;
2867c478bd9Sstevel@tonic-gate 		size = hash->dtah_size * sizeof (dt_ahashent_t *);
2877c478bd9Sstevel@tonic-gate 
2887c478bd9Sstevel@tonic-gate 		if ((hash->dtah_hash = malloc(size)) == NULL)
2897c478bd9Sstevel@tonic-gate 			return (dt_set_errno(dtp, EDT_NOMEM));
2907c478bd9Sstevel@tonic-gate 
2917c478bd9Sstevel@tonic-gate 		bzero(hash->dtah_hash, size);
2927c478bd9Sstevel@tonic-gate 	}
2937c478bd9Sstevel@tonic-gate 
2947c478bd9Sstevel@tonic-gate 	for (offs = 0; offs < buf->dtbd_size; ) {
2957c478bd9Sstevel@tonic-gate 		/*
2967c478bd9Sstevel@tonic-gate 		 * We're guaranteed to have an ID.
2977c478bd9Sstevel@tonic-gate 		 */
2987c478bd9Sstevel@tonic-gate 		id = *((dtrace_epid_t *)((uintptr_t)buf->dtbd_data +
2997c478bd9Sstevel@tonic-gate 		    (uintptr_t)offs));
3007c478bd9Sstevel@tonic-gate 
3017c478bd9Sstevel@tonic-gate 		if (id == DTRACE_AGGIDNONE) {
3027c478bd9Sstevel@tonic-gate 			/*
3037c478bd9Sstevel@tonic-gate 			 * This is filler to assure proper alignment of the
3047c478bd9Sstevel@tonic-gate 			 * next record; we simply ignore it.
3057c478bd9Sstevel@tonic-gate 			 */
3067c478bd9Sstevel@tonic-gate 			offs += sizeof (id);
3077c478bd9Sstevel@tonic-gate 			continue;
3087c478bd9Sstevel@tonic-gate 		}
3097c478bd9Sstevel@tonic-gate 
3107c478bd9Sstevel@tonic-gate 		if ((rval = dt_aggid_lookup(dtp, id, &agg)) != 0)
3117c478bd9Sstevel@tonic-gate 			return (rval);
3127c478bd9Sstevel@tonic-gate 
3137c478bd9Sstevel@tonic-gate 		addr = buf->dtbd_data + offs;
3147c478bd9Sstevel@tonic-gate 		size = agg->dtagd_size;
3157c478bd9Sstevel@tonic-gate 		hashval = 0;
3167c478bd9Sstevel@tonic-gate 
3177c478bd9Sstevel@tonic-gate 		for (j = 0; j < agg->dtagd_nrecs - 1; j++) {
3187c478bd9Sstevel@tonic-gate 			rec = &agg->dtagd_rec[j];
3197c478bd9Sstevel@tonic-gate 			roffs = rec->dtrd_offset;
3207c478bd9Sstevel@tonic-gate 
321*a1b5e537Sbmc 			switch (rec->dtrd_action) {
322*a1b5e537Sbmc 			case DTRACEACT_USYM:
323*a1b5e537Sbmc 				dt_aggregate_usym(dtp,
324*a1b5e537Sbmc 				    /* LINTED - alignment */
325*a1b5e537Sbmc 				    (uint64_t *)&addr[roffs]);
326*a1b5e537Sbmc 				break;
327*a1b5e537Sbmc 
328*a1b5e537Sbmc 			case DTRACEACT_UMOD:
329*a1b5e537Sbmc 				dt_aggregate_umod(dtp,
330*a1b5e537Sbmc 				    /* LINTED - alignment */
331*a1b5e537Sbmc 				    (uint64_t *)&addr[roffs]);
332*a1b5e537Sbmc 				break;
333*a1b5e537Sbmc 
334*a1b5e537Sbmc 			case DTRACEACT_SYM:
335*a1b5e537Sbmc 				/* LINTED - alignment */
336*a1b5e537Sbmc 				dt_aggregate_sym(dtp, (uint64_t *)&addr[roffs]);
337*a1b5e537Sbmc 				break;
338*a1b5e537Sbmc 
339*a1b5e537Sbmc 			case DTRACEACT_MOD:
340*a1b5e537Sbmc 				/* LINTED - alignment */
341*a1b5e537Sbmc 				dt_aggregate_mod(dtp, (uint64_t *)&addr[roffs]);
342*a1b5e537Sbmc 				break;
343*a1b5e537Sbmc 
344*a1b5e537Sbmc 			default:
345*a1b5e537Sbmc 				break;
346*a1b5e537Sbmc 			}
347*a1b5e537Sbmc 
3487c478bd9Sstevel@tonic-gate 			for (i = 0; i < rec->dtrd_size; i++)
3497c478bd9Sstevel@tonic-gate 				hashval += addr[roffs + i];
3507c478bd9Sstevel@tonic-gate 		}
3517c478bd9Sstevel@tonic-gate 
3527c478bd9Sstevel@tonic-gate 		ndx = hashval % hash->dtah_size;
3537c478bd9Sstevel@tonic-gate 
3547c478bd9Sstevel@tonic-gate 		for (h = hash->dtah_hash[ndx]; h != NULL; h = h->dtahe_next) {
3557c478bd9Sstevel@tonic-gate 			if (h->dtahe_hashval != hashval)
3567c478bd9Sstevel@tonic-gate 				continue;
3577c478bd9Sstevel@tonic-gate 
3587c478bd9Sstevel@tonic-gate 			if (h->dtahe_size != size)
3597c478bd9Sstevel@tonic-gate 				continue;
3607c478bd9Sstevel@tonic-gate 
3617c478bd9Sstevel@tonic-gate 			aggdata = &h->dtahe_data;
3627c478bd9Sstevel@tonic-gate 			data = aggdata->dtada_data;
3637c478bd9Sstevel@tonic-gate 
3647c478bd9Sstevel@tonic-gate 			for (j = 0; j < agg->dtagd_nrecs - 1; j++) {
3657c478bd9Sstevel@tonic-gate 				rec = &agg->dtagd_rec[j];
3667c478bd9Sstevel@tonic-gate 				roffs = rec->dtrd_offset;
3677c478bd9Sstevel@tonic-gate 
3687c478bd9Sstevel@tonic-gate 				for (i = 0; i < rec->dtrd_size; i++)
3697c478bd9Sstevel@tonic-gate 					if (addr[roffs + i] != data[roffs + i])
3707c478bd9Sstevel@tonic-gate 						goto hashnext;
3717c478bd9Sstevel@tonic-gate 			}
3727c478bd9Sstevel@tonic-gate 
3737c478bd9Sstevel@tonic-gate 			/*
3747c478bd9Sstevel@tonic-gate 			 * We found it.  Now we need to apply the aggregating
3757c478bd9Sstevel@tonic-gate 			 * action on the data here.
3767c478bd9Sstevel@tonic-gate 			 */
3777c478bd9Sstevel@tonic-gate 			rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
3787c478bd9Sstevel@tonic-gate 			roffs = rec->dtrd_offset;
3797c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
3807c478bd9Sstevel@tonic-gate 			h->dtahe_aggregate((uint64_t *)&data[roffs],
3817c478bd9Sstevel@tonic-gate 			    /* LINTED - alignment */
3827c478bd9Sstevel@tonic-gate 			    (uint64_t *)&addr[roffs], rec->dtrd_size);
3837c478bd9Sstevel@tonic-gate 
3847c478bd9Sstevel@tonic-gate 			/*
3857c478bd9Sstevel@tonic-gate 			 * If we're keeping per CPU data, apply the aggregating
3867c478bd9Sstevel@tonic-gate 			 * action there as well.
3877c478bd9Sstevel@tonic-gate 			 */
3887c478bd9Sstevel@tonic-gate 			if (aggdata->dtada_percpu != NULL) {
3897c478bd9Sstevel@tonic-gate 				data = aggdata->dtada_percpu[cpu];
3907c478bd9Sstevel@tonic-gate 
3917c478bd9Sstevel@tonic-gate 				/* LINTED - alignment */
3927c478bd9Sstevel@tonic-gate 				h->dtahe_aggregate((uint64_t *)data,
3937c478bd9Sstevel@tonic-gate 				    /* LINTED - alignment */
3947c478bd9Sstevel@tonic-gate 				    (uint64_t *)&addr[roffs], rec->dtrd_size);
3957c478bd9Sstevel@tonic-gate 			}
3967c478bd9Sstevel@tonic-gate 
3977c478bd9Sstevel@tonic-gate 			goto bufnext;
3987c478bd9Sstevel@tonic-gate hashnext:
3997c478bd9Sstevel@tonic-gate 			continue;
4007c478bd9Sstevel@tonic-gate 		}
4017c478bd9Sstevel@tonic-gate 
4027c478bd9Sstevel@tonic-gate 		/*
4037c478bd9Sstevel@tonic-gate 		 * If we're here, we couldn't find an entry for this record.
4047c478bd9Sstevel@tonic-gate 		 */
4057c478bd9Sstevel@tonic-gate 		if ((h = malloc(sizeof (dt_ahashent_t))) == NULL)
4067c478bd9Sstevel@tonic-gate 			return (dt_set_errno(dtp, EDT_NOMEM));
4077c478bd9Sstevel@tonic-gate 		bzero(h, sizeof (dt_ahashent_t));
4087c478bd9Sstevel@tonic-gate 		aggdata = &h->dtahe_data;
4097c478bd9Sstevel@tonic-gate 
4107c478bd9Sstevel@tonic-gate 		if ((aggdata->dtada_data = malloc(size)) == NULL) {
4117c478bd9Sstevel@tonic-gate 			free(h);
4127c478bd9Sstevel@tonic-gate 			return (dt_set_errno(dtp, EDT_NOMEM));
4137c478bd9Sstevel@tonic-gate 		}
4147c478bd9Sstevel@tonic-gate 
4157c478bd9Sstevel@tonic-gate 		bcopy(addr, aggdata->dtada_data, size);
4167c478bd9Sstevel@tonic-gate 		aggdata->dtada_size = size;
4177c478bd9Sstevel@tonic-gate 		aggdata->dtada_desc = agg;
4187c478bd9Sstevel@tonic-gate 		aggdata->dtada_handle = dtp;
4197c478bd9Sstevel@tonic-gate 		(void) dt_epid_lookup(dtp, agg->dtagd_epid,
4207c478bd9Sstevel@tonic-gate 		    &aggdata->dtada_edesc, &aggdata->dtada_pdesc);
4217c478bd9Sstevel@tonic-gate 		aggdata->dtada_normal = 1;
4227c478bd9Sstevel@tonic-gate 
4237c478bd9Sstevel@tonic-gate 		h->dtahe_hashval = hashval;
4247c478bd9Sstevel@tonic-gate 		h->dtahe_size = size;
4257c478bd9Sstevel@tonic-gate 
4267c478bd9Sstevel@tonic-gate 		rec = &agg->dtagd_rec[agg->dtagd_nrecs - 1];
4277c478bd9Sstevel@tonic-gate 
4287c478bd9Sstevel@tonic-gate 		if (flags & DTRACE_A_PERCPU) {
4297c478bd9Sstevel@tonic-gate 			int max_cpus = agp->dtat_maxcpu;
4307c478bd9Sstevel@tonic-gate 			caddr_t *percpu = malloc(max_cpus * sizeof (caddr_t));
4317c478bd9Sstevel@tonic-gate 
4327c478bd9Sstevel@tonic-gate 			if (percpu == NULL) {
4337c478bd9Sstevel@tonic-gate 				free(aggdata->dtada_data);
4347c478bd9Sstevel@tonic-gate 				free(h);
4357c478bd9Sstevel@tonic-gate 				return (dt_set_errno(dtp, EDT_NOMEM));
4367c478bd9Sstevel@tonic-gate 			}
4377c478bd9Sstevel@tonic-gate 
4387c478bd9Sstevel@tonic-gate 			for (j = 0; j < max_cpus; j++) {
4397c478bd9Sstevel@tonic-gate 				percpu[j] = malloc(rec->dtrd_size);
4407c478bd9Sstevel@tonic-gate 
4417c478bd9Sstevel@tonic-gate 				if (percpu[j] == NULL) {
4427c478bd9Sstevel@tonic-gate 					while (--j >= 0)
4437c478bd9Sstevel@tonic-gate 						free(percpu[j]);
4447c478bd9Sstevel@tonic-gate 
4457c478bd9Sstevel@tonic-gate 					free(aggdata->dtada_data);
4467c478bd9Sstevel@tonic-gate 					free(h);
4477c478bd9Sstevel@tonic-gate 					return (dt_set_errno(dtp, EDT_NOMEM));
4487c478bd9Sstevel@tonic-gate 				}
4497c478bd9Sstevel@tonic-gate 
4507c478bd9Sstevel@tonic-gate 				if (j == cpu) {
4517c478bd9Sstevel@tonic-gate 					bcopy(&addr[rec->dtrd_offset],
4527c478bd9Sstevel@tonic-gate 					    percpu[j], rec->dtrd_size);
4537c478bd9Sstevel@tonic-gate 				} else {
4547c478bd9Sstevel@tonic-gate 					bzero(percpu[j], rec->dtrd_size);
4557c478bd9Sstevel@tonic-gate 				}
4567c478bd9Sstevel@tonic-gate 			}
4577c478bd9Sstevel@tonic-gate 
4587c478bd9Sstevel@tonic-gate 			aggdata->dtada_percpu = percpu;
4597c478bd9Sstevel@tonic-gate 		}
4607c478bd9Sstevel@tonic-gate 
4617c478bd9Sstevel@tonic-gate 		switch (rec->dtrd_action) {
4627c478bd9Sstevel@tonic-gate 		case DTRACEAGG_MIN:
4637c478bd9Sstevel@tonic-gate 			h->dtahe_aggregate = dt_aggregate_min;
4647c478bd9Sstevel@tonic-gate 			break;
4657c478bd9Sstevel@tonic-gate 
4667c478bd9Sstevel@tonic-gate 		case DTRACEAGG_MAX:
4677c478bd9Sstevel@tonic-gate 			h->dtahe_aggregate = dt_aggregate_max;
4687c478bd9Sstevel@tonic-gate 			break;
4697c478bd9Sstevel@tonic-gate 
4707c478bd9Sstevel@tonic-gate 		case DTRACEAGG_LQUANTIZE:
4717c478bd9Sstevel@tonic-gate 			h->dtahe_aggregate = dt_aggregate_lquantize;
4727c478bd9Sstevel@tonic-gate 			break;
4737c478bd9Sstevel@tonic-gate 
4747c478bd9Sstevel@tonic-gate 		case DTRACEAGG_COUNT:
4757c478bd9Sstevel@tonic-gate 		case DTRACEAGG_SUM:
4767c478bd9Sstevel@tonic-gate 		case DTRACEAGG_AVG:
4777c478bd9Sstevel@tonic-gate 		case DTRACEAGG_QUANTIZE:
4787c478bd9Sstevel@tonic-gate 			h->dtahe_aggregate = dt_aggregate_count;
4797c478bd9Sstevel@tonic-gate 			break;
4807c478bd9Sstevel@tonic-gate 
4817c478bd9Sstevel@tonic-gate 		default:
4827c478bd9Sstevel@tonic-gate 			return (dt_set_errno(dtp, EDT_BADAGG));
4837c478bd9Sstevel@tonic-gate 		}
4847c478bd9Sstevel@tonic-gate 
4857c478bd9Sstevel@tonic-gate 		if (hash->dtah_hash[ndx] != NULL)
4867c478bd9Sstevel@tonic-gate 			hash->dtah_hash[ndx]->dtahe_prev = h;
4877c478bd9Sstevel@tonic-gate 
4887c478bd9Sstevel@tonic-gate 		h->dtahe_next = hash->dtah_hash[ndx];
4897c478bd9Sstevel@tonic-gate 		hash->dtah_hash[ndx] = h;
4907c478bd9Sstevel@tonic-gate 
4917c478bd9Sstevel@tonic-gate 		if (hash->dtah_all != NULL)
4927c478bd9Sstevel@tonic-gate 			hash->dtah_all->dtahe_prevall = h;
4937c478bd9Sstevel@tonic-gate 
4947c478bd9Sstevel@tonic-gate 		h->dtahe_nextall = hash->dtah_all;
4957c478bd9Sstevel@tonic-gate 		hash->dtah_all = h;
4967c478bd9Sstevel@tonic-gate bufnext:
4977c478bd9Sstevel@tonic-gate 		offs += agg->dtagd_size;
4987c478bd9Sstevel@tonic-gate 	}
4997c478bd9Sstevel@tonic-gate 
5007c478bd9Sstevel@tonic-gate 	return (0);
5017c478bd9Sstevel@tonic-gate }
5027c478bd9Sstevel@tonic-gate 
5037c478bd9Sstevel@tonic-gate int
5047c478bd9Sstevel@tonic-gate dtrace_aggregate_snap(dtrace_hdl_t *dtp)
5057c478bd9Sstevel@tonic-gate {
5067c478bd9Sstevel@tonic-gate 	int i, rval;
5077c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
5087c478bd9Sstevel@tonic-gate 	hrtime_t now = gethrtime();
5097c478bd9Sstevel@tonic-gate 	dtrace_optval_t interval = dtp->dt_options[DTRACEOPT_AGGRATE];
5107c478bd9Sstevel@tonic-gate 
5117c478bd9Sstevel@tonic-gate 	if (dtp->dt_lastagg != 0) {
5127c478bd9Sstevel@tonic-gate 		if (now - dtp->dt_lastagg < interval)
5137c478bd9Sstevel@tonic-gate 			return (0);
5147c478bd9Sstevel@tonic-gate 
5157c478bd9Sstevel@tonic-gate 		dtp->dt_lastagg += interval;
5167c478bd9Sstevel@tonic-gate 	} else {
5177c478bd9Sstevel@tonic-gate 		dtp->dt_lastagg = now;
5187c478bd9Sstevel@tonic-gate 	}
5197c478bd9Sstevel@tonic-gate 
5207c478bd9Sstevel@tonic-gate 	if (!dtp->dt_active)
5217c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, EINVAL));
5227c478bd9Sstevel@tonic-gate 
5237c478bd9Sstevel@tonic-gate 	if (agp->dtat_buf.dtbd_size == 0)
5247c478bd9Sstevel@tonic-gate 		return (0);
5257c478bd9Sstevel@tonic-gate 
5267c478bd9Sstevel@tonic-gate 	for (i = 0; i < agp->dtat_ncpus; i++) {
5277c478bd9Sstevel@tonic-gate 		if (rval = dt_aggregate_snap_cpu(dtp, agp->dtat_cpus[i]))
5287c478bd9Sstevel@tonic-gate 			return (rval);
5297c478bd9Sstevel@tonic-gate 	}
5307c478bd9Sstevel@tonic-gate 
5317c478bd9Sstevel@tonic-gate 	return (0);
5327c478bd9Sstevel@tonic-gate }
5337c478bd9Sstevel@tonic-gate 
5347c478bd9Sstevel@tonic-gate static int
5357c478bd9Sstevel@tonic-gate dt_aggregate_hashcmp(const void *lhs, const void *rhs)
5367c478bd9Sstevel@tonic-gate {
5377c478bd9Sstevel@tonic-gate 	dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
5387c478bd9Sstevel@tonic-gate 	dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
5397c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
5407c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
5417c478bd9Sstevel@tonic-gate 
5427c478bd9Sstevel@tonic-gate 	if (lagg->dtagd_nrecs < ragg->dtagd_nrecs)
5437c478bd9Sstevel@tonic-gate 		return (-1);
5447c478bd9Sstevel@tonic-gate 
5457c478bd9Sstevel@tonic-gate 	if (lagg->dtagd_nrecs > ragg->dtagd_nrecs)
5467c478bd9Sstevel@tonic-gate 		return (1);
5477c478bd9Sstevel@tonic-gate 
5487c478bd9Sstevel@tonic-gate 	return (0);
5497c478bd9Sstevel@tonic-gate }
5507c478bd9Sstevel@tonic-gate 
5517c478bd9Sstevel@tonic-gate static int
5527c478bd9Sstevel@tonic-gate dt_aggregate_varcmp(const void *lhs, const void *rhs)
5537c478bd9Sstevel@tonic-gate {
5547c478bd9Sstevel@tonic-gate 	dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
5557c478bd9Sstevel@tonic-gate 	dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
5567c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
5577c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
5587c478bd9Sstevel@tonic-gate 	caddr_t ldata = lh->dtahe_data.dtada_data;
5597c478bd9Sstevel@tonic-gate 	caddr_t rdata = rh->dtahe_data.dtada_data;
5607c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *lrec, *rrec;
5617c478bd9Sstevel@tonic-gate 	uint64_t lid, rid;
5627c478bd9Sstevel@tonic-gate 
5637c478bd9Sstevel@tonic-gate 	/*
5647c478bd9Sstevel@tonic-gate 	 * We know that we have a compiler-generated ID as the first record.
5657c478bd9Sstevel@tonic-gate 	 */
5667c478bd9Sstevel@tonic-gate 	lrec = lagg->dtagd_rec;
5677c478bd9Sstevel@tonic-gate 	rrec = ragg->dtagd_rec;
5687c478bd9Sstevel@tonic-gate 
5697c478bd9Sstevel@tonic-gate 	lid = *((uint64_t *)(uintptr_t)(ldata + lrec->dtrd_offset));
5707c478bd9Sstevel@tonic-gate 	rid = *((uint64_t *)(uintptr_t)(rdata + rrec->dtrd_offset));
5717c478bd9Sstevel@tonic-gate 
5727c478bd9Sstevel@tonic-gate 	if (lid < rid)
5737c478bd9Sstevel@tonic-gate 		return (-1);
5747c478bd9Sstevel@tonic-gate 
5757c478bd9Sstevel@tonic-gate 	if (lid > rid)
5767c478bd9Sstevel@tonic-gate 		return (1);
5777c478bd9Sstevel@tonic-gate 
5787c478bd9Sstevel@tonic-gate 	return (0);
5797c478bd9Sstevel@tonic-gate }
5807c478bd9Sstevel@tonic-gate 
5817c478bd9Sstevel@tonic-gate static int
5827c478bd9Sstevel@tonic-gate dt_aggregate_keycmp(const void *lhs, const void *rhs)
5837c478bd9Sstevel@tonic-gate {
5847c478bd9Sstevel@tonic-gate 	dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
5857c478bd9Sstevel@tonic-gate 	dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
5867c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
5877c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
5887c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *lrec, *rrec;
5897c478bd9Sstevel@tonic-gate 	char *ldata, *rdata;
590a67d40caSbmc 	int rval, i, j;
5917c478bd9Sstevel@tonic-gate 
5927c478bd9Sstevel@tonic-gate 	if ((rval = dt_aggregate_hashcmp(lhs, rhs)) != 0)
5937c478bd9Sstevel@tonic-gate 		return (rval);
5947c478bd9Sstevel@tonic-gate 
5957c478bd9Sstevel@tonic-gate 	for (i = 1; i < lagg->dtagd_nrecs - 1; i++) {
5967c478bd9Sstevel@tonic-gate 		uint64_t lval, rval;
5977c478bd9Sstevel@tonic-gate 
5987c478bd9Sstevel@tonic-gate 		lrec = &lagg->dtagd_rec[i];
5997c478bd9Sstevel@tonic-gate 		rrec = &ragg->dtagd_rec[i];
6007c478bd9Sstevel@tonic-gate 
6017c478bd9Sstevel@tonic-gate 		ldata = lh->dtahe_data.dtada_data + lrec->dtrd_offset;
6027c478bd9Sstevel@tonic-gate 		rdata = rh->dtahe_data.dtada_data + rrec->dtrd_offset;
6037c478bd9Sstevel@tonic-gate 
6047c478bd9Sstevel@tonic-gate 		if (lrec->dtrd_size < rrec->dtrd_size)
6057c478bd9Sstevel@tonic-gate 			return (-1);
6067c478bd9Sstevel@tonic-gate 
6077c478bd9Sstevel@tonic-gate 		if (lrec->dtrd_size > rrec->dtrd_size)
6087c478bd9Sstevel@tonic-gate 			return (1);
6097c478bd9Sstevel@tonic-gate 
6107c478bd9Sstevel@tonic-gate 		switch (lrec->dtrd_size) {
6117c478bd9Sstevel@tonic-gate 		case sizeof (uint64_t):
6127c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
6137c478bd9Sstevel@tonic-gate 			lval = *((uint64_t *)ldata);
6147c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
6157c478bd9Sstevel@tonic-gate 			rval = *((uint64_t *)rdata);
6167c478bd9Sstevel@tonic-gate 			break;
6177c478bd9Sstevel@tonic-gate 
6187c478bd9Sstevel@tonic-gate 		case sizeof (uint32_t):
6197c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
6207c478bd9Sstevel@tonic-gate 			lval = *((uint32_t *)ldata);
6217c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
6227c478bd9Sstevel@tonic-gate 			rval = *((uint32_t *)rdata);
6237c478bd9Sstevel@tonic-gate 			break;
6247c478bd9Sstevel@tonic-gate 
6257c478bd9Sstevel@tonic-gate 		case sizeof (uint16_t):
6267c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
6277c478bd9Sstevel@tonic-gate 			lval = *((uint16_t *)ldata);
6287c478bd9Sstevel@tonic-gate 			/* LINTED - alignment */
6297c478bd9Sstevel@tonic-gate 			rval = *((uint16_t *)rdata);
6307c478bd9Sstevel@tonic-gate 			break;
6317c478bd9Sstevel@tonic-gate 
6327c478bd9Sstevel@tonic-gate 		case sizeof (uint8_t):
6337c478bd9Sstevel@tonic-gate 			lval = *((uint8_t *)ldata);
6347c478bd9Sstevel@tonic-gate 			rval = *((uint8_t *)rdata);
6357c478bd9Sstevel@tonic-gate 			break;
6367c478bd9Sstevel@tonic-gate 
6377c478bd9Sstevel@tonic-gate 		default:
638a67d40caSbmc 			for (j = 0; j < lrec->dtrd_size; j++) {
639a67d40caSbmc 				lval = ((uint8_t *)ldata)[j];
640a67d40caSbmc 				rval = ((uint8_t *)rdata)[j];
6417c478bd9Sstevel@tonic-gate 
6427c478bd9Sstevel@tonic-gate 				if (lval < rval)
6437c478bd9Sstevel@tonic-gate 					return (-1);
6447c478bd9Sstevel@tonic-gate 
6457c478bd9Sstevel@tonic-gate 				if (lval > rval)
6467c478bd9Sstevel@tonic-gate 					return (1);
6477c478bd9Sstevel@tonic-gate 			}
6487c478bd9Sstevel@tonic-gate 
6497c478bd9Sstevel@tonic-gate 			continue;
6507c478bd9Sstevel@tonic-gate 		}
6517c478bd9Sstevel@tonic-gate 
6527c478bd9Sstevel@tonic-gate 		if (lval < rval)
6537c478bd9Sstevel@tonic-gate 			return (-1);
6547c478bd9Sstevel@tonic-gate 
6557c478bd9Sstevel@tonic-gate 		if (lval > rval)
6567c478bd9Sstevel@tonic-gate 			return (1);
6577c478bd9Sstevel@tonic-gate 	}
6587c478bd9Sstevel@tonic-gate 
6597c478bd9Sstevel@tonic-gate 	return (0);
6607c478bd9Sstevel@tonic-gate }
6617c478bd9Sstevel@tonic-gate 
6627c478bd9Sstevel@tonic-gate static int
6637c478bd9Sstevel@tonic-gate dt_aggregate_valcmp(const void *lhs, const void *rhs)
6647c478bd9Sstevel@tonic-gate {
6657c478bd9Sstevel@tonic-gate 	dt_ahashent_t *lh = *((dt_ahashent_t **)lhs);
6667c478bd9Sstevel@tonic-gate 	dt_ahashent_t *rh = *((dt_ahashent_t **)rhs);
6677c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *lagg = lh->dtahe_data.dtada_desc;
6687c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *ragg = rh->dtahe_data.dtada_desc;
6697c478bd9Sstevel@tonic-gate 	caddr_t ldata = lh->dtahe_data.dtada_data;
6707c478bd9Sstevel@tonic-gate 	caddr_t rdata = rh->dtahe_data.dtada_data;
6717c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *lrec, *rrec;
6727c478bd9Sstevel@tonic-gate 	uint64_t *laddr, *raddr;
6737c478bd9Sstevel@tonic-gate 	int rval, i;
6747c478bd9Sstevel@tonic-gate 
6757c478bd9Sstevel@tonic-gate 	if ((rval = dt_aggregate_hashcmp(lhs, rhs)) != 0)
6767c478bd9Sstevel@tonic-gate 		return (rval);
6777c478bd9Sstevel@tonic-gate 
6787c478bd9Sstevel@tonic-gate 	if (lagg->dtagd_nrecs < ragg->dtagd_nrecs)
6797c478bd9Sstevel@tonic-gate 		return (-1);
6807c478bd9Sstevel@tonic-gate 
6817c478bd9Sstevel@tonic-gate 	if (lagg->dtagd_nrecs > ragg->dtagd_nrecs)
6827c478bd9Sstevel@tonic-gate 		return (1);
6837c478bd9Sstevel@tonic-gate 
6847c478bd9Sstevel@tonic-gate 	for (i = 0; i < lagg->dtagd_nrecs; i++) {
6857c478bd9Sstevel@tonic-gate 		lrec = &lagg->dtagd_rec[i];
6867c478bd9Sstevel@tonic-gate 		rrec = &ragg->dtagd_rec[i];
6877c478bd9Sstevel@tonic-gate 
6887c478bd9Sstevel@tonic-gate 		if (lrec->dtrd_offset < rrec->dtrd_offset)
6897c478bd9Sstevel@tonic-gate 			return (-1);
6907c478bd9Sstevel@tonic-gate 
6917c478bd9Sstevel@tonic-gate 		if (lrec->dtrd_offset > rrec->dtrd_offset)
6927c478bd9Sstevel@tonic-gate 			return (1);
6937c478bd9Sstevel@tonic-gate 
6947c478bd9Sstevel@tonic-gate 		if (lrec->dtrd_action < rrec->dtrd_action)
6957c478bd9Sstevel@tonic-gate 			return (-1);
6967c478bd9Sstevel@tonic-gate 
6977c478bd9Sstevel@tonic-gate 		if (lrec->dtrd_action > rrec->dtrd_action)
6987c478bd9Sstevel@tonic-gate 			return (1);
6997c478bd9Sstevel@tonic-gate 	}
7007c478bd9Sstevel@tonic-gate 
7017c478bd9Sstevel@tonic-gate 	laddr = (uint64_t *)(uintptr_t)(ldata + lrec->dtrd_offset);
7027c478bd9Sstevel@tonic-gate 	raddr = (uint64_t *)(uintptr_t)(rdata + rrec->dtrd_offset);
7037c478bd9Sstevel@tonic-gate 
7047c478bd9Sstevel@tonic-gate 	switch (lrec->dtrd_action) {
7057c478bd9Sstevel@tonic-gate 	case DTRACEAGG_AVG:
7067c478bd9Sstevel@tonic-gate 		rval = dt_aggregate_averagecmp(laddr, raddr);
7077c478bd9Sstevel@tonic-gate 		break;
7087c478bd9Sstevel@tonic-gate 
7097c478bd9Sstevel@tonic-gate 	case DTRACEAGG_QUANTIZE:
7107c478bd9Sstevel@tonic-gate 		rval = dt_aggregate_quantizedcmp(laddr, raddr);
7117c478bd9Sstevel@tonic-gate 		break;
7127c478bd9Sstevel@tonic-gate 
7137c478bd9Sstevel@tonic-gate 	case DTRACEAGG_LQUANTIZE:
7147c478bd9Sstevel@tonic-gate 		rval = dt_aggregate_lquantizedcmp(laddr, raddr);
7157c478bd9Sstevel@tonic-gate 		break;
7167c478bd9Sstevel@tonic-gate 
7177c478bd9Sstevel@tonic-gate 	case DTRACEAGG_COUNT:
7187c478bd9Sstevel@tonic-gate 	case DTRACEAGG_SUM:
7197c478bd9Sstevel@tonic-gate 	case DTRACEAGG_MIN:
7207c478bd9Sstevel@tonic-gate 	case DTRACEAGG_MAX:
7217c478bd9Sstevel@tonic-gate 		rval = dt_aggregate_countcmp(laddr, raddr);
7227c478bd9Sstevel@tonic-gate 		break;
7237c478bd9Sstevel@tonic-gate 
7247c478bd9Sstevel@tonic-gate 	default:
7257c478bd9Sstevel@tonic-gate 		assert(0);
7267c478bd9Sstevel@tonic-gate 	}
7277c478bd9Sstevel@tonic-gate 
7287c478bd9Sstevel@tonic-gate 	if (rval != 0)
7297c478bd9Sstevel@tonic-gate 		return (rval);
7307c478bd9Sstevel@tonic-gate 
7317c478bd9Sstevel@tonic-gate 	/*
7327c478bd9Sstevel@tonic-gate 	 * If we're here, the values for the two aggregation elements are
7337c478bd9Sstevel@tonic-gate 	 * equal.  We already know that the key layout is the same for the two
7347c478bd9Sstevel@tonic-gate 	 * elements; we must now compare the keys themselves as a tie-breaker.
7357c478bd9Sstevel@tonic-gate 	 */
7367c478bd9Sstevel@tonic-gate 	return (dt_aggregate_keycmp(lhs, rhs));
7377c478bd9Sstevel@tonic-gate }
7387c478bd9Sstevel@tonic-gate 
7397c478bd9Sstevel@tonic-gate static int
7407c478bd9Sstevel@tonic-gate dt_aggregate_keyvarcmp(const void *lhs, const void *rhs)
7417c478bd9Sstevel@tonic-gate {
7427c478bd9Sstevel@tonic-gate 	int rval;
7437c478bd9Sstevel@tonic-gate 
7447c478bd9Sstevel@tonic-gate 	if ((rval = dt_aggregate_keycmp(lhs, rhs)) != 0)
7457c478bd9Sstevel@tonic-gate 		return (rval);
7467c478bd9Sstevel@tonic-gate 
7477c478bd9Sstevel@tonic-gate 	return (dt_aggregate_varcmp(lhs, rhs));
7487c478bd9Sstevel@tonic-gate }
7497c478bd9Sstevel@tonic-gate 
7507c478bd9Sstevel@tonic-gate static int
7517c478bd9Sstevel@tonic-gate dt_aggregate_varkeycmp(const void *lhs, const void *rhs)
7527c478bd9Sstevel@tonic-gate {
7537c478bd9Sstevel@tonic-gate 	int rval;
7547c478bd9Sstevel@tonic-gate 
7557c478bd9Sstevel@tonic-gate 	if ((rval = dt_aggregate_varcmp(lhs, rhs)) != 0)
7567c478bd9Sstevel@tonic-gate 		return (rval);
7577c478bd9Sstevel@tonic-gate 
7587c478bd9Sstevel@tonic-gate 	return (dt_aggregate_keycmp(lhs, rhs));
7597c478bd9Sstevel@tonic-gate }
7607c478bd9Sstevel@tonic-gate 
7617c478bd9Sstevel@tonic-gate static int
7627c478bd9Sstevel@tonic-gate dt_aggregate_valvarcmp(const void *lhs, const void *rhs)
7637c478bd9Sstevel@tonic-gate {
7647c478bd9Sstevel@tonic-gate 	int rval;
7657c478bd9Sstevel@tonic-gate 
7667c478bd9Sstevel@tonic-gate 	if ((rval = dt_aggregate_valcmp(lhs, rhs)) != 0)
7677c478bd9Sstevel@tonic-gate 		return (rval);
7687c478bd9Sstevel@tonic-gate 
7697c478bd9Sstevel@tonic-gate 	return (dt_aggregate_varcmp(lhs, rhs));
7707c478bd9Sstevel@tonic-gate }
7717c478bd9Sstevel@tonic-gate 
7727c478bd9Sstevel@tonic-gate static int
7737c478bd9Sstevel@tonic-gate dt_aggregate_varvalcmp(const void *lhs, const void *rhs)
7747c478bd9Sstevel@tonic-gate {
7757c478bd9Sstevel@tonic-gate 	int rval;
7767c478bd9Sstevel@tonic-gate 
7777c478bd9Sstevel@tonic-gate 	if ((rval = dt_aggregate_varcmp(lhs, rhs)) != 0)
7787c478bd9Sstevel@tonic-gate 		return (rval);
7797c478bd9Sstevel@tonic-gate 
7807c478bd9Sstevel@tonic-gate 	return (dt_aggregate_valcmp(lhs, rhs));
7817c478bd9Sstevel@tonic-gate }
7827c478bd9Sstevel@tonic-gate 
7837c478bd9Sstevel@tonic-gate static int
7847c478bd9Sstevel@tonic-gate dt_aggregate_keyvarrevcmp(const void *lhs, const void *rhs)
7857c478bd9Sstevel@tonic-gate {
7867c478bd9Sstevel@tonic-gate 	return (dt_aggregate_keyvarcmp(rhs, lhs));
7877c478bd9Sstevel@tonic-gate }
7887c478bd9Sstevel@tonic-gate 
7897c478bd9Sstevel@tonic-gate static int
7907c478bd9Sstevel@tonic-gate dt_aggregate_varkeyrevcmp(const void *lhs, const void *rhs)
7917c478bd9Sstevel@tonic-gate {
7927c478bd9Sstevel@tonic-gate 	return (dt_aggregate_varkeycmp(rhs, lhs));
7937c478bd9Sstevel@tonic-gate }
7947c478bd9Sstevel@tonic-gate 
7957c478bd9Sstevel@tonic-gate static int
7967c478bd9Sstevel@tonic-gate dt_aggregate_valvarrevcmp(const void *lhs, const void *rhs)
7977c478bd9Sstevel@tonic-gate {
7987c478bd9Sstevel@tonic-gate 	return (dt_aggregate_valvarcmp(rhs, lhs));
7997c478bd9Sstevel@tonic-gate }
8007c478bd9Sstevel@tonic-gate 
8017c478bd9Sstevel@tonic-gate static int
8027c478bd9Sstevel@tonic-gate dt_aggregate_varvalrevcmp(const void *lhs, const void *rhs)
8037c478bd9Sstevel@tonic-gate {
8047c478bd9Sstevel@tonic-gate 	return (dt_aggregate_varvalcmp(rhs, lhs));
8057c478bd9Sstevel@tonic-gate }
8067c478bd9Sstevel@tonic-gate 
8077c478bd9Sstevel@tonic-gate int
8087c478bd9Sstevel@tonic-gate dt_aggregate_go(dtrace_hdl_t *dtp)
8097c478bd9Sstevel@tonic-gate {
8107c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
8117c478bd9Sstevel@tonic-gate 	dtrace_optval_t size, cpu;
8127c478bd9Sstevel@tonic-gate 	dtrace_bufdesc_t *buf = &agp->dtat_buf;
8137c478bd9Sstevel@tonic-gate 	int rval, i;
8147c478bd9Sstevel@tonic-gate 
8157c478bd9Sstevel@tonic-gate 	assert(agp->dtat_maxcpu == 0);
8167c478bd9Sstevel@tonic-gate 	assert(agp->dtat_ncpu == 0);
8177c478bd9Sstevel@tonic-gate 	assert(agp->dtat_cpus == NULL);
8187c478bd9Sstevel@tonic-gate 
8197c478bd9Sstevel@tonic-gate 	agp->dtat_maxcpu = dt_sysconf(dtp, _SC_CPUID_MAX) + 1;
8207c478bd9Sstevel@tonic-gate 	agp->dtat_ncpu = dt_sysconf(dtp, _SC_NPROCESSORS_MAX);
8217c478bd9Sstevel@tonic-gate 	agp->dtat_cpus = malloc(agp->dtat_ncpu * sizeof (processorid_t));
8227c478bd9Sstevel@tonic-gate 
8237c478bd9Sstevel@tonic-gate 	if (agp->dtat_cpus == NULL)
8247c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, EDT_NOMEM));
8257c478bd9Sstevel@tonic-gate 
8267c478bd9Sstevel@tonic-gate 	/*
8277c478bd9Sstevel@tonic-gate 	 * Use the aggregation buffer size as reloaded from the kernel.
8287c478bd9Sstevel@tonic-gate 	 */
8297c478bd9Sstevel@tonic-gate 	size = dtp->dt_options[DTRACEOPT_AGGSIZE];
8307c478bd9Sstevel@tonic-gate 
8317c478bd9Sstevel@tonic-gate 	rval = dtrace_getopt(dtp, "aggsize", &size);
8327c478bd9Sstevel@tonic-gate 	assert(rval == 0);
8337c478bd9Sstevel@tonic-gate 
8347c478bd9Sstevel@tonic-gate 	if (size == 0 || size == DTRACEOPT_UNSET)
8357c478bd9Sstevel@tonic-gate 		return (0);
8367c478bd9Sstevel@tonic-gate 
8377c478bd9Sstevel@tonic-gate 	buf = &agp->dtat_buf;
8387c478bd9Sstevel@tonic-gate 	buf->dtbd_size = size;
8397c478bd9Sstevel@tonic-gate 
8407c478bd9Sstevel@tonic-gate 	if ((buf->dtbd_data = malloc(buf->dtbd_size)) == NULL)
8417c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, EDT_NOMEM));
8427c478bd9Sstevel@tonic-gate 
8437c478bd9Sstevel@tonic-gate 	/*
8447c478bd9Sstevel@tonic-gate 	 * Now query for the CPUs enabled.
8457c478bd9Sstevel@tonic-gate 	 */
8467c478bd9Sstevel@tonic-gate 	rval = dtrace_getopt(dtp, "cpu", &cpu);
8477c478bd9Sstevel@tonic-gate 	assert(rval == 0 && cpu != DTRACEOPT_UNSET);
8487c478bd9Sstevel@tonic-gate 
8497c478bd9Sstevel@tonic-gate 	if (cpu != DTRACE_CPUALL) {
8507c478bd9Sstevel@tonic-gate 		assert(cpu < agp->dtat_ncpu);
8517c478bd9Sstevel@tonic-gate 		agp->dtat_cpus[agp->dtat_ncpus++] = (processorid_t)cpu;
8527c478bd9Sstevel@tonic-gate 
8537c478bd9Sstevel@tonic-gate 		return (0);
8547c478bd9Sstevel@tonic-gate 	}
8557c478bd9Sstevel@tonic-gate 
8567c478bd9Sstevel@tonic-gate 	agp->dtat_ncpus = 0;
8577c478bd9Sstevel@tonic-gate 	for (i = 0; i < agp->dtat_maxcpu; i++) {
8587c478bd9Sstevel@tonic-gate 		if (dt_status(dtp, i) == -1)
8597c478bd9Sstevel@tonic-gate 			continue;
8607c478bd9Sstevel@tonic-gate 
8617c478bd9Sstevel@tonic-gate 		agp->dtat_cpus[agp->dtat_ncpus++] = i;
8627c478bd9Sstevel@tonic-gate 	}
8637c478bd9Sstevel@tonic-gate 
8647c478bd9Sstevel@tonic-gate 	return (0);
8657c478bd9Sstevel@tonic-gate }
8667c478bd9Sstevel@tonic-gate 
8677c478bd9Sstevel@tonic-gate static int
8687c478bd9Sstevel@tonic-gate dt_aggwalk_rval(dtrace_hdl_t *dtp, dt_ahashent_t *h, int rval)
8697c478bd9Sstevel@tonic-gate {
8707c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
8717c478bd9Sstevel@tonic-gate 	dtrace_aggdata_t *data;
8727c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *aggdesc;
8737c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *rec;
8747c478bd9Sstevel@tonic-gate 	int i;
8757c478bd9Sstevel@tonic-gate 
8767c478bd9Sstevel@tonic-gate 	switch (rval) {
8777c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_NEXT:
8787c478bd9Sstevel@tonic-gate 		break;
8797c478bd9Sstevel@tonic-gate 
8807c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_CLEAR: {
8817c478bd9Sstevel@tonic-gate 		uint32_t size, offs = 0;
8827c478bd9Sstevel@tonic-gate 
8837c478bd9Sstevel@tonic-gate 		aggdesc = h->dtahe_data.dtada_desc;
8847c478bd9Sstevel@tonic-gate 		rec = &aggdesc->dtagd_rec[aggdesc->dtagd_nrecs - 1];
8857c478bd9Sstevel@tonic-gate 		size = rec->dtrd_size;
8867c478bd9Sstevel@tonic-gate 		data = &h->dtahe_data;
8877c478bd9Sstevel@tonic-gate 
8887c478bd9Sstevel@tonic-gate 		if (rec->dtrd_action == DTRACEAGG_LQUANTIZE) {
8897c478bd9Sstevel@tonic-gate 			offs = sizeof (uint64_t);
8907c478bd9Sstevel@tonic-gate 			size -= sizeof (uint64_t);
8917c478bd9Sstevel@tonic-gate 		}
8927c478bd9Sstevel@tonic-gate 
8937c478bd9Sstevel@tonic-gate 		bzero(&data->dtada_data[rec->dtrd_offset] + offs, size);
8947c478bd9Sstevel@tonic-gate 
8957c478bd9Sstevel@tonic-gate 		if (data->dtada_percpu == NULL)
8967c478bd9Sstevel@tonic-gate 			break;
8977c478bd9Sstevel@tonic-gate 
8987c478bd9Sstevel@tonic-gate 		for (i = 0; i < dtp->dt_aggregate.dtat_maxcpu; i++)
8997c478bd9Sstevel@tonic-gate 			bzero(data->dtada_percpu[i] + offs, size);
9007c478bd9Sstevel@tonic-gate 		break;
9017c478bd9Sstevel@tonic-gate 	}
9027c478bd9Sstevel@tonic-gate 
9037c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_ERROR:
9047c478bd9Sstevel@tonic-gate 		/*
9057c478bd9Sstevel@tonic-gate 		 * We assume that errno is already set in this case.
9067c478bd9Sstevel@tonic-gate 		 */
9077c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, errno));
9087c478bd9Sstevel@tonic-gate 
9097c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_ABORT:
9107c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, EDT_DIRABORT));
9117c478bd9Sstevel@tonic-gate 
9127c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_DENORMALIZE:
9137c478bd9Sstevel@tonic-gate 		h->dtahe_data.dtada_normal = 1;
9147c478bd9Sstevel@tonic-gate 		return (0);
9157c478bd9Sstevel@tonic-gate 
9167c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_NORMALIZE:
9177c478bd9Sstevel@tonic-gate 		if (h->dtahe_data.dtada_normal == 0) {
9187c478bd9Sstevel@tonic-gate 			h->dtahe_data.dtada_normal = 1;
9197c478bd9Sstevel@tonic-gate 			return (dt_set_errno(dtp, EDT_BADRVAL));
9207c478bd9Sstevel@tonic-gate 		}
9217c478bd9Sstevel@tonic-gate 
9227c478bd9Sstevel@tonic-gate 		return (0);
9237c478bd9Sstevel@tonic-gate 
9247c478bd9Sstevel@tonic-gate 	case DTRACE_AGGWALK_REMOVE: {
9257c478bd9Sstevel@tonic-gate 		dtrace_aggdata_t *aggdata = &h->dtahe_data;
9267c478bd9Sstevel@tonic-gate 		int i, max_cpus = agp->dtat_maxcpu;
9277c478bd9Sstevel@tonic-gate 
9287c478bd9Sstevel@tonic-gate 		/*
9297c478bd9Sstevel@tonic-gate 		 * First, remove this hash entry from its hash chain.
9307c478bd9Sstevel@tonic-gate 		 */
9317c478bd9Sstevel@tonic-gate 		if (h->dtahe_prev != NULL) {
9327c478bd9Sstevel@tonic-gate 			h->dtahe_prev->dtahe_next = h->dtahe_next;
9337c478bd9Sstevel@tonic-gate 		} else {
9347c478bd9Sstevel@tonic-gate 			dt_ahash_t *hash = &agp->dtat_hash;
9357c478bd9Sstevel@tonic-gate 			size_t ndx = h->dtahe_hashval % hash->dtah_size;
9367c478bd9Sstevel@tonic-gate 
9377c478bd9Sstevel@tonic-gate 			assert(hash->dtah_hash[ndx] == h);
9387c478bd9Sstevel@tonic-gate 			hash->dtah_hash[ndx] = h->dtahe_next;
9397c478bd9Sstevel@tonic-gate 		}
9407c478bd9Sstevel@tonic-gate 
9417c478bd9Sstevel@tonic-gate 		if (h->dtahe_next != NULL)
9427c478bd9Sstevel@tonic-gate 			h->dtahe_next->dtahe_prev = h->dtahe_prev;
9437c478bd9Sstevel@tonic-gate 
9447c478bd9Sstevel@tonic-gate 		/*
9457c478bd9Sstevel@tonic-gate 		 * Now remove it from the list of all hash entries.
9467c478bd9Sstevel@tonic-gate 		 */
9477c478bd9Sstevel@tonic-gate 		if (h->dtahe_prevall != NULL) {
9487c478bd9Sstevel@tonic-gate 			h->dtahe_prevall->dtahe_nextall = h->dtahe_nextall;
9497c478bd9Sstevel@tonic-gate 		} else {
9507c478bd9Sstevel@tonic-gate 			dt_ahash_t *hash = &agp->dtat_hash;
9517c478bd9Sstevel@tonic-gate 
9527c478bd9Sstevel@tonic-gate 			assert(hash->dtah_all == h);
9537c478bd9Sstevel@tonic-gate 			hash->dtah_all = h->dtahe_nextall;
9547c478bd9Sstevel@tonic-gate 		}
9557c478bd9Sstevel@tonic-gate 
9567c478bd9Sstevel@tonic-gate 		if (h->dtahe_nextall != NULL)
9577c478bd9Sstevel@tonic-gate 			h->dtahe_nextall->dtahe_prevall = h->dtahe_prevall;
9587c478bd9Sstevel@tonic-gate 
9597c478bd9Sstevel@tonic-gate 		/*
9607c478bd9Sstevel@tonic-gate 		 * We're unlinked.  We can safely destroy the data.
9617c478bd9Sstevel@tonic-gate 		 */
9627c478bd9Sstevel@tonic-gate 		if (aggdata->dtada_percpu != NULL) {
9637c478bd9Sstevel@tonic-gate 			for (i = 0; i < max_cpus; i++)
9647c478bd9Sstevel@tonic-gate 				free(aggdata->dtada_percpu[i]);
9657c478bd9Sstevel@tonic-gate 			free(aggdata->dtada_percpu);
9667c478bd9Sstevel@tonic-gate 		}
9677c478bd9Sstevel@tonic-gate 
9687c478bd9Sstevel@tonic-gate 		free(aggdata->dtada_data);
9697c478bd9Sstevel@tonic-gate 		free(h);
9707c478bd9Sstevel@tonic-gate 
9717c478bd9Sstevel@tonic-gate 		return (0);
9727c478bd9Sstevel@tonic-gate 	}
9737c478bd9Sstevel@tonic-gate 
9747c478bd9Sstevel@tonic-gate 	default:
9757c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, EDT_BADRVAL));
9767c478bd9Sstevel@tonic-gate 	}
9777c478bd9Sstevel@tonic-gate 
9787c478bd9Sstevel@tonic-gate 	return (0);
9797c478bd9Sstevel@tonic-gate }
9807c478bd9Sstevel@tonic-gate 
9817c478bd9Sstevel@tonic-gate int
9827c478bd9Sstevel@tonic-gate dtrace_aggregate_walk(dtrace_hdl_t *dtp, dtrace_aggregate_f *func, void *arg)
9837c478bd9Sstevel@tonic-gate {
9847c478bd9Sstevel@tonic-gate 	dt_ahashent_t *h, *next;
9857c478bd9Sstevel@tonic-gate 	dt_ahash_t *hash = &dtp->dt_aggregate.dtat_hash;
9867c478bd9Sstevel@tonic-gate 
9877c478bd9Sstevel@tonic-gate 	for (h = hash->dtah_all; h != NULL; h = next) {
9887c478bd9Sstevel@tonic-gate 		/*
9897c478bd9Sstevel@tonic-gate 		 * dt_aggwalk_rval() can potentially remove the current hash
9907c478bd9Sstevel@tonic-gate 		 * entry; we need to load the next hash entry before calling
9917c478bd9Sstevel@tonic-gate 		 * into it.
9927c478bd9Sstevel@tonic-gate 		 */
9937c478bd9Sstevel@tonic-gate 		next = h->dtahe_nextall;
9947c478bd9Sstevel@tonic-gate 
9957c478bd9Sstevel@tonic-gate 		if (dt_aggwalk_rval(dtp, h, func(&h->dtahe_data, arg)) == -1)
9967c478bd9Sstevel@tonic-gate 			return (-1);
9977c478bd9Sstevel@tonic-gate 	}
9987c478bd9Sstevel@tonic-gate 
9997c478bd9Sstevel@tonic-gate 	return (0);
10007c478bd9Sstevel@tonic-gate }
10017c478bd9Sstevel@tonic-gate 
10027c478bd9Sstevel@tonic-gate static int
10037c478bd9Sstevel@tonic-gate dt_aggregate_walk_sorted(dtrace_hdl_t *dtp,
10047c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg,
10057c478bd9Sstevel@tonic-gate     int (*sfunc)(const void *, const void *))
10067c478bd9Sstevel@tonic-gate {
10077c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
10087c478bd9Sstevel@tonic-gate 	dt_ahashent_t *h, **sorted;
10097c478bd9Sstevel@tonic-gate 	dt_ahash_t *hash = &agp->dtat_hash;
10107c478bd9Sstevel@tonic-gate 	size_t i, nentries = 0;
10117c478bd9Sstevel@tonic-gate 
10127c478bd9Sstevel@tonic-gate 	for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall)
10137c478bd9Sstevel@tonic-gate 		nentries++;
10147c478bd9Sstevel@tonic-gate 
10157c478bd9Sstevel@tonic-gate 	sorted = malloc(nentries * sizeof (dt_ahashent_t *));
10167c478bd9Sstevel@tonic-gate 
10177c478bd9Sstevel@tonic-gate 	if (sorted == NULL)
10187c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, EDT_NOMEM));
10197c478bd9Sstevel@tonic-gate 
10207c478bd9Sstevel@tonic-gate 	for (h = hash->dtah_all, i = 0; h != NULL; h = h->dtahe_nextall)
10217c478bd9Sstevel@tonic-gate 		sorted[i++] = h;
10227c478bd9Sstevel@tonic-gate 
10237c478bd9Sstevel@tonic-gate 	qsort(sorted, nentries, sizeof (dt_ahashent_t *), sfunc);
10247c478bd9Sstevel@tonic-gate 
10257c478bd9Sstevel@tonic-gate 	for (i = 0; i < nentries; i++) {
10267c478bd9Sstevel@tonic-gate 		h = sorted[i];
10277c478bd9Sstevel@tonic-gate 
10287c478bd9Sstevel@tonic-gate 		if (dt_aggwalk_rval(dtp, h, func(&h->dtahe_data, arg)) == -1)
10297c478bd9Sstevel@tonic-gate 			return (-1);
10307c478bd9Sstevel@tonic-gate 	}
10317c478bd9Sstevel@tonic-gate 
10327c478bd9Sstevel@tonic-gate 	free(sorted);
10337c478bd9Sstevel@tonic-gate 	return (0);
10347c478bd9Sstevel@tonic-gate }
10357c478bd9Sstevel@tonic-gate 
10367c478bd9Sstevel@tonic-gate int
10377c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_keysorted(dtrace_hdl_t *dtp,
10387c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
10397c478bd9Sstevel@tonic-gate {
10407c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
10417c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_varkeycmp));
10427c478bd9Sstevel@tonic-gate }
10437c478bd9Sstevel@tonic-gate 
10447c478bd9Sstevel@tonic-gate int
10457c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_valsorted(dtrace_hdl_t *dtp,
10467c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
10477c478bd9Sstevel@tonic-gate {
10487c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
10497c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_varvalcmp));
10507c478bd9Sstevel@tonic-gate }
10517c478bd9Sstevel@tonic-gate 
10527c478bd9Sstevel@tonic-gate int
10537c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_keyvarsorted(dtrace_hdl_t *dtp,
10547c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
10557c478bd9Sstevel@tonic-gate {
10567c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
10577c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_keyvarcmp));
10587c478bd9Sstevel@tonic-gate }
10597c478bd9Sstevel@tonic-gate 
10607c478bd9Sstevel@tonic-gate int
10617c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_valvarsorted(dtrace_hdl_t *dtp,
10627c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
10637c478bd9Sstevel@tonic-gate {
10647c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
10657c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_valvarcmp));
10667c478bd9Sstevel@tonic-gate }
10677c478bd9Sstevel@tonic-gate 
10687c478bd9Sstevel@tonic-gate int
10697c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_keyrevsorted(dtrace_hdl_t *dtp,
10707c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
10717c478bd9Sstevel@tonic-gate {
10727c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
10737c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_varkeyrevcmp));
10747c478bd9Sstevel@tonic-gate }
10757c478bd9Sstevel@tonic-gate 
10767c478bd9Sstevel@tonic-gate int
10777c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_valrevsorted(dtrace_hdl_t *dtp,
10787c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
10797c478bd9Sstevel@tonic-gate {
10807c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
10817c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_varvalrevcmp));
10827c478bd9Sstevel@tonic-gate }
10837c478bd9Sstevel@tonic-gate 
10847c478bd9Sstevel@tonic-gate int
10857c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_keyvarrevsorted(dtrace_hdl_t *dtp,
10867c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
10877c478bd9Sstevel@tonic-gate {
10887c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
10897c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_keyvarrevcmp));
10907c478bd9Sstevel@tonic-gate }
10917c478bd9Sstevel@tonic-gate 
10927c478bd9Sstevel@tonic-gate int
10937c478bd9Sstevel@tonic-gate dtrace_aggregate_walk_valvarrevsorted(dtrace_hdl_t *dtp,
10947c478bd9Sstevel@tonic-gate     dtrace_aggregate_f *func, void *arg)
10957c478bd9Sstevel@tonic-gate {
10967c478bd9Sstevel@tonic-gate 	return (dt_aggregate_walk_sorted(dtp, func,
10977c478bd9Sstevel@tonic-gate 	    arg, dt_aggregate_valvarrevcmp));
10987c478bd9Sstevel@tonic-gate }
10997c478bd9Sstevel@tonic-gate 
11007c478bd9Sstevel@tonic-gate int
11017c478bd9Sstevel@tonic-gate dtrace_aggregate_print(dtrace_hdl_t *dtp, FILE *fp,
11027c478bd9Sstevel@tonic-gate     dtrace_aggregate_walk_f *func)
11037c478bd9Sstevel@tonic-gate {
11047c478bd9Sstevel@tonic-gate 	dt_print_aggdata_t pd;
11057c478bd9Sstevel@tonic-gate 
11067c478bd9Sstevel@tonic-gate 	pd.dtpa_dtp = dtp;
11077c478bd9Sstevel@tonic-gate 	pd.dtpa_fp = fp;
11087c478bd9Sstevel@tonic-gate 	pd.dtpa_allunprint = 1;
11097c478bd9Sstevel@tonic-gate 
11107c478bd9Sstevel@tonic-gate 	if (func == NULL)
11117c478bd9Sstevel@tonic-gate 		func = dtrace_aggregate_walk_valsorted;
11127c478bd9Sstevel@tonic-gate 
11137c478bd9Sstevel@tonic-gate 	if ((*func)(dtp, dt_print_agg, &pd) == -1)
11147c478bd9Sstevel@tonic-gate 		return (dt_set_errno(dtp, dtp->dt_errno));
11157c478bd9Sstevel@tonic-gate 
11167c478bd9Sstevel@tonic-gate 	return (0);
11177c478bd9Sstevel@tonic-gate }
11187c478bd9Sstevel@tonic-gate 
11197c478bd9Sstevel@tonic-gate void
11207c478bd9Sstevel@tonic-gate dtrace_aggregate_clear(dtrace_hdl_t *dtp)
11217c478bd9Sstevel@tonic-gate {
11227c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
11237c478bd9Sstevel@tonic-gate 	dt_ahash_t *hash = &agp->dtat_hash;
11247c478bd9Sstevel@tonic-gate 	dt_ahashent_t *h;
11257c478bd9Sstevel@tonic-gate 	dtrace_aggdata_t *data;
11267c478bd9Sstevel@tonic-gate 	dtrace_aggdesc_t *aggdesc;
11277c478bd9Sstevel@tonic-gate 	dtrace_recdesc_t *rec;
11287c478bd9Sstevel@tonic-gate 	int i, max_cpus = agp->dtat_maxcpu;
11297c478bd9Sstevel@tonic-gate 
11307c478bd9Sstevel@tonic-gate 	for (h = hash->dtah_all; h != NULL; h = h->dtahe_nextall) {
11317c478bd9Sstevel@tonic-gate 		aggdesc = h->dtahe_data.dtada_desc;
11327c478bd9Sstevel@tonic-gate 		rec = &aggdesc->dtagd_rec[aggdesc->dtagd_nrecs - 1];
11337c478bd9Sstevel@tonic-gate 		data = &h->dtahe_data;
11347c478bd9Sstevel@tonic-gate 
11357c478bd9Sstevel@tonic-gate 		bzero(&data->dtada_data[rec->dtrd_offset], rec->dtrd_size);
11367c478bd9Sstevel@tonic-gate 
11377c478bd9Sstevel@tonic-gate 		if (data->dtada_percpu == NULL)
11387c478bd9Sstevel@tonic-gate 			continue;
11397c478bd9Sstevel@tonic-gate 
11407c478bd9Sstevel@tonic-gate 		for (i = 0; i < max_cpus; i++)
11417c478bd9Sstevel@tonic-gate 			bzero(data->dtada_percpu[i], rec->dtrd_size);
11427c478bd9Sstevel@tonic-gate 	}
11437c478bd9Sstevel@tonic-gate }
11447c478bd9Sstevel@tonic-gate 
11457c478bd9Sstevel@tonic-gate void
11467c478bd9Sstevel@tonic-gate dt_aggregate_destroy(dtrace_hdl_t *dtp)
11477c478bd9Sstevel@tonic-gate {
11487c478bd9Sstevel@tonic-gate 	dt_aggregate_t *agp = &dtp->dt_aggregate;
11497c478bd9Sstevel@tonic-gate 	dt_ahash_t *hash = &agp->dtat_hash;
11507c478bd9Sstevel@tonic-gate 	dt_ahashent_t *h, *next;
11517c478bd9Sstevel@tonic-gate 	dtrace_aggdata_t *aggdata;
11527c478bd9Sstevel@tonic-gate 	int i, max_cpus = agp->dtat_maxcpu;
11537c478bd9Sstevel@tonic-gate 
11547c478bd9Sstevel@tonic-gate 	if (hash->dtah_hash == NULL) {
11557c478bd9Sstevel@tonic-gate 		assert(hash->dtah_all == NULL);
11567c478bd9Sstevel@tonic-gate 	} else {
11577c478bd9Sstevel@tonic-gate 		free(hash->dtah_hash);
11587c478bd9Sstevel@tonic-gate 
11597c478bd9Sstevel@tonic-gate 		for (h = hash->dtah_all; h != NULL; h = next) {
11607c478bd9Sstevel@tonic-gate 			next = h->dtahe_nextall;
11617c478bd9Sstevel@tonic-gate 
11627c478bd9Sstevel@tonic-gate 			aggdata = &h->dtahe_data;
11637c478bd9Sstevel@tonic-gate 
11647c478bd9Sstevel@tonic-gate 			if (aggdata->dtada_percpu != NULL) {
11657c478bd9Sstevel@tonic-gate 				for (i = 0; i < max_cpus; i++)
11667c478bd9Sstevel@tonic-gate 					free(aggdata->dtada_percpu[i]);
11677c478bd9Sstevel@tonic-gate 				free(aggdata->dtada_percpu);
11687c478bd9Sstevel@tonic-gate 			}
11697c478bd9Sstevel@tonic-gate 
11707c478bd9Sstevel@tonic-gate 			free(aggdata->dtada_data);
11717c478bd9Sstevel@tonic-gate 			free(h);
11727c478bd9Sstevel@tonic-gate 		}
11737c478bd9Sstevel@tonic-gate 
11747c478bd9Sstevel@tonic-gate 		hash->dtah_hash = NULL;
11757c478bd9Sstevel@tonic-gate 		hash->dtah_all = NULL;
11767c478bd9Sstevel@tonic-gate 		hash->dtah_size = 0;
11777c478bd9Sstevel@tonic-gate 	}
11787c478bd9Sstevel@tonic-gate 
11797c478bd9Sstevel@tonic-gate 	free(agp->dtat_buf.dtbd_data);
11807c478bd9Sstevel@tonic-gate 	free(agp->dtat_cpus);
11817c478bd9Sstevel@tonic-gate }
1182