xref: /illumos-gate/usr/src/uts/common/dtrace/dtrace.c (revision 35b7f6cc898f7c73f5367459c83198f1bb897fff)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * DTrace - Dynamic Tracing for Solaris
31  *
32  * This is the implementation of the Solaris Dynamic Tracing framework
33  * (DTrace).  The user-visible interface to DTrace is described at length in
34  * the "Solaris Dynamic Tracing Guide".  The interfaces between the libdtrace
35  * library, the in-kernel DTrace framework, and the DTrace providers are
36  * described in the block comments in the <sys/dtrace.h> header file.  The
37  * internal architecture of DTrace is described in the block comments in the
38  * <sys/dtrace_impl.h> header file.  The comments contained within the DTrace
39  * implementation very much assume mastery of all of these sources; if one has
40  * an unanswered question about the implementation, one should consult them
41  * first.
42  *
43  * The functions here are ordered roughly as follows:
44  *
45  *   - Probe context functions
46  *   - Probe hashing functions
47  *   - Non-probe context utility functions
48  *   - Matching functions
49  *   - Provider-to-Framework API functions
50  *   - Probe management functions
51  *   - DIF object functions
52  *   - Format functions
53  *   - Predicate functions
54  *   - ECB functions
55  *   - Buffer functions
56  *   - Enabling functions
57  *   - DOF functions
58  *   - Anonymous enabling functions
59  *   - Consumer state functions
60  *   - Helper functions
61  *   - Hook functions
62  *   - Driver cookbook functions
63  *
64  * Each group of functions begins with a block comment labelled the "DTrace
65  * [Group] Functions", allowing one to find each block by searching forward
66  * on capital-f functions.
67  */
68 #include <sys/errno.h>
69 #include <sys/stat.h>
70 #include <sys/modctl.h>
71 #include <sys/conf.h>
72 #include <sys/systm.h>
73 #include <sys/ddi.h>
74 #include <sys/sunddi.h>
75 #include <sys/cpuvar.h>
76 #include <sys/kmem.h>
77 #include <sys/strsubr.h>
78 #include <sys/sysmacros.h>
79 #include <sys/dtrace_impl.h>
80 #include <sys/atomic.h>
81 #include <sys/cmn_err.h>
82 #include <sys/mutex_impl.h>
83 #include <sys/rwlock_impl.h>
84 #include <sys/ctf_api.h>
85 #include <sys/panic.h>
86 #include <sys/priv_impl.h>
87 #include <sys/policy.h>
88 #include <sys/cred_impl.h>
89 #include <sys/procfs_isa.h>
90 #include <sys/taskq.h>
91 #include <sys/mkdev.h>
92 #include <sys/kdi.h>
93 #include <sys/zone.h>
94 
95 /*
96  * DTrace Tunable Variables
97  *
98  * The following variables may be tuned by adding a line to /etc/system that
99  * includes both the name of the DTrace module ("dtrace") and the name of the
100  * variable.  For example:
101  *
102  *   set dtrace:dtrace_destructive_disallow = 1
103  *
104  * In general, the only variables that one should be tuning this way are those
105  * that affect system-wide DTrace behavior, and for which the default behavior
106  * is undesirable.  Most of these variables are tunable on a per-consumer
107  * basis using DTrace options, and need not be tuned on a system-wide basis.
108  * When tuning these variables, avoid pathological values; while some attempt
109  * is made to verify the integrity of these variables, they are not considered
110  * part of the supported interface to DTrace, and they are therefore not
111  * checked comprehensively.  Further, these variables should not be tuned
112  * dynamically via "mdb -kw" or other means; they should only be tuned via
113  * /etc/system.
114  */
115 int		dtrace_destructive_disallow = 0;
116 dtrace_optval_t	dtrace_nonroot_maxsize = (16 * 1024 * 1024);
117 size_t		dtrace_difo_maxsize = (256 * 1024);
118 dtrace_optval_t	dtrace_dof_maxsize = (256 * 1024);
119 size_t		dtrace_global_maxsize = (16 * 1024);
120 size_t		dtrace_actions_max = (16 * 1024);
121 size_t		dtrace_retain_max = 1024;
122 dtrace_optval_t	dtrace_helper_actions_max = 32;
123 dtrace_optval_t	dtrace_helper_providers_max = 32;
124 dtrace_optval_t	dtrace_dstate_defsize = (1 * 1024 * 1024);
125 size_t		dtrace_strsize_default = 256;
126 dtrace_optval_t	dtrace_cleanrate_default = 9900990;		/* 101 hz */
127 dtrace_optval_t	dtrace_cleanrate_min = 200000;			/* 5000 hz */
128 dtrace_optval_t	dtrace_cleanrate_max = (uint64_t)60 * NANOSEC;	/* 1/minute */
129 dtrace_optval_t	dtrace_aggrate_default = NANOSEC;		/* 1 hz */
130 dtrace_optval_t	dtrace_statusrate_default = NANOSEC;		/* 1 hz */
131 dtrace_optval_t dtrace_statusrate_max = (hrtime_t)10 * NANOSEC;	 /* 6/minute */
132 dtrace_optval_t	dtrace_switchrate_default = NANOSEC;		/* 1 hz */
133 dtrace_optval_t	dtrace_nspec_default = 1;
134 dtrace_optval_t	dtrace_specsize_default = 32 * 1024;
135 dtrace_optval_t dtrace_stackframes_default = 20;
136 dtrace_optval_t dtrace_ustackframes_default = 20;
137 dtrace_optval_t dtrace_jstackframes_default = 50;
138 dtrace_optval_t dtrace_jstackstrsize_default = 512;
139 int		dtrace_msgdsize_max = 128;
140 hrtime_t	dtrace_chill_max = 500 * (NANOSEC / MILLISEC);	/* 500 ms */
141 hrtime_t	dtrace_chill_interval = NANOSEC;		/* 1000 ms */
142 int		dtrace_devdepth_max = 32;
143 int		dtrace_err_verbose;
144 hrtime_t	dtrace_deadman_interval = NANOSEC;
145 hrtime_t	dtrace_deadman_timeout = (hrtime_t)10 * NANOSEC;
146 hrtime_t	dtrace_deadman_user = (hrtime_t)30 * NANOSEC;
147 
148 /*
149  * DTrace External Variables
150  *
151  * As dtrace(7D) is a kernel module, any DTrace variables are obviously
152  * available to DTrace consumers via the backtick (`) syntax.  One of these,
153  * dtrace_zero, is made deliberately so:  it is provided as a source of
154  * well-known, zero-filled memory.  While this variable is not documented,
155  * it is used by some translators as an implementation detail.
156  */
157 const char	dtrace_zero[256] = { 0 };	/* zero-filled memory */
158 
159 /*
160  * DTrace Internal Variables
161  */
162 static dev_info_t	*dtrace_devi;		/* device info */
163 static vmem_t		*dtrace_arena;		/* probe ID arena */
164 static vmem_t		*dtrace_minor;		/* minor number arena */
165 static taskq_t		*dtrace_taskq;		/* task queue */
166 static dtrace_probe_t	**dtrace_probes;	/* array of all probes */
167 static int		dtrace_nprobes;		/* number of probes */
168 static dtrace_provider_t *dtrace_provider;	/* provider list */
169 static dtrace_meta_t	*dtrace_meta_pid;	/* user-land meta provider */
170 static int		dtrace_opens;		/* number of opens */
171 static int		dtrace_helpers;		/* number of helpers */
172 static void		*dtrace_softstate;	/* softstate pointer */
173 static dtrace_hash_t	*dtrace_bymod;		/* probes hashed by module */
174 static dtrace_hash_t	*dtrace_byfunc;		/* probes hashed by function */
175 static dtrace_hash_t	*dtrace_byname;		/* probes hashed by name */
176 static dtrace_toxrange_t *dtrace_toxrange;	/* toxic range array */
177 static int		dtrace_toxranges;	/* number of toxic ranges */
178 static int		dtrace_toxranges_max;	/* size of toxic range array */
179 static dtrace_anon_t	dtrace_anon;		/* anonymous enabling */
180 static kmem_cache_t	*dtrace_state_cache;	/* cache for dynamic state */
181 static uint64_t		dtrace_vtime_references; /* number of vtimestamp refs */
182 static kthread_t	*dtrace_panicked;	/* panicking thread */
183 static dtrace_ecb_t	*dtrace_ecb_create_cache; /* cached created ECB */
184 static dtrace_genid_t	dtrace_probegen;	/* current probe generation */
185 static dtrace_helpers_t *dtrace_deferred_pid;	/* deferred helper list */
186 static dtrace_enabling_t *dtrace_retained;	/* list of retained enablings */
187 static dtrace_state_t	*dtrace_state;		/* temporary variable */
188 static int		dtrace_err;		/* temporary variable */
189 
190 /*
191  * DTrace Locking
192  * DTrace is protected by three (relatively coarse-grained) locks:
193  *
194  * (1) dtrace_lock is required to manipulate essentially any DTrace state,
195  *     including enabling state, probes, ECBs, consumer state, helper state,
196  *     etc.  Importantly, dtrace_lock is _not_ required when in probe context;
197  *     probe context is lock-free -- synchronization is handled via the
198  *     dtrace_sync() cross call mechanism.
199  *
200  * (2) dtrace_provider_lock is required when manipulating provider state, or
201  *     when provider state must be held constant.
202  *
203  * (3) dtrace_meta_lock is required when manipulating meta provider state, or
204  *     when meta provider state must be held constant.
205  *
206  * The lock ordering between these three locks is dtrace_meta_lock before
207  * dtrace_provider_lock before dtrace_lock.  (In particular, there are
208  * several places where dtrace_provider_lock is held by the framework as it
209  * calls into the providers -- which then call back into the framework,
210  * grabbing dtrace_lock.)
211  *
212  * There are two other locks in the mix:  mod_lock and cpu_lock.  With respect
213  * to dtrace_provider_lock and dtrace_lock, cpu_lock continues its historical
214  * role as a coarse-grained lock; it is acquired before both of these locks.
215  * With respect to dtrace_meta_lock, its behavior is stranger:  cpu_lock must
216  * be acquired _between_ dtrace_meta_lock and any other DTrace locks.
217  * mod_lock is similar with respect to dtrace_provider_lock in that it must be
218  * acquired _between_ dtrace_provider_lock and dtrace_lock.
219  */
220 static kmutex_t		dtrace_lock;		/* probe state lock */
221 static kmutex_t		dtrace_provider_lock;	/* provider state lock */
222 static kmutex_t		dtrace_meta_lock;	/* meta-provider state lock */
223 
224 /*
225  * DTrace Provider Variables
226  *
227  * These are the variables relating to DTrace as a provider (that is, the
228  * provider of the BEGIN, END, and ERROR probes).
229  */
230 static dtrace_pattr_t	dtrace_provider_attr = {
231 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
232 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
233 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
234 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
235 { DTRACE_STABILITY_STABLE, DTRACE_STABILITY_STABLE, DTRACE_CLASS_COMMON },
236 };
237 
238 static void
239 dtrace_nullop(void)
240 {}
241 
242 static dtrace_pops_t	dtrace_provider_ops = {
243 	(void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop,
244 	(void (*)(void *, struct modctl *))dtrace_nullop,
245 	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
246 	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
247 	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
248 	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop,
249 	NULL,
250 	NULL,
251 	NULL,
252 	(void (*)(void *, dtrace_id_t, void *))dtrace_nullop
253 };
254 
255 static dtrace_id_t	dtrace_probeid_begin;	/* special BEGIN probe */
256 static dtrace_id_t	dtrace_probeid_end;	/* special END probe */
257 dtrace_id_t		dtrace_probeid_error;	/* special ERROR probe */
258 
259 /*
260  * DTrace Helper Tracing Variables
261  */
262 uint32_t dtrace_helptrace_next = 0;
263 uint32_t dtrace_helptrace_nlocals;
264 char	*dtrace_helptrace_buffer;
265 int	dtrace_helptrace_bufsize = 512 * 1024;
266 
267 #ifdef DEBUG
268 int	dtrace_helptrace_enabled = 1;
269 #else
270 int	dtrace_helptrace_enabled = 0;
271 #endif
272 
273 /*
274  * DTrace Error Hashing
275  *
276  * On DEBUG kernels, DTrace will track the errors that has seen in a hash
277  * table.  This is very useful for checking coverage of tests that are
278  * expected to induce DIF or DOF processing errors, and may be useful for
279  * debugging problems in the DIF code generator or in DOF generation .  The
280  * error hash may be examined with the ::dtrace_errhash MDB dcmd.
281  */
282 #ifdef DEBUG
283 static dtrace_errhash_t	dtrace_errhash[DTRACE_ERRHASHSZ];
284 static const char *dtrace_errlast;
285 static kthread_t *dtrace_errthread;
286 static kmutex_t dtrace_errlock;
287 #endif
288 
289 /*
290  * DTrace Macros and Constants
291  *
292  * These are various macros that are useful in various spots in the
293  * implementation, along with a few random constants that have no meaning
294  * outside of the implementation.  There is no real structure to this cpp
295  * mishmash -- but is there ever?
296  */
297 #define	DTRACE_HASHSTR(hash, probe)	\
298 	dtrace_hash_str(*((char **)((uintptr_t)(probe) + (hash)->dth_stroffs)))
299 
300 #define	DTRACE_HASHNEXT(hash, probe)	\
301 	(dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_nextoffs)
302 
303 #define	DTRACE_HASHPREV(hash, probe)	\
304 	(dtrace_probe_t **)((uintptr_t)(probe) + (hash)->dth_prevoffs)
305 
306 #define	DTRACE_HASHEQ(hash, lhs, rhs)	\
307 	(strcmp(*((char **)((uintptr_t)(lhs) + (hash)->dth_stroffs)), \
308 	    *((char **)((uintptr_t)(rhs) + (hash)->dth_stroffs))) == 0)
309 
310 #define	DTRACE_AGGHASHSIZE_SLEW		17
311 
312 /*
313  * The key for a thread-local variable consists of the lower 61 bits of the
314  * t_did, plus the 3 bits of the highest active interrupt above LOCK_LEVEL.
315  * We add DIF_VARIABLE_MAX to t_did to assure that the thread key is never
316  * equal to a variable identifier.  This is necessary (but not sufficient) to
317  * assure that global associative arrays never collide with thread-local
318  * variables.  To guarantee that they cannot collide, we must also define the
319  * order for keying dynamic variables.  That order is:
320  *
321  *   [ key0 ] ... [ keyn ] [ variable-key ] [ tls-key ]
322  *
323  * Because the variable-key and the tls-key are in orthogonal spaces, there is
324  * no way for a global variable key signature to match a thread-local key
325  * signature.
326  */
327 #define	DTRACE_TLS_THRKEY(where) { \
328 	uint_t intr = 0; \
329 	uint_t actv = CPU->cpu_intr_actv >> (LOCK_LEVEL + 1); \
330 	for (; actv; actv >>= 1) \
331 		intr++; \
332 	ASSERT(intr < (1 << 3)); \
333 	(where) = ((curthread->t_did + DIF_VARIABLE_MAX) & \
334 	    (((uint64_t)1 << 61) - 1)) | ((uint64_t)intr << 61); \
335 }
336 
337 #define	DTRACE_STORE(type, tomax, offset, what) \
338 	*((type *)((uintptr_t)(tomax) + (uintptr_t)offset)) = (type)(what);
339 
340 #ifndef __i386
341 #define	DTRACE_ALIGNCHECK(addr, size, flags)				\
342 	if (addr & (size - 1)) {					\
343 		*flags |= CPU_DTRACE_BADALIGN;				\
344 		cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;	\
345 		return (0);						\
346 	}
347 #else
348 #define	DTRACE_ALIGNCHECK(addr, size, flags)
349 #endif
350 
351 #define	DTRACE_LOADFUNC(bits)						\
352 /*CSTYLED*/								\
353 uint##bits##_t								\
354 dtrace_load##bits(uintptr_t addr)					\
355 {									\
356 	size_t size = bits / NBBY;					\
357 	/*CSTYLED*/							\
358 	uint##bits##_t rval;						\
359 	int i;								\
360 	volatile uint16_t *flags = (volatile uint16_t *)		\
361 	    &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;			\
362 									\
363 	DTRACE_ALIGNCHECK(addr, size, flags);				\
364 									\
365 	for (i = 0; i < dtrace_toxranges; i++) {			\
366 		if (addr >= dtrace_toxrange[i].dtt_limit)		\
367 			continue;					\
368 									\
369 		if (addr + size <= dtrace_toxrange[i].dtt_base)		\
370 			continue;					\
371 									\
372 		/*							\
373 		 * This address falls within a toxic region; return 0.	\
374 		 */							\
375 		*flags |= CPU_DTRACE_BADADDR;				\
376 		cpu_core[CPU->cpu_id].cpuc_dtrace_illval = addr;	\
377 		return (0);						\
378 	}								\
379 									\
380 	*flags |= CPU_DTRACE_NOFAULT;					\
381 	/*CSTYLED*/							\
382 	rval = *((volatile uint##bits##_t *)addr);			\
383 	*flags &= ~CPU_DTRACE_NOFAULT;					\
384 									\
385 	return (rval);							\
386 }
387 
388 #ifdef _LP64
389 #define	dtrace_loadptr	dtrace_load64
390 #else
391 #define	dtrace_loadptr	dtrace_load32
392 #endif
393 
394 #define	DTRACE_MATCH_NEXT	0
395 #define	DTRACE_MATCH_DONE	1
396 #define	DTRACE_ANCHORED(probe)	((probe)->dtpr_func[0] != '\0')
397 #define	DTRACE_STATE_ALIGN	64
398 
399 #define	DTRACE_FLAGS2FLT(flags)						\
400 	(((flags) & CPU_DTRACE_BADADDR) ? DTRACEFLT_BADADDR :		\
401 	((flags) & CPU_DTRACE_ILLOP) ? DTRACEFLT_ILLOP :		\
402 	((flags) & CPU_DTRACE_DIVZERO) ? DTRACEFLT_DIVZERO :		\
403 	((flags) & CPU_DTRACE_KPRIV) ? DTRACEFLT_KPRIV :		\
404 	((flags) & CPU_DTRACE_UPRIV) ? DTRACEFLT_UPRIV :		\
405 	((flags) & CPU_DTRACE_TUPOFLOW) ?  DTRACEFLT_TUPOFLOW :		\
406 	((flags) & CPU_DTRACE_BADALIGN) ?  DTRACEFLT_BADALIGN :		\
407 	((flags) & CPU_DTRACE_NOSCRATCH) ?  DTRACEFLT_NOSCRATCH :	\
408 	DTRACEFLT_UNKNOWN)
409 
410 static dtrace_probe_t *dtrace_probe_lookup_id(dtrace_id_t id);
411 static void dtrace_enabling_provide(dtrace_provider_t *);
412 static int dtrace_enabling_match(dtrace_enabling_t *, int *);
413 static void dtrace_enabling_matchall(void);
414 static dtrace_state_t *dtrace_anon_grab(void);
415 static uint64_t dtrace_helper(int, dtrace_mstate_t *,
416     dtrace_state_t *, uint64_t, uint64_t);
417 static dtrace_helpers_t *dtrace_helpers_create(proc_t *);
418 static void dtrace_buffer_drop(dtrace_buffer_t *);
419 static intptr_t dtrace_buffer_reserve(dtrace_buffer_t *, size_t, size_t,
420     dtrace_state_t *, dtrace_mstate_t *);
421 static int dtrace_state_option(dtrace_state_t *, dtrace_optid_t,
422     dtrace_optval_t);
423 static int dtrace_ecb_create_enable(dtrace_probe_t *, void *);
424 
425 /*
426  * DTrace Probe Context Functions
427  *
428  * These functions are called from probe context.  Because probe context is
429  * any context in which C may be called, arbitrarily locks may be held,
430  * interrupts may be disabled, we may be in arbitrary dispatched state, etc.
431  * As a result, functions called from probe context may only call other DTrace
432  * support functions -- they may not interact at all with the system at large.
433  * (Note that the ASSERT macro is made probe-context safe by redefining it in
434  * terms of dtrace_assfail(), a probe-context safe function.) If arbitrary
435  * loads are to be performed from probe context, they _must_ be in terms of
436  * the safe dtrace_load*() variants.
437  *
438  * Some functions in this block are not actually called from probe context;
439  * for these functions, there will be a comment above the function reading
440  * "Note:  not called from probe context."
441  */
442 void
443 dtrace_panic(const char *format, ...)
444 {
445 	va_list alist;
446 
447 	va_start(alist, format);
448 	dtrace_vpanic(format, alist);
449 	va_end(alist);
450 }
451 
452 int
453 dtrace_assfail(const char *a, const char *f, int l)
454 {
455 	dtrace_panic("assertion failed: %s, file: %s, line: %d", a, f, l);
456 
457 	/*
458 	 * We just need something here that even the most clever compiler
459 	 * cannot optimize away.
460 	 */
461 	return (a[(uintptr_t)f]);
462 }
463 
464 /*
465  * Atomically increment a specified error counter from probe context.
466  */
467 static void
468 dtrace_error(uint32_t *counter)
469 {
470 	/*
471 	 * Most counters stored to in probe context are per-CPU counters.
472 	 * However, there are some error conditions that are sufficiently
473 	 * arcane that they don't merit per-CPU storage.  If these counters
474 	 * are incremented concurrently on different CPUs, scalability will be
475 	 * adversely affected -- but we don't expect them to be white-hot in a
476 	 * correctly constructed enabling...
477 	 */
478 	uint32_t oval, nval;
479 
480 	do {
481 		oval = *counter;
482 
483 		if ((nval = oval + 1) == 0) {
484 			/*
485 			 * If the counter would wrap, set it to 1 -- assuring
486 			 * that the counter is never zero when we have seen
487 			 * errors.  (The counter must be 32-bits because we
488 			 * aren't guaranteed a 64-bit compare&swap operation.)
489 			 * To save this code both the infamy of being fingered
490 			 * by a priggish news story and the indignity of being
491 			 * the target of a neo-puritan witch trial, we're
492 			 * carefully avoiding any colorful description of the
493 			 * likelihood of this condition -- but suffice it to
494 			 * say that it is only slightly more likely than the
495 			 * overflow of predicate cache IDs, as discussed in
496 			 * dtrace_predicate_create().
497 			 */
498 			nval = 1;
499 		}
500 	} while (dtrace_cas32(counter, oval, nval) != oval);
501 }
502 
503 /*
504  * Use the DTRACE_LOADFUNC macro to define functions for each of loading a
505  * uint8_t, a uint16_t, a uint32_t and a uint64_t.
506  */
507 DTRACE_LOADFUNC(8)
508 DTRACE_LOADFUNC(16)
509 DTRACE_LOADFUNC(32)
510 DTRACE_LOADFUNC(64)
511 
512 static int
513 dtrace_inscratch(uintptr_t dest, size_t size, dtrace_mstate_t *mstate)
514 {
515 	if (dest < mstate->dtms_scratch_base)
516 		return (0);
517 
518 	if (dest + size < dest)
519 		return (0);
520 
521 	if (dest + size > mstate->dtms_scratch_ptr)
522 		return (0);
523 
524 	return (1);
525 }
526 
527 static int
528 dtrace_canstore_statvar(uint64_t addr, size_t sz,
529     dtrace_statvar_t **svars, int nsvars)
530 {
531 	int i;
532 
533 	for (i = 0; i < nsvars; i++) {
534 		dtrace_statvar_t *svar = svars[i];
535 
536 		if (svar == NULL || svar->dtsv_size == 0)
537 			continue;
538 
539 		if (addr - svar->dtsv_data < svar->dtsv_size &&
540 		    addr + sz <= svar->dtsv_data + svar->dtsv_size)
541 			return (1);
542 	}
543 
544 	return (0);
545 }
546 
547 /*
548  * Check to see if the address is within a memory region to which a store may
549  * be issued.  This includes the DTrace scratch areas, and any DTrace variable
550  * region.  The caller of dtrace_canstore() is responsible for performing any
551  * alignment checks that are needed before stores are actually executed.
552  */
553 static int
554 dtrace_canstore(uint64_t addr, size_t sz, dtrace_mstate_t *mstate,
555     dtrace_vstate_t *vstate)
556 {
557 	uintptr_t a;
558 	size_t s;
559 
560 	/*
561 	 * First, check to see if the address is in scratch space...
562 	 */
563 	a = mstate->dtms_scratch_base;
564 	s = mstate->dtms_scratch_size;
565 
566 	if (addr - a < s && addr + sz <= a + s)
567 		return (1);
568 
569 	/*
570 	 * Now check to see if it's a dynamic variable.  This check will pick
571 	 * up both thread-local variables and any global dynamically-allocated
572 	 * variables.
573 	 */
574 	a = (uintptr_t)vstate->dtvs_dynvars.dtds_base;
575 	s = vstate->dtvs_dynvars.dtds_size;
576 	if (addr - a < s && addr + sz <= a + s)
577 		return (1);
578 
579 	/*
580 	 * Finally, check the static local and global variables.  These checks
581 	 * take the longest, so we perform them last.
582 	 */
583 	if (dtrace_canstore_statvar(addr, sz,
584 	    vstate->dtvs_locals, vstate->dtvs_nlocals))
585 		return (1);
586 
587 	if (dtrace_canstore_statvar(addr, sz,
588 	    vstate->dtvs_globals, vstate->dtvs_nglobals))
589 		return (1);
590 
591 	return (0);
592 }
593 
594 /*
595  * Compare two strings using safe loads.
596  */
597 static int
598 dtrace_strncmp(char *s1, char *s2, size_t limit)
599 {
600 	uint8_t c1, c2;
601 	volatile uint16_t *flags;
602 
603 	if (s1 == s2 || limit == 0)
604 		return (0);
605 
606 	flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
607 
608 	do {
609 		if (s1 == NULL) {
610 			c1 = '\0';
611 		} else {
612 			c1 = dtrace_load8((uintptr_t)s1++);
613 		}
614 
615 		if (s2 == NULL) {
616 			c2 = '\0';
617 		} else {
618 			c2 = dtrace_load8((uintptr_t)s2++);
619 		}
620 
621 		if (c1 != c2)
622 			return (c1 - c2);
623 	} while (--limit && c1 != '\0' && !(*flags & CPU_DTRACE_FAULT));
624 
625 	return (0);
626 }
627 
628 /*
629  * Compute strlen(s) for a string using safe memory accesses.  The additional
630  * len parameter is used to specify a maximum length to ensure completion.
631  */
632 static size_t
633 dtrace_strlen(const char *s, size_t lim)
634 {
635 	uint_t len;
636 
637 	for (len = 0; len != lim; len++) {
638 		if (dtrace_load8((uintptr_t)s++) == '\0')
639 			break;
640 	}
641 
642 	return (len);
643 }
644 
645 /*
646  * Check if an address falls within a toxic region.
647  */
648 static int
649 dtrace_istoxic(uintptr_t kaddr, size_t size)
650 {
651 	uintptr_t taddr, tsize;
652 	int i;
653 
654 	for (i = 0; i < dtrace_toxranges; i++) {
655 		taddr = dtrace_toxrange[i].dtt_base;
656 		tsize = dtrace_toxrange[i].dtt_limit - taddr;
657 
658 		if (kaddr - taddr < tsize) {
659 			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
660 			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = kaddr;
661 			return (1);
662 		}
663 
664 		if (taddr - kaddr < size) {
665 			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
666 			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = taddr;
667 			return (1);
668 		}
669 	}
670 
671 	return (0);
672 }
673 
674 /*
675  * Copy src to dst using safe memory accesses.  The src is assumed to be unsafe
676  * memory specified by the DIF program.  The dst is assumed to be safe memory
677  * that we can store to directly because it is managed by DTrace.  As with
678  * standard bcopy, overlapping copies are handled properly.
679  */
680 static void
681 dtrace_bcopy(const void *src, void *dst, size_t len)
682 {
683 	if (len != 0) {
684 		uint8_t *s1 = dst;
685 		const uint8_t *s2 = src;
686 
687 		if (s1 <= s2) {
688 			do {
689 				*s1++ = dtrace_load8((uintptr_t)s2++);
690 			} while (--len != 0);
691 		} else {
692 			s2 += len;
693 			s1 += len;
694 
695 			do {
696 				*--s1 = dtrace_load8((uintptr_t)--s2);
697 			} while (--len != 0);
698 		}
699 	}
700 }
701 
702 /*
703  * Copy src to dst using safe memory accesses, up to either the specified
704  * length, or the point that a nul byte is encountered.  The src is assumed to
705  * be unsafe memory specified by the DIF program.  The dst is assumed to be
706  * safe memory that we can store to directly because it is managed by DTrace.
707  * Unlike dtrace_bcopy(), overlapping regions are not handled.
708  */
709 static void
710 dtrace_strcpy(const void *src, void *dst, size_t len)
711 {
712 	if (len != 0) {
713 		uint8_t *s1 = dst, c;
714 		const uint8_t *s2 = src;
715 
716 		do {
717 			*s1++ = c = dtrace_load8((uintptr_t)s2++);
718 		} while (--len != 0 && c != '\0');
719 	}
720 }
721 
722 /*
723  * Copy src to dst, deriving the size and type from the specified (BYREF)
724  * variable type.  The src is assumed to be unsafe memory specified by the DIF
725  * program.  The dst is assumed to be DTrace variable memory that is of the
726  * specified type; we assume that we can store to directly.
727  */
728 static void
729 dtrace_vcopy(void *src, void *dst, dtrace_diftype_t *type)
730 {
731 	ASSERT(type->dtdt_flags & DIF_TF_BYREF);
732 
733 	if (type->dtdt_kind == DIF_TYPE_STRING) {
734 		dtrace_strcpy(src, dst, type->dtdt_size);
735 	} else {
736 		dtrace_bcopy(src, dst, type->dtdt_size);
737 	}
738 }
739 
740 /*
741  * Compare s1 to s2 using safe memory accesses.  The s1 data is assumed to be
742  * unsafe memory specified by the DIF program.  The s2 data is assumed to be
743  * safe memory that we can access directly because it is managed by DTrace.
744  */
745 static int
746 dtrace_bcmp(const void *s1, const void *s2, size_t len)
747 {
748 	volatile uint16_t *flags;
749 
750 	flags = (volatile uint16_t *)&cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
751 
752 	if (s1 == s2)
753 		return (0);
754 
755 	if (s1 == NULL || s2 == NULL)
756 		return (1);
757 
758 	if (s1 != s2 && len != 0) {
759 		const uint8_t *ps1 = s1;
760 		const uint8_t *ps2 = s2;
761 
762 		do {
763 			if (dtrace_load8((uintptr_t)ps1++) != *ps2++)
764 				return (1);
765 		} while (--len != 0 && !(*flags & CPU_DTRACE_FAULT));
766 	}
767 	return (0);
768 }
769 
770 /*
771  * Zero the specified region using a simple byte-by-byte loop.  Note that this
772  * is for safe DTrace-managed memory only.
773  */
774 static void
775 dtrace_bzero(void *dst, size_t len)
776 {
777 	uchar_t *cp;
778 
779 	for (cp = dst; len != 0; len--)
780 		*cp++ = 0;
781 }
782 
783 /*
784  * This privilege checks should be used by actions and subroutines to
785  * verify the credentials of the process that enabled the invoking ECB.
786  */
787 static int
788 dtrace_priv_proc_common(dtrace_state_t *state)
789 {
790 	uid_t uid = state->dts_cred.dcr_uid;
791 	gid_t gid = state->dts_cred.dcr_gid;
792 	cred_t *cr;
793 	proc_t *proc;
794 
795 	if ((cr = CRED()) != NULL &&
796 	    uid == cr->cr_uid &&
797 	    uid == cr->cr_ruid &&
798 	    uid == cr->cr_suid &&
799 	    gid == cr->cr_gid &&
800 	    gid == cr->cr_rgid &&
801 	    gid == cr->cr_sgid &&
802 	    (proc = ttoproc(curthread)) != NULL &&
803 	    !(proc->p_flag & SNOCD))
804 		return (1);
805 
806 	cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
807 
808 	return (0);
809 }
810 
811 static int
812 dtrace_priv_proc_destructive(dtrace_state_t *state)
813 {
814 	if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_DESTRUCTIVE)
815 		return (1);
816 
817 	return (dtrace_priv_proc_common(state));
818 }
819 
820 static int
821 dtrace_priv_proc_control(dtrace_state_t *state)
822 {
823 	if (state->dts_cred.dcr_action & DTRACE_CRA_PROC_CONTROL)
824 		return (1);
825 
826 	return (dtrace_priv_proc_common(state));
827 }
828 
829 static int
830 dtrace_priv_proc(dtrace_state_t *state)
831 {
832 	if (state->dts_cred.dcr_action & DTRACE_CRA_PROC)
833 		return (1);
834 
835 	cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_UPRIV;
836 
837 	return (0);
838 }
839 
840 static int
841 dtrace_priv_kernel(dtrace_state_t *state)
842 {
843 	if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL)
844 		return (1);
845 
846 	cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
847 
848 	return (0);
849 }
850 
851 static int
852 dtrace_priv_kernel_destructive(dtrace_state_t *state)
853 {
854 	if (state->dts_cred.dcr_action & DTRACE_CRA_KERNEL_DESTRUCTIVE)
855 		return (1);
856 
857 	cpu_core[CPU->cpu_id].cpuc_dtrace_flags |= CPU_DTRACE_KPRIV;
858 
859 	return (0);
860 }
861 
862 /*
863  * Note:  not called from probe context.  This function is called
864  * asynchronously (and at a regular interval) from outside of probe context to
865  * clean the dirty dynamic variable lists on all CPUs.  Dynamic variable
866  * cleaning is explained in detail in <sys/dtrace_impl.h>.
867  */
868 void
869 dtrace_dynvar_clean(dtrace_dstate_t *dstate)
870 {
871 	dtrace_dynvar_t *dirty;
872 	dtrace_dstate_percpu_t *dcpu;
873 	int i, work = 0;
874 
875 	for (i = 0; i < NCPU; i++) {
876 		dcpu = &dstate->dtds_percpu[i];
877 
878 		ASSERT(dcpu->dtdsc_rinsing == NULL);
879 
880 		/*
881 		 * If the dirty list is NULL, there is no dirty work to do.
882 		 */
883 		if (dcpu->dtdsc_dirty == NULL)
884 			continue;
885 
886 		/*
887 		 * If the clean list is non-NULL, then we're not going to do
888 		 * any work for this CPU -- it means that there has not been
889 		 * a dtrace_dynvar() allocation on this CPU (or from this CPU)
890 		 * since the last time we cleaned house.
891 		 */
892 		if (dcpu->dtdsc_clean != NULL)
893 			continue;
894 
895 		work = 1;
896 
897 		/*
898 		 * Atomically move the dirty list aside.
899 		 */
900 		do {
901 			dirty = dcpu->dtdsc_dirty;
902 
903 			/*
904 			 * Before we zap the dirty list, set the rinsing list.
905 			 * (This allows for a potential assertion in
906 			 * dtrace_dynvar():  if a free dynamic variable appears
907 			 * on a hash chain, either the dirty list or the
908 			 * rinsing list for some CPU must be non-NULL.)
909 			 */
910 			dcpu->dtdsc_rinsing = dirty;
911 			dtrace_membar_producer();
912 		} while (dtrace_casptr(&dcpu->dtdsc_dirty,
913 		    dirty, NULL) != dirty);
914 	}
915 
916 	if (!work) {
917 		/*
918 		 * We have no work to do; we can simply return.
919 		 */
920 		return;
921 	}
922 
923 	dtrace_sync();
924 
925 	for (i = 0; i < NCPU; i++) {
926 		dcpu = &dstate->dtds_percpu[i];
927 
928 		if (dcpu->dtdsc_rinsing == NULL)
929 			continue;
930 
931 		/*
932 		 * We are now guaranteed that no hash chain contains a pointer
933 		 * into this dirty list; we can make it clean.
934 		 */
935 		ASSERT(dcpu->dtdsc_clean == NULL);
936 		dcpu->dtdsc_clean = dcpu->dtdsc_rinsing;
937 		dcpu->dtdsc_rinsing = NULL;
938 	}
939 
940 	/*
941 	 * Before we actually set the state to be DTRACE_DSTATE_CLEAN, make
942 	 * sure that all CPUs have seen all of the dtdsc_clean pointers.
943 	 * This prevents a race whereby a CPU incorrectly decides that
944 	 * the state should be something other than DTRACE_DSTATE_CLEAN
945 	 * after dtrace_dynvar_clean() has completed.
946 	 */
947 	dtrace_sync();
948 
949 	dstate->dtds_state = DTRACE_DSTATE_CLEAN;
950 }
951 
952 /*
953  * Depending on the value of the op parameter, this function looks-up,
954  * allocates or deallocates an arbitrarily-keyed dynamic variable.  If an
955  * allocation is requested, this function will return a pointer to a
956  * dtrace_dynvar_t corresponding to the allocated variable -- or NULL if no
957  * variable can be allocated.  If NULL is returned, the appropriate counter
958  * will be incremented.
959  */
960 dtrace_dynvar_t *
961 dtrace_dynvar(dtrace_dstate_t *dstate, uint_t nkeys,
962     dtrace_key_t *key, size_t dsize, dtrace_dynvar_op_t op)
963 {
964 	uint64_t hashval = 1;
965 	dtrace_dynhash_t *hash = dstate->dtds_hash;
966 	dtrace_dynvar_t *free, *new_free, *next, *dvar, *start, *prev = NULL;
967 	processorid_t me = CPU->cpu_id, cpu = me;
968 	dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[me];
969 	size_t bucket, ksize;
970 	size_t chunksize = dstate->dtds_chunksize;
971 	uintptr_t kdata, lock, nstate;
972 	uint_t i;
973 
974 	ASSERT(nkeys != 0);
975 
976 	/*
977 	 * Hash the key.  As with aggregations, we use Jenkins' "One-at-a-time"
978 	 * algorithm.  For the by-value portions, we perform the algorithm in
979 	 * 16-bit chunks (as opposed to 8-bit chunks).  This speeds things up a
980 	 * bit, and seems to have only a minute effect on distribution.  For
981 	 * the by-reference data, we perform "One-at-a-time" iterating (safely)
982 	 * over each referenced byte.  It's painful to do this, but it's much
983 	 * better than pathological hash distribution.  The efficacy of the
984 	 * hashing algorithm (and a comparison with other algorithms) may be
985 	 * found by running the ::dtrace_dynstat MDB dcmd.
986 	 */
987 	for (i = 0; i < nkeys; i++) {
988 		if (key[i].dttk_size == 0) {
989 			uint64_t val = key[i].dttk_value;
990 
991 			hashval += (val >> 48) & 0xffff;
992 			hashval += (hashval << 10);
993 			hashval ^= (hashval >> 6);
994 
995 			hashval += (val >> 32) & 0xffff;
996 			hashval += (hashval << 10);
997 			hashval ^= (hashval >> 6);
998 
999 			hashval += (val >> 16) & 0xffff;
1000 			hashval += (hashval << 10);
1001 			hashval ^= (hashval >> 6);
1002 
1003 			hashval += val & 0xffff;
1004 			hashval += (hashval << 10);
1005 			hashval ^= (hashval >> 6);
1006 		} else {
1007 			/*
1008 			 * This is incredibly painful, but it beats the hell
1009 			 * out of the alternative.
1010 			 */
1011 			uint64_t j, size = key[i].dttk_size;
1012 			uintptr_t base = (uintptr_t)key[i].dttk_value;
1013 
1014 			for (j = 0; j < size; j++) {
1015 				hashval += dtrace_load8(base + j);
1016 				hashval += (hashval << 10);
1017 				hashval ^= (hashval >> 6);
1018 			}
1019 		}
1020 	}
1021 
1022 	hashval += (hashval << 3);
1023 	hashval ^= (hashval >> 11);
1024 	hashval += (hashval << 15);
1025 
1026 	/*
1027 	 * There is a remote chance (ideally, 1 in 2^32) that our hashval
1028 	 * comes out to be 0.  We rely on a zero hashval denoting a free
1029 	 * element; if this actually happens, we set the hashval to 1.
1030 	 */
1031 	if (hashval == 0)
1032 		hashval = 1;
1033 
1034 	/*
1035 	 * Yes, it's painful to do a divide here.  If the cycle count becomes
1036 	 * important here, tricks can be pulled to reduce it.  (However, it's
1037 	 * critical that hash collisions be kept to an absolute minimum;
1038 	 * they're much more painful than a divide.)  It's better to have a
1039 	 * solution that generates few collisions and still keeps things
1040 	 * relatively simple.
1041 	 */
1042 	bucket = hashval % dstate->dtds_hashsize;
1043 
1044 	if (op == DTRACE_DYNVAR_DEALLOC) {
1045 		volatile uintptr_t *lockp = &hash[bucket].dtdh_lock;
1046 
1047 		for (;;) {
1048 			while ((lock = *lockp) & 1)
1049 				continue;
1050 
1051 			if (dtrace_casptr((void *)lockp,
1052 			    (void *)lock, (void *)(lock + 1)) == (void *)lock)
1053 				break;
1054 		}
1055 
1056 		dtrace_membar_producer();
1057 	}
1058 
1059 top:
1060 	prev = NULL;
1061 	lock = hash[bucket].dtdh_lock;
1062 
1063 	dtrace_membar_consumer();
1064 
1065 	start = hash[bucket].dtdh_chain;
1066 	ASSERT(start == NULL || start->dtdv_hashval != 0 ||
1067 	    op != DTRACE_DYNVAR_DEALLOC);
1068 
1069 	for (dvar = start; dvar != NULL; dvar = dvar->dtdv_next) {
1070 		dtrace_tuple_t *dtuple = &dvar->dtdv_tuple;
1071 		dtrace_key_t *dkey = &dtuple->dtt_key[0];
1072 
1073 		if (dvar->dtdv_hashval != hashval) {
1074 			if (dvar->dtdv_hashval == 0) {
1075 				/*
1076 				 * We've gone off the rails.  Somewhere
1077 				 * along the line, one of the members of this
1078 				 * hash chain was deleted.  We could assert
1079 				 * that either the dirty list or the rinsing
1080 				 * list is non-NULL.  (The dtrace_sync() in
1081 				 * dtrace_dynvar_clean() would validate this
1082 				 * assertion.)
1083 				 */
1084 				ASSERT(op != DTRACE_DYNVAR_DEALLOC);
1085 				goto top;
1086 			}
1087 
1088 			goto next;
1089 		}
1090 
1091 		if (dtuple->dtt_nkeys != nkeys)
1092 			goto next;
1093 
1094 		for (i = 0; i < nkeys; i++, dkey++) {
1095 			if (dkey->dttk_size != key[i].dttk_size)
1096 				goto next; /* size or type mismatch */
1097 
1098 			if (dkey->dttk_size != 0) {
1099 				if (dtrace_bcmp(
1100 				    (void *)(uintptr_t)key[i].dttk_value,
1101 				    (void *)(uintptr_t)dkey->dttk_value,
1102 				    dkey->dttk_size))
1103 					goto next;
1104 			} else {
1105 				if (dkey->dttk_value != key[i].dttk_value)
1106 					goto next;
1107 			}
1108 		}
1109 
1110 		if (op != DTRACE_DYNVAR_DEALLOC)
1111 			return (dvar);
1112 
1113 		ASSERT(dvar->dtdv_next == NULL ||
1114 		    dvar->dtdv_next->dtdv_hashval != 0);
1115 
1116 		if (prev != NULL) {
1117 			ASSERT(hash[bucket].dtdh_chain != dvar);
1118 			ASSERT(start != dvar);
1119 			ASSERT(prev->dtdv_next == dvar);
1120 			prev->dtdv_next = dvar->dtdv_next;
1121 		} else {
1122 			if (dtrace_casptr(&hash[bucket].dtdh_chain,
1123 			    start, dvar->dtdv_next) != start) {
1124 				/*
1125 				 * We have failed to atomically swing the
1126 				 * hash table head pointer, presumably because
1127 				 * of a conflicting allocation on another CPU.
1128 				 * We need to reread the hash chain and try
1129 				 * again.
1130 				 */
1131 				goto top;
1132 			}
1133 		}
1134 
1135 		dtrace_membar_producer();
1136 
1137 		/*
1138 		 * Now clear the hash value to indicate that it's free.
1139 		 */
1140 		ASSERT(hash[bucket].dtdh_chain != dvar);
1141 		dvar->dtdv_hashval = 0;
1142 
1143 		dtrace_membar_producer();
1144 
1145 		/*
1146 		 * Set the next pointer to point at the dirty list, and
1147 		 * atomically swing the dirty pointer to the newly freed dvar.
1148 		 */
1149 		do {
1150 			next = dcpu->dtdsc_dirty;
1151 			dvar->dtdv_next = next;
1152 		} while (dtrace_casptr(&dcpu->dtdsc_dirty, next, dvar) != next);
1153 
1154 		/*
1155 		 * Finally, unlock this hash bucket.
1156 		 */
1157 		ASSERT(hash[bucket].dtdh_lock == lock);
1158 		ASSERT(lock & 1);
1159 		hash[bucket].dtdh_lock++;
1160 
1161 		return (NULL);
1162 next:
1163 		prev = dvar;
1164 		continue;
1165 	}
1166 
1167 	if (op != DTRACE_DYNVAR_ALLOC) {
1168 		/*
1169 		 * If we are not to allocate a new variable, we want to
1170 		 * return NULL now.  Before we return, check that the value
1171 		 * of the lock word hasn't changed.  If it has, we may have
1172 		 * seen an inconsistent snapshot.
1173 		 */
1174 		if (op == DTRACE_DYNVAR_NOALLOC) {
1175 			if (hash[bucket].dtdh_lock != lock)
1176 				goto top;
1177 		} else {
1178 			ASSERT(op == DTRACE_DYNVAR_DEALLOC);
1179 			ASSERT(hash[bucket].dtdh_lock == lock);
1180 			ASSERT(lock & 1);
1181 			hash[bucket].dtdh_lock++;
1182 		}
1183 
1184 		return (NULL);
1185 	}
1186 
1187 	/*
1188 	 * We need to allocate a new dynamic variable.  The size we need is the
1189 	 * size of dtrace_dynvar plus the size of nkeys dtrace_key_t's plus the
1190 	 * size of any auxiliary key data (rounded up to 8-byte alignment) plus
1191 	 * the size of any referred-to data (dsize).  We then round the final
1192 	 * size up to the chunksize for allocation.
1193 	 */
1194 	for (ksize = 0, i = 0; i < nkeys; i++)
1195 		ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
1196 
1197 	/*
1198 	 * This should be pretty much impossible, but could happen if, say,
1199 	 * strange DIF specified the tuple.  Ideally, this should be an
1200 	 * assertion and not an error condition -- but that requires that the
1201 	 * chunksize calculation in dtrace_difo_chunksize() be absolutely
1202 	 * bullet-proof.  (That is, it must not be able to be fooled by
1203 	 * malicious DIF.)  Given the lack of backwards branches in DIF,
1204 	 * solving this would presumably not amount to solving the Halting
1205 	 * Problem -- but it still seems awfully hard.
1206 	 */
1207 	if (sizeof (dtrace_dynvar_t) + sizeof (dtrace_key_t) * (nkeys - 1) +
1208 	    ksize + dsize > chunksize) {
1209 		dcpu->dtdsc_drops++;
1210 		return (NULL);
1211 	}
1212 
1213 	nstate = DTRACE_DSTATE_EMPTY;
1214 
1215 	do {
1216 retry:
1217 		free = dcpu->dtdsc_free;
1218 
1219 		if (free == NULL) {
1220 			dtrace_dynvar_t *clean = dcpu->dtdsc_clean;
1221 			void *rval;
1222 
1223 			if (clean == NULL) {
1224 				/*
1225 				 * We're out of dynamic variable space on
1226 				 * this CPU.  Unless we have tried all CPUs,
1227 				 * we'll try to allocate from a different
1228 				 * CPU.
1229 				 */
1230 				switch (dstate->dtds_state) {
1231 				case DTRACE_DSTATE_CLEAN: {
1232 					void *sp = &dstate->dtds_state;
1233 
1234 					if (++cpu >= NCPU)
1235 						cpu = 0;
1236 
1237 					if (dcpu->dtdsc_dirty != NULL &&
1238 					    nstate == DTRACE_DSTATE_EMPTY)
1239 						nstate = DTRACE_DSTATE_DIRTY;
1240 
1241 					if (dcpu->dtdsc_rinsing != NULL)
1242 						nstate = DTRACE_DSTATE_RINSING;
1243 
1244 					dcpu = &dstate->dtds_percpu[cpu];
1245 
1246 					if (cpu != me)
1247 						goto retry;
1248 
1249 					(void) dtrace_cas32(sp,
1250 					    DTRACE_DSTATE_CLEAN, nstate);
1251 
1252 					/*
1253 					 * To increment the correct bean
1254 					 * counter, take another lap.
1255 					 */
1256 					goto retry;
1257 				}
1258 
1259 				case DTRACE_DSTATE_DIRTY:
1260 					dcpu->dtdsc_dirty_drops++;
1261 					break;
1262 
1263 				case DTRACE_DSTATE_RINSING:
1264 					dcpu->dtdsc_rinsing_drops++;
1265 					break;
1266 
1267 				case DTRACE_DSTATE_EMPTY:
1268 					dcpu->dtdsc_drops++;
1269 					break;
1270 				}
1271 
1272 				DTRACE_CPUFLAG_SET(CPU_DTRACE_DROP);
1273 				return (NULL);
1274 			}
1275 
1276 			/*
1277 			 * The clean list appears to be non-empty.  We want to
1278 			 * move the clean list to the free list; we start by
1279 			 * moving the clean pointer aside.
1280 			 */
1281 			if (dtrace_casptr(&dcpu->dtdsc_clean,
1282 			    clean, NULL) != clean) {
1283 				/*
1284 				 * We are in one of two situations:
1285 				 *
1286 				 *  (a)	The clean list was switched to the
1287 				 *	free list by another CPU.
1288 				 *
1289 				 *  (b)	The clean list was added to by the
1290 				 *	cleansing cyclic.
1291 				 *
1292 				 * In either of these situations, we can
1293 				 * just reattempt the free list allocation.
1294 				 */
1295 				goto retry;
1296 			}
1297 
1298 			ASSERT(clean->dtdv_hashval == 0);
1299 
1300 			/*
1301 			 * Now we'll move the clean list to the free list.
1302 			 * It's impossible for this to fail:  the only way
1303 			 * the free list can be updated is through this
1304 			 * code path, and only one CPU can own the clean list.
1305 			 * Thus, it would only be possible for this to fail if
1306 			 * this code were racing with dtrace_dynvar_clean().
1307 			 * (That is, if dtrace_dynvar_clean() updated the clean
1308 			 * list, and we ended up racing to update the free
1309 			 * list.)  This race is prevented by the dtrace_sync()
1310 			 * in dtrace_dynvar_clean() -- which flushes the
1311 			 * owners of the clean lists out before resetting
1312 			 * the clean lists.
1313 			 */
1314 			rval = dtrace_casptr(&dcpu->dtdsc_free, NULL, clean);
1315 			ASSERT(rval == NULL);
1316 			goto retry;
1317 		}
1318 
1319 		dvar = free;
1320 		new_free = dvar->dtdv_next;
1321 	} while (dtrace_casptr(&dcpu->dtdsc_free, free, new_free) != free);
1322 
1323 	/*
1324 	 * We have now allocated a new chunk.  We copy the tuple keys into the
1325 	 * tuple array and copy any referenced key data into the data space
1326 	 * following the tuple array.  As we do this, we relocate dttk_value
1327 	 * in the final tuple to point to the key data address in the chunk.
1328 	 */
1329 	kdata = (uintptr_t)&dvar->dtdv_tuple.dtt_key[nkeys];
1330 	dvar->dtdv_data = (void *)(kdata + ksize);
1331 	dvar->dtdv_tuple.dtt_nkeys = nkeys;
1332 
1333 	for (i = 0; i < nkeys; i++) {
1334 		dtrace_key_t *dkey = &dvar->dtdv_tuple.dtt_key[i];
1335 		size_t kesize = key[i].dttk_size;
1336 
1337 		if (kesize != 0) {
1338 			dtrace_bcopy(
1339 			    (const void *)(uintptr_t)key[i].dttk_value,
1340 			    (void *)kdata, kesize);
1341 			dkey->dttk_value = kdata;
1342 			kdata += P2ROUNDUP(kesize, sizeof (uint64_t));
1343 		} else {
1344 			dkey->dttk_value = key[i].dttk_value;
1345 		}
1346 
1347 		dkey->dttk_size = kesize;
1348 	}
1349 
1350 	ASSERT(dvar->dtdv_hashval == 0);
1351 	dvar->dtdv_hashval = hashval;
1352 	dvar->dtdv_next = start;
1353 
1354 	if (dtrace_casptr(&hash[bucket].dtdh_chain, start, dvar) == start)
1355 		return (dvar);
1356 
1357 	/*
1358 	 * The cas has failed.  Either another CPU is adding an element to
1359 	 * this hash chain, or another CPU is deleting an element from this
1360 	 * hash chain.  The simplest way to deal with both of these cases
1361 	 * (though not necessarily the most efficient) is to free our
1362 	 * allocated block and tail-call ourselves.  Note that the free is
1363 	 * to the dirty list and _not_ to the free list.  This is to prevent
1364 	 * races with allocators, above.
1365 	 */
1366 	dvar->dtdv_hashval = 0;
1367 
1368 	dtrace_membar_producer();
1369 
1370 	do {
1371 		free = dcpu->dtdsc_dirty;
1372 		dvar->dtdv_next = free;
1373 	} while (dtrace_casptr(&dcpu->dtdsc_dirty, free, dvar) != free);
1374 
1375 	return (dtrace_dynvar(dstate, nkeys, key, dsize, op));
1376 }
1377 
1378 /*ARGSUSED*/
1379 static void
1380 dtrace_aggregate_min(uint64_t *oval, uint64_t nval, uint64_t arg)
1381 {
1382 	if (nval < *oval)
1383 		*oval = nval;
1384 }
1385 
1386 /*ARGSUSED*/
1387 static void
1388 dtrace_aggregate_max(uint64_t *oval, uint64_t nval, uint64_t arg)
1389 {
1390 	if (nval > *oval)
1391 		*oval = nval;
1392 }
1393 
1394 static void
1395 dtrace_aggregate_quantize(uint64_t *quanta, uint64_t nval, uint64_t incr)
1396 {
1397 	int i, zero = DTRACE_QUANTIZE_ZEROBUCKET;
1398 	int64_t val = (int64_t)nval;
1399 
1400 	if (val < 0) {
1401 		for (i = 0; i < zero; i++) {
1402 			if (val <= DTRACE_QUANTIZE_BUCKETVAL(i)) {
1403 				quanta[i] += incr;
1404 				return;
1405 			}
1406 		}
1407 	} else {
1408 		for (i = zero + 1; i < DTRACE_QUANTIZE_NBUCKETS; i++) {
1409 			if (val < DTRACE_QUANTIZE_BUCKETVAL(i)) {
1410 				quanta[i - 1] += incr;
1411 				return;
1412 			}
1413 		}
1414 
1415 		quanta[DTRACE_QUANTIZE_NBUCKETS - 1] += incr;
1416 		return;
1417 	}
1418 
1419 	ASSERT(0);
1420 }
1421 
1422 static void
1423 dtrace_aggregate_lquantize(uint64_t *lquanta, uint64_t nval, uint64_t incr)
1424 {
1425 	uint64_t arg = *lquanta++;
1426 	int32_t base = DTRACE_LQUANTIZE_BASE(arg);
1427 	uint16_t step = DTRACE_LQUANTIZE_STEP(arg);
1428 	uint16_t levels = DTRACE_LQUANTIZE_LEVELS(arg);
1429 	int32_t val = (int32_t)nval, level;
1430 
1431 	ASSERT(step != 0);
1432 	ASSERT(levels != 0);
1433 
1434 	if (val < base) {
1435 		/*
1436 		 * This is an underflow.
1437 		 */
1438 		lquanta[0] += incr;
1439 		return;
1440 	}
1441 
1442 	level = (val - base) / step;
1443 
1444 	if (level < levels) {
1445 		lquanta[level + 1] += incr;
1446 		return;
1447 	}
1448 
1449 	/*
1450 	 * This is an overflow.
1451 	 */
1452 	lquanta[levels + 1] += incr;
1453 }
1454 
1455 /*ARGSUSED*/
1456 static void
1457 dtrace_aggregate_avg(uint64_t *data, uint64_t nval, uint64_t arg)
1458 {
1459 	data[0]++;
1460 	data[1] += nval;
1461 }
1462 
1463 /*ARGSUSED*/
1464 static void
1465 dtrace_aggregate_count(uint64_t *oval, uint64_t nval, uint64_t arg)
1466 {
1467 	*oval = *oval + 1;
1468 }
1469 
1470 /*ARGSUSED*/
1471 static void
1472 dtrace_aggregate_sum(uint64_t *oval, uint64_t nval, uint64_t arg)
1473 {
1474 	*oval += nval;
1475 }
1476 
1477 /*
1478  * Aggregate given the tuple in the principal data buffer, and the aggregating
1479  * action denoted by the specified dtrace_aggregation_t.  The aggregation
1480  * buffer is specified as the buf parameter.  This routine does not return
1481  * failure; if there is no space in the aggregation buffer, the data will be
1482  * dropped, and a corresponding counter incremented.
1483  */
1484 static void
1485 dtrace_aggregate(dtrace_aggregation_t *agg, dtrace_buffer_t *dbuf,
1486     intptr_t offset, dtrace_buffer_t *buf, uint64_t expr, uint64_t arg)
1487 {
1488 	dtrace_recdesc_t *rec = &agg->dtag_action.dta_rec;
1489 	uint32_t i, ndx, size, fsize;
1490 	uint32_t align = sizeof (uint64_t) - 1;
1491 	dtrace_aggbuffer_t *agb;
1492 	dtrace_aggkey_t *key;
1493 	uint32_t hashval = 0;
1494 	caddr_t tomax, data, kdata;
1495 	dtrace_actkind_t action;
1496 	uintptr_t offs;
1497 
1498 	if (buf == NULL)
1499 		return;
1500 
1501 	if (!agg->dtag_hasarg) {
1502 		/*
1503 		 * Currently, only quantize() and lquantize() take additional
1504 		 * arguments, and they have the same semantics:  an increment
1505 		 * value that defaults to 1 when not present.  If additional
1506 		 * aggregating actions take arguments, the setting of the
1507 		 * default argument value will presumably have to become more
1508 		 * sophisticated...
1509 		 */
1510 		arg = 1;
1511 	}
1512 
1513 	action = agg->dtag_action.dta_kind - DTRACEACT_AGGREGATION;
1514 	size = rec->dtrd_offset - agg->dtag_base;
1515 	fsize = size + rec->dtrd_size;
1516 
1517 	ASSERT(dbuf->dtb_tomax != NULL);
1518 	data = dbuf->dtb_tomax + offset + agg->dtag_base;
1519 
1520 	if ((tomax = buf->dtb_tomax) == NULL) {
1521 		dtrace_buffer_drop(buf);
1522 		return;
1523 	}
1524 
1525 	/*
1526 	 * The metastructure is always at the bottom of the buffer.
1527 	 */
1528 	agb = (dtrace_aggbuffer_t *)(tomax + buf->dtb_size -
1529 	    sizeof (dtrace_aggbuffer_t));
1530 
1531 	if (buf->dtb_offset == 0) {
1532 		/*
1533 		 * We just kludge up approximately 1/8th of the size to be
1534 		 * buckets.  If this guess ends up being routinely
1535 		 * off-the-mark, we may need to dynamically readjust this
1536 		 * based on past performance.
1537 		 */
1538 		uintptr_t hashsize = (buf->dtb_size >> 3) / sizeof (uintptr_t);
1539 
1540 		if ((uintptr_t)agb - hashsize * sizeof (dtrace_aggkey_t *) <
1541 		    (uintptr_t)tomax || hashsize == 0) {
1542 			/*
1543 			 * We've been given a ludicrously small buffer;
1544 			 * increment our drop count and leave.
1545 			 */
1546 			dtrace_buffer_drop(buf);
1547 			return;
1548 		}
1549 
1550 		/*
1551 		 * And now, a pathetic attempt to try to get a an odd (or
1552 		 * perchance, a prime) hash size for better hash distribution.
1553 		 */
1554 		if (hashsize > (DTRACE_AGGHASHSIZE_SLEW << 3))
1555 			hashsize -= DTRACE_AGGHASHSIZE_SLEW;
1556 
1557 		agb->dtagb_hashsize = hashsize;
1558 		agb->dtagb_hash = (dtrace_aggkey_t **)((uintptr_t)agb -
1559 		    agb->dtagb_hashsize * sizeof (dtrace_aggkey_t *));
1560 		agb->dtagb_free = (uintptr_t)agb->dtagb_hash;
1561 
1562 		for (i = 0; i < agb->dtagb_hashsize; i++)
1563 			agb->dtagb_hash[i] = NULL;
1564 	}
1565 
1566 	/*
1567 	 * Calculate the hash value based on the key.  Note that we _don't_
1568 	 * include the aggid in the hashing (but we will store it as part of
1569 	 * the key).  The hashing algorithm is Bob Jenkins' "One-at-a-time"
1570 	 * algorithm: a simple, quick algorithm that has no known funnels, and
1571 	 * gets good distribution in practice.  The efficacy of the hashing
1572 	 * algorithm (and a comparison with other algorithms) may be found by
1573 	 * running the ::dtrace_aggstat MDB dcmd.
1574 	 */
1575 	for (i = sizeof (dtrace_aggid_t); i < size; i++) {
1576 		hashval += data[i];
1577 		hashval += (hashval << 10);
1578 		hashval ^= (hashval >> 6);
1579 	}
1580 
1581 	hashval += (hashval << 3);
1582 	hashval ^= (hashval >> 11);
1583 	hashval += (hashval << 15);
1584 
1585 	/*
1586 	 * Yes, the divide here is expensive.  If the cycle count here becomes
1587 	 * prohibitive, we can do tricks to eliminate it.
1588 	 */
1589 	ndx = hashval % agb->dtagb_hashsize;
1590 
1591 	for (key = agb->dtagb_hash[ndx]; key != NULL; key = key->dtak_next) {
1592 		ASSERT((caddr_t)key >= tomax);
1593 		ASSERT((caddr_t)key < tomax + buf->dtb_size);
1594 
1595 		if (hashval != key->dtak_hashval || key->dtak_size != size)
1596 			continue;
1597 
1598 		kdata = key->dtak_data;
1599 		ASSERT(kdata >= tomax && kdata < tomax + buf->dtb_size);
1600 
1601 		for (i = sizeof (dtrace_aggid_t); i < size; i++) {
1602 			if (kdata[i] != data[i])
1603 				goto next;
1604 		}
1605 
1606 		if (action != key->dtak_action) {
1607 			/*
1608 			 * We are aggregating on the same value in the same
1609 			 * aggregation with two different aggregating actions.
1610 			 * (This should have been picked up in the compiler,
1611 			 * so we may be dealing with errant or devious DIF.)
1612 			 * This is an error condition; we indicate as much,
1613 			 * and return.
1614 			 */
1615 			DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
1616 			return;
1617 		}
1618 
1619 		/*
1620 		 * This is a hit:  we need to apply the aggregator to
1621 		 * the value at this key.
1622 		 */
1623 		agg->dtag_aggregate((uint64_t *)(kdata + size), expr, arg);
1624 		return;
1625 next:
1626 		continue;
1627 	}
1628 
1629 	/*
1630 	 * We didn't find it.  We need to allocate some zero-filled space,
1631 	 * link it into the hash table appropriately, and apply the aggregator
1632 	 * to the (zero-filled) value.
1633 	 */
1634 	offs = buf->dtb_offset;
1635 	while (offs & (align - 1))
1636 		offs += sizeof (uint32_t);
1637 
1638 	/*
1639 	 * If we don't have enough room to both allocate a new key _and_
1640 	 * its associated data, increment the drop count and return.
1641 	 */
1642 	if ((uintptr_t)tomax + offs + fsize >
1643 	    agb->dtagb_free - sizeof (dtrace_aggkey_t)) {
1644 		dtrace_buffer_drop(buf);
1645 		return;
1646 	}
1647 
1648 	/*CONSTCOND*/
1649 	ASSERT(!(sizeof (dtrace_aggkey_t) & (sizeof (uintptr_t) - 1)));
1650 	key = (dtrace_aggkey_t *)(agb->dtagb_free - sizeof (dtrace_aggkey_t));
1651 	agb->dtagb_free -= sizeof (dtrace_aggkey_t);
1652 
1653 	key->dtak_data = kdata = tomax + offs;
1654 	buf->dtb_offset = offs + fsize;
1655 
1656 	/*
1657 	 * Now copy the data across.
1658 	 */
1659 	*((dtrace_aggid_t *)kdata) = agg->dtag_id;
1660 
1661 	for (i = sizeof (dtrace_aggid_t); i < size; i++)
1662 		kdata[i] = data[i];
1663 
1664 	for (i = size; i < fsize; i++)
1665 		kdata[i] = 0;
1666 
1667 	key->dtak_hashval = hashval;
1668 	key->dtak_size = size;
1669 	key->dtak_action = action;
1670 	key->dtak_next = agb->dtagb_hash[ndx];
1671 	agb->dtagb_hash[ndx] = key;
1672 
1673 	/*
1674 	 * Finally, apply the aggregator.
1675 	 */
1676 	*((uint64_t *)(key->dtak_data + size)) = agg->dtag_initial;
1677 	agg->dtag_aggregate((uint64_t *)(key->dtak_data + size), expr, arg);
1678 }
1679 
1680 /*
1681  * Given consumer state, this routine finds a speculation in the INACTIVE
1682  * state and transitions it into the ACTIVE state.  If there is no speculation
1683  * in the INACTIVE state, 0 is returned.  In this case, no error counter is
1684  * incremented -- it is up to the caller to take appropriate action.
1685  */
1686 static int
1687 dtrace_speculation(dtrace_state_t *state)
1688 {
1689 	int i = 0;
1690 	dtrace_speculation_state_t current;
1691 	uint32_t *stat = &state->dts_speculations_unavail, count;
1692 
1693 	while (i < state->dts_nspeculations) {
1694 		dtrace_speculation_t *spec = &state->dts_speculations[i];
1695 
1696 		current = spec->dtsp_state;
1697 
1698 		if (current != DTRACESPEC_INACTIVE) {
1699 			if (current == DTRACESPEC_COMMITTINGMANY ||
1700 			    current == DTRACESPEC_COMMITTING ||
1701 			    current == DTRACESPEC_DISCARDING)
1702 				stat = &state->dts_speculations_busy;
1703 			i++;
1704 			continue;
1705 		}
1706 
1707 		if (dtrace_cas32((uint32_t *)&spec->dtsp_state,
1708 		    current, DTRACESPEC_ACTIVE) == current)
1709 			return (i + 1);
1710 	}
1711 
1712 	/*
1713 	 * We couldn't find a speculation.  If we found as much as a single
1714 	 * busy speculation buffer, we'll attribute this failure as "busy"
1715 	 * instead of "unavail".
1716 	 */
1717 	do {
1718 		count = *stat;
1719 	} while (dtrace_cas32(stat, count, count + 1) != count);
1720 
1721 	return (0);
1722 }
1723 
1724 /*
1725  * This routine commits an active speculation.  If the specified speculation
1726  * is not in a valid state to perform a commit(), this routine will silently do
1727  * nothing.  The state of the specified speculation is transitioned according
1728  * to the state transition diagram outlined in <sys/dtrace_impl.h>
1729  */
1730 static void
1731 dtrace_speculation_commit(dtrace_state_t *state, processorid_t cpu,
1732     dtrace_specid_t which)
1733 {
1734 	dtrace_speculation_t *spec;
1735 	dtrace_buffer_t *src, *dest;
1736 	uintptr_t daddr, saddr, dlimit;
1737 	dtrace_speculation_state_t current, new;
1738 	intptr_t offs;
1739 
1740 	if (which == 0)
1741 		return;
1742 
1743 	if (which > state->dts_nspeculations) {
1744 		cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
1745 		return;
1746 	}
1747 
1748 	spec = &state->dts_speculations[which - 1];
1749 	src = &spec->dtsp_buffer[cpu];
1750 	dest = &state->dts_buffer[cpu];
1751 
1752 	do {
1753 		current = spec->dtsp_state;
1754 
1755 		if (current == DTRACESPEC_COMMITTINGMANY)
1756 			break;
1757 
1758 		switch (current) {
1759 		case DTRACESPEC_INACTIVE:
1760 		case DTRACESPEC_DISCARDING:
1761 			return;
1762 
1763 		case DTRACESPEC_COMMITTING:
1764 			/*
1765 			 * This is only possible if we are (a) commit()'ing
1766 			 * without having done a prior speculate() on this CPU
1767 			 * and (b) racing with another commit() on a different
1768 			 * CPU.  There's nothing to do -- we just assert that
1769 			 * our offset is 0.
1770 			 */
1771 			ASSERT(src->dtb_offset == 0);
1772 			return;
1773 
1774 		case DTRACESPEC_ACTIVE:
1775 			new = DTRACESPEC_COMMITTING;
1776 			break;
1777 
1778 		case DTRACESPEC_ACTIVEONE:
1779 			/*
1780 			 * This speculation is active on one CPU.  If our
1781 			 * buffer offset is non-zero, we know that the one CPU
1782 			 * must be us.  Otherwise, we are committing on a
1783 			 * different CPU from the speculate(), and we must
1784 			 * rely on being asynchronously cleaned.
1785 			 */
1786 			if (src->dtb_offset != 0) {
1787 				new = DTRACESPEC_COMMITTING;
1788 				break;
1789 			}
1790 			/*FALLTHROUGH*/
1791 
1792 		case DTRACESPEC_ACTIVEMANY:
1793 			new = DTRACESPEC_COMMITTINGMANY;
1794 			break;
1795 
1796 		default:
1797 			ASSERT(0);
1798 		}
1799 	} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
1800 	    current, new) != current);
1801 
1802 	/*
1803 	 * We have set the state to indicate that we are committing this
1804 	 * speculation.  Now reserve the necessary space in the destination
1805 	 * buffer.
1806 	 */
1807 	if ((offs = dtrace_buffer_reserve(dest, src->dtb_offset,
1808 	    sizeof (uint64_t), state, NULL)) < 0) {
1809 		dtrace_buffer_drop(dest);
1810 		goto out;
1811 	}
1812 
1813 	/*
1814 	 * We have the space; copy the buffer across.  (Note that this is a
1815 	 * highly subobtimal bcopy(); in the unlikely event that this becomes
1816 	 * a serious performance issue, a high-performance DTrace-specific
1817 	 * bcopy() should obviously be invented.)
1818 	 */
1819 	daddr = (uintptr_t)dest->dtb_tomax + offs;
1820 	dlimit = daddr + src->dtb_offset;
1821 	saddr = (uintptr_t)src->dtb_tomax;
1822 
1823 	/*
1824 	 * First, the aligned portion.
1825 	 */
1826 	while (dlimit - daddr >= sizeof (uint64_t)) {
1827 		*((uint64_t *)daddr) = *((uint64_t *)saddr);
1828 
1829 		daddr += sizeof (uint64_t);
1830 		saddr += sizeof (uint64_t);
1831 	}
1832 
1833 	/*
1834 	 * Now any left-over bit...
1835 	 */
1836 	while (dlimit - daddr)
1837 		*((uint8_t *)daddr++) = *((uint8_t *)saddr++);
1838 
1839 	/*
1840 	 * Finally, commit the reserved space in the destination buffer.
1841 	 */
1842 	dest->dtb_offset = offs + src->dtb_offset;
1843 
1844 out:
1845 	/*
1846 	 * If we're lucky enough to be the only active CPU on this speculation
1847 	 * buffer, we can just set the state back to DTRACESPEC_INACTIVE.
1848 	 */
1849 	if (current == DTRACESPEC_ACTIVE ||
1850 	    (current == DTRACESPEC_ACTIVEONE && new == DTRACESPEC_COMMITTING)) {
1851 		uint32_t rval = dtrace_cas32((uint32_t *)&spec->dtsp_state,
1852 		    DTRACESPEC_COMMITTING, DTRACESPEC_INACTIVE);
1853 
1854 		ASSERT(rval == DTRACESPEC_COMMITTING);
1855 	}
1856 
1857 	src->dtb_offset = 0;
1858 	src->dtb_xamot_drops += src->dtb_drops;
1859 	src->dtb_drops = 0;
1860 }
1861 
1862 /*
1863  * This routine discards an active speculation.  If the specified speculation
1864  * is not in a valid state to perform a discard(), this routine will silently
1865  * do nothing.  The state of the specified speculation is transitioned
1866  * according to the state transition diagram outlined in <sys/dtrace_impl.h>
1867  */
1868 static void
1869 dtrace_speculation_discard(dtrace_state_t *state, processorid_t cpu,
1870     dtrace_specid_t which)
1871 {
1872 	dtrace_speculation_t *spec;
1873 	dtrace_speculation_state_t current, new;
1874 	dtrace_buffer_t *buf;
1875 
1876 	if (which == 0)
1877 		return;
1878 
1879 	if (which > state->dts_nspeculations) {
1880 		cpu_core[cpu].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
1881 		return;
1882 	}
1883 
1884 	spec = &state->dts_speculations[which - 1];
1885 	buf = &spec->dtsp_buffer[cpu];
1886 
1887 	do {
1888 		current = spec->dtsp_state;
1889 
1890 		switch (current) {
1891 		case DTRACESPEC_INACTIVE:
1892 		case DTRACESPEC_COMMITTINGMANY:
1893 		case DTRACESPEC_COMMITTING:
1894 		case DTRACESPEC_DISCARDING:
1895 			return;
1896 
1897 		case DTRACESPEC_ACTIVE:
1898 		case DTRACESPEC_ACTIVEMANY:
1899 			new = DTRACESPEC_DISCARDING;
1900 			break;
1901 
1902 		case DTRACESPEC_ACTIVEONE:
1903 			if (buf->dtb_offset != 0) {
1904 				new = DTRACESPEC_INACTIVE;
1905 			} else {
1906 				new = DTRACESPEC_DISCARDING;
1907 			}
1908 			break;
1909 
1910 		default:
1911 			ASSERT(0);
1912 		}
1913 	} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
1914 	    current, new) != current);
1915 
1916 	buf->dtb_offset = 0;
1917 	buf->dtb_drops = 0;
1918 }
1919 
1920 /*
1921  * Note:  not called from probe context.  This function is called
1922  * asynchronously from cross call context to clean any speculations that are
1923  * in the COMMITTINGMANY or DISCARDING states.  These speculations may not be
1924  * transitioned back to the INACTIVE state until all CPUs have cleaned the
1925  * speculation.
1926  */
1927 static void
1928 dtrace_speculation_clean_here(dtrace_state_t *state)
1929 {
1930 	dtrace_icookie_t cookie;
1931 	processorid_t cpu = CPU->cpu_id;
1932 	dtrace_buffer_t *dest = &state->dts_buffer[cpu];
1933 	dtrace_specid_t i;
1934 
1935 	cookie = dtrace_interrupt_disable();
1936 
1937 	if (dest->dtb_tomax == NULL) {
1938 		dtrace_interrupt_enable(cookie);
1939 		return;
1940 	}
1941 
1942 	for (i = 0; i < state->dts_nspeculations; i++) {
1943 		dtrace_speculation_t *spec = &state->dts_speculations[i];
1944 		dtrace_buffer_t *src = &spec->dtsp_buffer[cpu];
1945 
1946 		if (src->dtb_tomax == NULL)
1947 			continue;
1948 
1949 		if (spec->dtsp_state == DTRACESPEC_DISCARDING) {
1950 			src->dtb_offset = 0;
1951 			continue;
1952 		}
1953 
1954 		if (spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
1955 			continue;
1956 
1957 		if (src->dtb_offset == 0)
1958 			continue;
1959 
1960 		dtrace_speculation_commit(state, cpu, i + 1);
1961 	}
1962 
1963 	dtrace_interrupt_enable(cookie);
1964 }
1965 
1966 /*
1967  * Note:  not called from probe context.  This function is called
1968  * asynchronously (and at a regular interval) to clean any speculations that
1969  * are in the COMMITTINGMANY or DISCARDING states.  If it discovers that there
1970  * is work to be done, it cross calls all CPUs to perform that work;
1971  * COMMITMANY and DISCARDING speculations may not be transitioned back to the
1972  * INACTIVE state until they have been cleaned by all CPUs.
1973  */
1974 static void
1975 dtrace_speculation_clean(dtrace_state_t *state)
1976 {
1977 	int work = 0, rv;
1978 	dtrace_specid_t i;
1979 
1980 	for (i = 0; i < state->dts_nspeculations; i++) {
1981 		dtrace_speculation_t *spec = &state->dts_speculations[i];
1982 
1983 		ASSERT(!spec->dtsp_cleaning);
1984 
1985 		if (spec->dtsp_state != DTRACESPEC_DISCARDING &&
1986 		    spec->dtsp_state != DTRACESPEC_COMMITTINGMANY)
1987 			continue;
1988 
1989 		work++;
1990 		spec->dtsp_cleaning = 1;
1991 	}
1992 
1993 	if (!work)
1994 		return;
1995 
1996 	dtrace_xcall(DTRACE_CPUALL,
1997 	    (dtrace_xcall_t)dtrace_speculation_clean_here, state);
1998 
1999 	/*
2000 	 * We now know that all CPUs have committed or discarded their
2001 	 * speculation buffers, as appropriate.  We can now set the state
2002 	 * to inactive.
2003 	 */
2004 	for (i = 0; i < state->dts_nspeculations; i++) {
2005 		dtrace_speculation_t *spec = &state->dts_speculations[i];
2006 		dtrace_speculation_state_t current, new;
2007 
2008 		if (!spec->dtsp_cleaning)
2009 			continue;
2010 
2011 		current = spec->dtsp_state;
2012 		ASSERT(current == DTRACESPEC_DISCARDING ||
2013 		    current == DTRACESPEC_COMMITTINGMANY);
2014 
2015 		new = DTRACESPEC_INACTIVE;
2016 
2017 		rv = dtrace_cas32((uint32_t *)&spec->dtsp_state, current, new);
2018 		ASSERT(rv == current);
2019 		spec->dtsp_cleaning = 0;
2020 	}
2021 }
2022 
2023 /*
2024  * Called as part of a speculate() to get the speculative buffer associated
2025  * with a given speculation.  Returns NULL if the specified speculation is not
2026  * in an ACTIVE state.  If the speculation is in the ACTIVEONE state -- and
2027  * the active CPU is not the specified CPU -- the speculation will be
2028  * atomically transitioned into the ACTIVEMANY state.
2029  */
2030 static dtrace_buffer_t *
2031 dtrace_speculation_buffer(dtrace_state_t *state, processorid_t cpuid,
2032     dtrace_specid_t which)
2033 {
2034 	dtrace_speculation_t *spec;
2035 	dtrace_speculation_state_t current, new;
2036 	dtrace_buffer_t *buf;
2037 
2038 	if (which == 0)
2039 		return (NULL);
2040 
2041 	if (which > state->dts_nspeculations) {
2042 		cpu_core[cpuid].cpuc_dtrace_flags |= CPU_DTRACE_ILLOP;
2043 		return (NULL);
2044 	}
2045 
2046 	spec = &state->dts_speculations[which - 1];
2047 	buf = &spec->dtsp_buffer[cpuid];
2048 
2049 	do {
2050 		current = spec->dtsp_state;
2051 
2052 		switch (current) {
2053 		case DTRACESPEC_INACTIVE:
2054 		case DTRACESPEC_COMMITTINGMANY:
2055 		case DTRACESPEC_DISCARDING:
2056 			return (NULL);
2057 
2058 		case DTRACESPEC_COMMITTING:
2059 			ASSERT(buf->dtb_offset == 0);
2060 			return (NULL);
2061 
2062 		case DTRACESPEC_ACTIVEONE:
2063 			/*
2064 			 * This speculation is currently active on one CPU.
2065 			 * Check the offset in the buffer; if it's non-zero,
2066 			 * that CPU must be us (and we leave the state alone).
2067 			 * If it's zero, assume that we're starting on a new
2068 			 * CPU -- and change the state to indicate that the
2069 			 * speculation is active on more than one CPU.
2070 			 */
2071 			if (buf->dtb_offset != 0)
2072 				return (buf);
2073 
2074 			new = DTRACESPEC_ACTIVEMANY;
2075 			break;
2076 
2077 		case DTRACESPEC_ACTIVEMANY:
2078 			return (buf);
2079 
2080 		case DTRACESPEC_ACTIVE:
2081 			new = DTRACESPEC_ACTIVEONE;
2082 			break;
2083 
2084 		default:
2085 			ASSERT(0);
2086 		}
2087 	} while (dtrace_cas32((uint32_t *)&spec->dtsp_state,
2088 	    current, new) != current);
2089 
2090 	ASSERT(new == DTRACESPEC_ACTIVEONE || new == DTRACESPEC_ACTIVEMANY);
2091 	return (buf);
2092 }
2093 
2094 /*
2095  * This function implements the DIF emulator's variable lookups.  The emulator
2096  * passes a reserved variable identifier and optional built-in array index.
2097  */
2098 static uint64_t
2099 dtrace_dif_variable(dtrace_mstate_t *mstate, dtrace_state_t *state, uint64_t v,
2100     uint64_t ndx)
2101 {
2102 	/*
2103 	 * If we're accessing one of the uncached arguments, we'll turn this
2104 	 * into a reference in the args array.
2105 	 */
2106 	if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9) {
2107 		ndx = v - DIF_VAR_ARG0;
2108 		v = DIF_VAR_ARGS;
2109 	}
2110 
2111 	switch (v) {
2112 	case DIF_VAR_ARGS:
2113 		ASSERT(mstate->dtms_present & DTRACE_MSTATE_ARGS);
2114 		if (ndx >= sizeof (mstate->dtms_arg) /
2115 		    sizeof (mstate->dtms_arg[0])) {
2116 			int aframes = mstate->dtms_probe->dtpr_aframes + 2;
2117 			dtrace_provider_t *pv;
2118 			uint64_t val;
2119 
2120 			pv = mstate->dtms_probe->dtpr_provider;
2121 			if (pv->dtpv_pops.dtps_getargval != NULL)
2122 				val = pv->dtpv_pops.dtps_getargval(pv->dtpv_arg,
2123 				    mstate->dtms_probe->dtpr_id,
2124 				    mstate->dtms_probe->dtpr_arg, ndx, aframes);
2125 			else
2126 				val = dtrace_getarg(ndx, aframes);
2127 
2128 			/*
2129 			 * This is regrettably required to keep the compiler
2130 			 * from tail-optimizing the call to dtrace_getarg().
2131 			 * The condition always evaluates to true, but the
2132 			 * compiler has no way of figuring that out a priori.
2133 			 * (None of this would be necessary if the compiler
2134 			 * could be relied upon to _always_ tail-optimize
2135 			 * the call to dtrace_getarg() -- but it can't.)
2136 			 */
2137 			if (mstate->dtms_probe != NULL)
2138 				return (val);
2139 
2140 			ASSERT(0);
2141 		}
2142 
2143 		return (mstate->dtms_arg[ndx]);
2144 
2145 	case DIF_VAR_UREGS: {
2146 		klwp_t *lwp;
2147 
2148 		if (!dtrace_priv_proc(state))
2149 			return (0);
2150 
2151 		if ((lwp = curthread->t_lwp) == NULL) {
2152 			DTRACE_CPUFLAG_SET(CPU_DTRACE_BADADDR);
2153 			cpu_core[CPU->cpu_id].cpuc_dtrace_illval = NULL;
2154 			return (0);
2155 		}
2156 
2157 		return (dtrace_getreg(lwp->lwp_regs, ndx));
2158 	}
2159 
2160 	case DIF_VAR_CURTHREAD:
2161 		if (!dtrace_priv_kernel(state))
2162 			return (0);
2163 		return ((uint64_t)(uintptr_t)curthread);
2164 
2165 	case DIF_VAR_TIMESTAMP:
2166 		if (!(mstate->dtms_present & DTRACE_MSTATE_TIMESTAMP)) {
2167 			mstate->dtms_timestamp = dtrace_gethrtime();
2168 			mstate->dtms_present |= DTRACE_MSTATE_TIMESTAMP;
2169 		}
2170 		return (mstate->dtms_timestamp);
2171 
2172 	case DIF_VAR_VTIMESTAMP:
2173 		ASSERT(dtrace_vtime_references != 0);
2174 		return (curthread->t_dtrace_vtime);
2175 
2176 	case DIF_VAR_WALLTIMESTAMP:
2177 		if (!(mstate->dtms_present & DTRACE_MSTATE_WALLTIMESTAMP)) {
2178 			mstate->dtms_walltimestamp = dtrace_gethrestime();
2179 			mstate->dtms_present |= DTRACE_MSTATE_WALLTIMESTAMP;
2180 		}
2181 		return (mstate->dtms_walltimestamp);
2182 
2183 	case DIF_VAR_IPL:
2184 		if (!dtrace_priv_kernel(state))
2185 			return (0);
2186 		if (!(mstate->dtms_present & DTRACE_MSTATE_IPL)) {
2187 			mstate->dtms_ipl = dtrace_getipl();
2188 			mstate->dtms_present |= DTRACE_MSTATE_IPL;
2189 		}
2190 		return (mstate->dtms_ipl);
2191 
2192 	case DIF_VAR_EPID:
2193 		ASSERT(mstate->dtms_present & DTRACE_MSTATE_EPID);
2194 		return (mstate->dtms_epid);
2195 
2196 	case DIF_VAR_ID:
2197 		ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
2198 		return (mstate->dtms_probe->dtpr_id);
2199 
2200 	case DIF_VAR_STACKDEPTH:
2201 		if (!dtrace_priv_kernel(state))
2202 			return (0);
2203 		if (!(mstate->dtms_present & DTRACE_MSTATE_STACKDEPTH)) {
2204 			int aframes = mstate->dtms_probe->dtpr_aframes + 2;
2205 
2206 			mstate->dtms_stackdepth = dtrace_getstackdepth(aframes);
2207 			mstate->dtms_present |= DTRACE_MSTATE_STACKDEPTH;
2208 		}
2209 		return (mstate->dtms_stackdepth);
2210 
2211 	case DIF_VAR_USTACKDEPTH:
2212 		if (!dtrace_priv_proc(state))
2213 			return (0);
2214 		if (!(mstate->dtms_present & DTRACE_MSTATE_USTACKDEPTH)) {
2215 			/*
2216 			 * See comment in DIF_VAR_PID.
2217 			 */
2218 			if (DTRACE_ANCHORED(mstate->dtms_probe) &&
2219 			    CPU_ON_INTR(CPU)) {
2220 				mstate->dtms_ustackdepth = 0;
2221 			} else {
2222 				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2223 				mstate->dtms_ustackdepth =
2224 				    dtrace_getustackdepth();
2225 				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
2226 			}
2227 			mstate->dtms_present |= DTRACE_MSTATE_USTACKDEPTH;
2228 		}
2229 		return (mstate->dtms_ustackdepth);
2230 
2231 	case DIF_VAR_CALLER:
2232 		if (!dtrace_priv_kernel(state))
2233 			return (0);
2234 		if (!(mstate->dtms_present & DTRACE_MSTATE_CALLER)) {
2235 			int aframes = mstate->dtms_probe->dtpr_aframes + 2;
2236 
2237 			if (!DTRACE_ANCHORED(mstate->dtms_probe)) {
2238 				/*
2239 				 * If this is an unanchored probe, we are
2240 				 * required to go through the slow path:
2241 				 * dtrace_caller() only guarantees correct
2242 				 * results for anchored probes.
2243 				 */
2244 				pc_t caller[2];
2245 
2246 				dtrace_getpcstack(caller, 2, aframes,
2247 				    (uint32_t *)(uintptr_t)mstate->dtms_arg[0]);
2248 				mstate->dtms_caller = caller[1];
2249 			} else if ((mstate->dtms_caller =
2250 			    dtrace_caller(aframes)) == -1) {
2251 				/*
2252 				 * We have failed to do this the quick way;
2253 				 * we must resort to the slower approach of
2254 				 * calling dtrace_getpcstack().
2255 				 */
2256 				pc_t caller;
2257 
2258 				dtrace_getpcstack(&caller, 1, aframes, NULL);
2259 				mstate->dtms_caller = caller;
2260 			}
2261 
2262 			mstate->dtms_present |= DTRACE_MSTATE_CALLER;
2263 		}
2264 		return (mstate->dtms_caller);
2265 
2266 	case DIF_VAR_UCALLER:
2267 		if (!dtrace_priv_proc(state))
2268 			return (0);
2269 
2270 		if (!(mstate->dtms_present & DTRACE_MSTATE_UCALLER)) {
2271 			uint64_t ustack[3];
2272 
2273 			/*
2274 			 * dtrace_getupcstack() fills in the first uint64_t
2275 			 * with the current PID.  The second uint64_t will
2276 			 * be the program counter at user-level.  The third
2277 			 * uint64_t will contain the caller, which is what
2278 			 * we're after.
2279 			 */
2280 			ustack[2] = NULL;
2281 			dtrace_getupcstack(ustack, 3);
2282 			mstate->dtms_ucaller = ustack[2];
2283 			mstate->dtms_present |= DTRACE_MSTATE_UCALLER;
2284 		}
2285 
2286 		return (mstate->dtms_ucaller);
2287 
2288 	case DIF_VAR_PROBEPROV:
2289 		ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
2290 		return ((uint64_t)(uintptr_t)
2291 		    mstate->dtms_probe->dtpr_provider->dtpv_name);
2292 
2293 	case DIF_VAR_PROBEMOD:
2294 		ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
2295 		return ((uint64_t)(uintptr_t)
2296 		    mstate->dtms_probe->dtpr_mod);
2297 
2298 	case DIF_VAR_PROBEFUNC:
2299 		ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
2300 		return ((uint64_t)(uintptr_t)
2301 		    mstate->dtms_probe->dtpr_func);
2302 
2303 	case DIF_VAR_PROBENAME:
2304 		ASSERT(mstate->dtms_present & DTRACE_MSTATE_PROBE);
2305 		return ((uint64_t)(uintptr_t)
2306 		    mstate->dtms_probe->dtpr_name);
2307 
2308 	case DIF_VAR_PID:
2309 		if (!dtrace_priv_proc(state))
2310 			return (0);
2311 
2312 		/*
2313 		 * Note that we are assuming that an unanchored probe is
2314 		 * always due to a high-level interrupt.  (And we're assuming
2315 		 * that there is only a single high level interrupt.)
2316 		 */
2317 		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
2318 			return (pid0.pid_id);
2319 
2320 		/*
2321 		 * It is always safe to dereference one's own t_procp pointer:
2322 		 * it always points to a valid, allocated proc structure.
2323 		 * Further, it is always safe to dereference the p_pidp member
2324 		 * of one's own proc structure.  (These are truisms becuase
2325 		 * threads and processes don't clean up their own state --
2326 		 * they leave that task to whomever reaps them.)
2327 		 */
2328 		return ((uint64_t)curthread->t_procp->p_pidp->pid_id);
2329 
2330 	case DIF_VAR_TID:
2331 		/*
2332 		 * See comment in DIF_VAR_PID.
2333 		 */
2334 		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
2335 			return (0);
2336 
2337 		return ((uint64_t)curthread->t_tid);
2338 
2339 	case DIF_VAR_EXECNAME:
2340 		if (!dtrace_priv_proc(state))
2341 			return (0);
2342 
2343 		/*
2344 		 * See comment in DIF_VAR_PID.
2345 		 */
2346 		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
2347 			return ((uint64_t)(uintptr_t)p0.p_user.u_comm);
2348 
2349 		/*
2350 		 * It is always safe to dereference one's own t_procp pointer:
2351 		 * it always points to a valid, allocated proc structure.
2352 		 * (This is true because threads don't clean up their own
2353 		 * state -- they leave that task to whomever reaps them.)
2354 		 */
2355 		return ((uint64_t)(uintptr_t)
2356 		    curthread->t_procp->p_user.u_comm);
2357 
2358 	case DIF_VAR_ZONENAME:
2359 		if (!dtrace_priv_proc(state))
2360 			return (0);
2361 
2362 		/*
2363 		 * See comment in DIF_VAR_PID.
2364 		 */
2365 		if (DTRACE_ANCHORED(mstate->dtms_probe) && CPU_ON_INTR(CPU))
2366 			return ((uint64_t)(uintptr_t)p0.p_zone->zone_name);
2367 
2368 		/*
2369 		 * It is always safe to dereference one's own t_procp pointer:
2370 		 * it always points to a valid, allocated proc structure.
2371 		 * (This is true because threads don't clean up their own
2372 		 * state -- they leave that task to whomever reaps them.)
2373 		 */
2374 		return ((uint64_t)(uintptr_t)
2375 		    curthread->t_procp->p_zone->zone_name);
2376 
2377 	default:
2378 		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
2379 		return (0);
2380 	}
2381 }
2382 
2383 /*
2384  * Emulate the execution of DTrace ID subroutines invoked by the call opcode.
2385  * Notice that we don't bother validating the proper number of arguments or
2386  * their types in the tuple stack.  This isn't needed because all argument
2387  * interpretation is safe because of our load safety -- the worst that can
2388  * happen is that a bogus program can obtain bogus results.
2389  */
2390 static void
2391 dtrace_dif_subr(uint_t subr, uint_t rd, uint64_t *regs,
2392     dtrace_key_t *tupregs, int nargs,
2393     dtrace_mstate_t *mstate, dtrace_state_t *state)
2394 {
2395 	volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
2396 	volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
2397 
2398 	union {
2399 		mutex_impl_t mi;
2400 		uint64_t mx;
2401 	} m;
2402 
2403 	union {
2404 		krwlock_t ri;
2405 		uintptr_t rw;
2406 	} r;
2407 
2408 	switch (subr) {
2409 	case DIF_SUBR_RAND:
2410 		regs[rd] = (dtrace_gethrtime() * 2416 + 374441) % 1771875;
2411 		break;
2412 
2413 	case DIF_SUBR_MUTEX_OWNED:
2414 		m.mx = dtrace_load64(tupregs[0].dttk_value);
2415 		if (MUTEX_TYPE_ADAPTIVE(&m.mi))
2416 			regs[rd] = MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER;
2417 		else
2418 			regs[rd] = LOCK_HELD(&m.mi.m_spin.m_spinlock);
2419 		break;
2420 
2421 	case DIF_SUBR_MUTEX_OWNER:
2422 		m.mx = dtrace_load64(tupregs[0].dttk_value);
2423 		if (MUTEX_TYPE_ADAPTIVE(&m.mi) &&
2424 		    MUTEX_OWNER(&m.mi) != MUTEX_NO_OWNER)
2425 			regs[rd] = (uintptr_t)MUTEX_OWNER(&m.mi);
2426 		else
2427 			regs[rd] = 0;
2428 		break;
2429 
2430 	case DIF_SUBR_MUTEX_TYPE_ADAPTIVE:
2431 		m.mx = dtrace_load64(tupregs[0].dttk_value);
2432 		regs[rd] = MUTEX_TYPE_ADAPTIVE(&m.mi);
2433 		break;
2434 
2435 	case DIF_SUBR_MUTEX_TYPE_SPIN:
2436 		m.mx = dtrace_load64(tupregs[0].dttk_value);
2437 		regs[rd] = MUTEX_TYPE_SPIN(&m.mi);
2438 		break;
2439 
2440 	case DIF_SUBR_RW_READ_HELD: {
2441 		uintptr_t tmp;
2442 
2443 		r.rw = dtrace_loadptr(tupregs[0].dttk_value);
2444 		regs[rd] = _RW_READ_HELD(&r.ri, tmp);
2445 		break;
2446 	}
2447 
2448 	case DIF_SUBR_RW_WRITE_HELD:
2449 		r.rw = dtrace_loadptr(tupregs[0].dttk_value);
2450 		regs[rd] = _RW_WRITE_HELD(&r.ri);
2451 		break;
2452 
2453 	case DIF_SUBR_RW_ISWRITER:
2454 		r.rw = dtrace_loadptr(tupregs[0].dttk_value);
2455 		regs[rd] = _RW_ISWRITER(&r.ri);
2456 		break;
2457 
2458 	case DIF_SUBR_BCOPY: {
2459 		/*
2460 		 * We need to be sure that the destination is in the scratch
2461 		 * region -- no other region is allowed.
2462 		 */
2463 		uintptr_t src = tupregs[0].dttk_value;
2464 		uintptr_t dest = tupregs[1].dttk_value;
2465 		size_t size = tupregs[2].dttk_value;
2466 
2467 		if (!dtrace_inscratch(dest, size, mstate)) {
2468 			*flags |= CPU_DTRACE_BADADDR;
2469 			*illval = regs[rd];
2470 			break;
2471 		}
2472 
2473 		dtrace_bcopy((void *)src, (void *)dest, size);
2474 		break;
2475 	}
2476 
2477 	case DIF_SUBR_ALLOCA:
2478 	case DIF_SUBR_COPYIN: {
2479 		uintptr_t dest = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
2480 		uint64_t size =
2481 		    tupregs[subr == DIF_SUBR_ALLOCA ? 0 : 1].dttk_value;
2482 		size_t scratch_size = (dest - mstate->dtms_scratch_ptr) + size;
2483 
2484 		/*
2485 		 * This action doesn't require any credential checks since
2486 		 * probes will not activate in user contexts to which the
2487 		 * enabling user does not have permissions.
2488 		 */
2489 		if (mstate->dtms_scratch_ptr + scratch_size >
2490 		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
2491 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
2492 			regs[rd] = NULL;
2493 			break;
2494 		}
2495 
2496 		if (subr == DIF_SUBR_COPYIN) {
2497 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2498 			dtrace_copyin(tupregs[0].dttk_value, dest, size);
2499 			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
2500 		}
2501 
2502 		mstate->dtms_scratch_ptr += scratch_size;
2503 		regs[rd] = dest;
2504 		break;
2505 	}
2506 
2507 	case DIF_SUBR_COPYINTO: {
2508 		uint64_t size = tupregs[1].dttk_value;
2509 		uintptr_t dest = tupregs[2].dttk_value;
2510 
2511 		/*
2512 		 * This action doesn't require any credential checks since
2513 		 * probes will not activate in user contexts to which the
2514 		 * enabling user does not have permissions.
2515 		 */
2516 		if (!dtrace_inscratch(dest, size, mstate)) {
2517 			*flags |= CPU_DTRACE_BADADDR;
2518 			*illval = regs[rd];
2519 			break;
2520 		}
2521 
2522 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2523 		dtrace_copyin(tupregs[0].dttk_value, dest, size);
2524 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
2525 		break;
2526 	}
2527 
2528 	case DIF_SUBR_COPYINSTR: {
2529 		uintptr_t dest = mstate->dtms_scratch_ptr;
2530 		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
2531 
2532 		if (nargs > 1 && tupregs[1].dttk_value < size)
2533 			size = tupregs[1].dttk_value + 1;
2534 
2535 		/*
2536 		 * This action doesn't require any credential checks since
2537 		 * probes will not activate in user contexts to which the
2538 		 * enabling user does not have permissions.
2539 		 */
2540 		if (mstate->dtms_scratch_ptr + size >
2541 		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
2542 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
2543 			regs[rd] = NULL;
2544 			break;
2545 		}
2546 
2547 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2548 		dtrace_copyinstr(tupregs[0].dttk_value, dest, size);
2549 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
2550 
2551 		((char *)dest)[size - 1] = '\0';
2552 		mstate->dtms_scratch_ptr += size;
2553 		regs[rd] = dest;
2554 		break;
2555 	}
2556 
2557 	case DIF_SUBR_MSGSIZE:
2558 	case DIF_SUBR_MSGDSIZE: {
2559 		uintptr_t baddr = tupregs[0].dttk_value, daddr;
2560 		uintptr_t wptr, rptr;
2561 		size_t count = 0;
2562 		int cont = 0;
2563 
2564 		while (baddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
2565 			wptr = dtrace_loadptr(baddr +
2566 			    offsetof(mblk_t, b_wptr));
2567 
2568 			rptr = dtrace_loadptr(baddr +
2569 			    offsetof(mblk_t, b_rptr));
2570 
2571 			if (wptr < rptr) {
2572 				*flags |= CPU_DTRACE_BADADDR;
2573 				*illval = tupregs[0].dttk_value;
2574 				break;
2575 			}
2576 
2577 			daddr = dtrace_loadptr(baddr +
2578 			    offsetof(mblk_t, b_datap));
2579 
2580 			baddr = dtrace_loadptr(baddr +
2581 			    offsetof(mblk_t, b_cont));
2582 
2583 			/*
2584 			 * We want to prevent against denial-of-service here,
2585 			 * so we're only going to search the list for
2586 			 * dtrace_msgdsize_max mblks.
2587 			 */
2588 			if (cont++ > dtrace_msgdsize_max) {
2589 				*flags |= CPU_DTRACE_ILLOP;
2590 				break;
2591 			}
2592 
2593 			if (subr == DIF_SUBR_MSGDSIZE) {
2594 				if (dtrace_load8(daddr +
2595 				    offsetof(dblk_t, db_type)) != M_DATA)
2596 					continue;
2597 			}
2598 
2599 			count += wptr - rptr;
2600 		}
2601 
2602 		if (!(*flags & CPU_DTRACE_FAULT))
2603 			regs[rd] = count;
2604 
2605 		break;
2606 	}
2607 
2608 	case DIF_SUBR_PROGENYOF: {
2609 		pid_t pid = tupregs[0].dttk_value;
2610 		proc_t *p;
2611 		int rval = 0;
2612 
2613 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2614 
2615 		for (p = curthread->t_procp; p != NULL; p = p->p_parent) {
2616 			if (p->p_pidp->pid_id == pid) {
2617 				rval = 1;
2618 				break;
2619 			}
2620 		}
2621 
2622 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
2623 
2624 		regs[rd] = rval;
2625 		break;
2626 	}
2627 
2628 	case DIF_SUBR_SPECULATION:
2629 		regs[rd] = dtrace_speculation(state);
2630 		break;
2631 
2632 	case DIF_SUBR_COPYOUT: {
2633 		uintptr_t kaddr = tupregs[0].dttk_value;
2634 		uintptr_t uaddr = tupregs[1].dttk_value;
2635 		uint64_t size = tupregs[2].dttk_value;
2636 
2637 		if (!dtrace_destructive_disallow &&
2638 		    dtrace_priv_proc_control(state) &&
2639 		    !dtrace_istoxic(kaddr, size)) {
2640 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2641 			dtrace_copyout(kaddr, uaddr, size);
2642 			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
2643 		}
2644 		break;
2645 	}
2646 
2647 	case DIF_SUBR_COPYOUTSTR: {
2648 		uintptr_t kaddr = tupregs[0].dttk_value;
2649 		uintptr_t uaddr = tupregs[1].dttk_value;
2650 		uint64_t size = tupregs[2].dttk_value;
2651 
2652 		if (!dtrace_destructive_disallow &&
2653 		    dtrace_priv_proc_control(state) &&
2654 		    !dtrace_istoxic(kaddr, size)) {
2655 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2656 			dtrace_copyoutstr(kaddr, uaddr, size);
2657 			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
2658 		}
2659 		break;
2660 	}
2661 
2662 	case DIF_SUBR_STRLEN:
2663 		regs[rd] = dtrace_strlen((char *)(uintptr_t)
2664 		    tupregs[0].dttk_value,
2665 		    state->dts_options[DTRACEOPT_STRSIZE]);
2666 		break;
2667 
2668 	case DIF_SUBR_STRCHR:
2669 	case DIF_SUBR_STRRCHR: {
2670 		/*
2671 		 * We're going to iterate over the string looking for the
2672 		 * specified character.  We will iterate until we have reached
2673 		 * the string length or we have found the character.  If this
2674 		 * is DIF_SUBR_STRRCHR, we will look for the last occurrence
2675 		 * of the specified character instead of the first.
2676 		 */
2677 		uintptr_t addr = tupregs[0].dttk_value;
2678 		uintptr_t limit = addr + state->dts_options[DTRACEOPT_STRSIZE];
2679 		char c, target = (char)tupregs[1].dttk_value;
2680 
2681 		for (regs[rd] = NULL; addr < limit; addr++) {
2682 			if ((c = dtrace_load8(addr)) == target) {
2683 				regs[rd] = addr;
2684 
2685 				if (subr == DIF_SUBR_STRCHR)
2686 					break;
2687 			}
2688 
2689 			if (c == '\0')
2690 				break;
2691 		}
2692 
2693 		break;
2694 	}
2695 
2696 	case DIF_SUBR_STRSTR:
2697 	case DIF_SUBR_INDEX:
2698 	case DIF_SUBR_RINDEX: {
2699 		/*
2700 		 * We're going to iterate over the string looking for the
2701 		 * specified string.  We will iterate until we have reached
2702 		 * the string length or we have found the string.  (Yes, this
2703 		 * is done in the most naive way possible -- but considering
2704 		 * that the string we're searching for is likely to be
2705 		 * relatively short, the complexity of Rabin-Karp or similar
2706 		 * hardly seems merited.)
2707 		 */
2708 		char *addr = (char *)(uintptr_t)tupregs[0].dttk_value;
2709 		char *substr = (char *)(uintptr_t)tupregs[1].dttk_value;
2710 		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
2711 		size_t len = dtrace_strlen(addr, size);
2712 		size_t sublen = dtrace_strlen(substr, size);
2713 		char *limit = addr + len, *orig = addr;
2714 		int notfound = subr == DIF_SUBR_STRSTR ? 0 : -1;
2715 		int inc = 1;
2716 
2717 		regs[rd] = notfound;
2718 
2719 		/*
2720 		 * strstr() and index()/rindex() have similar semantics if
2721 		 * both strings are the empty string: strstr() returns a
2722 		 * pointer to the (empty) string, and index() and rindex()
2723 		 * both return index 0 (regardless of any position argument).
2724 		 */
2725 		if (sublen == 0 && len == 0) {
2726 			if (subr == DIF_SUBR_STRSTR)
2727 				regs[rd] = (uintptr_t)addr;
2728 			else
2729 				regs[rd] = 0;
2730 			break;
2731 		}
2732 
2733 		if (subr != DIF_SUBR_STRSTR) {
2734 			if (subr == DIF_SUBR_RINDEX) {
2735 				limit = orig - 1;
2736 				addr += len;
2737 				inc = -1;
2738 			}
2739 
2740 			/*
2741 			 * Both index() and rindex() take an optional position
2742 			 * argument that denotes the starting position.
2743 			 */
2744 			if (nargs == 3) {
2745 				int64_t pos = (int64_t)tupregs[2].dttk_value;
2746 
2747 				/*
2748 				 * If the position argument to index() is
2749 				 * negative, Perl implicitly clamps it at
2750 				 * zero.  This semantic is a little surprising
2751 				 * given the special meaning of negative
2752 				 * positions to similar Perl functions like
2753 				 * substr(), but it appears to reflect a
2754 				 * notion that index() can start from a
2755 				 * negative index and increment its way up to
2756 				 * the string.  Given this notion, Perl's
2757 				 * rindex() is at least self-consistent in
2758 				 * that it implicitly clamps positions greater
2759 				 * than the string length to be the string
2760 				 * length.  Where Perl completely loses
2761 				 * coherence, however, is when the specified
2762 				 * substring is the empty string ("").  In
2763 				 * this case, even if the position is
2764 				 * negative, rindex() returns 0 -- and even if
2765 				 * the position is greater than the length,
2766 				 * index() returns the string length.  These
2767 				 * semantics violate the notion that index()
2768 				 * should never return a value less than the
2769 				 * specified position and that rindex() should
2770 				 * never return a value greater than the
2771 				 * specified position.  (One assumes that
2772 				 * these semantics are artifacts of Perl's
2773 				 * implementation and not the results of
2774 				 * deliberate design -- it beggars belief that
2775 				 * even Larry Wall could desire such oddness.)
2776 				 * While in the abstract one would wish for
2777 				 * consistent position semantics across
2778 				 * substr(), index() and rindex() -- or at the
2779 				 * very least self-consistent position
2780 				 * semantics for index() and rindex() -- we
2781 				 * instead opt to keep with the extant Perl
2782 				 * semantics, in all their broken glory.  (Do
2783 				 * we have more desire to maintain Perl's
2784 				 * semantics than Perl does?  Probably.)
2785 				 */
2786 				if (subr == DIF_SUBR_RINDEX) {
2787 					if (pos < 0) {
2788 						if (sublen == 0)
2789 							regs[rd] = 0;
2790 						break;
2791 					}
2792 
2793 					if (pos > len)
2794 						pos = len;
2795 				} else {
2796 					if (pos < 0)
2797 						pos = 0;
2798 
2799 					if (pos >= len) {
2800 						if (sublen == 0)
2801 							regs[rd] = len;
2802 						break;
2803 					}
2804 				}
2805 
2806 				addr = orig + pos;
2807 			}
2808 		}
2809 
2810 		for (regs[rd] = notfound; addr != limit; addr += inc) {
2811 			if (dtrace_strncmp(addr, substr, sublen) == 0) {
2812 				if (subr != DIF_SUBR_STRSTR) {
2813 					/*
2814 					 * As D index() and rindex() are
2815 					 * modeled on Perl (and not on awk),
2816 					 * we return a zero-based (and not a
2817 					 * one-based) index.  (For you Perl
2818 					 * weenies: no, we're not going to add
2819 					 * $[ -- and shouldn't you be at a con
2820 					 * or something?)
2821 					 */
2822 					regs[rd] = (uintptr_t)(addr - orig);
2823 					break;
2824 				}
2825 
2826 				ASSERT(subr == DIF_SUBR_STRSTR);
2827 				regs[rd] = (uintptr_t)addr;
2828 				break;
2829 			}
2830 		}
2831 
2832 		break;
2833 	}
2834 
2835 	case DIF_SUBR_STRTOK: {
2836 		uintptr_t addr = tupregs[0].dttk_value;
2837 		uintptr_t tokaddr = tupregs[1].dttk_value;
2838 		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
2839 		uintptr_t limit, toklimit = tokaddr + size;
2840 		uint8_t c, tokmap[32];	 /* 256 / 8 */
2841 		char *dest = (char *)mstate->dtms_scratch_ptr;
2842 		int i;
2843 
2844 		if (mstate->dtms_scratch_ptr + size >
2845 		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
2846 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
2847 			regs[rd] = NULL;
2848 			break;
2849 		}
2850 
2851 		if (addr == NULL) {
2852 			/*
2853 			 * If the address specified is NULL, we use our saved
2854 			 * strtok pointer from the mstate.  Note that this
2855 			 * means that the saved strtok pointer is _only_
2856 			 * valid within multiple enablings of the same probe --
2857 			 * it behaves like an implicit clause-local variable.
2858 			 */
2859 			addr = mstate->dtms_strtok;
2860 		}
2861 
2862 		/*
2863 		 * First, zero the token map, and then process the token
2864 		 * string -- setting a bit in the map for every character
2865 		 * found in the token string.
2866 		 */
2867 		for (i = 0; i < sizeof (tokmap); i++)
2868 			tokmap[i] = 0;
2869 
2870 		for (; tokaddr < toklimit; tokaddr++) {
2871 			if ((c = dtrace_load8(tokaddr)) == '\0')
2872 				break;
2873 
2874 			ASSERT((c >> 3) < sizeof (tokmap));
2875 			tokmap[c >> 3] |= (1 << (c & 0x7));
2876 		}
2877 
2878 		for (limit = addr + size; addr < limit; addr++) {
2879 			/*
2880 			 * We're looking for a character that is _not_ contained
2881 			 * in the token string.
2882 			 */
2883 			if ((c = dtrace_load8(addr)) == '\0')
2884 				break;
2885 
2886 			if (!(tokmap[c >> 3] & (1 << (c & 0x7))))
2887 				break;
2888 		}
2889 
2890 		if (c == '\0') {
2891 			/*
2892 			 * We reached the end of the string without finding
2893 			 * any character that was not in the token string.
2894 			 * We return NULL in this case, and we set the saved
2895 			 * address to NULL as well.
2896 			 */
2897 			regs[rd] = NULL;
2898 			mstate->dtms_strtok = NULL;
2899 			break;
2900 		}
2901 
2902 		/*
2903 		 * From here on, we're copying into the destination string.
2904 		 */
2905 		for (i = 0; addr < limit && i < size - 1; addr++) {
2906 			if ((c = dtrace_load8(addr)) == '\0')
2907 				break;
2908 
2909 			if (tokmap[c >> 3] & (1 << (c & 0x7)))
2910 				break;
2911 
2912 			ASSERT(i < size);
2913 			dest[i++] = c;
2914 		}
2915 
2916 		ASSERT(i < size);
2917 		dest[i] = '\0';
2918 		regs[rd] = (uintptr_t)dest;
2919 		mstate->dtms_scratch_ptr += size;
2920 		mstate->dtms_strtok = addr;
2921 		break;
2922 	}
2923 
2924 	case DIF_SUBR_SUBSTR: {
2925 		uintptr_t s = tupregs[0].dttk_value;
2926 		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
2927 		char *d = (char *)mstate->dtms_scratch_ptr;
2928 		int64_t index = (int64_t)tupregs[1].dttk_value;
2929 		int64_t remaining = (int64_t)tupregs[2].dttk_value;
2930 		size_t len = dtrace_strlen((char *)s, size);
2931 		int64_t i = 0;
2932 
2933 		if (nargs <= 2)
2934 			remaining = (int64_t)size;
2935 
2936 		if (mstate->dtms_scratch_ptr + size >
2937 		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
2938 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
2939 			regs[rd] = NULL;
2940 			break;
2941 		}
2942 
2943 		if (index < 0) {
2944 			index += len;
2945 
2946 			if (index < 0 && index + remaining > 0) {
2947 				remaining += index;
2948 				index = 0;
2949 			}
2950 		}
2951 
2952 		if (index >= len || index < 0)
2953 			index = len;
2954 
2955 		for (d[0] = '\0'; remaining > 0; remaining--) {
2956 			if ((d[i++] = dtrace_load8(s++ + index)) == '\0')
2957 				break;
2958 
2959 			if (i == size) {
2960 				d[i - 1] = '\0';
2961 				break;
2962 			}
2963 		}
2964 
2965 		mstate->dtms_scratch_ptr += size;
2966 		regs[rd] = (uintptr_t)d;
2967 		break;
2968 	}
2969 
2970 	case DIF_SUBR_GETMAJOR:
2971 #ifdef _LP64
2972 		regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR64) & MAXMAJ64;
2973 #else
2974 		regs[rd] = (tupregs[0].dttk_value >> NBITSMINOR) & MAXMAJ;
2975 #endif
2976 		break;
2977 
2978 	case DIF_SUBR_GETMINOR:
2979 #ifdef _LP64
2980 		regs[rd] = tupregs[0].dttk_value & MAXMIN64;
2981 #else
2982 		regs[rd] = tupregs[0].dttk_value & MAXMIN;
2983 #endif
2984 		break;
2985 
2986 	case DIF_SUBR_DDI_PATHNAME: {
2987 		/*
2988 		 * This one is a galactic mess.  We are going to roughly
2989 		 * emulate ddi_pathname(), but it's made more complicated
2990 		 * by the fact that we (a) want to include the minor name and
2991 		 * (b) must proceed iteratively instead of recursively.
2992 		 */
2993 		uintptr_t dest = mstate->dtms_scratch_ptr;
2994 		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
2995 		char *start = (char *)dest, *end = start + size - 1;
2996 		uintptr_t daddr = tupregs[0].dttk_value;
2997 		int64_t minor = (int64_t)tupregs[1].dttk_value;
2998 		char *s;
2999 		int i, len, depth = 0;
3000 
3001 		if (size == 0 || mstate->dtms_scratch_ptr + size >
3002 		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
3003 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3004 			regs[rd] = NULL;
3005 			break;
3006 		}
3007 
3008 		*end = '\0';
3009 
3010 		/*
3011 		 * We want to have a name for the minor.  In order to do this,
3012 		 * we need to walk the minor list from the devinfo.  We want
3013 		 * to be sure that we don't infinitely walk a circular list,
3014 		 * so we check for circularity by sending a scout pointer
3015 		 * ahead two elements for every element that we iterate over;
3016 		 * if the list is circular, these will ultimately point to the
3017 		 * same element.  You may recognize this little trick as the
3018 		 * answer to a stupid interview question -- one that always
3019 		 * seems to be asked by those who had to have it laboriously
3020 		 * explained to them, and who can't even concisely describe
3021 		 * the conditions under which one would be forced to resort to
3022 		 * this technique.  Needless to say, those conditions are
3023 		 * found here -- and probably only here.  Is this is the only
3024 		 * use of this infamous trick in shipping, production code?
3025 		 * If it isn't, it probably should be...
3026 		 */
3027 		if (minor != -1) {
3028 			uintptr_t maddr = dtrace_loadptr(daddr +
3029 			    offsetof(struct dev_info, devi_minor));
3030 
3031 			uintptr_t next = offsetof(struct ddi_minor_data, next);
3032 			uintptr_t name = offsetof(struct ddi_minor_data,
3033 			    d_minor) + offsetof(struct ddi_minor, name);
3034 			uintptr_t dev = offsetof(struct ddi_minor_data,
3035 			    d_minor) + offsetof(struct ddi_minor, dev);
3036 			uintptr_t scout;
3037 
3038 			if (maddr != NULL)
3039 				scout = dtrace_loadptr(maddr + next);
3040 
3041 			while (maddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
3042 				uint64_t m;
3043 #ifdef _LP64
3044 				m = dtrace_load64(maddr + dev) & MAXMIN64;
3045 #else
3046 				m = dtrace_load32(maddr + dev) & MAXMIN;
3047 #endif
3048 				if (m != minor) {
3049 					maddr = dtrace_loadptr(maddr + next);
3050 
3051 					if (scout == NULL)
3052 						continue;
3053 
3054 					scout = dtrace_loadptr(scout + next);
3055 
3056 					if (scout == NULL)
3057 						continue;
3058 
3059 					scout = dtrace_loadptr(scout + next);
3060 
3061 					if (scout == NULL)
3062 						continue;
3063 
3064 					if (scout == maddr) {
3065 						*flags |= CPU_DTRACE_ILLOP;
3066 						break;
3067 					}
3068 
3069 					continue;
3070 				}
3071 
3072 				/*
3073 				 * We have the minor data.  Now we need to
3074 				 * copy the minor's name into the end of the
3075 				 * pathname.
3076 				 */
3077 				s = (char *)dtrace_loadptr(maddr + name);
3078 				len = dtrace_strlen(s, size);
3079 
3080 				if (*flags & CPU_DTRACE_FAULT)
3081 					break;
3082 
3083 				if (len != 0) {
3084 					if ((end -= (len + 1)) < start)
3085 						break;
3086 
3087 					*end = ':';
3088 				}
3089 
3090 				for (i = 1; i <= len; i++)
3091 					end[i] = dtrace_load8((uintptr_t)s++);
3092 				break;
3093 			}
3094 		}
3095 
3096 		while (daddr != NULL && !(*flags & CPU_DTRACE_FAULT)) {
3097 			ddi_node_state_t devi_state;
3098 
3099 			devi_state = dtrace_load32(daddr +
3100 			    offsetof(struct dev_info, devi_node_state));
3101 
3102 			if (*flags & CPU_DTRACE_FAULT)
3103 				break;
3104 
3105 			if (devi_state >= DS_INITIALIZED) {
3106 				s = (char *)dtrace_loadptr(daddr +
3107 				    offsetof(struct dev_info, devi_addr));
3108 				len = dtrace_strlen(s, size);
3109 
3110 				if (*flags & CPU_DTRACE_FAULT)
3111 					break;
3112 
3113 				if (len != 0) {
3114 					if ((end -= (len + 1)) < start)
3115 						break;
3116 
3117 					*end = '@';
3118 				}
3119 
3120 				for (i = 1; i <= len; i++)
3121 					end[i] = dtrace_load8((uintptr_t)s++);
3122 			}
3123 
3124 			/*
3125 			 * Now for the node name...
3126 			 */
3127 			s = (char *)dtrace_loadptr(daddr +
3128 			    offsetof(struct dev_info, devi_node_name));
3129 
3130 			daddr = dtrace_loadptr(daddr +
3131 			    offsetof(struct dev_info, devi_parent));
3132 
3133 			/*
3134 			 * If our parent is NULL (that is, if we're the root
3135 			 * node), we're going to use the special path
3136 			 * "devices".
3137 			 */
3138 			if (daddr == NULL)
3139 				s = "devices";
3140 
3141 			len = dtrace_strlen(s, size);
3142 			if (*flags & CPU_DTRACE_FAULT)
3143 				break;
3144 
3145 			if ((end -= (len + 1)) < start)
3146 				break;
3147 
3148 			for (i = 1; i <= len; i++)
3149 				end[i] = dtrace_load8((uintptr_t)s++);
3150 			*end = '/';
3151 
3152 			if (depth++ > dtrace_devdepth_max) {
3153 				*flags |= CPU_DTRACE_ILLOP;
3154 				break;
3155 			}
3156 		}
3157 
3158 		if (end < start)
3159 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3160 
3161 		if (daddr == NULL) {
3162 			regs[rd] = (uintptr_t)end;
3163 			mstate->dtms_scratch_ptr += size;
3164 		}
3165 
3166 		break;
3167 	}
3168 
3169 	case DIF_SUBR_STRJOIN: {
3170 		char *d = (char *)mstate->dtms_scratch_ptr;
3171 		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3172 		uintptr_t s1 = tupregs[0].dttk_value;
3173 		uintptr_t s2 = tupregs[1].dttk_value;
3174 		int i = 0;
3175 
3176 		if (mstate->dtms_scratch_ptr + size >
3177 		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
3178 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3179 			regs[rd] = NULL;
3180 			break;
3181 		}
3182 
3183 		for (;;) {
3184 			if (i >= size) {
3185 				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3186 				regs[rd] = NULL;
3187 				break;
3188 			}
3189 
3190 			if ((d[i++] = dtrace_load8(s1++)) == '\0') {
3191 				i--;
3192 				break;
3193 			}
3194 		}
3195 
3196 		for (;;) {
3197 			if (i >= size) {
3198 				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3199 				regs[rd] = NULL;
3200 				break;
3201 			}
3202 
3203 			if ((d[i++] = dtrace_load8(s2++)) == '\0')
3204 				break;
3205 		}
3206 
3207 		if (i < size) {
3208 			mstate->dtms_scratch_ptr += i;
3209 			regs[rd] = (uintptr_t)d;
3210 		}
3211 
3212 		break;
3213 	}
3214 
3215 	case DIF_SUBR_LLTOSTR: {
3216 		int64_t i = (int64_t)tupregs[0].dttk_value;
3217 		int64_t val = i < 0 ? i * -1 : i;
3218 		uint64_t size = 22;	/* enough room for 2^64 in decimal */
3219 		char *end = (char *)mstate->dtms_scratch_ptr + size - 1;
3220 
3221 		if (mstate->dtms_scratch_ptr + size >
3222 		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
3223 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3224 			regs[rd] = NULL;
3225 			break;
3226 		}
3227 
3228 		for (*end-- = '\0'; val; val /= 10)
3229 			*end-- = '0' + (val % 10);
3230 
3231 		if (i == 0)
3232 			*end-- = '0';
3233 
3234 		if (i < 0)
3235 			*end-- = '-';
3236 
3237 		regs[rd] = (uintptr_t)end + 1;
3238 		mstate->dtms_scratch_ptr += size;
3239 		break;
3240 	}
3241 
3242 	case DIF_SUBR_DIRNAME:
3243 	case DIF_SUBR_BASENAME: {
3244 		char *dest = (char *)mstate->dtms_scratch_ptr;
3245 		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3246 		uintptr_t src = tupregs[0].dttk_value;
3247 		int i, j, len = dtrace_strlen((char *)src, size);
3248 		int lastbase = -1, firstbase = -1, lastdir = -1;
3249 		int start, end;
3250 
3251 		if (mstate->dtms_scratch_ptr + size >
3252 		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
3253 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3254 			regs[rd] = NULL;
3255 			break;
3256 		}
3257 
3258 		/*
3259 		 * The basename and dirname for a zero-length string is
3260 		 * defined to be "."
3261 		 */
3262 		if (len == 0) {
3263 			len = 1;
3264 			src = (uintptr_t)".";
3265 		}
3266 
3267 		/*
3268 		 * Start from the back of the string, moving back toward the
3269 		 * front until we see a character that isn't a slash.  That
3270 		 * character is the last character in the basename.
3271 		 */
3272 		for (i = len - 1; i >= 0; i--) {
3273 			if (dtrace_load8(src + i) != '/')
3274 				break;
3275 		}
3276 
3277 		if (i >= 0)
3278 			lastbase = i;
3279 
3280 		/*
3281 		 * Starting from the last character in the basename, move
3282 		 * towards the front until we find a slash.  The character
3283 		 * that we processed immediately before that is the first
3284 		 * character in the basename.
3285 		 */
3286 		for (; i >= 0; i--) {
3287 			if (dtrace_load8(src + i) == '/')
3288 				break;
3289 		}
3290 
3291 		if (i >= 0)
3292 			firstbase = i + 1;
3293 
3294 		/*
3295 		 * Now keep going until we find a non-slash character.  That
3296 		 * character is the last character in the dirname.
3297 		 */
3298 		for (; i >= 0; i--) {
3299 			if (dtrace_load8(src + i) != '/')
3300 				break;
3301 		}
3302 
3303 		if (i >= 0)
3304 			lastdir = i;
3305 
3306 		ASSERT(!(lastbase == -1 && firstbase != -1));
3307 		ASSERT(!(firstbase == -1 && lastdir != -1));
3308 
3309 		if (lastbase == -1) {
3310 			/*
3311 			 * We didn't find a non-slash character.  We know that
3312 			 * the length is non-zero, so the whole string must be
3313 			 * slashes.  In either the dirname or the basename
3314 			 * case, we return '/'.
3315 			 */
3316 			ASSERT(firstbase == -1);
3317 			firstbase = lastbase = lastdir = 0;
3318 		}
3319 
3320 		if (firstbase == -1) {
3321 			/*
3322 			 * The entire string consists only of a basename
3323 			 * component.  If we're looking for dirname, we need
3324 			 * to change our string to be just "."; if we're
3325 			 * looking for a basename, we'll just set the first
3326 			 * character of the basename to be 0.
3327 			 */
3328 			if (subr == DIF_SUBR_DIRNAME) {
3329 				ASSERT(lastdir == -1);
3330 				src = (uintptr_t)".";
3331 				lastdir = 0;
3332 			} else {
3333 				firstbase = 0;
3334 			}
3335 		}
3336 
3337 		if (subr == DIF_SUBR_DIRNAME) {
3338 			if (lastdir == -1) {
3339 				/*
3340 				 * We know that we have a slash in the name --
3341 				 * or lastdir would be set to 0, above.  And
3342 				 * because lastdir is -1, we know that this
3343 				 * slash must be the first character.  (That
3344 				 * is, the full string must be of the form
3345 				 * "/basename".)  In this case, the last
3346 				 * character of the directory name is 0.
3347 				 */
3348 				lastdir = 0;
3349 			}
3350 
3351 			start = 0;
3352 			end = lastdir;
3353 		} else {
3354 			ASSERT(subr == DIF_SUBR_BASENAME);
3355 			ASSERT(firstbase != -1 && lastbase != -1);
3356 			start = firstbase;
3357 			end = lastbase;
3358 		}
3359 
3360 		for (i = start, j = 0; i <= end && j < size - 1; i++, j++)
3361 			dest[j] = dtrace_load8(src + i);
3362 
3363 		dest[j] = '\0';
3364 		regs[rd] = (uintptr_t)dest;
3365 		mstate->dtms_scratch_ptr += size;
3366 		break;
3367 	}
3368 
3369 	case DIF_SUBR_CLEANPATH: {
3370 		char *dest = (char *)mstate->dtms_scratch_ptr, c;
3371 		uint64_t size = state->dts_options[DTRACEOPT_STRSIZE];
3372 		uintptr_t src = tupregs[0].dttk_value;
3373 		int i = 0, j = 0;
3374 
3375 		if (mstate->dtms_scratch_ptr + size >
3376 		    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
3377 			DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
3378 			regs[rd] = NULL;
3379 			break;
3380 		}
3381 
3382 		/*
3383 		 * Move forward, loading each character.
3384 		 */
3385 		do {
3386 			c = dtrace_load8(src + i++);
3387 next:
3388 			if (j + 5 >= size)	/* 5 = strlen("/..c\0") */
3389 				break;
3390 
3391 			if (c != '/') {
3392 				dest[j++] = c;
3393 				continue;
3394 			}
3395 
3396 			c = dtrace_load8(src + i++);
3397 
3398 			if (c == '/') {
3399 				/*
3400 				 * We have two slashes -- we can just advance
3401 				 * to the next character.
3402 				 */
3403 				goto next;
3404 			}
3405 
3406 			if (c != '.') {
3407 				/*
3408 				 * This is not "." and it's not ".." -- we can
3409 				 * just store the "/" and this character and
3410 				 * drive on.
3411 				 */
3412 				dest[j++] = '/';
3413 				dest[j++] = c;
3414 				continue;
3415 			}
3416 
3417 			c = dtrace_load8(src + i++);
3418 
3419 			if (c == '/') {
3420 				/*
3421 				 * This is a "/./" component.  We're not going
3422 				 * to store anything in the destination buffer;
3423 				 * we're just going to go to the next component.
3424 				 */
3425 				goto next;
3426 			}
3427 
3428 			if (c != '.') {
3429 				/*
3430 				 * This is not ".." -- we can just store the
3431 				 * "/." and this character and continue
3432 				 * processing.
3433 				 */
3434 				dest[j++] = '/';
3435 				dest[j++] = '.';
3436 				dest[j++] = c;
3437 				continue;
3438 			}
3439 
3440 			c = dtrace_load8(src + i++);
3441 
3442 			if (c != '/' && c != '\0') {
3443 				/*
3444 				 * This is not ".." -- it's "..[mumble]".
3445 				 * We'll store the "/.." and this character
3446 				 * and continue processing.
3447 				 */
3448 				dest[j++] = '/';
3449 				dest[j++] = '.';
3450 				dest[j++] = '.';
3451 				dest[j++] = c;
3452 				continue;
3453 			}
3454 
3455 			/*
3456 			 * This is "/../" or "/..\0".  We need to back up
3457 			 * our destination pointer until we find a "/".
3458 			 */
3459 			i--;
3460 			while (j != 0 && dest[--j] != '/')
3461 				continue;
3462 
3463 			if (c == '\0')
3464 				dest[++j] = '/';
3465 		} while (c != '\0');
3466 
3467 		dest[j] = '\0';
3468 		regs[rd] = (uintptr_t)dest;
3469 		mstate->dtms_scratch_ptr += size;
3470 		break;
3471 	}
3472 	}
3473 }
3474 
3475 /*
3476  * Emulate the execution of DTrace IR instructions specified by the given
3477  * DIF object.  This function is deliberately void of assertions as all of
3478  * the necessary checks are handled by a call to dtrace_difo_validate().
3479  */
3480 static uint64_t
3481 dtrace_dif_emulate(dtrace_difo_t *difo, dtrace_mstate_t *mstate,
3482     dtrace_vstate_t *vstate, dtrace_state_t *state)
3483 {
3484 	const dif_instr_t *text = difo->dtdo_buf;
3485 	const uint_t textlen = difo->dtdo_len;
3486 	const char *strtab = difo->dtdo_strtab;
3487 	const uint64_t *inttab = difo->dtdo_inttab;
3488 
3489 	uint64_t rval = 0;
3490 	dtrace_statvar_t *svar;
3491 	dtrace_dstate_t *dstate = &vstate->dtvs_dynvars;
3492 	dtrace_difv_t *v;
3493 	volatile uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
3494 	volatile uintptr_t *illval = &cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
3495 
3496 	dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
3497 	uint64_t regs[DIF_DIR_NREGS];
3498 	uint64_t *tmp;
3499 
3500 	uint8_t cc_n = 0, cc_z = 0, cc_v = 0, cc_c = 0;
3501 	int64_t cc_r;
3502 	uint_t pc = 0, id, opc;
3503 	uint8_t ttop = 0;
3504 	dif_instr_t instr;
3505 	uint_t r1, r2, rd;
3506 
3507 	regs[DIF_REG_R0] = 0; 		/* %r0 is fixed at zero */
3508 
3509 	while (pc < textlen && !(*flags & CPU_DTRACE_FAULT)) {
3510 		opc = pc;
3511 
3512 		instr = text[pc++];
3513 		r1 = DIF_INSTR_R1(instr);
3514 		r2 = DIF_INSTR_R2(instr);
3515 		rd = DIF_INSTR_RD(instr);
3516 
3517 		switch (DIF_INSTR_OP(instr)) {
3518 		case DIF_OP_OR:
3519 			regs[rd] = regs[r1] | regs[r2];
3520 			break;
3521 		case DIF_OP_XOR:
3522 			regs[rd] = regs[r1] ^ regs[r2];
3523 			break;
3524 		case DIF_OP_AND:
3525 			regs[rd] = regs[r1] & regs[r2];
3526 			break;
3527 		case DIF_OP_SLL:
3528 			regs[rd] = regs[r1] << regs[r2];
3529 			break;
3530 		case DIF_OP_SRL:
3531 			regs[rd] = regs[r1] >> regs[r2];
3532 			break;
3533 		case DIF_OP_SUB:
3534 			regs[rd] = regs[r1] - regs[r2];
3535 			break;
3536 		case DIF_OP_ADD:
3537 			regs[rd] = regs[r1] + regs[r2];
3538 			break;
3539 		case DIF_OP_MUL:
3540 			regs[rd] = regs[r1] * regs[r2];
3541 			break;
3542 		case DIF_OP_SDIV:
3543 			if (regs[r2] == 0) {
3544 				regs[rd] = 0;
3545 				*flags |= CPU_DTRACE_DIVZERO;
3546 			} else {
3547 				regs[rd] = (int64_t)regs[r1] /
3548 				    (int64_t)regs[r2];
3549 			}
3550 			break;
3551 
3552 		case DIF_OP_UDIV:
3553 			if (regs[r2] == 0) {
3554 				regs[rd] = 0;
3555 				*flags |= CPU_DTRACE_DIVZERO;
3556 			} else {
3557 				regs[rd] = regs[r1] / regs[r2];
3558 			}
3559 			break;
3560 
3561 		case DIF_OP_SREM:
3562 			if (regs[r2] == 0) {
3563 				regs[rd] = 0;
3564 				*flags |= CPU_DTRACE_DIVZERO;
3565 			} else {
3566 				regs[rd] = (int64_t)regs[r1] %
3567 				    (int64_t)regs[r2];
3568 			}
3569 			break;
3570 
3571 		case DIF_OP_UREM:
3572 			if (regs[r2] == 0) {
3573 				regs[rd] = 0;
3574 				*flags |= CPU_DTRACE_DIVZERO;
3575 			} else {
3576 				regs[rd] = regs[r1] % regs[r2];
3577 			}
3578 			break;
3579 
3580 		case DIF_OP_NOT:
3581 			regs[rd] = ~regs[r1];
3582 			break;
3583 		case DIF_OP_MOV:
3584 			regs[rd] = regs[r1];
3585 			break;
3586 		case DIF_OP_CMP:
3587 			cc_r = regs[r1] - regs[r2];
3588 			cc_n = cc_r < 0;
3589 			cc_z = cc_r == 0;
3590 			cc_v = 0;
3591 			cc_c = regs[r1] < regs[r2];
3592 			break;
3593 		case DIF_OP_TST:
3594 			cc_n = cc_v = cc_c = 0;
3595 			cc_z = regs[r1] == 0;
3596 			break;
3597 		case DIF_OP_BA:
3598 			pc = DIF_INSTR_LABEL(instr);
3599 			break;
3600 		case DIF_OP_BE:
3601 			if (cc_z)
3602 				pc = DIF_INSTR_LABEL(instr);
3603 			break;
3604 		case DIF_OP_BNE:
3605 			if (cc_z == 0)
3606 				pc = DIF_INSTR_LABEL(instr);
3607 			break;
3608 		case DIF_OP_BG:
3609 			if ((cc_z | (cc_n ^ cc_v)) == 0)
3610 				pc = DIF_INSTR_LABEL(instr);
3611 			break;
3612 		case DIF_OP_BGU:
3613 			if ((cc_c | cc_z) == 0)
3614 				pc = DIF_INSTR_LABEL(instr);
3615 			break;
3616 		case DIF_OP_BGE:
3617 			if ((cc_n ^ cc_v) == 0)
3618 				pc = DIF_INSTR_LABEL(instr);
3619 			break;
3620 		case DIF_OP_BGEU:
3621 			if (cc_c == 0)
3622 				pc = DIF_INSTR_LABEL(instr);
3623 			break;
3624 		case DIF_OP_BL:
3625 			if (cc_n ^ cc_v)
3626 				pc = DIF_INSTR_LABEL(instr);
3627 			break;
3628 		case DIF_OP_BLU:
3629 			if (cc_c)
3630 				pc = DIF_INSTR_LABEL(instr);
3631 			break;
3632 		case DIF_OP_BLE:
3633 			if (cc_z | (cc_n ^ cc_v))
3634 				pc = DIF_INSTR_LABEL(instr);
3635 			break;
3636 		case DIF_OP_BLEU:
3637 			if (cc_c | cc_z)
3638 				pc = DIF_INSTR_LABEL(instr);
3639 			break;
3640 		case DIF_OP_RLDSB:
3641 			if (!dtrace_canstore(regs[r1], 1, mstate, vstate)) {
3642 				*flags |= CPU_DTRACE_KPRIV;
3643 				*illval = regs[r1];
3644 				break;
3645 			}
3646 			/*FALLTHROUGH*/
3647 		case DIF_OP_LDSB:
3648 			regs[rd] = (int8_t)dtrace_load8(regs[r1]);
3649 			break;
3650 		case DIF_OP_RLDSH:
3651 			if (!dtrace_canstore(regs[r1], 2, mstate, vstate)) {
3652 				*flags |= CPU_DTRACE_KPRIV;
3653 				*illval = regs[r1];
3654 				break;
3655 			}
3656 			/*FALLTHROUGH*/
3657 		case DIF_OP_LDSH:
3658 			regs[rd] = (int16_t)dtrace_load16(regs[r1]);
3659 			break;
3660 		case DIF_OP_RLDSW:
3661 			if (!dtrace_canstore(regs[r1], 4, mstate, vstate)) {
3662 				*flags |= CPU_DTRACE_KPRIV;
3663 				*illval = regs[r1];
3664 				break;
3665 			}
3666 			/*FALLTHROUGH*/
3667 		case DIF_OP_LDSW:
3668 			regs[rd] = (int32_t)dtrace_load32(regs[r1]);
3669 			break;
3670 		case DIF_OP_RLDUB:
3671 			if (!dtrace_canstore(regs[r1], 1, mstate, vstate)) {
3672 				*flags |= CPU_DTRACE_KPRIV;
3673 				*illval = regs[r1];
3674 				break;
3675 			}
3676 			/*FALLTHROUGH*/
3677 		case DIF_OP_LDUB:
3678 			regs[rd] = dtrace_load8(regs[r1]);
3679 			break;
3680 		case DIF_OP_RLDUH:
3681 			if (!dtrace_canstore(regs[r1], 2, mstate, vstate)) {
3682 				*flags |= CPU_DTRACE_KPRIV;
3683 				*illval = regs[r1];
3684 				break;
3685 			}
3686 			/*FALLTHROUGH*/
3687 		case DIF_OP_LDUH:
3688 			regs[rd] = dtrace_load16(regs[r1]);
3689 			break;
3690 		case DIF_OP_RLDUW:
3691 			if (!dtrace_canstore(regs[r1], 4, mstate, vstate)) {
3692 				*flags |= CPU_DTRACE_KPRIV;
3693 				*illval = regs[r1];
3694 				break;
3695 			}
3696 			/*FALLTHROUGH*/
3697 		case DIF_OP_LDUW:
3698 			regs[rd] = dtrace_load32(regs[r1]);
3699 			break;
3700 		case DIF_OP_RLDX:
3701 			if (!dtrace_canstore(regs[r1], 8, mstate, vstate)) {
3702 				*flags |= CPU_DTRACE_KPRIV;
3703 				*illval = regs[r1];
3704 				break;
3705 			}
3706 			/*FALLTHROUGH*/
3707 		case DIF_OP_LDX:
3708 			regs[rd] = dtrace_load64(regs[r1]);
3709 			break;
3710 		case DIF_OP_ULDSB:
3711 			regs[rd] = (int8_t)
3712 			    dtrace_fuword8((void *)(uintptr_t)regs[r1]);
3713 			break;
3714 		case DIF_OP_ULDSH:
3715 			regs[rd] = (int16_t)
3716 			    dtrace_fuword16((void *)(uintptr_t)regs[r1]);
3717 			break;
3718 		case DIF_OP_ULDSW:
3719 			regs[rd] = (int32_t)
3720 			    dtrace_fuword32((void *)(uintptr_t)regs[r1]);
3721 			break;
3722 		case DIF_OP_ULDUB:
3723 			regs[rd] =
3724 			    dtrace_fuword8((void *)(uintptr_t)regs[r1]);
3725 			break;
3726 		case DIF_OP_ULDUH:
3727 			regs[rd] =
3728 			    dtrace_fuword16((void *)(uintptr_t)regs[r1]);
3729 			break;
3730 		case DIF_OP_ULDUW:
3731 			regs[rd] =
3732 			    dtrace_fuword32((void *)(uintptr_t)regs[r1]);
3733 			break;
3734 		case DIF_OP_ULDX:
3735 			regs[rd] =
3736 			    dtrace_fuword64((void *)(uintptr_t)regs[r1]);
3737 			break;
3738 		case DIF_OP_RET:
3739 			rval = regs[rd];
3740 			break;
3741 		case DIF_OP_NOP:
3742 			break;
3743 		case DIF_OP_SETX:
3744 			regs[rd] = inttab[DIF_INSTR_INTEGER(instr)];
3745 			break;
3746 		case DIF_OP_SETS:
3747 			regs[rd] = (uint64_t)(uintptr_t)
3748 			    (strtab + DIF_INSTR_STRING(instr));
3749 			break;
3750 		case DIF_OP_SCMP:
3751 			cc_r = dtrace_strncmp((char *)(uintptr_t)regs[r1],
3752 			    (char *)(uintptr_t)regs[r2],
3753 			    state->dts_options[DTRACEOPT_STRSIZE]);
3754 
3755 			cc_n = cc_r < 0;
3756 			cc_z = cc_r == 0;
3757 			cc_v = cc_c = 0;
3758 			break;
3759 		case DIF_OP_LDGA:
3760 			regs[rd] = dtrace_dif_variable(mstate, state,
3761 			    r1, regs[r2]);
3762 			break;
3763 		case DIF_OP_LDGS:
3764 			id = DIF_INSTR_VAR(instr);
3765 
3766 			if (id >= DIF_VAR_OTHER_UBASE) {
3767 				uintptr_t a;
3768 
3769 				id -= DIF_VAR_OTHER_UBASE;
3770 				svar = vstate->dtvs_globals[id];
3771 				ASSERT(svar != NULL);
3772 				v = &svar->dtsv_var;
3773 
3774 				if (!(v->dtdv_type.dtdt_flags & DIF_TF_BYREF)) {
3775 					regs[rd] = svar->dtsv_data;
3776 					break;
3777 				}
3778 
3779 				a = (uintptr_t)svar->dtsv_data;
3780 
3781 				if (*(uint8_t *)a == UINT8_MAX) {
3782 					/*
3783 					 * If the 0th byte is set to UINT8_MAX
3784 					 * then this is to be treated as a
3785 					 * reference to a NULL variable.
3786 					 */
3787 					regs[rd] = NULL;
3788 				} else {
3789 					regs[rd] = a + sizeof (uint64_t);
3790 				}
3791 
3792 				break;
3793 			}
3794 
3795 			regs[rd] = dtrace_dif_variable(mstate, state, id, 0);
3796 			break;
3797 
3798 		case DIF_OP_STGS:
3799 			id = DIF_INSTR_VAR(instr);
3800 
3801 			ASSERT(id >= DIF_VAR_OTHER_UBASE);
3802 			id -= DIF_VAR_OTHER_UBASE;
3803 
3804 			svar = vstate->dtvs_globals[id];
3805 			ASSERT(svar != NULL);
3806 			v = &svar->dtsv_var;
3807 
3808 			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
3809 				uintptr_t a = (uintptr_t)svar->dtsv_data;
3810 
3811 				ASSERT(a != NULL);
3812 				ASSERT(svar->dtsv_size != 0);
3813 
3814 				if (regs[rd] == NULL) {
3815 					*(uint8_t *)a = UINT8_MAX;
3816 					break;
3817 				} else {
3818 					*(uint8_t *)a = 0;
3819 					a += sizeof (uint64_t);
3820 				}
3821 
3822 				dtrace_vcopy((void *)(uintptr_t)regs[rd],
3823 				    (void *)a, &v->dtdv_type);
3824 				break;
3825 			}
3826 
3827 			svar->dtsv_data = regs[rd];
3828 			break;
3829 
3830 		case DIF_OP_LDTA:
3831 			/*
3832 			 * There are no DTrace built-in thread-local arrays at
3833 			 * present.  This opcode is saved for future work.
3834 			 */
3835 			*flags |= CPU_DTRACE_ILLOP;
3836 			regs[rd] = 0;
3837 			break;
3838 
3839 		case DIF_OP_LDLS:
3840 			id = DIF_INSTR_VAR(instr);
3841 
3842 			if (id < DIF_VAR_OTHER_UBASE) {
3843 				/*
3844 				 * For now, this has no meaning.
3845 				 */
3846 				regs[rd] = 0;
3847 				break;
3848 			}
3849 
3850 			id -= DIF_VAR_OTHER_UBASE;
3851 
3852 			ASSERT(id < vstate->dtvs_nlocals);
3853 			ASSERT(vstate->dtvs_locals != NULL);
3854 
3855 			svar = vstate->dtvs_locals[id];
3856 			ASSERT(svar != NULL);
3857 			v = &svar->dtsv_var;
3858 
3859 			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
3860 				uintptr_t a = (uintptr_t)svar->dtsv_data;
3861 				size_t sz = v->dtdv_type.dtdt_size;
3862 
3863 				sz += sizeof (uint64_t);
3864 				ASSERT(svar->dtsv_size == NCPU * sz);
3865 				a += CPU->cpu_id * sz;
3866 
3867 				if (*(uint8_t *)a == UINT8_MAX) {
3868 					/*
3869 					 * If the 0th byte is set to UINT8_MAX
3870 					 * then this is to be treated as a
3871 					 * reference to a NULL variable.
3872 					 */
3873 					regs[rd] = NULL;
3874 				} else {
3875 					regs[rd] = a + sizeof (uint64_t);
3876 				}
3877 
3878 				break;
3879 			}
3880 
3881 			ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
3882 			tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
3883 			regs[rd] = tmp[CPU->cpu_id];
3884 			break;
3885 
3886 		case DIF_OP_STLS:
3887 			id = DIF_INSTR_VAR(instr);
3888 
3889 			ASSERT(id >= DIF_VAR_OTHER_UBASE);
3890 			id -= DIF_VAR_OTHER_UBASE;
3891 			ASSERT(id < vstate->dtvs_nlocals);
3892 
3893 			ASSERT(vstate->dtvs_locals != NULL);
3894 			svar = vstate->dtvs_locals[id];
3895 			ASSERT(svar != NULL);
3896 			v = &svar->dtsv_var;
3897 
3898 			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
3899 				uintptr_t a = (uintptr_t)svar->dtsv_data;
3900 				size_t sz = v->dtdv_type.dtdt_size;
3901 
3902 				sz += sizeof (uint64_t);
3903 				ASSERT(svar->dtsv_size == NCPU * sz);
3904 				a += CPU->cpu_id * sz;
3905 
3906 				if (regs[rd] == NULL) {
3907 					*(uint8_t *)a = UINT8_MAX;
3908 					break;
3909 				} else {
3910 					*(uint8_t *)a = 0;
3911 					a += sizeof (uint64_t);
3912 				}
3913 
3914 				dtrace_vcopy((void *)(uintptr_t)regs[rd],
3915 				    (void *)a, &v->dtdv_type);
3916 				break;
3917 			}
3918 
3919 			ASSERT(svar->dtsv_size == NCPU * sizeof (uint64_t));
3920 			tmp = (uint64_t *)(uintptr_t)svar->dtsv_data;
3921 			tmp[CPU->cpu_id] = regs[rd];
3922 			break;
3923 
3924 		case DIF_OP_LDTS: {
3925 			dtrace_dynvar_t *dvar;
3926 			dtrace_key_t *key;
3927 
3928 			id = DIF_INSTR_VAR(instr);
3929 			ASSERT(id >= DIF_VAR_OTHER_UBASE);
3930 			id -= DIF_VAR_OTHER_UBASE;
3931 			v = &vstate->dtvs_tlocals[id];
3932 
3933 			key = &tupregs[DIF_DTR_NREGS];
3934 			key[0].dttk_value = (uint64_t)id;
3935 			key[0].dttk_size = 0;
3936 			DTRACE_TLS_THRKEY(key[1].dttk_value);
3937 			key[1].dttk_size = 0;
3938 
3939 			dvar = dtrace_dynvar(dstate, 2, key,
3940 			    sizeof (uint64_t), DTRACE_DYNVAR_NOALLOC);
3941 
3942 			if (dvar == NULL) {
3943 				regs[rd] = 0;
3944 				break;
3945 			}
3946 
3947 			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
3948 				regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
3949 			} else {
3950 				regs[rd] = *((uint64_t *)dvar->dtdv_data);
3951 			}
3952 
3953 			break;
3954 		}
3955 
3956 		case DIF_OP_STTS: {
3957 			dtrace_dynvar_t *dvar;
3958 			dtrace_key_t *key;
3959 
3960 			id = DIF_INSTR_VAR(instr);
3961 			ASSERT(id >= DIF_VAR_OTHER_UBASE);
3962 			id -= DIF_VAR_OTHER_UBASE;
3963 
3964 			key = &tupregs[DIF_DTR_NREGS];
3965 			key[0].dttk_value = (uint64_t)id;
3966 			key[0].dttk_size = 0;
3967 			DTRACE_TLS_THRKEY(key[1].dttk_value);
3968 			key[1].dttk_size = 0;
3969 			v = &vstate->dtvs_tlocals[id];
3970 
3971 			dvar = dtrace_dynvar(dstate, 2, key,
3972 			    v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
3973 			    v->dtdv_type.dtdt_size : sizeof (uint64_t),
3974 			    regs[rd] ? DTRACE_DYNVAR_ALLOC :
3975 			    DTRACE_DYNVAR_DEALLOC);
3976 
3977 			/*
3978 			 * Given that we're storing to thread-local data,
3979 			 * we need to flush our predicate cache.
3980 			 */
3981 			curthread->t_predcache = NULL;
3982 
3983 			if (dvar == NULL)
3984 				break;
3985 
3986 			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
3987 				dtrace_vcopy((void *)(uintptr_t)regs[rd],
3988 				    dvar->dtdv_data, &v->dtdv_type);
3989 			} else {
3990 				*((uint64_t *)dvar->dtdv_data) = regs[rd];
3991 			}
3992 
3993 			break;
3994 		}
3995 
3996 		case DIF_OP_SRA:
3997 			regs[rd] = (int64_t)regs[r1] >> regs[r2];
3998 			break;
3999 
4000 		case DIF_OP_CALL:
4001 			dtrace_dif_subr(DIF_INSTR_SUBR(instr), rd,
4002 			    regs, tupregs, ttop, mstate, state);
4003 			break;
4004 
4005 		case DIF_OP_PUSHTR:
4006 			if (ttop == DIF_DTR_NREGS) {
4007 				*flags |= CPU_DTRACE_TUPOFLOW;
4008 				break;
4009 			}
4010 
4011 			if (r1 == DIF_TYPE_STRING) {
4012 				/*
4013 				 * If this is a string type and the size is 0,
4014 				 * we'll use the system-wide default string
4015 				 * size.  Note that we are _not_ looking at
4016 				 * the value of the DTRACEOPT_STRSIZE option;
4017 				 * had this been set, we would expect to have
4018 				 * a non-zero size value in the "pushtr".
4019 				 */
4020 				tupregs[ttop].dttk_size =
4021 				    dtrace_strlen((char *)(uintptr_t)regs[rd],
4022 				    regs[r2] ? regs[r2] :
4023 				    dtrace_strsize_default) + 1;
4024 			} else {
4025 				tupregs[ttop].dttk_size = regs[r2];
4026 			}
4027 
4028 			tupregs[ttop++].dttk_value = regs[rd];
4029 			break;
4030 
4031 		case DIF_OP_PUSHTV:
4032 			if (ttop == DIF_DTR_NREGS) {
4033 				*flags |= CPU_DTRACE_TUPOFLOW;
4034 				break;
4035 			}
4036 
4037 			tupregs[ttop].dttk_value = regs[rd];
4038 			tupregs[ttop++].dttk_size = 0;
4039 			break;
4040 
4041 		case DIF_OP_POPTS:
4042 			if (ttop != 0)
4043 				ttop--;
4044 			break;
4045 
4046 		case DIF_OP_FLUSHTS:
4047 			ttop = 0;
4048 			break;
4049 
4050 		case DIF_OP_LDGAA:
4051 		case DIF_OP_LDTAA: {
4052 			dtrace_dynvar_t *dvar;
4053 			dtrace_key_t *key = tupregs;
4054 			uint_t nkeys = ttop;
4055 
4056 			id = DIF_INSTR_VAR(instr);
4057 			ASSERT(id >= DIF_VAR_OTHER_UBASE);
4058 			id -= DIF_VAR_OTHER_UBASE;
4059 
4060 			key[nkeys].dttk_value = (uint64_t)id;
4061 			key[nkeys++].dttk_size = 0;
4062 
4063 			if (DIF_INSTR_OP(instr) == DIF_OP_LDTAA) {
4064 				DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
4065 				key[nkeys++].dttk_size = 0;
4066 				v = &vstate->dtvs_tlocals[id];
4067 			} else {
4068 				v = &vstate->dtvs_globals[id]->dtsv_var;
4069 			}
4070 
4071 			dvar = dtrace_dynvar(dstate, nkeys, key,
4072 			    v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
4073 			    v->dtdv_type.dtdt_size : sizeof (uint64_t),
4074 			    DTRACE_DYNVAR_NOALLOC);
4075 
4076 			if (dvar == NULL) {
4077 				regs[rd] = 0;
4078 				break;
4079 			}
4080 
4081 			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
4082 				regs[rd] = (uint64_t)(uintptr_t)dvar->dtdv_data;
4083 			} else {
4084 				regs[rd] = *((uint64_t *)dvar->dtdv_data);
4085 			}
4086 
4087 			break;
4088 		}
4089 
4090 		case DIF_OP_STGAA:
4091 		case DIF_OP_STTAA: {
4092 			dtrace_dynvar_t *dvar;
4093 			dtrace_key_t *key = tupregs;
4094 			uint_t nkeys = ttop;
4095 
4096 			id = DIF_INSTR_VAR(instr);
4097 			ASSERT(id >= DIF_VAR_OTHER_UBASE);
4098 			id -= DIF_VAR_OTHER_UBASE;
4099 
4100 			key[nkeys].dttk_value = (uint64_t)id;
4101 			key[nkeys++].dttk_size = 0;
4102 
4103 			if (DIF_INSTR_OP(instr) == DIF_OP_STTAA) {
4104 				DTRACE_TLS_THRKEY(key[nkeys].dttk_value);
4105 				key[nkeys++].dttk_size = 0;
4106 				v = &vstate->dtvs_tlocals[id];
4107 			} else {
4108 				v = &vstate->dtvs_globals[id]->dtsv_var;
4109 			}
4110 
4111 			dvar = dtrace_dynvar(dstate, nkeys, key,
4112 			    v->dtdv_type.dtdt_size > sizeof (uint64_t) ?
4113 			    v->dtdv_type.dtdt_size : sizeof (uint64_t),
4114 			    regs[rd] ? DTRACE_DYNVAR_ALLOC :
4115 			    DTRACE_DYNVAR_DEALLOC);
4116 
4117 			if (dvar == NULL)
4118 				break;
4119 
4120 			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF) {
4121 				dtrace_vcopy((void *)(uintptr_t)regs[rd],
4122 				    dvar->dtdv_data, &v->dtdv_type);
4123 			} else {
4124 				*((uint64_t *)dvar->dtdv_data) = regs[rd];
4125 			}
4126 
4127 			break;
4128 		}
4129 
4130 		case DIF_OP_ALLOCS: {
4131 			uintptr_t ptr = P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
4132 			size_t size = ptr - mstate->dtms_scratch_ptr + regs[r1];
4133 
4134 			if (mstate->dtms_scratch_ptr + size >
4135 			    mstate->dtms_scratch_base +
4136 			    mstate->dtms_scratch_size) {
4137 				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4138 				regs[rd] = NULL;
4139 			} else {
4140 				dtrace_bzero((void *)
4141 				    mstate->dtms_scratch_ptr, size);
4142 				mstate->dtms_scratch_ptr += size;
4143 				regs[rd] = ptr;
4144 			}
4145 			break;
4146 		}
4147 
4148 		case DIF_OP_COPYS:
4149 			if (!dtrace_canstore(regs[rd], regs[r2],
4150 			    mstate, vstate)) {
4151 				*flags |= CPU_DTRACE_BADADDR;
4152 				*illval = regs[rd];
4153 				break;
4154 			}
4155 
4156 			dtrace_bcopy((void *)(uintptr_t)regs[r1],
4157 			    (void *)(uintptr_t)regs[rd], (size_t)regs[r2]);
4158 			break;
4159 
4160 		case DIF_OP_STB:
4161 			if (!dtrace_canstore(regs[rd], 1, mstate, vstate)) {
4162 				*flags |= CPU_DTRACE_BADADDR;
4163 				*illval = regs[rd];
4164 				break;
4165 			}
4166 			*((uint8_t *)(uintptr_t)regs[rd]) = (uint8_t)regs[r1];
4167 			break;
4168 
4169 		case DIF_OP_STH:
4170 			if (!dtrace_canstore(regs[rd], 2, mstate, vstate)) {
4171 				*flags |= CPU_DTRACE_BADADDR;
4172 				*illval = regs[rd];
4173 				break;
4174 			}
4175 			if (regs[rd] & 1) {
4176 				*flags |= CPU_DTRACE_BADALIGN;
4177 				*illval = regs[rd];
4178 				break;
4179 			}
4180 			*((uint16_t *)(uintptr_t)regs[rd]) = (uint16_t)regs[r1];
4181 			break;
4182 
4183 		case DIF_OP_STW:
4184 			if (!dtrace_canstore(regs[rd], 4, mstate, vstate)) {
4185 				*flags |= CPU_DTRACE_BADADDR;
4186 				*illval = regs[rd];
4187 				break;
4188 			}
4189 			if (regs[rd] & 3) {
4190 				*flags |= CPU_DTRACE_BADALIGN;
4191 				*illval = regs[rd];
4192 				break;
4193 			}
4194 			*((uint32_t *)(uintptr_t)regs[rd]) = (uint32_t)regs[r1];
4195 			break;
4196 
4197 		case DIF_OP_STX:
4198 			if (!dtrace_canstore(regs[rd], 8, mstate, vstate)) {
4199 				*flags |= CPU_DTRACE_BADADDR;
4200 				*illval = regs[rd];
4201 				break;
4202 			}
4203 			if (regs[rd] & 7) {
4204 				*flags |= CPU_DTRACE_BADALIGN;
4205 				*illval = regs[rd];
4206 				break;
4207 			}
4208 			*((uint64_t *)(uintptr_t)regs[rd]) = regs[r1];
4209 			break;
4210 		}
4211 	}
4212 
4213 	if (!(*flags & CPU_DTRACE_FAULT))
4214 		return (rval);
4215 
4216 	mstate->dtms_fltoffs = opc * sizeof (dif_instr_t);
4217 	mstate->dtms_present |= DTRACE_MSTATE_FLTOFFS;
4218 
4219 	return (0);
4220 }
4221 
4222 static void
4223 dtrace_action_breakpoint(dtrace_ecb_t *ecb)
4224 {
4225 	dtrace_probe_t *probe = ecb->dte_probe;
4226 	dtrace_provider_t *prov = probe->dtpr_provider;
4227 	char c[DTRACE_FULLNAMELEN + 80], *str;
4228 	char *msg = "dtrace: breakpoint action at probe ";
4229 	char *ecbmsg = " (ecb ";
4230 	uintptr_t mask = (0xf << (sizeof (uintptr_t) * NBBY / 4));
4231 	uintptr_t val = (uintptr_t)ecb;
4232 	int shift = (sizeof (uintptr_t) * NBBY) - 4, i = 0;
4233 
4234 	if (dtrace_destructive_disallow)
4235 		return;
4236 
4237 	/*
4238 	 * It's impossible to be taking action on the NULL probe.
4239 	 */
4240 	ASSERT(probe != NULL);
4241 
4242 	/*
4243 	 * This is a poor man's (destitute man's?) sprintf():  we want to
4244 	 * print the provider name, module name, function name and name of
4245 	 * the probe, along with the hex address of the ECB with the breakpoint
4246 	 * action -- all of which we must place in the character buffer by
4247 	 * hand.
4248 	 */
4249 	while (*msg != '\0')
4250 		c[i++] = *msg++;
4251 
4252 	for (str = prov->dtpv_name; *str != '\0'; str++)
4253 		c[i++] = *str;
4254 	c[i++] = ':';
4255 
4256 	for (str = probe->dtpr_mod; *str != '\0'; str++)
4257 		c[i++] = *str;
4258 	c[i++] = ':';
4259 
4260 	for (str = probe->dtpr_func; *str != '\0'; str++)
4261 		c[i++] = *str;
4262 	c[i++] = ':';
4263 
4264 	for (str = probe->dtpr_name; *str != '\0'; str++)
4265 		c[i++] = *str;
4266 
4267 	while (*ecbmsg != '\0')
4268 		c[i++] = *ecbmsg++;
4269 
4270 	while (shift >= 0) {
4271 		mask = (uintptr_t)0xf << shift;
4272 
4273 		if (val >= ((uintptr_t)1 << shift))
4274 			c[i++] = "0123456789abcdef"[(val & mask) >> shift];
4275 		shift -= 4;
4276 	}
4277 
4278 	c[i++] = ')';
4279 	c[i] = '\0';
4280 
4281 	debug_enter(c);
4282 }
4283 
4284 static void
4285 dtrace_action_panic(dtrace_ecb_t *ecb)
4286 {
4287 	dtrace_probe_t *probe = ecb->dte_probe;
4288 
4289 	/*
4290 	 * It's impossible to be taking action on the NULL probe.
4291 	 */
4292 	ASSERT(probe != NULL);
4293 
4294 	if (dtrace_destructive_disallow)
4295 		return;
4296 
4297 	if (dtrace_panicked != NULL)
4298 		return;
4299 
4300 	if (dtrace_casptr(&dtrace_panicked, NULL, curthread) != NULL)
4301 		return;
4302 
4303 	/*
4304 	 * We won the right to panic.  (We want to be sure that only one
4305 	 * thread calls panic() from dtrace_probe(), and that panic() is
4306 	 * called exactly once.)
4307 	 */
4308 	dtrace_panic("dtrace: panic action at probe %s:%s:%s:%s (ecb %p)",
4309 	    probe->dtpr_provider->dtpv_name, probe->dtpr_mod,
4310 	    probe->dtpr_func, probe->dtpr_name, (void *)ecb);
4311 }
4312 
4313 static void
4314 dtrace_action_raise(uint64_t sig)
4315 {
4316 	if (dtrace_destructive_disallow)
4317 		return;
4318 
4319 	if (sig >= NSIG) {
4320 		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
4321 		return;
4322 	}
4323 
4324 	/*
4325 	 * raise() has a queue depth of 1 -- we ignore all subsequent
4326 	 * invocations of the raise() action.
4327 	 */
4328 	if (curthread->t_dtrace_sig == 0)
4329 		curthread->t_dtrace_sig = (uint8_t)sig;
4330 
4331 	curthread->t_sig_check = 1;
4332 	aston(curthread);
4333 }
4334 
4335 static void
4336 dtrace_action_stop(void)
4337 {
4338 	if (dtrace_destructive_disallow)
4339 		return;
4340 
4341 	if (!curthread->t_dtrace_stop) {
4342 		curthread->t_dtrace_stop = 1;
4343 		curthread->t_sig_check = 1;
4344 		aston(curthread);
4345 	}
4346 }
4347 
4348 static void
4349 dtrace_action_chill(dtrace_mstate_t *mstate, hrtime_t val)
4350 {
4351 	hrtime_t now;
4352 	volatile uint16_t *flags;
4353 	cpu_t *cpu = CPU;
4354 
4355 	if (dtrace_destructive_disallow)
4356 		return;
4357 
4358 	flags = (volatile uint16_t *)&cpu_core[cpu->cpu_id].cpuc_dtrace_flags;
4359 
4360 	now = dtrace_gethrtime();
4361 
4362 	if (now - cpu->cpu_dtrace_chillmark > dtrace_chill_interval) {
4363 		/*
4364 		 * We need to advance the mark to the current time.
4365 		 */
4366 		cpu->cpu_dtrace_chillmark = now;
4367 		cpu->cpu_dtrace_chilled = 0;
4368 	}
4369 
4370 	/*
4371 	 * Now check to see if the requested chill time would take us over
4372 	 * the maximum amount of time allowed in the chill interval.  (Or
4373 	 * worse, if the calculation itself induces overflow.)
4374 	 */
4375 	if (cpu->cpu_dtrace_chilled + val > dtrace_chill_max ||
4376 	    cpu->cpu_dtrace_chilled + val < cpu->cpu_dtrace_chilled) {
4377 		*flags |= CPU_DTRACE_ILLOP;
4378 		return;
4379 	}
4380 
4381 	while (dtrace_gethrtime() - now < val)
4382 		continue;
4383 
4384 	/*
4385 	 * Normally, we assure that the value of the variable "timestamp" does
4386 	 * not change within an ECB.  The presence of chill() represents an
4387 	 * exception to this rule, however.
4388 	 */
4389 	mstate->dtms_present &= ~DTRACE_MSTATE_TIMESTAMP;
4390 	cpu->cpu_dtrace_chilled += val;
4391 }
4392 
4393 static void
4394 dtrace_action_ustack(dtrace_mstate_t *mstate, dtrace_state_t *state,
4395     uint64_t *buf, uint64_t arg)
4396 {
4397 	int nframes = DTRACE_USTACK_NFRAMES(arg);
4398 	int strsize = DTRACE_USTACK_STRSIZE(arg);
4399 	uint64_t *pcs = &buf[1], *fps;
4400 	char *str = (char *)&pcs[nframes];
4401 	int size, offs = 0, i, j;
4402 	uintptr_t old = mstate->dtms_scratch_ptr, saved;
4403 	uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
4404 	char *sym;
4405 
4406 	/*
4407 	 * Should be taking a faster path if string space has not been
4408 	 * allocated.
4409 	 */
4410 	ASSERT(strsize != 0);
4411 
4412 	/*
4413 	 * We will first allocate some temporary space for the frame pointers.
4414 	 */
4415 	fps = (uint64_t *)P2ROUNDUP(mstate->dtms_scratch_ptr, 8);
4416 	size = (uintptr_t)fps - mstate->dtms_scratch_ptr +
4417 	    (nframes * sizeof (uint64_t));
4418 
4419 	if (mstate->dtms_scratch_ptr + size >
4420 	    mstate->dtms_scratch_base + mstate->dtms_scratch_size) {
4421 		/*
4422 		 * Not enough room for our frame pointers -- need to indicate
4423 		 * that we ran out of scratch space.
4424 		 */
4425 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOSCRATCH);
4426 		return;
4427 	}
4428 
4429 	mstate->dtms_scratch_ptr += size;
4430 	saved = mstate->dtms_scratch_ptr;
4431 
4432 	/*
4433 	 * Now get a stack with both program counters and frame pointers.
4434 	 */
4435 	DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4436 	dtrace_getufpstack(buf, fps, nframes + 1);
4437 	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4438 
4439 	/*
4440 	 * If that faulted, we're cooked.
4441 	 */
4442 	if (*flags & CPU_DTRACE_FAULT)
4443 		goto out;
4444 
4445 	/*
4446 	 * Now we want to walk up the stack, calling the USTACK helper.  For
4447 	 * each iteration, we restore the scratch pointer.
4448 	 */
4449 	for (i = 0; i < nframes; i++) {
4450 		mstate->dtms_scratch_ptr = saved;
4451 
4452 		if (offs >= strsize)
4453 			break;
4454 
4455 		sym = (char *)(uintptr_t)dtrace_helper(
4456 		    DTRACE_HELPER_ACTION_USTACK,
4457 		    mstate, state, pcs[i], fps[i]);
4458 
4459 		/*
4460 		 * If we faulted while running the helper, we're going to
4461 		 * clear the fault and null out the corresponding string.
4462 		 */
4463 		if (*flags & CPU_DTRACE_FAULT) {
4464 			*flags &= ~CPU_DTRACE_FAULT;
4465 			str[offs++] = '\0';
4466 			continue;
4467 		}
4468 
4469 		if (sym == NULL) {
4470 			str[offs++] = '\0';
4471 			continue;
4472 		}
4473 
4474 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4475 
4476 		/*
4477 		 * Now copy in the string that the helper returned to us.
4478 		 */
4479 		for (j = 0; offs + j < strsize; j++) {
4480 			if ((str[offs + j] = sym[j]) == '\0')
4481 				break;
4482 		}
4483 
4484 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4485 
4486 		offs += j + 1;
4487 	}
4488 
4489 	if (offs >= strsize) {
4490 		/*
4491 		 * If we didn't have room for all of the strings, we don't
4492 		 * abort processing -- this needn't be a fatal error -- but we
4493 		 * still want to increment a counter (dts_stkstroverflows) to
4494 		 * allow this condition to be warned about.  (If this is from
4495 		 * a jstack() action, it is easily tuned via jstackstrsize.)
4496 		 */
4497 		dtrace_error(&state->dts_stkstroverflows);
4498 	}
4499 
4500 	while (offs < strsize)
4501 		str[offs++] = '\0';
4502 
4503 out:
4504 	mstate->dtms_scratch_ptr = old;
4505 }
4506 
4507 /*
4508  * If you're looking for the epicenter of DTrace, you just found it.  This
4509  * is the function called by the provider to fire a probe -- from which all
4510  * subsequent probe-context DTrace activity emanates.
4511  */
4512 void
4513 dtrace_probe(dtrace_id_t id, uintptr_t arg0, uintptr_t arg1,
4514     uintptr_t arg2, uintptr_t arg3, uintptr_t arg4)
4515 {
4516 	processorid_t cpuid;
4517 	dtrace_icookie_t cookie;
4518 	dtrace_probe_t *probe;
4519 	dtrace_mstate_t mstate;
4520 	dtrace_ecb_t *ecb;
4521 	dtrace_action_t *act;
4522 	intptr_t offs;
4523 	size_t size;
4524 	int vtime, onintr;
4525 	volatile uint16_t *flags;
4526 	hrtime_t now;
4527 
4528 	/*
4529 	 * Kick out immediately if this CPU is still being born (in which case
4530 	 * curthread will be set to -1)
4531 	 */
4532 	if ((uintptr_t)curthread & 1)
4533 		return;
4534 
4535 	cookie = dtrace_interrupt_disable();
4536 	probe = dtrace_probes[id - 1];
4537 	cpuid = CPU->cpu_id;
4538 	onintr = CPU_ON_INTR(CPU);
4539 
4540 	if (!onintr && probe->dtpr_predcache != DTRACE_CACHEIDNONE &&
4541 	    probe->dtpr_predcache == curthread->t_predcache) {
4542 		/*
4543 		 * We have hit in the predicate cache; we know that
4544 		 * this predicate would evaluate to be false.
4545 		 */
4546 		dtrace_interrupt_enable(cookie);
4547 		return;
4548 	}
4549 
4550 	if (panic_quiesce) {
4551 		/*
4552 		 * We don't trace anything if we're panicking.
4553 		 */
4554 		dtrace_interrupt_enable(cookie);
4555 		return;
4556 	}
4557 
4558 	now = dtrace_gethrtime();
4559 	vtime = dtrace_vtime_references != 0;
4560 
4561 	if (vtime && curthread->t_dtrace_start)
4562 		curthread->t_dtrace_vtime += now - curthread->t_dtrace_start;
4563 
4564 	mstate.dtms_probe = probe;
4565 	mstate.dtms_arg[0] = arg0;
4566 	mstate.dtms_arg[1] = arg1;
4567 	mstate.dtms_arg[2] = arg2;
4568 	mstate.dtms_arg[3] = arg3;
4569 	mstate.dtms_arg[4] = arg4;
4570 
4571 	flags = (volatile uint16_t *)&cpu_core[cpuid].cpuc_dtrace_flags;
4572 
4573 	for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
4574 		dtrace_predicate_t *pred = ecb->dte_predicate;
4575 		dtrace_state_t *state = ecb->dte_state;
4576 		dtrace_buffer_t *buf = &state->dts_buffer[cpuid];
4577 		dtrace_buffer_t *aggbuf = &state->dts_aggbuffer[cpuid];
4578 		dtrace_vstate_t *vstate = &state->dts_vstate;
4579 		dtrace_provider_t *prov = probe->dtpr_provider;
4580 		int committed = 0;
4581 		caddr_t tomax;
4582 
4583 		/*
4584 		 * A little subtlety with the following (seemingly innocuous)
4585 		 * declaration of the automatic 'val':  by looking at the
4586 		 * code, you might think that it could be declared in the
4587 		 * action processing loop, below.  (That is, it's only used in
4588 		 * the action processing loop.)  However, it must be declared
4589 		 * out of that scope because in the case of DIF expression
4590 		 * arguments to aggregating actions, one iteration of the
4591 		 * action loop will use the last iteration's value.
4592 		 */
4593 #ifdef lint
4594 		uint64_t val = 0;
4595 #else
4596 		uint64_t val;
4597 #endif
4598 
4599 		mstate.dtms_present = DTRACE_MSTATE_ARGS | DTRACE_MSTATE_PROBE;
4600 		*flags &= ~CPU_DTRACE_ERROR;
4601 
4602 		if (prov == dtrace_provider) {
4603 			/*
4604 			 * If dtrace itself is the provider of this probe,
4605 			 * we're only going to continue processing the ECB if
4606 			 * arg0 (the dtrace_state_t) is equal to the ECB's
4607 			 * creating state.  (This prevents disjoint consumers
4608 			 * from seeing one another's metaprobes.)
4609 			 */
4610 			if (arg0 != (uint64_t)(uintptr_t)state)
4611 				continue;
4612 		}
4613 
4614 		if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE) {
4615 			/*
4616 			 * We're not currently active.  If our provider isn't
4617 			 * the dtrace pseudo provider, we're not interested.
4618 			 */
4619 			if (prov != dtrace_provider)
4620 				continue;
4621 
4622 			/*
4623 			 * Now we must further check if we are in the BEGIN
4624 			 * probe.  If we are, we will only continue processing
4625 			 * if we're still in WARMUP -- if one BEGIN enabling
4626 			 * has invoked the exit() action, we don't want to
4627 			 * evaluate subsequent BEGIN enablings.
4628 			 */
4629 			if (probe->dtpr_id == dtrace_probeid_begin &&
4630 			    state->dts_activity != DTRACE_ACTIVITY_WARMUP) {
4631 				ASSERT(state->dts_activity ==
4632 				    DTRACE_ACTIVITY_DRAINING);
4633 				continue;
4634 			}
4635 		}
4636 
4637 		if (ecb->dte_cond) {
4638 			/*
4639 			 * If the dte_cond bits indicate that this
4640 			 * consumer is only allowed to see user-mode firings
4641 			 * of this probe, call the provider's dtps_usermode()
4642 			 * entry point to check that the probe was fired
4643 			 * while in a user context. Skip this ECB if that's
4644 			 * not the case.
4645 			 */
4646 			if ((ecb->dte_cond & DTRACE_COND_USERMODE) &&
4647 			    prov->dtpv_pops.dtps_usermode(prov->dtpv_arg,
4648 			    probe->dtpr_id, probe->dtpr_arg) == 0)
4649 				continue;
4650 
4651 			/*
4652 			 * This is more subtle than it looks. We have to be
4653 			 * absolutely certain that CRED() isn't going to
4654 			 * change out from under us so it's only legit to
4655 			 * examine that structure if we're in constrained
4656 			 * situations. Currently, the only times we'll this
4657 			 * check is if a non-super-user has enabled the
4658 			 * profile or syscall providers -- providers that
4659 			 * allow visibility of all processes. For the
4660 			 * profile case, the check above will ensure that
4661 			 * we're examining a user context.
4662 			 */
4663 			if (ecb->dte_cond & DTRACE_COND_OWNER) {
4664 				uid_t uid = ecb->dte_state->dts_cred.dcr_uid;
4665 				gid_t gid = ecb->dte_state->dts_cred.dcr_gid;
4666 				cred_t *cr;
4667 				proc_t *proc;
4668 
4669 				if ((cr = CRED()) == NULL ||
4670 				    uid != cr->cr_uid ||
4671 				    uid != cr->cr_ruid ||
4672 				    uid != cr->cr_suid ||
4673 				    gid != cr->cr_gid ||
4674 				    gid != cr->cr_rgid ||
4675 				    gid != cr->cr_sgid ||
4676 				    (proc = ttoproc(curthread)) == NULL ||
4677 				    (proc->p_flag & SNOCD))
4678 					continue;
4679 
4680 			}
4681 		}
4682 
4683 		if (now - state->dts_alive > dtrace_deadman_timeout) {
4684 			/*
4685 			 * We seem to be dead.  Unless we (a) have kernel
4686 			 * destructive permissions (b) have expicitly enabled
4687 			 * destructive actions and (c) destructive actions have
4688 			 * not been disabled, we're going to transition into
4689 			 * the KILLED state, from which no further processing
4690 			 * on this state will be performed.
4691 			 */
4692 			if (!dtrace_priv_kernel_destructive(state) ||
4693 			    !state->dts_cred.dcr_destructive ||
4694 			    dtrace_destructive_disallow) {
4695 				void *activity = &state->dts_activity;
4696 				dtrace_activity_t current;
4697 
4698 				do {
4699 					current = state->dts_activity;
4700 				} while (dtrace_cas32(activity, current,
4701 				    DTRACE_ACTIVITY_KILLED) != current);
4702 
4703 				continue;
4704 			}
4705 		}
4706 
4707 		if ((offs = dtrace_buffer_reserve(buf, ecb->dte_needed,
4708 		    ecb->dte_alignment, state, &mstate)) < 0)
4709 			continue;
4710 
4711 		tomax = buf->dtb_tomax;
4712 		ASSERT(tomax != NULL);
4713 
4714 		if (ecb->dte_size != 0)
4715 			DTRACE_STORE(uint32_t, tomax, offs, ecb->dte_epid);
4716 
4717 		mstate.dtms_epid = ecb->dte_epid;
4718 		mstate.dtms_present |= DTRACE_MSTATE_EPID;
4719 
4720 		if (pred != NULL) {
4721 			dtrace_difo_t *dp = pred->dtp_difo;
4722 			int rval;
4723 
4724 			rval = dtrace_dif_emulate(dp, &mstate, vstate, state);
4725 
4726 			if (!(*flags & CPU_DTRACE_ERROR) && !rval) {
4727 				dtrace_cacheid_t cid = probe->dtpr_predcache;
4728 
4729 				if (cid != DTRACE_CACHEIDNONE && !onintr) {
4730 					/*
4731 					 * Update the predicate cache...
4732 					 */
4733 					ASSERT(cid == pred->dtp_cacheid);
4734 					curthread->t_predcache = cid;
4735 				}
4736 
4737 				continue;
4738 			}
4739 		}
4740 
4741 		for (act = ecb->dte_action; !(*flags & CPU_DTRACE_ERROR) &&
4742 		    act != NULL; act = act->dta_next) {
4743 			size_t valoffs;
4744 			dtrace_difo_t *dp;
4745 			dtrace_recdesc_t *rec = &act->dta_rec;
4746 
4747 			size = rec->dtrd_size;
4748 			valoffs = offs + rec->dtrd_offset;
4749 
4750 			if (DTRACEACT_ISAGG(act->dta_kind)) {
4751 				uint64_t v = 0xbad;
4752 				dtrace_aggregation_t *agg;
4753 
4754 				agg = (dtrace_aggregation_t *)act;
4755 
4756 				if ((dp = act->dta_difo) != NULL)
4757 					v = dtrace_dif_emulate(dp,
4758 					    &mstate, vstate, state);
4759 
4760 				if (*flags & CPU_DTRACE_ERROR)
4761 					continue;
4762 
4763 				/*
4764 				 * Note that we always pass the expression
4765 				 * value from the previous iteration of the
4766 				 * action loop.  This value will only be used
4767 				 * if there is an expression argument to the
4768 				 * aggregating action, denoted by the
4769 				 * dtag_hasarg field.
4770 				 */
4771 				dtrace_aggregate(agg, buf,
4772 				    offs, aggbuf, v, val);
4773 				continue;
4774 			}
4775 
4776 			switch (act->dta_kind) {
4777 			case DTRACEACT_STOP:
4778 				if (dtrace_priv_proc_destructive(state))
4779 					dtrace_action_stop();
4780 				continue;
4781 
4782 			case DTRACEACT_BREAKPOINT:
4783 				if (dtrace_priv_kernel_destructive(state))
4784 					dtrace_action_breakpoint(ecb);
4785 				continue;
4786 
4787 			case DTRACEACT_PANIC:
4788 				if (dtrace_priv_kernel_destructive(state))
4789 					dtrace_action_panic(ecb);
4790 				continue;
4791 
4792 			case DTRACEACT_STACK:
4793 				if (!dtrace_priv_kernel(state))
4794 					continue;
4795 
4796 				dtrace_getpcstack((pc_t *)(tomax + valoffs),
4797 				    size / sizeof (pc_t), probe->dtpr_aframes,
4798 				    DTRACE_ANCHORED(probe) ? NULL :
4799 				    (uint32_t *)arg0);
4800 
4801 				continue;
4802 
4803 			case DTRACEACT_JSTACK:
4804 			case DTRACEACT_USTACK:
4805 				if (!dtrace_priv_proc(state))
4806 					continue;
4807 
4808 				/*
4809 				 * See comment in DIF_VAR_PID.
4810 				 */
4811 				if (DTRACE_ANCHORED(mstate.dtms_probe) &&
4812 				    CPU_ON_INTR(CPU)) {
4813 					int depth = DTRACE_USTACK_NFRAMES(
4814 					    rec->dtrd_arg) + 1;
4815 
4816 					dtrace_bzero((void *)(tomax + valoffs),
4817 					    DTRACE_USTACK_STRSIZE(rec->dtrd_arg)
4818 					    + depth * sizeof (uint64_t));
4819 
4820 					continue;
4821 				}
4822 
4823 				if (DTRACE_USTACK_STRSIZE(rec->dtrd_arg) != 0 &&
4824 				    curproc->p_dtrace_helpers != NULL) {
4825 					/*
4826 					 * This is the slow path -- we have
4827 					 * allocated string space, and we're
4828 					 * getting the stack of a process that
4829 					 * has helpers.  Call into a separate
4830 					 * routine to perform this processing.
4831 					 */
4832 					dtrace_action_ustack(&mstate, state,
4833 					    (uint64_t *)(tomax + valoffs),
4834 					    rec->dtrd_arg);
4835 					continue;
4836 				}
4837 
4838 				DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
4839 				dtrace_getupcstack((uint64_t *)
4840 				    (tomax + valoffs),
4841 				    DTRACE_USTACK_NFRAMES(rec->dtrd_arg) + 1);
4842 				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
4843 				continue;
4844 
4845 			default:
4846 				break;
4847 			}
4848 
4849 			dp = act->dta_difo;
4850 			ASSERT(dp != NULL);
4851 
4852 			val = dtrace_dif_emulate(dp, &mstate, vstate, state);
4853 
4854 			if (*flags & CPU_DTRACE_ERROR)
4855 				continue;
4856 
4857 			switch (act->dta_kind) {
4858 			case DTRACEACT_SPECULATE:
4859 				ASSERT(buf == &state->dts_buffer[cpuid]);
4860 				buf = dtrace_speculation_buffer(state,
4861 				    cpuid, val);
4862 
4863 				if (buf == NULL) {
4864 					*flags |= CPU_DTRACE_DROP;
4865 					continue;
4866 				}
4867 
4868 				offs = dtrace_buffer_reserve(buf,
4869 				    ecb->dte_needed, ecb->dte_alignment,
4870 				    state, NULL);
4871 
4872 				if (offs < 0) {
4873 					*flags |= CPU_DTRACE_DROP;
4874 					continue;
4875 				}
4876 
4877 				tomax = buf->dtb_tomax;
4878 				ASSERT(tomax != NULL);
4879 
4880 				if (ecb->dte_size != 0)
4881 					DTRACE_STORE(uint32_t, tomax, offs,
4882 					    ecb->dte_epid);
4883 				continue;
4884 
4885 			case DTRACEACT_CHILL:
4886 				if (dtrace_priv_kernel_destructive(state))
4887 					dtrace_action_chill(&mstate, val);
4888 				continue;
4889 
4890 			case DTRACEACT_RAISE:
4891 				if (dtrace_priv_proc_destructive(state))
4892 					dtrace_action_raise(val);
4893 				continue;
4894 
4895 			case DTRACEACT_COMMIT:
4896 				ASSERT(!committed);
4897 
4898 				/*
4899 				 * We need to commit our buffer state.
4900 				 */
4901 				if (ecb->dte_size)
4902 					buf->dtb_offset = offs + ecb->dte_size;
4903 				buf = &state->dts_buffer[cpuid];
4904 				dtrace_speculation_commit(state, cpuid, val);
4905 				committed = 1;
4906 				continue;
4907 
4908 			case DTRACEACT_DISCARD:
4909 				dtrace_speculation_discard(state, cpuid, val);
4910 				continue;
4911 
4912 			case DTRACEACT_DIFEXPR:
4913 			case DTRACEACT_LIBACT:
4914 			case DTRACEACT_PRINTF:
4915 			case DTRACEACT_PRINTA:
4916 			case DTRACEACT_SYSTEM:
4917 			case DTRACEACT_FREOPEN:
4918 				break;
4919 
4920 			case DTRACEACT_SYM:
4921 			case DTRACEACT_MOD:
4922 				if (!dtrace_priv_kernel(state))
4923 					continue;
4924 				break;
4925 
4926 			case DTRACEACT_USYM:
4927 			case DTRACEACT_UMOD:
4928 			case DTRACEACT_UADDR: {
4929 				struct pid *pid = curthread->t_procp->p_pidp;
4930 
4931 				if (!dtrace_priv_proc(state))
4932 					continue;
4933 
4934 				DTRACE_STORE(uint64_t, tomax,
4935 				    valoffs, (uint64_t)pid->pid_id);
4936 				DTRACE_STORE(uint64_t, tomax,
4937 				    valoffs + sizeof (uint64_t), val);
4938 
4939 				continue;
4940 			}
4941 
4942 			case DTRACEACT_EXIT: {
4943 				/*
4944 				 * For the exit action, we are going to attempt
4945 				 * to atomically set our activity to be
4946 				 * draining.  If this fails (either because
4947 				 * another CPU has beat us to the exit action,
4948 				 * or because our current activity is something
4949 				 * other than ACTIVE or WARMUP), we will
4950 				 * continue.  This assures that the exit action
4951 				 * can be successfully recorded at most once
4952 				 * when we're in the ACTIVE state.  If we're
4953 				 * encountering the exit() action while in
4954 				 * COOLDOWN, however, we want to honor the new
4955 				 * status code.  (We know that we're the only
4956 				 * thread in COOLDOWN, so there is no race.)
4957 				 */
4958 				void *activity = &state->dts_activity;
4959 				dtrace_activity_t current = state->dts_activity;
4960 
4961 				if (current == DTRACE_ACTIVITY_COOLDOWN)
4962 					break;
4963 
4964 				if (current != DTRACE_ACTIVITY_WARMUP)
4965 					current = DTRACE_ACTIVITY_ACTIVE;
4966 
4967 				if (dtrace_cas32(activity, current,
4968 				    DTRACE_ACTIVITY_DRAINING) != current) {
4969 					*flags |= CPU_DTRACE_DROP;
4970 					continue;
4971 				}
4972 
4973 				break;
4974 			}
4975 
4976 			default:
4977 				ASSERT(0);
4978 			}
4979 
4980 			if (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF) {
4981 				uintptr_t end = valoffs + size;
4982 
4983 				/*
4984 				 * If this is a string, we're going to only
4985 				 * load until we find the zero byte -- after
4986 				 * which we'll store zero bytes.
4987 				 */
4988 				if (dp->dtdo_rtype.dtdt_kind ==
4989 				    DIF_TYPE_STRING) {
4990 					char c = '\0' + 1;
4991 					size_t s;
4992 
4993 					for (s = 0; s < size; s++) {
4994 						if (c != '\0')
4995 							c = dtrace_load8(val++);
4996 
4997 						DTRACE_STORE(uint8_t, tomax,
4998 						    valoffs++, c);
4999 					}
5000 
5001 					continue;
5002 				}
5003 
5004 				while (valoffs < end) {
5005 					DTRACE_STORE(uint8_t, tomax, valoffs++,
5006 					    dtrace_load8(val++));
5007 				}
5008 
5009 				continue;
5010 			}
5011 
5012 			switch (size) {
5013 			case 0:
5014 				break;
5015 
5016 			case sizeof (uint8_t):
5017 				DTRACE_STORE(uint8_t, tomax, valoffs, val);
5018 				break;
5019 			case sizeof (uint16_t):
5020 				DTRACE_STORE(uint16_t, tomax, valoffs, val);
5021 				break;
5022 			case sizeof (uint32_t):
5023 				DTRACE_STORE(uint32_t, tomax, valoffs, val);
5024 				break;
5025 			case sizeof (uint64_t):
5026 				DTRACE_STORE(uint64_t, tomax, valoffs, val);
5027 				break;
5028 			default:
5029 				/*
5030 				 * Any other size should have been returned by
5031 				 * reference, not by value.
5032 				 */
5033 				ASSERT(0);
5034 				break;
5035 			}
5036 		}
5037 
5038 		if (*flags & CPU_DTRACE_DROP)
5039 			continue;
5040 
5041 		if (*flags & CPU_DTRACE_FAULT) {
5042 			int ndx;
5043 			dtrace_action_t *err;
5044 
5045 			buf->dtb_errors++;
5046 
5047 			if (probe->dtpr_id == dtrace_probeid_error) {
5048 				/*
5049 				 * There's nothing we can do -- we had an
5050 				 * error on the error probe.  We bump an
5051 				 * error counter to at least indicate that
5052 				 * this condition happened.
5053 				 */
5054 				dtrace_error(&state->dts_dblerrors);
5055 				continue;
5056 			}
5057 
5058 			if (vtime) {
5059 				/*
5060 				 * Before recursing on dtrace_probe(), we
5061 				 * need to explicitly clear out our start
5062 				 * time to prevent it from being accumulated
5063 				 * into t_dtrace_vtime.
5064 				 */
5065 				curthread->t_dtrace_start = 0;
5066 			}
5067 
5068 			/*
5069 			 * Iterate over the actions to figure out which action
5070 			 * we were processing when we experienced the error.
5071 			 * Note that act points _past_ the faulting action; if
5072 			 * act is ecb->dte_action, the fault was in the
5073 			 * predicate, if it's ecb->dte_action->dta_next it's
5074 			 * in action #1, and so on.
5075 			 */
5076 			for (err = ecb->dte_action, ndx = 0;
5077 			    err != act; err = err->dta_next, ndx++)
5078 				continue;
5079 
5080 			dtrace_probe_error(state, ecb->dte_epid, ndx,
5081 			    (mstate.dtms_present & DTRACE_MSTATE_FLTOFFS) ?
5082 			    mstate.dtms_fltoffs : -1, DTRACE_FLAGS2FLT(*flags),
5083 			    cpu_core[cpuid].cpuc_dtrace_illval);
5084 
5085 			continue;
5086 		}
5087 
5088 		if (!committed)
5089 			buf->dtb_offset = offs + ecb->dte_size;
5090 	}
5091 
5092 	if (vtime)
5093 		curthread->t_dtrace_start = dtrace_gethrtime();
5094 
5095 	dtrace_interrupt_enable(cookie);
5096 }
5097 
5098 /*
5099  * DTrace Probe Hashing Functions
5100  *
5101  * The functions in this section (and indeed, the functions in remaining
5102  * sections) are not _called_ from probe context.  (Any exceptions to this are
5103  * marked with a "Note:".)  Rather, they are called from elsewhere in the
5104  * DTrace framework to look-up probes in, add probes to and remove probes from
5105  * the DTrace probe hashes.  (Each probe is hashed by each element of the
5106  * probe tuple -- allowing for fast lookups, regardless of what was
5107  * specified.)
5108  */
5109 static uint_t
5110 dtrace_hash_str(char *p)
5111 {
5112 	unsigned int g;
5113 	uint_t hval = 0;
5114 
5115 	while (*p) {
5116 		hval = (hval << 4) + *p++;
5117 		if ((g = (hval & 0xf0000000)) != 0)
5118 			hval ^= g >> 24;
5119 		hval &= ~g;
5120 	}
5121 	return (hval);
5122 }
5123 
5124 static dtrace_hash_t *
5125 dtrace_hash_create(uintptr_t stroffs, uintptr_t nextoffs, uintptr_t prevoffs)
5126 {
5127 	dtrace_hash_t *hash = kmem_zalloc(sizeof (dtrace_hash_t), KM_SLEEP);
5128 
5129 	hash->dth_stroffs = stroffs;
5130 	hash->dth_nextoffs = nextoffs;
5131 	hash->dth_prevoffs = prevoffs;
5132 
5133 	hash->dth_size = 1;
5134 	hash->dth_mask = hash->dth_size - 1;
5135 
5136 	hash->dth_tab = kmem_zalloc(hash->dth_size *
5137 	    sizeof (dtrace_hashbucket_t *), KM_SLEEP);
5138 
5139 	return (hash);
5140 }
5141 
5142 static void
5143 dtrace_hash_destroy(dtrace_hash_t *hash)
5144 {
5145 #ifdef DEBUG
5146 	int i;
5147 
5148 	for (i = 0; i < hash->dth_size; i++)
5149 		ASSERT(hash->dth_tab[i] == NULL);
5150 #endif
5151 
5152 	kmem_free(hash->dth_tab,
5153 	    hash->dth_size * sizeof (dtrace_hashbucket_t *));
5154 	kmem_free(hash, sizeof (dtrace_hash_t));
5155 }
5156 
5157 static void
5158 dtrace_hash_resize(dtrace_hash_t *hash)
5159 {
5160 	int size = hash->dth_size, i, ndx;
5161 	int new_size = hash->dth_size << 1;
5162 	int new_mask = new_size - 1;
5163 	dtrace_hashbucket_t **new_tab, *bucket, *next;
5164 
5165 	ASSERT((new_size & new_mask) == 0);
5166 
5167 	new_tab = kmem_zalloc(new_size * sizeof (void *), KM_SLEEP);
5168 
5169 	for (i = 0; i < size; i++) {
5170 		for (bucket = hash->dth_tab[i]; bucket != NULL; bucket = next) {
5171 			dtrace_probe_t *probe = bucket->dthb_chain;
5172 
5173 			ASSERT(probe != NULL);
5174 			ndx = DTRACE_HASHSTR(hash, probe) & new_mask;
5175 
5176 			next = bucket->dthb_next;
5177 			bucket->dthb_next = new_tab[ndx];
5178 			new_tab[ndx] = bucket;
5179 		}
5180 	}
5181 
5182 	kmem_free(hash->dth_tab, hash->dth_size * sizeof (void *));
5183 	hash->dth_tab = new_tab;
5184 	hash->dth_size = new_size;
5185 	hash->dth_mask = new_mask;
5186 }
5187 
5188 static void
5189 dtrace_hash_add(dtrace_hash_t *hash, dtrace_probe_t *new)
5190 {
5191 	int hashval = DTRACE_HASHSTR(hash, new);
5192 	int ndx = hashval & hash->dth_mask;
5193 	dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
5194 	dtrace_probe_t **nextp, **prevp;
5195 
5196 	for (; bucket != NULL; bucket = bucket->dthb_next) {
5197 		if (DTRACE_HASHEQ(hash, bucket->dthb_chain, new))
5198 			goto add;
5199 	}
5200 
5201 	if ((hash->dth_nbuckets >> 1) > hash->dth_size) {
5202 		dtrace_hash_resize(hash);
5203 		dtrace_hash_add(hash, new);
5204 		return;
5205 	}
5206 
5207 	bucket = kmem_zalloc(sizeof (dtrace_hashbucket_t), KM_SLEEP);
5208 	bucket->dthb_next = hash->dth_tab[ndx];
5209 	hash->dth_tab[ndx] = bucket;
5210 	hash->dth_nbuckets++;
5211 
5212 add:
5213 	nextp = DTRACE_HASHNEXT(hash, new);
5214 	ASSERT(*nextp == NULL && *(DTRACE_HASHPREV(hash, new)) == NULL);
5215 	*nextp = bucket->dthb_chain;
5216 
5217 	if (bucket->dthb_chain != NULL) {
5218 		prevp = DTRACE_HASHPREV(hash, bucket->dthb_chain);
5219 		ASSERT(*prevp == NULL);
5220 		*prevp = new;
5221 	}
5222 
5223 	bucket->dthb_chain = new;
5224 	bucket->dthb_len++;
5225 }
5226 
5227 static dtrace_probe_t *
5228 dtrace_hash_lookup(dtrace_hash_t *hash, dtrace_probe_t *template)
5229 {
5230 	int hashval = DTRACE_HASHSTR(hash, template);
5231 	int ndx = hashval & hash->dth_mask;
5232 	dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
5233 
5234 	for (; bucket != NULL; bucket = bucket->dthb_next) {
5235 		if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
5236 			return (bucket->dthb_chain);
5237 	}
5238 
5239 	return (NULL);
5240 }
5241 
5242 static int
5243 dtrace_hash_collisions(dtrace_hash_t *hash, dtrace_probe_t *template)
5244 {
5245 	int hashval = DTRACE_HASHSTR(hash, template);
5246 	int ndx = hashval & hash->dth_mask;
5247 	dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
5248 
5249 	for (; bucket != NULL; bucket = bucket->dthb_next) {
5250 		if (DTRACE_HASHEQ(hash, bucket->dthb_chain, template))
5251 			return (bucket->dthb_len);
5252 	}
5253 
5254 	return (NULL);
5255 }
5256 
5257 static void
5258 dtrace_hash_remove(dtrace_hash_t *hash, dtrace_probe_t *probe)
5259 {
5260 	int ndx = DTRACE_HASHSTR(hash, probe) & hash->dth_mask;
5261 	dtrace_hashbucket_t *bucket = hash->dth_tab[ndx];
5262 
5263 	dtrace_probe_t **prevp = DTRACE_HASHPREV(hash, probe);
5264 	dtrace_probe_t **nextp = DTRACE_HASHNEXT(hash, probe);
5265 
5266 	/*
5267 	 * Find the bucket that we're removing this probe from.
5268 	 */
5269 	for (; bucket != NULL; bucket = bucket->dthb_next) {
5270 		if (DTRACE_HASHEQ(hash, bucket->dthb_chain, probe))
5271 			break;
5272 	}
5273 
5274 	ASSERT(bucket != NULL);
5275 
5276 	if (*prevp == NULL) {
5277 		if (*nextp == NULL) {
5278 			/*
5279 			 * The removed probe was the only probe on this
5280 			 * bucket; we need to remove the bucket.
5281 			 */
5282 			dtrace_hashbucket_t *b = hash->dth_tab[ndx];
5283 
5284 			ASSERT(bucket->dthb_chain == probe);
5285 			ASSERT(b != NULL);
5286 
5287 			if (b == bucket) {
5288 				hash->dth_tab[ndx] = bucket->dthb_next;
5289 			} else {
5290 				while (b->dthb_next != bucket)
5291 					b = b->dthb_next;
5292 				b->dthb_next = bucket->dthb_next;
5293 			}
5294 
5295 			ASSERT(hash->dth_nbuckets > 0);
5296 			hash->dth_nbuckets--;
5297 			kmem_free(bucket, sizeof (dtrace_hashbucket_t));
5298 			return;
5299 		}
5300 
5301 		bucket->dthb_chain = *nextp;
5302 	} else {
5303 		*(DTRACE_HASHNEXT(hash, *prevp)) = *nextp;
5304 	}
5305 
5306 	if (*nextp != NULL)
5307 		*(DTRACE_HASHPREV(hash, *nextp)) = *prevp;
5308 }
5309 
5310 /*
5311  * DTrace Utility Functions
5312  *
5313  * These are random utility functions that are _not_ called from probe context.
5314  */
5315 static int
5316 dtrace_badattr(const dtrace_attribute_t *a)
5317 {
5318 	return (a->dtat_name > DTRACE_STABILITY_MAX ||
5319 	    a->dtat_data > DTRACE_STABILITY_MAX ||
5320 	    a->dtat_class > DTRACE_CLASS_MAX);
5321 }
5322 
5323 /*
5324  * Return a duplicate copy of a string.  If the specified string is NULL,
5325  * this function returns a zero-length string.
5326  */
5327 static char *
5328 dtrace_strdup(const char *str)
5329 {
5330 	char *new = kmem_zalloc((str != NULL ? strlen(str) : 0) + 1, KM_SLEEP);
5331 
5332 	if (str != NULL)
5333 		(void) strcpy(new, str);
5334 
5335 	return (new);
5336 }
5337 
5338 #define	DTRACE_ISALPHA(c)	\
5339 	(((c) >= 'a' && (c) <= 'z') || ((c) >= 'A' && (c) <= 'Z'))
5340 
5341 static int
5342 dtrace_badname(const char *s)
5343 {
5344 	char c;
5345 
5346 	if (s == NULL || (c = *s++) == '\0')
5347 		return (0);
5348 
5349 	if (!DTRACE_ISALPHA(c) && c != '-' && c != '_' && c != '.')
5350 		return (1);
5351 
5352 	while ((c = *s++) != '\0') {
5353 		if (!DTRACE_ISALPHA(c) && (c < '0' || c > '9') &&
5354 		    c != '-' && c != '_' && c != '.' && c != '`')
5355 			return (1);
5356 	}
5357 
5358 	return (0);
5359 }
5360 
5361 static void
5362 dtrace_cred2priv(cred_t *cr, uint32_t *privp, uid_t *uidp)
5363 {
5364 	uint32_t priv;
5365 
5366 	*uidp = crgetuid(cr);
5367 	if (PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
5368 		priv = DTRACE_PRIV_ALL;
5369 	} else {
5370 		priv = 0;
5371 		if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE))
5372 			priv |= DTRACE_PRIV_KERNEL | DTRACE_PRIV_USER;
5373 		else if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE))
5374 			priv |= DTRACE_PRIV_USER;
5375 		if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE))
5376 			priv |= DTRACE_PRIV_PROC;
5377 		if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
5378 			priv |= DTRACE_PRIV_OWNER;
5379 	}
5380 
5381 	*privp = priv;
5382 }
5383 
5384 #ifdef DTRACE_ERRDEBUG
5385 static void
5386 dtrace_errdebug(const char *str)
5387 {
5388 	int hval = dtrace_hash_str((char *)str) % DTRACE_ERRHASHSZ;
5389 	int occupied = 0;
5390 
5391 	mutex_enter(&dtrace_errlock);
5392 	dtrace_errlast = str;
5393 	dtrace_errthread = curthread;
5394 
5395 	while (occupied++ < DTRACE_ERRHASHSZ) {
5396 		if (dtrace_errhash[hval].dter_msg == str) {
5397 			dtrace_errhash[hval].dter_count++;
5398 			goto out;
5399 		}
5400 
5401 		if (dtrace_errhash[hval].dter_msg != NULL) {
5402 			hval = (hval + 1) % DTRACE_ERRHASHSZ;
5403 			continue;
5404 		}
5405 
5406 		dtrace_errhash[hval].dter_msg = str;
5407 		dtrace_errhash[hval].dter_count = 1;
5408 		goto out;
5409 	}
5410 
5411 	panic("dtrace: undersized error hash");
5412 out:
5413 	mutex_exit(&dtrace_errlock);
5414 }
5415 #endif
5416 
5417 /*
5418  * DTrace Matching Functions
5419  *
5420  * These functions are used to match groups of probes, given some elements of
5421  * a probe tuple, or some globbed expressions for elements of a probe tuple.
5422  */
5423 static int
5424 dtrace_match_priv(const dtrace_probe_t *prp, uint32_t priv, uid_t uid)
5425 {
5426 	if (priv != DTRACE_PRIV_ALL) {
5427 		uint32_t ppriv = prp->dtpr_provider->dtpv_priv.dtpp_flags;
5428 		uint32_t match = priv & ppriv;
5429 
5430 		/*
5431 		 * No PRIV_DTRACE_* privileges...
5432 		 */
5433 		if ((priv & (DTRACE_PRIV_PROC | DTRACE_PRIV_USER |
5434 		    DTRACE_PRIV_KERNEL)) == 0)
5435 			return (0);
5436 
5437 		/*
5438 		 * No matching bits, but there were bits to match...
5439 		 */
5440 		if (match == 0 && ppriv != 0)
5441 			return (0);
5442 
5443 		/*
5444 		 * Need to have permissions to the process, but don't...
5445 		 */
5446 		if (((ppriv & ~match) & DTRACE_PRIV_OWNER) != 0 &&
5447 		    uid != prp->dtpr_provider->dtpv_priv.dtpp_uid)
5448 			return (0);
5449 	}
5450 
5451 	return (1);
5452 }
5453 
5454 /*
5455  * dtrace_match_probe compares a dtrace_probe_t to a pre-compiled key, which
5456  * consists of input pattern strings and an ops-vector to evaluate them.
5457  * This function returns >0 for match, 0 for no match, and <0 for error.
5458  */
5459 static int
5460 dtrace_match_probe(const dtrace_probe_t *prp, const dtrace_probekey_t *pkp,
5461     uint32_t priv, uid_t uid)
5462 {
5463 	dtrace_provider_t *pvp = prp->dtpr_provider;
5464 	int rv;
5465 
5466 	if (pvp->dtpv_defunct)
5467 		return (0);
5468 
5469 	if ((rv = pkp->dtpk_pmatch(pvp->dtpv_name, pkp->dtpk_prov, 0)) <= 0)
5470 		return (rv);
5471 
5472 	if ((rv = pkp->dtpk_mmatch(prp->dtpr_mod, pkp->dtpk_mod, 0)) <= 0)
5473 		return (rv);
5474 
5475 	if ((rv = pkp->dtpk_fmatch(prp->dtpr_func, pkp->dtpk_func, 0)) <= 0)
5476 		return (rv);
5477 
5478 	if ((rv = pkp->dtpk_nmatch(prp->dtpr_name, pkp->dtpk_name, 0)) <= 0)
5479 		return (rv);
5480 
5481 	if (dtrace_match_priv(prp, priv, uid) == 0)
5482 		return (0);
5483 
5484 	return (rv);
5485 }
5486 
5487 /*
5488  * dtrace_match_glob() is a safe kernel implementation of the gmatch(3GEN)
5489  * interface for matching a glob pattern 'p' to an input string 's'.  Unlike
5490  * libc's version, the kernel version only applies to 8-bit ASCII strings.
5491  * In addition, all of the recursion cases except for '*' matching have been
5492  * unwound.  For '*', we still implement recursive evaluation, but a depth
5493  * counter is maintained and matching is aborted if we recurse too deep.
5494  * The function returns 0 if no match, >0 if match, and <0 if recursion error.
5495  */
5496 static int
5497 dtrace_match_glob(const char *s, const char *p, int depth)
5498 {
5499 	const char *olds;
5500 	char s1, c;
5501 	int gs;
5502 
5503 	if (depth > DTRACE_PROBEKEY_MAXDEPTH)
5504 		return (-1);
5505 
5506 	if (s == NULL)
5507 		s = ""; /* treat NULL as empty string */
5508 
5509 top:
5510 	olds = s;
5511 	s1 = *s++;
5512 
5513 	if (p == NULL)
5514 		return (0);
5515 
5516 	if ((c = *p++) == '\0')
5517 		return (s1 == '\0');
5518 
5519 	switch (c) {
5520 	case '[': {
5521 		int ok = 0, notflag = 0;
5522 		char lc = '\0';
5523 
5524 		if (s1 == '\0')
5525 			return (0);
5526 
5527 		if (*p == '!') {
5528 			notflag = 1;
5529 			p++;
5530 		}
5531 
5532 		if ((c = *p++) == '\0')
5533 			return (0);
5534 
5535 		do {
5536 			if (c == '-' && lc != '\0' && *p != ']') {
5537 				if ((c = *p++) == '\0')
5538 					return (0);
5539 				if (c == '\\' && (c = *p++) == '\0')
5540 					return (0);
5541 
5542 				if (notflag) {
5543 					if (s1 < lc || s1 > c)
5544 						ok++;
5545 					else
5546 						return (0);
5547 				} else if (lc <= s1 && s1 <= c)
5548 					ok++;
5549 
5550 			} else if (c == '\\' && (c = *p++) == '\0')
5551 				return (0);
5552 
5553 			lc = c; /* save left-hand 'c' for next iteration */
5554 
5555 			if (notflag) {
5556 				if (s1 != c)
5557 					ok++;
5558 				else
5559 					return (0);
5560 			} else if (s1 == c)
5561 				ok++;
5562 
5563 			if ((c = *p++) == '\0')
5564 				return (0);
5565 
5566 		} while (c != ']');
5567 
5568 		if (ok)
5569 			goto top;
5570 
5571 		return (0);
5572 	}
5573 
5574 	case '\\':
5575 		if ((c = *p++) == '\0')
5576 			return (0);
5577 		/*FALLTHRU*/
5578 
5579 	default:
5580 		if (c != s1)
5581 			return (0);
5582 		/*FALLTHRU*/
5583 
5584 	case '?':
5585 		if (s1 != '\0')
5586 			goto top;
5587 		return (0);
5588 
5589 	case '*':
5590 		while (*p == '*')
5591 			p++; /* consecutive *'s are identical to a single one */
5592 
5593 		if (*p == '\0')
5594 			return (1);
5595 
5596 		for (s = olds; *s != '\0'; s++) {
5597 			if ((gs = dtrace_match_glob(s, p, depth + 1)) != 0)
5598 				return (gs);
5599 		}
5600 
5601 		return (0);
5602 	}
5603 }
5604 
5605 /*ARGSUSED*/
5606 static int
5607 dtrace_match_string(const char *s, const char *p, int depth)
5608 {
5609 	return (s != NULL && strcmp(s, p) == 0);
5610 }
5611 
5612 /*ARGSUSED*/
5613 static int
5614 dtrace_match_nul(const char *s, const char *p, int depth)
5615 {
5616 	return (1); /* always match the empty pattern */
5617 }
5618 
5619 /*ARGSUSED*/
5620 static int
5621 dtrace_match_nonzero(const char *s, const char *p, int depth)
5622 {
5623 	return (s != NULL && s[0] != '\0');
5624 }
5625 
5626 static int
5627 dtrace_match(const dtrace_probekey_t *pkp, uint32_t priv, uid_t uid,
5628     int (*matched)(dtrace_probe_t *, void *), void *arg)
5629 {
5630 	dtrace_probe_t template, *probe;
5631 	dtrace_hash_t *hash = NULL;
5632 	int len, best = INT_MAX, nmatched = 0;
5633 	dtrace_id_t i;
5634 
5635 	ASSERT(MUTEX_HELD(&dtrace_lock));
5636 
5637 	/*
5638 	 * If the probe ID is specified in the key, just lookup by ID and
5639 	 * invoke the match callback once if a matching probe is found.
5640 	 */
5641 	if (pkp->dtpk_id != DTRACE_IDNONE) {
5642 		if ((probe = dtrace_probe_lookup_id(pkp->dtpk_id)) != NULL &&
5643 		    dtrace_match_probe(probe, pkp, priv, uid) > 0) {
5644 			(void) (*matched)(probe, arg);
5645 			nmatched++;
5646 		}
5647 		return (nmatched);
5648 	}
5649 
5650 	template.dtpr_mod = (char *)pkp->dtpk_mod;
5651 	template.dtpr_func = (char *)pkp->dtpk_func;
5652 	template.dtpr_name = (char *)pkp->dtpk_name;
5653 
5654 	/*
5655 	 * We want to find the most distinct of the module name, function
5656 	 * name, and name.  So for each one that is not a glob pattern or
5657 	 * empty string, we perform a lookup in the corresponding hash and
5658 	 * use the hash table with the fewest collisions to do our search.
5659 	 */
5660 	if (pkp->dtpk_mmatch == &dtrace_match_string &&
5661 	    (len = dtrace_hash_collisions(dtrace_bymod, &template)) < best) {
5662 		best = len;
5663 		hash = dtrace_bymod;
5664 	}
5665 
5666 	if (pkp->dtpk_fmatch == &dtrace_match_string &&
5667 	    (len = dtrace_hash_collisions(dtrace_byfunc, &template)) < best) {
5668 		best = len;
5669 		hash = dtrace_byfunc;
5670 	}
5671 
5672 	if (pkp->dtpk_nmatch == &dtrace_match_string &&
5673 	    (len = dtrace_hash_collisions(dtrace_byname, &template)) < best) {
5674 		best = len;
5675 		hash = dtrace_byname;
5676 	}
5677 
5678 	/*
5679 	 * If we did not select a hash table, iterate over every probe and
5680 	 * invoke our callback for each one that matches our input probe key.
5681 	 */
5682 	if (hash == NULL) {
5683 		for (i = 0; i < dtrace_nprobes; i++) {
5684 			if ((probe = dtrace_probes[i]) == NULL ||
5685 			    dtrace_match_probe(probe, pkp, priv, uid) <= 0)
5686 				continue;
5687 
5688 			nmatched++;
5689 
5690 			if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT)
5691 				break;
5692 		}
5693 
5694 		return (nmatched);
5695 	}
5696 
5697 	/*
5698 	 * If we selected a hash table, iterate over each probe of the same key
5699 	 * name and invoke the callback for every probe that matches the other
5700 	 * attributes of our input probe key.
5701 	 */
5702 	for (probe = dtrace_hash_lookup(hash, &template); probe != NULL;
5703 	    probe = *(DTRACE_HASHNEXT(hash, probe))) {
5704 
5705 		if (dtrace_match_probe(probe, pkp, priv, uid) <= 0)
5706 			continue;
5707 
5708 		nmatched++;
5709 
5710 		if ((*matched)(probe, arg) != DTRACE_MATCH_NEXT)
5711 			break;
5712 	}
5713 
5714 	return (nmatched);
5715 }
5716 
5717 /*
5718  * Return the function pointer dtrace_probecmp() should use to compare the
5719  * specified pattern with a string.  For NULL or empty patterns, we select
5720  * dtrace_match_nul().  For glob pattern strings, we use dtrace_match_glob().
5721  * For non-empty non-glob strings, we use dtrace_match_string().
5722  */
5723 static dtrace_probekey_f *
5724 dtrace_probekey_func(const char *p)
5725 {
5726 	char c;
5727 
5728 	if (p == NULL || *p == '\0')
5729 		return (&dtrace_match_nul);
5730 
5731 	while ((c = *p++) != '\0') {
5732 		if (c == '[' || c == '?' || c == '*' || c == '\\')
5733 			return (&dtrace_match_glob);
5734 	}
5735 
5736 	return (&dtrace_match_string);
5737 }
5738 
5739 /*
5740  * Build a probe comparison key for use with dtrace_match_probe() from the
5741  * given probe description.  By convention, a null key only matches anchored
5742  * probes: if each field is the empty string, reset dtpk_fmatch to
5743  * dtrace_match_nonzero().
5744  */
5745 static void
5746 dtrace_probekey(const dtrace_probedesc_t *pdp, dtrace_probekey_t *pkp)
5747 {
5748 	pkp->dtpk_prov = pdp->dtpd_provider;
5749 	pkp->dtpk_pmatch = dtrace_probekey_func(pdp->dtpd_provider);
5750 
5751 	pkp->dtpk_mod = pdp->dtpd_mod;
5752 	pkp->dtpk_mmatch = dtrace_probekey_func(pdp->dtpd_mod);
5753 
5754 	pkp->dtpk_func = pdp->dtpd_func;
5755 	pkp->dtpk_fmatch = dtrace_probekey_func(pdp->dtpd_func);
5756 
5757 	pkp->dtpk_name = pdp->dtpd_name;
5758 	pkp->dtpk_nmatch = dtrace_probekey_func(pdp->dtpd_name);
5759 
5760 	pkp->dtpk_id = pdp->dtpd_id;
5761 
5762 	if (pkp->dtpk_id == DTRACE_IDNONE &&
5763 	    pkp->dtpk_pmatch == &dtrace_match_nul &&
5764 	    pkp->dtpk_mmatch == &dtrace_match_nul &&
5765 	    pkp->dtpk_fmatch == &dtrace_match_nul &&
5766 	    pkp->dtpk_nmatch == &dtrace_match_nul)
5767 		pkp->dtpk_fmatch = &dtrace_match_nonzero;
5768 }
5769 
5770 /*
5771  * DTrace Provider-to-Framework API Functions
5772  *
5773  * These functions implement much of the Provider-to-Framework API, as
5774  * described in <sys/dtrace.h>.  The parts of the API not in this section are
5775  * the functions in the API for probe management (found below), and
5776  * dtrace_probe() itself (found above).
5777  */
5778 
5779 /*
5780  * Register the calling provider with the DTrace framework.  This should
5781  * generally be called by DTrace providers in their attach(9E) entry point.
5782  */
5783 int
5784 dtrace_register(const char *name, const dtrace_pattr_t *pap, uint32_t priv,
5785     uid_t uid, const dtrace_pops_t *pops, void *arg, dtrace_provider_id_t *idp)
5786 {
5787 	dtrace_provider_t *provider;
5788 
5789 	if (name == NULL || pap == NULL || pops == NULL || idp == NULL) {
5790 		cmn_err(CE_WARN, "failed to register provider '%s': invalid "
5791 		    "arguments", name ? name : "<NULL>");
5792 		return (EINVAL);
5793 	}
5794 
5795 	if (name[0] == '\0' || dtrace_badname(name)) {
5796 		cmn_err(CE_WARN, "failed to register provider '%s': invalid "
5797 		    "provider name", name);
5798 		return (EINVAL);
5799 	}
5800 
5801 	if ((pops->dtps_provide == NULL && pops->dtps_provide_module == NULL) ||
5802 	    pops->dtps_enable == NULL || pops->dtps_disable == NULL ||
5803 	    pops->dtps_destroy == NULL ||
5804 	    ((pops->dtps_resume == NULL) != (pops->dtps_suspend == NULL))) {
5805 		cmn_err(CE_WARN, "failed to register provider '%s': invalid "
5806 		    "provider ops", name);
5807 		return (EINVAL);
5808 	}
5809 
5810 	if (dtrace_badattr(&pap->dtpa_provider) ||
5811 	    dtrace_badattr(&pap->dtpa_mod) ||
5812 	    dtrace_badattr(&pap->dtpa_func) ||
5813 	    dtrace_badattr(&pap->dtpa_name) ||
5814 	    dtrace_badattr(&pap->dtpa_args)) {
5815 		cmn_err(CE_WARN, "failed to register provider '%s': invalid "
5816 		    "provider attributes", name);
5817 		return (EINVAL);
5818 	}
5819 
5820 	if (priv & ~DTRACE_PRIV_ALL) {
5821 		cmn_err(CE_WARN, "failed to register provider '%s': invalid "
5822 		    "privilege attributes", name);
5823 		return (EINVAL);
5824 	}
5825 
5826 	if ((priv & DTRACE_PRIV_KERNEL) &&
5827 	    (priv & (DTRACE_PRIV_USER | DTRACE_PRIV_OWNER)) &&
5828 	    pops->dtps_usermode == NULL) {
5829 		cmn_err(CE_WARN, "failed to register provider '%s': need "
5830 		    "dtps_usermode() op for given privilege attributes", name);
5831 		return (EINVAL);
5832 	}
5833 
5834 	provider = kmem_zalloc(sizeof (dtrace_provider_t), KM_SLEEP);
5835 	provider->dtpv_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
5836 	(void) strcpy(provider->dtpv_name, name);
5837 
5838 	provider->dtpv_attr = *pap;
5839 	provider->dtpv_priv.dtpp_flags = priv;
5840 	provider->dtpv_priv.dtpp_uid = uid;
5841 	provider->dtpv_pops = *pops;
5842 
5843 	if (pops->dtps_provide == NULL) {
5844 		ASSERT(pops->dtps_provide_module != NULL);
5845 		provider->dtpv_pops.dtps_provide =
5846 		    (void (*)(void *, const dtrace_probedesc_t *))dtrace_nullop;
5847 	}
5848 
5849 	if (pops->dtps_provide_module == NULL) {
5850 		ASSERT(pops->dtps_provide != NULL);
5851 		provider->dtpv_pops.dtps_provide_module =
5852 		    (void (*)(void *, struct modctl *))dtrace_nullop;
5853 	}
5854 
5855 	if (pops->dtps_suspend == NULL) {
5856 		ASSERT(pops->dtps_resume == NULL);
5857 		provider->dtpv_pops.dtps_suspend =
5858 		    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
5859 		provider->dtpv_pops.dtps_resume =
5860 		    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop;
5861 	}
5862 
5863 	provider->dtpv_arg = arg;
5864 	*idp = (dtrace_provider_id_t)provider;
5865 
5866 	if (pops == &dtrace_provider_ops) {
5867 		ASSERT(MUTEX_HELD(&dtrace_provider_lock));
5868 		ASSERT(MUTEX_HELD(&dtrace_lock));
5869 		ASSERT(dtrace_anon.dta_enabling == NULL);
5870 
5871 		/*
5872 		 * We make sure that the DTrace provider is at the head of
5873 		 * the provider chain.
5874 		 */
5875 		provider->dtpv_next = dtrace_provider;
5876 		dtrace_provider = provider;
5877 		return (0);
5878 	}
5879 
5880 	mutex_enter(&dtrace_provider_lock);
5881 	mutex_enter(&dtrace_lock);
5882 
5883 	/*
5884 	 * If there is at least one provider registered, we'll add this
5885 	 * provider after the first provider.
5886 	 */
5887 	if (dtrace_provider != NULL) {
5888 		provider->dtpv_next = dtrace_provider->dtpv_next;
5889 		dtrace_provider->dtpv_next = provider;
5890 	} else {
5891 		dtrace_provider = provider;
5892 	}
5893 
5894 	if (dtrace_retained != NULL) {
5895 		dtrace_enabling_provide(provider);
5896 
5897 		/*
5898 		 * Now we need to call dtrace_enabling_matchall() -- which
5899 		 * will acquire cpu_lock and dtrace_lock.  We therefore need
5900 		 * to drop all of our locks before calling into it...
5901 		 */
5902 		mutex_exit(&dtrace_lock);
5903 		mutex_exit(&dtrace_provider_lock);
5904 		dtrace_enabling_matchall();
5905 
5906 		return (0);
5907 	}
5908 
5909 	mutex_exit(&dtrace_lock);
5910 	mutex_exit(&dtrace_provider_lock);
5911 
5912 	return (0);
5913 }
5914 
5915 /*
5916  * Unregister the specified provider from the DTrace framework.  This should
5917  * generally be called by DTrace providers in their detach(9E) entry point.
5918  */
5919 int
5920 dtrace_unregister(dtrace_provider_id_t id)
5921 {
5922 	dtrace_provider_t *old = (dtrace_provider_t *)id;
5923 	dtrace_provider_t *prev = NULL;
5924 	int i, self = 0;
5925 	dtrace_probe_t *probe, *first = NULL;
5926 
5927 	if (old->dtpv_pops.dtps_enable ==
5928 	    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop) {
5929 		/*
5930 		 * If DTrace itself is the provider, we're called with locks
5931 		 * already held.
5932 		 */
5933 		ASSERT(old == dtrace_provider);
5934 		ASSERT(dtrace_devi != NULL);
5935 		ASSERT(MUTEX_HELD(&dtrace_provider_lock));
5936 		ASSERT(MUTEX_HELD(&dtrace_lock));
5937 		self = 1;
5938 
5939 		if (dtrace_provider->dtpv_next != NULL) {
5940 			/*
5941 			 * There's another provider here; return failure.
5942 			 */
5943 			return (EBUSY);
5944 		}
5945 	} else {
5946 		mutex_enter(&dtrace_provider_lock);
5947 		mutex_enter(&mod_lock);
5948 		mutex_enter(&dtrace_lock);
5949 	}
5950 
5951 	/*
5952 	 * If anyone has /dev/dtrace open, or if there are anonymous enabled
5953 	 * probes, we refuse to let providers slither away, unless this
5954 	 * provider has already been explicitly invalidated.
5955 	 */
5956 	if (!old->dtpv_defunct &&
5957 	    (dtrace_opens || (dtrace_anon.dta_state != NULL &&
5958 	    dtrace_anon.dta_state->dts_necbs > 0))) {
5959 		if (!self) {
5960 			mutex_exit(&dtrace_lock);
5961 			mutex_exit(&mod_lock);
5962 			mutex_exit(&dtrace_provider_lock);
5963 		}
5964 		return (EBUSY);
5965 	}
5966 
5967 	/*
5968 	 * Attempt to destroy the probes associated with this provider.
5969 	 */
5970 	for (i = 0; i < dtrace_nprobes; i++) {
5971 		if ((probe = dtrace_probes[i]) == NULL)
5972 			continue;
5973 
5974 		if (probe->dtpr_provider != old)
5975 			continue;
5976 
5977 		if (probe->dtpr_ecb == NULL)
5978 			continue;
5979 
5980 		/*
5981 		 * We have at least one ECB; we can't remove this provider.
5982 		 */
5983 		if (!self) {
5984 			mutex_exit(&dtrace_lock);
5985 			mutex_exit(&mod_lock);
5986 			mutex_exit(&dtrace_provider_lock);
5987 		}
5988 		return (EBUSY);
5989 	}
5990 
5991 	/*
5992 	 * All of the probes for this provider are disabled; we can safely
5993 	 * remove all of them from their hash chains and from the probe array.
5994 	 */
5995 	for (i = 0; i < dtrace_nprobes; i++) {
5996 		if ((probe = dtrace_probes[i]) == NULL)
5997 			continue;
5998 
5999 		if (probe->dtpr_provider != old)
6000 			continue;
6001 
6002 		dtrace_probes[i] = NULL;
6003 
6004 		dtrace_hash_remove(dtrace_bymod, probe);
6005 		dtrace_hash_remove(dtrace_byfunc, probe);
6006 		dtrace_hash_remove(dtrace_byname, probe);
6007 
6008 		if (first == NULL) {
6009 			first = probe;
6010 			probe->dtpr_nextmod = NULL;
6011 		} else {
6012 			probe->dtpr_nextmod = first;
6013 			first = probe;
6014 		}
6015 	}
6016 
6017 	/*
6018 	 * The provider's probes have been removed from the hash chains and
6019 	 * from the probe array.  Now issue a dtrace_sync() to be sure that
6020 	 * everyone has cleared out from any probe array processing.
6021 	 */
6022 	dtrace_sync();
6023 
6024 	for (probe = first; probe != NULL; probe = first) {
6025 		first = probe->dtpr_nextmod;
6026 
6027 		old->dtpv_pops.dtps_destroy(old->dtpv_arg, probe->dtpr_id,
6028 		    probe->dtpr_arg);
6029 		kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
6030 		kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
6031 		kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
6032 		vmem_free(dtrace_arena, (void *)(uintptr_t)(probe->dtpr_id), 1);
6033 		kmem_free(probe, sizeof (dtrace_probe_t));
6034 	}
6035 
6036 	if ((prev = dtrace_provider) == old) {
6037 		ASSERT(self || dtrace_devi == NULL);
6038 		ASSERT(old->dtpv_next == NULL || dtrace_devi == NULL);
6039 		dtrace_provider = old->dtpv_next;
6040 	} else {
6041 		while (prev != NULL && prev->dtpv_next != old)
6042 			prev = prev->dtpv_next;
6043 
6044 		if (prev == NULL) {
6045 			panic("attempt to unregister non-existent "
6046 			    "dtrace provider %p\n", (void *)id);
6047 		}
6048 
6049 		prev->dtpv_next = old->dtpv_next;
6050 	}
6051 
6052 	if (!self) {
6053 		mutex_exit(&dtrace_lock);
6054 		mutex_exit(&mod_lock);
6055 		mutex_exit(&dtrace_provider_lock);
6056 	}
6057 
6058 	kmem_free(old->dtpv_name, strlen(old->dtpv_name) + 1);
6059 	kmem_free(old, sizeof (dtrace_provider_t));
6060 
6061 	return (0);
6062 }
6063 
6064 /*
6065  * Invalidate the specified provider.  All subsequent probe lookups for the
6066  * specified provider will fail, but its probes will not be removed.
6067  */
6068 void
6069 dtrace_invalidate(dtrace_provider_id_t id)
6070 {
6071 	dtrace_provider_t *pvp = (dtrace_provider_t *)id;
6072 
6073 	ASSERT(pvp->dtpv_pops.dtps_enable !=
6074 	    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop);
6075 
6076 	mutex_enter(&dtrace_provider_lock);
6077 	mutex_enter(&dtrace_lock);
6078 
6079 	pvp->dtpv_defunct = 1;
6080 
6081 	mutex_exit(&dtrace_lock);
6082 	mutex_exit(&dtrace_provider_lock);
6083 }
6084 
6085 /*
6086  * Indicate whether or not DTrace has attached.
6087  */
6088 int
6089 dtrace_attached(void)
6090 {
6091 	/*
6092 	 * dtrace_provider will be non-NULL iff the DTrace driver has
6093 	 * attached.  (It's non-NULL because DTrace is always itself a
6094 	 * provider.)
6095 	 */
6096 	return (dtrace_provider != NULL);
6097 }
6098 
6099 /*
6100  * Remove all the unenabled probes for the given provider.  This function is
6101  * not unlike dtrace_unregister(), except that it doesn't remove the provider
6102  * -- just as many of its associated probes as it can.
6103  */
6104 int
6105 dtrace_condense(dtrace_provider_id_t id)
6106 {
6107 	dtrace_provider_t *prov = (dtrace_provider_t *)id;
6108 	int i;
6109 	dtrace_probe_t *probe;
6110 
6111 	/*
6112 	 * Make sure this isn't the dtrace provider itself.
6113 	 */
6114 	ASSERT(prov->dtpv_pops.dtps_enable !=
6115 	    (void (*)(void *, dtrace_id_t, void *))dtrace_nullop);
6116 
6117 	mutex_enter(&dtrace_provider_lock);
6118 	mutex_enter(&dtrace_lock);
6119 
6120 	/*
6121 	 * Attempt to destroy the probes associated with this provider.
6122 	 */
6123 	for (i = 0; i < dtrace_nprobes; i++) {
6124 		if ((probe = dtrace_probes[i]) == NULL)
6125 			continue;
6126 
6127 		if (probe->dtpr_provider != prov)
6128 			continue;
6129 
6130 		if (probe->dtpr_ecb != NULL)
6131 			continue;
6132 
6133 		dtrace_probes[i] = NULL;
6134 
6135 		dtrace_hash_remove(dtrace_bymod, probe);
6136 		dtrace_hash_remove(dtrace_byfunc, probe);
6137 		dtrace_hash_remove(dtrace_byname, probe);
6138 
6139 		prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, i + 1,
6140 		    probe->dtpr_arg);
6141 		kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
6142 		kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
6143 		kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
6144 		kmem_free(probe, sizeof (dtrace_probe_t));
6145 		vmem_free(dtrace_arena, (void *)((uintptr_t)i + 1), 1);
6146 	}
6147 
6148 	mutex_exit(&dtrace_lock);
6149 	mutex_exit(&dtrace_provider_lock);
6150 
6151 	return (0);
6152 }
6153 
6154 /*
6155  * DTrace Probe Management Functions
6156  *
6157  * The functions in this section perform the DTrace probe management,
6158  * including functions to create probes, look-up probes, and call into the
6159  * providers to request that probes be provided.  Some of these functions are
6160  * in the Provider-to-Framework API; these functions can be identified by the
6161  * fact that they are not declared "static".
6162  */
6163 
6164 /*
6165  * Create a probe with the specified module name, function name, and name.
6166  */
6167 dtrace_id_t
6168 dtrace_probe_create(dtrace_provider_id_t prov, const char *mod,
6169     const char *func, const char *name, int aframes, void *arg)
6170 {
6171 	dtrace_probe_t *probe, **probes;
6172 	dtrace_provider_t *provider = (dtrace_provider_t *)prov;
6173 	dtrace_id_t id;
6174 
6175 	if (provider == dtrace_provider) {
6176 		ASSERT(MUTEX_HELD(&dtrace_lock));
6177 	} else {
6178 		mutex_enter(&dtrace_lock);
6179 	}
6180 
6181 	id = (dtrace_id_t)(uintptr_t)vmem_alloc(dtrace_arena, 1,
6182 	    VM_BESTFIT | VM_SLEEP);
6183 	probe = kmem_zalloc(sizeof (dtrace_probe_t), KM_SLEEP);
6184 
6185 	probe->dtpr_id = id;
6186 	probe->dtpr_gen = dtrace_probegen++;
6187 	probe->dtpr_mod = dtrace_strdup(mod);
6188 	probe->dtpr_func = dtrace_strdup(func);
6189 	probe->dtpr_name = dtrace_strdup(name);
6190 	probe->dtpr_arg = arg;
6191 	probe->dtpr_aframes = aframes;
6192 	probe->dtpr_provider = provider;
6193 
6194 	dtrace_hash_add(dtrace_bymod, probe);
6195 	dtrace_hash_add(dtrace_byfunc, probe);
6196 	dtrace_hash_add(dtrace_byname, probe);
6197 
6198 	if (id - 1 >= dtrace_nprobes) {
6199 		size_t osize = dtrace_nprobes * sizeof (dtrace_probe_t *);
6200 		size_t nsize = osize << 1;
6201 
6202 		if (nsize == 0) {
6203 			ASSERT(osize == 0);
6204 			ASSERT(dtrace_probes == NULL);
6205 			nsize = sizeof (dtrace_probe_t *);
6206 		}
6207 
6208 		probes = kmem_zalloc(nsize, KM_SLEEP);
6209 
6210 		if (dtrace_probes == NULL) {
6211 			ASSERT(osize == 0);
6212 			dtrace_probes = probes;
6213 			dtrace_nprobes = 1;
6214 		} else {
6215 			dtrace_probe_t **oprobes = dtrace_probes;
6216 
6217 			bcopy(oprobes, probes, osize);
6218 			dtrace_membar_producer();
6219 			dtrace_probes = probes;
6220 
6221 			dtrace_sync();
6222 
6223 			/*
6224 			 * All CPUs are now seeing the new probes array; we can
6225 			 * safely free the old array.
6226 			 */
6227 			kmem_free(oprobes, osize);
6228 			dtrace_nprobes <<= 1;
6229 		}
6230 
6231 		ASSERT(id - 1 < dtrace_nprobes);
6232 	}
6233 
6234 	ASSERT(dtrace_probes[id - 1] == NULL);
6235 	dtrace_probes[id - 1] = probe;
6236 
6237 	if (provider != dtrace_provider)
6238 		mutex_exit(&dtrace_lock);
6239 
6240 	return (id);
6241 }
6242 
6243 static dtrace_probe_t *
6244 dtrace_probe_lookup_id(dtrace_id_t id)
6245 {
6246 	ASSERT(MUTEX_HELD(&dtrace_lock));
6247 
6248 	if (id == 0 || id > dtrace_nprobes)
6249 		return (NULL);
6250 
6251 	return (dtrace_probes[id - 1]);
6252 }
6253 
6254 static int
6255 dtrace_probe_lookup_match(dtrace_probe_t *probe, void *arg)
6256 {
6257 	*((dtrace_id_t *)arg) = probe->dtpr_id;
6258 
6259 	return (DTRACE_MATCH_DONE);
6260 }
6261 
6262 /*
6263  * Look up a probe based on provider and one or more of module name, function
6264  * name and probe name.
6265  */
6266 dtrace_id_t
6267 dtrace_probe_lookup(dtrace_provider_id_t prid, const char *mod,
6268     const char *func, const char *name)
6269 {
6270 	dtrace_probekey_t pkey;
6271 	dtrace_id_t id;
6272 	int match;
6273 
6274 	pkey.dtpk_prov = ((dtrace_provider_t *)prid)->dtpv_name;
6275 	pkey.dtpk_pmatch = &dtrace_match_string;
6276 	pkey.dtpk_mod = mod;
6277 	pkey.dtpk_mmatch = mod ? &dtrace_match_string : &dtrace_match_nul;
6278 	pkey.dtpk_func = func;
6279 	pkey.dtpk_fmatch = func ? &dtrace_match_string : &dtrace_match_nul;
6280 	pkey.dtpk_name = name;
6281 	pkey.dtpk_nmatch = name ? &dtrace_match_string : &dtrace_match_nul;
6282 	pkey.dtpk_id = DTRACE_IDNONE;
6283 
6284 	mutex_enter(&dtrace_lock);
6285 	match = dtrace_match(&pkey, DTRACE_PRIV_ALL, 0,
6286 	    dtrace_probe_lookup_match, &id);
6287 	mutex_exit(&dtrace_lock);
6288 
6289 	ASSERT(match == 1 || match == 0);
6290 	return (match ? id : 0);
6291 }
6292 
6293 /*
6294  * Returns the probe argument associated with the specified probe.
6295  */
6296 void *
6297 dtrace_probe_arg(dtrace_provider_id_t id, dtrace_id_t pid)
6298 {
6299 	dtrace_probe_t *probe;
6300 	void *rval = NULL;
6301 
6302 	mutex_enter(&dtrace_lock);
6303 
6304 	if ((probe = dtrace_probe_lookup_id(pid)) != NULL &&
6305 	    probe->dtpr_provider == (dtrace_provider_t *)id)
6306 		rval = probe->dtpr_arg;
6307 
6308 	mutex_exit(&dtrace_lock);
6309 
6310 	return (rval);
6311 }
6312 
6313 /*
6314  * Copy a probe into a probe description.
6315  */
6316 static void
6317 dtrace_probe_description(const dtrace_probe_t *prp, dtrace_probedesc_t *pdp)
6318 {
6319 	bzero(pdp, sizeof (dtrace_probedesc_t));
6320 	pdp->dtpd_id = prp->dtpr_id;
6321 
6322 	(void) strncpy(pdp->dtpd_provider,
6323 	    prp->dtpr_provider->dtpv_name, DTRACE_PROVNAMELEN - 1);
6324 
6325 	(void) strncpy(pdp->dtpd_mod, prp->dtpr_mod, DTRACE_MODNAMELEN - 1);
6326 	(void) strncpy(pdp->dtpd_func, prp->dtpr_func, DTRACE_FUNCNAMELEN - 1);
6327 	(void) strncpy(pdp->dtpd_name, prp->dtpr_name, DTRACE_NAMELEN - 1);
6328 }
6329 
6330 /*
6331  * Called to indicate that a probe -- or probes -- should be provided by a
6332  * specfied provider.  If the specified description is NULL, the provider will
6333  * be told to provide all of its probes.  (This is done whenever a new
6334  * consumer comes along, or whenever a retained enabling is to be matched.) If
6335  * the specified description is non-NULL, the provider is given the
6336  * opportunity to dynamically provide the specified probe, allowing providers
6337  * to support the creation of probes on-the-fly.  (So-called _autocreated_
6338  * probes.)  If the provider is NULL, the operations will be applied to all
6339  * providers; if the provider is non-NULL the operations will only be applied
6340  * to the specified provider.  The dtrace_provider_lock must be held, and the
6341  * dtrace_lock must _not_ be held -- the provider's dtps_provide() operation
6342  * will need to grab the dtrace_lock when it reenters the framework through
6343  * dtrace_probe_lookup(), dtrace_probe_create(), etc.
6344  */
6345 static void
6346 dtrace_probe_provide(dtrace_probedesc_t *desc, dtrace_provider_t *prv)
6347 {
6348 	struct modctl *ctl;
6349 	int all = 0;
6350 
6351 	ASSERT(MUTEX_HELD(&dtrace_provider_lock));
6352 
6353 	if (prv == NULL) {
6354 		all = 1;
6355 		prv = dtrace_provider;
6356 	}
6357 
6358 	do {
6359 		/*
6360 		 * First, call the blanket provide operation.
6361 		 */
6362 		prv->dtpv_pops.dtps_provide(prv->dtpv_arg, desc);
6363 
6364 		/*
6365 		 * Now call the per-module provide operation.  We will grab
6366 		 * mod_lock to prevent the list from being modified.  Note
6367 		 * that this also prevents the mod_busy bits from changing.
6368 		 * (mod_busy can only be changed with mod_lock held.)
6369 		 */
6370 		mutex_enter(&mod_lock);
6371 
6372 		ctl = &modules;
6373 		do {
6374 			if (ctl->mod_busy || ctl->mod_mp == NULL)
6375 				continue;
6376 
6377 			prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
6378 
6379 		} while ((ctl = ctl->mod_next) != &modules);
6380 
6381 		mutex_exit(&mod_lock);
6382 	} while (all && (prv = prv->dtpv_next) != NULL);
6383 }
6384 
6385 /*
6386  * Iterate over each probe, and call the Framework-to-Provider API function
6387  * denoted by offs.
6388  */
6389 static void
6390 dtrace_probe_foreach(uintptr_t offs)
6391 {
6392 	dtrace_provider_t *prov;
6393 	void (*func)(void *, dtrace_id_t, void *);
6394 	dtrace_probe_t *probe;
6395 	dtrace_icookie_t cookie;
6396 	int i;
6397 
6398 	/*
6399 	 * We disable interrupts to walk through the probe array.  This is
6400 	 * safe -- the dtrace_sync() in dtrace_unregister() assures that we
6401 	 * won't see stale data.
6402 	 */
6403 	cookie = dtrace_interrupt_disable();
6404 
6405 	for (i = 0; i < dtrace_nprobes; i++) {
6406 		if ((probe = dtrace_probes[i]) == NULL)
6407 			continue;
6408 
6409 		if (probe->dtpr_ecb == NULL) {
6410 			/*
6411 			 * This probe isn't enabled -- don't call the function.
6412 			 */
6413 			continue;
6414 		}
6415 
6416 		prov = probe->dtpr_provider;
6417 		func = *((void(**)(void *, dtrace_id_t, void *))
6418 		    ((uintptr_t)&prov->dtpv_pops + offs));
6419 
6420 		func(prov->dtpv_arg, i + 1, probe->dtpr_arg);
6421 	}
6422 
6423 	dtrace_interrupt_enable(cookie);
6424 }
6425 
6426 static int
6427 dtrace_probe_enable(const dtrace_probedesc_t *desc, dtrace_enabling_t *enab)
6428 {
6429 	dtrace_probekey_t pkey;
6430 	uint32_t priv;
6431 	uid_t uid;
6432 
6433 	ASSERT(MUTEX_HELD(&dtrace_lock));
6434 	dtrace_ecb_create_cache = NULL;
6435 
6436 	if (desc == NULL) {
6437 		/*
6438 		 * If we're passed a NULL description, we're being asked to
6439 		 * create an ECB with a NULL probe.
6440 		 */
6441 		(void) dtrace_ecb_create_enable(NULL, enab);
6442 		return (0);
6443 	}
6444 
6445 	dtrace_probekey(desc, &pkey);
6446 	dtrace_cred2priv(CRED(), &priv, &uid);
6447 
6448 	return (dtrace_match(&pkey, priv, uid, dtrace_ecb_create_enable, enab));
6449 }
6450 
6451 /*
6452  * DTrace Helper Provider Functions
6453  */
6454 static void
6455 dtrace_dofattr2attr(dtrace_attribute_t *attr, const dof_attr_t dofattr)
6456 {
6457 	attr->dtat_name = DOF_ATTR_NAME(dofattr);
6458 	attr->dtat_data = DOF_ATTR_DATA(dofattr);
6459 	attr->dtat_class = DOF_ATTR_CLASS(dofattr);
6460 }
6461 
6462 static void
6463 dtrace_dofprov2hprov(dtrace_helper_provdesc_t *hprov,
6464     const dof_provider_t *dofprov, char *strtab)
6465 {
6466 	hprov->dthpv_provname = strtab + dofprov->dofpv_name;
6467 	dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_provider,
6468 	    dofprov->dofpv_provattr);
6469 	dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_mod,
6470 	    dofprov->dofpv_modattr);
6471 	dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_func,
6472 	    dofprov->dofpv_funcattr);
6473 	dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_name,
6474 	    dofprov->dofpv_nameattr);
6475 	dtrace_dofattr2attr(&hprov->dthpv_pattr.dtpa_args,
6476 	    dofprov->dofpv_argsattr);
6477 }
6478 
6479 static void
6480 dtrace_helper_provide_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
6481 {
6482 	uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
6483 	dof_hdr_t *dof = (dof_hdr_t *)daddr;
6484 	dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec;
6485 	dof_provider_t *provider;
6486 	dof_probe_t *probe;
6487 	uint32_t *off;
6488 	uint8_t *arg;
6489 	char *strtab;
6490 	uint_t i, nprobes;
6491 	dtrace_helper_provdesc_t dhpv;
6492 	dtrace_helper_probedesc_t dhpb;
6493 	dtrace_meta_t *meta = dtrace_meta_pid;
6494 	dtrace_mops_t *mops = &meta->dtm_mops;
6495 	void *parg;
6496 
6497 	provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
6498 	str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
6499 	    provider->dofpv_strtab * dof->dofh_secsize);
6500 	prb_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
6501 	    provider->dofpv_probes * dof->dofh_secsize);
6502 	arg_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
6503 	    provider->dofpv_prargs * dof->dofh_secsize);
6504 	off_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
6505 	    provider->dofpv_proffs * dof->dofh_secsize);
6506 
6507 	strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
6508 	off = (uint32_t *)(uintptr_t)(daddr + off_sec->dofs_offset);
6509 	arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
6510 
6511 	nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
6512 
6513 	/*
6514 	 * Create the provider.
6515 	 */
6516 	dtrace_dofprov2hprov(&dhpv, provider, strtab);
6517 
6518 	if ((parg = mops->dtms_provide_pid(meta->dtm_arg, &dhpv, pid)) == NULL)
6519 		return;
6520 
6521 	meta->dtm_count++;
6522 
6523 	/*
6524 	 * Create the probes.
6525 	 */
6526 	for (i = 0; i < nprobes; i++) {
6527 		probe = (dof_probe_t *)(uintptr_t)(daddr +
6528 		    prb_sec->dofs_offset + i * prb_sec->dofs_entsize);
6529 
6530 		dhpb.dthpb_mod = dhp->dofhp_mod;
6531 		dhpb.dthpb_func = strtab + probe->dofpr_func;
6532 		dhpb.dthpb_name = strtab + probe->dofpr_name;
6533 		dhpb.dthpb_base = probe->dofpr_addr;
6534 		dhpb.dthpb_offs = off + probe->dofpr_offidx;
6535 		dhpb.dthpb_noffs = probe->dofpr_noffs;
6536 		dhpb.dthpb_args = arg + probe->dofpr_argidx;
6537 		dhpb.dthpb_nargc = probe->dofpr_nargc;
6538 		dhpb.dthpb_xargc = probe->dofpr_xargc;
6539 		dhpb.dthpb_ntypes = strtab + probe->dofpr_nargv;
6540 		dhpb.dthpb_xtypes = strtab + probe->dofpr_xargv;
6541 
6542 		mops->dtms_create_probe(meta->dtm_arg, parg, &dhpb);
6543 	}
6544 }
6545 
6546 static void
6547 dtrace_helper_provide(dof_helper_t *dhp, pid_t pid)
6548 {
6549 	uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
6550 	dof_hdr_t *dof = (dof_hdr_t *)daddr;
6551 	int i;
6552 
6553 	ASSERT(MUTEX_HELD(&dtrace_meta_lock));
6554 
6555 	for (i = 0; i < dof->dofh_secnum; i++) {
6556 		dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
6557 		    dof->dofh_secoff + i * dof->dofh_secsize);
6558 
6559 		if (sec->dofs_type != DOF_SECT_PROVIDER)
6560 			continue;
6561 
6562 		dtrace_helper_provide_one(dhp, sec, pid);
6563 	}
6564 
6565 	/*
6566 	 * We may have just created probes, so we must now rematch against
6567 	 * any retained enablings.  Note that this call will acquire both
6568 	 * cpu_lock and dtrace_lock; the fact that we are holding
6569 	 * dtrace_meta_lock now is what defines the ordering with respect to
6570 	 * these three locks.
6571 	 */
6572 	dtrace_enabling_matchall();
6573 }
6574 
6575 static void
6576 dtrace_helper_remove_one(dof_helper_t *dhp, dof_sec_t *sec, pid_t pid)
6577 {
6578 	uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
6579 	dof_hdr_t *dof = (dof_hdr_t *)daddr;
6580 	dof_sec_t *str_sec;
6581 	dof_provider_t *provider;
6582 	char *strtab;
6583 	dtrace_helper_provdesc_t dhpv;
6584 	dtrace_meta_t *meta = dtrace_meta_pid;
6585 	dtrace_mops_t *mops = &meta->dtm_mops;
6586 
6587 	provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
6588 	str_sec = (dof_sec_t *)(uintptr_t)(daddr + dof->dofh_secoff +
6589 	    provider->dofpv_strtab * dof->dofh_secsize);
6590 
6591 	strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
6592 
6593 	/*
6594 	 * Create the provider.
6595 	 */
6596 	dtrace_dofprov2hprov(&dhpv, provider, strtab);
6597 
6598 	mops->dtms_remove_pid(meta->dtm_arg, &dhpv, pid);
6599 
6600 	meta->dtm_count--;
6601 }
6602 
6603 static void
6604 dtrace_helper_remove(dof_helper_t *dhp, pid_t pid)
6605 {
6606 	uintptr_t daddr = (uintptr_t)dhp->dofhp_dof;
6607 	dof_hdr_t *dof = (dof_hdr_t *)daddr;
6608 	int i;
6609 
6610 	ASSERT(MUTEX_HELD(&dtrace_meta_lock));
6611 
6612 	for (i = 0; i < dof->dofh_secnum; i++) {
6613 		dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
6614 		    dof->dofh_secoff + i * dof->dofh_secsize);
6615 
6616 		if (sec->dofs_type != DOF_SECT_PROVIDER)
6617 			continue;
6618 
6619 		dtrace_helper_remove_one(dhp, sec, pid);
6620 	}
6621 }
6622 
6623 /*
6624  * DTrace Meta Provider-to-Framework API Functions
6625  *
6626  * These functions implement the Meta Provider-to-Framework API, as described
6627  * in <sys/dtrace.h>.
6628  */
6629 int
6630 dtrace_meta_register(const char *name, const dtrace_mops_t *mops, void *arg,
6631     dtrace_meta_provider_id_t *idp)
6632 {
6633 	dtrace_meta_t *meta;
6634 	dtrace_helpers_t *help, *next;
6635 	int i;
6636 
6637 	*idp = DTRACE_METAPROVNONE;
6638 
6639 	/*
6640 	 * We strictly don't need the name, but we hold onto it for
6641 	 * debuggability. All hail error queues!
6642 	 */
6643 	if (name == NULL) {
6644 		cmn_err(CE_WARN, "failed to register meta-provider: "
6645 		    "invalid name");
6646 		return (EINVAL);
6647 	}
6648 
6649 	if (mops == NULL ||
6650 	    mops->dtms_create_probe == NULL ||
6651 	    mops->dtms_provide_pid == NULL ||
6652 	    mops->dtms_remove_pid == NULL) {
6653 		cmn_err(CE_WARN, "failed to register meta-register %s: "
6654 		    "invalid ops", name);
6655 		return (EINVAL);
6656 	}
6657 
6658 	meta = kmem_zalloc(sizeof (dtrace_meta_t), KM_SLEEP);
6659 	meta->dtm_mops = *mops;
6660 	meta->dtm_name = kmem_alloc(strlen(name) + 1, KM_SLEEP);
6661 	(void) strcpy(meta->dtm_name, name);
6662 	meta->dtm_arg = arg;
6663 
6664 	mutex_enter(&dtrace_meta_lock);
6665 	mutex_enter(&dtrace_lock);
6666 
6667 	if (dtrace_meta_pid != NULL) {
6668 		mutex_exit(&dtrace_lock);
6669 		mutex_exit(&dtrace_meta_lock);
6670 		cmn_err(CE_WARN, "failed to register meta-register %s: "
6671 		    "user-land meta-provider exists", name);
6672 		kmem_free(meta->dtm_name, strlen(meta->dtm_name) + 1);
6673 		kmem_free(meta, sizeof (dtrace_meta_t));
6674 		return (EINVAL);
6675 	}
6676 
6677 	dtrace_meta_pid = meta;
6678 	*idp = (dtrace_meta_provider_id_t)meta;
6679 
6680 	/*
6681 	 * If there are providers and probes ready to go, pass them
6682 	 * off to the new meta provider now.
6683 	 */
6684 
6685 	help = dtrace_deferred_pid;
6686 	dtrace_deferred_pid = NULL;
6687 
6688 	mutex_exit(&dtrace_lock);
6689 
6690 	while (help != NULL) {
6691 		for (i = 0; i < help->dthps_nprovs; i++) {
6692 			dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
6693 			    help->dthps_pid);
6694 		}
6695 
6696 		next = help->dthps_next;
6697 		help->dthps_next = NULL;
6698 		help->dthps_prev = NULL;
6699 		help = next;
6700 	}
6701 
6702 	mutex_exit(&dtrace_meta_lock);
6703 
6704 	return (0);
6705 }
6706 
6707 int
6708 dtrace_meta_unregister(dtrace_meta_provider_id_t id)
6709 {
6710 	dtrace_meta_t **pp, *old = (dtrace_meta_t *)id;
6711 
6712 	mutex_enter(&dtrace_meta_lock);
6713 	mutex_enter(&dtrace_lock);
6714 
6715 	if (old == dtrace_meta_pid) {
6716 		pp = &dtrace_meta_pid;
6717 	} else {
6718 		panic("attempt to unregister non-existent "
6719 		    "dtrace meta-provider %p\n", (void *)old);
6720 	}
6721 
6722 	if (old->dtm_count != 0) {
6723 		mutex_exit(&dtrace_lock);
6724 		mutex_exit(&dtrace_meta_lock);
6725 		return (EBUSY);
6726 	}
6727 
6728 	*pp = NULL;
6729 
6730 	mutex_exit(&dtrace_lock);
6731 	mutex_exit(&dtrace_meta_lock);
6732 
6733 	kmem_free(old->dtm_name, strlen(old->dtm_name) + 1);
6734 	kmem_free(old, sizeof (dtrace_meta_t));
6735 
6736 	return (0);
6737 }
6738 
6739 
6740 /*
6741  * DTrace DIF Object Functions
6742  */
6743 static int
6744 dtrace_difo_err(uint_t pc, const char *format, ...)
6745 {
6746 	if (dtrace_err_verbose) {
6747 		va_list alist;
6748 
6749 		(void) uprintf("dtrace DIF object error: [%u]: ", pc);
6750 		va_start(alist, format);
6751 		(void) vuprintf(format, alist);
6752 		va_end(alist);
6753 	}
6754 
6755 #ifdef DTRACE_ERRDEBUG
6756 	dtrace_errdebug(format);
6757 #endif
6758 	return (1);
6759 }
6760 
6761 /*
6762  * Validate a DTrace DIF object by checking the IR instructions.  The following
6763  * rules are currently enforced by dtrace_difo_validate():
6764  *
6765  * 1. Each instruction must have a valid opcode
6766  * 2. Each register, string, variable, or subroutine reference must be valid
6767  * 3. No instruction can modify register %r0 (must be zero)
6768  * 4. All instruction reserved bits must be set to zero
6769  * 5. The last instruction must be a "ret" instruction
6770  * 6. All branch targets must reference a valid instruction _after_ the branch
6771  */
6772 static int
6773 dtrace_difo_validate(dtrace_difo_t *dp, dtrace_vstate_t *vstate, uint_t nregs,
6774     cred_t *cr)
6775 {
6776 	int err = 0, i;
6777 	int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
6778 	int kcheck;
6779 	uint_t pc;
6780 
6781 	kcheck = cr == NULL ||
6782 	    PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE) == 0;
6783 
6784 	dp->dtdo_destructive = 0;
6785 
6786 	for (pc = 0; pc < dp->dtdo_len && err == 0; pc++) {
6787 		dif_instr_t instr = dp->dtdo_buf[pc];
6788 
6789 		uint_t r1 = DIF_INSTR_R1(instr);
6790 		uint_t r2 = DIF_INSTR_R2(instr);
6791 		uint_t rd = DIF_INSTR_RD(instr);
6792 		uint_t rs = DIF_INSTR_RS(instr);
6793 		uint_t label = DIF_INSTR_LABEL(instr);
6794 		uint_t v = DIF_INSTR_VAR(instr);
6795 		uint_t subr = DIF_INSTR_SUBR(instr);
6796 		uint_t type = DIF_INSTR_TYPE(instr);
6797 		uint_t op = DIF_INSTR_OP(instr);
6798 
6799 		switch (op) {
6800 		case DIF_OP_OR:
6801 		case DIF_OP_XOR:
6802 		case DIF_OP_AND:
6803 		case DIF_OP_SLL:
6804 		case DIF_OP_SRL:
6805 		case DIF_OP_SRA:
6806 		case DIF_OP_SUB:
6807 		case DIF_OP_ADD:
6808 		case DIF_OP_MUL:
6809 		case DIF_OP_SDIV:
6810 		case DIF_OP_UDIV:
6811 		case DIF_OP_SREM:
6812 		case DIF_OP_UREM:
6813 		case DIF_OP_COPYS:
6814 			if (r1 >= nregs)
6815 				err += efunc(pc, "invalid register %u\n", r1);
6816 			if (r2 >= nregs)
6817 				err += efunc(pc, "invalid register %u\n", r2);
6818 			if (rd >= nregs)
6819 				err += efunc(pc, "invalid register %u\n", rd);
6820 			if (rd == 0)
6821 				err += efunc(pc, "cannot write to %r0\n");
6822 			break;
6823 		case DIF_OP_NOT:
6824 		case DIF_OP_MOV:
6825 		case DIF_OP_ALLOCS:
6826 			if (r1 >= nregs)
6827 				err += efunc(pc, "invalid register %u\n", r1);
6828 			if (r2 != 0)
6829 				err += efunc(pc, "non-zero reserved bits\n");
6830 			if (rd >= nregs)
6831 				err += efunc(pc, "invalid register %u\n", rd);
6832 			if (rd == 0)
6833 				err += efunc(pc, "cannot write to %r0\n");
6834 			break;
6835 		case DIF_OP_LDSB:
6836 		case DIF_OP_LDSH:
6837 		case DIF_OP_LDSW:
6838 		case DIF_OP_LDUB:
6839 		case DIF_OP_LDUH:
6840 		case DIF_OP_LDUW:
6841 		case DIF_OP_LDX:
6842 			if (r1 >= nregs)
6843 				err += efunc(pc, "invalid register %u\n", r1);
6844 			if (r2 != 0)
6845 				err += efunc(pc, "non-zero reserved bits\n");
6846 			if (rd >= nregs)
6847 				err += efunc(pc, "invalid register %u\n", rd);
6848 			if (rd == 0)
6849 				err += efunc(pc, "cannot write to %r0\n");
6850 			if (kcheck)
6851 				dp->dtdo_buf[pc] = DIF_INSTR_LOAD(op +
6852 				    DIF_OP_RLDSB - DIF_OP_LDSB, r1, rd);
6853 			break;
6854 		case DIF_OP_RLDSB:
6855 		case DIF_OP_RLDSH:
6856 		case DIF_OP_RLDSW:
6857 		case DIF_OP_RLDUB:
6858 		case DIF_OP_RLDUH:
6859 		case DIF_OP_RLDUW:
6860 		case DIF_OP_RLDX:
6861 			if (r1 >= nregs)
6862 				err += efunc(pc, "invalid register %u\n", r1);
6863 			if (r2 != 0)
6864 				err += efunc(pc, "non-zero reserved bits\n");
6865 			if (rd >= nregs)
6866 				err += efunc(pc, "invalid register %u\n", rd);
6867 			if (rd == 0)
6868 				err += efunc(pc, "cannot write to %r0\n");
6869 			break;
6870 		case DIF_OP_ULDSB:
6871 		case DIF_OP_ULDSH:
6872 		case DIF_OP_ULDSW:
6873 		case DIF_OP_ULDUB:
6874 		case DIF_OP_ULDUH:
6875 		case DIF_OP_ULDUW:
6876 		case DIF_OP_ULDX:
6877 			if (r1 >= nregs)
6878 				err += efunc(pc, "invalid register %u\n", r1);
6879 			if (r2 != 0)
6880 				err += efunc(pc, "non-zero reserved bits\n");
6881 			if (rd >= nregs)
6882 				err += efunc(pc, "invalid register %u\n", rd);
6883 			if (rd == 0)
6884 				err += efunc(pc, "cannot write to %r0\n");
6885 			break;
6886 		case DIF_OP_STB:
6887 		case DIF_OP_STH:
6888 		case DIF_OP_STW:
6889 		case DIF_OP_STX:
6890 			if (r1 >= nregs)
6891 				err += efunc(pc, "invalid register %u\n", r1);
6892 			if (r2 != 0)
6893 				err += efunc(pc, "non-zero reserved bits\n");
6894 			if (rd >= nregs)
6895 				err += efunc(pc, "invalid register %u\n", rd);
6896 			if (rd == 0)
6897 				err += efunc(pc, "cannot write to 0 address\n");
6898 			break;
6899 		case DIF_OP_CMP:
6900 		case DIF_OP_SCMP:
6901 			if (r1 >= nregs)
6902 				err += efunc(pc, "invalid register %u\n", r1);
6903 			if (r2 >= nregs)
6904 				err += efunc(pc, "invalid register %u\n", r2);
6905 			if (rd != 0)
6906 				err += efunc(pc, "non-zero reserved bits\n");
6907 			break;
6908 		case DIF_OP_TST:
6909 			if (r1 >= nregs)
6910 				err += efunc(pc, "invalid register %u\n", r1);
6911 			if (r2 != 0 || rd != 0)
6912 				err += efunc(pc, "non-zero reserved bits\n");
6913 			break;
6914 		case DIF_OP_BA:
6915 		case DIF_OP_BE:
6916 		case DIF_OP_BNE:
6917 		case DIF_OP_BG:
6918 		case DIF_OP_BGU:
6919 		case DIF_OP_BGE:
6920 		case DIF_OP_BGEU:
6921 		case DIF_OP_BL:
6922 		case DIF_OP_BLU:
6923 		case DIF_OP_BLE:
6924 		case DIF_OP_BLEU:
6925 			if (label >= dp->dtdo_len) {
6926 				err += efunc(pc, "invalid branch target %u\n",
6927 				    label);
6928 			}
6929 			if (label <= pc) {
6930 				err += efunc(pc, "backward branch to %u\n",
6931 				    label);
6932 			}
6933 			break;
6934 		case DIF_OP_RET:
6935 			if (r1 != 0 || r2 != 0)
6936 				err += efunc(pc, "non-zero reserved bits\n");
6937 			if (rd >= nregs)
6938 				err += efunc(pc, "invalid register %u\n", rd);
6939 			break;
6940 		case DIF_OP_NOP:
6941 		case DIF_OP_POPTS:
6942 		case DIF_OP_FLUSHTS:
6943 			if (r1 != 0 || r2 != 0 || rd != 0)
6944 				err += efunc(pc, "non-zero reserved bits\n");
6945 			break;
6946 		case DIF_OP_SETX:
6947 			if (DIF_INSTR_INTEGER(instr) >= dp->dtdo_intlen) {
6948 				err += efunc(pc, "invalid integer ref %u\n",
6949 				    DIF_INSTR_INTEGER(instr));
6950 			}
6951 			if (rd >= nregs)
6952 				err += efunc(pc, "invalid register %u\n", rd);
6953 			if (rd == 0)
6954 				err += efunc(pc, "cannot write to %r0\n");
6955 			break;
6956 		case DIF_OP_SETS:
6957 			if (DIF_INSTR_STRING(instr) >= dp->dtdo_strlen) {
6958 				err += efunc(pc, "invalid string ref %u\n",
6959 				    DIF_INSTR_STRING(instr));
6960 			}
6961 			if (rd >= nregs)
6962 				err += efunc(pc, "invalid register %u\n", rd);
6963 			if (rd == 0)
6964 				err += efunc(pc, "cannot write to %r0\n");
6965 			break;
6966 		case DIF_OP_LDGA:
6967 		case DIF_OP_LDTA:
6968 			if (r1 > DIF_VAR_ARRAY_MAX)
6969 				err += efunc(pc, "invalid array %u\n", r1);
6970 			if (r2 >= nregs)
6971 				err += efunc(pc, "invalid register %u\n", r2);
6972 			if (rd >= nregs)
6973 				err += efunc(pc, "invalid register %u\n", rd);
6974 			if (rd == 0)
6975 				err += efunc(pc, "cannot write to %r0\n");
6976 			break;
6977 		case DIF_OP_LDGS:
6978 		case DIF_OP_LDTS:
6979 		case DIF_OP_LDLS:
6980 		case DIF_OP_LDGAA:
6981 		case DIF_OP_LDTAA:
6982 			if (v < DIF_VAR_OTHER_MIN || v > DIF_VAR_OTHER_MAX)
6983 				err += efunc(pc, "invalid variable %u\n", v);
6984 			if (rd >= nregs)
6985 				err += efunc(pc, "invalid register %u\n", rd);
6986 			if (rd == 0)
6987 				err += efunc(pc, "cannot write to %r0\n");
6988 			break;
6989 		case DIF_OP_STGS:
6990 		case DIF_OP_STTS:
6991 		case DIF_OP_STLS:
6992 		case DIF_OP_STGAA:
6993 		case DIF_OP_STTAA:
6994 			if (v < DIF_VAR_OTHER_UBASE || v > DIF_VAR_OTHER_MAX)
6995 				err += efunc(pc, "invalid variable %u\n", v);
6996 			if (rs >= nregs)
6997 				err += efunc(pc, "invalid register %u\n", rd);
6998 			break;
6999 		case DIF_OP_CALL:
7000 			if (subr > DIF_SUBR_MAX)
7001 				err += efunc(pc, "invalid subr %u\n", subr);
7002 			if (rd >= nregs)
7003 				err += efunc(pc, "invalid register %u\n", rd);
7004 			if (rd == 0)
7005 				err += efunc(pc, "cannot write to %r0\n");
7006 
7007 			if (subr == DIF_SUBR_COPYOUT ||
7008 			    subr == DIF_SUBR_COPYOUTSTR) {
7009 				dp->dtdo_destructive = 1;
7010 			}
7011 			break;
7012 		case DIF_OP_PUSHTR:
7013 			if (type != DIF_TYPE_STRING && type != DIF_TYPE_CTF)
7014 				err += efunc(pc, "invalid ref type %u\n", type);
7015 			if (r2 >= nregs)
7016 				err += efunc(pc, "invalid register %u\n", r2);
7017 			if (rs >= nregs)
7018 				err += efunc(pc, "invalid register %u\n", rs);
7019 			break;
7020 		case DIF_OP_PUSHTV:
7021 			if (type != DIF_TYPE_CTF)
7022 				err += efunc(pc, "invalid val type %u\n", type);
7023 			if (r2 >= nregs)
7024 				err += efunc(pc, "invalid register %u\n", r2);
7025 			if (rs >= nregs)
7026 				err += efunc(pc, "invalid register %u\n", rs);
7027 			break;
7028 		default:
7029 			err += efunc(pc, "invalid opcode %u\n",
7030 			    DIF_INSTR_OP(instr));
7031 		}
7032 	}
7033 
7034 	if (dp->dtdo_len != 0 &&
7035 	    DIF_INSTR_OP(dp->dtdo_buf[dp->dtdo_len - 1]) != DIF_OP_RET) {
7036 		err += efunc(dp->dtdo_len - 1,
7037 		    "expected 'ret' as last DIF instruction\n");
7038 	}
7039 
7040 	if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF)) {
7041 		/*
7042 		 * If we're not returning by reference, the size must be either
7043 		 * 0 or the size of one of the base types.
7044 		 */
7045 		switch (dp->dtdo_rtype.dtdt_size) {
7046 		case 0:
7047 		case sizeof (uint8_t):
7048 		case sizeof (uint16_t):
7049 		case sizeof (uint32_t):
7050 		case sizeof (uint64_t):
7051 			break;
7052 
7053 		default:
7054 			err += efunc(dp->dtdo_len - 1, "bad return size");
7055 		}
7056 	}
7057 
7058 	for (i = 0; i < dp->dtdo_varlen && err == 0; i++) {
7059 		dtrace_difv_t *v = &dp->dtdo_vartab[i], *existing = NULL;
7060 		dtrace_diftype_t *vt, *et;
7061 		uint_t id, ndx;
7062 
7063 		if (v->dtdv_scope != DIFV_SCOPE_GLOBAL &&
7064 		    v->dtdv_scope != DIFV_SCOPE_THREAD &&
7065 		    v->dtdv_scope != DIFV_SCOPE_LOCAL) {
7066 			err += efunc(i, "unrecognized variable scope %d\n",
7067 			    v->dtdv_scope);
7068 			break;
7069 		}
7070 
7071 		if (v->dtdv_kind != DIFV_KIND_ARRAY &&
7072 		    v->dtdv_kind != DIFV_KIND_SCALAR) {
7073 			err += efunc(i, "unrecognized variable type %d\n",
7074 			    v->dtdv_kind);
7075 			break;
7076 		}
7077 
7078 		if ((id = v->dtdv_id) > DIF_VARIABLE_MAX) {
7079 			err += efunc(i, "%d exceeds variable id limit\n", id);
7080 			break;
7081 		}
7082 
7083 		if (id < DIF_VAR_OTHER_UBASE)
7084 			continue;
7085 
7086 		/*
7087 		 * For user-defined variables, we need to check that this
7088 		 * definition is identical to any previous definition that we
7089 		 * encountered.
7090 		 */
7091 		ndx = id - DIF_VAR_OTHER_UBASE;
7092 
7093 		switch (v->dtdv_scope) {
7094 		case DIFV_SCOPE_GLOBAL:
7095 			if (ndx < vstate->dtvs_nglobals) {
7096 				dtrace_statvar_t *svar;
7097 
7098 				if ((svar = vstate->dtvs_globals[ndx]) != NULL)
7099 					existing = &svar->dtsv_var;
7100 			}
7101 
7102 			break;
7103 
7104 		case DIFV_SCOPE_THREAD:
7105 			if (ndx < vstate->dtvs_ntlocals)
7106 				existing = &vstate->dtvs_tlocals[ndx];
7107 			break;
7108 
7109 		case DIFV_SCOPE_LOCAL:
7110 			if (ndx < vstate->dtvs_nlocals) {
7111 				dtrace_statvar_t *svar;
7112 
7113 				if ((svar = vstate->dtvs_locals[ndx]) != NULL)
7114 					existing = &svar->dtsv_var;
7115 			}
7116 
7117 			break;
7118 		}
7119 
7120 		vt = &v->dtdv_type;
7121 
7122 		if (vt->dtdt_flags & DIF_TF_BYREF) {
7123 			if (vt->dtdt_size == 0) {
7124 				err += efunc(i, "zero-sized variable\n");
7125 				break;
7126 			}
7127 
7128 			if (v->dtdv_scope == DIFV_SCOPE_GLOBAL &&
7129 			    vt->dtdt_size > dtrace_global_maxsize) {
7130 				err += efunc(i, "oversized by-ref global\n");
7131 				break;
7132 			}
7133 		}
7134 
7135 		if (existing == NULL || existing->dtdv_id == 0)
7136 			continue;
7137 
7138 		ASSERT(existing->dtdv_id == v->dtdv_id);
7139 		ASSERT(existing->dtdv_scope == v->dtdv_scope);
7140 
7141 		if (existing->dtdv_kind != v->dtdv_kind)
7142 			err += efunc(i, "%d changed variable kind\n", id);
7143 
7144 		et = &existing->dtdv_type;
7145 
7146 		if (vt->dtdt_flags != et->dtdt_flags) {
7147 			err += efunc(i, "%d changed variable type flags\n", id);
7148 			break;
7149 		}
7150 
7151 		if (vt->dtdt_size != 0 && vt->dtdt_size != et->dtdt_size) {
7152 			err += efunc(i, "%d changed variable type size\n", id);
7153 			break;
7154 		}
7155 	}
7156 
7157 	return (err);
7158 }
7159 
7160 /*
7161  * Validate a DTrace DIF object that it is to be used as a helper.  Helpers
7162  * are much more constrained than normal DIFOs.  Specifically, they may
7163  * not:
7164  *
7165  * 1. Make calls to subroutines other than copyin(), copyinstr() or
7166  *    miscellaneous string routines
7167  * 2. Access DTrace variables other than the args[] array, and the
7168  *    curthread, pid, tid and execname variables.
7169  * 3. Have thread-local variables.
7170  * 4. Have dynamic variables.
7171  */
7172 static int
7173 dtrace_difo_validate_helper(dtrace_difo_t *dp)
7174 {
7175 	int (*efunc)(uint_t pc, const char *, ...) = dtrace_difo_err;
7176 	int err = 0;
7177 	uint_t pc;
7178 
7179 	for (pc = 0; pc < dp->dtdo_len; pc++) {
7180 		dif_instr_t instr = dp->dtdo_buf[pc];
7181 
7182 		uint_t v = DIF_INSTR_VAR(instr);
7183 		uint_t subr = DIF_INSTR_SUBR(instr);
7184 		uint_t op = DIF_INSTR_OP(instr);
7185 
7186 		switch (op) {
7187 		case DIF_OP_OR:
7188 		case DIF_OP_XOR:
7189 		case DIF_OP_AND:
7190 		case DIF_OP_SLL:
7191 		case DIF_OP_SRL:
7192 		case DIF_OP_SRA:
7193 		case DIF_OP_SUB:
7194 		case DIF_OP_ADD:
7195 		case DIF_OP_MUL:
7196 		case DIF_OP_SDIV:
7197 		case DIF_OP_UDIV:
7198 		case DIF_OP_SREM:
7199 		case DIF_OP_UREM:
7200 		case DIF_OP_COPYS:
7201 		case DIF_OP_NOT:
7202 		case DIF_OP_MOV:
7203 		case DIF_OP_RLDSB:
7204 		case DIF_OP_RLDSH:
7205 		case DIF_OP_RLDSW:
7206 		case DIF_OP_RLDUB:
7207 		case DIF_OP_RLDUH:
7208 		case DIF_OP_RLDUW:
7209 		case DIF_OP_RLDX:
7210 		case DIF_OP_ULDSB:
7211 		case DIF_OP_ULDSH:
7212 		case DIF_OP_ULDSW:
7213 		case DIF_OP_ULDUB:
7214 		case DIF_OP_ULDUH:
7215 		case DIF_OP_ULDUW:
7216 		case DIF_OP_ULDX:
7217 		case DIF_OP_STB:
7218 		case DIF_OP_STH:
7219 		case DIF_OP_STW:
7220 		case DIF_OP_STX:
7221 		case DIF_OP_ALLOCS:
7222 		case DIF_OP_CMP:
7223 		case DIF_OP_SCMP:
7224 		case DIF_OP_TST:
7225 		case DIF_OP_BA:
7226 		case DIF_OP_BE:
7227 		case DIF_OP_BNE:
7228 		case DIF_OP_BG:
7229 		case DIF_OP_BGU:
7230 		case DIF_OP_BGE:
7231 		case DIF_OP_BGEU:
7232 		case DIF_OP_BL:
7233 		case DIF_OP_BLU:
7234 		case DIF_OP_BLE:
7235 		case DIF_OP_BLEU:
7236 		case DIF_OP_RET:
7237 		case DIF_OP_NOP:
7238 		case DIF_OP_POPTS:
7239 		case DIF_OP_FLUSHTS:
7240 		case DIF_OP_SETX:
7241 		case DIF_OP_SETS:
7242 		case DIF_OP_LDGA:
7243 		case DIF_OP_LDLS:
7244 		case DIF_OP_STGS:
7245 		case DIF_OP_STLS:
7246 		case DIF_OP_PUSHTR:
7247 		case DIF_OP_PUSHTV:
7248 			break;
7249 
7250 		case DIF_OP_LDGS:
7251 			if (v >= DIF_VAR_OTHER_UBASE)
7252 				break;
7253 
7254 			if (v >= DIF_VAR_ARG0 && v <= DIF_VAR_ARG9)
7255 				break;
7256 
7257 			if (v == DIF_VAR_CURTHREAD || v == DIF_VAR_PID ||
7258 			    v == DIF_VAR_TID || v == DIF_VAR_EXECNAME ||
7259 			    v == DIF_VAR_ZONENAME)
7260 				break;
7261 
7262 			err += efunc(pc, "illegal variable %u\n", v);
7263 			break;
7264 
7265 		case DIF_OP_LDTA:
7266 		case DIF_OP_LDTS:
7267 		case DIF_OP_LDGAA:
7268 		case DIF_OP_LDTAA:
7269 			err += efunc(pc, "illegal dynamic variable load\n");
7270 			break;
7271 
7272 		case DIF_OP_STTS:
7273 		case DIF_OP_STGAA:
7274 		case DIF_OP_STTAA:
7275 			err += efunc(pc, "illegal dynamic variable store\n");
7276 			break;
7277 
7278 		case DIF_OP_CALL:
7279 			if (subr == DIF_SUBR_ALLOCA ||
7280 			    subr == DIF_SUBR_BCOPY ||
7281 			    subr == DIF_SUBR_COPYIN ||
7282 			    subr == DIF_SUBR_COPYINTO ||
7283 			    subr == DIF_SUBR_COPYINSTR ||
7284 			    subr == DIF_SUBR_INDEX ||
7285 			    subr == DIF_SUBR_LLTOSTR ||
7286 			    subr == DIF_SUBR_RINDEX ||
7287 			    subr == DIF_SUBR_STRCHR ||
7288 			    subr == DIF_SUBR_STRJOIN ||
7289 			    subr == DIF_SUBR_STRRCHR ||
7290 			    subr == DIF_SUBR_STRSTR)
7291 				break;
7292 
7293 			err += efunc(pc, "invalid subr %u\n", subr);
7294 			break;
7295 
7296 		default:
7297 			err += efunc(pc, "invalid opcode %u\n",
7298 			    DIF_INSTR_OP(instr));
7299 		}
7300 	}
7301 
7302 	return (err);
7303 }
7304 
7305 /*
7306  * Returns 1 if the expression in the DIF object can be cached on a per-thread
7307  * basis; 0 if not.
7308  */
7309 static int
7310 dtrace_difo_cacheable(dtrace_difo_t *dp)
7311 {
7312 	int i;
7313 
7314 	if (dp == NULL)
7315 		return (0);
7316 
7317 	for (i = 0; i < dp->dtdo_varlen; i++) {
7318 		dtrace_difv_t *v = &dp->dtdo_vartab[i];
7319 
7320 		if (v->dtdv_scope != DIFV_SCOPE_GLOBAL)
7321 			continue;
7322 
7323 		switch (v->dtdv_id) {
7324 		case DIF_VAR_CURTHREAD:
7325 		case DIF_VAR_PID:
7326 		case DIF_VAR_TID:
7327 		case DIF_VAR_EXECNAME:
7328 		case DIF_VAR_ZONENAME:
7329 			break;
7330 
7331 		default:
7332 			return (0);
7333 		}
7334 	}
7335 
7336 	/*
7337 	 * This DIF object may be cacheable.  Now we need to look for any
7338 	 * array loading instructions, any memory loading instructions, or
7339 	 * any stores to thread-local variables.
7340 	 */
7341 	for (i = 0; i < dp->dtdo_len; i++) {
7342 		uint_t op = DIF_INSTR_OP(dp->dtdo_buf[i]);
7343 
7344 		if ((op >= DIF_OP_LDSB && op <= DIF_OP_LDX) ||
7345 		    (op >= DIF_OP_ULDSB && op <= DIF_OP_ULDX) ||
7346 		    (op >= DIF_OP_RLDSB && op <= DIF_OP_RLDX) ||
7347 		    op == DIF_OP_LDGA || op == DIF_OP_STTS)
7348 			return (0);
7349 	}
7350 
7351 	return (1);
7352 }
7353 
7354 static void
7355 dtrace_difo_hold(dtrace_difo_t *dp)
7356 {
7357 	int i;
7358 
7359 	ASSERT(MUTEX_HELD(&dtrace_lock));
7360 
7361 	dp->dtdo_refcnt++;
7362 	ASSERT(dp->dtdo_refcnt != 0);
7363 
7364 	/*
7365 	 * We need to check this DIF object for references to the variable
7366 	 * DIF_VAR_VTIMESTAMP.
7367 	 */
7368 	for (i = 0; i < dp->dtdo_varlen; i++) {
7369 		dtrace_difv_t *v = &dp->dtdo_vartab[i];
7370 
7371 		if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
7372 			continue;
7373 
7374 		if (dtrace_vtime_references++ == 0)
7375 			dtrace_vtime_enable();
7376 	}
7377 }
7378 
7379 /*
7380  * This routine calculates the dynamic variable chunksize for a given DIF
7381  * object.  The calculation is not fool-proof, and can probably be tricked by
7382  * malicious DIF -- but it works for all compiler-generated DIF.  Because this
7383  * calculation is likely imperfect, dtrace_dynvar() is able to gracefully fail
7384  * if a dynamic variable size exceeds the chunksize.
7385  */
7386 static void
7387 dtrace_difo_chunksize(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
7388 {
7389 	uint64_t sval;
7390 	dtrace_key_t tupregs[DIF_DTR_NREGS + 2]; /* +2 for thread and id */
7391 	const dif_instr_t *text = dp->dtdo_buf;
7392 	uint_t pc, srd = 0;
7393 	uint_t ttop = 0;
7394 	size_t size, ksize;
7395 	uint_t id, i;
7396 
7397 	for (pc = 0; pc < dp->dtdo_len; pc++) {
7398 		dif_instr_t instr = text[pc];
7399 		uint_t op = DIF_INSTR_OP(instr);
7400 		uint_t rd = DIF_INSTR_RD(instr);
7401 		uint_t r1 = DIF_INSTR_R1(instr);
7402 		uint_t nkeys = 0;
7403 		uchar_t scope;
7404 
7405 		dtrace_key_t *key = tupregs;
7406 
7407 		switch (op) {
7408 		case DIF_OP_SETX:
7409 			sval = dp->dtdo_inttab[DIF_INSTR_INTEGER(instr)];
7410 			srd = rd;
7411 			continue;
7412 
7413 		case DIF_OP_STTS:
7414 			key = &tupregs[DIF_DTR_NREGS];
7415 			key[0].dttk_size = 0;
7416 			key[1].dttk_size = 0;
7417 			nkeys = 2;
7418 			scope = DIFV_SCOPE_THREAD;
7419 			break;
7420 
7421 		case DIF_OP_STGAA:
7422 		case DIF_OP_STTAA:
7423 			nkeys = ttop;
7424 
7425 			if (DIF_INSTR_OP(instr) == DIF_OP_STTAA)
7426 				key[nkeys++].dttk_size = 0;
7427 
7428 			key[nkeys++].dttk_size = 0;
7429 
7430 			if (op == DIF_OP_STTAA) {
7431 				scope = DIFV_SCOPE_THREAD;
7432 			} else {
7433 				scope = DIFV_SCOPE_GLOBAL;
7434 			}
7435 
7436 			break;
7437 
7438 		case DIF_OP_PUSHTR:
7439 			if (ttop == DIF_DTR_NREGS)
7440 				return;
7441 
7442 			if ((srd == 0 || sval == 0) && r1 == DIF_TYPE_STRING) {
7443 				/*
7444 				 * If the register for the size of the "pushtr"
7445 				 * is %r0 (or the value is 0) and the type is
7446 				 * a string, we'll use the system-wide default
7447 				 * string size.
7448 				 */
7449 				tupregs[ttop++].dttk_size =
7450 				    dtrace_strsize_default;
7451 			} else {
7452 				if (srd == 0)
7453 					return;
7454 
7455 				tupregs[ttop++].dttk_size = sval;
7456 			}
7457 
7458 			break;
7459 
7460 		case DIF_OP_PUSHTV:
7461 			if (ttop == DIF_DTR_NREGS)
7462 				return;
7463 
7464 			tupregs[ttop++].dttk_size = 0;
7465 			break;
7466 
7467 		case DIF_OP_FLUSHTS:
7468 			ttop = 0;
7469 			break;
7470 
7471 		case DIF_OP_POPTS:
7472 			if (ttop != 0)
7473 				ttop--;
7474 			break;
7475 		}
7476 
7477 		sval = 0;
7478 		srd = 0;
7479 
7480 		if (nkeys == 0)
7481 			continue;
7482 
7483 		/*
7484 		 * We have a dynamic variable allocation; calculate its size.
7485 		 */
7486 		for (ksize = 0, i = 0; i < nkeys; i++)
7487 			ksize += P2ROUNDUP(key[i].dttk_size, sizeof (uint64_t));
7488 
7489 		size = sizeof (dtrace_dynvar_t);
7490 		size += sizeof (dtrace_key_t) * (nkeys - 1);
7491 		size += ksize;
7492 
7493 		/*
7494 		 * Now we need to determine the size of the stored data.
7495 		 */
7496 		id = DIF_INSTR_VAR(instr);
7497 
7498 		for (i = 0; i < dp->dtdo_varlen; i++) {
7499 			dtrace_difv_t *v = &dp->dtdo_vartab[i];
7500 
7501 			if (v->dtdv_id == id && v->dtdv_scope == scope) {
7502 				size += v->dtdv_type.dtdt_size;
7503 				break;
7504 			}
7505 		}
7506 
7507 		if (i == dp->dtdo_varlen)
7508 			return;
7509 
7510 		/*
7511 		 * We have the size.  If this is larger than the chunk size
7512 		 * for our dynamic variable state, reset the chunk size.
7513 		 */
7514 		size = P2ROUNDUP(size, sizeof (uint64_t));
7515 
7516 		if (size > vstate->dtvs_dynvars.dtds_chunksize)
7517 			vstate->dtvs_dynvars.dtds_chunksize = size;
7518 	}
7519 }
7520 
7521 static void
7522 dtrace_difo_init(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
7523 {
7524 	int i, oldsvars, osz, nsz, otlocals, ntlocals;
7525 	uint_t id;
7526 
7527 	ASSERT(MUTEX_HELD(&dtrace_lock));
7528 	ASSERT(dp->dtdo_buf != NULL && dp->dtdo_len != 0);
7529 
7530 	for (i = 0; i < dp->dtdo_varlen; i++) {
7531 		dtrace_difv_t *v = &dp->dtdo_vartab[i];
7532 		dtrace_statvar_t *svar, ***svarp;
7533 		size_t dsize = 0;
7534 		uint8_t scope = v->dtdv_scope;
7535 		int *np;
7536 
7537 		if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
7538 			continue;
7539 
7540 		id -= DIF_VAR_OTHER_UBASE;
7541 
7542 		switch (scope) {
7543 		case DIFV_SCOPE_THREAD:
7544 			while (id >= (otlocals = vstate->dtvs_ntlocals)) {
7545 				dtrace_difv_t *tlocals;
7546 
7547 				if ((ntlocals = (otlocals << 1)) == 0)
7548 					ntlocals = 1;
7549 
7550 				osz = otlocals * sizeof (dtrace_difv_t);
7551 				nsz = ntlocals * sizeof (dtrace_difv_t);
7552 
7553 				tlocals = kmem_zalloc(nsz, KM_SLEEP);
7554 
7555 				if (osz != 0) {
7556 					bcopy(vstate->dtvs_tlocals,
7557 					    tlocals, osz);
7558 					kmem_free(vstate->dtvs_tlocals, osz);
7559 				}
7560 
7561 				vstate->dtvs_tlocals = tlocals;
7562 				vstate->dtvs_ntlocals = ntlocals;
7563 			}
7564 
7565 			vstate->dtvs_tlocals[id] = *v;
7566 			continue;
7567 
7568 		case DIFV_SCOPE_LOCAL:
7569 			np = &vstate->dtvs_nlocals;
7570 			svarp = &vstate->dtvs_locals;
7571 
7572 			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
7573 				dsize = NCPU * (v->dtdv_type.dtdt_size +
7574 				    sizeof (uint64_t));
7575 			else
7576 				dsize = NCPU * sizeof (uint64_t);
7577 
7578 			break;
7579 
7580 		case DIFV_SCOPE_GLOBAL:
7581 			np = &vstate->dtvs_nglobals;
7582 			svarp = &vstate->dtvs_globals;
7583 
7584 			if (v->dtdv_type.dtdt_flags & DIF_TF_BYREF)
7585 				dsize = v->dtdv_type.dtdt_size +
7586 				    sizeof (uint64_t);
7587 
7588 			break;
7589 
7590 		default:
7591 			ASSERT(0);
7592 		}
7593 
7594 		while (id >= (oldsvars = *np)) {
7595 			dtrace_statvar_t **statics;
7596 			int newsvars, oldsize, newsize;
7597 
7598 			if ((newsvars = (oldsvars << 1)) == 0)
7599 				newsvars = 1;
7600 
7601 			oldsize = oldsvars * sizeof (dtrace_statvar_t *);
7602 			newsize = newsvars * sizeof (dtrace_statvar_t *);
7603 
7604 			statics = kmem_zalloc(newsize, KM_SLEEP);
7605 
7606 			if (oldsize != 0) {
7607 				bcopy(*svarp, statics, oldsize);
7608 				kmem_free(*svarp, oldsize);
7609 			}
7610 
7611 			*svarp = statics;
7612 			*np = newsvars;
7613 		}
7614 
7615 		if ((svar = (*svarp)[id]) == NULL) {
7616 			svar = kmem_zalloc(sizeof (dtrace_statvar_t), KM_SLEEP);
7617 			svar->dtsv_var = *v;
7618 
7619 			if ((svar->dtsv_size = dsize) != 0) {
7620 				svar->dtsv_data = (uint64_t)(uintptr_t)
7621 				    kmem_zalloc(dsize, KM_SLEEP);
7622 			}
7623 
7624 			(*svarp)[id] = svar;
7625 		}
7626 
7627 		svar->dtsv_refcnt++;
7628 	}
7629 
7630 	dtrace_difo_chunksize(dp, vstate);
7631 	dtrace_difo_hold(dp);
7632 }
7633 
7634 static dtrace_difo_t *
7635 dtrace_difo_duplicate(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
7636 {
7637 	dtrace_difo_t *new;
7638 	size_t sz;
7639 
7640 	ASSERT(dp->dtdo_buf != NULL);
7641 	ASSERT(dp->dtdo_refcnt != 0);
7642 
7643 	new = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
7644 
7645 	ASSERT(dp->dtdo_buf != NULL);
7646 	sz = dp->dtdo_len * sizeof (dif_instr_t);
7647 	new->dtdo_buf = kmem_alloc(sz, KM_SLEEP);
7648 	bcopy(dp->dtdo_buf, new->dtdo_buf, sz);
7649 	new->dtdo_len = dp->dtdo_len;
7650 
7651 	if (dp->dtdo_strtab != NULL) {
7652 		ASSERT(dp->dtdo_strlen != 0);
7653 		new->dtdo_strtab = kmem_alloc(dp->dtdo_strlen, KM_SLEEP);
7654 		bcopy(dp->dtdo_strtab, new->dtdo_strtab, dp->dtdo_strlen);
7655 		new->dtdo_strlen = dp->dtdo_strlen;
7656 	}
7657 
7658 	if (dp->dtdo_inttab != NULL) {
7659 		ASSERT(dp->dtdo_intlen != 0);
7660 		sz = dp->dtdo_intlen * sizeof (uint64_t);
7661 		new->dtdo_inttab = kmem_alloc(sz, KM_SLEEP);
7662 		bcopy(dp->dtdo_inttab, new->dtdo_inttab, sz);
7663 		new->dtdo_intlen = dp->dtdo_intlen;
7664 	}
7665 
7666 	if (dp->dtdo_vartab != NULL) {
7667 		ASSERT(dp->dtdo_varlen != 0);
7668 		sz = dp->dtdo_varlen * sizeof (dtrace_difv_t);
7669 		new->dtdo_vartab = kmem_alloc(sz, KM_SLEEP);
7670 		bcopy(dp->dtdo_vartab, new->dtdo_vartab, sz);
7671 		new->dtdo_varlen = dp->dtdo_varlen;
7672 	}
7673 
7674 	dtrace_difo_init(new, vstate);
7675 	return (new);
7676 }
7677 
7678 static void
7679 dtrace_difo_destroy(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
7680 {
7681 	int i;
7682 
7683 	ASSERT(dp->dtdo_refcnt == 0);
7684 
7685 	for (i = 0; i < dp->dtdo_varlen; i++) {
7686 		dtrace_difv_t *v = &dp->dtdo_vartab[i];
7687 		dtrace_statvar_t *svar, **svarp;
7688 		uint_t id;
7689 		uint8_t scope = v->dtdv_scope;
7690 		int *np;
7691 
7692 		switch (scope) {
7693 		case DIFV_SCOPE_THREAD:
7694 			continue;
7695 
7696 		case DIFV_SCOPE_LOCAL:
7697 			np = &vstate->dtvs_nlocals;
7698 			svarp = vstate->dtvs_locals;
7699 			break;
7700 
7701 		case DIFV_SCOPE_GLOBAL:
7702 			np = &vstate->dtvs_nglobals;
7703 			svarp = vstate->dtvs_globals;
7704 			break;
7705 
7706 		default:
7707 			ASSERT(0);
7708 		}
7709 
7710 		if ((id = v->dtdv_id) < DIF_VAR_OTHER_UBASE)
7711 			continue;
7712 
7713 		id -= DIF_VAR_OTHER_UBASE;
7714 		ASSERT(id < *np);
7715 
7716 		svar = svarp[id];
7717 		ASSERT(svar != NULL);
7718 		ASSERT(svar->dtsv_refcnt > 0);
7719 
7720 		if (--svar->dtsv_refcnt > 0)
7721 			continue;
7722 
7723 		if (svar->dtsv_size != 0) {
7724 			ASSERT(svar->dtsv_data != NULL);
7725 			kmem_free((void *)(uintptr_t)svar->dtsv_data,
7726 			    svar->dtsv_size);
7727 		}
7728 
7729 		kmem_free(svar, sizeof (dtrace_statvar_t));
7730 		svarp[id] = NULL;
7731 	}
7732 
7733 	kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
7734 	kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
7735 	kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
7736 	kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
7737 
7738 	kmem_free(dp, sizeof (dtrace_difo_t));
7739 }
7740 
7741 static void
7742 dtrace_difo_release(dtrace_difo_t *dp, dtrace_vstate_t *vstate)
7743 {
7744 	int i;
7745 
7746 	ASSERT(MUTEX_HELD(&dtrace_lock));
7747 	ASSERT(dp->dtdo_refcnt != 0);
7748 
7749 	for (i = 0; i < dp->dtdo_varlen; i++) {
7750 		dtrace_difv_t *v = &dp->dtdo_vartab[i];
7751 
7752 		if (v->dtdv_id != DIF_VAR_VTIMESTAMP)
7753 			continue;
7754 
7755 		ASSERT(dtrace_vtime_references > 0);
7756 		if (--dtrace_vtime_references == 0)
7757 			dtrace_vtime_disable();
7758 	}
7759 
7760 	if (--dp->dtdo_refcnt == 0)
7761 		dtrace_difo_destroy(dp, vstate);
7762 }
7763 
7764 /*
7765  * DTrace Format Functions
7766  */
7767 static uint16_t
7768 dtrace_format_add(dtrace_state_t *state, char *str)
7769 {
7770 	char *fmt, **new;
7771 	uint16_t ndx, len = strlen(str) + 1;
7772 
7773 	fmt = kmem_zalloc(len, KM_SLEEP);
7774 	bcopy(str, fmt, len);
7775 
7776 	for (ndx = 0; ndx < state->dts_nformats; ndx++) {
7777 		if (state->dts_formats[ndx] == NULL) {
7778 			state->dts_formats[ndx] = fmt;
7779 			return (ndx + 1);
7780 		}
7781 	}
7782 
7783 	if (state->dts_nformats == USHRT_MAX) {
7784 		/*
7785 		 * This is only likely if a denial-of-service attack is being
7786 		 * attempted.  As such, it's okay to fail silently here.
7787 		 */
7788 		kmem_free(fmt, len);
7789 		return (0);
7790 	}
7791 
7792 	/*
7793 	 * For simplicity, we always resize the formats array to be exactly the
7794 	 * number of formats.
7795 	 */
7796 	ndx = state->dts_nformats++;
7797 	new = kmem_alloc((ndx + 1) * sizeof (char *), KM_SLEEP);
7798 
7799 	if (state->dts_formats != NULL) {
7800 		ASSERT(ndx != 0);
7801 		bcopy(state->dts_formats, new, ndx * sizeof (char *));
7802 		kmem_free(state->dts_formats, ndx * sizeof (char *));
7803 	}
7804 
7805 	state->dts_formats = new;
7806 	state->dts_formats[ndx] = fmt;
7807 
7808 	return (ndx + 1);
7809 }
7810 
7811 static void
7812 dtrace_format_remove(dtrace_state_t *state, uint16_t format)
7813 {
7814 	char *fmt;
7815 
7816 	ASSERT(state->dts_formats != NULL);
7817 	ASSERT(format <= state->dts_nformats);
7818 	ASSERT(state->dts_formats[format - 1] != NULL);
7819 
7820 	fmt = state->dts_formats[format - 1];
7821 	kmem_free(fmt, strlen(fmt) + 1);
7822 	state->dts_formats[format - 1] = NULL;
7823 }
7824 
7825 static void
7826 dtrace_format_destroy(dtrace_state_t *state)
7827 {
7828 	int i;
7829 
7830 	if (state->dts_nformats == 0) {
7831 		ASSERT(state->dts_formats == NULL);
7832 		return;
7833 	}
7834 
7835 	ASSERT(state->dts_formats != NULL);
7836 
7837 	for (i = 0; i < state->dts_nformats; i++) {
7838 		char *fmt = state->dts_formats[i];
7839 
7840 		if (fmt == NULL)
7841 			continue;
7842 
7843 		kmem_free(fmt, strlen(fmt) + 1);
7844 	}
7845 
7846 	kmem_free(state->dts_formats, state->dts_nformats * sizeof (char *));
7847 	state->dts_nformats = 0;
7848 	state->dts_formats = NULL;
7849 }
7850 
7851 /*
7852  * DTrace Predicate Functions
7853  */
7854 static dtrace_predicate_t *
7855 dtrace_predicate_create(dtrace_difo_t *dp)
7856 {
7857 	dtrace_predicate_t *pred;
7858 
7859 	ASSERT(MUTEX_HELD(&dtrace_lock));
7860 	ASSERT(dp->dtdo_refcnt != 0);
7861 
7862 	pred = kmem_zalloc(sizeof (dtrace_predicate_t), KM_SLEEP);
7863 	pred->dtp_difo = dp;
7864 	pred->dtp_refcnt = 1;
7865 
7866 	if (!dtrace_difo_cacheable(dp))
7867 		return (pred);
7868 
7869 	if (dtrace_predcache_id == DTRACE_CACHEIDNONE) {
7870 		/*
7871 		 * This is only theoretically possible -- we have had 2^32
7872 		 * cacheable predicates on this machine.  We cannot allow any
7873 		 * more predicates to become cacheable:  as unlikely as it is,
7874 		 * there may be a thread caching a (now stale) predicate cache
7875 		 * ID. (N.B.: the temptation is being successfully resisted to
7876 		 * have this cmn_err() "Holy shit -- we executed this code!")
7877 		 */
7878 		return (pred);
7879 	}
7880 
7881 	pred->dtp_cacheid = dtrace_predcache_id++;
7882 
7883 	return (pred);
7884 }
7885 
7886 static void
7887 dtrace_predicate_hold(dtrace_predicate_t *pred)
7888 {
7889 	ASSERT(MUTEX_HELD(&dtrace_lock));
7890 	ASSERT(pred->dtp_difo != NULL && pred->dtp_difo->dtdo_refcnt != 0);
7891 	ASSERT(pred->dtp_refcnt > 0);
7892 
7893 	pred->dtp_refcnt++;
7894 }
7895 
7896 static void
7897 dtrace_predicate_release(dtrace_predicate_t *pred, dtrace_vstate_t *vstate)
7898 {
7899 	dtrace_difo_t *dp = pred->dtp_difo;
7900 
7901 	ASSERT(MUTEX_HELD(&dtrace_lock));
7902 	ASSERT(dp != NULL && dp->dtdo_refcnt != 0);
7903 	ASSERT(pred->dtp_refcnt > 0);
7904 
7905 	if (--pred->dtp_refcnt == 0) {
7906 		dtrace_difo_release(pred->dtp_difo, vstate);
7907 		kmem_free(pred, sizeof (dtrace_predicate_t));
7908 	}
7909 }
7910 
7911 /*
7912  * DTrace Action Description Functions
7913  */
7914 static dtrace_actdesc_t *
7915 dtrace_actdesc_create(dtrace_actkind_t kind, uint32_t ntuple,
7916     uint64_t uarg, uint64_t arg)
7917 {
7918 	dtrace_actdesc_t *act;
7919 
7920 	ASSERT(!DTRACEACT_ISPRINTFLIKE(kind) || (arg != NULL &&
7921 	    arg >= KERNELBASE) || (arg == NULL && kind == DTRACEACT_PRINTA));
7922 
7923 	act = kmem_zalloc(sizeof (dtrace_actdesc_t), KM_SLEEP);
7924 	act->dtad_kind = kind;
7925 	act->dtad_ntuple = ntuple;
7926 	act->dtad_uarg = uarg;
7927 	act->dtad_arg = arg;
7928 	act->dtad_refcnt = 1;
7929 
7930 	return (act);
7931 }
7932 
7933 static void
7934 dtrace_actdesc_hold(dtrace_actdesc_t *act)
7935 {
7936 	ASSERT(act->dtad_refcnt >= 1);
7937 	act->dtad_refcnt++;
7938 }
7939 
7940 static void
7941 dtrace_actdesc_release(dtrace_actdesc_t *act, dtrace_vstate_t *vstate)
7942 {
7943 	dtrace_actkind_t kind = act->dtad_kind;
7944 	dtrace_difo_t *dp;
7945 
7946 	ASSERT(act->dtad_refcnt >= 1);
7947 
7948 	if (--act->dtad_refcnt != 0)
7949 		return;
7950 
7951 	if ((dp = act->dtad_difo) != NULL)
7952 		dtrace_difo_release(dp, vstate);
7953 
7954 	if (DTRACEACT_ISPRINTFLIKE(kind)) {
7955 		char *str = (char *)(uintptr_t)act->dtad_arg;
7956 
7957 		ASSERT((str != NULL && (uintptr_t)str >= KERNELBASE) ||
7958 		    (str == NULL && act->dtad_kind == DTRACEACT_PRINTA));
7959 
7960 		if (str != NULL)
7961 			kmem_free(str, strlen(str) + 1);
7962 	}
7963 
7964 	kmem_free(act, sizeof (dtrace_actdesc_t));
7965 }
7966 
7967 /*
7968  * DTrace ECB Functions
7969  */
7970 static dtrace_ecb_t *
7971 dtrace_ecb_add(dtrace_state_t *state, dtrace_probe_t *probe)
7972 {
7973 	dtrace_ecb_t *ecb;
7974 	dtrace_epid_t epid;
7975 
7976 	ASSERT(MUTEX_HELD(&dtrace_lock));
7977 
7978 	ecb = kmem_zalloc(sizeof (dtrace_ecb_t), KM_SLEEP);
7979 	ecb->dte_predicate = NULL;
7980 	ecb->dte_probe = probe;
7981 
7982 	/*
7983 	 * The default size is the size of the default action: recording
7984 	 * the epid.
7985 	 */
7986 	ecb->dte_size = ecb->dte_needed = sizeof (dtrace_epid_t);
7987 	ecb->dte_alignment = sizeof (dtrace_epid_t);
7988 
7989 	epid = state->dts_epid++;
7990 
7991 	if (epid - 1 >= state->dts_necbs) {
7992 		dtrace_ecb_t **oecbs = state->dts_ecbs, **ecbs;
7993 		int necbs = state->dts_necbs << 1;
7994 
7995 		ASSERT(epid == state->dts_necbs + 1);
7996 
7997 		if (necbs == 0) {
7998 			ASSERT(oecbs == NULL);
7999 			necbs = 1;
8000 		}
8001 
8002 		ecbs = kmem_zalloc(necbs * sizeof (*ecbs), KM_SLEEP);
8003 
8004 		if (oecbs != NULL)
8005 			bcopy(oecbs, ecbs, state->dts_necbs * sizeof (*ecbs));
8006 
8007 		dtrace_membar_producer();
8008 		state->dts_ecbs = ecbs;
8009 
8010 		if (oecbs != NULL) {
8011 			/*
8012 			 * If this state is active, we must dtrace_sync()
8013 			 * before we can free the old dts_ecbs array:  we're
8014 			 * coming in hot, and there may be active ring
8015 			 * buffer processing (which indexes into the dts_ecbs
8016 			 * array) on another CPU.
8017 			 */
8018 			if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
8019 				dtrace_sync();
8020 
8021 			kmem_free(oecbs, state->dts_necbs * sizeof (*ecbs));
8022 		}
8023 
8024 		dtrace_membar_producer();
8025 		state->dts_necbs = necbs;
8026 	}
8027 
8028 	ecb->dte_state = state;
8029 
8030 	ASSERT(state->dts_ecbs[epid - 1] == NULL);
8031 	dtrace_membar_producer();
8032 	state->dts_ecbs[(ecb->dte_epid = epid) - 1] = ecb;
8033 
8034 	return (ecb);
8035 }
8036 
8037 static void
8038 dtrace_ecb_enable(dtrace_ecb_t *ecb)
8039 {
8040 	dtrace_probe_t *probe = ecb->dte_probe;
8041 
8042 	ASSERT(MUTEX_HELD(&cpu_lock));
8043 	ASSERT(MUTEX_HELD(&dtrace_lock));
8044 	ASSERT(ecb->dte_next == NULL);
8045 
8046 	if (probe == NULL) {
8047 		/*
8048 		 * This is the NULL probe -- there's nothing to do.
8049 		 */
8050 		return;
8051 	}
8052 
8053 	if (probe->dtpr_ecb == NULL) {
8054 		dtrace_provider_t *prov = probe->dtpr_provider;
8055 
8056 		/*
8057 		 * We're the first ECB on this probe.
8058 		 */
8059 		probe->dtpr_ecb = probe->dtpr_ecb_last = ecb;
8060 
8061 		if (ecb->dte_predicate != NULL)
8062 			probe->dtpr_predcache = ecb->dte_predicate->dtp_cacheid;
8063 
8064 		prov->dtpv_pops.dtps_enable(prov->dtpv_arg,
8065 		    probe->dtpr_id, probe->dtpr_arg);
8066 	} else {
8067 		/*
8068 		 * This probe is already active.  Swing the last pointer to
8069 		 * point to the new ECB, and issue a dtrace_sync() to assure
8070 		 * that all CPUs have seen the change.
8071 		 */
8072 		ASSERT(probe->dtpr_ecb_last != NULL);
8073 		probe->dtpr_ecb_last->dte_next = ecb;
8074 		probe->dtpr_ecb_last = ecb;
8075 		probe->dtpr_predcache = 0;
8076 
8077 		dtrace_sync();
8078 	}
8079 }
8080 
8081 static void
8082 dtrace_ecb_resize(dtrace_ecb_t *ecb)
8083 {
8084 	uint32_t maxalign = sizeof (dtrace_epid_t);
8085 	uint32_t align = sizeof (uint8_t), offs, diff;
8086 	dtrace_action_t *act;
8087 	int wastuple = 0;
8088 	uint32_t aggbase = UINT32_MAX;
8089 	dtrace_state_t *state = ecb->dte_state;
8090 
8091 	/*
8092 	 * If we record anything, we always record the epid.  (And we always
8093 	 * record it first.)
8094 	 */
8095 	offs = sizeof (dtrace_epid_t);
8096 	ecb->dte_size = ecb->dte_needed = sizeof (dtrace_epid_t);
8097 
8098 	for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
8099 		dtrace_recdesc_t *rec = &act->dta_rec;
8100 
8101 		if ((align = rec->dtrd_alignment) > maxalign)
8102 			maxalign = align;
8103 
8104 		if (!wastuple && act->dta_intuple) {
8105 			/*
8106 			 * This is the first record in a tuple.  Align the
8107 			 * offset to be at offset 4 in an 8-byte aligned
8108 			 * block.
8109 			 */
8110 			diff = offs + sizeof (dtrace_aggid_t);
8111 
8112 			if (diff = (diff & (sizeof (uint64_t) - 1)))
8113 				offs += sizeof (uint64_t) - diff;
8114 
8115 			aggbase = offs - sizeof (dtrace_aggid_t);
8116 			ASSERT(!(aggbase & (sizeof (uint64_t) - 1)));
8117 		}
8118 
8119 		/*LINTED*/
8120 		if (rec->dtrd_size != 0 && (diff = (offs & (align - 1)))) {
8121 			/*
8122 			 * The current offset is not properly aligned; align it.
8123 			 */
8124 			offs += align - diff;
8125 		}
8126 
8127 		rec->dtrd_offset = offs;
8128 
8129 		if (offs + rec->dtrd_size > ecb->dte_needed) {
8130 			ecb->dte_needed = offs + rec->dtrd_size;
8131 
8132 			if (ecb->dte_needed > state->dts_needed)
8133 				state->dts_needed = ecb->dte_needed;
8134 		}
8135 
8136 		if (DTRACEACT_ISAGG(act->dta_kind)) {
8137 			dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
8138 			dtrace_action_t *first = agg->dtag_first, *prev;
8139 
8140 			ASSERT(rec->dtrd_size != 0 && first != NULL);
8141 			ASSERT(wastuple);
8142 			ASSERT(aggbase != UINT32_MAX);
8143 
8144 			agg->dtag_base = aggbase;
8145 
8146 			while ((prev = first->dta_prev) != NULL &&
8147 			    DTRACEACT_ISAGG(prev->dta_kind)) {
8148 				agg = (dtrace_aggregation_t *)prev;
8149 				first = agg->dtag_first;
8150 			}
8151 
8152 			if (prev != NULL) {
8153 				offs = prev->dta_rec.dtrd_offset +
8154 				    prev->dta_rec.dtrd_size;
8155 			} else {
8156 				offs = sizeof (dtrace_epid_t);
8157 			}
8158 			wastuple = 0;
8159 		} else {
8160 			if (!act->dta_intuple)
8161 				ecb->dte_size = offs + rec->dtrd_size;
8162 
8163 			offs += rec->dtrd_size;
8164 		}
8165 
8166 		wastuple = act->dta_intuple;
8167 	}
8168 
8169 	if ((act = ecb->dte_action) != NULL &&
8170 	    !(act->dta_kind == DTRACEACT_SPECULATE && act->dta_next == NULL) &&
8171 	    ecb->dte_size == sizeof (dtrace_epid_t)) {
8172 		/*
8173 		 * If the size is still sizeof (dtrace_epid_t), then all
8174 		 * actions store no data; set the size to 0.
8175 		 */
8176 		ecb->dte_alignment = maxalign;
8177 		ecb->dte_size = 0;
8178 
8179 		/*
8180 		 * If the needed space is still sizeof (dtrace_epid_t), then
8181 		 * all actions need no additional space; set the needed
8182 		 * size to 0.
8183 		 */
8184 		if (ecb->dte_needed == sizeof (dtrace_epid_t))
8185 			ecb->dte_needed = 0;
8186 
8187 		return;
8188 	}
8189 
8190 	/*
8191 	 * Set our alignment, and make sure that the dte_size and dte_needed
8192 	 * are aligned to the size of an EPID.
8193 	 */
8194 	ecb->dte_alignment = maxalign;
8195 	ecb->dte_size = (ecb->dte_size + (sizeof (dtrace_epid_t) - 1)) &
8196 	    ~(sizeof (dtrace_epid_t) - 1);
8197 	ecb->dte_needed = (ecb->dte_needed + (sizeof (dtrace_epid_t) - 1)) &
8198 	    ~(sizeof (dtrace_epid_t) - 1);
8199 	ASSERT(ecb->dte_size <= ecb->dte_needed);
8200 }
8201 
8202 static dtrace_action_t *
8203 dtrace_ecb_aggregation_create(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
8204 {
8205 	dtrace_aggregation_t *agg;
8206 	size_t size = sizeof (uint64_t);
8207 	int ntuple = desc->dtad_ntuple;
8208 	dtrace_action_t *act;
8209 	dtrace_recdesc_t *frec;
8210 	dtrace_aggid_t aggid;
8211 	dtrace_state_t *state = ecb->dte_state;
8212 
8213 	agg = kmem_zalloc(sizeof (dtrace_aggregation_t), KM_SLEEP);
8214 	agg->dtag_ecb = ecb;
8215 
8216 	ASSERT(DTRACEACT_ISAGG(desc->dtad_kind));
8217 
8218 	switch (desc->dtad_kind) {
8219 	case DTRACEAGG_MIN:
8220 		agg->dtag_initial = UINT64_MAX;
8221 		agg->dtag_aggregate = dtrace_aggregate_min;
8222 		break;
8223 
8224 	case DTRACEAGG_MAX:
8225 		agg->dtag_aggregate = dtrace_aggregate_max;
8226 		break;
8227 
8228 	case DTRACEAGG_COUNT:
8229 		agg->dtag_aggregate = dtrace_aggregate_count;
8230 		break;
8231 
8232 	case DTRACEAGG_QUANTIZE:
8233 		agg->dtag_aggregate = dtrace_aggregate_quantize;
8234 		size = (((sizeof (uint64_t) * NBBY) - 1) * 2 + 1) *
8235 		    sizeof (uint64_t);
8236 		break;
8237 
8238 	case DTRACEAGG_LQUANTIZE: {
8239 		uint16_t step = DTRACE_LQUANTIZE_STEP(desc->dtad_arg);
8240 		uint16_t levels = DTRACE_LQUANTIZE_LEVELS(desc->dtad_arg);
8241 
8242 		agg->dtag_initial = desc->dtad_arg;
8243 		agg->dtag_aggregate = dtrace_aggregate_lquantize;
8244 
8245 		if (step == 0 || levels == 0)
8246 			goto err;
8247 
8248 		size = levels * sizeof (uint64_t) + 3 * sizeof (uint64_t);
8249 		break;
8250 	}
8251 
8252 	case DTRACEAGG_AVG:
8253 		agg->dtag_aggregate = dtrace_aggregate_avg;
8254 		size = sizeof (uint64_t) * 2;
8255 		break;
8256 
8257 	case DTRACEAGG_SUM:
8258 		agg->dtag_aggregate = dtrace_aggregate_sum;
8259 		break;
8260 
8261 	default:
8262 		goto err;
8263 	}
8264 
8265 	agg->dtag_action.dta_rec.dtrd_size = size;
8266 
8267 	if (ntuple == 0)
8268 		goto err;
8269 
8270 	/*
8271 	 * We must make sure that we have enough actions for the n-tuple.
8272 	 */
8273 	for (act = ecb->dte_action_last; act != NULL; act = act->dta_prev) {
8274 		if (DTRACEACT_ISAGG(act->dta_kind))
8275 			break;
8276 
8277 		if (--ntuple == 0) {
8278 			/*
8279 			 * This is the action with which our n-tuple begins.
8280 			 */
8281 			agg->dtag_first = act;
8282 			goto success;
8283 		}
8284 	}
8285 
8286 	/*
8287 	 * This n-tuple is short by ntuple elements.  Return failure.
8288 	 */
8289 	ASSERT(ntuple != 0);
8290 err:
8291 	kmem_free(agg, sizeof (dtrace_aggregation_t));
8292 	return (NULL);
8293 
8294 success:
8295 	/*
8296 	 * If the last action in the tuple has a size of zero, it's actually
8297 	 * an expression argument for the aggregating action.
8298 	 */
8299 	ASSERT(ecb->dte_action_last != NULL);
8300 	act = ecb->dte_action_last;
8301 
8302 	if (act->dta_kind == DTRACEACT_DIFEXPR) {
8303 		ASSERT(act->dta_difo != NULL);
8304 
8305 		if (act->dta_difo->dtdo_rtype.dtdt_size == 0)
8306 			agg->dtag_hasarg = 1;
8307 	}
8308 
8309 	/*
8310 	 * We need to allocate an id for this aggregation.
8311 	 */
8312 	aggid = (dtrace_aggid_t)(uintptr_t)vmem_alloc(state->dts_aggid_arena, 1,
8313 	    VM_BESTFIT | VM_SLEEP);
8314 
8315 	if (aggid - 1 >= state->dts_naggregations) {
8316 		dtrace_aggregation_t **oaggs = state->dts_aggregations;
8317 		dtrace_aggregation_t **aggs;
8318 		int naggs = state->dts_naggregations << 1;
8319 		int onaggs = state->dts_naggregations;
8320 
8321 		ASSERT(aggid == state->dts_naggregations + 1);
8322 
8323 		if (naggs == 0) {
8324 			ASSERT(oaggs == NULL);
8325 			naggs = 1;
8326 		}
8327 
8328 		aggs = kmem_zalloc(naggs * sizeof (*aggs), KM_SLEEP);
8329 
8330 		if (oaggs != NULL) {
8331 			bcopy(oaggs, aggs, onaggs * sizeof (*aggs));
8332 			kmem_free(oaggs, onaggs * sizeof (*aggs));
8333 		}
8334 
8335 		state->dts_aggregations = aggs;
8336 		state->dts_naggregations = naggs;
8337 	}
8338 
8339 	ASSERT(state->dts_aggregations[aggid - 1] == NULL);
8340 	state->dts_aggregations[(agg->dtag_id = aggid) - 1] = agg;
8341 
8342 	frec = &agg->dtag_first->dta_rec;
8343 	if (frec->dtrd_alignment < sizeof (dtrace_aggid_t))
8344 		frec->dtrd_alignment = sizeof (dtrace_aggid_t);
8345 
8346 	for (act = agg->dtag_first; act != NULL; act = act->dta_next) {
8347 		ASSERT(!act->dta_intuple);
8348 		act->dta_intuple = 1;
8349 	}
8350 
8351 	return (&agg->dtag_action);
8352 }
8353 
8354 static void
8355 dtrace_ecb_aggregation_destroy(dtrace_ecb_t *ecb, dtrace_action_t *act)
8356 {
8357 	dtrace_aggregation_t *agg = (dtrace_aggregation_t *)act;
8358 	dtrace_state_t *state = ecb->dte_state;
8359 	dtrace_aggid_t aggid = agg->dtag_id;
8360 
8361 	ASSERT(DTRACEACT_ISAGG(act->dta_kind));
8362 	vmem_free(state->dts_aggid_arena, (void *)(uintptr_t)aggid, 1);
8363 
8364 	ASSERT(state->dts_aggregations[aggid - 1] == agg);
8365 	state->dts_aggregations[aggid - 1] = NULL;
8366 
8367 	kmem_free(agg, sizeof (dtrace_aggregation_t));
8368 }
8369 
8370 static int
8371 dtrace_ecb_action_add(dtrace_ecb_t *ecb, dtrace_actdesc_t *desc)
8372 {
8373 	dtrace_action_t *action, *last;
8374 	dtrace_difo_t *dp = desc->dtad_difo;
8375 	uint32_t size = 0, align = sizeof (uint8_t), mask;
8376 	uint16_t format = 0;
8377 	dtrace_recdesc_t *rec;
8378 	dtrace_state_t *state = ecb->dte_state;
8379 	dtrace_optval_t *opt = state->dts_options, nframes, strsize;
8380 	uint64_t arg = desc->dtad_arg;
8381 
8382 	ASSERT(MUTEX_HELD(&dtrace_lock));
8383 	ASSERT(ecb->dte_action == NULL || ecb->dte_action->dta_refcnt == 1);
8384 
8385 	if (DTRACEACT_ISAGG(desc->dtad_kind)) {
8386 		/*
8387 		 * If this is an aggregating action, there must be neither
8388 		 * a speculate nor a commit on the action chain.
8389 		 */
8390 		dtrace_action_t *act;
8391 
8392 		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
8393 			if (act->dta_kind == DTRACEACT_COMMIT)
8394 				return (EINVAL);
8395 
8396 			if (act->dta_kind == DTRACEACT_SPECULATE)
8397 				return (EINVAL);
8398 		}
8399 
8400 		action = dtrace_ecb_aggregation_create(ecb, desc);
8401 
8402 		if (action == NULL)
8403 			return (EINVAL);
8404 	} else {
8405 		if (DTRACEACT_ISDESTRUCTIVE(desc->dtad_kind) ||
8406 		    (desc->dtad_kind == DTRACEACT_DIFEXPR &&
8407 		    dp != NULL && dp->dtdo_destructive)) {
8408 			state->dts_destructive = 1;
8409 		}
8410 
8411 		switch (desc->dtad_kind) {
8412 		case DTRACEACT_PRINTF:
8413 		case DTRACEACT_PRINTA:
8414 		case DTRACEACT_SYSTEM:
8415 		case DTRACEACT_FREOPEN:
8416 			/*
8417 			 * We know that our arg is a string -- turn it into a
8418 			 * format.
8419 			 */
8420 			if (arg == NULL) {
8421 				ASSERT(desc->dtad_kind == DTRACEACT_PRINTA);
8422 				format = 0;
8423 			} else {
8424 				ASSERT(arg != NULL);
8425 				ASSERT(arg > KERNELBASE);
8426 				format = dtrace_format_add(state,
8427 				    (char *)(uintptr_t)arg);
8428 			}
8429 
8430 			/*FALLTHROUGH*/
8431 		case DTRACEACT_LIBACT:
8432 		case DTRACEACT_DIFEXPR:
8433 			if (dp == NULL)
8434 				return (EINVAL);
8435 
8436 			if ((size = dp->dtdo_rtype.dtdt_size) != 0)
8437 				break;
8438 
8439 			if (dp->dtdo_rtype.dtdt_kind == DIF_TYPE_STRING) {
8440 				if (!(dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
8441 					return (EINVAL);
8442 
8443 				size = opt[DTRACEOPT_STRSIZE];
8444 			}
8445 
8446 			break;
8447 
8448 		case DTRACEACT_STACK:
8449 			if ((nframes = arg) == 0) {
8450 				nframes = opt[DTRACEOPT_STACKFRAMES];
8451 				ASSERT(nframes > 0);
8452 				arg = nframes;
8453 			}
8454 
8455 			size = nframes * sizeof (pc_t);
8456 			break;
8457 
8458 		case DTRACEACT_JSTACK:
8459 			if ((strsize = DTRACE_USTACK_STRSIZE(arg)) == 0)
8460 				strsize = opt[DTRACEOPT_JSTACKSTRSIZE];
8461 
8462 			if ((nframes = DTRACE_USTACK_NFRAMES(arg)) == 0)
8463 				nframes = opt[DTRACEOPT_JSTACKFRAMES];
8464 
8465 			arg = DTRACE_USTACK_ARG(nframes, strsize);
8466 
8467 			/*FALLTHROUGH*/
8468 		case DTRACEACT_USTACK:
8469 			if (desc->dtad_kind != DTRACEACT_JSTACK &&
8470 			    (nframes = DTRACE_USTACK_NFRAMES(arg)) == 0) {
8471 				strsize = DTRACE_USTACK_STRSIZE(arg);
8472 				nframes = opt[DTRACEOPT_USTACKFRAMES];
8473 				ASSERT(nframes > 0);
8474 				arg = DTRACE_USTACK_ARG(nframes, strsize);
8475 			}
8476 
8477 			/*
8478 			 * Save a slot for the pid.
8479 			 */
8480 			size = (nframes + 1) * sizeof (uint64_t);
8481 			size += DTRACE_USTACK_STRSIZE(arg);
8482 			size = P2ROUNDUP(size, (uint32_t)(sizeof (uintptr_t)));
8483 
8484 			break;
8485 
8486 		case DTRACEACT_SYM:
8487 		case DTRACEACT_MOD:
8488 			if (dp == NULL || ((size = dp->dtdo_rtype.dtdt_size) !=
8489 			    sizeof (uint64_t)) ||
8490 			    (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
8491 				return (EINVAL);
8492 			break;
8493 
8494 		case DTRACEACT_USYM:
8495 		case DTRACEACT_UMOD:
8496 		case DTRACEACT_UADDR:
8497 			if (dp == NULL ||
8498 			    (dp->dtdo_rtype.dtdt_size != sizeof (uint64_t)) ||
8499 			    (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
8500 				return (EINVAL);
8501 
8502 			/*
8503 			 * We have a slot for the pid, plus a slot for the
8504 			 * argument.  To keep things simple (aligned with
8505 			 * bitness-neutral sizing), we store each as a 64-bit
8506 			 * quantity.
8507 			 */
8508 			size = 2 * sizeof (uint64_t);
8509 			break;
8510 
8511 		case DTRACEACT_STOP:
8512 		case DTRACEACT_BREAKPOINT:
8513 		case DTRACEACT_PANIC:
8514 			break;
8515 
8516 		case DTRACEACT_CHILL:
8517 		case DTRACEACT_DISCARD:
8518 		case DTRACEACT_RAISE:
8519 			if (dp == NULL)
8520 				return (EINVAL);
8521 			break;
8522 
8523 		case DTRACEACT_EXIT:
8524 			if (dp == NULL ||
8525 			    (size = dp->dtdo_rtype.dtdt_size) != sizeof (int) ||
8526 			    (dp->dtdo_rtype.dtdt_flags & DIF_TF_BYREF))
8527 				return (EINVAL);
8528 			break;
8529 
8530 		case DTRACEACT_SPECULATE:
8531 			if (ecb->dte_size > sizeof (dtrace_epid_t))
8532 				return (EINVAL);
8533 
8534 			if (dp == NULL)
8535 				return (EINVAL);
8536 
8537 			state->dts_speculates = 1;
8538 			break;
8539 
8540 		case DTRACEACT_COMMIT: {
8541 			dtrace_action_t *act = ecb->dte_action;
8542 
8543 			for (; act != NULL; act = act->dta_next) {
8544 				if (act->dta_kind == DTRACEACT_COMMIT)
8545 					return (EINVAL);
8546 			}
8547 
8548 			if (dp == NULL)
8549 				return (EINVAL);
8550 			break;
8551 		}
8552 
8553 		default:
8554 			return (EINVAL);
8555 		}
8556 
8557 		if (size != 0 || desc->dtad_kind == DTRACEACT_SPECULATE) {
8558 			/*
8559 			 * If this is a data-storing action or a speculate,
8560 			 * we must be sure that there isn't a commit on the
8561 			 * action chain.
8562 			 */
8563 			dtrace_action_t *act = ecb->dte_action;
8564 
8565 			for (; act != NULL; act = act->dta_next) {
8566 				if (act->dta_kind == DTRACEACT_COMMIT)
8567 					return (EINVAL);
8568 			}
8569 		}
8570 
8571 		action = kmem_zalloc(sizeof (dtrace_action_t), KM_SLEEP);
8572 		action->dta_rec.dtrd_size = size;
8573 	}
8574 
8575 	action->dta_refcnt = 1;
8576 	rec = &action->dta_rec;
8577 	size = rec->dtrd_size;
8578 
8579 	for (mask = sizeof (uint64_t) - 1; size != 0 && mask > 0; mask >>= 1) {
8580 		if (!(size & mask)) {
8581 			align = mask + 1;
8582 			break;
8583 		}
8584 	}
8585 
8586 	action->dta_kind = desc->dtad_kind;
8587 
8588 	if ((action->dta_difo = dp) != NULL)
8589 		dtrace_difo_hold(dp);
8590 
8591 	rec->dtrd_action = action->dta_kind;
8592 	rec->dtrd_arg = arg;
8593 
8594 	if (ecb->dte_state == dtrace_anon.dta_state) {
8595 		/*
8596 		 * If this is an anonymous enabling, explicitly clear the uarg.
8597 		 */
8598 		rec->dtrd_uarg = 0;
8599 	} else {
8600 		rec->dtrd_uarg = desc->dtad_uarg;
8601 	}
8602 
8603 	rec->dtrd_alignment = (uint16_t)align;
8604 	rec->dtrd_format = format;
8605 
8606 	if ((last = ecb->dte_action_last) != NULL) {
8607 		ASSERT(ecb->dte_action != NULL);
8608 		action->dta_prev = last;
8609 		last->dta_next = action;
8610 	} else {
8611 		ASSERT(ecb->dte_action == NULL);
8612 		ecb->dte_action = action;
8613 	}
8614 
8615 	ecb->dte_action_last = action;
8616 
8617 	return (0);
8618 }
8619 
8620 static void
8621 dtrace_ecb_action_remove(dtrace_ecb_t *ecb)
8622 {
8623 	dtrace_action_t *act = ecb->dte_action, *next;
8624 	dtrace_vstate_t *vstate = &ecb->dte_state->dts_vstate;
8625 	dtrace_difo_t *dp;
8626 	uint16_t format;
8627 
8628 	if (act != NULL && act->dta_refcnt > 1) {
8629 		ASSERT(act->dta_next == NULL || act->dta_next->dta_refcnt == 1);
8630 		act->dta_refcnt--;
8631 	} else {
8632 		for (; act != NULL; act = next) {
8633 			next = act->dta_next;
8634 			ASSERT(next != NULL || act == ecb->dte_action_last);
8635 			ASSERT(act->dta_refcnt == 1);
8636 
8637 			if ((format = act->dta_rec.dtrd_format) != 0)
8638 				dtrace_format_remove(ecb->dte_state, format);
8639 
8640 			if ((dp = act->dta_difo) != NULL)
8641 				dtrace_difo_release(dp, vstate);
8642 
8643 			if (DTRACEACT_ISAGG(act->dta_kind)) {
8644 				dtrace_ecb_aggregation_destroy(ecb, act);
8645 			} else {
8646 				kmem_free(act, sizeof (dtrace_action_t));
8647 			}
8648 		}
8649 	}
8650 
8651 	ecb->dte_action = NULL;
8652 	ecb->dte_action_last = NULL;
8653 	ecb->dte_size = sizeof (dtrace_epid_t);
8654 }
8655 
8656 static void
8657 dtrace_ecb_disable(dtrace_ecb_t *ecb)
8658 {
8659 	/*
8660 	 * We disable the ECB by removing it from its probe.
8661 	 */
8662 	dtrace_ecb_t *pecb, *prev = NULL;
8663 	dtrace_probe_t *probe = ecb->dte_probe;
8664 
8665 	ASSERT(MUTEX_HELD(&dtrace_lock));
8666 
8667 	if (probe == NULL) {
8668 		/*
8669 		 * This is the NULL probe; there is nothing to disable.
8670 		 */
8671 		return;
8672 	}
8673 
8674 	for (pecb = probe->dtpr_ecb; pecb != NULL; pecb = pecb->dte_next) {
8675 		if (pecb == ecb)
8676 			break;
8677 		prev = pecb;
8678 	}
8679 
8680 	ASSERT(pecb != NULL);
8681 
8682 	if (prev == NULL) {
8683 		probe->dtpr_ecb = ecb->dte_next;
8684 	} else {
8685 		prev->dte_next = ecb->dte_next;
8686 	}
8687 
8688 	if (ecb == probe->dtpr_ecb_last) {
8689 		ASSERT(ecb->dte_next == NULL);
8690 		probe->dtpr_ecb_last = prev;
8691 	}
8692 
8693 	/*
8694 	 * The ECB has been disconnected from the probe; now sync to assure
8695 	 * that all CPUs have seen the change before returning.
8696 	 */
8697 	dtrace_sync();
8698 
8699 	if (probe->dtpr_ecb == NULL) {
8700 		/*
8701 		 * That was the last ECB on the probe; clear the predicate
8702 		 * cache ID for the probe, disable it and sync one more time
8703 		 * to assure that we'll never hit it again.
8704 		 */
8705 		dtrace_provider_t *prov = probe->dtpr_provider;
8706 
8707 		ASSERT(ecb->dte_next == NULL);
8708 		ASSERT(probe->dtpr_ecb_last == NULL);
8709 		probe->dtpr_predcache = DTRACE_CACHEIDNONE;
8710 		prov->dtpv_pops.dtps_disable(prov->dtpv_arg,
8711 		    probe->dtpr_id, probe->dtpr_arg);
8712 		dtrace_sync();
8713 	} else {
8714 		/*
8715 		 * There is at least one ECB remaining on the probe.  If there
8716 		 * is _exactly_ one, set the probe's predicate cache ID to be
8717 		 * the predicate cache ID of the remaining ECB.
8718 		 */
8719 		ASSERT(probe->dtpr_ecb_last != NULL);
8720 		ASSERT(probe->dtpr_predcache == DTRACE_CACHEIDNONE);
8721 
8722 		if (probe->dtpr_ecb == probe->dtpr_ecb_last) {
8723 			dtrace_predicate_t *p = probe->dtpr_ecb->dte_predicate;
8724 
8725 			ASSERT(probe->dtpr_ecb->dte_next == NULL);
8726 
8727 			if (p != NULL)
8728 				probe->dtpr_predcache = p->dtp_cacheid;
8729 		}
8730 
8731 		ecb->dte_next = NULL;
8732 	}
8733 }
8734 
8735 static void
8736 dtrace_ecb_destroy(dtrace_ecb_t *ecb)
8737 {
8738 	dtrace_state_t *state = ecb->dte_state;
8739 	dtrace_vstate_t *vstate = &state->dts_vstate;
8740 	dtrace_predicate_t *pred;
8741 	dtrace_epid_t epid = ecb->dte_epid;
8742 
8743 	ASSERT(MUTEX_HELD(&dtrace_lock));
8744 	ASSERT(ecb->dte_next == NULL);
8745 	ASSERT(ecb->dte_probe == NULL || ecb->dte_probe->dtpr_ecb != ecb);
8746 
8747 	if ((pred = ecb->dte_predicate) != NULL)
8748 		dtrace_predicate_release(pred, vstate);
8749 
8750 	dtrace_ecb_action_remove(ecb);
8751 
8752 	ASSERT(state->dts_ecbs[epid - 1] == ecb);
8753 	state->dts_ecbs[epid - 1] = NULL;
8754 
8755 	kmem_free(ecb, sizeof (dtrace_ecb_t));
8756 }
8757 
8758 static dtrace_ecb_t *
8759 dtrace_ecb_create(dtrace_state_t *state, dtrace_probe_t *probe,
8760     dtrace_enabling_t *enab)
8761 {
8762 	dtrace_ecb_t *ecb;
8763 	dtrace_predicate_t *pred;
8764 	dtrace_actdesc_t *act;
8765 	dtrace_provider_t *prov;
8766 	dtrace_ecbdesc_t *desc = enab->dten_current;
8767 
8768 	ASSERT(MUTEX_HELD(&dtrace_lock));
8769 	ASSERT(state != NULL);
8770 
8771 	ecb = dtrace_ecb_add(state, probe);
8772 	ecb->dte_uarg = desc->dted_uarg;
8773 
8774 	if ((pred = desc->dted_pred.dtpdd_predicate) != NULL) {
8775 		dtrace_predicate_hold(pred);
8776 		ecb->dte_predicate = pred;
8777 	}
8778 
8779 	if (probe != NULL) {
8780 		/*
8781 		 * If the provider shows more leg than the consumer is old
8782 		 * enough to see, we need to enable the appropriate implicit
8783 		 * predicate bits to prevent the ecb from activating at
8784 		 * revealing times.
8785 		 */
8786 		prov = probe->dtpr_provider;
8787 		if (!(state->dts_cred.dcr_visible & DTRACE_CRV_ALLPROC) &&
8788 		    (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_USER))
8789 			ecb->dte_cond |= DTRACE_COND_OWNER;
8790 
8791 		if (!(state->dts_cred.dcr_visible & DTRACE_CRV_KERNEL) &&
8792 		    (prov->dtpv_priv.dtpp_flags & DTRACE_PRIV_KERNEL))
8793 			ecb->dte_cond |= DTRACE_COND_USERMODE;
8794 	}
8795 
8796 	if (dtrace_ecb_create_cache != NULL) {
8797 		/*
8798 		 * If we have a cached ecb, we'll use its action list instead
8799 		 * of creating our own (saving both time and space).
8800 		 */
8801 		dtrace_ecb_t *cached = dtrace_ecb_create_cache;
8802 		dtrace_action_t *act = cached->dte_action;
8803 
8804 		if (act != NULL) {
8805 			ASSERT(act->dta_refcnt > 0);
8806 			act->dta_refcnt++;
8807 			ecb->dte_action = act;
8808 			ecb->dte_action_last = cached->dte_action_last;
8809 			ecb->dte_needed = cached->dte_needed;
8810 			ecb->dte_size = cached->dte_size;
8811 			ecb->dte_alignment = cached->dte_alignment;
8812 		}
8813 
8814 		return (ecb);
8815 	}
8816 
8817 	for (act = desc->dted_action; act != NULL; act = act->dtad_next) {
8818 		if ((enab->dten_error = dtrace_ecb_action_add(ecb, act)) != 0) {
8819 			dtrace_ecb_destroy(ecb);
8820 			return (NULL);
8821 		}
8822 	}
8823 
8824 	dtrace_ecb_resize(ecb);
8825 
8826 	return (dtrace_ecb_create_cache = ecb);
8827 }
8828 
8829 static int
8830 dtrace_ecb_create_enable(dtrace_probe_t *probe, void *arg)
8831 {
8832 	dtrace_ecb_t *ecb;
8833 	dtrace_enabling_t *enab = arg;
8834 	dtrace_state_t *state = enab->dten_vstate->dtvs_state;
8835 
8836 	ASSERT(state != NULL);
8837 
8838 	if (probe != NULL && probe->dtpr_gen < enab->dten_probegen) {
8839 		/*
8840 		 * This probe was created in a generation for which this
8841 		 * enabling has previously created ECBs; we don't want to
8842 		 * enable it again, so just kick out.
8843 		 */
8844 		return (DTRACE_MATCH_NEXT);
8845 	}
8846 
8847 	if ((ecb = dtrace_ecb_create(state, probe, enab)) == NULL)
8848 		return (DTRACE_MATCH_DONE);
8849 
8850 	dtrace_ecb_enable(ecb);
8851 	return (DTRACE_MATCH_NEXT);
8852 }
8853 
8854 static dtrace_ecb_t *
8855 dtrace_epid2ecb(dtrace_state_t *state, dtrace_epid_t id)
8856 {
8857 	dtrace_ecb_t *ecb;
8858 
8859 	ASSERT(MUTEX_HELD(&dtrace_lock));
8860 
8861 	if (id == 0 || id > state->dts_necbs)
8862 		return (NULL);
8863 
8864 	ASSERT(state->dts_necbs > 0 && state->dts_ecbs != NULL);
8865 	ASSERT((ecb = state->dts_ecbs[id - 1]) == NULL || ecb->dte_epid == id);
8866 
8867 	return (state->dts_ecbs[id - 1]);
8868 }
8869 
8870 static dtrace_aggregation_t *
8871 dtrace_aggid2agg(dtrace_state_t *state, dtrace_aggid_t id)
8872 {
8873 	dtrace_aggregation_t *agg;
8874 
8875 	ASSERT(MUTEX_HELD(&dtrace_lock));
8876 
8877 	if (id == 0 || id > state->dts_naggregations)
8878 		return (NULL);
8879 
8880 	ASSERT(state->dts_naggregations > 0 && state->dts_aggregations != NULL);
8881 	ASSERT((agg = state->dts_aggregations[id - 1]) == NULL ||
8882 	    agg->dtag_id == id);
8883 
8884 	return (state->dts_aggregations[id - 1]);
8885 }
8886 
8887 /*
8888  * DTrace Buffer Functions
8889  *
8890  * The following functions manipulate DTrace buffers.  Most of these functions
8891  * are called in the context of establishing or processing consumer state;
8892  * exceptions are explicitly noted.
8893  */
8894 
8895 /*
8896  * Note:  called from cross call context.  This function switches the two
8897  * buffers on a given CPU.  The atomicity of this operation is assured by
8898  * disabling interrupts while the actual switch takes place; the disabling of
8899  * interrupts serializes the execution with any execution of dtrace_probe() on
8900  * the same CPU.
8901  */
8902 static void
8903 dtrace_buffer_switch(dtrace_buffer_t *buf)
8904 {
8905 	caddr_t tomax = buf->dtb_tomax;
8906 	caddr_t xamot = buf->dtb_xamot;
8907 	dtrace_icookie_t cookie;
8908 
8909 	ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
8910 	ASSERT(!(buf->dtb_flags & DTRACEBUF_RING));
8911 
8912 	cookie = dtrace_interrupt_disable();
8913 	buf->dtb_tomax = xamot;
8914 	buf->dtb_xamot = tomax;
8915 	buf->dtb_xamot_drops = buf->dtb_drops;
8916 	buf->dtb_xamot_offset = buf->dtb_offset;
8917 	buf->dtb_xamot_errors = buf->dtb_errors;
8918 	buf->dtb_xamot_flags = buf->dtb_flags;
8919 	buf->dtb_offset = 0;
8920 	buf->dtb_drops = 0;
8921 	buf->dtb_errors = 0;
8922 	buf->dtb_flags &= ~(DTRACEBUF_ERROR | DTRACEBUF_DROPPED);
8923 	dtrace_interrupt_enable(cookie);
8924 }
8925 
8926 /*
8927  * Note:  called from cross call context.  This function activates a buffer
8928  * on a CPU.  As with dtrace_buffer_switch(), the atomicity of the operation
8929  * is guaranteed by the disabling of interrupts.
8930  */
8931 static void
8932 dtrace_buffer_activate(dtrace_state_t *state)
8933 {
8934 	dtrace_buffer_t *buf;
8935 	dtrace_icookie_t cookie = dtrace_interrupt_disable();
8936 
8937 	buf = &state->dts_buffer[CPU->cpu_id];
8938 
8939 	if (buf->dtb_tomax != NULL) {
8940 		/*
8941 		 * We might like to assert that the buffer is marked inactive,
8942 		 * but this isn't necessarily true:  the buffer for the CPU
8943 		 * that processes the BEGIN probe has its buffer activated
8944 		 * manually.  In this case, we take the (harmless) action
8945 		 * re-clearing the bit INACTIVE bit.
8946 		 */
8947 		buf->dtb_flags &= ~DTRACEBUF_INACTIVE;
8948 	}
8949 
8950 	dtrace_interrupt_enable(cookie);
8951 }
8952 
8953 static int
8954 dtrace_buffer_alloc(dtrace_buffer_t *bufs, size_t size, int flags,
8955     processorid_t cpu)
8956 {
8957 	cpu_t *cp;
8958 	dtrace_buffer_t *buf;
8959 
8960 	ASSERT(MUTEX_HELD(&cpu_lock));
8961 	ASSERT(MUTEX_HELD(&dtrace_lock));
8962 
8963 	if (crgetuid(CRED()) != 0 && size > dtrace_nonroot_maxsize)
8964 		return (EFBIG);
8965 
8966 	cp = cpu_list;
8967 
8968 	do {
8969 		if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
8970 			continue;
8971 
8972 		buf = &bufs[cp->cpu_id];
8973 
8974 		/*
8975 		 * If there is already a buffer allocated for this CPU, it
8976 		 * is only possible that this is a DR event.  In this case,
8977 		 * the buffer size must match our specified size.
8978 		 */
8979 		if (buf->dtb_tomax != NULL) {
8980 			ASSERT(buf->dtb_size == size);
8981 			continue;
8982 		}
8983 
8984 		ASSERT(buf->dtb_xamot == NULL);
8985 
8986 		if ((buf->dtb_tomax = kmem_zalloc(size, KM_NOSLEEP)) == NULL)
8987 			goto err;
8988 
8989 		buf->dtb_size = size;
8990 		buf->dtb_flags = flags;
8991 		buf->dtb_offset = 0;
8992 		buf->dtb_drops = 0;
8993 
8994 		if (flags & DTRACEBUF_NOSWITCH)
8995 			continue;
8996 
8997 		if ((buf->dtb_xamot = kmem_zalloc(size, KM_NOSLEEP)) == NULL)
8998 			goto err;
8999 	} while ((cp = cp->cpu_next) != cpu_list);
9000 
9001 	return (0);
9002 
9003 err:
9004 	cp = cpu_list;
9005 
9006 	do {
9007 		if (cpu != DTRACE_CPUALL && cpu != cp->cpu_id)
9008 			continue;
9009 
9010 		buf = &bufs[cp->cpu_id];
9011 
9012 		if (buf->dtb_xamot != NULL) {
9013 			ASSERT(buf->dtb_tomax != NULL);
9014 			ASSERT(buf->dtb_size == size);
9015 			kmem_free(buf->dtb_xamot, size);
9016 		}
9017 
9018 		if (buf->dtb_tomax != NULL) {
9019 			ASSERT(buf->dtb_size == size);
9020 			kmem_free(buf->dtb_tomax, size);
9021 		}
9022 
9023 		buf->dtb_tomax = NULL;
9024 		buf->dtb_xamot = NULL;
9025 		buf->dtb_size = 0;
9026 	} while ((cp = cp->cpu_next) != cpu_list);
9027 
9028 	return (ENOMEM);
9029 }
9030 
9031 /*
9032  * Note:  called from probe context.  This function just increments the drop
9033  * count on a buffer.  It has been made a function to allow for the
9034  * possibility of understanding the source of mysterious drop counts.  (A
9035  * problem for which one may be particularly disappointed that DTrace cannot
9036  * be used to understand DTrace.)
9037  */
9038 static void
9039 dtrace_buffer_drop(dtrace_buffer_t *buf)
9040 {
9041 	buf->dtb_drops++;
9042 }
9043 
9044 /*
9045  * Note:  called from probe context.  This function is called to reserve space
9046  * in a buffer.  If mstate is non-NULL, sets the scratch base and size in the
9047  * mstate.  Returns the new offset in the buffer, or a negative value if an
9048  * error has occurred.
9049  */
9050 static intptr_t
9051 dtrace_buffer_reserve(dtrace_buffer_t *buf, size_t needed, size_t align,
9052     dtrace_state_t *state, dtrace_mstate_t *mstate)
9053 {
9054 	intptr_t offs = buf->dtb_offset, soffs;
9055 	intptr_t woffs;
9056 	caddr_t tomax;
9057 	size_t total;
9058 
9059 	if (buf->dtb_flags & DTRACEBUF_INACTIVE)
9060 		return (-1);
9061 
9062 	if ((tomax = buf->dtb_tomax) == NULL) {
9063 		dtrace_buffer_drop(buf);
9064 		return (-1);
9065 	}
9066 
9067 	if (!(buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL))) {
9068 		while (offs & (align - 1)) {
9069 			/*
9070 			 * Assert that our alignment is off by a number which
9071 			 * is itself sizeof (uint32_t) aligned.
9072 			 */
9073 			ASSERT(!((align - (offs & (align - 1))) &
9074 			    (sizeof (uint32_t) - 1)));
9075 			DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
9076 			offs += sizeof (uint32_t);
9077 		}
9078 
9079 		if ((soffs = offs + needed) > buf->dtb_size) {
9080 			dtrace_buffer_drop(buf);
9081 			return (-1);
9082 		}
9083 
9084 		if (mstate == NULL)
9085 			return (offs);
9086 
9087 		mstate->dtms_scratch_base = (uintptr_t)tomax + soffs;
9088 		mstate->dtms_scratch_size = buf->dtb_size - soffs;
9089 		mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
9090 
9091 		return (offs);
9092 	}
9093 
9094 	if (buf->dtb_flags & DTRACEBUF_FILL) {
9095 		if (state->dts_activity != DTRACE_ACTIVITY_COOLDOWN &&
9096 		    (buf->dtb_flags & DTRACEBUF_FULL))
9097 			return (-1);
9098 		goto out;
9099 	}
9100 
9101 	total = needed + (offs & (align - 1));
9102 
9103 	/*
9104 	 * For a ring buffer, life is quite a bit more complicated.  Before
9105 	 * we can store any padding, we need to adjust our wrapping offset.
9106 	 * (If we've never before wrapped or we're not about to, no adjustment
9107 	 * is required.)
9108 	 */
9109 	if ((buf->dtb_flags & DTRACEBUF_WRAPPED) ||
9110 	    offs + total > buf->dtb_size) {
9111 		woffs = buf->dtb_xamot_offset;
9112 
9113 		if (offs + total > buf->dtb_size) {
9114 			/*
9115 			 * We can't fit in the end of the buffer.  First, a
9116 			 * sanity check that we can fit in the buffer at all.
9117 			 */
9118 			if (total > buf->dtb_size) {
9119 				dtrace_buffer_drop(buf);
9120 				return (-1);
9121 			}
9122 
9123 			/*
9124 			 * We're going to be storing at the top of the buffer,
9125 			 * so now we need to deal with the wrapped offset.  We
9126 			 * only reset our wrapped offset to 0 if it is
9127 			 * currently greater than the current offset.  If it
9128 			 * is less than the current offset, it is because a
9129 			 * previous allocation induced a wrap -- but the
9130 			 * allocation didn't subsequently take the space due
9131 			 * to an error or false predicate evaluation.  In this
9132 			 * case, we'll just leave the wrapped offset alone: if
9133 			 * the wrapped offset hasn't been advanced far enough
9134 			 * for this allocation, it will be adjusted in the
9135 			 * lower loop.
9136 			 */
9137 			if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
9138 				if (woffs >= offs)
9139 					woffs = 0;
9140 			} else {
9141 				woffs = 0;
9142 			}
9143 
9144 			/*
9145 			 * Now we know that we're going to be storing to the
9146 			 * top of the buffer and that there is room for us
9147 			 * there.  We need to clear the buffer from the current
9148 			 * offset to the end (there may be old gunk there).
9149 			 */
9150 			while (offs < buf->dtb_size)
9151 				tomax[offs++] = 0;
9152 
9153 			/*
9154 			 * We need to set our offset to zero.  And because we
9155 			 * are wrapping, we need to set the bit indicating as
9156 			 * much.  We can also adjust our needed space back
9157 			 * down to the space required by the ECB -- we know
9158 			 * that the top of the buffer is aligned.
9159 			 */
9160 			offs = 0;
9161 			total = needed;
9162 			buf->dtb_flags |= DTRACEBUF_WRAPPED;
9163 		} else {
9164 			/*
9165 			 * There is room for us in the buffer, so we simply
9166 			 * need to check the wrapped offset.
9167 			 */
9168 			if (woffs < offs) {
9169 				/*
9170 				 * The wrapped offset is less than the offset.
9171 				 * This can happen if we allocated buffer space
9172 				 * that induced a wrap, but then we didn't
9173 				 * subsequently take the space due to an error
9174 				 * or false predicate evaluation.  This is
9175 				 * okay; we know that _this_ allocation isn't
9176 				 * going to induce a wrap.  We still can't
9177 				 * reset the wrapped offset to be zero,
9178 				 * however: the space may have been trashed in
9179 				 * the previous failed probe attempt.  But at
9180 				 * least the wrapped offset doesn't need to
9181 				 * be adjusted at all...
9182 				 */
9183 				goto out;
9184 			}
9185 		}
9186 
9187 		while (offs + total > woffs) {
9188 			dtrace_epid_t epid = *(uint32_t *)(tomax + woffs);
9189 			size_t size;
9190 
9191 			if (epid == DTRACE_EPIDNONE) {
9192 				size = sizeof (uint32_t);
9193 			} else {
9194 				ASSERT(epid <= state->dts_necbs);
9195 				ASSERT(state->dts_ecbs[epid - 1] != NULL);
9196 
9197 				size = state->dts_ecbs[epid - 1]->dte_size;
9198 			}
9199 
9200 			ASSERT(woffs + size <= buf->dtb_size);
9201 			ASSERT(size != 0);
9202 
9203 			if (woffs + size == buf->dtb_size) {
9204 				/*
9205 				 * We've reached the end of the buffer; we want
9206 				 * to set the wrapped offset to 0 and break
9207 				 * out.  However, if the offs is 0, then we're
9208 				 * in a strange edge-condition:  the amount of
9209 				 * space that we want to reserve plus the size
9210 				 * of the record that we're overwriting is
9211 				 * greater than the size of the buffer.  This
9212 				 * is problematic because if we reserve the
9213 				 * space but subsequently don't consume it (due
9214 				 * to a failed predicate or error) the wrapped
9215 				 * offset will be 0 -- yet the EPID at offset 0
9216 				 * will not be committed.  This situation is
9217 				 * relatively easy to deal with:  if we're in
9218 				 * this case, the buffer is indistinguishable
9219 				 * from one that hasn't wrapped; we need only
9220 				 * finish the job by clearing the wrapped bit,
9221 				 * explicitly setting the offset to be 0, and
9222 				 * zero'ing out the old data in the buffer.
9223 				 */
9224 				if (offs == 0) {
9225 					buf->dtb_flags &= ~DTRACEBUF_WRAPPED;
9226 					buf->dtb_offset = 0;
9227 					woffs = total;
9228 
9229 					while (woffs < buf->dtb_size)
9230 						tomax[woffs++] = 0;
9231 				}
9232 
9233 				woffs = 0;
9234 				break;
9235 			}
9236 
9237 			woffs += size;
9238 		}
9239 
9240 		/*
9241 		 * We have a wrapped offset.  It may be that the wrapped offset
9242 		 * has become zero -- that's okay.
9243 		 */
9244 		buf->dtb_xamot_offset = woffs;
9245 	}
9246 
9247 out:
9248 	/*
9249 	 * Now we can plow the buffer with any necessary padding.
9250 	 */
9251 	while (offs & (align - 1)) {
9252 		/*
9253 		 * Assert that our alignment is off by a number which
9254 		 * is itself sizeof (uint32_t) aligned.
9255 		 */
9256 		ASSERT(!((align - (offs & (align - 1))) &
9257 		    (sizeof (uint32_t) - 1)));
9258 		DTRACE_STORE(uint32_t, tomax, offs, DTRACE_EPIDNONE);
9259 		offs += sizeof (uint32_t);
9260 	}
9261 
9262 	if (buf->dtb_flags & DTRACEBUF_FILL) {
9263 		if (offs + needed > buf->dtb_size - state->dts_reserve) {
9264 			buf->dtb_flags |= DTRACEBUF_FULL;
9265 			return (-1);
9266 		}
9267 	}
9268 
9269 	if (mstate == NULL)
9270 		return (offs);
9271 
9272 	/*
9273 	 * For ring buffers and fill buffers, the scratch space is always
9274 	 * the inactive buffer.
9275 	 */
9276 	mstate->dtms_scratch_base = (uintptr_t)buf->dtb_xamot;
9277 	mstate->dtms_scratch_size = buf->dtb_size;
9278 	mstate->dtms_scratch_ptr = mstate->dtms_scratch_base;
9279 
9280 	return (offs);
9281 }
9282 
9283 static void
9284 dtrace_buffer_polish(dtrace_buffer_t *buf)
9285 {
9286 	ASSERT(buf->dtb_flags & DTRACEBUF_RING);
9287 	ASSERT(MUTEX_HELD(&dtrace_lock));
9288 
9289 	if (!(buf->dtb_flags & DTRACEBUF_WRAPPED))
9290 		return;
9291 
9292 	/*
9293 	 * We need to polish the ring buffer.  There are three cases:
9294 	 *
9295 	 * - The first (and presumably most common) is that there is no gap
9296 	 *   between the buffer offset and the wrapped offset.  In this case,
9297 	 *   there is nothing in the buffer that isn't valid data; we can
9298 	 *   mark the buffer as polished and return.
9299 	 *
9300 	 * - The second (less common than the first but still more common
9301 	 *   than the third) is that there is a gap between the buffer offset
9302 	 *   and the wrapped offset, and the wrapped offset is larger than the
9303 	 *   buffer offset.  This can happen because of an alignment issue, or
9304 	 *   can happen because of a call to dtrace_buffer_reserve() that
9305 	 *   didn't subsequently consume the buffer space.  In this case,
9306 	 *   we need to zero the data from the buffer offset to the wrapped
9307 	 *   offset.
9308 	 *
9309 	 * - The third (and least common) is that there is a gap between the
9310 	 *   buffer offset and the wrapped offset, but the wrapped offset is
9311 	 *   _less_ than the buffer offset.  This can only happen because a
9312 	 *   call to dtrace_buffer_reserve() induced a wrap, but the space
9313 	 *   was not subsequently consumed.  In this case, we need to zero the
9314 	 *   space from the offset to the end of the buffer _and_ from the
9315 	 *   top of the buffer to the wrapped offset.
9316 	 */
9317 	if (buf->dtb_offset < buf->dtb_xamot_offset) {
9318 		bzero(buf->dtb_tomax + buf->dtb_offset,
9319 		    buf->dtb_xamot_offset - buf->dtb_offset);
9320 	}
9321 
9322 	if (buf->dtb_offset > buf->dtb_xamot_offset) {
9323 		bzero(buf->dtb_tomax + buf->dtb_offset,
9324 		    buf->dtb_size - buf->dtb_offset);
9325 		bzero(buf->dtb_tomax, buf->dtb_xamot_offset);
9326 	}
9327 }
9328 
9329 static void
9330 dtrace_buffer_free(dtrace_buffer_t *bufs)
9331 {
9332 	int i;
9333 
9334 	for (i = 0; i < NCPU; i++) {
9335 		dtrace_buffer_t *buf = &bufs[i];
9336 
9337 		if (buf->dtb_tomax == NULL) {
9338 			ASSERT(buf->dtb_xamot == NULL);
9339 			ASSERT(buf->dtb_size == 0);
9340 			continue;
9341 		}
9342 
9343 		if (buf->dtb_xamot != NULL) {
9344 			ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
9345 			kmem_free(buf->dtb_xamot, buf->dtb_size);
9346 		}
9347 
9348 		kmem_free(buf->dtb_tomax, buf->dtb_size);
9349 		buf->dtb_size = 0;
9350 		buf->dtb_tomax = NULL;
9351 		buf->dtb_xamot = NULL;
9352 	}
9353 }
9354 
9355 /*
9356  * DTrace Enabling Functions
9357  */
9358 static dtrace_enabling_t *
9359 dtrace_enabling_create(dtrace_vstate_t *vstate)
9360 {
9361 	dtrace_enabling_t *enab;
9362 
9363 	enab = kmem_zalloc(sizeof (dtrace_enabling_t), KM_SLEEP);
9364 	enab->dten_vstate = vstate;
9365 
9366 	return (enab);
9367 }
9368 
9369 static void
9370 dtrace_enabling_add(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb)
9371 {
9372 	dtrace_ecbdesc_t **ndesc;
9373 	size_t osize, nsize;
9374 
9375 	/*
9376 	 * We can't add to enablings after we've enabled them, or after we've
9377 	 * retained them.
9378 	 */
9379 	ASSERT(enab->dten_probegen == 0);
9380 	ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
9381 
9382 	if (enab->dten_ndesc < enab->dten_maxdesc) {
9383 		enab->dten_desc[enab->dten_ndesc++] = ecb;
9384 		return;
9385 	}
9386 
9387 	osize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
9388 
9389 	if (enab->dten_maxdesc == 0) {
9390 		enab->dten_maxdesc = 1;
9391 	} else {
9392 		enab->dten_maxdesc <<= 1;
9393 	}
9394 
9395 	ASSERT(enab->dten_ndesc < enab->dten_maxdesc);
9396 
9397 	nsize = enab->dten_maxdesc * sizeof (dtrace_enabling_t *);
9398 	ndesc = kmem_zalloc(nsize, KM_SLEEP);
9399 	bcopy(enab->dten_desc, ndesc, osize);
9400 	kmem_free(enab->dten_desc, osize);
9401 
9402 	enab->dten_desc = ndesc;
9403 	enab->dten_desc[enab->dten_ndesc++] = ecb;
9404 }
9405 
9406 static void
9407 dtrace_enabling_addlike(dtrace_enabling_t *enab, dtrace_ecbdesc_t *ecb,
9408     dtrace_probedesc_t *pd)
9409 {
9410 	dtrace_ecbdesc_t *new;
9411 	dtrace_predicate_t *pred;
9412 	dtrace_actdesc_t *act;
9413 
9414 	/*
9415 	 * We're going to create a new ECB description that matches the
9416 	 * specified ECB in every way, but has the specified probe description.
9417 	 */
9418 	new = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
9419 
9420 	if ((pred = ecb->dted_pred.dtpdd_predicate) != NULL)
9421 		dtrace_predicate_hold(pred);
9422 
9423 	for (act = ecb->dted_action; act != NULL; act = act->dtad_next)
9424 		dtrace_actdesc_hold(act);
9425 
9426 	new->dted_action = ecb->dted_action;
9427 	new->dted_pred = ecb->dted_pred;
9428 	new->dted_probe = *pd;
9429 	new->dted_uarg = ecb->dted_uarg;
9430 
9431 	dtrace_enabling_add(enab, new);
9432 }
9433 
9434 static void
9435 dtrace_enabling_dump(dtrace_enabling_t *enab)
9436 {
9437 	int i;
9438 
9439 	for (i = 0; i < enab->dten_ndesc; i++) {
9440 		dtrace_probedesc_t *desc = &enab->dten_desc[i]->dted_probe;
9441 
9442 		cmn_err(CE_NOTE, "enabling probe %d (%s:%s:%s:%s)", i,
9443 		    desc->dtpd_provider, desc->dtpd_mod,
9444 		    desc->dtpd_func, desc->dtpd_name);
9445 	}
9446 }
9447 
9448 static void
9449 dtrace_enabling_destroy(dtrace_enabling_t *enab)
9450 {
9451 	int i;
9452 	dtrace_ecbdesc_t *ep;
9453 	dtrace_vstate_t *vstate = enab->dten_vstate;
9454 
9455 	ASSERT(MUTEX_HELD(&dtrace_lock));
9456 
9457 	for (i = 0; i < enab->dten_ndesc; i++) {
9458 		dtrace_actdesc_t *act, *next;
9459 		dtrace_predicate_t *pred;
9460 
9461 		ep = enab->dten_desc[i];
9462 
9463 		if ((pred = ep->dted_pred.dtpdd_predicate) != NULL)
9464 			dtrace_predicate_release(pred, vstate);
9465 
9466 		for (act = ep->dted_action; act != NULL; act = next) {
9467 			next = act->dtad_next;
9468 			dtrace_actdesc_release(act, vstate);
9469 		}
9470 
9471 		kmem_free(ep, sizeof (dtrace_ecbdesc_t));
9472 	}
9473 
9474 	kmem_free(enab->dten_desc,
9475 	    enab->dten_maxdesc * sizeof (dtrace_enabling_t *));
9476 
9477 	/*
9478 	 * If this was a retained enabling, decrement the dts_nretained count
9479 	 * and take it off of the dtrace_retained list.
9480 	 */
9481 	if (enab->dten_prev != NULL || enab->dten_next != NULL ||
9482 	    dtrace_retained == enab) {
9483 		ASSERT(enab->dten_vstate->dtvs_state != NULL);
9484 		ASSERT(enab->dten_vstate->dtvs_state->dts_nretained > 0);
9485 		enab->dten_vstate->dtvs_state->dts_nretained--;
9486 	}
9487 
9488 	if (enab->dten_prev == NULL) {
9489 		if (dtrace_retained == enab) {
9490 			dtrace_retained = enab->dten_next;
9491 
9492 			if (dtrace_retained != NULL)
9493 				dtrace_retained->dten_prev = NULL;
9494 		}
9495 	} else {
9496 		ASSERT(enab != dtrace_retained);
9497 		ASSERT(dtrace_retained != NULL);
9498 		enab->dten_prev->dten_next = enab->dten_next;
9499 	}
9500 
9501 	if (enab->dten_next != NULL) {
9502 		ASSERT(dtrace_retained != NULL);
9503 		enab->dten_next->dten_prev = enab->dten_prev;
9504 	}
9505 
9506 	kmem_free(enab, sizeof (dtrace_enabling_t));
9507 }
9508 
9509 static int
9510 dtrace_enabling_retain(dtrace_enabling_t *enab)
9511 {
9512 	dtrace_state_t *state;
9513 
9514 	ASSERT(MUTEX_HELD(&dtrace_lock));
9515 	ASSERT(enab->dten_next == NULL && enab->dten_prev == NULL);
9516 	ASSERT(enab->dten_vstate != NULL);
9517 
9518 	state = enab->dten_vstate->dtvs_state;
9519 	ASSERT(state != NULL);
9520 
9521 	/*
9522 	 * We only allow each state to retain dtrace_retain_max enablings.
9523 	 */
9524 	if (state->dts_nretained >= dtrace_retain_max)
9525 		return (ENOSPC);
9526 
9527 	state->dts_nretained++;
9528 
9529 	if (dtrace_retained == NULL) {
9530 		dtrace_retained = enab;
9531 		return (0);
9532 	}
9533 
9534 	enab->dten_next = dtrace_retained;
9535 	dtrace_retained->dten_prev = enab;
9536 	dtrace_retained = enab;
9537 
9538 	return (0);
9539 }
9540 
9541 static int
9542 dtrace_enabling_replicate(dtrace_state_t *state, dtrace_probedesc_t *match,
9543     dtrace_probedesc_t *create)
9544 {
9545 	dtrace_enabling_t *new, *enab;
9546 	int found = 0, err = ENOENT;
9547 
9548 	ASSERT(MUTEX_HELD(&dtrace_lock));
9549 	ASSERT(strlen(match->dtpd_provider) < DTRACE_PROVNAMELEN);
9550 	ASSERT(strlen(match->dtpd_mod) < DTRACE_MODNAMELEN);
9551 	ASSERT(strlen(match->dtpd_func) < DTRACE_FUNCNAMELEN);
9552 	ASSERT(strlen(match->dtpd_name) < DTRACE_NAMELEN);
9553 
9554 	new = dtrace_enabling_create(&state->dts_vstate);
9555 
9556 	/*
9557 	 * Iterate over all retained enablings, looking for enablings that
9558 	 * match the specified state.
9559 	 */
9560 	for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
9561 		int i;
9562 
9563 		/*
9564 		 * dtvs_state can only be NULL for helper enablings -- and
9565 		 * helper enablings can't be retained.
9566 		 */
9567 		ASSERT(enab->dten_vstate->dtvs_state != NULL);
9568 
9569 		if (enab->dten_vstate->dtvs_state != state)
9570 			continue;
9571 
9572 		/*
9573 		 * Now iterate over each probe description; we're looking for
9574 		 * an exact match to the specified probe description.
9575 		 */
9576 		for (i = 0; i < enab->dten_ndesc; i++) {
9577 			dtrace_ecbdesc_t *ep = enab->dten_desc[i];
9578 			dtrace_probedesc_t *pd = &ep->dted_probe;
9579 
9580 			if (strcmp(pd->dtpd_provider, match->dtpd_provider))
9581 				continue;
9582 
9583 			if (strcmp(pd->dtpd_mod, match->dtpd_mod))
9584 				continue;
9585 
9586 			if (strcmp(pd->dtpd_func, match->dtpd_func))
9587 				continue;
9588 
9589 			if (strcmp(pd->dtpd_name, match->dtpd_name))
9590 				continue;
9591 
9592 			/*
9593 			 * We have a winning probe!  Add it to our growing
9594 			 * enabling.
9595 			 */
9596 			found = 1;
9597 			dtrace_enabling_addlike(new, ep, create);
9598 		}
9599 	}
9600 
9601 	if (!found || (err = dtrace_enabling_retain(new)) != 0) {
9602 		dtrace_enabling_destroy(new);
9603 		return (err);
9604 	}
9605 
9606 	return (0);
9607 }
9608 
9609 static void
9610 dtrace_enabling_retract(dtrace_state_t *state)
9611 {
9612 	dtrace_enabling_t *enab, *next;
9613 
9614 	ASSERT(MUTEX_HELD(&dtrace_lock));
9615 
9616 	/*
9617 	 * Iterate over all retained enablings, destroy the enablings retained
9618 	 * for the specified state.
9619 	 */
9620 	for (enab = dtrace_retained; enab != NULL; enab = next) {
9621 		next = enab->dten_next;
9622 
9623 		/*
9624 		 * dtvs_state can only be NULL for helper enablings -- and
9625 		 * helper enablings can't be retained.
9626 		 */
9627 		ASSERT(enab->dten_vstate->dtvs_state != NULL);
9628 
9629 		if (enab->dten_vstate->dtvs_state == state) {
9630 			ASSERT(state->dts_nretained > 0);
9631 			dtrace_enabling_destroy(enab);
9632 		}
9633 	}
9634 
9635 	ASSERT(state->dts_nretained == 0);
9636 }
9637 
9638 static int
9639 dtrace_enabling_match(dtrace_enabling_t *enab, int *nmatched)
9640 {
9641 	int i = 0;
9642 	int matched = 0;
9643 
9644 	ASSERT(MUTEX_HELD(&cpu_lock));
9645 	ASSERT(MUTEX_HELD(&dtrace_lock));
9646 
9647 	for (i = 0; i < enab->dten_ndesc; i++) {
9648 		dtrace_ecbdesc_t *ep = enab->dten_desc[i];
9649 
9650 		enab->dten_current = ep;
9651 		enab->dten_error = 0;
9652 
9653 		matched += dtrace_probe_enable(&ep->dted_probe, enab);
9654 
9655 		if (enab->dten_error != 0) {
9656 			/*
9657 			 * If we get an error half-way through enabling the
9658 			 * probes, we kick out -- perhaps with some number of
9659 			 * them enabled.  Leaving enabled probes enabled may
9660 			 * be slightly confusing for user-level, but we expect
9661 			 * that no one will attempt to actually drive on in
9662 			 * the face of such errors.  If this is an anonymous
9663 			 * enabling (indicated with a NULL nmatched pointer),
9664 			 * we cmn_err() a message.  We aren't expecting to
9665 			 * get such an error -- such as it can exist at all,
9666 			 * it would be a result of corrupted DOF in the driver
9667 			 * properties.
9668 			 */
9669 			if (nmatched == NULL) {
9670 				cmn_err(CE_WARN, "dtrace_enabling_match() "
9671 				    "error on %p: %d", (void *)ep,
9672 				    enab->dten_error);
9673 			}
9674 
9675 			return (enab->dten_error);
9676 		}
9677 	}
9678 
9679 	enab->dten_probegen = dtrace_probegen;
9680 	if (nmatched != NULL)
9681 		*nmatched = matched;
9682 
9683 	return (0);
9684 }
9685 
9686 static void
9687 dtrace_enabling_matchall(void)
9688 {
9689 	dtrace_enabling_t *enab;
9690 
9691 	mutex_enter(&cpu_lock);
9692 	mutex_enter(&dtrace_lock);
9693 
9694 	/*
9695 	 * Because we can be called after dtrace_detach() has been called, we
9696 	 * cannot assert that there are retained enablings.  We can safely
9697 	 * load from dtrace_retained, however:  the taskq_destroy() at the
9698 	 * end of dtrace_detach() will block pending our completion.
9699 	 */
9700 	for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next)
9701 		(void) dtrace_enabling_match(enab, NULL);
9702 
9703 	mutex_exit(&dtrace_lock);
9704 	mutex_exit(&cpu_lock);
9705 }
9706 
9707 static int
9708 dtrace_enabling_matchstate(dtrace_state_t *state, int *nmatched)
9709 {
9710 	dtrace_enabling_t *enab;
9711 	int matched, total = 0, err;
9712 
9713 	ASSERT(MUTEX_HELD(&cpu_lock));
9714 	ASSERT(MUTEX_HELD(&dtrace_lock));
9715 
9716 	for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
9717 		ASSERT(enab->dten_vstate->dtvs_state != NULL);
9718 
9719 		if (enab->dten_vstate->dtvs_state != state)
9720 			continue;
9721 
9722 		if ((err = dtrace_enabling_match(enab, &matched)) != 0)
9723 			return (err);
9724 
9725 		total += matched;
9726 	}
9727 
9728 	if (nmatched != NULL)
9729 		*nmatched = total;
9730 
9731 	return (0);
9732 }
9733 
9734 /*
9735  * If an enabling is to be enabled without having matched probes (that is, if
9736  * dtrace_state_go() is to be called on the underlying dtrace_state_t), the
9737  * enabling must be _primed_ by creating an ECB for every ECB description.
9738  * This must be done to assure that we know the number of speculations, the
9739  * number of aggregations, the minimum buffer size needed, etc. before we
9740  * transition out of DTRACE_ACTIVITY_INACTIVE.  To do this without actually
9741  * enabling any probes, we create ECBs for every ECB decription, but with a
9742  * NULL probe -- which is exactly what this function does.
9743  */
9744 static void
9745 dtrace_enabling_prime(dtrace_state_t *state)
9746 {
9747 	dtrace_enabling_t *enab;
9748 	int i;
9749 
9750 	for (enab = dtrace_retained; enab != NULL; enab = enab->dten_next) {
9751 		ASSERT(enab->dten_vstate->dtvs_state != NULL);
9752 
9753 		if (enab->dten_vstate->dtvs_state != state)
9754 			continue;
9755 
9756 		/*
9757 		 * We don't want to prime an enabling more than once, lest
9758 		 * we allow a malicious user to induce resource exhaustion.
9759 		 * (The ECBs that result from priming an enabling aren't
9760 		 * leaked -- but they also aren't deallocated until the
9761 		 * consumer state is destroyed.)
9762 		 */
9763 		if (enab->dten_primed)
9764 			continue;
9765 
9766 		for (i = 0; i < enab->dten_ndesc; i++) {
9767 			enab->dten_current = enab->dten_desc[i];
9768 			(void) dtrace_probe_enable(NULL, enab);
9769 		}
9770 
9771 		enab->dten_primed = 1;
9772 	}
9773 }
9774 
9775 /*
9776  * Called to indicate that probes should be provided due to retained
9777  * enablings.  This is implemented in terms of dtrace_probe_provide(), but it
9778  * must take an initial lap through the enabling calling the dtps_provide()
9779  * entry point explicitly to allow for autocreated probes.
9780  */
9781 static void
9782 dtrace_enabling_provide(dtrace_provider_t *prv)
9783 {
9784 	int i, all = 0;
9785 	dtrace_probedesc_t desc;
9786 
9787 	ASSERT(MUTEX_HELD(&dtrace_lock));
9788 	ASSERT(MUTEX_HELD(&dtrace_provider_lock));
9789 
9790 	if (prv == NULL) {
9791 		all = 1;
9792 		prv = dtrace_provider;
9793 	}
9794 
9795 	do {
9796 		dtrace_enabling_t *enab = dtrace_retained;
9797 		void *parg = prv->dtpv_arg;
9798 
9799 		for (; enab != NULL; enab = enab->dten_next) {
9800 			for (i = 0; i < enab->dten_ndesc; i++) {
9801 				desc = enab->dten_desc[i]->dted_probe;
9802 				mutex_exit(&dtrace_lock);
9803 				prv->dtpv_pops.dtps_provide(parg, &desc);
9804 				mutex_enter(&dtrace_lock);
9805 			}
9806 		}
9807 	} while (all && (prv = prv->dtpv_next) != NULL);
9808 
9809 	mutex_exit(&dtrace_lock);
9810 	dtrace_probe_provide(NULL, all ? NULL : prv);
9811 	mutex_enter(&dtrace_lock);
9812 }
9813 
9814 /*
9815  * DTrace DOF Functions
9816  */
9817 /*ARGSUSED*/
9818 static void
9819 dtrace_dof_error(dof_hdr_t *dof, const char *str)
9820 {
9821 	if (dtrace_err_verbose)
9822 		cmn_err(CE_WARN, "failed to process DOF: %s", str);
9823 
9824 #ifdef DTRACE_ERRDEBUG
9825 	dtrace_errdebug(str);
9826 #endif
9827 }
9828 
9829 /*
9830  * Create DOF out of a currently enabled state.  Right now, we only create
9831  * DOF containing the run-time options -- but this could be expanded to create
9832  * complete DOF representing the enabled state.
9833  */
9834 static dof_hdr_t *
9835 dtrace_dof_create(dtrace_state_t *state)
9836 {
9837 	dof_hdr_t *dof;
9838 	dof_sec_t *sec;
9839 	dof_optdesc_t *opt;
9840 	int i, len = sizeof (dof_hdr_t) +
9841 	    roundup(sizeof (dof_sec_t), sizeof (uint64_t)) +
9842 	    sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
9843 
9844 	ASSERT(MUTEX_HELD(&dtrace_lock));
9845 
9846 	dof = kmem_zalloc(len, KM_SLEEP);
9847 	dof->dofh_ident[DOF_ID_MAG0] = DOF_MAG_MAG0;
9848 	dof->dofh_ident[DOF_ID_MAG1] = DOF_MAG_MAG1;
9849 	dof->dofh_ident[DOF_ID_MAG2] = DOF_MAG_MAG2;
9850 	dof->dofh_ident[DOF_ID_MAG3] = DOF_MAG_MAG3;
9851 
9852 	dof->dofh_ident[DOF_ID_MODEL] = DOF_MODEL_NATIVE;
9853 	dof->dofh_ident[DOF_ID_ENCODING] = DOF_ENCODE_NATIVE;
9854 	dof->dofh_ident[DOF_ID_VERSION] = DOF_VERSION_1;
9855 	dof->dofh_ident[DOF_ID_DIFVERS] = DIF_VERSION;
9856 	dof->dofh_ident[DOF_ID_DIFIREG] = DIF_DIR_NREGS;
9857 	dof->dofh_ident[DOF_ID_DIFTREG] = DIF_DTR_NREGS;
9858 
9859 	dof->dofh_flags = 0;
9860 	dof->dofh_hdrsize = sizeof (dof_hdr_t);
9861 	dof->dofh_secsize = sizeof (dof_sec_t);
9862 	dof->dofh_secnum = 1;	/* only DOF_SECT_OPTDESC */
9863 	dof->dofh_secoff = sizeof (dof_hdr_t);
9864 	dof->dofh_loadsz = len;
9865 	dof->dofh_filesz = len;
9866 	dof->dofh_pad = 0;
9867 
9868 	/*
9869 	 * Fill in the option section header...
9870 	 */
9871 	sec = (dof_sec_t *)((uintptr_t)dof + sizeof (dof_hdr_t));
9872 	sec->dofs_type = DOF_SECT_OPTDESC;
9873 	sec->dofs_align = sizeof (uint64_t);
9874 	sec->dofs_flags = DOF_SECF_LOAD;
9875 	sec->dofs_entsize = sizeof (dof_optdesc_t);
9876 
9877 	opt = (dof_optdesc_t *)((uintptr_t)sec +
9878 	    roundup(sizeof (dof_sec_t), sizeof (uint64_t)));
9879 
9880 	sec->dofs_offset = (uintptr_t)opt - (uintptr_t)dof;
9881 	sec->dofs_size = sizeof (dof_optdesc_t) * DTRACEOPT_MAX;
9882 
9883 	for (i = 0; i < DTRACEOPT_MAX; i++) {
9884 		opt[i].dofo_option = i;
9885 		opt[i].dofo_strtab = DOF_SECIDX_NONE;
9886 		opt[i].dofo_value = state->dts_options[i];
9887 	}
9888 
9889 	return (dof);
9890 }
9891 
9892 static dof_hdr_t *
9893 dtrace_dof_copyin(uintptr_t uarg, int *errp)
9894 {
9895 	dof_hdr_t hdr, *dof;
9896 
9897 	ASSERT(!MUTEX_HELD(&dtrace_lock));
9898 
9899 	/*
9900 	 * First, we're going to copyin() the sizeof (dof_hdr_t).
9901 	 */
9902 	if (copyin((void *)uarg, &hdr, sizeof (hdr)) != 0) {
9903 		dtrace_dof_error(NULL, "failed to copyin DOF header");
9904 		*errp = EFAULT;
9905 		return (NULL);
9906 	}
9907 
9908 	/*
9909 	 * Now we'll allocate the entire DOF and copy it in -- provided
9910 	 * that the length isn't outrageous.
9911 	 */
9912 	if (hdr.dofh_loadsz >= dtrace_dof_maxsize) {
9913 		dtrace_dof_error(&hdr, "load size exceeds maximum");
9914 		*errp = E2BIG;
9915 		return (NULL);
9916 	}
9917 
9918 	if (hdr.dofh_loadsz < sizeof (hdr)) {
9919 		dtrace_dof_error(&hdr, "invalid load size");
9920 		*errp = EINVAL;
9921 		return (NULL);
9922 	}
9923 
9924 	dof = kmem_alloc(hdr.dofh_loadsz, KM_SLEEP);
9925 
9926 	if (copyin((void *)uarg, dof, hdr.dofh_loadsz) != 0) {
9927 		kmem_free(dof, hdr.dofh_loadsz);
9928 		*errp = EFAULT;
9929 		return (NULL);
9930 	}
9931 
9932 	return (dof);
9933 }
9934 
9935 static dof_hdr_t *
9936 dtrace_dof_property(const char *name)
9937 {
9938 	uchar_t *buf;
9939 	uint64_t loadsz;
9940 	unsigned int len, i;
9941 	dof_hdr_t *dof;
9942 
9943 	/*
9944 	 * Unfortunately, array of values in .conf files are always (and
9945 	 * only) interpreted to be integer arrays.  We must read our DOF
9946 	 * as an integer array, and then squeeze it into a byte array.
9947 	 */
9948 	if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dtrace_devi, 0,
9949 	    (char *)name, (int **)&buf, &len) != DDI_PROP_SUCCESS)
9950 		return (NULL);
9951 
9952 	for (i = 0; i < len; i++)
9953 		buf[i] = (uchar_t)(((int *)buf)[i]);
9954 
9955 	if (len < sizeof (dof_hdr_t)) {
9956 		ddi_prop_free(buf);
9957 		dtrace_dof_error(NULL, "truncated header");
9958 		return (NULL);
9959 	}
9960 
9961 	if (len < (loadsz = ((dof_hdr_t *)buf)->dofh_loadsz)) {
9962 		ddi_prop_free(buf);
9963 		dtrace_dof_error(NULL, "truncated DOF");
9964 		return (NULL);
9965 	}
9966 
9967 	if (loadsz >= dtrace_dof_maxsize) {
9968 		ddi_prop_free(buf);
9969 		dtrace_dof_error(NULL, "oversized DOF");
9970 		return (NULL);
9971 	}
9972 
9973 	dof = kmem_alloc(loadsz, KM_SLEEP);
9974 	bcopy(buf, dof, loadsz);
9975 	ddi_prop_free(buf);
9976 
9977 	return (dof);
9978 }
9979 
9980 static void
9981 dtrace_dof_destroy(dof_hdr_t *dof)
9982 {
9983 	kmem_free(dof, dof->dofh_loadsz);
9984 }
9985 
9986 /*
9987  * Return the dof_sec_t pointer corresponding to a given section index.  If the
9988  * index is not valid, dtrace_dof_error() is called and NULL is returned.  If
9989  * a type other than DOF_SECT_NONE is specified, the header is checked against
9990  * this type and NULL is returned if the types do not match.
9991  */
9992 static dof_sec_t *
9993 dtrace_dof_sect(dof_hdr_t *dof, uint32_t type, dof_secidx_t i)
9994 {
9995 	dof_sec_t *sec = (dof_sec_t *)(uintptr_t)
9996 	    ((uintptr_t)dof + dof->dofh_secoff + i * dof->dofh_secsize);
9997 
9998 	if (i >= dof->dofh_secnum) {
9999 		dtrace_dof_error(dof, "referenced section index is invalid");
10000 		return (NULL);
10001 	}
10002 
10003 	if (!(sec->dofs_flags & DOF_SECF_LOAD)) {
10004 		dtrace_dof_error(dof, "referenced section is not loadable");
10005 		return (NULL);
10006 	}
10007 
10008 	if (type != DOF_SECT_NONE && type != sec->dofs_type) {
10009 		dtrace_dof_error(dof, "referenced section is the wrong type");
10010 		return (NULL);
10011 	}
10012 
10013 	return (sec);
10014 }
10015 
10016 static dtrace_probedesc_t *
10017 dtrace_dof_probedesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_probedesc_t *desc)
10018 {
10019 	dof_probedesc_t *probe;
10020 	dof_sec_t *strtab;
10021 	uintptr_t daddr = (uintptr_t)dof;
10022 	uintptr_t str;
10023 	size_t size;
10024 
10025 	if (sec->dofs_type != DOF_SECT_PROBEDESC) {
10026 		dtrace_dof_error(dof, "invalid probe section");
10027 		return (NULL);
10028 	}
10029 
10030 	if (sec->dofs_align != sizeof (dof_secidx_t)) {
10031 		dtrace_dof_error(dof, "bad alignment in probe description");
10032 		return (NULL);
10033 	}
10034 
10035 	if (sec->dofs_offset + sizeof (dof_probedesc_t) > dof->dofh_loadsz) {
10036 		dtrace_dof_error(dof, "truncated probe description");
10037 		return (NULL);
10038 	}
10039 
10040 	probe = (dof_probedesc_t *)(uintptr_t)(daddr + sec->dofs_offset);
10041 	strtab = dtrace_dof_sect(dof, DOF_SECT_STRTAB, probe->dofp_strtab);
10042 
10043 	if (strtab == NULL)
10044 		return (NULL);
10045 
10046 	str = daddr + strtab->dofs_offset;
10047 	size = strtab->dofs_size;
10048 
10049 	if (probe->dofp_provider >= strtab->dofs_size) {
10050 		dtrace_dof_error(dof, "corrupt probe provider");
10051 		return (NULL);
10052 	}
10053 
10054 	(void) strncpy(desc->dtpd_provider,
10055 	    (char *)(str + probe->dofp_provider),
10056 	    MIN(DTRACE_PROVNAMELEN - 1, size - probe->dofp_provider));
10057 
10058 	if (probe->dofp_mod >= strtab->dofs_size) {
10059 		dtrace_dof_error(dof, "corrupt probe module");
10060 		return (NULL);
10061 	}
10062 
10063 	(void) strncpy(desc->dtpd_mod, (char *)(str + probe->dofp_mod),
10064 	    MIN(DTRACE_MODNAMELEN - 1, size - probe->dofp_mod));
10065 
10066 	if (probe->dofp_func >= strtab->dofs_size) {
10067 		dtrace_dof_error(dof, "corrupt probe function");
10068 		return (NULL);
10069 	}
10070 
10071 	(void) strncpy(desc->dtpd_func, (char *)(str + probe->dofp_func),
10072 	    MIN(DTRACE_FUNCNAMELEN - 1, size - probe->dofp_func));
10073 
10074 	if (probe->dofp_name >= strtab->dofs_size) {
10075 		dtrace_dof_error(dof, "corrupt probe name");
10076 		return (NULL);
10077 	}
10078 
10079 	(void) strncpy(desc->dtpd_name, (char *)(str + probe->dofp_name),
10080 	    MIN(DTRACE_NAMELEN - 1, size - probe->dofp_name));
10081 
10082 	return (desc);
10083 }
10084 
10085 static dtrace_difo_t *
10086 dtrace_dof_difo(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
10087     cred_t *cr)
10088 {
10089 	dtrace_difo_t *dp;
10090 	size_t ttl = 0;
10091 	dof_difohdr_t *dofd;
10092 	uintptr_t daddr = (uintptr_t)dof;
10093 	size_t max = dtrace_difo_maxsize;
10094 	int i, l, n;
10095 
10096 	static const struct {
10097 		int section;
10098 		int bufoffs;
10099 		int lenoffs;
10100 		int entsize;
10101 		int align;
10102 		const char *msg;
10103 	} difo[] = {
10104 		{ DOF_SECT_DIF, offsetof(dtrace_difo_t, dtdo_buf),
10105 		offsetof(dtrace_difo_t, dtdo_len), sizeof (dif_instr_t),
10106 		sizeof (dif_instr_t), "multiple DIF sections" },
10107 
10108 		{ DOF_SECT_INTTAB, offsetof(dtrace_difo_t, dtdo_inttab),
10109 		offsetof(dtrace_difo_t, dtdo_intlen), sizeof (uint64_t),
10110 		sizeof (uint64_t), "multiple integer tables" },
10111 
10112 		{ DOF_SECT_STRTAB, offsetof(dtrace_difo_t, dtdo_strtab),
10113 		offsetof(dtrace_difo_t, dtdo_strlen), 0,
10114 		sizeof (char), "multiple string tables" },
10115 
10116 		{ DOF_SECT_VARTAB, offsetof(dtrace_difo_t, dtdo_vartab),
10117 		offsetof(dtrace_difo_t, dtdo_varlen), sizeof (dtrace_difv_t),
10118 		sizeof (uint_t), "multiple variable tables" },
10119 
10120 		{ DOF_SECT_NONE, 0, 0, 0, NULL }
10121 	};
10122 
10123 	if (sec->dofs_type != DOF_SECT_DIFOHDR) {
10124 		dtrace_dof_error(dof, "invalid DIFO header section");
10125 		return (NULL);
10126 	}
10127 
10128 	if (sec->dofs_align != sizeof (dof_secidx_t)) {
10129 		dtrace_dof_error(dof, "bad alignment in DIFO header");
10130 		return (NULL);
10131 	}
10132 
10133 	if (sec->dofs_size < sizeof (dof_difohdr_t) ||
10134 	    sec->dofs_size % sizeof (dof_secidx_t)) {
10135 		dtrace_dof_error(dof, "bad size in DIFO header");
10136 		return (NULL);
10137 	}
10138 
10139 	dofd = (dof_difohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
10140 	n = (sec->dofs_size - sizeof (*dofd)) / sizeof (dof_secidx_t) + 1;
10141 
10142 	dp = kmem_zalloc(sizeof (dtrace_difo_t), KM_SLEEP);
10143 	dp->dtdo_rtype = dofd->dofd_rtype;
10144 
10145 	for (l = 0; l < n; l++) {
10146 		dof_sec_t *subsec;
10147 		void **bufp;
10148 		uint32_t *lenp;
10149 
10150 		if ((subsec = dtrace_dof_sect(dof, DOF_SECT_NONE,
10151 		    dofd->dofd_links[l])) == NULL)
10152 			goto err; /* invalid section link */
10153 
10154 		if (ttl + subsec->dofs_size > max) {
10155 			dtrace_dof_error(dof, "exceeds maximum size");
10156 			goto err;
10157 		}
10158 
10159 		ttl += subsec->dofs_size;
10160 
10161 		for (i = 0; difo[i].section != DOF_SECT_NONE; i++) {
10162 			if (subsec->dofs_type != difo[i].section)
10163 				continue;
10164 
10165 			if (!(subsec->dofs_flags & DOF_SECF_LOAD)) {
10166 				dtrace_dof_error(dof, "section not loaded");
10167 				goto err;
10168 			}
10169 
10170 			if (subsec->dofs_align != difo[i].align) {
10171 				dtrace_dof_error(dof, "bad alignment");
10172 				goto err;
10173 			}
10174 
10175 			bufp = (void **)((uintptr_t)dp + difo[i].bufoffs);
10176 			lenp = (uint32_t *)((uintptr_t)dp + difo[i].lenoffs);
10177 
10178 			if (*bufp != NULL) {
10179 				dtrace_dof_error(dof, difo[i].msg);
10180 				goto err;
10181 			}
10182 
10183 			if (difo[i].entsize != subsec->dofs_entsize) {
10184 				dtrace_dof_error(dof, "entry size mismatch");
10185 				goto err;
10186 			}
10187 
10188 			if (subsec->dofs_entsize != 0 &&
10189 			    (subsec->dofs_size % subsec->dofs_entsize) != 0) {
10190 				dtrace_dof_error(dof, "corrupt entry size");
10191 				goto err;
10192 			}
10193 
10194 			*lenp = subsec->dofs_size;
10195 			*bufp = kmem_alloc(subsec->dofs_size, KM_SLEEP);
10196 			bcopy((char *)(uintptr_t)(daddr + subsec->dofs_offset),
10197 			    *bufp, subsec->dofs_size);
10198 
10199 			if (subsec->dofs_entsize != 0)
10200 				*lenp /= subsec->dofs_entsize;
10201 
10202 			break;
10203 		}
10204 
10205 		/*
10206 		 * If we encounter a loadable DIFO sub-section that is not
10207 		 * known to us, assume this is a broken program and fail.
10208 		 */
10209 		if (difo[i].section == DOF_SECT_NONE &&
10210 		    (subsec->dofs_flags & DOF_SECF_LOAD)) {
10211 			dtrace_dof_error(dof, "unrecognized DIFO subsection");
10212 			goto err;
10213 		}
10214 	}
10215 
10216 	if (dp->dtdo_buf == NULL) {
10217 		/*
10218 		 * We can't have a DIF object without DIF text.
10219 		 */
10220 		dtrace_dof_error(dof, "missing DIF text");
10221 		goto err;
10222 	}
10223 
10224 	/*
10225 	 * Before we validate the DIF object, run through the variable table
10226 	 * looking for the strings -- if any of their size are under, we'll set
10227 	 * their size to be the system-wide default string size.  Note that
10228 	 * this should _not_ happen if the "strsize" option has been set --
10229 	 * in this case, the compiler should have set the size to reflect the
10230 	 * setting of the option.
10231 	 */
10232 	for (i = 0; i < dp->dtdo_varlen; i++) {
10233 		dtrace_difv_t *v = &dp->dtdo_vartab[i];
10234 		dtrace_diftype_t *t = &v->dtdv_type;
10235 
10236 		if (v->dtdv_id < DIF_VAR_OTHER_UBASE)
10237 			continue;
10238 
10239 		if (t->dtdt_kind == DIF_TYPE_STRING && t->dtdt_size == 0)
10240 			t->dtdt_size = dtrace_strsize_default;
10241 	}
10242 
10243 	if (dtrace_difo_validate(dp, vstate, DIF_DIR_NREGS, cr) != 0)
10244 		goto err;
10245 
10246 	dtrace_difo_init(dp, vstate);
10247 	return (dp);
10248 
10249 err:
10250 	kmem_free(dp->dtdo_buf, dp->dtdo_len * sizeof (dif_instr_t));
10251 	kmem_free(dp->dtdo_inttab, dp->dtdo_intlen * sizeof (uint64_t));
10252 	kmem_free(dp->dtdo_strtab, dp->dtdo_strlen);
10253 	kmem_free(dp->dtdo_vartab, dp->dtdo_varlen * sizeof (dtrace_difv_t));
10254 
10255 	kmem_free(dp, sizeof (dtrace_difo_t));
10256 	return (NULL);
10257 }
10258 
10259 static dtrace_predicate_t *
10260 dtrace_dof_predicate(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
10261     cred_t *cr)
10262 {
10263 	dtrace_difo_t *dp;
10264 
10265 	if ((dp = dtrace_dof_difo(dof, sec, vstate, cr)) == NULL)
10266 		return (NULL);
10267 
10268 	return (dtrace_predicate_create(dp));
10269 }
10270 
10271 static dtrace_actdesc_t *
10272 dtrace_dof_actdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
10273     cred_t *cr)
10274 {
10275 	dtrace_actdesc_t *act, *first = NULL, *last = NULL, *next;
10276 	dof_actdesc_t *desc;
10277 	dof_sec_t *difosec;
10278 	size_t offs;
10279 	uintptr_t daddr = (uintptr_t)dof;
10280 	uint64_t arg;
10281 	dtrace_actkind_t kind;
10282 
10283 	if (sec->dofs_type != DOF_SECT_ACTDESC) {
10284 		dtrace_dof_error(dof, "invalid action section");
10285 		return (NULL);
10286 	}
10287 
10288 	if (sec->dofs_offset + sizeof (dof_actdesc_t) > dof->dofh_loadsz) {
10289 		dtrace_dof_error(dof, "truncated action description");
10290 		return (NULL);
10291 	}
10292 
10293 	if (sec->dofs_align != sizeof (uint64_t)) {
10294 		dtrace_dof_error(dof, "bad alignment in action description");
10295 		return (NULL);
10296 	}
10297 
10298 	if (sec->dofs_size < sec->dofs_entsize) {
10299 		dtrace_dof_error(dof, "section entry size exceeds total size");
10300 		return (NULL);
10301 	}
10302 
10303 	if (sec->dofs_entsize != sizeof (dof_actdesc_t)) {
10304 		dtrace_dof_error(dof, "bad entry size in action description");
10305 		return (NULL);
10306 	}
10307 
10308 	if (sec->dofs_size / sec->dofs_entsize > dtrace_actions_max) {
10309 		dtrace_dof_error(dof, "actions exceed dtrace_actions_max");
10310 		return (NULL);
10311 	}
10312 
10313 	for (offs = 0; offs < sec->dofs_size; offs += sec->dofs_entsize) {
10314 		desc = (dof_actdesc_t *)(daddr +
10315 		    (uintptr_t)sec->dofs_offset + offs);
10316 		kind = (dtrace_actkind_t)desc->dofa_kind;
10317 
10318 		if (DTRACEACT_ISPRINTFLIKE(kind) &&
10319 		    (kind != DTRACEACT_PRINTA ||
10320 		    desc->dofa_strtab != DOF_SECIDX_NONE)) {
10321 			dof_sec_t *strtab;
10322 			char *str, *fmt;
10323 			uint64_t i;
10324 
10325 			/*
10326 			 * printf()-like actions must have a format string.
10327 			 */
10328 			if ((strtab = dtrace_dof_sect(dof,
10329 			    DOF_SECT_STRTAB, desc->dofa_strtab)) == NULL)
10330 				goto err;
10331 
10332 			str = (char *)((uintptr_t)dof +
10333 			    (uintptr_t)strtab->dofs_offset);
10334 
10335 			for (i = desc->dofa_arg; i < strtab->dofs_size; i++) {
10336 				if (str[i] == '\0')
10337 					break;
10338 			}
10339 
10340 			if (i >= strtab->dofs_size) {
10341 				dtrace_dof_error(dof, "bogus format string");
10342 				goto err;
10343 			}
10344 
10345 			if (i == desc->dofa_arg) {
10346 				dtrace_dof_error(dof, "empty format string");
10347 				goto err;
10348 			}
10349 
10350 			i -= desc->dofa_arg;
10351 			fmt = kmem_alloc(i + 1, KM_SLEEP);
10352 			bcopy(&str[desc->dofa_arg], fmt, i + 1);
10353 			arg = (uint64_t)(uintptr_t)fmt;
10354 		} else {
10355 			if (kind == DTRACEACT_PRINTA) {
10356 				ASSERT(desc->dofa_strtab == DOF_SECIDX_NONE);
10357 				arg = 0;
10358 			} else {
10359 				arg = desc->dofa_arg;
10360 			}
10361 		}
10362 
10363 		act = dtrace_actdesc_create(kind, desc->dofa_ntuple,
10364 		    desc->dofa_uarg, arg);
10365 
10366 		if (last != NULL) {
10367 			last->dtad_next = act;
10368 		} else {
10369 			first = act;
10370 		}
10371 
10372 		last = act;
10373 
10374 		if (desc->dofa_difo == DOF_SECIDX_NONE)
10375 			continue;
10376 
10377 		if ((difosec = dtrace_dof_sect(dof,
10378 		    DOF_SECT_DIFOHDR, desc->dofa_difo)) == NULL)
10379 			goto err;
10380 
10381 		act->dtad_difo = dtrace_dof_difo(dof, difosec, vstate, cr);
10382 
10383 		if (act->dtad_difo == NULL)
10384 			goto err;
10385 	}
10386 
10387 	ASSERT(first != NULL);
10388 	return (first);
10389 
10390 err:
10391 	for (act = first; act != NULL; act = next) {
10392 		next = act->dtad_next;
10393 		dtrace_actdesc_release(act, vstate);
10394 	}
10395 
10396 	return (NULL);
10397 }
10398 
10399 static dtrace_ecbdesc_t *
10400 dtrace_dof_ecbdesc(dof_hdr_t *dof, dof_sec_t *sec, dtrace_vstate_t *vstate,
10401     cred_t *cr)
10402 {
10403 	dtrace_ecbdesc_t *ep;
10404 	dof_ecbdesc_t *ecb;
10405 	dtrace_probedesc_t *desc;
10406 	dtrace_predicate_t *pred = NULL;
10407 
10408 	if (sec->dofs_size < sizeof (dof_ecbdesc_t)) {
10409 		dtrace_dof_error(dof, "truncated ECB description");
10410 		return (NULL);
10411 	}
10412 
10413 	if (sec->dofs_align != sizeof (uint64_t)) {
10414 		dtrace_dof_error(dof, "bad alignment in ECB description");
10415 		return (NULL);
10416 	}
10417 
10418 	ecb = (dof_ecbdesc_t *)((uintptr_t)dof + (uintptr_t)sec->dofs_offset);
10419 	sec = dtrace_dof_sect(dof, DOF_SECT_PROBEDESC, ecb->dofe_probes);
10420 
10421 	if (sec == NULL)
10422 		return (NULL);
10423 
10424 	ep = kmem_zalloc(sizeof (dtrace_ecbdesc_t), KM_SLEEP);
10425 	ep->dted_uarg = ecb->dofe_uarg;
10426 	desc = &ep->dted_probe;
10427 
10428 	if (dtrace_dof_probedesc(dof, sec, desc) == NULL)
10429 		goto err;
10430 
10431 	if (ecb->dofe_pred != DOF_SECIDX_NONE) {
10432 		if ((sec = dtrace_dof_sect(dof,
10433 		    DOF_SECT_DIFOHDR, ecb->dofe_pred)) == NULL)
10434 			goto err;
10435 
10436 		if ((pred = dtrace_dof_predicate(dof, sec, vstate, cr)) == NULL)
10437 			goto err;
10438 
10439 		ep->dted_pred.dtpdd_predicate = pred;
10440 	}
10441 
10442 	if (ecb->dofe_actions != DOF_SECIDX_NONE) {
10443 		if ((sec = dtrace_dof_sect(dof,
10444 		    DOF_SECT_ACTDESC, ecb->dofe_actions)) == NULL)
10445 			goto err;
10446 
10447 		ep->dted_action = dtrace_dof_actdesc(dof, sec, vstate, cr);
10448 
10449 		if (ep->dted_action == NULL)
10450 			goto err;
10451 	}
10452 
10453 	return (ep);
10454 
10455 err:
10456 	if (pred != NULL)
10457 		dtrace_predicate_release(pred, vstate);
10458 	kmem_free(ep, sizeof (dtrace_ecbdesc_t));
10459 	return (NULL);
10460 }
10461 
10462 /*
10463  * Apply the relocations from the specified 'sec' (a DOF_SECT_URELHDR) to the
10464  * specified DOF.  At present, this amounts to simply adding 'ubase' to the
10465  * site of any user SETX relocations to account for load object base address.
10466  * In the future, if we need other relocations, this function can be extended.
10467  */
10468 static int
10469 dtrace_dof_relocate(dof_hdr_t *dof, dof_sec_t *sec, uint64_t ubase)
10470 {
10471 	uintptr_t daddr = (uintptr_t)dof;
10472 	dof_relohdr_t *dofr =
10473 	    (dof_relohdr_t *)(uintptr_t)(daddr + sec->dofs_offset);
10474 	dof_sec_t *ss, *rs, *ts;
10475 	dof_relodesc_t *r;
10476 	uint_t i, n;
10477 
10478 	if (sec->dofs_size < sizeof (dof_relohdr_t) ||
10479 	    sec->dofs_align != sizeof (dof_secidx_t)) {
10480 		dtrace_dof_error(dof, "invalid relocation header");
10481 		return (-1);
10482 	}
10483 
10484 	ss = dtrace_dof_sect(dof, DOF_SECT_STRTAB, dofr->dofr_strtab);
10485 	rs = dtrace_dof_sect(dof, DOF_SECT_RELTAB, dofr->dofr_relsec);
10486 	ts = dtrace_dof_sect(dof, DOF_SECT_NONE, dofr->dofr_tgtsec);
10487 
10488 	if (ss == NULL || rs == NULL || ts == NULL)
10489 		return (-1); /* dtrace_dof_error() has been called already */
10490 
10491 	if (rs->dofs_entsize < sizeof (dof_relodesc_t) ||
10492 	    rs->dofs_align != sizeof (uint64_t)) {
10493 		dtrace_dof_error(dof, "invalid relocation section");
10494 		return (-1);
10495 	}
10496 
10497 	r = (dof_relodesc_t *)(uintptr_t)(daddr + rs->dofs_offset);
10498 	n = rs->dofs_size / rs->dofs_entsize;
10499 
10500 	for (i = 0; i < n; i++) {
10501 		uintptr_t taddr = daddr + ts->dofs_offset + r->dofr_offset;
10502 
10503 		switch (r->dofr_type) {
10504 		case DOF_RELO_NONE:
10505 			break;
10506 		case DOF_RELO_SETX:
10507 			if (r->dofr_offset >= ts->dofs_size || r->dofr_offset +
10508 			    sizeof (uint64_t) > ts->dofs_size) {
10509 				dtrace_dof_error(dof, "bad relocation offset");
10510 				return (-1);
10511 			}
10512 
10513 			if (!IS_P2ALIGNED(taddr, sizeof (uint64_t))) {
10514 				dtrace_dof_error(dof, "misaligned setx relo");
10515 				return (-1);
10516 			}
10517 
10518 			*(uint64_t *)taddr += ubase;
10519 			break;
10520 		default:
10521 			dtrace_dof_error(dof, "invalid relocation type");
10522 			return (-1);
10523 		}
10524 
10525 		r = (dof_relodesc_t *)((uintptr_t)r + rs->dofs_entsize);
10526 	}
10527 
10528 	return (0);
10529 }
10530 
10531 /*
10532  * The dof_hdr_t passed to dtrace_dof_slurp() should be a partially validated
10533  * header:  it should be at the front of a memory region that is at least
10534  * sizeof (dof_hdr_t) in size -- and then at least dof_hdr.dofh_loadsz in
10535  * size.  It need not be validated in any other way.
10536  */
10537 static int
10538 dtrace_dof_slurp(dof_hdr_t *dof, dtrace_vstate_t *vstate, cred_t *cr,
10539     dtrace_enabling_t **enabp, uint64_t ubase, int noprobes)
10540 {
10541 	uint64_t len = dof->dofh_loadsz, seclen;
10542 	uintptr_t daddr = (uintptr_t)dof;
10543 	dtrace_ecbdesc_t *ep;
10544 	dtrace_enabling_t *enab;
10545 	uint_t i;
10546 
10547 	ASSERT(MUTEX_HELD(&dtrace_lock));
10548 	ASSERT(dof->dofh_loadsz >= sizeof (dof_hdr_t));
10549 
10550 	/*
10551 	 * Check the DOF header identification bytes.  In addition to checking
10552 	 * valid settings, we also verify that unused bits/bytes are zeroed so
10553 	 * we can use them later without fear of regressing existing binaries.
10554 	 */
10555 	if (bcmp(&dof->dofh_ident[DOF_ID_MAG0],
10556 	    DOF_MAG_STRING, DOF_MAG_STRLEN) != 0) {
10557 		dtrace_dof_error(dof, "DOF magic string mismatch");
10558 		return (-1);
10559 	}
10560 
10561 	if (dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_ILP32 &&
10562 	    dof->dofh_ident[DOF_ID_MODEL] != DOF_MODEL_LP64) {
10563 		dtrace_dof_error(dof, "DOF has invalid data model");
10564 		return (-1);
10565 	}
10566 
10567 	if (dof->dofh_ident[DOF_ID_ENCODING] != DOF_ENCODE_NATIVE) {
10568 		dtrace_dof_error(dof, "DOF encoding mismatch");
10569 		return (-1);
10570 	}
10571 
10572 	if (dof->dofh_ident[DOF_ID_VERSION] != DOF_VERSION_1) {
10573 		dtrace_dof_error(dof, "DOF version mismatch");
10574 		return (-1);
10575 	}
10576 
10577 	if (dof->dofh_ident[DOF_ID_DIFVERS] != DIF_VERSION_2) {
10578 		dtrace_dof_error(dof, "DOF uses unsupported instruction set");
10579 		return (-1);
10580 	}
10581 
10582 	if (dof->dofh_ident[DOF_ID_DIFIREG] > DIF_DIR_NREGS) {
10583 		dtrace_dof_error(dof, "DOF uses too many integer registers");
10584 		return (-1);
10585 	}
10586 
10587 	if (dof->dofh_ident[DOF_ID_DIFTREG] > DIF_DTR_NREGS) {
10588 		dtrace_dof_error(dof, "DOF uses too many tuple registers");
10589 		return (-1);
10590 	}
10591 
10592 	for (i = DOF_ID_PAD; i < DOF_ID_SIZE; i++) {
10593 		if (dof->dofh_ident[i] != 0) {
10594 			dtrace_dof_error(dof, "DOF has invalid ident byte set");
10595 			return (-1);
10596 		}
10597 	}
10598 
10599 	if (dof->dofh_flags & ~DOF_FL_VALID) {
10600 		dtrace_dof_error(dof, "DOF has invalid flag bits set");
10601 		return (-1);
10602 	}
10603 
10604 	if (dof->dofh_secsize == 0) {
10605 		dtrace_dof_error(dof, "zero section header size");
10606 		return (-1);
10607 	}
10608 
10609 	/*
10610 	 * Check that the section headers don't exceed the amount of DOF
10611 	 * data.  Note that we cast the section size and number of sections
10612 	 * to uint64_t's to prevent possible overflow in the multiplication.
10613 	 */
10614 	seclen = (uint64_t)dof->dofh_secnum * (uint64_t)dof->dofh_secsize;
10615 
10616 	if (dof->dofh_secoff > len || seclen > len ||
10617 	    dof->dofh_secoff + seclen > len) {
10618 		dtrace_dof_error(dof, "truncated section headers");
10619 		return (-1);
10620 	}
10621 
10622 	if (!IS_P2ALIGNED(dof->dofh_secoff, sizeof (uint64_t))) {
10623 		dtrace_dof_error(dof, "misaligned section headers");
10624 		return (-1);
10625 	}
10626 
10627 	if (!IS_P2ALIGNED(dof->dofh_secsize, sizeof (uint64_t))) {
10628 		dtrace_dof_error(dof, "misaligned section size");
10629 		return (-1);
10630 	}
10631 
10632 	/*
10633 	 * Take an initial pass through the section headers to be sure that
10634 	 * the headers don't have stray offsets.  If the 'noprobes' flag is
10635 	 * set, do not permit sections relating to providers, probes, or args.
10636 	 */
10637 	for (i = 0; i < dof->dofh_secnum; i++) {
10638 		dof_sec_t *sec = (dof_sec_t *)(daddr +
10639 		    (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
10640 
10641 		if (noprobes) {
10642 			switch (sec->dofs_type) {
10643 			case DOF_SECT_PROVIDER:
10644 			case DOF_SECT_PROBES:
10645 			case DOF_SECT_PRARGS:
10646 			case DOF_SECT_PROFFS:
10647 				dtrace_dof_error(dof, "illegal sections "
10648 				    "for enabling");
10649 				return (-1);
10650 			}
10651 		}
10652 
10653 		if (!(sec->dofs_flags & DOF_SECF_LOAD))
10654 			continue; /* just ignore non-loadable sections */
10655 
10656 		if (sec->dofs_align & (sec->dofs_align - 1)) {
10657 			dtrace_dof_error(dof, "bad section alignment");
10658 			return (-1);
10659 		}
10660 
10661 		if (sec->dofs_offset & (sec->dofs_align - 1)) {
10662 			dtrace_dof_error(dof, "misaligned section");
10663 			return (-1);
10664 		}
10665 
10666 		if (sec->dofs_offset > len || sec->dofs_size > len ||
10667 		    sec->dofs_offset + sec->dofs_size > len) {
10668 			dtrace_dof_error(dof, "corrupt section header");
10669 			return (-1);
10670 		}
10671 
10672 		if (sec->dofs_type == DOF_SECT_STRTAB && *((char *)daddr +
10673 		    sec->dofs_offset + sec->dofs_size - 1) != '\0') {
10674 			dtrace_dof_error(dof, "non-terminating string table");
10675 			return (-1);
10676 		}
10677 	}
10678 
10679 	/*
10680 	 * Take a second pass through the sections and locate and perform any
10681 	 * relocations that are present.  We do this after the first pass to
10682 	 * be sure that all sections have had their headers validated.
10683 	 */
10684 	for (i = 0; i < dof->dofh_secnum; i++) {
10685 		dof_sec_t *sec = (dof_sec_t *)(daddr +
10686 		    (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
10687 
10688 		if (!(sec->dofs_flags & DOF_SECF_LOAD))
10689 			continue; /* skip sections that are not loadable */
10690 
10691 		switch (sec->dofs_type) {
10692 		case DOF_SECT_URELHDR:
10693 			if (dtrace_dof_relocate(dof, sec, ubase) != 0)
10694 				return (-1);
10695 			break;
10696 		}
10697 	}
10698 
10699 	if ((enab = *enabp) == NULL)
10700 		enab = *enabp = dtrace_enabling_create(vstate);
10701 
10702 	for (i = 0; i < dof->dofh_secnum; i++) {
10703 		dof_sec_t *sec = (dof_sec_t *)(daddr +
10704 		    (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
10705 
10706 		if (sec->dofs_type != DOF_SECT_ECBDESC)
10707 			continue;
10708 
10709 		if ((ep = dtrace_dof_ecbdesc(dof, sec, vstate, cr)) == NULL) {
10710 			dtrace_enabling_destroy(enab);
10711 			*enabp = NULL;
10712 			return (-1);
10713 		}
10714 
10715 		dtrace_enabling_add(enab, ep);
10716 	}
10717 
10718 	return (0);
10719 }
10720 
10721 /*
10722  * Process DOF for any options.  This routine assumes that the DOF has been
10723  * at least processed by dtrace_dof_slurp().
10724  */
10725 static int
10726 dtrace_dof_options(dof_hdr_t *dof, dtrace_state_t *state)
10727 {
10728 	int i, rval;
10729 	uint32_t entsize;
10730 	size_t offs;
10731 	dof_optdesc_t *desc;
10732 
10733 	for (i = 0; i < dof->dofh_secnum; i++) {
10734 		dof_sec_t *sec = (dof_sec_t *)((uintptr_t)dof +
10735 		    (uintptr_t)dof->dofh_secoff + i * dof->dofh_secsize);
10736 
10737 		if (sec->dofs_type != DOF_SECT_OPTDESC)
10738 			continue;
10739 
10740 		if (sec->dofs_align != sizeof (uint64_t)) {
10741 			dtrace_dof_error(dof, "bad alignment in "
10742 			    "option description");
10743 			return (EINVAL);
10744 		}
10745 
10746 		if ((entsize = sec->dofs_entsize) == 0) {
10747 			dtrace_dof_error(dof, "zeroed option entry size");
10748 			return (EINVAL);
10749 		}
10750 
10751 		if (entsize < sizeof (dof_optdesc_t)) {
10752 			dtrace_dof_error(dof, "bad option entry size");
10753 			return (EINVAL);
10754 		}
10755 
10756 		for (offs = 0; offs < sec->dofs_size; offs += entsize) {
10757 			desc = (dof_optdesc_t *)((uintptr_t)dof +
10758 			    (uintptr_t)sec->dofs_offset + offs);
10759 
10760 			if (desc->dofo_strtab != DOF_SECIDX_NONE) {
10761 				dtrace_dof_error(dof, "non-zero option string");
10762 				return (EINVAL);
10763 			}
10764 
10765 			if (desc->dofo_value == DTRACEOPT_UNSET) {
10766 				dtrace_dof_error(dof, "unset option");
10767 				return (EINVAL);
10768 			}
10769 
10770 			if ((rval = dtrace_state_option(state,
10771 			    desc->dofo_option, desc->dofo_value)) != 0) {
10772 				dtrace_dof_error(dof, "rejected option");
10773 				return (rval);
10774 			}
10775 		}
10776 	}
10777 
10778 	return (0);
10779 }
10780 
10781 /*
10782  * DTrace Consumer State Functions
10783  */
10784 int
10785 dtrace_dstate_init(dtrace_dstate_t *dstate, size_t size)
10786 {
10787 	size_t hashsize, maxper, min, chunksize = dstate->dtds_chunksize;
10788 	void *base;
10789 	uintptr_t limit;
10790 	dtrace_dynvar_t *dvar, *next, *start;
10791 	int i;
10792 
10793 	ASSERT(MUTEX_HELD(&dtrace_lock));
10794 	ASSERT(dstate->dtds_base == NULL && dstate->dtds_percpu == NULL);
10795 
10796 	bzero(dstate, sizeof (dtrace_dstate_t));
10797 
10798 	if ((dstate->dtds_chunksize = chunksize) == 0)
10799 		dstate->dtds_chunksize = DTRACE_DYNVAR_CHUNKSIZE;
10800 
10801 	if (size < (min = dstate->dtds_chunksize + sizeof (dtrace_dynhash_t)))
10802 		size = min;
10803 
10804 	if ((base = kmem_zalloc(size, KM_NOSLEEP)) == NULL)
10805 		return (ENOMEM);
10806 
10807 	dstate->dtds_size = size;
10808 	dstate->dtds_base = base;
10809 	dstate->dtds_percpu = kmem_cache_alloc(dtrace_state_cache, KM_SLEEP);
10810 	bzero(dstate->dtds_percpu, NCPU * sizeof (dtrace_dstate_percpu_t));
10811 
10812 	hashsize = size / (dstate->dtds_chunksize + sizeof (dtrace_dynhash_t));
10813 
10814 	if (hashsize != 1 && (hashsize & 1))
10815 		hashsize--;
10816 
10817 	dstate->dtds_hashsize = hashsize;
10818 	dstate->dtds_hash = dstate->dtds_base;
10819 
10820 	/*
10821 	 * Determine number of active CPUs.  Divide free list evenly among
10822 	 * active CPUs.
10823 	 */
10824 	start = (dtrace_dynvar_t *)
10825 	    ((uintptr_t)base + hashsize * sizeof (dtrace_dynhash_t));
10826 	limit = (uintptr_t)base + size;
10827 
10828 	maxper = (limit - (uintptr_t)start) / NCPU;
10829 	maxper = (maxper / dstate->dtds_chunksize) * dstate->dtds_chunksize;
10830 
10831 	for (i = 0; i < NCPU; i++) {
10832 		dstate->dtds_percpu[i].dtdsc_free = dvar = start;
10833 
10834 		/*
10835 		 * If we don't even have enough chunks to make it once through
10836 		 * NCPUs, we're just going to allocate everything to the first
10837 		 * CPU.  And if we're on the last CPU, we're going to allocate
10838 		 * whatever is left over.  In either case, we set the limit to
10839 		 * be the limit of the dynamic variable space.
10840 		 */
10841 		if (maxper == 0 || i == NCPU - 1) {
10842 			limit = (uintptr_t)base + size;
10843 			start = NULL;
10844 		} else {
10845 			limit = (uintptr_t)start + maxper;
10846 			start = (dtrace_dynvar_t *)limit;
10847 		}
10848 
10849 		ASSERT(limit <= (uintptr_t)base + size);
10850 
10851 		for (;;) {
10852 			next = (dtrace_dynvar_t *)((uintptr_t)dvar +
10853 			    dstate->dtds_chunksize);
10854 
10855 			if ((uintptr_t)next + dstate->dtds_chunksize >= limit)
10856 				break;
10857 
10858 			dvar->dtdv_next = next;
10859 			dvar = next;
10860 		}
10861 
10862 		if (maxper == 0)
10863 			break;
10864 	}
10865 
10866 	return (0);
10867 }
10868 
10869 void
10870 dtrace_dstate_fini(dtrace_dstate_t *dstate)
10871 {
10872 	ASSERT(MUTEX_HELD(&cpu_lock));
10873 
10874 	if (dstate->dtds_base == NULL)
10875 		return;
10876 
10877 	kmem_free(dstate->dtds_base, dstate->dtds_size);
10878 	kmem_cache_free(dtrace_state_cache, dstate->dtds_percpu);
10879 }
10880 
10881 static void
10882 dtrace_vstate_fini(dtrace_vstate_t *vstate)
10883 {
10884 	/*
10885 	 * Logical XOR, where are you?
10886 	 */
10887 	ASSERT((vstate->dtvs_nglobals == 0) ^ (vstate->dtvs_globals != NULL));
10888 
10889 	if (vstate->dtvs_nglobals > 0) {
10890 		kmem_free(vstate->dtvs_globals, vstate->dtvs_nglobals *
10891 		    sizeof (dtrace_statvar_t *));
10892 	}
10893 
10894 	if (vstate->dtvs_ntlocals > 0) {
10895 		kmem_free(vstate->dtvs_tlocals, vstate->dtvs_ntlocals *
10896 		    sizeof (dtrace_difv_t));
10897 	}
10898 
10899 	ASSERT((vstate->dtvs_nlocals == 0) ^ (vstate->dtvs_locals != NULL));
10900 
10901 	if (vstate->dtvs_nlocals > 0) {
10902 		kmem_free(vstate->dtvs_locals, vstate->dtvs_nlocals *
10903 		    sizeof (dtrace_statvar_t *));
10904 	}
10905 }
10906 
10907 static void
10908 dtrace_state_clean(dtrace_state_t *state)
10909 {
10910 	if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE)
10911 		return;
10912 
10913 	dtrace_dynvar_clean(&state->dts_vstate.dtvs_dynvars);
10914 	dtrace_speculation_clean(state);
10915 }
10916 
10917 static void
10918 dtrace_state_deadman(dtrace_state_t *state)
10919 {
10920 	hrtime_t now;
10921 
10922 	dtrace_sync();
10923 
10924 	now = dtrace_gethrtime();
10925 
10926 	if (state != dtrace_anon.dta_state &&
10927 	    now - state->dts_laststatus >= dtrace_deadman_user)
10928 		return;
10929 
10930 	/*
10931 	 * We must be sure that dts_alive never appears to be less than the
10932 	 * value upon entry to dtrace_state_deadman(), and because we lack a
10933 	 * dtrace_cas64(), we cannot store to it atomically.  We thus instead
10934 	 * store INT64_MAX to it, followed by a memory barrier, followed by
10935 	 * the new value.  This assures that dts_alive never appears to be
10936 	 * less than its true value, regardless of the order in which the
10937 	 * stores to the underlying storage are issued.
10938 	 */
10939 	state->dts_alive = INT64_MAX;
10940 	dtrace_membar_producer();
10941 	state->dts_alive = now;
10942 }
10943 
10944 dtrace_state_t *
10945 dtrace_state_create(dev_t *devp, cred_t *cr)
10946 {
10947 	minor_t minor;
10948 	major_t major;
10949 	char c[30];
10950 	dtrace_state_t *state;
10951 	dtrace_optval_t *opt;
10952 	int bufsize = NCPU * sizeof (dtrace_buffer_t), i;
10953 
10954 	ASSERT(MUTEX_HELD(&dtrace_lock));
10955 	ASSERT(MUTEX_HELD(&cpu_lock));
10956 
10957 	minor = (minor_t)(uintptr_t)vmem_alloc(dtrace_minor, 1,
10958 	    VM_BESTFIT | VM_SLEEP);
10959 
10960 	if (ddi_soft_state_zalloc(dtrace_softstate, minor) != DDI_SUCCESS) {
10961 		vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
10962 		return (NULL);
10963 	}
10964 
10965 	state = ddi_get_soft_state(dtrace_softstate, minor);
10966 	state->dts_epid = DTRACE_EPIDNONE + 1;
10967 
10968 	(void) snprintf(c, sizeof (c), "dtrace_aggid_%d", minor);
10969 	state->dts_aggid_arena = vmem_create(c, (void *)1, UINT32_MAX, 1,
10970 	    NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
10971 
10972 	if (devp != NULL) {
10973 		major = getemajor(*devp);
10974 	} else {
10975 		major = ddi_driver_major(dtrace_devi);
10976 	}
10977 
10978 	state->dts_dev = makedevice(major, minor);
10979 
10980 	if (devp != NULL)
10981 		*devp = state->dts_dev;
10982 
10983 	/*
10984 	 * We allocate NCPU buffers.  On the one hand, this can be quite
10985 	 * a bit of memory per instance (nearly 36K on a Starcat).  On the
10986 	 * other hand, it saves an additional memory reference in the probe
10987 	 * path.
10988 	 */
10989 	state->dts_buffer = kmem_zalloc(bufsize, KM_SLEEP);
10990 	state->dts_aggbuffer = kmem_zalloc(bufsize, KM_SLEEP);
10991 	state->dts_cleaner = CYCLIC_NONE;
10992 	state->dts_deadman = CYCLIC_NONE;
10993 	state->dts_vstate.dtvs_state = state;
10994 
10995 	for (i = 0; i < DTRACEOPT_MAX; i++)
10996 		state->dts_options[i] = DTRACEOPT_UNSET;
10997 
10998 	/*
10999 	 * Set the default options.
11000 	 */
11001 	opt = state->dts_options;
11002 	opt[DTRACEOPT_BUFPOLICY] = DTRACEOPT_BUFPOLICY_SWITCH;
11003 	opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_AUTO;
11004 	opt[DTRACEOPT_NSPEC] = dtrace_nspec_default;
11005 	opt[DTRACEOPT_SPECSIZE] = dtrace_specsize_default;
11006 	opt[DTRACEOPT_CPU] = (dtrace_optval_t)DTRACE_CPUALL;
11007 	opt[DTRACEOPT_STRSIZE] = dtrace_strsize_default;
11008 	opt[DTRACEOPT_STACKFRAMES] = dtrace_stackframes_default;
11009 	opt[DTRACEOPT_USTACKFRAMES] = dtrace_ustackframes_default;
11010 	opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_default;
11011 	opt[DTRACEOPT_AGGRATE] = dtrace_aggrate_default;
11012 	opt[DTRACEOPT_SWITCHRATE] = dtrace_switchrate_default;
11013 	opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_default;
11014 	opt[DTRACEOPT_JSTACKFRAMES] = dtrace_jstackframes_default;
11015 	opt[DTRACEOPT_JSTACKSTRSIZE] = dtrace_jstackstrsize_default;
11016 
11017 	state->dts_activity = DTRACE_ACTIVITY_INACTIVE;
11018 
11019 	/*
11020 	 * Set up the credentials for this instantiation.
11021 	 */
11022 	if (cr == NULL || PRIV_POLICY_ONLY(cr, PRIV_ALL, B_FALSE)) {
11023 		state->dts_cred.dcr_visible = DTRACE_CRV_ALL;
11024 		state->dts_cred.dcr_action = DTRACE_CRA_ALL;
11025 	} else {
11026 		state->dts_cred.dcr_uid = crgetuid(cr);
11027 		state->dts_cred.dcr_gid = crgetgid(cr);
11028 
11029 		if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) ||
11030 		    PRIV_POLICY_ONLY(cr, PRIV_DTRACE_PROC, B_FALSE)) {
11031 			state->dts_cred.dcr_action |= DTRACE_CRA_PROC;
11032 		}
11033 
11034 		if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_USER, B_FALSE) &&
11035 		    PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE)) {
11036 			state->dts_cred.dcr_visible |= DTRACE_CRV_ALLPROC;
11037 			state->dts_cred.dcr_action |=
11038 			    DTRACE_CRA_PROC_DESTRUCTIVE;
11039 		}
11040 
11041 		if (PRIV_POLICY_ONLY(cr, PRIV_DTRACE_KERNEL, B_FALSE)) {
11042 			state->dts_cred.dcr_visible |= DTRACE_CRV_KERNEL |
11043 			    DTRACE_CRV_ALLPROC;
11044 			state->dts_cred.dcr_action |= DTRACE_CRA_KERNEL |
11045 			    DTRACE_CRA_PROC;
11046 
11047 			if (PRIV_POLICY_ONLY(cr, PRIV_PROC_OWNER, B_FALSE))
11048 				state->dts_cred.dcr_action |=
11049 				    DTRACE_CRA_PROC_DESTRUCTIVE;
11050 		}
11051 	}
11052 
11053 	return (state);
11054 }
11055 
11056 static int
11057 dtrace_state_buffer(dtrace_state_t *state, dtrace_buffer_t *buf, int which)
11058 {
11059 	dtrace_optval_t *opt = state->dts_options, size;
11060 	processorid_t cpu;
11061 	int flags = 0, rval;
11062 
11063 	ASSERT(MUTEX_HELD(&dtrace_lock));
11064 	ASSERT(MUTEX_HELD(&cpu_lock));
11065 	ASSERT(which < DTRACEOPT_MAX);
11066 	ASSERT(state->dts_activity == DTRACE_ACTIVITY_INACTIVE ||
11067 	    (state == dtrace_anon.dta_state &&
11068 	    state->dts_activity == DTRACE_ACTIVITY_ACTIVE));
11069 
11070 	if (opt[which] == DTRACEOPT_UNSET || opt[which] == 0)
11071 		return (0);
11072 
11073 	if (opt[DTRACEOPT_CPU] != DTRACEOPT_UNSET)
11074 		cpu = opt[DTRACEOPT_CPU];
11075 
11076 	if (which == DTRACEOPT_SPECSIZE)
11077 		flags |= DTRACEBUF_NOSWITCH;
11078 
11079 	if (which == DTRACEOPT_BUFSIZE) {
11080 		if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_RING)
11081 			flags |= DTRACEBUF_RING;
11082 
11083 		if (opt[DTRACEOPT_BUFPOLICY] == DTRACEOPT_BUFPOLICY_FILL)
11084 			flags |= DTRACEBUF_FILL;
11085 
11086 		flags |= DTRACEBUF_INACTIVE;
11087 	}
11088 
11089 	for (size = opt[which]; size >= sizeof (uint64_t); size >>= 1) {
11090 		/*
11091 		 * The size must be 8-byte aligned.  If the size is not 8-byte
11092 		 * aligned, drop it down by the difference.
11093 		 */
11094 		if (size & (sizeof (uint64_t) - 1))
11095 			size -= size & (sizeof (uint64_t) - 1);
11096 
11097 		if (size < state->dts_reserve) {
11098 			/*
11099 			 * Buffers always must be large enough to accommodate
11100 			 * their prereserved space.  We return E2BIG instead
11101 			 * of ENOMEM in this case to allow for user-level
11102 			 * software to differentiate the cases.
11103 			 */
11104 			return (E2BIG);
11105 		}
11106 
11107 		rval = dtrace_buffer_alloc(buf, size, flags, cpu);
11108 
11109 		if (rval != ENOMEM) {
11110 			opt[which] = size;
11111 			return (rval);
11112 		}
11113 
11114 		if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
11115 			return (rval);
11116 	}
11117 
11118 	return (ENOMEM);
11119 }
11120 
11121 static int
11122 dtrace_state_buffers(dtrace_state_t *state)
11123 {
11124 	dtrace_speculation_t *spec = state->dts_speculations;
11125 	int rval, i;
11126 
11127 	if ((rval = dtrace_state_buffer(state, state->dts_buffer,
11128 	    DTRACEOPT_BUFSIZE)) != 0)
11129 		return (rval);
11130 
11131 	if ((rval = dtrace_state_buffer(state, state->dts_aggbuffer,
11132 	    DTRACEOPT_AGGSIZE)) != 0)
11133 		return (rval);
11134 
11135 	for (i = 0; i < state->dts_nspeculations; i++) {
11136 		if ((rval = dtrace_state_buffer(state,
11137 		    spec[i].dtsp_buffer, DTRACEOPT_SPECSIZE)) != 0)
11138 			return (rval);
11139 	}
11140 
11141 	return (0);
11142 }
11143 
11144 static void
11145 dtrace_state_prereserve(dtrace_state_t *state)
11146 {
11147 	dtrace_ecb_t *ecb;
11148 	dtrace_probe_t *probe;
11149 
11150 	state->dts_reserve = 0;
11151 
11152 	if (state->dts_options[DTRACEOPT_BUFPOLICY] != DTRACEOPT_BUFPOLICY_FILL)
11153 		return;
11154 
11155 	/*
11156 	 * If our buffer policy is a "fill" buffer policy, we need to set the
11157 	 * prereserved space to be the space required by the END probes.
11158 	 */
11159 	probe = dtrace_probes[dtrace_probeid_end - 1];
11160 	ASSERT(probe != NULL);
11161 
11162 	for (ecb = probe->dtpr_ecb; ecb != NULL; ecb = ecb->dte_next) {
11163 		if (ecb->dte_state != state)
11164 			continue;
11165 
11166 		state->dts_reserve += ecb->dte_needed + ecb->dte_alignment;
11167 	}
11168 }
11169 
11170 static int
11171 dtrace_state_go(dtrace_state_t *state, processorid_t *cpu)
11172 {
11173 	dtrace_optval_t *opt = state->dts_options, sz, nspec;
11174 	dtrace_speculation_t *spec;
11175 	dtrace_buffer_t *buf;
11176 	cyc_handler_t hdlr;
11177 	cyc_time_t when;
11178 	int rval = 0, i, bufsize = NCPU * sizeof (dtrace_buffer_t);
11179 	dtrace_icookie_t cookie;
11180 
11181 	mutex_enter(&cpu_lock);
11182 	mutex_enter(&dtrace_lock);
11183 
11184 	if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
11185 		rval = EBUSY;
11186 		goto out;
11187 	}
11188 
11189 	/*
11190 	 * Before we can perform any checks, we must prime all of the
11191 	 * retained enablings that correspond to this state.
11192 	 */
11193 	dtrace_enabling_prime(state);
11194 
11195 	if (state->dts_destructive && !state->dts_cred.dcr_destructive) {
11196 		rval = EACCES;
11197 		goto out;
11198 	}
11199 
11200 	dtrace_state_prereserve(state);
11201 
11202 	/*
11203 	 * Now we want to do is try to allocate our speculations.
11204 	 * We do not automatically resize the number of speculations; if
11205 	 * this fails, we will fail the operation.
11206 	 */
11207 	nspec = opt[DTRACEOPT_NSPEC];
11208 	ASSERT(nspec != DTRACEOPT_UNSET);
11209 
11210 	if (nspec > INT_MAX) {
11211 		rval = ENOMEM;
11212 		goto out;
11213 	}
11214 
11215 	spec = kmem_zalloc(nspec * sizeof (dtrace_speculation_t), KM_NOSLEEP);
11216 
11217 	if (spec == NULL) {
11218 		rval = ENOMEM;
11219 		goto out;
11220 	}
11221 
11222 	state->dts_speculations = spec;
11223 	state->dts_nspeculations = (int)nspec;
11224 
11225 	for (i = 0; i < nspec; i++) {
11226 		if ((buf = kmem_zalloc(bufsize, KM_NOSLEEP)) == NULL) {
11227 			rval = ENOMEM;
11228 			goto err;
11229 		}
11230 
11231 		spec[i].dtsp_buffer = buf;
11232 	}
11233 
11234 	if (opt[DTRACEOPT_GRABANON] != DTRACEOPT_UNSET) {
11235 		if (dtrace_anon.dta_state == NULL) {
11236 			rval = ENOENT;
11237 			goto out;
11238 		}
11239 
11240 		if (state->dts_necbs != 0) {
11241 			rval = EALREADY;
11242 			goto out;
11243 		}
11244 
11245 		state->dts_anon = dtrace_anon_grab();
11246 		ASSERT(state->dts_anon != NULL);
11247 
11248 		*cpu = dtrace_anon.dta_beganon;
11249 
11250 		/*
11251 		 * If the anonymous state is active (as it almost certainly
11252 		 * is if the anonymous enabling ultimately matched anything),
11253 		 * we don't allow any further option processing -- but we
11254 		 * don't return failure.
11255 		 */
11256 		state = state->dts_anon;
11257 
11258 		if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
11259 			goto out;
11260 	}
11261 
11262 	if (opt[DTRACEOPT_AGGSIZE] != DTRACEOPT_UNSET &&
11263 	    opt[DTRACEOPT_AGGSIZE] != 0) {
11264 		if (state->dts_aggregations == NULL) {
11265 			/*
11266 			 * We're not going to create an aggregation buffer
11267 			 * because we don't have any ECBs that contain
11268 			 * aggregations -- set this option to 0.
11269 			 */
11270 			opt[DTRACEOPT_AGGSIZE] = 0;
11271 		} else {
11272 			/*
11273 			 * If we have an aggregation buffer, we must also have
11274 			 * a buffer to use as scratch.
11275 			 */
11276 			if (opt[DTRACEOPT_BUFSIZE] == DTRACEOPT_UNSET ||
11277 			    opt[DTRACEOPT_BUFSIZE] < state->dts_needed) {
11278 				opt[DTRACEOPT_BUFSIZE] = state->dts_needed;
11279 			}
11280 		}
11281 	}
11282 
11283 	if (opt[DTRACEOPT_SPECSIZE] != DTRACEOPT_UNSET &&
11284 	    opt[DTRACEOPT_SPECSIZE] != 0) {
11285 		if (!state->dts_speculates) {
11286 			/*
11287 			 * We're not going to create speculation buffers
11288 			 * because we don't have any ECBs that actually
11289 			 * speculate -- set the speculation size to 0.
11290 			 */
11291 			opt[DTRACEOPT_SPECSIZE] = 0;
11292 		}
11293 	}
11294 
11295 	/*
11296 	 * The bare minimum size for any buffer that we're actually going to
11297 	 * do anything to is sizeof (uint64_t).
11298 	 */
11299 	sz = sizeof (uint64_t);
11300 
11301 	if ((state->dts_needed != 0 && opt[DTRACEOPT_BUFSIZE] < sz) ||
11302 	    (state->dts_speculates && opt[DTRACEOPT_SPECSIZE] < sz) ||
11303 	    (state->dts_aggregations != NULL && opt[DTRACEOPT_AGGSIZE] < sz)) {
11304 		/*
11305 		 * A buffer size has been explicitly set to 0 (or to a size
11306 		 * that will be adjusted to 0) and we need the space -- we
11307 		 * need to return failure.  We return ENOSPC to differentiate
11308 		 * it from failing to allocate a buffer due to failure to meet
11309 		 * the reserve (for which we return E2BIG).
11310 		 */
11311 		rval = ENOSPC;
11312 		goto out;
11313 	}
11314 
11315 	if ((rval = dtrace_state_buffers(state)) != 0)
11316 		goto err;
11317 
11318 	if ((sz = opt[DTRACEOPT_DYNVARSIZE]) == DTRACEOPT_UNSET)
11319 		sz = dtrace_dstate_defsize;
11320 
11321 	do {
11322 		rval = dtrace_dstate_init(&state->dts_vstate.dtvs_dynvars, sz);
11323 
11324 		if (rval == 0)
11325 			break;
11326 
11327 		if (opt[DTRACEOPT_BUFRESIZE] == DTRACEOPT_BUFRESIZE_MANUAL)
11328 			goto err;
11329 	} while (sz >>= 1);
11330 
11331 	opt[DTRACEOPT_DYNVARSIZE] = sz;
11332 
11333 	if (rval != 0)
11334 		goto err;
11335 
11336 	if (opt[DTRACEOPT_STATUSRATE] > dtrace_statusrate_max)
11337 		opt[DTRACEOPT_STATUSRATE] = dtrace_statusrate_max;
11338 
11339 	if (opt[DTRACEOPT_CLEANRATE] == 0)
11340 		opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
11341 
11342 	if (opt[DTRACEOPT_CLEANRATE] < dtrace_cleanrate_min)
11343 		opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_min;
11344 
11345 	if (opt[DTRACEOPT_CLEANRATE] > dtrace_cleanrate_max)
11346 		opt[DTRACEOPT_CLEANRATE] = dtrace_cleanrate_max;
11347 
11348 	hdlr.cyh_func = (cyc_func_t)dtrace_state_clean;
11349 	hdlr.cyh_arg = state;
11350 	hdlr.cyh_level = CY_LOW_LEVEL;
11351 
11352 	when.cyt_when = 0;
11353 	when.cyt_interval = opt[DTRACEOPT_CLEANRATE];
11354 
11355 	state->dts_cleaner = cyclic_add(&hdlr, &when);
11356 
11357 	hdlr.cyh_func = (cyc_func_t)dtrace_state_deadman;
11358 	hdlr.cyh_arg = state;
11359 	hdlr.cyh_level = CY_LOW_LEVEL;
11360 
11361 	when.cyt_when = 0;
11362 	when.cyt_interval = dtrace_deadman_interval;
11363 
11364 	state->dts_alive = state->dts_laststatus = dtrace_gethrtime();
11365 	state->dts_deadman = cyclic_add(&hdlr, &when);
11366 
11367 	state->dts_activity = DTRACE_ACTIVITY_WARMUP;
11368 
11369 	/*
11370 	 * Now it's time to actually fire the BEGIN probe.  We need to disable
11371 	 * interrupts here both to record the CPU on which we fired the BEGIN
11372 	 * probe (the data from this CPU will be processed first at user
11373 	 * level) and to manually activate the buffer for this CPU.
11374 	 */
11375 	cookie = dtrace_interrupt_disable();
11376 	*cpu = CPU->cpu_id;
11377 	ASSERT(state->dts_buffer[*cpu].dtb_flags & DTRACEBUF_INACTIVE);
11378 	state->dts_buffer[*cpu].dtb_flags &= ~DTRACEBUF_INACTIVE;
11379 
11380 	dtrace_probe(dtrace_probeid_begin,
11381 	    (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
11382 	dtrace_interrupt_enable(cookie);
11383 	/*
11384 	 * We may have had an exit action from a BEGIN probe; only change our
11385 	 * state to ACTIVE if we're still in WARMUP.
11386 	 */
11387 	ASSERT(state->dts_activity == DTRACE_ACTIVITY_WARMUP ||
11388 	    state->dts_activity == DTRACE_ACTIVITY_DRAINING);
11389 
11390 	if (state->dts_activity == DTRACE_ACTIVITY_WARMUP)
11391 		state->dts_activity = DTRACE_ACTIVITY_ACTIVE;
11392 
11393 	/*
11394 	 * Regardless of whether or not now we're in ACTIVE or DRAINING, we
11395 	 * want each CPU to transition its principal buffer out of the
11396 	 * INACTIVE state.  Doing this assures that no CPU will suddenly begin
11397 	 * processing an ECB halfway down a probe's ECB chain; all CPUs will
11398 	 * atomically transition from processing none of a state's ECBs to
11399 	 * processing all of them.
11400 	 */
11401 	dtrace_xcall(DTRACE_CPUALL,
11402 	    (dtrace_xcall_t)dtrace_buffer_activate, state);
11403 	goto out;
11404 
11405 err:
11406 	dtrace_buffer_free(state->dts_buffer);
11407 	dtrace_buffer_free(state->dts_aggbuffer);
11408 
11409 	if ((nspec = state->dts_nspeculations) == 0) {
11410 		ASSERT(state->dts_speculations == NULL);
11411 		goto out;
11412 	}
11413 
11414 	spec = state->dts_speculations;
11415 	ASSERT(spec != NULL);
11416 
11417 	for (i = 0; i < state->dts_nspeculations; i++) {
11418 		if ((buf = spec[i].dtsp_buffer) == NULL)
11419 			break;
11420 
11421 		dtrace_buffer_free(buf);
11422 		kmem_free(buf, bufsize);
11423 	}
11424 
11425 	kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
11426 	state->dts_nspeculations = 0;
11427 	state->dts_speculations = NULL;
11428 
11429 out:
11430 	mutex_exit(&dtrace_lock);
11431 	mutex_exit(&cpu_lock);
11432 
11433 	return (rval);
11434 }
11435 
11436 static int
11437 dtrace_state_stop(dtrace_state_t *state, processorid_t *cpu)
11438 {
11439 	dtrace_icookie_t cookie;
11440 
11441 	ASSERT(MUTEX_HELD(&dtrace_lock));
11442 
11443 	if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE &&
11444 	    state->dts_activity != DTRACE_ACTIVITY_DRAINING)
11445 		return (EINVAL);
11446 
11447 	/*
11448 	 * We'll set the activity to DTRACE_ACTIVITY_DRAINING, and issue a sync
11449 	 * to be sure that every CPU has seen it.  See below for the details
11450 	 * on why this is done.
11451 	 */
11452 	state->dts_activity = DTRACE_ACTIVITY_DRAINING;
11453 	dtrace_sync();
11454 
11455 	/*
11456 	 * By this point, it is impossible for any CPU to be still processing
11457 	 * with DTRACE_ACTIVITY_ACTIVE.  We can thus set our activity to
11458 	 * DTRACE_ACTIVITY_COOLDOWN and know that we're not racing with any
11459 	 * other CPU in dtrace_buffer_reserve().  This allows dtrace_probe()
11460 	 * and callees to know that the activity is DTRACE_ACTIVITY_COOLDOWN
11461 	 * iff we're in the END probe.
11462 	 */
11463 	state->dts_activity = DTRACE_ACTIVITY_COOLDOWN;
11464 	dtrace_sync();
11465 	ASSERT(state->dts_activity == DTRACE_ACTIVITY_COOLDOWN);
11466 
11467 	/*
11468 	 * Finally, we can release the reserve and call the END probe.  We
11469 	 * disable interrupts across calling the END probe to allow us to
11470 	 * return the CPU on which we actually called the END probe.  This
11471 	 * allows user-land to be sure that this CPU's principal buffer is
11472 	 * processed last.
11473 	 */
11474 	state->dts_reserve = 0;
11475 
11476 	cookie = dtrace_interrupt_disable();
11477 	*cpu = CPU->cpu_id;
11478 	dtrace_probe(dtrace_probeid_end,
11479 	    (uint64_t)(uintptr_t)state, 0, 0, 0, 0);
11480 	dtrace_interrupt_enable(cookie);
11481 
11482 	state->dts_activity = DTRACE_ACTIVITY_STOPPED;
11483 	dtrace_sync();
11484 
11485 	return (0);
11486 }
11487 
11488 static int
11489 dtrace_state_option(dtrace_state_t *state, dtrace_optid_t option,
11490     dtrace_optval_t val)
11491 {
11492 	ASSERT(MUTEX_HELD(&dtrace_lock));
11493 
11494 	if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE)
11495 		return (EBUSY);
11496 
11497 	if (option >= DTRACEOPT_MAX)
11498 		return (EINVAL);
11499 
11500 	if (option != DTRACEOPT_CPU && val < 0)
11501 		return (EINVAL);
11502 
11503 	switch (option) {
11504 	case DTRACEOPT_DESTRUCTIVE:
11505 		if (dtrace_destructive_disallow)
11506 			return (EACCES);
11507 
11508 		state->dts_cred.dcr_destructive = 1;
11509 		break;
11510 
11511 	case DTRACEOPT_BUFSIZE:
11512 	case DTRACEOPT_DYNVARSIZE:
11513 	case DTRACEOPT_AGGSIZE:
11514 	case DTRACEOPT_SPECSIZE:
11515 	case DTRACEOPT_STRSIZE:
11516 		if (val < 0)
11517 			return (EINVAL);
11518 
11519 		if (val >= LONG_MAX) {
11520 			/*
11521 			 * If this is an otherwise negative value, set it to
11522 			 * the highest multiple of 128m less than LONG_MAX.
11523 			 * Technically, we're adjusting the size without
11524 			 * regard to the buffer resizing policy, but in fact,
11525 			 * this has no effect -- if we set the buffer size to
11526 			 * ~LONG_MAX and the buffer policy is ultimately set to
11527 			 * be "manual", the buffer allocation is guaranteed to
11528 			 * fail, if only because the allocation requires two
11529 			 * buffers.  (We set the the size to the highest
11530 			 * multiple of 128m because it ensures that the size
11531 			 * will remain a multiple of a megabyte when
11532 			 * repeatedly halved -- all the way down to 15m.)
11533 			 */
11534 			val = LONG_MAX - (1 << 27) + 1;
11535 		}
11536 	}
11537 
11538 	state->dts_options[option] = val;
11539 
11540 	return (0);
11541 }
11542 
11543 static void
11544 dtrace_state_destroy(dtrace_state_t *state)
11545 {
11546 	dtrace_ecb_t *ecb;
11547 	dtrace_vstate_t *vstate = &state->dts_vstate;
11548 	minor_t minor = getminor(state->dts_dev);
11549 	int i, bufsize = NCPU * sizeof (dtrace_buffer_t);
11550 	dtrace_speculation_t *spec = state->dts_speculations;
11551 	int nspec = state->dts_nspeculations;
11552 	uint32_t match;
11553 
11554 	ASSERT(MUTEX_HELD(&dtrace_lock));
11555 	ASSERT(MUTEX_HELD(&cpu_lock));
11556 
11557 	/*
11558 	 * First, retract any retained enablings for this state.
11559 	 */
11560 	dtrace_enabling_retract(state);
11561 	ASSERT(state->dts_nretained == 0);
11562 
11563 	/*
11564 	 * Now we need to disable and destroy any enabled probes.  Because any
11565 	 * DTRACE_PRIV_KERNEL probes may actually be slowing our progress
11566 	 * (especially if they're all enabled), we take two passes through
11567 	 * the ECBs:  in the first, we disable just DTRACE_PRIV_KERNEL probes,
11568 	 * and in the second we disable whatever is left over.
11569 	 */
11570 	for (match = DTRACE_PRIV_KERNEL; ; match = 0) {
11571 		for (i = 0; i < state->dts_necbs; i++) {
11572 			if ((ecb = state->dts_ecbs[i]) == NULL)
11573 				continue;
11574 
11575 			if (match && ecb->dte_probe != NULL) {
11576 				dtrace_probe_t *probe = ecb->dte_probe;
11577 				dtrace_provider_t *prov = probe->dtpr_provider;
11578 
11579 				if (!(prov->dtpv_priv.dtpp_flags & match))
11580 					continue;
11581 			}
11582 
11583 			dtrace_ecb_disable(ecb);
11584 			dtrace_ecb_destroy(ecb);
11585 		}
11586 
11587 		if (!match)
11588 			break;
11589 	}
11590 
11591 	/*
11592 	 * Before we free the buffers, perform one more sync to assure that
11593 	 * every CPU is out of probe context.
11594 	 */
11595 	dtrace_sync();
11596 
11597 	dtrace_buffer_free(state->dts_buffer);
11598 	dtrace_buffer_free(state->dts_aggbuffer);
11599 
11600 	for (i = 0; i < nspec; i++)
11601 		dtrace_buffer_free(spec[i].dtsp_buffer);
11602 
11603 	if (state->dts_cleaner != CYCLIC_NONE)
11604 		cyclic_remove(state->dts_cleaner);
11605 
11606 	if (state->dts_deadman != CYCLIC_NONE)
11607 		cyclic_remove(state->dts_deadman);
11608 
11609 	dtrace_dstate_fini(&vstate->dtvs_dynvars);
11610 	dtrace_vstate_fini(vstate);
11611 	kmem_free(state->dts_ecbs, state->dts_necbs * sizeof (dtrace_ecb_t *));
11612 
11613 	if (state->dts_aggregations != NULL) {
11614 #ifdef DEBUG
11615 		for (i = 0; i < state->dts_naggregations; i++)
11616 			ASSERT(state->dts_aggregations[i] == NULL);
11617 #endif
11618 		ASSERT(state->dts_naggregations > 0);
11619 		kmem_free(state->dts_aggregations,
11620 		    state->dts_naggregations * sizeof (dtrace_aggregation_t *));
11621 	}
11622 
11623 	kmem_free(state->dts_buffer, bufsize);
11624 	kmem_free(state->dts_aggbuffer, bufsize);
11625 
11626 	for (i = 0; i < nspec; i++)
11627 		kmem_free(spec[i].dtsp_buffer, bufsize);
11628 
11629 	kmem_free(spec, nspec * sizeof (dtrace_speculation_t));
11630 
11631 	dtrace_format_destroy(state);
11632 
11633 	vmem_destroy(state->dts_aggid_arena);
11634 	ddi_soft_state_free(dtrace_softstate, minor);
11635 	vmem_free(dtrace_minor, (void *)(uintptr_t)minor, 1);
11636 }
11637 
11638 /*
11639  * DTrace Anonymous Enabling Functions
11640  */
11641 static dtrace_state_t *
11642 dtrace_anon_grab(void)
11643 {
11644 	dtrace_state_t *state;
11645 
11646 	ASSERT(MUTEX_HELD(&dtrace_lock));
11647 
11648 	if ((state = dtrace_anon.dta_state) == NULL) {
11649 		ASSERT(dtrace_anon.dta_enabling == NULL);
11650 		return (NULL);
11651 	}
11652 
11653 	ASSERT(dtrace_anon.dta_enabling != NULL);
11654 	ASSERT(dtrace_retained != NULL);
11655 
11656 	dtrace_enabling_destroy(dtrace_anon.dta_enabling);
11657 	dtrace_anon.dta_enabling = NULL;
11658 	dtrace_anon.dta_state = NULL;
11659 
11660 	return (state);
11661 }
11662 
11663 static void
11664 dtrace_anon_property(void)
11665 {
11666 	int i, rv;
11667 	dtrace_state_t *state;
11668 	dof_hdr_t *dof;
11669 	char c[32];		/* enough for "dof-data-" + digits */
11670 
11671 	ASSERT(MUTEX_HELD(&dtrace_lock));
11672 	ASSERT(MUTEX_HELD(&cpu_lock));
11673 
11674 	for (i = 0; ; i++) {
11675 		(void) snprintf(c, sizeof (c), "dof-data-%d", i);
11676 
11677 		dtrace_err_verbose = 1;
11678 
11679 		if ((dof = dtrace_dof_property(c)) == NULL) {
11680 			dtrace_err_verbose = 0;
11681 			break;
11682 		}
11683 
11684 		/*
11685 		 * We want to create anonymous state, so we need to transition
11686 		 * the kernel debugger to indicate that DTrace is active.  If
11687 		 * this fails (e.g. because the debugger has modified text in
11688 		 * some way), we won't continue with the processing.
11689 		 */
11690 		if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
11691 			cmn_err(CE_NOTE, "kernel debugger active; anonymous "
11692 			    "enabling ignored.");
11693 			dtrace_dof_destroy(dof);
11694 			break;
11695 		}
11696 
11697 		/*
11698 		 * If we haven't allocated an anonymous state, we'll do so now.
11699 		 */
11700 		if ((state = dtrace_anon.dta_state) == NULL) {
11701 			state = dtrace_state_create(NULL, NULL);
11702 			dtrace_anon.dta_state = state;
11703 
11704 			if (state == NULL) {
11705 				/*
11706 				 * This basically shouldn't happen:  the only
11707 				 * failure mode from dtrace_state_create() is a
11708 				 * failure of ddi_soft_state_zalloc() that
11709 				 * itself should never happen.  Still, the
11710 				 * interface allows for a failure mode, and
11711 				 * we want to fail as gracefully as possible:
11712 				 * we'll emit an error message and cease
11713 				 * processing anonymous state in this case.
11714 				 */
11715 				cmn_err(CE_WARN, "failed to create "
11716 				    "anonymous state");
11717 				dtrace_dof_destroy(dof);
11718 				break;
11719 			}
11720 		}
11721 
11722 		rv = dtrace_dof_slurp(dof, &state->dts_vstate, CRED(),
11723 		    &dtrace_anon.dta_enabling, 0, B_TRUE);
11724 
11725 		if (rv == 0)
11726 			rv = dtrace_dof_options(dof, state);
11727 
11728 		dtrace_err_verbose = 0;
11729 		dtrace_dof_destroy(dof);
11730 
11731 		if (rv != 0) {
11732 			/*
11733 			 * This is malformed DOF; chuck any anonymous state
11734 			 * that we created.
11735 			 */
11736 			ASSERT(dtrace_anon.dta_enabling == NULL);
11737 			dtrace_state_destroy(state);
11738 			dtrace_anon.dta_state = NULL;
11739 			break;
11740 		}
11741 
11742 		ASSERT(dtrace_anon.dta_enabling != NULL);
11743 	}
11744 
11745 	if (dtrace_anon.dta_enabling != NULL) {
11746 		int rval;
11747 
11748 		/*
11749 		 * dtrace_enabling_retain() can only fail because we are
11750 		 * trying to retain more enablings than are allowed -- but
11751 		 * we only have one anonymous enabling, and we are guaranteed
11752 		 * to be allowed at least one retained enabling; we assert
11753 		 * that dtrace_enabling_retain() returns success.
11754 		 */
11755 		rval = dtrace_enabling_retain(dtrace_anon.dta_enabling);
11756 		ASSERT(rval == 0);
11757 
11758 		dtrace_enabling_dump(dtrace_anon.dta_enabling);
11759 	}
11760 }
11761 
11762 /*
11763  * DTrace Helper Functions
11764  */
11765 static void
11766 dtrace_helper_trace(dtrace_helper_action_t *helper,
11767     dtrace_mstate_t *mstate, dtrace_vstate_t *vstate, int where)
11768 {
11769 	uint32_t size, next, nnext, i;
11770 	dtrace_helptrace_t *ent;
11771 	uint16_t flags = cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
11772 
11773 	if (!dtrace_helptrace_enabled)
11774 		return;
11775 
11776 	ASSERT(vstate->dtvs_nlocals <= dtrace_helptrace_nlocals);
11777 
11778 	/*
11779 	 * What would a tracing framework be without its own tracing
11780 	 * framework?  (Well, a hell of a lot simpler, for starters...)
11781 	 */
11782 	size = sizeof (dtrace_helptrace_t) + dtrace_helptrace_nlocals *
11783 	    sizeof (uint64_t) - sizeof (uint64_t);
11784 
11785 	/*
11786 	 * Iterate until we can allocate a slot in the trace buffer.
11787 	 */
11788 	do {
11789 		next = dtrace_helptrace_next;
11790 
11791 		if (next + size < dtrace_helptrace_bufsize) {
11792 			nnext = next + size;
11793 		} else {
11794 			nnext = size;
11795 		}
11796 	} while (dtrace_cas32(&dtrace_helptrace_next, next, nnext) != next);
11797 
11798 	/*
11799 	 * We have our slot; fill it in.
11800 	 */
11801 	if (nnext == size)
11802 		next = 0;
11803 
11804 	ent = (dtrace_helptrace_t *)&dtrace_helptrace_buffer[next];
11805 	ent->dtht_helper = helper;
11806 	ent->dtht_where = where;
11807 	ent->dtht_nlocals = vstate->dtvs_nlocals;
11808 
11809 	ent->dtht_fltoffs = (mstate->dtms_present & DTRACE_MSTATE_FLTOFFS) ?
11810 	    mstate->dtms_fltoffs : -1;
11811 	ent->dtht_fault = DTRACE_FLAGS2FLT(flags);
11812 	ent->dtht_illval = cpu_core[CPU->cpu_id].cpuc_dtrace_illval;
11813 
11814 	for (i = 0; i < vstate->dtvs_nlocals; i++) {
11815 		dtrace_statvar_t *svar;
11816 
11817 		if ((svar = vstate->dtvs_locals[i]) == NULL)
11818 			continue;
11819 
11820 		ASSERT(svar->dtsv_size >= NCPU * sizeof (uint64_t));
11821 		ent->dtht_locals[i] =
11822 		    ((uint64_t *)(uintptr_t)svar->dtsv_data)[CPU->cpu_id];
11823 	}
11824 }
11825 
11826 static uint64_t
11827 dtrace_helper(int which, dtrace_mstate_t *mstate,
11828     dtrace_state_t *state, uint64_t arg0, uint64_t arg1)
11829 {
11830 	uint16_t *flags = &cpu_core[CPU->cpu_id].cpuc_dtrace_flags;
11831 	uint64_t sarg0 = mstate->dtms_arg[0];
11832 	uint64_t sarg1 = mstate->dtms_arg[1];
11833 	uint64_t rval;
11834 	dtrace_helpers_t *helpers = curproc->p_dtrace_helpers;
11835 	dtrace_helper_action_t *helper;
11836 	dtrace_vstate_t *vstate;
11837 	dtrace_difo_t *pred;
11838 	int i, trace = dtrace_helptrace_enabled;
11839 
11840 	ASSERT(which >= 0 && which < DTRACE_NHELPER_ACTIONS);
11841 
11842 	if (helpers == NULL)
11843 		return (0);
11844 
11845 	if ((helper = helpers->dthps_actions[which]) == NULL)
11846 		return (0);
11847 
11848 	vstate = &helpers->dthps_vstate;
11849 	mstate->dtms_arg[0] = arg0;
11850 	mstate->dtms_arg[1] = arg1;
11851 
11852 	/*
11853 	 * Now iterate over each helper.  If its predicate evaluates to 'true',
11854 	 * we'll call the corresponding actions.  Note that the below calls
11855 	 * to dtrace_dif_emulate() may set faults in machine state.  This is
11856 	 * okay:  our caller (the outer dtrace_dif_emulate()) will simply plow
11857 	 * the stored DIF offset with its own (which is the desired behavior).
11858 	 * Also, note the calls to dtrace_dif_emulate() may allocate scratch
11859 	 * from machine state; this is okay, too.
11860 	 */
11861 	for (; helper != NULL; helper = helper->dthp_next) {
11862 		if ((pred = helper->dthp_predicate) != NULL) {
11863 			if (trace)
11864 				dtrace_helper_trace(helper, mstate, vstate, 0);
11865 
11866 			if (!dtrace_dif_emulate(pred, mstate, vstate, state))
11867 				goto next;
11868 
11869 			if (*flags & CPU_DTRACE_FAULT)
11870 				goto err;
11871 		}
11872 
11873 		for (i = 0; i < helper->dthp_nactions; i++) {
11874 			if (trace)
11875 				dtrace_helper_trace(helper,
11876 				    mstate, vstate, i + 1);
11877 
11878 			rval = dtrace_dif_emulate(helper->dthp_actions[i],
11879 			    mstate, vstate, state);
11880 
11881 			if (*flags & CPU_DTRACE_FAULT)
11882 				goto err;
11883 		}
11884 
11885 next:
11886 		if (trace)
11887 			dtrace_helper_trace(helper, mstate, vstate,
11888 			    DTRACE_HELPTRACE_NEXT);
11889 	}
11890 
11891 	if (trace)
11892 		dtrace_helper_trace(helper, mstate, vstate,
11893 		    DTRACE_HELPTRACE_DONE);
11894 
11895 	/*
11896 	 * Restore the arg0 that we saved upon entry.
11897 	 */
11898 	mstate->dtms_arg[0] = sarg0;
11899 	mstate->dtms_arg[1] = sarg1;
11900 
11901 	return (rval);
11902 
11903 err:
11904 	if (trace)
11905 		dtrace_helper_trace(helper, mstate, vstate,
11906 		    DTRACE_HELPTRACE_ERR);
11907 
11908 	/*
11909 	 * Restore the arg0 that we saved upon entry.
11910 	 */
11911 	mstate->dtms_arg[0] = sarg0;
11912 	mstate->dtms_arg[1] = sarg1;
11913 
11914 	return (NULL);
11915 }
11916 
11917 static void
11918 dtrace_helper_destroy(dtrace_helper_action_t *helper, dtrace_vstate_t *vstate)
11919 {
11920 	int i;
11921 
11922 	if (helper->dthp_predicate != NULL)
11923 		dtrace_difo_release(helper->dthp_predicate, vstate);
11924 
11925 	for (i = 0; i < helper->dthp_nactions; i++) {
11926 		ASSERT(helper->dthp_actions[i] != NULL);
11927 		dtrace_difo_release(helper->dthp_actions[i], vstate);
11928 	}
11929 
11930 	kmem_free(helper->dthp_actions,
11931 	    helper->dthp_nactions * sizeof (dtrace_difo_t *));
11932 	kmem_free(helper, sizeof (dtrace_helper_action_t));
11933 }
11934 
11935 static int
11936 dtrace_helper_destroygen(int gen)
11937 {
11938 	dtrace_helpers_t *help = curproc->p_dtrace_helpers;
11939 	dtrace_vstate_t *vstate;
11940 	int i;
11941 
11942 	ASSERT(MUTEX_HELD(&dtrace_lock));
11943 
11944 	if (help == NULL || gen > help->dthps_generation)
11945 		return (EINVAL);
11946 
11947 	vstate = &help->dthps_vstate;
11948 
11949 	for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
11950 		dtrace_helper_action_t *last = NULL, *h, *next;
11951 
11952 		for (h = help->dthps_actions[i]; h != NULL; h = next) {
11953 			next = h->dthp_next;
11954 
11955 			if (h->dthp_generation == gen) {
11956 				if (last != NULL) {
11957 					last->dthp_next = next;
11958 				} else {
11959 					help->dthps_actions[i] = next;
11960 				}
11961 
11962 				dtrace_helper_destroy(h, vstate);
11963 			} else {
11964 				last = h;
11965 			}
11966 		}
11967 	}
11968 
11969 	return (0);
11970 }
11971 
11972 static int
11973 dtrace_helper_validate(dtrace_helper_action_t *helper)
11974 {
11975 	int err = 0, i;
11976 	dtrace_difo_t *dp;
11977 
11978 	if ((dp = helper->dthp_predicate) != NULL)
11979 		err += dtrace_difo_validate_helper(dp);
11980 
11981 	for (i = 0; i < helper->dthp_nactions; i++)
11982 		err += dtrace_difo_validate_helper(helper->dthp_actions[i]);
11983 
11984 	return (err == 0);
11985 }
11986 
11987 static int
11988 dtrace_helper_action_add(int which, dtrace_ecbdesc_t *ep)
11989 {
11990 	dtrace_helpers_t *help;
11991 	dtrace_helper_action_t *helper, *last;
11992 	dtrace_actdesc_t *act;
11993 	dtrace_vstate_t *vstate;
11994 	dtrace_predicate_t *pred;
11995 	int count = 0, nactions = 0, i;
11996 
11997 	if (which < 0 || which >= DTRACE_NHELPER_ACTIONS)
11998 		return (EINVAL);
11999 
12000 	help = curproc->p_dtrace_helpers;
12001 	last = help->dthps_actions[which];
12002 	vstate = &help->dthps_vstate;
12003 
12004 	for (count = 0; last != NULL; last = last->dthp_next) {
12005 		count++;
12006 		if (last->dthp_next == NULL)
12007 			break;
12008 	}
12009 
12010 	/*
12011 	 * If we already have dtrace_helper_actions_max helper actions for this
12012 	 * helper action type, we'll refuse to add a new one.
12013 	 */
12014 	if (count >= dtrace_helper_actions_max)
12015 		return (ENOSPC);
12016 
12017 	helper = kmem_zalloc(sizeof (dtrace_helper_action_t), KM_SLEEP);
12018 	helper->dthp_generation = help->dthps_generation;
12019 
12020 	if ((pred = ep->dted_pred.dtpdd_predicate) != NULL) {
12021 		ASSERT(pred->dtp_difo != NULL);
12022 		dtrace_difo_hold(pred->dtp_difo);
12023 		helper->dthp_predicate = pred->dtp_difo;
12024 	}
12025 
12026 	for (act = ep->dted_action; act != NULL; act = act->dtad_next) {
12027 		if (act->dtad_kind != DTRACEACT_DIFEXPR)
12028 			goto err;
12029 
12030 		if (act->dtad_difo == NULL)
12031 			goto err;
12032 
12033 		nactions++;
12034 	}
12035 
12036 	helper->dthp_actions = kmem_zalloc(sizeof (dtrace_difo_t *) *
12037 	    (helper->dthp_nactions = nactions), KM_SLEEP);
12038 
12039 	for (act = ep->dted_action, i = 0; act != NULL; act = act->dtad_next) {
12040 		dtrace_difo_hold(act->dtad_difo);
12041 		helper->dthp_actions[i++] = act->dtad_difo;
12042 	}
12043 
12044 	if (!dtrace_helper_validate(helper))
12045 		goto err;
12046 
12047 	if (last == NULL) {
12048 		help->dthps_actions[which] = helper;
12049 	} else {
12050 		last->dthp_next = helper;
12051 	}
12052 
12053 	if (vstate->dtvs_nlocals > dtrace_helptrace_nlocals) {
12054 		dtrace_helptrace_nlocals = vstate->dtvs_nlocals;
12055 		dtrace_helptrace_next = 0;
12056 	}
12057 
12058 	return (0);
12059 err:
12060 	dtrace_helper_destroy(helper, vstate);
12061 	return (EINVAL);
12062 }
12063 
12064 static void
12065 dtrace_helper_provider_register(proc_t *p, dtrace_helpers_t *help,
12066     dof_helper_t *dofhp)
12067 {
12068 	ASSERT(MUTEX_NOT_HELD(&dtrace_lock));
12069 
12070 	mutex_enter(&dtrace_meta_lock);
12071 	mutex_enter(&dtrace_lock);
12072 
12073 	if (!dtrace_attached() || dtrace_meta_pid == NULL) {
12074 		/*
12075 		 * If the dtrace module is loaded but not attached, or if
12076 		 * there aren't isn't a meta provider registered to deal with
12077 		 * these provider descriptions, we need to postpone creating
12078 		 * the actual providers until later.
12079 		 */
12080 
12081 		if (help->dthps_next == NULL && help->dthps_prev == NULL &&
12082 		    dtrace_deferred_pid != help) {
12083 			help->dthps_pid = p->p_pid;
12084 			help->dthps_next = dtrace_deferred_pid;
12085 			help->dthps_prev = NULL;
12086 			if (dtrace_deferred_pid != NULL)
12087 				dtrace_deferred_pid->dthps_prev = help;
12088 			dtrace_deferred_pid = help;
12089 		}
12090 
12091 		mutex_exit(&dtrace_lock);
12092 
12093 	} else if (dofhp != NULL) {
12094 		/*
12095 		 * If the dtrace module is loaded and we have a particular
12096 		 * helper provider description, pass that off to the
12097 		 * meta provider.
12098 		 */
12099 
12100 		mutex_exit(&dtrace_lock);
12101 
12102 		dtrace_helper_provide(dofhp, p->p_pid);
12103 
12104 	} else {
12105 		/*
12106 		 * Otherwise, just pass all the helper provider descriptions
12107 		 * off to the meta provider.
12108 		 */
12109 
12110 		int i;
12111 		mutex_exit(&dtrace_lock);
12112 
12113 		for (i = 0; i < help->dthps_nprovs; i++) {
12114 			dtrace_helper_provide(&help->dthps_provs[i]->dthp_prov,
12115 			    p->p_pid);
12116 		}
12117 	}
12118 
12119 	mutex_exit(&dtrace_meta_lock);
12120 }
12121 
12122 static int
12123 dtrace_helper_provider_add(dof_helper_t *dofhp)
12124 {
12125 	dtrace_helpers_t *help;
12126 	dtrace_helper_provider_t *hprov, **tmp_provs;
12127 	uint_t tmp_nprovs, i;
12128 
12129 	help = curproc->p_dtrace_helpers;
12130 	ASSERT(help != NULL);
12131 
12132 	/*
12133 	 * If we already have dtrace_helper_providers_max helper providers,
12134 	 * we're refuse to add a new one.
12135 	 */
12136 	if (help->dthps_nprovs >= dtrace_helper_providers_max)
12137 		return (ENOSPC);
12138 
12139 	/*
12140 	 * Check to make sure this isn't a duplicate.
12141 	 */
12142 	for (i = 0; i < help->dthps_nprovs; i++) {
12143 		if (dofhp->dofhp_addr ==
12144 		    help->dthps_provs[i]->dthp_prov.dofhp_addr)
12145 			return (EALREADY);
12146 	}
12147 
12148 	hprov = kmem_zalloc(sizeof (dtrace_helper_provider_t), KM_SLEEP);
12149 	hprov->dthp_prov = *dofhp;
12150 	hprov->dthp_ref = 1;
12151 
12152 	tmp_nprovs = help->dthps_nprovs;
12153 	tmp_provs = help->dthps_provs;
12154 	help->dthps_nprovs++;
12155 	help->dthps_provs = kmem_zalloc(help->dthps_nprovs *
12156 	    sizeof (dtrace_helper_provider_t *), KM_SLEEP);
12157 
12158 	help->dthps_provs[tmp_nprovs] = hprov;
12159 	if (tmp_provs != NULL) {
12160 		bcopy(tmp_provs, help->dthps_provs, tmp_nprovs *
12161 		    sizeof (dtrace_helper_provider_t *));
12162 		kmem_free(tmp_provs, tmp_nprovs *
12163 		    sizeof (dtrace_helper_provider_t *));
12164 	}
12165 
12166 	return (0);
12167 }
12168 
12169 static void
12170 dtrace_helper_provider_remove(dtrace_helper_provider_t *hprov)
12171 {
12172 	mutex_enter(&dtrace_lock);
12173 
12174 	if (--hprov->dthp_ref == 0) {
12175 		dof_hdr_t *dof;
12176 		mutex_exit(&dtrace_lock);
12177 		dof = (dof_hdr_t *)(uintptr_t)hprov->dthp_prov.dofhp_dof;
12178 		dtrace_dof_destroy(dof);
12179 		kmem_free(hprov, sizeof (dtrace_helper_provider_t));
12180 	} else {
12181 		mutex_exit(&dtrace_lock);
12182 	}
12183 }
12184 
12185 static int
12186 dtrace_helper_provider_validate(dof_hdr_t *dof, dof_sec_t *sec)
12187 {
12188 	uintptr_t daddr = (uintptr_t)dof;
12189 	dof_sec_t *str_sec, *prb_sec, *arg_sec, *off_sec;
12190 	dof_provider_t *provider;
12191 	dof_probe_t *probe;
12192 	uint8_t *arg;
12193 	char *strtab, *typestr;
12194 	dof_stridx_t typeidx;
12195 	size_t typesz;
12196 	uint_t nprobes, j, k;
12197 
12198 	ASSERT(sec->dofs_type == DOF_SECT_PROVIDER);
12199 
12200 	if (sec->dofs_offset & (sizeof (uint_t) - 1)) {
12201 		dtrace_dof_error(dof, "misaligned section offset");
12202 		return (-1);
12203 	}
12204 
12205 	provider = (dof_provider_t *)(uintptr_t)(daddr + sec->dofs_offset);
12206 	str_sec = dtrace_dof_sect(dof, DOF_SECT_STRTAB, provider->dofpv_strtab);
12207 	prb_sec = dtrace_dof_sect(dof, DOF_SECT_PROBES, provider->dofpv_probes);
12208 	arg_sec = dtrace_dof_sect(dof, DOF_SECT_PRARGS, provider->dofpv_prargs);
12209 	off_sec = dtrace_dof_sect(dof, DOF_SECT_PROFFS, provider->dofpv_proffs);
12210 
12211 	if (str_sec == NULL || prb_sec == NULL ||
12212 	    arg_sec == NULL || off_sec == NULL)
12213 		return (-1);
12214 
12215 	strtab = (char *)(uintptr_t)(daddr + str_sec->dofs_offset);
12216 
12217 	if (provider->dofpv_name >= str_sec->dofs_size ||
12218 	    strlen(strtab + provider->dofpv_name) >= DTRACE_PROVNAMELEN) {
12219 		dtrace_dof_error(dof, "invalid provider name");
12220 		return (-1);
12221 	}
12222 
12223 	if (prb_sec->dofs_entsize == 0 ||
12224 	    prb_sec->dofs_entsize > prb_sec->dofs_size) {
12225 		dtrace_dof_error(dof, "invalid entry size");
12226 		return (-1);
12227 	}
12228 
12229 	if (prb_sec->dofs_entsize & (sizeof (uintptr_t) - 1)) {
12230 		dtrace_dof_error(dof, "misaligned entry size");
12231 		return (-1);
12232 	}
12233 
12234 	if (off_sec->dofs_entsize != sizeof (uint32_t)) {
12235 		dtrace_dof_error(dof, "invalid entry size");
12236 		return (-1);
12237 	}
12238 
12239 	if (off_sec->dofs_offset & (sizeof (uint32_t) - 1)) {
12240 		dtrace_dof_error(dof, "misaligned section offset");
12241 		return (-1);
12242 	}
12243 
12244 	if (arg_sec->dofs_entsize != sizeof (uint8_t)) {
12245 		dtrace_dof_error(dof, "invalid entry size");
12246 		return (-1);
12247 	}
12248 
12249 	arg = (uint8_t *)(uintptr_t)(daddr + arg_sec->dofs_offset);
12250 
12251 	nprobes = prb_sec->dofs_size / prb_sec->dofs_entsize;
12252 
12253 	/*
12254 	 * Take a pass through the probes to check for errors.
12255 	 */
12256 	for (j = 0; j < nprobes; j++) {
12257 		probe = (dof_probe_t *)(uintptr_t)(daddr +
12258 		    prb_sec->dofs_offset + j * prb_sec->dofs_entsize);
12259 
12260 		if (probe->dofpr_func >= str_sec->dofs_size) {
12261 			dtrace_dof_error(dof, "invalid function name");
12262 			return (-1);
12263 		}
12264 
12265 		if (strlen(strtab + probe->dofpr_func) >= DTRACE_FUNCNAMELEN) {
12266 			dtrace_dof_error(dof, "function name too long");
12267 			return (-1);
12268 		}
12269 
12270 		if (probe->dofpr_name >= str_sec->dofs_size ||
12271 		    strlen(strtab + probe->dofpr_name) >= DTRACE_NAMELEN) {
12272 			dtrace_dof_error(dof, "invalid probe name");
12273 			return (-1);
12274 		}
12275 
12276 
12277 		if (probe->dofpr_offidx + probe->dofpr_noffs <
12278 		    probe->dofpr_offidx ||
12279 		    (probe->dofpr_offidx + probe->dofpr_noffs) *
12280 		    off_sec->dofs_entsize > off_sec->dofs_size) {
12281 			dtrace_dof_error(dof, "invalid probe offset");
12282 			return (-1);
12283 		}
12284 
12285 		if (probe->dofpr_argidx + probe->dofpr_xargc <
12286 		    probe->dofpr_argidx ||
12287 		    (probe->dofpr_argidx + probe->dofpr_xargc) *
12288 		    arg_sec->dofs_entsize > arg_sec->dofs_size) {
12289 			dtrace_dof_error(dof, "invalid args");
12290 			return (-1);
12291 		}
12292 
12293 		typeidx = probe->dofpr_nargv;
12294 		typestr = strtab + probe->dofpr_nargv;
12295 		for (k = 0; k < probe->dofpr_nargc; k++) {
12296 			if (typeidx >= str_sec->dofs_size) {
12297 				dtrace_dof_error(dof, "bad "
12298 				    "native argument type");
12299 				return (-1);
12300 			}
12301 
12302 			typesz = strlen(typestr) + 1;
12303 			if (typesz > DTRACE_ARGTYPELEN) {
12304 				dtrace_dof_error(dof, "native "
12305 				    "argument type too long");
12306 				return (-1);
12307 			}
12308 			typeidx += typesz;
12309 			typestr += typesz;
12310 		}
12311 
12312 		typeidx = probe->dofpr_xargv;
12313 		typestr = strtab + probe->dofpr_xargv;
12314 		for (k = 0; k < probe->dofpr_xargc; k++) {
12315 			if (arg[probe->dofpr_argidx + k] > probe->dofpr_nargc) {
12316 				dtrace_dof_error(dof, "bad "
12317 				    "native argument index");
12318 				return (-1);
12319 			}
12320 
12321 			if (typeidx >= str_sec->dofs_size) {
12322 				dtrace_dof_error(dof, "bad "
12323 				    "translated argument type");
12324 				return (-1);
12325 			}
12326 
12327 			typesz = strlen(typestr) + 1;
12328 			if (typesz > DTRACE_ARGTYPELEN) {
12329 				dtrace_dof_error(dof, "translated argument "
12330 				    "type too long");
12331 				return (-1);
12332 			}
12333 
12334 			typeidx += typesz;
12335 			typestr += typesz;
12336 		}
12337 	}
12338 
12339 	return (0);
12340 }
12341 
12342 static int
12343 dtrace_helper_slurp(dof_hdr_t *dof, dof_helper_t *dhp)
12344 {
12345 	dtrace_helpers_t *help;
12346 	dtrace_vstate_t *vstate;
12347 	dtrace_enabling_t *enab = NULL;
12348 	int i, gen, rv, nhelpers = 0, destroy = 1;
12349 
12350 	ASSERT(MUTEX_HELD(&dtrace_lock));
12351 
12352 	if ((help = curproc->p_dtrace_helpers) == NULL)
12353 		help = dtrace_helpers_create(curproc);
12354 
12355 	vstate = &help->dthps_vstate;
12356 
12357 	if ((rv = dtrace_dof_slurp(dof, vstate, NULL, &enab,
12358 	    dhp != NULL ? dhp->dofhp_addr : 0, B_FALSE)) != 0) {
12359 		dtrace_dof_destroy(dof);
12360 		return (rv);
12361 	}
12362 
12363 	/*
12364 	 * Now we need to walk through the ECB descriptions in the enabling.
12365 	 */
12366 	for (i = 0; i < enab->dten_ndesc; i++) {
12367 		dtrace_ecbdesc_t *ep = enab->dten_desc[i];
12368 		dtrace_probedesc_t *desc = &ep->dted_probe;
12369 
12370 		if (strcmp(desc->dtpd_provider, "dtrace") != 0)
12371 			continue;
12372 
12373 		if (strcmp(desc->dtpd_mod, "helper") != 0)
12374 			continue;
12375 
12376 		if (strcmp(desc->dtpd_func, "ustack") != 0)
12377 			continue;
12378 
12379 		if ((rv = dtrace_helper_action_add(DTRACE_HELPER_ACTION_USTACK,
12380 		    ep)) != 0) {
12381 			/*
12382 			 * Adding this helper action failed -- we are now going
12383 			 * to rip out the entire generation and return failure.
12384 			 */
12385 			(void) dtrace_helper_destroygen(help->dthps_generation);
12386 			dtrace_enabling_destroy(enab);
12387 			dtrace_dof_destroy(dof);
12388 			dtrace_err = rv;
12389 			return (-1);
12390 		}
12391 
12392 		nhelpers++;
12393 	}
12394 
12395 	if (nhelpers < enab->dten_ndesc)
12396 		dtrace_dof_error(dof, "unmatched helpers");
12397 
12398 	if (dhp != NULL) {
12399 		uintptr_t daddr = (uintptr_t)dof;
12400 		int err = 0;
12401 
12402 		/*
12403 		 * Look for helper probes.
12404 		 */
12405 		for (i = 0; i < dof->dofh_secnum; i++) {
12406 			dof_sec_t *sec = (dof_sec_t *)(uintptr_t)(daddr +
12407 			    dof->dofh_secoff + i * dof->dofh_secsize);
12408 
12409 			if (sec->dofs_type != DOF_SECT_PROVIDER)
12410 				continue;
12411 
12412 			if (dtrace_helper_provider_validate(dof, sec) != 0) {
12413 				err = 1;
12414 				break;
12415 			}
12416 		}
12417 
12418 		dhp->dofhp_dof = (uint64_t)(uintptr_t)dof;
12419 		if (err == 0 && dtrace_helper_provider_add(dhp) == 0)
12420 			destroy = 0;
12421 		else
12422 			dhp = NULL;
12423 	}
12424 
12425 	gen = help->dthps_generation++;
12426 	dtrace_enabling_destroy(enab);
12427 
12428 	if (dhp != NULL) {
12429 		mutex_exit(&dtrace_lock);
12430 		dtrace_helper_provider_register(curproc, help, dhp);
12431 		mutex_enter(&dtrace_lock);
12432 	}
12433 
12434 	if (destroy)
12435 		dtrace_dof_destroy(dof);
12436 
12437 	return (gen);
12438 }
12439 
12440 static dtrace_helpers_t *
12441 dtrace_helpers_create(proc_t *p)
12442 {
12443 	dtrace_helpers_t *help;
12444 
12445 	ASSERT(MUTEX_HELD(&dtrace_lock));
12446 	ASSERT(p->p_dtrace_helpers == NULL);
12447 
12448 	help = kmem_zalloc(sizeof (dtrace_helpers_t), KM_SLEEP);
12449 	help->dthps_actions = kmem_zalloc(sizeof (dtrace_helper_action_t *) *
12450 	    DTRACE_NHELPER_ACTIONS, KM_SLEEP);
12451 
12452 	p->p_dtrace_helpers = help;
12453 	dtrace_helpers++;
12454 
12455 	return (help);
12456 }
12457 
12458 static void
12459 dtrace_helpers_destroy(void)
12460 {
12461 	dtrace_helpers_t *help;
12462 	dtrace_vstate_t *vstate;
12463 	proc_t *p = curproc;
12464 	int i;
12465 
12466 	mutex_enter(&dtrace_lock);
12467 
12468 	ASSERT(p->p_dtrace_helpers != NULL);
12469 	ASSERT(dtrace_helpers > 0);
12470 
12471 	help = p->p_dtrace_helpers;
12472 	vstate = &help->dthps_vstate;
12473 
12474 	/*
12475 	 * We're now going to lose the help from this process.
12476 	 */
12477 	p->p_dtrace_helpers = NULL;
12478 	dtrace_sync();
12479 
12480 	/*
12481 	 * Destory the helper actions.
12482 	 */
12483 	for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
12484 		dtrace_helper_action_t *h, *next;
12485 
12486 		for (h = help->dthps_actions[i]; h != NULL; h = next) {
12487 			next = h->dthp_next;
12488 			dtrace_helper_destroy(h, vstate);
12489 			h = next;
12490 		}
12491 	}
12492 
12493 	mutex_exit(&dtrace_lock);
12494 
12495 	/*
12496 	 * Destroy the helper providers.
12497 	 */
12498 	if (help->dthps_nprovs > 0) {
12499 		mutex_enter(&dtrace_meta_lock);
12500 		if (dtrace_meta_pid != NULL) {
12501 			ASSERT(dtrace_deferred_pid == NULL);
12502 
12503 			for (i = 0; i < help->dthps_nprovs; i++) {
12504 				dtrace_helper_remove(
12505 				    &help->dthps_provs[i]->dthp_prov, p->p_pid);
12506 			}
12507 		} else {
12508 			mutex_enter(&dtrace_lock);
12509 			ASSERT(dtrace_deferred_pid != NULL);
12510 
12511 			/*
12512 			 * Remove the helper from the deferred list.
12513 			 */
12514 			if (help->dthps_next != NULL)
12515 				help->dthps_next->dthps_prev = help->dthps_prev;
12516 			if (help->dthps_prev != NULL)
12517 				help->dthps_prev->dthps_next = help->dthps_next;
12518 			if (dtrace_deferred_pid == help) {
12519 				dtrace_deferred_pid = help->dthps_next;
12520 				ASSERT(help->dthps_prev == NULL);
12521 			}
12522 
12523 			mutex_exit(&dtrace_lock);
12524 		}
12525 
12526 		mutex_exit(&dtrace_meta_lock);
12527 
12528 		for (i = 0; i < help->dthps_nprovs; i++) {
12529 			dtrace_helper_provider_remove(help->dthps_provs[i]);
12530 		}
12531 
12532 		kmem_free(help->dthps_provs, help->dthps_nprovs *
12533 		    sizeof (dtrace_helper_provider_t *));
12534 	}
12535 
12536 	mutex_enter(&dtrace_lock);
12537 
12538 	dtrace_vstate_fini(&help->dthps_vstate);
12539 	kmem_free(help->dthps_actions,
12540 	    sizeof (dtrace_helper_action_t *) * DTRACE_NHELPER_ACTIONS);
12541 	kmem_free(help, sizeof (dtrace_helpers_t));
12542 
12543 	--dtrace_helpers;
12544 	mutex_exit(&dtrace_lock);
12545 }
12546 
12547 static void
12548 dtrace_helpers_duplicate(proc_t *from, proc_t *to)
12549 {
12550 	dtrace_helpers_t *help, *newhelp;
12551 	dtrace_helper_action_t *helper, *new, *last;
12552 	dtrace_difo_t *dp;
12553 	dtrace_vstate_t *vstate;
12554 	int i, j, sz, hasprovs = 0;
12555 
12556 	mutex_enter(&dtrace_lock);
12557 	ASSERT(from->p_dtrace_helpers != NULL);
12558 	ASSERT(dtrace_helpers > 0);
12559 
12560 	help = from->p_dtrace_helpers;
12561 	newhelp = dtrace_helpers_create(to);
12562 	ASSERT(to->p_dtrace_helpers != NULL);
12563 
12564 	newhelp->dthps_generation = help->dthps_generation;
12565 	vstate = &newhelp->dthps_vstate;
12566 
12567 	/*
12568 	 * Duplicate the helper actions.
12569 	 */
12570 	for (i = 0; i < DTRACE_NHELPER_ACTIONS; i++) {
12571 		if ((helper = help->dthps_actions[i]) == NULL)
12572 			continue;
12573 
12574 		for (last = NULL; helper != NULL; helper = helper->dthp_next) {
12575 			new = kmem_zalloc(sizeof (dtrace_helper_action_t),
12576 			    KM_SLEEP);
12577 			new->dthp_generation = helper->dthp_generation;
12578 
12579 			if ((dp = helper->dthp_predicate) != NULL) {
12580 				dp = dtrace_difo_duplicate(dp, vstate);
12581 				new->dthp_predicate = dp;
12582 			}
12583 
12584 			new->dthp_nactions = helper->dthp_nactions;
12585 			sz = sizeof (dtrace_difo_t *) * new->dthp_nactions;
12586 			new->dthp_actions = kmem_alloc(sz, KM_SLEEP);
12587 
12588 			for (j = 0; j < new->dthp_nactions; j++) {
12589 				dtrace_difo_t *dp = helper->dthp_actions[j];
12590 
12591 				ASSERT(dp != NULL);
12592 				dp = dtrace_difo_duplicate(dp, vstate);
12593 				new->dthp_actions[j] = dp;
12594 			}
12595 
12596 			if (last != NULL) {
12597 				last->dthp_next = new;
12598 			} else {
12599 				newhelp->dthps_actions[i] = new;
12600 			}
12601 
12602 			last = new;
12603 		}
12604 	}
12605 
12606 	/*
12607 	 * Duplicate the helper providers and register them with the
12608 	 * DTrace framework.
12609 	 */
12610 	if (help->dthps_nprovs > 0) {
12611 		newhelp->dthps_nprovs = help->dthps_nprovs;
12612 		newhelp->dthps_provs = kmem_alloc(newhelp->dthps_nprovs *
12613 		    sizeof (dtrace_helper_provider_t *), KM_SLEEP);
12614 		for (i = 0; i < newhelp->dthps_nprovs; i++) {
12615 			newhelp->dthps_provs[i] = help->dthps_provs[i];
12616 			newhelp->dthps_provs[i]->dthp_ref++;
12617 		}
12618 
12619 		hasprovs = 1;
12620 	}
12621 
12622 	mutex_exit(&dtrace_lock);
12623 
12624 	if (hasprovs)
12625 		dtrace_helper_provider_register(to, newhelp, NULL);
12626 }
12627 
12628 /*
12629  * DTrace Hook Functions
12630  */
12631 static void
12632 dtrace_module_loaded(struct modctl *ctl)
12633 {
12634 	dtrace_provider_t *prv;
12635 
12636 	mutex_enter(&dtrace_provider_lock);
12637 	mutex_enter(&mod_lock);
12638 
12639 	ASSERT(ctl->mod_busy);
12640 
12641 	/*
12642 	 * We're going to call each providers per-module provide operation
12643 	 * specifying only this module.
12644 	 */
12645 	for (prv = dtrace_provider; prv != NULL; prv = prv->dtpv_next)
12646 		prv->dtpv_pops.dtps_provide_module(prv->dtpv_arg, ctl);
12647 
12648 	mutex_exit(&mod_lock);
12649 	mutex_exit(&dtrace_provider_lock);
12650 
12651 	/*
12652 	 * If we have any retained enablings, we need to match against them.
12653 	 * Enabling probes requires that cpu_lock be held, and we cannot hold
12654 	 * cpu_lock here -- it is legal for cpu_lock to be held when loading a
12655 	 * module.  (In particular, this happens when loading scheduling
12656 	 * classes.)  So if we have any retained enablings, we need to dispatch
12657 	 * our task queue to do the match for us.
12658 	 */
12659 	mutex_enter(&dtrace_lock);
12660 
12661 	if (dtrace_retained == NULL) {
12662 		mutex_exit(&dtrace_lock);
12663 		return;
12664 	}
12665 
12666 	(void) taskq_dispatch(dtrace_taskq,
12667 	    (task_func_t *)dtrace_enabling_matchall, NULL, TQ_SLEEP);
12668 
12669 	mutex_exit(&dtrace_lock);
12670 
12671 	/*
12672 	 * And now, for a little heuristic sleaze:  in general, we want to
12673 	 * match modules as soon as they load.  However, we cannot guarantee
12674 	 * this, because it would lead us to the lock ordering violation
12675 	 * outlined above.  The common case, of course, is that cpu_lock is
12676 	 * _not_ held -- so we delay here for a clock tick, hoping that that's
12677 	 * long enough for the task queue to do its work.  If it's not, it's
12678 	 * not a serious problem -- it just means that the module that we
12679 	 * just loaded may not be immediately instrumentable.
12680 	 */
12681 	delay(1);
12682 }
12683 
12684 static void
12685 dtrace_module_unloaded(struct modctl *ctl)
12686 {
12687 	dtrace_probe_t template, *probe, *first, *next;
12688 	dtrace_provider_t *prov;
12689 
12690 	template.dtpr_mod = ctl->mod_modname;
12691 
12692 	mutex_enter(&dtrace_provider_lock);
12693 	mutex_enter(&mod_lock);
12694 	mutex_enter(&dtrace_lock);
12695 
12696 	if (dtrace_bymod == NULL) {
12697 		/*
12698 		 * The DTrace module is loaded (obviously) but not attached;
12699 		 * we don't have any work to do.
12700 		 */
12701 		mutex_exit(&dtrace_provider_lock);
12702 		mutex_exit(&mod_lock);
12703 		mutex_exit(&dtrace_lock);
12704 		return;
12705 	}
12706 
12707 	for (probe = first = dtrace_hash_lookup(dtrace_bymod, &template);
12708 	    probe != NULL; probe = probe->dtpr_nextmod) {
12709 		if (probe->dtpr_ecb != NULL) {
12710 			mutex_exit(&dtrace_provider_lock);
12711 			mutex_exit(&mod_lock);
12712 			mutex_exit(&dtrace_lock);
12713 
12714 			/*
12715 			 * This shouldn't _actually_ be possible -- we're
12716 			 * unloading a module that has an enabled probe in it.
12717 			 * (It's normally up to the provider to make sure that
12718 			 * this can't happen.)  However, because dtps_enable()
12719 			 * doesn't have a failure mode, there can be an
12720 			 * enable/unload race.  Upshot:  we don't want to
12721 			 * assert, but we're not going to disable the
12722 			 * probe, either.
12723 			 */
12724 			if (dtrace_err_verbose) {
12725 				cmn_err(CE_WARN, "unloaded module '%s' had "
12726 				    "enabled probes", ctl->mod_modname);
12727 			}
12728 
12729 			return;
12730 		}
12731 	}
12732 
12733 	probe = first;
12734 
12735 	for (first = NULL; probe != NULL; probe = next) {
12736 		ASSERT(dtrace_probes[probe->dtpr_id - 1] == probe);
12737 
12738 		dtrace_probes[probe->dtpr_id - 1] = NULL;
12739 
12740 		next = probe->dtpr_nextmod;
12741 		dtrace_hash_remove(dtrace_bymod, probe);
12742 		dtrace_hash_remove(dtrace_byfunc, probe);
12743 		dtrace_hash_remove(dtrace_byname, probe);
12744 
12745 		if (first == NULL) {
12746 			first = probe;
12747 			probe->dtpr_nextmod = NULL;
12748 		} else {
12749 			probe->dtpr_nextmod = first;
12750 			first = probe;
12751 		}
12752 	}
12753 
12754 	/*
12755 	 * We've removed all of the module's probes from the hash chains and
12756 	 * from the probe array.  Now issue a dtrace_sync() to be sure that
12757 	 * everyone has cleared out from any probe array processing.
12758 	 */
12759 	dtrace_sync();
12760 
12761 	for (probe = first; probe != NULL; probe = first) {
12762 		first = probe->dtpr_nextmod;
12763 		prov = probe->dtpr_provider;
12764 		prov->dtpv_pops.dtps_destroy(prov->dtpv_arg, probe->dtpr_id,
12765 		    probe->dtpr_arg);
12766 		kmem_free(probe->dtpr_mod, strlen(probe->dtpr_mod) + 1);
12767 		kmem_free(probe->dtpr_func, strlen(probe->dtpr_func) + 1);
12768 		kmem_free(probe->dtpr_name, strlen(probe->dtpr_name) + 1);
12769 		vmem_free(dtrace_arena, (void *)(uintptr_t)probe->dtpr_id, 1);
12770 		kmem_free(probe, sizeof (dtrace_probe_t));
12771 	}
12772 
12773 	mutex_exit(&dtrace_lock);
12774 	mutex_exit(&mod_lock);
12775 	mutex_exit(&dtrace_provider_lock);
12776 }
12777 
12778 void
12779 dtrace_suspend(void)
12780 {
12781 	dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_suspend));
12782 }
12783 
12784 void
12785 dtrace_resume(void)
12786 {
12787 	dtrace_probe_foreach(offsetof(dtrace_pops_t, dtps_resume));
12788 }
12789 
12790 static int
12791 dtrace_cpu_setup(cpu_setup_t what, processorid_t cpu)
12792 {
12793 	ASSERT(MUTEX_HELD(&cpu_lock));
12794 	mutex_enter(&dtrace_lock);
12795 
12796 	switch (what) {
12797 	case CPU_CONFIG: {
12798 		dtrace_state_t *state;
12799 		dtrace_optval_t *opt, rs, c;
12800 
12801 		/*
12802 		 * For now, we only allocate a new buffer for anonymous state.
12803 		 */
12804 		if ((state = dtrace_anon.dta_state) == NULL)
12805 			break;
12806 
12807 		if (state->dts_activity != DTRACE_ACTIVITY_ACTIVE)
12808 			break;
12809 
12810 		opt = state->dts_options;
12811 		c = opt[DTRACEOPT_CPU];
12812 
12813 		if (c != DTRACE_CPUALL && c != DTRACEOPT_UNSET && c != cpu)
12814 			break;
12815 
12816 		/*
12817 		 * Regardless of what the actual policy is, we're going to
12818 		 * temporarily set our resize policy to be manual.  We're
12819 		 * also going to temporarily set our CPU option to denote
12820 		 * the newly configured CPU.
12821 		 */
12822 		rs = opt[DTRACEOPT_BUFRESIZE];
12823 		opt[DTRACEOPT_BUFRESIZE] = DTRACEOPT_BUFRESIZE_MANUAL;
12824 		opt[DTRACEOPT_CPU] = (dtrace_optval_t)cpu;
12825 
12826 		(void) dtrace_state_buffers(state);
12827 
12828 		opt[DTRACEOPT_BUFRESIZE] = rs;
12829 		opt[DTRACEOPT_CPU] = c;
12830 
12831 		break;
12832 	}
12833 
12834 	case CPU_UNCONFIG:
12835 		/*
12836 		 * We don't free the buffer in the CPU_UNCONFIG case.  (The
12837 		 * buffer will be freed when the consumer exits.)
12838 		 */
12839 		break;
12840 
12841 	default:
12842 		break;
12843 	}
12844 
12845 	mutex_exit(&dtrace_lock);
12846 	return (0);
12847 }
12848 
12849 static void
12850 dtrace_cpu_setup_initial(processorid_t cpu)
12851 {
12852 	(void) dtrace_cpu_setup(CPU_CONFIG, cpu);
12853 }
12854 
12855 static void
12856 dtrace_toxrange_add(uintptr_t base, uintptr_t limit)
12857 {
12858 	if (dtrace_toxranges >= dtrace_toxranges_max) {
12859 		int osize, nsize;
12860 		dtrace_toxrange_t *range;
12861 
12862 		osize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
12863 
12864 		if (osize == 0) {
12865 			ASSERT(dtrace_toxrange == NULL);
12866 			ASSERT(dtrace_toxranges_max == 0);
12867 			dtrace_toxranges_max = 1;
12868 		} else {
12869 			dtrace_toxranges_max <<= 1;
12870 		}
12871 
12872 		nsize = dtrace_toxranges_max * sizeof (dtrace_toxrange_t);
12873 		range = kmem_zalloc(nsize, KM_SLEEP);
12874 
12875 		if (dtrace_toxrange != NULL) {
12876 			ASSERT(osize != 0);
12877 			bcopy(dtrace_toxrange, range, osize);
12878 			kmem_free(dtrace_toxrange, osize);
12879 		}
12880 
12881 		dtrace_toxrange = range;
12882 	}
12883 
12884 	ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_base == NULL);
12885 	ASSERT(dtrace_toxrange[dtrace_toxranges].dtt_limit == NULL);
12886 
12887 	dtrace_toxrange[dtrace_toxranges].dtt_base = base;
12888 	dtrace_toxrange[dtrace_toxranges].dtt_limit = limit;
12889 	dtrace_toxranges++;
12890 }
12891 
12892 /*
12893  * DTrace Driver Cookbook Functions
12894  */
12895 /*ARGSUSED*/
12896 static int
12897 dtrace_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
12898 {
12899 	dtrace_provider_id_t id;
12900 	dtrace_state_t *state = NULL;
12901 	dtrace_enabling_t *enab;
12902 
12903 	mutex_enter(&cpu_lock);
12904 	mutex_enter(&dtrace_provider_lock);
12905 	mutex_enter(&dtrace_lock);
12906 
12907 	if (ddi_soft_state_init(&dtrace_softstate, sizeof (dtrace_state_t) +
12908 	    NCPU * sizeof (dtrace_buffer_t), 0) != 0) {
12909 		cmn_err(CE_NOTE, "/dev/dtrace failed to initialize soft state");
12910 		mutex_exit(&cpu_lock);
12911 		mutex_exit(&dtrace_provider_lock);
12912 		mutex_exit(&dtrace_lock);
12913 		return (DDI_FAILURE);
12914 	}
12915 
12916 	if (ddi_create_minor_node(devi, DTRACEMNR_DTRACE, S_IFCHR,
12917 	    DTRACEMNRN_DTRACE, DDI_PSEUDO, NULL) == DDI_FAILURE ||
12918 	    ddi_create_minor_node(devi, DTRACEMNR_HELPER, S_IFCHR,
12919 	    DTRACEMNRN_HELPER, DDI_PSEUDO, NULL) == DDI_FAILURE) {
12920 		cmn_err(CE_NOTE, "/dev/dtrace couldn't create minor nodes");
12921 		ddi_remove_minor_node(devi, NULL);
12922 		ddi_soft_state_fini(&dtrace_softstate);
12923 		mutex_exit(&cpu_lock);
12924 		mutex_exit(&dtrace_provider_lock);
12925 		mutex_exit(&dtrace_lock);
12926 		return (DDI_FAILURE);
12927 	}
12928 
12929 	ddi_report_dev(devi);
12930 	dtrace_devi = devi;
12931 
12932 	dtrace_modload = dtrace_module_loaded;
12933 	dtrace_modunload = dtrace_module_unloaded;
12934 	dtrace_cpu_init = dtrace_cpu_setup_initial;
12935 	dtrace_helpers_cleanup = dtrace_helpers_destroy;
12936 	dtrace_helpers_fork = dtrace_helpers_duplicate;
12937 	dtrace_cpustart_init = dtrace_suspend;
12938 	dtrace_cpustart_fini = dtrace_resume;
12939 	dtrace_debugger_init = dtrace_suspend;
12940 	dtrace_debugger_fini = dtrace_resume;
12941 	dtrace_kreloc_init = dtrace_suspend;
12942 	dtrace_kreloc_fini = dtrace_resume;
12943 
12944 	register_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
12945 
12946 	ASSERT(MUTEX_HELD(&cpu_lock));
12947 
12948 	dtrace_arena = vmem_create("dtrace", (void *)1, UINT32_MAX, 1,
12949 	    NULL, NULL, NULL, 0, VM_SLEEP | VMC_IDENTIFIER);
12950 	dtrace_minor = vmem_create("dtrace_minor", (void *)DTRACEMNRN_CLONE,
12951 	    UINT32_MAX - DTRACEMNRN_CLONE, 1, NULL, NULL, NULL, 0,
12952 	    VM_SLEEP | VMC_IDENTIFIER);
12953 	dtrace_taskq = taskq_create("dtrace_taskq", 1, maxclsyspri,
12954 	    1, INT_MAX, 0);
12955 
12956 	dtrace_state_cache = kmem_cache_create("dtrace_state_cache",
12957 	    sizeof (dtrace_dstate_percpu_t) * NCPU, DTRACE_STATE_ALIGN,
12958 	    NULL, NULL, NULL, NULL, NULL, 0);
12959 
12960 	ASSERT(MUTEX_HELD(&cpu_lock));
12961 	dtrace_bymod = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_mod),
12962 	    offsetof(dtrace_probe_t, dtpr_nextmod),
12963 	    offsetof(dtrace_probe_t, dtpr_prevmod));
12964 
12965 	dtrace_byfunc = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_func),
12966 	    offsetof(dtrace_probe_t, dtpr_nextfunc),
12967 	    offsetof(dtrace_probe_t, dtpr_prevfunc));
12968 
12969 	dtrace_byname = dtrace_hash_create(offsetof(dtrace_probe_t, dtpr_name),
12970 	    offsetof(dtrace_probe_t, dtpr_nextname),
12971 	    offsetof(dtrace_probe_t, dtpr_prevname));
12972 
12973 	if (dtrace_retain_max < 1) {
12974 		cmn_err(CE_WARN, "illegal value (%lu) for dtrace_retain_max; "
12975 		    "setting to 1", dtrace_retain_max);
12976 		dtrace_retain_max = 1;
12977 	}
12978 
12979 	/*
12980 	 * Now discover our toxic ranges.
12981 	 */
12982 	dtrace_toxic_ranges(dtrace_toxrange_add);
12983 
12984 	/*
12985 	 * Before we register ourselves as a provider to our own framework,
12986 	 * we would like to assert that dtrace_provider is NULL -- but that's
12987 	 * not true if we were loaded as a dependency of a DTrace provider.
12988 	 * Once we've registered, we can assert that dtrace_provider is our
12989 	 * pseudo provider.
12990 	 */
12991 	(void) dtrace_register("dtrace", &dtrace_provider_attr,
12992 	    DTRACE_PRIV_NONE, 0, &dtrace_provider_ops, NULL, &id);
12993 
12994 	ASSERT(dtrace_provider != NULL);
12995 	ASSERT((dtrace_provider_id_t)dtrace_provider == id);
12996 
12997 	dtrace_probeid_begin = dtrace_probe_create((dtrace_provider_id_t)
12998 	    dtrace_provider, NULL, NULL, "BEGIN", 0, NULL);
12999 	dtrace_probeid_end = dtrace_probe_create((dtrace_provider_id_t)
13000 	    dtrace_provider, NULL, NULL, "END", 0, NULL);
13001 	dtrace_probeid_error = dtrace_probe_create((dtrace_provider_id_t)
13002 	    dtrace_provider, NULL, NULL, "ERROR", 1, NULL);
13003 
13004 	dtrace_anon_property();
13005 	mutex_exit(&cpu_lock);
13006 
13007 	/*
13008 	 * If DTrace helper tracing is enabled, we need to allocate the
13009 	 * trace buffer and initialize the values.
13010 	 */
13011 	if (dtrace_helptrace_enabled) {
13012 		ASSERT(dtrace_helptrace_buffer == NULL);
13013 		dtrace_helptrace_buffer =
13014 		    kmem_zalloc(dtrace_helptrace_bufsize, KM_SLEEP);
13015 		dtrace_helptrace_next = 0;
13016 	}
13017 
13018 	/*
13019 	 * If there are already providers, we must ask them to provide their
13020 	 * probes, and then match any anonymous enabling against them.  Note
13021 	 * that there should be no other retained enablings at this time:
13022 	 * the only retained enablings at this time should be the anonymous
13023 	 * enabling.
13024 	 */
13025 	if (dtrace_anon.dta_enabling != NULL) {
13026 		ASSERT(dtrace_retained == dtrace_anon.dta_enabling);
13027 
13028 		dtrace_enabling_provide(NULL);
13029 		state = dtrace_anon.dta_state;
13030 
13031 		/*
13032 		 * We couldn't hold cpu_lock across the above call to
13033 		 * dtrace_enabling_provide(), but we must hold it to actually
13034 		 * enable the probes.  We have to drop all of our locks, pick
13035 		 * up cpu_lock, and regain our locks before matching the
13036 		 * retained anonymous enabling.
13037 		 */
13038 		mutex_exit(&dtrace_lock);
13039 		mutex_exit(&dtrace_provider_lock);
13040 
13041 		mutex_enter(&cpu_lock);
13042 		mutex_enter(&dtrace_provider_lock);
13043 		mutex_enter(&dtrace_lock);
13044 
13045 		if ((enab = dtrace_anon.dta_enabling) != NULL)
13046 			(void) dtrace_enabling_match(enab, NULL);
13047 
13048 		mutex_exit(&cpu_lock);
13049 	}
13050 
13051 	mutex_exit(&dtrace_lock);
13052 	mutex_exit(&dtrace_provider_lock);
13053 
13054 	if (state != NULL) {
13055 		/*
13056 		 * If we created any anonymous state, set it going now.
13057 		 */
13058 		(void) dtrace_state_go(state, &dtrace_anon.dta_beganon);
13059 	}
13060 
13061 	return (DDI_SUCCESS);
13062 }
13063 
13064 /*ARGSUSED*/
13065 static int
13066 dtrace_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
13067 {
13068 	dtrace_state_t *state;
13069 	uint32_t priv;
13070 	uid_t uid;
13071 
13072 	if (getminor(*devp) == DTRACEMNRN_HELPER)
13073 		return (0);
13074 
13075 	/*
13076 	 * If this wasn't an open with the "helper" minor, then it must be
13077 	 * the "dtrace" minor.
13078 	 */
13079 	ASSERT(getminor(*devp) == DTRACEMNRN_DTRACE);
13080 
13081 	/*
13082 	 * If no DTRACE_PRIV_* bits are set in the credential, then the
13083 	 * caller lacks sufficient permission to do anything with DTrace.
13084 	 */
13085 	dtrace_cred2priv(cred_p, &priv, &uid);
13086 	if (priv == DTRACE_PRIV_NONE)
13087 		return (EACCES);
13088 
13089 	/*
13090 	 * Ask all providers to provide all their probes.
13091 	 */
13092 	mutex_enter(&dtrace_provider_lock);
13093 	dtrace_probe_provide(NULL, NULL);
13094 	mutex_exit(&dtrace_provider_lock);
13095 
13096 	mutex_enter(&cpu_lock);
13097 	mutex_enter(&dtrace_lock);
13098 	dtrace_opens++;
13099 	dtrace_membar_producer();
13100 
13101 	/*
13102 	 * If the kernel debugger is active (that is, if the kernel debugger
13103 	 * modified text in some way), we won't allow the open.
13104 	 */
13105 	if (kdi_dtrace_set(KDI_DTSET_DTRACE_ACTIVATE) != 0) {
13106 		dtrace_opens--;
13107 		mutex_exit(&cpu_lock);
13108 		mutex_exit(&dtrace_lock);
13109 		return (EBUSY);
13110 	}
13111 
13112 	state = dtrace_state_create(devp, cred_p);
13113 	mutex_exit(&cpu_lock);
13114 
13115 	if (state == NULL) {
13116 		if (--dtrace_opens == 0)
13117 			(void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
13118 		mutex_exit(&dtrace_lock);
13119 		return (EAGAIN);
13120 	}
13121 
13122 	mutex_exit(&dtrace_lock);
13123 
13124 	return (0);
13125 }
13126 
13127 /*ARGSUSED*/
13128 static int
13129 dtrace_close(dev_t dev, int flag, int otyp, cred_t *cred_p)
13130 {
13131 	minor_t minor = getminor(dev);
13132 	dtrace_state_t *state;
13133 
13134 	if (minor == DTRACEMNRN_HELPER)
13135 		return (0);
13136 
13137 	state = ddi_get_soft_state(dtrace_softstate, minor);
13138 
13139 	mutex_enter(&cpu_lock);
13140 	mutex_enter(&dtrace_lock);
13141 
13142 	if (state->dts_anon) {
13143 		/*
13144 		 * There is anonymous state. Destroy that first.
13145 		 */
13146 		ASSERT(dtrace_anon.dta_state == NULL);
13147 		dtrace_state_destroy(state->dts_anon);
13148 	}
13149 
13150 	dtrace_state_destroy(state);
13151 	ASSERT(dtrace_opens > 0);
13152 	if (--dtrace_opens == 0)
13153 		(void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
13154 
13155 	mutex_exit(&dtrace_lock);
13156 	mutex_exit(&cpu_lock);
13157 
13158 	return (0);
13159 }
13160 
13161 /*ARGSUSED*/
13162 static int
13163 dtrace_ioctl_helper(int cmd, intptr_t arg, int *rv)
13164 {
13165 	int rval;
13166 	dof_helper_t help, *dhp = NULL;
13167 
13168 	switch (cmd) {
13169 	case DTRACEHIOC_ADDDOF:
13170 		if (copyin((void *)arg, &help, sizeof (help)) != 0) {
13171 			dtrace_dof_error(NULL, "failed to copyin DOF helper");
13172 			return (EFAULT);
13173 		}
13174 
13175 		dhp = &help;
13176 		arg = (intptr_t)help.dofhp_dof;
13177 		/*FALLTHROUGH*/
13178 
13179 	case DTRACEHIOC_ADD: {
13180 		dof_hdr_t *dof = dtrace_dof_copyin(arg, &rval);
13181 
13182 		if (dof == NULL)
13183 			return (rval);
13184 
13185 		mutex_enter(&dtrace_lock);
13186 		dtrace_err = 0;
13187 
13188 		/*
13189 		 * dtrace_helper_slurp() takes responsibility for the dof --
13190 		 * it may free it now or it may save it and free it later.
13191 		 */
13192 		if ((rval = dtrace_helper_slurp(dof, dhp)) != -1) {
13193 			*rv = rval;
13194 			rval = 0;
13195 		} else {
13196 			rval = EINVAL;
13197 		}
13198 
13199 		mutex_exit(&dtrace_lock);
13200 		return (rval);
13201 	}
13202 
13203 	case DTRACEHIOC_REMOVE: {
13204 		mutex_enter(&dtrace_lock);
13205 		rval = dtrace_helper_destroygen(arg);
13206 		mutex_exit(&dtrace_lock);
13207 
13208 		return (rval);
13209 	}
13210 
13211 	default:
13212 		break;
13213 	}
13214 
13215 	return (ENOTTY);
13216 }
13217 
13218 /*ARGSUSED*/
13219 static int
13220 dtrace_ioctl(dev_t dev, int cmd, intptr_t arg, int md, cred_t *cr, int *rv)
13221 {
13222 	minor_t minor = getminor(dev);
13223 	dtrace_state_t *state;
13224 	int rval;
13225 
13226 	if (minor == DTRACEMNRN_HELPER)
13227 		return (dtrace_ioctl_helper(cmd, arg, rv));
13228 
13229 	state = ddi_get_soft_state(dtrace_softstate, minor);
13230 
13231 	if (state->dts_anon) {
13232 		ASSERT(dtrace_anon.dta_state == NULL);
13233 		state = state->dts_anon;
13234 	}
13235 
13236 	switch (cmd) {
13237 	case DTRACEIOC_PROVIDER: {
13238 		dtrace_providerdesc_t pvd;
13239 		dtrace_provider_t *pvp;
13240 
13241 		if (copyin((void *)arg, &pvd, sizeof (pvd)) != 0)
13242 			return (EFAULT);
13243 
13244 		pvd.dtvd_name[DTRACE_PROVNAMELEN - 1] = '\0';
13245 		mutex_enter(&dtrace_provider_lock);
13246 
13247 		for (pvp = dtrace_provider; pvp != NULL; pvp = pvp->dtpv_next) {
13248 			if (strcmp(pvp->dtpv_name, pvd.dtvd_name) == 0)
13249 				break;
13250 		}
13251 
13252 		mutex_exit(&dtrace_provider_lock);
13253 
13254 		if (pvp == NULL)
13255 			return (ESRCH);
13256 
13257 		bcopy(&pvp->dtpv_priv, &pvd.dtvd_priv, sizeof (dtrace_ppriv_t));
13258 		bcopy(&pvp->dtpv_attr, &pvd.dtvd_attr, sizeof (dtrace_pattr_t));
13259 		if (copyout(&pvd, (void *)arg, sizeof (pvd)) != 0)
13260 			return (EFAULT);
13261 
13262 		return (0);
13263 	}
13264 
13265 	case DTRACEIOC_EPROBE: {
13266 		dtrace_eprobedesc_t epdesc;
13267 		dtrace_ecb_t *ecb;
13268 		dtrace_action_t *act;
13269 		void *buf;
13270 		size_t size;
13271 		uintptr_t dest;
13272 		int nrecs;
13273 
13274 		if (copyin((void *)arg, &epdesc, sizeof (epdesc)) != 0)
13275 			return (EFAULT);
13276 
13277 		mutex_enter(&dtrace_lock);
13278 
13279 		if ((ecb = dtrace_epid2ecb(state, epdesc.dtepd_epid)) == NULL) {
13280 			mutex_exit(&dtrace_lock);
13281 			return (EINVAL);
13282 		}
13283 
13284 		if (ecb->dte_probe == NULL) {
13285 			mutex_exit(&dtrace_lock);
13286 			return (EINVAL);
13287 		}
13288 
13289 		epdesc.dtepd_probeid = ecb->dte_probe->dtpr_id;
13290 		epdesc.dtepd_uarg = ecb->dte_uarg;
13291 		epdesc.dtepd_size = ecb->dte_size;
13292 
13293 		nrecs = epdesc.dtepd_nrecs;
13294 		epdesc.dtepd_nrecs = 0;
13295 		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
13296 			if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
13297 				continue;
13298 
13299 			epdesc.dtepd_nrecs++;
13300 		}
13301 
13302 		/*
13303 		 * Now that we have the size, we need to allocate a temporary
13304 		 * buffer in which to store the complete description.  We need
13305 		 * the temporary buffer to be able to drop dtrace_lock()
13306 		 * across the copyout(), below.
13307 		 */
13308 		size = sizeof (dtrace_eprobedesc_t) +
13309 		    (epdesc.dtepd_nrecs * sizeof (dtrace_recdesc_t));
13310 
13311 		buf = kmem_alloc(size, KM_SLEEP);
13312 		dest = (uintptr_t)buf;
13313 
13314 		bcopy(&epdesc, (void *)dest, sizeof (epdesc));
13315 		dest += offsetof(dtrace_eprobedesc_t, dtepd_rec[0]);
13316 
13317 		for (act = ecb->dte_action; act != NULL; act = act->dta_next) {
13318 			if (DTRACEACT_ISAGG(act->dta_kind) || act->dta_intuple)
13319 				continue;
13320 
13321 			if (nrecs-- == 0)
13322 				break;
13323 
13324 			bcopy(&act->dta_rec, (void *)dest,
13325 			    sizeof (dtrace_recdesc_t));
13326 			dest += sizeof (dtrace_recdesc_t);
13327 		}
13328 
13329 		mutex_exit(&dtrace_lock);
13330 
13331 		if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
13332 			kmem_free(buf, size);
13333 			return (EFAULT);
13334 		}
13335 
13336 		kmem_free(buf, size);
13337 		return (0);
13338 	}
13339 
13340 	case DTRACEIOC_AGGDESC: {
13341 		dtrace_aggdesc_t aggdesc;
13342 		dtrace_action_t *act;
13343 		dtrace_aggregation_t *agg;
13344 		int nrecs;
13345 		uint32_t offs;
13346 		dtrace_recdesc_t *lrec;
13347 		void *buf;
13348 		size_t size;
13349 		uintptr_t dest;
13350 
13351 		if (copyin((void *)arg, &aggdesc, sizeof (aggdesc)) != 0)
13352 			return (EFAULT);
13353 
13354 		mutex_enter(&dtrace_lock);
13355 
13356 		if ((agg = dtrace_aggid2agg(state, aggdesc.dtagd_id)) == NULL) {
13357 			mutex_exit(&dtrace_lock);
13358 			return (EINVAL);
13359 		}
13360 
13361 		aggdesc.dtagd_epid = agg->dtag_ecb->dte_epid;
13362 
13363 		nrecs = aggdesc.dtagd_nrecs;
13364 		aggdesc.dtagd_nrecs = 0;
13365 
13366 		offs = agg->dtag_base;
13367 		lrec = &agg->dtag_action.dta_rec;
13368 		aggdesc.dtagd_size = lrec->dtrd_offset + lrec->dtrd_size - offs;
13369 
13370 		for (act = agg->dtag_first; ; act = act->dta_next) {
13371 			ASSERT(act->dta_intuple ||
13372 			    DTRACEACT_ISAGG(act->dta_kind));
13373 			aggdesc.dtagd_nrecs++;
13374 
13375 			if (act == &agg->dtag_action)
13376 				break;
13377 		}
13378 
13379 		/*
13380 		 * Now that we have the size, we need to allocate a temporary
13381 		 * buffer in which to store the complete description.  We need
13382 		 * the temporary buffer to be able to drop dtrace_lock()
13383 		 * across the copyout(), below.
13384 		 */
13385 		size = sizeof (dtrace_aggdesc_t) +
13386 		    (aggdesc.dtagd_nrecs * sizeof (dtrace_recdesc_t));
13387 
13388 		buf = kmem_alloc(size, KM_SLEEP);
13389 		dest = (uintptr_t)buf;
13390 
13391 		bcopy(&aggdesc, (void *)dest, sizeof (aggdesc));
13392 		dest += offsetof(dtrace_aggdesc_t, dtagd_rec[0]);
13393 
13394 		for (act = agg->dtag_first; ; act = act->dta_next) {
13395 			dtrace_recdesc_t rec = act->dta_rec;
13396 
13397 			if (nrecs-- == 0)
13398 				break;
13399 
13400 			rec.dtrd_offset -= offs;
13401 			bcopy(&rec, (void *)dest, sizeof (rec));
13402 			dest += sizeof (dtrace_recdesc_t);
13403 
13404 			if (act == &agg->dtag_action)
13405 				break;
13406 		}
13407 
13408 		mutex_exit(&dtrace_lock);
13409 
13410 		if (copyout(buf, (void *)arg, dest - (uintptr_t)buf) != 0) {
13411 			kmem_free(buf, size);
13412 			return (EFAULT);
13413 		}
13414 
13415 		kmem_free(buf, size);
13416 		return (0);
13417 	}
13418 
13419 	case DTRACEIOC_ENABLE: {
13420 		dof_hdr_t *dof;
13421 		dtrace_enabling_t *enab = NULL;
13422 		dtrace_vstate_t *vstate;
13423 		int err = 0;
13424 
13425 		*rv = 0;
13426 
13427 		/*
13428 		 * If a NULL argument has been passed, we take this as our
13429 		 * cue to reevaluate our enablings.
13430 		 */
13431 		if (arg == NULL) {
13432 			mutex_enter(&cpu_lock);
13433 			mutex_enter(&dtrace_lock);
13434 			err = dtrace_enabling_matchstate(state, rv);
13435 			mutex_exit(&dtrace_lock);
13436 			mutex_exit(&cpu_lock);
13437 
13438 			return (err);
13439 		}
13440 
13441 		if ((dof = dtrace_dof_copyin(arg, &rval)) == NULL)
13442 			return (rval);
13443 
13444 		mutex_enter(&cpu_lock);
13445 		mutex_enter(&dtrace_lock);
13446 		vstate = &state->dts_vstate;
13447 
13448 		if (state->dts_activity != DTRACE_ACTIVITY_INACTIVE) {
13449 			mutex_exit(&dtrace_lock);
13450 			mutex_exit(&cpu_lock);
13451 			dtrace_dof_destroy(dof);
13452 			return (EBUSY);
13453 		}
13454 
13455 		if (dtrace_dof_slurp(dof, vstate, cr, &enab, 0, B_TRUE) != 0) {
13456 			mutex_exit(&dtrace_lock);
13457 			mutex_exit(&cpu_lock);
13458 			dtrace_dof_destroy(dof);
13459 			return (EINVAL);
13460 		}
13461 
13462 		if ((rval = dtrace_dof_options(dof, state)) != 0) {
13463 			dtrace_enabling_destroy(enab);
13464 			mutex_exit(&dtrace_lock);
13465 			mutex_exit(&cpu_lock);
13466 			dtrace_dof_destroy(dof);
13467 			return (rval);
13468 		}
13469 
13470 		if ((err = dtrace_enabling_match(enab, rv)) == 0) {
13471 			err = dtrace_enabling_retain(enab);
13472 		} else {
13473 			dtrace_enabling_destroy(enab);
13474 		}
13475 
13476 		mutex_exit(&cpu_lock);
13477 		mutex_exit(&dtrace_lock);
13478 		dtrace_dof_destroy(dof);
13479 
13480 		return (err);
13481 	}
13482 
13483 	case DTRACEIOC_REPLICATE: {
13484 		dtrace_repldesc_t desc;
13485 		dtrace_probedesc_t *match = &desc.dtrpd_match;
13486 		dtrace_probedesc_t *create = &desc.dtrpd_create;
13487 		int err;
13488 
13489 		if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
13490 			return (EFAULT);
13491 
13492 		match->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
13493 		match->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
13494 		match->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
13495 		match->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
13496 
13497 		create->dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
13498 		create->dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
13499 		create->dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
13500 		create->dtpd_name[DTRACE_NAMELEN - 1] = '\0';
13501 
13502 		mutex_enter(&dtrace_lock);
13503 		err = dtrace_enabling_replicate(state, match, create);
13504 		mutex_exit(&dtrace_lock);
13505 
13506 		return (err);
13507 	}
13508 
13509 	case DTRACEIOC_PROBEMATCH:
13510 	case DTRACEIOC_PROBES: {
13511 		dtrace_probe_t *probe = NULL;
13512 		dtrace_probedesc_t desc;
13513 		dtrace_probekey_t pkey;
13514 		dtrace_id_t i;
13515 		int m = 0;
13516 		uint32_t priv;
13517 		uid_t uid;
13518 
13519 		if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
13520 			return (EFAULT);
13521 
13522 		desc.dtpd_provider[DTRACE_PROVNAMELEN - 1] = '\0';
13523 		desc.dtpd_mod[DTRACE_MODNAMELEN - 1] = '\0';
13524 		desc.dtpd_func[DTRACE_FUNCNAMELEN - 1] = '\0';
13525 		desc.dtpd_name[DTRACE_NAMELEN - 1] = '\0';
13526 
13527 		/*
13528 		 * Before we attempt to match this probe, we want to give
13529 		 * all providers the opportunity to provide it.
13530 		 */
13531 		if (desc.dtpd_id == DTRACE_IDNONE) {
13532 			mutex_enter(&dtrace_provider_lock);
13533 			dtrace_probe_provide(&desc, NULL);
13534 			mutex_exit(&dtrace_provider_lock);
13535 			desc.dtpd_id++;
13536 		}
13537 
13538 		if (cmd == DTRACEIOC_PROBEMATCH)  {
13539 			dtrace_probekey(&desc, &pkey);
13540 			pkey.dtpk_id = DTRACE_IDNONE;
13541 		}
13542 
13543 		uid = crgetuid(cr);
13544 		dtrace_cred2priv(cr, &priv, &uid);
13545 
13546 		mutex_enter(&dtrace_lock);
13547 
13548 		if (cmd == DTRACEIOC_PROBEMATCH) {
13549 			for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
13550 				if ((probe = dtrace_probes[i - 1]) != NULL &&
13551 				    (m = dtrace_match_probe(probe, &pkey,
13552 				    priv, uid)) != 0)
13553 					break;
13554 			}
13555 
13556 			if (m < 0) {
13557 				mutex_exit(&dtrace_lock);
13558 				return (EINVAL);
13559 			}
13560 
13561 		} else {
13562 			for (i = desc.dtpd_id; i <= dtrace_nprobes; i++) {
13563 				if ((probe = dtrace_probes[i - 1]) != NULL &&
13564 				    dtrace_match_priv(probe, priv, uid))
13565 					break;
13566 			}
13567 		}
13568 
13569 		if (probe == NULL) {
13570 			mutex_exit(&dtrace_lock);
13571 			return (ESRCH);
13572 		}
13573 
13574 		dtrace_probe_description(probe, &desc);
13575 		mutex_exit(&dtrace_lock);
13576 
13577 		if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
13578 			return (EFAULT);
13579 
13580 		return (0);
13581 	}
13582 
13583 	case DTRACEIOC_PROBEARG: {
13584 		dtrace_argdesc_t desc;
13585 		dtrace_probe_t *probe;
13586 		dtrace_provider_t *prov;
13587 
13588 		if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
13589 			return (EFAULT);
13590 
13591 		if (desc.dtargd_id == DTRACE_IDNONE)
13592 			return (EINVAL);
13593 
13594 		if (desc.dtargd_ndx == DTRACE_ARGNONE)
13595 			return (EINVAL);
13596 
13597 		mutex_enter(&dtrace_provider_lock);
13598 		mutex_enter(&mod_lock);
13599 		mutex_enter(&dtrace_lock);
13600 
13601 		if (desc.dtargd_id > dtrace_nprobes) {
13602 			mutex_exit(&dtrace_lock);
13603 			mutex_exit(&mod_lock);
13604 			mutex_exit(&dtrace_provider_lock);
13605 			return (EINVAL);
13606 		}
13607 
13608 		if ((probe = dtrace_probes[desc.dtargd_id - 1]) == NULL) {
13609 			mutex_exit(&dtrace_lock);
13610 			mutex_exit(&mod_lock);
13611 			mutex_exit(&dtrace_provider_lock);
13612 			return (EINVAL);
13613 		}
13614 
13615 		mutex_exit(&dtrace_lock);
13616 
13617 		prov = probe->dtpr_provider;
13618 
13619 		if (prov->dtpv_pops.dtps_getargdesc == NULL) {
13620 			/*
13621 			 * There isn't any typed information for this probe.
13622 			 * Set the argument number to DTRACE_ARGNONE.
13623 			 */
13624 			desc.dtargd_ndx = DTRACE_ARGNONE;
13625 		} else {
13626 			desc.dtargd_native[0] = '\0';
13627 			desc.dtargd_xlate[0] = '\0';
13628 			desc.dtargd_mapping = desc.dtargd_ndx;
13629 
13630 			prov->dtpv_pops.dtps_getargdesc(prov->dtpv_arg,
13631 			    probe->dtpr_id, probe->dtpr_arg, &desc);
13632 		}
13633 
13634 		mutex_exit(&mod_lock);
13635 		mutex_exit(&dtrace_provider_lock);
13636 
13637 		if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
13638 			return (EFAULT);
13639 
13640 		return (0);
13641 	}
13642 
13643 	case DTRACEIOC_GO: {
13644 		processorid_t cpuid;
13645 		rval = dtrace_state_go(state, &cpuid);
13646 
13647 		if (rval != 0)
13648 			return (rval);
13649 
13650 		if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
13651 			return (EFAULT);
13652 
13653 		return (0);
13654 	}
13655 
13656 	case DTRACEIOC_STOP: {
13657 		processorid_t cpuid;
13658 
13659 		mutex_enter(&dtrace_lock);
13660 		rval = dtrace_state_stop(state, &cpuid);
13661 		mutex_exit(&dtrace_lock);
13662 
13663 		if (rval != 0)
13664 			return (rval);
13665 
13666 		if (copyout(&cpuid, (void *)arg, sizeof (cpuid)) != 0)
13667 			return (EFAULT);
13668 
13669 		return (0);
13670 	}
13671 
13672 	case DTRACEIOC_DOFGET: {
13673 		dof_hdr_t hdr, *dof;
13674 		uint64_t len;
13675 
13676 		if (copyin((void *)arg, &hdr, sizeof (hdr)) != 0)
13677 			return (EFAULT);
13678 
13679 		mutex_enter(&dtrace_lock);
13680 		dof = dtrace_dof_create(state);
13681 		mutex_exit(&dtrace_lock);
13682 
13683 		len = MIN(hdr.dofh_loadsz, dof->dofh_loadsz);
13684 		rval = copyout(dof, (void *)arg, len);
13685 		dtrace_dof_destroy(dof);
13686 
13687 		return (rval == 0 ? 0 : EFAULT);
13688 	}
13689 
13690 	case DTRACEIOC_AGGSNAP:
13691 	case DTRACEIOC_BUFSNAP: {
13692 		dtrace_bufdesc_t desc;
13693 		caddr_t cached;
13694 		dtrace_buffer_t *buf;
13695 
13696 		if (copyin((void *)arg, &desc, sizeof (desc)) != 0)
13697 			return (EFAULT);
13698 
13699 		if (desc.dtbd_cpu < 0 || desc.dtbd_cpu >= NCPU)
13700 			return (EINVAL);
13701 
13702 		mutex_enter(&dtrace_lock);
13703 
13704 		if (cmd == DTRACEIOC_BUFSNAP) {
13705 			buf = &state->dts_buffer[desc.dtbd_cpu];
13706 		} else {
13707 			buf = &state->dts_aggbuffer[desc.dtbd_cpu];
13708 		}
13709 
13710 		if (buf->dtb_flags & (DTRACEBUF_RING | DTRACEBUF_FILL)) {
13711 			size_t sz = buf->dtb_offset;
13712 
13713 			if (state->dts_activity != DTRACE_ACTIVITY_STOPPED) {
13714 				mutex_exit(&dtrace_lock);
13715 				return (EBUSY);
13716 			}
13717 
13718 			/*
13719 			 * If this buffer has already been consumed, we're
13720 			 * going to indicate that there's nothing left here
13721 			 * to consume.
13722 			 */
13723 			if (buf->dtb_flags & DTRACEBUF_CONSUMED) {
13724 				mutex_exit(&dtrace_lock);
13725 
13726 				desc.dtbd_size = 0;
13727 				desc.dtbd_drops = 0;
13728 				desc.dtbd_errors = 0;
13729 				desc.dtbd_oldest = 0;
13730 				sz = sizeof (desc);
13731 
13732 				if (copyout(&desc, (void *)arg, sz) != 0)
13733 					return (EFAULT);
13734 
13735 				return (0);
13736 			}
13737 
13738 			/*
13739 			 * If this is a ring buffer that has wrapped, we want
13740 			 * to copy the whole thing out.
13741 			 */
13742 			if (buf->dtb_flags & DTRACEBUF_WRAPPED) {
13743 				dtrace_buffer_polish(buf);
13744 				sz = buf->dtb_size;
13745 			}
13746 
13747 			if (copyout(buf->dtb_tomax, desc.dtbd_data, sz) != 0) {
13748 				mutex_exit(&dtrace_lock);
13749 				return (EFAULT);
13750 			}
13751 
13752 			desc.dtbd_size = sz;
13753 			desc.dtbd_drops = buf->dtb_drops;
13754 			desc.dtbd_errors = buf->dtb_errors;
13755 			desc.dtbd_oldest = buf->dtb_xamot_offset;
13756 
13757 			mutex_exit(&dtrace_lock);
13758 
13759 			if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
13760 				return (EFAULT);
13761 
13762 			buf->dtb_flags |= DTRACEBUF_CONSUMED;
13763 
13764 			return (0);
13765 		}
13766 
13767 		if (buf->dtb_tomax == NULL) {
13768 			ASSERT(buf->dtb_xamot == NULL);
13769 			mutex_exit(&dtrace_lock);
13770 			return (ENOENT);
13771 		}
13772 
13773 		cached = buf->dtb_tomax;
13774 		ASSERT(!(buf->dtb_flags & DTRACEBUF_NOSWITCH));
13775 
13776 		dtrace_xcall(desc.dtbd_cpu,
13777 		    (dtrace_xcall_t)dtrace_buffer_switch, buf);
13778 
13779 		state->dts_errors += buf->dtb_xamot_errors;
13780 
13781 		/*
13782 		 * If the buffers did not actually switch, then the cross call
13783 		 * did not take place -- presumably because the given CPU is
13784 		 * not in the ready set.  If this is the case, we'll return
13785 		 * ENOENT.
13786 		 */
13787 		if (buf->dtb_tomax == cached) {
13788 			ASSERT(buf->dtb_xamot != cached);
13789 			mutex_exit(&dtrace_lock);
13790 			return (ENOENT);
13791 		}
13792 
13793 		ASSERT(cached == buf->dtb_xamot);
13794 
13795 		/*
13796 		 * We have our snapshot; now copy it out.
13797 		 */
13798 		if (copyout(buf->dtb_xamot, desc.dtbd_data,
13799 		    buf->dtb_xamot_offset) != 0) {
13800 			mutex_exit(&dtrace_lock);
13801 			return (EFAULT);
13802 		}
13803 
13804 		desc.dtbd_size = buf->dtb_xamot_offset;
13805 		desc.dtbd_drops = buf->dtb_xamot_drops;
13806 		desc.dtbd_errors = buf->dtb_xamot_errors;
13807 		desc.dtbd_oldest = 0;
13808 
13809 		mutex_exit(&dtrace_lock);
13810 
13811 		/*
13812 		 * Finally, copy out the buffer description.
13813 		 */
13814 		if (copyout(&desc, (void *)arg, sizeof (desc)) != 0)
13815 			return (EFAULT);
13816 
13817 		return (0);
13818 	}
13819 
13820 	case DTRACEIOC_CONF: {
13821 		dtrace_conf_t conf;
13822 
13823 		bzero(&conf, sizeof (conf));
13824 		conf.dtc_difversion = DIF_VERSION;
13825 		conf.dtc_difintregs = DIF_DIR_NREGS;
13826 		conf.dtc_diftupregs = DIF_DTR_NREGS;
13827 		conf.dtc_ctfmodel = CTF_MODEL_NATIVE;
13828 
13829 		if (copyout(&conf, (void *)arg, sizeof (conf)) != 0)
13830 			return (EFAULT);
13831 
13832 		return (0);
13833 	}
13834 
13835 	case DTRACEIOC_STATUS: {
13836 		dtrace_status_t stat;
13837 		dtrace_dstate_t *dstate;
13838 		int i, j;
13839 		uint64_t nerrs;
13840 
13841 		/*
13842 		 * See the comment in dtrace_state_deadman() for the reason
13843 		 * for setting dts_laststatus to INT64_MAX before setting
13844 		 * it to the correct value.
13845 		 */
13846 		state->dts_laststatus = INT64_MAX;
13847 		dtrace_membar_producer();
13848 		state->dts_laststatus = dtrace_gethrtime();
13849 
13850 		bzero(&stat, sizeof (stat));
13851 
13852 		mutex_enter(&dtrace_lock);
13853 
13854 		if (state->dts_activity == DTRACE_ACTIVITY_INACTIVE) {
13855 			mutex_exit(&dtrace_lock);
13856 			return (ENOENT);
13857 		}
13858 
13859 		if (state->dts_activity == DTRACE_ACTIVITY_DRAINING)
13860 			stat.dtst_exiting = 1;
13861 
13862 		nerrs = state->dts_errors;
13863 		dstate = &state->dts_vstate.dtvs_dynvars;
13864 
13865 		for (i = 0; i < NCPU; i++) {
13866 			dtrace_dstate_percpu_t *dcpu = &dstate->dtds_percpu[i];
13867 
13868 			stat.dtst_dyndrops += dcpu->dtdsc_drops;
13869 			stat.dtst_dyndrops_dirty += dcpu->dtdsc_dirty_drops;
13870 			stat.dtst_dyndrops_rinsing += dcpu->dtdsc_rinsing_drops;
13871 
13872 			if (state->dts_buffer[i].dtb_flags & DTRACEBUF_FULL)
13873 				stat.dtst_filled++;
13874 
13875 			nerrs += state->dts_buffer[i].dtb_errors;
13876 
13877 			for (j = 0; j < state->dts_nspeculations; j++) {
13878 				dtrace_speculation_t *spec;
13879 				dtrace_buffer_t *buf;
13880 
13881 				spec = &state->dts_speculations[j];
13882 				buf = &spec->dtsp_buffer[i];
13883 				stat.dtst_specdrops += buf->dtb_xamot_drops;
13884 			}
13885 		}
13886 
13887 		stat.dtst_specdrops_busy = state->dts_speculations_busy;
13888 		stat.dtst_specdrops_unavail = state->dts_speculations_unavail;
13889 		stat.dtst_stkstroverflows = state->dts_stkstroverflows;
13890 		stat.dtst_dblerrors = state->dts_dblerrors;
13891 		stat.dtst_killed =
13892 		    (state->dts_activity == DTRACE_ACTIVITY_KILLED);
13893 		stat.dtst_errors = nerrs;
13894 
13895 		mutex_exit(&dtrace_lock);
13896 
13897 		if (copyout(&stat, (void *)arg, sizeof (stat)) != 0)
13898 			return (EFAULT);
13899 
13900 		return (0);
13901 	}
13902 
13903 	case DTRACEIOC_FORMAT: {
13904 		dtrace_fmtdesc_t fmt;
13905 		char *str;
13906 		int len;
13907 
13908 		if (copyin((void *)arg, &fmt, sizeof (fmt)) != 0)
13909 			return (EFAULT);
13910 
13911 		mutex_enter(&dtrace_lock);
13912 
13913 		if (fmt.dtfd_format == 0 ||
13914 		    fmt.dtfd_format > state->dts_nformats) {
13915 			mutex_exit(&dtrace_lock);
13916 			return (EINVAL);
13917 		}
13918 
13919 		/*
13920 		 * Format strings are allocated contiguously and they are
13921 		 * never freed; if a format index is less than the number
13922 		 * of formats, we can assert that the format map is non-NULL
13923 		 * and that the format for the specified index is non-NULL.
13924 		 */
13925 		ASSERT(state->dts_formats != NULL);
13926 		str = state->dts_formats[fmt.dtfd_format - 1];
13927 		ASSERT(str != NULL);
13928 
13929 		len = strlen(str) + 1;
13930 
13931 		if (len > fmt.dtfd_length) {
13932 			fmt.dtfd_length = len;
13933 
13934 			if (copyout(&fmt, (void *)arg, sizeof (fmt)) != 0) {
13935 				mutex_exit(&dtrace_lock);
13936 				return (EINVAL);
13937 			}
13938 		} else {
13939 			if (copyout(str, fmt.dtfd_string, len) != 0) {
13940 				mutex_exit(&dtrace_lock);
13941 				return (EINVAL);
13942 			}
13943 		}
13944 
13945 		mutex_exit(&dtrace_lock);
13946 		return (0);
13947 	}
13948 
13949 	default:
13950 		break;
13951 	}
13952 
13953 	return (ENOTTY);
13954 }
13955 
13956 /*ARGSUSED*/
13957 static int
13958 dtrace_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
13959 {
13960 	dtrace_state_t *state;
13961 
13962 	switch (cmd) {
13963 	case DDI_DETACH:
13964 		break;
13965 
13966 	case DDI_SUSPEND:
13967 		return (DDI_SUCCESS);
13968 
13969 	default:
13970 		return (DDI_FAILURE);
13971 	}
13972 
13973 	mutex_enter(&cpu_lock);
13974 	mutex_enter(&dtrace_provider_lock);
13975 	mutex_enter(&dtrace_lock);
13976 
13977 	ASSERT(dtrace_opens == 0);
13978 
13979 	if (dtrace_helpers > 0) {
13980 		mutex_exit(&dtrace_provider_lock);
13981 		mutex_exit(&dtrace_lock);
13982 		mutex_exit(&cpu_lock);
13983 		return (DDI_FAILURE);
13984 	}
13985 
13986 	if (dtrace_unregister((dtrace_provider_id_t)dtrace_provider) != 0) {
13987 		mutex_exit(&dtrace_provider_lock);
13988 		mutex_exit(&dtrace_lock);
13989 		mutex_exit(&cpu_lock);
13990 		return (DDI_FAILURE);
13991 	}
13992 
13993 	dtrace_provider = NULL;
13994 
13995 	if ((state = dtrace_anon_grab()) != NULL) {
13996 		/*
13997 		 * If there were ECBs on this state, the provider should
13998 		 * have not been allowed to detach; assert that there is
13999 		 * none.
14000 		 */
14001 		ASSERT(state->dts_necbs == 0);
14002 		dtrace_state_destroy(state);
14003 
14004 		/*
14005 		 * If we're being detached with anonymous state, we need to
14006 		 * indicate to the kernel debugger that DTrace is now inactive.
14007 		 */
14008 		(void) kdi_dtrace_set(KDI_DTSET_DTRACE_DEACTIVATE);
14009 	}
14010 
14011 	bzero(&dtrace_anon, sizeof (dtrace_anon_t));
14012 	unregister_cpu_setup_func((cpu_setup_func_t *)dtrace_cpu_setup, NULL);
14013 	dtrace_cpu_init = NULL;
14014 	dtrace_helpers_cleanup = NULL;
14015 	dtrace_helpers_fork = NULL;
14016 	dtrace_cpustart_init = NULL;
14017 	dtrace_cpustart_fini = NULL;
14018 	dtrace_debugger_init = NULL;
14019 	dtrace_debugger_fini = NULL;
14020 	dtrace_kreloc_init = NULL;
14021 	dtrace_kreloc_fini = NULL;
14022 	dtrace_modload = NULL;
14023 	dtrace_modunload = NULL;
14024 
14025 	mutex_exit(&cpu_lock);
14026 
14027 	if (dtrace_helptrace_enabled) {
14028 		kmem_free(dtrace_helptrace_buffer, dtrace_helptrace_bufsize);
14029 		dtrace_helptrace_buffer = NULL;
14030 	}
14031 
14032 	kmem_free(dtrace_probes, dtrace_nprobes * sizeof (dtrace_probe_t *));
14033 	dtrace_probes = NULL;
14034 	dtrace_nprobes = 0;
14035 
14036 	dtrace_hash_destroy(dtrace_bymod);
14037 	dtrace_hash_destroy(dtrace_byfunc);
14038 	dtrace_hash_destroy(dtrace_byname);
14039 	dtrace_bymod = NULL;
14040 	dtrace_byfunc = NULL;
14041 	dtrace_byname = NULL;
14042 
14043 	kmem_cache_destroy(dtrace_state_cache);
14044 	vmem_destroy(dtrace_minor);
14045 	vmem_destroy(dtrace_arena);
14046 
14047 	if (dtrace_toxrange != NULL) {
14048 		kmem_free(dtrace_toxrange,
14049 		    dtrace_toxranges_max * sizeof (dtrace_toxrange_t));
14050 		dtrace_toxrange = NULL;
14051 		dtrace_toxranges = 0;
14052 		dtrace_toxranges_max = 0;
14053 	}
14054 
14055 	ddi_remove_minor_node(dtrace_devi, NULL);
14056 	dtrace_devi = NULL;
14057 
14058 	ddi_soft_state_fini(&dtrace_softstate);
14059 
14060 	ASSERT(dtrace_vtime_references == 0);
14061 	ASSERT(dtrace_opens == 0);
14062 	ASSERT(dtrace_retained == NULL);
14063 
14064 	mutex_exit(&dtrace_lock);
14065 	mutex_exit(&dtrace_provider_lock);
14066 
14067 	/*
14068 	 * We don't destroy the task queue until after we have dropped our
14069 	 * locks (taskq_destroy() may block on running tasks).  To prevent
14070 	 * attempting to do work after we have effectively detached but before
14071 	 * the task queue has been destroyed, all tasks dispatched via the
14072 	 * task queue must check that DTrace is still attached before
14073 	 * performing any operation.
14074 	 */
14075 	taskq_destroy(dtrace_taskq);
14076 	dtrace_taskq = NULL;
14077 
14078 	return (DDI_SUCCESS);
14079 }
14080 
14081 /*ARGSUSED*/
14082 static int
14083 dtrace_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
14084 {
14085 	int error;
14086 
14087 	switch (infocmd) {
14088 	case DDI_INFO_DEVT2DEVINFO:
14089 		*result = (void *)dtrace_devi;
14090 		error = DDI_SUCCESS;
14091 		break;
14092 	case DDI_INFO_DEVT2INSTANCE:
14093 		*result = (void *)0;
14094 		error = DDI_SUCCESS;
14095 		break;
14096 	default:
14097 		error = DDI_FAILURE;
14098 	}
14099 	return (error);
14100 }
14101 
14102 static struct cb_ops dtrace_cb_ops = {
14103 	dtrace_open,		/* open */
14104 	dtrace_close,		/* close */
14105 	nulldev,		/* strategy */
14106 	nulldev,		/* print */
14107 	nodev,			/* dump */
14108 	nodev,			/* read */
14109 	nodev,			/* write */
14110 	dtrace_ioctl,		/* ioctl */
14111 	nodev,			/* devmap */
14112 	nodev,			/* mmap */
14113 	nodev,			/* segmap */
14114 	nochpoll,		/* poll */
14115 	ddi_prop_op,		/* cb_prop_op */
14116 	0,			/* streamtab  */
14117 	D_NEW | D_MP		/* Driver compatibility flag */
14118 };
14119 
14120 static struct dev_ops dtrace_ops = {
14121 	DEVO_REV,		/* devo_rev */
14122 	0,			/* refcnt */
14123 	dtrace_info,		/* get_dev_info */
14124 	nulldev,		/* identify */
14125 	nulldev,		/* probe */
14126 	dtrace_attach,		/* attach */
14127 	dtrace_detach,		/* detach */
14128 	nodev,			/* reset */
14129 	&dtrace_cb_ops,		/* driver operations */
14130 	NULL,			/* bus operations */
14131 	nodev			/* dev power */
14132 };
14133 
14134 static struct modldrv modldrv = {
14135 	&mod_driverops,		/* module type (this is a pseudo driver) */
14136 	"Dynamic Tracing",	/* name of module */
14137 	&dtrace_ops,		/* driver ops */
14138 };
14139 
14140 static struct modlinkage modlinkage = {
14141 	MODREV_1,
14142 	(void *)&modldrv,
14143 	NULL
14144 };
14145 
14146 int
14147 _init(void)
14148 {
14149 	return (mod_install(&modlinkage));
14150 }
14151 
14152 int
14153 _info(struct modinfo *modinfop)
14154 {
14155 	return (mod_info(&modlinkage, modinfop));
14156 }
14157 
14158 int
14159 _fini(void)
14160 {
14161 	return (mod_remove(&modlinkage));
14162 }
14163