17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5ac448965Sahl  * Common Development and Distribution License (the "License").
6ac448965Sahl  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
2175521904Sraf 
227c478bd9Sstevel@tonic-gate /*
2373427c57Sahl  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
277c478bd9Sstevel@tonic-gate #include <sys/fasttrap_isa.h>
287c478bd9Sstevel@tonic-gate #include <sys/fasttrap_impl.h>
297c478bd9Sstevel@tonic-gate #include <sys/dtrace.h>
307c478bd9Sstevel@tonic-gate #include <sys/dtrace_impl.h>
317c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
327c478bd9Sstevel@tonic-gate #include <sys/frame.h>
337c478bd9Sstevel@tonic-gate #include <sys/stack.h>
347c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
357c478bd9Sstevel@tonic-gate #include <sys/trap.h>
367c478bd9Sstevel@tonic-gate 
377c478bd9Sstevel@tonic-gate #include <v9/sys/machpcb.h>
387c478bd9Sstevel@tonic-gate #include <v9/sys/privregs.h>
397c478bd9Sstevel@tonic-gate 
407c478bd9Sstevel@tonic-gate /*
417c478bd9Sstevel@tonic-gate  * Lossless User-Land Tracing on SPARC
427c478bd9Sstevel@tonic-gate  * -----------------------------------
437c478bd9Sstevel@tonic-gate  *
447c478bd9Sstevel@tonic-gate  * The Basic Idea
457c478bd9Sstevel@tonic-gate  *
467c478bd9Sstevel@tonic-gate  * The most important design constraint is, of course, correct execution of
477c478bd9Sstevel@tonic-gate  * the user thread above all else. The next most important goal is rapid
487c478bd9Sstevel@tonic-gate  * execution. We combine execution of instructions in user-land with
497c478bd9Sstevel@tonic-gate  * emulation of certain instructions in the kernel to aim for complete
507c478bd9Sstevel@tonic-gate  * correctness and maximal performance.
517c478bd9Sstevel@tonic-gate  *
527c478bd9Sstevel@tonic-gate  * We take advantage of the split PC/NPC architecture to speed up logical
537c478bd9Sstevel@tonic-gate  * single-stepping; when we copy an instruction out to the scratch space in
547c478bd9Sstevel@tonic-gate  * the ulwp_t structure (held in the %g7 register on SPARC), we can
557c478bd9Sstevel@tonic-gate  * effectively single step by setting the PC to our scratch space and leaving
567c478bd9Sstevel@tonic-gate  * the NPC alone. This executes the replaced instruction and then continues
577c478bd9Sstevel@tonic-gate  * on without having to reenter the kernel as with single- stepping. The
587c478bd9Sstevel@tonic-gate  * obvious caveat is for instructions whose execution is PC dependant --
597c478bd9Sstevel@tonic-gate  * branches, call and link instructions (call and jmpl), and the rdpc
607c478bd9Sstevel@tonic-gate  * instruction. These instructions cannot be executed in the manner described
617c478bd9Sstevel@tonic-gate  * so they must be emulated in the kernel.
627c478bd9Sstevel@tonic-gate  *
637c478bd9Sstevel@tonic-gate  * Emulation for this small set of instructions if fairly simple; the most
647c478bd9Sstevel@tonic-gate  * difficult part being emulating branch conditions.
657c478bd9Sstevel@tonic-gate  *
667c478bd9Sstevel@tonic-gate  *
677c478bd9Sstevel@tonic-gate  * A Cache Heavy Portfolio
687c478bd9Sstevel@tonic-gate  *
697c478bd9Sstevel@tonic-gate  * It's important to note at this time that copying an instruction out to the
707c478bd9Sstevel@tonic-gate  * ulwp_t scratch space in user-land is rather complicated. SPARC has
717c478bd9Sstevel@tonic-gate  * separate data and instruction caches so any writes to the D$ (using a
727c478bd9Sstevel@tonic-gate  * store instruction for example) aren't necessarily reflected in the I$.
737c478bd9Sstevel@tonic-gate  * The flush instruction can be used to synchronize the two and must be used
747c478bd9Sstevel@tonic-gate  * for any self-modifying code, but the flush instruction only applies to the
757c478bd9Sstevel@tonic-gate  * primary address space (the absence of a flusha analogue to the flush
767c478bd9Sstevel@tonic-gate  * instruction that accepts an ASI argument is an obvious omission from SPARC
777c478bd9Sstevel@tonic-gate  * v9 where the notion of the alternate address space was introduced on
787c478bd9Sstevel@tonic-gate  * SPARC). To correctly copy out the instruction we must use a block store
797c478bd9Sstevel@tonic-gate  * that doesn't allocate in the D$ and ensures synchronization with the I$;
807c478bd9Sstevel@tonic-gate  * see dtrace_blksuword32() for the implementation  (this function uses
817c478bd9Sstevel@tonic-gate  * ASI_BLK_COMMIT_S to write a block through the secondary ASI in the manner
827c478bd9Sstevel@tonic-gate  * described). Refer to the UltraSPARC I/II manual for details on the
837c478bd9Sstevel@tonic-gate  * ASI_BLK_COMMIT_S ASI.
847c478bd9Sstevel@tonic-gate  *
857c478bd9Sstevel@tonic-gate  *
867c478bd9Sstevel@tonic-gate  * Return Subtleties
877c478bd9Sstevel@tonic-gate  *
887c478bd9Sstevel@tonic-gate  * When we're firing a return probe we need to expose the value returned by
897c478bd9Sstevel@tonic-gate  * the function being traced. Since the function can set the return value
907c478bd9Sstevel@tonic-gate  * in its last instruction, we need to fire the return probe only _after_
917c478bd9Sstevel@tonic-gate  * the effects of the instruction are apparent. For instructions that we
927c478bd9Sstevel@tonic-gate  * emulate, we can call dtrace_probe() after we've performed the emulation;
937c478bd9Sstevel@tonic-gate  * for instructions that we execute after we return to user-land, we set
947c478bd9Sstevel@tonic-gate  * %pc to the instruction we copied out (as described above) and set %npc
957c478bd9Sstevel@tonic-gate  * to a trap instruction stashed in the ulwp_t structure. After the traced
967c478bd9Sstevel@tonic-gate  * instruction is executed, the trap instruction returns control to the
977c478bd9Sstevel@tonic-gate  * kernel where we can fire the return probe.
987c478bd9Sstevel@tonic-gate  *
997c478bd9Sstevel@tonic-gate  * This need for a second trap in cases where we execute the traced
1007c478bd9Sstevel@tonic-gate  * instruction makes it all the more important to emulate the most common
1017c478bd9Sstevel@tonic-gate  * instructions to avoid the second trip in and out of the kernel.
1027c478bd9Sstevel@tonic-gate  *
1037c478bd9Sstevel@tonic-gate  *
1047c478bd9Sstevel@tonic-gate  * Making it Fast
1057c478bd9Sstevel@tonic-gate  *
1067c478bd9Sstevel@tonic-gate  * Since copying out an instruction is neither simple nor inexpensive for the
1077c478bd9Sstevel@tonic-gate  * CPU, we should attempt to avoid doing it in as many cases as possible.
1087c478bd9Sstevel@tonic-gate  * Since function entry and return are usually the most interesting probe
1097c478bd9Sstevel@tonic-gate  * sites, we attempt to tune the performance of the fasttrap provider around
1107c478bd9Sstevel@tonic-gate  * instructions typically in those places.
1117c478bd9Sstevel@tonic-gate  *
1127c478bd9Sstevel@tonic-gate  * Looking at a bunch of functions in libraries and executables reveals that
1137c478bd9Sstevel@tonic-gate  * most functions begin with either a save or a sethi (to setup a larger
1147c478bd9Sstevel@tonic-gate  * argument to the save) and end with a restore or an or (in the case of leaf
1157c478bd9Sstevel@tonic-gate  * functions). To try to improve performance, we emulate all of these
1167c478bd9Sstevel@tonic-gate  * instructions in the kernel.
1177c478bd9Sstevel@tonic-gate  *
1187c478bd9Sstevel@tonic-gate  * The save and restore instructions are a little tricky since they perform
1197c478bd9Sstevel@tonic-gate  * register window maniplulation. Rather than trying to tinker with the
1207c478bd9Sstevel@tonic-gate  * register windows from the kernel, we emulate the implicit add that takes
1217c478bd9Sstevel@tonic-gate  * place as part of those instructions and set the %pc to point to a simple
1227c478bd9Sstevel@tonic-gate  * save or restore we've hidden in the ulwp_t structure. If we're in a return
1237c478bd9Sstevel@tonic-gate  * probe so want to make it seem as though the tracepoint has been completely
1247c478bd9Sstevel@tonic-gate  * executed we need to remember that we've pulled this trick with restore and
1257c478bd9Sstevel@tonic-gate  * pull registers from the previous window (the one that we'll switch to once
1267c478bd9Sstevel@tonic-gate  * the simple store instruction is executed) rather than the current one. This
1277c478bd9Sstevel@tonic-gate  * is why in the case of emulating a restore we set the DTrace CPU flag
1287c478bd9Sstevel@tonic-gate  * CPU_DTRACE_FAKERESTORE before calling dtrace_probe() for the return probes
1297c478bd9Sstevel@tonic-gate  * (see fasttrap_return_common()).
1307c478bd9Sstevel@tonic-gate  */
1317c478bd9Sstevel@tonic-gate 
1327c478bd9Sstevel@tonic-gate #define	OP(x)		((x) >> 30)
1337c478bd9Sstevel@tonic-gate #define	OP2(x)		(((x) >> 22) & 0x07)
1347c478bd9Sstevel@tonic-gate #define	OP3(x)		(((x) >> 19) & 0x3f)
1357c478bd9Sstevel@tonic-gate #define	RCOND(x)	(((x) >> 25) & 0x07)
1367c478bd9Sstevel@tonic-gate #define	COND(x)		(((x) >> 25) & 0x0f)
1377c478bd9Sstevel@tonic-gate #define	A(x)		(((x) >> 29) & 0x01)
1387c478bd9Sstevel@tonic-gate #define	I(x)		(((x) >> 13) & 0x01)
1397c478bd9Sstevel@tonic-gate #define	RD(x)		(((x) >> 25) & 0x1f)
1407c478bd9Sstevel@tonic-gate #define	RS1(x)		(((x) >> 14) & 0x1f)
1417c478bd9Sstevel@tonic-gate #define	RS2(x)		(((x) >> 0) & 0x1f)
1427c478bd9Sstevel@tonic-gate #define	CC(x)		(((x) >> 20) & 0x03)
1437c478bd9Sstevel@tonic-gate #define	DISP16(x)	((((x) >> 6) & 0xc000) | ((x) & 0x3fff))
1447c478bd9Sstevel@tonic-gate #define	DISP22(x)	((x) & 0x3fffff)
1457c478bd9Sstevel@tonic-gate #define	DISP19(x)	((x) & 0x7ffff)
1467c478bd9Sstevel@tonic-gate #define	DISP30(x)	((x) & 0x3fffffff)
1477c478bd9Sstevel@tonic-gate #define	SW_TRAP(x)	((x) & 0x7f)
1487c478bd9Sstevel@tonic-gate 
1497c478bd9Sstevel@tonic-gate #define	OP3_OR		0x02
1507c478bd9Sstevel@tonic-gate #define	OP3_RD		0x28
1517c478bd9Sstevel@tonic-gate #define	OP3_JMPL	0x38
1527c478bd9Sstevel@tonic-gate #define	OP3_RETURN	0x39
1537c478bd9Sstevel@tonic-gate #define	OP3_TCC		0x3a
1547c478bd9Sstevel@tonic-gate #define	OP3_SAVE	0x3c
1557c478bd9Sstevel@tonic-gate #define	OP3_RESTORE	0x3d
1567c478bd9Sstevel@tonic-gate 
1577c478bd9Sstevel@tonic-gate #define	OP3_PREFETCH	0x2d
1587c478bd9Sstevel@tonic-gate #define	OP3_CASA	0x3c
1597c478bd9Sstevel@tonic-gate #define	OP3_PREFETCHA	0x3d
1607c478bd9Sstevel@tonic-gate #define	OP3_CASXA	0x3e
1617c478bd9Sstevel@tonic-gate 
1627c478bd9Sstevel@tonic-gate #define	OP2_ILLTRAP	0x0
1637c478bd9Sstevel@tonic-gate #define	OP2_BPcc	0x1
1647c478bd9Sstevel@tonic-gate #define	OP2_Bicc	0x2
1657c478bd9Sstevel@tonic-gate #define	OP2_BPr		0x3
1667c478bd9Sstevel@tonic-gate #define	OP2_SETHI	0x4
1677c478bd9Sstevel@tonic-gate #define	OP2_FBPfcc	0x5
1687c478bd9Sstevel@tonic-gate #define	OP2_FBfcc	0x6
1697c478bd9Sstevel@tonic-gate 
1707c478bd9Sstevel@tonic-gate #define	R_G0		0
1717c478bd9Sstevel@tonic-gate #define	R_O0		8
1727c478bd9Sstevel@tonic-gate #define	R_SP		14
1737c478bd9Sstevel@tonic-gate #define	R_I0		24
1747c478bd9Sstevel@tonic-gate #define	R_I1		25
1757c478bd9Sstevel@tonic-gate #define	R_I2		26
1767c478bd9Sstevel@tonic-gate #define	R_I3		27
17792e807e6Sahl #define	R_I4		28
1787c478bd9Sstevel@tonic-gate 
1797c478bd9Sstevel@tonic-gate /*
1807c478bd9Sstevel@tonic-gate  * Check the comment in fasttrap.h when changing these offsets or adding
1817c478bd9Sstevel@tonic-gate  * new instructions.
1827c478bd9Sstevel@tonic-gate  */
1837c478bd9Sstevel@tonic-gate #define	FASTTRAP_OFF_SAVE	64
1847c478bd9Sstevel@tonic-gate #define	FASTTRAP_OFF_RESTORE	68
1857c478bd9Sstevel@tonic-gate #define	FASTTRAP_OFF_FTRET	72
1867c478bd9Sstevel@tonic-gate #define	FASTTRAP_OFF_RETURN	76
1877c478bd9Sstevel@tonic-gate 
1887c478bd9Sstevel@tonic-gate #define	BREAKPOINT_INSTR	0x91d02001	/* ta 1 */
1897c478bd9Sstevel@tonic-gate 
1907c478bd9Sstevel@tonic-gate /*
1917c478bd9Sstevel@tonic-gate  * Tunable to let users turn off the fancy save instruction optimization.
1927c478bd9Sstevel@tonic-gate  * If a program is non-ABI compliant, there's a possibility that the save
1937c478bd9Sstevel@tonic-gate  * instruction optimization could cause an error.
1947c478bd9Sstevel@tonic-gate  */
1957c478bd9Sstevel@tonic-gate int fasttrap_optimize_save = 1;
1967c478bd9Sstevel@tonic-gate 
1977c478bd9Sstevel@tonic-gate static uint64_t
fasttrap_anarg(struct regs * rp,int argno)1987c478bd9Sstevel@tonic-gate fasttrap_anarg(struct regs *rp, int argno)
1997c478bd9Sstevel@tonic-gate {
2007c478bd9Sstevel@tonic-gate 	uint64_t value;
2017c478bd9Sstevel@tonic-gate 
2027c478bd9Sstevel@tonic-gate 	if (argno < 6)
2037c478bd9Sstevel@tonic-gate 		return ((&rp->r_o0)[argno]);
2047c478bd9Sstevel@tonic-gate 
2057c478bd9Sstevel@tonic-gate 	if (curproc->p_model == DATAMODEL_NATIVE) {
2067c478bd9Sstevel@tonic-gate 		struct frame *fr = (struct frame *)(rp->r_sp + STACK_BIAS);
2077c478bd9Sstevel@tonic-gate 
2087c478bd9Sstevel@tonic-gate 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2097c478bd9Sstevel@tonic-gate 		value = dtrace_fulword(&fr->fr_argd[argno]);
2107c478bd9Sstevel@tonic-gate 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR |
2117c478bd9Sstevel@tonic-gate 		    CPU_DTRACE_BADALIGN);
2127c478bd9Sstevel@tonic-gate 	} else {
2137c478bd9Sstevel@tonic-gate 		struct frame32 *fr = (struct frame32 *)rp->r_sp;
2147c478bd9Sstevel@tonic-gate 
2157c478bd9Sstevel@tonic-gate 		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
2167c478bd9Sstevel@tonic-gate 		value = dtrace_fuword32(&fr->fr_argd[argno]);
2177c478bd9Sstevel@tonic-gate 		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR |
2187c478bd9Sstevel@tonic-gate 		    CPU_DTRACE_BADALIGN);
2197c478bd9Sstevel@tonic-gate 	}
2207c478bd9Sstevel@tonic-gate 
2217c478bd9Sstevel@tonic-gate 	return (value);
2227c478bd9Sstevel@tonic-gate }
2237c478bd9Sstevel@tonic-gate 
2247c478bd9Sstevel@tonic-gate static ulong_t fasttrap_getreg(struct regs *, uint_t);
2257c478bd9Sstevel@tonic-gate static void fasttrap_putreg(struct regs *, uint_t, ulong_t);
2267c478bd9Sstevel@tonic-gate 
2277c478bd9Sstevel@tonic-gate static void
fasttrap_usdt_args(fasttrap_probe_t * probe,struct regs * rp,uint_t fake_restore,int argc,uintptr_t * argv)22892e807e6Sahl fasttrap_usdt_args(fasttrap_probe_t *probe, struct regs *rp,
22992e807e6Sahl     uint_t fake_restore, int argc, uintptr_t *argv)
2307c478bd9Sstevel@tonic-gate {
2317c478bd9Sstevel@tonic-gate 	int i, x, cap = MIN(argc, probe->ftp_nargs);
23292e807e6Sahl 	int inc = (fake_restore ? 16 : 0);
23392e807e6Sahl 
23492e807e6Sahl 	/*
23592e807e6Sahl 	 * The only way we'll hit the fake_restore case is if a USDT probe is
23692e807e6Sahl 	 * invoked as a tail-call. While it wouldn't be incorrect, we can
23792e807e6Sahl 	 * avoid a call to fasttrap_getreg(), and safely use rp->r_sp
23892e807e6Sahl 	 * directly since a tail-call can't be made if the invoked function
23992e807e6Sahl 	 * would use the argument dump space (i.e. if there were more than
24092e807e6Sahl 	 * 6 arguments). We take this shortcut because unconditionally rooting
24192e807e6Sahl 	 * around for R_FP (R_SP + 16) would be unnecessarily painful.
24292e807e6Sahl 	 */
2437c478bd9Sstevel@tonic-gate 
2447c478bd9Sstevel@tonic-gate 	if (curproc->p_model == DATAMODEL_NATIVE) {
2457c478bd9Sstevel@tonic-gate 		struct frame *fr = (struct frame *)(rp->r_sp + STACK_BIAS);
2467c478bd9Sstevel@tonic-gate 		uintptr_t v;
2477c478bd9Sstevel@tonic-gate 
2487c478bd9Sstevel@tonic-gate 		for (i = 0; i < cap; i++) {
2497c478bd9Sstevel@tonic-gate 			x = probe->ftp_argmap[i];
2507c478bd9Sstevel@tonic-gate 
2517c478bd9Sstevel@tonic-gate 			if (x < 6)
25292e807e6Sahl 				argv[i] = fasttrap_getreg(rp, R_O0 + x + inc);
2537c478bd9Sstevel@tonic-gate 			else if (fasttrap_fulword(&fr->fr_argd[x], &v) != 0)
2547c478bd9Sstevel@tonic-gate 				argv[i] = 0;
2557c478bd9Sstevel@tonic-gate 		}
2567c478bd9Sstevel@tonic-gate 
2577c478bd9Sstevel@tonic-gate 	} else {
2587c478bd9Sstevel@tonic-gate 		struct frame32 *fr = (struct frame32 *)rp->r_sp;
2597c478bd9Sstevel@tonic-gate 		uint32_t v;
2607c478bd9Sstevel@tonic-gate 
2617c478bd9Sstevel@tonic-gate 		for (i = 0; i < cap; i++) {
2627c478bd9Sstevel@tonic-gate 			x = probe->ftp_argmap[i];
2637c478bd9Sstevel@tonic-gate 
2647c478bd9Sstevel@tonic-gate 			if (x < 6)
26592e807e6Sahl 				argv[i] = fasttrap_getreg(rp, R_O0 + x + inc);
2667c478bd9Sstevel@tonic-gate 			else if (fasttrap_fuword32(&fr->fr_argd[x], &v) != 0)
2677c478bd9Sstevel@tonic-gate 				argv[i] = 0;
2687c478bd9Sstevel@tonic-gate 		}
2697c478bd9Sstevel@tonic-gate 	}
2707c478bd9Sstevel@tonic-gate 
2717c478bd9Sstevel@tonic-gate 	for (; i < argc; i++) {
2727c478bd9Sstevel@tonic-gate 		argv[i] = 0;
2737c478bd9Sstevel@tonic-gate 	}
2747c478bd9Sstevel@tonic-gate }
2757c478bd9Sstevel@tonic-gate 
2767c478bd9Sstevel@tonic-gate static void
fasttrap_return_common(struct regs * rp,uintptr_t pc,pid_t pid,uint_t fake_restore)2777c478bd9Sstevel@tonic-gate fasttrap_return_common(struct regs *rp, uintptr_t pc, pid_t pid,
2787c478bd9Sstevel@tonic-gate     uint_t fake_restore)
2797c478bd9Sstevel@tonic-gate {
2807c478bd9Sstevel@tonic-gate 	fasttrap_tracepoint_t *tp;
2817c478bd9Sstevel@tonic-gate 	fasttrap_bucket_t *bucket;
2827c478bd9Sstevel@tonic-gate 	fasttrap_id_t *id;
2837c478bd9Sstevel@tonic-gate 	kmutex_t *pid_mtx;
2847c478bd9Sstevel@tonic-gate 	dtrace_icookie_t cookie;
2857c478bd9Sstevel@tonic-gate 
2867c478bd9Sstevel@tonic-gate 	pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
2877c478bd9Sstevel@tonic-gate 	mutex_enter(pid_mtx);
2887c478bd9Sstevel@tonic-gate 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
2897c478bd9Sstevel@tonic-gate 
2907c478bd9Sstevel@tonic-gate 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
2917c478bd9Sstevel@tonic-gate 		if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
292038dc6b3Sahl 		    tp->ftt_proc->ftpc_acount != 0)
2937c478bd9Sstevel@tonic-gate 			break;
2947c478bd9Sstevel@tonic-gate 	}
2957c478bd9Sstevel@tonic-gate 
2967c478bd9Sstevel@tonic-gate 	/*
2977c478bd9Sstevel@tonic-gate 	 * Don't sweat it if we can't find the tracepoint again; unlike
2987c478bd9Sstevel@tonic-gate 	 * when we're in fasttrap_pid_probe(), finding the tracepoint here
2997c478bd9Sstevel@tonic-gate 	 * is not essential to the correct execution of the process.
3007c478bd9Sstevel@tonic-gate 	 */
3017c478bd9Sstevel@tonic-gate 	if (tp == NULL || tp->ftt_retids == NULL) {
3027c478bd9Sstevel@tonic-gate 		mutex_exit(pid_mtx);
3037c478bd9Sstevel@tonic-gate 		return;
3047c478bd9Sstevel@tonic-gate 	}
3057c478bd9Sstevel@tonic-gate 
3067c478bd9Sstevel@tonic-gate 	for (id = tp->ftt_retids; id != NULL; id = id->fti_next) {
3077c478bd9Sstevel@tonic-gate 		fasttrap_probe_t *probe = id->fti_probe;
3087c478bd9Sstevel@tonic-gate 
309ac448965Sahl 		if (id->fti_ptype == DTFTP_POST_OFFSETS) {
31092e807e6Sahl 			if (probe->ftp_argmap != NULL && fake_restore) {
31192e807e6Sahl 				uintptr_t t[5];
31292e807e6Sahl 
31392e807e6Sahl 				fasttrap_usdt_args(probe, rp, fake_restore,
31492e807e6Sahl 				    sizeof (t) / sizeof (t[0]), t);
31592e807e6Sahl 
31692e807e6Sahl 				cookie = dtrace_interrupt_disable();
31792e807e6Sahl 				DTRACE_CPUFLAG_SET(CPU_DTRACE_FAKERESTORE);
31892e807e6Sahl 				dtrace_probe(probe->ftp_id, t[0], t[1],
31992e807e6Sahl 				    t[2], t[3], t[4]);
32092e807e6Sahl 				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_FAKERESTORE);
32192e807e6Sahl 				dtrace_interrupt_enable(cookie);
32292e807e6Sahl 
32392e807e6Sahl 			} else if (probe->ftp_argmap != NULL) {
3247c478bd9Sstevel@tonic-gate 				uintptr_t t[5];
3257c478bd9Sstevel@tonic-gate 
32692e807e6Sahl 				fasttrap_usdt_args(probe, rp, fake_restore,
3277c478bd9Sstevel@tonic-gate 				    sizeof (t) / sizeof (t[0]), t);
3287c478bd9Sstevel@tonic-gate 
3297c478bd9Sstevel@tonic-gate 				dtrace_probe(probe->ftp_id, t[0], t[1],
3307c478bd9Sstevel@tonic-gate 				    t[2], t[3], t[4]);
33192e807e6Sahl 
33292e807e6Sahl 			} else if (fake_restore) {
33392e807e6Sahl 				uintptr_t arg0 = fasttrap_getreg(rp, R_I0);
33492e807e6Sahl 				uintptr_t arg1 = fasttrap_getreg(rp, R_I1);
33592e807e6Sahl 				uintptr_t arg2 = fasttrap_getreg(rp, R_I2);
33692e807e6Sahl 				uintptr_t arg3 = fasttrap_getreg(rp, R_I3);
33792e807e6Sahl 				uintptr_t arg4 = fasttrap_getreg(rp, R_I4);
33892e807e6Sahl 
33992e807e6Sahl 				cookie = dtrace_interrupt_disable();
34092e807e6Sahl 				DTRACE_CPUFLAG_SET(CPU_DTRACE_FAKERESTORE);
34192e807e6Sahl 				dtrace_probe(probe->ftp_id, arg0, arg1,
34292e807e6Sahl 				    arg2, arg3, arg4);
34392e807e6Sahl 				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_FAKERESTORE);
34492e807e6Sahl 				dtrace_interrupt_enable(cookie);
34592e807e6Sahl 
34692e807e6Sahl 			} else {
34792e807e6Sahl 				dtrace_probe(probe->ftp_id, rp->r_o0, rp->r_o1,
34892e807e6Sahl 				    rp->r_o2, rp->r_o3, rp->r_o4);
3497c478bd9Sstevel@tonic-gate 			}
35092e807e6Sahl 
3517c478bd9Sstevel@tonic-gate 			continue;
3527c478bd9Sstevel@tonic-gate 		}
3537c478bd9Sstevel@tonic-gate 
3547c478bd9Sstevel@tonic-gate 		/*
3557c478bd9Sstevel@tonic-gate 		 * If this is only a possible return point, we must
3567c478bd9Sstevel@tonic-gate 		 * be looking at a potential tail call in leaf context.
3577c478bd9Sstevel@tonic-gate 		 * If the %npc is still within this function, then we
3587c478bd9Sstevel@tonic-gate 		 * must have misidentified a jmpl as a tail-call when it
3597c478bd9Sstevel@tonic-gate 		 * is, in fact, part of a jump table. It would be nice to
3607c478bd9Sstevel@tonic-gate 		 * remove this tracepoint, but this is neither the time
3617c478bd9Sstevel@tonic-gate 		 * nor the place.
3627c478bd9Sstevel@tonic-gate 		 */
3637c478bd9Sstevel@tonic-gate 		if ((tp->ftt_flags & FASTTRAP_F_RETMAYBE) &&
3647c478bd9Sstevel@tonic-gate 		    rp->r_npc - probe->ftp_faddr < probe->ftp_fsize)
3657c478bd9Sstevel@tonic-gate 			continue;
3667c478bd9Sstevel@tonic-gate 
3677c478bd9Sstevel@tonic-gate 		/*
3687c478bd9Sstevel@tonic-gate 		 * It's possible for a function to branch to the delay slot
3697c478bd9Sstevel@tonic-gate 		 * of an instruction that we've identified as a return site.
3707c478bd9Sstevel@tonic-gate 		 * We can dectect this spurious return probe activation by
3717c478bd9Sstevel@tonic-gate 		 * observing that in this case %npc will be %pc + 4 and %npc
3727c478bd9Sstevel@tonic-gate 		 * will be inside the current function (unless the user is
3737c478bd9Sstevel@tonic-gate 		 * doing _crazy_ instruction picking in which case there's
3747c478bd9Sstevel@tonic-gate 		 * very little we can do). The second check is important
3757c478bd9Sstevel@tonic-gate 		 * in case the last instructions of a function make a tail-
3767c478bd9Sstevel@tonic-gate 		 * call to the function located immediately subsequent.
3777c478bd9Sstevel@tonic-gate 		 */
3787c478bd9Sstevel@tonic-gate 		if (rp->r_npc == rp->r_pc + 4 &&
3797c478bd9Sstevel@tonic-gate 		    rp->r_npc - probe->ftp_faddr < probe->ftp_fsize)
3807c478bd9Sstevel@tonic-gate 			continue;
3817c478bd9Sstevel@tonic-gate 
3827c478bd9Sstevel@tonic-gate 		/*
3837c478bd9Sstevel@tonic-gate 		 * The first argument is the offset of return tracepoint
3847c478bd9Sstevel@tonic-gate 		 * in the function; the remaining arguments are the return
3857c478bd9Sstevel@tonic-gate 		 * values.
3867c478bd9Sstevel@tonic-gate 		 *
3877c478bd9Sstevel@tonic-gate 		 * If fake_restore is set, we need to pull the return values
3887c478bd9Sstevel@tonic-gate 		 * out of the %i's rather than the %o's -- a little trickier.
3897c478bd9Sstevel@tonic-gate 		 */
3907c478bd9Sstevel@tonic-gate 		if (!fake_restore) {
3917c478bd9Sstevel@tonic-gate 			dtrace_probe(probe->ftp_id, pc - probe->ftp_faddr,
3927c478bd9Sstevel@tonic-gate 			    rp->r_o0, rp->r_o1, rp->r_o2, rp->r_o3);
3937c478bd9Sstevel@tonic-gate 		} else {
3947c478bd9Sstevel@tonic-gate 			uintptr_t arg0 = fasttrap_getreg(rp, R_I0);
3957c478bd9Sstevel@tonic-gate 			uintptr_t arg1 = fasttrap_getreg(rp, R_I1);
3967c478bd9Sstevel@tonic-gate 			uintptr_t arg2 = fasttrap_getreg(rp, R_I2);
3977c478bd9Sstevel@tonic-gate 			uintptr_t arg3 = fasttrap_getreg(rp, R_I3);
3987c478bd9Sstevel@tonic-gate 
3997c478bd9Sstevel@tonic-gate 			cookie = dtrace_interrupt_disable();
4007c478bd9Sstevel@tonic-gate 			DTRACE_CPUFLAG_SET(CPU_DTRACE_FAKERESTORE);
4017c478bd9Sstevel@tonic-gate 			dtrace_probe(probe->ftp_id, pc - probe->ftp_faddr,
4027c478bd9Sstevel@tonic-gate 			    arg0, arg1, arg2, arg3);
4037c478bd9Sstevel@tonic-gate 			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_FAKERESTORE);
4047c478bd9Sstevel@tonic-gate 			dtrace_interrupt_enable(cookie);
4057c478bd9Sstevel@tonic-gate 		}
4067c478bd9Sstevel@tonic-gate 	}
4077c478bd9Sstevel@tonic-gate 
4087c478bd9Sstevel@tonic-gate 	mutex_exit(pid_mtx);
4097c478bd9Sstevel@tonic-gate }
4107c478bd9Sstevel@tonic-gate 
4117c478bd9Sstevel@tonic-gate int
fasttrap_pid_probe(struct regs * rp)4127c478bd9Sstevel@tonic-gate fasttrap_pid_probe(struct regs *rp)
4137c478bd9Sstevel@tonic-gate {
4147c478bd9Sstevel@tonic-gate 	proc_t *p = curproc;
4157c478bd9Sstevel@tonic-gate 	fasttrap_tracepoint_t *tp, tp_local;
4167c478bd9Sstevel@tonic-gate 	fasttrap_id_t *id;
4177c478bd9Sstevel@tonic-gate 	pid_t pid;
4187c478bd9Sstevel@tonic-gate 	uintptr_t pc = rp->r_pc;
4197c478bd9Sstevel@tonic-gate 	uintptr_t npc = rp->r_npc;
4207c478bd9Sstevel@tonic-gate 	uintptr_t orig_pc = pc;
4217c478bd9Sstevel@tonic-gate 	fasttrap_bucket_t *bucket;
4227c478bd9Sstevel@tonic-gate 	kmutex_t *pid_mtx;
423ac448965Sahl 	uint_t fake_restore = 0, is_enabled = 0;
4247c478bd9Sstevel@tonic-gate 	dtrace_icookie_t cookie;
4257c478bd9Sstevel@tonic-gate 
4267c478bd9Sstevel@tonic-gate 	/*
4277c478bd9Sstevel@tonic-gate 	 * It's possible that a user (in a veritable orgy of bad planning)
4287c478bd9Sstevel@tonic-gate 	 * could redirect this thread's flow of control before it reached the
4297c478bd9Sstevel@tonic-gate 	 * return probe fasttrap. In this case we need to kill the process
4307c478bd9Sstevel@tonic-gate 	 * since it's in a unrecoverable state.
4317c478bd9Sstevel@tonic-gate 	 */
4327c478bd9Sstevel@tonic-gate 	if (curthread->t_dtrace_step) {
4337c478bd9Sstevel@tonic-gate 		ASSERT(curthread->t_dtrace_on);
4347c478bd9Sstevel@tonic-gate 		fasttrap_sigtrap(p, curthread, pc);
4357c478bd9Sstevel@tonic-gate 		return (0);
4367c478bd9Sstevel@tonic-gate 	}
4377c478bd9Sstevel@tonic-gate 
4387c478bd9Sstevel@tonic-gate 	/*
4397c478bd9Sstevel@tonic-gate 	 * Clear all user tracing flags.
4407c478bd9Sstevel@tonic-gate 	 */
4417c478bd9Sstevel@tonic-gate 	curthread->t_dtrace_ft = 0;
4427c478bd9Sstevel@tonic-gate 	curthread->t_dtrace_pc = 0;
4437c478bd9Sstevel@tonic-gate 	curthread->t_dtrace_npc = 0;
4447c478bd9Sstevel@tonic-gate 	curthread->t_dtrace_scrpc = 0;
4457c478bd9Sstevel@tonic-gate 	curthread->t_dtrace_astpc = 0;
4467c478bd9Sstevel@tonic-gate 
4477c478bd9Sstevel@tonic-gate 	/*
4487c478bd9Sstevel@tonic-gate 	 * Treat a child created by a call to vfork(2) as if it were its
4497c478bd9Sstevel@tonic-gate 	 * parent. We know that there's only one thread of control in such a
4507c478bd9Sstevel@tonic-gate 	 * process: this one.
4517c478bd9Sstevel@tonic-gate 	 */
4527c478bd9Sstevel@tonic-gate 	while (p->p_flag & SVFORK) {
4537c478bd9Sstevel@tonic-gate 		p = p->p_parent;
4547c478bd9Sstevel@tonic-gate 	}
4557c478bd9Sstevel@tonic-gate 
4567c478bd9Sstevel@tonic-gate 	pid = p->p_pid;
4577c478bd9Sstevel@tonic-gate 	pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
4587c478bd9Sstevel@tonic-gate 	mutex_enter(pid_mtx);
4597c478bd9Sstevel@tonic-gate 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
4607c478bd9Sstevel@tonic-gate 
4617c478bd9Sstevel@tonic-gate 	/*
4627c478bd9Sstevel@tonic-gate 	 * Lookup the tracepoint that the process just hit.
4637c478bd9Sstevel@tonic-gate 	 */
4647c478bd9Sstevel@tonic-gate 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
4657c478bd9Sstevel@tonic-gate 		if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
466038dc6b3Sahl 		    tp->ftt_proc->ftpc_acount != 0)
4677c478bd9Sstevel@tonic-gate 			break;
4687c478bd9Sstevel@tonic-gate 	}
4697c478bd9Sstevel@tonic-gate 
4707c478bd9Sstevel@tonic-gate 	/*
4717c478bd9Sstevel@tonic-gate 	 * If we couldn't find a matching tracepoint, either a tracepoint has
4727c478bd9Sstevel@tonic-gate 	 * been inserted without using the pid<pid> ioctl interface (see
4737c478bd9Sstevel@tonic-gate 	 * fasttrap_ioctl), or somehow we have mislaid this tracepoint.
4747c478bd9Sstevel@tonic-gate 	 */
4757c478bd9Sstevel@tonic-gate 	if (tp == NULL) {
4767c478bd9Sstevel@tonic-gate 		mutex_exit(pid_mtx);
4777c478bd9Sstevel@tonic-gate 		return (-1);
4787c478bd9Sstevel@tonic-gate 	}
4797c478bd9Sstevel@tonic-gate 
4807c478bd9Sstevel@tonic-gate 	for (id = tp->ftt_ids; id != NULL; id = id->fti_next) {
4817c478bd9Sstevel@tonic-gate 		fasttrap_probe_t *probe = id->fti_probe;
482ac448965Sahl 		int isentry = (id->fti_ptype == DTFTP_ENTRY);
483ac448965Sahl 
484ac448965Sahl 		if (id->fti_ptype == DTFTP_IS_ENABLED) {
485ac448965Sahl 			is_enabled = 1;
486ac448965Sahl 			continue;
487ac448965Sahl 		}
488ac448965Sahl 
4897c478bd9Sstevel@tonic-gate 		/*
4907c478bd9Sstevel@tonic-gate 		 * We note that this was an entry probe to help ustack() find
4917c478bd9Sstevel@tonic-gate 		 * the first caller.
4927c478bd9Sstevel@tonic-gate 		 */
493ac448965Sahl 		if (isentry) {
4947c478bd9Sstevel@tonic-gate 			cookie = dtrace_interrupt_disable();
4957c478bd9Sstevel@tonic-gate 			DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY);
4967c478bd9Sstevel@tonic-gate 		}
4977c478bd9Sstevel@tonic-gate 		dtrace_probe(probe->ftp_id, rp->r_o0, rp->r_o1, rp->r_o2,
4987c478bd9Sstevel@tonic-gate 		    rp->r_o3, rp->r_o4);
4997c478bd9Sstevel@tonic-gate 		if (isentry) {
5007c478bd9Sstevel@tonic-gate 			DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY);
5017c478bd9Sstevel@tonic-gate 			dtrace_interrupt_enable(cookie);
5027c478bd9Sstevel@tonic-gate 		}
5037c478bd9Sstevel@tonic-gate 	}
5047c478bd9Sstevel@tonic-gate 
5057c478bd9Sstevel@tonic-gate 	/*
5067c478bd9Sstevel@tonic-gate 	 * We're about to do a bunch of work so we cache a local copy of
5077c478bd9Sstevel@tonic-gate 	 * the tracepoint to emulate the instruction, and then find the
5087c478bd9Sstevel@tonic-gate 	 * tracepoint again later if we need to light up any return probes.
5097c478bd9Sstevel@tonic-gate 	 */
5107c478bd9Sstevel@tonic-gate 	tp_local = *tp;
5117c478bd9Sstevel@tonic-gate 	mutex_exit(pid_mtx);
5127c478bd9Sstevel@tonic-gate 	tp = &tp_local;
5137c478bd9Sstevel@tonic-gate 
5147c478bd9Sstevel@tonic-gate 	/*
515ac448965Sahl 	 * If there's an is-enabled probe conntected to this tracepoint it
516ac448965Sahl 	 * means that there was a 'mov %g0, %o0' instruction that was placed
517ac448965Sahl 	 * there by DTrace when the binary was linked. As this probe is, in
518ac448965Sahl 	 * fact, enabled, we need to stuff 1 into %o0. Accordingly, we can
519ac448965Sahl 	 * bypass all the instruction emulation logic since we know the
520ac448965Sahl 	 * inevitable result. It's possible that a user could construct a
521ac448965Sahl 	 * scenario where the 'is-enabled' probe was on some other
522ac448965Sahl 	 * instruction, but that would be a rather exotic way to shoot oneself
523ac448965Sahl 	 * in the foot.
524ac448965Sahl 	 */
525ac448965Sahl 	if (is_enabled) {
526ac448965Sahl 		rp->r_o0 = 1;
527ac448965Sahl 		pc = rp->r_npc;
528ac448965Sahl 		npc = pc + 4;
529ac448965Sahl 		goto done;
530ac448965Sahl 	}
531ac448965Sahl 
532ac448965Sahl 	/*
533ac448965Sahl 	 * We emulate certain types of instructions to ensure correctness
5347c478bd9Sstevel@tonic-gate 	 * (in the case of position dependent instructions) or optimize
5357c478bd9Sstevel@tonic-gate 	 * common cases. The rest we have the thread execute back in user-
5367c478bd9Sstevel@tonic-gate 	 * land.
5377c478bd9Sstevel@tonic-gate 	 */
5387c478bd9Sstevel@tonic-gate 	switch (tp->ftt_type) {
5397c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_SAVE:
5407c478bd9Sstevel@tonic-gate 	{
5417c478bd9Sstevel@tonic-gate 		int32_t imm;
5427c478bd9Sstevel@tonic-gate 
5437c478bd9Sstevel@tonic-gate 		/*
5447c478bd9Sstevel@tonic-gate 		 * This an optimization to let us handle function entry
5457c478bd9Sstevel@tonic-gate 		 * probes more efficiently. Many functions begin with a save
5467c478bd9Sstevel@tonic-gate 		 * instruction that follows the pattern:
5477c478bd9Sstevel@tonic-gate 		 *	save	%sp, <imm>, %sp
5487c478bd9Sstevel@tonic-gate 		 *
5497c478bd9Sstevel@tonic-gate 		 * Meanwhile, we've stashed the instruction:
5507c478bd9Sstevel@tonic-gate 		 *	save	%g1, %g0, %sp
5517c478bd9Sstevel@tonic-gate 		 *
5527c478bd9Sstevel@tonic-gate 		 * off of %g7, so all we have to do is stick the right value
5537c478bd9Sstevel@tonic-gate 		 * into %g1 and reset %pc to point to the instruction we've
5547c478bd9Sstevel@tonic-gate 		 * cleverly hidden (%npc should not be touched).
5557c478bd9Sstevel@tonic-gate 		 */
5567c478bd9Sstevel@tonic-gate 
5577c478bd9Sstevel@tonic-gate 		imm = tp->ftt_instr << 19;
5587c478bd9Sstevel@tonic-gate 		imm >>= 19;
5597c478bd9Sstevel@tonic-gate 		rp->r_g1 = rp->r_sp + imm;
5607c478bd9Sstevel@tonic-gate 		pc = rp->r_g7 + FASTTRAP_OFF_SAVE;
5617c478bd9Sstevel@tonic-gate 		break;
5627c478bd9Sstevel@tonic-gate 	}
5637c478bd9Sstevel@tonic-gate 
5647c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_RESTORE:
5657c478bd9Sstevel@tonic-gate 	{
5667c478bd9Sstevel@tonic-gate 		ulong_t value;
5677c478bd9Sstevel@tonic-gate 		uint_t rd;
5687c478bd9Sstevel@tonic-gate 
5697c478bd9Sstevel@tonic-gate 		/*
5707c478bd9Sstevel@tonic-gate 		 * This is an optimization to let us handle function
5717c478bd9Sstevel@tonic-gate 		 * return probes more efficiently. Most non-leaf functions
5727c478bd9Sstevel@tonic-gate 		 * end with the sequence:
5737c478bd9Sstevel@tonic-gate 		 *	ret
5747c478bd9Sstevel@tonic-gate 		 *	restore	<reg>, <reg_or_imm>, %oX
5757c478bd9Sstevel@tonic-gate 		 *
5767c478bd9Sstevel@tonic-gate 		 * We've stashed the instruction:
5777c478bd9Sstevel@tonic-gate 		 *	restore	%g0, %g0, %g0
5787c478bd9Sstevel@tonic-gate 		 *
5797c478bd9Sstevel@tonic-gate 		 * off of %g7 so we just need to place the correct value
5807c478bd9Sstevel@tonic-gate 		 * in the right %i register (since after our fake-o
5817c478bd9Sstevel@tonic-gate 		 * restore, the %i's will become the %o's) and set the %pc
5827c478bd9Sstevel@tonic-gate 		 * to point to our hidden restore. We also set fake_restore to
5837c478bd9Sstevel@tonic-gate 		 * let fasttrap_return_common() know that it will find the
5847c478bd9Sstevel@tonic-gate 		 * return values in the %i's rather than the %o's.
5857c478bd9Sstevel@tonic-gate 		 */
5867c478bd9Sstevel@tonic-gate 
5877c478bd9Sstevel@tonic-gate 		if (I(tp->ftt_instr)) {
5887c478bd9Sstevel@tonic-gate 			int32_t imm;
5897c478bd9Sstevel@tonic-gate 
5907c478bd9Sstevel@tonic-gate 			imm = tp->ftt_instr << 19;
5917c478bd9Sstevel@tonic-gate 			imm >>= 19;
5927c478bd9Sstevel@tonic-gate 			value = fasttrap_getreg(rp, RS1(tp->ftt_instr)) + imm;
5937c478bd9Sstevel@tonic-gate 		} else {
5947c478bd9Sstevel@tonic-gate 			value = fasttrap_getreg(rp, RS1(tp->ftt_instr)) +
5957c478bd9Sstevel@tonic-gate 			    fasttrap_getreg(rp, RS2(tp->ftt_instr));
5967c478bd9Sstevel@tonic-gate 		}
5977c478bd9Sstevel@tonic-gate 
5987c478bd9Sstevel@tonic-gate 		/*
5997c478bd9Sstevel@tonic-gate 		 * Convert %o's to %i's; leave %g's as they are.
6007c478bd9Sstevel@tonic-gate 		 */
6017c478bd9Sstevel@tonic-gate 		rd = RD(tp->ftt_instr);
6027c478bd9Sstevel@tonic-gate 		fasttrap_putreg(rp, ((rd & 0x18) == 0x8) ? rd + 16 : rd, value);
6037c478bd9Sstevel@tonic-gate 
6047c478bd9Sstevel@tonic-gate 		pc = rp->r_g7 + FASTTRAP_OFF_RESTORE;
6057c478bd9Sstevel@tonic-gate 		fake_restore = 1;
6067c478bd9Sstevel@tonic-gate 		break;
6077c478bd9Sstevel@tonic-gate 	}
6087c478bd9Sstevel@tonic-gate 
6097c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_RETURN:
6107c478bd9Sstevel@tonic-gate 	{
6117c478bd9Sstevel@tonic-gate 		uintptr_t target;
6127c478bd9Sstevel@tonic-gate 
6137c478bd9Sstevel@tonic-gate 		/*
6147c478bd9Sstevel@tonic-gate 		 * A return instruction is like a jmpl (without the link
6157c478bd9Sstevel@tonic-gate 		 * part) that executes an implicit restore. We've stashed
6167c478bd9Sstevel@tonic-gate 		 * the instruction:
6177c478bd9Sstevel@tonic-gate 		 *	return %o0
6187c478bd9Sstevel@tonic-gate 		 *
6197c478bd9Sstevel@tonic-gate 		 * off of %g7 so we just need to place the target in %o0
6207c478bd9Sstevel@tonic-gate 		 * and set the %pc to point to the stashed return instruction.
6217c478bd9Sstevel@tonic-gate 		 * We use %o0 since that register disappears after the return
6227c478bd9Sstevel@tonic-gate 		 * executes, erasing any evidence of this tampering.
6237c478bd9Sstevel@tonic-gate 		 */
6247c478bd9Sstevel@tonic-gate 		if (I(tp->ftt_instr)) {
6257c478bd9Sstevel@tonic-gate 			int32_t imm;
6267c478bd9Sstevel@tonic-gate 
6277c478bd9Sstevel@tonic-gate 			imm = tp->ftt_instr << 19;
6287c478bd9Sstevel@tonic-gate 			imm >>= 19;
6297c478bd9Sstevel@tonic-gate 			target = fasttrap_getreg(rp, RS1(tp->ftt_instr)) + imm;
6307c478bd9Sstevel@tonic-gate 		} else {
6317c478bd9Sstevel@tonic-gate 			target = fasttrap_getreg(rp, RS1(tp->ftt_instr)) +
6327c478bd9Sstevel@tonic-gate 			    fasttrap_getreg(rp, RS2(tp->ftt_instr));
6337c478bd9Sstevel@tonic-gate 		}
6347c478bd9Sstevel@tonic-gate 
6357c478bd9Sstevel@tonic-gate 		fasttrap_putreg(rp, R_O0, target);
6367c478bd9Sstevel@tonic-gate 
6377c478bd9Sstevel@tonic-gate 		pc = rp->r_g7 + FASTTRAP_OFF_RETURN;
6387c478bd9Sstevel@tonic-gate 		fake_restore = 1;
6397c478bd9Sstevel@tonic-gate 		break;
6407c478bd9Sstevel@tonic-gate 	}
6417c478bd9Sstevel@tonic-gate 
6427c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_OR:
6437c478bd9Sstevel@tonic-gate 	{
6447c478bd9Sstevel@tonic-gate 		ulong_t value;
6457c478bd9Sstevel@tonic-gate 
6467c478bd9Sstevel@tonic-gate 		if (I(tp->ftt_instr)) {
6477c478bd9Sstevel@tonic-gate 			int32_t imm;
6487c478bd9Sstevel@tonic-gate 
6497c478bd9Sstevel@tonic-gate 			imm = tp->ftt_instr << 19;
6507c478bd9Sstevel@tonic-gate 			imm >>= 19;
6517c478bd9Sstevel@tonic-gate 			value = fasttrap_getreg(rp, RS1(tp->ftt_instr)) | imm;
6527c478bd9Sstevel@tonic-gate 		} else {
6537c478bd9Sstevel@tonic-gate 			value = fasttrap_getreg(rp, RS1(tp->ftt_instr)) |
6547c478bd9Sstevel@tonic-gate 			    fasttrap_getreg(rp, RS2(tp->ftt_instr));
6557c478bd9Sstevel@tonic-gate 		}
6567c478bd9Sstevel@tonic-gate 
6577c478bd9Sstevel@tonic-gate 		fasttrap_putreg(rp, RD(tp->ftt_instr), value);
6587c478bd9Sstevel@tonic-gate 		pc = rp->r_npc;
6597c478bd9Sstevel@tonic-gate 		npc = pc + 4;
6607c478bd9Sstevel@tonic-gate 		break;
6617c478bd9Sstevel@tonic-gate 	}
6627c478bd9Sstevel@tonic-gate 
6637c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_SETHI:
6647c478bd9Sstevel@tonic-gate 		if (RD(tp->ftt_instr) != R_G0) {
6657c478bd9Sstevel@tonic-gate 			uint32_t imm32 = tp->ftt_instr << 10;
6667c478bd9Sstevel@tonic-gate 			fasttrap_putreg(rp, RD(tp->ftt_instr), (ulong_t)imm32);
6677c478bd9Sstevel@tonic-gate 		}
6687c478bd9Sstevel@tonic-gate 		pc = rp->r_npc;
6697c478bd9Sstevel@tonic-gate 		npc = pc + 4;
6707c478bd9Sstevel@tonic-gate 		break;
6717c478bd9Sstevel@tonic-gate 
6727c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_CCR:
6737c478bd9Sstevel@tonic-gate 	{
6747c478bd9Sstevel@tonic-gate 		uint_t c, v, z, n, taken;
6757c478bd9Sstevel@tonic-gate 		uint_t ccr = rp->r_tstate >> TSTATE_CCR_SHIFT;
6767c478bd9Sstevel@tonic-gate 
6777c478bd9Sstevel@tonic-gate 		if (tp->ftt_cc != 0)
6787c478bd9Sstevel@tonic-gate 			ccr >>= 4;
6797c478bd9Sstevel@tonic-gate 
6807c478bd9Sstevel@tonic-gate 		c = (ccr >> 0) & 1;
6817c478bd9Sstevel@tonic-gate 		v = (ccr >> 1) & 1;
6827c478bd9Sstevel@tonic-gate 		z = (ccr >> 2) & 1;
6837c478bd9Sstevel@tonic-gate 		n = (ccr >> 3) & 1;
6847c478bd9Sstevel@tonic-gate 
6857c478bd9Sstevel@tonic-gate 		switch (tp->ftt_code) {
6867c478bd9Sstevel@tonic-gate 		case 0x0:	/* BN */
6877c478bd9Sstevel@tonic-gate 			taken = 0;		break;
6887c478bd9Sstevel@tonic-gate 		case 0x1:	/* BE */
6897c478bd9Sstevel@tonic-gate 			taken = z;		break;
6907c478bd9Sstevel@tonic-gate 		case 0x2:	/* BLE */
6917c478bd9Sstevel@tonic-gate 			taken = z | (n ^ v);	break;
6927c478bd9Sstevel@tonic-gate 		case 0x3:	/* BL */
6937c478bd9Sstevel@tonic-gate 			taken = n ^ v;		break;
6947c478bd9Sstevel@tonic-gate 		case 0x4:	/* BLEU */
6957c478bd9Sstevel@tonic-gate 			taken = c | z;		break;
6967c478bd9Sstevel@tonic-gate 		case 0x5:	/* BCS (BLU) */
6977c478bd9Sstevel@tonic-gate 			taken = c;		break;
6987c478bd9Sstevel@tonic-gate 		case 0x6:	/* BNEG */
6997c478bd9Sstevel@tonic-gate 			taken = n;		break;
7007c478bd9Sstevel@tonic-gate 		case 0x7:	/* BVS */
7017c478bd9Sstevel@tonic-gate 			taken = v;		break;
7027c478bd9Sstevel@tonic-gate 		case 0x8:	/* BA */
7037c478bd9Sstevel@tonic-gate 			/*
7047c478bd9Sstevel@tonic-gate 			 * We handle the BA case differently since the annul
7057c478bd9Sstevel@tonic-gate 			 * bit means something slightly different.
7067c478bd9Sstevel@tonic-gate 			 */
7077c478bd9Sstevel@tonic-gate 			panic("fasttrap: mishandled a branch");
7087c478bd9Sstevel@tonic-gate 			taken = 1;		break;
7097c478bd9Sstevel@tonic-gate 		case 0x9:	/* BNE */
7107c478bd9Sstevel@tonic-gate 			taken = ~z;		break;
7117c478bd9Sstevel@tonic-gate 		case 0xa:	/* BG */
7127c478bd9Sstevel@tonic-gate 			taken = ~(z | (n ^ v));	break;
7137c478bd9Sstevel@tonic-gate 		case 0xb:	/* BGE */
7147c478bd9Sstevel@tonic-gate 			taken = ~(n ^ v);	break;
7157c478bd9Sstevel@tonic-gate 		case 0xc:	/* BGU */
7167c478bd9Sstevel@tonic-gate 			taken = ~(c | z);	break;
7177c478bd9Sstevel@tonic-gate 		case 0xd:	/* BCC (BGEU) */
7187c478bd9Sstevel@tonic-gate 			taken = ~c;		break;
7197c478bd9Sstevel@tonic-gate 		case 0xe:	/* BPOS */
7207c478bd9Sstevel@tonic-gate 			taken = ~n;		break;
7217c478bd9Sstevel@tonic-gate 		case 0xf:	/* BVC */
7227c478bd9Sstevel@tonic-gate 			taken = ~v;		break;
7237c478bd9Sstevel@tonic-gate 		}
7247c478bd9Sstevel@tonic-gate 
7257c478bd9Sstevel@tonic-gate 		if (taken & 1) {
7267c478bd9Sstevel@tonic-gate 			pc = rp->r_npc;
7277c478bd9Sstevel@tonic-gate 			npc = tp->ftt_dest;
7287c478bd9Sstevel@tonic-gate 		} else if (tp->ftt_flags & FASTTRAP_F_ANNUL) {
7297c478bd9Sstevel@tonic-gate 			/*
7307c478bd9Sstevel@tonic-gate 			 * Untaken annulled branches don't execute the
7317c478bd9Sstevel@tonic-gate 			 * instruction in the delay slot.
7327c478bd9Sstevel@tonic-gate 			 */
7337c478bd9Sstevel@tonic-gate 			pc = rp->r_npc + 4;
7347c478bd9Sstevel@tonic-gate 			npc = pc + 4;
7357c478bd9Sstevel@tonic-gate 		} else {
7367c478bd9Sstevel@tonic-gate 			pc = rp->r_npc;
7377c478bd9Sstevel@tonic-gate 			npc = pc + 4;
7387c478bd9Sstevel@tonic-gate 		}
7397c478bd9Sstevel@tonic-gate 		break;
7407c478bd9Sstevel@tonic-gate 	}
7417c478bd9Sstevel@tonic-gate 
7427c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_FCC:
7437c478bd9Sstevel@tonic-gate 	{
7447c478bd9Sstevel@tonic-gate 		uint_t fcc;
7457c478bd9Sstevel@tonic-gate 		uint_t taken;
7467c478bd9Sstevel@tonic-gate 		uint64_t fsr;
7477c478bd9Sstevel@tonic-gate 
7487c478bd9Sstevel@tonic-gate 		dtrace_getfsr(&fsr);
7497c478bd9Sstevel@tonic-gate 
7507c478bd9Sstevel@tonic-gate 		if (tp->ftt_cc == 0) {
7517c478bd9Sstevel@tonic-gate 			fcc = (fsr >> 10) & 0x3;
7527c478bd9Sstevel@tonic-gate 		} else {
7537c478bd9Sstevel@tonic-gate 			uint_t shift;
7547c478bd9Sstevel@tonic-gate 			ASSERT(tp->ftt_cc <= 3);
7557c478bd9Sstevel@tonic-gate 			shift = 30 + tp->ftt_cc * 2;
7567c478bd9Sstevel@tonic-gate 			fcc = (fsr >> shift) & 0x3;
7577c478bd9Sstevel@tonic-gate 		}
7587c478bd9Sstevel@tonic-gate 
7597c478bd9Sstevel@tonic-gate 		switch (tp->ftt_code) {
7607c478bd9Sstevel@tonic-gate 		case 0x0:	/* FBN */
7617c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (0|0|0|0);	break;
7627c478bd9Sstevel@tonic-gate 		case 0x1:	/* FBNE */
7637c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (8|4|2|0);	break;
7647c478bd9Sstevel@tonic-gate 		case 0x2:	/* FBLG */
7657c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (0|4|2|0);	break;
7667c478bd9Sstevel@tonic-gate 		case 0x3:	/* FBUL */
7677c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (8|0|2|0);	break;
7687c478bd9Sstevel@tonic-gate 		case 0x4:	/* FBL */
7697c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (0|0|2|0);	break;
7707c478bd9Sstevel@tonic-gate 		case 0x5:	/* FBUG */
7717c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (8|4|0|0);	break;
7727c478bd9Sstevel@tonic-gate 		case 0x6:	/* FBG */
7737c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (0|4|0|0);	break;
7747c478bd9Sstevel@tonic-gate 		case 0x7:	/* FBU */
7757c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (8|0|0|0);	break;
7767c478bd9Sstevel@tonic-gate 		case 0x8:	/* FBA */
7777c478bd9Sstevel@tonic-gate 			/*
7787c478bd9Sstevel@tonic-gate 			 * We handle the FBA case differently since the annul
7797c478bd9Sstevel@tonic-gate 			 * bit means something slightly different.
7807c478bd9Sstevel@tonic-gate 			 */
7817c478bd9Sstevel@tonic-gate 			panic("fasttrap: mishandled a branch");
7827c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (8|4|2|1);	break;
7837c478bd9Sstevel@tonic-gate 		case 0x9:	/* FBE */
7847c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (0|0|0|1);	break;
7857c478bd9Sstevel@tonic-gate 		case 0xa:	/* FBUE */
7867c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (8|0|0|1);	break;
7877c478bd9Sstevel@tonic-gate 		case 0xb:	/* FBGE */
7887c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (0|4|0|1);	break;
7897c478bd9Sstevel@tonic-gate 		case 0xc:	/* FBUGE */
7907c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (8|4|0|1);	break;
7917c478bd9Sstevel@tonic-gate 		case 0xd:	/* FBLE */
7927c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (0|0|2|1);	break;
7937c478bd9Sstevel@tonic-gate 		case 0xe:	/* FBULE */
7947c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (8|0|2|1);	break;
7957c478bd9Sstevel@tonic-gate 		case 0xf:	/* FBO */
7967c478bd9Sstevel@tonic-gate 			taken = (1 << fcc) & (0|4|2|1);	break;
7977c478bd9Sstevel@tonic-gate 		}
7987c478bd9Sstevel@tonic-gate 
7997c478bd9Sstevel@tonic-gate 		if (taken) {
8007c478bd9Sstevel@tonic-gate 			pc = rp->r_npc;
8017c478bd9Sstevel@tonic-gate 			npc = tp->ftt_dest;
8027c478bd9Sstevel@tonic-gate 		} else if (tp->ftt_flags & FASTTRAP_F_ANNUL) {
8037c478bd9Sstevel@tonic-gate 			/*
8047c478bd9Sstevel@tonic-gate 			 * Untaken annulled branches don't execute the
8057c478bd9Sstevel@tonic-gate 			 * instruction in the delay slot.
8067c478bd9Sstevel@tonic-gate 			 */
8077c478bd9Sstevel@tonic-gate 			pc = rp->r_npc + 4;
8087c478bd9Sstevel@tonic-gate 			npc = pc + 4;
8097c478bd9Sstevel@tonic-gate 		} else {
8107c478bd9Sstevel@tonic-gate 			pc = rp->r_npc;
8117c478bd9Sstevel@tonic-gate 			npc = pc + 4;
8127c478bd9Sstevel@tonic-gate 		}
8137c478bd9Sstevel@tonic-gate 		break;
8147c478bd9Sstevel@tonic-gate 	}
8157c478bd9Sstevel@tonic-gate 
8167c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_REG:
8177c478bd9Sstevel@tonic-gate 	{
81873427c57Sahl 		int64_t value;
8197c478bd9Sstevel@tonic-gate 		uint_t taken;
8207c478bd9Sstevel@tonic-gate 		uint_t reg = RS1(tp->ftt_instr);
8217c478bd9Sstevel@tonic-gate 
8227c478bd9Sstevel@tonic-gate 		/*
8237c478bd9Sstevel@tonic-gate 		 * An ILP32 process shouldn't be using a branch predicated on
8247c478bd9Sstevel@tonic-gate 		 * an %i or an %l since it would violate the ABI. It's a
8257c478bd9Sstevel@tonic-gate 		 * violation of the ABI because we can't ensure deterministic
8267c478bd9Sstevel@tonic-gate 		 * behavior. We should have identified this case when we
8277c478bd9Sstevel@tonic-gate 		 * enabled the probe.
8287c478bd9Sstevel@tonic-gate 		 */
8297c478bd9Sstevel@tonic-gate 		ASSERT(p->p_model == DATAMODEL_LP64 || reg < 16);
8307c478bd9Sstevel@tonic-gate 
83173427c57Sahl 		value = (int64_t)fasttrap_getreg(rp, reg);
8327c478bd9Sstevel@tonic-gate 
8337c478bd9Sstevel@tonic-gate 		switch (tp->ftt_code) {
8347c478bd9Sstevel@tonic-gate 		case 0x1:	/* BRZ */
8357c478bd9Sstevel@tonic-gate 			taken = (value == 0);	break;
8367c478bd9Sstevel@tonic-gate 		case 0x2:	/* BRLEZ */
8377c478bd9Sstevel@tonic-gate 			taken = (value <= 0);	break;
8387c478bd9Sstevel@tonic-gate 		case 0x3:	/* BRLZ */
8397c478bd9Sstevel@tonic-gate 			taken = (value < 0);	break;
8407c478bd9Sstevel@tonic-gate 		case 0x5:	/* BRNZ */
8417c478bd9Sstevel@tonic-gate 			taken = (value != 0);	break;
8427c478bd9Sstevel@tonic-gate 		case 0x6:	/* BRGZ */
8437c478bd9Sstevel@tonic-gate 			taken = (value > 0);	break;
8447c478bd9Sstevel@tonic-gate 		case 0x7:	/* BRGEZ */
84573427c57Sahl 			taken = (value >= 0);	break;
8467c478bd9Sstevel@tonic-gate 		default:
8477c478bd9Sstevel@tonic-gate 		case 0x0:
8487c478bd9Sstevel@tonic-gate 		case 0x4:
8497c478bd9Sstevel@tonic-gate 			panic("fasttrap: mishandled a branch");
8507c478bd9Sstevel@tonic-gate 		}
8517c478bd9Sstevel@tonic-gate 
8527c478bd9Sstevel@tonic-gate 		if (taken) {
8537c478bd9Sstevel@tonic-gate 			pc = rp->r_npc;
8547c478bd9Sstevel@tonic-gate 			npc = tp->ftt_dest;
8557c478bd9Sstevel@tonic-gate 		} else if (tp->ftt_flags & FASTTRAP_F_ANNUL) {
8567c478bd9Sstevel@tonic-gate 			/*
8577c478bd9Sstevel@tonic-gate 			 * Untaken annulled branches don't execute the
8587c478bd9Sstevel@tonic-gate 			 * instruction in the delay slot.
8597c478bd9Sstevel@tonic-gate 			 */
8607c478bd9Sstevel@tonic-gate 			pc = rp->r_npc + 4;
8617c478bd9Sstevel@tonic-gate 			npc = pc + 4;
8627c478bd9Sstevel@tonic-gate 		} else {
8637c478bd9Sstevel@tonic-gate 			pc = rp->r_npc;
8647c478bd9Sstevel@tonic-gate 			npc = pc + 4;
8657c478bd9Sstevel@tonic-gate 		}
8667c478bd9Sstevel@tonic-gate 		break;
8677c478bd9Sstevel@tonic-gate 	}
8687c478bd9Sstevel@tonic-gate 
8697c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_ALWAYS:
8707c478bd9Sstevel@tonic-gate 		/*
8717c478bd9Sstevel@tonic-gate 		 * BAs, BA,As...
8727c478bd9Sstevel@tonic-gate 		 */
8737c478bd9Sstevel@tonic-gate 
8747c478bd9Sstevel@tonic-gate 		if (tp->ftt_flags & FASTTRAP_F_ANNUL) {
8757c478bd9Sstevel@tonic-gate 			/*
8767c478bd9Sstevel@tonic-gate 			 * Annulled branch always instructions never execute
8777c478bd9Sstevel@tonic-gate 			 * the instruction in the delay slot.
8787c478bd9Sstevel@tonic-gate 			 */
8797c478bd9Sstevel@tonic-gate 			pc = tp->ftt_dest;
8807c478bd9Sstevel@tonic-gate 			npc = tp->ftt_dest + 4;
8817c478bd9Sstevel@tonic-gate 		} else {
8827c478bd9Sstevel@tonic-gate 			pc = rp->r_npc;
8837c478bd9Sstevel@tonic-gate 			npc = tp->ftt_dest;
8847c478bd9Sstevel@tonic-gate 		}
8857c478bd9Sstevel@tonic-gate 		break;
8867c478bd9Sstevel@tonic-gate 
8877c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_RDPC:
8887c478bd9Sstevel@tonic-gate 		fasttrap_putreg(rp, RD(tp->ftt_instr), rp->r_pc);
8897c478bd9Sstevel@tonic-gate 		pc = rp->r_npc;
8907c478bd9Sstevel@tonic-gate 		npc = pc + 4;
8917c478bd9Sstevel@tonic-gate 		break;
8927c478bd9Sstevel@tonic-gate 
8937c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_CALL:
8947c478bd9Sstevel@tonic-gate 		/*
8957c478bd9Sstevel@tonic-gate 		 * It's a call _and_ link remember...
8967c478bd9Sstevel@tonic-gate 		 */
8977c478bd9Sstevel@tonic-gate 		rp->r_o7 = rp->r_pc;
8987c478bd9Sstevel@tonic-gate 		pc = rp->r_npc;
8997c478bd9Sstevel@tonic-gate 		npc = tp->ftt_dest;
9007c478bd9Sstevel@tonic-gate 		break;
9017c478bd9Sstevel@tonic-gate 
9027c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_JMPL:
9037c478bd9Sstevel@tonic-gate 		pc = rp->r_npc;
9047c478bd9Sstevel@tonic-gate 
9057c478bd9Sstevel@tonic-gate 		if (I(tp->ftt_instr)) {
9067c478bd9Sstevel@tonic-gate 			uint_t rs1 = RS1(tp->ftt_instr);
9077c478bd9Sstevel@tonic-gate 			int32_t imm;
9087c478bd9Sstevel@tonic-gate 
9097c478bd9Sstevel@tonic-gate 			imm = tp->ftt_instr << 19;
9107c478bd9Sstevel@tonic-gate 			imm >>= 19;
9117c478bd9Sstevel@tonic-gate 			npc = fasttrap_getreg(rp, rs1) + imm;
9127c478bd9Sstevel@tonic-gate 		} else {
9137c478bd9Sstevel@tonic-gate 			uint_t rs1 = RS1(tp->ftt_instr);
9147c478bd9Sstevel@tonic-gate 			uint_t rs2 = RS2(tp->ftt_instr);
9157c478bd9Sstevel@tonic-gate 
9167c478bd9Sstevel@tonic-gate 			npc = fasttrap_getreg(rp, rs1) +
9177c478bd9Sstevel@tonic-gate 			    fasttrap_getreg(rp, rs2);
9187c478bd9Sstevel@tonic-gate 		}
9197c478bd9Sstevel@tonic-gate 
9207c478bd9Sstevel@tonic-gate 		/*
9217c478bd9Sstevel@tonic-gate 		 * Do the link part of the jump-and-link instruction.
9227c478bd9Sstevel@tonic-gate 		 */
9237c478bd9Sstevel@tonic-gate 		fasttrap_putreg(rp, RD(tp->ftt_instr), rp->r_pc);
9247c478bd9Sstevel@tonic-gate 
9257c478bd9Sstevel@tonic-gate 		break;
9267c478bd9Sstevel@tonic-gate 
9277c478bd9Sstevel@tonic-gate 	case FASTTRAP_T_COMMON:
9287c478bd9Sstevel@tonic-gate 	{
9297c478bd9Sstevel@tonic-gate 		curthread->t_dtrace_scrpc = rp->r_g7;
9307c478bd9Sstevel@tonic-gate 		curthread->t_dtrace_astpc = rp->r_g7 + FASTTRAP_OFF_FTRET;
9317c478bd9Sstevel@tonic-gate 
9327c478bd9Sstevel@tonic-gate 		/*
9337c478bd9Sstevel@tonic-gate 		 * Copy the instruction to a reserved location in the
9347c478bd9Sstevel@tonic-gate 		 * user-land thread structure, then set the PC to that
9357c478bd9Sstevel@tonic-gate 		 * location and leave the NPC alone. We take pains to ensure
9367c478bd9Sstevel@tonic-gate 		 * consistency in the instruction stream (See SPARC
9377c478bd9Sstevel@tonic-gate 		 * Architecture Manual Version 9, sections 8.4.7, A.20, and
9387c478bd9Sstevel@tonic-gate 		 * H.1.6; UltraSPARC I/II User's Manual, sections 3.1.1.1,
9397c478bd9Sstevel@tonic-gate 		 * and 13.6.4) by using the ASI ASI_BLK_COMMIT_S to copy the
9407c478bd9Sstevel@tonic-gate 		 * instruction into the user's address space without
9417c478bd9Sstevel@tonic-gate 		 * bypassing the I$. There's no AS_USER version of this ASI
9427c478bd9Sstevel@tonic-gate 		 * (as exist for other ASIs) so we use the lofault
9437c478bd9Sstevel@tonic-gate 		 * mechanism to catch faults.
9447c478bd9Sstevel@tonic-gate 		 */
9457c478bd9Sstevel@tonic-gate 		if (dtrace_blksuword32(rp->r_g7, &tp->ftt_instr, 1) == -1) {
9467c478bd9Sstevel@tonic-gate 			/*
9477c478bd9Sstevel@tonic-gate 			 * If the copyout fails, then the process's state
9487c478bd9Sstevel@tonic-gate 			 * is not consistent (the effects of the traced
9497c478bd9Sstevel@tonic-gate 			 * instruction will never be seen). This process
9507c478bd9Sstevel@tonic-gate 			 * cannot be allowed to continue execution.
9517c478bd9Sstevel@tonic-gate 			 */
9527c478bd9Sstevel@tonic-gate 			fasttrap_sigtrap(curproc, curthread, pc);
9537c478bd9Sstevel@tonic-gate 			return (0);
9547c478bd9Sstevel@tonic-gate 		}
9557c478bd9Sstevel@tonic-gate 
9567c478bd9Sstevel@tonic-gate 		curthread->t_dtrace_pc = pc;
9577c478bd9Sstevel@tonic-gate 		curthread->t_dtrace_npc = npc;
9587c478bd9Sstevel@tonic-gate 		curthread->t_dtrace_on = 1;
9597c478bd9Sstevel@tonic-gate 
9607c478bd9Sstevel@tonic-gate 		pc = curthread->t_dtrace_scrpc;
9617c478bd9Sstevel@tonic-gate 
9627c478bd9Sstevel@tonic-gate 		if (tp->ftt_retids != NULL) {
9637c478bd9Sstevel@tonic-gate 			curthread->t_dtrace_step = 1;
9647c478bd9Sstevel@tonic-gate 			curthread->t_dtrace_ret = 1;
9657c478bd9Sstevel@tonic-gate 			npc = curthread->t_dtrace_astpc;
9667c478bd9Sstevel@tonic-gate 		}
9677c478bd9Sstevel@tonic-gate 		break;
9687c478bd9Sstevel@tonic-gate 	}
9697c478bd9Sstevel@tonic-gate 
9707c478bd9Sstevel@tonic-gate 	default:
9717c478bd9Sstevel@tonic-gate 		panic("fasttrap: mishandled an instruction");
9727c478bd9Sstevel@tonic-gate 	}
9737c478bd9Sstevel@tonic-gate 
9747c478bd9Sstevel@tonic-gate 	/*
9757c478bd9Sstevel@tonic-gate 	 * This bit me in the ass a couple of times, so lets toss this
9767c478bd9Sstevel@tonic-gate 	 * in as a cursory sanity check.
9777c478bd9Sstevel@tonic-gate 	 */
9787c478bd9Sstevel@tonic-gate 	ASSERT(pc != rp->r_g7 + 4);
9797c478bd9Sstevel@tonic-gate 	ASSERT(pc != rp->r_g7 + 8);
9807c478bd9Sstevel@tonic-gate 
981ac448965Sahl done:
9827c478bd9Sstevel@tonic-gate 	/*
9837c478bd9Sstevel@tonic-gate 	 * If there were no return probes when we first found the tracepoint,
9847c478bd9Sstevel@tonic-gate 	 * we should feel no obligation to honor any return probes that were
9857c478bd9Sstevel@tonic-gate 	 * subsequently enabled -- they'll just have to wait until the next
9867c478bd9Sstevel@tonic-gate 	 * time around.
9877c478bd9Sstevel@tonic-gate 	 */
9887c478bd9Sstevel@tonic-gate 	if (tp->ftt_retids != NULL) {
9897c478bd9Sstevel@tonic-gate 		/*
9907c478bd9Sstevel@tonic-gate 		 * We need to wait until the results of the instruction are
9917c478bd9Sstevel@tonic-gate 		 * apparent before invoking any return probes. If this
9927c478bd9Sstevel@tonic-gate 		 * instruction was emulated we can just call
9937c478bd9Sstevel@tonic-gate 		 * fasttrap_return_common(); if it needs to be executed, we
9947c478bd9Sstevel@tonic-gate 		 * need to wait until we return to the kernel.
9957c478bd9Sstevel@tonic-gate 		 */
9967c478bd9Sstevel@tonic-gate 		if (tp->ftt_type != FASTTRAP_T_COMMON) {
9977c478bd9Sstevel@tonic-gate 			fasttrap_return_common(rp, orig_pc, pid, fake_restore);
9987c478bd9Sstevel@tonic-gate 		} else {
9997c478bd9Sstevel@tonic-gate 			ASSERT(curthread->t_dtrace_ret != 0);
10007c478bd9Sstevel@tonic-gate 			ASSERT(curthread->t_dtrace_pc == orig_pc);
10017c478bd9Sstevel@tonic-gate 			ASSERT(curthread->t_dtrace_scrpc == rp->r_g7);
10027c478bd9Sstevel@tonic-gate 			ASSERT(npc == curthread->t_dtrace_astpc);
10037c478bd9Sstevel@tonic-gate 		}
10047c478bd9Sstevel@tonic-gate 	}
10057c478bd9Sstevel@tonic-gate 
10067c478bd9Sstevel@tonic-gate 	ASSERT(pc != 0);
10077c478bd9Sstevel@tonic-gate 	rp->r_pc = pc;
10087c478bd9Sstevel@tonic-gate 	rp->r_npc = npc;
10097c478bd9Sstevel@tonic-gate 
10107c478bd9Sstevel@tonic-gate 	return (0);
10117c478bd9Sstevel@tonic-gate }
10127c478bd9Sstevel@tonic-gate 
10137c478bd9Sstevel@tonic-gate int
fasttrap_return_probe(struct regs * rp)10147c478bd9Sstevel@tonic-gate fasttrap_return_probe(struct regs *rp)
10157c478bd9Sstevel@tonic-gate {
10167c478bd9Sstevel@tonic-gate 	proc_t *p = ttoproc(curthread);
10177c478bd9Sstevel@tonic-gate 	pid_t pid;
10187c478bd9Sstevel@tonic-gate 	uintptr_t pc = curthread->t_dtrace_pc;
10197c478bd9Sstevel@tonic-gate 	uintptr_t npc = curthread->t_dtrace_npc;
10207c478bd9Sstevel@tonic-gate 
10217c478bd9Sstevel@tonic-gate 	curthread->t_dtrace_pc = 0;
10227c478bd9Sstevel@tonic-gate 	curthread->t_dtrace_npc = 0;
10237c478bd9Sstevel@tonic-gate 	curthread->t_dtrace_scrpc = 0;
10247c478bd9Sstevel@tonic-gate 	curthread->t_dtrace_astpc = 0;
10257c478bd9Sstevel@tonic-gate 
10267c478bd9Sstevel@tonic-gate 	/*
10277c478bd9Sstevel@tonic-gate 	 * Treat a child created by a call to vfork(2) as if it were its
10287c478bd9Sstevel@tonic-gate 	 * parent. We know there's only one thread of control in such a
10297c478bd9Sstevel@tonic-gate 	 * process: this one.
10307c478bd9Sstevel@tonic-gate 	 */
10317c478bd9Sstevel@tonic-gate 	while (p->p_flag & SVFORK) {
10327c478bd9Sstevel@tonic-gate 		p = p->p_parent;
10337c478bd9Sstevel@tonic-gate 	}
10347c478bd9Sstevel@tonic-gate 
10357c478bd9Sstevel@tonic-gate 	/*
10367c478bd9Sstevel@tonic-gate 	 * We set the %pc and %npc to their values when the traced
10377c478bd9Sstevel@tonic-gate 	 * instruction was initially executed so that it appears to
10387c478bd9Sstevel@tonic-gate 	 * dtrace_probe() that we're on the original instruction, and so that
10397c478bd9Sstevel@tonic-gate 	 * the user can't easily detect our complex web of lies.
10407c478bd9Sstevel@tonic-gate 	 * dtrace_return_probe() (our caller) will correctly set %pc and %npc
10417c478bd9Sstevel@tonic-gate 	 * after we return.
10427c478bd9Sstevel@tonic-gate 	 */
10437c478bd9Sstevel@tonic-gate 	rp->r_pc = pc;
10447c478bd9Sstevel@tonic-gate 	rp->r_npc = npc;
10457c478bd9Sstevel@tonic-gate 
10467c478bd9Sstevel@tonic-gate 	pid = p->p_pid;
10477c478bd9Sstevel@tonic-gate 	fasttrap_return_common(rp, pc, pid, 0);
10487c478bd9Sstevel@tonic-gate 
10497c478bd9Sstevel@tonic-gate 	return (0);
10507c478bd9Sstevel@tonic-gate }
10517c478bd9Sstevel@tonic-gate 
10527c478bd9Sstevel@tonic-gate int
fasttrap_tracepoint_install(proc_t * p,fasttrap_tracepoint_t * tp)10537c478bd9Sstevel@tonic-gate fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp)
10547c478bd9Sstevel@tonic-gate {
10557c478bd9Sstevel@tonic-gate 	fasttrap_instr_t instr = FASTTRAP_INSTR;
10567c478bd9Sstevel@tonic-gate 
10577c478bd9Sstevel@tonic-gate 	if (uwrite(p, &instr, 4, tp->ftt_pc) != 0)
10587c478bd9Sstevel@tonic-gate 		return (-1);
10597c478bd9Sstevel@tonic-gate 
10607c478bd9Sstevel@tonic-gate 	return (0);
10617c478bd9Sstevel@tonic-gate }
10627c478bd9Sstevel@tonic-gate 
10637c478bd9Sstevel@tonic-gate int
fasttrap_tracepoint_remove(proc_t * p,fasttrap_tracepoint_t * tp)10647c478bd9Sstevel@tonic-gate fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp)
10657c478bd9Sstevel@tonic-gate {
10667c478bd9Sstevel@tonic-gate 	fasttrap_instr_t instr;
10677c478bd9Sstevel@tonic-gate 
10687c478bd9Sstevel@tonic-gate 	/*
10697c478bd9Sstevel@tonic-gate 	 * Distinguish between read or write failures and a changed
10707c478bd9Sstevel@tonic-gate 	 * instruction.
10717c478bd9Sstevel@tonic-gate 	 */
10727c478bd9Sstevel@tonic-gate 	if (uread(p, &instr, 4, tp->ftt_pc) != 0)
10737c478bd9Sstevel@tonic-gate 		return (0);
10747c478bd9Sstevel@tonic-gate 	if (instr != FASTTRAP_INSTR && instr != BREAKPOINT_INSTR)
10757c478bd9Sstevel@tonic-gate 		return (0);
10767c478bd9Sstevel@tonic-gate 	if (uwrite(p, &tp->ftt_instr, 4, tp->ftt_pc) != 0)
10777c478bd9Sstevel@tonic-gate 		return (-1);
10787c478bd9Sstevel@tonic-gate 
10797c478bd9Sstevel@tonic-gate 	return (0);
10807c478bd9Sstevel@tonic-gate }
10817c478bd9Sstevel@tonic-gate 
10827c478bd9Sstevel@tonic-gate int
fasttrap_tracepoint_init(proc_t * p,fasttrap_tracepoint_t * tp,uintptr_t pc,fasttrap_probe_type_t type)1083ac448965Sahl fasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, uintptr_t pc,
1084ac448965Sahl     fasttrap_probe_type_t type)
10857c478bd9Sstevel@tonic-gate {
10867c478bd9Sstevel@tonic-gate 	uint32_t instr;
10877c478bd9Sstevel@tonic-gate 	int32_t disp;
10887c478bd9Sstevel@tonic-gate 
10897c478bd9Sstevel@tonic-gate 	/*
10907c478bd9Sstevel@tonic-gate 	 * Read the instruction at the given address out of the process's
10917c478bd9Sstevel@tonic-gate 	 * address space. We don't have to worry about a debugger
10927c478bd9Sstevel@tonic-gate 	 * changing this instruction before we overwrite it with our trap
10937c478bd9Sstevel@tonic-gate 	 * instruction since P_PR_LOCK is set.
10947c478bd9Sstevel@tonic-gate 	 */
10957c478bd9Sstevel@tonic-gate 	if (uread(p, &instr, 4, pc) != 0)
10967c478bd9Sstevel@tonic-gate 		return (-1);
10977c478bd9Sstevel@tonic-gate 
10987c478bd9Sstevel@tonic-gate 	/*
10997c478bd9Sstevel@tonic-gate 	 * Decode the instruction to fill in the probe flags. We can have
11007c478bd9Sstevel@tonic-gate 	 * the process execute most instructions on its own using a pc/npc
11017c478bd9Sstevel@tonic-gate 	 * trick, but pc-relative control transfer present a problem since
11027c478bd9Sstevel@tonic-gate 	 * we're relocating the instruction. We emulate these instructions
11037c478bd9Sstevel@tonic-gate 	 * in the kernel. We assume a default type and over-write that as
11047c478bd9Sstevel@tonic-gate 	 * needed.
11057c478bd9Sstevel@tonic-gate 	 *
11067c478bd9Sstevel@tonic-gate 	 * pc-relative instructions must be emulated for correctness;
11077c478bd9Sstevel@tonic-gate 	 * other instructions (which represent a large set of commonly traced
11087c478bd9Sstevel@tonic-gate 	 * instructions) are emulated or otherwise optimized for performance.
11097c478bd9Sstevel@tonic-gate 	 */
11107c478bd9Sstevel@tonic-gate 	tp->ftt_type = FASTTRAP_T_COMMON;
11117c478bd9Sstevel@tonic-gate 	if (OP(instr) == 1) {
11127c478bd9Sstevel@tonic-gate 		/*
11137c478bd9Sstevel@tonic-gate 		 * Call instructions.
11147c478bd9Sstevel@tonic-gate 		 */
11157c478bd9Sstevel@tonic-gate 		tp->ftt_type = FASTTRAP_T_CALL;
11167c478bd9Sstevel@tonic-gate 		disp = DISP30(instr) << 2;
11177c478bd9Sstevel@tonic-gate 		tp->ftt_dest = pc + (intptr_t)disp;
11187c478bd9Sstevel@tonic-gate 
11197c478bd9Sstevel@tonic-gate 	} else if (OP(instr) == 0) {
11207c478bd9Sstevel@tonic-gate 		/*
11217c478bd9Sstevel@tonic-gate 		 * Branch instructions.
11227c478bd9Sstevel@tonic-gate 		 *
11237c478bd9Sstevel@tonic-gate 		 * Unconditional branches need careful attention when they're
11247c478bd9Sstevel@tonic-gate 		 * annulled: annulled unconditional branches never execute
11257c478bd9Sstevel@tonic-gate 		 * the instruction in the delay slot.
11267c478bd9Sstevel@tonic-gate 		 */
11277c478bd9Sstevel@tonic-gate 		switch (OP2(instr)) {
11287c478bd9Sstevel@tonic-gate 		case OP2_ILLTRAP:
11297c478bd9Sstevel@tonic-gate 		case 0x7:
11307c478bd9Sstevel@tonic-gate 			/*
11317c478bd9Sstevel@tonic-gate 			 * The compiler may place an illtrap after a call to
11327c478bd9Sstevel@tonic-gate 			 * a function that returns a structure. In the case of
11337c478bd9Sstevel@tonic-gate 			 * a returned structure, the compiler places an illtrap
11347c478bd9Sstevel@tonic-gate 			 * whose const22 field is the size of the returned
11357c478bd9Sstevel@tonic-gate 			 * structure immediately following the delay slot of
11367c478bd9Sstevel@tonic-gate 			 * the call. To stay out of the way, we refuse to
11377c478bd9Sstevel@tonic-gate 			 * place tracepoints on top of illtrap instructions.
11387c478bd9Sstevel@tonic-gate 			 *
11397c478bd9Sstevel@tonic-gate 			 * This is one of the dumbest architectural decisions
11407c478bd9Sstevel@tonic-gate 			 * I've ever had to work around.
11417c478bd9Sstevel@tonic-gate 			 *
11427c478bd9Sstevel@tonic-gate 			 * We also identify the only illegal op2 value (See
11437c478bd9Sstevel@tonic-gate 			 * SPARC Architecture Manual Version 9, E.2 table 31).
11447c478bd9Sstevel@tonic-gate 			 */
11457c478bd9Sstevel@tonic-gate 			return (-1);
11467c478bd9Sstevel@tonic-gate 
11477c478bd9Sstevel@tonic-gate 		case OP2_BPcc:
11487c478bd9Sstevel@tonic-gate 			if (COND(instr) == 8) {
11497c478bd9Sstevel@tonic-gate 				tp->ftt_type = FASTTRAP_T_ALWAYS;
11507c478bd9Sstevel@tonic-gate 			} else {
11517c478bd9Sstevel@tonic-gate 				/*
11527c478bd9Sstevel@tonic-gate 				 * Check for an illegal instruction.
11537c478bd9Sstevel@tonic-gate 				 */
11547c478bd9Sstevel@tonic-gate 				if (CC(instr) & 1)
11557c478bd9Sstevel@tonic-gate 					return (-1);
1156