xref: /illumos-gate/usr/src/uts/sparc/dtrace/fbt.c (revision 692bcae7)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5ad4023c4Sdp  * Common Development and Distribution License (the "License").
6ad4023c4Sdp  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22b9e93c10SJonathan Haslam  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate 
277c478bd9Sstevel@tonic-gate #include <sys/errno.h>
287c478bd9Sstevel@tonic-gate #include <sys/stat.h>
297c478bd9Sstevel@tonic-gate #include <sys/modctl.h>
307c478bd9Sstevel@tonic-gate #include <sys/conf.h>
317c478bd9Sstevel@tonic-gate #include <sys/systm.h>
327c478bd9Sstevel@tonic-gate #include <sys/ddi.h>
337c478bd9Sstevel@tonic-gate #include <sys/sunddi.h>
347c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h>
357c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
367c478bd9Sstevel@tonic-gate #include <sys/strsubr.h>
377c478bd9Sstevel@tonic-gate #include <sys/dtrace.h>
387c478bd9Sstevel@tonic-gate #include <sys/kobj.h>
397c478bd9Sstevel@tonic-gate #include <sys/modctl.h>
407c478bd9Sstevel@tonic-gate #include <sys/atomic.h>
417c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
427c478bd9Sstevel@tonic-gate #include <sys/stack.h>
437c478bd9Sstevel@tonic-gate #include <sys/ctf_api.h>
447c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
457c478bd9Sstevel@tonic-gate 
467c478bd9Sstevel@tonic-gate static dev_info_t		*fbt_devi;
477c478bd9Sstevel@tonic-gate static dtrace_provider_id_t	fbt_id;
487c478bd9Sstevel@tonic-gate static uintptr_t		fbt_trampoline;
497c478bd9Sstevel@tonic-gate static caddr_t			fbt_trampoline_window;
507c478bd9Sstevel@tonic-gate static size_t			fbt_trampoline_size;
517c478bd9Sstevel@tonic-gate static int			fbt_verbose = 0;
527c478bd9Sstevel@tonic-gate 
537c478bd9Sstevel@tonic-gate /*
547c478bd9Sstevel@tonic-gate  * Various interesting bean counters.
557c478bd9Sstevel@tonic-gate  */
567c478bd9Sstevel@tonic-gate static int			fbt_entry;
577c478bd9Sstevel@tonic-gate static int			fbt_ret;
587c478bd9Sstevel@tonic-gate static int			fbt_retl;
597c478bd9Sstevel@tonic-gate static int			fbt_retl_jmptab;
607c478bd9Sstevel@tonic-gate static int			fbt_retl_twoinstr;
617c478bd9Sstevel@tonic-gate static int			fbt_retl_tailcall;
627c478bd9Sstevel@tonic-gate static int			fbt_retl_tailjmpl;
637c478bd9Sstevel@tonic-gate static int			fbt_leaf_functions;
647c478bd9Sstevel@tonic-gate 
657c478bd9Sstevel@tonic-gate extern char			stubs_base[];
667c478bd9Sstevel@tonic-gate extern char			stubs_end[];
677c478bd9Sstevel@tonic-gate 
687c478bd9Sstevel@tonic-gate #define	FBT_REG_G0		0
697c478bd9Sstevel@tonic-gate #define	FBT_REG_G1		1
707c478bd9Sstevel@tonic-gate #define	FBT_REG_O0		8
717c478bd9Sstevel@tonic-gate #define	FBT_REG_O1		9
727c478bd9Sstevel@tonic-gate #define	FBT_REG_O2		10
737c478bd9Sstevel@tonic-gate #define	FBT_REG_O3		11
747c478bd9Sstevel@tonic-gate #define	FBT_REG_O4		12
757c478bd9Sstevel@tonic-gate #define	FBT_REG_O5		13
767c478bd9Sstevel@tonic-gate #define	FBT_REG_O6		14
777c478bd9Sstevel@tonic-gate #define	FBT_REG_O7		15
787c478bd9Sstevel@tonic-gate #define	FBT_REG_I0		24
797c478bd9Sstevel@tonic-gate #define	FBT_REG_I1		25
807c478bd9Sstevel@tonic-gate #define	FBT_REG_I2		26
817c478bd9Sstevel@tonic-gate #define	FBT_REG_I3		27
827c478bd9Sstevel@tonic-gate #define	FBT_REG_I4		28
837c478bd9Sstevel@tonic-gate #define	FBT_REG_I7		31
847c478bd9Sstevel@tonic-gate #define	FBT_REG_L0		16
857c478bd9Sstevel@tonic-gate #define	FBT_REG_L1		17
867c478bd9Sstevel@tonic-gate #define	FBT_REG_L2		18
877c478bd9Sstevel@tonic-gate #define	FBT_REG_L3		19
887c478bd9Sstevel@tonic-gate #define	FBT_REG_PC		5
897c478bd9Sstevel@tonic-gate 
907c478bd9Sstevel@tonic-gate #define	FBT_REG_ISGLOBAL(r)	((r) < 8)
917c478bd9Sstevel@tonic-gate #define	FBT_REG_ISOUTPUT(r)	((r) >= 8 && (r) < 16)
927c478bd9Sstevel@tonic-gate #define	FBT_REG_ISLOCAL(r)	((r) >= 16 && (r) < 24)
937c478bd9Sstevel@tonic-gate #define	FBT_REG_ISVOLATILE(r)	\
947c478bd9Sstevel@tonic-gate 	((FBT_REG_ISGLOBAL(r) || FBT_REG_ISOUTPUT(r)) && (r) != FBT_REG_G0)
957c478bd9Sstevel@tonic-gate #define	FBT_REG_NLOCALS		8
967c478bd9Sstevel@tonic-gate 
977c478bd9Sstevel@tonic-gate #define	FBT_REG_MARKLOCAL(locals, r)	\
987c478bd9Sstevel@tonic-gate 	if (FBT_REG_ISLOCAL(r)) \
997c478bd9Sstevel@tonic-gate 		(locals)[(r) - FBT_REG_L0] = 1;
1007c478bd9Sstevel@tonic-gate 
1017c478bd9Sstevel@tonic-gate #define	FBT_REG_INITLOCALS(local, locals)	\
1027c478bd9Sstevel@tonic-gate 	for ((local) = 0; (local) < FBT_REG_NLOCALS; (local)++)  \
1037c478bd9Sstevel@tonic-gate 		(locals)[(local)] = 0; \
1047c478bd9Sstevel@tonic-gate 	(local) = FBT_REG_L0
1057c478bd9Sstevel@tonic-gate 
1067c478bd9Sstevel@tonic-gate #define	FBT_REG_ALLOCLOCAL(local, locals)	\
1077c478bd9Sstevel@tonic-gate 	while ((locals)[(local) - FBT_REG_L0]) \
1087c478bd9Sstevel@tonic-gate 		(local)++; \
1097c478bd9Sstevel@tonic-gate 	(locals)[(local) - FBT_REG_L0] = 1;
1107c478bd9Sstevel@tonic-gate 
1117c478bd9Sstevel@tonic-gate #define	FBT_OP_MASK		0xc0000000
1127c478bd9Sstevel@tonic-gate #define	FBT_OP_SHIFT		30
1137c478bd9Sstevel@tonic-gate #define	FBT_OP(val)		((val) & FBT_FMT1_MASK)
1147c478bd9Sstevel@tonic-gate 
1157c478bd9Sstevel@tonic-gate #define	FBT_SIMM13_MASK		0x1fff
1167c478bd9Sstevel@tonic-gate #define	FBT_SIMM13_MAX		((int32_t)0xfff)
1177c478bd9Sstevel@tonic-gate #define	FBT_IMM22_MASK		0x3fffff
1187c478bd9Sstevel@tonic-gate #define	FBT_IMM22_SHIFT		10
1197c478bd9Sstevel@tonic-gate #define	FBT_IMM10_MASK		0x3ff
1207c478bd9Sstevel@tonic-gate 
1217c478bd9Sstevel@tonic-gate #define	FBT_DISP30_MASK		0x3fffffff
1227c478bd9Sstevel@tonic-gate #define	FBT_DISP30(from, to)	\
1237c478bd9Sstevel@tonic-gate 	(((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP30_MASK)
1247c478bd9Sstevel@tonic-gate 
1257c478bd9Sstevel@tonic-gate #define	FBT_DISP22_MASK		0x3fffff
1267c478bd9Sstevel@tonic-gate #define	FBT_DISP22(from, to)	\
1277c478bd9Sstevel@tonic-gate 	(((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP22_MASK)
1287c478bd9Sstevel@tonic-gate 
1297c478bd9Sstevel@tonic-gate #define	FBT_DISP19_MASK		0x7ffff
1307c478bd9Sstevel@tonic-gate #define	FBT_DISP19(from, to)	\
1317c478bd9Sstevel@tonic-gate 	(((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP19_MASK)
1327c478bd9Sstevel@tonic-gate 
1337c478bd9Sstevel@tonic-gate #define	FBT_DISP16_HISHIFT	20
1347c478bd9Sstevel@tonic-gate #define	FBT_DISP16_HIMASK	(0x3 << FBT_DISP16_HISHIFT)
1357c478bd9Sstevel@tonic-gate #define	FBT_DISP16_LOMASK	(0x3fff)
1367c478bd9Sstevel@tonic-gate #define	FBT_DISP16_MASK		(FBT_DISP16_HIMASK | FBT_DISP16_LOMASK)
1377c478bd9Sstevel@tonic-gate #define	FBT_DISP16(val)	\
1387c478bd9Sstevel@tonic-gate 	((((val) & FBT_DISP16_HIMASK) >> 6) | ((val) & FBT_DISP16_LOMASK))
1397c478bd9Sstevel@tonic-gate 
1407c478bd9Sstevel@tonic-gate #define	FBT_DISP14_MASK		0x3fff
1417c478bd9Sstevel@tonic-gate #define	FBT_DISP14(from, to)	\
1427c478bd9Sstevel@tonic-gate 	(((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP14_MASK)
1437c478bd9Sstevel@tonic-gate 
1447c478bd9Sstevel@tonic-gate #define	FBT_OP0			(((uint32_t)0) << FBT_OP_SHIFT)
1457c478bd9Sstevel@tonic-gate #define	FBT_OP1			(((uint32_t)1) << FBT_OP_SHIFT)
1467c478bd9Sstevel@tonic-gate #define	FBT_OP2			(((uint32_t)2) << FBT_OP_SHIFT)
1477c478bd9Sstevel@tonic-gate #define	FBT_ILLTRAP		0
1487c478bd9Sstevel@tonic-gate 
1497c478bd9Sstevel@tonic-gate #define	FBT_ANNUL_SHIFT		29
1507c478bd9Sstevel@tonic-gate #define	FBT_ANNUL		(1 << FBT_ANNUL_SHIFT)
1517c478bd9Sstevel@tonic-gate 
1527c478bd9Sstevel@tonic-gate #define	FBT_FMT3_OP3_SHIFT	19
1537c478bd9Sstevel@tonic-gate #define	FBT_FMT3_OP_MASK	0xc1f80000
1547c478bd9Sstevel@tonic-gate #define	FBT_FMT3_OP(val)	((val) & FBT_FMT3_OP_MASK)
1557c478bd9Sstevel@tonic-gate 
1567c478bd9Sstevel@tonic-gate #define	FBT_FMT3_RD_SHIFT	25
1577c478bd9Sstevel@tonic-gate #define	FBT_FMT3_RD_MASK	(0x1f << FBT_FMT3_RD_SHIFT)
1587c478bd9Sstevel@tonic-gate #define	FBT_FMT3_RD(val)	\
1597c478bd9Sstevel@tonic-gate 	(((val) & FBT_FMT3_RD_MASK) >> FBT_FMT3_RD_SHIFT)
1607c478bd9Sstevel@tonic-gate 
1617c478bd9Sstevel@tonic-gate #define	FBT_FMT3_RS1_SHIFT	14
1627c478bd9Sstevel@tonic-gate #define	FBT_FMT3_RS1_MASK	(0x1f << FBT_FMT3_RS1_SHIFT)
1637c478bd9Sstevel@tonic-gate #define	FBT_FMT3_RS1(val)	\
1647c478bd9Sstevel@tonic-gate 	(((val) & FBT_FMT3_RS1_MASK) >> FBT_FMT3_RS1_SHIFT)
1657c478bd9Sstevel@tonic-gate #define	FBT_FMT3_RS1_SET(val, rs1) \
1667c478bd9Sstevel@tonic-gate 	(val) = ((val) & ~FBT_FMT3_RS1_MASK) | ((rs1) << FBT_FMT3_RS1_SHIFT)
1677c478bd9Sstevel@tonic-gate 
1687c478bd9Sstevel@tonic-gate #define	FBT_FMT3_RS2_SHIFT	0
1697c478bd9Sstevel@tonic-gate #define	FBT_FMT3_RS2_MASK	(0x1f << FBT_FMT3_RS2_SHIFT)
1707c478bd9Sstevel@tonic-gate #define	FBT_FMT3_RS2(val)	\
1717c478bd9Sstevel@tonic-gate 	(((val) & FBT_FMT3_RS2_MASK) >> FBT_FMT3_RS2_SHIFT)
1727c478bd9Sstevel@tonic-gate #define	FBT_FMT3_RS2_SET(val, rs2) \
1737c478bd9Sstevel@tonic-gate 	(val) = ((val) & ~FBT_FMT3_RS2_MASK) | ((rs2) << FBT_FMT3_RS2_SHIFT)
1747c478bd9Sstevel@tonic-gate 
1757c478bd9Sstevel@tonic-gate #define	FBT_FMT3_IMM_SHIFT	13
1767c478bd9Sstevel@tonic-gate #define	FBT_FMT3_IMM		(1 << FBT_FMT3_IMM_SHIFT)
1777c478bd9Sstevel@tonic-gate #define	FBT_FMT3_SIMM13_MASK	FBT_SIMM13_MASK
1787c478bd9Sstevel@tonic-gate 
1797c478bd9Sstevel@tonic-gate #define	FBT_FMT3_ISIMM(val)	((val) & FBT_FMT3_IMM)
1807c478bd9Sstevel@tonic-gate #define	FBT_FMT3_SIMM13(val)	((val) & FBT_FMT3_SIMM13_MASK)
1817c478bd9Sstevel@tonic-gate 
1827c478bd9Sstevel@tonic-gate #define	FBT_FMT2_OP2_SHIFT	22
1837c478bd9Sstevel@tonic-gate #define	FBT_FMT2_OP2_MASK	(0x7 << FBT_FMT2_OP2_SHIFT)
1847c478bd9Sstevel@tonic-gate #define	FBT_FMT2_RD_SHIFT	25
1857c478bd9Sstevel@tonic-gate 
1867c478bd9Sstevel@tonic-gate #define	FBT_FMT1_OP(val)	((val) & FBT_OP_MASK)
1877c478bd9Sstevel@tonic-gate #define	FBT_FMT1_DISP30(val)	((val) & FBT_DISP30_MASK)
1887c478bd9Sstevel@tonic-gate 
1897c478bd9Sstevel@tonic-gate #define	FBT_FMT2_OP2_BPCC	(0x01 << FBT_FMT2_OP2_SHIFT)
1907c478bd9Sstevel@tonic-gate #define	FBT_FMT2_OP2_BCC	(0x02 << FBT_FMT2_OP2_SHIFT)
1917c478bd9Sstevel@tonic-gate #define	FBT_FMT2_OP2_BPR	(0x03 << FBT_FMT2_OP2_SHIFT)
1927c478bd9Sstevel@tonic-gate #define	FBT_FMT2_OP2_SETHI	(0x04 << FBT_FMT2_OP2_SHIFT)
1937c478bd9Sstevel@tonic-gate 
1947c478bd9Sstevel@tonic-gate #define	FBT_FMT2_COND_SHIFT	25
1957c478bd9Sstevel@tonic-gate #define	FBT_FMT2_COND_BA	(0x8 << FBT_FMT2_COND_SHIFT)
1967c478bd9Sstevel@tonic-gate #define	FBT_FMT2_COND_BL	(0x3 << FBT_FMT2_COND_SHIFT)
1977c478bd9Sstevel@tonic-gate #define	FBT_FMT2_COND_BGE	(0xb << FBT_FMT2_COND_SHIFT)
1987c478bd9Sstevel@tonic-gate 
1997c478bd9Sstevel@tonic-gate #define	FBT_OP_RESTORE		(FBT_OP2 | (0x3d << FBT_FMT3_OP3_SHIFT))
2007c478bd9Sstevel@tonic-gate #define	FBT_OP_SAVE		(FBT_OP2 | (0x3c << FBT_FMT3_OP3_SHIFT))
2017c478bd9Sstevel@tonic-gate #define	FBT_OP_JMPL		(FBT_OP2 | (0x38 << FBT_FMT3_OP3_SHIFT))
2027c478bd9Sstevel@tonic-gate #define	FBT_OP_RETURN		(FBT_OP2 | (0x39 << FBT_FMT3_OP3_SHIFT))
2037c478bd9Sstevel@tonic-gate #define	FBT_OP_CALL		FBT_OP1
2047c478bd9Sstevel@tonic-gate #define	FBT_OP_SETHI		(FBT_OP0 | FBT_FMT2_OP2_SETHI)
2057c478bd9Sstevel@tonic-gate #define	FBT_OP_ADD		(FBT_OP2 | (0x00 << FBT_FMT3_OP3_SHIFT))
2067c478bd9Sstevel@tonic-gate #define	FBT_OP_OR		(FBT_OP2 | (0x02 << FBT_FMT3_OP3_SHIFT))
2077c478bd9Sstevel@tonic-gate #define	FBT_OP_SUB		(FBT_OP2 | (0x04 << FBT_FMT3_OP3_SHIFT))
2087c478bd9Sstevel@tonic-gate #define	FBT_OP_CC		(FBT_OP2 | (0x10 << FBT_FMT3_OP3_SHIFT))
2097c478bd9Sstevel@tonic-gate #define	FBT_OP_BA		(FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BA)
2107c478bd9Sstevel@tonic-gate #define	FBT_OP_BL		(FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BL)
2117c478bd9Sstevel@tonic-gate #define	FBT_OP_BGE		(FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BGE)
2127c478bd9Sstevel@tonic-gate #define	FBT_OP_BAPCC		(FBT_OP0 | FBT_FMT2_OP2_BPCC | FBT_FMT2_COND_BA)
2137c478bd9Sstevel@tonic-gate #define	FBT_OP_RD		(FBT_OP2 | (0x28 << FBT_FMT3_OP3_SHIFT))
2147c478bd9Sstevel@tonic-gate 
2157c478bd9Sstevel@tonic-gate #define	FBT_ORLO(rs, val, rd) \
2167c478bd9Sstevel@tonic-gate 	(FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
2177c478bd9Sstevel@tonic-gate 	((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_IMM10_MASK))
2187c478bd9Sstevel@tonic-gate 
2197c478bd9Sstevel@tonic-gate #define	FBT_ORSIMM13(rs, val, rd) \
2207c478bd9Sstevel@tonic-gate 	(FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
2217c478bd9Sstevel@tonic-gate 	((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
2227c478bd9Sstevel@tonic-gate 
2237c478bd9Sstevel@tonic-gate #define	FBT_ADDSIMM13(rs, val, rd) \
2247c478bd9Sstevel@tonic-gate 	(FBT_OP_ADD | ((rs) << FBT_FMT3_RS1_SHIFT) | \
2257c478bd9Sstevel@tonic-gate 	((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
2267c478bd9Sstevel@tonic-gate 
2277c478bd9Sstevel@tonic-gate #define	FBT_ADD(rs1, rs2, rd) \
2287c478bd9Sstevel@tonic-gate 	(FBT_OP_ADD | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
2297c478bd9Sstevel@tonic-gate 	((rs2) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))
2307c478bd9Sstevel@tonic-gate 
2317c478bd9Sstevel@tonic-gate #define	FBT_CMP(rs1, rs2) \
2327c478bd9Sstevel@tonic-gate 	(FBT_OP_SUB | FBT_OP_CC | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
2337c478bd9Sstevel@tonic-gate 	((rs2) << FBT_FMT3_RS2_SHIFT) | (FBT_REG_G0 << FBT_FMT3_RD_SHIFT))
2347c478bd9Sstevel@tonic-gate 
2357c478bd9Sstevel@tonic-gate #define	FBT_MOV(rs, rd) \
2367c478bd9Sstevel@tonic-gate 	(FBT_OP_OR | (FBT_REG_G0 << FBT_FMT3_RS1_SHIFT) | \
2377c478bd9Sstevel@tonic-gate 	((rs) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))
2387c478bd9Sstevel@tonic-gate 
2397c478bd9Sstevel@tonic-gate #define	FBT_SETHI(val, reg)	\
2407c478bd9Sstevel@tonic-gate 	(FBT_OP_SETHI | (reg << FBT_FMT2_RD_SHIFT) | \
2417c478bd9Sstevel@tonic-gate 	((val >> FBT_IMM22_SHIFT) & FBT_IMM22_MASK))
2427c478bd9Sstevel@tonic-gate 
2437c478bd9Sstevel@tonic-gate #define	FBT_CALL(orig, dest)	(FBT_OP_CALL | FBT_DISP30(orig, dest))
2447c478bd9Sstevel@tonic-gate 
2457c478bd9Sstevel@tonic-gate #define	FBT_RET \
2467c478bd9Sstevel@tonic-gate 	(FBT_OP_JMPL | (FBT_REG_I7 << FBT_FMT3_RS1_SHIFT) | \
2477c478bd9Sstevel@tonic-gate 	(FBT_REG_G0 << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | (sizeof (pc_t) << 1))
2487c478bd9Sstevel@tonic-gate 
2497c478bd9Sstevel@tonic-gate #define	FBT_SAVEIMM(rd, val, rs1)	\
2507c478bd9Sstevel@tonic-gate 	(FBT_OP_SAVE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
2517c478bd9Sstevel@tonic-gate 	((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
2527c478bd9Sstevel@tonic-gate 
2537c478bd9Sstevel@tonic-gate #define	FBT_RESTORE(rd, rs1, rs2)	\
2547c478bd9Sstevel@tonic-gate 	(FBT_OP_RESTORE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
2557c478bd9Sstevel@tonic-gate 	((rd) << FBT_FMT3_RD_SHIFT) | ((rs2) << FBT_FMT3_RS2_SHIFT))
2567c478bd9Sstevel@tonic-gate 
2577c478bd9Sstevel@tonic-gate #define	FBT_RETURN(rs1, val)		\
2587c478bd9Sstevel@tonic-gate 	(FBT_OP_RETURN | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
2597c478bd9Sstevel@tonic-gate 	FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
2607c478bd9Sstevel@tonic-gate 
2617c478bd9Sstevel@tonic-gate #define	FBT_BA(orig, dest)	(FBT_OP_BA | FBT_DISP22(orig, dest))
2627c478bd9Sstevel@tonic-gate #define	FBT_BAA(orig, dest)	(FBT_BA(orig, dest) | FBT_ANNUL)
2637c478bd9Sstevel@tonic-gate #define	FBT_BL(orig, dest)	(FBT_OP_BL | FBT_DISP22(orig, dest))
2647c478bd9Sstevel@tonic-gate #define	FBT_BGE(orig, dest)	(FBT_OP_BGE | FBT_DISP22(orig, dest))
2657c478bd9Sstevel@tonic-gate #define	FBT_BDEST(va, instr)	((uintptr_t)(va) + \
2667c478bd9Sstevel@tonic-gate 	(((int32_t)(((instr) & FBT_DISP22_MASK) << 10)) >> 8))
2677c478bd9Sstevel@tonic-gate #define	FBT_BPCCDEST(va, instr)	((uintptr_t)(va) + \
2687c478bd9Sstevel@tonic-gate 	(((int32_t)(((instr) & FBT_DISP19_MASK) << 13)) >> 11))
2697c478bd9Sstevel@tonic-gate #define	FBT_BPRDEST(va, instr)	((uintptr_t)(va) + \
2707c478bd9Sstevel@tonic-gate 	(((int32_t)((FBT_DISP16(instr)) << 16)) >> 14))
2717c478bd9Sstevel@tonic-gate 
2727c478bd9Sstevel@tonic-gate /*
2737c478bd9Sstevel@tonic-gate  * We're only going to treat a save as safe if (a) both rs1 and rd are
2747c478bd9Sstevel@tonic-gate  * %sp and (b) if the instruction has a simm, the value isn't 0.
2757c478bd9Sstevel@tonic-gate  */
2767c478bd9Sstevel@tonic-gate #define	FBT_IS_SAVE(instr)	\
2777c478bd9Sstevel@tonic-gate 	(FBT_FMT3_OP(instr) == FBT_OP_SAVE && \
2787c478bd9Sstevel@tonic-gate 	FBT_FMT3_RD(instr) == FBT_REG_O6 && \
2797c478bd9Sstevel@tonic-gate 	FBT_FMT3_RS1(instr) == FBT_REG_O6 && \
2807c478bd9Sstevel@tonic-gate 	!(FBT_FMT3_ISIMM(instr) && FBT_FMT3_SIMM13(instr) == 0))
2817c478bd9Sstevel@tonic-gate 
2827c478bd9Sstevel@tonic-gate #define	FBT_IS_BA(instr)	(((instr) & ~FBT_DISP22_MASK) == FBT_OP_BA)
2837c478bd9Sstevel@tonic-gate #define	FBT_IS_BAPCC(instr)	(((instr) & ~FBT_DISP22_MASK) == FBT_OP_BAPCC)
2847c478bd9Sstevel@tonic-gate 
2857c478bd9Sstevel@tonic-gate #define	FBT_IS_RDPC(instr)	((FBT_FMT3_OP(instr) == FBT_OP_RD) && \
2867c478bd9Sstevel@tonic-gate 	(FBT_FMT3_RD(instr) == FBT_REG_PC))
2877c478bd9Sstevel@tonic-gate 
2887c478bd9Sstevel@tonic-gate #define	FBT_IS_PCRELATIVE(instr)	\
2897c478bd9Sstevel@tonic-gate 	((((instr) & FBT_OP_MASK) == FBT_OP0 && \
2907c478bd9Sstevel@tonic-gate 	((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
2917c478bd9Sstevel@tonic-gate 	((instr) & FBT_OP_MASK) == FBT_OP1 || \
2927c478bd9Sstevel@tonic-gate 	FBT_IS_RDPC(instr))
2937c478bd9Sstevel@tonic-gate 
2947c478bd9Sstevel@tonic-gate #define	FBT_IS_CTI(instr)	\
2957c478bd9Sstevel@tonic-gate 	((((instr) & FBT_OP_MASK) == FBT_OP0 && \
2967c478bd9Sstevel@tonic-gate 	((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
2977c478bd9Sstevel@tonic-gate 	((instr) & FBT_OP_MASK) == FBT_OP1 || \
2987c478bd9Sstevel@tonic-gate 	(FBT_FMT3_OP(instr) == FBT_OP_JMPL) || \
2997c478bd9Sstevel@tonic-gate 	(FBT_FMT3_OP(instr) == FBT_OP_RETURN))
3007c478bd9Sstevel@tonic-gate 
3017c478bd9Sstevel@tonic-gate #define	FBT_PROBENAME_ENTRY	"entry"
3027c478bd9Sstevel@tonic-gate #define	FBT_PROBENAME_RETURN	"return"
3037c478bd9Sstevel@tonic-gate #define	FBT_ESTIMATE_ID		(UINT32_MAX)
3047c478bd9Sstevel@tonic-gate #define	FBT_COUNTER(id, count)	if ((id) != FBT_ESTIMATE_ID) (count)++
3057c478bd9Sstevel@tonic-gate 
3067c478bd9Sstevel@tonic-gate #define	FBT_ENTENT_MAXSIZE	(16 * sizeof (uint32_t))
3077c478bd9Sstevel@tonic-gate #define	FBT_RETENT_MAXSIZE	(11 * sizeof (uint32_t))
3087c478bd9Sstevel@tonic-gate #define	FBT_RETLENT_MAXSIZE	(23 * sizeof (uint32_t))
3097c478bd9Sstevel@tonic-gate #define	FBT_ENT_MAXSIZE		\
3107c478bd9Sstevel@tonic-gate 	MAX(MAX(FBT_ENTENT_MAXSIZE, FBT_RETENT_MAXSIZE), FBT_RETLENT_MAXSIZE)
3117c478bd9Sstevel@tonic-gate 
3127c478bd9Sstevel@tonic-gate typedef struct fbt_probe {
3137c478bd9Sstevel@tonic-gate 	char		*fbtp_name;
3147c478bd9Sstevel@tonic-gate 	dtrace_id_t	fbtp_id;
3157c478bd9Sstevel@tonic-gate 	uintptr_t	fbtp_addr;
3167c478bd9Sstevel@tonic-gate 	struct modctl	*fbtp_ctl;
3177c478bd9Sstevel@tonic-gate 	int		fbtp_loadcnt;
3187c478bd9Sstevel@tonic-gate 	int		fbtp_symndx;
3197c478bd9Sstevel@tonic-gate 	int		fbtp_primary;
3207c478bd9Sstevel@tonic-gate 	int		fbtp_return;
3217c478bd9Sstevel@tonic-gate 	uint32_t	*fbtp_patchpoint;
3227c478bd9Sstevel@tonic-gate 	uint32_t	fbtp_patchval;
3237c478bd9Sstevel@tonic-gate 	uint32_t	fbtp_savedval;
3247c478bd9Sstevel@tonic-gate 	struct fbt_probe *fbtp_next;
3257c478bd9Sstevel@tonic-gate } fbt_probe_t;
3267c478bd9Sstevel@tonic-gate 
3277c478bd9Sstevel@tonic-gate typedef struct fbt_trampoline {
3287c478bd9Sstevel@tonic-gate 	uintptr_t	fbtt_va;
3297c478bd9Sstevel@tonic-gate 	uintptr_t	fbtt_limit;
3307c478bd9Sstevel@tonic-gate 	uintptr_t	fbtt_next;
3317c478bd9Sstevel@tonic-gate } fbt_trampoline_t;
3327c478bd9Sstevel@tonic-gate 
3337c478bd9Sstevel@tonic-gate static caddr_t
fbt_trampoline_map(uintptr_t tramp,size_t size)3347c478bd9Sstevel@tonic-gate fbt_trampoline_map(uintptr_t tramp, size_t size)
3357c478bd9Sstevel@tonic-gate {
3367c478bd9Sstevel@tonic-gate 	uintptr_t offs;
3377c478bd9Sstevel@tonic-gate 	page_t **ppl;
3387c478bd9Sstevel@tonic-gate 
3397c478bd9Sstevel@tonic-gate 	ASSERT(fbt_trampoline_window == NULL);
3407c478bd9Sstevel@tonic-gate 	ASSERT(fbt_trampoline_size == 0);
3417c478bd9Sstevel@tonic-gate 	ASSERT(fbt_trampoline == NULL);
3427c478bd9Sstevel@tonic-gate 
3437c478bd9Sstevel@tonic-gate 	size += tramp & PAGEOFFSET;
3447c478bd9Sstevel@tonic-gate 	fbt_trampoline = tramp & PAGEMASK;
3457c478bd9Sstevel@tonic-gate 	fbt_trampoline_size = (size + PAGESIZE - 1) & PAGEMASK;
3467c478bd9Sstevel@tonic-gate 	fbt_trampoline_window =
3477c478bd9Sstevel@tonic-gate 	    vmem_alloc(heap_arena, fbt_trampoline_size, VM_SLEEP);
3487c478bd9Sstevel@tonic-gate 
3497c478bd9Sstevel@tonic-gate 	(void) as_pagelock(&kas, &ppl, (caddr_t)fbt_trampoline,
3507c478bd9Sstevel@tonic-gate 	    fbt_trampoline_size, S_WRITE);
3517c478bd9Sstevel@tonic-gate 
3527c478bd9Sstevel@tonic-gate 	for (offs = 0; offs < fbt_trampoline_size; offs += PAGESIZE) {
3537c478bd9Sstevel@tonic-gate 		hat_devload(kas.a_hat, fbt_trampoline_window + offs, PAGESIZE,
3547c478bd9Sstevel@tonic-gate 		    hat_getpfnum(kas.a_hat, (caddr_t)fbt_trampoline + offs),
3557c478bd9Sstevel@tonic-gate 		    PROT_READ | PROT_WRITE,
3567c478bd9Sstevel@tonic-gate 		    HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
3577c478bd9Sstevel@tonic-gate 	}
3587c478bd9Sstevel@tonic-gate 
3597c478bd9Sstevel@tonic-gate 	as_pageunlock(&kas, ppl, (caddr_t)fbt_trampoline, fbt_trampoline_size,
3607c478bd9Sstevel@tonic-gate 	    S_WRITE);
3617c478bd9Sstevel@tonic-gate 
3627c478bd9Sstevel@tonic-gate 	return (fbt_trampoline_window + (tramp & PAGEOFFSET));
3637c478bd9Sstevel@tonic-gate }
3647c478bd9Sstevel@tonic-gate 
3657c478bd9Sstevel@tonic-gate static void
fbt_trampoline_unmap()3667c478bd9Sstevel@tonic-gate fbt_trampoline_unmap()
3677c478bd9Sstevel@tonic-gate {
3687c478bd9Sstevel@tonic-gate 	ASSERT(fbt_trampoline_window != NULL);
3697c478bd9Sstevel@tonic-gate 	ASSERT(fbt_trampoline_size != 0);
3707c478bd9Sstevel@tonic-gate 	ASSERT(fbt_trampoline != NULL);
3717c478bd9Sstevel@tonic-gate 
3727c478bd9Sstevel@tonic-gate 	membar_enter();
3737c478bd9Sstevel@tonic-gate 	sync_icache((caddr_t)fbt_trampoline, fbt_trampoline_size);
3747c478bd9Sstevel@tonic-gate 	sync_icache(fbt_trampoline_window, fbt_trampoline_size);
3757c478bd9Sstevel@tonic-gate 
3767c478bd9Sstevel@tonic-gate 	hat_unload(kas.a_hat, fbt_trampoline_window, fbt_trampoline_size,
3777c478bd9Sstevel@tonic-gate 	    HAT_UNLOAD_UNLOCK);
3787c478bd9Sstevel@tonic-gate 
3797c478bd9Sstevel@tonic-gate 	vmem_free(heap_arena, fbt_trampoline_window, fbt_trampoline_size);
3807c478bd9Sstevel@tonic-gate 
3817c478bd9Sstevel@tonic-gate 	fbt_trampoline_window = NULL;
382*692bcae7SToomas Soome 	fbt_trampoline = 0;
3837c478bd9Sstevel@tonic-gate 	fbt_trampoline_size = 0;
3847c478bd9Sstevel@tonic-gate }
3857c478bd9Sstevel@tonic-gate 
3867c478bd9Sstevel@tonic-gate static uintptr_t
fbt_patch_entry(uint32_t * instr,uint32_t id,fbt_trampoline_t * tramp,int nargs)3877c478bd9Sstevel@tonic-gate fbt_patch_entry(uint32_t *instr, uint32_t id, fbt_trampoline_t *tramp,
3887c478bd9Sstevel@tonic-gate     int nargs)
3897c478bd9Sstevel@tonic-gate {
3907c478bd9Sstevel@tonic-gate 	uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
3917c478bd9Sstevel@tonic-gate 	uint32_t first = *instr;
3927c478bd9Sstevel@tonic-gate 	uintptr_t va = tramp->fbtt_va;
3937c478bd9Sstevel@tonic-gate 	uintptr_t base = tramp->fbtt_next;
3947c478bd9Sstevel@tonic-gate 
3957c478bd9Sstevel@tonic-gate 	if (tramp->fbtt_next + FBT_ENTENT_MAXSIZE > tramp->fbtt_limit) {
3967c478bd9Sstevel@tonic-gate 		/*
3977c478bd9Sstevel@tonic-gate 		 * There isn't sufficient room for this entry; return failure.
3987c478bd9Sstevel@tonic-gate 		 */
3997c478bd9Sstevel@tonic-gate 		return (0);
4007c478bd9Sstevel@tonic-gate 	}
4017c478bd9Sstevel@tonic-gate 
4027c478bd9Sstevel@tonic-gate 	FBT_COUNTER(id, fbt_entry);
4037c478bd9Sstevel@tonic-gate 
4047c478bd9Sstevel@tonic-gate 	if (FBT_IS_SAVE(first)) {
4057c478bd9Sstevel@tonic-gate 		*tinstr++ = first;
4067c478bd9Sstevel@tonic-gate 	} else {
4077c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
4087c478bd9Sstevel@tonic-gate 	}
4097c478bd9Sstevel@tonic-gate 
4107c478bd9Sstevel@tonic-gate 	if (id > (uint32_t)FBT_SIMM13_MAX) {
4117c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_SETHI(id, FBT_REG_O0);
4127c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
4137c478bd9Sstevel@tonic-gate 	} else {
4147c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
4157c478bd9Sstevel@tonic-gate 	}
4167c478bd9Sstevel@tonic-gate 
4177c478bd9Sstevel@tonic-gate 	if (nargs >= 1)
4187c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O1);
4197c478bd9Sstevel@tonic-gate 
4207c478bd9Sstevel@tonic-gate 	if (nargs >= 2)
4217c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_MOV(FBT_REG_I1, FBT_REG_O2);
4227c478bd9Sstevel@tonic-gate 
4237c478bd9Sstevel@tonic-gate 	if (nargs >= 3)
4247c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_MOV(FBT_REG_I2, FBT_REG_O3);
4257c478bd9Sstevel@tonic-gate 
4267c478bd9Sstevel@tonic-gate 	if (nargs >= 4)
4277c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_MOV(FBT_REG_I3, FBT_REG_O4);
4287c478bd9Sstevel@tonic-gate 
4297c478bd9Sstevel@tonic-gate 	if (nargs >= 5)
4307c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_MOV(FBT_REG_I4, FBT_REG_O5);
4317c478bd9Sstevel@tonic-gate 
4327c478bd9Sstevel@tonic-gate 	if (FBT_IS_SAVE(first)) {
4337c478bd9Sstevel@tonic-gate 		uintptr_t ret = (uintptr_t)instr - sizeof (uint32_t);
4347c478bd9Sstevel@tonic-gate 
4357c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_SETHI(ret, FBT_REG_G1);
4367c478bd9Sstevel@tonic-gate 		*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
4377c478bd9Sstevel@tonic-gate 		tinstr++;
4387c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_ORLO(FBT_REG_G1, ret, FBT_REG_O7);
4397c478bd9Sstevel@tonic-gate 	} else {
4407c478bd9Sstevel@tonic-gate 		uintptr_t slot = *--tinstr;
4417c478bd9Sstevel@tonic-gate 		uintptr_t ret = (uintptr_t)instr + sizeof (uint32_t);
4427c478bd9Sstevel@tonic-gate 		uint32_t delay = first;
4437c478bd9Sstevel@tonic-gate 
4447c478bd9Sstevel@tonic-gate 		*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
4457c478bd9Sstevel@tonic-gate 		tinstr++;
4467c478bd9Sstevel@tonic-gate 		*tinstr++ = slot;
4477c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
4487c478bd9Sstevel@tonic-gate 
4497c478bd9Sstevel@tonic-gate 		if (FBT_IS_BA(first) || FBT_IS_BAPCC(first)) {
4507c478bd9Sstevel@tonic-gate 			/*
4517c478bd9Sstevel@tonic-gate 			 * This is a special case:  we are instrumenting a
4527c478bd9Sstevel@tonic-gate 			 * a non-annulled branch-always (or variant).  We'll
4537c478bd9Sstevel@tonic-gate 			 * return directly to the destination of the branch,
4547c478bd9Sstevel@tonic-gate 			 * copying the instruction in the delay slot here,
4557c478bd9Sstevel@tonic-gate 			 * and then executing it in the slot of a ba.
4567c478bd9Sstevel@tonic-gate 			 */
4577c478bd9Sstevel@tonic-gate 			if (FBT_IS_BA(first)) {
4587c478bd9Sstevel@tonic-gate 				ret = FBT_BDEST(instr, *instr);
4597c478bd9Sstevel@tonic-gate 			} else {
4607c478bd9Sstevel@tonic-gate 				ret = FBT_BPCCDEST(instr, *instr);
4617c478bd9Sstevel@tonic-gate 			}
4627c478bd9Sstevel@tonic-gate 
4637c478bd9Sstevel@tonic-gate 			delay = *(instr + 1);
4647c478bd9Sstevel@tonic-gate 		}
4657c478bd9Sstevel@tonic-gate 
4667c478bd9Sstevel@tonic-gate 		if ((first & FBT_OP_MASK) != FBT_OP0 ||
4677c478bd9Sstevel@tonic-gate 		    (first & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
4687c478bd9Sstevel@tonic-gate 			*tinstr = FBT_BA((uintptr_t)tinstr - base + va, ret);
4697c478bd9Sstevel@tonic-gate 			tinstr++;
4707c478bd9Sstevel@tonic-gate 			*tinstr++ = delay;
4717c478bd9Sstevel@tonic-gate 		} else {
4727c478bd9Sstevel@tonic-gate 			/*
4737c478bd9Sstevel@tonic-gate 			 * If this is a branch-on-register, we have a little
4747c478bd9Sstevel@tonic-gate 			 * more work to do:  because the displacement is only
4757c478bd9Sstevel@tonic-gate 			 * sixteen bits, we're going to thunk the branch into
4767c478bd9Sstevel@tonic-gate 			 * the trampoline, and then ba,a to the appropriate
4777c478bd9Sstevel@tonic-gate 			 * destination in the branch targets.  That is, we're
4787c478bd9Sstevel@tonic-gate 			 * constructing this sequence in the trampoline:
4797c478bd9Sstevel@tonic-gate 			 *
4807c478bd9Sstevel@tonic-gate 			 *		br[cc]	%[rs], 1f
4817c478bd9Sstevel@tonic-gate 			 *		<delay-instruction>
4827c478bd9Sstevel@tonic-gate 			 *		ba,a	<not-taken-destination>
4837c478bd9Sstevel@tonic-gate 			 *	1:	ba,a	<taken-destination>
4847c478bd9Sstevel@tonic-gate 			 *
4857c478bd9Sstevel@tonic-gate 			 */
4867c478bd9Sstevel@tonic-gate 			uintptr_t targ = FBT_BPRDEST(instr, first);
4877c478bd9Sstevel@tonic-gate 
4887c478bd9Sstevel@tonic-gate 			*tinstr = first & ~(FBT_DISP16_MASK);
4897c478bd9Sstevel@tonic-gate 			*tinstr |= FBT_DISP14(tinstr, &tinstr[3]);
4907c478bd9Sstevel@tonic-gate 			tinstr++;
4917c478bd9Sstevel@tonic-gate 			*tinstr++ = *(instr + 1);
4927c478bd9Sstevel@tonic-gate 			*tinstr = FBT_BAA((uintptr_t)tinstr - base + va,
4937c478bd9Sstevel@tonic-gate 			    ret + sizeof (uint32_t));
4947c478bd9Sstevel@tonic-gate 			tinstr++;
4957c478bd9Sstevel@tonic-gate 			*tinstr = FBT_BAA((uintptr_t)tinstr - base + va, targ);
4967c478bd9Sstevel@tonic-gate 			tinstr++;
4977c478bd9Sstevel@tonic-gate 		}
4987c478bd9Sstevel@tonic-gate 	}
4997c478bd9Sstevel@tonic-gate 
5007c478bd9Sstevel@tonic-gate 	tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
5017c478bd9Sstevel@tonic-gate 	tramp->fbtt_next = (uintptr_t)tinstr;
5027c478bd9Sstevel@tonic-gate 
5037c478bd9Sstevel@tonic-gate 	return (1);
5047c478bd9Sstevel@tonic-gate }
5057c478bd9Sstevel@tonic-gate 
5067c478bd9Sstevel@tonic-gate /*
5077c478bd9Sstevel@tonic-gate  * We are patching control-transfer/restore couplets.  There are three
5087c478bd9Sstevel@tonic-gate  * variants of couplet:
5097c478bd9Sstevel@tonic-gate  *
5107c478bd9Sstevel@tonic-gate  * (a)	return		rs1 + imm
5117c478bd9Sstevel@tonic-gate  *	delay
5127c478bd9Sstevel@tonic-gate  *
5137c478bd9Sstevel@tonic-gate  * (b)	jmpl		rs1 + (rs2 | offset), rd
5147c478bd9Sstevel@tonic-gate  *	restore		rs1, rs2 | imm, rd
5157c478bd9Sstevel@tonic-gate  *
5167c478bd9Sstevel@tonic-gate  * (c)	call		displacement
5177c478bd9Sstevel@tonic-gate  *	restore		rs1, rs2 | imm, rd
5187c478bd9Sstevel@tonic-gate  *
5197c478bd9Sstevel@tonic-gate  * If rs1 in (a) is anything other than %i7, or imm is anything other than 8,
5207c478bd9Sstevel@tonic-gate  * or delay is a DCTI, we fail.  If rd from the jmpl in (b) is something other
5217c478bd9Sstevel@tonic-gate  * than %g0 (a ret or a tail-call through a function pointer) or %o7 (a call
5227c478bd9Sstevel@tonic-gate  * through a register), we fail.
5237c478bd9Sstevel@tonic-gate  *
5247c478bd9Sstevel@tonic-gate  * Note that rs1 and rs2 in the restore instructions in (b) and (c) are
5257c478bd9Sstevel@tonic-gate  * potentially outputs and/or globals.  Because these registers cannot be
5267c478bd9Sstevel@tonic-gate  * relied upon across the call to dtrace_probe(), we move rs1 into an unused
5277c478bd9Sstevel@tonic-gate  * local, ls0, and rs2 into an unused local, ls1, and restructure the restore
5287c478bd9Sstevel@tonic-gate  * to be:
5297c478bd9Sstevel@tonic-gate  *
5307c478bd9Sstevel@tonic-gate  *	restore		ls0, ls1, rd
5317c478bd9Sstevel@tonic-gate  *
5327c478bd9Sstevel@tonic-gate  * Likewise, rs1 and rs2 in the jmpl of case (b) may be outputs and/or globals.
5337c478bd9Sstevel@tonic-gate  * If the jmpl uses outputs or globals, we restructure it to be:
5347c478bd9Sstevel@tonic-gate  *
535*692bcae7SToomas Soome  *	jmpl		ls2 + (ls3 | offset), (%g0 | %o7)
5367c478bd9Sstevel@tonic-gate  *
5377c478bd9Sstevel@tonic-gate  */
5387c478bd9Sstevel@tonic-gate /*ARGSUSED*/
5397c478bd9Sstevel@tonic-gate static int
fbt_canpatch_return(uint32_t * instr,int offset,const char * name)5407c478bd9Sstevel@tonic-gate fbt_canpatch_return(uint32_t *instr, int offset, const char *name)
5417c478bd9Sstevel@tonic-gate {
5427c478bd9Sstevel@tonic-gate 	int rd;
5437c478bd9Sstevel@tonic-gate 
5447c478bd9Sstevel@tonic-gate 	if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
5457c478bd9Sstevel@tonic-gate 		uint32_t delay = *(instr + 1);
5467c478bd9Sstevel@tonic-gate 
5477c478bd9Sstevel@tonic-gate 		if (*instr != FBT_RETURN(FBT_REG_I7, 8)) {
5487c478bd9Sstevel@tonic-gate 			/*
5497c478bd9Sstevel@tonic-gate 			 * It's unclear if we should warn about this or not.
5507c478bd9Sstevel@tonic-gate 			 * We really wouldn't expect the compiler to generate
5517c478bd9Sstevel@tonic-gate 			 * return instructions with something other than %i7
5527c478bd9Sstevel@tonic-gate 			 * as rs1 and 8 as the simm13 -- it would just be
5537c478bd9Sstevel@tonic-gate 			 * mean-spirited.  That said, such a construct isn't
5547c478bd9Sstevel@tonic-gate 			 * necessarily incorrect.  Sill, we err on the side of
5557c478bd9Sstevel@tonic-gate 			 * caution and warn about it...
5567c478bd9Sstevel@tonic-gate 			 */
5577c478bd9Sstevel@tonic-gate 			cmn_err(CE_NOTE, "cannot instrument return of %s at "
5587c478bd9Sstevel@tonic-gate 			    "%p: non-canonical return instruction", name,
5597c478bd9Sstevel@tonic-gate 			    (void *)instr);
5607c478bd9Sstevel@tonic-gate 			return (0);
5617c478bd9Sstevel@tonic-gate 		}
5627c478bd9Sstevel@tonic-gate 
5637c478bd9Sstevel@tonic-gate 		if (FBT_IS_CTI(delay)) {
5647c478bd9Sstevel@tonic-gate 			/*
5657c478bd9Sstevel@tonic-gate 			 * This is even weirder -- a DCTI coupled with a
5667c478bd9Sstevel@tonic-gate 			 * return instruction.  Similar constructs are used to
5677c478bd9Sstevel@tonic-gate 			 * return from utraps, but these typically have the
5687c478bd9Sstevel@tonic-gate 			 * return in the slot -- and we wouldn't expect to see
5697c478bd9Sstevel@tonic-gate 			 * it in the kernel regardless.  At any rate, we don't
5707c478bd9Sstevel@tonic-gate 			 * want to try to instrument this construct, whatever
5717c478bd9Sstevel@tonic-gate 			 * it may be.
5727c478bd9Sstevel@tonic-gate 			 */
5737c478bd9Sstevel@tonic-gate 			cmn_err(CE_NOTE, "cannot instrument return of %s at "
5747c478bd9Sstevel@tonic-gate 			    "%p: CTI in delay slot of return instruction",
5757c478bd9Sstevel@tonic-gate 			    name, (void *)instr);
5767c478bd9Sstevel@tonic-gate 			return (0);
5777c478bd9Sstevel@tonic-gate 		}
5787c478bd9Sstevel@tonic-gate 
5797c478bd9Sstevel@tonic-gate 		if (FBT_IS_PCRELATIVE(delay)) {
5807c478bd9Sstevel@tonic-gate 			/*
5817c478bd9Sstevel@tonic-gate 			 * This is also very weird, but might be correct code
5827c478bd9Sstevel@tonic-gate 			 * if the function is (for example) returning the
5837c478bd9Sstevel@tonic-gate 			 * address of the delay instruction of the return as
5847c478bd9Sstevel@tonic-gate 			 * its return value (e.g. "rd %pc, %o0" in the slot).
5857c478bd9Sstevel@tonic-gate 			 * Perhaps correct, but still too weird to not warn
5867c478bd9Sstevel@tonic-gate 			 * about it...
5877c478bd9Sstevel@tonic-gate 			 */
5887c478bd9Sstevel@tonic-gate 			cmn_err(CE_NOTE, "cannot instrument return of %s at "
5897c478bd9Sstevel@tonic-gate 			    "%p: PC-relative instruction in delay slot of "
5907c478bd9Sstevel@tonic-gate 			    "return instruction", name, (void *)instr);
5917c478bd9Sstevel@tonic-gate 			return (0);
5927c478bd9Sstevel@tonic-gate 		}
5937c478bd9Sstevel@tonic-gate 
5947c478bd9Sstevel@tonic-gate 		return (1);
5957c478bd9Sstevel@tonic-gate 	}
5967c478bd9Sstevel@tonic-gate 
5977c478bd9Sstevel@tonic-gate 	if (FBT_FMT3_OP(*(instr + 1)) != FBT_OP_RESTORE)
5987c478bd9Sstevel@tonic-gate 		return (0);
5997c478bd9Sstevel@tonic-gate 
6007c478bd9Sstevel@tonic-gate 	if (FBT_FMT1_OP(*instr) == FBT_OP_CALL)
6017c478bd9Sstevel@tonic-gate 		return (1);
6027c478bd9Sstevel@tonic-gate 
6037c478bd9Sstevel@tonic-gate 	if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
6047c478bd9Sstevel@tonic-gate 		return (0);
6057c478bd9Sstevel@tonic-gate 
6067c478bd9Sstevel@tonic-gate 	rd = FBT_FMT3_RD(*instr);
6077c478bd9Sstevel@tonic-gate 
6087c478bd9Sstevel@tonic-gate 	if (rd == FBT_REG_I7 || rd == FBT_REG_O7 || rd == FBT_REG_G0)
6097c478bd9Sstevel@tonic-gate 		return (1);
6107c478bd9Sstevel@tonic-gate 
6117c478bd9Sstevel@tonic-gate 	/*
6127c478bd9Sstevel@tonic-gate 	 * We have encountered a jmpl that is storing the calling %pc in
6137c478bd9Sstevel@tonic-gate 	 * some register besides %i7, %o7 or %g0.  This is strange; emit
6147c478bd9Sstevel@tonic-gate 	 * a warning and fail.
6157c478bd9Sstevel@tonic-gate 	 */
6167c478bd9Sstevel@tonic-gate 	cmn_err(CE_NOTE, "cannot instrument return of %s at %p: unexpected "
6177c478bd9Sstevel@tonic-gate 	    "jmpl destination register", name, (void *)instr);
6187c478bd9Sstevel@tonic-gate 	return (0);
6197c478bd9Sstevel@tonic-gate }
6207c478bd9Sstevel@tonic-gate 
6217c478bd9Sstevel@tonic-gate static int
fbt_canpatch_retl(uint32_t * instr,int offset,const char * name)6227c478bd9Sstevel@tonic-gate fbt_canpatch_retl(uint32_t *instr, int offset, const char *name)
6237c478bd9Sstevel@tonic-gate {
6247c478bd9Sstevel@tonic-gate 	if (FBT_FMT1_OP(*instr) == FBT_OP_CALL ||
6257c478bd9Sstevel@tonic-gate 	    (FBT_FMT3_OP(*instr) == FBT_OP_JMPL &&
6267c478bd9Sstevel@tonic-gate 	    FBT_FMT3_RD(*instr) == FBT_REG_O7)) {
6277c478bd9Sstevel@tonic-gate 		/*
6287c478bd9Sstevel@tonic-gate 		 * If this is a call (or a jmpl that links into %o7), we can
6297c478bd9Sstevel@tonic-gate 		 * patch it iff the next instruction uses %o7 as a destination
6307c478bd9Sstevel@tonic-gate 		 * register.  Because there is an ABI responsibility to
6317c478bd9Sstevel@tonic-gate 		 * restore %o7 to the value before the call/jmpl, we don't
6327c478bd9Sstevel@tonic-gate 		 * particularly care how this routine is managing to restore
6337c478bd9Sstevel@tonic-gate 		 * it (mov, add, ld or divx for all we care).  If it doesn't
6347c478bd9Sstevel@tonic-gate 		 * seem to be restoring it at all, however, we'll refuse
6357c478bd9Sstevel@tonic-gate 		 * to patch it.
6367c478bd9Sstevel@tonic-gate 		 */
6377c478bd9Sstevel@tonic-gate 		uint32_t delay = *(instr + 1);
6387c478bd9Sstevel@tonic-gate 		uint32_t op, rd;
6397c478bd9Sstevel@tonic-gate 
6407c478bd9Sstevel@tonic-gate 		op = FBT_FMT1_OP(delay);
6417c478bd9Sstevel@tonic-gate 		rd = FBT_FMT3_RD(delay);
6427c478bd9Sstevel@tonic-gate 
6437c478bd9Sstevel@tonic-gate 		if (op != FBT_OP2 || rd != FBT_REG_O7) {
6447c478bd9Sstevel@tonic-gate 			/*
6457c478bd9Sstevel@tonic-gate 			 * This is odd.  Before we assume that we're looking
6467c478bd9Sstevel@tonic-gate 			 * at something bizarre (and warn accordingly), we'll
6477c478bd9Sstevel@tonic-gate 			 * check to see if it's obviously a jump table entry.
6487c478bd9Sstevel@tonic-gate 			 */
6497c478bd9Sstevel@tonic-gate 			if (*instr < (uintptr_t)instr &&
6507c478bd9Sstevel@tonic-gate 			    *instr >= (uintptr_t)instr - offset)
6517c478bd9Sstevel@tonic-gate 				return (0);
6527c478bd9Sstevel@tonic-gate 
6537c478bd9Sstevel@tonic-gate 			cmn_err(CE_NOTE, "cannot instrument return of %s at "
6547c478bd9Sstevel@tonic-gate 			    "%p: leaf jmpl/call delay isn't restoring %%o7",
6557c478bd9Sstevel@tonic-gate 			    name, (void *)instr);
6567c478bd9Sstevel@tonic-gate 			return (0);
6577c478bd9Sstevel@tonic-gate 		}
6587c478bd9Sstevel@tonic-gate 
6597c478bd9Sstevel@tonic-gate 		return (1);
6607c478bd9Sstevel@tonic-gate 	}
6617c478bd9Sstevel@tonic-gate 
6627c478bd9Sstevel@tonic-gate 	if (offset == sizeof (uint32_t)) {
6637c478bd9Sstevel@tonic-gate 		/*
6647c478bd9Sstevel@tonic-gate 		 * If this is the second instruction in the function, we're
6657c478bd9Sstevel@tonic-gate 		 * going to allow it to be patched if the first instruction
6667c478bd9Sstevel@tonic-gate 		 * is a patchable return-from-leaf instruction.
6677c478bd9Sstevel@tonic-gate 		 */
6687c478bd9Sstevel@tonic-gate 		if (fbt_canpatch_retl(instr - 1, 0, name))
6697c478bd9Sstevel@tonic-gate 			return (1);
6707c478bd9Sstevel@tonic-gate 	}
6717c478bd9Sstevel@tonic-gate 
6727c478bd9Sstevel@tonic-gate 	if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
6737c478bd9Sstevel@tonic-gate 		return (0);
6747c478bd9Sstevel@tonic-gate 
6757c478bd9Sstevel@tonic-gate 	if (FBT_FMT3_RD(*instr) != FBT_REG_G0)
6767c478bd9Sstevel@tonic-gate 		return (0);
6777c478bd9Sstevel@tonic-gate 
6787c478bd9Sstevel@tonic-gate 	return (1);
6797c478bd9Sstevel@tonic-gate }
6807c478bd9Sstevel@tonic-gate 
6817c478bd9Sstevel@tonic-gate /*ARGSUSED*/
6827c478bd9Sstevel@tonic-gate static uint32_t
fbt_patch_return(uint32_t * instr,uint32_t * funcbase,uint32_t * funclim,int offset,uint32_t id,fbt_trampoline_t * tramp,const char * name)6837c478bd9Sstevel@tonic-gate fbt_patch_return(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
6847c478bd9Sstevel@tonic-gate     int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
6857c478bd9Sstevel@tonic-gate {
6867c478bd9Sstevel@tonic-gate 	uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
6877c478bd9Sstevel@tonic-gate 	uint32_t cti = *instr, restore = *(instr + 1), rs1, dest;
6887c478bd9Sstevel@tonic-gate 	uintptr_t va = tramp->fbtt_va;
6897c478bd9Sstevel@tonic-gate 	uintptr_t base = tramp->fbtt_next;
6907c478bd9Sstevel@tonic-gate 	uint32_t locals[FBT_REG_NLOCALS], local;
6917c478bd9Sstevel@tonic-gate 
6927c478bd9Sstevel@tonic-gate 	if (tramp->fbtt_next + FBT_RETENT_MAXSIZE > tramp->fbtt_limit) {
6937c478bd9Sstevel@tonic-gate 		/*
6947c478bd9Sstevel@tonic-gate 		 * There isn't sufficient room for this entry; return failure.
6957c478bd9Sstevel@tonic-gate 		 */
6967c478bd9Sstevel@tonic-gate 		return (FBT_ILLTRAP);
6977c478bd9Sstevel@tonic-gate 	}
6987c478bd9Sstevel@tonic-gate 
6997c478bd9Sstevel@tonic-gate 	FBT_COUNTER(id, fbt_ret);
7007c478bd9Sstevel@tonic-gate 
7017c478bd9Sstevel@tonic-gate 	if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
7027c478bd9Sstevel@tonic-gate 		/*
7037c478bd9Sstevel@tonic-gate 		 * To handle the case of the return instruction, we'll emit a
7047c478bd9Sstevel@tonic-gate 		 * restore, followed by the instruction in the slot (which
7057c478bd9Sstevel@tonic-gate 		 * we'll transplant here), and then another save.  While it
7067c478bd9Sstevel@tonic-gate 		 * may seem intellectually unsatisfying to emit the additional
7077c478bd9Sstevel@tonic-gate 		 * restore/save couplet, one can take solace in the fact that
7087c478bd9Sstevel@tonic-gate 		 * we don't do this if the instruction in the return delay
7097c478bd9Sstevel@tonic-gate 		 * slot is a nop -- which it is nearly 90% of the time with
7107c478bd9Sstevel@tonic-gate 		 * gcc.  (And besides, this couplet can't induce unnecessary
7117c478bd9Sstevel@tonic-gate 		 * spill/fill traps; rewriting the delay instruction to be
7127c478bd9Sstevel@tonic-gate 		 * in terms of the current window hardly seems worth the
7137c478bd9Sstevel@tonic-gate 		 * trouble -- let alone the risk.)
7147c478bd9Sstevel@tonic-gate 		 */
7157c478bd9Sstevel@tonic-gate 		uint32_t delay = *(instr + 1);
7167c478bd9Sstevel@tonic-gate 		ASSERT(*instr == FBT_RETURN(FBT_REG_I7, 8));
7177c478bd9Sstevel@tonic-gate 
7187c478bd9Sstevel@tonic-gate 		cti = FBT_RET;
7197c478bd9Sstevel@tonic-gate 		restore = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
7207c478bd9Sstevel@tonic-gate 
7217c478bd9Sstevel@tonic-gate 		if (delay != FBT_SETHI(0, FBT_REG_G0)) {
7227c478bd9Sstevel@tonic-gate 			*tinstr++ = restore;
7237c478bd9Sstevel@tonic-gate 			*tinstr++ = delay;
7247c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_SAVEIMM(FBT_REG_O6,
7257c478bd9Sstevel@tonic-gate 			    -SA(MINFRAME), FBT_REG_O6);
7267c478bd9Sstevel@tonic-gate 		}
7277c478bd9Sstevel@tonic-gate 	}
7287c478bd9Sstevel@tonic-gate 
7297c478bd9Sstevel@tonic-gate 	FBT_REG_INITLOCALS(local, locals);
7307c478bd9Sstevel@tonic-gate 
7317c478bd9Sstevel@tonic-gate 	/*
7327c478bd9Sstevel@tonic-gate 	 * Mark the locals used in the jmpl.
7337c478bd9Sstevel@tonic-gate 	 */
7347c478bd9Sstevel@tonic-gate 	if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
7357c478bd9Sstevel@tonic-gate 		uint32_t rs1 = FBT_FMT3_RS1(cti);
7367c478bd9Sstevel@tonic-gate 		FBT_REG_MARKLOCAL(locals, rs1);
7377c478bd9Sstevel@tonic-gate 
7387c478bd9Sstevel@tonic-gate 		if (!FBT_FMT3_ISIMM(cti)) {
7397c478bd9Sstevel@tonic-gate 			uint32_t rs2 = FBT_FMT3_RS2(cti);
7407c478bd9Sstevel@tonic-gate 			FBT_REG_MARKLOCAL(locals, rs2);
7417c478bd9Sstevel@tonic-gate 		}
7427c478bd9Sstevel@tonic-gate 	}
7437c478bd9Sstevel@tonic-gate 
7447c478bd9Sstevel@tonic-gate 	/*
7457c478bd9Sstevel@tonic-gate 	 * And mark the locals used in the restore.
7467c478bd9Sstevel@tonic-gate 	 */
7477c478bd9Sstevel@tonic-gate 	rs1 = FBT_FMT3_RS1(restore);
7487c478bd9Sstevel@tonic-gate 	FBT_REG_MARKLOCAL(locals, rs1);
7497c478bd9Sstevel@tonic-gate 
7507c478bd9Sstevel@tonic-gate 	if (!FBT_FMT3_ISIMM(restore)) {
7517c478bd9Sstevel@tonic-gate 		uint32_t rs2 = FBT_FMT3_RS2(restore);
7527c478bd9Sstevel@tonic-gate 		FBT_REG_MARKLOCAL(locals, rs2);
7537c478bd9Sstevel@tonic-gate 	}
7547c478bd9Sstevel@tonic-gate 
7557c478bd9Sstevel@tonic-gate 	if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
7567c478bd9Sstevel@tonic-gate 		uint32_t rs1 = FBT_FMT3_RS1(cti);
7577c478bd9Sstevel@tonic-gate 
7587c478bd9Sstevel@tonic-gate 		if (FBT_REG_ISVOLATILE(rs1)) {
7597c478bd9Sstevel@tonic-gate 			FBT_REG_ALLOCLOCAL(local, locals);
7607c478bd9Sstevel@tonic-gate 			FBT_FMT3_RS1_SET(cti, local);
7617c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_MOV(rs1, local);
7627c478bd9Sstevel@tonic-gate 		}
7637c478bd9Sstevel@tonic-gate 
7647c478bd9Sstevel@tonic-gate 		if (!FBT_FMT3_ISIMM(cti)) {
7657c478bd9Sstevel@tonic-gate 			uint32_t rs2 = FBT_FMT3_RS2(cti);
7667c478bd9Sstevel@tonic-gate 
7677c478bd9Sstevel@tonic-gate 			if (FBT_REG_ISVOLATILE(rs2)) {
7687c478bd9Sstevel@tonic-gate 				FBT_REG_ALLOCLOCAL(local, locals);
7697c478bd9Sstevel@tonic-gate 				FBT_FMT3_RS2_SET(cti, local);
7707c478bd9Sstevel@tonic-gate 				*tinstr++ = FBT_MOV(rs2, local);
7717c478bd9Sstevel@tonic-gate 			}
7727c478bd9Sstevel@tonic-gate 		}
7737c478bd9Sstevel@tonic-gate 	}
7747c478bd9Sstevel@tonic-gate 
7757c478bd9Sstevel@tonic-gate 	rs1 = FBT_FMT3_RS1(restore);
7767c478bd9Sstevel@tonic-gate 
7777c478bd9Sstevel@tonic-gate 	if (FBT_REG_ISVOLATILE(rs1)) {
7787c478bd9Sstevel@tonic-gate 		FBT_REG_ALLOCLOCAL(local, locals);
7797c478bd9Sstevel@tonic-gate 		FBT_FMT3_RS1_SET(restore, local);
7807c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_MOV(rs1, local);
7817c478bd9Sstevel@tonic-gate 	}
7827c478bd9Sstevel@tonic-gate 
7837c478bd9Sstevel@tonic-gate 	if (!FBT_FMT3_ISIMM(restore)) {
7847c478bd9Sstevel@tonic-gate 		uint32_t rs2 = FBT_FMT3_RS2(restore);
7857c478bd9Sstevel@tonic-gate 
7867c478bd9Sstevel@tonic-gate 		if (FBT_REG_ISVOLATILE(rs2)) {
7877c478bd9Sstevel@tonic-gate 			FBT_REG_ALLOCLOCAL(local, locals);
7887c478bd9Sstevel@tonic-gate 			FBT_FMT3_RS2_SET(restore, local);
7897c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_MOV(rs2, local);
7907c478bd9Sstevel@tonic-gate 		}
7917c478bd9Sstevel@tonic-gate 	}
7927c478bd9Sstevel@tonic-gate 
7937c478bd9Sstevel@tonic-gate 	if (id > (uint32_t)FBT_SIMM13_MAX) {
7947c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_SETHI(id, FBT_REG_O0);
7957c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
7967c478bd9Sstevel@tonic-gate 	} else {
7977c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
7987c478bd9Sstevel@tonic-gate 	}
7997c478bd9Sstevel@tonic-gate 
8007c478bd9Sstevel@tonic-gate 	if (offset > (uint32_t)FBT_SIMM13_MAX) {
8017c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
8027c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
8037c478bd9Sstevel@tonic-gate 	} else {
8047c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
8057c478bd9Sstevel@tonic-gate 	}
8067c478bd9Sstevel@tonic-gate 
8077c478bd9Sstevel@tonic-gate 	*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
8087c478bd9Sstevel@tonic-gate 	tinstr++;
8097c478bd9Sstevel@tonic-gate 
8107c478bd9Sstevel@tonic-gate 	if (FBT_FMT3_RD(restore) == FBT_REG_O0) {
8117c478bd9Sstevel@tonic-gate 		/*
8127c478bd9Sstevel@tonic-gate 		 * If the destination register of the restore is %o0, we
8137c478bd9Sstevel@tonic-gate 		 * need to perform the implied calculation to derive the
8147c478bd9Sstevel@tonic-gate 		 * return value.
8157c478bd9Sstevel@tonic-gate 		 */
8167c478bd9Sstevel@tonic-gate 		uint32_t add = (restore & ~FBT_FMT3_OP_MASK) | FBT_OP_ADD;
8177c478bd9Sstevel@tonic-gate 		add &= ~FBT_FMT3_RD_MASK;
8187c478bd9Sstevel@tonic-gate 		*tinstr++ = add | (FBT_REG_O2 << FBT_FMT3_RD_SHIFT);
8197c478bd9Sstevel@tonic-gate 	} else {
8207c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
8217c478bd9Sstevel@tonic-gate 	}
8227c478bd9Sstevel@tonic-gate 
8237c478bd9Sstevel@tonic-gate 	/*
8247c478bd9Sstevel@tonic-gate 	 * If the control transfer instruction is %pc-relative (i.e. a
8257c478bd9Sstevel@tonic-gate 	 * call), we need to reset it appropriately.
8267c478bd9Sstevel@tonic-gate 	 */
8277c478bd9Sstevel@tonic-gate 	if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
8287c478bd9Sstevel@tonic-gate 		dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
8297c478bd9Sstevel@tonic-gate 		*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
8307c478bd9Sstevel@tonic-gate 		tinstr++;
8317c478bd9Sstevel@tonic-gate 	} else {
8327c478bd9Sstevel@tonic-gate 		*tinstr++ = cti;
8337c478bd9Sstevel@tonic-gate 	}
8347c478bd9Sstevel@tonic-gate 
8357c478bd9Sstevel@tonic-gate 	*tinstr++ = restore;
8367c478bd9Sstevel@tonic-gate 	tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
8377c478bd9Sstevel@tonic-gate 	tramp->fbtt_next = (uintptr_t)tinstr;
8387c478bd9Sstevel@tonic-gate 
8397c478bd9Sstevel@tonic-gate 	return (FBT_BAA(instr, va));
8407c478bd9Sstevel@tonic-gate }
8417c478bd9Sstevel@tonic-gate 
8427c478bd9Sstevel@tonic-gate static uint32_t
fbt_patch_retl(uint32_t * instr,uint32_t * funcbase,uint32_t * funclim,int offset,uint32_t id,fbt_trampoline_t * tramp,const char * name)8437c478bd9Sstevel@tonic-gate fbt_patch_retl(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
8447c478bd9Sstevel@tonic-gate     int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
8457c478bd9Sstevel@tonic-gate {
8467c478bd9Sstevel@tonic-gate 	uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
8477c478bd9Sstevel@tonic-gate 	uintptr_t va = tramp->fbtt_va;
8487c478bd9Sstevel@tonic-gate 	uintptr_t base = tramp->fbtt_next;
8497c478bd9Sstevel@tonic-gate 	uint32_t cti = *instr, dest;
8507c478bd9Sstevel@tonic-gate 	int annul = 0;
8517c478bd9Sstevel@tonic-gate 
8527c478bd9Sstevel@tonic-gate 	FBT_COUNTER(id, fbt_retl);
8537c478bd9Sstevel@tonic-gate 
8547c478bd9Sstevel@tonic-gate 	if (tramp->fbtt_next + FBT_RETLENT_MAXSIZE > tramp->fbtt_limit) {
8557c478bd9Sstevel@tonic-gate 		/*
8567c478bd9Sstevel@tonic-gate 		 * There isn't sufficient room for this entry; return failure.
8577c478bd9Sstevel@tonic-gate 		 */
8587c478bd9Sstevel@tonic-gate 		return (FBT_ILLTRAP);
8597c478bd9Sstevel@tonic-gate 	}
8607c478bd9Sstevel@tonic-gate 
8617c478bd9Sstevel@tonic-gate 	if (offset == sizeof (uint32_t) &&
8627c478bd9Sstevel@tonic-gate 	    fbt_canpatch_retl(instr - 1, 0, name)) {
8637c478bd9Sstevel@tonic-gate 		*tinstr++ = *instr;
8647c478bd9Sstevel@tonic-gate 		annul = 1;
8657c478bd9Sstevel@tonic-gate 		FBT_COUNTER(id, fbt_retl_twoinstr);
8667c478bd9Sstevel@tonic-gate 	} else {
8677c478bd9Sstevel@tonic-gate 		if (FBT_FMT3_OP(cti) == FBT_OP_JMPL &&
8687c478bd9Sstevel@tonic-gate 		    FBT_FMT3_RD(cti) != FBT_REG_O7 &&
8697c478bd9Sstevel@tonic-gate 		    FBT_FMT3_RS1(cti) != FBT_REG_O7) {
8707c478bd9Sstevel@tonic-gate 			annul = 1;
8717c478bd9Sstevel@tonic-gate 			*tinstr++ = *(instr + 1);
8727c478bd9Sstevel@tonic-gate 		}
8737c478bd9Sstevel@tonic-gate 	}
8747c478bd9Sstevel@tonic-gate 
8757c478bd9Sstevel@tonic-gate 	*tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
8767c478bd9Sstevel@tonic-gate 
8777c478bd9Sstevel@tonic-gate 	if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
8787c478bd9Sstevel@tonic-gate 		uint32_t rs1, rs2, o2i = FBT_REG_I0 - FBT_REG_O0;
8797c478bd9Sstevel@tonic-gate 
8807c478bd9Sstevel@tonic-gate 		/*
8817c478bd9Sstevel@tonic-gate 		 * If we have a jmpl and it's in terms of output registers, we
8827c478bd9Sstevel@tonic-gate 		 * need to rewrite it to be in terms of the corresponding input
8837c478bd9Sstevel@tonic-gate 		 * registers.  If it's in terms of the globals, we'll rewrite
8847c478bd9Sstevel@tonic-gate 		 * it to be in terms of locals.
8857c478bd9Sstevel@tonic-gate 		 */
8867c478bd9Sstevel@tonic-gate 		rs1 = FBT_FMT3_RS1(cti);
8877c478bd9Sstevel@tonic-gate 
8887c478bd9Sstevel@tonic-gate 		if (FBT_REG_ISOUTPUT(rs1))
8897c478bd9Sstevel@tonic-gate 			rs1 += o2i;
8907c478bd9Sstevel@tonic-gate 
8917c478bd9Sstevel@tonic-gate 		if (FBT_REG_ISGLOBAL(rs1)) {
8927c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_MOV(rs1, FBT_REG_L0);
8937c478bd9Sstevel@tonic-gate 			rs1 = FBT_REG_L0;
8947c478bd9Sstevel@tonic-gate 		}
8957c478bd9Sstevel@tonic-gate 
8967c478bd9Sstevel@tonic-gate 		FBT_FMT3_RS1_SET(cti, rs1);
8977c478bd9Sstevel@tonic-gate 
8987c478bd9Sstevel@tonic-gate 		if (!FBT_FMT3_ISIMM(cti)) {
8997c478bd9Sstevel@tonic-gate 			rs2 = FBT_FMT3_RS2(cti);
9007c478bd9Sstevel@tonic-gate 
9017c478bd9Sstevel@tonic-gate 			if (FBT_REG_ISOUTPUT(rs2))
9027c478bd9Sstevel@tonic-gate 				rs2 += o2i;
9037c478bd9Sstevel@tonic-gate 
9047c478bd9Sstevel@tonic-gate 			if (FBT_REG_ISGLOBAL(rs2)) {
9057c478bd9Sstevel@tonic-gate 				*tinstr++ = FBT_MOV(rs2, FBT_REG_L1);
9067c478bd9Sstevel@tonic-gate 				rs2 = FBT_REG_L1;
9077c478bd9Sstevel@tonic-gate 			}
9087c478bd9Sstevel@tonic-gate 
9097c478bd9Sstevel@tonic-gate 			FBT_FMT3_RS2_SET(cti, rs2);
9107c478bd9Sstevel@tonic-gate 		}
9117c478bd9Sstevel@tonic-gate 
9127c478bd9Sstevel@tonic-gate 		/*
9137c478bd9Sstevel@tonic-gate 		 * Now we need to check the rd and source register for the jmpl;
9147c478bd9Sstevel@tonic-gate 		 * If neither rd nor the source register is %o7, then we might
9157c478bd9Sstevel@tonic-gate 		 * have a jmp that is actually part of a jump table.  We need
9167c478bd9Sstevel@tonic-gate 		 * to generate the code to compare it to the base and limit of
9177c478bd9Sstevel@tonic-gate 		 * the function.
9187c478bd9Sstevel@tonic-gate 		 */
9197c478bd9Sstevel@tonic-gate 		if (FBT_FMT3_RD(cti) != FBT_REG_O7 && rs1 != FBT_REG_I7) {
9207c478bd9Sstevel@tonic-gate 			uintptr_t base = (uintptr_t)funcbase;
9217c478bd9Sstevel@tonic-gate 			uintptr_t limit = (uintptr_t)funclim;
9227c478bd9Sstevel@tonic-gate 
9237c478bd9Sstevel@tonic-gate 			FBT_COUNTER(id, fbt_retl_jmptab);
9247c478bd9Sstevel@tonic-gate 
9257c478bd9Sstevel@tonic-gate 			if (FBT_FMT3_ISIMM(cti)) {
9267c478bd9Sstevel@tonic-gate 				*tinstr++ = FBT_ADDSIMM13(rs1,
9277c478bd9Sstevel@tonic-gate 				    FBT_FMT3_SIMM13(cti), FBT_REG_L2);
9287c478bd9Sstevel@tonic-gate 			} else {
9297c478bd9Sstevel@tonic-gate 				*tinstr++ = FBT_ADD(rs1, rs2, FBT_REG_L2);
9307c478bd9Sstevel@tonic-gate 			}
9317c478bd9Sstevel@tonic-gate 
9327c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_SETHI(base, FBT_REG_L3);
9337c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_ORLO(FBT_REG_L3, base, FBT_REG_L3);
9347c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
9357c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_BL(0, 8 * sizeof (uint32_t));
9367c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_SETHI(limit, FBT_REG_L3);
9377c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_ORLO(FBT_REG_L3, limit, FBT_REG_L3);
9387c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
9397c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_BGE(0, 4 * sizeof (uint32_t));
9407c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_SETHI(0, FBT_REG_G0);
9417c478bd9Sstevel@tonic-gate 			*tinstr++ = cti;
9427c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_RESTORE(FBT_REG_G0,
9437c478bd9Sstevel@tonic-gate 			    FBT_REG_G0, FBT_REG_G0);
9447c478bd9Sstevel@tonic-gate 		}
9457c478bd9Sstevel@tonic-gate 	}
9467c478bd9Sstevel@tonic-gate 
9477c478bd9Sstevel@tonic-gate 	if (id > (uint32_t)FBT_SIMM13_MAX) {
9487c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_SETHI(id, FBT_REG_O0);
9497c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
9507c478bd9Sstevel@tonic-gate 	} else {
9517c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
9527c478bd9Sstevel@tonic-gate 	}
9537c478bd9Sstevel@tonic-gate 
9547c478bd9Sstevel@tonic-gate 	if (offset > (uint32_t)FBT_SIMM13_MAX) {
9557c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
9567c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
9577c478bd9Sstevel@tonic-gate 	} else {
9587c478bd9Sstevel@tonic-gate 		*tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
9597c478bd9Sstevel@tonic-gate 	}
9607c478bd9Sstevel@tonic-gate 
9617c478bd9Sstevel@tonic-gate 	*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
9627c478bd9Sstevel@tonic-gate 	tinstr++;
9637c478bd9Sstevel@tonic-gate 	*tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
9647c478bd9Sstevel@tonic-gate 
9657c478bd9Sstevel@tonic-gate 	/*
9667c478bd9Sstevel@tonic-gate 	 * If the control transfer instruction is %pc-relative (i.e. a
9677c478bd9Sstevel@tonic-gate 	 * call), we need to reset it appropriately.
9687c478bd9Sstevel@tonic-gate 	 */
9697c478bd9Sstevel@tonic-gate 	if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
9707c478bd9Sstevel@tonic-gate 		FBT_COUNTER(id, fbt_retl_tailcall);
9717c478bd9Sstevel@tonic-gate 		dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
9727c478bd9Sstevel@tonic-gate 		*tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
9737c478bd9Sstevel@tonic-gate 		tinstr++;
9747c478bd9Sstevel@tonic-gate 		annul = 1;
9757c478bd9Sstevel@tonic-gate 	} else {
9767c478bd9Sstevel@tonic-gate 		if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
9777c478bd9Sstevel@tonic-gate 			*tinstr++ = cti;
9787c478bd9Sstevel@tonic-gate 
9797c478bd9Sstevel@tonic-gate 			if (FBT_FMT3_RD(cti) == FBT_REG_O7) {
9807c478bd9Sstevel@tonic-gate 				FBT_COUNTER(id, fbt_retl_tailjmpl);
9817c478bd9Sstevel@tonic-gate 				annul = 1;
9827c478bd9Sstevel@tonic-gate 			}
9837c478bd9Sstevel@tonic-gate 		} else {
9847c478bd9Sstevel@tonic-gate 			*tinstr++ = FBT_RET;
9857c478bd9Sstevel@tonic-gate 		}
9867c478bd9Sstevel@tonic-gate 	}
9877c478bd9Sstevel@tonic-gate 
9887c478bd9Sstevel@tonic-gate 	*tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
9897c478bd9Sstevel@tonic-gate 
9907c478bd9Sstevel@tonic-gate 	tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
9917c478bd9Sstevel@tonic-gate 	tramp->fbtt_next = (uintptr_t)tinstr;
9927c478bd9Sstevel@tonic-gate 
9937c478bd9Sstevel@tonic-gate 	return (annul ? FBT_BAA(instr, va) : FBT_BA(instr, va));
9947c478bd9Sstevel@tonic-gate }
9957c478bd9Sstevel@tonic-gate 
9967c478bd9Sstevel@tonic-gate /*ARGSUSED*/
9977c478bd9Sstevel@tonic-gate static void
fbt_provide_module(void * arg,struct modctl * ctl)9987c478bd9Sstevel@tonic-gate fbt_provide_module(void *arg, struct modctl *ctl)
9997c478bd9Sstevel@tonic-gate {
10007c478bd9Sstevel@tonic-gate 	struct module *mp = ctl->mod_mp;
10017c478bd9Sstevel@tonic-gate 	char *modname = ctl->mod_modname;
10027c478bd9Sstevel@tonic-gate 	char *str = mp->strings;
10037c478bd9Sstevel@tonic-gate 	int nsyms = mp->nsyms;
10047c478bd9Sstevel@tonic-gate 	Shdr *symhdr = mp->symhdr;
10057c478bd9Sstevel@tonic-gate 	size_t symsize;
10067c478bd9Sstevel@tonic-gate 	char *name;
10077c478bd9Sstevel@tonic-gate 	int i;
10087c478bd9Sstevel@tonic-gate 	fbt_probe_t *fbt, *retfbt;
10097c478bd9Sstevel@tonic-gate 	fbt_trampoline_t tramp;
10107c478bd9Sstevel@tonic-gate 	uintptr_t offset;
10117c478bd9Sstevel@tonic-gate 	int primary = 0;
10127c478bd9Sstevel@tonic-gate 	ctf_file_t *fp = NULL;
10137c478bd9Sstevel@tonic-gate 	int error;
10147c478bd9Sstevel@tonic-gate 	int estimate = 1;
10157c478bd9Sstevel@tonic-gate 	uint32_t faketramp[50];
10167c478bd9Sstevel@tonic-gate 	size_t fbt_size = 0;
10177c478bd9Sstevel@tonic-gate 
10187c478bd9Sstevel@tonic-gate 	/*
10197c478bd9Sstevel@tonic-gate 	 * Employees of dtrace and their families are ineligible.  Void
10207c478bd9Sstevel@tonic-gate 	 * where prohibited.
10217c478bd9Sstevel@tonic-gate 	 */
10227c478bd9Sstevel@tonic-gate 	if (strcmp(modname, "dtrace") == 0)
10237c478bd9Sstevel@tonic-gate 		return;
10247c478bd9Sstevel@tonic-gate 
10257c478bd9Sstevel@tonic-gate 	if (ctl->mod_requisites != NULL) {
10267c478bd9Sstevel@tonic-gate 		struct modctl_list *list;
10277c478bd9Sstevel@tonic-gate 
10287c478bd9Sstevel@tonic-gate 		list = (struct modctl_list *)ctl->mod_requisites;
10297c478bd9Sstevel@tonic-gate 
10307c478bd9Sstevel@tonic-gate 		for (; list != NULL; list = list->modl_next) {
10317c478bd9Sstevel@tonic-gate 			if (strcmp(list->modl_modp->mod_modname, "dtrace") == 0)
10327c478bd9Sstevel@tonic-gate 				return;
10337c478bd9Sstevel@tonic-gate 		}
10347c478bd9Sstevel@tonic-gate 	}
10357c478bd9Sstevel@tonic-gate 
10367c478bd9Sstevel@tonic-gate 	/*
10377c478bd9Sstevel@tonic-gate 	 * KMDB is ineligible for instrumentation -- it may execute in
10387c478bd9Sstevel@tonic-gate 	 * any context, including probe context.
10397c478bd9Sstevel@tonic-gate 	 */
10407c478bd9Sstevel@tonic-gate 	if (strcmp(modname, "kmdbmod") == 0)
10417c478bd9Sstevel@tonic-gate 		return;
10427c478bd9Sstevel@tonic-gate 
1043*692bcae7SToomas Soome 	if (str == NULL || symhdr == NULL || symhdr->sh_addr == 0) {
10447c478bd9Sstevel@tonic-gate 		/*
10457c478bd9Sstevel@tonic-gate 		 * If this module doesn't (yet) have its string or symbol
10467c478bd9Sstevel@tonic-gate 		 * table allocated, clear out.
10477c478bd9Sstevel@tonic-gate 		 */
10487c478bd9Sstevel@tonic-gate 		return;
10497c478bd9Sstevel@tonic-gate 	}
10507c478bd9Sstevel@tonic-gate 
10517c478bd9Sstevel@tonic-gate 	symsize = symhdr->sh_entsize;
10527c478bd9Sstevel@tonic-gate 
10537c478bd9Sstevel@tonic-gate 	if (mp->fbt_nentries) {
10547c478bd9Sstevel@tonic-gate 		/*
10557c478bd9Sstevel@tonic-gate 		 * This module has some FBT entries allocated; we're afraid
10567c478bd9Sstevel@tonic-gate 		 * to screw with it.
10577c478bd9Sstevel@tonic-gate 		 */
10587c478bd9Sstevel@tonic-gate 		return;
10597c478bd9Sstevel@tonic-gate 	}
10607c478bd9Sstevel@tonic-gate 
10617c478bd9Sstevel@tonic-gate 	if (mp->fbt_tab != NULL)
10627c478bd9Sstevel@tonic-gate 		estimate = 0;
10637c478bd9Sstevel@tonic-gate 
10647c478bd9Sstevel@tonic-gate 	/*
10657c478bd9Sstevel@tonic-gate 	 * This is a hack for unix/genunix/krtld.
10667c478bd9Sstevel@tonic-gate 	 */
10677c478bd9Sstevel@tonic-gate 	primary = vmem_contains(heap_arena, (void *)ctl,
10687c478bd9Sstevel@tonic-gate 	    sizeof (struct modctl)) == 0;
10697c478bd9Sstevel@tonic-gate 	kobj_textwin_alloc(mp);
10707c478bd9Sstevel@tonic-gate 
10717c478bd9Sstevel@tonic-gate 	/*
10727c478bd9Sstevel@tonic-gate 	 * Open the CTF data for the module.  We'll use this to determine the
10737c478bd9Sstevel@tonic-gate 	 * functions that can be instrumented.  Note that this call can fail,
10747c478bd9Sstevel@tonic-gate 	 * in which case we'll use heuristics to determine the functions that
10757c478bd9Sstevel@tonic-gate 	 * can be instrumented.  (But in particular, leaf functions will not be
10767c478bd9Sstevel@tonic-gate 	 * instrumented.)
10777c478bd9Sstevel@tonic-gate 	 */
10787c478bd9Sstevel@tonic-gate 	fp = ctf_modopen(mp, &error);
10797c478bd9Sstevel@tonic-gate 
10807c478bd9Sstevel@tonic-gate forreal:
10817c478bd9Sstevel@tonic-gate 	if (!estimate) {
10827c478bd9Sstevel@tonic-gate 		tramp.fbtt_next =
10837c478bd9Sstevel@tonic-gate 		    (uintptr_t)fbt_trampoline_map((uintptr_t)mp->fbt_tab,
10847c478bd9Sstevel@tonic-gate 		    mp->fbt_size);
10857c478bd9Sstevel@tonic-gate 		tramp.fbtt_limit = tramp.fbtt_next + mp->fbt_size;
10867c478bd9Sstevel@tonic-gate 		tramp.fbtt_va = (uintptr_t)mp->fbt_tab;
10877c478bd9Sstevel@tonic-gate 	}
10887c478bd9Sstevel@tonic-gate 
10897c478bd9Sstevel@tonic-gate 	for (i = 1; i < nsyms; i++) {
10907c478bd9Sstevel@tonic-gate 		ctf_funcinfo_t f;
10917c478bd9Sstevel@tonic-gate 		uint32_t *instr, *base, *limit;
10927c478bd9Sstevel@tonic-gate 		Sym *sym = (Sym *)(symhdr->sh_addr + i * symsize);
10937c478bd9Sstevel@tonic-gate 		int have_ctf = 0, is_leaf = 0, nargs, cti = 0;
10947c478bd9Sstevel@tonic-gate 		int (*canpatch)(uint32_t *, int, const char *);
10957c478bd9Sstevel@tonic-gate 		uint32_t (*patch)(uint32_t *, uint32_t *, uint32_t *, int,
10967c478bd9Sstevel@tonic-gate 		    uint32_t, fbt_trampoline_t *, const char *);
10977c478bd9Sstevel@tonic-gate 
10987c478bd9Sstevel@tonic-gate 		if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
10997c478bd9Sstevel@tonic-gate 			continue;
11007c478bd9Sstevel@tonic-gate 
11017c478bd9Sstevel@tonic-gate 		/*
11027c478bd9Sstevel@tonic-gate 		 * Weak symbols are not candidates.  This could be made to
11037c478bd9Sstevel@tonic-gate 		 * work (where weak functions and their underlying function
11047c478bd9Sstevel@tonic-gate 		 * appear as two disjoint probes), but it's not simple.
11057c478bd9Sstevel@tonic-gate 		 */
11067c478bd9Sstevel@tonic-gate 		if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
11077c478bd9Sstevel@tonic-gate 			continue;
11087c478bd9Sstevel@tonic-gate 
11097c478bd9Sstevel@tonic-gate 		name = str + sym->st_name;
11107c478bd9Sstevel@tonic-gate 
11117c478bd9Sstevel@tonic-gate 		if (strstr(name, "dtrace_") == name &&
11127c478bd9Sstevel@tonic-gate 		    strstr(name, "dtrace_safe_") != name) {
11137c478bd9Sstevel@tonic-gate 			/*
11147c478bd9Sstevel@tonic-gate 			 * Anything beginning with "dtrace_" may be called
11157c478bd9Sstevel@tonic-gate 			 * from probe context unless it explitly indicates
11167c478bd9Sstevel@tonic-gate 			 * that it won't be called from probe context by
11177c478bd9Sstevel@tonic-gate 			 * using the prefix "dtrace_safe_".
11187c478bd9Sstevel@tonic-gate 			 */
11197c478bd9Sstevel@tonic-gate 			continue;
11207c478bd9Sstevel@tonic-gate 		}
11217c478bd9Sstevel@tonic-gate 
1122a1b5e537Sbmc 		if (strstr(name, "kdi_") == name ||
1123a1b5e537Sbmc 		    strstr(name, "_kdi_") != NULL) {
11247c478bd9Sstevel@tonic-gate 			/*
1125a1b5e537Sbmc 			 * Any function name beginning with "kdi_" or
1126a1b5e537Sbmc 			 * containing the string "_kdi_" is a part of the
11277c478bd9Sstevel@tonic-gate 			 * kernel debugger interface and may be called in
11287c478bd9Sstevel@tonic-gate 			 * arbitrary context -- including probe context.
11297c478bd9Sstevel@tonic-gate 			 */
11307c478bd9Sstevel@tonic-gate 			continue;
11317c478bd9Sstevel@tonic-gate 		}
11327c478bd9Sstevel@tonic-gate 
11337c478bd9Sstevel@tonic-gate 		if (strstr(name, "__relocatable") != NULL) {
11347c478bd9Sstevel@tonic-gate 			/*
11357c478bd9Sstevel@tonic-gate 			 * Anything with the string "__relocatable" anywhere
1136