1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26
27 #include <sys/errno.h>
28 #include <sys/stat.h>
29 #include <sys/modctl.h>
30 #include <sys/conf.h>
31 #include <sys/systm.h>
32 #include <sys/ddi.h>
33 #include <sys/sunddi.h>
34 #include <sys/cpuvar.h>
35 #include <sys/kmem.h>
36 #include <sys/strsubr.h>
37 #include <sys/dtrace.h>
38 #include <sys/kobj.h>
39 #include <sys/modctl.h>
40 #include <sys/atomic.h>
41 #include <vm/seg_kmem.h>
42 #include <sys/stack.h>
43 #include <sys/ctf_api.h>
44 #include <sys/sysmacros.h>
45
46 static dev_info_t *fbt_devi;
47 static dtrace_provider_id_t fbt_id;
48 static uintptr_t fbt_trampoline;
49 static caddr_t fbt_trampoline_window;
50 static size_t fbt_trampoline_size;
51 static int fbt_verbose = 0;
52
53 /*
54 * Various interesting bean counters.
55 */
56 static int fbt_entry;
57 static int fbt_ret;
58 static int fbt_retl;
59 static int fbt_retl_jmptab;
60 static int fbt_retl_twoinstr;
61 static int fbt_retl_tailcall;
62 static int fbt_retl_tailjmpl;
63 static int fbt_leaf_functions;
64
65 extern char stubs_base[];
66 extern char stubs_end[];
67
68 #define FBT_REG_G0 0
69 #define FBT_REG_G1 1
70 #define FBT_REG_O0 8
71 #define FBT_REG_O1 9
72 #define FBT_REG_O2 10
73 #define FBT_REG_O3 11
74 #define FBT_REG_O4 12
75 #define FBT_REG_O5 13
76 #define FBT_REG_O6 14
77 #define FBT_REG_O7 15
78 #define FBT_REG_I0 24
79 #define FBT_REG_I1 25
80 #define FBT_REG_I2 26
81 #define FBT_REG_I3 27
82 #define FBT_REG_I4 28
83 #define FBT_REG_I7 31
84 #define FBT_REG_L0 16
85 #define FBT_REG_L1 17
86 #define FBT_REG_L2 18
87 #define FBT_REG_L3 19
88 #define FBT_REG_PC 5
89
90 #define FBT_REG_ISGLOBAL(r) ((r) < 8)
91 #define FBT_REG_ISOUTPUT(r) ((r) >= 8 && (r) < 16)
92 #define FBT_REG_ISLOCAL(r) ((r) >= 16 && (r) < 24)
93 #define FBT_REG_ISVOLATILE(r) \
94 ((FBT_REG_ISGLOBAL(r) || FBT_REG_ISOUTPUT(r)) && (r) != FBT_REG_G0)
95 #define FBT_REG_NLOCALS 8
96
97 #define FBT_REG_MARKLOCAL(locals, r) \
98 if (FBT_REG_ISLOCAL(r)) \
99 (locals)[(r) - FBT_REG_L0] = 1;
100
101 #define FBT_REG_INITLOCALS(local, locals) \
102 for ((local) = 0; (local) < FBT_REG_NLOCALS; (local)++) \
103 (locals)[(local)] = 0; \
104 (local) = FBT_REG_L0
105
106 #define FBT_REG_ALLOCLOCAL(local, locals) \
107 while ((locals)[(local) - FBT_REG_L0]) \
108 (local)++; \
109 (locals)[(local) - FBT_REG_L0] = 1;
110
111 #define FBT_OP_MASK 0xc0000000
112 #define FBT_OP_SHIFT 30
113 #define FBT_OP(val) ((val) & FBT_FMT1_MASK)
114
115 #define FBT_SIMM13_MASK 0x1fff
116 #define FBT_SIMM13_MAX ((int32_t)0xfff)
117 #define FBT_IMM22_MASK 0x3fffff
118 #define FBT_IMM22_SHIFT 10
119 #define FBT_IMM10_MASK 0x3ff
120
121 #define FBT_DISP30_MASK 0x3fffffff
122 #define FBT_DISP30(from, to) \
123 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP30_MASK)
124
125 #define FBT_DISP22_MASK 0x3fffff
126 #define FBT_DISP22(from, to) \
127 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP22_MASK)
128
129 #define FBT_DISP19_MASK 0x7ffff
130 #define FBT_DISP19(from, to) \
131 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP19_MASK)
132
133 #define FBT_DISP16_HISHIFT 20
134 #define FBT_DISP16_HIMASK (0x3 << FBT_DISP16_HISHIFT)
135 #define FBT_DISP16_LOMASK (0x3fff)
136 #define FBT_DISP16_MASK (FBT_DISP16_HIMASK | FBT_DISP16_LOMASK)
137 #define FBT_DISP16(val) \
138 ((((val) & FBT_DISP16_HIMASK) >> 6) | ((val) & FBT_DISP16_LOMASK))
139
140 #define FBT_DISP14_MASK 0x3fff
141 #define FBT_DISP14(from, to) \
142 (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP14_MASK)
143
144 #define FBT_OP0 (((uint32_t)0) << FBT_OP_SHIFT)
145 #define FBT_OP1 (((uint32_t)1) << FBT_OP_SHIFT)
146 #define FBT_OP2 (((uint32_t)2) << FBT_OP_SHIFT)
147 #define FBT_ILLTRAP 0
148
149 #define FBT_ANNUL_SHIFT 29
150 #define FBT_ANNUL (1 << FBT_ANNUL_SHIFT)
151
152 #define FBT_FMT3_OP3_SHIFT 19
153 #define FBT_FMT3_OP_MASK 0xc1f80000
154 #define FBT_FMT3_OP(val) ((val) & FBT_FMT3_OP_MASK)
155
156 #define FBT_FMT3_RD_SHIFT 25
157 #define FBT_FMT3_RD_MASK (0x1f << FBT_FMT3_RD_SHIFT)
158 #define FBT_FMT3_RD(val) \
159 (((val) & FBT_FMT3_RD_MASK) >> FBT_FMT3_RD_SHIFT)
160
161 #define FBT_FMT3_RS1_SHIFT 14
162 #define FBT_FMT3_RS1_MASK (0x1f << FBT_FMT3_RS1_SHIFT)
163 #define FBT_FMT3_RS1(val) \
164 (((val) & FBT_FMT3_RS1_MASK) >> FBT_FMT3_RS1_SHIFT)
165 #define FBT_FMT3_RS1_SET(val, rs1) \
166 (val) = ((val) & ~FBT_FMT3_RS1_MASK) | ((rs1) << FBT_FMT3_RS1_SHIFT)
167
168 #define FBT_FMT3_RS2_SHIFT 0
169 #define FBT_FMT3_RS2_MASK (0x1f << FBT_FMT3_RS2_SHIFT)
170 #define FBT_FMT3_RS2(val) \
171 (((val) & FBT_FMT3_RS2_MASK) >> FBT_FMT3_RS2_SHIFT)
172 #define FBT_FMT3_RS2_SET(val, rs2) \
173 (val) = ((val) & ~FBT_FMT3_RS2_MASK) | ((rs2) << FBT_FMT3_RS2_SHIFT)
174
175 #define FBT_FMT3_IMM_SHIFT 13
176 #define FBT_FMT3_IMM (1 << FBT_FMT3_IMM_SHIFT)
177 #define FBT_FMT3_SIMM13_MASK FBT_SIMM13_MASK
178
179 #define FBT_FMT3_ISIMM(val) ((val) & FBT_FMT3_IMM)
180 #define FBT_FMT3_SIMM13(val) ((val) & FBT_FMT3_SIMM13_MASK)
181
182 #define FBT_FMT2_OP2_SHIFT 22
183 #define FBT_FMT2_OP2_MASK (0x7 << FBT_FMT2_OP2_SHIFT)
184 #define FBT_FMT2_RD_SHIFT 25
185
186 #define FBT_FMT1_OP(val) ((val) & FBT_OP_MASK)
187 #define FBT_FMT1_DISP30(val) ((val) & FBT_DISP30_MASK)
188
189 #define FBT_FMT2_OP2_BPCC (0x01 << FBT_FMT2_OP2_SHIFT)
190 #define FBT_FMT2_OP2_BCC (0x02 << FBT_FMT2_OP2_SHIFT)
191 #define FBT_FMT2_OP2_BPR (0x03 << FBT_FMT2_OP2_SHIFT)
192 #define FBT_FMT2_OP2_SETHI (0x04 << FBT_FMT2_OP2_SHIFT)
193
194 #define FBT_FMT2_COND_SHIFT 25
195 #define FBT_FMT2_COND_BA (0x8 << FBT_FMT2_COND_SHIFT)
196 #define FBT_FMT2_COND_BL (0x3 << FBT_FMT2_COND_SHIFT)
197 #define FBT_FMT2_COND_BGE (0xb << FBT_FMT2_COND_SHIFT)
198
199 #define FBT_OP_RESTORE (FBT_OP2 | (0x3d << FBT_FMT3_OP3_SHIFT))
200 #define FBT_OP_SAVE (FBT_OP2 | (0x3c << FBT_FMT3_OP3_SHIFT))
201 #define FBT_OP_JMPL (FBT_OP2 | (0x38 << FBT_FMT3_OP3_SHIFT))
202 #define FBT_OP_RETURN (FBT_OP2 | (0x39 << FBT_FMT3_OP3_SHIFT))
203 #define FBT_OP_CALL FBT_OP1
204 #define FBT_OP_SETHI (FBT_OP0 | FBT_FMT2_OP2_SETHI)
205 #define FBT_OP_ADD (FBT_OP2 | (0x00 << FBT_FMT3_OP3_SHIFT))
206 #define FBT_OP_OR (FBT_OP2 | (0x02 << FBT_FMT3_OP3_SHIFT))
207 #define FBT_OP_SUB (FBT_OP2 | (0x04 << FBT_FMT3_OP3_SHIFT))
208 #define FBT_OP_CC (FBT_OP2 | (0x10 << FBT_FMT3_OP3_SHIFT))
209 #define FBT_OP_BA (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BA)
210 #define FBT_OP_BL (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BL)
211 #define FBT_OP_BGE (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BGE)
212 #define FBT_OP_BAPCC (FBT_OP0 | FBT_FMT2_OP2_BPCC | FBT_FMT2_COND_BA)
213 #define FBT_OP_RD (FBT_OP2 | (0x28 << FBT_FMT3_OP3_SHIFT))
214
215 #define FBT_ORLO(rs, val, rd) \
216 (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
217 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_IMM10_MASK))
218
219 #define FBT_ORSIMM13(rs, val, rd) \
220 (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
221 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
222
223 #define FBT_ADDSIMM13(rs, val, rd) \
224 (FBT_OP_ADD | ((rs) << FBT_FMT3_RS1_SHIFT) | \
225 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
226
227 #define FBT_ADD(rs1, rs2, rd) \
228 (FBT_OP_ADD | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
229 ((rs2) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))
230
231 #define FBT_CMP(rs1, rs2) \
232 (FBT_OP_SUB | FBT_OP_CC | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
233 ((rs2) << FBT_FMT3_RS2_SHIFT) | (FBT_REG_G0 << FBT_FMT3_RD_SHIFT))
234
235 #define FBT_MOV(rs, rd) \
236 (FBT_OP_OR | (FBT_REG_G0 << FBT_FMT3_RS1_SHIFT) | \
237 ((rs) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))
238
239 #define FBT_SETHI(val, reg) \
240 (FBT_OP_SETHI | (reg << FBT_FMT2_RD_SHIFT) | \
241 ((val >> FBT_IMM22_SHIFT) & FBT_IMM22_MASK))
242
243 #define FBT_CALL(orig, dest) (FBT_OP_CALL | FBT_DISP30(orig, dest))
244
245 #define FBT_RET \
246 (FBT_OP_JMPL | (FBT_REG_I7 << FBT_FMT3_RS1_SHIFT) | \
247 (FBT_REG_G0 << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | (sizeof (pc_t) << 1))
248
249 #define FBT_SAVEIMM(rd, val, rs1) \
250 (FBT_OP_SAVE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
251 ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
252
253 #define FBT_RESTORE(rd, rs1, rs2) \
254 (FBT_OP_RESTORE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
255 ((rd) << FBT_FMT3_RD_SHIFT) | ((rs2) << FBT_FMT3_RS2_SHIFT))
256
257 #define FBT_RETURN(rs1, val) \
258 (FBT_OP_RETURN | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
259 FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))
260
261 #define FBT_BA(orig, dest) (FBT_OP_BA | FBT_DISP22(orig, dest))
262 #define FBT_BAA(orig, dest) (FBT_BA(orig, dest) | FBT_ANNUL)
263 #define FBT_BL(orig, dest) (FBT_OP_BL | FBT_DISP22(orig, dest))
264 #define FBT_BGE(orig, dest) (FBT_OP_BGE | FBT_DISP22(orig, dest))
265 #define FBT_BDEST(va, instr) ((uintptr_t)(va) + \
266 (((int32_t)(((instr) & FBT_DISP22_MASK) << 10)) >> 8))
267 #define FBT_BPCCDEST(va, instr) ((uintptr_t)(va) + \
268 (((int32_t)(((instr) & FBT_DISP19_MASK) << 13)) >> 11))
269 #define FBT_BPRDEST(va, instr) ((uintptr_t)(va) + \
270 (((int32_t)((FBT_DISP16(instr)) << 16)) >> 14))
271
272 /*
273 * We're only going to treat a save as safe if (a) both rs1 and rd are
274 * %sp and (b) if the instruction has a simm, the value isn't 0.
275 */
276 #define FBT_IS_SAVE(instr) \
277 (FBT_FMT3_OP(instr) == FBT_OP_SAVE && \
278 FBT_FMT3_RD(instr) == FBT_REG_O6 && \
279 FBT_FMT3_RS1(instr) == FBT_REG_O6 && \
280 !(FBT_FMT3_ISIMM(instr) && FBT_FMT3_SIMM13(instr) == 0))
281
282 #define FBT_IS_BA(instr) (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BA)
283 #define FBT_IS_BAPCC(instr) (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BAPCC)
284
285 #define FBT_IS_RDPC(instr) ((FBT_FMT3_OP(instr) == FBT_OP_RD) && \
286 (FBT_FMT3_RD(instr) == FBT_REG_PC))
287
288 #define FBT_IS_PCRELATIVE(instr) \
289 ((((instr) & FBT_OP_MASK) == FBT_OP0 && \
290 ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
291 ((instr) & FBT_OP_MASK) == FBT_OP1 || \
292 FBT_IS_RDPC(instr))
293
294 #define FBT_IS_CTI(instr) \
295 ((((instr) & FBT_OP_MASK) == FBT_OP0 && \
296 ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
297 ((instr) & FBT_OP_MASK) == FBT_OP1 || \
298 (FBT_FMT3_OP(instr) == FBT_OP_JMPL) || \
299 (FBT_FMT3_OP(instr) == FBT_OP_RETURN))
300
301 #define FBT_PROBENAME_ENTRY "entry"
302 #define FBT_PROBENAME_RETURN "return"
303 #define FBT_ESTIMATE_ID (UINT32_MAX)
304 #define FBT_COUNTER(id, count) if ((id) != FBT_ESTIMATE_ID) (count)++
305
306 #define FBT_ENTENT_MAXSIZE (16 * sizeof (uint32_t))
307 #define FBT_RETENT_MAXSIZE (11 * sizeof (uint32_t))
308 #define FBT_RETLENT_MAXSIZE (23 * sizeof (uint32_t))
309 #define FBT_ENT_MAXSIZE \
310 MAX(MAX(FBT_ENTENT_MAXSIZE, FBT_RETENT_MAXSIZE), FBT_RETLENT_MAXSIZE)
311
312 typedef struct fbt_probe {
313 char *fbtp_name;
314 dtrace_id_t fbtp_id;
315 uintptr_t fbtp_addr;
316 struct modctl *fbtp_ctl;
317 int fbtp_loadcnt;
318 int fbtp_symndx;
319 int fbtp_primary;
320 int fbtp_return;
321 uint32_t *fbtp_patchpoint;
322 uint32_t fbtp_patchval;
323 uint32_t fbtp_savedval;
324 struct fbt_probe *fbtp_next;
325 } fbt_probe_t;
326
327 typedef struct fbt_trampoline {
328 uintptr_t fbtt_va;
329 uintptr_t fbtt_limit;
330 uintptr_t fbtt_next;
331 } fbt_trampoline_t;
332
333 static caddr_t
fbt_trampoline_map(uintptr_t tramp,size_t size)334 fbt_trampoline_map(uintptr_t tramp, size_t size)
335 {
336 uintptr_t offs;
337 page_t **ppl;
338
339 ASSERT(fbt_trampoline_window == NULL);
340 ASSERT(fbt_trampoline_size == 0);
341 ASSERT(fbt_trampoline == NULL);
342
343 size += tramp & PAGEOFFSET;
344 fbt_trampoline = tramp & PAGEMASK;
345 fbt_trampoline_size = (size + PAGESIZE - 1) & PAGEMASK;
346 fbt_trampoline_window =
347 vmem_alloc(heap_arena, fbt_trampoline_size, VM_SLEEP);
348
349 (void) as_pagelock(&kas, &ppl, (caddr_t)fbt_trampoline,
350 fbt_trampoline_size, S_WRITE);
351
352 for (offs = 0; offs < fbt_trampoline_size; offs += PAGESIZE) {
353 hat_devload(kas.a_hat, fbt_trampoline_window + offs, PAGESIZE,
354 hat_getpfnum(kas.a_hat, (caddr_t)fbt_trampoline + offs),
355 PROT_READ | PROT_WRITE,
356 HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
357 }
358
359 as_pageunlock(&kas, ppl, (caddr_t)fbt_trampoline, fbt_trampoline_size,
360 S_WRITE);
361
362 return (fbt_trampoline_window + (tramp & PAGEOFFSET));
363 }
364
365 static void
fbt_trampoline_unmap()366 fbt_trampoline_unmap()
367 {
368 ASSERT(fbt_trampoline_window != NULL);
369 ASSERT(fbt_trampoline_size != 0);
370 ASSERT(fbt_trampoline != NULL);
371
372 membar_enter();
373 sync_icache((caddr_t)fbt_trampoline, fbt_trampoline_size);
374 sync_icache(fbt_trampoline_window, fbt_trampoline_size);
375
376 hat_unload(kas.a_hat, fbt_trampoline_window, fbt_trampoline_size,
377 HAT_UNLOAD_UNLOCK);
378
379 vmem_free(heap_arena, fbt_trampoline_window, fbt_trampoline_size);
380
381 fbt_trampoline_window = NULL;
382 fbt_trampoline = 0;
383 fbt_trampoline_size = 0;
384 }
385
386 static uintptr_t
fbt_patch_entry(uint32_t * instr,uint32_t id,fbt_trampoline_t * tramp,int nargs)387 fbt_patch_entry(uint32_t *instr, uint32_t id, fbt_trampoline_t *tramp,
388 int nargs)
389 {
390 uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
391 uint32_t first = *instr;
392 uintptr_t va = tramp->fbtt_va;
393 uintptr_t base = tramp->fbtt_next;
394
395 if (tramp->fbtt_next + FBT_ENTENT_MAXSIZE > tramp->fbtt_limit) {
396 /*
397 * There isn't sufficient room for this entry; return failure.
398 */
399 return (0);
400 }
401
402 FBT_COUNTER(id, fbt_entry);
403
404 if (FBT_IS_SAVE(first)) {
405 *tinstr++ = first;
406 } else {
407 *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
408 }
409
410 if (id > (uint32_t)FBT_SIMM13_MAX) {
411 *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
412 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
413 } else {
414 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
415 }
416
417 if (nargs >= 1)
418 *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O1);
419
420 if (nargs >= 2)
421 *tinstr++ = FBT_MOV(FBT_REG_I1, FBT_REG_O2);
422
423 if (nargs >= 3)
424 *tinstr++ = FBT_MOV(FBT_REG_I2, FBT_REG_O3);
425
426 if (nargs >= 4)
427 *tinstr++ = FBT_MOV(FBT_REG_I3, FBT_REG_O4);
428
429 if (nargs >= 5)
430 *tinstr++ = FBT_MOV(FBT_REG_I4, FBT_REG_O5);
431
432 if (FBT_IS_SAVE(first)) {
433 uintptr_t ret = (uintptr_t)instr - sizeof (uint32_t);
434
435 *tinstr++ = FBT_SETHI(ret, FBT_REG_G1);
436 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
437 tinstr++;
438 *tinstr++ = FBT_ORLO(FBT_REG_G1, ret, FBT_REG_O7);
439 } else {
440 uintptr_t slot = *--tinstr;
441 uintptr_t ret = (uintptr_t)instr + sizeof (uint32_t);
442 uint32_t delay = first;
443
444 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
445 tinstr++;
446 *tinstr++ = slot;
447 *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
448
449 if (FBT_IS_BA(first) || FBT_IS_BAPCC(first)) {
450 /*
451 * This is a special case: we are instrumenting a
452 * a non-annulled branch-always (or variant). We'll
453 * return directly to the destination of the branch,
454 * copying the instruction in the delay slot here,
455 * and then executing it in the slot of a ba.
456 */
457 if (FBT_IS_BA(first)) {
458 ret = FBT_BDEST(instr, *instr);
459 } else {
460 ret = FBT_BPCCDEST(instr, *instr);
461 }
462
463 delay = *(instr + 1);
464 }
465
466 if ((first & FBT_OP_MASK) != FBT_OP0 ||
467 (first & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
468 *tinstr = FBT_BA((uintptr_t)tinstr - base + va, ret);
469 tinstr++;
470 *tinstr++ = delay;
471 } else {
472 /*
473 * If this is a branch-on-register, we have a little
474 * more work to do: because the displacement is only
475 * sixteen bits, we're going to thunk the branch into
476 * the trampoline, and then ba,a to the appropriate
477 * destination in the branch targets. That is, we're
478 * constructing this sequence in the trampoline:
479 *
480 * br[cc] %[rs], 1f
481 * <delay-instruction>
482 * ba,a <not-taken-destination>
483 * 1: ba,a <taken-destination>
484 *
485 */
486 uintptr_t targ = FBT_BPRDEST(instr, first);
487
488 *tinstr = first & ~(FBT_DISP16_MASK);
489 *tinstr |= FBT_DISP14(tinstr, &tinstr[3]);
490 tinstr++;
491 *tinstr++ = *(instr + 1);
492 *tinstr = FBT_BAA((uintptr_t)tinstr - base + va,
493 ret + sizeof (uint32_t));
494 tinstr++;
495 *tinstr = FBT_BAA((uintptr_t)tinstr - base + va, targ);
496 tinstr++;
497 }
498 }
499
500 tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
501 tramp->fbtt_next = (uintptr_t)tinstr;
502
503 return (1);
504 }
505
506 /*
507 * We are patching control-transfer/restore couplets. There are three
508 * variants of couplet:
509 *
510 * (a) return rs1 + imm
511 * delay
512 *
513 * (b) jmpl rs1 + (rs2 | offset), rd
514 * restore rs1, rs2 | imm, rd
515 *
516 * (c) call displacement
517 * restore rs1, rs2 | imm, rd
518 *
519 * If rs1 in (a) is anything other than %i7, or imm is anything other than 8,
520 * or delay is a DCTI, we fail. If rd from the jmpl in (b) is something other
521 * than %g0 (a ret or a tail-call through a function pointer) or %o7 (a call
522 * through a register), we fail.
523 *
524 * Note that rs1 and rs2 in the restore instructions in (b) and (c) are
525 * potentially outputs and/or globals. Because these registers cannot be
526 * relied upon across the call to dtrace_probe(), we move rs1 into an unused
527 * local, ls0, and rs2 into an unused local, ls1, and restructure the restore
528 * to be:
529 *
530 * restore ls0, ls1, rd
531 *
532 * Likewise, rs1 and rs2 in the jmpl of case (b) may be outputs and/or globals.
533 * If the jmpl uses outputs or globals, we restructure it to be:
534 *
535 * jmpl ls2 + (ls3 | offset), (%g0 | %o7)
536 *
537 */
538 /*ARGSUSED*/
539 static int
fbt_canpatch_return(uint32_t * instr,int offset,const char * name)540 fbt_canpatch_return(uint32_t *instr, int offset, const char *name)
541 {
542 int rd;
543
544 if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
545 uint32_t delay = *(instr + 1);
546
547 if (*instr != FBT_RETURN(FBT_REG_I7, 8)) {
548 /*
549 * It's unclear if we should warn about this or not.
550 * We really wouldn't expect the compiler to generate
551 * return instructions with something other than %i7
552 * as rs1 and 8 as the simm13 -- it would just be
553 * mean-spirited. That said, such a construct isn't
554 * necessarily incorrect. Sill, we err on the side of
555 * caution and warn about it...
556 */
557 cmn_err(CE_NOTE, "cannot instrument return of %s at "
558 "%p: non-canonical return instruction", name,
559 (void *)instr);
560 return (0);
561 }
562
563 if (FBT_IS_CTI(delay)) {
564 /*
565 * This is even weirder -- a DCTI coupled with a
566 * return instruction. Similar constructs are used to
567 * return from utraps, but these typically have the
568 * return in the slot -- and we wouldn't expect to see
569 * it in the kernel regardless. At any rate, we don't
570 * want to try to instrument this construct, whatever
571 * it may be.
572 */
573 cmn_err(CE_NOTE, "cannot instrument return of %s at "
574 "%p: CTI in delay slot of return instruction",
575 name, (void *)instr);
576 return (0);
577 }
578
579 if (FBT_IS_PCRELATIVE(delay)) {
580 /*
581 * This is also very weird, but might be correct code
582 * if the function is (for example) returning the
583 * address of the delay instruction of the return as
584 * its return value (e.g. "rd %pc, %o0" in the slot).
585 * Perhaps correct, but still too weird to not warn
586 * about it...
587 */
588 cmn_err(CE_NOTE, "cannot instrument return of %s at "
589 "%p: PC-relative instruction in delay slot of "
590 "return instruction", name, (void *)instr);
591 return (0);
592 }
593
594 return (1);
595 }
596
597 if (FBT_FMT3_OP(*(instr + 1)) != FBT_OP_RESTORE)
598 return (0);
599
600 if (FBT_FMT1_OP(*instr) == FBT_OP_CALL)
601 return (1);
602
603 if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
604 return (0);
605
606 rd = FBT_FMT3_RD(*instr);
607
608 if (rd == FBT_REG_I7 || rd == FBT_REG_O7 || rd == FBT_REG_G0)
609 return (1);
610
611 /*
612 * We have encountered a jmpl that is storing the calling %pc in
613 * some register besides %i7, %o7 or %g0. This is strange; emit
614 * a warning and fail.
615 */
616 cmn_err(CE_NOTE, "cannot instrument return of %s at %p: unexpected "
617 "jmpl destination register", name, (void *)instr);
618 return (0);
619 }
620
621 static int
fbt_canpatch_retl(uint32_t * instr,int offset,const char * name)622 fbt_canpatch_retl(uint32_t *instr, int offset, const char *name)
623 {
624 if (FBT_FMT1_OP(*instr) == FBT_OP_CALL ||
625 (FBT_FMT3_OP(*instr) == FBT_OP_JMPL &&
626 FBT_FMT3_RD(*instr) == FBT_REG_O7)) {
627 /*
628 * If this is a call (or a jmpl that links into %o7), we can
629 * patch it iff the next instruction uses %o7 as a destination
630 * register. Because there is an ABI responsibility to
631 * restore %o7 to the value before the call/jmpl, we don't
632 * particularly care how this routine is managing to restore
633 * it (mov, add, ld or divx for all we care). If it doesn't
634 * seem to be restoring it at all, however, we'll refuse
635 * to patch it.
636 */
637 uint32_t delay = *(instr + 1);
638 uint32_t op, rd;
639
640 op = FBT_FMT1_OP(delay);
641 rd = FBT_FMT3_RD(delay);
642
643 if (op != FBT_OP2 || rd != FBT_REG_O7) {
644 /*
645 * This is odd. Before we assume that we're looking
646 * at something bizarre (and warn accordingly), we'll
647 * check to see if it's obviously a jump table entry.
648 */
649 if (*instr < (uintptr_t)instr &&
650 *instr >= (uintptr_t)instr - offset)
651 return (0);
652
653 cmn_err(CE_NOTE, "cannot instrument return of %s at "
654 "%p: leaf jmpl/call delay isn't restoring %%o7",
655 name, (void *)instr);
656 return (0);
657 }
658
659 return (1);
660 }
661
662 if (offset == sizeof (uint32_t)) {
663 /*
664 * If this is the second instruction in the function, we're
665 * going to allow it to be patched if the first instruction
666 * is a patchable return-from-leaf instruction.
667 */
668 if (fbt_canpatch_retl(instr - 1, 0, name))
669 return (1);
670 }
671
672 if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
673 return (0);
674
675 if (FBT_FMT3_RD(*instr) != FBT_REG_G0)
676 return (0);
677
678 return (1);
679 }
680
681 /*ARGSUSED*/
682 static uint32_t
fbt_patch_return(uint32_t * instr,uint32_t * funcbase,uint32_t * funclim,int offset,uint32_t id,fbt_trampoline_t * tramp,const char * name)683 fbt_patch_return(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
684 int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
685 {
686 uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
687 uint32_t cti = *instr, restore = *(instr + 1), rs1, dest;
688 uintptr_t va = tramp->fbtt_va;
689 uintptr_t base = tramp->fbtt_next;
690 uint32_t locals[FBT_REG_NLOCALS], local;
691
692 if (tramp->fbtt_next + FBT_RETENT_MAXSIZE > tramp->fbtt_limit) {
693 /*
694 * There isn't sufficient room for this entry; return failure.
695 */
696 return (FBT_ILLTRAP);
697 }
698
699 FBT_COUNTER(id, fbt_ret);
700
701 if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
702 /*
703 * To handle the case of the return instruction, we'll emit a
704 * restore, followed by the instruction in the slot (which
705 * we'll transplant here), and then another save. While it
706 * may seem intellectually unsatisfying to emit the additional
707 * restore/save couplet, one can take solace in the fact that
708 * we don't do this if the instruction in the return delay
709 * slot is a nop -- which it is nearly 90% of the time with
710 * gcc. (And besides, this couplet can't induce unnecessary
711 * spill/fill traps; rewriting the delay instruction to be
712 * in terms of the current window hardly seems worth the
713 * trouble -- let alone the risk.)
714 */
715 uint32_t delay = *(instr + 1);
716 ASSERT(*instr == FBT_RETURN(FBT_REG_I7, 8));
717
718 cti = FBT_RET;
719 restore = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
720
721 if (delay != FBT_SETHI(0, FBT_REG_G0)) {
722 *tinstr++ = restore;
723 *tinstr++ = delay;
724 *tinstr++ = FBT_SAVEIMM(FBT_REG_O6,
725 -SA(MINFRAME), FBT_REG_O6);
726 }
727 }
728
729 FBT_REG_INITLOCALS(local, locals);
730
731 /*
732 * Mark the locals used in the jmpl.
733 */
734 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
735 uint32_t rs1 = FBT_FMT3_RS1(cti);
736 FBT_REG_MARKLOCAL(locals, rs1);
737
738 if (!FBT_FMT3_ISIMM(cti)) {
739 uint32_t rs2 = FBT_FMT3_RS2(cti);
740 FBT_REG_MARKLOCAL(locals, rs2);
741 }
742 }
743
744 /*
745 * And mark the locals used in the restore.
746 */
747 rs1 = FBT_FMT3_RS1(restore);
748 FBT_REG_MARKLOCAL(locals, rs1);
749
750 if (!FBT_FMT3_ISIMM(restore)) {
751 uint32_t rs2 = FBT_FMT3_RS2(restore);
752 FBT_REG_MARKLOCAL(locals, rs2);
753 }
754
755 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
756 uint32_t rs1 = FBT_FMT3_RS1(cti);
757
758 if (FBT_REG_ISVOLATILE(rs1)) {
759 FBT_REG_ALLOCLOCAL(local, locals);
760 FBT_FMT3_RS1_SET(cti, local);
761 *tinstr++ = FBT_MOV(rs1, local);
762 }
763
764 if (!FBT_FMT3_ISIMM(cti)) {
765 uint32_t rs2 = FBT_FMT3_RS2(cti);
766
767 if (FBT_REG_ISVOLATILE(rs2)) {
768 FBT_REG_ALLOCLOCAL(local, locals);
769 FBT_FMT3_RS2_SET(cti, local);
770 *tinstr++ = FBT_MOV(rs2, local);
771 }
772 }
773 }
774
775 rs1 = FBT_FMT3_RS1(restore);
776
777 if (FBT_REG_ISVOLATILE(rs1)) {
778 FBT_REG_ALLOCLOCAL(local, locals);
779 FBT_FMT3_RS1_SET(restore, local);
780 *tinstr++ = FBT_MOV(rs1, local);
781 }
782
783 if (!FBT_FMT3_ISIMM(restore)) {
784 uint32_t rs2 = FBT_FMT3_RS2(restore);
785
786 if (FBT_REG_ISVOLATILE(rs2)) {
787 FBT_REG_ALLOCLOCAL(local, locals);
788 FBT_FMT3_RS2_SET(restore, local);
789 *tinstr++ = FBT_MOV(rs2, local);
790 }
791 }
792
793 if (id > (uint32_t)FBT_SIMM13_MAX) {
794 *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
795 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
796 } else {
797 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
798 }
799
800 if (offset > (uint32_t)FBT_SIMM13_MAX) {
801 *tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
802 *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
803 } else {
804 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
805 }
806
807 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
808 tinstr++;
809
810 if (FBT_FMT3_RD(restore) == FBT_REG_O0) {
811 /*
812 * If the destination register of the restore is %o0, we
813 * need to perform the implied calculation to derive the
814 * return value.
815 */
816 uint32_t add = (restore & ~FBT_FMT3_OP_MASK) | FBT_OP_ADD;
817 add &= ~FBT_FMT3_RD_MASK;
818 *tinstr++ = add | (FBT_REG_O2 << FBT_FMT3_RD_SHIFT);
819 } else {
820 *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
821 }
822
823 /*
824 * If the control transfer instruction is %pc-relative (i.e. a
825 * call), we need to reset it appropriately.
826 */
827 if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
828 dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
829 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
830 tinstr++;
831 } else {
832 *tinstr++ = cti;
833 }
834
835 *tinstr++ = restore;
836 tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
837 tramp->fbtt_next = (uintptr_t)tinstr;
838
839 return (FBT_BAA(instr, va));
840 }
841
842 static uint32_t
fbt_patch_retl(uint32_t * instr,uint32_t * funcbase,uint32_t * funclim,int offset,uint32_t id,fbt_trampoline_t * tramp,const char * name)843 fbt_patch_retl(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
844 int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
845 {
846 uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
847 uintptr_t va = tramp->fbtt_va;
848 uintptr_t base = tramp->fbtt_next;
849 uint32_t cti = *instr, dest;
850 int annul = 0;
851
852 FBT_COUNTER(id, fbt_retl);
853
854 if (tramp->fbtt_next + FBT_RETLENT_MAXSIZE > tramp->fbtt_limit) {
855 /*
856 * There isn't sufficient room for this entry; return failure.
857 */
858 return (FBT_ILLTRAP);
859 }
860
861 if (offset == sizeof (uint32_t) &&
862 fbt_canpatch_retl(instr - 1, 0, name)) {
863 *tinstr++ = *instr;
864 annul = 1;
865 FBT_COUNTER(id, fbt_retl_twoinstr);
866 } else {
867 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL &&
868 FBT_FMT3_RD(cti) != FBT_REG_O7 &&
869 FBT_FMT3_RS1(cti) != FBT_REG_O7) {
870 annul = 1;
871 *tinstr++ = *(instr + 1);
872 }
873 }
874
875 *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
876
877 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
878 uint32_t rs1, rs2, o2i = FBT_REG_I0 - FBT_REG_O0;
879
880 /*
881 * If we have a jmpl and it's in terms of output registers, we
882 * need to rewrite it to be in terms of the corresponding input
883 * registers. If it's in terms of the globals, we'll rewrite
884 * it to be in terms of locals.
885 */
886 rs1 = FBT_FMT3_RS1(cti);
887
888 if (FBT_REG_ISOUTPUT(rs1))
889 rs1 += o2i;
890
891 if (FBT_REG_ISGLOBAL(rs1)) {
892 *tinstr++ = FBT_MOV(rs1, FBT_REG_L0);
893 rs1 = FBT_REG_L0;
894 }
895
896 FBT_FMT3_RS1_SET(cti, rs1);
897
898 if (!FBT_FMT3_ISIMM(cti)) {
899 rs2 = FBT_FMT3_RS2(cti);
900
901 if (FBT_REG_ISOUTPUT(rs2))
902 rs2 += o2i;
903
904 if (FBT_REG_ISGLOBAL(rs2)) {
905 *tinstr++ = FBT_MOV(rs2, FBT_REG_L1);
906 rs2 = FBT_REG_L1;
907 }
908
909 FBT_FMT3_RS2_SET(cti, rs2);
910 }
911
912 /*
913 * Now we need to check the rd and source register for the jmpl;
914 * If neither rd nor the source register is %o7, then we might
915 * have a jmp that is actually part of a jump table. We need
916 * to generate the code to compare it to the base and limit of
917 * the function.
918 */
919 if (FBT_FMT3_RD(cti) != FBT_REG_O7 && rs1 != FBT_REG_I7) {
920 uintptr_t base = (uintptr_t)funcbase;
921 uintptr_t limit = (uintptr_t)funclim;
922
923 FBT_COUNTER(id, fbt_retl_jmptab);
924
925 if (FBT_FMT3_ISIMM(cti)) {
926 *tinstr++ = FBT_ADDSIMM13(rs1,
927 FBT_FMT3_SIMM13(cti), FBT_REG_L2);
928 } else {
929 *tinstr++ = FBT_ADD(rs1, rs2, FBT_REG_L2);
930 }
931
932 *tinstr++ = FBT_SETHI(base, FBT_REG_L3);
933 *tinstr++ = FBT_ORLO(FBT_REG_L3, base, FBT_REG_L3);
934 *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
935 *tinstr++ = FBT_BL(0, 8 * sizeof (uint32_t));
936 *tinstr++ = FBT_SETHI(limit, FBT_REG_L3);
937 *tinstr++ = FBT_ORLO(FBT_REG_L3, limit, FBT_REG_L3);
938 *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
939 *tinstr++ = FBT_BGE(0, 4 * sizeof (uint32_t));
940 *tinstr++ = FBT_SETHI(0, FBT_REG_G0);
941 *tinstr++ = cti;
942 *tinstr++ = FBT_RESTORE(FBT_REG_G0,
943 FBT_REG_G0, FBT_REG_G0);
944 }
945 }
946
947 if (id > (uint32_t)FBT_SIMM13_MAX) {
948 *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
949 *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
950 } else {
951 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
952 }
953
954 if (offset > (uint32_t)FBT_SIMM13_MAX) {
955 *tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
956 *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
957 } else {
958 *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
959 }
960
961 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
962 tinstr++;
963 *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
964
965 /*
966 * If the control transfer instruction is %pc-relative (i.e. a
967 * call), we need to reset it appropriately.
968 */
969 if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
970 FBT_COUNTER(id, fbt_retl_tailcall);
971 dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
972 *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
973 tinstr++;
974 annul = 1;
975 } else {
976 if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
977 *tinstr++ = cti;
978
979 if (FBT_FMT3_RD(cti) == FBT_REG_O7) {
980 FBT_COUNTER(id, fbt_retl_tailjmpl);
981 annul = 1;
982 }
983 } else {
984 *tinstr++ = FBT_RET;
985 }
986 }
987
988 *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);
989
990 tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
991 tramp->fbtt_next = (uintptr_t)tinstr;
992
993 return (annul ? FBT_BAA(instr, va) : FBT_BA(instr, va));
994 }
995
996 /*ARGSUSED*/
997 static void
fbt_provide_module(void * arg,struct modctl * ctl)998 fbt_provide_module(void *arg, struct modctl *ctl)
999 {
1000 struct module *mp = ctl->mod_mp;
1001 char *modname = ctl->mod_modname;
1002 char *str = mp->strings;
1003 int nsyms = mp->nsyms;
1004 Shdr *symhdr = mp->symhdr;
1005 size_t symsize;
1006 char *name;
1007 int i;
1008 fbt_probe_t *fbt, *retfbt;
1009 fbt_trampoline_t tramp;
1010 uintptr_t offset;
1011 int primary = 0;
1012 ctf_file_t *fp = NULL;
1013 int error;
1014 int estimate = 1;
1015 uint32_t faketramp[50];
1016 size_t fbt_size = 0;
1017
1018 /*
1019 * Employees of dtrace and their families are ineligible. Void
1020 * where prohibited.
1021 */
1022 if (strcmp(modname, "dtrace") == 0)
1023 return;
1024
1025 if (ctl->mod_requisites != NULL) {
1026 struct modctl_list *list;
1027
1028 list = (struct modctl_list *)ctl->mod_requisites;
1029
1030 for (; list != NULL; list = list->modl_next) {
1031 if (strcmp(list->modl_modp->mod_modname, "dtrace") == 0)
1032 return;
1033 }
1034 }
1035
1036 /*
1037 * KMDB is ineligible for instrumentation -- it may execute in
1038 * any context, including probe context.
1039 */
1040 if (strcmp(modname, "kmdbmod") == 0)
1041 return;
1042
1043 if (str == NULL || symhdr == NULL || symhdr->sh_addr == 0) {
1044 /*
1045 * If this module doesn't (yet) have its string or symbol
1046 * table allocated, clear out.
1047 */
1048 return;
1049 }
1050
1051 symsize = symhdr->sh_entsize;
1052
1053 if (mp->fbt_nentries) {
1054 /*
1055 * This module has some FBT entries allocated; we're afraid
1056 * to screw with it.
1057 */
1058 return;
1059 }
1060
1061 if (mp->fbt_tab != NULL)
1062 estimate = 0;
1063
1064 /*
1065 * This is a hack for unix/genunix/krtld.
1066 */
1067 primary = vmem_contains(heap_arena, (void *)ctl,
1068 sizeof (struct modctl)) == 0;
1069 kobj_textwin_alloc(mp);
1070
1071 /*
1072 * Open the CTF data for the module. We'll use this to determine the
1073 * functions that can be instrumented. Note that this call can fail,
1074 * in which case we'll use heuristics to determine the functions that
1075 * can be instrumented. (But in particular, leaf functions will not be
1076 * instrumented.)
1077 */
1078 fp = ctf_modopen(mp, &error);
1079
1080 forreal:
1081 if (!estimate) {
1082 tramp.fbtt_next =
1083 (uintptr_t)fbt_trampoline_map((uintptr_t)mp->fbt_tab,
1084 mp->fbt_size);
1085 tramp.fbtt_limit = tramp.fbtt_next + mp->fbt_size;
1086 tramp.fbtt_va = (uintptr_t)mp->fbt_tab;
1087 }
1088
1089 for (i = 1; i < nsyms; i++) {
1090 ctf_funcinfo_t f;
1091 uint32_t *instr, *base, *limit;
1092 Sym *sym = (Sym *)(symhdr->sh_addr + i * symsize);
1093 int have_ctf = 0, is_leaf = 0, nargs, cti = 0;
1094 int (*canpatch)(uint32_t *, int, const char *);
1095 uint32_t (*patch)(uint32_t *, uint32_t *, uint32_t *, int,
1096 uint32_t, fbt_trampoline_t *, const char *);
1097
1098 if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
1099 continue;
1100
1101 /*
1102 * Weak symbols are not candidates. This could be made to
1103 * work (where weak functions and their underlying function
1104 * appear as two disjoint probes), but it's not simple.
1105 */
1106 if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
1107 continue;
1108
1109 name = str + sym->st_name;
1110
1111 if (strstr(name, "dtrace_") == name &&
1112 strstr(name, "dtrace_safe_") != name) {
1113 /*
1114 * Anything beginning with "dtrace_" may be called
1115 * from probe context unless it explitly indicates
1116 * that it won't be called from probe context by
1117 * using the prefix "dtrace_safe_".
1118 */
1119 continue;
1120 }
1121
1122 if (strstr(name, "kdi_") == name ||
1123 strstr(name, "_kdi_") != NULL) {
1124 /*
1125 * Any function name beginning with "kdi_" or
1126 * containing the string "_kdi_" is a part of the
1127 * kernel debugger interface and may be called in
1128 * arbitrary context -- including probe context.
1129 */
1130 continue;
1131 }
1132
1133 if (strstr(name, "__relocatable") != NULL) {
1134 /*
1135 * Anything with the string "__relocatable" anywhere
1136 * in the function name is considered to be a function
1137 * that may be manually relocated before execution.
1138 * Because FBT uses a PC-relative technique for
1139 * instrumentation, these functions cannot safely
1140 * be instrumented by us.
1141 */
1142 continue;
1143 }
1144
1145 if (strstr(name, "ip_ocsum") == name) {
1146 /*
1147 * The ip_ocsum_* family of routines are all ABI
1148 * violators. (They expect incoming arguments in the
1149 * globals!) Break the ABI? No soup for you!
1150 */
1151 continue;
1152 }
1153
1154 /*
1155 * We want to scan the function for one (and only one) save.
1156 * Any more indicates that something fancy is going on.
1157 */
1158 base = (uint32_t *)sym->st_value;
1159 limit = (uint32_t *)(sym->st_value + sym->st_size);
1160
1161 /*
1162 * We don't want to interpose on the module stubs.
1163 */
1164 if (base >= (uint32_t *)stubs_base &&
1165 base <= (uint32_t *)stubs_end)
1166 continue;
1167
1168 /*
1169 * We can't safely trace a zero-length function...
1170 */
1171 if (base == limit)
1172 continue;
1173
1174 /*
1175 * Due to 4524008, _init and _fini may have a bloated st_size.
1176 * While this bug was fixed quite some time ago, old drivers
1177 * may be lurking. We need to develop a better solution to
1178 * this problem, such that correct _init and _fini functions
1179 * (the vast majority) may be correctly traced. One solution
1180 * may be to scan through the entire symbol table to see if
1181 * any symbol overlaps with _init. If none does, set a bit in
1182 * the module structure that this module has correct _init and
1183 * _fini sizes. This will cause some pain the first time a
1184 * module is scanned, but at least it would be O(N) instead of
1185 * O(N log N)...
1186 */
1187 if (strcmp(name, "_init") == 0)
1188 continue;
1189
1190 if (strcmp(name, "_fini") == 0)
1191 continue;
1192
1193 instr = base;
1194
1195 /*
1196 * While we try hard to only trace safe functions (that is,
1197 * functions at TL=0), one unsafe function manages to otherwise
1198 * appear safe: prom_trap(). We could discover prom_trap()
1199 * if we added an additional rule: in order to trace a
1200 * function, we must either (a) discover a restore or (b)
1201 * determine that the function does not have any unlinked
1202 * control transfers to another function (i.e., the function
1203 * never returns). Unfortunately, as of this writing, one
1204 * legitimate function (resume_from_zombie()) transfers
1205 * control to a different function (_resume_from_idle())
1206 * without executing a restore. Barring a rule to figure out
1207 * that resume_from_zombie() is safe while prom_trap() is not,
1208 * we resort to hard-coding prom_trap() here.
1209 */
1210 if (strcmp(name, "prom_trap") == 0)
1211 continue;
1212
1213 if (fp != NULL && ctf_func_info(fp, i, &f) != CTF_ERR) {
1214 nargs = f.ctc_argc;
1215 have_ctf = 1;
1216 } else {
1217 nargs = 32;
1218 }
1219
1220 /*
1221 * If the first instruction of the function is a branch and
1222 * it's not a branch-always-not-annulled, we're going to refuse
1223 * to patch it.
1224 */
1225 if ((*instr & FBT_OP_MASK) == FBT_OP0 &&
1226 (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI &&
1227 (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
1228 if (!FBT_IS_BA(*instr) && !FBT_IS_BAPCC(*instr)) {
1229 if (have_ctf) {
1230 cmn_err(CE_NOTE, "cannot instrument %s:"
1231 " begins with non-ba, "
1232 "non-br CTI", name);
1233 }
1234 continue;
1235 }
1236 }
1237
1238 while (!FBT_IS_SAVE(*instr)) {
1239 /*
1240 * Before we assume that this is a leaf routine, check
1241 * forward in the basic block for a save.
1242 */
1243 int op = *instr & FBT_OP_MASK;
1244 int op2 = *instr & FBT_FMT2_OP2_MASK;
1245
1246 if (op == FBT_OP0 && op2 != FBT_FMT2_OP2_SETHI) {
1247 /*
1248 * This is a CTI. If we see a subsequent
1249 * save, we will refuse to process this
1250 * routine unless both of the following are
1251 * true:
1252 *
1253 * (a) The branch is not annulled
1254 *
1255 * (b) The subsequent save is in the delay
1256 * slot of the branch
1257 */
1258 if ((*instr & FBT_ANNUL) ||
1259 !FBT_IS_SAVE(*(instr + 1))) {
1260 cti = 1;
1261 } else {
1262 instr++;
1263 break;
1264 }
1265 }
1266
1267 if (op == FBT_OP1)
1268 cti = 1;
1269
1270 if (++instr == limit)
1271 break;
1272 }
1273
1274 if (instr < limit && cti) {
1275 /*
1276 * If we found a CTI before the save, we need to not
1277 * do anything. But if we have CTF information, this
1278 * is weird enough that it merits a message.
1279 */
1280 if (!have_ctf)
1281 continue;
1282
1283 cmn_err(CE_NOTE, "cannot instrument %s: "
1284 "save not in first basic block", name);
1285 continue;
1286 }
1287
1288 if (instr == limit) {
1289 if (!have_ctf)
1290 continue;
1291 is_leaf = 1;
1292
1293 if (!estimate)
1294 fbt_leaf_functions++;
1295
1296 canpatch = fbt_canpatch_retl;
1297 patch = fbt_patch_retl;
1298 } else {
1299 canpatch = fbt_canpatch_return;
1300 patch = fbt_patch_return;
1301 }
1302
1303 if (!have_ctf && !is_leaf) {
1304 /*
1305 * Before we assume that this isn't something tricky,
1306 * look for other saves. If we find them, there are
1307 * multiple entry points here (or something), and we'll
1308 * leave it alone.
1309 */
1310 while (++instr < limit) {
1311 if (FBT_IS_SAVE(*instr))
1312 break;
1313 }
1314
1315 if (instr != limit)
1316 continue;
1317 }
1318
1319 instr = base;
1320
1321 if (FBT_IS_CTI(*instr)) {
1322 /*
1323 * If we have a CTI, we want to be sure that we don't
1324 * have a CTI or a PC-relative instruction in the
1325 * delay slot -- we want to be able to thunk the
1326 * instruction into the trampoline without worrying
1327 * about either DCTIs or relocations. It would be
1328 * very odd for the compiler to generate this kind of
1329 * code, so we warn about it if we have CTF
1330 * information.
1331 */
1332 if (FBT_IS_CTI(*(instr + 1))) {
1333 if (!have_ctf)
1334 continue;
1335
1336 cmn_err(CE_NOTE, "cannot instrument %s: "
1337 "CTI in delay slot of first instruction",
1338 name);
1339 continue;
1340 }
1341
1342 if (FBT_IS_PCRELATIVE(*(instr + 1))) {
1343 if (!have_ctf)
1344 continue;
1345
1346 cmn_err(CE_NOTE, "cannot instrument %s: "
1347 "PC-relative instruction in delay slot of"
1348 " first instruction", name);
1349 continue;
1350 }
1351 }
1352
1353 if (estimate) {
1354 tramp.fbtt_next = (uintptr_t)faketramp;
1355 tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
1356 (void) fbt_patch_entry(instr, FBT_ESTIMATE_ID,
1357 &tramp, nargs);
1358 fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;
1359 } else {
1360 fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
1361 fbt->fbtp_name = name;
1362 fbt->fbtp_ctl = ctl;
1363 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
1364 name, FBT_PROBENAME_ENTRY, 1, fbt);
1365 fbt->fbtp_patchval = FBT_BAA(instr, tramp.fbtt_va);
1366
1367 if (!fbt_patch_entry(instr, fbt->fbtp_id,
1368 &tramp, nargs)) {
1369 cmn_err(CE_WARN, "unexpectedly short FBT table "
1370 "in module %s (sym %d of %d)", modname,
1371 i, nsyms);
1372 break;
1373 }
1374
1375 fbt->fbtp_patchpoint =
1376 (uint32_t *)((uintptr_t)mp->textwin +
1377 ((uintptr_t)instr - (uintptr_t)mp->text));
1378 fbt->fbtp_savedval = *instr;
1379
1380 fbt->fbtp_loadcnt = ctl->mod_loadcnt;
1381 fbt->fbtp_primary = primary;
1382 fbt->fbtp_symndx = i;
1383 mp->fbt_nentries++;
1384 }
1385
1386 retfbt = NULL;
1387 again:
1388 if (++instr == limit)
1389 continue;
1390
1391 offset = (uintptr_t)instr - (uintptr_t)base;
1392
1393 if (!(*canpatch)(instr, offset, name))
1394 goto again;
1395
1396 if (estimate) {
1397 tramp.fbtt_next = (uintptr_t)faketramp;
1398 tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
1399 (void) (*patch)(instr, base, limit,
1400 offset, FBT_ESTIMATE_ID, &tramp, name);
1401 fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;
1402
1403 goto again;
1404 }
1405
1406 fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
1407 fbt->fbtp_name = name;
1408 fbt->fbtp_ctl = ctl;
1409
1410 if (retfbt == NULL) {
1411 fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
1412 name, FBT_PROBENAME_RETURN, 1, fbt);
1413 } else {
1414 retfbt->fbtp_next = fbt;
1415 fbt->fbtp_id = retfbt->fbtp_id;
1416 }
1417
1418 fbt->fbtp_return = 1;
1419 retfbt = fbt;
1420
1421 if ((fbt->fbtp_patchval = (*patch)(instr, base, limit, offset,
1422 fbt->fbtp_id, &tramp, name)) == FBT_ILLTRAP) {
1423 cmn_err(CE_WARN, "unexpectedly short FBT table "
1424 "in module %s (sym %d of %d)", modname, i, nsyms);
1425 break;
1426 }
1427
1428 fbt->fbtp_patchpoint = (uint32_t *)((uintptr_t)mp->textwin +
1429 ((uintptr_t)instr - (uintptr_t)mp->text));
1430 fbt->fbtp_savedval = *instr;
1431 fbt->fbtp_loadcnt = ctl->mod_loadcnt;
1432 fbt->fbtp_primary = primary;
1433 fbt->fbtp_symndx = i;
1434 mp->fbt_nentries++;
1435
1436 goto again;
1437 }
1438
1439 if (estimate) {
1440 /*
1441 * Slosh on another entry's worth...
1442 */
1443 fbt_size += FBT_ENT_MAXSIZE;
1444 mp->fbt_size = fbt_size;
1445 mp->fbt_tab = kobj_texthole_alloc(mp->text, fbt_size);
1446
1447 if (mp->fbt_tab == NULL) {
1448 cmn_err(CE_WARN, "couldn't allocate FBT table "
1449 "for module %s", modname);
1450 } else {
1451 estimate = 0;
1452 goto forreal;
1453 }
1454 } else {
1455 fbt_trampoline_unmap();
1456 }
1457
1458 error:
1459 if (fp != NULL)
1460 ctf_close(fp);
1461 }
1462
1463 /*ARGSUSED*/
1464 static void
fbt_destroy(void * arg,dtrace_id_t id,void * parg)1465 fbt_destroy(void *arg, dtrace_id_t id, void *parg)
1466 {
1467 fbt_probe_t *fbt = parg, *next;
1468 struct modctl *ctl = fbt->fbtp_ctl;
1469
1470 do {
1471 if (ctl != NULL && ctl->mod_loadcnt == fbt->fbtp_loadcnt) {
1472 if ((ctl->mod_loadcnt == fbt->fbtp_loadcnt &&
1473 ctl->mod_loaded) || fbt->fbtp_primary) {
1474 ((struct module *)
1475 (ctl->mod_mp))->fbt_nentries--;
1476 }
1477 }
1478
1479 next = fbt->fbtp_next;
1480 kmem_free(fbt, sizeof (fbt_probe_t));
1481 fbt = next;
1482 } while (fbt != NULL);
1483 }
1484
1485 /*ARGSUSED*/
1486 static int
fbt_enable(void * arg,dtrace_id_t id,void * parg)1487 fbt_enable(void *arg, dtrace_id_t id, void *parg)
1488 {
1489 fbt_probe_t *fbt = parg, *f;
1490 struct modctl *ctl = fbt->fbtp_ctl;
1491
1492 ctl->mod_nenabled++;
1493
1494 for (f = fbt; f != NULL; f = f->fbtp_next) {
1495 if (f->fbtp_patchpoint == NULL) {
1496 /*
1497 * Due to a shortened FBT table, this entry was never
1498 * completed; refuse to enable it.
1499 */
1500 if (fbt_verbose) {
1501 cmn_err(CE_NOTE, "fbt is failing for probe %s "
1502 "(short FBT table in %s)",
1503 fbt->fbtp_name, ctl->mod_modname);
1504 }
1505
1506 return (0);
1507 }
1508 }
1509
1510 /*
1511 * If this module has disappeared since we discovered its probes,
1512 * refuse to enable it.
1513 */
1514 if (!fbt->fbtp_primary && !ctl->mod_loaded) {
1515 if (fbt_verbose) {
1516 cmn_err(CE_NOTE, "fbt is failing for probe %s "
1517 "(module %s unloaded)",
1518 fbt->fbtp_name, ctl->mod_modname);
1519 }
1520
1521 return (0);
1522 }
1523
1524 /*
1525 * Now check that our modctl has the expected load count. If it
1526 * doesn't, this module must have been unloaded and reloaded -- and
1527 * we're not going to touch it.
1528 */
1529 if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) {
1530 if (fbt_verbose) {
1531 cmn_err(CE_NOTE, "fbt is failing for probe %s "
1532 "(module %s reloaded)",
1533 fbt->fbtp_name, ctl->mod_modname);
1534 }
1535
1536 return (0);
1537 }
1538
1539 for (; fbt != NULL; fbt = fbt->fbtp_next)
1540 *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1541
1542 return (0);
1543 }
1544
1545 /*ARGSUSED*/
1546 static void
fbt_disable(void * arg,dtrace_id_t id,void * parg)1547 fbt_disable(void *arg, dtrace_id_t id, void *parg)
1548 {
1549 fbt_probe_t *fbt = parg, *f;
1550 struct modctl *ctl = fbt->fbtp_ctl;
1551
1552 ASSERT(ctl->mod_nenabled > 0);
1553 ctl->mod_nenabled--;
1554
1555 for (f = fbt; f != NULL; f = f->fbtp_next) {
1556 if (f->fbtp_patchpoint == NULL)
1557 return;
1558 }
1559
1560 if ((!fbt->fbtp_primary && !ctl->mod_loaded) ||
1561 (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
1562 return;
1563
1564 for (; fbt != NULL; fbt = fbt->fbtp_next)
1565 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1566 }
1567
1568 /*ARGSUSED*/
1569 static void
fbt_suspend(void * arg,dtrace_id_t id,void * parg)1570 fbt_suspend(void *arg, dtrace_id_t id, void *parg)
1571 {
1572 fbt_probe_t *fbt = parg;
1573 struct modctl *ctl = fbt->fbtp_ctl;
1574
1575 if (!fbt->fbtp_primary && !ctl->mod_loaded)
1576 return;
1577
1578 if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
1579 return;
1580
1581 ASSERT(ctl->mod_nenabled > 0);
1582
1583 for (; fbt != NULL; fbt = fbt->fbtp_next)
1584 *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
1585 }
1586
1587 /*ARGSUSED*/
1588 static void
fbt_resume(void * arg,dtrace_id_t id,void * parg)1589 fbt_resume(void *arg, dtrace_id_t id, void *parg)
1590 {
1591 fbt_probe_t *fbt = parg;
1592 struct modctl *ctl = fbt->fbtp_ctl;
1593
1594 if (!fbt->fbtp_primary && !ctl->mod_loaded)
1595 return;
1596
1597 if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
1598 return;
1599
1600 ASSERT(ctl->mod_nenabled > 0);
1601
1602 for (; fbt != NULL; fbt = fbt->fbtp_next)
1603 *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
1604 }
1605
1606 /*ARGSUSED*/
1607 static void
fbt_getargdesc(void * arg,dtrace_id_t id,void * parg,dtrace_argdesc_t * desc)1608 fbt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
1609 {
1610 fbt_probe_t *fbt = parg;
1611 struct modctl *ctl = fbt->fbtp_ctl;
1612 struct module *mp = ctl->mod_mp;
1613 ctf_file_t *fp = NULL, *pfp;
1614 ctf_funcinfo_t f;
1615 int error;
1616 ctf_id_t argv[32], type;
1617 int argc = sizeof (argv) / sizeof (ctf_id_t);
1618 const char *parent;
1619
1620 if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
1621 goto err;
1622
1623 if (fbt->fbtp_return && desc->dtargd_ndx == 0) {
1624 (void) strcpy(desc->dtargd_native, "int");
1625 return;
1626 }
1627
1628 if ((fp = ctf_modopen(mp, &error)) == NULL) {
1629 /*
1630 * We have no CTF information for this module -- and therefore
1631 * no args[] information.
1632 */
1633 goto err;
1634 }
1635
1636 /*
1637 * If we have a parent container, we must manually import it.
1638 */
1639 if ((parent = ctf_parent_name(fp)) != NULL) {
1640 struct modctl *mp = &modules;
1641 struct modctl *mod = NULL;
1642
1643 /*
1644 * We must iterate over all modules to find the module that
1645 * is our parent.
1646 */
1647 do {
1648 if (strcmp(mp->mod_modname, parent) == 0) {
1649 mod = mp;
1650 break;
1651 }
1652 } while ((mp = mp->mod_next) != &modules);
1653
1654 if (mod == NULL)
1655 goto err;
1656
1657 if ((pfp = ctf_modopen(mod->mod_mp, &error)) == NULL)
1658 goto err;
1659
1660 if (ctf_import(fp, pfp) != 0) {
1661 ctf_close(pfp);
1662 goto err;
1663 }
1664
1665 ctf_close(pfp);
1666 }
1667
1668 if (ctf_func_info(fp, fbt->fbtp_symndx, &f) == CTF_ERR)
1669 goto err;
1670
1671 if (fbt->fbtp_return) {
1672 if (desc->dtargd_ndx > 1)
1673 goto err;
1674
1675 ASSERT(desc->dtargd_ndx == 1);
1676 type = f.ctc_return;
1677 } else {
1678 if (desc->dtargd_ndx + 1 > f.ctc_argc)
1679 goto err;
1680
1681 if (ctf_func_args(fp, fbt->fbtp_symndx, argc, argv) == CTF_ERR)
1682 goto err;
1683
1684 type = argv[desc->dtargd_ndx];
1685 }
1686
1687 if (ctf_type_name(fp, type, desc->dtargd_native,
1688 DTRACE_ARGTYPELEN) != NULL) {
1689 ctf_close(fp);
1690 return;
1691 }
1692 err:
1693 if (fp != NULL)
1694 ctf_close(fp);
1695
1696 desc->dtargd_ndx = DTRACE_ARGNONE;
1697 }
1698
1699 static dtrace_pattr_t fbt_attr = {
1700 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1701 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1702 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
1703 { DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
1704 { DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
1705 };
1706
1707 static dtrace_pops_t fbt_pops = {
1708 NULL,
1709 fbt_provide_module,
1710 fbt_enable,
1711 fbt_disable,
1712 fbt_suspend,
1713 fbt_resume,
1714 fbt_getargdesc,
1715 NULL,
1716 NULL,
1717 fbt_destroy
1718 };
1719
1720 static int
fbt_attach(dev_info_t * devi,ddi_attach_cmd_t cmd)1721 fbt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
1722 {
1723 switch (cmd) {
1724 case DDI_ATTACH:
1725 break;
1726 case DDI_RESUME:
1727 return (DDI_SUCCESS);
1728 default:
1729 return (DDI_FAILURE);
1730 }
1731
1732 if (ddi_create_minor_node(devi, "fbt", S_IFCHR, 0,
1733 DDI_PSEUDO, 0) == DDI_FAILURE ||
1734 dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_KERNEL, NULL,
1735 &fbt_pops, NULL, &fbt_id) != 0) {
1736 ddi_remove_minor_node(devi, NULL);
1737 return (DDI_FAILURE);
1738 }
1739
1740 ddi_report_dev(devi);
1741 fbt_devi = devi;
1742 return (DDI_SUCCESS);
1743 }
1744
1745 static int
fbt_detach(dev_info_t * devi,ddi_detach_cmd_t cmd)1746 fbt_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
1747 {
1748 switch (cmd) {
1749 case DDI_DETACH:
1750 break;
1751 case DDI_SUSPEND:
1752 return (DDI_SUCCESS);
1753 default:
1754 return (DDI_FAILURE);
1755 }
1756
1757 if (dtrace_unregister(fbt_id) != 0)
1758 return (DDI_FAILURE);
1759
1760 ddi_remove_minor_node(devi, NULL);
1761 return (DDI_SUCCESS);
1762 }
1763
1764 /*ARGSUSED*/
1765 static int
fbt_info(dev_info_t * dip,ddi_info_cmd_t infocmd,void * arg,void ** result)1766 fbt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
1767 {
1768 int error;
1769
1770 switch (infocmd) {
1771 case DDI_INFO_DEVT2DEVINFO:
1772 *result = (void *)fbt_devi;
1773 error = DDI_SUCCESS;
1774 break;
1775 case DDI_INFO_DEVT2INSTANCE:
1776 *result = (void *)0;
1777 error = DDI_SUCCESS;
1778 break;
1779 default:
1780 error = DDI_FAILURE;
1781 }
1782 return (error);
1783 }
1784
1785 /*ARGSUSED*/
1786 static int
fbt_open(dev_t * devp,int flag,int otyp,cred_t * cred_p)1787 fbt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
1788 {
1789 return (0);
1790 }
1791
1792 static struct cb_ops fbt_cb_ops = {
1793 fbt_open, /* open */
1794 nodev, /* close */
1795 nulldev, /* strategy */
1796 nulldev, /* print */
1797 nodev, /* dump */
1798 nodev, /* read */
1799 nodev, /* write */
1800 nodev, /* ioctl */
1801 nodev, /* devmap */
1802 nodev, /* mmap */
1803 nodev, /* segmap */
1804 nochpoll, /* poll */
1805 ddi_prop_op, /* cb_prop_op */
1806 0, /* streamtab */
1807 D_NEW | D_MP /* Driver compatibility flag */
1808 };
1809
1810 static struct dev_ops fbt_ops = {
1811 DEVO_REV, /* devo_rev */
1812 0, /* refcnt */
1813 fbt_info, /* get_dev_info */
1814 nulldev, /* identify */
1815 nulldev, /* probe */
1816 fbt_attach, /* attach */
1817 fbt_detach, /* detach */
1818 nodev, /* reset */
1819 &fbt_cb_ops, /* driver operations */
1820 NULL, /* bus operations */
1821 nodev, /* dev power */
1822 ddi_quiesce_not_needed, /* quiesce */
1823 };
1824
1825 /*
1826 * Module linkage information for the kernel.
1827 */
1828 static struct modldrv modldrv = {
1829 &mod_driverops, /* module type (this is a pseudo driver) */
1830 "Function Boundary Tracing", /* name of module */
1831 &fbt_ops, /* driver ops */
1832 };
1833
1834 static struct modlinkage modlinkage = {
1835 MODREV_1,
1836 (void *)&modldrv,
1837 NULL
1838 };
1839
1840 int
_init(void)1841 _init(void)
1842 {
1843 return (mod_install(&modlinkage));
1844 }
1845
1846 int
_info(struct modinfo * modinfop)1847 _info(struct modinfo *modinfop)
1848 {
1849 return (mod_info(&modlinkage, modinfop));
1850 }
1851
1852 int
_fini(void)1853 _fini(void)
1854 {
1855 return (mod_remove(&modlinkage));
1856 }
1857