1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27/*
28 * Copyright (c) 2011, Joyent, Inc. All rights reserved.
29 */
30
31#include <sys/dtrace.h>
32#include <sys/fasttrap.h>
33#include <sys/x_call.h>
34#include <sys/cmn_err.h>
35#include <sys/trap.h>
36#include <sys/psw.h>
37#include <sys/privregs.h>
38#include <sys/machsystm.h>
39#include <vm/seg_kmem.h>
40
41typedef struct dtrace_invop_hdlr {
42	int (*dtih_func)(uintptr_t, uintptr_t *, uintptr_t);
43	struct dtrace_invop_hdlr *dtih_next;
44} dtrace_invop_hdlr_t;
45
46dtrace_invop_hdlr_t *dtrace_invop_hdlr;
47
48int
49dtrace_invop(uintptr_t addr, uintptr_t *stack, uintptr_t eax)
50{
51	dtrace_invop_hdlr_t *hdlr;
52	int rval;
53
54	for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next) {
55		if ((rval = hdlr->dtih_func(addr, stack, eax)) != 0)
56			return (rval);
57	}
58
59	return (0);
60}
61
62void
63dtrace_invop_add(int (*func)(uintptr_t, uintptr_t *, uintptr_t))
64{
65	dtrace_invop_hdlr_t *hdlr;
66
67	hdlr = kmem_alloc(sizeof (dtrace_invop_hdlr_t), KM_SLEEP);
68	hdlr->dtih_func = func;
69	hdlr->dtih_next = dtrace_invop_hdlr;
70	dtrace_invop_hdlr = hdlr;
71}
72
73void
74dtrace_invop_remove(int (*func)(uintptr_t, uintptr_t *, uintptr_t))
75{
76	dtrace_invop_hdlr_t *hdlr = dtrace_invop_hdlr, *prev = NULL;
77
78	for (;;) {
79		if (hdlr == NULL)
80			panic("attempt to remove non-existent invop handler");
81
82		if (hdlr->dtih_func == func)
83			break;
84
85		prev = hdlr;
86		hdlr = hdlr->dtih_next;
87	}
88
89	if (prev == NULL) {
90		ASSERT(dtrace_invop_hdlr == hdlr);
91		dtrace_invop_hdlr = hdlr->dtih_next;
92	} else {
93		ASSERT(dtrace_invop_hdlr != hdlr);
94		prev->dtih_next = hdlr->dtih_next;
95	}
96
97	kmem_free(hdlr, sizeof (dtrace_invop_hdlr_t));
98}
99
100int
101dtrace_getipl(void)
102{
103	return (CPU->cpu_pri);
104}
105
106/*ARGSUSED*/
107void
108dtrace_toxic_ranges(void (*func)(uintptr_t base, uintptr_t limit))
109{
110#ifdef __amd64
111	extern uintptr_t toxic_addr;
112	extern size_t toxic_size;
113
114	(*func)(0, _userlimit);
115
116	if (hole_end > hole_start)
117		(*func)(hole_start, hole_end);
118	(*func)(toxic_addr, toxic_addr + toxic_size);
119#else
120	extern void *device_arena_contains(void *, size_t, size_t *);
121	caddr_t	vaddr;
122	size_t	len;
123
124	for (vaddr = (caddr_t)kernelbase; vaddr < (caddr_t)KERNEL_TEXT;
125	    vaddr += len) {
126		len = (caddr_t)KERNEL_TEXT - vaddr;
127		vaddr = device_arena_contains(vaddr, len, &len);
128		if (vaddr == NULL)
129			break;
130		(*func)((uintptr_t)vaddr, (uintptr_t)vaddr + len);
131	}
132#endif
133	(*func)(0, _userlimit);
134}
135
136static int
137dtrace_xcall_func(xc_arg_t arg1, xc_arg_t arg2, xc_arg_t arg3 __unused)
138{
139	dtrace_xcall_t func = (dtrace_xcall_t)arg1;
140	(*func)((void*)arg2);
141
142	return (0);
143}
144
145/*ARGSUSED*/
146void
147dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg)
148{
149	cpuset_t set;
150
151	CPUSET_ZERO(set);
152
153	if (cpu == DTRACE_CPUALL) {
154		CPUSET_ALL(set);
155	} else {
156		CPUSET_ADD(set, cpu);
157	}
158
159	kpreempt_disable();
160	xc_sync((xc_arg_t)func, (xc_arg_t)arg, 0, CPUSET2BV(set),
161	    dtrace_xcall_func);
162	kpreempt_enable();
163}
164
165void
166dtrace_sync_func(void)
167{}
168
169void
170dtrace_sync(void)
171{
172	dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL);
173}
174
175int (*dtrace_pid_probe_ptr)(struct regs *);
176int (*dtrace_return_probe_ptr)(struct regs *);
177
178void
179dtrace_user_probe(struct regs *rp, caddr_t addr, processorid_t cpuid)
180{
181	krwlock_t *rwp;
182	proc_t *p = curproc;
183	extern void trap(struct regs *, caddr_t, processorid_t);
184
185	if (USERMODE(rp->r_cs) || (rp->r_ps & PS_VM)) {
186		if (curthread->t_cred != p->p_cred) {
187			cred_t *oldcred = curthread->t_cred;
188			/*
189			 * DTrace accesses t_cred in probe context.  t_cred
190			 * must always be either NULL, or point to a valid,
191			 * allocated cred structure.
192			 */
193			curthread->t_cred = crgetcred();
194			crfree(oldcred);
195		}
196	}
197
198	if (rp->r_trapno == T_DTRACE_RET) {
199		uint8_t step = curthread->t_dtrace_step;
200		uint8_t ret = curthread->t_dtrace_ret;
201		uintptr_t npc = curthread->t_dtrace_npc;
202
203		if (curthread->t_dtrace_ast) {
204			aston(curthread);
205			curthread->t_sig_check = 1;
206		}
207
208		/*
209		 * Clear all user tracing flags.
210		 */
211		curthread->t_dtrace_ft = 0;
212
213		/*
214		 * If we weren't expecting to take a return probe trap, kill
215		 * the process as though it had just executed an unassigned
216		 * trap instruction.
217		 */
218		if (step == 0) {
219			tsignal(curthread, SIGILL);
220			return;
221		}
222
223		/*
224		 * If we hit this trap unrelated to a return probe, we're
225		 * just here to reset the AST flag since we deferred a signal
226		 * until after we logically single-stepped the instruction we
227		 * copied out.
228		 */
229		if (ret == 0) {
230			rp->r_pc = npc;
231			return;
232		}
233
234		/*
235		 * We need to wait until after we've called the
236		 * dtrace_return_probe_ptr function pointer to set %pc.
237		 */
238		rwp = &CPU->cpu_ft_lock;
239		rw_enter(rwp, RW_READER);
240		if (dtrace_return_probe_ptr != NULL)
241			(void) (*dtrace_return_probe_ptr)(rp);
242		rw_exit(rwp);
243		rp->r_pc = npc;
244
245	} else if (rp->r_trapno == T_BPTFLT) {
246		uint8_t instr, instr2;
247		caddr_t linearpc;
248		rwp = &CPU->cpu_ft_lock;
249
250		/*
251		 * The DTrace fasttrap provider uses the breakpoint trap
252		 * (int 3). We let DTrace take the first crack at handling
253		 * this trap; if it's not a probe that DTrace knowns about,
254		 * we call into the trap() routine to handle it like a
255		 * breakpoint placed by a conventional debugger.
256		 */
257		rw_enter(rwp, RW_READER);
258		if (dtrace_pid_probe_ptr != NULL &&
259		    (*dtrace_pid_probe_ptr)(rp) == 0) {
260			rw_exit(rwp);
261			return;
262		}
263		rw_exit(rwp);
264
265		if (dtrace_linear_pc(rp, p, &linearpc) != 0) {
266			trap(rp, addr, cpuid);
267			return;
268		}
269
270		/*
271		 * If the instruction that caused the breakpoint trap doesn't
272		 * look like an int 3 anymore, it may be that this tracepoint
273		 * was removed just after the user thread executed it. In
274		 * that case, return to user land to retry the instuction.
275		 * Note that we assume the length of the instruction to retry
276		 * is 1 byte because that's the length of FASTTRAP_INSTR.
277		 * We check for r_pc > 0 and > 2 so that we don't have to
278		 * deal with segment wraparound.
279		 */
280		if (rp->r_pc > 0 && fuword8(linearpc - 1, &instr) == 0 &&
281		    instr != FASTTRAP_INSTR &&
282		    (instr != 3 || (rp->r_pc >= 2 &&
283		    (fuword8(linearpc - 2, &instr2) != 0 || instr2 != 0xCD)))) {
284			rp->r_pc--;
285			return;
286		}
287
288		trap(rp, addr, cpuid);
289
290	} else {
291		trap(rp, addr, cpuid);
292	}
293}
294
295void
296dtrace_safe_synchronous_signal(void)
297{
298	kthread_t *t = curthread;
299	struct regs *rp = lwptoregs(ttolwp(t));
300	size_t isz = t->t_dtrace_npc - t->t_dtrace_pc;
301
302	ASSERT(t->t_dtrace_on);
303
304	/*
305	 * If we're not in the range of scratch addresses, we're not actually
306	 * tracing user instructions so turn off the flags. If the instruction
307	 * we copied out caused a synchonous trap, reset the pc back to its
308	 * original value and turn off the flags.
309	 */
310	if (rp->r_pc < t->t_dtrace_scrpc ||
311	    rp->r_pc > t->t_dtrace_astpc + isz) {
312		t->t_dtrace_ft = 0;
313	} else if (rp->r_pc == t->t_dtrace_scrpc ||
314	    rp->r_pc == t->t_dtrace_astpc) {
315		rp->r_pc = t->t_dtrace_pc;
316		t->t_dtrace_ft = 0;
317	}
318}
319
320int
321dtrace_safe_defer_signal(void)
322{
323	kthread_t *t = curthread;
324	struct regs *rp = lwptoregs(ttolwp(t));
325	size_t isz = t->t_dtrace_npc - t->t_dtrace_pc;
326
327	ASSERT(t->t_dtrace_on);
328
329	/*
330	 * If we're not in the range of scratch addresses, we're not actually
331	 * tracing user instructions so turn off the flags.
332	 */
333	if (rp->r_pc < t->t_dtrace_scrpc ||
334	    rp->r_pc > t->t_dtrace_astpc + isz) {
335		t->t_dtrace_ft = 0;
336		return (0);
337	}
338
339	/*
340	 * If we have executed the original instruction, but we have performed
341	 * neither the jmp back to t->t_dtrace_npc nor the clean up of any
342	 * registers used to emulate %rip-relative instructions in 64-bit mode,
343	 * we'll save ourselves some effort by doing that here and taking the
344	 * signal right away.  We detect this condition by seeing if the program
345	 * counter is the range [scrpc + isz, astpc).
346	 */
347	if (rp->r_pc >= t->t_dtrace_scrpc + isz &&
348	    rp->r_pc < t->t_dtrace_astpc) {
349#ifdef __amd64
350		/*
351		 * If there is a scratch register and we're on the
352		 * instruction immediately after the modified instruction,
353		 * restore the value of that scratch register.
354		 */
355		if (t->t_dtrace_reg != 0 &&
356		    rp->r_pc == t->t_dtrace_scrpc + isz) {
357			switch (t->t_dtrace_reg) {
358			case REG_RAX:
359				rp->r_rax = t->t_dtrace_regv;
360				break;
361			case REG_RCX:
362				rp->r_rcx = t->t_dtrace_regv;
363				break;
364			case REG_R8:
365				rp->r_r8 = t->t_dtrace_regv;
366				break;
367			case REG_R9:
368				rp->r_r9 = t->t_dtrace_regv;
369				break;
370			}
371		}
372#endif
373		rp->r_pc = t->t_dtrace_npc;
374		t->t_dtrace_ft = 0;
375		return (0);
376	}
377
378	/*
379	 * Otherwise, make sure we'll return to the kernel after executing
380	 * the copied out instruction and defer the signal.
381	 */
382	if (!t->t_dtrace_step) {
383		ASSERT(rp->r_pc < t->t_dtrace_astpc);
384		rp->r_pc += t->t_dtrace_astpc - t->t_dtrace_scrpc;
385		t->t_dtrace_step = 1;
386	}
387
388	t->t_dtrace_ast = 1;
389
390	return (1);
391}
392
393/*
394 * Additional artificial frames for the machine type. For i86pc, we're already
395 * accounted for, so return 0. On the hypervisor, we have an additional frame
396 * (xen_callback_handler).
397 */
398int
399dtrace_mach_aframes(void)
400{
401#ifdef __xpv
402	return (1);
403#else
404	return (0);
405#endif
406}
407