xref: /illumos-gate/usr/src/uts/intel/ia32/os/archdep.c (revision cff040f3)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 /*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
26 /*	  All Rights Reserved	*/
27 /*
28  * Copyright (c) 2018, Joyent, Inc.
29  * Copyright 2012 Nexenta Systems, Inc.  All rights reserved.
30  */
31 
32 #include <sys/param.h>
33 #include <sys/types.h>
34 #include <sys/vmparam.h>
35 #include <sys/systm.h>
36 #include <sys/signal.h>
37 #include <sys/stack.h>
38 #include <sys/regset.h>
39 #include <sys/privregs.h>
40 #include <sys/frame.h>
41 #include <sys/proc.h>
42 #include <sys/psw.h>
43 #include <sys/siginfo.h>
44 #include <sys/cpuvar.h>
45 #include <sys/asm_linkage.h>
46 #include <sys/kmem.h>
47 #include <sys/errno.h>
48 #include <sys/bootconf.h>
49 #include <sys/archsystm.h>
50 #include <sys/debug.h>
51 #include <sys/elf.h>
52 #include <sys/spl.h>
53 #include <sys/time.h>
54 #include <sys/atomic.h>
55 #include <sys/sysmacros.h>
56 #include <sys/cmn_err.h>
57 #include <sys/modctl.h>
58 #include <sys/kobj.h>
59 #include <sys/panic.h>
60 #include <sys/reboot.h>
61 #include <sys/time.h>
62 #include <sys/fp.h>
63 #include <sys/x86_archext.h>
64 #include <sys/auxv.h>
65 #include <sys/auxv_386.h>
66 #include <sys/dtrace.h>
67 #include <sys/brand.h>
68 #include <sys/machbrand.h>
69 #include <sys/cmn_err.h>
70 
71 /*
72  * Map an fnsave-formatted save area into an fxsave-formatted save area.
73  *
74  * Most fields are the same width, content and semantics.  However
75  * the tag word is compressed.
76  */
77 static void
fnsave_to_fxsave(const struct fnsave_state * fn,struct fxsave_state * fx)78 fnsave_to_fxsave(const struct fnsave_state *fn, struct fxsave_state *fx)
79 {
80 	uint_t i, tagbits;
81 
82 	fx->fx_fcw = fn->f_fcw;
83 	fx->fx_fsw = fn->f_fsw;
84 
85 	/*
86 	 * copy element by element (because of holes)
87 	 */
88 	for (i = 0; i < 8; i++)
89 		bcopy(&fn->f_st[i].fpr_16[0], &fx->fx_st[i].fpr_16[0],
90 		    sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
91 
92 	/*
93 	 * synthesize compressed tag bits
94 	 */
95 	fx->fx_fctw = 0;
96 	for (tagbits = fn->f_ftw, i = 0; i < 8; i++, tagbits >>= 2)
97 		if ((tagbits & 3) != 3)
98 			fx->fx_fctw |= (1 << i);
99 
100 	fx->fx_fop = fn->f_fop;
101 
102 #if defined(__amd64)
103 	fx->fx_rip = (uint64_t)fn->f_eip;
104 	fx->fx_rdp = (uint64_t)fn->f_dp;
105 #else
106 	fx->fx_eip = fn->f_eip;
107 	fx->fx_cs = fn->f_cs;
108 	fx->__fx_ign0 = 0;
109 	fx->fx_dp = fn->f_dp;
110 	fx->fx_ds = fn->f_ds;
111 	fx->__fx_ign1 = 0;
112 #endif
113 }
114 
115 /*
116  * Map from an fxsave-format save area to an fnsave-format save area.
117  */
118 static void
fxsave_to_fnsave(const struct fxsave_state * fx,struct fnsave_state * fn)119 fxsave_to_fnsave(const struct fxsave_state *fx, struct fnsave_state *fn)
120 {
121 	uint_t i, top, tagbits;
122 
123 	fn->f_fcw = fx->fx_fcw;
124 	fn->__f_ign0 = 0;
125 	fn->f_fsw = fx->fx_fsw;
126 	fn->__f_ign1 = 0;
127 
128 	top = (fx->fx_fsw & FPS_TOP) >> 11;
129 
130 	/*
131 	 * copy element by element (because of holes)
132 	 */
133 	for (i = 0; i < 8; i++)
134 		bcopy(&fx->fx_st[i].fpr_16[0], &fn->f_st[i].fpr_16[0],
135 		    sizeof (fn->f_st[0].fpr_16)); /* 80-bit x87-style floats */
136 
137 	/*
138 	 * synthesize uncompressed tag bits
139 	 */
140 	fn->f_ftw = 0;
141 	for (tagbits = fx->fx_fctw, i = 0; i < 8; i++, tagbits >>= 1) {
142 		uint_t ibit, expo;
143 		const uint16_t *fpp;
144 		static const uint16_t zero[5] = { 0, 0, 0, 0, 0 };
145 
146 		if ((tagbits & 1) == 0) {
147 			fn->f_ftw |= 3 << (i << 1);	/* empty */
148 			continue;
149 		}
150 
151 		/*
152 		 * (tags refer to *physical* registers)
153 		 */
154 		fpp = &fx->fx_st[(i - top + 8) & 7].fpr_16[0];
155 		ibit = fpp[3] >> 15;
156 		expo = fpp[4] & 0x7fff;
157 
158 		if (ibit && expo != 0 && expo != 0x7fff)
159 			continue;			/* valid fp number */
160 
161 		if (bcmp(fpp, &zero, sizeof (zero)))
162 			fn->f_ftw |= 2 << (i << 1);	/* NaN */
163 		else
164 			fn->f_ftw |= 1 << (i << 1);	/* fp zero */
165 	}
166 
167 	fn->f_fop = fx->fx_fop;
168 
169 	fn->__f_ign2 = 0;
170 #if defined(__amd64)
171 	fn->f_eip = (uint32_t)fx->fx_rip;
172 	fn->f_cs = U32CS_SEL;
173 	fn->f_dp = (uint32_t)fx->fx_rdp;
174 	fn->f_ds = UDS_SEL;
175 #else
176 	fn->f_eip = fx->fx_eip;
177 	fn->f_cs = fx->fx_cs;
178 	fn->f_dp = fx->fx_dp;
179 	fn->f_ds = fx->fx_ds;
180 #endif
181 	fn->__f_ign3 = 0;
182 }
183 
184 /*
185  * Map from an fpregset_t into an fxsave-format save area
186  */
187 static void
fpregset_to_fxsave(const fpregset_t * fp,struct fxsave_state * fx)188 fpregset_to_fxsave(const fpregset_t *fp, struct fxsave_state *fx)
189 {
190 #if defined(__amd64)
191 	bcopy(fp, fx, sizeof (*fx));
192 #else
193 	const struct _fpchip_state *fc = &fp->fp_reg_set.fpchip_state;
194 
195 	fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
196 	fx->fx_mxcsr = fc->mxcsr;
197 	bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
198 #endif
199 	/*
200 	 * avoid useless #gp exceptions - mask reserved bits
201 	 */
202 	fx->fx_mxcsr &= sse_mxcsr_mask;
203 }
204 
205 /*
206  * Map from an fxsave-format save area into a fpregset_t
207  */
208 static void
fxsave_to_fpregset(const struct fxsave_state * fx,fpregset_t * fp)209 fxsave_to_fpregset(const struct fxsave_state *fx, fpregset_t *fp)
210 {
211 #if defined(__amd64)
212 	bcopy(fx, fp, sizeof (*fx));
213 #else
214 	struct _fpchip_state *fc = &fp->fp_reg_set.fpchip_state;
215 
216 	fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
217 	fc->mxcsr = fx->fx_mxcsr;
218 	bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
219 #endif
220 }
221 
222 #if defined(_SYSCALL32_IMPL)
223 static void
fpregset32_to_fxsave(const fpregset32_t * fp,struct fxsave_state * fx)224 fpregset32_to_fxsave(const fpregset32_t *fp, struct fxsave_state *fx)
225 {
226 	const struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
227 
228 	fnsave_to_fxsave((const struct fnsave_state *)fc, fx);
229 	/*
230 	 * avoid useless #gp exceptions - mask reserved bits
231 	 */
232 	fx->fx_mxcsr = sse_mxcsr_mask & fc->mxcsr;
233 	bcopy(&fc->xmm[0], &fx->fx_xmm[0], sizeof (fc->xmm));
234 }
235 
236 static void
fxsave_to_fpregset32(const struct fxsave_state * fx,fpregset32_t * fp)237 fxsave_to_fpregset32(const struct fxsave_state *fx, fpregset32_t *fp)
238 {
239 	struct fpchip32_state *fc = &fp->fp_reg_set.fpchip_state;
240 
241 	fxsave_to_fnsave(fx, (struct fnsave_state *)fc);
242 	fc->mxcsr = fx->fx_mxcsr;
243 	bcopy(&fx->fx_xmm[0], &fc->xmm[0], sizeof (fc->xmm));
244 }
245 
246 static void
fpregset_nto32(const fpregset_t * src,fpregset32_t * dst)247 fpregset_nto32(const fpregset_t *src, fpregset32_t *dst)
248 {
249 	fxsave_to_fpregset32((struct fxsave_state *)src, dst);
250 	dst->fp_reg_set.fpchip_state.status =
251 	    src->fp_reg_set.fpchip_state.status;
252 	dst->fp_reg_set.fpchip_state.xstatus =
253 	    src->fp_reg_set.fpchip_state.xstatus;
254 }
255 
256 static void
fpregset_32ton(const fpregset32_t * src,fpregset_t * dst)257 fpregset_32ton(const fpregset32_t *src, fpregset_t *dst)
258 {
259 	fpregset32_to_fxsave(src, (struct fxsave_state *)dst);
260 	dst->fp_reg_set.fpchip_state.status =
261 	    src->fp_reg_set.fpchip_state.status;
262 	dst->fp_reg_set.fpchip_state.xstatus =
263 	    src->fp_reg_set.fpchip_state.xstatus;
264 }
265 #endif
266 
267 /*
268  * Set floating-point registers from a native fpregset_t.
269  */
270 void
setfpregs(klwp_t * lwp,fpregset_t * fp)271 setfpregs(klwp_t *lwp, fpregset_t *fp)
272 {
273 	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
274 
275 	if (fpu->fpu_flags & FPU_EN) {
276 		if (!(fpu->fpu_flags & FPU_VALID)) {
277 			/*
278 			 * FPU context is still active, release the
279 			 * ownership.
280 			 */
281 			fp_free(fpu, 0);
282 		}
283 	}
284 	/*
285 	 * Else: if we are trying to change the FPU state of a thread which
286 	 * hasn't yet initialized floating point, store the state in
287 	 * the pcb and indicate that the state is valid.  When the
288 	 * thread enables floating point, it will use this state instead
289 	 * of the default state.
290 	 */
291 
292 	switch (fp_save_mech) {
293 #if defined(__i386)
294 	case FP_FNSAVE:
295 		bcopy(fp, fpu->fpu_regs.kfpu_u.kfpu_fn,
296 		    sizeof (*fpu->fpu_regs.kfpu_u.kfpu_fn));
297 		break;
298 #endif
299 	case FP_FXSAVE:
300 		fpregset_to_fxsave(fp, fpu->fpu_regs.kfpu_u.kfpu_fx);
301 		fpu->fpu_regs.kfpu_xstatus =
302 		    fp->fp_reg_set.fpchip_state.xstatus;
303 		break;
304 
305 	case FP_XSAVE:
306 		fpregset_to_fxsave(fp,
307 		    &fpu->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave);
308 		fpu->fpu_regs.kfpu_xstatus =
309 		    fp->fp_reg_set.fpchip_state.xstatus;
310 		fpu->fpu_regs.kfpu_u.kfpu_xs->xs_xstate_bv |=
311 		    (XFEATURE_LEGACY_FP | XFEATURE_SSE);
312 		break;
313 	default:
314 		panic("Invalid fp_save_mech");
315 		/*NOTREACHED*/
316 	}
317 
318 	fpu->fpu_regs.kfpu_status = fp->fp_reg_set.fpchip_state.status;
319 	fpu->fpu_flags |= FPU_VALID;
320 	PCB_SET_UPDATE_FPU(&lwp->lwp_pcb);
321 }
322 
323 /*
324  * Get floating-point registers into a native fpregset_t.
325  */
326 void
getfpregs(klwp_t * lwp,fpregset_t * fp)327 getfpregs(klwp_t *lwp, fpregset_t *fp)
328 {
329 	struct fpu_ctx *fpu = &lwp->lwp_pcb.pcb_fpu;
330 
331 	kpreempt_disable();
332 	if (fpu->fpu_flags & FPU_EN) {
333 		/*
334 		 * If we have FPU hw and the thread's pcb doesn't have
335 		 * a valid FPU state then get the state from the hw.
336 		 */
337 		if (fpu_exists && ttolwp(curthread) == lwp &&
338 		    !(fpu->fpu_flags & FPU_VALID))
339 			fp_save(fpu); /* get the current FPU state */
340 	}
341 
342 	/*
343 	 * There are 3 possible cases we have to be aware of here:
344 	 *
345 	 * 1. FPU is enabled.  FPU state is stored in the current LWP.
346 	 *
347 	 * 2. FPU is not enabled, and there have been no intervening /proc
348 	 *    modifications.  Return initial FPU state.
349 	 *
350 	 * 3. FPU is not enabled, but a /proc consumer has modified FPU state.
351 	 *    FPU state is stored in the current LWP.
352 	 */
353 	if ((fpu->fpu_flags & FPU_EN) || (fpu->fpu_flags & FPU_VALID)) {
354 		/*
355 		 * Cases 1 and 3.
356 		 */
357 		switch (fp_save_mech) {
358 #if defined(__i386)
359 		case FP_FNSAVE:
360 			bcopy(fpu->fpu_regs.kfpu_u.kfpu_fn, fp,
361 			    sizeof (*fpu->fpu_regs.kfpu_u.kfpu_fn));
362 			break;
363 #endif
364 		case FP_FXSAVE:
365 			fxsave_to_fpregset(fpu->fpu_regs.kfpu_u.kfpu_fx, fp);
366 			fp->fp_reg_set.fpchip_state.xstatus =
367 			    fpu->fpu_regs.kfpu_xstatus;
368 			break;
369 		case FP_XSAVE:
370 			fxsave_to_fpregset(
371 			    &fpu->fpu_regs.kfpu_u.kfpu_xs->xs_fxsave, fp);
372 			fp->fp_reg_set.fpchip_state.xstatus =
373 			    fpu->fpu_regs.kfpu_xstatus;
374 			break;
375 		default:
376 			panic("Invalid fp_save_mech");
377 			/*NOTREACHED*/
378 		}
379 		fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
380 	} else {
381 		/*
382 		 * Case 2.
383 		 */
384 		switch (fp_save_mech) {
385 #if defined(__i386)
386 		case FP_FNSAVE:
387 			bcopy(&x87_initial, fp, sizeof (x87_initial));
388 			break;
389 #endif
390 		case FP_FXSAVE:
391 		case FP_XSAVE:
392 			/*
393 			 * For now, we don't have any AVX specific field in ABI.
394 			 * If we add any in the future, we need to initial them
395 			 * as well.
396 			 */
397 			fxsave_to_fpregset(&sse_initial, fp);
398 			fp->fp_reg_set.fpchip_state.xstatus =
399 			    fpu->fpu_regs.kfpu_xstatus;
400 			break;
401 		default:
402 			panic("Invalid fp_save_mech");
403 			/*NOTREACHED*/
404 		}
405 		fp->fp_reg_set.fpchip_state.status = fpu->fpu_regs.kfpu_status;
406 	}
407 	kpreempt_enable();
408 }
409 
410 #if defined(_SYSCALL32_IMPL)
411 
412 /*
413  * Set floating-point registers from an fpregset32_t.
414  */
415 void
setfpregs32(klwp_t * lwp,fpregset32_t * fp)416 setfpregs32(klwp_t *lwp, fpregset32_t *fp)
417 {
418 	fpregset_t fpregs;
419 
420 	fpregset_32ton(fp, &fpregs);
421 	setfpregs(lwp, &fpregs);
422 }
423 
424 /*
425  * Get floating-point registers into an fpregset32_t.
426  */
427 void
getfpregs32(klwp_t * lwp,fpregset32_t * fp)428 getfpregs32(klwp_t *lwp, fpregset32_t *fp)
429 {
430 	fpregset_t fpregs;
431 
432 	getfpregs(lwp, &fpregs);
433 	fpregset_nto32(&fpregs, fp);
434 }
435 
436 #endif	/* _SYSCALL32_IMPL */
437 
438 /*
439  * Return the general registers
440  */
441 void
getgregs(klwp_t * lwp,gregset_t grp)442 getgregs(klwp_t *lwp, gregset_t grp)
443 {
444 	struct regs *rp = lwptoregs(lwp);
445 #if defined(__amd64)
446 	struct pcb *pcb = &lwp->lwp_pcb;
447 	int thisthread = lwptot(lwp) == curthread;
448 
449 	grp[REG_RDI] = rp->r_rdi;
450 	grp[REG_RSI] = rp->r_rsi;
451 	grp[REG_RDX] = rp->r_rdx;
452 	grp[REG_RCX] = rp->r_rcx;
453 	grp[REG_R8] = rp->r_r8;
454 	grp[REG_R9] = rp->r_r9;
455 	grp[REG_RAX] = rp->r_rax;
456 	grp[REG_RBX] = rp->r_rbx;
457 	grp[REG_RBP] = rp->r_rbp;
458 	grp[REG_R10] = rp->r_r10;
459 	grp[REG_R11] = rp->r_r11;
460 	grp[REG_R12] = rp->r_r12;
461 	grp[REG_R13] = rp->r_r13;
462 	grp[REG_R14] = rp->r_r14;
463 	grp[REG_R15] = rp->r_r15;
464 	grp[REG_FSBASE] = pcb->pcb_fsbase;
465 	grp[REG_GSBASE] = pcb->pcb_gsbase;
466 	if (thisthread)
467 		kpreempt_disable();
468 	if (PCB_NEED_UPDATE_SEGS(pcb)) {
469 		grp[REG_DS] = pcb->pcb_ds;
470 		grp[REG_ES] = pcb->pcb_es;
471 		grp[REG_FS] = pcb->pcb_fs;
472 		grp[REG_GS] = pcb->pcb_gs;
473 	} else {
474 		grp[REG_DS] = rp->r_ds;
475 		grp[REG_ES] = rp->r_es;
476 		grp[REG_FS] = rp->r_fs;
477 		grp[REG_GS] = rp->r_gs;
478 	}
479 	if (thisthread)
480 		kpreempt_enable();
481 	grp[REG_TRAPNO] = rp->r_trapno;
482 	grp[REG_ERR] = rp->r_err;
483 	grp[REG_RIP] = rp->r_rip;
484 	grp[REG_CS] = rp->r_cs;
485 	grp[REG_SS] = rp->r_ss;
486 	grp[REG_RFL] = rp->r_rfl;
487 	grp[REG_RSP] = rp->r_rsp;
488 #else
489 	bcopy(&rp->r_gs, grp, sizeof (gregset_t));
490 #endif
491 }
492 
493 #if defined(_SYSCALL32_IMPL)
494 
495 void
getgregs32(klwp_t * lwp,gregset32_t grp)496 getgregs32(klwp_t *lwp, gregset32_t grp)
497 {
498 	struct regs *rp = lwptoregs(lwp);
499 	struct pcb *pcb = &lwp->lwp_pcb;
500 	int thisthread = lwptot(lwp) == curthread;
501 
502 	if (thisthread)
503 		kpreempt_disable();
504 	if (PCB_NEED_UPDATE_SEGS(pcb)) {
505 		grp[GS] = (uint16_t)pcb->pcb_gs;
506 		grp[FS] = (uint16_t)pcb->pcb_fs;
507 		grp[DS] = (uint16_t)pcb->pcb_ds;
508 		grp[ES] = (uint16_t)pcb->pcb_es;
509 	} else {
510 		grp[GS] = (uint16_t)rp->r_gs;
511 		grp[FS] = (uint16_t)rp->r_fs;
512 		grp[DS] = (uint16_t)rp->r_ds;
513 		grp[ES] = (uint16_t)rp->r_es;
514 	}
515 	if (thisthread)
516 		kpreempt_enable();
517 	grp[EDI] = (greg32_t)rp->r_rdi;
518 	grp[ESI] = (greg32_t)rp->r_rsi;
519 	grp[EBP] = (greg32_t)rp->r_rbp;
520 	grp[ESP] = 0;
521 	grp[EBX] = (greg32_t)rp->r_rbx;
522 	grp[EDX] = (greg32_t)rp->r_rdx;
523 	grp[ECX] = (greg32_t)rp->r_rcx;
524 	grp[EAX] = (greg32_t)rp->r_rax;
525 	grp[TRAPNO] = (greg32_t)rp->r_trapno;
526 	grp[ERR] = (greg32_t)rp->r_err;
527 	grp[EIP] = (greg32_t)rp->r_rip;
528 	grp[CS] = (uint16_t)rp->r_cs;
529 	grp[EFL] = (greg32_t)rp->r_rfl;
530 	grp[UESP] = (greg32_t)rp->r_rsp;
531 	grp[SS] = (uint16_t)rp->r_ss;
532 }
533 
534 void
ucontext_32ton(const ucontext32_t * src,ucontext_t * dst)535 ucontext_32ton(const ucontext32_t *src, ucontext_t *dst)
536 {
537 	mcontext_t *dmc = &dst->uc_mcontext;
538 	const mcontext32_t *smc = &src->uc_mcontext;
539 
540 	bzero(dst, sizeof (*dst));
541 	dst->uc_flags = src->uc_flags;
542 	dst->uc_link = (ucontext_t *)(uintptr_t)src->uc_link;
543 
544 	bcopy(&src->uc_sigmask, &dst->uc_sigmask, sizeof (dst->uc_sigmask));
545 
546 	dst->uc_stack.ss_sp = (void *)(uintptr_t)src->uc_stack.ss_sp;
547 	dst->uc_stack.ss_size = (size_t)src->uc_stack.ss_size;
548 	dst->uc_stack.ss_flags = src->uc_stack.ss_flags;
549 
550 	dmc->gregs[REG_GS] = (greg_t)(uint32_t)smc->gregs[GS];
551 	dmc->gregs[REG_FS] = (greg_t)(uint32_t)smc->gregs[FS];
552 	dmc->gregs[REG_ES] = (greg_t)(uint32_t)smc->gregs[ES];
553 	dmc->gregs[REG_DS] = (greg_t)(uint32_t)smc->gregs[DS];
554 	dmc->gregs[REG_RDI] = (greg_t)(uint32_t)smc->gregs[EDI];
555 	dmc->gregs[REG_RSI] = (greg_t)(uint32_t)smc->gregs[ESI];
556 	dmc->gregs[REG_RBP] = (greg_t)(uint32_t)smc->gregs[EBP];
557 	dmc->gregs[REG_RBX] = (greg_t)(uint32_t)smc->gregs[EBX];
558 	dmc->gregs[REG_RDX] = (greg_t)(uint32_t)smc->gregs[EDX];
559 	dmc->gregs[REG_RCX] = (greg_t)(uint32_t)smc->gregs[ECX];
560 	dmc->gregs[REG_RAX] = (greg_t)(uint32_t)smc->gregs[EAX];
561 	dmc->gregs[REG_TRAPNO] = (greg_t)(uint32_t)smc->gregs[TRAPNO];
562 	dmc->gregs[REG_ERR] = (greg_t)(uint32_t)smc->gregs[ERR];
563 	dmc->gregs[REG_RIP] = (greg_t)(uint32_t)smc->gregs[EIP];
564 	dmc->gregs[REG_CS] = (greg_t)(uint32_t)smc->gregs[CS];
565 	dmc->gregs[REG_RFL] = (greg_t)(uint32_t)smc->gregs[EFL];
566 	dmc->gregs[REG_RSP] = (greg_t)(uint32_t)smc->gregs[UESP];
567 	dmc->gregs[REG_SS] = (greg_t)(uint32_t)smc->gregs[SS];
568 
569 	/*
570 	 * A valid fpregs is only copied in if uc.uc_flags has UC_FPU set
571 	 * otherwise there is no guarantee that anything in fpregs is valid.
572 	 */
573 	if (src->uc_flags & UC_FPU)
574 		fpregset_32ton(&src->uc_mcontext.fpregs,
575 		    &dst->uc_mcontext.fpregs);
576 }
577 
578 #endif	/* _SYSCALL32_IMPL */
579 
580 /*
581  * Return the user-level PC.
582  * If in a system call, return the address of the syscall trap.
583  */
584 greg_t
getuserpc()585 getuserpc()
586 {
587 	greg_t upc = lwptoregs(ttolwp(curthread))->r_pc;
588 	uint32_t insn;
589 
590 	if (curthread->t_sysnum == 0)
591 		return (upc);
592 
593 	/*
594 	 * We might've gotten here from sysenter (0xf 0x34),
595 	 * syscall (0xf 0x5) or lcall (0x9a 0 0 0 0 0x27 0).
596 	 *
597 	 * Go peek at the binary to figure it out..
598 	 */
599 	if (fuword32((void *)(upc - 2), &insn) != -1 &&
600 	    (insn & 0xffff) == 0x340f || (insn & 0xffff) == 0x050f)
601 		return (upc - 2);
602 	return (upc - 7);
603 }
604 
605 /*
606  * Protect segment registers from non-user privilege levels and GDT selectors
607  * other than USER_CS, USER_DS and lwp FS and GS values.  If the segment
608  * selector is non-null and not USER_CS/USER_DS, we make sure that the
609  * TI bit is set to point into the LDT and that the RPL is set to 3.
610  *
611  * Since struct regs stores each 16-bit segment register as a 32-bit greg_t, we
612  * also explicitly zero the top 16 bits since they may be coming from the
613  * user's address space via setcontext(2) or /proc.
614  *
615  * Note about null selector. When running on the hypervisor if we allow a
616  * process to set its %cs to null selector with RPL of 0 the hypervisor will
617  * crash the domain. If running on bare metal we would get a #gp fault and
618  * be able to kill the process and continue on. Therefore we make sure to
619  * force RPL to SEL_UPL even for null selector when setting %cs.
620  */
621 
622 #if defined(IS_CS) || defined(IS_NOT_CS)
623 #error	"IS_CS and IS_NOT_CS already defined"
624 #endif
625 
626 #define	IS_CS		1
627 #define	IS_NOT_CS	0
628 
629 /*ARGSUSED*/
630 static greg_t
fix_segreg(greg_t sr,int iscs,model_t datamodel)631 fix_segreg(greg_t sr, int iscs, model_t datamodel)
632 {
633 	switch (sr &= 0xffff) {
634 
635 	case 0:
636 		if (iscs == IS_CS)
637 			return (0 | SEL_UPL);
638 		else
639 			return (0);
640 
641 #if defined(__amd64)
642 	/*
643 	 * If lwp attempts to switch data model then force their
644 	 * code selector to be null selector.
645 	 */
646 	case U32CS_SEL:
647 		if (datamodel == DATAMODEL_NATIVE)
648 			return (0 | SEL_UPL);
649 		else
650 			return (sr);
651 
652 	case UCS_SEL:
653 		if (datamodel == DATAMODEL_ILP32)
654 			return (0 | SEL_UPL);
655 #elif defined(__i386)
656 	case UCS_SEL:
657 #endif
658 	/*FALLTHROUGH*/
659 	case UDS_SEL:
660 	case LWPFS_SEL:
661 	case LWPGS_SEL:
662 	case SEL_UPL:
663 		return (sr);
664 	default:
665 		break;
666 	}
667 
668 	/*
669 	 * Force it into the LDT in ring 3 for 32-bit processes, which by
670 	 * default do not have an LDT, so that any attempt to use an invalid
671 	 * selector will reference the (non-existant) LDT, and cause a #gp
672 	 * fault for the process.
673 	 *
674 	 * 64-bit processes get the null gdt selector since they
675 	 * are not allowed to have a private LDT.
676 	 */
677 #if defined(__amd64)
678 	if (datamodel == DATAMODEL_ILP32) {
679 		return (sr | SEL_TI_LDT | SEL_UPL);
680 	} else {
681 		if (iscs == IS_CS)
682 			return (0 | SEL_UPL);
683 		else
684 			return (0);
685 	}
686 
687 #elif defined(__i386)
688 	return (sr | SEL_TI_LDT | SEL_UPL);
689 #endif
690 }
691 
692 /*
693  * Set general registers.
694  */
695 void
setgregs(klwp_t * lwp,gregset_t grp)696 setgregs(klwp_t *lwp, gregset_t grp)
697 {
698 	struct regs *rp = lwptoregs(lwp);
699 	model_t	datamodel = lwp_getdatamodel(lwp);
700 
701 #if defined(__amd64)
702 	struct pcb *pcb = &lwp->lwp_pcb;
703 	int thisthread = lwptot(lwp) == curthread;
704 
705 	if (datamodel == DATAMODEL_NATIVE) {
706 
707 		if (thisthread)
708 			(void) save_syscall_args();	/* copy the args */
709 
710 		rp->r_rdi = grp[REG_RDI];
711 		rp->r_rsi = grp[REG_RSI];
712 		rp->r_rdx = grp[REG_RDX];
713 		rp->r_rcx = grp[REG_RCX];
714 		rp->r_r8 = grp[REG_R8];
715 		rp->r_r9 = grp[REG_R9];
716 		rp->r_rax = grp[REG_RAX];
717 		rp->r_rbx = grp[REG_RBX];
718 		rp->r_rbp = grp[REG_RBP];
719 		rp->r_r10 = grp[REG_R10];
720 		rp->r_r11 = grp[REG_R11];
721 		rp->r_r12 = grp[REG_R12];
722 		rp->r_r13 = grp[REG_R13];
723 		rp->r_r14 = grp[REG_R14];
724 		rp->r_r15 = grp[REG_R15];
725 		rp->r_trapno = grp[REG_TRAPNO];
726 		rp->r_err = grp[REG_ERR];
727 		rp->r_rip = grp[REG_RIP];
728 		/*
729 		 * Setting %cs or %ss to anything else is quietly but
730 		 * quite definitely forbidden!
731 		 */
732 		rp->r_cs = UCS_SEL;
733 		rp->r_ss = UDS_SEL;
734 		rp->r_rsp = grp[REG_RSP];
735 
736 		if (thisthread)
737 			kpreempt_disable();
738 
739 		pcb->pcb_ds = UDS_SEL;
740 		pcb->pcb_es = UDS_SEL;
741 
742 		/*
743 		 * 64-bit processes -are- allowed to set their fsbase/gsbase
744 		 * values directly, but only if they're using the segment
745 		 * selectors that allow that semantic.
746 		 *
747 		 * (32-bit processes must use lwp_set_private().)
748 		 */
749 		pcb->pcb_fsbase = grp[REG_FSBASE];
750 		pcb->pcb_gsbase = grp[REG_GSBASE];
751 		pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
752 		pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
753 
754 		/*
755 		 * Ensure that we go out via update_sregs
756 		 */
757 		PCB_SET_UPDATE_SEGS(pcb);
758 		lwptot(lwp)->t_post_sys = 1;
759 		if (thisthread)
760 			kpreempt_enable();
761 #if defined(_SYSCALL32_IMPL)
762 	} else {
763 		rp->r_rdi = (uint32_t)grp[REG_RDI];
764 		rp->r_rsi = (uint32_t)grp[REG_RSI];
765 		rp->r_rdx = (uint32_t)grp[REG_RDX];
766 		rp->r_rcx = (uint32_t)grp[REG_RCX];
767 		rp->r_rax = (uint32_t)grp[REG_RAX];
768 		rp->r_rbx = (uint32_t)grp[REG_RBX];
769 		rp->r_rbp = (uint32_t)grp[REG_RBP];
770 		rp->r_trapno = (uint32_t)grp[REG_TRAPNO];
771 		rp->r_err = (uint32_t)grp[REG_ERR];
772 		rp->r_rip = (uint32_t)grp[REG_RIP];
773 
774 		rp->r_cs = fix_segreg(grp[REG_CS], IS_CS, datamodel);
775 		rp->r_ss = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
776 
777 		rp->r_rsp = (uint32_t)grp[REG_RSP];
778 
779 		if (thisthread)
780 			kpreempt_disable();
781 
782 		pcb->pcb_ds = fix_segreg(grp[REG_DS], IS_NOT_CS, datamodel);
783 		pcb->pcb_es = fix_segreg(grp[REG_ES], IS_NOT_CS, datamodel);
784 
785 		/*
786 		 * (See fsbase/gsbase commentary above)
787 		 */
788 		pcb->pcb_fs = fix_segreg(grp[REG_FS], IS_NOT_CS, datamodel);
789 		pcb->pcb_gs = fix_segreg(grp[REG_GS], IS_NOT_CS, datamodel);
790 
791 		/*
792 		 * Ensure that we go out via update_sregs
793 		 */
794 		PCB_SET_UPDATE_SEGS(pcb);
795 		lwptot(lwp)->t_post_sys = 1;
796 		if (thisthread)
797 			kpreempt_enable();
798 #endif
799 	}
800 
801 	/*
802 	 * Only certain bits of the flags register can be modified.
803 	 */
804 	rp->r_rfl = (rp->r_rfl & ~PSL_USERMASK) |
805 	    (grp[REG_RFL] & PSL_USERMASK);
806 
807 #elif defined(__i386)
808 
809 	/*
810 	 * Only certain bits of the flags register can be modified.
811 	 */
812 	grp[EFL] = (rp->r_efl & ~PSL_USERMASK) | (grp[EFL] & PSL_USERMASK);
813 
814 	/*
815 	 * Copy saved registers from user stack.
816 	 */
817 	bcopy(grp, &rp->r_gs, sizeof (gregset_t));
818 
819 	rp->r_cs = fix_segreg(rp->r_cs, IS_CS, datamodel);
820 	rp->r_ss = fix_segreg(rp->r_ss, IS_NOT_CS, datamodel);
821 	rp->r_ds = fix_segreg(rp->r_ds, IS_NOT_CS, datamodel);
822 	rp->r_es = fix_segreg(rp->r_es, IS_NOT_CS, datamodel);
823 	rp->r_fs = fix_segreg(rp->r_fs, IS_NOT_CS, datamodel);
824 	rp->r_gs = fix_segreg(rp->r_gs, IS_NOT_CS, datamodel);
825 
826 #endif	/* __i386 */
827 }
828 
829 /*
830  * Determine whether eip is likely to have an interrupt frame
831  * on the stack.  We do this by comparing the address to the
832  * range of addresses spanned by several well-known routines.
833  */
834 extern void _interrupt();
835 extern void _allsyscalls();
836 extern void _cmntrap();
837 extern void fakesoftint();
838 
839 extern size_t _interrupt_size;
840 extern size_t _allsyscalls_size;
841 extern size_t _cmntrap_size;
842 extern size_t _fakesoftint_size;
843 
844 /*
845  * Get a pc-only stacktrace.  Used for kmem_alloc() buffer ownership tracking.
846  * Returns MIN(current stack depth, pcstack_limit).
847  */
848 int
getpcstack(pc_t * pcstack,int pcstack_limit)849 getpcstack(pc_t *pcstack, int pcstack_limit)
850 {
851 	struct frame *fp = (struct frame *)getfp();
852 	struct frame *nextfp, *minfp, *stacktop;
853 	int depth = 0;
854 	int on_intr;
855 	uintptr_t pc;
856 
857 	if ((on_intr = CPU_ON_INTR(CPU)) != 0)
858 		stacktop = (struct frame *)(CPU->cpu_intr_stack + SA(MINFRAME));
859 	else
860 		stacktop = (struct frame *)curthread->t_stk;
861 	minfp = fp;
862 
863 	pc = ((struct regs *)fp)->r_pc;
864 
865 	while (depth < pcstack_limit) {
866 		nextfp = (struct frame *)fp->fr_savfp;
867 		pc = fp->fr_savpc;
868 		if (nextfp <= minfp || nextfp >= stacktop) {
869 			if (on_intr) {
870 				/*
871 				 * Hop from interrupt stack to thread stack.
872 				 */
873 				stacktop = (struct frame *)curthread->t_stk;
874 				minfp = (struct frame *)curthread->t_stkbase;
875 				on_intr = 0;
876 				continue;
877 			}
878 			break;
879 		}
880 		pcstack[depth++] = (pc_t)pc;
881 		fp = nextfp;
882 		minfp = fp;
883 	}
884 	return (depth);
885 }
886 
887 /*
888  * The following ELF header fields are defined as processor-specific
889  * in the V8 ABI:
890  *
891  *	e_ident[EI_DATA]	encoding of the processor-specific
892  *				data in the object file
893  *	e_machine		processor identification
894  *	e_flags			processor-specific flags associated
895  *				with the file
896  */
897 
898 /*
899  * The value of at_flags reflects a platform's cpu module support.
900  * at_flags is used to check for allowing a binary to execute and
901  * is passed as the value of the AT_FLAGS auxiliary vector.
902  */
903 int at_flags = 0;
904 
905 /*
906  * Check the processor-specific fields of an ELF header.
907  *
908  * returns 1 if the fields are valid, 0 otherwise
909  */
910 /*ARGSUSED2*/
911 int
elfheadcheck(unsigned char e_data,Elf32_Half e_machine,Elf32_Word e_flags)912 elfheadcheck(
913 	unsigned char e_data,
914 	Elf32_Half e_machine,
915 	Elf32_Word e_flags)
916 {
917 	if (e_data != ELFDATA2LSB)
918 		return (0);
919 #if defined(__amd64)
920 	if (e_machine == EM_AMD64)
921 		return (1);
922 #endif
923 	return (e_machine == EM_386);
924 }
925 
926 uint_t auxv_hwcap_include = 0;	/* patch to enable unrecognized features */
927 uint_t auxv_hwcap_include_2 = 0;	/* second word */
928 uint_t auxv_hwcap_exclude = 0;	/* patch for broken cpus, debugging */
929 uint_t auxv_hwcap_exclude_2 = 0;	/* second word */
930 #if defined(_SYSCALL32_IMPL)
931 uint_t auxv_hwcap32_include = 0;	/* ditto for 32-bit apps */
932 uint_t auxv_hwcap32_include_2 = 0;	/* ditto for 32-bit apps */
933 uint_t auxv_hwcap32_exclude = 0;	/* ditto for 32-bit apps */
934 uint_t auxv_hwcap32_exclude_2 = 0;	/* ditto for 32-bit apps */
935 #endif
936 
937 /*
938  * Gather information about the processor and place it into auxv_hwcap
939  * so that it can be exported to the linker via the aux vector.
940  *
941  * We use this seemingly complicated mechanism so that we can ensure
942  * that /etc/system can be used to override what the system can or
943  * cannot discover for itself.
944  */
945 void
bind_hwcap(void)946 bind_hwcap(void)
947 {
948 	uint_t cpu_hwcap_flags[2];
949 	cpuid_pass4(NULL, cpu_hwcap_flags);
950 
951 	auxv_hwcap = (auxv_hwcap_include | cpu_hwcap_flags[0]) &
952 	    ~auxv_hwcap_exclude;
953 	auxv_hwcap_2 = (auxv_hwcap_include_2 | cpu_hwcap_flags[1]) &
954 	    ~auxv_hwcap_exclude_2;
955 
956 #if defined(__amd64)
957 	/*
958 	 * On AMD processors, sysenter just doesn't work at all
959 	 * when the kernel is in long mode.  On IA-32e processors
960 	 * it does, but there's no real point in all the alternate
961 	 * mechanism when syscall works on both.
962 	 *
963 	 * Besides, the kernel's sysenter handler is expecting a
964 	 * 32-bit lwp ...
965 	 */
966 	auxv_hwcap &= ~AV_386_SEP;
967 #else
968 	/*
969 	 * 32-bit processes can -always- use the lahf/sahf instructions
970 	 */
971 	auxv_hwcap |= AV_386_AHF;
972 #endif
973 
974 	if (auxv_hwcap_include || auxv_hwcap_exclude || auxv_hwcap_include_2 ||
975 	    auxv_hwcap_exclude_2) {
976 		/*
977 		 * The below assignment is regrettably required to get lint
978 		 * to accept the validity of our format string.  The format
979 		 * string is in fact valid, but whatever intelligence in lint
980 		 * understands the cmn_err()-specific %b appears to have an
981 		 * off-by-one error:  it (mistakenly) complains about bit
982 		 * number 32 (even though this is explicitly permitted).
983 		 * Normally, one would will away such warnings with a "LINTED"
984 		 * directive, but for reasons unclear and unknown, lint
985 		 * refuses to be assuaged in this case.  Fortunately, lint
986 		 * doesn't pretend to have solved the Halting Problem --
987 		 * and as soon as the format string is programmatic, it
988 		 * knows enough to shut up.
989 		 */
990 		char *fmt = "?user ABI extensions: %b\n";
991 		cmn_err(CE_CONT, fmt, auxv_hwcap, FMT_AV_386);
992 		fmt = "?user ABI extensions (word 2): %b\n";
993 		cmn_err(CE_CONT, fmt, auxv_hwcap_2, FMT_AV_386_2);
994 	}
995 
996 #if defined(_SYSCALL32_IMPL)
997 	auxv_hwcap32 = (auxv_hwcap32_include | cpu_hwcap_flags[0]) &
998 	    ~auxv_hwcap32_exclude;
999 	auxv_hwcap32_2 = (auxv_hwcap32_include_2 | cpu_hwcap_flags[1]) &
1000 	    ~auxv_hwcap32_exclude_2;
1001 
1002 #if defined(__amd64)
1003 	/*
1004 	 * If this is an amd64 architecture machine from Intel, then
1005 	 * syscall -doesn't- work in compatibility mode, only sysenter does.
1006 	 *
1007 	 * Sigh.
1008 	 */
1009 	if (!cpuid_syscall32_insn(NULL))
1010 		auxv_hwcap32 &= ~AV_386_AMD_SYSC;
1011 
1012 	/*
1013 	 * 32-bit processes can -always- use the lahf/sahf instructions
1014 	 */
1015 	auxv_hwcap32 |= AV_386_AHF;
1016 
1017 	/*
1018 	 * 32-bit processes can -never- use fsgsbase instructions.
1019 	 */
1020 	auxv_hwcap32_2 &= ~AV_386_2_FSGSBASE;
1021 #endif
1022 
1023 	if (auxv_hwcap32_include || auxv_hwcap32_exclude ||
1024 	    auxv_hwcap32_include_2 || auxv_hwcap32_exclude_2) {
1025 		/*
1026 		 * See the block comment in the cmn_err() of auxv_hwcap, above.
1027 		 */
1028 		char *fmt = "?32-bit user ABI extensions: %b\n";
1029 		cmn_err(CE_CONT, fmt, auxv_hwcap32, FMT_AV_386);
1030 		fmt = "?32-bit user ABI extensions (word 2): %b\n";
1031 		cmn_err(CE_CONT, fmt, auxv_hwcap32_2, FMT_AV_386_2);
1032 	}
1033 #endif
1034 }
1035 
1036 /*
1037  *	sync_icache() - this is called
1038  *	in proc/fs/prusrio.c. x86 has an unified cache and therefore
1039  *	this is a nop.
1040  */
1041 /* ARGSUSED */
1042 void
sync_icache(caddr_t addr,uint_t len)1043 sync_icache(caddr_t addr, uint_t len)
1044 {
1045 	/* Do nothing for now */
1046 }
1047 
1048 /*ARGSUSED*/
1049 void
sync_data_memory(caddr_t va,size_t len)1050 sync_data_memory(caddr_t va, size_t len)
1051 {
1052 	/* Not implemented for this platform */
1053 }
1054 
1055 int
__ipltospl(int ipl)1056 __ipltospl(int ipl)
1057 {
1058 	return (ipltospl(ipl));
1059 }
1060 
1061 /*
1062  * The panic code invokes panic_saveregs() to record the contents of a
1063  * regs structure into the specified panic_data structure for debuggers.
1064  */
1065 void
panic_saveregs(panic_data_t * pdp,struct regs * rp)1066 panic_saveregs(panic_data_t *pdp, struct regs *rp)
1067 {
1068 	panic_nv_t *pnv = PANICNVGET(pdp);
1069 
1070 	struct cregs	creg;
1071 
1072 	getcregs(&creg);
1073 
1074 #if defined(__amd64)
1075 	PANICNVADD(pnv, "rdi", rp->r_rdi);
1076 	PANICNVADD(pnv, "rsi", rp->r_rsi);
1077 	PANICNVADD(pnv, "rdx", rp->r_rdx);
1078 	PANICNVADD(pnv, "rcx", rp->r_rcx);
1079 	PANICNVADD(pnv, "r8", rp->r_r8);
1080 	PANICNVADD(pnv, "r9", rp->r_r9);
1081 	PANICNVADD(pnv, "rax", rp->r_rax);
1082 	PANICNVADD(pnv, "rbx", rp->r_rbx);
1083 	PANICNVADD(pnv, "rbp", rp->r_rbp);
1084 	PANICNVADD(pnv, "r10", rp->r_r10);
1085 	PANICNVADD(pnv, "r11", rp->r_r11);
1086 	PANICNVADD(pnv, "r12", rp->r_r12);
1087 	PANICNVADD(pnv, "r13", rp->r_r13);
1088 	PANICNVADD(pnv, "r14", rp->r_r14);
1089 	PANICNVADD(pnv, "r15", rp->r_r15);
1090 	PANICNVADD(pnv, "fsbase", rdmsr(MSR_AMD_FSBASE));
1091 	PANICNVADD(pnv, "gsbase", rdmsr(MSR_AMD_GSBASE));
1092 	PANICNVADD(pnv, "ds", rp->r_ds);
1093 	PANICNVADD(pnv, "es", rp->r_es);
1094 	PANICNVADD(pnv, "fs", rp->r_fs);
1095 	PANICNVADD(pnv, "gs", rp->r_gs);
1096 	PANICNVADD(pnv, "trapno", rp->r_trapno);
1097 	PANICNVADD(pnv, "err", rp->r_err);
1098 	PANICNVADD(pnv, "rip", rp->r_rip);
1099 	PANICNVADD(pnv, "cs", rp->r_cs);
1100 	PANICNVADD(pnv, "rflags", rp->r_rfl);
1101 	PANICNVADD(pnv, "rsp", rp->r_rsp);
1102 	PANICNVADD(pnv, "ss", rp->r_ss);
1103 	PANICNVADD(pnv, "gdt_hi", (uint64_t)(creg.cr_gdt._l[3]));
1104 	PANICNVADD(pnv, "gdt_lo", (uint64_t)(creg.cr_gdt._l[0]));
1105 	PANICNVADD(pnv, "idt_hi", (uint64_t)(creg.cr_idt._l[3]));
1106 	PANICNVADD(pnv, "idt_lo", (uint64_t)(creg.cr_idt._l[0]));
1107 #elif defined(__i386)
1108 	PANICNVADD(pnv, "gs", (uint32_t)rp->r_gs);
1109 	PANICNVADD(pnv, "fs", (uint32_t)rp->r_fs);
1110 	PANICNVADD(pnv, "es", (uint32_t)rp->r_es);
1111 	PANICNVADD(pnv, "ds", (uint32_t)rp->r_ds);
1112 	PANICNVADD(pnv, "edi", (uint32_t)rp->r_edi);
1113 	PANICNVADD(pnv, "esi", (uint32_t)rp->r_esi);
1114 	PANICNVADD(pnv, "ebp", (uint32_t)rp->r_ebp);
1115 	PANICNVADD(pnv, "esp", (uint32_t)rp->r_esp);
1116 	PANICNVADD(pnv, "ebx", (uint32_t)rp->r_ebx);
1117 	PANICNVADD(pnv, "edx", (uint32_t)rp->r_edx);
1118 	PANICNVADD(pnv, "ecx", (uint32_t)rp->r_ecx);
1119 	PANICNVADD(pnv, "eax", (uint32_t)rp->r_eax);
1120 	PANICNVADD(pnv, "trapno", (uint32_t)rp->r_trapno);
1121 	PANICNVADD(pnv, "err", (uint32_t)rp->r_err);
1122 	PANICNVADD(pnv, "eip", (uint32_t)rp->r_eip);
1123 	PANICNVADD(pnv, "cs", (uint32_t)rp->r_cs);
1124 	PANICNVADD(pnv, "eflags", (uint32_t)rp->r_efl);
1125 	PANICNVADD(pnv, "uesp", (uint32_t)rp->r_uesp);
1126 	PANICNVADD(pnv, "ss", (uint32_t)rp->r_ss);
1127 	PANICNVADD(pnv, "gdt", creg.cr_gdt);
1128 	PANICNVADD(pnv, "idt", creg.cr_idt);
1129 #endif	/* __i386 */
1130 
1131 	PANICNVADD(pnv, "ldt", creg.cr_ldt);
1132 	PANICNVADD(pnv, "task", creg.cr_task);
1133 	PANICNVADD(pnv, "cr0", creg.cr_cr0);
1134 	PANICNVADD(pnv, "cr2", creg.cr_cr2);
1135 	PANICNVADD(pnv, "cr3", creg.cr_cr3);
1136 	if (creg.cr_cr4)
1137 		PANICNVADD(pnv, "cr4", creg.cr_cr4);
1138 
1139 	PANICNVSET(pdp, pnv);
1140 }
1141 
1142 #define	TR_ARG_MAX 6	/* Max args to print, same as SPARC */
1143 
1144 #if !defined(__amd64)
1145 
1146 /*
1147  * Given a return address (%eip), determine the likely number of arguments
1148  * that were pushed on the stack prior to its execution.  We do this by
1149  * expecting that a typical call sequence consists of pushing arguments on
1150  * the stack, executing a call instruction, and then performing an add
1151  * on %esp to restore it to the value prior to pushing the arguments for
1152  * the call.  We attempt to detect such an add, and divide the addend
1153  * by the size of a word to determine the number of pushed arguments.
1154  *
1155  * If we do not find such an add, we punt and return TR_ARG_MAX. It is not
1156  * possible to reliably determine if a function took no arguments (i.e. was
1157  * void) because assembler routines do not reliably perform an add on %esp
1158  * immediately upon returning (eg. _sys_call()), so returning TR_ARG_MAX is
1159  * safer than returning 0.
1160  */
1161 static ulong_t
argcount(uintptr_t eip)1162 argcount(uintptr_t eip)
1163 {
1164 	const uint8_t *ins = (const uint8_t *)eip;
1165 	ulong_t n;
1166 
1167 	enum {
1168 		M_MODRM_ESP = 0xc4,	/* Mod/RM byte indicates %esp */
1169 		M_ADD_IMM32 = 0x81,	/* ADD imm32 to r/m32 */
1170 		M_ADD_IMM8  = 0x83	/* ADD imm8 to r/m32 */
1171 	};
1172 
1173 	if (eip < KERNELBASE || ins[1] != M_MODRM_ESP)
1174 		return (TR_ARG_MAX);
1175 
1176 	switch (ins[0]) {
1177 	case M_ADD_IMM32:
1178 		n = ins[2] + (ins[3] << 8) + (ins[4] << 16) + (ins[5] << 24);
1179 		break;
1180 
1181 	case M_ADD_IMM8:
1182 		n = ins[2];
1183 		break;
1184 
1185 	default:
1186 		return (TR_ARG_MAX);
1187 	}
1188 
1189 	n /= sizeof (long);
1190 	return (MIN(n, TR_ARG_MAX));
1191 }
1192 
1193 #endif	/* !__amd64 */
1194 
1195 /*
1196  * Print a stack backtrace using the specified frame pointer.  We delay two
1197  * seconds before continuing, unless this is the panic traceback.
1198  * If we are in the process of panicking, we also attempt to write the
1199  * stack backtrace to a staticly assigned buffer, to allow the panic
1200  * code to find it and write it in to uncompressed pages within the
1201  * system crash dump.
1202  * Note that the frame for the starting stack pointer value is omitted because
1203  * the corresponding %eip is not known.
1204  */
1205 
1206 extern char *dump_stack_scratch;
1207 
1208 #if defined(__amd64)
1209 
1210 void
traceback(caddr_t fpreg)1211 traceback(caddr_t fpreg)
1212 {
1213 	struct frame	*fp = (struct frame *)fpreg;
1214 	struct frame	*nextfp;
1215 	uintptr_t	pc, nextpc;
1216 	ulong_t		off;
1217 	char		args[TR_ARG_MAX * 2 + 16], *sym;
1218 	uint_t	  offset = 0;
1219 	uint_t	  next_offset = 0;
1220 	char	    stack_buffer[1024];
1221 
1222 	if (!panicstr)
1223 		printf("traceback: %%fp = %p\n", (void *)fp);
1224 
1225 	if (panicstr && !dump_stack_scratch) {
1226 		printf("Warning - stack not written to the dump buffer\n");
1227 	}
1228 
1229 	fp = (struct frame *)plat_traceback(fpreg);
1230 	if ((uintptr_t)fp < KERNELBASE)
1231 		goto out;
1232 
1233 	pc = fp->fr_savpc;
1234 	fp = (struct frame *)fp->fr_savfp;
1235 
1236 	while ((uintptr_t)fp >= KERNELBASE) {
1237 		/*
1238 		 * XX64 Until port is complete tolerate 8-byte aligned
1239 		 * frame pointers but flag with a warning so they can
1240 		 * be fixed.
1241 		 */
1242 		if (((uintptr_t)fp & (STACK_ALIGN - 1)) != 0) {
1243 			if (((uintptr_t)fp & (8 - 1)) == 0) {
1244 				printf("  >> warning! 8-byte"
1245 				    " aligned %%fp = %p\n", (void *)fp);
1246 			} else {
1247 				printf(
1248 				    "  >> mis-aligned %%fp = %p\n", (void *)fp);
1249 				break;
1250 			}
1251 		}
1252 
1253 		args[0] = '\0';
1254 		nextpc = (uintptr_t)fp->fr_savpc;
1255 		nextfp = (struct frame *)fp->fr_savfp;
1256 		if ((sym = kobj_getsymname(pc, &off)) != NULL) {
1257 			printf("%016lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
1258 			    mod_containing_pc((caddr_t)pc), sym, off, args);
1259 			(void) snprintf(stack_buffer, sizeof (stack_buffer),
1260 			    "%s:%s+%lx (%s) | ",
1261 			    mod_containing_pc((caddr_t)pc), sym, off, args);
1262 		} else {
1263 			printf("%016lx %lx (%s)\n",
1264 			    (uintptr_t)fp, pc, args);
1265 			(void) snprintf(stack_buffer, sizeof (stack_buffer),
1266 			    "%lx (%s) | ", pc, args);
1267 		}
1268 
1269 		if (panicstr && dump_stack_scratch) {
1270 			next_offset = offset + strlen(stack_buffer);
1271 			if (next_offset < STACK_BUF_SIZE) {
1272 				bcopy(stack_buffer, dump_stack_scratch + offset,
1273 				    strlen(stack_buffer));
1274 				offset = next_offset;
1275 			} else {
1276 				/*
1277 				 * In attempting to save the panic stack
1278 				 * to the dumpbuf we have overflowed that area.
1279 				 * Print a warning and continue to printf the
1280 				 * stack to the msgbuf
1281 				 */
1282 				printf("Warning: stack in the dump buffer"
1283 				    " may be incomplete\n");
1284 				offset = next_offset;
1285 			}
1286 		}
1287 
1288 		pc = nextpc;
1289 		fp = nextfp;
1290 	}
1291 out:
1292 	if (!panicstr) {
1293 		printf("end of traceback\n");
1294 		DELAY(2 * MICROSEC);
1295 	} else if (dump_stack_scratch) {
1296 		dump_stack_scratch[offset] = '\0';
1297 	}
1298 }
1299 
1300 #elif defined(__i386)
1301 
1302 void
traceback(caddr_t fpreg)1303 traceback(caddr_t fpreg)
1304 {
1305 	struct frame *fp = (struct frame *)fpreg;
1306 	struct frame *nextfp, *minfp, *stacktop;
1307 	uintptr_t pc, nextpc;
1308 	uint_t	  offset = 0;
1309 	uint_t	  next_offset = 0;
1310 	char	    stack_buffer[1024];
1311 
1312 	cpu_t *cpu;
1313 
1314 	/*
1315 	 * args[] holds TR_ARG_MAX hex long args, plus ", " or '\0'.
1316 	 */
1317 	char args[TR_ARG_MAX * 2 + 8], *p;
1318 
1319 	int on_intr;
1320 	ulong_t off;
1321 	char *sym;
1322 
1323 	if (!panicstr)
1324 		printf("traceback: %%fp = %p\n", (void *)fp);
1325 
1326 	if (panicstr && !dump_stack_scratch) {
1327 		printf("Warning - stack not written to the dumpbuf\n");
1328 	}
1329 
1330 	/*
1331 	 * If we are panicking, all high-level interrupt information in
1332 	 * CPU was overwritten.  panic_cpu has the correct values.
1333 	 */
1334 	kpreempt_disable();			/* prevent migration */
1335 
1336 	cpu = (panicstr && CPU->cpu_id == panic_cpu.cpu_id)? &panic_cpu : CPU;
1337 
1338 	if ((on_intr = CPU_ON_INTR(cpu)) != 0)
1339 		stacktop = (struct frame *)(cpu->cpu_intr_stack + SA(MINFRAME));
1340 	else
1341 		stacktop = (struct frame *)curthread->t_stk;
1342 
1343 	kpreempt_enable();
1344 
1345 	fp = (struct frame *)plat_traceback(fpreg);
1346 	if ((uintptr_t)fp < KERNELBASE)
1347 		goto out;
1348 
1349 	minfp = fp;	/* Baseline minimum frame pointer */
1350 	pc = fp->fr_savpc;
1351 	fp = (struct frame *)fp->fr_savfp;
1352 
1353 	while ((uintptr_t)fp >= KERNELBASE) {
1354 		ulong_t argc;
1355 		long *argv;
1356 
1357 		if (fp <= minfp || fp >= stacktop) {
1358 			if (on_intr) {
1359 				/*
1360 				 * Hop from interrupt stack to thread stack.
1361 				 */
1362 				stacktop = (struct frame *)curthread->t_stk;
1363 				minfp = (struct frame *)curthread->t_stkbase;
1364 				on_intr = 0;
1365 				continue;
1366 			}
1367 			break; /* we're outside of the expected range */
1368 		}
1369 
1370 		if ((uintptr_t)fp & (STACK_ALIGN - 1)) {
1371 			printf("  >> mis-aligned %%fp = %p\n", (void *)fp);
1372 			break;
1373 		}
1374 
1375 		nextpc = fp->fr_savpc;
1376 		nextfp = (struct frame *)fp->fr_savfp;
1377 		argc = argcount(nextpc);
1378 		argv = (long *)((char *)fp + sizeof (struct frame));
1379 
1380 		args[0] = '\0';
1381 		p = args;
1382 		while (argc-- > 0 && argv < (long *)stacktop) {
1383 			p += snprintf(p, args + sizeof (args) - p,
1384 			    "%s%lx", (p == args) ? "" : ", ", *argv++);
1385 		}
1386 
1387 		if ((sym = kobj_getsymname(pc, &off)) != NULL) {
1388 			printf("%08lx %s:%s+%lx (%s)\n", (uintptr_t)fp,
1389 			    mod_containing_pc((caddr_t)pc), sym, off, args);
1390 			(void) snprintf(stack_buffer, sizeof (stack_buffer),
1391 			    "%s:%s+%lx (%s) | ",
1392 			    mod_containing_pc((caddr_t)pc), sym, off, args);
1393 
1394 		} else {
1395 			printf("%08lx %lx (%s)\n",
1396 			    (uintptr_t)fp, pc, args);
1397 			(void) snprintf(stack_buffer, sizeof (stack_buffer),
1398 			    "%lx (%s) | ", pc, args);
1399 
1400 		}
1401 
1402 		if (panicstr && dump_stack_scratch) {
1403 			next_offset = offset + strlen(stack_buffer);
1404 			if (next_offset < STACK_BUF_SIZE) {
1405 				bcopy(stack_buffer, dump_stack_scratch + offset,
1406 				    strlen(stack_buffer));
1407 				offset = next_offset;
1408 			} else {
1409 				/*
1410 				 * In attempting to save the panic stack
1411 				 * to the dumpbuf we have overflowed that area.
1412 				 * Print a warning and continue to printf the
1413 				 * stack to the msgbuf
1414 				 */
1415 				printf("Warning: stack in the dumpbuf"
1416 				    " may be incomplete\n");
1417 				offset = next_offset;
1418 			}
1419 		}
1420 
1421 		minfp = fp;
1422 		pc = nextpc;
1423 		fp = nextfp;
1424 	}
1425 out:
1426 	if (!panicstr) {
1427 		printf("end of traceback\n");
1428 		DELAY(2 * MICROSEC);
1429 	} else if (dump_stack_scratch) {
1430 		dump_stack_scratch[offset] = '\0';
1431 	}
1432 
1433 }
1434 
1435 #endif	/* __i386 */
1436 
1437 /*
1438  * Generate a stack backtrace from a saved register set.
1439  */
1440 void
traceregs(struct regs * rp)1441 traceregs(struct regs *rp)
1442 {
1443 	traceback((caddr_t)rp->r_fp);
1444 }
1445 
1446 void
exec_set_sp(size_t stksize)1447 exec_set_sp(size_t stksize)
1448 {
1449 	klwp_t *lwp = ttolwp(curthread);
1450 
1451 	lwptoregs(lwp)->r_sp = (uintptr_t)curproc->p_usrstack - stksize;
1452 }
1453 
1454 hrtime_t
gethrtime_waitfree(void)1455 gethrtime_waitfree(void)
1456 {
1457 	return (dtrace_gethrtime());
1458 }
1459 
1460 hrtime_t
gethrtime(void)1461 gethrtime(void)
1462 {
1463 	return (gethrtimef());
1464 }
1465 
1466 hrtime_t
gethrtime_unscaled(void)1467 gethrtime_unscaled(void)
1468 {
1469 	return (gethrtimeunscaledf());
1470 }
1471 
1472 void
scalehrtime(hrtime_t * hrt)1473 scalehrtime(hrtime_t *hrt)
1474 {
1475 	scalehrtimef(hrt);
1476 }
1477 
1478 uint64_t
unscalehrtime(hrtime_t nsecs)1479 unscalehrtime(hrtime_t nsecs)
1480 {
1481 	return (unscalehrtimef(nsecs));
1482 }
1483 
1484 void
gethrestime(timespec_t * tp)1485 gethrestime(timespec_t *tp)
1486 {
1487 	gethrestimef(tp);
1488 }
1489 
1490 #if defined(__amd64)
1491 /*
1492  * Part of the implementation of hres_tick(); this routine is
1493  * easier in C than assembler .. called with the hres_lock held.
1494  *
1495  * XX64	Many of these timekeeping variables need to be extern'ed in a header
1496  */
1497 
1498 #include <sys/time.h>
1499 #include <sys/machlock.h>
1500 
1501 extern int one_sec;
1502 extern int max_hres_adj;
1503 
1504 void
__adj_hrestime(void)1505 __adj_hrestime(void)
1506 {
1507 	long long adj;
1508 
1509 	if (hrestime_adj == 0)
1510 		adj = 0;
1511 	else if (hrestime_adj > 0) {
1512 		if (hrestime_adj < max_hres_adj)
1513 			adj = hrestime_adj;
1514 		else
1515 			adj = max_hres_adj;
1516 	} else {
1517 		if (hrestime_adj < -max_hres_adj)
1518 			adj = -max_hres_adj;
1519 		else
1520 			adj = hrestime_adj;
1521 	}
1522 
1523 	timedelta -= adj;
1524 	hrestime_adj = timedelta;
1525 	hrestime.tv_nsec += adj;
1526 
1527 	while (hrestime.tv_nsec >= NANOSEC) {
1528 		one_sec++;
1529 		hrestime.tv_sec++;
1530 		hrestime.tv_nsec -= NANOSEC;
1531 	}
1532 }
1533 #endif
1534 
1535 /*
1536  * Wrapper functions to maintain backwards compability
1537  */
1538 int
xcopyin(const void * uaddr,void * kaddr,size_t count)1539 xcopyin(const void *uaddr, void *kaddr, size_t count)
1540 {
1541 	return (xcopyin_nta(uaddr, kaddr, count, UIO_COPY_CACHED));
1542 }
1543 
1544 int
xcopyout(const void * kaddr,void * uaddr,size_t count)1545 xcopyout(const void *kaddr, void *uaddr, size_t count)
1546 {
1547 	return (xcopyout_nta(kaddr, uaddr, count, UIO_COPY_CACHED));
1548 }
1549