1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21/* 22 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2019 Joyent, Inc. 24 * Copyright (c) 2016 by Delphix. All rights reserved. 25 * Copyright 2024 MNX Cloud, Inc. 26 */ 27 28#include <sys/asm_linkage.h> 29#include <sys/asm_misc.h> 30#include <sys/regset.h> 31#include <sys/privregs.h> 32#include <sys/psw.h> 33#include <sys/machbrand.h> 34 35#include <sys/segments.h> 36#include <sys/pcb.h> 37#include <sys/trap.h> 38#include <sys/ftrace.h> 39#include <sys/traptrace.h> 40#include <sys/clock.h> 41#include <sys/model.h> 42#include <sys/panic.h> 43 44#if defined(__xpv) 45#include <sys/hypervisor.h> 46#endif 47 48#include "assym.h" 49 50/* 51 * We implement five flavours of system call entry points 52 * 53 * - syscall/sysretq (amd64 generic) 54 * - syscall/sysretl (i386 plus SYSC bit) 55 * - sysenter/sysexit (i386 plus SEP bit) 56 * - int/iret (i386 generic) 57 * - lcall/iret (i386 generic) 58 * 59 * The current libc included in Solaris uses int/iret as the base unoptimized 60 * kernel entry method. Older libc implementations and legacy binaries may use 61 * the lcall call gate, so it must continue to be supported. 62 * 63 * System calls that use an lcall call gate are processed in trap() via a 64 * segment-not-present trap, i.e. lcalls are extremely slow(!). 65 * 66 * The basic pattern used in the 32-bit SYSC handler at this point in time is 67 * to have the bare minimum of assembler, and get to the C handlers as 68 * quickly as possible. 69 * 70 * The 64-bit handler is much closer to the sparcv9 handler; that's 71 * because of passing arguments in registers. The 32-bit world still 72 * passes arguments on the stack -- that makes that handler substantially 73 * more complex. 74 * 75 * The two handlers share a few code fragments which are broken 76 * out into preprocessor macros below. 77 * 78 * XX64 come back and speed all this up later. The 32-bit stuff looks 79 * especially easy to speed up the argument copying part .. 80 * 81 * 82 * Notes about segment register usage (c.f. the 32-bit kernel) 83 * 84 * In the 32-bit kernel, segment registers are dutifully saved and 85 * restored on all mode transitions because the kernel uses them directly. 86 * When the processor is running in 64-bit mode, segment registers are 87 * largely ignored. 88 * 89 * %cs and %ss 90 * controlled by the hardware mechanisms that make mode transitions 91 * 92 * The remaining segment registers have to either be pointing at a valid 93 * descriptor i.e. with the 'present' bit set, or they can NULL descriptors 94 * 95 * %ds and %es 96 * always ignored 97 * 98 * %fs and %gs 99 * fsbase and gsbase are used to control the place they really point at. 100 * The kernel only depends on %gs, and controls its own gsbase via swapgs 101 * 102 * Note that loading segment registers is still costly because the GDT 103 * lookup still happens (this is because the hardware can't know that we're 104 * not setting up these segment registers for a 32-bit program). Thus we 105 * avoid doing this in the syscall path, and defer them to lwp context switch 106 * handlers, so the register values remain virtualized to the lwp. 107 */ 108 109#if defined(SYSCALLTRACE) 110#define ORL_SYSCALLTRACE(r32) \ 111 orl syscalltrace(%rip), r32 112#else 113#define ORL_SYSCALLTRACE(r32) 114#endif 115 116/* 117 * In the 32-bit kernel, we do absolutely nothing before getting into the 118 * brand callback checks. In 64-bit land, we do swapgs and then come here. 119 * We assume that the %rsp- and %r15-stashing fields in the CPU structure 120 * are still unused. 121 * 122 * Check if a brand_mach_ops callback is defined for the specified callback_id 123 * type. If so invoke it with the kernel's %gs value loaded and the following 124 * data on the stack: 125 * 126 * stack: -------------------------------------- 127 * 32 | callback pointer | 128 * | 24 | user (or interrupt) stack pointer | 129 * | 16 | lwp pointer | 130 * v 8 | userland return address | 131 * 0 | callback wrapper return addr | 132 * -------------------------------------- 133 * 134 * Since we're pushing the userland return address onto the kernel stack 135 * we need to get that address without accessing the user's stack (since we 136 * can't trust that data). There are different ways to get the userland 137 * return address depending on how the syscall trap was made: 138 * 139 * a) For sys_syscall and sys_syscall32 the return address is in %rcx. 140 * b) For sys_sysenter the return address is in %rdx. 141 * c) For sys_int80 and sys_syscall_int (int91), upon entry into the macro, 142 * the stack pointer points at the state saved when we took the interrupt: 143 * ------------------------ 144 * | | user's %ss | 145 * | | user's %esp | 146 * | | EFLAGS register | 147 * v | user's %cs | 148 * | user's %eip | 149 * ------------------------ 150 * 151 * The 2nd parameter to the BRAND_CALLBACK macro is either the 152 * BRAND_URET_FROM_REG or BRAND_URET_FROM_INTR_STACK macro. These macros are 153 * used to generate the proper code to get the userland return address for 154 * each syscall entry point. 155 * 156 * The interface to the brand callbacks on the 64-bit kernel assumes %r15 157 * is available as a scratch register within the callback. If the callback 158 * returns within the kernel then this macro will restore %r15. If the 159 * callback is going to return directly to userland then it should restore 160 * %r15 before returning to userland. 161 */ 162#define BRAND_URET_FROM_REG(rip_reg) \ 163 pushq rip_reg /* push the return address */ 164 165/* 166 * The interrupt stack pointer we saved on entry to the BRAND_CALLBACK macro 167 * is currently pointing at the user return address (%eip). 168 */ 169#define BRAND_URET_FROM_INTR_STACK() \ 170 movq %gs:CPU_RTMP_RSP, %r15 /* grab the intr. stack pointer */ ;\ 171 pushq (%r15) /* push the return address */ 172 173#define BRAND_CALLBACK(callback_id, push_userland_ret) \ 174 movq %rsp, %gs:CPU_RTMP_RSP /* save the stack pointer */ ;\ 175 movq %r15, %gs:CPU_RTMP_R15 /* save %r15 */ ;\ 176 movq %gs:CPU_THREAD, %r15 /* load the thread pointer */ ;\ 177 movq T_STACK(%r15), %rsp /* switch to the kernel stack */ ;\ 178 subq $16, %rsp /* save space for 2 pointers */ ;\ 179 pushq %r14 /* save %r14 */ ;\ 180 movq %gs:CPU_RTMP_RSP, %r14 ;\ 181 movq %r14, 8(%rsp) /* stash the user stack pointer */ ;\ 182 popq %r14 /* restore %r14 */ ;\ 183 movq T_LWP(%r15), %r15 /* load the lwp pointer */ ;\ 184 pushq %r15 /* push the lwp pointer */ ;\ 185 movq LWP_PROCP(%r15), %r15 /* load the proc pointer */ ;\ 186 movq P_BRAND(%r15), %r15 /* load the brand pointer */ ;\ 187 movq B_MACHOPS(%r15), %r15 /* load the machops pointer */ ;\ 188 movq _CONST(_MUL(callback_id, CPTRSIZE))(%r15), %r15 ;\ 189 cmpq $0, %r15 ;\ 190 je 1f ;\ 191 movq %r15, 16(%rsp) /* save the callback pointer */ ;\ 192 push_userland_ret /* push the return address */ ;\ 193 movq 24(%rsp), %r15 /* load callback pointer */ ;\ 194 INDIRECT_CALL_REG(r15) /* call callback */ ;\ 1951: movq %gs:CPU_RTMP_R15, %r15 /* restore %r15 */ ;\ 196 movq %gs:CPU_RTMP_RSP, %rsp /* restore the stack pointer */ 197 198#define MSTATE_TRANSITION(from, to) \ 199 movl $from, %edi; \ 200 movl $to, %esi; \ 201 call syscall_mstate 202 203/* 204 * Check to see if a simple (direct) return is possible i.e. 205 * 206 * if (t->t_post_sys_ast | syscalltrace | 207 * lwp->lwp_pcb.pcb_rupdate == 1) 208 * do full version ; 209 * 210 * Preconditions: 211 * - t is curthread 212 * Postconditions: 213 * - condition code NE is set if post-sys is too complex 214 * - rtmp is zeroed if it isn't (we rely on this!) 215 * - ltmp is smashed 216 */ 217#define CHECK_POSTSYS_NE(t, ltmp, rtmp) \ 218 movq T_LWP(t), ltmp; \ 219 movzbl PCB_RUPDATE(ltmp), rtmp; \ 220 ORL_SYSCALLTRACE(rtmp); \ 221 orl T_POST_SYS_AST(t), rtmp; \ 222 cmpl $0, rtmp 223 224/* 225 * Fix up the lwp, thread, and eflags for a successful return 226 * 227 * Preconditions: 228 * - zwreg contains zero 229 */ 230#define SIMPLE_SYSCALL_POSTSYS(t, lwp, zwreg) \ 231 movb $LWP_USER, LWP_STATE(lwp); \ 232 movw zwreg, T_SYSNUM(t); \ 233 andb $_CONST(0xffff - PS_C), REGOFF_RFL(%rsp) 234 235/* 236 * ASSERT(lwptoregs(lwp) == rp); 237 * 238 * This may seem obvious, but very odd things happen if this 239 * assertion is false 240 * 241 * Preconditions: 242 * (%rsp is ready for normal call sequence) 243 * Postconditions (if assertion is true): 244 * %r11 is smashed 245 * 246 * ASSERT(rp->r_cs == descnum) 247 * 248 * The code selector is written into the regs structure when the 249 * lwp stack is created. We use this ASSERT to validate that 250 * the regs structure really matches how we came in. 251 * 252 * Preconditions: 253 * (%rsp is ready for normal call sequence) 254 * Postconditions (if assertion is true): 255 * -none- 256 * 257 * ASSERT(lwp->lwp_pcb.pcb_rupdate == 0); 258 * 259 * If this is false, it meant that we returned to userland without 260 * updating the segment registers as we were supposed to. 261 * 262 * Note that we must ensure no interrupts or other traps intervene 263 * between entering privileged mode and performing the assertion, 264 * otherwise we may perform a context switch on the thread, which 265 * will end up setting pcb_rupdate to 1 again. 266 * 267 * ASSERT(%cr0 & CR0_TS == 0); 268 * Preconditions: 269 * (%rsp is ready for normal call sequence) 270 * Postconditions (if assertion is true): 271 * (specified register is clobbered) 272 * 273 * Check to make sure that we are returning to user land and that CR0.TS 274 * is not set. This is required as part of the eager FPU (see 275 * uts/intel/os/fpu.c for more information). 276 */ 277 278#if defined(DEBUG) 279 280__lwptoregs_msg: 281 .string "syscall_asm_amd64.s:%d lwptoregs(%p) [%p] != rp [%p]" 282 283__codesel_msg: 284 .string "syscall_asm_amd64.s:%d rp->r_cs [%ld] != %ld" 285 286__no_rupdate_msg: 287 .string "syscall_asm_amd64.s:%d lwp %p, pcb_rupdate != 0" 288 289__bad_ts_msg: 290 .string "syscall_asm_amd64.s:%d CR0.TS set on user return" 291 292#define ASSERT_LWPTOREGS(lwp, rp) \ 293 movq LWP_REGS(lwp), %r11; \ 294 cmpq rp, %r11; \ 295 je 7f; \ 296 leaq __lwptoregs_msg(%rip), %rdi; \ 297 movl $__LINE__, %esi; \ 298 movq lwp, %rdx; \ 299 movq %r11, %rcx; \ 300 movq rp, %r8; \ 301 xorl %eax, %eax; \ 302 call panic; \ 3037: 304 305#define ASSERT_NO_RUPDATE_PENDING(lwp) \ 306 testb $0x1, PCB_RUPDATE(lwp); \ 307 je 8f; \ 308 movq lwp, %rdx; \ 309 leaq __no_rupdate_msg(%rip), %rdi; \ 310 movl $__LINE__, %esi; \ 311 xorl %eax, %eax; \ 312 call panic; \ 3138: 314 315#define ASSERT_CR0TS_ZERO(reg) \ 316 movq %cr0, reg; \ 317 testq $CR0_TS, reg; \ 318 jz 9f; \ 319 leaq __bad_ts_msg(%rip), %rdi; \ 320 movl $__LINE__, %esi; \ 321 xorl %eax, %eax; \ 322 call panic; \ 3239: 324 325#else 326#define ASSERT_LWPTOREGS(lwp, rp) 327#define ASSERT_NO_RUPDATE_PENDING(lwp) 328#define ASSERT_CR0TS_ZERO(reg) 329#endif 330 331/* 332 * Do the traptrace thing and restore any registers we used 333 * in situ. Assumes that %rsp is pointing at the base of 334 * the struct regs, obviously .. 335 */ 336#ifdef TRAPTRACE 337#define SYSCALL_TRAPTRACE(ttype) \ 338 TRACE_PTR(%rdi, %rbx, %ebx, %rcx, ttype); \ 339 TRACE_REGS(%rdi, %rsp, %rbx, %rcx); \ 340 TRACE_STAMP(%rdi); /* rdtsc clobbers %eax, %edx */ \ 341 movq REGOFF_RAX(%rsp), %rax; \ 342 movq REGOFF_RBX(%rsp), %rbx; \ 343 movq REGOFF_RCX(%rsp), %rcx; \ 344 movq REGOFF_RDX(%rsp), %rdx; \ 345 movl %eax, TTR_SYSNUM(%rdi); \ 346 movq REGOFF_RDI(%rsp), %rdi 347 348#define SYSCALL_TRAPTRACE32(ttype) \ 349 SYSCALL_TRAPTRACE(ttype); \ 350 /* paranoia: clean the top 32-bits of the registers */ \ 351 orl %eax, %eax; \ 352 orl %ebx, %ebx; \ 353 orl %ecx, %ecx; \ 354 orl %edx, %edx; \ 355 orl %edi, %edi 356#else /* TRAPTRACE */ 357#define SYSCALL_TRAPTRACE(ttype) 358#define SYSCALL_TRAPTRACE32(ttype) 359#endif /* TRAPTRACE */ 360 361/* 362 * The 64-bit libc syscall wrapper does this: 363 * 364 * fn(<args>) 365 * { 366 * movq %rcx, %r10 -- because syscall smashes %rcx 367 * movl $CODE, %eax 368 * syscall 369 * <error processing> 370 * } 371 * 372 * Thus when we come into the kernel: 373 * 374 * %rdi, %rsi, %rdx, %r10, %r8, %r9 contain first six args 375 * %rax is the syscall number 376 * %r12-%r15 contain caller state 377 * 378 * The syscall instruction arranges that: 379 * 380 * %rcx contains the return %rip 381 * %r11d contains bottom 32-bits of %rflags 382 * %rflags is masked (as determined by the SFMASK msr) 383 * %cs is set to UCS_SEL (as determined by the STAR msr) 384 * %ss is set to UDS_SEL (as determined by the STAR msr) 385 * %rip is set to sys_syscall (as determined by the LSTAR msr) 386 * 387 * Or in other words, we have no registers available at all. 388 * Only swapgs can save us! 389 * 390 * Under the hypervisor, the swapgs has happened already. However, the 391 * state of the world is very different from that we're familiar with. 392 * 393 * In particular, we have a stack structure like that for interrupt 394 * gates, except that the %cs and %ss registers are modified for reasons 395 * that are not entirely clear. Critically, the %rcx/%r11 values do 396 * *not* reflect the usage of those registers under a 'real' syscall[1]; 397 * the stack, therefore, looks like this: 398 * 399 * 0x0(rsp) potentially junk %rcx 400 * 0x8(rsp) potentially junk %r11 401 * 0x10(rsp) user %rip 402 * 0x18(rsp) modified %cs 403 * 0x20(rsp) user %rflags 404 * 0x28(rsp) user %rsp 405 * 0x30(rsp) modified %ss 406 * 407 * 408 * and before continuing on, we must load the %rip into %rcx and the 409 * %rflags into %r11. 410 * 411 * [1] They used to, and we relied on it, but this was broken in 3.1.1. 412 * Sigh. 413 */ 414#if defined(__xpv) 415#define XPV_SYSCALL_PROD \ 416 movq 0x10(%rsp), %rcx; \ 417 movq 0x20(%rsp), %r11; \ 418 movq 0x28(%rsp), %rsp 419#else 420#define XPV_SYSCALL_PROD /* nothing */ 421#endif 422 423 ENTRY_NP2(brand_sys_syscall,_allsyscalls) 424 SWAPGS /* kernel gsbase */ 425 XPV_SYSCALL_PROD 426 BRAND_CALLBACK(BRAND_CB_SYSCALL, BRAND_URET_FROM_REG(%rcx)) 427 jmp noprod_sys_syscall 428 429 ALTENTRY(sys_syscall) 430 SWAPGS /* kernel gsbase */ 431 XPV_SYSCALL_PROD 432 433noprod_sys_syscall: 434 movq %r15, %gs:CPU_RTMP_R15 435 movq %rsp, %gs:CPU_RTMP_RSP 436 437 movq %gs:CPU_THREAD, %r15 438 movq T_STACK(%r15), %rsp /* switch from user to kernel stack */ 439 440 ASSERT_UPCALL_MASK_IS_SET 441 442 movl $UCS_SEL, REGOFF_CS(%rsp) 443 movq %rcx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */ 444 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */ 445 movl $UDS_SEL, REGOFF_SS(%rsp) 446 447 movl %eax, %eax /* wrapper: sysc# -> %eax */ 448 movq %rdi, REGOFF_RDI(%rsp) 449 movq %rsi, REGOFF_RSI(%rsp) 450 movq %rdx, REGOFF_RDX(%rsp) 451 movq %r10, REGOFF_RCX(%rsp) /* wrapper: %rcx -> %r10 */ 452 movq %r10, %rcx /* arg[3] for direct calls */ 453 454 movq %r8, REGOFF_R8(%rsp) 455 movq %r9, REGOFF_R9(%rsp) 456 movq %rax, REGOFF_RAX(%rsp) 457 movq %rbx, REGOFF_RBX(%rsp) 458 459 movq %rbp, REGOFF_RBP(%rsp) 460 movq %r10, REGOFF_R10(%rsp) 461 movq %gs:CPU_RTMP_RSP, %r11 462 movq %r11, REGOFF_RSP(%rsp) 463 movq %r12, REGOFF_R12(%rsp) 464 465 movq %r13, REGOFF_R13(%rsp) 466 movq %r14, REGOFF_R14(%rsp) 467 movq %gs:CPU_RTMP_R15, %r10 468 movq %r10, REGOFF_R15(%rsp) 469 movq $0, REGOFF_SAVFP(%rsp) 470 movq $0, REGOFF_SAVPC(%rsp) 471 472 /* 473 * Copy these registers here in case we end up stopped with 474 * someone (like, say, /proc) messing with our register state. 475 * We don't -restore- them unless we have to in update_sregs. 476 * 477 * Since userland -can't- change fsbase or gsbase directly, 478 * and capturing them involves two serializing instructions, 479 * we don't bother to capture them here. 480 */ 481 xorl %ebx, %ebx 482 movw %ds, %bx 483 movq %rbx, REGOFF_DS(%rsp) 484 movw %es, %bx 485 movq %rbx, REGOFF_ES(%rsp) 486 movw %fs, %bx 487 movq %rbx, REGOFF_FS(%rsp) 488 movw %gs, %bx 489 movq %rbx, REGOFF_GS(%rsp) 490 491 /* 492 * If we're trying to use TRAPTRACE though, I take that back: we're 493 * probably debugging some problem in the SWAPGS logic and want to know 494 * what the incoming gsbase was. 495 * 496 * Since we already did SWAPGS, record the KGSBASE. 497 */ 498#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv) 499 movl $MSR_AMD_KGSBASE, %ecx 500 rdmsr 501 movl %eax, REGOFF_GSBASE(%rsp) 502 movl %edx, REGOFF_GSBASE+4(%rsp) 503#endif 504 505 /* 506 * Machine state saved in the regs structure on the stack 507 * First six args in %rdi, %rsi, %rdx, %rcx, %r8, %r9 508 * %eax is the syscall number 509 * %rsp is the thread's stack, %r15 is curthread 510 * REG_RSP(%rsp) is the user's stack 511 */ 512 513 SYSCALL_TRAPTRACE($TT_SYSC64) 514 515 movq %rsp, %rbp 516 517 movq T_LWP(%r15), %r14 518 ASSERT_NO_RUPDATE_PENDING(%r14) 519 ENABLE_INTR_FLAGS 520 521 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 522 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */ 523 524 ASSERT_LWPTOREGS(%r14, %rsp) 525 526 movb $LWP_SYS, LWP_STATE(%r14) 527 incq LWP_RU_SYSC(%r14) 528 movb $NORMALRETURN, LWP_EOSYS(%r14) 529 530 incq %gs:CPU_STATS_SYS_SYSCALL 531 532 movw %ax, T_SYSNUM(%r15) 533 movzbl T_PRE_SYS(%r15), %ebx 534 ORL_SYSCALLTRACE(%ebx) 535 testl %ebx, %ebx 536 jne _syscall_pre 537 538_syscall_invoke: 539 movq REGOFF_RDI(%rbp), %rdi 540 movq REGOFF_RSI(%rbp), %rsi 541 movq REGOFF_RDX(%rbp), %rdx 542 movq REGOFF_RCX(%rbp), %rcx 543 movq REGOFF_R8(%rbp), %r8 544 movq REGOFF_R9(%rbp), %r9 545 546 cmpl $NSYSCALL, %eax 547 jae _syscall_ill 548 shll $SYSENT_SIZE_SHIFT, %eax 549 leaq sysent(%rax), %rbx 550 551 movq SY_CALLC(%rbx), %rax 552 INDIRECT_CALL_REG(rax) 553 554 movq %rax, %r12 555 movq %rdx, %r13 556 557 /* 558 * If the handler returns two ints, then we need to split the 559 * 64-bit return value into two 32-bit values. 560 */ 561 testw $SE_32RVAL2, SY_FLAGS(%rbx) 562 je 5f 563 movq %r12, %r13 564 shrq $32, %r13 /* upper 32-bits into %edx */ 565 movl %r12d, %r12d /* lower 32-bits into %eax */ 5665: 567 /* 568 * Optimistically assume that there's no post-syscall 569 * work to do. (This is to avoid having to call syscall_mstate() 570 * with interrupts disabled) 571 */ 572 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 573 574 /* 575 * We must protect ourselves from being descheduled here; 576 * If we were, and we ended up on another cpu, or another 577 * lwp got in ahead of us, it could change the segment 578 * registers without us noticing before we return to userland. 579 */ 580 CLI(%r14) 581 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 582 jne _syscall_post 583 584 /* 585 * We need to protect ourselves against non-canonical return values 586 * because Intel doesn't check for them on sysret (AMD does). Canonical 587 * addresses on current amd64 processors only use 48-bits for VAs; an 588 * address is canonical if all upper bits (47-63) are identical. If we 589 * find a non-canonical %rip, we opt to go through the full 590 * _syscall_post path which takes us into an iretq which is not 591 * susceptible to the same problems sysret is. 592 * 593 * We're checking for a canonical address by first doing an arithmetic 594 * shift. This will fill in the remaining bits with the value of bit 63. 595 * If the address were canonical, the register would now have either all 596 * zeroes or all ones in it. Therefore we add one (inducing overflow) 597 * and compare against 1. A canonical address will either be zero or one 598 * at this point, hence the use of ja. 599 * 600 * At this point, r12 and r13 have the return value so we can't use 601 * those registers. 602 */ 603 movq REGOFF_RIP(%rsp), %rcx 604 sarq $47, %rcx 605 incq %rcx 606 cmpq $1, %rcx 607 ja _syscall_post 608 609 610 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 611 612 movq %r12, REGOFF_RAX(%rsp) 613 movq %r13, REGOFF_RDX(%rsp) 614 615 /* 616 * Clobber %r11 as we check CR0.TS. 617 */ 618 ASSERT_CR0TS_ZERO(%r11) 619 620 /* 621 * To get back to userland, we need the return %rip in %rcx and 622 * the return %rfl in %r11d. The sysretq instruction also arranges 623 * to fix up %cs and %ss; everything else is our responsibility. 624 */ 625 movq REGOFF_RDI(%rsp), %rdi 626 movq REGOFF_RSI(%rsp), %rsi 627 movq REGOFF_RDX(%rsp), %rdx 628 /* %rcx used to restore %rip value */ 629 630 movq REGOFF_R8(%rsp), %r8 631 movq REGOFF_R9(%rsp), %r9 632 movq REGOFF_RAX(%rsp), %rax 633 movq REGOFF_RBX(%rsp), %rbx 634 635 movq REGOFF_RBP(%rsp), %rbp 636 movq REGOFF_R10(%rsp), %r10 637 /* %r11 used to restore %rfl value */ 638 movq REGOFF_R12(%rsp), %r12 639 640 movq REGOFF_R13(%rsp), %r13 641 movq REGOFF_R14(%rsp), %r14 642 movq REGOFF_R15(%rsp), %r15 643 644 movq REGOFF_RIP(%rsp), %rcx 645 movl REGOFF_RFL(%rsp), %r11d 646 647 /* 648 * Unlike other cases, because we need to restore the user stack pointer 649 * before exiting the kernel we must clear the microarch state before 650 * getting here. This should be safe because it means that the only 651 * values on the bus after this are based on the user's registers and 652 * potentially the addresses where we stored them. Given the constraints 653 * of sysret, that's how it has to be. 654 */ 655 call x86_md_clear 656 657#if defined(__xpv) 658 addq $REGOFF_RIP, %rsp 659#else 660 movq REGOFF_RSP(%rsp), %rsp 661#endif 662 663 /* 664 * There can be no instructions between the ALTENTRY below and 665 * SYSRET or we could end up breaking brand support. See label usage 666 * in sn1_brand_syscall_callback for an example. 667 */ 668 ASSERT_UPCALL_MASK_IS_SET 669#if defined(__xpv) 670 SYSRETQ 671 ALTENTRY(nopop_sys_syscall_swapgs_sysretq) 672 673 /* 674 * We can only get here after executing a brand syscall 675 * interposition callback handler and simply need to 676 * "sysretq" back to userland. On the hypervisor this 677 * involves the iret hypercall which requires us to construct 678 * just enough of the stack needed for the hypercall. 679 * (rip, cs, rflags, rsp, ss). 680 */ 681 movq %rsp, %gs:CPU_RTMP_RSP /* save user's rsp */ 682 movq %gs:CPU_THREAD, %r11 683 movq T_STACK(%r11), %rsp 684 685 movq %rcx, REGOFF_RIP(%rsp) 686 movl $UCS_SEL, REGOFF_CS(%rsp) 687 movq %gs:CPU_RTMP_RSP, %r11 688 movq %r11, REGOFF_RSP(%rsp) 689 pushfq 690 popq %r11 /* hypercall enables ints */ 691 movq %r11, REGOFF_RFL(%rsp) 692 movl $UDS_SEL, REGOFF_SS(%rsp) 693 addq $REGOFF_RIP, %rsp 694 /* 695 * XXPV: see comment in SYSRETQ definition for future optimization 696 * we could take. 697 */ 698 ASSERT_UPCALL_MASK_IS_SET 699 SYSRETQ 700#else 701 ALTENTRY(nopop_sys_syscall_swapgs_sysretq) 702 jmp tr_sysretq 703#endif 704 /*NOTREACHED*/ 705 SET_SIZE(nopop_sys_syscall_swapgs_sysretq) 706 707_syscall_pre: 708 call pre_syscall 709 movl %eax, %r12d 710 testl %eax, %eax 711 jne _syscall_post_call 712 /* 713 * Didn't abort, so reload the syscall args and invoke the handler. 714 */ 715 movzwl T_SYSNUM(%r15), %eax 716 jmp _syscall_invoke 717 718_syscall_ill: 719 call nosys 720 movq %rax, %r12 721 movq %rdx, %r13 722 jmp _syscall_post_call 723 724_syscall_post: 725 STI 726 /* 727 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM 728 * so that we can account for the extra work it takes us to finish. 729 */ 730 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 731_syscall_post_call: 732 movq %r12, %rdi 733 movq %r13, %rsi 734 call post_syscall 735 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 736 jmp _sys_rtt 737 SET_SIZE(sys_syscall) 738 SET_SIZE(brand_sys_syscall) 739 740 ENTRY_NP(brand_sys_syscall32) 741 SWAPGS /* kernel gsbase */ 742 XPV_TRAP_POP 743 BRAND_CALLBACK(BRAND_CB_SYSCALL32, BRAND_URET_FROM_REG(%rcx)) 744 jmp nopop_sys_syscall32 745 746 ALTENTRY(sys_syscall32) 747 SWAPGS /* kernel gsbase */ 748 XPV_TRAP_POP 749 750nopop_sys_syscall32: 751 movl %esp, %r10d 752 movq %gs:CPU_THREAD, %r15 753 movq T_STACK(%r15), %rsp 754 movl %eax, %eax 755 756 movl $U32CS_SEL, REGOFF_CS(%rsp) 757 movl %ecx, REGOFF_RIP(%rsp) /* syscall: %rip -> %rcx */ 758 movq %r11, REGOFF_RFL(%rsp) /* syscall: %rfl -> %r11d */ 759 movq %r10, REGOFF_RSP(%rsp) 760 movl $UDS_SEL, REGOFF_SS(%rsp) 761 762_syscall32_save: 763 movl %edi, REGOFF_RDI(%rsp) 764 movl %esi, REGOFF_RSI(%rsp) 765 movl %ebp, REGOFF_RBP(%rsp) 766 movl %ebx, REGOFF_RBX(%rsp) 767 movl %edx, REGOFF_RDX(%rsp) 768 movl %ecx, REGOFF_RCX(%rsp) 769 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */ 770 movq $0, REGOFF_SAVFP(%rsp) 771 movq $0, REGOFF_SAVPC(%rsp) 772 773 /* 774 * Copy these registers here in case we end up stopped with 775 * someone (like, say, /proc) messing with our register state. 776 * We don't -restore- them unless we have to in update_sregs. 777 * 778 * Since userland -can't- change fsbase or gsbase directly, 779 * we don't bother to capture them here. 780 */ 781 xorl %ebx, %ebx 782 movw %ds, %bx 783 movq %rbx, REGOFF_DS(%rsp) 784 movw %es, %bx 785 movq %rbx, REGOFF_ES(%rsp) 786 movw %fs, %bx 787 movq %rbx, REGOFF_FS(%rsp) 788 movw %gs, %bx 789 movq %rbx, REGOFF_GS(%rsp) 790 791 /* 792 * If we're trying to use TRAPTRACE though, I take that back: we're 793 * probably debugging some problem in the SWAPGS logic and want to know 794 * what the incoming gsbase was. 795 * 796 * Since we already did SWAPGS, record the KGSBASE. 797 */ 798#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv) 799 movl $MSR_AMD_KGSBASE, %ecx 800 rdmsr 801 movl %eax, REGOFF_GSBASE(%rsp) 802 movl %edx, REGOFF_GSBASE+4(%rsp) 803#endif 804 805 /* 806 * Application state saved in the regs structure on the stack 807 * %eax is the syscall number 808 * %rsp is the thread's stack, %r15 is curthread 809 * REG_RSP(%rsp) is the user's stack 810 */ 811 812 SYSCALL_TRAPTRACE32($TT_SYSC) 813 814 movq %rsp, %rbp 815 816 movq T_LWP(%r15), %r14 817 ASSERT_NO_RUPDATE_PENDING(%r14) 818 819 ENABLE_INTR_FLAGS 820 821 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 822 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate call) */ 823 824 ASSERT_LWPTOREGS(%r14, %rsp) 825 826 incq %gs:CPU_STATS_SYS_SYSCALL 827 828 /* 829 * Make some space for MAXSYSARGS (currently 8) 32-bit args placed 830 * into 64-bit (long) arg slots, maintaining 16 byte alignment. Or 831 * more succinctly: 832 * 833 * SA(MAXSYSARGS * sizeof (long)) == 64 834 */ 835#define SYS_DROP 64 /* drop for args */ 836 subq $SYS_DROP, %rsp 837 movb $LWP_SYS, LWP_STATE(%r14) 838 movq %r15, %rdi 839 movq %rsp, %rsi 840 call syscall_entry 841 842 /* 843 * Fetch the arguments copied onto the kernel stack and put 844 * them in the right registers to invoke a C-style syscall handler. 845 * %rax contains the handler address. 846 * 847 * Ideas for making all this go faster of course include simply 848 * forcibly fetching 6 arguments from the user stack under lofault 849 * protection, reverting to copyin_args only when watchpoints 850 * are in effect. 851 * 852 * (If we do this, make sure that exec and libthread leave 853 * enough space at the top of the stack to ensure that we'll 854 * never do a fetch from an invalid page.) 855 * 856 * Lots of ideas here, but they won't really help with bringup B-) 857 * Correctness can't wait, performance can wait a little longer .. 858 */ 859 860 movq %rax, %rbx 861 movl 0(%rsp), %edi 862 movl 8(%rsp), %esi 863 movl 0x10(%rsp), %edx 864 movl 0x18(%rsp), %ecx 865 movl 0x20(%rsp), %r8d 866 movl 0x28(%rsp), %r9d 867 868 movq SY_CALLC(%rbx), %rax 869 INDIRECT_CALL_REG(rax) 870 871 movq %rbp, %rsp /* pop the args */ 872 873 /* 874 * amd64 syscall handlers -always- return a 64-bit value in %rax. 875 * On the 32-bit kernel, they always return that value in %eax:%edx 876 * as required by the 32-bit ABI. 877 * 878 * Simulate the same behaviour by unconditionally splitting the 879 * return value in the same way. 880 */ 881 movq %rax, %r13 882 shrq $32, %r13 /* upper 32-bits into %edx */ 883 movl %eax, %r12d /* lower 32-bits into %eax */ 884 885 /* 886 * Optimistically assume that there's no post-syscall 887 * work to do. (This is to avoid having to call syscall_mstate() 888 * with interrupts disabled) 889 */ 890 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 891 892 /* 893 * We must protect ourselves from being descheduled here; 894 * If we were, and we ended up on another cpu, or another 895 * lwp got in ahead of us, it could change the segment 896 * registers without us noticing before we return to userland. 897 */ 898 CLI(%r14) 899 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 900 jne _full_syscall_postsys32 901 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 902 903 /* 904 * Clobber %r11 as we check CR0.TS. 905 */ 906 ASSERT_CR0TS_ZERO(%r11) 907 908 /* 909 * To get back to userland, we need to put the return %rip in %rcx and 910 * the return %rfl in %r11d. The sysret instruction also arranges 911 * to fix up %cs and %ss; everything else is our responsibility. 912 */ 913 914 movl %r12d, %eax /* %eax: rval1 */ 915 movl REGOFF_RBX(%rsp), %ebx 916 /* %ecx used for return pointer */ 917 movl %r13d, %edx /* %edx: rval2 */ 918 movl REGOFF_RBP(%rsp), %ebp 919 movl REGOFF_RSI(%rsp), %esi 920 movl REGOFF_RDI(%rsp), %edi 921 922 movl REGOFF_RFL(%rsp), %r11d /* %r11 -> eflags */ 923 movl REGOFF_RIP(%rsp), %ecx /* %ecx -> %eip */ 924 /* 925 * Unlike other cases, because we need to restore the user stack pointer 926 * before exiting the kernel we must clear the microarch state before 927 * getting here. This should be safe because it means that the only 928 * values on the bus after this are based on the user's registers and 929 * potentially the addresses where we stored them. Given the constraints 930 * of sysret, that's how it has to be. 931 */ 932 call x86_md_clear 933 934 movl REGOFF_RSP(%rsp), %esp 935 936 ASSERT_UPCALL_MASK_IS_SET 937 ALTENTRY(nopop_sys_syscall32_swapgs_sysretl) 938 jmp tr_sysretl 939 SET_SIZE(nopop_sys_syscall32_swapgs_sysretl) 940 /*NOTREACHED*/ 941 942_full_syscall_postsys32: 943 STI 944 /* 945 * Sigh, our optimism wasn't justified, put it back to LMS_SYSTEM 946 * so that we can account for the extra work it takes us to finish. 947 */ 948 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 949 movq %r15, %rdi 950 movq %r12, %rsi /* rval1 - %eax */ 951 movq %r13, %rdx /* rval2 - %edx */ 952 call syscall_exit 953 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 954 jmp _sys_rtt 955 SET_SIZE(sys_syscall32) 956 SET_SIZE(brand_sys_syscall32) 957 958/* 959 * System call handler via the sysenter instruction 960 * Used only for 32-bit system calls on the 64-bit kernel. 961 * 962 * The caller in userland has arranged that: 963 * 964 * - %eax contains the syscall number 965 * - %ecx contains the user %esp 966 * - %edx contains the return %eip 967 * - the user stack contains the args to the syscall 968 * 969 * Hardware and (privileged) initialization code have arranged that by 970 * the time the sysenter instructions completes: 971 * 972 * - %rip is pointing to sys_sysenter (below). 973 * - %cs and %ss are set to kernel text and stack (data) selectors. 974 * - %rsp is pointing at the lwp's stack 975 * - interrupts have been disabled. 976 * 977 * Note that we are unable to return both "rvals" to userland with 978 * this call, as %edx is used by the sysexit instruction. 979 * 980 * One final complication in this routine is its interaction with 981 * single-stepping in a debugger. For most of the system call mechanisms, the 982 * CPU automatically clears the single-step flag before we enter the kernel. 983 * The sysenter mechanism does not clear the flag, so a user single-stepping 984 * through a libc routine may suddenly find themself single-stepping through the 985 * kernel. To detect this, kmdb and trap() both compare the trap %pc to the 986 * [brand_]sys_enter addresses on each single-step trap. If it finds that we 987 * have single-stepped to a sysenter entry point, it explicitly clears the flag 988 * and executes the sys_sysenter routine. 989 * 990 * One final complication in this final complication is the fact that we have 991 * two different entry points for sysenter: brand_sys_sysenter and sys_sysenter. 992 * If we enter at brand_sys_sysenter and start single-stepping through the 993 * kernel with kmdb, we will eventually hit the instruction at sys_sysenter. 994 * kmdb cannot distinguish between that valid single-step and the undesirable 995 * one mentioned above. To avoid this situation, we simply add a jump over the 996 * instruction at sys_sysenter to make it impossible to single-step to it. 997 */ 998 999 ENTRY_NP(brand_sys_sysenter) 1000 SWAPGS /* kernel gsbase */ 1001 ALTENTRY(_brand_sys_sysenter_post_swapgs) 1002 1003 BRAND_CALLBACK(BRAND_CB_SYSENTER, BRAND_URET_FROM_REG(%rdx)) 1004 /* 1005 * Jump over sys_sysenter to allow single-stepping as described 1006 * above. 1007 */ 1008 jmp _sys_sysenter_post_swapgs 1009 1010 ALTENTRY(sys_sysenter) 1011 SWAPGS /* kernel gsbase */ 1012 ALTENTRY(_sys_sysenter_post_swapgs) 1013 1014 movq %gs:CPU_THREAD, %r15 1015 1016 movl $U32CS_SEL, REGOFF_CS(%rsp) 1017 movl %ecx, REGOFF_RSP(%rsp) /* wrapper: %esp -> %ecx */ 1018 movl %edx, REGOFF_RIP(%rsp) /* wrapper: %eip -> %edx */ 1019 /* 1020 * NOTE: none of the instructions that run before we get here should 1021 * clobber bits in (R)FLAGS! This includes the kpti trampoline. 1022 */ 1023 pushfq 1024 popq %r10 1025 movl $UDS_SEL, REGOFF_SS(%rsp) 1026 1027 /* 1028 * Set the interrupt flag before storing the flags to the 1029 * flags image on the stack so we can return to user with 1030 * interrupts enabled if we return via sys_rtt_syscall32 1031 */ 1032 orq $PS_IE, %r10 1033 movq %r10, REGOFF_RFL(%rsp) 1034 1035 movl %edi, REGOFF_RDI(%rsp) 1036 movl %esi, REGOFF_RSI(%rsp) 1037 movl %ebp, REGOFF_RBP(%rsp) 1038 movl %ebx, REGOFF_RBX(%rsp) 1039 movl %edx, REGOFF_RDX(%rsp) 1040 movl %ecx, REGOFF_RCX(%rsp) 1041 movl %eax, REGOFF_RAX(%rsp) /* wrapper: sysc# -> %eax */ 1042 movq $0, REGOFF_SAVFP(%rsp) 1043 movq $0, REGOFF_SAVPC(%rsp) 1044 1045 /* 1046 * Copy these registers here in case we end up stopped with 1047 * someone (like, say, /proc) messing with our register state. 1048 * We don't -restore- them unless we have to in update_sregs. 1049 * 1050 * Since userland -can't- change fsbase or gsbase directly, 1051 * we don't bother to capture them here. 1052 */ 1053 xorl %ebx, %ebx 1054 movw %ds, %bx 1055 movq %rbx, REGOFF_DS(%rsp) 1056 movw %es, %bx 1057 movq %rbx, REGOFF_ES(%rsp) 1058 movw %fs, %bx 1059 movq %rbx, REGOFF_FS(%rsp) 1060 movw %gs, %bx 1061 movq %rbx, REGOFF_GS(%rsp) 1062 1063 /* 1064 * If we're trying to use TRAPTRACE though, I take that back: we're 1065 * probably debugging some problem in the SWAPGS logic and want to know 1066 * what the incoming gsbase was. 1067 * 1068 * Since we already did SWAPGS, record the KGSBASE. 1069 */ 1070#if defined(DEBUG) && defined(TRAPTRACE) && !defined(__xpv) 1071 movl $MSR_AMD_KGSBASE, %ecx 1072 rdmsr 1073 movl %eax, REGOFF_GSBASE(%rsp) 1074 movl %edx, REGOFF_GSBASE+4(%rsp) 1075#endif 1076 1077 /* 1078 * Application state saved in the regs structure on the stack 1079 * %eax is the syscall number 1080 * %rsp is the thread's stack, %r15 is curthread 1081 * REG_RSP(%rsp) is the user's stack 1082 */ 1083 1084 SYSCALL_TRAPTRACE($TT_SYSENTER) 1085 1086 movq %rsp, %rbp 1087 1088 movq T_LWP(%r15), %r14 1089 ASSERT_NO_RUPDATE_PENDING(%r14) 1090 1091 ENABLE_INTR_FLAGS 1092 1093 /* 1094 * Catch 64-bit process trying to issue sysenter instruction 1095 * on Nocona based systems. 1096 */ 1097 movq LWP_PROCP(%r14), %rax 1098 cmpq $DATAMODEL_ILP32, P_MODEL(%rax) 1099 je 7f 1100 1101 /* 1102 * For a non-32-bit process, simulate a #ud, since that's what 1103 * native hardware does. The traptrace entry (above) will 1104 * let you know what really happened. 1105 */ 1106 movq $T_ILLINST, REGOFF_TRAPNO(%rsp) 1107 movq REGOFF_CS(%rsp), %rdi 1108 movq %rdi, REGOFF_ERR(%rsp) 1109 movq %rsp, %rdi 1110 movq REGOFF_RIP(%rsp), %rsi 1111 movl %gs:CPU_ID, %edx 1112 call trap 1113 jmp _sys_rtt 11147: 1115 1116 MSTATE_TRANSITION(LMS_USER, LMS_SYSTEM) 1117 movl REGOFF_RAX(%rsp), %eax /* (%rax damaged by mstate calls) */ 1118 1119 ASSERT_LWPTOREGS(%r14, %rsp) 1120 1121 incq %gs:CPU_STATS_SYS_SYSCALL 1122 1123 /* 1124 * Make some space for MAXSYSARGS (currently 8) 32-bit args 1125 * placed into 64-bit (long) arg slots, plus one 64-bit 1126 * (long) arg count, maintaining 16 byte alignment. 1127 */ 1128 subq $SYS_DROP, %rsp 1129 movb $LWP_SYS, LWP_STATE(%r14) 1130 movq %r15, %rdi 1131 movq %rsp, %rsi 1132 call syscall_entry 1133 1134 /* 1135 * Fetch the arguments copied onto the kernel stack and put 1136 * them in the right registers to invoke a C-style syscall handler. 1137 * %rax contains the handler address. 1138 */ 1139 movq %rax, %rbx 1140 movl 0(%rsp), %edi 1141 movl 8(%rsp), %esi 1142 movl 0x10(%rsp), %edx 1143 movl 0x18(%rsp), %ecx 1144 movl 0x20(%rsp), %r8d 1145 movl 0x28(%rsp), %r9d 1146 1147 movq SY_CALLC(%rbx), %rax 1148 INDIRECT_CALL_REG(rax) 1149 1150 movq %rbp, %rsp /* pop the args */ 1151 1152 /* 1153 * amd64 syscall handlers -always- return a 64-bit value in %rax. 1154 * On the 32-bit kernel, the always return that value in %eax:%edx 1155 * as required by the 32-bit ABI. 1156 * 1157 * Simulate the same behaviour by unconditionally splitting the 1158 * return value in the same way. 1159 */ 1160 movq %rax, %r13 1161 shrq $32, %r13 /* upper 32-bits into %edx */ 1162 movl %eax, %r12d /* lower 32-bits into %eax */ 1163 1164 /* 1165 * Optimistically assume that there's no post-syscall 1166 * work to do. (This is to avoid having to call syscall_mstate() 1167 * with interrupts disabled) 1168 */ 1169 MSTATE_TRANSITION(LMS_SYSTEM, LMS_USER) 1170 1171 /* 1172 * We must protect ourselves from being descheduled here; 1173 * If we were, and we ended up on another cpu, or another 1174 * lwp got int ahead of us, it could change the segment 1175 * registers without us noticing before we return to userland. 1176 * 1177 * This cli is undone in the tr_sysexit trampoline code. 1178 */ 1179 cli 1180 CHECK_POSTSYS_NE(%r15, %r14, %ebx) 1181 jne _full_syscall_postsys32 1182 SIMPLE_SYSCALL_POSTSYS(%r15, %r14, %bx) 1183 1184 /* 1185 * To get back to userland, load up the 32-bit registers and 1186 * sysexit back where we came from. 1187 */ 1188 1189 /* 1190 * Interrupts will be turned on by the 'sti' executed just before 1191 * sysexit. The following ensures that restoring the user's rflags 1192 * doesn't enable interrupts too soon. 1193 */ 1194 andq $_BITNOT(PS_IE), REGOFF_RFL(%rsp) 1195 1196 /* 1197 * Clobber %r11 as we check CR0.TS. 1198 */ 1199 ASSERT_CR0TS_ZERO(%r11) 1200 1201 /* 1202 * (There's no point in loading up %edx because the sysexit 1203 * mechanism smashes it.) 1204 */ 1205 movl %r12d, %eax 1206 movl REGOFF_RBX(%rsp), %ebx 1207 movl REGOFF_RBP(%rsp), %ebp 1208 movl REGOFF_RSI(%rsp), %esi 1209 movl REGOFF_RDI(%rsp), %edi 1210 1211 movl REGOFF_RIP(%rsp), %edx /* sysexit: %edx -> %eip */ 1212 pushq REGOFF_RFL(%rsp) 1213 popfq 1214 movl REGOFF_RSP(%rsp), %ecx /* sysexit: %ecx -> %esp */ 1215 ALTENTRY(sys_sysenter_swapgs_sysexit) 1216 call x86_md_clear 1217 jmp tr_sysexit 1218 SET_SIZE(sys_sysenter_swapgs_sysexit) 1219 SET_SIZE(sys_sysenter) 1220 SET_SIZE(_sys_sysenter_post_swapgs) 1221 SET_SIZE(brand_sys_sysenter) 1222 1223/* 1224 * This is the destination of the "int $T_SYSCALLINT" interrupt gate, used by 1225 * the generic i386 libc to do system calls. We do a small amount of setup 1226 * before jumping into the existing sys_syscall32 path. 1227 */ 1228 1229 ENTRY_NP(brand_sys_syscall_int) 1230 SWAPGS /* kernel gsbase */ 1231 XPV_TRAP_POP 1232 call smap_enable 1233 BRAND_CALLBACK(BRAND_CB_INT91, BRAND_URET_FROM_INTR_STACK()) 1234 jmp nopop_syscall_int 1235 1236 ALTENTRY(sys_syscall_int) 1237 SWAPGS /* kernel gsbase */ 1238 XPV_TRAP_POP 1239 call smap_enable 1240 1241nopop_syscall_int: 1242 movq %gs:CPU_THREAD, %r15 1243 movq T_STACK(%r15), %rsp 1244 movl %eax, %eax 1245 /* 1246 * Set t_post_sys on this thread to force ourselves out via the slow 1247 * path. It might be possible at some later date to optimize this out 1248 * and use a faster return mechanism. 1249 */ 1250 movb $1, T_POST_SYS(%r15) 1251 CLEAN_CS 1252 jmp _syscall32_save 1253 /* 1254 * There should be no instructions between this label and SWAPGS/IRET 1255 * or we could end up breaking branded zone support. See the usage of 1256 * this label in lx_brand_int80_callback and sn1_brand_int91_callback 1257 * for examples. 1258 * 1259 * We want to swapgs to maintain the invariant that all entries into 1260 * tr_iret_user are done on the user gsbase. 1261 */ 1262 ALTENTRY(sys_sysint_swapgs_iret) 1263 call x86_md_clear 1264 SWAPGS 1265 jmp tr_iret_user 1266 /*NOTREACHED*/ 1267 SET_SIZE(sys_sysint_swapgs_iret) 1268 SET_SIZE(sys_syscall_int) 1269 SET_SIZE(brand_sys_syscall_int) 1270 1271/* 1272 * Legacy 32-bit applications and old libc implementations do lcalls; 1273 * we should never get here because the LDT entry containing the syscall 1274 * segment descriptor has the "segment present" bit cleared, which means 1275 * we end up processing those system calls in trap() via a not-present trap. 1276 * 1277 * We do it this way because a call gate unhelpfully does -nothing- to the 1278 * interrupt flag bit, so an interrupt can run us just after the lcall 1279 * completes, but just before the swapgs takes effect. Thus the INTR_PUSH and 1280 * INTR_POP paths would have to be slightly more complex to dance around 1281 * this problem, and end up depending explicitly on the first 1282 * instruction of this handler being either swapgs or cli. 1283 */ 1284 1285 ENTRY_NP(sys_lcall32) 1286 SWAPGS /* kernel gsbase */ 1287 pushq $0 1288 pushq %rbp 1289 movq %rsp, %rbp 1290 leaq __lcall_panic_str(%rip), %rdi 1291 xorl %eax, %eax 1292 call panic 1293 SET_SIZE(sys_lcall32) 1294 1295__lcall_panic_str: 1296 .string "sys_lcall32: shouldn't be here!" 1297 1298/* 1299 * Declare a uintptr_t which covers the entire pc range of syscall 1300 * handlers for the stack walkers that need this. 1301 */ 1302 .align CPTRSIZE 1303 .globl _allsyscalls_size 1304 .type _allsyscalls_size, @object 1305_allsyscalls_size: 1306 .NWORD . - _allsyscalls 1307 SET_SIZE(_allsyscalls_size) 1308 1309/* 1310 * These are the thread context handlers for lwps using sysenter/sysexit. 1311 */ 1312 1313 /* 1314 * setting this value to zero as we switch away causes the 1315 * stack-pointer-on-sysenter to be NULL, ensuring that we 1316 * don't silently corrupt another (preempted) thread stack 1317 * when running an lwp that (somehow) didn't get sep_restore'd 1318 */ 1319 ENTRY_NP(sep_save) 1320 xorl %edx, %edx 1321 xorl %eax, %eax 1322 movl $MSR_INTC_SEP_ESP, %ecx 1323 wrmsr 1324 ret 1325 SET_SIZE(sep_save) 1326 1327 /* 1328 * Update the kernel stack pointer as we resume onto this cpu. 1329 */ 1330 ENTRY_NP(sep_restore) 1331 movq %rdi, %rdx 1332 shrq $32, %rdx 1333 movl %edi, %eax 1334 movl $MSR_INTC_SEP_ESP, %ecx 1335 wrmsr 1336 ret 1337 SET_SIZE(sep_restore) 1338