xref: /illumos-gate/usr/src/lib/commpage/i386/cp_subr.S (revision 5d9d9091)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2019 Joyent, Inc.
 * Copyright 2020 Oxide Computer Company
 */

#include <sys/asm_linkage.h>
#include <sys/segments.h>
#include <sys/time_impl.h>
#include <sys/tsc.h>
#include <cp_offsets.h>

#define	GETCPU_GDT_OFFSET	SEL_GDT(GDT_CPUID, SEL_UPL)

/*
 * For __cp_tsc_read calls which incur looping retries due to CPU migration,
 * this represents the maximum number of tries before bailing out.
 */
#define	TSC_READ_MAXLOOP	0x4

	.file	"cp_subr.s"

/*
 * hrtime_t
 * __cp_tsc_read(comm_page_t *cp)
 *
 * Stack usage: 0x18 bytes
 *
 * %ebp-0x00 - frame pointer
 * %ebp-0x04 - saved %edi
 * %ebp-0x08 - saved %esi
 * %ebp-0x0c - CPU ID
 * %ebp-0x10 - loop count
 * %ebp-0x14 - saved %ebx (during cpuid)
 */
	ENTRY_NP(__cp_tsc_read)
	pushl	%ebp
	movl	%esp, %ebp
	pushl	%edi
	pushl	%esi
	subl	$0x8, %esp

	movl	0x8(%ebp), %edi
	movl	CP_TSC_TYPE(%edi), %eax
	movl	CP_TSC_NCPU(%edi), %esi
	cmpl	$TSC_TSCP, %eax
	jne	3f
	rdtscp
	cmpl	$0, %esi
	jne	2f
1:
	addl	$0x8, %esp
	popl	%esi
	popl	%edi
	leave
	ret
2:
	/*
	 * When cp_tsc_ncpu is non-zero, it indicates the length of the
	 * cp_tsc_sync_tick_delta array, which contains per-CPU offsets for the
	 * TSC.  The CPU ID furnished by the IA32_TSC_AUX register via rdtscp
	 * is used to look up an offset value in that array and apply it to the
	 * TSC reading.
	 */
	leal	CP_TSC_SYNC_TICK_DELTA(%edi), %esi
	leal	(%esi, %ecx, 8), %ecx
	addl	(%ecx), %eax
	adcl	0x4(%ecx), %edx
	jmp	1b

3:
	testl	%esi, %esi
	jz	4f

	movl	$0, (%esp)
	mov	$GETCPU_GDT_OFFSET, %eax
	lsl	%eax, %eax
	movl	%eax, 0x4(%esp)
	movl	CP_TSC_TYPE(%edi), %eax

4:
	/*
	 * TSC_RDTSC_MFENCE was used in the past for AMD chips, but has been
	 * supplanted by TSC_RDTSC_LFENCE, which works on Intel and AMD (when
	 * lfence can be confirmed as serializing).
	 */

5:
	cmpl	$TSC_RDTSC_LFENCE, %eax
	jne	6f
	lfence
	rdtsc
	jmp	8f

6:
	cmpl	$TSC_RDTSC_CPUID, %eax
	jne	7f
	pushl	%ebx
	xorl	%eax, %eax
	cpuid
	rdtsc
	popl	%ebx
	jmp	8f

7:
	/*
	 * Other protections should have prevented this function from being
	 * called in the first place.  Since callers must handle a failure from
	 * CPU migration looping, yield the same result as a bail-out: 0
	 */
	xorl	%eax, %eax
	xorl	%edx, %edx
	jmp	1b

8:

	cmpl	$0, %esi
	je	1b
	/*
	 * With a TSC reading in-hand, confirm that the thread has not migrated
	 * since the cpu_id was first checked.
	 */
	movl	$GETCPU_GDT_OFFSET, %ecx
	lsl	%ecx, %ecx
	movl	0x4(%esp), %esi
	cmpl	%ecx, %esi
	jne	9f

	/* Grab the per-cpu offset and add it to the TSC result */
	leal	CP_TSC_SYNC_TICK_DELTA(%edi), %esi
	leal	(%esi, %ecx, 8), %ecx
	addl	(%ecx), %eax
	adcl	0x4(%ecx), %edx
	jmp	1b

9:
	/*
	 * It appears that a migration has occurred between the first CPU ID
	 * query and now.  Check if the loop limit has been broken and retry if
	 * that's not the case.
	 */
	movl	(%esp), %eax
	cmpl	$TSC_READ_MAXLOOP, %eax
	jge	10f

	incl	%eax
	movl	%eax, (%esp)
	movl	%ecx, 0x4(%esp)
	movl	CP_TSC_NCPU(%edi), %esi
	movl	CP_TSC_TYPE(%edi), %eax
	jmp	4b

10:
	/* Loop limit was reached. Return bail-out value of 0. */
	xorl	%eax, %eax
	xorl	%edx, %edx
	jmp	1b

	SET_SIZE(__cp_tsc_read)

/*
 * hrtime_t
 * __cp_gethrtime_fasttrap()
 */
	ENTRY_NP(__cp_gethrtime_fasttrap)
	movl	$T_GETHRTIME, %eax
	int	$T_FASTTRAP
	ret
	SET_SIZE(__cp_gethrtime_fasttrap)

/*
 * uint_t
 * __cp_getcpu(uint_t cp_tsc_type)
 */
	ENTRY_NP(__cp_getcpu)
	/*
	 * If RDTSCP is available, it is a quick way to grab the cpu_id which
	 * is stored in the TSC_AUX MSR by the kernel.
	 */
	movl	4(%esp), %eax
	movl	CP_TSC_TYPE(%eax), %eax
	cmpl	$TSC_TSCP, %eax
	jne	1f
	rdtscp
	movl	%ecx, %eax
	ret
1:
	mov	$GETCPU_GDT_OFFSET, %eax
	lsl	%eax, %eax
	ret
	SET_SIZE(__cp_getcpu)