1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21/*
22 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23 * Use is subject to license terms.
24 */
25/*
26 * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
27 * Copyright (c) 2014 by Delphix. All rights reserved.
28 * Copyright 2018 Joyent, Inc.
29 */
30
31#include <sys/cpuvar.h>
32#include <sys/psm.h>
33#include <sys/archsystm.h>
34#include <sys/apic.h>
35#include <sys/apic_common.h>
36#include <sys/sunddi.h>
37#include <sys/ddi_impldefs.h>
38#include <sys/mach_intr.h>
39#include <sys/sysmacros.h>
40#include <sys/trap.h>
41#include <sys/x86_archext.h>
42#include <sys/privregs.h>
43#include <sys/psm_common.h>
44
45/* Function prototypes of X2APIC */
46static uint64_t local_x2apic_read(uint32_t msr);
47static void local_x2apic_write(uint32_t msr, uint64_t value);
48static int get_local_x2apic_pri(void);
49static void local_x2apic_write_task_reg(uint64_t value);
50static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
51
52/*
53 * According to the X2APIC specification:
54 *
55 *   xAPIC global enable    X2APIC enable         Description
56 *   (IA32_APIC_BASE[11])   (IA32_APIC_BASE[10])
57 * -----------------------------------------------------------
58 *	0			0	APIC is disabled
59 *	0			1	Invalid
60 *	1			0	APIC is enabled in xAPIC mode
61 *	1			1	APIC is enabled in X2APIC mode
62 * -----------------------------------------------------------
63 */
64int	x2apic_enable = 1;
65
66/* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
67static apic_reg_ops_t x2apic_regs_ops = {
68	local_x2apic_read,
69	local_x2apic_write,
70	get_local_x2apic_pri,
71	local_x2apic_write_task_reg,
72	local_x2apic_write_int_cmd,
73	apic_send_EOI,
74};
75
76/*
77 * X2APIC Implementation.
78 */
79static uint64_t
80local_x2apic_read(uint32_t msr)
81{
82	uint64_t i;
83
84	i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
85	return (i);
86}
87
88static void
89local_x2apic_write(uint32_t msr, uint64_t value)
90{
91	uint64_t tmp;
92
93	if (msr != APIC_EOI_REG) {
94		tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
95		tmp = (tmp & 0xffffffff00000000) | value;
96	} else {
97		tmp = 0;
98	}
99
100	wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
101}
102
103static int
104get_local_x2apic_pri(void)
105{
106	return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
107}
108
109static void
110local_x2apic_write_task_reg(uint64_t value)
111{
112	X2APIC_WRITE(APIC_TASK_REG, value);
113}
114
115static void
116local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
117{
118	wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
119	    (((uint64_t)cpu_id << 32) | cmd1));
120}
121
122int
123apic_detect_x2apic(void)
124{
125	if (x2apic_enable == 0)
126		return (0);
127
128	return (is_x86_feature(x86_featureset, X86FSET_X2APIC));
129}
130
131void
132apic_enable_x2apic(void)
133{
134	uint64_t apic_base_msr;
135
136	if (apic_local_mode() == LOCAL_X2APIC) {
137		/* BIOS apparently has enabled X2APIC */
138		if (apic_mode != LOCAL_X2APIC)
139			x2apic_update_psm();
140		return;
141	}
142
143	/*
144	 * This is the first time we are enabling X2APIC on this CPU
145	 */
146	apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
147	apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
148	wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
149
150	if (apic_mode != LOCAL_X2APIC)
151		x2apic_update_psm();
152}
153
154/*
155 * Change apic_reg_ops depending upon the apic_mode.
156 */
157void
158apic_change_ops()
159{
160	if (apic_mode == LOCAL_APIC)
161		apic_reg_ops = &local_apic_regs_ops;
162	else if (apic_mode == LOCAL_X2APIC)
163		apic_reg_ops = &x2apic_regs_ops;
164}
165
166/*
167 * Generates an interprocessor interrupt to another CPU when X2APIC mode is
168 * enabled.
169 */
170void
171x2apic_send_ipi(int cpun, int ipl)
172{
173	int vector;
174	ulong_t flag;
175
176	ASSERT(apic_mode == LOCAL_X2APIC);
177
178	/*
179	 * With X2APIC, Intel relaxed the semantics of the
180	 * WRMSR instruction such that references to the X2APIC
181	 * MSR registers are no longer serializing instructions.
182	 * The code that initiates IPIs assumes that some sort
183	 * of memory serialization occurs. The old APIC code
184	 * did a write to uncachable memory mapped registers.
185	 * Any reference to uncached memory is a serializing
186	 * operation. To mimic those semantics here, we do an
187	 * atomic operation, which translates to a LOCK OR instruction,
188	 * which is serializing.
189	 */
190	atomic_or_ulong(&flag, 1);
191
192	vector = apic_resv_vector[ipl];
193
194	flag = intr_clear();
195
196	/*
197	 * According to X2APIC specification in section '2.3.5.1' of
198	 * Interrupt Command Register Semantics, the semantics of
199	 * programming Interrupt Command Register to dispatch an interrupt
200	 * is simplified. A single MSR write to the 64-bit ICR is required
201	 * for dispatching an interrupt. Specifically with the 64-bit MSR
202	 * interface to ICR, system software is not required to check the
203	 * status of the delivery status bit prior to writing to the ICR
204	 * to send an IPI. With the removal of the Delivery Status bit,
205	 * system software no longer has a reason to read the ICR. It remains
206	 * readable only to aid in debugging.
207	 */
208#ifdef	DEBUG
209	APIC_AV_PENDING_SET();
210#endif	/* DEBUG */
211
212	if ((cpun == psm_get_cpu_id())) {
213		X2APIC_WRITE(X2APIC_SELF_IPI, vector);
214	} else {
215		apic_reg_ops->apic_write_int_cmd(
216		    apic_cpus[cpun].aci_local_id, vector);
217	}
218
219	intr_restore(flag);
220}
221
222void
223x2apic_send_pir_ipi(processorid_t cpun)
224{
225	const int vector = apic_pir_vect;
226	ulong_t flag;
227
228	ASSERT(apic_mode == LOCAL_X2APIC);
229	ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
230
231	/* Serialize as described in x2apic_send_ipi() above. */
232	atomic_or_ulong(&flag, 1);
233
234	flag = intr_clear();
235
236	/* Self-IPI for inducing PIR makes no sense. */
237	if ((cpun != psm_get_cpu_id())) {
238#ifdef	DEBUG
239		/* Only for debugging. (again, see: x2apic_send_ipi) */
240		APIC_AV_PENDING_SET();
241#endif	/* DEBUG */
242
243		apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
244		    vector);
245	}
246
247	intr_restore(flag);
248}
249
250/*
251 * Generates IPI to another CPU depending on the local APIC mode.
252 * apic_send_ipi() and x2apic_send_ipi() depends on the configured
253 * mode of the local APIC, but that may not match the actual mode
254 * early in CPU startup.
255 *
256 * Any changes made to this routine must be accompanied by similar
257 * changes to apic_send_ipi().
258 */
259void
260apic_common_send_ipi(int cpun, int ipl)
261{
262	int vector;
263	ulong_t flag;
264	int mode = apic_local_mode();
265
266	if (mode == LOCAL_X2APIC) {
267		x2apic_send_ipi(cpun, ipl);
268		return;
269	}
270
271	ASSERT(mode == LOCAL_APIC);
272
273	vector = apic_resv_vector[ipl];
274	ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
275	flag = intr_clear();
276	while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
277		apic_ret();
278	local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
279	    vector);
280	intr_restore(flag);
281}
282
283void
284apic_common_send_pir_ipi(processorid_t cpun)
285{
286	const int mode = apic_local_mode();
287
288	if (mode == LOCAL_X2APIC) {
289		x2apic_send_pir_ipi(cpun);
290		return;
291	}
292
293	apic_send_pir_ipi(cpun);
294}
295