1583cd330SHans Rosenfeld /*
2583cd330SHans Rosenfeld * CDDL HEADER START
3583cd330SHans Rosenfeld *
4583cd330SHans Rosenfeld * The contents of this file are subject to the terms of the
5583cd330SHans Rosenfeld * Common Development and Distribution License (the "License").
6583cd330SHans Rosenfeld * You may not use this file except in compliance with the License.
7583cd330SHans Rosenfeld *
8583cd330SHans Rosenfeld * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9583cd330SHans Rosenfeld * or http://www.opensolaris.org/os/licensing.
10583cd330SHans Rosenfeld * See the License for the specific language governing permissions
11583cd330SHans Rosenfeld * and limitations under the License.
12583cd330SHans Rosenfeld *
13583cd330SHans Rosenfeld * When distributing Covered Code, include this CDDL HEADER in each
14583cd330SHans Rosenfeld * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15583cd330SHans Rosenfeld * If applicable, add the following below this CDDL HEADER, with the
16583cd330SHans Rosenfeld * fields enclosed by brackets "[]" replaced with your own identifying
17583cd330SHans Rosenfeld * information: Portions Copyright [yyyy] [name of copyright owner]
18583cd330SHans Rosenfeld *
19583cd330SHans Rosenfeld * CDDL HEADER END
20583cd330SHans Rosenfeld */
21583cd330SHans Rosenfeld /*
22583cd330SHans Rosenfeld * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23583cd330SHans Rosenfeld * Use is subject to license terms.
24583cd330SHans Rosenfeld */
25583cd330SHans Rosenfeld /*
26583cd330SHans Rosenfeld * Copyright 2014 Josef 'Jeff' Sipek <jeffpc@josefsipek.net>
27583cd330SHans Rosenfeld * Copyright (c) 2014 by Delphix. All rights reserved.
28*1c2d0470SPatrick Mooney * Copyright 2018 Joyent, Inc.
29583cd330SHans Rosenfeld */
30583cd330SHans Rosenfeld
31583cd330SHans Rosenfeld #include <sys/cpuvar.h>
32583cd330SHans Rosenfeld #include <sys/psm.h>
33583cd330SHans Rosenfeld #include <sys/archsystm.h>
34583cd330SHans Rosenfeld #include <sys/apic.h>
35*1c2d0470SPatrick Mooney #include <sys/apic_common.h>
36583cd330SHans Rosenfeld #include <sys/sunddi.h>
37583cd330SHans Rosenfeld #include <sys/ddi_impldefs.h>
38583cd330SHans Rosenfeld #include <sys/mach_intr.h>
39583cd330SHans Rosenfeld #include <sys/sysmacros.h>
40583cd330SHans Rosenfeld #include <sys/trap.h>
41583cd330SHans Rosenfeld #include <sys/x86_archext.h>
42583cd330SHans Rosenfeld #include <sys/privregs.h>
43583cd330SHans Rosenfeld #include <sys/psm_common.h>
44583cd330SHans Rosenfeld
45583cd330SHans Rosenfeld /* Function prototypes of X2APIC */
46583cd330SHans Rosenfeld static uint64_t local_x2apic_read(uint32_t msr);
47583cd330SHans Rosenfeld static void local_x2apic_write(uint32_t msr, uint64_t value);
48583cd330SHans Rosenfeld static int get_local_x2apic_pri(void);
49583cd330SHans Rosenfeld static void local_x2apic_write_task_reg(uint64_t value);
50583cd330SHans Rosenfeld static void local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1);
51583cd330SHans Rosenfeld
52583cd330SHans Rosenfeld /*
53583cd330SHans Rosenfeld * According to the X2APIC specification:
54583cd330SHans Rosenfeld *
55583cd330SHans Rosenfeld * xAPIC global enable X2APIC enable Description
56583cd330SHans Rosenfeld * (IA32_APIC_BASE[11]) (IA32_APIC_BASE[10])
57583cd330SHans Rosenfeld * -----------------------------------------------------------
58583cd330SHans Rosenfeld * 0 0 APIC is disabled
59583cd330SHans Rosenfeld * 0 1 Invalid
60583cd330SHans Rosenfeld * 1 0 APIC is enabled in xAPIC mode
61583cd330SHans Rosenfeld * 1 1 APIC is enabled in X2APIC mode
62583cd330SHans Rosenfeld * -----------------------------------------------------------
63583cd330SHans Rosenfeld */
64583cd330SHans Rosenfeld int x2apic_enable = 1;
65583cd330SHans Rosenfeld
66583cd330SHans Rosenfeld /* X2APIC : Uses RDMSR/WRMSR instructions to access APIC registers */
67583cd330SHans Rosenfeld static apic_reg_ops_t x2apic_regs_ops = {
68583cd330SHans Rosenfeld local_x2apic_read,
69583cd330SHans Rosenfeld local_x2apic_write,
70583cd330SHans Rosenfeld get_local_x2apic_pri,
71583cd330SHans Rosenfeld local_x2apic_write_task_reg,
72583cd330SHans Rosenfeld local_x2apic_write_int_cmd,
73583cd330SHans Rosenfeld apic_send_EOI,
74583cd330SHans Rosenfeld };
75583cd330SHans Rosenfeld
76583cd330SHans Rosenfeld /*
77583cd330SHans Rosenfeld * X2APIC Implementation.
78583cd330SHans Rosenfeld */
79583cd330SHans Rosenfeld static uint64_t
local_x2apic_read(uint32_t msr)80583cd330SHans Rosenfeld local_x2apic_read(uint32_t msr)
81583cd330SHans Rosenfeld {
82583cd330SHans Rosenfeld uint64_t i;
83583cd330SHans Rosenfeld
84583cd330SHans Rosenfeld i = (uint64_t)(rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2)) & 0xffffffff);
85583cd330SHans Rosenfeld return (i);
86583cd330SHans Rosenfeld }
87583cd330SHans Rosenfeld
88583cd330SHans Rosenfeld static void
local_x2apic_write(uint32_t msr,uint64_t value)89583cd330SHans Rosenfeld local_x2apic_write(uint32_t msr, uint64_t value)
90583cd330SHans Rosenfeld {
91583cd330SHans Rosenfeld uint64_t tmp;
92583cd330SHans Rosenfeld
93583cd330SHans Rosenfeld if (msr != APIC_EOI_REG) {
94583cd330SHans Rosenfeld tmp = rdmsr(REG_X2APIC_BASE_MSR + (msr >> 2));
95583cd330SHans Rosenfeld tmp = (tmp & 0xffffffff00000000) | value;
96583cd330SHans Rosenfeld } else {
97583cd330SHans Rosenfeld tmp = 0;
98583cd330SHans Rosenfeld }
99583cd330SHans Rosenfeld
100583cd330SHans Rosenfeld wrmsr((REG_X2APIC_BASE_MSR + (msr >> 2)), tmp);
101583cd330SHans Rosenfeld }
102583cd330SHans Rosenfeld
103583cd330SHans Rosenfeld static int
get_local_x2apic_pri(void)104583cd330SHans Rosenfeld get_local_x2apic_pri(void)
105583cd330SHans Rosenfeld {
106583cd330SHans Rosenfeld return (rdmsr(REG_X2APIC_BASE_MSR + (APIC_TASK_REG >> 2)));
107583cd330SHans Rosenfeld }
108583cd330SHans Rosenfeld
109583cd330SHans Rosenfeld static void
local_x2apic_write_task_reg(uint64_t value)110583cd330SHans Rosenfeld local_x2apic_write_task_reg(uint64_t value)
111583cd330SHans Rosenfeld {
112583cd330SHans Rosenfeld X2APIC_WRITE(APIC_TASK_REG, value);
113583cd330SHans Rosenfeld }
114583cd330SHans Rosenfeld
115583cd330SHans Rosenfeld static void
local_x2apic_write_int_cmd(uint32_t cpu_id,uint32_t cmd1)116583cd330SHans Rosenfeld local_x2apic_write_int_cmd(uint32_t cpu_id, uint32_t cmd1)
117583cd330SHans Rosenfeld {
118583cd330SHans Rosenfeld wrmsr((REG_X2APIC_BASE_MSR + (APIC_INT_CMD1 >> 2)),
119583cd330SHans Rosenfeld (((uint64_t)cpu_id << 32) | cmd1));
120583cd330SHans Rosenfeld }
121583cd330SHans Rosenfeld
122583cd330SHans Rosenfeld int
apic_detect_x2apic(void)123583cd330SHans Rosenfeld apic_detect_x2apic(void)
124583cd330SHans Rosenfeld {
125583cd330SHans Rosenfeld if (x2apic_enable == 0)
126583cd330SHans Rosenfeld return (0);
127583cd330SHans Rosenfeld
128583cd330SHans Rosenfeld return (is_x86_feature(x86_featureset, X86FSET_X2APIC));
129583cd330SHans Rosenfeld }
130583cd330SHans Rosenfeld
131583cd330SHans Rosenfeld void
apic_enable_x2apic(void)132583cd330SHans Rosenfeld apic_enable_x2apic(void)
133583cd330SHans Rosenfeld {
134583cd330SHans Rosenfeld uint64_t apic_base_msr;
135583cd330SHans Rosenfeld
136583cd330SHans Rosenfeld if (apic_local_mode() == LOCAL_X2APIC) {
137583cd330SHans Rosenfeld /* BIOS apparently has enabled X2APIC */
138583cd330SHans Rosenfeld if (apic_mode != LOCAL_X2APIC)
139583cd330SHans Rosenfeld x2apic_update_psm();
140583cd330SHans Rosenfeld return;
141583cd330SHans Rosenfeld }
142583cd330SHans Rosenfeld
143583cd330SHans Rosenfeld /*
144583cd330SHans Rosenfeld * This is the first time we are enabling X2APIC on this CPU
145583cd330SHans Rosenfeld */
146583cd330SHans Rosenfeld apic_base_msr = rdmsr(REG_APIC_BASE_MSR);
147583cd330SHans Rosenfeld apic_base_msr = apic_base_msr | (0x1 << X2APIC_ENABLE_BIT);
148583cd330SHans Rosenfeld wrmsr(REG_APIC_BASE_MSR, apic_base_msr);
149583cd330SHans Rosenfeld
150583cd330SHans Rosenfeld if (apic_mode != LOCAL_X2APIC)
151583cd330SHans Rosenfeld x2apic_update_psm();
152583cd330SHans Rosenfeld }
153583cd330SHans Rosenfeld
154583cd330SHans Rosenfeld /*
155583cd330SHans Rosenfeld * Change apic_reg_ops depending upon the apic_mode.
156583cd330SHans Rosenfeld */
157583cd330SHans Rosenfeld void
apic_change_ops()158583cd330SHans Rosenfeld apic_change_ops()
159583cd330SHans Rosenfeld {
160583cd330SHans Rosenfeld if (apic_mode == LOCAL_APIC)
161583cd330SHans Rosenfeld apic_reg_ops = &local_apic_regs_ops;
162583cd330SHans Rosenfeld else if (apic_mode == LOCAL_X2APIC)
163583cd330SHans Rosenfeld apic_reg_ops = &x2apic_regs_ops;
164583cd330SHans Rosenfeld }
165583cd330SHans Rosenfeld
166583cd330SHans Rosenfeld /*
167583cd330SHans Rosenfeld * Generates an interprocessor interrupt to another CPU when X2APIC mode is
168583cd330SHans Rosenfeld * enabled.
169583cd330SHans Rosenfeld */
170583cd330SHans Rosenfeld void
x2apic_send_ipi(int cpun,int ipl)171583cd330SHans Rosenfeld x2apic_send_ipi(int cpun, int ipl)
172583cd330SHans Rosenfeld {
173583cd330SHans Rosenfeld int vector;
174583cd330SHans Rosenfeld ulong_t flag;
175583cd330SHans Rosenfeld
176583cd330SHans Rosenfeld ASSERT(apic_mode == LOCAL_X2APIC);
177583cd330SHans Rosenfeld
178583cd330SHans Rosenfeld /*
179583cd330SHans Rosenfeld * With X2APIC, Intel relaxed the semantics of the
180583cd330SHans Rosenfeld * WRMSR instruction such that references to the X2APIC
181583cd330SHans Rosenfeld * MSR registers are no longer serializing instructions.
182583cd330SHans Rosenfeld * The code that initiates IPIs assumes that some sort
183583cd330SHans Rosenfeld * of memory serialization occurs. The old APIC code
184583cd330SHans Rosenfeld * did a write to uncachable memory mapped registers.
185583cd330SHans Rosenfeld * Any reference to uncached memory is a serializing
186583cd330SHans Rosenfeld * operation. To mimic those semantics here, we do an
187583cd330SHans Rosenfeld * atomic operation, which translates to a LOCK OR instruction,
188583cd330SHans Rosenfeld * which is serializing.
189583cd330SHans Rosenfeld */
190583cd330SHans Rosenfeld atomic_or_ulong(&flag, 1);
191583cd330SHans Rosenfeld
192583cd330SHans Rosenfeld vector = apic_resv_vector[ipl];
193583cd330SHans Rosenfeld
194583cd330SHans Rosenfeld flag = intr_clear();
195583cd330SHans Rosenfeld
196583cd330SHans Rosenfeld /*
197583cd330SHans Rosenfeld * According to X2APIC specification in section '2.3.5.1' of
198583cd330SHans Rosenfeld * Interrupt Command Register Semantics, the semantics of
199583cd330SHans Rosenfeld * programming Interrupt Command Register to dispatch an interrupt
200583cd330SHans Rosenfeld * is simplified. A single MSR write to the 64-bit ICR is required
201583cd330SHans Rosenfeld * for dispatching an interrupt. Specifically with the 64-bit MSR
202583cd330SHans Rosenfeld * interface to ICR, system software is not required to check the
203583cd330SHans Rosenfeld * status of the delivery status bit prior to writing to the ICR
204583cd330SHans Rosenfeld * to send an IPI. With the removal of the Delivery Status bit,
205583cd330SHans Rosenfeld * system software no longer has a reason to read the ICR. It remains
206583cd330SHans Rosenfeld * readable only to aid in debugging.
207583cd330SHans Rosenfeld */
208583cd330SHans Rosenfeld #ifdef DEBUG
209583cd330SHans Rosenfeld APIC_AV_PENDING_SET();
210583cd330SHans Rosenfeld #endif /* DEBUG */
211583cd330SHans Rosenfeld
212583cd330SHans Rosenfeld if ((cpun == psm_get_cpu_id())) {
213583cd330SHans Rosenfeld X2APIC_WRITE(X2APIC_SELF_IPI, vector);
214583cd330SHans Rosenfeld } else {
215583cd330SHans Rosenfeld apic_reg_ops->apic_write_int_cmd(
216583cd330SHans Rosenfeld apic_cpus[cpun].aci_local_id, vector);
217583cd330SHans Rosenfeld }
218583cd330SHans Rosenfeld
219583cd330SHans Rosenfeld intr_restore(flag);
220583cd330SHans Rosenfeld }
221583cd330SHans Rosenfeld
222*1c2d0470SPatrick Mooney void
x2apic_send_pir_ipi(processorid_t cpun)223*1c2d0470SPatrick Mooney x2apic_send_pir_ipi(processorid_t cpun)
224*1c2d0470SPatrick Mooney {
225*1c2d0470SPatrick Mooney const int vector = apic_pir_vect;
226*1c2d0470SPatrick Mooney ulong_t flag;
227*1c2d0470SPatrick Mooney
228*1c2d0470SPatrick Mooney ASSERT(apic_mode == LOCAL_X2APIC);
229*1c2d0470SPatrick Mooney ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
230*1c2d0470SPatrick Mooney
231*1c2d0470SPatrick Mooney /* Serialize as described in x2apic_send_ipi() above. */
232*1c2d0470SPatrick Mooney atomic_or_ulong(&flag, 1);
233*1c2d0470SPatrick Mooney
234*1c2d0470SPatrick Mooney flag = intr_clear();
235*1c2d0470SPatrick Mooney
236*1c2d0470SPatrick Mooney /* Self-IPI for inducing PIR makes no sense. */
237*1c2d0470SPatrick Mooney if ((cpun != psm_get_cpu_id())) {
238*1c2d0470SPatrick Mooney #ifdef DEBUG
239*1c2d0470SPatrick Mooney /* Only for debugging. (again, see: x2apic_send_ipi) */
240*1c2d0470SPatrick Mooney APIC_AV_PENDING_SET();
241*1c2d0470SPatrick Mooney #endif /* DEBUG */
242*1c2d0470SPatrick Mooney
243*1c2d0470SPatrick Mooney apic_reg_ops->apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
244*1c2d0470SPatrick Mooney vector);
245*1c2d0470SPatrick Mooney }
246*1c2d0470SPatrick Mooney
247*1c2d0470SPatrick Mooney intr_restore(flag);
248*1c2d0470SPatrick Mooney }
249*1c2d0470SPatrick Mooney
250583cd330SHans Rosenfeld /*
251583cd330SHans Rosenfeld * Generates IPI to another CPU depending on the local APIC mode.
252583cd330SHans Rosenfeld * apic_send_ipi() and x2apic_send_ipi() depends on the configured
253583cd330SHans Rosenfeld * mode of the local APIC, but that may not match the actual mode
254583cd330SHans Rosenfeld * early in CPU startup.
255583cd330SHans Rosenfeld *
256583cd330SHans Rosenfeld * Any changes made to this routine must be accompanied by similar
257583cd330SHans Rosenfeld * changes to apic_send_ipi().
258583cd330SHans Rosenfeld */
259583cd330SHans Rosenfeld void
apic_common_send_ipi(int cpun,int ipl)260583cd330SHans Rosenfeld apic_common_send_ipi(int cpun, int ipl)
261583cd330SHans Rosenfeld {
262583cd330SHans Rosenfeld int vector;
263583cd330SHans Rosenfeld ulong_t flag;
264583cd330SHans Rosenfeld int mode = apic_local_mode();
265583cd330SHans Rosenfeld
266583cd330SHans Rosenfeld if (mode == LOCAL_X2APIC) {
267583cd330SHans Rosenfeld x2apic_send_ipi(cpun, ipl);
268583cd330SHans Rosenfeld return;
269583cd330SHans Rosenfeld }
270583cd330SHans Rosenfeld
271583cd330SHans Rosenfeld ASSERT(mode == LOCAL_APIC);
272583cd330SHans Rosenfeld
273583cd330SHans Rosenfeld vector = apic_resv_vector[ipl];
274583cd330SHans Rosenfeld ASSERT((vector >= APIC_BASE_VECT) && (vector <= APIC_SPUR_INTR));
275583cd330SHans Rosenfeld flag = intr_clear();
276583cd330SHans Rosenfeld while (local_apic_regs_ops.apic_read(APIC_INT_CMD1) & AV_PENDING)
277583cd330SHans Rosenfeld apic_ret();
278583cd330SHans Rosenfeld local_apic_regs_ops.apic_write_int_cmd(apic_cpus[cpun].aci_local_id,
279583cd330SHans Rosenfeld vector);
280583cd330SHans Rosenfeld intr_restore(flag);
281583cd330SHans Rosenfeld }
282*1c2d0470SPatrick Mooney
283*1c2d0470SPatrick Mooney void
apic_common_send_pir_ipi(processorid_t cpun)284*1c2d0470SPatrick Mooney apic_common_send_pir_ipi(processorid_t cpun)
285*1c2d0470SPatrick Mooney {
286*1c2d0470SPatrick Mooney const int mode = apic_local_mode();
287*1c2d0470SPatrick Mooney
288*1c2d0470SPatrick Mooney if (mode == LOCAL_X2APIC) {
289*1c2d0470SPatrick Mooney x2apic_send_pir_ipi(cpun);
290*1c2d0470SPatrick Mooney return;
291*1c2d0470SPatrick Mooney }
292*1c2d0470SPatrick Mooney
293*1c2d0470SPatrick Mooney apic_send_pir_ipi(cpun);
294*1c2d0470SPatrick Mooney }
295