1/*-
2 * Copyright (c) 2015 The FreeBSD Foundation
3 * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com>
4 * All rights reserved.
5 *
6 * Portions of this software were developed by Andrew Turner under
7 * sponsorship from the FreeBSD Foundation.
8 *
9 * Portions of this software were developed by SRI International and the
10 * University of Cambridge Computer Laboratory under DARPA/AFRL contract
11 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme.
12 *
13 * Portions of this software were developed by the University of Cambridge
14 * Computer Laboratory as part of the CTSRD Project, with support from the
15 * UK Higher Education Innovation Fund (HEIF).
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 * 1. Redistributions of source code must retain the above copyright
21 *    notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 *    notice, this list of conditions and the following disclaimer in the
24 *    documentation and/or other materials provided with the distribution.
25 *
26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
29 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
36 * SUCH DAMAGE.
37 */
38
39#include "opt_kstack_pages.h"
40#include "opt_platform.h"
41
42#include <sys/cdefs.h>
43__FBSDID("$FreeBSD$");
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/bus.h>
48#include <sys/cpu.h>
49#include <sys/kernel.h>
50#include <sys/ktr.h>
51#include <sys/malloc.h>
52#include <sys/module.h>
53#include <sys/mutex.h>
54#include <sys/proc.h>
55#include <sys/sched.h>
56#include <sys/smp.h>
57
58#include <vm/vm.h>
59#include <vm/pmap.h>
60#include <vm/vm_extern.h>
61#include <vm/vm_kern.h>
62#include <vm/vm_map.h>
63
64#include <machine/intr.h>
65#include <machine/smp.h>
66#include <machine/sbi.h>
67
68#ifdef FDT
69#include <dev/ofw/openfirm.h>
70#include <dev/ofw/ofw_cpu.h>
71#endif
72
73boolean_t ofw_cpu_reg(phandle_t node, u_int, cell_t *);
74
75uint32_t __riscv_boot_ap[MAXCPU];
76
77static enum {
78	CPUS_UNKNOWN,
79#ifdef FDT
80	CPUS_FDT,
81#endif
82} cpu_enum_method;
83
84static device_identify_t riscv64_cpu_identify;
85static device_probe_t riscv64_cpu_probe;
86static device_attach_t riscv64_cpu_attach;
87
88static int ipi_handler(void *);
89
90struct pcb stoppcbs[MAXCPU];
91
92extern uint32_t boot_hart;
93extern cpuset_t all_harts;
94
95#ifdef INVARIANTS
96static uint32_t cpu_reg[MAXCPU][2];
97#endif
98static device_t cpu_list[MAXCPU];
99
100void mpentry(u_long hartid);
101void init_secondary(uint64_t);
102
103static struct mtx ap_boot_mtx;
104
105/* Stacks for AP initialization, discarded once idle threads are started. */
106void *bootstack;
107static void *bootstacks[MAXCPU];
108
109/* Count of started APs, used to synchronize access to bootstack. */
110static volatile int aps_started;
111
112/* Set to 1 once we're ready to let the APs out of the pen. */
113static volatile int aps_ready;
114
115/* Temporary variables for init_secondary()  */
116void *dpcpu[MAXCPU - 1];
117
118static device_method_t riscv64_cpu_methods[] = {
119	/* Device interface */
120	DEVMETHOD(device_identify,	riscv64_cpu_identify),
121	DEVMETHOD(device_probe,		riscv64_cpu_probe),
122	DEVMETHOD(device_attach,	riscv64_cpu_attach),
123
124	DEVMETHOD_END
125};
126
127static devclass_t riscv64_cpu_devclass;
128static driver_t riscv64_cpu_driver = {
129	"riscv64_cpu",
130	riscv64_cpu_methods,
131	0
132};
133
134DRIVER_MODULE(riscv64_cpu, cpu, riscv64_cpu_driver, riscv64_cpu_devclass, 0, 0);
135
136static void
137riscv64_cpu_identify(driver_t *driver, device_t parent)
138{
139
140	if (device_find_child(parent, "riscv64_cpu", -1) != NULL)
141		return;
142	if (BUS_ADD_CHILD(parent, 0, "riscv64_cpu", -1) == NULL)
143		device_printf(parent, "add child failed\n");
144}
145
146static int
147riscv64_cpu_probe(device_t dev)
148{
149	u_int cpuid;
150
151	cpuid = device_get_unit(dev);
152	if (cpuid >= MAXCPU || cpuid > mp_maxid)
153		return (EINVAL);
154
155	device_quiet(dev);
156	return (0);
157}
158
159static int
160riscv64_cpu_attach(device_t dev)
161{
162	const uint32_t *reg;
163	size_t reg_size;
164	u_int cpuid;
165	int i;
166
167	cpuid = device_get_unit(dev);
168
169	if (cpuid >= MAXCPU || cpuid > mp_maxid)
170		return (EINVAL);
171	KASSERT(cpu_list[cpuid] == NULL, ("Already have cpu %u", cpuid));
172
173	reg = cpu_get_cpuid(dev, &reg_size);
174	if (reg == NULL)
175		return (EINVAL);
176
177	if (bootverbose) {
178		device_printf(dev, "register <");
179		for (i = 0; i < reg_size; i++)
180			printf("%s%x", (i == 0) ? "" : " ", reg[i]);
181		printf(">\n");
182	}
183
184	/* Set the device to start it later */
185	cpu_list[cpuid] = dev;
186
187	return (0);
188}
189
190static void
191release_aps(void *dummy __unused)
192{
193	cpuset_t mask;
194	int i;
195
196	if (mp_ncpus == 1)
197		return;
198
199	/* Setup the IPI handler */
200	riscv_setup_ipihandler(ipi_handler);
201
202	atomic_store_rel_int(&aps_ready, 1);
203
204	/* Wake up the other CPUs */
205	mask = all_harts;
206	CPU_CLR(boot_hart, &mask);
207
208	printf("Release APs\n");
209
210	sbi_send_ipi(mask.__bits);
211
212	for (i = 0; i < 2000; i++) {
213		if (smp_started)
214			return;
215		DELAY(1000);
216	}
217
218	printf("APs not started\n");
219}
220SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL);
221
222void
223init_secondary(uint64_t hart)
224{
225	struct pcpu *pcpup;
226	u_int cpuid;
227
228	/* Renumber this cpu */
229	cpuid = hart;
230	if (cpuid < boot_hart)
231		cpuid += mp_maxid + 1;
232	cpuid -= boot_hart;
233
234	/* Setup the pcpu pointer */
235	pcpup = &__pcpu[cpuid];
236	__asm __volatile("mv tp, %0" :: "r"(pcpup));
237
238	/* Workaround: make sure wfi doesn't halt the hart */
239	csr_set(sie, SIE_SSIE);
240	csr_set(sip, SIE_SSIE);
241
242	/* Signal the BSP and spin until it has released all APs. */
243	atomic_add_int(&aps_started, 1);
244	while (!atomic_load_int(&aps_ready))
245		__asm __volatile("wfi");
246
247	/* Initialize curthread */
248	KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread"));
249	pcpup->pc_curthread = pcpup->pc_idlethread;
250
251	/*
252	 * Identify current CPU. This is necessary to setup
253	 * affinity registers and to provide support for
254	 * runtime chip identification.
255	 */
256	identify_cpu();
257
258	/* Enable software interrupts */
259	riscv_unmask_ipi();
260
261#ifndef EARLY_AP_STARTUP
262	/* Start per-CPU event timers. */
263	cpu_initclocks_ap();
264#endif
265
266	/* Enable external (PLIC) interrupts */
267	csr_set(sie, SIE_SEIE);
268
269	/* Activate process 0's pmap. */
270	pmap_activate_boot(vmspace_pmap(proc0.p_vmspace));
271
272	mtx_lock_spin(&ap_boot_mtx);
273
274	atomic_add_rel_32(&smp_cpus, 1);
275
276	if (smp_cpus == mp_ncpus) {
277		/* enable IPI's, tlb shootdown, freezes etc */
278		atomic_store_rel_int(&smp_started, 1);
279	}
280
281	mtx_unlock_spin(&ap_boot_mtx);
282
283	/*
284	 * Assert that smp_after_idle_runnable condition is reasonable.
285	 */
286	MPASS(PCPU_GET(curpcb) == NULL);
287
288	/* Enter the scheduler */
289	sched_throw(NULL);
290
291	panic("scheduler returned us to init_secondary");
292	/* NOTREACHED */
293}
294
295static void
296smp_after_idle_runnable(void *arg __unused)
297{
298	struct pcpu *pc;
299	int cpu;
300
301	for (cpu = 1; cpu <= mp_maxid; cpu++) {
302		if (bootstacks[cpu] != NULL) {
303			pc = pcpu_find(cpu);
304			while (atomic_load_ptr(&pc->pc_curpcb) == NULL)
305				cpu_spinwait();
306			kmem_free((vm_offset_t)bootstacks[cpu], PAGE_SIZE);
307		}
308	}
309}
310SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY,
311    smp_after_idle_runnable, NULL);
312
313static int
314ipi_handler(void *arg)
315{
316	u_int ipi_bitmap;
317	u_int cpu, ipi;
318	int bit;
319
320	sbi_clear_ipi();
321
322	cpu = PCPU_GET(cpuid);
323
324	mb();
325
326	ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis));
327	if (ipi_bitmap == 0)
328		return (FILTER_HANDLED);
329
330	while ((bit = ffs(ipi_bitmap))) {
331		bit = (bit - 1);
332		ipi = (1 << bit);
333		ipi_bitmap &= ~ipi;
334
335		mb();
336
337		switch (ipi) {
338		case IPI_AST:
339			CTR0(KTR_SMP, "IPI_AST");
340			break;
341		case IPI_PREEMPT:
342			CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__);
343			sched_preempt(curthread);
344			break;
345		case IPI_RENDEZVOUS:
346			CTR0(KTR_SMP, "IPI_RENDEZVOUS");
347			smp_rendezvous_action();
348			break;
349		case IPI_STOP:
350		case IPI_STOP_HARD:
351			CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD");
352			savectx(&stoppcbs[cpu]);
353
354			/* Indicate we are stopped */
355			CPU_SET_ATOMIC(cpu, &stopped_cpus);
356
357			/* Wait for restart */
358			while (!CPU_ISSET(cpu, &started_cpus))
359				cpu_spinwait();
360
361			CPU_CLR_ATOMIC(cpu, &started_cpus);
362			CPU_CLR_ATOMIC(cpu, &stopped_cpus);
363			CTR0(KTR_SMP, "IPI_STOP (restart)");
364
365			/*
366			 * The kernel debugger might have set a breakpoint,
367			 * so flush the instruction cache.
368			 */
369			fence_i();
370			break;
371		case IPI_HARDCLOCK:
372			CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__);
373			hardclockintr();
374			break;
375		default:
376			panic("Unknown IPI %#0x on cpu %d", ipi, curcpu);
377		}
378	}
379
380	return (FILTER_HANDLED);
381}
382
383struct cpu_group *
384cpu_topo(void)
385{
386
387	return (smp_topo_none());
388}
389
390/* Determine if we running MP machine */
391int
392cpu_mp_probe(void)
393{
394
395	return (mp_ncpus > 1);
396}
397
398#ifdef FDT
399static boolean_t
400cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg)
401{
402	struct pcpu *pcpup;
403	vm_paddr_t start_addr;
404	uint64_t hart;
405	u_int cpuid;
406	int naps;
407	int error;
408
409	/* Check if this hart supports MMU. */
410	if (OF_getproplen(node, "mmu-type") < 0)
411		return (0);
412
413	KASSERT(id < MAXCPU, ("Too many CPUs"));
414
415	KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size"));
416#ifdef INVARIANTS
417	cpu_reg[id][0] = reg[0];
418	if (addr_size == 2)
419		cpu_reg[id][1] = reg[1];
420#endif
421
422	hart = reg[0];
423	if (addr_size == 2) {
424		hart <<= 32;
425		hart |= reg[1];
426	}
427
428	KASSERT(hart < MAXCPU, ("Too many harts."));
429
430	/* We are already running on this cpu */
431	if (hart == boot_hart)
432		return (1);
433
434	/*
435	 * Rotate the CPU IDs to put the boot CPU as CPU 0.
436	 * We keep the other CPUs ordered.
437	 */
438	cpuid = hart;
439	if (cpuid < boot_hart)
440		cpuid += mp_maxid + 1;
441	cpuid -= boot_hart;
442
443	/* Check if we are able to start this cpu */
444	if (cpuid > mp_maxid)
445		return (0);
446
447	/*
448	 * Depending on the SBI implementation, APs are waiting either in
449	 * locore.S or to be activated explicitly, via SBI call.
450	 */
451	if (sbi_probe_extension(SBI_EXT_ID_HSM) != 0) {
452		start_addr = pmap_kextract((vm_offset_t)mpentry);
453		error = sbi_hsm_hart_start(hart, start_addr, 0);
454		if (error != 0) {
455			mp_ncpus--;
456
457			/* Send a warning to the user and continue. */
458			printf("AP %u (hart %lu) failed to start, error %d\n",
459			    cpuid, hart, error);
460			return (0);
461		}
462	}
463
464	pcpup = &__pcpu[cpuid];
465	pcpu_init(pcpup, cpuid, sizeof(struct pcpu));
466	pcpup->pc_hart = hart;
467
468	dpcpu[cpuid - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO);
469	dpcpu_init(dpcpu[cpuid - 1], cpuid);
470
471	bootstacks[cpuid] = (void *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO);
472
473	naps = atomic_load_int(&aps_started);
474	bootstack = (char *)bootstacks[cpuid] + PAGE_SIZE;
475
476	printf("Starting CPU %u (hart %lx)\n", cpuid, hart);
477	atomic_store_32(&__riscv_boot_ap[hart], 1);
478
479	/* Wait for the AP to switch to its boot stack. */
480	while (atomic_load_int(&aps_started) < naps + 1)
481		cpu_spinwait();
482
483	CPU_SET(cpuid, &all_cpus);
484	CPU_SET(hart, &all_harts);
485
486	return (1);
487}
488#endif
489
490/* Initialize and fire up non-boot processors */
491void
492cpu_mp_start(void)
493{
494
495	mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN);
496
497	CPU_SET(0, &all_cpus);
498	CPU_SET(boot_hart, &all_harts);
499
500	switch(cpu_enum_method) {
501#ifdef FDT
502	case CPUS_FDT:
503		ofw_cpu_early_foreach(cpu_init_fdt, true);
504		break;
505#endif
506	case CPUS_UNKNOWN:
507		break;
508	}
509}
510
511/* Introduce rest of cores to the world */
512void
513cpu_mp_announce(void)
514{
515}
516
517static boolean_t
518cpu_check_mmu(u_int id, phandle_t node, u_int addr_size, pcell_t *reg)
519{
520
521	/* Check if this hart supports MMU. */
522	if (OF_getproplen(node, "mmu-type") < 0)
523		return (0);
524
525	return (1);
526}
527
528void
529cpu_mp_setmaxid(void)
530{
531#ifdef FDT
532	int cores;
533
534	cores = ofw_cpu_early_foreach(cpu_check_mmu, true);
535	if (cores > 0) {
536		cores = MIN(cores, MAXCPU);
537		if (bootverbose)
538			printf("Found %d CPUs in the device tree\n", cores);
539		mp_ncpus = cores;
540		mp_maxid = cores - 1;
541		cpu_enum_method = CPUS_FDT;
542		return;
543	}
544#endif
545
546	if (bootverbose)
547		printf("No CPU data, limiting to 1 core\n");
548	mp_ncpus = 1;
549	mp_maxid = 0;
550}
551