103831d35Sstevel /*
203831d35Sstevel  * CDDL HEADER START
303831d35Sstevel  *
403831d35Sstevel  * The contents of this file are subject to the terms of the
503831d35Sstevel  * Common Development and Distribution License (the "License").
603831d35Sstevel  * You may not use this file except in compliance with the License.
703831d35Sstevel  *
803831d35Sstevel  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
903831d35Sstevel  * or http://www.opensolaris.org/os/licensing.
1003831d35Sstevel  * See the License for the specific language governing permissions
1103831d35Sstevel  * and limitations under the License.
1203831d35Sstevel  *
1303831d35Sstevel  * When distributing Covered Code, include this CDDL HEADER in each
1403831d35Sstevel  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
1503831d35Sstevel  * If applicable, add the following below this CDDL HEADER, with the
1603831d35Sstevel  * fields enclosed by brackets "[]" replaced with your own identifying
1703831d35Sstevel  * information: Portions Copyright [yyyy] [name of copyright owner]
1803831d35Sstevel  *
1903831d35Sstevel  * CDDL HEADER END
2003831d35Sstevel  */
2103831d35Sstevel 
2203831d35Sstevel /*
2307d06da5SSurya Prakki  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
2403831d35Sstevel  * Use is subject to license terms.
25*48bbca81SDaniel Hoffman  * Copyright (c) 2016 by Delphix. All rights reserved.
2603831d35Sstevel  */
2703831d35Sstevel 
2803831d35Sstevel /*
2903831d35Sstevel  * CPU management for serengeti DR
3003831d35Sstevel  *
3103831d35Sstevel  * There are three states a CPU can be in:
3203831d35Sstevel  *
3303831d35Sstevel  *	disconnected:		In reset
3403831d35Sstevel  *	connect,unconfigured:	Idling in OBP's idle loop
3503831d35Sstevel  *	configured:		Running Solaris
3603831d35Sstevel  *
3703831d35Sstevel  * State transitions:
3803831d35Sstevel  *
3903831d35Sstevel  *                connect              configure
4003831d35Sstevel  *              ------------>         ------------>
4103831d35Sstevel  * disconnected              connected             configured
4203831d35Sstevel  *                          unconfigured
4303831d35Sstevel  *              <-----------         <-------------
4403831d35Sstevel  *                disconnect           unconfigure
4503831d35Sstevel  *
4603831d35Sstevel  * Firmware involvements
4703831d35Sstevel  *
4803831d35Sstevel  *              start_cpu(SC)
4903831d35Sstevel  *      prom_serengeti_wakeupcpu(OBP)
5003831d35Sstevel  *              ------------>         ------------------------->
5103831d35Sstevel  * disconnected              connected                         configured
5203831d35Sstevel  *                          unconfigured
5303831d35Sstevel  *              <-----------          <-------------------------
5403831d35Sstevel  *      prom_serengeti_cpu_off(OBP)  prom_serengeti_cpu_off(OBP)
5503831d35Sstevel  *               stop_cpu(SC)        prom_serengeti_wakeupcpu(OBP)
5603831d35Sstevel  *
5703831d35Sstevel  * SIR (Software Initiated Reset) is used to unconfigure a CPU.
5803831d35Sstevel  * After the CPU has completed flushing the caches, it issues an
5903831d35Sstevel  * sir instruction to put itself through POST.  POST detects that
6003831d35Sstevel  * it is an SIR, and re-enters OBP as a slave.  When the operation
6103831d35Sstevel  * completes successfully, the CPU will be idling in OBP.
6203831d35Sstevel  */
6303831d35Sstevel 
6403831d35Sstevel #include <sys/obpdefs.h>
6503831d35Sstevel #include <sys/types.h>
6603831d35Sstevel #include <sys/cmn_err.h>
6703831d35Sstevel #include <sys/cpuvar.h>
6803831d35Sstevel #include <sys/membar.h>
6903831d35Sstevel #include <sys/x_call.h>
7003831d35Sstevel #include <sys/machsystm.h>
7103831d35Sstevel #include <sys/cpu_sgnblk_defs.h>
7203831d35Sstevel #include <sys/pte.h>
7303831d35Sstevel #include <vm/hat_sfmmu.h>
7403831d35Sstevel #include <sys/promif.h>
7503831d35Sstevel #include <sys/note.h>
7603831d35Sstevel #include <sys/vmsystm.h>
7703831d35Sstevel #include <vm/seg_kmem.h>
7803831d35Sstevel 
7903831d35Sstevel #include <sys/sbd_ioctl.h>
8003831d35Sstevel #include <sys/sbd.h>
8103831d35Sstevel #include <sys/sbdp_priv.h>
8203831d35Sstevel #include <sys/sbdp_mem.h>
8303831d35Sstevel #include <sys/sbdp_error.h>
8403831d35Sstevel #include <sys/sgsbbc_iosram.h>
8503831d35Sstevel #include <sys/prom_plat.h>
8603831d35Sstevel #include <sys/cheetahregs.h>
8703831d35Sstevel 
8803831d35Sstevel uint64_t	*sbdp_valp;
8903831d35Sstevel extern uint64_t	va_to_pa(void *);
9003831d35Sstevel static int	sbdp_cpu_ntries = 50000;
9103831d35Sstevel static int	sbdp_cpu_delay = 100;
9203831d35Sstevel void		sbdp_get_cpu_sram_addr(uint64_t, uint64_t);
9303831d35Sstevel static int	cpusram_map(caddr_t *, pgcnt_t *);
9403831d35Sstevel static void	cpusram_unmap(caddr_t *, pgcnt_t);
9503831d35Sstevel extern int	prom_serengeti_wakeupcpu(pnode_t);
9603831d35Sstevel extern int	prom_serengeti_cpu_off(pnode_t);
9703831d35Sstevel extern sbdp_wnode_t *sbdp_get_wnodep(int);
9803831d35Sstevel extern caddr_t	sbdp_shutdown_va;
9903831d35Sstevel static int	sbdp_prom_get_cpu(void *arg, int changed);
1008682d1efSRichard Lowe static void	sbdp_cpu_shutdown_self(void);
10103831d35Sstevel 
10203831d35Sstevel int
sbdp_disconnect_cpu(sbdp_handle_t * hp,dev_info_t * dip,processorid_t cpuid)10303831d35Sstevel sbdp_disconnect_cpu(sbdp_handle_t *hp, dev_info_t *dip, processorid_t cpuid)
10403831d35Sstevel {
10503831d35Sstevel 	pnode_t		nodeid;
10603831d35Sstevel 	int		bd, wnode;
10703831d35Sstevel 	sbdp_wnode_t	*wnodep;
10803831d35Sstevel 	sbdp_bd_t	*bdp = NULL;
10903831d35Sstevel 	int		rv = 0;
11003831d35Sstevel 	processorid_t	cpu = cpuid;
11103831d35Sstevel 	processorid_t	portid;
11203831d35Sstevel 	static fn_t	f = "sbdp_disconnect_cpu";
11303831d35Sstevel 
11403831d35Sstevel 	SBDP_DBG_FUNC("%s\n", f);
11503831d35Sstevel 
11603831d35Sstevel 	nodeid = ddi_get_nodeid(dip);
11703831d35Sstevel 
11803831d35Sstevel 	/*
11903831d35Sstevel 	 * Get board number and node number
12003831d35Sstevel 	 * The check for determining if nodeid is valid is done inside
12103831d35Sstevel 	 * sbdp_get_bd_and_wnode_num.
12203831d35Sstevel 	 */
12303831d35Sstevel 	if (SBDP_INJECT_ERROR(f, 0) ||
12403831d35Sstevel 	    sbdp_get_bd_and_wnode_num(nodeid, &bd, &wnode) != 0) {
12503831d35Sstevel 
12603831d35Sstevel 		rv = -1;
12703831d35Sstevel 		goto out;
12803831d35Sstevel 	}
12903831d35Sstevel 
13003831d35Sstevel 	/*
13103831d35Sstevel 	 * Grab the lock to prevent status threads from accessing
13203831d35Sstevel 	 * registers on the CPU when it is being put into reset.
13303831d35Sstevel 	 */
13403831d35Sstevel 	wnodep = sbdp_get_wnodep(wnode);
13503831d35Sstevel 	bdp = &wnodep->bds[bd];
13603831d35Sstevel 	ASSERT(bdp);
13703831d35Sstevel 	mutex_enter(&bdp->bd_mutex);
13803831d35Sstevel 
13903831d35Sstevel 	/*
14003831d35Sstevel 	 * Mark the CPU in reset.  This should be done before calling
14103831d35Sstevel 	 * the SC because we won't know at which stage it failed if
14203831d35Sstevel 	 * the SC call returns failure.
14303831d35Sstevel 	 */
14403831d35Sstevel 	sbdp_cpu_in_reset(wnode, bd, SG_CPUID_TO_CPU_UNIT(cpuid), 1);
14503831d35Sstevel 
14603831d35Sstevel 	/*
14703831d35Sstevel 	 * Ask OBP to mark the CPU as in POST
14803831d35Sstevel 	 */
14903831d35Sstevel 	if (SBDP_INJECT_ERROR(f, 1) || prom_serengeti_cpu_off(nodeid) != 0) {
15003831d35Sstevel 
15103831d35Sstevel 		rv = -1;
15203831d35Sstevel 		goto out;
15303831d35Sstevel 	}
15403831d35Sstevel 
15503831d35Sstevel 	/*
15603831d35Sstevel 	 * Ask the SC to put the CPU into reset. If the first
15703831d35Sstevel 	 * core is not present, the stop CPU interface needs
15803831d35Sstevel 	 * to be called with the portid rather than the cpuid.
15903831d35Sstevel 	 */
16003831d35Sstevel 	portid = SG_CPUID_TO_PORTID(cpuid);
16103831d35Sstevel 	if (!SBDP_IS_CPU_PRESENT(bdp, SG_CPUID_TO_CPU_UNIT(portid))) {
16203831d35Sstevel 		cpu = portid;
16303831d35Sstevel 	}
16403831d35Sstevel 
16503831d35Sstevel 	if (SBDP_INJECT_ERROR(f, 2) || sbdp_stop_cpu(cpu) != 0) {
16603831d35Sstevel 
16703831d35Sstevel 		rv = -1;
16803831d35Sstevel 		goto out;
16903831d35Sstevel 	}
17003831d35Sstevel 
17103831d35Sstevel out:
17203831d35Sstevel 	if (bdp != NULL) {
17303831d35Sstevel 		mutex_exit(&bdp->bd_mutex);
17403831d35Sstevel 	}
17503831d35Sstevel 
17603831d35Sstevel 	if (rv != 0) {
17703831d35Sstevel 		sbdp_set_err(hp->h_err, ESGT_STOPCPU, NULL);
17803831d35Sstevel 	}
17903831d35Sstevel 
18003831d35Sstevel 	return (rv);
18103831d35Sstevel }
18203831d35Sstevel 
18303831d35Sstevel int
sbdp_connect_cpu(sbdp_handle_t * hp,dev_info_t * dip,processorid_t cpuid)18403831d35Sstevel sbdp_connect_cpu(sbdp_handle_t *hp, dev_info_t *dip, processorid_t cpuid)
18503831d35Sstevel {
18603831d35Sstevel 	pnode_t		nodeid;
18703831d35Sstevel 	sbd_error_t	*sep;
18803831d35Sstevel 	int		i;
18903831d35Sstevel 	int		bd, wnode;
19003831d35Sstevel 	int		rv = 0;
19103831d35Sstevel 	static fn_t	f = "sbdp_connect_cpu";
19203831d35Sstevel 
19303831d35Sstevel 	SBDP_DBG_FUNC("%s\n", f);
19403831d35Sstevel 
19503831d35Sstevel 	sep = hp->h_err;
19603831d35Sstevel 
19703831d35Sstevel 	nodeid = ddi_get_nodeid(dip);
19803831d35Sstevel 
19903831d35Sstevel 	/*
20003831d35Sstevel 	 * The check for determining if nodeid is valid is done inside
20103831d35Sstevel 	 * sbdp_get_bd_and_wnode_num.
20203831d35Sstevel 	 */
20303831d35Sstevel 	if (SBDP_INJECT_ERROR(f, 0) ||
20403831d35Sstevel 	    sbdp_get_bd_and_wnode_num(nodeid, &bd, &wnode) != 0) {
20503831d35Sstevel 
20603831d35Sstevel 		rv = -1;
20703831d35Sstevel 		goto out;
20803831d35Sstevel 	}
20903831d35Sstevel 
21003831d35Sstevel 	/*
21103831d35Sstevel 	 * Ask the SC to bring the CPU out of reset.
21203831d35Sstevel 	 * At this point, the sb_dev_present bit is not set for the CPU.
21303831d35Sstevel 	 * From sbd point of view the CPU is not present yet.  No
21403831d35Sstevel 	 * status threads will try to read registers off the CPU.
21503831d35Sstevel 	 * Since we are already holding sb_mutex, it is not necessary
21603831d35Sstevel 	 * to grab the board mutex when checking and setting the
21703831d35Sstevel 	 * cpus_in_reset bit.
21803831d35Sstevel 	 */
21903831d35Sstevel 	if (sbdp_is_cpu_in_reset(wnode, bd, SG_CPUID_TO_CPU_UNIT(cpuid))) {
22003831d35Sstevel 
22103831d35Sstevel 		sbdp_wnode_t	*wnodep;
22203831d35Sstevel 		sbdp_bd_t	*bdp = NULL;
22303831d35Sstevel 		processorid_t	cpu = cpuid;
22403831d35Sstevel 		processorid_t	portid;
22503831d35Sstevel 
22603831d35Sstevel 		wnodep = sbdp_get_wnodep(wnode);
22703831d35Sstevel 		bdp = &wnodep->bds[bd];
22803831d35Sstevel 		ASSERT(bdp);
22903831d35Sstevel 
23003831d35Sstevel 		/*
23103831d35Sstevel 		 * If the first core is not present, the start CPU
23203831d35Sstevel 		 * interface needs to be called with the portid rather
23303831d35Sstevel 		 * than the cpuid.
23403831d35Sstevel 		 */
23503831d35Sstevel 		portid = SG_CPUID_TO_PORTID(cpuid);
23603831d35Sstevel 		if (!SBDP_IS_CPU_PRESENT(bdp, SG_CPUID_TO_CPU_UNIT(portid))) {
23703831d35Sstevel 			cpu = portid;
23803831d35Sstevel 		}
23903831d35Sstevel 
24003831d35Sstevel 		if (SBDP_INJECT_ERROR(f, 1) || sbdp_start_cpu(cpu) != 0) {
24103831d35Sstevel 
24203831d35Sstevel 			rv = -1;
24303831d35Sstevel 			goto out;
24403831d35Sstevel 		}
24503831d35Sstevel 
24603831d35Sstevel 		if (SBDP_INJECT_ERROR(f, 2) ||
24703831d35Sstevel 		    prom_serengeti_wakeupcpu(nodeid) != 0) {
24803831d35Sstevel 
24903831d35Sstevel 			rv = -1;
25003831d35Sstevel 			goto out;
25103831d35Sstevel 		}
25203831d35Sstevel 	}
25303831d35Sstevel 
25403831d35Sstevel 	/*
25503831d35Sstevel 	 * Mark the CPU out of reset.
25603831d35Sstevel 	 */
25703831d35Sstevel 	sbdp_cpu_in_reset(wnode, bd, SG_CPUID_TO_CPU_UNIT(cpuid), 0);
25803831d35Sstevel 
25903831d35Sstevel 	/*
26003831d35Sstevel 	 * Refresh the bd info
26103831d35Sstevel 	 * we need to wait until all cpus are out of reset
26203831d35Sstevel 	 */
26303831d35Sstevel 	for (i = 0; i < SG_MAX_CPUS_PER_BD; i++)
26403831d35Sstevel 		if (sbdp_is_cpu_present(wnode, bd, i) &&
26503831d35Sstevel 		    sbdp_is_cpu_in_reset(wnode, bd, i) == 1) {
26603831d35Sstevel 			break;
26703831d35Sstevel 		}
26803831d35Sstevel 
26903831d35Sstevel 	if (i == SG_MAX_CPUS_PER_BD) {
27003831d35Sstevel 		/*
27103831d35Sstevel 		 * All cpus are out of reset so it is safe to
27203831d35Sstevel 		 * update the bd info
27303831d35Sstevel 		 */
27403831d35Sstevel 		sbdp_add_new_bd_info(wnode, bd);
27503831d35Sstevel 	}
276