/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * i86pc Memory Scrubbing
 *
 * On detection of a correctable memory ECC error, the i86pc hardware
 * returns the corrected data to the requester and may re-write it
 * to memory (DRAM or NVRAM). Machines which do not re-write this to
 * memory should add an NMI handler to correct and rewrite.
 *
 * Scrubbing thus reduces the likelyhood that multiple transient errors
 * will occur in the same memory word, making uncorrectable errors due
 * to transients less likely.
 *
 * Thus is born the desire that every memory location be periodically
 * accessed.
 *
 * This file implements a memory scrubbing thread.  This scrubber
 * guarantees that all of physical memory is accessed periodically
 * (memscrub_period_sec -- 12 hours).
 *
 * It attempts to do this as unobtrusively as possible.  The thread
 * schedules itself to wake up at an interval such that if it reads
 * memscrub_span_pages (4MB) on each wakeup, it will read all of physical
 * memory in in memscrub_period_sec (12 hours).
 *
 * The scrubber uses the REP LODS so it reads 4MB in 0.15 secs (on P5-200).
 * When it completes a span, if all the CPUs are idle, it reads another span.
 * Typically it soaks up idle time this way to reach its deadline early
 * -- and sleeps until the next period begins.
 *
 * Maximal Cost Estimate:  8GB @ xxMB/s = xxx seconds spent in 640 wakeups
 * that run for 0.15 seconds at intervals of 67 seconds.
 *
 * In practice, the scrubber finds enough idle time to finish in a few
 * minutes, and sleeps until its 12 hour deadline.
 *
 * The scrubber maintains a private copy of the phys_install memory list
 * to keep track of what memory should be scrubbed.
 *
 * The following parameters can be set via /etc/system
 *
 * memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES (4MB)
 * memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC (12 hours)
 * memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI (0)
 * memscrub_delay_start_sec = (10 seconds)
 * disable_memscrub = (0)
 *
 * the scrubber will exit (or never be started) if it finds the variable
 * "disable_memscrub" set.
 *
 * MEMSCRUB_DFL_SPAN_PAGES  is based on the guess that 0.15 sec
 * is a "good" amount of minimum time for the thread to run at a time.
 *
 * MEMSCRUB_DFL_PERIOD_SEC (12 hours) is nearly a total guess --
 * twice the frequency the hardware folk estimated would be necessary.
 *
 * MEMSCRUB_DFL_THREAD_PRI (0) is based on the assumption that nearly
 * any other use of the system should be higher priority than scrubbing.
 */

#include <sys/types.h>
#include <sys/systm.h>		/* timeout, types, t_lock */
#include <sys/cmn_err.h>
#include <sys/sysmacros.h>	/* MIN */
#include <sys/memlist.h>	/* memlist */
#include <sys/kmem.h>		/* KMEM_NOSLEEP */
#include <sys/cpuvar.h>		/* ncpus_online */
#include <sys/debug.h>		/* ASSERTs */
#include <sys/vmem.h>
#include <sys/mman.h>
#include <vm/seg_kmem.h>
#include <vm/seg_kpm.h>
#include <vm/hat_i86.h>
#include <sys/callb.h>		/* CPR callback */

static caddr_t	memscrub_window;
static hat_mempte_t memscrub_pte;

/*
 * Global Data:
 */
/*
 * scan all of physical memory at least once every MEMSCRUB_PERIOD_SEC
 */
#define	MEMSCRUB_DFL_PERIOD_SEC	(12 * 60 * 60)	/* 12 hours */

/*
 * start only if at least MEMSCRUB_MIN_PAGES in system
 */
#define	MEMSCRUB_MIN_PAGES	((32 * 1024 * 1024) / PAGESIZE)

/*
 * scan at least MEMSCRUB_DFL_SPAN_PAGES each iteration
 */
#define	MEMSCRUB_DFL_SPAN_PAGES	((4 * 1024 * 1024) / PAGESIZE)

/*
 * almost anything is higher priority than scrubbing
 */
#define	MEMSCRUB_DFL_THREAD_PRI	0

/*
 * we can patch these defaults in /etc/system if necessary
 */
uint_t disable_memscrub = 0;
static uint_t disable_memscrub_quietly = 0;
pgcnt_t memscrub_min_pages = MEMSCRUB_MIN_PAGES;
pgcnt_t memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES;
time_t memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC;
uint_t memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI;
time_t memscrub_delay_start_sec = 10;

/*
 * Static Routines
 */
static void memscrubber(void);
static int system_is_idle(void);
static int memscrub_add_span(uint64_t, uint64_t);

/*
 * Static Data
 */
static struct memlist *memscrub_memlist;
static uint_t memscrub_phys_pages;

static kcondvar_t memscrub_cv;
static kmutex_t memscrub_lock;

/*
 * memscrub_lock protects memscrub_memlist
 */
uint_t memscrub_scans_done;

uint_t memscrub_done_early;
uint_t memscrub_early_sec;

uint_t memscrub_done_late;
time_t memscrub_late_sec;

/*
 * create memscrub_memlist from phys_install list
 * initialize locks, set memscrub_phys_pages.
 */
void
memscrub_init()
{
	struct memlist *src;

	if (physmem < memscrub_min_pages)
		return;

	if (!kpm_enable) {
		memscrub_window = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
		memscrub_pte = hat_mempte_setup(memscrub_window);
	}

	/*
	 * copy phys_install to memscrub_memlist
	 */
	for (src = phys_install; src; src = src->ml_next) {
		if (memscrub_add_span(src->ml_address, src->ml_size)) {
			cmn_err(CE_WARN,
			    "Software memory scrubber failed to initialize\n");
			return;
		}
	}

	mutex_init(&memscrub_lock, NULL, MUTEX_DRIVER, NULL);
	cv_init(&memscrub_cv, NULL, CV_DRIVER, NULL);

	/*
	 * create memscrubber thread
	 */
	(void) thread_create(NULL, 0, (void (*)())memscrubber, NULL, 0, &p0,
	    TS_RUN, memscrub_thread_pri);
}

/*
 * Function to cause the software memscrubber to exit quietly if the
 * platform support has located a hardware scrubber and enabled it.
 */
void
memscrub_disable(void)
{
	disable_memscrub_quietly = 1;
}

#ifdef MEMSCRUB_DEBUG
static void
memscrub_printmemlist(char *title, struct memlist *listp)
{
	struct memlist *list;

	cmn_err(CE_CONT, "%s:\n", title);

	for (list = listp; list; list = list->next) {
		cmn_err(CE_CONT, "addr = 0x%llx, size = 0x%llx\n",
		    list->address, list->size);
	}
}
#endif /* MEMSCRUB_DEBUG */

/* ARGSUSED */
static void
memscrub_wakeup(void *c)
{
	/*
	 * grab mutex to guarantee that our wakeup call
	 * arrives after we go to sleep -- so we can't sleep forever.
	 */
	mutex_enter(&memscrub_lock);
	cv_signal(&memscrub_cv);
	mutex_exit(&memscrub_lock);
}

/*
 * this calculation doesn't account for the time that the actual scan
 * consumes -- so we'd fall slightly behind schedule with this
 * interval_sec.  but the idle loop optimization below usually makes us
 * come in way ahead of schedule.
 */
static int
compute_interval_sec()
{
	if (memscrub_phys_pages <= memscrub_span_pages)
		return (memscrub_period_sec);
	else
		return (memscrub_period_sec/
		    (memscrub_phys_pages/memscrub_span_pages));
}

static void
memscrubber()
{
	time_t deadline;
	uint64_t mlp_last_addr;
	uint64_t mlp_next_addr;
	int reached_end = 1;
	time_t interval_sec = 0;
	struct memlist *mlp;

	extern void scan_memory(caddr_t, size_t);
	callb_cpr_t cprinfo;

	/*
	 * notify CPR of our existence
	 */
	CALLB_CPR_INIT(&cprinfo, &memscrub_lock, callb_generic_cpr, "memscrub");

	if (memscrub_memlist == NULL) {
		cmn_err(CE_WARN, "memscrub_memlist not initialized.");
		goto memscrub_exit;
	}

	mlp = memscrub_memlist;
	mlp_next_addr = mlp->ml_address;
	mlp_last_addr = mlp->ml_address + mlp->ml_size;

	deadline = gethrestime_sec() + memscrub_delay_start_sec;

	for (;;) {
		if (disable_memscrub || disable_memscrub_quietly)
			break;

		mutex_enter(&memscrub_lock);

		/*
		 * did we just reach the end of memory?
		 */
		if (reached_end) {
			time_t now = gethrestime_sec();

			if (now >= deadline) {
				memscrub_done_late++;
				memscrub_late_sec += (now - deadline);
				/*
				 * past deadline, start right away
				 */
				interval_sec = 0;

				deadline = now + memscrub_period_sec;
			} else {
				/*
				 * we finished ahead of schedule.
				 * wait till previous dealine before re-start.
				 */
				interval_sec = deadline - now;
				memscrub_done_early++;
				memscrub_early_sec += interval_sec;
				deadline += memscrub_period_sec;
			}
		} else {
			interval_sec = compute_interval_sec();
		}

		/*
		 * it is safe from our standpoint for CPR to
		 * suspend the system
		 */
		CALLB_CPR_SAFE_BEGIN(&cprinfo);

		/*
		 * hit the snooze bar
		 */
		(void) timeout(memscrub_wakeup, NULL, interval_sec * hz);

		/*
		 * go to sleep
		 */
		cv_wait(&memscrub_cv, &memscrub_lock);

		/* we need to goto work */
		CALLB_CPR_SAFE_END(&cprinfo, &memscrub_lock);

		mutex_exit(&memscrub_lock);

		do {
			pgcnt_t pages = memscrub_span_pages;
			uint64_t address = mlp_next_addr;

			if (disable_memscrub || disable_memscrub_quietly)
				break;

			mutex_enter(&memscrub_lock);

			/*
			 * Make sure we don't try to scan beyond the end of
			 * the current memlist.  If we would, then resize
			 * our scan target for this iteration, and prepare
			 * to read the next memlist entry on the next
			 * iteration.
			 */
			reached_end = 0;
			if (address + mmu_ptob(pages) >= mlp_last_addr) {
				pages = mmu_btop(mlp_last_addr - address);
				mlp = mlp->ml_next;
				if (mlp == NULL) {
					reached_end = 1;
					mlp = memscrub_memlist;
				}
				mlp_next_addr = mlp->ml_address;
				mlp_last_addr = mlp->ml_address + mlp->ml_size;
			} else {
				mlp_next_addr += mmu_ptob(pages);
			}

			mutex_exit(&memscrub_lock);

			while (pages--) {
				pfn_t pfn = btop(address);

				/*
				 * Without segkpm, the memscrubber cannot
				 * be allowed to migrate across CPUs, as
				 * the CPU-specific mapping of
				 * memscrub_window would be incorrect.
				 * With segkpm, switching CPUs is legal, but
				 * inefficient.  We don't use
				 * kpreempt_disable as it might hold a
				 * higher priority thread (eg, RT) too long
				 * off CPU.
				 */
				thread_affinity_set(curthread, CPU_CURRENT);
				if (kpm_enable)
					memscrub_window = hat_kpm_pfn2va(pfn);
				else
					hat_mempte_remap(pfn, memscrub_window,
					    memscrub_pte,
					    PROT_READ, HAT_LOAD_NOCONSIST);

				scan_memory(memscrub_window, PAGESIZE);

				thread_affinity_clear(curthread);
				address += MMU_PAGESIZE;
			}

			memscrub_scans_done++;
		} while (!reached_end && system_is_idle());
	}

memscrub_exit:

	if (!disable_memscrub_quietly)
		cmn_err(CE_NOTE, "Software memory scrubber exiting.");
	/*
	 * We are about to bail, but don't have the memscrub_lock,
	 * and it is needed for CALLB_CPR_EXIT.
	 */
	mutex_enter(&memscrub_lock);
	CALLB_CPR_EXIT(&cprinfo);

	cv_destroy(&memscrub_cv);

	thread_exit();
}


/*
 * return 1 if we're MP and all the other CPUs are idle
 */
static int
system_is_idle()
{
	int cpu_id;
	int found = 0;

	if (1 == ncpus_online)
		return (0);

	for (cpu_id = 0; cpu_id < NCPU; ++cpu_id) {
		if (!cpu[cpu_id])
			continue;

		found++;

		if (cpu[cpu_id]->cpu_thread != cpu[cpu_id]->cpu_idle_thread) {
			if (CPU->cpu_id == cpu_id &&
			    CPU->cpu_disp->disp_nrunnable == 0)
				continue;
			return (0);
		}

		if (found == ncpus)
			break;
	}
	return (1);
}

/*
 * add a span to the memscrub list
 */
static int
memscrub_add_span(uint64_t start, uint64_t bytes)
{
	struct memlist *dst;
	struct memlist *prev, *next;
	uint64_t end = start + bytes - 1;
	int retval = 0;

	mutex_enter(&memscrub_lock);

#ifdef MEMSCRUB_DEBUG
	memscrub_printmemlist("memscrub_memlist before", memscrub_memlist);
	cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
	cmn_err(CE_CONT, "memscrub_add_span: address: 0x%llx"
	    " size: 0x%llx\n", start, bytes);
#endif /* MEMSCRUB_DEBUG */

	/*
	 * Scan through the list to find the proper place to install it.
	 */
	prev = NULL;
	next = memscrub_memlist;
	while (next) {
		uint64_t ns = next->ml_address;
		uint64_t ne = next->ml_address + next->ml_size - 1;

		/*
		 * If this span overlaps with an existing span, then
		 * something has gone horribly wrong with the phys_install
		 * list.  In fact, I'm surprised we made it this far.
		 */
		if ((start >= ns && start <= ne) || (end >= ns && end <= ne) ||
		    (start < ns && end > ne))
			panic("memscrub found overlapping memory ranges "
			    "(0x%p-0x%p) and (0x%p-0x%p)",
			    (void *)(uintptr_t)start, (void *)(uintptr_t)end,
			    (void *)(uintptr_t)ns, (void *)(uintptr_t)ne);

		/*
		 * New span can be appended to an existing one.
		 */
		if (start == ne + 1) {
			next->ml_size += bytes;
			goto add_done;
		}

		/*
		 * New span can be prepended to an existing one.
		 */
		if (end + 1 == ns) {
			next->ml_size += bytes;
			next->ml_address = start;
			goto add_done;
		}

		/*
		 * If the next span has a higher start address than the new
		 * one, then we have found the right spot for our
		 * insertion.
		 */
		if (ns > start)
			break;

		prev = next;
		next = next->ml_next;
	}

	/*
	 * allocate a new struct memlist
	 */
	dst = kmem_alloc(sizeof (struct memlist), KM_NOSLEEP);
	if (dst == NULL) {
		retval = -1;
		goto add_done;
	}
	dst->ml_address = start;
	dst->ml_size = bytes;
	dst->ml_prev = prev;
	dst->ml_next = next;

	if (prev)
		prev->ml_next = dst;
	else
		memscrub_memlist = dst;

	if (next)
		next->ml_prev = dst;

add_done:

	if (retval != -1)
		memscrub_phys_pages += mmu_btop(bytes);

#ifdef MEMSCRUB_DEBUG
	memscrub_printmemlist("memscrub_memlist after", memscrub_memlist);
	cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
#endif /* MEMSCRUB_DEBUG */

	mutex_exit(&memscrub_lock);
	return (retval);
}