17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
5ae115bc7Smrj * Common Development and Distribution License (the "License").
6ae115bc7Smrj * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217aec1d6eScindi
227c478bd9Sstevel@tonic-gate /*
23*56f33205SJonathan Adams * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
247c478bd9Sstevel@tonic-gate * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate */
267c478bd9Sstevel@tonic-gate
277c478bd9Sstevel@tonic-gate /*
287c478bd9Sstevel@tonic-gate * i86pc Memory Scrubbing
297c478bd9Sstevel@tonic-gate *
307c478bd9Sstevel@tonic-gate * On detection of a correctable memory ECC error, the i86pc hardware
317c478bd9Sstevel@tonic-gate * returns the corrected data to the requester and may re-write it
327c478bd9Sstevel@tonic-gate * to memory (DRAM or NVRAM). Machines which do not re-write this to
337c478bd9Sstevel@tonic-gate * memory should add an NMI handler to correct and rewrite.
347c478bd9Sstevel@tonic-gate *
357c478bd9Sstevel@tonic-gate * Scrubbing thus reduces the likelyhood that multiple transient errors
367c478bd9Sstevel@tonic-gate * will occur in the same memory word, making uncorrectable errors due
377c478bd9Sstevel@tonic-gate * to transients less likely.
387c478bd9Sstevel@tonic-gate *
397c478bd9Sstevel@tonic-gate * Thus is born the desire that every memory location be periodically
407c478bd9Sstevel@tonic-gate * accessed.
417c478bd9Sstevel@tonic-gate *
427c478bd9Sstevel@tonic-gate * This file implements a memory scrubbing thread. This scrubber
437c478bd9Sstevel@tonic-gate * guarantees that all of physical memory is accessed periodically
447c478bd9Sstevel@tonic-gate * (memscrub_period_sec -- 12 hours).
457c478bd9Sstevel@tonic-gate *
467c478bd9Sstevel@tonic-gate * It attempts to do this as unobtrusively as possible. The thread
477c478bd9Sstevel@tonic-gate * schedules itself to wake up at an interval such that if it reads
487c478bd9Sstevel@tonic-gate * memscrub_span_pages (4MB) on each wakeup, it will read all of physical
497c478bd9Sstevel@tonic-gate * memory in in memscrub_period_sec (12 hours).
507c478bd9Sstevel@tonic-gate *
517c478bd9Sstevel@tonic-gate * The scrubber uses the REP LODS so it reads 4MB in 0.15 secs (on P5-200).
527c478bd9Sstevel@tonic-gate * When it completes a span, if all the CPUs are idle, it reads another span.
537c478bd9Sstevel@tonic-gate * Typically it soaks up idle time this way to reach its deadline early
547c478bd9Sstevel@tonic-gate * -- and sleeps until the next period begins.
557c478bd9Sstevel@tonic-gate *
567c478bd9Sstevel@tonic-gate * Maximal Cost Estimate: 8GB @ xxMB/s = xxx seconds spent in 640 wakeups
577c478bd9Sstevel@tonic-gate * that run for 0.15 seconds at intervals of 67 seconds.
587c478bd9Sstevel@tonic-gate *
597c478bd9Sstevel@tonic-gate * In practice, the scrubber finds enough idle time to finish in a few
607c478bd9Sstevel@tonic-gate * minutes, and sleeps until its 12 hour deadline.
617c478bd9Sstevel@tonic-gate *
627c478bd9Sstevel@tonic-gate * The scrubber maintains a private copy of the phys_install memory list
637c478bd9Sstevel@tonic-gate * to keep track of what memory should be scrubbed.
647c478bd9Sstevel@tonic-gate *
657c478bd9Sstevel@tonic-gate * The following parameters can be set via /etc/system
667c478bd9Sstevel@tonic-gate *
677c478bd9Sstevel@tonic-gate * memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES (4MB)
687c478bd9Sstevel@tonic-gate * memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC (12 hours)
697c478bd9Sstevel@tonic-gate * memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI (0)
707c478bd9Sstevel@tonic-gate * memscrub_delay_start_sec = (10 seconds)
717c478bd9Sstevel@tonic-gate * disable_memscrub = (0)
727c478bd9Sstevel@tonic-gate *
737c478bd9Sstevel@tonic-gate * the scrubber will exit (or never be started) if it finds the variable
747c478bd9Sstevel@tonic-gate * "disable_memscrub" set.
757c478bd9Sstevel@tonic-gate *
767c478bd9Sstevel@tonic-gate * MEMSCRUB_DFL_SPAN_PAGES is based on the guess that 0.15 sec
777c478bd9Sstevel@tonic-gate * is a "good" amount of minimum time for the thread to run at a time.
787c478bd9Sstevel@tonic-gate *
797c478bd9Sstevel@tonic-gate * MEMSCRUB_DFL_PERIOD_SEC (12 hours) is nearly a total guess --
807c478bd9Sstevel@tonic-gate * twice the frequency the hardware folk estimated would be necessary.
817c478bd9Sstevel@tonic-gate *
827c478bd9Sstevel@tonic-gate * MEMSCRUB_DFL_THREAD_PRI (0) is based on the assumption that nearly
837c478bd9Sstevel@tonic-gate * any other use of the system should be higher priority than scrubbing.
847c478bd9Sstevel@tonic-gate */
857c478bd9Sstevel@tonic-gate
867c478bd9Sstevel@tonic-gate #include <sys/types.h>
877c478bd9Sstevel@tonic-gate #include <sys/systm.h> /* timeout, types, t_lock */
887c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
897c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> /* MIN */
907c478bd9Sstevel@tonic-gate #include <sys/memlist.h> /* memlist */
917c478bd9Sstevel@tonic-gate #include <sys/kmem.h> /* KMEM_NOSLEEP */
927c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> /* ncpus_online */
937c478bd9Sstevel@tonic-gate #include <sys/debug.h> /* ASSERTs */
947c478bd9Sstevel@tonic-gate #include <sys/vmem.h>
957c478bd9Sstevel@tonic-gate #include <sys/mman.h>
967c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
977c478bd9Sstevel@tonic-gate #include <vm/seg_kpm.h>
987c478bd9Sstevel@tonic-gate #include <vm/hat_i86.h>
992df1fe9cSrandyf #include <sys/callb.h> /* CPR callback */
1007c478bd9Sstevel@tonic-gate
1017c478bd9Sstevel@tonic-gate static caddr_t memscrub_window;
102ae115bc7Smrj static hat_mempte_t memscrub_pte;
1037c478bd9Sstevel@tonic-gate
1047c478bd9Sstevel@tonic-gate /*
1057c478bd9Sstevel@tonic-gate * Global Data:
1067c478bd9Sstevel@tonic-gate */
1077c478bd9Sstevel@tonic-gate /*
1087c478bd9Sstevel@tonic-gate * scan all of physical memory at least once every MEMSCRUB_PERIOD_SEC
1097c478bd9Sstevel@tonic-gate */
1107c478bd9Sstevel@tonic-gate #define MEMSCRUB_DFL_PERIOD_SEC (12 * 60 * 60) /* 12 hours */
1117c478bd9Sstevel@tonic-gate
1127c478bd9Sstevel@tonic-gate /*
1137c478bd9Sstevel@tonic-gate * start only if at least MEMSCRUB_MIN_PAGES in system
1147c478bd9Sstevel@tonic-gate */
1157c478bd9Sstevel@tonic-gate #define MEMSCRUB_MIN_PAGES ((32 * 1024 * 1024) / PAGESIZE)
1167c478bd9Sstevel@tonic-gate
1177c478bd9Sstevel@tonic-gate /*
1187c478bd9Sstevel@tonic-gate * scan at least MEMSCRUB_DFL_SPAN_PAGES each iteration
1197c478bd9Sstevel@tonic-gate */
1207c478bd9Sstevel@tonic-gate #define MEMSCRUB_DFL_SPAN_PAGES ((4 * 1024 * 1024) / PAGESIZE)
1217c478bd9Sstevel@tonic-gate
1227c478bd9Sstevel@tonic-gate /*
1237c478bd9Sstevel@tonic-gate * almost anything is higher priority than scrubbing
1247c478bd9Sstevel@tonic-gate */
1257c478bd9Sstevel@tonic-gate #define MEMSCRUB_DFL_THREAD_PRI 0
1267c478bd9Sstevel@tonic-gate
1277c478bd9Sstevel@tonic-gate /*
1287c478bd9Sstevel@tonic-gate * we can patch these defaults in /etc/system if necessary
1297c478bd9Sstevel@tonic-gate */
1307c478bd9Sstevel@tonic-gate uint_t disable_memscrub = 0;
1317aec1d6eScindi static uint_t disable_memscrub_quietly = 0;
1327c478bd9Sstevel@tonic-gate pgcnt_t memscrub_min_pages = MEMSCRUB_MIN_PAGES;
1337c478bd9Sstevel@tonic-gate pgcnt_t memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES;
1347c478bd9Sstevel@tonic-gate time_t memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC;
1357c478bd9Sstevel@tonic-gate uint_t memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI;
1367c478bd9Sstevel@tonic-gate time_t memscrub_delay_start_sec = 10;
1377c478bd9Sstevel@tonic-gate
1387c478bd9Sstevel@tonic-gate /*
1397c478bd9Sstevel@tonic-gate * Static Routines
1407c478bd9Sstevel@tonic-gate */
1417c478bd9Sstevel@tonic-gate static void memscrubber(void);
1427c478bd9Sstevel@tonic-gate static int system_is_idle(void);
1437c478bd9Sstevel@tonic-gate static int memscrub_add_span(uint64_t, uint64_t);
1447c478bd9Sstevel@tonic-gate
1457c478bd9Sstevel@tonic-gate /*
1467c478bd9Sstevel@tonic-gate * Static Data
1477c478bd9Sstevel@tonic-gate */
1487c478bd9Sstevel@tonic-gate static struct memlist *memscrub_memlist;
1497c478bd9Sstevel@tonic-gate static uint_t memscrub_phys_pages;
1507c478bd9Sstevel@tonic-gate
1517c478bd9Sstevel@tonic-gate static kcondvar_t memscrub_cv;
1527c478bd9Sstevel@tonic-gate static kmutex_t memscrub_lock;
1537aec1d6eScindi
1547c478bd9Sstevel@tonic-gate /*
1557c478bd9Sstevel@tonic-gate * memscrub_lock protects memscrub_memlist
1567c478bd9Sstevel@tonic-gate */
1577c478bd9Sstevel@tonic-gate uint_t memscrub_scans_done;
1587c478bd9Sstevel@tonic-gate
1597c478bd9Sstevel@tonic-gate uint_t memscrub_done_early;
1607c478bd9Sstevel@tonic-gate uint_t memscrub_early_sec;
1617c478bd9Sstevel@tonic-gate
1627c478bd9Sstevel@tonic-gate uint_t memscrub_done_late;
1637c478bd9Sstevel@tonic-gate time_t memscrub_late_sec;
1647c478bd9Sstevel@tonic-gate
1657c478bd9Sstevel@tonic-gate /*
1667c478bd9Sstevel@tonic-gate * create memscrub_memlist from phys_install list
1677c478bd9Sstevel@tonic-gate * initialize locks, set memscrub_phys_pages.
1687c478bd9Sstevel@tonic-gate */
1697c478bd9Sstevel@tonic-gate void
memscrub_init()1707c478bd9Sstevel@tonic-gate memscrub_init()
1717c478bd9Sstevel@tonic-gate {
1727c478bd9Sstevel@tonic-gate struct memlist *src;
1737c478bd9Sstevel@tonic-gate
1747c478bd9Sstevel@tonic-gate if (physmem < memscrub_min_pages)
1757c478bd9Sstevel@tonic-gate return;
1767c478bd9Sstevel@tonic-gate
1777c478bd9Sstevel@tonic-gate if (!kpm_enable) {
1787c478bd9Sstevel@tonic-gate memscrub_window = vmem_alloc(heap_arena, PAGESIZE, VM_SLEEP);
1797c478bd9Sstevel@tonic-gate memscrub_pte = hat_mempte_setup(memscrub_window);
1807c478bd9Sstevel@tonic-gate }
1817c478bd9Sstevel@tonic-gate
1827c478bd9Sstevel@tonic-gate /*
1837c478bd9Sstevel@tonic-gate * copy phys_install to memscrub_memlist
1847c478bd9Sstevel@tonic-gate */
185*56f33205SJonathan Adams for (src = phys_install; src; src = src->ml_next) {
186*56f33205SJonathan Adams if (memscrub_add_span(src->ml_address, src->ml_size)) {
1877c478bd9Sstevel@tonic-gate cmn_err(CE_WARN,
188e4b86885SCheng Sean Ye "Software memory scrubber failed to initialize\n");
1897c478bd9Sstevel@tonic-gate return;
1907c478bd9Sstevel@tonic-gate }
1917c478bd9Sstevel@tonic-gate }
1927c478bd9Sstevel@tonic-gate
1937c478bd9Sstevel@tonic-gate mutex_init(&memscrub_lock, NULL, MUTEX_DRIVER, NULL);
1947c478bd9Sstevel@tonic-gate cv_init(&memscrub_cv, NULL, CV_DRIVER, NULL);
1957c478bd9Sstevel@tonic-gate
1967c478bd9Sstevel@tonic-gate /*
1977c478bd9Sstevel@tonic-gate * create memscrubber thread
1987c478bd9Sstevel@tonic-gate */
1997c478bd9Sstevel@tonic-gate (void) thread_create(NULL, 0, (void (*)())memscrubber, NULL, 0, &p0,
2007c478bd9Sstevel@tonic-gate TS_RUN, memscrub_thread_pri);
2017c478bd9Sstevel@tonic-gate }
2027c478bd9Sstevel@tonic-gate
2037aec1d6eScindi /*
2047aec1d6eScindi * Function to cause the software memscrubber to exit quietly if the
2057aec1d6eScindi * platform support has located a hardware scrubber and enabled it.
2067aec1d6eScindi */
2077aec1d6eScindi void
memscrub_disable(void)2087aec1d6eScindi memscrub_disable(void)
2097aec1d6eScindi {
2107aec1d6eScindi disable_memscrub_quietly = 1;
2117aec1d6eScindi }
2127aec1d6eScindi
2137c478bd9Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG
214e4b86885SCheng Sean Ye static void
memscrub_printmemlist(char * title,struct memlist * listp)2157c478bd9Sstevel@tonic-gate memscrub_printmemlist(char *title, struct memlist *listp)
2167c478bd9Sstevel@tonic-gate {
2177c478bd9Sstevel@tonic-gate struct memlist *list;
2187c478bd9Sstevel@tonic-gate
2197c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "%s:\n", title);
2207c478bd9Sstevel@tonic-gate
2217c478bd9Sstevel@tonic-gate for (list = listp; list; list = list->next) {
2227c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "addr = 0x%llx, size = 0x%llx\n",
2237c478bd9Sstevel@tonic-gate list->address, list->size);
2247c478bd9Sstevel@tonic-gate }
2257c478bd9Sstevel@tonic-gate }
2267c478bd9Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */
2277c478bd9Sstevel@tonic-gate
2287c478bd9Sstevel@tonic-gate /* ARGSUSED */
229e4b86885SCheng Sean Ye static void
memscrub_wakeup(void * c)2307c478bd9Sstevel@tonic-gate memscrub_wakeup(void *c)
2317c478bd9Sstevel@tonic-gate {
2327c478bd9Sstevel@tonic-gate /*
2337c478bd9Sstevel@tonic-gate * grab mutex to guarantee that our wakeup call
2347c478bd9Sstevel@tonic-gate * arrives after we go to sleep -- so we can't sleep forever.
2357c478bd9Sstevel@tonic-gate */
2367c478bd9Sstevel@tonic-gate mutex_enter(&memscrub_lock);
2377c478bd9Sstevel@tonic-gate cv_signal(&memscrub_cv);
2387c478bd9Sstevel@tonic-gate mutex_exit(&memscrub_lock);
2397c478bd9Sstevel@tonic-gate }
2407c478bd9Sstevel@tonic-gate
2417c478bd9Sstevel@tonic-gate /*
2427c478bd9Sstevel@tonic-gate * this calculation doesn't account for the time that the actual scan
2437c478bd9Sstevel@tonic-gate * consumes -- so we'd fall slightly behind schedule with this
2447c478bd9Sstevel@tonic-gate * interval_sec. but the idle loop optimization below usually makes us
2457c478bd9Sstevel@tonic-gate * come in way ahead of schedule.
2467c478bd9Sstevel@tonic-gate */
2477c478bd9Sstevel@tonic-gate static int
compute_interval_sec()2487c478bd9Sstevel@tonic-gate compute_interval_sec()
2497c478bd9Sstevel@tonic-gate {
2507c478bd9Sstevel@tonic-gate if (memscrub_phys_pages <= memscrub_span_pages)
2517c478bd9Sstevel@tonic-gate return (memscrub_period_sec);
2527c478bd9Sstevel@tonic-gate else
2537c478bd9Sstevel@tonic-gate return (memscrub_period_sec/
2542df1fe9cSrandyf (memscrub_phys_pages/memscrub_span_pages));
2557c478bd9Sstevel@tonic-gate }
2567c478bd9Sstevel@tonic-gate
257e4b86885SCheng Sean Ye static void
memscrubber()2587c478bd9Sstevel@tonic-gate memscrubber()
2597c478bd9Sstevel@tonic-gate {
2607c478bd9Sstevel@tonic-gate time_t deadline;
2617c478bd9Sstevel@tonic-gate uint64_t mlp_last_addr;
2627c478bd9Sstevel@tonic-gate uint64_t mlp_next_addr;
2637c478bd9Sstevel@tonic-gate int reached_end = 1;
2647c478bd9Sstevel@tonic-gate time_t interval_sec = 0;
2657c478bd9Sstevel@tonic-gate struct memlist *mlp;
2667c478bd9Sstevel@tonic-gate
2677c478bd9Sstevel@tonic-gate extern void scan_memory(caddr_t, size_t);
2682df1fe9cSrandyf callb_cpr_t cprinfo;
2692df1fe9cSrandyf
2702df1fe9cSrandyf /*
2712df1fe9cSrandyf * notify CPR of our existence
2722df1fe9cSrandyf */
2732df1fe9cSrandyf CALLB_CPR_INIT(&cprinfo, &memscrub_lock, callb_generic_cpr, "memscrub");
2747c478bd9Sstevel@tonic-gate
2757c478bd9Sstevel@tonic-gate if (memscrub_memlist == NULL) {
2767c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "memscrub_memlist not initialized.");
2777c478bd9Sstevel@tonic-gate goto memscrub_exit;
2787c478bd9Sstevel@tonic-gate }
2797c478bd9Sstevel@tonic-gate
2807c478bd9Sstevel@tonic-gate mlp = memscrub_memlist;
281*56f33205SJonathan Adams mlp_next_addr = mlp->ml_address;
282*56f33205SJonathan Adams mlp_last_addr = mlp->ml_address + mlp->ml_size;
2837c478bd9Sstevel@tonic-gate
2847c478bd9Sstevel@tonic-gate deadline = gethrestime_sec() + memscrub_delay_start_sec;
2857c478bd9Sstevel@tonic-gate
2867c478bd9Sstevel@tonic-gate for (;;) {
2877aec1d6eScindi if (disable_memscrub || disable_memscrub_quietly)
2887c478bd9Sstevel@tonic-gate break;
2897c478bd9Sstevel@tonic-gate
2907c478bd9Sstevel@tonic-gate mutex_enter(&memscrub_lock);
2917c478bd9Sstevel@tonic-gate
2927c478bd9Sstevel@tonic-gate /*
2937c478bd9Sstevel@tonic-gate * did we just reach the end of memory?
2947c478bd9Sstevel@tonic-gate */
2957c478bd9Sstevel@tonic-gate if (reached_end) {
2967c478bd9Sstevel@tonic-gate time_t now = gethrestime_sec();
2977c478bd9Sstevel@tonic-gate
2987c478bd9Sstevel@tonic-gate if (now >= deadline) {
2997c478bd9Sstevel@tonic-gate memscrub_done_late++;
3007c478bd9Sstevel@tonic-gate memscrub_late_sec += (now - deadline);
3017c478bd9Sstevel@tonic-gate /*
3027c478bd9Sstevel@tonic-gate * past deadline, start right away
3037c478bd9Sstevel@tonic-gate */
3047c478bd9Sstevel@tonic-gate interval_sec = 0;
3057c478bd9Sstevel@tonic-gate
3067c478bd9Sstevel@tonic-gate deadline = now + memscrub_period_sec;
3077c478bd9Sstevel@tonic-gate } else {
3087c478bd9Sstevel@tonic-gate /*
3097c478bd9Sstevel@tonic-gate * we finished ahead of schedule.
3107c478bd9Sstevel@tonic-gate * wait till previous dealine before re-start.
3117c478bd9Sstevel@tonic-gate */
3127c478bd9Sstevel@tonic-gate interval_sec = deadline - now;
3137c478bd9Sstevel@tonic-gate memscrub_done_early++;
3147c478bd9Sstevel@tonic-gate memscrub_early_sec += interval_sec;
3157c478bd9Sstevel@tonic-gate deadline += memscrub_period_sec;
3167c478bd9Sstevel@tonic-gate }
3177c478bd9Sstevel@tonic-gate } else {
3187c478bd9Sstevel@tonic-gate interval_sec = compute_interval_sec();
3197c478bd9Sstevel@tonic-gate }
3207c478bd9Sstevel@tonic-gate
3212df1fe9cSrandyf /*
3222df1fe9cSrandyf * it is safe from our standpoint for CPR to
3232df1fe9cSrandyf * suspend the system
3242df1fe9cSrandyf */
3252df1fe9cSrandyf CALLB_CPR_SAFE_BEGIN(&cprinfo);
3262df1fe9cSrandyf
3277c478bd9Sstevel@tonic-gate /*
3287c478bd9Sstevel@tonic-gate * hit the snooze bar
3297c478bd9Sstevel@tonic-gate */
3307c478bd9Sstevel@tonic-gate (void) timeout(memscrub_wakeup, NULL, interval_sec * hz);
3317c478bd9Sstevel@tonic-gate
3327c478bd9Sstevel@tonic-gate /*
3337c478bd9Sstevel@tonic-gate * go to sleep
3347c478bd9Sstevel@tonic-gate */
3357c478bd9Sstevel@tonic-gate cv_wait(&memscrub_cv, &memscrub_lock);
3367c478bd9Sstevel@tonic-gate
3372df1fe9cSrandyf /* we need to goto work */
3382df1fe9cSrandyf CALLB_CPR_SAFE_END(&cprinfo, &memscrub_lock);
3392df1fe9cSrandyf
3407c478bd9Sstevel@tonic-gate mutex_exit(&memscrub_lock);
3417c478bd9Sstevel@tonic-gate
3427c478bd9Sstevel@tonic-gate do {
3437c478bd9Sstevel@tonic-gate pgcnt_t pages = memscrub_span_pages;
3447c478bd9Sstevel@tonic-gate uint64_t address = mlp_next_addr;
3457c478bd9Sstevel@tonic-gate
3467aec1d6eScindi if (disable_memscrub || disable_memscrub_quietly)
3477c478bd9Sstevel@tonic-gate break;
3487c478bd9Sstevel@tonic-gate
3497c478bd9Sstevel@tonic-gate mutex_enter(&memscrub_lock);
3507c478bd9Sstevel@tonic-gate
3517c478bd9Sstevel@tonic-gate /*
3527c478bd9Sstevel@tonic-gate * Make sure we don't try to scan beyond the end of
3537c478bd9Sstevel@tonic-gate * the current memlist. If we would, then resize
3547c478bd9Sstevel@tonic-gate * our scan target for this iteration, and prepare
3557c478bd9Sstevel@tonic-gate * to read the next memlist entry on the next
3567c478bd9Sstevel@tonic-gate * iteration.
3577c478bd9Sstevel@tonic-gate */
3587c478bd9Sstevel@tonic-gate reached_end = 0;
3597c478bd9Sstevel@tonic-gate if (address + mmu_ptob(pages) >= mlp_last_addr) {
3607c478bd9Sstevel@tonic-gate pages = mmu_btop(mlp_last_addr - address);
361*56f33205SJonathan Adams mlp = mlp->ml_next;
3627c478bd9Sstevel@tonic-gate if (mlp == NULL) {
3637c478bd9Sstevel@tonic-gate reached_end = 1;
3647c478bd9Sstevel@tonic-gate mlp = memscrub_memlist;
3657c478bd9Sstevel@tonic-gate }
366*56f33205SJonathan Adams mlp_next_addr = mlp->ml_address;
367*56f33205SJonathan Adams mlp_last_addr = mlp->ml_address + mlp->ml_size;
3687c478bd9Sstevel@tonic-gate } else {
3697c478bd9Sstevel@tonic-gate mlp_next_addr += mmu_ptob(pages);
3707c478bd9Sstevel@tonic-gate }
3717c478bd9Sstevel@tonic-gate
3727c478bd9Sstevel@tonic-gate mutex_exit(&memscrub_lock);
3737c478bd9Sstevel@tonic-gate
3747c478bd9Sstevel@tonic-gate while (pages--) {
3757c478bd9Sstevel@tonic-gate pfn_t pfn = btop(address);
3767c478bd9Sstevel@tonic-gate
3777c478bd9Sstevel@tonic-gate /*
3787c478bd9Sstevel@tonic-gate * Without segkpm, the memscrubber cannot
3797c478bd9Sstevel@tonic-gate * be allowed to migrate across CPUs, as
3807c478bd9Sstevel@tonic-gate * the CPU-specific mapping of
3817c478bd9Sstevel@tonic-gate * memscrub_window would be incorrect.
3827c478bd9Sstevel@tonic-gate * With segkpm, switching CPUs is legal, but
3837c478bd9Sstevel@tonic-gate * inefficient. We don't use
3847c478bd9Sstevel@tonic-gate * kpreempt_disable as it might hold a
3857c478bd9Sstevel@tonic-gate * higher priority thread (eg, RT) too long
3867c478bd9Sstevel@tonic-gate * off CPU.
3877c478bd9Sstevel@tonic-gate */
3887c478bd9Sstevel@tonic-gate thread_affinity_set(curthread, CPU_CURRENT);
3897c478bd9Sstevel@tonic-gate if (kpm_enable)
3907c478bd9Sstevel@tonic-gate memscrub_window = hat_kpm_pfn2va(pfn);
3917c478bd9Sstevel@tonic-gate else
3927c478bd9Sstevel@tonic-gate hat_mempte_remap(pfn, memscrub_window,
3937c478bd9Sstevel@tonic-gate memscrub_pte,
3947c478bd9Sstevel@tonic-gate PROT_READ, HAT_LOAD_NOCONSIST);
3957c478bd9Sstevel@tonic-gate
3967c478bd9Sstevel@tonic-gate scan_memory(memscrub_window, PAGESIZE);
3977c478bd9Sstevel@tonic-gate
3987c478bd9Sstevel@tonic-gate thread_affinity_clear(curthread);
3997c478bd9Sstevel@tonic-gate address += MMU_PAGESIZE;
4007c478bd9Sstevel@tonic-gate }
4017c478bd9Sstevel@tonic-gate
4027c478bd9Sstevel@tonic-gate memscrub_scans_done++;
4037c478bd9Sstevel@tonic-gate } while (!reached_end && system_is_idle());
4047c478bd9Sstevel@tonic-gate }
4057c478bd9Sstevel@tonic-gate
4067c478bd9Sstevel@tonic-gate memscrub_exit:
4077c478bd9Sstevel@tonic-gate
4087aec1d6eScindi if (!disable_memscrub_quietly)
409e4b86885SCheng Sean Ye cmn_err(CE_NOTE, "Software memory scrubber exiting.");
4102df1fe9cSrandyf /*
4112df1fe9cSrandyf * We are about to bail, but don't have the memscrub_lock,
4122df1fe9cSrandyf * and it is needed for CALLB_CPR_EXIT.
4132df1fe9cSrandyf */
4142df1fe9cSrandyf mutex_enter(&memscrub_lock);
4152df1fe9cSrandyf CALLB_CPR_EXIT(&cprinfo);
4167c478bd9Sstevel@tonic-gate
4177c478bd9Sstevel@tonic-gate cv_destroy(&memscrub_cv);
4187c478bd9Sstevel@tonic-gate
4197c478bd9Sstevel@tonic-gate thread_exit();
4207c478bd9Sstevel@tonic-gate }
4217c478bd9Sstevel@tonic-gate
4227c478bd9Sstevel@tonic-gate
4237c478bd9Sstevel@tonic-gate /*
4247c478bd9Sstevel@tonic-gate * return 1 if we're MP and all the other CPUs are idle
4257c478bd9Sstevel@tonic-gate */
4267c478bd9Sstevel@tonic-gate static int
system_is_idle()4277c478bd9Sstevel@tonic-gate system_is_idle()
4287c478bd9Sstevel@tonic-gate {
4297c478bd9Sstevel@tonic-gate int cpu_id;
4307c478bd9Sstevel@tonic-gate int found = 0;
4317c478bd9Sstevel@tonic-gate
4327c478bd9Sstevel@tonic-gate if (1 == ncpus_online)
4337c478bd9Sstevel@tonic-gate return (0);
4347c478bd9Sstevel@tonic-gate
4357c478bd9Sstevel@tonic-gate for (cpu_id = 0; cpu_id < NCPU; ++cpu_id) {
4367c478bd9Sstevel@tonic-gate if (!cpu[cpu_id])
4377c478bd9Sstevel@tonic-gate continue;
4387c478bd9Sstevel@tonic-gate
4397c478bd9Sstevel@tonic-gate found++;
4407c478bd9Sstevel@tonic-gate
4417c478bd9Sstevel@tonic-gate if (cpu[cpu_id]->cpu_thread != cpu[cpu_id]->cpu_idle_thread) {
4427c478bd9Sstevel@tonic-gate if (CPU->cpu_id == cpu_id &&
4437c478bd9Sstevel@tonic-gate CPU->cpu_disp->disp_nrunnable == 0)
4447c478bd9Sstevel@tonic-gate continue;
4457c478bd9Sstevel@tonic-gate return (0);
4467c478bd9Sstevel@tonic-gate }
4477c478bd9Sstevel@tonic-gate
4487c478bd9Sstevel@tonic-gate if (found == ncpus)
4497c478bd9Sstevel@tonic-gate break;
4507c478bd9Sstevel@tonic-gate }
4517c478bd9Sstevel@tonic-gate return (1);
4527c478bd9Sstevel@tonic-gate }
4537c478bd9Sstevel@tonic-gate
4547c478bd9Sstevel@tonic-gate /*
4557c478bd9Sstevel@tonic-gate * add a span to the memscrub list
4567c478bd9Sstevel@tonic-gate */
4577c478bd9Sstevel@tonic-gate static int
memscrub_add_span(uint64_t start,uint64_t bytes)4587c478bd9Sstevel@tonic-gate memscrub_add_span(uint64_t start, uint64_t bytes)
4597c478bd9Sstevel@tonic-gate {
4607c478bd9Sstevel@tonic-gate struct memlist *dst;
4617c478bd9Sstevel@tonic-gate struct memlist *prev, *next;
4627c478bd9Sstevel@tonic-gate uint64_t end = start + bytes - 1;
4637c478bd9Sstevel@tonic-gate int retval = 0;
4647c478bd9Sstevel@tonic-gate
4657c478bd9Sstevel@tonic-gate mutex_enter(&memscrub_lock);
4667c478bd9Sstevel@tonic-gate
4677c478bd9Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG
4687c478bd9Sstevel@tonic-gate memscrub_printmemlist("memscrub_memlist before", memscrub_memlist);
4697c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
4707c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_add_span: address: 0x%llx"
4712df1fe9cSrandyf " size: 0x%llx\n", start, bytes);
4727c478bd9Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */
4737c478bd9Sstevel@tonic-gate
4747c478bd9Sstevel@tonic-gate /*
4757c478bd9Sstevel@tonic-gate * Scan through the list to find the proper place to install it.
4767c478bd9Sstevel@tonic-gate */
4777c478bd9Sstevel@tonic-gate prev = NULL;
4787c478bd9Sstevel@tonic-gate next = memscrub_memlist;
4797c478bd9Sstevel@tonic-gate while (next) {
480*56f33205SJonathan Adams uint64_t ns = next->ml_address;
481*56f33205SJonathan Adams uint64_t ne = next->ml_address + next->ml_size - 1;
4827c478bd9Sstevel@tonic-gate
4837c478bd9Sstevel@tonic-gate /*
4847c478bd9Sstevel@tonic-gate * If this span overlaps with an existing span, then
4857c478bd9Sstevel@tonic-gate * something has gone horribly wrong with the phys_install
4867c478bd9Sstevel@tonic-gate * list. In fact, I'm surprised we made it this far.
4877c478bd9Sstevel@tonic-gate */
4887c478bd9Sstevel@tonic-gate if ((start >= ns && start <= ne) || (end >= ns && end <= ne) ||
4897c478bd9Sstevel@tonic-gate (start < ns && end > ne))
4907c478bd9Sstevel@tonic-gate panic("memscrub found overlapping memory ranges "
4917c478bd9Sstevel@tonic-gate "(0x%p-0x%p) and (0x%p-0x%p)",
4923dfcd6dcSdmick (void *)(uintptr_t)start, (void *)(uintptr_t)end,
4933dfcd6dcSdmick (void *)(uintptr_t)ns, (void *)(uintptr_t)ne);
4947c478bd9Sstevel@tonic-gate
4957c478bd9Sstevel@tonic-gate /*
4967c478bd9Sstevel@tonic-gate * New span can be appended to an existing one.
4977c478bd9Sstevel@tonic-gate */
4987c478bd9Sstevel@tonic-gate if (start == ne + 1) {
499*56f33205SJonathan Adams next->ml_size += bytes;
5007c478bd9Sstevel@tonic-gate goto add_done;
5017c478bd9Sstevel@tonic-gate }
5027c478bd9Sstevel@tonic-gate
5037c478bd9Sstevel@tonic-gate /*
5047c478bd9Sstevel@tonic-gate * New span can be prepended to an existing one.
5057c478bd9Sstevel@tonic-gate */
5067c478bd9Sstevel@tonic-gate if (end + 1 == ns) {
507*56f33205SJonathan Adams next->ml_size += bytes;
508*56f33205SJonathan Adams next->ml_address = start;
5097c478bd9Sstevel@tonic-gate goto add_done;
5107c478bd9Sstevel@tonic-gate }
5117c478bd9Sstevel@tonic-gate
5127c478bd9Sstevel@tonic-gate /*
5137c478bd9Sstevel@tonic-gate * If the next span has a higher start address than the new
5147c478bd9Sstevel@tonic-gate * one, then we have found the right spot for our
5157c478bd9Sstevel@tonic-gate * insertion.
5167c478bd9Sstevel@tonic-gate */
5177c478bd9Sstevel@tonic-gate if (ns > start)
5187c478bd9Sstevel@tonic-gate break;
5197c478bd9Sstevel@tonic-gate
5207c478bd9Sstevel@tonic-gate prev = next;
521*56f33205SJonathan Adams next = next->ml_next;
5227c478bd9Sstevel@tonic-gate }
5237c478bd9Sstevel@tonic-gate
5247c478bd9Sstevel@tonic-gate /*
5257c478bd9Sstevel@tonic-gate * allocate a new struct memlist
5267c478bd9Sstevel@tonic-gate */
5277c478bd9Sstevel@tonic-gate dst = kmem_alloc(sizeof (struct memlist), KM_NOSLEEP);
5287c478bd9Sstevel@tonic-gate if (dst == NULL) {
5297c478bd9Sstevel@tonic-gate retval = -1;
5307c478bd9Sstevel@tonic-gate goto add_done;
5317c478bd9Sstevel@tonic-gate }
532*56f33205SJonathan Adams dst->ml_address = start;
533*56f33205SJonathan Adams dst->ml_size = bytes;
534*56f33205SJonathan Adams dst->ml_prev = prev;
535*56f33205SJonathan Adams dst->ml_next = next;
5367c478bd9Sstevel@tonic-gate
5377c478bd9Sstevel@tonic-gate if (prev)
538*56f33205SJonathan Adams prev->ml_next = dst;
5397c478bd9Sstevel@tonic-gate else
5407c478bd9Sstevel@tonic-gate memscrub_memlist = dst;
5417c478bd9Sstevel@tonic-gate
5427c478bd9Sstevel@tonic-gate if (next)
543*56f33205SJonathan Adams next->ml_prev = dst;
5447c478bd9Sstevel@tonic-gate
5457c478bd9Sstevel@tonic-gate add_done:
5467c478bd9Sstevel@tonic-gate
5477c478bd9Sstevel@tonic-gate if (retval != -1)
5487c478bd9Sstevel@tonic-gate memscrub_phys_pages += mmu_btop(bytes);
5497c478bd9Sstevel@tonic-gate
5507c478bd9Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG
5517c478bd9Sstevel@tonic-gate memscrub_printmemlist("memscrub_memlist after", memscrub_memlist);
5527c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
5537c478bd9Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */
5547c478bd9Sstevel@tonic-gate
5557c478bd9Sstevel@tonic-gate mutex_exit(&memscrub_lock);
5567c478bd9Sstevel@tonic-gate return (retval);
5577c478bd9Sstevel@tonic-gate }
558