17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate * CDDL HEADER START
37c478bd9Sstevel@tonic-gate *
47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the
561ef38f7Svb * Common Development and Distribution License (the "License").
661ef38f7Svb * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate *
87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate * and limitations under the License.
127c478bd9Sstevel@tonic-gate *
137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate *
197c478bd9Sstevel@tonic-gate * CDDL HEADER END
207c478bd9Sstevel@tonic-gate */
217c478bd9Sstevel@tonic-gate /*
2256f33205SJonathan Adams * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
237c478bd9Sstevel@tonic-gate * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate */
257c478bd9Sstevel@tonic-gate
267c478bd9Sstevel@tonic-gate /*
277c478bd9Sstevel@tonic-gate * sun4u Memory Scrubbing
287c478bd9Sstevel@tonic-gate *
297c478bd9Sstevel@tonic-gate * On detection of a correctable memory ECC error, the sun4u kernel
307c478bd9Sstevel@tonic-gate * returns the corrected data to the requester and re-writes it
317c478bd9Sstevel@tonic-gate * to memory (DRAM). So if the correctable error was transient,
327c478bd9Sstevel@tonic-gate * the read has effectively been cleaned (scrubbed) from memory.
337c478bd9Sstevel@tonic-gate *
347c478bd9Sstevel@tonic-gate * Scrubbing thus reduces the likelyhood that multiple transient errors
357c478bd9Sstevel@tonic-gate * will occur in the same memory word, making uncorrectable errors due
367c478bd9Sstevel@tonic-gate * to transients less likely.
377c478bd9Sstevel@tonic-gate *
387c478bd9Sstevel@tonic-gate * Thus is born the desire that every memory location be periodically
397c478bd9Sstevel@tonic-gate * accessed.
407c478bd9Sstevel@tonic-gate *
417c478bd9Sstevel@tonic-gate * This file implements a memory scrubbing thread. This scrubber
427c478bd9Sstevel@tonic-gate * guarantees that all of physical memory is accessed periodically
437c478bd9Sstevel@tonic-gate * (memscrub_period_sec -- 12 hours).
447c478bd9Sstevel@tonic-gate *
457c478bd9Sstevel@tonic-gate * It attempts to do this as unobtrusively as possible. The thread
467c478bd9Sstevel@tonic-gate * schedules itself to wake up at an interval such that if it reads
4794836d42Spt * memscrub_span_pages (32MB) on each wakeup, it will read all of physical
487c478bd9Sstevel@tonic-gate * memory in in memscrub_period_sec (12 hours).
497c478bd9Sstevel@tonic-gate *
5094836d42Spt * The scrubber uses the block load and prefetch hardware to read memory
5194836d42Spt * @ 1300MB/s, so it reads spans of 32MB in 0.025 seconds. Unlike the
5294836d42Spt * original sun4d scrubber the sun4u scrubber does not read ahead if the
5394836d42Spt * system is idle because we can read memory very efficently.
547c478bd9Sstevel@tonic-gate *
557c478bd9Sstevel@tonic-gate * The scrubber maintains a private copy of the phys_install memory list
567c478bd9Sstevel@tonic-gate * to keep track of what memory should be scrubbed.
577c478bd9Sstevel@tonic-gate *
587c478bd9Sstevel@tonic-gate * The global routines memscrub_add_span() and memscrub_delete_span() are
597c478bd9Sstevel@tonic-gate * used to add and delete from this list. If hotplug memory is later
607c478bd9Sstevel@tonic-gate * supported these two routines can be used to notify the scrubber of
617c478bd9Sstevel@tonic-gate * memory configuration changes.
627c478bd9Sstevel@tonic-gate *
637c478bd9Sstevel@tonic-gate * The following parameters can be set via /etc/system
647c478bd9Sstevel@tonic-gate *
657c478bd9Sstevel@tonic-gate * memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES (8MB)
667c478bd9Sstevel@tonic-gate * memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC (12 hours)
677c478bd9Sstevel@tonic-gate * memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI (MINCLSYSPRI)
687c478bd9Sstevel@tonic-gate * memscrub_delay_start_sec = (5 minutes)
697c478bd9Sstevel@tonic-gate * memscrub_verbose = (0)
707c478bd9Sstevel@tonic-gate * memscrub_override_ticks = (1 tick)
717c478bd9Sstevel@tonic-gate * disable_memscrub = (0)
727c478bd9Sstevel@tonic-gate * pause_memscrub = (0)
737c478bd9Sstevel@tonic-gate * read_all_memscrub = (0)
747c478bd9Sstevel@tonic-gate *
757c478bd9Sstevel@tonic-gate * The scrubber will print NOTICE messages of what it is doing if
767c478bd9Sstevel@tonic-gate * "memscrub_verbose" is set.
777c478bd9Sstevel@tonic-gate *
787c478bd9Sstevel@tonic-gate * If the scrubber's sleep time calculation drops to zero ticks,
797c478bd9Sstevel@tonic-gate * memscrub_override_ticks will be used as the sleep time instead. The
8094836d42Spt * sleep time should only drop to zero on a system with over 131.84
817c478bd9Sstevel@tonic-gate * terabytes of memory, or where the default scrubber parameters have
827c478bd9Sstevel@tonic-gate * been adjusted. For example, reducing memscrub_span_pages or
837c478bd9Sstevel@tonic-gate * memscrub_period_sec causes the sleep time to drop to zero with less
847c478bd9Sstevel@tonic-gate * memory. Note that since the sleep time is calculated in clock ticks,
857c478bd9Sstevel@tonic-gate * using hires clock ticks allows for more memory before the sleep time
867c478bd9Sstevel@tonic-gate * becomes zero.
877c478bd9Sstevel@tonic-gate *
887c478bd9Sstevel@tonic-gate * The scrubber will exit (or never be started) if it finds the variable
897c478bd9Sstevel@tonic-gate * "disable_memscrub" set.
907c478bd9Sstevel@tonic-gate *
917c478bd9Sstevel@tonic-gate * The scrubber will pause (not read memory) when "pause_memscrub"
927c478bd9Sstevel@tonic-gate * is set. It will check the state of pause_memscrub at each wakeup
937c478bd9Sstevel@tonic-gate * period. The scrubber will not make up for lost time. If you
947c478bd9Sstevel@tonic-gate * pause the scrubber for a prolonged period of time you can use
957c478bd9Sstevel@tonic-gate * the "read_all_memscrub" switch (see below) to catch up. In addition,
967c478bd9Sstevel@tonic-gate * pause_memscrub is used internally by the post memory DR callbacks.
977c478bd9Sstevel@tonic-gate * It is set for the small period of time during which the callbacks
987c478bd9Sstevel@tonic-gate * are executing. This ensures "memscrub_lock" will be released,
997c478bd9Sstevel@tonic-gate * allowing the callbacks to finish.
1007c478bd9Sstevel@tonic-gate *
1017c478bd9Sstevel@tonic-gate * The scrubber will read all memory if "read_all_memscrub" is set.
1027c478bd9Sstevel@tonic-gate * The normal span read will also occur during the wakeup.
1037c478bd9Sstevel@tonic-gate *
1047c478bd9Sstevel@tonic-gate * MEMSCRUB_MIN_PAGES (32MB) is the minimum amount of memory a system
1057c478bd9Sstevel@tonic-gate * must have before we'll start the scrubber.
1067c478bd9Sstevel@tonic-gate *
10794836d42Spt * MEMSCRUB_DFL_SPAN_PAGES (32MB) is based on the guess that 0.025 sec
1087c478bd9Sstevel@tonic-gate * is a "good" amount of minimum time for the thread to run at a time.
1097c478bd9Sstevel@tonic-gate *
1107c478bd9Sstevel@tonic-gate * MEMSCRUB_DFL_PERIOD_SEC (12 hours) is nearly a total guess --
1117c478bd9Sstevel@tonic-gate * twice the frequency the hardware folk estimated would be necessary.
1127c478bd9Sstevel@tonic-gate *
1137c478bd9Sstevel@tonic-gate * MEMSCRUB_DFL_THREAD_PRI (MINCLSYSPRI) is based on the assumption
1147c478bd9Sstevel@tonic-gate * that the scurbber should get its fair share of time (since it
1157c478bd9Sstevel@tonic-gate * is short). At a priority of 0 the scrubber will be starved.
1167c478bd9Sstevel@tonic-gate */
1177c478bd9Sstevel@tonic-gate
1187c478bd9Sstevel@tonic-gate #include <sys/systm.h> /* timeout, types, t_lock */
1197c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
1207c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h> /* MIN */
1217c478bd9Sstevel@tonic-gate #include <sys/memlist.h> /* memlist */
1227c478bd9Sstevel@tonic-gate #include <sys/mem_config.h> /* memory add/delete */
1237c478bd9Sstevel@tonic-gate #include <sys/kmem.h> /* KMEM_NOSLEEP */
1247c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> /* ncpus_online */
1257c478bd9Sstevel@tonic-gate #include <sys/debug.h> /* ASSERTs */
1267c478bd9Sstevel@tonic-gate #include <sys/machsystm.h> /* lddphys */
1277c478bd9Sstevel@tonic-gate #include <sys/cpu_module.h> /* vtag_flushpage */
1287c478bd9Sstevel@tonic-gate #include <sys/kstat.h>
1297c478bd9Sstevel@tonic-gate #include <sys/atomic.h> /* atomic_add_32 */
1307c478bd9Sstevel@tonic-gate
1317c478bd9Sstevel@tonic-gate #include <vm/hat.h>
1327c478bd9Sstevel@tonic-gate #include <vm/seg_kmem.h>
1337c478bd9Sstevel@tonic-gate #include <vm/hat_sfmmu.h> /* XXX FIXME - delete */
1347c478bd9Sstevel@tonic-gate
1357c478bd9Sstevel@tonic-gate #include <sys/time.h>
1367c478bd9Sstevel@tonic-gate #include <sys/callb.h> /* CPR callback */
1377c478bd9Sstevel@tonic-gate #include <sys/ontrap.h>
1387c478bd9Sstevel@tonic-gate
1397c478bd9Sstevel@tonic-gate /*
1407c478bd9Sstevel@tonic-gate * Should really have paddr_t defined, but it is broken. Use
1417c478bd9Sstevel@tonic-gate * ms_paddr_t in the meantime to make the code cleaner
1427c478bd9Sstevel@tonic-gate */
1437c478bd9Sstevel@tonic-gate typedef uint64_t ms_paddr_t;
1447c478bd9Sstevel@tonic-gate
1457c478bd9Sstevel@tonic-gate /*
1467c478bd9Sstevel@tonic-gate * Global Routines:
1477c478bd9Sstevel@tonic-gate */
1487c478bd9Sstevel@tonic-gate int memscrub_add_span(pfn_t pfn, pgcnt_t pages);
1497c478bd9Sstevel@tonic-gate int memscrub_delete_span(pfn_t pfn, pgcnt_t pages);
1507c478bd9Sstevel@tonic-gate int memscrub_init(void);
15161ef38f7Svb void memscrub_induced_error(void);
1527c478bd9Sstevel@tonic-gate
1537c478bd9Sstevel@tonic-gate /*
1547c478bd9Sstevel@tonic-gate * Global Data:
1557c478bd9Sstevel@tonic-gate */
1567c478bd9Sstevel@tonic-gate
1577c478bd9Sstevel@tonic-gate /*
1587c478bd9Sstevel@tonic-gate * scrub if we have at least this many pages
1597c478bd9Sstevel@tonic-gate */
1607c478bd9Sstevel@tonic-gate #define MEMSCRUB_MIN_PAGES (32 * 1024 * 1024 / PAGESIZE)
1617c478bd9Sstevel@tonic-gate
1627c478bd9Sstevel@tonic-gate /*
1637c478bd9Sstevel@tonic-gate * scan all of physical memory at least once every MEMSCRUB_PERIOD_SEC
1647c478bd9Sstevel@tonic-gate */
1657c478bd9Sstevel@tonic-gate #define MEMSCRUB_DFL_PERIOD_SEC (12 * 60 * 60) /* 12 hours */
1667c478bd9Sstevel@tonic-gate
1677c478bd9Sstevel@tonic-gate /*
1687c478bd9Sstevel@tonic-gate * scan at least MEMSCRUB_DFL_SPAN_PAGES each iteration
1697c478bd9Sstevel@tonic-gate */
17094836d42Spt #define MEMSCRUB_DFL_SPAN_PAGES ((32 * 1024 * 1024) / PAGESIZE)
1717c478bd9Sstevel@tonic-gate
1727c478bd9Sstevel@tonic-gate /*
1737c478bd9Sstevel@tonic-gate * almost anything is higher priority than scrubbing
1747c478bd9Sstevel@tonic-gate */
1757c478bd9Sstevel@tonic-gate #define MEMSCRUB_DFL_THREAD_PRI MINCLSYSPRI
1767c478bd9Sstevel@tonic-gate
1777c478bd9Sstevel@tonic-gate /*
1787c478bd9Sstevel@tonic-gate * size used when scanning memory
1797c478bd9Sstevel@tonic-gate */
1807c478bd9Sstevel@tonic-gate #define MEMSCRUB_BLOCK_SIZE 256
1817c478bd9Sstevel@tonic-gate #define MEMSCRUB_BLOCK_SIZE_SHIFT 8 /* log2(MEMSCRUB_BLOCK_SIZE) */
1827c478bd9Sstevel@tonic-gate #define MEMSCRUB_BLOCKS_PER_PAGE (PAGESIZE >> MEMSCRUB_BLOCK_SIZE_SHIFT)
1837c478bd9Sstevel@tonic-gate
1847c478bd9Sstevel@tonic-gate #define MEMSCRUB_BPP4M MMU_PAGESIZE4M >> MEMSCRUB_BLOCK_SIZE_SHIFT
1857c478bd9Sstevel@tonic-gate #define MEMSCRUB_BPP512K MMU_PAGESIZE512K >> MEMSCRUB_BLOCK_SIZE_SHIFT
1867c478bd9Sstevel@tonic-gate #define MEMSCRUB_BPP64K MMU_PAGESIZE64K >> MEMSCRUB_BLOCK_SIZE_SHIFT
1877c478bd9Sstevel@tonic-gate #define MEMSCRUB_BPP MMU_PAGESIZE >> MEMSCRUB_BLOCK_SIZE_SHIFT
1887c478bd9Sstevel@tonic-gate
1897c478bd9Sstevel@tonic-gate /*
1907c478bd9Sstevel@tonic-gate * This message indicates that we have exceeded the limitations of
1917c478bd9Sstevel@tonic-gate * the memscrubber. See the comments above regarding what would
1927c478bd9Sstevel@tonic-gate * cause the sleep time to become zero. In DEBUG mode, this message
1937c478bd9Sstevel@tonic-gate * is logged on the console and in the messages file. In non-DEBUG
1947c478bd9Sstevel@tonic-gate * mode, it is only logged in the messages file.
1957c478bd9Sstevel@tonic-gate */
1967c478bd9Sstevel@tonic-gate #ifdef DEBUG
1977c478bd9Sstevel@tonic-gate #define MEMSCRUB_OVERRIDE_MSG "Memory scrubber sleep time is zero " \
1987c478bd9Sstevel@tonic-gate "seconds, consuming entire CPU."
1997c478bd9Sstevel@tonic-gate #else
2007c478bd9Sstevel@tonic-gate #define MEMSCRUB_OVERRIDE_MSG "!Memory scrubber sleep time is zero " \
2017c478bd9Sstevel@tonic-gate "seconds, consuming entire CPU."
2027c478bd9Sstevel@tonic-gate #endif /* DEBUG */
2037c478bd9Sstevel@tonic-gate
2047c478bd9Sstevel@tonic-gate /*
2057c478bd9Sstevel@tonic-gate * we can patch these defaults in /etc/system if necessary
2067c478bd9Sstevel@tonic-gate */
2077c478bd9Sstevel@tonic-gate uint_t disable_memscrub = 0;
2087c478bd9Sstevel@tonic-gate uint_t pause_memscrub = 0;
2097c478bd9Sstevel@tonic-gate uint_t read_all_memscrub = 0;
2107c478bd9Sstevel@tonic-gate uint_t memscrub_verbose = 0;
2117c478bd9Sstevel@tonic-gate uint_t memscrub_all_idle = 0;
2127c478bd9Sstevel@tonic-gate uint_t memscrub_span_pages = MEMSCRUB_DFL_SPAN_PAGES;
2137c478bd9Sstevel@tonic-gate uint_t memscrub_period_sec = MEMSCRUB_DFL_PERIOD_SEC;
2147c478bd9Sstevel@tonic-gate uint_t memscrub_thread_pri = MEMSCRUB_DFL_THREAD_PRI;
2157c478bd9Sstevel@tonic-gate uint_t memscrub_delay_start_sec = 5 * 60;
2167c478bd9Sstevel@tonic-gate uint_t memscrub_override_ticks = 1;
2177c478bd9Sstevel@tonic-gate
2187c478bd9Sstevel@tonic-gate /*
2197c478bd9Sstevel@tonic-gate * Static Routines
2207c478bd9Sstevel@tonic-gate */
2217c478bd9Sstevel@tonic-gate static void memscrubber(void);
2227c478bd9Sstevel@tonic-gate static void memscrub_cleanup(void);
2237c478bd9Sstevel@tonic-gate static int memscrub_add_span_gen(pfn_t, pgcnt_t, struct memlist **, uint_t *);
2247c478bd9Sstevel@tonic-gate static int memscrub_verify_span(ms_paddr_t *addrp, pgcnt_t *pagesp);
2257c478bd9Sstevel@tonic-gate static void memscrub_scan(uint_t blks, ms_paddr_t src);
2267c478bd9Sstevel@tonic-gate
2277c478bd9Sstevel@tonic-gate /*
2287c478bd9Sstevel@tonic-gate * Static Data
2297c478bd9Sstevel@tonic-gate */
2307c478bd9Sstevel@tonic-gate
2317c478bd9Sstevel@tonic-gate static struct memlist *memscrub_memlist;
2327c478bd9Sstevel@tonic-gate static uint_t memscrub_phys_pages;
2337c478bd9Sstevel@tonic-gate
2347c478bd9Sstevel@tonic-gate static kcondvar_t memscrub_cv;
2357c478bd9Sstevel@tonic-gate static kmutex_t memscrub_lock;
2367c478bd9Sstevel@tonic-gate /*
2377c478bd9Sstevel@tonic-gate * memscrub_lock protects memscrub_memlist, interval_ticks, cprinfo, ...
2387c478bd9Sstevel@tonic-gate */
2397c478bd9Sstevel@tonic-gate static void memscrub_init_mem_config(void);
2407c478bd9Sstevel@tonic-gate static void memscrub_uninit_mem_config(void);
2417c478bd9Sstevel@tonic-gate
24261ef38f7Svb /*
24361ef38f7Svb * Linked list of memscrub aware spans having retired pages.
24461ef38f7Svb * Currently enabled only on sun4u USIII-based platforms.
24561ef38f7Svb */
24661ef38f7Svb typedef struct memscrub_page_retire_span {
24761ef38f7Svb ms_paddr_t address;
24861ef38f7Svb struct memscrub_page_retire_span *next;
24961ef38f7Svb } memscrub_page_retire_span_t;
25061ef38f7Svb
25161ef38f7Svb static memscrub_page_retire_span_t *memscrub_page_retire_span_list = NULL;
25261ef38f7Svb
25361ef38f7Svb static void memscrub_page_retire_span_add(ms_paddr_t);
25461ef38f7Svb static void memscrub_page_retire_span_delete(ms_paddr_t);
25561ef38f7Svb static int memscrub_page_retire_span_search(ms_paddr_t);
25661ef38f7Svb static void memscrub_page_retire_span_list_update(void);
25761ef38f7Svb
25861ef38f7Svb /*
25961ef38f7Svb * add_to_page_retire_list: Set by cpu_async_log_err() routine
26061ef38f7Svb * by calling memscrub_induced_error() when CE/UE occurs on a retired
26161ef38f7Svb * page due to memscrub reading. Cleared by memscrub after updating
26261ef38f7Svb * global page retire span list. Piggybacking on protection of
26361ef38f7Svb * memscrub_lock, which is held during set and clear.
26461ef38f7Svb * Note: When cpu_async_log_err() calls memscrub_induced_error(), it is running
26561ef38f7Svb * on softint context, which gets fired on a cpu memscrub thread currently
26661ef38f7Svb * running. Memscrub thread has affinity set during memscrub_read(), hence
26761ef38f7Svb * migration to new cpu not expected.
26861ef38f7Svb */
26961ef38f7Svb static int add_to_page_retire_list = 0;
27061ef38f7Svb
2717c478bd9Sstevel@tonic-gate /*
2727c478bd9Sstevel@tonic-gate * Keep track of some interesting statistics
2737c478bd9Sstevel@tonic-gate */
2747c478bd9Sstevel@tonic-gate static struct memscrub_kstats {
2757c478bd9Sstevel@tonic-gate kstat_named_t done_early; /* ahead of schedule */
2767c478bd9Sstevel@tonic-gate kstat_named_t early_sec; /* by cumulative num secs */
2777c478bd9Sstevel@tonic-gate kstat_named_t done_late; /* behind schedule */
2787c478bd9Sstevel@tonic-gate kstat_named_t late_sec; /* by cumulative num secs */
2797c478bd9Sstevel@tonic-gate kstat_named_t interval_ticks; /* num ticks between intervals */
2807c478bd9Sstevel@tonic-gate kstat_named_t force_run; /* forced to run, non-timeout */
2817c478bd9Sstevel@tonic-gate kstat_named_t errors_found; /* num errors found by memscrub */
2827c478bd9Sstevel@tonic-gate } memscrub_counts = {
2837c478bd9Sstevel@tonic-gate { "done_early", KSTAT_DATA_UINT32 },
2847c478bd9Sstevel@tonic-gate { "early_sec", KSTAT_DATA_UINT32 },
2857c478bd9Sstevel@tonic-gate { "done_late", KSTAT_DATA_UINT32 },
2867c478bd9Sstevel@tonic-gate { "late_sec", KSTAT_DATA_UINT32 },
2877c478bd9Sstevel@tonic-gate { "interval_ticks", KSTAT_DATA_UINT32 },
2887c478bd9Sstevel@tonic-gate { "force_run", KSTAT_DATA_UINT32 },
2897c478bd9Sstevel@tonic-gate { "errors_found", KSTAT_DATA_UINT32 },
2907c478bd9Sstevel@tonic-gate };
2911a3c9a5aSVijay S Balakrishna
2921a3c9a5aSVijay S Balakrishna #define MEMSCRUB_STAT_INC(stat) memscrub_counts.stat.value.ui32++
2931a3c9a5aSVijay S Balakrishna #define MEMSCRUB_STAT_SET(stat, val) memscrub_counts.stat.value.ui32 = (val)
2941a3c9a5aSVijay S Balakrishna #define MEMSCRUB_STAT_NINC(stat, val) memscrub_counts.stat.value.ui32 += (val)
2951a3c9a5aSVijay S Balakrishna
2967c478bd9Sstevel@tonic-gate static struct kstat *memscrub_ksp = (struct kstat *)NULL;
2977c478bd9Sstevel@tonic-gate
2987c478bd9Sstevel@tonic-gate static timeout_id_t memscrub_tid = 0; /* keep track of timeout id */
2997c478bd9Sstevel@tonic-gate
3007c478bd9Sstevel@tonic-gate /*
3017c478bd9Sstevel@tonic-gate * create memscrub_memlist from phys_install list
3027c478bd9Sstevel@tonic-gate * initialize locks, set memscrub_phys_pages.
3037c478bd9Sstevel@tonic-gate */
3047c478bd9Sstevel@tonic-gate int
memscrub_init(void)3057c478bd9Sstevel@tonic-gate memscrub_init(void)
3067c478bd9Sstevel@tonic-gate {
3077c478bd9Sstevel@tonic-gate struct memlist *src;
3087c478bd9Sstevel@tonic-gate
3097c478bd9Sstevel@tonic-gate /*
3107c478bd9Sstevel@tonic-gate * only startup the scrubber if we have a minimum
3117c478bd9Sstevel@tonic-gate * number of pages
3127c478bd9Sstevel@tonic-gate */
3137c478bd9Sstevel@tonic-gate if (physinstalled >= MEMSCRUB_MIN_PAGES) {
3147c478bd9Sstevel@tonic-gate
3157c478bd9Sstevel@tonic-gate /*
3167c478bd9Sstevel@tonic-gate * initialize locks
3177c478bd9Sstevel@tonic-gate */
3187c478bd9Sstevel@tonic-gate mutex_init(&memscrub_lock, NULL, MUTEX_DRIVER, NULL);
3197c478bd9Sstevel@tonic-gate cv_init(&memscrub_cv, NULL, CV_DRIVER, NULL);
3207c478bd9Sstevel@tonic-gate
3217c478bd9Sstevel@tonic-gate /*
3227c478bd9Sstevel@tonic-gate * copy phys_install to memscrub_memlist
3237c478bd9Sstevel@tonic-gate */
32456f33205SJonathan Adams for (src = phys_install; src; src = src->ml_next) {
3257c478bd9Sstevel@tonic-gate if (memscrub_add_span(
32656f33205SJonathan Adams (pfn_t)(src->ml_address >> PAGESHIFT),
32756f33205SJonathan Adams (pgcnt_t)(src->ml_size >> PAGESHIFT))) {
3287c478bd9Sstevel@tonic-gate memscrub_cleanup();
3297c478bd9Sstevel@tonic-gate return (-1);
3307c478bd9Sstevel@tonic-gate }
3317c478bd9Sstevel@tonic-gate }
3327c478bd9Sstevel@tonic-gate
3337c478bd9Sstevel@tonic-gate /*
3347c478bd9Sstevel@tonic-gate * initialize kstats
3357c478bd9Sstevel@tonic-gate */
3367c478bd9Sstevel@tonic-gate memscrub_ksp = kstat_create("unix", 0, "memscrub_kstat",
33756f33205SJonathan Adams "misc", KSTAT_TYPE_NAMED,
33856f33205SJonathan Adams sizeof (memscrub_counts) / sizeof (kstat_named_t),
33956f33205SJonathan Adams KSTAT_FLAG_VIRTUAL | KSTAT_FLAG_WRITABLE);
3407c478bd9Sstevel@tonic-gate
3417c478bd9Sstevel@tonic-gate if (memscrub_ksp) {
3427c478bd9Sstevel@tonic-gate memscrub_ksp->ks_data = (void *)&memscrub_counts;
3437c478bd9Sstevel@tonic-gate kstat_install(memscrub_ksp);
3447c478bd9Sstevel@tonic-gate } else {
3457c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "Memscrubber cannot create kstats\n");
3467c478bd9Sstevel@tonic-gate }
3477c478bd9Sstevel@tonic-gate
3487c478bd9Sstevel@tonic-gate /*
3497c478bd9Sstevel@tonic-gate * create memscrubber thread
3507c478bd9Sstevel@tonic-gate */
3517c478bd9Sstevel@tonic-gate (void) thread_create(NULL, 0, (void (*)())memscrubber,
3527c478bd9Sstevel@tonic-gate NULL, 0, &p0, TS_RUN, memscrub_thread_pri);
3537c478bd9Sstevel@tonic-gate
3547c478bd9Sstevel@tonic-gate /*
3557c478bd9Sstevel@tonic-gate * We don't want call backs changing the list
3567c478bd9Sstevel@tonic-gate * if there is no thread running. We do not
3577c478bd9Sstevel@tonic-gate * attempt to deal with stopping/starting scrubbing
3587c478bd9Sstevel@tonic-gate * on memory size changes.
3597c478bd9Sstevel@tonic-gate */
3607c478bd9Sstevel@tonic-gate memscrub_init_mem_config();
3617c478bd9Sstevel@tonic-gate }
3627c478bd9Sstevel@tonic-gate
3637c478bd9Sstevel@tonic-gate return (0);
3647c478bd9Sstevel@tonic-gate }
3657c478bd9Sstevel@tonic-gate
3667c478bd9Sstevel@tonic-gate static void
memscrub_cleanup(void)3677c478bd9Sstevel@tonic-gate memscrub_cleanup(void)
3687c478bd9Sstevel@tonic-gate {
3697c478bd9Sstevel@tonic-gate memscrub_uninit_mem_config();
3707c478bd9Sstevel@tonic-gate while (memscrub_memlist) {
3717c478bd9Sstevel@tonic-gate (void) memscrub_delete_span(
37256f33205SJonathan Adams (pfn_t)(memscrub_memlist->ml_address >> PAGESHIFT),
37356f33205SJonathan Adams (pgcnt_t)(memscrub_memlist->ml_size >> PAGESHIFT));
3747c478bd9Sstevel@tonic-gate }
3757c478bd9Sstevel@tonic-gate if (memscrub_ksp)
3767c478bd9Sstevel@tonic-gate kstat_delete(memscrub_ksp);
3777c478bd9Sstevel@tonic-gate cv_destroy(&memscrub_cv);
3787c478bd9Sstevel@tonic-gate mutex_destroy(&memscrub_lock);
3797c478bd9Sstevel@tonic-gate }
3807c478bd9Sstevel@tonic-gate
3817c478bd9Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG
3827c478bd9Sstevel@tonic-gate static void
memscrub_printmemlist(char * title,struct memlist * listp)3837c478bd9Sstevel@tonic-gate memscrub_printmemlist(char *title, struct memlist *listp)
3847c478bd9Sstevel@tonic-gate {
3857c478bd9Sstevel@tonic-gate struct memlist *list;
3867c478bd9Sstevel@tonic-gate
3877c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "%s:\n", title);
3887c478bd9Sstevel@tonic-gate
38956f33205SJonathan Adams for (list = listp; list; list = list->ml_next) {
3907c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "addr = 0x%llx, size = 0x%llx\n",
39156f33205SJonathan Adams list->ml_address, list->ml_size);
3927c478bd9Sstevel@tonic-gate }
3937c478bd9Sstevel@tonic-gate }
3947c478bd9Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */
3957c478bd9Sstevel@tonic-gate
3967c478bd9Sstevel@tonic-gate /* ARGSUSED */
3977c478bd9Sstevel@tonic-gate static void
memscrub_wakeup(void * c)3987c478bd9Sstevel@tonic-gate memscrub_wakeup(void *c)
3997c478bd9Sstevel@tonic-gate {
4007c478bd9Sstevel@tonic-gate /*
4017c478bd9Sstevel@tonic-gate * grab mutex to guarantee that our wakeup call
4027c478bd9Sstevel@tonic-gate * arrives after we go to sleep -- so we can't sleep forever.
4037c478bd9Sstevel@tonic-gate */
4047c478bd9Sstevel@tonic-gate mutex_enter(&memscrub_lock);
4057c478bd9Sstevel@tonic-gate cv_signal(&memscrub_cv);
4067c478bd9Sstevel@tonic-gate mutex_exit(&memscrub_lock);
4077c478bd9Sstevel@tonic-gate }
4087c478bd9Sstevel@tonic-gate
4097c478bd9Sstevel@tonic-gate /*
4107c478bd9Sstevel@tonic-gate * provide an interface external to the memscrubber
4117c478bd9Sstevel@tonic-gate * which will force the memscrub thread to run vs.
4127c478bd9Sstevel@tonic-gate * waiting for the timeout, if one is set
4137c478bd9Sstevel@tonic-gate */
4147c478bd9Sstevel@tonic-gate void
memscrub_run(void)4157c478bd9Sstevel@tonic-gate memscrub_run(void)
4167c478bd9Sstevel@tonic-gate {
4171a3c9a5aSVijay S Balakrishna MEMSCRUB_STAT_INC(force_run);
4187c478bd9Sstevel@tonic-gate if (memscrub_tid) {
4197c478bd9Sstevel@tonic-gate (void) untimeout(memscrub_tid);
4207c478bd9Sstevel@tonic-gate memscrub_wakeup((void *)NULL);
4217c478bd9Sstevel@tonic-gate }
4227c478bd9Sstevel@tonic-gate }
4237c478bd9Sstevel@tonic-gate
4247c478bd9Sstevel@tonic-gate /*
4257c478bd9Sstevel@tonic-gate * this calculation doesn't account for the time
4267c478bd9Sstevel@tonic-gate * that the actual scan consumes -- so we'd fall
4277c478bd9Sstevel@tonic-gate * slightly behind schedule with this interval.
4287c478bd9Sstevel@tonic-gate * It's very small.
4297c478bd9Sstevel@tonic-gate */
4307c478bd9Sstevel@tonic-gate
4317c478bd9Sstevel@tonic-gate static uint_t
compute_interval_ticks(void)4327c478bd9Sstevel@tonic-gate compute_interval_ticks(void)
4337c478bd9Sstevel@tonic-gate {
4347c478bd9Sstevel@tonic-gate /*
4357c478bd9Sstevel@tonic-gate * We use msp_safe mpp_safe below to insure somebody
4367c478bd9Sstevel@tonic-gate * doesn't set memscrub_span_pages or memscrub_phys_pages
4377c478bd9Sstevel@tonic-gate * to 0 on us.
4387c478bd9Sstevel@tonic-gate */
4397c478bd9Sstevel@tonic-gate static uint_t msp_safe, mpp_safe;
4407c478bd9Sstevel@tonic-gate static uint_t interval_ticks, period_ticks;
4417c478bd9Sstevel@tonic-gate msp_safe = memscrub_span_pages;
4427c478bd9Sstevel@tonic-gate mpp_safe = memscrub_phys_pages;
4437c478bd9Sstevel@tonic-gate
4447c478bd9Sstevel@tonic-gate period_ticks = memscrub_period_sec * hz;
4457c478bd9Sstevel@tonic-gate interval_ticks = period_ticks;
4467c478bd9Sstevel@tonic-gate
4477c478bd9Sstevel@tonic-gate ASSERT(mutex_owned(&memscrub_lock));
4487c478bd9Sstevel@tonic-gate
4497c478bd9Sstevel@tonic-gate if ((msp_safe != 0) && (mpp_safe != 0)) {
4507c478bd9Sstevel@tonic-gate if (memscrub_phys_pages <= msp_safe) {
4517c478bd9Sstevel@tonic-gate interval_ticks = period_ticks;
4527c478bd9Sstevel@tonic-gate } else {
4537c478bd9Sstevel@tonic-gate interval_ticks = (period_ticks /
4547c478bd9Sstevel@tonic-gate (mpp_safe / msp_safe));
4557c478bd9Sstevel@tonic-gate }
4567c478bd9Sstevel@tonic-gate }
4577c478bd9Sstevel@tonic-gate return (interval_ticks);
4587c478bd9Sstevel@tonic-gate }
4597c478bd9Sstevel@tonic-gate
4607c478bd9Sstevel@tonic-gate void
memscrubber(void)4617c478bd9Sstevel@tonic-gate memscrubber(void)
4627c478bd9Sstevel@tonic-gate {
4637c478bd9Sstevel@tonic-gate ms_paddr_t address, addr;
4647c478bd9Sstevel@tonic-gate time_t deadline;
4657c478bd9Sstevel@tonic-gate pgcnt_t pages;
4667c478bd9Sstevel@tonic-gate uint_t reached_end = 1;
4677c478bd9Sstevel@tonic-gate uint_t paused_message = 0;
4687c478bd9Sstevel@tonic-gate uint_t interval_ticks = 0;
4697c478bd9Sstevel@tonic-gate uint_t sleep_warn_printed = 0;
4707c478bd9Sstevel@tonic-gate callb_cpr_t cprinfo;
4717c478bd9Sstevel@tonic-gate
4727c478bd9Sstevel@tonic-gate /*
4737c478bd9Sstevel@tonic-gate * notify CPR of our existence
4747c478bd9Sstevel@tonic-gate */
4757c478bd9Sstevel@tonic-gate CALLB_CPR_INIT(&cprinfo, &memscrub_lock, callb_generic_cpr, "memscrub");
4767c478bd9Sstevel@tonic-gate
4777c478bd9Sstevel@tonic-gate mutex_enter(&memscrub_lock);
4787c478bd9Sstevel@tonic-gate
4797c478bd9Sstevel@tonic-gate if (memscrub_memlist == NULL) {
4807c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "memscrub_memlist not initialized.");
4817c478bd9Sstevel@tonic-gate goto memscrub_exit;
4827c478bd9Sstevel@tonic-gate }
4837c478bd9Sstevel@tonic-gate
48456f33205SJonathan Adams address = memscrub_memlist->ml_address;
4857c478bd9Sstevel@tonic-gate
4867c478bd9Sstevel@tonic-gate deadline = gethrestime_sec() + memscrub_delay_start_sec;
4877c478bd9Sstevel@tonic-gate
4887c478bd9Sstevel@tonic-gate for (;;) {
4897c478bd9Sstevel@tonic-gate if (disable_memscrub)
4907c478bd9Sstevel@tonic-gate break;
4917c478bd9Sstevel@tonic-gate
4927c478bd9Sstevel@tonic-gate /*
4937c478bd9Sstevel@tonic-gate * compute interval_ticks
4947c478bd9Sstevel@tonic-gate */
4957c478bd9Sstevel@tonic-gate interval_ticks = compute_interval_ticks();
4967c478bd9Sstevel@tonic-gate
4977c478bd9Sstevel@tonic-gate /*
4987c478bd9Sstevel@tonic-gate * If the calculated sleep time is zero, and pause_memscrub
4997c478bd9Sstevel@tonic-gate * has been set, make sure we sleep so that another thread
5007c478bd9Sstevel@tonic-gate * can acquire memscrub_lock.
5017c478bd9Sstevel@tonic-gate */
5027c478bd9Sstevel@tonic-gate if (interval_ticks == 0 && pause_memscrub) {
5037c478bd9Sstevel@tonic-gate interval_ticks = hz;
5047c478bd9Sstevel@tonic-gate }
5057c478bd9Sstevel@tonic-gate
5067c478bd9Sstevel@tonic-gate /*
5077c478bd9Sstevel@tonic-gate * And as a fail safe, under normal non-paused operation, do
5087c478bd9Sstevel@tonic-gate * not allow the sleep time to be zero.
5097c478bd9Sstevel@tonic-gate */
5107c478bd9Sstevel@tonic-gate if (interval_ticks == 0) {
5117c478bd9Sstevel@tonic-gate interval_ticks = memscrub_override_ticks;
5127c478bd9Sstevel@tonic-gate if (!sleep_warn_printed) {
5137c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, MEMSCRUB_OVERRIDE_MSG);
5147c478bd9Sstevel@tonic-gate sleep_warn_printed = 1;
5157c478bd9Sstevel@tonic-gate }
5167c478bd9Sstevel@tonic-gate }
5177c478bd9Sstevel@tonic-gate
5181a3c9a5aSVijay S Balakrishna MEMSCRUB_STAT_SET(interval_ticks, interval_ticks);
5197c478bd9Sstevel@tonic-gate
5207c478bd9Sstevel@tonic-gate /*
5217c478bd9Sstevel@tonic-gate * Did we just reach the end of memory? If we are at the
5227c478bd9Sstevel@tonic-gate * end of memory, delay end of memory processing until
5237c478bd9Sstevel@tonic-gate * pause_memscrub is not set.
5247c478bd9Sstevel@tonic-gate */
5257c478bd9Sstevel@tonic-gate if (reached_end && !pause_memscrub) {
5267c478bd9Sstevel@tonic-gate time_t now = gethrestime_sec();
5277c478bd9Sstevel@tonic-gate
5287c478bd9Sstevel@tonic-gate if (now >= deadline) {
5291a3c9a5aSVijay S Balakrishna MEMSCRUB_STAT_INC(done_late);
5301a3c9a5aSVijay S Balakrishna MEMSCRUB_STAT_NINC(late_sec, now - deadline);
5317c478bd9Sstevel@tonic-gate /*
5327c478bd9Sstevel@tonic-gate * past deadline, start right away
5337c478bd9Sstevel@tonic-gate */
5347c478bd9Sstevel@tonic-gate interval_ticks = 0;
5357c478bd9Sstevel@tonic-gate
5367c478bd9Sstevel@tonic-gate deadline = now + memscrub_period_sec;
5377c478bd9Sstevel@tonic-gate } else {
5387c478bd9Sstevel@tonic-gate /*
5397c478bd9Sstevel@tonic-gate * we finished ahead of schedule.
5407c478bd9Sstevel@tonic-gate * wait till previous deadline before re-start.
5417c478bd9Sstevel@tonic-gate */
5427c478bd9Sstevel@tonic-gate interval_ticks = (deadline - now) * hz;
5431a3c9a5aSVijay S Balakrishna MEMSCRUB_STAT_INC(done_early);
5441a3c9a5aSVijay S Balakrishna MEMSCRUB_STAT_NINC(early_sec, deadline - now);
5457c478bd9Sstevel@tonic-gate deadline += memscrub_period_sec;
5467c478bd9Sstevel@tonic-gate }
5477c478bd9Sstevel@tonic-gate reached_end = 0;
5487c478bd9Sstevel@tonic-gate sleep_warn_printed = 0;
5497c478bd9Sstevel@tonic-gate }
5507c478bd9Sstevel@tonic-gate
5517c478bd9Sstevel@tonic-gate if (interval_ticks != 0) {
5527c478bd9Sstevel@tonic-gate /*
5537c478bd9Sstevel@tonic-gate * it is safe from our standpoint for CPR to
5547c478bd9Sstevel@tonic-gate * suspend the system
5557c478bd9Sstevel@tonic-gate */
5567c478bd9Sstevel@tonic-gate CALLB_CPR_SAFE_BEGIN(&cprinfo);
5577c478bd9Sstevel@tonic-gate
5587c478bd9Sstevel@tonic-gate /*
5597c478bd9Sstevel@tonic-gate * hit the snooze bar
5607c478bd9Sstevel@tonic-gate */
5617c478bd9Sstevel@tonic-gate memscrub_tid = timeout(memscrub_wakeup, NULL,
5627c478bd9Sstevel@tonic-gate interval_ticks);
5637c478bd9Sstevel@tonic-gate
5647c478bd9Sstevel@tonic-gate /*
5657c478bd9Sstevel@tonic-gate * go to sleep
5667c478bd9Sstevel@tonic-gate */
5677c478bd9Sstevel@tonic-gate cv_wait(&memscrub_cv, &memscrub_lock);
5687c478bd9Sstevel@tonic-gate
5697c478bd9Sstevel@tonic-gate /*
5707c478bd9Sstevel@tonic-gate * at this point, no timeout should be set
5717c478bd9Sstevel@tonic-gate */
5727c478bd9Sstevel@tonic-gate memscrub_tid = 0;
5737c478bd9Sstevel@tonic-gate
5747c478bd9Sstevel@tonic-gate /*
5757c478bd9Sstevel@tonic-gate * we need to goto work and will be modifying
5767c478bd9Sstevel@tonic-gate * our internal state and mapping/unmapping
5777c478bd9Sstevel@tonic-gate * TTEs
5787c478bd9Sstevel@tonic-gate */
5797c478bd9Sstevel@tonic-gate CALLB_CPR_SAFE_END(&cprinfo, &memscrub_lock);
5807c478bd9Sstevel@tonic-gate }
5817c478bd9Sstevel@tonic-gate
5827c478bd9Sstevel@tonic-gate
5837c478bd9Sstevel@tonic-gate if (memscrub_phys_pages == 0) {
5847c478bd9Sstevel@tonic-gate cmn_err(CE_WARN, "Memory scrubber has 0 pages to read");
5857c478bd9Sstevel@tonic-gate goto memscrub_exit;
5867c478bd9Sstevel@tonic-gate }
5877c478bd9Sstevel@tonic-gate
5887c478bd9Sstevel@tonic-gate if (!pause_memscrub) {
5897c478bd9Sstevel@tonic-gate if (paused_message) {
5907c478bd9Sstevel@tonic-gate paused_message = 0;
5917c478bd9Sstevel@tonic-gate if (memscrub_verbose)
5927c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "Memory scrubber "
5937c478bd9Sstevel@tonic-gate "resuming");
5947c478bd9Sstevel@tonic-gate }
5957c478bd9Sstevel@tonic-gate
5967c478bd9Sstevel@tonic-gate if (read_all_memscrub) {
5977c478bd9Sstevel@tonic-gate if (memscrub_verbose)
5987c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "Memory scrubber "
5997c478bd9Sstevel@tonic-gate "reading all memory per request");
6007c478bd9Sstevel@tonic-gate
60156f33205SJonathan Adams addr = memscrub_memlist->ml_address;
6027c478bd9Sstevel@tonic-gate reached_end = 0;
6037c478bd9Sstevel@tonic-gate while (!reached_end) {
6047c478bd9Sstevel@tonic-gate if (disable_memscrub)
6057c478bd9Sstevel@tonic-gate break;
6067c478bd9Sstevel@tonic-gate pages = memscrub_phys_pages;
6077c478bd9Sstevel@tonic-gate reached_end = memscrub_verify_span(
6087c478bd9Sstevel@tonic-gate &addr, &pages);
6097c478bd9Sstevel@tonic-gate memscrub_scan(pages *
6107c478bd9Sstevel@tonic-gate MEMSCRUB_BLOCKS_PER_PAGE, addr);
6117c478bd9Sstevel@tonic-gate addr += ((uint64_t)pages * PAGESIZE);
6127c478bd9Sstevel@tonic-gate }
6137c478bd9Sstevel@tonic-gate read_all_memscrub = 0;
6147c478bd9Sstevel@tonic-gate }
6157c478bd9Sstevel@tonic-gate
6167c478bd9Sstevel@tonic-gate /*
6177c478bd9Sstevel@tonic-gate * read 1 span
6187c478bd9Sstevel@tonic-gate */
6197c478bd9Sstevel@tonic-gate pages = memscrub_span_pages;
6207c478bd9Sstevel@tonic-gate
6217c478bd9Sstevel@tonic-gate if (disable_memscrub)
6227c478bd9Sstevel@tonic-gate break;
6237c478bd9Sstevel@tonic-gate
6247c478bd9Sstevel@tonic-gate /*
6257c478bd9Sstevel@tonic-gate * determine physical address range
6267c478bd9Sstevel@tonic-gate */
6277c478bd9Sstevel@tonic-gate reached_end = memscrub_verify_span(&address,
6287c478bd9Sstevel@tonic-gate &pages);
6297c478bd9Sstevel@tonic-gate
6307c478bd9Sstevel@tonic-gate memscrub_scan(pages * MEMSCRUB_BLOCKS_PER_PAGE,
6317c478bd9Sstevel@tonic-gate address);
6327c478bd9Sstevel@tonic-gate
6337c478bd9Sstevel@tonic-gate address += ((uint64_t)pages * PAGESIZE);
6347c478bd9Sstevel@tonic-gate }
6357c478bd9Sstevel@tonic-gate
6367c478bd9Sstevel@tonic-gate if (pause_memscrub && !paused_message) {
6377c478bd9Sstevel@tonic-gate paused_message = 1;
6387c478bd9Sstevel@tonic-gate if (memscrub_verbose)
6397c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "Memory scrubber paused");
6407c478bd9Sstevel@tonic-gate }
6417c478bd9Sstevel@tonic-gate }
6427c478bd9Sstevel@tonic-gate
6437c478bd9Sstevel@tonic-gate memscrub_exit:
6447c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "Memory scrubber exiting");
6457c478bd9Sstevel@tonic-gate CALLB_CPR_EXIT(&cprinfo);
6467c478bd9Sstevel@tonic-gate memscrub_cleanup();
6477c478bd9Sstevel@tonic-gate thread_exit();
6487c478bd9Sstevel@tonic-gate /* NOTREACHED */
6497c478bd9Sstevel@tonic-gate }
6507c478bd9Sstevel@tonic-gate
6517c478bd9Sstevel@tonic-gate /*
6527c478bd9Sstevel@tonic-gate * condition address and size
6537c478bd9Sstevel@tonic-gate * such that they span legal physical addresses.
6547c478bd9Sstevel@tonic-gate *
6557c478bd9Sstevel@tonic-gate * when appropriate, address will be rounded up to start of next
6567c478bd9Sstevel@tonic-gate * struct memlist, and pages will be rounded down to the end of the
6577c478bd9Sstevel@tonic-gate * memlist size.
6587c478bd9Sstevel@tonic-gate *
6597c478bd9Sstevel@tonic-gate * returns 1 if reached end of list, else returns 0.
6607c478bd9Sstevel@tonic-gate */
6617c478bd9Sstevel@tonic-gate static int
memscrub_verify_span(ms_paddr_t * addrp,pgcnt_t * pagesp)6627c478bd9Sstevel@tonic-gate memscrub_verify_span(ms_paddr_t *addrp, pgcnt_t *pagesp)
6637c478bd9Sstevel@tonic-gate {
6647c478bd9Sstevel@tonic-gate struct memlist *mlp;
6657c478bd9Sstevel@tonic-gate ms_paddr_t address = *addrp;
6667c478bd9Sstevel@tonic-gate uint64_t bytes = (uint64_t)*pagesp * PAGESIZE;
6677c478bd9Sstevel@tonic-gate uint64_t bytes_remaining;
6687c478bd9Sstevel@tonic-gate int reached_end = 0;
6697c478bd9Sstevel@tonic-gate
6707c478bd9Sstevel@tonic-gate ASSERT(mutex_owned(&memscrub_lock));
6717c478bd9Sstevel@tonic-gate
6727c478bd9Sstevel@tonic-gate /*
6737c478bd9Sstevel@tonic-gate * find memlist struct that contains addrp
6747c478bd9Sstevel@tonic-gate * assumes memlist is sorted by ascending address.
6757c478bd9Sstevel@tonic-gate */
67656f33205SJonathan Adams for (mlp = memscrub_memlist; mlp != NULL; mlp = mlp->ml_next) {
6777c478bd9Sstevel@tonic-gate /*
6787c478bd9Sstevel@tonic-gate * if before this chunk, round up to beginning
6797c478bd9Sstevel@tonic-gate */
68056f33205SJonathan Adams if (address < mlp->ml_address) {
68156f33205SJonathan Adams address = mlp->ml_address;
6827c478bd9Sstevel@tonic-gate break;
6837c478bd9Sstevel@tonic-gate }
6847c478bd9Sstevel@tonic-gate /*
6857c478bd9Sstevel@tonic-gate * if before end of chunk, then we found it
6867c478bd9Sstevel@tonic-gate */
68756f33205SJonathan Adams if (address < (mlp->ml_address + mlp->ml_size))
6887c478bd9Sstevel@tonic-gate break;
6897c478bd9Sstevel@tonic-gate
6907c478bd9Sstevel@tonic-gate /* else go to next struct memlist */
6917c478bd9Sstevel@tonic-gate }
6927c478bd9Sstevel@tonic-gate /*
6937c478bd9Sstevel@tonic-gate * if we hit end of list, start at beginning
6947c478bd9Sstevel@tonic-gate */
6957c478bd9Sstevel@tonic-gate if (mlp == NULL) {
6967c478bd9Sstevel@tonic-gate mlp = memscrub_memlist;
69756f33205SJonathan Adams address = mlp->ml_address;
6987c478bd9Sstevel@tonic-gate }
6997c478bd9Sstevel@tonic-gate
7007c478bd9Sstevel@tonic-gate /*
7017c478bd9Sstevel@tonic-gate * now we have legal address, and its mlp, condition bytes
7027c478bd9Sstevel@tonic-gate */
70356f33205SJonathan Adams bytes_remaining = (mlp->ml_address + mlp->ml_size) - address;
7047c478bd9Sstevel@tonic-gate
7057c478bd9Sstevel@tonic-gate if (bytes > bytes_remaining)
7067c478bd9Sstevel@tonic-gate bytes = bytes_remaining;
7077c478bd9Sstevel@tonic-gate
7087c478bd9Sstevel@tonic-gate /*
7097c478bd9Sstevel@tonic-gate * will this span take us to end of list?
7107c478bd9Sstevel@tonic-gate */
71156f33205SJonathan Adams if ((mlp->ml_next == NULL) &&
71256f33205SJonathan Adams ((mlp->ml_address + mlp->ml_size) == (address + bytes)))
7137c478bd9Sstevel@tonic-gate reached_end = 1;
7147c478bd9Sstevel@tonic-gate
7157c478bd9Sstevel@tonic-gate /* return values */
7167c478bd9Sstevel@tonic-gate *addrp = address;
7177c478bd9Sstevel@tonic-gate *pagesp = bytes / PAGESIZE;
7187c478bd9Sstevel@tonic-gate
7197c478bd9Sstevel@tonic-gate return (reached_end);
7207c478bd9Sstevel@tonic-gate }
7217c478bd9Sstevel@tonic-gate
7227c478bd9Sstevel@tonic-gate /*
7237c478bd9Sstevel@tonic-gate * add a span to the memscrub list
7247c478bd9Sstevel@tonic-gate * add to memscrub_phys_pages
7257c478bd9Sstevel@tonic-gate */
7267c478bd9Sstevel@tonic-gate int
memscrub_add_span(pfn_t pfn,pgcnt_t pages)7277c478bd9Sstevel@tonic-gate memscrub_add_span(pfn_t pfn, pgcnt_t pages)
7287c478bd9Sstevel@tonic-gate {
7297c478bd9Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG
7307c478bd9Sstevel@tonic-gate ms_paddr_t address = (ms_paddr_t)pfn << PAGESHIFT;
7317c478bd9Sstevel@tonic-gate uint64_t bytes = (uint64_t)pages << PAGESHIFT;
7327c478bd9Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */
7337c478bd9Sstevel@tonic-gate
7347c478bd9Sstevel@tonic-gate int retval;
7357c478bd9Sstevel@tonic-gate
7367c478bd9Sstevel@tonic-gate mutex_enter(&memscrub_lock);
7377c478bd9Sstevel@tonic-gate
7387c478bd9Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG
7397c478bd9Sstevel@tonic-gate memscrub_printmemlist("memscrub_memlist before", memscrub_memlist);
7407c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
7417c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_add_span: address: 0x%llx"
7427c478bd9Sstevel@tonic-gate " size: 0x%llx\n", address, bytes);
7437c478bd9Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */
7447c478bd9Sstevel@tonic-gate
7457c478bd9Sstevel@tonic-gate retval = memscrub_add_span_gen(pfn, pages, &memscrub_memlist,
7467c478bd9Sstevel@tonic-gate &memscrub_phys_pages);
7477c478bd9Sstevel@tonic-gate
7487c478bd9Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG
7497c478bd9Sstevel@tonic-gate memscrub_printmemlist("memscrub_memlist after", memscrub_memlist);
7507c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
7517c478bd9Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */
7527c478bd9Sstevel@tonic-gate
7537c478bd9Sstevel@tonic-gate mutex_exit(&memscrub_lock);
7547c478bd9Sstevel@tonic-gate
7557c478bd9Sstevel@tonic-gate return (retval);
7567c478bd9Sstevel@tonic-gate }
7577c478bd9Sstevel@tonic-gate
7587c478bd9Sstevel@tonic-gate static int
memscrub_add_span_gen(pfn_t pfn,pgcnt_t pages,struct memlist ** list,uint_t * npgs)7597c478bd9Sstevel@tonic-gate memscrub_add_span_gen(
7607c478bd9Sstevel@tonic-gate pfn_t pfn,
7617c478bd9Sstevel@tonic-gate pgcnt_t pages,
7627c478bd9Sstevel@tonic-gate struct memlist **list,
7637c478bd9Sstevel@tonic-gate uint_t *npgs)
7647c478bd9Sstevel@tonic-gate {
7657c478bd9Sstevel@tonic-gate ms_paddr_t address = (ms_paddr_t)pfn << PAGESHIFT;
7667c478bd9Sstevel@tonic-gate uint64_t bytes = (uint64_t)pages << PAGESHIFT;
7677c478bd9Sstevel@tonic-gate struct memlist *dst;
7687c478bd9Sstevel@tonic-gate struct memlist *prev, *next;
7697c478bd9Sstevel@tonic-gate int retval = 0;
7707c478bd9Sstevel@tonic-gate
7717c478bd9Sstevel@tonic-gate /*
7727c478bd9Sstevel@tonic-gate * allocate a new struct memlist
7737c478bd9Sstevel@tonic-gate */
7747c478bd9Sstevel@tonic-gate
7757c478bd9Sstevel@tonic-gate dst = (struct memlist *)
7767c478bd9Sstevel@tonic-gate kmem_alloc(sizeof (struct memlist), KM_NOSLEEP);
7777c478bd9Sstevel@tonic-gate
7787c478bd9Sstevel@tonic-gate if (dst == NULL) {
7797c478bd9Sstevel@tonic-gate retval = -1;
7807c478bd9Sstevel@tonic-gate goto add_done;
7817c478bd9Sstevel@tonic-gate }
7827c478bd9Sstevel@tonic-gate
78356f33205SJonathan Adams dst->ml_address = address;
78456f33205SJonathan Adams dst->ml_size = bytes;
7857c478bd9Sstevel@tonic-gate
7867c478bd9Sstevel@tonic-gate /*
7877c478bd9Sstevel@tonic-gate * first insert
7887c478bd9Sstevel@tonic-gate */
7897c478bd9Sstevel@tonic-gate if (*list == NULL) {
79056f33205SJonathan Adams dst->ml_prev = NULL;
79156f33205SJonathan Adams dst->ml_next = NULL;
7927c478bd9Sstevel@tonic-gate *list = dst;
7937c478bd9Sstevel@tonic-gate
7947c478bd9Sstevel@tonic-gate goto add_done;
7957c478bd9Sstevel@tonic-gate }
7967c478bd9Sstevel@tonic-gate
7977c478bd9Sstevel@tonic-gate /*
7987c478bd9Sstevel@tonic-gate * insert into sorted list
7997c478bd9Sstevel@tonic-gate */
8007c478bd9Sstevel@tonic-gate for (prev = NULL, next = *list;
8017c478bd9Sstevel@tonic-gate next != NULL;
80256f33205SJonathan Adams prev = next, next = next->ml_next) {
80356f33205SJonathan Adams if (address > (next->ml_address + next->ml_size))
8047c478bd9Sstevel@tonic-gate continue;
8057c478bd9Sstevel@tonic-gate
8067c478bd9Sstevel@tonic-gate /*
8077c478bd9Sstevel@tonic-gate * else insert here
8087c478bd9Sstevel@tonic-gate */
8097c478bd9Sstevel@tonic-gate
8107c478bd9Sstevel@tonic-gate /*
8117c478bd9Sstevel@tonic-gate * prepend to next
8127c478bd9Sstevel@tonic-gate */
81356f33205SJonathan Adams if ((address + bytes) == next->ml_address) {
8147c478bd9Sstevel@tonic-gate kmem_free(dst, sizeof (struct memlist));
8157c478bd9Sstevel@tonic-gate
81656f33205SJonathan Adams next->ml_address = address;
81756f33205SJonathan Adams next->ml_size += bytes;
8187c478bd9Sstevel@tonic-gate
8197c478bd9Sstevel@tonic-gate goto add_done;
8207c478bd9Sstevel@tonic-gate }
8217c478bd9Sstevel@tonic-gate
8227c478bd9Sstevel@tonic-gate /*
8237c478bd9Sstevel@tonic-gate * append to next
8247c478bd9Sstevel@tonic-gate */
82556f33205SJonathan Adams if (address == (next->ml_address + next->ml_size)) {
8267c478bd9Sstevel@tonic-gate kmem_free(dst, sizeof (struct memlist));
8277c478bd9Sstevel@tonic-gate
82856f33205SJonathan Adams if (next->ml_next) {
8297c478bd9Sstevel@tonic-gate /*
83056f33205SJonathan Adams * don't overlap with next->ml_next
8317c478bd9Sstevel@tonic-gate */
83256f33205SJonathan Adams if ((address + bytes) >
83356f33205SJonathan Adams next->ml_next->ml_address) {
8347c478bd9Sstevel@tonic-gate retval = -1;
8357c478bd9Sstevel@tonic-gate goto add_done;
8367c478bd9Sstevel@tonic-gate }
8377c478bd9Sstevel@tonic-gate /*
83856f33205SJonathan Adams * concatenate next and next->ml_next
8397c478bd9Sstevel@tonic-gate */
84056f33205SJonathan Adams if ((address + bytes) ==
84156f33205SJonathan Adams next->ml_next->ml_address) {
84256f33205SJonathan Adams struct memlist *mlp = next->ml_next;
8437c478bd9Sstevel@tonic-gate
8447c478bd9Sstevel@tonic-gate if (next == *list)
84556f33205SJonathan Adams *list = next->ml_next;
8467c478bd9Sstevel@tonic-gate
84756f33205SJonathan Adams mlp->ml_address = next->ml_address;
84856f33205SJonathan Adams mlp->ml_size += next->ml_size;
84956f33205SJonathan Adams mlp->ml_size += bytes;
8507c478bd9Sstevel@tonic-gate
85156f33205SJonathan Adams if (next->ml_prev)
85256f33205SJonathan Adams next->ml_prev->ml_next = mlp;
85356f33205SJonathan Adams mlp->ml_prev = next->ml_prev;
8547c478bd9Sstevel@tonic-gate
8557c478bd9Sstevel@tonic-gate kmem_free(next,
85656f33205SJonathan Adams sizeof (struct memlist));
8577c478bd9Sstevel@tonic-gate goto add_done;
8587c478bd9Sstevel@tonic-gate }
8597c478bd9Sstevel@tonic-gate }
8607c478bd9Sstevel@tonic-gate
86156f33205SJonathan Adams next->ml_size += bytes;
8627c478bd9Sstevel@tonic-gate
8637c478bd9Sstevel@tonic-gate goto add_done;
8647c478bd9Sstevel@tonic-gate }
8657c478bd9Sstevel@tonic-gate
8667c478bd9Sstevel@tonic-gate /* don't overlap with next */
86756f33205SJonathan Adams if ((address + bytes) > next->ml_address) {
8687c478bd9Sstevel@tonic-gate retval = -1;
8697c478bd9Sstevel@tonic-gate kmem_free(dst, sizeof (struct memlist));
8707c478bd9Sstevel@tonic-gate goto add_done;
8717c478bd9Sstevel@tonic-gate }
8727c478bd9Sstevel@tonic-gate
8737c478bd9Sstevel@tonic-gate /*
8747c478bd9Sstevel@tonic-gate * insert before next
8757c478bd9Sstevel@tonic-gate */
87656f33205SJonathan Adams dst->ml_prev = prev;
87756f33205SJonathan Adams dst->ml_next = next;
87856f33205SJonathan Adams next->ml_prev = dst;
8797c478bd9Sstevel@tonic-gate if (prev == NULL) {
8807c478bd9Sstevel@tonic-gate *list = dst;
8817c478bd9Sstevel@tonic-gate } else {
88256f33205SJonathan Adams prev->ml_next = dst;
8837c478bd9Sstevel@tonic-gate }
8847c478bd9Sstevel@tonic-gate goto add_done;
8857c478bd9Sstevel@tonic-gate } /* end for */
8867c478bd9Sstevel@tonic-gate
8877c478bd9Sstevel@tonic-gate /*
8887c478bd9Sstevel@tonic-gate * end of list, prev is valid and next is NULL
8897c478bd9Sstevel@tonic-gate */
89056f33205SJonathan Adams prev->ml_next = dst;
89156f33205SJonathan Adams dst->ml_prev = prev;
89256f33205SJonathan Adams dst->ml_next = NULL;
8937c478bd9Sstevel@tonic-gate
8947c478bd9Sstevel@tonic-gate add_done:
8957c478bd9Sstevel@tonic-gate
8967c478bd9Sstevel@tonic-gate if (retval != -1)
8977c478bd9Sstevel@tonic-gate *npgs += pages;
8987c478bd9Sstevel@tonic-gate
8997c478bd9Sstevel@tonic-gate return (retval);
9007c478bd9Sstevel@tonic-gate }
9017c478bd9Sstevel@tonic-gate
9027c478bd9Sstevel@tonic-gate /*
9037c478bd9Sstevel@tonic-gate * delete a span from the memscrub list
9047c478bd9Sstevel@tonic-gate * subtract from memscrub_phys_pages
9057c478bd9Sstevel@tonic-gate */
9067c478bd9Sstevel@tonic-gate int
memscrub_delete_span(pfn_t pfn,pgcnt_t pages)9077c478bd9Sstevel@tonic-gate memscrub_delete_span(pfn_t pfn, pgcnt_t pages)
9087c478bd9Sstevel@tonic-gate {
9097c478bd9Sstevel@tonic-gate ms_paddr_t address = (ms_paddr_t)pfn << PAGESHIFT;
9107c478bd9Sstevel@tonic-gate uint64_t bytes = (uint64_t)pages << PAGESHIFT;
9117c478bd9Sstevel@tonic-gate struct memlist *dst, *next;
9127c478bd9Sstevel@tonic-gate int retval = 0;
9137c478bd9Sstevel@tonic-gate
9147c478bd9Sstevel@tonic-gate mutex_enter(&memscrub_lock);
9157c478bd9Sstevel@tonic-gate
9167c478bd9Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG
9177c478bd9Sstevel@tonic-gate memscrub_printmemlist("memscrub_memlist Before", memscrub_memlist);
9187c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
9197c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_delete_span: 0x%llx 0x%llx\n",
9207c478bd9Sstevel@tonic-gate address, bytes);
9217c478bd9Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */
9227c478bd9Sstevel@tonic-gate
9237c478bd9Sstevel@tonic-gate /*
9247c478bd9Sstevel@tonic-gate * find struct memlist containing page
9257c478bd9Sstevel@tonic-gate */
92656f33205SJonathan Adams for (next = memscrub_memlist; next != NULL; next = next->ml_next) {
92756f33205SJonathan Adams if ((address >= next->ml_address) &&
92856f33205SJonathan Adams (address < next->ml_address + next->ml_size))
9297c478bd9Sstevel@tonic-gate break;
9307c478bd9Sstevel@tonic-gate }
9317c478bd9Sstevel@tonic-gate
9327c478bd9Sstevel@tonic-gate /*
9337c478bd9Sstevel@tonic-gate * if start address not in list
9347c478bd9Sstevel@tonic-gate */
9357c478bd9Sstevel@tonic-gate if (next == NULL) {
9367c478bd9Sstevel@tonic-gate retval = -1;
9377c478bd9Sstevel@tonic-gate goto delete_done;
9387c478bd9Sstevel@tonic-gate }
9397c478bd9Sstevel@tonic-gate
9407c478bd9Sstevel@tonic-gate /*
9417c478bd9Sstevel@tonic-gate * error if size goes off end of this struct memlist
9427c478bd9Sstevel@tonic-gate */
94356f33205SJonathan Adams if (address + bytes > next->ml_address + next->ml_size) {
9447c478bd9Sstevel@tonic-gate retval = -1;
9457c478bd9Sstevel@tonic-gate goto delete_done;
9467c478bd9Sstevel@tonic-gate }
9477c478bd9Sstevel@tonic-gate
9487c478bd9Sstevel@tonic-gate /*
9497c478bd9Sstevel@tonic-gate * pages at beginning of struct memlist
9507c478bd9Sstevel@tonic-gate */
95156f33205SJonathan Adams if (address == next->ml_address) {
9527c478bd9Sstevel@tonic-gate /*
9537c478bd9Sstevel@tonic-gate * if start & size match, delete from list
9547c478bd9Sstevel@tonic-gate */
95556f33205SJonathan Adams if (bytes == next->ml_size) {
9567c478bd9Sstevel@tonic-gate if (next == memscrub_memlist)
95756f33205SJonathan Adams memscrub_memlist = next->ml_next;
95856f33205SJonathan Adams if (next->ml_prev != NULL)
95956f33205SJonathan Adams next->ml_prev->ml_next = next->ml_next;
96056f33205SJonathan Adams if (next->ml_next != NULL)
96156f33205SJonathan Adams next->ml_next->ml_prev = next->ml_prev;
9627c478bd9Sstevel@tonic-gate
9637c478bd9Sstevel@tonic-gate kmem_free(next, sizeof (struct memlist));
9647c478bd9Sstevel@tonic-gate } else {
9657c478bd9Sstevel@tonic-gate /*
9667c478bd9Sstevel@tonic-gate * increment start address by bytes
9677c478bd9Sstevel@tonic-gate */
96856f33205SJonathan Adams next->ml_address += bytes;
96956f33205SJonathan Adams next->ml_size -= bytes;
9707c478bd9Sstevel@tonic-gate }
9717c478bd9Sstevel@tonic-gate goto delete_done;
9727c478bd9Sstevel@tonic-gate }
9737c478bd9Sstevel@tonic-gate
9747c478bd9Sstevel@tonic-gate /*
9757c478bd9Sstevel@tonic-gate * pages at end of struct memlist
9767c478bd9Sstevel@tonic-gate */
97756f33205SJonathan Adams if (address + bytes == next->ml_address + next->ml_size) {
9787c478bd9Sstevel@tonic-gate /*
9797c478bd9Sstevel@tonic-gate * decrement size by bytes
9807c478bd9Sstevel@tonic-gate */
98156f33205SJonathan Adams next->ml_size -= bytes;
9827c478bd9Sstevel@tonic-gate goto delete_done;
9837c478bd9Sstevel@tonic-gate }
9847c478bd9Sstevel@tonic-gate
9857c478bd9Sstevel@tonic-gate /*
9867c478bd9Sstevel@tonic-gate * delete a span in the middle of the struct memlist
9877c478bd9Sstevel@tonic-gate */
9887c478bd9Sstevel@tonic-gate {
9897c478bd9Sstevel@tonic-gate /*
9907c478bd9Sstevel@tonic-gate * create a new struct memlist
9917c478bd9Sstevel@tonic-gate */
9927c478bd9Sstevel@tonic-gate dst = (struct memlist *)
9937c478bd9Sstevel@tonic-gate kmem_alloc(sizeof (struct memlist), KM_NOSLEEP);
9947c478bd9Sstevel@tonic-gate
9957c478bd9Sstevel@tonic-gate if (dst == NULL) {
9967c478bd9Sstevel@tonic-gate retval = -1;
9977c478bd9Sstevel@tonic-gate goto delete_done;
9987c478bd9Sstevel@tonic-gate }
9997c478bd9Sstevel@tonic-gate
10007c478bd9Sstevel@tonic-gate /*
10017c478bd9Sstevel@tonic-gate * existing struct memlist gets address
10027c478bd9Sstevel@tonic-gate * and size up to pfn
10037c478bd9Sstevel@tonic-gate */
100456f33205SJonathan Adams dst->ml_address = address + bytes;
100556f33205SJonathan Adams dst->ml_size =
100656f33205SJonathan Adams (next->ml_address + next->ml_size) - dst->ml_address;
100756f33205SJonathan Adams next->ml_size = address - next->ml_address;
10087c478bd9Sstevel@tonic-gate
10097c478bd9Sstevel@tonic-gate /*
10107c478bd9Sstevel@tonic-gate * new struct memlist gets address starting
10117c478bd9Sstevel@tonic-gate * after pfn, until end
10127c478bd9Sstevel@tonic-gate */
10137c478bd9Sstevel@tonic-gate
10147c478bd9Sstevel@tonic-gate /*
10157c478bd9Sstevel@tonic-gate * link in new memlist after old
10167c478bd9Sstevel@tonic-gate */
101756f33205SJonathan Adams dst->ml_next = next->ml_next;
101856f33205SJonathan Adams dst->ml_prev = next;
10197c478bd9Sstevel@tonic-gate
102056f33205SJonathan Adams if (next->ml_next != NULL)
102156f33205SJonathan Adams next->ml_next->ml_prev = dst;
102256f33205SJonathan Adams next->ml_next = dst;
10237c478bd9Sstevel@tonic-gate }
10247c478bd9Sstevel@tonic-gate
10257c478bd9Sstevel@tonic-gate delete_done:
10267c478bd9Sstevel@tonic-gate if (retval != -1) {
10277c478bd9Sstevel@tonic-gate memscrub_phys_pages -= pages;
10287c478bd9Sstevel@tonic-gate if (memscrub_phys_pages == 0)
10297c478bd9Sstevel@tonic-gate disable_memscrub = 1;
10307c478bd9Sstevel@tonic-gate }
10317c478bd9Sstevel@tonic-gate
10327c478bd9Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG
10337c478bd9Sstevel@tonic-gate memscrub_printmemlist("memscrub_memlist After", memscrub_memlist);
10347c478bd9Sstevel@tonic-gate cmn_err(CE_CONT, "memscrub_phys_pages: 0x%x\n", memscrub_phys_pages);
10357c478bd9Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */
10367c478bd9Sstevel@tonic-gate
10377c478bd9Sstevel@tonic-gate mutex_exit(&memscrub_lock);
10387c478bd9Sstevel@tonic-gate return (retval);
10397c478bd9Sstevel@tonic-gate }
10407c478bd9Sstevel@tonic-gate
10417c478bd9Sstevel@tonic-gate static void
memscrub_scan(uint_t blks,ms_paddr_t src)10427c478bd9Sstevel@tonic-gate memscrub_scan(uint_t blks, ms_paddr_t src)
10437c478bd9Sstevel@tonic-gate {
10447c478bd9Sstevel@tonic-gate uint_t psz, bpp, pgsread;
10457c478bd9Sstevel@tonic-gate pfn_t pfn;
10467c478bd9Sstevel@tonic-gate ms_paddr_t pa;
10477c478bd9Sstevel@tonic-gate caddr_t va;
10487c478bd9Sstevel@tonic-gate on_trap_data_t otd;
104961ef38f7Svb int scan_mmu_pagesize = 0;
105061ef38f7Svb int retired_pages = 0;
10517c478bd9Sstevel@tonic-gate
10527c478bd9Sstevel@tonic-gate extern void memscrub_read(caddr_t src, uint_t blks);
10537c478bd9Sstevel@tonic-gate
10547c478bd9Sstevel@tonic-gate ASSERT(mutex_owned(&memscrub_lock));
10557c478bd9Sstevel@tonic-gate
10567c478bd9Sstevel@tonic-gate pgsread = 0;
10577c478bd9Sstevel@tonic-gate pa = src;
10587c478bd9Sstevel@tonic-gate
105961ef38f7Svb if (memscrub_page_retire_span_list != NULL) {
106061ef38f7Svb if (memscrub_page_retire_span_search(src)) {
106161ef38f7Svb /* retired pages in current span */
106261ef38f7Svb scan_mmu_pagesize = 1;
106361ef38f7Svb }
106461ef38f7Svb }
106561ef38f7Svb
106661ef38f7Svb #ifdef MEMSCRUB_DEBUG
106761ef38f7Svb cmn_err(CE_NOTE, "scan_mmu_pagesize = %d\n" scan_mmu_pagesize);
106861ef38f7Svb #endif /* MEMSCRUB_DEBUG */
106961ef38f7Svb
10707c478bd9Sstevel@tonic-gate while (blks != 0) {
10717c478bd9Sstevel@tonic-gate /* Ensure the PA is properly aligned */
10727c478bd9Sstevel@tonic-gate if (((pa & MMU_PAGEMASK4M) == pa) &&
107356f33205SJonathan Adams (blks >= MEMSCRUB_BPP4M)) {
10747c478bd9Sstevel@tonic-gate psz = MMU_PAGESIZE4M;
10757c478bd9Sstevel@tonic-gate bpp = MEMSCRUB_BPP4M;
10767c478bd9Sstevel@tonic-gate } else if (((pa & MMU_PAGEMASK512K) == pa) &&
107756f33205SJonathan Adams (blks >= MEMSCRUB_BPP512K)) {
10787c478bd9Sstevel@tonic-gate psz = MMU_PAGESIZE512K;
10797c478bd9Sstevel@tonic-gate bpp = MEMSCRUB_BPP512K;
10807c478bd9Sstevel@tonic-gate } else if (((pa & MMU_PAGEMASK64K) == pa) &&
108156f33205SJonathan Adams (blks >= MEMSCRUB_BPP64K)) {
10827c478bd9Sstevel@tonic-gate psz = MMU_PAGESIZE64K;
10837c478bd9Sstevel@tonic-gate bpp = MEMSCRUB_BPP64K;
10847c478bd9Sstevel@tonic-gate } else if ((pa & MMU_PAGEMASK) == pa) {
10857c478bd9Sstevel@tonic-gate psz = MMU_PAGESIZE;
10867c478bd9Sstevel@tonic-gate bpp = MEMSCRUB_BPP;
10877c478bd9Sstevel@tonic-gate } else {
10887c478bd9Sstevel@tonic-gate if (memscrub_verbose) {
10897c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "Memory scrubber ignoring "
10907c478bd9Sstevel@tonic-gate "non-page aligned block starting at 0x%"
10917c478bd9Sstevel@tonic-gate PRIx64, src);
10927c478bd9Sstevel@tonic-gate }
10937c478bd9Sstevel@tonic-gate return;
10947c478bd9Sstevel@tonic-gate }
10957c478bd9Sstevel@tonic-gate if (blks < bpp) bpp = blks;
10967c478bd9Sstevel@tonic-gate
10977c478bd9Sstevel@tonic-gate #ifdef MEMSCRUB_DEBUG
10987c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "Going to run psz=%x, "
10997c478bd9Sstevel@tonic-gate "bpp=%x pa=%llx\n", psz, bpp, pa);
11007c478bd9Sstevel@tonic-gate #endif /* MEMSCRUB_DEBUG */
11017c478bd9Sstevel@tonic-gate
11027c478bd9Sstevel@tonic-gate /*
11037c478bd9Sstevel@tonic-gate * MEMSCRUBBASE is a 4MB aligned page in the
11047c478bd9Sstevel@tonic-gate * kernel so that we can quickly map the PA
11057c478bd9Sstevel@tonic-gate * to a VA for the block loads performed in
11067c478bd9Sstevel@tonic-gate * memscrub_read.
11077c478bd9Sstevel@tonic-gate */
11087c478bd9Sstevel@tonic-gate pfn = mmu_btop(pa);
11097c478bd9Sstevel@tonic-gate va = (caddr_t)MEMSCRUBBASE;
11107c478bd9Sstevel@tonic-gate hat_devload(kas.a_hat, va, psz, pfn, PROT_READ,
111156f33205SJonathan Adams HAT_LOAD_NOCONSIST | HAT_LOAD_LOCK);
11127c478bd9Sstevel@tonic-gate
11137c478bd9Sstevel@tonic-gate /*
11147c478bd9Sstevel@tonic-gate * Can't allow the memscrubber to migrate across CPUs as
11157c478bd9Sstevel@tonic-gate * we need to know whether CEEN is enabled for the current
11167c478bd9Sstevel@tonic-gate * CPU to enable us to scrub the memory. Don't use
11177c478bd9Sstevel@tonic-gate * kpreempt_disable as the time we take to scan a span (even
11187c478bd9Sstevel@tonic-gate * without cpu_check_ce having to manually cpu_check_block)
11197c478bd9Sstevel@tonic-gate * is too long to hold a higher priority thread (eg, RT)
11207c478bd9Sstevel@tonic-gate * off cpu.
11217c478bd9Sstevel@tonic-gate */
11227c478bd9Sstevel@tonic-gate thread_affinity_set(curthread, CPU_CURRENT);
11237c478bd9Sstevel@tonic-gate
11247c478bd9Sstevel@tonic-gate /*
11257c478bd9Sstevel@tonic-gate * Protect read scrub from async faults. For now, we simply
11267c478bd9Sstevel@tonic-gate * maintain a count of such faults caught.
11277c478bd9Sstevel@tonic-gate */
11287c478bd9Sstevel@tonic-gate
1129df5afd94SVijay S Balakrishna if (!on_trap(&otd, OT_DATA_EC) && !scan_mmu_pagesize) {
11307c478bd9Sstevel@tonic-gate memscrub_read(va, bpp);
11317c478bd9Sstevel@tonic-gate /*
11327c478bd9Sstevel@tonic-gate * Check if CEs require logging
11337c478bd9Sstevel@tonic-gate */
11347c478bd9Sstevel@tonic-gate cpu_check_ce(SCRUBBER_CEEN_CHECK,
11357c478bd9Sstevel@tonic-gate (uint64_t)pa, va, psz);
1136a08365b4Srjnoe no_trap();
11377c478bd9Sstevel@tonic-gate thread_affinity_clear(curthread);
11387c478bd9Sstevel@tonic-gate } else {
11397c478bd9Sstevel@tonic-gate no_trap();
11407c478bd9Sstevel@tonic-gate thread_affinity_clear(curthread);
11417c478bd9Sstevel@tonic-gate
11427c478bd9Sstevel@tonic-gate /*
11437c478bd9Sstevel@tonic-gate * Got an async error..
11447c478bd9Sstevel@tonic-gate * Try rescanning it at MMU_PAGESIZE
11457c478bd9Sstevel@tonic-gate * granularity if we were trying to
11467c478bd9Sstevel@tonic-gate * read at a larger page size.
11477c478bd9Sstevel@tonic-gate * This is to ensure we continue to
11487c478bd9Sstevel@tonic-gate * scan the rest of the span.
114961ef38f7Svb * OR scanning MMU_PAGESIZE granularity to avoid
115061ef38f7Svb * reading retired pages memory when scan_mmu_pagesize
115161ef38f7Svb * is set.
11527c478bd9Sstevel@tonic-gate */
115361ef38f7Svb if (psz > MMU_PAGESIZE || scan_mmu_pagesize) {
11541a3c9a5aSVijay S Balakrishna caddr_t vaddr = va;
11551a3c9a5aSVijay S Balakrishna ms_paddr_t paddr = pa;
11561a3c9a5aSVijay S Balakrishna int tmp = 0;
11571a3c9a5aSVijay S Balakrishna for (; tmp < bpp; tmp += MEMSCRUB_BPP) {
11581a3c9a5aSVijay S Balakrishna /* Don't scrub retired pages */
11591a3c9a5aSVijay S Balakrishna if (page_retire_check(paddr, NULL)
11601a3c9a5aSVijay S Balakrishna == 0) {
11611a3c9a5aSVijay S Balakrishna vaddr += MMU_PAGESIZE;
11621a3c9a5aSVijay S Balakrishna paddr += MMU_PAGESIZE;
11631a3c9a5aSVijay S Balakrishna retired_pages++;
11641a3c9a5aSVijay S Balakrishna continue;
11651a3c9a5aSVijay S Balakrishna }
11661a3c9a5aSVijay S Balakrishna thread_affinity_set(curthread,
11671a3c9a5aSVijay S Balakrishna CPU_CURRENT);
11681a3c9a5aSVijay S Balakrishna if (!on_trap(&otd, OT_DATA_EC)) {
11691a3c9a5aSVijay S Balakrishna memscrub_read(vaddr,
11701a3c9a5aSVijay S Balakrishna MEMSCRUB_BPP);
11711a3c9a5aSVijay S Balakrishna cpu_check_ce(
11721a3c9a5aSVijay S Balakrishna SCRUBBER_CEEN_CHECK,
11731a3c9a5aSVijay S Balakrishna (uint64_t)paddr, vaddr,
11741a3c9a5aSVijay S Balakrishna MMU_PAGESIZE);
11751a3c9a5aSVijay S Balakrishna no_trap();
11761a3c9a5aSVijay S Balakrishna } else {
11771a3c9a5aSVijay S Balakrishna no_trap();
11781a3c9a5aSVijay S Balakrishna MEMSCRUB_STAT_INC(errors_found);
11791a3c9a5aSVijay S Balakrishna }
11801a3c9a5aSVijay S Balakrishna thread_affinity_clear(curthread);
118161ef38f7Svb vaddr += MMU_PAGESIZE;
118261ef38f7Svb paddr += MMU_PAGESIZE;
1183a08365b4Srjnoe }
11847c478bd9Sstevel@tonic-gate }
11857c478bd9Sstevel@tonic-gate }
11867c478bd9Sstevel@tonic-gate hat_unload(kas.a_hat, va, psz, HAT_UNLOAD_UNLOCK);
11877c478bd9Sstevel@tonic-gate
11887c478bd9Sstevel@tonic-gate blks -= bpp;
11897c478bd9Sstevel@tonic-gate pa += psz;
11907c478bd9Sstevel@tonic-gate pgsread++;
11917c478bd9Sstevel@tonic-gate }
119261ef38f7Svb
119361ef38f7Svb /*
119461ef38f7Svb * If just finished scrubbing MMU_PAGESIZE at a time, but no retired
119561ef38f7Svb * pages found so delete span from global list.
119661ef38f7Svb */
119761ef38f7Svb if (scan_mmu_pagesize && retired_pages == 0)
119861ef38f7Svb memscrub_page_retire_span_delete(src);
119961ef38f7Svb
120061ef38f7Svb /*
120161ef38f7Svb * Encountered CE/UE on a retired page during memscrub read of current
120261ef38f7Svb * span. Adding span to global list to enable avoid reading further.
120361ef38f7Svb */
120461ef38f7Svb if (add_to_page_retire_list) {
120561ef38f7Svb if (!memscrub_page_retire_span_search(src))
120661ef38f7Svb memscrub_page_retire_span_add(src);
120761ef38f7Svb add_to_page_retire_list = 0;
120861ef38f7Svb }
120961ef38f7Svb
12107c478bd9Sstevel@tonic-gate if (memscrub_verbose) {
12117c478bd9Sstevel@tonic-gate cmn_err(CE_NOTE, "Memory scrubber read 0x%x pages starting "
12127c478bd9Sstevel@tonic-gate "at 0x%" PRIx64, pgsread, src);
12137c478bd9Sstevel@tonic-gate }
12147c478bd9Sstevel@tonic-gate }
12157c478bd9Sstevel@tonic-gate
121661ef38f7Svb /*
121761ef38f7Svb * Called by cpu_async_log_err() when memscrub read causes
121861ef38f7Svb * CE/UE on a retired page.
121961ef38f7Svb */
122061ef38f7Svb void
memscrub_induced_error(void)122161ef38f7Svb memscrub_induced_error(void)
122261ef38f7Svb {
122361ef38f7Svb add_to_page_retire_list = 1;
122461ef38f7Svb }
122561ef38f7Svb
12261a3c9a5aSVijay S Balakrishna /*
12271a3c9a5aSVijay S Balakrishna * Called by page_retire() when toxic pages cannot be retired
12281a3c9a5aSVijay S Balakrishna * immediately and are scheduled for retire. Memscrubber stops
12291a3c9a5aSVijay S Balakrishna * scrubbing them to avoid further CE/UEs.
12301a3c9a5aSVijay S Balakrishna */
12311a3c9a5aSVijay S Balakrishna void
memscrub_notify(ms_paddr_t pa)12321a3c9a5aSVijay S Balakrishna memscrub_notify(ms_paddr_t pa)
12331a3c9a5aSVijay S Balakrishna {
12341a3c9a5aSVijay S Balakrishna mutex_enter(&memscrub_lock);
12351a3c9a5aSVijay S Balakrishna if (!memscrub_page_retire_span_search(pa))
12361a3c9a5aSVijay S Balakrishna memscrub_page_retire_span_add(pa);
12371a3c9a5aSVijay S Balakrishna mutex_exit(&memscrub_lock);
12381a3c9a5aSVijay S Balakrishna }
123961ef38f7Svb
124061ef38f7Svb /*
12411a3c9a5aSVijay S Balakrishna * Called by memscrub_scan() and memscrub_notify().
124261ef38f7Svb * pa: physical address of span with CE/UE, add to global list.
124361ef38f7Svb */
124461ef38f7Svb static void
memscrub_page_retire_span_add(ms_paddr_t pa)124561ef38f7Svb memscrub_page_retire_span_add(ms_paddr_t pa)
124661ef38f7Svb {
124761ef38f7Svb memscrub_page_retire_span_t *new_span;
124861ef38f7Svb
124961ef38f7Svb new_span = (memscrub_page_retire_span_t *)
125061ef38f7Svb kmem_zalloc(sizeof (memscrub_page_retire_span_t), KM_NOSLEEP);
125161ef38f7Svb
125261ef38f7Svb if (new_span == NULL) {
125361ef38f7Svb #ifdef MEMSCRUB_DEBUG
125461ef38f7Svb cmn_err(CE_NOTE, "failed to allocate new span - span with"
125561ef38f7Svb " retired page/s not tracked.\n");
125661ef38f7Svb #endif /* MEMSCRUB_DEBUG */
125761ef38f7Svb return;
125861ef38f7Svb }
125961ef38f7Svb
126061ef38f7Svb new_span->address = pa;
126161ef38f7Svb new_span->next = memscrub_page_retire_span_list;
126261ef38f7Svb memscrub_page_retire_span_list = new_span;
126361ef38f7Svb }
126461ef38f7Svb
126561ef38f7Svb /*
126661ef38f7Svb * Called by memscrub_scan().
126761ef38f7Svb * pa: physical address of span to be removed from global list.
126861ef38f7Svb */
126961ef38f7Svb static void
memscrub_page_retire_span_delete(ms_paddr_t pa)127061ef38f7Svb memscrub_page_retire_span_delete(ms_paddr_t pa)
127161ef38f7Svb {
127261ef38f7Svb memscrub_page_retire_span_t *prev_span, *next_span;
127361ef38f7Svb
127461ef38f7Svb prev_span = memscrub_page_retire_span_list;
127561ef38f7Svb next_span = memscrub_page_retire_span_list->next;
127661ef38f7Svb
127761ef38f7Svb if (pa == prev_span->address) {
127861ef38f7Svb memscrub_page_retire_span_list = next_span;
127961ef38f7Svb kmem_free(prev_span, sizeof (memscrub_page_retire_span_t));
128061ef38f7Svb return;
128161ef38f7Svb }
128261ef38f7Svb
128361ef38f7Svb while (next_span) {
128461ef38f7Svb if (pa == next_span->address) {
128561ef38f7Svb prev_span->next = next_span->next;
128661ef38f7Svb kmem_free(next_span,
128761ef38f7Svb sizeof (memscrub_page_retire_span_t));
128861ef38f7Svb return;
128961ef38f7Svb }
129061ef38f7Svb prev_span = next_span;
129161ef38f7Svb next_span = next_span->next;
129261ef38f7Svb }
129361ef38f7Svb }
129461ef38f7Svb
129561ef38f7Svb /*
12961a3c9a5aSVijay S Balakrishna * Called by memscrub_scan() and memscrub_notify().
129761ef38f7Svb * pa: physical address of span to be searched in global list.
129861ef38f7Svb */
129961ef38f7Svb static int
memscrub_page_retire_span_search(ms_paddr_t pa)130061ef38f7Svb memscrub_page_retire_span_search(ms_paddr_t pa)
130161ef38f7Svb {
130261ef38f7Svb memscrub_page_retire_span_t *next_span = memscrub_page_retire_span_list;
130361ef38f7Svb
130461ef38f7Svb while (next_span) {
130561ef38f7Svb if (pa == next_span->address)
130661ef38f7Svb return (1);
130761ef38f7Svb next_span = next_span->next;
130861ef38f7Svb }
130961ef38f7Svb return (0);
131061ef38f7Svb }
131161ef38f7Svb
131261ef38f7Svb /*
131361ef38f7Svb * Called from new_memscrub() as a result of memory delete.
131461ef38f7Svb * Using page_numtopp_nolock() to determine if we have valid PA.
131561ef38f7Svb */
131661ef38f7Svb static void
memscrub_page_retire_span_list_update(void)131761ef38f7Svb memscrub_page_retire_span_list_update(void)
131861ef38f7Svb {
131961ef38f7Svb memscrub_page_retire_span_t *prev, *cur, *next;
132061ef38f7Svb
132161ef38f7Svb if (memscrub_page_retire_span_list == NULL)
132261ef38f7Svb return;
132361ef38f7Svb
132461ef38f7Svb prev = cur = memscrub_page_retire_span_list;
132561ef38f7Svb next = cur->next;
132661ef38f7Svb
132761ef38f7Svb while (cur) {
132861ef38f7Svb if (page_numtopp_nolock(mmu_btop(cur->address)) == NULL) {
132961ef38f7Svb if (cur == memscrub_page_retire_span_list) {
133061ef38f7Svb memscrub_page_retire_span_list = next;
133161ef38f7Svb kmem_free(cur,
133261ef38f7Svb sizeof (memscrub_page_retire_span_t));
133361ef38f7Svb prev = cur = memscrub_page_retire_span_list;
133461ef38f7Svb } else {
133561ef38f7Svb prev->next = cur->next;
133661ef38f7Svb kmem_free(cur,
133761ef38f7Svb sizeof (memscrub_page_retire_span_t));
133861ef38f7Svb cur = next;
133961ef38f7Svb }
134061ef38f7Svb } else {
134161ef38f7Svb prev = cur;
134261ef38f7Svb cur = next;
134361ef38f7Svb }
134461ef38f7Svb if (cur != NULL)
134561ef38f7Svb next = cur->next;
134661ef38f7Svb }
134761ef38f7Svb }
134861ef38f7Svb
13497c478bd9Sstevel@tonic-gate /*
13507c478bd9Sstevel@tonic-gate * The memory add/delete callback mechanism does not pass in the
13517c478bd9Sstevel@tonic-gate * page ranges. The phys_install list has been updated though, so
13527c478bd9Sstevel@tonic-gate * create a new scrub list from it.
13537c478bd9Sstevel@tonic-gate */
13547c478bd9Sstevel@tonic-gate
13557c478bd9Sstevel@tonic-gate static int
new_memscrub(int update_page_retire_list)135661ef38f7Svb new_memscrub(int update_page_retire_list)
13577c478bd9Sstevel@tonic-gate {
13587c478bd9Sstevel@tonic-gate struct memlist *src, *list, *old_list;
13597c478bd9Sstevel@tonic-gate uint_t npgs;
13607c478bd9Sstevel@tonic-gate
13617c478bd9Sstevel@tonic-gate /*
13627c478bd9Sstevel@tonic-gate * copy phys_install to memscrub_memlist
13637c478bd9Sstevel@tonic-gate */
13647c478bd9Sstevel@tonic-gate list = NULL;
13657c478bd9Sstevel@tonic-gate npgs = 0;
13667c478bd9Sstevel@tonic-gate memlist_read_lock();
136756f33205SJonathan Adams for (src = phys_install; src; src = src->ml_next) {
136856f33205SJonathan Adams if (memscrub_add_span_gen((pfn_t)(src->ml_address >> PAGESHIFT),
136956f33205SJonathan Adams (pgcnt_t)(src->ml_size >> PAGESHIFT), &list, &npgs)) {
13707c478bd9Sstevel@tonic-gate memlist_read_unlock();
13717c478bd9Sstevel@tonic-gate while (list) {
13727c478bd9Sstevel@tonic-gate struct memlist *el;
13737c478bd9Sstevel@tonic-gate
13747c478bd9Sstevel@tonic-gate el = list;
137556f33205SJonathan Adams list = list->ml_next;
13767c478bd9Sstevel@tonic-gate kmem_free(el, sizeof (struct memlist));
13777c478bd9Sstevel@tonic-gate }
13787c478bd9Sstevel@tonic-gate return (-1);
13797c478bd9Sstevel@tonic-gate }
13807c478bd9Sstevel@tonic-gate }
13817c478bd9Sstevel@tonic-gate memlist_read_unlock();
13827c478bd9Sstevel@tonic-gate
13837c478bd9Sstevel@tonic-gate mutex_enter(&memscrub_lock);
13847c478bd9Sstevel@tonic-gate memscrub_phys_pages = npgs;
13857c478bd9Sstevel@tonic-gate old_list = memscrub_memlist;
13867c478bd9Sstevel@tonic-gate memscrub_memlist = list;
138761ef38f7Svb
138861ef38f7Svb if (update_page_retire_list)
138961ef38f7Svb memscrub_page_retire_span_list_update();
139061ef38f7Svb
13917c478bd9Sstevel@tonic-gate mutex_exit(&memscrub_lock);
13927c478bd9Sstevel@tonic-gate
13937c478bd9Sstevel@tonic-gate while (old_list) {
13947c478bd9Sstevel@tonic-gate struct memlist *el;
13957c478bd9Sstevel@tonic-gate
13967c478bd9Sstevel@tonic-gate el = old_list;
139756f33205SJonathan Adams old_list = old_list->ml_next;
13987c478bd9Sstevel@tonic-gate kmem_free(el, sizeof (struct memlist));
13997c478bd9Sstevel@tonic-gate }
140061ef38f7Svb
14017c478bd9Sstevel@tonic-gate return (0);
14027c478bd9Sstevel@tonic-gate }
14037c478bd9Sstevel@tonic-gate
14047c478bd9Sstevel@tonic-gate /*ARGSUSED*/
14057c478bd9Sstevel@tonic-gate static void
memscrub_mem_config_post_add(void * arg,pgcnt_t delta_pages)14067c478bd9Sstevel@tonic-gate memscrub_mem_config_post_add(
14077c478bd9Sstevel@tonic-gate void *arg,
14087c478bd9Sstevel@tonic-gate pgcnt_t delta_pages)
14097c478bd9Sstevel@tonic-gate {
14107c478bd9Sstevel@tonic-gate /*
14117c478bd9Sstevel@tonic-gate * We increment pause_memscrub before entering new_memscrub(). This
14127c478bd9Sstevel@tonic-gate * will force the memscrubber to sleep, allowing the DR callback
14137c478bd9Sstevel@tonic-gate * thread to acquire memscrub_lock in new_memscrub(). The use of
14147c478bd9Sstevel@tonic-gate * atomic_add_32() allows concurrent memory DR operations to use the
14157c478bd9Sstevel@tonic-gate * callbacks safely.
14167c478bd9Sstevel@tonic-gate */
1417*1a5e258fSJosef 'Jeff' Sipek atomic_inc_32(&pause_memscrub);
14187c478bd9Sstevel@tonic-gate ASSERT(pause_memscrub != 0);
14197c478bd9Sstevel@tonic-gate
14207c478bd9Sstevel@tonic-gate /*
14217c478bd9Sstevel@tonic-gate * "Don't care" if we are not scrubbing new memory.
14227c478bd9Sstevel@tonic-gate */
142361ef38f7Svb (void) new_memscrub(0); /* retain page retire list */
14247c478bd9Sstevel@tonic-gate
14257c478bd9Sstevel@tonic-gate /* Restore the pause setting. */
1426*1a5e258fSJosef 'Jeff' Sipek atomic_dec_32(&pause_memscrub);
14277c478bd9Sstevel@tonic-gate }
14287c478bd9Sstevel@tonic-gate
14297c478bd9Sstevel@tonic-gate /*ARGSUSED*/
14307c478bd9Sstevel@tonic-gate static int
memscrub_mem_config_pre_del(void * arg,pgcnt_t delta_pages)14317c478bd9Sstevel@tonic-gate memscrub_mem_config_pre_del(
14327c478bd9Sstevel@tonic-gate void *arg,
14337c478bd9Sstevel@tonic-gate pgcnt_t delta_pages)
14347c478bd9Sstevel@tonic-gate {
14357c478bd9Sstevel@tonic-gate /* Nothing to do. */
14367c478bd9Sstevel@tonic-gate return (0);
14377c478bd9Sstevel@tonic-gate }
14387c478bd9Sstevel@tonic-gate
14397c478bd9Sstevel@tonic-gate /*ARGSUSED*/
14407c478bd9Sstevel@tonic-gate static void
memscrub_mem_config_post_del(void * arg,pgcnt_t delta_pages,int cancelled)14417c478bd9Sstevel@tonic-gate memscrub_mem_config_post_del(
14427c478bd9Sstevel@tonic-gate void *arg,
14437c478bd9Sstevel@tonic-gate pgcnt_t delta_pages,
14447c478bd9Sstevel@tonic-gate int cancelled)
14457c478bd9Sstevel@tonic-gate {
14467c478bd9Sstevel@tonic-gate /*
14477c478bd9Sstevel@tonic-gate * We increment pause_memscrub before entering new_memscrub(). This
14487c478bd9Sstevel@tonic-gate * will force the memscrubber to sleep, allowing the DR callback
14497c478bd9Sstevel@tonic-gate * thread to acquire memscrub_lock in new_memscrub(). The use of
14507c478bd9Sstevel@tonic-gate * atomic_add_32() allows concurrent memory DR operations to use the
14517c478bd9Sstevel@tonic-gate * callbacks safely.
14527c478bd9Sstevel@tonic-gate */
1453*1a5e258fSJosef 'Jeff' Sipek atomic_inc_32(&pause_memscrub);
14547c478bd9Sstevel@tonic-gate ASSERT(pause_memscrub != 0);
14557c478bd9Sstevel@tonic-gate
14567c478bd9Sstevel@tonic-gate /*
14577c478bd9Sstevel@tonic-gate * Must stop scrubbing deleted memory as it may be disconnected.
14587c478bd9Sstevel@tonic-gate */
145961ef38f7Svb if (new_memscrub(1)) { /* update page retire list */
14607c478bd9Sstevel@tonic-gate disable_memscrub = 1;
14617c478bd9Sstevel@tonic-gate }
14627c478bd9Sstevel@tonic-gate
14637c478bd9Sstevel@tonic-gate /* Restore the pause setting. */
1464*1a5e258fSJosef 'Jeff' Sipek atomic_dec_32(&pause_memscrub);
14657c478bd9Sstevel@tonic-gate }
14667c478bd9Sstevel@tonic-gate
14677c478bd9Sstevel@tonic-gate static kphysm_setup_vector_t memscrub_mem_config_vec = {
14687c478bd9Sstevel@tonic-gate KPHYSM_SETUP_VECTOR_VERSION,
14697c478bd9Sstevel@tonic-gate memscrub_mem_config_post_add,
14707c478bd9Sstevel@tonic-gate memscrub_mem_config_pre_del,
14717c478bd9Sstevel@tonic-gate memscrub_mem_config_post_del,
14727c478bd9Sstevel@tonic-gate };
14737c478bd9Sstevel@tonic-gate
14747c478bd9Sstevel@tonic-gate static void
memscrub_init_mem_config()14757c478bd9Sstevel@tonic-gate memscrub_init_mem_config()
14767c478bd9Sstevel@tonic-gate {
14777c478bd9Sstevel@tonic-gate int ret;
14787c478bd9Sstevel@tonic-gate
14797c478bd9Sstevel@tonic-gate ret = kphysm_setup_func_register(&memscrub_mem_config_vec,
14807c478bd9Sstevel@tonic-gate (void *)NULL);
14817c478bd9Sstevel@tonic-gate ASSERT(ret == 0);
14827c478bd9Sstevel@tonic-gate }
14837c478bd9Sstevel@tonic-gate
14847c478bd9Sstevel@tonic-gate static void
memscrub_uninit_mem_config()14857c478bd9Sstevel@tonic-gate memscrub_uninit_mem_config()
14867c478bd9Sstevel@tonic-gate {
14877c478bd9Sstevel@tonic-gate /* This call is OK if the register call was not done. */
14887c478bd9Sstevel@tonic-gate kphysm_setup_func_unregister(&memscrub_mem_config_vec, (void *)NULL);
14897c478bd9Sstevel@tonic-gate }
1490