xref: /illumos-gate/usr/src/uts/common/vm/page_retire.c (revision 04909c8c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  * Copyright (c) 2016 by Delphix. All rights reserved.
25  * Copyright 2018 Joyent, Inc.
26  */
27 
28 /*
29  * Page Retire - Big Theory Statement.
30  *
31  * This file handles removing sections of faulty memory from use when the
32  * user land FMA Diagnosis Engine requests that a page be removed or when
33  * a CE or UE is detected by the hardware.
34  *
35  * In the bad old days, the kernel side of Page Retire did a lot of the work
36  * on its own. Now, with the DE keeping track of errors, the kernel side is
37  * rather simple minded on most platforms.
38  *
39  * Errors are all reflected to the DE, and after digesting the error and
40  * looking at all previously reported errors, the DE decides what should
41  * be done about the current error. If the DE wants a particular page to
42  * be retired, then the kernel page retire code is invoked via an ioctl.
43  * On non-FMA platforms, the ue_drain and ce_drain paths ends up calling
44  * page retire to handle the error. Since page retire is just a simple
45  * mechanism it doesn't need to differentiate between the different callers.
46  *
47  * The p_toxic field in the page_t is used to indicate which errors have
48  * occurred and what action has been taken on a given page. Because errors are
49  * reported without regard to the locked state of a page, no locks are used
50  * to SET the error bits in p_toxic. However, in order to clear the error
51  * bits, the page_t must be held exclusively locked.
52  *
53  * When page_retire() is called, it must be able to acquire locks, sleep, etc.
54  * It must not be called from high-level interrupt context.
55  *
56  * Depending on how the requested page is being used at the time of the retire
57  * request (and on the availability of sufficient system resources), the page
58  * may be retired immediately, or just marked for retirement later. For
59  * example, locked pages are marked, while free pages are retired. Multiple
60  * requests may be made to retire the same page, although there is no need
61  * to: once the p_toxic flags are set, the page will be retired as soon as it
62  * can be exclusively locked.
63  *
64  * The retire mechanism is driven centrally out of page_unlock(). To expedite
65  * the retirement of pages, further requests for SE_SHARED locks are denied
66  * as long as a page retirement is pending. In addition, as long as pages are
67  * pending retirement a background thread runs periodically trying to retire
68  * those pages. Pages which could not be retired while the system is running
69  * are scrubbed prior to rebooting to avoid latent errors on the next boot.
70  *
71  * UE pages without persistent errors are scrubbed and returned to service.
72  * Recidivist pages, as well as FMA-directed requests for retirement, result
73  * in the page being taken out of service. Once the decision is made to take
74  * a page out of service, the page is cleared, hashed onto the retired_pages
75  * vnode, marked as retired, and it is unlocked.  No other requesters (except
76  * for unretire) are allowed to lock retired pages.
77  *
78  * The public routines return (sadly) 0 if they worked and a non-zero error
79  * value if something went wrong. This is done for the ioctl side of the
80  * world to allow errors to be reflected all the way out to user land. The
81  * non-zero values are explained in comments atop each function.
82  */
83 
84 /*
85  * Things to fix:
86  *
87  * 	1. Trying to retire non-relocatable kvp pages may result in a
88  *      quagmire. This is because seg_kmem() no longer keeps its pages locked,
89  *      and calls page_lookup() in the free path; since kvp pages are modified
90  *      and don't have a usable backing store, page_retire() can't do anything
91  *      with them, and we'll keep denying the lock to seg_kmem_free() in a
92  *      vicious cycle. To prevent that, we don't deny locks to kvp pages, and
93  *      hence only try to retire a page from page_unlock() in the free path.
94  *      Since most kernel pages are indefinitely held anyway, and don't
95  *      participate in I/O, this is of little consequence.
96  *
97  *      2. Low memory situations will be interesting. If we don't have
98  *      enough memory for page_relocate() to succeed, we won't be able to
99  *      retire dirty pages; nobody will be able to push them out to disk
100  *      either, since we aggressively deny the page lock. We could change
101  *      fsflush so it can recognize this situation, grab the lock, and push
102  *      the page out, where we'll catch it in the free path and retire it.
103  *
104  *	3. Beware of places that have code like this in them:
105  *
106  *		if (! page_tryupgrade(pp)) {
107  *			page_unlock(pp);
108  *			while (! page_lock(pp, SE_EXCL, NULL, P_RECLAIM)) {
109  *				/ *NOTHING* /
110  *			}
111  *		}
112  *		page_free(pp);
113  *
114  *	The problem is that pp can change identity right after the
115  *	page_unlock() call.  In particular, page_retire() can step in
116  *	there, change pp's identity, and hash pp onto the retired_vnode.
117  *
118  *	Of course, other functions besides page_retire() can have the
119  *	same effect. A kmem reader can waltz by, set up a mapping to the
120  *	page, and then unlock the page. Page_free() will then go castors
121  *	up. So if anybody is doing this, it's already a bug.
122  *
123  *      4. mdboot()'s call into page_retire_mdboot() should probably be
124  *      moved lower. Where the call is made now, we can get into trouble
125  *      by scrubbing a kernel page that is then accessed later.
126  */
127 
128 #include <sys/types.h>
129 #include <sys/param.h>
130 #include <sys/systm.h>
131 #include <sys/mman.h>
132 #include <sys/vnode.h>
133 #include <sys/vfs_opreg.h>
134 #include <sys/cmn_err.h>
135 #include <sys/ksynch.h>
136 #include <sys/thread.h>
137 #include <sys/disp.h>
138 #include <sys/ontrap.h>
139 #include <sys/vmsystm.h>
140 #include <sys/mem_config.h>
141 #include <sys/atomic.h>
142 #include <sys/callb.h>
143 #include <sys/kobj.h>
144 #include <vm/page.h>
145 #include <vm/vm_dep.h>
146 #include <vm/as.h>
147 #include <vm/hat.h>
148 #include <vm/seg_kmem.h>
149 
150 /*
151  * vnode for all pages which are retired from the VM system;
152  */
153 vnode_t *retired_pages;
154 
155 static int page_retire_pp_finish(page_t *, void *, uint_t);
156 
157 /*
158  * Make a list of all of the pages that have been marked for retirement
159  * but are not yet retired.  At system shutdown, we will scrub all of the
160  * pages in the list in case there are outstanding UEs.  Then, we
161  * cross-check this list against the number of pages that are yet to be
162  * retired, and if we find inconsistencies, we scan every page_t in the
163  * whole system looking for any pages that need to be scrubbed for UEs.
164  * The background thread also uses this queue to determine which pages
165  * it should keep trying to retire.
166  */
167 #ifdef	DEBUG
168 #define	PR_PENDING_QMAX	32
169 #else	/* DEBUG */
170 #define	PR_PENDING_QMAX	256
171 #endif	/* DEBUG */
172 page_t		*pr_pending_q[PR_PENDING_QMAX];
173 kmutex_t	pr_q_mutex;
174 
175 /*
176  * Page retire global kstats
177  */
178 struct page_retire_kstat {
179 	kstat_named_t	pr_retired;
180 	kstat_named_t	pr_requested;
181 	kstat_named_t	pr_requested_free;
182 	kstat_named_t	pr_enqueue_fail;
183 	kstat_named_t	pr_dequeue_fail;
184 	kstat_named_t	pr_pending;
185 	kstat_named_t	pr_pending_kas;
186 	kstat_named_t	pr_failed;
187 	kstat_named_t	pr_failed_kernel;
188 	kstat_named_t	pr_limit;
189 	kstat_named_t	pr_limit_exceeded;
190 	kstat_named_t	pr_fma;
191 	kstat_named_t	pr_mce;
192 	kstat_named_t	pr_ue;
193 	kstat_named_t	pr_ue_cleared_retire;
194 	kstat_named_t	pr_ue_cleared_free;
195 	kstat_named_t	pr_ue_persistent;
196 	kstat_named_t	pr_unretired;
197 };
198 
199 static struct page_retire_kstat page_retire_kstat = {
200 	{ "pages_retired",		KSTAT_DATA_UINT64},
201 	{ "pages_retire_request",	KSTAT_DATA_UINT64},
202 	{ "pages_retire_request_free",	KSTAT_DATA_UINT64},
203 	{ "pages_notenqueued", 		KSTAT_DATA_UINT64},
204 	{ "pages_notdequeued", 		KSTAT_DATA_UINT64},
205 	{ "pages_pending", 		KSTAT_DATA_UINT64},
206 	{ "pages_pending_kas", 		KSTAT_DATA_UINT64},
207 	{ "pages_deferred",		KSTAT_DATA_UINT64},
208 	{ "pages_deferred_kernel",	KSTAT_DATA_UINT64},
209 	{ "pages_limit",		KSTAT_DATA_UINT64},
210 	{ "pages_limit_exceeded",	KSTAT_DATA_UINT64},
211 	{ "pages_fma",			KSTAT_DATA_UINT64},
212 	{ "pages_multiple_ce",		KSTAT_DATA_UINT64},
213 	{ "pages_ue",			KSTAT_DATA_UINT64},
214 	{ "pages_ue_cleared_retired",	KSTAT_DATA_UINT64},
215 	{ "pages_ue_cleared_freed",	KSTAT_DATA_UINT64},
216 	{ "pages_ue_persistent",	KSTAT_DATA_UINT64},
217 	{ "pages_unretired",		KSTAT_DATA_UINT64},
218 };
219 
220 static kstat_t  *page_retire_ksp = NULL;
221 
222 #define	PR_INCR_KSTAT(stat)	\
223 	atomic_inc_64(&(page_retire_kstat.stat.value.ui64))
224 #define	PR_DECR_KSTAT(stat)	\
225 	atomic_dec_64(&(page_retire_kstat.stat.value.ui64))
226 
227 #define	PR_KSTAT_RETIRED_CE	(page_retire_kstat.pr_mce.value.ui64)
228 #define	PR_KSTAT_RETIRED_FMA	(page_retire_kstat.pr_fma.value.ui64)
229 #define	PR_KSTAT_RETIRED_NOTUE	(PR_KSTAT_RETIRED_CE + PR_KSTAT_RETIRED_FMA)
230 #define	PR_KSTAT_PENDING	(page_retire_kstat.pr_pending.value.ui64)
231 #define	PR_KSTAT_PENDING_KAS	(page_retire_kstat.pr_pending_kas.value.ui64)
232 #define	PR_KSTAT_EQFAIL		(page_retire_kstat.pr_enqueue_fail.value.ui64)
233 #define	PR_KSTAT_DQFAIL		(page_retire_kstat.pr_dequeue_fail.value.ui64)
234 
235 /*
236  * page retire kstats to list all retired pages
237  */
238 static int pr_list_kstat_update(kstat_t *ksp, int rw);
239 static int pr_list_kstat_snapshot(kstat_t *ksp, void *buf, int rw);
240 kmutex_t pr_list_kstat_mutex;
241 
242 /*
243  * Limit the number of multiple CE page retires.
244  * The default is 0.1% of physmem, or 1 in 1000 pages. This is set in
245  * basis points, where 100 basis points equals one percent.
246  */
247 #define	MCE_BPT	10
248 uint64_t	max_pages_retired_bps = MCE_BPT;
249 #define	PAGE_RETIRE_LIMIT	((physmem * max_pages_retired_bps) / 10000)
250 
251 /*
252  * Control over the verbosity of page retirement.
253  *
254  * When set to zero (the default), no messages will be printed.
255  * When set to one, summary messages will be printed.
256  * When set > one, all messages will be printed.
257  *
258  * A value of one will trigger detailed messages for retirement operations,
259  * and is intended as a platform tunable for processors where FMA's DE does
260  * not run (e.g., spitfire). Values > one are intended for debugging only.
261  */
262 int page_retire_messages = 0;
263 
264 /*
265  * Control whether or not we return scrubbed UE pages to service.
266  * By default we do not since FMA wants to run its diagnostics first
267  * and then ask us to unretire the page if it passes. Non-FMA platforms
268  * may set this to zero so we will only retire recidivist pages. It should
269  * not be changed by the user.
270  */
271 int page_retire_first_ue = 1;
272 
273 /*
274  * Master enable for page retire. This prevents a CE or UE early in boot
275  * from trying to retire a page before page_retire_init() has finished
276  * setting things up. This is internal only and is not a tunable!
277  */
278 static int pr_enable = 0;
279 
280 static void (*memscrub_notify_func)(uint64_t);
281 
282 #ifdef	DEBUG
283 struct page_retire_debug {
284 	int prd_dup1;
285 	int prd_dup2;
286 	int prd_qdup;
287 	int prd_noaction;
288 	int prd_queued;
289 	int prd_notqueued;
290 	int prd_dequeue;
291 	int prd_top;
292 	int prd_locked;
293 	int prd_reloc;
294 	int prd_relocfail;
295 	int prd_mod;
296 	int prd_mod_late;
297 	int prd_kern;
298 	int prd_free;
299 	int prd_noreclaim;
300 	int prd_hashout;
301 	int prd_fma;
302 	int prd_uescrubbed;
303 	int prd_uenotscrubbed;
304 	int prd_mce;
305 	int prd_prlocked;
306 	int prd_prnotlocked;
307 	int prd_prretired;
308 	int prd_ulocked;
309 	int prd_unotretired;
310 	int prd_udestroy;
311 	int prd_uhashout;
312 	int prd_uunretired;
313 	int prd_unotlocked;
314 	int prd_checkhit;
315 	int prd_checkmiss_pend;
316 	int prd_checkmiss_noerr;
317 	int prd_tctop;
318 	int prd_tclocked;
319 	int prd_hunt;
320 	int prd_dohunt;
321 	int prd_earlyhunt;
322 	int prd_latehunt;
323 	int prd_nofreedemote;
324 	int prd_nodemote;
325 	int prd_demoted;
326 } pr_debug;
327 
328 #define	PR_DEBUG(foo)	((pr_debug.foo)++)
329 
330 /*
331  * A type histogram. We record the incidence of the various toxic
332  * flag combinations along with the interesting page attributes. The
333  * goal is to get as many combinations as we can while driving all
334  * pr_debug values nonzero (indicating we've exercised all possible
335  * code paths across all possible page types). Not all combinations
336  * will make sense -- e.g. PRT_MOD|PRT_KERNEL.
337  *
338  * pr_type offset bit encoding (when examining with a debugger):
339  *
340  *    PRT_NAMED  - 0x4
341  *    PRT_KERNEL - 0x8
342  *    PRT_FREE   - 0x10
343  *    PRT_MOD    - 0x20
344  *    PRT_FMA    - 0x0
345  *    PRT_MCE    - 0x40
346  *    PRT_UE     - 0x80
347  */
348 
349 #define	PRT_NAMED	0x01
350 #define	PRT_KERNEL	0x02
351 #define	PRT_FREE	0x04
352 #define	PRT_MOD		0x08
353 #define	PRT_FMA		0x00	/* yes, this is not a mistake */
354 #define	PRT_MCE		0x10
355 #define	PRT_UE		0x20
356 #define	PRT_ALL		0x3F
357 
358 int pr_types[PRT_ALL+1];
359 
360 #define	PR_TYPES(pp)	{			\
361 	int whichtype = 0;			\
362 	if (pp->p_vnode)			\
363 		whichtype |= PRT_NAMED;		\
364 	if (PP_ISKAS(pp))			\
365 		whichtype |= PRT_KERNEL;	\
366 	if (PP_ISFREE(pp))			\
367 		whichtype |= PRT_FREE;		\
368 	if (hat_ismod(pp))			\
369 		whichtype |= PRT_MOD;		\
370 	if (pp->p_toxic & PR_UE)		\
371 		whichtype |= PRT_UE;		\
372 	if (pp->p_toxic & PR_MCE)		\
373 		whichtype |= PRT_MCE;		\
374 	pr_types[whichtype]++;			\
375 }
376 
377 int recl_calls;
378 int recl_mtbf = 3;
379 int reloc_calls;
380 int reloc_mtbf = 7;
381 int pr_calls;
382 int pr_mtbf = 15;
383 
384 #define	MTBF(v, f)	(((++(v)) & (f)) != (f))
385 
386 #else	/* DEBUG */
387 
388 #define	PR_DEBUG(foo)	/* nothing */
389 #define	PR_TYPES(foo)	/* nothing */
390 #define	MTBF(v, f)	(1)
391 
392 #endif	/* DEBUG */
393 
394 /*
395  * page_retire_done() - completion processing
396  *
397  * Used by the page_retire code for common completion processing.
398  * It keeps track of how many times a given result has happened,
399  * and writes out an occasional message.
400  *
401  * May be called with a NULL pp (PRD_INVALID_PA case).
402  */
403 #define	PRD_INVALID_KEY		-1
404 #define	PRD_SUCCESS		0
405 #define	PRD_PENDING		1
406 #define	PRD_FAILED		2
407 #define	PRD_DUPLICATE		3
408 #define	PRD_INVALID_PA		4
409 #define	PRD_LIMIT		5
410 #define	PRD_UE_SCRUBBED		6
411 #define	PRD_UNR_SUCCESS		7
412 #define	PRD_UNR_CANTLOCK	8
413 #define	PRD_UNR_NOT		9
414 
415 typedef struct page_retire_op {
416 	int	pr_key;		/* one of the PRD_* defines from above */
417 	int	pr_count;	/* How many times this has happened */
418 	int	pr_retval;	/* return value */
419 	int	pr_msglvl;	/* message level - when to print */
420 	char	*pr_message;	/* Cryptic message for field service */
421 } page_retire_op_t;
422 
423 static page_retire_op_t page_retire_ops[] = {
424 	/* key			count	retval	msglvl	message */
425 	{PRD_SUCCESS,		0,	0,	1,
426 		"Page 0x%08x.%08x removed from service"},
427 	{PRD_PENDING,		0,	EAGAIN,	2,
428 		"Page 0x%08x.%08x will be retired on free"},
429 	{PRD_FAILED,		0,	EAGAIN,	0, NULL},
430 	{PRD_DUPLICATE,		0,	EIO,	2,
431 		"Page 0x%08x.%08x already retired or pending"},
432 	{PRD_INVALID_PA,	0,	EINVAL, 2,
433 		"PA 0x%08x.%08x is not a relocatable page"},
434 	{PRD_LIMIT,		0,	0,	1,
435 		"Page 0x%08x.%08x not retired due to limit exceeded"},
436 	{PRD_UE_SCRUBBED,	0,	0,	1,
437 		"Previously reported error on page 0x%08x.%08x cleared"},
438 	{PRD_UNR_SUCCESS,	0,	0,	1,
439 		"Page 0x%08x.%08x returned to service"},
440 	{PRD_UNR_CANTLOCK,	0,	EAGAIN,	2,
441 		"Page 0x%08x.%08x could not be unretired"},
442 	{PRD_UNR_NOT,		0,	EIO,	2,
443 		"Page 0x%08x.%08x is not retired"},
444 	{PRD_INVALID_KEY,	0,	0,	0, NULL} /* MUST BE LAST! */
445 };
446 
447 /*
448  * print a message if page_retire_messages is true.
449  */
450 #define	PR_MESSAGE(debuglvl, msglvl, msg, pa)				\
451 {									\
452 	uint64_t p = (uint64_t)pa;					\
453 	if (page_retire_messages >= msglvl && msg != NULL) {		\
454 		cmn_err(debuglvl, msg,					\
455 		    (uint32_t)(p >> 32), (uint32_t)p);			\
456 	}								\
457 }
458 
459 /*
460  * Note that multiple bits may be set in a single settoxic operation.
461  * May be called without the page locked.
462  */
463 void
page_settoxic(page_t * pp,uchar_t bits)464 page_settoxic(page_t *pp, uchar_t bits)
465 {
466 	atomic_or_8(&pp->p_toxic, bits);
467 }
468 
469 /*
470  * Note that multiple bits may cleared in a single clrtoxic operation.
471  * Must be called with the page exclusively locked to prevent races which
472  * may attempt to retire a page without any toxic bits set.
473  * Note that the PR_CAPTURE bit can be cleared without the exclusive lock
474  * being held as there is a separate mutex which protects that bit.
475  */
476 void
page_clrtoxic(page_t * pp,uchar_t bits)477 page_clrtoxic(page_t *pp, uchar_t bits)
478 {
479 	ASSERT((bits & PR_CAPTURE) || PAGE_EXCL(pp));
480 	atomic_and_8(&pp->p_toxic, ~bits);
481 }
482 
483 /*
484  * Prints any page retire messages to the user, and decides what
485  * error code is appropriate for the condition reported.
486  */
487 static int
page_retire_done(page_t * pp,int code)488 page_retire_done(page_t *pp, int code)
489 {
490 	page_retire_op_t *prop;
491 	uint64_t	pa = 0;
492 	int		i;
493 
494 	if (pp != NULL) {
495 		pa = mmu_ptob((uint64_t)pp->p_pagenum);
496 	}
497 
498 	prop = NULL;
499 	for (i = 0; page_retire_ops[i].pr_key != PRD_INVALID_KEY; i++) {
500 		if (page_retire_ops[i].pr_key == code) {
501 			prop = &page_retire_ops[i];
502 			break;
503 		}
504 	}
505 
506 #ifdef	DEBUG
507 	if (page_retire_ops[i].pr_key == PRD_INVALID_KEY) {
508 		cmn_err(CE_PANIC, "page_retire_done: Invalid opcode %d", code);
509 	}
510 #endif
511 
512 	ASSERT(prop->pr_key == code);
513 
514 	prop->pr_count++;
515 
516 	PR_MESSAGE(CE_NOTE, prop->pr_msglvl, prop->pr_message, pa);
517 	if (pp != NULL) {
518 		page_settoxic(pp, PR_MSG);
519 	}
520 
521 	return (prop->pr_retval);
522 }
523 
524 /*
525  * Act like page_destroy(), but instead of freeing the page, hash it onto
526  * the retired_pages vnode, and mark it retired.
527  *
528  * For fun, we try to scrub the page until it's squeaky clean.
529  * availrmem is adjusted here.
530  */
531 static void
page_retire_destroy(page_t * pp)532 page_retire_destroy(page_t *pp)
533 {
534 	u_offset_t off = (u_offset_t)((uintptr_t)pp);
535 
536 	ASSERT(PAGE_EXCL(pp));
537 	ASSERT(!PP_ISFREE(pp));
538 	ASSERT(pp->p_szc == 0);
539 	ASSERT(!hat_page_is_mapped(pp));
540 	ASSERT(!pp->p_vnode);
541 
542 	page_clr_all_props(pp);
543 	pagescrub(pp, 0, MMU_PAGESIZE);
544 
545 	pp->p_next = NULL;
546 	pp->p_prev = NULL;
547 	if (page_hashin(pp, retired_pages, off, NULL) == 0) {
548 		cmn_err(CE_PANIC, "retired page %p hashin failed", (void *)pp);
549 	}
550 
551 	page_settoxic(pp, PR_RETIRED);
552 	PR_INCR_KSTAT(pr_retired);
553 
554 	if (pp->p_toxic & PR_FMA) {
555 		PR_INCR_KSTAT(pr_fma);
556 	} else if (pp->p_toxic & PR_UE) {
557 		PR_INCR_KSTAT(pr_ue);
558 	} else {
559 		PR_INCR_KSTAT(pr_mce);
560 	}
561 
562 	mutex_enter(&freemem_lock);
563 	availrmem--;
564 	mutex_exit(&freemem_lock);
565 
566 	page_unlock(pp);
567 }
568 
569 /*
570  * Check whether the number of pages which have been retired already exceeds
571  * the maximum allowable percentage of memory which may be retired.
572  *
573  * Returns 1 if the limit has been exceeded.
574  */
575 static int
page_retire_limit(void)576 page_retire_limit(void)
577 {
578 	if (PR_KSTAT_RETIRED_NOTUE >= (uint64_t)PAGE_RETIRE_LIMIT) {
579 		PR_INCR_KSTAT(pr_limit_exceeded);
580 		return (1);
581 	}
582 
583 	return (0);
584 }
585 
586 #define	MSG_DM	"Data Mismatch occurred at PA 0x%08x.%08x"		\
587 	"[ 0x%x != 0x%x ] while attempting to clear previously "	\
588 	"reported error; page removed from service"
589 
590 #define	MSG_UE	"Uncorrectable Error occurred at PA 0x%08x.%08x while "	\
591 	"attempting to clear previously reported error; page removed "	\
592 	"from service"
593 
594 /*
595  * Attempt to clear a UE from a page.
596  * Returns 1 if the error has been successfully cleared.
597  */
598 static int
page_clear_transient_ue(page_t * pp)599 page_clear_transient_ue(page_t *pp)
600 {
601 	caddr_t		kaddr;
602 	uint8_t		rb, wb;
603 	uint64_t	pa;
604 	uint32_t	pa_hi, pa_lo;
605 	on_trap_data_t	otd;
606 	int		errors = 0;
607 	int		i;
608 
609 	ASSERT(PAGE_EXCL(pp));
610 	ASSERT(PP_PR_REQ(pp));
611 	ASSERT(pp->p_szc == 0);
612 	ASSERT(!hat_page_is_mapped(pp));
613 
614 	/*
615 	 * Clear the page and attempt to clear the UE.  If we trap
616 	 * on the next access to the page, we know the UE has recurred.
617 	 */
618 	pagescrub(pp, 0, PAGESIZE);
619 
620 	/*
621 	 * Map the page and write a bunch of bit patterns to compare
622 	 * what we wrote with what we read back.  This isn't a perfect
623 	 * test but it should be good enough to catch most of the
624 	 * recurring UEs. If this fails to catch a recurrent UE, we'll
625 	 * retire the page the next time we see a UE on the page.
626 	 */
627 	kaddr = ppmapin(pp, PROT_READ|PROT_WRITE, (caddr_t)-1);
628 
629 	pa = ptob((uint64_t)page_pptonum(pp));
630 	pa_hi = (uint32_t)(pa >> 32);
631 	pa_lo = (uint32_t)pa;
632 
633 	/*
634 	 * Disable preemption to prevent the off chance that
635 	 * we migrate while in the middle of running through
636 	 * the bit pattern and run on a different processor
637 	 * than what we started on.
638 	 */
639 	kpreempt_disable();
640 
641 	/*
642 	 * Fill the page with each (0x00 - 0xFF] bit pattern, flushing
643 	 * the cache in between reading and writing.  We do this under
644 	 * on_trap() protection to avoid recursion.
645 	 */
646 	if (on_trap(&otd, OT_DATA_EC)) {
647 		PR_MESSAGE(CE_WARN, 1, MSG_UE, pa);
648 		errors = 1;
649 	} else {
650 		for (wb = 0xff; wb > 0; wb--) {
651 			for (i = 0; i < PAGESIZE; i++) {
652 				kaddr[i] = wb;
653 			}
654 
655 			sync_data_memory(kaddr, PAGESIZE);
656 
657 			for (i = 0; i < PAGESIZE; i++) {
658 				rb = kaddr[i];
659 				if (rb != wb) {
660 					/*
661 					 * We had a mismatch without a trap.
662 					 * Uh-oh. Something is really wrong
663 					 * with this system.
664 					 */
665 					if (page_retire_messages) {
666 						cmn_err(CE_WARN, MSG_DM,
667 						    pa_hi, pa_lo, rb, wb);
668 					}
669 					errors = 1;
670 					goto out;	/* double break */
671 				}
672 			}
673 		}
674 	}
675 out:
676 	no_trap();
677 	kpreempt_enable();
678 	ppmapout(kaddr);
679 
680 	return (errors ? 0 : 1);
681 }
682 
683 /*
684  * Try to clear a page_t with a single UE. If the UE was transient, it is
685  * returned to service, and we return 1. Otherwise we return 0 meaning
686  * that further processing is required to retire the page.
687  */
688 static int
page_retire_transient_ue(page_t * pp)689 page_retire_transient_ue(page_t *pp)
690 {
691 	ASSERT(PAGE_EXCL(pp));
692 	ASSERT(!hat_page_is_mapped(pp));
693 
694 	/*
695 	 * If this page is a repeat offender, retire it under the
696 	 * "two strikes and you're out" rule. The caller is responsible
697 	 * for scrubbing the page to try to clear the error.
698 	 */
699 	if (pp->p_toxic & PR_UE_SCRUBBED) {
700 		PR_INCR_KSTAT(pr_ue_persistent);
701 		return (0);
702 	}
703 
704 	if (page_clear_transient_ue(pp)) {
705 		/*
706 		 * We set the PR_SCRUBBED_UE bit; if we ever see this
707 		 * page again, we will retire it, no questions asked.
708 		 */
709 		page_settoxic(pp, PR_UE_SCRUBBED);
710 
711 		if (page_retire_first_ue) {
712 			PR_INCR_KSTAT(pr_ue_cleared_retire);
713 			return (0);
714 		} else {
715 			PR_INCR_KSTAT(pr_ue_cleared_free);
716 
717 			page_clrtoxic(pp, PR_UE | PR_MCE | PR_MSG);
718 
719 			/* LINTED: CONSTCOND */
720 			VN_DISPOSE(pp, B_FREE, 1, kcred);
721 			return (1);
722 		}
723 	}
724 
725 	PR_INCR_KSTAT(pr_ue_persistent);
726 	return (0);
727 }
728 
729 /*
730  * Update the statistics dynamically when our kstat is read.
731  */
732 static int
page_retire_kstat_update(kstat_t * ksp,int rw)733 page_retire_kstat_update(kstat_t *ksp, int rw)
734 {
735 	struct page_retire_kstat *pr;
736 
737 	if (ksp == NULL)
738 		return (EINVAL);
739 
740 	switch (rw) {
741 
742 	case KSTAT_READ:
743 		pr = (struct page_retire_kstat *)ksp->ks_data;
744 		ASSERT(pr == &page_retire_kstat);
745 		pr->pr_limit.value.ui64 = PAGE_RETIRE_LIMIT;
746 		return (0);
747 
748 	case KSTAT_WRITE:
749 		return (EACCES);
750 
751 	default:
752 		return (EINVAL);
753 	}
754 	/*NOTREACHED*/
755 }
756 
757 static int
pr_list_kstat_update(kstat_t * ksp,int rw)758 pr_list_kstat_update(kstat_t *ksp, int rw)
759 {
760 	uint_t count;
761 	page_t *pp;
762 	kmutex_t *vphm;
763 
764 	if (rw == KSTAT_WRITE)
765 		return (EACCES);
766 
767 	vphm = page_vnode_mutex(retired_pages);
768 	mutex_enter(vphm);
769 	/* Needs to be under a lock so that for loop will work right */
770 	if (retired_pages->v_pages == NULL) {
771 		mutex_exit(vphm);
772 		ksp->ks_ndata = 0;
773 		ksp->ks_data_size = 0;
774 		return (0);
775 	}
776 
777 	count = 1;
778 	for (pp = retired_pages->v_pages->p_vpnext;
779 	    pp != retired_pages->v_pages; pp = pp->p_vpnext) {
780 		count++;
781 	}
782 	mutex_exit(vphm);
783 
784 	ksp->ks_ndata = count;
785 	ksp->ks_data_size = count * 2 * sizeof (uint64_t);
786 
787 	return (0);
788 }
789 
790 /*
791  * all spans will be pagesize and no coalescing will be done with the
792  * list produced.
793  */
794 static int
pr_list_kstat_snapshot(kstat_t * ksp,void * buf,int rw)795 pr_list_kstat_snapshot(kstat_t *ksp, void *buf, int rw)
796 {
797 	kmutex_t *vphm;
798 	page_t *pp;
799 	struct memunit {
800 		uint64_t address;
801 		uint64_t size;
802 	} *kspmem;
803 
804 	if (rw == KSTAT_WRITE)
805 		return (EACCES);
806 
807 	ksp->ks_snaptime = gethrtime();
808 
809 	kspmem = (struct memunit *)buf;
810 
811 	vphm = page_vnode_mutex(retired_pages);
812 	mutex_enter(vphm);
813 	pp = retired_pages->v_pages;
814 	if (((caddr_t)kspmem >= (caddr_t)buf + ksp->ks_data_size) ||
815 	    (pp == NULL)) {
816 		mutex_exit(vphm);
817 		return (0);
818 	}
819 	kspmem->address = ptob(pp->p_pagenum);
820 	kspmem->size = PAGESIZE;
821 	kspmem++;
822 	for (pp = pp->p_vpnext; pp != retired_pages->v_pages;
823 	    pp = pp->p_vpnext, kspmem++) {
824 		if ((caddr_t)kspmem >= (caddr_t)buf + ksp->ks_data_size)
825 			break;
826 		kspmem->address = ptob(pp->p_pagenum);
827 		kspmem->size = PAGESIZE;
828 	}
829 	mutex_exit(vphm);
830 
831 	return (0);
832 }
833 
834 /*
835  * page_retire_pend_count -- helper function for page_capture_thread,
836  * returns the number of pages pending retirement.
837  */
838 uint64_t
page_retire_pend_count(void)839 page_retire_pend_count(void)
840 {
841 	return (PR_KSTAT_PENDING);
842 }
843 
844 uint64_t
page_retire_pend_kas_count(void)845 page_retire_pend_kas_count(void)
846 {
847 	return (PR_KSTAT_PENDING_KAS);
848 }
849 
850 void
page_retire_incr_pend_count(void * datap)851 page_retire_incr_pend_count(void *datap)
852 {
853 	PR_INCR_KSTAT(pr_pending);
854 
855 	if (datap == &kvp || datap == &kvps[KV_ZVP] || datap == &kvps[KV_VVP])
856 		PR_INCR_KSTAT(pr_pending_kas);
857 }
858 
859 void
page_retire_decr_pend_count(void * datap)860 page_retire_decr_pend_count(void *datap)
861 {
862 	PR_DECR_KSTAT(pr_pending);
863 
864 	if (datap == &kvp || datap == &kvps[KV_ZVP] || datap == &kvps[KV_VVP])
865 		PR_DECR_KSTAT(pr_pending_kas);
866 }
867 
868 /*
869  * Initialize the page retire mechanism:
870  *
871  *   - Establish the correctable error retire limit.
872  *   - Initialize locks.
873  *   - Build the retired_pages vnode.
874  *   - Set up the kstats.
875  *   - Fire off the background thread.
876  *   - Tell page_retire() it's OK to start retiring pages.
877  */
878 void
page_retire_init(void)879 page_retire_init(void)
880 {
881 	const fs_operation_def_t retired_vnodeops_template[] = {
882 		{ NULL, NULL }
883 	};
884 	struct vnodeops *vops;
885 	kstat_t *ksp;
886 
887 	const uint_t page_retire_ndata =
888 	    sizeof (page_retire_kstat) / sizeof (kstat_named_t);
889 
890 	ASSERT(page_retire_ksp == NULL);
891 
892 	if (max_pages_retired_bps <= 0) {
893 		max_pages_retired_bps = MCE_BPT;
894 	}
895 
896 	mutex_init(&pr_q_mutex, NULL, MUTEX_DEFAULT, NULL);
897 
898 	retired_pages = vn_alloc(KM_SLEEP);
899 	if (vn_make_ops("retired_pages", retired_vnodeops_template, &vops)) {
900 		cmn_err(CE_PANIC,
901 		    "page_retired_init: can't make retired vnodeops");
902 	}
903 	vn_setops(retired_pages, vops);
904 
905 	if ((page_retire_ksp = kstat_create("unix", 0, "page_retire",
906 	    "misc", KSTAT_TYPE_NAMED, page_retire_ndata,
907 	    KSTAT_FLAG_VIRTUAL)) == NULL) {
908 		cmn_err(CE_WARN, "kstat_create for page_retire failed");
909 	} else {
910 		page_retire_ksp->ks_data = (void *)&page_retire_kstat;
911 		page_retire_ksp->ks_update = page_retire_kstat_update;
912 		kstat_install(page_retire_ksp);
913 	}
914 
915 	mutex_init(&pr_list_kstat_mutex, NULL, MUTEX_DEFAULT, NULL);
916 	ksp = kstat_create("unix", 0, "page_retire_list", "misc",
917 	    KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VAR_SIZE | KSTAT_FLAG_VIRTUAL);
918 	if (ksp != NULL) {
919 		ksp->ks_update = pr_list_kstat_update;
920 		ksp->ks_snapshot = pr_list_kstat_snapshot;
921 		ksp->ks_lock = &pr_list_kstat_mutex;
922 		kstat_install(ksp);
923 	}
924 
925 	memscrub_notify_func =
926 	    (void(*)(uint64_t))kobj_getsymvalue("memscrub_notify", 0);
927 
928 	page_capture_register_callback(PC_RETIRE, -1, page_retire_pp_finish);
929 	pr_enable = 1;
930 }
931 
932 /*
933  * page_retire_hunt() callback for the retire thread.
934  */
935 static void
page_retire_thread_cb(page_t * pp)936 page_retire_thread_cb(page_t *pp)
937 {
938 	PR_DEBUG(prd_tctop);
939 	if (!PP_ISKAS(pp) && page_trylock(pp, SE_EXCL)) {
940 		PR_DEBUG(prd_tclocked);
941 		page_unlock(pp);
942 	}
943 }
944 
945 /*
946  * Callback used by page_trycapture() to finish off retiring a page.
947  * The page has already been cleaned and we've been given sole access to
948  * it.
949  * Always returns 0 to indicate that callback succeded as the callback never
950  * fails to finish retiring the given page.
951  */
952 /*ARGSUSED*/
953 static int
page_retire_pp_finish(page_t * pp,void * notused,uint_t flags)954 page_retire_pp_finish(page_t *pp, void *notused, uint_t flags)
955 {
956 	int		toxic;
957 
958 	ASSERT(PAGE_EXCL(pp));
959 	ASSERT(pp->p_iolock_state == 0);
960 	ASSERT(pp->p_szc == 0);
961 
962 	toxic = pp->p_toxic;
963 
964 	/*
965 	 * The problem page is locked, demoted, unmapped, not free,
966 	 * hashed out, and not COW or mlocked (whew!).
967 	 *
968 	 * Now we select our ammunition, take it around back, and shoot it.
969 	 */
970 	if (toxic & PR_UE) {
971 ue_error:
972 		if (page_retire_transient_ue(pp)) {
973 			PR_DEBUG(prd_uescrubbed);
974 			(void) page_retire_done(pp, PRD_UE_SCRUBBED);
975 		} else {
976 			PR_DEBUG(prd_uenotscrubbed);
977 			page_retire_destroy(pp);
978 			(void) page_retire_done(pp, PRD_SUCCESS);
979 		}
980 		return (0);
981 	} else if (toxic & PR_FMA) {
982 		PR_DEBUG(prd_fma);
983 		page_retire_destroy(pp);
984 		(void) page_retire_done(pp, PRD_SUCCESS);
985 		return (0);
986 	} else if (toxic & PR_MCE) {
987 		PR_DEBUG(prd_mce);
988 		page_retire_destroy(pp);
989 		(void) page_retire_done(pp, PRD_SUCCESS);
990 		return (0);
991 	}
992 
993 	/*
994 	 * When page_retire_first_ue is set to zero and a UE occurs which is
995 	 * transient, it's possible that we clear some flags set by a second
996 	 * UE error on the page which occurs while the first is currently being
997 	 * handled and thus we need to handle the case where none of the above
998 	 * are set.  In this instance, PR_UE_SCRUBBED should be set and thus
999 	 * we should execute the UE code above.
1000 	 */
1001 	if (toxic & PR_UE_SCRUBBED) {
1002 		goto ue_error;
1003 	}
1004 
1005 	/*
1006 	 * It's impossible to get here.
1007 	 */
1008 	panic("bad toxic flags 0x%x in page_retire_pp_finish\n", toxic);
1009 	return (0);
1010 }
1011 
1012 /*
1013  * page_retire() - the front door in to retire a page.
1014  *
1015  * Ideally, page_retire() would instantly retire the requested page.
1016  * Unfortunately, some pages are locked or otherwise tied up and cannot be
1017  * retired right away.  We use the page capture logic to deal with this
1018  * situation as it will continuously try to retire the page in the background
1019  * if the first attempt fails.  Success is determined by looking to see whether
1020  * the page has been retired after the page_trycapture() attempt.
1021  *
1022  * Returns:
1023  *
1024  *   - 0 on success,
1025  *   - EINVAL when the PA is whacko,
1026  *   - EIO if the page is already retired or already pending retirement, or
1027  *   - EAGAIN if the page could not be _immediately_ retired but is pending.
1028  */
1029 int
page_retire(uint64_t pa,uchar_t reason)1030 page_retire(uint64_t pa, uchar_t reason)
1031 {
1032 	page_t	*pp;
1033 
1034 	ASSERT(reason & PR_REASONS);		/* there must be a reason */
1035 	ASSERT(!(reason & ~PR_REASONS));	/* but no other bits */
1036 
1037 	pp = page_numtopp_nolock(mmu_btop(pa));
1038 	if (pp == NULL) {
1039 		PR_MESSAGE(CE_WARN, 1, "Cannot schedule clearing of error on"
1040 		    " page 0x%08x.%08x; page is not relocatable memory", pa);
1041 		return (page_retire_done(pp, PRD_INVALID_PA));
1042 	}
1043 	if (PP_RETIRED(pp)) {
1044 		PR_DEBUG(prd_dup1);
1045 		return (page_retire_done(pp, PRD_DUPLICATE));
1046 	}
1047 
1048 	if (memscrub_notify_func != NULL) {
1049 		(void) memscrub_notify_func(pa);
1050 	}
1051 
1052 	if ((reason & PR_UE) && !PP_TOXIC(pp)) {
1053 		PR_MESSAGE(CE_NOTE, 1, "Scheduling clearing of error on"
1054 		    " page 0x%08x.%08x", pa);
1055 	} else if (PP_PR_REQ(pp)) {
1056 		PR_DEBUG(prd_dup2);
1057 		return (page_retire_done(pp, PRD_DUPLICATE));
1058 	} else {
1059 		PR_MESSAGE(CE_NOTE, 1, "Scheduling removal of"
1060 		    " page 0x%08x.%08x", pa);
1061 	}
1062 
1063 	/* Avoid setting toxic bits in the first place */
1064 	if ((reason & (PR_FMA | PR_MCE)) && !(reason & PR_UE) &&
1065 	    page_retire_limit()) {
1066 		return (page_retire_done(pp, PRD_LIMIT));
1067 	}
1068 
1069 	if (MTBF(pr_calls, pr_mtbf)) {
1070 		page_settoxic(pp, reason);
1071 		if (page_trycapture(pp, 0, CAPTURE_RETIRE, pp->p_vnode) == 0) {
1072 			PR_DEBUG(prd_prlocked);
1073 		} else {
1074 			PR_DEBUG(prd_prnotlocked);
1075 		}
1076 	} else {
1077 		PR_DEBUG(prd_prnotlocked);
1078 	}
1079 
1080 	if (PP_RETIRED(pp)) {
1081 		PR_DEBUG(prd_prretired);
1082 		return (0);
1083 	} else {
1084 		cv_signal(&pc_cv);
1085 		PR_INCR_KSTAT(pr_failed);
1086 
1087 		if (pp->p_toxic & PR_MSG) {
1088 			return (page_retire_done(pp, PRD_FAILED));
1089 		} else {
1090 			return (page_retire_done(pp, PRD_PENDING));
1091 		}
1092 	}
1093 }
1094 
1095 /*
1096  * Take a retired page off the retired-pages vnode and clear the toxic flags.
1097  * If "free" is nonzero, lock it and put it back on the freelist. If "free"
1098  * is zero, the caller already holds SE_EXCL lock so we simply unretire it
1099  * and don't do anything else with it.
1100  *
1101  * Any unretire messages are printed from this routine.
1102  *
1103  * Returns 0 if page pp was unretired; else an error code.
1104  *
1105  * If flags is:
1106  *	PR_UNR_FREE - lock the page, clear the toxic flags and free it
1107  *	    to the freelist.
1108  *	PR_UNR_TEMP - lock the page, unretire it, leave the toxic
1109  *	    bits set as is and return it to the caller.
1110  *	PR_UNR_CLEAN - page is SE_EXCL locked, unretire it, clear the
1111  *	    toxic flags and return it to caller as is.
1112  */
1113 int
page_unretire_pp(page_t * pp,int flags)1114 page_unretire_pp(page_t *pp, int flags)
1115 {
1116 	/*
1117 	 * To be retired, a page has to be hashed onto the retired_pages vnode
1118 	 * and have PR_RETIRED set in p_toxic.
1119 	 */
1120 	if (flags == PR_UNR_CLEAN ||
1121 	    page_try_reclaim_lock(pp, SE_EXCL, SE_RETIRED)) {
1122 		ASSERT(PAGE_EXCL(pp));
1123 		PR_DEBUG(prd_ulocked);
1124 		if (!PP_RETIRED(pp)) {
1125 			PR_DEBUG(prd_unotretired);
1126 			page_unlock(pp);
1127 			return (page_retire_done(pp, PRD_UNR_NOT));
1128 		}
1129 
1130 		PR_MESSAGE(CE_NOTE, 1, "unretiring retired"
1131 		    " page 0x%08x.%08x", mmu_ptob((uint64_t)pp->p_pagenum));
1132 		if (pp->p_toxic & PR_FMA) {
1133 			PR_DECR_KSTAT(pr_fma);
1134 		} else if (pp->p_toxic & PR_UE) {
1135 			PR_DECR_KSTAT(pr_ue);
1136 		} else {
1137 			PR_DECR_KSTAT(pr_mce);
1138 		}
1139 
1140 		if (flags == PR_UNR_TEMP)
1141 			page_clrtoxic(pp, PR_RETIRED);
1142 		else
1143 			page_clrtoxic(pp, PR_TOXICFLAGS);
1144 
1145 		if (flags == PR_UNR_FREE) {
1146 			PR_DEBUG(prd_udestroy);
1147 			page_destroy(pp, 0);
1148 		} else {
1149 			PR_DEBUG(prd_uhashout);
1150 			page_hashout(pp, NULL);
1151 		}
1152 
1153 		mutex_enter(&freemem_lock);
1154 		availrmem++;
1155 		mutex_exit(&freemem_lock);
1156 
1157 		PR_DEBUG(prd_uunretired);
1158 		PR_DECR_KSTAT(pr_retired);
1159 		PR_INCR_KSTAT(pr_unretired);
1160 		return (page_retire_done(pp, PRD_UNR_SUCCESS));
1161 	}
1162 	PR_DEBUG(prd_unotlocked);
1163 	return (page_retire_done(pp, PRD_UNR_CANTLOCK));
1164 }
1165 
1166 /*
1167  * Return a page to service by moving it from the retired_pages vnode
1168  * onto the freelist.
1169  *
1170  * Called from mmioctl_page_retire() on behalf of the FMA DE.
1171  *
1172  * Returns:
1173  *
1174  *   - 0 if the page is unretired,
1175  *   - EAGAIN if the pp can not be locked,
1176  *   - EINVAL if the PA is whacko, and
1177  *   - EIO if the pp is not retired.
1178  */
1179 int
page_unretire(uint64_t pa)1180 page_unretire(uint64_t pa)
1181 {
1182 	page_t	*pp;
1183 
1184 	pp = page_numtopp_nolock(mmu_btop(pa));
1185 	if (pp == NULL) {
1186 		return (page_retire_done(pp, PRD_INVALID_PA));
1187 	}
1188 
1189 	return (page_unretire_pp(pp, PR_UNR_FREE));
1190 }
1191 
1192 /*
1193  * Test a page to see if it is retired. If errors is non-NULL, the toxic
1194  * bits of the page are returned. Returns 0 on success, error code on failure.
1195  */
1196 int
page_retire_check_pp(page_t * pp,uint64_t * errors)1197 page_retire_check_pp(page_t *pp, uint64_t *errors)
1198 {
1199 	int rc;
1200 
1201 	if (PP_RETIRED(pp)) {
1202 		PR_DEBUG(prd_checkhit);
1203 		rc = 0;
1204 	} else if (PP_PR_REQ(pp)) {
1205 		PR_DEBUG(prd_checkmiss_pend);
1206 		rc = EAGAIN;
1207 	} else {
1208 		PR_DEBUG(prd_checkmiss_noerr);
1209 		rc = EIO;
1210 	}
1211 
1212 	/*
1213 	 * We have magically arranged the bit values returned to fmd(1M)
1214 	 * to line up with the FMA, MCE, and UE bits of the page_t.
1215 	 */
1216 	if (errors) {
1217 		uint64_t toxic = (uint64_t)(pp->p_toxic & PR_ERRMASK);
1218 		if (toxic & PR_UE_SCRUBBED) {
1219 			toxic &= ~PR_UE_SCRUBBED;
1220 			toxic |= PR_UE;
1221 		}
1222 		*errors = toxic;
1223 	}
1224 
1225 	return (rc);
1226 }
1227 
1228 /*
1229  * Test to see if the page_t for a given PA is retired, and return the
1230  * hardware errors we have seen on the page if requested.
1231  *
1232  * Called from mmioctl_page_retire on behalf of the FMA DE.
1233  *
1234  * Returns:
1235  *
1236  *   - 0 if the page is retired,
1237  *   - EIO if the page is not retired and has no errors,
1238  *   - EAGAIN if the page is not retired but is pending; and
1239  *   - EINVAL if the PA is whacko.
1240  */
1241 int
page_retire_check(uint64_t pa,uint64_t * errors)1242 page_retire_check(uint64_t pa, uint64_t *errors)
1243 {
1244 	page_t	*pp;
1245 
1246 	if (errors) {
1247 		*errors = 0;
1248 	}
1249 
1250 	pp = page_numtopp_nolock(mmu_btop(pa));
1251 	if (pp == NULL) {
1252 		return (page_retire_done(pp, PRD_INVALID_PA));
1253 	}
1254 
1255 	return (page_retire_check_pp(pp, errors));
1256 }
1257 
1258 /*
1259  * Page retire self-test. For now, it always returns 0.
1260  */
1261 int
page_retire_test(void)1262 page_retire_test(void)
1263 {
1264 	page_t *first, *pp, *cpp, *cpp2, *lpp;
1265 
1266 	/*
1267 	 * Tests the corner case where a large page can't be retired
1268 	 * because one of the constituent pages is locked. We mark
1269 	 * one page to be retired and try to retire it, and mark the
1270 	 * other page to be retired but don't try to retire it, so
1271 	 * that page_unlock() in the failure path will recurse and try
1272 	 * to retire THAT page. This is the worst possible situation
1273 	 * we can get ourselves into.
1274 	 */
1275 	memsegs_lock(0);
1276 	pp = first = page_first();
1277 	do {
1278 		if (pp->p_szc && PP_PAGEROOT(pp) == pp) {
1279 			cpp = pp + 1;
1280 			lpp = PP_ISFREE(pp)? pp : pp + 2;
1281 			cpp2 = pp + 3;
1282 			if (!page_trylock(lpp, pp == lpp? SE_EXCL : SE_SHARED))
1283 				continue;
1284 			if (!page_trylock(cpp, SE_EXCL)) {
1285 				page_unlock(lpp);
1286 				continue;
1287 			}
1288 
1289 			/* fails */
1290 			(void) page_retire(ptob(cpp->p_pagenum), PR_FMA);
1291 
1292 			page_unlock(lpp);
1293 			page_unlock(cpp);
1294 			(void) page_retire(ptob(cpp->p_pagenum), PR_FMA);
1295 			(void) page_retire(ptob(cpp2->p_pagenum), PR_FMA);
1296 		}
1297 	} while ((pp = page_next(pp)) != first);
1298 	memsegs_unlock(0);
1299 
1300 	return (0);
1301 }
1302