1db874c57Selowe /*
2db874c57Selowe * CDDL HEADER START
3db874c57Selowe *
4db874c57Selowe * The contents of this file are subject to the terms of the
58b464eb8Smec * Common Development and Distribution License (the "License").
68b464eb8Smec * You may not use this file except in compliance with the License.
7db874c57Selowe *
8db874c57Selowe * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9db874c57Selowe * or http://www.opensolaris.org/os/licensing.
10db874c57Selowe * See the License for the specific language governing permissions
11db874c57Selowe * and limitations under the License.
12db874c57Selowe *
13db874c57Selowe * When distributing Covered Code, include this CDDL HEADER in each
14db874c57Selowe * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15db874c57Selowe * If applicable, add the following below this CDDL HEADER, with the
16db874c57Selowe * fields enclosed by brackets "[]" replaced with your own identifying
17db874c57Selowe * information: Portions Copyright [yyyy] [name of copyright owner]
18db874c57Selowe *
19db874c57Selowe * CDDL HEADER END
20db874c57Selowe */
21db874c57Selowe /*
221a3c9a5aSVijay S Balakrishna * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23db874c57Selowe * Use is subject to license terms.
2448bbca81SDaniel Hoffman * Copyright (c) 2016 by Delphix. All rights reserved.
2504909c8cSJohn Levon * Copyright 2018 Joyent, Inc.
26db874c57Selowe */
27db874c57Selowe
28db874c57Selowe /*
29db874c57Selowe * Page Retire - Big Theory Statement.
30db874c57Selowe *
31db874c57Selowe * This file handles removing sections of faulty memory from use when the
32db874c57Selowe * user land FMA Diagnosis Engine requests that a page be removed or when
33db874c57Selowe * a CE or UE is detected by the hardware.
34db874c57Selowe *
35db874c57Selowe * In the bad old days, the kernel side of Page Retire did a lot of the work
36db874c57Selowe * on its own. Now, with the DE keeping track of errors, the kernel side is
37db874c57Selowe * rather simple minded on most platforms.
38db874c57Selowe *
39db874c57Selowe * Errors are all reflected to the DE, and after digesting the error and
40db874c57Selowe * looking at all previously reported errors, the DE decides what should
41db874c57Selowe * be done about the current error. If the DE wants a particular page to
42db874c57Selowe * be retired, then the kernel page retire code is invoked via an ioctl.
43db874c57Selowe * On non-FMA platforms, the ue_drain and ce_drain paths ends up calling
44db874c57Selowe * page retire to handle the error. Since page retire is just a simple
45db874c57Selowe * mechanism it doesn't need to differentiate between the different callers.
46db874c57Selowe *
47db874c57Selowe * The p_toxic field in the page_t is used to indicate which errors have
48db874c57Selowe * occurred and what action has been taken on a given page. Because errors are
49db874c57Selowe * reported without regard to the locked state of a page, no locks are used
50db874c57Selowe * to SET the error bits in p_toxic. However, in order to clear the error
51db874c57Selowe * bits, the page_t must be held exclusively locked.
52db874c57Selowe *
53db874c57Selowe * When page_retire() is called, it must be able to acquire locks, sleep, etc.
54db874c57Selowe * It must not be called from high-level interrupt context.
55db874c57Selowe *
56db874c57Selowe * Depending on how the requested page is being used at the time of the retire
57db874c57Selowe * request (and on the availability of sufficient system resources), the page
58db874c57Selowe * may be retired immediately, or just marked for retirement later. For
59db874c57Selowe * example, locked pages are marked, while free pages are retired. Multiple
60db874c57Selowe * requests may be made to retire the same page, although there is no need
61db874c57Selowe * to: once the p_toxic flags are set, the page will be retired as soon as it
62db874c57Selowe * can be exclusively locked.
63db874c57Selowe *
64db874c57Selowe * The retire mechanism is driven centrally out of page_unlock(). To expedite
65db874c57Selowe * the retirement of pages, further requests for SE_SHARED locks are denied
66db874c57Selowe * as long as a page retirement is pending. In addition, as long as pages are
67db874c57Selowe * pending retirement a background thread runs periodically trying to retire
68db874c57Selowe * those pages. Pages which could not be retired while the system is running
69db874c57Selowe * are scrubbed prior to rebooting to avoid latent errors on the next boot.
70db874c57Selowe *
714fc2445aSelowe * UE pages without persistent errors are scrubbed and returned to service.
724fc2445aSelowe * Recidivist pages, as well as FMA-directed requests for retirement, result
734fc2445aSelowe * in the page being taken out of service. Once the decision is made to take
744fc2445aSelowe * a page out of service, the page is cleared, hashed onto the retired_pages
754fc2445aSelowe * vnode, marked as retired, and it is unlocked. No other requesters (except
764fc2445aSelowe * for unretire) are allowed to lock retired pages.
77db874c57Selowe *
78db874c57Selowe * The public routines return (sadly) 0 if they worked and a non-zero error
79db874c57Selowe * value if something went wrong. This is done for the ioctl side of the
80db874c57Selowe * world to allow errors to be reflected all the way out to user land. The
81db874c57Selowe * non-zero values are explained in comments atop each function.
82db874c57Selowe */
83db874c57Selowe
84db874c57Selowe /*
85db874c57Selowe * Things to fix:
86db874c57Selowe *
878b464eb8Smec * 1. Trying to retire non-relocatable kvp pages may result in a
88db874c57Selowe * quagmire. This is because seg_kmem() no longer keeps its pages locked,
89db874c57Selowe * and calls page_lookup() in the free path; since kvp pages are modified
90db874c57Selowe * and don't have a usable backing store, page_retire() can't do anything
91db874c57Selowe * with them, and we'll keep denying the lock to seg_kmem_free() in a
92db874c57Selowe * vicious cycle. To prevent that, we don't deny locks to kvp pages, and
938b464eb8Smec * hence only try to retire a page from page_unlock() in the free path.
94db874c57Selowe * Since most kernel pages are indefinitely held anyway, and don't
95db874c57Selowe * participate in I/O, this is of little consequence.
96db874c57Selowe *
978b464eb8Smec * 2. Low memory situations will be interesting. If we don't have
98db874c57Selowe * enough memory for page_relocate() to succeed, we won't be able to
99db874c57Selowe * retire dirty pages; nobody will be able to push them out to disk
100db874c57Selowe * either, since we aggressively deny the page lock. We could change
101db874c57Selowe * fsflush so it can recognize this situation, grab the lock, and push
102db874c57Selowe * the page out, where we'll catch it in the free path and retire it.
103db874c57Selowe *
1048b464eb8Smec * 3. Beware of places that have code like this in them:
105db874c57Selowe *
106db874c57Selowe * if (! page_tryupgrade(pp)) {
107db874c57Selowe * page_unlock(pp);
108db874c57Selowe * while (! page_lock(pp, SE_EXCL, NULL, P_RECLAIM)) {
109db874c57Selowe * / *NOTHING* /
110db874c57Selowe * }
111db874c57Selowe * }
112db874c57Selowe * page_free(pp);
113db874c57Selowe *
114db874c57Selowe * The problem is that pp can change identity right after the
115db874c57Selowe * page_unlock() call. In particular, page_retire() can step in
116db874c57Selowe * there, change pp's identity, and hash pp onto the retired_vnode.
117db874c57Selowe *
118db874c57Selowe * Of course, other functions besides page_retire() can have the
119db874c57Selowe * same effect. A kmem reader can waltz by, set up a mapping to the
120db874c57Selowe * page, and then unlock the page. Page_free() will then go castors
121db874c57Selowe * up. So if anybody is doing this, it's already a bug.
122db874c57Selowe *
1238b464eb8Smec * 4. mdboot()'s call into page_retire_mdboot() should probably be
124db874c57Selowe * moved lower. Where the call is made now, we can get into trouble
125db874c57Selowe * by scrubbing a kernel page that is then accessed later.
126db874c57Selowe */
127db874c57Selowe
128db874c57Selowe #include <sys/types.h>
129db874c57Selowe #include <sys/param.h>
130db874c57Selowe #include <sys/systm.h>
131db874c57Selowe #include <sys/mman.h>
132db874c57Selowe #include <sys/vnode.h>
133aa59c4cbSrsb #include <sys/vfs_opreg.h>
134db874c57Selowe #include <sys/cmn_err.h>
135db874c57Selowe #include <sys/ksynch.h>
136db874c57Selowe #include <sys/thread.h>
137db874c57Selowe #include <sys/disp.h>
138db874c57Selowe #include <sys/ontrap.h>
139db874c57Selowe #include <sys/vmsystm.h>
140db874c57Selowe #include <sys/mem_config.h>
141db874c57Selowe #include <sys/atomic.h>
142db874c57Selowe #include <sys/callb.h>
1431a3c9a5aSVijay S Balakrishna #include <sys/kobj.h>
144db874c57Selowe #include <vm/page.h>
145db874c57Selowe #include <vm/vm_dep.h>
146db874c57Selowe #include <vm/as.h>
147db874c57Selowe #include <vm/hat.h>
148af4c679fSSean McEnroe #include <vm/seg_kmem.h>
149db874c57Selowe
150db874c57Selowe /*
151db874c57Selowe * vnode for all pages which are retired from the VM system;
152db874c57Selowe */
153db874c57Selowe vnode_t *retired_pages;
154db874c57Selowe
1558b464eb8Smec static int page_retire_pp_finish(page_t *, void *, uint_t);
156db874c57Selowe
157db874c57Selowe /*
158db874c57Selowe * Make a list of all of the pages that have been marked for retirement
159db874c57Selowe * but are not yet retired. At system shutdown, we will scrub all of the
160db874c57Selowe * pages in the list in case there are outstanding UEs. Then, we
161db874c57Selowe * cross-check this list against the number of pages that are yet to be
162db874c57Selowe * retired, and if we find inconsistencies, we scan every page_t in the
163db874c57Selowe * whole system looking for any pages that need to be scrubbed for UEs.
164db874c57Selowe * The background thread also uses this queue to determine which pages
165db874c57Selowe * it should keep trying to retire.
166db874c57Selowe */
167db874c57Selowe #ifdef DEBUG
168db874c57Selowe #define PR_PENDING_QMAX 32
169db874c57Selowe #else /* DEBUG */
170db874c57Selowe #define PR_PENDING_QMAX 256
171db874c57Selowe #endif /* DEBUG */
172db874c57Selowe page_t *pr_pending_q[PR_PENDING_QMAX];
173db874c57Selowe kmutex_t pr_q_mutex;
174db874c57Selowe
175db874c57Selowe /*
176db874c57Selowe * Page retire global kstats
177db874c57Selowe */
178db874c57Selowe struct page_retire_kstat {
179db874c57Selowe kstat_named_t pr_retired;
180db874c57Selowe kstat_named_t pr_requested;
181db874c57Selowe kstat_named_t pr_requested_free;
182db874c57Selowe kstat_named_t pr_enqueue_fail;
183db874c57Selowe kstat_named_t pr_dequeue_fail;
184db874c57Selowe kstat_named_t pr_pending;
185704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States kstat_named_t pr_pending_kas;
186db874c57Selowe kstat_named_t pr_failed;
187db874c57Selowe kstat_named_t pr_failed_kernel;
188db874c57Selowe kstat_named_t pr_limit;
189db874c57Selowe kstat_named_t pr_limit_exceeded;
190db874c57Selowe kstat_named_t pr_fma;
191db874c57Selowe kstat_named_t pr_mce;
192db874c57Selowe kstat_named_t pr_ue;
193db874c57Selowe kstat_named_t pr_ue_cleared_retire;
194db874c57Selowe kstat_named_t pr_ue_cleared_free;
195db874c57Selowe kstat_named_t pr_ue_persistent;
196db874c57Selowe kstat_named_t pr_unretired;
197db874c57Selowe };
198db874c57Selowe
199db874c57Selowe static struct page_retire_kstat page_retire_kstat = {
200db874c57Selowe { "pages_retired", KSTAT_DATA_UINT64},
201db874c57Selowe { "pages_retire_request", KSTAT_DATA_UINT64},
202db874c57Selowe { "pages_retire_request_free", KSTAT_DATA_UINT64},
203db874c57Selowe { "pages_notenqueued", KSTAT_DATA_UINT64},
204db874c57Selowe { "pages_notdequeued", KSTAT_DATA_UINT64},
205db874c57Selowe { "pages_pending", KSTAT_DATA_UINT64},
206704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States { "pages_pending_kas", KSTAT_DATA_UINT64},
207db874c57Selowe { "pages_deferred", KSTAT_DATA_UINT64},
208db874c57Selowe { "pages_deferred_kernel", KSTAT_DATA_UINT64},
209db874c57Selowe { "pages_limit", KSTAT_DATA_UINT64},
210db874c57Selowe { "pages_limit_exceeded", KSTAT_DATA_UINT64},
211db874c57Selowe { "pages_fma", KSTAT_DATA_UINT64},
212db874c57Selowe { "pages_multiple_ce", KSTAT_DATA_UINT64},
213db874c57Selowe { "pages_ue", KSTAT_DATA_UINT64},
214db874c57Selowe { "pages_ue_cleared_retired", KSTAT_DATA_UINT64},
215db874c57Selowe { "pages_ue_cleared_freed", KSTAT_DATA_UINT64},
216db874c57Selowe { "pages_ue_persistent", KSTAT_DATA_UINT64},
217db874c57Selowe { "pages_unretired", KSTAT_DATA_UINT64},
218db874c57Selowe };
219db874c57Selowe
220db874c57Selowe static kstat_t *page_retire_ksp = NULL;
221db874c57Selowe
222db874c57Selowe #define PR_INCR_KSTAT(stat) \
2231a5e258fSJosef 'Jeff' Sipek atomic_inc_64(&(page_retire_kstat.stat.value.ui64))
224db874c57Selowe #define PR_DECR_KSTAT(stat) \
2251a5e258fSJosef 'Jeff' Sipek atomic_dec_64(&(page_retire_kstat.stat.value.ui64))
226db874c57Selowe
227db874c57Selowe #define PR_KSTAT_RETIRED_CE (page_retire_kstat.pr_mce.value.ui64)
228db874c57Selowe #define PR_KSTAT_RETIRED_FMA (page_retire_kstat.pr_fma.value.ui64)
229db874c57Selowe #define PR_KSTAT_RETIRED_NOTUE (PR_KSTAT_RETIRED_CE + PR_KSTAT_RETIRED_FMA)
230db874c57Selowe #define PR_KSTAT_PENDING (page_retire_kstat.pr_pending.value.ui64)
231704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States #define PR_KSTAT_PENDING_KAS (page_retire_kstat.pr_pending_kas.value.ui64)
232db874c57Selowe #define PR_KSTAT_EQFAIL (page_retire_kstat.pr_enqueue_fail.value.ui64)
233db874c57Selowe #define PR_KSTAT_DQFAIL (page_retire_kstat.pr_dequeue_fail.value.ui64)
234db874c57Selowe
2358b464eb8Smec /*
2368b464eb8Smec * page retire kstats to list all retired pages
2378b464eb8Smec */
2388b464eb8Smec static int pr_list_kstat_update(kstat_t *ksp, int rw);
2398b464eb8Smec static int pr_list_kstat_snapshot(kstat_t *ksp, void *buf, int rw);
2408b464eb8Smec kmutex_t pr_list_kstat_mutex;
2418b464eb8Smec
242db874c57Selowe /*
243db874c57Selowe * Limit the number of multiple CE page retires.
244db874c57Selowe * The default is 0.1% of physmem, or 1 in 1000 pages. This is set in
245db874c57Selowe * basis points, where 100 basis points equals one percent.
246db874c57Selowe */
247db874c57Selowe #define MCE_BPT 10
248db874c57Selowe uint64_t max_pages_retired_bps = MCE_BPT;
249db874c57Selowe #define PAGE_RETIRE_LIMIT ((physmem * max_pages_retired_bps) / 10000)
250db874c57Selowe
251db874c57Selowe /*
252db874c57Selowe * Control over the verbosity of page retirement.
253db874c57Selowe *
254db874c57Selowe * When set to zero (the default), no messages will be printed.
255db874c57Selowe * When set to one, summary messages will be printed.
256db874c57Selowe * When set > one, all messages will be printed.
257db874c57Selowe *
258db874c57Selowe * A value of one will trigger detailed messages for retirement operations,
259db874c57Selowe * and is intended as a platform tunable for processors where FMA's DE does
260db874c57Selowe * not run (e.g., spitfire). Values > one are intended for debugging only.
261db874c57Selowe */
262db874c57Selowe int page_retire_messages = 0;
263db874c57Selowe
264db874c57Selowe /*
265db874c57Selowe * Control whether or not we return scrubbed UE pages to service.
266db874c57Selowe * By default we do not since FMA wants to run its diagnostics first
267db874c57Selowe * and then ask us to unretire the page if it passes. Non-FMA platforms
268db874c57Selowe * may set this to zero so we will only retire recidivist pages. It should
269db874c57Selowe * not be changed by the user.
270db874c57Selowe */
271db874c57Selowe int page_retire_first_ue = 1;
272db874c57Selowe
273db874c57Selowe /*
274db874c57Selowe * Master enable for page retire. This prevents a CE or UE early in boot
275db874c57Selowe * from trying to retire a page before page_retire_init() has finished
276db874c57Selowe * setting things up. This is internal only and is not a tunable!
277db874c57Selowe */
278db874c57Selowe static int pr_enable = 0;
279db874c57Selowe
2801a3c9a5aSVijay S Balakrishna static void (*memscrub_notify_func)(uint64_t);
2811a3c9a5aSVijay S Balakrishna
282db874c57Selowe #ifdef DEBUG
283db874c57Selowe struct page_retire_debug {
2842708b84eSelowe int prd_dup1;
2852708b84eSelowe int prd_dup2;
2862708b84eSelowe int prd_qdup;
287db874c57Selowe int prd_noaction;
288db874c57Selowe int prd_queued;
289db874c57Selowe int prd_notqueued;
290db874c57Selowe int prd_dequeue;
291db874c57Selowe int prd_top;
292db874c57Selowe int prd_locked;
293db874c57Selowe int prd_reloc;
29424e9c58bSelowe int prd_relocfail;
29524e9c58bSelowe int prd_mod;
29624e9c58bSelowe int prd_mod_late;
297db874c57Selowe int prd_kern;
298db874c57Selowe int prd_free;
299db874c57Selowe int prd_noreclaim;
300db874c57Selowe int prd_hashout;
301db874c57Selowe int prd_fma;
302db874c57Selowe int prd_uescrubbed;
303db874c57Selowe int prd_uenotscrubbed;
304db874c57Selowe int prd_mce;
305db874c57Selowe int prd_prlocked;
306db874c57Selowe int prd_prnotlocked;
307db874c57Selowe int prd_prretired;
308db874c57Selowe int prd_ulocked;
309db874c57Selowe int prd_unotretired;
310db874c57Selowe int prd_udestroy;
311db874c57Selowe int prd_uhashout;
312db874c57Selowe int prd_uunretired;
313db874c57Selowe int prd_unotlocked;
314db874c57Selowe int prd_checkhit;
3152708b84eSelowe int prd_checkmiss_pend;
3162708b84eSelowe int prd_checkmiss_noerr;
317db874c57Selowe int prd_tctop;
318db874c57Selowe int prd_tclocked;
319db874c57Selowe int prd_hunt;
320db874c57Selowe int prd_dohunt;
321db874c57Selowe int prd_earlyhunt;
322db874c57Selowe int prd_latehunt;
323db874c57Selowe int prd_nofreedemote;
324db874c57Selowe int prd_nodemote;
325db874c57Selowe int prd_demoted;
326db874c57Selowe } pr_debug;
327db874c57Selowe
328db874c57Selowe #define PR_DEBUG(foo) ((pr_debug.foo)++)
329db874c57Selowe
330db874c57Selowe /*
331db874c57Selowe * A type histogram. We record the incidence of the various toxic
332db874c57Selowe * flag combinations along with the interesting page attributes. The
333db874c57Selowe * goal is to get as many combinations as we can while driving all
334db874c57Selowe * pr_debug values nonzero (indicating we've exercised all possible
335db874c57Selowe * code paths across all possible page types). Not all combinations
336db874c57Selowe * will make sense -- e.g. PRT_MOD|PRT_KERNEL.
337db874c57Selowe *
338db874c57Selowe * pr_type offset bit encoding (when examining with a debugger):
339db874c57Selowe *
340db874c57Selowe * PRT_NAMED - 0x4
341db874c57Selowe * PRT_KERNEL - 0x8
342db874c57Selowe * PRT_FREE - 0x10
343db874c57Selowe * PRT_MOD - 0x20
344db874c57Selowe * PRT_FMA - 0x0
345db874c57Selowe * PRT_MCE - 0x40
346db874c57Selowe * PRT_UE - 0x80
347db874c57Selowe */
348db874c57Selowe
349db874c57Selowe #define PRT_NAMED 0x01
350db874c57Selowe #define PRT_KERNEL 0x02
351db874c57Selowe #define PRT_FREE 0x04
352db874c57Selowe #define PRT_MOD 0x08
353db874c57Selowe #define PRT_FMA 0x00 /* yes, this is not a mistake */
354db874c57Selowe #define PRT_MCE 0x10
355db874c57Selowe #define PRT_UE 0x20
356db874c57Selowe #define PRT_ALL 0x3F
357db874c57Selowe
358db874c57Selowe int pr_types[PRT_ALL+1];
359db874c57Selowe
360db874c57Selowe #define PR_TYPES(pp) { \
361db874c57Selowe int whichtype = 0; \
362db874c57Selowe if (pp->p_vnode) \
363db874c57Selowe whichtype |= PRT_NAMED; \
364ad23a2dbSjohansen if (PP_ISKAS(pp)) \
365db874c57Selowe whichtype |= PRT_KERNEL; \
366db874c57Selowe if (PP_ISFREE(pp)) \
367db874c57Selowe whichtype |= PRT_FREE; \
368db874c57Selowe if (hat_ismod(pp)) \
369db874c57Selowe whichtype |= PRT_MOD; \
370db874c57Selowe if (pp->p_toxic & PR_UE) \
371db874c57Selowe whichtype |= PRT_UE; \
372db874c57Selowe if (pp->p_toxic & PR_MCE) \
373db874c57Selowe whichtype |= PRT_MCE; \
374db874c57Selowe pr_types[whichtype]++; \
375db874c57Selowe }
376db874c57Selowe
377db874c57Selowe int recl_calls;
378db874c57Selowe int recl_mtbf = 3;
379db874c57Selowe int reloc_calls;
380db874c57Selowe int reloc_mtbf = 7;
381db874c57Selowe int pr_calls;
382db874c57Selowe int pr_mtbf = 15;
383db874c57Selowe
384db874c57Selowe #define MTBF(v, f) (((++(v)) & (f)) != (f))
385db874c57Selowe
386db874c57Selowe #else /* DEBUG */
387db874c57Selowe
388db874c57Selowe #define PR_DEBUG(foo) /* nothing */
389db874c57Selowe #define PR_TYPES(foo) /* nothing */
390db874c57Selowe #define MTBF(v, f) (1)
391db874c57Selowe
392db874c57Selowe #endif /* DEBUG */
393db874c57Selowe
394db874c57Selowe /*
395db874c57Selowe * page_retire_done() - completion processing
396db874c57Selowe *
397db874c57Selowe * Used by the page_retire code for common completion processing.
398db874c57Selowe * It keeps track of how many times a given result has happened,
399db874c57Selowe * and writes out an occasional message.
400db874c57Selowe *
401db874c57Selowe * May be called with a NULL pp (PRD_INVALID_PA case).
402db874c57Selowe */
403db874c57Selowe #define PRD_INVALID_KEY -1
404db874c57Selowe #define PRD_SUCCESS 0
405db874c57Selowe #define PRD_PENDING 1
406db874c57Selowe #define PRD_FAILED 2
407db874c57Selowe #define PRD_DUPLICATE 3
408db874c57Selowe #define PRD_INVALID_PA 4
409db874c57Selowe #define PRD_LIMIT 5
410db874c57Selowe #define PRD_UE_SCRUBBED 6
411db874c57Selowe #define PRD_UNR_SUCCESS 7
412db874c57Selowe #define PRD_UNR_CANTLOCK 8
413db874c57Selowe #define PRD_UNR_NOT 9
414db874c57Selowe
415db874c57Selowe typedef struct page_retire_op {
416db874c57Selowe int pr_key; /* one of the PRD_* defines from above */
417db874c57Selowe int pr_count; /* How many times this has happened */
418db874c57Selowe int pr_retval; /* return value */
419db874c57Selowe int pr_msglvl; /* message level - when to print */
420db874c57Selowe char *pr_message; /* Cryptic message for field service */
421db874c57Selowe } page_retire_op_t;
422db874c57Selowe
423db874c57Selowe static page_retire_op_t page_retire_ops[] = {
424db874c57Selowe /* key count retval msglvl message */
425db874c57Selowe {PRD_SUCCESS, 0, 0, 1,
426db874c57Selowe "Page 0x%08x.%08x removed from service"},
427db874c57Selowe {PRD_PENDING, 0, EAGAIN, 2,
428db874c57Selowe "Page 0x%08x.%08x will be retired on free"},
429db874c57Selowe {PRD_FAILED, 0, EAGAIN, 0, NULL},
4302708b84eSelowe {PRD_DUPLICATE, 0, EIO, 2,
4312708b84eSelowe "Page 0x%08x.%08x already retired or pending"},
432db874c57Selowe {PRD_INVALID_PA, 0, EINVAL, 2,
433db874c57Selowe "PA 0x%08x.%08x is not a relocatable page"},
434db874c57Selowe {PRD_LIMIT, 0, 0, 1,
435db874c57Selowe "Page 0x%08x.%08x not retired due to limit exceeded"},
436db874c57Selowe {PRD_UE_SCRUBBED, 0, 0, 1,
437db874c57Selowe "Previously reported error on page 0x%08x.%08x cleared"},
438db874c57Selowe {PRD_UNR_SUCCESS, 0, 0, 1,
439db874c57Selowe "Page 0x%08x.%08x returned to service"},
440db874c57Selowe {PRD_UNR_CANTLOCK, 0, EAGAIN, 2,
441db874c57Selowe "Page 0x%08x.%08x could not be unretired"},
4422708b84eSelowe {PRD_UNR_NOT, 0, EIO, 2,
443db874c57Selowe "Page 0x%08x.%08x is not retired"},
444db874c57Selowe {PRD_INVALID_KEY, 0, 0, 0, NULL} /* MUST BE LAST! */
445db874c57Selowe };
446db874c57Selowe
447db874c57Selowe /*
448db874c57Selowe * print a message if page_retire_messages is true.
449db874c57Selowe */
450db874c57Selowe #define PR_MESSAGE(debuglvl, msglvl, msg, pa) \
451db874c57Selowe { \
452db874c57Selowe uint64_t p = (uint64_t)pa; \
453db874c57Selowe if (page_retire_messages >= msglvl && msg != NULL) { \
454db874c57Selowe cmn_err(debuglvl, msg, \
455db874c57Selowe (uint32_t)(p >> 32), (uint32_t)p); \
456db874c57Selowe } \
457db874c57Selowe }
458db874c57Selowe
459db874c57Selowe /*
460db874c57Selowe * Note that multiple bits may be set in a single settoxic operation.
461db874c57Selowe * May be called without the page locked.
462db874c57Selowe */
463db874c57Selowe void
page_settoxic(page_t * pp,uchar_t bits)464db874c57Selowe page_settoxic(page_t *pp, uchar_t bits)
465db874c57Selowe {
466db874c57Selowe atomic_or_8(&pp->p_toxic, bits);
467db874c57Selowe }
468db874c57Selowe
469db874c57Selowe /*
470db874c57Selowe * Note that multiple bits may cleared in a single clrtoxic operation.
4714fc2445aSelowe * Must be called with the page exclusively locked to prevent races which
4724fc2445aSelowe * may attempt to retire a page without any toxic bits set.
4738b464eb8Smec * Note that the PR_CAPTURE bit can be cleared without the exclusive lock
4748b464eb8Smec * being held as there is a separate mutex which protects that bit.
475db874c57Selowe */
476db874c57Selowe void
page_clrtoxic(page_t * pp,uchar_t bits)477db874c57Selowe page_clrtoxic(page_t *pp, uchar_t bits)
478db874c57Selowe {
4798b464eb8Smec ASSERT((bits & PR_CAPTURE) || PAGE_EXCL(pp));
480db874c57Selowe atomic_and_8(&pp->p_toxic, ~bits);
481db874c57Selowe }
482db874c57Selowe
483db874c57Selowe /*
484db874c57Selowe * Prints any page retire messages to the user, and decides what
485db874c57Selowe * error code is appropriate for the condition reported.
486db874c57Selowe */
487db874c57Selowe static int
page_retire_done(page_t * pp,int code)488db874c57Selowe page_retire_done(page_t *pp, int code)
489db874c57Selowe {
490db874c57Selowe page_retire_op_t *prop;
491db874c57Selowe uint64_t pa = 0;
492db874c57Selowe int i;
493db874c57Selowe
494db874c57Selowe if (pp != NULL) {
4954fc2445aSelowe pa = mmu_ptob((uint64_t)pp->p_pagenum);
496db874c57Selowe }
497db874c57Selowe
498db874c57Selowe prop = NULL;
499db874c57Selowe for (i = 0; page_retire_ops[i].pr_key != PRD_INVALID_KEY; i++) {
500db874c57Selowe if (page_retire_ops[i].pr_key == code) {
501db874c57Selowe prop = &page_retire_ops[i];
502db874c57Selowe break;
503db874c57Selowe }
504db874c57Selowe }
505db874c57Selowe
506db874c57Selowe #ifdef DEBUG
507db874c57Selowe if (page_retire_ops[i].pr_key == PRD_INVALID_KEY) {
508db874c57Selowe cmn_err(CE_PANIC, "page_retire_done: Invalid opcode %d", code);
509db874c57Selowe }
510db874c57Selowe #endif
511db874c57Selowe
512db874c57Selowe ASSERT(prop->pr_key == code);
513db874c57Selowe
514db874c57Selowe prop->pr_count++;
515db874c57Selowe
516db874c57Selowe PR_MESSAGE(CE_NOTE, prop->pr_msglvl, prop->pr_message, pa);
517db874c57Selowe if (pp != NULL) {
518db874c57Selowe page_settoxic(pp, PR_MSG);
519db874c57Selowe }
520db874c57Selowe
521db874c57Selowe return (prop->pr_retval);
522db874c57Selowe }
523db874c57Selowe
524db874c57Selowe /*
525db874c57Selowe * Act like page_destroy(), but instead of freeing the page, hash it onto
526db874c57Selowe * the retired_pages vnode, and mark it retired.
527db874c57Selowe *
528db874c57Selowe * For fun, we try to scrub the page until it's squeaky clean.
529db874c57Selowe * availrmem is adjusted here.
530db874c57Selowe */
531db874c57Selowe static void
page_retire_destroy(page_t * pp)532db874c57Selowe page_retire_destroy(page_t *pp)
533db874c57Selowe {
53424e9c58bSelowe u_offset_t off = (u_offset_t)((uintptr_t)pp);
53524e9c58bSelowe
536db874c57Selowe ASSERT(PAGE_EXCL(pp));
537db874c57Selowe ASSERT(!PP_ISFREE(pp));
538db874c57Selowe ASSERT(pp->p_szc == 0);
539db874c57Selowe ASSERT(!hat_page_is_mapped(pp));
540db874c57Selowe ASSERT(!pp->p_vnode);
541db874c57Selowe
5429d0d62adSJason Beloro page_clr_all_props(pp);
543db874c57Selowe pagescrub(pp, 0, MMU_PAGESIZE);
544db874c57Selowe
545db874c57Selowe pp->p_next = NULL;
546db874c57Selowe pp->p_prev = NULL;
54724e9c58bSelowe if (page_hashin(pp, retired_pages, off, NULL) == 0) {
548db874c57Selowe cmn_err(CE_PANIC, "retired page %p hashin failed", (void *)pp);
549db874c57Selowe }
550db874c57Selowe
551db874c57Selowe page_settoxic(pp, PR_RETIRED);
552db874c57Selowe PR_INCR_KSTAT(pr_retired);
553db874c57Selowe
554db874c57Selowe if (pp->p_toxic & PR_FMA) {
555db874c57Selowe PR_INCR_KSTAT(pr_fma);
556db874c57Selowe } else if (pp->p_toxic & PR_UE) {
557db874c57Selowe PR_INCR_KSTAT(pr_ue);
558db874c57Selowe } else {
559db874c57Selowe PR_INCR_KSTAT(pr_mce);
560db874c57Selowe }
561db874c57Selowe
562db874c57Selowe mutex_enter(&freemem_lock);
563db874c57Selowe availrmem--;
564db874c57Selowe mutex_exit(&freemem_lock);
565db874c57Selowe
566db874c57Selowe page_unlock(pp);
567db874c57Selowe }
568db874c57Selowe
569db874c57Selowe /*
570db874c57Selowe * Check whether the number of pages which have been retired already exceeds
571db874c57Selowe * the maximum allowable percentage of memory which may be retired.
572db874c57Selowe *
573db874c57Selowe * Returns 1 if the limit has been exceeded.
574db874c57Selowe */
575db874c57Selowe static int
page_retire_limit(void)576db874c57Selowe page_retire_limit(void)
577db874c57Selowe {
578db874c57Selowe if (PR_KSTAT_RETIRED_NOTUE >= (uint64_t)PAGE_RETIRE_LIMIT) {
579db874c57Selowe PR_INCR_KSTAT(pr_limit_exceeded);
580db874c57Selowe return (1);
581db874c57Selowe }
582db874c57Selowe
583db874c57Selowe return (0);
584db874c57Selowe }
585db874c57Selowe
586db874c57Selowe #define MSG_DM "Data Mismatch occurred at PA 0x%08x.%08x" \
587db874c57Selowe "[ 0x%x != 0x%x ] while attempting to clear previously " \
588db874c57Selowe "reported error; page removed from service"
589db874c57Selowe
590db874c57Selowe #define MSG_UE "Uncorrectable Error occurred at PA 0x%08x.%08x while " \
591db874c57Selowe "attempting to clear previously reported error; page removed " \
592db874c57Selowe "from service"
593db874c57Selowe
594db874c57Selowe /*
595db874c57Selowe * Attempt to clear a UE from a page.
596db874c57Selowe * Returns 1 if the error has been successfully cleared.
597db874c57Selowe */
598db874c57Selowe static int
page_clear_transient_ue(page_t * pp)599db874c57Selowe page_clear_transient_ue(page_t *pp)
600db874c57Selowe {
601db874c57Selowe caddr_t kaddr;
602db874c57Selowe uint8_t rb, wb;
603db874c57Selowe uint64_t pa;
604db874c57Selowe uint32_t pa_hi, pa_lo;
605db874c57Selowe on_trap_data_t otd;
606*55fea89dSDan Cross int errors;
607db874c57Selowe int i;
608db874c57Selowe
609db874c57Selowe ASSERT(PAGE_EXCL(pp));
610db874c57Selowe ASSERT(PP_PR_REQ(pp));
611db874c57Selowe ASSERT(pp->p_szc == 0);
612db874c57Selowe ASSERT(!hat_page_is_mapped(pp));
613db874c57Selowe
614db874c57Selowe /*
615db874c57Selowe * Clear the page and attempt to clear the UE. If we trap
616db874c57Selowe * on the next access to the page, we know the UE has recurred.
617db874c57Selowe */
618db874c57Selowe pagescrub(pp, 0, PAGESIZE);
619db874c57Selowe
620db874c57Selowe /*
621db874c57Selowe * Map the page and write a bunch of bit patterns to compare
622db874c57Selowe * what we wrote with what we read back. This isn't a perfect
623db874c57Selowe * test but it should be good enough to catch most of the
624db874c57Selowe * recurring UEs. If this fails to catch a recurrent UE, we'll
625db874c57Selowe * retire the page the next time we see a UE on the page.
626db874c57Selowe */
627db874c57Selowe kaddr = ppmapin(pp, PROT_READ|PROT_WRITE, (caddr_t)-1);
628db874c57Selowe
629db874c57Selowe pa = ptob((uint64_t)page_pptonum(pp));
630db874c57Selowe pa_hi = (uint32_t)(pa >> 32);
631db874c57Selowe pa_lo = (uint32_t)pa;
632db874c57Selowe
633bb9e3cfdSChristopher Baumbauer - Sun Microsystems - San Diego United States /*
634bb9e3cfdSChristopher Baumbauer - Sun Microsystems - San Diego United States * Disable preemption to prevent the off chance that
635bb9e3cfdSChristopher Baumbauer - Sun Microsystems - San Diego United States * we migrate while in the middle of running through
636bb9e3cfdSChristopher Baumbauer - Sun Microsystems - San Diego United States * the bit pattern and run on a different processor
637bb9e3cfdSChristopher Baumbauer - Sun Microsystems - San Diego United States * than what we started on.
638bb9e3cfdSChristopher Baumbauer - Sun Microsystems - San Diego United States */
639bb9e3cfdSChristopher Baumbauer - Sun Microsystems - San Diego United States kpreempt_disable();
640bb9e3cfdSChristopher Baumbauer - Sun Microsystems - San Diego United States
641db874c57Selowe /*
642db874c57Selowe * Fill the page with each (0x00 - 0xFF] bit pattern, flushing
643db874c57Selowe * the cache in between reading and writing. We do this under
644db874c57Selowe * on_trap() protection to avoid recursion.
645db874c57Selowe */
646db874c57Selowe if (on_trap(&otd, OT_DATA_EC)) {
647db874c57Selowe PR_MESSAGE(CE_WARN, 1, MSG_UE, pa);
648db874c57Selowe errors = 1;
649db874c57Selowe } else {
6503df2e8b2SRobert Mustacchi errors = 0;
651db874c57Selowe for (wb = 0xff; wb > 0; wb--) {
652db874c57Selowe for (i = 0; i < PAGESIZE; i++) {
653db874c57Selowe kaddr[i] = wb;
654db874c57Selowe }
655db874c57Selowe
656db874c57Selowe sync_data_memory(kaddr, PAGESIZE);
657db874c57Selowe
658db874c57Selowe for (i = 0; i < PAGESIZE; i++) {
659db874c57Selowe rb = kaddr[i];
660db874c57Selowe if (rb != wb) {
661db874c57Selowe /*
662db874c57Selowe * We had a mismatch without a trap.
663db874c57Selowe * Uh-oh. Something is really wrong
664db874c57Selowe * with this system.
665db874c57Selowe */
666db874c57Selowe if (page_retire_messages) {
667db874c57Selowe cmn_err(CE_WARN, MSG_DM,
668db874c57Selowe pa_hi, pa_lo, rb, wb);
669db874c57Selowe }
670db874c57Selowe errors = 1;
671db874c57Selowe goto out; /* double break */
672db874c57Selowe }
673db874c57Selowe }
674db874c57Selowe }
675db874c57Selowe }
676db874c57Selowe out:
677db874c57Selowe no_trap();
678bb9e3cfdSChristopher Baumbauer - Sun Microsystems - San Diego United States kpreempt_enable();
679db874c57Selowe ppmapout(kaddr);
680db874c57Selowe
681db874c57Selowe return (errors ? 0 : 1);
682db874c57Selowe }
683db874c57Selowe
684db874c57Selowe /*
685db874c57Selowe * Try to clear a page_t with a single UE. If the UE was transient, it is
686db874c57Selowe * returned to service, and we return 1. Otherwise we return 0 meaning
687db874c57Selowe * that further processing is required to retire the page.
688db874c57Selowe */
689db874c57Selowe static int
page_retire_transient_ue(page_t * pp)690db874c57Selowe page_retire_transient_ue(page_t *pp)
691db874c57Selowe {
692db874c57Selowe ASSERT(PAGE_EXCL(pp));
693db874c57Selowe ASSERT(!hat_page_is_mapped(pp));
694db874c57Selowe
695db874c57Selowe /*
69648bbca81SDaniel Hoffman * If this page is a repeat offender, retire it under the
697db874c57Selowe * "two strikes and you're out" rule. The caller is responsible
698db874c57Selowe * for scrubbing the page to try to clear the error.
699db874c57Selowe */
700db874c57Selowe if (pp->p_toxic & PR_UE_SCRUBBED) {
701db874c57Selowe PR_INCR_KSTAT(pr_ue_persistent);
702db874c57Selowe return (0);
703db874c57Selowe }
704db874c57Selowe
705db874c57Selowe if (page_clear_transient_ue(pp)) {
706db874c57Selowe /*
707db874c57Selowe * We set the PR_SCRUBBED_UE bit; if we ever see this
708db874c57Selowe * page again, we will retire it, no questions asked.
709db874c57Selowe */
710db874c57Selowe page_settoxic(pp, PR_UE_SCRUBBED);
711db874c57Selowe
712db874c57Selowe if (page_retire_first_ue) {
713db874c57Selowe PR_INCR_KSTAT(pr_ue_cleared_retire);
714db874c57Selowe return (0);
715db874c57Selowe } else {
716db874c57Selowe PR_INCR_KSTAT(pr_ue_cleared_free);
717db874c57Selowe
7188b464eb8Smec page_clrtoxic(pp, PR_UE | PR_MCE | PR_MSG);
719db874c57Selowe
720db874c57Selowe /* LINTED: CONSTCOND */
721db874c57Selowe VN_DISPOSE(pp, B_FREE, 1, kcred);
722db874c57Selowe return (1);
723db874c57Selowe }
724db874c57Selowe }
725db874c57Selowe
726db874c57Selowe PR_INCR_KSTAT(pr_ue_persistent);
727db874c57Selowe return (0);
728db874c57Selowe }
729db874c57Selowe
730db874c57Selowe /*
731db874c57Selowe * Update the statistics dynamically when our kstat is read.
732db874c57Selowe */
733db874c57Selowe static int
page_retire_kstat_update(kstat_t * ksp,int rw)734db874c57Selowe page_retire_kstat_update(kstat_t *ksp, int rw)
735db874c57Selowe {
736db874c57Selowe struct page_retire_kstat *pr;
737db874c57Selowe
738db874c57Selowe if (ksp == NULL)
739bb9e3cfdSChristopher Baumbauer - Sun Microsystems - San Diego United States return (EINVAL);
740db874c57Selowe
741db874c57Selowe switch (rw) {
742db874c57Selowe
743db874c57Selowe case KSTAT_READ:
744db874c57Selowe pr = (struct page_retire_kstat *)ksp->ks_data;
745db874c57Selowe ASSERT(pr == &page_retire_kstat);
746db874c57Selowe pr->pr_limit.value.ui64 = PAGE_RETIRE_LIMIT;
747db874c57Selowe return (0);
748db874c57Selowe
749db874c57Selowe case KSTAT_WRITE:
750db874c57Selowe return (EACCES);
751db874c57Selowe
752db874c57Selowe default:
753db874c57Selowe return (EINVAL);
754db874c57Selowe }
755db874c57Selowe /*NOTREACHED*/
756db874c57Selowe }
757db874c57Selowe
7588b464eb8Smec static int
pr_list_kstat_update(kstat_t * ksp,int rw)7598b464eb8Smec pr_list_kstat_update(kstat_t *ksp, int rw)
7608b464eb8Smec {
7618b464eb8Smec uint_t count;
7628b464eb8Smec page_t *pp;
7638b464eb8Smec kmutex_t *vphm;
7648b464eb8Smec
7658b464eb8Smec if (rw == KSTAT_WRITE)
7668b464eb8Smec return (EACCES);
7678b464eb8Smec
7688b464eb8Smec vphm = page_vnode_mutex(retired_pages);
7698b464eb8Smec mutex_enter(vphm);
7708b464eb8Smec /* Needs to be under a lock so that for loop will work right */
7718b464eb8Smec if (retired_pages->v_pages == NULL) {
7728b464eb8Smec mutex_exit(vphm);
7738b464eb8Smec ksp->ks_ndata = 0;
7748b464eb8Smec ksp->ks_data_size = 0;
7758b464eb8Smec return (0);
7768b464eb8Smec }
7778b464eb8Smec
7788b464eb8Smec count = 1;
7798b464eb8Smec for (pp = retired_pages->v_pages->p_vpnext;
7808b464eb8Smec pp != retired_pages->v_pages; pp = pp->p_vpnext) {
7818b464eb8Smec count++;
7828b464eb8Smec }
7838b464eb8Smec mutex_exit(vphm);
7848b464eb8Smec
7858b464eb8Smec ksp->ks_ndata = count;
7868b464eb8Smec ksp->ks_data_size = count * 2 * sizeof (uint64_t);
7878b464eb8Smec
7888b464eb8Smec return (0);
7898b464eb8Smec }
7908b464eb8Smec
7918b464eb8Smec /*
7928b464eb8Smec * all spans will be pagesize and no coalescing will be done with the
7938b464eb8Smec * list produced.
7948b464eb8Smec */
7958b464eb8Smec static int
pr_list_kstat_snapshot(kstat_t * ksp,void * buf,int rw)7968b464eb8Smec pr_list_kstat_snapshot(kstat_t *ksp, void *buf, int rw)
7978b464eb8Smec {
7988b464eb8Smec kmutex_t *vphm;
7998b464eb8Smec page_t *pp;
8008b464eb8Smec struct memunit {
8018b464eb8Smec uint64_t address;
8028b464eb8Smec uint64_t size;
8038b464eb8Smec } *kspmem;
8048b464eb8Smec
8058b464eb8Smec if (rw == KSTAT_WRITE)
8068b464eb8Smec return (EACCES);
8078b464eb8Smec
8088b464eb8Smec ksp->ks_snaptime = gethrtime();
8098b464eb8Smec
8108b464eb8Smec kspmem = (struct memunit *)buf;
8118b464eb8Smec
8128b464eb8Smec vphm = page_vnode_mutex(retired_pages);
8138b464eb8Smec mutex_enter(vphm);
8148b464eb8Smec pp = retired_pages->v_pages;
8158b464eb8Smec if (((caddr_t)kspmem >= (caddr_t)buf + ksp->ks_data_size) ||
8168b464eb8Smec (pp == NULL)) {
8178b464eb8Smec mutex_exit(vphm);
8188b464eb8Smec return (0);
8198b464eb8Smec }
8208b464eb8Smec kspmem->address = ptob(pp->p_pagenum);
8218b464eb8Smec kspmem->size = PAGESIZE;
8228b464eb8Smec kspmem++;
8238b464eb8Smec for (pp = pp->p_vpnext; pp != retired_pages->v_pages;
8248b464eb8Smec pp = pp->p_vpnext, kspmem++) {
8258b464eb8Smec if ((caddr_t)kspmem >= (caddr_t)buf + ksp->ks_data_size)
8268b464eb8Smec break;
8278b464eb8Smec kspmem->address = ptob(pp->p_pagenum);
8288b464eb8Smec kspmem->size = PAGESIZE;
8298b464eb8Smec }
8308b464eb8Smec mutex_exit(vphm);
8318b464eb8Smec
8328b464eb8Smec return (0);
8338b464eb8Smec }
8348b464eb8Smec
835cee1d74bSjfrank /*
836cee1d74bSjfrank * page_retire_pend_count -- helper function for page_capture_thread,
837cee1d74bSjfrank * returns the number of pages pending retirement.
838cee1d74bSjfrank */
839cee1d74bSjfrank uint64_t
page_retire_pend_count(void)840cee1d74bSjfrank page_retire_pend_count(void)
841cee1d74bSjfrank {
842cee1d74bSjfrank return (PR_KSTAT_PENDING);
843cee1d74bSjfrank }
844cee1d74bSjfrank
845704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States uint64_t
page_retire_pend_kas_count(void)846704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States page_retire_pend_kas_count(void)
847704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States {
848704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States return (PR_KSTAT_PENDING_KAS);
849704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States }
850704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States
851cee1d74bSjfrank void
page_retire_incr_pend_count(void * datap)852704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States page_retire_incr_pend_count(void *datap)
853cee1d74bSjfrank {
854cee1d74bSjfrank PR_INCR_KSTAT(pr_pending);
855704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States
85604909c8cSJohn Levon if (datap == &kvp || datap == &kvps[KV_ZVP] || datap == &kvps[KV_VVP])
857704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States PR_INCR_KSTAT(pr_pending_kas);
858cee1d74bSjfrank }
859cee1d74bSjfrank
860cee1d74bSjfrank void
page_retire_decr_pend_count(void * datap)861704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States page_retire_decr_pend_count(void *datap)
862cee1d74bSjfrank {
863cee1d74bSjfrank PR_DECR_KSTAT(pr_pending);
864704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States
86504909c8cSJohn Levon if (datap == &kvp || datap == &kvps[KV_ZVP] || datap == &kvps[KV_VVP])
866704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States PR_DECR_KSTAT(pr_pending_kas);
867cee1d74bSjfrank }
868cee1d74bSjfrank
869db874c57Selowe /*
870db874c57Selowe * Initialize the page retire mechanism:
871db874c57Selowe *
872db874c57Selowe * - Establish the correctable error retire limit.
873db874c57Selowe * - Initialize locks.
874db874c57Selowe * - Build the retired_pages vnode.
875db874c57Selowe * - Set up the kstats.
876db874c57Selowe * - Fire off the background thread.
8778b464eb8Smec * - Tell page_retire() it's OK to start retiring pages.
878db874c57Selowe */
879db874c57Selowe void
page_retire_init(void)880db874c57Selowe page_retire_init(void)
881db874c57Selowe {
882aa59c4cbSrsb const fs_operation_def_t retired_vnodeops_template[] = {
883aa59c4cbSrsb { NULL, NULL }
884aa59c4cbSrsb };
885db874c57Selowe struct vnodeops *vops;
8868b464eb8Smec kstat_t *ksp;
887db874c57Selowe
888db874c57Selowe const uint_t page_retire_ndata =
889db874c57Selowe sizeof (page_retire_kstat) / sizeof (kstat_named_t);
890db874c57Selowe
891db874c57Selowe ASSERT(page_retire_ksp == NULL);
892db874c57Selowe
893db874c57Selowe if (max_pages_retired_bps <= 0) {
894db874c57Selowe max_pages_retired_bps = MCE_BPT;
895db874c57Selowe }
896db874c57Selowe
897db874c57Selowe mutex_init(&pr_q_mutex, NULL, MUTEX_DEFAULT, NULL);
898db874c57Selowe
899db874c57Selowe retired_pages = vn_alloc(KM_SLEEP);
900db874c57Selowe if (vn_make_ops("retired_pages", retired_vnodeops_template, &vops)) {
901db874c57Selowe cmn_err(CE_PANIC,
902db874c57Selowe "page_retired_init: can't make retired vnodeops");
903db874c57Selowe }
904db874c57Selowe vn_setops(retired_pages, vops);
905db874c57Selowe
906db874c57Selowe if ((page_retire_ksp = kstat_create("unix", 0, "page_retire",
907db874c57Selowe "misc", KSTAT_TYPE_NAMED, page_retire_ndata,
908db874c57Selowe KSTAT_FLAG_VIRTUAL)) == NULL) {
909db874c57Selowe cmn_err(CE_WARN, "kstat_create for page_retire failed");
910db874c57Selowe } else {
911db874c57Selowe page_retire_ksp->ks_data = (void *)&page_retire_kstat;
912db874c57Selowe page_retire_ksp->ks_update = page_retire_kstat_update;
913db874c57Selowe kstat_install(page_retire_ksp);
914db874c57Selowe }
915db874c57Selowe
9168b464eb8Smec mutex_init(&pr_list_kstat_mutex, NULL, MUTEX_DEFAULT, NULL);
9178b464eb8Smec ksp = kstat_create("unix", 0, "page_retire_list", "misc",
9188b464eb8Smec KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VAR_SIZE | KSTAT_FLAG_VIRTUAL);
9198b464eb8Smec if (ksp != NULL) {
9208b464eb8Smec ksp->ks_update = pr_list_kstat_update;
9218b464eb8Smec ksp->ks_snapshot = pr_list_kstat_snapshot;
9228b464eb8Smec ksp->ks_lock = &pr_list_kstat_mutex;
9238b464eb8Smec kstat_install(ksp);
9248b464eb8Smec }
925db874c57Selowe
9261a3c9a5aSVijay S Balakrishna memscrub_notify_func =
9271a3c9a5aSVijay S Balakrishna (void(*)(uint64_t))kobj_getsymvalue("memscrub_notify", 0);
9281a3c9a5aSVijay S Balakrishna
9298b464eb8Smec page_capture_register_callback(PC_RETIRE, -1, page_retire_pp_finish);
930db874c57Selowe pr_enable = 1;
931db874c57Selowe }
932db874c57Selowe
933db874c57Selowe /*
934db874c57Selowe * page_retire_hunt() callback for the retire thread.
935db874c57Selowe */
936db874c57Selowe static void
page_retire_thread_cb(page_t * pp)937db874c57Selowe page_retire_thread_cb(page_t *pp)
938db874c57Selowe {
939db874c57Selowe PR_DEBUG(prd_tctop);
940ad23a2dbSjohansen if (!PP_ISKAS(pp) && page_trylock(pp, SE_EXCL)) {
941db874c57Selowe PR_DEBUG(prd_tclocked);
942db874c57Selowe page_unlock(pp);
943db874c57Selowe }
944db874c57Selowe }
945db874c57Selowe
946db874c57Selowe /*
9478b464eb8Smec * Callback used by page_trycapture() to finish off retiring a page.
9488b464eb8Smec * The page has already been cleaned and we've been given sole access to
9498b464eb8Smec * it.
9508b464eb8Smec * Always returns 0 to indicate that callback succeded as the callback never
9518b464eb8Smec * fails to finish retiring the given page.
952db874c57Selowe */
9538b464eb8Smec /*ARGSUSED*/
954db874c57Selowe static int
page_retire_pp_finish(page_t * pp,void * notused,uint_t flags)9558b464eb8Smec page_retire_pp_finish(page_t *pp, void *notused, uint_t flags)
956db874c57Selowe {
957db874c57Selowe int toxic;
958db874c57Selowe
959db874c57Selowe ASSERT(PAGE_EXCL(pp));
960db874c57Selowe ASSERT(pp->p_iolock_state == 0);
961db874c57Selowe ASSERT(pp->p_szc == 0);
962db874c57Selowe
963db874c57Selowe toxic = pp->p_toxic;
964db874c57Selowe
965db874c57Selowe /*
966db874c57Selowe * The problem page is locked, demoted, unmapped, not free,
967db874c57Selowe * hashed out, and not COW or mlocked (whew!).
968db874c57Selowe *
969db874c57Selowe * Now we select our ammunition, take it around back, and shoot it.
970db874c57Selowe */
971db874c57Selowe if (toxic & PR_UE) {
9728b464eb8Smec ue_error:
973db874c57Selowe if (page_retire_transient_ue(pp)) {
974db874c57Selowe PR_DEBUG(prd_uescrubbed);
9758b464eb8Smec (void) page_retire_done(pp, PRD_UE_SCRUBBED);
976db874c57Selowe } else {
977db874c57Selowe PR_DEBUG(prd_uenotscrubbed);
978db874c57Selowe page_retire_destroy(pp);
9798b464eb8Smec (void) page_retire_done(pp, PRD_SUCCESS);
980db874c57Selowe }
9818b464eb8Smec return (0);
982db874c57Selowe } else if (toxic & PR_FMA) {
983db874c57Selowe PR_DEBUG(prd_fma);
984db874c57Selowe page_retire_destroy(pp);
9858b464eb8Smec (void) page_retire_done(pp, PRD_SUCCESS);
9868b464eb8Smec return (0);
987db874c57Selowe } else if (toxic & PR_MCE) {
988db874c57Selowe PR_DEBUG(prd_mce);
989db874c57Selowe page_retire_destroy(pp);
9908b464eb8Smec (void) page_retire_done(pp, PRD_SUCCESS);
9918b464eb8Smec return (0);
992db874c57Selowe }
993db874c57Selowe
9948b464eb8Smec /*
9958b464eb8Smec * When page_retire_first_ue is set to zero and a UE occurs which is
9968b464eb8Smec * transient, it's possible that we clear some flags set by a second
9978b464eb8Smec * UE error on the page which occurs while the first is currently being
9988b464eb8Smec * handled and thus we need to handle the case where none of the above
9998b464eb8Smec * are set. In this instance, PR_UE_SCRUBBED should be set and thus
10008b464eb8Smec * we should execute the UE code above.
10018b464eb8Smec */
10028b464eb8Smec if (toxic & PR_UE_SCRUBBED) {
10038b464eb8Smec goto ue_error;
1004db874c57Selowe }
1005db874c57Selowe
1006db874c57Selowe /*
10078b464eb8Smec * It's impossible to get here.
1008db874c57Selowe */
10098b464eb8Smec panic("bad toxic flags 0x%x in page_retire_pp_finish\n", toxic);
10108b464eb8Smec return (0);
1011db874c57Selowe }
1012db874c57Selowe
1013db874c57Selowe /*
1014db874c57Selowe * page_retire() - the front door in to retire a page.
1015db874c57Selowe *
1016db874c57Selowe * Ideally, page_retire() would instantly retire the requested page.
1017db874c57Selowe * Unfortunately, some pages are locked or otherwise tied up and cannot be
10188b464eb8Smec * retired right away. We use the page capture logic to deal with this
10198b464eb8Smec * situation as it will continuously try to retire the page in the background
10208b464eb8Smec * if the first attempt fails. Success is determined by looking to see whether
10218b464eb8Smec * the page has been retired after the page_trycapture() attempt.
1022db874c57Selowe *
1023db874c57Selowe * Returns:
1024db874c57Selowe *
1025db874c57Selowe * - 0 on success,
1026db874c57Selowe * - EINVAL when the PA is whacko,
10272708b84eSelowe * - EIO if the page is already retired or already pending retirement, or
10282708b84eSelowe * - EAGAIN if the page could not be _immediately_ retired but is pending.
1029db874c57Selowe */
1030db874c57Selowe int
page_retire(uint64_t pa,uchar_t reason)1031db874c57Selowe page_retire(uint64_t pa, uchar_t reason)
1032db874c57Selowe {
1033db874c57Selowe page_t *pp;
1034db874c57Selowe
1035db874c57Selowe ASSERT(reason & PR_REASONS); /* there must be a reason */
1036db874c57Selowe ASSERT(!(reason & ~PR_REASONS)); /* but no other bits */
1037db874c57Selowe
1038db874c57Selowe pp = page_numtopp_nolock(mmu_btop(pa));
1039db874c57Selowe if (pp == NULL) {
1040db874c57Selowe PR_MESSAGE(CE_WARN, 1, "Cannot schedule clearing of error on"
1041db874c57Selowe " page 0x%08x.%08x; page is not relocatable memory", pa);
1042db874c57Selowe return (page_retire_done(pp, PRD_INVALID_PA));
1043db874c57Selowe }
1044db874c57Selowe if (PP_RETIRED(pp)) {
10452708b84eSelowe PR_DEBUG(prd_dup1);
1046db874c57Selowe return (page_retire_done(pp, PRD_DUPLICATE));
1047db874c57Selowe }
1048db874c57Selowe
10491a3c9a5aSVijay S Balakrishna if (memscrub_notify_func != NULL) {
10501a3c9a5aSVijay S Balakrishna (void) memscrub_notify_func(pa);
10511a3c9a5aSVijay S Balakrishna }
10521a3c9a5aSVijay S Balakrishna
10532708b84eSelowe if ((reason & PR_UE) && !PP_TOXIC(pp)) {
1054db874c57Selowe PR_MESSAGE(CE_NOTE, 1, "Scheduling clearing of error on"
1055db874c57Selowe " page 0x%08x.%08x", pa);
10562708b84eSelowe } else if (PP_PR_REQ(pp)) {
10572708b84eSelowe PR_DEBUG(prd_dup2);
10582708b84eSelowe return (page_retire_done(pp, PRD_DUPLICATE));
1059db874c57Selowe } else {
1060db874c57Selowe PR_MESSAGE(CE_NOTE, 1, "Scheduling removal of"
1061db874c57Selowe " page 0x%08x.%08x", pa);
1062db874c57Selowe }
1063db874c57Selowe
10648b464eb8Smec /* Avoid setting toxic bits in the first place */
10658b464eb8Smec if ((reason & (PR_FMA | PR_MCE)) && !(reason & PR_UE) &&
10668b464eb8Smec page_retire_limit()) {
10678b464eb8Smec return (page_retire_done(pp, PRD_LIMIT));
10688b464eb8Smec }
10698b464eb8Smec
10708b464eb8Smec if (MTBF(pr_calls, pr_mtbf)) {
10718b464eb8Smec page_settoxic(pp, reason);
1072704b9682SChristopher Baumbauer - Sun Microsystems - San Diego United States if (page_trycapture(pp, 0, CAPTURE_RETIRE, pp->p_vnode) == 0) {
10738b464eb8Smec PR_DEBUG(prd_prlocked);
10748b464eb8Smec } else {
10758b464eb8Smec PR_DEBUG(prd_prnotlocked);
10768b464eb8Smec }
1077db874c57Selowe } else {
1078db874c57Selowe PR_DEBUG(prd_prnotlocked);
1079db874c57Selowe }
1080db874c57Selowe
1081db874c57Selowe if (PP_RETIRED(pp)) {
1082db874c57Selowe PR_DEBUG(prd_prretired);
1083db874c57Selowe return (0);
1084db874c57Selowe } else {
10858b464eb8Smec cv_signal(&pc_cv);
1086db874c57Selowe PR_INCR_KSTAT(pr_failed);
1087db874c57Selowe
1088db874c57Selowe if (pp->p_toxic & PR_MSG) {
1089db874c57Selowe return (page_retire_done(pp, PRD_FAILED));
1090db874c57Selowe } else {
1091db874c57Selowe return (page_retire_done(pp, PRD_PENDING));
1092db874c57Selowe }
1093db874c57Selowe }
1094db874c57Selowe }
1095db874c57Selowe
1096db874c57Selowe /*
1097db874c57Selowe * Take a retired page off the retired-pages vnode and clear the toxic flags.
1098db874c57Selowe * If "free" is nonzero, lock it and put it back on the freelist. If "free"
1099db874c57Selowe * is zero, the caller already holds SE_EXCL lock so we simply unretire it
1100db874c57Selowe * and don't do anything else with it.
1101db874c57Selowe *
1102db874c57Selowe * Any unretire messages are printed from this routine.
1103db874c57Selowe *
1104db874c57Selowe * Returns 0 if page pp was unretired; else an error code.
11058b464eb8Smec *
11068b464eb8Smec * If flags is:
11078b464eb8Smec * PR_UNR_FREE - lock the page, clear the toxic flags and free it
11088b464eb8Smec * to the freelist.
11098b464eb8Smec * PR_UNR_TEMP - lock the page, unretire it, leave the toxic
11108b464eb8Smec * bits set as is and return it to the caller.
11118b464eb8Smec * PR_UNR_CLEAN - page is SE_EXCL locked, unretire it, clear the
11128b464eb8Smec * toxic flags and return it to caller as is.
1113db874c57Selowe */
1114db874c57Selowe int
page_unretire_pp(page_t * pp,int flags)11158b464eb8Smec page_unretire_pp(page_t *pp, int flags)
1116db874c57Selowe {
1117db874c57Selowe /*
1118db874c57Selowe * To be retired, a page has to be hashed onto the retired_pages vnode
1119db874c57Selowe * and have PR_RETIRED set in p_toxic.
1120db874c57Selowe */
11218b464eb8Smec if (flags == PR_UNR_CLEAN ||
11228b464eb8Smec page_try_reclaim_lock(pp, SE_EXCL, SE_RETIRED)) {
1123db874c57Selowe ASSERT(PAGE_EXCL(pp));
1124db874c57Selowe PR_DEBUG(prd_ulocked);
1125db874c57Selowe if (!PP_RETIRED(pp)) {
1126db874c57Selowe PR_DEBUG(prd_unotretired);
1127db874c57Selowe page_unlock(pp);
1128db874c57Selowe return (page_retire_done(pp, PRD_UNR_NOT));
1129db874c57Selowe }
1130db874c57Selowe
1131db874c57Selowe PR_MESSAGE(CE_NOTE, 1, "unretiring retired"
11324fc2445aSelowe " page 0x%08x.%08x", mmu_ptob((uint64_t)pp->p_pagenum));
1133db874c57Selowe if (pp->p_toxic & PR_FMA) {
1134db874c57Selowe PR_DECR_KSTAT(pr_fma);
1135db874c57Selowe } else if (pp->p_toxic & PR_UE) {
1136db874c57Selowe PR_DECR_KSTAT(pr_ue);
1137db874c57Selowe } else {
1138db874c57Selowe PR_DECR_KSTAT(pr_mce);
1139db874c57Selowe }
1140db874c57Selowe
11418b464eb8Smec if (flags == PR_UNR_TEMP)
11428b464eb8Smec page_clrtoxic(pp, PR_RETIRED);
11438b464eb8Smec else
11448b464eb8Smec page_clrtoxic(pp, PR_TOXICFLAGS);
11458b464eb8Smec
11468b464eb8Smec if (flags == PR_UNR_FREE) {
1147db874c57Selowe PR_DEBUG(prd_udestroy);
1148db874c57Selowe page_destroy(pp, 0);
1149db874c57Selowe } else {
1150db874c57Selowe PR_DEBUG(prd_uhashout);
1151db874c57Selowe page_hashout(pp, NULL);
1152db874c57Selowe }
1153db874c57Selowe
1154db874c57Selowe mutex_enter(&freemem_lock);
1155db874c57Selowe availrmem++;
1156db874c57Selowe mutex_exit(&freemem_lock);
1157db874c57Selowe
1158db874c57Selowe PR_DEBUG(prd_uunretired);
1159db874c57Selowe PR_DECR_KSTAT(pr_retired);
1160db874c57Selowe PR_INCR_KSTAT(pr_unretired);
1161db874c57Selowe return (page_retire_done(pp, PRD_UNR_SUCCESS));
1162db874c57Selowe }
1163db874c57Selowe PR_DEBUG(prd_unotlocked);
1164db874c57Selowe return (page_retire_done(pp, PRD_UNR_CANTLOCK));
1165db874c57Selowe }
1166db874c57Selowe
1167db874c57Selowe /*
1168db874c57Selowe * Return a page to service by moving it from the retired_pages vnode
1169db874c57Selowe * onto the freelist.
1170db874c57Selowe *
1171db874c57Selowe * Called from mmioctl_page_retire() on behalf of the FMA DE.
1172db874c57Selowe *
1173db874c57Selowe * Returns:
1174db874c57Selowe *
1175db874c57Selowe * - 0 if the page is unretired,
1176db874c57Selowe * - EAGAIN if the pp can not be locked,
1177db874c57Selowe * - EINVAL if the PA is whacko, and
11782708b84eSelowe * - EIO if the pp is not retired.
1179db874c57Selowe */
1180db874c57Selowe int
page_unretire(uint64_t pa)1181db874c57Selowe page_unretire(uint64_t pa)
1182db874c57Selowe {
1183db874c57Selowe page_t *pp;
1184db874c57Selowe
1185db874c57Selowe pp = page_numtopp_nolock(mmu_btop(pa));
1186db874c57Selowe if (pp == NULL) {
1187db874c57Selowe return (page_retire_done(pp, PRD_INVALID_PA));
1188db874c57Selowe }
1189db874c57Selowe
11908b464eb8Smec return (page_unretire_pp(pp, PR_UNR_FREE));
1191db874c57Selowe }
1192db874c57Selowe
1193db874c57Selowe /*
1194db874c57Selowe * Test a page to see if it is retired. If errors is non-NULL, the toxic
1195db874c57Selowe * bits of the page are returned. Returns 0 on success, error code on failure.
1196db874c57Selowe */
1197db874c57Selowe int
page_retire_check_pp(page_t * pp,uint64_t * errors)1198db874c57Selowe page_retire_check_pp(page_t *pp, uint64_t *errors)
1199db874c57Selowe {
1200db874c57Selowe int rc;
1201db874c57Selowe
1202db874c57Selowe if (PP_RETIRED(pp)) {
1203db874c57Selowe PR_DEBUG(prd_checkhit);
1204db874c57Selowe rc = 0;
12052708b84eSelowe } else if (PP_PR_REQ(pp)) {
12062708b84eSelowe PR_DEBUG(prd_checkmiss_pend);
1207db874c57Selowe rc = EAGAIN;
12082708b84eSelowe } else {
12092708b84eSelowe PR_DEBUG(prd_checkmiss_noerr);
12102708b84eSelowe rc = EIO;
1211db874c57Selowe }
1212db874c57Selowe
1213db874c57Selowe /*
1214bbf21555SRichard Lowe * We have magically arranged the bit values returned to fmd(8)
1215db874c57Selowe * to line up with the FMA, MCE, and UE bits of the page_t.
1216db874c57Selowe */
1217db874c57Selowe if (errors) {
1218db874c57Selowe uint64_t toxic = (uint64_t)(pp->p_toxic & PR_ERRMASK);
1219db874c57Selowe if (toxic & PR_UE_SCRUBBED) {
1220db874c57Selowe toxic &= ~PR_UE_SCRUBBED;
1221db874c57Selowe toxic |= PR_UE;
1222db874c57Selowe }
1223db874c57Selowe *errors = toxic;
1224db874c57Selowe }
1225db874c57Selowe
1226db874c57Selowe return (rc);
1227db874c57Selowe }
1228db874c57Selowe
1229db874c57Selowe /*
1230db874c57Selowe * Test to see if the page_t for a given PA is retired, and return the
1231db874c57Selowe * hardware errors we have seen on the page if requested.
1232db874c57Selowe *
1233db874c57Selowe * Called from mmioctl_page_retire on behalf of the FMA DE.
1234db874c57Selowe *
1235db874c57Selowe * Returns:
1236db874c57Selowe *
1237db874c57Selowe * - 0 if the page is retired,
12382708b84eSelowe * - EIO if the page is not retired and has no errors,
12392708b84eSelowe * - EAGAIN if the page is not retired but is pending; and
1240db874c57Selowe * - EINVAL if the PA is whacko.
1241db874c57Selowe */
1242db874c57Selowe int
page_retire_check(uint64_t pa,uint64_t * errors)1243db874c57Selowe page_retire_check(uint64_t pa, uint64_t *errors)
1244db874c57Selowe {
1245db874c57Selowe page_t *pp;
1246db874c57Selowe
1247db874c57Selowe if (errors) {
1248db874c57Selowe *errors = 0;
1249db874c57Selowe }
1250db874c57Selowe
1251db874c57Selowe pp = page_numtopp_nolock(mmu_btop(pa));
1252db874c57Selowe if (pp == NULL) {
1253db874c57Selowe return (page_retire_done(pp, PRD_INVALID_PA));
1254db874c57Selowe }
1255db874c57Selowe
1256db874c57Selowe return (page_retire_check_pp(pp, errors));
1257db874c57Selowe }
1258db874c57Selowe
1259db874c57Selowe /*
1260db874c57Selowe * Page retire self-test. For now, it always returns 0.
1261db874c57Selowe */
1262db874c57Selowe int
page_retire_test(void)1263db874c57Selowe page_retire_test(void)
1264db874c57Selowe {
1265db874c57Selowe page_t *first, *pp, *cpp, *cpp2, *lpp;
1266db874c57Selowe
1267db874c57Selowe /*
1268db874c57Selowe * Tests the corner case where a large page can't be retired
1269db874c57Selowe * because one of the constituent pages is locked. We mark
1270db874c57Selowe * one page to be retired and try to retire it, and mark the
1271db874c57Selowe * other page to be retired but don't try to retire it, so
1272db874c57Selowe * that page_unlock() in the failure path will recurse and try
1273db874c57Selowe * to retire THAT page. This is the worst possible situation
1274db874c57Selowe * we can get ourselves into.
1275db874c57Selowe */
1276db874c57Selowe memsegs_lock(0);
1277db874c57Selowe pp = first = page_first();
1278db874c57Selowe do {
1279db874c57Selowe if (pp->p_szc && PP_PAGEROOT(pp) == pp) {
1280db874c57Selowe cpp = pp + 1;
1281db874c57Selowe lpp = PP_ISFREE(pp)? pp : pp + 2;
1282db874c57Selowe cpp2 = pp + 3;
1283db874c57Selowe if (!page_trylock(lpp, pp == lpp? SE_EXCL : SE_SHARED))
1284db874c57Selowe continue;
1285db874c57Selowe if (!page_trylock(cpp, SE_EXCL)) {
1286db874c57Selowe page_unlock(lpp);
1287db874c57Selowe continue;
1288db874c57Selowe }
12898b464eb8Smec
12908b464eb8Smec /* fails */
12918b464eb8Smec (void) page_retire(ptob(cpp->p_pagenum), PR_FMA);
12928b464eb8Smec
1293db874c57Selowe page_unlock(lpp);
12948b464eb8Smec page_unlock(cpp);
12958b464eb8Smec (void) page_retire(ptob(cpp->p_pagenum), PR_FMA);
12968b464eb8Smec (void) page_retire(ptob(cpp2->p_pagenum), PR_FMA);
1297db874c57Selowe }
1298db874c57Selowe } while ((pp = page_next(pp)) != first);
1299db874c57Selowe memsegs_unlock(0);
1300db874c57Selowe
1301db874c57Selowe return (0);
1302db874c57Selowe }
1303