1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 * Copyright (c) 2016 by Delphix. All rights reserved.
25 * Copyright 2018 Joyent, Inc.
26 */
27
28 /*
29 * Page Retire - Big Theory Statement.
30 *
31 * This file handles removing sections of faulty memory from use when the
32 * user land FMA Diagnosis Engine requests that a page be removed or when
33 * a CE or UE is detected by the hardware.
34 *
35 * In the bad old days, the kernel side of Page Retire did a lot of the work
36 * on its own. Now, with the DE keeping track of errors, the kernel side is
37 * rather simple minded on most platforms.
38 *
39 * Errors are all reflected to the DE, and after digesting the error and
40 * looking at all previously reported errors, the DE decides what should
41 * be done about the current error. If the DE wants a particular page to
42 * be retired, then the kernel page retire code is invoked via an ioctl.
43 * On non-FMA platforms, the ue_drain and ce_drain paths ends up calling
44 * page retire to handle the error. Since page retire is just a simple
45 * mechanism it doesn't need to differentiate between the different callers.
46 *
47 * The p_toxic field in the page_t is used to indicate which errors have
48 * occurred and what action has been taken on a given page. Because errors are
49 * reported without regard to the locked state of a page, no locks are used
50 * to SET the error bits in p_toxic. However, in order to clear the error
51 * bits, the page_t must be held exclusively locked.
52 *
53 * When page_retire() is called, it must be able to acquire locks, sleep, etc.
54 * It must not be called from high-level interrupt context.
55 *
56 * Depending on how the requested page is being used at the time of the retire
57 * request (and on the availability of sufficient system resources), the page
58 * may be retired immediately, or just marked for retirement later. For
59 * example, locked pages are marked, while free pages are retired. Multiple
60 * requests may be made to retire the same page, although there is no need
61 * to: once the p_toxic flags are set, the page will be retired as soon as it
62 * can be exclusively locked.
63 *
64 * The retire mechanism is driven centrally out of page_unlock(). To expedite
65 * the retirement of pages, further requests for SE_SHARED locks are denied
66 * as long as a page retirement is pending. In addition, as long as pages are
67 * pending retirement a background thread runs periodically trying to retire
68 * those pages. Pages which could not be retired while the system is running
69 * are scrubbed prior to rebooting to avoid latent errors on the next boot.
70 *
71 * UE pages without persistent errors are scrubbed and returned to service.
72 * Recidivist pages, as well as FMA-directed requests for retirement, result
73 * in the page being taken out of service. Once the decision is made to take
74 * a page out of service, the page is cleared, hashed onto the retired_pages
75 * vnode, marked as retired, and it is unlocked. No other requesters (except
76 * for unretire) are allowed to lock retired pages.
77 *
78 * The public routines return (sadly) 0 if they worked and a non-zero error
79 * value if something went wrong. This is done for the ioctl side of the
80 * world to allow errors to be reflected all the way out to user land. The
81 * non-zero values are explained in comments atop each function.
82 */
83
84 /*
85 * Things to fix:
86 *
87 * 1. Trying to retire non-relocatable kvp pages may result in a
88 * quagmire. This is because seg_kmem() no longer keeps its pages locked,
89 * and calls page_lookup() in the free path; since kvp pages are modified
90 * and don't have a usable backing store, page_retire() can't do anything
91 * with them, and we'll keep denying the lock to seg_kmem_free() in a
92 * vicious cycle. To prevent that, we don't deny locks to kvp pages, and
93 * hence only try to retire a page from page_unlock() in the free path.
94 * Since most kernel pages are indefinitely held anyway, and don't
95 * participate in I/O, this is of little consequence.
96 *
97 * 2. Low memory situations will be interesting. If we don't have
98 * enough memory for page_relocate() to succeed, we won't be able to
99 * retire dirty pages; nobody will be able to push them out to disk
100 * either, since we aggressively deny the page lock. We could change
101 * fsflush so it can recognize this situation, grab the lock, and push
102 * the page out, where we'll catch it in the free path and retire it.
103 *
104 * 3. Beware of places that have code like this in them:
105 *
106 * if (! page_tryupgrade(pp)) {
107 * page_unlock(pp);
108 * while (! page_lock(pp, SE_EXCL, NULL, P_RECLAIM)) {
109 * / *NOTHING* /
110 * }
111 * }
112 * page_free(pp);
113 *
114 * The problem is that pp can change identity right after the
115 * page_unlock() call. In particular, page_retire() can step in
116 * there, change pp's identity, and hash pp onto the retired_vnode.
117 *
118 * Of course, other functions besides page_retire() can have the
119 * same effect. A kmem reader can waltz by, set up a mapping to the
120 * page, and then unlock the page. Page_free() will then go castors
121 * up. So if anybody is doing this, it's already a bug.
122 *
123 * 4. mdboot()'s call into page_retire_mdboot() should probably be
124 * moved lower. Where the call is made now, we can get into trouble
125 * by scrubbing a kernel page that is then accessed later.
126 */
127
128 #include <sys/types.h>
129 #include <sys/param.h>
130 #include <sys/systm.h>
131 #include <sys/mman.h>
132 #include <sys/vnode.h>
133 #include <sys/vfs_opreg.h>
134 #include <sys/cmn_err.h>
135 #include <sys/ksynch.h>
136 #include <sys/thread.h>
137 #include <sys/disp.h>
138 #include <sys/ontrap.h>
139 #include <sys/vmsystm.h>
140 #include <sys/mem_config.h>
141 #include <sys/atomic.h>
142 #include <sys/callb.h>
143 #include <sys/kobj.h>
144 #include <vm/page.h>
145 #include <vm/vm_dep.h>
146 #include <vm/as.h>
147 #include <vm/hat.h>
148 #include <vm/seg_kmem.h>
149
150 /*
151 * vnode for all pages which are retired from the VM system;
152 */
153 vnode_t *retired_pages;
154
155 static int page_retire_pp_finish(page_t *, void *, uint_t);
156
157 /*
158 * Make a list of all of the pages that have been marked for retirement
159 * but are not yet retired. At system shutdown, we will scrub all of the
160 * pages in the list in case there are outstanding UEs. Then, we
161 * cross-check this list against the number of pages that are yet to be
162 * retired, and if we find inconsistencies, we scan every page_t in the
163 * whole system looking for any pages that need to be scrubbed for UEs.
164 * The background thread also uses this queue to determine which pages
165 * it should keep trying to retire.
166 */
167 #ifdef DEBUG
168 #define PR_PENDING_QMAX 32
169 #else /* DEBUG */
170 #define PR_PENDING_QMAX 256
171 #endif /* DEBUG */
172 page_t *pr_pending_q[PR_PENDING_QMAX];
173 kmutex_t pr_q_mutex;
174
175 /*
176 * Page retire global kstats
177 */
178 struct page_retire_kstat {
179 kstat_named_t pr_retired;
180 kstat_named_t pr_requested;
181 kstat_named_t pr_requested_free;
182 kstat_named_t pr_enqueue_fail;
183 kstat_named_t pr_dequeue_fail;
184 kstat_named_t pr_pending;
185 kstat_named_t pr_pending_kas;
186 kstat_named_t pr_failed;
187 kstat_named_t pr_failed_kernel;
188 kstat_named_t pr_limit;
189 kstat_named_t pr_limit_exceeded;
190 kstat_named_t pr_fma;
191 kstat_named_t pr_mce;
192 kstat_named_t pr_ue;
193 kstat_named_t pr_ue_cleared_retire;
194 kstat_named_t pr_ue_cleared_free;
195 kstat_named_t pr_ue_persistent;
196 kstat_named_t pr_unretired;
197 };
198
199 static struct page_retire_kstat page_retire_kstat = {
200 { "pages_retired", KSTAT_DATA_UINT64},
201 { "pages_retire_request", KSTAT_DATA_UINT64},
202 { "pages_retire_request_free", KSTAT_DATA_UINT64},
203 { "pages_notenqueued", KSTAT_DATA_UINT64},
204 { "pages_notdequeued", KSTAT_DATA_UINT64},
205 { "pages_pending", KSTAT_DATA_UINT64},
206 { "pages_pending_kas", KSTAT_DATA_UINT64},
207 { "pages_deferred", KSTAT_DATA_UINT64},
208 { "pages_deferred_kernel", KSTAT_DATA_UINT64},
209 { "pages_limit", KSTAT_DATA_UINT64},
210 { "pages_limit_exceeded", KSTAT_DATA_UINT64},
211 { "pages_fma", KSTAT_DATA_UINT64},
212 { "pages_multiple_ce", KSTAT_DATA_UINT64},
213 { "pages_ue", KSTAT_DATA_UINT64},
214 { "pages_ue_cleared_retired", KSTAT_DATA_UINT64},
215 { "pages_ue_cleared_freed", KSTAT_DATA_UINT64},
216 { "pages_ue_persistent", KSTAT_DATA_UINT64},
217 { "pages_unretired", KSTAT_DATA_UINT64},
218 };
219
220 static kstat_t *page_retire_ksp = NULL;
221
222 #define PR_INCR_KSTAT(stat) \
223 atomic_inc_64(&(page_retire_kstat.stat.value.ui64))
224 #define PR_DECR_KSTAT(stat) \
225 atomic_dec_64(&(page_retire_kstat.stat.value.ui64))
226
227 #define PR_KSTAT_RETIRED_CE (page_retire_kstat.pr_mce.value.ui64)
228 #define PR_KSTAT_RETIRED_FMA (page_retire_kstat.pr_fma.value.ui64)
229 #define PR_KSTAT_RETIRED_NOTUE (PR_KSTAT_RETIRED_CE + PR_KSTAT_RETIRED_FMA)
230 #define PR_KSTAT_PENDING (page_retire_kstat.pr_pending.value.ui64)
231 #define PR_KSTAT_PENDING_KAS (page_retire_kstat.pr_pending_kas.value.ui64)
232 #define PR_KSTAT_EQFAIL (page_retire_kstat.pr_enqueue_fail.value.ui64)
233 #define PR_KSTAT_DQFAIL (page_retire_kstat.pr_dequeue_fail.value.ui64)
234
235 /*
236 * page retire kstats to list all retired pages
237 */
238 static int pr_list_kstat_update(kstat_t *ksp, int rw);
239 static int pr_list_kstat_snapshot(kstat_t *ksp, void *buf, int rw);
240 kmutex_t pr_list_kstat_mutex;
241
242 /*
243 * Limit the number of multiple CE page retires.
244 * The default is 0.1% of physmem, or 1 in 1000 pages. This is set in
245 * basis points, where 100 basis points equals one percent.
246 */
247 #define MCE_BPT 10
248 uint64_t max_pages_retired_bps = MCE_BPT;
249 #define PAGE_RETIRE_LIMIT ((physmem * max_pages_retired_bps) / 10000)
250
251 /*
252 * Control over the verbosity of page retirement.
253 *
254 * When set to zero (the default), no messages will be printed.
255 * When set to one, summary messages will be printed.
256 * When set > one, all messages will be printed.
257 *
258 * A value of one will trigger detailed messages for retirement operations,
259 * and is intended as a platform tunable for processors where FMA's DE does
260 * not run (e.g., spitfire). Values > one are intended for debugging only.
261 */
262 int page_retire_messages = 0;
263
264 /*
265 * Control whether or not we return scrubbed UE pages to service.
266 * By default we do not since FMA wants to run its diagnostics first
267 * and then ask us to unretire the page if it passes. Non-FMA platforms
268 * may set this to zero so we will only retire recidivist pages. It should
269 * not be changed by the user.
270 */
271 int page_retire_first_ue = 1;
272
273 /*
274 * Master enable for page retire. This prevents a CE or UE early in boot
275 * from trying to retire a page before page_retire_init() has finished
276 * setting things up. This is internal only and is not a tunable!
277 */
278 static int pr_enable = 0;
279
280 static void (*memscrub_notify_func)(uint64_t);
281
282 #ifdef DEBUG
283 struct page_retire_debug {
284 int prd_dup1;
285 int prd_dup2;
286 int prd_qdup;
287 int prd_noaction;
288 int prd_queued;
289 int prd_notqueued;
290 int prd_dequeue;
291 int prd_top;
292 int prd_locked;
293 int prd_reloc;
294 int prd_relocfail;
295 int prd_mod;
296 int prd_mod_late;
297 int prd_kern;
298 int prd_free;
299 int prd_noreclaim;
300 int prd_hashout;
301 int prd_fma;
302 int prd_uescrubbed;
303 int prd_uenotscrubbed;
304 int prd_mce;
305 int prd_prlocked;
306 int prd_prnotlocked;
307 int prd_prretired;
308 int prd_ulocked;
309 int prd_unotretired;
310 int prd_udestroy;
311 int prd_uhashout;
312 int prd_uunretired;
313 int prd_unotlocked;
314 int prd_checkhit;
315 int prd_checkmiss_pend;
316 int prd_checkmiss_noerr;
317 int prd_tctop;
318 int prd_tclocked;
319 int prd_hunt;
320 int prd_dohunt;
321 int prd_earlyhunt;
322 int prd_latehunt;
323 int prd_nofreedemote;
324 int prd_nodemote;
325 int prd_demoted;
326 } pr_debug;
327
328 #define PR_DEBUG(foo) ((pr_debug.foo)++)
329
330 /*
331 * A type histogram. We record the incidence of the various toxic
332 * flag combinations along with the interesting page attributes. The
333 * goal is to get as many combinations as we can while driving all
334 * pr_debug values nonzero (indicating we've exercised all possible
335 * code paths across all possible page types). Not all combinations
336 * will make sense -- e.g. PRT_MOD|PRT_KERNEL.
337 *
338 * pr_type offset bit encoding (when examining with a debugger):
339 *
340 * PRT_NAMED - 0x4
341 * PRT_KERNEL - 0x8
342 * PRT_FREE - 0x10
343 * PRT_MOD - 0x20
344 * PRT_FMA - 0x0
345 * PRT_MCE - 0x40
346 * PRT_UE - 0x80
347 */
348
349 #define PRT_NAMED 0x01
350 #define PRT_KERNEL 0x02
351 #define PRT_FREE 0x04
352 #define PRT_MOD 0x08
353 #define PRT_FMA 0x00 /* yes, this is not a mistake */
354 #define PRT_MCE 0x10
355 #define PRT_UE 0x20
356 #define PRT_ALL 0x3F
357
358 int pr_types[PRT_ALL+1];
359
360 #define PR_TYPES(pp) { \
361 int whichtype = 0; \
362 if (pp->p_vnode) \
363 whichtype |= PRT_NAMED; \
364 if (PP_ISKAS(pp)) \
365 whichtype |= PRT_KERNEL; \
366 if (PP_ISFREE(pp)) \
367 whichtype |= PRT_FREE; \
368 if (hat_ismod(pp)) \
369 whichtype |= PRT_MOD; \
370 if (pp->p_toxic & PR_UE) \
371 whichtype |= PRT_UE; \
372 if (pp->p_toxic & PR_MCE) \
373 whichtype |= PRT_MCE; \
374 pr_types[whichtype]++; \
375 }
376
377 int recl_calls;
378 int recl_mtbf = 3;
379 int reloc_calls;
380 int reloc_mtbf = 7;
381 int pr_calls;
382 int pr_mtbf = 15;
383
384 #define MTBF(v, f) (((++(v)) & (f)) != (f))
385
386 #else /* DEBUG */
387
388 #define PR_DEBUG(foo) /* nothing */
389 #define PR_TYPES(foo) /* nothing */
390 #define MTBF(v, f) (1)
391
392 #endif /* DEBUG */
393
394 /*
395 * page_retire_done() - completion processing
396 *
397 * Used by the page_retire code for common completion processing.
398 * It keeps track of how many times a given result has happened,
399 * and writes out an occasional message.
400 *
401 * May be called with a NULL pp (PRD_INVALID_PA case).
402 */
403 #define PRD_INVALID_KEY -1
404 #define PRD_SUCCESS 0
405 #define PRD_PENDING 1
406 #define PRD_FAILED 2
407 #define PRD_DUPLICATE 3
408 #define PRD_INVALID_PA 4
409 #define PRD_LIMIT 5
410 #define PRD_UE_SCRUBBED 6
411 #define PRD_UNR_SUCCESS 7
412 #define PRD_UNR_CANTLOCK 8
413 #define PRD_UNR_NOT 9
414
415 typedef struct page_retire_op {
416 int pr_key; /* one of the PRD_* defines from above */
417 int pr_count; /* How many times this has happened */
418 int pr_retval; /* return value */
419 int pr_msglvl; /* message level - when to print */
420 char *pr_message; /* Cryptic message for field service */
421 } page_retire_op_t;
422
423 static page_retire_op_t page_retire_ops[] = {
424 /* key count retval msglvl message */
425 {PRD_SUCCESS, 0, 0, 1,
426 "Page 0x%08x.%08x removed from service"},
427 {PRD_PENDING, 0, EAGAIN, 2,
428 "Page 0x%08x.%08x will be retired on free"},
429 {PRD_FAILED, 0, EAGAIN, 0, NULL},
430 {PRD_DUPLICATE, 0, EIO, 2,
431 "Page 0x%08x.%08x already retired or pending"},
432 {PRD_INVALID_PA, 0, EINVAL, 2,
433 "PA 0x%08x.%08x is not a relocatable page"},
434 {PRD_LIMIT, 0, 0, 1,
435 "Page 0x%08x.%08x not retired due to limit exceeded"},
436 {PRD_UE_SCRUBBED, 0, 0, 1,
437 "Previously reported error on page 0x%08x.%08x cleared"},
438 {PRD_UNR_SUCCESS, 0, 0, 1,
439 "Page 0x%08x.%08x returned to service"},
440 {PRD_UNR_CANTLOCK, 0, EAGAIN, 2,
441 "Page 0x%08x.%08x could not be unretired"},
442 {PRD_UNR_NOT, 0, EIO, 2,
443 "Page 0x%08x.%08x is not retired"},
444 {PRD_INVALID_KEY, 0, 0, 0, NULL} /* MUST BE LAST! */
445 };
446
447 /*
448 * print a message if page_retire_messages is true.
449 */
450 #define PR_MESSAGE(debuglvl, msglvl, msg, pa) \
451 { \
452 uint64_t p = (uint64_t)pa; \
453 if (page_retire_messages >= msglvl && msg != NULL) { \
454 cmn_err(debuglvl, msg, \
455 (uint32_t)(p >> 32), (uint32_t)p); \
456 } \
457 }
458
459 /*
460 * Note that multiple bits may be set in a single settoxic operation.
461 * May be called without the page locked.
462 */
463 void
page_settoxic(page_t * pp,uchar_t bits)464 page_settoxic(page_t *pp, uchar_t bits)
465 {
466 atomic_or_8(&pp->p_toxic, bits);
467 }
468
469 /*
470 * Note that multiple bits may cleared in a single clrtoxic operation.
471 * Must be called with the page exclusively locked to prevent races which
472 * may attempt to retire a page without any toxic bits set.
473 * Note that the PR_CAPTURE bit can be cleared without the exclusive lock
474 * being held as there is a separate mutex which protects that bit.
475 */
476 void
page_clrtoxic(page_t * pp,uchar_t bits)477 page_clrtoxic(page_t *pp, uchar_t bits)
478 {
479 ASSERT((bits & PR_CAPTURE) || PAGE_EXCL(pp));
480 atomic_and_8(&pp->p_toxic, ~bits);
481 }
482
483 /*
484 * Prints any page retire messages to the user, and decides what
485 * error code is appropriate for the condition reported.
486 */
487 static int
page_retire_done(page_t * pp,int code)488 page_retire_done(page_t *pp, int code)
489 {
490 page_retire_op_t *prop;
491 uint64_t pa = 0;
492 int i;
493
494 if (pp != NULL) {
495 pa = mmu_ptob((uint64_t)pp->p_pagenum);
496 }
497
498 prop = NULL;
499 for (i = 0; page_retire_ops[i].pr_key != PRD_INVALID_KEY; i++) {
500 if (page_retire_ops[i].pr_key == code) {
501 prop = &page_retire_ops[i];
502 break;
503 }
504 }
505
506 #ifdef DEBUG
507 if (page_retire_ops[i].pr_key == PRD_INVALID_KEY) {
508 cmn_err(CE_PANIC, "page_retire_done: Invalid opcode %d", code);
509 }
510 #endif
511
512 ASSERT(prop->pr_key == code);
513
514 prop->pr_count++;
515
516 PR_MESSAGE(CE_NOTE, prop->pr_msglvl, prop->pr_message, pa);
517 if (pp != NULL) {
518 page_settoxic(pp, PR_MSG);
519 }
520
521 return (prop->pr_retval);
522 }
523
524 /*
525 * Act like page_destroy(), but instead of freeing the page, hash it onto
526 * the retired_pages vnode, and mark it retired.
527 *
528 * For fun, we try to scrub the page until it's squeaky clean.
529 * availrmem is adjusted here.
530 */
531 static void
page_retire_destroy(page_t * pp)532 page_retire_destroy(page_t *pp)
533 {
534 u_offset_t off = (u_offset_t)((uintptr_t)pp);
535
536 ASSERT(PAGE_EXCL(pp));
537 ASSERT(!PP_ISFREE(pp));
538 ASSERT(pp->p_szc == 0);
539 ASSERT(!hat_page_is_mapped(pp));
540 ASSERT(!pp->p_vnode);
541
542 page_clr_all_props(pp);
543 pagescrub(pp, 0, MMU_PAGESIZE);
544
545 pp->p_next = NULL;
546 pp->p_prev = NULL;
547 if (page_hashin(pp, retired_pages, off, NULL) == 0) {
548 cmn_err(CE_PANIC, "retired page %p hashin failed", (void *)pp);
549 }
550
551 page_settoxic(pp, PR_RETIRED);
552 PR_INCR_KSTAT(pr_retired);
553
554 if (pp->p_toxic & PR_FMA) {
555 PR_INCR_KSTAT(pr_fma);
556 } else if (pp->p_toxic & PR_UE) {
557 PR_INCR_KSTAT(pr_ue);
558 } else {
559 PR_INCR_KSTAT(pr_mce);
560 }
561
562 mutex_enter(&freemem_lock);
563 availrmem--;
564 mutex_exit(&freemem_lock);
565
566 page_unlock(pp);
567 }
568
569 /*
570 * Check whether the number of pages which have been retired already exceeds
571 * the maximum allowable percentage of memory which may be retired.
572 *
573 * Returns 1 if the limit has been exceeded.
574 */
575 static int
page_retire_limit(void)576 page_retire_limit(void)
577 {
578 if (PR_KSTAT_RETIRED_NOTUE >= (uint64_t)PAGE_RETIRE_LIMIT) {
579 PR_INCR_KSTAT(pr_limit_exceeded);
580 return (1);
581 }
582
583 return (0);
584 }
585
586 #define MSG_DM "Data Mismatch occurred at PA 0x%08x.%08x" \
587 "[ 0x%x != 0x%x ] while attempting to clear previously " \
588 "reported error; page removed from service"
589
590 #define MSG_UE "Uncorrectable Error occurred at PA 0x%08x.%08x while " \
591 "attempting to clear previously reported error; page removed " \
592 "from service"
593
594 /*
595 * Attempt to clear a UE from a page.
596 * Returns 1 if the error has been successfully cleared.
597 */
598 static int
page_clear_transient_ue(page_t * pp)599 page_clear_transient_ue(page_t *pp)
600 {
601 caddr_t kaddr;
602 uint8_t rb, wb;
603 uint64_t pa;
604 uint32_t pa_hi, pa_lo;
605 on_trap_data_t otd;
606 int errors;
607 int i;
608
609 ASSERT(PAGE_EXCL(pp));
610 ASSERT(PP_PR_REQ(pp));
611 ASSERT(pp->p_szc == 0);
612 ASSERT(!hat_page_is_mapped(pp));
613
614 /*
615 * Clear the page and attempt to clear the UE. If we trap
616 * on the next access to the page, we know the UE has recurred.
617 */
618 pagescrub(pp, 0, PAGESIZE);
619
620 /*
621 * Map the page and write a bunch of bit patterns to compare
622 * what we wrote with what we read back. This isn't a perfect
623 * test but it should be good enough to catch most of the
624 * recurring UEs. If this fails to catch a recurrent UE, we'll
625 * retire the page the next time we see a UE on the page.
626 */
627 kaddr = ppmapin(pp, PROT_READ|PROT_WRITE, (caddr_t)-1);
628
629 pa = ptob((uint64_t)page_pptonum(pp));
630 pa_hi = (uint32_t)(pa >> 32);
631 pa_lo = (uint32_t)pa;
632
633 /*
634 * Disable preemption to prevent the off chance that
635 * we migrate while in the middle of running through
636 * the bit pattern and run on a different processor
637 * than what we started on.
638 */
639 kpreempt_disable();
640
641 /*
642 * Fill the page with each (0x00 - 0xFF] bit pattern, flushing
643 * the cache in between reading and writing. We do this under
644 * on_trap() protection to avoid recursion.
645 */
646 if (on_trap(&otd, OT_DATA_EC)) {
647 PR_MESSAGE(CE_WARN, 1, MSG_UE, pa);
648 errors = 1;
649 } else {
650 errors = 0;
651 for (wb = 0xff; wb > 0; wb--) {
652 for (i = 0; i < PAGESIZE; i++) {
653 kaddr[i] = wb;
654 }
655
656 sync_data_memory(kaddr, PAGESIZE);
657
658 for (i = 0; i < PAGESIZE; i++) {
659 rb = kaddr[i];
660 if (rb != wb) {
661 /*
662 * We had a mismatch without a trap.
663 * Uh-oh. Something is really wrong
664 * with this system.
665 */
666 if (page_retire_messages) {
667 cmn_err(CE_WARN, MSG_DM,
668 pa_hi, pa_lo, rb, wb);
669 }
670 errors = 1;
671 goto out; /* double break */
672 }
673 }
674 }
675 }
676 out:
677 no_trap();
678 kpreempt_enable();
679 ppmapout(kaddr);
680
681 return (errors ? 0 : 1);
682 }
683
684 /*
685 * Try to clear a page_t with a single UE. If the UE was transient, it is
686 * returned to service, and we return 1. Otherwise we return 0 meaning
687 * that further processing is required to retire the page.
688 */
689 static int
page_retire_transient_ue(page_t * pp)690 page_retire_transient_ue(page_t *pp)
691 {
692 ASSERT(PAGE_EXCL(pp));
693 ASSERT(!hat_page_is_mapped(pp));
694
695 /*
696 * If this page is a repeat offender, retire it under the
697 * "two strikes and you're out" rule. The caller is responsible
698 * for scrubbing the page to try to clear the error.
699 */
700 if (pp->p_toxic & PR_UE_SCRUBBED) {
701 PR_INCR_KSTAT(pr_ue_persistent);
702 return (0);
703 }
704
705 if (page_clear_transient_ue(pp)) {
706 /*
707 * We set the PR_SCRUBBED_UE bit; if we ever see this
708 * page again, we will retire it, no questions asked.
709 */
710 page_settoxic(pp, PR_UE_SCRUBBED);
711
712 if (page_retire_first_ue) {
713 PR_INCR_KSTAT(pr_ue_cleared_retire);
714 return (0);
715 } else {
716 PR_INCR_KSTAT(pr_ue_cleared_free);
717
718 page_clrtoxic(pp, PR_UE | PR_MCE | PR_MSG);
719
720 /* LINTED: CONSTCOND */
721 VN_DISPOSE(pp, B_FREE, 1, kcred);
722 return (1);
723 }
724 }
725
726 PR_INCR_KSTAT(pr_ue_persistent);
727 return (0);
728 }
729
730 /*
731 * Update the statistics dynamically when our kstat is read.
732 */
733 static int
page_retire_kstat_update(kstat_t * ksp,int rw)734 page_retire_kstat_update(kstat_t *ksp, int rw)
735 {
736 struct page_retire_kstat *pr;
737
738 if (ksp == NULL)
739 return (EINVAL);
740
741 switch (rw) {
742
743 case KSTAT_READ:
744 pr = (struct page_retire_kstat *)ksp->ks_data;
745 ASSERT(pr == &page_retire_kstat);
746 pr->pr_limit.value.ui64 = PAGE_RETIRE_LIMIT;
747 return (0);
748
749 case KSTAT_WRITE:
750 return (EACCES);
751
752 default:
753 return (EINVAL);
754 }
755 /*NOTREACHED*/
756 }
757
758 static int
pr_list_kstat_update(kstat_t * ksp,int rw)759 pr_list_kstat_update(kstat_t *ksp, int rw)
760 {
761 uint_t count;
762 page_t *pp;
763 kmutex_t *vphm;
764
765 if (rw == KSTAT_WRITE)
766 return (EACCES);
767
768 vphm = page_vnode_mutex(retired_pages);
769 mutex_enter(vphm);
770 /* Needs to be under a lock so that for loop will work right */
771 if (retired_pages->v_pages == NULL) {
772 mutex_exit(vphm);
773 ksp->ks_ndata = 0;
774 ksp->ks_data_size = 0;
775 return (0);
776 }
777
778 count = 1;
779 for (pp = retired_pages->v_pages->p_vpnext;
780 pp != retired_pages->v_pages; pp = pp->p_vpnext) {
781 count++;
782 }
783 mutex_exit(vphm);
784
785 ksp->ks_ndata = count;
786 ksp->ks_data_size = count * 2 * sizeof (uint64_t);
787
788 return (0);
789 }
790
791 /*
792 * all spans will be pagesize and no coalescing will be done with the
793 * list produced.
794 */
795 static int
pr_list_kstat_snapshot(kstat_t * ksp,void * buf,int rw)796 pr_list_kstat_snapshot(kstat_t *ksp, void *buf, int rw)
797 {
798 kmutex_t *vphm;
799 page_t *pp;
800 struct memunit {
801 uint64_t address;
802 uint64_t size;
803 } *kspmem;
804
805 if (rw == KSTAT_WRITE)
806 return (EACCES);
807
808 ksp->ks_snaptime = gethrtime();
809
810 kspmem = (struct memunit *)buf;
811
812 vphm = page_vnode_mutex(retired_pages);
813 mutex_enter(vphm);
814 pp = retired_pages->v_pages;
815 if (((caddr_t)kspmem >= (caddr_t)buf + ksp->ks_data_size) ||
816 (pp == NULL)) {
817 mutex_exit(vphm);
818 return (0);
819 }
820 kspmem->address = ptob(pp->p_pagenum);
821 kspmem->size = PAGESIZE;
822 kspmem++;
823 for (pp = pp->p_vpnext; pp != retired_pages->v_pages;
824 pp = pp->p_vpnext, kspmem++) {
825 if ((caddr_t)kspmem >= (caddr_t)buf + ksp->ks_data_size)
826 break;
827 kspmem->address = ptob(pp->p_pagenum);
828 kspmem->size = PAGESIZE;
829 }
830 mutex_exit(vphm);
831
832 return (0);
833 }
834
835 /*
836 * page_retire_pend_count -- helper function for page_capture_thread,
837 * returns the number of pages pending retirement.
838 */
839 uint64_t
page_retire_pend_count(void)840 page_retire_pend_count(void)
841 {
842 return (PR_KSTAT_PENDING);
843 }
844
845 uint64_t
page_retire_pend_kas_count(void)846 page_retire_pend_kas_count(void)
847 {
848 return (PR_KSTAT_PENDING_KAS);
849 }
850
851 void
page_retire_incr_pend_count(void * datap)852 page_retire_incr_pend_count(void *datap)
853 {
854 PR_INCR_KSTAT(pr_pending);
855
856 if (datap == &kvp || datap == &kvps[KV_ZVP] || datap == &kvps[KV_VVP])
857 PR_INCR_KSTAT(pr_pending_kas);
858 }
859
860 void
page_retire_decr_pend_count(void * datap)861 page_retire_decr_pend_count(void *datap)
862 {
863 PR_DECR_KSTAT(pr_pending);
864
865 if (datap == &kvp || datap == &kvps[KV_ZVP] || datap == &kvps[KV_VVP])
866 PR_DECR_KSTAT(pr_pending_kas);
867 }
868
869 /*
870 * Initialize the page retire mechanism:
871 *
872 * - Establish the correctable error retire limit.
873 * - Initialize locks.
874 * - Build the retired_pages vnode.
875 * - Set up the kstats.
876 * - Fire off the background thread.
877 * - Tell page_retire() it's OK to start retiring pages.
878 */
879 void
page_retire_init(void)880 page_retire_init(void)
881 {
882 const fs_operation_def_t retired_vnodeops_template[] = {
883 { NULL, NULL }
884 };
885 struct vnodeops *vops;
886 kstat_t *ksp;
887
888 const uint_t page_retire_ndata =
889 sizeof (page_retire_kstat) / sizeof (kstat_named_t);
890
891 ASSERT(page_retire_ksp == NULL);
892
893 if (max_pages_retired_bps <= 0) {
894 max_pages_retired_bps = MCE_BPT;
895 }
896
897 mutex_init(&pr_q_mutex, NULL, MUTEX_DEFAULT, NULL);
898
899 retired_pages = vn_alloc(KM_SLEEP);
900 if (vn_make_ops("retired_pages", retired_vnodeops_template, &vops)) {
901 cmn_err(CE_PANIC,
902 "page_retired_init: can't make retired vnodeops");
903 }
904 vn_setops(retired_pages, vops);
905
906 if ((page_retire_ksp = kstat_create("unix", 0, "page_retire",
907 "misc", KSTAT_TYPE_NAMED, page_retire_ndata,
908 KSTAT_FLAG_VIRTUAL)) == NULL) {
909 cmn_err(CE_WARN, "kstat_create for page_retire failed");
910 } else {
911 page_retire_ksp->ks_data = (void *)&page_retire_kstat;
912 page_retire_ksp->ks_update = page_retire_kstat_update;
913 kstat_install(page_retire_ksp);
914 }
915
916 mutex_init(&pr_list_kstat_mutex, NULL, MUTEX_DEFAULT, NULL);
917 ksp = kstat_create("unix", 0, "page_retire_list", "misc",
918 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VAR_SIZE | KSTAT_FLAG_VIRTUAL);
919 if (ksp != NULL) {
920 ksp->ks_update = pr_list_kstat_update;
921 ksp->ks_snapshot = pr_list_kstat_snapshot;
922 ksp->ks_lock = &pr_list_kstat_mutex;
923 kstat_install(ksp);
924 }
925
926 memscrub_notify_func =
927 (void(*)(uint64_t))kobj_getsymvalue("memscrub_notify", 0);
928
929 page_capture_register_callback(PC_RETIRE, -1, page_retire_pp_finish);
930 pr_enable = 1;
931 }
932
933 /*
934 * page_retire_hunt() callback for the retire thread.
935 */
936 static void
page_retire_thread_cb(page_t * pp)937 page_retire_thread_cb(page_t *pp)
938 {
939 PR_DEBUG(prd_tctop);
940 if (!PP_ISKAS(pp) && page_trylock(pp, SE_EXCL)) {
941 PR_DEBUG(prd_tclocked);
942 page_unlock(pp);
943 }
944 }
945
946 /*
947 * Callback used by page_trycapture() to finish off retiring a page.
948 * The page has already been cleaned and we've been given sole access to
949 * it.
950 * Always returns 0 to indicate that callback succeded as the callback never
951 * fails to finish retiring the given page.
952 */
953 /*ARGSUSED*/
954 static int
page_retire_pp_finish(page_t * pp,void * notused,uint_t flags)955 page_retire_pp_finish(page_t *pp, void *notused, uint_t flags)
956 {
957 int toxic;
958
959 ASSERT(PAGE_EXCL(pp));
960 ASSERT(pp->p_iolock_state == 0);
961 ASSERT(pp->p_szc == 0);
962
963 toxic = pp->p_toxic;
964
965 /*
966 * The problem page is locked, demoted, unmapped, not free,
967 * hashed out, and not COW or mlocked (whew!).
968 *
969 * Now we select our ammunition, take it around back, and shoot it.
970 */
971 if (toxic & PR_UE) {
972 ue_error:
973 if (page_retire_transient_ue(pp)) {
974 PR_DEBUG(prd_uescrubbed);
975 (void) page_retire_done(pp, PRD_UE_SCRUBBED);
976 } else {
977 PR_DEBUG(prd_uenotscrubbed);
978 page_retire_destroy(pp);
979 (void) page_retire_done(pp, PRD_SUCCESS);
980 }
981 return (0);
982 } else if (toxic & PR_FMA) {
983 PR_DEBUG(prd_fma);
984 page_retire_destroy(pp);
985 (void) page_retire_done(pp, PRD_SUCCESS);
986 return (0);
987 } else if (toxic & PR_MCE) {
988 PR_DEBUG(prd_mce);
989 page_retire_destroy(pp);
990 (void) page_retire_done(pp, PRD_SUCCESS);
991 return (0);
992 }
993
994 /*
995 * When page_retire_first_ue is set to zero and a UE occurs which is
996 * transient, it's possible that we clear some flags set by a second
997 * UE error on the page which occurs while the first is currently being
998 * handled and thus we need to handle the case where none of the above
999 * are set. In this instance, PR_UE_SCRUBBED should be set and thus
1000 * we should execute the UE code above.
1001 */
1002 if (toxic & PR_UE_SCRUBBED) {
1003 goto ue_error;
1004 }
1005
1006 /*
1007 * It's impossible to get here.
1008 */
1009 panic("bad toxic flags 0x%x in page_retire_pp_finish\n", toxic);
1010 return (0);
1011 }
1012
1013 /*
1014 * page_retire() - the front door in to retire a page.
1015 *
1016 * Ideally, page_retire() would instantly retire the requested page.
1017 * Unfortunately, some pages are locked or otherwise tied up and cannot be
1018 * retired right away. We use the page capture logic to deal with this
1019 * situation as it will continuously try to retire the page in the background
1020 * if the first attempt fails. Success is determined by looking to see whether
1021 * the page has been retired after the page_trycapture() attempt.
1022 *
1023 * Returns:
1024 *
1025 * - 0 on success,
1026 * - EINVAL when the PA is whacko,
1027 * - EIO if the page is already retired or already pending retirement, or
1028 * - EAGAIN if the page could not be _immediately_ retired but is pending.
1029 */
1030 int
page_retire(uint64_t pa,uchar_t reason)1031 page_retire(uint64_t pa, uchar_t reason)
1032 {
1033 page_t *pp;
1034
1035 ASSERT(reason & PR_REASONS); /* there must be a reason */
1036 ASSERT(!(reason & ~PR_REASONS)); /* but no other bits */
1037
1038 pp = page_numtopp_nolock(mmu_btop(pa));
1039 if (pp == NULL) {
1040 PR_MESSAGE(CE_WARN, 1, "Cannot schedule clearing of error on"
1041 " page 0x%08x.%08x; page is not relocatable memory", pa);
1042 return (page_retire_done(pp, PRD_INVALID_PA));
1043 }
1044 if (PP_RETIRED(pp)) {
1045 PR_DEBUG(prd_dup1);
1046 return (page_retire_done(pp, PRD_DUPLICATE));
1047 }
1048
1049 if (memscrub_notify_func != NULL) {
1050 (void) memscrub_notify_func(pa);
1051 }
1052
1053 if ((reason & PR_UE) && !PP_TOXIC(pp)) {
1054 PR_MESSAGE(CE_NOTE, 1, "Scheduling clearing of error on"
1055 " page 0x%08x.%08x", pa);
1056 } else if (PP_PR_REQ(pp)) {
1057 PR_DEBUG(prd_dup2);
1058 return (page_retire_done(pp, PRD_DUPLICATE));
1059 } else {
1060 PR_MESSAGE(CE_NOTE, 1, "Scheduling removal of"
1061 " page 0x%08x.%08x", pa);
1062 }
1063
1064 /* Avoid setting toxic bits in the first place */
1065 if ((reason & (PR_FMA | PR_MCE)) && !(reason & PR_UE) &&
1066 page_retire_limit()) {
1067 return (page_retire_done(pp, PRD_LIMIT));
1068 }
1069
1070 if (MTBF(pr_calls, pr_mtbf)) {
1071 page_settoxic(pp, reason);
1072 if (page_trycapture(pp, 0, CAPTURE_RETIRE, pp->p_vnode) == 0) {
1073 PR_DEBUG(prd_prlocked);
1074 } else {
1075 PR_DEBUG(prd_prnotlocked);
1076 }
1077 } else {
1078 PR_DEBUG(prd_prnotlocked);
1079 }
1080
1081 if (PP_RETIRED(pp)) {
1082 PR_DEBUG(prd_prretired);
1083 return (0);
1084 } else {
1085 cv_signal(&pc_cv);
1086 PR_INCR_KSTAT(pr_failed);
1087
1088 if (pp->p_toxic & PR_MSG) {
1089 return (page_retire_done(pp, PRD_FAILED));
1090 } else {
1091 return (page_retire_done(pp, PRD_PENDING));
1092 }
1093 }
1094 }
1095
1096 /*
1097 * Take a retired page off the retired-pages vnode and clear the toxic flags.
1098 * If "free" is nonzero, lock it and put it back on the freelist. If "free"
1099 * is zero, the caller already holds SE_EXCL lock so we simply unretire it
1100 * and don't do anything else with it.
1101 *
1102 * Any unretire messages are printed from this routine.
1103 *
1104 * Returns 0 if page pp was unretired; else an error code.
1105 *
1106 * If flags is:
1107 * PR_UNR_FREE - lock the page, clear the toxic flags and free it
1108 * to the freelist.
1109 * PR_UNR_TEMP - lock the page, unretire it, leave the toxic
1110 * bits set as is and return it to the caller.
1111 * PR_UNR_CLEAN - page is SE_EXCL locked, unretire it, clear the
1112 * toxic flags and return it to caller as is.
1113 */
1114 int
page_unretire_pp(page_t * pp,int flags)1115 page_unretire_pp(page_t *pp, int flags)
1116 {
1117 /*
1118 * To be retired, a page has to be hashed onto the retired_pages vnode
1119 * and have PR_RETIRED set in p_toxic.
1120 */
1121 if (flags == PR_UNR_CLEAN ||
1122 page_try_reclaim_lock(pp, SE_EXCL, SE_RETIRED)) {
1123 ASSERT(PAGE_EXCL(pp));
1124 PR_DEBUG(prd_ulocked);
1125 if (!PP_RETIRED(pp)) {
1126 PR_DEBUG(prd_unotretired);
1127 page_unlock(pp);
1128 return (page_retire_done(pp, PRD_UNR_NOT));
1129 }
1130
1131 PR_MESSAGE(CE_NOTE, 1, "unretiring retired"
1132 " page 0x%08x.%08x", mmu_ptob((uint64_t)pp->p_pagenum));
1133 if (pp->p_toxic & PR_FMA) {
1134 PR_DECR_KSTAT(pr_fma);
1135 } else if (pp->p_toxic & PR_UE) {
1136 PR_DECR_KSTAT(pr_ue);
1137 } else {
1138 PR_DECR_KSTAT(pr_mce);
1139 }
1140
1141 if (flags == PR_UNR_TEMP)
1142 page_clrtoxic(pp, PR_RETIRED);
1143 else
1144 page_clrtoxic(pp, PR_TOXICFLAGS);
1145
1146 if (flags == PR_UNR_FREE) {
1147 PR_DEBUG(prd_udestroy);
1148 page_destroy(pp, 0);
1149 } else {
1150 PR_DEBUG(prd_uhashout);
1151 page_hashout(pp, NULL);
1152 }
1153
1154 mutex_enter(&freemem_lock);
1155 availrmem++;
1156 mutex_exit(&freemem_lock);
1157
1158 PR_DEBUG(prd_uunretired);
1159 PR_DECR_KSTAT(pr_retired);
1160 PR_INCR_KSTAT(pr_unretired);
1161 return (page_retire_done(pp, PRD_UNR_SUCCESS));
1162 }
1163 PR_DEBUG(prd_unotlocked);
1164 return (page_retire_done(pp, PRD_UNR_CANTLOCK));
1165 }
1166
1167 /*
1168 * Return a page to service by moving it from the retired_pages vnode
1169 * onto the freelist.
1170 *
1171 * Called from mmioctl_page_retire() on behalf of the FMA DE.
1172 *
1173 * Returns:
1174 *
1175 * - 0 if the page is unretired,
1176 * - EAGAIN if the pp can not be locked,
1177 * - EINVAL if the PA is whacko, and
1178 * - EIO if the pp is not retired.
1179 */
1180 int
page_unretire(uint64_t pa)1181 page_unretire(uint64_t pa)
1182 {
1183 page_t *pp;
1184
1185 pp = page_numtopp_nolock(mmu_btop(pa));
1186 if (pp == NULL) {
1187 return (page_retire_done(pp, PRD_INVALID_PA));
1188 }
1189
1190 return (page_unretire_pp(pp, PR_UNR_FREE));
1191 }
1192
1193 /*
1194 * Test a page to see if it is retired. If errors is non-NULL, the toxic
1195 * bits of the page are returned. Returns 0 on success, error code on failure.
1196 */
1197 int
page_retire_check_pp(page_t * pp,uint64_t * errors)1198 page_retire_check_pp(page_t *pp, uint64_t *errors)
1199 {
1200 int rc;
1201
1202 if (PP_RETIRED(pp)) {
1203 PR_DEBUG(prd_checkhit);
1204 rc = 0;
1205 } else if (PP_PR_REQ(pp)) {
1206 PR_DEBUG(prd_checkmiss_pend);
1207 rc = EAGAIN;
1208 } else {
1209 PR_DEBUG(prd_checkmiss_noerr);
1210 rc = EIO;
1211 }
1212
1213 /*
1214 * We have magically arranged the bit values returned to fmd(8)
1215 * to line up with the FMA, MCE, and UE bits of the page_t.
1216 */
1217 if (errors) {
1218 uint64_t toxic = (uint64_t)(pp->p_toxic & PR_ERRMASK);
1219 if (toxic & PR_UE_SCRUBBED) {
1220 toxic &= ~PR_UE_SCRUBBED;
1221 toxic |= PR_UE;
1222 }
1223 *errors = toxic;
1224 }
1225
1226 return (rc);
1227 }
1228
1229 /*
1230 * Test to see if the page_t for a given PA is retired, and return the
1231 * hardware errors we have seen on the page if requested.
1232 *
1233 * Called from mmioctl_page_retire on behalf of the FMA DE.
1234 *
1235 * Returns:
1236 *
1237 * - 0 if the page is retired,
1238 * - EIO if the page is not retired and has no errors,
1239 * - EAGAIN if the page is not retired but is pending; and
1240 * - EINVAL if the PA is whacko.
1241 */
1242 int
page_retire_check(uint64_t pa,uint64_t * errors)1243 page_retire_check(uint64_t pa, uint64_t *errors)
1244 {
1245 page_t *pp;
1246
1247 if (errors) {
1248 *errors = 0;
1249 }
1250
1251 pp = page_numtopp_nolock(mmu_btop(pa));
1252 if (pp == NULL) {
1253 return (page_retire_done(pp, PRD_INVALID_PA));
1254 }
1255
1256 return (page_retire_check_pp(pp, errors));
1257 }
1258
1259 /*
1260 * Page retire self-test. For now, it always returns 0.
1261 */
1262 int
page_retire_test(void)1263 page_retire_test(void)
1264 {
1265 page_t *first, *pp, *cpp, *cpp2, *lpp;
1266
1267 /*
1268 * Tests the corner case where a large page can't be retired
1269 * because one of the constituent pages is locked. We mark
1270 * one page to be retired and try to retire it, and mark the
1271 * other page to be retired but don't try to retire it, so
1272 * that page_unlock() in the failure path will recurse and try
1273 * to retire THAT page. This is the worst possible situation
1274 * we can get ourselves into.
1275 */
1276 memsegs_lock(0);
1277 pp = first = page_first();
1278 do {
1279 if (pp->p_szc && PP_PAGEROOT(pp) == pp) {
1280 cpp = pp + 1;
1281 lpp = PP_ISFREE(pp)? pp : pp + 2;
1282 cpp2 = pp + 3;
1283 if (!page_trylock(lpp, pp == lpp? SE_EXCL : SE_SHARED))
1284 continue;
1285 if (!page_trylock(cpp, SE_EXCL)) {
1286 page_unlock(lpp);
1287 continue;
1288 }
1289
1290 /* fails */
1291 (void) page_retire(ptob(cpp->p_pagenum), PR_FMA);
1292
1293 page_unlock(lpp);
1294 page_unlock(cpp);
1295 (void) page_retire(ptob(cpp->p_pagenum), PR_FMA);
1296 (void) page_retire(ptob(cpp2->p_pagenum), PR_FMA);
1297 }
1298 } while ((pp = page_next(pp)) != first);
1299 memsegs_unlock(0);
1300
1301 return (0);
1302 }
1303