17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
5a307a255Sgavinm  * Common Development and Distribution License (the "License").
6a307a255Sgavinm  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217aec1d6eScindi 
227c478bd9Sstevel@tonic-gate /*
235f149bcaScy  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
247c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
257c478bd9Sstevel@tonic-gate  */
267c478bd9Sstevel@tonic-gate 
277c478bd9Sstevel@tonic-gate /*
287c478bd9Sstevel@tonic-gate  * Page retirement can be an extended process due to the fact that a retirement
297c478bd9Sstevel@tonic-gate  * may not be possible when the original request is made.  The kernel will
307c478bd9Sstevel@tonic-gate  * repeatedly attempt to retire a given page, but will not let us know when the
317c478bd9Sstevel@tonic-gate  * page has been retired.  We therefore have to poll to see if the retirement
327c478bd9Sstevel@tonic-gate  * has been completed.  This poll is implemented with a bounded exponential
337c478bd9Sstevel@tonic-gate  * backoff to reduce the burden which we impose upon the system.
347c478bd9Sstevel@tonic-gate  *
357c478bd9Sstevel@tonic-gate  * To reduce the burden on fmd in the face of retirement storms, we schedule
367c478bd9Sstevel@tonic-gate  * all retries as a group.  In the simplest case, we attempt to retire a single
377c478bd9Sstevel@tonic-gate  * page.  When forced to retry, we initially schedule a retry at a configurable
387c478bd9Sstevel@tonic-gate  * interval t.  If the retry fails, we schedule another at 2 * t, and so on,
397c478bd9Sstevel@tonic-gate  * until t reaches the maximum interval (also configurable).  Future retries
407c478bd9Sstevel@tonic-gate  * for that page will occur with t equal to the maximum interval value.  We
417c478bd9Sstevel@tonic-gate  * will never give up on a retirement.
427c478bd9Sstevel@tonic-gate  *
437c478bd9Sstevel@tonic-gate  * With multiple retirements, the situation gets slightly more complicated.  As
447c478bd9Sstevel@tonic-gate  * indicated above, we schedule retries as a group.  We don't want to deny new
457c478bd9Sstevel@tonic-gate  * pages their short retry intervals, so we'll (re)set the retry interval to the
467c478bd9Sstevel@tonic-gate  * value appropriate for the newest page.
477c478bd9Sstevel@tonic-gate  */
487c478bd9Sstevel@tonic-gate 
497c478bd9Sstevel@tonic-gate #include <cma.h>
507c478bd9Sstevel@tonic-gate 
517c478bd9Sstevel@tonic-gate #include <time.h>
527c478bd9Sstevel@tonic-gate #include <errno.h>
537c478bd9Sstevel@tonic-gate #include <unistd.h>
547c478bd9Sstevel@tonic-gate #include <strings.h>
557c478bd9Sstevel@tonic-gate #include <fm/fmd_api.h>
567aec1d6eScindi #include <fm/libtopo.h>
57e4b86885SCheng Sean Ye #include <fm/fmd_fmri.h>
58e4b86885SCheng Sean Ye #include <fm/fmd_agent.h>
597c478bd9Sstevel@tonic-gate #include <sys/fm/protocol.h>
607c478bd9Sstevel@tonic-gate 
617c478bd9Sstevel@tonic-gate static void
cma_page_free(fmd_hdl_t * hdl,cma_page_t * page)627c478bd9Sstevel@tonic-gate cma_page_free(fmd_hdl_t *hdl, cma_page_t *page)
637c478bd9Sstevel@tonic-gate {
64*3f1e69beSCheng Sean Ye 	nvlist_free(page->pg_asru);
65*3f1e69beSCheng Sean Ye 	nvlist_free(page->pg_rsrc);
667c478bd9Sstevel@tonic-gate 	fmd_hdl_free(hdl, page, sizeof (cma_page_t));
677c478bd9Sstevel@tonic-gate }
687c478bd9Sstevel@tonic-gate 
697aec1d6eScindi /*
707aec1d6eScindi  * Retire the specified ASRU, referring to a memory page by PA or by DIMM
717aec1d6eScindi  * offset (i.e. the encoded coordinates internal bank, row, and column).
727aec1d6eScindi  * In the initial FMA implementation, fault.memory.page exported an ASRU
737aec1d6eScindi  * with an explicit physical address, which is valid at the initial time of
747aec1d6eScindi  * diagnosis but may not be later following DR, DIMM removal, or interleave
757aec1d6eScindi  * changes.  On SPARC, this issue was solved by exporting the DIMM offset
767aec1d6eScindi  * and pushing the entire FMRI to the platform memory controller through
77e4b86885SCheng Sean Ye  * /dev/fm so it can derive the current PA from the DIMM and offset.
78e4b86885SCheng Sean Ye  * On x86, we also encode DIMM and offset in hc-specific, which is then used
79e4b86885SCheng Sean Ye  * by the x64 memory controller driver.
807aec1d6eScindi  * At some point these three approaches need to be rationalized: all platforms
817aec1d6eScindi  * should use the same scheme, either with decoding in the kernel or decoding
827aec1d6eScindi  * in userland (i.e. with a libtopo method to compute and update the PA).
837aec1d6eScindi  */
847c478bd9Sstevel@tonic-gate /*ARGSUSED*/
8525cf1a30Sjl int
cma_page_retire(fmd_hdl_t * hdl,nvlist_t * nvl,nvlist_t * asru,const char * uuid,boolean_t repair)865f149bcaScy cma_page_retire(fmd_hdl_t *hdl, nvlist_t *nvl, nvlist_t *asru,
875f149bcaScy     const char *uuid, boolean_t repair)
887c478bd9Sstevel@tonic-gate {
897c478bd9Sstevel@tonic-gate 	cma_page_t *page;
907c478bd9Sstevel@tonic-gate 	uint64_t pageaddr;
915f149bcaScy 	const char *action = repair ? "unretire" : "retire";
92e4b86885SCheng Sean Ye 	int rc;
93*3f1e69beSCheng Sean Ye 	nvlist_t *rsrc = NULL, *asrucp = NULL, *hcsp;
947c478bd9Sstevel@tonic-gate 
95*3f1e69beSCheng Sean Ye 	(void) nvlist_lookup_nvlist(nvl, FM_FAULT_RESOURCE, &rsrc);
96*3f1e69beSCheng Sean Ye 
97*3f1e69beSCheng Sean Ye 	if (nvlist_dup(asru, &asrucp, 0) != 0) {
988ec6c6f1Stsien 		fmd_hdl_debug(hdl, "page retire nvlist dup failed\n");
998ec6c6f1Stsien 		return (CMA_RA_FAILURE);
1008ec6c6f1Stsien 	}
1018ec6c6f1Stsien 
1027c478bd9Sstevel@tonic-gate 	/* It should already be expanded, but we'll do it again anyway */
103*3f1e69beSCheng Sean Ye 	if (fmd_nvl_fmri_expand(hdl, asrucp) < 0) {
1047c478bd9Sstevel@tonic-gate 		fmd_hdl_debug(hdl, "failed to expand page asru\n");
1057c478bd9Sstevel@tonic-gate 		cma_stats.bad_flts.fmds_value.ui64++;
106*3f1e69beSCheng Sean Ye 		nvlist_free(asrucp);
10725cf1a30Sjl 		return (CMA_RA_FAILURE);
1087c478bd9Sstevel@tonic-gate 	}
1097c478bd9Sstevel@tonic-gate 
110*3f1e69beSCheng Sean Ye 	if (!repair && !fmd_nvl_fmri_present(hdl, asrucp)) {
1115f149bcaScy 		fmd_hdl_debug(hdl, "page retire overtaken by events\n");
1125f149bcaScy 		cma_stats.page_nonent.fmds_value.ui64++;
113*3f1e69beSCheng Sean Ye 		nvlist_free(asrucp);
1145f149bcaScy 		return (CMA_RA_SUCCESS);
1155f149bcaScy 	}
1165f149bcaScy 
117*3f1e69beSCheng Sean Ye 	/* Figure out physaddr from resource or asru */
118*3f1e69beSCheng Sean Ye 	if (rsrc == NULL ||
119*3f1e69beSCheng Sean Ye 	    nvlist_lookup_nvlist(rsrc, FM_FMRI_HC_SPECIFIC, &hcsp) != 0 ||
120e4b86885SCheng Sean Ye 	    (nvlist_lookup_uint64(hcsp, "asru-" FM_FMRI_HC_SPECIFIC_PHYSADDR,
121e4b86885SCheng Sean Ye 	    &pageaddr) != 0 && nvlist_lookup_uint64(hcsp,
122e4b86885SCheng Sean Ye 	    FM_FMRI_HC_SPECIFIC_PHYSADDR, &pageaddr) != 0)) {
123*3f1e69beSCheng Sean Ye 		if (nvlist_lookup_uint64(asrucp, FM_FMRI_MEM_PHYSADDR,
124*3f1e69beSCheng Sean Ye 		    &pageaddr) != 0) {
125*3f1e69beSCheng Sean Ye 			fmd_hdl_debug(hdl, "mem fault missing 'physaddr'\n");
126*3f1e69beSCheng Sean Ye 			cma_stats.bad_flts.fmds_value.ui64++;
127*3f1e69beSCheng Sean Ye 			nvlist_free(asrucp);
128*3f1e69beSCheng Sean Ye 			return (CMA_RA_FAILURE);
129*3f1e69beSCheng Sean Ye 		}
1307c478bd9Sstevel@tonic-gate 	}
1317c478bd9Sstevel@tonic-gate 
1325f149bcaScy 	if (repair) {
1335f149bcaScy 		if (!cma.cma_page_dounretire) {
1345f149bcaScy 			fmd_hdl_debug(hdl, "suppressed unretire of page %llx\n",
1355f149bcaScy 			    (u_longlong_t)pageaddr);
1365f149bcaScy 			cma_stats.page_supp.fmds_value.ui64++;
137*3f1e69beSCheng Sean Ye 			nvlist_free(asrucp);
1385f149bcaScy 			return (CMA_RA_SUCCESS);
1395f149bcaScy 		}
140*3f1e69beSCheng Sean Ye 		/* If unretire via topo fails, we fall back to legacy way */
141*3f1e69beSCheng Sean Ye 		if (rsrc == NULL || (rc = fmd_nvl_fmri_unretire(hdl, rsrc)) < 0)
142*3f1e69beSCheng Sean Ye 			rc = cma_fmri_page_unretire(hdl, asrucp);
1435f149bcaScy 	} else {
1445f149bcaScy 		if (!cma.cma_page_doretire) {
1455f149bcaScy 			fmd_hdl_debug(hdl, "suppressed retire of page %llx\n",
1465f149bcaScy 			    (u_longlong_t)pageaddr);
1475f149bcaScy 			cma_stats.page_supp.fmds_value.ui64++;
148*3f1e69beSCheng Sean Ye 			nvlist_free(asrucp);
14925cf1a30Sjl 			return (CMA_RA_FAILURE);
1507aec1d6eScindi 		}
151*3f1e69beSCheng Sean Ye 		/* If retire via topo fails, we fall back to legacy way */
152*3f1e69beSCheng Sean Ye 		if (rsrc == NULL || (rc = fmd_nvl_fmri_retire(hdl, rsrc)) < 0)
153*3f1e69beSCheng Sean Ye 			rc = cma_fmri_page_retire(hdl, asrucp);
1547aec1d6eScindi 	}
1557aec1d6eScindi 
156e4b86885SCheng Sean Ye 	if (rc == FMD_AGENT_RETIRE_DONE) {
1575f149bcaScy 		fmd_hdl_debug(hdl, "%sd page 0x%llx\n",
1585f149bcaScy 		    action, (u_longlong_t)pageaddr);
1595f149bcaScy 		if (repair)
1605f149bcaScy 			cma_stats.page_repairs.fmds_value.ui64++;
1615f149bcaScy 		else
1625f149bcaScy 			cma_stats.page_flts.fmds_value.ui64++;
163*3f1e69beSCheng Sean Ye 		nvlist_free(asrucp);
16425cf1a30Sjl 		return (CMA_RA_SUCCESS);
165e4b86885SCheng Sean Ye 	} else if (repair || rc != FMD_AGENT_RETIRE_ASYNC) {
1665f149bcaScy 		fmd_hdl_debug(hdl, "%s of page 0x%llx failed, will not "
1675f149bcaScy 		    "retry: %s\n", action, (u_longlong_t)pageaddr,
1685f149bcaScy 		    strerror(errno));
1695f149bcaScy 
1705f149bcaScy 		cma_stats.page_fails.fmds_value.ui64++;
171*3f1e69beSCheng Sean Ye 		nvlist_free(asrucp);
17225cf1a30Sjl 		return (CMA_RA_FAILURE);
1737c478bd9Sstevel@tonic-gate 	}
1747c478bd9Sstevel@tonic-gate 
1757c478bd9Sstevel@tonic-gate 	/*
1767c478bd9Sstevel@tonic-gate 	 * The page didn't immediately retire.  We'll need to periodically
1777c478bd9Sstevel@tonic-gate 	 * check to see if it has been retired.
1787c478bd9Sstevel@tonic-gate 	 */
1797c478bd9Sstevel@tonic-gate 	fmd_hdl_debug(hdl, "page didn't retire - sleeping\n");
1807c478bd9Sstevel@tonic-gate 
1817c478bd9Sstevel@tonic-gate 	page = fmd_hdl_zalloc(hdl, sizeof (cma_page_t), FMD_SLEEP);
1827c478bd9Sstevel@tonic-gate 	page->pg_addr = pageaddr;
183*3f1e69beSCheng Sean Ye 	if (rsrc != NULL)
184*3f1e69beSCheng Sean Ye 		(void) nvlist_dup(rsrc, &page->pg_rsrc, 0);
185*3f1e69beSCheng Sean Ye 	page->pg_asru = asrucp;
1867c478bd9Sstevel@tonic-gate 	if (uuid != NULL)
1877c478bd9Sstevel@tonic-gate 		page->pg_uuid = fmd_hdl_strdup(hdl, uuid, FMD_SLEEP);
1887c478bd9Sstevel@tonic-gate 
1897c478bd9Sstevel@tonic-gate 	page->pg_next = cma.cma_pages;
1907c478bd9Sstevel@tonic-gate 	cma.cma_pages = page;
1917c478bd9Sstevel@tonic-gate 
1927c478bd9Sstevel@tonic-gate 	if (cma.cma_page_timerid != 0)
1937c478bd9Sstevel@tonic-gate 		fmd_timer_remove(hdl, cma.cma_page_timerid);
1947c478bd9Sstevel@tonic-gate 
1957c478bd9Sstevel@tonic-gate 	cma.cma_page_curdelay = cma.cma_page_mindelay;
1967c478bd9Sstevel@tonic-gate 
1977c478bd9Sstevel@tonic-gate 	cma.cma_page_timerid =
1987c478bd9Sstevel@tonic-gate 	    fmd_timer_install(hdl, NULL, NULL, cma.cma_page_curdelay);
19925cf1a30Sjl 
200*3f1e69beSCheng Sean Ye 	/* Don't free asrucp here.  This FMRI will be needed for retry. */
20125cf1a30Sjl 	return (CMA_RA_FAILURE);
2027c478bd9Sstevel@tonic-gate }
2037c478bd9Sstevel@tonic-gate 
2047c478bd9Sstevel@tonic-gate static int
page_retry(fmd_hdl_t * hdl,cma_page_t * page)2057c478bd9Sstevel@tonic-gate page_retry(fmd_hdl_t *hdl, cma_page_t *page)
2067c478bd9Sstevel@tonic-gate {
207e4b86885SCheng Sean Ye 	int rc;
208e4b86885SCheng Sean Ye 
209*3f1e69beSCheng Sean Ye 	if (page->pg_asru != NULL &&
210*3f1e69beSCheng Sean Ye 	    !fmd_nvl_fmri_present(hdl, page->pg_asru)) {
2117c478bd9Sstevel@tonic-gate 		fmd_hdl_debug(hdl, "page retire overtaken by events");
2127c478bd9Sstevel@tonic-gate 		cma_stats.page_nonent.fmds_value.ui64++;
2137c478bd9Sstevel@tonic-gate 
2147c478bd9Sstevel@tonic-gate 		if (page->pg_uuid != NULL)
2157c478bd9Sstevel@tonic-gate 			fmd_case_uuclose(hdl, page->pg_uuid);
2167c478bd9Sstevel@tonic-gate 		return (1); /* no longer a page to retire */
2177c478bd9Sstevel@tonic-gate 	}
2187c478bd9Sstevel@tonic-gate 
219*3f1e69beSCheng Sean Ye 	if (page->pg_rsrc == NULL ||
220*3f1e69beSCheng Sean Ye 	    (rc = fmd_nvl_fmri_service_state(hdl, page->pg_rsrc)) < 0)
221*3f1e69beSCheng Sean Ye 		rc = cma_fmri_page_service_state(hdl, page->pg_asru);
222*3f1e69beSCheng Sean Ye 
223e4b86885SCheng Sean Ye 	if (rc == FMD_SERVICE_STATE_UNUSABLE) {
2247c478bd9Sstevel@tonic-gate 		fmd_hdl_debug(hdl, "retired page 0x%llx on retry %u\n",
2257c478bd9Sstevel@tonic-gate 		    page->pg_addr, page->pg_nretries);
2267c478bd9Sstevel@tonic-gate 		cma_stats.page_flts.fmds_value.ui64++;
2277c478bd9Sstevel@tonic-gate 
2287c478bd9Sstevel@tonic-gate 		if (page->pg_uuid != NULL)
2297c478bd9Sstevel@tonic-gate 			fmd_case_uuclose(hdl, page->pg_uuid);
2307c478bd9Sstevel@tonic-gate 		return (1); /* page retired */
2317c478bd9Sstevel@tonic-gate 	}
2327c478bd9Sstevel@tonic-gate 
233e4b86885SCheng Sean Ye 	if (rc == FMD_SERVICE_STATE_ISOLATE_PENDING) {
2347c478bd9Sstevel@tonic-gate 		fmd_hdl_debug(hdl, "scheduling another retry for 0x%llx\n",
2357c478bd9Sstevel@tonic-gate 		    page->pg_addr);
2367c478bd9Sstevel@tonic-gate 		return (0); /* schedule another retry */
2377c478bd9Sstevel@tonic-gate 	} else {
238e4b86885SCheng Sean Ye 		fmd_hdl_debug(hdl, "failed to retry page 0x%llx "
239e4b86885SCheng Sean Ye 		    "retirement: %s\n", page->pg_addr,
240e4b86885SCheng Sean Ye 		    strerror(errno));
2417c478bd9Sstevel@tonic-gate 
2427c478bd9Sstevel@tonic-gate 		cma_stats.page_fails.fmds_value.ui64++;
2437c478bd9Sstevel@tonic-gate 		return (1); /* give up */
2447c478bd9Sstevel@tonic-gate 	}
2457c478bd9Sstevel@tonic-gate }
2467c478bd9Sstevel@tonic-gate 
2477c478bd9Sstevel@tonic-gate void
cma_page_retry(fmd_hdl_t * hdl)2487c478bd9Sstevel@tonic-gate cma_page_retry(fmd_hdl_t *hdl)
2497c478bd9Sstevel@tonic-gate {
2507c478bd9Sstevel@tonic-gate 	cma_page_t **pagep;
2517c478bd9Sstevel@tonic-gate 
2527c478bd9Sstevel@tonic-gate 	cma.cma_page_timerid = 0;
2537c478bd9Sstevel@tonic-gate 
2547c478bd9Sstevel@tonic-gate 	fmd_hdl_debug(hdl, "page_retry: timer fired\n");
2557c478bd9Sstevel@tonic-gate 
2567c478bd9Sstevel@tonic-gate 	pagep = &cma.cma_pages;
2577c478bd9Sstevel@tonic-gate 	while (*pagep != NULL) {
2587c478bd9Sstevel@tonic-gate 		cma_page_t *page = *pagep;
2597c478bd9Sstevel@tonic-gate 
2607c478bd9Sstevel@tonic-gate 		if (page_retry(hdl, page)) {
2617c478bd9Sstevel@tonic-gate 			/*
2627c478bd9Sstevel@tonic-gate 			 * Successful retry or we're giving up - remove from
2637c478bd9Sstevel@tonic-gate 			 * the list
2647c478bd9Sstevel@tonic-gate 			 */
2657c478bd9Sstevel@tonic-gate 			*pagep = page->pg_next;
2667c478bd9Sstevel@tonic-gate 
2677c478bd9Sstevel@tonic-gate 			if (page->pg_uuid != NULL)
2687c478bd9Sstevel@tonic-gate 				fmd_hdl_strfree(hdl, page->pg_uuid);
2697c478bd9Sstevel@tonic-gate 
2707c478bd9Sstevel@tonic-gate 			cma_page_free(hdl, page);
27178432d5eScy 		} else {
2727c478bd9Sstevel@tonic-gate 			page->pg_nretries++;
2737c478bd9Sstevel@tonic-gate 			pagep = &page->pg_next;
2747c478bd9Sstevel@tonic-gate 		}
2757c478bd9Sstevel@tonic-gate 	}
2767c478bd9Sstevel@tonic-gate 
2777c478bd9Sstevel@tonic-gate 	if (cma.cma_pages == NULL)
2787c478bd9Sstevel@tonic-gate 		return; /* no more retirements */
2797c478bd9Sstevel@tonic-gate 
2807c478bd9Sstevel@tonic-gate 	/*
2817c478bd9Sstevel@tonic-gate 	 * We still have retirements that haven't completed.  Back the delay
2827c478bd9Sstevel@tonic-gate 	 * off, and schedule a retry.
2837c478bd9Sstevel@tonic-gate 	 */
2847c478bd9Sstevel@tonic-gate 	cma.cma_page_curdelay = MIN(cma.cma_page_curdelay * 2,
2857c478bd9Sstevel@tonic-gate 	    cma.cma_page_maxdelay);
2867c478bd9Sstevel@tonic-gate 
2877c478bd9Sstevel@tonic-gate 	fmd_hdl_debug(hdl, "scheduled page retirement retry for %llu secs\n",
2887c478bd9Sstevel@tonic-gate 	    (u_longlong_t)(cma.cma_page_curdelay / NANOSEC));
2897c478bd9Sstevel@tonic-gate 
2907c478bd9Sstevel@tonic-gate 	cma.cma_page_timerid =
2917c478bd9Sstevel@tonic-gate 	    fmd_timer_install(hdl, NULL, NULL, cma.cma_page_curdelay);
2927c478bd9Sstevel@tonic-gate }
2937c478bd9Sstevel@tonic-gate 
2947c478bd9Sstevel@tonic-gate void
cma_page_fini(fmd_hdl_t * hdl)2957c478bd9Sstevel@tonic-gate cma_page_fini(fmd_hdl_t *hdl)
2967c478bd9Sstevel@tonic-gate {
2977c478bd9Sstevel@tonic-gate 	cma_page_t *page;
2987c478bd9Sstevel@tonic-gate 
2997c478bd9Sstevel@tonic-gate 	while ((page = cma.cma_pages) != NULL) {
3007c478bd9Sstevel@tonic-gate 		cma.cma_pages = page->pg_next;
301*3f1e69beSCheng Sean Ye 		if (page->pg_uuid != NULL)
302*3f1e69beSCheng Sean Ye 			fmd_hdl_strfree(hdl, page->pg_uuid);
3037c478bd9Sstevel@tonic-gate 		cma_page_free(hdl, page);
3047c478bd9Sstevel@tonic-gate 	}
3057c478bd9Sstevel@tonic-gate }
306