17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
57f0b8309SEdward Pilatowicz  * Common Development and Distribution License (the "License").
67f0b8309SEdward Pilatowicz  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
227f0b8309SEdward Pilatowicz  *	Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  *	Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #include <sys/cred.h>
277c478bd9Sstevel@tonic-gate #include <sys/kstat.h>
287c478bd9Sstevel@tonic-gate #include <sys/list.h>
297c478bd9Sstevel@tonic-gate #include <sys/systm.h>
307c478bd9Sstevel@tonic-gate #include <sys/vfs.h>
317c478bd9Sstevel@tonic-gate #include <sys/vnode.h>
327c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
337c478bd9Sstevel@tonic-gate 
347c478bd9Sstevel@tonic-gate #include <nfs/nfs4_clnt.h>
357c478bd9Sstevel@tonic-gate #include <nfs/rnode4.h>
367c478bd9Sstevel@tonic-gate 
377c478bd9Sstevel@tonic-gate /*
387c478bd9Sstevel@tonic-gate  * Recovery kstats
397c478bd9Sstevel@tonic-gate  */
407c478bd9Sstevel@tonic-gate typedef struct rkstat {
417c478bd9Sstevel@tonic-gate 	kstat_named_t	badhandle;
427c478bd9Sstevel@tonic-gate 	kstat_named_t	badowner;
437c478bd9Sstevel@tonic-gate 	kstat_named_t	clientid;
447c478bd9Sstevel@tonic-gate 	kstat_named_t	dead_file;
457c478bd9Sstevel@tonic-gate 	kstat_named_t	delay;
467c478bd9Sstevel@tonic-gate 	kstat_named_t	fail_relock;
477c478bd9Sstevel@tonic-gate 	kstat_named_t	file_diff;
487c478bd9Sstevel@tonic-gate 	kstat_named_t	no_grace;
497c478bd9Sstevel@tonic-gate 	kstat_named_t	not_responding;
507c478bd9Sstevel@tonic-gate 	kstat_named_t	opens_changed;
517c478bd9Sstevel@tonic-gate 	kstat_named_t	siglost;
527c478bd9Sstevel@tonic-gate 	kstat_named_t	unexp_action;
537c478bd9Sstevel@tonic-gate 	kstat_named_t	unexp_errno;
547c478bd9Sstevel@tonic-gate 	kstat_named_t	unexp_status;
557c478bd9Sstevel@tonic-gate 	kstat_named_t	wrongsec;
567c478bd9Sstevel@tonic-gate 	kstat_named_t	lost_state_bad_op;
577c478bd9Sstevel@tonic-gate } rkstat_t;
587c478bd9Sstevel@tonic-gate 
597c478bd9Sstevel@tonic-gate static rkstat_t rkstat_template = {
607c478bd9Sstevel@tonic-gate 	{ "badhandle",		KSTAT_DATA_ULONG },
617c478bd9Sstevel@tonic-gate 	{ "badowner",		KSTAT_DATA_ULONG },
627c478bd9Sstevel@tonic-gate 	{ "clientid",		KSTAT_DATA_ULONG },
637c478bd9Sstevel@tonic-gate 	{ "dead_file",		KSTAT_DATA_ULONG },
647c478bd9Sstevel@tonic-gate 	{ "delay",		KSTAT_DATA_ULONG },
657c478bd9Sstevel@tonic-gate 	{ "fail_relock",	KSTAT_DATA_ULONG },
667c478bd9Sstevel@tonic-gate 	{ "file_diff",		KSTAT_DATA_ULONG },
677c478bd9Sstevel@tonic-gate 	{ "no_grace",		KSTAT_DATA_ULONG },
687c478bd9Sstevel@tonic-gate 	{ "not_responding",	KSTAT_DATA_ULONG },
697c478bd9Sstevel@tonic-gate 	{ "opens_changed",	KSTAT_DATA_ULONG },
707c478bd9Sstevel@tonic-gate 	{ "siglost",		KSTAT_DATA_ULONG },
717c478bd9Sstevel@tonic-gate 	{ "unexp_action",	KSTAT_DATA_ULONG },
727c478bd9Sstevel@tonic-gate 	{ "unexp_errno",	KSTAT_DATA_ULONG },
737c478bd9Sstevel@tonic-gate 	{ "unexp_status",	KSTAT_DATA_ULONG },
747c478bd9Sstevel@tonic-gate 	{ "wrongsec",		KSTAT_DATA_ULONG },
757c478bd9Sstevel@tonic-gate 	{ "bad_op",		KSTAT_DATA_ULONG },
767c478bd9Sstevel@tonic-gate };
777c478bd9Sstevel@tonic-gate 
787c478bd9Sstevel@tonic-gate /* maximum number of messages allowed on the mi's mi_msg_list */
797c478bd9Sstevel@tonic-gate int nfs4_msg_max = NFS4_MSG_MAX;
807c478bd9Sstevel@tonic-gate #define	DEFAULT_LEASE	180
817c478bd9Sstevel@tonic-gate 
827c478bd9Sstevel@tonic-gate /*
837c478bd9Sstevel@tonic-gate  * Sets the appropiate fields of "ep", given "id" and various parameters.
847c478bd9Sstevel@tonic-gate  * Assumes that ep's fields have been initialized to zero/null, except for
857c478bd9Sstevel@tonic-gate  * re_type and mount point info, which are already set.
867c478bd9Sstevel@tonic-gate  */
877c478bd9Sstevel@tonic-gate static void
set_event(nfs4_event_type_t id,nfs4_revent_t * ep,mntinfo4_t * mi,rnode4_t * rp1,rnode4_t * rp2,uint_t count,pid_t pid,nfsstat4 nfs4_error,char * server1,char * why,nfs4_tag_type_t tag1,nfs4_tag_type_t tag2,seqid4 seqid1,seqid4 seqid2)887c478bd9Sstevel@tonic-gate set_event(nfs4_event_type_t id, nfs4_revent_t *ep, mntinfo4_t *mi,
897c478bd9Sstevel@tonic-gate     rnode4_t *rp1, rnode4_t *rp2, uint_t count, pid_t pid, nfsstat4 nfs4_error,
907c478bd9Sstevel@tonic-gate     char *server1, char *why, nfs4_tag_type_t tag1, nfs4_tag_type_t tag2,
917c478bd9Sstevel@tonic-gate     seqid4 seqid1, seqid4 seqid2)
927c478bd9Sstevel@tonic-gate {
937c478bd9Sstevel@tonic-gate 	int len;
947c478bd9Sstevel@tonic-gate 
957c478bd9Sstevel@tonic-gate 	switch (id) {
967c478bd9Sstevel@tonic-gate 	case RE_BAD_SEQID:
977c478bd9Sstevel@tonic-gate 		ep->re_mi = mi;
987c478bd9Sstevel@tonic-gate 
997c478bd9Sstevel@tonic-gate 		/* bad seqid'd file <path/component name> */
1007c478bd9Sstevel@tonic-gate 		if (rp1 && rp1->r_svnode.sv_name)
1017c478bd9Sstevel@tonic-gate 			ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
1027c478bd9Sstevel@tonic-gate 		else
1037c478bd9Sstevel@tonic-gate 			ep->re_char1 = NULL;
1047c478bd9Sstevel@tonic-gate 		ep->re_rp1 = rp1;
1057c478bd9Sstevel@tonic-gate 
1067c478bd9Sstevel@tonic-gate 		/* for LOCK/LOCKU */
1077c478bd9Sstevel@tonic-gate 		ep->re_pid = pid;
1087c478bd9Sstevel@tonic-gate 
1097c478bd9Sstevel@tonic-gate 		ep->re_stat4 = nfs4_error;
1107c478bd9Sstevel@tonic-gate 		ep->re_tag1 = tag1;
1117c478bd9Sstevel@tonic-gate 		ep->re_tag2 = tag2;
1127c478bd9Sstevel@tonic-gate 		ep->re_seqid1 = seqid1;
1137c478bd9Sstevel@tonic-gate 		ep->re_seqid2 = seqid2;
1147c478bd9Sstevel@tonic-gate 		break;
1157c478bd9Sstevel@tonic-gate 	case RE_BADHANDLE:
1167c478bd9Sstevel@tonic-gate 		ASSERT(rp1 != NULL);
1177c478bd9Sstevel@tonic-gate 
1187c478bd9Sstevel@tonic-gate 		/* dead file <path/component name> */
1197c478bd9Sstevel@tonic-gate 		if (rp1->r_svnode.sv_name)
1207c478bd9Sstevel@tonic-gate 			ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
1217c478bd9Sstevel@tonic-gate 		else
1227c478bd9Sstevel@tonic-gate 			ep->re_char1 = NULL;
1237c478bd9Sstevel@tonic-gate 		ep->re_rp1 = rp1;
1247c478bd9Sstevel@tonic-gate 		break;
1257c478bd9Sstevel@tonic-gate 	case RE_CLIENTID:
1267c478bd9Sstevel@tonic-gate 		ep->re_mi = mi;
1277c478bd9Sstevel@tonic-gate 
1287c478bd9Sstevel@tonic-gate 		/* the error we failed with */
1297c478bd9Sstevel@tonic-gate 		ep->re_uint = count;
1307c478bd9Sstevel@tonic-gate 		ep->re_stat4 = nfs4_error;
1317c478bd9Sstevel@tonic-gate 		break;
1327c478bd9Sstevel@tonic-gate 	case RE_DEAD_FILE:
1337c478bd9Sstevel@tonic-gate 		ASSERT(rp1 != NULL);
1347c478bd9Sstevel@tonic-gate 
1357c478bd9Sstevel@tonic-gate 		/* dead file <path/component name> */
1367c478bd9Sstevel@tonic-gate 		if (rp1->r_svnode.sv_name)
1377c478bd9Sstevel@tonic-gate 			ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
1387c478bd9Sstevel@tonic-gate 		else
1397c478bd9Sstevel@tonic-gate 			ep->re_char1 = NULL;
1407c478bd9Sstevel@tonic-gate 		ep->re_rp1 = rp1;
1417c478bd9Sstevel@tonic-gate 
1427c478bd9Sstevel@tonic-gate 		/* why the file got killed */
1437c478bd9Sstevel@tonic-gate 		if (why) {
1447c478bd9Sstevel@tonic-gate 			len = strlen(why);
1457c478bd9Sstevel@tonic-gate 			ep->re_char2 = kmem_alloc(len + 1, KM_SLEEP);
1467c478bd9Sstevel@tonic-gate 			bcopy(why, ep->re_char2, len);
1477c478bd9Sstevel@tonic-gate 			ep->re_char2[len] = '\0';
1487c478bd9Sstevel@tonic-gate 		} else
1497c478bd9Sstevel@tonic-gate 			ep->re_char2 = NULL;
1507c478bd9Sstevel@tonic-gate 
1517c478bd9Sstevel@tonic-gate 		ep->re_stat4 = nfs4_error;
1527c478bd9Sstevel@tonic-gate 		break;
1537c478bd9Sstevel@tonic-gate 	case RE_END:
1547c478bd9Sstevel@tonic-gate 		/* first rnode */
1557c478bd9Sstevel@tonic-gate 		if (rp1 && rp1->r_svnode.sv_name)
1567c478bd9Sstevel@tonic-gate 			ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
1577c478bd9Sstevel@tonic-gate 		else
1587c478bd9Sstevel@tonic-gate 			ep->re_char1 = NULL;
1597c478bd9Sstevel@tonic-gate 		ep->re_rp1 = rp1;
1607c478bd9Sstevel@tonic-gate 
1617c478bd9Sstevel@tonic-gate 		/* second rnode */
1627c478bd9Sstevel@tonic-gate 		if (rp2 && rp2->r_svnode.sv_name)
1637c478bd9Sstevel@tonic-gate 			ep->re_char2 = fn_path(rp2->r_svnode.sv_name);
1647c478bd9Sstevel@tonic-gate 		else
1657c478bd9Sstevel@tonic-gate 			ep->re_char2 = NULL;
1667c478bd9Sstevel@tonic-gate 		ep->re_rp2 = rp2;
1677c478bd9Sstevel@tonic-gate 
1687c478bd9Sstevel@tonic-gate 		ep->re_mi = mi;
1697c478bd9Sstevel@tonic-gate 		break;
1707c478bd9Sstevel@tonic-gate 	case RE_FAIL_RELOCK:
1717c478bd9Sstevel@tonic-gate 		ASSERT(rp1 != NULL);
1727c478bd9Sstevel@tonic-gate 
1737c478bd9Sstevel@tonic-gate 		/* error on fail relock */
1747c478bd9Sstevel@tonic-gate 		ep->re_uint = count;
1757c478bd9Sstevel@tonic-gate 
1767c478bd9Sstevel@tonic-gate 		/* process that failed */
1777c478bd9Sstevel@tonic-gate 		ep->re_pid = pid;
1787c478bd9Sstevel@tonic-gate 
1797c478bd9Sstevel@tonic-gate 		/* nfs4 error */
1807c478bd9Sstevel@tonic-gate 		ep->re_stat4 = nfs4_error;
1817c478bd9Sstevel@tonic-gate 
1827c478bd9Sstevel@tonic-gate 		/* file <path/component name> */
1837c478bd9Sstevel@tonic-gate 		if (rp1->r_svnode.sv_name)
1847c478bd9Sstevel@tonic-gate 			ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
1857c478bd9Sstevel@tonic-gate 		else
1867c478bd9Sstevel@tonic-gate 			ep->re_char1 = NULL;
1877c478bd9Sstevel@tonic-gate 		ep->re_rp1 = rp1;
1887c478bd9Sstevel@tonic-gate 		break;
1897c478bd9Sstevel@tonic-gate 	case RE_FAIL_REMAP_LEN:
1907c478bd9Sstevel@tonic-gate 		/* length of returned filehandle */
1917c478bd9Sstevel@tonic-gate 		ep->re_uint = count;
1927c478bd9Sstevel@tonic-gate 		break;
1937c478bd9Sstevel@tonic-gate 	case RE_FAIL_REMAP_OP:
1947c478bd9Sstevel@tonic-gate 		break;
1957c478bd9Sstevel@tonic-gate 	case RE_FAILOVER:
1967c478bd9Sstevel@tonic-gate 		/* server we're failing over to (if not picking original) */
1977c478bd9Sstevel@tonic-gate 		if (server1 != NULL) {
1987c478bd9Sstevel@tonic-gate 			len = strlen(server1);
1997c478bd9Sstevel@tonic-gate 			ep->re_char1 = kmem_alloc(len + 1, KM_SLEEP);
2007c478bd9Sstevel@tonic-gate 			bcopy(server1, ep->re_char1, len);
2017c478bd9Sstevel@tonic-gate 			ep->re_char1[len] = '\0';
2027c478bd9Sstevel@tonic-gate 		} else {
2037c478bd9Sstevel@tonic-gate 			ep->re_char1 = NULL;
2047c478bd9Sstevel@tonic-gate 		}
2057c478bd9Sstevel@tonic-gate 		break;
2067c478bd9Sstevel@tonic-gate 	case RE_FILE_DIFF:
2077c478bd9Sstevel@tonic-gate 		ASSERT(rp1 != NULL);
2087c478bd9Sstevel@tonic-gate 
2097c478bd9Sstevel@tonic-gate 		/* dead file <path/component name> */
2107c478bd9Sstevel@tonic-gate 		if (rp1->r_svnode.sv_name)
2117c478bd9Sstevel@tonic-gate 			ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
2127c478bd9Sstevel@tonic-gate 		else
2137c478bd9Sstevel@tonic-gate 			ep->re_char1 = NULL;
2147c478bd9Sstevel@tonic-gate 		ep->re_rp1 = rp1;
2157c478bd9Sstevel@tonic-gate 		break;
2167c478bd9Sstevel@tonic-gate 	case RE_LOST_STATE:
2177c478bd9Sstevel@tonic-gate 		ep->re_uint = count;		/* op number */
2187c478bd9Sstevel@tonic-gate 		if (rp1 && rp1->r_svnode.sv_name)
2197c478bd9Sstevel@tonic-gate 			ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
2207c478bd9Sstevel@tonic-gate 		else
2217c478bd9Sstevel@tonic-gate 			ep->re_char1 = NULL;
2227c478bd9Sstevel@tonic-gate 		ep->re_rp1 = rp1;
2237c478bd9Sstevel@tonic-gate 		if (rp2 && rp2->r_svnode.sv_name)
2247c478bd9Sstevel@tonic-gate 			ep->re_char2 = fn_path(rp2->r_svnode.sv_name);
2257c478bd9Sstevel@tonic-gate 		else
2267c478bd9Sstevel@tonic-gate 			ep->re_char2 = NULL;
2277c478bd9Sstevel@tonic-gate 		ep->re_rp2 = rp2;
2287c478bd9Sstevel@tonic-gate 		break;
2297c478bd9Sstevel@tonic-gate 	case RE_OPENS_CHANGED:
2307c478bd9Sstevel@tonic-gate 		ep->re_mi = mi;
2317c478bd9Sstevel@tonic-gate 
2327c478bd9Sstevel@tonic-gate 		/* original number of open files */
2337c478bd9Sstevel@tonic-gate 		ep->re_uint = count;
2347c478bd9Sstevel@tonic-gate 		/* new number of open files */
2357c478bd9Sstevel@tonic-gate 		ep->re_pid = pid;
2367c478bd9Sstevel@tonic-gate 		break;
2377c478bd9Sstevel@tonic-gate 	case RE_SIGLOST:
2387c478bd9Sstevel@tonic-gate 	case RE_SIGLOST_NO_DUMP:
2397c478bd9Sstevel@tonic-gate 		ASSERT(rp1 != NULL);
2407c478bd9Sstevel@tonic-gate 
2417c478bd9Sstevel@tonic-gate 		/* file <path/component name> */
2427c478bd9Sstevel@tonic-gate 		if (rp1->r_svnode.sv_name)
2437c478bd9Sstevel@tonic-gate 			ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
2447c478bd9Sstevel@tonic-gate 		else
2457c478bd9Sstevel@tonic-gate 			ep->re_char1 = NULL;
2467c478bd9Sstevel@tonic-gate 		ep->re_rp1 = rp1;
2477c478bd9Sstevel@tonic-gate 		ep->re_pid = pid;
2487c478bd9Sstevel@tonic-gate 		ep->re_uint = count;
2497c478bd9Sstevel@tonic-gate 		ep->re_stat4 = nfs4_error;
2507c478bd9Sstevel@tonic-gate 		break;
2517c478bd9Sstevel@tonic-gate 	case RE_START:
2527c478bd9Sstevel@tonic-gate 		/* file <path/component name> */
2537c478bd9Sstevel@tonic-gate 		if (rp1 && rp1->r_svnode.sv_name)
2547c478bd9Sstevel@tonic-gate 			ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
2557c478bd9Sstevel@tonic-gate 		else
2567c478bd9Sstevel@tonic-gate 			ep->re_char1 = NULL;
2577c478bd9Sstevel@tonic-gate 		ep->re_rp1 = rp1;
2587c478bd9Sstevel@tonic-gate 
2597c478bd9Sstevel@tonic-gate 		/* file <path/component name> */
2607c478bd9Sstevel@tonic-gate 		if (rp2 && rp2->r_svnode.sv_name)
2617c478bd9Sstevel@tonic-gate 			ep->re_char2 = fn_path(rp2->r_svnode.sv_name);
2627c478bd9Sstevel@tonic-gate 		else
2637c478bd9Sstevel@tonic-gate 			ep->re_char2 = NULL;
2647c478bd9Sstevel@tonic-gate 		ep->re_rp2 = rp2;
2657c478bd9Sstevel@tonic-gate 
2667c478bd9Sstevel@tonic-gate 		ep->re_mi = mi;
2677c478bd9Sstevel@tonic-gate 		ep->re_uint = count;
2687c478bd9Sstevel@tonic-gate 		break;
2697c478bd9Sstevel@tonic-gate 	case RE_UNEXPECTED_ACTION:
2707c478bd9Sstevel@tonic-gate 	case RE_UNEXPECTED_ERRNO:
2717c478bd9Sstevel@tonic-gate 		/* the error that is unexpected */
2727c478bd9Sstevel@tonic-gate 		ep->re_uint = count;
2737c478bd9Sstevel@tonic-gate 		break;
2747c478bd9Sstevel@tonic-gate 	case RE_UNEXPECTED_STATUS:
2757c478bd9Sstevel@tonic-gate 		/* nfsstat4 error */
2767c478bd9Sstevel@tonic-gate 		ep->re_stat4 = nfs4_error;
2777c478bd9Sstevel@tonic-gate 		break;
2787c478bd9Sstevel@tonic-gate 	case RE_WRONGSEC:
2797c478bd9Sstevel@tonic-gate 		/* the error we failed with */
2807c478bd9Sstevel@tonic-gate 		ep->re_uint = count;
2817c478bd9Sstevel@tonic-gate 
2827c478bd9Sstevel@tonic-gate 		/* file <path/component name> */
2837c478bd9Sstevel@tonic-gate 		if (rp1 && rp1->r_svnode.sv_name)
2847c478bd9Sstevel@tonic-gate 			ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
2857c478bd9Sstevel@tonic-gate 		else
2867c478bd9Sstevel@tonic-gate 			ep->re_char1 = NULL;
2877c478bd9Sstevel@tonic-gate 		ep->re_rp1 = rp1;
2887c478bd9Sstevel@tonic-gate 
2897c478bd9Sstevel@tonic-gate 		/* file <path/component name> */
2907c478bd9Sstevel@tonic-gate 		if (rp2 && rp2->r_svnode.sv_name)
2917c478bd9Sstevel@tonic-gate 			ep->re_char2 = fn_path(rp2->r_svnode.sv_name);
2927c478bd9Sstevel@tonic-gate 		else
2937c478bd9Sstevel@tonic-gate 			ep->re_char2 = NULL;
2947c478bd9Sstevel@tonic-gate 		ep->re_rp2 = rp2;
2957c478bd9Sstevel@tonic-gate 		break;
2967c478bd9Sstevel@tonic-gate 	case RE_LOST_STATE_BAD_OP:
2977c478bd9Sstevel@tonic-gate 		ep->re_uint = count;	/* the unexpected op */
2987c478bd9Sstevel@tonic-gate 		ep->re_pid = pid;
2997c478bd9Sstevel@tonic-gate 		ep->re_rp1 = rp1;
3007c478bd9Sstevel@tonic-gate 		if (rp1 != NULL && rp1->r_svnode.sv_name != NULL)
3017c478bd9Sstevel@tonic-gate 			ep->re_char1 = fn_path(rp1->r_svnode.sv_name);
3027c478bd9Sstevel@tonic-gate 		ep->re_rp2 = rp2;
3037c478bd9Sstevel@tonic-gate 		if (rp2 != NULL && rp2->r_svnode.sv_name != NULL)
3047c478bd9Sstevel@tonic-gate 			ep->re_char2 = fn_path(rp2->r_svnode.sv_name);
3057c478bd9Sstevel@tonic-gate 		break;
306*2f172c55SRobert Thurlow 	case RE_REFERRAL:
307*2f172c55SRobert Thurlow 		/* server we're being referred to */
308*2f172c55SRobert Thurlow 		if (server1 != NULL) {
309*2f172c55SRobert Thurlow 			len = strlen(server1);
310*2f172c55SRobert Thurlow 			ep->re_char1 = kmem_alloc(len + 1, KM_SLEEP);
311*2f172c55SRobert Thurlow 			bcopy(server1, ep->re_char1, len);
312*2f172c55SRobert Thurlow 			ep->re_char1[len] = '\0';
313*2f172c55SRobert Thurlow 		} else {
314*2f172c55SRobert Thurlow 			ep->re_char1 = NULL;
315*2f172c55SRobert Thurlow 		}
316*2f172c55SRobert Thurlow 		break;
3177c478bd9Sstevel@tonic-gate 	default:
3187c478bd9Sstevel@tonic-gate 		break;
3197c478bd9Sstevel@tonic-gate 	}
3207c478bd9Sstevel@tonic-gate }
3217c478bd9Sstevel@tonic-gate 
3227c478bd9Sstevel@tonic-gate /*
3237c478bd9Sstevel@tonic-gate  * Sets the appropiate fields of the 'fact' for this 'id'.
3247c478bd9Sstevel@tonic-gate  */
3257c478bd9Sstevel@tonic-gate static void
set_fact(nfs4_fact_type_t id,nfs4_rfact_t * fp,nfsstat4 stat4,nfs4_recov_t raction,nfs_opnum4 op,bool_t reboot,int error,vnode_t * vp)3267c478bd9Sstevel@tonic-gate set_fact(nfs4_fact_type_t id, nfs4_rfact_t *fp, nfsstat4 stat4,
3277c478bd9Sstevel@tonic-gate     nfs4_recov_t raction, nfs_opnum4 op, bool_t reboot, int error,
3287c478bd9Sstevel@tonic-gate     vnode_t *vp)
3297c478bd9Sstevel@tonic-gate {
3307c478bd9Sstevel@tonic-gate 	rnode4_t *rp1;
3317c478bd9Sstevel@tonic-gate 
3327c478bd9Sstevel@tonic-gate 	switch (id) {
3337c478bd9Sstevel@tonic-gate 	case RF_BADOWNER:
3347c478bd9Sstevel@tonic-gate 		fp->rf_op = op;
3357c478bd9Sstevel@tonic-gate 		fp->rf_reboot = reboot;
3367c478bd9Sstevel@tonic-gate 		fp->rf_stat4 = stat4;
3377c478bd9Sstevel@tonic-gate 		break;
3387c478bd9Sstevel@tonic-gate 	case RF_RENEW_EXPIRED:
3397c478bd9Sstevel@tonic-gate 		break;
3407c478bd9Sstevel@tonic-gate 	case RF_ERR:
3417c478bd9Sstevel@tonic-gate 		fp->rf_op = op;
3427c478bd9Sstevel@tonic-gate 		fp->rf_reboot = reboot;
3437c478bd9Sstevel@tonic-gate 		fp->rf_stat4 = stat4;
3447c478bd9Sstevel@tonic-gate 		fp->rf_action = raction;
3457c478bd9Sstevel@tonic-gate 		fp->rf_error = error;
3467c478bd9Sstevel@tonic-gate 		break;
3477c478bd9Sstevel@tonic-gate 	case RF_SRV_OK:
3487c478bd9Sstevel@tonic-gate 		break;
3497c478bd9Sstevel@tonic-gate 	case RF_SRV_NOT_RESPOND:
3507c478bd9Sstevel@tonic-gate 		break;
3517c478bd9Sstevel@tonic-gate 	case RF_SRVS_OK:
3527c478bd9Sstevel@tonic-gate 		break;
3537c478bd9Sstevel@tonic-gate 	case RF_SRVS_NOT_RESPOND:
3547c478bd9Sstevel@tonic-gate 		gethrestime(&fp->rf_time);
3557c478bd9Sstevel@tonic-gate 		break;
3567c478bd9Sstevel@tonic-gate 	case RF_DELMAP_CB_ERR:
3577c478bd9Sstevel@tonic-gate 		fp->rf_op = op;
3587c478bd9Sstevel@tonic-gate 		fp->rf_stat4 = stat4;
3597c478bd9Sstevel@tonic-gate 
3607c478bd9Sstevel@tonic-gate 		rp1 = VTOR4(vp);
3617c478bd9Sstevel@tonic-gate 		fp->rf_rp1 = rp1;
3627c478bd9Sstevel@tonic-gate 		if (rp1 && rp1->r_svnode.sv_name)
3637c478bd9Sstevel@tonic-gate 			fp->rf_char1 = fn_path(rp1->r_svnode.sv_name);
3647c478bd9Sstevel@tonic-gate 		else
3657c478bd9Sstevel@tonic-gate 			fp->rf_char1 = NULL;
3667c478bd9Sstevel@tonic-gate 		break;
367e280ed37SDai Ngo 	case RF_SENDQ_FULL:
368e280ed37SDai Ngo 		break;
3697c478bd9Sstevel@tonic-gate 	default:
3707c478bd9Sstevel@tonic-gate 		zcmn_err(getzoneid(), CE_NOTE, "illegal fact %d", id);
3717c478bd9Sstevel@tonic-gate 		break;
3727c478bd9Sstevel@tonic-gate 	}
3737c478bd9Sstevel@tonic-gate }
3747c478bd9Sstevel@tonic-gate 
3757c478bd9Sstevel@tonic-gate /*
3767c478bd9Sstevel@tonic-gate  * Returns 1 if the event/fact is of a successful communication
3777c478bd9Sstevel@tonic-gate  * from the server; 0 otherwise.
3787c478bd9Sstevel@tonic-gate  */
3797c478bd9Sstevel@tonic-gate static int
successful_comm(nfs4_debug_msg_t * msgp)3807c478bd9Sstevel@tonic-gate successful_comm(nfs4_debug_msg_t *msgp)
3817c478bd9Sstevel@tonic-gate {
3827c478bd9Sstevel@tonic-gate 	if (msgp->msg_type == RM_EVENT) {
3837c478bd9Sstevel@tonic-gate 		switch (msgp->rmsg_u.msg_event.re_type) {
3847c478bd9Sstevel@tonic-gate 		case RE_BAD_SEQID:
3857c478bd9Sstevel@tonic-gate 		case RE_BADHANDLE:
3867c478bd9Sstevel@tonic-gate 		case RE_FAIL_REMAP_LEN:
3877c478bd9Sstevel@tonic-gate 		case RE_FAIL_REMAP_OP:
3887c478bd9Sstevel@tonic-gate 		case RE_FILE_DIFF:
3897c478bd9Sstevel@tonic-gate 		case RE_START:
3907c478bd9Sstevel@tonic-gate 		case RE_UNEXPECTED_ACTION:
3917c478bd9Sstevel@tonic-gate 		case RE_UNEXPECTED_ERRNO:
3927c478bd9Sstevel@tonic-gate 		case RE_UNEXPECTED_STATUS:
3937c478bd9Sstevel@tonic-gate 		case RE_WRONGSEC:
3947c478bd9Sstevel@tonic-gate 			return (1);
3957c478bd9Sstevel@tonic-gate 		case RE_CLIENTID:
3967c478bd9Sstevel@tonic-gate 		case RE_DEAD_FILE:
3977c478bd9Sstevel@tonic-gate 		case RE_END:
3987c478bd9Sstevel@tonic-gate 		case RE_FAIL_RELOCK:
3997c478bd9Sstevel@tonic-gate 		case RE_FAILOVER:
4007c478bd9Sstevel@tonic-gate 		case RE_LOST_STATE:
4017c478bd9Sstevel@tonic-gate 		case RE_OPENS_CHANGED:
4027c478bd9Sstevel@tonic-gate 		case RE_SIGLOST:
4037c478bd9Sstevel@tonic-gate 		case RE_SIGLOST_NO_DUMP:
4047c478bd9Sstevel@tonic-gate 		case RE_LOST_STATE_BAD_OP:
405*2f172c55SRobert Thurlow 		case RE_REFERRAL:
406*2f172c55SRobert Thurlow 			/* placeholder */
4077c478bd9Sstevel@tonic-gate 			return (0);
4087c478bd9Sstevel@tonic-gate 		default:
4097c478bd9Sstevel@tonic-gate 			return (0);
4107c478bd9Sstevel@tonic-gate 		}
4117c478bd9Sstevel@tonic-gate 	} else {
4127c478bd9Sstevel@tonic-gate 		switch (msgp->rmsg_u.msg_fact.rf_type) {
4137c478bd9Sstevel@tonic-gate 		case RF_BADOWNER:
4147c478bd9Sstevel@tonic-gate 		case RF_ERR:
4157c478bd9Sstevel@tonic-gate 		case RF_RENEW_EXPIRED:
4167c478bd9Sstevel@tonic-gate 		case RF_SRV_OK:
4177c478bd9Sstevel@tonic-gate 		case RF_SRVS_OK:
4187c478bd9Sstevel@tonic-gate 		case RF_DELMAP_CB_ERR:
4197c478bd9Sstevel@tonic-gate 			return (1);
4207c478bd9Sstevel@tonic-gate 		case RF_SRV_NOT_RESPOND:
4217c478bd9Sstevel@tonic-gate 		case RF_SRVS_NOT_RESPOND:
422e280ed37SDai Ngo 		case RF_SENDQ_FULL:
4237c478bd9Sstevel@tonic-gate 			return (0);
4247c478bd9Sstevel@tonic-gate 		default:
4257c478bd9Sstevel@tonic-gate 			return (0);
4267c478bd9Sstevel@tonic-gate 		}
4277c478bd9Sstevel@tonic-gate 	}
4287c478bd9Sstevel@tonic-gate }
4297c478bd9Sstevel@tonic-gate 
4307c478bd9Sstevel@tonic-gate /*
4317c478bd9Sstevel@tonic-gate  * Iterate backwards through the mi's mi_msg_list to find the earliest
4327c478bd9Sstevel@tonic-gate  * message that we should find relevant facts to investigate.
4337c478bd9Sstevel@tonic-gate  */
4347c478bd9Sstevel@tonic-gate static nfs4_debug_msg_t *
find_beginning(nfs4_debug_msg_t * first_msg,mntinfo4_t * mi)4357c478bd9Sstevel@tonic-gate find_beginning(nfs4_debug_msg_t *first_msg, mntinfo4_t *mi)
4367c478bd9Sstevel@tonic-gate {
4377c478bd9Sstevel@tonic-gate 	nfs4_debug_msg_t	*oldest_msg, *cur_msg;
4387c478bd9Sstevel@tonic-gate 	time_t			lease;
4397c478bd9Sstevel@tonic-gate 
4407c478bd9Sstevel@tonic-gate 	ASSERT(mutex_owned(&mi->mi_msg_list_lock));
4417c478bd9Sstevel@tonic-gate 	if (mi->mi_lease_period > 0)
4427c478bd9Sstevel@tonic-gate 		lease = 2 * mi->mi_lease_period;
4437c478bd9Sstevel@tonic-gate 	else
4447c478bd9Sstevel@tonic-gate 		lease = DEFAULT_LEASE;
4457c478bd9Sstevel@tonic-gate 
4467c478bd9Sstevel@tonic-gate 	oldest_msg = first_msg;
4477c478bd9Sstevel@tonic-gate 	cur_msg = list_prev(&mi->mi_msg_list, first_msg);
4487c478bd9Sstevel@tonic-gate 	while (cur_msg &&
4497c478bd9Sstevel@tonic-gate 	    first_msg->msg_time.tv_sec - cur_msg->msg_time.tv_sec < lease) {
4507c478bd9Sstevel@tonic-gate 		oldest_msg = cur_msg;
4517c478bd9Sstevel@tonic-gate 		if ((cur_msg->msg_type == RM_FACT) &&
4527c478bd9Sstevel@tonic-gate 		    (cur_msg->rmsg_u.msg_fact.rf_type == RF_SRV_OK)) {
4537c478bd9Sstevel@tonic-gate 			/* find where we lost contact with the server */
4547c478bd9Sstevel@tonic-gate 			while (cur_msg) {
4557c478bd9Sstevel@tonic-gate 				if ((cur_msg->msg_type == RM_FACT) &&
4567c478bd9Sstevel@tonic-gate 				    (cur_msg->rmsg_u.msg_fact.rf_type ==
4577c478bd9Sstevel@tonic-gate 				    RF_SRV_NOT_RESPOND))
4587c478bd9Sstevel@tonic-gate 					break;
4597c478bd9Sstevel@tonic-gate 				oldest_msg = cur_msg;
4607c478bd9Sstevel@tonic-gate 				cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
4617c478bd9Sstevel@tonic-gate 			}
4627c478bd9Sstevel@tonic-gate 			/*
4637c478bd9Sstevel@tonic-gate 			 * Find the first successful message before
4647c478bd9Sstevel@tonic-gate 			 * we lost contact with the server.
4657c478bd9Sstevel@tonic-gate 			 */
4667c478bd9Sstevel@tonic-gate 			if (cur_msg) {
4677c478bd9Sstevel@tonic-gate 				cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
4687c478bd9Sstevel@tonic-gate 				while (cur_msg && !successful_comm(cur_msg)) {
4697c478bd9Sstevel@tonic-gate 					oldest_msg = cur_msg;
4707c478bd9Sstevel@tonic-gate 					cur_msg = list_prev(&mi->mi_msg_list,
4717c478bd9Sstevel@tonic-gate 					    cur_msg);
4727c478bd9Sstevel@tonic-gate 				}
4737c478bd9Sstevel@tonic-gate 			}
4747c478bd9Sstevel@tonic-gate 			/*
4757c478bd9Sstevel@tonic-gate 			 * If we're not at the dummy head pointer,
4767c478bd9Sstevel@tonic-gate 			 * set the oldest and current message.
4777c478bd9Sstevel@tonic-gate 			 */
4787c478bd9Sstevel@tonic-gate 			if (cur_msg) {
4797c478bd9Sstevel@tonic-gate 				first_msg = cur_msg;
4807c478bd9Sstevel@tonic-gate 				oldest_msg = cur_msg;
4817c478bd9Sstevel@tonic-gate 				cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
4827c478bd9Sstevel@tonic-gate 			}
4837c478bd9Sstevel@tonic-gate 		} else
4847c478bd9Sstevel@tonic-gate 			cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
4857c478bd9Sstevel@tonic-gate 	}
4867c478bd9Sstevel@tonic-gate 
4877c478bd9Sstevel@tonic-gate 	return (oldest_msg);
4887c478bd9Sstevel@tonic-gate }
4897c478bd9Sstevel@tonic-gate 
4907c478bd9Sstevel@tonic-gate /*
4917c478bd9Sstevel@tonic-gate  * Returns 1 if facts have been found; 0 otherwise.
4927c478bd9Sstevel@tonic-gate  */
4937c478bd9Sstevel@tonic-gate static int
get_facts(nfs4_debug_msg_t * msgp,nfs4_rfact_t * ret_fp,char ** mnt_pt,mntinfo4_t * mi)4947c478bd9Sstevel@tonic-gate get_facts(nfs4_debug_msg_t *msgp, nfs4_rfact_t *ret_fp, char **mnt_pt,
4957c478bd9Sstevel@tonic-gate mntinfo4_t *mi)
4967c478bd9Sstevel@tonic-gate {
4977c478bd9Sstevel@tonic-gate 	nfs4_debug_msg_t	*cur_msg, *oldest_msg;
4987c478bd9Sstevel@tonic-gate 	nfs4_rfact_t		*cur_fp;
4997c478bd9Sstevel@tonic-gate 	int			found_a_fact = 0;
5007c478bd9Sstevel@tonic-gate 	int			len;
5017c478bd9Sstevel@tonic-gate 
5027c478bd9Sstevel@tonic-gate 	cur_msg = msgp;
5037c478bd9Sstevel@tonic-gate 
5047c478bd9Sstevel@tonic-gate 	/* find the oldest msg to search backwards to */
5057c478bd9Sstevel@tonic-gate 	oldest_msg = find_beginning(cur_msg, mi);
5067c478bd9Sstevel@tonic-gate 	ASSERT(oldest_msg != NULL);
5077c478bd9Sstevel@tonic-gate 
5087c478bd9Sstevel@tonic-gate 	/*
5097c478bd9Sstevel@tonic-gate 	 * Create a fact sheet by searching from our current message
5107c478bd9Sstevel@tonic-gate 	 * backwards to the 'oldest_msg', recording facts along the way
5117c478bd9Sstevel@tonic-gate 	 * until we found facts that have been inspected by another time.
5127c478bd9Sstevel@tonic-gate 	 */
5137c478bd9Sstevel@tonic-gate 	while (cur_msg && cur_msg != list_prev(&mi->mi_msg_list, oldest_msg)) {
5147c478bd9Sstevel@tonic-gate 		if (cur_msg->msg_type != RM_FACT) {
5157c478bd9Sstevel@tonic-gate 			cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
5167c478bd9Sstevel@tonic-gate 			continue;
5177c478bd9Sstevel@tonic-gate 		}
5187c478bd9Sstevel@tonic-gate 
5197c478bd9Sstevel@tonic-gate 		cur_fp = &cur_msg->rmsg_u.msg_fact;
5207c478bd9Sstevel@tonic-gate 		/*
5217c478bd9Sstevel@tonic-gate 		 * If this fact has already been looked at, then so
5227c478bd9Sstevel@tonic-gate 		 * have all preceding facts.  Return Now.
5237c478bd9Sstevel@tonic-gate 		 */
5247c478bd9Sstevel@tonic-gate 		if (cur_fp->rf_status == RFS_INSPECT)
5257c478bd9Sstevel@tonic-gate 			return (found_a_fact);
5267c478bd9Sstevel@tonic-gate 
5277c478bd9Sstevel@tonic-gate 		cur_fp->rf_status = RFS_INSPECT;
5287c478bd9Sstevel@tonic-gate 		found_a_fact = 1;
5297c478bd9Sstevel@tonic-gate 		switch (cur_fp->rf_type) {
5307c478bd9Sstevel@tonic-gate 		case RF_BADOWNER:
5317c478bd9Sstevel@tonic-gate 			break;
5327c478bd9Sstevel@tonic-gate 		case RF_ERR:
5337c478bd9Sstevel@tonic-gate 			/*
5347c478bd9Sstevel@tonic-gate 			 * Don't want to overwrite a fact that was
5357c478bd9Sstevel@tonic-gate 			 * previously found during our current search.
5367c478bd9Sstevel@tonic-gate 			 */
5377c478bd9Sstevel@tonic-gate 			if (!ret_fp->rf_reboot)
5387c478bd9Sstevel@tonic-gate 				ret_fp->rf_reboot = cur_fp->rf_reboot;
5397c478bd9Sstevel@tonic-gate 			if (!ret_fp->rf_stat4)
5407c478bd9Sstevel@tonic-gate 				ret_fp->rf_stat4 = cur_fp->rf_stat4;
5417c478bd9Sstevel@tonic-gate 			if (!ret_fp->rf_action)
5427c478bd9Sstevel@tonic-gate 				ret_fp->rf_action = cur_fp->rf_action;
5437c478bd9Sstevel@tonic-gate 			break;
5447c478bd9Sstevel@tonic-gate 		case RF_RENEW_EXPIRED:
5457c478bd9Sstevel@tonic-gate 			if (cur_msg->msg_mntpt && !(*mnt_pt)) {
5467c478bd9Sstevel@tonic-gate 				len = strlen(cur_msg->msg_mntpt) + 1;
5477c478bd9Sstevel@tonic-gate 				*mnt_pt = kmem_alloc(len, KM_SLEEP);
5487c478bd9Sstevel@tonic-gate 				bcopy(cur_msg->msg_mntpt, *mnt_pt, len);
5497c478bd9Sstevel@tonic-gate 			}
5507c478bd9Sstevel@tonic-gate 			break;
5517c478bd9Sstevel@tonic-gate 		case RF_SRV_OK:
5527c478bd9Sstevel@tonic-gate 			break;
5537c478bd9Sstevel@tonic-gate 		case RF_SRV_NOT_RESPOND:
5547c478bd9Sstevel@tonic-gate 			/*
5557c478bd9Sstevel@tonic-gate 			 * Okay to overwrite this fact as
5567c478bd9Sstevel@tonic-gate 			 * we want the earliest time.
5577c478bd9Sstevel@tonic-gate 			 */
5587c478bd9Sstevel@tonic-gate 			ret_fp->rf_time = cur_fp->rf_time;
5597c478bd9Sstevel@tonic-gate 			break;
5607c478bd9Sstevel@tonic-gate 		case RF_SRVS_OK:
5617c478bd9Sstevel@tonic-gate 			break;
5627c478bd9Sstevel@tonic-gate 		case RF_SRVS_NOT_RESPOND:
5637c478bd9Sstevel@tonic-gate 			break;
5647c478bd9Sstevel@tonic-gate 		case RF_DELMAP_CB_ERR:
5657c478bd9Sstevel@tonic-gate 			break;
566e280ed37SDai Ngo 		case RF_SENDQ_FULL:
567e280ed37SDai Ngo 			break;
5687c478bd9Sstevel@tonic-gate 		default:
5697c478bd9Sstevel@tonic-gate 			zcmn_err(getzoneid(), CE_NOTE,
5707c478bd9Sstevel@tonic-gate 			    "get facts: illegal fact %d", cur_fp->rf_type);
5717c478bd9Sstevel@tonic-gate 			break;
5727c478bd9Sstevel@tonic-gate 		}
5737c478bd9Sstevel@tonic-gate 		cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
5747c478bd9Sstevel@tonic-gate 	}
5757c478bd9Sstevel@tonic-gate 
5767c478bd9Sstevel@tonic-gate 	return (found_a_fact);
5777c478bd9Sstevel@tonic-gate }
5787c478bd9Sstevel@tonic-gate 
5797c478bd9Sstevel@tonic-gate /*
5807c478bd9Sstevel@tonic-gate  * Returns 1 if this fact is identical to the last fact recorded
5817c478bd9Sstevel@tonic-gate  * (only checks for a match within the last 2 lease periods).
5827c478bd9Sstevel@tonic-gate  */
5837c478bd9Sstevel@tonic-gate static int
facts_same(nfs4_debug_msg_t * cur_msg,nfs4_debug_msg_t * new_msg,mntinfo4_t * mi)5847c478bd9Sstevel@tonic-gate facts_same(nfs4_debug_msg_t *cur_msg, nfs4_debug_msg_t *new_msg,
5857c478bd9Sstevel@tonic-gate     mntinfo4_t *mi)
5867c478bd9Sstevel@tonic-gate {
5877c478bd9Sstevel@tonic-gate 	nfs4_rfact_t	*fp1, *fp2;
5887c478bd9Sstevel@tonic-gate 	int		lease, len;
5897c478bd9Sstevel@tonic-gate 
5907c478bd9Sstevel@tonic-gate 	ASSERT(mutex_owned(&mi->mi_msg_list_lock));
5917c478bd9Sstevel@tonic-gate 	if (mi->mi_lease_period > 0)
5927c478bd9Sstevel@tonic-gate 		lease = 2 * mi->mi_lease_period;
5937c478bd9Sstevel@tonic-gate 	else
5947c478bd9Sstevel@tonic-gate 		lease = DEFAULT_LEASE;
5957c478bd9Sstevel@tonic-gate 
5967c478bd9Sstevel@tonic-gate 	fp2 = &new_msg->rmsg_u.msg_fact;
5977c478bd9Sstevel@tonic-gate 
5987c478bd9Sstevel@tonic-gate 	while (cur_msg &&
5997c478bd9Sstevel@tonic-gate 	    new_msg->msg_time.tv_sec - cur_msg->msg_time.tv_sec < lease) {
6007c478bd9Sstevel@tonic-gate 		if (cur_msg->msg_type != RM_FACT) {
6017c478bd9Sstevel@tonic-gate 			cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
6027c478bd9Sstevel@tonic-gate 			continue;
6037c478bd9Sstevel@tonic-gate 		}
6047c478bd9Sstevel@tonic-gate 		fp1 = &cur_msg->rmsg_u.msg_fact;
6057c478bd9Sstevel@tonic-gate 		if (fp1->rf_type != fp2->rf_type)
6067c478bd9Sstevel@tonic-gate 			return (0);
6077c478bd9Sstevel@tonic-gate 
6087c478bd9Sstevel@tonic-gate 		/* now actually compare the facts */
6097c478bd9Sstevel@tonic-gate 		if (fp1->rf_action != fp2->rf_action)
6107c478bd9Sstevel@tonic-gate 			return (0);
6117c478bd9Sstevel@tonic-gate 		if (fp1->rf_stat4 != fp2->rf_stat4)
6127c478bd9Sstevel@tonic-gate 			return (0);
6137c478bd9Sstevel@tonic-gate 		if (fp1->rf_reboot != fp2->rf_reboot)
6147c478bd9Sstevel@tonic-gate 			return (0);
6157c478bd9Sstevel@tonic-gate 		if (fp1->rf_op != fp2->rf_op)
6167c478bd9Sstevel@tonic-gate 			return (0);
6177c478bd9Sstevel@tonic-gate 		if (fp1->rf_time.tv_sec != fp2->rf_time.tv_sec)
6187c478bd9Sstevel@tonic-gate 			return (0);
6197c478bd9Sstevel@tonic-gate 		if (fp1->rf_error != fp2->rf_error)
6207c478bd9Sstevel@tonic-gate 			return (0);
6217c478bd9Sstevel@tonic-gate 		if (fp1->rf_rp1 != fp2->rf_rp1)
6227c478bd9Sstevel@tonic-gate 			return (0);
6237c478bd9Sstevel@tonic-gate 		if (cur_msg->msg_srv != NULL) {
6247c478bd9Sstevel@tonic-gate 			if (new_msg->msg_srv == NULL)
6257c478bd9Sstevel@tonic-gate 				return (0);
6267c478bd9Sstevel@tonic-gate 			len = strlen(cur_msg->msg_srv);
6277c478bd9Sstevel@tonic-gate 			if (strncmp(cur_msg->msg_srv, new_msg->msg_srv,
6287f0b8309SEdward Pilatowicz 			    len) != 0)
6297c478bd9Sstevel@tonic-gate 				return (0);
6307c478bd9Sstevel@tonic-gate 		} else if (new_msg->msg_srv != NULL) {
6317c478bd9Sstevel@tonic-gate 			return (0);
6327c478bd9Sstevel@tonic-gate 		}
6337c478bd9Sstevel@tonic-gate 		if (cur_msg->msg_mntpt != NULL) {
6347c478bd9Sstevel@tonic-gate 			if (new_msg->msg_mntpt == NULL)
6357c478bd9Sstevel@tonic-gate 				return (0);
6367c478bd9Sstevel@tonic-gate 			len = strlen(cur_msg->msg_mntpt);
6377c478bd9Sstevel@tonic-gate 			if (strncmp(cur_msg->msg_mntpt, new_msg->msg_mntpt,
6387f0b8309SEdward Pilatowicz 			    len) != 0)
6397c478bd9Sstevel@tonic-gate 				return (0);
6407c478bd9Sstevel@tonic-gate 		} else if (new_msg->msg_mntpt != NULL) {
6417c478bd9Sstevel@tonic-gate 			return (0);
6427c478bd9Sstevel@tonic-gate 		}
6437c478bd9Sstevel@tonic-gate 		if (fp1->rf_char1 != NULL) {
6447c478bd9Sstevel@tonic-gate 			if (fp2->rf_char1 == NULL)
6457c478bd9Sstevel@tonic-gate 				return (0);
6467c478bd9Sstevel@tonic-gate 			len = strlen(fp1->rf_char1);
6477c478bd9Sstevel@tonic-gate 			if (strncmp(fp1->rf_char1, fp2->rf_char1, len) != 0)
6487c478bd9Sstevel@tonic-gate 				return (0);
6497c478bd9Sstevel@tonic-gate 		} else if (fp2->rf_char1 != NULL) {
6507c478bd9Sstevel@tonic-gate 			return (0);
6517c478bd9Sstevel@tonic-gate 		}
6527c478bd9Sstevel@tonic-gate 		return (1);
6537c478bd9Sstevel@tonic-gate 	}
6547c478bd9Sstevel@tonic-gate 
6557c478bd9Sstevel@tonic-gate 	return (0);
6567c478bd9Sstevel@tonic-gate }
6577c478bd9Sstevel@tonic-gate 
6587c478bd9Sstevel@tonic-gate /*
6597c478bd9Sstevel@tonic-gate  * Returns 1 if these two messages are identical; 0 otherwise.
6607c478bd9Sstevel@tonic-gate  */
6617c478bd9Sstevel@tonic-gate static int
events_same(nfs4_debug_msg_t * cur_msg,nfs4_debug_msg_t * new_msg,mntinfo4_t * mi)6627c478bd9Sstevel@tonic-gate events_same(nfs4_debug_msg_t *cur_msg, nfs4_debug_msg_t *new_msg,
6637c478bd9Sstevel@tonic-gate     mntinfo4_t *mi)
6647c478bd9Sstevel@tonic-gate {
6657c478bd9Sstevel@tonic-gate 	nfs4_revent_t	*ep1, *ep2;
6667c478bd9Sstevel@tonic-gate 	int		len;
6677c478bd9Sstevel@tonic-gate 
6687c478bd9Sstevel@tonic-gate 	/* find the last event, bypassing all facts */
6697c478bd9Sstevel@tonic-gate 	while (cur_msg && cur_msg->msg_type != RM_EVENT)
6707c478bd9Sstevel@tonic-gate 		cur_msg = list_prev(&mi->mi_msg_list, cur_msg);
6717c478bd9Sstevel@tonic-gate 
6727c478bd9Sstevel@tonic-gate 	if (!cur_msg)
6737c478bd9Sstevel@tonic-gate 		return (0);
6747c478bd9Sstevel@tonic-gate 
6757c478bd9Sstevel@tonic-gate 	if (cur_msg->msg_type != RM_EVENT)
6767c478bd9Sstevel@tonic-gate 		return (0);
6777c478bd9Sstevel@tonic-gate 
6787c478bd9Sstevel@tonic-gate 	ep1 = &cur_msg->rmsg_u.msg_event;
6797c478bd9Sstevel@tonic-gate 	ep2 = &new_msg->rmsg_u.msg_event;
6807c478bd9Sstevel@tonic-gate 	if (ep1->re_type != ep2->re_type)
6817c478bd9Sstevel@tonic-gate 		return (0);
6827c478bd9Sstevel@tonic-gate 
6837c478bd9Sstevel@tonic-gate 	/*
6847c478bd9Sstevel@tonic-gate 	 * Since we zalloc the buffer, then the two nfs4_debug_msg's
6857c478bd9Sstevel@tonic-gate 	 * must match up even if all the fields weren't filled in
6867c478bd9Sstevel@tonic-gate 	 * the first place.
6877c478bd9Sstevel@tonic-gate 	 */
6887c478bd9Sstevel@tonic-gate 	if (ep1->re_mi != ep2->re_mi)
6897c478bd9Sstevel@tonic-gate 		return (0);
6907c478bd9Sstevel@tonic-gate 	if (ep1->re_uint != ep2->re_uint)
6917c478bd9Sstevel@tonic-gate 		return (0);
6927c478bd9Sstevel@tonic-gate 	if (ep1->re_stat4 != ep2->re_stat4)
6937c478bd9Sstevel@tonic-gate 		return (0);
6947c478bd9Sstevel@tonic-gate 	if (ep1->re_pid != ep2->re_pid)
6957c478bd9Sstevel@tonic-gate 		return (0);
6967c478bd9Sstevel@tonic-gate 	if (ep1->re_rp1 != ep2->re_rp1)
6977c478bd9Sstevel@tonic-gate 		return (0);
6987c478bd9Sstevel@tonic-gate 	if (ep1->re_rp2 != ep2->re_rp2)
6997c478bd9Sstevel@tonic-gate 		return (0);
7007c478bd9Sstevel@tonic-gate 	if (ep1->re_tag1 != ep2->re_tag1)
7017c478bd9Sstevel@tonic-gate 		return (0);
7027c478bd9Sstevel@tonic-gate 	if (ep1->re_tag2 != ep2->re_tag2)
7037c478bd9Sstevel@tonic-gate 		return (0);
7047c478bd9Sstevel@tonic-gate 	if (ep1->re_seqid1 != ep2->re_seqid1)
7057c478bd9Sstevel@tonic-gate 		return (0);
7067c478bd9Sstevel@tonic-gate 	if (ep1->re_seqid2 != ep2->re_seqid2)
7077c478bd9Sstevel@tonic-gate 		return (0);
7087c478bd9Sstevel@tonic-gate 
7097c478bd9Sstevel@tonic-gate 	if (cur_msg->msg_srv != NULL) {
7107c478bd9Sstevel@tonic-gate 		if (new_msg->msg_srv == NULL)
7117c478bd9Sstevel@tonic-gate 			return (0);
7127c478bd9Sstevel@tonic-gate 		len = strlen(cur_msg->msg_srv);
7137c478bd9Sstevel@tonic-gate 		if (strncmp(cur_msg->msg_srv, new_msg->msg_srv, len) != 0)
7147c478bd9Sstevel@tonic-gate 			return (0);
7157c478bd9Sstevel@tonic-gate 	} else if (new_msg->msg_srv != NULL) {
7167c478bd9Sstevel@tonic-gate 		return (0);
7177c478bd9Sstevel@tonic-gate 	}
7187c478bd9Sstevel@tonic-gate 
7197c478bd9Sstevel@tonic-gate 	if (ep1->re_char1 != NULL) {
7207c478bd9Sstevel@tonic-gate 		if (ep2->re_char1 == NULL)
7217c478bd9Sstevel@tonic-gate 			return (0);
7227c478bd9Sstevel@tonic-gate 		len = strlen(ep1->re_char1);
7237c478bd9Sstevel@tonic-gate 		if (strncmp(ep1->re_char1, ep2->re_char1, len) != 0)
7247c478bd9Sstevel@tonic-gate 			return (0);
7257c478bd9Sstevel@tonic-gate 	} else if (ep2->re_char1 != NULL) {
7267c478bd9Sstevel@tonic-gate 		return (0);
7277c478bd9Sstevel@tonic-gate 	}
7287c478bd9Sstevel@tonic-gate 
7297c478bd9Sstevel@tonic-gate 	if (ep1->re_char2 != NULL) {
7307c478bd9Sstevel@tonic-gate 		if (ep2->re_char2 == NULL)
7317c478bd9Sstevel@tonic-gate 			return (0);
7327c478bd9Sstevel@tonic-gate 		len = strlen(ep1->re_char2);
7337c478bd9Sstevel@tonic-gate 		if (strncmp(ep1->re_char2, ep2->re_char2, len) != 0)
7347c478bd9Sstevel@tonic-gate 			return (0);
7357c478bd9Sstevel@tonic-gate 	} else if (ep2->re_char2 != NULL) {
7367c478bd9Sstevel@tonic-gate 		return (0);
7377c478bd9Sstevel@tonic-gate 	}
7387c478bd9Sstevel@tonic-gate 
7397c478bd9Sstevel@tonic-gate 	if (cur_msg->msg_mntpt != NULL) {
7407c478bd9Sstevel@tonic-gate 		if (new_msg->msg_mntpt == NULL)
7417c478bd9Sstevel@tonic-gate 			return (0);
7427c478bd9Sstevel@tonic-gate 		len = strlen(cur_msg->msg_mntpt);
7437c478bd9Sstevel@tonic-gate 		if (strncmp(cur_msg->msg_mntpt, cur_msg->msg_mntpt, len) != 0)
7447c478bd9Sstevel@tonic-gate 			return (0);
7457c478bd9Sstevel@tonic-gate 	} else if (new_msg->msg_mntpt != NULL) {
7467c478bd9Sstevel@tonic-gate 		return (0);
7477c478bd9Sstevel@tonic-gate 	}
7487c478bd9Sstevel@tonic-gate 
7497c478bd9Sstevel@tonic-gate 	return (1);
7507c478bd9Sstevel@tonic-gate }
7517c478bd9Sstevel@tonic-gate 
7527c478bd9Sstevel@tonic-gate /*
7537c478bd9Sstevel@tonic-gate  * Free up a recovery event.
7547c478bd9Sstevel@tonic-gate  */
7557c478bd9Sstevel@tonic-gate static void
free_event(nfs4_revent_t * ep)7567c478bd9Sstevel@tonic-gate free_event(nfs4_revent_t *ep)
7577c478bd9Sstevel@tonic-gate {
7587c478bd9Sstevel@tonic-gate 	int	len;
7597c478bd9Sstevel@tonic-gate 
7607c478bd9Sstevel@tonic-gate 	if (ep->re_char1) {
7617c478bd9Sstevel@tonic-gate 		len = strlen(ep->re_char1) + 1;
7627c478bd9Sstevel@tonic-gate 		kmem_free(ep->re_char1, len);
7637c478bd9Sstevel@tonic-gate 	}
7647c478bd9Sstevel@tonic-gate 	if (ep->re_char2) {
7657c478bd9Sstevel@tonic-gate 		len = strlen(ep->re_char2) + 1;
7667c478bd9Sstevel@tonic-gate 		kmem_free(ep->re_char2, len);
7677c478bd9Sstevel@tonic-gate 	}
7687c478bd9Sstevel@tonic-gate }
7697c478bd9Sstevel@tonic-gate 
7707c478bd9Sstevel@tonic-gate /*
7717c478bd9Sstevel@tonic-gate  * Free up a recovery fact.
7727c478bd9Sstevel@tonic-gate  */
7737c478bd9Sstevel@tonic-gate static void
free_fact(nfs4_rfact_t * fp)7747c478bd9Sstevel@tonic-gate free_fact(nfs4_rfact_t *fp)
7757c478bd9Sstevel@tonic-gate {
7767c478bd9Sstevel@tonic-gate 	int	len;
7777c478bd9Sstevel@tonic-gate 
7787c478bd9Sstevel@tonic-gate 	if (fp->rf_char1) {
7797c478bd9Sstevel@tonic-gate 		len = strlen(fp->rf_char1) + 1;
7807c478bd9Sstevel@tonic-gate 		kmem_free(fp->rf_char1, len);
7817c478bd9Sstevel@tonic-gate 	}
7827c478bd9Sstevel@tonic-gate }
7837c478bd9Sstevel@tonic-gate 
7847c478bd9Sstevel@tonic-gate /*
7857c478bd9Sstevel@tonic-gate  * Free up the message.
7867c478bd9Sstevel@tonic-gate  */
7877c478bd9Sstevel@tonic-gate void
nfs4_free_msg(nfs4_debug_msg_t * msg)7887c478bd9Sstevel@tonic-gate nfs4_free_msg(nfs4_debug_msg_t *msg)
7897c478bd9Sstevel@tonic-gate {
7907c478bd9Sstevel@tonic-gate 	int len;
7917c478bd9Sstevel@tonic-gate 
7927c478bd9Sstevel@tonic-gate 	if (msg->msg_type == RM_EVENT)
7937c478bd9Sstevel@tonic-gate 		free_event(&msg->rmsg_u.msg_event);
7947c478bd9Sstevel@tonic-gate 	else
7957c478bd9Sstevel@tonic-gate 		free_fact(&msg->rmsg_u.msg_fact);
7967c478bd9Sstevel@tonic-gate 
7977c478bd9Sstevel@tonic-gate 	if (msg->msg_srv) {
7987c478bd9Sstevel@tonic-gate 		len = strlen(msg->msg_srv) + 1;
7997c478bd9Sstevel@tonic-gate 		kmem_free(msg->msg_srv, len);
8007c478bd9Sstevel@tonic-gate 	}
8017c478bd9Sstevel@tonic-gate 
8027c478bd9Sstevel@tonic-gate 	if (msg->msg_mntpt) {
8037c478bd9Sstevel@tonic-gate 		len = strlen(msg->msg_mntpt) + 1;
8047c478bd9Sstevel@tonic-gate 		kmem_free(msg->msg_mntpt, len);
8057c478bd9Sstevel@tonic-gate 	}
8067c478bd9Sstevel@tonic-gate 
8077c478bd9Sstevel@tonic-gate 	/* free up the data structure itself */
8087c478bd9Sstevel@tonic-gate 	kmem_free(msg, sizeof (*msg));
8097c478bd9Sstevel@tonic-gate }
8107c478bd9Sstevel@tonic-gate 
8117c478bd9Sstevel@tonic-gate /*
8127c478bd9Sstevel@tonic-gate  * Prints out the interesting facts for recovery events:
8137c478bd9Sstevel@tonic-gate  * -DEAD_FILE
8147c478bd9Sstevel@tonic-gate  * -SIGLOST(_NO_DUMP)
8157c478bd9Sstevel@tonic-gate  */
8167c478bd9Sstevel@tonic-gate static void
print_facts(nfs4_debug_msg_t * msg,mntinfo4_t * mi)8177c478bd9Sstevel@tonic-gate print_facts(nfs4_debug_msg_t *msg, mntinfo4_t *mi)
8187c478bd9Sstevel@tonic-gate {
8197c478bd9Sstevel@tonic-gate 	nfs4_rfact_t *fp;
8207c478bd9Sstevel@tonic-gate 	char *mount_pt;
8217c478bd9Sstevel@tonic-gate 	int len;
8227c478bd9Sstevel@tonic-gate 
8237c478bd9Sstevel@tonic-gate 	if (msg->rmsg_u.msg_event.re_type != RE_DEAD_FILE &&
8247c478bd9Sstevel@tonic-gate 	    msg->rmsg_u.msg_event.re_type != RE_SIGLOST &&
8257c478bd9Sstevel@tonic-gate 	    msg->rmsg_u.msg_event.re_type != RE_SIGLOST_NO_DUMP)
8267c478bd9Sstevel@tonic-gate 		return;
8277c478bd9Sstevel@tonic-gate 
8287c478bd9Sstevel@tonic-gate 	fp = kmem_zalloc(sizeof (*fp), KM_SLEEP);
8297c478bd9Sstevel@tonic-gate 	mount_pt = NULL;
8307c478bd9Sstevel@tonic-gate 
8317c478bd9Sstevel@tonic-gate 	if (get_facts(msg, fp, &mount_pt, mi)) {
8327c478bd9Sstevel@tonic-gate 		char	time[256];
8337c478bd9Sstevel@tonic-gate 
8347c478bd9Sstevel@tonic-gate 
8357c478bd9Sstevel@tonic-gate 		if (fp->rf_time.tv_sec)
8367c478bd9Sstevel@tonic-gate 			(void) snprintf(time, 256, "%ld",
8377c478bd9Sstevel@tonic-gate 			    (gethrestime_sec() - fp->rf_time.tv_sec)/60);
8387c478bd9Sstevel@tonic-gate 		zcmn_err(mi->mi_zone->zone_id, CE_NOTE,
8397c478bd9Sstevel@tonic-gate 		    "!NFS4 FACT SHEET: %s%s %s%s %s %s%s%s %s%s",
8407c478bd9Sstevel@tonic-gate 		    fp->rf_action ? "\n Action: " : "",
8417c478bd9Sstevel@tonic-gate 		    fp->rf_action ? nfs4_recov_action_to_str(fp->rf_action) :
8427c478bd9Sstevel@tonic-gate 		    "",
8437c478bd9Sstevel@tonic-gate 		    fp->rf_stat4 ? "\n NFS4 error: " : "",
8447c478bd9Sstevel@tonic-gate 		    fp->rf_stat4 ? nfs4_stat_to_str(fp->rf_stat4) : "",
8457c478bd9Sstevel@tonic-gate 		    fp->rf_reboot ? "\n Suspected server reboot. " : "",
8467c478bd9Sstevel@tonic-gate 		    fp->rf_time.tv_sec ? "\n Server was down for " : "",
8477c478bd9Sstevel@tonic-gate 		    fp->rf_time.tv_sec ? time : "",
8487c478bd9Sstevel@tonic-gate 		    fp->rf_time.tv_sec ? " minutes." : "",
8497c478bd9Sstevel@tonic-gate 		    mount_pt ? " \n Client's lease expired on mount " : "",
8507c478bd9Sstevel@tonic-gate 		    mount_pt ? mount_pt : "");
8517c478bd9Sstevel@tonic-gate 	}
8527c478bd9Sstevel@tonic-gate 
8537c478bd9Sstevel@tonic-gate 	if (mount_pt) {
8547c478bd9Sstevel@tonic-gate 		len = strlen(mount_pt) + 1;
8557c478bd9Sstevel@tonic-gate 		kmem_free(mount_pt, len);
8567c478bd9Sstevel@tonic-gate 	}
8577c478bd9Sstevel@tonic-gate 
8587c478bd9Sstevel@tonic-gate 	/* free the fact struct itself */
8597c478bd9Sstevel@tonic-gate 	if (fp)
8607c478bd9Sstevel@tonic-gate 		kmem_free(fp, sizeof (*fp));
8617c478bd9Sstevel@tonic-gate }
8627c478bd9Sstevel@tonic-gate 
8637c478bd9Sstevel@tonic-gate /*
8647c478bd9Sstevel@tonic-gate  * Print an event message to /var/adm/messages
8657c478bd9Sstevel@tonic-gate  * The last argument to this fuction dictates the repeat status
8667c478bd9Sstevel@tonic-gate  * of the event. If set to 1, it means that we are dumping this
8677c478bd9Sstevel@tonic-gate  * event and it will _never_ be printed after this time. Else if
8687c478bd9Sstevel@tonic-gate  * set to 0 it will be printed again.
8697c478bd9Sstevel@tonic-gate  */
8707c478bd9Sstevel@tonic-gate static void
queue_print_event(nfs4_debug_msg_t * msg,mntinfo4_t * mi,int dump)8717c478bd9Sstevel@tonic-gate queue_print_event(nfs4_debug_msg_t *msg, mntinfo4_t *mi, int dump)
8727c478bd9Sstevel@tonic-gate {
8737c478bd9Sstevel@tonic-gate 	nfs4_revent_t		*ep;
8747c478bd9Sstevel@tonic-gate 	zoneid_t		zoneid;
8757c478bd9Sstevel@tonic-gate 
8767c478bd9Sstevel@tonic-gate 	ep = &msg->rmsg_u.msg_event;
8777c478bd9Sstevel@tonic-gate 	zoneid = mi->mi_zone->zone_id;
8787c478bd9Sstevel@tonic-gate 
8797c478bd9Sstevel@tonic-gate 	switch (ep->re_type) {
8807c478bd9Sstevel@tonic-gate 	case RE_BAD_SEQID:
8817c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
8827c478bd9Sstevel@tonic-gate 		    "Operation %s for file %s (rnode_pt 0x%p), pid %d using "
8837c478bd9Sstevel@tonic-gate 		    "seqid %d got %s.  Last good seqid was %d for "
8847c478bd9Sstevel@tonic-gate 		    "operation %s.",
8857c478bd9Sstevel@tonic-gate 		    msg->msg_srv, msg->msg_mntpt,
8867c478bd9Sstevel@tonic-gate 		    nfs4_ctags[ep->re_tag1].ct_str, ep->re_char1,
8877c478bd9Sstevel@tonic-gate 		    (void *)ep->re_rp1, ep->re_pid, ep->re_seqid1,
8887c478bd9Sstevel@tonic-gate 		    nfs4_stat_to_str(ep->re_stat4), ep->re_seqid2,
8897c478bd9Sstevel@tonic-gate 		    nfs4_ctags[ep->re_tag2].ct_str);
8907c478bd9Sstevel@tonic-gate 		break;
8917c478bd9Sstevel@tonic-gate 	case RE_BADHANDLE:
8927c478bd9Sstevel@tonic-gate 		ASSERT(ep->re_rp1 != NULL);
8937c478bd9Sstevel@tonic-gate 		if (ep->re_char1 != NULL) {
8947c478bd9Sstevel@tonic-gate 			zcmn_err(zoneid, CE_NOTE,
8957c478bd9Sstevel@tonic-gate 			    "![NFS4][Server: %s][Mntpt: %s]"
8967c478bd9Sstevel@tonic-gate 			    "server %s said filehandle was "
8977c478bd9Sstevel@tonic-gate 			    "invalid for file: %s (rnode_pt 0x%p) on mount %s",
8987c478bd9Sstevel@tonic-gate 			    msg->msg_srv, msg->msg_mntpt, msg->msg_srv,
8997c478bd9Sstevel@tonic-gate 			    ep->re_char1, (void *)ep->re_rp1, msg->msg_mntpt);
9007c478bd9Sstevel@tonic-gate 		} else {
9017c478bd9Sstevel@tonic-gate 			zcmn_err(zoneid, CE_NOTE,
9027c478bd9Sstevel@tonic-gate 			    "![NFS4][Server: %s][Mntpt: %s]"
9037c478bd9Sstevel@tonic-gate 			    "server %s said filehandle was "
9047c478bd9Sstevel@tonic-gate 			    "invalid for file: (rnode_pt 0x%p) on mount %s"
9057c478bd9Sstevel@tonic-gate 			    " for fh:", msg->msg_srv, msg->msg_mntpt,
9067c478bd9Sstevel@tonic-gate 			    msg->msg_srv, (void *)ep->re_rp1, msg->msg_mntpt);
9077c478bd9Sstevel@tonic-gate 			sfh4_printfhandle(ep->re_rp1->r_fh);
9087c478bd9Sstevel@tonic-gate 		}
9097c478bd9Sstevel@tonic-gate 		break;
9107c478bd9Sstevel@tonic-gate 	case RE_CLIENTID:
9117c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
9127c478bd9Sstevel@tonic-gate 		    "Can't recover clientid on mount point %s "
9137c478bd9Sstevel@tonic-gate 		    "(mi 0x%p) due to error %d (%s), for server %s.  Marking "
9147c478bd9Sstevel@tonic-gate 		    "file system as unusable.",
9157c478bd9Sstevel@tonic-gate 		    msg->msg_srv, msg->msg_mntpt, msg->msg_mntpt,
9167c478bd9Sstevel@tonic-gate 		    (void *)ep->re_mi, ep->re_uint,
9177c478bd9Sstevel@tonic-gate 		    nfs4_stat_to_str(ep->re_stat4),
9187c478bd9Sstevel@tonic-gate 		    msg->msg_srv);
9197c478bd9Sstevel@tonic-gate 		break;
9207c478bd9Sstevel@tonic-gate 	case RE_DEAD_FILE:
9217c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
9227c478bd9Sstevel@tonic-gate 		    "File %s (rnode_pt: %p) was closed due to NFS "
9237c478bd9Sstevel@tonic-gate 		    "recovery error on server %s(%s %s)", msg->msg_srv,
9247c478bd9Sstevel@tonic-gate 		    msg->msg_mntpt, ep->re_char1, (void *)ep->re_rp1,
9257c478bd9Sstevel@tonic-gate 		    msg->msg_srv, ep->re_char2 ? ep->re_char2 : "",
9267c478bd9Sstevel@tonic-gate 		    ep->re_stat4 ? nfs4_stat_to_str(ep->re_stat4) : "");
9277c478bd9Sstevel@tonic-gate 		break;
9287c478bd9Sstevel@tonic-gate 	case RE_END:
9297c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
9307c478bd9Sstevel@tonic-gate 		    "NFS Recovery done for mount %s (mi 0x%p) "
9317c478bd9Sstevel@tonic-gate 		    "on server %s, rnode_pt1 %s (0x%p), "
9327c478bd9Sstevel@tonic-gate 		    "rnode_pt2 %s (0x%p)", msg->msg_srv, msg->msg_mntpt,
9337c478bd9Sstevel@tonic-gate 		    msg->msg_mntpt, (void *)ep->re_mi, msg->msg_srv,
9347c478bd9Sstevel@tonic-gate 		    ep->re_char1, (void *)ep->re_rp1, ep->re_char2,
9357c478bd9Sstevel@tonic-gate 		    (void *)ep->re_rp2);
9367c478bd9Sstevel@tonic-gate 		break;
9377c478bd9Sstevel@tonic-gate 	case RE_FAIL_RELOCK:
9387c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
9397c478bd9Sstevel@tonic-gate 		    "Couldn't reclaim lock for pid %d for "
9407c478bd9Sstevel@tonic-gate 		    "file %s (rnode_pt 0x%p) on (server %s): error %d",
9417c478bd9Sstevel@tonic-gate 		    msg->msg_srv, msg->msg_mntpt, ep->re_pid, ep->re_char1,
9427c478bd9Sstevel@tonic-gate 		    (void *)ep->re_rp1, msg->msg_srv,
9437c478bd9Sstevel@tonic-gate 		    ep->re_uint ? ep->re_uint : ep->re_stat4);
9447c478bd9Sstevel@tonic-gate 		break;
9457c478bd9Sstevel@tonic-gate 	case RE_FAIL_REMAP_LEN:
9467c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
9477c478bd9Sstevel@tonic-gate 		    "remap_lookup: server %s returned bad "
9487c478bd9Sstevel@tonic-gate 		    "fhandle length (%d)", msg->msg_srv, msg->msg_mntpt,
9497c478bd9Sstevel@tonic-gate 		    msg->msg_srv, ep->re_uint);
9507c478bd9Sstevel@tonic-gate 		break;
9517c478bd9Sstevel@tonic-gate 	case RE_FAIL_REMAP_OP:
9527c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
9537c478bd9Sstevel@tonic-gate 		    "remap_lookup: didn't get expected OP_GETFH"
9547c478bd9Sstevel@tonic-gate 		    " for server %s", msg->msg_srv, msg->msg_mntpt,
9557c478bd9Sstevel@tonic-gate 		    msg->msg_srv);
9567c478bd9Sstevel@tonic-gate 		break;
9577c478bd9Sstevel@tonic-gate 	case RE_FAILOVER:
9587c478bd9Sstevel@tonic-gate 		if (ep->re_char1)
9597c478bd9Sstevel@tonic-gate 			zcmn_err(zoneid, CE_NOTE,
9607c478bd9Sstevel@tonic-gate 			    "![NFS4][Server: %s][Mntpt: %s]"
9617c478bd9Sstevel@tonic-gate 			    "failing over from %s to %s", msg->msg_srv,
9627c478bd9Sstevel@tonic-gate 			    msg->msg_mntpt, msg->msg_srv, ep->re_char1);
9637c478bd9Sstevel@tonic-gate 		else
9647c478bd9Sstevel@tonic-gate 			zcmn_err(zoneid, CE_NOTE,
9657c478bd9Sstevel@tonic-gate 			    "![NFS4][Server: %s][Mntpt: %s]"
9667c478bd9Sstevel@tonic-gate 			    "NFS4: failing over: selecting "
9677c478bd9Sstevel@tonic-gate 			    "original server %s", msg->msg_srv, msg->msg_mntpt,
9687c478bd9Sstevel@tonic-gate 			    msg->msg_srv);
9697c478bd9Sstevel@tonic-gate 		break;
9707c478bd9Sstevel@tonic-gate 	case RE_FILE_DIFF:
9717c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
9727c478bd9Sstevel@tonic-gate 		    "File %s (rnode_pt: %p) on server %s was closed "
9737c478bd9Sstevel@tonic-gate 		    "and failed attempted failover since its is different than "
9747c478bd9Sstevel@tonic-gate 		    "the original file", msg->msg_srv, msg->msg_mntpt,
9757c478bd9Sstevel@tonic-gate 		    ep->re_char1, (void *)ep->re_rp1, msg->msg_srv);
9767c478bd9Sstevel@tonic-gate 		break;
9777c478bd9Sstevel@tonic-gate 	case RE_LOST_STATE:
9787c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
9797c478bd9Sstevel@tonic-gate 		    "Lost %s request for fs %s, file %s (rnode_pt: 0x%p), "
9807c478bd9Sstevel@tonic-gate 		    "dir %s (0x%p) for server %s", msg->msg_srv, msg->msg_mntpt,
9817c478bd9Sstevel@tonic-gate 		    nfs4_op_to_str(ep->re_uint), msg->msg_mntpt,
9827c478bd9Sstevel@tonic-gate 		    ep->re_char1, (void *)ep->re_rp1, ep->re_char2,
9837c478bd9Sstevel@tonic-gate 		    (void *)ep->re_rp2, msg->msg_srv);
9847c478bd9Sstevel@tonic-gate 		break;
9857c478bd9Sstevel@tonic-gate 	case RE_OPENS_CHANGED:
9867c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
9877c478bd9Sstevel@tonic-gate 		    "The number of open files to reopen changed "
9887c478bd9Sstevel@tonic-gate 		    "for mount %s mi 0x%p (old %d, new %d) on server %s",
9897c478bd9Sstevel@tonic-gate 		    msg->msg_srv, msg->msg_mntpt, msg->msg_mntpt,
9907c478bd9Sstevel@tonic-gate 		    (void *)ep->re_mi, ep->re_uint, ep->re_pid, msg->msg_srv);
9917c478bd9Sstevel@tonic-gate 		break;
9927c478bd9Sstevel@tonic-gate 	case RE_SIGLOST:
9937c478bd9Sstevel@tonic-gate 	case RE_SIGLOST_NO_DUMP:
9947c478bd9Sstevel@tonic-gate 		if (ep->re_uint)
9957c478bd9Sstevel@tonic-gate 			zcmn_err(zoneid, CE_NOTE,
9967c478bd9Sstevel@tonic-gate 			    "![NFS4][Server: %s][Mntpt: %s]"
9977c478bd9Sstevel@tonic-gate 			    "Process %d lost its locks on "
9987c478bd9Sstevel@tonic-gate 			    "file %s (rnode_pt: %p) due to NFS recovery error "
9997c478bd9Sstevel@tonic-gate 			    "(%d) on server %s.", msg->msg_srv, msg->msg_mntpt,
10007c478bd9Sstevel@tonic-gate 			    ep->re_pid, ep->re_char1, (void *)ep->re_rp1,
10017c478bd9Sstevel@tonic-gate 			    ep->re_uint, msg->msg_srv);
10027c478bd9Sstevel@tonic-gate 		else
10037c478bd9Sstevel@tonic-gate 			zcmn_err(zoneid, CE_NOTE,
10047c478bd9Sstevel@tonic-gate 			    "![NFS4][Server: %s][Mntpt: %s]"
10057c478bd9Sstevel@tonic-gate 			    "Process %d lost its locks on "
10067c478bd9Sstevel@tonic-gate 			    "file %s (rnode_pt: %p) due to NFS recovery error "
10077c478bd9Sstevel@tonic-gate 			    "(%s) on server %s.", msg->msg_srv, msg->msg_mntpt,
10087c478bd9Sstevel@tonic-gate 			    ep->re_pid, ep->re_char1, (void *)ep->re_rp1,
10097c478bd9Sstevel@tonic-gate 			    nfs4_stat_to_str(ep->re_stat4), msg->msg_srv);
10107c478bd9Sstevel@tonic-gate 		break;
10117c478bd9Sstevel@tonic-gate 	case RE_START:
10127c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
10137c478bd9Sstevel@tonic-gate 		    "NFS Starting recovery for mount %s "
10147c478bd9Sstevel@tonic-gate 		    "(mi 0x%p mi_recovflags [0x%x]) on server %s, "
10157c478bd9Sstevel@tonic-gate 		    "rnode_pt1 %s (0x%p), rnode_pt2 %s (0x%p)", msg->msg_srv,
10167c478bd9Sstevel@tonic-gate 		    msg->msg_mntpt, msg->msg_mntpt, (void *)ep->re_mi,
10177c478bd9Sstevel@tonic-gate 		    ep->re_uint, msg->msg_srv, ep->re_char1, (void *)ep->re_rp1,
10187c478bd9Sstevel@tonic-gate 		    ep->re_char2, (void *)ep->re_rp2);
10197c478bd9Sstevel@tonic-gate 		break;
10207c478bd9Sstevel@tonic-gate 	case RE_UNEXPECTED_ACTION:
10217c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
10227c478bd9Sstevel@tonic-gate 		    "NFS recovery: unexpected action (%s) on server %s",
10237c478bd9Sstevel@tonic-gate 		    msg->msg_srv, msg->msg_mntpt,
10247c478bd9Sstevel@tonic-gate 		    nfs4_recov_action_to_str(ep->re_uint), msg->msg_srv);
10257c478bd9Sstevel@tonic-gate 		break;
10267c478bd9Sstevel@tonic-gate 	case RE_UNEXPECTED_ERRNO:
10277c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
10287c478bd9Sstevel@tonic-gate 		    "NFS recovery: unexpected errno (%d) on server %s",
10297c478bd9Sstevel@tonic-gate 		    msg->msg_srv, msg->msg_mntpt, ep->re_uint, msg->msg_srv);
10307c478bd9Sstevel@tonic-gate 		break;
10317c478bd9Sstevel@tonic-gate 	case RE_UNEXPECTED_STATUS:
10327c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
10337c478bd9Sstevel@tonic-gate 		    "NFS recovery: unexpected NFS status code (%s) "
10347c478bd9Sstevel@tonic-gate 		    "on server %s", msg->msg_srv, msg->msg_mntpt,
10357c478bd9Sstevel@tonic-gate 		    nfs4_stat_to_str(ep->re_stat4),
10367c478bd9Sstevel@tonic-gate 		    msg->msg_srv);
10377c478bd9Sstevel@tonic-gate 		break;
10387c478bd9Sstevel@tonic-gate 	case RE_WRONGSEC:
10397c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
10407c478bd9Sstevel@tonic-gate 		    "NFS can't recover from NFS4ERR_WRONGSEC."
10417c478bd9Sstevel@tonic-gate 		    "  error %d for server %s: rnode_pt1 %s (0x%p)"
10427c478bd9Sstevel@tonic-gate 		    " rnode_pt2 %s (0x%p)", msg->msg_srv, msg->msg_mntpt,
10437c478bd9Sstevel@tonic-gate 		    ep->re_uint, msg->msg_srv, ep->re_char1, (void *)ep->re_rp1,
10447c478bd9Sstevel@tonic-gate 		    ep->re_char2, (void *)ep->re_rp2);
10457c478bd9Sstevel@tonic-gate 		break;
10467c478bd9Sstevel@tonic-gate 	case RE_LOST_STATE_BAD_OP:
10477c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
10487c478bd9Sstevel@tonic-gate 		    "NFS lost state with unrecognized op (%d)."
10497c478bd9Sstevel@tonic-gate 		    "  fs %s, server %s, pid %d, file %s (rnode_pt: 0x%p), "
10507c478bd9Sstevel@tonic-gate 		    "dir %s (0x%p)", msg->msg_srv, msg->msg_mntpt,
10517c478bd9Sstevel@tonic-gate 		    ep->re_uint, msg->msg_mntpt, msg->msg_srv, ep->re_pid,
10527c478bd9Sstevel@tonic-gate 		    ep->re_char1, (void *)ep->re_rp1, ep->re_char2,
10537c478bd9Sstevel@tonic-gate 		    (void *)ep->re_rp2);
10547c478bd9Sstevel@tonic-gate 		break;
1055*2f172c55SRobert Thurlow 	case RE_REFERRAL:
1056*2f172c55SRobert Thurlow 		if (ep->re_char1)
1057*2f172c55SRobert Thurlow 			zcmn_err(zoneid, CE_NOTE,
1058*2f172c55SRobert Thurlow 			    "![NFS4][Server: %s][Mntpt: %s]"
1059*2f172c55SRobert Thurlow 			    "being referred from %s to %s", msg->msg_srv,
1060*2f172c55SRobert Thurlow 			    msg->msg_mntpt, msg->msg_srv, ep->re_char1);
1061*2f172c55SRobert Thurlow 		else
1062*2f172c55SRobert Thurlow 			zcmn_err(zoneid, CE_NOTE,
1063*2f172c55SRobert Thurlow 			    "![NFS4][Server: %s][Mntpt: %s]"
1064*2f172c55SRobert Thurlow 			    "NFS4: being referred from %s to unknown server",
1065*2f172c55SRobert Thurlow 			    msg->msg_srv, msg->msg_mntpt, msg->msg_srv);
1066*2f172c55SRobert Thurlow 		break;
10677c478bd9Sstevel@tonic-gate 	default:
10687c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_WARN,
10697c478bd9Sstevel@tonic-gate 		    "!queue_print_event: illegal event %d", ep->re_type);
10707c478bd9Sstevel@tonic-gate 		break;
10717c478bd9Sstevel@tonic-gate 	}
10727c478bd9Sstevel@tonic-gate 
10737c478bd9Sstevel@tonic-gate 	print_facts(msg, mi);
10747c478bd9Sstevel@tonic-gate 
10757c478bd9Sstevel@tonic-gate 	/*
10767c478bd9Sstevel@tonic-gate 	 * If set this event will not be printed again and is considered
10777c478bd9Sstevel@tonic-gate 	 * dumped.
10787c478bd9Sstevel@tonic-gate 	 */
10797c478bd9Sstevel@tonic-gate 	if (dump)
10807c478bd9Sstevel@tonic-gate 		msg->msg_status = NFS4_MS_NO_DUMP;
10817c478bd9Sstevel@tonic-gate }
10827c478bd9Sstevel@tonic-gate 
10837c478bd9Sstevel@tonic-gate /*
10847c478bd9Sstevel@tonic-gate  * Print a fact message to /var/adm/messages
10857c478bd9Sstevel@tonic-gate  */
10867c478bd9Sstevel@tonic-gate static void
queue_print_fact(nfs4_debug_msg_t * msg,int dump)10877c478bd9Sstevel@tonic-gate queue_print_fact(nfs4_debug_msg_t *msg, int dump)
10887c478bd9Sstevel@tonic-gate {
10897c478bd9Sstevel@tonic-gate 	nfs4_rfact_t	*fp;
10907c478bd9Sstevel@tonic-gate 	zoneid_t	zoneid;
10917c478bd9Sstevel@tonic-gate 
10927c478bd9Sstevel@tonic-gate 	fp = &msg->rmsg_u.msg_fact;
10937c478bd9Sstevel@tonic-gate 	zoneid = getzoneid();
10947c478bd9Sstevel@tonic-gate 
10957c478bd9Sstevel@tonic-gate 	switch (fp->rf_type) {
10967c478bd9Sstevel@tonic-gate 	case RF_BADOWNER:
10977c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
10987c478bd9Sstevel@tonic-gate 		    "NFSMAPID_DOMAIN does not match the server: %s domain\n"
10997c478bd9Sstevel@tonic-gate 		    "Please check configuration", msg->msg_srv, msg->msg_mntpt,
11007c478bd9Sstevel@tonic-gate 		    msg->msg_srv);
11017c478bd9Sstevel@tonic-gate 		break;
11027c478bd9Sstevel@tonic-gate 	case RF_ERR:
11037c478bd9Sstevel@tonic-gate 		if (fp->rf_error)
11047c478bd9Sstevel@tonic-gate 			zcmn_err(zoneid, CE_NOTE,
11057c478bd9Sstevel@tonic-gate 			    "![NFS4][Server: %s][Mntpt: %s]NFS op %s got "
11067c478bd9Sstevel@tonic-gate 			    "error %d causing recovery action %s.%s",
11077c478bd9Sstevel@tonic-gate 			    msg->msg_srv, msg->msg_mntpt,
11087c478bd9Sstevel@tonic-gate 			    nfs4_op_to_str(fp->rf_op), fp->rf_error,
11097c478bd9Sstevel@tonic-gate 			    nfs4_recov_action_to_str(fp->rf_action),
11107c478bd9Sstevel@tonic-gate 			    fp->rf_reboot ?
11117c478bd9Sstevel@tonic-gate 			    "  Client also suspects that the server rebooted,"
11127c478bd9Sstevel@tonic-gate 			    " or experienced a network partition." : "");
11137c478bd9Sstevel@tonic-gate 		else
11147c478bd9Sstevel@tonic-gate 			zcmn_err(zoneid, CE_NOTE,
11157c478bd9Sstevel@tonic-gate 			    "![NFS4][Server: %s][Mntpt: %s]NFS op %s got "
11167c478bd9Sstevel@tonic-gate 			    "error %s causing recovery action %s.%s",
11177c478bd9Sstevel@tonic-gate 			    msg->msg_srv, msg->msg_mntpt,
11187c478bd9Sstevel@tonic-gate 			    nfs4_op_to_str(fp->rf_op),
11197c478bd9Sstevel@tonic-gate 			    nfs4_stat_to_str(fp->rf_stat4),
11207c478bd9Sstevel@tonic-gate 			    nfs4_recov_action_to_str(fp->rf_action),
11217c478bd9Sstevel@tonic-gate 			    fp->rf_reboot ?
11227c478bd9Sstevel@tonic-gate 			    "  Client also suspects that the server rebooted,"
11237c478bd9Sstevel@tonic-gate 			    " or experienced a network partition." : "");
11247c478bd9Sstevel@tonic-gate 		break;
11257c478bd9Sstevel@tonic-gate 	case RF_RENEW_EXPIRED:
11267c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
11277c478bd9Sstevel@tonic-gate 		    "NFS4 renew thread detected client's "
11287c478bd9Sstevel@tonic-gate 		    "lease has expired. Current open files/locks/IO may fail",
11297c478bd9Sstevel@tonic-gate 		    msg->msg_srv, msg->msg_mntpt);
11307c478bd9Sstevel@tonic-gate 		break;
11317c478bd9Sstevel@tonic-gate 	case RF_SRV_NOT_RESPOND:
11327c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
11337c478bd9Sstevel@tonic-gate 		    "NFS server %s not responding; still trying\n",
11347c478bd9Sstevel@tonic-gate 		    msg->msg_srv, msg->msg_mntpt, msg->msg_srv);
11357c478bd9Sstevel@tonic-gate 		break;
11367c478bd9Sstevel@tonic-gate 	case RF_SRV_OK:
11377c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
11387c478bd9Sstevel@tonic-gate 		    "NFS server %s ok", msg->msg_srv, msg->msg_mntpt,
11397c478bd9Sstevel@tonic-gate 		    msg->msg_srv);
11407c478bd9Sstevel@tonic-gate 		break;
11417c478bd9Sstevel@tonic-gate 	case RF_SRVS_NOT_RESPOND:
11427c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
11437c478bd9Sstevel@tonic-gate 		    "NFS servers %s not responding; still trying", msg->msg_srv,
11447c478bd9Sstevel@tonic-gate 		    msg->msg_mntpt, msg->msg_srv);
11457c478bd9Sstevel@tonic-gate 		break;
11467c478bd9Sstevel@tonic-gate 	case RF_SRVS_OK:
11477c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
11487c478bd9Sstevel@tonic-gate 		    "NFS servers %s ok", msg->msg_srv, msg->msg_mntpt,
11497c478bd9Sstevel@tonic-gate 		    msg->msg_srv);
11507c478bd9Sstevel@tonic-gate 		break;
11517c478bd9Sstevel@tonic-gate 	case RF_DELMAP_CB_ERR:
11527c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
11537c478bd9Sstevel@tonic-gate 		    "NFS op %s got error %s when executing delmap on file %s "
11547c478bd9Sstevel@tonic-gate 		    "(rnode_pt 0x%p).",
11557c478bd9Sstevel@tonic-gate 		    msg->msg_srv, msg->msg_mntpt, nfs4_op_to_str(fp->rf_op),
11567c478bd9Sstevel@tonic-gate 		    nfs4_stat_to_str(fp->rf_stat4), fp->rf_char1,
11577c478bd9Sstevel@tonic-gate 		    (void *)fp->rf_rp1);
11587c478bd9Sstevel@tonic-gate 		break;
1159e280ed37SDai Ngo 	case RF_SENDQ_FULL:
1160e280ed37SDai Ngo 		zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]"
1161e280ed37SDai Ngo 		    "send queue to NFS server %s is full; still trying\n",
1162e280ed37SDai Ngo 		    msg->msg_srv, msg->msg_mntpt, msg->msg_srv);
1163e280ed37SDai Ngo 		break;
1164e280ed37SDai Ngo 
11657c478bd9Sstevel@tonic-gate 	default:
11667c478bd9Sstevel@tonic-gate 		zcmn_err(zoneid, CE_WARN, "!queue_print_fact: illegal fact %d",
11677c478bd9Sstevel@tonic-gate 		    fp->rf_type);
11687c478bd9Sstevel@tonic-gate 	}
11697c478bd9Sstevel@tonic-gate 
11707c478bd9Sstevel@tonic-gate 	/*
11717c478bd9Sstevel@tonic-gate 	 * If set this fact will not be printed again and is considered
11727c478bd9Sstevel@tonic-gate 	 * dumped.
11737c478bd9Sstevel@tonic-gate 	 */
11747c478bd9Sstevel@tonic-gate 	if (dump)
11757c478bd9Sstevel@tonic-gate 		msg->msg_status = NFS4_MS_NO_DUMP;
11767c478bd9Sstevel@tonic-gate }
11777c478bd9Sstevel@tonic-gate 
11787c478bd9Sstevel@tonic-gate /*
11797c478bd9Sstevel@tonic-gate  * Returns 1 if the entire queue should be dumped, 0 otherwise.
11807c478bd9Sstevel@tonic-gate  */
11817c478bd9Sstevel@tonic-gate static int
id_to_dump_queue(nfs4_event_type_t id)11827c478bd9Sstevel@tonic-gate id_to_dump_queue(nfs4_event_type_t id)
11837c478bd9Sstevel@tonic-gate {
11847c478bd9Sstevel@tonic-gate 	switch (id) {
11857c478bd9Sstevel@tonic-gate 	case RE_DEAD_FILE:
11867c478bd9Sstevel@tonic-gate 	case RE_SIGLOST:
11877c478bd9Sstevel@tonic-gate 	case RE_WRONGSEC:
11887c478bd9Sstevel@tonic-gate 	case RE_CLIENTID:
11897c478bd9Sstevel@tonic-gate 		return (1);
11907c478bd9Sstevel@tonic-gate 	default:
11917c478bd9Sstevel@tonic-gate 		return (0);
11927c478bd9Sstevel@tonic-gate 	}
11937c478bd9Sstevel@tonic-gate }
11947c478bd9Sstevel@tonic-gate 
11957c478bd9Sstevel@tonic-gate /*
11967c478bd9Sstevel@tonic-gate  * Returns 1 if the event (but not the entire queue) should be printed;
11977c478bd9Sstevel@tonic-gate  * 0 otherwise.
11987c478bd9Sstevel@tonic-gate  */
11997c478bd9Sstevel@tonic-gate static int
id_to_dump_solo_event(nfs4_event_type_t id)12007c478bd9Sstevel@tonic-gate id_to_dump_solo_event(nfs4_event_type_t id)
12017c478bd9Sstevel@tonic-gate {
12027c478bd9Sstevel@tonic-gate 	switch (id) {
12037c478bd9Sstevel@tonic-gate 	case RE_BAD_SEQID:
12047c478bd9Sstevel@tonic-gate 	case RE_BADHANDLE:
12057c478bd9Sstevel@tonic-gate 	case RE_FAIL_REMAP_LEN:
12067c478bd9Sstevel@tonic-gate 	case RE_FAIL_REMAP_OP:
12077c478bd9Sstevel@tonic-gate 	case RE_FAILOVER:
12087c478bd9Sstevel@tonic-gate 	case RE_OPENS_CHANGED:
12097c478bd9Sstevel@tonic-gate 	case RE_SIGLOST_NO_DUMP:
12107c478bd9Sstevel@tonic-gate 	case RE_UNEXPECTED_ACTION:
12117c478bd9Sstevel@tonic-gate 	case RE_UNEXPECTED_ERRNO:
12127c478bd9Sstevel@tonic-gate 	case RE_UNEXPECTED_STATUS:
12137c478bd9Sstevel@tonic-gate 	case RE_LOST_STATE_BAD_OP:
1214*2f172c55SRobert Thurlow 	case RE_REFERRAL:
12157c478bd9Sstevel@tonic-gate 		return (1);
12167c478bd9Sstevel@tonic-gate 	default:
12177c478bd9Sstevel@tonic-gate 		return (0);
12187c478bd9Sstevel@tonic-gate 	}
12197c478bd9Sstevel@tonic-gate }
12207c478bd9Sstevel@tonic-gate 
12217c478bd9Sstevel@tonic-gate /*
12227c478bd9Sstevel@tonic-gate  * Returns 1 if the fact (but not the entire queue) should be printed;
12237c478bd9Sstevel@tonic-gate  * 0 otherwise.
12247c478bd9Sstevel@tonic-gate  */
12257c478bd9Sstevel@tonic-gate static int
id_to_dump_solo_fact(nfs4_fact_type_t id)12267c478bd9Sstevel@tonic-gate id_to_dump_solo_fact(nfs4_fact_type_t id)
12277c478bd9Sstevel@tonic-gate {
12287c478bd9Sstevel@tonic-gate 	switch (id) {
12297c478bd9Sstevel@tonic-gate 	case RF_SRV_NOT_RESPOND:
12307c478bd9Sstevel@tonic-gate 	case RF_SRV_OK:
12317c478bd9Sstevel@tonic-gate 	case RF_SRVS_NOT_RESPOND:
12327c478bd9Sstevel@tonic-gate 	case RF_SRVS_OK:
1233e280ed37SDai Ngo 	case RF_SENDQ_FULL:
12347c478bd9Sstevel@tonic-gate 		return (1);
12357c478bd9Sstevel@tonic-gate 	default:
12367c478bd9Sstevel@tonic-gate 		return (0);
12377c478bd9Sstevel@tonic-gate 	}
12387c478bd9Sstevel@tonic-gate }
12397c478bd9Sstevel@tonic-gate 
12407c478bd9Sstevel@tonic-gate /*
12417c478bd9Sstevel@tonic-gate  * Update a kernel stat
12427c478bd9Sstevel@tonic-gate  */
12437c478bd9Sstevel@tonic-gate static void
update_recov_kstats(nfs4_debug_msg_t * msg,mntinfo4_t * mi)12447c478bd9Sstevel@tonic-gate update_recov_kstats(nfs4_debug_msg_t *msg, mntinfo4_t *mi)
12457c478bd9Sstevel@tonic-gate {
12467c478bd9Sstevel@tonic-gate 	rkstat_t	*rsp;
12477c478bd9Sstevel@tonic-gate 
12487c478bd9Sstevel@tonic-gate 	if (!mi->mi_recov_ksp)
12497c478bd9Sstevel@tonic-gate 		return;
12507c478bd9Sstevel@tonic-gate 
12517c478bd9Sstevel@tonic-gate 	rsp = (rkstat_t *)mi->mi_recov_ksp->ks_data;
12527c478bd9Sstevel@tonic-gate 
12537c478bd9Sstevel@tonic-gate 	if (msg->msg_type == RM_EVENT) {
12547c478bd9Sstevel@tonic-gate 		switch (msg->rmsg_u.msg_event.re_type) {
12557c478bd9Sstevel@tonic-gate 		case RE_BADHANDLE:
12567c478bd9Sstevel@tonic-gate 			rsp->badhandle.value.ul++;
12577c478bd9Sstevel@tonic-gate 			break;
12587c478bd9Sstevel@tonic-gate 		case RE_CLIENTID:
12597c478bd9Sstevel@tonic-gate 			rsp->clientid.value.ul++;
12607c478bd9Sstevel@tonic-gate 			break;
12617c478bd9Sstevel@tonic-gate 		case RE_DEAD_FILE:
12627c478bd9Sstevel@tonic-gate 			rsp->dead_file.value.ul++;
12637c478bd9Sstevel@tonic-gate 			break;
12647c478bd9Sstevel@tonic-gate 		case RE_FAIL_RELOCK:
12657c478bd9Sstevel@tonic-gate 			rsp->fail_relock.value.ul++;
12667c478bd9Sstevel@tonic-gate 			break;
12677c478bd9Sstevel@tonic-gate 		case RE_FILE_DIFF:
12687c478bd9Sstevel@tonic-gate 			rsp->file_diff.value.ul++;
12697c478bd9Sstevel@tonic-gate 			break;
12707c478bd9Sstevel@tonic-gate 		case RE_OPENS_CHANGED:
12717c478bd9Sstevel@tonic-gate 			rsp->opens_changed.value.ul++;
12727c478bd9Sstevel@tonic-gate 			break;
12737c478bd9Sstevel@tonic-gate 		case RE_SIGLOST:
12747c478bd9Sstevel@tonic-gate 		case RE_SIGLOST_NO_DUMP:
12757c478bd9Sstevel@tonic-gate 			rsp->siglost.value.ul++;
12767c478bd9Sstevel@tonic-gate 			break;
12777c478bd9Sstevel@tonic-gate 		case RE_UNEXPECTED_ACTION:
12787c478bd9Sstevel@tonic-gate 			rsp->unexp_action.value.ul++;
12797c478bd9Sstevel@tonic-gate 			break;
12807c478bd9Sstevel@tonic-gate 		case RE_UNEXPECTED_ERRNO:
12817c478bd9Sstevel@tonic-gate 			rsp->unexp_errno.value.ul++;
12827c478bd9Sstevel@tonic-gate 			break;
12837c478bd9Sstevel@tonic-gate 		case RE_UNEXPECTED_STATUS:
12847c478bd9Sstevel@tonic-gate 			rsp->unexp_status.value.ul++;
12857c478bd9Sstevel@tonic-gate 			break;
12867c478bd9Sstevel@tonic-gate 		case RE_WRONGSEC:
12877c478bd9Sstevel@tonic-gate 			rsp->wrongsec.value.ul++;
12887c478bd9Sstevel@tonic-gate 			break;
12897c478bd9Sstevel@tonic-gate 		case RE_LOST_STATE_BAD_OP:
12907c478bd9Sstevel@tonic-gate 			rsp->lost_state_bad_op.value.ul++;
12917c478bd9Sstevel@tonic-gate 			break;
12927c478bd9Sstevel@tonic-gate 		default:
12937c478bd9Sstevel@tonic-gate 			break;
12947c478bd9Sstevel@tonic-gate 		}
12957c478bd9Sstevel@tonic-gate 	} else if (msg->msg_type == RM_FACT) {
12967c478bd9Sstevel@tonic-gate 		switch (msg->rmsg_u.msg_fact.rf_type) {
12977c478bd9Sstevel@tonic-gate 		case RF_BADOWNER:
12987c478bd9Sstevel@tonic-gate 			rsp->badowner.value.ul++;
12997c478bd9Sstevel@tonic-gate 			break;
13007c478bd9Sstevel@tonic-gate 		case RF_SRV_NOT_RESPOND:
13017c478bd9Sstevel@tonic-gate 			rsp->not_responding.value.ul++;
13027c478bd9Sstevel@tonic-gate 			break;
13037c478bd9Sstevel@tonic-gate 		default:
13047c478bd9Sstevel@tonic-gate 			break;
13057c478bd9Sstevel@tonic-gate 		}
13067c478bd9Sstevel@tonic-gate 	}
13077c478bd9Sstevel@tonic-gate }
13087c478bd9Sstevel@tonic-gate 
13097c478bd9Sstevel@tonic-gate /*
13107c478bd9Sstevel@tonic-gate  * Dump the mi's mi_msg_list of recovery messages.
13117c478bd9Sstevel@tonic-gate  */
13127c478bd9Sstevel@tonic-gate static void
dump_queue(mntinfo4_t * mi,nfs4_debug_msg_t * msg)13137c478bd9Sstevel@tonic-gate dump_queue(mntinfo4_t *mi, nfs4_debug_msg_t *msg)
13147c478bd9Sstevel@tonic-gate {
13157c478bd9Sstevel@tonic-gate 	nfs4_debug_msg_t *tmp_msg;
13167c478bd9Sstevel@tonic-gate 
13177c478bd9Sstevel@tonic-gate 	ASSERT(mutex_owned(&mi->mi_msg_list_lock));
13187c478bd9Sstevel@tonic-gate 
13197c478bd9Sstevel@tonic-gate 	/* update kstats */
13207c478bd9Sstevel@tonic-gate 	update_recov_kstats(msg, mi);
13217c478bd9Sstevel@tonic-gate 
13227c478bd9Sstevel@tonic-gate 	/*
13237c478bd9Sstevel@tonic-gate 	 * If we aren't supposed to dump the queue then see if we
13247c478bd9Sstevel@tonic-gate 	 * should just print this single message, then return.
13257c478bd9Sstevel@tonic-gate 	 */
13267c478bd9Sstevel@tonic-gate 	if (!id_to_dump_queue(msg->rmsg_u.msg_event.re_type)) {
13277c478bd9Sstevel@tonic-gate 		if (id_to_dump_solo_event(msg->rmsg_u.msg_event.re_type))
13287c478bd9Sstevel@tonic-gate 			queue_print_event(msg, mi, 0);
13297c478bd9Sstevel@tonic-gate 		return;
13307c478bd9Sstevel@tonic-gate 	}
13317c478bd9Sstevel@tonic-gate 
13327c478bd9Sstevel@tonic-gate 	/*
13337c478bd9Sstevel@tonic-gate 	 * Write all events/facts in the queue that haven't been
13347c478bd9Sstevel@tonic-gate 	 * previously written to disk.
13357c478bd9Sstevel@tonic-gate 	 */
13367c478bd9Sstevel@tonic-gate 	tmp_msg = list_head(&mi->mi_msg_list);
13377c478bd9Sstevel@tonic-gate 	while (tmp_msg) {
13387c478bd9Sstevel@tonic-gate 		if (tmp_msg->msg_status == NFS4_MS_DUMP) {
13397c478bd9Sstevel@tonic-gate 			if (tmp_msg->msg_type == RM_EVENT)
13407c478bd9Sstevel@tonic-gate 				queue_print_event(tmp_msg, mi, 1);
13417c478bd9Sstevel@tonic-gate 			else if (tmp_msg->msg_type == RM_FACT)
13427c478bd9Sstevel@tonic-gate 				queue_print_fact(tmp_msg, 1);
13437c478bd9Sstevel@tonic-gate 		}
13447c478bd9Sstevel@tonic-gate 		tmp_msg = list_next(&mi->mi_msg_list, tmp_msg);
13457c478bd9Sstevel@tonic-gate 	}
13467c478bd9Sstevel@tonic-gate }
13477c478bd9Sstevel@tonic-gate 
13487c478bd9Sstevel@tonic-gate /*
13497c478bd9Sstevel@tonic-gate  * Places the event into mi's debug recovery message queue.  Some of the
13507c478bd9Sstevel@tonic-gate  * fields can be overloaded to be a generic value, depending on the event
13517c478bd9Sstevel@tonic-gate  * type.  These include "count", "why".
13527c478bd9Sstevel@tonic-gate  */
13537c478bd9Sstevel@tonic-gate void
nfs4_queue_event(nfs4_event_type_t id,mntinfo4_t * mi,char * server1,uint_t count,vnode_t * vp1,vnode_t * vp2,nfsstat4 nfs4_error,char * why,pid_t pid,nfs4_tag_type_t tag1,nfs4_tag_type_t tag2,seqid4 seqid1,seqid4 seqid2)13547c478bd9Sstevel@tonic-gate nfs4_queue_event(nfs4_event_type_t id, mntinfo4_t *mi, char *server1,
13557c478bd9Sstevel@tonic-gate     uint_t count, vnode_t *vp1, vnode_t *vp2, nfsstat4 nfs4_error,
13567c478bd9Sstevel@tonic-gate     char *why, pid_t pid, nfs4_tag_type_t tag1, nfs4_tag_type_t tag2,
13577c478bd9Sstevel@tonic-gate     seqid4 seqid1, seqid4 seqid2)
13587c478bd9Sstevel@tonic-gate {
13597c478bd9Sstevel@tonic-gate 	nfs4_debug_msg_t	*msg;
13607c478bd9Sstevel@tonic-gate 	nfs4_revent_t		*ep;
13617c478bd9Sstevel@tonic-gate 	char			*cur_srv;
13627c478bd9Sstevel@tonic-gate 	rnode4_t		*rp1 = NULL, *rp2 = NULL;
13637c478bd9Sstevel@tonic-gate 	refstr_t		*mntpt;
13647c478bd9Sstevel@tonic-gate 
13657c478bd9Sstevel@tonic-gate 	ASSERT(mi != NULL);
13667c478bd9Sstevel@tonic-gate 	if (vp1)
13677c478bd9Sstevel@tonic-gate 		rp1 = VTOR4(vp1);
13687c478bd9Sstevel@tonic-gate 	if (vp2)
13697c478bd9Sstevel@tonic-gate 		rp2 = VTOR4(vp2);
13707c478bd9Sstevel@tonic-gate 
13717c478bd9Sstevel@tonic-gate 	/*
137263360950Smp 	 * Initialize the message with the relevant server/mount_pt/time
13737c478bd9Sstevel@tonic-gate 	 * information. Also place the relevent event related info.
13747c478bd9Sstevel@tonic-gate 	 */
13757c478bd9Sstevel@tonic-gate 	msg = kmem_zalloc(sizeof (*msg), KM_SLEEP);
13767c478bd9Sstevel@tonic-gate 	msg->msg_type = RM_EVENT;
13777c478bd9Sstevel@tonic-gate 	msg->msg_status = NFS4_MS_DUMP;
13787c478bd9Sstevel@tonic-gate 	ep = &msg->rmsg_u.msg_event;
13797c478bd9Sstevel@tonic-gate 	ep->re_type = id;
13807c478bd9Sstevel@tonic-gate 	gethrestime(&msg->msg_time);
13817c478bd9Sstevel@tonic-gate 
13827c478bd9Sstevel@tonic-gate 	cur_srv = mi->mi_curr_serv->sv_hostname;
13837c478bd9Sstevel@tonic-gate 	msg->msg_srv = strdup(cur_srv);
13847c478bd9Sstevel@tonic-gate 	mntpt = vfs_getmntpoint(mi->mi_vfsp);
13857c478bd9Sstevel@tonic-gate 	msg->msg_mntpt = strdup(refstr_value(mntpt));
13867c478bd9Sstevel@tonic-gate 	refstr_rele(mntpt);
13877c478bd9Sstevel@tonic-gate 
13887c478bd9Sstevel@tonic-gate 	set_event(id, ep, mi, rp1, rp2, count, pid, nfs4_error, server1,
13897c478bd9Sstevel@tonic-gate 	    why, tag1, tag2, seqid1, seqid2);
13907c478bd9Sstevel@tonic-gate 
13917c478bd9Sstevel@tonic-gate 	mutex_enter(&mi->mi_msg_list_lock);
13927c478bd9Sstevel@tonic-gate 
13937c478bd9Sstevel@tonic-gate 	/* if this event is the same as the last event, drop it */
13947c478bd9Sstevel@tonic-gate 	if (events_same(list_tail(&mi->mi_msg_list), msg, mi)) {
13957c478bd9Sstevel@tonic-gate 		mutex_exit(&mi->mi_msg_list_lock);
13967c478bd9Sstevel@tonic-gate 		nfs4_free_msg(msg);
13977c478bd9Sstevel@tonic-gate 		return;
13987c478bd9Sstevel@tonic-gate 	}
13997c478bd9Sstevel@tonic-gate 
14007c478bd9Sstevel@tonic-gate 	/* queue the message at the end of the list */
14017c478bd9Sstevel@tonic-gate 	list_insert_tail(&mi->mi_msg_list, msg);
14027c478bd9Sstevel@tonic-gate 
14037c478bd9Sstevel@tonic-gate 	dump_queue(mi, msg);
14047c478bd9Sstevel@tonic-gate 
14057c478bd9Sstevel@tonic-gate 	if (mi->mi_msg_count == nfs4_msg_max) {
14067c478bd9Sstevel@tonic-gate 		nfs4_debug_msg_t *rm_msg;
14077c478bd9Sstevel@tonic-gate 
14087c478bd9Sstevel@tonic-gate 		/* remove the queue'd message at the front of the list */
14097c478bd9Sstevel@tonic-gate 		rm_msg = list_head(&mi->mi_msg_list);
14107c478bd9Sstevel@tonic-gate 		list_remove(&mi->mi_msg_list, rm_msg);
14117c478bd9Sstevel@tonic-gate 		mutex_exit(&mi->mi_msg_list_lock);
14127c478bd9Sstevel@tonic-gate 		nfs4_free_msg(rm_msg);
14137c478bd9Sstevel@tonic-gate 	} else {
14147c478bd9Sstevel@tonic-gate 		mi->mi_msg_count++;
14157c478bd9Sstevel@tonic-gate 		mutex_exit(&mi->mi_msg_list_lock);
14167c478bd9Sstevel@tonic-gate 	}
14177c478bd9Sstevel@tonic-gate }
14187c478bd9Sstevel@tonic-gate 
14197c478bd9Sstevel@tonic-gate /*
14207c478bd9Sstevel@tonic-gate  * Places the fact into mi's debug recovery messages queue.
14217c478bd9Sstevel@tonic-gate  */
14227c478bd9Sstevel@tonic-gate void
nfs4_queue_fact(nfs4_fact_type_t fid,mntinfo4_t * mi,nfsstat4 stat4,nfs4_recov_t raction,nfs_opnum4 op,bool_t reboot,char * srvname,int error,vnode_t * vp)14237c478bd9Sstevel@tonic-gate nfs4_queue_fact(nfs4_fact_type_t fid, mntinfo4_t *mi, nfsstat4 stat4,
14247c478bd9Sstevel@tonic-gate     nfs4_recov_t raction, nfs_opnum4 op, bool_t reboot, char *srvname,
14257c478bd9Sstevel@tonic-gate     int error, vnode_t *vp)
14267c478bd9Sstevel@tonic-gate {
14277c478bd9Sstevel@tonic-gate 	nfs4_debug_msg_t	*msg;
14287c478bd9Sstevel@tonic-gate 	nfs4_rfact_t		*fp;
14297c478bd9Sstevel@tonic-gate 	char			*cur_srv;
14307c478bd9Sstevel@tonic-gate 	refstr_t		*mntpt;
14317c478bd9Sstevel@tonic-gate 
14327c478bd9Sstevel@tonic-gate 	/*
143363360950Smp 	 * Initialize the message with the relevant server/mount_pt/time
143463360950Smp 	 * information. Also place the relevant fact related info.
14357c478bd9Sstevel@tonic-gate 	 */
14367c478bd9Sstevel@tonic-gate 	msg = kmem_zalloc(sizeof (*msg), KM_SLEEP);
14377c478bd9Sstevel@tonic-gate 	msg->msg_type = RM_FACT;
14387c478bd9Sstevel@tonic-gate 	msg->msg_status = NFS4_MS_DUMP;
14397c478bd9Sstevel@tonic-gate 	gethrestime(&msg->msg_time);
14407c478bd9Sstevel@tonic-gate 
14417c478bd9Sstevel@tonic-gate 	if (srvname)
14427c478bd9Sstevel@tonic-gate 		cur_srv = srvname;
14437c478bd9Sstevel@tonic-gate 	else
14447c478bd9Sstevel@tonic-gate 		cur_srv = mi->mi_curr_serv->sv_hostname;
14457c478bd9Sstevel@tonic-gate 
14467c478bd9Sstevel@tonic-gate 	msg->msg_srv = strdup(cur_srv);
14477c478bd9Sstevel@tonic-gate 	mntpt = vfs_getmntpoint(mi->mi_vfsp);
14487c478bd9Sstevel@tonic-gate 	msg->msg_mntpt = strdup(refstr_value(mntpt));
14497c478bd9Sstevel@tonic-gate 	refstr_rele(mntpt);
14507c478bd9Sstevel@tonic-gate 
14517c478bd9Sstevel@tonic-gate 	fp = &msg->rmsg_u.msg_fact;
14527c478bd9Sstevel@tonic-gate 	fp->rf_type = fid;
14537c478bd9Sstevel@tonic-gate 	fp->rf_status = RFS_NO_INSPECT;
14547c478bd9Sstevel@tonic-gate 	set_fact(fid, fp, stat4, raction, op, reboot, error, vp);
14557c478bd9Sstevel@tonic-gate 
14567c478bd9Sstevel@tonic-gate 	update_recov_kstats(msg, mi);
14577c478bd9Sstevel@tonic-gate 
14587c478bd9Sstevel@tonic-gate 	mutex_enter(&mi->mi_msg_list_lock);
14597c478bd9Sstevel@tonic-gate 
14607c478bd9Sstevel@tonic-gate 	/* if this fact is the same as the last fact, drop it */
14617c478bd9Sstevel@tonic-gate 	if (facts_same(list_tail(&mi->mi_msg_list), msg, mi)) {
14627c478bd9Sstevel@tonic-gate 		mutex_exit(&mi->mi_msg_list_lock);
14637c478bd9Sstevel@tonic-gate 		nfs4_free_msg(msg);
14647c478bd9Sstevel@tonic-gate 		return;
14657c478bd9Sstevel@tonic-gate 	}
14667c478bd9Sstevel@tonic-gate 
14677c478bd9Sstevel@tonic-gate 	/* queue the message at the end of the list */
14687c478bd9Sstevel@tonic-gate 	list_insert_tail(&mi->mi_msg_list, msg);
14697c478bd9Sstevel@tonic-gate 
14707c478bd9Sstevel@tonic-gate 	if (id_to_dump_solo_fact(msg->rmsg_u.msg_fact.rf_type))
14717c478bd9Sstevel@tonic-gate 		queue_print_fact(msg, 0);
14727c478bd9Sstevel@tonic-gate 
14737c478bd9Sstevel@tonic-gate 	if (mi->mi_msg_count == nfs4_msg_max) {
14747c478bd9Sstevel@tonic-gate 		nfs4_debug_msg_t *rm_msg;
14757c478bd9Sstevel@tonic-gate 
14767c478bd9Sstevel@tonic-gate 		/* remove the queue'd message at the front of the list */
14777c478bd9Sstevel@tonic-gate 		rm_msg = list_head(&mi->mi_msg_list);
14787c478bd9Sstevel@tonic-gate 		list_remove(&mi->mi_msg_list, rm_msg);
14797c478bd9Sstevel@tonic-gate 		mutex_exit(&mi->mi_msg_list_lock);
14807c478bd9Sstevel@tonic-gate 		nfs4_free_msg(rm_msg);
14817c478bd9Sstevel@tonic-gate 	} else {
14827c478bd9Sstevel@tonic-gate 		mi->mi_msg_count++;
14837c478bd9Sstevel@tonic-gate 		mutex_exit(&mi->mi_msg_list_lock);
14847c478bd9Sstevel@tonic-gate 	}
14857c478bd9Sstevel@tonic-gate }
14867c478bd9Sstevel@tonic-gate 
14877c478bd9Sstevel@tonic-gate /*
14887c478bd9Sstevel@tonic-gate  * Initialize the 'mi_recov_kstat' kstat.
14897c478bd9Sstevel@tonic-gate  */
14907c478bd9Sstevel@tonic-gate void
nfs4_mnt_recov_kstat_init(vfs_t * vfsp)14917c478bd9Sstevel@tonic-gate nfs4_mnt_recov_kstat_init(vfs_t *vfsp)
14927c478bd9Sstevel@tonic-gate {
14937c478bd9Sstevel@tonic-gate 	mntinfo4_t *mi = VFTOMI4(vfsp);
14947c478bd9Sstevel@tonic-gate 	kstat_t		*ksp;
14957c478bd9Sstevel@tonic-gate 	zoneid_t	zoneid = mi->mi_zone->zone_id;
14967c478bd9Sstevel@tonic-gate 
14977c478bd9Sstevel@tonic-gate 	/*
14987c478bd9Sstevel@tonic-gate 	 * Create the version specific kstats.
14997c478bd9Sstevel@tonic-gate 	 *
15007c478bd9Sstevel@tonic-gate 	 * PSARC 2001/697 Contract Private Interface
15017c478bd9Sstevel@tonic-gate 	 * All nfs kstats are under SunMC contract
15027c478bd9Sstevel@tonic-gate 	 * Please refer to the PSARC listed above and contact
15037c478bd9Sstevel@tonic-gate 	 * SunMC before making any changes!
15047c478bd9Sstevel@tonic-gate 	 *
15057c478bd9Sstevel@tonic-gate 	 * Changes must be reviewed by Solaris File Sharing
15067c478bd9Sstevel@tonic-gate 	 * Changes must be communicated to contract-2001-697@sun.com
15077c478bd9Sstevel@tonic-gate 	 *
15087c478bd9Sstevel@tonic-gate 	 */
15097c478bd9Sstevel@tonic-gate 
15107c478bd9Sstevel@tonic-gate 	if ((ksp = kstat_create_zone("nfs", getminor(vfsp->vfs_dev),
15117c478bd9Sstevel@tonic-gate 	    "mi_recov_kstat", "misc", KSTAT_TYPE_NAMED,
15127c478bd9Sstevel@tonic-gate 	    sizeof (rkstat_t) / sizeof (kstat_named_t),
15137c478bd9Sstevel@tonic-gate 	    KSTAT_FLAG_WRITABLE, zoneid)) == NULL) {
15147c478bd9Sstevel@tonic-gate 		mi->mi_recov_ksp = NULL;
15157c478bd9Sstevel@tonic-gate 		zcmn_err(GLOBAL_ZONEID, CE_NOTE,
15167c478bd9Sstevel@tonic-gate 		    "!mi_recov_kstat for mi %p failed\n",
15177c478bd9Sstevel@tonic-gate 		    (void *)mi);
15187c478bd9Sstevel@tonic-gate 		return;
15197c478bd9Sstevel@tonic-gate 	}
15207c478bd9Sstevel@tonic-gate 	if (zoneid != GLOBAL_ZONEID)
15217c478bd9Sstevel@tonic-gate 		kstat_zone_add(ksp, GLOBAL_ZONEID);
15227c478bd9Sstevel@tonic-gate 	mi->mi_recov_ksp = ksp;
15237c478bd9Sstevel@tonic-gate 	bcopy(&rkstat_template, ksp->ks_data, sizeof (rkstat_t));
15247c478bd9Sstevel@tonic-gate 	kstat_install(ksp);
15257c478bd9Sstevel@tonic-gate }
15267c478bd9Sstevel@tonic-gate 
15277c478bd9Sstevel@tonic-gate /*
15287c478bd9Sstevel@tonic-gate  * Increment the "delay" kstat.
15297c478bd9Sstevel@tonic-gate  */
15307c478bd9Sstevel@tonic-gate void
nfs4_mi_kstat_inc_delay(mntinfo4_t * mi)15317c478bd9Sstevel@tonic-gate nfs4_mi_kstat_inc_delay(mntinfo4_t *mi)
15327c478bd9Sstevel@tonic-gate {
15337c478bd9Sstevel@tonic-gate 	rkstat_t    *rsp;
15347c478bd9Sstevel@tonic-gate 
15357c478bd9Sstevel@tonic-gate 	if (!mi->mi_recov_ksp)
15367c478bd9Sstevel@tonic-gate 		return;
15377c478bd9Sstevel@tonic-gate 
15387c478bd9Sstevel@tonic-gate 	rsp = (rkstat_t *)mi->mi_recov_ksp->ks_data;
15397c478bd9Sstevel@tonic-gate 	rsp->delay.value.ul++;
15407c478bd9Sstevel@tonic-gate }
15417c478bd9Sstevel@tonic-gate 
15427c478bd9Sstevel@tonic-gate /*
15437c478bd9Sstevel@tonic-gate  * Increment the "no_grace" kstat.
15447c478bd9Sstevel@tonic-gate  */
15457c478bd9Sstevel@tonic-gate void
nfs4_mi_kstat_inc_no_grace(mntinfo4_t * mi)15467c478bd9Sstevel@tonic-gate nfs4_mi_kstat_inc_no_grace(mntinfo4_t *mi)
15477c478bd9Sstevel@tonic-gate {
15487c478bd9Sstevel@tonic-gate 	rkstat_t	*rsp;
15497c478bd9Sstevel@tonic-gate 
15507c478bd9Sstevel@tonic-gate 	if (!mi->mi_recov_ksp)
15517c478bd9Sstevel@tonic-gate 		return;
15527c478bd9Sstevel@tonic-gate 
15537c478bd9Sstevel@tonic-gate 	rsp = (rkstat_t *)mi->mi_recov_ksp->ks_data;
15547c478bd9Sstevel@tonic-gate 	rsp->no_grace.value.ul++;
15557c478bd9Sstevel@tonic-gate }
1556