17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 57f0b8309SEdward Pilatowicz * Common Development and Distribution License (the "License"). 67f0b8309SEdward Pilatowicz * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* 227f0b8309SEdward Pilatowicz * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 237c478bd9Sstevel@tonic-gate * Use is subject to license terms. 247c478bd9Sstevel@tonic-gate */ 257c478bd9Sstevel@tonic-gate 267c478bd9Sstevel@tonic-gate #include <sys/cred.h> 277c478bd9Sstevel@tonic-gate #include <sys/kstat.h> 287c478bd9Sstevel@tonic-gate #include <sys/list.h> 297c478bd9Sstevel@tonic-gate #include <sys/systm.h> 307c478bd9Sstevel@tonic-gate #include <sys/vfs.h> 317c478bd9Sstevel@tonic-gate #include <sys/vnode.h> 327c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 337c478bd9Sstevel@tonic-gate 347c478bd9Sstevel@tonic-gate #include <nfs/nfs4_clnt.h> 357c478bd9Sstevel@tonic-gate #include <nfs/rnode4.h> 367c478bd9Sstevel@tonic-gate 377c478bd9Sstevel@tonic-gate /* 387c478bd9Sstevel@tonic-gate * Recovery kstats 397c478bd9Sstevel@tonic-gate */ 407c478bd9Sstevel@tonic-gate typedef struct rkstat { 417c478bd9Sstevel@tonic-gate kstat_named_t badhandle; 427c478bd9Sstevel@tonic-gate kstat_named_t badowner; 437c478bd9Sstevel@tonic-gate kstat_named_t clientid; 447c478bd9Sstevel@tonic-gate kstat_named_t dead_file; 457c478bd9Sstevel@tonic-gate kstat_named_t delay; 467c478bd9Sstevel@tonic-gate kstat_named_t fail_relock; 477c478bd9Sstevel@tonic-gate kstat_named_t file_diff; 487c478bd9Sstevel@tonic-gate kstat_named_t no_grace; 497c478bd9Sstevel@tonic-gate kstat_named_t not_responding; 507c478bd9Sstevel@tonic-gate kstat_named_t opens_changed; 517c478bd9Sstevel@tonic-gate kstat_named_t siglost; 527c478bd9Sstevel@tonic-gate kstat_named_t unexp_action; 537c478bd9Sstevel@tonic-gate kstat_named_t unexp_errno; 547c478bd9Sstevel@tonic-gate kstat_named_t unexp_status; 557c478bd9Sstevel@tonic-gate kstat_named_t wrongsec; 567c478bd9Sstevel@tonic-gate kstat_named_t lost_state_bad_op; 577c478bd9Sstevel@tonic-gate } rkstat_t; 587c478bd9Sstevel@tonic-gate 597c478bd9Sstevel@tonic-gate static rkstat_t rkstat_template = { 607c478bd9Sstevel@tonic-gate { "badhandle", KSTAT_DATA_ULONG }, 617c478bd9Sstevel@tonic-gate { "badowner", KSTAT_DATA_ULONG }, 627c478bd9Sstevel@tonic-gate { "clientid", KSTAT_DATA_ULONG }, 637c478bd9Sstevel@tonic-gate { "dead_file", KSTAT_DATA_ULONG }, 647c478bd9Sstevel@tonic-gate { "delay", KSTAT_DATA_ULONG }, 657c478bd9Sstevel@tonic-gate { "fail_relock", KSTAT_DATA_ULONG }, 667c478bd9Sstevel@tonic-gate { "file_diff", KSTAT_DATA_ULONG }, 677c478bd9Sstevel@tonic-gate { "no_grace", KSTAT_DATA_ULONG }, 687c478bd9Sstevel@tonic-gate { "not_responding", KSTAT_DATA_ULONG }, 697c478bd9Sstevel@tonic-gate { "opens_changed", KSTAT_DATA_ULONG }, 707c478bd9Sstevel@tonic-gate { "siglost", KSTAT_DATA_ULONG }, 717c478bd9Sstevel@tonic-gate { "unexp_action", KSTAT_DATA_ULONG }, 727c478bd9Sstevel@tonic-gate { "unexp_errno", KSTAT_DATA_ULONG }, 737c478bd9Sstevel@tonic-gate { "unexp_status", KSTAT_DATA_ULONG }, 747c478bd9Sstevel@tonic-gate { "wrongsec", KSTAT_DATA_ULONG }, 757c478bd9Sstevel@tonic-gate { "bad_op", KSTAT_DATA_ULONG }, 767c478bd9Sstevel@tonic-gate }; 777c478bd9Sstevel@tonic-gate 787c478bd9Sstevel@tonic-gate /* maximum number of messages allowed on the mi's mi_msg_list */ 797c478bd9Sstevel@tonic-gate int nfs4_msg_max = NFS4_MSG_MAX; 807c478bd9Sstevel@tonic-gate #define DEFAULT_LEASE 180 817c478bd9Sstevel@tonic-gate 827c478bd9Sstevel@tonic-gate /* 837c478bd9Sstevel@tonic-gate * Sets the appropiate fields of "ep", given "id" and various parameters. 847c478bd9Sstevel@tonic-gate * Assumes that ep's fields have been initialized to zero/null, except for 857c478bd9Sstevel@tonic-gate * re_type and mount point info, which are already set. 867c478bd9Sstevel@tonic-gate */ 877c478bd9Sstevel@tonic-gate static void 887c478bd9Sstevel@tonic-gate set_event(nfs4_event_type_t id, nfs4_revent_t *ep, mntinfo4_t *mi, 897c478bd9Sstevel@tonic-gate rnode4_t *rp1, rnode4_t *rp2, uint_t count, pid_t pid, nfsstat4 nfs4_error, 907c478bd9Sstevel@tonic-gate char *server1, char *why, nfs4_tag_type_t tag1, nfs4_tag_type_t tag2, 917c478bd9Sstevel@tonic-gate seqid4 seqid1, seqid4 seqid2) 927c478bd9Sstevel@tonic-gate { 937c478bd9Sstevel@tonic-gate int len; 947c478bd9Sstevel@tonic-gate 957c478bd9Sstevel@tonic-gate switch (id) { 967c478bd9Sstevel@tonic-gate case RE_BAD_SEQID: 977c478bd9Sstevel@tonic-gate ep->re_mi = mi; 987c478bd9Sstevel@tonic-gate 997c478bd9Sstevel@tonic-gate /* bad seqid'd file <path/component name> */ 1007c478bd9Sstevel@tonic-gate if (rp1 && rp1->r_svnode.sv_name) 1017c478bd9Sstevel@tonic-gate ep->re_char1 = fn_path(rp1->r_svnode.sv_name); 1027c478bd9Sstevel@tonic-gate else 1037c478bd9Sstevel@tonic-gate ep->re_char1 = NULL; 1047c478bd9Sstevel@tonic-gate ep->re_rp1 = rp1; 1057c478bd9Sstevel@tonic-gate 1067c478bd9Sstevel@tonic-gate /* for LOCK/LOCKU */ 1077c478bd9Sstevel@tonic-gate ep->re_pid = pid; 1087c478bd9Sstevel@tonic-gate 1097c478bd9Sstevel@tonic-gate ep->re_stat4 = nfs4_error; 1107c478bd9Sstevel@tonic-gate ep->re_tag1 = tag1; 1117c478bd9Sstevel@tonic-gate ep->re_tag2 = tag2; 1127c478bd9Sstevel@tonic-gate ep->re_seqid1 = seqid1; 1137c478bd9Sstevel@tonic-gate ep->re_seqid2 = seqid2; 1147c478bd9Sstevel@tonic-gate break; 1157c478bd9Sstevel@tonic-gate case RE_BADHANDLE: 1167c478bd9Sstevel@tonic-gate ASSERT(rp1 != NULL); 1177c478bd9Sstevel@tonic-gate 1187c478bd9Sstevel@tonic-gate /* dead file <path/component name> */ 1197c478bd9Sstevel@tonic-gate if (rp1->r_svnode.sv_name) 1207c478bd9Sstevel@tonic-gate ep->re_char1 = fn_path(rp1->r_svnode.sv_name); 1217c478bd9Sstevel@tonic-gate else 1227c478bd9Sstevel@tonic-gate ep->re_char1 = NULL; 1237c478bd9Sstevel@tonic-gate ep->re_rp1 = rp1; 1247c478bd9Sstevel@tonic-gate break; 1257c478bd9Sstevel@tonic-gate case RE_CLIENTID: 1267c478bd9Sstevel@tonic-gate ep->re_mi = mi; 1277c478bd9Sstevel@tonic-gate 1287c478bd9Sstevel@tonic-gate /* the error we failed with */ 1297c478bd9Sstevel@tonic-gate ep->re_uint = count; 1307c478bd9Sstevel@tonic-gate ep->re_stat4 = nfs4_error; 1317c478bd9Sstevel@tonic-gate break; 1327c478bd9Sstevel@tonic-gate case RE_DEAD_FILE: 1337c478bd9Sstevel@tonic-gate ASSERT(rp1 != NULL); 1347c478bd9Sstevel@tonic-gate 1357c478bd9Sstevel@tonic-gate /* dead file <path/component name> */ 1367c478bd9Sstevel@tonic-gate if (rp1->r_svnode.sv_name) 1377c478bd9Sstevel@tonic-gate ep->re_char1 = fn_path(rp1->r_svnode.sv_name); 1387c478bd9Sstevel@tonic-gate else 1397c478bd9Sstevel@tonic-gate ep->re_char1 = NULL; 1407c478bd9Sstevel@tonic-gate ep->re_rp1 = rp1; 1417c478bd9Sstevel@tonic-gate 1427c478bd9Sstevel@tonic-gate /* why the file got killed */ 1437c478bd9Sstevel@tonic-gate if (why) { 1447c478bd9Sstevel@tonic-gate len = strlen(why); 1457c478bd9Sstevel@tonic-gate ep->re_char2 = kmem_alloc(len + 1, KM_SLEEP); 1467c478bd9Sstevel@tonic-gate bcopy(why, ep->re_char2, len); 1477c478bd9Sstevel@tonic-gate ep->re_char2[len] = '\0'; 1487c478bd9Sstevel@tonic-gate } else 1497c478bd9Sstevel@tonic-gate ep->re_char2 = NULL; 1507c478bd9Sstevel@tonic-gate 1517c478bd9Sstevel@tonic-gate ep->re_stat4 = nfs4_error; 1527c478bd9Sstevel@tonic-gate break; 1537c478bd9Sstevel@tonic-gate case RE_END: 1547c478bd9Sstevel@tonic-gate /* first rnode */ 1557c478bd9Sstevel@tonic-gate if (rp1 && rp1->r_svnode.sv_name) 1567c478bd9Sstevel@tonic-gate ep->re_char1 = fn_path(rp1->r_svnode.sv_name); 1577c478bd9Sstevel@tonic-gate else 1587c478bd9Sstevel@tonic-gate ep->re_char1 = NULL; 1597c478bd9Sstevel@tonic-gate ep->re_rp1 = rp1; 1607c478bd9Sstevel@tonic-gate 1617c478bd9Sstevel@tonic-gate /* second rnode */ 1627c478bd9Sstevel@tonic-gate if (rp2 && rp2->r_svnode.sv_name) 1637c478bd9Sstevel@tonic-gate ep->re_char2 = fn_path(rp2->r_svnode.sv_name); 1647c478bd9Sstevel@tonic-gate else 1657c478bd9Sstevel@tonic-gate ep->re_char2 = NULL; 1667c478bd9Sstevel@tonic-gate ep->re_rp2 = rp2; 1677c478bd9Sstevel@tonic-gate 1687c478bd9Sstevel@tonic-gate ep->re_mi = mi; 1697c478bd9Sstevel@tonic-gate break; 1707c478bd9Sstevel@tonic-gate case RE_FAIL_RELOCK: 1717c478bd9Sstevel@tonic-gate ASSERT(rp1 != NULL); 1727c478bd9Sstevel@tonic-gate 1737c478bd9Sstevel@tonic-gate /* error on fail relock */ 1747c478bd9Sstevel@tonic-gate ep->re_uint = count; 1757c478bd9Sstevel@tonic-gate 1767c478bd9Sstevel@tonic-gate /* process that failed */ 1777c478bd9Sstevel@tonic-gate ep->re_pid = pid; 1787c478bd9Sstevel@tonic-gate 1797c478bd9Sstevel@tonic-gate /* nfs4 error */ 1807c478bd9Sstevel@tonic-gate ep->re_stat4 = nfs4_error; 1817c478bd9Sstevel@tonic-gate 1827c478bd9Sstevel@tonic-gate /* file <path/component name> */ 1837c478bd9Sstevel@tonic-gate if (rp1->r_svnode.sv_name) 1847c478bd9Sstevel@tonic-gate ep->re_char1 = fn_path(rp1->r_svnode.sv_name); 1857c478bd9Sstevel@tonic-gate else 1867c478bd9Sstevel@tonic-gate ep->re_char1 = NULL; 1877c478bd9Sstevel@tonic-gate ep->re_rp1 = rp1; 1887c478bd9Sstevel@tonic-gate break; 1897c478bd9Sstevel@tonic-gate case RE_FAIL_REMAP_LEN: 1907c478bd9Sstevel@tonic-gate /* length of returned filehandle */ 1917c478bd9Sstevel@tonic-gate ep->re_uint = count; 1927c478bd9Sstevel@tonic-gate break; 1937c478bd9Sstevel@tonic-gate case RE_FAIL_REMAP_OP: 1947c478bd9Sstevel@tonic-gate break; 1957c478bd9Sstevel@tonic-gate case RE_FAILOVER: 1967c478bd9Sstevel@tonic-gate /* server we're failing over to (if not picking original) */ 1977c478bd9Sstevel@tonic-gate if (server1 != NULL) { 1987c478bd9Sstevel@tonic-gate len = strlen(server1); 1997c478bd9Sstevel@tonic-gate ep->re_char1 = kmem_alloc(len + 1, KM_SLEEP); 2007c478bd9Sstevel@tonic-gate bcopy(server1, ep->re_char1, len); 2017c478bd9Sstevel@tonic-gate ep->re_char1[len] = '\0'; 2027c478bd9Sstevel@tonic-gate } else { 2037c478bd9Sstevel@tonic-gate ep->re_char1 = NULL; 2047c478bd9Sstevel@tonic-gate } 2057c478bd9Sstevel@tonic-gate break; 2067c478bd9Sstevel@tonic-gate case RE_FILE_DIFF: 2077c478bd9Sstevel@tonic-gate ASSERT(rp1 != NULL); 2087c478bd9Sstevel@tonic-gate 2097c478bd9Sstevel@tonic-gate /* dead file <path/component name> */ 2107c478bd9Sstevel@tonic-gate if (rp1->r_svnode.sv_name) 2117c478bd9Sstevel@tonic-gate ep->re_char1 = fn_path(rp1->r_svnode.sv_name); 2127c478bd9Sstevel@tonic-gate else 2137c478bd9Sstevel@tonic-gate ep->re_char1 = NULL; 2147c478bd9Sstevel@tonic-gate ep->re_rp1 = rp1; 2157c478bd9Sstevel@tonic-gate break; 2167c478bd9Sstevel@tonic-gate case RE_LOST_STATE: 2177c478bd9Sstevel@tonic-gate ep->re_uint = count; /* op number */ 2187c478bd9Sstevel@tonic-gate if (rp1 && rp1->r_svnode.sv_name) 2197c478bd9Sstevel@tonic-gate ep->re_char1 = fn_path(rp1->r_svnode.sv_name); 2207c478bd9Sstevel@tonic-gate else 2217c478bd9Sstevel@tonic-gate ep->re_char1 = NULL; 2227c478bd9Sstevel@tonic-gate ep->re_rp1 = rp1; 2237c478bd9Sstevel@tonic-gate if (rp2 && rp2->r_svnode.sv_name) 2247c478bd9Sstevel@tonic-gate ep->re_char2 = fn_path(rp2->r_svnode.sv_name); 2257c478bd9Sstevel@tonic-gate else 2267c478bd9Sstevel@tonic-gate ep->re_char2 = NULL; 2277c478bd9Sstevel@tonic-gate ep->re_rp2 = rp2; 2287c478bd9Sstevel@tonic-gate break; 2297c478bd9Sstevel@tonic-gate case RE_OPENS_CHANGED: 2307c478bd9Sstevel@tonic-gate ep->re_mi = mi; 2317c478bd9Sstevel@tonic-gate 2327c478bd9Sstevel@tonic-gate /* original number of open files */ 2337c478bd9Sstevel@tonic-gate ep->re_uint = count; 2347c478bd9Sstevel@tonic-gate /* new number of open files */ 2357c478bd9Sstevel@tonic-gate ep->re_pid = pid; 2367c478bd9Sstevel@tonic-gate break; 2377c478bd9Sstevel@tonic-gate case RE_SIGLOST: 2387c478bd9Sstevel@tonic-gate case RE_SIGLOST_NO_DUMP: 2397c478bd9Sstevel@tonic-gate ASSERT(rp1 != NULL); 2407c478bd9Sstevel@tonic-gate 2417c478bd9Sstevel@tonic-gate /* file <path/component name> */ 2427c478bd9Sstevel@tonic-gate if (rp1->r_svnode.sv_name) 2437c478bd9Sstevel@tonic-gate ep->re_char1 = fn_path(rp1->r_svnode.sv_name); 2447c478bd9Sstevel@tonic-gate else 2457c478bd9Sstevel@tonic-gate ep->re_char1 = NULL; 2467c478bd9Sstevel@tonic-gate ep->re_rp1 = rp1; 2477c478bd9Sstevel@tonic-gate ep->re_pid = pid; 2487c478bd9Sstevel@tonic-gate ep->re_uint = count; 2497c478bd9Sstevel@tonic-gate ep->re_stat4 = nfs4_error; 2507c478bd9Sstevel@tonic-gate break; 2517c478bd9Sstevel@tonic-gate case RE_START: 2527c478bd9Sstevel@tonic-gate /* file <path/component name> */ 2537c478bd9Sstevel@tonic-gate if (rp1 && rp1->r_svnode.sv_name) 2547c478bd9Sstevel@tonic-gate ep->re_char1 = fn_path(rp1->r_svnode.sv_name); 2557c478bd9Sstevel@tonic-gate else 2567c478bd9Sstevel@tonic-gate ep->re_char1 = NULL; 2577c478bd9Sstevel@tonic-gate ep->re_rp1 = rp1; 2587c478bd9Sstevel@tonic-gate 2597c478bd9Sstevel@tonic-gate /* file <path/component name> */ 2607c478bd9Sstevel@tonic-gate if (rp2 && rp2->r_svnode.sv_name) 2617c478bd9Sstevel@tonic-gate ep->re_char2 = fn_path(rp2->r_svnode.sv_name); 2627c478bd9Sstevel@tonic-gate else 2637c478bd9Sstevel@tonic-gate ep->re_char2 = NULL; 2647c478bd9Sstevel@tonic-gate ep->re_rp2 = rp2; 2657c478bd9Sstevel@tonic-gate 2667c478bd9Sstevel@tonic-gate ep->re_mi = mi; 2677c478bd9Sstevel@tonic-gate ep->re_uint = count; 2687c478bd9Sstevel@tonic-gate break; 2697c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_ACTION: 2707c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_ERRNO: 2717c478bd9Sstevel@tonic-gate /* the error that is unexpected */ 2727c478bd9Sstevel@tonic-gate ep->re_uint = count; 2737c478bd9Sstevel@tonic-gate break; 2747c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_STATUS: 2757c478bd9Sstevel@tonic-gate /* nfsstat4 error */ 2767c478bd9Sstevel@tonic-gate ep->re_stat4 = nfs4_error; 2777c478bd9Sstevel@tonic-gate break; 2787c478bd9Sstevel@tonic-gate case RE_WRONGSEC: 2797c478bd9Sstevel@tonic-gate /* the error we failed with */ 2807c478bd9Sstevel@tonic-gate ep->re_uint = count; 2817c478bd9Sstevel@tonic-gate 2827c478bd9Sstevel@tonic-gate /* file <path/component name> */ 2837c478bd9Sstevel@tonic-gate if (rp1 && rp1->r_svnode.sv_name) 2847c478bd9Sstevel@tonic-gate ep->re_char1 = fn_path(rp1->r_svnode.sv_name); 2857c478bd9Sstevel@tonic-gate else 2867c478bd9Sstevel@tonic-gate ep->re_char1 = NULL; 2877c478bd9Sstevel@tonic-gate ep->re_rp1 = rp1; 2887c478bd9Sstevel@tonic-gate 2897c478bd9Sstevel@tonic-gate /* file <path/component name> */ 2907c478bd9Sstevel@tonic-gate if (rp2 && rp2->r_svnode.sv_name) 2917c478bd9Sstevel@tonic-gate ep->re_char2 = fn_path(rp2->r_svnode.sv_name); 2927c478bd9Sstevel@tonic-gate else 2937c478bd9Sstevel@tonic-gate ep->re_char2 = NULL; 2947c478bd9Sstevel@tonic-gate ep->re_rp2 = rp2; 2957c478bd9Sstevel@tonic-gate break; 2967c478bd9Sstevel@tonic-gate case RE_LOST_STATE_BAD_OP: 2977c478bd9Sstevel@tonic-gate ep->re_uint = count; /* the unexpected op */ 2987c478bd9Sstevel@tonic-gate ep->re_pid = pid; 2997c478bd9Sstevel@tonic-gate ep->re_rp1 = rp1; 3007c478bd9Sstevel@tonic-gate if (rp1 != NULL && rp1->r_svnode.sv_name != NULL) 3017c478bd9Sstevel@tonic-gate ep->re_char1 = fn_path(rp1->r_svnode.sv_name); 3027c478bd9Sstevel@tonic-gate ep->re_rp2 = rp2; 3037c478bd9Sstevel@tonic-gate if (rp2 != NULL && rp2->r_svnode.sv_name != NULL) 3047c478bd9Sstevel@tonic-gate ep->re_char2 = fn_path(rp2->r_svnode.sv_name); 3057c478bd9Sstevel@tonic-gate break; 3067c478bd9Sstevel@tonic-gate default: 3077c478bd9Sstevel@tonic-gate break; 3087c478bd9Sstevel@tonic-gate } 3097c478bd9Sstevel@tonic-gate } 3107c478bd9Sstevel@tonic-gate 3117c478bd9Sstevel@tonic-gate /* 3127c478bd9Sstevel@tonic-gate * Sets the appropiate fields of the 'fact' for this 'id'. 3137c478bd9Sstevel@tonic-gate */ 3147c478bd9Sstevel@tonic-gate static void 3157c478bd9Sstevel@tonic-gate set_fact(nfs4_fact_type_t id, nfs4_rfact_t *fp, nfsstat4 stat4, 3167c478bd9Sstevel@tonic-gate nfs4_recov_t raction, nfs_opnum4 op, bool_t reboot, int error, 3177c478bd9Sstevel@tonic-gate vnode_t *vp) 3187c478bd9Sstevel@tonic-gate { 3197c478bd9Sstevel@tonic-gate rnode4_t *rp1; 3207c478bd9Sstevel@tonic-gate 3217c478bd9Sstevel@tonic-gate switch (id) { 3227c478bd9Sstevel@tonic-gate case RF_BADOWNER: 3237c478bd9Sstevel@tonic-gate fp->rf_op = op; 3247c478bd9Sstevel@tonic-gate fp->rf_reboot = reboot; 3257c478bd9Sstevel@tonic-gate fp->rf_stat4 = stat4; 3267c478bd9Sstevel@tonic-gate break; 3277c478bd9Sstevel@tonic-gate case RF_RENEW_EXPIRED: 3287c478bd9Sstevel@tonic-gate break; 3297c478bd9Sstevel@tonic-gate case RF_ERR: 3307c478bd9Sstevel@tonic-gate fp->rf_op = op; 3317c478bd9Sstevel@tonic-gate fp->rf_reboot = reboot; 3327c478bd9Sstevel@tonic-gate fp->rf_stat4 = stat4; 3337c478bd9Sstevel@tonic-gate fp->rf_action = raction; 3347c478bd9Sstevel@tonic-gate fp->rf_error = error; 3357c478bd9Sstevel@tonic-gate break; 3367c478bd9Sstevel@tonic-gate case RF_SRV_OK: 3377c478bd9Sstevel@tonic-gate break; 3387c478bd9Sstevel@tonic-gate case RF_SRV_NOT_RESPOND: 3397c478bd9Sstevel@tonic-gate break; 3407c478bd9Sstevel@tonic-gate case RF_SRVS_OK: 3417c478bd9Sstevel@tonic-gate break; 3427c478bd9Sstevel@tonic-gate case RF_SRVS_NOT_RESPOND: 3437c478bd9Sstevel@tonic-gate gethrestime(&fp->rf_time); 3447c478bd9Sstevel@tonic-gate break; 3457c478bd9Sstevel@tonic-gate case RF_DELMAP_CB_ERR: 3467c478bd9Sstevel@tonic-gate fp->rf_op = op; 3477c478bd9Sstevel@tonic-gate fp->rf_stat4 = stat4; 3487c478bd9Sstevel@tonic-gate 3497c478bd9Sstevel@tonic-gate rp1 = VTOR4(vp); 3507c478bd9Sstevel@tonic-gate fp->rf_rp1 = rp1; 3517c478bd9Sstevel@tonic-gate if (rp1 && rp1->r_svnode.sv_name) 3527c478bd9Sstevel@tonic-gate fp->rf_char1 = fn_path(rp1->r_svnode.sv_name); 3537c478bd9Sstevel@tonic-gate else 3547c478bd9Sstevel@tonic-gate fp->rf_char1 = NULL; 3557c478bd9Sstevel@tonic-gate break; 356*e280ed37SDai Ngo case RF_SENDQ_FULL: 357*e280ed37SDai Ngo break; 3587c478bd9Sstevel@tonic-gate default: 3597c478bd9Sstevel@tonic-gate zcmn_err(getzoneid(), CE_NOTE, "illegal fact %d", id); 3607c478bd9Sstevel@tonic-gate break; 3617c478bd9Sstevel@tonic-gate } 3627c478bd9Sstevel@tonic-gate } 3637c478bd9Sstevel@tonic-gate 3647c478bd9Sstevel@tonic-gate /* 3657c478bd9Sstevel@tonic-gate * Returns 1 if the event/fact is of a successful communication 3667c478bd9Sstevel@tonic-gate * from the server; 0 otherwise. 3677c478bd9Sstevel@tonic-gate */ 3687c478bd9Sstevel@tonic-gate static int 3697c478bd9Sstevel@tonic-gate successful_comm(nfs4_debug_msg_t *msgp) 3707c478bd9Sstevel@tonic-gate { 3717c478bd9Sstevel@tonic-gate if (msgp->msg_type == RM_EVENT) { 3727c478bd9Sstevel@tonic-gate switch (msgp->rmsg_u.msg_event.re_type) { 3737c478bd9Sstevel@tonic-gate case RE_BAD_SEQID: 3747c478bd9Sstevel@tonic-gate case RE_BADHANDLE: 3757c478bd9Sstevel@tonic-gate case RE_FAIL_REMAP_LEN: 3767c478bd9Sstevel@tonic-gate case RE_FAIL_REMAP_OP: 3777c478bd9Sstevel@tonic-gate case RE_FILE_DIFF: 3787c478bd9Sstevel@tonic-gate case RE_START: 3797c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_ACTION: 3807c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_ERRNO: 3817c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_STATUS: 3827c478bd9Sstevel@tonic-gate case RE_WRONGSEC: 3837c478bd9Sstevel@tonic-gate return (1); 3847c478bd9Sstevel@tonic-gate case RE_CLIENTID: 3857c478bd9Sstevel@tonic-gate case RE_DEAD_FILE: 3867c478bd9Sstevel@tonic-gate case RE_END: 3877c478bd9Sstevel@tonic-gate case RE_FAIL_RELOCK: 3887c478bd9Sstevel@tonic-gate case RE_FAILOVER: 3897c478bd9Sstevel@tonic-gate case RE_LOST_STATE: 3907c478bd9Sstevel@tonic-gate case RE_OPENS_CHANGED: 3917c478bd9Sstevel@tonic-gate case RE_SIGLOST: 3927c478bd9Sstevel@tonic-gate case RE_SIGLOST_NO_DUMP: 3937c478bd9Sstevel@tonic-gate case RE_LOST_STATE_BAD_OP: 3947c478bd9Sstevel@tonic-gate return (0); 3957c478bd9Sstevel@tonic-gate default: 3967c478bd9Sstevel@tonic-gate return (0); 3977c478bd9Sstevel@tonic-gate } 3987c478bd9Sstevel@tonic-gate } else { 3997c478bd9Sstevel@tonic-gate switch (msgp->rmsg_u.msg_fact.rf_type) { 4007c478bd9Sstevel@tonic-gate case RF_BADOWNER: 4017c478bd9Sstevel@tonic-gate case RF_ERR: 4027c478bd9Sstevel@tonic-gate case RF_RENEW_EXPIRED: 4037c478bd9Sstevel@tonic-gate case RF_SRV_OK: 4047c478bd9Sstevel@tonic-gate case RF_SRVS_OK: 4057c478bd9Sstevel@tonic-gate case RF_DELMAP_CB_ERR: 4067c478bd9Sstevel@tonic-gate return (1); 4077c478bd9Sstevel@tonic-gate case RF_SRV_NOT_RESPOND: 4087c478bd9Sstevel@tonic-gate case RF_SRVS_NOT_RESPOND: 409*e280ed37SDai Ngo case RF_SENDQ_FULL: 4107c478bd9Sstevel@tonic-gate return (0); 4117c478bd9Sstevel@tonic-gate default: 4127c478bd9Sstevel@tonic-gate return (0); 4137c478bd9Sstevel@tonic-gate } 4147c478bd9Sstevel@tonic-gate } 4157c478bd9Sstevel@tonic-gate } 4167c478bd9Sstevel@tonic-gate 4177c478bd9Sstevel@tonic-gate /* 4187c478bd9Sstevel@tonic-gate * Iterate backwards through the mi's mi_msg_list to find the earliest 4197c478bd9Sstevel@tonic-gate * message that we should find relevant facts to investigate. 4207c478bd9Sstevel@tonic-gate */ 4217c478bd9Sstevel@tonic-gate static nfs4_debug_msg_t * 4227c478bd9Sstevel@tonic-gate find_beginning(nfs4_debug_msg_t *first_msg, mntinfo4_t *mi) 4237c478bd9Sstevel@tonic-gate { 4247c478bd9Sstevel@tonic-gate nfs4_debug_msg_t *oldest_msg, *cur_msg; 4257c478bd9Sstevel@tonic-gate time_t lease; 4267c478bd9Sstevel@tonic-gate 4277c478bd9Sstevel@tonic-gate ASSERT(mutex_owned(&mi->mi_msg_list_lock)); 4287c478bd9Sstevel@tonic-gate if (mi->mi_lease_period > 0) 4297c478bd9Sstevel@tonic-gate lease = 2 * mi->mi_lease_period; 4307c478bd9Sstevel@tonic-gate else 4317c478bd9Sstevel@tonic-gate lease = DEFAULT_LEASE; 4327c478bd9Sstevel@tonic-gate 4337c478bd9Sstevel@tonic-gate oldest_msg = first_msg; 4347c478bd9Sstevel@tonic-gate cur_msg = list_prev(&mi->mi_msg_list, first_msg); 4357c478bd9Sstevel@tonic-gate while (cur_msg && 4367c478bd9Sstevel@tonic-gate first_msg->msg_time.tv_sec - cur_msg->msg_time.tv_sec < lease) { 4377c478bd9Sstevel@tonic-gate oldest_msg = cur_msg; 4387c478bd9Sstevel@tonic-gate if ((cur_msg->msg_type == RM_FACT) && 4397c478bd9Sstevel@tonic-gate (cur_msg->rmsg_u.msg_fact.rf_type == RF_SRV_OK)) { 4407c478bd9Sstevel@tonic-gate /* find where we lost contact with the server */ 4417c478bd9Sstevel@tonic-gate while (cur_msg) { 4427c478bd9Sstevel@tonic-gate if ((cur_msg->msg_type == RM_FACT) && 4437c478bd9Sstevel@tonic-gate (cur_msg->rmsg_u.msg_fact.rf_type == 4447c478bd9Sstevel@tonic-gate RF_SRV_NOT_RESPOND)) 4457c478bd9Sstevel@tonic-gate break; 4467c478bd9Sstevel@tonic-gate oldest_msg = cur_msg; 4477c478bd9Sstevel@tonic-gate cur_msg = list_prev(&mi->mi_msg_list, cur_msg); 4487c478bd9Sstevel@tonic-gate } 4497c478bd9Sstevel@tonic-gate /* 4507c478bd9Sstevel@tonic-gate * Find the first successful message before 4517c478bd9Sstevel@tonic-gate * we lost contact with the server. 4527c478bd9Sstevel@tonic-gate */ 4537c478bd9Sstevel@tonic-gate if (cur_msg) { 4547c478bd9Sstevel@tonic-gate cur_msg = list_prev(&mi->mi_msg_list, cur_msg); 4557c478bd9Sstevel@tonic-gate while (cur_msg && !successful_comm(cur_msg)) { 4567c478bd9Sstevel@tonic-gate oldest_msg = cur_msg; 4577c478bd9Sstevel@tonic-gate cur_msg = list_prev(&mi->mi_msg_list, 4587c478bd9Sstevel@tonic-gate cur_msg); 4597c478bd9Sstevel@tonic-gate } 4607c478bd9Sstevel@tonic-gate } 4617c478bd9Sstevel@tonic-gate /* 4627c478bd9Sstevel@tonic-gate * If we're not at the dummy head pointer, 4637c478bd9Sstevel@tonic-gate * set the oldest and current message. 4647c478bd9Sstevel@tonic-gate */ 4657c478bd9Sstevel@tonic-gate if (cur_msg) { 4667c478bd9Sstevel@tonic-gate first_msg = cur_msg; 4677c478bd9Sstevel@tonic-gate oldest_msg = cur_msg; 4687c478bd9Sstevel@tonic-gate cur_msg = list_prev(&mi->mi_msg_list, cur_msg); 4697c478bd9Sstevel@tonic-gate } 4707c478bd9Sstevel@tonic-gate } else 4717c478bd9Sstevel@tonic-gate cur_msg = list_prev(&mi->mi_msg_list, cur_msg); 4727c478bd9Sstevel@tonic-gate } 4737c478bd9Sstevel@tonic-gate 4747c478bd9Sstevel@tonic-gate return (oldest_msg); 4757c478bd9Sstevel@tonic-gate } 4767c478bd9Sstevel@tonic-gate 4777c478bd9Sstevel@tonic-gate /* 4787c478bd9Sstevel@tonic-gate * Returns 1 if facts have been found; 0 otherwise. 4797c478bd9Sstevel@tonic-gate */ 4807c478bd9Sstevel@tonic-gate static int 4817c478bd9Sstevel@tonic-gate get_facts(nfs4_debug_msg_t *msgp, nfs4_rfact_t *ret_fp, char **mnt_pt, 4827c478bd9Sstevel@tonic-gate mntinfo4_t *mi) 4837c478bd9Sstevel@tonic-gate { 4847c478bd9Sstevel@tonic-gate nfs4_debug_msg_t *cur_msg, *oldest_msg; 4857c478bd9Sstevel@tonic-gate nfs4_rfact_t *cur_fp; 4867c478bd9Sstevel@tonic-gate int found_a_fact = 0; 4877c478bd9Sstevel@tonic-gate int len; 4887c478bd9Sstevel@tonic-gate 4897c478bd9Sstevel@tonic-gate cur_msg = msgp; 4907c478bd9Sstevel@tonic-gate 4917c478bd9Sstevel@tonic-gate /* find the oldest msg to search backwards to */ 4927c478bd9Sstevel@tonic-gate oldest_msg = find_beginning(cur_msg, mi); 4937c478bd9Sstevel@tonic-gate ASSERT(oldest_msg != NULL); 4947c478bd9Sstevel@tonic-gate 4957c478bd9Sstevel@tonic-gate /* 4967c478bd9Sstevel@tonic-gate * Create a fact sheet by searching from our current message 4977c478bd9Sstevel@tonic-gate * backwards to the 'oldest_msg', recording facts along the way 4987c478bd9Sstevel@tonic-gate * until we found facts that have been inspected by another time. 4997c478bd9Sstevel@tonic-gate */ 5007c478bd9Sstevel@tonic-gate while (cur_msg && cur_msg != list_prev(&mi->mi_msg_list, oldest_msg)) { 5017c478bd9Sstevel@tonic-gate if (cur_msg->msg_type != RM_FACT) { 5027c478bd9Sstevel@tonic-gate cur_msg = list_prev(&mi->mi_msg_list, cur_msg); 5037c478bd9Sstevel@tonic-gate continue; 5047c478bd9Sstevel@tonic-gate } 5057c478bd9Sstevel@tonic-gate 5067c478bd9Sstevel@tonic-gate cur_fp = &cur_msg->rmsg_u.msg_fact; 5077c478bd9Sstevel@tonic-gate /* 5087c478bd9Sstevel@tonic-gate * If this fact has already been looked at, then so 5097c478bd9Sstevel@tonic-gate * have all preceding facts. Return Now. 5107c478bd9Sstevel@tonic-gate */ 5117c478bd9Sstevel@tonic-gate if (cur_fp->rf_status == RFS_INSPECT) 5127c478bd9Sstevel@tonic-gate return (found_a_fact); 5137c478bd9Sstevel@tonic-gate 5147c478bd9Sstevel@tonic-gate cur_fp->rf_status = RFS_INSPECT; 5157c478bd9Sstevel@tonic-gate found_a_fact = 1; 5167c478bd9Sstevel@tonic-gate switch (cur_fp->rf_type) { 5177c478bd9Sstevel@tonic-gate case RF_BADOWNER: 5187c478bd9Sstevel@tonic-gate break; 5197c478bd9Sstevel@tonic-gate case RF_ERR: 5207c478bd9Sstevel@tonic-gate /* 5217c478bd9Sstevel@tonic-gate * Don't want to overwrite a fact that was 5227c478bd9Sstevel@tonic-gate * previously found during our current search. 5237c478bd9Sstevel@tonic-gate */ 5247c478bd9Sstevel@tonic-gate if (!ret_fp->rf_reboot) 5257c478bd9Sstevel@tonic-gate ret_fp->rf_reboot = cur_fp->rf_reboot; 5267c478bd9Sstevel@tonic-gate if (!ret_fp->rf_stat4) 5277c478bd9Sstevel@tonic-gate ret_fp->rf_stat4 = cur_fp->rf_stat4; 5287c478bd9Sstevel@tonic-gate if (!ret_fp->rf_action) 5297c478bd9Sstevel@tonic-gate ret_fp->rf_action = cur_fp->rf_action; 5307c478bd9Sstevel@tonic-gate break; 5317c478bd9Sstevel@tonic-gate case RF_RENEW_EXPIRED: 5327c478bd9Sstevel@tonic-gate if (cur_msg->msg_mntpt && !(*mnt_pt)) { 5337c478bd9Sstevel@tonic-gate len = strlen(cur_msg->msg_mntpt) + 1; 5347c478bd9Sstevel@tonic-gate *mnt_pt = kmem_alloc(len, KM_SLEEP); 5357c478bd9Sstevel@tonic-gate bcopy(cur_msg->msg_mntpt, *mnt_pt, len); 5367c478bd9Sstevel@tonic-gate } 5377c478bd9Sstevel@tonic-gate break; 5387c478bd9Sstevel@tonic-gate case RF_SRV_OK: 5397c478bd9Sstevel@tonic-gate break; 5407c478bd9Sstevel@tonic-gate case RF_SRV_NOT_RESPOND: 5417c478bd9Sstevel@tonic-gate /* 5427c478bd9Sstevel@tonic-gate * Okay to overwrite this fact as 5437c478bd9Sstevel@tonic-gate * we want the earliest time. 5447c478bd9Sstevel@tonic-gate */ 5457c478bd9Sstevel@tonic-gate ret_fp->rf_time = cur_fp->rf_time; 5467c478bd9Sstevel@tonic-gate break; 5477c478bd9Sstevel@tonic-gate case RF_SRVS_OK: 5487c478bd9Sstevel@tonic-gate break; 5497c478bd9Sstevel@tonic-gate case RF_SRVS_NOT_RESPOND: 5507c478bd9Sstevel@tonic-gate break; 5517c478bd9Sstevel@tonic-gate case RF_DELMAP_CB_ERR: 5527c478bd9Sstevel@tonic-gate break; 553*e280ed37SDai Ngo case RF_SENDQ_FULL: 554*e280ed37SDai Ngo break; 5557c478bd9Sstevel@tonic-gate default: 5567c478bd9Sstevel@tonic-gate zcmn_err(getzoneid(), CE_NOTE, 5577c478bd9Sstevel@tonic-gate "get facts: illegal fact %d", cur_fp->rf_type); 5587c478bd9Sstevel@tonic-gate break; 5597c478bd9Sstevel@tonic-gate } 5607c478bd9Sstevel@tonic-gate cur_msg = list_prev(&mi->mi_msg_list, cur_msg); 5617c478bd9Sstevel@tonic-gate } 5627c478bd9Sstevel@tonic-gate 5637c478bd9Sstevel@tonic-gate return (found_a_fact); 5647c478bd9Sstevel@tonic-gate } 5657c478bd9Sstevel@tonic-gate 5667c478bd9Sstevel@tonic-gate /* 5677c478bd9Sstevel@tonic-gate * Returns 1 if this fact is identical to the last fact recorded 5687c478bd9Sstevel@tonic-gate * (only checks for a match within the last 2 lease periods). 5697c478bd9Sstevel@tonic-gate */ 5707c478bd9Sstevel@tonic-gate static int 5717c478bd9Sstevel@tonic-gate facts_same(nfs4_debug_msg_t *cur_msg, nfs4_debug_msg_t *new_msg, 5727c478bd9Sstevel@tonic-gate mntinfo4_t *mi) 5737c478bd9Sstevel@tonic-gate { 5747c478bd9Sstevel@tonic-gate nfs4_rfact_t *fp1, *fp2; 5757c478bd9Sstevel@tonic-gate int lease, len; 5767c478bd9Sstevel@tonic-gate 5777c478bd9Sstevel@tonic-gate ASSERT(mutex_owned(&mi->mi_msg_list_lock)); 5787c478bd9Sstevel@tonic-gate if (mi->mi_lease_period > 0) 5797c478bd9Sstevel@tonic-gate lease = 2 * mi->mi_lease_period; 5807c478bd9Sstevel@tonic-gate else 5817c478bd9Sstevel@tonic-gate lease = DEFAULT_LEASE; 5827c478bd9Sstevel@tonic-gate 5837c478bd9Sstevel@tonic-gate fp2 = &new_msg->rmsg_u.msg_fact; 5847c478bd9Sstevel@tonic-gate 5857c478bd9Sstevel@tonic-gate while (cur_msg && 5867c478bd9Sstevel@tonic-gate new_msg->msg_time.tv_sec - cur_msg->msg_time.tv_sec < lease) { 5877c478bd9Sstevel@tonic-gate if (cur_msg->msg_type != RM_FACT) { 5887c478bd9Sstevel@tonic-gate cur_msg = list_prev(&mi->mi_msg_list, cur_msg); 5897c478bd9Sstevel@tonic-gate continue; 5907c478bd9Sstevel@tonic-gate } 5917c478bd9Sstevel@tonic-gate fp1 = &cur_msg->rmsg_u.msg_fact; 5927c478bd9Sstevel@tonic-gate if (fp1->rf_type != fp2->rf_type) 5937c478bd9Sstevel@tonic-gate return (0); 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate /* now actually compare the facts */ 5967c478bd9Sstevel@tonic-gate if (fp1->rf_action != fp2->rf_action) 5977c478bd9Sstevel@tonic-gate return (0); 5987c478bd9Sstevel@tonic-gate if (fp1->rf_stat4 != fp2->rf_stat4) 5997c478bd9Sstevel@tonic-gate return (0); 6007c478bd9Sstevel@tonic-gate if (fp1->rf_reboot != fp2->rf_reboot) 6017c478bd9Sstevel@tonic-gate return (0); 6027c478bd9Sstevel@tonic-gate if (fp1->rf_op != fp2->rf_op) 6037c478bd9Sstevel@tonic-gate return (0); 6047c478bd9Sstevel@tonic-gate if (fp1->rf_time.tv_sec != fp2->rf_time.tv_sec) 6057c478bd9Sstevel@tonic-gate return (0); 6067c478bd9Sstevel@tonic-gate if (fp1->rf_error != fp2->rf_error) 6077c478bd9Sstevel@tonic-gate return (0); 6087c478bd9Sstevel@tonic-gate if (fp1->rf_rp1 != fp2->rf_rp1) 6097c478bd9Sstevel@tonic-gate return (0); 6107c478bd9Sstevel@tonic-gate if (cur_msg->msg_srv != NULL) { 6117c478bd9Sstevel@tonic-gate if (new_msg->msg_srv == NULL) 6127c478bd9Sstevel@tonic-gate return (0); 6137c478bd9Sstevel@tonic-gate len = strlen(cur_msg->msg_srv); 6147c478bd9Sstevel@tonic-gate if (strncmp(cur_msg->msg_srv, new_msg->msg_srv, 6157f0b8309SEdward Pilatowicz len) != 0) 6167c478bd9Sstevel@tonic-gate return (0); 6177c478bd9Sstevel@tonic-gate } else if (new_msg->msg_srv != NULL) { 6187c478bd9Sstevel@tonic-gate return (0); 6197c478bd9Sstevel@tonic-gate } 6207c478bd9Sstevel@tonic-gate if (cur_msg->msg_mntpt != NULL) { 6217c478bd9Sstevel@tonic-gate if (new_msg->msg_mntpt == NULL) 6227c478bd9Sstevel@tonic-gate return (0); 6237c478bd9Sstevel@tonic-gate len = strlen(cur_msg->msg_mntpt); 6247c478bd9Sstevel@tonic-gate if (strncmp(cur_msg->msg_mntpt, new_msg->msg_mntpt, 6257f0b8309SEdward Pilatowicz len) != 0) 6267c478bd9Sstevel@tonic-gate return (0); 6277c478bd9Sstevel@tonic-gate } else if (new_msg->msg_mntpt != NULL) { 6287c478bd9Sstevel@tonic-gate return (0); 6297c478bd9Sstevel@tonic-gate } 6307c478bd9Sstevel@tonic-gate if (fp1->rf_char1 != NULL) { 6317c478bd9Sstevel@tonic-gate if (fp2->rf_char1 == NULL) 6327c478bd9Sstevel@tonic-gate return (0); 6337c478bd9Sstevel@tonic-gate len = strlen(fp1->rf_char1); 6347c478bd9Sstevel@tonic-gate if (strncmp(fp1->rf_char1, fp2->rf_char1, len) != 0) 6357c478bd9Sstevel@tonic-gate return (0); 6367c478bd9Sstevel@tonic-gate } else if (fp2->rf_char1 != NULL) { 6377c478bd9Sstevel@tonic-gate return (0); 6387c478bd9Sstevel@tonic-gate } 6397c478bd9Sstevel@tonic-gate return (1); 6407c478bd9Sstevel@tonic-gate } 6417c478bd9Sstevel@tonic-gate 6427c478bd9Sstevel@tonic-gate return (0); 6437c478bd9Sstevel@tonic-gate } 6447c478bd9Sstevel@tonic-gate 6457c478bd9Sstevel@tonic-gate /* 6467c478bd9Sstevel@tonic-gate * Returns 1 if these two messages are identical; 0 otherwise. 6477c478bd9Sstevel@tonic-gate */ 6487c478bd9Sstevel@tonic-gate static int 6497c478bd9Sstevel@tonic-gate events_same(nfs4_debug_msg_t *cur_msg, nfs4_debug_msg_t *new_msg, 6507c478bd9Sstevel@tonic-gate mntinfo4_t *mi) 6517c478bd9Sstevel@tonic-gate { 6527c478bd9Sstevel@tonic-gate nfs4_revent_t *ep1, *ep2; 6537c478bd9Sstevel@tonic-gate int len; 6547c478bd9Sstevel@tonic-gate 6557c478bd9Sstevel@tonic-gate /* find the last event, bypassing all facts */ 6567c478bd9Sstevel@tonic-gate while (cur_msg && cur_msg->msg_type != RM_EVENT) 6577c478bd9Sstevel@tonic-gate cur_msg = list_prev(&mi->mi_msg_list, cur_msg); 6587c478bd9Sstevel@tonic-gate 6597c478bd9Sstevel@tonic-gate if (!cur_msg) 6607c478bd9Sstevel@tonic-gate return (0); 6617c478bd9Sstevel@tonic-gate 6627c478bd9Sstevel@tonic-gate if (cur_msg->msg_type != RM_EVENT) 6637c478bd9Sstevel@tonic-gate return (0); 6647c478bd9Sstevel@tonic-gate 6657c478bd9Sstevel@tonic-gate ep1 = &cur_msg->rmsg_u.msg_event; 6667c478bd9Sstevel@tonic-gate ep2 = &new_msg->rmsg_u.msg_event; 6677c478bd9Sstevel@tonic-gate if (ep1->re_type != ep2->re_type) 6687c478bd9Sstevel@tonic-gate return (0); 6697c478bd9Sstevel@tonic-gate 6707c478bd9Sstevel@tonic-gate /* 6717c478bd9Sstevel@tonic-gate * Since we zalloc the buffer, then the two nfs4_debug_msg's 6727c478bd9Sstevel@tonic-gate * must match up even if all the fields weren't filled in 6737c478bd9Sstevel@tonic-gate * the first place. 6747c478bd9Sstevel@tonic-gate */ 6757c478bd9Sstevel@tonic-gate if (ep1->re_mi != ep2->re_mi) 6767c478bd9Sstevel@tonic-gate return (0); 6777c478bd9Sstevel@tonic-gate if (ep1->re_uint != ep2->re_uint) 6787c478bd9Sstevel@tonic-gate return (0); 6797c478bd9Sstevel@tonic-gate if (ep1->re_stat4 != ep2->re_stat4) 6807c478bd9Sstevel@tonic-gate return (0); 6817c478bd9Sstevel@tonic-gate if (ep1->re_pid != ep2->re_pid) 6827c478bd9Sstevel@tonic-gate return (0); 6837c478bd9Sstevel@tonic-gate if (ep1->re_rp1 != ep2->re_rp1) 6847c478bd9Sstevel@tonic-gate return (0); 6857c478bd9Sstevel@tonic-gate if (ep1->re_rp2 != ep2->re_rp2) 6867c478bd9Sstevel@tonic-gate return (0); 6877c478bd9Sstevel@tonic-gate if (ep1->re_tag1 != ep2->re_tag1) 6887c478bd9Sstevel@tonic-gate return (0); 6897c478bd9Sstevel@tonic-gate if (ep1->re_tag2 != ep2->re_tag2) 6907c478bd9Sstevel@tonic-gate return (0); 6917c478bd9Sstevel@tonic-gate if (ep1->re_seqid1 != ep2->re_seqid1) 6927c478bd9Sstevel@tonic-gate return (0); 6937c478bd9Sstevel@tonic-gate if (ep1->re_seqid2 != ep2->re_seqid2) 6947c478bd9Sstevel@tonic-gate return (0); 6957c478bd9Sstevel@tonic-gate 6967c478bd9Sstevel@tonic-gate if (cur_msg->msg_srv != NULL) { 6977c478bd9Sstevel@tonic-gate if (new_msg->msg_srv == NULL) 6987c478bd9Sstevel@tonic-gate return (0); 6997c478bd9Sstevel@tonic-gate len = strlen(cur_msg->msg_srv); 7007c478bd9Sstevel@tonic-gate if (strncmp(cur_msg->msg_srv, new_msg->msg_srv, len) != 0) 7017c478bd9Sstevel@tonic-gate return (0); 7027c478bd9Sstevel@tonic-gate } else if (new_msg->msg_srv != NULL) { 7037c478bd9Sstevel@tonic-gate return (0); 7047c478bd9Sstevel@tonic-gate } 7057c478bd9Sstevel@tonic-gate 7067c478bd9Sstevel@tonic-gate if (ep1->re_char1 != NULL) { 7077c478bd9Sstevel@tonic-gate if (ep2->re_char1 == NULL) 7087c478bd9Sstevel@tonic-gate return (0); 7097c478bd9Sstevel@tonic-gate len = strlen(ep1->re_char1); 7107c478bd9Sstevel@tonic-gate if (strncmp(ep1->re_char1, ep2->re_char1, len) != 0) 7117c478bd9Sstevel@tonic-gate return (0); 7127c478bd9Sstevel@tonic-gate } else if (ep2->re_char1 != NULL) { 7137c478bd9Sstevel@tonic-gate return (0); 7147c478bd9Sstevel@tonic-gate } 7157c478bd9Sstevel@tonic-gate 7167c478bd9Sstevel@tonic-gate if (ep1->re_char2 != NULL) { 7177c478bd9Sstevel@tonic-gate if (ep2->re_char2 == NULL) 7187c478bd9Sstevel@tonic-gate return (0); 7197c478bd9Sstevel@tonic-gate len = strlen(ep1->re_char2); 7207c478bd9Sstevel@tonic-gate if (strncmp(ep1->re_char2, ep2->re_char2, len) != 0) 7217c478bd9Sstevel@tonic-gate return (0); 7227c478bd9Sstevel@tonic-gate } else if (ep2->re_char2 != NULL) { 7237c478bd9Sstevel@tonic-gate return (0); 7247c478bd9Sstevel@tonic-gate } 7257c478bd9Sstevel@tonic-gate 7267c478bd9Sstevel@tonic-gate if (cur_msg->msg_mntpt != NULL) { 7277c478bd9Sstevel@tonic-gate if (new_msg->msg_mntpt == NULL) 7287c478bd9Sstevel@tonic-gate return (0); 7297c478bd9Sstevel@tonic-gate len = strlen(cur_msg->msg_mntpt); 7307c478bd9Sstevel@tonic-gate if (strncmp(cur_msg->msg_mntpt, cur_msg->msg_mntpt, len) != 0) 7317c478bd9Sstevel@tonic-gate return (0); 7327c478bd9Sstevel@tonic-gate } else if (new_msg->msg_mntpt != NULL) { 7337c478bd9Sstevel@tonic-gate return (0); 7347c478bd9Sstevel@tonic-gate } 7357c478bd9Sstevel@tonic-gate 7367c478bd9Sstevel@tonic-gate return (1); 7377c478bd9Sstevel@tonic-gate } 7387c478bd9Sstevel@tonic-gate 7397c478bd9Sstevel@tonic-gate /* 7407c478bd9Sstevel@tonic-gate * Free up a recovery event. 7417c478bd9Sstevel@tonic-gate */ 7427c478bd9Sstevel@tonic-gate static void 7437c478bd9Sstevel@tonic-gate free_event(nfs4_revent_t *ep) 7447c478bd9Sstevel@tonic-gate { 7457c478bd9Sstevel@tonic-gate int len; 7467c478bd9Sstevel@tonic-gate 7477c478bd9Sstevel@tonic-gate if (ep->re_char1) { 7487c478bd9Sstevel@tonic-gate len = strlen(ep->re_char1) + 1; 7497c478bd9Sstevel@tonic-gate kmem_free(ep->re_char1, len); 7507c478bd9Sstevel@tonic-gate } 7517c478bd9Sstevel@tonic-gate if (ep->re_char2) { 7527c478bd9Sstevel@tonic-gate len = strlen(ep->re_char2) + 1; 7537c478bd9Sstevel@tonic-gate kmem_free(ep->re_char2, len); 7547c478bd9Sstevel@tonic-gate } 7557c478bd9Sstevel@tonic-gate } 7567c478bd9Sstevel@tonic-gate 7577c478bd9Sstevel@tonic-gate /* 7587c478bd9Sstevel@tonic-gate * Free up a recovery fact. 7597c478bd9Sstevel@tonic-gate */ 7607c478bd9Sstevel@tonic-gate static void 7617c478bd9Sstevel@tonic-gate free_fact(nfs4_rfact_t *fp) 7627c478bd9Sstevel@tonic-gate { 7637c478bd9Sstevel@tonic-gate int len; 7647c478bd9Sstevel@tonic-gate 7657c478bd9Sstevel@tonic-gate if (fp->rf_char1) { 7667c478bd9Sstevel@tonic-gate len = strlen(fp->rf_char1) + 1; 7677c478bd9Sstevel@tonic-gate kmem_free(fp->rf_char1, len); 7687c478bd9Sstevel@tonic-gate } 7697c478bd9Sstevel@tonic-gate } 7707c478bd9Sstevel@tonic-gate 7717c478bd9Sstevel@tonic-gate /* 7727c478bd9Sstevel@tonic-gate * Free up the message. 7737c478bd9Sstevel@tonic-gate */ 7747c478bd9Sstevel@tonic-gate void 7757c478bd9Sstevel@tonic-gate nfs4_free_msg(nfs4_debug_msg_t *msg) 7767c478bd9Sstevel@tonic-gate { 7777c478bd9Sstevel@tonic-gate int len; 7787c478bd9Sstevel@tonic-gate 7797c478bd9Sstevel@tonic-gate if (msg->msg_type == RM_EVENT) 7807c478bd9Sstevel@tonic-gate free_event(&msg->rmsg_u.msg_event); 7817c478bd9Sstevel@tonic-gate else 7827c478bd9Sstevel@tonic-gate free_fact(&msg->rmsg_u.msg_fact); 7837c478bd9Sstevel@tonic-gate 7847c478bd9Sstevel@tonic-gate if (msg->msg_srv) { 7857c478bd9Sstevel@tonic-gate len = strlen(msg->msg_srv) + 1; 7867c478bd9Sstevel@tonic-gate kmem_free(msg->msg_srv, len); 7877c478bd9Sstevel@tonic-gate } 7887c478bd9Sstevel@tonic-gate 7897c478bd9Sstevel@tonic-gate if (msg->msg_mntpt) { 7907c478bd9Sstevel@tonic-gate len = strlen(msg->msg_mntpt) + 1; 7917c478bd9Sstevel@tonic-gate kmem_free(msg->msg_mntpt, len); 7927c478bd9Sstevel@tonic-gate } 7937c478bd9Sstevel@tonic-gate 7947c478bd9Sstevel@tonic-gate /* free up the data structure itself */ 7957c478bd9Sstevel@tonic-gate kmem_free(msg, sizeof (*msg)); 7967c478bd9Sstevel@tonic-gate } 7977c478bd9Sstevel@tonic-gate 7987c478bd9Sstevel@tonic-gate /* 7997c478bd9Sstevel@tonic-gate * Prints out the interesting facts for recovery events: 8007c478bd9Sstevel@tonic-gate * -DEAD_FILE 8017c478bd9Sstevel@tonic-gate * -SIGLOST(_NO_DUMP) 8027c478bd9Sstevel@tonic-gate */ 8037c478bd9Sstevel@tonic-gate static void 8047c478bd9Sstevel@tonic-gate print_facts(nfs4_debug_msg_t *msg, mntinfo4_t *mi) 8057c478bd9Sstevel@tonic-gate { 8067c478bd9Sstevel@tonic-gate nfs4_rfact_t *fp; 8077c478bd9Sstevel@tonic-gate char *mount_pt; 8087c478bd9Sstevel@tonic-gate int len; 8097c478bd9Sstevel@tonic-gate 8107c478bd9Sstevel@tonic-gate if (msg->rmsg_u.msg_event.re_type != RE_DEAD_FILE && 8117c478bd9Sstevel@tonic-gate msg->rmsg_u.msg_event.re_type != RE_SIGLOST && 8127c478bd9Sstevel@tonic-gate msg->rmsg_u.msg_event.re_type != RE_SIGLOST_NO_DUMP) 8137c478bd9Sstevel@tonic-gate return; 8147c478bd9Sstevel@tonic-gate 8157c478bd9Sstevel@tonic-gate fp = kmem_zalloc(sizeof (*fp), KM_SLEEP); 8167c478bd9Sstevel@tonic-gate mount_pt = NULL; 8177c478bd9Sstevel@tonic-gate 8187c478bd9Sstevel@tonic-gate if (get_facts(msg, fp, &mount_pt, mi)) { 8197c478bd9Sstevel@tonic-gate char time[256]; 8207c478bd9Sstevel@tonic-gate 8217c478bd9Sstevel@tonic-gate 8227c478bd9Sstevel@tonic-gate if (fp->rf_time.tv_sec) 8237c478bd9Sstevel@tonic-gate (void) snprintf(time, 256, "%ld", 8247c478bd9Sstevel@tonic-gate (gethrestime_sec() - fp->rf_time.tv_sec)/60); 8257c478bd9Sstevel@tonic-gate zcmn_err(mi->mi_zone->zone_id, CE_NOTE, 8267c478bd9Sstevel@tonic-gate "!NFS4 FACT SHEET: %s%s %s%s %s %s%s%s %s%s", 8277c478bd9Sstevel@tonic-gate fp->rf_action ? "\n Action: " : "", 8287c478bd9Sstevel@tonic-gate fp->rf_action ? nfs4_recov_action_to_str(fp->rf_action) : 8297c478bd9Sstevel@tonic-gate "", 8307c478bd9Sstevel@tonic-gate fp->rf_stat4 ? "\n NFS4 error: " : "", 8317c478bd9Sstevel@tonic-gate fp->rf_stat4 ? nfs4_stat_to_str(fp->rf_stat4) : "", 8327c478bd9Sstevel@tonic-gate fp->rf_reboot ? "\n Suspected server reboot. " : "", 8337c478bd9Sstevel@tonic-gate fp->rf_time.tv_sec ? "\n Server was down for " : "", 8347c478bd9Sstevel@tonic-gate fp->rf_time.tv_sec ? time : "", 8357c478bd9Sstevel@tonic-gate fp->rf_time.tv_sec ? " minutes." : "", 8367c478bd9Sstevel@tonic-gate mount_pt ? " \n Client's lease expired on mount " : "", 8377c478bd9Sstevel@tonic-gate mount_pt ? mount_pt : ""); 8387c478bd9Sstevel@tonic-gate } 8397c478bd9Sstevel@tonic-gate 8407c478bd9Sstevel@tonic-gate if (mount_pt) { 8417c478bd9Sstevel@tonic-gate len = strlen(mount_pt) + 1; 8427c478bd9Sstevel@tonic-gate kmem_free(mount_pt, len); 8437c478bd9Sstevel@tonic-gate } 8447c478bd9Sstevel@tonic-gate 8457c478bd9Sstevel@tonic-gate /* free the fact struct itself */ 8467c478bd9Sstevel@tonic-gate if (fp) 8477c478bd9Sstevel@tonic-gate kmem_free(fp, sizeof (*fp)); 8487c478bd9Sstevel@tonic-gate } 8497c478bd9Sstevel@tonic-gate 8507c478bd9Sstevel@tonic-gate /* 8517c478bd9Sstevel@tonic-gate * Print an event message to /var/adm/messages 8527c478bd9Sstevel@tonic-gate * The last argument to this fuction dictates the repeat status 8537c478bd9Sstevel@tonic-gate * of the event. If set to 1, it means that we are dumping this 8547c478bd9Sstevel@tonic-gate * event and it will _never_ be printed after this time. Else if 8557c478bd9Sstevel@tonic-gate * set to 0 it will be printed again. 8567c478bd9Sstevel@tonic-gate */ 8577c478bd9Sstevel@tonic-gate static void 8587c478bd9Sstevel@tonic-gate queue_print_event(nfs4_debug_msg_t *msg, mntinfo4_t *mi, int dump) 8597c478bd9Sstevel@tonic-gate { 8607c478bd9Sstevel@tonic-gate nfs4_revent_t *ep; 8617c478bd9Sstevel@tonic-gate zoneid_t zoneid; 8627c478bd9Sstevel@tonic-gate 8637c478bd9Sstevel@tonic-gate ep = &msg->rmsg_u.msg_event; 8647c478bd9Sstevel@tonic-gate zoneid = mi->mi_zone->zone_id; 8657c478bd9Sstevel@tonic-gate 8667c478bd9Sstevel@tonic-gate switch (ep->re_type) { 8677c478bd9Sstevel@tonic-gate case RE_BAD_SEQID: 8687c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 8697c478bd9Sstevel@tonic-gate "Operation %s for file %s (rnode_pt 0x%p), pid %d using " 8707c478bd9Sstevel@tonic-gate "seqid %d got %s. Last good seqid was %d for " 8717c478bd9Sstevel@tonic-gate "operation %s.", 8727c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt, 8737c478bd9Sstevel@tonic-gate nfs4_ctags[ep->re_tag1].ct_str, ep->re_char1, 8747c478bd9Sstevel@tonic-gate (void *)ep->re_rp1, ep->re_pid, ep->re_seqid1, 8757c478bd9Sstevel@tonic-gate nfs4_stat_to_str(ep->re_stat4), ep->re_seqid2, 8767c478bd9Sstevel@tonic-gate nfs4_ctags[ep->re_tag2].ct_str); 8777c478bd9Sstevel@tonic-gate break; 8787c478bd9Sstevel@tonic-gate case RE_BADHANDLE: 8797c478bd9Sstevel@tonic-gate ASSERT(ep->re_rp1 != NULL); 8807c478bd9Sstevel@tonic-gate if (ep->re_char1 != NULL) { 8817c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, 8827c478bd9Sstevel@tonic-gate "![NFS4][Server: %s][Mntpt: %s]" 8837c478bd9Sstevel@tonic-gate "server %s said filehandle was " 8847c478bd9Sstevel@tonic-gate "invalid for file: %s (rnode_pt 0x%p) on mount %s", 8857c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt, msg->msg_srv, 8867c478bd9Sstevel@tonic-gate ep->re_char1, (void *)ep->re_rp1, msg->msg_mntpt); 8877c478bd9Sstevel@tonic-gate } else { 8887c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, 8897c478bd9Sstevel@tonic-gate "![NFS4][Server: %s][Mntpt: %s]" 8907c478bd9Sstevel@tonic-gate "server %s said filehandle was " 8917c478bd9Sstevel@tonic-gate "invalid for file: (rnode_pt 0x%p) on mount %s" 8927c478bd9Sstevel@tonic-gate " for fh:", msg->msg_srv, msg->msg_mntpt, 8937c478bd9Sstevel@tonic-gate msg->msg_srv, (void *)ep->re_rp1, msg->msg_mntpt); 8947c478bd9Sstevel@tonic-gate sfh4_printfhandle(ep->re_rp1->r_fh); 8957c478bd9Sstevel@tonic-gate } 8967c478bd9Sstevel@tonic-gate break; 8977c478bd9Sstevel@tonic-gate case RE_CLIENTID: 8987c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 8997c478bd9Sstevel@tonic-gate "Can't recover clientid on mount point %s " 9007c478bd9Sstevel@tonic-gate "(mi 0x%p) due to error %d (%s), for server %s. Marking " 9017c478bd9Sstevel@tonic-gate "file system as unusable.", 9027c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt, msg->msg_mntpt, 9037c478bd9Sstevel@tonic-gate (void *)ep->re_mi, ep->re_uint, 9047c478bd9Sstevel@tonic-gate nfs4_stat_to_str(ep->re_stat4), 9057c478bd9Sstevel@tonic-gate msg->msg_srv); 9067c478bd9Sstevel@tonic-gate break; 9077c478bd9Sstevel@tonic-gate case RE_DEAD_FILE: 9087c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 9097c478bd9Sstevel@tonic-gate "File %s (rnode_pt: %p) was closed due to NFS " 9107c478bd9Sstevel@tonic-gate "recovery error on server %s(%s %s)", msg->msg_srv, 9117c478bd9Sstevel@tonic-gate msg->msg_mntpt, ep->re_char1, (void *)ep->re_rp1, 9127c478bd9Sstevel@tonic-gate msg->msg_srv, ep->re_char2 ? ep->re_char2 : "", 9137c478bd9Sstevel@tonic-gate ep->re_stat4 ? nfs4_stat_to_str(ep->re_stat4) : ""); 9147c478bd9Sstevel@tonic-gate break; 9157c478bd9Sstevel@tonic-gate case RE_END: 9167c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 9177c478bd9Sstevel@tonic-gate "NFS Recovery done for mount %s (mi 0x%p) " 9187c478bd9Sstevel@tonic-gate "on server %s, rnode_pt1 %s (0x%p), " 9197c478bd9Sstevel@tonic-gate "rnode_pt2 %s (0x%p)", msg->msg_srv, msg->msg_mntpt, 9207c478bd9Sstevel@tonic-gate msg->msg_mntpt, (void *)ep->re_mi, msg->msg_srv, 9217c478bd9Sstevel@tonic-gate ep->re_char1, (void *)ep->re_rp1, ep->re_char2, 9227c478bd9Sstevel@tonic-gate (void *)ep->re_rp2); 9237c478bd9Sstevel@tonic-gate break; 9247c478bd9Sstevel@tonic-gate case RE_FAIL_RELOCK: 9257c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 9267c478bd9Sstevel@tonic-gate "Couldn't reclaim lock for pid %d for " 9277c478bd9Sstevel@tonic-gate "file %s (rnode_pt 0x%p) on (server %s): error %d", 9287c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt, ep->re_pid, ep->re_char1, 9297c478bd9Sstevel@tonic-gate (void *)ep->re_rp1, msg->msg_srv, 9307c478bd9Sstevel@tonic-gate ep->re_uint ? ep->re_uint : ep->re_stat4); 9317c478bd9Sstevel@tonic-gate break; 9327c478bd9Sstevel@tonic-gate case RE_FAIL_REMAP_LEN: 9337c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 9347c478bd9Sstevel@tonic-gate "remap_lookup: server %s returned bad " 9357c478bd9Sstevel@tonic-gate "fhandle length (%d)", msg->msg_srv, msg->msg_mntpt, 9367c478bd9Sstevel@tonic-gate msg->msg_srv, ep->re_uint); 9377c478bd9Sstevel@tonic-gate break; 9387c478bd9Sstevel@tonic-gate case RE_FAIL_REMAP_OP: 9397c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 9407c478bd9Sstevel@tonic-gate "remap_lookup: didn't get expected OP_GETFH" 9417c478bd9Sstevel@tonic-gate " for server %s", msg->msg_srv, msg->msg_mntpt, 9427c478bd9Sstevel@tonic-gate msg->msg_srv); 9437c478bd9Sstevel@tonic-gate break; 9447c478bd9Sstevel@tonic-gate case RE_FAILOVER: 9457c478bd9Sstevel@tonic-gate if (ep->re_char1) 9467c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, 9477c478bd9Sstevel@tonic-gate "![NFS4][Server: %s][Mntpt: %s]" 9487c478bd9Sstevel@tonic-gate "failing over from %s to %s", msg->msg_srv, 9497c478bd9Sstevel@tonic-gate msg->msg_mntpt, msg->msg_srv, ep->re_char1); 9507c478bd9Sstevel@tonic-gate else 9517c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, 9527c478bd9Sstevel@tonic-gate "![NFS4][Server: %s][Mntpt: %s]" 9537c478bd9Sstevel@tonic-gate "NFS4: failing over: selecting " 9547c478bd9Sstevel@tonic-gate "original server %s", msg->msg_srv, msg->msg_mntpt, 9557c478bd9Sstevel@tonic-gate msg->msg_srv); 9567c478bd9Sstevel@tonic-gate break; 9577c478bd9Sstevel@tonic-gate case RE_FILE_DIFF: 9587c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 9597c478bd9Sstevel@tonic-gate "File %s (rnode_pt: %p) on server %s was closed " 9607c478bd9Sstevel@tonic-gate "and failed attempted failover since its is different than " 9617c478bd9Sstevel@tonic-gate "the original file", msg->msg_srv, msg->msg_mntpt, 9627c478bd9Sstevel@tonic-gate ep->re_char1, (void *)ep->re_rp1, msg->msg_srv); 9637c478bd9Sstevel@tonic-gate break; 9647c478bd9Sstevel@tonic-gate case RE_LOST_STATE: 9657c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 9667c478bd9Sstevel@tonic-gate "Lost %s request for fs %s, file %s (rnode_pt: 0x%p), " 9677c478bd9Sstevel@tonic-gate "dir %s (0x%p) for server %s", msg->msg_srv, msg->msg_mntpt, 9687c478bd9Sstevel@tonic-gate nfs4_op_to_str(ep->re_uint), msg->msg_mntpt, 9697c478bd9Sstevel@tonic-gate ep->re_char1, (void *)ep->re_rp1, ep->re_char2, 9707c478bd9Sstevel@tonic-gate (void *)ep->re_rp2, msg->msg_srv); 9717c478bd9Sstevel@tonic-gate break; 9727c478bd9Sstevel@tonic-gate case RE_OPENS_CHANGED: 9737c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 9747c478bd9Sstevel@tonic-gate "The number of open files to reopen changed " 9757c478bd9Sstevel@tonic-gate "for mount %s mi 0x%p (old %d, new %d) on server %s", 9767c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt, msg->msg_mntpt, 9777c478bd9Sstevel@tonic-gate (void *)ep->re_mi, ep->re_uint, ep->re_pid, msg->msg_srv); 9787c478bd9Sstevel@tonic-gate break; 9797c478bd9Sstevel@tonic-gate case RE_SIGLOST: 9807c478bd9Sstevel@tonic-gate case RE_SIGLOST_NO_DUMP: 9817c478bd9Sstevel@tonic-gate if (ep->re_uint) 9827c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, 9837c478bd9Sstevel@tonic-gate "![NFS4][Server: %s][Mntpt: %s]" 9847c478bd9Sstevel@tonic-gate "Process %d lost its locks on " 9857c478bd9Sstevel@tonic-gate "file %s (rnode_pt: %p) due to NFS recovery error " 9867c478bd9Sstevel@tonic-gate "(%d) on server %s.", msg->msg_srv, msg->msg_mntpt, 9877c478bd9Sstevel@tonic-gate ep->re_pid, ep->re_char1, (void *)ep->re_rp1, 9887c478bd9Sstevel@tonic-gate ep->re_uint, msg->msg_srv); 9897c478bd9Sstevel@tonic-gate else 9907c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, 9917c478bd9Sstevel@tonic-gate "![NFS4][Server: %s][Mntpt: %s]" 9927c478bd9Sstevel@tonic-gate "Process %d lost its locks on " 9937c478bd9Sstevel@tonic-gate "file %s (rnode_pt: %p) due to NFS recovery error " 9947c478bd9Sstevel@tonic-gate "(%s) on server %s.", msg->msg_srv, msg->msg_mntpt, 9957c478bd9Sstevel@tonic-gate ep->re_pid, ep->re_char1, (void *)ep->re_rp1, 9967c478bd9Sstevel@tonic-gate nfs4_stat_to_str(ep->re_stat4), msg->msg_srv); 9977c478bd9Sstevel@tonic-gate break; 9987c478bd9Sstevel@tonic-gate case RE_START: 9997c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 10007c478bd9Sstevel@tonic-gate "NFS Starting recovery for mount %s " 10017c478bd9Sstevel@tonic-gate "(mi 0x%p mi_recovflags [0x%x]) on server %s, " 10027c478bd9Sstevel@tonic-gate "rnode_pt1 %s (0x%p), rnode_pt2 %s (0x%p)", msg->msg_srv, 10037c478bd9Sstevel@tonic-gate msg->msg_mntpt, msg->msg_mntpt, (void *)ep->re_mi, 10047c478bd9Sstevel@tonic-gate ep->re_uint, msg->msg_srv, ep->re_char1, (void *)ep->re_rp1, 10057c478bd9Sstevel@tonic-gate ep->re_char2, (void *)ep->re_rp2); 10067c478bd9Sstevel@tonic-gate break; 10077c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_ACTION: 10087c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 10097c478bd9Sstevel@tonic-gate "NFS recovery: unexpected action (%s) on server %s", 10107c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt, 10117c478bd9Sstevel@tonic-gate nfs4_recov_action_to_str(ep->re_uint), msg->msg_srv); 10127c478bd9Sstevel@tonic-gate break; 10137c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_ERRNO: 10147c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 10157c478bd9Sstevel@tonic-gate "NFS recovery: unexpected errno (%d) on server %s", 10167c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt, ep->re_uint, msg->msg_srv); 10177c478bd9Sstevel@tonic-gate break; 10187c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_STATUS: 10197c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 10207c478bd9Sstevel@tonic-gate "NFS recovery: unexpected NFS status code (%s) " 10217c478bd9Sstevel@tonic-gate "on server %s", msg->msg_srv, msg->msg_mntpt, 10227c478bd9Sstevel@tonic-gate nfs4_stat_to_str(ep->re_stat4), 10237c478bd9Sstevel@tonic-gate msg->msg_srv); 10247c478bd9Sstevel@tonic-gate break; 10257c478bd9Sstevel@tonic-gate case RE_WRONGSEC: 10267c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 10277c478bd9Sstevel@tonic-gate "NFS can't recover from NFS4ERR_WRONGSEC." 10287c478bd9Sstevel@tonic-gate " error %d for server %s: rnode_pt1 %s (0x%p)" 10297c478bd9Sstevel@tonic-gate " rnode_pt2 %s (0x%p)", msg->msg_srv, msg->msg_mntpt, 10307c478bd9Sstevel@tonic-gate ep->re_uint, msg->msg_srv, ep->re_char1, (void *)ep->re_rp1, 10317c478bd9Sstevel@tonic-gate ep->re_char2, (void *)ep->re_rp2); 10327c478bd9Sstevel@tonic-gate break; 10337c478bd9Sstevel@tonic-gate case RE_LOST_STATE_BAD_OP: 10347c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 10357c478bd9Sstevel@tonic-gate "NFS lost state with unrecognized op (%d)." 10367c478bd9Sstevel@tonic-gate " fs %s, server %s, pid %d, file %s (rnode_pt: 0x%p), " 10377c478bd9Sstevel@tonic-gate "dir %s (0x%p)", msg->msg_srv, msg->msg_mntpt, 10387c478bd9Sstevel@tonic-gate ep->re_uint, msg->msg_mntpt, msg->msg_srv, ep->re_pid, 10397c478bd9Sstevel@tonic-gate ep->re_char1, (void *)ep->re_rp1, ep->re_char2, 10407c478bd9Sstevel@tonic-gate (void *)ep->re_rp2); 10417c478bd9Sstevel@tonic-gate break; 10427c478bd9Sstevel@tonic-gate default: 10437c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_WARN, 10447c478bd9Sstevel@tonic-gate "!queue_print_event: illegal event %d", ep->re_type); 10457c478bd9Sstevel@tonic-gate break; 10467c478bd9Sstevel@tonic-gate } 10477c478bd9Sstevel@tonic-gate 10487c478bd9Sstevel@tonic-gate print_facts(msg, mi); 10497c478bd9Sstevel@tonic-gate 10507c478bd9Sstevel@tonic-gate /* 10517c478bd9Sstevel@tonic-gate * If set this event will not be printed again and is considered 10527c478bd9Sstevel@tonic-gate * dumped. 10537c478bd9Sstevel@tonic-gate */ 10547c478bd9Sstevel@tonic-gate if (dump) 10557c478bd9Sstevel@tonic-gate msg->msg_status = NFS4_MS_NO_DUMP; 10567c478bd9Sstevel@tonic-gate } 10577c478bd9Sstevel@tonic-gate 10587c478bd9Sstevel@tonic-gate /* 10597c478bd9Sstevel@tonic-gate * Print a fact message to /var/adm/messages 10607c478bd9Sstevel@tonic-gate */ 10617c478bd9Sstevel@tonic-gate static void 10627c478bd9Sstevel@tonic-gate queue_print_fact(nfs4_debug_msg_t *msg, int dump) 10637c478bd9Sstevel@tonic-gate { 10647c478bd9Sstevel@tonic-gate nfs4_rfact_t *fp; 10657c478bd9Sstevel@tonic-gate zoneid_t zoneid; 10667c478bd9Sstevel@tonic-gate 10677c478bd9Sstevel@tonic-gate fp = &msg->rmsg_u.msg_fact; 10687c478bd9Sstevel@tonic-gate zoneid = getzoneid(); 10697c478bd9Sstevel@tonic-gate 10707c478bd9Sstevel@tonic-gate switch (fp->rf_type) { 10717c478bd9Sstevel@tonic-gate case RF_BADOWNER: 10727c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 10737c478bd9Sstevel@tonic-gate "NFSMAPID_DOMAIN does not match the server: %s domain\n" 10747c478bd9Sstevel@tonic-gate "Please check configuration", msg->msg_srv, msg->msg_mntpt, 10757c478bd9Sstevel@tonic-gate msg->msg_srv); 10767c478bd9Sstevel@tonic-gate break; 10777c478bd9Sstevel@tonic-gate case RF_ERR: 10787c478bd9Sstevel@tonic-gate if (fp->rf_error) 10797c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, 10807c478bd9Sstevel@tonic-gate "![NFS4][Server: %s][Mntpt: %s]NFS op %s got " 10817c478bd9Sstevel@tonic-gate "error %d causing recovery action %s.%s", 10827c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt, 10837c478bd9Sstevel@tonic-gate nfs4_op_to_str(fp->rf_op), fp->rf_error, 10847c478bd9Sstevel@tonic-gate nfs4_recov_action_to_str(fp->rf_action), 10857c478bd9Sstevel@tonic-gate fp->rf_reboot ? 10867c478bd9Sstevel@tonic-gate " Client also suspects that the server rebooted," 10877c478bd9Sstevel@tonic-gate " or experienced a network partition." : ""); 10887c478bd9Sstevel@tonic-gate else 10897c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, 10907c478bd9Sstevel@tonic-gate "![NFS4][Server: %s][Mntpt: %s]NFS op %s got " 10917c478bd9Sstevel@tonic-gate "error %s causing recovery action %s.%s", 10927c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt, 10937c478bd9Sstevel@tonic-gate nfs4_op_to_str(fp->rf_op), 10947c478bd9Sstevel@tonic-gate nfs4_stat_to_str(fp->rf_stat4), 10957c478bd9Sstevel@tonic-gate nfs4_recov_action_to_str(fp->rf_action), 10967c478bd9Sstevel@tonic-gate fp->rf_reboot ? 10977c478bd9Sstevel@tonic-gate " Client also suspects that the server rebooted," 10987c478bd9Sstevel@tonic-gate " or experienced a network partition." : ""); 10997c478bd9Sstevel@tonic-gate break; 11007c478bd9Sstevel@tonic-gate case RF_RENEW_EXPIRED: 11017c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 11027c478bd9Sstevel@tonic-gate "NFS4 renew thread detected client's " 11037c478bd9Sstevel@tonic-gate "lease has expired. Current open files/locks/IO may fail", 11047c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt); 11057c478bd9Sstevel@tonic-gate break; 11067c478bd9Sstevel@tonic-gate case RF_SRV_NOT_RESPOND: 11077c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 11087c478bd9Sstevel@tonic-gate "NFS server %s not responding; still trying\n", 11097c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt, msg->msg_srv); 11107c478bd9Sstevel@tonic-gate break; 11117c478bd9Sstevel@tonic-gate case RF_SRV_OK: 11127c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 11137c478bd9Sstevel@tonic-gate "NFS server %s ok", msg->msg_srv, msg->msg_mntpt, 11147c478bd9Sstevel@tonic-gate msg->msg_srv); 11157c478bd9Sstevel@tonic-gate break; 11167c478bd9Sstevel@tonic-gate case RF_SRVS_NOT_RESPOND: 11177c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 11187c478bd9Sstevel@tonic-gate "NFS servers %s not responding; still trying", msg->msg_srv, 11197c478bd9Sstevel@tonic-gate msg->msg_mntpt, msg->msg_srv); 11207c478bd9Sstevel@tonic-gate break; 11217c478bd9Sstevel@tonic-gate case RF_SRVS_OK: 11227c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 11237c478bd9Sstevel@tonic-gate "NFS servers %s ok", msg->msg_srv, msg->msg_mntpt, 11247c478bd9Sstevel@tonic-gate msg->msg_srv); 11257c478bd9Sstevel@tonic-gate break; 11267c478bd9Sstevel@tonic-gate case RF_DELMAP_CB_ERR: 11277c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 11287c478bd9Sstevel@tonic-gate "NFS op %s got error %s when executing delmap on file %s " 11297c478bd9Sstevel@tonic-gate "(rnode_pt 0x%p).", 11307c478bd9Sstevel@tonic-gate msg->msg_srv, msg->msg_mntpt, nfs4_op_to_str(fp->rf_op), 11317c478bd9Sstevel@tonic-gate nfs4_stat_to_str(fp->rf_stat4), fp->rf_char1, 11327c478bd9Sstevel@tonic-gate (void *)fp->rf_rp1); 11337c478bd9Sstevel@tonic-gate break; 1134*e280ed37SDai Ngo case RF_SENDQ_FULL: 1135*e280ed37SDai Ngo zcmn_err(zoneid, CE_NOTE, "![NFS4][Server: %s][Mntpt: %s]" 1136*e280ed37SDai Ngo "send queue to NFS server %s is full; still trying\n", 1137*e280ed37SDai Ngo msg->msg_srv, msg->msg_mntpt, msg->msg_srv); 1138*e280ed37SDai Ngo break; 1139*e280ed37SDai Ngo 11407c478bd9Sstevel@tonic-gate default: 11417c478bd9Sstevel@tonic-gate zcmn_err(zoneid, CE_WARN, "!queue_print_fact: illegal fact %d", 11427c478bd9Sstevel@tonic-gate fp->rf_type); 11437c478bd9Sstevel@tonic-gate } 11447c478bd9Sstevel@tonic-gate 11457c478bd9Sstevel@tonic-gate /* 11467c478bd9Sstevel@tonic-gate * If set this fact will not be printed again and is considered 11477c478bd9Sstevel@tonic-gate * dumped. 11487c478bd9Sstevel@tonic-gate */ 11497c478bd9Sstevel@tonic-gate if (dump) 11507c478bd9Sstevel@tonic-gate msg->msg_status = NFS4_MS_NO_DUMP; 11517c478bd9Sstevel@tonic-gate } 11527c478bd9Sstevel@tonic-gate 11537c478bd9Sstevel@tonic-gate /* 11547c478bd9Sstevel@tonic-gate * Returns 1 if the entire queue should be dumped, 0 otherwise. 11557c478bd9Sstevel@tonic-gate */ 11567c478bd9Sstevel@tonic-gate static int 11577c478bd9Sstevel@tonic-gate id_to_dump_queue(nfs4_event_type_t id) 11587c478bd9Sstevel@tonic-gate { 11597c478bd9Sstevel@tonic-gate switch (id) { 11607c478bd9Sstevel@tonic-gate case RE_DEAD_FILE: 11617c478bd9Sstevel@tonic-gate case RE_SIGLOST: 11627c478bd9Sstevel@tonic-gate case RE_WRONGSEC: 11637c478bd9Sstevel@tonic-gate case RE_CLIENTID: 11647c478bd9Sstevel@tonic-gate return (1); 11657c478bd9Sstevel@tonic-gate default: 11667c478bd9Sstevel@tonic-gate return (0); 11677c478bd9Sstevel@tonic-gate } 11687c478bd9Sstevel@tonic-gate } 11697c478bd9Sstevel@tonic-gate 11707c478bd9Sstevel@tonic-gate /* 11717c478bd9Sstevel@tonic-gate * Returns 1 if the event (but not the entire queue) should be printed; 11727c478bd9Sstevel@tonic-gate * 0 otherwise. 11737c478bd9Sstevel@tonic-gate */ 11747c478bd9Sstevel@tonic-gate static int 11757c478bd9Sstevel@tonic-gate id_to_dump_solo_event(nfs4_event_type_t id) 11767c478bd9Sstevel@tonic-gate { 11777c478bd9Sstevel@tonic-gate switch (id) { 11787c478bd9Sstevel@tonic-gate case RE_BAD_SEQID: 11797c478bd9Sstevel@tonic-gate case RE_BADHANDLE: 11807c478bd9Sstevel@tonic-gate case RE_FAIL_REMAP_LEN: 11817c478bd9Sstevel@tonic-gate case RE_FAIL_REMAP_OP: 11827c478bd9Sstevel@tonic-gate case RE_FAILOVER: 11837c478bd9Sstevel@tonic-gate case RE_OPENS_CHANGED: 11847c478bd9Sstevel@tonic-gate case RE_SIGLOST_NO_DUMP: 11857c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_ACTION: 11867c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_ERRNO: 11877c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_STATUS: 11887c478bd9Sstevel@tonic-gate case RE_LOST_STATE_BAD_OP: 11897c478bd9Sstevel@tonic-gate return (1); 11907c478bd9Sstevel@tonic-gate default: 11917c478bd9Sstevel@tonic-gate return (0); 11927c478bd9Sstevel@tonic-gate } 11937c478bd9Sstevel@tonic-gate } 11947c478bd9Sstevel@tonic-gate 11957c478bd9Sstevel@tonic-gate /* 11967c478bd9Sstevel@tonic-gate * Returns 1 if the fact (but not the entire queue) should be printed; 11977c478bd9Sstevel@tonic-gate * 0 otherwise. 11987c478bd9Sstevel@tonic-gate */ 11997c478bd9Sstevel@tonic-gate static int 12007c478bd9Sstevel@tonic-gate id_to_dump_solo_fact(nfs4_fact_type_t id) 12017c478bd9Sstevel@tonic-gate { 12027c478bd9Sstevel@tonic-gate switch (id) { 12037c478bd9Sstevel@tonic-gate case RF_SRV_NOT_RESPOND: 12047c478bd9Sstevel@tonic-gate case RF_SRV_OK: 12057c478bd9Sstevel@tonic-gate case RF_SRVS_NOT_RESPOND: 12067c478bd9Sstevel@tonic-gate case RF_SRVS_OK: 1207*e280ed37SDai Ngo case RF_SENDQ_FULL: 12087c478bd9Sstevel@tonic-gate return (1); 12097c478bd9Sstevel@tonic-gate default: 12107c478bd9Sstevel@tonic-gate return (0); 12117c478bd9Sstevel@tonic-gate } 12127c478bd9Sstevel@tonic-gate } 12137c478bd9Sstevel@tonic-gate 12147c478bd9Sstevel@tonic-gate /* 12157c478bd9Sstevel@tonic-gate * Update a kernel stat 12167c478bd9Sstevel@tonic-gate */ 12177c478bd9Sstevel@tonic-gate static void 12187c478bd9Sstevel@tonic-gate update_recov_kstats(nfs4_debug_msg_t *msg, mntinfo4_t *mi) 12197c478bd9Sstevel@tonic-gate { 12207c478bd9Sstevel@tonic-gate rkstat_t *rsp; 12217c478bd9Sstevel@tonic-gate 12227c478bd9Sstevel@tonic-gate if (!mi->mi_recov_ksp) 12237c478bd9Sstevel@tonic-gate return; 12247c478bd9Sstevel@tonic-gate 12257c478bd9Sstevel@tonic-gate rsp = (rkstat_t *)mi->mi_recov_ksp->ks_data; 12267c478bd9Sstevel@tonic-gate 12277c478bd9Sstevel@tonic-gate if (msg->msg_type == RM_EVENT) { 12287c478bd9Sstevel@tonic-gate switch (msg->rmsg_u.msg_event.re_type) { 12297c478bd9Sstevel@tonic-gate case RE_BADHANDLE: 12307c478bd9Sstevel@tonic-gate rsp->badhandle.value.ul++; 12317c478bd9Sstevel@tonic-gate break; 12327c478bd9Sstevel@tonic-gate case RE_CLIENTID: 12337c478bd9Sstevel@tonic-gate rsp->clientid.value.ul++; 12347c478bd9Sstevel@tonic-gate break; 12357c478bd9Sstevel@tonic-gate case RE_DEAD_FILE: 12367c478bd9Sstevel@tonic-gate rsp->dead_file.value.ul++; 12377c478bd9Sstevel@tonic-gate break; 12387c478bd9Sstevel@tonic-gate case RE_FAIL_RELOCK: 12397c478bd9Sstevel@tonic-gate rsp->fail_relock.value.ul++; 12407c478bd9Sstevel@tonic-gate break; 12417c478bd9Sstevel@tonic-gate case RE_FILE_DIFF: 12427c478bd9Sstevel@tonic-gate rsp->file_diff.value.ul++; 12437c478bd9Sstevel@tonic-gate break; 12447c478bd9Sstevel@tonic-gate case RE_OPENS_CHANGED: 12457c478bd9Sstevel@tonic-gate rsp->opens_changed.value.ul++; 12467c478bd9Sstevel@tonic-gate break; 12477c478bd9Sstevel@tonic-gate case RE_SIGLOST: 12487c478bd9Sstevel@tonic-gate case RE_SIGLOST_NO_DUMP: 12497c478bd9Sstevel@tonic-gate rsp->siglost.value.ul++; 12507c478bd9Sstevel@tonic-gate break; 12517c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_ACTION: 12527c478bd9Sstevel@tonic-gate rsp->unexp_action.value.ul++; 12537c478bd9Sstevel@tonic-gate break; 12547c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_ERRNO: 12557c478bd9Sstevel@tonic-gate rsp->unexp_errno.value.ul++; 12567c478bd9Sstevel@tonic-gate break; 12577c478bd9Sstevel@tonic-gate case RE_UNEXPECTED_STATUS: 12587c478bd9Sstevel@tonic-gate rsp->unexp_status.value.ul++; 12597c478bd9Sstevel@tonic-gate break; 12607c478bd9Sstevel@tonic-gate case RE_WRONGSEC: 12617c478bd9Sstevel@tonic-gate rsp->wrongsec.value.ul++; 12627c478bd9Sstevel@tonic-gate break; 12637c478bd9Sstevel@tonic-gate case RE_LOST_STATE_BAD_OP: 12647c478bd9Sstevel@tonic-gate rsp->lost_state_bad_op.value.ul++; 12657c478bd9Sstevel@tonic-gate break; 12667c478bd9Sstevel@tonic-gate default: 12677c478bd9Sstevel@tonic-gate break; 12687c478bd9Sstevel@tonic-gate } 12697c478bd9Sstevel@tonic-gate } else if (msg->msg_type == RM_FACT) { 12707c478bd9Sstevel@tonic-gate switch (msg->rmsg_u.msg_fact.rf_type) { 12717c478bd9Sstevel@tonic-gate case RF_BADOWNER: 12727c478bd9Sstevel@tonic-gate rsp->badowner.value.ul++; 12737c478bd9Sstevel@tonic-gate break; 12747c478bd9Sstevel@tonic-gate case RF_SRV_NOT_RESPOND: 12757c478bd9Sstevel@tonic-gate rsp->not_responding.value.ul++; 12767c478bd9Sstevel@tonic-gate break; 12777c478bd9Sstevel@tonic-gate default: 12787c478bd9Sstevel@tonic-gate break; 12797c478bd9Sstevel@tonic-gate } 12807c478bd9Sstevel@tonic-gate } 12817c478bd9Sstevel@tonic-gate } 12827c478bd9Sstevel@tonic-gate 12837c478bd9Sstevel@tonic-gate /* 12847c478bd9Sstevel@tonic-gate * Dump the mi's mi_msg_list of recovery messages. 12857c478bd9Sstevel@tonic-gate */ 12867c478bd9Sstevel@tonic-gate static void 12877c478bd9Sstevel@tonic-gate dump_queue(mntinfo4_t *mi, nfs4_debug_msg_t *msg) 12887c478bd9Sstevel@tonic-gate { 12897c478bd9Sstevel@tonic-gate nfs4_debug_msg_t *tmp_msg; 12907c478bd9Sstevel@tonic-gate 12917c478bd9Sstevel@tonic-gate ASSERT(mutex_owned(&mi->mi_msg_list_lock)); 12927c478bd9Sstevel@tonic-gate 12937c478bd9Sstevel@tonic-gate /* update kstats */ 12947c478bd9Sstevel@tonic-gate update_recov_kstats(msg, mi); 12957c478bd9Sstevel@tonic-gate 12967c478bd9Sstevel@tonic-gate /* 12977c478bd9Sstevel@tonic-gate * If we aren't supposed to dump the queue then see if we 12987c478bd9Sstevel@tonic-gate * should just print this single message, then return. 12997c478bd9Sstevel@tonic-gate */ 13007c478bd9Sstevel@tonic-gate if (!id_to_dump_queue(msg->rmsg_u.msg_event.re_type)) { 13017c478bd9Sstevel@tonic-gate if (id_to_dump_solo_event(msg->rmsg_u.msg_event.re_type)) 13027c478bd9Sstevel@tonic-gate queue_print_event(msg, mi, 0); 13037c478bd9Sstevel@tonic-gate return; 13047c478bd9Sstevel@tonic-gate } 13057c478bd9Sstevel@tonic-gate 13067c478bd9Sstevel@tonic-gate /* 13077c478bd9Sstevel@tonic-gate * Write all events/facts in the queue that haven't been 13087c478bd9Sstevel@tonic-gate * previously written to disk. 13097c478bd9Sstevel@tonic-gate */ 13107c478bd9Sstevel@tonic-gate tmp_msg = list_head(&mi->mi_msg_list); 13117c478bd9Sstevel@tonic-gate while (tmp_msg) { 13127c478bd9Sstevel@tonic-gate if (tmp_msg->msg_status == NFS4_MS_DUMP) { 13137c478bd9Sstevel@tonic-gate if (tmp_msg->msg_type == RM_EVENT) 13147c478bd9Sstevel@tonic-gate queue_print_event(tmp_msg, mi, 1); 13157c478bd9Sstevel@tonic-gate else if (tmp_msg->msg_type == RM_FACT) 13167c478bd9Sstevel@tonic-gate queue_print_fact(tmp_msg, 1); 13177c478bd9Sstevel@tonic-gate } 13187c478bd9Sstevel@tonic-gate tmp_msg = list_next(&mi->mi_msg_list, tmp_msg); 13197c478bd9Sstevel@tonic-gate } 13207c478bd9Sstevel@tonic-gate } 13217c478bd9Sstevel@tonic-gate 13227c478bd9Sstevel@tonic-gate /* 13237c478bd9Sstevel@tonic-gate * Places the event into mi's debug recovery message queue. Some of the 13247c478bd9Sstevel@tonic-gate * fields can be overloaded to be a generic value, depending on the event 13257c478bd9Sstevel@tonic-gate * type. These include "count", "why". 13267c478bd9Sstevel@tonic-gate */ 13277c478bd9Sstevel@tonic-gate void 13287c478bd9Sstevel@tonic-gate nfs4_queue_event(nfs4_event_type_t id, mntinfo4_t *mi, char *server1, 13297c478bd9Sstevel@tonic-gate uint_t count, vnode_t *vp1, vnode_t *vp2, nfsstat4 nfs4_error, 13307c478bd9Sstevel@tonic-gate char *why, pid_t pid, nfs4_tag_type_t tag1, nfs4_tag_type_t tag2, 13317c478bd9Sstevel@tonic-gate seqid4 seqid1, seqid4 seqid2) 13327c478bd9Sstevel@tonic-gate { 13337c478bd9Sstevel@tonic-gate nfs4_debug_msg_t *msg; 13347c478bd9Sstevel@tonic-gate nfs4_revent_t *ep; 13357c478bd9Sstevel@tonic-gate char *cur_srv; 13367c478bd9Sstevel@tonic-gate rnode4_t *rp1 = NULL, *rp2 = NULL; 13377c478bd9Sstevel@tonic-gate refstr_t *mntpt; 13387c478bd9Sstevel@tonic-gate 13397c478bd9Sstevel@tonic-gate ASSERT(mi != NULL); 13407c478bd9Sstevel@tonic-gate if (vp1) 13417c478bd9Sstevel@tonic-gate rp1 = VTOR4(vp1); 13427c478bd9Sstevel@tonic-gate if (vp2) 13437c478bd9Sstevel@tonic-gate rp2 = VTOR4(vp2); 13447c478bd9Sstevel@tonic-gate 13457c478bd9Sstevel@tonic-gate /* 134663360950Smp * Initialize the message with the relevant server/mount_pt/time 13477c478bd9Sstevel@tonic-gate * information. Also place the relevent event related info. 13487c478bd9Sstevel@tonic-gate */ 13497c478bd9Sstevel@tonic-gate msg = kmem_zalloc(sizeof (*msg), KM_SLEEP); 13507c478bd9Sstevel@tonic-gate msg->msg_type = RM_EVENT; 13517c478bd9Sstevel@tonic-gate msg->msg_status = NFS4_MS_DUMP; 13527c478bd9Sstevel@tonic-gate ep = &msg->rmsg_u.msg_event; 13537c478bd9Sstevel@tonic-gate ep->re_type = id; 13547c478bd9Sstevel@tonic-gate gethrestime(&msg->msg_time); 13557c478bd9Sstevel@tonic-gate 13567c478bd9Sstevel@tonic-gate cur_srv = mi->mi_curr_serv->sv_hostname; 13577c478bd9Sstevel@tonic-gate msg->msg_srv = strdup(cur_srv); 13587c478bd9Sstevel@tonic-gate mntpt = vfs_getmntpoint(mi->mi_vfsp); 13597c478bd9Sstevel@tonic-gate msg->msg_mntpt = strdup(refstr_value(mntpt)); 13607c478bd9Sstevel@tonic-gate refstr_rele(mntpt); 13617c478bd9Sstevel@tonic-gate 13627c478bd9Sstevel@tonic-gate set_event(id, ep, mi, rp1, rp2, count, pid, nfs4_error, server1, 13637c478bd9Sstevel@tonic-gate why, tag1, tag2, seqid1, seqid2); 13647c478bd9Sstevel@tonic-gate 13657c478bd9Sstevel@tonic-gate mutex_enter(&mi->mi_msg_list_lock); 13667c478bd9Sstevel@tonic-gate 13677c478bd9Sstevel@tonic-gate /* if this event is the same as the last event, drop it */ 13687c478bd9Sstevel@tonic-gate if (events_same(list_tail(&mi->mi_msg_list), msg, mi)) { 13697c478bd9Sstevel@tonic-gate mutex_exit(&mi->mi_msg_list_lock); 13707c478bd9Sstevel@tonic-gate nfs4_free_msg(msg); 13717c478bd9Sstevel@tonic-gate return; 13727c478bd9Sstevel@tonic-gate } 13737c478bd9Sstevel@tonic-gate 13747c478bd9Sstevel@tonic-gate /* queue the message at the end of the list */ 13757c478bd9Sstevel@tonic-gate list_insert_tail(&mi->mi_msg_list, msg); 13767c478bd9Sstevel@tonic-gate 13777c478bd9Sstevel@tonic-gate dump_queue(mi, msg); 13787c478bd9Sstevel@tonic-gate 13797c478bd9Sstevel@tonic-gate if (mi->mi_msg_count == nfs4_msg_max) { 13807c478bd9Sstevel@tonic-gate nfs4_debug_msg_t *rm_msg; 13817c478bd9Sstevel@tonic-gate 13827c478bd9Sstevel@tonic-gate /* remove the queue'd message at the front of the list */ 13837c478bd9Sstevel@tonic-gate rm_msg = list_head(&mi->mi_msg_list); 13847c478bd9Sstevel@tonic-gate list_remove(&mi->mi_msg_list, rm_msg); 13857c478bd9Sstevel@tonic-gate mutex_exit(&mi->mi_msg_list_lock); 13867c478bd9Sstevel@tonic-gate nfs4_free_msg(rm_msg); 13877c478bd9Sstevel@tonic-gate } else { 13887c478bd9Sstevel@tonic-gate mi->mi_msg_count++; 13897c478bd9Sstevel@tonic-gate mutex_exit(&mi->mi_msg_list_lock); 13907c478bd9Sstevel@tonic-gate } 13917c478bd9Sstevel@tonic-gate } 13927c478bd9Sstevel@tonic-gate 13937c478bd9Sstevel@tonic-gate /* 13947c478bd9Sstevel@tonic-gate * Places the fact into mi's debug recovery messages queue. 13957c478bd9Sstevel@tonic-gate */ 13967c478bd9Sstevel@tonic-gate void 13977c478bd9Sstevel@tonic-gate nfs4_queue_fact(nfs4_fact_type_t fid, mntinfo4_t *mi, nfsstat4 stat4, 13987c478bd9Sstevel@tonic-gate nfs4_recov_t raction, nfs_opnum4 op, bool_t reboot, char *srvname, 13997c478bd9Sstevel@tonic-gate int error, vnode_t *vp) 14007c478bd9Sstevel@tonic-gate { 14017c478bd9Sstevel@tonic-gate nfs4_debug_msg_t *msg; 14027c478bd9Sstevel@tonic-gate nfs4_rfact_t *fp; 14037c478bd9Sstevel@tonic-gate char *cur_srv; 14047c478bd9Sstevel@tonic-gate refstr_t *mntpt; 14057c478bd9Sstevel@tonic-gate 14067c478bd9Sstevel@tonic-gate /* 140763360950Smp * Initialize the message with the relevant server/mount_pt/time 140863360950Smp * information. Also place the relevant fact related info. 14097c478bd9Sstevel@tonic-gate */ 14107c478bd9Sstevel@tonic-gate msg = kmem_zalloc(sizeof (*msg), KM_SLEEP); 14117c478bd9Sstevel@tonic-gate msg->msg_type = RM_FACT; 14127c478bd9Sstevel@tonic-gate msg->msg_status = NFS4_MS_DUMP; 14137c478bd9Sstevel@tonic-gate gethrestime(&msg->msg_time); 14147c478bd9Sstevel@tonic-gate 14157c478bd9Sstevel@tonic-gate if (srvname) 14167c478bd9Sstevel@tonic-gate cur_srv = srvname; 14177c478bd9Sstevel@tonic-gate else 14187c478bd9Sstevel@tonic-gate cur_srv = mi->mi_curr_serv->sv_hostname; 14197c478bd9Sstevel@tonic-gate 14207c478bd9Sstevel@tonic-gate msg->msg_srv = strdup(cur_srv); 14217c478bd9Sstevel@tonic-gate mntpt = vfs_getmntpoint(mi->mi_vfsp); 14227c478bd9Sstevel@tonic-gate msg->msg_mntpt = strdup(refstr_value(mntpt)); 14237c478bd9Sstevel@tonic-gate refstr_rele(mntpt); 14247c478bd9Sstevel@tonic-gate 14257c478bd9Sstevel@tonic-gate fp = &msg->rmsg_u.msg_fact; 14267c478bd9Sstevel@tonic-gate fp->rf_type = fid; 14277c478bd9Sstevel@tonic-gate fp->rf_status = RFS_NO_INSPECT; 14287c478bd9Sstevel@tonic-gate set_fact(fid, fp, stat4, raction, op, reboot, error, vp); 14297c478bd9Sstevel@tonic-gate 14307c478bd9Sstevel@tonic-gate update_recov_kstats(msg, mi); 14317c478bd9Sstevel@tonic-gate 14327c478bd9Sstevel@tonic-gate mutex_enter(&mi->mi_msg_list_lock); 14337c478bd9Sstevel@tonic-gate 14347c478bd9Sstevel@tonic-gate /* if this fact is the same as the last fact, drop it */ 14357c478bd9Sstevel@tonic-gate if (facts_same(list_tail(&mi->mi_msg_list), msg, mi)) { 14367c478bd9Sstevel@tonic-gate mutex_exit(&mi->mi_msg_list_lock); 14377c478bd9Sstevel@tonic-gate nfs4_free_msg(msg); 14387c478bd9Sstevel@tonic-gate return; 14397c478bd9Sstevel@tonic-gate } 14407c478bd9Sstevel@tonic-gate 14417c478bd9Sstevel@tonic-gate /* queue the message at the end of the list */ 14427c478bd9Sstevel@tonic-gate list_insert_tail(&mi->mi_msg_list, msg); 14437c478bd9Sstevel@tonic-gate 14447c478bd9Sstevel@tonic-gate if (id_to_dump_solo_fact(msg->rmsg_u.msg_fact.rf_type)) 14457c478bd9Sstevel@tonic-gate queue_print_fact(msg, 0); 14467c478bd9Sstevel@tonic-gate 14477c478bd9Sstevel@tonic-gate if (mi->mi_msg_count == nfs4_msg_max) { 14487c478bd9Sstevel@tonic-gate nfs4_debug_msg_t *rm_msg; 14497c478bd9Sstevel@tonic-gate 14507c478bd9Sstevel@tonic-gate /* remove the queue'd message at the front of the list */ 14517c478bd9Sstevel@tonic-gate rm_msg = list_head(&mi->mi_msg_list); 14527c478bd9Sstevel@tonic-gate list_remove(&mi->mi_msg_list, rm_msg); 14537c478bd9Sstevel@tonic-gate mutex_exit(&mi->mi_msg_list_lock); 14547c478bd9Sstevel@tonic-gate nfs4_free_msg(rm_msg); 14557c478bd9Sstevel@tonic-gate } else { 14567c478bd9Sstevel@tonic-gate mi->mi_msg_count++; 14577c478bd9Sstevel@tonic-gate mutex_exit(&mi->mi_msg_list_lock); 14587c478bd9Sstevel@tonic-gate } 14597c478bd9Sstevel@tonic-gate } 14607c478bd9Sstevel@tonic-gate 14617c478bd9Sstevel@tonic-gate /* 14627c478bd9Sstevel@tonic-gate * Initialize the 'mi_recov_kstat' kstat. 14637c478bd9Sstevel@tonic-gate */ 14647c478bd9Sstevel@tonic-gate void 14657c478bd9Sstevel@tonic-gate nfs4_mnt_recov_kstat_init(vfs_t *vfsp) 14667c478bd9Sstevel@tonic-gate { 14677c478bd9Sstevel@tonic-gate mntinfo4_t *mi = VFTOMI4(vfsp); 14687c478bd9Sstevel@tonic-gate kstat_t *ksp; 14697c478bd9Sstevel@tonic-gate zoneid_t zoneid = mi->mi_zone->zone_id; 14707c478bd9Sstevel@tonic-gate 14717c478bd9Sstevel@tonic-gate /* 14727c478bd9Sstevel@tonic-gate * Create the version specific kstats. 14737c478bd9Sstevel@tonic-gate * 14747c478bd9Sstevel@tonic-gate * PSARC 2001/697 Contract Private Interface 14757c478bd9Sstevel@tonic-gate * All nfs kstats are under SunMC contract 14767c478bd9Sstevel@tonic-gate * Please refer to the PSARC listed above and contact 14777c478bd9Sstevel@tonic-gate * SunMC before making any changes! 14787c478bd9Sstevel@tonic-gate * 14797c478bd9Sstevel@tonic-gate * Changes must be reviewed by Solaris File Sharing 14807c478bd9Sstevel@tonic-gate * Changes must be communicated to contract-2001-697@sun.com 14817c478bd9Sstevel@tonic-gate * 14827c478bd9Sstevel@tonic-gate */ 14837c478bd9Sstevel@tonic-gate 14847c478bd9Sstevel@tonic-gate if ((ksp = kstat_create_zone("nfs", getminor(vfsp->vfs_dev), 14857c478bd9Sstevel@tonic-gate "mi_recov_kstat", "misc", KSTAT_TYPE_NAMED, 14867c478bd9Sstevel@tonic-gate sizeof (rkstat_t) / sizeof (kstat_named_t), 14877c478bd9Sstevel@tonic-gate KSTAT_FLAG_WRITABLE, zoneid)) == NULL) { 14887c478bd9Sstevel@tonic-gate mi->mi_recov_ksp = NULL; 14897c478bd9Sstevel@tonic-gate zcmn_err(GLOBAL_ZONEID, CE_NOTE, 14907c478bd9Sstevel@tonic-gate "!mi_recov_kstat for mi %p failed\n", 14917c478bd9Sstevel@tonic-gate (void *)mi); 14927c478bd9Sstevel@tonic-gate return; 14937c478bd9Sstevel@tonic-gate } 14947c478bd9Sstevel@tonic-gate if (zoneid != GLOBAL_ZONEID) 14957c478bd9Sstevel@tonic-gate kstat_zone_add(ksp, GLOBAL_ZONEID); 14967c478bd9Sstevel@tonic-gate mi->mi_recov_ksp = ksp; 14977c478bd9Sstevel@tonic-gate bcopy(&rkstat_template, ksp->ks_data, sizeof (rkstat_t)); 14987c478bd9Sstevel@tonic-gate kstat_install(ksp); 14997c478bd9Sstevel@tonic-gate } 15007c478bd9Sstevel@tonic-gate 15017c478bd9Sstevel@tonic-gate /* 15027c478bd9Sstevel@tonic-gate * Increment the "delay" kstat. 15037c478bd9Sstevel@tonic-gate */ 15047c478bd9Sstevel@tonic-gate void 15057c478bd9Sstevel@tonic-gate nfs4_mi_kstat_inc_delay(mntinfo4_t *mi) 15067c478bd9Sstevel@tonic-gate { 15077c478bd9Sstevel@tonic-gate rkstat_t *rsp; 15087c478bd9Sstevel@tonic-gate 15097c478bd9Sstevel@tonic-gate if (!mi->mi_recov_ksp) 15107c478bd9Sstevel@tonic-gate return; 15117c478bd9Sstevel@tonic-gate 15127c478bd9Sstevel@tonic-gate rsp = (rkstat_t *)mi->mi_recov_ksp->ks_data; 15137c478bd9Sstevel@tonic-gate rsp->delay.value.ul++; 15147c478bd9Sstevel@tonic-gate } 15157c478bd9Sstevel@tonic-gate 15167c478bd9Sstevel@tonic-gate /* 15177c478bd9Sstevel@tonic-gate * Increment the "no_grace" kstat. 15187c478bd9Sstevel@tonic-gate */ 15197c478bd9Sstevel@tonic-gate void 15207c478bd9Sstevel@tonic-gate nfs4_mi_kstat_inc_no_grace(mntinfo4_t *mi) 15217c478bd9Sstevel@tonic-gate { 15227c478bd9Sstevel@tonic-gate rkstat_t *rsp; 15237c478bd9Sstevel@tonic-gate 15247c478bd9Sstevel@tonic-gate if (!mi->mi_recov_ksp) 15257c478bd9Sstevel@tonic-gate return; 15267c478bd9Sstevel@tonic-gate 15277c478bd9Sstevel@tonic-gate rsp = (rkstat_t *)mi->mi_recov_ksp->ks_data; 15287c478bd9Sstevel@tonic-gate rsp->no_grace.value.ul++; 15297c478bd9Sstevel@tonic-gate } 1530