xref: /illumos-gate/usr/src/uts/common/fs/ufs/ufs_panic.c (revision 1d48f750)
17c478bd9Sstevel@tonic-gate /*
27c478bd9Sstevel@tonic-gate  * CDDL HEADER START
37c478bd9Sstevel@tonic-gate  *
47c478bd9Sstevel@tonic-gate  * The contents of this file are subject to the terms of the
566c9f83dSowenr  * Common Development and Distribution License (the "License").
666c9f83dSowenr  * You may not use this file except in compliance with the License.
77c478bd9Sstevel@tonic-gate  *
87c478bd9Sstevel@tonic-gate  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
97c478bd9Sstevel@tonic-gate  * or http://www.opensolaris.org/os/licensing.
107c478bd9Sstevel@tonic-gate  * See the License for the specific language governing permissions
117c478bd9Sstevel@tonic-gate  * and limitations under the License.
127c478bd9Sstevel@tonic-gate  *
137c478bd9Sstevel@tonic-gate  * When distributing Covered Code, include this CDDL HEADER in each
147c478bd9Sstevel@tonic-gate  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
157c478bd9Sstevel@tonic-gate  * If applicable, add the following below this CDDL HEADER, with the
167c478bd9Sstevel@tonic-gate  * fields enclosed by brackets "[]" replaced with your own identifying
177c478bd9Sstevel@tonic-gate  * information: Portions Copyright [yyyy] [name of copyright owner]
187c478bd9Sstevel@tonic-gate  *
197c478bd9Sstevel@tonic-gate  * CDDL HEADER END
207c478bd9Sstevel@tonic-gate  */
217c478bd9Sstevel@tonic-gate /*
22d3d50737SRafael Vanoni  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
237c478bd9Sstevel@tonic-gate  * Use is subject to license terms.
247c478bd9Sstevel@tonic-gate  */
257c478bd9Sstevel@tonic-gate 
267c478bd9Sstevel@tonic-gate #include <sys/types.h>
277c478bd9Sstevel@tonic-gate #include <sys/param.h>
287c478bd9Sstevel@tonic-gate #include <sys/systm.h>
297c478bd9Sstevel@tonic-gate #include <sys/errno.h>
307c478bd9Sstevel@tonic-gate #include <sys/mode.h>
317c478bd9Sstevel@tonic-gate #include <sys/sysmacros.h>
327c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h>
337c478bd9Sstevel@tonic-gate #include <sys/varargs.h>
347c478bd9Sstevel@tonic-gate #include <sys/time.h>
357c478bd9Sstevel@tonic-gate #include <sys/buf.h>
367c478bd9Sstevel@tonic-gate #include <sys/kmem.h>
377c478bd9Sstevel@tonic-gate #include <sys/t_lock.h>
387c478bd9Sstevel@tonic-gate #include <sys/poll.h>
397c478bd9Sstevel@tonic-gate #include <sys/debug.h>
407c478bd9Sstevel@tonic-gate #include <sys/cred.h>
417c478bd9Sstevel@tonic-gate #include <sys/lockfs.h>
427c478bd9Sstevel@tonic-gate #include <sys/fs/ufs_fs.h>
437c478bd9Sstevel@tonic-gate #include <sys/fs/ufs_inode.h>
447c478bd9Sstevel@tonic-gate #include <sys/fs/ufs_panic.h>
457c478bd9Sstevel@tonic-gate #include <sys/fs/ufs_lockfs.h>
467c478bd9Sstevel@tonic-gate #include <sys/fs/ufs_trans.h>
477c478bd9Sstevel@tonic-gate #include <sys/fs/ufs_mount.h>
487c478bd9Sstevel@tonic-gate #include <sys/fs/ufs_prot.h>
497c478bd9Sstevel@tonic-gate #include <sys/fs/ufs_bio.h>
507c478bd9Sstevel@tonic-gate #include <sys/pathname.h>
517c478bd9Sstevel@tonic-gate #include <sys/utsname.h>
527c478bd9Sstevel@tonic-gate #include <sys/conf.h>
537c478bd9Sstevel@tonic-gate 
547c478bd9Sstevel@tonic-gate /* handy */
557c478bd9Sstevel@tonic-gate #define	abs(x)		((x) < 0? -(x): (x))
567c478bd9Sstevel@tonic-gate 
577c478bd9Sstevel@tonic-gate #if defined(DEBUG)
587c478bd9Sstevel@tonic-gate 
597c478bd9Sstevel@tonic-gate #define	DBGLVL_NONE	0x00000000
607c478bd9Sstevel@tonic-gate #define	DBGLVL_MAJOR	0x00000100
617c478bd9Sstevel@tonic-gate #define	DBGLVL_MINOR	0x00000200
627c478bd9Sstevel@tonic-gate #define	DBGLVL_MINUTE	0x00000400
637c478bd9Sstevel@tonic-gate #define	DBGLVL_TRIVIA	0x00000800
647c478bd9Sstevel@tonic-gate #define	DBGLVL_HIDEOUS	0x00001000
657c478bd9Sstevel@tonic-gate 
667c478bd9Sstevel@tonic-gate #define	DBGFLG_NONE		0x00000000
677c478bd9Sstevel@tonic-gate #define	DBGFLG_NOPANIC		0x00000001
687c478bd9Sstevel@tonic-gate #define	DBGFLG_LVLONLY		0x00000002
697c478bd9Sstevel@tonic-gate #define	DBGFLG_FIXWOULDPANIC	0x00000004
707c478bd9Sstevel@tonic-gate 
717c478bd9Sstevel@tonic-gate #define	DBGFLG_FLAGMASK		0x0000000F
727c478bd9Sstevel@tonic-gate #define	DBGFLG_LEVELMASK	~DBGFLG_FLAGMASK
737c478bd9Sstevel@tonic-gate 
747c478bd9Sstevel@tonic-gate #define	DEBUG_FLAGS	(ufs_fix_failure_dbg & DBGFLG_FLAGMASK)
757c478bd9Sstevel@tonic-gate #define	DEBUG_LEVEL	(ufs_fix_failure_dbg & DBGFLG_LEVELMASK)
767c478bd9Sstevel@tonic-gate 
777c478bd9Sstevel@tonic-gate unsigned int ufs_fix_failure_dbg =	DBGLVL_NONE | DBGFLG_NONE;
787c478bd9Sstevel@tonic-gate 
797c478bd9Sstevel@tonic-gate #define	DCALL(dbg_level, call)						\
807c478bd9Sstevel@tonic-gate 	{								\
817c478bd9Sstevel@tonic-gate 		if (DEBUG_LEVEL != DBGLVL_NONE) {			\
827c478bd9Sstevel@tonic-gate 			if (DEBUG_FLAGS & DBGFLG_LVLONLY) {		\
837c478bd9Sstevel@tonic-gate 				if (DEBUG_LEVEL & dbg_level) {		\
847c478bd9Sstevel@tonic-gate 					call;				\
857c478bd9Sstevel@tonic-gate 				}					\
867c478bd9Sstevel@tonic-gate 			} else {					\
877c478bd9Sstevel@tonic-gate 				if (dbg_level <= DEBUG_LEVEL) {		\
887c478bd9Sstevel@tonic-gate 					call;				\
897c478bd9Sstevel@tonic-gate 				}					\
907c478bd9Sstevel@tonic-gate 			}						\
917c478bd9Sstevel@tonic-gate 		}							\
927c478bd9Sstevel@tonic-gate 	}
937c478bd9Sstevel@tonic-gate 
947c478bd9Sstevel@tonic-gate #define	DPRINTF(dbg_level, msg)		DCALL(dbg_level, printf msg)
957c478bd9Sstevel@tonic-gate 
967c478bd9Sstevel@tonic-gate #define	MAJOR(msg)			DPRINTF(DBGLVL_MAJOR, msg)
977c478bd9Sstevel@tonic-gate #define	MINOR(msg)			DPRINTF(DBGLVL_MINOR, msg)
987c478bd9Sstevel@tonic-gate #define	MINUTE(msg)			DPRINTF(DBGLVL_MINUTE, msg)
997c478bd9Sstevel@tonic-gate #define	TRIVIA(msg)			DPRINTF(DBGLVL_TRIVIA, msg)
1007c478bd9Sstevel@tonic-gate #define	HIDEOUS(msg)			DPRINTF(DBGLVL_HIDEOUS, msg)
1017c478bd9Sstevel@tonic-gate 
1027c478bd9Sstevel@tonic-gate #else	/* !DEBUG */
1037c478bd9Sstevel@tonic-gate 
1047c478bd9Sstevel@tonic-gate #define	DCALL(ignored_dbg_level, ignored_routine)
1057c478bd9Sstevel@tonic-gate #define	MAJOR(ignored)
1067c478bd9Sstevel@tonic-gate #define	MINOR(ignored)
1077c478bd9Sstevel@tonic-gate #define	MINUTE(ignored)
1087c478bd9Sstevel@tonic-gate #define	TRIVIA(ignored)
1097c478bd9Sstevel@tonic-gate #define	HIDEOUS(ignored)
1107c478bd9Sstevel@tonic-gate 
1117c478bd9Sstevel@tonic-gate #endif /* DEBUG */
1127c478bd9Sstevel@tonic-gate 
1137c478bd9Sstevel@tonic-gate #define	NULLSTR(str)	(!(str) || *(str) == '\0'? "<null>" : (str))
1147c478bd9Sstevel@tonic-gate #define	NULSTRING	""
1157c478bd9Sstevel@tonic-gate 
1167c478bd9Sstevel@tonic-gate /* somewhat arbitrary limits, in seconds */
1177c478bd9Sstevel@tonic-gate /* all probably ought to be different, but these are convenient for debugging */
1187c478bd9Sstevel@tonic-gate const time_t	UF_TOO_LONG		= 128;	/* max. wait for fsck start */
1197c478bd9Sstevel@tonic-gate 
1207c478bd9Sstevel@tonic-gate /* all of these are in units of seconds used for retry period while ... */
1217c478bd9Sstevel@tonic-gate const time_t	UF_FIXSTART_PERIOD	= 16;	/* awaiting fsck start */
1227c478bd9Sstevel@tonic-gate const time_t	UF_FIXPOLL_PERIOD	= 256;	/* awaiting fsck finish */
1237c478bd9Sstevel@tonic-gate const time_t	UF_SHORT_ERROR_PERIOD	= 4;	/* after (lockfs) error */
1247c478bd9Sstevel@tonic-gate const time_t	UF_LONG_ERROR_PERIOD	= 512;	/* after (lockfs) error */
1257c478bd9Sstevel@tonic-gate 
1267c478bd9Sstevel@tonic-gate #define	NO_ERROR		0
1277c478bd9Sstevel@tonic-gate #define	LOCKFS_OLOCK		LOCKFS_MAXLOCK+1
1287c478bd9Sstevel@tonic-gate 
1297c478bd9Sstevel@tonic-gate const ulong_t	GB			= 1024 * 1024 * 1024;
1307c478bd9Sstevel@tonic-gate const ulong_t	SecondsPerGig		= 1024;	/* ~17 minutes (overestimate) */
1317c478bd9Sstevel@tonic-gate 
1327c478bd9Sstevel@tonic-gate /*
1337c478bd9Sstevel@tonic-gate  * per filesystem flags
1347c478bd9Sstevel@tonic-gate  */
1357c478bd9Sstevel@tonic-gate const int	UFSFX_PANIC		= (UFSMNT_ONERROR_PANIC >> 4);
1367c478bd9Sstevel@tonic-gate const int	UFSFX_LCKONLY		= (UFSMNT_ONERROR_LOCK >> 4);
1377c478bd9Sstevel@tonic-gate const int	UFSFX_LCKUMOUNT		= (UFSMNT_ONERROR_UMOUNT >> 4);
1387c478bd9Sstevel@tonic-gate const int	UFSFX_DEFAULT		= (UFSMNT_ONERROR_DEFAULT >> 4);
1397c478bd9Sstevel@tonic-gate const int	UFSFX_REPAIR_START	= 0x10000000;
1407c478bd9Sstevel@tonic-gate 
1417c478bd9Sstevel@tonic-gate /* return protocols */
1427c478bd9Sstevel@tonic-gate 
1437c478bd9Sstevel@tonic-gate typedef enum triage_return_code {
1447c478bd9Sstevel@tonic-gate 	TRIAGE_DEAD = -1,
1457c478bd9Sstevel@tonic-gate 	TRIAGE_NO_SPIRIT,
1467c478bd9Sstevel@tonic-gate 	TRIAGE_ATTEND_TO
1477c478bd9Sstevel@tonic-gate } triage_t;
1487c478bd9Sstevel@tonic-gate 
1497c478bd9Sstevel@tonic-gate typedef enum statefunc_return_code {
1507c478bd9Sstevel@tonic-gate 	SFRC_SUCCESS = 1,
1517c478bd9Sstevel@tonic-gate 	SFRC_FAIL = 0
1527c478bd9Sstevel@tonic-gate } sfrc_t;
1537c478bd9Sstevel@tonic-gate 
1547c478bd9Sstevel@tonic-gate /* external references */
1557c478bd9Sstevel@tonic-gate /* in ufs_thread.c */
1567c478bd9Sstevel@tonic-gate extern int	ufs_thread_run(struct ufs_q *, callb_cpr_t *cprinfop);
1577c478bd9Sstevel@tonic-gate extern int	ufs_checkaccton(vnode_t *);		/* in ufs_lockfs.c */
1587c478bd9Sstevel@tonic-gate extern int	ufs_checkswapon(vnode_t *);		/* in ufs_lockfs.c */
1597c478bd9Sstevel@tonic-gate 
1607c478bd9Sstevel@tonic-gate extern struct pollhead		ufs_pollhd;		/* in ufs_vnops.c */
1617c478bd9Sstevel@tonic-gate 
1627c478bd9Sstevel@tonic-gate /* globals */
1637c478bd9Sstevel@tonic-gate struct	ufs_q	 ufs_fix;
1647c478bd9Sstevel@tonic-gate 
1657c478bd9Sstevel@tonic-gate /*
1667c478bd9Sstevel@tonic-gate  * patchable constants:
1677c478bd9Sstevel@tonic-gate  *   These are set in ufsfx_init() [called at modload]
1687c478bd9Sstevel@tonic-gate  */
1697c478bd9Sstevel@tonic-gate struct ufs_failure_tunable {
1707c478bd9Sstevel@tonic-gate 	long	 uft_too_long;		/* limit repair startup time */
1717c478bd9Sstevel@tonic-gate 	long	 uft_fixstart_period;	/* pre-repair start period */
1727c478bd9Sstevel@tonic-gate 	long	 uft_fixpoll_period;	/* post-fsck start period */
1737c478bd9Sstevel@tonic-gate 	long	 uft_short_err_period;	/* post-error short period */
1747c478bd9Sstevel@tonic-gate 	long	 uft_long_err_period;	/* post-error long period */
1757c478bd9Sstevel@tonic-gate } ufsfx_tune;
1767c478bd9Sstevel@tonic-gate 
1777c478bd9Sstevel@tonic-gate /* internal statistics of events */
1787c478bd9Sstevel@tonic-gate struct uf_statistics {
1797c478bd9Sstevel@tonic-gate 	ulong_t		ufst_lock_violations;
1807c478bd9Sstevel@tonic-gate 	ulong_t		ufst_current_races;
1817c478bd9Sstevel@tonic-gate 	ulong_t		ufst_unmount_failures;
1827c478bd9Sstevel@tonic-gate 	ulong_t		ufst_num_fixed;
1837c478bd9Sstevel@tonic-gate 	ulong_t		ufst_num_failed;
1847c478bd9Sstevel@tonic-gate 	ulong_t		ufst_cpu_waste;
1857c478bd9Sstevel@tonic-gate 	time_t		ufst_last_start_tm;
1867c478bd9Sstevel@tonic-gate 	kmutex_t	ufst_mutex;
1877c478bd9Sstevel@tonic-gate } uf_stats;
1887c478bd9Sstevel@tonic-gate 
1897c478bd9Sstevel@tonic-gate typedef enum state_action {
1907c478bd9Sstevel@tonic-gate 	UFA_ERROR = -1,		/* internal error */
1917c478bd9Sstevel@tonic-gate 	UFA_FOUND,		/* found uf in state */
1927c478bd9Sstevel@tonic-gate 	UFA_SET			/* change uf to state */
1937c478bd9Sstevel@tonic-gate } ufsa_t;
1947c478bd9Sstevel@tonic-gate 
1957c478bd9Sstevel@tonic-gate /* state definition */
1967c478bd9Sstevel@tonic-gate typedef struct uf_state_desc {
1977c478bd9Sstevel@tonic-gate 	int	  ud_v;					/* value */
1987c478bd9Sstevel@tonic-gate 	char	 *ud_name;				/* name */
1997c478bd9Sstevel@tonic-gate 	sfrc_t	(*ud_sfp)(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
2007c478bd9Sstevel@tonic-gate 							/* per-state actions */
2017c478bd9Sstevel@tonic-gate 	ufs_failure_states_t	  ud_prev;		/* valid prev. states */
2027c478bd9Sstevel@tonic-gate 
2037c478bd9Sstevel@tonic-gate 	struct uf_state_desc_attr {
2047c478bd9Sstevel@tonic-gate 		unsigned	terminal:1;	/* no action req. if found */
2057c478bd9Sstevel@tonic-gate 		unsigned	at_fail:1;	/* state set by thread */
2067c478bd9Sstevel@tonic-gate 						/* encountering the error */
2077c478bd9Sstevel@tonic-gate 		unsigned	unused;
2087c478bd9Sstevel@tonic-gate 	} ud_attr;
2097c478bd9Sstevel@tonic-gate } ufsd_t;
2107c478bd9Sstevel@tonic-gate 
2117c478bd9Sstevel@tonic-gate /*
2127c478bd9Sstevel@tonic-gate  * forward references
2137c478bd9Sstevel@tonic-gate  */
2147c478bd9Sstevel@tonic-gate 
2157c478bd9Sstevel@tonic-gate /* thread to watch for failures */
2167c478bd9Sstevel@tonic-gate static void	ufsfx_thread_fix_failures(void *);
217645a7badSToomas Soome static int	ufsfx_do_failure_q(void);
2187c478bd9Sstevel@tonic-gate static void	ufsfx_kill_fix_failure_thread(void *);
2197c478bd9Sstevel@tonic-gate 
2207c478bd9Sstevel@tonic-gate /* routines called when failure occurs */
2217c478bd9Sstevel@tonic-gate static int		 ufs_fault_v(vnode_t *, char *, va_list)
2227c478bd9Sstevel@tonic-gate 	__KVPRINTFLIKE(2);
2237c478bd9Sstevel@tonic-gate static ufs_failure_t	*init_failure(vnode_t *, char *, va_list)
2247c478bd9Sstevel@tonic-gate 	__KVPRINTFLIKE(2);
2257c478bd9Sstevel@tonic-gate static void		 queue_failure(ufs_failure_t *);
2267c478bd9Sstevel@tonic-gate /*PRINTFLIKE2*/
2277c478bd9Sstevel@tonic-gate static void		 real_panic(ufs_failure_t *, const char *, ...)
2287c478bd9Sstevel@tonic-gate 	__KPRINTFLIKE(2);
2297c478bd9Sstevel@tonic-gate static void		 real_panic_v(ufs_failure_t *, const char *, va_list)
2307c478bd9Sstevel@tonic-gate 	__KVPRINTFLIKE(2);
2317c478bd9Sstevel@tonic-gate static triage_t		 triage(vnode_t *);
2327c478bd9Sstevel@tonic-gate 
2337c478bd9Sstevel@tonic-gate /* routines called when failure record is acted upon */
2347c478bd9Sstevel@tonic-gate static sfrc_t	set_state(ufs_failure_t *, ufs_failure_states_t);
2357c478bd9Sstevel@tonic-gate static int	state_trans_valid(ufs_failure_states_t, ufs_failure_states_t);
2367c478bd9Sstevel@tonic-gate static int	terminal_state(ufs_failure_states_t);
2377c478bd9Sstevel@tonic-gate 
2387c478bd9Sstevel@tonic-gate /* routines called when states entered/found */
2397c478bd9Sstevel@tonic-gate static sfrc_t	sf_minimum(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
2407c478bd9Sstevel@tonic-gate static sfrc_t	sf_undef(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
2417c478bd9Sstevel@tonic-gate static sfrc_t	sf_init(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
2427c478bd9Sstevel@tonic-gate static sfrc_t	sf_queue(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
2437c478bd9Sstevel@tonic-gate static sfrc_t	sf_found_queue(ufs_failure_t *);
2447c478bd9Sstevel@tonic-gate static sfrc_t	sf_nonterm_cmn(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
2457c478bd9Sstevel@tonic-gate static sfrc_t	sf_term_cmn(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
2467c478bd9Sstevel@tonic-gate static sfrc_t	sf_panic(ufs_failure_t *, ufsa_t, ufs_failure_states_t);
2477c478bd9Sstevel@tonic-gate static sfrc_t	sf_set_trylck(ufs_failure_t *);
2487c478bd9Sstevel@tonic-gate static sfrc_t	sf_set_locked(ufs_failure_t *);
2497c478bd9Sstevel@tonic-gate static sfrc_t	sf_found_trylck(ufs_failure_t *);
2507c478bd9Sstevel@tonic-gate static sfrc_t	sf_found_lock_fix_cmn(ufs_failure_t *, ufs_failure_states_t);
2517c478bd9Sstevel@tonic-gate static sfrc_t	sf_found_umount(ufs_failure_t *);
2527c478bd9Sstevel@tonic-gate 
2537c478bd9Sstevel@tonic-gate /* support routines, called by sf_nonterm_cmn and sf_term_cmn */
254645a7badSToomas Soome static time_t	trylock_time_exceeded(ufs_failure_t *);
255645a7badSToomas Soome static void	pester_msg(ufs_failure_t *, int);
256645a7badSToomas Soome static int	get_lockfs_status(ufs_failure_t *, struct lockfs *);
257645a7badSToomas Soome static void	alloc_lockfs_comment(ufs_failure_t *, struct lockfs *);
258645a7badSToomas Soome static int	set_lockfs(ufs_failure_t *, struct lockfs *);
259645a7badSToomas Soome static int	lockfs_failure(ufs_failure_t *);
260645a7badSToomas Soome static int	lockfs_success(ufs_failure_t *);
2617c478bd9Sstevel@tonic-gate static int	fsck_active(ufs_failure_t *);
2627c478bd9Sstevel@tonic-gate 
2637c478bd9Sstevel@tonic-gate /* low-level support routines */
2647c478bd9Sstevel@tonic-gate static ufsd_t	*get_state_desc(ufs_failure_states_t);
2657c478bd9Sstevel@tonic-gate static char	*fs_name(ufs_failure_t *);
2667c478bd9Sstevel@tonic-gate 
2677c478bd9Sstevel@tonic-gate #if defined(DEBUG)
2687c478bd9Sstevel@tonic-gate static char	*state_name(ufs_failure_states_t);
2697c478bd9Sstevel@tonic-gate static char	*lock_name(struct lockfs *);
2707c478bd9Sstevel@tonic-gate static char	*err_name(int);
2717c478bd9Sstevel@tonic-gate static char	*act_name(ufsa_t);
2727c478bd9Sstevel@tonic-gate static void	 dump_uf_list(char *msg);
2737c478bd9Sstevel@tonic-gate static void	 dump_uf(ufs_failure_t *, int i);
2747c478bd9Sstevel@tonic-gate #endif /* DEBUG */
2757c478bd9Sstevel@tonic-gate /*
2767c478bd9Sstevel@tonic-gate  *
2777c478bd9Sstevel@tonic-gate  * State Transitions:
2787c478bd9Sstevel@tonic-gate  *
2797c478bd9Sstevel@tonic-gate  * normally:
2807c478bd9Sstevel@tonic-gate  * if flagged to be locked but not unmounted:	(UFSMNT_ONERROR_LOCK)
2817c478bd9Sstevel@tonic-gate  *	UNDEF -> INIT -> QUEUE -> TRYLCK -> LOCKED -> FIXING -> FIXED
2827c478bd9Sstevel@tonic-gate  *
2837c478bd9Sstevel@tonic-gate  * The only difference between these two is that the fsck must be started
2847c478bd9Sstevel@tonic-gate  * manually.
2857c478bd9Sstevel@tonic-gate  *
2867c478bd9Sstevel@tonic-gate  * if flagged to be unmounted:			(UFSMNT_ONERROR_UMOUNT)
2877c478bd9Sstevel@tonic-gate  *	UNDEF -> INIT -> QUEUE -> TRYLCK -> LOCKED -> UMOUNT -> NOTFIX
2887c478bd9Sstevel@tonic-gate  *
2897c478bd9Sstevel@tonic-gate  * if flagged to panic:				(UFSMNT_ONERROR_PANIC)
2907c478bd9Sstevel@tonic-gate  *	UNDEF -> INIT -> PANIC
2917c478bd9Sstevel@tonic-gate  *
2927c478bd9Sstevel@tonic-gate  * if a secondary panic on a file system which has an active failure
2937c478bd9Sstevel@tonic-gate  * record:
2947c478bd9Sstevel@tonic-gate  *	UNDEF -> INIT -> QUEUE -> REPLICA
2957c478bd9Sstevel@tonic-gate  *
2967c478bd9Sstevel@tonic-gate  * UNDEF, INIT, QUEUE all are set in the context of the failing thread.
2977c478bd9Sstevel@tonic-gate  * All other states (except possibly PANIC) are set in by the monitor
2987c478bd9Sstevel@tonic-gate  * (lock) thread.
2997c478bd9Sstevel@tonic-gate  *
3007c478bd9Sstevel@tonic-gate  */
3017c478bd9Sstevel@tonic-gate 
3027c478bd9Sstevel@tonic-gate ufsd_t	state_desc[] =
3037c478bd9Sstevel@tonic-gate {
3047c478bd9Sstevel@tonic-gate 	{ UF_ILLEGAL,	"in an unknown state",	sf_minimum,	UF_ILLEGAL,
3057c478bd9Sstevel@tonic-gate 								{ 0, 1, 0 } },
3067c478bd9Sstevel@tonic-gate 	{ UF_UNDEF,	"undefined",		sf_undef,	UF_UNDEF,
3077c478bd9Sstevel@tonic-gate 								{ 0, 1, 0 } },
3087c478bd9Sstevel@tonic-gate 	{ UF_INIT,	"being initialized",	sf_init,	UF_UNDEF,
3097c478bd9Sstevel@tonic-gate 								{ 0, 1, 0 } },
3107c478bd9Sstevel@tonic-gate 	{ UF_QUEUE,	"queued",		sf_queue,	UF_INIT,
3117c478bd9Sstevel@tonic-gate 								{ 0, 1, 0 } },
3127c478bd9Sstevel@tonic-gate 	{ UF_TRYLCK,	"trying to be locked",	sf_nonterm_cmn,
3137c478bd9Sstevel@tonic-gate 						UF_QUEUE,	{ 0, 0, 0 } },
3147c478bd9Sstevel@tonic-gate 	{ UF_LOCKED,	"locked",		sf_nonterm_cmn,
3157c478bd9Sstevel@tonic-gate 					UF_TRYLCK | UF_FIXING,	{ 0, 0, 0 } },
3167c478bd9Sstevel@tonic-gate 	{ UF_UMOUNT,	"being unmounted",	sf_nonterm_cmn,
3177c478bd9Sstevel@tonic-gate 
3187c478bd9Sstevel@tonic-gate #if defined(DEBUG)
3197c478bd9Sstevel@tonic-gate 					UF_PANIC |
3207c478bd9Sstevel@tonic-gate #endif /* DEBUG */
321645a7badSToomas Soome 					UF_TRYLCK | UF_LOCKED,	{ 0, 0, 0 } },
3227c478bd9Sstevel@tonic-gate 	{ UF_FIXING,	"being fixed",		sf_nonterm_cmn,
3237c478bd9Sstevel@tonic-gate 						UF_LOCKED,	{ 0, 0, 0 } },
3247c478bd9Sstevel@tonic-gate 	{ UF_FIXED,	"fixed",		sf_term_cmn,
3257c478bd9Sstevel@tonic-gate 						UF_FIXING,	{ 1, 0, 0 } },
3267c478bd9Sstevel@tonic-gate 	{ UF_NOTFIX,	"not fixed",		sf_term_cmn,
3277c478bd9Sstevel@tonic-gate 
3287c478bd9Sstevel@tonic-gate #if defined(DEBUG)
3297c478bd9Sstevel@tonic-gate 							UF_PANIC |
3307c478bd9Sstevel@tonic-gate #endif /* DEBUG */
3317c478bd9Sstevel@tonic-gate 
3327c478bd9Sstevel@tonic-gate 	    UF_QUEUE | UF_TRYLCK | UF_LOCKED | UF_UMOUNT | UF_FIXING,
3337c478bd9Sstevel@tonic-gate 								{ 1, 0, 0 } },
3347c478bd9Sstevel@tonic-gate 	{ UF_REPLICA,	"a replica",		sf_term_cmn,
3357c478bd9Sstevel@tonic-gate 						UF_QUEUE,	{ 1, 0, 0 } },
3367c478bd9Sstevel@tonic-gate 	{ UF_PANIC,	"panicking",		sf_panic,
3377c478bd9Sstevel@tonic-gate 		/* XXX make this narrower */	UF_ALLSTATES,	{ 0, 0, 0 } },
3387c478bd9Sstevel@tonic-gate 	{ UF_UNDEF,	NULL,			((sfrc_t (*)()) NULL),
339645a7badSToomas Soome 						UF_UNDEF,	{ 0, 0, 0 } }
3407c478bd9Sstevel@tonic-gate };
3417c478bd9Sstevel@tonic-gate 
3427c478bd9Sstevel@tonic-gate /* unified collection */
3437c478bd9Sstevel@tonic-gate struct ufsfx_info {
3447c478bd9Sstevel@tonic-gate 	struct uf_statistics		*ufi_statp;
3457c478bd9Sstevel@tonic-gate 	struct ufs_failure_tunable	*ufi_tunep;
3467c478bd9Sstevel@tonic-gate 	ufsd_t				*ufi_statetab;
3477c478bd9Sstevel@tonic-gate } uffsinfo;
3487c478bd9Sstevel@tonic-gate 
3497c478bd9Sstevel@tonic-gate #if defined(DEBUG)
3507c478bd9Sstevel@tonic-gate struct action_description {
3517c478bd9Sstevel@tonic-gate 	ufsa_t	 ad_v;
3527c478bd9Sstevel@tonic-gate 	char	*ad_name;
3537c478bd9Sstevel@tonic-gate };
3547c478bd9Sstevel@tonic-gate 
3557c478bd9Sstevel@tonic-gate #define	EUNK		(-1)
3567c478bd9Sstevel@tonic-gate 
3577c478bd9Sstevel@tonic-gate struct error_description {
3587c478bd9Sstevel@tonic-gate 	int	 ed_errno;
3597c478bd9Sstevel@tonic-gate 	char	*ed_name;
3607c478bd9Sstevel@tonic-gate } err_desc[] =
3617c478bd9Sstevel@tonic-gate {
3627c478bd9Sstevel@tonic-gate 	{ EUNK,		"<unexpected errno?>"	},
3637c478bd9Sstevel@tonic-gate 	{ EINVAL,	"EINVAL"		},
3647c478bd9Sstevel@tonic-gate 	{ EACCES,	"EACCES"		},
3657c478bd9Sstevel@tonic-gate 	{ EPERM,	"EPERM"			},
3667c478bd9Sstevel@tonic-gate 	{ EIO,		"EIO"			},
3677c478bd9Sstevel@tonic-gate 	{ EDEADLK,	"EDEADLK"		},
3687c478bd9Sstevel@tonic-gate 	{ EBUSY,	"EBUSY"			},
3697c478bd9Sstevel@tonic-gate 	{ EAGAIN,	"EAGAIN"		},
3707c478bd9Sstevel@tonic-gate 	{ ERESTART,	"ERESTART"		},
3717c478bd9Sstevel@tonic-gate 	{ ETIMEDOUT,	"ETIMEDOUT"		},
3727c478bd9Sstevel@tonic-gate 	{ NO_ERROR,	"Ok"			},
373645a7badSToomas Soome 	{ EUNK,		NULL			}
3747c478bd9Sstevel@tonic-gate };
3757c478bd9Sstevel@tonic-gate 
3767c478bd9Sstevel@tonic-gate struct action_description act_desc[] =
3777c478bd9Sstevel@tonic-gate {
3787c478bd9Sstevel@tonic-gate 	{ UFA_ERROR,	"<unexpected action?>"	},
3797c478bd9Sstevel@tonic-gate 	{ UFA_FOUND,	"\"found\""	},
3807c478bd9Sstevel@tonic-gate 	{ UFA_SET,	"\"set\""	},
3817c478bd9Sstevel@tonic-gate 	{ UFA_ERROR,	NULL			},
3827c478bd9Sstevel@tonic-gate };
3837c478bd9Sstevel@tonic-gate 
3847c478bd9Sstevel@tonic-gate #define	LOCKFS_BADLOCK	(-1)
3857c478bd9Sstevel@tonic-gate 
3867c478bd9Sstevel@tonic-gate struct lock_description {
3877c478bd9Sstevel@tonic-gate 	int	 ld_type;
3887c478bd9Sstevel@tonic-gate 	char	*ld_name;
3897c478bd9Sstevel@tonic-gate } lock_desc[] =
3907c478bd9Sstevel@tonic-gate {
3917c478bd9Sstevel@tonic-gate 	{ LOCKFS_BADLOCK,	"<unexpected lock?>"	},
3927c478bd9Sstevel@tonic-gate 	{ LOCKFS_ULOCK,		"Unlock"		},
3937c478bd9Sstevel@tonic-gate 	{ LOCKFS_ELOCK,		"Error Lock"		},
3947c478bd9Sstevel@tonic-gate 	{ LOCKFS_HLOCK,		"Hard Lock"		},
3957c478bd9Sstevel@tonic-gate 	{ LOCKFS_OLOCK,		"Old Lock"		},
3967c478bd9Sstevel@tonic-gate 	{ LOCKFS_BADLOCK,	NULL			}
3977c478bd9Sstevel@tonic-gate };
3987c478bd9Sstevel@tonic-gate 
3997c478bd9Sstevel@tonic-gate #endif /* DEBUG */
4007c478bd9Sstevel@tonic-gate 
4017c478bd9Sstevel@tonic-gate /*
4027c478bd9Sstevel@tonic-gate  * ufs_fault, ufs_fault_v
4037c478bd9Sstevel@tonic-gate  *
4047c478bd9Sstevel@tonic-gate  *  called instead of cmn_err(CE_PANIC, ...) by ufs routines
4057c478bd9Sstevel@tonic-gate  *  when a failure is detected to put the file system into an
4067c478bd9Sstevel@tonic-gate  *  error state (if possible) or to devolve to a panic otherwise
4077c478bd9Sstevel@tonic-gate  *
4087c478bd9Sstevel@tonic-gate  * vnode is some vnode in this file system, used to find the way
4097c478bd9Sstevel@tonic-gate  * to ufsvfs, vfsp etc.  Since a panic can be called from many
4107c478bd9Sstevel@tonic-gate  * levels, the vnode is the most convenient hook to pass through.
4117c478bd9Sstevel@tonic-gate  *
4127c478bd9Sstevel@tonic-gate  */
4137c478bd9Sstevel@tonic-gate 
4147c478bd9Sstevel@tonic-gate /*PRINTFLIKE2*/
4157c478bd9Sstevel@tonic-gate int
ufs_fault(vnode_t * vp,char * fmt,...)4167c478bd9Sstevel@tonic-gate ufs_fault(vnode_t *vp, char *fmt, ...)
4177c478bd9Sstevel@tonic-gate {
4187c478bd9Sstevel@tonic-gate 	va_list	adx;
4197c478bd9Sstevel@tonic-gate 	int	error;
4207c478bd9Sstevel@tonic-gate 
4217c478bd9Sstevel@tonic-gate 	MINOR(("[ufs_fault"));
4227c478bd9Sstevel@tonic-gate 
4237c478bd9Sstevel@tonic-gate 	va_start(adx, fmt);
4247c478bd9Sstevel@tonic-gate 	error = ufs_fault_v(vp, fmt, adx);
4257c478bd9Sstevel@tonic-gate 	va_end(adx);
4267c478bd9Sstevel@tonic-gate 
4277c478bd9Sstevel@tonic-gate 	MINOR((": %s (%d)]\n", err_name(error), error));
4287c478bd9Sstevel@tonic-gate 	return (error);
4297c478bd9Sstevel@tonic-gate }
4307c478bd9Sstevel@tonic-gate 
4317c478bd9Sstevel@tonic-gate const char *nullfmt = "<null format?>";
4327c478bd9Sstevel@tonic-gate 
4337c478bd9Sstevel@tonic-gate static int
ufs_fault_v(vnode_t * vp,char * fmt,va_list adx)4347c478bd9Sstevel@tonic-gate ufs_fault_v(vnode_t *vp, char *fmt, va_list adx)
4357c478bd9Sstevel@tonic-gate {
4367c478bd9Sstevel@tonic-gate 	ufs_failure_t		*new = NULL;
4377c478bd9Sstevel@tonic-gate 	ufsvfs_t		*ufsvfsp;
4387c478bd9Sstevel@tonic-gate 	triage_t		 fix;
4397c478bd9Sstevel@tonic-gate 	int			 err = ERESTART;
4407c478bd9Sstevel@tonic-gate 	int			need_vfslock;
4417c478bd9Sstevel@tonic-gate 
4427c478bd9Sstevel@tonic-gate 	MINOR(("[ufs_fault_v"));
4437c478bd9Sstevel@tonic-gate 
4447c478bd9Sstevel@tonic-gate 	if (fmt == NULL)
4457c478bd9Sstevel@tonic-gate 		fmt = (char *)nullfmt;
4467c478bd9Sstevel@tonic-gate 
4477c478bd9Sstevel@tonic-gate 	fix = triage(vp);
4487c478bd9Sstevel@tonic-gate 
4497c478bd9Sstevel@tonic-gate 	if (vp) {
4507c478bd9Sstevel@tonic-gate 		ufsvfsp = (struct ufsvfs *)vp->v_vfsp->vfs_data;
4517c478bd9Sstevel@tonic-gate 
4527c478bd9Sstevel@tonic-gate 		/*
4537c478bd9Sstevel@tonic-gate 		 * Something bad has happened. That is why we are here.
4547c478bd9Sstevel@tonic-gate 		 *
4557c478bd9Sstevel@tonic-gate 		 * In order for the bad thing to be recorded in the superblock
4567c478bd9Sstevel@tonic-gate 		 * we need to write to the superblock directly.
4577c478bd9Sstevel@tonic-gate 		 * In the case that logging is enabled the logging code
4587c478bd9Sstevel@tonic-gate 		 * would normally intercept our write as a delta to the log,
4597c478bd9Sstevel@tonic-gate 		 * thus we mark the filesystem FSBAD in any case.
4607c478bd9Sstevel@tonic-gate 		 */
4617c478bd9Sstevel@tonic-gate 		need_vfslock = !MUTEX_HELD(&ufsvfsp->vfs_lock);
4627c478bd9Sstevel@tonic-gate 
4637c478bd9Sstevel@tonic-gate 		if (need_vfslock) {
4647c478bd9Sstevel@tonic-gate 			mutex_enter(&ufsvfsp->vfs_lock);
4657c478bd9Sstevel@tonic-gate 		}
4667c478bd9Sstevel@tonic-gate 
4677c478bd9Sstevel@tonic-gate 		ufsvfsp->vfs_fs->fs_clean = FSBAD;
4687c478bd9Sstevel@tonic-gate 		ASSERT(SEMA_HELD(&ufsvfsp->vfs_bufp->b_sem));
46980d34432Sfrankho 		ufsvfsp->vfs_bufp->b_flags &=
47080d34432Sfrankho 		    ~(B_ASYNC | B_READ | B_DONE | B_ERROR | B_DELWRI);
4717c478bd9Sstevel@tonic-gate 
4727c478bd9Sstevel@tonic-gate 		(void) bdev_strategy(ufsvfsp->vfs_bufp);
4737c478bd9Sstevel@tonic-gate 		(void) biowait(ufsvfsp->vfs_bufp);
4747c478bd9Sstevel@tonic-gate 
4757c478bd9Sstevel@tonic-gate 		if (need_vfslock) {
4767c478bd9Sstevel@tonic-gate 			mutex_exit(&ufsvfsp->vfs_lock);
4777c478bd9Sstevel@tonic-gate 		}
4787c478bd9Sstevel@tonic-gate 	}
4797c478bd9Sstevel@tonic-gate 
4807c478bd9Sstevel@tonic-gate 	switch (fix) {
4817c478bd9Sstevel@tonic-gate 
4827c478bd9Sstevel@tonic-gate 	default:
4837c478bd9Sstevel@tonic-gate 	case TRIAGE_DEAD:
4847c478bd9Sstevel@tonic-gate 	case TRIAGE_NO_SPIRIT:
4857c478bd9Sstevel@tonic-gate 
4867c478bd9Sstevel@tonic-gate 		real_panic_v(new, fmt, adx);
4877c478bd9Sstevel@tonic-gate 		/* LINTED: warning: logical expression always true: op "||" */
4887c478bd9Sstevel@tonic-gate 		ASSERT(DEBUG);
4897c478bd9Sstevel@tonic-gate 		err = EAGAIN;
4907c478bd9Sstevel@tonic-gate 
4917c478bd9Sstevel@tonic-gate #if defined(DEBUG)
4927c478bd9Sstevel@tonic-gate 		if (!(DEBUG_FLAGS & DBGFLG_FIXWOULDPANIC)) {
4937c478bd9Sstevel@tonic-gate 			break;
4947c478bd9Sstevel@tonic-gate 		}
4957c478bd9Sstevel@tonic-gate #else
4967c478bd9Sstevel@tonic-gate 		break;
4977c478bd9Sstevel@tonic-gate 
4987c478bd9Sstevel@tonic-gate #endif /* DEBUG */
499645a7badSToomas Soome 		/* FALLTHROUGH */
5007c478bd9Sstevel@tonic-gate 
5017c478bd9Sstevel@tonic-gate 	case TRIAGE_ATTEND_TO:
5027c478bd9Sstevel@tonic-gate 
5037c478bd9Sstevel@tonic-gate 		/* q thread not running yet? */
50466c9f83dSowenr 		if (mutex_tryenter(&ufs_fix.uq_mutex)) {
50566c9f83dSowenr 			if (!ufs_fix.uq_threadp) {
50666c9f83dSowenr 				mutex_exit(&ufs_fix.uq_mutex);
50766c9f83dSowenr 				ufs_thread_start(&ufs_fix,
50866c9f83dSowenr 				    ufsfx_thread_fix_failures, NULL);
50966c9f83dSowenr 				ufs_fix.uq_threadp->t_flag |= T_DONTBLOCK;
51066c9f83dSowenr 				mutex_enter(&ufs_fix.uq_mutex);
51166c9f83dSowenr 			} else {
51266c9f83dSowenr 				/*
51366c9f83dSowenr 				 * We got the lock but we are not the current
51466c9f83dSowenr 				 * threadp so we have to release the lock.
51566c9f83dSowenr 				 */
51666c9f83dSowenr 				mutex_exit(&ufs_fix.uq_mutex);
51766c9f83dSowenr 			}
5187c478bd9Sstevel@tonic-gate 		} else {
5197c478bd9Sstevel@tonic-gate 			MINOR((": fix failure thread already running "));
52066c9f83dSowenr 			/*
52166c9f83dSowenr 			 * No need to log another failure as one is already
52266c9f83dSowenr 			 * being logged.
52366c9f83dSowenr 			 */
52466c9f83dSowenr 			break;
5257c478bd9Sstevel@tonic-gate 		}
5267c478bd9Sstevel@tonic-gate 
5277c478bd9Sstevel@tonic-gate 		if (ufs_fix.uq_threadp && ufs_fix.uq_threadp == curthread) {
5287c478bd9Sstevel@tonic-gate 			mutex_exit(&ufs_fix.uq_mutex);
5297c478bd9Sstevel@tonic-gate 			cmn_err(CE_WARN, "ufs_fault_v: recursive ufs_fault");
5307c478bd9Sstevel@tonic-gate 		} else {
53166c9f83dSowenr 			/*
53266c9f83dSowenr 			 * Must check if we actually still own the lock and
53366c9f83dSowenr 			 * if so then release the lock and move on with life.
53466c9f83dSowenr 			 */
53566c9f83dSowenr 			if (mutex_owner(&ufs_fix.uq_mutex) == curthread)
53666c9f83dSowenr 				mutex_exit(&ufs_fix.uq_mutex);
5377c478bd9Sstevel@tonic-gate 		}
5387c478bd9Sstevel@tonic-gate 
5397c478bd9Sstevel@tonic-gate 		new = init_failure(vp, fmt, adx);
5407c478bd9Sstevel@tonic-gate 		if (new != NULL) {
5417c478bd9Sstevel@tonic-gate 			queue_failure(new);
5427c478bd9Sstevel@tonic-gate 			break;
5437c478bd9Sstevel@tonic-gate 		}
5447c478bd9Sstevel@tonic-gate 		real_panic_v(new, fmt, adx);
5457c478bd9Sstevel@tonic-gate 		break;
5467c478bd9Sstevel@tonic-gate 
5477c478bd9Sstevel@tonic-gate 	}
5487c478bd9Sstevel@tonic-gate 	MINOR(("] "));
5497c478bd9Sstevel@tonic-gate 	return (err);
5507c478bd9Sstevel@tonic-gate }
5517c478bd9Sstevel@tonic-gate 
5527c478bd9Sstevel@tonic-gate /*
5537c478bd9Sstevel@tonic-gate  * triage()
5547c478bd9Sstevel@tonic-gate  *
5557c478bd9Sstevel@tonic-gate  *  Attempt to fix iff:
5567c478bd9Sstevel@tonic-gate  *    - the system is not already panicking
5577c478bd9Sstevel@tonic-gate  *    - this file system isn't explicitly marked not to be fixed
5587c478bd9Sstevel@tonic-gate  *    - we can connect to the user-level daemon
5597c478bd9Sstevel@tonic-gate  * These conditions are detectable later, but if we can determine
5607c478bd9Sstevel@tonic-gate  * them in the failing threads context the core dump may be more
5617c478bd9Sstevel@tonic-gate  * useful.
5627c478bd9Sstevel@tonic-gate  *
5637c478bd9Sstevel@tonic-gate  */
5647c478bd9Sstevel@tonic-gate 
5657c478bd9Sstevel@tonic-gate static triage_t
triage(vnode_t * vp)5667c478bd9Sstevel@tonic-gate triage(vnode_t *vp)
5677c478bd9Sstevel@tonic-gate {
5687c478bd9Sstevel@tonic-gate 	struct inode	 *ip;
5697c478bd9Sstevel@tonic-gate 	int		  need_unlock_vfs;
5707c478bd9Sstevel@tonic-gate 	int		  fs_flags;
5717c478bd9Sstevel@tonic-gate 
5727c478bd9Sstevel@tonic-gate 	MINUTE(("[triage"));
5737c478bd9Sstevel@tonic-gate 
5747c478bd9Sstevel@tonic-gate 	if (panicstr) {
5757c478bd9Sstevel@tonic-gate 		MINUTE((
5767c478bd9Sstevel@tonic-gate 		": already panicking: \"%s\" => TRIAGE_DEAD]\n", panicstr));
5777c478bd9Sstevel@tonic-gate 		return (TRIAGE_DEAD);
5787c478bd9Sstevel@tonic-gate 	}
5797c478bd9Sstevel@tonic-gate 
5807c478bd9Sstevel@tonic-gate 	if (!vp || !(ip = VTOI(vp)) || !ip->i_ufsvfs) {
5817c478bd9Sstevel@tonic-gate 		MINUTE((
5827c478bd9Sstevel@tonic-gate 	": vp, ip or ufsvfs is NULL; can't determine fs => TRIAGE_DEAD]\n"));
5837c478bd9Sstevel@tonic-gate 		return (TRIAGE_DEAD);
5847c478bd9Sstevel@tonic-gate 	}
5857c478bd9Sstevel@tonic-gate 
5867c478bd9Sstevel@tonic-gate 	/* use tryenter and continue no matter what since we're panicky */
5877c478bd9Sstevel@tonic-gate 	need_unlock_vfs = !MUTEX_HELD(&ip->i_ufsvfs->vfs_lock);
5887c478bd9Sstevel@tonic-gate 	if (need_unlock_vfs)
5897c478bd9Sstevel@tonic-gate 		need_unlock_vfs = mutex_tryenter(&ip->i_ufsvfs->vfs_lock);
5907c478bd9Sstevel@tonic-gate 
5917c478bd9Sstevel@tonic-gate 	fs_flags = ip->i_ufsvfs->vfs_fsfx.fx_flags;
5927c478bd9Sstevel@tonic-gate 	if (need_unlock_vfs)
5937c478bd9Sstevel@tonic-gate 		mutex_exit(&ip->i_ufsvfs->vfs_lock);
5947c478bd9Sstevel@tonic-gate 
5957c478bd9Sstevel@tonic-gate 	if (fs_flags & UFSFX_PANIC) {
5967c478bd9Sstevel@tonic-gate 		MINUTE((
5977c478bd9Sstevel@tonic-gate 		": filesystem marked \"panic\" => TRIAGE_NO_SPIRIT]\n"));
5987c478bd9Sstevel@tonic-gate 		return (TRIAGE_NO_SPIRIT);
5997c478bd9Sstevel@tonic-gate 	}
6007c478bd9Sstevel@tonic-gate 
6017c478bd9Sstevel@tonic-gate 	if (ufs_checkaccton(vp) != 0) {
6027c478bd9Sstevel@tonic-gate 		MINUTE((
6037c478bd9Sstevel@tonic-gate 		": filesystem would deadlock (accounting) => TRIAGE_DEAD]\n"));
6047c478bd9Sstevel@tonic-gate 		return (TRIAGE_DEAD);
6057c478bd9Sstevel@tonic-gate 	}
6067c478bd9Sstevel@tonic-gate 
6077c478bd9Sstevel@tonic-gate 	if (ufs_checkswapon(vp) != 0) {
6087c478bd9Sstevel@tonic-gate 		MINUTE((
6097c478bd9Sstevel@tonic-gate 		": filesystem would deadlock (swapping) => TRIAGE_DEAD]\n"));
6107c478bd9Sstevel@tonic-gate 		return (TRIAGE_DEAD);
6117c478bd9Sstevel@tonic-gate 	}
6127c478bd9Sstevel@tonic-gate 
6137c478bd9Sstevel@tonic-gate 	MINUTE((": return TRIAGE_ATTEND_TO] "));
6147c478bd9Sstevel@tonic-gate 	return (TRIAGE_ATTEND_TO);
6157c478bd9Sstevel@tonic-gate }
6167c478bd9Sstevel@tonic-gate 
6177c478bd9Sstevel@tonic-gate /*
6187c478bd9Sstevel@tonic-gate  * init failure
6197c478bd9Sstevel@tonic-gate  *
6207c478bd9Sstevel@tonic-gate  * This routine allocates a failure struct and initializes
6217c478bd9Sstevel@tonic-gate  * it's member elements.
6227c478bd9Sstevel@tonic-gate  * Space is allocated for copies of dynamic identifying fs structures
6237c478bd9Sstevel@tonic-gate  * passed in.  Without a much more segmented kernel architecture
6247c478bd9Sstevel@tonic-gate  * this is as protected as we can make it (for now.)
6257c478bd9Sstevel@tonic-gate  */
6267c478bd9Sstevel@tonic-gate static ufs_failure_t *
init_failure(vnode_t * vp,char * fmt,va_list adx)6277c478bd9Sstevel@tonic-gate init_failure(vnode_t *vp, char *fmt, va_list adx)
6287c478bd9Sstevel@tonic-gate {
6297c478bd9Sstevel@tonic-gate 	ufs_failure_t	*new;
6307c478bd9Sstevel@tonic-gate 	struct inode	*ip;
6317c478bd9Sstevel@tonic-gate 	int		 initialization_worked = 0;
6327c478bd9Sstevel@tonic-gate 	int		 need_vfs_unlock;
6337c478bd9Sstevel@tonic-gate 
6347c478bd9Sstevel@tonic-gate 	MINOR(("[init_failure"));
6357c478bd9Sstevel@tonic-gate 
6367c478bd9Sstevel@tonic-gate 	new = kmem_zalloc(sizeof (ufs_failure_t), KM_NOSLEEP);
6377c478bd9Sstevel@tonic-gate 	if (!new) {
6387c478bd9Sstevel@tonic-gate 		MINOR((": kmem_zalloc failed]\n"));
6397c478bd9Sstevel@tonic-gate 		return (NULL);
6407c478bd9Sstevel@tonic-gate 	}
6417c478bd9Sstevel@tonic-gate 
6427c478bd9Sstevel@tonic-gate 	/*
6437c478bd9Sstevel@tonic-gate 	 * enough information to make a fix attempt possible?
6447c478bd9Sstevel@tonic-gate 	 */
6457c478bd9Sstevel@tonic-gate 	if (!vp || !(ip = VTOI(vp)) || !ip->i_ufsvfs || !vp->v_vfsp ||
6467c478bd9Sstevel@tonic-gate 	    !ip->i_ufsvfs->vfs_bufp || !ITOF(ip) || !fmt)
6477c478bd9Sstevel@tonic-gate 		goto errout;
6487c478bd9Sstevel@tonic-gate 
6497c478bd9Sstevel@tonic-gate 	if (vp->v_type != VREG && vp->v_type != VDIR &&
6507c478bd9Sstevel@tonic-gate 	    vp->v_type != VBLK && vp->v_type != VCHR &&
6517c478bd9Sstevel@tonic-gate 	    vp->v_type != VLNK && vp->v_type != VFIFO &&
6527c478bd9Sstevel@tonic-gate 	    vp->v_type != VSOCK)
6537c478bd9Sstevel@tonic-gate 		goto errout;
6547c478bd9Sstevel@tonic-gate 
6557c478bd9Sstevel@tonic-gate 	if (ip->i_ufsvfs->vfs_root->v_type != VREG &&
6567c478bd9Sstevel@tonic-gate 	    ip->i_ufsvfs->vfs_root->v_type != VDIR &&
6577c478bd9Sstevel@tonic-gate 	    ip->i_ufsvfs->vfs_root->v_type != VBLK &&
6587c478bd9Sstevel@tonic-gate 	    ip->i_ufsvfs->vfs_root->v_type != VCHR &&
6597c478bd9Sstevel@tonic-gate 	    ip->i_ufsvfs->vfs_root->v_type != VLNK &&
6607c478bd9Sstevel@tonic-gate 	    ip->i_ufsvfs->vfs_root->v_type != VFIFO &&
6617c478bd9Sstevel@tonic-gate 	    ip->i_ufsvfs->vfs_root->v_type != VSOCK)
6627c478bd9Sstevel@tonic-gate 		goto errout;
6637c478bd9Sstevel@tonic-gate 
6647c478bd9Sstevel@tonic-gate 	if ((ITOF(ip)->fs_magic != FS_MAGIC) &&
6657c478bd9Sstevel@tonic-gate 	    (ITOF(ip)->fs_magic != MTB_UFS_MAGIC))
6667c478bd9Sstevel@tonic-gate 		goto errout;
6677c478bd9Sstevel@tonic-gate 
6687c478bd9Sstevel@tonic-gate 	/* intialize values */
6697c478bd9Sstevel@tonic-gate 
6707c478bd9Sstevel@tonic-gate 	(void) vsnprintf(new->uf_panic_str, LOCKFS_MAXCOMMENTLEN - 1, fmt, adx);
6717c478bd9Sstevel@tonic-gate 
6727c478bd9Sstevel@tonic-gate 	new->uf_ufsvfsp = ip->i_ufsvfs;
6737c478bd9Sstevel@tonic-gate 	new->uf_vfsp    = ip->i_vfs;
6747c478bd9Sstevel@tonic-gate 
6757c478bd9Sstevel@tonic-gate 	mutex_init(&new->uf_mutex, NULL, MUTEX_DEFAULT, NULL);
6767c478bd9Sstevel@tonic-gate 	need_vfs_unlock = !MUTEX_HELD(&ip->i_ufsvfs->vfs_lock);
6777c478bd9Sstevel@tonic-gate 
6787c478bd9Sstevel@tonic-gate 	if (need_vfs_unlock) {
6797c478bd9Sstevel@tonic-gate 		if (!mutex_tryenter(&ip->i_ufsvfs->vfs_lock)) {
6807c478bd9Sstevel@tonic-gate 			/*
6817c478bd9Sstevel@tonic-gate 			 * not much alternative here, but we're panicking
6827c478bd9Sstevel@tonic-gate 			 * already, it couldn't be worse - so just
6837c478bd9Sstevel@tonic-gate 			 * proceed optimistically and take note.
6847c478bd9Sstevel@tonic-gate 			 */
6857c478bd9Sstevel@tonic-gate 			mutex_enter(&uf_stats.ufst_mutex);
6867c478bd9Sstevel@tonic-gate 			uf_stats.ufst_lock_violations++;
6877c478bd9Sstevel@tonic-gate 			mutex_exit(&uf_stats.ufst_mutex);
6887c478bd9Sstevel@tonic-gate 			MINOR((": couldn't get vfs lock"))
6897c478bd9Sstevel@tonic-gate 			need_vfs_unlock = 0;
6907c478bd9Sstevel@tonic-gate 		}
6917c478bd9Sstevel@tonic-gate 	}
6927c478bd9Sstevel@tonic-gate 
6937c478bd9Sstevel@tonic-gate 	if (mutex_tryenter(&new->uf_mutex)) {
6947c478bd9Sstevel@tonic-gate 		initialization_worked = set_state(new, UF_INIT);
6957c478bd9Sstevel@tonic-gate 		mutex_exit(&new->uf_mutex);
6967c478bd9Sstevel@tonic-gate 	}
6977c478bd9Sstevel@tonic-gate 
6987c478bd9Sstevel@tonic-gate 	if (need_vfs_unlock)
6997c478bd9Sstevel@tonic-gate 		mutex_exit(&ip->i_ufsvfs->vfs_lock);
7007c478bd9Sstevel@tonic-gate 
7017c478bd9Sstevel@tonic-gate 	if (initialization_worked) {
7027c478bd9Sstevel@tonic-gate 		MINOR(("] "));
7037c478bd9Sstevel@tonic-gate 		return (new);
7047c478bd9Sstevel@tonic-gate 	}
7057c478bd9Sstevel@tonic-gate 	/* FALLTHROUGH */
7067c478bd9Sstevel@tonic-gate 
7077c478bd9Sstevel@tonic-gate errout:
7087c478bd9Sstevel@tonic-gate 	if (new)
7097c478bd9Sstevel@tonic-gate 		kmem_free(new, sizeof (ufs_failure_t));
7107c478bd9Sstevel@tonic-gate 	MINOR((": failed]\n"));
7117c478bd9Sstevel@tonic-gate 	return (NULL);
7127c478bd9Sstevel@tonic-gate }
7137c478bd9Sstevel@tonic-gate 
7147c478bd9Sstevel@tonic-gate static void
queue_failure(ufs_failure_t * new)7157c478bd9Sstevel@tonic-gate queue_failure(ufs_failure_t *new)
7167c478bd9Sstevel@tonic-gate {
7177c478bd9Sstevel@tonic-gate 	MINOR(("[queue_failure"));
7187c478bd9Sstevel@tonic-gate 
7197c478bd9Sstevel@tonic-gate 	mutex_enter(&ufs_fix.uq_mutex);
7207c478bd9Sstevel@tonic-gate 
7217c478bd9Sstevel@tonic-gate 	if (ufs_fix.uq_ufhead)
7227c478bd9Sstevel@tonic-gate 		insque(new, &ufs_fix.uq_ufhead);
7237c478bd9Sstevel@tonic-gate 	else
7247c478bd9Sstevel@tonic-gate 		ufs_fix.uq_ufhead = new;
7257c478bd9Sstevel@tonic-gate 
7267c478bd9Sstevel@tonic-gate 	if (mutex_tryenter(&new->uf_mutex)) {
7277c478bd9Sstevel@tonic-gate 		(void) set_state(new, UF_QUEUE);
7287c478bd9Sstevel@tonic-gate 		mutex_exit(&new->uf_mutex);
7297c478bd9Sstevel@tonic-gate 	}
7307c478bd9Sstevel@tonic-gate 
7317c478bd9Sstevel@tonic-gate 	mutex_enter(&uf_stats.ufst_mutex);		/* force wakeup */
7327c478bd9Sstevel@tonic-gate 	ufs_fix.uq_ne = ufs_fix.uq_lowat = uf_stats.ufst_num_failed;
7337c478bd9Sstevel@tonic-gate 	mutex_exit(&uf_stats.ufst_mutex);
7347c478bd9Sstevel@tonic-gate 
7357c478bd9Sstevel@tonic-gate 	cv_broadcast(&ufs_fix.uq_cv);
7367c478bd9Sstevel@tonic-gate 
73780d34432Sfrankho 	DCALL(DBGLVL_MAJOR, cmn_err(CE_WARN, new->uf_panic_str ?
73880d34432Sfrankho 	    new->uf_panic_str : "queue_failure: NULL panic str?"));
7397c478bd9Sstevel@tonic-gate 	mutex_exit(&ufs_fix.uq_mutex);
7407c478bd9Sstevel@tonic-gate 
7417c478bd9Sstevel@tonic-gate 	MINOR(("] "));
7427c478bd9Sstevel@tonic-gate }
7437c478bd9Sstevel@tonic-gate 
7447c478bd9Sstevel@tonic-gate /*PRINTFLIKE2*/
7457c478bd9Sstevel@tonic-gate static void
real_panic(ufs_failure_t * f,const char * fmt,...)7467c478bd9Sstevel@tonic-gate real_panic(ufs_failure_t *f, const char *fmt, ...)
7477c478bd9Sstevel@tonic-gate {
7487c478bd9Sstevel@tonic-gate 	va_list	adx;
7497c478bd9Sstevel@tonic-gate 
7507c478bd9Sstevel@tonic-gate 	MINUTE(("[real_panic "));
7517c478bd9Sstevel@tonic-gate 
7527c478bd9Sstevel@tonic-gate 	va_start(adx, fmt);
7537c478bd9Sstevel@tonic-gate 	real_panic_v(f, fmt, adx);
7547c478bd9Sstevel@tonic-gate 	va_end(adx);
7557c478bd9Sstevel@tonic-gate 
7567c478bd9Sstevel@tonic-gate 	MINUTE((": return?!]\n"));
7577c478bd9Sstevel@tonic-gate }
7587c478bd9Sstevel@tonic-gate 
7597c478bd9Sstevel@tonic-gate static void
real_panic_v(ufs_failure_t * f,const char * fmt,va_list adx)7607c478bd9Sstevel@tonic-gate real_panic_v(ufs_failure_t *f, const char *fmt, va_list adx)
7617c478bd9Sstevel@tonic-gate {
7627c478bd9Sstevel@tonic-gate 	int seriousness = CE_PANIC;
7637c478bd9Sstevel@tonic-gate 	int need_unlock;
7647c478bd9Sstevel@tonic-gate 
7657c478bd9Sstevel@tonic-gate 	MINUTE(("[real_panic_v "));
7667c478bd9Sstevel@tonic-gate 
7677c478bd9Sstevel@tonic-gate 	if (f && f->uf_ufsvfsp)
7687c478bd9Sstevel@tonic-gate 		TRANS_SETERROR(f->uf_ufsvfsp);
7697c478bd9Sstevel@tonic-gate 
7707c478bd9Sstevel@tonic-gate #if defined(DEBUG)
7717c478bd9Sstevel@tonic-gate 	if (DEBUG_FLAGS & DBGFLG_NOPANIC) {
7727c478bd9Sstevel@tonic-gate 		seriousness = CE_WARN;
7737c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "real_panic: EWOULDPANIC\n");
7747c478bd9Sstevel@tonic-gate 	}
7757c478bd9Sstevel@tonic-gate #endif /* DEBUG */
7767c478bd9Sstevel@tonic-gate 
7777c478bd9Sstevel@tonic-gate 	delay(hz >> 1);			/* allow previous warnings to get out */
7787c478bd9Sstevel@tonic-gate 
7797c478bd9Sstevel@tonic-gate 	if (!f && fmt)
7807c478bd9Sstevel@tonic-gate 		vcmn_err(seriousness, fmt, adx);
7817c478bd9Sstevel@tonic-gate 	else
782*1d48f750SToomas Soome 		cmn_err(seriousness, f != NULL && f->uf_panic_str[0] != '\0' ?
783*1d48f750SToomas Soome 		    f->uf_panic_str: "real_panic: <unknown panic?>");
7847c478bd9Sstevel@tonic-gate 
7857c478bd9Sstevel@tonic-gate 	if (f) {
7867c478bd9Sstevel@tonic-gate 		need_unlock = !MUTEX_HELD(&f->uf_mutex);
7877c478bd9Sstevel@tonic-gate 		if (need_unlock) {
7887c478bd9Sstevel@tonic-gate 			mutex_enter(&f->uf_mutex);
7897c478bd9Sstevel@tonic-gate 		}
7907c478bd9Sstevel@tonic-gate 
7917c478bd9Sstevel@tonic-gate 		f->uf_retry = -1;
7927c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_PANIC);
7937c478bd9Sstevel@tonic-gate 
7947c478bd9Sstevel@tonic-gate 		if (need_unlock) {
7957c478bd9Sstevel@tonic-gate 			mutex_exit(&f->uf_mutex);
7967c478bd9Sstevel@tonic-gate 		}
7977c478bd9Sstevel@tonic-gate 	}
7987c478bd9Sstevel@tonic-gate 	MINUTE((": return?!]\n"));
7997c478bd9Sstevel@tonic-gate }
8007c478bd9Sstevel@tonic-gate 
8017c478bd9Sstevel@tonic-gate /*
8027c478bd9Sstevel@tonic-gate  * initializes ufs panic structs, locks, etc
8037c478bd9Sstevel@tonic-gate  */
8047c478bd9Sstevel@tonic-gate void
ufsfx_init(void)8057c478bd9Sstevel@tonic-gate ufsfx_init(void)
8067c478bd9Sstevel@tonic-gate {
8077c478bd9Sstevel@tonic-gate 
8087c478bd9Sstevel@tonic-gate 	MINUTE(("[ufsfx_init"));
8097c478bd9Sstevel@tonic-gate 
8107c478bd9Sstevel@tonic-gate 	/* patchable; unchanged while running, so no lock is needed */
8117c478bd9Sstevel@tonic-gate 	ufsfx_tune.uft_too_long		= UF_TOO_LONG;
8127c478bd9Sstevel@tonic-gate 	ufsfx_tune.uft_fixstart_period	= UF_FIXSTART_PERIOD;
8137c478bd9Sstevel@tonic-gate 	ufsfx_tune.uft_fixpoll_period	= UF_FIXPOLL_PERIOD;
8147c478bd9Sstevel@tonic-gate 	ufsfx_tune.uft_short_err_period	= UF_SHORT_ERROR_PERIOD;
8157c478bd9Sstevel@tonic-gate 	ufsfx_tune.uft_long_err_period	= UF_LONG_ERROR_PERIOD;
8167c478bd9Sstevel@tonic-gate 
8177c478bd9Sstevel@tonic-gate 	uffsinfo.ufi_statp	= &uf_stats;
8187c478bd9Sstevel@tonic-gate 	uffsinfo.ufi_tunep	= &ufsfx_tune;
8197c478bd9Sstevel@tonic-gate 	uffsinfo.ufi_statetab	= &state_desc[0];
8207c478bd9Sstevel@tonic-gate 
8217c478bd9Sstevel@tonic-gate 	mutex_init(&uf_stats.ufst_mutex, NULL, MUTEX_DEFAULT, NULL);
8227c478bd9Sstevel@tonic-gate 	ufs_thread_init(&ufs_fix, /* maxne */ 1);
8237c478bd9Sstevel@tonic-gate 
8247c478bd9Sstevel@tonic-gate 	MINUTE(("] "));
8257c478bd9Sstevel@tonic-gate }
8267c478bd9Sstevel@tonic-gate 
8277c478bd9Sstevel@tonic-gate /*
8287c478bd9Sstevel@tonic-gate  * initializes per-ufs values
8297c478bd9Sstevel@tonic-gate  * returns 0 (ok) or errno
8307c478bd9Sstevel@tonic-gate  */
8317c478bd9Sstevel@tonic-gate int
ufsfx_mount(struct ufsvfs * ufsvfsp,int flags)8327c478bd9Sstevel@tonic-gate ufsfx_mount(struct ufsvfs *ufsvfsp, int flags)
8337c478bd9Sstevel@tonic-gate {
8347c478bd9Sstevel@tonic-gate 	MINUTE(("[ufsfx_mount (%d)", flags));
8357c478bd9Sstevel@tonic-gate 	/* don't check/need vfs_lock because it's still being initialized */
8367c478bd9Sstevel@tonic-gate 
8377c478bd9Sstevel@tonic-gate 	ufsvfsp->vfs_fsfx.fx_flags = (flags & UFSMNT_ONERROR_FLGMASK) >> 4;
8387c478bd9Sstevel@tonic-gate 
8397c478bd9Sstevel@tonic-gate 	MINUTE((": %s: fx_flags:%ld,",
84080d34432Sfrankho 	    ufsvfsp->vfs_fs->fs_fsmnt, ufsvfsp->vfs_fsfx.fx_flags));
8417c478bd9Sstevel@tonic-gate 	/*
8427c478bd9Sstevel@tonic-gate 	 *	onerror={panic ^ lock only ^ unmount}
8437c478bd9Sstevel@tonic-gate 	 */
8447c478bd9Sstevel@tonic-gate 
8457c478bd9Sstevel@tonic-gate 	if (ufsvfsp->vfs_fsfx.fx_flags & UFSFX_PANIC) {
8467c478bd9Sstevel@tonic-gate 		MINUTE((" PANIC"));
8477c478bd9Sstevel@tonic-gate 
8487c478bd9Sstevel@tonic-gate 	} else if (ufsvfsp->vfs_fsfx.fx_flags & UFSFX_LCKONLY) {
8497c478bd9Sstevel@tonic-gate 		MINUTE((" LCKONLY"));
8507c478bd9Sstevel@tonic-gate 
8517c478bd9Sstevel@tonic-gate 	} else if (ufsvfsp->vfs_fsfx.fx_flags & UFSFX_LCKUMOUNT) {
8527c478bd9Sstevel@tonic-gate 		MINUTE((" LCKUMOUNT"));
8537c478bd9Sstevel@tonic-gate 
8547c478bd9Sstevel@tonic-gate 	} else {
8557c478bd9Sstevel@tonic-gate 		ufsvfsp->vfs_fsfx.fx_flags = UFSFX_DEFAULT;
8567c478bd9Sstevel@tonic-gate 		ASSERT(ufsvfsp->vfs_fsfx.fx_flags &
85780d34432Sfrankho 		    (UFSMNT_ONERROR_FLGMASK >> 4));
8587c478bd9Sstevel@tonic-gate 		MINUTE((" DEFAULT"));
8597c478bd9Sstevel@tonic-gate 	}
8607c478bd9Sstevel@tonic-gate 
8617c478bd9Sstevel@tonic-gate 	pollwakeup(&ufs_pollhd, POLLPRI);
8627c478bd9Sstevel@tonic-gate 	MINUTE(("]\n"));
8637c478bd9Sstevel@tonic-gate 	return (0);
8647c478bd9Sstevel@tonic-gate }
8657c478bd9Sstevel@tonic-gate 
8667c478bd9Sstevel@tonic-gate /*
8677c478bd9Sstevel@tonic-gate  * ufsfx_unmount
8687c478bd9Sstevel@tonic-gate  *
8697c478bd9Sstevel@tonic-gate  * called during unmount
8707c478bd9Sstevel@tonic-gate  */
8717c478bd9Sstevel@tonic-gate void
ufsfx_unmount(struct ufsvfs * ufsvfsp)8727c478bd9Sstevel@tonic-gate ufsfx_unmount(struct ufsvfs *ufsvfsp)
8737c478bd9Sstevel@tonic-gate {
8747c478bd9Sstevel@tonic-gate 	ufs_failure_t	*f;
8757c478bd9Sstevel@tonic-gate 	int		 must_unlock_list;
8767c478bd9Sstevel@tonic-gate 
8777c478bd9Sstevel@tonic-gate 	MINUTE(("[ufsfx_unmount"));
8787c478bd9Sstevel@tonic-gate 
8797c478bd9Sstevel@tonic-gate 	if (!ufsvfsp) {
8807c478bd9Sstevel@tonic-gate 		MINUTE((": no ufsvfsp]"));
8817c478bd9Sstevel@tonic-gate 		return;
8827c478bd9Sstevel@tonic-gate 	}
8837c478bd9Sstevel@tonic-gate 
8847c478bd9Sstevel@tonic-gate 	if ((must_unlock_list = !MUTEX_HELD(&ufs_fix.uq_mutex)) != 0)
8857c478bd9Sstevel@tonic-gate 		mutex_enter(&ufs_fix.uq_mutex);
8867c478bd9Sstevel@tonic-gate 
8877c478bd9Sstevel@tonic-gate 	for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {
8887c478bd9Sstevel@tonic-gate 		int must_unlock_failure;
8897c478bd9Sstevel@tonic-gate 
8907c478bd9Sstevel@tonic-gate 		must_unlock_failure = !MUTEX_HELD(&f->uf_mutex);
8917c478bd9Sstevel@tonic-gate 		if (must_unlock_failure) {
8927c478bd9Sstevel@tonic-gate 			mutex_enter(&f->uf_mutex);
8937c478bd9Sstevel@tonic-gate 		}
8947c478bd9Sstevel@tonic-gate 
8957c478bd9Sstevel@tonic-gate 		if (f->uf_ufsvfsp == ufsvfsp) {
8967c478bd9Sstevel@tonic-gate 
8977c478bd9Sstevel@tonic-gate 			/*
8987c478bd9Sstevel@tonic-gate 			 * if we owned the failure record lock, then this
8997c478bd9Sstevel@tonic-gate 			 * is probably a fix failure-triggered unmount, so
9007c478bd9Sstevel@tonic-gate 			 * the warning is not appropriate or needed
9017c478bd9Sstevel@tonic-gate 			 */
9027c478bd9Sstevel@tonic-gate 
9037c478bd9Sstevel@tonic-gate 			/* XXX if rebooting don't print this? */
9047c478bd9Sstevel@tonic-gate 			if (!terminal_state(f->uf_s) && must_unlock_failure) {
9057c478bd9Sstevel@tonic-gate 				cmn_err(CE_WARN,
90680d34432Sfrankho 				    "Unmounting %s while error-locked",
90780d34432Sfrankho 				    fs_name(f));
9087c478bd9Sstevel@tonic-gate 			}
9097c478bd9Sstevel@tonic-gate 
9107c478bd9Sstevel@tonic-gate 			f->uf_ufsvfsp		= NULL;
9117c478bd9Sstevel@tonic-gate 			f->uf_vfs_ufsfxp	= NULL;
9127c478bd9Sstevel@tonic-gate 			f->uf_vfs_lockp		= NULL;
9137c478bd9Sstevel@tonic-gate 			f->uf_bp		= NULL;
9147c478bd9Sstevel@tonic-gate 			f->uf_vfsp		= NULL;
9157c478bd9Sstevel@tonic-gate 			f->uf_retry		= -1;
9167c478bd9Sstevel@tonic-gate 		}
9177c478bd9Sstevel@tonic-gate 
9187c478bd9Sstevel@tonic-gate 		if (must_unlock_failure)
9197c478bd9Sstevel@tonic-gate 			mutex_exit(&f->uf_mutex);
9207c478bd9Sstevel@tonic-gate 	}
9217c478bd9Sstevel@tonic-gate 	if (must_unlock_list)
9227c478bd9Sstevel@tonic-gate 		mutex_exit(&ufs_fix.uq_mutex);
9237c478bd9Sstevel@tonic-gate 
9247c478bd9Sstevel@tonic-gate 	pollwakeup(&ufs_pollhd, POLLPRI | POLLHUP);
9257c478bd9Sstevel@tonic-gate 	MINUTE(("] "));
9267c478bd9Sstevel@tonic-gate }
9277c478bd9Sstevel@tonic-gate 
9287c478bd9Sstevel@tonic-gate /*
9297c478bd9Sstevel@tonic-gate  * ufsfx_(un)lockfs
9307c478bd9Sstevel@tonic-gate  *
9317c478bd9Sstevel@tonic-gate  * provides hook from lockfs code so we can recognize unlock/relock
9327c478bd9Sstevel@tonic-gate  *  This is called after it is certain that the (un)lock will succeed.
9337c478bd9Sstevel@tonic-gate  */
9347c478bd9Sstevel@tonic-gate void
ufsfx_unlockfs(struct ufsvfs * ufsvfsp)9357c478bd9Sstevel@tonic-gate ufsfx_unlockfs(struct ufsvfs *ufsvfsp)
9367c478bd9Sstevel@tonic-gate {
9377c478bd9Sstevel@tonic-gate 	ufs_failure_t	*f;
9387c478bd9Sstevel@tonic-gate 	int		 need_unlock;
9397c478bd9Sstevel@tonic-gate 	int		 need_unlock_list;
9407c478bd9Sstevel@tonic-gate 	int		 informed = 0;
9417c478bd9Sstevel@tonic-gate 
9427c478bd9Sstevel@tonic-gate 	MINUTE(("[ufsfx_unlockfs"));
9437c478bd9Sstevel@tonic-gate 
9447c478bd9Sstevel@tonic-gate 	if (!ufsvfsp)
9457c478bd9Sstevel@tonic-gate 		return;
9467c478bd9Sstevel@tonic-gate 
9477c478bd9Sstevel@tonic-gate 	need_unlock_list = !MUTEX_HELD(&ufs_fix.uq_mutex);
9487c478bd9Sstevel@tonic-gate 
9497c478bd9Sstevel@tonic-gate 	if (need_unlock_list)
9507c478bd9Sstevel@tonic-gate 		mutex_enter(&ufs_fix.uq_mutex);
9517c478bd9Sstevel@tonic-gate 
9527c478bd9Sstevel@tonic-gate 	for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {
9537c478bd9Sstevel@tonic-gate 
9547c478bd9Sstevel@tonic-gate 		need_unlock = !MUTEX_HELD(&f->uf_mutex);
9557c478bd9Sstevel@tonic-gate 		if (need_unlock)
9567c478bd9Sstevel@tonic-gate 			mutex_enter(&f->uf_mutex);
9577c478bd9Sstevel@tonic-gate 
9587c478bd9Sstevel@tonic-gate 		if (f->uf_ufsvfsp == ufsvfsp && !terminal_state(f->uf_s)) {
9597c478bd9Sstevel@tonic-gate 			if (!(f->uf_s & UF_FIXING)) {
9607c478bd9Sstevel@tonic-gate 				/*
9617c478bd9Sstevel@tonic-gate 				 * This might happen if we don't notice that
9627c478bd9Sstevel@tonic-gate 				 * the fs gets marked FSFIX before it is
9637c478bd9Sstevel@tonic-gate 				 * marked FSCLEAN, as might occur if the
9647c478bd9Sstevel@tonic-gate 				 * the superblock was hammered directly.
9657c478bd9Sstevel@tonic-gate 				 */
9667c478bd9Sstevel@tonic-gate 				if (!informed) {
9677c478bd9Sstevel@tonic-gate 					informed = 1;
9687c478bd9Sstevel@tonic-gate 					cmn_err(CE_NOTE,
96980d34432Sfrankho 					    "Unlock of %s succeeded before "
97080d34432Sfrankho 					    "fs_clean marked FSFIX?",
97180d34432Sfrankho 					    fs_name(f));
9727c478bd9Sstevel@tonic-gate 				}
9737c478bd9Sstevel@tonic-gate 
9747c478bd9Sstevel@tonic-gate 				/*
9757c478bd9Sstevel@tonic-gate 				 * pass through fixing state so
9767c478bd9Sstevel@tonic-gate 				 * transition protocol is satisfied
9777c478bd9Sstevel@tonic-gate 				 */
9787c478bd9Sstevel@tonic-gate 				if (!set_state(f, UF_FIXING)) {
9797c478bd9Sstevel@tonic-gate 					MINUTE((": failed] "));
9807c478bd9Sstevel@tonic-gate 				}
9817c478bd9Sstevel@tonic-gate 			}
9827c478bd9Sstevel@tonic-gate 
9837c478bd9Sstevel@tonic-gate 			if (!set_state(f, UF_FIXED)) {
9847c478bd9Sstevel@tonic-gate 				/* it's already fixed, so don't panic now */
9857c478bd9Sstevel@tonic-gate 				MINUTE((": failed] "));
9867c478bd9Sstevel@tonic-gate 			}
9877c478bd9Sstevel@tonic-gate 		}
9887c478bd9Sstevel@tonic-gate 
9897c478bd9Sstevel@tonic-gate 		if (need_unlock)
9907c478bd9Sstevel@tonic-gate 			mutex_exit(&f->uf_mutex);
9917c478bd9Sstevel@tonic-gate 	}
9927c478bd9Sstevel@tonic-gate 	if (need_unlock_list)
9937c478bd9Sstevel@tonic-gate 		mutex_exit(&ufs_fix.uq_mutex);
9947c478bd9Sstevel@tonic-gate 	MINUTE(("] "));
9957c478bd9Sstevel@tonic-gate }
9967c478bd9Sstevel@tonic-gate 
9977c478bd9Sstevel@tonic-gate void
ufsfx_lockfs(struct ufsvfs * ufsvfsp)9987c478bd9Sstevel@tonic-gate ufsfx_lockfs(struct ufsvfs *ufsvfsp)
9997c478bd9Sstevel@tonic-gate {
10007c478bd9Sstevel@tonic-gate 	ufs_failure_t	*f;
10017c478bd9Sstevel@tonic-gate 	int		 need_unlock;
10027c478bd9Sstevel@tonic-gate 	int		 need_unlock_list;
10037c478bd9Sstevel@tonic-gate 
10047c478bd9Sstevel@tonic-gate 	MINUTE(("[ufsfx_lockfs"));
10057c478bd9Sstevel@tonic-gate 
10067c478bd9Sstevel@tonic-gate 	if (!ufsvfsp)
10077c478bd9Sstevel@tonic-gate 		return;
10087c478bd9Sstevel@tonic-gate 
10097c478bd9Sstevel@tonic-gate 	need_unlock_list = !MUTEX_HELD(&ufs_fix.uq_mutex);
10107c478bd9Sstevel@tonic-gate 
10117c478bd9Sstevel@tonic-gate 	if (need_unlock_list)
10127c478bd9Sstevel@tonic-gate 		mutex_enter(&ufs_fix.uq_mutex);
10137c478bd9Sstevel@tonic-gate 
10147c478bd9Sstevel@tonic-gate 	for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {
10157c478bd9Sstevel@tonic-gate 
10167c478bd9Sstevel@tonic-gate 		need_unlock = !MUTEX_HELD(&f->uf_mutex);
10177c478bd9Sstevel@tonic-gate 		if (need_unlock)
10187c478bd9Sstevel@tonic-gate 			mutex_enter(&f->uf_mutex);
10197c478bd9Sstevel@tonic-gate 
10207c478bd9Sstevel@tonic-gate 		if (f->uf_ufsvfsp == ufsvfsp && !terminal_state(f->uf_s) &&
10217c478bd9Sstevel@tonic-gate 		    f->uf_s != UF_PANIC) {
10227c478bd9Sstevel@tonic-gate 			switch (f->uf_s) {
10237c478bd9Sstevel@tonic-gate 
10247c478bd9Sstevel@tonic-gate 			default:
10257c478bd9Sstevel@tonic-gate 				cmn_err(CE_WARN,
102680d34432Sfrankho 				    "fs %s not in state "
102780d34432Sfrankho 				    "UF_TRYLCK, UF_LOCKED or UF_FIXING",
102880d34432Sfrankho 				    fs_name(f));
10297c478bd9Sstevel@tonic-gate 				break;
10307c478bd9Sstevel@tonic-gate 
10317c478bd9Sstevel@tonic-gate 			case UF_TRYLCK:
10327c478bd9Sstevel@tonic-gate 				if (!set_state(f, UF_LOCKED)) {
10337c478bd9Sstevel@tonic-gate 					MINUTE((": failed] "));
10347c478bd9Sstevel@tonic-gate 				}
10357c478bd9Sstevel@tonic-gate 				break;
10367c478bd9Sstevel@tonic-gate 
10377c478bd9Sstevel@tonic-gate 			case UF_LOCKED:
10387c478bd9Sstevel@tonic-gate 				if (!set_state(f, UF_FIXING)) {
10397c478bd9Sstevel@tonic-gate 					MINUTE((": failed] "));
10407c478bd9Sstevel@tonic-gate 				}
10417c478bd9Sstevel@tonic-gate 				break;
10427c478bd9Sstevel@tonic-gate 
10437c478bd9Sstevel@tonic-gate 			case UF_FIXING:
10447c478bd9Sstevel@tonic-gate 				break;
10457c478bd9Sstevel@tonic-gate 
10467c478bd9Sstevel@tonic-gate 			}
10477c478bd9Sstevel@tonic-gate 		}
10487c478bd9Sstevel@tonic-gate 
10497c478bd9Sstevel@tonic-gate 		if (need_unlock)
10507c478bd9Sstevel@tonic-gate 			mutex_exit(&f->uf_mutex);
10517c478bd9Sstevel@tonic-gate 	}
10527c478bd9Sstevel@tonic-gate 	if (need_unlock_list)
10537c478bd9Sstevel@tonic-gate 		mutex_exit(&ufs_fix.uq_mutex);
10547c478bd9Sstevel@tonic-gate 
10557c478bd9Sstevel@tonic-gate 	MINUTE(("] "));
10567c478bd9Sstevel@tonic-gate }
10577c478bd9Sstevel@tonic-gate 
10587c478bd9Sstevel@tonic-gate /*
10597c478bd9Sstevel@tonic-gate  * error lock, trigger fsck and unlock those fs with failures
10607c478bd9Sstevel@tonic-gate  * blatantly copied from the hlock routine, although this routine
10617c478bd9Sstevel@tonic-gate  * triggers differently in order to use uq_ne as meaningful data.
10627c478bd9Sstevel@tonic-gate  */
10637c478bd9Sstevel@tonic-gate /* ARGSUSED */
10647c478bd9Sstevel@tonic-gate void
ufsfx_thread_fix_failures(void * ignored)10657c478bd9Sstevel@tonic-gate ufsfx_thread_fix_failures(void *ignored)
10667c478bd9Sstevel@tonic-gate {
10677c478bd9Sstevel@tonic-gate 	int		retry;
10687c478bd9Sstevel@tonic-gate 	callb_cpr_t	cprinfo;
10697c478bd9Sstevel@tonic-gate 
10707c478bd9Sstevel@tonic-gate 	CALLB_CPR_INIT(&cprinfo, &ufs_fix.uq_mutex, callb_generic_cpr,
10717c478bd9Sstevel@tonic-gate 	    "ufsfixfail");
10727c478bd9Sstevel@tonic-gate 
10737c478bd9Sstevel@tonic-gate 	MINUTE(("[ufsfx_thread_fix_failures] "));
10747c478bd9Sstevel@tonic-gate 
10757c478bd9Sstevel@tonic-gate 	for (;;) {
10767c478bd9Sstevel@tonic-gate 		/* sleep until there is work to do */
10777c478bd9Sstevel@tonic-gate 
10787c478bd9Sstevel@tonic-gate 		mutex_enter(&ufs_fix.uq_mutex);
10797c478bd9Sstevel@tonic-gate 		(void) ufs_thread_run(&ufs_fix, &cprinfo);
10807c478bd9Sstevel@tonic-gate 		ufs_fix.uq_ne = 0;
10817c478bd9Sstevel@tonic-gate 		mutex_exit(&ufs_fix.uq_mutex);
10827c478bd9Sstevel@tonic-gate 
10837c478bd9Sstevel@tonic-gate 		/* process failures on our q */
10847c478bd9Sstevel@tonic-gate 		do {
10857c478bd9Sstevel@tonic-gate 			retry = ufsfx_do_failure_q();
10867c478bd9Sstevel@tonic-gate 			if (retry) {
10877c478bd9Sstevel@tonic-gate 				mutex_enter(&ufs_fix.uq_mutex);
10887c478bd9Sstevel@tonic-gate 				CALLB_CPR_SAFE_BEGIN(&cprinfo);
1089d3d50737SRafael Vanoni 				(void) cv_reltimedwait(&ufs_fix.uq_cv,
1090d3d50737SRafael Vanoni 				    &ufs_fix.uq_mutex, (hz * retry),
1091d3d50737SRafael Vanoni 				    TR_CLOCK_TICK);
10927c478bd9Sstevel@tonic-gate 				CALLB_CPR_SAFE_END(&cprinfo,
10937c478bd9Sstevel@tonic-gate 				    &ufs_fix.uq_mutex);
10947c478bd9Sstevel@tonic-gate 				mutex_exit(&ufs_fix.uq_mutex);
10957c478bd9Sstevel@tonic-gate 			}
10967c478bd9Sstevel@tonic-gate 		} while (retry);
10977c478bd9Sstevel@tonic-gate 	}
10987c478bd9Sstevel@tonic-gate 	/* NOTREACHED */
10997c478bd9Sstevel@tonic-gate }
11007c478bd9Sstevel@tonic-gate 
11017c478bd9Sstevel@tonic-gate 
11027c478bd9Sstevel@tonic-gate /*
11037c478bd9Sstevel@tonic-gate  * watch for fix-on-panic work
11047c478bd9Sstevel@tonic-gate  *
11057c478bd9Sstevel@tonic-gate  * returns # of seconds to sleep before trying again
11067c478bd9Sstevel@tonic-gate  * and zero if no retry is needed
11077c478bd9Sstevel@tonic-gate  */
11087c478bd9Sstevel@tonic-gate 
11097c478bd9Sstevel@tonic-gate int
ufsfx_do_failure_q(void)11107c478bd9Sstevel@tonic-gate ufsfx_do_failure_q(void)
11117c478bd9Sstevel@tonic-gate {
11127c478bd9Sstevel@tonic-gate 	ufs_failure_t	*f;
11137c478bd9Sstevel@tonic-gate 	long		 retry = 1;
11147c478bd9Sstevel@tonic-gate 	ufsd_t		*s;
11157c478bd9Sstevel@tonic-gate 
11167c478bd9Sstevel@tonic-gate 	MAJOR(("[ufsfx_do_failure_q"));
11177c478bd9Sstevel@tonic-gate 	DCALL(DBGLVL_HIDEOUS, dump_uf_list(NULL));
11187c478bd9Sstevel@tonic-gate 
11197c478bd9Sstevel@tonic-gate 	if (!mutex_tryenter(&ufs_fix.uq_mutex))
11207c478bd9Sstevel@tonic-gate 		return (retry);
11217c478bd9Sstevel@tonic-gate 
11227c478bd9Sstevel@tonic-gate 	retry = 0;
11237c478bd9Sstevel@tonic-gate rescan_q:
11247c478bd9Sstevel@tonic-gate 
11257c478bd9Sstevel@tonic-gate 	/*
11267c478bd9Sstevel@tonic-gate 	 * walk down failure list
11277c478bd9Sstevel@tonic-gate 	 *  depending on state of each failure, do whatever
11287c478bd9Sstevel@tonic-gate 	 *  is appropriate to move it to the next state
11297c478bd9Sstevel@tonic-gate 	 *  taking note of whether retry gets set
11307c478bd9Sstevel@tonic-gate 	 *
11317c478bd9Sstevel@tonic-gate 	 * retry protocol:
11327c478bd9Sstevel@tonic-gate 	 * wakeup in shortest required time for any failure
11337c478bd9Sstevel@tonic-gate 	 *   retry == 0; nothing more to do (terminal state)
11347c478bd9Sstevel@tonic-gate 	 *   retry < 0; reprocess queue immediately, retry will
11357c478bd9Sstevel@tonic-gate 	 *		be abs(retry) for the next cycle
11367c478bd9Sstevel@tonic-gate 	 *   retry > 0; schedule wakeup for retry seconds
11377c478bd9Sstevel@tonic-gate 	 */
11387c478bd9Sstevel@tonic-gate 
11397c478bd9Sstevel@tonic-gate 	for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {
11407c478bd9Sstevel@tonic-gate 
11417c478bd9Sstevel@tonic-gate 		if (!mutex_tryenter(&f->uf_mutex)) {
11427c478bd9Sstevel@tonic-gate 			retry = 1;
11437c478bd9Sstevel@tonic-gate 			continue;
11447c478bd9Sstevel@tonic-gate 		}
11457c478bd9Sstevel@tonic-gate 		s = get_state_desc(f->uf_s);
11467c478bd9Sstevel@tonic-gate 
11477c478bd9Sstevel@tonic-gate 		MINOR((": found%s: %s, \"%s: %s\"\n",
114880d34432Sfrankho 		    s->ud_attr.terminal ? " old" : "",
114980d34432Sfrankho 		    fs_name(f), state_name(f->uf_s), f->uf_panic_str));
11507c478bd9Sstevel@tonic-gate 
11517c478bd9Sstevel@tonic-gate 		if (s->ud_attr.terminal) {
11527c478bd9Sstevel@tonic-gate 			mutex_exit(&f->uf_mutex);
11537c478bd9Sstevel@tonic-gate 			continue;
11547c478bd9Sstevel@tonic-gate 		}
11557c478bd9Sstevel@tonic-gate 
11567c478bd9Sstevel@tonic-gate 		if (s->ud_sfp)
11577c478bd9Sstevel@tonic-gate 			(*s->ud_sfp)(f, UFA_FOUND, f->uf_s);
11587c478bd9Sstevel@tonic-gate 
11597c478bd9Sstevel@tonic-gate 		ASSERT(terminal_state(f->uf_s) || f->uf_retry != 0);
11607c478bd9Sstevel@tonic-gate 
11617c478bd9Sstevel@tonic-gate 		if (f->uf_retry != 0) {
11627c478bd9Sstevel@tonic-gate 			if (retry > f->uf_retry || retry == 0)
11637c478bd9Sstevel@tonic-gate 				retry = f->uf_retry;
11647c478bd9Sstevel@tonic-gate 			if (f->uf_retry < 0)
11657c478bd9Sstevel@tonic-gate 				f->uf_retry = abs(f->uf_retry);
11667c478bd9Sstevel@tonic-gate 		}
11677c478bd9Sstevel@tonic-gate 		mutex_exit(&f->uf_mutex);
11687c478bd9Sstevel@tonic-gate 	}
11697c478bd9Sstevel@tonic-gate 
11707c478bd9Sstevel@tonic-gate 
11717c478bd9Sstevel@tonic-gate 	if (retry < 0) {
11727c478bd9Sstevel@tonic-gate 		retry = abs(retry);
11737c478bd9Sstevel@tonic-gate 		goto rescan_q;
11747c478bd9Sstevel@tonic-gate 	}
11757c478bd9Sstevel@tonic-gate 
11767c478bd9Sstevel@tonic-gate 	mutex_exit(&ufs_fix.uq_mutex);
11777c478bd9Sstevel@tonic-gate 
11787c478bd9Sstevel@tonic-gate 	DCALL(DBGLVL_HIDEOUS, dump_uf_list(NULL));
11797c478bd9Sstevel@tonic-gate 	MAJOR((": retry=%ld, good night]\n\n", retry));
11807c478bd9Sstevel@tonic-gate 
11817c478bd9Sstevel@tonic-gate 	return (retry);
11827c478bd9Sstevel@tonic-gate }
11837c478bd9Sstevel@tonic-gate 
11847c478bd9Sstevel@tonic-gate static void
pester_msg(ufs_failure_t * f,int seriousness)11857c478bd9Sstevel@tonic-gate pester_msg(ufs_failure_t *f, int seriousness)
11867c478bd9Sstevel@tonic-gate {
11877c478bd9Sstevel@tonic-gate 	MINUTE(("[pester_msg"));
11887c478bd9Sstevel@tonic-gate 	ASSERT(f->uf_s & (UF_LOCKED | UF_FIXING));
11897c478bd9Sstevel@tonic-gate 
11907c478bd9Sstevel@tonic-gate 	/*
11917c478bd9Sstevel@tonic-gate 	 * XXX if seems too long for this fs, poke administrator
11927c478bd9Sstevel@tonic-gate 	 * XXX to run fsck manually (and change retry time?)
11937c478bd9Sstevel@tonic-gate 	 */
119480d34432Sfrankho 	cmn_err(seriousness, "Waiting for repair of %s to %s",
119580d34432Sfrankho 	    fs_name(f), f->uf_s & UF_LOCKED ? "start" : "finish");
11967c478bd9Sstevel@tonic-gate 	MINUTE(("]"));
11977c478bd9Sstevel@tonic-gate }
11987c478bd9Sstevel@tonic-gate 
11997c478bd9Sstevel@tonic-gate static time_t
trylock_time_exceeded(ufs_failure_t * f)12007c478bd9Sstevel@tonic-gate trylock_time_exceeded(ufs_failure_t *f)
12017c478bd9Sstevel@tonic-gate {
12027c478bd9Sstevel@tonic-gate 	time_t		toolong;
12037c478bd9Sstevel@tonic-gate 	extern time_t	time;
12047c478bd9Sstevel@tonic-gate 
12057c478bd9Sstevel@tonic-gate 	MINUTE(("[trylock_time_exceeded"));
12067c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&f->uf_mutex));
12077c478bd9Sstevel@tonic-gate 
12087c478bd9Sstevel@tonic-gate 	toolong = (time_t)ufsfx_tune.uft_too_long + f->uf_entered_tm;
12097c478bd9Sstevel@tonic-gate 	if (time > toolong)
12107c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "error-lock timeout exceeded: %s", fs_name(f));
12117c478bd9Sstevel@tonic-gate 
12127c478bd9Sstevel@tonic-gate 	MINUTE(("] "));
12137c478bd9Sstevel@tonic-gate 	return (time <= toolong? 0: time - toolong);
12147c478bd9Sstevel@tonic-gate }
12157c478bd9Sstevel@tonic-gate 
12167c478bd9Sstevel@tonic-gate static int
get_lockfs_status(ufs_failure_t * f,struct lockfs * lfp)12177c478bd9Sstevel@tonic-gate get_lockfs_status(ufs_failure_t *f, struct lockfs *lfp)
12187c478bd9Sstevel@tonic-gate {
12197c478bd9Sstevel@tonic-gate 	MINUTE(("[get_lockfs_status"));
12207c478bd9Sstevel@tonic-gate 
12217c478bd9Sstevel@tonic-gate 	if (!f->uf_ufsvfsp) {
12227c478bd9Sstevel@tonic-gate 		MINUTE((": ufsvfsp is NULL]\n"));
12237c478bd9Sstevel@tonic-gate 		return (0);
12247c478bd9Sstevel@tonic-gate 	}
12257c478bd9Sstevel@tonic-gate 
12267c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&f->uf_mutex));
12277c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));
12287c478bd9Sstevel@tonic-gate 	ASSERT(!vfs_lock_held(f->uf_vfsp));
12297c478bd9Sstevel@tonic-gate 	ASSERT(f->uf_ufsvfsp->vfs_root != NULL);
12307c478bd9Sstevel@tonic-gate 
12317c478bd9Sstevel@tonic-gate 	f->uf_lf_err = ufs_fiolfss(f->uf_ufsvfsp->vfs_root, lfp);
12327c478bd9Sstevel@tonic-gate 
12337c478bd9Sstevel@tonic-gate 	if (f->uf_lf_err) {
12347c478bd9Sstevel@tonic-gate 		f->uf_retry = ufsfx_tune.uft_short_err_period;
12357c478bd9Sstevel@tonic-gate 	}
12367c478bd9Sstevel@tonic-gate 
12377c478bd9Sstevel@tonic-gate 	MINUTE(("] "));
12387c478bd9Sstevel@tonic-gate 	return (1);
12397c478bd9Sstevel@tonic-gate }
12407c478bd9Sstevel@tonic-gate 
12417c478bd9Sstevel@tonic-gate static sfrc_t
set_state(ufs_failure_t * f,ufs_failure_states_t new_state)12427c478bd9Sstevel@tonic-gate set_state(ufs_failure_t *f, ufs_failure_states_t new_state)
12437c478bd9Sstevel@tonic-gate {
12447c478bd9Sstevel@tonic-gate 	ufsd_t		*s;
12457c478bd9Sstevel@tonic-gate 	sfrc_t		 sfrc = SFRC_FAIL;
12467c478bd9Sstevel@tonic-gate 	int		 need_unlock;
12477c478bd9Sstevel@tonic-gate 	extern time_t	 time;
12487c478bd9Sstevel@tonic-gate 
12497c478bd9Sstevel@tonic-gate 	HIDEOUS(("[set_state: new state:%s", state_name(new_state)));
12507c478bd9Sstevel@tonic-gate 	ASSERT(f);
12517c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&f->uf_mutex));
12527c478bd9Sstevel@tonic-gate 
12537c478bd9Sstevel@tonic-gate 	/*
12547c478bd9Sstevel@tonic-gate 	 * if someone else is panicking, just let panic sync proceed
12557c478bd9Sstevel@tonic-gate 	 */
12567c478bd9Sstevel@tonic-gate 	if (panicstr) {
12577c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_NOTFIX);
12587c478bd9Sstevel@tonic-gate 		HIDEOUS((": state reset: not fixed] "));
12597c478bd9Sstevel@tonic-gate 		return (sfrc);
12607c478bd9Sstevel@tonic-gate 	}
12617c478bd9Sstevel@tonic-gate 
12627c478bd9Sstevel@tonic-gate 	/*
12637c478bd9Sstevel@tonic-gate 	 * bad state transition, an internal error
12647c478bd9Sstevel@tonic-gate 	 */
12657c478bd9Sstevel@tonic-gate 	if (!state_trans_valid(f->uf_s, new_state)) {
12667c478bd9Sstevel@tonic-gate 		/* recursion */
12677c478bd9Sstevel@tonic-gate 		if (!(f->uf_s & UF_PANIC) && !(new_state & UF_PANIC))
12687c478bd9Sstevel@tonic-gate 			(void) set_state(f, UF_PANIC);
12697c478bd9Sstevel@tonic-gate 		MINOR((": state reset: transition failure (\"%s\"->\"%s\")] ",
127080d34432Sfrankho 		    state_name(f->uf_s), state_name(new_state)));
12717c478bd9Sstevel@tonic-gate 		return (sfrc);
12727c478bd9Sstevel@tonic-gate 	}
12737c478bd9Sstevel@tonic-gate 
12747c478bd9Sstevel@tonic-gate 	s = get_state_desc(new_state);
12757c478bd9Sstevel@tonic-gate 
12767c478bd9Sstevel@tonic-gate 	need_unlock = !MUTEX_HELD(&ufs_fix.uq_mutex);
12777c478bd9Sstevel@tonic-gate 	if (need_unlock)
12787c478bd9Sstevel@tonic-gate 		mutex_enter(&ufs_fix.uq_mutex);
12797c478bd9Sstevel@tonic-gate 
12807c478bd9Sstevel@tonic-gate 	if (s->ud_attr.at_fail && ufs_fix.uq_threadp &&
12817c478bd9Sstevel@tonic-gate 	    curthread == ufs_fix.uq_threadp) {
12827c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "set_state: probable recursive panic of %s",
128380d34432Sfrankho 		    fs_name(f));
12847c478bd9Sstevel@tonic-gate 	}
12857c478bd9Sstevel@tonic-gate 	if (need_unlock)
12867c478bd9Sstevel@tonic-gate 		mutex_exit(&ufs_fix.uq_mutex);
12877c478bd9Sstevel@tonic-gate 
12887c478bd9Sstevel@tonic-gate 	/* NULL state functions always succeed */
12897c478bd9Sstevel@tonic-gate 	sfrc = !s->ud_sfp? SFRC_SUCCESS: (*s->ud_sfp)(f, UFA_SET, new_state);
12907c478bd9Sstevel@tonic-gate 
12917c478bd9Sstevel@tonic-gate 	if (sfrc == SFRC_SUCCESS && f->uf_s != new_state) {
12927c478bd9Sstevel@tonic-gate 		f->uf_s = new_state;
12937c478bd9Sstevel@tonic-gate 		f->uf_entered_tm = time;
12947c478bd9Sstevel@tonic-gate 		f->uf_counter = 0;
12957c478bd9Sstevel@tonic-gate 	}
12967c478bd9Sstevel@tonic-gate 
12977c478bd9Sstevel@tonic-gate 	HIDEOUS(("]\n"));
12987c478bd9Sstevel@tonic-gate 	return (sfrc);
12997c478bd9Sstevel@tonic-gate }
13007c478bd9Sstevel@tonic-gate 
13017c478bd9Sstevel@tonic-gate static ufsd_t *
get_state_desc(ufs_failure_states_t state)13027c478bd9Sstevel@tonic-gate get_state_desc(ufs_failure_states_t state)
13037c478bd9Sstevel@tonic-gate {
13047c478bd9Sstevel@tonic-gate 	ufsd_t *s;
13057c478bd9Sstevel@tonic-gate 
13067c478bd9Sstevel@tonic-gate 	HIDEOUS(("[get_state_desc"));
13077c478bd9Sstevel@tonic-gate 
13087c478bd9Sstevel@tonic-gate 	for (s = &state_desc[1]; s->ud_name != NULL; s++) {
13097c478bd9Sstevel@tonic-gate 		if (s->ud_v == state) {
13107c478bd9Sstevel@tonic-gate 			HIDEOUS(("] "));
13117c478bd9Sstevel@tonic-gate 			return (s);
13127c478bd9Sstevel@tonic-gate 		}
13137c478bd9Sstevel@tonic-gate 	}
13147c478bd9Sstevel@tonic-gate 
13157c478bd9Sstevel@tonic-gate 	HIDEOUS(("] "));
13167c478bd9Sstevel@tonic-gate 	return (&state_desc[0]);	/* default */
13177c478bd9Sstevel@tonic-gate }
13187c478bd9Sstevel@tonic-gate 
13197c478bd9Sstevel@tonic-gate static sfrc_t
sf_undef(ufs_failure_t * f,ufsa_t a,ufs_failure_states_t s)13207c478bd9Sstevel@tonic-gate sf_undef(ufs_failure_t *f, ufsa_t a, ufs_failure_states_t s)
13217c478bd9Sstevel@tonic-gate {
13227c478bd9Sstevel@tonic-gate 	sfrc_t rc;
13237c478bd9Sstevel@tonic-gate 
13247c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_undef, action is %s, state is %s\n",
132580d34432Sfrankho 	    act_name(a), state_name(s)));
13267c478bd9Sstevel@tonic-gate 	ASSERT(s == UF_UNDEF);
13277c478bd9Sstevel@tonic-gate 
13287c478bd9Sstevel@tonic-gate 	/* shouldn't find null failure records or ever set one */
13297c478bd9Sstevel@tonic-gate 	rc = set_state(f, UF_NOTFIX);
13307c478bd9Sstevel@tonic-gate 
13317c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
13327c478bd9Sstevel@tonic-gate 	return (rc);
13337c478bd9Sstevel@tonic-gate }
13347c478bd9Sstevel@tonic-gate 
13357c478bd9Sstevel@tonic-gate 
13367c478bd9Sstevel@tonic-gate static sfrc_t
sf_init(ufs_failure_t * f,ufsa_t a,ufs_failure_states_t s)13377c478bd9Sstevel@tonic-gate sf_init(
13387c478bd9Sstevel@tonic-gate 	ufs_failure_t	*f,
13397c478bd9Sstevel@tonic-gate 	ufsa_t	 a,
13407c478bd9Sstevel@tonic-gate 	ufs_failure_states_t	 s)
13417c478bd9Sstevel@tonic-gate {
13427c478bd9Sstevel@tonic-gate 	sfrc_t		rc = SFRC_FAIL;
13437c478bd9Sstevel@tonic-gate 	extern time_t	time;
13447c478bd9Sstevel@tonic-gate 
13457c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_init, action is %s", act_name(a)));
13467c478bd9Sstevel@tonic-gate 	ASSERT(s & UF_INIT);
13477c478bd9Sstevel@tonic-gate 
13487c478bd9Sstevel@tonic-gate 	switch (a) {
13497c478bd9Sstevel@tonic-gate 	case UFA_SET:
13507c478bd9Sstevel@tonic-gate 		f->uf_begin_tm = time;
13517c478bd9Sstevel@tonic-gate 		f->uf_retry = 1;
13527c478bd9Sstevel@tonic-gate 		if (!f->uf_ufsvfsp) {
13537c478bd9Sstevel@tonic-gate 			(void) set_state(f, UF_PANIC);
13547c478bd9Sstevel@tonic-gate 			TRIVIA((": NULL ufsvfsp]\n"));
13557c478bd9Sstevel@tonic-gate 			return (rc);
13567c478bd9Sstevel@tonic-gate 		}
13577c478bd9Sstevel@tonic-gate 		/*
13587c478bd9Sstevel@tonic-gate 		 * because we can call panic from many different levels,
13597c478bd9Sstevel@tonic-gate 		 * we can't be sure that we've got the vfs_lock at this
13607c478bd9Sstevel@tonic-gate 		 * point.  However, there's not much alternative and if
13617c478bd9Sstevel@tonic-gate 		 * we don't (have the lock) the worst case is we'll just
13627c478bd9Sstevel@tonic-gate 		 * panic again
13637c478bd9Sstevel@tonic-gate 		 */
13647c478bd9Sstevel@tonic-gate 		f->uf_vfs_lockp		= &f->uf_ufsvfsp->vfs_lock;
13657c478bd9Sstevel@tonic-gate 		f->uf_vfs_ufsfxp	= &f->uf_ufsvfsp->vfs_fsfx;
13667c478bd9Sstevel@tonic-gate 
13677c478bd9Sstevel@tonic-gate 		if (!f->uf_ufsvfsp->vfs_bufp) {
13687c478bd9Sstevel@tonic-gate 			(void) set_state(f, UF_PANIC);
13697c478bd9Sstevel@tonic-gate 			TRIVIA((": NULL vfs_bufp]\n"));
13707c478bd9Sstevel@tonic-gate 			return (rc);
13717c478bd9Sstevel@tonic-gate 		}
13727c478bd9Sstevel@tonic-gate 		f->uf_bp = f->uf_ufsvfsp->vfs_bufp;
13737c478bd9Sstevel@tonic-gate 
13747c478bd9Sstevel@tonic-gate 		if (!f->uf_ufsvfsp->vfs_bufp->b_un.b_fs) {
13757c478bd9Sstevel@tonic-gate 			(void) set_state(f, UF_PANIC);
13767c478bd9Sstevel@tonic-gate 			TRIVIA((": NULL vfs_fs]\n"));
13777c478bd9Sstevel@tonic-gate 			return (rc);
13787c478bd9Sstevel@tonic-gate 		}
13797c478bd9Sstevel@tonic-gate 
13807c478bd9Sstevel@tonic-gate 		/* vfs_fs = vfs_bufp->b_un.b_fs */
13817c478bd9Sstevel@tonic-gate 		bcopy(f->uf_ufsvfsp->vfs_fs->fs_fsmnt, f->uf_fsname, MAXMNTLEN);
13827c478bd9Sstevel@tonic-gate 
13837c478bd9Sstevel@tonic-gate 		f->uf_lf.lf_lock  = LOCKFS_ELOCK;	/* primer */
13847c478bd9Sstevel@tonic-gate 
13857c478bd9Sstevel@tonic-gate 		if (!f->uf_vfsp || f->uf_vfsp->vfs_dev == NODEV) {
13867c478bd9Sstevel@tonic-gate 			(void) set_state(f, UF_PANIC);
13877c478bd9Sstevel@tonic-gate 			TRIVIA((": NULL vfsp or vfs_dev == NODEV"));
13887c478bd9Sstevel@tonic-gate 			return (rc);
13897c478bd9Sstevel@tonic-gate 		}
13907c478bd9Sstevel@tonic-gate 		f->uf_dev = f->uf_vfsp->vfs_dev;
13917c478bd9Sstevel@tonic-gate 
13927c478bd9Sstevel@tonic-gate 		rc = SFRC_SUCCESS;
13937c478bd9Sstevel@tonic-gate 		break;
13947c478bd9Sstevel@tonic-gate 
13957c478bd9Sstevel@tonic-gate 	case UFA_FOUND:
13967c478bd9Sstevel@tonic-gate 	default:
13977c478bd9Sstevel@tonic-gate 		/* failures marked init shouldn't even be on the queue yet */
13987c478bd9Sstevel@tonic-gate 		rc = set_state(f, UF_QUEUE);
13997c478bd9Sstevel@tonic-gate 		TRIVIA((": found failure with state init]\n"));
14007c478bd9Sstevel@tonic-gate 	}
14017c478bd9Sstevel@tonic-gate 
14027c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
14037c478bd9Sstevel@tonic-gate 	return (rc);
14047c478bd9Sstevel@tonic-gate }
14057c478bd9Sstevel@tonic-gate 
14067c478bd9Sstevel@tonic-gate static sfrc_t
sf_queue(ufs_failure_t * f,ufsa_t a,ufs_failure_states_t s)14077c478bd9Sstevel@tonic-gate sf_queue(
14087c478bd9Sstevel@tonic-gate 	ufs_failure_t	*f,
14097c478bd9Sstevel@tonic-gate 	ufsa_t	 a,
14107c478bd9Sstevel@tonic-gate 	ufs_failure_states_t	 s)
14117c478bd9Sstevel@tonic-gate {
14127c478bd9Sstevel@tonic-gate 	sfrc_t		rc = SFRC_FAIL;
14137c478bd9Sstevel@tonic-gate 
14147c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_queue, action is %s", act_name(a)));
14157c478bd9Sstevel@tonic-gate 	ASSERT(s & UF_QUEUE);
14167c478bd9Sstevel@tonic-gate 
14177c478bd9Sstevel@tonic-gate 	if (!f->uf_ufsvfsp) {
14187c478bd9Sstevel@tonic-gate 		TRIVIA((": NULL ufsvfsp]\n"));
14197c478bd9Sstevel@tonic-gate 		return (rc);
14207c478bd9Sstevel@tonic-gate 	}
14217c478bd9Sstevel@tonic-gate 
14227c478bd9Sstevel@tonic-gate 	switch (a) {
14237c478bd9Sstevel@tonic-gate 	case UFA_FOUND:
14247c478bd9Sstevel@tonic-gate 		rc = sf_found_queue(f);
14257c478bd9Sstevel@tonic-gate 		break;
14267c478bd9Sstevel@tonic-gate 
14277c478bd9Sstevel@tonic-gate 	case UFA_SET:
14287c478bd9Sstevel@tonic-gate 
14297c478bd9Sstevel@tonic-gate 		ASSERT(MUTEX_HELD(&ufs_fix.uq_mutex));
14307c478bd9Sstevel@tonic-gate 
14317c478bd9Sstevel@tonic-gate 		mutex_enter(&uf_stats.ufst_mutex);
14327c478bd9Sstevel@tonic-gate 		uf_stats.ufst_num_failed++;
14337c478bd9Sstevel@tonic-gate 		mutex_exit(&uf_stats.ufst_mutex);
14347c478bd9Sstevel@tonic-gate 
14357c478bd9Sstevel@tonic-gate 		/*
14367c478bd9Sstevel@tonic-gate 		 * if can't get the vfs lock, just wait until
14377c478bd9Sstevel@tonic-gate 		 * UF_TRYLCK to set fx_current
14387c478bd9Sstevel@tonic-gate 		 */
14397c478bd9Sstevel@tonic-gate 		if (mutex_tryenter(f->uf_vfs_lockp)) {
14407c478bd9Sstevel@tonic-gate 			f->uf_vfs_ufsfxp->fx_current = f;
14417c478bd9Sstevel@tonic-gate 			mutex_exit(f->uf_vfs_lockp);
14427c478bd9Sstevel@tonic-gate 		} else {
14437c478bd9Sstevel@tonic-gate 			mutex_enter(&uf_stats.ufst_mutex);
14447c478bd9Sstevel@tonic-gate 			uf_stats.ufst_current_races++;
14457c478bd9Sstevel@tonic-gate 			mutex_exit(&uf_stats.ufst_mutex);
14467c478bd9Sstevel@tonic-gate 		}
14477c478bd9Sstevel@tonic-gate 
14487c478bd9Sstevel@tonic-gate 		f->uf_retry = 1;
14497c478bd9Sstevel@tonic-gate 		rc = SFRC_SUCCESS;
14507c478bd9Sstevel@tonic-gate 		TRIVIA(("] "));
14517c478bd9Sstevel@tonic-gate 		break;
14527c478bd9Sstevel@tonic-gate 
14537c478bd9Sstevel@tonic-gate 	default:
14547c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_PANIC);
14557c478bd9Sstevel@tonic-gate 		TRIVIA((": failed] "));
14567c478bd9Sstevel@tonic-gate 	}
14577c478bd9Sstevel@tonic-gate 
14587c478bd9Sstevel@tonic-gate 	return (rc);
14597c478bd9Sstevel@tonic-gate }
14607c478bd9Sstevel@tonic-gate 
14617c478bd9Sstevel@tonic-gate static sfrc_t
sf_found_queue(ufs_failure_t * f)14627c478bd9Sstevel@tonic-gate sf_found_queue(ufs_failure_t *f)
14637c478bd9Sstevel@tonic-gate {
14647c478bd9Sstevel@tonic-gate 	int		replica;
14657c478bd9Sstevel@tonic-gate 	sfrc_t		rc = SFRC_FAIL;
14667c478bd9Sstevel@tonic-gate 
14677c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_found_queue"));
14687c478bd9Sstevel@tonic-gate 
14697c478bd9Sstevel@tonic-gate 	/*
14707c478bd9Sstevel@tonic-gate 	 * don't need to check for null ufsvfsp because
14717c478bd9Sstevel@tonic-gate 	 * unmount must own list's ufs_fix.uq_mutex
14727c478bd9Sstevel@tonic-gate 	 * to mark it null and we own that lock since
14737c478bd9Sstevel@tonic-gate 	 * we got here.
14747c478bd9Sstevel@tonic-gate 	 */
14757c478bd9Sstevel@tonic-gate 
14767c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&ufs_fix.uq_mutex));
14777c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));
14787c478bd9Sstevel@tonic-gate 
14797c478bd9Sstevel@tonic-gate 	if (!mutex_tryenter(f->uf_vfs_lockp)) {
14807c478bd9Sstevel@tonic-gate 		TRIVIA((": tryenter(vfslockp) failed; retry]\n"));
14817c478bd9Sstevel@tonic-gate 		f->uf_retry = 1;
14827c478bd9Sstevel@tonic-gate 		return (rc);
14837c478bd9Sstevel@tonic-gate 	}
14847c478bd9Sstevel@tonic-gate 
14857c478bd9Sstevel@tonic-gate 	replica = f->uf_vfs_ufsfxp && f->uf_vfs_ufsfxp->fx_current != NULL &&
148680d34432Sfrankho 	    f->uf_vfs_ufsfxp->fx_current != f &&
148780d34432Sfrankho 	    !terminal_state(f->uf_vfs_ufsfxp->fx_current->uf_s);
14887c478bd9Sstevel@tonic-gate 
14897c478bd9Sstevel@tonic-gate 	/*
14907c478bd9Sstevel@tonic-gate 	 * copy general flags to this ufs_failure so we don't
14917c478bd9Sstevel@tonic-gate 	 * need to refer back to the ufsvfs, or, more importantly,
14927c478bd9Sstevel@tonic-gate 	 * don't need to keep acquiring (trying to acquire) vfs_lockp
14937c478bd9Sstevel@tonic-gate 	 *
14947c478bd9Sstevel@tonic-gate 	 * The most restrictive option wins:
14957c478bd9Sstevel@tonic-gate 	 *  panic > errlock only > errlock+unmount > repair
14967c478bd9Sstevel@tonic-gate 	 * XXX panic > elock > elock > elock+umount
14977c478bd9Sstevel@tonic-gate 	 */
14987c478bd9Sstevel@tonic-gate 	if (f->uf_vfs_ufsfxp->fx_flags & UFSFX_PANIC) {
14997c478bd9Sstevel@tonic-gate 		if (!set_state(f, UF_PANIC)) {
15007c478bd9Sstevel@tonic-gate 			TRIVIA((": marked panic but was queued?"));
15017c478bd9Sstevel@tonic-gate 			real_panic(f, " ");
15027c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
15037c478bd9Sstevel@tonic-gate 		}
15047c478bd9Sstevel@tonic-gate 		mutex_exit(f->uf_vfs_lockp);
15057c478bd9Sstevel@tonic-gate 		return (rc);
15067c478bd9Sstevel@tonic-gate 	}
15077c478bd9Sstevel@tonic-gate 	f->uf_flags = f->uf_vfs_ufsfxp->fx_flags;
15087c478bd9Sstevel@tonic-gate 
15097c478bd9Sstevel@tonic-gate 	if (replica) {
15107c478bd9Sstevel@tonic-gate 		if (!set_state(f, UF_REPLICA)) {
15117c478bd9Sstevel@tonic-gate 			f->uf_retry = 1;
15127c478bd9Sstevel@tonic-gate 			TRIVIA((": set to replica failed] "));
15137c478bd9Sstevel@tonic-gate 		} else {
15147c478bd9Sstevel@tonic-gate 			TRIVIA(("] "));
15157c478bd9Sstevel@tonic-gate 		}
15167c478bd9Sstevel@tonic-gate 		mutex_exit(f->uf_vfs_lockp);
15177c478bd9Sstevel@tonic-gate 		return (rc);
15187c478bd9Sstevel@tonic-gate 	}
15197c478bd9Sstevel@tonic-gate 	mutex_exit(f->uf_vfs_lockp);
15207c478bd9Sstevel@tonic-gate 
15217c478bd9Sstevel@tonic-gate 	if (!set_state(f, UF_TRYLCK)) {
15227c478bd9Sstevel@tonic-gate 		TRIVIA((": failed] "));
15237c478bd9Sstevel@tonic-gate 	} else {
15247c478bd9Sstevel@tonic-gate 		rc = SFRC_SUCCESS;
15257c478bd9Sstevel@tonic-gate 	}
15267c478bd9Sstevel@tonic-gate 	return (rc);
15277c478bd9Sstevel@tonic-gate }
15287c478bd9Sstevel@tonic-gate 
15297c478bd9Sstevel@tonic-gate static sfrc_t
sf_nonterm_cmn(ufs_failure_t * f,ufsa_t a,ufs_failure_states_t s)15307c478bd9Sstevel@tonic-gate sf_nonterm_cmn(ufs_failure_t *f, ufsa_t a, ufs_failure_states_t s)
15317c478bd9Sstevel@tonic-gate {
15327c478bd9Sstevel@tonic-gate 	sfrc_t	rc = SFRC_FAIL;
15337c478bd9Sstevel@tonic-gate 
15347c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_nonterm_cmn, action: %s, %s", act_name(a), state_name(s)));
15357c478bd9Sstevel@tonic-gate 	ASSERT(s & (UF_TRYLCK | UF_LOCKED | UF_UMOUNT | UF_FIXING));
15367c478bd9Sstevel@tonic-gate 	ASSERT(!terminal_state(s));
15377c478bd9Sstevel@tonic-gate 
15387c478bd9Sstevel@tonic-gate 	if (!f->uf_ufsvfsp && !(f->uf_s & UF_UMOUNT)) {
15397c478bd9Sstevel@tonic-gate 		TRIVIA((": NULL ufsvfsp (state != UMOUNT)]\n"));
15407c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_NOTFIX);
15417c478bd9Sstevel@tonic-gate 		return (rc);
15427c478bd9Sstevel@tonic-gate 	}
15437c478bd9Sstevel@tonic-gate 
15447c478bd9Sstevel@tonic-gate 	switch (a) {
15457c478bd9Sstevel@tonic-gate 	case UFA_SET:
15467c478bd9Sstevel@tonic-gate 		switch (s) {
15477c478bd9Sstevel@tonic-gate 		case UF_TRYLCK:
15487c478bd9Sstevel@tonic-gate 			ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));
15497c478bd9Sstevel@tonic-gate 			rc = sf_set_trylck(f);
15507c478bd9Sstevel@tonic-gate 			break;
15517c478bd9Sstevel@tonic-gate 
15527c478bd9Sstevel@tonic-gate 		case UF_LOCKED:
15537c478bd9Sstevel@tonic-gate 			rc = sf_set_locked(f);
15547c478bd9Sstevel@tonic-gate 			break;
15557c478bd9Sstevel@tonic-gate 
15567c478bd9Sstevel@tonic-gate 		case UF_FIXING:
15577c478bd9Sstevel@tonic-gate 			f->uf_flags |= UFSFX_REPAIR_START;
15587c478bd9Sstevel@tonic-gate 			f->uf_retry  = ufsfx_tune.uft_fixpoll_period;
15597c478bd9Sstevel@tonic-gate 			rc = SFRC_SUCCESS;
15607c478bd9Sstevel@tonic-gate 			break;
15617c478bd9Sstevel@tonic-gate 
15627c478bd9Sstevel@tonic-gate 		case UF_UMOUNT:
15637c478bd9Sstevel@tonic-gate 			f->uf_retry = -ufsfx_tune.uft_short_err_period;
15647c478bd9Sstevel@tonic-gate 			rc = SFRC_SUCCESS;
15657c478bd9Sstevel@tonic-gate 			break;
15667c478bd9Sstevel@tonic-gate 
15677c478bd9Sstevel@tonic-gate 		default:
15687c478bd9Sstevel@tonic-gate 			(void) set_state(f, UF_PANIC);
15697c478bd9Sstevel@tonic-gate 			TRIVIA((": failed] "));
15707c478bd9Sstevel@tonic-gate 		}
15717c478bd9Sstevel@tonic-gate 		break;
15727c478bd9Sstevel@tonic-gate 
15737c478bd9Sstevel@tonic-gate 	case UFA_FOUND:
15747c478bd9Sstevel@tonic-gate 
15757c478bd9Sstevel@tonic-gate 		switch (s) {
15767c478bd9Sstevel@tonic-gate 		case UF_TRYLCK:
15777c478bd9Sstevel@tonic-gate 			rc = sf_found_trylck(f);
15787c478bd9Sstevel@tonic-gate 			break;
15797c478bd9Sstevel@tonic-gate 
15807c478bd9Sstevel@tonic-gate 		case UF_LOCKED:
15817c478bd9Sstevel@tonic-gate 		case UF_FIXING:
15827c478bd9Sstevel@tonic-gate 			rc = sf_found_lock_fix_cmn(f, s);
15837c478bd9Sstevel@tonic-gate 			break;
15847c478bd9Sstevel@tonic-gate 
15857c478bd9Sstevel@tonic-gate 		case UF_UMOUNT:
15867c478bd9Sstevel@tonic-gate 			rc = sf_found_umount(f);
15877c478bd9Sstevel@tonic-gate 			break;
15887c478bd9Sstevel@tonic-gate 
15897c478bd9Sstevel@tonic-gate 		default:
15907c478bd9Sstevel@tonic-gate 			(void) set_state(f, UF_PANIC);
15917c478bd9Sstevel@tonic-gate 			TRIVIA((": failed] "));
15927c478bd9Sstevel@tonic-gate 			break;
15937c478bd9Sstevel@tonic-gate 		}
15947c478bd9Sstevel@tonic-gate 		break;
15957c478bd9Sstevel@tonic-gate 	default:
15967c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_PANIC);
15977c478bd9Sstevel@tonic-gate 		TRIVIA((": failed] "));
15987c478bd9Sstevel@tonic-gate 		break;
15997c478bd9Sstevel@tonic-gate 	}
16007c478bd9Sstevel@tonic-gate 
16017c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
16027c478bd9Sstevel@tonic-gate 	return (rc);
16037c478bd9Sstevel@tonic-gate }
16047c478bd9Sstevel@tonic-gate 
16057c478bd9Sstevel@tonic-gate static sfrc_t
sf_set_trylck(ufs_failure_t * f)16067c478bd9Sstevel@tonic-gate sf_set_trylck(ufs_failure_t *f)
16077c478bd9Sstevel@tonic-gate {
16087c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_set_trylck"));
16097c478bd9Sstevel@tonic-gate 
16107c478bd9Sstevel@tonic-gate 	if (!mutex_tryenter(f->uf_vfs_lockp)) {
16117c478bd9Sstevel@tonic-gate 		TRIVIA((": tryenter(vfslockp) failed; retry]\n"));
16127c478bd9Sstevel@tonic-gate 		f->uf_retry = 1;
16137c478bd9Sstevel@tonic-gate 		return (SFRC_FAIL);
16147c478bd9Sstevel@tonic-gate 	}
16157c478bd9Sstevel@tonic-gate 
16167c478bd9Sstevel@tonic-gate 	if (!f->uf_vfs_ufsfxp->fx_current)
16177c478bd9Sstevel@tonic-gate 		f->uf_vfs_ufsfxp->fx_current = f;
16187c478bd9Sstevel@tonic-gate 
16197c478bd9Sstevel@tonic-gate 	mutex_exit(f->uf_vfs_lockp);
16207c478bd9Sstevel@tonic-gate 
16217c478bd9Sstevel@tonic-gate 	f->uf_lf.lf_flags = 0;
16227c478bd9Sstevel@tonic-gate 	f->uf_lf.lf_lock  = LOCKFS_ELOCK;
16237c478bd9Sstevel@tonic-gate 	f->uf_retry = -ufsfx_tune.uft_fixstart_period;
16247c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
16257c478bd9Sstevel@tonic-gate 	return (SFRC_SUCCESS);
16267c478bd9Sstevel@tonic-gate }
16277c478bd9Sstevel@tonic-gate 
16287c478bd9Sstevel@tonic-gate static sfrc_t
sf_found_trylck(ufs_failure_t * f)16297c478bd9Sstevel@tonic-gate sf_found_trylck(ufs_failure_t *f)
16307c478bd9Sstevel@tonic-gate {
16317c478bd9Sstevel@tonic-gate 	struct lockfs lockfs_status;
16327c478bd9Sstevel@tonic-gate 
16337c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_found_trylck"));
16347c478bd9Sstevel@tonic-gate 
16357c478bd9Sstevel@tonic-gate 	if (trylock_time_exceeded(f) > 0) {
16367c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_PANIC);
16377c478bd9Sstevel@tonic-gate 		TRIVIA((": failed] "));
16387c478bd9Sstevel@tonic-gate 		return (SFRC_FAIL);
16397c478bd9Sstevel@tonic-gate 	}
16407c478bd9Sstevel@tonic-gate 
16417c478bd9Sstevel@tonic-gate 	if (!get_lockfs_status(f, &lockfs_status)) {
16427c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_PANIC);
16437c478bd9Sstevel@tonic-gate 		TRIVIA((": failed] "));
16447c478bd9Sstevel@tonic-gate 		return (SFRC_FAIL);
16457c478bd9Sstevel@tonic-gate 	}
16467c478bd9Sstevel@tonic-gate 
16477c478bd9Sstevel@tonic-gate 	if (f->uf_lf_err == NO_ERROR)
16487c478bd9Sstevel@tonic-gate 		f->uf_lf.lf_key = lockfs_status.lf_key;
16497c478bd9Sstevel@tonic-gate 
16507c478bd9Sstevel@tonic-gate 	if (!set_lockfs(f, &lockfs_status)) {
16517c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_PANIC);
16527c478bd9Sstevel@tonic-gate 		TRIVIA((": failed] "));
16537c478bd9Sstevel@tonic-gate 		return (SFRC_FAIL);
16547c478bd9Sstevel@tonic-gate 	}
16557c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
16567c478bd9Sstevel@tonic-gate 	return (SFRC_SUCCESS);
16577c478bd9Sstevel@tonic-gate }
16587c478bd9Sstevel@tonic-gate 
16597c478bd9Sstevel@tonic-gate static sfrc_t
sf_set_locked(ufs_failure_t * f)16607c478bd9Sstevel@tonic-gate sf_set_locked(ufs_failure_t *f)
16617c478bd9Sstevel@tonic-gate {
16627c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_set_locked"));
16637c478bd9Sstevel@tonic-gate 
16647c478bd9Sstevel@tonic-gate 	f->uf_retry = -ufsfx_tune.uft_fixstart_period;
16657c478bd9Sstevel@tonic-gate 
16667c478bd9Sstevel@tonic-gate #if defined(DEBUG)
16677c478bd9Sstevel@tonic-gate 	if (f->uf_flags & UFSFX_REPAIR_START)
16687c478bd9Sstevel@tonic-gate 		TRIVIA(("clearing UFSFX_REPAIR_START "));
16697c478bd9Sstevel@tonic-gate #endif /* DEBUG */
16707c478bd9Sstevel@tonic-gate 
16717c478bd9Sstevel@tonic-gate 	f->uf_flags &= ~UFSFX_REPAIR_START;
16727c478bd9Sstevel@tonic-gate 
16737c478bd9Sstevel@tonic-gate 	if (f->uf_s & UF_TRYLCK) {
16747c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "Error-locked %s: \"%s\"",
167580d34432Sfrankho 		    fs_name(f), f->uf_panic_str);
16767c478bd9Sstevel@tonic-gate 
16777c478bd9Sstevel@tonic-gate 		if (f->uf_flags & UFSFX_LCKONLY)
16787c478bd9Sstevel@tonic-gate 			cmn_err(CE_WARN, "Manual repair of %s required",
167980d34432Sfrankho 			    fs_name(f));
16807c478bd9Sstevel@tonic-gate 	}
16817c478bd9Sstevel@tonic-gate 
16827c478bd9Sstevel@tonic-gate 	/*
16837c478bd9Sstevel@tonic-gate 	 * just reset to current state
16847c478bd9Sstevel@tonic-gate 	 */
16857c478bd9Sstevel@tonic-gate #if defined(DEBUG)
16867c478bd9Sstevel@tonic-gate 	TRIVIA(("locked->locked "));
16877c478bd9Sstevel@tonic-gate #endif /* DEBUG */
16887c478bd9Sstevel@tonic-gate 
16897c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
16907c478bd9Sstevel@tonic-gate 	return (SFRC_SUCCESS);
16917c478bd9Sstevel@tonic-gate }
16927c478bd9Sstevel@tonic-gate 
16937c478bd9Sstevel@tonic-gate static sfrc_t
sf_found_lock_fix_cmn(ufs_failure_t * f,ufs_failure_states_t s)16947c478bd9Sstevel@tonic-gate sf_found_lock_fix_cmn(ufs_failure_t *f, ufs_failure_states_t s)
16957c478bd9Sstevel@tonic-gate {
16967c478bd9Sstevel@tonic-gate 	time_t		toolong;
16977c478bd9Sstevel@tonic-gate 	extern time_t	time;
16987c478bd9Sstevel@tonic-gate 	struct buf	*bp			= NULL;
16997c478bd9Sstevel@tonic-gate 	struct fs	*dfs;
17007c478bd9Sstevel@tonic-gate 	time_t		 concerned, anxious;
17017c478bd9Sstevel@tonic-gate 	sfrc_t		 rc			= SFRC_FAIL;
17027c478bd9Sstevel@tonic-gate 	ulong_t		 gb_size;
17037c478bd9Sstevel@tonic-gate 
17047c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_found_lock_fix_cmn (\"%s\")", state_name(s)));
17057c478bd9Sstevel@tonic-gate 
17067c478bd9Sstevel@tonic-gate 	if (s & UF_LOCKED) {
17077c478bd9Sstevel@tonic-gate 		ASSERT(MUTEX_HELD(&f->uf_mutex));
17087c478bd9Sstevel@tonic-gate 
170980d34432Sfrankho 		toolong =
171080d34432Sfrankho 		    time > (ufsfx_tune.uft_too_long + f->uf_entered_tm);
17117c478bd9Sstevel@tonic-gate 		TRIVIA(("%stoolong", !toolong? "not": ""));
17127c478bd9Sstevel@tonic-gate 		HIDEOUS((": time:%ld, too long:%ld, entered_tm:%ld ",
17137c478bd9Sstevel@tonic-gate 		    time, ufsfx_tune.uft_too_long, f->uf_entered_tm));
17147c478bd9Sstevel@tonic-gate 
17157c478bd9Sstevel@tonic-gate 		if (f->uf_flags & UFSFX_LCKUMOUNT) {
17167c478bd9Sstevel@tonic-gate 			if (set_state(f, UF_UMOUNT)) {
17177c478bd9Sstevel@tonic-gate 				TRIVIA(("] "));
17187c478bd9Sstevel@tonic-gate 				rc = SFRC_SUCCESS;
17197c478bd9Sstevel@tonic-gate 			} else {
17207c478bd9Sstevel@tonic-gate 				TRIVIA((": failed] "));
17217c478bd9Sstevel@tonic-gate 				f->uf_retry = 1;
17227c478bd9Sstevel@tonic-gate 			}
17237c478bd9Sstevel@tonic-gate 			return (rc);
17247c478bd9Sstevel@tonic-gate 		}
17257c478bd9Sstevel@tonic-gate 		if (!toolong) {
17267c478bd9Sstevel@tonic-gate 			rc = SFRC_SUCCESS;
17277c478bd9Sstevel@tonic-gate 		} else {
17287c478bd9Sstevel@tonic-gate 			if (!(f->uf_flags & UFSFX_REPAIR_START)) {
17297c478bd9Sstevel@tonic-gate 				cmn_err(CE_WARN, "%s repair of %s not started.",
173080d34432Sfrankho 				    (f->uf_flags & UFSFX_LCKONLY) ?
173180d34432Sfrankho 				    "Manual" : "Automatic", fs_name(f));
17327c478bd9Sstevel@tonic-gate 
17337c478bd9Sstevel@tonic-gate 				f->uf_retry = ufsfx_tune.uft_long_err_period;
17347c478bd9Sstevel@tonic-gate 			} else {
17357c478bd9Sstevel@tonic-gate 				f->uf_retry = ufsfx_tune.uft_long_err_period;
173680d34432Sfrankho 				cmn_err(CE_WARN, "Repair of %s is not timely; "
173780d34432Sfrankho 				    "operator attention is required.",
173880d34432Sfrankho 				    fs_name(f));
17397c478bd9Sstevel@tonic-gate 			}
17407c478bd9Sstevel@tonic-gate 			TRIVIA(("] "));
17417c478bd9Sstevel@tonic-gate 			return (rc);
17427c478bd9Sstevel@tonic-gate 		}
17437c478bd9Sstevel@tonic-gate 	}
17447c478bd9Sstevel@tonic-gate 
17457c478bd9Sstevel@tonic-gate #if defined(DEBUG)
17467c478bd9Sstevel@tonic-gate 	else {
17477c478bd9Sstevel@tonic-gate 		ASSERT(s & UF_FIXING);
17487c478bd9Sstevel@tonic-gate 	}
17497c478bd9Sstevel@tonic-gate #endif /* DEBUG */
17507c478bd9Sstevel@tonic-gate 
17517c478bd9Sstevel@tonic-gate 	/*
17527c478bd9Sstevel@tonic-gate 	 * get on disk superblock; force it to really
17537c478bd9Sstevel@tonic-gate 	 * come from the disk
17547c478bd9Sstevel@tonic-gate 	 */
17557c478bd9Sstevel@tonic-gate 	(void) bfinval(f->uf_dev, 0);
17567c478bd9Sstevel@tonic-gate 	bp = UFS_BREAD(f->uf_ufsvfsp, f->uf_dev, SBLOCK, SBSIZE);
17577c478bd9Sstevel@tonic-gate 	if (bp) {
17587c478bd9Sstevel@tonic-gate 		bp->b_flags |= (B_STALE | B_AGE);
17597c478bd9Sstevel@tonic-gate 		dfs = bp->b_un.b_fs;
17607c478bd9Sstevel@tonic-gate 	}
17617c478bd9Sstevel@tonic-gate 
17627c478bd9Sstevel@tonic-gate 	if (!bp || (bp->b_flags & B_ERROR) || ((dfs->fs_magic != FS_MAGIC) &&
17637c478bd9Sstevel@tonic-gate 	    (dfs->fs_magic != MTB_UFS_MAGIC))) {
17647c478bd9Sstevel@tonic-gate 		TRIVIA((": UFS_BREAD(SBLOCK) failed]\n"));
17657c478bd9Sstevel@tonic-gate 		f->uf_retry = 1;
17667c478bd9Sstevel@tonic-gate 		goto out;
17677c478bd9Sstevel@tonic-gate 	}
17687c478bd9Sstevel@tonic-gate 
17697c478bd9Sstevel@tonic-gate 	/* fsck started but we haven't noticed yet? */
17707c478bd9Sstevel@tonic-gate 	if (!(s & UF_FIXING) && dfs->fs_clean == FSFIX) {
17717c478bd9Sstevel@tonic-gate 		if (!set_state(f, UF_FIXING)) {
17727c478bd9Sstevel@tonic-gate 			TRIVIA((": failed]\n"));
17737c478bd9Sstevel@tonic-gate 			f->uf_retry = 1;
17747c478bd9Sstevel@tonic-gate 			goto out;
17757c478bd9Sstevel@tonic-gate 		}
17767c478bd9Sstevel@tonic-gate 	}
17777c478bd9Sstevel@tonic-gate 
17787c478bd9Sstevel@tonic-gate 	/* fsck started but didn't succeed? */
17797c478bd9Sstevel@tonic-gate 	if ((s & UF_FIXING) && ((dfs->fs_clean == FSBAD) || !fsck_active(f))) {
17807c478bd9Sstevel@tonic-gate 		TRIVIA((": fs_clean: %d", (int)dfs->fs_clean));
17817c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_LOCKED);
17827c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "%s: Manual repair is necessary.", fs_name(f));
17837c478bd9Sstevel@tonic-gate 		f->uf_retry = ufsfx_tune.uft_long_err_period;
17847c478bd9Sstevel@tonic-gate 		goto out;
17857c478bd9Sstevel@tonic-gate 	}
17867c478bd9Sstevel@tonic-gate 
17877c478bd9Sstevel@tonic-gate 	gb_size = (dfs->fs_size * dfs->fs_bshift) / GB;
17887c478bd9Sstevel@tonic-gate 	toolong = (time_t)((gb_size == 0? 1: gb_size) * SecondsPerGig);
17897c478bd9Sstevel@tonic-gate 
17907c478bd9Sstevel@tonic-gate 	/* fsck started but doesn't seem to be proceeding? */
17917c478bd9Sstevel@tonic-gate 	if ((s & UF_FIXING) && dfs->fs_clean == FSFIX) {
17927c478bd9Sstevel@tonic-gate 		if (time > f->uf_entered_tm + toolong) {
17937c478bd9Sstevel@tonic-gate 
17947c478bd9Sstevel@tonic-gate 			cmn_err(CE_WARN,
179580d34432Sfrankho 			    "Repair completion timeout exceeded on %s; "
179680d34432Sfrankho 			    "manual fsck may be required", fs_name(f));
17977c478bd9Sstevel@tonic-gate 			f->uf_retry = ufsfx_tune.uft_long_err_period;
17987c478bd9Sstevel@tonic-gate 		}
17997c478bd9Sstevel@tonic-gate 	}
18007c478bd9Sstevel@tonic-gate 
18017c478bd9Sstevel@tonic-gate 	concerned = f->uf_entered_tm + (toolong / 3);
18027c478bd9Sstevel@tonic-gate 	anxious = f->uf_entered_tm + ((2 * toolong) / 3);
18037c478bd9Sstevel@tonic-gate 
18047c478bd9Sstevel@tonic-gate 	if (time > concerned)
18057c478bd9Sstevel@tonic-gate 		pester_msg(f, time > anxious? CE_WARN: CE_NOTE);
18067c478bd9Sstevel@tonic-gate 
18077c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
18087c478bd9Sstevel@tonic-gate 
18097c478bd9Sstevel@tonic-gate out:
18107c478bd9Sstevel@tonic-gate 	if (bp)
18117c478bd9Sstevel@tonic-gate 		brelse(bp);
18127c478bd9Sstevel@tonic-gate 
18137c478bd9Sstevel@tonic-gate 	return (rc);
18147c478bd9Sstevel@tonic-gate }
18157c478bd9Sstevel@tonic-gate 
18167c478bd9Sstevel@tonic-gate static sfrc_t
sf_found_umount(ufs_failure_t * f)18177c478bd9Sstevel@tonic-gate sf_found_umount(ufs_failure_t *f)
18187c478bd9Sstevel@tonic-gate {
18197c478bd9Sstevel@tonic-gate 	extern time_t	 time;
18207c478bd9Sstevel@tonic-gate 	sfrc_t		 rc			= SFRC_FAIL;
18217c478bd9Sstevel@tonic-gate 	struct vfs	*vfsp			= f->uf_vfsp;
18227c478bd9Sstevel@tonic-gate 	struct ufsvfs	*ufsvfsp		= f->uf_ufsvfsp;
18237c478bd9Sstevel@tonic-gate 	int		 toolong		= 0;
18247c478bd9Sstevel@tonic-gate 	int		 err			= 0;
18257c478bd9Sstevel@tonic-gate 
18267c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_found_umount"));
18277c478bd9Sstevel@tonic-gate 
18287c478bd9Sstevel@tonic-gate 	toolong = time > ufsfx_tune.uft_too_long + f->uf_entered_tm;
18297c478bd9Sstevel@tonic-gate 	if (toolong) {
18307c478bd9Sstevel@tonic-gate 		TRIVIA((": unmount time limit exceeded] "));
18317c478bd9Sstevel@tonic-gate 		goto out;
18327c478bd9Sstevel@tonic-gate 	}
18337c478bd9Sstevel@tonic-gate 
18347c478bd9Sstevel@tonic-gate 	if (!vfsp || !ufsvfsp) {	/* trivial case */
18357c478bd9Sstevel@tonic-gate 		TRIVIA((": NULL vfsp and/or ufsvfsp, already unmounted?] "));
18367c478bd9Sstevel@tonic-gate 		goto out;
18377c478bd9Sstevel@tonic-gate 	}
18387c478bd9Sstevel@tonic-gate 
18397c478bd9Sstevel@tonic-gate 	if (!ULOCKFS_IS_ELOCK(&ufsvfsp->vfs_ulockfs)) {
18407c478bd9Sstevel@tonic-gate 		TRIVIA((": !not error locked?"));
18417c478bd9Sstevel@tonic-gate 		err = EINVAL;
18427c478bd9Sstevel@tonic-gate 		goto out;
18437c478bd9Sstevel@tonic-gate 	}
18447c478bd9Sstevel@tonic-gate 
18457c478bd9Sstevel@tonic-gate 	/* The vn_vfsunlock will be done in dounmount() [.../common/fs/vfs.c] */
1846d5dbd18dSbatschul 	if (vn_vfswlock(vfsp->vfs_vnodecovered)) {
18477c478bd9Sstevel@tonic-gate 		TRIVIA((": couldn't lock coveredvp"));
18487c478bd9Sstevel@tonic-gate 		err = EBUSY;
18497c478bd9Sstevel@tonic-gate 		goto out;
18507c478bd9Sstevel@tonic-gate 	}
18517c478bd9Sstevel@tonic-gate 
18527c478bd9Sstevel@tonic-gate 	if ((err = dounmount(vfsp, 0, kcred)) != 0) {
18537c478bd9Sstevel@tonic-gate 
18547c478bd9Sstevel@tonic-gate 		/* take note, but not many alternatives here */
18557c478bd9Sstevel@tonic-gate 		mutex_enter(&uf_stats.ufst_mutex);
18567c478bd9Sstevel@tonic-gate 		uf_stats.ufst_unmount_failures++;
18577c478bd9Sstevel@tonic-gate 		mutex_exit(&uf_stats.ufst_mutex);
18587c478bd9Sstevel@tonic-gate 
18597c478bd9Sstevel@tonic-gate 		TRIVIA((": unmount failed] "));
18607c478bd9Sstevel@tonic-gate 	} else {
18617c478bd9Sstevel@tonic-gate 		cmn_err(CE_NOTE, "unmounted error-locked %s", fs_name(f));
18627c478bd9Sstevel@tonic-gate 	}
18637c478bd9Sstevel@tonic-gate 
18647c478bd9Sstevel@tonic-gate out:
18657c478bd9Sstevel@tonic-gate 	if (toolong || (err != EBUSY && err != EAGAIN))
18667c478bd9Sstevel@tonic-gate 		rc = set_state(f, UF_NOTFIX);
18677c478bd9Sstevel@tonic-gate 
18687c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
18697c478bd9Sstevel@tonic-gate 	return (rc);
18707c478bd9Sstevel@tonic-gate }
18717c478bd9Sstevel@tonic-gate 
18727c478bd9Sstevel@tonic-gate static sfrc_t
sf_term_cmn(ufs_failure_t * f,ufsa_t a,ufs_failure_states_t s)18737c478bd9Sstevel@tonic-gate sf_term_cmn(ufs_failure_t *f, ufsa_t a, ufs_failure_states_t s)
18747c478bd9Sstevel@tonic-gate {
18757c478bd9Sstevel@tonic-gate 	extern time_t	time;
18767c478bd9Sstevel@tonic-gate 	sfrc_t		rc = SFRC_FAIL;
18777c478bd9Sstevel@tonic-gate 
18787c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_term_cmn, action is %s, state is %s",
187980d34432Sfrankho 	    act_name(a), state_name(s)));
18807c478bd9Sstevel@tonic-gate 	ASSERT(s & (UF_FIXED | UF_NOTFIX | UF_REPLICA));
18817c478bd9Sstevel@tonic-gate 	ASSERT(terminal_state(s));
18827c478bd9Sstevel@tonic-gate 
18837c478bd9Sstevel@tonic-gate 	if (!f->uf_ufsvfsp && !(f->uf_s & (UF_UMOUNT | UF_NOTFIX))) {
18847c478bd9Sstevel@tonic-gate 		TRIVIA((": NULL ufsvfsp (state != UMOUNT | NOTFIX)]\n"));
18857c478bd9Sstevel@tonic-gate 		return (rc);
18867c478bd9Sstevel@tonic-gate 	}
18877c478bd9Sstevel@tonic-gate 
18887c478bd9Sstevel@tonic-gate 	switch (a) {
18897c478bd9Sstevel@tonic-gate 	case UFA_SET:
18907c478bd9Sstevel@tonic-gate 		switch (s) {
18917c478bd9Sstevel@tonic-gate 		case UF_NOTFIX:
18927c478bd9Sstevel@tonic-gate 		case UF_FIXED:
189380d34432Sfrankho 		{
189480d34432Sfrankho 			int need_lock_vfs;
18957c478bd9Sstevel@tonic-gate 
18967c478bd9Sstevel@tonic-gate 			if (f->uf_ufsvfsp && f->uf_vfs_lockp)
18977c478bd9Sstevel@tonic-gate 				need_lock_vfs = !MUTEX_HELD(f->uf_vfs_lockp);
18987c478bd9Sstevel@tonic-gate 			else
18997c478bd9Sstevel@tonic-gate 				need_lock_vfs = 0;
19007c478bd9Sstevel@tonic-gate 
19017c478bd9Sstevel@tonic-gate 			if (need_lock_vfs && !mutex_tryenter(f->uf_vfs_lockp)) {
19027c478bd9Sstevel@tonic-gate 				TRIVIA((": tryenter(vfslockp) fail; retry]\n"));
19037c478bd9Sstevel@tonic-gate 				f->uf_retry = 1;
19047c478bd9Sstevel@tonic-gate 				break;
19057c478bd9Sstevel@tonic-gate 			}
19067c478bd9Sstevel@tonic-gate 
19077c478bd9Sstevel@tonic-gate 			f->uf_end_tm = time;
19087c478bd9Sstevel@tonic-gate 			f->uf_lf.lf_lock = LOCKFS_OLOCK;
19097c478bd9Sstevel@tonic-gate 			f->uf_retry = 0;
19107c478bd9Sstevel@tonic-gate 
19117c478bd9Sstevel@tonic-gate 			if (f->uf_vfs_ufsfxp)
19127c478bd9Sstevel@tonic-gate 				f->uf_vfs_ufsfxp->fx_current = NULL;
19137c478bd9Sstevel@tonic-gate 
19147c478bd9Sstevel@tonic-gate 			if (need_lock_vfs)
19157c478bd9Sstevel@tonic-gate 				mutex_exit(f->uf_vfs_lockp);
19167c478bd9Sstevel@tonic-gate 
19177c478bd9Sstevel@tonic-gate 			cmn_err(CE_NOTE, (s & UF_NOTFIX)? "Could not fix %s":
191880d34432Sfrankho 			    "%s is now accessible", fs_name(f));
19197c478bd9Sstevel@tonic-gate 
19207c478bd9Sstevel@tonic-gate 			if (s & UF_FIXED) {
19217c478bd9Sstevel@tonic-gate 				mutex_enter(&uf_stats.ufst_mutex);
19227c478bd9Sstevel@tonic-gate 				uf_stats.ufst_num_fixed++;
19237c478bd9Sstevel@tonic-gate 				mutex_exit(&uf_stats.ufst_mutex);
19247c478bd9Sstevel@tonic-gate 			}
19257c478bd9Sstevel@tonic-gate 			(void) timeout(ufsfx_kill_fix_failure_thread,
19267c478bd9Sstevel@tonic-gate 			    (void *)(ufsfx_tune.uft_short_err_period * hz),
19277c478bd9Sstevel@tonic-gate 			    ufsfx_tune.uft_short_err_period * hz);
19287c478bd9Sstevel@tonic-gate 			rc = SFRC_SUCCESS;
19297c478bd9Sstevel@tonic-gate 			break;
19307c478bd9Sstevel@tonic-gate 		}
19317c478bd9Sstevel@tonic-gate 		case UF_REPLICA:
19327c478bd9Sstevel@tonic-gate 
19337c478bd9Sstevel@tonic-gate 			ASSERT(MUTEX_HELD(f->uf_vfs_lockp));
19347c478bd9Sstevel@tonic-gate 
19357c478bd9Sstevel@tonic-gate 			/* not actually a replica? */
19367c478bd9Sstevel@tonic-gate 			if (f->uf_vfs_ufsfxp && f->uf_vfs_ufsfxp->fx_current &&
19377c478bd9Sstevel@tonic-gate 			    f->uf_vfs_ufsfxp->fx_current != f &&
193880d34432Sfrankho 			    !terminal_state(
193980d34432Sfrankho 			    f->uf_vfs_ufsfxp->fx_current->uf_s)) {
19407c478bd9Sstevel@tonic-gate 
19417c478bd9Sstevel@tonic-gate 				f->uf_orig = f->uf_vfs_ufsfxp->fx_current;
19427c478bd9Sstevel@tonic-gate 				f->uf_retry = 0;
19437c478bd9Sstevel@tonic-gate 				rc = SFRC_SUCCESS;
19447c478bd9Sstevel@tonic-gate 			} else {
19457c478bd9Sstevel@tonic-gate 				TRIVIA((": NULL fx_current]\n"));
19467c478bd9Sstevel@tonic-gate 				f->uf_retry = 1;
19477c478bd9Sstevel@tonic-gate 			}
19487c478bd9Sstevel@tonic-gate 
19497c478bd9Sstevel@tonic-gate 			break;
19507c478bd9Sstevel@tonic-gate 
19517c478bd9Sstevel@tonic-gate 		default:
19527c478bd9Sstevel@tonic-gate 			rc = set_state(f, UF_PANIC);
19537c478bd9Sstevel@tonic-gate 			TRIVIA((": failed] "));
19547c478bd9Sstevel@tonic-gate 			break;
19557c478bd9Sstevel@tonic-gate 		}
19567c478bd9Sstevel@tonic-gate 		break;
19577c478bd9Sstevel@tonic-gate 
19587c478bd9Sstevel@tonic-gate 	case UFA_FOUND:
19597c478bd9Sstevel@tonic-gate 		/*
19607c478bd9Sstevel@tonic-gate 		 * XXX de-allocate these after some period?
19617c478bd9Sstevel@tonic-gate 		 * XXX or move to an historical list?
19627c478bd9Sstevel@tonic-gate 		 * XXX or have an ioctl which reaps them?
19637c478bd9Sstevel@tonic-gate 		 */
19647c478bd9Sstevel@tonic-gate 		/*
19657c478bd9Sstevel@tonic-gate 		 * For now, since we don't expect lots of failures
19667c478bd9Sstevel@tonic-gate 		 * to occur (to the point of memory shortages),
19677c478bd9Sstevel@tonic-gate 		 * just punt
19687c478bd9Sstevel@tonic-gate 		 */
19697c478bd9Sstevel@tonic-gate 
19707c478bd9Sstevel@tonic-gate 		/* be sure we're not wasting cpu on old failures */
19717c478bd9Sstevel@tonic-gate 		if (f->uf_retry != 0) {
19727c478bd9Sstevel@tonic-gate 			mutex_enter(&uf_stats.ufst_mutex);
19737c478bd9Sstevel@tonic-gate 			uf_stats.ufst_cpu_waste++;
19747c478bd9Sstevel@tonic-gate 			mutex_exit(&uf_stats.ufst_mutex);
19757c478bd9Sstevel@tonic-gate 			f->uf_retry = 0;
19767c478bd9Sstevel@tonic-gate 		}
19777c478bd9Sstevel@tonic-gate 		rc = SFRC_SUCCESS;
19787c478bd9Sstevel@tonic-gate 		break;
19797c478bd9Sstevel@tonic-gate 
19807c478bd9Sstevel@tonic-gate 	default:
19817c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_PANIC);
19827c478bd9Sstevel@tonic-gate 		TRIVIA((": failed] "));
19837c478bd9Sstevel@tonic-gate 		break;
19847c478bd9Sstevel@tonic-gate 	}
19857c478bd9Sstevel@tonic-gate 
19867c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
19877c478bd9Sstevel@tonic-gate 	return (rc);
19887c478bd9Sstevel@tonic-gate }
19897c478bd9Sstevel@tonic-gate 
19907c478bd9Sstevel@tonic-gate static sfrc_t
sf_panic(ufs_failure_t * f,ufsa_t a,ufs_failure_states_t s)19917c478bd9Sstevel@tonic-gate sf_panic(
19927c478bd9Sstevel@tonic-gate 	ufs_failure_t	*f,
19937c478bd9Sstevel@tonic-gate 	ufsa_t	 a,
19947c478bd9Sstevel@tonic-gate 	ufs_failure_states_t	 s)
19957c478bd9Sstevel@tonic-gate {
19967c478bd9Sstevel@tonic-gate 	sfrc_t	rc = SFRC_FAIL;
19977c478bd9Sstevel@tonic-gate 
19987c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_panic, action is %s, prev. state is %s",
199980d34432Sfrankho 	    act_name(a), state_name(f->uf_s)));
20007c478bd9Sstevel@tonic-gate 	ASSERT(s & UF_PANIC);
20017c478bd9Sstevel@tonic-gate 
20027c478bd9Sstevel@tonic-gate 	switch (a) {
20037c478bd9Sstevel@tonic-gate 	case UFA_SET:
20047c478bd9Sstevel@tonic-gate 		f->uf_retry = -ufsfx_tune.uft_short_err_period;
20057c478bd9Sstevel@tonic-gate 		rc = SFRC_SUCCESS;
20067c478bd9Sstevel@tonic-gate 		break;
20077c478bd9Sstevel@tonic-gate 
20087c478bd9Sstevel@tonic-gate 	case UFA_FOUND:
20097c478bd9Sstevel@tonic-gate 	default:
20107c478bd9Sstevel@tonic-gate 		real_panic(f, " ");
20117c478bd9Sstevel@tonic-gate 
20127c478bd9Sstevel@tonic-gate 		/* LINTED: warning: logical expression always true: op "||" */
20137c478bd9Sstevel@tonic-gate 		ASSERT(DEBUG);
20147c478bd9Sstevel@tonic-gate 
20157c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_UMOUNT);	/* XXX UF_NOTFIX? */
20167c478bd9Sstevel@tonic-gate 
20177c478bd9Sstevel@tonic-gate 		break;
20187c478bd9Sstevel@tonic-gate 	}
20197c478bd9Sstevel@tonic-gate 
20207c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
20217c478bd9Sstevel@tonic-gate 	return (rc);
20227c478bd9Sstevel@tonic-gate }
20237c478bd9Sstevel@tonic-gate 
20247c478bd9Sstevel@tonic-gate /*
20257c478bd9Sstevel@tonic-gate  * minimum state function
20267c478bd9Sstevel@tonic-gate  */
20277c478bd9Sstevel@tonic-gate static sfrc_t
sf_minimum(ufs_failure_t * f,ufsa_t a,ufs_failure_states_t ignored)20287c478bd9Sstevel@tonic-gate sf_minimum(
20297c478bd9Sstevel@tonic-gate 	ufs_failure_t	*f,
20307c478bd9Sstevel@tonic-gate 	ufsa_t	 a, /* LINTED argument unused in function: ignored */
20317c478bd9Sstevel@tonic-gate 	ufs_failure_states_t	 ignored)
20327c478bd9Sstevel@tonic-gate {
20337c478bd9Sstevel@tonic-gate 	sfrc_t rc = SFRC_FAIL;
20347c478bd9Sstevel@tonic-gate 
20357c478bd9Sstevel@tonic-gate 	TRIVIA(("[sf_minimum, action is %s", act_name(a)));
20367c478bd9Sstevel@tonic-gate 
20377c478bd9Sstevel@tonic-gate 	switch (a) {
20387c478bd9Sstevel@tonic-gate 	case UFA_SET:
20397c478bd9Sstevel@tonic-gate 		f->uf_retry = 0;
20407c478bd9Sstevel@tonic-gate 		/* FALLTHROUGH */
20417c478bd9Sstevel@tonic-gate 
20427c478bd9Sstevel@tonic-gate 	case UFA_FOUND:
20437c478bd9Sstevel@tonic-gate 		rc = SFRC_SUCCESS;
20447c478bd9Sstevel@tonic-gate 		break;
20457c478bd9Sstevel@tonic-gate 
20467c478bd9Sstevel@tonic-gate 	default:
20477c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_PANIC);
20487c478bd9Sstevel@tonic-gate 		TRIVIA((": failed] "));
20497c478bd9Sstevel@tonic-gate 		break;
20507c478bd9Sstevel@tonic-gate 	}
20517c478bd9Sstevel@tonic-gate 
20527c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
20537c478bd9Sstevel@tonic-gate 	return (rc);
20547c478bd9Sstevel@tonic-gate }
20557c478bd9Sstevel@tonic-gate 
20567c478bd9Sstevel@tonic-gate static int
state_trans_valid(ufs_failure_states_t from,ufs_failure_states_t to)20577c478bd9Sstevel@tonic-gate state_trans_valid(ufs_failure_states_t from, ufs_failure_states_t to)
20587c478bd9Sstevel@tonic-gate {
20597c478bd9Sstevel@tonic-gate 	ufsd_t	*s;
20607c478bd9Sstevel@tonic-gate 	int	 valid;
20617c478bd9Sstevel@tonic-gate 
20627c478bd9Sstevel@tonic-gate 	HIDEOUS(("[state_trans_valid"));
20637c478bd9Sstevel@tonic-gate 
20647c478bd9Sstevel@tonic-gate 	if (from & to)
20657c478bd9Sstevel@tonic-gate 		return (1);
20667c478bd9Sstevel@tonic-gate 
20677c478bd9Sstevel@tonic-gate 	s = get_state_desc(to);
20687c478bd9Sstevel@tonic-gate 
20697c478bd9Sstevel@tonic-gate 	/*
20707c478bd9Sstevel@tonic-gate 	 * extra test is necessary since we want UF_UNDEF = 0,
20717c478bd9Sstevel@tonic-gate 	 * (to detect freshly allocated memory)
20727c478bd9Sstevel@tonic-gate 	 * but can't check for that value with a bit test
20737c478bd9Sstevel@tonic-gate 	 */
20747c478bd9Sstevel@tonic-gate 	valid = (to & UF_INIT)? from == s->ud_prev: from & s->ud_prev;
20757c478bd9Sstevel@tonic-gate 
20767c478bd9Sstevel@tonic-gate 	HIDEOUS((": %svalid] ", valid? "": "in"));
20777c478bd9Sstevel@tonic-gate 	return (valid);
20787c478bd9Sstevel@tonic-gate }
20797c478bd9Sstevel@tonic-gate 
20807c478bd9Sstevel@tonic-gate static int
terminal_state(ufs_failure_states_t state)20817c478bd9Sstevel@tonic-gate terminal_state(ufs_failure_states_t state)
20827c478bd9Sstevel@tonic-gate {
20837c478bd9Sstevel@tonic-gate 	ufsd_t	*s;
20847c478bd9Sstevel@tonic-gate 
20857c478bd9Sstevel@tonic-gate 	HIDEOUS(("[terminal_state"));
20867c478bd9Sstevel@tonic-gate 
20877c478bd9Sstevel@tonic-gate 	s = get_state_desc(state);
20887c478bd9Sstevel@tonic-gate 
20897c478bd9Sstevel@tonic-gate 	HIDEOUS((": %sterminal] ", s->ud_attr.terminal? "": "not "));
20907c478bd9Sstevel@tonic-gate 	return ((int)s->ud_attr.terminal);
20917c478bd9Sstevel@tonic-gate }
20927c478bd9Sstevel@tonic-gate 
20937c478bd9Sstevel@tonic-gate static void
alloc_lockfs_comment(ufs_failure_t * f,struct lockfs * lfp)20947c478bd9Sstevel@tonic-gate alloc_lockfs_comment(ufs_failure_t *f, struct lockfs *lfp)
20957c478bd9Sstevel@tonic-gate {
20967c478bd9Sstevel@tonic-gate 	MINUTE(("[alloc_lockfs_comment"));
20977c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&f->uf_mutex));
20987c478bd9Sstevel@tonic-gate 
20997c478bd9Sstevel@tonic-gate 	/*
21007c478bd9Sstevel@tonic-gate 	 * ufs_fiolfs expects a kmem_alloc'ed comment;
21017c478bd9Sstevel@tonic-gate 	 * it frees the comment if the lock fails
21027c478bd9Sstevel@tonic-gate 	 * or else when the lock is unlocked.
21037c478bd9Sstevel@tonic-gate 	 */
21047c478bd9Sstevel@tonic-gate 
21057c478bd9Sstevel@tonic-gate 	f->uf_lf.lf_comment = kmem_zalloc(LOCKFS_MAXCOMMENTLEN, KM_NOSLEEP);
21067c478bd9Sstevel@tonic-gate 	if (f->uf_lf.lf_comment) {
21077c478bd9Sstevel@tonic-gate 		char	*from;
21087c478bd9Sstevel@tonic-gate 		size_t	 len;
21097c478bd9Sstevel@tonic-gate 
21107c478bd9Sstevel@tonic-gate 		/*
21117c478bd9Sstevel@tonic-gate 		 * use panic string if there's no previous comment
21127c478bd9Sstevel@tonic-gate 		 * or if we're setting the error lock
21137c478bd9Sstevel@tonic-gate 		 */
21147c478bd9Sstevel@tonic-gate 		if ((LOCKFS_IS_ELOCK(&f->uf_lf) || !lfp->lf_comment ||
21157c478bd9Sstevel@tonic-gate 		    lfp->lf_comlen <= 0)) {
21167c478bd9Sstevel@tonic-gate 			from = f->uf_panic_str;
21177c478bd9Sstevel@tonic-gate 			len = LOCKFS_MAXCOMMENTLEN;
21187c478bd9Sstevel@tonic-gate 		} else {
21197c478bd9Sstevel@tonic-gate 			from = lfp->lf_comment;
21207c478bd9Sstevel@tonic-gate 			len = lfp->lf_comlen;
21217c478bd9Sstevel@tonic-gate 		}
21227c478bd9Sstevel@tonic-gate 
21237c478bd9Sstevel@tonic-gate 		bcopy(from, f->uf_lf.lf_comment, len);
21247c478bd9Sstevel@tonic-gate 		f->uf_lf.lf_comlen = len;
21257c478bd9Sstevel@tonic-gate 
21267c478bd9Sstevel@tonic-gate 	} else {
21277c478bd9Sstevel@tonic-gate 		f->uf_lf.lf_comlen = 0;
21287c478bd9Sstevel@tonic-gate 	}
21297c478bd9Sstevel@tonic-gate 	MINUTE(("] "));
21307c478bd9Sstevel@tonic-gate }
21317c478bd9Sstevel@tonic-gate 
21327c478bd9Sstevel@tonic-gate static int
set_lockfs(ufs_failure_t * f,struct lockfs * lfp)21337c478bd9Sstevel@tonic-gate set_lockfs(ufs_failure_t *f, struct lockfs *lfp)
21347c478bd9Sstevel@tonic-gate {
21357c478bd9Sstevel@tonic-gate 	int	(*handle_lockfs_rc)(ufs_failure_t *);
21367c478bd9Sstevel@tonic-gate 	int	  rc;
21377c478bd9Sstevel@tonic-gate 
21387c478bd9Sstevel@tonic-gate 	MINUTE(("[set_lockfs"));
21397c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&f->uf_mutex));
21407c478bd9Sstevel@tonic-gate 	ASSERT(!vfs_lock_held(f->uf_vfsp));
21417c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));
21427c478bd9Sstevel@tonic-gate 
21437c478bd9Sstevel@tonic-gate 	if (!f->uf_ufsvfsp) {
21447c478bd9Sstevel@tonic-gate 		MINUTE((": ufsvfsp is NULL]\n"));
21457c478bd9Sstevel@tonic-gate 		return (0);
21467c478bd9Sstevel@tonic-gate 	}
21477c478bd9Sstevel@tonic-gate 
21487c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(&f->uf_ufsvfsp->vfs_ulockfs.ul_lock));
21497c478bd9Sstevel@tonic-gate 
21507c478bd9Sstevel@tonic-gate 	if (!f->uf_ufsvfsp->vfs_root) {
21517c478bd9Sstevel@tonic-gate 		MINUTE((": vfs_root is NULL]\n"));
21527c478bd9Sstevel@tonic-gate 		return (0);
21537c478bd9Sstevel@tonic-gate 	}
21547c478bd9Sstevel@tonic-gate 
21557c478bd9Sstevel@tonic-gate 	alloc_lockfs_comment(f, lfp);
21567c478bd9Sstevel@tonic-gate 	f->uf_lf_err = 0;
21577c478bd9Sstevel@tonic-gate 
21587c478bd9Sstevel@tonic-gate 	if (!LOCKFS_IS_ELOCK(lfp)) {
21597c478bd9Sstevel@tonic-gate 		lfp->lf_lock = f->uf_lf.lf_lock = LOCKFS_ELOCK;
21607c478bd9Sstevel@tonic-gate 		VN_HOLD(f->uf_ufsvfsp->vfs_root);
216180d34432Sfrankho 		f->uf_lf_err =
216280d34432Sfrankho 		    ufs__fiolfs(f->uf_ufsvfsp->vfs_root,
216380d34432Sfrankho 		    &f->uf_lf, /* from_user */ 0, /* from_log */ 0);
21647c478bd9Sstevel@tonic-gate 		VN_RELE(f->uf_ufsvfsp->vfs_root);
21657c478bd9Sstevel@tonic-gate 	}
21667c478bd9Sstevel@tonic-gate 
21677c478bd9Sstevel@tonic-gate 	handle_lockfs_rc = f->uf_lf_err != 0? lockfs_failure: lockfs_success;
21687c478bd9Sstevel@tonic-gate 	rc = handle_lockfs_rc(f);
21697c478bd9Sstevel@tonic-gate 
21707c478bd9Sstevel@tonic-gate 	MINUTE(("] "));
21717c478bd9Sstevel@tonic-gate 	return (rc);
21727c478bd9Sstevel@tonic-gate }
21737c478bd9Sstevel@tonic-gate 
21747c478bd9Sstevel@tonic-gate static int
lockfs_failure(ufs_failure_t * f)21757c478bd9Sstevel@tonic-gate lockfs_failure(ufs_failure_t *f)
21767c478bd9Sstevel@tonic-gate {
21777c478bd9Sstevel@tonic-gate 	int	error;
21787c478bd9Sstevel@tonic-gate 	ufs_failure_states_t	s;
21797c478bd9Sstevel@tonic-gate 
21807c478bd9Sstevel@tonic-gate 	TRIVIA(("[lockfs_failure"));
21817c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&f->uf_mutex));
21827c478bd9Sstevel@tonic-gate 
21837c478bd9Sstevel@tonic-gate 	if (!f->uf_ufsvfsp) {
21847c478bd9Sstevel@tonic-gate 		TRIVIA((": ufsvfsp is NULL]\n"));
21857c478bd9Sstevel@tonic-gate 		return (0);
21867c478bd9Sstevel@tonic-gate 	}
21877c478bd9Sstevel@tonic-gate 
21887c478bd9Sstevel@tonic-gate 	error = f->uf_lf_err;
21897c478bd9Sstevel@tonic-gate 	switch (error) {
21907c478bd9Sstevel@tonic-gate 			/* non-transient errors: */
21917c478bd9Sstevel@tonic-gate 	case EACCES:	/* disk/in-core metadata reconciliation failed  */
21927c478bd9Sstevel@tonic-gate 	case EPERM:	/* inode reconciliation failed; incore inode changed? */
21937c478bd9Sstevel@tonic-gate 	case EIO:	/* device is hard-locked or not responding */
21947c478bd9Sstevel@tonic-gate 	case EROFS:	/* device is write-locked */
21957c478bd9Sstevel@tonic-gate 	case EDEADLK:	/* can't lockfs; deadlock would result; */
21967c478bd9Sstevel@tonic-gate 			/* Swapping or saving accounting records */
21977c478bd9Sstevel@tonic-gate 			/* onto this fs can cause this errno. */
21987c478bd9Sstevel@tonic-gate 
21997c478bd9Sstevel@tonic-gate 		MINOR(("ufs_fiolfs(\"%s\") of %s failed: %s (%d)",
220080d34432Sfrankho 		    fs_name(f), lock_name(&f->uf_lf),
220180d34432Sfrankho 		    err_name(error), error));
22027c478bd9Sstevel@tonic-gate 
22037c478bd9Sstevel@tonic-gate 		/*
22047c478bd9Sstevel@tonic-gate 		 * if can't get lock, then fallback to panic, unless
22057c478bd9Sstevel@tonic-gate 		 * unless unmount was requested (although unmount will
22067c478bd9Sstevel@tonic-gate 		 * probably fail if the lock failed, so we'll panic
22077c478bd9Sstevel@tonic-gate 		 * anyway
22087c478bd9Sstevel@tonic-gate 		 */
22097c478bd9Sstevel@tonic-gate 
221080d34432Sfrankho 		s = ((f->uf_flags & UFSFX_LCKUMOUNT) && error != EDEADLK) ?
221180d34432Sfrankho 		    UF_UMOUNT: UF_PANIC;
22127c478bd9Sstevel@tonic-gate 
22137c478bd9Sstevel@tonic-gate 		if (!set_state(f, s)) {
22147c478bd9Sstevel@tonic-gate 			real_panic(f, " ");
22157c478bd9Sstevel@tonic-gate 			/*NOTREACHED*/
22167c478bd9Sstevel@tonic-gate 			break;
22177c478bd9Sstevel@tonic-gate 		}
22187c478bd9Sstevel@tonic-gate 		break;
22197c478bd9Sstevel@tonic-gate 
22207c478bd9Sstevel@tonic-gate 
22217c478bd9Sstevel@tonic-gate 	case EBUSY:
22227c478bd9Sstevel@tonic-gate 	case EAGAIN:
22237c478bd9Sstevel@tonic-gate 
22247c478bd9Sstevel@tonic-gate 		f->uf_retry = ufsfx_tune.uft_short_err_period;
22257c478bd9Sstevel@tonic-gate 		if (curthread->t_flag & T_DONTPEND) {
22267c478bd9Sstevel@tonic-gate 			curthread->t_flag &= ~T_DONTPEND;
22277c478bd9Sstevel@tonic-gate 
22287c478bd9Sstevel@tonic-gate 		} else if (!(f->uf_s & (UF_LOCKED | UF_FIXING))) {
22297c478bd9Sstevel@tonic-gate 			ufs_failure_states_t state;
22307c478bd9Sstevel@tonic-gate 			/*
22317c478bd9Sstevel@tonic-gate 			 * if we didn't know that the fix had started,
22327c478bd9Sstevel@tonic-gate 			 * take note
22337c478bd9Sstevel@tonic-gate 			 */
22347c478bd9Sstevel@tonic-gate 			state = error == EBUSY? UF_LOCKED: UF_FIXING;
22357c478bd9Sstevel@tonic-gate 			if (!set_state(f, state)) {
22367c478bd9Sstevel@tonic-gate 				TRIVIA((": failed] "));
22377c478bd9Sstevel@tonic-gate 				return (0);
22387c478bd9Sstevel@tonic-gate 			}
22397c478bd9Sstevel@tonic-gate 		}
22407c478bd9Sstevel@tonic-gate 		break;
22417c478bd9Sstevel@tonic-gate 
22427c478bd9Sstevel@tonic-gate 	default:	/* some other non-fatal error */
22437c478bd9Sstevel@tonic-gate 		MINOR(("lockfs(\"%s\") of %s returned %s (%d)",
224480d34432Sfrankho 		    lock_name(&f->uf_lf), fs_name(f),
224580d34432Sfrankho 		    err_name(f->uf_lf_err), f->uf_lf_err));
22467c478bd9Sstevel@tonic-gate 
22477c478bd9Sstevel@tonic-gate 		f->uf_retry = ufsfx_tune.uft_short_err_period;
22487c478bd9Sstevel@tonic-gate 		break;
22497c478bd9Sstevel@tonic-gate 
22507c478bd9Sstevel@tonic-gate 	case EINVAL:	/* unmounted? */
22517c478bd9Sstevel@tonic-gate 		(void) set_state(f, UF_NOTFIX);
22527c478bd9Sstevel@tonic-gate 		break;
22537c478bd9Sstevel@tonic-gate 	}
22547c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
22557c478bd9Sstevel@tonic-gate 	return (1);
22567c478bd9Sstevel@tonic-gate }
22577c478bd9Sstevel@tonic-gate 
22587c478bd9Sstevel@tonic-gate static int
lockfs_success(ufs_failure_t * f)22597c478bd9Sstevel@tonic-gate lockfs_success(ufs_failure_t *f)
22607c478bd9Sstevel@tonic-gate {
22617c478bd9Sstevel@tonic-gate 	TRIVIA(("[lockfs_success"));
22627c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&f->uf_mutex));
22637c478bd9Sstevel@tonic-gate 
22647c478bd9Sstevel@tonic-gate 	if (!f->uf_ufsvfsp) {
22657c478bd9Sstevel@tonic-gate 		TRIVIA((": ufsvfsp is NULL]\n"));
22667c478bd9Sstevel@tonic-gate 		return (0);
22677c478bd9Sstevel@tonic-gate 	}
22687c478bd9Sstevel@tonic-gate 
22697c478bd9Sstevel@tonic-gate 	switch (f->uf_lf.lf_lock) {
22707c478bd9Sstevel@tonic-gate 	case LOCKFS_ELOCK:	/* error lock worked */
22717c478bd9Sstevel@tonic-gate 
22727c478bd9Sstevel@tonic-gate 		if (!set_state(f, UF_LOCKED)) {
22737c478bd9Sstevel@tonic-gate 			TRIVIA((": failed] "));
22747c478bd9Sstevel@tonic-gate 			return (0);
22757c478bd9Sstevel@tonic-gate 		}
22767c478bd9Sstevel@tonic-gate 		break;
22777c478bd9Sstevel@tonic-gate 
2278645a7badSToomas Soome 	case LOCKFS_ULOCK:	/* unlock worked */
22797c478bd9Sstevel@tonic-gate 		/*
22807c478bd9Sstevel@tonic-gate 		 * how'd we get here?
22817c478bd9Sstevel@tonic-gate 		 * This should be done from fsck's unlock,
22827c478bd9Sstevel@tonic-gate 		 * not from this thread's context.
22837c478bd9Sstevel@tonic-gate 		 */
22847c478bd9Sstevel@tonic-gate 		cmn_err(CE_WARN, "Unlocked error-lock of %s", fs_name(f));
22857c478bd9Sstevel@tonic-gate 		ufsfx_unlockfs(f->uf_ufsvfsp);
22867c478bd9Sstevel@tonic-gate 		break;
22877c478bd9Sstevel@tonic-gate 
22887c478bd9Sstevel@tonic-gate 	default:
22897c478bd9Sstevel@tonic-gate 		if (!set_state(f, UF_NOTFIX)) {
22907c478bd9Sstevel@tonic-gate 			TRIVIA((": failed] "));
22917c478bd9Sstevel@tonic-gate 			return (0);
22927c478bd9Sstevel@tonic-gate 		}
22937c478bd9Sstevel@tonic-gate 		break;
22947c478bd9Sstevel@tonic-gate 	}
22957c478bd9Sstevel@tonic-gate 	TRIVIA(("] "));
22967c478bd9Sstevel@tonic-gate 	return (1);
22977c478bd9Sstevel@tonic-gate }
22987c478bd9Sstevel@tonic-gate 
22997c478bd9Sstevel@tonic-gate /*
23007c478bd9Sstevel@tonic-gate  * when fsck is running it puts its pid into the lockfs
23017c478bd9Sstevel@tonic-gate  * comment structure, prefaced by PIDSTR
23027c478bd9Sstevel@tonic-gate  */
23037c478bd9Sstevel@tonic-gate const char *PIDSTR = "[pid:";
23047c478bd9Sstevel@tonic-gate static int
fsck_active(ufs_failure_t * f)23057c478bd9Sstevel@tonic-gate fsck_active(ufs_failure_t *f)
23067c478bd9Sstevel@tonic-gate {
23077c478bd9Sstevel@tonic-gate 	char		*cp;
23087c478bd9Sstevel@tonic-gate 	int		 i, found, errlocked;
23097c478bd9Sstevel@tonic-gate 	size_t		 comlen;
23107c478bd9Sstevel@tonic-gate 	const int	 PIDSTRLEN = (int)strlen(PIDSTR);
23117c478bd9Sstevel@tonic-gate 	struct ulockfs	*ulp = &f->uf_ufsvfsp->vfs_ulockfs;
23127c478bd9Sstevel@tonic-gate 
23137c478bd9Sstevel@tonic-gate 	TRIVIA(("[fsck_active"));
23147c478bd9Sstevel@tonic-gate 
23157c478bd9Sstevel@tonic-gate 	ASSERT(f);
23167c478bd9Sstevel@tonic-gate 	ASSERT(f->uf_s & UF_FIXING);
23177c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&f->uf_mutex));
23187c478bd9Sstevel@tonic-gate 	ASSERT(f->uf_ufsvfsp);
23197c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(f->uf_vfs_lockp));
23207c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_NOT_HELD(&ulp->ul_lock));
23217c478bd9Sstevel@tonic-gate 
23227c478bd9Sstevel@tonic-gate 	mutex_enter(&ulp->ul_lock);
23237c478bd9Sstevel@tonic-gate 	cp = ulp->ul_lockfs.lf_comment;
23247c478bd9Sstevel@tonic-gate 	comlen = ulp->ul_lockfs.lf_comlen;
23257c478bd9Sstevel@tonic-gate 	errlocked = (int)ULOCKFS_IS_ELOCK(ulp);
23267c478bd9Sstevel@tonic-gate 	mutex_exit(&ulp->ul_lock);
23277c478bd9Sstevel@tonic-gate 
23287c478bd9Sstevel@tonic-gate 	if (!cp || comlen == 0) {
23297c478bd9Sstevel@tonic-gate 		TRIVIA((": null comment or comlen <= 0, found:0]"));
23307c478bd9Sstevel@tonic-gate 		return (0);
23317c478bd9Sstevel@tonic-gate 	}
23327c478bd9Sstevel@tonic-gate 
23337c478bd9Sstevel@tonic-gate 	for (found = i = 0; !found && i < (comlen - PIDSTRLEN); i++, cp++)
23347c478bd9Sstevel@tonic-gate 		found = strncmp(cp, PIDSTR, PIDSTRLEN) == 0;
23357c478bd9Sstevel@tonic-gate 
23367c478bd9Sstevel@tonic-gate 	TRIVIA(("found:%d, is_elock:%d]", found, errlocked));
23377c478bd9Sstevel@tonic-gate 	return (errlocked & found);
23387c478bd9Sstevel@tonic-gate }
23397c478bd9Sstevel@tonic-gate 
23407c478bd9Sstevel@tonic-gate static const char unknown_fs[]		= "<unknown fs>";
23417c478bd9Sstevel@tonic-gate static const char null_failure[] = "<NULL ufs failure record; unknown fs>";
23427c478bd9Sstevel@tonic-gate static const char mutated_vfs_bufp[]	= "<mutated vfs_bufp, unknown fs>";
23437c478bd9Sstevel@tonic-gate static const char mutated_vfs_fs[]	= "<mutated vfs_fs, unknown fs>";
23447c478bd9Sstevel@tonic-gate 
23457c478bd9Sstevel@tonic-gate static char *
fs_name(ufs_failure_t * f)23467c478bd9Sstevel@tonic-gate fs_name(ufs_failure_t *f)
23477c478bd9Sstevel@tonic-gate {
23487c478bd9Sstevel@tonic-gate 	HIDEOUS(("[fs_name"));
23497c478bd9Sstevel@tonic-gate 	ASSERT(MUTEX_HELD(&f->uf_mutex));
23507c478bd9Sstevel@tonic-gate 
23517c478bd9Sstevel@tonic-gate 	if (!f) {
23527c478bd9Sstevel@tonic-gate 		HIDEOUS((": failure ptr is NULL]\n"));
23537c478bd9Sstevel@tonic-gate 		return ((char *)null_failure);
23547c478bd9Sstevel@tonic-gate 	}
23557c478bd9Sstevel@tonic-gate 
23567c478bd9Sstevel@tonic-gate 	if (f->uf_fsname[0] != '\0') {
23577c478bd9Sstevel@tonic-gate 		HIDEOUS((": return (uf_fsname)]\n"));
23587c478bd9Sstevel@tonic-gate 		return (f->uf_fsname);
23597c478bd9Sstevel@tonic-gate 	}
23607c478bd9Sstevel@tonic-gate 
23617c478bd9Sstevel@tonic-gate 	if (MUTEX_HELD(f->uf_vfs_lockp)) {
23627c478bd9Sstevel@tonic-gate 		if (f->uf_bp != f->uf_ufsvfsp->vfs_bufp) {
23637c478bd9Sstevel@tonic-gate 			HIDEOUS((": vfs_bufp mutated from 0x%p to 0x%p\n",
23647c478bd9Sstevel@tonic-gate 			    (void *)f->uf_bp, (void *)f->uf_ufsvfsp->vfs_bufp));
23657c478bd9Sstevel@tonic-gate 			return ((char *)mutated_vfs_bufp);
23667c478bd9Sstevel@tonic-gate 		}
23677c478bd9Sstevel@tonic-gate 		if (f->uf_fs != f->uf_ufsvfsp->vfs_fs) {
23687c478bd9Sstevel@tonic-gate 			HIDEOUS((": vfs_bufp mutated from 0x%p to 0x%p\n",
23697c478bd9Sstevel@tonic-gate 			    (void *)f->uf_fs, (void *)f->uf_ufsvfsp->vfs_fs));
23707c478bd9Sstevel@tonic-gate 			return ((char *)mutated_vfs_fs);
23717c478bd9Sstevel@tonic-gate 		}
23727c478bd9Sstevel@tonic-gate 		if (f->uf_ufsvfsp && f->uf_bp && f->uf_fs &&
23737c478bd9Sstevel@tonic-gate 		    *f->uf_fs->fs_fsmnt != '\0') {
23747c478bd9Sstevel@tonic-gate 			HIDEOUS((": return (fs_fsmnt)]\n"));
23757c478bd9Sstevel@tonic-gate 			return (f->uf_fs->fs_fsmnt);
23767c478bd9Sstevel@tonic-gate 		}
23777c478bd9Sstevel@tonic-gate 	}
23787c478bd9Sstevel@tonic-gate 
23797c478bd9Sstevel@tonic-gate 	HIDEOUS((": unknown file system]\n"));
23807c478bd9Sstevel@tonic-gate 	return ((char *)unknown_fs);
23817c478bd9Sstevel@tonic-gate }
23827c478bd9Sstevel@tonic-gate 
23837c478bd9Sstevel@tonic-gate #if defined(DEBUG)
23847c478bd9Sstevel@tonic-gate static char *
lock_name(struct lockfs * lfp)23857c478bd9Sstevel@tonic-gate lock_name(struct lockfs *lfp)
23867c478bd9Sstevel@tonic-gate {
23877c478bd9Sstevel@tonic-gate 	struct lock_description	*l;
23887c478bd9Sstevel@tonic-gate 	char			*lname;
23897c478bd9Sstevel@tonic-gate 
23907c478bd9Sstevel@tonic-gate 	HIDEOUS(("[lock_name"));
23917c478bd9Sstevel@tonic-gate 
23927c478bd9Sstevel@tonic-gate 	lname = lock_desc[0].ld_name;
23937c478bd9Sstevel@tonic-gate 	for (l = &lock_desc[1]; l->ld_name != NULL; l++) {
23947c478bd9Sstevel@tonic-gate 		if (lfp && lfp->lf_lock == l->ld_type) {
23957c478bd9Sstevel@tonic-gate 			lname = l->ld_name;
23967c478bd9Sstevel@tonic-gate 			break;
23977c478bd9Sstevel@tonic-gate 		}
23987c478bd9Sstevel@tonic-gate 	}
23997c478bd9Sstevel@tonic-gate 	HIDEOUS(("]"));
24007c478bd9Sstevel@tonic-gate 	return (lname);
24017c478bd9Sstevel@tonic-gate }
24027c478bd9Sstevel@tonic-gate 
24037c478bd9Sstevel@tonic-gate static char *
state_name(ufs_failure_states_t state)24047c478bd9Sstevel@tonic-gate state_name(ufs_failure_states_t state)
24057c478bd9Sstevel@tonic-gate {
24067c478bd9Sstevel@tonic-gate 	ufsd_t	*s;
24077c478bd9Sstevel@tonic-gate 
24087c478bd9Sstevel@tonic-gate 	HIDEOUS(("[state_name"));
24097c478bd9Sstevel@tonic-gate 
24107c478bd9Sstevel@tonic-gate 	s = get_state_desc(state);
24117c478bd9Sstevel@tonic-gate 
24127c478bd9Sstevel@tonic-gate 	HIDEOUS(("]"));
24137c478bd9Sstevel@tonic-gate 	return (s->ud_name);
24147c478bd9Sstevel@tonic-gate }
24157c478bd9Sstevel@tonic-gate 
24167c478bd9Sstevel@tonic-gate static char *
err_name(int error)24177c478bd9Sstevel@tonic-gate err_name(int error)
24187c478bd9Sstevel@tonic-gate {
24197c478bd9Sstevel@tonic-gate 	struct error_description *e;
24207c478bd9Sstevel@tonic-gate 
24217c478bd9Sstevel@tonic-gate 	HIDEOUS(("[err_name"));
24227c478bd9Sstevel@tonic-gate 
24237c478bd9Sstevel@tonic-gate 	for (e = &err_desc[1]; e->ed_name != NULL; e++) {
24247c478bd9Sstevel@tonic-gate 		if (error == e->ed_errno) {
24257c478bd9Sstevel@tonic-gate 			HIDEOUS(("]"));
24267c478bd9Sstevel@tonic-gate 			return (e->ed_name);
24277c478bd9Sstevel@tonic-gate 		}
24287c478bd9Sstevel@tonic-gate 	}
24297c478bd9Sstevel@tonic-gate 	HIDEOUS(("]"));
24307c478bd9Sstevel@tonic-gate 	return (err_desc[0].ed_name);
24317c478bd9Sstevel@tonic-gate }
24327c478bd9Sstevel@tonic-gate 
24337c478bd9Sstevel@tonic-gate static char *
act_name(ufsa_t action)24347c478bd9Sstevel@tonic-gate act_name(ufsa_t action)
24357c478bd9Sstevel@tonic-gate {
24367c478bd9Sstevel@tonic-gate 	struct action_description *a;
24377c478bd9Sstevel@tonic-gate 
24387c478bd9Sstevel@tonic-gate 	HIDEOUS(("[act_name"));
24397c478bd9Sstevel@tonic-gate 
24407c478bd9Sstevel@tonic-gate 	for (a = &act_desc[1]; a->ad_name != NULL; a++) {
24417c478bd9Sstevel@tonic-gate 		if (action == a->ad_v) {
24427c478bd9Sstevel@tonic-gate 			HIDEOUS(("]"));
24437c478bd9Sstevel@tonic-gate 			return (a->ad_name);
24447c478bd9Sstevel@tonic-gate 		}
24457c478bd9Sstevel@tonic-gate 	}
24467c478bd9Sstevel@tonic-gate 	HIDEOUS(("]"));
24477c478bd9Sstevel@tonic-gate 	return (act_desc[0].ad_name);
24487c478bd9Sstevel@tonic-gate }
24497c478bd9Sstevel@tonic-gate 
24507c478bd9Sstevel@tonic-gate /*
24517c478bd9Sstevel@tonic-gate  * dump failure list
24527c478bd9Sstevel@tonic-gate  */
24537c478bd9Sstevel@tonic-gate static void
dump_uf_list(char * msg)24547c478bd9Sstevel@tonic-gate dump_uf_list(char *msg)
24557c478bd9Sstevel@tonic-gate {
24567c478bd9Sstevel@tonic-gate 	ufs_failure_t	*f;
24577c478bd9Sstevel@tonic-gate 	int		 i;
24587c478bd9Sstevel@tonic-gate 	int		 list_was_locked = MUTEX_HELD(&ufs_fix.uq_mutex);
24597c478bd9Sstevel@tonic-gate 
24607c478bd9Sstevel@tonic-gate 	if (!list_was_locked && !mutex_tryenter(&ufs_fix.uq_mutex)) {
24617c478bd9Sstevel@tonic-gate 		printf("dump_uf_list: couldn't get list lock\n");
24627c478bd9Sstevel@tonic-gate 		return;
24637c478bd9Sstevel@tonic-gate 	}
24647c478bd9Sstevel@tonic-gate 
24657c478bd9Sstevel@tonic-gate 	if (msg) {
24667c478bd9Sstevel@tonic-gate 		printf("\n%s", msg);
24677c478bd9Sstevel@tonic-gate 	}
24687c478bd9Sstevel@tonic-gate 	printf("\ndump_uf_list:\n\tuq_lowat: %d, uq_ne: %d\n",
246980d34432Sfrankho 	    ufs_fix.uq_lowat, ufs_fix.uq_ne);
24707c478bd9Sstevel@tonic-gate 
24717c478bd9Sstevel@tonic-gate 	mutex_enter(&uf_stats.ufst_mutex);
24727c478bd9Sstevel@tonic-gate 	printf("\tuf_stats.current_races: %ld\n", uf_stats.ufst_current_races);
24737c478bd9Sstevel@tonic-gate 	printf("\tuf_stats.num_failed: %ld\n", uf_stats.ufst_num_failed);
24747c478bd9Sstevel@tonic-gate 	printf("\tuf_stats.num_fixed: %ld\n", uf_stats.ufst_num_fixed);
24757c478bd9Sstevel@tonic-gate 	printf("\tuf_stats.cpu_waste: %ld\n", uf_stats.ufst_cpu_waste);
24767c478bd9Sstevel@tonic-gate 	printf("\tuf_stats.lock_violations: %ld, unmount_failures: %ld\n",
247780d34432Sfrankho 	    uf_stats.ufst_lock_violations, uf_stats.ufst_unmount_failures);
24787c478bd9Sstevel@tonic-gate 	mutex_exit(&uf_stats.ufst_mutex);
24797c478bd9Sstevel@tonic-gate 
24807c478bd9Sstevel@tonic-gate 	for (f = ufs_fix.uq_ufhead, i = 1; f; f = f->uf_next, i++) {
24817c478bd9Sstevel@tonic-gate 
24827c478bd9Sstevel@tonic-gate 		if (!mutex_tryenter(&f->uf_mutex)) {
24837c478bd9Sstevel@tonic-gate 			printf("%d.\t\"skipped - try enter failed\"\n", i);
24847c478bd9Sstevel@tonic-gate 			continue;
24857c478bd9Sstevel@tonic-gate 		}
24867c478bd9Sstevel@tonic-gate 
24877c478bd9Sstevel@tonic-gate 		dump_uf(f, i);
24887c478bd9Sstevel@tonic-gate 
24897c478bd9Sstevel@tonic-gate 		mutex_exit(&f->uf_mutex);
24907c478bd9Sstevel@tonic-gate 	}
24917c478bd9Sstevel@tonic-gate 
24927c478bd9Sstevel@tonic-gate 	printf("\n");
24937c478bd9Sstevel@tonic-gate 
24947c478bd9Sstevel@tonic-gate 	if (!list_was_locked)
24957c478bd9Sstevel@tonic-gate 		mutex_exit(&ufs_fix.uq_mutex);
24967c478bd9Sstevel@tonic-gate }
24977c478bd9Sstevel@tonic-gate 
24987c478bd9Sstevel@tonic-gate static void
dump_uf(ufs_failure_t * f,int i)24997c478bd9Sstevel@tonic-gate dump_uf(ufs_failure_t *f, int i)
25007c478bd9Sstevel@tonic-gate {
25017c478bd9Sstevel@tonic-gate 	if (!f) {
25027c478bd9Sstevel@tonic-gate 		printf("dump_uf: NULL failure record\n");
25037c478bd9Sstevel@tonic-gate 		return;
25047c478bd9Sstevel@tonic-gate 	}
25057c478bd9Sstevel@tonic-gate 
25067c478bd9Sstevel@tonic-gate 	printf("%d.\t\"%s\" is %s.\n",
250780d34432Sfrankho 	    i, fs_name(f), state_name(f->uf_s));
25087c478bd9Sstevel@tonic-gate 	printf("\t\"%s\"\tAddr: 0x%p\n", f->uf_panic_str, (void *)f);
25097c478bd9Sstevel@tonic-gate 	printf("\tNext: 0x%p\t\tPrev: 0x%p\n",
251080d34432Sfrankho 	    (void *)f->uf_next, (void *)f->uf_prev);
25117c478bd9Sstevel@tonic-gate 
25127c478bd9Sstevel@tonic-gate 	if (f->uf_orig)
25137c478bd9Sstevel@tonic-gate 		printf("\tOriginal failure: 0x%p \"%s\"\n",
25147c478bd9Sstevel@tonic-gate 		    (void *)f->uf_orig, f->uf_orig->uf_panic_str);
25157c478bd9Sstevel@tonic-gate 
25167c478bd9Sstevel@tonic-gate 	printf("\tUfsvfs: 0x%p\t\tVfs_lockp: 0x%p\n",
251780d34432Sfrankho 	    (void *)f->uf_ufsvfsp, (void *)f->uf_vfs_lockp);
25187c478bd9Sstevel@tonic-gate 	printf("\tVfs_fsfxp: 0x%p\n", (void *)f->uf_vfs_ufsfxp);
25197c478bd9Sstevel@tonic-gate 	printf("\tVfs_bufp: 0x%p", (void *)f->uf_bp);
25207c478bd9Sstevel@tonic-gate 
25217c478bd9Sstevel@tonic-gate 	if (f->uf_bp)
25227c478bd9Sstevel@tonic-gate 		printf("\t\tVfs_fs: 0x%p\n", (void *)f->uf_fs);
25237c478bd9Sstevel@tonic-gate 	else
25247c478bd9Sstevel@tonic-gate 		printf("\n");
25257c478bd9Sstevel@tonic-gate 
25267c478bd9Sstevel@tonic-gate 	printf("\tBegin: 0x%lx\tEntered: 0x%lx\tEnd: 0x%lx\n",
25277c478bd9Sstevel@tonic-gate 	    f->uf_begin_tm, f->uf_entered_tm, f->uf_end_tm);
25287c478bd9Sstevel@tonic-gate 
25297c478bd9Sstevel@tonic-gate 	printf("\tFlags: (%d) %s%s%s%s", f->uf_flags,
253080d34432Sfrankho 	    f->uf_flags & UFSFX_LCKONLY?	 "\"lock only\" "	: "",
253180d34432Sfrankho 	    f->uf_flags & UFSFX_LCKUMOUNT?	 "\"lock+unmount\" "	: "",
253280d34432Sfrankho 	    f->uf_flags & UFSFX_REPAIR_START? "\"started repair\" "	: "",
253380d34432Sfrankho 	    f->uf_flags == 0?                "<none>"               : "");
25347c478bd9Sstevel@tonic-gate 
25357c478bd9Sstevel@tonic-gate 	printf("\tRetry: %ld seconds\n", f->uf_retry);
25367c478bd9Sstevel@tonic-gate 
25377c478bd9Sstevel@tonic-gate 	printf("\tLockfs:\ttype: %s\terror: %s (%d)\n",
253880d34432Sfrankho 	    lock_name(&f->uf_lf), err_name(f->uf_lf_err), f->uf_lf_err);
25397c478bd9Sstevel@tonic-gate 
25407c478bd9Sstevel@tonic-gate }
25417c478bd9Sstevel@tonic-gate #endif /* DEBUG */
25427c478bd9Sstevel@tonic-gate 
25437c478bd9Sstevel@tonic-gate /*
25447c478bd9Sstevel@tonic-gate  * returns # of ufs_failures in a non-terminal state on queue
25457c478bd9Sstevel@tonic-gate  * used to coordinate with hlock thread (see ufs_thread.c)
25467c478bd9Sstevel@tonic-gate  * and to determine when the error lock thread may exit
25477c478bd9Sstevel@tonic-gate  */
25487c478bd9Sstevel@tonic-gate 
25497c478bd9Sstevel@tonic-gate int
ufsfx_get_failure_qlen(void)25507c478bd9Sstevel@tonic-gate ufsfx_get_failure_qlen(void)
25517c478bd9Sstevel@tonic-gate {
25527c478bd9Sstevel@tonic-gate 	ufs_failure_t	*f;
25537c478bd9Sstevel@tonic-gate 	ufsd_t		*s;
25547c478bd9Sstevel@tonic-gate 	int		 qlen = 0;
25557c478bd9Sstevel@tonic-gate 
25567c478bd9Sstevel@tonic-gate 	MINUTE(("[ufsfx_get_failure_qlen"));
25577c478bd9Sstevel@tonic-gate 
25587c478bd9Sstevel@tonic-gate 	if (!mutex_tryenter(&ufs_fix.uq_mutex))
25597c478bd9Sstevel@tonic-gate 		return (-1);
25607c478bd9Sstevel@tonic-gate 
25617c478bd9Sstevel@tonic-gate 	/*
25627c478bd9Sstevel@tonic-gate 	 * walk down failure list
25637c478bd9Sstevel@tonic-gate 	 */
25647c478bd9Sstevel@tonic-gate 
25657c478bd9Sstevel@tonic-gate 	for (f = ufs_fix.uq_ufhead; f; f = f->uf_next) {
25667c478bd9Sstevel@tonic-gate 
25677c478bd9Sstevel@tonic-gate 		if (!mutex_tryenter(&f->uf_mutex))
25687c478bd9Sstevel@tonic-gate 			continue;
25697c478bd9Sstevel@tonic-gate 
25707c478bd9Sstevel@tonic-gate 		s = get_state_desc(f->uf_s);
25717c478bd9Sstevel@tonic-gate 
25727c478bd9Sstevel@tonic-gate 		if (s->ud_attr.terminal) {
25737c478bd9Sstevel@tonic-gate 			mutex_exit(&f->uf_mutex);
25747c478bd9Sstevel@tonic-gate 			continue;
25757c478bd9Sstevel@tonic-gate 		}
25767c478bd9Sstevel@tonic-gate 
25777c478bd9Sstevel@tonic-gate 		MINUTE((": found: %s, \"%s: %s\"\n",
257880d34432Sfrankho 		    fs_name(f), state_name(f->uf_s), f->uf_panic_str));
25797c478bd9Sstevel@tonic-gate 
25807c478bd9Sstevel@tonic-gate 		qlen++;
25817c478bd9Sstevel@tonic-gate 		mutex_exit(&f->uf_mutex);
25827c478bd9Sstevel@tonic-gate 	}
25837c478bd9Sstevel@tonic-gate 
25847c478bd9Sstevel@tonic-gate 	mutex_exit(&ufs_fix.uq_mutex);
25857c478bd9Sstevel@tonic-gate 
25867c478bd9Sstevel@tonic-gate 	MINUTE((": qlen=%d]\n", qlen));
25877c478bd9Sstevel@tonic-gate 
25887c478bd9Sstevel@tonic-gate 	return (qlen);
25897c478bd9Sstevel@tonic-gate }
25907c478bd9Sstevel@tonic-gate 
25917c478bd9Sstevel@tonic-gate /*
25927c478bd9Sstevel@tonic-gate  * timeout routine
25937c478bd9Sstevel@tonic-gate  *  called to shutdown fix failure thread and server daemon
25947c478bd9Sstevel@tonic-gate  */
25957c478bd9Sstevel@tonic-gate static void
ufsfx_kill_fix_failure_thread(void * arg)25967c478bd9Sstevel@tonic-gate ufsfx_kill_fix_failure_thread(void *arg)
25977c478bd9Sstevel@tonic-gate {
25987c478bd9Sstevel@tonic-gate 	clock_t odelta = (clock_t)arg;
25997c478bd9Sstevel@tonic-gate 	int	qlen;
26007c478bd9Sstevel@tonic-gate 
26017c478bd9Sstevel@tonic-gate 	MAJOR(("[ufsfx_kill_fix_failure_thread"));
26027c478bd9Sstevel@tonic-gate 
26037c478bd9Sstevel@tonic-gate 	qlen = ufsfx_get_failure_qlen();
26047c478bd9Sstevel@tonic-gate 
26057c478bd9Sstevel@tonic-gate 	if (qlen < 0) {
26067c478bd9Sstevel@tonic-gate 		clock_t delta;
26077c478bd9Sstevel@tonic-gate 
26087c478bd9Sstevel@tonic-gate 		delta = odelta << 1;
26097c478bd9Sstevel@tonic-gate 		if (delta <= 0)
26107c478bd9Sstevel@tonic-gate 			delta = INT_MAX;
26117c478bd9Sstevel@tonic-gate 
26127c478bd9Sstevel@tonic-gate 		(void) timeout(ufsfx_kill_fix_failure_thread,
26137c478bd9Sstevel@tonic-gate 		    (void *)delta, delta);
26147c478bd9Sstevel@tonic-gate 		MAJOR((": rescheduled"));
26157c478bd9Sstevel@tonic-gate 
26167c478bd9Sstevel@tonic-gate 	} else if (qlen == 0) {
26177c478bd9Sstevel@tonic-gate 		ufs_thread_exit(&ufs_fix);
26187c478bd9Sstevel@tonic-gate 		MAJOR((": killed"));
26197c478bd9Sstevel@tonic-gate 	}
26207c478bd9Sstevel@tonic-gate 	/*
26217c478bd9Sstevel@tonic-gate 	 * else
26227c478bd9Sstevel@tonic-gate 	 *  let timeout expire
26237c478bd9Sstevel@tonic-gate 	 */
26247c478bd9Sstevel@tonic-gate 	MAJOR(("]\n"));
26257c478bd9Sstevel@tonic-gate }
2626