1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 1984, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2018 Joyent, Inc.
25 * Copyright (c) 2016 by Delphix. All rights reserved.
26 */
27
28/*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T	*/
29/*	  All Rights Reserved	*/
30
31/*
32 * Portions of this source code were derived from Berkeley 4.3 BSD
33 * under license from the Regents of the University of California.
34 */
35
36#include <sys/types.h>
37#include <sys/t_lock.h>
38#include <sys/ksynch.h>
39#include <sys/param.h>
40#include <sys/time.h>
41#include <sys/systm.h>
42#include <sys/sysmacros.h>
43#include <sys/resource.h>
44#include <sys/signal.h>
45#include <sys/cred.h>
46#include <sys/user.h>
47#include <sys/buf.h>
48#include <sys/vfs.h>
49#include <sys/vfs_opreg.h>
50#include <sys/vnode.h>
51#include <sys/proc.h>
52#include <sys/disp.h>
53#include <sys/file.h>
54#include <sys/fcntl.h>
55#include <sys/flock.h>
56#include <sys/atomic.h>
57#include <sys/kmem.h>
58#include <sys/uio.h>
59#include <sys/dnlc.h>
60#include <sys/conf.h>
61#include <sys/mman.h>
62#include <sys/pathname.h>
63#include <sys/debug.h>
64#include <sys/vmsystm.h>
65#include <sys/cmn_err.h>
66#include <sys/filio.h>
67#include <sys/policy.h>
68
69#include <sys/fs/ufs_fs.h>
70#include <sys/fs/ufs_lockfs.h>
71#include <sys/fs/ufs_filio.h>
72#include <sys/fs/ufs_inode.h>
73#include <sys/fs/ufs_fsdir.h>
74#include <sys/fs/ufs_quota.h>
75#include <sys/fs/ufs_log.h>
76#include <sys/fs/ufs_snap.h>
77#include <sys/fs/ufs_trans.h>
78#include <sys/fs/ufs_panic.h>
79#include <sys/fs/ufs_bio.h>
80#include <sys/dirent.h>		/* must be AFTER <sys/fs/fsdir.h>! */
81#include <sys/errno.h>
82#include <sys/fssnap_if.h>
83#include <sys/unistd.h>
84#include <sys/sunddi.h>
85
86#include <sys/filio.h>		/* _FIOIO */
87
88#include <vm/hat.h>
89#include <vm/page.h>
90#include <vm/pvn.h>
91#include <vm/as.h>
92#include <vm/seg.h>
93#include <vm/seg_map.h>
94#include <vm/seg_vn.h>
95#include <vm/seg_kmem.h>
96#include <vm/rm.h>
97#include <sys/swap.h>
98
99#include <fs/fs_subr.h>
100
101#include <sys/fs/decomp.h>
102
103static struct instats ins;
104
105static	int ufs_getpage_ra(struct vnode *, u_offset_t, struct seg *, caddr_t);
106static	int ufs_getpage_miss(struct vnode *, u_offset_t, size_t, struct seg *,
107		caddr_t, struct page **, size_t, enum seg_rw, int);
108static	int ufs_open(struct vnode **, int, struct cred *, caller_context_t *);
109static	int ufs_close(struct vnode *, int, int, offset_t, struct cred *,
110		caller_context_t *);
111static	int ufs_read(struct vnode *, struct uio *, int, struct cred *,
112		struct caller_context *);
113static	int ufs_write(struct vnode *, struct uio *, int, struct cred *,
114		struct caller_context *);
115static	int ufs_ioctl(struct vnode *, int, intptr_t, int, struct cred *,
116		int *, caller_context_t *);
117static	int ufs_getattr(struct vnode *, struct vattr *, int, struct cred *,
118		caller_context_t *);
119static	int ufs_setattr(struct vnode *, struct vattr *, int, struct cred *,
120		caller_context_t *);
121static	int ufs_access(struct vnode *, int, int, struct cred *,
122		caller_context_t *);
123static	int ufs_lookup(struct vnode *, char *, struct vnode **,
124		struct pathname *, int, struct vnode *, struct cred *,
125		caller_context_t *, int *, pathname_t *);
126static	int ufs_create(struct vnode *, char *, struct vattr *, enum vcexcl,
127		int, struct vnode **, struct cred *, int,
128		caller_context_t *, vsecattr_t  *);
129static	int ufs_remove(struct vnode *, char *, struct cred *,
130		caller_context_t *, int);
131static	int ufs_link(struct vnode *, struct vnode *, char *, struct cred *,
132		caller_context_t *, int);
133static	int ufs_rename(struct vnode *, char *, struct vnode *, char *,
134		struct cred *, caller_context_t *, int);
135static	int ufs_mkdir(struct vnode *, char *, struct vattr *, struct vnode **,
136		struct cred *, caller_context_t *, int, vsecattr_t *);
137static	int ufs_rmdir(struct vnode *, char *, struct vnode *, struct cred *,
138		caller_context_t *, int);
139static	int ufs_readdir(struct vnode *, struct uio *, struct cred *, int *,
140		caller_context_t *, int);
141static	int ufs_symlink(struct vnode *, char *, struct vattr *, char *,
142		struct cred *, caller_context_t *, int);
143static	int ufs_readlink(struct vnode *, struct uio *, struct cred *,
144		caller_context_t *);
145static	int ufs_fsync(struct vnode *, int, struct cred *, caller_context_t *);
146static	void ufs_inactive(struct vnode *, struct cred *, caller_context_t *);
147static	int ufs_fid(struct vnode *, struct fid *, caller_context_t *);
148static	int ufs_rwlock(struct vnode *, int, caller_context_t *);
149static	void ufs_rwunlock(struct vnode *, int, caller_context_t *);
150static	int ufs_seek(struct vnode *, offset_t, offset_t *, caller_context_t *);
151static	int ufs_frlock(struct vnode *, int, struct flock64 *, int, offset_t,
152		struct flk_callback *, struct cred *,
153		caller_context_t *);
154static  int ufs_space(struct vnode *, int, struct flock64 *, int, offset_t,
155		cred_t *, caller_context_t *);
156static	int ufs_getpage(struct vnode *, offset_t, size_t, uint_t *,
157		struct page **, size_t, struct seg *, caddr_t,
158		enum seg_rw, struct cred *, caller_context_t *);
159static	int ufs_putpage(struct vnode *, offset_t, size_t, int, struct cred *,
160		caller_context_t *);
161static	int ufs_putpages(struct vnode *, offset_t, size_t, int, struct cred *);
162static	int ufs_map(struct vnode *, offset_t, struct as *, caddr_t *, size_t,
163		uchar_t, uchar_t, uint_t, struct cred *, caller_context_t *);
164static	int ufs_addmap(struct vnode *, offset_t, struct as *, caddr_t,  size_t,
165		uchar_t, uchar_t, uint_t, struct cred *, caller_context_t *);
166static	int ufs_delmap(struct vnode *, offset_t, struct as *, caddr_t,  size_t,
167		uint_t, uint_t, uint_t, struct cred *, caller_context_t *);
168static	int ufs_poll(vnode_t *, short, int, short *, struct pollhead **,
169		caller_context_t *);
170static	int ufs_dump(vnode_t *, caddr_t, offset_t, offset_t,
171    caller_context_t *);
172static	int ufs_l_pathconf(struct vnode *, int, ulong_t *, struct cred *,
173		caller_context_t *);
174static	int ufs_pageio(struct vnode *, struct page *, u_offset_t, size_t, int,
175		struct cred *, caller_context_t *);
176static	int ufs_dumpctl(vnode_t *, int, offset_t *, caller_context_t *);
177static	daddr32_t *save_dblks(struct inode *, struct ufsvfs *, daddr32_t *,
178		daddr32_t *, int, int);
179static	int ufs_getsecattr(struct vnode *, vsecattr_t *, int, struct cred *,
180		caller_context_t *);
181static	int ufs_setsecattr(struct vnode *, vsecattr_t *, int, struct cred *,
182		caller_context_t *);
183static	int ufs_priv_access(void *, int, struct cred *);
184static	int ufs_eventlookup(struct vnode *, char *, struct cred *,
185    struct vnode **);
186
187/*
188 * For lockfs: ulockfs begin/end is now inlined in the ufs_xxx functions.
189 *
190 * XXX - ULOCKFS in fs_pathconf and ufs_ioctl is not inlined yet.
191 */
192struct vnodeops *ufs_vnodeops;
193
194/* NOTE: "not blkd" below  means that the operation isn't blocked by lockfs */
195const fs_operation_def_t ufs_vnodeops_template[] = {
196	VOPNAME_OPEN,		{ .vop_open = ufs_open },	/* not blkd */
197	VOPNAME_CLOSE,		{ .vop_close = ufs_close },	/* not blkd */
198	VOPNAME_READ,		{ .vop_read = ufs_read },
199	VOPNAME_WRITE,		{ .vop_write = ufs_write },
200	VOPNAME_IOCTL,		{ .vop_ioctl = ufs_ioctl },
201	VOPNAME_GETATTR,	{ .vop_getattr = ufs_getattr },
202	VOPNAME_SETATTR,	{ .vop_setattr = ufs_setattr },
203	VOPNAME_ACCESS,		{ .vop_access = ufs_access },
204	VOPNAME_LOOKUP,		{ .vop_lookup = ufs_lookup },
205	VOPNAME_CREATE,		{ .vop_create = ufs_create },
206	VOPNAME_REMOVE,		{ .vop_remove = ufs_remove },
207	VOPNAME_LINK,		{ .vop_link = ufs_link },
208	VOPNAME_RENAME,		{ .vop_rename = ufs_rename },
209	VOPNAME_MKDIR,		{ .vop_mkdir = ufs_mkdir },
210	VOPNAME_RMDIR,		{ .vop_rmdir = ufs_rmdir },
211	VOPNAME_READDIR,	{ .vop_readdir = ufs_readdir },
212	VOPNAME_SYMLINK,	{ .vop_symlink = ufs_symlink },
213	VOPNAME_READLINK,	{ .vop_readlink = ufs_readlink },
214	VOPNAME_FSYNC,		{ .vop_fsync = ufs_fsync },
215	VOPNAME_INACTIVE,	{ .vop_inactive = ufs_inactive }, /* not blkd */
216	VOPNAME_FID,		{ .vop_fid = ufs_fid },
217	VOPNAME_RWLOCK,		{ .vop_rwlock = ufs_rwlock },	/* not blkd */
218	VOPNAME_RWUNLOCK,	{ .vop_rwunlock = ufs_rwunlock }, /* not blkd */
219	VOPNAME_SEEK,		{ .vop_seek = ufs_seek },
220	VOPNAME_FRLOCK,		{ .vop_frlock = ufs_frlock },
221	VOPNAME_SPACE,		{ .vop_space = ufs_space },
222	VOPNAME_GETPAGE,	{ .vop_getpage = ufs_getpage },
223	VOPNAME_PUTPAGE,	{ .vop_putpage = ufs_putpage },
224	VOPNAME_MAP,		{ .vop_map = ufs_map },
225	VOPNAME_ADDMAP,		{ .vop_addmap = ufs_addmap },	/* not blkd */
226	VOPNAME_DELMAP,		{ .vop_delmap = ufs_delmap },	/* not blkd */
227	VOPNAME_POLL,		{ .vop_poll = ufs_poll },	/* not blkd */
228	VOPNAME_DUMP,		{ .vop_dump = ufs_dump },
229	VOPNAME_PATHCONF,	{ .vop_pathconf = ufs_l_pathconf },
230	VOPNAME_PAGEIO,		{ .vop_pageio = ufs_pageio },
231	VOPNAME_DUMPCTL,	{ .vop_dumpctl = ufs_dumpctl },
232	VOPNAME_GETSECATTR,	{ .vop_getsecattr = ufs_getsecattr },
233	VOPNAME_SETSECATTR,	{ .vop_setsecattr = ufs_setsecattr },
234	VOPNAME_VNEVENT,	{ .vop_vnevent = fs_vnevent_support },
235	NULL,			NULL
236};
237
238#define	MAX_BACKFILE_COUNT	9999
239
240/*
241 * Created by ufs_dumpctl() to store a file's disk block info into memory.
242 * Used by ufs_dump() to dump data to disk directly.
243 */
244struct dump {
245	struct inode	*ip;		/* the file we contain */
246	daddr_t		fsbs;		/* number of blocks stored */
247	struct timeval32 time;		/* time stamp for the struct */
248	daddr32_t	dblk[1];	/* place holder for block info */
249};
250
251static struct dump *dump_info = NULL;
252
253/*
254 * Previously there was no special action required for ordinary files.
255 * (Devices are handled through the device file system.)
256 * Now we support Large Files and Large File API requires open to
257 * fail if file is large.
258 * We could take care to prevent data corruption
259 * by doing an atomic check of size and truncate if file is opened with
260 * FTRUNC flag set but traditionally this is being done by the vfs/vnode
261 * layers. So taking care of truncation here is a change in the existing
262 * semantics of VOP_OPEN and therefore we chose not to implement any thing
263 * here. The check for the size of the file > 2GB is being done at the
264 * vfs layer in routine vn_open().
265 */
266
267/* ARGSUSED */
268static int
269ufs_open(struct vnode **vpp, int flag, struct cred *cr, caller_context_t *ct)
270{
271	return (0);
272}
273
274/*ARGSUSED*/
275static int
276ufs_close(struct vnode *vp, int flag, int count, offset_t offset,
277    struct cred *cr, caller_context_t *ct)
278{
279	cleanlocks(vp, ttoproc(curthread)->p_pid, 0);
280	cleanshares(vp, ttoproc(curthread)->p_pid);
281
282	/*
283	 * Push partially filled cluster at last close.
284	 * ``last close'' is approximated because the dnlc
285	 * may have a hold on the vnode.
286	 * Checking for VBAD here will also act as a forced umount check.
287	 */
288	if (vp->v_count <= 2 && vp->v_type != VBAD) {
289		struct inode *ip = VTOI(vp);
290		if (ip->i_delaylen) {
291			ins.in_poc.value.ul++;
292			(void) ufs_putpages(vp, ip->i_delayoff, ip->i_delaylen,
293			    B_ASYNC | B_FREE, cr);
294			ip->i_delaylen = 0;
295		}
296	}
297
298	return (0);
299}
300
301/*ARGSUSED*/
302static int
303ufs_read(struct vnode *vp, struct uio *uiop, int ioflag, struct cred *cr,
304    struct caller_context *ct)
305{
306	struct inode *ip = VTOI(vp);
307	struct ufsvfs *ufsvfsp;
308	struct ulockfs *ulp = NULL;
309	int error = 0;
310	int intrans = 0;
311
312	ASSERT(RW_READ_HELD(&ip->i_rwlock));
313
314	/*
315	 * Mandatory locking needs to be done before ufs_lockfs_begin()
316	 * and TRANS_BEGIN_SYNC() calls since mandatory locks can sleep.
317	 */
318	if (MANDLOCK(vp, ip->i_mode)) {
319		/*
320		 * ufs_getattr ends up being called by chklock
321		 */
322		error = chklock(vp, FREAD, uiop->uio_loffset,
323		    uiop->uio_resid, uiop->uio_fmode, ct);
324		if (error)
325			goto out;
326	}
327
328	ufsvfsp = ip->i_ufsvfs;
329	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_READ_MASK);
330	if (error)
331		goto out;
332
333	/*
334	 * In the case that a directory is opened for reading as a file
335	 * (eg "cat .") with the  O_RSYNC, O_SYNC and O_DSYNC flags set.
336	 * The locking order had to be changed to avoid a deadlock with
337	 * an update taking place on that directory at the same time.
338	 */
339	if ((ip->i_mode & IFMT) == IFDIR) {
340
341		rw_enter(&ip->i_contents, RW_READER);
342		error = rdip(ip, uiop, ioflag, cr);
343		rw_exit(&ip->i_contents);
344
345		if (error) {
346			if (ulp)
347				ufs_lockfs_end(ulp);
348			goto out;
349		}
350
351		if (ulp && (ioflag & FRSYNC) && (ioflag & (FSYNC | FDSYNC)) &&
352		    TRANS_ISTRANS(ufsvfsp)) {
353			rw_exit(&ip->i_rwlock);
354			TRANS_BEGIN_SYNC(ufsvfsp, TOP_READ_SYNC, TOP_READ_SIZE,
355			    error);
356			ASSERT(!error);
357			TRANS_END_SYNC(ufsvfsp, error, TOP_READ_SYNC,
358			    TOP_READ_SIZE);
359			rw_enter(&ip->i_rwlock, RW_READER);
360		}
361	} else {
362		/*
363		 * Only transact reads to files opened for sync-read and
364		 * sync-write on a file system that is not write locked.
365		 *
366		 * The ``not write locked'' check prevents problems with
367		 * enabling/disabling logging on a busy file system.  E.g.,
368		 * logging exists at the beginning of the read but does not
369		 * at the end.
370		 *
371		 */
372		if (ulp && (ioflag & FRSYNC) && (ioflag & (FSYNC | FDSYNC)) &&
373		    TRANS_ISTRANS(ufsvfsp)) {
374			TRANS_BEGIN_SYNC(ufsvfsp, TOP_READ_SYNC, TOP_READ_SIZE,
375			    error);
376			ASSERT(!error);
377			intrans = 1;
378		}
379
380		rw_enter(&ip->i_contents, RW_READER);
381		error = rdip(ip, uiop, ioflag, cr);
382		rw_exit(&ip->i_contents);
383
384		if (intrans) {
385			TRANS_END_SYNC(ufsvfsp, error, TOP_READ_SYNC,
386			    TOP_READ_SIZE);
387		}
388	}
389
390	if (ulp) {
391		ufs_lockfs_end(ulp);
392	}
393out:
394
395	return (error);
396}
397
398extern	int	ufs_HW;		/* high water mark */
399extern	int	ufs_LW;		/* low water mark */
400int	ufs_WRITES = 1;		/* XXX - enable/disable */
401int	ufs_throttles = 0;	/* throttling count */
402int	ufs_allow_shared_writes = 1;	/* directio shared writes */
403
404static int
405ufs_check_rewrite(struct inode *ip, struct uio *uiop, int ioflag)
406{
407	int	shared_write;
408
409	/*
410	 * If the FDSYNC flag is set then ignore the global
411	 * ufs_allow_shared_writes in this case.
412	 */
413	shared_write = (ioflag & FDSYNC) | ufs_allow_shared_writes;
414
415	/*
416	 * Filter to determine if this request is suitable as a
417	 * concurrent rewrite. This write must not allocate blocks
418	 * by extending the file or filling in holes. No use trying
419	 * through FSYNC descriptors as the inode will be synchronously
420	 * updated after the write. The uio structure has not yet been
421	 * checked for sanity, so assume nothing.
422	 */
423	return (((ip->i_mode & IFMT) == IFREG) && !(ioflag & FAPPEND) &&
424	    (uiop->uio_loffset >= (offset_t)0) &&
425	    (uiop->uio_loffset < ip->i_size) && (uiop->uio_resid > 0) &&
426	    ((ip->i_size - uiop->uio_loffset) >= uiop->uio_resid) &&
427	    !(ioflag & FSYNC) && !bmap_has_holes(ip) &&
428	    shared_write);
429}
430
431/*ARGSUSED*/
432static int
433ufs_write(struct vnode *vp, struct uio *uiop, int ioflag, cred_t *cr,
434    caller_context_t *ct)
435{
436	struct inode *ip = VTOI(vp);
437	struct ufsvfs *ufsvfsp;
438	struct ulockfs *ulp;
439	int retry = 1;
440	int error, resv, resid = 0;
441	int directio_status;
442	int exclusive;
443	int rewriteflg;
444	long start_resid = uiop->uio_resid;
445
446	ASSERT(RW_LOCK_HELD(&ip->i_rwlock));
447
448retry_mandlock:
449	/*
450	 * Mandatory locking needs to be done before ufs_lockfs_begin()
451	 * and TRANS_BEGIN_[A]SYNC() calls since mandatory locks can sleep.
452	 * Check for forced unmounts normally done in ufs_lockfs_begin().
453	 */
454	if ((ufsvfsp = ip->i_ufsvfs) == NULL) {
455		error = EIO;
456		goto out;
457	}
458	if (MANDLOCK(vp, ip->i_mode)) {
459
460		ASSERT(RW_WRITE_HELD(&ip->i_rwlock));
461
462		/*
463		 * ufs_getattr ends up being called by chklock
464		 */
465		error = chklock(vp, FWRITE, uiop->uio_loffset,
466		    uiop->uio_resid, uiop->uio_fmode, ct);
467		if (error)
468			goto out;
469	}
470
471	/* i_rwlock can change in chklock */
472	exclusive = rw_write_held(&ip->i_rwlock);
473	rewriteflg = ufs_check_rewrite(ip, uiop, ioflag);
474
475	/*
476	 * Check for fast-path special case of directio re-writes.
477	 */
478	if ((ip->i_flag & IDIRECTIO || ufsvfsp->vfs_forcedirectio) &&
479	    !exclusive && rewriteflg) {
480
481		error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_WRITE_MASK);
482		if (error)
483			goto out;
484
485		rw_enter(&ip->i_contents, RW_READER);
486		error = ufs_directio_write(ip, uiop, ioflag, 1, cr,
487		    &directio_status);
488		if (directio_status == DIRECTIO_SUCCESS) {
489			uint_t i_flag_save;
490
491			if (start_resid != uiop->uio_resid)
492				error = 0;
493			/*
494			 * Special treatment of access times for re-writes.
495			 * If IMOD is not already set, then convert it
496			 * to IMODACC for this operation. This defers
497			 * entering a delta into the log until the inode
498			 * is flushed. This mimics what is done for read
499			 * operations and inode access time.
500			 */
501			mutex_enter(&ip->i_tlock);
502			i_flag_save = ip->i_flag;
503			ip->i_flag |= IUPD | ICHG;
504			ip->i_seq++;
505			ITIMES_NOLOCK(ip);
506			if ((i_flag_save & IMOD) == 0) {
507				ip->i_flag &= ~IMOD;
508				ip->i_flag |= IMODACC;
509			}
510			mutex_exit(&ip->i_tlock);
511			rw_exit(&ip->i_contents);
512			if (ulp)
513				ufs_lockfs_end(ulp);
514			goto out;
515		}
516		rw_exit(&ip->i_contents);
517		if (ulp)
518			ufs_lockfs_end(ulp);
519	}
520
521	if (!exclusive && !rw_tryupgrade(&ip->i_rwlock)) {
522		rw_exit(&ip->i_rwlock);
523		rw_enter(&ip->i_rwlock, RW_WRITER);
524		/*
525		 * Mandatory locking could have been enabled
526		 * after dropping the i_rwlock.
527		 */
528		if (MANDLOCK(vp, ip->i_mode))
529			goto retry_mandlock;
530	}
531
532	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_WRITE_MASK);
533	if (error)
534		goto out;
535
536	/*
537	 * Amount of log space needed for this write
538	 */
539	if (!rewriteflg || !(ioflag & FDSYNC))
540		TRANS_WRITE_RESV(ip, uiop, ulp, &resv, &resid);
541
542	/*
543	 * Throttle writes.
544	 */
545	if (ufs_WRITES && (ip->i_writes > ufs_HW)) {
546		mutex_enter(&ip->i_tlock);
547		while (ip->i_writes > ufs_HW) {
548			ufs_throttles++;
549			cv_wait(&ip->i_wrcv, &ip->i_tlock);
550		}
551		mutex_exit(&ip->i_tlock);
552	}
553
554	/*
555	 * Enter Transaction
556	 *
557	 * If the write is a rewrite there is no need to open a transaction
558	 * if the FDSYNC flag is set and not the FSYNC.  In this case just
559	 * set the IMODACC flag to modify do the update at a later time
560	 * thus avoiding the overhead of the logging transaction that is
561	 * not required.
562	 */
563	if (ioflag & (FSYNC|FDSYNC)) {
564		if (ulp) {
565			if (rewriteflg) {
566				uint_t i_flag_save;
567
568				rw_enter(&ip->i_contents, RW_READER);
569				mutex_enter(&ip->i_tlock);
570				i_flag_save = ip->i_flag;
571				ip->i_flag |= IUPD | ICHG;
572				ip->i_seq++;
573				ITIMES_NOLOCK(ip);
574				if ((i_flag_save & IMOD) == 0) {
575					ip->i_flag &= ~IMOD;
576					ip->i_flag |= IMODACC;
577				}
578				mutex_exit(&ip->i_tlock);
579				rw_exit(&ip->i_contents);
580			} else {
581				int terr = 0;
582				TRANS_BEGIN_SYNC(ufsvfsp, TOP_WRITE_SYNC, resv,
583				    terr);
584				ASSERT(!terr);
585			}
586		}
587	} else {
588		if (ulp)
589			TRANS_BEGIN_ASYNC(ufsvfsp, TOP_WRITE, resv);
590	}
591
592	/*
593	 * Write the file
594	 */
595	rw_enter(&ufsvfsp->vfs_dqrwlock, RW_READER);
596	rw_enter(&ip->i_contents, RW_WRITER);
597	if ((ioflag & FAPPEND) != 0 && (ip->i_mode & IFMT) == IFREG) {
598		/*
599		 * In append mode start at end of file.
600		 */
601		uiop->uio_loffset = ip->i_size;
602	}
603
604	/*
605	 * Mild optimisation, don't call ufs_trans_write() unless we have to
606	 * Also, suppress file system full messages if we will retry.
607	 */
608	if (retry)
609		ip->i_flag |= IQUIET;
610	if (resid) {
611		TRANS_WRITE(ip, uiop, ioflag, error, ulp, cr, resv, resid);
612	} else {
613		error = wrip(ip, uiop, ioflag, cr);
614	}
615	ip->i_flag &= ~IQUIET;
616
617	rw_exit(&ip->i_contents);
618	rw_exit(&ufsvfsp->vfs_dqrwlock);
619
620	/*
621	 * Leave Transaction
622	 */
623	if (ulp) {
624		if (ioflag & (FSYNC|FDSYNC)) {
625			if (!rewriteflg) {
626				int terr = 0;
627
628				TRANS_END_SYNC(ufsvfsp, terr, TOP_WRITE_SYNC,
629				    resv);
630				if (error == 0)
631					error = terr;
632			}
633		} else {
634			TRANS_END_ASYNC(ufsvfsp, TOP_WRITE, resv);
635		}
636		ufs_lockfs_end(ulp);
637	}
638out:
639	if ((error == ENOSPC) && retry && TRANS_ISTRANS(ufsvfsp)) {
640		/*
641		 * Any blocks tied up in pending deletes?
642		 */
643		ufs_delete_drain_wait(ufsvfsp, 1);
644		retry = 0;
645		goto retry_mandlock;
646	}
647
648	if (error == ENOSPC && (start_resid != uiop->uio_resid))
649		error = 0;
650
651	return (error);
652}
653
654/*
655 * Don't cache write blocks to files with the sticky bit set.
656 * Used to keep swap files from blowing the page cache on a server.
657 */
658int stickyhack = 1;
659
660/*
661 * wrip does the real work of write requests for ufs.
662 */
663int
664wrip(struct inode *ip, struct uio *uio, int ioflag, struct cred *cr)
665{
666	rlim64_t limit = uio->uio_llimit;
667	u_offset_t off;
668	u_offset_t old_i_size;
669	struct fs *fs;
670	struct vnode *vp;
671	struct ufsvfs *ufsvfsp;
672	caddr_t base;
673	long start_resid = uio->uio_resid;	/* save starting resid */
674	long premove_resid;			/* resid before uiomove() */
675	uint_t flags;
676	int newpage;
677	int iupdat_flag, directio_status;
678	int n, on, mapon;
679	int error, pagecreate;
680	int do_dqrwlock;		/* drop/reacquire vfs_dqrwlock */
681	int32_t	iblocks;
682	int	new_iblocks;
683
684	/*
685	 * ip->i_size is incremented before the uiomove
686	 * is done on a write.  If the move fails (bad user
687	 * address) reset ip->i_size.
688	 * The better way would be to increment ip->i_size
689	 * only if the uiomove succeeds.
690	 */
691	int i_size_changed = 0;
692	o_mode_t type;
693	int i_seq_needed = 0;
694
695	vp = ITOV(ip);
696
697	/*
698	 * check for forced unmount - should not happen as
699	 * the request passed the lockfs checks.
700	 */
701	if ((ufsvfsp = ip->i_ufsvfs) == NULL)
702		return (EIO);
703
704	fs = ip->i_fs;
705
706	ASSERT(RW_WRITE_HELD(&ip->i_contents));
707
708	/* check for valid filetype */
709	type = ip->i_mode & IFMT;
710	if ((type != IFREG) && (type != IFDIR) && (type != IFATTRDIR) &&
711	    (type != IFLNK) && (type != IFSHAD)) {
712		return (EIO);
713	}
714
715	/*
716	 * the actual limit of UFS file size
717	 * is UFS_MAXOFFSET_T
718	 */
719	if (limit == RLIM64_INFINITY || limit > MAXOFFSET_T)
720		limit = MAXOFFSET_T;
721
722	if (uio->uio_loffset >= limit) {
723		proc_t *p = ttoproc(curthread);
724
725		mutex_enter(&p->p_lock);
726		(void) rctl_action(rctlproc_legacy[RLIMIT_FSIZE], p->p_rctls,
727		    p, RCA_UNSAFE_SIGINFO);
728		mutex_exit(&p->p_lock);
729		return (EFBIG);
730	}
731
732	/*
733	 * if largefiles are disallowed, the limit is
734	 * the pre-largefiles value of 2GB
735	 */
736	if (ufsvfsp->vfs_lfflags & UFS_LARGEFILES)
737		limit = MIN(UFS_MAXOFFSET_T, limit);
738	else
739		limit = MIN(MAXOFF32_T, limit);
740
741	if (uio->uio_loffset < (offset_t)0) {
742		return (EINVAL);
743	}
744	if (uio->uio_resid == 0) {
745		return (0);
746	}
747
748	if (uio->uio_loffset >= limit)
749		return (EFBIG);
750
751	ip->i_flag |= INOACC;	/* don't update ref time in getpage */
752
753	if (ioflag & (FSYNC|FDSYNC)) {
754		ip->i_flag |= ISYNC;
755		iupdat_flag = 1;
756	}
757	/*
758	 * Try to go direct
759	 */
760	if (ip->i_flag & IDIRECTIO || ufsvfsp->vfs_forcedirectio) {
761		uio->uio_llimit = limit;
762		error = ufs_directio_write(ip, uio, ioflag, 0, cr,
763		    &directio_status);
764		/*
765		 * If ufs_directio wrote to the file or set the flags,
766		 * we need to update i_seq, but it may be deferred.
767		 */
768		if (start_resid != uio->uio_resid ||
769		    (ip->i_flag & (ICHG|IUPD))) {
770			i_seq_needed = 1;
771			ip->i_flag |= ISEQ;
772		}
773		if (directio_status == DIRECTIO_SUCCESS)
774			goto out;
775	}
776
777	/*
778	 * Behavior with respect to dropping/reacquiring vfs_dqrwlock:
779	 *
780	 * o shadow inodes: vfs_dqrwlock is not held at all
781	 * o quota updates: vfs_dqrwlock is read or write held
782	 * o other updates: vfs_dqrwlock is read held
783	 *
784	 * The first case is the only one where we do not hold
785	 * vfs_dqrwlock at all while entering wrip().
786	 * We must make sure not to downgrade/drop vfs_dqrwlock if we
787	 * have it as writer, i.e. if we are updating the quota inode.
788	 * There is no potential deadlock scenario in this case as
789	 * ufs_getpage() takes care of this and avoids reacquiring
790	 * vfs_dqrwlock in that case.
791	 *
792	 * This check is done here since the above conditions do not change
793	 * and we possibly loop below, so save a few cycles.
794	 */
795	if ((type == IFSHAD) ||
796	    (rw_owner(&ufsvfsp->vfs_dqrwlock) == curthread)) {
797		do_dqrwlock = 0;
798	} else {
799		do_dqrwlock = 1;
800	}
801
802	/*
803	 * Large Files: We cast MAXBMASK to offset_t
804	 * inorder to mask out the higher bits. Since offset_t
805	 * is a signed value, the high order bit set in MAXBMASK
806	 * value makes it do the right thing by having all bits 1
807	 * in the higher word. May be removed for _SOLARIS64_.
808	 */
809
810	fs = ip->i_fs;
811	do {
812		u_offset_t uoff = uio->uio_loffset;
813		off = uoff & (offset_t)MAXBMASK;
814		mapon = (int)(uoff & (offset_t)MAXBOFFSET);
815		on = (int)blkoff(fs, uoff);
816		n = (int)MIN(fs->fs_bsize - on, uio->uio_resid);
817		new_iblocks = 1;
818
819		if (type == IFREG && uoff + n >= limit) {
820			if (uoff >= limit) {
821				error = EFBIG;
822				goto out;
823			}
824			/*
825			 * since uoff + n >= limit,
826			 * therefore n >= limit - uoff, and n is an int
827			 * so it is safe to cast it to an int
828			 */
829			n = (int)(limit - (rlim64_t)uoff);
830		}
831		if (uoff + n > ip->i_size) {
832			/*
833			 * We are extending the length of the file.
834			 * bmap is used so that we are sure that
835			 * if we need to allocate new blocks, that it
836			 * is done here before we up the file size.
837			 */
838			error = bmap_write(ip, uoff, (int)(on + n),
839			    mapon == 0, NULL, cr);
840			/*
841			 * bmap_write never drops i_contents so if
842			 * the flags are set it changed the file.
843			 */
844			if (ip->i_flag & (ICHG|IUPD)) {
845				i_seq_needed = 1;
846				ip->i_flag |= ISEQ;
847			}
848			if (error)
849				break;
850			/*
851			 * There is a window of vulnerability here.
852			 * The sequence of operations: allocate file
853			 * system blocks, uiomove the data into pages,
854			 * and then update the size of the file in the
855			 * inode, must happen atomically.  However, due
856			 * to current locking constraints, this can not
857			 * be done.
858			 */
859			ASSERT(ip->i_writer == NULL);
860			ip->i_writer = curthread;
861			i_size_changed = 1;
862			/*
863			 * If we are writing from the beginning of
864			 * the mapping, we can just create the
865			 * pages without having to read them.
866			 */
867			pagecreate = (mapon == 0);
868		} else if (n == MAXBSIZE) {
869			/*
870			 * Going to do a whole mappings worth,
871			 * so we can just create the pages w/o
872			 * having to read them in.  But before
873			 * we do that, we need to make sure any
874			 * needed blocks are allocated first.
875			 */
876			iblocks = ip->i_blocks;
877			error = bmap_write(ip, uoff, (int)(on + n),
878			    BI_ALLOC_ONLY, NULL, cr);
879			/*
880			 * bmap_write never drops i_contents so if
881			 * the flags are set it changed the file.
882			 */
883			if (ip->i_flag & (ICHG|IUPD)) {
884				i_seq_needed = 1;
885				ip->i_flag |= ISEQ;
886			}
887			if (error)
888				break;
889			pagecreate = 1;
890			/*
891			 * check if the new created page needed the
892			 * allocation of new disk blocks.
893			 */
894			if (iblocks == ip->i_blocks)
895				new_iblocks = 0; /* no new blocks allocated */
896		} else {
897			pagecreate = 0;
898			/*
899			 * In sync mode flush the indirect blocks which
900			 * may have been allocated and not written on
901			 * disk. In above cases bmap_write will allocate
902			 * in sync mode.
903			 */
904			if (ioflag & (FSYNC|FDSYNC)) {
905				error = ufs_indirblk_sync(ip, uoff);
906				if (error)
907					break;
908			}
909		}
910
911		/*
912		 * At this point we can enter ufs_getpage() in one
913		 * of two ways:
914		 * 1) segmap_getmapflt() calls ufs_getpage() when the
915		 *    forcefault parameter is true (pagecreate == 0)
916		 * 2) uiomove() causes a page fault.
917		 *
918		 * We have to drop the contents lock to prevent the VM
919		 * system from trying to reacquire it in ufs_getpage()
920		 * should the uiomove cause a pagefault.
921		 *
922		 * We have to drop the reader vfs_dqrwlock here as well.
923		 */
924		rw_exit(&ip->i_contents);
925		if (do_dqrwlock) {
926			ASSERT(RW_LOCK_HELD(&ufsvfsp->vfs_dqrwlock));
927			ASSERT(!(RW_WRITE_HELD(&ufsvfsp->vfs_dqrwlock)));
928			rw_exit(&ufsvfsp->vfs_dqrwlock);
929		}
930
931		newpage = 0;
932		premove_resid = uio->uio_resid;
933
934		/*
935		 * Touch the page and fault it in if it is not in core
936		 * before segmap_getmapflt or vpm_data_copy can lock it.
937		 * This is to avoid the deadlock if the buffer is mapped
938		 * to the same file through mmap which we want to write.
939		 */
940		uio_prefaultpages((long)n, uio);
941
942		if (vpm_enable) {
943			/*
944			 * Copy data. If new pages are created, part of
945			 * the page that is not written will be initizliazed
946			 * with zeros.
947			 */
948			error = vpm_data_copy(vp, (off + mapon), (uint_t)n,
949			    uio, !pagecreate, &newpage, 0, S_WRITE);
950		} else {
951
952			base = segmap_getmapflt(segkmap, vp, (off + mapon),
953			    (uint_t)n, !pagecreate, S_WRITE);
954
955			/*
956			 * segmap_pagecreate() returns 1 if it calls
957			 * page_create_va() to allocate any pages.
958			 */
959
960			if (pagecreate)
961				newpage = segmap_pagecreate(segkmap, base,
962				    (size_t)n, 0);
963
964			error = uiomove(base + mapon, (long)n, UIO_WRITE, uio);
965		}
966
967		/*
968		 * If "newpage" is set, then a new page was created and it
969		 * does not contain valid data, so it needs to be initialized
970		 * at this point.
971		 * Otherwise the page contains old data, which was overwritten
972		 * partially or as a whole in uiomove.
973		 * If there is only one iovec structure within uio, then
974		 * on error uiomove will not be able to update uio->uio_loffset
975		 * and we would zero the whole page here!
976		 *
977		 * If uiomove fails because of an error, the old valid data
978		 * is kept instead of filling the rest of the page with zero's.
979		 */
980		if (!vpm_enable && newpage &&
981		    uio->uio_loffset < roundup(off + mapon + n, PAGESIZE)) {
982			/*
983			 * We created pages w/o initializing them completely,
984			 * thus we need to zero the part that wasn't set up.
985			 * This happens on most EOF write cases and if
986			 * we had some sort of error during the uiomove.
987			 */
988			int nzero, nmoved;
989
990			nmoved = (int)(uio->uio_loffset - (off + mapon));
991			ASSERT(nmoved >= 0 && nmoved <= n);
992			nzero = roundup(on + n, PAGESIZE) - nmoved;
993			ASSERT(nzero > 0 && mapon + nmoved + nzero <= MAXBSIZE);
994			(void) kzero(base + mapon + nmoved, (uint_t)nzero);
995		}
996
997		/*
998		 * Unlock the pages allocated by page_create_va()
999		 * in segmap_pagecreate()
1000		 */
1001		if (!vpm_enable && newpage)
1002			segmap_pageunlock(segkmap, base, (size_t)n, S_WRITE);
1003
1004		/*
1005		 * If the size of the file changed, then update the
1006		 * size field in the inode now.  This can't be done
1007		 * before the call to segmap_pageunlock or there is
1008		 * a potential deadlock with callers to ufs_putpage().
1009		 * They will be holding i_contents and trying to lock
1010		 * a page, while this thread is holding a page locked
1011		 * and trying to acquire i_contents.
1012		 */
1013		if (i_size_changed) {
1014			rw_enter(&ip->i_contents, RW_WRITER);
1015			old_i_size = ip->i_size;
1016			UFS_SET_ISIZE(uoff + n, ip);
1017			TRANS_INODE(ufsvfsp, ip);
1018			/*
1019			 * file has grown larger than 2GB. Set flag
1020			 * in superblock to indicate this, if it
1021			 * is not already set.
1022			 */
1023			if ((ip->i_size > MAXOFF32_T) &&
1024			    !(fs->fs_flags & FSLARGEFILES)) {
1025				ASSERT(ufsvfsp->vfs_lfflags & UFS_LARGEFILES);
1026				mutex_enter(&ufsvfsp->vfs_lock);
1027				fs->fs_flags |= FSLARGEFILES;
1028				ufs_sbwrite(ufsvfsp);
1029				mutex_exit(&ufsvfsp->vfs_lock);
1030			}
1031			mutex_enter(&ip->i_tlock);
1032			ip->i_writer = NULL;
1033			cv_broadcast(&ip->i_wrcv);
1034			mutex_exit(&ip->i_tlock);
1035			rw_exit(&ip->i_contents);
1036		}
1037
1038		if (error) {
1039			/*
1040			 * If we failed on a write, we may have already
1041			 * allocated file blocks as well as pages.  It's
1042			 * hard to undo the block allocation, but we must
1043			 * be sure to invalidate any pages that may have
1044			 * been allocated.
1045			 *
1046			 * If the page was created without initialization
1047			 * then we must check if it should be possible
1048			 * to destroy the new page and to keep the old data
1049			 * on the disk.
1050			 *
1051			 * It is possible to destroy the page without
1052			 * having to write back its contents only when
1053			 * - the size of the file keeps unchanged
1054			 * - bmap_write() did not allocate new disk blocks
1055			 *   it is possible to create big files using "seek" and
1056			 *   write to the end of the file. A "write" to a
1057			 *   position before the end of the file would not
1058			 *   change the size of the file but it would allocate
1059			 *   new disk blocks.
1060			 * - uiomove intended to overwrite the whole page.
1061			 * - a new page was created (newpage == 1).
1062			 */
1063
1064			if (i_size_changed == 0 && new_iblocks == 0 &&
1065			    newpage) {
1066
1067				/* unwind what uiomove eventually last did */
1068				uio->uio_resid = premove_resid;
1069
1070				/*
1071				 * destroy the page, do not write ambiguous
1072				 * data to the disk.
1073				 */
1074				flags = SM_DESTROY;
1075			} else {
1076				/*
1077				 * write the page back to the disk, if dirty,
1078				 * and remove the page from the cache.
1079				 */
1080				flags = SM_INVAL;
1081			}
1082
1083			if (vpm_enable) {
1084				/*
1085				 *  Flush pages.
1086				 */
1087				(void) vpm_sync_pages(vp, off, n, flags);
1088			} else {
1089				(void) segmap_release(segkmap, base, flags);
1090			}
1091		} else {
1092			flags = 0;
1093			/*
1094			 * Force write back for synchronous write cases.
1095			 */
1096			if ((ioflag & (FSYNC|FDSYNC)) || type == IFDIR) {
1097				/*
1098				 * If the sticky bit is set but the
1099				 * execute bit is not set, we do a
1100				 * synchronous write back and free
1101				 * the page when done.  We set up swap
1102				 * files to be handled this way to
1103				 * prevent servers from keeping around
1104				 * the client's swap pages too long.
1105				 * XXX - there ought to be a better way.
1106				 */
1107				if (IS_SWAPVP(vp)) {
1108					flags = SM_WRITE | SM_FREE |
1109					    SM_DONTNEED;
1110					iupdat_flag = 0;
1111				} else {
1112					flags = SM_WRITE;
1113				}
1114			} else if (n + on == MAXBSIZE || IS_SWAPVP(vp)) {
1115				/*
1116				 * Have written a whole block.
1117				 * Start an asynchronous write and
1118				 * mark the buffer to indicate that
1119				 * it won't be needed again soon.
1120				 */
1121				flags = SM_WRITE | SM_ASYNC | SM_DONTNEED;
1122			}
1123			if (vpm_enable) {
1124				/*
1125				 * Flush pages.
1126				 */
1127				error = vpm_sync_pages(vp, off, n, flags);
1128			} else {
1129				error = segmap_release(segkmap, base, flags);
1130			}
1131			/*
1132			 * If the operation failed and is synchronous,
1133			 * then we need to unwind what uiomove() last
1134			 * did so we can potentially return an error to
1135			 * the caller.  If this write operation was
1136			 * done in two pieces and the first succeeded,
1137			 * then we won't return an error for the second
1138			 * piece that failed.  However, we only want to
1139			 * return a resid value that reflects what was
1140			 * really done.
1141			 *
1142			 * Failures for non-synchronous operations can
1143			 * be ignored since the page subsystem will
1144			 * retry the operation until it succeeds or the
1145			 * file system is unmounted.
1146			 */
1147			if (error) {
1148				if ((ioflag & (FSYNC | FDSYNC)) ||
1149				    type == IFDIR) {
1150					uio->uio_resid = premove_resid;
1151				} else {
1152					error = 0;
1153				}
1154			}
1155		}
1156
1157		/*
1158		 * Re-acquire contents lock.
1159		 * If it was dropped, reacquire reader vfs_dqrwlock as well.
1160		 */
1161		if (do_dqrwlock)
1162			rw_enter(&ufsvfsp->vfs_dqrwlock, RW_READER);
1163		rw_enter(&ip->i_contents, RW_WRITER);
1164
1165		/*
1166		 * If the uiomove() failed or if a synchronous
1167		 * page push failed, fix up i_size.
1168		 */
1169		if (error) {
1170			if (i_size_changed) {
1171				/*
1172				 * The uiomove failed, and we
1173				 * allocated blocks,so get rid
1174				 * of them.
1175				 */
1176				(void) ufs_itrunc(ip, old_i_size, 0, cr);
1177			}
1178		} else {
1179			/*
1180			 * XXX - Can this be out of the loop?
1181			 */
1182			ip->i_flag |= IUPD | ICHG;
1183			/*
1184			 * Only do one increase of i_seq for multiple
1185			 * pieces.  Because we drop locks, record
1186			 * the fact that we changed the timestamp and
1187			 * are deferring the increase in case another thread
1188			 * pushes our timestamp update.
1189			 */
1190			i_seq_needed = 1;
1191			ip->i_flag |= ISEQ;
1192			if (i_size_changed)
1193				ip->i_flag |= IATTCHG;
1194			if ((ip->i_mode & (IEXEC | (IEXEC >> 3) |
1195			    (IEXEC >> 6))) != 0 &&
1196			    (ip->i_mode & (ISUID | ISGID)) != 0 &&
1197			    secpolicy_vnode_setid_retain(cr,
1198			    (ip->i_mode & ISUID) != 0 && ip->i_uid == 0) != 0) {
1199				/*
1200				 * Clear Set-UID & Set-GID bits on
1201				 * successful write if not privileged
1202				 * and at least one of the execute bits
1203				 * is set.  If we always clear Set-GID,
1204				 * mandatory file and record locking is
1205				 * unuseable.
1206				 */
1207				ip->i_mode &= ~(ISUID | ISGID);
1208			}
1209		}
1210		/*
1211		 * In the case the FDSYNC flag is set and this is a
1212		 * "rewrite" we won't log a delta.
1213		 * The FSYNC flag overrides all cases.
1214		 */
1215		if (!ufs_check_rewrite(ip, uio, ioflag) || !(ioflag & FDSYNC)) {
1216			TRANS_INODE(ufsvfsp, ip);
1217		}
1218	} while (error == 0 && uio->uio_resid > 0 && n != 0);
1219
1220out:
1221	/*
1222	 * Make sure i_seq is increased at least once per write
1223	 */
1224	if (i_seq_needed) {
1225		ip->i_seq++;
1226		ip->i_flag &= ~ISEQ;	/* no longer deferred */
1227	}
1228
1229	/*
1230	 * Inode is updated according to this table -
1231	 *
1232	 *   FSYNC	  FDSYNC(posix.4)
1233	 *   --------------------------
1234	 *   always@	  IATTCHG|IBDWRITE
1235	 *
1236	 * @ -	If we are doing synchronous write the only time we should
1237	 *	not be sync'ing the ip here is if we have the stickyhack
1238	 *	activated, the file is marked with the sticky bit and
1239	 *	no exec bit, the file length has not been changed and
1240	 *	no new blocks have been allocated during this write.
1241	 */
1242
1243	if ((ip->i_flag & ISYNC) != 0) {
1244		/*
1245		 * we have eliminated nosync
1246		 */
1247		if ((ip->i_flag & (IATTCHG|IBDWRITE)) ||
1248		    ((ioflag & FSYNC) && iupdat_flag)) {
1249			ufs_iupdat(ip, 1);
1250		}
1251	}
1252
1253	/*
1254	 * If we've already done a partial-write, terminate
1255	 * the write but return no error unless the error is ENOSPC
1256	 * because the caller can detect this and free resources and
1257	 * try again.
1258	 */
1259	if ((start_resid != uio->uio_resid) && (error != ENOSPC))
1260		error = 0;
1261
1262	ip->i_flag &= ~(INOACC | ISYNC);
1263	ITIMES_NOLOCK(ip);
1264	return (error);
1265}
1266
1267/*
1268 * rdip does the real work of read requests for ufs.
1269 */
1270int
1271rdip(struct inode *ip, struct uio *uio, int ioflag, cred_t *cr)
1272{
1273	u_offset_t off;
1274	caddr_t base;
1275	struct fs *fs;
1276	struct ufsvfs *ufsvfsp;
1277	struct vnode *vp;
1278	long oresid = uio->uio_resid;
1279	u_offset_t n, on, mapon;
1280	int error = 0;
1281	int doupdate = 1;
1282	uint_t flags;
1283	int directio_status;
1284	krw_t rwtype;
1285	o_mode_t type;
1286
1287	vp = ITOV(ip);
1288
1289	ASSERT(RW_LOCK_HELD(&ip->i_contents));
1290
1291	ufsvfsp = ip->i_ufsvfs;
1292
1293	if (ufsvfsp == NULL)
1294		return (EIO);
1295
1296	fs = ufsvfsp->vfs_fs;
1297
1298	/* check for valid filetype */
1299	type = ip->i_mode & IFMT;
1300	if ((type != IFREG) && (type != IFDIR) && (type != IFATTRDIR) &&
1301	    (type != IFLNK) && (type != IFSHAD)) {
1302		return (EIO);
1303	}
1304
1305	if (uio->uio_loffset > UFS_MAXOFFSET_T) {
1306		error = 0;
1307		goto out;
1308	}
1309	if (uio->uio_loffset < (offset_t)0) {
1310		return (EINVAL);
1311	}
1312	if (uio->uio_resid == 0) {
1313		return (0);
1314	}
1315
1316	if (!ULOCKFS_IS_NOIACC(ITOUL(ip)) && (fs->fs_ronly == 0) &&
1317	    (!ufsvfsp->vfs_noatime)) {
1318		mutex_enter(&ip->i_tlock);
1319		ip->i_flag |= IACC;
1320		mutex_exit(&ip->i_tlock);
1321	}
1322	/*
1323	 * Try to go direct
1324	 */
1325	if (ip->i_flag & IDIRECTIO || ufsvfsp->vfs_forcedirectio) {
1326		error = ufs_directio_read(ip, uio, cr, &directio_status);
1327		if (directio_status == DIRECTIO_SUCCESS)
1328			goto out;
1329	}
1330
1331	rwtype = (rw_write_held(&ip->i_contents)?RW_WRITER:RW_READER);
1332
1333	do {
1334		offset_t diff;
1335		u_offset_t uoff = uio->uio_loffset;
1336		off = uoff & (offset_t)MAXBMASK;
1337		mapon = (u_offset_t)(uoff & (offset_t)MAXBOFFSET);
1338		on = (u_offset_t)blkoff(fs, uoff);
1339		n = MIN((u_offset_t)fs->fs_bsize - on,
1340		    (u_offset_t)uio->uio_resid);
1341
1342		diff = ip->i_size - uoff;
1343
1344		if (diff <= (offset_t)0) {
1345			error = 0;
1346			goto out;
1347		}
1348		if (diff < (offset_t)n)
1349			n = (int)diff;
1350
1351		/*
1352		 * At this point we can enter ufs_getpage() in one of two
1353		 * ways:
1354		 * 1) segmap_getmapflt() calls ufs_getpage() when the
1355		 *    forcefault parameter is true (value of 1 is passed)
1356		 * 2) uiomove() causes a page fault.
1357		 *
1358		 * We cannot hold onto an i_contents reader lock without
1359		 * risking deadlock in ufs_getpage() so drop a reader lock.
1360		 * The ufs_getpage() dolock logic already allows for a
1361		 * thread holding i_contents as writer to work properly
1362		 * so we keep a writer lock.
1363		 */
1364		if (rwtype == RW_READER)
1365			rw_exit(&ip->i_contents);
1366
1367		if (vpm_enable) {
1368			/*
1369			 * Copy data.
1370			 */
1371			error = vpm_data_copy(vp, (off + mapon), (uint_t)n,
1372			    uio, 1, NULL, 0, S_READ);
1373		} else {
1374			base = segmap_getmapflt(segkmap, vp, (off + mapon),
1375			    (uint_t)n, 1, S_READ);
1376			error = uiomove(base + mapon, (long)n, UIO_READ, uio);
1377		}
1378
1379		flags = 0;
1380		if (!error) {
1381			/*
1382			 * In POSIX SYNC (FSYNC and FDSYNC) read mode,
1383			 * we want to make sure that the page which has
1384			 * been read, is written on disk if it is dirty.
1385			 * And corresponding indirect blocks should also
1386			 * be flushed out.
1387			 */
1388			if ((ioflag & FRSYNC) && (ioflag & (FSYNC|FDSYNC))) {
1389				flags |= SM_WRITE;
1390			}
1391			if (vpm_enable) {
1392				error = vpm_sync_pages(vp, off, n, flags);
1393			} else {
1394				error = segmap_release(segkmap, base, flags);
1395			}
1396		} else {
1397			if (vpm_enable) {
1398				(void) vpm_sync_pages(vp, off, n, flags);
1399			} else {
1400				(void) segmap_release(segkmap, base, flags);
1401			}
1402		}
1403
1404		if (rwtype == RW_READER)
1405			rw_enter(&ip->i_contents, rwtype);
1406	} while (error == 0 && uio->uio_resid > 0 && n != 0);
1407out:
1408	/*
1409	 * Inode is updated according to this table if FRSYNC is set.
1410	 *
1411	 *   FSYNC	  FDSYNC(posix.4)
1412	 *   --------------------------
1413	 *   always	  IATTCHG|IBDWRITE
1414	 */
1415	/*
1416	 * The inode is not updated if we're logging and the inode is a
1417	 * directory with FRSYNC, FSYNC and FDSYNC flags set.
1418	 */
1419	if (ioflag & FRSYNC) {
1420		if (TRANS_ISTRANS(ufsvfsp) && ((ip->i_mode & IFMT) == IFDIR)) {
1421			doupdate = 0;
1422		}
1423		if (doupdate) {
1424			if ((ioflag & FSYNC) ||
1425			    ((ioflag & FDSYNC) &&
1426			    (ip->i_flag & (IATTCHG|IBDWRITE)))) {
1427				ufs_iupdat(ip, 1);
1428			}
1429		}
1430	}
1431	/*
1432	 * If we've already done a partial read, terminate
1433	 * the read but return no error.
1434	 */
1435	if (oresid != uio->uio_resid)
1436		error = 0;
1437	ITIMES(ip);
1438
1439	return (error);
1440}
1441
1442/* ARGSUSED */
1443static int
1444ufs_ioctl(
1445	struct vnode	*vp,
1446	int		cmd,
1447	intptr_t	arg,
1448	int		flag,
1449	struct cred	*cr,
1450	int		*rvalp,
1451	caller_context_t *ct)
1452{
1453	struct lockfs	lockfs, lockfs_out;
1454	struct ufsvfs	*ufsvfsp = VTOI(vp)->i_ufsvfs;
1455	char		*comment, *original_comment;
1456	struct fs	*fs;
1457	struct ulockfs	*ulp;
1458	offset_t	off;
1459	extern int	maxphys;
1460	int		error;
1461	int		issync;
1462	int		trans_size;
1463
1464
1465	/*
1466	 * forcibly unmounted
1467	 */
1468	if (ufsvfsp == NULL || vp->v_vfsp == NULL ||
1469	    vp->v_vfsp->vfs_flag & VFS_UNMOUNTED)
1470		return (EIO);
1471	fs = ufsvfsp->vfs_fs;
1472
1473	if (cmd == Q_QUOTACTL) {
1474		error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_QUOTA_MASK);
1475		if (error)
1476			return (error);
1477
1478		if (ulp) {
1479			TRANS_BEGIN_ASYNC(ufsvfsp, TOP_QUOTA,
1480			    TOP_SETQUOTA_SIZE(fs));
1481		}
1482
1483		error = quotactl(vp, arg, flag, cr);
1484
1485		if (ulp) {
1486			TRANS_END_ASYNC(ufsvfsp, TOP_QUOTA,
1487			    TOP_SETQUOTA_SIZE(fs));
1488			ufs_lockfs_end(ulp);
1489		}
1490		return (error);
1491	}
1492
1493	switch (cmd) {
1494		case _FIOLFS:
1495			/*
1496			 * file system locking
1497			 */
1498			if (secpolicy_fs_config(cr, ufsvfsp->vfs_vfs) != 0)
1499				return (EPERM);
1500
1501			if ((flag & DATAMODEL_MASK) == DATAMODEL_NATIVE) {
1502				if (copyin((caddr_t)arg, &lockfs,
1503				    sizeof (struct lockfs)))
1504					return (EFAULT);
1505			}
1506#ifdef _SYSCALL32_IMPL
1507			else {
1508				struct lockfs32	lockfs32;
1509				/* Translate ILP32 lockfs to LP64 lockfs */
1510				if (copyin((caddr_t)arg, &lockfs32,
1511				    sizeof (struct lockfs32)))
1512					return (EFAULT);
1513				lockfs.lf_lock = (ulong_t)lockfs32.lf_lock;
1514				lockfs.lf_flags = (ulong_t)lockfs32.lf_flags;
1515				lockfs.lf_key = (ulong_t)lockfs32.lf_key;
1516				lockfs.lf_comlen = (ulong_t)lockfs32.lf_comlen;
1517				lockfs.lf_comment =
1518				    (caddr_t)(uintptr_t)lockfs32.lf_comment;
1519			}
1520#endif /* _SYSCALL32_IMPL */
1521
1522			if (lockfs.lf_comlen) {
1523				if (lockfs.lf_comlen > LOCKFS_MAXCOMMENTLEN)
1524					return (ENAMETOOLONG);
1525				comment =
1526				    kmem_alloc(lockfs.lf_comlen, KM_SLEEP);
1527				if (copyin(lockfs.lf_comment, comment,
1528				    lockfs.lf_comlen)) {
1529					kmem_free(comment, lockfs.lf_comlen);
1530					return (EFAULT);
1531				}
1532				original_comment = lockfs.lf_comment;
1533				lockfs.lf_comment = comment;
1534			}
1535			if ((error = ufs_fiolfs(vp, &lockfs, 0)) == 0) {
1536				lockfs.lf_comment = original_comment;
1537
1538				if ((flag & DATAMODEL_MASK) ==
1539				    DATAMODEL_NATIVE) {
1540					(void) copyout(&lockfs, (caddr_t)arg,
1541					    sizeof (struct lockfs));
1542				}
1543#ifdef _SYSCALL32_IMPL
1544				else {
1545					struct lockfs32	lockfs32;
1546					/* Translate LP64 to ILP32 lockfs */
1547					lockfs32.lf_lock =
1548					    (uint32_t)lockfs.lf_lock;
1549					lockfs32.lf_flags =
1550					    (uint32_t)lockfs.lf_flags;
1551					lockfs32.lf_key =
1552					    (uint32_t)lockfs.lf_key;
1553					lockfs32.lf_comlen =
1554					    (uint32_t)lockfs.lf_comlen;
1555					lockfs32.lf_comment =
1556					    (uint32_t)(uintptr_t)
1557					    lockfs.lf_comment;
1558					(void) copyout(&lockfs32, (caddr_t)arg,
1559					    sizeof (struct lockfs32));
1560				}
1561#endif /* _SYSCALL32_IMPL */
1562
1563			} else {
1564				if (lockfs.lf_comlen)
1565					kmem_free(comment, lockfs.lf_comlen);
1566			}
1567			return (error);
1568
1569		case _FIOLFSS:
1570			/*
1571			 * get file system locking status
1572			 */
1573
1574			if ((flag & DATAMODEL_MASK) == DATAMODEL_NATIVE) {
1575				if (copyin((caddr_t)arg, &lockfs,
1576				    sizeof (struct lockfs)))
1577					return (EFAULT);
1578			}
1579#ifdef _SYSCALL32_IMPL
1580			else {
1581				struct lockfs32	lockfs32;
1582				/* Translate ILP32 lockfs to LP64 lockfs */
1583				if (copyin((caddr_t)arg, &lockfs32,
1584				    sizeof (struct lockfs32)))
1585					return (EFAULT);
1586				lockfs.lf_lock = (ulong_t)lockfs32.lf_lock;
1587				lockfs.lf_flags = (ulong_t)lockfs32.lf_flags;
1588				lockfs.lf_key = (ulong_t)lockfs32.lf_key;
1589				lockfs.lf_comlen = (ulong_t)lockfs32.lf_comlen;
1590				lockfs.lf_comment =
1591				    (caddr_t)(uintptr_t)lockfs32.lf_comment;
1592			}
1593#endif /* _SYSCALL32_IMPL */
1594
1595			if (error =  ufs_fiolfss(vp, &lockfs_out))
1596				return (error);
1597			lockfs.lf_lock = lockfs_out.lf_lock;
1598			lockfs.lf_key = lockfs_out.lf_key;
1599			lockfs.lf_flags = lockfs_out.lf_flags;
1600			lockfs.lf_comlen = MIN(lockfs.lf_comlen,
1601			    lockfs_out.lf_comlen);
1602
1603			if ((flag & DATAMODEL_MASK) == DATAMODEL_NATIVE) {
1604				if (copyout(&lockfs, (caddr_t)arg,
1605				    sizeof (struct lockfs)))
1606					return (EFAULT);
1607			}
1608#ifdef _SYSCALL32_IMPL
1609			else {
1610				/* Translate LP64 to ILP32 lockfs */
1611				struct lockfs32	lockfs32;
1612				lockfs32.lf_lock = (uint32_t)lockfs.lf_lock;
1613				lockfs32.lf_flags = (uint32_t)lockfs.lf_flags;
1614				lockfs32.lf_key = (uint32_t)lockfs.lf_key;
1615				lockfs32.lf_comlen = (uint32_t)lockfs.lf_comlen;
1616				lockfs32.lf_comment =
1617				    (uint32_t)(uintptr_t)lockfs.lf_comment;
1618				if (copyout(&lockfs32, (caddr_t)arg,
1619				    sizeof (struct lockfs32)))
1620					return (EFAULT);
1621			}
1622#endif /* _SYSCALL32_IMPL */
1623
1624			if (lockfs.lf_comlen &&
1625			    lockfs.lf_comment && lockfs_out.lf_comment)
1626				if (copyout(lockfs_out.lf_comment,
1627				    lockfs.lf_comment, lockfs.lf_comlen))
1628					return (EFAULT);
1629			return (0);
1630
1631		case _FIOSATIME:
1632			/*
1633			 * set access time
1634			 */
1635
1636			/*
1637			 * if mounted w/o atime, return quietly.
1638			 * I briefly thought about returning ENOSYS, but
1639			 * figured that most apps would consider this fatal
1640			 * but the idea is to make this as seamless as poss.
1641			 */
1642			if (ufsvfsp->vfs_noatime)
1643				return (0);
1644
1645			error = ufs_lockfs_begin(ufsvfsp, &ulp,
1646			    ULOCKFS_SETATTR_MASK);
1647			if (error)
1648				return (error);
1649
1650			if (ulp) {
1651				trans_size = (int)TOP_SETATTR_SIZE(VTOI(vp));
1652				TRANS_BEGIN_CSYNC(ufsvfsp, issync,
1653				    TOP_SETATTR, trans_size);
1654			}
1655
1656			error = ufs_fiosatime(vp, (struct timeval *)arg,
1657			    flag, cr);
1658
1659			if (ulp) {
1660				TRANS_END_CSYNC(ufsvfsp, error, issync,
1661				    TOP_SETATTR, trans_size);
1662				ufs_lockfs_end(ulp);
1663			}
1664			return (error);
1665
1666		case _FIOSDIO:
1667			/*
1668			 * set delayed-io
1669			 */
1670			return (ufs_fiosdio(vp, (uint_t *)arg, flag, cr));
1671
1672		case _FIOGDIO:
1673			/*
1674			 * get delayed-io
1675			 */
1676			return (ufs_fiogdio(vp, (uint_t *)arg, flag, cr));
1677
1678		case _FIOIO:
1679			/*
1680			 * inode open
1681			 */
1682			error = ufs_lockfs_begin(ufsvfsp, &ulp,
1683			    ULOCKFS_VGET_MASK);
1684			if (error)
1685				return (error);
1686
1687			error = ufs_fioio(vp, (struct fioio *)arg, flag, cr);
1688
1689			if (ulp) {
1690				ufs_lockfs_end(ulp);
1691			}
1692			return (error);
1693
1694		case _FIOFFS:
1695			/*
1696			 * file system flush (push w/invalidate)
1697			 */
1698			if ((caddr_t)arg != NULL)
1699				return (EINVAL);
1700			return (ufs_fioffs(vp, NULL, cr));
1701
1702		case _FIOISBUSY:
1703			/*
1704			 * Contract-private interface for Legato
1705			 * Purge this vnode from the DNLC and decide
1706			 * if this vnode is busy (*arg == 1) or not
1707			 * (*arg == 0)
1708			 */
1709			if (secpolicy_fs_config(cr, ufsvfsp->vfs_vfs) != 0)
1710				return (EPERM);
1711			error = ufs_fioisbusy(vp, (int *)arg, cr);
1712			return (error);
1713
1714		case _FIODIRECTIO:
1715			return (ufs_fiodirectio(vp, (int)arg, cr));
1716
1717		case _FIOTUNE:
1718			/*
1719			 * Tune the file system (aka setting fs attributes)
1720			 */
1721			error = ufs_lockfs_begin(ufsvfsp, &ulp,
1722			    ULOCKFS_SETATTR_MASK);
1723			if (error)
1724				return (error);
1725
1726			error = ufs_fiotune(vp, (struct fiotune *)arg, cr);
1727
1728			if (ulp)
1729				ufs_lockfs_end(ulp);
1730			return (error);
1731
1732		case _FIOLOGENABLE:
1733			if (secpolicy_fs_config(cr, ufsvfsp->vfs_vfs) != 0)
1734				return (EPERM);
1735			return (ufs_fiologenable(vp, (void *)arg, cr, flag));
1736
1737		case _FIOLOGDISABLE:
1738			if (secpolicy_fs_config(cr, ufsvfsp->vfs_vfs) != 0)
1739				return (EPERM);
1740			return (ufs_fiologdisable(vp, (void *)arg, cr, flag));
1741
1742		case _FIOISLOG:
1743			return (ufs_fioislog(vp, (void *)arg, cr, flag));
1744
1745		case _FIOSNAPSHOTCREATE_MULTI:
1746		{
1747			struct fiosnapcreate_multi	fc, *fcp;
1748			size_t	fcm_size;
1749
1750			if (copyin((void *)arg, &fc, sizeof (fc)))
1751				return (EFAULT);
1752			if (fc.backfilecount > MAX_BACKFILE_COUNT)
1753				return (EINVAL);
1754			fcm_size = sizeof (struct fiosnapcreate_multi) +
1755			    (fc.backfilecount - 1) * sizeof (int);
1756			fcp = (struct fiosnapcreate_multi *)
1757			    kmem_alloc(fcm_size, KM_SLEEP);
1758			if (copyin((void *)arg, fcp, fcm_size)) {
1759				kmem_free(fcp, fcm_size);
1760				return (EFAULT);
1761			}
1762			error = ufs_snap_create(vp, fcp, cr);
1763			/*
1764			 * Do copyout even if there is an error because
1765			 * the details of error is stored in fcp.
1766			 */
1767			if (copyout(fcp, (void *)arg, fcm_size))
1768				error = EFAULT;
1769			kmem_free(fcp, fcm_size);
1770			return (error);
1771		}
1772
1773		case _FIOSNAPSHOTDELETE:
1774		{
1775			struct fiosnapdelete	fc;
1776
1777			if (copyin((void *)arg, &fc, sizeof (fc)))
1778				return (EFAULT);
1779			error = ufs_snap_delete(vp, &fc, cr);
1780			if (!error && copyout(&fc, (void *)arg, sizeof (fc)))
1781				error = EFAULT;
1782			return (error);
1783		}
1784
1785		case _FIOGETSUPERBLOCK:
1786			if (copyout(fs, (void *)arg, SBSIZE))
1787				return (EFAULT);
1788			return (0);
1789
1790		case _FIOGETMAXPHYS:
1791			if (copyout(&maxphys, (void *)arg, sizeof (maxphys)))
1792				return (EFAULT);
1793			return (0);
1794
1795		/*
1796		 * The following 3 ioctls are for TSufs support
1797		 * although could potentially be used elsewhere
1798		 */
1799		case _FIO_SET_LUFS_DEBUG:
1800			if (secpolicy_fs_config(cr, ufsvfsp->vfs_vfs) != 0)
1801				return (EPERM);
1802			lufs_debug = (uint32_t)arg;
1803			return (0);
1804
1805		case _FIO_SET_LUFS_ERROR:
1806			if (secpolicy_fs_config(cr, ufsvfsp->vfs_vfs) != 0)
1807				return (EPERM);
1808			TRANS_SETERROR(ufsvfsp);
1809			return (0);
1810
1811		case _FIO_GET_TOP_STATS:
1812		{
1813			fio_lufs_stats_t *ls;
1814			ml_unit_t *ul = ufsvfsp->vfs_log;
1815
1816			ls = kmem_zalloc(sizeof (*ls), KM_SLEEP);
1817			ls->ls_debug = ul->un_debug; /* return debug value */
1818			/* Copy stucture if statistics are being kept */
1819			if (ul->un_logmap->mtm_tops) {
1820				ls->ls_topstats = *(ul->un_logmap->mtm_tops);
1821			}
1822			error = 0;
1823			if (copyout(ls, (void *)arg, sizeof (*ls)))
1824				error = EFAULT;
1825			kmem_free(ls, sizeof (*ls));
1826			return (error);
1827		}
1828
1829		case _FIO_SEEK_DATA:
1830		case _FIO_SEEK_HOLE:
1831			if (ddi_copyin((void *)arg, &off, sizeof (off), flag))
1832				return (EFAULT);
1833			/* offset paramater is in/out */
1834			error = ufs_fio_holey(vp, cmd, &off);
1835			if (error)
1836				return (error);
1837			if (ddi_copyout(&off, (void *)arg, sizeof (off), flag))
1838				return (EFAULT);
1839			return (0);
1840
1841		case _FIO_COMPRESSED:
1842		{
1843			/*
1844			 * This is a project private ufs ioctl() to mark
1845			 * the inode as that belonging to a compressed
1846			 * file. This is used to mark individual
1847			 * compressed files in a miniroot archive.
1848			 * The files compressed in this manner are
1849			 * automatically decompressed by the dcfs filesystem
1850			 * (via an interception in ufs_lookup - see decompvp())
1851			 * which is layered on top of ufs on a system running
1852			 * from the archive. See uts/common/fs/dcfs for details.
1853			 * This ioctl only marks the file as compressed - the
1854			 * actual compression is done by fiocompress (a
1855			 * userland utility) which invokes this ioctl().
1856			 */
1857			struct inode *ip = VTOI(vp);
1858
1859			error = ufs_lockfs_begin(ufsvfsp, &ulp,
1860			    ULOCKFS_SETATTR_MASK);
1861			if (error)
1862				return (error);
1863
1864			if (ulp) {
1865				TRANS_BEGIN_ASYNC(ufsvfsp, TOP_IUPDAT,
1866				    TOP_IUPDAT_SIZE(ip));
1867			}
1868
1869			error = ufs_mark_compressed(vp);
1870
1871			if (ulp) {
1872				TRANS_END_ASYNC(ufsvfsp, TOP_IUPDAT,
1873				    TOP_IUPDAT_SIZE(ip));
1874				ufs_lockfs_end(ulp);
1875			}
1876
1877			return (error);
1878
1879		}
1880
1881		default:
1882			return (ENOTTY);
1883	}
1884}
1885
1886
1887/* ARGSUSED */
1888static int
1889ufs_getattr(struct vnode *vp, struct vattr *vap, int flags,
1890    struct cred *cr, caller_context_t *ct)
1891{
1892	struct inode *ip = VTOI(vp);
1893	struct ufsvfs *ufsvfsp;
1894	int err;
1895
1896	if (vap->va_mask == AT_SIZE) {
1897		/*
1898		 * for performance, if only the size is requested don't bother
1899		 * with anything else.
1900		 */
1901		UFS_GET_ISIZE(&vap->va_size, ip);
1902		return (0);
1903	}
1904
1905	/*
1906	 * inlined lockfs checks
1907	 */
1908	ufsvfsp = ip->i_ufsvfs;
1909	if ((ufsvfsp == NULL) || ULOCKFS_IS_HLOCK(&ufsvfsp->vfs_ulockfs)) {
1910		err = EIO;
1911		goto out;
1912	}
1913
1914	rw_enter(&ip->i_contents, RW_READER);
1915	/*
1916	 * Return all the attributes.  This should be refined so
1917	 * that it only returns what's asked for.
1918	 */
1919
1920	/*
1921	 * Copy from inode table.
1922	 */
1923	vap->va_type = vp->v_type;
1924	vap->va_mode = ip->i_mode & MODEMASK;
1925	/*
1926	 * If there is an ACL and there is a mask entry, then do the
1927	 * extra work that completes the equivalent of an acltomode(3)
1928	 * call.  According to POSIX P1003.1e, the acl mask should be
1929	 * returned in the group permissions field.
1930	 *
1931	 * - start with the original permission and mode bits (from above)
1932	 * - clear the group owner bits
1933	 * - add in the mask bits.
1934	 */
1935	if (ip->i_ufs_acl && ip->i_ufs_acl->aclass.acl_ismask) {
1936		vap->va_mode &= ~((VREAD | VWRITE | VEXEC) >> 3);
1937		vap->va_mode |=
1938		    (ip->i_ufs_acl->aclass.acl_maskbits & PERMMASK) << 3;
1939	}
1940	vap->va_uid = ip->i_uid;
1941	vap->va_gid = ip->i_gid;
1942	vap->va_fsid = ip->i_dev;
1943	vap->va_nodeid = (ino64_t)ip->i_number;
1944	vap->va_nlink = ip->i_nlink;
1945	vap->va_size = ip->i_size;
1946	if (vp->v_type == VCHR || vp->v_type == VBLK)
1947		vap->va_rdev = ip->i_rdev;
1948	else
1949		vap->va_rdev = 0;	/* not a b/c spec. */
1950	mutex_enter(&ip->i_tlock);
1951	ITIMES_NOLOCK(ip);	/* mark correct time in inode */
1952	vap->va_seq = ip->i_seq;
1953	vap->va_atime.tv_sec = (time_t)ip->i_atime.tv_sec;
1954	vap->va_atime.tv_nsec = ip->i_atime.tv_usec*1000;
1955	vap->va_mtime.tv_sec = (time_t)ip->i_mtime.tv_sec;
1956	vap->va_mtime.tv_nsec = ip->i_mtime.tv_usec*1000;
1957	vap->va_ctime.tv_sec = (time_t)ip->i_ctime.tv_sec;
1958	vap->va_ctime.tv_nsec = ip->i_ctime.tv_usec*1000;
1959	mutex_exit(&ip->i_tlock);
1960
1961	switch (ip->i_mode & IFMT) {
1962
1963	case IFBLK:
1964		vap->va_blksize = MAXBSIZE;		/* was BLKDEV_IOSIZE */
1965		break;
1966
1967	case IFCHR:
1968		vap->va_blksize = MAXBSIZE;
1969		break;
1970
1971	default:
1972		vap->va_blksize = ip->i_fs->fs_bsize;
1973		break;
1974	}
1975	vap->va_nblocks = (fsblkcnt64_t)ip->i_blocks;
1976	rw_exit(&ip->i_contents);
1977	err = 0;
1978
1979out:
1980	return (err);
1981}
1982
1983/*
1984 * Special wrapper to provide a callback for secpolicy_vnode_setattr().
1985 * The i_contents lock is already held by the caller and we need to
1986 * declare the inode as 'void *' argument.
1987 */
1988static int
1989ufs_priv_access(void *vip, int mode, struct cred *cr)
1990{
1991	struct inode *ip = vip;
1992
1993	return (ufs_iaccess(ip, mode, cr, 0));
1994}
1995
1996/*ARGSUSED4*/
1997static int
1998ufs_setattr(struct vnode *vp, struct vattr *vap, int flags, struct cred *cr,
1999    caller_context_t *ct)
2000{
2001	struct inode *ip = VTOI(vp);
2002	struct ufsvfs *ufsvfsp = ip->i_ufsvfs;
2003	struct fs *fs;
2004	struct ulockfs *ulp;
2005	char *errmsg1;
2006	char *errmsg2;
2007	long blocks;
2008	long int mask = vap->va_mask;
2009	size_t len1, len2;
2010	int issync;
2011	int trans_size;
2012	int dotrans;
2013	int dorwlock;
2014	int error;
2015	int owner_change;
2016	int dodqlock;
2017	timestruc_t now;
2018	vattr_t oldva;
2019	int retry = 1;
2020	int indeadlock;
2021
2022	/*
2023	 * Cannot set these attributes.
2024	 */
2025	if ((mask & AT_NOSET) || (mask & AT_XVATTR))
2026		return (EINVAL);
2027
2028	/*
2029	 * check for forced unmount
2030	 */
2031	if (ufsvfsp == NULL)
2032		return (EIO);
2033
2034	fs = ufsvfsp->vfs_fs;
2035	if (fs->fs_ronly != 0)
2036		return (EROFS);
2037
2038again:
2039	errmsg1 = NULL;
2040	errmsg2 = NULL;
2041	dotrans = 0;
2042	dorwlock = 0;
2043	dodqlock = 0;
2044
2045	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_SETATTR_MASK);
2046	if (error)
2047		goto out;
2048
2049	/*
2050	 * Acquire i_rwlock before TRANS_BEGIN_CSYNC() if this is a file.
2051	 * This follows the protocol for read()/write().
2052	 */
2053	if (vp->v_type != VDIR) {
2054		/*
2055		 * ufs_tryirwlock uses rw_tryenter and checks for SLOCK to
2056		 * avoid i_rwlock, ufs_lockfs_begin deadlock. If deadlock
2057		 * possible, retries the operation.
2058		 */
2059		ufs_tryirwlock(&ip->i_rwlock, RW_WRITER, retry_file);
2060		if (indeadlock) {
2061			if (ulp)
2062				ufs_lockfs_end(ulp);
2063			goto again;
2064		}
2065		dorwlock = 1;
2066	}
2067
2068	/*
2069	 * Truncate file.  Must have write permission and not be a directory.
2070	 */
2071	if (mask & AT_SIZE) {
2072		rw_enter(&ip->i_contents, RW_WRITER);
2073		if (vp->v_type == VDIR) {
2074			error = EISDIR;
2075			goto update_inode;
2076		}
2077		if (error = ufs_iaccess(ip, IWRITE, cr, 0))
2078			goto update_inode;
2079
2080		rw_exit(&ip->i_contents);
2081		error = TRANS_ITRUNC(ip, vap->va_size, 0, cr);
2082		if (error) {
2083			rw_enter(&ip->i_contents, RW_WRITER);
2084			goto update_inode;
2085		}
2086
2087		if (error == 0 && vap->va_size)
2088			vnevent_truncate(vp, ct);
2089	}
2090
2091	if (ulp) {
2092		trans_size = (int)TOP_SETATTR_SIZE(ip);
2093		TRANS_BEGIN_CSYNC(ufsvfsp, issync, TOP_SETATTR, trans_size);
2094		++dotrans;
2095	}
2096
2097	/*
2098	 * Acquire i_rwlock after TRANS_BEGIN_CSYNC() if this is a directory.
2099	 * This follows the protocol established by
2100	 * ufs_link/create/remove/rename/mkdir/rmdir/symlink.
2101	 */
2102	if (vp->v_type == VDIR) {
2103		ufs_tryirwlock_trans(&ip->i_rwlock, RW_WRITER, TOP_SETATTR,
2104		    retry_dir);
2105		if (indeadlock)
2106			goto again;
2107		dorwlock = 1;
2108	}
2109
2110	/*
2111	 * Grab quota lock if we are changing the file's owner.
2112	 */
2113	if (mask & AT_UID) {
2114		rw_enter(&ufsvfsp->vfs_dqrwlock, RW_READER);
2115		dodqlock = 1;
2116	}
2117	rw_enter(&ip->i_contents, RW_WRITER);
2118
2119	oldva.va_mode = ip->i_mode;
2120	oldva.va_uid = ip->i_uid;
2121	oldva.va_gid = ip->i_gid;
2122
2123	vap->va_mask &= ~AT_SIZE;
2124
2125	error = secpolicy_vnode_setattr(cr, vp, vap, &oldva, flags,
2126	    ufs_priv_access, ip);
2127	if (error)
2128		goto update_inode;
2129
2130	mask = vap->va_mask;
2131
2132	/*
2133	 * Change file access modes.
2134	 */
2135	if (mask & AT_MODE) {
2136		ip->i_mode = (ip->i_mode & IFMT) | (vap->va_mode & ~IFMT);
2137		TRANS_INODE(ufsvfsp, ip);
2138		ip->i_flag |= ICHG;
2139		if (stickyhack) {
2140			mutex_enter(&vp->v_lock);
2141			if ((ip->i_mode & (ISVTX | IEXEC | IFDIR)) == ISVTX)
2142				vp->v_flag |= VSWAPLIKE;
2143			else
2144				vp->v_flag &= ~VSWAPLIKE;
2145			mutex_exit(&vp->v_lock);
2146		}
2147	}
2148	if (mask & (AT_UID|AT_GID)) {
2149		if (mask & AT_UID) {
2150			/*
2151			 * Don't change ownership of the quota inode.
2152			 */
2153			if (ufsvfsp->vfs_qinod == ip) {
2154				ASSERT(ufsvfsp->vfs_qflags & MQ_ENABLED);
2155				error = EINVAL;
2156				goto update_inode;
2157			}
2158
2159			/*
2160			 * No real ownership change.
2161			 */
2162			if (ip->i_uid == vap->va_uid) {
2163				blocks = 0;
2164				owner_change = 0;
2165			}
2166			/*
2167			 * Remove the blocks and the file, from the old user's
2168			 * quota.
2169			 */
2170			else {
2171				blocks = ip->i_blocks;
2172				owner_change = 1;
2173
2174				(void) chkdq(ip, -blocks, /* force */ 1, cr,
2175				    (char **)NULL, (size_t *)NULL);
2176				(void) chkiq(ufsvfsp, /* change */ -1, ip,
2177				    (uid_t)ip->i_uid, /* force */ 1, cr,
2178				    (char **)NULL, (size_t *)NULL);
2179				dqrele(ip->i_dquot);
2180			}
2181
2182			ip->i_uid = vap->va_uid;
2183
2184			/*
2185			 * There is a real ownership change.
2186			 */
2187			if (owner_change) {
2188				/*
2189				 * Add the blocks and the file to the new
2190				 * user's quota.
2191				 */
2192				ip->i_dquot = getinoquota(ip);
2193				(void) chkdq(ip, blocks, /* force */ 1, cr,
2194				    &errmsg1, &len1);
2195				(void) chkiq(ufsvfsp, /* change */ 1,
2196				    (struct inode *)NULL, (uid_t)ip->i_uid,
2197				    /* force */ 1, cr, &errmsg2, &len2);
2198			}
2199		}
2200		if (mask & AT_GID) {
2201			ip->i_gid = vap->va_gid;
2202		}
2203		TRANS_INODE(ufsvfsp, ip);
2204		ip->i_flag |= ICHG;
2205	}
2206	/*
2207	 * Change file access or modified times.
2208	 */
2209	if (mask & (AT_ATIME|AT_MTIME)) {
2210		/* Check that the time value is within ufs range */
2211		if (((mask & AT_ATIME) && TIMESPEC_OVERFLOW(&vap->va_atime)) ||
2212		    ((mask & AT_MTIME) && TIMESPEC_OVERFLOW(&vap->va_mtime))) {
2213			error = EOVERFLOW;
2214			goto update_inode;
2215		}
2216
2217		/*
2218		 * if the "noaccess" mount option is set and only atime
2219		 * update is requested, do nothing. No error is returned.
2220		 */
2221		if ((ufsvfsp->vfs_noatime) &&
2222		    ((mask & (AT_ATIME|AT_MTIME)) == AT_ATIME))
2223			goto skip_atime;
2224
2225		if (mask & AT_ATIME) {
2226			ip->i_atime.tv_sec = vap->va_atime.tv_sec;
2227			ip->i_atime.tv_usec = vap->va_atime.tv_nsec / 1000;
2228			ip->i_flag &= ~IACC;
2229		}
2230		if (mask & AT_MTIME) {
2231			ip->i_mtime.tv_sec = vap->va_mtime.tv_sec;
2232			ip->i_mtime.tv_usec = vap->va_mtime.tv_nsec / 1000;
2233			gethrestime(&now);
2234			if (now.tv_sec > TIME32_MAX) {
2235				/*
2236				 * In 2038, ctime sticks forever..
2237				 */
2238				ip->i_ctime.tv_sec = TIME32_MAX;
2239				ip->i_ctime.tv_usec = 0;
2240			} else {
2241				ip->i_ctime.tv_sec = now.tv_sec;
2242				ip->i_ctime.tv_usec = now.tv_nsec / 1000;
2243			}
2244			ip->i_flag &= ~(IUPD|ICHG);
2245			ip->i_flag |= IMODTIME;
2246		}
2247		TRANS_INODE(ufsvfsp, ip);
2248		ip->i_flag |= IMOD;
2249	}
2250
2251skip_atime:
2252	/*
2253	 * The presence of a shadow inode may indicate an ACL, but does
2254	 * not imply an ACL.  Future FSD types should be handled here too
2255	 * and check for the presence of the attribute-specific data
2256	 * before referencing it.
2257	 */
2258	if (ip->i_shadow) {
2259		/*
2260		 * XXX if ufs_iupdat is changed to sandbagged write fix
2261		 * ufs_acl_setattr to push ip to keep acls consistent
2262		 *
2263		 * Suppress out of inodes messages if we will retry.
2264		 */
2265		if (retry)
2266			ip->i_flag |= IQUIET;
2267		error = ufs_acl_setattr(ip, vap, cr);
2268		ip->i_flag &= ~IQUIET;
2269	}
2270
2271update_inode:
2272	/*
2273	 * Setattr always increases the sequence number
2274	 */
2275	ip->i_seq++;
2276
2277	/*
2278	 * if nfsd and not logging; push synchronously
2279	 */
2280	if ((curthread->t_flag & T_DONTPEND) && !TRANS_ISTRANS(ufsvfsp)) {
2281		ufs_iupdat(ip, 1);
2282	} else {
2283		ITIMES_NOLOCK(ip);
2284	}
2285
2286	rw_exit(&ip->i_contents);
2287	if (dodqlock) {
2288		rw_exit(&ufsvfsp->vfs_dqrwlock);
2289	}
2290	if (dorwlock)
2291		rw_exit(&ip->i_rwlock);
2292
2293	if (ulp) {
2294		if (dotrans) {
2295			int terr = 0;
2296			TRANS_END_CSYNC(ufsvfsp, terr, issync, TOP_SETATTR,
2297			    trans_size);
2298			if (error == 0)
2299				error = terr;
2300		}
2301		ufs_lockfs_end(ulp);
2302	}
2303out:
2304	/*
2305	 * If out of inodes or blocks, see if we can free something
2306	 * up from the delete queue.
2307	 */
2308	if ((error == ENOSPC) && retry && TRANS_ISTRANS(ufsvfsp)) {
2309		ufs_delete_drain_wait(ufsvfsp, 1);
2310		retry = 0;
2311		if (errmsg1 != NULL)
2312			kmem_free(errmsg1, len1);
2313		if (errmsg2 != NULL)
2314			kmem_free(errmsg2, len2);
2315		goto again;
2316	}
2317	if (errmsg1 != NULL) {
2318		uprintf(errmsg1);
2319		kmem_free(errmsg1, len1);
2320	}
2321	if (errmsg2 != NULL) {
2322		uprintf(errmsg2);
2323		kmem_free(errmsg2, len2);
2324	}
2325	return (error);
2326}
2327
2328/*ARGSUSED*/
2329static int
2330ufs_access(struct vnode *vp, int mode, int flags, struct cred *cr,
2331    caller_context_t *ct)
2332{
2333	struct inode *ip = VTOI(vp);
2334
2335	if (ip->i_ufsvfs == NULL)
2336		return (EIO);
2337
2338	/*
2339	 * The ufs_iaccess function wants to be called with
2340	 * mode bits expressed as "ufs specific" bits.
2341	 * I.e., VWRITE|VREAD|VEXEC do not make sense to
2342	 * ufs_iaccess() but IWRITE|IREAD|IEXEC do.
2343	 * But since they're the same we just pass the vnode mode
2344	 * bit but just verify that assumption at compile time.
2345	 */
2346#if IWRITE != VWRITE || IREAD != VREAD || IEXEC != VEXEC
2347#error "ufs_access needs to map Vmodes to Imodes"
2348#endif
2349	return (ufs_iaccess(ip, mode, cr, 1));
2350}
2351
2352/* ARGSUSED */
2353static int
2354ufs_readlink(struct vnode *vp, struct uio *uiop, struct cred *cr,
2355    caller_context_t *ct)
2356{
2357	struct inode *ip = VTOI(vp);
2358	struct ufsvfs *ufsvfsp;
2359	struct ulockfs *ulp;
2360	int error;
2361	int fastsymlink;
2362
2363	if (vp->v_type != VLNK) {
2364		error = EINVAL;
2365		goto nolockout;
2366	}
2367
2368	/*
2369	 * If the symbolic link is empty there is nothing to read.
2370	 * Fast-track these empty symbolic links
2371	 */
2372	if (ip->i_size == 0) {
2373		error = 0;
2374		goto nolockout;
2375	}
2376
2377	ufsvfsp = ip->i_ufsvfs;
2378	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_READLINK_MASK);
2379	if (error)
2380		goto nolockout;
2381	/*
2382	 * The ip->i_rwlock protects the data blocks used for FASTSYMLINK
2383	 */
2384again:
2385	fastsymlink = 0;
2386	if (ip->i_flag & IFASTSYMLNK) {
2387		rw_enter(&ip->i_rwlock, RW_READER);
2388		rw_enter(&ip->i_contents, RW_READER);
2389		if (ip->i_flag & IFASTSYMLNK) {
2390			if (!ULOCKFS_IS_NOIACC(ITOUL(ip)) &&
2391			    (ip->i_fs->fs_ronly == 0) &&
2392			    (!ufsvfsp->vfs_noatime)) {
2393				mutex_enter(&ip->i_tlock);
2394				ip->i_flag |= IACC;
2395				mutex_exit(&ip->i_tlock);
2396			}
2397			error = uiomove((caddr_t)&ip->i_db[1],
2398			    MIN(ip->i_size, uiop->uio_resid),
2399			    UIO_READ, uiop);
2400			ITIMES(ip);
2401			++fastsymlink;
2402		}
2403		rw_exit(&ip->i_contents);
2404		rw_exit(&ip->i_rwlock);
2405	}
2406	if (!fastsymlink) {
2407		ssize_t size;	/* number of bytes read  */
2408		caddr_t basep;	/* pointer to input data */
2409		ino_t ino;
2410		long  igen;
2411		struct uio tuio;	/* temp uio struct */
2412		struct uio *tuiop;
2413		iovec_t tiov;		/* temp iovec struct */
2414		char kbuf[FSL_SIZE];	/* buffer to hold fast symlink */
2415		int tflag = 0;		/* flag to indicate temp vars used */
2416
2417		ino = ip->i_number;
2418		igen = ip->i_gen;
2419		size = uiop->uio_resid;
2420		basep = uiop->uio_iov->iov_base;
2421		tuiop = uiop;
2422
2423		rw_enter(&ip->i_rwlock, RW_WRITER);
2424		rw_enter(&ip->i_contents, RW_WRITER);
2425		if (ip->i_flag & IFASTSYMLNK) {
2426			rw_exit(&ip->i_contents);
2427			rw_exit(&ip->i_rwlock);
2428			goto again;
2429		}
2430
2431		/* can this be a fast symlink and is it a user buffer? */
2432		if (ip->i_size <= FSL_SIZE &&
2433		    (uiop->uio_segflg == UIO_USERSPACE ||
2434		    uiop->uio_segflg == UIO_USERISPACE)) {
2435
2436			bzero(&tuio, sizeof (struct uio));
2437			/*
2438			 * setup a kernel buffer to read link into.  this
2439			 * is to fix a race condition where the user buffer
2440			 * got corrupted before copying it into the inode.
2441			 */
2442			size = ip->i_size;
2443			tiov.iov_len = size;
2444			tiov.iov_base = kbuf;
2445			tuio.uio_iov = &tiov;
2446			tuio.uio_iovcnt = 1;
2447			tuio.uio_offset = uiop->uio_offset;
2448			tuio.uio_segflg = UIO_SYSSPACE;
2449			tuio.uio_fmode = uiop->uio_fmode;
2450			tuio.uio_extflg = uiop->uio_extflg;
2451			tuio.uio_limit = uiop->uio_limit;
2452			tuio.uio_resid = size;
2453
2454			basep = tuio.uio_iov->iov_base;
2455			tuiop = &tuio;
2456			tflag = 1;
2457		}
2458
2459		error = rdip(ip, tuiop, 0, cr);
2460		if (!(error == 0 && ip->i_number == ino && ip->i_gen == igen)) {
2461			rw_exit(&ip->i_contents);
2462			rw_exit(&ip->i_rwlock);
2463			goto out;
2464		}
2465
2466		if (tflag == 0)
2467			size -= uiop->uio_resid;
2468
2469		if ((tflag == 0 && ip->i_size <= FSL_SIZE &&
2470		    ip->i_size == size) || (tflag == 1 &&
2471		    tuio.uio_resid == 0)) {
2472			error = kcopy(basep, &ip->i_db[1], ip->i_size);
2473			if (error == 0) {
2474				ip->i_flag |= IFASTSYMLNK;
2475				/*
2476				 * free page
2477				 */
2478				(void) VOP_PUTPAGE(ITOV(ip),
2479				    (offset_t)0, PAGESIZE,
2480				    (B_DONTNEED | B_FREE | B_FORCE | B_ASYNC),
2481				    cr, ct);
2482			} else {
2483				int i;
2484				/* error, clear garbage left behind */
2485				for (i = 1; i < NDADDR; i++)
2486					ip->i_db[i] = 0;
2487				for (i = 0; i < NIADDR; i++)
2488					ip->i_ib[i] = 0;
2489			}
2490		}
2491		if (tflag == 1) {
2492			/* now, copy it into the user buffer */
2493			error = uiomove((caddr_t)kbuf,
2494			    MIN(size, uiop->uio_resid),
2495			    UIO_READ, uiop);
2496		}
2497		rw_exit(&ip->i_contents);
2498		rw_exit(&ip->i_rwlock);
2499	}
2500out:
2501	if (ulp) {
2502		ufs_lockfs_end(ulp);
2503	}
2504nolockout:
2505	return (error);
2506}
2507
2508/* ARGSUSED */
2509static int
2510ufs_fsync(struct vnode *vp, int syncflag, struct cred *cr, caller_context_t *ct)
2511{
2512	struct inode *ip = VTOI(vp);
2513	struct ufsvfs *ufsvfsp = ip->i_ufsvfs;
2514	struct ulockfs *ulp;
2515	int error;
2516
2517	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_FSYNC_MASK);
2518	if (error)
2519		return (error);
2520
2521	if (TRANS_ISTRANS(ufsvfsp)) {
2522		/*
2523		 * First push out any data pages
2524		 */
2525		if (vn_has_cached_data(vp) && !(syncflag & FNODSYNC) &&
2526		    (vp->v_type != VCHR) && !(IS_SWAPVP(vp))) {
2527			error = VOP_PUTPAGE(vp, (offset_t)0, (size_t)0,
2528			    0, CRED(), ct);
2529			if (error)
2530				goto out;
2531		}
2532
2533		/*
2534		 * Delta any delayed inode times updates
2535		 * and push inode to log.
2536		 * All other inode deltas will have already been delta'd
2537		 * and will be pushed during the commit.
2538		 */
2539		if (!(syncflag & FDSYNC) &&
2540		    ((ip->i_flag & (IMOD|IMODACC)) == IMODACC)) {
2541			if (ulp) {
2542				TRANS_BEGIN_ASYNC(ufsvfsp, TOP_FSYNC,
2543				    TOP_SYNCIP_SIZE);
2544			}
2545			rw_enter(&ip->i_contents, RW_READER);
2546			mutex_enter(&ip->i_tlock);
2547			ip->i_flag &= ~IMODTIME;
2548			mutex_exit(&ip->i_tlock);
2549			ufs_iupdat(ip, I_SYNC);
2550			rw_exit(&ip->i_contents);
2551			if (ulp) {
2552				TRANS_END_ASYNC(ufsvfsp, TOP_FSYNC,
2553				    TOP_SYNCIP_SIZE);
2554			}
2555		}
2556
2557		/*
2558		 * Commit the Moby transaction
2559		 *
2560		 * Deltas have already been made so we just need to
2561		 * commit them with a synchronous transaction.
2562		 * TRANS_BEGIN_SYNC() will return an error
2563		 * if there are no deltas to commit, for an
2564		 * empty transaction.
2565		 */
2566		if (ulp) {
2567			TRANS_BEGIN_SYNC(ufsvfsp, TOP_FSYNC, TOP_COMMIT_SIZE,
2568			    error);
2569			if (error) {
2570				error = 0; /* commit wasn't needed */
2571				goto out;
2572			}
2573			TRANS_END_SYNC(ufsvfsp, error, TOP_FSYNC,
2574			    TOP_COMMIT_SIZE);
2575		}
2576	} else {	/* not logging */
2577		if (!(IS_SWAPVP(vp)))
2578			if (syncflag & FNODSYNC) {
2579				/* Just update the inode only */
2580				TRANS_IUPDAT(ip, 1);
2581				error = 0;
2582			} else if (syncflag & FDSYNC)
2583				/* Do data-synchronous writes */
2584				error = TRANS_SYNCIP(ip, 0, I_DSYNC, TOP_FSYNC);
2585			else
2586				/* Do synchronous writes */
2587				error = TRANS_SYNCIP(ip, 0, I_SYNC, TOP_FSYNC);
2588
2589		rw_enter(&ip->i_contents, RW_WRITER);
2590		if (!error)
2591			error = ufs_sync_indir(ip);
2592		rw_exit(&ip->i_contents);
2593	}
2594out:
2595	if (ulp) {
2596		ufs_lockfs_end(ulp);
2597	}
2598	return (error);
2599}
2600
2601/*ARGSUSED*/
2602static void
2603ufs_inactive(struct vnode *vp, struct cred *cr, caller_context_t *ct)
2604{
2605	ufs_iinactive(VTOI(vp));
2606}
2607
2608/*
2609 * Unix file system operations having to do with directory manipulation.
2610 */
2611int ufs_lookup_idle_count = 2;	/* Number of inodes to idle each time */
2612/* ARGSUSED */
2613static int
2614ufs_lookup(struct vnode *dvp, char *nm, struct vnode **vpp,
2615    struct pathname *pnp, int flags, struct vnode *rdir, struct cred *cr,
2616    caller_context_t *ct, int *direntflags, pathname_t *realpnp)
2617{
2618	struct inode *ip;
2619	struct inode *sip;
2620	struct inode *xip;
2621	struct ufsvfs *ufsvfsp;
2622	struct ulockfs *ulp;
2623	struct vnode *vp;
2624	int error;
2625
2626	/*
2627	 * Check flags for type of lookup (regular file or attribute file)
2628	 */
2629
2630	ip = VTOI(dvp);
2631
2632	if (flags & LOOKUP_XATTR) {
2633
2634		/*
2635		 * If not mounted with XATTR support then return EINVAL
2636		 */
2637
2638		if (!(ip->i_ufsvfs->vfs_vfs->vfs_flag & VFS_XATTR))
2639			return (EINVAL);
2640		/*
2641		 * We don't allow recursive attributes...
2642		 * Maybe someday we will.
2643		 */
2644		if ((ip->i_cflags & IXATTR)) {
2645			return (EINVAL);
2646		}
2647
2648		if ((vp = dnlc_lookup(dvp, XATTR_DIR_NAME)) == NULL) {
2649			error = ufs_xattr_getattrdir(dvp, &sip, flags, cr);
2650			if (error) {
2651				*vpp = NULL;
2652				goto out;
2653			}
2654
2655			vp = ITOV(sip);
2656			dnlc_update(dvp, XATTR_DIR_NAME, vp);
2657		}
2658
2659		/*
2660		 * Check accessibility of directory.
2661		 */
2662		if (vp == DNLC_NO_VNODE) {
2663			VN_RELE(vp);
2664			error = ENOENT;
2665			goto out;
2666		}
2667		if ((error = ufs_iaccess(VTOI(vp), IEXEC, cr, 1)) != 0) {
2668			VN_RELE(vp);
2669			goto out;
2670		}
2671
2672		*vpp = vp;
2673		return (0);
2674	}
2675
2676	/*
2677	 * Check for a null component, which we should treat as
2678	 * looking at dvp from within it's parent, so we don't
2679	 * need a call to ufs_iaccess(), as it has already been
2680	 * done.
2681	 */
2682	if (nm[0] == 0) {
2683		VN_HOLD(dvp);
2684		error = 0;
2685		*vpp = dvp;
2686		goto out;
2687	}
2688
2689	/*
2690	 * Check for "." ie itself. this is a quick check and
2691	 * avoids adding "." into the dnlc (which have been seen
2692	 * to occupy >10% of the cache).
2693	 */
2694	if ((nm[0] == '.') && (nm[1] == 0)) {
2695		/*
2696		 * Don't return without checking accessibility
2697		 * of the directory. We only need the lock if
2698		 * we are going to return it.
2699		 */
2700		if ((error = ufs_iaccess(ip, IEXEC, cr, 1)) == 0) {
2701			VN_HOLD(dvp);
2702			*vpp = dvp;
2703		}
2704		goto out;
2705	}
2706
2707	/*
2708	 * Fast path: Check the directory name lookup cache.
2709	 */
2710	if (vp = dnlc_lookup(dvp, nm)) {
2711		/*
2712		 * Check accessibility of directory.
2713		 */
2714		if ((error = ufs_iaccess(ip, IEXEC, cr, 1)) != 0) {
2715			VN_RELE(vp);
2716			goto out;
2717		}
2718		if (vp == DNLC_NO_VNODE) {
2719			VN_RELE(vp);
2720			error = ENOENT;
2721			goto out;
2722		}
2723		xip = VTOI(vp);
2724		ulp = NULL;
2725		goto fastpath;
2726	}
2727
2728	/*
2729	 * Keep the idle queue from getting too long by
2730	 * idling two inodes before attempting to allocate another.
2731	 *    This operation must be performed before entering
2732	 *    lockfs or a transaction.
2733	 */
2734	if (ufs_idle_q.uq_ne > ufs_idle_q.uq_hiwat)
2735		if ((curthread->t_flag & T_DONTBLOCK) == 0) {
2736			ins.in_lidles.value.ul += ufs_lookup_idle_count;
2737			ufs_idle_some(ufs_lookup_idle_count);
2738		}
2739
2740retry_lookup:
2741	/*
2742	 * Check accessibility of directory.
2743	 */
2744	if (error = ufs_diraccess(ip, IEXEC, cr))
2745		goto out;
2746
2747	ufsvfsp = ip->i_ufsvfs;
2748	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_LOOKUP_MASK);
2749	if (error)
2750		goto out;
2751
2752	error = ufs_dirlook(ip, nm, &xip, cr, 1, 0);
2753
2754fastpath:
2755	if (error == 0) {
2756		ip = xip;
2757		*vpp = ITOV(ip);
2758
2759		/*
2760		 * If vnode is a device return special vnode instead.
2761		 */
2762		if (IS_DEVVP(*vpp)) {
2763			struct vnode *newvp;
2764
2765			newvp = specvp(*vpp, (*vpp)->v_rdev, (*vpp)->v_type,
2766			    cr);
2767			VN_RELE(*vpp);
2768			if (newvp == NULL)
2769				error = ENOSYS;
2770			else
2771				*vpp = newvp;
2772		} else if (ip->i_cflags & ICOMPRESS) {
2773			struct vnode *newvp;
2774
2775			/*
2776			 * Compressed file, substitute dcfs vnode
2777			 */
2778			newvp = decompvp(*vpp, cr, ct);
2779			VN_RELE(*vpp);
2780			if (newvp == NULL)
2781				error = ENOSYS;
2782			else
2783				*vpp = newvp;
2784		}
2785	}
2786	if (ulp) {
2787		ufs_lockfs_end(ulp);
2788	}
2789
2790	if (error == EAGAIN)
2791		goto retry_lookup;
2792
2793out:
2794	return (error);
2795}
2796
2797/*ARGSUSED*/
2798static int
2799ufs_create(struct vnode *dvp, char *name, struct vattr *vap, enum vcexcl excl,
2800    int mode, struct vnode **vpp, struct cred *cr, int flag,
2801    caller_context_t *ct, vsecattr_t *vsecp)
2802{
2803	struct inode *ip;
2804	struct inode *xip;
2805	struct inode *dip;
2806	struct vnode *xvp;
2807	struct ufsvfs *ufsvfsp;
2808	struct ulockfs *ulp;
2809	int error;
2810	int issync;
2811	int truncflag;
2812	int trans_size;
2813	int noentry;
2814	int defer_dip_seq_update = 0;	/* need to defer update of dip->i_seq */
2815	int retry = 1;
2816	int indeadlock;
2817
2818again:
2819	ip = VTOI(dvp);
2820	ufsvfsp = ip->i_ufsvfs;
2821	truncflag = 0;
2822
2823	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_CREATE_MASK);
2824	if (error)
2825		goto out;
2826
2827	if (ulp) {
2828		trans_size = (int)TOP_CREATE_SIZE(ip);
2829		TRANS_BEGIN_CSYNC(ufsvfsp, issync, TOP_CREATE, trans_size);
2830	}
2831
2832	if ((vap->va_mode & VSVTX) && secpolicy_vnode_stky_modify(cr) != 0)
2833		vap->va_mode &= ~VSVTX;
2834
2835	if (*name == '\0') {
2836		/*
2837		 * Null component name refers to the directory itself.
2838		 */
2839		VN_HOLD(dvp);
2840		/*
2841		 * Even though this is an error case, we need to grab the
2842		 * quota lock since the error handling code below is common.
2843		 */
2844		rw_enter(&ufsvfsp->vfs_dqrwlock, RW_READER);
2845		rw_enter(&ip->i_contents, RW_WRITER);
2846		error = EEXIST;
2847	} else {
2848		xip = NULL;
2849		noentry = 0;
2850		/*
2851		 * ufs_tryirwlock_trans uses rw_tryenter and checks for SLOCK
2852		 * to avoid i_rwlock, ufs_lockfs_begin deadlock. If deadlock
2853		 * possible, retries the operation.
2854		 */
2855		ufs_tryirwlock_trans(&ip->i_rwlock, RW_WRITER, TOP_CREATE,
2856		    retry_dir);
2857		if (indeadlock)
2858			goto again;
2859
2860		xvp = dnlc_lookup(dvp, name);
2861		if (xvp == DNLC_NO_VNODE) {
2862			noentry = 1;
2863			VN_RELE(xvp);
2864			xvp = NULL;
2865		}
2866		if (xvp) {
2867			rw_exit(&ip->i_rwlock);
2868			if (error = ufs_iaccess(ip, IEXEC, cr, 1)) {
2869				VN_RELE(xvp);
2870			} else {
2871				error = EEXIST;
2872				xip = VTOI(xvp);
2873			}
2874		} else {
2875			/*
2876			 * Suppress file system full message if we will retry
2877			 */
2878			error = ufs_direnter_cm(ip, name, DE_CREATE,
2879			    vap, &xip, cr, (noentry | (retry ? IQUIET : 0)));
2880			if (error == EAGAIN) {
2881				if (ulp) {
2882					TRANS_END_CSYNC(ufsvfsp, error, issync,
2883					    TOP_CREATE, trans_size);
2884					ufs_lockfs_end(ulp);
2885				}
2886				goto again;
2887			}
2888			rw_exit(&ip->i_rwlock);
2889		}
2890		ip = xip;
2891		if (ip != NULL) {
2892			rw_enter(&ufsvfsp->vfs_dqrwlock, RW_READER);
2893			rw_enter(&ip->i_contents, RW_WRITER);
2894		}
2895	}
2896
2897	/*
2898	 * If the file already exists and this is a non-exclusive create,
2899	 * check permissions and allow access for non-directories.
2900	 * Read-only create of an existing directory is also allowed.
2901	 * We fail an exclusive create of anything which already exists.
2902	 */
2903	if (error == EEXIST) {
2904		dip = VTOI(dvp);
2905		if (excl == NONEXCL) {
2906			if ((((ip->i_mode & IFMT) == IFDIR) ||
2907			    ((ip->i_mode & IFMT) == IFATTRDIR)) &&
2908			    (mode & IWRITE))
2909				error = EISDIR;
2910			else if (mode)
2911				error = ufs_iaccess(ip, mode, cr, 0);
2912			else
2913				error = 0;
2914		}
2915		if (error) {
2916			rw_exit(&ip->i_contents);
2917			rw_exit(&ufsvfsp->vfs_dqrwlock);
2918			VN_RELE(ITOV(ip));
2919			goto unlock;
2920		}
2921		/*
2922		 * If the error EEXIST was set, then i_seq can not
2923		 * have been updated. The sequence number interface
2924		 * is defined such that a non-error VOP_CREATE must
2925		 * increase the dir va_seq it by at least one. If we
2926		 * have cleared the error, increase i_seq. Note that
2927		 * we are increasing the dir i_seq and in rare cases
2928		 * ip may actually be from the dvp, so we already have
2929		 * the locks and it will not be subject to truncation.
2930		 * In case we have to update i_seq of the parent
2931		 * directory dip, we have to defer it till we have
2932		 * released our locks on ip due to lock ordering requirements.
2933		 */
2934		if (ip != dip)
2935			defer_dip_seq_update = 1;
2936		else
2937			ip->i_seq++;
2938
2939		if (((ip->i_mode & IFMT) == IFREG) &&
2940		    (vap->va_mask & AT_SIZE) && vap->va_size == 0) {
2941			/*
2942			 * Truncate regular files, if requested by caller.
2943			 * Grab i_rwlock to make sure no one else is
2944			 * currently writing to the file (we promised
2945			 * bmap we would do this).
2946			 * Must get the locks in the correct order.
2947			 */
2948			if (ip->i_size == 0) {
2949				ip->i_flag |= ICHG | IUPD;
2950				ip->i_seq++;
2951				TRANS_INODE(ufsvfsp, ip);
2952			} else {
2953				/*
2954				 * Large Files: Why this check here?
2955				 * Though we do it in vn_create() we really
2956				 * want to guarantee that we do not destroy
2957				 * Large file data by atomically checking
2958				 * the size while holding the contents
2959				 * lock.
2960				 */
2961				if (flag && !(flag & FOFFMAX) &&
2962				    ((ip->i_mode & IFMT) == IFREG) &&
2963				    (ip->i_size > (offset_t)MAXOFF32_T)) {
2964					rw_exit(&ip->i_contents);
2965					rw_exit(&ufsvfsp->vfs_dqrwlock);
2966					error = EOVERFLOW;
2967					goto unlock;
2968				}
2969				if (TRANS_ISTRANS(ufsvfsp))
2970					truncflag++;
2971				else {
2972					rw_exit(&ip->i_contents);
2973					rw_exit(&ufsvfsp->vfs_dqrwlock);
2974					ufs_tryirwlock_trans(&ip->i_rwlock,
2975					    RW_WRITER, TOP_CREATE,
2976					    retry_file);
2977					if (indeadlock) {
2978						VN_RELE(ITOV(ip));
2979						goto again;
2980					}
2981					rw_enter(&ufsvfsp->vfs_dqrwlock,
2982					    RW_READER);
2983					rw_enter(&ip->i_contents, RW_WRITER);
2984					(void) ufs_itrunc(ip, (u_offset_t)0, 0,
2985					    cr);
2986					rw_exit(&ip->i_rwlock);
2987				}
2988
2989			}
2990			if (error == 0) {
2991				vnevent_create(ITOV(ip), ct);
2992			}
2993		}
2994	}
2995
2996	if (error) {
2997		if (ip != NULL) {
2998			rw_exit(&ufsvfsp->vfs_dqrwlock);
2999			rw_exit(&ip->i_contents);
3000		}
3001		goto unlock;
3002	}
3003
3004	*vpp = ITOV(ip);
3005	ITIMES(ip);
3006	rw_exit(&ip->i_contents);
3007	rw_exit(&ufsvfsp->vfs_dqrwlock);
3008
3009	/*
3010	 * If vnode is a device return special vnode instead.
3011	 */
3012	if (!error && IS_DEVVP(*vpp)) {
3013		struct vnode *newvp;
3014
3015		newvp = specvp(*vpp, (*vpp)->v_rdev, (*vpp)->v_type, cr);
3016		VN_RELE(*vpp);
3017		if (newvp == NULL) {
3018			error = ENOSYS;
3019			goto unlock;
3020		}
3021		truncflag = 0;
3022		*vpp = newvp;
3023	}
3024unlock:
3025
3026	/*
3027	 * Do the deferred update of the parent directory's sequence
3028	 * number now.
3029	 */
3030	if (defer_dip_seq_update == 1) {
3031		rw_enter(&dip->i_contents, RW_READER);
3032		mutex_enter(&dip->i_tlock);
3033		dip->i_seq++;
3034		mutex_exit(&dip->i_tlock);
3035		rw_exit(&dip->i_contents);
3036	}
3037
3038	if (ulp) {
3039		int terr = 0;
3040
3041		TRANS_END_CSYNC(ufsvfsp, terr, issync, TOP_CREATE,
3042		    trans_size);
3043
3044		/*
3045		 * If we haven't had a more interesting failure
3046		 * already, then anything that might've happened
3047		 * here should be reported.
3048		 */
3049		if (error == 0)
3050			error = terr;
3051	}
3052
3053	if (!error && truncflag) {
3054		ufs_tryirwlock(&ip->i_rwlock, RW_WRITER, retry_trunc);
3055		if (indeadlock) {
3056			if (ulp)
3057				ufs_lockfs_end(ulp);
3058			VN_RELE(ITOV(ip));
3059			goto again;
3060		}
3061		(void) TRANS_ITRUNC(ip, (u_offset_t)0, 0, cr);
3062		rw_exit(&ip->i_rwlock);
3063	}
3064
3065	if (ulp)
3066		ufs_lockfs_end(ulp);
3067
3068	/*
3069	 * If no inodes available, try to free one up out of the
3070	 * pending delete queue.
3071	 */
3072	if ((error == ENOSPC) && retry && TRANS_ISTRANS(ufsvfsp)) {
3073		ufs_delete_drain_wait(ufsvfsp, 1);
3074		retry = 0;
3075		goto again;
3076	}
3077
3078out:
3079	return (error);
3080}
3081
3082extern int ufs_idle_max;
3083/*ARGSUSED*/
3084static int
3085ufs_remove(struct vnode *vp, char *nm, struct cred *cr, caller_context_t *ct,
3086    int flags)
3087{
3088	struct inode *ip = VTOI(vp);
3089	struct ufsvfs *ufsvfsp	= ip->i_ufsvfs;
3090	struct ulockfs *ulp;
3091	vnode_t *rmvp = NULL;	/* Vnode corresponding to name being removed */
3092	int indeadlock;
3093	int error;
3094	int issync;
3095	int trans_size;
3096
3097	/*
3098	 * don't let the delete queue get too long
3099	 */
3100	if (ufsvfsp == NULL) {
3101		error = EIO;
3102		goto out;
3103	}
3104	if (ufsvfsp->vfs_delete.uq_ne > ufs_idle_max)
3105		ufs_delete_drain(vp->v_vfsp, 1, 1);
3106
3107	error = ufs_eventlookup(vp, nm, cr, &rmvp);
3108	if (rmvp != NULL) {
3109		/* Only send the event if there were no errors */
3110		if (error == 0)
3111			vnevent_remove(rmvp, vp, nm, ct);
3112		VN_RELE(rmvp);
3113	}
3114
3115retry_remove:
3116	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_REMOVE_MASK);
3117	if (error)
3118		goto out;
3119
3120	if (ulp)
3121		TRANS_BEGIN_CSYNC(ufsvfsp, issync, TOP_REMOVE,
3122		    trans_size = (int)TOP_REMOVE_SIZE(VTOI(vp)));
3123
3124	/*
3125	 * ufs_tryirwlock_trans uses rw_tryenter and checks for SLOCK
3126	 * to avoid i_rwlock, ufs_lockfs_begin deadlock. If deadlock
3127	 * possible, retries the operation.
3128	 */
3129	ufs_tryirwlock_trans(&ip->i_rwlock, RW_WRITER, TOP_REMOVE, retry);
3130	if (indeadlock)
3131		goto retry_remove;
3132	error = ufs_dirremove(ip, nm, (struct inode *)0, (struct vnode *)0,
3133	    DR_REMOVE, cr);
3134	rw_exit(&ip->i_rwlock);
3135
3136	if (ulp) {
3137		TRANS_END_CSYNC(ufsvfsp, error, issync, TOP_REMOVE, trans_size);
3138		ufs_lockfs_end(ulp);
3139	}
3140
3141out:
3142	return (error);
3143}
3144
3145/*
3146 * Link a file or a directory.  Only privileged processes are allowed to
3147 * make links to directories.
3148 */
3149/*ARGSUSED*/
3150static int
3151ufs_link(struct vnode *tdvp, struct vnode *svp, char *tnm, struct cred *cr,
3152    caller_context_t *ct, int flags)
3153{
3154	struct inode *sip;
3155	struct inode *tdp = VTOI(tdvp);
3156	struct ufsvfs *ufsvfsp = tdp->i_ufsvfs;
3157	struct ulockfs *ulp;
3158	struct vnode *realvp;
3159	int error;
3160	int issync;
3161	int trans_size;
3162	int isdev;
3163	int indeadlock;
3164
3165retry_link:
3166	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_LINK_MASK);
3167	if (error)
3168		goto out;
3169
3170	if (ulp)
3171		TRANS_BEGIN_CSYNC(ufsvfsp, issync, TOP_LINK,
3172		    trans_size = (int)TOP_LINK_SIZE(VTOI(tdvp)));
3173
3174	if (VOP_REALVP(svp, &realvp, ct) == 0)
3175		svp = realvp;
3176
3177	/*
3178	 * Make sure link for extended attributes is valid
3179	 * We only support hard linking of attr in ATTRDIR to ATTRDIR
3180	 *
3181	 * Make certain we don't attempt to look at a device node as
3182	 * a ufs inode.
3183	 */
3184
3185	isdev = IS_DEVVP(svp);
3186	if (((isdev == 0) && ((VTOI(svp)->i_cflags & IXATTR) == 0) &&
3187	    ((tdp->i_mode & IFMT) == IFATTRDIR)) ||
3188	    ((isdev == 0) && (VTOI(svp)->i_cflags & IXATTR) &&
3189	    ((tdp->i_mode & IFMT) == IFDIR))) {
3190		error = EINVAL;
3191		goto unlock;
3192	}
3193
3194	sip = VTOI(svp);
3195	if ((svp->v_type == VDIR &&
3196	    secpolicy_fs_linkdir(cr, ufsvfsp->vfs_vfs) != 0) ||
3197	    (sip->i_uid != crgetuid(cr) && secpolicy_basic_link(cr) != 0)) {
3198		error = EPERM;
3199		goto unlock;
3200	}
3201
3202	/*
3203	 * ufs_tryirwlock_trans uses rw_tryenter and checks for SLOCK
3204	 * to avoid i_rwlock, ufs_lockfs_begin deadlock. If deadlock
3205	 * possible, retries the operation.
3206	 */
3207	ufs_tryirwlock_trans(&tdp->i_rwlock, RW_WRITER, TOP_LINK, retry);
3208	if (indeadlock)
3209		goto retry_link;
3210	error = ufs_direnter_lr(tdp, tnm, DE_LINK, (struct inode *)0,
3211	    sip, cr);
3212	rw_exit(&tdp->i_rwlock);
3213
3214unlock:
3215	if (ulp) {
3216		TRANS_END_CSYNC(ufsvfsp, error, issync, TOP_LINK, trans_size);
3217		ufs_lockfs_end(ulp);
3218	}
3219
3220	if (!error) {
3221		vnevent_link(svp, ct);
3222	}
3223out:
3224	return (error);
3225}
3226
3227uint64_t ufs_rename_retry_cnt;
3228uint64_t ufs_rename_upgrade_retry_cnt;
3229uint64_t ufs_rename_dircheck_retry_cnt;
3230clock_t	 ufs_rename_backoff_delay = 1;
3231
3232/*
3233 * Rename a file or directory.
3234 * We are given the vnode and entry string of the source and the
3235 * vnode and entry string of the place we want to move the source
3236 * to (the target). The essential operation is:
3237 *	unlink(target);
3238 *	link(source, target);
3239 *	unlink(source);
3240 * but "atomically".  Can't do full commit without saving state in
3241 * the inode on disk, which isn't feasible at this time.  Best we
3242 * can do is always guarantee that the TARGET exists.
3243 */
3244
3245/*ARGSUSED*/
3246static int
3247ufs_rename(struct vnode *sdvp, char *snm, struct vnode *tdvp, char *tnm,
3248    struct cred *cr, caller_context_t *ct, int flags)
3249{
3250	struct inode *sip = NULL;	/* source inode */
3251	struct inode *ip = NULL;	/* check inode */
3252	struct inode *sdp;		/* old (source) parent inode */
3253	struct inode *tdp;		/* new (target) parent inode */
3254	struct vnode *svp = NULL;	/* source vnode */
3255	struct vnode *tvp = NULL;	/* target vnode, if it exists */
3256	struct vnode *realvp;
3257	struct ufsvfs *ufsvfsp;
3258	struct ulockfs *ulp = NULL;
3259	struct ufs_slot slot;
3260	timestruc_t now;
3261	int error;
3262	int issync;
3263	int trans_size;
3264	krwlock_t *first_lock;
3265	krwlock_t *second_lock;
3266	krwlock_t *reverse_lock;
3267	int serr, terr;
3268
3269	sdp = VTOI(sdvp);
3270	slot.fbp = NULL;
3271	ufsvfsp = sdp->i_ufsvfs;
3272
3273	if (VOP_REALVP(tdvp, &realvp, ct) == 0)
3274		tdvp = realvp;
3275
3276	/* Must do this before taking locks in case of DNLC miss */
3277	terr = ufs_eventlookup(tdvp, tnm, cr, &tvp);
3278	serr = ufs_eventlookup(sdvp, snm, cr, &svp);
3279
3280	if ((serr == 0) && ((terr == 0) || (terr == ENOENT))) {
3281		if (tvp != NULL)
3282			vnevent_pre_rename_dest(tvp, tdvp, tnm, ct);
3283
3284		/*
3285		 * Notify the target directory of the rename event
3286		 * if source and target directories are not the same.
3287		 */
3288		if (sdvp != tdvp)
3289			vnevent_pre_rename_dest_dir(tdvp, svp, tnm, ct);
3290
3291		if (svp != NULL)
3292			vnevent_pre_rename_src(svp, sdvp, snm, ct);
3293	}
3294
3295	if (svp != NULL)
3296		VN_RELE(svp);
3297
3298retry_rename:
3299	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_RENAME_MASK);
3300	if (error)
3301		goto unlock;
3302
3303	if (ulp)
3304		TRANS_BEGIN_CSYNC(ufsvfsp, issync, TOP_RENAME,
3305		    trans_size = (int)TOP_RENAME_SIZE(sdp));
3306
3307	if (VOP_REALVP(tdvp, &realvp, ct) == 0)
3308		tdvp = realvp;
3309
3310	tdp = VTOI(tdvp);
3311
3312	/*
3313	 * We only allow renaming of attributes from ATTRDIR to ATTRDIR.
3314	 */
3315	if ((tdp->i_mode & IFMT) != (sdp->i_mode & IFMT)) {
3316		error = EINVAL;
3317		goto unlock;
3318	}
3319
3320	/*
3321	 * Check accessibility of directory.
3322	 */
3323	if (error = ufs_diraccess(sdp, IEXEC, cr))
3324		goto unlock;
3325
3326	/*
3327	 * Look up inode of file we're supposed to rename.
3328	 */
3329	gethrestime(&now);
3330	if (error = ufs_dirlook(sdp, snm, &sip, cr, 0, 0)) {
3331		if (error == EAGAIN) {
3332			if (ulp) {
3333				TRANS_END_CSYNC(ufsvfsp, error, issync,
3334				    TOP_RENAME, trans_size);
3335				ufs_lockfs_end(ulp);
3336			}
3337			goto retry_rename;
3338		}
3339
3340		goto unlock;
3341	}
3342
3343	/*
3344	 * Lock both the source and target directories (they may be
3345	 * the same) to provide the atomicity semantics that was
3346	 * previously provided by the per file system vfs_rename_lock
3347	 *
3348	 * with vfs_rename_lock removed to allow simultaneous renames
3349	 * within a file system, ufs_dircheckpath can deadlock while
3350	 * traversing back to ensure that source is not a parent directory
3351	 * of target parent directory. This is because we get into
3352	 * ufs_dircheckpath with the sdp and tdp locks held as RW_WRITER.
3353	 * If the tdp and sdp of the simultaneous renames happen to be
3354	 * in the path of each other, it can lead to a deadlock. This
3355	 * can be avoided by getting the locks as RW_READER here and then
3356	 * upgrading to RW_WRITER after completing the ufs_dircheckpath.
3357	 *
3358	 * We hold the target directory's i_rwlock after calling
3359	 * ufs_lockfs_begin but in many other operations (like ufs_readdir)
3360	 * VOP_RWLOCK is explicitly called by the filesystem independent code
3361	 * before calling the file system operation. In these cases the order
3362	 * is reversed (i.e i_rwlock is taken first and then ufs_lockfs_begin
3363	 * is called). This is fine as long as ufs_lockfs_begin acts as a VOP
3364	 * counter but with ufs_quiesce setting the SLOCK bit this becomes a
3365	 * synchronizing object which might lead to a deadlock. So we use
3366	 * rw_tryenter instead of rw_enter. If we fail to get this lock and
3367	 * find that SLOCK bit is set, we call ufs_lockfs_end and restart the
3368	 * operation.
3369	 */
3370retry:
3371	first_lock = &tdp->i_rwlock;
3372	second_lock = &sdp->i_rwlock;
3373retry_firstlock:
3374	if (!rw_tryenter(first_lock, RW_READER)) {
3375		/*
3376		 * We didn't get the lock. Check if the SLOCK is set in the
3377		 * ufsvfs. If yes, we might be in a deadlock. Safer to give up
3378		 * and wait for SLOCK to be cleared.
3379		 */
3380
3381		if (ulp && ULOCKFS_IS_SLOCK(ulp)) {
3382			TRANS_END_CSYNC(ufsvfsp, error, issync, TOP_RENAME,
3383			    trans_size);
3384			ufs_lockfs_end(ulp);
3385			goto retry_rename;
3386
3387		} else {
3388			/*
3389			 * SLOCK isn't set so this is a genuine synchronization
3390			 * case. Let's try again after giving them a breather.
3391			 */
3392			delay(RETRY_LOCK_DELAY);
3393			goto  retry_firstlock;
3394		}
3395	}
3396	/*
3397	 * Need to check if the tdp and sdp are same !!!
3398	 */
3399	if ((tdp != sdp) && (!rw_tryenter(second_lock, RW_READER))) {
3400		/*
3401		 * We didn't get the lock. Check if the SLOCK is set in the
3402		 * ufsvfs. If yes, we might be in a deadlock. Safer to give up
3403		 * and wait for SLOCK to be cleared.
3404		 */
3405
3406		rw_exit(first_lock);
3407		if (ulp && ULOCKFS_IS_SLOCK(ulp)) {
3408			TRANS_END_CSYNC(ufsvfsp, error, issync, TOP_RENAME,
3409			    trans_size);
3410			ufs_lockfs_end(ulp);
3411			goto retry_rename;
3412
3413		} else {
3414			/*
3415			 * So we couldn't get the second level peer lock *and*
3416			 * the SLOCK bit isn't set. Too bad we can be
3417			 * contentding with someone wanting these locks otherway
3418			 * round. Reverse the locks in case there is a heavy
3419			 * contention for the second level lock.
3420			 */
3421			reverse_lock = first_lock;
3422			first_lock = second_lock;
3423			second_lock = reverse_lock;
3424			ufs_rename_retry_cnt++;
3425			goto  retry_firstlock;
3426		}
3427	}
3428
3429	if (sip == tdp) {
3430		error = EINVAL;
3431		goto errout;
3432	}
3433	/*
3434	 * Make sure we can delete the source entry.  This requires
3435	 * write permission on the containing directory.
3436	 * Check for sticky directories.
3437	 */
3438	rw_enter(&sdp->i_contents, RW_READER);
3439	rw_enter(&sip->i_contents, RW_READER);
3440	if ((error = ufs_iaccess(sdp, IWRITE, cr, 0)) != 0 ||
3441	    (error = ufs_sticky_remove_access(sdp, sip, cr)) != 0) {
3442		rw_exit(&sip->i_contents);
3443		rw_exit(&sdp->i_contents);
3444		goto errout;
3445	}
3446
3447	/*
3448	 * If this is a rename of a directory and the parent is
3449	 * different (".." must be changed), then the source
3450	 * directory must not be in the directory hierarchy
3451	 * above the target, as this would orphan everything
3452	 * below the source directory.  Also the user must have
3453	 * write permission in the source so as to be able to
3454	 * change "..".
3455	 */
3456	if ((((sip->i_mode & IFMT) == IFDIR) ||
3457	    ((sip->i_mode & IFMT) == IFATTRDIR)) && sdp != tdp) {
3458		ino_t	inum;
3459
3460		if (error = ufs_iaccess(sip, IWRITE, cr, 0)) {
3461			rw_exit(&sip->i_contents);
3462			rw_exit(&sdp->i_contents);
3463			goto errout;
3464		}
3465		inum = sip->i_number;
3466		rw_exit(&sip->i_contents);
3467		rw_exit(&sdp->i_contents);
3468		if ((error = ufs_dircheckpath(inum, tdp, sdp, cr))) {
3469			/*
3470			 * If we got EAGAIN ufs_dircheckpath detected a
3471			 * potential deadlock and backed out. We need
3472			 * to retry the operation since sdp and tdp have
3473			 * to be released to avoid the deadlock.
3474			 */
3475			if (error == EAGAIN) {
3476				rw_exit(&tdp->i_rwlock);
3477				if (tdp != sdp)
3478					rw_exit(&sdp->i_rwlock);
3479				delay(ufs_rename_backoff_delay);
3480				ufs_rename_dircheck_retry_cnt++;
3481				goto retry;
3482			}
3483			goto errout;
3484		}
3485	} else {
3486		rw_exit(&sip->i_contents);
3487		rw_exit(&sdp->i_contents);
3488	}
3489
3490
3491	/*
3492	 * Check for renaming '.' or '..' or alias of '.'
3493	 */
3494	if (strcmp(snm, ".") == 0 || strcmp(snm, "..") == 0 || sdp == sip) {
3495		error = EINVAL;
3496		goto errout;
3497	}
3498
3499	/*
3500	 * Simultaneous renames can deadlock in ufs_dircheckpath since it
3501	 * tries to traverse back the file tree with both tdp and sdp held
3502	 * as RW_WRITER. To avoid that we have to hold the tdp and sdp locks
3503	 * as RW_READERS  till ufs_dircheckpath is done.
3504	 * Now that ufs_dircheckpath is done with, we can upgrade the locks
3505	 * to RW_WRITER.
3506	 */
3507	if (!rw_tryupgrade(&tdp->i_rwlock)) {
3508		/*
3509		 * The upgrade failed. We got to give away the lock
3510		 * as to avoid deadlocking with someone else who is
3511		 * waiting for writer lock. With the lock gone, we
3512		 * cannot be sure the checks done above will hold
3513		 * good when we eventually get them back as writer.
3514		 * So if we can't upgrade we drop the locks and retry
3515		 * everything again.
3516		 */
3517		rw_exit(&tdp->i_rwlock);
3518		if (tdp != sdp)
3519			rw_exit(&sdp->i_rwlock);
3520		delay(ufs_rename_backoff_delay);
3521		ufs_rename_upgrade_retry_cnt++;
3522		goto retry;
3523	}
3524	if (tdp != sdp) {
3525		if (!rw_tryupgrade(&sdp->i_rwlock)) {
3526			/*
3527			 * The upgrade failed. We got to give away the lock
3528			 * as to avoid deadlocking with someone else who is
3529			 * waiting for writer lock. With the lock gone, we
3530			 * cannot be sure the checks done above will hold
3531			 * good when we eventually get them back as writer.
3532			 * So if we can't upgrade we drop the locks and retry
3533			 * everything again.
3534			 */
3535			rw_exit(&tdp->i_rwlock);
3536			rw_exit(&sdp->i_rwlock);
3537			delay(ufs_rename_backoff_delay);
3538			ufs_rename_upgrade_retry_cnt++;
3539			goto retry;
3540		}
3541	}
3542
3543	/*
3544	 * Now that all the locks are held check to make sure another thread
3545	 * didn't slip in and take out the sip.
3546	 */
3547	slot.status = NONE;
3548	if ((sip->i_ctime.tv_usec * 1000) > now.tv_nsec ||
3549	    sip->i_ctime.tv_sec > now.tv_sec) {
3550		rw_enter(&sdp->i_ufsvfs->vfs_dqrwlock, RW_READER);
3551		rw_enter(&sdp->i_contents, RW_WRITER);
3552		error = ufs_dircheckforname(sdp, snm, strlen(snm), &slot,
3553		    &ip, cr, 0);
3554		rw_exit(&sdp->i_contents);
3555		rw_exit(&sdp->i_ufsvfs->vfs_dqrwlock);
3556		if (error) {
3557			goto errout;
3558		}
3559		if (ip == NULL) {
3560			error = ENOENT;
3561			goto errout;
3562		} else {
3563			/*
3564			 * If the inode was found need to drop the v_count
3565			 * so as not to keep the filesystem from being
3566			 * unmounted at a later time.
3567			 */
3568			VN_RELE(ITOV(ip));
3569		}
3570
3571		/*
3572		 * Release the slot.fbp that has the page mapped and
3573		 * locked SE_SHARED, and could be used in in
3574		 * ufs_direnter_lr() which needs to get the SE_EXCL lock
3575		 * on said page.
3576		 */
3577		if (slot.fbp) {
3578			fbrelse(slot.fbp, S_OTHER);
3579			slot.fbp = NULL;
3580		}
3581	}
3582
3583	/*
3584	 * Link source to the target.
3585	 */
3586	if (error = ufs_direnter_lr(tdp, tnm, DE_RENAME, sdp, sip, cr)) {
3587		/*
3588		 * ESAME isn't really an error; it indicates that the
3589		 * operation should not be done because the source and target
3590		 * are the same file, but that no error should be reported.
3591		 */
3592		if (error == ESAME)
3593			error = 0;
3594		goto errout;
3595	}
3596
3597	if (error == 0 && tvp != NULL)
3598		vnevent_rename_dest(tvp, tdvp, tnm, ct);
3599
3600	/*
3601	 * Unlink the source.
3602	 * Remove the source entry.  ufs_dirremove() checks that the entry
3603	 * still reflects sip, and returns an error if it doesn't.
3604	 * If the entry has changed just forget about it.  Release
3605	 * the source inode.
3606	 */
3607	if ((error = ufs_dirremove(sdp, snm, sip, (struct vnode *)0,
3608	    DR_RENAME, cr)) == ENOENT)
3609		error = 0;
3610
3611	if (error == 0) {
3612		vnevent_rename_src(ITOV(sip), sdvp, snm, ct);
3613		/*
3614		 * Notify the target directory of the rename event
3615		 * if source and target directories are not the same.
3616		 */
3617		if (sdvp != tdvp)
3618			vnevent_rename_dest_dir(tdvp, ct);
3619	}
3620
3621errout:
3622	if (slot.fbp)
3623		fbrelse(slot.fbp, S_OTHER);
3624
3625	rw_exit(&tdp->i_rwlock);
3626	if (sdp != tdp) {
3627		rw_exit(&sdp->i_rwlock);
3628	}
3629
3630unlock:
3631	if (tvp != NULL)
3632		VN_RELE(tvp);
3633	if (sip != NULL)
3634		VN_RELE(ITOV(sip));
3635
3636	if (ulp) {
3637		TRANS_END_CSYNC(ufsvfsp, error, issync, TOP_RENAME, trans_size);
3638		ufs_lockfs_end(ulp);
3639	}
3640
3641	return (error);
3642}
3643
3644/*ARGSUSED*/
3645static int
3646ufs_mkdir(struct vnode *dvp, char *dirname, struct vattr *vap,
3647    struct vnode **vpp, struct cred *cr, caller_context_t *ct, int flags,
3648    vsecattr_t *vsecp)
3649{
3650	struct inode *ip;
3651	struct inode *xip;
3652	struct ufsvfs *ufsvfsp;
3653	struct ulockfs *ulp;
3654	int error;
3655	int issync;
3656	int trans_size;
3657	int indeadlock;
3658	int retry = 1;
3659
3660	ASSERT((vap->va_mask & (AT_TYPE|AT_MODE)) == (AT_TYPE|AT_MODE));
3661
3662	/*
3663	 * Can't make directory in attr hidden dir
3664	 */
3665	if ((VTOI(dvp)->i_mode & IFMT) == IFATTRDIR)
3666		return (EINVAL);
3667
3668again:
3669	ip = VTOI(dvp);
3670	ufsvfsp = ip->i_ufsvfs;
3671	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_MKDIR_MASK);
3672	if (error)
3673		goto out;
3674	if (ulp)
3675		TRANS_BEGIN_CSYNC(ufsvfsp, issync, TOP_MKDIR,
3676		    trans_size = (int)TOP_MKDIR_SIZE(ip));
3677
3678	/*
3679	 * ufs_tryirwlock_trans uses rw_tryenter and checks for SLOCK
3680	 * to avoid i_rwlock, ufs_lockfs_begin deadlock. If deadlock
3681	 * possible, retries the operation.
3682	 */
3683	ufs_tryirwlock_trans(&ip->i_rwlock, RW_WRITER, TOP_MKDIR, retry);
3684	if (indeadlock)
3685		goto again;
3686
3687	error = ufs_direnter_cm(ip, dirname, DE_MKDIR, vap, &xip, cr,
3688	    (retry ? IQUIET : 0));
3689	if (error == EAGAIN) {
3690		if (ulp) {
3691			TRANS_END_CSYNC(ufsvfsp, error, issync, TOP_MKDIR,
3692			    trans_size);
3693			ufs_lockfs_end(ulp);
3694		}
3695		goto again;
3696	}
3697
3698	rw_exit(&ip->i_rwlock);
3699	if (error == 0) {
3700		ip = xip;
3701		*vpp = ITOV(ip);
3702	} else if (error == EEXIST)
3703		VN_RELE(ITOV(xip));
3704
3705	if (ulp) {
3706		int terr = 0;
3707		TRANS_END_CSYNC(ufsvfsp, terr, issync, TOP_MKDIR, trans_size);
3708		ufs_lockfs_end(ulp);
3709		if (error == 0)
3710			error = terr;
3711	}
3712out:
3713	if ((error == ENOSPC) && retry && TRANS_ISTRANS(ufsvfsp)) {
3714		ufs_delete_drain_wait(ufsvfsp, 1);
3715		retry = 0;
3716		goto again;
3717	}
3718
3719	return (error);
3720}
3721
3722/*ARGSUSED*/
3723static int
3724ufs_rmdir(struct vnode *vp, char *nm, struct vnode *cdir, struct cred *cr,
3725    caller_context_t *ct, int flags)
3726{
3727	struct inode *ip = VTOI(vp);
3728	struct ufsvfs *ufsvfsp = ip->i_ufsvfs;
3729	struct ulockfs *ulp;
3730	vnode_t *rmvp = NULL;	/* Vnode of removed directory */
3731	int error;
3732	int issync;
3733	int trans_size;
3734	int indeadlock;
3735
3736	/*
3737	 * don't let the delete queue get too long
3738	 */
3739	if (ufsvfsp == NULL) {
3740		error = EIO;
3741		goto out;
3742	}
3743	if (ufsvfsp->vfs_delete.uq_ne > ufs_idle_max)
3744		ufs_delete_drain(vp->v_vfsp, 1, 1);
3745
3746	error = ufs_eventlookup(vp, nm, cr, &rmvp);
3747	if (rmvp != NULL) {
3748		/* Only send the event if there were no errors */
3749		if (error == 0)
3750			vnevent_rmdir(rmvp, vp, nm, ct);
3751		VN_RELE(rmvp);
3752	}
3753
3754retry_rmdir:
3755	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_RMDIR_MASK);
3756	if (error)
3757		goto out;
3758
3759	if (ulp)
3760		TRANS_BEGIN_CSYNC(ufsvfsp, issync, TOP_RMDIR,
3761		    trans_size = TOP_RMDIR_SIZE);
3762
3763	/*
3764	 * ufs_tryirwlock_trans uses rw_tryenter and checks for SLOCK
3765	 * to avoid i_rwlock, ufs_lockfs_begin deadlock. If deadlock
3766	 * possible, retries the operation.
3767	 */
3768	ufs_tryirwlock_trans(&ip->i_rwlock, RW_WRITER, TOP_RMDIR, retry);
3769	if (indeadlock)
3770		goto retry_rmdir;
3771	error = ufs_dirremove(ip, nm, (struct inode *)0, cdir, DR_RMDIR, cr);
3772
3773	rw_exit(&ip->i_rwlock);
3774
3775	if (ulp) {
3776		TRANS_END_CSYNC(ufsvfsp, error, issync, TOP_RMDIR,
3777		    trans_size);
3778		ufs_lockfs_end(ulp);
3779	}
3780
3781out:
3782	return (error);
3783}
3784
3785/* ARGSUSED */
3786static int
3787ufs_readdir(struct vnode *vp, struct uio *uiop, struct cred *cr, int *eofp,
3788    caller_context_t *ct, int flags)
3789{
3790	struct iovec *iovp;
3791	struct inode *ip;
3792	struct direct *idp;
3793	struct dirent64 *odp;
3794	struct fbuf *fbp;
3795	struct ufsvfs *ufsvfsp;
3796	struct ulockfs *ulp;
3797	caddr_t outbuf;
3798	size_t bufsize;
3799	uint_t offset;
3800	uint_t bytes_wanted, total_bytes_wanted;
3801	int incount = 0;
3802	int outcount = 0;
3803	int error;
3804
3805	ip = VTOI(vp);
3806	ASSERT(RW_READ_HELD(&ip->i_rwlock));
3807
3808	if (uiop->uio_loffset >= MAXOFF32_T) {
3809		if (eofp)
3810			*eofp = 1;
3811		return (0);
3812	}
3813
3814	/*
3815	 * Check if we have been called with a valid iov_len
3816	 * and bail out if not, otherwise we may potentially loop
3817	 * forever further down.
3818	 */
3819	if (uiop->uio_iov->iov_len <= 0) {
3820		error = EINVAL;
3821		goto out;
3822	}
3823
3824	/*
3825	 * Large Files: When we come here we are guaranteed that
3826	 * uio_offset can be used safely. The high word is zero.
3827	 */
3828
3829	ufsvfsp = ip->i_ufsvfs;
3830	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_READDIR_MASK);
3831	if (error)
3832		goto out;
3833
3834	iovp = uiop->uio_iov;
3835	total_bytes_wanted = iovp->iov_len;
3836
3837	/* Large Files: directory files should not be "large" */
3838
3839	ASSERT(ip->i_size <= MAXOFF32_T);
3840
3841	/* Force offset to be valid (to guard against bogus lseek() values) */
3842	offset = (uint_t)uiop->uio_offset & ~(DIRBLKSIZ - 1);
3843
3844	/* Quit if at end of file or link count of zero (posix) */
3845	if (offset >= (uint_t)ip->i_size || ip->i_nlink <= 0) {
3846		if (eofp)
3847			*eofp = 1;
3848		error = 0;
3849		goto unlock;
3850	}
3851
3852	/*
3853	 * Get space to change directory entries into fs independent format.
3854	 * Do fast alloc for the most commonly used-request size (filesystem
3855	 * block size).
3856	 */
3857	if (uiop->uio_segflg != UIO_SYSSPACE || uiop->uio_iovcnt != 1) {
3858		bufsize = total_bytes_wanted;
3859		outbuf = kmem_alloc(bufsize, KM_SLEEP);
3860		odp = (struct dirent64 *)outbuf;
3861	} else {
3862		bufsize = total_bytes_wanted;
3863		odp = (struct dirent64 *)iovp->iov_base;
3864	}
3865
3866nextblk:
3867	bytes_wanted = total_bytes_wanted;
3868
3869	/* Truncate request to file size */
3870	if (offset + bytes_wanted > (int)ip->i_size)
3871		bytes_wanted = (int)(ip->i_size - offset);
3872
3873	/* Comply with MAXBSIZE boundary restrictions of fbread() */
3874	if ((offset & MAXBOFFSET) + bytes_wanted > MAXBSIZE)
3875		bytes_wanted = MAXBSIZE - (offset & MAXBOFFSET);
3876
3877	/*
3878	 * Read in the next chunk.
3879	 * We are still holding the i_rwlock.
3880	 */
3881	error = fbread(vp, (offset_t)offset, bytes_wanted, S_OTHER, &fbp);
3882
3883	if (error)
3884		goto update_inode;
3885	if (!ULOCKFS_IS_NOIACC(ITOUL(ip)) && (ip->i_fs->fs_ronly == 0) &&
3886	    (!ufsvfsp->vfs_noatime)) {
3887		ip->i_flag |= IACC;
3888	}
3889	incount = 0;
3890	idp = (struct direct *)fbp->fb_addr;
3891	if (idp->d_ino == 0 && idp->d_reclen == 0 && idp->d_namlen == 0) {
3892		cmn_err(CE_WARN, "ufs_readdir: bad dir, inumber = %llu, "
3893		    "fs = %s\n",
3894		    (u_longlong_t)ip->i_number, ufsvfsp->vfs_fs->fs_fsmnt);
3895		fbrelse(fbp, S_OTHER);
3896		error = ENXIO;
3897		goto update_inode;
3898	}
3899	/* Transform to file-system independent format */
3900	while (incount < bytes_wanted) {
3901		/*
3902		 * If the current directory entry is mangled, then skip
3903		 * to the next block.  It would be nice to set the FSBAD
3904		 * flag in the super-block so that a fsck is forced on
3905		 * next reboot, but locking is a problem.
3906		 */
3907		if (idp->d_reclen & 0x3) {
3908			offset = (offset + DIRBLKSIZ) & ~(DIRBLKSIZ-1);
3909			break;
3910		}
3911
3912		/* Skip to requested offset and skip empty entries */
3913		if (idp->d_ino != 0 && offset >= (uint_t)uiop->uio_offset) {
3914			ushort_t this_reclen =
3915			    DIRENT64_RECLEN(idp->d_namlen);
3916			/* Buffer too small for any entries */
3917			if (!outcount && this_reclen > bufsize) {
3918				fbrelse(fbp, S_OTHER);
3919				error = EINVAL;
3920				goto update_inode;
3921			}
3922			/* If would overrun the buffer, quit */
3923			if (outcount + this_reclen > bufsize) {
3924				break;
3925			}
3926			/* Take this entry */
3927			odp->d_ino = (ino64_t)idp->d_ino;
3928			odp->d_reclen = (ushort_t)this_reclen;
3929			odp->d_off = (offset_t)(offset + idp->d_reclen);
3930
3931			/* use strncpy(9f) to zero out uninitialized bytes */
3932
3933			ASSERT(strlen(idp->d_name) + 1 <=
3934			    DIRENT64_NAMELEN(this_reclen));
3935			(void) strncpy(odp->d_name, idp->d_name,
3936			    DIRENT64_NAMELEN(this_reclen));
3937			outcount += odp->d_reclen;
3938			odp = (struct dirent64 *)
3939			    ((intptr_t)odp + odp->d_reclen);
3940			ASSERT(outcount <= bufsize);
3941		}
3942		if (idp->d_reclen) {
3943			incount += idp->d_reclen;
3944			offset += idp->d_reclen;
3945			idp = (struct direct *)((intptr_t)idp + idp->d_reclen);
3946		} else {
3947			offset = (offset + DIRBLKSIZ) & ~(DIRBLKSIZ-1);
3948			break;
3949		}
3950	}
3951	/* Release the chunk */
3952	fbrelse(fbp, S_OTHER);
3953
3954	/* Read whole block, but got no entries, read another if not eof */
3955
3956	/*
3957	 * Large Files: casting i_size to int here is not a problem
3958	 * because directory sizes are always less than MAXOFF32_T.
3959	 * See assertion above.
3960	 */
3961
3962	if (offset < (int)ip->i_size && !outcount)
3963		goto nextblk;
3964
3965	/* Copy out the entry data */
3966	if (uiop->uio_segflg == UIO_SYSSPACE && uiop->uio_iovcnt == 1) {
3967		iovp->iov_base += outcount;
3968		iovp->iov_len -= outcount;
3969		uiop->uio_resid -= outcount;
3970		uiop->uio_offset = offset;
3971	} else if ((error = uiomove(outbuf, (long)outcount, UIO_READ,
3972	    uiop)) == 0)
3973		uiop->uio_offset = offset;
3974update_inode:
3975	ITIMES(ip);
3976	if (uiop->uio_segflg != UIO_SYSSPACE || uiop->uio_iovcnt != 1)
3977		kmem_free(outbuf, bufsize);
3978
3979	if (eofp && error == 0)
3980		*eofp = (uiop->uio_offset >= (int)ip->i_size);
3981unlock:
3982	if (ulp) {
3983		ufs_lockfs_end(ulp);
3984	}
3985out:
3986	return (error);
3987}
3988
3989/*ARGSUSED*/
3990static int
3991ufs_symlink(struct vnode *dvp, char *linkname, struct vattr *vap, char *target,
3992    struct cred *cr, caller_context_t *ct, int flags)
3993{
3994	struct inode *ip, *dip = VTOI(dvp);
3995	struct ufsvfs *ufsvfsp = dip->i_ufsvfs;
3996	struct ulockfs *ulp;
3997	int error;
3998	int issync;
3999	int trans_size;
4000	int residual;
4001	int ioflag;
4002	int retry = 1;
4003
4004	/*
4005	 * No symlinks in attrdirs at this time
4006	 */
4007	if ((VTOI(dvp)->i_mode & IFMT) == IFATTRDIR)
4008		return (EINVAL);
4009
4010again:
4011	ip = (struct inode *)NULL;
4012	vap->va_type = VLNK;
4013	vap->va_rdev = 0;
4014
4015	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_SYMLINK_MASK);
4016	if (error)
4017		goto out;
4018
4019	if (ulp)
4020		TRANS_BEGIN_CSYNC(ufsvfsp, issync, TOP_SYMLINK,
4021		    trans_size = (int)TOP_SYMLINK_SIZE(dip));
4022
4023	/*
4024	 * We must create the inode before the directory entry, to avoid
4025	 * racing with readlink().  ufs_dirmakeinode requires that we
4026	 * hold the quota lock as reader, and directory locks as writer.
4027	 */
4028
4029	rw_enter(&dip->i_rwlock, RW_WRITER);
4030	rw_enter(&ufsvfsp->vfs_dqrwlock, RW_READER);
4031	rw_enter(&dip->i_contents, RW_WRITER);
4032
4033	/*
4034	 * Suppress any out of inodes messages if we will retry on
4035	 * ENOSP
4036	 */
4037	if (retry)
4038		dip->i_flag |= IQUIET;
4039
4040	error = ufs_dirmakeinode(dip, &ip, vap, DE_SYMLINK, cr);
4041
4042	dip->i_flag &= ~IQUIET;
4043
4044	rw_exit(&dip->i_contents);
4045	rw_exit(&ufsvfsp->vfs_dqrwlock);
4046	rw_exit(&dip->i_rwlock);
4047
4048	if (error)
4049		goto unlock;
4050
4051	/*
4052	 * OK.  The inode has been created.  Write out the data of the
4053	 * symbolic link.  Since symbolic links are metadata, and should
4054	 * remain consistent across a system crash, we need to force the
4055	 * data out synchronously.
4056	 *
4057	 * (This is a change from the semantics in earlier releases, which
4058	 * only created symbolic links synchronously if the semi-documented
4059	 * 'syncdir' option was set, or if we were being invoked by the NFS
4060	 * server, which requires symbolic links to be created synchronously.)
4061	 *
4062	 * We need to pass in a pointer for the residual length; otherwise
4063	 * ufs_rdwri() will always return EIO if it can't write the data,
4064	 * even if the error was really ENOSPC or EDQUOT.
4065	 */
4066
4067	ioflag = FWRITE | FDSYNC;
4068	residual = 0;
4069
4070	rw_enter(&ufsvfsp->vfs_dqrwlock, RW_READER);
4071	rw_enter(&ip->i_contents, RW_WRITER);
4072
4073	/*
4074	 * Suppress file system full messages if we will retry
4075	 */
4076	if (retry)
4077		ip->i_flag |= IQUIET;
4078
4079	error = ufs_rdwri(UIO_WRITE, ioflag, ip, target, strlen(target),
4080	    (offset_t)0, UIO_SYSSPACE, &residual, cr);
4081
4082	ip->i_flag &= ~IQUIET;
4083
4084	if (error) {
4085		rw_exit(&ip->i_contents);
4086		rw_exit(&ufsvfsp->vfs_dqrwlock);
4087		goto remove;
4088	}
4089
4090	/*
4091	 * If the link's data is small enough, we can cache it in the inode.
4092	 * This is a "fast symbolic link".  We don't use the first direct
4093	 * block because that's actually used to point at the symbolic link's
4094	 * contents on disk; but we know that none of the other direct or
4095	 * indirect blocks can be used because symbolic links are restricted
4096	 * to be smaller than a file system block.
4097	 */
4098
4099	ASSERT(MAXPATHLEN <= VBSIZE(ITOV(ip)));
4100
4101	if (ip->i_size > 0 && ip->i_size <= FSL_SIZE) {
4102		if (kcopy(target, &ip->i_db[1], ip->i_size) == 0) {
4103			ip->i_flag |= IFASTSYMLNK;
4104		} else {
4105			int i;
4106			/* error, clear garbage left behind */
4107			for (i = 1; i < NDADDR; i++)
4108				ip->i_db[i] = 0;
4109			for (i = 0; i < NIADDR; i++)
4110				ip->i_ib[i] = 0;
4111		}
4112	}
4113
4114	rw_exit(&ip->i_contents);
4115	rw_exit(&ufsvfsp->vfs_dqrwlock);
4116
4117	/*
4118	 * OK.  We've successfully created the symbolic link.  All that
4119	 * remains is to insert it into the appropriate directory.
4120	 */
4121
4122	rw_enter(&dip->i_rwlock, RW_WRITER);
4123	error = ufs_direnter_lr(dip, linkname, DE_SYMLINK, NULL, ip, cr);
4124	rw_exit(&dip->i_rwlock);
4125
4126	/*
4127	 * Fall through into remove-on-error code.  We're either done, or we
4128	 * need to remove the inode (if we couldn't insert it).
4129	 */
4130
4131remove:
4132	if (error && (ip != NULL)) {
4133		rw_enter(&ip->i_contents, RW_WRITER);
4134		ip->i_nlink--;
4135		ip->i_flag |= ICHG;
4136		ip->i_seq++;
4137		ufs_setreclaim(ip);
4138		rw_exit(&ip->i_contents);
4139	}
4140
4141unlock:
4142	if (ip != NULL)
4143		VN_RELE(ITOV(ip));
4144
4145	if (ulp) {
4146		int terr = 0;
4147
4148		TRANS_END_CSYNC(ufsvfsp, terr, issync, TOP_SYMLINK,
4149		    trans_size);
4150		ufs_lockfs_end(ulp);
4151		if (error == 0)
4152			error = terr;
4153	}
4154
4155	/*
4156	 * We may have failed due to lack of an inode or of a block to
4157	 * store the target in.  Try flushing the delete queue to free
4158	 * logically-available things up and try again.
4159	 */
4160	if ((error == ENOSPC) && retry && TRANS_ISTRANS(ufsvfsp)) {
4161		ufs_delete_drain_wait(ufsvfsp, 1);
4162		retry = 0;
4163		goto again;
4164	}
4165
4166out:
4167	return (error);
4168}
4169
4170/*
4171 * Ufs specific routine used to do ufs io.
4172 */
4173int
4174ufs_rdwri(enum uio_rw rw, int ioflag, struct inode *ip, caddr_t base,
4175    ssize_t len, offset_t offset, enum uio_seg seg, int *aresid,
4176    struct cred *cr)
4177{
4178	struct uio auio;
4179	struct iovec aiov;
4180	int error;
4181
4182	ASSERT(RW_LOCK_HELD(&ip->i_contents));
4183
4184	bzero((caddr_t)&auio, sizeof (uio_t));
4185	bzero((caddr_t)&aiov, sizeof (iovec_t));
4186
4187	aiov.iov_base = base;
4188	aiov.iov_len = len;
4189	auio.uio_iov = &aiov;
4190	auio.uio_iovcnt = 1;
4191	auio.uio_loffset = offset;
4192	auio.uio_segflg = (short)seg;
4193	auio.uio_resid = len;
4194
4195	if (rw == UIO_WRITE) {
4196		auio.uio_fmode = FWRITE;
4197		auio.uio_extflg = UIO_COPY_DEFAULT;
4198		auio.uio_llimit = curproc->p_fsz_ctl;
4199		error = wrip(ip, &auio, ioflag, cr);
4200	} else {
4201		auio.uio_fmode = FREAD;
4202		auio.uio_extflg = UIO_COPY_CACHED;
4203		auio.uio_llimit = MAXOFFSET_T;
4204		error = rdip(ip, &auio, ioflag, cr);
4205	}
4206
4207	if (aresid) {
4208		*aresid = auio.uio_resid;
4209	} else if (auio.uio_resid) {
4210		error = EIO;
4211	}
4212	return (error);
4213}
4214
4215/*ARGSUSED*/
4216static int
4217ufs_fid(struct vnode *vp, struct fid *fidp, caller_context_t *ct)
4218{
4219	struct ufid *ufid;
4220	struct inode *ip = VTOI(vp);
4221
4222	if (ip->i_ufsvfs == NULL)
4223		return (EIO);
4224
4225	if (fidp->fid_len < (sizeof (struct ufid) - sizeof (ushort_t))) {
4226		fidp->fid_len = sizeof (struct ufid) - sizeof (ushort_t);
4227		return (ENOSPC);
4228	}
4229
4230	ufid = (struct ufid *)fidp;
4231	bzero((char *)ufid, sizeof (struct ufid));
4232	ufid->ufid_len = sizeof (struct ufid) - sizeof (ushort_t);
4233	ufid->ufid_ino = ip->i_number;
4234	ufid->ufid_gen = ip->i_gen;
4235
4236	return (0);
4237}
4238
4239/* ARGSUSED2 */
4240static int
4241ufs_rwlock(struct vnode *vp, int write_lock, caller_context_t *ctp)
4242{
4243	struct inode	*ip = VTOI(vp);
4244	struct ufsvfs	*ufsvfsp;
4245	int		forcedirectio;
4246
4247	/*
4248	 * Read case is easy.
4249	 */
4250	if (!write_lock) {
4251		rw_enter(&ip->i_rwlock, RW_READER);
4252		return (V_WRITELOCK_FALSE);
4253	}
4254
4255	/*
4256	 * Caller has requested a writer lock, but that inhibits any
4257	 * concurrency in the VOPs that follow. Acquire the lock shared
4258	 * and defer exclusive access until it is known to be needed in
4259	 * other VOP handlers. Some cases can be determined here.
4260	 */
4261
4262	/*
4263	 * If directio is not set, there is no chance of concurrency,
4264	 * so just acquire the lock exclusive. Beware of a forced
4265	 * unmount before looking at the mount option.
4266	 */
4267	ufsvfsp = ip->i_ufsvfs;
4268	forcedirectio = ufsvfsp ? ufsvfsp->vfs_forcedirectio : 0;
4269	if (!(ip->i_flag & IDIRECTIO || forcedirectio) ||
4270	    !ufs_allow_shared_writes) {
4271		rw_enter(&ip->i_rwlock, RW_WRITER);
4272		return (V_WRITELOCK_TRUE);
4273	}
4274
4275	/*
4276	 * Mandatory locking forces acquiring i_rwlock exclusive.
4277	 */
4278	if (MANDLOCK(vp, ip->i_mode)) {
4279		rw_enter(&ip->i_rwlock, RW_WRITER);
4280		return (V_WRITELOCK_TRUE);
4281	}
4282
4283	/*
4284	 * Acquire the lock shared in case a concurrent write follows.
4285	 * Mandatory locking could have become enabled before the lock
4286	 * was acquired. Re-check and upgrade if needed.
4287	 */
4288	rw_enter(&ip->i_rwlock, RW_READER);
4289	if (MANDLOCK(vp, ip->i_mode)) {
4290		rw_exit(&ip->i_rwlock);
4291		rw_enter(&ip->i_rwlock, RW_WRITER);
4292		return (V_WRITELOCK_TRUE);
4293	}
4294	return (V_WRITELOCK_FALSE);
4295}
4296
4297/*ARGSUSED*/
4298static void
4299ufs_rwunlock(struct vnode *vp, int write_lock, caller_context_t *ctp)
4300{
4301	struct inode	*ip = VTOI(vp);
4302
4303	rw_exit(&ip->i_rwlock);
4304}
4305
4306/* ARGSUSED */
4307static int
4308ufs_seek(struct vnode *vp, offset_t ooff, offset_t *noffp, caller_context_t *ct)
4309{
4310	return ((*noffp < 0 || *noffp > MAXOFFSET_T) ? EINVAL : 0);
4311}
4312
4313/* ARGSUSED */
4314static int
4315ufs_frlock(struct vnode *vp, int cmd, struct flock64 *bfp, int flag,
4316    offset_t offset, struct flk_callback *flk_cbp, struct cred *cr,
4317    caller_context_t *ct)
4318{
4319	struct inode *ip = VTOI(vp);
4320
4321	if (ip->i_ufsvfs == NULL)
4322		return (EIO);
4323
4324	/*
4325	 * If file is being mapped, disallow frlock.
4326	 * XXX I am not holding tlock while checking i_mapcnt because the
4327	 * current locking strategy drops all locks before calling fs_frlock.
4328	 * So, mapcnt could change before we enter fs_frlock making is
4329	 * meaningless to have held tlock in the first place.
4330	 */
4331	if (ip->i_mapcnt > 0 && MANDLOCK(vp, ip->i_mode))
4332		return (EAGAIN);
4333	return (fs_frlock(vp, cmd, bfp, flag, offset, flk_cbp, cr, ct));
4334}
4335
4336/* ARGSUSED */
4337static int
4338ufs_space(struct vnode *vp, int cmd, struct flock64 *bfp, int flag,
4339    offset_t offset, cred_t *cr, caller_context_t *ct)
4340{
4341	struct ufsvfs *ufsvfsp = VTOI(vp)->i_ufsvfs;
4342	struct ulockfs *ulp;
4343	int error;
4344
4345	if ((error = convoff(vp, bfp, 0, offset)) == 0) {
4346		if (cmd == F_FREESP) {
4347			error = ufs_lockfs_begin(ufsvfsp, &ulp,
4348			    ULOCKFS_SPACE_MASK);
4349			if (error)
4350				return (error);
4351			error = ufs_freesp(vp, bfp, flag, cr);
4352
4353			if (error == 0 && bfp->l_start == 0)
4354				vnevent_truncate(vp, ct);
4355		} else if (cmd == F_ALLOCSP) {
4356			error = ufs_lockfs_begin(ufsvfsp, &ulp,
4357			    ULOCKFS_FALLOCATE_MASK);
4358			if (error)
4359				return (error);
4360			error = ufs_allocsp(vp, bfp, cr);
4361		} else
4362			return (EINVAL); /* Command not handled here */
4363
4364		if (ulp)
4365			ufs_lockfs_end(ulp);
4366
4367	}
4368	return (error);
4369}
4370
4371/*
4372 * Used to determine if read ahead should be done. Also used to
4373 * to determine when write back occurs.
4374 */
4375#define	CLUSTSZ(ip)		((ip)->i_ufsvfs->vfs_ioclustsz)
4376
4377/*
4378 * A faster version of ufs_getpage.
4379 *
4380 * We optimize by inlining the pvn_getpages iterator, eliminating
4381 * calls to bmap_read if file doesn't have UFS holes, and avoiding
4382 * the overhead of page_exists().
4383 *
4384 * When files has UFS_HOLES and ufs_getpage is called with S_READ,
4385 * we set *protp to PROT_READ to avoid calling bmap_read. This approach
4386 * victimizes performance when a file with UFS holes is faulted
4387 * first in the S_READ mode, and then in the S_WRITE mode. We will get
4388 * two MMU faults in this case.
4389 *
4390 * XXX - the inode fields which control the sequential mode are not
4391 *	 protected by any mutex. The read ahead will act wild if
4392 *	 multiple processes will access the file concurrently and
4393 *	 some of them in sequential mode. One particulary bad case
4394 *	 is if another thread will change the value of i_nextrio between
4395 *	 the time this thread tests the i_nextrio value and then reads it
4396 *	 again to use it as the offset for the read ahead.
4397 */
4398/*ARGSUSED*/
4399static int
4400ufs_getpage(struct vnode *vp, offset_t off, size_t len, uint_t *protp,
4401    page_t *plarr[], size_t plsz, struct seg *seg, caddr_t addr,
4402    enum seg_rw rw, struct cred *cr, caller_context_t *ct)
4403{
4404	u_offset_t	uoff = (u_offset_t)off; /* type conversion */
4405	u_offset_t	pgoff;
4406	u_offset_t	eoff;
4407	struct inode	*ip = VTOI(vp);
4408	struct ufsvfs	*ufsvfsp = ip->i_ufsvfs;
4409	struct fs	*fs;
4410	struct ulockfs	*ulp;
4411	page_t		**pl;
4412	caddr_t		pgaddr;
4413	krw_t		rwtype;
4414	int		err;
4415	int		has_holes;
4416	int		beyond_eof;
4417	int		seqmode;
4418	int		pgsize = PAGESIZE;
4419	int		dolock;
4420	int		do_qlock;
4421	int		trans_size;
4422
4423	ASSERT((uoff & PAGEOFFSET) == 0);
4424
4425	if (protp)
4426		*protp = PROT_ALL;
4427
4428	/*
4429	 * Obey the lockfs protocol
4430	 */
4431	err = ufs_lockfs_begin_getpage(ufsvfsp, &ulp, seg,
4432	    rw == S_READ || rw == S_EXEC, protp);
4433	if (err)
4434		goto out;
4435
4436	fs = ufsvfsp->vfs_fs;
4437
4438	if (ulp && (rw == S_CREATE || rw == S_WRITE) &&
4439	    !(vp->v_flag & VISSWAP)) {
4440		/*
4441		 * Try to start a transaction, will return if blocking is
4442		 * expected to occur and the address space is not the
4443		 * kernel address space.
4444		 */
4445		trans_size = TOP_GETPAGE_SIZE(ip);
4446		if (seg->s_as != &kas) {
4447			TRANS_TRY_BEGIN_ASYNC(ufsvfsp, TOP_GETPAGE,
4448			    trans_size, err)
4449			if (err == EWOULDBLOCK) {
4450				/*
4451				 * Use EDEADLK here because the VM code
4452				 * can normally never see this error.
4453				 */
4454				err = EDEADLK;
4455				ufs_lockfs_end(ulp);
4456				goto out;
4457			}
4458		} else {
4459			TRANS_BEGIN_ASYNC(ufsvfsp, TOP_GETPAGE, trans_size);
4460		}
4461	}
4462
4463	if (vp->v_flag & VNOMAP) {
4464		err = ENOSYS;
4465		goto unlock;
4466	}
4467
4468	seqmode = ip->i_nextr == uoff && rw != S_CREATE;
4469
4470	rwtype = RW_READER;		/* start as a reader */
4471	dolock = (rw_owner(&ip->i_contents) != curthread);
4472	/*
4473	 * If this thread owns the lock, i.e., this thread grabbed it
4474	 * as writer somewhere above, then we don't need to grab the
4475	 * lock as reader in this routine.
4476	 */
4477	do_qlock = (rw_owner(&ufsvfsp->vfs_dqrwlock) != curthread);
4478
4479retrylock:
4480	if (dolock) {
4481		/*
4482		 * Grab the quota lock if we need to call
4483		 * bmap_write() below (with i_contents as writer).
4484		 */
4485		if (do_qlock && rwtype == RW_WRITER)
4486			rw_enter(&ufsvfsp->vfs_dqrwlock, RW_READER);
4487		rw_enter(&ip->i_contents, rwtype);
4488	}
4489
4490	/*
4491	 * We may be getting called as a side effect of a bmap using
4492	 * fbread() when the blocks might be being allocated and the
4493	 * size has not yet been up'ed.  In this case we want to be
4494	 * able to return zero pages if we get back UFS_HOLE from
4495	 * calling bmap for a non write case here.  We also might have
4496	 * to read some frags from the disk into a page if we are
4497	 * extending the number of frags for a given lbn in bmap().
4498	 * Large Files: The read of i_size here is atomic because
4499	 * i_contents is held here. If dolock is zero, the lock
4500	 * is held in bmap routines.
4501	 */
4502	beyond_eof = uoff + len >
4503	    P2ROUNDUP_TYPED(ip->i_size, PAGESIZE, u_offset_t);
4504	if (beyond_eof && seg != segkmap) {
4505		if (dolock) {
4506			rw_exit(&ip->i_contents);
4507			if (do_qlock && rwtype == RW_WRITER)
4508				rw_exit(&ufsvfsp->vfs_dqrwlock);
4509		}
4510		err = EFAULT;
4511		goto unlock;
4512	}
4513
4514	/*
4515	 * Must hold i_contents lock throughout the call to pvn_getpages
4516	 * since locked pages are returned from each call to ufs_getapage.
4517	 * Must *not* return locked pages and then try for contents lock
4518	 * due to lock ordering requirements (inode > page)
4519	 */
4520
4521	has_holes = bmap_has_holes(ip);
4522
4523	if ((rw == S_WRITE || rw == S_CREATE) && has_holes && !beyond_eof) {
4524		int	blk_size;
4525		u_offset_t offset;
4526
4527		/*
4528		 * We must acquire the RW_WRITER lock in order to
4529		 * call bmap_write().
4530		 */
4531		if (dolock && rwtype == RW_READER) {
4532			rwtype = RW_WRITER;
4533
4534			/*
4535			 * Grab the quota lock before
4536			 * upgrading i_contents, but if we can't grab it
4537			 * don't wait here due to lock order:
4538			 * vfs_dqrwlock > i_contents.
4539			 */
4540			if (do_qlock &&
4541			    rw_tryenter(&ufsvfsp->vfs_dqrwlock, RW_READER)
4542			    == 0) {
4543				rw_exit(&ip->i_contents);
4544				goto retrylock;
4545			}
4546			if (!rw_tryupgrade(&ip->i_contents)) {
4547				rw_exit(&ip->i_contents);
4548				if (do_qlock)
4549					rw_exit(&ufsvfsp->vfs_dqrwlock);
4550				goto retrylock;
4551			}
4552		}
4553
4554		/*
4555		 * May be allocating disk blocks for holes here as
4556		 * a result of mmap faults. write(2) does the bmap_write
4557		 * in rdip/wrip, not here. We are not dealing with frags
4558		 * in this case.
4559		 */
4560		/*
4561		 * Large Files: We cast fs_bmask field to offset_t
4562		 * just as we do for MAXBMASK because uoff is a 64-bit
4563		 * data type. fs_bmask will still be a 32-bit type
4564		 * as we cannot change any ondisk data structures.
4565		 */
4566
4567		offset = uoff & (offset_t)fs->fs_bmask;
4568		while (offset < uoff + len) {
4569			blk_size = (int)blksize(fs, ip, lblkno(fs, offset));
4570			err = bmap_write(ip, offset, blk_size,
4571			    BI_NORMAL, NULL, cr);
4572			if (ip->i_flag & (ICHG|IUPD))
4573				ip->i_seq++;
4574			if (err)
4575				goto update_inode;
4576			offset += blk_size; /* XXX - make this contig */
4577		}
4578	}
4579
4580	/*
4581	 * Can be a reader from now on.
4582	 */
4583	if (dolock && rwtype == RW_WRITER) {
4584		rw_downgrade(&ip->i_contents);
4585		/*
4586		 * We can release vfs_dqrwlock early so do it, but make
4587		 * sure we don't try to release it again at the bottom.
4588		 */
4589		if (do_qlock) {
4590			rw_exit(&ufsvfsp->vfs_dqrwlock);
4591			do_qlock = 0;
4592		}
4593	}
4594
4595	/*
4596	 * We remove PROT_WRITE in cases when the file has UFS holes
4597	 * because we don't  want to call bmap_read() to check each
4598	 * page if it is backed with a disk block.
4599	 */
4600	if (protp && has_holes && rw != S_WRITE && rw != S_CREATE)
4601		*protp &= ~PROT_WRITE;
4602
4603	err = 0;
4604
4605	/*
4606	 * The loop looks up pages in the range [off, off + len).
4607	 * For each page, we first check if we should initiate an asynchronous
4608	 * read ahead before we call page_lookup (we may sleep in page_lookup
4609	 * for a previously initiated disk read).
4610	 */
4611	eoff = (uoff + len);
4612	for (pgoff = uoff, pgaddr = addr, pl = plarr;
4613	    pgoff < eoff; /* empty */) {
4614		page_t	*pp;
4615		u_offset_t	nextrio;
4616		se_t	se;
4617		int retval;
4618
4619		se = ((rw == S_CREATE || rw == S_OTHER) ? SE_EXCL : SE_SHARED);
4620
4621		/* Handle async getpage (faultahead) */
4622		if (plarr == NULL) {
4623			ip->i_nextrio = pgoff;
4624			(void) ufs_getpage_ra(vp, pgoff, seg, pgaddr);
4625			pgoff += pgsize;
4626			pgaddr += pgsize;
4627			continue;
4628		}
4629		/*
4630		 * Check if we should initiate read ahead of next cluster.
4631		 * We call page_exists only when we need to confirm that
4632		 * we have the current page before we initiate the read ahead.
4633		 */
4634		nextrio = ip->i_nextrio;
4635		if (seqmode &&
4636		    pgoff + CLUSTSZ(ip) >= nextrio && pgoff <= nextrio &&
4637		    nextrio < ip->i_size && page_exists(vp, pgoff)) {
4638			retval = ufs_getpage_ra(vp, pgoff, seg, pgaddr);
4639			/*
4640			 * We always read ahead the next cluster of data
4641			 * starting from i_nextrio. If the page (vp,nextrio)
4642			 * is actually in core at this point, the routine
4643			 * ufs_getpage_ra() will stop pre-fetching data
4644			 * until we read that page in a synchronized manner
4645			 * through ufs_getpage_miss(). So, we should increase
4646			 * i_nextrio if the page (vp, nextrio) exists.
4647			 */
4648			if ((retval == 0) && page_exists(vp, nextrio)) {
4649				ip->i_nextrio = nextrio + pgsize;
4650			}
4651		}
4652
4653		if ((pp = page_lookup(vp, pgoff, se)) != NULL) {
4654			/*
4655			 * We found the page in the page cache.
4656			 */
4657			*pl++ = pp;
4658			pgoff += pgsize;
4659			pgaddr += pgsize;
4660			len -= pgsize;
4661			plsz -= pgsize;
4662		} else  {
4663			/*
4664			 * We have to create the page, or read it from disk.
4665			 */
4666			if (err = ufs_getpage_miss(vp, pgoff, len, seg, pgaddr,
4667			    pl, plsz, rw, seqmode))
4668				goto error;
4669
4670			while (*pl != NULL) {
4671				pl++;
4672				pgoff += pgsize;
4673				pgaddr += pgsize;
4674				len -= pgsize;
4675				plsz -= pgsize;
4676			}
4677		}
4678	}
4679
4680	/*
4681	 * Return pages up to plsz if they are in the page cache.
4682	 * We cannot return pages if there is a chance that they are
4683	 * backed with a UFS hole and rw is S_WRITE or S_CREATE.
4684	 */
4685	if (plarr && !(has_holes && (rw == S_WRITE || rw == S_CREATE))) {
4686
4687		ASSERT((protp == NULL) ||
4688		    !(has_holes && (*protp & PROT_WRITE)));
4689
4690		eoff = pgoff + plsz;
4691		while (pgoff < eoff) {
4692			page_t		*pp;
4693
4694			if ((pp = page_lookup_nowait(vp, pgoff,
4695			    SE_SHARED)) == NULL)
4696				break;
4697
4698			*pl++ = pp;
4699			pgoff += pgsize;
4700			plsz -= pgsize;
4701		}
4702	}
4703
4704	if (plarr)
4705		*pl = NULL;			/* Terminate page list */
4706	ip->i_nextr = pgoff;
4707
4708error:
4709	if (err && plarr) {
4710		/*
4711		 * Release any pages we have locked.
4712		 */
4713		while (pl > &plarr[0])
4714			page_unlock(*--pl);
4715
4716		plarr[0] = NULL;
4717	}
4718
4719update_inode:
4720	/*
4721	 * If the inode is not already marked for IACC (in rdip() for read)
4722	 * and the inode is not marked for no access time update (in wrip()
4723	 * for write) then update the inode access time and mod time now.
4724	 */
4725	if ((ip->i_flag & (IACC | INOACC)) == 0) {
4726		if ((rw != S_OTHER) && (ip->i_mode & IFMT) != IFDIR) {
4727			if (!ULOCKFS_IS_NOIACC(ITOUL(ip)) &&
4728			    (fs->fs_ronly == 0) &&
4729			    (!ufsvfsp->vfs_noatime)) {
4730				mutex_enter(&ip->i_tlock);
4731				ip->i_flag |= IACC;
4732				ITIMES_NOLOCK(ip);
4733				mutex_exit(&ip->i_tlock);
4734			}
4735		}
4736	}
4737
4738	if (dolock) {
4739		rw_exit(&ip->i_contents);
4740		if (do_qlock && rwtype == RW_WRITER)
4741			rw_exit(&ufsvfsp->vfs_dqrwlock);
4742	}
4743
4744unlock:
4745	if (ulp) {
4746		if ((rw == S_CREATE || rw == S_WRITE) &&
4747		    !(vp->v_flag & VISSWAP)) {
4748			TRANS_END_ASYNC(ufsvfsp, TOP_GETPAGE, trans_size);
4749		}
4750		ufs_lockfs_end(ulp);
4751	}
4752out:
4753	return (err);
4754}
4755
4756/*
4757 * ufs_getpage_miss is called when ufs_getpage missed the page in the page
4758 * cache. The page is either read from the disk, or it's created.
4759 * A page is created (without disk read) if rw == S_CREATE, or if
4760 * the page is not backed with a real disk block (UFS hole).
4761 */
4762/* ARGSUSED */
4763static int
4764ufs_getpage_miss(struct vnode *vp, u_offset_t off, size_t len, struct seg *seg,
4765    caddr_t addr, page_t *pl[], size_t plsz, enum seg_rw rw, int seq)
4766{
4767	struct inode	*ip = VTOI(vp);
4768	page_t		*pp;
4769	daddr_t		bn;
4770	size_t		io_len;
4771	int		crpage = 0;
4772	int		err;
4773	int		contig;
4774	int		bsize = ip->i_fs->fs_bsize;
4775
4776	/*
4777	 * Figure out whether the page can be created, or must be
4778	 * must be read from the disk.
4779	 */
4780	if (rw == S_CREATE)
4781		crpage = 1;
4782	else {
4783		contig = 0;
4784		if (err = bmap_read(ip, off, &bn, &contig))
4785			return (err);
4786
4787		crpage = (bn == UFS_HOLE);
4788
4789		/*
4790		 * If its also a fallocated block that hasn't been written to
4791		 * yet, we will treat it just like a UFS_HOLE and create
4792		 * a zero page for it
4793		 */
4794		if (ISFALLOCBLK(ip, bn))
4795			crpage = 1;
4796	}
4797
4798	if (crpage) {
4799		if ((pp = page_create_va(vp, off, PAGESIZE, PG_WAIT, seg,
4800		    addr)) == NULL) {
4801			return (ufs_fault(vp,
4802			    "ufs_getpage_miss: page_create == NULL"));
4803		}
4804
4805		if (rw != S_CREATE)
4806			pagezero(pp, 0, PAGESIZE);
4807
4808		io_len = PAGESIZE;
4809	} else {
4810		u_offset_t	io_off;
4811		uint_t	xlen;
4812		struct buf	*bp;
4813		ufsvfs_t	*ufsvfsp = ip->i_ufsvfs;
4814
4815		/*
4816		 * If access is not in sequential order, we read from disk
4817		 * in bsize units.
4818		 *
4819		 * We limit the size of the transfer to bsize if we are reading
4820		 * from the beginning of the file. Note in this situation we
4821		 * will hedge our bets and initiate an async read ahead of
4822		 * the second block.
4823		 */
4824		if (!seq || off == 0)
4825			contig = MIN(contig, bsize);
4826
4827		pp = pvn_read_kluster(vp, off, seg, addr, &io_off,
4828		    &io_len, off, contig, 0);
4829
4830		/*
4831		 * Some other thread has entered the page.
4832		 * ufs_getpage will retry page_lookup.
4833		 */
4834		if (pp == NULL) {
4835			pl[0] = NULL;
4836			return (0);
4837		}
4838
4839		/*
4840		 * Zero part of the page which we are not
4841		 * going to read from the disk.
4842		 */
4843		xlen = io_len & PAGEOFFSET;
4844		if (xlen != 0)
4845			pagezero(pp->p_prev, xlen, PAGESIZE - xlen);
4846
4847		bp = pageio_setup(pp, io_len, ip->i_devvp, B_READ);
4848		bp->b_edev = ip->i_dev;
4849		bp->b_dev = cmpdev(ip->i_dev);
4850		bp->b_blkno = bn;
4851		bp->b_un.b_addr = (caddr_t)0;
4852		bp->b_file = ip->i_vnode;
4853		bp->b_offset = off;
4854
4855		if (ufsvfsp->vfs_log) {
4856			lufs_read_strategy(ufsvfsp->vfs_log, bp);
4857		} else if (ufsvfsp->vfs_snapshot) {
4858			fssnap_strategy(&ufsvfsp->vfs_snapshot, bp);
4859		} else {
4860			ufsvfsp->vfs_iotstamp = ddi_get_lbolt();
4861			ub.ub_getpages.value.ul++;
4862			(void) bdev_strategy(bp);
4863			lwp_stat_update(LWP_STAT_INBLK, 1);
4864		}
4865
4866		ip->i_nextrio = off + ((io_len + PAGESIZE - 1) & PAGEMASK);
4867
4868		/*
4869		 * If the file access is sequential, initiate read ahead
4870		 * of the next cluster.
4871		 */
4872		if (seq && ip->i_nextrio < ip->i_size)
4873			(void) ufs_getpage_ra(vp, off, seg, addr);
4874		err = biowait(bp);
4875		pageio_done(bp);
4876
4877		if (err) {
4878			pvn_read_done(pp, B_ERROR);
4879			return (err);
4880		}
4881	}
4882
4883	pvn_plist_init(pp, pl, plsz, off, io_len, rw);
4884	return (0);
4885}
4886
4887/*
4888 * Read ahead a cluster from the disk. Returns the length in bytes.
4889 */
4890static int
4891ufs_getpage_ra(struct vnode *vp, u_offset_t off, struct seg *seg, caddr_t addr)
4892{
4893	struct inode	*ip = VTOI(vp);
4894	page_t		*pp;
4895	u_offset_t	io_off = ip->i_nextrio;
4896	ufsvfs_t	*ufsvfsp;
4897	caddr_t		addr2 = addr + (io_off - off);
4898	struct buf	*bp;
4899	daddr_t		bn;
4900	size_t		io_len;
4901	int		err;
4902	int		contig;
4903	int		xlen;
4904	int		bsize = ip->i_fs->fs_bsize;
4905
4906	/*
4907	 * If the directio advisory is in effect on this file,
4908	 * then do not do buffered read ahead. Read ahead makes
4909	 * it more difficult on threads using directio as they
4910	 * will be forced to flush the pages from this vnode.
4911	 */
4912	if ((ufsvfsp = ip->i_ufsvfs) == NULL)
4913		return (0);
4914	if (ip->i_flag & IDIRECTIO || ufsvfsp->vfs_forcedirectio)
4915		return (0);
4916
4917	/*
4918	 * Is this test needed?
4919	 */
4920	if (addr2 >= seg->s_base + seg->s_size)
4921		return (0);
4922
4923	contig = 0;
4924	err = bmap_read(ip, io_off, &bn, &contig);
4925	/*
4926	 * If its a UFS_HOLE or a fallocated block, do not perform
4927	 * any read ahead's since there probably is nothing to read ahead
4928	 */
4929	if (err || bn == UFS_HOLE || ISFALLOCBLK(ip, bn))
4930		return (0);
4931
4932	/*
4933	 * Limit the transfer size to bsize if this is the 2nd block.
4934	 */
4935	if (io_off == (u_offset_t)bsize)
4936		contig = MIN(contig, bsize);
4937
4938	if ((pp = pvn_read_kluster(vp, io_off, seg, addr2, &io_off,
4939	    &io_len, io_off, contig, 1)) == NULL)
4940		return (0);
4941
4942	/*
4943	 * Zero part of page which we are not going to read from disk
4944	 */
4945	if ((xlen = (io_len & PAGEOFFSET)) > 0)
4946		pagezero(pp->p_prev, xlen, PAGESIZE - xlen);
4947
4948	ip->i_nextrio = (io_off + io_len + PAGESIZE - 1) & PAGEMASK;
4949
4950	bp = pageio_setup(pp, io_len, ip->i_devvp, B_READ | B_ASYNC);
4951	bp->b_edev = ip->i_dev;
4952	bp->b_dev = cmpdev(ip->i_dev);
4953	bp->b_blkno = bn;
4954	bp->b_un.b_addr = (caddr_t)0;
4955	bp->b_file = ip->i_vnode;
4956	bp->b_offset = off;
4957
4958	if (ufsvfsp->vfs_log) {
4959		lufs_read_strategy(ufsvfsp->vfs_log, bp);
4960	} else if (ufsvfsp->vfs_snapshot) {
4961		fssnap_strategy(&ufsvfsp->vfs_snapshot, bp);
4962	} else {
4963		ufsvfsp->vfs_iotstamp = ddi_get_lbolt();
4964		ub.ub_getras.value.ul++;
4965		(void) bdev_strategy(bp);
4966		lwp_stat_update(LWP_STAT_INBLK, 1);
4967	}
4968
4969	return (io_len);
4970}
4971
4972int	ufs_delay = 1;
4973/*
4974 * Flags are composed of {B_INVAL, B_FREE, B_DONTNEED, B_FORCE, B_ASYNC}
4975 *
4976 * LMXXX - the inode really ought to contain a pointer to one of these
4977 * async args.  Stuff gunk in there and just hand the whole mess off.
4978 * This would replace i_delaylen, i_delayoff.
4979 */
4980/*ARGSUSED*/
4981static int
4982ufs_putpage(struct vnode *vp, offset_t off, size_t len, int flags,
4983    struct cred *cr, caller_context_t *ct)
4984{
4985	struct inode *ip = VTOI(vp);
4986	int err = 0;
4987
4988	if (vp->v_count == 0) {
4989		return (ufs_fault(vp, "ufs_putpage: bad v_count == 0"));
4990	}
4991
4992	/*
4993	 * XXX - Why should this check be made here?
4994	 */
4995	if (vp->v_flag & VNOMAP) {
4996		err = ENOSYS;
4997		goto errout;
4998	}
4999
5000	if (ip->i_ufsvfs == NULL) {
5001		err = EIO;
5002		goto errout;
5003	}
5004
5005	if (flags & B_ASYNC) {
5006		if (ufs_delay && len &&
5007		    (flags & ~(B_ASYNC|B_DONTNEED|B_FREE)) == 0) {
5008			mutex_enter(&ip->i_tlock);
5009			/*
5010			 * If nobody stalled, start a new cluster.
5011			 */
5012			if (ip->i_delaylen == 0) {
5013				ip->i_delayoff = off;
5014				ip->i_delaylen = len;
5015				mutex_exit(&ip->i_tlock);
5016				goto errout;
5017			}
5018			/*
5019			 * If we have a full cluster or they are not contig,
5020			 * then push last cluster and start over.
5021			 */
5022			if (ip->i_delaylen >= CLUSTSZ(ip) ||
5023			    ip->i_delayoff + ip->i_delaylen != off) {
5024				u_offset_t doff;
5025				size_t dlen;
5026
5027				doff = ip->i_delayoff;
5028				dlen = ip->i_delaylen;
5029				ip->i_delayoff = off;
5030				ip->i_delaylen = len;
5031				mutex_exit(&ip->i_tlock);
5032				err = ufs_putpages(vp, doff, dlen,
5033				    flags, cr);
5034				/* LMXXX - flags are new val, not old */
5035				goto errout;
5036			}
5037			/*
5038			 * There is something there, it's not full, and
5039			 * it is contig.
5040			 */
5041			ip->i_delaylen += len;
5042			mutex_exit(&ip->i_tlock);
5043			goto errout;
5044		}
5045		/*
5046		 * Must have weird flags or we are not clustering.
5047		 */
5048	}
5049
5050	err = ufs_putpages(vp, off, len, flags, cr);
5051
5052errout:
5053	return (err);
5054}
5055
5056/*
5057 * If len == 0, do from off to EOF.
5058 *
5059 * The normal cases should be len == 0 & off == 0 (entire vp list),
5060 * len == MAXBSIZE (from segmap_release actions), and len == PAGESIZE
5061 * (from pageout).
5062 */
5063/*ARGSUSED*/
5064static int
5065ufs_putpages(struct vnode *vp, offset_t off, size_t len, int flags,
5066    struct cred *cr)
5067{
5068	u_offset_t io_off;
5069	u_offset_t eoff;
5070	struct inode *ip = VTOI(vp);
5071	page_t *pp;
5072	size_t io_len;
5073	int err = 0;
5074	int dolock;
5075
5076	if (vp->v_count == 0)
5077		return (ufs_fault(vp, "ufs_putpages: v_count == 0"));
5078	/*
5079	 * Acquire the readers/write inode lock before locking
5080	 * any pages in this inode.
5081	 * The inode lock is held during i/o.
5082	 */
5083	if (len == 0) {
5084		mutex_enter(&ip->i_tlock);
5085		ip->i_delayoff = ip->i_delaylen = 0;
5086		mutex_exit(&ip->i_tlock);
5087	}
5088	dolock = (rw_owner(&ip->i_contents) != curthread);
5089	if (dolock) {
5090		/*
5091		 * Must synchronize this thread and any possible thread
5092		 * operating in the window of vulnerability in wrip().
5093		 * It is dangerous to allow both a thread doing a putpage
5094		 * and a thread writing, so serialize them.  The exception
5095		 * is when the thread in wrip() does something which causes
5096		 * a putpage operation.  Then, the thread must be allowed
5097		 * to continue.  It may encounter a bmap_read problem in
5098		 * ufs_putapage, but that is handled in ufs_putapage.
5099		 * Allow async writers to proceed, we don't want to block
5100		 * the pageout daemon.
5101		 */
5102		if (ip->i_writer == curthread)
5103			rw_enter(&ip->i_contents, RW_READER);
5104		else {
5105			for (;;) {
5106				rw_enter(&ip->i_contents, RW_READER);
5107				mutex_enter(&ip->i_tlock);
5108				/*
5109				 * If there is no thread in the critical
5110				 * section of wrip(), then proceed.
5111				 * Otherwise, wait until there isn't one.
5112				 */
5113				if (ip->i_writer == NULL) {
5114					mutex_exit(&ip->i_tlock);
5115					break;
5116				}
5117				rw_exit(&ip->i_contents);
5118				/*
5119				 * Bounce async writers when we have a writer
5120				 * working on this file so we don't deadlock
5121				 * the pageout daemon.
5122				 */
5123				if (flags & B_ASYNC) {
5124					mutex_exit(&ip->i_tlock);
5125					return (0);
5126				}
5127				cv_wait(&ip->i_wrcv, &ip->i_tlock);
5128				mutex_exit(&ip->i_tlock);
5129			}
5130		}
5131	}
5132
5133	if (!vn_has_cached_data(vp)) {
5134		if (dolock)
5135			rw_exit(&ip->i_contents);
5136		return (0);
5137	}
5138
5139	if (len == 0) {
5140		/*
5141		 * Search the entire vp list for pages >= off.
5142		 */
5143		err = pvn_vplist_dirty(vp, (u_offset_t)off, ufs_putapage,
5144		    flags, cr);
5145	} else {
5146		/*
5147		 * Loop over all offsets in the range looking for
5148		 * pages to deal with.
5149		 */
5150		if ((eoff = blkroundup(ip->i_fs, ip->i_size)) != 0)
5151			eoff = MIN(off + len, eoff);
5152		else
5153			eoff = off + len;
5154
5155		for (io_off = off; io_off < eoff; io_off += io_len) {
5156			/*
5157			 * If we are not invalidating, synchronously
5158			 * freeing or writing pages, use the routine
5159			 * page_lookup_nowait() to prevent reclaiming
5160			 * them from the free list.
5161			 */
5162			if ((flags & B_INVAL) || ((flags & B_ASYNC) == 0)) {
5163				pp = page_lookup(vp, io_off,
5164				    (flags & (B_INVAL | B_FREE)) ?
5165				    SE_EXCL : SE_SHARED);
5166			} else {
5167				pp = page_lookup_nowait(vp, io_off,
5168				    (flags & B_FREE) ? SE_EXCL : SE_SHARED);
5169			}
5170
5171			if (pp == NULL || pvn_getdirty(pp, flags) == 0)
5172				io_len = PAGESIZE;
5173			else {
5174				u_offset_t *io_offp = &io_off;
5175
5176				err = ufs_putapage(vp, pp, io_offp, &io_len,
5177				    flags, cr);
5178				if (err != 0)
5179					break;
5180				/*
5181				 * "io_off" and "io_len" are returned as
5182				 * the range of pages we actually wrote.
5183				 * This allows us to skip ahead more quickly
5184				 * since several pages may've been dealt
5185				 * with by this iteration of the loop.
5186				 */
5187			}
5188		}
5189	}
5190	if (err == 0 && off == 0 && (len == 0 || len >= ip->i_size)) {
5191		/*
5192		 * We have just sync'ed back all the pages on
5193		 * the inode, turn off the IMODTIME flag.
5194		 */
5195		mutex_enter(&ip->i_tlock);
5196		ip->i_flag &= ~IMODTIME;
5197		mutex_exit(&ip->i_tlock);
5198	}
5199	if (dolock)
5200		rw_exit(&ip->i_contents);
5201	return (err);
5202}
5203
5204static int
5205ufs_iodone(buf_t *bp)
5206{
5207	struct inode *ip;
5208
5209	ASSERT((bp->b_pages->p_vnode != NULL) && !(bp->b_flags & B_READ));
5210
5211	bp->b_iodone = NULL;
5212
5213	ip = VTOI(bp->b_pages->p_vnode);
5214
5215	mutex_enter(&ip->i_tlock);
5216	if (ip->i_writes >= ufs_LW) {
5217		if ((ip->i_writes -= bp->b_bcount) <= ufs_LW)
5218			if (ufs_WRITES)
5219				cv_broadcast(&ip->i_wrcv); /* wake all up */
5220	} else {
5221		ip->i_writes -= bp->b_bcount;
5222	}
5223
5224	mutex_exit(&ip->i_tlock);
5225	iodone(bp);
5226	return (0);
5227}
5228
5229/*
5230 * Write out a single page, possibly klustering adjacent
5231 * dirty pages.  The inode lock must be held.
5232 *
5233 * LMXXX - bsize < pagesize not done.
5234 */
5235/*ARGSUSED*/
5236int
5237ufs_putapage(struct vnode *vp, page_t *pp, u_offset_t *offp, size_t *lenp,
5238    int flags, struct cred *cr)
5239{
5240	u_offset_t io_off;
5241	u_offset_t off;
5242	struct inode *ip = VTOI(vp);
5243	struct ufsvfs *ufsvfsp = ip->i_ufsvfs;
5244	struct fs *fs;
5245	struct buf *bp;
5246	size_t io_len;
5247	daddr_t bn;
5248	int err;
5249	int contig;
5250	int dotrans;
5251
5252	ASSERT(RW_LOCK_HELD(&ip->i_contents));
5253
5254	if (ufsvfsp == NULL) {
5255		err = EIO;
5256		goto out_trace;
5257	}
5258
5259	fs = ip->i_fs;
5260	ASSERT(fs->fs_ronly == 0);
5261
5262	/*
5263	 * If the modified time on the inode has not already been
5264	 * set elsewhere (e.g. for write/setattr) we set the time now.
5265	 * This gives us approximate modified times for mmap'ed files
5266	 * which are modified via stores in the user address space.
5267	 */
5268	if ((ip->i_flag & IMODTIME) == 0) {
5269		mutex_enter(&ip->i_tlock);
5270		ip->i_flag |= IUPD;
5271		ip->i_seq++;
5272		ITIMES_NOLOCK(ip);
5273		mutex_exit(&ip->i_tlock);
5274	}
5275
5276	/*
5277	 * Align the request to a block boundry (for old file systems),
5278	 * and go ask bmap() how contiguous things are for this file.
5279	 */
5280	off = pp->p_offset & (offset_t)fs->fs_bmask;	/* block align it */
5281	contig = 0;
5282	err = bmap_read(ip, off, &bn, &contig);
5283	if (err)
5284		goto out;
5285	if (bn == UFS_HOLE) {			/* putpage never allocates */
5286		/*
5287		 * logging device is in error mode; simply return EIO
5288		 */
5289		if (TRANS_ISERROR(ufsvfsp)) {
5290			err = EIO;
5291			goto out;
5292		}
5293		/*
5294		 * Oops, the thread in the window in wrip() did some
5295		 * sort of operation which caused a putpage in the bad
5296		 * range.  In this case, just return an error which will
5297		 * cause the software modified bit on the page to set
5298		 * and the page will get written out again later.
5299		 */
5300		if (ip->i_writer == curthread) {
5301			err = EIO;
5302			goto out;
5303		}
5304		/*
5305		 * If the pager is trying to push a page in the bad range
5306		 * just tell it to try again later when things are better.
5307		 */
5308		if (flags & B_ASYNC) {
5309			err = EAGAIN;
5310			goto out;
5311		}
5312		err = ufs_fault(ITOV(ip), "ufs_putapage: bn == UFS_HOLE");
5313		goto out;
5314	}
5315
5316	/*
5317	 * If it is an fallocate'd block, reverse the negativity since
5318	 * we are now writing to it
5319	 */
5320	if (ISFALLOCBLK(ip, bn)) {
5321		err = bmap_set_bn(vp, off, dbtofsb(fs, -bn));
5322		if (err)
5323			goto out;
5324
5325		bn = -bn;
5326	}
5327
5328	/*
5329	 * Take the length (of contiguous bytes) passed back from bmap()
5330	 * and _try_ and get a set of pages covering that extent.
5331	 */
5332	pp = pvn_write_kluster(vp, pp, &io_off, &io_len, off, contig, flags);
5333
5334	/*
5335	 * May have run out of memory and not clustered backwards.
5336	 * off		p_offset
5337	 * [  pp - 1  ][   pp   ]
5338	 * [	block		]
5339	 * We told bmap off, so we have to adjust the bn accordingly.
5340	 */
5341	if (io_off > off) {
5342		bn += btod(io_off - off);
5343		contig -= (io_off - off);
5344	}
5345
5346	/*
5347	 * bmap was carefull to tell us the right size so use that.
5348	 * There might be unallocated frags at the end.
5349	 * LMXXX - bzero the end of the page?  We must be writing after EOF.
5350	 */
5351	if (io_len > contig) {
5352		ASSERT(io_len - contig < fs->fs_bsize);
5353		io_len -= (io_len - contig);
5354	}
5355
5356	/*
5357	 * Handle the case where we are writing the last page after EOF.
5358	 *
5359	 * XXX - just a patch for i-mt3.
5360	 */
5361	if (io_len == 0) {
5362		ASSERT(pp->p_offset >=
5363		    (u_offset_t)(roundup(ip->i_size, PAGESIZE)));
5364		io_len = PAGESIZE;
5365	}
5366
5367	bp = pageio_setup(pp, io_len, ip->i_devvp, B_WRITE | flags);
5368
5369	ULOCKFS_SET_MOD(ITOUL(ip));
5370
5371	bp->b_edev = ip->i_dev;
5372	bp->b_dev = cmpdev(ip->i_dev);
5373	bp->b_blkno = bn;
5374	bp->b_un.b_addr = (caddr_t)0;
5375	bp->b_file = ip->i_vnode;
5376
5377	/*
5378	 * File contents of shadow or quota inodes are metadata, and updates
5379	 * to these need to be put into a logging transaction. All direct
5380	 * callers in UFS do that, but fsflush can come here _before_ the
5381	 * normal codepath. An example would be updating ACL information, for
5382	 * which the normal codepath would be:
5383	 *	ufs_si_store()
5384	 *	ufs_rdwri()
5385	 *	wrip()
5386	 *	segmap_release()
5387	 *	VOP_PUTPAGE()
5388	 * Here, fsflush can pick up the dirty page before segmap_release()
5389	 * forces it out. If that happens, there's no transaction.
5390	 * We therefore need to test whether a transaction exists, and if not
5391	 * create one - for fsflush.
5392	 */
5393	dotrans =
5394	    (((ip->i_mode & IFMT) == IFSHAD || ufsvfsp->vfs_qinod == ip) &&
5395	    ((curthread->t_flag & T_DONTBLOCK) == 0) &&
5396	    (TRANS_ISTRANS(ufsvfsp)));
5397
5398	if (dotrans) {
5399		curthread->t_flag |= T_DONTBLOCK;
5400		TRANS_BEGIN_ASYNC(ufsvfsp, TOP_PUTPAGE, TOP_PUTPAGE_SIZE(ip));
5401	}
5402	if (TRANS_ISTRANS(ufsvfsp)) {
5403		if ((ip->i_mode & IFMT) == IFSHAD) {
5404			TRANS_BUF(ufsvfsp, 0, io_len, bp, DT_SHAD);
5405		} else if (ufsvfsp->vfs_qinod == ip) {
5406			TRANS_DELTA(ufsvfsp, ldbtob(bn), bp->b_bcount, DT_QR,
5407			    0, 0);
5408		}
5409	}
5410	if (dotrans) {
5411		TRANS_END_ASYNC(ufsvfsp, TOP_PUTPAGE, TOP_PUTPAGE_SIZE(ip));
5412		curthread->t_flag &= ~T_DONTBLOCK;
5413	}
5414
5415	/* write throttle */
5416
5417	ASSERT(bp->b_iodone == NULL);
5418	bp->b_iodone = ufs_iodone;
5419	mutex_enter(&ip->i_tlock);
5420	ip->i_writes += bp->b_bcount;
5421	mutex_exit(&ip->i_tlock);
5422
5423	if (bp->b_flags & B_ASYNC) {
5424		if (ufsvfsp->vfs_log) {
5425			lufs_write_strategy(ufsvfsp->vfs_log, bp);
5426		} else if (ufsvfsp->vfs_snapshot) {
5427			fssnap_strategy(&ufsvfsp->vfs_snapshot, bp);
5428		} else {
5429			ufsvfsp->vfs_iotstamp = ddi_get_lbolt();
5430			ub.ub_putasyncs.value.ul++;
5431			(void) bdev_strategy(bp);
5432			lwp_stat_update(LWP_STAT_OUBLK, 1);
5433		}
5434	} else {
5435		if (ufsvfsp->vfs_log) {
5436			lufs_write_strategy(ufsvfsp->vfs_log, bp);
5437		} else if (ufsvfsp->vfs_snapshot) {
5438			fssnap_strategy(&ufsvfsp->vfs_snapshot, bp);
5439		} else {
5440			ufsvfsp->vfs_iotstamp = ddi_get_lbolt();
5441			ub.ub_putsyncs.value.ul++;
5442			(void) bdev_strategy(bp);
5443			lwp_stat_update(LWP_STAT_OUBLK, 1);
5444		}
5445		err = biowait(bp);
5446		pageio_done(bp);
5447		pvn_write_done(pp, ((err) ? B_ERROR : 0) | B_WRITE | flags);
5448	}
5449
5450	pp = NULL;
5451
5452out:
5453	if (err != 0 && pp != NULL)
5454		pvn_write_done(pp, B_ERROR | B_WRITE | flags);
5455
5456	if (offp)
5457		*offp = io_off;
5458	if (lenp)
5459		*lenp = io_len;
5460out_trace:
5461	return (err);
5462}
5463
5464uint64_t ufs_map_alock_retry_cnt;
5465uint64_t ufs_map_lockfs_retry_cnt;
5466
5467/* ARGSUSED */
5468static int
5469ufs_map(struct vnode *vp, offset_t off, struct as *as, caddr_t *addrp,
5470    size_t len, uchar_t prot, uchar_t maxprot, uint_t flags, struct cred *cr,
5471    caller_context_t *ct)
5472{
5473	struct segvn_crargs vn_a;
5474	struct ufsvfs *ufsvfsp = VTOI(vp)->i_ufsvfs;
5475	struct ulockfs *ulp;
5476	int error, sig;
5477	k_sigset_t smask;
5478	caddr_t hint = *addrp;
5479
5480	if (vp->v_flag & VNOMAP) {
5481		error = ENOSYS;
5482		goto out;
5483	}
5484
5485	if (off < (offset_t)0 || (offset_t)(off + len) < (offset_t)0) {
5486		error = ENXIO;
5487		goto out;
5488	}
5489
5490	if (vp->v_type != VREG) {
5491		error = ENODEV;
5492		goto out;
5493	}
5494
5495retry_map:
5496	*addrp = hint;
5497	/*
5498	 * If file is being locked, disallow mapping.
5499	 */
5500	if (vn_has_mandatory_locks(vp, VTOI(vp)->i_mode)) {
5501		error = EAGAIN;
5502		goto out;
5503	}
5504
5505	as_rangelock(as);
5506	/*
5507	 * Note that if we are retrying (because ufs_lockfs_trybegin failed in
5508	 * the previous attempt), some other thread could have grabbed
5509	 * the same VA range if MAP_FIXED is set. In that case, choose_addr
5510	 * would unmap the valid VA range, that is ok.
5511	 */
5512	error = choose_addr(as, addrp, len, off, ADDR_VACALIGN, flags);
5513	if (error != 0) {
5514		as_rangeunlock(as);
5515		goto out;
5516	}
5517
5518	/*
5519	 * a_lock has to be acquired before entering the lockfs protocol
5520	 * because that is the order in which pagefault works. Also we cannot
5521	 * block on a_lock here because this waiting writer will prevent
5522	 * further readers like ufs_read from progressing and could cause
5523	 * deadlock between ufs_read/ufs_map/pagefault when a quiesce is
5524	 * pending.
5525	 */
5526	while (!AS_LOCK_TRYENTER(as, RW_WRITER)) {
5527		ufs_map_alock_retry_cnt++;
5528		delay(RETRY_LOCK_DELAY);
5529	}
5530
5531	/*
5532	 * We can't hold as->a_lock and wait for lockfs to succeed because
5533	 * the proc tools might hang on a_lock, so call ufs_lockfs_trybegin()
5534	 * instead.
5535	 */
5536	if (error = ufs_lockfs_trybegin(ufsvfsp, &ulp, ULOCKFS_MAP_MASK)) {
5537		/*
5538		 * ufs_lockfs_trybegin() did not succeed. It is safer to give up
5539		 * as->a_lock and wait for ulp->ul_fs_lock status to change.
5540		 */
5541		ufs_map_lockfs_retry_cnt++;
5542		AS_LOCK_EXIT(as);
5543		as_rangeunlock(as);
5544		if (error == EIO)
5545			goto out;
5546
5547		mutex_enter(&ulp->ul_lock);
5548		while (ulp->ul_fs_lock & ULOCKFS_MAP_MASK) {
5549			if (ULOCKFS_IS_SLOCK(ulp) || ufsvfsp->vfs_nointr) {
5550				cv_wait(&ulp->ul_cv, &ulp->ul_lock);
5551			} else {
5552				sigintr(&smask, 1);
5553				sig = cv_wait_sig(&ulp->ul_cv, &ulp->ul_lock);
5554				sigunintr(&smask);
5555				if (((ulp->ul_fs_lock & ULOCKFS_MAP_MASK) &&
5556				    !sig) || ufsvfsp->vfs_dontblock) {
5557					mutex_exit(&ulp->ul_lock);
5558					return (EINTR);
5559				}
5560			}
5561		}
5562		mutex_exit(&ulp->ul_lock);
5563		goto retry_map;
5564	}
5565
5566	vn_a.vp = vp;
5567	vn_a.offset = (u_offset_t)off;
5568	vn_a.type = flags & MAP_TYPE;
5569	vn_a.prot = prot;
5570	vn_a.maxprot = maxprot;
5571	vn_a.cred = cr;
5572	vn_a.amp = NULL;
5573	vn_a.flags = flags & ~MAP_TYPE;
5574	vn_a.szc = 0;
5575	vn_a.lgrp_mem_policy_flags = 0;
5576
5577	error = as_map_locked(as, *addrp, len, segvn_create, &vn_a);
5578	if (ulp)
5579		ufs_lockfs_end(ulp);
5580	as_rangeunlock(as);
5581out:
5582	return (error);
5583}
5584
5585/* ARGSUSED */
5586static int
5587ufs_addmap(struct vnode *vp, offset_t off, struct as *as, caddr_t addr,
5588    size_t len, uchar_t  prot, uchar_t  maxprot, uint_t    flags,
5589    struct cred *cr, caller_context_t *ct)
5590{
5591	struct inode *ip = VTOI(vp);
5592
5593	if (vp->v_flag & VNOMAP) {
5594		return (ENOSYS);
5595	}
5596
5597	mutex_enter(&ip->i_tlock);
5598	ip->i_mapcnt += btopr(len);
5599	mutex_exit(&ip->i_tlock);
5600	return (0);
5601}
5602
5603/*ARGSUSED*/
5604static int
5605ufs_delmap(struct vnode *vp, offset_t off, struct as *as, caddr_t addr,
5606    size_t len, uint_t prot,  uint_t maxprot,  uint_t flags, struct cred *cr,
5607    caller_context_t *ct)
5608{
5609	struct inode *ip = VTOI(vp);
5610
5611	if (vp->v_flag & VNOMAP) {
5612		return (ENOSYS);
5613	}
5614
5615	mutex_enter(&ip->i_tlock);
5616	ip->i_mapcnt -= btopr(len);	/* Count released mappings */
5617	ASSERT(ip->i_mapcnt >= 0);
5618	mutex_exit(&ip->i_tlock);
5619	return (0);
5620}
5621/*
5622 * Return the answer requested to poll() for non-device files
5623 */
5624struct pollhead ufs_pollhd;
5625
5626/* ARGSUSED */
5627int
5628ufs_poll(vnode_t *vp, short ev, int any, short *revp, struct pollhead **phpp,
5629    caller_context_t *ct)
5630{
5631	struct ufsvfs	*ufsvfsp;
5632
5633	/*
5634	 * Regular files reject epollers (and edge-triggered pollers).
5635	 * See the comment in fs_poll() for a more detailed explanation.
5636	 */
5637	if (fs_reject_epoll() || (ev & POLLET) != 0) {
5638		return (EPERM);
5639	}
5640
5641	*revp = 0;
5642	ufsvfsp = VTOI(vp)->i_ufsvfs;
5643
5644	if (!ufsvfsp) {
5645		*revp = POLLHUP;
5646		goto out;
5647	}
5648
5649	if (ULOCKFS_IS_HLOCK(&ufsvfsp->vfs_ulockfs) ||
5650	    ULOCKFS_IS_ELOCK(&ufsvfsp->vfs_ulockfs)) {
5651		*revp |= POLLERR;
5652
5653	} else {
5654		if ((ev & POLLOUT) && !ufsvfsp->vfs_fs->fs_ronly &&
5655		    !ULOCKFS_IS_WLOCK(&ufsvfsp->vfs_ulockfs))
5656			*revp |= POLLOUT;
5657
5658		if ((ev & POLLWRBAND) && !ufsvfsp->vfs_fs->fs_ronly &&
5659		    !ULOCKFS_IS_WLOCK(&ufsvfsp->vfs_ulockfs))
5660			*revp |= POLLWRBAND;
5661
5662		if (ev & POLLIN)
5663			*revp |= POLLIN;
5664
5665		if (ev & POLLRDNORM)
5666			*revp |= POLLRDNORM;
5667
5668		if (ev & POLLRDBAND)
5669			*revp |= POLLRDBAND;
5670	}
5671
5672	if ((ev & POLLPRI) && (*revp & (POLLERR|POLLHUP)))
5673		*revp |= POLLPRI;
5674out:
5675	if (*revp == 0 && ! any) {
5676		*phpp = &ufs_pollhd;
5677	}
5678
5679	return (0);
5680}
5681
5682/* ARGSUSED */
5683static int
5684ufs_l_pathconf(struct vnode *vp, int cmd, ulong_t *valp, struct cred *cr,
5685    caller_context_t *ct)
5686{
5687	struct ufsvfs	*ufsvfsp = VTOI(vp)->i_ufsvfs;
5688	struct ulockfs	*ulp = NULL;
5689	struct inode	*sip = NULL;
5690	int		error;
5691	struct inode	*ip = VTOI(vp);
5692	int		issync;
5693
5694	error = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_PATHCONF_MASK);
5695	if (error)
5696		return (error);
5697
5698	switch (cmd) {
5699		/*
5700		 * Have to handle _PC_NAME_MAX here, because the normal way
5701		 * [fs_pathconf() -> VOP_STATVFS() -> ufs_statvfs()]
5702		 * results in a lock ordering reversal between
5703		 * ufs_lockfs_{begin,end}() and
5704		 * ufs_thread_{suspend,continue}().
5705		 *
5706		 * Keep in sync with ufs_statvfs().
5707		 */
5708	case _PC_NAME_MAX:
5709		*valp = MAXNAMLEN;
5710		break;
5711
5712	case _PC_FILESIZEBITS:
5713		if (ufsvfsp->vfs_lfflags & UFS_LARGEFILES)
5714			*valp = UFS_FILESIZE_BITS;
5715		else
5716			*valp = 32;
5717		break;
5718
5719	case _PC_XATTR_EXISTS:
5720		if (vp->v_vfsp->vfs_flag & VFS_XATTR) {
5721
5722			error =
5723			    ufs_xattr_getattrdir(vp, &sip, LOOKUP_XATTR, cr);
5724			if (error ==  0 && sip != NULL) {
5725				/* Start transaction */
5726				if (ulp) {
5727					TRANS_BEGIN_CSYNC(ufsvfsp, issync,
5728					    TOP_RMDIR, TOP_RMDIR_SIZE);
5729				}
5730				/*
5731				 * Is directory empty
5732				 */
5733				rw_enter(&sip->i_rwlock, RW_WRITER);
5734				rw_enter(&sip->i_contents, RW_WRITER);
5735				if (ufs_xattrdirempty(sip,
5736				    sip->i_number, CRED())) {
5737					rw_enter(&ip->i_contents, RW_WRITER);
5738					ufs_unhook_shadow(ip, sip);
5739					rw_exit(&ip->i_contents);
5740
5741					*valp = 0;
5742
5743				} else
5744					*valp = 1;
5745				rw_exit(&sip->i_contents);
5746				rw_exit(&sip->i_rwlock);
5747				if (ulp) {
5748					TRANS_END_CSYNC(ufsvfsp, error, issync,
5749					    TOP_RMDIR, TOP_RMDIR_SIZE);
5750				}
5751				VN_RELE(ITOV(sip));
5752			} else if (error == ENOENT) {
5753				*valp = 0;
5754				error = 0;
5755			}
5756		} else {
5757			error = fs_pathconf(vp, cmd, valp, cr, ct);
5758		}
5759		break;
5760
5761	case _PC_ACL_ENABLED:
5762		*valp = _ACL_ACLENT_ENABLED;
5763		break;
5764
5765	case _PC_MIN_HOLE_SIZE:
5766		*valp = (ulong_t)ip->i_fs->fs_bsize;
5767		break;
5768
5769	case _PC_SATTR_ENABLED:
5770	case _PC_SATTR_EXISTS:
5771		*valp = vfs_has_feature(vp->v_vfsp, VFSFT_SYSATTR_VIEWS) &&
5772		    (vp->v_type == VREG || vp->v_type == VDIR);
5773		break;
5774
5775	case _PC_TIMESTAMP_RESOLUTION:
5776		/*
5777		 * UFS keeps only microsecond timestamp resolution.
5778		 * This is historical and will probably never change.
5779		 */
5780		*valp = 1000L;
5781		break;
5782
5783	default:
5784		error = fs_pathconf(vp, cmd, valp, cr, ct);
5785		break;
5786	}
5787
5788	if (ulp != NULL) {
5789		ufs_lockfs_end(ulp);
5790	}
5791	return (error);
5792}
5793
5794int ufs_pageio_writes, ufs_pageio_reads;
5795
5796/*ARGSUSED*/
5797static int
5798ufs_pageio(struct vnode *vp, page_t *pp, u_offset_t io_off, size_t io_len,
5799    int flags, struct cred *cr, caller_context_t *ct)
5800{
5801	struct inode *ip = VTOI(vp);
5802	struct ufsvfs *ufsvfsp;
5803	page_t *npp = NULL, *opp = NULL, *cpp = pp;
5804	struct buf *bp;
5805	daddr_t bn;
5806	size_t done_len = 0, cur_len = 0;
5807	int err = 0;
5808	int contig = 0;
5809	int dolock;
5810	int vmpss = 0;
5811	struct ulockfs *ulp;
5812
5813	if ((flags & B_READ) && pp != NULL && pp->p_vnode == vp &&
5814	    vp->v_mpssdata != NULL) {
5815		vmpss = 1;
5816	}
5817
5818	dolock = (rw_owner(&ip->i_contents) != curthread);
5819	/*
5820	 * We need a better check.  Ideally, we would use another
5821	 * vnodeops so that hlocked and forcibly unmounted file
5822	 * systems would return EIO where appropriate and w/o the
5823	 * need for these checks.
5824	 */
5825	if ((ufsvfsp = ip->i_ufsvfs) == NULL)
5826		return (EIO);
5827
5828	/*
5829	 * For vmpss (pp can be NULL) case respect the quiesce protocol.
5830	 * ul_lock must be taken before locking pages so we can't use it here
5831	 * if pp is non NULL because segvn already locked pages
5832	 * SE_EXCL. Instead we rely on the fact that a forced umount or
5833	 * applying a filesystem lock via ufs_fiolfs() will block in the
5834	 * implicit call to ufs_flush() until we unlock the pages after the
5835	 * return to segvn. Other ufs_quiesce() callers keep ufs_quiesce_pend
5836	 * above 0 until they are done. We have to be careful not to increment
5837	 * ul_vnops_cnt here after forceful unmount hlocks the file system.
5838	 *
5839	 * If pp is NULL use ul_lock to make sure we don't increment
5840	 * ul_vnops_cnt after forceful unmount hlocks the file system.
5841	 */
5842	if (vmpss || pp == NULL) {
5843		ulp = &ufsvfsp->vfs_ulockfs;
5844		if (pp == NULL)
5845			mutex_enter(&ulp->ul_lock);
5846		if (ulp->ul_fs_lock & ULOCKFS_GETREAD_MASK) {
5847			if (pp == NULL) {
5848				mutex_exit(&ulp->ul_lock);
5849			}
5850			return (vmpss ? EIO : EINVAL);
5851		}
5852		atomic_inc_ulong(&ulp->ul_vnops_cnt);
5853		if (pp == NULL)
5854			mutex_exit(&ulp->ul_lock);
5855		if (ufs_quiesce_pend) {
5856			if (!atomic_dec_ulong_nv(&ulp->ul_vnops_cnt))
5857				cv_broadcast(&ulp->ul_cv);
5858			return (vmpss ? EIO : EINVAL);
5859		}
5860	}
5861
5862	if (dolock) {
5863		/*
5864		 * segvn may call VOP_PAGEIO() instead of VOP_GETPAGE() to
5865		 * handle a fault against a segment that maps vnode pages with
5866		 * large mappings.  Segvn creates pages and holds them locked
5867		 * SE_EXCL during VOP_PAGEIO() call. In this case we have to
5868		 * use rw_tryenter() to avoid a potential deadlock since in
5869		 * lock order i_contents needs to be taken first.
5870		 * Segvn will retry via VOP_GETPAGE() if VOP_PAGEIO() fails.
5871		 */
5872		if (!vmpss) {
5873			rw_enter(&ip->i_contents, RW_READER);
5874		} else if (!rw_tryenter(&ip->i_contents, RW_READER)) {
5875			if (!atomic_dec_ulong_nv(&ulp->ul_vnops_cnt))
5876				cv_broadcast(&ulp->ul_cv);
5877			return (EDEADLK);
5878		}
5879	}
5880
5881	/*
5882	 * Return an error to segvn because the pagefault request is beyond
5883	 * PAGESIZE rounded EOF.
5884	 */
5885	if (vmpss && btopr(io_off + io_len) > btopr(ip->i_size)) {
5886		if (dolock)
5887			rw_exit(&ip->i_contents);
5888		if (!atomic_dec_ulong_nv(&ulp->ul_vnops_cnt))
5889			cv_broadcast(&ulp->ul_cv);
5890		return (EFAULT);
5891	}
5892
5893	if (pp == NULL) {
5894		if (bmap_has_holes(ip)) {
5895			err = ENOSYS;
5896		} else {
5897			err = EINVAL;
5898		}
5899		if (dolock)
5900			rw_exit(&ip->i_contents);
5901		if (!atomic_dec_ulong_nv(&ulp->ul_vnops_cnt))
5902			cv_broadcast(&ulp->ul_cv);
5903		return (err);
5904	}
5905
5906	/*
5907	 * Break the io request into chunks, one for each contiguous
5908	 * stretch of disk blocks in the target file.
5909	 */
5910	while (done_len < io_len) {
5911		ASSERT(cpp);
5912		contig = 0;
5913		if (err = bmap_read(ip, (u_offset_t)(io_off + done_len),
5914		    &bn, &contig))
5915			break;
5916
5917		if (bn == UFS_HOLE) {	/* No holey swapfiles */
5918			if (vmpss) {
5919				err = EFAULT;
5920				break;
5921			}
5922			err = ufs_fault(ITOV(ip), "ufs_pageio: bn == UFS_HOLE");
5923			break;
5924		}
5925
5926		cur_len = MIN(io_len - done_len, contig);
5927		/*
5928		 * Zero out a page beyond EOF, when the last block of
5929		 * a file is a UFS fragment so that ufs_pageio() can be used
5930		 * instead of ufs_getpage() to handle faults against
5931		 * segvn segments that use large pages.
5932		 */
5933		page_list_break(&cpp, &npp, btopr(cur_len));
5934		if ((flags & B_READ) && (cur_len & PAGEOFFSET)) {
5935			size_t xlen = cur_len & PAGEOFFSET;
5936			pagezero(cpp->p_prev, xlen, PAGESIZE - xlen);
5937		}
5938
5939		bp = pageio_setup(cpp, cur_len, ip->i_devvp, flags);
5940		ASSERT(bp != NULL);
5941
5942		bp->b_edev = ip->i_dev;
5943		bp->b_dev = cmpdev(ip->i_dev);
5944		bp->b_blkno = bn;
5945		bp->b_un.b_addr = (caddr_t)0;
5946		bp->b_file = ip->i_vnode;
5947
5948		ufsvfsp->vfs_iotstamp = ddi_get_lbolt();
5949		ub.ub_pageios.value.ul++;
5950		if (ufsvfsp->vfs_snapshot)
5951			fssnap_strategy(&(ufsvfsp->vfs_snapshot), bp);
5952		else
5953			(void) bdev_strategy(bp);
5954
5955		if (flags & B_READ)
5956			ufs_pageio_reads++;
5957		else
5958			ufs_pageio_writes++;
5959		if (flags & B_READ)
5960			lwp_stat_update(LWP_STAT_INBLK, 1);
5961		else
5962			lwp_stat_update(LWP_STAT_OUBLK, 1);
5963		/*
5964		 * If the request is not B_ASYNC, wait for i/o to complete
5965		 * and re-assemble the page list to return to the caller.
5966		 * If it is B_ASYNC we leave the page list in pieces and
5967		 * cleanup() will dispose of them.
5968		 */
5969		if ((flags & B_ASYNC) == 0) {
5970			err = biowait(bp);
5971			pageio_done(bp);
5972			if (err)
5973				break;
5974			page_list_concat(&opp, &cpp);
5975		}
5976		cpp = npp;
5977		npp = NULL;
5978		if (flags & B_READ)
5979			cur_len = P2ROUNDUP_TYPED(cur_len, PAGESIZE, size_t);
5980		done_len += cur_len;
5981	}
5982	ASSERT(err || (cpp == NULL && npp == NULL && done_len == io_len));
5983	if (err) {
5984		if (flags & B_ASYNC) {
5985			/* Cleanup unprocessed parts of list */
5986			page_list_concat(&cpp, &npp);
5987			if (flags & B_READ)
5988				pvn_read_done(cpp, B_ERROR);
5989			else
5990				pvn_write_done(cpp, B_ERROR);
5991		} else {
5992			/* Re-assemble list and let caller clean up */
5993			page_list_concat(&opp, &cpp);
5994			page_list_concat(&opp, &npp);
5995		}
5996	}
5997
5998	if (vmpss && !(ip->i_flag & IACC) && !ULOCKFS_IS_NOIACC(ulp) &&
5999	    ufsvfsp->vfs_fs->fs_ronly == 0 && !ufsvfsp->vfs_noatime) {
6000		mutex_enter(&ip->i_tlock);
6001		ip->i_flag |= IACC;
6002		ITIMES_NOLOCK(ip);
6003		mutex_exit(&ip->i_tlock);
6004	}
6005
6006	if (dolock)
6007		rw_exit(&ip->i_contents);
6008	if (vmpss && !atomic_dec_ulong_nv(&ulp->ul_vnops_cnt))
6009		cv_broadcast(&ulp->ul_cv);
6010	return (err);
6011}
6012
6013/*
6014 * Called when the kernel is in a frozen state to dump data
6015 * directly to the device. It uses a private dump data structure,
6016 * set up by dump_ctl, to locate the correct disk block to which to dump.
6017 */
6018/*ARGSUSED*/
6019static int
6020ufs_dump(vnode_t *vp, caddr_t addr, offset_t ldbn, offset_t dblks,
6021    caller_context_t *ct)
6022{
6023	u_offset_t	file_size;
6024	struct inode    *ip = VTOI(vp);
6025	struct fs	*fs = ip->i_fs;
6026	daddr_t		dbn, lfsbn;
6027	int		disk_blks = fs->fs_bsize >> DEV_BSHIFT;
6028	int		error = 0;
6029	int		ndbs, nfsbs;
6030
6031	/*
6032	 * forced unmount case
6033	 */
6034	if (ip->i_ufsvfs == NULL)
6035		return (EIO);
6036	/*
6037	 * Validate the inode that it has not been modified since
6038	 * the dump structure is allocated.
6039	 */
6040	mutex_enter(&ip->i_tlock);
6041	if ((dump_info == NULL) ||
6042	    (dump_info->ip != ip) ||
6043	    (dump_info->time.tv_sec != ip->i_mtime.tv_sec) ||
6044	    (dump_info->time.tv_usec != ip->i_mtime.tv_usec)) {
6045		mutex_exit(&ip->i_tlock);
6046		return (-1);
6047	}
6048	mutex_exit(&ip->i_tlock);
6049
6050	/*
6051	 * See that the file has room for this write
6052	 */
6053	UFS_GET_ISIZE(&file_size, ip);
6054
6055	if (ldbtob(ldbn + dblks) > file_size)
6056		return (ENOSPC);
6057
6058	/*
6059	 * Find the physical disk block numbers from the dump
6060	 * private data structure directly and write out the data
6061	 * in contiguous block lumps
6062	 */
6063	while (dblks > 0 && !error) {
6064		lfsbn = (daddr_t)lblkno(fs, ldbtob(ldbn));
6065		dbn = fsbtodb(fs, dump_info->dblk[lfsbn]) + ldbn % disk_blks;
6066		nfsbs = 1;
6067		ndbs = disk_blks - ldbn % disk_blks;
6068		while (ndbs < dblks && fsbtodb(fs, dump_info->dblk[lfsbn +
6069		    nfsbs]) == dbn + ndbs) {
6070			nfsbs++;
6071			ndbs += disk_blks;
6072		}
6073		if (ndbs > dblks)
6074			ndbs = dblks;
6075		error = bdev_dump(ip->i_dev, addr, dbn, ndbs);
6076		addr += ldbtob((offset_t)ndbs);
6077		dblks -= ndbs;
6078		ldbn += ndbs;
6079	}
6080	return (error);
6081
6082}
6083
6084/*
6085 * Prepare the file system before and after the dump operation.
6086 *
6087 * action = DUMP_ALLOC:
6088 * Preparation before dump, allocate dump private data structure
6089 * to hold all the direct and indirect block info for dump.
6090 *
6091 * action = DUMP_FREE:
6092 * Clean up after dump, deallocate the dump private data structure.
6093 *
6094 * action = DUMP_SCAN:
6095 * Scan dump_info for *blkp DEV_BSIZE blocks of contig fs space;
6096 * if found, the starting file-relative DEV_BSIZE lbn is written
6097 * to *bklp; that lbn is intended for use with VOP_DUMP()
6098 */
6099/*ARGSUSED*/
6100static int
6101ufs_dumpctl(vnode_t *vp, int action, offset_t *blkp, caller_context_t *ct)
6102{
6103	struct inode	*ip = VTOI(vp);
6104	ufsvfs_t	*ufsvfsp = ip->i_ufsvfs;
6105	struct fs	*fs;
6106	daddr32_t	*dblk, *storeblk;
6107	daddr32_t	*nextblk, *endblk;
6108	struct buf	*bp;
6109	int		i, entry, entries;
6110	int		n, ncontig;
6111
6112	/*
6113	 * check for forced unmount
6114	 */
6115	if (ufsvfsp == NULL)
6116		return (EIO);
6117
6118	if (action == DUMP_ALLOC) {
6119		/*
6120		 * alloc and record dump_info
6121		 */
6122		if (dump_info != NULL)
6123			return (EINVAL);
6124
6125		ASSERT(vp->v_type == VREG);
6126		fs = ufsvfsp->vfs_fs;
6127
6128		rw_enter(&ip->i_contents, RW_READER);
6129
6130		if (bmap_has_holes(ip)) {
6131			rw_exit(&ip->i_contents);
6132			return (EFAULT);
6133		}
6134
6135		/*
6136		 * calculate and allocate space needed according to i_size
6137		 */
6138		entries = (int)lblkno(fs, blkroundup(fs, ip->i_size));
6139		dump_info = kmem_alloc(sizeof (struct dump) +
6140		    (entries - 1) * sizeof (daddr32_t), KM_NOSLEEP);
6141		if (dump_info == NULL) {
6142			rw_exit(&ip->i_contents);
6143			return (ENOMEM);
6144		}
6145
6146		/* Start saving the info */
6147		dump_info->fsbs = entries;
6148		dump_info->ip = ip;
6149		storeblk = &dump_info->dblk[0];
6150
6151		/* Direct Blocks */
6152		for (entry = 0; entry < NDADDR && entry < entries; entry++)
6153			*storeblk++ = ip->i_db[entry];
6154
6155		/* Indirect Blocks */
6156		for (i = 0; i < NIADDR; i++) {
6157			int error = 0;
6158
6159			bp = UFS_BREAD(ufsvfsp,
6160			    ip->i_dev, fsbtodb(fs, ip->i_ib[i]), fs->fs_bsize);
6161			if (bp->b_flags & B_ERROR)
6162				error = EIO;
6163			else {
6164				dblk = bp->b_un.b_daddr;
6165				if ((storeblk = save_dblks(ip, ufsvfsp,
6166				    storeblk, dblk, i, entries)) == NULL)
6167					error = EIO;
6168			}
6169
6170			brelse(bp);
6171
6172			if (error != 0) {
6173				kmem_free(dump_info, sizeof (struct dump) +
6174				    (entries - 1) * sizeof (daddr32_t));
6175				rw_exit(&ip->i_contents);
6176				dump_info = NULL;
6177				return (error);
6178			}
6179		}
6180		/* and time stamp the information */
6181		mutex_enter(&ip->i_tlock);
6182		dump_info->time = ip->i_mtime;
6183		mutex_exit(&ip->i_tlock);
6184
6185		rw_exit(&ip->i_contents);
6186	} else if (action == DUMP_FREE) {
6187		/*
6188		 * free dump_info
6189		 */
6190		if (dump_info == NULL)
6191			return (EINVAL);
6192		entries = dump_info->fsbs - 1;
6193		kmem_free(dump_info, sizeof (struct dump) +
6194		    entries * sizeof (daddr32_t));
6195		dump_info = NULL;
6196	} else if (action == DUMP_SCAN) {
6197		/*
6198		 * scan dump_info
6199		 */
6200		if (dump_info == NULL)
6201			return (EINVAL);
6202
6203		dblk = dump_info->dblk;
6204		nextblk = dblk + 1;
6205		endblk = dblk + dump_info->fsbs - 1;
6206		fs = ufsvfsp->vfs_fs;
6207		ncontig = *blkp >> (fs->fs_bshift - DEV_BSHIFT);
6208
6209		/*
6210		 * scan dblk[] entries; contig fs space is found when:
6211		 * ((current blkno + frags per block) == next blkno)
6212		 */
6213		n = 0;
6214		while (n < ncontig && dblk < endblk) {
6215			if ((*dblk + fs->fs_frag) == *nextblk)
6216				n++;
6217			else
6218				n = 0;
6219			dblk++;
6220			nextblk++;
6221		}
6222
6223		/*
6224		 * index is where size bytes of contig space begins;
6225		 * conversion from index to the file's DEV_BSIZE lbn
6226		 * is equivalent to:  (index * fs_bsize) / DEV_BSIZE
6227		 */
6228		if (n == ncontig) {
6229			i = (dblk - dump_info->dblk) - ncontig;
6230			*blkp = i << (fs->fs_bshift - DEV_BSHIFT);
6231		} else
6232			return (EFAULT);
6233	}
6234	return (0);
6235}
6236
6237/*
6238 * Recursive helper function for ufs_dumpctl().  It follows the indirect file
6239 * system  blocks until it reaches the the disk block addresses, which are
6240 * then stored into the given buffer, storeblk.
6241 */
6242static daddr32_t *
6243save_dblks(struct inode *ip, struct ufsvfs *ufsvfsp,  daddr32_t *storeblk,
6244    daddr32_t *dblk, int level, int entries)
6245{
6246	struct fs	*fs = ufsvfsp->vfs_fs;
6247	struct buf	*bp;
6248	int		i;
6249
6250	if (level == 0) {
6251		for (i = 0; i < NINDIR(fs); i++) {
6252			if (storeblk - dump_info->dblk >= entries)
6253				break;
6254			*storeblk++ = dblk[i];
6255		}
6256		return (storeblk);
6257	}
6258	for (i = 0; i < NINDIR(fs); i++) {
6259		if (storeblk - dump_info->dblk >= entries)
6260			break;
6261		bp = UFS_BREAD(ufsvfsp,
6262		    ip->i_dev, fsbtodb(fs, dblk[i]), fs->fs_bsize);
6263		if (bp->b_flags & B_ERROR) {
6264			brelse(bp);
6265			return (NULL);
6266		}
6267		storeblk = save_dblks(ip, ufsvfsp, storeblk, bp->b_un.b_daddr,
6268		    level - 1, entries);
6269		brelse(bp);
6270
6271		if (storeblk == NULL)
6272			return (NULL);
6273	}
6274	return (storeblk);
6275}
6276
6277/* ARGSUSED */
6278static int
6279ufs_getsecattr(struct vnode *vp, vsecattr_t *vsap, int flag,
6280    struct cred *cr, caller_context_t *ct)
6281{
6282	struct inode	*ip = VTOI(vp);
6283	struct ulockfs	*ulp;
6284	struct ufsvfs	*ufsvfsp = ip->i_ufsvfs;
6285	ulong_t		vsa_mask = vsap->vsa_mask;
6286	int		err = EINVAL;
6287
6288	vsa_mask &= (VSA_ACL | VSA_ACLCNT | VSA_DFACL | VSA_DFACLCNT);
6289
6290	/*
6291	 * Only grab locks if needed - they're not needed to check vsa_mask
6292	 * or if the mask contains no acl flags.
6293	 */
6294	if (vsa_mask != 0) {
6295		if (err = ufs_lockfs_begin(ufsvfsp, &ulp,
6296		    ULOCKFS_GETATTR_MASK))
6297			return (err);
6298
6299		rw_enter(&ip->i_contents, RW_READER);
6300		err = ufs_acl_get(ip, vsap, flag, cr);
6301		rw_exit(&ip->i_contents);
6302
6303		if (ulp)
6304			ufs_lockfs_end(ulp);
6305	}
6306	return (err);
6307}
6308
6309/* ARGSUSED */
6310static int
6311ufs_setsecattr(struct vnode *vp, vsecattr_t *vsap, int flag, struct cred *cr,
6312    caller_context_t *ct)
6313{
6314	struct inode	*ip = VTOI(vp);
6315	struct ulockfs	*ulp = NULL;
6316	struct ufsvfs	*ufsvfsp = VTOI(vp)->i_ufsvfs;
6317	ulong_t		vsa_mask = vsap->vsa_mask;
6318	int		err;
6319	int		haverwlock = 1;
6320	int		trans_size;
6321	int		donetrans = 0;
6322	int		retry = 1;
6323
6324	ASSERT(RW_LOCK_HELD(&ip->i_rwlock));
6325
6326	/* Abort now if the request is either empty or invalid. */
6327	vsa_mask &= (VSA_ACL | VSA_ACLCNT | VSA_DFACL | VSA_DFACLCNT);
6328	if ((vsa_mask == 0) ||
6329	    ((vsap->vsa_aclentp == NULL) &&
6330	    (vsap->vsa_dfaclentp == NULL))) {
6331		err = EINVAL;
6332		goto out;
6333	}
6334
6335	/*
6336	 * Following convention, if this is a directory then we acquire the
6337	 * inode's i_rwlock after starting a UFS logging transaction;
6338	 * otherwise, we acquire it beforehand. Since we were called (and
6339	 * must therefore return) with the lock held, we will have to drop it,
6340	 * and later reacquire it, if operating on a directory.
6341	 */
6342	if (vp->v_type == VDIR) {
6343		rw_exit(&ip->i_rwlock);
6344		haverwlock = 0;
6345	} else {
6346		/* Upgrade the lock if required. */
6347		if (!rw_write_held(&ip->i_rwlock)) {
6348			rw_exit(&ip->i_rwlock);
6349			rw_enter(&ip->i_rwlock, RW_WRITER);
6350		}
6351	}
6352
6353again:
6354	ASSERT(!(vp->v_type == VDIR && haverwlock));
6355	if (err = ufs_lockfs_begin(ufsvfsp, &ulp, ULOCKFS_SETATTR_MASK)) {
6356		ulp = NULL;
6357		retry = 0;
6358		goto out;
6359	}
6360
6361	/*
6362	 * Check that the file system supports this operation. Note that
6363	 * ufs_lockfs_begin() will have checked that the file system had
6364	 * not been forcibly unmounted.
6365	 */
6366	if (ufsvfsp->vfs_fs->fs_ronly) {
6367		err = EROFS;
6368		goto out;
6369	}
6370	if (ufsvfsp->vfs_nosetsec) {
6371		err = ENOSYS;
6372		goto out;
6373	}
6374
6375	if (ulp) {
6376		TRANS_BEGIN_ASYNC(ufsvfsp, TOP_SETSECATTR,
6377		    trans_size = TOP_SETSECATTR_SIZE(VTOI(vp)));
6378		donetrans =