1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright (c) 1988, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2020 Joyent, Inc.
25 * Copyright 2016 Nexenta Systems, Inc.  All rights reserved.
26 * Copyright (c) 2011, 2017 by Delphix. All rights reserved.
27 */
28
29/*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T	*/
30/*	  All Rights Reserved	*/
31
32/*
33 * University Copyright- Copyright (c) 1982, 1986, 1988
34 * The Regents of the University of California
35 * All Rights Reserved
36 *
37 * University Acknowledgment- Portions of this document are derived from
38 * software developed by the University of California, Berkeley, and its
39 * contributors.
40 */
41
42#include <sys/types.h>
43#include <sys/param.h>
44#include <sys/t_lock.h>
45#include <sys/errno.h>
46#include <sys/cred.h>
47#include <sys/user.h>
48#include <sys/uio.h>
49#include <sys/file.h>
50#include <sys/pathname.h>
51#include <sys/vfs.h>
52#include <sys/vfs_opreg.h>
53#include <sys/vnode.h>
54#include <sys/filio.h>
55#include <sys/rwstlock.h>
56#include <sys/fem.h>
57#include <sys/stat.h>
58#include <sys/mode.h>
59#include <sys/conf.h>
60#include <sys/sysmacros.h>
61#include <sys/cmn_err.h>
62#include <sys/systm.h>
63#include <sys/kmem.h>
64#include <sys/debug.h>
65#include <c2/audit.h>
66#include <sys/acl.h>
67#include <sys/nbmlock.h>
68#include <sys/fcntl.h>
69#include <fs/fs_subr.h>
70#include <sys/taskq.h>
71#include <fs/fs_reparse.h>
72#include <sys/time.h>
73#include <sys/sdt.h>
74
75/* Determine if this vnode is a file that is read-only */
76#define	ISROFILE(vp)	\
77	((vp)->v_type != VCHR && (vp)->v_type != VBLK && \
78	    (vp)->v_type != VFIFO && vn_is_readonly(vp))
79
80/* Tunable via /etc/system; used only by admin/install */
81int nfs_global_client_only;
82
83/*
84 * Array of vopstats_t for per-FS-type vopstats.  This array has the same
85 * number of entries as and parallel to the vfssw table.  (Arguably, it could
86 * be part of the vfssw table.)  Once it's initialized, it's accessed using
87 * the same fstype index that is used to index into the vfssw table.
88 */
89vopstats_t **vopstats_fstype;
90
91/* vopstats initialization template used for fast initialization via bcopy() */
92static vopstats_t *vs_templatep;
93
94/* Kmem cache handle for vsk_anchor_t allocations */
95kmem_cache_t *vsk_anchor_cache;
96
97/* file events cleanup routine */
98extern void free_fopdata(vnode_t *);
99
100/*
101 * Root of AVL tree for the kstats associated with vopstats.  Lock protects
102 * updates to vsktat_tree.
103 */
104avl_tree_t	vskstat_tree;
105kmutex_t	vskstat_tree_lock;
106
107/* Global variable which enables/disables the vopstats collection */
108int vopstats_enabled = 1;
109
110/* Global used for empty/invalid v_path */
111char *vn_vpath_empty = "";
112
113/*
114 * forward declarations for internal vnode specific data (vsd)
115 */
116static void *vsd_realloc(void *, size_t, size_t);
117
118/*
119 * forward declarations for reparse point functions
120 */
121static int fs_reparse_mark(char *target, vattr_t *vap, xvattr_t *xvattr);
122
123/*
124 * VSD -- VNODE SPECIFIC DATA
125 * The v_data pointer is typically used by a file system to store a
126 * pointer to the file system's private node (e.g. ufs inode, nfs rnode).
127 * However, there are times when additional project private data needs
128 * to be stored separately from the data (node) pointed to by v_data.
129 * This additional data could be stored by the file system itself or
130 * by a completely different kernel entity.  VSD provides a way for
131 * callers to obtain a key and store a pointer to private data associated
132 * with a vnode.
133 *
134 * Callers are responsible for protecting the vsd by holding v_vsd_lock
135 * for calls to vsd_set() and vsd_get().
136 */
137
138/*
139 * vsd_lock protects:
140 *   vsd_nkeys - creation and deletion of vsd keys
141 *   vsd_list - insertion and deletion of vsd_node in the vsd_list
142 *   vsd_destructor - adding and removing destructors to the list
143 */
144static kmutex_t		vsd_lock;
145static uint_t		vsd_nkeys;	 /* size of destructor array */
146/* list of vsd_node's */
147static list_t *vsd_list = NULL;
148/* per-key destructor funcs */
149static void		(**vsd_destructor)(void *);
150
151/*
152 * The following is the common set of actions needed to update the
153 * vopstats structure from a vnode op.  Both VOPSTATS_UPDATE() and
154 * VOPSTATS_UPDATE_IO() do almost the same thing, except for the
155 * recording of the bytes transferred.  Since the code is similar
156 * but small, it is nearly a duplicate.  Consequently any changes
157 * to one may need to be reflected in the other.
158 * Rundown of the variables:
159 * vp - Pointer to the vnode
160 * counter - Partial name structure member to update in vopstats for counts
161 * bytecounter - Partial name structure member to update in vopstats for bytes
162 * bytesval - Value to update in vopstats for bytes
163 * fstype - Index into vsanchor_fstype[], same as index into vfssw[]
164 * vsp - Pointer to vopstats structure (either in vfs or vsanchor_fstype[i])
165 */
166
167#define	VOPSTATS_UPDATE(vp, counter) {					\
168	vfs_t *vfsp = (vp)->v_vfsp;					\
169	if (vfsp && vfsp->vfs_implp &&					\
170	    (vfsp->vfs_flag & VFS_STATS) && (vp)->v_type != VBAD) {	\
171		vopstats_t *vsp = &vfsp->vfs_vopstats;			\
172		uint64_t *stataddr = &(vsp->n##counter.value.ui64);	\
173		extern void __dtrace_probe___fsinfo_##counter(vnode_t *, \
174		    size_t, uint64_t *);				\
175		__dtrace_probe___fsinfo_##counter(vp, 0, stataddr);	\
176		(*stataddr)++;						\
177		if ((vsp = vfsp->vfs_fstypevsp) != NULL) {		\
178			vsp->n##counter.value.ui64++;			\
179		}							\
180	}								\
181}
182
183#define	VOPSTATS_UPDATE_IO(vp, counter, bytecounter, bytesval) {	\
184	vfs_t *vfsp = (vp)->v_vfsp;					\
185	if (vfsp && vfsp->vfs_implp &&					\
186	    (vfsp->vfs_flag & VFS_STATS) && (vp)->v_type != VBAD) {	\
187		vopstats_t *vsp = &vfsp->vfs_vopstats;			\
188		uint64_t *stataddr = &(vsp->n##counter.value.ui64);	\
189		extern void __dtrace_probe___fsinfo_##counter(vnode_t *, \
190		    size_t, uint64_t *);				\
191		__dtrace_probe___fsinfo_##counter(vp, bytesval, stataddr); \
192		(*stataddr)++;						\
193		vsp->bytecounter.value.ui64 += bytesval;		\
194		if ((vsp = vfsp->vfs_fstypevsp) != NULL) {		\
195			vsp->n##counter.value.ui64++;			\
196			vsp->bytecounter.value.ui64 += bytesval;	\
197		}							\
198	}								\
199}
200
201/*
202 * If the filesystem does not support XIDs map credential
203 * If the vfsp is NULL, perhaps we should also map?
204 */
205#define	VOPXID_MAP_CR(vp, cr)	{					\
206	vfs_t *vfsp = (vp)->v_vfsp;					\
207	if (vfsp != NULL && (vfsp->vfs_flag & VFS_XID) == 0)		\
208		cr = crgetmapped(cr);					\
209	}
210
211/*
212 * Convert stat(2) formats to vnode types and vice versa.  (Knows about
213 * numerical order of S_IFMT and vnode types.)
214 */
215enum vtype iftovt_tab[] = {
216	VNON, VFIFO, VCHR, VNON, VDIR, VNON, VBLK, VNON,
217	VREG, VNON, VLNK, VNON, VSOCK, VNON, VNON, VNON
218};
219
220ushort_t vttoif_tab[] = {
221	0, S_IFREG, S_IFDIR, S_IFBLK, S_IFCHR, S_IFLNK, S_IFIFO,
222	S_IFDOOR, 0, S_IFSOCK, S_IFPORT, 0
223};
224
225/*
226 * The system vnode cache.
227 */
228
229kmem_cache_t *vn_cache;
230
231
232/*
233 * Vnode operations vector.
234 */
235
236static const fs_operation_trans_def_t vn_ops_table[] = {
237	VOPNAME_OPEN, offsetof(struct vnodeops, vop_open),
238	    fs_nosys, fs_nosys,
239
240	VOPNAME_CLOSE, offsetof(struct vnodeops, vop_close),
241	    fs_nosys, fs_nosys,
242
243	VOPNAME_READ, offsetof(struct vnodeops, vop_read),
244	    fs_nosys, fs_nosys,
245
246	VOPNAME_WRITE, offsetof(struct vnodeops, vop_write),
247	    fs_nosys, fs_nosys,
248
249	VOPNAME_IOCTL, offsetof(struct vnodeops, vop_ioctl),
250	    fs_nosys, fs_nosys,
251
252	VOPNAME_SETFL, offsetof(struct vnodeops, vop_setfl),
253	    fs_setfl, fs_nosys,
254
255	VOPNAME_GETATTR, offsetof(struct vnodeops, vop_getattr),
256	    fs_nosys, fs_nosys,
257
258	VOPNAME_SETATTR, offsetof(struct vnodeops, vop_setattr),
259	    fs_nosys, fs_nosys,
260
261	VOPNAME_ACCESS, offsetof(struct vnodeops, vop_access),
262	    fs_nosys, fs_nosys,
263
264	VOPNAME_LOOKUP, offsetof(struct vnodeops, vop_lookup),
265	    fs_nosys, fs_nosys,
266
267	VOPNAME_CREATE, offsetof(struct vnodeops, vop_create),
268	    fs_nosys, fs_nosys,
269
270	VOPNAME_REMOVE, offsetof(struct vnodeops, vop_remove),
271	    fs_nosys, fs_nosys,
272
273	VOPNAME_LINK, offsetof(struct vnodeops, vop_link),
274	    fs_nosys, fs_nosys,
275
276	VOPNAME_RENAME, offsetof(struct vnodeops, vop_rename),
277	    fs_nosys, fs_nosys,
278
279	VOPNAME_MKDIR, offsetof(struct vnodeops, vop_mkdir),
280	    fs_nosys, fs_nosys,
281
282	VOPNAME_RMDIR, offsetof(struct vnodeops, vop_rmdir),
283	    fs_nosys, fs_nosys,
284
285	VOPNAME_READDIR, offsetof(struct vnodeops, vop_readdir),
286	    fs_nosys, fs_nosys,
287
288	VOPNAME_SYMLINK, offsetof(struct vnodeops, vop_symlink),
289	    fs_nosys, fs_nosys,
290
291	VOPNAME_READLINK, offsetof(struct vnodeops, vop_readlink),
292	    fs_nosys, fs_nosys,
293
294	VOPNAME_FSYNC, offsetof(struct vnodeops, vop_fsync),
295	    fs_nosys, fs_nosys,
296
297	VOPNAME_INACTIVE, offsetof(struct vnodeops, vop_inactive),
298	    fs_nosys, fs_nosys,
299
300	VOPNAME_FID, offsetof(struct vnodeops, vop_fid),
301	    fs_nosys, fs_nosys,
302
303	VOPNAME_RWLOCK, offsetof(struct vnodeops, vop_rwlock),
304	    fs_rwlock, fs_rwlock,
305
306	VOPNAME_RWUNLOCK, offsetof(struct vnodeops, vop_rwunlock),
307	    (fs_generic_func_p)(uintptr_t)fs_rwunlock,
308	    (fs_generic_func_p)(uintptr_t)fs_rwunlock,	/* no errors allowed */
309
310	VOPNAME_SEEK, offsetof(struct vnodeops, vop_seek),
311	    fs_nosys, fs_nosys,
312
313	VOPNAME_CMP, offsetof(struct vnodeops, vop_cmp),
314	    fs_cmp, fs_cmp,		/* no errors allowed */
315
316	VOPNAME_FRLOCK, offsetof(struct vnodeops, vop_frlock),
317	    fs_frlock, fs_nosys,
318
319	VOPNAME_SPACE, offsetof(struct vnodeops, vop_space),
320	    fs_nosys, fs_nosys,
321
322	VOPNAME_REALVP, offsetof(struct vnodeops, vop_realvp),
323	    fs_nosys, fs_nosys,
324
325	VOPNAME_GETPAGE, offsetof(struct vnodeops, vop_getpage),
326	    fs_nosys, fs_nosys,
327
328	VOPNAME_PUTPAGE, offsetof(struct vnodeops, vop_putpage),
329	    fs_nosys, fs_nosys,
330
331	VOPNAME_MAP, offsetof(struct vnodeops, vop_map),
332	    (fs_generic_func_p) fs_nosys_map,
333	    (fs_generic_func_p) fs_nosys_map,
334
335	VOPNAME_ADDMAP, offsetof(struct vnodeops, vop_addmap),
336	    (fs_generic_func_p) fs_nosys_addmap,
337	    (fs_generic_func_p) fs_nosys_addmap,
338
339	VOPNAME_DELMAP, offsetof(struct vnodeops, vop_delmap),
340	    fs_nosys, fs_nosys,
341
342	VOPNAME_POLL, offsetof(struct vnodeops, vop_poll),
343	    (fs_generic_func_p) fs_poll, (fs_generic_func_p) fs_nosys_poll,
344
345	VOPNAME_DUMP, offsetof(struct vnodeops, vop_dump),
346	    fs_nosys, fs_nosys,
347
348	VOPNAME_PATHCONF, offsetof(struct vnodeops, vop_pathconf),
349	    fs_pathconf, fs_nosys,
350
351	VOPNAME_PAGEIO, offsetof(struct vnodeops, vop_pageio),
352	    fs_nosys, fs_nosys,
353
354	VOPNAME_DUMPCTL, offsetof(struct vnodeops, vop_dumpctl),
355	    fs_nosys, fs_nosys,
356
357	VOPNAME_DISPOSE, offsetof(struct vnodeops, vop_dispose),
358	    (fs_generic_func_p)(uintptr_t)fs_dispose,
359	    (fs_generic_func_p)(uintptr_t)fs_nodispose,
360
361	VOPNAME_SETSECATTR, offsetof(struct vnodeops, vop_setsecattr),
362	    fs_nosys, fs_nosys,
363
364	VOPNAME_GETSECATTR, offsetof(struct vnodeops, vop_getsecattr),
365	    fs_fab_acl, fs_nosys,
366
367	VOPNAME_SHRLOCK, offsetof(struct vnodeops, vop_shrlock),
368	    fs_shrlock, fs_nosys,
369
370	VOPNAME_VNEVENT, offsetof(struct vnodeops, vop_vnevent),
371	    (fs_generic_func_p) fs_vnevent_nosupport,
372	    (fs_generic_func_p) fs_vnevent_nosupport,
373
374	VOPNAME_REQZCBUF, offsetof(struct vnodeops, vop_reqzcbuf),
375	    fs_nosys, fs_nosys,
376
377	VOPNAME_RETZCBUF, offsetof(struct vnodeops, vop_retzcbuf),
378	    fs_nosys, fs_nosys,
379
380	NULL, 0, NULL, NULL
381};
382
383/* Extensible attribute (xva) routines. */
384
385/*
386 * Zero out the structure, set the size of the requested/returned bitmaps,
387 * set AT_XVATTR in the embedded vattr_t's va_mask, and set up the pointer
388 * to the returned attributes array.
389 */
390void
391xva_init(xvattr_t *xvap)
392{
393	bzero(xvap, sizeof (xvattr_t));
394	xvap->xva_mapsize = XVA_MAPSIZE;
395	xvap->xva_magic = XVA_MAGIC;
396	xvap->xva_vattr.va_mask = AT_XVATTR;
397	xvap->xva_rtnattrmapp = &(xvap->xva_rtnattrmap)[0];
398}
399
400/*
401 * If AT_XVATTR is set, returns a pointer to the embedded xoptattr_t
402 * structure.  Otherwise, returns NULL.
403 */
404xoptattr_t *
405xva_getxoptattr(xvattr_t *xvap)
406{
407	xoptattr_t *xoap = NULL;
408	if (xvap->xva_vattr.va_mask & AT_XVATTR)
409		xoap = &xvap->xva_xoptattrs;
410	return (xoap);
411}
412
413/*
414 * Used by the AVL routines to compare two vsk_anchor_t structures in the tree.
415 * We use the f_fsid reported by VFS_STATVFS() since we use that for the
416 * kstat name.
417 */
418static int
419vska_compar(const void *n1, const void *n2)
420{
421	int ret;
422	ulong_t p1 = ((vsk_anchor_t *)n1)->vsk_fsid;
423	ulong_t p2 = ((vsk_anchor_t *)n2)->vsk_fsid;
424
425	if (p1 < p2) {
426		ret = -1;
427	} else if (p1 > p2) {
428		ret = 1;
429	} else {
430		ret = 0;
431	}
432
433	return (ret);
434}
435
436/*
437 * Used to create a single template which will be bcopy()ed to a newly
438 * allocated vsanchor_combo_t structure in new_vsanchor(), below.
439 */
440static vopstats_t *
441create_vopstats_template()
442{
443	vopstats_t		*vsp;
444
445	vsp = kmem_alloc(sizeof (vopstats_t), KM_SLEEP);
446	bzero(vsp, sizeof (*vsp));	/* Start fresh */
447
448	/* VOP_OPEN */
449	kstat_named_init(&vsp->nopen, "nopen", KSTAT_DATA_UINT64);
450	/* VOP_CLOSE */
451	kstat_named_init(&vsp->nclose, "nclose", KSTAT_DATA_UINT64);
452	/* VOP_READ I/O */
453	kstat_named_init(&vsp->nread, "nread", KSTAT_DATA_UINT64);
454	kstat_named_init(&vsp->read_bytes, "read_bytes", KSTAT_DATA_UINT64);
455	/* VOP_WRITE I/O */
456	kstat_named_init(&vsp->nwrite, "nwrite", KSTAT_DATA_UINT64);
457	kstat_named_init(&vsp->write_bytes, "write_bytes", KSTAT_DATA_UINT64);
458	/* VOP_IOCTL */
459	kstat_named_init(&vsp->nioctl, "nioctl", KSTAT_DATA_UINT64);
460	/* VOP_SETFL */
461	kstat_named_init(&vsp->nsetfl, "nsetfl", KSTAT_DATA_UINT64);
462	/* VOP_GETATTR */
463	kstat_named_init(&vsp->ngetattr, "ngetattr", KSTAT_DATA_UINT64);
464	/* VOP_SETATTR */
465	kstat_named_init(&vsp->nsetattr, "nsetattr", KSTAT_DATA_UINT64);
466	/* VOP_ACCESS */
467	kstat_named_init(&vsp->naccess, "naccess", KSTAT_DATA_UINT64);
468	/* VOP_LOOKUP */
469	kstat_named_init(&vsp->nlookup, "nlookup", KSTAT_DATA_UINT64);
470	/* VOP_CREATE */
471	kstat_named_init(&vsp->ncreate, "ncreate", KSTAT_DATA_UINT64);
472	/* VOP_REMOVE */
473	kstat_named_init(&vsp->nremove, "nremove", KSTAT_DATA_UINT64);
474	/* VOP_LINK */
475	kstat_named_init(&vsp->nlink, "nlink", KSTAT_DATA_UINT64);
476	/* VOP_RENAME */
477	kstat_named_init(&vsp->nrename, "nrename", KSTAT_DATA_UINT64);
478	/* VOP_MKDIR */
479	kstat_named_init(&vsp->nmkdir, "nmkdir", KSTAT_DATA_UINT64);
480	/* VOP_RMDIR */
481	kstat_named_init(&vsp->nrmdir, "nrmdir", KSTAT_DATA_UINT64);
482	/* VOP_READDIR I/O */
483	kstat_named_init(&vsp->nreaddir, "nreaddir", KSTAT_DATA_UINT64);
484	kstat_named_init(&vsp->readdir_bytes, "readdir_bytes",
485	    KSTAT_DATA_UINT64);
486	/* VOP_SYMLINK */
487	kstat_named_init(&vsp->nsymlink, "nsymlink", KSTAT_DATA_UINT64);
488	/* VOP_READLINK */
489	kstat_named_init(&vsp->nreadlink, "nreadlink", KSTAT_DATA_UINT64);
490	/* VOP_FSYNC */
491	kstat_named_init(&vsp->nfsync, "nfsync", KSTAT_DATA_UINT64);
492	/* VOP_INACTIVE */
493	kstat_named_init(&vsp->ninactive, "ninactive", KSTAT_DATA_UINT64);
494	/* VOP_FID */
495	kstat_named_init(&vsp->nfid, "nfid", KSTAT_DATA_UINT64);
496	/* VOP_RWLOCK */
497	kstat_named_init(&vsp->nrwlock, "nrwlock", KSTAT_DATA_UINT64);
498	/* VOP_RWUNLOCK */
499	kstat_named_init(&vsp->nrwunlock, "nrwunlock", KSTAT_DATA_UINT64);
500	/* VOP_SEEK */
501	kstat_named_init(&vsp->nseek, "nseek", KSTAT_DATA_UINT64);
502	/* VOP_CMP */
503	kstat_named_init(&vsp->ncmp, "ncmp", KSTAT_DATA_UINT64);
504	/* VOP_FRLOCK */
505	kstat_named_init(&vsp->nfrlock, "nfrlock", KSTAT_DATA_UINT64);
506	/* VOP_SPACE */
507	kstat_named_init(&vsp->nspace, "nspace", KSTAT_DATA_UINT64);
508	/* VOP_REALVP */
509	kstat_named_init(&vsp->nrealvp, "nrealvp", KSTAT_DATA_UINT64);
510	/* VOP_GETPAGE */
511	kstat_named_init(&vsp->ngetpage, "ngetpage", KSTAT_DATA_UINT64);
512	/* VOP_PUTPAGE */
513	kstat_named_init(&vsp->nputpage, "nputpage", KSTAT_DATA_UINT64);
514	/* VOP_MAP */
515	kstat_named_init(&vsp->nmap, "nmap", KSTAT_DATA_UINT64);
516	/* VOP_ADDMAP */
517	kstat_named_init(&vsp->naddmap, "naddmap", KSTAT_DATA_UINT64);
518	/* VOP_DELMAP */
519	kstat_named_init(&vsp->ndelmap, "ndelmap", KSTAT_DATA_UINT64);
520	/* VOP_POLL */
521	kstat_named_init(&vsp->npoll, "npoll", KSTAT_DATA_UINT64);
522	/* VOP_DUMP */
523	kstat_named_init(&vsp->ndump, "ndump", KSTAT_DATA_UINT64);
524	/* VOP_PATHCONF */
525	kstat_named_init(&vsp->npathconf, "npathconf", KSTAT_DATA_UINT64);
526	/* VOP_PAGEIO */
527	kstat_named_init(&vsp->npageio, "npageio", KSTAT_DATA_UINT64);
528	/* VOP_DUMPCTL */
529	kstat_named_init(&vsp->ndumpctl, "ndumpctl", KSTAT_DATA_UINT64);
530	/* VOP_DISPOSE */
531	kstat_named_init(&vsp->ndispose, "ndispose", KSTAT_DATA_UINT64);
532	/* VOP_SETSECATTR */
533	kstat_named_init(&vsp->nsetsecattr, "nsetsecattr", KSTAT_DATA_UINT64);
534	/* VOP_GETSECATTR */
535	kstat_named_init(&vsp->ngetsecattr, "ngetsecattr", KSTAT_DATA_UINT64);
536	/* VOP_SHRLOCK */
537	kstat_named_init(&vsp->nshrlock, "nshrlock", KSTAT_DATA_UINT64);
538	/* VOP_VNEVENT */
539	kstat_named_init(&vsp->nvnevent, "nvnevent", KSTAT_DATA_UINT64);
540	/* VOP_REQZCBUF */
541	kstat_named_init(&vsp->nreqzcbuf, "nreqzcbuf", KSTAT_DATA_UINT64);
542	/* VOP_RETZCBUF */
543	kstat_named_init(&vsp->nretzcbuf, "nretzcbuf", KSTAT_DATA_UINT64);
544
545	return (vsp);
546}
547
548/*
549 * Creates a kstat structure associated with a vopstats structure.
550 */
551kstat_t *
552new_vskstat(char *ksname, vopstats_t *vsp)
553{
554	kstat_t		*ksp;
555
556	if (!vopstats_enabled) {
557		return (NULL);
558	}
559
560	ksp = kstat_create("unix", 0, ksname, "misc", KSTAT_TYPE_NAMED,
561	    sizeof (vopstats_t)/sizeof (kstat_named_t),
562	    KSTAT_FLAG_VIRTUAL|KSTAT_FLAG_WRITABLE);
563	if (ksp) {
564		ksp->ks_data = vsp;
565		kstat_install(ksp);
566	}
567
568	return (ksp);
569}
570
571/*
572 * Called from vfsinit() to initialize the support mechanisms for vopstats
573 */
574void
575vopstats_startup()
576{
577	if (!vopstats_enabled)
578		return;
579
580	/*
581	 * Creates the AVL tree which holds per-vfs vopstat anchors.  This
582	 * is necessary since we need to check if a kstat exists before we
583	 * attempt to create it.  Also, initialize its lock.
584	 */
585	avl_create(&vskstat_tree, vska_compar, sizeof (vsk_anchor_t),
586	    offsetof(vsk_anchor_t, vsk_node));
587	mutex_init(&vskstat_tree_lock, NULL, MUTEX_DEFAULT, NULL);
588
589	vsk_anchor_cache = kmem_cache_create("vsk_anchor_cache",
590	    sizeof (vsk_anchor_t), sizeof (uintptr_t), NULL, NULL, NULL,
591	    NULL, NULL, 0);
592
593	/*
594	 * Set up the array of pointers for the vopstats-by-FS-type.
595	 * The entries will be allocated/initialized as each file system
596	 * goes through modload/mod_installfs.
597	 */
598	vopstats_fstype = (vopstats_t **)kmem_zalloc(
599	    (sizeof (vopstats_t *) * nfstype), KM_SLEEP);
600
601	/* Set up the global vopstats initialization template */
602	vs_templatep = create_vopstats_template();
603}
604
605/*
606 * We need to have the all of the counters zeroed.
607 * The initialization of the vopstats_t includes on the order of
608 * 50 calls to kstat_named_init().  Rather that do that on every call,
609 * we do it once in a template (vs_templatep) then bcopy it over.
610 */
611void
612initialize_vopstats(vopstats_t *vsp)
613{
614	if (vsp == NULL)
615		return;
616
617	bcopy(vs_templatep, vsp, sizeof (vopstats_t));
618}
619
620/*
621 * If possible, determine which vopstats by fstype to use and
622 * return a pointer to the caller.
623 */
624vopstats_t *
625get_fstype_vopstats(vfs_t *vfsp, struct vfssw *vswp)
626{
627	int		fstype = 0;	/* Index into vfssw[] */
628	vopstats_t	*vsp = NULL;
629
630	if (vfsp == NULL || (vfsp->vfs_flag & VFS_STATS) == 0 ||
631	    !vopstats_enabled)
632		return (NULL);
633	/*
634	 * Set up the fstype.  We go to so much trouble because all versions
635	 * of NFS use the same fstype in their vfs even though they have
636	 * distinct entries in the vfssw[] table.
637	 * NOTE: A special vfs (e.g., EIO_vfs) may not have an entry.
638	 */
639	if (vswp) {
640		fstype = vswp - vfssw;	/* Gets us the index */
641	} else {
642		fstype = vfsp->vfs_fstype;
643	}
644
645	/*
646	 * Point to the per-fstype vopstats. The only valid values are
647	 * non-zero positive values less than the number of vfssw[] table
648	 * entries.
649	 */
650	if (fstype > 0 && fstype < nfstype) {
651		vsp = vopstats_fstype[fstype];
652	}
653
654	return (vsp);
655}
656
657/*
658 * Generate a kstat name, create the kstat structure, and allocate a
659 * vsk_anchor_t to hold it together.  Return the pointer to the vsk_anchor_t
660 * to the caller.  This must only be called from a mount.
661 */
662vsk_anchor_t *
663get_vskstat_anchor(vfs_t *vfsp)
664{
665	char		kstatstr[KSTAT_STRLEN]; /* kstat name for vopstats */
666	statvfs64_t	statvfsbuf;		/* Needed to find f_fsid */
667	vsk_anchor_t	*vskp = NULL;		/* vfs <--> kstat anchor */
668	kstat_t		*ksp;			/* Ptr to new kstat */
669	avl_index_t	where;			/* Location in the AVL tree */
670
671	if (vfsp == NULL || vfsp->vfs_implp == NULL ||
672	    (vfsp->vfs_flag & VFS_STATS) == 0 || !vopstats_enabled)
673		return (NULL);
674
675	/* Need to get the fsid to build a kstat name */
676	if (VFS_STATVFS(vfsp, &statvfsbuf) == 0) {
677		/* Create a name for our kstats based on fsid */
678		(void) snprintf(kstatstr, KSTAT_STRLEN, "%s%lx",
679		    VOPSTATS_STR, statvfsbuf.f_fsid);
680
681		/* Allocate and initialize the vsk_anchor_t */
682		vskp = kmem_cache_alloc(vsk_anchor_cache, KM_SLEEP);
683		bzero(vskp, sizeof (*vskp));
684		vskp->vsk_fsid = statvfsbuf.f_fsid;
685
686		mutex_enter(&vskstat_tree_lock);
687		if (avl_find(&vskstat_tree, vskp, &where) == NULL) {
688			avl_insert(&vskstat_tree, vskp, where);
689			mutex_exit(&vskstat_tree_lock);
690
691			/*
692			 * Now that we've got the anchor in the AVL
693			 * tree, we can create the kstat.
694			 */
695			ksp = new_vskstat(kstatstr, &vfsp->vfs_vopstats);
696			if (ksp) {
697				vskp->vsk_ksp = ksp;
698			}
699		} else {
700			/* Oops, found one! Release memory and lock. */
701			mutex_exit(&vskstat_tree_lock);
702			kmem_cache_free(vsk_anchor_cache, vskp);
703			vskp = NULL;
704		}
705	}
706	return (vskp);
707}
708
709/*
710 * We're in the process of tearing down the vfs and need to cleanup
711 * the data structures associated with the vopstats. Must only be called
712 * from dounmount().
713 */
714void
715teardown_vopstats(vfs_t *vfsp)
716{
717	vsk_anchor_t	*vskap;
718	avl_index_t	where;
719
720	if (vfsp == NULL || vfsp->vfs_implp == NULL ||
721	    (vfsp->vfs_flag & VFS_STATS) == 0 || !vopstats_enabled)
722		return;
723
724	/* This is a safe check since VFS_STATS must be set (see above) */
725	if ((vskap = vfsp->vfs_vskap) == NULL)
726		return;
727
728	/* Whack the pointer right away */
729	vfsp->vfs_vskap = NULL;
730
731	/* Lock the tree, remove the node, and delete the kstat */
732	mutex_enter(&vskstat_tree_lock);
733	if (avl_find(&vskstat_tree, vskap, &where)) {
734		avl_remove(&vskstat_tree, vskap);
735	}
736
737	if (vskap->vsk_ksp) {
738		kstat_delete(vskap->vsk_ksp);
739	}
740	mutex_exit(&vskstat_tree_lock);
741
742	kmem_cache_free(vsk_anchor_cache, vskap);
743}
744
745/*
746 * Read or write a vnode.  Called from kernel code.
747 */
748int
749vn_rdwr(
750	enum uio_rw rw,
751	struct vnode *vp,
752	caddr_t base,
753	ssize_t len,
754	offset_t offset,
755	enum uio_seg seg,
756	int ioflag,
757	rlim64_t ulimit,	/* meaningful only if rw is UIO_WRITE */
758	cred_t *cr,
759	ssize_t *residp)
760{
761	struct uio uio;
762	struct iovec iov;
763	int error;
764	int in_crit = 0;
765
766	if (rw == UIO_WRITE && ISROFILE(vp))
767		return (EROFS);
768
769	if (len < 0)
770		return (EIO);
771
772	VOPXID_MAP_CR(vp, cr);
773
774	iov.iov_base = base;
775	iov.iov_len = len;
776	uio.uio_iov = &iov;
777	uio.uio_iovcnt = 1;
778	uio.uio_loffset = offset;
779	uio.uio_segflg = (short)seg;
780	uio.uio_resid = len;
781	uio.uio_llimit = ulimit;
782
783	/*
784	 * We have to enter the critical region before calling VOP_RWLOCK
785	 * to avoid a deadlock with ufs.
786	 */
787	if (nbl_need_check(vp)) {
788		int svmand;
789
790		nbl_start_crit(vp, RW_READER);
791		in_crit = 1;
792		error = nbl_svmand(vp, cr, &svmand);
793		if (error != 0)
794			goto done;
795		if (nbl_conflict(vp, rw == UIO_WRITE ? NBL_WRITE : NBL_READ,
796		    uio.uio_offset, uio.uio_resid, svmand, NULL)) {
797			error = EACCES;
798			goto done;
799		}
800	}
801
802	(void) VOP_RWLOCK(vp,
803	    rw == UIO_WRITE ? V_WRITELOCK_TRUE : V_WRITELOCK_FALSE, NULL);
804	if (rw == UIO_WRITE) {
805		uio.uio_fmode = FWRITE;
806		uio.uio_extflg = UIO_COPY_DEFAULT;
807		error = VOP_WRITE(vp, &uio, ioflag, cr, NULL);
808	} else {
809		uio.uio_fmode = FREAD;
810		uio.uio_extflg = UIO_COPY_CACHED;
811		error = VOP_READ(vp, &uio, ioflag, cr, NULL);
812	}
813	VOP_RWUNLOCK(vp,
814	    rw == UIO_WRITE ? V_WRITELOCK_TRUE : V_WRITELOCK_FALSE, NULL);
815	if (residp)
816		*residp = uio.uio_resid;
817	else if (uio.uio_resid)
818		error = EIO;
819
820done:
821	if (in_crit)
822		nbl_end_crit(vp);
823	return (error);
824}
825
826/*
827 * Release a vnode.  Call VOP_INACTIVE on last reference or
828 * decrement reference count.
829 *
830 * To avoid race conditions, the v_count is left at 1 for
831 * the call to VOP_INACTIVE. This prevents another thread
832 * from reclaiming and releasing the vnode *before* the
833 * VOP_INACTIVE routine has a chance to destroy the vnode.
834 * We can't have more than 1 thread calling VOP_INACTIVE
835 * on a vnode.
836 */
837void
838vn_rele(vnode_t *vp)
839{
840	VERIFY(vp->v_count > 0);
841	mutex_enter(&vp->v_lock);
842	if (vp->v_count == 1) {
843		mutex_exit(&vp->v_lock);
844		VOP_INACTIVE(vp, CRED(), NULL);
845		return;
846	}
847	VN_RELE_LOCKED(vp);
848	mutex_exit(&vp->v_lock);
849}
850
851/*
852 * Release a vnode referenced by the DNLC. Multiple DNLC references are treated
853 * as a single reference, so v_count is not decremented until the last DNLC hold
854 * is released. This makes it possible to distinguish vnodes that are referenced
855 * only by the DNLC.
856 */
857void
858vn_rele_dnlc(vnode_t *vp)
859{
860	VERIFY((vp->v_count > 0) && (vp->v_count_dnlc > 0));
861	mutex_enter(&vp->v_lock);
862	if (--vp->v_count_dnlc == 0) {
863		if (vp->v_count == 1) {
864			mutex_exit(&vp->v_lock);
865			VOP_INACTIVE(vp, CRED(), NULL);
866			return;
867		}
868		VN_RELE_LOCKED(vp);
869	}
870	mutex_exit(&vp->v_lock);
871}
872
873/*
874 * Like vn_rele() except that it clears v_stream under v_lock.
875 * This is used by sockfs when it dismantles the association between
876 * the sockfs node and the vnode in the underlying file system.
877 * v_lock has to be held to prevent a thread coming through the lookupname
878 * path from accessing a stream head that is going away.
879 */
880void
881vn_rele_stream(vnode_t *vp)
882{
883	VERIFY(vp->v_count > 0);
884	mutex_enter(&vp->v_lock);
885	vp->v_stream = NULL;
886	if (vp->v_count == 1) {
887		mutex_exit(&vp->v_lock);
888		VOP_INACTIVE(vp, CRED(), NULL);
889		return;
890	}
891	VN_RELE_LOCKED(vp);
892	mutex_exit(&vp->v_lock);
893}
894
895static void
896vn_rele_inactive(vnode_t *vp)
897{
898	VOP_INACTIVE(vp, CRED(), NULL);
899}
900
901/*
902 * Like vn_rele() except if we are going to call VOP_INACTIVE() then do it
903 * asynchronously using a taskq. This can avoid deadlocks caused by re-entering
904 * the file system as a result of releasing the vnode. Note, file systems
905 * already have to handle the race where the vnode is incremented before the
906 * inactive routine is called and does its locking.
907 *
908 * Warning: Excessive use of this routine can lead to performance problems.
909 * This is because taskqs throttle back allocation if too many are created.
910 */
911void
912vn_rele_async(vnode_t *vp, taskq_t *taskq)
913{
914	VERIFY(vp->v_count > 0);
915	mutex_enter(&vp->v_lock);
916	if (vp->v_count == 1) {
917		mutex_exit(&vp->v_lock);
918		VERIFY(taskq_dispatch(taskq, (task_func_t *)vn_rele_inactive,
919		    vp, TQ_SLEEP) != TASKQID_INVALID);
920		return;
921	}
922	VN_RELE_LOCKED(vp);
923	mutex_exit(&vp->v_lock);
924}
925
926int
927vn_open(
928	char *pnamep,
929	enum uio_seg seg,
930	int filemode,
931	int createmode,
932	struct vnode **vpp,
933	enum create crwhy,
934	mode_t umask)
935{
936	return (vn_openat(pnamep, seg, filemode, createmode, vpp, crwhy,
937	    umask, NULL, -1));
938}
939
940
941/*
942 * Open/create a vnode.
943 * This may be callable by the kernel, the only known use
944 * of user context being that the current user credentials
945 * are used for permissions.  crwhy is defined iff filemode & FCREAT.
946 */
947int
948vn_openat(
949	char *pnamep,
950	enum uio_seg seg,
951	int filemode,
952	int createmode,
953	struct vnode **vpp,
954	enum create crwhy,
955	mode_t umask,
956	struct vnode *startvp,
957	int fd)
958{
959	struct vnode *vp;
960	int mode;
961	int accessflags;
962	int error;
963	int in_crit = 0;
964	int open_done = 0;
965	int shrlock_done = 0;
966	struct vattr vattr;
967	enum symfollow follow;
968	int estale_retry = 0;
969	struct shrlock shr;
970	struct shr_locowner shr_own;
971	boolean_t create;
972
973	mode = 0;
974	accessflags = 0;
975	if (filemode & FREAD)
976		mode |= VREAD;
977	if (filemode & (FWRITE|FTRUNC))
978		mode |= VWRITE;
979	if (filemode & (FSEARCH|FEXEC|FXATTRDIROPEN))
980		mode |= VEXEC;
981
982	/* symlink interpretation */
983	if (filemode & FNOFOLLOW)
984		follow = NO_FOLLOW;
985	else
986		follow = FOLLOW;
987
988	if (filemode & FAPPEND)
989		accessflags |= V_APPEND;
990
991	/*
992	 * We need to handle the case of FCREAT | FDIRECTORY and the case of
993	 * FEXCL. If all three are specified, then we always fail because we
994	 * cannot create a directory through this interface and FEXCL says we
995	 * need to fail the request if we can't create it. If, however, only
996	 * FCREAT | FDIRECTORY are specified, then we can treat this as the case
997	 * of opening a file that already exists. If it exists, we can do
998	 * something and if not, we fail. Effectively FCREAT | FDIRECTORY is
999	 * treated as FDIRECTORY.
1000	 */
1001	if ((filemode & (FCREAT | FDIRECTORY | FEXCL)) ==
1002	    (FCREAT | FDIRECTORY | FEXCL)) {
1003		return (EINVAL);
1004	}
1005
1006	if ((filemode & (FCREAT | FDIRECTORY)) == (FCREAT | FDIRECTORY)) {
1007		create = B_FALSE;
1008	} else if ((filemode & FCREAT) != 0) {
1009		create = B_TRUE;
1010	} else {
1011		create = B_FALSE;
1012	}
1013
1014top:
1015	if (create) {
1016		enum vcexcl excl;
1017
1018		/*
1019		 * Wish to create a file.
1020		 */
1021		vattr.va_type = VREG;
1022		vattr.va_mode = createmode;
1023		vattr.va_mask = AT_TYPE|AT_MODE;
1024		if (filemode & FTRUNC) {
1025			vattr.va_size = 0;
1026			vattr.va_mask |= AT_SIZE;
1027		}
1028		if (filemode & FEXCL)
1029			excl = EXCL;
1030		else
1031			excl = NONEXCL;
1032
1033		if (error =
1034		    vn_createat(pnamep, seg, &vattr, excl, mode, &vp, crwhy,
1035		    (filemode & ~(FTRUNC|FEXCL)), umask, startvp))
1036			return (error);
1037	} else {
1038		/*
1039		 * Wish to open a file.  Just look it up.
1040		 */
1041		if (error = lookupnameat(pnamep, seg, follow,
1042		    NULLVPP, &vp, startvp)) {
1043			if ((error == ESTALE) &&
1044			    fs_need_estale_retry(estale_retry++))
1045				goto top;
1046			return (error);
1047		}
1048
1049		/*
1050		 * Get the attributes to check whether file is large.
1051		 * We do this only if the FOFFMAX flag is not set and
1052		 * only for regular files.
1053		 */
1054
1055		if (!(filemode & FOFFMAX) && (vp->v_type == VREG)) {
1056			vattr.va_mask = AT_SIZE;
1057			if ((error = VOP_GETATTR(vp, &vattr, 0,
1058			    CRED(), NULL))) {
1059				goto out;
1060			}
1061			if (vattr.va_size > (u_offset_t)MAXOFF32_T) {
1062				/*
1063				 * Large File API - regular open fails
1064				 * if FOFFMAX flag is set in file mode
1065				 */
1066				error = EOVERFLOW;
1067				goto out;
1068			}
1069		}
1070		/*
1071		 * Can't write directories, active texts, or
1072		 * read-only filesystems.  Can't truncate files
1073		 * on which mandatory locking is in effect.
1074		 */
1075		if (filemode & (FWRITE|FTRUNC)) {
1076			/*
1077			 * Allow writable directory if VDIROPEN flag is set.
1078			 */
1079			if (vp->v_type == VDIR && !(vp->v_flag & VDIROPEN)) {
1080				error = EISDIR;
1081				goto out;
1082			}
1083			if (ISROFILE(vp)) {
1084				error = EROFS;
1085				goto out;
1086			}
1087			/*
1088			 * Can't truncate files on which
1089			 * sysv mandatory locking is in effect.
1090			 */
1091			if (filemode & FTRUNC) {
1092				vnode_t *rvp;
1093
1094				if (VOP_REALVP(vp, &rvp, NULL) != 0)
1095					rvp = vp;
1096				if (rvp->v_filocks != NULL) {
1097					vattr.va_mask = AT_MODE;
1098					if ((error = VOP_GETATTR(vp,
1099					    &vattr, 0, CRED(), NULL)) == 0 &&
1100					    MANDLOCK(vp, vattr.va_mode))
1101						error = EAGAIN;
1102				}
1103			}
1104			if (error)
1105				goto out;
1106		}
1107		/*
1108		 * Check permissions.
1109		 */
1110		if (error = VOP_ACCESS(vp, mode, accessflags, CRED(), NULL))
1111			goto out;
1112
1113		/*
1114		 * Require FSEARCH and FDIRECTORY to return a directory. Require
1115		 * FEXEC to return a regular file.
1116		 */
1117		if ((filemode & (FSEARCH|FDIRECTORY)) != 0 &&
1118		    vp->v_type != VDIR) {
1119			error = ENOTDIR;
1120			goto out;
1121		}
1122		if ((filemode & FEXEC) && vp->v_type != VREG) {
1123			error = ENOEXEC;	/* XXX: error code? */
1124			goto out;
1125		}
1126	}
1127
1128	/*
1129	 * Do remaining checks for FNOFOLLOW and FNOLINKS.
1130	 */
1131	if ((filemode & FNOFOLLOW) && vp->v_type == VLNK) {
1132		error = ELOOP;
1133		goto out;
1134	}
1135	if (filemode & FNOLINKS) {
1136		vattr.va_mask = AT_NLINK;
1137		if ((error = VOP_GETATTR(vp, &vattr, 0, CRED(), NULL))) {
1138			goto out;
1139		}
1140		if (vattr.va_nlink != 1) {
1141			error = EMLINK;
1142			goto out;
1143		}
1144	}
1145
1146	/*
1147	 * Opening a socket corresponding to the AF_UNIX pathname
1148	 * in the filesystem name space is not supported.
1149	 * However, VSOCK nodes in namefs are supported in order
1150	 * to make fattach work for sockets.
1151	 *
1152	 * XXX This uses VOP_REALVP to distinguish between
1153	 * an unopened namefs node (where VOP_REALVP returns a
1154	 * different VSOCK vnode) and a VSOCK created by vn_create
1155	 * in some file system (where VOP_REALVP would never return
1156	 * a different vnode).
1157	 */
1158	if (vp->v_type == VSOCK) {
1159		struct vnode *nvp;
1160
1161		error = VOP_REALVP(vp, &nvp, NULL);
1162		if (error != 0 || nvp == NULL || nvp == vp ||
1163		    nvp->v_type != VSOCK) {
1164			error = EOPNOTSUPP;
1165			goto out;
1166		}
1167	}
1168
1169	if ((vp->v_type == VREG) && nbl_need_check(vp)) {
1170		/* get share reservation */
1171		shr.s_access = 0;
1172		if (filemode & FWRITE)
1173			shr.s_access |= F_WRACC;
1174		if (filemode & FREAD)
1175			shr.s_access |= F_RDACC;
1176		shr.s_deny = 0;
1177		shr.s_sysid = 0;
1178		shr.s_pid = ttoproc(curthread)->p_pid;
1179		shr_own.sl_pid = shr.s_pid;
1180		shr_own.sl_id = fd;
1181		shr.s_own_len = sizeof (shr_own);
1182		shr.s_owner = (caddr_t)&shr_own;
1183		error = VOP_SHRLOCK(vp, F_SHARE_NBMAND, &shr, filemode, CRED(),
1184		    NULL);
1185		if (error)
1186			goto out;
1187		shrlock_done = 1;
1188
1189		/* nbmand conflict check if truncating file */
1190		if ((filemode & FTRUNC) && !(filemode & FCREAT)) {
1191			nbl_start_crit(vp, RW_READER);
1192			in_crit = 1;
1193
1194			vattr.va_mask = AT_SIZE;
1195			if (error = VOP_GETATTR(vp, &vattr, 0, CRED(), NULL))
1196				goto out;
1197			if (nbl_conflict(vp, NBL_WRITE, 0, vattr.va_size, 0,
1198			    NULL)) {
1199				error = EACCES;
1200				goto out;
1201			}
1202		}
1203	}
1204
1205	/*
1206	 * Do opening protocol.
1207	 */
1208	error = VOP_OPEN(&vp, filemode, CRED(), NULL);
1209	if (error)
1210		goto out;
1211	open_done = 1;
1212
1213	/*
1214	 * Truncate if required.
1215	 */
1216	if ((filemode & FTRUNC) && !(filemode & FCREAT)) {
1217		vattr.va_size = 0;
1218		vattr.va_mask = AT_SIZE;
1219		if ((error = VOP_SETATTR(vp, &vattr, 0, CRED(), NULL)) != 0)
1220			goto out;
1221	}
1222
1223	/*
1224	 * Turn on directio, if requested.
1225	 */
1226	if (filemode & FDIRECT) {
1227		if ((error = VOP_IOCTL(vp, _FIODIRECTIO, DIRECTIO_ON, 0,
1228		    CRED(), NULL, NULL)) != 0) {
1229			/*
1230			 * On Linux, O_DIRECT returns EINVAL when the file
1231			 * system does not support directio, so we'll do the
1232			 * same.
1233			 */
1234			error = EINVAL;
1235			goto out;
1236		}
1237	}
1238out:
1239	ASSERT(vp->v_count > 0);
1240
1241	if (in_crit) {
1242		nbl_end_crit(vp);
1243		in_crit = 0;
1244	}
1245	if (error) {
1246		if (open_done) {
1247			(void) VOP_CLOSE(vp, filemode, 1, (offset_t)0, CRED(),
1248			    NULL);
1249			open_done = 0;
1250			shrlock_done = 0;
1251		}
1252		if (shrlock_done) {
1253			(void) VOP_SHRLOCK(vp, F_UNSHARE, &shr, 0, CRED(),
1254			    NULL);
1255			shrlock_done = 0;
1256		}
1257
1258		/*
1259		 * The following clause was added to handle a problem
1260		 * with NFS consistency.  It is possible that a lookup
1261		 * of the file to be opened succeeded, but the file
1262		 * itself doesn't actually exist on the server.  This
1263		 * is chiefly due to the DNLC containing an entry for
1264		 * the file which has been removed on the server.  In
1265		 * this case, we just start over.  If there was some
1266		 * other cause for the ESTALE error, then the lookup
1267		 * of the file will fail and the error will be returned
1268		 * above instead of looping around from here.
1269		 */
1270		VN_RELE(vp);
1271		if ((error == ESTALE) && fs_need_estale_retry(estale_retry++))
1272			goto top;
1273	} else
1274		*vpp = vp;
1275	return (error);
1276}
1277
1278/*
1279 * The following two accessor functions are for the NFSv4 server.  Since there
1280 * is no VOP_OPEN_UP/DOWNGRADE we need a way for the NFS server to keep the
1281 * vnode open counts correct when a client "upgrades" an open or does an
1282 * open_downgrade.  In NFS, an upgrade or downgrade can not only change the
1283 * open mode (add or subtract read or write), but also change the share/deny
1284 * modes.  However, share reservations are not integrated with OPEN, yet, so
1285 * we need to handle each separately.  These functions are cleaner than having
1286 * the NFS server manipulate the counts directly, however, nobody else should
1287 * use these functions.
1288 */
1289void
1290vn_open_upgrade(
1291	vnode_t *vp,
1292	int filemode)
1293{
1294	ASSERT(vp->v_type == VREG);
1295
1296	if (filemode & FREAD)
1297		atomic_inc_32(&vp->v_rdcnt);
1298	if (filemode & FWRITE)
1299		atomic_inc_32(&vp->v_wrcnt);
1300
1301}
1302
1303void
1304vn_open_downgrade(
1305	vnode_t *vp,
1306	int filemode)
1307{
1308	ASSERT(vp->v_type == VREG);
1309
1310	if (filemode & FREAD) {
1311		ASSERT(vp->v_rdcnt > 0);
1312		atomic_dec_32(&vp->v_rdcnt);
1313	}
1314	if (filemode & FWRITE) {
1315		ASSERT(vp->v_wrcnt > 0);
1316		atomic_dec_32(&vp->v_wrcnt);
1317	}
1318
1319}
1320
1321int
1322vn_create(
1323	char *pnamep,
1324	enum uio_seg seg,
1325	struct vattr *vap,
1326	enum vcexcl excl,
1327	int mode,
1328	struct vnode **vpp,
1329	enum create why,
1330	int flag,
1331	mode_t umask)
1332{
1333	return (vn_createat(pnamep, seg, vap, excl, mode, vpp, why, flag,
1334	    umask, NULL));
1335}
1336
1337/*
1338 * Create a vnode (makenode).
1339 */
1340int
1341vn_createat(
1342	char *pnamep,
1343	enum uio_seg seg,
1344	struct vattr *vap,
1345	enum vcexcl excl,
1346	int mode,
1347	struct vnode **vpp,
1348	enum create why,
1349	int flag,
1350	mode_t umask,
1351	struct vnode *startvp)
1352{
1353	struct vnode *dvp;	/* ptr to parent dir vnode */
1354	struct vnode *vp = NULL;
1355	struct pathname pn;
1356	int error;
1357	int in_crit = 0;
1358	struct vattr vattr;
1359	enum symfollow follow;
1360	int estale_retry = 0;
1361	uint32_t auditing = AU_AUDITING();
1362
1363	ASSERT((vap->va_mask & (AT_TYPE|AT_MODE)) == (AT_TYPE|AT_MODE));
1364
1365	/* symlink interpretation */
1366	if ((flag & FNOFOLLOW) || excl == EXCL)
1367		follow = NO_FOLLOW;
1368	else
1369		follow = FOLLOW;
1370	flag &= ~(FNOFOLLOW|FNOLINKS);
1371
1372top:
1373	/*
1374	 * Lookup directory.
1375	 * If new object is a file, call lower level to create it.
1376	 * Note that it is up to the lower level to enforce exclusive
1377	 * creation, if the file is already there.
1378	 * This allows the lower level to do whatever
1379	 * locking or protocol that is needed to prevent races.
1380	 * If the new object is directory call lower level to make
1381	 * the new directory, with "." and "..".
1382	 */
1383	if (error = pn_get(pnamep, seg, &pn))
1384		return (error);
1385	if (auditing)
1386		audit_vncreate_start();
1387	dvp = NULL;
1388	*vpp = NULL;
1389	/*
1390	 * lookup will find the parent directory for the vnode.
1391	 * When it is done the pn holds the name of the entry
1392	 * in the directory.
1393	 * If this is a non-exclusive create we also find the node itself.
1394	 */
1395	error = lookuppnat(&pn, NULL, follow, &dvp,
1396	    (excl == EXCL) ? NULLVPP : vpp, startvp);
1397	if (error) {
1398		pn_free(&pn);
1399		if ((error == ESTALE) && fs_need_estale_retry(estale_retry++))
1400			goto top;
1401		if (why == CRMKDIR && error == EINVAL)
1402			error = EEXIST;		/* SVID */
1403		return (error);
1404	}
1405
1406	if (why != CRMKNOD)
1407		vap->va_mode &= ~VSVTX;
1408
1409	/*
1410	 * If default ACLs are defined for the directory don't apply the
1411	 * umask if umask is passed.
1412	 */
1413
1414	if (umask) {
1415
1416		vsecattr_t vsec;
1417
1418		vsec.vsa_aclcnt = 0;
1419		vsec.vsa_aclentp = NULL;
1420		vsec.vsa_dfaclcnt = 0;
1421		vsec.vsa_dfaclentp = NULL;
1422		vsec.vsa_mask = VSA_DFACLCNT;
1423		error = VOP_GETSECATTR(dvp, &vsec, 0, CRED(), NULL);
1424		/*
1425		 * If error is ENOSYS then treat it as no error
1426		 * Don't want to force all file systems to support
1427		 * aclent_t style of ACL's.
1428		 */
1429		if (error == ENOSYS)
1430			error = 0;
1431		if (error) {
1432			if (*vpp != NULL)
1433				VN_RELE(*vpp);
1434			goto out;
1435		} else {
1436			/*
1437			 * Apply the umask if no default ACLs.
1438			 */
1439			if (vsec.vsa_dfaclcnt == 0)
1440				vap->va_mode &= ~umask;
1441
1442			/*
1443			 * VOP_GETSECATTR() may have allocated memory for
1444			 * ACLs we didn't request, so double-check and
1445			 * free it if necessary.
1446			 */
1447			if (vsec.vsa_aclcnt && vsec.vsa_aclentp != NULL)
1448				kmem_free((caddr_t)vsec.vsa_aclentp,
1449				    vsec.vsa_aclcnt * sizeof (aclent_t));
1450			if (vsec.vsa_dfaclcnt && vsec.vsa_dfaclentp != NULL)
1451				kmem_free((caddr_t)vsec.vsa_dfaclentp,
1452				    vsec.vsa_dfaclcnt * sizeof (aclent_t));
1453		}
1454	}
1455
1456	/*
1457	 * In general we want to generate EROFS if the file system is
1458	 * readonly.  However, POSIX (IEEE Std. 1003.1) section 5.3.1
1459	 * documents the open system call, and it says that O_CREAT has no
1460	 * effect if the file already exists.  Bug 1119649 states
1461	 * that open(path, O_CREAT, ...) fails when attempting to open an
1462	 * existing file on a read only file system.  Thus, the first part
1463	 * of the following if statement has 3 checks:
1464	 *	if the file exists &&
1465	 *		it is being open with write access &&
1466	 *		the file system is read only
1467	 *	then generate EROFS
1468	 */
1469	if ((*vpp != NULL && (mode & VWRITE) && ISROFILE(*vpp)) ||
1470	    (*vpp == NULL && dvp->v_vfsp->vfs_flag & VFS_RDONLY)) {
1471		if (*vpp)
1472			VN_RELE(*vpp);
1473		error = EROFS;
1474	} else if (excl == NONEXCL && *vpp != NULL) {
1475		vnode_t *rvp;
1476
1477		/*
1478		 * File already exists.  If a mandatory lock has been
1479		 * applied, return error.
1480		 */
1481		vp = *vpp;
1482		if (VOP_REALVP(vp, &rvp, NULL) != 0)
1483			rvp = vp;
1484		if ((vap->va_mask & AT_SIZE) && nbl_need_check(vp)) {
1485			nbl_start_crit(vp, RW_READER);
1486			in_crit = 1;
1487		}
1488		if (rvp->v_filocks != NULL || rvp->v_shrlocks != NULL) {
1489			vattr.va_mask = AT_MODE|AT_SIZE;
1490			if (error = VOP_GETATTR(vp, &vattr, 0, CRED(), NULL)) {
1491				goto out;
1492			}
1493			if (MANDLOCK(vp, vattr.va_mode)) {
1494				error = EAGAIN;
1495				goto out;
1496			}
1497			/*
1498			 * File cannot be truncated if non-blocking mandatory
1499			 * locks are currently on the file.
1500			 */
1501			if ((vap->va_mask & AT_SIZE) && in_crit) {
1502				u_offset_t offset;
1503				ssize_t length;
1504
1505				offset = vap->va_size > vattr.va_size ?
1506				    vattr.va_size : vap->va_size;
1507				length = vap->va_size > vattr.va_size ?
1508				    vap->va_size - vattr.va_size :
1509				    vattr.va_size - vap->va_size;
1510				if (nbl_conflict(vp, NBL_WRITE, offset,
1511				    length, 0, NULL)) {
1512					error = EACCES;
1513					goto out;
1514				}
1515			}
1516		}
1517
1518		/*
1519		 * If the file is the root of a VFS, we've crossed a
1520		 * mount point and the "containing" directory that we
1521		 * acquired above (dvp) is irrelevant because it's in
1522		 * a different file system.  We apply VOP_CREATE to the
1523		 * target itself instead of to the containing directory
1524		 * and supply a null path name to indicate (conventionally)
1525		 * the node itself as the "component" of interest.
1526		 *
1527		 * The call to VOP_CREATE() is necessary to ensure
1528		 * that the appropriate permission checks are made,
1529		 * i.e. EISDIR, EACCES, etc.  We already know that vpp
1530		 * exists since we are in the else condition where this
1531		 * was checked.
1532		 */
1533		if (vp->v_flag & VROOT) {
1534			ASSERT(why != CRMKDIR);
1535			error = VOP_CREATE(vp, "", vap, excl, mode, vpp,
1536			    CRED(), flag, NULL, NULL);
1537			/*
1538			 * If the create succeeded, it will have created a
1539			 * new reference on a new vnode (*vpp) in the child
1540			 * file system, so we want to drop our reference on
1541			 * the old (vp) upon exit.
1542			 */
1543			goto out;
1544		}
1545
1546		/*
1547		 * Large File API - non-large open (FOFFMAX flag not set)
1548		 * of regular file fails if the file size exceeds MAXOFF32_T.
1549		 */
1550		if (why != CRMKDIR &&
1551		    !(flag & FOFFMAX) &&
1552		    (vp->v_type == VREG)) {
1553			vattr.va_mask = AT_SIZE;
1554			if ((error = VOP_GETATTR(vp, &vattr, 0,
1555			    CRED(), NULL))) {
1556				goto out;
1557			}
1558			if ((vattr.va_size > (u_offset_t)MAXOFF32_T)) {
1559				error = EOVERFLOW;
1560				goto out;
1561			}
1562		}
1563	}
1564
1565	if (error == 0) {
1566		/*
1567		 * Call mkdir() if specified, otherwise create().
1568		 */
1569		int must_be_dir = pn_fixslash(&pn);	/* trailing '/'? */
1570
1571		if (why == CRMKDIR)
1572			/*
1573			 * N.B., if vn_createat() ever requests
1574			 * case-insensitive behavior then it will need
1575			 * to be passed to VOP_MKDIR().  VOP_CREATE()
1576			 * will already get it via "flag"
1577			 */
1578			error = VOP_MKDIR(dvp, pn.pn_path, vap, vpp, CRED(),
1579			    NULL, 0, NULL);
1580		else if (!must_be_dir)
1581			error = VOP_CREATE(dvp, pn.pn_path, vap,
1582			    excl, mode, vpp, CRED(), flag, NULL, NULL);
1583		else
1584			error = ENOTDIR;
1585	}
1586
1587out:
1588
1589	if (auditing)
1590		audit_vncreate_finish(*vpp, error);
1591	if (in_crit) {
1592		nbl_end_crit(vp);
1593		in_crit = 0;
1594	}
1595	if (vp != NULL) {
1596		VN_RELE(vp);
1597		vp = NULL;
1598	}
1599	pn_free(&pn);
1600	VN_RELE(dvp);
1601	/*
1602	 * The following clause was added to handle a problem
1603	 * with NFS consistency.  It is possible that a lookup
1604	 * of the file to be created succeeded, but the file
1605	 * itself doesn't actually exist on the server.  This
1606	 * is chiefly due to the DNLC containing an entry for
1607	 * the file which has been removed on the server.  In
1608	 * this case, we just start over.  If there was some
1609	 * other cause for the ESTALE error, then the lookup
1610	 * of the file will fail and the error will be returned
1611	 * above instead of looping around from here.
1612	 */
1613	if ((error == ESTALE) && fs_need_estale_retry(estale_retry++))
1614		goto top;
1615	return (error);
1616}
1617
1618int
1619vn_link(char *from, char *to, enum uio_seg seg)
1620{
1621	return (vn_linkat(NULL, from, NO_FOLLOW, NULL, to, seg));
1622}
1623
1624int
1625vn_linkat(vnode_t *fstartvp, char *from, enum symfollow follow,
1626    vnode_t *tstartvp, char *to, enum uio_seg seg)
1627{
1628	struct vnode *fvp;		/* from vnode ptr */
1629	struct vnode *tdvp;		/* to directory vnode ptr */
1630	struct pathname pn;
1631	int error;
1632	struct vattr vattr;
1633	dev_t fsid;
1634	int estale_retry = 0;
1635	uint32_t auditing = AU_AUDITING();
1636
1637top:
1638	fvp = tdvp = NULL;
1639	if (error = pn_get(to, seg, &pn))
1640		return (error);
1641	if (auditing && fstartvp != NULL)
1642		audit_setfsat_path(1);
1643	if (error = lookupnameat(from, seg, follow, NULLVPP, &fvp, fstartvp))
1644		goto out;
1645	if (auditing && tstartvp != NULL)
1646		audit_setfsat_path(3);
1647	if (error = lookuppnat(&pn, NULL, NO_FOLLOW, &tdvp, NULLVPP, tstartvp))
1648		goto out;
1649	/*
1650	 * Make sure both source vnode and target directory vnode are
1651	 * in the same vfs and that it is writeable.
1652	 */
1653	vattr.va_mask = AT_FSID;
1654	if (error = VOP_GETATTR(fvp, &vattr, 0, CRED(), NULL))
1655		goto out;
1656	fsid = vattr.va_fsid;
1657	vattr.va_mask = AT_FSID;
1658	if (error = VOP_GETATTR(tdvp, &vattr, 0, CRED(), NULL))
1659		goto out;
1660	if (fsid != vattr.va_fsid) {
1661		error = EXDEV;
1662		goto out;
1663	}
1664	if (tdvp->v_vfsp->vfs_flag & VFS_RDONLY) {
1665		error = EROFS;
1666		goto out;
1667	}
1668	/*
1669	 * Do the link.
1670	 */
1671	(void) pn_fixslash(&pn);
1672	error = VOP_LINK(tdvp, fvp, pn.pn_path, CRED(), NULL, 0);
1673out:
1674	pn_free(&pn);
1675	if (fvp)
1676		VN_RELE(fvp);
1677	if (tdvp)
1678		VN_RELE(tdvp);
1679	if ((error == ESTALE) && fs_need_estale_retry(estale_retry++))
1680		goto top;
1681	return (error);
1682}
1683
1684int
1685vn_rename(char *from, char *to, enum uio_seg seg)
1686{
1687	return (vn_renameat(NULL, from, NULL, to, seg));
1688}
1689
1690int
1691vn_renameat(vnode_t *fdvp, char *fname, vnode_t *tdvp,
1692    char *tname, enum uio_seg seg)
1693{
1694	int error;
1695	struct vattr vattr;
1696	struct pathname fpn;		/* from pathname */
1697	struct pathname tpn;		/* to pathname */
1698	dev_t fsid;
1699	int in_crit_src, in_crit_targ;
1700	vnode_t *fromvp, *fvp;
1701	vnode_t *tovp, *targvp;
1702	int estale_retry = 0;
1703	uint32_t auditing = AU_AUDITING();
1704
1705top:
1706	fvp = fromvp = tovp = targvp = NULL;
1707	in_crit_src = in_crit_targ = 0;
1708	/*
1709	 * Get to and from pathnames.
1710	 */
1711	if (error = pn_get(fname, seg, &fpn))
1712		return (error);
1713	if (error = pn_get(tname, seg, &tpn)) {
1714		pn_free(&fpn);
1715		return (error);
1716	}
1717
1718	/*
1719	 * First we need to resolve the correct directories
1720	 * The passed in directories may only be a starting point,
1721	 * but we need the real directories the file(s) live in.
1722	 * For example the fname may be something like usr/lib/sparc
1723	 * and we were passed in the / directory, but we need to
1724	 * use the lib directory for the rename.
1725	 */
1726
1727	if (auditing && fdvp != NULL)
1728		audit_setfsat_path(1);
1729	/*
1730	 * Lookup to and from directories.
1731	 */
1732	if (error = lookuppnat(&fpn, NULL, NO_FOLLOW, &fromvp, &fvp, fdvp)) {
1733		goto out;
1734	}
1735
1736	/*
1737	 * Make sure there is an entry.
1738	 */
1739	if (fvp == NULL) {
1740		error = ENOENT;
1741		goto out;
1742	}
1743
1744	if (auditing && tdvp != NULL)
1745		audit_setfsat_path(3);
1746	if (error = lookuppnat(&tpn, NULL, NO_FOLLOW, &tovp, &targvp, tdvp)) {
1747		goto out;
1748	}
1749
1750	/*
1751	 * Make sure both the from vnode directory and the to directory
1752	 * are in the same vfs and the to directory is writable.
1753	 * We check fsid's, not vfs pointers, so loopback fs works.
1754	 */
1755	if (fromvp != tovp) {
1756		vattr.va_mask = AT_FSID;
1757		if (error = VOP_GETATTR(fromvp, &vattr, 0, CRED(), NULL))
1758			goto out;
1759		fsid = vattr.va_fsid;
1760		vattr.va_mask = AT_FSID;
1761		if (error = VOP_GETATTR(tovp, &vattr, 0, CRED(), NULL))
1762			goto out;
1763		if (fsid != vattr.va_fsid) {
1764			error = EXDEV;
1765			goto out;
1766		}
1767	}
1768
1769	if (tovp->v_vfsp->vfs_flag & VFS_RDONLY) {
1770		error = EROFS;
1771		goto out;
1772	}
1773
1774	/*
1775	 * Make sure "from" vp is not a mount point.
1776	 * Note, lookup did traverse() already, so
1777	 * we'll be looking at the mounted FS root.
1778	 * (but allow files like mnttab)
1779	 */
1780	if ((fvp->v_flag & VROOT) != 0 && fvp->v_type == VDIR) {
1781		error = EBUSY;
1782		goto out;
1783	}
1784
1785	if (targvp && (fvp != targvp)) {
1786		nbl_start_crit(targvp, RW_READER);
1787		in_crit_targ = 1;
1788		if (nbl_conflict(targvp, NBL_REMOVE, 0, 0, 0, NULL)) {
1789			error = EACCES;
1790			goto out;
1791		}
1792	}
1793
1794	if (nbl_need_check(fvp)) {
1795		nbl_start_crit(fvp, RW_READER);
1796		in_crit_src = 1;
1797		if (nbl_conflict(fvp, NBL_RENAME, 0, 0, 0, NULL)) {
1798			error = EACCES;
1799			goto out;
1800		}
1801	}
1802
1803	/*
1804	 * Do the rename.
1805	 */
1806	(void) pn_fixslash(&tpn);
1807	error = VOP_RENAME(fromvp, fpn.pn_path, tovp, tpn.pn_path, CRED(),
1808	    NULL, 0);
1809
1810out:
1811	pn_free(&fpn);
1812	pn_free(&tpn);
1813	if (in_crit_src)
1814		nbl_end_crit(fvp);
1815	if (in_crit_targ)
1816		nbl_end_crit(targvp);
1817	if (fromvp)
1818		VN_RELE(fromvp);
1819	if (tovp)
1820		VN_RELE(tovp);
1821	if (targvp)
1822		VN_RELE(targvp);
1823	if (fvp)
1824		VN_RELE(fvp);
1825	if ((error == ESTALE) && fs_need_estale_retry(estale_retry++))
1826		goto top;
1827	return (error);
1828}
1829
1830/*
1831 * Remove a file or directory.
1832 */
1833int
1834vn_remove(char *fnamep, enum uio_seg seg, enum rm dirflag)
1835{
1836	return (vn_removeat(NULL, fnamep, seg, dirflag));
1837}
1838
1839int
1840vn_removeat(vnode_t *startvp, char *fnamep, enum uio_seg seg, enum rm dirflag)
1841{
1842	struct vnode *vp;		/* entry vnode */
1843	struct vnode *dvp;		/* ptr to parent dir vnode */
1844	struct vnode *coveredvp;
1845	struct pathname pn;		/* name of entry */
1846	enum vtype vtype;
1847	int error;
1848	struct vfs *vfsp;
1849	struct vfs *dvfsp;	/* ptr to parent dir vfs */
1850	int in_crit = 0;
1851	int estale_retry = 0;
1852
1853top:
1854	if (error = pn_get(fnamep, seg, &pn))
1855		return (error);
1856	dvp = vp = NULL;
1857	if (error = lookuppnat(&pn, NULL, NO_FOLLOW, &dvp, &vp, startvp)) {
1858		pn_free(&pn);
1859		if ((error == ESTALE) && fs_need_estale_retry(estale_retry++))
1860			goto top;
1861		return (error);
1862	}
1863
1864	/*
1865	 * Make sure there is an entry.
1866	 */
1867	if (vp == NULL) {
1868		error = ENOENT;
1869		goto out;
1870	}
1871
1872	vfsp = vp->v_vfsp;
1873	dvfsp = dvp->v_vfsp;
1874
1875	/*
1876	 * If the named file is the root of a mounted filesystem, fail,
1877	 * unless it's marked unlinkable.  In that case, unmount the
1878	 * filesystem and proceed to unlink the covered vnode.  (If the
1879	 * covered vnode is a directory, use rmdir instead of unlink,
1880	 * to avoid file system corruption.)
1881	 */
1882	if (vp->v_flag & VROOT) {
1883		if ((vfsp->vfs_flag & VFS_UNLINKABLE) == 0) {
1884			error = EBUSY;
1885			goto out;
1886		}
1887
1888		/*
1889		 * Namefs specific code starts here.
1890		 */
1891
1892		if (dirflag == RMDIRECTORY) {
1893			/*
1894			 * User called rmdir(2) on a file that has
1895			 * been namefs mounted on top of.  Since
1896			 * namefs doesn't allow directories to
1897			 * be mounted on other files we know
1898			 * vp is not of type VDIR so fail to operation.
1899			 */
1900			error = ENOTDIR;
1901			goto out;
1902		}
1903
1904		/*
1905		 * If VROOT is still set after grabbing vp->v_lock,
1906		 * noone has finished nm_unmount so far and coveredvp
1907		 * is valid.
1908		 * If we manage to grab vn_vfswlock(coveredvp) before releasing
1909		 * vp->v_lock, any race window is eliminated.
1910		 */
1911
1912		mutex_enter(&vp->v_lock);
1913		if ((vp->v_flag & VROOT) == 0) {
1914			/* Someone beat us to the unmount */
1915			mutex_exit(&vp->v_lock);
1916			error = EBUSY;
1917			goto out;
1918		}
1919		vfsp = vp->v_vfsp;
1920		coveredvp = vfsp->vfs_vnodecovered;
1921		ASSERT(coveredvp);
1922		/*
1923		 * Note: Implementation of vn_vfswlock shows that ordering of
1924		 * v_lock / vn_vfswlock is not an issue here.
1925		 */
1926		error = vn_vfswlock(coveredvp);
1927		mutex_exit(&vp->v_lock);
1928
1929		if (error)
1930			goto out;
1931
1932		VN_HOLD(coveredvp);
1933		VN_RELE(vp);
1934		error = dounmount(vfsp, 0, CRED());
1935
1936		/*
1937		 * Unmounted the namefs file system; now get
1938		 * the object it was mounted over.
1939		 */
1940		vp = coveredvp;
1941		/*
1942		 * If namefs was mounted over a directory, then
1943		 * we want to use rmdir() instead of unlink().
1944		 */
1945		if (vp->v_type == VDIR)
1946			dirflag = RMDIRECTORY;
1947
1948		if (error)
1949			goto out;
1950	}
1951
1952	/*
1953	 * Make sure filesystem is writeable.
1954	 * We check the parent directory's vfs in case this is an lofs vnode.
1955	 */
1956	if (dvfsp && dvfsp->vfs_flag & VFS_RDONLY) {
1957		error = EROFS;
1958		goto out;
1959	}
1960
1961	vtype = vp->v_type;
1962
1963	/*
1964	 * If there is the possibility of an nbmand share reservation, make
1965	 * sure it's okay to remove the file.  Keep a reference to the
1966	 * vnode, so that we can exit the nbl critical region after
1967	 * calling VOP_REMOVE.
1968	 * If there is no possibility of an nbmand share reservation,
1969	 * release the vnode reference now.  Filesystems like NFS may
1970	 * behave differently if there is an extra reference, so get rid of
1971	 * this one.  Fortunately, we can't have nbmand mounts on NFS
1972	 * filesystems.
1973	 */
1974	if (nbl_need_check(vp)) {
1975		nbl_start_crit(vp, RW_READER);
1976		in_crit = 1;
1977		if (nbl_conflict(vp, NBL_REMOVE, 0, 0, 0, NULL)) {
1978			error = EACCES;
1979			goto out;
1980		}
1981	} else {
1982		VN_RELE(vp);
1983		vp = NULL;
1984	}
1985
1986	if (dirflag == RMDIRECTORY) {
1987		/*
1988		 * Caller is using rmdir(2), which can only be applied to
1989		 * directories.
1990		 */
1991		if (vtype != VDIR) {
1992			error = ENOTDIR;
1993		} else {
1994			vnode_t *cwd;
1995			proc_t *pp = curproc;
1996
1997			mutex_enter(&pp->p_lock);
1998			cwd = PTOU(pp)->u_cdir;
1999			VN_HOLD(cwd);
2000			mutex_exit(&pp->p_lock);
2001			error = VOP_RMDIR(dvp, pn.pn_path, cwd, CRED(),
2002			    NULL, 0);
2003			VN_RELE(cwd);
2004		}
2005	} else {
2006		/*
2007		 * Unlink(2) can be applied to anything.
2008		 */
2009		error = VOP_REMOVE(dvp, pn.pn_path, CRED(), NULL, 0);
2010	}
2011
2012out:
2013	pn_free(&pn);
2014	if (in_crit) {
2015		nbl_end_crit(vp);
2016		in_crit = 0;
2017	}
2018	if (vp != NULL)
2019		VN_RELE(vp);
2020	if (dvp != NULL)
2021		VN_RELE(dvp);
2022	if ((error == ESTALE) && fs_need_estale_retry(estale_retry++))
2023		goto top;
2024	return (error);
2025}
2026
2027/*
2028 * Utility function to compare equality of vnodes.
2029 * Compare the underlying real vnodes, if there are underlying vnodes.
2030 * This is a more thorough comparison than the VN_CMP() macro provides.
2031 */
2032int
2033vn_compare(vnode_t *vp1, vnode_t *vp2)
2034{
2035	vnode_t *realvp;
2036
2037	if (vp1 != NULL && VOP_REALVP(vp1, &realvp, NULL) == 0)
2038		vp1 = realvp;
2039	if (vp2 != NULL && VOP_REALVP(vp2, &realvp, NULL) == 0)
2040		vp2 = realvp;
2041	return (VN_CMP(vp1, vp2));
2042}
2043
2044/*
2045 * The number of locks to hash into.  This value must be a power
2046 * of 2 minus 1 and should probably also be prime.
2047 */
2048#define	NUM_BUCKETS	1023
2049
2050struct  vn_vfslocks_bucket {
2051	kmutex_t vb_lock;
2052	vn_vfslocks_entry_t *vb_list;
2053	char pad[64 - sizeof (kmutex_t) - sizeof (void *)];
2054};
2055
2056/*
2057 * Total number of buckets will be NUM_BUCKETS + 1 .
2058 */
2059
2060#pragma	align	64(vn_vfslocks_buckets)
2061static	struct vn_vfslocks_bucket	vn_vfslocks_buckets[NUM_BUCKETS + 1];
2062
2063#define	VN_VFSLOCKS_SHIFT	9
2064
2065#define	VN_VFSLOCKS_HASH(vfsvpptr)	\
2066	((((intptr_t)(vfsvpptr)) >> VN_VFSLOCKS_SHIFT) & NUM_BUCKETS)
2067
2068/*
2069 * vn_vfslocks_getlock() uses an HASH scheme to generate
2070 * rwstlock using vfs/vnode pointer passed to it.
2071 *
2072 * vn_vfslocks_rele() releases a reference in the
2073 * HASH table which allows the entry allocated by
2074 * vn_vfslocks_getlock() to be freed at a later
2075 * stage when the refcount drops to zero.
2076 */
2077
2078vn_vfslocks_entry_t *
2079vn_vfslocks_getlock(void *vfsvpptr)
2080{
2081	struct vn_vfslocks_bucket *bp;
2082	vn_vfslocks_entry_t *vep;
2083	vn_vfslocks_entry_t *tvep;
2084
2085	ASSERT(vfsvpptr != NULL);
2086	bp = &vn_vfslocks_buckets[VN_VFSLOCKS_HASH(vfsvpptr)];
2087
2088	mutex_enter(&bp->vb_lock);
2089	for (vep = bp->vb_list; vep != NULL; vep = vep->ve_next) {
2090		if (vep->ve_vpvfs == vfsvpptr) {
2091			vep->ve_refcnt++;
2092			mutex_exit(&bp->vb_lock);
2093			return (vep);
2094		}
2095	}
2096	mutex_exit(&bp->vb_lock);
2097	vep = kmem_alloc(sizeof (*vep), KM_SLEEP);
2098	rwst_init(&vep->ve_lock, NULL, RW_DEFAULT, NULL);
2099	vep->ve_vpvfs = (char *)vfsvpptr;
2100	vep->ve_refcnt = 1;
2101	mutex_enter(&bp->vb_lock);
2102	for (tvep = bp->vb_list; tvep != NULL; tvep = tvep->ve_next) {
2103		if (tvep->ve_vpvfs == vfsvpptr) {
2104			tvep->ve_refcnt++;
2105			mutex_exit(&bp->vb_lock);
2106
2107			/*
2108			 * There is already an entry in the hash
2109			 * destroy what we just allocated.
2110			 */
2111			rwst_destroy(&vep->ve_lock);
2112			kmem_free(vep, sizeof (*vep));
2113			return (tvep);
2114		}
2115	}
2116	vep->ve_next = bp->vb_list;
2117	bp->vb_list = vep;
2118	mutex_exit(&bp->vb_lock);
2119	return (vep);
2120}
2121
2122void
2123vn_vfslocks_rele(vn_vfslocks_entry_t *vepent)
2124{
2125	struct vn_vfslocks_bucket *bp;
2126	vn_vfslocks_entry_t *vep;
2127	vn_vfslocks_entry_t *pvep;
2128
2129	ASSERT(vepent != NULL);
2130	ASSERT(vepent->ve_vpvfs != NULL);
2131
2132	bp = &vn_vfslocks_buckets[VN_VFSLOCKS_HASH(vepent->ve_vpvfs)];
2133
2134	mutex_enter(&bp->vb_lock);
2135	vepent->ve_refcnt--;
2136
2137	if ((int32_t)vepent->ve_refcnt < 0)
2138		cmn_err(CE_PANIC, "vn_vfslocks_rele: refcount negative");
2139
2140	pvep = NULL;
2141	if (vepent->ve_refcnt == 0) {
2142		for (vep = bp->vb_list; vep != NULL; vep = vep->ve_next) {
2143			if (vep->ve_vpvfs == vepent->ve_vpvfs) {
2144				if (pvep == NULL)
2145					bp->vb_list = vep->ve_next;
2146				else {
2147					pvep->ve_next = vep->ve_next;
2148				}
2149				mutex_exit(&bp->vb_lock);
2150				rwst_destroy(&vep->ve_lock);
2151				kmem_free(vep, sizeof (*vep));
2152				return;
2153			}
2154			pvep = vep;
2155		}
2156		cmn_err(CE_PANIC, "vn_vfslocks_rele: vp/vfs not found");
2157	}
2158	mutex_exit(&bp->vb_lock);
2159}
2160
2161/*
2162 * vn_vfswlock_wait is used to implement a lock which is logically a writers
2163 * lock protecting the v_vfsmountedhere field.
2164 * vn_vfswlock_wait has been modified to be similar to vn_vfswlock,
2165 * except that it blocks to acquire the lock VVFSLOCK.
2166 *
2167 * traverse() and routines re-implementing part of traverse (e.g. autofs)
2168 * need to hold this lock. mount(), vn_rename(), vn_remove() and so on
2169 * need the non-blocking version of the writers lock i.e. vn_vfswlock
2170 */
2171int
2172vn_vfswlock_wait(vnode_t *vp)
2173{
2174	int retval;
2175	vn_vfslocks_entry_t *vpvfsentry;
2176	ASSERT(vp != NULL);
2177
2178	vpvfsentry = vn_vfslocks_getlock(vp);
2179	retval = rwst_enter_sig(&vpvfsentry->ve_lock, RW_WRITER);
2180
2181	if (retval == EINTR) {
2182		vn_vfslocks_rele(vpvfsentry);
2183		return (EINTR);
2184	}
2185	return (retval);
2186}
2187
2188int
2189vn_vfsrlock_wait(vnode_t *vp)
2190{
2191	int retval;
2192	vn_vfslocks_entry_t *vpvfsentry;
2193	ASSERT(vp != NULL);
2194
2195	vpvfsentry = vn_vfslocks_getlock(vp);
2196	retval = rwst_enter_sig(&vpvfsentry->ve_lock, RW_READER);
2197
2198	if (retval == EINTR) {
2199		vn_vfslocks_rele(vpvfsentry);
2200		return (EINTR);
2201	}
2202
2203	return (retval);
2204}
2205
2206
2207/*
2208 * vn_vfswlock is used to implement a lock which is logically a writers lock
2209 * protecting the v_vfsmountedhere field.
2210 */
2211int
2212vn_vfswlock(vnode_t *vp)
2213{
2214	vn_vfslocks_entry_t *vpvfsentry;
2215
2216	/*
2217	 * If vp is NULL then somebody is trying to lock the covered vnode
2218	 * of /.  (vfs_vnodecovered is NULL for /).  This situation will
2219	 * only happen when unmounting /.  Since that operation will fail
2220	 * anyway, return EBUSY here instead of in VFS_UNMOUNT.
2221	 */
2222	if (vp == NULL)
2223		return (EBUSY);
2224
2225	vpvfsentry = vn_vfslocks_getlock(vp);
2226
2227	if (rwst_tryenter(&vpvfsentry->ve_lock, RW_WRITER))
2228		return (0);
2229
2230	vn_vfslocks_rele(vpvfsentry);
2231	return (EBUSY);
2232}
2233
2234int
2235vn_vfsrlock(vnode_t *vp)
2236{
2237	vn_vfslocks_entry_t *vpvfsentry;
2238
2239	/*
2240	 * If vp is NULL then somebody is trying to lock the covered vnode
2241	 * of /.  (vfs_vnodecovered is NULL for /).  This situation will
2242	 * only happen when unmounting /.  Since that operation will fail
2243	 * anyway, return EBUSY here instead of in VFS_UNMOUNT.
2244	 */
2245	if (vp == NULL)
2246		return (EBUSY);
2247
2248	vpvfsentry = vn_vfslocks_getlock(vp);
2249
2250	if (rwst_tryenter(&vpvfsentry->ve_lock, RW_READER))
2251		return (0);
2252
2253	vn_vfslocks_rele(vpvfsentry);
2254	return (EBUSY);
2255}
2256
2257void
2258vn_vfsunlock(vnode_t *vp)
2259{
2260	vn_vfslocks_entry_t *vpvfsentry;
2261
2262	/*
2263	 * ve_refcnt needs to be decremented twice.
2264	 * 1. To release refernce after a call to vn_vfslocks_getlock()
2265	 * 2. To release the reference from the locking routines like
2266	 *    vn_vfsrlock/vn_vfswlock etc,.
2267	 */
2268	vpvfsentry = vn_vfslocks_getlock(vp);
2269	vn_vfslocks_rele(vpvfsentry);
2270
2271	rwst_exit(&vpvfsentry->ve_lock);
2272	vn_vfslocks_rele(vpvfsentry);
2273}
2274
2275int
2276vn_vfswlock_held(vnode_t *vp)
2277{
2278	int held;
2279	vn_vfslocks_entry_t *vpvfsentry;
2280
2281	ASSERT(vp != NULL);
2282
2283	vpvfsentry = vn_vfslocks_getlock(vp);
2284	held = rwst_lock_held(&vpvfsentry->ve_lock, RW_WRITER);
2285
2286	vn_vfslocks_rele(vpvfsentry);
2287	return (held);
2288}
2289
2290
2291int
2292vn_make_ops(
2293	const char *name,			/* Name of file system */
2294	const fs_operation_def_t *templ,	/* Operation specification */
2295	vnodeops_t **actual)			/* Return the vnodeops */
2296{
2297	int unused_ops;
2298	int error;
2299
2300	*actual = (vnodeops_t *)kmem_alloc(sizeof (vnodeops_t), KM_SLEEP);
2301
2302	(*actual)->vnop_name = name;
2303
2304	error = fs_build_vector(*actual, &unused_ops, vn_ops_table, templ);
2305	if (error) {
2306		kmem_free(*actual, sizeof (vnodeops_t));
2307	}
2308
2309#if DEBUG
2310	if (unused_ops != 0)
2311		cmn_err(CE_WARN, "vn_make_ops: %s: %d operations supplied "
2312		    "but not used", name, unused_ops);
2313#endif
2314
2315	return (error);
2316}
2317
2318/*
2319 * Free the vnodeops created as a result of vn_make_ops()
2320 */
2321void
2322vn_freevnodeops(vnodeops_t *vnops)
2323{
2324	kmem_free(vnops, sizeof (vnodeops_t));
2325}
2326
2327/*
2328 * Vnode cache.
2329 */
2330
2331/* ARGSUSED */
2332static int
2333vn_cache_constructor(void *buf, void *cdrarg, int kmflags)
2334{
2335	struct vnode *vp;
2336
2337	vp = buf;
2338
2339	mutex_init(&vp->v_lock, NULL, MUTEX_DEFAULT, NULL);
2340	mutex_init(&vp->v_vsd_lock, NULL, MUTEX_DEFAULT, NULL);
2341	cv_init(&vp->v_cv, NULL, CV_DEFAULT, NULL);
2342	rw_init(&vp->v_nbllock, NULL, RW_DEFAULT, NULL);
2343	vp->v_femhead = NULL;	/* Must be done before vn_reinit() */
2344	vp->v_path = vn_vpath_empty;
2345	vp->v_path_stamp = 0;
2346	vp->v_mpssdata = NULL;
2347	vp->v_vsd = NULL;
2348	vp->v_fopdata = NULL;
2349
2350	return (0);
2351}
2352
2353/* ARGSUSED */
2354static void
2355vn_cache_destructor(void *buf, void *cdrarg)
2356{
2357	struct vnode *vp;
2358
2359	vp = buf;
2360
2361	rw_destroy(&vp->v_nbllock);
2362	cv_destroy(&vp->v_cv);
2363	mutex_destroy(&vp->v_vsd_lock);
2364	mutex_destroy(&vp->v_lock);
2365}
2366
2367void
2368vn_create_cache(void)
2369{
2370	/* LINTED */
2371	ASSERT((1 << VNODE_ALIGN_LOG2) ==
2372	    P2ROUNDUP(sizeof (struct vnode), VNODE_ALIGN));
2373	vn_cache = kmem_cache_create("vn_cache", sizeof (struct vnode),
2374	    VNODE_ALIGN, vn_cache_constructor, vn_cache_destructor, NULL, NULL,
2375	    NULL, 0);
2376}
2377
2378void
2379vn_destroy_cache(void)
2380{
2381	kmem_cache_destroy(vn_cache);
2382}
2383
2384/*
2385 * Used by file systems when fs-specific nodes (e.g., ufs inodes) are
2386 * cached by the file system and vnodes remain associated.
2387 */
2388void
2389vn_recycle(vnode_t *vp)
2390{
2391	ASSERT(vp->v_pages == NULL);
2392	VERIFY(vp->v_path != NULL);
2393
2394	/*
2395	 * XXX - This really belongs in vn_reinit(), but we have some issues
2396	 * with the counts.  Best to have it here for clean initialization.
2397	 */
2398	vp->v_rdcnt = 0;
2399	vp->v_wrcnt = 0;
2400	vp->v_mmap_read = 0;
2401	vp->v_mmap_write = 0;
2402
2403	/*
2404	 * If FEM was in use, make sure everything gets cleaned up
2405	 * NOTE: vp->v_femhead is initialized to NULL in the vnode
2406	 * constructor.
2407	 */
2408	if (vp->v_femhead) {
2409		/* XXX - There should be a free_femhead() that does all this */
2410		ASSERT(vp->v_femhead->femh_list == NULL);
2411		mutex_destroy(&vp->v_femhead->femh_lock);
2412		kmem_free(vp->v_femhead, sizeof (*(vp->v_femhead)));
2413		vp->v_femhead = NULL;
2414	}
2415	if (vp->v_path != vn_vpath_empty) {
2416		kmem_free(vp->v_path, strlen(vp->v_path) + 1);
2417		vp->v_path = vn_vpath_empty;
2418	}
2419	vp->v_path_stamp = 0;
2420
2421	if (vp->v_fopdata != NULL) {
2422		free_fopdata(vp);
2423	}
2424	vp->v_mpssdata = NULL;
2425	vsd_free(vp);
2426}
2427
2428/*
2429 * Used to reset the vnode fields including those that are directly accessible
2430 * as well as those which require an accessor function.
2431 *
2432 * Does not initialize:
2433 *	synchronization objects: v_lock, v_vsd_lock, v_nbllock, v_cv
2434 *	v_data (since FS-nodes and vnodes point to each other and should
2435 *		be updated simultaneously)
2436 *	v_op (in case someone needs to make a VOP call on this object)
2437 */
2438void
2439vn_reinit(vnode_t *vp)
2440{
2441	vp->v_count = 1;
2442	vp->v_count_dnlc = 0;
2443	vp->v_vfsp = NULL;
2444	vp->v_stream = NULL;
2445	vp->v_vfsmountedhere = NULL;
2446	vp->v_flag = 0;
2447	vp->v_type = VNON;
2448	vp->v_rdev = NODEV;
2449
2450	vp->v_filocks = NULL;
2451	vp->v_shrlocks = NULL;
2452	vp->v_pages = NULL;
2453
2454	vp->v_locality = NULL;
2455	vp->v_xattrdir = NULL;
2456
2457	/*
2458	 * In a few specific instances, vn_reinit() is used to initialize
2459	 * locally defined vnode_t instances.  Lacking the construction offered
2460	 * by vn_alloc(), these vnodes require v_path initialization.
2461	 */
2462	if (vp->v_path == NULL) {
2463		vp->v_path = vn_vpath_empty;
2464	}
2465
2466	/* Handles v_femhead, v_path, and the r/w/map counts */
2467	vn_recycle(vp);
2468}
2469
2470vnode_t *
2471vn_alloc(int kmflag)
2472{
2473	vnode_t *vp;
2474
2475	vp = kmem_cache_alloc(vn_cache, kmflag);
2476
2477	if (vp != NULL) {
2478		vp->v_femhead = NULL;	/* Must be done before vn_reinit() */
2479		vp->v_fopdata = NULL;
2480		vn_reinit(vp);
2481	}
2482
2483	return (vp);
2484}
2485
2486void
2487vn_free(vnode_t *vp)
2488{
2489	ASSERT(vp->v_shrlocks == NULL);
2490	ASSERT(vp->v_filocks == NULL);
2491
2492	/*
2493	 * Some file systems call vn_free() with v_count of zero,
2494	 * some with v_count of 1.  In any case, the value should
2495	 * never be anything else.
2496	 */
2497	ASSERT((vp->v_count == 0) || (vp->v_count == 1));
2498	ASSERT(vp->v_count_dnlc == 0);
2499	VERIFY(vp->v_path != NULL);
2500	if (vp->v_path != vn_vpath_empty) {
2501		kmem_free(vp->v_path, strlen(vp->v_path) + 1);
2502		vp->v_path = vn_vpath_empty;
2503	}
2504
2505	/* If FEM was in use, make sure everything gets cleaned up */
2506	if (vp->v_femhead) {
2507		/* XXX - There should be a free_femhead() that does all this */
2508		ASSERT(vp->v_femhead->femh_list == NULL);
2509		mutex_destroy(&vp->v_femhead->femh_lock);
2510		kmem_free(vp->v_femhead, sizeof (*(vp->v_femhead)));
2511		vp->v_femhead = NULL;
2512	}
2513
2514	if (vp->v_fopdata != NULL) {
2515		free_fopdata(vp);
2516	}
2517	vp->v_mpssdata = NULL;
2518	vsd_free(vp);
2519	kmem_cache_free(vn_cache, vp);
2520}
2521
2522/*
2523 * vnode status changes, should define better states than 1, 0.
2524 */
2525void
2526vn_reclaim(vnode_t *vp)
2527{
2528	vfs_t   *vfsp = vp->v_vfsp;
2529
2530	if (vfsp == NULL ||
2531	    vfsp->vfs_implp == NULL || vfsp->vfs_femhead == NULL) {
2532		return;
2533	}
2534	(void) VFS_VNSTATE(vfsp, vp, VNTRANS_RECLAIMED);
2535}
2536
2537void
2538vn_idle(vnode_t *vp)
2539{
2540	vfs_t   *vfsp = vp->v_vfsp;
2541
2542	if (vfsp == NULL ||
2543	    vfsp->vfs_implp == NULL || vfsp->vfs_femhead == NULL) {
2544		return;
2545	}
2546	(void) VFS_VNSTATE(vfsp, vp, VNTRANS_IDLED);
2547}
2548void
2549vn_exists(vnode_t *vp)
2550{
2551	vfs_t   *vfsp = vp->v_vfsp;
2552
2553	if (vfsp == NULL ||
2554	    vfsp->vfs_implp == NULL || vfsp->vfs_femhead == NULL) {
2555		return;
2556	}
2557	(void) VFS_VNSTATE(vfsp, vp, VNTRANS_EXISTS);
2558}
2559
2560void
2561vn_invalid(vnode_t *vp)
2562{
2563	vfs_t   *vfsp = vp->v_vfsp;
2564
2565	if (vfsp == NULL ||
2566	    vfsp->vfs_implp == NULL || vfsp->vfs_femhead == NULL) {
2567		return;
2568	}
2569	(void) VFS_VNSTATE(vfsp, vp, VNTRANS_DESTROYED);
2570}
2571
2572/* Vnode event notification */
2573
2574int
2575vnevent_support(vnode_t *vp, caller_context_t *ct)
2576{
2577	if (vp == NULL)
2578		return (EINVAL);
2579
2580	return (VOP_VNEVENT(vp, VE_SUPPORT, NULL, NULL, ct));
2581}
2582
2583void
2584vnevent_rename_src(vnode_t *vp, vnode_t *dvp, char *name, caller_context_t *ct)
2585{
2586	if (vp == NULL || vp->v_femhead == NULL) {
2587		return;
2588	}
2589	(void) VOP_VNEVENT(vp, VE_RENAME_SRC, dvp, name, ct);
2590}
2591
2592void
2593vnevent_rename_dest(vnode_t *vp, vnode_t *dvp, char *name,
2594    caller_context_t *ct)
2595{
2596	if (vp == NULL || vp->v_femhead == NULL) {
2597		return;
2598	}
2599	(void) VOP_VNEVENT(vp, VE_RENAME_DEST, dvp, name, ct);
2600}
2601
2602void
2603vnevent_rename_dest_dir(vnode_t *vp, caller_context_t *ct)
2604{
2605	if (vp == NULL || vp->v_femhead == NULL) {
2606		return;
2607	}
2608	(void) VOP_VNEVENT(vp, VE_RENAME_DEST_DIR, NULL, NULL, ct);
2609}
2610
2611void
2612vnevent_remove(vnode_t *vp, vnode_t *dvp, char *name, caller_context_t *ct)
2613{
2614	if (vp == NULL || vp->v_femhead == NULL) {
2615		return;
2616	}
2617	(void) VOP_VNEVENT(vp, VE_REMOVE, dvp, name, ct);
2618}
2619
2620void
2621vnevent_rmdir(vnode_t *vp, vnode_t *dvp, char *name, caller_context_t *ct)
2622{
2623	if (vp == NULL || vp->v_femhead == NULL) {
2624		return;
2625	}
2626	(void) VOP_VNEVENT(vp, VE_RMDIR, dvp, name, ct);
2627}
2628
2629void
2630vnevent_pre_rename_src(vnode_t *vp, vnode_t *dvp, char *name,
2631    caller_context_t *ct)
2632{
2633	if (vp == NULL || vp->v_femhead == NULL) {
2634		return;
2635	}
2636	(void) VOP_VNEVENT(vp, VE_PRE_RENAME_SRC, dvp, name, ct);
2637}
2638
2639void
2640vnevent_pre_rename_dest(vnode_t *vp, vnode_t *dvp, char *name,
2641    caller_context_t *ct)
2642{
2643	if (vp == NULL || vp->v_femhead == NULL) {
2644		return;
2645	}
2646	(void) VOP_VNEVENT(vp, VE_PRE_RENAME_DEST, dvp, name, ct);
2647}
2648
2649void
2650vnevent_pre_rename_dest_dir(vnode_t *vp, vnode_t *nvp, char *name,
2651    caller_context_t *ct)
2652{
2653	if (vp == NULL || vp->v_femhead == NULL) {
2654		return;
2655	}
2656	(void) VOP_VNEVENT(vp, VE_PRE_RENAME_DEST_DIR, nvp, name, ct);
2657}
2658
2659void
2660vnevent_create(vnode_t *vp, caller_context_t *ct)
2661{
2662	if (vp == NULL || vp->v_femhead == NULL) {
2663		return;
2664	}
2665	(void) VOP_VNEVENT(vp, VE_CREATE, NULL, NULL, ct);
2666}
2667
2668void
2669vnevent_link(vnode_t *vp, caller_context_t *ct)
2670{
2671	if (vp == NULL || vp->v_femhead == NULL) {
2672		return;
2673	}
2674	(void) VOP_VNEVENT(vp, VE_LINK, NULL, NULL, ct);
2675}
2676
2677void
2678vnevent_mountedover(vnode_t *vp, caller_context_t *ct)
2679{
2680	if (vp == NULL || vp->v_femhead == NULL) {
2681		return;
2682	}
2683	(void) VOP_VNEVENT(vp, VE_MOUNTEDOVER, NULL, NULL, ct);
2684}
2685
2686void
2687vnevent_truncate(vnode_t *vp, caller_context_t *ct)
2688{
2689	if (vp == NULL || vp->v_femhead == NULL) {
2690		return;
2691	}
2692	(void) VOP_VNEVENT(vp, VE_TRUNCATE, NULL, NULL, ct);
2693}
2694
2695/*
2696 * Vnode accessors.
2697 */
2698
2699int
2700vn_is_readonly(vnode_t *vp)
2701{
2702	return (vp->v_vfsp->vfs_flag & VFS_RDONLY);
2703}
2704
2705int
2706vn_has_flocks(vnode_t *vp)
2707{
2708	return (vp->v_filocks != NULL);
2709}
2710
2711int
2712vn_has_mandatory_locks(vnode_t *vp, int mode)
2713{
2714	return ((vp->v_filocks != NULL) && (MANDLOCK(vp, mode)));
2715}
2716
2717int
2718vn_has_cached_data(vnode_t *vp)
2719{
2720	return (vp->v_pages != NULL);
2721}
2722
2723/*
2724 * Return 0 if the vnode in question shouldn't be permitted into a zone via
2725 * zone_enter(2).
2726 */
2727int
2728vn_can_change_zones(vnode_t *vp)
2729{
2730	struct vfssw *vswp;
2731	int allow = 1;
2732	vnode_t *rvp;
2733
2734	if (nfs_global_client_only != 0)
2735		return (1);
2736
2737	/*
2738	 * We always want to look at the underlying vnode if there is one.
2739	 */
2740	if (VOP_REALVP(vp, &rvp, NULL) != 0)
2741		rvp = vp;
2742	/*
2743	 * Some pseudo filesystems (including doorfs) don't actually register
2744	 * their vfsops_t, so the following may return NULL; we happily let
2745	 * such vnodes switch zones.
2746	 */
2747	vswp = vfs_getvfsswbyvfsops(vfs_getops(rvp->v_vfsp));
2748	if (vswp != NULL) {
2749		if (vswp->vsw_flag & VSW_NOTZONESAFE)
2750			allow = 0;
2751		vfs_unrefvfssw(vswp);
2752	}
2753	return (allow);
2754}
2755
2756/*
2757 * Return nonzero if the vnode is a mount point, zero if not.
2758 */
2759int
2760vn_ismntpt(vnode_t *vp)
2761{
2762	return (vp->v_vfsmountedhere != NULL);
2763}
2764
2765/* Retrieve the vfs (if any) mounted on this vnode */
2766vfs_t *
2767vn_mountedvfs(vnode_t *vp)
2768{
2769	return (vp->v_vfsmountedhere);
2770}
2771
2772/*
2773 * Return nonzero if the vnode is referenced by the dnlc, zero if not.
2774 */
2775int
2776vn_in_dnlc(vnode_t *vp)
2777{
2778	return (vp->v_count_dnlc > 0);
2779}
2780
2781/*
2782 * vn_has_other_opens() checks whether a particular file is opened by more than
2783 * just the caller and whether the open is for read and/or write.
2784 * This routine is for calling after the caller has already called VOP_OPEN()
2785 * and the caller wishes to know if they are the only one with it open for
2786 * the mode(s) specified.
2787 *
2788 * Vnode counts are only kept on regular files (v_type=VREG).
2789 */
2790int
2791vn_has_other_opens(
2792	vnode_t *vp,
2793	v_mode_t mode)
2794{
2795
2796	ASSERT(vp != NULL);
2797
2798	switch (mode) {
2799	case V_WRITE:
2800		if (vp->v_wrcnt > 1)
2801			return (V_TRUE);
2802		break;
2803	case V_RDORWR:
2804		if ((vp->v_rdcnt > 1) || (vp->v_wrcnt > 1))
2805			return (V_TRUE);
2806		break;
2807	case V_RDANDWR:
2808		if ((vp->v_rdcnt > 1) && (vp->v_wrcnt > 1))
2809			return (V_TRUE);
2810		break;
2811	case V_READ:
2812		if (vp->v_rdcnt > 1)
2813			return (V_TRUE);
2814		break;
2815	}
2816
2817	return (V_FALSE);
2818}
2819
2820/*
2821 * vn_is_opened() checks whether a particular file is opened and
2822 * whether the open is for read and/or write.
2823 *
2824 * Vnode counts are only kept on regular files (v_type=VREG).
2825 */
2826int
2827vn_is_opened(
2828	vnode_t *vp,
2829	v_mode_t mode)
2830{
2831
2832	ASSERT(vp != NULL);
2833
2834	switch (mode) {
2835	case V_WRITE:
2836		if (vp->v_wrcnt)
2837			return (V_TRUE);
2838		break;
2839	case V_RDANDWR:
2840		if (vp->v_rdcnt && vp->v_wrcnt)
2841			return (V_TRUE);
2842		break;
2843	case V_RDORWR:
2844		if (vp->v_rdcnt || vp->v_wrcnt)
2845			return (V_TRUE);
2846		break;
2847	case V_READ:
2848		if (vp->v_rdcnt)
2849			return (V_TRUE);
2850		break;
2851	}
2852
2853	return (V_FALSE);
2854}
2855
2856/*
2857 * vn_is_mapped() checks whether a particular file is mapped and whether
2858 * the file is mapped read and/or write.
2859 */
2860int
2861vn_is_mapped(
2862	vnode_t *vp,
2863	v_mode_t mode)
2864{
2865
2866	ASSERT(vp != NULL);
2867
2868#if !defined(_LP64)
2869	switch (mode) {
2870	/*
2871	 * The atomic_add_64_nv functions force atomicity in the
2872	 * case of 32 bit architectures. Otherwise the 64 bit values
2873	 * require two fetches. The value of the fields may be
2874	 * (potentially) changed between the first fetch and the
2875	 * second
2876	 */
2877	case V_WRITE:
2878		if (atomic_add_64_nv((&(vp->v_mmap_write)), 0))
2879			return (V_TRUE);
2880		break;
2881	case V_RDANDWR:
2882		if ((atomic_add_64_nv((&(vp->v_mmap_read)), 0)) &&
2883		    (atomic_add_64_nv((&(vp->v_mmap_write)), 0)))
2884			return (V_TRUE);
2885		break;
2886	case V_RDORWR:
2887		if ((atomic_add_64_nv((&(vp->v_mmap_read)), 0)) ||
2888		    (atomic_add_64_nv((&(vp->v_mmap_write)), 0)))
2889			return (V_TRUE);
2890		break;
2891	case V_READ:
2892		if (atomic_add_64_nv((&(vp->v_mmap_read)), 0))
2893			return (V_TRUE);
2894		break;
2895	}
2896#else
2897	switch (mode) {
2898	case V_WRITE:
2899		if (vp->v_mmap_write)
2900			return (V_TRUE);
2901		break;
2902	case V_RDANDWR:
2903		if (vp->v_mmap_read && vp->v_mmap_write)
2904			return (V_TRUE);
2905		break;
2906	case V_RDORWR:
2907		if (vp->v_mmap_read || vp->v_mmap_write)
2908			return (V_TRUE);
2909		break;
2910	case V_READ:
2911		if (vp->v_mmap_read)
2912			return (V_TRUE);
2913		break;
2914	}
2915#endif
2916
2917	return (V_FALSE);
2918}
2919
2920/*
2921 * Set the operations vector for a vnode.
2922 *
2923 * FEM ensures that the v_femhead pointer is filled in before the
2924 * v_op pointer is changed.  This means that if the v_femhead pointer
2925 * is NULL, and the v_op field hasn't changed since before which checked
2926 * the v_femhead pointer; then our update is ok - we are not racing with
2927 * FEM.
2928 */
2929void
2930vn_setops(vnode_t *vp, vnodeops_t *vnodeops)
2931{
2932	vnodeops_t	*op;
2933
2934	ASSERT(vp != NULL);
2935	ASSERT(vnodeops != NULL);
2936
2937	op = vp->v_op;
2938	membar_consumer();
2939	/*
2940	 * If vp->v_femhead == NULL, then we'll call atomic_cas_ptr() to do
2941	 * the compare-and-swap on vp->v_op.  If either fails, then FEM is
2942	 * in effect on the vnode and we need to have FEM deal with it.
2943	 */
2944	if (vp->v_femhead != NULL || atomic_cas_ptr(&vp->v_op, op, vnodeops) !=
2945	    op) {
2946		fem_setvnops(vp, vnodeops);
2947	}
2948}
2949
2950/*
2951 * Retrieve the operations vector for a vnode
2952 * As with vn_setops(above); make sure we aren't racing with FEM.
2953 * FEM sets the v_op to a special, internal, vnodeops that wouldn't
2954 * make sense to the callers of this routine.
2955 */
2956vnodeops_t *
2957vn_getops(vnode_t *vp)
2958{
2959	vnodeops_t	*op;
2960
2961	ASSERT(vp != NULL);
2962
2963	op = vp->v_op;
2964	membar_consumer();
2965	if (vp->v_femhead == NULL && op == vp->v_op) {
2966		return (op);
2967	} else {
2968		return (fem_getvnops(vp));
2969	}
2970}
2971
2972/*
2973 * Returns non-zero (1) if the vnodeops matches that of the vnode.
2974 * Returns zero (0) if not.
2975 */
2976int
2977vn_matchops(vnode_t *vp, vnodeops_t *vnodeops)
2978{
2979	return (vn_getops(vp) == vnodeops);
2980}
2981
2982/*
2983 * Returns non-zero (1) if the specified operation matches the
2984 * corresponding operation for that the vnode.
2985 * Returns zero (0) if not.
2986 */
2987
2988#define	MATCHNAME(n1, n2) (((n1)[0] == (n2)[0]) && (strcmp((n1), (n2)) == 0))
2989
2990int
2991vn_matchopval(vnode_t *vp, char *vopname, fs_generic_func_p funcp)
2992{
2993	const fs_operation_trans_def_t *otdp;
2994	fs_generic_func_p *loc = NULL;
2995	vnodeops_t	*vop = vn_getops(vp);
2996
2997	ASSERT(vopname != NULL);
2998
2999	for (otdp = vn_ops_table; otdp->name != NULL; otdp++) {
3000		if (MATCHNAME(otdp->name, vopname)) {
3001			loc = (fs_generic_func_p *)
3002			    ((char *)(vop) + otdp->offset);
3003			break;
3004		}
3005	}
3006
3007	return ((loc != NULL) && (*loc == funcp));
3008}
3009
3010/*
3011 * fs_new_caller_id() needs to return a unique ID on a given local system.
3012 * The IDs do not need to survive across reboots.  These are primarily
3013 * used so that (FEM) monitors can detect particular callers (such as
3014 * the NFS server) to a given vnode/vfs operation.
3015 */
3016u_longlong_t
3017fs_new_caller_id()
3018{
3019	static uint64_t next_caller_id = 0LL; /* First call returns 1 */
3020
3021	return ((u_longlong_t)atomic_inc_64_nv(&next_caller_id));
3022}
3023
3024/*
3025 * The value stored in v_path is relative to rootdir, located in the global
3026 * zone.  Zones or chroot environments which reside deeper inside the VFS
3027 * hierarchy will have a relative view of MAXPATHLEN since they are unaware of
3028 * what lies below their perceived root.  In order to keep v_path usable for
3029 * these child environments, its allocations are allowed to exceed MAXPATHLEN.
3030 *
3031 * An upper bound of max_vnode_path is placed upon v_path allocations to
3032 * prevent the system from going too wild at the behest of pathological
3033 * behavior from the operator.
3034 */
3035size_t max_vnode_path = 4 * MAXPATHLEN;
3036
3037
3038void
3039vn_clearpath(vnode_t *vp, hrtime_t compare_stamp)
3040{
3041	char *buf;
3042
3043	mutex_enter(&vp->v_lock);
3044	/*
3045	 * If the snapshot of v_path_stamp passed in via compare_stamp does not
3046	 * match the present value on the vnode, it indicates that subsequent
3047	 * changes have occurred.  The v_path value is not cleared in this case
3048	 * since the new value may be valid.
3049	 */
3050	if (compare_stamp != 0 && vp->v_path_stamp != compare_stamp) {
3051		mutex_exit(&vp->v_lock);
3052		return;
3053	}
3054	buf = vp->v_path;
3055	vp->v_path = vn_vpath_empty;
3056	vp->v_path_stamp = 0;
3057	mutex_exit(&vp->v_lock);
3058	if (buf != vn_vpath_empty) {
3059		kmem_free(buf, strlen(buf) + 1);
3060	}
3061}
3062
3063static void
3064vn_setpath_common(vnode_t *pvp, vnode_t *vp, const char *name, size_t len,
3065    boolean_t is_rename)
3066{
3067	char *buf, *oldbuf;
3068	hrtime_t pstamp;
3069	size_t baselen, buflen = 0;
3070
3071	/* Handle the vn_setpath_str case. */
3072	if (pvp == NULL) {
3073		if (len + 1 > max_vnode_path) {
3074			DTRACE_PROBE4(vn__setpath__too__long, vnode_t *, pvp,
3075			    vnode_t *, vp, char *, name, size_t, len + 1);
3076			return;
3077		}
3078		buf = kmem_alloc(len + 1, KM_SLEEP);
3079		bcopy(name, buf, len);
3080		buf[len] = '\0';
3081
3082		mutex_enter(&vp->v_lock);
3083		oldbuf = vp->v_path;
3084		vp->v_path = buf;
3085		vp->v_path_stamp = gethrtime();
3086		mutex_exit(&vp->v_lock);
3087		if (oldbuf != vn_vpath_empty) {
3088			kmem_free(oldbuf, strlen(oldbuf) + 1);
3089		}
3090		return;
3091	}
3092
3093	/* Take snapshot of parent dir */
3094	mutex_enter(&pvp->v_lock);
3095
3096	if ((pvp->v_flag & VTRAVERSE) != 0) {
3097		/*
3098		 * When the parent vnode has VTRAVERSE set in its flags, normal
3099		 * assumptions about v_path calculation no longer apply.  The
3100		 * primary situation where this occurs is via the VFS tricks
3101		 * which procfs plays in order to allow /proc/PID/(root|cwd) to
3102		 * yield meaningful results.
3103		 *
3104		 * When this flag is set, v_path on the child must not be
3105		 * updated since the calculated value is likely to be
3106		 * incorrect, given the current context.
3107		 */
3108		mutex_exit(&pvp->v_lock);
3109		return;
3110	}
3111
3112retrybuf:
3113	if (pvp->v_path == vn_vpath_empty) {
3114		/*
3115		 * Without v_path from the parent directory, generating a child
3116		 * path from the name is impossible.
3117		 */
3118		if (len > 0) {
3119			pstamp = pvp->v_path_stamp;
3120			mutex_exit(&pvp->v_lock);
3121			vn_clearpath(vp, pstamp);
3122			return;
3123		}
3124
3125		/*
3126		 * The only feasible case here is where a NUL lookup is being
3127		 * performed on rootdir prior to its v_path being populated.
3128		 */
3129		ASSERT(pvp->v_path_stamp == 0);
3130		baselen = 0;
3131		pstamp = 0;
3132	} else {
3133		pstamp = pvp->v_path_stamp;
3134		baselen = strlen(pvp->v_path);
3135		/* ignore a trailing slash if present */
3136		if (pvp->v_path[baselen - 1] == '/') {
3137			/* This should only the be case for rootdir */
3138			ASSERT(baselen == 1 && pvp == rootdir);
3139			baselen--;
3140		}
3141	}
3142	mutex_exit(&pvp->v_lock);
3143
3144	if (buflen != 0) {
3145		/* Free the existing (mis-sized) buffer in case of retry */
3146		kmem_free(buf, buflen);
3147	}
3148	/* base, '/', name and trailing NUL */
3149	buflen = baselen + len + 2;
3150	if (buflen > max_vnode_path) {
3151		DTRACE_PROBE4(vn__setpath_too__long, vnode_t *, pvp,
3152		    vnode_t *, vp, char *, name, size_t, buflen);
3153		return;
3154	}
3155	buf = kmem_alloc(buflen, KM_SLEEP);
3156
3157	mutex_enter(&pvp->v_lock);
3158	if (pvp->v_path_stamp != pstamp) {
3159		size_t vlen;
3160
3161		/*
3162		 * Since v_path_stamp changed on the parent, it is likely that
3163		 * v_path has been altered as well.  If the length does not
3164		 * exactly match what was previously measured, the buffer
3165		 * allocation must be repeated for proper sizing.
3166		 */
3167		if (pvp->v_path == vn_vpath_empty) {
3168			/* Give up if parent lack v_path */
3169			mutex_exit(&pvp->v_lock);
3170			kmem_free(buf, buflen);
3171			return;
3172		}
3173		vlen = strlen(pvp->v_path);
3174		if (pvp->v_path[vlen - 1] == '/') {
3175			vlen--;
3176		}
3177		if (vlen != baselen) {
3178			goto retrybuf;
3179		}
3180	}
3181	bcopy(pvp->v_path, buf, baselen);
3182	mutex_exit(&pvp->v_lock);
3183
3184	buf[baselen] = '/';
3185	baselen++;
3186	bcopy(name, &buf[baselen], len + 1);
3187
3188	mutex_enter(&vp->v_lock);
3189	if (vp->v_path_stamp == 0) {
3190		/* never-visited vnode can inherit stamp from parent */
3191		ASSERT(vp->v_path == vn_vpath_empty);
3192		vp->v_path_stamp = pstamp;
3193		vp->v_path = buf;
3194		mutex_exit(&vp->v_lock);
3195	} else if (vp->v_path_stamp < pstamp || is_rename) {
3196		/*
3197		 * Install the updated path and stamp, ensuring that the v_path
3198		 * pointer is valid at all times for dtrace.
3199		 */
3200		oldbuf = vp->v_path;
3201		vp->v_path = buf;
3202		vp->v_path_stamp = gethrtime();
3203		mutex_exit(&vp->v_lock);
3204		kmem_free(oldbuf, strlen(oldbuf) + 1);
3205	} else {
3206		/*
3207		 * If the timestamp matches or is greater, it means another
3208		 * thread performed the update first while locks were dropped
3209		 * here to make the allocation.  We defer to the newer value.
3210		 */
3211		mutex_exit(&vp->v_lock);
3212		kmem_free(buf, buflen);
3213	}
3214	ASSERT(MUTEX_NOT_HELD(&vp->v_lock));
3215}
3216
3217void
3218vn_updatepath(vnode_t *pvp, vnode_t *vp, const char *name)
3219{
3220	size_t len;
3221
3222	/*
3223	 * If the parent is older or empty, there's nothing further to do.
3224	 */
3225	if (pvp->v_path == vn_vpath_empty ||
3226	    pvp->v_path_stamp <= vp->v_path_stamp) {
3227		return;
3228	}
3229
3230	/*
3231	 * Given the lack of appropriate context, meaningful updates to v_path
3232	 * cannot be made for during lookups for the '.' or '..' entries.
3233	 */
3234	len = strlen(name);
3235	if (len == 0 || (len == 1 && name[0] == '.') ||
3236	    (len == 2 && name[0] == '.' && name[1] == '.')) {
3237		return;
3238	}
3239
3240	vn_setpath_common(pvp, vp, name, len, B_FALSE);
3241}
3242
3243/*
3244 * Given a starting vnode and a path, updates the path in the target vnode in
3245 * a safe manner.  If the vnode already has path information embedded, then the
3246 * cached path is left untouched.
3247 */
3248/* ARGSUSED */
3249void
3250vn_setpath(vnode_t *rootvp, vnode_t *pvp, vnode_t *vp, const char *name,
3251    size_t len)
3252{
3253	vn_setpath_common(pvp, vp, name, len, B_FALSE);
3254}
3255
3256/*
3257 * Sets the path to the vnode to be the given string, regardless of current
3258 * context.  The string must be a complete path from rootdir.  This is only used
3259 * by fsop_root() for setting the path based on the mountpoint.
3260 */
3261void
3262vn_setpath_str(vnode_t *vp, const char *str, size_t len)
3263{
3264	vn_setpath_common(NULL, vp, str, len, B_FALSE);
3265}
3266
3267/*
3268 * Called from within filesystem's vop_rename() to handle renames once the
3269 * target vnode is available.
3270 */
3271void
3272vn_renamepath(vnode_t *pvp, vnode_t *vp, const char *name, size_t len)
3273{
3274	vn_setpath_common(pvp, vp, name, len, B_TRUE);
3275}
3276
3277/*
3278 * Similar to vn_setpath_str(), this function sets the path of the destination
3279 * vnode to the be the same as the source vnode.
3280 */
3281void
3282vn_copypath(struct vnode *src, struct vnode *dst)
3283{
3284	char *buf;
3285	hrtime_t stamp;
3286	size_t buflen;
3287
3288	mutex_enter(&src->v_lock);
3289	if (src->v_path == vn_vpath_empty) {
3290		mutex_exit(&src->v_lock);
3291		return;
3292	}
3293	buflen = strlen(src->v_path) + 1;
3294	mutex_exit(&src->v_lock);
3295
3296	buf = kmem_alloc(buflen, KM_SLEEP);
3297
3298	mutex_enter(&src->v_lock);
3299	if (src->v_path == vn_vpath_empty ||
3300	    strlen(src->v_path) + 1 != buflen) {
3301		mutex_exit(&src->v_lock);
3302		kmem_free(buf, buflen);
3303		return;
3304	}
3305	bcopy(src->v_path, buf, buflen);
3306	stamp = src->v_path_stamp;
3307	mutex_exit(&src->v_lock);
3308
3309	mutex_enter(&dst->v_lock);
3310	if (dst->v_path != vn_vpath_empty) {
3311		mutex_exit(&dst->v_lock);
3312		kmem_free(buf, buflen);
3313		return;
3314	}
3315	dst->v_path = buf;
3316	dst->v_path_stamp = stamp;
3317	mutex_exit(&dst->v_lock);
3318}
3319
3320
3321/*
3322 * XXX Private interface for segvn routines that handle vnode
3323 * large page segments.
3324 *
3325 * return 1 if vp's file system VOP_PAGEIO() implementation
3326 * can be safely used instead of VOP_GETPAGE() for handling
3327 * pagefaults against regular non swap files. VOP_PAGEIO()
3328 * interface is considered safe here if its implementation
3329 * is very close to VOP_GETPAGE() implementation.
3330 * e.g. It zero's out the part of the page beyond EOF. Doesn't
3331 * panic if there're file holes but instead returns an error.
3332 * Doesn't assume file won't be changed by user writes, etc.
3333 *
3334 * return 0 otherwise.
3335 *
3336 * For now allow segvn to only use VOP_PAGEIO() with ufs and nfs.
3337 */
3338int
3339vn_vmpss_usepageio(vnode_t *vp)
3340{
3341	vfs_t   *vfsp = vp->v_vfsp;
3342	char *fsname = vfssw[vfsp->vfs_fstype].vsw_name;
3343	char *pageio_ok_fss[] = {"ufs", "nfs", NULL};
3344	char **fsok = pageio_ok_fss;
3345
3346	if (fsname == NULL) {
3347		return (0);
3348	}
3349
3350	for (; *fsok; fsok++) {
3351		if (strcmp(*fsok, fsname) == 0) {
3352			return (1);
3353		}
3354	}
3355	return (0);
3356}
3357
3358/* VOP_XXX() macros call the corresponding fop_xxx() function */
3359
3360int
3361fop_open(
3362	vnode_t **vpp,
3363	int mode,
3364	cred_t *cr,
3365	caller_context_t *ct)
3366{
3367	int ret;
3368	vnode_t *vp = *vpp;
3369
3370	VN_HOLD(vp);
3371	/*
3372	 * Adding to the vnode counts before calling open
3373	 * avoids the need for a mutex. It circumvents a race
3374	 * condition where a query made on the vnode counts results in a
3375	 * false negative. The inquirer goes away believing the file is
3376	 * not open when there is an open on the file already under way.
3377	 *
3378	 * The counts are meant to prevent NFS from granting a delegation
3379	 * when it would be dangerous to do so.
3380	 *
3381	 * The vnode counts are only kept on regular files
3382	 */
3383	if ((*vpp)->v_type == VREG) {
3384		if (mode & FREAD)
3385			atomic_inc_32(&(*vpp)->v_rdcnt);
3386		if (mode & FWRITE)
3387			atomic_inc_32(&(*vpp)->v_wrcnt);
3388	}
3389
3390	VOPXID_MAP_CR(vp, cr);
3391
3392	ret = (*(*(vpp))->v_op->vop_open)(vpp, mode, cr, ct);
3393
3394	if (ret) {
3395		/*
3396		 * Use the saved vp just in case the vnode ptr got trashed
3397		 * by the error.
3398		 */
3399		VOPSTATS_UPDATE(vp, open);
3400		if ((vp->v_type == VREG) && (mode & FREAD))
3401			atomic_dec_32(&vp->v_rdcnt);
3402		if ((vp->v_type == VREG) && (mode & FWRITE))
3403			atomic_dec_32(&vp->v_wrcnt);
3404	} else {
3405		/*
3406		 * Some filesystems will return a different vnode,
3407		 * but the same path was still used to open it.
3408		 * So if we do change the vnode and need to
3409		 * copy over the path, do so here, rather than special
3410		 * casing each filesystem. Adjust the vnode counts to
3411		 * reflect the vnode switch.
3412		 */
3413		VOPSTATS_UPDATE(*vpp, open);
3414		if (*vpp != vp) {
3415			vn_copypath(vp, *vpp);
3416			if (((*vpp)->v_type == VREG) && (mode & FREAD))
3417				atomic_inc_32(&(*vpp)->v_rdcnt);
3418			if ((vp->v_type == VREG) && (mode & FREAD))
3419				atomic_dec_32(&vp->v_rdcnt);
3420			if (((*vpp)->v_type == VREG) && (mode & FWRITE))
3421				atomic_inc_32(&(*vpp)->v_wrcnt);
3422			if ((vp->v_type == VREG) && (mode & FWRITE))
3423				atomic_dec_32(&vp->v_wrcnt);
3424		}
3425	}
3426	VN_RELE(vp);
3427	return (ret);
3428}
3429
3430int
3431fop_close(
3432	vnode_t *vp,
3433	int flag,
3434	int count,
3435	offset_t offset,
3436	cred_t *cr,
3437	caller_context_t *ct)
3438{
3439	int err;
3440
3441	VOPXID_MAP_CR(vp, cr);
3442
3443	err = (*(vp)->v_op->vop_close)(vp, flag, count, offset, cr, ct);
3444	VOPSTATS_UPDATE(vp, close);
3445	/*
3446	 * Check passed in count to handle possible dups. Vnode counts are only
3447	 * kept on regular files
3448	 */
3449	if ((vp->v_type == VREG) && (count == 1))  {
3450		if (flag & FREAD) {
3451			ASSERT(vp->v_rdcnt > 0);
3452			atomic_dec_32(&vp->v_rdcnt);
3453		}
3454		if (flag & FWRITE) {
3455			ASSERT(vp->v_wrcnt > 0);
3456			atomic_dec_32(&vp->v_wrcnt);
3457		}
3458	}
3459	return (err);
3460}
3461
3462int
3463fop_read(
3464	vnode_t *vp,
3465	uio_t *uiop,
3466	int ioflag,
3467	cred_t *cr,
3468	caller_context_t *ct)
3469{
3470	int	err;
3471	ssize_t	resid_start = uiop->uio_resid;
3472
3473	VOPXID_MAP_CR(vp, cr);
3474
3475	err = (*(vp)->v_op->vop_read)(vp, uiop, ioflag, cr, ct);
3476	VOPSTATS_UPDATE_IO(vp, read,
3477	    read_bytes, (resid_start - uiop->uio_resid));
3478	return (err);
3479}
3480
3481int
3482fop_write(
3483	vnode_t *vp,
3484	uio_t *uiop,
3485	int ioflag,
3486	cred_t *cr,
3487	caller_context_t *ct)
3488{
3489	int	err;
3490	ssize_t	resid_start = uiop->uio_resid;
3491
3492	VOPXID_MAP_CR(vp, cr);
3493
3494	err = (*(vp)->v_op->vop_write)(vp, uiop, ioflag, cr, ct);
3495	VOPSTATS_UPDATE_IO(vp, write,
3496	    write_bytes, (resid_start - uiop->uio_resid));
3497	return (err);
3498}
3499
3500int
3501fop_ioctl(
3502	vnode_t *vp,
3503	int cmd,
3504	intptr_t arg,
3505	int flag,
3506	cred_t *cr,
3507	int *rvalp,
3508	caller_context_t *ct)
3509{
3510	int	err;
3511
3512	VOPXID_MAP_CR(vp, cr);
3513
3514	err = (*(vp)->v_op->vop_ioctl)(vp, cmd, arg, flag, cr, rvalp, ct);
3515	VOPSTATS_UPDATE(vp, ioctl);
3516	return (err);
3517}
3518
3519int
3520fop_setfl(
3521	vnode_t *vp,
3522	int oflags,
3523	int nflags,
3524	cred_t *cr,
3525	caller_context_t *ct)
3526{
3527	int	err;
3528
3529	VOPXID_MAP_CR(vp, cr);
3530
3531	err = (*(vp)->v_op->vop_setfl)(vp, oflags, nflags, cr, ct);
3532	VOPSTATS_UPDATE(vp, setfl);
3533	return (err);
3534}
3535
3536int
3537fop_getattr(
3538	vnode_t *vp,
3539	vattr_t *vap,
3540	int flags,
3541	cred_t *cr,
3542	caller_context_t *ct)
3543{
3544	int	err;
3545
3546	VOPXID_MAP_CR(vp, cr);
3547
3548	/*
3549	 * If this file system doesn't understand the xvattr extensions
3550	 * then turn off the xvattr bit.
3551	 */
3552	if (vfs_has_feature(vp->v_vfsp, VFSFT_XVATTR) == 0) {
3553		vap->va_mask &= ~AT_XVATTR;
3554	}
3555
3556	/*
3557	 * We're only allowed to skip the ACL check iff we used a 32 bit
3558	 * ACE mask with VOP_ACCESS() to determine permissions.
3559	 */
3560	if ((flags & ATTR_NOACLCHECK) &&
3561	    vfs_has_feature(vp->v_vfsp, VFSFT_ACEMASKONACCESS) == 0) {
3562		return (EINVAL);
3563	}
3564	err = (*(vp)->v_op->vop_getattr)(vp, vap, flags, cr, ct);
3565	VOPSTATS_UPDATE(vp, getattr);
3566	return (err);
3567}
3568
3569int
3570fop_setattr(
3571	vnode_t *vp,
3572	vattr_t *vap,
3573	int flags,
3574	cred_t *cr,
3575	caller_context_t *ct)
3576{
3577	int	err;
3578
3579	VOPXID_MAP_CR(vp, cr);
3580
3581	/*
3582	 * If this file system doesn't understand the xvattr extensions
3583	 * then turn off the xvattr bit.
3584	 */
3585	if (vfs_has_feature(vp->v_vfsp, VFSFT_XVATTR) == 0) {
3586		vap->va_mask &= ~AT_XVATTR;
3587	}
3588
3589	/*
3590	 * We're only allowed to skip the ACL check iff we used a 32 bit
3591	 * ACE mask with VOP_ACCESS() to determine permissions.
3592	 */
3593	if ((flags & ATTR_NOACLCHECK) &&
3594	    vfs_has_feature(vp->v_vfsp, VFSFT_ACEMASKONACCESS) == 0) {
3595		return (EINVAL);
3596	}
3597	err = (*(vp)->v_op->vop_setattr)(vp, vap, flags, cr, ct);
3598	VOPSTATS_UPDATE(vp, setattr);
3599	return (err);
3600}
3601
3602int
3603fop_access(
3604	vnode_t *vp,
3605	int mode,
3606	int flags,
3607	cred_t *cr,
3608	caller_context_t *ct)
3609{
3610	int	err;
3611
3612	if ((flags & V_ACE_MASK) &&
3613	    vfs_has_feature(vp->v_vfsp, VFSFT_ACEMASKONACCESS) == 0) {
3614		return (EINVAL);
3615	}
3616
3617	VOPXID_MAP_CR(vp, cr);
3618
3619	err = (*(vp)->v_op->vop_access)(vp, mode, flags, cr, ct);
3620	VOPSTATS_UPDATE(vp, access);
3621	return (err);
3622}
3623
3624int
3625fop_lookup(
3626	vnode_t *dvp,
3627	char *nm,
3628	vnode_t **vpp,
3629	pathname_t *pnp,
3630	int flags,
3631	vnode_t *rdir,
3632	cred_t *cr,
3633	caller_context_t *ct,
3634	int *deflags,		/* Returned per-dirent flags */
3635	pathname_t *ppnp)	/* Returned case-preserved name in directory */
3636{
3637	int ret;
3638
3639	/*
3640	 * If this file system doesn't support case-insensitive access
3641	 * and said access is requested, fail quickly.  It is required
3642	 * that if the vfs supports case-insensitive lookup, it also
3643	 * supports extended dirent flags.
3644	 */
3645	if (flags & FIGNORECASE &&
3646	    (vfs_has_feature(dvp->v_vfsp, VFSFT_CASEINSENSITIVE) == 0 &&
3647	    vfs_has_feature(dvp->v_vfsp, VFSFT_NOCASESENSITIVE) == 0))
3648		return (EINVAL);
3649
3650	VOPXID_MAP_CR(dvp, cr);
3651
3652	if ((flags & LOOKUP_XATTR) && (flags & LOOKUP_HAVE_SYSATTR_DIR) == 0) {
3653		ret = xattr_dir_lookup(dvp, vpp, flags, cr);
3654	} else {
3655		ret = (*(dvp)->v_op->vop_lookup)
3656		    (dvp, nm, vpp, pnp, flags, rdir, cr, ct, deflags, ppnp);
3657	}
3658	if (ret == 0 && *vpp) {
3659		VOPSTATS_UPDATE(*vpp, lookup);
3660		vn_updatepath(dvp, *vpp, nm);
3661	}
3662
3663	return (ret);
3664}
3665
3666int
3667fop_create(
3668	vnode_t *dvp,
3669	char *name,
3670	vattr_t *vap,
3671	vcexcl_t excl,
3672	int mode,
3673	vnode_t **vpp,
3674	cred_t *cr,
3675	int flags,
3676	caller_context_t *ct,
3677	vsecattr_t *vsecp)	/* ACL to set during create */
3678{
3679	int ret;
3680
3681	if (vsecp != NULL &&
3682	    vfs_has_feature(dvp->v_vfsp, VFSFT_ACLONCREATE) == 0) {
3683		return (EINVAL);
3684	}
3685	/*
3686	 * If this file system doesn't support case-insensitive access
3687	 * and said access is requested, fail quickly.
3688	 */
3689	if (flags & FIGNORECASE &&
3690	    (vfs_has_feature(dvp->v_vfsp, VFSFT_CASEINSENSITIVE) == 0 &&
3691	    vfs_has_feature(dvp->v_vfsp, VFSFT_NOCASESENSITIVE) == 0))
3692		return (EINVAL);
3693
3694	VOPXID_MAP_CR(dvp, cr);
3695
3696	ret = (*(dvp)->v_op->vop_create)
3697	    (dvp, name, vap, excl, mode, vpp, cr, flags, ct, vsecp);
3698	if (ret == 0 && *vpp) {
3699		VOPSTATS_UPDATE(*vpp, create);
3700		vn_updatepath(dvp, *vpp, name);
3701	}
3702
3703	return (ret);
3704}
3705
3706int
3707fop_remove(
3708	vnode_t *dvp,
3709	char *nm,
3710	cred_t *cr,
3711	caller_context_t *ct,
3712	int flags)
3713{
3714	int	err;
3715
3716	/*
3717	 * If this file system doesn't support case-insensitive access
3718	 * and said access is requested, fail quickly.
3719	 */
3720	if (flags & FIGNORECASE &&
3721	    (vfs_has_feature(dvp->v_vfsp, VFSFT_CASEINSENSITIVE) == 0 &&
3722	    vfs_has_feature(dvp->v_vfsp, VFSFT_NOCASESENSITIVE) == 0))
3723		return (EINVAL);
3724
3725	VOPXID_MAP_CR(dvp, cr);
3726
3727	err = (*(dvp)->v_op->vop_remove)(dvp, nm, cr, ct, flags);
3728	VOPSTATS_UPDATE(dvp, remove);
3729	return (err);
3730}
3731
3732int
3733fop_link(
3734	vnode_t *tdvp,
3735	vnode_t *svp,
3736	char *tnm,
3737	cred_t *cr,
3738	caller_context_t *ct,
3739	int flags)
3740{
3741	int	err;
3742
3743	/*
3744	 * If the target file system doesn't support case-insensitive access
3745	 * and said access is requested, fail quickly.
3746	 */
3747	if (flags & FIGNORECASE &&
3748	    (vfs_has_feature(tdvp->v_vfsp, VFSFT_CASEINSENSITIVE) == 0 &&
3749	    vfs_has_feature(tdvp->v_vfsp, VFSFT_NOCASESENSITIVE) == 0))
3750		return (EINVAL);
3751
3752	VOPXID_MAP_CR(tdvp, cr);
3753
3754	err = (*(tdvp)->v_op->vop_link)(tdvp, svp, tnm, cr, ct, flags);
3755	VOPSTATS_UPDATE(tdvp, link);
3756	return (err);
3757}
3758
3759int
3760fop_rename(
3761	vnode_t *sdvp,
3762	char *snm,
3763	vnode_t *tdvp,
3764	char *tnm,
3765	cred_t *cr,
3766	caller_context_t *ct,
3767	int flags)
3768{
3769	int	err;
3770
3771	/*
3772	 * If the file system involved does not support
3773	 * case-insensitive access and said access is requested, fail
3774	 * quickly.
3775	 */
3776	if (flags & FIGNORECASE &&
3777	    ((vfs_has_feature(sdvp->v_vfsp, VFSFT_CASEINSENSITIVE) == 0 &&
3778	    vfs_has_feature(sdvp->v_vfsp, VFSFT_NOCASESENSITIVE) == 0)))
3779		return (EINVAL);
3780
3781	VOPXID_MAP_CR(tdvp, cr);
3782
3783	err = (*(sdvp)->v_op->vop_rename)(sdvp, snm, tdvp, tnm, cr, ct, flags);
3784	VOPSTATS_UPDATE(sdvp, rename);
3785	return (err);
3786}
3787
3788int
3789fop_mkdir(
3790	vnode_t *dvp,
3791	char *dirname,
3792	vattr_t *vap,
3793	vnode_t **vpp,
3794	cred_t *cr,
3795	caller_context_t *ct,
3796	int flags,
3797	vsecattr_t *vsecp)	/* ACL to set during create */
3798{
3799	int ret;
3800
3801	if (vsecp != NULL &&
3802	    vfs_has_feature(dvp->v_vfsp, VFSFT_ACLONCREATE) == 0) {
3803		return (EINVAL);
3804	}
3805	/*
3806	 * If this file system doesn't support case-insensitive access
3807	 * and said access is requested, fail quickly.
3808	 */
3809	if (flags & FIGNORECASE &&
3810	    (vfs_has_feature(dvp->v_vfsp, VFSFT_CASEINSENSITIVE) == 0 &&
3811	    vfs_has_feature(dvp->v_vfsp, VFSFT_NOCASESENSITIVE) == 0))
3812		return (EINVAL);
3813
3814	VOPXID_MAP_CR(dvp, cr);
3815
3816	ret = (*(dvp)->v_op->vop_mkdir)
3817	    (dvp, dirname, vap, vpp, cr, ct, flags, vsecp);
3818	if (ret == 0 && *vpp) {
3819		VOPSTATS_UPDATE(*vpp, mkdir);
3820		vn_updatepath(dvp, *vpp, dirname);
3821	}
3822
3823	return (ret);
3824}
3825
3826int
3827fop_rmdir(
3828	vnode_t *dvp,
3829	char *nm,
3830	vnode_t *cdir,
3831	cred_t *cr,
3832	caller_context_t *ct,
3833	int flags)
3834{
3835	int	err;
3836
3837	/*
3838	 * If this file system doesn't support case-insensitive access
3839	 * and said access is requested, fail quickly.
3840	 */
3841	if (flags & FIGNORECASE &&
3842	    (vfs_has_feature(dvp->v_vfsp, VFSFT_CASEINSENSITIVE) == 0 &&
3843	    vfs_has_feature(dvp->v_vfsp, VFSFT_NOCASESENSITIVE) == 0))
3844		return (EINVAL);
3845
3846	VOPXID_MAP_CR(dvp, cr);
3847
3848	err = (*(dvp)->v_op->vop_rmdir)(dvp, nm, cdir, cr, ct, flags);
3849	VOPSTATS_UPDATE(dvp, rmdir);
3850	return (err);
3851}
3852
3853int
3854fop_readdir(
3855	vnode_t *vp,
3856	uio_t *uiop,
3857	cred_t *cr,
3858	int *eofp,
3859	caller_context_t *ct,
3860	int flags)
3861{
3862	int	err;
3863	ssize_t	resid_start = uiop->uio_resid;
3864
3865	/*
3866	 * If this file system doesn't support retrieving directory
3867	 * entry flags and said access is requested, fail quickly.
3868	 */
3869	if (flags & V_RDDIR_ENTFLAGS &&
3870	    vfs_has_feature(vp->v_vfsp, VFSFT_DIRENTFLAGS) == 0)
3871		return (EINVAL);
3872
3873	VOPXID_MAP_CR(vp, cr);
3874
3875	err = (*(vp)->v_op->vop_readdir)(vp, uiop, cr, eofp, ct, flags);
3876	VOPSTATS_UPDATE_IO(vp, readdir,
3877	    readdir_bytes, (resid_start - uiop->uio_resid));
3878	return (err);
3879}
3880
3881int
3882fop_symlink(
3883	vnode_t *dvp,
3884	char *linkname,
3885	vattr_t *vap,
3886	char *target,
3887	cred_t *cr,
3888	caller_context_t *ct,
3889	int flags)
3890{
3891	int	err;
3892	xvattr_t xvattr;
3893
3894	/*
3895	 * If this file system doesn't support case-insensitive access
3896	 * and said access is requested, fail quickly.
3897	 */
3898	if (flags & FIGNORECASE &&
3899	    (vfs_has_feature(dvp->v_vfsp, VFSFT_CASEINSENSITIVE) == 0 &&
3900	    vfs_has_feature(dvp->v_vfsp, VFSFT_NOCASESENSITIVE) == 0))
3901		return (EINVAL);
3902
3903	VOPXID_MAP_CR(dvp, cr);
3904
3905	/* check for reparse point */
3906	if ((vfs_has_feature(dvp->v_vfsp, VFSFT_REPARSE)) &&
3907	    (strncmp(target, FS_REPARSE_TAG_STR,
3908	    strlen(FS_REPARSE_TAG_STR)) == 0)) {
3909		if (!fs_reparse_mark(target, vap, &xvattr))
3910			vap = (vattr_t *)&xvattr;
3911	}
3912
3913	err = (*(dvp)->v_op->vop_symlink)
3914	    (dvp, linkname, vap, target, cr, ct, flags);
3915	VOPSTATS_UPDATE(dvp, symlink);
3916	return (err);
3917}
3918
3919int
3920fop_readlink(
3921	vnode_t *vp,
3922	uio_t *uiop,
3923	cred_t *cr,
3924	caller_context_t *ct)
3925{
3926	int	err;
3927
3928	VOPXID_MAP_CR(vp, cr);
3929
3930	err = (*(vp)->v_op->vop_readlink)(vp, uiop, cr, ct);
3931	VOPSTATS_UPDATE(vp, readlink);
3932	return (err);
3933}
3934
3935int
3936fop_fsync(
3937	vnode_t *vp,
3938	int syncflag,
3939	cred_t *cr,
3940	caller_context_t *ct)
3941{
3942	int	err;
3943
3944	VOPXID_MAP_CR(vp, cr);
3945
3946	err = (*(vp)->v_op->vop_fsync)(vp, syncflag, cr, ct);
3947	VOPSTATS_UPDATE(vp, fsync);
3948	return (err);
3949}
3950
3951void
3952fop_inactive(
3953	vnode_t *vp,
3954	cred_t *cr,
3955	caller_context_t *ct)
3956{
3957	/* Need to update stats before vop call since we may lose the vnode */
3958	VOPSTATS_UPDATE(vp, inactive);
3959
3960	VOPXID_MAP_CR(vp, cr);
3961
3962	(*(vp)->v_op->vop_inactive)(vp, cr, ct);
3963}
3964
3965int
3966fop_fid(
3967	vnode_t *vp,
3968	fid_t *fidp,
3969	caller_context_t *ct)
3970{
3971	int	err;
3972
3973	err = (*(vp)->v_op->vop_fid)(vp, fidp, ct);
3974	VOPSTATS_UPDATE(vp, fid);
3975	return (err);
3976}
3977
3978int
3979fop_rwlock(
3980	vnode_t *vp,
3981	int write_lock,
3982	caller_context_t *ct)
3983{
3984	int	ret;
3985
3986	ret = ((*(vp)->v_op->vop_rwlock)(vp, write_lock, ct));
3987	VOPSTATS_UPDATE(vp, rwlock);
3988	return (ret);
3989}
3990
3991void
3992fop_rwunlock(
3993	vnode_t *vp,
3994	int write_lock,
3995	caller_context_t *ct)
3996{
3997	(*(vp)->v_op->vop_rwunlock)(vp, write_lock, ct);
3998	VOPSTATS_UPDATE(vp, rwunlock);
3999}
4000
4001int
4002fop_seek(
4003	vnode_t *vp,
4004	offset_t ooff,
4005	offset_t *noffp,
4006	caller_context_t *ct)
4007{
4008	int	err;
4009
4010	err = (*(vp)->v_op->vop_seek)(vp, ooff, noffp, ct);
4011	VOPSTATS_UPDATE(vp, seek);
4012	return (err);
4013}
4014
4015int
4016fop_cmp(
4017	vnode_t *vp1,
4018	vnode_t *vp2,
4019	caller_context_t *ct)
4020{
4021	int	err;
4022
4023	err = (*(vp1)->v_op->vop_cmp)(vp1, vp2, ct);
4024	VOPSTATS_UPDATE(vp1, cmp);
4025	return (err);
4026}
4027
4028int
4029fop_frlock(
4030	vnode_t *vp,
4031	int cmd,
4032	flock64_t *bfp,
4033	int flag,
4034	offset_t offset,
4035	struct flk_callback *flk_cbp,
4036	cred_t *cr,
4037	caller_context_t *ct)
4038{
4039	int	err;
4040
4041	VOPXID_MAP_CR(vp, cr);
4042
4043	err = (*(vp)->v_op->vop_frlock)
4044	    (vp, cmd, bfp, flag, offset, flk_cbp, cr, ct);
4045	VOPSTATS_UPDATE(vp, frlock);
4046	return (err);
4047}
4048
4049int
4050fop_space(
4051	vnode_t *vp,
4052	int cmd,
4053	flock64_t *bfp,
4054	int flag,
4055	offset_t offset,
4056	cred_t *cr,
4057	caller_context_t *ct)
4058{
4059	int	err;
4060
4061	VOPXID_MAP_CR(vp, cr);
4062
4063	err = (*(vp)->v_op->vop_space)(vp, cmd, bfp, flag, offset, cr, ct);
4064	VOPSTATS_UPDATE(vp, space);
4065	return (err);
4066}
4067
4068int
4069fop_realvp(
4070	vnode_t *vp,
4071	vnode_t **vpp,
4072	caller_context_t *ct)
4073{
4074	int	err;
4075
4076	err = (*(vp)->v_op->vop_realvp)(vp, vpp, ct);
4077	VOPSTATS_UPDATE(vp, realvp);
4078	return (err);
4079}
4080
4081int
4082fop_getpage(
4083	vnode_t *vp,
4084	offset_t off,
4085	size_t len,
4086	uint_t *protp,
4087	page_t **plarr,
4088	size_t plsz,
4089	struct seg *seg,
4090	caddr_t addr,
4091	enum seg_rw rw,
4092	cred_t *cr,
4093	caller_context_t *ct)
4094{
4095	int	err;
4096
4097	VOPXID_MAP_CR(vp, cr);
4098
4099	err = (*(vp)->v_op->vop_getpage)
4100	    (vp, off, len, protp, plarr, plsz, seg, addr, rw, cr, ct);
4101	VOPSTATS_UPDATE(vp, getpage);
4102	return (err);
4103}
4104
4105int
4106fop_putpage(
4107	vnode_t *vp,
4108	offset_t off,
4109	size_t len,
4110	int flags,
4111	cred_t *cr,
4112	caller_context_t *ct)
4113{
4114	int	err;
4115
4116	VOPXID_MAP_CR(vp, cr);
4117
4118	err = (*(vp)->v_op->vop_putpage)(vp, off, len, flags, cr, ct);
4119	VOPSTATS_UPDATE(vp, putpage);
4120	return (err);
4121}
4122
4123int
4124fop_map(
4125	vnode_t *vp,
4126	offset_t off,
4127	struct as *as,
4128	caddr_t *addrp,
4129	size_t len,
4130	uchar_t prot,
4131	uchar_t maxprot,
4132	uint_t flags,
4133	cred_t *cr,
4134	caller_context_t *ct)
4135{
4136	int	err;
4137
4138	VOPXID_MAP_CR(vp, cr);
4139
4140	err = (*(vp)->v_op->vop_map)
4141	    (vp, off, as, addrp, len, prot, maxprot, flags, cr, ct);
4142	VOPSTATS_UPDATE(vp, map);
4143	return (err);
4144}
4145
4146int
4147fop_addmap(
4148	vnode_t *vp,
4149	offset_t off,
4150	struct as *as,
4151	caddr_t addr,
4152	size_t len,
4153	uchar_t prot,
4154	uchar_t maxprot,
4155	uint_t flags,
4156	cred_t *cr,
4157	caller_context_t *ct)
4158{
4159	int error;
4160	u_longlong_t delta;
4161
4162	VOPXID_MAP_CR(vp, cr);
4163
4164	error = (*(vp)->v_op->vop_addmap)
4165	    (vp, off, as, addr, len, prot, maxprot, flags, cr, ct);
4166
4167	if ((!error) && (vp->v_type == VREG)) {
4168		delta = (u_longlong_t)btopr(len);
4169		/*
4170		 * If file is declared MAP_PRIVATE, it can't be written back
4171		 * even if open for write. Handle as read.
4172		 */
4173		if (flags & MAP_PRIVATE) {
4174			atomic_add_64((uint64_t *)(&(vp->v_mmap_read)),
4175			    (int64_t)delta);
4176		} else {
4177			/*
4178			 * atomic_add_64 forces the fetch of a 64 bit value to
4179			 * be atomic on 32 bit machines
4180			 */
4181			if (maxprot & PROT_WRITE)
4182				atomic_add_64((uint64_t *)(&(vp->v_mmap_write)),
4183				    (int64_t)delta);
4184			if (maxprot & PROT_READ)
4185				atomic_add_64((uint64_t *)(&(vp->v_mmap_read)),
4186				    (int64_t)delta);
4187			if (maxprot & PROT_EXEC)
4188				atomic_add_64((uint64_t *)(&(vp->v_mmap_read)),
4189				    (int64_t)delta);
4190		}
4191	}
4192	VOPSTATS_UPDATE(vp, addmap);
4193	return (error);
4194}
4195
4196int
4197fop_delmap(
4198	vnode_t *vp,
4199	offset_t off,
4200	struct as *as,
4201	caddr_t addr,
4202	size_t len,
4203	uint_t prot,
4204	uint_t maxprot,
4205	uint_t flags,
4206	cred_t *cr,
4207	caller_context_t *ct)
4208{
4209	int error;
4210	u_longlong_t delta;
4211
4212	VOPXID_MAP_CR(vp, cr);
4213
4214	error = (*(vp)->v_op->vop_delmap)
4215	    (vp, off, as, addr, len, prot, maxprot, flags, cr, ct);
4216
4217	/*
4218	 * NFS calls into delmap twice, the first time
4219	 * it simply establishes a callback mechanism and returns EAGAIN
4220	 * while the real work is being done upon the second invocation.
4221	 * We have to detect this here and only decrement the counts upon
4222	 * the second delmap request.
4223	 */
4224	if ((error != EAGAIN) && (vp->v_type == VREG)) {
4225
4226		delta = (u_longlong_t)btopr(len);
4227
4228		if (flags & MAP_PRIVATE) {
4229			atomic_add_64((uint64_t *)(&(vp->v_mmap_read)),
4230			    (int64_t)(-delta));
4231		} else {
4232			/*
4233			 * atomic_add_64 forces the fetch of a 64 bit value
4234			 * to be atomic on 32 bit machines
4235			 */
4236			if (maxprot & PROT_WRITE)
4237				atomic_add_64((uint64_t *)(&(vp->v_mmap_write)),
4238				    (int64_t)(-delta));
4239			if (maxprot & PROT_READ)
4240				atomic_add_64((uint64_t *)(&(vp->v_mmap_read)),
4241				    (int64_t)(-delta));
4242			if (maxprot & PROT_EXEC)
4243				atomic_add_64((uint64_t *)(&(vp->v_mmap_read)),
4244				    (int64_t)(-delta));
4245		}
4246	}
4247	VOPSTATS_UPDATE(vp, delmap);
4248	return (error);
4249}
4250
4251
4252int
4253fop_poll(
4254	vnode_t *vp,
4255	short events,
4256	int anyyet,
4257	short *reventsp,
4258	struct pollhead **phpp,
4259	caller_context_t *ct)
4260{
4261	int	err;
4262
4263	err = (*(vp)->v_op->vop_poll)(vp, events, anyyet, reventsp, phpp, ct);
4264	VOPSTATS_UPDATE(vp, poll);
4265	return (err);
4266}
4267
4268int
4269fop_dump(
4270	vnode_t *vp,
4271	caddr_t addr,
4272	offset_t lbdn,
4273	offset_t dblks,
4274	caller_context_t *ct)
4275{
4276	int	err;
4277
4278	/* ensure lbdn and dblks can be passed safely to bdev_dump */
4279	if ((lbdn != (daddr_t)lbdn) || (dblks != (int)dblks))
4280		return (EIO);
4281
4282	err = (*(vp)->v_op->vop_dump)(vp, addr, lbdn, dblks, ct);
4283	VOPSTATS_UPDATE(vp, dump);
4284	return (err);
4285}
4286
4287int
4288fop_pathconf(
4289	vnode_t *vp,
4290	int cmd,
4291	ulong_t *valp,
4292	cred_t *cr,
4293	caller_context_t *ct)
4294{
4295	int	err;
4296
4297	VOPXID_MAP_CR(vp, cr);
4298
4299	err = (*(vp)->v_op->vop_pathconf)(vp, cmd, valp, cr, ct);
4300	VOPSTATS_UPDATE(vp, pathconf);
4301	return (err);
4302}
4303
4304int
4305fop_pageio(
4306	vnode_t *vp,
4307	struct page *pp,
4308	u_offset_t io_off,
4309	size_t io_len,
4310	int flags,
4311	cred_t *cr,
4312	caller_context_t *ct)
4313{
4314	int	err;
4315
4316	VOPXID_MAP_CR(vp, cr);
4317
4318	err = (*(vp)->v_op->vop_pageio)(vp, pp, io_off, io_len, flags, cr, ct);
4319	VOPSTATS_UPDATE(vp, pageio);
4320	return (err);
4321}
4322
4323int
4324fop_dumpctl(
4325	vnode_t *vp,
4326	int action,
4327	offset_t *blkp,
4328	caller_context_t *ct)
4329{
4330	int	err;
4331	err = (*(vp)->v_op->vop_dumpctl)(vp, action, blkp, ct);
4332	VOPSTATS_UPDATE(vp, dumpctl);
4333	return (err);
4334}
4335
4336void
4337fop_dispose(
4338	vnode_t *vp,
4339	page_t *pp,
4340	int flag,
4341	int dn,
4342	cred_t *cr,
4343	caller_context_t *ct)
4344{
4345	/* Must do stats first since it's possible to lose the vnode */
4346	VOPSTATS_UPDATE(vp, dispose);
4347
4348	VOPXID_MAP_CR(vp, cr);
4349
4350	(*(vp)->v_op->vop_dispose)(vp, pp, flag, dn, cr, ct);
4351}
4352
4353int
4354fop_setsecattr(
4355	vnode_t *vp,
4356	vsecattr_t *vsap,
4357	int flag,
4358	cred_t *cr,
4359	caller_context_t *ct)
4360{
4361	int	err;
4362
4363	VOPXID_MAP_CR(vp, cr);
4364
4365	/*
4366	 * We're only allowed to skip the ACL check iff we used a 32 bit
4367	 * ACE mask with VOP_ACCESS() to determine permissions.
4368	 */
4369	if ((flag & ATTR_NOACLCHECK) &&
4370	    vfs_has_feature(vp->v_vfsp, VFSFT_ACEMASKONACCESS) == 0) {
4371		return (EINVAL);
4372	}
4373	err = (*(vp)->v_op->vop_setsecattr) (vp, vsap, flag, cr, ct);
4374	VOPSTATS_UPDATE(vp, setsecattr);
4375	return (err);
4376}
4377
4378int
4379fop_getsecattr(
4380	vnode_t *vp,
4381	vsecattr_t *vsap,
4382	int flag,
4383	cred_t *cr,
4384	caller_context_t *ct)
4385{
4386	int	err;
4387
4388	/*
4389	 * We're only allowed to skip the ACL check iff we used a 32 bit
4390	 * ACE mask with VOP_ACCESS() to determine permissions.
4391	 */
4392	if ((flag & ATTR_NOACLCHECK) &&
4393	    vfs_has_feature(vp->v_vfsp, VFSFT_ACEMASKONACCESS) == 0) {
4394		return (EINVAL);
4395	}
4396
4397	VOPXID_MAP_CR(vp, cr);
4398
4399	err = (*(vp)->v_op->vop_getsecattr) (vp, vsap, flag, cr, ct);
4400	VOPSTATS_UPDATE(vp, getsecattr);
4401	return (err);
4402}
4403
4404int
4405fop_shrlock(
4406	vnode_t *vp,
4407	int cmd,
4408	struct shrlock *shr,
4409	int flag,
4410	cred_t *cr,
4411	caller_context_t *ct)
4412{
4413	int	err;
4414
4415	VOPXID_MAP_CR(vp, cr);
4416
4417	err = (*(vp)->v_op->vop_shrlock)(vp, cmd, shr, flag, cr, ct);
4418	VOPSTATS_UPDATE(vp, shrlock);
4419	return (err);
4420}
4421
4422int
4423fop_vnevent(vnode_t *vp, vnevent_t vnevent, vnode_t *dvp, char *fnm,
4424    caller_context_t *ct)
4425{
4426	int	err;
4427
4428	err = (*(vp)->v_op->vop_vnevent)(vp, vnevent, dvp, fnm, ct);
4429	VOPSTATS_UPDATE(vp, vnevent);
4430	return (err);
4431}
4432
4433int
4434fop_reqzcbuf(vnode_t *vp, enum uio_rw ioflag, xuio_t *uiop, cred_t *cr,
4435    caller_context_t *ct)
4436{
4437	int err;
4438
4439	if (vfs_has_feature(vp->v_vfsp, VFSFT_ZEROCOPY_SUPPORTED) == 0)
4440		return (ENOTSUP);
4441	err = (*(vp)->v_op->vop_reqzcbuf)(vp, ioflag, uiop, cr, ct);
4442	VOPSTATS_UPDATE(vp, reqzcbuf);
4443	return (err);
4444}
4445
4446int
4447fop_retzcbuf(vnode_t *vp, xuio_t *uiop, cred_t *cr, caller_context_t *ct)
4448{
4449	int err;
4450
4451	if (vfs_has_feature(vp->v_vfsp, VFSFT_ZEROCOPY_SUPPORTED) == 0)
4452		return (ENOTSUP);
4453	err = (*(vp)->v_op->vop_retzcbuf)(vp, uiop, cr, ct);
4454	VOPSTATS_UPDATE(vp, retzcbuf);
4455	return (err);
4456}
4457
4458/*
4459 * Default destructor
4460 *	Needed because NULL destructor means that the key is unused
4461 */
4462/* ARGSUSED */
4463void
4464vsd_defaultdestructor(void *value)
4465{}
4466
4467/*
4468 * Create a key (index into per vnode array)
4469 *	Locks out vsd_create, vsd_destroy, and vsd_free
4470 *	May allocate memory with lock held
4471 */
4472void
4473vsd_create(uint_t *keyp, void (*destructor)(void *))
4474{
4475	int	i;
4476	uint_t	nkeys;
4477
4478	/*
4479	 * if key is allocated, do nothing
4480	 */
4481	mutex_enter(&vsd_lock);
4482	if (*keyp) {
4483		mutex_exit(&vsd_lock);
4484		return;
4485	}
4486	/*
4487	 * find an unused key
4488	 */
4489	if (destructor == NULL)
4490		destructor = vsd_defaultdestructor;
4491
4492	for (i = 0; i < vsd_nkeys; ++i)
4493		if (vsd_destructor[i] == NULL)
4494			break;
4495
4496	/*
4497	 * if no unused keys, increase the size of the destructor array
4498	 */
4499	if (i == vsd_nkeys) {
4500		if ((nkeys = (vsd_nkeys << 1)) == 0)
4501			nkeys = 1;
4502		vsd_destructor =
4503		    (void (**)(void *))vsd_realloc((void *)vsd_destructor,
4504		    (size_t)(vsd_nkeys * sizeof (void (*)(void *))),
4505		    (size_t)(nkeys * sizeof (void (*)(void *))));
4506		vsd_nkeys = nkeys;
4507	}
4508
4509	/*
4510	 * allocate the next available unused key
4511	 */
4512	vsd_destructor[i] = destructor;
4513	*keyp = i + 1;
4514
4515	/* create vsd_list, if it doesn't exist */
4516	if (vsd_list == NULL) {
4517		vsd_list = kmem_alloc(sizeof (list_t), KM_SLEEP);
4518		list_create(vsd_list, sizeof (struct vsd_node),
4519		    offsetof(struct vsd_node, vs_nodes));
4520	}
4521
4522	mutex_exit(&vsd_lock);
4523}
4524
4525/*
4526 * Destroy a key
4527 *
4528 * Assumes that the caller is preventing vsd_set and vsd_get
4529 * Locks out vsd_create, vsd_destroy, and vsd_free
4530 * May free memory with lock held
4531 */
4532void
4533vsd_destroy(uint_t *keyp)
4534{
4535	uint_t key;
4536	struct vsd_node *vsd;
4537
4538	/*
4539	 * protect the key namespace and our destructor lists
4540	 */
4541	mutex_enter(&vsd_lock);
4542	key = *keyp;
4543	*keyp = 0;
4544
4545	ASSERT(key <= vsd_nkeys);
4546
4547	/*
4548	 * if the key is valid
4549	 */
4550	if (key != 0) {
4551		uint_t k = key - 1;
4552		/*
4553		 * for every vnode with VSD, call key's destructor
4554		 */
4555		for (vsd = list_head(vsd_list); vsd != NULL;
4556		    vsd = list_next(vsd_list, vsd)) {
4557			/*
4558			 * no VSD for key in this vnode
4559			 */
4560			if (key > vsd->vs_nkeys)
4561				continue;
4562			/*
4563			 * call destructor for key
4564			 */
4565			if (vsd->vs_value[k] && vsd_destructor[k])
4566				(*vsd_destructor[k])(vsd->vs_value[k]);
4567			/*
4568			 * reset value for key
4569			 */
4570			vsd->vs_value[k] = NULL;
4571		}
4572		/*
4573		 * actually free the key (NULL destructor == unused)
4574		 */
4575		vsd_destructor[k] = NULL;
4576	}
4577
4578	mutex_exit(&vsd_lock);
4579}
4580
4581/*
4582 * Quickly return the per vnode value that was stored with the specified key
4583 * Assumes the caller is protecting key from vsd_create and vsd_destroy
4584 * Assumes the caller is holding v_vsd_lock to protect the vsd.
4585 */
4586void *
4587vsd_get(vnode_t *vp, uint_t key)
4588{
4589	struct vsd_node *vsd;
4590
4591	ASSERT(vp != NULL);
4592	ASSERT(mutex_owned(&vp->v_vsd_lock));
4593
4594	vsd = vp->v_vsd;
4595
4596	if (key && vsd != NULL && key <= vsd->vs_nkeys)
4597		return (vsd->vs_value[key - 1]);
4598	return (NULL);
4599}
4600
4601/*
4602 * Set a per vnode value indexed with the specified key
4603 * Assumes the caller is holding v_vsd_lock to protect the vsd.
4604 */
4605int
4606vsd_set(vnode_t *vp, uint_t key, void *value)
4607{
4608	struct vsd_node *vsd;
4609
4610	ASSERT(vp != NULL);
4611	ASSERT(mutex_owned(&vp->v_vsd_lock));
4612
4613	if (key == 0)
4614		return (EINVAL);
4615
4616	vsd = vp->v_vsd;
4617	if (vsd == NULL)
4618		vsd = vp->v_vsd = kmem_zalloc(sizeof (*vsd), KM_SLEEP);
4619
4620	/*
4621	 * If the vsd was just allocated, vs_nkeys will be 0, so the following
4622	 * code won't happen and we will continue down and allocate space for
4623	 * the vs_value array.
4624	 * If the caller is replacing one value with another, then it is up
4625	 * to the caller to free/rele/destroy the previous value (if needed).
4626	 */
4627	if (key <= vsd->vs_nkeys) {
4628		vsd->vs_value[key - 1] = value;
4629		return (0);
4630	}
4631
4632	ASSERT(key <= vsd_nkeys);
4633
4634	if (vsd->vs_nkeys == 0) {
4635		mutex_enter(&vsd_lock);	/* lock out vsd_destroy() */
4636		/*
4637		 * Link onto list of all VSD nodes.
4638		 */
4639		list_insert_head(vsd_list, vsd);
4640		mutex_exit(&vsd_lock);
4641	}
4642
4643	/*
4644	 * Allocate vnode local storage and set the value for key
4645	 */
4646	vsd->vs_value = vsd_realloc(vsd->vs_value,
4647	    vsd->vs_nkeys * sizeof (void *),
4648	    key * sizeof (void *));
4649	vsd->vs_nkeys = key;
4650	vsd->vs_value[key - 1] = value;
4651
4652	return (0);
4653}
4654
4655/*
4656 * Called from vn_free() to run the destructor function for each vsd
4657 *	Locks out vsd_create and vsd_destroy
4658 *	Assumes that the destructor *DOES NOT* use vsd
4659 */
4660void
4661vsd_free(vnode_t *vp)
4662{
4663	int i;
4664	struct vsd_node *vsd = vp->v_vsd;
4665
4666	if (vsd == NULL)
4667		return;
4668
4669	if (vsd->vs_nkeys == 0) {
4670		kmem_free(vsd, sizeof (*vsd));
4671		vp->v_vsd = NULL;
4672		return;
4673	}
4674
4675	/*
4676	 * lock out vsd_create and vsd_destroy, call
4677	 * the destructor, and mark the value as destroyed.
4678	 */
4679	mutex_enter(&vsd_lock);
4680
4681	for (i = 0; i < vsd->vs_nkeys; i++) {
4682		if (vsd->vs_value[i] && vsd_destructor[i])
4683			(*vsd_destructor[i])(vsd->vs_value[i]);
4684		vsd->vs_value[i] = NULL;
4685	}
4686
4687	/*
4688	 * remove from linked list of VSD nodes
4689	 */
4690	list_remove(vsd_list, vsd);
4691
4692	mutex_exit(&vsd_lock);
4693
4694	/*
4695	 * free up the VSD
4696	 */
4697	kmem_free(vsd->vs_value, vsd->vs_nkeys * sizeof (void *));
4698	kmem_free(vsd, sizeof (struct vsd_node));
4699	vp->v_vsd = NULL;
4700}
4701
4702/*
4703 * realloc
4704 */
4705static void *
4706vsd_realloc(void *old, size_t osize, size_t nsize)
4707{
4708	void *new;
4709
4710	new = kmem_zalloc(nsize, KM_SLEEP);
4711	if (old) {
4712		bcopy(old, new, osize);
4713		kmem_free(old, osize);
4714	}
4715	return (new);
4716}
4717
4718/*
4719 * Setup the extensible system attribute for creating a reparse point.
4720 * The symlink data 'target' is validated for proper format of a reparse
4721 * string and a check also made to make sure the symlink data does not
4722 * point to an existing file.
4723 *
4724 * return 0 if ok else -1.
4725 */
4726static int
4727fs_reparse_mark(char *target, vattr_t *vap, xvattr_t *xvattr)
4728{
4729	xoptattr_t *xoap;
4730
4731	if ((!target) || (!vap) || (!xvattr))
4732		return (-1);
4733
4734	/* validate reparse string */
4735	if (reparse_validate((const char *)target))
4736		return (-1);
4737
4738	xva_init(xvattr);
4739	xvattr->xva_vattr = *vap;
4740	xvattr->xva_vattr.va_mask |= AT_XVATTR;
4741	xoap = xva_getxoptattr(xvattr);
4742	ASSERT(xoap);
4743	XVA_SET_REQ(xvattr, XAT_REPARSE);
4744	xoap->xoa_reparse = 1;
4745
4746	return (0);
4747}
4748
4749/*
4750 * Function to check whether a symlink is a reparse point.
4751 * Return B_TRUE if it is a reparse point, else return B_FALSE
4752 */
4753boolean_t
4754vn_is_reparse(vnode_t *vp, cred_t *cr, caller_context_t *ct)
4755{
4756	xvattr_t xvattr;
4757	xoptattr_t *xoap;
4758
4759	if ((vp->v_type != VLNK) ||
4760	    !(vfs_has_feature(vp->v_vfsp, VFSFT_XVATTR)))
4761		return (B_FALSE);
4762
4763	xva_init(&xvattr);
4764	xoap = xva_getxoptattr(&xvattr);
4765	ASSERT(xoap);
4766	XVA_SET_REQ(&xvattr, XAT_REPARSE);
4767
4768	if (VOP_GETATTR(vp, &xvattr.xva_vattr, 0, cr, ct))
4769		return (B_FALSE);
4770
4771	if ((!(xvattr.xva_vattr.va_mask & AT_XVATTR)) ||
4772	    (!(XVA_ISSET_RTN(&xvattr, XAT_REPARSE))))
4773		return (B_FALSE);
4774
4775	return (xoap->xoa_reparse ? B_TRUE : B_FALSE);
4776}
4777