xref: /illumos-gate/usr/src/uts/common/nfs/nfs4.h (revision f44e1126)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 /*
28  * Copyright 2018 Nexenta Systems, Inc.
29  * Copyright 2019 Nexenta by DDN, Inc.
30  */
31 
32 #ifndef _NFS4_H
33 #define	_NFS4_H
34 
35 #include <sys/types.h>
36 #include <sys/vnode.h>
37 #include <sys/fem.h>
38 #include <rpc/rpc.h>
39 #include <nfs/nfs.h>
40 
41 #ifdef _KERNEL
42 #include <nfs/nfs4_kprot.h>
43 #include <sys/nvpair.h>
44 #else
45 #include <rpcsvc/nfs4_prot.h>
46 #endif
47 #include <nfs/nfs4_attr.h>
48 #include <sys/acl.h>
49 #include <sys/list.h>
50 #include <nfs/nfs4x.h>
51 
52 #ifdef	__cplusplus
53 extern "C" {
54 #endif
55 
56 #define	NFS4_MAX_SECOID4	65536
57 #define	NFS4_MAX_UTF8STRING	65536
58 #define	NFS4_MAX_LINKTEXT4	65536
59 #define	NFS4_MAX_PATHNAME4	65536
60 
61 struct nfs_fsl_info {
62 	uint_t netbuf_len;
63 	uint_t netnm_len;
64 	uint_t knconf_len;
65 	char *netname;
66 	struct netbuf *addr;
67 	struct knetconfig *knconf;
68 };
69 
70 #ifdef _KERNEL
71 
72 typedef struct nfs4_fhandle {
73 	int fh_len;
74 	char fh_buf[NFS4_FHSIZE];
75 } nfs4_fhandle_t;
76 
77 typedef uint8_t nfs4_minor_t;
78 
79 #define	NFS4_MINORVERSION	0
80 #define	CB4_MINORVERSION	0
81 
82 #define	FIRST_NFS4_OP   OP_ACCESS
83 #define	LAST_NFS40_OP   OP_RELEASE_LOCKOWNER
84 #define	LAST_NFS41_OP   OP_RECLAIM_COMPLETE
85 #define	LAST_NFS42_OP   OP_RECLAIM_COMPLETE
86 #define	LAST_NFS4_OP    LAST_NFS42_OP
87 
88 /*
89  * Set the fattr4_change variable using a time struct. Note that change
90  * is 64 bits, but timestruc_t is 128 bits in a 64-bit kernel.
91  */
92 #define	NFS4_SET_FATTR4_CHANGE(change, ts)			\
93 {							\
94 	change = (ts).tv_sec;				\
95 	change <<= 32;					\
96 	change |= (uint32_t)((ts).tv_nsec);		\
97 }
98 
99 /*
100  * Server lease period.  Value is in seconds;  Also used for grace period
101  */
102 extern time_t rfs4_lease_time;
103 
104 /*
105  * This set of typedefs and interfaces represent the core or base set
106  * of functionality that backs the NFSv4 server's state related data
107  * structures.  Since the NFSv4 server needs inter-RPC state to be
108  * available that is unrelated to the filesystem (in other words,
109  * soft-state), this functionality is needed to maintain that and is
110  * written to be somewhat flexible to adapt to the various types of
111  * data structures contained within the server.
112  *
113  * The basic structure at this level is that the server maintains a
114  * global "database" which consists of a set of tables.  Each table
115  * contains a set of like data structures.  Each table is indexed by
116  * at least one hash function and in most cases two hashes.  Each
117  * table's characteristics is set when it is created at run-time via
118  * rfs4_table_create().  All table creation and related functions are
119  * located in nfs4_state.c.  The generic database functionality is
120  * located in nfs4_db.c.
121  */
122 
123 typedef struct rfs4_dbe rfs4_dbe_t;		/* basic opaque db entry */
124 typedef struct rfs4_table rfs4_table_t;		/* basic table type */
125 typedef struct rfs4_index rfs4_index_t;		/* index */
126 typedef struct rfs4_database rfs4_database_t;	/* and database */
127 
128 typedef struct {		/* opaque entry type for later use */
129 	rfs4_dbe_t *dbe;
130 } *rfs4_entry_t;
131 
132 /*
133  * NFSv4 server state databases
134  *
135  * Initialized when the module is loaded and used by NFSv4 state tables.
136  * These kmem_cache free pools are used globally, the NFSv4 state tables
137  * which make use of these kmem_cache free pools are per zone.
138  */
139 extern kmem_cache_t *rfs4_client_mem_cache;
140 extern kmem_cache_t *rfs4_clntIP_mem_cache;
141 extern kmem_cache_t *rfs4_openown_mem_cache;
142 extern kmem_cache_t *rfs4_openstID_mem_cache;
143 extern kmem_cache_t *rfs4_lockstID_mem_cache;
144 extern kmem_cache_t *rfs4_lockown_mem_cache;
145 extern kmem_cache_t *rfs4_file_mem_cache;
146 extern kmem_cache_t *rfs4_delegstID_mem_cache;
147 extern kmem_cache_t *rfs4_session_mem_cache;
148 
149 /* database, table, index creation entry points */
150 extern rfs4_database_t *rfs4_database_create(uint32_t);
151 extern void		rfs4_database_shutdown(rfs4_database_t *);
152 extern void		rfs4_database_destroy(rfs4_database_t *);
153 
154 extern void		rfs4_database_destroy(rfs4_database_t *);
155 
156 extern kmem_cache_t	*nfs4_init_mem_cache(char *, uint32_t, uint32_t,
157 				uint32_t);
158 extern rfs4_table_t	*rfs4_table_create(rfs4_database_t *, char *,
159 				time_t, uint32_t,
160 				bool_t (*create)(rfs4_entry_t, void *),
161 				void (*destroy)(rfs4_entry_t),
162 				bool_t (*expiry)(rfs4_entry_t),
163 				uint32_t, uint32_t, uint32_t, id_t);
164 extern void		rfs4_table_destroy(rfs4_database_t *, rfs4_table_t *);
165 extern rfs4_index_t	*rfs4_index_create(rfs4_table_t *, char *,
166 				uint32_t (*hash)(void *),
167 				bool_t (compare)(rfs4_entry_t, void *),
168 				void *(*mkkey)(rfs4_entry_t), bool_t);
169 extern void		rfs4_index_destroy(rfs4_index_t *);
170 
171 /* Type used to direct rfs4_dbsearch() in what types of records to inspect */
172 typedef enum {RFS4_DBS_VALID, RFS4_DBS_INVALID} rfs4_dbsearch_type_t;
173 /* search and db entry manipulation entry points */
174 extern rfs4_entry_t	rfs4_dbsearch(rfs4_index_t *, void *,
175 				bool_t *, void *, rfs4_dbsearch_type_t);
176 extern void		rfs4_dbe_lock(rfs4_dbe_t *);
177 extern void		rfs4_dbe_unlock(rfs4_dbe_t *);
178 extern clock_t		rfs4_dbe_twait(rfs4_dbe_t *, clock_t);
179 extern void		rfs4_dbe_cv_broadcast(rfs4_dbe_t *);
180 extern void		rfs4_dbe_hold(rfs4_dbe_t *);
181 extern void		rfs4_dbe_hold_nolock(rfs4_dbe_t *);
182 extern void		rfs4_dbe_rele_nolock(rfs4_dbe_t *);
183 extern void		rfs4_dbe_rele(rfs4_dbe_t *);
184 extern uint32_t	rfs4_dbe_refcnt(rfs4_dbe_t *);
185 extern id_t		rfs4_dbe_getid(rfs4_dbe_t *);
186 extern void		rfs4_dbe_invalidate(rfs4_dbe_t *);
187 extern bool_t		rfs4_dbe_is_invalid(rfs4_dbe_t *);
188 extern time_t		rfs4_dbe_get_timerele(rfs4_dbe_t *);
189 extern void		rfs4_dbe_hide(rfs4_dbe_t *);
190 extern void		rfs4_dbe_unhide(rfs4_dbe_t *);
191 #ifdef DEBUG
192 extern bool_t		rfs4_dbe_islocked(rfs4_dbe_t *);
193 #endif
194 extern void		rfs4_dbe_walk(rfs4_table_t *,
195 			void (*callout)(rfs4_entry_t, void *), void *);
196 
197 /*
198  * Minimal server stable storage.
199  *
200  * Currently the NFSv4 server will only save the client
201  * ID (the long version) so that it will be able to
202  * grant possible reclaim requests during the infamous
203  * grace_period.
204  */
205 
206 #define	RFS4_SS_DIRSIZE	64 * 1024
207 #define	NFS4_SS_VERSION 1
208 
209 /* handy pathname structure */
210 typedef struct ss_pn {
211 	char *leaf;
212 	char pn[MAXPATHLEN];
213 } rfs4_ss_pn_t;
214 
215 /*
216  * The server will build this link list on startup. It represents the
217  * clients that have had valid state on the server in a prior instance.
218  *
219  */
220 typedef struct rfs4_oldstate {
221 	struct rfs4_oldstate	*next;
222 	struct rfs4_oldstate	*prev;
223 	rfs4_ss_pn_t		*ss_pn;
224 	nfs_client_id4		cl_id4;
225 } rfs4_oldstate_t;
226 
227 /*
228  * This union is used to overlay the server's internal treatment of
229  * the protocols stateid4 datatype.  Therefore, "bits" must not exceed
230  * the size of stateid4 and more importantly should match the size of
231  * stateid4.  The chgseq field must the first entry since it overlays
232  * stateid4.seqid.
233  */
234 typedef union {
235 	stateid4 stateid;
236 	struct {
237 		uint32_t chgseq;	/* State changes / protocol's seqid */
238 		uint32_t boottime;	/* boot time  */
239 		uint32_t type:2;	/* stateid_type_t as define below */
240 		uint32_t clnodeid:8;	/* cluster server nodeid */
241 		uint32_t ident:22;	/* 2^22-1 openowner x fhs */
242 		pid_t	 pid;		/* pid of corresponding lock owner */
243 	} bits;
244 } stateid_t;
245 /*
246  * Note that the way the type field above is defined, this enum must
247  * not have more than 4 members.
248  */
249 typedef enum {OPENID, LOCKID, DELEGID} stateid_type_t;
250 
251 /*
252  * "wait" struct for use in the open open and lock owner state
253  * structures to provide serialization between server threads that are
254  * handling requests for the same open owner or lock stateid.  This
255  * way only one thread will be updating things like sequence ids,
256  * replay cache and stateid at a time.
257  */
258 typedef struct rfs4_state_wait {
259 	uint32_t		sw_active;
260 	uint32_t		sw_wait_count;
261 	kmutex_t		sw_cv_lock[1];
262 	kcondvar_t		sw_cv[1];
263 } rfs4_state_wait_t;
264 
265 extern void	rfs4_sw_enter(rfs4_state_wait_t *);
266 extern void	rfs4_sw_exit(rfs4_state_wait_t *);
267 
268 /*
269  * This enum and the following rfs4_cbinfo_t struct are used to
270  * maintain information about the callback path used from the server
271  * to client for operations like CB_GETATTR and CB_RECALL.  The
272  * rfs4_cbinfo_t struct is meant to be encompassed in the client
273  * struct and managed within that structure's locking scheme.
274  *
275  * The various states of the callback path are used by the server to
276  * determine if delegations should initially be provided to a client
277  * and then later on if connectivity has been lost and delegations
278  * should be revoked.
279  */
280 
281 /*
282  * CB_NOCHANGE - Special value used for interfaces within the delegation
283  *		code to signify that "no change" has occurred to the
284  *		callback path
285  * CB_UNINIT	- No callback info provided by the client
286  * CB_NONE	- Callback info provided but CB_NULL call
287  *		  has yet to be attempted
288  * CB_OK	- Callback path tested with CB_NULL with success
289  * CB_INPROG	- Callback path currently being tested with CB_NULL
290  * CB_FAILED	- Callback path was == CB_OK but has failed
291  *		  with timeout/rpc error
292  * CB_BAD	- Callback info provided but CB_NULL failed
293  */
294 typedef enum {
295 	CB_NOCHANGE = 0,
296 	CB_UNINIT = 1,
297 	CB_NONE = 2,
298 	CB_OK = 3,
299 	CB_INPROG = 4,
300 	CB_FAILED = 5,
301 	CB_BAD = 6
302 } rfs4_cbstate_t;
303 
304 #define	RFS4_CBCH_MAX	10	/* size callback client handle cache */
305 /*
306  * Callback info for a client.
307  * Client only provides: cb_client4 and cb_ident
308  * The rest of the information is used to track callback path status
309  * and usage.
310  *
311  * cb_state - used as comments for the rfs4_cbstate_t enum indicate
312  * cb_notified_of_cb_path_down - if the callback path was once CB_OK and
313  *	has hence CB_FAILED, the client needs to be notified via RENEW.
314  * cb_timefailed - current time when cb_state transitioned from
315  *	CB_OK -> CB_FAILED.  Meant for observability.  When did that happen?
316  * cb_chc_free/cb_chc - cache of client handles for the callback path
317  * cb_ident - SETCLIENTID provided callback_ident value
318  * callback - SETCLIENTID provided cb_client4 value
319  * cb_refcnt - current number of users of this structure's content
320  *	protected by cb_lock
321  * cb_badbehavior - how many times did a client do something we didn't like?
322  * cb_lock - lock for contents of cbinfo
323  * cb_cv - used to allow threads to wait on CB_NULL completion
324  * cb_nullcaller - is there a thread currently taking care of
325  *	new callback information?
326  * cb_cv_nullcaller - used by the thread doing CB_NULL to wait on
327  *	threads that may be using client handles of the current
328  *	client handle cache.
329  * newer - new callback info provided by a client and awaiting
330  *	CB_NULL testing and move to regular cbinfo.
331  */
332 typedef struct {
333 	rfs4_cbstate_t	cb_state;
334 	unsigned	cb_notified_of_cb_path_down:1;
335 	time_t		cb_timefailed;
336 	int		cb_chc_free;
337 	CLIENT		*cb_chc[RFS4_CBCH_MAX];
338 	uint32_t	cb_ident;
339 	cb_client4	cb_callback;
340 	uint32_t	cb_refcnt;
341 	uint32_t	cb_badbehavior;
342 	kmutex_t	cb_lock[1];
343 	kcondvar_t	cb_cv[1];
344 	bool_t		cb_nullcaller;
345 	kcondvar_t	cb_cv_nullcaller[1];
346 	struct {
347 		bool_t		cb_new;
348 		bool_t		cb_confirmed;
349 		uint32_t	cb_ident;
350 		cb_client4	cb_callback;
351 	} cb_newer;
352 } rfs4_cbinfo_t;
353 
354 /*
355  * A server instance. We can associate sets of clients - via a pointer in
356  * rfs4_client_t - with a given server instance, allowing us to treat clients
357  * in the set differently to clients in other sets.
358  *
359  * Currently used only for Sun Cluster HA-NFS support, to group clients
360  * on NFS resource failover so each set of clients gets its own dedicated
361  * grace period and distributed stable storage data.
362  */
363 typedef struct rfs4_servinst {
364 	int			dss_npaths;
365 	krwlock_t		rwlock;
366 	krwlock_t		oldstate_lock;
367 	time_t			start_time;
368 	time_t			grace_period;
369 	uint_t			nreclaim;	/* number reclaim clients  */
370 	rfs4_oldstate_t		*oldstate;
371 	struct rfs4_dss_path	**dss_paths;
372 	struct rfs4_servinst	*next;
373 	struct rfs4_servinst	*prev;
374 } rfs4_servinst_t;
375 
376 /*
377  * DSS: distributed stable storage
378  */
379 
380 typedef struct rfs4_dss_path {
381 	struct rfs4_dss_path	*next; /* for insque/remque */
382 	struct rfs4_dss_path	*prev; /* for insque/remque */
383 	char			*path;
384 	struct rfs4_servinst	*sip;
385 	unsigned		index; /* offset in servinst's array */
386 } rfs4_dss_path_t;
387 
388 /* array of paths passed-in from nfsd command-line; stored in nvlist */
389 extern char		**rfs4_dss_newpaths;
390 extern uint_t		rfs4_dss_numnewpaths;
391 
392 /* nvlists of all DSS paths: current, and before last warmstart */
393 extern nvlist_t *rfs4_dss_paths, *rfs4_dss_oldpaths;
394 
395 /*
396  * The server maintains a set of state on a per client basis that
397  * matches that of the protocol requirements.  A client's state is
398  * rooted with the rfs4_client_t struct of which there is one per
399  * client and is created when SETCLIENTID/SETCLIENTID_CONFIRM are
400  * received.  From there, the server then creates rfs4_openowner_t
401  * structs for each new open owner from that client and are initiated
402  * at OPEN/OPEN_CONFIRM (when the open owner is new to the server).
403  * At OPEN, at least two other structures are created, and potentially a
404  * third.  rfs4_state_t is created to track the association between an
405  * open owner and a particular file. An rfs4_file_t struct may be
406  * created (if the file is not already open) at OPEN as well.  The
407  * rfs4_file_t struct is the only one that is per server and not per
408  * client.  The rfs4_deleg_state_t struct is created in the
409  * instance that the server is going to provide a delegation for the
410  * file being OPENed.  Finally, the rfs4_lockowner_t is created at the
411  * first use of a lock owner at the server and is a result of the LOCK
412  * operation.  The rfs4_lo_state_t struct is then created to represent
413  * the relation between the lock owner and the file.
414  *
415  */
416 /*
417  * The following ascii art represents each of these data structs and
418  * their references to each other.  Note: "<-(x)->" represents the
419  * doubly link lists (list_t).
420  *
421  *                          ____________________
422  *                         |                    |
423  *                         |    rfs4_client_t   |
424  *                       ->|         (1)        |<-
425  *                      /  |____________________|  \
426  *                     /              ^             \
427  *                    /               |              \
428  *  ____________________    ____________________    ____________________
429  * |                    |  |                    |  |                    |
430  * |  rfs4_lockowner_t  |  |  rfs4_openowner_t  |  | rfs4_deleg_state_t |
431  * |                    |  |     (3)    <-(1)-> |  |            <-(2)-> |
432  * |____________________|  |____________________|  |____________________|
433  *           ^                        ^                       |
434  *           |                        |                       V
435  *  ____________________    ____________________    ____________________
436  * |                    |  |                    |  |                    |
437  * |  rfs4_lo_state_t   |->|    rfs4_state_t    |->|     rfs4_file_t    |
438  * |            <-(4)-> |  |     (4)    <-(3)-> |  |        (2)         |
439  * |____________________|  |____________________|  |____________________|
440  */
441 /*
442  * Each of these data types are kept in a separate rfs4_table_t and is
443  * actually encapsulated within a rfs4_dbe_t struct.  The various
444  * tables and their construction is done in nfs4_state.c but
445  * documented here to completeness.
446  *
447  * Table		Data struct stored	Indexed by
448  * -----		------------------	----------
449  * rfs4_client_tab	rfs4_client_t		nfs_client_id4
450  *						clientid4
451  *
452  * rfs4_openowner_tab	rfs4_openowner_t	open_owner4
453  *
454  * rfs4_state_tab	rfs4_state_t		open_owner4 | file
455  *						stateid
456  *
457  * rfs4_lo_state_tab	rfs4_lo_state_t		lockowner | stateid
458  *						lock_stateid
459  *
460  * rfs4_lockowner_tab	rfs4_lockowner_t	lockowner
461  *						pid
462  *
463  * rfs4_file_tab	rfs4_file_t		filehandle
464  *
465  * rfs4_deleg_state_tab	rfs4_deleg_state_t	clientid4 | file
466  *						deleg_stateid
467  */
468 
469 /*
470  * The client struct, it is the root of all state for a particular
471  * client.  The client is identified by the nfs_client_id4 via
472  * SETCLIENTID and the server returns the clientid4 as short hand reference
473  */
474 /*
475  * Client struct - as mentioned above it is the root of all state for
476  * a single client as identified by the client supplied nfs_client_id4
477  *
478  * dbe - encapsulation struct
479  * clientid - server assigned short hand reference to client
480  * nfs_client - client supplied identifier for itself
481  * confirm_verf - the value provided to the client for SETCLIENTID_CONFIRM
482  * need_confirm - does this client need to be SETCLIENTID_CONFIRMed?
483  *
484  * unlksys_completed - has an F_UNLKSYS been done for this client which
485  *		says that the use of cleanlocks() on individual files
486  *		is not required?
487  * can_reclaim - indicates if client is allowed to reclaim after server
488  *		start-up (client had previous state at server)
489  * ss_remove - indicates that the rfs4_client_destroy function should
490  *		clean up stable storage file.
491  * forced_expire - set if the sysadmin has used clear_locks for this client.
492  * no_referrals - set if the client is Solaris and pre-dates referrals
493  * deleg_revoked - how many delegations have been revoked for this client?
494  *
495  * cp_confirmed - this refers to a confirmed client struct that has
496  * the same nfs_client_id4 as this client struct.  When/if this client
497  * struct is confirmed via SETCLINETID_CONFIRM, the previously
498  * confirmed client struct will be "closed" and hence this reference.
499  *
500  * last_access - used to determine if the client has let its lease expire
501  * cbinfo - struct containing all callback related information
502  * cr_set - credentials used for the SETCLIENTID/SETCLIENTID_CONFIRM pair
503  * sysid - the lock manager sysid allocated for this client's file locks
504  * openownerlist - root of openowners list associated with this client
505  * ss_pn - Pathname to the stable storage file.
506  * cl_addr - Clients network address.
507  * server_instance - pointer to the currently associated server instance
508  */
509 typedef struct rfs4_client {
510 	rfs4_dbe_t		*rc_dbe;
511 	clientid4		rc_clientid;
512 	nfs_client_id4		rc_nfs_client;
513 	verifier4		rc_confirm_verf;
514 	unsigned		rc_need_confirm:1;
515 	unsigned		rc_unlksys_completed:1;
516 	unsigned		rc_can_reclaim:1;
517 	unsigned		rc_ss_remove:1;
518 	unsigned		rc_forced_expire:1;
519 	unsigned		rc_reclaim_completed:1;
520 	uint_t			rc_deleg_revoked;
521 	struct rfs4_client	*rc_cp_confirmed;
522 	time_t			rc_last_access;
523 	rfs4_cbinfo_t		rc_cbinfo;
524 	cred_set_t		rc_cr_set;
525 	sysid_t			rc_sysidt;
526 	list_t			rc_openownerlist;
527 	rfs4_ss_pn_t		*rc_ss_pn;
528 	struct sockaddr_storage rc_addr;
529 	rfs4_servinst_t		*rc_server_instance;
530 
531 	/* nfsv4.1 */
532 	rfs41_csr_t		rc_contrived;
533 	rfs41_sprot_t		rc_state_prot;
534 	list_t			rc_sessions;
535 	unsigned		rc_destroying:1;    /* flag: going to destroy */
536 } rfs4_client_t;
537 
538 /*
539  * ClntIP struct - holds the diagnosis about whether the client
540  * cannot support referrals.  Set to true for old Solaris clients.
541  */
542 
543 typedef struct rfs4_clntip {
544 	rfs4_dbe_t		*ri_dbe;
545 	struct sockaddr_storage ri_addr;
546 	unsigned		ri_no_referrals:1;
547 } rfs4_clntip_t;
548 
549 /*
550  * The openowner contains the client supplied open_owner4 as well as
551  * the matching sequence id and is used to track the client's usage of
552  * the open_owner4.  Note that a reply is saved here as well for
553  * processing of retransmissions.
554  *
555  * dbe - encapsulation struct
556  * client - reference to rfs4_client_t for this openowner
557  * owner - actual client supplied open_owner4
558  * need_confirm - does this openowner need to be OPEN_CONFIRMed
559  * postpone_confirm - set if error received on first use of open_owner
560  * state2confirm - what stateid4 should be used on the OPEN_CONFIRM
561  * open_seqid - what is the next open_seqid expected for this openowner
562  * oo_sw - used to serialize access to the open seqid/reply handling
563  * statelist - root of state struct list associated with this openowner
564  * node - node for client struct list of openowners
565  * reply_fh - open replay processing needs the filehandle so that it is
566  *	able to reset the current filehandle for appropriate compound
567  *	processing and reply.
568  * reply - last reply sent in relation to this openowner
569  */
570 typedef struct rfs4_openowner {
571 	rfs4_dbe_t		*ro_dbe;
572 	rfs4_client_t		*ro_client;
573 	open_owner4		ro_owner;
574 	unsigned		ro_need_confirm:1;
575 	unsigned		ro_postpone_confirm:1;
576 	seqid4			ro_open_seqid;
577 	rfs4_state_wait_t	ro_sw;
578 	list_t			ro_statelist;
579 	list_node_t		ro_node;
580 	nfs_fh4			ro_reply_fh;
581 	nfs_resop4		ro_reply;
582 } rfs4_openowner_t;
583 
584 /*
585  * This state struct represents the association between an openowner
586  * and a file that has been OPENed by that openowner.
587  *
588  * dbe - encapsulation struct
589  * stateid - server provided stateid
590  * owner - reference back to the openowner for this state
591  * finfo - reference to the open file for this state
592  * open_access - how did the openowner OPEN the file (access)
593  * open_deny - how did the openowner OPEN the file (deny)
594  * share_access - what share reservation is on the file (access)
595  * share_deny - what share reservation is on the file (deny)
596  * closed - has this file been closed?
597  * lostatelist - root of list of lo_state associated with this state/file
598  * node - node for state struct list of states
599  */
600 typedef struct rfs4_state {
601 	rfs4_dbe_t		*rs_dbe;
602 	stateid_t		rs_stateid;
603 	rfs4_openowner_t	*rs_owner;
604 	struct rfs4_file	*rs_finfo;
605 	uint32_t		rs_open_access;
606 	uint32_t		rs_open_deny;
607 	uint32_t		rs_share_access;
608 	uint32_t		rs_share_deny;
609 	unsigned		rs_closed:1;
610 	list_t			rs_lostatelist;
611 	list_node_t		rs_node;
612 } rfs4_state_t;
613 
614 /*
615  * Lockowner - track the lockowner and its related info
616  *
617  * dbe - encapsulation struct
618  * client - reference to the client
619  * owner - lockowner supplied by the client
620  * pid - local identifier used for file locking
621  */
622 typedef struct rfs4_lockowner {
623 	rfs4_dbe_t		*rl_dbe;
624 	rfs4_client_t		*rl_client;
625 	lock_owner4		rl_owner;
626 	pid_t			rl_pid;
627 } rfs4_lockowner_t;
628 
629 /*
630  * Lockowner_state associated with a state struct and lockowner
631  *
632  * dbe - encapsulation struct
633  * state - reference back to state struct for open file
634  * lockid - stateid for this lockowner/state
635  * locker - reference to lockowner
636  * seqid - sequence id for this lockowner/state
637  * skip_seqid_check - used on initialization of struct
638  * locks_cleaned - have all locks been released for this lockowner/file?
639  * lock_completed - successful LOCK with lockowner/file?
640  * ls_sw - used to serialize update seqid/reply/stateid handling
641  * node - node for state struct list of lo_states
642  * reply - last reply sent in relation to this lockowner/state
643  */
644 typedef struct rfs4_lo_state {
645 	rfs4_dbe_t		*rls_dbe;
646 	rfs4_state_t		*rls_state;
647 	stateid_t		rls_lockid;
648 	rfs4_lockowner_t	*rls_locker;
649 	seqid4			rls_seqid;
650 	unsigned		rls_skip_seqid_check:1;
651 	unsigned		rls_locks_cleaned:1;
652 	unsigned		rls_lock_completed:1;
653 	rfs4_state_wait_t	rls_sw;
654 	list_node_t		rls_node;
655 	nfs_resop4		rls_reply;
656 } rfs4_lo_state_t;
657 
658 /*
659  * Delegation state - per client
660  *
661  * dbe - encapsulation struct
662  * dtype - type of delegation (NONE, READ, WRITE)
663  * delegid - stateid for this delegation
664  * time_granted - time this delegation was assigned to client
665  * time_recalled - time when the server started recall process
666  * time_revoked - if revoked, time that the revoke occurred
667  * finfo - reference to the file associated with this delegation
668  * client - reference to client for which this delegation is associated
669  * node - list of delegations for the file (WRITE == 1, READ == )
670  */
671 typedef struct rfs4_deleg_state {
672 	rfs4_dbe_t		*rds_dbe;
673 	open_delegation_type4	rds_dtype;
674 	stateid_t		rds_delegid;
675 	time_t			rds_time_granted;
676 	time_t			rds_time_recalled;
677 	time_t			rds_time_revoked;
678 	struct rfs4_file	*rds_finfo;
679 	rfs4_client_t		*rds_client;
680 	list_node_t		rds_node;
681 } rfs4_deleg_state_t;
682 
683 /*
684  * Delegation info associated with the file
685  *
686  * dtype - type of delegation for file (NONE, READ, WRITE)
687  * time_returned - time that last delegation was returned for file
688  * time_recalled - time that recall sequence started
689  * time_lastgrant - time that last delegation was provided to a client
690  * time_lastwrite - time of last write to use the delegation stateid
691  * time_rm_delayed - time of last remove/rename which was DELAYed
692  * rdgrants - how many read delegations have been provided for this file
693  * wrgrants - how many write delegations provided (can only be one)
694  * recall_count - how many recall threads are outstanding
695  * recall_lock - lock to protect contents of this struct
696  * recall_cv - condition var for the "parent" thread to wait upon
697  * deleg_change_grant - value for change attribute at time of write grant
698  * deleg_change - most recent value of change obtained from client
699  * deleg_change_ts - time of last deleg_change update
700  * ever_recalled - has this particular delegation ever been recalled?
701  * dont_grant - file deletion is impending, don't grant a delegation
702  * conflicted_client - clientid of the client that caused a CB_RECALL
703  *	to occur. This is used for delegation policy (should a delegation
704  *	be granted shortly after it has been returned?)
705  */
706 typedef struct rfs4_dinfo {
707 	open_delegation_type4 rd_dtype;
708 	time_t		rd_time_returned;
709 	time_t		rd_time_recalled;
710 	time_t		rd_time_lastgrant;
711 	time_t		rd_time_lastwrite;
712 	time_t		rd_time_rm_delayed;
713 	uint32_t	rd_rdgrants;
714 	uint32_t	rd_wrgrants;
715 	int32_t		rd_recall_count;
716 	kmutex_t	rd_recall_lock[1];
717 	kcondvar_t	rd_recall_cv[1];
718 	bool_t		rd_ever_recalled;
719 	uint32_t	rd_hold_grant;
720 	clientid4	rd_conflicted_client;
721 } rfs4_dinfo_t;
722 
723 /*
724  * File
725  *
726  * dbe - encapsulation struct
727  * vp - vnode for the file that is open or has a delegation
728  * filehandle - the filehandle generated by the server for this file
729  * delegstatelist - root of delegation list for this file
730  * dinfo - see struct definition above
731  * share_deny - union of all deny modes on file
732  * share_access - union of all access modes on file
733  * access_read - count of read access
734  * access_write - count of write access
735  * deny_read - count of deny reads
736  * deny_write - count of deny writes
737  * file_rwlock - lock for serializing the removal of a file while
738  *	the state structures are active within the server
739  *
740  *	The only requirement for locking file_rwlock is that the
741  *	caller have a reference to the containing rfs4_file.  The dbe
742  *	lock may or may not be held for lock/unlock of file_rwlock.
743  *	As mentioned above, the file_rwlock is used for serialization
744  *	of file removal and more specifically reference to the held
745  *	vnode (e.g. vp).
746  */
747 typedef struct rfs4_file {
748 	rfs4_dbe_t	*rf_dbe;
749 	vnode_t		*rf_vp;
750 	nfs_fh4		rf_filehandle;
751 	list_t		rf_delegstatelist;
752 	rfs4_dinfo_t	rf_dinfo;
753 	uint32_t	rf_share_deny;
754 	uint32_t	rf_share_access;
755 	uint32_t	rf_access_read;
756 	uint32_t	rf_access_write;
757 	uint32_t	rf_deny_read;
758 	uint32_t	rf_deny_write;
759 	krwlock_t	rf_file_rwlock;
760 } rfs4_file_t;
761 
762 /*
763  * nfs4_deleg_policy is used to signify the server's global delegation
764  * policy.  The default is to NEVER delegate files and the
765  * administrator must configure the server to enable delegations.
766  *
767  * The disable/enable delegation functions are used to eliminate a
768  * race with exclusive creates.
769  */
770 typedef enum {
771 	SRV_NEVER_DELEGATE = 0,
772 	SRV_NORMAL_DELEGATE = 1
773 } srv_deleg_policy_t;
774 
775 extern void rfs4_disable_delegation(void), rfs4_enable_delegation(void);
776 
777 /*
778  * Request types for delegation. These correspond with
779  * open_delegation_type4 with the addition of a new value, DELEG_ANY,
780  * to reqequest any delegation.
781  */
782 typedef enum {
783 	DELEG_NONE = 0,		/* Corresponds to OPEN_DELEG_NONE */
784 	DELEG_READ = 1,		/* Corresponds to OPEN_DELEG_READ */
785 	DELEG_WRITE = 2,	/* Corresponds to OPEN_DELEG_WRITE */
786 	DELEG_ANY = -1		/* New value to request any delegation type */
787 } delegreq_t;
788 
789 #define	NFS4_DELEG4TYPE2REQTYPE(x) (delegreq_t)(x)
790 
791 /*
792  * Zone global variables of NFSv4 server
793  */
794 typedef struct nfs4_srv {
795 	/* Unique write verifier */
796 	verifier4	write4verf;
797 	/* Delegation lock */
798 	kmutex_t	deleg_lock;
799 	/* Used to serialize create/destroy of nfs4_server_state database */
800 	kmutex_t	state_lock;
801 	rfs4_database_t *nfs4_server_state;
802 	/* Used to manage access to server instance linked list */
803 	kmutex_t	servinst_lock;
804 	rfs4_servinst_t *nfs4_cur_servinst;
805 	/* Used to manage access to nfs4_deleg_policy */
806 	krwlock_t	deleg_policy_lock;
807 	srv_deleg_policy_t nfs4_deleg_policy;
808 	/* Allowed minors, nfs4 minors start from 0, which is always allowed */
809 	nfs4_minor_t	nfs4_minor_max;
810 	/* Set first time we see one */
811 	int		seen_first_compound;
812 	/*
813 	 * Circular double-linked list of paths for currently-served RGs.
814 	 * No locking required -- only changed on server start.
815 	 * Managed with insque/remque.
816 	 */
817 	rfs4_dss_path_t	*dss_pathlist;
818 	/* Duplicate request cache */
819 	struct rfs4_drc	*nfs4_drc;
820 	/* nfsv4 server start time */
821 	time_t rfs4_start_time;
822 	/* Used to serialize lookups of clientids */
823 	krwlock_t rfs4_findclient_lock;
824 
825 	/* NFSv4 server state client tables */
826 	/* table expiry times */
827 	time_t rfs4_client_cache_time;
828 	time_t rfs4_openowner_cache_time;
829 	time_t rfs4_state_cache_time;
830 	time_t rfs4_lo_state_cache_time;
831 	time_t rfs4_lockowner_cache_time;
832 	time_t rfs4_file_cache_time;
833 	time_t rfs4_deleg_state_cache_time;
834 	time_t rfs4_clntip_cache_time;
835 	/* tables and indexes */
836 	/* client table */
837 	rfs4_table_t *rfs4_client_tab;
838 	rfs4_index_t *rfs4_clientid_idx;
839 	rfs4_index_t *rfs4_nfsclnt_idx;
840 	/* client IP table */
841 	rfs4_table_t *rfs4_clntip_tab;
842 	rfs4_index_t *rfs4_clntip_idx;
843 	/* Open Owner table */
844 	rfs4_table_t *rfs4_openowner_tab;
845 	rfs4_index_t *rfs4_openowner_idx;
846 	/* Open State ID table */
847 	rfs4_table_t *rfs4_state_tab;
848 	rfs4_index_t *rfs4_state_idx;
849 	rfs4_index_t *rfs4_state_owner_file_idx;
850 	rfs4_index_t *rfs4_state_file_idx;
851 	/* Lock State ID table */
852 	rfs4_table_t *rfs4_lo_state_tab;
853 	rfs4_index_t *rfs4_lo_state_idx;
854 	rfs4_index_t *rfs4_lo_state_owner_idx;
855 	/* Lock owner table */
856 	rfs4_table_t *rfs4_lockowner_tab;
857 	rfs4_index_t *rfs4_lockowner_idx;
858 	rfs4_index_t *rfs4_lockowner_pid_idx;
859 	/* File table */
860 	rfs4_table_t *rfs4_file_tab;
861 	rfs4_index_t *rfs4_file_idx;
862 	/* Deleg State table */
863 	rfs4_table_t *rfs4_deleg_state_tab;
864 	rfs4_index_t *rfs4_deleg_idx;
865 	rfs4_index_t *rfs4_deleg_state_idx;
866 
867 	/* nfs4.x */
868 	rfs4_table_t	*rfs4_session_tab;
869 	rfs4_index_t	*rfs4_session_idx;
870 
871 	/* client stable storage */
872 	int rfs4_ss_enabled;
873 } nfs4_srv_t;
874 
875 /*
876  * max length of the NFSv4 server database name
877  */
878 #define	RFS4_MAX_MEM_CACHE_NAME 48
879 
880 /*
881  * global NFSv4 server kmem caches
882  * r_db_name - The name of the state database and the table that will use it
883  *             These tables are defined in nfs4_srv_t
884  * r_db_mem_cache - The kmem cache associated with the state database name
885  */
886 typedef struct rfs4_db_mem_cache {
887 	char		r_db_name[RFS4_MAX_MEM_CACHE_NAME];
888 	kmem_cache_t	*r_db_mem_cache;
889 } rfs4_db_mem_cache_t;
890 
891 #define	RFS4_DB_MEM_CACHE_NUM 9
892 
893 extern rfs4_db_mem_cache_t rfs4_db_mem_cache_table[RFS4_DB_MEM_CACHE_NUM];
894 
895 extern srv_deleg_policy_t nfs4_get_deleg_policy();
896 
897 extern void		rfs4_servinst_create(nfs4_srv_t *, int, int, char **);
898 extern void		rfs4_servinst_destroy_all(nfs4_srv_t *);
899 extern void		rfs4_servinst_assign(nfs4_srv_t *, rfs4_client_t *,
900 			    rfs4_servinst_t *);
901 extern rfs4_servinst_t	*rfs4_servinst(rfs4_client_t *);
902 extern int		rfs4_clnt_in_grace(rfs4_client_t *);
903 extern int		rfs4_servinst_in_grace(rfs4_servinst_t *);
904 extern int		rfs4_servinst_grace_new(rfs4_servinst_t *);
905 extern void		rfs4_grace_start(rfs4_servinst_t *);
906 extern void		rfs4_grace_start_new(nfs4_srv_t *);
907 extern void		rfs4_grace_reset_all(nfs4_srv_t *);
908 extern void		rfs4_dss_readstate(nfs4_srv_t *, int, char **);
909 
910 /*
911  * Various interfaces to manipulate the state structures introduced
912  * above
913  */
914 extern	void		rfs4_free_reply(nfs_resop4 *);
915 extern	void		rfs4_copy_reply(nfs_resop4 *, nfs_resop4 *);
916 
917 /* rfs4_client_t handling */
918 extern	rfs4_client_t	*rfs4_findclient(nfs_client_id4 *,
919 					bool_t *, rfs4_client_t *);
920 extern	rfs4_client_t	*rfs4_findclient_by_id(clientid4, bool_t);
921 extern	rfs4_client_t	*rfs4_findclient_by_addr(struct sockaddr *);
922 extern	void		rfs4_client_rele(rfs4_client_t *);
923 extern	void		rfs4_client_close(rfs4_client_t *);
924 extern	void		rfs4_client_state_remove(rfs4_client_t *);
925 extern	void		rfs4_client_scv_next(rfs4_client_t *);
926 extern	void		rfs4_update_lease(rfs4_client_t *);
927 extern	bool_t		rfs4_lease_expired(rfs4_client_t *);
928 extern	nfsstat4	rfs4_check_clientid(clientid4 *, int);
929 
930 /* rfs4_clntip_t handling */
931 extern	rfs4_clntip_t	*rfs4_find_clntip(struct sockaddr *, bool_t *);
932 extern	void		rfs4_invalidate_clntip(struct sockaddr *);
933 
934 /* rfs4_openowner_t handling */
935 extern	rfs4_openowner_t *rfs4_findopenowner(open_owner4 *, bool_t *, seqid4);
936 extern	void		rfs4_update_open_sequence(rfs4_openowner_t *);
937 extern	void		rfs4_update_open_resp(rfs4_openowner_t *,
938 					nfs_resop4 *, nfs_fh4 *);
939 extern	void		rfs4_openowner_rele(rfs4_openowner_t *);
940 extern	void		rfs4_free_opens(rfs4_openowner_t *, bool_t, bool_t);
941 
942 /* rfs4_lockowner_t handling */
943 extern	rfs4_lockowner_t *rfs4_findlockowner(lock_owner4 *, bool_t *);
944 extern	rfs4_lockowner_t *rfs4_findlockowner_by_pid(pid_t);
945 extern	void		rfs4_lockowner_rele(rfs4_lockowner_t *);
946 
947 /* rfs4_state_t handling */
948 extern	rfs4_state_t	*rfs4_findstate_by_owner_file(rfs4_openowner_t *,
949 					rfs4_file_t *, bool_t *);
950 extern	void		rfs4_state_rele(rfs4_state_t *);
951 extern	void		rfs4_state_close(rfs4_state_t *, bool_t,
952 					bool_t, cred_t *);
953 extern	void		rfs4_release_share_lock_state(rfs4_state_t *,
954 					cred_t *, bool_t);
955 extern	void		rfs4_close_all_state(rfs4_file_t *);
956 
957 /* rfs4_lo_state_t handling */
958 extern	rfs4_lo_state_t *rfs4_findlo_state_by_owner(rfs4_lockowner_t *,
959 						rfs4_state_t *, bool_t *);
960 extern	void		rfs4_lo_state_rele(rfs4_lo_state_t *, bool_t);
961 extern	void		rfs4_update_lock_sequence(rfs4_lo_state_t *);
962 extern	void		rfs4_update_lock_resp(rfs4_lo_state_t *,
963 					nfs_resop4 *);
964 
965 /* rfs4_file_t handling */
966 extern	rfs4_file_t	*rfs4_findfile(vnode_t *, nfs_fh4 *, bool_t *);
967 extern	rfs4_file_t	*rfs4_findfile_withlock(vnode_t *, nfs_fh4 *,
968 						bool_t *);
969 extern	void		rfs4_file_rele(rfs4_file_t *);
970 
971 /* General collection of "get state" functions */
972 extern	nfsstat4	rfs4_get_state(stateid4 *, rfs4_state_t **,
973 					rfs4_dbsearch_type_t);
974 extern	nfsstat4	rfs4_get_deleg_state(stateid4 *,
975 					rfs4_deleg_state_t **);
976 extern	nfsstat4	rfs4_get_lo_state(stateid4 *, rfs4_lo_state_t **,
977 					bool_t);
978 struct compound_state;
979 extern	nfsstat4	rfs4_check_stateid(int, vnode_t *, stateid4 *,
980 					bool_t, bool_t *, bool_t,
981 					caller_context_t *,
982 					struct compound_state *);
983 extern	int		rfs4_check_stateid_seqid(rfs4_state_t *, stateid4 *,
984 					const struct compound_state *);
985 extern	int		rfs4_check_lo_stateid_seqid(rfs4_lo_state_t *,
986 					stateid4 *,
987 					const struct compound_state *);
988 
989 /* return values for rfs4_check_stateid_seqid() */
990 #define	NFS4_CHECK_STATEID_OKAY	1
991 #define	NFS4_CHECK_STATEID_OLD	2
992 #define	NFS4_CHECK_STATEID_BAD	3
993 #define	NFS4_CHECK_STATEID_EXPIRED	4
994 #define	NFS4_CHECK_STATEID_REPLAY	5
995 #define	NFS4_CHECK_STATEID_CLOSED	6
996 #define	NFS4_CHECK_STATEID_UNCONFIRMED	7
997 
998 /* delay() time that server is willing to briefly wait for a delegreturn */
999 #define	NFS4_DELEGATION_CONFLICT_DELAY	(hz/10)
1000 
1001 /*
1002  * Interfaces for handling of callback's client handle cache and
1003  * callback interfaces themselves.
1004  */
1005 extern	void		rfs4_cbinfo_free(rfs4_cbinfo_t *);
1006 extern	void		rfs4_client_setcb(rfs4_client_t *, cb_client4 *,
1007 					uint32_t);
1008 extern	void		rfs4_deleg_cb_check(rfs4_client_t *);
1009 extern	nfsstat4	rfs4_vop_getattr(vnode_t *, vattr_t *, int, cred_t *);
1010 
1011 /* rfs4_deleg_state_t handling and other delegation interfaces */
1012 extern	rfs4_deleg_state_t *rfs4_finddeleg(rfs4_state_t *, bool_t *);
1013 extern	rfs4_deleg_state_t *rfs4_finddelegstate(stateid_t *);
1014 extern	bool_t		rfs4_check_recall(rfs4_state_t *, uint32_t);
1015 extern	void		rfs4_recall_deleg(rfs4_file_t *,
1016 				bool_t, rfs4_client_t *);
1017 extern	int		rfs4_get_deleg(rfs4_state_t *,  open_delegation_type4,
1018 			open_delegation_type4 (*policy)(rfs4_state_t *,
1019 				open_delegation_type4 dtype));
1020 extern	rfs4_deleg_state_t *rfs4_grant_delegation(delegreq_t, rfs4_state_t *,
1021 				int *);
1022 extern	void		rfs4_set_deleg_response(rfs4_deleg_state_t *,
1023 				open_delegation4 *, nfsace4 *, int);
1024 extern	void		rfs4_return_deleg(rfs4_deleg_state_t *, bool_t);
1025 extern	bool_t		rfs4_is_deleg(rfs4_state_t *);
1026 extern	void		rfs4_deleg_state_rele(rfs4_deleg_state_t *);
1027 extern	bool_t		rfs4_check_delegated_byfp(int, rfs4_file_t *,
1028 					bool_t, bool_t, bool_t, clientid4 *);
1029 extern	void		rfs4_clear_dont_grant(rfs4_file_t *);
1030 
1031 /*
1032  * nfs4 monitored operations.
1033  */
1034 extern int deleg_rd_open(femarg_t *, int, cred_t *, caller_context_t *);
1035 extern int deleg_wr_open(femarg_t *, int, cred_t *, caller_context_t *);
1036 extern int deleg_wr_read(femarg_t *, uio_t *, int, cred_t *,
1037 	    caller_context_t *);
1038 extern int deleg_rd_write(femarg_t *, uio_t *, int, cred_t *,
1039 	    caller_context_t *);
1040 extern int deleg_wr_write(femarg_t *, uio_t *, int, cred_t *,
1041 	    caller_context_t *);
1042 extern int deleg_rd_setattr(femarg_t *, vattr_t *, int, cred_t *,
1043 		caller_context_t *);
1044 extern int deleg_wr_setattr(femarg_t *, vattr_t *, int, cred_t *,
1045 		caller_context_t *);
1046 extern int deleg_rd_rwlock(femarg_t *, int, caller_context_t *);
1047 extern int deleg_wr_rwlock(femarg_t *, int, caller_context_t *);
1048 extern int deleg_rd_space(femarg_t *, int, flock64_t *, int, offset_t, cred_t *,
1049 		caller_context_t *);
1050 extern int deleg_wr_space(femarg_t *, int, flock64_t *, int, offset_t, cred_t *,
1051 		caller_context_t *);
1052 extern int deleg_rd_setsecattr(femarg_t *, vsecattr_t *, int, cred_t *,
1053 		caller_context_t *);
1054 extern int deleg_wr_setsecattr(femarg_t *, vsecattr_t *, int, cred_t *,
1055 		caller_context_t *);
1056 extern int deleg_rd_vnevent(femarg_t *, vnevent_t, vnode_t *, char *,
1057 		caller_context_t *);
1058 extern int deleg_wr_vnevent(femarg_t *, vnevent_t, vnode_t *, char *,
1059 		caller_context_t *);
1060 
1061 extern void rfs4_mon_hold(void *);
1062 extern void rfs4_mon_rele(void *);
1063 
1064 extern fem_t	*deleg_rdops;
1065 extern fem_t	*deleg_wrops;
1066 
1067 extern int rfs4_share(rfs4_state_t *, uint32_t, uint32_t);
1068 extern int rfs4_unshare(rfs4_state_t *);
1069 extern void rfs4_set_deleg_policy(nfs4_srv_t *, srv_deleg_policy_t);
1070 extern void rfs4_hold_deleg_policy(nfs4_srv_t *);
1071 extern void rfs4_rele_deleg_policy(nfs4_srv_t *);
1072 
1073 #ifdef DEBUG
1074 #define	NFS4_DEBUG(var, args) if (var) cmn_err args
1075 
1076 extern int rfs4_debug;
1077 extern int nfs4_client_attr_debug;
1078 extern int nfs4_client_state_debug;
1079 extern int nfs4_client_shadow_debug;
1080 extern int nfs4_client_lock_debug;
1081 extern int nfs4_client_lease_debug;
1082 extern int nfs4_seqid_sync;
1083 extern int nfs4_client_map_debug;
1084 extern int nfs4_client_inactive_debug;
1085 extern int nfs4_client_recov_debug;
1086 extern int nfs4_client_failover_debug;
1087 extern int nfs4_client_call_debug;
1088 extern int nfs4_client_foo_debug;
1089 extern int nfs4_client_zone_debug;
1090 extern int nfs4_lost_rqst_debug;
1091 extern int nfs4_open_stream_debug;
1092 extern int nfs4_client_open_dg;
1093 extern int nfs4_srvmnt_debug;
1094 extern int nfs4_utf8_debug;
1095 
1096 void rfs4_dbe_debug(rfs4_dbe_t *e);
1097 
1098 #ifdef NFS4_DEBUG_MUTEX
1099 void nfs4_debug_mutex_enter(kmutex_t *, char *, int);
1100 void nfs4_debug_mutex_exit(kmutex_t *, char *, int);
1101 
1102 #define	mutex_enter(m) nfs4_debug_mutex_enter((m), __FILE__, __LINE__)
1103 #define	mutex_exit(m) nfs4_debug_mutex_exit((m), __FILE__, __LINE__)
1104 #endif /* NFS4_DEBUG_MUTEX */
1105 
1106 #else  /* ! DEBUG */
1107 #define	NFS4_DEBUG(var, args)
1108 #endif /* DEBUG */
1109 
1110 /*
1111  * XXX - temporary for testing of volatile fh
1112  */
1113 
1114 #ifdef VOLATILE_FH_TEST
1115 
1116 struct nfs_fh4_fmt {
1117 	fhandle4_t	fh4_i;
1118 	uint32_t	fh4_flag;
1119 	uint32_t	fh4_volatile_id;
1120 };
1121 
1122 #else /* VOLATILE_FH_TEST */
1123 
1124 struct nfs_fh4_fmt {
1125 	fhandle4_t	fh4_i;
1126 	uint32_t	fh4_flag;
1127 };
1128 
1129 #endif /* VOLATILE_FH_TEST */
1130 
1131 #define	FH4_NAMEDATTR	1
1132 #define	FH4_ATTRDIR	2
1133 
1134 #define	fh4_fsid	fh4_i.fhx_fsid
1135 #define	fh4_len		fh4_i.fhx_len	/* fid length */
1136 #define	fh4_data	fh4_i.fhx_data	/* fid bytes */
1137 #define	fh4_xlen	fh4_i.fhx_xlen
1138 #define	fh4_xdata	fh4_i.fhx_xdata
1139 typedef struct nfs_fh4_fmt nfs_fh4_fmt_t;
1140 
1141 #define	fh4_to_fmt4(fh4p) ((nfs_fh4_fmt_t *)(fh4p)->nfs_fh4_val)
1142 #define	get_fh4_flag(fh4p, flag) ((fh4_to_fmt4(fh4p)->fh4_flag) & (flag))
1143 #define	set_fh4_flag(fh4p, flag) ((fh4_to_fmt4(fh4p)->fh4_flag) |= (flag))
1144 #define	clr_fh4_flag(fh4p, flag) ((fh4_to_fmt4(fh4p)->fh4_flag) &= ~(flag))
1145 
1146 #define	NFS_FH4_LEN	sizeof (nfs_fh4_fmt_t)
1147 
1148 /*
1149  * Copy fields from external (fhandle_t) to in-memory (nfs_fh4_fmt_t)
1150  * format to support export info checking.  It does not copy over
1151  * the complete filehandle, just the fsid, xlen and xdata.  It may
1152  * need to be changed to be used in other places.
1153  *
1154  * NOTE: The macro expects the space to be  pre-allocated for
1155  * the contents of nfs_fh4_fmt_t.
1156  */
1157 #define	FH_TO_FMT4(exifh, nfs_fmt) {				\
1158 	bzero((nfs_fmt), NFS_FH4_LEN);				\
1159 	(nfs_fmt)->fh4_fsid = (exifh)->fh_fsid;			\
1160 	(nfs_fmt)->fh4_xlen = (exifh)->fh_xlen;			\
1161 	bcopy((exifh)->fh_xdata, (nfs_fmt)->fh4_xdata,		\
1162 	    (exifh)->fh_xlen);					\
1163 }
1164 
1165 /*
1166  * A few definitions of repeatedly used constructs for nfsv4
1167  */
1168 #define	UTF8STRING_FREE(str)	{				\
1169 	kmem_free((str).utf8string_val,	(str).utf8string_len);	\
1170 	(str).utf8string_val = NULL;				\
1171 	(str).utf8string_len = 0;				\
1172 }
1173 
1174 /*
1175  * NFS4_VOLATILE_FH yields non-zero if the filesystem uses non-persistent
1176  * filehandles.
1177  */
1178 #define	NFS4_VOLATILE_FH(mi)					\
1179 	((mi)->mi_fh_expire_type &				\
1180 	(FH4_VOLATILE_ANY | FH4_VOL_MIGRATION | FH4_VOL_RENAME))
1181 
1182 /*
1183  * NFS_IS_DOTNAME checks if the name given represents a dot or dotdot entry
1184  */
1185 #define	NFS_IS_DOTNAME(name)					\
1186 	(((name)[0] == '.') &&					\
1187 	(((name)[1] == '\0') || (((name)[1] == '.') && ((name)[2] == '\0'))))
1188 
1189 /*
1190  * Define the number of bits in a bitmap word (uint32)
1191  */
1192 #define	NFS4_BITMAP4_BITSPERWORD	(sizeof (uint32_t) * 8)
1193 
1194 /*
1195  * Define the value for the access field of the compound_state structure
1196  * based on the result of nfsauth access checking.
1197  */
1198 #define	CS_ACCESS_OK		0x1
1199 #define	CS_ACCESS_DENIED	0x2
1200 #define	CS_ACCESS_LIMITED	0x4
1201 
1202 /*
1203  * compound state in nfsv4 server
1204  */
1205 struct compound_state {
1206 	struct exportinfo *exi;
1207 	struct exportinfo *saved_exi;	/* export struct for saved_vp */
1208 	cred_t		*basecr;	/* UNIX cred:  only RPC request */
1209 	caddr_t		principal;
1210 	int		nfsflavor;
1211 	cred_t		*cr;		/* UNIX cred: RPC request and */
1212 					/* target export */
1213 	bool_t		cont;
1214 	uint_t		access;		/* access perm on vp per request */
1215 	bool_t		deleg;		/* TRUE if current fh has */
1216 					/* write delegated */
1217 	vnode_t		*vp;		/* modified by PUTFH, and by ops that */
1218 					/* input to GETFH */
1219 	bool_t		mandlock;	/* Is mandatory locking in effect */
1220 					/* for vp */
1221 	vnode_t		*saved_vp;	/* modified by SAVEFH, copied to */
1222 					/* vp by RESTOREFH */
1223 	nfsstat4	*statusp;
1224 	nfs_fh4		fh;		/* ditto. valid only if vp != NULL */
1225 	nfs_fh4		saved_fh;	/* ditto. valid only if */
1226 					/*	saved_vp != NULL */
1227 	struct svc_req	*req;
1228 	char		fhbuf[NFS4_FHSIZE];
1229 
1230 	/* NFSv4.1 */
1231 	uint8_t		minorversion;	/* NFS4 minor version */
1232 	rfs4_session_t	*sp;		/* OP_SEQUENCE set it */
1233 	slotid4		slotno;
1234 	rfs4_slot_t	*slot;
1235 	rfs4_client_t	*client;
1236 	uint16_t	op_pos;
1237 	uint16_t	op_len;		/* number operations in compound req */
1238 #define	RFS4_DISPATCH_DONE	(1 << 0)
1239 	uint8_t		cs_flags;
1240 	bool_t		cachethis;
1241 	COMPOUND4res	*cmpresp;
1242 };
1243 
1244 typedef struct compound_state compound_state_t;
1245 
1246 static inline bool_t
rfs4_has_session(const compound_state_t * cs)1247 rfs4_has_session(const compound_state_t *cs)
1248 {
1249 	return (cs->slot != NULL);
1250 }
1251 
1252 /*
1253  * Conversion commands for nfsv4 server attr checking
1254  */
1255 enum nfs4_attr_cmd {
1256 	NFS4ATTR_SUPPORTED = 0,		/* check which attrs supported */
1257 	NFS4ATTR_GETIT = 1,		/* getattr - sys to fattr4 (r) */
1258 	NFS4ATTR_SETIT = 2,		/* setattr - fattr4 to sys (w) */
1259 	NFS4ATTR_VERIT = 3,		/* verify - fattr4 to sys (r) */
1260 	NFS4ATTR_FREEIT = 4		/* free any alloc'd space for attr */
1261 };
1262 
1263 typedef enum nfs4_attr_cmd nfs4_attr_cmd_t;
1264 
1265 struct nfs4_svgetit_arg {
1266 	nfs4_attr_cmd_t op;		/* getit or setit */
1267 	struct compound_state *cs;
1268 	struct statvfs64 *sbp;
1269 	uint_t		flag;		/* VOP_GETATTR/VOP_SETATTR flag */
1270 	uint_t		xattr;		/* object is xattr */
1271 	bool_t		rdattr_error_req; /* if readdir & client wants */
1272 						/* rdattr_error */
1273 	nfsstat4	rdattr_error;	/* used for per-entry status */
1274 					/* (if rdattr_err) */
1275 	bool_t		is_referral;	/* because sometimes we tell lies */
1276 	bool_t		mntdfid_set;
1277 
1278 	fattr4_mounted_on_fileid
1279 			mounted_on_fileid;
1280 					/* readdir op can always return	*/
1281 					/* d_ino from server fs dirent  */
1282 					/* for mounted_on_fileid attr.	*/
1283 					/* This field holds d_ino so	*/
1284 					/* srv attr conv code can avoid */
1285 					/* doing an untraverse.		*/
1286 	vattr_t		vap[1];
1287 };
1288 
1289 struct nfs4_ntov_map {
1290 	bitmap4		fbit;		/* FATTR4_XXX_MASKY */
1291 	uint_t		vbit;		/* AT_XXX */
1292 	bool_t		vfsstat;
1293 	bool_t		mandatory;	/* attribute mandatory to implement? */
1294 	uint_t		nval;
1295 	int		xdr_size;	/* Size of XDR'd attr */
1296 	xdrproc_t	xfunc;
1297 	int (*sv_getit)(nfs4_attr_cmd_t, struct nfs4_svgetit_arg *,
1298 		union nfs4_attr_u *);	/* subroutine for getting attr. */
1299 	char		*prtstr;	/* string attr for printing */
1300 };
1301 
1302 struct nfs4attr_to_vattr {
1303 	vnode_t		*vp;
1304 	vattr_t		*vap;
1305 	nfs_fh4		*fhp;
1306 	nfsstat4	rdattr_error;
1307 	uint32_t	flag;
1308 	fattr4_change	change;
1309 	fattr4_fsid	srv_fsid;
1310 	fattr4_mounted_on_fileid	mntd_fid;
1311 };
1312 
1313 typedef struct nfs4attr_to_vattr ntov4_t;
1314 
1315 /*
1316  * nfs4attr_to_vattr flags
1317  */
1318 #define	NTOV_FHP_VALID			0x01
1319 #define	NTOV_RDATTR_ERROR_VALID		0x02
1320 #define	NTOV_CHANGE_VALID		0x04
1321 #define	NTOV_SUPP_VALID			0x08
1322 #define	NTOV_SRV_FSID_VALID		0x10
1323 #define	NTOV_MOUNTED_ON_FILEID_VALID	0x20
1324 
1325 
1326 #define	FATTR4_MANDATTR_MASK (		\
1327 	FATTR4_SUPPORTED_ATTRS_MASK |	\
1328 	FATTR4_TYPE_MASK |		\
1329 	FATTR4_FH_EXPIRE_TYPE_MASK |	\
1330 	FATTR4_CHANGE_MASK |		\
1331 	FATTR4_SIZE_MASK |		\
1332 	FATTR4_LINK_SUPPORT_MASK |	\
1333 	FATTR4_SYMLINK_SUPPORT_MASK |	\
1334 	FATTR4_NAMED_ATTR_MASK |	\
1335 	FATTR4_FSID_MASK |		\
1336 	FATTR4_UNIQUE_HANDLES_MASK |	\
1337 	FATTR4_LEASE_TIME_MASK |	\
1338 	FATTR4_RDATTR_ERROR_MASK |	\
1339 	FATTR4_FILEHANDLE_MASK)
1340 
1341 
1342 struct nfs4attr_to_osattr {
1343 	void *attrconv_arg;
1344 	uint_t mask;
1345 };
1346 
1347 struct mntinfo4;
1348 
1349 /*
1350  * lkp4_attr_setup lists the different options for attributes when calling
1351  * nfs4lookup_setup - either no attributes (just lookups - e.g., secinfo),
1352  * one component only (normal component lookup), get attributes for the
1353  * last component (e.g., mount), attributes for each component (e.g.,
1354  * failovers later), just the filehandle for the last component (e.g.,
1355  * volatile filehandle recovery), or stuff that needs OPENATTR (e.g.
1356  * looking up a named attribute or it's hidden directory).
1357  */
1358 enum lkp4_attr_setup {
1359 	LKP4_NO_ATTRIBUTES = 0,		/* no attrs or filehandles */
1360 	LKP4_ALL_ATTRIBUTES = 3,	/* multi-comp: attrs for all comps */
1361 	LKP4_LAST_NAMED_ATTR = 5,	/* multi-comp: named attr & attrdir */
1362 	LKP4_LAST_ATTRDIR = 6,		/* multi-comp: just attrdir */
1363 	LKP4_ALL_ATTR_SECINFO = 7	/* multi-comp: attrs for all comp and */
1364 					/*	secinfo for last comp */
1365 };
1366 
1367 /*
1368  * lookup4_param a set of parameters to nfs4lookup_setup -
1369  * used to setup a path lookup compound request.
1370  */
1371 typedef struct lookup4_param {
1372 	enum lkp4_attr_setup l4_getattrs; /* (in) get attrs in the lookup? */
1373 	int		header_len;	/* (in) num ops before first lookup  */
1374 	int		trailer_len;	/* (in) num ops after last	*/
1375 					/*	Lookup/Getattr		*/
1376 	bitmap4		ga_bits;	/* (in) Which attributes for Getattr */
1377 	COMPOUND4args_clnt *argsp;	/* (in/out) args for compound struct */
1378 	COMPOUND4res_clnt  *resp;	/* (in/out) res for compound  struct */
1379 	int		arglen;		/* (out) argop buffer alloc'd length */
1380 	struct mntinfo4 *mi;
1381 } lookup4_param_t;
1382 
1383 
1384 #define	NFS4_FATTR4_FINISH	-1	/* fattr4 index indicating finish */
1385 
1386 typedef int (*nfs4attr_to_os_t)(int, union nfs4_attr_u *,
1387 		struct nfs4attr_to_osattr *);
1388 
1389 /*
1390  * The nfs4_error_t is the basic structure to return error values
1391  * from rfs4call.  It encapsulates the unix errno
1392  * value, the nfsstat4 value and the rpc status value into a single
1393  * structure.
1394  *
1395  * If error is set, then stat is ignored and rpc_status may be
1396  * set if the error occurred as the result of a CLNT_CALL.  If
1397  * stat is set, then rpc request succeeded, error and
1398  * rpc_status are set to 0 and stat contains the result of
1399  * operation, NFS4_OK or one of the NFS4ERR_* values.
1400  *
1401  * Functions which want to generate errors independently from
1402  * rfs4call should set error to the desired errno value and
1403  * set stat and rpc_status to 0.  nfs4_error_init() is a
1404  * convenient function to do this.
1405  */
1406 typedef struct {
1407 	int		error;
1408 	nfsstat4	stat;
1409 	enum clnt_stat	rpc_status;
1410 } nfs4_error_t;
1411 
1412 /*
1413  * Shared functions
1414  */
1415 extern void	rfs4_op_readdir(nfs_argop4 *, nfs_resop4 *,
1416 			struct svc_req *, struct compound_state *);
1417 extern void	nfs_fh4_copy(nfs_fh4 *, nfs_fh4 *);
1418 
1419 extern void	nfs4_fattr4_free(fattr4 *);
1420 
1421 extern int	nfs4lookup_setup(char *, lookup4_param_t *, int);
1422 extern void	nfs4_getattr_otw_norecovery(vnode_t *,
1423 			nfs4_ga_res_t *, nfs4_error_t *, cred_t *, int);
1424 extern int	nfs4_getattr_otw(vnode_t *, nfs4_ga_res_t *, cred_t *, int);
1425 extern int	nfs4cmpfh(const nfs_fh4 *, const nfs_fh4 *);
1426 extern int	nfs4cmpfhandle(nfs4_fhandle_t *, nfs4_fhandle_t *);
1427 extern int	nfs4getattr(vnode_t *, struct vattr *, cred_t *);
1428 extern int	nfs4_waitfor_purge_complete(vnode_t *);
1429 extern int	nfs4_validate_caches(vnode_t *, cred_t *);
1430 extern int	nfs4init(int, char *);
1431 extern void	nfs4fini(void);
1432 extern int	nfs4_vfsinit(void);
1433 extern void	nfs4_vfsfini(void);
1434 
1435 extern void	nfs4_vnops_init(void);
1436 extern void	nfs4_vnops_fini(void);
1437 extern void	nfs_idmap_init(void);
1438 extern void	nfs_idmap_flush(int);
1439 extern void	nfs_idmap_fini(void);
1440 extern int	nfs4_rnode_init(void);
1441 extern int	nfs4_rnode_fini(void);
1442 extern int	nfs4_shadow_init(void);
1443 extern int	nfs4_shadow_fini(void);
1444 extern int	nfs4_acache_init(void);
1445 extern int	nfs4_acache_fini(void);
1446 extern int	nfs4_subr_init(void);
1447 extern int	nfs4_subr_fini(void);
1448 extern void	nfs4_acl_init(void);
1449 extern void	nfs4_acl_free_cache(vsecattr_t *);
1450 
1451 extern int	geterrno4(nfsstat4);
1452 extern nfsstat4	puterrno4(int);
1453 extern int	nfs4_need_to_bump_seqid(COMPOUND4res_clnt *);
1454 extern int	nfs4tsize(void);
1455 extern int	checkauth4(struct compound_state *, struct svc_req *);
1456 extern nfsstat4 call_checkauth4(struct compound_state *, struct svc_req *);
1457 extern int	is_exported_sec(int, struct exportinfo *);
1458 extern void	nfs4_vmask_to_nmask(uint_t, bitmap4 *);
1459 extern void	nfs4_vmask_to_nmask_set(uint_t, bitmap4 *);
1460 extern int	nfs_idmap_str_uid(utf8string *u8s, uid_t *, bool_t);
1461 extern int	nfs_idmap_str_gid(utf8string *u8s, gid_t *, bool_t);
1462 extern int	nfs_idmap_uid_str(uid_t, utf8string *u8s, bool_t);
1463 extern int	nfs_idmap_gid_str(gid_t gid, utf8string *u8s, bool_t);
1464 extern int	nfs4_time_ntov(nfstime4 *, timestruc_t *);
1465 extern int	nfs4_time_vton(timestruc_t *, nfstime4 *);
1466 extern char	*utf8_to_str(utf8string *, uint_t *, char *);
1467 extern char	*utf8_to_fn(utf8string *, uint_t *, char *);
1468 extern utf8string *str_to_utf8(char *, utf8string *);
1469 extern utf8string *utf8_copy(utf8string *, utf8string *);
1470 extern int	utf8_compare(const utf8string *, const utf8string *);
1471 extern nfsstat4	utf8_dir_verify(utf8string *);
1472 extern char	*utf8_strchr(utf8string *, const char);
1473 extern int	ln_ace4_cmp(nfsace4 *, nfsace4 *, int);
1474 extern int	vs_aent_to_ace4(vsecattr_t *, vsecattr_t *, int, int);
1475 extern int	vs_ace4_to_aent(vsecattr_t *, vsecattr_t *, uid_t, gid_t,
1476     int, int);
1477 extern int	vs_ace4_to_acet(vsecattr_t *, vsecattr_t *, uid_t, gid_t,
1478     int);
1479 extern int	vs_acet_to_ace4(vsecattr_t *, vsecattr_t *, int);
1480 extern void	vs_acet_destroy(vsecattr_t *);
1481 extern void	vs_ace4_destroy(vsecattr_t *);
1482 extern void	vs_aent_destroy(vsecattr_t *);
1483 
1484 extern int	vn_find_nfs_record(vnode_t *, nvlist_t **, char **, char **);
1485 extern int	vn_is_nfs_reparse(vnode_t *, cred_t *);
1486 extern fs_locations4 *fetch_referral(vnode_t *, cred_t *);
1487 extern char	*build_symlink(vnode_t *, cred_t *, size_t *);
1488 
1489 extern int	stateid4_cmp(stateid4 *, stateid4 *);
1490 
1491 extern vtype_t	nf4_to_vt[];
1492 
1493 extern struct nfs4_ntov_map nfs4_ntov_map[];
1494 extern uint_t nfs4_ntov_map_size;
1495 
1496 extern struct vfsops	*nfs4_vfsops;
1497 extern struct vnodeops	*nfs4_vnodeops;
1498 extern const struct	fs_operation_def nfs4_vnodeops_template[];
1499 extern vnodeops_t	*nfs4_trigger_vnodeops;
1500 extern const struct	fs_operation_def nfs4_trigger_vnodeops_template[];
1501 
1502 extern uint_t nfs4_tsize(struct knetconfig *);
1503 extern uint_t rfs4_tsize(struct svc_req *);
1504 
1505 extern bool_t	xdr_inline_decode_nfs_fh4(uint32_t *, nfs_fh4_fmt_t *,
1506 			uint32_t);
1507 extern bool_t	xdr_inline_encode_nfs_fh4(uint32_t **, uint32_t *,
1508 			nfs_fh4_fmt_t *);
1509 
1510 #ifdef DEBUG
1511 extern int		rfs4_do_pre_op_attr;
1512 extern int		rfs4_do_post_op_attr;
1513 #endif
1514 
1515 extern stateid4 clnt_special0;
1516 extern stateid4 clnt_special1;
1517 #define	CLNT_ISSPECIAL(id) (stateid4_cmp(id, &clnt_special0) || \
1518 				stateid4_cmp(id, &clnt_special1))
1519 
1520 /* State's functions */
1521 extern void rfs4_ss_clid(nfs4_srv_t *nsrv4, rfs4_client_t *);
1522 extern void rfs4_ss_chkclid(nfs4_srv_t *nsrv4, rfs4_client_t *);
1523 
1524 /* Declarations for nfs4.x */
1525 nfsstat4 do_rfs4_op_secinfo(struct compound_state *, char *, SECINFO4res *);
1526 
1527 /*
1528  * The NFS Version 4 service procedures.
1529  */
1530 
1531 extern void	rfs4_do_server_start(int, int, nfs4_minor_t, int);
1532 extern void	rfs4_compound(COMPOUND4args *, COMPOUND4res *,
1533 			compound_state_t *, struct svc_req *, int *);
1534 extern void rfs4_init_compound_state(struct compound_state *);
1535 extern void rfs4_fini_compound_state(struct compound_state *);
1536 
1537 struct rpcdisp;
1538 extern int rfs4_dispatch(struct rpcdisp *, struct svc_req *, SVCXPRT *, char *);
1539 extern void	rfs4_compound_free(COMPOUND4res *);
1540 extern void	rfs4_compound_flagproc(COMPOUND4args *, int *);
1541 
1542 extern void	rfs4_srvrinit(void);
1543 extern void	rfs4_srvrfini(void);
1544 extern void	rfs4_srv_zone_init(nfs_globals_t *);
1545 extern void	rfs4_srv_zone_fini(nfs_globals_t *);
1546 extern void	rfs4_state_g_init(void);
1547 extern void	rfs4_state_zone_init(nfs4_srv_t *);
1548 extern void	rfs4_state_g_fini(void);
1549 extern void	rfs4_state_zone_fini(void);
1550 extern nfs4_srv_t *nfs4_get_srv(void);
1551 
1552 #endif
1553 #ifdef	__cplusplus
1554 }
1555 #endif
1556 
1557 #endif /* _NFS4_H */
1558