1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License, Version 1.0 only
6 * (the "License").  You may not use this file except in compliance
7 * with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or http://www.opensolaris.org/os/licensing.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22/*
23 * Copyright 1999-2003 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#ifndef _RSM_IN_H
28#define	_RSM_IN_H
29
30#pragma ident	"%Z%%M%	%I%	%E% SMI"
31
32#ifdef	__cplusplus
33extern "C" {
34#endif
35
36#include <sys/ddi.h>
37#include <sys/sunddi.h>
38#include <sys/rsm/rsm.h>
39#include <sys/rsm/rsmpi.h>
40
41#define	DRIVER_NAME	"rsm"
42
43
44#define	RSM_DRIVER_MINOR	0
45
46#define	RSM_CNUM	8
47
48#define	RSMIPC_SZ	10	/* number of outstanding requests, max: 256 */
49
50#define	RSMIPC_MAX_MESSAGES	64 /* max msgs that receiver can buffer */
51#define	RSMIPC_LOTSFREE_MSGBUFS	16 /* chunks of credits sent to sender  */
52
53/*
54 * The base for Sun RSMAPI Kernel Agent service idenitifiers is RSM_INTR_T_KA
55 * as defined below. This is as per the RSMPI specification. Thus,
56 * in the kernel agent, we need to use this value as the service identifier
57 * while registering the service handlers.
58 */
59#define	RSM_INTR_T_KA	0x88
60#define	RSM_SERVICE	RSM_INTR_T_KA
61
62#define	RSM_PRI		2
63#define	RSM_QUEUE_SZ	256
64
65#define	RSM_LOCK	0
66#define	RSM_NOLOCK	1
67
68#define	RSM_MAX_NUM_SEG	4095	/* default value for max imp and exp segs */
69
70#define	RSM_MAX_NODE	64	/* maximum number of nodes in the cluster */
71
72#define	RSM_MAX_CTRL	32	/* maximum number of controllers per node */
73
74/*
75 * The following defines UINT_MAX rounded down to a page aligned value.
76 */
77#define	RSM_MAXSZ_PAGE_ALIGNED (UINT_MAX & PAGEMASK)
78/*
79 * Define TRASHSIZE as the maximum possible size which is page aligned
80 * This value cannot be 0xffffffffffffe000 since this is taken as a
81 * negative value in the devmap_umem_remap call, thus causing the call
82 * to fail.
83 */
84#define	TRASHSIZE 0x7fffffffffffe000
85
86#define	RSM_ACCESS_READ				0444
87#define	RSM_ACCESS_WRITE			0222
88#define	RSM_ACCESS_TRUSTED			0666
89
90/* flag values for rsmseg_unload */
91#define	DISCONNECT	1
92#define	NO_DISCONNECT	0
93
94struct rsm_driver_data {
95	kmutex_t	drv_lock;
96	kcondvar_t	drv_cv;
97	int		drv_state;	/* RSM_DRV_YYYY states */
98	int		drv_memdel_cnt; /* number of memdel callbacks */
99};
100
101/* rsm driver state */
102#define	RSM_DRV_NEW			0
103#define	RSM_DRV_OK			1
104#define	RSM_DRV_PREDEL_STARTED		2
105#define	RSM_DRV_PREDEL_COMPLETED	3
106#define	RSM_DRV_POSTDEL_IN_PROGRESS	4
107#define	RSM_DRV_DR_IN_PROGRESS		5
108#define	RSM_DRV_REG_PROCESSING		6
109#define	RSM_DRV_UNREG_PROCESSING	7
110
111/* internal flags */
112#define	RSM_DR_QUIESCE		0
113#define	RSM_DR_UNQUIESCE	1
114
115typedef enum {
116	RSM_STATE_NEW = 0,
117	RSM_STATE_NEW_QUIESCED,
118	RSM_STATE_BIND,
119	RSM_STATE_BIND_QUIESCED,
120	RSM_STATE_EXPORT,
121	RSM_STATE_EXPORT_QUIESCING,
122	RSM_STATE_EXPORT_QUIESCED,
123	RSM_STATE_ZOMBIE,
124	RSM_STATE_CONNECTING,
125	RSM_STATE_ABORT_CONNECT,
126	RSM_STATE_CONNECT,
127	RSM_STATE_CONN_QUIESCE,
128	RSM_STATE_MAPPING,
129	RSM_STATE_ACTIVE,
130	RSM_STATE_MAP_QUIESCE,
131	RSM_STATE_DISCONNECT,
132	RSM_STATE_END
133} rsm_resource_state_t;
134
135typedef enum {
136	RSM_RESOURCE_EXPORT_SEGMENT,
137	RSM_RESOURCE_IMPORT_SEGMENT,
138	RSM_RESOURCE_BAR
139}rsm_resource_type_t;
140
141/*
142 * All resources have the only common info. whether it is a segment or
143 * a notification queue.
144 */
145typedef struct rsm_resource {
146	kmutex_t		rsmrc_lock;	/* sync on resource */
147	minor_t			rsmrc_num;	/* (minor) number */
148	rsm_memseg_id_t		rsmrc_key;	/* user key */
149	mode_t			rsmrc_mode;	/* access permission */
150	struct adapter		*rsmrc_adapter;	/* controller number */
151	rsm_node_id_t		rsmrc_node;	/*  nodeid */
152	rsm_resource_type_t	rsmrc_type;	/* type of this resource */
153	rsm_resource_state_t	rsmrc_state;	/* segment state */
154	struct rsm_resource	*rsmrc_next;
155} rsmresource_t;
156
157#define	RSMRC_BLKSZ	16
158#define	RSMRC_RESERVED	((rsmresource_t *)0x1)
159
160#define	RSM_HASHSZ	128
161
162#define	RSM_USER_MEMORY		0x1
163#define	RSM_KERNEL_MEMORY	0x2
164#define	RSM_EXPORT_WAIT		0x4
165#define	RSM_SEGMENT_POLL	0x8
166#define	RSM_FORCE_DISCONNECT	0x10
167#define	RSM_IMPORT_DUMMY	0x20
168/*
169 * The following macro is used within the kernel agent to indicate that
170 * rebind/unbind is allowed for an exported segment. It is a part of the
171 * segment's s_flags field.
172 */
173#define	RSMKA_ALLOW_UNBIND_REBIND	0x40
174#define	RSM_REPUBLISH_WAIT	0x80
175#define	RSM_DR_INPROGRESS	0x100
176#define	RSM_FORCE_DESTROY_WAIT	0x200
177#define	RSMKA_SET_RESOURCE_DONTWAIT	0x400
178
179#define	RSMRC_LOCK(p)	mutex_enter(&(p)->rsmrc_lock)
180#define	RSMRC_UNLOCK(p)	mutex_exit(&(p)->rsmrc_lock)
181#define	RSMRC_HELD(p)	MUTEX_HELD(&(p)->rsmrc_lock)
182#define	RSMRC_TRY(p)	mutex_tryenter(&(p)->rsmrc_lock)
183
184typedef struct rsm_region {
185	caddr_t		r_vaddr;	/* exported virtual address */
186	size_t		r_len;		/* length of export region */
187	offset_t	r_off;		/* offset of this region in segment */
188	struct as	*r_asp;
189	struct rsm_region *r_next;	/* next region of segment */
190}rsm_region;
191
192typedef struct rsm_cookie {
193	devmap_cookie_t		c_dhp;		/* devmap cookie handle */
194	offset_t		c_off;		/* offset of mapping	*/
195	size_t			c_len;		/* len of mapping	*/
196	struct rsm_cookie	*c_next;	/* next handle		*/
197}rsmcookie_t;
198
199typedef struct rsm_mapinfo {
200	dev_info_t	*dip;
201	uint_t		dev_register;
202	off_t		dev_offset;
203	off_t		start_offset;
204	size_t		individual_len;
205	struct rsm_mapinfo *next;
206} rsm_mapinfo_t;
207
208
209
210/*
211 * Shared Importer data structure
212 *
213 */
214typedef struct rsm_import_share {
215	kmutex_t	rsmsi_lock;	/* lock for shared importers	*/
216	kcondvar_t	rsmsi_cv;	/* condvar to wait at		*/
217	rsm_node_id_t	rsmsi_node;
218	rsm_memseg_id_t	rsmsi_segid;
219	size_t		rsmsi_seglen;
220	rsm_memseg_import_handle_t	rsmsi_handle; /* RSMPI handle */
221	uint_t		rsmsi_state;
222#define	RSMSI_STATE_NEW			0x0001
223#define	RSMSI_STATE_CONNECTING		0x0002
224#define	RSMSI_STATE_ABORT_CONNECT	0x0004
225#define	RSMSI_STATE_CONNECTED		0x0008
226#define	RSMSI_STATE_CONN_QUIESCE	0x0010
227#define	RSMSI_STATE_MAPPED		0x0020
228#define	RSMSI_STATE_MAP_QUIESCE		0x0040
229#define	RSMSI_STATE_DISCONNECTED	0x0080
230
231	uint_t		rsmsi_refcnt;	/* ref count of importers	*/
232	uint_t		rsmsi_mapcnt;	/* count of mapped importers	*/
233	mode_t		rsmsi_mode;	/* mode of last (re)publish	*/
234	uid_t		rsmsi_uid;
235	gid_t		rsmsi_gid;
236	rsm_mapinfo_t	*rsmsi_mapinfo;	/* register, offset, len values */
237	uint_t		rsmsi_flags;	/* flags			*/
238#define	RSMSI_FLAGS_ABORTDONE	0x0001	/* NOT_IMPORTING msg for abort conn */
239					/* has been sent		    */
240	void		*rsmsi_cookie;	/* cookie of the first seg connect */
241} rsm_import_share_t;
242
243#define	RSMSI_LOCK(sharep)	mutex_enter(&(sharep)->rsmsi_lock)
244#define	RSMSI_UNLOCK(sharep)	mutex_exit(&(sharep)->rsmsi_lock)
245#define	RSMSI_HELD(sharep)	MUTEX_HELD(&(sharep)->rsmsi_lock)
246#define	RSMSI_TRY(sharep)	mutex_tryenter(&(sharep)->rsmsi_lock)
247
248typedef struct rsm_seginfo {
249	rsmresource_t		s_hdr;		/* resource hdr */
250#define	s_state	s_hdr.rsmrc_state	/* segment state */
251#define	s_adapter s_hdr.rsmrc_adapter
252#define	s_node	s_hdr.rsmrc_node
253#define	s_lock	s_hdr.rsmrc_lock
254#define	s_minor	s_hdr.rsmrc_num		/* minor # of segment */
255#define	s_key	s_hdr.rsmrc_key		/* user segment key */
256#define	s_mode	s_hdr.rsmrc_mode	/* user segment mode */
257#define	s_type	s_hdr.rsmrc_type	/* segment type */
258	uid_t			s_uid;		/* owner id */
259	gid_t			s_gid;		/* owner id */
260
261	size_t			s_len;		/* total segment size */
262	rsm_region		s_region;	/* regions of segment */
263
264	int			s_flags;
265	int			s_pollflag;	/* indicates poll status */
266
267	kcondvar_t		s_cv;		/* condition to wait on */
268
269	rsm_memseg_id_t		s_segid;	/* NIC segment id */
270
271	int		s_acl_len;		/* length of access list */
272	rsmapi_access_entry_t *s_acl;		/* access list */
273	rsm_access_entry_t *s_acl_in;		/* access list with hwaddr */
274
275	struct pollhead	s_poll;
276	uint32_t	s_pollevent;
277	pid_t 		s_pid;
278
279	rsmcookie_t	*s_ckl;		/* list of devmap cookie */
280
281	size_t		s_total_maplen;
282	rsm_mapinfo_t	*s_mapinfo;	/* register, offset, len  */
283
284	union {
285		rsm_memseg_import_handle_t	in;
286		rsm_memseg_export_handle_t	out;
287	} s_handle;			/* NIC handle for segment */
288
289	/*
290	 * This field is used to indicate the cookie returned by the
291	 * ddi_umem_lock when binding pages for an export segment.
292	 * Also, for importers on the same node as the export segment,
293	 * this field indicates the cookie used during import mapping.
294	 */
295	ddi_umem_cookie_t	s_cookie;
296	rsm_import_share_t	*s_share;	/* shared importer data	    */
297	/*
298	 * This field in an import segments indicates the number of
299	 * putv/getv operations in progress and in an export segment
300	 * it is the number of putv/getv ops currently using it as
301	 * a handle in the iovec.
302	 */
303	uint_t			s_rdmacnt;
304	struct proc		*s_proc;
305} rsmseg_t;
306
307#define	rsmseglock_acquire(p)	RSMRC_LOCK((rsmresource_t *)(p))
308#define	rsmseglock_release(p)	RSMRC_UNLOCK((rsmresource_t *)(p))
309#define	rsmseglock_held(p)	RSMRC_HELD((rsmresource_t *)(p))
310#define	rsmseglock_try(p)	RSMRC_TRY((rsmresource_t *)(p))
311
312#define	rsmsharelock_acquire(p)	RSMSI_LOCK(p->s_share)
313#define	rsmsharelock_release(p)	RSMSI_UNLOCK(p->s_share)
314#define	rsmsharelock_held(p)	RSMSI_HELD(p->s_share)
315#define	rsmsharelock_try(p)	RSMSI_TRY(p->s_share)
316
317/*
318 * Resource elements structure
319 */
320typedef struct {
321	int		rsmrcblk_avail;
322	rsmresource_t	*rsmrcblk_blks[RSMRC_BLKSZ];
323}rsmresource_blk_t;
324
325struct rsmresource_table {
326	krwlock_t	rsmrc_lock;
327	int		rsmrc_len;
328	int		rsmrc_sz;
329	rsmresource_blk_t **rsmrc_root;
330};
331
332/*
333 * Struct for advertised resource list
334 */
335/*
336 * Hashtable structs
337 * bucket points to an array of pointers, each entry in the bucket array
338 * points to a linked list of resource items.
339 * bucket index = bucket_address%RSM_HASHSZ
340 */
341typedef struct rsmhash_table {
342	krwlock_t		rsmhash_rw;
343	rsmresource_t		**bucket;
344} rsmhash_table_t;
345
346/*
347 * Remote messaging related structure
348 */
349
350/*
351 * Flags for ipc slot
352 */
353#define	RSMIPC_FREE	0x1			/* slot is free */
354#define	RSMIPC_PENDING	0x2			/* slot has pending request */
355
356#define	RSMIPC_SET(x, v)	((x)->rsmipc_flags |= (v))
357#define	RSMIPC_GET(x, v)	((x)->rsmipc_flags & (v))
358#define	RSMIPC_CLEAR(x, v)	((x)->rsmipc_flags &= ~(v))
359
360typedef struct rsmipc_slot {
361	kmutex_t 	rsmipc_lock;		/* lock for remote msgs */
362	kcondvar_t	rsmipc_cv;		/* condition var to wait on */
363	int		rsmipc_flags;
364	rsmipc_cookie_t	rsmipc_cookie;		/* cookie of request in wire */
365	void 		*rsmipc_data;		/* ptr to data to copy */
366}rsmipc_slot_t;
367
368/*
369 * Messaging struc
370 */
371typedef struct {
372	kmutex_t	lock;
373	kcondvar_t	cv;
374	int		count;
375	int		wanted;
376	int		sequence;
377	rsmipc_slot_t	slots[RSMIPC_SZ];
378}rsm_ipc_t;
379
380/*
381 * These tokens are used for building the list of remote node importers
382 * of a segment exported from the local node
383 */
384typedef struct importing_token {
385	struct importing_token	*next;
386	rsm_memseg_id_t		key;
387	rsm_node_id_t		importing_node;
388	void			*import_segment_cookie;
389	rsm_addr_t		importing_adapter_hwaddr;
390} importing_token_t;
391
392typedef struct {
393	kmutex_t		lock;
394	importing_token_t	**bucket;
395} importers_table_t;
396
397/*
398 * Used by the rsm_send_republish() fn
399 */
400typedef struct republish_token {
401	struct republish_token	*next;
402	rsm_memseg_id_t		key;
403	rsm_node_id_t		importing_node;
404	rsm_permission_t	permission;
405} republish_token_t;
406
407/*
408 * data strucuture for list manipulation
409 */
410typedef struct list_element {
411	struct list_element	*next;
412	rsm_node_id_t		nodeid;
413	uint32_t		flags;
414#define	RSM_SUSPEND_ACKPENDING	0x01
415#define	RSM_SUSPEND_NODEDEAD	0x02
416} list_element_t;
417
418typedef struct list_head {
419	struct list_element	*list_head;
420	kmutex_t		list_lock;
421} list_head_t;
422
423#ifdef	__cplusplus
424}
425#endif
426
427#endif	/* _RSM_IN_H */
428