1/*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22/*
23 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
24 * Use is subject to license terms.
25 */
26
27#ifndef	_ASYNCIO_H
28#define	_ASYNCIO_H
29
30#pragma ident	"%Z%%M%	%I%	%E% SMI"
31
32#ifdef	__cplusplus
33extern "C" {
34#endif
35
36#include <stdio.h>
37#include <stdlib.h>
38#include <unistd.h>
39#include <string.h>
40#include <errno.h>
41#include <sys/types.h>
42#include <sys/stat.h>
43#include <thread.h>
44#include <pthread.h>
45#include <setjmp.h>
46#include <signal.h>
47#include <siginfo.h>
48#include <aio.h>
49#include <limits.h>
50#include <ucontext.h>
51#include <sys/asynch.h>
52#include <sys/mman.h>
53
54#if !defined(_LP64)
55#define	AIOSTKSIZE	(64 * 1024)
56#else
57#define	AIOSTKSIZE	(128 * 1024)
58#endif
59
60#define	SIGAIOCANCEL		SIGLWP	/* special aio cancelation signal */
61
62#define	AIO_WAITN_MAXIOCBS	32768	/* max. iocbs per system call */
63
64/*
65 * Declare structure types.  The structures themselves are defined below.
66 */
67typedef struct aio_args		aio_args_t;
68typedef struct aio_lio		aio_lio_t;
69typedef struct notif_param	notif_param_t;
70typedef struct aio_req		aio_req_t;
71typedef struct aio_worker	aio_worker_t;
72typedef struct aio_hash		aio_hash_t;
73
74struct aio_args {
75	int 		fd;
76	caddr_t		buf;
77	size_t		bufsz;
78	offset_t	offset;
79};
80
81/*
82 * list head for UFS list I/O
83 */
84struct aio_lio {
85	mutex_t		lio_mutex;	/* list mutex */
86	cond_t		lio_cond_cv;	/* list notification for I/O done */
87	aio_lio_t	*lio_next;	/* pointer to next on freelist */
88	char		lio_mode;	/* LIO_WAIT/LIO_NOWAIT */
89	char		lio_canned;	/* lio was canceled */
90	char		lio_largefile;	/* largefile operation */
91	char		lio_waiting;	/* waiting in __lio_listio() */
92	int		lio_nent;	/* Number of list I/O's */
93	int		lio_refcnt;	/* outstanding I/O's */
94	int		lio_event;	/* Event number for notification */
95	int		lio_port;	/* Port number for notification */
96	int		lio_signo;	/* Signal number for notification */
97	union sigval	lio_sigval;	/* Signal parameter */
98	uintptr_t	lio_object;	/* for SIGEV_THREAD or SIGEV_PORT */
99	struct sigevent	*lio_sigevent;	/* Notification function and attr. */
100};
101
102/*
103 * Notification parameters
104 */
105struct notif_param {
106	int		np_signo;	/* SIGEV_SIGNAL */
107	int		np_port;	/* SIGEV_THREAD or SIGEV_PORT */
108	void		*np_user;
109	int		np_event;
110	uintptr_t	np_object;
111	int		np_lio_signo;	/* listio: SIGEV_SIGNAL */
112	int		np_lio_port;	/* listio: SIGEV_THREAD or SIGEV_PORT */
113	void		*np_lio_user;
114	int		np_lio_event;
115	uintptr_t	np_lio_object;
116};
117
118struct aio_req {
119	/*
120	 * fields protected by _aio_mutex lock.
121	 */
122	aio_req_t *req_link;		/* hash/freelist chain link */
123	/*
124	 * when req is on the doneq, then req_next is protected by
125	 * the _aio_mutex lock. when the req is on a work q, then
126	 * req_next is protected by a worker's work_qlock1 lock.
127	 */
128	aio_req_t *req_next;		/* request/done queue link */
129	aio_req_t *req_prev;		/* double linked list */
130	/*
131	 * fields protected by a worker's work_qlock1 lock.
132	 */
133	char		req_state;	/* AIO_REQ_QUEUED, ... */
134	/*
135	 * fields require no locking.
136	 */
137	char		req_type;	/* AIO_POSIX_REQ or not */
138	char		req_largefile;	/* largefile operation */
139	char		req_op;		/* AIOREAD, etc. */
140	aio_worker_t	*req_worker;	/* associate request with worker */
141	aio_result_t	*req_resultp;	/* address of result buffer */
142	aio_args_t	req_args;	/* arglist */
143	aio_lio_t	*req_head;	/* list head for LIO */
144	struct sigevent	req_sigevent;
145	void		*req_aiocbp;	/* ptr to aiocb or aiocb64 */
146	notif_param_t	req_notify;	/* notification parameters */
147};
148
149/* special lio type that destroys itself when lio refcnt becomes zero */
150#define	LIO_FSYNC	LIO_WAIT+1
151#define	LIO_DESTROY	LIO_FSYNC+1
152
153/* lio flags */
154#define	LIO_FSYNC_CANCELED	0x1
155
156/* values for aio_state */
157
158#define	AIO_REQ_QUEUED		1
159#define	AIO_REQ_INPROGRESS	2
160#define	AIO_REQ_CANCELED	3
161#define	AIO_REQ_DONE 		4
162#define	AIO_REQ_FREE		5
163#define	AIO_REQ_DONEQ 		6
164
165/* use KAIO in _aio_rw() */
166#define	AIO_NO_KAIO		0x0
167#define	AIO_KAIO		0x1
168#define	AIO_NO_DUPS		0x2
169
170#define	AIO_POSIX_REQ		0x1
171
172#define	CHECK			1
173#define	NOCHECK			2
174#define	CHECKED			3
175#define	USERAIO			4
176#define	USERAIO_DONE		5
177
178/* values for _aio_flags */
179
180/* if set, _aiodone() notifies aio_waitn about done requests */
181#define	AIO_WAIT_INPROGRESS	0x1
182/* if set, _aiodone() wakes up functions waiting for completed I/Os */
183#define	AIO_IO_WAITING		0x2
184#define	AIO_LIB_WAITN		0x4	/* aio_waitn in progress */
185#define	AIO_LIB_WAITN_PENDING	0x8	/* aio_waitn requests pending */
186
187/*
188 * Before a kaio() system call, the fd will be checked
189 * to ensure that kernel async. I/O is supported for this file.
190 * The only way to find out is if a kaio() call returns ENOTSUP,
191 * so the default will always be to try the kaio() call. Only in
192 * the specific instance of a kaio() call returning ENOTSUP
193 * will we stop submitting kaio() calls for that fd.
194 * If the fd is outside the array bounds, we will allow the kaio()
195 * call.
196 *
197 * The only way that an fd entry can go from ENOTSUP to supported
198 * is if that fd is freed up by a close(), and close will clear
199 * the entry for that fd.
200 *
201 * Each fd gets a bit in the array _kaio_supported[].
202 *
203 * uint32_t	_kaio_supported[MAX_KAIO_FDARRAY_SIZE];
204 *
205 * Array is MAX_KAIO_ARRAY_SIZE of 32-bit elements, for 8kb.
206 * If more than (MAX_KAIO_FDARRAY_SIZE * KAIO_FDARRAY_ELEM_SIZE)
207 * files are open, this can be expanded.
208 */
209
210#define	MAX_KAIO_FDARRAY_SIZE		2048
211#define	KAIO_FDARRAY_ELEM_SIZE		WORD_BIT	/* uint32_t */
212
213#define	MAX_KAIO_FDS	(MAX_KAIO_FDARRAY_SIZE * KAIO_FDARRAY_ELEM_SIZE)
214
215#define	VALID_FD(fdes)		((fdes) >= 0 && (fdes) < MAX_KAIO_FDS)
216
217#define	KAIO_SUPPORTED(fdes)						\
218	(!VALID_FD(fdes) || 						\
219		((_kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] &	\
220		(uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE))) == 0))
221
222#define	SET_KAIO_NOT_SUPPORTED(fdes)					\
223	if (VALID_FD(fdes))						\
224		_kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] |=	\
225		(uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE))
226
227#define	CLEAR_KAIO_SUPPORTED(fdes)					\
228	if (VALID_FD(fdes))						\
229		_kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] &=	\
230		~(uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE))
231
232struct aio_worker {
233	aio_worker_t *work_forw;	/* forward link in list of workers */
234	aio_worker_t *work_backw;	/* backwards link in list of workers */
235	mutex_t work_qlock1;		/* lock for work queue 1 */
236	cond_t work_idle_cv;		/* place to sleep when idle */
237	aio_req_t *work_head1;		/* head of work request queue 1 */
238	aio_req_t *work_tail1;		/* tail of work request queue 1 */
239	aio_req_t *work_next1;		/* work queue one's next pointer */
240	aio_req_t *work_prev1;		/* last request done from queue 1 */
241	aio_req_t *work_req;		/* active work request */
242	thread_t work_tid;		/* worker's thread-id */
243	int work_count1;		/* length of work queue one */
244	int work_done1;			/* number of requests done */
245	int work_minload1;		/* min length of queue */
246	int work_idleflg;		/* when set, worker is idle */
247	sigjmp_buf work_jmp_buf;	/* cancellation point */
248};
249
250struct aio_hash {			/* resultp hash table */
251	mutex_t		hash_lock;
252	aio_req_t	*hash_ptr;
253#if !defined(_LP64)
254	void		*hash_pad;	/* ensure sizeof (aio_hash_t) == 32 */
255#endif
256};
257
258extern aio_hash_t *_aio_hash;
259
260#define	HASHSZ			2048	/* power of 2 */
261#define	AIOHASH(resultp)	((((uintptr_t)(resultp) >> 17) ^ \
262				((uintptr_t)(resultp) >> 2)) & (HASHSZ - 1))
263#define	POSIX_AIO(x)		((x)->req_type == AIO_POSIX_REQ)
264
265extern int __uaio_init(void);
266extern void _kaio_init(void);
267extern intptr_t _kaio(int, ...);
268extern int _aiorw(int, caddr_t, int, offset_t, int, aio_result_t *, int);
269extern int _aio_rw(aiocb_t *, aio_lio_t *, aio_worker_t **, int, int);
270#if !defined(_LP64)
271extern int _aio_rw64(aiocb64_t *, aio_lio_t *, aio_worker_t **, int, int);
272#endif
273extern int _aio_create_worker(aio_req_t *, int);
274extern int _aio_cancel_req(aio_worker_t *, aio_req_t *, int *, int *);
275extern int aiocancel_all(int);
276extern void aio_panic(const char *);
277extern aio_req_t *_aio_hash_find(aio_result_t *);
278extern aio_req_t *_aio_hash_del(aio_result_t *);
279extern void _aio_req_mark_done(aio_req_t *);
280extern void _aio_waitn_wakeup(void);
281extern aio_worker_t *_aio_worker_alloc(void);
282extern void _aio_worker_free(void *);
283extern aio_req_t *_aio_req_alloc(void);
284extern void _aio_req_free(aio_req_t *);
285extern aio_lio_t *_aio_lio_alloc(void);
286extern void _aio_lio_free(aio_lio_t *);
287extern int _aio_idle(aio_worker_t *);
288extern void *_aio_do_request(void *);
289extern void *_aio_do_notify(void *);
290extern void _lio_remove(aio_req_t *);
291extern aio_req_t *_aio_req_remove(aio_req_t *);
292extern int _aio_get_timedelta(timespec_t *, timespec_t *);
293extern aio_result_t *_aio_req_done(void);
294extern void _aio_set_result(aio_req_t *, ssize_t, int);
295extern int _aio_sigev_thread_init(struct sigevent *);
296extern int _aio_sigev_thread(aiocb_t *);
297#if !defined(_LP64)
298extern int _aio_sigev_thread64(aiocb64_t *);
299#endif
300
301extern aio_worker_t *_kaiowp;		/* points to kaio cleanup thread */
302extern aio_worker_t *__workers_rw;	/* list of all rw workers */
303extern aio_worker_t *__nextworker_rw;	/* worker chosen for next rw request */
304extern int __rw_workerscnt;		/* number of rw workers */
305extern aio_worker_t *__workers_no;	/* list of all notification workers */
306extern aio_worker_t *__nextworker_no;	/* worker chosen, next notification */
307extern int __no_workerscnt;		/* number of notification workers */
308extern mutex_t __aio_initlock;		/* makes aio initialization atomic */
309extern cond_t __aio_initcv;
310extern int __aio_initbusy;
311extern mutex_t __aio_mutex;		/* global aio lock */
312extern cond_t _aio_iowait_cv;		/* wait for userland I/Os */
313extern cond_t _aio_waitn_cv;		/* wait for end of aio_waitn */
314extern int _max_workers;		/* max number of workers permitted */
315extern int _min_workers;		/* min number of workers */
316extern sigset_t _worker_set;		/* worker's signal mask */
317extern int _aio_worker_cnt;		/* number of AIO workers */
318extern int _sigio_enabled;		/* when set, send SIGIO signal */
319extern pid_t __pid;			/* process's PID */
320extern int __uaio_ok;			/* indicates if aio is initialized */
321extern int _kaio_ok;			/* indicates if kaio is initialized */
322extern pthread_key_t _aio_key;		/* for thread-specific data */
323extern aio_req_t *_aio_done_tail;	/* list of done requests */
324extern aio_req_t *_aio_done_head;
325extern aio_req_t *_aio_doneq;
326extern int _aio_freelist_cnt;
327extern int _aio_allocated_cnt;
328extern int _aio_donecnt;
329extern int _aio_doneq_cnt;
330extern int _aio_waitncnt;		/* # of requests for aio_waitn */
331extern int _aio_outstand_cnt;		/* # of outstanding requests */
332extern int _kaio_outstand_cnt;		/* # of outstanding kaio requests */
333extern int _aio_req_done_cnt;		/* req. done but not in "done queue" */
334extern int _aio_kernel_suspend;		/* active kernel kaio calls */
335extern int _aio_suscv_cnt;		/* aio_suspend calls waiting on cv's */
336extern int _aiowait_flag;		/* when set, aiowait() is inprogress */
337extern int _aio_flags;			/* see defines, above */
338extern uint32_t *_kaio_supported;
339
340extern const sigset_t maskset;		/* all maskable signals */
341
342#ifdef	__cplusplus
343}
344#endif
345
346#endif	/* _ASYNCIO_H */
347