1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 *    notice, this list of conditions and the following disclaimer in the
14 *    documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 *
28 * $FreeBSD$
29 */
30
31#ifndef	_G_RAID_H_
32#define	_G_RAID_H_
33
34#include <sys/param.h>
35#include <sys/kobj.h>
36#include <sys/bio.h>
37#include <sys/time.h>
38#ifdef _KERNEL
39#include <sys/sysctl.h>
40#endif
41
42#define	G_RAID_CLASS_NAME	"RAID"
43
44#define	G_RAID_MAGIC		"GEOM::RAID"
45
46#define	G_RAID_VERSION		0
47
48struct g_raid_md_object;
49struct g_raid_tr_object;
50
51#define	G_RAID_DEVICE_FLAG_NOAUTOSYNC	0x0000000000000001ULL
52#define	G_RAID_DEVICE_FLAG_NOFAILSYNC	0x0000000000000002ULL
53#define	G_RAID_DEVICE_FLAG_MASK	(G_RAID_DEVICE_FLAG_NOAUTOSYNC | \
54					 G_RAID_DEVICE_FLAG_NOFAILSYNC)
55
56#ifdef _KERNEL
57extern u_int g_raid_aggressive_spare;
58extern u_int g_raid_debug;
59extern int g_raid_enable;
60extern int g_raid_read_err_thresh;
61extern u_int g_raid_start_timeout;
62extern struct g_class g_raid_class;
63
64#define	G_RAID_DEBUG(lvl, ...) \
65    _GEOM_DEBUG("GEOM_RAID", g_raid_debug, (lvl), NULL, __VA_ARGS__)
66#define	G_RAID_DEBUG1(lvl, sc, fmt, ...)				\
67    _GEOM_DEBUG("GEOM_RAID", g_raid_debug, (lvl), NULL, "%s: " fmt,	\
68	(sc)->sc_name, ## __VA_ARGS__)
69#define	G_RAID_LOGREQ(lvl, bp, ...) \
70    _GEOM_DEBUG("GEOM_RAID", g_raid_debug, (lvl), (bp), __VA_ARGS__)
71
72/*
73 * Flags we use to distinguish I/O initiated by the TR layer to maintain
74 * the volume's characteristics, fix subdisks, extra copies of data, etc.
75 *
76 * G_RAID_BIO_FLAG_SYNC		I/O to update an extra copy of the data
77 *				for RAID volumes that maintain extra data
78 *				and need to rebuild that data.
79 * G_RAID_BIO_FLAG_REMAP	I/O done to try to provoke a subdisk into
80 *				doing some desirable action such as bad
81 *				block remapping after we detect a bad part
82 *				of the disk.
83 * G_RAID_BIO_FLAG_LOCKED	I/O holds range lock that should re released.
84 *
85 * and the following meta item:
86 * G_RAID_BIO_FLAG_SPECIAL	And of the I/O flags that need to make it
87 *				through the range locking which would
88 *				otherwise defer the I/O until after that
89 *				range is unlocked.
90 */
91#define	G_RAID_BIO_FLAG_SYNC		0x01
92#define	G_RAID_BIO_FLAG_REMAP		0x02
93#define	G_RAID_BIO_FLAG_SPECIAL \
94		(G_RAID_BIO_FLAG_SYNC|G_RAID_BIO_FLAG_REMAP)
95#define	G_RAID_BIO_FLAG_LOCKED		0x80
96
97struct g_raid_lock {
98	off_t			 l_offset;
99	off_t			 l_length;
100	void			*l_callback_arg;
101	int			 l_pending;
102	LIST_ENTRY(g_raid_lock)	 l_next;
103};
104
105#define	G_RAID_EVENT_WAIT	0x01
106#define	G_RAID_EVENT_VOLUME	0x02
107#define	G_RAID_EVENT_SUBDISK	0x04
108#define	G_RAID_EVENT_DISK	0x08
109#define	G_RAID_EVENT_DONE	0x10
110struct g_raid_event {
111	void			*e_tgt;
112	int			 e_event;
113	int			 e_flags;
114	int			 e_error;
115	TAILQ_ENTRY(g_raid_event) e_next;
116};
117#define G_RAID_DISK_S_NONE		0x00	/* State is unknown. */
118#define G_RAID_DISK_S_OFFLINE		0x01	/* Missing disk placeholder. */
119#define G_RAID_DISK_S_DISABLED		0x02	/* Disabled. */
120#define G_RAID_DISK_S_FAILED		0x03	/* Failed. */
121#define G_RAID_DISK_S_STALE_FAILED	0x04	/* Old failed. */
122#define G_RAID_DISK_S_SPARE		0x05	/* Hot-spare. */
123#define G_RAID_DISK_S_STALE		0x06	/* Old disk, unused now. */
124#define G_RAID_DISK_S_ACTIVE		0x07	/* Operational. */
125
126#define G_RAID_DISK_E_DISCONNECTED	0x01
127
128struct g_raid_disk {
129	struct g_raid_softc	*d_softc;	/* Back-pointer to softc. */
130	struct g_consumer	*d_consumer;	/* GEOM disk consumer. */
131	void			*d_md_data;	/* Disk's metadata storage. */
132	int			 d_candelete;	/* BIO_DELETE supported. */
133	uint64_t		 d_flags;	/* Additional flags. */
134	u_int			 d_state;	/* Disk state. */
135	u_int			 d_load;	/* Disk average load. */
136	off_t			 d_last_offset;	/* Last head offset. */
137	int			 d_read_errs;	/* Count of the read errors */
138	TAILQ_HEAD(, g_raid_subdisk)	 d_subdisks; /* List of subdisks. */
139	TAILQ_ENTRY(g_raid_disk)	 d_next;	/* Next disk in the node. */
140	struct g_kerneldump	 d_kd;		/* Kernel dumping method/args. */
141};
142
143#define G_RAID_SUBDISK_S_NONE		0x00	/* Absent. */
144#define G_RAID_SUBDISK_S_FAILED		0x01	/* Failed. */
145#define G_RAID_SUBDISK_S_NEW		0x02	/* Blank. */
146#define G_RAID_SUBDISK_S_REBUILD	0x03	/* Blank + rebuild. */
147#define G_RAID_SUBDISK_S_UNINITIALIZED	0x04	/* Disk of the new volume. */
148#define G_RAID_SUBDISK_S_STALE		0x05	/* Dirty. */
149#define G_RAID_SUBDISK_S_RESYNC		0x06	/* Dirty + check/repair. */
150#define G_RAID_SUBDISK_S_ACTIVE		0x07	/* Usable. */
151
152#define G_RAID_SUBDISK_E_NEW		0x01	/* A new subdisk has arrived */
153#define G_RAID_SUBDISK_E_FAILED		0x02	/* A subdisk failed, but remains in volume */
154#define G_RAID_SUBDISK_E_DISCONNECTED	0x03	/* A subdisk removed from volume. */
155#define G_RAID_SUBDISK_E_FIRST_TR_PRIVATE 0x80	/* translation private events */
156
157#define G_RAID_SUBDISK_POS(sd)						\
158    ((sd)->sd_disk ? ((sd)->sd_disk->d_last_offset - (sd)->sd_offset) : 0)
159#define G_RAID_SUBDISK_TRACK_SIZE	(1 * 1024 * 1024)
160#define G_RAID_SUBDISK_LOAD(sd)						\
161    ((sd)->sd_disk ? ((sd)->sd_disk->d_load) : 0)
162#define G_RAID_SUBDISK_LOAD_SCALE	256
163
164struct g_raid_subdisk {
165	struct g_raid_softc	*sd_softc;	/* Back-pointer to softc. */
166	struct g_raid_disk	*sd_disk;	/* Where this subdisk lives. */
167	struct g_raid_volume	*sd_volume;	/* Volume, sd is a part of. */
168	off_t			 sd_offset;	/* Offset on the disk. */
169	off_t			 sd_size;	/* Size on the disk. */
170	u_int			 sd_pos;	/* Position in volume. */
171	u_int			 sd_state;	/* Subdisk state. */
172	off_t			 sd_rebuild_pos; /* Rebuild position. */
173	int			 sd_recovery;	/* Count of recovery reqs. */
174	TAILQ_ENTRY(g_raid_subdisk)	 sd_next; /* Next subdisk on disk. */
175};
176
177#define G_RAID_MAX_SUBDISKS	16
178#define G_RAID_MAX_VOLUMENAME	32
179
180#define G_RAID_VOLUME_S_STARTING	0x00
181#define G_RAID_VOLUME_S_BROKEN		0x01
182#define G_RAID_VOLUME_S_DEGRADED	0x02
183#define G_RAID_VOLUME_S_SUBOPTIMAL	0x03
184#define G_RAID_VOLUME_S_OPTIMAL		0x04
185#define G_RAID_VOLUME_S_UNSUPPORTED	0x05
186#define G_RAID_VOLUME_S_STOPPED		0x06
187
188#define G_RAID_VOLUME_S_ALIVE(s)			\
189    ((s) == G_RAID_VOLUME_S_DEGRADED ||			\
190     (s) == G_RAID_VOLUME_S_SUBOPTIMAL ||		\
191     (s) == G_RAID_VOLUME_S_OPTIMAL)
192
193#define G_RAID_VOLUME_E_DOWN		0x00
194#define G_RAID_VOLUME_E_UP		0x01
195#define G_RAID_VOLUME_E_START		0x10
196#define G_RAID_VOLUME_E_STARTMD		0x11
197
198#define G_RAID_VOLUME_RL_RAID0		0x00
199#define G_RAID_VOLUME_RL_RAID1		0x01
200#define G_RAID_VOLUME_RL_RAID3		0x03
201#define G_RAID_VOLUME_RL_RAID4		0x04
202#define G_RAID_VOLUME_RL_RAID5		0x05
203#define G_RAID_VOLUME_RL_RAID6		0x06
204#define G_RAID_VOLUME_RL_RAIDMDF	0x07
205#define G_RAID_VOLUME_RL_RAID1E		0x11
206#define G_RAID_VOLUME_RL_SINGLE		0x0f
207#define G_RAID_VOLUME_RL_CONCAT		0x1f
208#define G_RAID_VOLUME_RL_RAID5E		0x15
209#define G_RAID_VOLUME_RL_RAID5EE	0x25
210#define G_RAID_VOLUME_RL_RAID5R		0x35
211#define G_RAID_VOLUME_RL_UNKNOWN	0xff
212
213#define G_RAID_VOLUME_RLQ_NONE		0x00
214#define G_RAID_VOLUME_RLQ_R1SM		0x00
215#define G_RAID_VOLUME_RLQ_R1MM		0x01
216#define G_RAID_VOLUME_RLQ_R3P0		0x00
217#define G_RAID_VOLUME_RLQ_R3PN		0x01
218#define G_RAID_VOLUME_RLQ_R4P0		0x00
219#define G_RAID_VOLUME_RLQ_R4PN		0x01
220#define G_RAID_VOLUME_RLQ_R5RA		0x00
221#define G_RAID_VOLUME_RLQ_R5RS		0x01
222#define G_RAID_VOLUME_RLQ_R5LA		0x02
223#define G_RAID_VOLUME_RLQ_R5LS		0x03
224#define G_RAID_VOLUME_RLQ_R6RA		0x00
225#define G_RAID_VOLUME_RLQ_R6RS		0x01
226#define G_RAID_VOLUME_RLQ_R6LA		0x02
227#define G_RAID_VOLUME_RLQ_R6LS		0x03
228#define G_RAID_VOLUME_RLQ_RMDFRA	0x00
229#define G_RAID_VOLUME_RLQ_RMDFRS	0x01
230#define G_RAID_VOLUME_RLQ_RMDFLA	0x02
231#define G_RAID_VOLUME_RLQ_RMDFLS	0x03
232#define G_RAID_VOLUME_RLQ_R1EA		0x00
233#define G_RAID_VOLUME_RLQ_R1EO		0x01
234#define G_RAID_VOLUME_RLQ_R5ERA		0x00
235#define G_RAID_VOLUME_RLQ_R5ERS		0x01
236#define G_RAID_VOLUME_RLQ_R5ELA		0x02
237#define G_RAID_VOLUME_RLQ_R5ELS		0x03
238#define G_RAID_VOLUME_RLQ_R5EERA	0x00
239#define G_RAID_VOLUME_RLQ_R5EERS	0x01
240#define G_RAID_VOLUME_RLQ_R5EELA	0x02
241#define G_RAID_VOLUME_RLQ_R5EELS	0x03
242#define G_RAID_VOLUME_RLQ_R5RRA		0x00
243#define G_RAID_VOLUME_RLQ_R5RRS		0x01
244#define G_RAID_VOLUME_RLQ_R5RLA		0x02
245#define G_RAID_VOLUME_RLQ_R5RLS		0x03
246#define G_RAID_VOLUME_RLQ_UNKNOWN	0xff
247
248struct g_raid_volume;
249
250struct g_raid_volume {
251	struct g_raid_softc	*v_softc;	/* Back-pointer to softc. */
252	struct g_provider	*v_provider;	/* GEOM provider. */
253	struct g_raid_subdisk	 v_subdisks[G_RAID_MAX_SUBDISKS];
254						/* Subdisks of this volume. */
255	void			*v_md_data;	/* Volume's metadata storage. */
256	struct g_raid_tr_object	*v_tr;		/* Transformation object. */
257	char			 v_name[G_RAID_MAX_VOLUMENAME];
258						/* Volume name. */
259	u_int			 v_state;	/* Volume state. */
260	u_int			 v_raid_level;	/* Array RAID level. */
261	u_int			 v_raid_level_qualifier; /* RAID level det. */
262	u_int			 v_disks_count;	/* Number of disks in array. */
263	u_int			 v_mdf_pdisks;	/* Number of parity disks
264						   in RAIDMDF array. */
265	uint16_t		 v_mdf_polynomial; /* Polynomial for RAIDMDF. */
266	uint8_t			 v_mdf_method;	/* Generation method for RAIDMDF. */
267	u_int			 v_strip_size;	/* Array strip size. */
268	u_int			 v_rotate_parity; /* Rotate RAID5R parity
269						   after numer of stripes. */
270	u_int			 v_sectorsize;	/* Volume sector size. */
271	off_t			 v_mediasize;	/* Volume media size.  */
272	struct bio_queue_head	 v_inflight;	/* In-flight write requests. */
273	struct bio_queue_head	 v_locked;	/* Blocked I/O requests. */
274	LIST_HEAD(, g_raid_lock) v_locks;	 /* List of locked regions. */
275	int			 v_pending_lock; /* writes to locked region */
276	int			 v_dirty;	/* Volume is DIRTY. */
277	struct timeval		 v_last_done;	/* Time of the last I/O. */
278	time_t			 v_last_write;	/* Time of the last write. */
279	u_int			 v_writes;	/* Number of active writes. */
280	struct root_hold_token	*v_rootmount;	/* Root mount delay token. */
281	int			 v_starting;	/* Volume is starting */
282	int			 v_stopping;	/* Volume is stopping */
283	int			 v_provider_open; /* Number of opens. */
284	int			 v_global_id;	/* Global volume ID (rX). */
285	int			 v_read_only;	/* Volume is read-only. */
286	TAILQ_ENTRY(g_raid_volume)	 v_next; /* List of volumes entry. */
287	LIST_ENTRY(g_raid_volume)	 v_global_next; /* Global list entry. */
288};
289
290#define G_RAID_NODE_E_WAKE	0x00
291#define G_RAID_NODE_E_START	0x01
292
293struct g_raid_softc {
294	struct g_raid_md_object	*sc_md;		/* Metadata object. */
295	struct g_geom		*sc_geom;	/* GEOM class instance. */
296	uint64_t		 sc_flags;	/* Additional flags. */
297	TAILQ_HEAD(, g_raid_volume)	 sc_volumes;	/* List of volumes. */
298	TAILQ_HEAD(, g_raid_disk)	 sc_disks;	/* List of disks. */
299	struct sx		 sc_lock;	/* Main node lock. */
300	struct proc		*sc_worker;	/* Worker process. */
301	struct mtx		 sc_queue_mtx;	/* Worker queues lock. */
302	TAILQ_HEAD(, g_raid_event) sc_events;	/* Worker events queue. */
303	struct bio_queue_head	 sc_queue;	/* Worker I/O queue. */
304	int			 sc_stopping;	/* Node is stopping */
305};
306#define	sc_name	sc_geom->name
307
308SYSCTL_DECL(_kern_geom_raid);
309
310/*
311 * KOBJ parent class of metadata processing modules.
312 */
313struct g_raid_md_class {
314	KOBJ_CLASS_FIELDS;
315	int		 mdc_enable;
316	int		 mdc_priority;
317	LIST_ENTRY(g_raid_md_class) mdc_list;
318};
319
320/*
321 * KOBJ instance of metadata processing module.
322 */
323struct g_raid_md_object {
324	KOBJ_FIELDS;
325	struct g_raid_md_class	*mdo_class;
326	struct g_raid_softc	*mdo_softc;	/* Back-pointer to softc. */
327};
328
329int g_raid_md_modevent(module_t, int, void *);
330
331#define	G_RAID_MD_DECLARE(name, label)				\
332    static moduledata_t g_raid_md_##name##_mod = {		\
333	"g_raid_md_" __XSTRING(name),				\
334	g_raid_md_modevent,					\
335	&g_raid_md_##name##_class				\
336    };								\
337    DECLARE_MODULE(g_raid_md_##name, g_raid_md_##name##_mod,	\
338	SI_SUB_DRIVERS, SI_ORDER_SECOND);			\
339    MODULE_DEPEND(g_raid_md_##name, geom_raid, 0, 0, 0);	\
340    SYSCTL_NODE(_kern_geom_raid, OID_AUTO, name,		\
341        CTLFLAG_RD | CTLFLAG_MPSAFE,				\
342	NULL, label " metadata module");			\
343    SYSCTL_INT(_kern_geom_raid_##name, OID_AUTO, enable,	\
344	CTLFLAG_RWTUN, &g_raid_md_##name##_class.mdc_enable, 0,	\
345	"Enable " label " metadata format taste")
346
347/*
348 * KOBJ parent class of data transformation modules.
349 */
350struct g_raid_tr_class {
351	KOBJ_CLASS_FIELDS;
352	int		 trc_enable;
353	int		 trc_priority;
354	int		 trc_accept_unmapped;
355	LIST_ENTRY(g_raid_tr_class) trc_list;
356};
357
358/*
359 * KOBJ instance of data transformation module.
360 */
361struct g_raid_tr_object {
362	KOBJ_FIELDS;
363	struct g_raid_tr_class	*tro_class;
364	struct g_raid_volume 	*tro_volume;	/* Back-pointer to volume. */
365};
366
367int g_raid_tr_modevent(module_t, int, void *);
368
369#define	G_RAID_TR_DECLARE(name, label)				\
370    static moduledata_t g_raid_tr_##name##_mod = {		\
371	"g_raid_tr_" __XSTRING(name),				\
372	g_raid_tr_modevent,					\
373	&g_raid_tr_##name##_class				\
374    };								\
375    DECLARE_MODULE(g_raid_tr_##name, g_raid_tr_##name##_mod,	\
376	SI_SUB_DRIVERS, SI_ORDER_FIRST);			\
377    MODULE_DEPEND(g_raid_tr_##name, geom_raid, 0, 0, 0);	\
378    SYSCTL_NODE(_kern_geom_raid, OID_AUTO, name,		\
379        CTLFLAG_RD | CTLFLAG_MPSAFE,				\
380	NULL, label " transformation module");			\
381    SYSCTL_INT(_kern_geom_raid_##name, OID_AUTO, enable,	\
382	CTLFLAG_RWTUN, &g_raid_tr_##name##_class.trc_enable, 0,	\
383	"Enable " label " transformation module taste")
384
385const char * g_raid_volume_level2str(int level, int qual);
386int g_raid_volume_str2level(const char *str, int *level, int *qual);
387const char * g_raid_volume_state2str(int state);
388const char * g_raid_subdisk_state2str(int state);
389const char * g_raid_disk_state2str(int state);
390
391struct g_raid_softc * g_raid_create_node(struct g_class *mp,
392    const char *name, struct g_raid_md_object *md);
393int g_raid_create_node_format(const char *format, struct gctl_req *req,
394    struct g_geom **gp);
395struct g_raid_volume * g_raid_create_volume(struct g_raid_softc *sc,
396    const char *name, int id);
397struct g_raid_disk * g_raid_create_disk(struct g_raid_softc *sc);
398const char * g_raid_get_diskname(struct g_raid_disk *disk);
399void g_raid_get_disk_info(struct g_raid_disk *disk);
400
401int g_raid_start_volume(struct g_raid_volume *vol);
402
403int g_raid_destroy_node(struct g_raid_softc *sc, int worker);
404int g_raid_destroy_volume(struct g_raid_volume *vol);
405int g_raid_destroy_disk(struct g_raid_disk *disk);
406
407void g_raid_iodone(struct bio *bp, int error);
408void g_raid_subdisk_iostart(struct g_raid_subdisk *sd, struct bio *bp);
409int g_raid_subdisk_kerneldump(struct g_raid_subdisk *sd,
410    void *virtual, vm_offset_t physical, off_t offset, size_t length);
411
412struct g_consumer *g_raid_open_consumer(struct g_raid_softc *sc,
413    const char *name);
414void g_raid_kill_consumer(struct g_raid_softc *sc, struct g_consumer *cp);
415
416void g_raid_report_disk_state(struct g_raid_disk *disk);
417void g_raid_change_disk_state(struct g_raid_disk *disk, int state);
418void g_raid_change_subdisk_state(struct g_raid_subdisk *sd, int state);
419void g_raid_change_volume_state(struct g_raid_volume *vol, int state);
420
421void g_raid_write_metadata(struct g_raid_softc *sc, struct g_raid_volume *vol,
422    struct g_raid_subdisk *sd, struct g_raid_disk *disk);
423void g_raid_fail_disk(struct g_raid_softc *sc,
424    struct g_raid_subdisk *sd, struct g_raid_disk *disk);
425
426void g_raid_tr_flush_common(struct g_raid_tr_object *tr, struct bio *bp);
427int g_raid_tr_kerneldump_common(struct g_raid_tr_object *tr,
428    void *virtual, vm_offset_t physical, off_t offset, size_t length);
429
430u_int g_raid_ndisks(struct g_raid_softc *sc, int state);
431u_int g_raid_nsubdisks(struct g_raid_volume *vol, int state);
432u_int g_raid_nopens(struct g_raid_softc *sc);
433struct g_raid_subdisk * g_raid_get_subdisk(struct g_raid_volume *vol,
434    int state);
435#define	G_RAID_DESTROY_SOFT		0
436#define	G_RAID_DESTROY_DELAYED	1
437#define	G_RAID_DESTROY_HARD		2
438int g_raid_destroy(struct g_raid_softc *sc, int how);
439int g_raid_event_send(void *arg, int event, int flags);
440int g_raid_lock_range(struct g_raid_volume *vol, off_t off, off_t len,
441    struct bio *ignore, void *argp);
442int g_raid_unlock_range(struct g_raid_volume *vol, off_t off, off_t len);
443
444g_ctl_req_t g_raid_ctl;
445#endif	/* _KERNEL */
446
447#endif	/* !_G_RAID_H_ */
448