18dab5b0mav/*-
2a82e3a8pfg * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3a82e3a8pfg *
48dab5b0mav * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
58dab5b0mav * All rights reserved.
68dab5b0mav *
78dab5b0mav * Redistribution and use in source and binary forms, with or without
88dab5b0mav * modification, are permitted provided that the following conditions
98dab5b0mav * are met:
108dab5b0mav * 1. Redistributions of source code must retain the above copyright
118dab5b0mav *    notice, this list of conditions and the following disclaimer.
128dab5b0mav * 2. Redistributions in binary form must reproduce the above copyright
138dab5b0mav *    notice, this list of conditions and the following disclaimer in the
148dab5b0mav *    documentation and/or other materials provided with the distribution.
158dab5b0mav *
168dab5b0mav * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
178dab5b0mav * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
188dab5b0mav * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
198dab5b0mav * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
208dab5b0mav * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
218dab5b0mav * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
228dab5b0mav * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
238dab5b0mav * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
248dab5b0mav * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
258dab5b0mav * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
268dab5b0mav * SUCH DAMAGE.
278dab5b0mav *
288dab5b0mav * $FreeBSD$
298dab5b0mav */
308dab5b0mav
318dab5b0mav#ifndef	_G_RAID_H_
328dab5b0mav#define	_G_RAID_H_
338dab5b0mav
348dab5b0mav#include <sys/param.h>
358dab5b0mav#include <sys/kobj.h>
368dab5b0mav#include <sys/bio.h>
378dab5b0mav#include <sys/time.h>
38db9e01amav#ifdef _KERNEL
39db9e01amav#include <sys/sysctl.h>
40db9e01amav#endif
418dab5b0mav
428dab5b0mav#define	G_RAID_CLASS_NAME	"RAID"
438dab5b0mav
448dab5b0mav#define	G_RAID_MAGIC		"GEOM::RAID"
458dab5b0mav
468dab5b0mav#define	G_RAID_VERSION		0
478dab5b0mav
488dab5b0mavstruct g_raid_md_object;
498dab5b0mavstruct g_raid_tr_object;
508dab5b0mav
518dab5b0mav#define	G_RAID_DEVICE_FLAG_NOAUTOSYNC	0x0000000000000001ULL
528dab5b0mav#define	G_RAID_DEVICE_FLAG_NOFAILSYNC	0x0000000000000002ULL
538dab5b0mav#define	G_RAID_DEVICE_FLAG_MASK	(G_RAID_DEVICE_FLAG_NOAUTOSYNC | \
548dab5b0mav					 G_RAID_DEVICE_FLAG_NOFAILSYNC)
558dab5b0mav
568dab5b0mav#ifdef _KERNEL
578dab5b0mavextern u_int g_raid_aggressive_spare;
588dab5b0mavextern u_int g_raid_debug;
59db9e01amavextern int g_raid_enable;
608dab5b0mavextern int g_raid_read_err_thresh;
618dab5b0mavextern u_int g_raid_start_timeout;
628dab5b0mavextern struct g_class g_raid_class;
638dab5b0mav
6410d53fccem#define	G_RAID_DEBUG(lvl, ...) \
6510d53fccem    _GEOM_DEBUG("GEOM_RAID", g_raid_debug, (lvl), NULL, __VA_ARGS__)
6610d53fccem#define	G_RAID_DEBUG1(lvl, sc, fmt, ...)				\
6710d53fccem    _GEOM_DEBUG("GEOM_RAID", g_raid_debug, (lvl), NULL, "%s: " fmt,	\
6810d53fccem	(sc)->sc_name, ## __VA_ARGS__)
6910d53fccem#define	G_RAID_LOGREQ(lvl, bp, ...) \
7010d53fccem    _GEOM_DEBUG("GEOM_RAID", g_raid_debug, (lvl), (bp), __VA_ARGS__)
718dab5b0mav
728dab5b0mav/*
738dab5b0mav * Flags we use to distinguish I/O initiated by the TR layer to maintain
748dab5b0mav * the volume's characteristics, fix subdisks, extra copies of data, etc.
758dab5b0mav *
768dab5b0mav * G_RAID_BIO_FLAG_SYNC		I/O to update an extra copy of the data
778dab5b0mav *				for RAID volumes that maintain extra data
788dab5b0mav *				and need to rebuild that data.
798dab5b0mav * G_RAID_BIO_FLAG_REMAP	I/O done to try to provoke a subdisk into
808dab5b0mav *				doing some desirable action such as bad
818dab5b0mav *				block remapping after we detect a bad part
828dab5b0mav *				of the disk.
838dab5b0mav * G_RAID_BIO_FLAG_LOCKED	I/O holds range lock that should re released.
848dab5b0mav *
858dab5b0mav * and the following meta item:
868dab5b0mav * G_RAID_BIO_FLAG_SPECIAL	And of the I/O flags that need to make it
878dab5b0mav *				through the range locking which would
888dab5b0mav *				otherwise defer the I/O until after that
898dab5b0mav *				range is unlocked.
908dab5b0mav */
918dab5b0mav#define	G_RAID_BIO_FLAG_SYNC		0x01
928dab5b0mav#define	G_RAID_BIO_FLAG_REMAP		0x02
938dab5b0mav#define	G_RAID_BIO_FLAG_SPECIAL \
948dab5b0mav		(G_RAID_BIO_FLAG_SYNC|G_RAID_BIO_FLAG_REMAP)
958dab5b0mav#define	G_RAID_BIO_FLAG_LOCKED		0x80
968dab5b0mav
978dab5b0mavstruct g_raid_lock {
988dab5b0mav	off_t			 l_offset;
998dab5b0mav	off_t			 l_length;
1008dab5b0mav	void			*l_callback_arg;
1018dab5b0mav	int			 l_pending;
1028dab5b0mav	LIST_ENTRY(g_raid_lock)	 l_next;
1038dab5b0mav};
1048dab5b0mav
1058dab5b0mav#define	G_RAID_EVENT_WAIT	0x01
1068dab5b0mav#define	G_RAID_EVENT_VOLUME	0x02
1078dab5b0mav#define	G_RAID_EVENT_SUBDISK	0x04
1088dab5b0mav#define	G_RAID_EVENT_DISK	0x08
1098dab5b0mav#define	G_RAID_EVENT_DONE	0x10
1108dab5b0mavstruct g_raid_event {
1118dab5b0mav	void			*e_tgt;
1128dab5b0mav	int			 e_event;
1138dab5b0mav	int			 e_flags;
1148dab5b0mav	int			 e_error;
1158dab5b0mav	TAILQ_ENTRY(g_raid_event) e_next;
1168dab5b0mav};
1178dab5b0mav#define G_RAID_DISK_S_NONE		0x00	/* State is unknown. */
1188dab5b0mav#define G_RAID_DISK_S_OFFLINE		0x01	/* Missing disk placeholder. */
1192a61b08mav#define G_RAID_DISK_S_DISABLED		0x02	/* Disabled. */
1202a61b08mav#define G_RAID_DISK_S_FAILED		0x03	/* Failed. */
1212a61b08mav#define G_RAID_DISK_S_STALE_FAILED	0x04	/* Old failed. */
1222a61b08mav#define G_RAID_DISK_S_SPARE		0x05	/* Hot-spare. */
1232a61b08mav#define G_RAID_DISK_S_STALE		0x06	/* Old disk, unused now. */
1242a61b08mav#define G_RAID_DISK_S_ACTIVE		0x07	/* Operational. */
1258dab5b0mav
1268dab5b0mav#define G_RAID_DISK_E_DISCONNECTED	0x01
1278dab5b0mav
1288dab5b0mavstruct g_raid_disk {
1298dab5b0mav	struct g_raid_softc	*d_softc;	/* Back-pointer to softc. */
1308dab5b0mav	struct g_consumer	*d_consumer;	/* GEOM disk consumer. */
1318dab5b0mav	void			*d_md_data;	/* Disk's metadata storage. */
132a43d540mav	int			 d_candelete;	/* BIO_DELETE supported. */
1338dab5b0mav	uint64_t		 d_flags;	/* Additional flags. */
1348dab5b0mav	u_int			 d_state;	/* Disk state. */
1358dab5b0mav	u_int			 d_load;	/* Disk average load. */
1368dab5b0mav	off_t			 d_last_offset;	/* Last head offset. */
1378dab5b0mav	int			 d_read_errs;	/* Count of the read errors */
1388dab5b0mav	TAILQ_HEAD(, g_raid_subdisk)	 d_subdisks; /* List of subdisks. */
1398dab5b0mav	TAILQ_ENTRY(g_raid_disk)	 d_next;	/* Next disk in the node. */
1406058a49cem	struct g_kerneldump	 d_kd;		/* Kernel dumping method/args. */
1418dab5b0mav};
1428dab5b0mav
1438dab5b0mav#define G_RAID_SUBDISK_S_NONE		0x00	/* Absent. */
1448dab5b0mav#define G_RAID_SUBDISK_S_FAILED		0x01	/* Failed. */
1458dab5b0mav#define G_RAID_SUBDISK_S_NEW		0x02	/* Blank. */
1468dab5b0mav#define G_RAID_SUBDISK_S_REBUILD	0x03	/* Blank + rebuild. */
1478dab5b0mav#define G_RAID_SUBDISK_S_UNINITIALIZED	0x04	/* Disk of the new volume. */
1488dab5b0mav#define G_RAID_SUBDISK_S_STALE		0x05	/* Dirty. */
1498dab5b0mav#define G_RAID_SUBDISK_S_RESYNC		0x06	/* Dirty + check/repair. */
1508dab5b0mav#define G_RAID_SUBDISK_S_ACTIVE		0x07	/* Usable. */
1518dab5b0mav
1528dab5b0mav#define G_RAID_SUBDISK_E_NEW		0x01	/* A new subdisk has arrived */
1538dab5b0mav#define G_RAID_SUBDISK_E_FAILED		0x02	/* A subdisk failed, but remains in volume */
1548dab5b0mav#define G_RAID_SUBDISK_E_DISCONNECTED	0x03	/* A subdisk removed from volume. */
1558dab5b0mav#define G_RAID_SUBDISK_E_FIRST_TR_PRIVATE 0x80	/* translation private events */
1568dab5b0mav
1578dab5b0mav#define G_RAID_SUBDISK_POS(sd)						\
1588dab5b0mav    ((sd)->sd_disk ? ((sd)->sd_disk->d_last_offset - (sd)->sd_offset) : 0)
1598dab5b0mav#define G_RAID_SUBDISK_TRACK_SIZE	(1 * 1024 * 1024)
1608dab5b0mav#define G_RAID_SUBDISK_LOAD(sd)						\
1618dab5b0mav    ((sd)->sd_disk ? ((sd)->sd_disk->d_load) : 0)
1628dab5b0mav#define G_RAID_SUBDISK_LOAD_SCALE	256
1638dab5b0mav
1648dab5b0mavstruct g_raid_subdisk {
1658dab5b0mav	struct g_raid_softc	*sd_softc;	/* Back-pointer to softc. */
1668dab5b0mav	struct g_raid_disk	*sd_disk;	/* Where this subdisk lives. */
1678dab5b0mav	struct g_raid_volume	*sd_volume;	/* Volume, sd is a part of. */
1688dab5b0mav	off_t			 sd_offset;	/* Offset on the disk. */
1698dab5b0mav	off_t			 sd_size;	/* Size on the disk. */
1708dab5b0mav	u_int			 sd_pos;	/* Position in volume. */
1718dab5b0mav	u_int			 sd_state;	/* Subdisk state. */
1728dab5b0mav	off_t			 sd_rebuild_pos; /* Rebuild position. */
1738dab5b0mav	int			 sd_recovery;	/* Count of recovery reqs. */
1748dab5b0mav	TAILQ_ENTRY(g_raid_subdisk)	 sd_next; /* Next subdisk on disk. */
1758dab5b0mav};
1768dab5b0mav
1778dab5b0mav#define G_RAID_MAX_SUBDISKS	16
1788dab5b0mav#define G_RAID_MAX_VOLUMENAME	32
1798dab5b0mav
1808dab5b0mav#define G_RAID_VOLUME_S_STARTING	0x00
1818dab5b0mav#define G_RAID_VOLUME_S_BROKEN		0x01
1828dab5b0mav#define G_RAID_VOLUME_S_DEGRADED	0x02
1838dab5b0mav#define G_RAID_VOLUME_S_SUBOPTIMAL	0x03
1848dab5b0mav#define G_RAID_VOLUME_S_OPTIMAL		0x04
1858dab5b0mav#define G_RAID_VOLUME_S_UNSUPPORTED	0x05
1868dab5b0mav#define G_RAID_VOLUME_S_STOPPED		0x06
1878dab5b0mav
1888dab5b0mav#define G_RAID_VOLUME_S_ALIVE(s)			\
1898dab5b0mav    ((s) == G_RAID_VOLUME_S_DEGRADED ||			\
1908dab5b0mav     (s) == G_RAID_VOLUME_S_SUBOPTIMAL ||		\
1918dab5b0mav     (s) == G_RAID_VOLUME_S_OPTIMAL)
1928dab5b0mav
1938dab5b0mav#define G_RAID_VOLUME_E_DOWN		0x00
1948dab5b0mav#define G_RAID_VOLUME_E_UP		0x01
1958dab5b0mav#define G_RAID_VOLUME_E_START		0x10
1968dab5b0mav#define G_RAID_VOLUME_E_STARTMD		0x11
1978dab5b0mav
1988dab5b0mav#define G_RAID_VOLUME_RL_RAID0		0x00
1998dab5b0mav#define G_RAID_VOLUME_RL_RAID1		0x01
2008dab5b0mav#define G_RAID_VOLUME_RL_RAID3		0x03
2018dab5b0mav#define G_RAID_VOLUME_RL_RAID4		0x04
2028dab5b0mav#define G_RAID_VOLUME_RL_RAID5		0x05
2038dab5b0mav#define G_RAID_VOLUME_RL_RAID6		0x06
2042e83ed7mav#define G_RAID_VOLUME_RL_RAIDMDF	0x07
2058dab5b0mav#define G_RAID_VOLUME_RL_RAID1E		0x11
2068dab5b0mav#define G_RAID_VOLUME_RL_SINGLE		0x0f
2078dab5b0mav#define G_RAID_VOLUME_RL_CONCAT		0x1f
2088dab5b0mav#define G_RAID_VOLUME_RL_RAID5E		0x15
2098dab5b0mav#define G_RAID_VOLUME_RL_RAID5EE	0x25
2102e83ed7mav#define G_RAID_VOLUME_RL_RAID5R		0x35
2118dab5b0mav#define G_RAID_VOLUME_RL_UNKNOWN	0xff
2128dab5b0mav
2138dab5b0mav#define G_RAID_VOLUME_RLQ_NONE		0x00
2142e83ed7mav#define G_RAID_VOLUME_RLQ_R1SM		0x00
2152e83ed7mav#define G_RAID_VOLUME_RLQ_R1MM		0x01
2162e83ed7mav#define G_RAID_VOLUME_RLQ_R3P0		0x00
2172e83ed7mav#define G_RAID_VOLUME_RLQ_R3PN		0x01
2182e83ed7mav#define G_RAID_VOLUME_RLQ_R4P0		0x00
2192e83ed7mav#define G_RAID_VOLUME_RLQ_R4PN		0x01
220fdb5671mav#define G_RAID_VOLUME_RLQ_R5RA		0x00
221fdb5671mav#define G_RAID_VOLUME_RLQ_R5RS		0x01
222fdb5671mav#define G_RAID_VOLUME_RLQ_R5LA		0x02
223fdb5671mav#define G_RAID_VOLUME_RLQ_R5LS		0x03
2242e83ed7mav#define G_RAID_VOLUME_RLQ_R6RA		0x00
2252e83ed7mav#define G_RAID_VOLUME_RLQ_R6RS		0x01
2262e83ed7mav#define G_RAID_VOLUME_RLQ_R6LA		0x02
2272e83ed7mav#define G_RAID_VOLUME_RLQ_R6LS		0x03
2282e83ed7mav#define G_RAID_VOLUME_RLQ_RMDFRA	0x00
2292e83ed7mav#define G_RAID_VOLUME_RLQ_RMDFRS	0x01
2302e83ed7mav#define G_RAID_VOLUME_RLQ_RMDFLA	0x02
2312e83ed7mav#define G_RAID_VOLUME_RLQ_RMDFLS	0x03
2322e83ed7mav#define G_RAID_VOLUME_RLQ_R1EA		0x00
2332e83ed7mav#define G_RAID_VOLUME_RLQ_R1EO		0x01
2342e83ed7mav#define G_RAID_VOLUME_RLQ_R5ERA		0x00
2352e83ed7mav#define G_RAID_VOLUME_RLQ_R5ERS		0x01
2362e83ed7mav#define G_RAID_VOLUME_RLQ_R5ELA		0x02
2372e83ed7mav#define G_RAID_VOLUME_RLQ_R5ELS		0x03
2382e83ed7mav#define G_RAID_VOLUME_RLQ_R5EERA	0x00
2392e83ed7mav#define G_RAID_VOLUME_RLQ_R5EERS	0x01
2402e83ed7mav#define G_RAID_VOLUME_RLQ_R5EELA	0x02
2412e83ed7mav#define G_RAID_VOLUME_RLQ_R5EELS	0x03
2422e83ed7mav#define G_RAID_VOLUME_RLQ_R5RRA		0x00
2432e83ed7mav#define G_RAID_VOLUME_RLQ_R5RRS		0x01
2442e83ed7mav#define G_RAID_VOLUME_RLQ_R5RLA		0x02
2452e83ed7mav#define G_RAID_VOLUME_RLQ_R5RLS		0x03
2468dab5b0mav#define G_RAID_VOLUME_RLQ_UNKNOWN	0xff
2478dab5b0mav
2488dab5b0mavstruct g_raid_volume;
2498dab5b0mav
2508dab5b0mavstruct g_raid_volume {
2518dab5b0mav	struct g_raid_softc	*v_softc;	/* Back-pointer to softc. */
2528dab5b0mav	struct g_provider	*v_provider;	/* GEOM provider. */
2538dab5b0mav	struct g_raid_subdisk	 v_subdisks[G_RAID_MAX_SUBDISKS];
2548dab5b0mav						/* Subdisks of this volume. */
2558dab5b0mav	void			*v_md_data;	/* Volume's metadata storage. */
2568dab5b0mav	struct g_raid_tr_object	*v_tr;		/* Transformation object. */
2578dab5b0mav	char			 v_name[G_RAID_MAX_VOLUMENAME];
2588dab5b0mav						/* Volume name. */
2598dab5b0mav	u_int			 v_state;	/* Volume state. */
2608dab5b0mav	u_int			 v_raid_level;	/* Array RAID level. */
2618dab5b0mav	u_int			 v_raid_level_qualifier; /* RAID level det. */
2628dab5b0mav	u_int			 v_disks_count;	/* Number of disks in array. */
2633d44dd0mav	u_int			 v_mdf_pdisks;	/* Number of parity disks
2643d44dd0mav						   in RAIDMDF array. */
2653d44dd0mav	uint16_t		 v_mdf_polynomial; /* Polynomial for RAIDMDF. */
2663d44dd0mav	uint8_t			 v_mdf_method;	/* Generation method for RAIDMDF. */
2678dab5b0mav	u_int			 v_strip_size;	/* Array strip size. */
2683d44dd0mav	u_int			 v_rotate_parity; /* Rotate RAID5R parity
2693d44dd0mav						   after numer of stripes. */
2708dab5b0mav	u_int			 v_sectorsize;	/* Volume sector size. */
2718dab5b0mav	off_t			 v_mediasize;	/* Volume media size.  */
2728dab5b0mav	struct bio_queue_head	 v_inflight;	/* In-flight write requests. */
2738dab5b0mav	struct bio_queue_head	 v_locked;	/* Blocked I/O requests. */
2748dab5b0mav	LIST_HEAD(, g_raid_lock) v_locks;	 /* List of locked regions. */
2758dab5b0mav	int			 v_pending_lock; /* writes to locked region */
2768dab5b0mav	int			 v_dirty;	/* Volume is DIRTY. */
2778dab5b0mav	struct timeval		 v_last_done;	/* Time of the last I/O. */
2788dab5b0mav	time_t			 v_last_write;	/* Time of the last write. */
2798dab5b0mav	u_int			 v_writes;	/* Number of active writes. */
2808dab5b0mav	struct root_hold_token	*v_rootmount;	/* Root mount delay token. */
2818dab5b0mav	int			 v_starting;	/* Volume is starting */
2828dab5b0mav	int			 v_stopping;	/* Volume is stopping */
2838dab5b0mav	int			 v_provider_open; /* Number of opens. */
2848dab5b0mav	int			 v_global_id;	/* Global volume ID (rX). */
285eba4a48mav	int			 v_read_only;	/* Volume is read-only. */
2868dab5b0mav	TAILQ_ENTRY(g_raid_volume)	 v_next; /* List of volumes entry. */
2878dab5b0mav	LIST_ENTRY(g_raid_volume)	 v_global_next; /* Global list entry. */
2888dab5b0mav};
2898dab5b0mav
2908dab5b0mav#define G_RAID_NODE_E_WAKE	0x00
2918dab5b0mav#define G_RAID_NODE_E_START	0x01
2928dab5b0mav
2938dab5b0mavstruct g_raid_softc {
2948dab5b0mav	struct g_raid_md_object	*sc_md;		/* Metadata object. */
2958dab5b0mav	struct g_geom		*sc_geom;	/* GEOM class instance. */
2968dab5b0mav	uint64_t		 sc_flags;	/* Additional flags. */
2978dab5b0mav	TAILQ_HEAD(, g_raid_volume)	 sc_volumes;	/* List of volumes. */
2988dab5b0mav	TAILQ_HEAD(, g_raid_disk)	 sc_disks;	/* List of disks. */
2998dab5b0mav	struct sx		 sc_lock;	/* Main node lock. */
3008dab5b0mav	struct proc		*sc_worker;	/* Worker process. */
3018dab5b0mav	struct mtx		 sc_queue_mtx;	/* Worker queues lock. */
3028dab5b0mav	TAILQ_HEAD(, g_raid_event) sc_events;	/* Worker events queue. */
3038dab5b0mav	struct bio_queue_head	 sc_queue;	/* Worker I/O queue. */
3048dab5b0mav	int			 sc_stopping;	/* Node is stopping */
3058dab5b0mav};
3068dab5b0mav#define	sc_name	sc_geom->name
3078dab5b0mav
308db9e01amavSYSCTL_DECL(_kern_geom_raid);
309db9e01amav
3108dab5b0mav/*
3118dab5b0mav * KOBJ parent class of metadata processing modules.
3128dab5b0mav */
3138dab5b0mavstruct g_raid_md_class {
3148dab5b0mav	KOBJ_CLASS_FIELDS;
315db9e01amav	int		 mdc_enable;
3168dab5b0mav	int		 mdc_priority;
3178dab5b0mav	LIST_ENTRY(g_raid_md_class) mdc_list;
3188dab5b0mav};
3198dab5b0mav
3208dab5b0mav/*
3218dab5b0mav * KOBJ instance of metadata processing module.
3228dab5b0mav */
3238dab5b0mavstruct g_raid_md_object {
3248dab5b0mav	KOBJ_FIELDS;
3258dab5b0mav	struct g_raid_md_class	*mdo_class;
3268dab5b0mav	struct g_raid_softc	*mdo_softc;	/* Back-pointer to softc. */
3278dab5b0mav};
3288dab5b0mav
3298dab5b0mavint g_raid_md_modevent(module_t, int, void *);
3308dab5b0mav
331db9e01amav#define	G_RAID_MD_DECLARE(name, label)				\
332db9e01amav    static moduledata_t g_raid_md_##name##_mod = {		\
333db9e01amav	"g_raid_md_" __XSTRING(name),				\
3348dab5b0mav	g_raid_md_modevent,					\
335db9e01amav	&g_raid_md_##name##_class				\
3368dab5b0mav    };								\
337db9e01amav    DECLARE_MODULE(g_raid_md_##name, g_raid_md_##name##_mod,	\
338db9e01amav	SI_SUB_DRIVERS, SI_ORDER_SECOND);			\
339db9e01amav    MODULE_DEPEND(g_raid_md_##name, geom_raid, 0, 0, 0);	\
340ad355b0kaktus    SYSCTL_NODE(_kern_geom_raid, OID_AUTO, name,		\
341ad355b0kaktus        CTLFLAG_RD | CTLFLAG_MPSAFE,				\
342db9e01amav	NULL, label " metadata module");			\
343db9e01amav    SYSCTL_INT(_kern_geom_raid_##name, OID_AUTO, enable,	\
344