1/*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2002 Poul-Henning Kamp
5 * Copyright (c) 2002 Networks Associates Technology, Inc.
6 * Copyright (c) 2013 The FreeBSD Foundation
7 * All rights reserved.
8 *
9 * This software was developed for the FreeBSD Project by Poul-Henning Kamp
10 * and NAI Labs, the Security Research Division of Network Associates, Inc.
11 * under DARPA/SPAWAR contract N66001-01-C-8035 ("CBOSS"), as part of the
12 * DARPA CHATS research program.
13 *
14 * Portions of this software were developed by Konstantin Belousov
15 * under sponsorship from the FreeBSD Foundation.
16 *
17 * Redistribution and use in source and binary forms, with or without
18 * modification, are permitted provided that the following conditions
19 * are met:
20 * 1. Redistributions of source code must retain the above copyright
21 *    notice, this list of conditions and the following disclaimer.
22 * 2. Redistributions in binary form must reproduce the above copyright
23 *    notice, this list of conditions and the following disclaimer in the
24 *    documentation and/or other materials provided with the distribution.
25 * 3. The names of the authors may not be used to endorse or promote
26 *    products derived from this software without specific prior written
27 *    permission.
28 *
29 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
30 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
31 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
32 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
33 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
34 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
35 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
36 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
37 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
38 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
39 * SUCH DAMAGE.
40 */
41
42#include <sys/cdefs.h>
43__FBSDID("$FreeBSD$");
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/kernel.h>
48#include <sys/malloc.h>
49#include <sys/bio.h>
50#include <sys/ktr.h>
51#include <sys/proc.h>
52#include <sys/sbuf.h>
53#include <sys/stack.h>
54#include <sys/sysctl.h>
55#include <sys/vmem.h>
56#include <machine/stdarg.h>
57
58#include <sys/errno.h>
59#include <geom/geom.h>
60#include <geom/geom_int.h>
61#include <sys/devicestat.h>
62
63#include <vm/uma.h>
64#include <vm/vm.h>
65#include <vm/vm_param.h>
66#include <vm/vm_kern.h>
67#include <vm/vm_page.h>
68#include <vm/vm_object.h>
69#include <vm/vm_extern.h>
70#include <vm/vm_map.h>
71
72static int	g_io_transient_map_bio(struct bio *bp);
73
74static struct g_bioq g_bio_run_down;
75static struct g_bioq g_bio_run_up;
76
77/*
78 * Pace is a hint that we've had some trouble recently allocating
79 * bios, so we should back off trying to send I/O down the stack
80 * a bit to let the problem resolve. When pacing, we also turn
81 * off direct dispatch to also reduce memory pressure from I/Os
82 * there, at the expxense of some added latency while the memory
83 * pressures exist. See g_io_schedule_down() for more details
84 * and limitations.
85 */
86static volatile u_int __read_mostly pace;
87
88static uma_zone_t __read_mostly biozone;
89
90#include <machine/atomic.h>
91
92static void
93g_bioq_lock(struct g_bioq *bq)
94{
95
96	mtx_lock(&bq->bio_queue_lock);
97}
98
99static void
100g_bioq_unlock(struct g_bioq *bq)
101{
102
103	mtx_unlock(&bq->bio_queue_lock);
104}
105
106#if 0
107static void
108g_bioq_destroy(struct g_bioq *bq)
109{
110
111	mtx_destroy(&bq->bio_queue_lock);
112}
113#endif
114
115static void
116g_bioq_init(struct g_bioq *bq)
117{
118
119	TAILQ_INIT(&bq->bio_queue);
120	mtx_init(&bq->bio_queue_lock, "bio queue", NULL, MTX_DEF);
121}
122
123static struct bio *
124g_bioq_first(struct g_bioq *bq)
125{
126	struct bio *bp;
127
128	bp = TAILQ_FIRST(&bq->bio_queue);
129	if (bp != NULL) {
130		KASSERT((bp->bio_flags & BIO_ONQUEUE),
131		    ("Bio not on queue bp=%p target %p", bp, bq));
132		bp->bio_flags &= ~BIO_ONQUEUE;
133		TAILQ_REMOVE(&bq->bio_queue, bp, bio_queue);
134		bq->bio_queue_length--;
135	}
136	return (bp);
137}
138
139struct bio *
140g_new_bio(void)
141{
142	struct bio *bp;
143
144	bp = uma_zalloc(biozone, M_NOWAIT | M_ZERO);
145#ifdef KTR
146	if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) {
147		struct stack st;
148
149		CTR1(KTR_GEOM, "g_new_bio(): %p", bp);
150		stack_save(&st);
151		CTRSTACK(KTR_GEOM, &st, 3);
152	}
153#endif
154	return (bp);
155}
156
157struct bio *
158g_alloc_bio(void)
159{
160	struct bio *bp;
161
162	bp = uma_zalloc(biozone, M_WAITOK | M_ZERO);
163#ifdef KTR
164	if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) {
165		struct stack st;
166
167		CTR1(KTR_GEOM, "g_alloc_bio(): %p", bp);
168		stack_save(&st);
169		CTRSTACK(KTR_GEOM, &st, 3);
170	}
171#endif
172	return (bp);
173}
174
175void
176g_destroy_bio(struct bio *bp)
177{
178#ifdef KTR
179	if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) {
180		struct stack st;
181
182		CTR1(KTR_GEOM, "g_destroy_bio(): %p", bp);
183		stack_save(&st);
184		CTRSTACK(KTR_GEOM, &st, 3);
185	}
186#endif
187	uma_zfree(biozone, bp);
188}
189
190struct bio *
191g_clone_bio(struct bio *bp)
192{
193	struct bio *bp2;
194
195	bp2 = uma_zalloc(biozone, M_NOWAIT | M_ZERO);
196	if (bp2 != NULL) {
197		bp2->bio_parent = bp;
198		bp2->bio_cmd = bp->bio_cmd;
199		/*
200		 *  BIO_ORDERED flag may be used by disk drivers to enforce
201		 *  ordering restrictions, so this flag needs to be cloned.
202		 *  BIO_UNMAPPED and BIO_VLIST should be inherited, to properly
203		 *  indicate which way the buffer is passed.
204		 *  Other bio flags are not suitable for cloning.
205		 */
206		bp2->bio_flags = bp->bio_flags &
207		    (BIO_ORDERED | BIO_UNMAPPED | BIO_VLIST);
208		bp2->bio_length = bp->bio_length;
209		bp2->bio_offset = bp->bio_offset;
210		bp2->bio_data = bp->bio_data;
211		bp2->bio_ma = bp->bio_ma;
212		bp2->bio_ma_n = bp->bio_ma_n;
213		bp2->bio_ma_offset = bp->bio_ma_offset;
214		bp2->bio_attribute = bp->bio_attribute;
215		if (bp->bio_cmd == BIO_ZONE)
216			bcopy(&bp->bio_zone, &bp2->bio_zone,
217			    sizeof(bp->bio_zone));
218#if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
219		bp2->bio_track_bp = bp->bio_track_bp;
220#endif
221		bp->bio_children++;
222	}
223#ifdef KTR
224	if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) {
225		struct stack st;
226
227		CTR2(KTR_GEOM, "g_clone_bio(%p): %p", bp, bp2);
228		stack_save(&st);
229		CTRSTACK(KTR_GEOM, &st, 3);
230	}
231#endif
232	return(bp2);
233}
234
235struct bio *
236g_duplicate_bio(struct bio *bp)
237{
238	struct bio *bp2;
239
240	bp2 = uma_zalloc(biozone, M_WAITOK | M_ZERO);
241	bp2->bio_flags = bp->bio_flags & (BIO_UNMAPPED | BIO_VLIST);
242	bp2->bio_parent = bp;
243	bp2->bio_cmd = bp->bio_cmd;
244	bp2->bio_length = bp->bio_length;
245	bp2->bio_offset = bp->bio_offset;
246	bp2->bio_data = bp->bio_data;
247	bp2->bio_ma = bp->bio_ma;
248	bp2->bio_ma_n = bp->bio_ma_n;
249	bp2->bio_ma_offset = bp->bio_ma_offset;
250	bp2->bio_attribute = bp->bio_attribute;
251	bp->bio_children++;
252#ifdef KTR
253	if ((KTR_COMPILE & KTR_GEOM) && (ktr_mask & KTR_GEOM)) {
254		struct stack st;
255
256		CTR2(KTR_GEOM, "g_duplicate_bio(%p): %p", bp, bp2);
257		stack_save(&st);
258		CTRSTACK(KTR_GEOM, &st, 3);
259	}
260#endif
261	return(bp2);
262}
263
264void
265g_reset_bio(struct bio *bp)
266{
267
268	bzero(bp, sizeof(*bp));
269}
270
271void
272g_io_init()
273{
274
275	g_bioq_init(&g_bio_run_down);
276	g_bioq_init(&g_bio_run_up);
277	biozone = uma_zcreate("g_bio", sizeof (struct bio),
278	    NULL, NULL,
279	    NULL, NULL,
280	    0, 0);
281}
282
283int
284g_io_getattr(const char *attr, struct g_consumer *cp, int *len, void *ptr)
285{
286	struct bio *bp;
287	int error;
288
289	g_trace(G_T_BIO, "bio_getattr(%s)", attr);
290	bp = g_alloc_bio();
291	bp->bio_cmd = BIO_GETATTR;
292	bp->bio_done = NULL;
293	bp->bio_attribute = attr;
294	bp->bio_length = *len;
295	bp->bio_data = ptr;
296	g_io_request(bp, cp);
297	error = biowait(bp, "ggetattr");
298	*len = bp->bio_completed;
299	g_destroy_bio(bp);
300	return (error);
301}
302
303int
304g_io_zonecmd(struct disk_zone_args *zone_args, struct g_consumer *cp)
305{
306	struct bio *bp;
307	int error;
308
309	g_trace(G_T_BIO, "bio_zone(%d)", zone_args->zone_cmd);
310	bp = g_alloc_bio();
311	bp->bio_cmd = BIO_ZONE;
312	bp->bio_done = NULL;
313	/*
314	 * XXX KDM need to handle report zone data.
315	 */
316	bcopy(zone_args, &bp->bio_zone, sizeof(*zone_args));
317	if (zone_args->zone_cmd == DISK_ZONE_REPORT_ZONES)
318		bp->bio_length =
319		    zone_args->zone_params.report.entries_allocated *
320		    sizeof(struct disk_zone_rep_entry);
321	else
322		bp->bio_length = 0;
323
324	g_io_request(bp, cp);
325	error = biowait(bp, "gzone");
326	bcopy(&bp->bio_zone, zone_args, sizeof(*zone_args));
327	g_destroy_bio(bp);
328	return (error);
329}
330
331/*
332 * Send a BIO_SPEEDUP down the stack. This is used to tell the lower layers that
333 * the upper layers have detected a resource shortage. The lower layers are
334 * advised to stop delaying I/O that they might be holding for performance
335 * reasons and to schedule it (non-trims) or complete it successfully (trims) as
336 * quickly as it can. bio_length is the amount of the shortage.  This call
337 * should be non-blocking. bio_resid is used to communicate back if the lower
338 * layers couldn't find bio_length worth of I/O to schedule or discard. A length
339 * of 0 means to do as much as you can (schedule the h/w queues full, discard
340 * all trims). flags are a hint from the upper layers to the lower layers what
341 * operation should be done.
342 */
343int
344g_io_speedup(size_t shortage, u_int flags, size_t *resid, struct g_consumer *cp)
345{
346	struct bio *bp;
347	int error;
348
349	KASSERT((flags & (BIO_SPEEDUP_TRIM | BIO_SPEEDUP_WRITE)) != 0,
350	    ("Invalid flags passed to g_io_speedup: %#x", flags));
351	g_trace(G_T_BIO, "bio_speedup(%s, %zu, %#x)", cp->provider->name,
352	    shortage, flags);
353	bp = g_new_bio();
354	if (bp == NULL)
355		return (ENOMEM);
356	bp->bio_cmd = BIO_SPEEDUP;
357	bp->bio_length = shortage;
358	bp->bio_done = NULL;
359	bp->bio_flags |= flags;
360	g_io_request(bp, cp);
361	error = biowait(bp, "gflush");
362	*resid = bp->bio_resid;
363	g_destroy_bio(bp);
364	return (error);
365}
366
367int
368g_io_flush(struct g_consumer *cp)
369{
370	struct bio *bp;
371	int error;
372
373	g_trace(G_T_BIO, "bio_flush(%s)", cp->provider->name);
374	bp = g_alloc_bio();
375	bp->bio_cmd = BIO_FLUSH;
376	bp->bio_flags |= BIO_ORDERED;
377	bp->bio_done = NULL;
378	bp->bio_attribute = NULL;
379	bp->bio_offset = cp->provider->mediasize;
380	bp->bio_length = 0;
381	bp->bio_data = NULL;
382	g_io_request(bp, cp);
383	error = biowait(bp, "gflush");
384	g_destroy_bio(bp);
385	return (error);
386}
387
388static int
389g_io_check(struct bio *bp)
390{
391	struct g_consumer *cp;
392	struct g_provider *pp;
393	off_t excess;
394	int error;
395
396	biotrack(bp, __func__);
397
398	cp = bp->bio_from;
399	pp = bp->bio_to;
400
401	/* Fail if access counters dont allow the operation */
402	switch(bp->bio_cmd) {
403	case BIO_READ:
404	case BIO_GETATTR:
405		if (cp->acr == 0)
406			return (EPERM);
407		break;
408	case BIO_WRITE:
409	case BIO_DELETE:
410	case BIO_SPEEDUP:
411	case BIO_FLUSH:
412		if (cp->acw == 0)
413			return (EPERM);
414		break;
415	case BIO_ZONE:
416		if ((bp->bio_zone.zone_cmd == DISK_ZONE_REPORT_ZONES) ||
417		    (bp->bio_zone.zone_cmd == DISK_ZONE_GET_PARAMS)) {
418			if (cp->acr == 0)
419				return (EPERM);
420		} else if (cp->acw == 0)
421			return (EPERM);
422		break;
423	default:
424		return (EPERM);
425	}
426	/* if provider is marked for error, don't disturb. */
427	if (pp->error)
428		return (pp->error);
429	if (cp->flags & G_CF_ORPHAN)
430		return (ENXIO);
431
432	switch(bp->bio_cmd) {
433	case BIO_READ:
434	case BIO_WRITE:
435	case BIO_DELETE:
436		/* Zero sectorsize or mediasize is probably a lack of media. */
437		if (pp->sectorsize == 0 || pp->mediasize == 0)
438			return (ENXIO);
439		/* Reject I/O not on sector boundary */
440		if (bp->bio_offset % pp->sectorsize)
441			return (EINVAL);
442		/* Reject I/O not integral sector long */
443		if (bp->bio_length % pp->sectorsize)
444			return (EINVAL);
445		/* Reject requests before or past the end of media. */
446		if (bp->bio_offset < 0)
447			return (EIO);
448		if (bp->bio_offset > pp->mediasize)
449			return (EIO);
450
451		/* Truncate requests to the end of providers media. */
452		excess = bp->bio_offset + bp->bio_length;
453		if (excess > bp->bio_to->mediasize) {
454			KASSERT((bp->bio_flags & BIO_UNMAPPED) == 0 ||
455			    round_page(bp->bio_ma_offset +
456			    bp->bio_length) / PAGE_SIZE == bp->bio_ma_n,
457			    ("excess bio %p too short", bp));
458			excess -= bp->bio_to->mediasize;
459			bp->bio_length -= excess;
460			if ((bp->bio_flags & BIO_UNMAPPED) != 0) {
461				bp->bio_ma_n = round_page(bp->bio_ma_offset +
462				    bp->bio_length) / PAGE_SIZE;
463			}
464			if (excess > 0)
465				CTR3(KTR_GEOM, "g_down truncated bio "
466				    "%p provider %s by %d", bp,
467				    bp->bio_to->name, excess);
468		}
469
470		/* Deliver zero length transfers right here. */
471		if (bp->bio_length == 0) {
472			CTR2(KTR_GEOM, "g_down terminated 0-length "
473			    "bp %p provider %s", bp, bp->bio_to->name);
474			return (0);
475		}
476
477		if ((bp->bio_flags & BIO_UNMAPPED) != 0 &&
478		    (bp->bio_to->flags & G_PF_ACCEPT_UNMAPPED) == 0 &&
479		    (bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE)) {
480			if ((error = g_io_transient_map_bio(bp)) >= 0)
481				return (error);
482		}
483		break;
484	default:
485		break;
486	}
487	return (EJUSTRETURN);
488}
489
490void
491g_io_request(struct bio *bp, struct g_consumer *cp)
492{
493	struct g_provider *pp;
494	int direct, error, first;
495	uint8_t cmd;
496
497	biotrack(bp, __func__);
498
499	KASSERT(cp != NULL, ("NULL cp in g_io_request"));
500	KASSERT(bp != NULL, ("NULL bp in g_io_request"));
501	pp = cp->provider;
502	KASSERT(pp != NULL, ("consumer not attached in g_io_request"));
503#ifdef DIAGNOSTIC
504	KASSERT(bp->bio_driver1 == NULL,
505	    ("bio_driver1 used by the consumer (geom %s)", cp->geom->name));
506	KASSERT(bp->bio_driver2 == NULL,
507	    ("bio_driver2 used by the consumer (geom %s)", cp->geom->name));
508	KASSERT(bp->bio_pflags == 0,
509	    ("bio_pflags used by the consumer (geom %s)", cp->geom->name));
510	/*
511	 * Remember consumer's private fields, so we can detect if they were
512	 * modified by the provider.
513	 */
514	bp->_bio_caller1 = bp->bio_caller1;
515	bp->_bio_caller2 = bp->bio_caller2;
516	bp->_bio_cflags = bp->bio_cflags;
517#endif
518
519	cmd = bp->bio_cmd;
520	if (cmd == BIO_READ || cmd == BIO_WRITE || cmd == BIO_GETATTR) {
521		KASSERT(bp->bio_data != NULL,
522		    ("NULL bp->data in g_io_request(cmd=%hu)", bp->bio_cmd));
523	}
524	if (cmd == BIO_DELETE || cmd == BIO_FLUSH) {
525		KASSERT(bp->bio_data == NULL,
526		    ("non-NULL bp->data in g_io_request(cmd=%hu)",
527		    bp->bio_cmd));
528	}
529	if (cmd == BIO_READ || cmd == BIO_WRITE || cmd == BIO_DELETE) {
530		KASSERT(bp->bio_offset % cp->provider->sectorsize == 0,
531		    ("wrong offset %jd for sectorsize %u",
532		    bp->bio_offset, cp->provider->sectorsize));
533		KASSERT(bp->bio_length % cp->provider->sectorsize == 0,
534		    ("wrong length %jd for sectorsize %u",
535		    bp->bio_length, cp->provider->sectorsize));
536	}
537
538	g_trace(G_T_BIO, "bio_request(%p) from %p(%s) to %p(%s) cmd %d",
539	    bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd);
540
541	bp->bio_from = cp;
542	bp->bio_to = pp;
543	bp->bio_error = 0;
544	bp->bio_completed = 0;
545
546	KASSERT(!(bp->bio_flags & BIO_ONQUEUE),
547	    ("Bio already on queue bp=%p", bp));
548
549	if ((g_collectstats & G_STATS_CONSUMERS) != 0 ||
550	    ((g_collectstats & G_STATS_PROVIDERS) != 0 && pp->stat != NULL))
551		binuptime(&bp->bio_t0);
552	else
553		getbinuptime(&bp->bio_t0);
554	if (g_collectstats & G_STATS_CONSUMERS)
555		devstat_start_transaction(cp->stat, &bp->bio_t0);
556	if (g_collectstats & G_STATS_PROVIDERS)
557		devstat_start_transaction(pp->stat, &bp->bio_t0);
558#ifdef INVARIANTS
559	atomic_add_int(&cp->nstart, 1);
560#endif
561
562#ifdef GET_STACK_USAGE
563	direct = (cp->flags & G_CF_DIRECT_SEND) != 0 &&
564	    (pp->flags & G_PF_DIRECT_RECEIVE) != 0 &&
565	    !g_is_geom_thread(curthread) &&
566	    ((pp->flags & G_PF_ACCEPT_UNMAPPED) != 0 ||
567	    (bp->bio_flags & BIO_UNMAPPED) == 0 || THREAD_CAN_SLEEP()) &&
568	    pace == 0;
569	if (direct) {
570		/* Block direct execution if less then half of stack left. */
571		size_t	st, su;
572		GET_STACK_USAGE(st, su);
573		if (su * 2 > st)
574			direct = 0;
575	}
576#else
577	direct = 0;
578#endif
579
580	if (direct) {
581		error = g_io_check(bp);
582		if (error >= 0) {
583			CTR3(KTR_GEOM, "g_io_request g_io_check on bp %p "
584			    "provider %s returned %d", bp, bp->bio_to->name,
585			    error);
586			g_io_deliver(bp, error);
587			return;
588		}
589		bp->bio_to->geom->start(bp);
590	} else {
591		g_bioq_lock(&g_bio_run_down);
592		first = TAILQ_EMPTY(&g_bio_run_down.bio_queue);
593		TAILQ_INSERT_TAIL(&g_bio_run_down.bio_queue, bp, bio_queue);
594		bp->bio_flags |= BIO_ONQUEUE;
595		g_bio_run_down.bio_queue_length++;
596		g_bioq_unlock(&g_bio_run_down);
597		/* Pass it on down. */
598		if (first)
599			wakeup(&g_wait_down);
600	}
601}
602
603void
604g_io_deliver(struct bio *bp, int error)
605{
606	struct bintime now;
607	struct g_consumer *cp;
608	struct g_provider *pp;
609	struct mtx *mtxp;
610	int direct, first;
611
612	biotrack(bp, __func__);
613
614	KASSERT(bp != NULL, ("NULL bp in g_io_deliver"));
615	pp = bp->bio_to;
616	KASSERT(pp != NULL, ("NULL bio_to in g_io_deliver"));
617	cp = bp->bio_from;
618	if (cp == NULL) {
619		bp->bio_error = error;
620		bp->bio_done(bp);
621		return;
622	}
623	KASSERT(cp != NULL, ("NULL bio_from in g_io_deliver"));
624	KASSERT(cp->geom != NULL, ("NULL bio_from->geom in g_io_deliver"));
625#ifdef DIAGNOSTIC
626	/*
627	 * Some classes - GJournal in particular - can modify bio's
628	 * private fields while the bio is in transit; G_GEOM_VOLATILE_BIO
629	 * flag means it's an expected behaviour for that particular geom.
630	 */
631	if ((cp->geom->flags & G_GEOM_VOLATILE_BIO) == 0) {
632		KASSERT(bp->bio_caller1 == bp->_bio_caller1,
633		    ("bio_caller1 used by the provider %s", pp->name));
634		KASSERT(bp->bio_caller2 == bp->_bio_caller2,
635		    ("bio_caller2 used by the provider %s", pp->name));
636		KASSERT(bp->bio_cflags == bp->_bio_cflags,
637		    ("bio_cflags used by the provider %s", pp->name));
638	}
639#endif
640	KASSERT(bp->bio_completed >= 0, ("bio_completed can't be less than 0"));
641	KASSERT(bp->bio_completed <= bp->bio_length,
642	    ("bio_completed can't be greater than bio_length"));
643
644	g_trace(G_T_BIO,
645"g_io_deliver(%p) from %p(%s) to %p(%s) cmd %d error %d off %jd len %jd",
646	    bp, cp, cp->geom->name, pp, pp->name, bp->bio_cmd, error,
647	    (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length);
648
649	KASSERT(!(bp->bio_flags & BIO_ONQUEUE),
650	    ("Bio already on queue bp=%p", bp));
651
652	/*
653	 * XXX: next two doesn't belong here
654	 */
655	bp->bio_bcount = bp->bio_length;
656	bp->bio_resid = bp->bio_bcount - bp->bio_completed;
657
658#ifdef GET_STACK_USAGE
659	direct = (pp->flags & G_PF_DIRECT_SEND) &&
660		 (cp->flags & G_CF_DIRECT_RECEIVE) &&
661		 !g_is_geom_thread(curthread);
662	if (direct) {
663		/* Block direct execution if less then half of stack left. */
664		size_t	st, su;
665		GET_STACK_USAGE(st, su);
666		if (su * 2 > st)
667			direct = 0;
668	}
669#else
670	direct = 0;
671#endif
672
673	/*
674	 * The statistics collection is lockless, as such, but we
675	 * can not update one instance of the statistics from more
676	 * than one thread at a time, so grab the lock first.
677	 */
678	if ((g_collectstats & G_STATS_CONSUMERS) != 0 ||
679	    ((g_collectstats & G_STATS_PROVIDERS) != 0 && pp->stat != NULL))
680		binuptime(&now);
681	mtxp = mtx_pool_find(mtxpool_sleep, cp);
682	mtx_lock(mtxp);
683	if (g_collectstats & G_STATS_PROVIDERS)
684		devstat_end_transaction_bio_bt(pp->stat, bp, &now);
685	if (g_collectstats & G_STATS_CONSUMERS)
686		devstat_end_transaction_bio_bt(cp->stat, bp, &now);
687#ifdef INVARIANTS
688	cp->nend++;
689#endif
690	mtx_unlock(mtxp);
691
692	if (error != ENOMEM) {
693		bp->bio_error = error;
694		if (direct) {
695			biodone(bp);
696		} else {
697			g_bioq_lock(&g_bio_run_up);
698			first = TAILQ_EMPTY(&g_bio_run_up.bio_queue);
699			TAILQ_INSERT_TAIL(&g_bio_run_up.bio_queue, bp, bio_queue);
700			bp->bio_flags |= BIO_ONQUEUE;
701			g_bio_run_up.bio_queue_length++;
702			g_bioq_unlock(&g_bio_run_up);
703			if (first)
704				wakeup(&g_wait_up);
705		}
706		return;
707	}
708
709	if (bootverbose)
710		printf("ENOMEM %p on %p(%s)\n", bp, pp, pp->name);
711	bp->bio_children = 0;
712	bp->bio_inbed = 0;
713	bp->bio_driver1 = NULL;
714	bp->bio_driver2 = NULL;
715	bp->bio_pflags = 0;
716	g_io_request(bp, cp);
717	pace = 1;
718	return;
719}
720
721SYSCTL_DECL(_kern_geom);
722
723static long transient_maps;
724SYSCTL_LONG(_kern_geom, OID_AUTO, transient_maps, CTLFLAG_RD,
725    &transient_maps, 0,
726    "Total count of the transient mapping requests");
727u_int transient_map_retries = 10;
728SYSCTL_UINT(_kern_geom, OID_AUTO, transient_map_retries, CTLFLAG_RW,
729    &transient_map_retries, 0,
730    "Max count of retries used before giving up on creating transient map");
731int transient_map_hard_failures;
732SYSCTL_INT(_kern_geom, OID_AUTO, transient_map_hard_failures, CTLFLAG_RD,
733    &transient_map_hard_failures, 0,
734    "Failures to establish the transient mapping due to retry attempts "
735    "exhausted");
736int transient_map_soft_failures;
737SYSCTL_INT(_kern_geom, OID_AUTO, transient_map_soft_failures, CTLFLAG_RD,
738    &transient_map_soft_failures, 0,
739    "Count of retried failures to establish the transient mapping");
740int inflight_transient_maps;
741SYSCTL_INT(_kern_geom, OID_AUTO, inflight_transient_maps, CTLFLAG_RD,
742    &inflight_transient_maps, 0,
743    "Current count of the active transient maps");
744
745static int
746g_io_transient_map_bio(struct bio *bp)
747{
748	vm_offset_t addr;
749	long size;
750	u_int retried;
751
752	KASSERT(unmapped_buf_allowed, ("unmapped disabled"));
753
754	size = round_page(bp->bio_ma_offset + bp->bio_length);
755	KASSERT(size / PAGE_SIZE == bp->bio_ma_n, ("Bio too short %p", bp));
756	addr = 0;
757	retried = 0;
758	atomic_add_long(&transient_maps, 1);
759retry:
760	if (vmem_alloc(transient_arena, size, M_BESTFIT | M_NOWAIT, &addr)) {
761		if (transient_map_retries != 0 &&
762		    retried >= transient_map_retries) {
763			CTR2(KTR_GEOM, "g_down cannot map bp %p provider %s",
764			    bp, bp->bio_to->name);
765			atomic_add_int(&transient_map_hard_failures, 1);
766			return (EDEADLK/* XXXKIB */);
767		} else {
768			/*
769			 * Naive attempt to quisce the I/O to get more
770			 * in-flight requests completed and defragment
771			 * the transient_arena.
772			 */
773			CTR3(KTR_GEOM, "g_down retrymap bp %p provider %s r %d",
774			    bp, bp->bio_to->name, retried);
775			pause("g_d_tra", hz / 10);
776			retried++;
777			atomic_add_int(&transient_map_soft_failures, 1);
778			goto retry;
779		}
780	}
781	atomic_add_int(&inflight_transient_maps, 1);
782	pmap_qenter((vm_offset_t)addr, bp->bio_ma, OFF_TO_IDX(size));
783	bp->bio_data = (caddr_t)addr + bp->bio_ma_offset;
784	bp->bio_flags |= BIO_TRANSIENT_MAPPING;
785	bp->bio_flags &= ~BIO_UNMAPPED;
786	return (EJUSTRETURN);
787}
788
789void
790g_io_schedule_down(struct thread *tp __unused)
791{
792	struct bio *bp;
793	int error;
794
795	for(;;) {
796		g_bioq_lock(&g_bio_run_down);
797		bp = g_bioq_first(&g_bio_run_down);
798		if (bp == NULL) {
799			CTR0(KTR_GEOM, "g_down going to sleep");
800			msleep(&g_wait_down, &g_bio_run_down.bio_queue_lock,
801			    PRIBIO | PDROP, "-", 0);
802			continue;
803		}
804		CTR0(KTR_GEOM, "g_down has work to do");
805		g_bioq_unlock(&g_bio_run_down);
806		biotrack(bp, __func__);
807		if (pace != 0) {
808			/*
809			 * There has been at least one memory allocation
810			 * failure since the last I/O completed. Pause 1ms to
811			 * give the system a chance to free up memory. We only
812			 * do this once because a large number of allocations
813			 * can fail in the direct dispatch case and there's no
814			 * relationship between the number of these failures and
815			 * the length of the outage. If there's still an outage,
816			 * we'll pause again and again until it's
817			 * resolved. Older versions paused longer and once per
818			 * allocation failure. This was OK for a single threaded
819			 * g_down, but with direct dispatch would lead to max of
820			 * 10 IOPs for minutes at a time when transient memory
821			 * issues prevented allocation for a batch of requests
822			 * from the upper layers.
823			 *
824			 * XXX This pacing is really lame. It needs to be solved
825			 * by other methods. This is OK only because the worst
826			 * case scenario is so rare. In the worst case scenario
827			 * all memory is tied up waiting for I/O to complete
828			 * which can never happen since we can't allocate bios
829			 * for that I/O.
830			 */
831			CTR0(KTR_GEOM, "g_down pacing self");
832			pause("g_down", min(hz/1000, 1));
833			pace = 0;
834		}
835		CTR2(KTR_GEOM, "g_down processing bp %p provider %s", bp,
836		    bp->bio_to->name);
837		error = g_io_check(bp);
838		if (error >= 0) {
839			CTR3(KTR_GEOM, "g_down g_io_check on bp %p provider "
840			    "%s returned %d", bp, bp->bio_to->name, error);
841			g_io_deliver(bp, error);
842			continue;
843		}
844		THREAD_NO_SLEEPING();
845		CTR4(KTR_GEOM, "g_down starting bp %p provider %s off %ld "
846		    "len %ld", bp, bp->bio_to->name, bp->bio_offset,
847		    bp->bio_length);
848		bp->bio_to->geom->start(bp);
849		THREAD_SLEEPING_OK();
850	}
851}
852
853void
854g_io_schedule_up(struct thread *tp __unused)
855{
856	struct bio *bp;
857
858	for(;;) {
859		g_bioq_lock(&g_bio_run_up);
860		bp = g_bioq_first(&g_bio_run_up);
861		if (bp == NULL) {
862			CTR0(KTR_GEOM, "g_up going to sleep");
863			msleep(&g_wait_up, &g_bio_run_up.bio_queue_lock,
864			    PRIBIO | PDROP, "-", 0);
865			continue;
866		}
867		g_bioq_unlock(&g_bio_run_up);
868		THREAD_NO_SLEEPING();
869		CTR4(KTR_GEOM, "g_up biodone bp %p provider %s off "
870		    "%jd len %ld", bp, bp->bio_to->name,
871		    bp->bio_offset, bp->bio_length);
872		biodone(bp);
873		THREAD_SLEEPING_OK();
874	}
875}
876
877void *
878g_read_data(struct g_consumer *cp, off_t offset, off_t length, int *error)
879{
880	struct bio *bp;
881	void *ptr;
882	int errorc;
883
884	KASSERT(length > 0 && length >= cp->provider->sectorsize &&
885	    length <= MAXPHYS, ("g_read_data(): invalid length %jd",
886	    (intmax_t)length));
887
888	bp = g_alloc_bio();
889	bp->bio_cmd = BIO_READ;
890	bp->bio_done = NULL;
891	bp->bio_offset = offset;
892	bp->bio_length = length;
893	ptr = g_malloc(length, M_WAITOK);
894	bp->bio_data = ptr;
895	g_io_request(bp, cp);
896	errorc = biowait(bp, "gread");
897	if (error != NULL)
898		*error = errorc;
899	g_destroy_bio(bp);
900	if (errorc) {
901		g_free(ptr);
902		ptr = NULL;
903	}
904	return (ptr);
905}
906
907/*
908 * A read function for use by ffs_sbget when used by GEOM-layer routines.
909 */
910int
911g_use_g_read_data(void *devfd, off_t loc, void **bufp, int size)
912{
913	struct g_consumer *cp;
914
915	KASSERT(*bufp == NULL,
916	    ("g_use_g_read_data: non-NULL *bufp %p\n", *bufp));
917
918	cp = (struct g_consumer *)devfd;
919	/*
920	 * Take care not to issue an invalid I/O request. The offset of
921	 * the superblock candidate must be multiples of the provider's
922	 * sector size, otherwise an FFS can't exist on the provider
923	 * anyway.
924	 */
925	if (loc % cp->provider->sectorsize != 0)
926		return (ENOENT);
927	*bufp = g_read_data(cp, loc, size, NULL);
928	if (*bufp == NULL)
929		return (ENOENT);
930	return (0);
931}
932
933int
934g_write_data(struct g_consumer *cp, off_t offset, void *ptr, off_t length)
935{
936	struct bio *bp;
937	int error;
938
939	KASSERT(length > 0 && length >= cp->provider->sectorsize &&
940	    length <= MAXPHYS, ("g_write_data(): invalid length %jd",
941	    (intmax_t)length));
942
943	bp = g_alloc_bio();
944	bp->bio_cmd = BIO_WRITE;
945	bp->bio_done = NULL;
946	bp->bio_offset = offset;
947	bp->bio_length = length;
948	bp->bio_data = ptr;
949	g_io_request(bp, cp);
950	error = biowait(bp, "gwrite");
951	g_destroy_bio(bp);
952	return (error);
953}
954
955/*
956 * A write function for use by ffs_sbput when used by GEOM-layer routines.
957 */
958int
959g_use_g_write_data(void *devfd, off_t loc, void *buf, int size)
960{
961
962	return (g_write_data((struct g_consumer *)devfd, loc, buf, size));
963}
964
965int
966g_delete_data(struct g_consumer *cp, off_t offset, off_t length)
967{
968	struct bio *bp;
969	int error;
970
971	KASSERT(length > 0 && length >= cp->provider->sectorsize,
972	    ("g_delete_data(): invalid length %jd", (intmax_t)length));
973
974	bp = g_alloc_bio();
975	bp->bio_cmd = BIO_DELETE;
976	bp->bio_done = NULL;
977	bp->bio_offset = offset;
978	bp->bio_length = length;
979	bp->bio_data = NULL;
980	g_io_request(bp, cp);
981	error = biowait(bp, "gdelete");
982	g_destroy_bio(bp);
983	return (error);
984}
985
986void
987g_print_bio(const char *prefix, const struct bio *bp, const char *fmtsuffix,
988    ...)
989{
990#ifndef PRINTF_BUFR_SIZE
991#define PRINTF_BUFR_SIZE 64
992#endif
993	char bufr[PRINTF_BUFR_SIZE];
994	struct sbuf sb, *sbp __unused;
995	va_list ap;
996
997	sbp = sbuf_new(&sb, bufr, sizeof(bufr), SBUF_FIXEDLEN);
998	KASSERT(sbp != NULL, ("sbuf_new misused?"));
999
1000	sbuf_set_drain(&sb, sbuf_printf_drain, NULL);
1001
1002	sbuf_cat(&sb, prefix);
1003	g_format_bio(&sb, bp);
1004
1005	va_start(ap, fmtsuffix);
1006	sbuf_vprintf(&sb, fmtsuffix, ap);
1007	va_end(ap);
1008
1009	sbuf_nl_terminate(&sb);
1010
1011	sbuf_finish(&sb);
1012	sbuf_delete(&sb);
1013}
1014
1015void
1016g_format_bio(struct sbuf *sb, const struct bio *bp)
1017{
1018	const char *pname, *cmd = NULL;
1019
1020	if (bp->bio_to != NULL)
1021		pname = bp->bio_to->name;
1022	else
1023		pname = "[unknown]";
1024
1025	switch (bp->bio_cmd) {
1026	case BIO_GETATTR:
1027		cmd = "GETATTR";
1028		sbuf_printf(sb, "%s[%s(attr=%s)]", pname, cmd,
1029		    bp->bio_attribute);
1030		return;
1031	case BIO_FLUSH:
1032		cmd = "FLUSH";
1033		sbuf_printf(sb, "%s[%s]", pname, cmd);
1034		return;
1035	case BIO_ZONE: {
1036		char *subcmd = NULL;
1037		cmd = "ZONE";
1038		switch (bp->bio_zone.zone_cmd) {
1039		case DISK_ZONE_OPEN:
1040			subcmd = "OPEN";
1041			break;
1042		case DISK_ZONE_CLOSE:
1043			subcmd = "CLOSE";
1044			break;
1045		case DISK_ZONE_FINISH:
1046			subcmd = "FINISH";
1047			break;
1048		case DISK_ZONE_RWP:
1049			subcmd = "RWP";
1050			break;
1051		case DISK_ZONE_REPORT_ZONES:
1052			subcmd = "REPORT ZONES";
1053			break;
1054		case DISK_ZONE_GET_PARAMS:
1055			subcmd = "GET PARAMS";
1056			break;
1057		default:
1058			subcmd = "UNKNOWN";
1059			break;
1060		}
1061		sbuf_printf(sb, "%s[%s,%s]", pname, cmd, subcmd);
1062		return;
1063	}
1064	case BIO_READ:
1065		cmd = "READ";
1066		break;
1067	case BIO_WRITE:
1068		cmd = "WRITE";
1069		break;
1070	case BIO_DELETE:
1071		cmd = "DELETE";
1072		break;
1073	default:
1074		cmd = "UNKNOWN";
1075		sbuf_printf(sb, "%s[%s()]", pname, cmd);
1076		return;
1077	}
1078	sbuf_printf(sb, "%s[%s(offset=%jd, length=%jd)]", pname, cmd,
1079	    (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length);
1080}
1081