1/*-
2 * SPDX-License-Identifier: BSD-4-Clause
3 *
4 * Copyright (c) 1996 John S. Dyson
5 * Copyright (c) 2012 Giovanni Trematerra
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice immediately at the beginning of the file, without modification,
13 *    this list of conditions, and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 * 3. Absolutely no warranty of function or purpose is made by the author
18 *    John S. Dyson.
19 * 4. Modifications may be freely made to this file if the above conditions
20 *    are met.
21 */
22
23/*
24 * This file contains a high-performance replacement for the socket-based
25 * pipes scheme originally used in FreeBSD/4.4Lite.  It does not support
26 * all features of sockets, but does do everything that pipes normally
27 * do.
28 */
29
30/*
31 * This code has two modes of operation, a small write mode and a large
32 * write mode.  The small write mode acts like conventional pipes with
33 * a kernel buffer.  If the buffer is less than PIPE_MINDIRECT, then the
34 * "normal" pipe buffering is done.  If the buffer is between PIPE_MINDIRECT
35 * and PIPE_SIZE in size, the sending process pins the underlying pages in
36 * memory, and the receiving process copies directly from these pinned pages
37 * in the sending process.
38 *
39 * If the sending process receives a signal, it is possible that it will
40 * go away, and certainly its address space can change, because control
41 * is returned back to the user-mode side.  In that case, the pipe code
42 * arranges to copy the buffer supplied by the user process, to a pageable
43 * kernel buffer, and the receiving process will grab the data from the
44 * pageable kernel buffer.  Since signals don't happen all that often,
45 * the copy operation is normally eliminated.
46 *
47 * The constant PIPE_MINDIRECT is chosen to make sure that buffering will
48 * happen for small transfers so that the system will not spend all of
49 * its time context switching.
50 *
51 * In order to limit the resource use of pipes, two sysctls exist:
52 *
53 * kern.ipc.maxpipekva - This is a hard limit on the amount of pageable
54 * address space available to us in pipe_map. This value is normally
55 * autotuned, but may also be loader tuned.
56 *
57 * kern.ipc.pipekva - This read-only sysctl tracks the current amount of
58 * memory in use by pipes.
59 *
60 * Based on how large pipekva is relative to maxpipekva, the following
61 * will happen:
62 *
63 * 0% - 50%:
64 *     New pipes are given 16K of memory backing, pipes may dynamically
65 *     grow to as large as 64K where needed.
66 * 50% - 75%:
67 *     New pipes are given 4K (or PAGE_SIZE) of memory backing,
68 *     existing pipes may NOT grow.
69 * 75% - 100%:
70 *     New pipes are given 4K (or PAGE_SIZE) of memory backing,
71 *     existing pipes will be shrunk down to 4K whenever possible.
72 *
73 * Resizing may be disabled by setting kern.ipc.piperesizeallowed=0.  If
74 * that is set,  the only resize that will occur is the 0 -> SMALL_PIPE_SIZE
75 * resize which MUST occur for reverse-direction pipes when they are
76 * first used.
77 *
78 * Additional information about the current state of pipes may be obtained
79 * from kern.ipc.pipes, kern.ipc.pipefragretry, kern.ipc.pipeallocfail,
80 * and kern.ipc.piperesizefail.
81 *
82 * Locking rules:  There are two locks present here:  A mutex, used via
83 * PIPE_LOCK, and a flag, used via pipelock().  All locking is done via
84 * the flag, as mutexes can not persist over uiomove.  The mutex
85 * exists only to guard access to the flag, and is not in itself a
86 * locking mechanism.  Also note that there is only a single mutex for
87 * both directions of a pipe.
88 *
89 * As pipelock() may have to sleep before it can acquire the flag, it
90 * is important to reread all data after a call to pipelock(); everything
91 * in the structure may have changed.
92 */
93
94#include <sys/cdefs.h>
95__FBSDID("$FreeBSD$");
96
97#include <sys/param.h>
98#include <sys/systm.h>
99#include <sys/conf.h>
100#include <sys/fcntl.h>
101#include <sys/file.h>
102#include <sys/filedesc.h>
103#include <sys/filio.h>
104#include <sys/kernel.h>
105#include <sys/lock.h>
106#include <sys/mutex.h>
107#include <sys/ttycom.h>
108#include <sys/stat.h>
109#include <sys/malloc.h>
110#include <sys/poll.h>
111#include <sys/selinfo.h>
112#include <sys/signalvar.h>
113#include <sys/syscallsubr.h>
114#include <sys/sysctl.h>
115#include <sys/sysproto.h>
116#include <sys/pipe.h>
117#include <sys/proc.h>
118#include <sys/vnode.h>
119#include <sys/uio.h>
120#include <sys/user.h>
121#include <sys/event.h>
122
123#include <security/mac/mac_framework.h>
124
125#include <vm/vm.h>
126#include <vm/vm_param.h>
127#include <vm/vm_object.h>
128#include <vm/vm_kern.h>
129#include <vm/vm_extern.h>
130#include <vm/pmap.h>
131#include <vm/vm_map.h>
132#include <vm/vm_page.h>
133#include <vm/uma.h>
134
135/*
136 * Use this define if you want to disable *fancy* VM things.  Expect an
137 * approx 30% decrease in transfer rate.  This could be useful for
138 * NetBSD or OpenBSD.
139 */
140/* #define PIPE_NODIRECT */
141
142#define PIPE_PEER(pipe)	\
143	(((pipe)->pipe_state & PIPE_NAMED) ? (pipe) : ((pipe)->pipe_peer))
144
145/*
146 * interfaces to the outside world
147 */
148static fo_rdwr_t	pipe_read;
149static fo_rdwr_t	pipe_write;
150static fo_truncate_t	pipe_truncate;
151static fo_ioctl_t	pipe_ioctl;
152static fo_poll_t	pipe_poll;
153static fo_kqfilter_t	pipe_kqfilter;
154static fo_stat_t	pipe_stat;
155static fo_close_t	pipe_close;
156static fo_chmod_t	pipe_chmod;
157static fo_chown_t	pipe_chown;
158static fo_fill_kinfo_t	pipe_fill_kinfo;
159
160struct fileops pipeops = {
161	.fo_read = pipe_read,
162	.fo_write = pipe_write,
163	.fo_truncate = pipe_truncate,
164	.fo_ioctl = pipe_ioctl,
165	.fo_poll = pipe_poll,
166	.fo_kqfilter = pipe_kqfilter,
167	.fo_stat = pipe_stat,
168	.fo_close = pipe_close,
169	.fo_chmod = pipe_chmod,
170	.fo_chown = pipe_chown,
171	.fo_sendfile = invfo_sendfile,
172	.fo_fill_kinfo = pipe_fill_kinfo,
173	.fo_flags = DFLAG_PASSABLE
174};
175
176static void	filt_pipedetach(struct knote *kn);
177static void	filt_pipedetach_notsup(struct knote *kn);
178static int	filt_pipenotsup(struct knote *kn, long hint);
179static int	filt_piperead(struct knote *kn, long hint);
180static int	filt_pipewrite(struct knote *kn, long hint);
181
182static struct filterops pipe_nfiltops = {
183	.f_isfd = 1,
184	.f_detach = filt_pipedetach_notsup,
185	.f_event = filt_pipenotsup
186};
187static struct filterops pipe_rfiltops = {
188	.f_isfd = 1,
189	.f_detach = filt_pipedetach,
190	.f_event = filt_piperead
191};
192static struct filterops pipe_wfiltops = {
193	.f_isfd = 1,
194	.f_detach = filt_pipedetach,
195	.f_event = filt_pipewrite
196};
197
198/*
199 * Default pipe buffer size(s), this can be kind-of large now because pipe
200 * space is pageable.  The pipe code will try to maintain locality of
201 * reference for performance reasons, so small amounts of outstanding I/O
202 * will not wipe the cache.
203 */
204#define MINPIPESIZE (PIPE_SIZE/3)
205#define MAXPIPESIZE (2*PIPE_SIZE/3)
206
207static long amountpipekva;
208static int pipefragretry;
209static int pipeallocfail;
210static int piperesizefail;
211static int piperesizeallowed = 1;
212
213SYSCTL_LONG(_kern_ipc, OID_AUTO, maxpipekva, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
214	   &maxpipekva, 0, "Pipe KVA limit");
215SYSCTL_LONG(_kern_ipc, OID_AUTO, pipekva, CTLFLAG_RD,
216	   &amountpipekva, 0, "Pipe KVA usage");
217SYSCTL_INT(_kern_ipc, OID_AUTO, pipefragretry, CTLFLAG_RD,
218	  &pipefragretry, 0, "Pipe allocation retries due to fragmentation");
219SYSCTL_INT(_kern_ipc, OID_AUTO, pipeallocfail, CTLFLAG_RD,
220	  &pipeallocfail, 0, "Pipe allocation failures");
221SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizefail, CTLFLAG_RD,
222	  &piperesizefail, 0, "Pipe resize failures");
223SYSCTL_INT(_kern_ipc, OID_AUTO, piperesizeallowed, CTLFLAG_RW,
224	  &piperesizeallowed, 0, "Pipe resizing allowed");
225
226static void pipeinit(void *dummy __unused);
227static void pipeclose(struct pipe *cpipe);
228static void pipe_free_kmem(struct pipe *cpipe);
229static int pipe_create(struct pipe *pipe, bool backing);
230static int pipe_paircreate(struct thread *td, struct pipepair **p_pp);
231static __inline int pipelock(struct pipe *cpipe, int catch);
232static __inline void pipeunlock(struct pipe *cpipe);
233#ifndef PIPE_NODIRECT
234static int pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio);
235static void pipe_destroy_write_buffer(struct pipe *wpipe);
236static int pipe_direct_write(struct pipe *wpipe, struct uio *uio);
237static void pipe_clone_write_buffer(struct pipe *wpipe);
238#endif
239static int pipespace(struct pipe *cpipe, int size);
240static int pipespace_new(struct pipe *cpipe, int size);
241
242static int	pipe_zone_ctor(void *mem, int size, void *arg, int flags);
243static int	pipe_zone_init(void *mem, int size, int flags);
244static void	pipe_zone_fini(void *mem, int size);
245
246static uma_zone_t pipe_zone;
247static struct unrhdr64 pipeino_unr;
248static dev_t pipedev_ino;
249
250SYSINIT(vfs, SI_SUB_VFS, SI_ORDER_ANY, pipeinit, NULL);
251
252static void
253pipeinit(void *dummy __unused)
254{
255
256	pipe_zone = uma_zcreate("pipe", sizeof(struct pipepair),
257	    pipe_zone_ctor, NULL, pipe_zone_init, pipe_zone_fini,
258	    UMA_ALIGN_PTR, 0);
259	KASSERT(pipe_zone != NULL, ("pipe_zone not initialized"));
260	new_unrhdr64(&pipeino_unr, 1);
261	pipedev_ino = devfs_alloc_cdp_inode();
262	KASSERT(pipedev_ino > 0, ("pipe dev inode not initialized"));
263}
264
265static int
266pipe_zone_ctor(void *mem, int size, void *arg, int flags)
267{
268	struct pipepair *pp;
269	struct pipe *rpipe, *wpipe;
270
271	KASSERT(size == sizeof(*pp), ("pipe_zone_ctor: wrong size"));
272
273	pp = (struct pipepair *)mem;
274
275	/*
276	 * We zero both pipe endpoints to make sure all the kmem pointers
277	 * are NULL, flag fields are zero'd, etc.  We timestamp both
278	 * endpoints with the same time.
279	 */
280	rpipe = &pp->pp_rpipe;
281	bzero(rpipe, sizeof(*rpipe));
282	vfs_timestamp(&rpipe->pipe_ctime);
283	rpipe->pipe_atime = rpipe->pipe_mtime = rpipe->pipe_ctime;
284
285	wpipe = &pp->pp_wpipe;
286	bzero(wpipe, sizeof(*wpipe));
287	wpipe->pipe_ctime = rpipe->pipe_ctime;
288	wpipe->pipe_atime = wpipe->pipe_mtime = rpipe->pipe_ctime;
289
290	rpipe->pipe_peer = wpipe;
291	rpipe->pipe_pair = pp;
292	wpipe->pipe_peer = rpipe;
293	wpipe->pipe_pair = pp;
294
295	/*
296	 * Mark both endpoints as present; they will later get free'd
297	 * one at a time.  When both are free'd, then the whole pair
298	 * is released.
299	 */
300	rpipe->pipe_present = PIPE_ACTIVE;
301	wpipe->pipe_present = PIPE_ACTIVE;
302
303	/*
304	 * Eventually, the MAC Framework may initialize the label
305	 * in ctor or init, but for now we do it elswhere to avoid
306	 * blocking in ctor or init.
307	 */
308	pp->pp_label = NULL;
309
310	return (0);
311}
312
313static int
314pipe_zone_init(void *mem, int size, int flags)
315{
316	struct pipepair *pp;
317
318	KASSERT(size == sizeof(*pp), ("pipe_zone_init: wrong size"));
319
320	pp = (struct pipepair *)mem;
321
322	mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_NEW);
323	return (0);
324}
325
326static void
327pipe_zone_fini(void *mem, int size)
328{
329	struct pipepair *pp;
330
331	KASSERT(size == sizeof(*pp), ("pipe_zone_fini: wrong size"));
332
333	pp = (struct pipepair *)mem;
334
335	mtx_destroy(&pp->pp_mtx);
336}
337
338static int
339pipe_paircreate(struct thread *td, struct pipepair **p_pp)
340{
341	struct pipepair *pp;
342	struct pipe *rpipe, *wpipe;
343	int error;
344
345	*p_pp = pp = uma_zalloc(pipe_zone, M_WAITOK);
346#ifdef MAC
347	/*
348	 * The MAC label is shared between the connected endpoints.  As a
349	 * result mac_pipe_init() and mac_pipe_create() are called once
350	 * for the pair, and not on the endpoints.
351	 */
352	mac_pipe_init(pp);
353	mac_pipe_create(td->td_ucred, pp);
354#endif
355	rpipe = &pp->pp_rpipe;
356	wpipe = &pp->pp_wpipe;
357
358	knlist_init_mtx(&rpipe->pipe_sel.si_note, PIPE_MTX(rpipe));
359	knlist_init_mtx(&wpipe->pipe_sel.si_note, PIPE_MTX(wpipe));
360
361	/*
362	 * Only the forward direction pipe is backed by big buffer by
363	 * default.
364	 */
365	error = pipe_create(rpipe, true);
366	if (error != 0)
367		goto fail;
368	error = pipe_create(wpipe, false);
369	if (error != 0) {
370		/*
371		 * This cleanup leaves the pipe inode number for rpipe
372		 * still allocated, but never used.  We do not free
373		 * inode numbers for opened pipes, which is required
374		 * for correctness because numbers must be unique.
375		 * But also it avoids any memory use by the unr
376		 * allocator, so stashing away the transient inode
377		 * number is reasonable.
378		 */
379		pipe_free_kmem(rpipe);
380		goto fail;
381	}
382
383	rpipe->pipe_state |= PIPE_DIRECTOK;
384	wpipe->pipe_state |= PIPE_DIRECTOK;
385	return (0);
386
387fail:
388	knlist_destroy(&rpipe->pipe_sel.si_note);
389	knlist_destroy(&wpipe->pipe_sel.si_note);
390#ifdef MAC
391	mac_pipe_destroy(pp);
392#endif
393	return (error);
394}
395
396int
397pipe_named_ctor(struct pipe **ppipe, struct thread *td)
398{
399	struct pipepair *pp;
400	int error;
401
402	error = pipe_paircreate(td, &pp);
403	if (error != 0)
404		return (error);
405	pp->pp_rpipe.pipe_state |= PIPE_NAMED;
406	*ppipe = &pp->pp_rpipe;
407	return (0);
408}
409
410void
411pipe_dtor(struct pipe *dpipe)
412{
413	struct pipe *peer;
414
415	peer = (dpipe->pipe_state & PIPE_NAMED) != 0 ? dpipe->pipe_peer : NULL;
416	funsetown(&dpipe->pipe_sigio);
417	pipeclose(dpipe);
418	if (peer != NULL) {
419		funsetown(&peer->pipe_sigio);
420		pipeclose(peer);
421	}
422}
423
424/*
425 * The pipe system call for the DTYPE_PIPE type of pipes.  If we fail, let
426 * the zone pick up the pieces via pipeclose().
427 */
428int
429kern_pipe(struct thread *td, int fildes[2], int flags, struct filecaps *fcaps1,
430    struct filecaps *fcaps2)
431{
432	struct file *rf, *wf;
433	struct pipe *rpipe, *wpipe;
434	struct pipepair *pp;
435	int fd, fflags, error;
436
437	error = pipe_paircreate(td, &pp);
438	if (error != 0)
439		return (error);
440	rpipe = &pp->pp_rpipe;
441	wpipe = &pp->pp_wpipe;
442	error = falloc_caps(td, &rf, &fd, flags, fcaps1);
443	if (error) {
444		pipeclose(rpipe);
445		pipeclose(wpipe);
446		return (error);
447	}
448	/* An extra reference on `rf' has been held for us by falloc_caps(). */
449	fildes[0] = fd;
450
451	fflags = FREAD | FWRITE;
452	if ((flags & O_NONBLOCK) != 0)
453		fflags |= FNONBLOCK;
454
455	/*
456	 * Warning: once we've gotten past allocation of the fd for the
457	 * read-side, we can only drop the read side via fdrop() in order
458	 * to avoid races against processes which manage to dup() the read
459	 * side while we are blocked trying to allocate the write side.
460	 */
461	finit(rf, fflags, DTYPE_PIPE, rpipe, &pipeops);
462	error = falloc_caps(td, &wf, &fd, flags, fcaps2);
463	if (error) {
464		fdclose(td, rf, fildes[0]);
465		fdrop(rf, td);
466		/* rpipe has been closed by fdrop(). */
467		pipeclose(wpipe);
468		return (error);
469	}
470	/* An extra reference on `wf' has been held for us by falloc_caps(). */
471	finit(wf, fflags, DTYPE_PIPE, wpipe, &pipeops);
472	fdrop(wf, td);
473	fildes[1] = fd;
474	fdrop(rf, td);
475
476	return (0);
477}
478
479#ifdef COMPAT_FREEBSD10
480/* ARGSUSED */
481int
482freebsd10_pipe(struct thread *td, struct freebsd10_pipe_args *uap __unused)
483{
484	int error;
485	int fildes[2];
486
487	error = kern_pipe(td, fildes, 0, NULL, NULL);
488	if (error)
489		return (error);
490
491	td->td_retval[0] = fildes[0];
492	td->td_retval[1] = fildes[1];
493
494	return (0);
495}
496#endif
497
498int
499sys_pipe2(struct thread *td, struct pipe2_args *uap)
500{
501	int error, fildes[2];
502
503	if (uap->flags & ~(O_CLOEXEC | O_NONBLOCK))
504		return (EINVAL);
505	error = kern_pipe(td, fildes, uap->flags, NULL, NULL);
506	if (error)
507		return (error);
508	error = copyout(fildes, uap->fildes, 2 * sizeof(int));
509	if (error) {
510		(void)kern_close(td, fildes[0]);
511		(void)kern_close(td, fildes[1]);
512	}
513	return (error);
514}
515
516/*
517 * Allocate kva for pipe circular buffer, the space is pageable
518 * This routine will 'realloc' the size of a pipe safely, if it fails
519 * it will retain the old buffer.
520 * If it fails it will return ENOMEM.
521 */
522static int
523pipespace_new(struct pipe *cpipe, int size)
524{
525	caddr_t buffer;
526	int error, cnt, firstseg;
527	static int curfail = 0;
528	static struct timeval lastfail;
529
530	KASSERT(!mtx_owned(PIPE_MTX(cpipe)), ("pipespace: pipe mutex locked"));
531	KASSERT(!(cpipe->pipe_state & PIPE_DIRECTW),
532		("pipespace: resize of direct writes not allowed"));
533retry:
534	cnt = cpipe->pipe_buffer.cnt;
535	if (cnt > size)
536		size = cnt;
537
538	size = round_page(size);
539	buffer = (caddr_t) vm_map_min(pipe_map);
540
541	error = vm_map_find(pipe_map, NULL, 0, (vm_offset_t *)&buffer, size, 0,
542	    VMFS_ANY_SPACE, VM_PROT_RW, VM_PROT_RW, 0);
543	if (error != KERN_SUCCESS) {
544		if (cpipe->pipe_buffer.buffer == NULL &&
545		    size > SMALL_PIPE_SIZE) {
546			size = SMALL_PIPE_SIZE;
547			pipefragretry++;
548			goto retry;
549		}
550		if (cpipe->pipe_buffer.buffer == NULL) {
551			pipeallocfail++;
552			if (ppsratecheck(&lastfail, &curfail, 1))
553				printf("kern.ipc.maxpipekva exceeded; see tuning(7)\n");
554		} else {
555			piperesizefail++;
556		}
557		return (ENOMEM);
558	}
559
560	/* copy data, then free old resources if we're resizing */
561	if (cnt > 0) {
562		if (cpipe->pipe_buffer.in <= cpipe->pipe_buffer.out) {
563			firstseg = cpipe->pipe_buffer.size - cpipe->pipe_buffer.out;
564			bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out],
565				buffer, firstseg);
566			if ((cnt - firstseg) > 0)
567				bcopy(cpipe->pipe_buffer.buffer, &buffer[firstseg],
568					cpipe->pipe_buffer.in);
569		} else {
570			bcopy(&cpipe->pipe_buffer.buffer[cpipe->pipe_buffer.out],
571				buffer, cnt);
572		}
573	}
574	pipe_free_kmem(cpipe);
575	cpipe->pipe_buffer.buffer = buffer;
576	cpipe->pipe_buffer.size = size;
577	cpipe->pipe_buffer.in = cnt;
578	cpipe->pipe_buffer.out = 0;
579	cpipe->pipe_buffer.cnt = cnt;
580	atomic_add_long(&amountpipekva, cpipe->pipe_buffer.size);
581	return (0);
582}
583
584/*
585 * Wrapper for pipespace_new() that performs locking assertions.
586 */
587static int
588pipespace(struct pipe *cpipe, int size)
589{
590
591	KASSERT(cpipe->pipe_state & PIPE_LOCKFL,
592	    ("Unlocked pipe passed to pipespace"));
593	return (pipespace_new(cpipe, size));
594}
595
596/*
597 * lock a pipe for I/O, blocking other access
598 */
599static __inline int
600pipelock(struct pipe *cpipe, int catch)
601{
602	int error;
603
604	PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
605	while (cpipe->pipe_state & PIPE_LOCKFL) {
606		cpipe->pipe_state |= PIPE_LWANT;
607		error = msleep(cpipe, PIPE_MTX(cpipe),
608		    catch ? (PRIBIO | PCATCH) : PRIBIO,
609		    "pipelk", 0);
610		if (error != 0)
611			return (error);
612	}
613	cpipe->pipe_state |= PIPE_LOCKFL;
614	return (0);
615}
616
617/*
618 * unlock a pipe I/O lock
619 */
620static __inline void
621pipeunlock(struct pipe *cpipe)
622{
623
624	PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
625	KASSERT(cpipe->pipe_state & PIPE_LOCKFL,
626		("Unlocked pipe passed to pipeunlock"));
627	cpipe->pipe_state &= ~PIPE_LOCKFL;
628	if (cpipe->pipe_state & PIPE_LWANT) {
629		cpipe->pipe_state &= ~PIPE_LWANT;
630		wakeup(cpipe);
631	}
632}
633
634void
635pipeselwakeup(struct pipe *cpipe)
636{
637
638	PIPE_LOCK_ASSERT(cpipe, MA_OWNED);
639	if (cpipe->pipe_state & PIPE_SEL) {
640		selwakeuppri(&cpipe->pipe_sel, PSOCK);
641		if (!SEL_WAITING(&cpipe->pipe_sel))
642			cpipe->pipe_state &= ~PIPE_SEL;
643	}
644	if ((cpipe->pipe_state & PIPE_ASYNC) && cpipe->pipe_sigio)
645		pgsigio(&cpipe->pipe_sigio, SIGIO, 0);
646	KNOTE_LOCKED(&cpipe->pipe_sel.si_note, 0);
647}
648
649/*
650 * Initialize and allocate VM and memory for pipe.  The structure
651 * will start out zero'd from the ctor, so we just manage the kmem.
652 */
653static int
654pipe_create(struct pipe *pipe, bool large_backing)
655{
656	int error;
657
658	error = pipespace_new(pipe, !large_backing || amountpipekva >
659	    maxpipekva / 2 ? SMALL_PIPE_SIZE : PIPE_SIZE);
660	if (error == 0)
661		pipe->pipe_ino = alloc_unr64(&pipeino_unr);
662	return (error);
663}
664
665/* ARGSUSED */
666static int
667pipe_read(struct file *fp, struct uio *uio, struct ucred *active_cred,
668    int flags, struct thread *td)
669{
670	struct pipe *rpipe;
671	int error;
672	int nread = 0;
673	int size;
674
675	rpipe = fp->f_data;
676	PIPE_LOCK(rpipe);
677	++rpipe->pipe_busy;
678	error = pipelock(rpipe, 1);
679	if (error)
680		goto unlocked_error;
681
682#ifdef MAC
683	error = mac_pipe_check_read(active_cred, rpipe->pipe_pair);
684	if (error)
685		goto locked_error;
686#endif
687	if (amountpipekva > (3 * maxpipekva) / 4) {
688		if ((rpipe->pipe_state & PIPE_DIRECTW) == 0 &&
689		    rpipe->pipe_buffer.size > SMALL_PIPE_SIZE &&
690		    rpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE &&
691		    piperesizeallowed == 1) {
692			PIPE_UNLOCK(rpipe);
693			pipespace(rpipe, SMALL_PIPE_SIZE);
694			PIPE_LOCK(rpipe);
695		}
696	}
697
698	while (uio->uio_resid) {
699		/*
700		 * normal pipe buffer receive
701		 */
702		if (rpipe->pipe_buffer.cnt > 0) {
703			size = rpipe->pipe_buffer.size - rpipe->pipe_buffer.out;
704			if (size > rpipe->pipe_buffer.cnt)
705				size = rpipe->pipe_buffer.cnt;
706			if (size > uio->uio_resid)
707				size = uio->uio_resid;
708
709			PIPE_UNLOCK(rpipe);
710			error = uiomove(
711			    &rpipe->pipe_buffer.buffer[rpipe->pipe_buffer.out],
712			    size, uio);
713			PIPE_LOCK(rpipe);
714			if (error)
715				break;
716
717			rpipe->pipe_buffer.out += size;
718			if (rpipe->pipe_buffer.out >= rpipe->pipe_buffer.size)
719				rpipe->pipe_buffer.out = 0;
720
721			rpipe->pipe_buffer.cnt -= size;
722
723			/*
724			 * If there is no more to read in the pipe, reset
725			 * its pointers to the beginning.  This improves
726			 * cache hit stats.
727			 */
728			if (rpipe->pipe_buffer.cnt == 0) {
729				rpipe->pipe_buffer.in = 0;
730				rpipe->pipe_buffer.out = 0;
731			}
732			nread += size;
733#ifndef PIPE_NODIRECT
734		/*
735		 * Direct copy, bypassing a kernel buffer.
736		 */
737		} else if ((size = rpipe->pipe_map.cnt) != 0) {
738			if (size > uio->uio_resid)
739				size = (u_int) uio->uio_resid;
740			PIPE_UNLOCK(rpipe);
741			error = uiomove_fromphys(rpipe->pipe_map.ms,
742			    rpipe->pipe_map.pos, size, uio);
743			PIPE_LOCK(rpipe);
744			if (error)
745				break;
746			nread += size;
747			rpipe->pipe_map.pos += size;
748			rpipe->pipe_map.cnt -= size;
749			if (rpipe->pipe_map.cnt == 0) {
750				rpipe->pipe_state &= ~PIPE_WANTW;
751				wakeup(rpipe);
752			}
753#endif
754		} else {
755			/*
756			 * detect EOF condition
757			 * read returns 0 on EOF, no need to set error
758			 */
759			if (rpipe->pipe_state & PIPE_EOF)
760				break;
761
762			/*
763			 * If the "write-side" has been blocked, wake it up now.
764			 */
765			if (rpipe->pipe_state & PIPE_WANTW) {
766				rpipe->pipe_state &= ~PIPE_WANTW;
767				wakeup(rpipe);
768			}
769
770			/*
771			 * Break if some data was read.
772			 */
773			if (nread > 0)
774				break;
775
776			/*
777			 * Unlock the pipe buffer for our remaining processing.
778			 * We will either break out with an error or we will
779			 * sleep and relock to loop.
780			 */
781			pipeunlock(rpipe);
782
783			/*
784			 * Handle non-blocking mode operation or
785			 * wait for more data.
786			 */
787			if (fp->f_flag & FNONBLOCK) {
788				error = EAGAIN;
789			} else {
790				rpipe->pipe_state |= PIPE_WANTR;
791				if ((error = msleep(rpipe, PIPE_MTX(rpipe),
792				    PRIBIO | PCATCH,
793				    "piperd", 0)) == 0)
794					error = pipelock(rpipe, 1);
795			}
796			if (error)
797				goto unlocked_error;
798		}
799	}
800#ifdef MAC
801locked_error:
802#endif
803	pipeunlock(rpipe);
804
805	/* XXX: should probably do this before getting any locks. */
806	if (error == 0)
807		vfs_timestamp(&rpipe->pipe_atime);
808unlocked_error:
809	--rpipe->pipe_busy;
810
811	/*
812	 * PIPE_WANT processing only makes sense if pipe_busy is 0.
813	 */
814	if ((rpipe->pipe_busy == 0) && (rpipe->pipe_state & PIPE_WANT)) {
815		rpipe->pipe_state &= ~(PIPE_WANT|PIPE_WANTW);
816		wakeup(rpipe);
817	} else if (rpipe->pipe_buffer.cnt < MINPIPESIZE) {
818		/*
819		 * Handle write blocking hysteresis.
820		 */
821		if (rpipe->pipe_state & PIPE_WANTW) {
822			rpipe->pipe_state &= ~PIPE_WANTW;
823			wakeup(rpipe);
824		}
825	}
826
827	/*
828	 * Only wake up writers if there was actually something read.
829	 * Otherwise, when calling read(2) at EOF, a spurious wakeup occurs.
830	 */
831	if (nread > 0 &&
832	    rpipe->pipe_buffer.size - rpipe->pipe_buffer.cnt >= PIPE_BUF)
833		pipeselwakeup(rpipe);
834
835	PIPE_UNLOCK(rpipe);
836	return (error);
837}
838
839#ifndef PIPE_NODIRECT
840/*
841 * Map the sending processes' buffer into kernel space and wire it.
842 * This is similar to a physical write operation.
843 */
844static int
845pipe_build_write_buffer(struct pipe *wpipe, struct uio *uio)
846{
847	u_int size;
848	int i;
849
850	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
851	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,
852	    ("%s: PIPE_DIRECTW set on %p", __func__, wpipe));
853	KASSERT(wpipe->pipe_map.cnt == 0,
854	    ("%s: pipe map for %p contains residual data", __func__, wpipe));
855
856	if (uio->uio_iov->iov_len > wpipe->pipe_buffer.size)
857                size = wpipe->pipe_buffer.size;
858	else
859                size = uio->uio_iov->iov_len;
860
861	wpipe->pipe_state |= PIPE_DIRECTW;
862	PIPE_UNLOCK(wpipe);
863	i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
864	    (vm_offset_t)uio->uio_iov->iov_base, size, VM_PROT_READ,
865	    wpipe->pipe_map.ms, PIPENPAGES);
866	PIPE_LOCK(wpipe);
867	if (i < 0) {
868		wpipe->pipe_state &= ~PIPE_DIRECTW;
869		return (EFAULT);
870	}
871
872	wpipe->pipe_map.npages = i;
873	wpipe->pipe_map.pos =
874	    ((vm_offset_t) uio->uio_iov->iov_base) & PAGE_MASK;
875	wpipe->pipe_map.cnt = size;
876
877	uio->uio_iov->iov_len -= size;
878	uio->uio_iov->iov_base = (char *)uio->uio_iov->iov_base + size;
879	if (uio->uio_iov->iov_len == 0)
880		uio->uio_iov++;
881	uio->uio_resid -= size;
882	uio->uio_offset += size;
883	return (0);
884}
885
886/*
887 * Unwire the process buffer.
888 */
889static void
890pipe_destroy_write_buffer(struct pipe *wpipe)
891{
892
893	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
894	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,
895	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));
896	KASSERT(wpipe->pipe_map.cnt == 0,
897	    ("%s: pipe map for %p contains residual data", __func__, wpipe));
898
899	wpipe->pipe_state &= ~PIPE_DIRECTW;
900	vm_page_unhold_pages(wpipe->pipe_map.ms, wpipe->pipe_map.npages);
901	wpipe->pipe_map.npages = 0;
902}
903
904/*
905 * In the case of a signal, the writing process might go away.  This
906 * code copies the data into the circular buffer so that the source
907 * pages can be freed without loss of data.
908 */
909static void
910pipe_clone_write_buffer(struct pipe *wpipe)
911{
912	struct uio uio;
913	struct iovec iov;
914	int size;
915	int pos;
916
917	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
918	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) != 0,
919	    ("%s: PIPE_DIRECTW not set on %p", __func__, wpipe));
920
921	size = wpipe->pipe_map.cnt;
922	pos = wpipe->pipe_map.pos;
923	wpipe->pipe_map.cnt = 0;
924
925	wpipe->pipe_buffer.in = size;
926	wpipe->pipe_buffer.out = 0;
927	wpipe->pipe_buffer.cnt = size;
928
929	PIPE_UNLOCK(wpipe);
930	iov.iov_base = wpipe->pipe_buffer.buffer;
931	iov.iov_len = size;
932	uio.uio_iov = &iov;
933	uio.uio_iovcnt = 1;
934	uio.uio_offset = 0;
935	uio.uio_resid = size;
936	uio.uio_segflg = UIO_SYSSPACE;
937	uio.uio_rw = UIO_READ;
938	uio.uio_td = curthread;
939	uiomove_fromphys(wpipe->pipe_map.ms, pos, size, &uio);
940	PIPE_LOCK(wpipe);
941	pipe_destroy_write_buffer(wpipe);
942}
943
944/*
945 * This implements the pipe buffer write mechanism.  Note that only
946 * a direct write OR a normal pipe write can be pending at any given time.
947 * If there are any characters in the pipe buffer, the direct write will
948 * be deferred until the receiving process grabs all of the bytes from
949 * the pipe buffer.  Then the direct mapping write is set-up.
950 */
951static int
952pipe_direct_write(struct pipe *wpipe, struct uio *uio)
953{
954	int error;
955
956retry:
957	PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
958	error = pipelock(wpipe, 1);
959	if (error != 0)
960		goto error1;
961	if ((wpipe->pipe_state & PIPE_EOF) != 0) {
962		error = EPIPE;
963		pipeunlock(wpipe);
964		goto error1;
965	}
966	if (wpipe->pipe_state & PIPE_DIRECTW) {
967		if (wpipe->pipe_state & PIPE_WANTR) {
968			wpipe->pipe_state &= ~PIPE_WANTR;
969			wakeup(wpipe);
970		}
971		pipeselwakeup(wpipe);
972		wpipe->pipe_state |= PIPE_WANTW;
973		pipeunlock(wpipe);
974		error = msleep(wpipe, PIPE_MTX(wpipe),
975		    PRIBIO | PCATCH, "pipdww", 0);
976		if (error)
977			goto error1;
978		else
979			goto retry;
980	}
981	if (wpipe->pipe_buffer.cnt > 0) {
982		if (wpipe->pipe_state & PIPE_WANTR) {
983			wpipe->pipe_state &= ~PIPE_WANTR;
984			wakeup(wpipe);
985		}
986		pipeselwakeup(wpipe);
987		wpipe->pipe_state |= PIPE_WANTW;
988		pipeunlock(wpipe);
989		error = msleep(wpipe, PIPE_MTX(wpipe),
990		    PRIBIO | PCATCH, "pipdwc", 0);
991		if (error)
992			goto error1;
993		else
994			goto retry;
995	}
996
997	error = pipe_build_write_buffer(wpipe, uio);
998	if (error) {
999		pipeunlock(wpipe);
1000		goto error1;
1001	}
1002
1003	while (wpipe->pipe_map.cnt != 0 &&
1004	    (wpipe->pipe_state & PIPE_EOF) == 0) {
1005		if (wpipe->pipe_state & PIPE_WANTR) {
1006			wpipe->pipe_state &= ~PIPE_WANTR;
1007			wakeup(wpipe);
1008		}
1009		pipeselwakeup(wpipe);
1010		wpipe->pipe_state |= PIPE_WANTW;
1011		pipeunlock(wpipe);
1012		error = msleep(wpipe, PIPE_MTX(wpipe), PRIBIO | PCATCH,
1013		    "pipdwt", 0);
1014		pipelock(wpipe, 0);
1015		if (error != 0)
1016			break;
1017	}
1018
1019	if ((wpipe->pipe_state & PIPE_EOF) != 0) {
1020		wpipe->pipe_map.cnt = 0;
1021		pipe_destroy_write_buffer(wpipe);
1022		pipeselwakeup(wpipe);
1023		error = EPIPE;
1024	} else if (error == EINTR || error == ERESTART) {
1025		pipe_clone_write_buffer(wpipe);
1026	} else {
1027		pipe_destroy_write_buffer(wpipe);
1028	}
1029	pipeunlock(wpipe);
1030	KASSERT((wpipe->pipe_state & PIPE_DIRECTW) == 0,
1031	    ("pipe %p leaked PIPE_DIRECTW", wpipe));
1032	return (error);
1033
1034error1:
1035	wakeup(wpipe);
1036	return (error);
1037}
1038#endif
1039
1040static int
1041pipe_write(struct file *fp, struct uio *uio, struct ucred *active_cred,
1042    int flags, struct thread *td)
1043{
1044	struct pipe *wpipe, *rpipe;
1045	ssize_t orig_resid;
1046	int desiredsize, error;
1047
1048	rpipe = fp->f_data;
1049	wpipe = PIPE_PEER(rpipe);
1050	PIPE_LOCK(rpipe);
1051	error = pipelock(wpipe, 1);
1052	if (error) {
1053		PIPE_UNLOCK(rpipe);
1054		return (error);
1055	}
1056	/*
1057	 * detect loss of pipe read side, issue SIGPIPE if lost.
1058	 */
1059	if (wpipe->pipe_present != PIPE_ACTIVE ||
1060	    (wpipe->pipe_state & PIPE_EOF)) {
1061		pipeunlock(wpipe);
1062		PIPE_UNLOCK(rpipe);
1063		return (EPIPE);
1064	}
1065#ifdef MAC
1066	error = mac_pipe_check_write(active_cred, wpipe->pipe_pair);
1067	if (error) {
1068		pipeunlock(wpipe);
1069		PIPE_UNLOCK(rpipe);
1070		return (error);
1071	}
1072#endif
1073	++wpipe->pipe_busy;
1074
1075	/* Choose a larger size if it's advantageous */
1076	desiredsize = max(SMALL_PIPE_SIZE, wpipe->pipe_buffer.size);
1077	while (desiredsize < wpipe->pipe_buffer.cnt + uio->uio_resid) {
1078		if (piperesizeallowed != 1)
1079			break;
1080		if (amountpipekva > maxpipekva / 2)
1081			break;
1082		if (desiredsize == BIG_PIPE_SIZE)
1083			break;
1084		desiredsize = desiredsize * 2;
1085	}
1086
1087	/* Choose a smaller size if we're in a OOM situation */
1088	if (amountpipekva > (3 * maxpipekva) / 4 &&
1089	    wpipe->pipe_buffer.size > SMALL_PIPE_SIZE &&
1090	    wpipe->pipe_buffer.cnt <= SMALL_PIPE_SIZE &&
1091	    piperesizeallowed == 1)
1092		desiredsize = SMALL_PIPE_SIZE;
1093
1094	/* Resize if the above determined that a new size was necessary */
1095	if (desiredsize != wpipe->pipe_buffer.size &&
1096	    (wpipe->pipe_state & PIPE_DIRECTW) == 0) {
1097		PIPE_UNLOCK(wpipe);
1098		pipespace(wpipe, desiredsize);
1099		PIPE_LOCK(wpipe);
1100	}
1101	MPASS(wpipe->pipe_buffer.size != 0);
1102
1103	pipeunlock(wpipe);
1104
1105	orig_resid = uio->uio_resid;
1106
1107	while (uio->uio_resid) {
1108		int space;
1109
1110		pipelock(wpipe, 0);
1111		if (wpipe->pipe_state & PIPE_EOF) {
1112			pipeunlock(wpipe);
1113			error = EPIPE;
1114			break;
1115		}
1116#ifndef PIPE_NODIRECT
1117		/*
1118		 * If the transfer is large, we can gain performance if
1119		 * we do process-to-process copies directly.
1120		 * If the write is non-blocking, we don't use the
1121		 * direct write mechanism.
1122		 *
1123		 * The direct write mechanism will detect the reader going
1124		 * away on us.
1125		 */
1126		if (uio->uio_segflg == UIO_USERSPACE &&
1127		    uio->uio_iov->iov_len >= PIPE_MINDIRECT &&
1128		    wpipe->pipe_buffer.size >= PIPE_MINDIRECT &&
1129		    (fp->f_flag & FNONBLOCK) == 0) {
1130			pipeunlock(wpipe);
1131			error = pipe_direct_write(wpipe, uio);
1132			if (error)
1133				break;
1134			continue;
1135		}
1136#endif
1137
1138		/*
1139		 * Pipe buffered writes cannot be coincidental with
1140		 * direct writes.  We wait until the currently executing
1141		 * direct write is completed before we start filling the
1142		 * pipe buffer.  We break out if a signal occurs or the
1143		 * reader goes away.
1144		 */
1145		if (wpipe->pipe_map.cnt != 0) {
1146			if (wpipe->pipe_state & PIPE_WANTR) {
1147				wpipe->pipe_state &= ~PIPE_WANTR;
1148				wakeup(wpipe);
1149			}
1150			pipeselwakeup(wpipe);
1151			wpipe->pipe_state |= PIPE_WANTW;
1152			pipeunlock(wpipe);
1153			error = msleep(wpipe, PIPE_MTX(rpipe), PRIBIO | PCATCH,
1154			    "pipbww", 0);
1155			if (error)
1156				break;
1157			else
1158				continue;
1159		}
1160
1161		space = wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt;
1162
1163		/* Writes of size <= PIPE_BUF must be atomic. */
1164		if ((space < uio->uio_resid) && (orig_resid <= PIPE_BUF))
1165			space = 0;
1166
1167		if (space > 0) {
1168			int size;	/* Transfer size */
1169			int segsize;	/* first segment to transfer */
1170
1171			/*
1172			 * Transfer size is minimum of uio transfer
1173			 * and free space in pipe buffer.
1174			 */
1175			if (space > uio->uio_resid)
1176				size = uio->uio_resid;
1177			else
1178				size = space;
1179			/*
1180			 * First segment to transfer is minimum of
1181			 * transfer size and contiguous space in
1182			 * pipe buffer.  If first segment to transfer
1183			 * is less than the transfer size, we've got
1184			 * a wraparound in the buffer.
1185			 */
1186			segsize = wpipe->pipe_buffer.size -
1187				wpipe->pipe_buffer.in;
1188			if (segsize > size)
1189				segsize = size;
1190
1191			/* Transfer first segment */
1192
1193			PIPE_UNLOCK(rpipe);
1194			error = uiomove(&wpipe->pipe_buffer.buffer[wpipe->pipe_buffer.in],
1195					segsize, uio);
1196			PIPE_LOCK(rpipe);
1197
1198			if (error == 0 && segsize < size) {
1199				KASSERT(wpipe->pipe_buffer.in + segsize ==
1200					wpipe->pipe_buffer.size,
1201					("Pipe buffer wraparound disappeared"));
1202				/*
1203				 * Transfer remaining part now, to
1204				 * support atomic writes.  Wraparound
1205				 * happened.
1206				 */
1207
1208				PIPE_UNLOCK(rpipe);
1209				error = uiomove(
1210				    &wpipe->pipe_buffer.buffer[0],
1211				    size - segsize, uio);
1212				PIPE_LOCK(rpipe);
1213			}
1214			if (error == 0) {
1215				wpipe->pipe_buffer.in += size;
1216				if (wpipe->pipe_buffer.in >=
1217				    wpipe->pipe_buffer.size) {
1218					KASSERT(wpipe->pipe_buffer.in ==
1219						size - segsize +
1220						wpipe->pipe_buffer.size,
1221						("Expected wraparound bad"));
1222					wpipe->pipe_buffer.in = size - segsize;
1223				}
1224
1225				wpipe->pipe_buffer.cnt += size;
1226				KASSERT(wpipe->pipe_buffer.cnt <=
1227					wpipe->pipe_buffer.size,
1228					("Pipe buffer overflow"));
1229			}
1230			pipeunlock(wpipe);
1231			if (error != 0)
1232				break;
1233		} else {
1234			/*
1235			 * If the "read-side" has been blocked, wake it up now.
1236			 */
1237			if (wpipe->pipe_state & PIPE_WANTR) {
1238				wpipe->pipe_state &= ~PIPE_WANTR;
1239				wakeup(wpipe);
1240			}
1241
1242			/*
1243			 * don't block on non-blocking I/O
1244			 */
1245			if (fp->f_flag & FNONBLOCK) {
1246				error = EAGAIN;
1247				pipeunlock(wpipe);
1248				break;
1249			}
1250
1251			/*
1252			 * We have no more space and have something to offer,
1253			 * wake up select/poll.
1254			 */
1255			pipeselwakeup(wpipe);
1256
1257			wpipe->pipe_state |= PIPE_WANTW;
1258			pipeunlock(wpipe);
1259			error = msleep(wpipe, PIPE_MTX(rpipe),
1260			    PRIBIO | PCATCH, "pipewr", 0);
1261			if (error != 0)
1262				break;
1263		}
1264	}
1265
1266	pipelock(wpipe, 0);
1267	--wpipe->pipe_busy;
1268
1269	if ((wpipe->pipe_busy == 0) && (wpipe->pipe_state & PIPE_WANT)) {
1270		wpipe->pipe_state &= ~(PIPE_WANT | PIPE_WANTR);
1271		wakeup(wpipe);
1272	} else if (wpipe->pipe_buffer.cnt > 0) {
1273		/*
1274		 * If we have put any characters in the buffer, we wake up
1275		 * the reader.
1276		 */
1277		if (wpipe->pipe_state & PIPE_WANTR) {
1278			wpipe->pipe_state &= ~PIPE_WANTR;
1279			wakeup(wpipe);
1280		}
1281	}
1282
1283	/*
1284	 * Don't return EPIPE if any byte was written.
1285	 * EINTR and other interrupts are handled by generic I/O layer.
1286	 * Do not pretend that I/O succeeded for obvious user error
1287	 * like EFAULT.
1288	 */
1289	if (uio->uio_resid != orig_resid && error == EPIPE)
1290		error = 0;
1291
1292	if (error == 0)
1293		vfs_timestamp(&wpipe->pipe_mtime);
1294
1295	/*
1296	 * We have something to offer,
1297	 * wake up select/poll.
1298	 */
1299	if (wpipe->pipe_buffer.cnt)
1300		pipeselwakeup(wpipe);
1301
1302	pipeunlock(wpipe);
1303	PIPE_UNLOCK(rpipe);
1304	return (error);
1305}
1306
1307/* ARGSUSED */
1308static int
1309pipe_truncate(struct file *fp, off_t length, struct ucred *active_cred,
1310    struct thread *td)
1311{
1312	struct pipe *cpipe;
1313	int error;
1314
1315	cpipe = fp->f_data;
1316	if (cpipe->pipe_state & PIPE_NAMED)
1317		error = vnops.fo_truncate(fp, length, active_cred, td);
1318	else
1319		error = invfo_truncate(fp, length, active_cred, td);
1320	return (error);
1321}
1322
1323/*
1324 * we implement a very minimal set of ioctls for compatibility with sockets.
1325 */
1326static int
1327pipe_ioctl(struct file *fp, u_long cmd, void *data, struct ucred *active_cred,
1328    struct thread *td)
1329{
1330	struct pipe *mpipe = fp->f_data;
1331	int error;
1332
1333	PIPE_LOCK(mpipe);
1334
1335#ifdef MAC
1336	error = mac_pipe_check_ioctl(active_cred, mpipe->pipe_pair, cmd, data);
1337	if (error) {
1338		PIPE_UNLOCK(mpipe);
1339		return (error);
1340	}
1341#endif
1342
1343	error = 0;
1344	switch (cmd) {
1345
1346	case FIONBIO:
1347		break;
1348
1349	case FIOASYNC:
1350		if (*(int *)data) {
1351			mpipe->pipe_state |= PIPE_ASYNC;
1352		} else {
1353			mpipe->pipe_state &= ~PIPE_ASYNC;
1354		}
1355		break;
1356
1357	case FIONREAD:
1358		if (!(fp->f_flag & FREAD)) {
1359			*(int *)data = 0;
1360			PIPE_UNLOCK(mpipe);
1361			return (0);
1362		}
1363		if (mpipe->pipe_map.cnt != 0)
1364			*(int *)data = mpipe->pipe_map.cnt;
1365		else
1366			*(int *)data = mpipe->pipe_buffer.cnt;
1367		break;
1368
1369	case FIOSETOWN:
1370		PIPE_UNLOCK(mpipe);
1371		error = fsetown(*(int *)data, &mpipe->pipe_sigio);
1372		goto out_unlocked;
1373
1374	case FIOGETOWN:
1375		*(int *)data = fgetown(&mpipe->pipe_sigio);
1376		break;
1377
1378	/* This is deprecated, FIOSETOWN should be used instead. */
1379	case TIOCSPGRP:
1380		PIPE_UNLOCK(mpipe);
1381		error = fsetown(-(*(int *)data), &mpipe->pipe_sigio);
1382		goto out_unlocked;
1383
1384	/* This is deprecated, FIOGETOWN should be used instead. */
1385	case TIOCGPGRP:
1386		*(int *)data = -fgetown(&mpipe->pipe_sigio);
1387		break;
1388
1389	default:
1390		error = ENOTTY;
1391		break;
1392	}
1393	PIPE_UNLOCK(mpipe);
1394out_unlocked:
1395	return (error);
1396}
1397
1398static int
1399pipe_poll(struct file *fp, int events, struct ucred *active_cred,
1400    struct thread *td)
1401{
1402	struct pipe *rpipe;
1403	struct pipe *wpipe;
1404	int levents, revents;
1405#ifdef MAC
1406	int error;
1407#endif
1408
1409	revents = 0;
1410	rpipe = fp->f_data;
1411	wpipe = PIPE_PEER(rpipe);
1412	PIPE_LOCK(rpipe);
1413#ifdef MAC
1414	error = mac_pipe_check_poll(active_cred, rpipe->pipe_pair);
1415	if (error)
1416		goto locked_error;
1417#endif
1418	if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM))
1419		if (rpipe->pipe_map.cnt > 0 || rpipe->pipe_buffer.cnt > 0)
1420			revents |= events & (POLLIN | POLLRDNORM);
1421
1422	if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM))
1423		if (wpipe->pipe_present != PIPE_ACTIVE ||
1424		    (wpipe->pipe_state & PIPE_EOF) ||
1425		    ((wpipe->pipe_state & PIPE_DIRECTW) == 0 &&
1426		     ((wpipe->pipe_buffer.size - wpipe->pipe_buffer.cnt) >= PIPE_BUF ||
1427			 wpipe->pipe_buffer.size == 0)))
1428			revents |= events & (POLLOUT | POLLWRNORM);
1429
1430	levents = events &
1431	    (POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM | POLLRDBAND);
1432	if (rpipe->pipe_state & PIPE_NAMED && fp->f_flag & FREAD && levents &&
1433	    fp->f_pipegen == rpipe->pipe_wgen)
1434		events |= POLLINIGNEOF;
1435
1436	if ((events & POLLINIGNEOF) == 0) {
1437		if (rpipe->pipe_state & PIPE_EOF) {
1438			if (fp->f_flag & FREAD)
1439				revents |= (events & (POLLIN | POLLRDNORM));
1440			if (wpipe->pipe_present != PIPE_ACTIVE ||
1441			    (wpipe->pipe_state & PIPE_EOF))
1442				revents |= POLLHUP;
1443		}
1444	}
1445
1446	if (revents == 0) {
1447		if (fp->f_flag & FREAD && events & (POLLIN | POLLRDNORM)) {
1448			selrecord(td, &rpipe->pipe_sel);
1449			if (SEL_WAITING(&rpipe->pipe_sel))
1450				rpipe->pipe_state |= PIPE_SEL;
1451		}
1452
1453		if (fp->f_flag & FWRITE && events & (POLLOUT | POLLWRNORM)) {
1454			selrecord(td, &wpipe->pipe_sel);
1455			if (SEL_WAITING(&wpipe->pipe_sel))
1456				wpipe->pipe_state |= PIPE_SEL;
1457		}
1458	}
1459#ifdef MAC
1460locked_error:
1461#endif
1462	PIPE_UNLOCK(rpipe);
1463
1464	return (revents);
1465}
1466
1467/*
1468 * We shouldn't need locks here as we're doing a read and this should
1469 * be a natural race.
1470 */
1471static int
1472pipe_stat(struct file *fp, struct stat *ub, struct ucred *active_cred,
1473    struct thread *td)
1474{
1475	struct pipe *pipe;
1476#ifdef MAC
1477	int error;
1478#endif
1479
1480	pipe = fp->f_data;
1481	PIPE_LOCK(pipe);
1482#ifdef MAC
1483	error = mac_pipe_check_stat(active_cred, pipe->pipe_pair);
1484	if (error) {
1485		PIPE_UNLOCK(pipe);
1486		return (error);
1487	}
1488#endif
1489
1490	/* For named pipes ask the underlying filesystem. */
1491	if (pipe->pipe_state & PIPE_NAMED) {
1492		PIPE_UNLOCK(pipe);
1493		return (vnops.fo_stat(fp, ub, active_cred, td));
1494	}
1495
1496	PIPE_UNLOCK(pipe);
1497
1498	bzero(ub, sizeof(*ub));
1499	ub->st_mode = S_IFIFO;
1500	ub->st_blksize = PAGE_SIZE;
1501	if (pipe->pipe_map.cnt != 0)
1502		ub->st_size = pipe->pipe_map.cnt;
1503	else
1504		ub->st_size = pipe->pipe_buffer.cnt;
1505	ub->st_blocks = howmany(ub->st_size, ub->st_blksize);
1506	ub->st_atim = pipe->pipe_atime;
1507	ub->st_mtim = pipe->pipe_mtime;
1508	ub->st_ctim = pipe->pipe_ctime;
1509	ub->st_uid = fp->f_cred->cr_uid;
1510	ub->st_gid = fp->f_cred->cr_gid;
1511	ub->st_dev = pipedev_ino;
1512	ub->st_ino = pipe->pipe_ino;
1513	/*
1514	 * Left as 0: st_nlink, st_rdev, st_flags, st_gen.
1515	 */
1516	return (0);
1517}
1518
1519/* ARGSUSED */
1520static int
1521pipe_close(struct file *fp, struct thread *td)
1522{
1523
1524	if (fp->f_vnode != NULL)
1525		return vnops.fo_close(fp, td);
1526	fp->f_ops = &badfileops;
1527	pipe_dtor(fp->f_data);
1528	fp->f_data = NULL;
1529	return (0);
1530}
1531
1532static int
1533pipe_chmod(struct file *fp, mode_t mode, struct ucred *active_cred, struct thread *td)
1534{
1535	struct pipe *cpipe;
1536	int error;
1537
1538	cpipe = fp->f_data;
1539	if (cpipe->pipe_state & PIPE_NAMED)
1540		error = vn_chmod(fp, mode, active_cred, td);
1541	else
1542		error = invfo_chmod(fp, mode, active_cred, td);
1543	return (error);
1544}
1545
1546static int
1547pipe_chown(struct file *fp, uid_t uid, gid_t gid, struct ucred *active_cred,
1548    struct thread *td)
1549{
1550	struct pipe *cpipe;
1551	int error;
1552
1553	cpipe = fp->f_data;
1554	if (cpipe->pipe_state & PIPE_NAMED)
1555		error = vn_chown(fp, uid, gid, active_cred, td);
1556	else
1557		error = invfo_chown(fp, uid, gid, active_cred, td);
1558	return (error);
1559}
1560
1561static int
1562pipe_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
1563{
1564	struct pipe *pi;
1565
1566	if (fp->f_type == DTYPE_FIFO)
1567		return (vn_fill_kinfo(fp, kif, fdp));
1568	kif->kf_type = KF_TYPE_PIPE;
1569	pi = fp->f_data;
1570	kif->kf_un.kf_pipe.kf_pipe_addr = (uintptr_t)pi;
1571	kif->kf_un.kf_pipe.kf_pipe_peer = (uintptr_t)pi->pipe_peer;
1572	kif->kf_un.kf_pipe.kf_pipe_buffer_cnt = pi->pipe_buffer.cnt;
1573	return (0);
1574}
1575
1576static void
1577pipe_free_kmem(struct pipe *cpipe)
1578{
1579
1580	KASSERT(!mtx_owned(PIPE_MTX(cpipe)),
1581	    ("pipe_free_kmem: pipe mutex locked"));
1582
1583	if (cpipe->pipe_buffer.buffer != NULL) {
1584		atomic_subtract_long(&amountpipekva, cpipe->pipe_buffer.size);
1585		vm_map_remove(pipe_map,
1586		    (vm_offset_t)cpipe->pipe_buffer.buffer,
1587		    (vm_offset_t)cpipe->pipe_buffer.buffer + cpipe->pipe_buffer.size);
1588		cpipe->pipe_buffer.buffer = NULL;
1589	}
1590#ifndef PIPE_NODIRECT
1591	{
1592		cpipe->pipe_map.cnt = 0;
1593		cpipe->pipe_map.pos = 0;
1594		cpipe->pipe_map.npages = 0;
1595	}
1596#endif
1597}
1598
1599/*
1600 * shutdown the pipe
1601 */
1602static void
1603pipeclose(struct pipe *cpipe)
1604{
1605	struct pipepair *pp;
1606	struct pipe *ppipe;
1607
1608	KASSERT(cpipe != NULL, ("pipeclose: cpipe == NULL"));
1609
1610	PIPE_LOCK(cpipe);
1611	pipelock(cpipe, 0);
1612	pp = cpipe->pipe_pair;
1613
1614	/*
1615	 * If the other side is blocked, wake it up saying that
1616	 * we want to close it down.
1617	 */
1618	cpipe->pipe_state |= PIPE_EOF;
1619	while (cpipe->pipe_busy) {
1620		wakeup(cpipe);
1621		cpipe->pipe_state |= PIPE_WANT;
1622		pipeunlock(cpipe);
1623		msleep(cpipe, PIPE_MTX(cpipe), PRIBIO, "pipecl", 0);
1624		pipelock(cpipe, 0);
1625	}
1626
1627	pipeselwakeup(cpipe);
1628
1629	/*
1630	 * Disconnect from peer, if any.
1631	 */
1632	ppipe = cpipe->pipe_peer;
1633	if (ppipe->pipe_present == PIPE_ACTIVE) {
1634		ppipe->pipe_state |= PIPE_EOF;
1635		wakeup(ppipe);
1636		pipeselwakeup(ppipe);
1637	}
1638
1639	/*
1640	 * Mark this endpoint as free.  Release kmem resources.  We
1641	 * don't mark this endpoint as unused until we've finished
1642	 * doing that, or the pipe might disappear out from under
1643	 * us.
1644	 */
1645	PIPE_UNLOCK(cpipe);
1646	pipe_free_kmem(cpipe);
1647	PIPE_LOCK(cpipe);
1648	cpipe->pipe_present = PIPE_CLOSING;
1649	pipeunlock(cpipe);
1650
1651	/*
1652	 * knlist_clear() may sleep dropping the PIPE_MTX. Set the
1653	 * PIPE_FINALIZED, that allows other end to free the
1654	 * pipe_pair, only after the knotes are completely dismantled.
1655	 */
1656	knlist_clear(&cpipe->pipe_sel.si_note, 1);
1657	cpipe->pipe_present = PIPE_FINALIZED;
1658	seldrain(&cpipe->pipe_sel);
1659	knlist_destroy(&cpipe->pipe_sel.si_note);
1660
1661	/*
1662	 * If both endpoints are now closed, release the memory for the
1663	 * pipe pair.  If not, unlock.
1664	 */
1665	if (ppipe->pipe_present == PIPE_FINALIZED) {
1666		PIPE_UNLOCK(cpipe);
1667#ifdef MAC
1668		mac_pipe_destroy(pp);
1669#endif
1670		uma_zfree(pipe_zone, cpipe->pipe_pair);
1671	} else
1672		PIPE_UNLOCK(cpipe);
1673}
1674
1675/*ARGSUSED*/
1676static int
1677pipe_kqfilter(struct file *fp, struct knote *kn)
1678{
1679	struct pipe *cpipe;
1680
1681	/*
1682	 * If a filter is requested that is not supported by this file
1683	 * descriptor, don't return an error, but also don't ever generate an
1684	 * event.
1685	 */
1686	if ((kn->kn_filter == EVFILT_READ) && !(fp->f_flag & FREAD)) {
1687		kn->kn_fop = &pipe_nfiltops;
1688		return (0);
1689	}
1690	if ((kn->kn_filter == EVFILT_WRITE) && !(fp->f_flag & FWRITE)) {
1691		kn->kn_fop = &pipe_nfiltops;
1692		return (0);
1693	}
1694	cpipe = fp->f_data;
1695	PIPE_LOCK(cpipe);
1696	switch (kn->kn_filter) {
1697	case EVFILT_READ:
1698		kn->kn_fop = &pipe_rfiltops;
1699		break;
1700	case EVFILT_WRITE:
1701		kn->kn_fop = &pipe_wfiltops;
1702		if (cpipe->pipe_peer->pipe_present != PIPE_ACTIVE) {
1703			/* other end of pipe has been closed */
1704			PIPE_UNLOCK(cpipe);
1705			return (EPIPE);
1706		}
1707		cpipe = PIPE_PEER(cpipe);
1708		break;
1709	default:
1710		PIPE_UNLOCK(cpipe);
1711		return (EINVAL);
1712	}
1713
1714	kn->kn_hook = cpipe;
1715	knlist_add(&cpipe->pipe_sel.si_note, kn, 1);
1716	PIPE_UNLOCK(cpipe);
1717	return (0);
1718}
1719
1720static void
1721filt_pipedetach(struct knote *kn)
1722{
1723	struct pipe *cpipe = kn->kn_hook;
1724
1725	PIPE_LOCK(cpipe);
1726	knlist_remove(&cpipe->pipe_sel.si_note, kn, 1);
1727	PIPE_UNLOCK(cpipe);
1728}
1729
1730/*ARGSUSED*/
1731static int
1732filt_piperead(struct knote *kn, long hint)
1733{
1734	struct file *fp = kn->kn_fp;
1735	struct pipe *rpipe = kn->kn_hook;
1736
1737	PIPE_LOCK_ASSERT(rpipe, MA_OWNED);
1738	kn->kn_data = rpipe->pipe_buffer.cnt;
1739	if (kn->kn_data == 0)
1740		kn->kn_data = rpipe->pipe_map.cnt;
1741
1742	if ((rpipe->pipe_state & PIPE_EOF) != 0 &&
1743	    ((rpipe->pipe_state & PIPE_NAMED) == 0 ||
1744	    fp->f_pipegen != rpipe->pipe_wgen)) {
1745		kn->kn_flags |= EV_EOF;
1746		return (1);
1747	}
1748	kn->kn_flags &= ~EV_EOF;
1749	return (kn->kn_data > 0);
1750}
1751
1752/*ARGSUSED*/
1753static int
1754filt_pipewrite(struct knote *kn, long hint)
1755{
1756	struct pipe *wpipe = kn->kn_hook;
1757
1758	/*
1759	 * If this end of the pipe is closed, the knote was removed from the
1760	 * knlist and the list lock (i.e., the pipe lock) is therefore not held.
1761	 */
1762	if (wpipe->pipe_present == PIPE_ACTIVE ||
1763	    (wpipe->pipe_state & PIPE_NAMED) != 0) {
1764		PIPE_LOCK_ASSERT(wpipe, MA_OWNED);
1765
1766		if (wpipe->pipe_state & PIPE_DIRECTW) {
1767			kn->kn_data = 0;
1768		} else if (wpipe->pipe_buffer.size > 0) {
1769			kn->kn_data = wpipe->pipe_buffer.size -
1770			    wpipe->pipe_buffer.cnt;
1771		} else {
1772			kn->kn_data = PIPE_BUF;
1773		}
1774	}
1775
1776	if (wpipe->pipe_present != PIPE_ACTIVE ||
1777	    (wpipe->pipe_state & PIPE_EOF)) {
1778		kn->kn_flags |= EV_EOF;
1779		return (1);
1780	}
1781	kn->kn_flags &= ~EV_EOF;
1782	return (kn->kn_data >= PIPE_BUF);
1783}
1784
1785static void
1786filt_pipedetach_notsup(struct knote *kn)
1787{
1788
1789}
1790
1791static int
1792filt_pipenotsup(struct knote *kn, long hint)
1793{
1794
1795	return (0);
1796}
1797