1/*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1990, 1993
5 *	The Regents of the University of California.
6 * Copyright (c) 2010-2011 Juniper Networks, Inc.
7 * All rights reserved.
8 *
9 * Portions of this software were developed by Robert N. M. Watson under
10 * contract to Juniper Networks, Inc.
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 *    notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 *    notice, this list of conditions and the following disclaimer in the
19 *    documentation and/or other materials provided with the distribution.
20 * 3. Neither the name of the University nor the names of its contributors
21 *    may be used to endorse or promote products derived from this software
22 *    without specific prior written permission.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 *
36 *	@(#)in_pcb.h	8.1 (Berkeley) 6/10/93
37 * $FreeBSD$
38 */
39
40#ifndef _NETINET_IN_PCB_H_
41#define _NETINET_IN_PCB_H_
42
43#include <sys/queue.h>
44#include <sys/epoch.h>
45#include <sys/_lock.h>
46#include <sys/_mutex.h>
47#include <sys/_rwlock.h>
48#include <net/route.h>
49
50#ifdef _KERNEL
51#include <sys/lock.h>
52#include <sys/rwlock.h>
53#include <net/vnet.h>
54#include <vm/uma.h>
55#endif
56#include <sys/ck.h>
57
58/*
59 * struct inpcb is the common protocol control block structure used in most
60 * IP transport protocols.
61 *
62 * Pointers to local and foreign host table entries, local and foreign socket
63 * numbers, and pointers up (to a socket structure) and down (to a
64 * protocol-specific control block) are stored here.
65 */
66CK_LIST_HEAD(inpcbhead, inpcb);
67CK_LIST_HEAD(inpcbporthead, inpcbport);
68CK_LIST_HEAD(inpcblbgrouphead, inpcblbgroup);
69typedef	uint64_t	inp_gen_t;
70
71/*
72 * PCB with AF_INET6 null bind'ed laddr can receive AF_INET input packet.
73 * So, AF_INET6 null laddr is also used as AF_INET null laddr, by utilizing
74 * the following structure.
75 */
76struct in_addr_4in6 {
77	u_int32_t	ia46_pad32[3];
78	struct	in_addr	ia46_addr4;
79};
80
81union in_dependaddr {
82	struct in_addr_4in6 id46_addr;
83	struct in6_addr	id6_addr;
84};
85
86/*
87 * NOTE: ipv6 addrs should be 64-bit aligned, per RFC 2553.  in_conninfo has
88 * some extra padding to accomplish this.
89 * NOTE 2: tcp_syncache.c uses first 5 32-bit words, which identify fport,
90 * lport, faddr to generate hash, so these fields shouldn't be moved.
91 */
92struct in_endpoints {
93	u_int16_t	ie_fport;		/* foreign port */
94	u_int16_t	ie_lport;		/* local port */
95	/* protocol dependent part, local and foreign addr */
96	union in_dependaddr ie_dependfaddr;	/* foreign host table entry */
97	union in_dependaddr ie_dependladdr;	/* local host table entry */
98#define	ie_faddr	ie_dependfaddr.id46_addr.ia46_addr4
99#define	ie_laddr	ie_dependladdr.id46_addr.ia46_addr4
100#define	ie6_faddr	ie_dependfaddr.id6_addr
101#define	ie6_laddr	ie_dependladdr.id6_addr
102	u_int32_t	ie6_zoneid;		/* scope zone id */
103};
104
105/*
106 * XXX The defines for inc_* are hacks and should be changed to direct
107 * references.
108 */
109struct in_conninfo {
110	u_int8_t	inc_flags;
111	u_int8_t	inc_len;
112	u_int16_t	inc_fibnum;	/* XXX was pad, 16 bits is plenty */
113	/* protocol dependent part */
114	struct	in_endpoints inc_ie;
115};
116
117/*
118 * Flags for inc_flags.
119 */
120#define	INC_ISIPV6	0x01
121#define	INC_IPV6MINMTU	0x02
122
123#define	inc_fport	inc_ie.ie_fport
124#define	inc_lport	inc_ie.ie_lport
125#define	inc_faddr	inc_ie.ie_faddr
126#define	inc_laddr	inc_ie.ie_laddr
127#define	inc6_faddr	inc_ie.ie6_faddr
128#define	inc6_laddr	inc_ie.ie6_laddr
129#define	inc6_zoneid	inc_ie.ie6_zoneid
130
131#if defined(_KERNEL) || defined(_WANT_INPCB)
132/*
133 * struct inpcb captures the network layer state for TCP, UDP, and raw IPv4 and
134 * IPv6 sockets.  In the case of TCP and UDP, further per-connection state is
135 * hung off of inp_ppcb most of the time.  Almost all fields of struct inpcb
136 * are static after creation or protected by a per-inpcb rwlock, inp_lock.  A
137 * few fields are protected by multiple locks as indicated in the locking notes
138 * below.  For these fields, all of the listed locks must be write-locked for
139 * any modifications.  However, these fields can be safely read while any one of
140 * the listed locks are read-locked.  This model can permit greater concurrency
141 * for read operations.  For example, connections can be looked up while only
142 * holding a read lock on the global pcblist lock.  This is important for
143 * performance when attempting to find the connection for a packet given its IP
144 * and port tuple.
145 *
146 * One noteworthy exception is that the global pcbinfo lock follows a different
147 * set of rules in relation to the inp_list field.  Rather than being
148 * write-locked for modifications and read-locked for list iterations, it must
149 * be read-locked during modifications and write-locked during list iterations.
150 * This ensures that the relatively rare global list iterations safely walk a
151 * stable snapshot of connections while allowing more common list modifications
152 * to safely grab the pcblist lock just while adding or removing a connection
153 * from the global list.
154 *
155 * Key:
156 * (b) - Protected by the hpts lock.
157 * (c) - Constant after initialization
158 * (e) - Protected by the net_epoch_prempt epoch
159 * (g) - Protected by the pcbgroup lock
160 * (i) - Protected by the inpcb lock
161 * (p) - Protected by the pcbinfo lock for the inpcb
162 * (l) - Protected by the pcblist lock for the inpcb
163 * (h) - Protected by the pcbhash lock for the inpcb
164 * (s) - Protected by another subsystem's locks
165 * (x) - Undefined locking
166 *
167 * Notes on the tcp_hpts:
168 *
169 * First Hpts lock order is
170 * 1) INP_WLOCK()
171 * 2) HPTS_LOCK() i.e. hpts->pmtx
172 *
173 * To insert a TCB on the hpts you *must* be holding the INP_WLOCK().
174 * You may check the inp->inp_in_hpts flag without the hpts lock.
175 * The hpts is the only one that will clear this flag holding
176 * only the hpts lock. This means that in your tcp_output()
177 * routine when you test for the inp_in_hpts flag to be 1
178 * it may be transitioning to 0 (by the hpts).
179 * That's ok since that will just mean an extra call to tcp_output
180 * that most likely will find the call you executed
181 * (when the mis-match occured) will have put the TCB back
182 * on the hpts and it will return. If your
183 * call did not add the inp back to the hpts then you will either
184 * over-send or the cwnd will block you from sending more.
185 *
186 * Note you should also be holding the INP_WLOCK() when you
187 * call the remove from the hpts as well. Though usually
188 * you are either doing this from a timer, where you need and have
189 * the INP_WLOCK() or from destroying your TCB where again
190 * you should already have the INP_WLOCK().
191 *
192 * The inp_hpts_cpu, inp_hpts_cpu_set, inp_input_cpu and
193 * inp_input_cpu_set fields are controlled completely by
194 * the hpts. Do not ever set these. The inp_hpts_cpu_set
195 * and inp_input_cpu_set fields indicate if the hpts has
196 * setup the respective cpu field. It is advised if this
197 * field is 0, to enqueue the packet with the appropriate
198 * hpts_immediate() call. If the _set field is 1, then
199 * you may compare the inp_*_cpu field to the curcpu and
200 * may want to again insert onto the hpts if these fields
201 * are not equal (i.e. you are not on the expected CPU).
202 *
203 * A note on inp_hpts_calls and inp_input_calls, these
204 * flags are set when the hpts calls either the output
205 * or do_segment routines respectively. If the routine
206 * being called wants to use this, then it needs to
207 * clear the flag before returning. The hpts will not
208 * clear the flag. The flags can be used to tell if
209 * the hpts is the function calling the respective
210 * routine.
211 *
212 * A few other notes:
213 *
214 * When a read lock is held, stability of the field is guaranteed; to write
215 * to a field, a write lock must generally be held.
216 *
217 * netinet/netinet6-layer code should not assume that the inp_socket pointer
218 * is safe to dereference without inp_lock being held, even for protocols
219 * other than TCP (where the inpcb persists during TIMEWAIT even after the
220 * socket has been freed), or there may be close(2)-related races.
221 *
222 * The inp_vflag field is overloaded, and would otherwise ideally be (c).
223 *
224 * TODO:  Currently only the TCP stack is leveraging the global pcbinfo lock
225 * read-lock usage during modification, this model can be applied to other
226 * protocols (especially SCTP).
227 */
228struct icmp6_filter;
229struct inpcbpolicy;
230struct m_snd_tag;
231struct inpcb {
232	/* Cache line #1 (amd64) */
233	CK_LIST_ENTRY(inpcb) inp_hash;	/* [w](h/i) [r](e/i)  hash list */
234	CK_LIST_ENTRY(inpcb) inp_pcbgrouphash;	/* (g/i) hash list */
235	struct rwlock	inp_lock;
236	/* Cache line #2 (amd64) */
237#define	inp_start_zero	inp_hpts
238#define	inp_zero_size	(sizeof(struct inpcb) - \
239			    offsetof(struct inpcb, inp_start_zero))
240	TAILQ_ENTRY(inpcb) inp_hpts;	/* pacing out queue next lock(b) */
241
242	uint32_t inp_hpts_request;	/* Current hpts request, zero if
243					 * fits in the pacing window (i&b). */
244	/*
245	 * Note the next fields are protected by a
246	 * different lock (hpts-lock). This means that
247	 * they must correspond in size to the smallest
248	 * protectable bit field (uint8_t on x86, and
249	 * other platfomrs potentially uint32_t?). Also
250	 * since CPU switches can occur at different times the two
251	 * fields can *not* be collapsed into a signal bit field.
252	 */
253#if defined(__amd64__) || defined(__i386__)
254	volatile uint8_t inp_in_hpts; /* on output hpts (lock b) */
255	volatile uint8_t inp_in_input; /* on input hpts (lock b) */
256#else
257	volatile uint32_t inp_in_hpts; /* on output hpts (lock b) */
258	volatile uint32_t inp_in_input; /* on input hpts (lock b) */
259#endif
260	volatile uint16_t  inp_hpts_cpu; /* Lock (i) */
261	u_int	inp_refcount;		/* (i) refcount */
262	int	inp_flags;		/* (i) generic IP/datagram flags */
263	int	inp_flags2;		/* (i) generic IP/datagram flags #2*/
264	volatile uint16_t  inp_input_cpu; /* Lock (i) */
265	volatile uint8_t inp_hpts_cpu_set :1,  /* on output hpts (i) */
266			 inp_input_cpu_set : 1,	/* on input hpts (i) */
267			 inp_hpts_calls :1,	/* (i) from output hpts */
268			 inp_input_calls :1,	/* (i) from input hpts */
269			 inp_spare_bits2 : 4;
270	uint8_t inp_numa_domain;	/* numa domain */
271	void	*inp_ppcb;		/* (i) pointer to per-protocol pcb */
272	struct	socket *inp_socket;	/* (i) back pointer to socket */
273	uint32_t 	 inp_hptsslot;	/* Hpts wheel slot this tcb is Lock(i&b) */
274	uint32_t         inp_hpts_drop_reas;	/* reason we are dropping the PCB (lock i&b) */
275	TAILQ_ENTRY(inpcb) inp_input;	/* pacing in  queue next lock(b) */
276	struct	inpcbinfo *inp_pcbinfo;	/* (c) PCB list info */
277	struct	inpcbgroup *inp_pcbgroup; /* (g/i) PCB group list */
278	CK_LIST_ENTRY(inpcb) inp_pcbgroup_wild; /* (g/i/h) group wildcard entry */
279	struct	ucred	*inp_cred;	/* (c) cache of socket cred */
280	u_int32_t inp_flow;		/* (i) IPv6 flow information */
281	u_char	inp_vflag;		/* (i) IP version flag (v4/v6) */
282	u_char	inp_ip_ttl;		/* (i) time to live proto */
283	u_char	inp_ip_p;		/* (c) protocol proto */
284	u_char	inp_ip_minttl;		/* (i) minimum TTL or drop */
285	uint32_t inp_flowid;		/* (x) flow id / queue id */
286	struct m_snd_tag *inp_snd_tag;	/* (i) send tag for outgoing mbufs */
287	uint32_t inp_flowtype;		/* (x) M_HASHTYPE value */
288	uint32_t inp_rss_listen_bucket;	/* (x) overridden RSS listen bucket */
289
290	/* Local and foreign ports, local and foreign addr. */
291	struct	in_conninfo inp_inc;	/* (i) list for PCB's local port */
292
293	/* MAC and IPSEC policy information. */
294	struct	label *inp_label;	/* (i) MAC label */
295	struct	inpcbpolicy *inp_sp;    /* (s) for IPSEC */
296
297	/* Protocol-dependent part; options. */
298	struct {
299		u_char	inp_ip_tos;		/* (i) type of service proto */
300		struct mbuf		*inp_options;	/* (i) IP options */
301		struct ip_moptions	*inp_moptions;	/* (i) mcast options */
302	};
303	struct {
304		/* (i) IP options */
305		struct mbuf		*in6p_options;
306		/* (i) IP6 options for outgoing packets */
307		struct ip6_pktopts	*in6p_outputopts;
308		/* (i) IP multicast options */
309		struct ip6_moptions	*in6p_moptions;
310		/* (i) ICMPv6 code type filter */
311		struct icmp6_filter	*in6p_icmp6filt;
312		/* (i) IPV6_CHECKSUM setsockopt */
313		int	in6p_cksum;
314		short	in6p_hops;
315	};
316	CK_LIST_ENTRY(inpcb) inp_portlist;	/* (i/h) */
317	struct	inpcbport *inp_phd;	/* (i/h) head of this list */
318	inp_gen_t	inp_gencnt;	/* (c) generation count */
319	void		*spare_ptr;	/* Spare pointer. */
320	rt_gen_t	inp_rt_cookie;	/* generation for route entry */
321	union {				/* cached L3 information */
322		struct route inp_route;
323		struct route_in6 inp_route6;
324	};
325	CK_LIST_ENTRY(inpcb) inp_list;	/* (p/l) list for all PCBs for proto */
326	                                /* (e[r]) for list iteration */
327	                                /* (p[w]/l) for addition/removal */
328	struct epoch_context inp_epoch_ctx;
329};
330#endif	/* _KERNEL */
331
332#define	inp_fport	inp_inc.inc_fport
333#define	inp_lport	inp_inc.inc_lport
334#define	inp_faddr	inp_inc.inc_faddr
335#define	inp_laddr	inp_inc.inc_laddr
336
337#define	in6p_faddr	inp_inc.inc6_faddr
338#define	in6p_laddr	inp_inc.inc6_laddr
339#define	in6p_zoneid	inp_inc.inc6_zoneid
340
341#define	inp_vnet	inp_pcbinfo->ipi_vnet
342
343/*
344 * The range of the generation count, as used in this implementation, is 9e19.
345 * We would have to create 300 billion connections per second for this number
346 * to roll over in a year.  This seems sufficiently unlikely that we simply
347 * don't concern ourselves with that possibility.
348 */
349
350/*
351 * Interface exported to userland by various protocols which use inpcbs.  Hack
352 * alert -- only define if struct xsocket is in scope.
353 * Fields prefixed with "xi_" are unique to this structure, and the rest
354 * match fields in the struct inpcb, to ease coding and porting.
355 *
356 * Legend:
357 * (s) - used by userland utilities in src
358 * (p) - used by utilities in ports
359 * (3) - is known to be used by third party software not in ports
360 * (n) - no known usage
361 */
362#ifdef _SYS_SOCKETVAR_H_
363struct xinpcb {
364	ksize_t		xi_len;			/* length of this structure */
365	struct xsocket	xi_socket;		/* (s,p) */
366	struct in_conninfo inp_inc;		/* (s,p) */
367	uint64_t	inp_gencnt;		/* (s,p) */
368	kvaddr_t	inp_ppcb;		/* (s) netstat(1) */
369	int64_t		inp_spare64[4];
370	uint32_t	inp_flow;		/* (s) */
371	uint32_t	inp_flowid;		/* (s) */
372	uint32_t	inp_flowtype;		/* (s) */
373	int32_t		inp_flags;		/* (s,p) */
374	int32_t		inp_flags2;		/* (s) */
375	int32_t		inp_rss_listen_bucket;	/* (n) */
376	int32_t		in6p_cksum;		/* (n) */
377	int32_t		inp_spare32[4];
378	uint16_t	in6p_hops;		/* (n) */
379	uint8_t		inp_ip_tos;		/* (n) */
380	int8_t		pad8;
381	uint8_t		inp_vflag;		/* (s,p) */
382	uint8_t		inp_ip_ttl;		/* (n) */
383	uint8_t		inp_ip_p;		/* (n) */
384	uint8_t		inp_ip_minttl;		/* (n) */
385	int8_t		inp_spare8[4];
386} __aligned(8);
387
388struct xinpgen {
389	ksize_t	xig_len;	/* length of this structure */
390	u_int		xig_count;	/* number of PCBs at this time */
391	uint32_t	_xig_spare32;
392	inp_gen_t	xig_gen;	/* generation count at this time */
393	so_gen_t	xig_sogen;	/* socket generation count this time */
394	uint64_t	_xig_spare64[4];
395} __aligned(8);
396#ifdef	_KERNEL
397void	in_pcbtoxinpcb(const struct inpcb *, struct xinpcb *);
398#endif
399#endif /* _SYS_SOCKETVAR_H_ */
400
401struct inpcbport {
402	struct epoch_context phd_epoch_ctx;
403	CK_LIST_ENTRY(inpcbport) phd_hash;
404	struct inpcbhead phd_pcblist;
405	u_short phd_port;
406};
407
408struct in_pcblist {
409	int il_count;
410	struct epoch_context il_epoch_ctx;
411	struct inpcbinfo *il_pcbinfo;
412	struct inpcb *il_inp_list[0];
413};
414
415/*-
416 * Global data structure for each high-level protocol (UDP, TCP, ...) in both
417 * IPv4 and IPv6.  Holds inpcb lists and information for managing them.
418 *
419 * Each pcbinfo is protected by three locks: ipi_lock, ipi_hash_lock and
420 * ipi_list_lock:
421 *  - ipi_lock covering the global pcb list stability during loop iteration,
422 *  - ipi_hash_lock covering the hashed lookup tables,
423 *  - ipi_list_lock covering mutable global fields (such as the global
424 *    pcb list)
425 *
426 * The lock order is:
427 *
428 *    ipi_lock (before)
429 *        inpcb locks (before)
430 *            ipi_list locks (before)
431 *                {ipi_hash_lock, pcbgroup locks}
432 *
433 * Locking key:
434 *
435 * (c) Constant or nearly constant after initialisation
436 * (e) - Protected by the net_epoch_prempt epoch
437 * (g) Locked by ipi_lock
438 * (l) Locked by ipi_list_lock
439 * (h) Read using either net_epoch_preempt or inpcb lock; write requires both ipi_hash_lock and inpcb lock
440 * (p) Protected by one or more pcbgroup locks
441 * (x) Synchronisation properties poorly defined
442 */
443struct inpcbinfo {
444	/*
445	 * Global lock protecting inpcb list modification
446	 */
447	struct mtx		 ipi_lock;
448
449	/*
450	 * Global list of inpcbs on the protocol.
451	 */
452	struct inpcbhead	*ipi_listhead;		/* [r](e) [w](g/l) */
453	u_int			 ipi_count;		/* (l) */
454
455	/*
456	 * Generation count -- incremented each time a connection is allocated
457	 * or freed.
458	 */
459	u_quad_t		 ipi_gencnt;		/* (l) */
460
461	/*
462	 * Fields associated with port lookup and allocation.
463	 */
464	u_short			 ipi_lastport;		/* (x) */
465	u_short			 ipi_lastlow;		/* (x) */
466	u_short			 ipi_lasthi;		/* (x) */
467
468	/*
469	 * UMA zone from which inpcbs are allocated for this protocol.
470	 */
471	struct	uma_zone	*ipi_zone;		/* (c) */
472
473	/*
474	 * Connection groups associated with this protocol.  These fields are
475	 * constant, but pcbgroup structures themselves are protected by
476	 * per-pcbgroup locks.
477	 */
478	struct inpcbgroup	*ipi_pcbgroups;		/* (c) */
479	u_int			 ipi_npcbgroups;	/* (c) */
480	u_int			 ipi_hashfields;	/* (c) */
481
482	/*
483	 * Global lock protecting modification non-pcbgroup hash lookup tables.
484	 */
485	struct mtx		 ipi_hash_lock;
486
487	/*
488	 * Global hash of inpcbs, hashed by local and foreign addresses and
489	 * port numbers.
490	 */
491	struct inpcbhead	*ipi_hashbase;		/* (h) */
492	u_long			 ipi_hashmask;		/* (h) */
493
494	/*
495	 * Global hash of inpcbs, hashed by only local port number.
496	 */
497	struct inpcbporthead	*ipi_porthashbase;	/* (h) */
498	u_long			 ipi_porthashmask;	/* (h) */
499
500	/*
501	 * List of wildcard inpcbs for use with pcbgroups.  In the past, was
502	 * per-pcbgroup but is now global.  All pcbgroup locks must be held
503	 * to modify the list, so any is sufficient to read it.
504	 */
505	struct inpcbhead	*ipi_wildbase;		/* (p) */
506	u_long			 ipi_wildmask;		/* (p) */
507
508	/*
509	 * Load balance groups used for the SO_REUSEPORT_LB option,
510	 * hashed by local port.
511	 */
512	struct	inpcblbgrouphead *ipi_lbgrouphashbase;	/* (h) */
513	u_long			 ipi_lbgrouphashmask;	/* (h) */
514
515	/*
516	 * Pointer to network stack instance
517	 */
518	struct vnet		*ipi_vnet;		/* (c) */
519
520	/*
521	 * general use 2
522	 */
523	void 			*ipi_pspare[2];
524
525	/*
526	 * Global lock protecting global inpcb list, inpcb count, etc.
527	 */
528	struct rwlock		 ipi_list_lock;
529};
530
531#ifdef _KERNEL
532/*
533 * Connection groups hold sets of connections that have similar CPU/thread
534 * affinity.  Each connection belongs to exactly one connection group.
535 */
536struct inpcbgroup {
537	/*
538	 * Per-connection group hash of inpcbs, hashed by local and foreign
539	 * addresses and port numbers.
540	 */
541	struct inpcbhead	*ipg_hashbase;		/* (c) */
542	u_long			 ipg_hashmask;		/* (c) */
543
544	/*
545	 * Notional affinity of this pcbgroup.
546	 */
547	u_int			 ipg_cpu;		/* (p) */
548
549	/*
550	 * Per-connection group lock, not to be confused with ipi_lock.
551	 * Protects the hash table hung off the group, but also the global
552	 * wildcard list in inpcbinfo.
553	 */
554	struct mtx		 ipg_lock;
555} __aligned(CACHE_LINE_SIZE);
556
557/*
558 * Load balance groups used for the SO_REUSEPORT_LB socket option. Each group
559 * (or unique address:port combination) can be re-used at most
560 * INPCBLBGROUP_SIZMAX (256) times. The inpcbs are stored in il_inp which
561 * is dynamically resized as processes bind/unbind to that specific group.
562 */
563struct inpcblbgroup {
564	CK_LIST_ENTRY(inpcblbgroup) il_list;
565	struct epoch_context il_epoch_ctx;
566	uint16_t	il_lport;			/* (c) */
567	u_char		il_vflag;			/* (c) */
568	u_char		il_pad;
569	uint32_t	il_pad2;
570	union in_dependaddr il_dependladdr;		/* (c) */
571#define	il_laddr	il_dependladdr.id46_addr.ia46_addr4
572#define	il6_laddr	il_dependladdr.id6_addr
573	uint32_t	il_inpsiz; /* max count in il_inp[] (h) */
574	uint32_t	il_inpcnt; /* cur count in il_inp[] (h) */
575	struct inpcb	*il_inp[];			/* (h) */
576};
577
578#define INP_LOCK_INIT(inp, d, t) \
579	rw_init_flags(&(inp)->inp_lock, (t), RW_RECURSE |  RW_DUPOK)
580#define INP_LOCK_DESTROY(inp)	rw_destroy(&(inp)->inp_lock)
581#define INP_RLOCK(inp)		rw_rlock(&(inp)->inp_lock)
582#define INP_WLOCK(inp)		rw_wlock(&(inp)->inp_lock)
583#define INP_TRY_RLOCK(inp)	rw_try_rlock(&(inp)->inp_lock)
584#define INP_TRY_WLOCK(inp)	rw_try_wlock(&(inp)->inp_lock)
585#define INP_RUNLOCK(inp)	rw_runlock(&(inp)->inp_lock)
586#define INP_WUNLOCK(inp)	rw_wunlock(&(inp)->inp_lock)
587#define INP_UNLOCK(inp)		rw_unlock(&(inp)->inp_lock)
588#define	INP_TRY_UPGRADE(inp)	rw_try_upgrade(&(inp)->inp_lock)
589#define	INP_DOWNGRADE(inp)	rw_downgrade(&(inp)->inp_lock)
590#define	INP_WLOCKED(inp)	rw_wowned(&(inp)->inp_lock)
591#define	INP_LOCK_ASSERT(inp)	rw_assert(&(inp)->inp_lock, RA_LOCKED)
592#define	INP_RLOCK_ASSERT(inp)	rw_assert(&(inp)->inp_lock, RA_RLOCKED)
593#define	INP_WLOCK_ASSERT(inp)	rw_assert(&(inp)->inp_lock, RA_WLOCKED)
594#define	INP_UNLOCK_ASSERT(inp)	rw_assert(&(inp)->inp_lock, RA_UNLOCKED)
595
596/*
597 * These locking functions are for inpcb consumers outside of sys/netinet,
598 * more specifically, they were added for the benefit of TOE drivers. The
599 * macros are reserved for use by the stack.
600 */
601void inp_wlock(struct inpcb *);
602void inp_wunlock(struct inpcb *);
603void inp_rlock(struct inpcb *);
604void inp_runlock(struct inpcb *);
605
606#ifdef INVARIANT_SUPPORT
607void inp_lock_assert(struct inpcb *);
608void inp_unlock_assert(struct inpcb *);
609#else
610#define	inp_lock_assert(inp)	do {} while (0)
611#define	inp_unlock_assert(inp)	do {} while (0)
612#endif
613
614void	inp_apply_all(void (*func)(struct inpcb *, void *), void *arg);
615int 	inp_ip_tos_get(const struct inpcb *inp);
616void 	inp_ip_tos_set(struct inpcb *inp, int val);
617struct socket *
618	inp_inpcbtosocket(struct inpcb *inp);
619struct tcpcb *
620	inp_inpcbtotcpcb(struct inpcb *inp);
621void 	inp_4tuple_get(struct inpcb *inp, uint32_t *laddr, uint16_t *lp,
622		uint32_t *faddr, uint16_t *fp);
623int	inp_so_options(const struct inpcb *inp);
624
625#endif /* _KERNEL */
626
627#define INP_INFO_LOCK_INIT(ipi, d) \
628	mtx_init(&(ipi)->ipi_lock, (d), NULL, MTX_DEF| MTX_RECURSE)
629#define INP_INFO_LOCK_DESTROY(ipi)  mtx_destroy(&(ipi)->ipi_lock)
630#define INP_INFO_WLOCK(ipi) mtx_lock(&(ipi)->ipi_lock)
631#define INP_INFO_TRY_WLOCK(ipi)	mtx_trylock(&(ipi)->ipi_lock)
632#define INP_INFO_WLOCKED(ipi)	mtx_owned(&(ipi)->ipi_lock)
633#define INP_INFO_WUNLOCK(ipi)	mtx_unlock(&(ipi)->ipi_lock)
634#define	INP_INFO_LOCK_ASSERT(ipi)	MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(ipi)->ipi_lock))
635#define INP_INFO_WLOCK_ASSERT(ipi)	mtx_assert(&(ipi)->ipi_lock, MA_OWNED)
636#define INP_INFO_WUNLOCK_ASSERT(ipi)	\
637	mtx_assert(&(ipi)->ipi_lock, MA_NOTOWNED)
638
639#define INP_LIST_LOCK_INIT(ipi, d) \
640        rw_init_flags(&(ipi)->ipi_list_lock, (d), 0)
641#define INP_LIST_LOCK_DESTROY(ipi)  rw_destroy(&(ipi)->ipi_list_lock)
642#define INP_LIST_RLOCK(ipi)     rw_rlock(&(ipi)->ipi_list_lock)
643#define INP_LIST_WLOCK(ipi)     rw_wlock(&(ipi)->ipi_list_lock)
644#define INP_LIST_TRY_RLOCK(ipi) rw_try_rlock(&(ipi)->ipi_list_lock)
645#define INP_LIST_TRY_WLOCK(ipi) rw_try_wlock(&(ipi)->ipi_list_lock)
646#define INP_LIST_TRY_UPGRADE(ipi)       rw_try_upgrade(&(ipi)->ipi_list_lock)
647#define INP_LIST_RUNLOCK(ipi)   rw_runlock(&(ipi)->ipi_list_lock)
648#define INP_LIST_WUNLOCK(ipi)   rw_wunlock(&(ipi)->ipi_list_lock)
649#define INP_LIST_LOCK_ASSERT(ipi) \
650	rw_assert(&(ipi)->ipi_list_lock, RA_LOCKED)
651#define INP_LIST_RLOCK_ASSERT(ipi) \
652	rw_assert(&(ipi)->ipi_list_lock, RA_RLOCKED)
653#define INP_LIST_WLOCK_ASSERT(ipi) \
654	rw_assert(&(ipi)->ipi_list_lock, RA_WLOCKED)
655#define INP_LIST_UNLOCK_ASSERT(ipi) \
656	rw_assert(&(ipi)->ipi_list_lock, RA_UNLOCKED)
657
658#define	INP_HASH_LOCK_INIT(ipi, d) mtx_init(&(ipi)->ipi_hash_lock, (d), NULL, MTX_DEF)
659#define	INP_HASH_LOCK_DESTROY(ipi)	mtx_destroy(&(ipi)->ipi_hash_lock)
660#define	INP_HASH_WLOCK(ipi)		mtx_lock(&(ipi)->ipi_hash_lock)
661#define	INP_HASH_WUNLOCK(ipi)		mtx_unlock(&(ipi)->ipi_hash_lock)
662#define	INP_HASH_LOCK_ASSERT(ipi)	MPASS(in_epoch(net_epoch_preempt) || mtx_owned(&(ipi)->ipi_hash_lock))
663#define	INP_HASH_WLOCK_ASSERT(ipi)	mtx_assert(&(ipi)->ipi_hash_lock, MA_OWNED);
664
665#define	INP_GROUP_LOCK_INIT(ipg, d)	mtx_init(&(ipg)->ipg_lock, (d), NULL, \
666					    MTX_DEF | MTX_DUPOK)
667#define	INP_GROUP_LOCK_DESTROY(ipg)	mtx_destroy(&(ipg)->ipg_lock)
668
669#define	INP_GROUP_LOCK(ipg)		mtx_lock(&(ipg)->ipg_lock)
670#define	INP_GROUP_LOCK_ASSERT(ipg)	mtx_assert(&(ipg)->ipg_lock, MA_OWNED)
671#define	INP_GROUP_UNLOCK(ipg)		mtx_unlock(&(ipg)->ipg_lock)
672
673#define INP_PCBHASH(faddr, lport, fport, mask) \
674	(((faddr) ^ ((faddr) >> 16) ^ ntohs((lport) ^ (fport))) & (mask))
675#define INP_PCBPORTHASH(lport, mask) \
676	(ntohs((lport)) & (mask))
677#define	INP_PCBLBGROUP_PKTHASH(faddr, lport, fport) \
678	((faddr) ^ ((faddr) >> 16) ^ ntohs((lport) ^ (fport)))
679#define	INP6_PCBHASHKEY(faddr)	((faddr)->s6_addr32[3])
680
681/*
682 * Flags for inp_vflags -- historically version flags only
683 */
684#define	INP_IPV4	0x1
685#define	INP_IPV6	0x2
686#define	INP_IPV6PROTO	0x4		/* opened under IPv6 protocol */
687
688/*
689 * Flags for inp_flags.
690 */
691#define	INP_RECVOPTS		0x00000001 /* receive incoming IP options */
692#define	INP_RECVRETOPTS		0x00000002 /* receive IP options for reply */
693#define	INP_RECVDSTADDR		0x00000004 /* receive IP dst address */
694#define	INP_HDRINCL		0x00000008 /* user supplies entire IP header */
695#define	INP_HIGHPORT		0x00000010 /* user wants "high" port binding */
696#define	INP_LOWPORT		0x00000020 /* user wants "low" port binding */
697#define	INP_ANONPORT		0x00000040 /* port chosen for user */
698#define	INP_RECVIF		0x00000080 /* receive incoming interface */
699#define	INP_MTUDISC		0x00000100 /* user can do MTU discovery */
700				   	   /* 0x000200 unused: was INP_FAITH */
701#define	INP_RECVTTL		0x00000400 /* receive incoming IP TTL */
702#define	INP_DONTFRAG		0x00000800 /* don't fragment packet */
703#define	INP_BINDANY		0x00001000 /* allow bind to any address */
704#define	INP_INHASHLIST		0x00002000 /* in_pcbinshash() has been called */
705#define	INP_RECVTOS		0x00004000 /* receive incoming IP TOS */
706#define	IN6P_IPV6_V6ONLY	0x00008000 /* restrict AF_INET6 socket for v6 */
707#define	IN6P_PKTINFO		0x00010000 /* receive IP6 dst and I/F */
708#define	IN6P_HOPLIMIT		0x00020000 /* receive hoplimit */
709#define	IN6P_HOPOPTS		0x00040000 /* receive hop-by-hop options */
710#define	IN6P_DSTOPTS		0x00080000 /* receive dst options after rthdr */
711#define	IN6P_RTHDR		0x00100000 /* receive routing header */
712#define	IN6P_RTHDRDSTOPTS	0x00200000 /* receive dstoptions before rthdr */
713#define	IN6P_TCLASS		0x00400000 /* receive traffic class value */
714#define	IN6P_AUTOFLOWLABEL	0x00800000 /* attach flowlabel automatically */
715#define	INP_TIMEWAIT		0x01000000 /* in TIMEWAIT, ppcb is tcptw */
716#define	INP_ONESBCAST		0x02000000 /* send all-ones broadcast */
717#define	INP_DROPPED		0x04000000 /* protocol drop flag */
718#define	INP_SOCKREF		0x08000000 /* strong socket reference */
719#define	INP_RESERVED_0          0x10000000 /* reserved field */
720#define	INP_RESERVED_1          0x20000000 /* reserved field */
721#define	IN6P_RFC2292		0x40000000 /* used RFC2292 API on the socket */
722#define	IN6P_MTU		0x80000000 /* receive path MTU */
723
724#define	INP_CONTROLOPTS		(INP_RECVOPTS|INP_RECVRETOPTS|INP_RECVDSTADDR|\
725				 INP_RECVIF|INP_RECVTTL|INP_RECVTOS|\
726				 IN6P_PKTINFO|IN6P_HOPLIMIT|IN6P_HOPOPTS|\
727				 IN6P_DSTOPTS|IN6P_RTHDR|IN6P_RTHDRDSTOPTS|\
728				 IN6P_TCLASS|IN6P_AUTOFLOWLABEL|IN6P_RFC2292|\
729				 IN6P_MTU)
730
731/*
732 * Flags for inp_flags2.
733 */
734#define	INP_2UNUSED1		0x00000001
735#define	INP_2UNUSED2		0x00000002
736#define	INP_PCBGROUPWILD	0x00000004 /* in pcbgroup wildcard list */
737#define	INP_REUSEPORT		0x00000008 /* SO_REUSEPORT option is set */
738#define	INP_FREED		0x00000010 /* inp itself is not valid */
739#define	INP_REUSEADDR		0x00000020 /* SO_REUSEADDR option is set */
740#define	INP_BINDMULTI		0x00000040 /* IP_BINDMULTI option is set */
741#define	INP_RSS_BUCKET_SET	0x00000080 /* IP_RSS_LISTEN_BUCKET is set */
742#define	INP_RECVFLOWID		0x00000100 /* populate recv datagram with flow info */
743#define	INP_RECVRSSBUCKETID	0x00000200 /* populate recv datagram with bucket id */
744#define	INP_RATE_LIMIT_CHANGED	0x00000400 /* rate limit needs attention */
745#define	INP_ORIGDSTADDR		0x00000800 /* receive IP dst address/port */
746#define INP_CANNOT_DO_ECN	0x00001000 /* The stack does not do ECN */
747#define	INP_REUSEPORT_LB	0x00002000 /* SO_REUSEPORT_LB option is set */
748#define INP_SUPPORTS_MBUFQ	0x00004000 /* Supports the mbuf queue method of LRO */
749#define INP_MBUF_QUEUE_READY	0x00008000 /* The transport is pacing, inputs can be queued */
750#define INP_DONT_SACK_QUEUE	0x00010000 /* If a sack arrives do not wake me */
751/*
752 * Flags passed to in_pcblookup*() functions.
753 */
754#define	INPLOOKUP_WILDCARD	0x00000001	/* Allow wildcard sockets. */
755#define	INPLOOKUP_RLOCKPCB	0x00000002	/* Return inpcb read-locked. */
756#define	INPLOOKUP_WLOCKPCB	0x00000004	/* Return inpcb write-locked. */
757
758#define	INPLOOKUP_MASK	(INPLOOKUP_WILDCARD | INPLOOKUP_RLOCKPCB | \
759			    INPLOOKUP_WLOCKPCB)
760
761#define	sotoinpcb(so)	((struct inpcb *)(so)->so_pcb)
762
763#define	INP_SOCKAF(so) so->so_proto->pr_domain->dom_family
764
765#define	INP_CHECK_SOCKAF(so, af)	(INP_SOCKAF(so) == af)
766
767/*
768 * Constants for pcbinfo.ipi_hashfields.
769 */
770#define	IPI_HASHFIELDS_NONE	0
771#define	IPI_HASHFIELDS_2TUPLE	1
772#define	IPI_HASHFIELDS_4TUPLE	2
773
774#ifdef _KERNEL
775VNET_DECLARE(int, ipport_reservedhigh);
776VNET_DECLARE(int, ipport_reservedlow);
777VNET_DECLARE(int, ipport_lowfirstauto);
778VNET_DECLARE(int, ipport_lowlastauto);
779VNET_DECLARE(int, ipport_firstauto);
780VNET_DECLARE(int, ipport_lastauto);
781VNET_DECLARE(int, ipport_hifirstauto);
782VNET_DECLARE(int, ipport_hilastauto);
783VNET_DECLARE(int, ipport_randomized);
784VNET_DECLARE(int, ipport_randomcps);
785VNET_DECLARE(int, ipport_randomtime);
786VNET_DECLARE(int, ipport_stoprandom);
787VNET_DECLARE(int, ipport_tcpallocs);
788
789#define	V_ipport_reservedhigh	VNET(ipport_reservedhigh)
790#define	V_ipport_reservedlow	VNET(ipport_reservedlow)
791#define	V_ipport_lowfirstauto	VNET(ipport_lowfirstauto)
792#define	V_ipport_lowlastauto	VNET(ipport_lowlastauto)
793#define	V_ipport_firstauto	VNET(ipport_firstauto)
794#define	V_ipport_lastauto	VNET(ipport_lastauto)
795#define	V_ipport_hifirstauto	VNET(ipport_hifirstauto)
796#define	V_ipport_hilastauto	VNET(ipport_hilastauto)
797#define	V_ipport_randomized	VNET(ipport_randomized)
798#define	V_ipport_randomcps	VNET(ipport_randomcps)
799#define	V_ipport_randomtime	VNET(ipport_randomtime)
800#define	V_ipport_stoprandom	VNET(ipport_stoprandom)
801#define	V_ipport_tcpallocs	VNET(ipport_tcpallocs)
802
803void	in_pcbinfo_destroy(struct inpcbinfo *);
804void	in_pcbinfo_init(struct inpcbinfo *, const char *, struct inpcbhead *,
805	    int, int, char *, uma_init, u_int);
806
807int	in_pcbbind_check_bindmulti(const struct inpcb *ni,
808	    const struct inpcb *oi);
809
810struct inpcbgroup *
811	in_pcbgroup_byhash(struct inpcbinfo *, u_int, uint32_t);
812struct inpcbgroup *
813	in_pcbgroup_byinpcb(struct inpcb *);
814struct inpcbgroup *
815	in_pcbgroup_bytuple(struct inpcbinfo *, struct in_addr, u_short,
816	    struct in_addr, u_short);
817void	in_pcbgroup_destroy(struct inpcbinfo *);
818int	in_pcbgroup_enabled(struct inpcbinfo *);
819void	in_pcbgroup_init(struct inpcbinfo *, u_int, int);
820void	in_pcbgroup_remove(struct inpcb *);
821void	in_pcbgroup_update(struct inpcb *);
822void	in_pcbgroup_update_mbuf(struct inpcb *, struct mbuf *);
823
824void	in_pcbpurgeif0(struct inpcbinfo *, struct ifnet *);
825int	in_pcballoc(struct socket *, struct inpcbinfo *);
826int	in_pcbbind(struct inpcb *, struct sockaddr *, struct ucred *);
827int	in_pcb_lport_dest(struct inpcb *inp, struct sockaddr *lsa,
828	    u_short *lportp, struct sockaddr *fsa, u_short fport,
829	    struct ucred *cred, int lookupflags);
830int	in_pcb_lport(struct inpcb *, struct in_addr *, u_short *,
831	    struct ucred *, int);
832int	in_pcbbind_setup(struct inpcb *, struct sockaddr *, in_addr_t *,
833	    u_short *, struct ucred *);
834int	in_pcbconnect(struct inpcb *, struct sockaddr *, struct ucred *);
835int	in_pcbconnect_mbuf(struct inpcb *, struct sockaddr *, struct ucred *,
836	    struct mbuf *, bool);
837int	in_pcbconnect_setup(struct inpcb *, struct sockaddr *, in_addr_t *,
838	    u_short *, in_addr_t *, u_short *, struct inpcb **,
839	    struct ucred *);
840void	in_pcbdetach(struct inpcb *);
841void	in_pcbdisconnect(struct inpcb *);
842void	in_pcbdrop(struct inpcb *);
843void	in_pcbfree(struct inpcb *);
844int	in_pcbinshash(struct inpcb *);
845int	in_pcbinshash_mbuf(struct inpcb *, struct mbuf *);
846int	in_pcbladdr(struct inpcb *, struct in_addr *, struct in_addr *,
847	    struct ucred *);
848struct inpcb *
849	in_pcblookup_local(struct inpcbinfo *,
850	    struct in_addr, u_short, int, struct ucred *);
851struct inpcb *
852	in_pcblookup(struct inpcbinfo *, struct in_addr, u_int,
853	    struct in_addr, u_int, int, struct ifnet *);
854struct inpcb *
855	in_pcblookup_mbuf(struct inpcbinfo *, struct in_addr, u_int,
856	    struct in_addr, u_int, int, struct ifnet *, struct mbuf *);
857void	in_pcbnotifyall(struct inpcbinfo *pcbinfo, struct in_addr,
858	    int, struct inpcb *(*)(struct inpcb *, int));
859void	in_pcbref(struct inpcb *);
860void	in_pcbrehash(struct inpcb *);
861void	in_pcbrehash_mbuf(struct inpcb *, struct mbuf *);
862int	in_pcbrele(struct inpcb *);
863int	in_pcbrele_rlocked(struct inpcb *);
864int	in_pcbrele_wlocked(struct inpcb *);
865void	in_pcblist_rele_rlocked(epoch_context_t ctx);
866void	in_losing(struct inpcb *);
867void	in_pcbsetsolabel(struct socket *so);
868int	in_getpeeraddr(struct socket *so, struct sockaddr **nam);
869int	in_getsockaddr(struct socket *so, struct sockaddr **nam);
870struct sockaddr *
871	in_sockaddr(in_port_t port, struct in_addr *addr);
872void	in_pcbsosetlabel(struct socket *so);
873#ifdef RATELIMIT
874int
875in_pcboutput_txrtlmt_locked(struct inpcb *, struct ifnet *,
876	    struct mbuf *, uint32_t);
877int	in_pcbattach_txrtlmt(struct inpcb *, struct ifnet *, uint32_t, uint32_t,
878	    uint32_t, struct m_snd_tag **);
879void	in_pcbdetach_txrtlmt(struct inpcb *);
880void    in_pcbdetach_tag(struct ifnet *ifp, struct m_snd_tag *mst);
881int	in_pcbmodify_txrtlmt(struct inpcb *, uint32_t);
882int	in_pcbquery_txrtlmt(struct inpcb *, uint32_t *);
883int	in_pcbquery_txrlevel(struct inpcb *, uint32_t *);
884void	in_pcboutput_txrtlmt(struct inpcb *, struct ifnet *, struct mbuf *);
885void	in_pcboutput_eagain(struct inpcb *);
886#endif
887#endif /* _KERNEL */
888
889#endif /* !_NETINET_IN_PCB_H_ */
890