1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Codel/FQ_Codel and PIE/FQ-PIE Code:
5 * Copyright (C) 2016 Centre for Advanced Internet Architectures,
6 *  Swinburne University of Technology, Melbourne, Australia.
7 * Portions of this code were made possible in part by a gift from
8 *  The Comcast Innovation Fund.
9 * Implemented by Rasool Al-Saadi <ralsaadi@swin.edu.au>
10 *
11 * Copyright (c) 1998-2002,2010 Luigi Rizzo, Universita` di Pisa
12 * Portions Copyright (c) 2000 Akamba Corp.
13 * All rights reserved
14 *
15 * Redistribution and use in source and binary forms, with or without
16 * modification, are permitted provided that the following conditions
17 * are met:
18 * 1. Redistributions of source code must retain the above copyright
19 *    notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 *    notice, this list of conditions and the following disclaimer in the
22 *    documentation and/or other materials provided with the distribution.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34 * SUCH DAMAGE.
35 */
36
37#include <sys/cdefs.h>
38__FBSDID("$FreeBSD$");
39
40/*
41 * Configuration and internal object management for dummynet.
42 */
43
44#include "opt_inet6.h"
45
46#include <sys/param.h>
47#include <sys/systm.h>
48#include <sys/malloc.h>
49#include <sys/mbuf.h>
50#include <sys/kernel.h>
51#include <sys/lock.h>
52#include <sys/module.h>
53#include <sys/mutex.h>
54#include <sys/priv.h>
55#include <sys/proc.h>
56#include <sys/rwlock.h>
57#include <sys/socket.h>
58#include <sys/socketvar.h>
59#include <sys/time.h>
60#include <sys/taskqueue.h>
61#include <net/if.h>	/* IFNAMSIZ, struct ifaddr, ifq head, lock.h mutex.h */
62#include <netinet/in.h>
63#include <netinet/ip_var.h>	/* ip_output(), IP_FORWARDING */
64#include <netinet/ip_fw.h>
65#include <netinet/ip_dummynet.h>
66
67#include <netpfil/ipfw/ip_fw_private.h>
68#include <netpfil/ipfw/dn_heap.h>
69#include <netpfil/ipfw/ip_dn_private.h>
70#ifdef NEW_AQM
71#include <netpfil/ipfw/dn_aqm.h>
72#endif
73#include <netpfil/ipfw/dn_sched.h>
74
75/* which objects to copy */
76#define DN_C_LINK 	0x01
77#define DN_C_SCH	0x02
78#define DN_C_FLOW	0x04
79#define DN_C_FS		0x08
80#define DN_C_QUEUE	0x10
81
82/* we use this argument in case of a schk_new */
83struct schk_new_arg {
84	struct dn_alg *fp;
85	struct dn_sch *sch;
86};
87
88/*---- callout hooks. ----*/
89static struct callout dn_timeout;
90static int dn_gone;
91static struct task	dn_task;
92static struct taskqueue	*dn_tq = NULL;
93
94static void
95dummynet(void *arg)
96{
97
98	(void)arg;	/* UNUSED */
99	taskqueue_enqueue(dn_tq, &dn_task);
100}
101
102void
103dn_reschedule(void)
104{
105
106	if (dn_gone != 0)
107		return;
108	callout_reset_sbt(&dn_timeout, tick_sbt, 0, dummynet, NULL,
109	    C_HARDCLOCK | C_DIRECT_EXEC);
110}
111/*----- end of callout hooks -----*/
112
113#ifdef NEW_AQM
114/* Return AQM descriptor for given type or name. */
115static struct dn_aqm *
116find_aqm_type(int type, char *name)
117{
118	struct dn_aqm *d;
119
120	SLIST_FOREACH(d, &dn_cfg.aqmlist, next) {
121		if (d->type == type || (name && !strcasecmp(d->name, name)))
122			return d;
123	}
124	return NULL; /* not found */
125}
126#endif
127
128/* Return a scheduler descriptor given the type or name. */
129static struct dn_alg *
130find_sched_type(int type, char *name)
131{
132	struct dn_alg *d;
133
134	SLIST_FOREACH(d, &dn_cfg.schedlist, next) {
135		if (d->type == type || (name && !strcasecmp(d->name, name)))
136			return d;
137	}
138	return NULL; /* not found */
139}
140
141int
142ipdn_bound_var(int *v, int dflt, int lo, int hi, const char *msg)
143{
144	int oldv = *v;
145	const char *op = NULL;
146	if (dflt < lo)
147		dflt = lo;
148	if (dflt > hi)
149		dflt = hi;
150	if (oldv < lo) {
151		*v = dflt;
152		op = "Bump";
153	} else if (oldv > hi) {
154		*v = hi;
155		op = "Clamp";
156	} else
157		return *v;
158	if (op && msg)
159		printf("%s %s to %d (was %d)\n", op, msg, *v, oldv);
160	return *v;
161}
162
163/*---- flow_id mask, hash and compare functions ---*/
164/*
165 * The flow_id includes the 5-tuple, the queue/pipe number
166 * which we store in the extra area in host order,
167 * and for ipv6 also the flow_id6.
168 * XXX see if we want the tos byte (can store in 'flags')
169 */
170static struct ipfw_flow_id *
171flow_id_mask(struct ipfw_flow_id *mask, struct ipfw_flow_id *id)
172{
173	int is_v6 = IS_IP6_FLOW_ID(id);
174
175	id->dst_port &= mask->dst_port;
176	id->src_port &= mask->src_port;
177	id->proto &= mask->proto;
178	id->extra &= mask->extra;
179	if (is_v6) {
180		APPLY_MASK(&id->dst_ip6, &mask->dst_ip6);
181		APPLY_MASK(&id->src_ip6, &mask->src_ip6);
182		id->flow_id6 &= mask->flow_id6;
183	} else {
184		id->dst_ip &= mask->dst_ip;
185		id->src_ip &= mask->src_ip;
186	}
187	return id;
188}
189
190/* computes an OR of two masks, result in dst and also returned */
191static struct ipfw_flow_id *
192flow_id_or(struct ipfw_flow_id *src, struct ipfw_flow_id *dst)
193{
194	int is_v6 = IS_IP6_FLOW_ID(dst);
195
196	dst->dst_port |= src->dst_port;
197	dst->src_port |= src->src_port;
198	dst->proto |= src->proto;
199	dst->extra |= src->extra;
200	if (is_v6) {
201#define OR_MASK(_d, _s)                          \
202    (_d)->__u6_addr.__u6_addr32[0] |= (_s)->__u6_addr.__u6_addr32[0]; \
203    (_d)->__u6_addr.__u6_addr32[1] |= (_s)->__u6_addr.__u6_addr32[1]; \
204    (_d)->__u6_addr.__u6_addr32[2] |= (_s)->__u6_addr.__u6_addr32[2]; \
205    (_d)->__u6_addr.__u6_addr32[3] |= (_s)->__u6_addr.__u6_addr32[3];
206		OR_MASK(&dst->dst_ip6, &src->dst_ip6);
207		OR_MASK(&dst->src_ip6, &src->src_ip6);
208#undef OR_MASK
209		dst->flow_id6 |= src->flow_id6;
210	} else {
211		dst->dst_ip |= src->dst_ip;
212		dst->src_ip |= src->src_ip;
213	}
214	return dst;
215}
216
217static int
218nonzero_mask(struct ipfw_flow_id *m)
219{
220	if (m->dst_port || m->src_port || m->proto || m->extra)
221		return 1;
222	if (IS_IP6_FLOW_ID(m)) {
223		return
224			m->dst_ip6.__u6_addr.__u6_addr32[0] ||
225			m->dst_ip6.__u6_addr.__u6_addr32[1] ||
226			m->dst_ip6.__u6_addr.__u6_addr32[2] ||
227			m->dst_ip6.__u6_addr.__u6_addr32[3] ||
228			m->src_ip6.__u6_addr.__u6_addr32[0] ||
229			m->src_ip6.__u6_addr.__u6_addr32[1] ||
230			m->src_ip6.__u6_addr.__u6_addr32[2] ||
231			m->src_ip6.__u6_addr.__u6_addr32[3] ||
232			m->flow_id6;
233	} else {
234		return m->dst_ip || m->src_ip;
235	}
236}
237
238/* XXX we may want a better hash function */
239static uint32_t
240flow_id_hash(struct ipfw_flow_id *id)
241{
242    uint32_t i;
243
244    if (IS_IP6_FLOW_ID(id)) {
245	uint32_t *d = (uint32_t *)&id->dst_ip6;
246	uint32_t *s = (uint32_t *)&id->src_ip6;
247        i = (d[0]      ) ^ (d[1])       ^
248            (d[2]      ) ^ (d[3])       ^
249            (d[0] >> 15) ^ (d[1] >> 15) ^
250            (d[2] >> 15) ^ (d[3] >> 15) ^
251            (s[0] <<  1) ^ (s[1] <<  1) ^
252            (s[2] <<  1) ^ (s[3] <<  1) ^
253            (s[0] << 16) ^ (s[1] << 16) ^
254            (s[2] << 16) ^ (s[3] << 16) ^
255            (id->dst_port << 1) ^ (id->src_port) ^
256	    (id->extra) ^
257            (id->proto ) ^ (id->flow_id6);
258    } else {
259        i = (id->dst_ip)        ^ (id->dst_ip >> 15) ^
260            (id->src_ip << 1)   ^ (id->src_ip >> 16) ^
261	    (id->extra) ^
262            (id->dst_port << 1) ^ (id->src_port)     ^ (id->proto);
263    }
264    return i;
265}
266
267/* Like bcmp, returns 0 if ids match, 1 otherwise. */
268static int
269flow_id_cmp(struct ipfw_flow_id *id1, struct ipfw_flow_id *id2)
270{
271	int is_v6 = IS_IP6_FLOW_ID(id1);
272
273	if (!is_v6) {
274	    if (IS_IP6_FLOW_ID(id2))
275		return 1; /* different address families */
276
277	    return (id1->dst_ip == id2->dst_ip &&
278		    id1->src_ip == id2->src_ip &&
279		    id1->dst_port == id2->dst_port &&
280		    id1->src_port == id2->src_port &&
281		    id1->proto == id2->proto &&
282		    id1->extra == id2->extra) ? 0 : 1;
283	}
284	/* the ipv6 case */
285	return (
286	    !bcmp(&id1->dst_ip6,&id2->dst_ip6, sizeof(id1->dst_ip6)) &&
287	    !bcmp(&id1->src_ip6,&id2->src_ip6, sizeof(id1->src_ip6)) &&
288	    id1->dst_port == id2->dst_port &&
289	    id1->src_port == id2->src_port &&
290	    id1->proto == id2->proto &&
291	    id1->extra == id2->extra &&
292	    id1->flow_id6 == id2->flow_id6) ? 0 : 1;
293}
294/*--------- end of flow-id mask, hash and compare ---------*/
295
296/*--- support functions for the qht hashtable ----
297 * Entries are hashed by flow-id
298 */
299static uint32_t
300q_hash(uintptr_t key, int flags, void *arg)
301{
302	/* compute the hash slot from the flow id */
303	struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ?
304		&((struct dn_queue *)key)->ni.fid :
305		(struct ipfw_flow_id *)key;
306
307	return flow_id_hash(id);
308}
309
310static int
311q_match(void *obj, uintptr_t key, int flags, void *arg)
312{
313	struct dn_queue *o = (struct dn_queue *)obj;
314	struct ipfw_flow_id *id2;
315
316	if (flags & DNHT_KEY_IS_OBJ) {
317		/* compare pointers */
318		id2 = &((struct dn_queue *)key)->ni.fid;
319	} else {
320		id2 = (struct ipfw_flow_id *)key;
321	}
322	return (0 == flow_id_cmp(&o->ni.fid,  id2));
323}
324
325/*
326 * create a new queue instance for the given 'key'.
327 */
328static void *
329q_new(uintptr_t key, int flags, void *arg)
330{
331	struct dn_queue *q, *template = arg;
332	struct dn_fsk *fs = template->fs;
333	int size = sizeof(*q) + fs->sched->fp->q_datalen;
334
335	q = malloc(size, M_DUMMYNET, M_NOWAIT | M_ZERO);
336	if (q == NULL) {
337		D("no memory for new queue");
338		return NULL;
339	}
340
341	set_oid(&q->ni.oid, DN_QUEUE, size);
342	if (fs->fs.flags & DN_QHT_HASH)
343		q->ni.fid = *(struct ipfw_flow_id *)key;
344	q->fs = fs;
345	q->_si = template->_si;
346	q->_si->q_count++;
347
348	if (fs->sched->fp->new_queue)
349		fs->sched->fp->new_queue(q);
350
351#ifdef NEW_AQM
352	/* call AQM init function after creating a queue*/
353	if (fs->aqmfp && fs->aqmfp->init)
354		if(fs->aqmfp->init(q))
355			D("unable to init AQM for fs %d", fs->fs.fs_nr);
356#endif
357	dn_cfg.queue_count++;
358
359	return q;
360}
361
362/*
363 * Notify schedulers that a queue is going away.
364 * If (flags & DN_DESTROY), also free the packets.
365 * The version for callbacks is called q_delete_cb().
366 */
367static void
368dn_delete_queue(struct dn_queue *q, int flags)
369{
370	struct dn_fsk *fs = q->fs;
371
372#ifdef NEW_AQM
373	/* clean up AQM status for queue 'q'
374	 * cleanup here is called just with MULTIQUEUE
375	 */
376	if (fs && fs->aqmfp && fs->aqmfp->cleanup)
377		fs->aqmfp->cleanup(q);
378#endif
379	// D("fs %p si %p\n", fs, q->_si);
380	/* notify the parent scheduler that the queue is going away */
381	if (fs && fs->sched->fp->free_queue)
382		fs->sched->fp->free_queue(q);
383	q->_si->q_count--;
384	q->_si = NULL;
385	if (flags & DN_DESTROY) {
386		if (q->mq.head)
387			dn_free_pkts(q->mq.head);
388		bzero(q, sizeof(*q));	// safety
389		free(q, M_DUMMYNET);
390		dn_cfg.queue_count--;
391	}
392}
393
394static int
395q_delete_cb(void *q, void *arg)
396{
397	int flags = (int)(uintptr_t)arg;
398	dn_delete_queue(q, flags);
399	return (flags & DN_DESTROY) ? DNHT_SCAN_DEL : 0;
400}
401
402/*
403 * calls dn_delete_queue/q_delete_cb on all queues,
404 * which notifies the parent scheduler and possibly drains packets.
405 * flags & DN_DESTROY: drains queues and destroy qht;
406 */
407static void
408qht_delete(struct dn_fsk *fs, int flags)
409{
410	ND("fs %d start flags %d qht %p",
411		fs->fs.fs_nr, flags, fs->qht);
412	if (!fs->qht)
413		return;
414	if (fs->fs.flags & DN_QHT_HASH) {
415		dn_ht_scan(fs->qht, q_delete_cb, (void *)(uintptr_t)flags);
416		if (flags & DN_DESTROY) {
417			dn_ht_free(fs->qht, 0);
418			fs->qht = NULL;
419		}
420	} else {
421		dn_delete_queue((struct dn_queue *)(fs->qht), flags);
422		if (flags & DN_DESTROY)
423			fs->qht = NULL;
424	}
425}
426
427/*
428 * Find and possibly create the queue for a MULTIQUEUE scheduler.
429 * We never call it for !MULTIQUEUE (the queue is in the sch_inst).
430 */
431struct dn_queue *
432ipdn_q_find(struct dn_fsk *fs, struct dn_sch_inst *si,
433	struct ipfw_flow_id *id)
434{
435	struct dn_queue template;
436
437	template._si = si;
438	template.fs = fs;
439
440	if (fs->fs.flags & DN_QHT_HASH) {
441		struct ipfw_flow_id masked_id;
442		if (fs->qht == NULL) {
443			fs->qht = dn_ht_init(NULL, fs->fs.buckets,
444				offsetof(struct dn_queue, q_next),
445				q_hash, q_match, q_new);
446			if (fs->qht == NULL)
447				return NULL;
448		}
449		masked_id = *id;
450		flow_id_mask(&fs->fsk_mask, &masked_id);
451		return dn_ht_find(fs->qht, (uintptr_t)&masked_id,
452			DNHT_INSERT, &template);
453	} else {
454		if (fs->qht == NULL)
455			fs->qht = q_new(0, 0, &template);
456		return (struct dn_queue *)fs->qht;
457	}
458}
459/*--- end of queue hash table ---*/
460
461/*--- support functions for the sch_inst hashtable ----
462 *
463 * These are hashed by flow-id
464 */
465static uint32_t
466si_hash(uintptr_t key, int flags, void *arg)
467{
468	/* compute the hash slot from the flow id */
469	struct ipfw_flow_id *id = (flags & DNHT_KEY_IS_OBJ) ?
470		&((struct dn_sch_inst *)key)->ni.fid :
471		(struct ipfw_flow_id *)key;
472
473	return flow_id_hash(id);
474}
475
476static int
477si_match(void *obj, uintptr_t key, int flags, void *arg)
478{
479	struct dn_sch_inst *o = obj;
480	struct ipfw_flow_id *id2;
481
482	id2 = (flags & DNHT_KEY_IS_OBJ) ?
483		&((struct dn_sch_inst *)key)->ni.fid :
484		(struct ipfw_flow_id *)key;
485	return flow_id_cmp(&o->ni.fid,  id2) == 0;
486}
487
488/*
489 * create a new instance for the given 'key'
490 * Allocate memory for instance, delay line and scheduler private data.
491 */
492static void *
493si_new(uintptr_t key, int flags, void *arg)
494{
495	struct dn_schk *s = arg;
496	struct dn_sch_inst *si;
497	int l = sizeof(*si) + s->fp->si_datalen;
498
499	si = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO);
500	if (si == NULL)
501		goto error;
502
503	/* Set length only for the part passed up to userland. */
504	set_oid(&si->ni.oid, DN_SCH_I, sizeof(struct dn_flow));
505	set_oid(&(si->dline.oid), DN_DELAY_LINE,
506		sizeof(struct delay_line));
507	/* mark si and dline as outside the event queue */
508	si->ni.oid.id = si->dline.oid.id = -1;
509
510	si->sched = s;
511	si->dline.si = si;
512
513	if (s->fp->new_sched && s->fp->new_sched(si)) {
514		D("new_sched error");
515		goto error;
516	}
517	if (s->sch.flags & DN_HAVE_MASK)
518		si->ni.fid = *(struct ipfw_flow_id *)key;
519
520#ifdef NEW_AQM
521	/* init AQM status for !DN_MULTIQUEUE sched*/
522	if (!(s->fp->flags & DN_MULTIQUEUE))
523		if (s->fs->aqmfp && s->fs->aqmfp->init)
524			if(s->fs->aqmfp->init((struct dn_queue *)(si + 1))) {
525				D("unable to init AQM for fs %d", s->fs->fs.fs_nr);
526				goto error;
527			}
528#endif
529
530	dn_cfg.si_count++;
531	return si;
532
533error:
534	if (si) {
535		bzero(si, sizeof(*si)); // safety
536		free(si, M_DUMMYNET);
537	}
538        return NULL;
539}
540
541/*
542 * Callback from siht to delete all scheduler instances. Remove
543 * si and delay line from the system heap, destroy all queues.
544 * We assume that all flowset have been notified and do not
545 * point to us anymore.
546 */
547static int
548si_destroy(void *_si, void *arg)
549{
550	struct dn_sch_inst *si = _si;
551	struct dn_schk *s = si->sched;
552	struct delay_line *dl = &si->dline;
553
554	if (dl->oid.subtype) /* remove delay line from event heap */
555		heap_extract(&dn_cfg.evheap, dl);
556	dn_free_pkts(dl->mq.head);	/* drain delay line */
557	if (si->kflags & DN_ACTIVE) /* remove si from event heap */
558		heap_extract(&dn_cfg.evheap, si);
559
560#ifdef NEW_AQM
561	/* clean up AQM status for !DN_MULTIQUEUE sched
562	 * Note that all queues belong to fs were cleaned up in fsk_detach.
563	 * When drain_scheduler is called s->fs and q->fs are pointing
564	 * to a correct fs, so we can use fs in this case.
565	 */
566	if (!(s->fp->flags & DN_MULTIQUEUE)) {
567		struct dn_queue *q = (struct dn_queue *)(si + 1);
568		if (q->aqm_status && q->fs->aqmfp)
569			if (q->fs->aqmfp->cleanup)
570				q->fs->aqmfp->cleanup(q);
571	}
572#endif
573	if (s->fp->free_sched)
574		s->fp->free_sched(si);
575	bzero(si, sizeof(*si));	/* safety */
576	free(si, M_DUMMYNET);
577	dn_cfg.si_count--;
578	return DNHT_SCAN_DEL;
579}
580
581/*
582 * Find the scheduler instance for this packet. If we need to apply
583 * a mask, do on a local copy of the flow_id to preserve the original.
584 * Assume siht is always initialized if we have a mask.
585 */
586struct dn_sch_inst *
587ipdn_si_find(struct dn_schk *s, struct ipfw_flow_id *id)
588{
589
590	if (s->sch.flags & DN_HAVE_MASK) {
591		struct ipfw_flow_id id_t = *id;
592		flow_id_mask(&s->sch.sched_mask, &id_t);
593		return dn_ht_find(s->siht, (uintptr_t)&id_t,
594			DNHT_INSERT, s);
595	}
596	if (!s->siht)
597		s->siht = si_new(0, 0, s);
598	return (struct dn_sch_inst *)s->siht;
599}
600
601/* callback to flush credit for the scheduler instance */
602static int
603si_reset_credit(void *_si, void *arg)
604{
605	struct dn_sch_inst *si = _si;
606	struct dn_link *p = &si->sched->link;
607
608	si->credit = p->burst + (dn_cfg.io_fast ?  p->bandwidth : 0);
609	return 0;
610}
611
612static void
613schk_reset_credit(struct dn_schk *s)
614{
615	if (s->sch.flags & DN_HAVE_MASK)
616		dn_ht_scan(s->siht, si_reset_credit, NULL);
617	else if (s->siht)
618		si_reset_credit(s->siht, NULL);
619}
620/*---- end of sch_inst hashtable ---------------------*/
621
622/*-------------------------------------------------------
623 * flowset hash (fshash) support. Entries are hashed by fs_nr.
624 * New allocations are put in the fsunlinked list, from which
625 * they are removed when they point to a specific scheduler.
626 */
627static uint32_t
628fsk_hash(uintptr_t key, int flags, void *arg)
629{
630	uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key :
631		((struct dn_fsk *)key)->fs.fs_nr;
632
633	return ( (i>>8)^(i>>4)^i );
634}
635
636static int
637fsk_match(void *obj, uintptr_t key, int flags, void *arg)
638{
639	struct dn_fsk *fs = obj;
640	int i = !(flags & DNHT_KEY_IS_OBJ) ? key :
641		((struct dn_fsk *)key)->fs.fs_nr;
642
643	return (fs->fs.fs_nr == i);
644}
645
646static void *
647fsk_new(uintptr_t key, int flags, void *arg)
648{
649	struct dn_fsk *fs;
650
651	fs = malloc(sizeof(*fs), M_DUMMYNET, M_NOWAIT | M_ZERO);
652	if (fs) {
653		set_oid(&fs->fs.oid, DN_FS, sizeof(fs->fs));
654		dn_cfg.fsk_count++;
655		fs->drain_bucket = 0;
656		SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain);
657	}
658	return fs;
659}
660
661#ifdef NEW_AQM
662/* callback function for cleaning up AQM queue status belongs to a flowset
663 * connected to scheduler instance '_si' (for !DN_MULTIQUEUE only).
664 */
665static int
666si_cleanup_q(void *_si, void *arg)
667{
668	struct dn_sch_inst *si = _si;
669
670	if (!(si->sched->fp->flags & DN_MULTIQUEUE)) {
671		if (si->sched->fs->aqmfp && si->sched->fs->aqmfp->cleanup)
672			si->sched->fs->aqmfp->cleanup((struct dn_queue *) (si+1));
673	}
674	return 0;
675}
676
677/* callback to clean up queue AQM status.*/
678static int
679q_cleanup_q(void *_q, void *arg)
680{
681	struct dn_queue *q = _q;
682	q->fs->aqmfp->cleanup(q);
683	return 0;
684}
685
686/* Clean up all AQM queues status belongs to flowset 'fs' and then
687 * deconfig AQM for flowset 'fs'
688 */
689static void
690aqm_cleanup_deconfig_fs(struct dn_fsk *fs)
691{
692	struct dn_sch_inst *si;
693
694	/* clean up AQM status for all queues for !DN_MULTIQUEUE sched*/
695	if (fs->fs.fs_nr > DN_MAX_ID) {
696		if (fs->sched && !(fs->sched->fp->flags & DN_MULTIQUEUE)) {
697			if (fs->sched->sch.flags & DN_HAVE_MASK)
698				dn_ht_scan(fs->sched->siht, si_cleanup_q, NULL);
699			else {
700					/* single si i.e. no sched mask */
701					si = (struct dn_sch_inst *) fs->sched->siht;
702					if (si && fs->aqmfp && fs->aqmfp->cleanup)
703						fs->aqmfp->cleanup((struct dn_queue *) (si+1));
704			}
705		}
706	}
707
708	/* clean up AQM status for all queues for DN_MULTIQUEUE sched*/
709	if (fs->sched && fs->sched->fp->flags & DN_MULTIQUEUE && fs->qht) {
710			if (fs->fs.flags & DN_QHT_HASH)
711				dn_ht_scan(fs->qht, q_cleanup_q, NULL);
712			else
713				fs->aqmfp->cleanup((struct dn_queue *)(fs->qht));
714	}
715
716	/* deconfig AQM */
717	if(fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig)
718		fs->aqmfp->deconfig(fs);
719}
720#endif
721
722/*
723 * detach flowset from its current scheduler. Flags as follows:
724 * DN_DETACH removes from the fsk_list
725 * DN_DESTROY deletes individual queues
726 * DN_DELETE_FS destroys the flowset (otherwise goes in unlinked).
727 */
728static void
729fsk_detach(struct dn_fsk *fs, int flags)
730{
731	if (flags & DN_DELETE_FS)
732		flags |= DN_DESTROY;
733	ND("fs %d from sched %d flags %s %s %s",
734		fs->fs.fs_nr, fs->fs.sched_nr,
735		(flags & DN_DELETE_FS) ? "DEL_FS":"",
736		(flags & DN_DESTROY) ? "DEL":"",
737		(flags & DN_DETACH) ? "DET":"");
738	if (flags & DN_DETACH) { /* detach from the list */
739		struct dn_fsk_head *h;
740		h = fs->sched ? &fs->sched->fsk_list : &dn_cfg.fsu;
741		SLIST_REMOVE(h, fs, dn_fsk, sch_chain);
742	}
743	/* Free the RED parameters, they will be recomputed on
744	 * subsequent attach if needed.
745	 */
746	if (fs->w_q_lookup)
747		free(fs->w_q_lookup, M_DUMMYNET);
748	fs->w_q_lookup = NULL;
749	qht_delete(fs, flags);
750#ifdef NEW_AQM
751	aqm_cleanup_deconfig_fs(fs);
752#endif
753
754	if (fs->sched && fs->sched->fp->free_fsk)
755		fs->sched->fp->free_fsk(fs);
756	fs->sched = NULL;
757	if (flags & DN_DELETE_FS) {
758		bzero(fs, sizeof(*fs));	/* safety */
759		free(fs, M_DUMMYNET);
760		dn_cfg.fsk_count--;
761	} else {
762		SLIST_INSERT_HEAD(&dn_cfg.fsu, fs, sch_chain);
763	}
764}
765
766/*
767 * Detach or destroy all flowsets in a list.
768 * flags specifies what to do:
769 * DN_DESTROY:	flush all queues
770 * DN_DELETE_FS:	DN_DESTROY + destroy flowset
771 *	DN_DELETE_FS implies DN_DESTROY
772 */
773static void
774fsk_detach_list(struct dn_fsk_head *h, int flags)
775{
776	struct dn_fsk *fs;
777	int n = 0; /* only for stats */
778
779	ND("head %p flags %x", h, flags);
780	while ((fs = SLIST_FIRST(h))) {
781		SLIST_REMOVE_HEAD(h, sch_chain);
782		n++;
783		fsk_detach(fs, flags);
784	}
785	ND("done %d flowsets", n);
786}
787
788/*
789 * called on 'queue X delete' -- removes the flowset from fshash,
790 * deletes all queues for the flowset, and removes the flowset.
791 */
792static int
793delete_fs(int i, int locked)
794{
795	struct dn_fsk *fs;
796	int err = 0;
797
798	if (!locked)
799		DN_BH_WLOCK();
800	fs = dn_ht_find(dn_cfg.fshash, i, DNHT_REMOVE, NULL);
801	ND("fs %d found %p", i, fs);
802	if (fs) {
803		fsk_detach(fs, DN_DETACH | DN_DELETE_FS);
804		err = 0;
805	} else
806		err = EINVAL;
807	if (!locked)
808		DN_BH_WUNLOCK();
809	return err;
810}
811
812/*----- end of flowset hashtable support -------------*/
813
814/*------------------------------------------------------------
815 * Scheduler hash. When searching by index we pass sched_nr,
816 * otherwise we pass struct dn_sch * which is the first field in
817 * struct dn_schk so we can cast between the two. We use this trick
818 * because in the create phase (but it should be fixed).
819 */
820static uint32_t
821schk_hash(uintptr_t key, int flags, void *_arg)
822{
823	uint32_t i = !(flags & DNHT_KEY_IS_OBJ) ? key :
824		((struct dn_schk *)key)->sch.sched_nr;
825	return ( (i>>8)^(i>>4)^i );
826}
827
828static int
829schk_match(void *obj, uintptr_t key, int flags, void *_arg)
830{
831	struct dn_schk *s = (struct dn_schk *)obj;
832	int i = !(flags & DNHT_KEY_IS_OBJ) ? key :
833		((struct dn_schk *)key)->sch.sched_nr;
834	return (s->sch.sched_nr == i);
835}
836
837/*
838 * Create the entry and intialize with the sched hash if needed.
839 * Leave s->fp unset so we can tell whether a dn_ht_find() returns
840 * a new object or a previously existing one.
841 */
842static void *
843schk_new(uintptr_t key, int flags, void *arg)
844{
845	struct schk_new_arg *a = arg;
846	struct dn_schk *s;
847	int l = sizeof(*s) +a->fp->schk_datalen;
848
849	s = malloc(l, M_DUMMYNET, M_NOWAIT | M_ZERO);
850	if (s == NULL)
851		return NULL;
852	set_oid(&s->link.oid, DN_LINK, sizeof(s->link));
853	s->sch = *a->sch; // copy initial values
854	s->link.link_nr = s->sch.sched_nr;
855	SLIST_INIT(&s->fsk_list);
856	/* initialize the hash table or create the single instance */
857	s->fp = a->fp;	/* si_new needs this */
858	s->drain_bucket = 0;
859	if (s->sch.flags & DN_HAVE_MASK) {
860		s->siht = dn_ht_init(NULL, s->sch.buckets,
861			offsetof(struct dn_sch_inst, si_next),
862			si_hash, si_match, si_new);
863		if (s->siht == NULL) {
864			free(s, M_DUMMYNET);
865			return NULL;
866		}
867	}
868	s->fp = NULL;	/* mark as a new scheduler */
869	dn_cfg.schk_count++;
870	return s;
871}
872
873/*
874 * Callback for sched delete. Notify all attached flowsets to
875 * detach from the scheduler, destroy the internal flowset, and
876 * all instances. The scheduler goes away too.
877 * arg is 0 (only detach flowsets and destroy instances)
878 * DN_DESTROY (detach & delete queues, delete schk)
879 * or DN_DELETE_FS (delete queues and flowsets, delete schk)
880 */
881static int
882schk_delete_cb(void *obj, void *arg)
883{
884	struct dn_schk *s = obj;
885#if 0
886	int a = (int)arg;
887	ND("sched %d arg %s%s",
888		s->sch.sched_nr,
889		a&DN_DESTROY ? "DEL ":"",
890		a&DN_DELETE_FS ? "DEL_FS":"");
891#endif
892	fsk_detach_list(&s->fsk_list, arg ? DN_DESTROY : 0);
893	/* no more flowset pointing to us now */
894	if (s->sch.flags & DN_HAVE_MASK) {
895		dn_ht_scan(s->siht, si_destroy, NULL);
896		dn_ht_free(s->siht, 0);
897	} else if (s->siht)
898		si_destroy(s->siht, NULL);
899	if (s->profile) {
900		free(s->profile, M_DUMMYNET);
901		s->profile = NULL;
902	}
903	s->siht = NULL;
904	if (s->fp->destroy)
905		s->fp->destroy(s);
906	bzero(s, sizeof(*s));	// safety
907	free(obj, M_DUMMYNET);
908	dn_cfg.schk_count--;
909	return DNHT_SCAN_DEL;
910}
911
912/*
913 * called on a 'sched X delete' command. Deletes a single scheduler.
914 * This is done by removing from the schedhash, unlinking all
915 * flowsets and deleting their traffic.
916 */
917static int
918delete_schk(int i)
919{
920	struct dn_schk *s;
921
922	s = dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL);
923	ND("%d %p", i, s);
924	if (!s)
925		return EINVAL;
926	delete_fs(i + DN_MAX_ID, 1); /* first delete internal fs */
927	/* then detach flowsets, delete traffic */
928	schk_delete_cb(s, (void*)(uintptr_t)DN_DESTROY);
929	return 0;
930}
931/*--- end of schk hashtable support ---*/
932
933static int
934copy_obj(char **start, char *end, void *_o, const char *msg, int i)
935{
936	struct dn_id o;
937	union {
938		struct dn_link l;
939		struct dn_schk s;
940	} dn;
941	int have = end - *start;
942
943	memcpy(&o, _o, sizeof(o));
944	if (have < o.len || o.len == 0 || o.type == 0) {
945		D("(WARN) type %d %s %d have %d need %d",
946		    o.type, msg, i, have, o.len);
947		return 1;
948	}
949	ND("type %d %s %d len %d", o.type, msg, i, o.len);
950	if (o.type == DN_LINK) {
951		memcpy(&dn.l, _o, sizeof(dn.l));
952		/* Adjust burst parameter for link */
953		dn.l.burst = div64(dn.l.burst, 8 * hz);
954		dn.l.delay = dn.l.delay * 1000 / hz;
955		memcpy(*start, &dn.l, sizeof(dn.l));
956	} else if (o.type == DN_SCH) {
957		/* Set dn.s.sch.oid.id to the number of instances */
958		memcpy(&dn.s, _o, sizeof(dn.s));
959		dn.s.sch.oid.id = (dn.s.sch.flags & DN_HAVE_MASK) ?
960		    dn_ht_entries(dn.s.siht) : (dn.s.siht ? 1 : 0);
961		memcpy(*start, &dn.s, sizeof(dn.s));
962	} else
963		memcpy(*start, _o, o.len);
964	*start += o.len;
965	return 0;
966}
967
968/* Specific function to copy a queue.
969 * Copies only the user-visible part of a queue (which is in
970 * a struct dn_flow), and sets len accordingly.
971 */
972static int
973copy_obj_q(char **start, char *end, void *_o, const char *msg, int i)
974{
975	struct dn_id *o = _o;
976	int have = end - *start;
977	int len = sizeof(struct dn_flow); /* see above comment */
978
979	if (have < len || o->len == 0 || o->type != DN_QUEUE) {
980		D("ERROR type %d %s %d have %d need %d",
981			o->type, msg, i, have, len);
982		return 1;
983	}
984	ND("type %d %s %d len %d", o->type, msg, i, len);
985	memcpy(*start, _o, len);
986	((struct dn_id*)(*start))->len = len;
987	*start += len;
988	return 0;
989}
990
991static int
992copy_q_cb(void *obj, void *arg)
993{
994	struct dn_queue *q = obj;
995	struct copy_args *a = arg;
996	struct dn_flow *ni = (struct dn_flow *)(*a->start);
997        if (copy_obj_q(a->start, a->end, &q->ni, "queue", -1))
998                return DNHT_SCAN_END;
999        ni->oid.type = DN_FLOW; /* override the DN_QUEUE */
1000        ni->oid.id = si_hash((uintptr_t)&ni->fid, 0, NULL);
1001        return 0;
1002}
1003
1004static int
1005copy_q(struct copy_args *a, struct dn_fsk *fs, int flags)
1006{
1007	if (!fs->qht)
1008		return 0;
1009	if (fs->fs.flags & DN_QHT_HASH)
1010		dn_ht_scan(fs->qht, copy_q_cb, a);
1011	else
1012		copy_q_cb(fs->qht, a);
1013	return 0;
1014}
1015
1016/*
1017 * This routine only copies the initial part of a profile ? XXX
1018 */
1019static int
1020copy_profile(struct copy_args *a, struct dn_profile *p)
1021{
1022	int have = a->end - *a->start;
1023	/* XXX here we check for max length */
1024	int profile_len = sizeof(struct dn_profile) -
1025		ED_MAX_SAMPLES_NO*sizeof(int);
1026
1027	if (p == NULL)
1028		return 0;
1029	if (have < profile_len) {
1030		D("error have %d need %d", have, profile_len);
1031		return 1;
1032	}
1033	memcpy(*a->start, p, profile_len);
1034	((struct dn_id *)(*a->start))->len = profile_len;
1035	*a->start += profile_len;
1036	return 0;
1037}
1038
1039static int
1040copy_flowset(struct copy_args *a, struct dn_fsk *fs, int flags)
1041{
1042	struct dn_fs *ufs = (struct dn_fs *)(*a->start);
1043	if (!fs)
1044		return 0;
1045	ND("flowset %d", fs->fs.fs_nr);
1046	if (copy_obj(a->start, a->end, &fs->fs, "flowset", fs->fs.fs_nr))
1047		return DNHT_SCAN_END;
1048	ufs->oid.id = (fs->fs.flags & DN_QHT_HASH) ?
1049		dn_ht_entries(fs->qht) : (fs->qht ? 1 : 0);
1050	if (flags) {	/* copy queues */
1051		copy_q(a, fs, 0);
1052	}
1053	return 0;
1054}
1055
1056static int
1057copy_si_cb(void *obj, void *arg)
1058{
1059	struct dn_sch_inst *si = obj;
1060	struct copy_args *a = arg;
1061	struct dn_flow *ni = (struct dn_flow *)(*a->start);
1062	if (copy_obj(a->start, a->end, &si->ni, "inst",
1063			si->sched->sch.sched_nr))
1064		return DNHT_SCAN_END;
1065	ni->oid.type = DN_FLOW; /* override the DN_SCH_I */
1066	ni->oid.id = si_hash((uintptr_t)si, DNHT_KEY_IS_OBJ, NULL);
1067	return 0;
1068}
1069
1070static int
1071copy_si(struct copy_args *a, struct dn_schk *s, int flags)
1072{
1073	if (s->sch.flags & DN_HAVE_MASK)
1074		dn_ht_scan(s->siht, copy_si_cb, a);
1075	else if (s->siht)
1076		copy_si_cb(s->siht, a);
1077	return 0;
1078}
1079
1080/*
1081 * compute a list of children of a scheduler and copy up
1082 */
1083static int
1084copy_fsk_list(struct copy_args *a, struct dn_schk *s, int flags)
1085{
1086	struct dn_fsk *fs;
1087	struct dn_id *o;
1088	uint32_t *p;
1089
1090	int n = 0, space = sizeof(*o);
1091	SLIST_FOREACH(fs, &s->fsk_list, sch_chain) {
1092		if (fs->fs.fs_nr < DN_MAX_ID)
1093			n++;
1094	}
1095	space += n * sizeof(uint32_t);
1096	DX(3, "sched %d has %d flowsets", s->sch.sched_nr, n);
1097	if (a->end - *(a->start) < space)
1098		return DNHT_SCAN_END;
1099	o = (struct dn_id *)(*(a->start));
1100	o->len = space;
1101	*a->start += o->len;
1102	o->type = DN_TEXT;
1103	p = (uint32_t *)(o+1);
1104	SLIST_FOREACH(fs, &s->fsk_list, sch_chain)
1105		if (fs->fs.fs_nr < DN_MAX_ID)
1106			*p++ = fs->fs.fs_nr;
1107	return 0;
1108}
1109
1110static int
1111copy_data_helper(void *_o, void *_arg)
1112{
1113	struct copy_args *a = _arg;
1114	uint32_t *r = a->extra->r; /* start of first range */
1115	uint32_t *lim;	/* first invalid pointer */
1116	int n;
1117
1118	lim = (uint32_t *)((char *)(a->extra) + a->extra->o.len);
1119
1120	if (a->type == DN_LINK || a->type == DN_SCH) {
1121		/* pipe|sched show, we receive a dn_schk */
1122		struct dn_schk *s = _o;
1123
1124		n = s->sch.sched_nr;
1125		if (a->type == DN_SCH && n >= DN_MAX_ID)
1126			return 0;	/* not a scheduler */
1127		if (a->type == DN_LINK && n <= DN_MAX_ID)
1128		    return 0;	/* not a pipe */
1129
1130		/* see if the object is within one of our ranges */
1131		for (;r < lim; r += 2) {
1132			if (n < r[0] || n > r[1])
1133				continue;
1134			/* Found a valid entry, copy and we are done */
1135			if (a->flags & DN_C_LINK) {
1136				if (copy_obj(a->start, a->end,
1137				    &s->link, "link", n))
1138					return DNHT_SCAN_END;
1139				if (copy_profile(a, s->profile))
1140					return DNHT_SCAN_END;
1141				if (copy_flowset(a, s->fs, 0))
1142					return DNHT_SCAN_END;
1143			}
1144			if (a->flags & DN_C_SCH) {
1145				if (copy_obj(a->start, a->end,
1146				    &s->sch, "sched", n))
1147					return DNHT_SCAN_END;
1148				/* list all attached flowsets */
1149				if (copy_fsk_list(a, s, 0))
1150					return DNHT_SCAN_END;
1151			}
1152			if (a->flags & DN_C_FLOW)
1153				copy_si(a, s, 0);
1154			break;
1155		}
1156	} else if (a->type == DN_FS) {
1157		/* queue show, skip internal flowsets */
1158		struct dn_fsk *fs = _o;
1159
1160		n = fs->fs.fs_nr;
1161		if (n >= DN_MAX_ID)
1162			return 0;
1163		/* see if the object is within one of our ranges */
1164		for (;r < lim; r += 2) {
1165			if (n < r[0] || n > r[1])
1166				continue;
1167			if (copy_flowset(a, fs, 0))
1168				return DNHT_SCAN_END;
1169			copy_q(a, fs, 0);
1170			break; /* we are done */
1171		}
1172	}
1173	return 0;
1174}
1175
1176static inline struct dn_schk *
1177locate_scheduler(int i)
1178{
1179	return dn_ht_find(dn_cfg.schedhash, i, 0, NULL);
1180}
1181
1182/*
1183 * red parameters are in fixed point arithmetic.
1184 */
1185static int
1186config_red(struct dn_fsk *fs)
1187{
1188	int64_t s, idle, weight, w0;
1189	int t, i;
1190
1191	fs->w_q = fs->fs.w_q;
1192	fs->max_p = fs->fs.max_p;
1193	ND("called");
1194	/* Doing stuff that was in userland */
1195	i = fs->sched->link.bandwidth;
1196	s = (i <= 0) ? 0 :
1197		hz * dn_cfg.red_avg_pkt_size * 8 * SCALE(1) / i;
1198
1199	idle = div64((s * 3) , fs->w_q); /* s, fs->w_q scaled; idle not scaled */
1200	fs->lookup_step = div64(idle , dn_cfg.red_lookup_depth);
1201	/* fs->lookup_step not scaled, */
1202	if (!fs->lookup_step)
1203		fs->lookup_step = 1;
1204	w0 = weight = SCALE(1) - fs->w_q; //fs->w_q scaled
1205
1206	for (t = fs->lookup_step; t > 1; --t)
1207		weight = SCALE_MUL(weight, w0);
1208	fs->lookup_weight = (int)(weight); // scaled
1209
1210	/* Now doing stuff that was in kerneland */
1211	fs->min_th = SCALE(fs->fs.min_th);
1212	fs->max_th = SCALE(fs->fs.max_th);
1213
1214	if (fs->fs.max_th == fs->fs.min_th)
1215		fs->c_1 = fs->max_p;
1216	else
1217		fs->c_1 = SCALE((int64_t)(fs->max_p)) / (fs->fs.max_th - fs->fs.min_th);
1218	fs->c_2 = SCALE_MUL(fs->c_1, SCALE(fs->fs.min_th));
1219
1220	if (fs->fs.flags & DN_IS_GENTLE_RED) {
1221		fs->c_3 = (SCALE(1) - fs->max_p) / fs->fs.max_th;
1222		fs->c_4 = SCALE(1) - 2 * fs->max_p;
1223	}
1224
1225	/* If the lookup table already exist, free and create it again. */
1226	if (fs->w_q_lookup) {
1227		free(fs->w_q_lookup, M_DUMMYNET);
1228		fs->w_q_lookup = NULL;
1229	}
1230	if (dn_cfg.red_lookup_depth == 0) {
1231		printf("\ndummynet: net.inet.ip.dummynet.red_lookup_depth"
1232		    "must be > 0\n");
1233		fs->fs.flags &= ~DN_IS_RED;
1234		fs->fs.flags &= ~DN_IS_GENTLE_RED;
1235		return (EINVAL);
1236	}
1237	fs->lookup_depth = dn_cfg.red_lookup_depth;
1238	fs->w_q_lookup = (u_int *)malloc(fs->lookup_depth * sizeof(int),
1239	    M_DUMMYNET, M_NOWAIT);
1240	if (fs->w_q_lookup == NULL) {
1241		printf("dummynet: sorry, cannot allocate red lookup table\n");
1242		fs->fs.flags &= ~DN_IS_RED;
1243		fs->fs.flags &= ~DN_IS_GENTLE_RED;
1244		return(ENOSPC);
1245	}
1246
1247	/* Fill the lookup table with (1 - w_q)^x */
1248	fs->w_q_lookup[0] = SCALE(1) - fs->w_q;
1249
1250	for (i = 1; i < fs->lookup_depth; i++)
1251		fs->w_q_lookup[i] =
1252		    SCALE_MUL(fs->w_q_lookup[i - 1], fs->lookup_weight);
1253
1254	if (dn_cfg.red_avg_pkt_size < 1)
1255		dn_cfg.red_avg_pkt_size = 512;
1256	fs->avg_pkt_size = dn_cfg.red_avg_pkt_size;
1257	if (dn_cfg.red_max_pkt_size < 1)
1258		dn_cfg.red_max_pkt_size = 1500;
1259	fs->max_pkt_size = dn_cfg.red_max_pkt_size;
1260	ND("exit");
1261	return 0;
1262}
1263
1264/* Scan all flowset attached to this scheduler and update red */
1265static void
1266update_red(struct dn_schk *s)
1267{
1268	struct dn_fsk *fs;
1269	SLIST_FOREACH(fs, &s->fsk_list, sch_chain) {
1270		if (fs && (fs->fs.flags & DN_IS_RED))
1271			config_red(fs);
1272	}
1273}
1274
1275/* attach flowset to scheduler s, possibly requeue */
1276static void
1277fsk_attach(struct dn_fsk *fs, struct dn_schk *s)
1278{
1279	ND("remove fs %d from fsunlinked, link to sched %d",
1280		fs->fs.fs_nr, s->sch.sched_nr);
1281	SLIST_REMOVE(&dn_cfg.fsu, fs, dn_fsk, sch_chain);
1282	fs->sched = s;
1283	SLIST_INSERT_HEAD(&s->fsk_list, fs, sch_chain);
1284	if (s->fp->new_fsk)
1285		s->fp->new_fsk(fs);
1286	/* XXX compute fsk_mask */
1287	fs->fsk_mask = fs->fs.flow_mask;
1288	if (fs->sched->sch.flags & DN_HAVE_MASK)
1289		flow_id_or(&fs->sched->sch.sched_mask, &fs->fsk_mask);
1290	if (fs->qht) {
1291		/*
1292		 * we must drain qht according to the old
1293		 * type, and reinsert according to the new one.
1294		 * The requeue is complex -- in general we need to
1295		 * reclassify every single packet.
1296		 * For the time being, let's hope qht is never set
1297		 * when we reach this point.
1298		 */
1299		D("XXX TODO requeue from fs %d to sch %d",
1300			fs->fs.fs_nr, s->sch.sched_nr);
1301		fs->qht = NULL;
1302	}
1303	/* set the new type for qht */
1304	if (nonzero_mask(&fs->fsk_mask))
1305		fs->fs.flags |= DN_QHT_HASH;
1306	else
1307		fs->fs.flags &= ~DN_QHT_HASH;
1308
1309	/* XXX config_red() can fail... */
1310	if (fs->fs.flags & DN_IS_RED)
1311		config_red(fs);
1312}
1313
1314/* update all flowsets which may refer to this scheduler */
1315static void
1316update_fs(struct dn_schk *s)
1317{
1318	struct dn_fsk *fs, *tmp;
1319
1320	SLIST_FOREACH_SAFE(fs, &dn_cfg.fsu, sch_chain, tmp) {
1321		if (s->sch.sched_nr != fs->fs.sched_nr) {
1322			D("fs %d for sch %d not %d still unlinked",
1323				fs->fs.fs_nr, fs->fs.sched_nr,
1324				s->sch.sched_nr);
1325			continue;
1326		}
1327		fsk_attach(fs, s);
1328	}
1329}
1330
1331#ifdef NEW_AQM
1332/* Retrieve AQM configurations to ipfw userland
1333 */
1334static int
1335get_aqm_parms(struct sockopt *sopt)
1336{
1337	struct dn_extra_parms  *ep;
1338	struct dn_fsk *fs;
1339	size_t sopt_valsize;
1340	int l, err = 0;
1341
1342	sopt_valsize = sopt->sopt_valsize;
1343	l = sizeof(*ep);
1344	if (sopt->sopt_valsize < l) {
1345		D("bad len sopt->sopt_valsize %d len %d",
1346			(int) sopt->sopt_valsize , l);
1347		err = EINVAL;
1348		return err;
1349	}
1350	ep = malloc(l, M_DUMMYNET, M_WAITOK);
1351	if(!ep) {
1352		err = ENOMEM ;
1353		return err;
1354	}
1355	do {
1356		err = sooptcopyin(sopt, ep, l, l);
1357		if(err)
1358			break;
1359		sopt->sopt_valsize = sopt_valsize;
1360		if (ep->oid.len < l) {
1361			err = EINVAL;
1362			break;
1363		}
1364
1365		fs = dn_ht_find(dn_cfg.fshash, ep->nr, 0, NULL);
1366		if (!fs) {
1367			D("fs %d not found", ep->nr);
1368			err = EINVAL;
1369			break;
1370		}
1371
1372		if (fs->aqmfp && fs->aqmfp->getconfig) {
1373			if(fs->aqmfp->getconfig(fs, ep)) {
1374				D("Error while trying to get AQM params");
1375				err = EINVAL;
1376				break;
1377			}
1378			ep->oid.len = l;
1379			err = sooptcopyout(sopt, ep, l);
1380		}
1381	}while(0);
1382
1383	free(ep, M_DUMMYNET);
1384	return err;
1385}
1386
1387/* Retrieve AQM configurations to ipfw userland
1388 */
1389static int
1390get_sched_parms(struct sockopt *sopt)
1391{
1392	struct dn_extra_parms  *ep;
1393	struct dn_schk *schk;
1394	size_t sopt_valsize;
1395	int l, err = 0;
1396
1397	sopt_valsize = sopt->sopt_valsize;
1398	l = sizeof(*ep);
1399	if (sopt->sopt_valsize < l) {
1400		D("bad len sopt->sopt_valsize %d len %d",
1401			(int) sopt->sopt_valsize , l);
1402		err = EINVAL;
1403		return err;
1404	}
1405	ep = malloc(l, M_DUMMYNET, M_WAITOK);
1406	if(!ep) {
1407		err = ENOMEM ;
1408		return err;
1409	}
1410	do {
1411		err = sooptcopyin(sopt, ep, l, l);
1412		if(err)
1413			break;
1414		sopt->sopt_valsize = sopt_valsize;
1415		if (ep->oid.len < l) {
1416			err = EINVAL;
1417			break;
1418		}
1419
1420		schk = locate_scheduler(ep->nr);
1421		if (!schk) {
1422			D("sched %d not found", ep->nr);
1423			err = EINVAL;
1424			break;
1425		}
1426
1427		if (schk->fp && schk->fp->getconfig) {
1428			if(schk->fp->getconfig(schk, ep)) {
1429				D("Error while trying to get sched params");
1430				err = EINVAL;
1431				break;
1432			}
1433			ep->oid.len = l;
1434			err = sooptcopyout(sopt, ep, l);
1435		}
1436	}while(0);
1437	free(ep, M_DUMMYNET);
1438
1439	return err;
1440}
1441
1442/* Configure AQM for flowset 'fs'.
1443 * extra parameters are passed from userland.
1444 */
1445static int
1446config_aqm(struct dn_fsk *fs, struct  dn_extra_parms *ep, int busy)
1447{
1448	int err = 0;
1449
1450	do {
1451		/* no configurations */
1452		if (!ep) {
1453			err = 0;
1454			break;
1455		}
1456
1457		/* no AQM for this flowset*/
1458		if (!strcmp(ep->name,"")) {
1459			err = 0;
1460			break;
1461		}
1462		if (ep->oid.len < sizeof(*ep)) {
1463			D("short aqm len %d", ep->oid.len);
1464				err = EINVAL;
1465				break;
1466		}
1467
1468		if (busy) {
1469			D("Unable to configure flowset, flowset busy!");
1470			err = EINVAL;
1471			break;
1472		}
1473
1474		/* deconfigure old aqm if exist */
1475		if (fs->aqmcfg && fs->aqmfp && fs->aqmfp->deconfig) {
1476			aqm_cleanup_deconfig_fs(fs);
1477		}
1478
1479		if (!(fs->aqmfp = find_aqm_type(0, ep->name))) {
1480			D("AQM functions not found for type %s!", ep->name);
1481			fs->fs.flags &= ~DN_IS_AQM;
1482			err = EINVAL;
1483			break;
1484		} else
1485			fs->fs.flags |= DN_IS_AQM;
1486
1487		if (ep->oid.subtype != DN_AQM_PARAMS) {
1488				D("Wrong subtype");
1489				err = EINVAL;
1490				break;
1491		}
1492
1493		if (fs->aqmfp->config) {
1494			err = fs->aqmfp->config(fs, ep, ep->oid.len);
1495			if (err) {
1496					D("Unable to configure AQM for FS %d", fs->fs.fs_nr );
1497					fs->fs.flags &= ~DN_IS_AQM;
1498					fs->aqmfp = NULL;
1499					break;
1500			}
1501		}
1502	} while(0);
1503
1504	return err;
1505}
1506#endif
1507
1508/*
1509 * Configuration -- to preserve backward compatibility we use
1510 * the following scheme (N is 65536)
1511 *	NUMBER		SCHED	LINK	FLOWSET
1512 *	   1 ..  N-1	(1)WFQ	(2)WFQ	(3)queue
1513 *	 N+1 .. 2N-1	(4)FIFO (5)FIFO	(6)FIFO for sched 1..N-1
1514 *	2N+1 .. 3N-1	--	--	(7)FIFO for sched N+1..2N-1
1515 *
1516 * "pipe i config" configures #1, #2 and #3
1517 * "sched i config" configures #1 and possibly #6
1518 * "queue i config" configures #3
1519 * #1 is configured with 'pipe i config' or 'sched i config'
1520 * #2 is configured with 'pipe i config', and created if not
1521 *	existing with 'sched i config'
1522 * #3 is configured with 'queue i config'
1523 * #4 is automatically configured after #1, can only be FIFO
1524 * #5 is automatically configured after #2
1525 * #6 is automatically created when #1 is !MULTIQUEUE,
1526 *	and can be updated.
1527 * #7 is automatically configured after #2
1528 */
1529
1530/*
1531 * configure a link (and its FIFO instance)
1532 */
1533static int
1534config_link(struct dn_link *p, struct dn_id *arg)
1535{
1536	int i;
1537
1538	if (p->oid.len != sizeof(*p)) {
1539		D("invalid pipe len %d", p->oid.len);
1540		return EINVAL;
1541	}
1542	i = p->link_nr;
1543	if (i <= 0 || i >= DN_MAX_ID)
1544		return EINVAL;
1545	/*
1546	 * The config program passes parameters as follows:
1547	 * bw = bits/second (0 means no limits),
1548	 * delay = ms, must be translated into ticks.
1549	 * qsize = slots/bytes
1550	 * burst ???
1551	 */
1552	p->delay = (p->delay * hz) / 1000;
1553	/* Scale burst size: bytes -> bits * hz */
1554	p->burst *= 8 * hz;
1555
1556	DN_BH_WLOCK();
1557	/* do it twice, base link and FIFO link */
1558	for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) {
1559	    struct dn_schk *s = locate_scheduler(i);
1560	    if (s == NULL) {
1561		DN_BH_WUNLOCK();
1562		D("sched %d not found", i);
1563		return EINVAL;
1564	    }
1565	    /* remove profile if exists */
1566	    if (s->profile) {
1567		free(s->profile, M_DUMMYNET);
1568		s->profile = NULL;
1569	    }
1570	    /* copy all parameters */
1571	    s->link.oid = p->oid;
1572	    s->link.link_nr = i;
1573	    s->link.delay = p->delay;
1574	    if (s->link.bandwidth != p->bandwidth) {
1575		/* XXX bandwidth changes, need to update red params */
1576	    s->link.bandwidth = p->bandwidth;
1577		update_red(s);
1578	    }
1579	    s->link.burst = p->burst;
1580	    schk_reset_credit(s);
1581	}
1582	dn_cfg.id++;
1583	DN_BH_WUNLOCK();
1584	return 0;
1585}
1586
1587/*
1588 * configure a flowset. Can be called from inside with locked=1,
1589 */
1590static struct dn_fsk *
1591config_fs(struct dn_fs *nfs, struct dn_id *arg, int locked)
1592{
1593	int i;
1594	struct dn_fsk *fs;
1595#ifdef NEW_AQM
1596	struct dn_extra_parms *ep;
1597#endif
1598
1599	if (nfs->oid.len != sizeof(*nfs)) {
1600		D("invalid flowset len %d", nfs->oid.len);
1601		return NULL;
1602	}
1603	i = nfs->fs_nr;
1604	if (i <= 0 || i >= 3*DN_MAX_ID)
1605		return NULL;
1606#ifdef NEW_AQM
1607	ep = NULL;
1608	if (arg != NULL) {
1609		ep = malloc(sizeof(*ep), M_TEMP, locked ? M_NOWAIT : M_WAITOK);
1610		if (ep == NULL)
1611			return (NULL);
1612		memcpy(ep, arg, sizeof(*ep));
1613	}
1614#endif
1615	ND("flowset %d", i);
1616	/* XXX other sanity checks */
1617        if (nfs->flags & DN_QSIZE_BYTES) {
1618		ipdn_bound_var(&nfs->qsize, 16384,
1619		    1500, dn_cfg.byte_limit, NULL); // "queue byte size");
1620        } else {
1621		ipdn_bound_var(&nfs->qsize, 50,
1622		    1, dn_cfg.slot_limit, NULL); // "queue slot size");
1623        }
1624	if (nfs->flags & DN_HAVE_MASK) {
1625		/* make sure we have some buckets */
1626		ipdn_bound_var((int *)&nfs->buckets, dn_cfg.hash_size,
1627			1, dn_cfg.max_hash_size, "flowset buckets");
1628	} else {
1629		nfs->buckets = 1;	/* we only need 1 */
1630	}
1631	if (!locked)
1632		DN_BH_WLOCK();
1633	do { /* exit with break when done */
1634	    struct dn_schk *s;
1635	    int flags = nfs->sched_nr ? DNHT_INSERT : 0;
1636	    int j;
1637	    int oldc = dn_cfg.fsk_count;
1638	    fs = dn_ht_find(dn_cfg.fshash, i, flags, NULL);
1639	    if (fs == NULL) {
1640		D("missing sched for flowset %d", i);
1641	        break;
1642	    }
1643	    /* grab some defaults from the existing one */
1644	    if (nfs->sched_nr == 0) /* reuse */
1645		nfs->sched_nr = fs->fs.sched_nr;
1646	    for (j = 0; j < sizeof(nfs->par)/sizeof(nfs->par[0]); j++) {
1647		if (nfs->par[j] == -1) /* reuse */
1648		    nfs->par[j] = fs->fs.par[j];
1649	    }
1650	    if (bcmp(&fs->fs, nfs, sizeof(*nfs)) == 0) {
1651		ND("flowset %d unchanged", i);
1652#ifdef NEW_AQM
1653		if (ep != NULL) {
1654			/*
1655			 * Reconfigure AQM as the parameters can be changed.
1656			 * We consider the flowset as busy if it has scheduler
1657			 * instance(s).
1658			 */
1659			s = locate_scheduler(nfs->sched_nr);
1660			config_aqm(fs, ep, s != NULL && s->siht != NULL);
1661		}
1662#endif
1663		break; /* no change, nothing to do */
1664	    }
1665	    if (oldc != dn_cfg.fsk_count)	/* new item */
1666		dn_cfg.id++;
1667	    s = locate_scheduler(nfs->sched_nr);
1668	    /* detach from old scheduler if needed, preserving
1669	     * queues if we need to reattach. Then update the
1670	     * configuration, and possibly attach to the new sched.
1671	     */
1672	    DX(2, "fs %d changed sched %d@%p to %d@%p",
1673		fs->fs.fs_nr,
1674		fs->fs.sched_nr, fs->sched, nfs->sched_nr, s);
1675	    if (fs->sched) {
1676		int flags = s ? DN_DETACH : (DN_DETACH | DN_DESTROY);
1677		flags |= DN_DESTROY; /* XXX temporary */
1678		fsk_detach(fs, flags);
1679	    }
1680	    fs->fs = *nfs; /* copy configuration */
1681#ifdef NEW_AQM
1682			fs->aqmfp = NULL;
1683			if (ep != NULL)
1684				config_aqm(fs, ep, s != NULL &&
1685				    s->siht != NULL);
1686#endif
1687	    if (s != NULL)
1688		fsk_attach(fs, s);
1689	} while (0);
1690	if (!locked)
1691		DN_BH_WUNLOCK();
1692#ifdef NEW_AQM
1693	if (ep != NULL)
1694		free(ep, M_TEMP);
1695#endif
1696	return fs;
1697}
1698
1699/*
1700 * config/reconfig a scheduler and its FIFO variant.
1701 * For !MULTIQUEUE schedulers, also set up the flowset.
1702 *
1703 * On reconfigurations (detected because s->fp is set),
1704 * detach existing flowsets preserving traffic, preserve link,
1705 * and delete the old scheduler creating a new one.
1706 */
1707static int
1708config_sched(struct dn_sch *_nsch, struct dn_id *arg)
1709{
1710	struct dn_schk *s;
1711	struct schk_new_arg a; /* argument for schk_new */
1712	int i;
1713	struct dn_link p;	/* copy of oldlink */
1714	struct dn_profile *pf = NULL;	/* copy of old link profile */
1715	/* Used to preserv mask parameter */
1716	struct ipfw_flow_id new_mask;
1717	int new_buckets = 0;
1718	int new_flags = 0;
1719	int pipe_cmd;
1720	int err = ENOMEM;
1721
1722	a.sch = _nsch;
1723	if (a.sch->oid.len != sizeof(*a.sch)) {
1724		D("bad sched len %d", a.sch->oid.len);
1725		return EINVAL;
1726	}
1727	i = a.sch->sched_nr;
1728	if (i <= 0 || i >= DN_MAX_ID)
1729		return EINVAL;
1730	/* make sure we have some buckets */
1731	if (a.sch->flags & DN_HAVE_MASK)
1732		ipdn_bound_var((int *)&a.sch->buckets, dn_cfg.hash_size,
1733			1, dn_cfg.max_hash_size, "sched buckets");
1734	/* XXX other sanity checks */
1735	bzero(&p, sizeof(p));
1736
1737	pipe_cmd = a.sch->flags & DN_PIPE_CMD;
1738	a.sch->flags &= ~DN_PIPE_CMD; //XXX do it even if is not set?
1739	if (pipe_cmd) {
1740		/* Copy mask parameter */
1741		new_mask = a.sch->sched_mask;
1742		new_buckets = a.sch->buckets;
1743		new_flags = a.sch->flags;
1744	}
1745	DN_BH_WLOCK();
1746again: /* run twice, for wfq and fifo */
1747	/*
1748	 * lookup the type. If not supplied, use the previous one
1749	 * or default to WF2Q+. Otherwise, return an error.
1750	 */
1751	dn_cfg.id++;
1752	a.fp = find_sched_type(a.sch->oid.subtype, a.sch->name);
1753	if (a.fp != NULL) {
1754		/* found. Lookup or create entry */
1755		s = dn_ht_find(dn_cfg.schedhash, i, DNHT_INSERT, &a);
1756	} else if (a.sch->oid.subtype == 0 && !a.sch->name[0]) {
1757		/* No type. search existing s* or retry with WF2Q+ */
1758		s = dn_ht_find(dn_cfg.schedhash, i, 0, &a);
1759		if (s != NULL) {
1760			a.fp = s->fp;
1761			/* Scheduler exists, skip to FIFO scheduler
1762			 * if command was pipe config...
1763			 */
1764			if (pipe_cmd)
1765				goto next;
1766		} else {
1767			/* New scheduler, create a wf2q+ with no mask
1768			 * if command was pipe config...
1769			 */
1770			if (pipe_cmd) {
1771				/* clear mask parameter */
1772				bzero(&a.sch->sched_mask, sizeof(new_mask));
1773				a.sch->buckets = 0;
1774				a.sch->flags &= ~DN_HAVE_MASK;
1775			}
1776			a.sch->oid.subtype = DN_SCHED_WF2QP;
1777			goto again;
1778		}
1779	} else {
1780		D("invalid scheduler type %d %s",
1781			a.sch->oid.subtype, a.sch->name);
1782		err = EINVAL;
1783		goto error;
1784	}
1785	/* normalize name and subtype */
1786	a.sch->oid.subtype = a.fp->type;
1787	bzero(a.sch->name, sizeof(a.sch->name));
1788	strlcpy(a.sch->name, a.fp->name, sizeof(a.sch->name));
1789	if (s == NULL) {
1790		D("cannot allocate scheduler %d", i);
1791		goto error;
1792	}
1793	/* restore existing link if any */
1794	if (p.link_nr) {
1795		s->link = p;
1796		if (!pf || pf->link_nr != p.link_nr) { /* no saved value */
1797			s->profile = NULL; /* XXX maybe not needed */
1798		} else {
1799			s->profile = malloc(sizeof(struct dn_profile),
1800					     M_DUMMYNET, M_NOWAIT | M_ZERO);
1801			if (s->profile == NULL) {
1802				D("cannot allocate profile");
1803				goto error; //XXX
1804			}
1805			memcpy(s->profile, pf, sizeof(*pf));
1806		}
1807	}
1808	p.link_nr = 0;
1809	if (s->fp == NULL) {
1810		DX(2, "sched %d new type %s", i, a.fp->name);
1811	} else if (s->fp != a.fp ||
1812			bcmp(a.sch, &s->sch, sizeof(*a.sch)) ) {
1813		/* already existing. */
1814		DX(2, "sched %d type changed from %s to %s",
1815			i, s->fp->name, a.fp->name);
1816		DX(4, "   type/sub %d/%d -> %d/%d",
1817			s->sch.oid.type, s->sch.oid.subtype,
1818			a.sch->oid.type, a.sch->oid.subtype);
1819		if (s->link.link_nr == 0)
1820			D("XXX WARNING link 0 for sched %d", i);
1821		p = s->link;	/* preserve link */
1822		if (s->profile) {/* preserve profile */
1823			if (!pf)
1824				pf = malloc(sizeof(*pf),
1825				    M_DUMMYNET, M_NOWAIT | M_ZERO);
1826			if (pf)	/* XXX should issue a warning otherwise */
1827				memcpy(pf, s->profile, sizeof(*pf));
1828		}
1829		/* remove from the hash */
1830		dn_ht_find(dn_cfg.schedhash, i, DNHT_REMOVE, NULL);
1831		/* Detach flowsets, preserve queues. */
1832		// schk_delete_cb(s, NULL);
1833		// XXX temporarily, kill queues
1834		schk_delete_cb(s, (void *)DN_DESTROY);
1835		goto again;
1836	} else {
1837		DX(4, "sched %d unchanged type %s", i, a.fp->name);
1838	}
1839	/* complete initialization */
1840	s->sch = *a.sch;
1841	s->fp = a.fp;
1842	s->cfg = arg;
1843	// XXX schk_reset_credit(s);
1844	/* create the internal flowset if needed,
1845	 * trying to reuse existing ones if available
1846	 */
1847	if (!(s->fp->flags & DN_MULTIQUEUE) && !s->fs) {
1848	        s->fs = dn_ht_find(dn_cfg.fshash, i, 0, NULL);
1849		if (!s->fs) {
1850			struct dn_fs fs;
1851			bzero(&fs, sizeof(fs));
1852			set_oid(&fs.oid, DN_FS, sizeof(fs));
1853			fs.fs_nr = i + DN_MAX_ID;
1854			fs.sched_nr = i;
1855			s->fs = config_fs(&fs, NULL, 1 /* locked */);
1856		}
1857		if (!s->fs) {
1858			schk_delete_cb(s, (void *)DN_DESTROY);
1859			D("error creating internal fs for %d", i);
1860			goto error;
1861		}
1862	}
1863	/* call init function after the flowset is created */
1864	if (s->fp->config)
1865		s->fp->config(s);
1866	update_fs(s);
1867next:
1868	if (i < DN_MAX_ID) { /* now configure the FIFO instance */
1869		i += DN_MAX_ID;
1870		if (pipe_cmd) {
1871			/* Restore mask parameter for FIFO */
1872			a.sch->sched_mask = new_mask;
1873			a.sch->buckets = new_buckets;
1874			a.sch->flags = new_flags;
1875		} else {
1876			/* sched config shouldn't modify the FIFO scheduler */
1877			if (dn_ht_find(dn_cfg.schedhash, i, 0, &a) != NULL) {
1878				/* FIFO already exist, don't touch it */
1879				err = 0; /* and this is not an error */
1880				goto error;
1881			}
1882		}
1883		a.sch->sched_nr = i;
1884		a.sch->oid.subtype = DN_SCHED_FIFO;
1885		bzero(a.sch->name, sizeof(a.sch->name));
1886		goto again;
1887	}
1888	err = 0;
1889error:
1890	DN_BH_WUNLOCK();
1891	if (pf)
1892		free(pf, M_DUMMYNET);
1893	return err;
1894}
1895
1896/*
1897 * attach a profile to a link
1898 */
1899static int
1900config_profile(struct dn_profile *pf, struct dn_id *arg)
1901{
1902	struct dn_schk *s;
1903	int i, olen, err = 0;
1904
1905	if (pf->oid.len < sizeof(*pf)) {
1906		D("short profile len %d", pf->oid.len);
1907		return EINVAL;
1908	}
1909	i = pf->link_nr;
1910	if (i <= 0 || i >= DN_MAX_ID)
1911		return EINVAL;
1912	/* XXX other sanity checks */
1913	DN_BH_WLOCK();
1914	for (; i < 2*DN_MAX_ID; i += DN_MAX_ID) {
1915		s = locate_scheduler(i);
1916
1917		if (s == NULL) {
1918			err = EINVAL;
1919			break;
1920		}
1921		dn_cfg.id++;
1922		/*
1923		 * If we had a profile and the new one does not fit,
1924		 * or it is deleted, then we need to free memory.
1925		 */
1926		if (s->profile && (pf->samples_no == 0 ||
1927		    s->profile->oid.len < pf->oid.len)) {
1928			free(s->profile, M_DUMMYNET);
1929			s->profile = NULL;
1930		}
1931		if (pf->samples_no == 0)
1932			continue;
1933		/*
1934		 * new profile, possibly allocate memory
1935		 * and copy data.
1936		 */
1937		if (s->profile == NULL)
1938			s->profile = malloc(pf->oid.len,
1939				    M_DUMMYNET, M_NOWAIT | M_ZERO);
1940		if (s->profile == NULL) {
1941			D("no memory for profile %d", i);
1942			err = ENOMEM;
1943			break;
1944		}
1945		/* preserve larger length XXX double check */
1946		olen = s->profile->oid.len;
1947		if (olen < pf->oid.len)
1948			olen = pf->oid.len;
1949		memcpy(s->profile, pf, pf->oid.len);
1950		s->profile->oid.len = olen;
1951	}
1952	DN_BH_WUNLOCK();
1953	return err;
1954}
1955
1956/*
1957 * Delete all objects:
1958 */
1959static void
1960dummynet_flush(void)
1961{
1962
1963	/* delete all schedulers and related links/queues/flowsets */
1964	dn_ht_scan(dn_cfg.schedhash, schk_delete_cb,
1965		(void *)(uintptr_t)DN_DELETE_FS);
1966	/* delete all remaining (unlinked) flowsets */
1967	DX(4, "still %d unlinked fs", dn_cfg.fsk_count);
1968	dn_ht_free(dn_cfg.fshash, DNHT_REMOVE);
1969	fsk_detach_list(&dn_cfg.fsu, DN_DELETE_FS);
1970	/* Reinitialize system heap... */
1971	heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id));
1972}
1973
1974/*
1975 * Main handler for configuration. We are guaranteed to be called
1976 * with an oid which is at least a dn_id.
1977 * - the first object is the command (config, delete, flush, ...)
1978 * - config_link must be issued after the corresponding config_sched
1979 * - parameters (DN_TXT) for an object must precede the object
1980 *   processed on a config_sched.
1981 */
1982int
1983do_config(void *p, int l)
1984{
1985	struct dn_id o;
1986	union {
1987		struct dn_profile profile;
1988		struct dn_fs fs;
1989		struct dn_link link;
1990		struct dn_sch sched;
1991	} *dn;
1992	struct dn_id *arg;
1993	uintptr_t a;
1994	int err, err2, off;
1995
1996	memcpy(&o, p, sizeof(o));
1997	if (o.id != DN_API_VERSION) {
1998		D("invalid api version got %d need %d", o.id, DN_API_VERSION);
1999		return EINVAL;
2000	}
2001	arg = NULL;
2002	dn = NULL;
2003	for (off = 0; l >= sizeof(o); memcpy(&o, (char *)p + off, sizeof(o))) {
2004		if (o.len < sizeof(o) || l < o.len) {
2005			D("bad len o.len %d len %d", o.len, l);
2006			err = EINVAL;
2007			break;
2008		}
2009		l -= o.len;
2010		err = 0;
2011		switch (o.type) {
2012		default:
2013			D("cmd %d not implemented", o.type);
2014			break;
2015
2016#ifdef EMULATE_SYSCTL
2017		/* sysctl emulation.
2018		 * if we recognize the command, jump to the correct
2019		 * handler and return
2020		 */
2021		case DN_SYSCTL_SET:
2022			err = kesysctl_emu_set(p, l);
2023			return err;
2024#endif
2025
2026		case DN_CMD_CONFIG: /* simply a header */
2027			break;
2028
2029		case DN_CMD_DELETE:
2030			/* the argument is in the first uintptr_t after o */
2031			if (o.len < sizeof(o) + sizeof(a)) {
2032				err = EINVAL;
2033				break;
2034			}
2035			memcpy(&a, (char *)p + off + sizeof(o), sizeof(a));
2036			switch (o.subtype) {
2037			case DN_LINK:
2038				/* delete base and derived schedulers */
2039				DN_BH_WLOCK();
2040				err = delete_schk(a);
2041				err2 = delete_schk(a + DN_MAX_ID);
2042				DN_BH_WUNLOCK();
2043				if (!err)
2044					err = err2;
2045				break;
2046
2047			default:
2048				D("invalid delete type %d", o.subtype);
2049				err = EINVAL;
2050				break;
2051
2052			case DN_FS:
2053				err = (a < 1 || a >= DN_MAX_ID) ?
2054				    EINVAL : delete_fs(a, 0) ;
2055				break;
2056			}
2057			break;
2058
2059		case DN_CMD_FLUSH:
2060			DN_BH_WLOCK();
2061			dummynet_flush();
2062			DN_BH_WUNLOCK();
2063			break;
2064		case DN_TEXT:	/* store argument of next block */
2065			if (arg != NULL)
2066				free(arg, M_TEMP);
2067			arg = malloc(o.len, M_TEMP, M_WAITOK);
2068			memcpy(arg, (char *)p + off, o.len);
2069			break;
2070		case DN_LINK:
2071			if (dn == NULL)
2072				dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK);
2073			memcpy(&dn->link, (char *)p + off, sizeof(dn->link));
2074			err = config_link(&dn->link, arg);
2075			break;
2076		case DN_PROFILE:
2077			if (dn == NULL)
2078				dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK);
2079			memcpy(&dn->profile, (char *)p + off,
2080			    sizeof(dn->profile));
2081			err = config_profile(&dn->profile, arg);
2082			break;
2083		case DN_SCH:
2084			if (dn == NULL)
2085				dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK);
2086			memcpy(&dn->sched, (char *)p + off,
2087			    sizeof(dn->sched));
2088			err = config_sched(&dn->sched, arg);
2089			break;
2090		case DN_FS:
2091			if (dn == NULL)
2092				dn = malloc(sizeof(*dn), M_TEMP, M_WAITOK);
2093			memcpy(&dn->fs, (char *)p + off, sizeof(dn->fs));
2094			err = (NULL == config_fs(&dn->fs, arg, 0));
2095			break;
2096		}
2097		if (err != 0)
2098			break;
2099		off += o.len;
2100	}
2101	if (arg != NULL)
2102		free(arg, M_TEMP);
2103	if (dn != NULL)
2104		free(dn, M_TEMP);
2105	return err;
2106}
2107
2108static int
2109compute_space(struct dn_id *cmd, struct copy_args *a)
2110{
2111	int x = 0, need = 0;
2112	int profile_size = sizeof(struct dn_profile) -
2113		ED_MAX_SAMPLES_NO*sizeof(int);
2114
2115	/* NOTE about compute space:
2116	 * NP 	= dn_cfg.schk_count
2117	 * NSI 	= dn_cfg.si_count
2118	 * NF 	= dn_cfg.fsk_count
2119	 * NQ 	= dn_cfg.queue_count
2120	 * - ipfw pipe show
2121	 *   (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler
2122	 *                             link, scheduler template, flowset
2123	 *                             integrated in scheduler and header
2124	 *                             for flowset list
2125	 *   (NSI)*(dn_flow) all scheduler instance (includes
2126	 *                              the queue instance)
2127	 * - ipfw sched show
2128	 *   (NP/2)*(dn_link + dn_sch + dn_id + dn_fs) only half scheduler
2129	 *                             link, scheduler template, flowset
2130	 *                             integrated in scheduler and header
2131	 *                             for flowset list
2132	 *   (NSI * dn_flow) all scheduler instances
2133	 *   (NF * sizeof(uint_32)) space for flowset list linked to scheduler
2134	 *   (NQ * dn_queue) all queue [XXXfor now not listed]
2135	 * - ipfw queue show
2136	 *   (NF * dn_fs) all flowset
2137	 *   (NQ * dn_queue) all queues
2138	 */
2139	switch (cmd->subtype) {
2140	default:
2141		return -1;
2142	/* XXX where do LINK and SCH differ ? */
2143	/* 'ipfw sched show' could list all queues associated to
2144	 * a scheduler. This feature for now is disabled
2145	 */
2146	case DN_LINK:	/* pipe show */
2147		x = DN_C_LINK | DN_C_SCH | DN_C_FLOW;
2148		need += dn_cfg.schk_count *
2149			(sizeof(struct dn_fs) + profile_size) / 2;
2150		need += dn_cfg.fsk_count * sizeof(uint32_t);
2151		break;
2152	case DN_SCH:	/* sched show */
2153		need += dn_cfg.schk_count *
2154			(sizeof(struct dn_fs) + profile_size) / 2;
2155		need += dn_cfg.fsk_count * sizeof(uint32_t);
2156		x = DN_C_SCH | DN_C_LINK | DN_C_FLOW;
2157		break;
2158	case DN_FS:	/* queue show */
2159		x = DN_C_FS | DN_C_QUEUE;
2160		break;
2161	case DN_GET_COMPAT:	/* compatibility mode */
2162		need =  dn_compat_calc_size();
2163		break;
2164	}
2165	a->flags = x;
2166	if (x & DN_C_SCH) {
2167		need += dn_cfg.schk_count * sizeof(struct dn_sch) / 2;
2168		/* NOT also, each fs might be attached to a sched */
2169		need += dn_cfg.schk_count * sizeof(struct dn_id) / 2;
2170	}
2171	if (x & DN_C_FS)
2172		need += dn_cfg.fsk_count * sizeof(struct dn_fs);
2173	if (x & DN_C_LINK) {
2174		need += dn_cfg.schk_count * sizeof(struct dn_link) / 2;
2175	}
2176	/*
2177	 * When exporting a queue to userland, only pass up the
2178	 * struct dn_flow, which is the only visible part.
2179	 */
2180
2181	if (x & DN_C_QUEUE)
2182		need += dn_cfg.queue_count * sizeof(struct dn_flow);
2183	if (x & DN_C_FLOW)
2184		need += dn_cfg.si_count * (sizeof(struct dn_flow));
2185	return need;
2186}
2187
2188/*
2189 * If compat != NULL dummynet_get is called in compatibility mode.
2190 * *compat will be the pointer to the buffer to pass to ipfw
2191 */
2192int
2193dummynet_get(struct sockopt *sopt, void **compat)
2194{
2195	int have, i, need, error;
2196	char *start = NULL, *buf;
2197	size_t sopt_valsize;
2198	struct dn_id *cmd;
2199	struct copy_args a;
2200	struct copy_range r;
2201	int l = sizeof(struct dn_id);
2202
2203	bzero(&a, sizeof(a));
2204	bzero(&r, sizeof(r));
2205
2206	/* save and restore original sopt_valsize around copyin */
2207	sopt_valsize = sopt->sopt_valsize;
2208
2209	cmd = &r.o;
2210
2211	if (!compat) {
2212		/* copy at least an oid, and possibly a full object */
2213		error = sooptcopyin(sopt, cmd, sizeof(r), sizeof(*cmd));
2214		sopt->sopt_valsize = sopt_valsize;
2215		if (error)
2216			goto done;
2217		l = cmd->len;
2218#ifdef EMULATE_SYSCTL
2219		/* sysctl emulation. */
2220		if (cmd->type == DN_SYSCTL_GET)
2221			return kesysctl_emu_get(sopt);
2222#endif
2223		if (l > sizeof(r)) {
2224			/* request larger than default, allocate buffer */
2225			cmd = malloc(l,  M_DUMMYNET, M_WAITOK);
2226			error = sooptcopyin(sopt, cmd, l, l);
2227			sopt->sopt_valsize = sopt_valsize;
2228			if (error)
2229				goto done;
2230		}
2231	} else { /* compatibility */
2232		error = 0;
2233		cmd->type = DN_CMD_GET;
2234		cmd->len = sizeof(struct dn_id);
2235		cmd->subtype = DN_GET_COMPAT;
2236		// cmd->id = sopt_valsize;
2237		D("compatibility mode");
2238	}
2239
2240#ifdef NEW_AQM
2241	/* get AQM params */
2242	if(cmd->subtype == DN_AQM_PARAMS) {
2243		error = get_aqm_parms(sopt);
2244		goto done;
2245	/* get Scheduler params */
2246	} else if (cmd->subtype == DN_SCH_PARAMS) {
2247		error = get_sched_parms(sopt);
2248		goto done;
2249	}
2250#endif
2251
2252	a.extra = (struct copy_range *)cmd;
2253	if (cmd->len == sizeof(*cmd)) { /* no range, create a default */
2254		uint32_t *rp = (uint32_t *)(cmd + 1);
2255		cmd->len += 2* sizeof(uint32_t);
2256		rp[0] = 1;
2257		rp[1] = DN_MAX_ID - 1;
2258		if (cmd->subtype == DN_LINK) {
2259			rp[0] += DN_MAX_ID;
2260			rp[1] += DN_MAX_ID;
2261		}
2262	}
2263	/* Count space (under lock) and allocate (outside lock).
2264	 * Exit with lock held if we manage to get enough buffer.
2265	 * Try a few times then give up.
2266	 */
2267	for (have = 0, i = 0; i < 10; i++) {
2268		DN_BH_WLOCK();
2269		need = compute_space(cmd, &a);
2270
2271		/* if there is a range, ignore value from compute_space() */
2272		if (l > sizeof(*cmd))
2273			need = sopt_valsize - sizeof(*cmd);
2274
2275		if (need < 0) {
2276			DN_BH_WUNLOCK();
2277			error = EINVAL;
2278			goto done;
2279		}
2280		need += sizeof(*cmd);
2281		cmd->id = need;
2282		if (have >= need)
2283			break;
2284
2285		DN_BH_WUNLOCK();
2286		if (start)
2287			free(start, M_DUMMYNET);
2288		start = NULL;
2289		if (need > sopt_valsize)
2290			break;
2291
2292		have = need;
2293		start = malloc(have, M_DUMMYNET, M_WAITOK | M_ZERO);
2294	}
2295
2296	if (start == NULL) {
2297		if (compat) {
2298			*compat = NULL;
2299			error =  1; // XXX
2300		} else {
2301			error = sooptcopyout(sopt, cmd, sizeof(*cmd));
2302		}
2303		goto done;
2304	}
2305	ND("have %d:%d sched %d, %d:%d links %d, %d:%d flowsets %d, "
2306		"%d:%d si %d, %d:%d queues %d",
2307		dn_cfg.schk_count, sizeof(struct dn_sch), DN_SCH,
2308		dn_cfg.schk_count, sizeof(struct dn_link), DN_LINK,
2309		dn_cfg.fsk_count, sizeof(struct dn_fs), DN_FS,
2310		dn_cfg.si_count, sizeof(struct dn_flow), DN_SCH_I,
2311		dn_cfg.queue_count, sizeof(struct dn_queue), DN_QUEUE);
2312	sopt->sopt_valsize = sopt_valsize;
2313	a.type = cmd->subtype;
2314
2315	if (compat == NULL) {
2316		memcpy(start, cmd, sizeof(*cmd));
2317		((struct dn_id*)(start))->len = sizeof(struct dn_id);
2318		buf = start + sizeof(*cmd);
2319	} else
2320		buf = start;
2321	a.start = &buf;
2322	a.end = start + have;
2323	/* start copying other objects */
2324	if (compat) {
2325		a.type = DN_COMPAT_PIPE;
2326		dn_ht_scan(dn_cfg.schedhash, copy_data_helper_compat, &a);
2327		a.type = DN_COMPAT_QUEUE;
2328		dn_ht_scan(dn_cfg.fshash, copy_data_helper_compat, &a);
2329	} else if (a.type == DN_FS) {
2330		dn_ht_scan(dn_cfg.fshash, copy_data_helper, &a);
2331	} else {
2332		dn_ht_scan(dn_cfg.schedhash, copy_data_helper, &a);
2333	}
2334	DN_BH_WUNLOCK();
2335
2336	if (compat) {
2337		*compat = start;
2338		sopt->sopt_valsize = buf - start;
2339		/* free() is done by ip_dummynet_compat() */
2340		start = NULL; //XXX hack
2341	} else {
2342		error = sooptcopyout(sopt, start, buf - start);
2343	}
2344done:
2345	if (cmd && cmd != &r.o)
2346		free(cmd, M_DUMMYNET);
2347	if (start)
2348		free(start, M_DUMMYNET);
2349	return error;
2350}
2351
2352/* Callback called on scheduler instance to delete it if idle */
2353static int
2354drain_scheduler_cb(void *_si, void *arg)
2355{
2356	struct dn_sch_inst *si = _si;
2357
2358	if ((si->kflags & DN_ACTIVE) || si->dline.mq.head != NULL)
2359		return 0;
2360
2361	if (si->sched->fp->flags & DN_MULTIQUEUE) {
2362		if (si->q_count == 0)
2363			return si_destroy(si, NULL);
2364		else
2365			return 0;
2366	} else { /* !DN_MULTIQUEUE */
2367		if ((si+1)->ni.length == 0)
2368			return si_destroy(si, NULL);
2369		else
2370			return 0;
2371	}
2372	return 0; /* unreachable */
2373}
2374
2375/* Callback called on scheduler to check if it has instances */
2376static int
2377drain_scheduler_sch_cb(void *_s, void *arg)
2378{
2379	struct dn_schk *s = _s;
2380
2381	if (s->sch.flags & DN_HAVE_MASK) {
2382		dn_ht_scan_bucket(s->siht, &s->drain_bucket,
2383				drain_scheduler_cb, NULL);
2384		s->drain_bucket++;
2385	} else {
2386		if (s->siht) {
2387			if (drain_scheduler_cb(s->siht, NULL) == DNHT_SCAN_DEL)
2388				s->siht = NULL;
2389		}
2390	}
2391	return 0;
2392}
2393
2394/* Called every tick, try to delete a 'bucket' of scheduler */
2395void
2396dn_drain_scheduler(void)
2397{
2398	dn_ht_scan_bucket(dn_cfg.schedhash, &dn_cfg.drain_sch,
2399			   drain_scheduler_sch_cb, NULL);
2400	dn_cfg.drain_sch++;
2401}
2402
2403/* Callback called on queue to delete if it is idle */
2404static int
2405drain_queue_cb(void *_q, void *arg)
2406{
2407	struct dn_queue *q = _q;
2408
2409	if (q->ni.length == 0) {
2410		dn_delete_queue(q, DN_DESTROY);
2411		return DNHT_SCAN_DEL; /* queue is deleted */
2412	}
2413
2414	return 0; /* queue isn't deleted */
2415}
2416
2417/* Callback called on flowset used to check if it has queues */
2418static int
2419drain_queue_fs_cb(void *_fs, void *arg)
2420{
2421	struct dn_fsk *fs = _fs;
2422
2423	if (fs->fs.flags & DN_QHT_HASH) {
2424		/* Flowset has a hash table for queues */
2425		dn_ht_scan_bucket(fs->qht, &fs->drain_bucket,
2426				drain_queue_cb, NULL);
2427		fs->drain_bucket++;
2428	} else {
2429		/* No hash table for this flowset, null the pointer
2430		 * if the queue is deleted
2431		 */
2432		if (fs->qht) {
2433			if (drain_queue_cb(fs->qht, NULL) == DNHT_SCAN_DEL)
2434				fs->qht = NULL;
2435		}
2436	}
2437	return 0;
2438}
2439
2440/* Called every tick, try to delete a 'bucket' of queue */
2441void
2442dn_drain_queue(void)
2443{
2444	/* scan a bucket of flowset */
2445	dn_ht_scan_bucket(dn_cfg.fshash, &dn_cfg.drain_fs,
2446                               drain_queue_fs_cb, NULL);
2447	dn_cfg.drain_fs++;
2448}
2449
2450/*
2451 * Handler for the various dummynet socket options
2452 */
2453static int
2454ip_dn_ctl(struct sockopt *sopt)
2455{
2456	void *p = NULL;
2457	int error, l;
2458
2459	error = priv_check(sopt->sopt_td, PRIV_NETINET_DUMMYNET);
2460	if (error)
2461		return (error);
2462
2463	/* Disallow sets in really-really secure mode. */
2464	if (sopt->sopt_dir == SOPT_SET) {
2465		error =  securelevel_ge(sopt->sopt_td->td_ucred, 3);
2466		if (error)
2467			return (error);
2468	}
2469
2470	switch (sopt->sopt_name) {
2471	default :
2472		D("dummynet: unknown option %d", sopt->sopt_name);
2473		error = EINVAL;
2474		break;
2475
2476	case IP_DUMMYNET_FLUSH:
2477	case IP_DUMMYNET_CONFIGURE:
2478	case IP_DUMMYNET_DEL:	/* remove a pipe or queue */
2479	case IP_DUMMYNET_GET:
2480		D("dummynet: compat option %d", sopt->sopt_name);
2481		error = ip_dummynet_compat(sopt);
2482		break;
2483
2484	case IP_DUMMYNET3 :
2485		if (sopt->sopt_dir == SOPT_GET) {
2486			error = dummynet_get(sopt, NULL);
2487			break;
2488		}
2489		l = sopt->sopt_valsize;
2490		if (l < sizeof(struct dn_id) || l > 12000) {
2491			D("argument len %d invalid", l);
2492			break;
2493		}
2494		p = malloc(l, M_TEMP, M_WAITOK); // XXX can it fail ?
2495		error = sooptcopyin(sopt, p, l, l);
2496		if (error)
2497			break ;
2498		error = do_config(p, l);
2499		break;
2500	}
2501
2502	if (p != NULL)
2503		free(p, M_TEMP);
2504
2505	return error ;
2506}
2507
2508
2509static void
2510ip_dn_init(void)
2511{
2512	if (dn_cfg.init_done)
2513		return;
2514	printf("DUMMYNET %p with IPv6 initialized (100409)\n", curvnet);
2515	dn_cfg.init_done = 1;
2516	/* Set defaults here. MSVC does not accept initializers,
2517	 * and this is also useful for vimages
2518	 */
2519	/* queue limits */
2520	dn_cfg.slot_limit = 100; /* Foot shooting limit for queues. */
2521	dn_cfg.byte_limit = 1024 * 1024;
2522	dn_cfg.expire = 1;
2523
2524	/* RED parameters */
2525	dn_cfg.red_lookup_depth = 256;	/* default lookup table depth */
2526	dn_cfg.red_avg_pkt_size = 512;	/* default medium packet size */
2527	dn_cfg.red_max_pkt_size = 1500;	/* default max packet size */
2528
2529	/* hash tables */
2530	dn_cfg.max_hash_size = 65536;	/* max in the hash tables */
2531	dn_cfg.hash_size = 64;		/* default hash size */
2532
2533	/* create hash tables for schedulers and flowsets.
2534	 * In both we search by key and by pointer.
2535	 */
2536	dn_cfg.schedhash = dn_ht_init(NULL, dn_cfg.hash_size,
2537		offsetof(struct dn_schk, schk_next),
2538		schk_hash, schk_match, schk_new);
2539	dn_cfg.fshash = dn_ht_init(NULL, dn_cfg.hash_size,
2540		offsetof(struct dn_fsk, fsk_next),
2541		fsk_hash, fsk_match, fsk_new);
2542
2543	/* bucket index to drain object */
2544	dn_cfg.drain_fs = 0;
2545	dn_cfg.drain_sch = 0;
2546
2547	heap_init(&dn_cfg.evheap, 16, offsetof(struct dn_id, id));
2548	SLIST_INIT(&dn_cfg.fsu);
2549	SLIST_INIT(&dn_cfg.schedlist);
2550
2551	DN_LOCK_INIT();
2552
2553	NET_TASK_INIT(&dn_task, 0, dummynet_task, curvnet);
2554	dn_tq = taskqueue_create_fast("dummynet", M_WAITOK,
2555	    taskqueue_thread_enqueue, &dn_tq);
2556	taskqueue_start_threads(&dn_tq, 1, PI_NET, "dummynet");
2557
2558	callout_init(&dn_timeout, 1);
2559	dn_reschedule();
2560
2561	/* Initialize curr_time adjustment mechanics. */
2562	getmicrouptime(&dn_cfg.prev_t);
2563}
2564
2565static void
2566ip_dn_destroy(int last)
2567{
2568	DN_BH_WLOCK();
2569	/* ensure no more callouts are started */
2570	dn_gone = 1;
2571
2572	/* check for last */
2573	if (last) {
2574		ND("removing last instance\n");
2575		ip_dn_ctl_ptr = NULL;
2576		ip_dn_io_ptr = NULL;
2577	}
2578
2579	dummynet_flush();
2580	DN_BH_WUNLOCK();
2581
2582	callout_drain(&dn_timeout);
2583	taskqueue_drain(dn_tq, &dn_task);
2584	taskqueue_free(dn_tq);
2585
2586	dn_ht_free(dn_cfg.schedhash, 0);
2587	dn_ht_free(dn_cfg.fshash, 0);
2588	heap_free(&dn_cfg.evheap);
2589
2590	DN_LOCK_DESTROY();
2591}
2592
2593static int
2594dummynet_modevent(module_t mod, int type, void *data)
2595{
2596
2597	if (type == MOD_LOAD) {
2598		if (ip_dn_io_ptr) {
2599			printf("DUMMYNET already loaded\n");
2600			return EEXIST ;
2601		}
2602		ip_dn_init();
2603		ip_dn_ctl_ptr = ip_dn_ctl;
2604		ip_dn_io_ptr = dummynet_io;
2605		return 0;
2606	} else if (type == MOD_UNLOAD) {
2607		ip_dn_destroy(1 /* last */);
2608		return 0;
2609	} else
2610		return EOPNOTSUPP;
2611}
2612
2613/* modevent helpers for the modules */
2614static int
2615load_dn_sched(struct dn_alg *d)
2616{
2617	struct dn_alg *s;
2618
2619	if (d == NULL)
2620		return 1; /* error */
2621	ip_dn_init();	/* just in case, we need the lock */
2622
2623	/* Check that mandatory funcs exists */
2624	if (d->enqueue == NULL || d->dequeue == NULL) {
2625		D("missing enqueue or dequeue for %s", d->name);
2626		return 1;
2627	}
2628
2629	/* Search if scheduler already exists */
2630	DN_BH_WLOCK();
2631	SLIST_FOREACH(s, &dn_cfg.schedlist, next) {
2632		if (strcmp(s->name, d->name) == 0) {
2633			D("%s already loaded", d->name);
2634			break; /* scheduler already exists */
2635		}
2636	}
2637	if (s == NULL)
2638		SLIST_INSERT_HEAD(&dn_cfg.schedlist, d, next);
2639	DN_BH_WUNLOCK();
2640	D("dn_sched %s %sloaded", d->name, s ? "not ":"");
2641	return s ? 1 : 0;
2642}
2643
2644static int
2645unload_dn_sched(struct dn_alg *s)
2646{
2647	struct dn_alg *tmp, *r;
2648	int err = EINVAL;
2649
2650	ND("called for %s", s->name);
2651
2652	DN_BH_WLOCK();
2653	SLIST_FOREACH_SAFE(r, &dn_cfg.schedlist, next, tmp) {
2654		if (strcmp(s->name, r->name) != 0)
2655			continue;
2656		ND("ref_count = %d", r->ref_count);
2657		err = (r->ref_count != 0) ? EBUSY : 0;
2658		if (err == 0)
2659			SLIST_REMOVE(&dn_cfg.schedlist, r, dn_alg, next);
2660		break;
2661	}
2662	DN_BH_WUNLOCK();
2663	D("dn_sched %s %sunloaded", s->name, err ? "not ":"");
2664	return err;
2665}
2666
2667int
2668dn_sched_modevent(module_t mod, int cmd, void *arg)
2669{
2670	struct dn_alg *sch = arg;
2671
2672	if (cmd == MOD_LOAD)
2673		return load_dn_sched(sch);
2674	else if (cmd == MOD_UNLOAD)
2675		return unload_dn_sched(sch);
2676	else
2677		return EINVAL;
2678}
2679
2680static moduledata_t dummynet_mod = {
2681	"dummynet", dummynet_modevent, NULL
2682};
2683
2684#define	DN_SI_SUB	SI_SUB_PROTO_FIREWALL
2685#define	DN_MODEV_ORD	(SI_ORDER_ANY - 128) /* after ipfw */
2686DECLARE_MODULE(dummynet, dummynet_mod, DN_SI_SUB, DN_MODEV_ORD);
2687MODULE_DEPEND(dummynet, ipfw, 3, 3, 3);
2688MODULE_VERSION(dummynet, 3);
2689
2690/*
2691 * Starting up. Done in order after dummynet_modevent() has been called.
2692 * VNET_SYSINIT is also called for each existing vnet and each new vnet.
2693 */
2694//VNET_SYSINIT(vnet_dn_init, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_init, NULL);
2695
2696/*
2697 * Shutdown handlers up shop. These are done in REVERSE ORDER, but still
2698 * after dummynet_modevent() has been called. Not called on reboot.
2699 * VNET_SYSUNINIT is also called for each exiting vnet as it exits.
2700 * or when the module is unloaded.
2701 */
2702//VNET_SYSUNINIT(vnet_dn_uninit, DN_SI_SUB, DN_MODEV_ORD+2, ip_dn_destroy, NULL);
2703
2704#ifdef NEW_AQM
2705
2706/* modevent helpers for the AQM modules */
2707static int
2708load_dn_aqm(struct dn_aqm *d)
2709{
2710	struct dn_aqm *aqm=NULL;
2711
2712	if (d == NULL)
2713		return 1; /* error */
2714	ip_dn_init();	/* just in case, we need the lock */
2715
2716	/* Check that mandatory funcs exists */
2717	if (d->enqueue == NULL || d->dequeue == NULL) {
2718		D("missing enqueue or dequeue for %s", d->name);
2719		return 1;
2720	}
2721
2722	/* Search if AQM already exists */
2723	DN_BH_WLOCK();
2724	SLIST_FOREACH(aqm, &dn_cfg.aqmlist, next) {
2725		if (strcmp(aqm->name, d->name) == 0) {
2726			D("%s already loaded", d->name);
2727			break; /* AQM already exists */
2728		}
2729	}
2730	if (aqm == NULL)
2731		SLIST_INSERT_HEAD(&dn_cfg.aqmlist, d, next);
2732	DN_BH_WUNLOCK();
2733	D("dn_aqm %s %sloaded", d->name, aqm ? "not ":"");
2734	return aqm ? 1 : 0;
2735}
2736
2737
2738/* Callback to clean up AQM status for queues connected to a flowset
2739 * and then deconfigure the flowset.
2740 * This function is called before an AQM module is unloaded
2741 */
2742static int
2743fs_cleanup(void *_fs, void *arg)
2744{
2745	struct dn_fsk *fs = _fs;
2746	uint32_t type = *(uint32_t *)arg;
2747
2748	if (fs->aqmfp && fs->aqmfp->type == type)
2749		aqm_cleanup_deconfig_fs(fs);
2750
2751	return 0;
2752}
2753
2754static int
2755unload_dn_aqm(struct dn_aqm *aqm)
2756{
2757	struct dn_aqm *tmp, *r;
2758	int err = EINVAL;
2759	err = 0;
2760	ND("called for %s", aqm->name);
2761
2762	DN_BH_WLOCK();
2763
2764	/* clean up AQM status and deconfig flowset */
2765	dn_ht_scan(dn_cfg.fshash, fs_cleanup, &aqm->type);
2766
2767	SLIST_FOREACH_SAFE(r, &dn_cfg.aqmlist, next, tmp) {
2768		if (strcmp(aqm->name, r->name) != 0)
2769			continue;
2770		ND("ref_count = %d", r->ref_count);
2771		err = (r->ref_count != 0 || r->cfg_ref_count != 0) ? EBUSY : 0;
2772		if (err == 0)
2773			SLIST_REMOVE(&dn_cfg.aqmlist, r, dn_aqm, next);
2774		break;
2775	}
2776	DN_BH_WUNLOCK();
2777	D("%s %sunloaded", aqm->name, err ? "not ":"");
2778	if (err)
2779		D("ref_count=%d, cfg_ref_count=%d", r->ref_count, r->cfg_ref_count);
2780	return err;
2781}
2782
2783int
2784dn_aqm_modevent(module_t mod, int cmd, void *arg)
2785{
2786	struct dn_aqm *aqm = arg;
2787
2788	if (cmd == MOD_LOAD)
2789		return load_dn_aqm(aqm);
2790	else if (cmd == MOD_UNLOAD)
2791		return unload_dn_aqm(aqm);
2792	else
2793		return EINVAL;
2794}
2795#endif
2796
2797/* end of file */
2798
2799