xref: /illumos-gate/usr/src/cmd/intrd/intrd.pl (revision d89fccd8)
1#!/usr/perl5/bin/perl
2#
3# CDDL HEADER START
4#
5# The contents of this file are subject to the terms of the
6# Common Development and Distribution License (the "License").
7# You may not use this file except in compliance with the License.
8#
9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10# or http://www.opensolaris.org/os/licensing.
11# See the License for the specific language governing permissions
12# and limitations under the License.
13#
14# When distributing Covered Code, include this CDDL HEADER in each
15# file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16# If applicable, add the following below this CDDL HEADER, with the
17# fields enclosed by brackets "[]" replaced with your own identifying
18# information: Portions Copyright [yyyy] [name of copyright owner]
19#
20# CDDL HEADER END
21#
22
23#
24# Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
25# Use is subject to license terms.
26#
27#ident	"%Z%%M%	%I%	%E% SMI"
28#
29
30require 5.6.1;
31use strict;
32use warnings;
33use POSIX;
34use File::Basename("basename");
35
36my $cmdname = basename($0);
37
38my $using_scengen = 0;	# 1 if using scenario simulator
39my $debug = 0;
40
41my $min_sleeptime = 1;
42my $max_sleeptime = 15;
43my $onecpu_sleeptime = (60 * 15);	# used if only 1 CPU on system
44my $sleeptime = $min_sleeptime;	# time to sleep between kstat updates
45
46# For timerange_foo variables, see comments at tail of &getstat()
47
48my $timerange_toohi    = .01;
49my $timerange_hithresh = .0003;
50my $timerange_lothresh = $timerange_hithresh / 2;
51my $unsafe_timerange   = .02;
52
53my $statslen = 60;	# time period (in secs) to keep in @deltas
54
55
56# Parse arguments. intrd does not accept any public arguments; the two
57# arguments below are meant for testing purposes. -D generates a significant
58# amount of syslog output. -S <filename> loads the filename as a perl
59# script. That file is expected to implement a kstat "simulator" which
60# can be used to feed information to intrd and verify intrd's responses.
61
62while ($_ = shift @ARGV) {
63	if ($_ eq "-S" && $#ARGV != -1) {
64		$using_scengen = 1;
65		do $ARGV[0];	# load simulator
66		shift @ARGV;
67	} elsif ($_ eq "-D") {
68		$debug = 1;
69	}
70}
71
72if ($using_scengen == 0) {
73	require Sun::Solaris::Kstat;
74	require Sun::Solaris::Intrs;
75	import Sun::Solaris::Intrs(qw(intrmove));
76	require Sys::Syslog;
77	import Sys::Syslog;
78	openlog($cmdname, 'pid', 'daemon');
79	setlogmask(Sys::Syslog::LOG_UPTO($debug > 0 ? &Sys::Syslog::LOG_DEBUG :
80	    &Sys::Syslog::LOG_INFO));
81}
82
83
84my $asserted = 0;
85my $assert_level = 'debug';	# syslog level for assertion failures
86sub VERIFY($@)
87{
88	my $bad = (shift() == 0);	# $_[0] == 0 means assert failed
89	if ($bad) {
90		my $msg = shift();
91		syslog($assert_level, "VERIFY: $msg", @_);
92		$asserted++;
93	}
94	return ($bad);
95}
96
97
98
99
100sub getstat($);
101sub generate_delta($$);
102sub compress_deltas($);
103sub dumpdelta($);
104
105sub goodness($);
106sub imbalanced($$);
107sub do_reconfig($);
108
109sub goodness_cpu($$);		# private function
110sub move_intr($$$$);		# private function
111sub ivecs_to_string(@);		# private function
112sub do_find_goal($$$$);		# private function
113sub find_goal($$);		# private function
114sub do_reconfig_cpu2cpu($$$$);	# private function
115sub do_reconfig_cpu($$$);	# private function
116
117
118#
119# What follow are the basic data structures routines of intrd.
120#
121# getstat() is responsible for reading the kstats and generating a "stat" hash.
122#
123# generate_delta() is responsible for taking two "stat" hashes and creating
124# a new "delta" hash that represents what has changed over time.
125#
126# compress_deltas() is responsible for taking a list of deltas and generating
127# a single delta hash that encompasses all the time periods described by the
128# deltas.
129
130
131#
132# getstat() is handed a reference to a kstat and generates a hash, returned
133# by reference, containing all the fields from the kstats which we need.
134# If it returns the scalar 0, it failed to gather the kstats, and the caller
135# should react accordingly.
136#
137# getstat() is also responsible for maintaining a reasonable $sleeptime.
138#
139# {"snaptime"}          kstat's snaptime
140# {<cpuid>}             one hash reference per online cpu
141#  ->{"tot"}            == cpu:<cpuid>:sys:cpu_nsec_{user + kernel + idle}
142#  ->{"crtime"}         == cpu:<cpuid>:sys:crtime
143#  ->{"ivecs"}
144#     ->{<cookie#>}     iterates over pci_intrs::<nexus>:cookie
145#        ->{"time"}     == pci_intrs:<ivec#>:<nexus>:time (in nsec)
146#        ->{"pil"}      == pci_intrs:<ivec#>:<nexus>:pil
147#        ->{"crtime"}   == pci_intrs:<ivec#>:<nexus>:crtime
148#        ->{"ino"}      == pci_intrs:<ivec#>:<nexus>:ino
149#        ->{"buspath"}  == pci_intrs:<ivec#>:<nexus>:buspath
150#        ->{"name"}     == pci_intrs:<ivec#>:<nexus>:name
151#        ->{"ihs"}      == pci_intrs:<ivec#>:<nexus>:ihs
152#
153
154sub getstat($)
155{
156	my ($ks) = @_;
157
158	my $cpucnt = 0;
159	my %stat = ();
160	my ($minsnap, $maxsnap);
161
162	# kstats are not generated atomically. Each kstat hierarchy will
163	# have been generated within the kernel at a different time. On a
164	# thrashing system, we may not run quickly enough in order to get
165	# coherent kstat timing information across all the kstats. To
166	# determine if this is occurring, $minsnap/$maxsnap are used to
167	# find the breadth between the first and last snaptime of all the
168	# kstats we access. $maxsnap - $minsnap roughly represents the
169	# total time taken up in getstat(). If this time approaches the
170	# time between snapshots, our results may not be useful.
171
172	$minsnap = -1;		# snaptime is always a positive number
173	$maxsnap = $minsnap;
174
175	# Iterate over the cpus in cpu:<cpuid>::. Check
176	# cpu_info:<cpuid>:cpu_info<cpuid>:state to make sure the
177	# processor is "on-line". If not, it isn't accepting interrupts
178	# and doesn't concern us.
179	#
180	# Record cpu:<cpuid>:sys:snaptime, and check $minsnap/$maxsnap.
181
182	while (my ($cpu, $cpst) = each %{$ks->{cpu}}) {
183		next if !exists($ks->{cpu_info}{$cpu}{"cpu_info$cpu"}{state});
184		my $state = $ks->{cpu_info}{$cpu}{"cpu_info$cpu"}{state};
185		next if ($state !~ /^on-line\0/);
186		my $cpu_sys = $cpst->{sys};
187
188		$stat{$cpu}{tot} = ($cpu_sys->{cpu_nsec_idle} +
189				    $cpu_sys->{cpu_nsec_user} +
190				    $cpu_sys->{cpu_nsec_kernel});
191		$stat{$cpu}{crtime} = $cpu_sys->{crtime};
192		$stat{$cpu}{ivecs} = {};
193
194		if ($minsnap == -1 || $cpu_sys->{snaptime} < $minsnap) {
195			$minsnap = $cpu_sys->{snaptime};
196		}
197		if ($cpu_sys->{snaptime} > $maxsnap) {
198			$maxsnap = $cpu_sys->{snaptime};
199		}
200		$cpucnt++;
201	}
202
203	if ($cpucnt <= 1) {
204		$sleeptime = $onecpu_sleeptime;
205		return (0);	# nothing to do with 1 CPU
206	}
207
208	# Iterate over the ivecs. If the cpu is not on-line, ignore the
209	# ivecs mapped to it, if any.
210	#
211	# Record pci_intrs:{inum}:<nexus>:time, snaptime, crtime, pil,
212	# ino, name, and buspath. Check $minsnap/$maxsnap.
213
214	foreach my $inst (values(%{$ks->{pci_intrs}})) {
215		my $intrcfg = (values(%$inst))[0];
216		my $cpu = $intrcfg->{cpu};
217
218		next unless exists $stat{$cpu};
219		next if ($intrcfg->{type} =~ /^disabled\0/);
220
221		if ($intrcfg->{snaptime} < $minsnap) {
222			$minsnap = $intrcfg->{snaptime};
223		} elsif ($intrcfg->{snaptime} > $maxsnap) {
224			$maxsnap = $intrcfg->{snaptime};
225		}
226
227		my $cookie = "$intrcfg->{buspath} $intrcfg->{ino}";
228		if (exists $stat{$cpu}{ivecs}{$cookie}) {
229			my $cookiestats = $stat{$cpu}{ivecs}{$cookie};
230
231			$cookiestats->{time} += $intrcfg->{time};
232			$cookiestats->{name} .= "/$intrcfg->{name}";
233
234			# If this new interrupt sharing $cookie represents a
235			# change from an earlier getstat, make sure that
236			# generate_delta will see the change by setting
237			# crtime to the most recent crtime of its components.
238
239			if ($intrcfg->{crtime} > $cookiestats->{crtime}) {
240				$cookiestats->{crtime} = $intrcfg->{crtime};
241			}
242			$cookiestats->{ihs}++;
243			next;
244		}
245		$stat{$cpu}{ivecs}{$cookie}{time} = $intrcfg->{time};
246		$stat{$cpu}{ivecs}{$cookie}{crtime} = $intrcfg->{crtime};
247		$stat{$cpu}{ivecs}{$cookie}{pil} = $intrcfg->{pil};
248		$stat{$cpu}{ivecs}{$cookie}{ino} = $intrcfg->{ino};
249		$stat{$cpu}{ivecs}{$cookie}{buspath} = $intrcfg->{buspath};
250		$stat{$cpu}{ivecs}{$cookie}{name} = $intrcfg->{name};
251		$stat{$cpu}{ivecs}{$cookie}{ihs} = 1;
252	}
253
254	# We define the timerange as the amount of time spent gathering the
255	# various kstats, divided by our sleeptime. If we take a lot of time
256	# to access the kstats, and then we create a delta comparing these
257	# kstats with a prior set of kstats, that delta will cover
258	# substaintially different amount of time depending upon which
259	# interrupt or CPU is being examined.
260	#
261	# By checking the timerange here, we guarantee that any deltas
262	# created from these kstats will contain self-consistent data,
263	# in that all CPUs and interrupts cover a similar span of time.
264	#
265	# We attempt to keep this timerange between $timerange_lothresh and
266	# $timerange_hithresh. If the timerange gets too large, not only are
267	# there the accuracy concerns above, but it means that intrd is using
268	# a lot of CPU time. If the timerange gets too small, that means our
269	# sleep time is large, and we could fail to react quickly enough to a
270	# sudden change.
271	#
272	# Finally, $timerange_toohi is the upper bound. Any timerange above
273	# this is thrown out as garbage. If the stat is safely within this
274	# bound, we treat the stat as representing an instant in time, rather
275	# than the time range it actually spans. We arbitrarily choose minsnap
276	# as the snaptime of the stat.
277
278	$stat{snaptime} = $minsnap;
279	my $timerange = ($maxsnap - $minsnap) / $sleeptime;
280	if ($sleeptime == $onecpu_sleeptime) {
281		$sleeptime = $min_sleeptime; # time to come out of idling
282	} elsif ($timerange > $timerange_hithresh &&
283	    $sleeptime < $max_sleeptime) {
284		$sleeptime++;
285	} elsif ($timerange < $timerange_lothresh &&
286	    $sleeptime > $min_sleeptime) {
287		$sleeptime--;
288	}
289	return (0) if ($timerange > $timerange_toohi);	# i.e. failure
290	return (\%stat);
291}
292
293#
294# dumpdelta takes a reference to our "delta" structure:
295# {"missing"}           "1" if the delta's component stats had inconsistencies
296# {"minsnap"}           time of the first kstat snaptime used in this delta
297# {"maxsnap"}           time of the last kstat snaptime used in this delta
298# {"goodness"}          cost function applied to this delta
299# {"avgintrload"}       avg of interrupt load across cpus, as a percentage
300# {"avgintrnsec"}       avg number of nsec spent in interrupts, per cpu
301# {<cpuid>}             iterates over on-line cpus
302#  ->{"intrs"}          cpu's movable intr time (sum of "time" for each ivec)
303#  ->{"tot"}            CPU load from all sources
304#  ->{"bigintr"}        largest value of {ivecs}{<ivec#>}{time} from below
305#  ->{"intrload"}       intrs / tot
306#  ->{"ivecs"}
307#     ->{<ivec#>}       iterates over ivecs for this cpu
308#        ->{"time"}     time used by this interrupt (in nsec)
309#        ->{"pil"}      pil level of this interrupt
310#        ->{"ino"}      interrupt number
311#        ->{"buspath"}  filename of the directory of the device's bus
312#        ->{"name"}     device name
313#        ->{"ihs"}      number of different handlers sharing this ino
314#
315# It prints out the delta structure in a nice, human readable display.
316#
317
318sub dumpdelta($)
319{
320	my ($delta) = @_;
321
322	# print global info
323
324	syslog('debug', "dumpdelta:");
325	syslog('debug', " RECONFIGURATION IN DELTA") if $delta->{missing} > 0;
326	syslog('debug', " avgintrload: %5.2f%%  avgintrnsec: %d",
327	       $delta->{avgintrload} * 100, $delta->{avgintrnsec});
328	syslog('debug', "    goodness: %5.2f%%", $delta->{goodness} * 100)
329	    if exists($delta->{goodness});
330
331	# iterate over cpus
332
333	while (my ($cpu, $cpst) = each %$delta) {
334		next if !ref($cpst);		# skip non-cpuid entries
335		my $tot = $cpst->{tot};
336		syslog('debug', "    cpu %3d intr %7.3f%%  (bigintr %7.3f%%)",
337		       $cpu, $cpst->{intrload}*100, $cpst->{bigintr}*100/$tot);
338		syslog('debug', "        intrs %d, bigintr %d",
339		       $cpst->{intrs}, $cpst->{bigintr});
340
341		# iterate over ivecs on this cpu
342
343		while (my ($ivec, $ivst) = each %{$cpst->{ivecs}}) {
344			syslog('debug', "    %15s:\"%s\": %7.3f%%  %d",
345			    ($ivst->{ihs} > 1 ? "$ivst->{name}($ivst->{ihs})" :
346			    $ivst->{name}), $ivec,
347			    $ivst->{time}*100 / $tot, $ivst->{time});
348		}
349	}
350}
351
352#
353# generate_delta($stat, $newstat) takes two stat references, returned from
354# getstat(), and creates a %delta. %delta (not surprisingly) contains the
355# same basic info as stat and newstat, but with the timestamps as deltas
356# instead of absolute times. We return a reference to the delta.
357#
358
359sub generate_delta($$)
360{
361	my ($stat, $newstat) = @_;
362
363	my %delta = ();
364	my $intrload;
365	my $intrnsec;
366	my $cpus;
367
368	# Take the worstcase timerange
369	$delta{minsnap} = $stat->{snaptime};
370	$delta{maxsnap} = $newstat->{snaptime};
371	if (VERIFY($delta{maxsnap} > $delta{minsnap},
372	    "generate_delta: stats aren't ascending")) {
373		$delta{missing} = 1;
374		return (\%delta);
375	}
376
377	# if there are a different number of cpus in the stats, set missing
378
379	$delta{missing} = (keys(%$stat) != keys(%$newstat));
380	if (VERIFY($delta{missing} == 0,
381	    "generate_delta: number of CPUs changed")) {
382		return (\%delta);
383	}
384
385	# scan through every cpu in %newstat and compare against %stat
386
387	while (my ($cpu, $newcpst) = each %$newstat) {
388		next if !ref($newcpst);		# skip non-cpuid fields
389
390		# If %stat is missing a cpu from %newstat, then it was just
391		# onlined. Mark missing.
392
393		if (VERIFY(exists $stat->{$cpu} &&
394		    $stat->{$cpu}{crtime} == $newcpst->{crtime},
395		    "generate_delta: cpu $cpu changed")) {
396			$delta{missing} = 1;
397			return (\%delta);
398		}
399		my $cpst = $stat->{$cpu};
400		$delta{$cpu}{tot} = $newcpst->{tot} - $cpst->{tot};
401		if (VERIFY($delta{$cpu}{tot} >= 0,
402		    "generate_delta: deltas are not ascending?")) {
403			$delta{missing} = 1;
404			delete($delta{$cpu});
405			return (\%delta);
406		}
407		# Avoid remote chance of division by zero
408		$delta{$cpu}{tot} = 1 if $delta{$cpu}{tot} == 0;
409		$delta{$cpu}{intrs} = 0;
410		$delta{$cpu}{bigintr} = 0;
411
412		my %ivecs = ();
413		$delta{$cpu}{ivecs} = \%ivecs;
414
415		# if the number of ivecs differs, set missing
416
417		if (VERIFY(keys(%{$cpst->{ivecs}}) ==
418			   keys(%{$newcpst->{ivecs}}),
419			   "generate_delta: cpu $cpu has more/less".
420			   " interrupts")) {
421			$delta{missing} = 1;
422			return (\%delta);
423		}
424
425		while (my ($inum, $newivec) = each %{$newcpst->{ivecs}}) {
426			# If this ivec doesn't exist in $stat, or if $stat
427			# shows a different crtime, set missing.
428
429			if (VERIFY(exists $cpst->{ivecs}{$inum} &&
430				   $cpst->{ivecs}{$inum}{crtime} ==
431				   $newivec->{crtime},
432				   "generate_delta: cpu $cpu inum $inum".
433				   " has changed")) {
434				$delta{missing} = 1;
435				return (\%delta);
436			}
437			my $ivec = $cpst->{ivecs}{$inum};
438
439			# Create $delta{$cpu}{ivecs}{$inum}.
440
441			my %dltivec = ();
442			$delta{$cpu}{ivecs}{$inum} = \%dltivec;
443
444			# calculate time used by this interrupt
445
446			my $time = $newivec->{time} - $ivec->{time};
447			if (VERIFY($time >= 0,
448				   "generate_delta: ivec went backwards?")) {
449				$delta{missing} = 1;
450				delete($delta{$cpu}{ivecs}{$inum});
451				return (\%delta);
452			}
453			$delta{$cpu}{intrs} += $time;
454			$dltivec{time} = $time;
455			if ($time > $delta{$cpu}{bigintr}) {
456				$delta{$cpu}{bigintr} = $time;
457			}
458
459			# Transfer over basic info about the kstat. We
460			# don't have to worry about discrepancies between
461			# ivec and newivec because we verified that both
462			# have the same crtime.
463
464			$dltivec{pil} = $newivec->{pil};
465			$dltivec{ino} = $newivec->{ino};
466			$dltivec{buspath} = $newivec->{buspath};
467			$dltivec{name} = $newivec->{name};
468			$dltivec{ihs} = $newivec->{ihs};
469		}
470		if ($delta{$cpu}{tot} < $delta{$cpu}{intrs}) {
471			# Ewww! Hopefully just a rounding error.
472			# Make something up.
473			$delta{$cpu}{tot} = $delta{$cpu}{intrs};
474		}
475		$delta{$cpu}{intrload} =
476		       $delta{$cpu}{intrs} / $delta{$cpu}{tot};
477		$intrload += $delta{$cpu}{intrload};
478		$intrnsec += $delta{$cpu}{intrs};
479		$cpus++;
480	}
481	if ($cpus > 0) {
482		$delta{avgintrload} = $intrload / $cpus;
483		$delta{avgintrnsec} = $intrnsec / $cpus;
484	} else {
485		$delta{avgintrload} = 0;
486		$delta{avgintrnsec} = 0;
487	}
488	return (\%delta);
489}
490
491
492# compress_delta takes a list of deltas, and returns a single new delta
493# which represents the combined information from all the deltas. The deltas
494# provided are assumed to be sequential in time. The resulting compressed
495# delta looks just like any other delta. This new delta is also more accurate
496# since its statistics are averaged over a longer period than any of the
497# original deltas.
498
499sub compress_deltas ($)
500{
501	my ($deltas) = @_;
502
503	my %newdelta = ();
504	my ($intrs, $tot);
505	my $cpus = 0;
506
507	if (VERIFY($#$deltas != -1,
508		   "compress_deltas: list of delta is empty?")) {
509		return (0);
510	}
511	$newdelta{minsnap} = $deltas->[0]{minsnap};
512	$newdelta{maxsnap} = $deltas->[$#$deltas]{maxsnap};
513	$newdelta{missing} = 0;
514
515	foreach my $delta (@$deltas) {
516		if (VERIFY($delta->{missing} == 0,
517		    "compressing bad deltas?")) {
518			return (0);
519		}
520		while (my ($cpuid, $cpu) = each %$delta) {
521			next if !ref($cpu);
522
523			$intrs += $cpu->{intrs};
524			$tot += $cpu->{tot};
525			$newdelta{$cpuid}{intrs} += $cpu->{intrs};
526			$newdelta{$cpuid}{tot} += $cpu->{tot};
527			if (!exists $newdelta{$cpuid}{ivecs}) {
528				my %ivecs = ();
529				$newdelta{$cpuid}{ivecs} = \%ivecs;
530			}
531			while (my ($inum, $ivec) = each %{$cpu->{ivecs}}) {
532				my $newivecs = $newdelta{$cpuid}{ivecs};
533				$newivecs->{$inum}{time} += $ivec->{time};
534				$newivecs->{$inum}{pil} = $ivec->{pil};
535				$newivecs->{$inum}{ino} = $ivec->{ino};
536				$newivecs->{$inum}{buspath} = $ivec->{buspath};
537				$newivecs->{$inum}{name} = $ivec->{name};
538				$newivecs->{$inum}{ihs} = $ivec->{ihs};
539			}
540		}
541	}
542	foreach my $cpu (values(%newdelta)) {
543		next if !ref($cpu); # ignore non-cpu fields
544		$cpus++;
545
546		my $bigintr = 0;
547		foreach my $ivec (values(%{$cpu->{ivecs}})) {
548			if ($ivec->{time} > $bigintr) {
549				$bigintr = $ivec->{time};
550			}
551		}
552		$cpu->{bigintr} = $bigintr;
553		$cpu->{intrload} = $cpu->{intrs} / $cpu->{tot};
554		$cpu->{tot} = 1 if $cpu->{tot} <= 0;
555	}
556	if ($cpus == 0) {
557		$newdelta{avgintrnsec} = 0;
558		$newdelta{avgintrload} = 0;
559	} else {
560		$newdelta{avgintrnsec} = $intrs / $cpus;
561		$newdelta{avgintrload} = $intrs / $tot;
562	}
563	return (\%newdelta);
564}
565
566
567
568
569
570# What follow are the core functions responsible for examining the deltas
571# generated above and deciding what to do about them.
572#
573# goodness() and its helper goodness_cpu() return a heuristic which describe
574# how good (or bad) the current interrupt balance is. The value returned will
575# be between 0 and 1, with 0 representing maximum goodness, and 1 representing
576# maximum badness.
577#
578# imbalanced() compares a current and historical value of goodness, and
579# determines if there has been enough change to warrant evaluating a
580# reconfiguration of the interrupts
581#
582# do_reconfig(), and its helpers, do_reconfig_cpu(), do_reconfig_cpu2cpu(),
583# find_goal(), do_find_goal(), and move_intr(), are responsible for examining
584# a delta and determining the best possible assignment of interrupts to CPUs.
585#
586# It is important that do_reconfig() be in alignment with goodness(). If
587# do_reconfig were to generate a new interrupt distribution that worsened
588# goodness, we could get into a pathological loop with intrd fighting itself,
589# constantly deciding that things are imbalanced, and then changing things
590# only to make them worse.
591
592
593
594# any goodness over $goodness_unsafe_load is considered really bad
595# goodness must drop by at least $goodness_mindelta for a reconfig
596
597my $goodness_unsafe_load = .9;
598my $goodness_mindelta = .1;
599
600# goodness(%delta) examines a delta and return its "goodness". goodness will
601# be between 0 (best) and 1 (major bad). goodness is determined by evaluating
602# the goodness of each individual cpu, and returning the worst case. This
603# helps on systems with many CPUs, where otherwise a single pathological CPU
604# might otherwise be ignored because the average was OK.
605#
606# To calculate the goodness of an individual CPU, we start by looking at its
607# load due to interrupts. If the load is above a certain high threshold and
608# there is more than one interrupt assigned to this CPU, we set goodness
609# to worst-case. If the load is below the average interrupt load of all CPUs,
610# then we return best-case, since what's to complain about?
611#
612# Otherwise we look at how much the load is above the average, and return
613# that as the goodness, with one caveat: we never return more than the CPU's
614# interrupt load ignoring its largest single interrupt source. This is
615# because a CPU with one high-load interrupt, and no other interrupts, is
616# perfectly balanced. Nothing can be done to improve the situation, and thus
617# it is perfectly balanced even if the interrupt's load is 100%.
618
619sub goodness($)
620{
621	my ($delta) = @_;
622
623	return (1) if $delta->{missing} > 0;
624
625	my $high_goodness = 0;
626	my $goodness;
627
628	foreach my $cpu (values(%$delta)) {
629		next if !ref($cpu);		# skip non-cpuid fields
630
631		$goodness = goodness_cpu($cpu, $delta->{avgintrload});
632		if (VERIFY($goodness >= 0 && $goodness <= 1,
633			   "goodness: cpu goodness out of range?")) {
634			dumpdelta($delta);
635			return (1);
636		}
637		if ($goodness == 1) {
638			return (1);	# worst case, no need to continue
639		}
640		if ($goodness > $high_goodness) {
641			$high_goodness = $goodness;
642		}
643	}
644	return ($high_goodness);
645}
646
647sub goodness_cpu($$)		# private function
648{
649	my ($cpu, $avgintrload) = @_;
650
651	my $goodness;
652	my $load = $cpu->{intrs} / $cpu->{tot};
653
654	return (0) if ($load < $avgintrload);	# low loads are perfectly good
655
656	# Calculate $load_no_bigintr, which represents the load
657	# due to interrupts, excluding the one biggest interrupt.
658	# This is the most gain we can get on this CPU from
659	# offloading interrupts.
660
661	my $load_no_bigintr = ($cpu->{intrs} - $cpu->{bigintr}) / $cpu->{tot};
662
663	# A major imbalance is indicated if a CPU is saturated
664	# with interrupt handling, and it has more than one
665	# source of interrupts. Those other interrupts could be
666	# starved if of a lower pil. Return a goodness of 1,
667	# which is the worst possible return value,
668	# which will effectively contaminate this entire delta.
669
670	my $cnt = keys(%{$cpu->{ivecs}});
671
672	if ($load > $goodness_unsafe_load && $cnt > 1) {
673		return (1);
674	}
675	$goodness = $load - $avgintrload;
676	if ($goodness > $load_no_bigintr) {
677		$goodness = $load_no_bigintr;
678	}
679	return ($goodness);
680}
681
682
683# imbalanced() is used by the main routine to determine if the goodness
684# has shifted far enough from our last baseline to warrant a reassignment
685# of interrupts. A very high goodness indicates that a CPU is way out of
686# whack. If the goodness has varied too much since the baseline, then
687# perhaps a reconfiguration is worth considering.
688
689sub imbalanced ($$)
690{
691	my ($goodness, $baseline) = @_;
692
693	# Return 1 if we are pathological, or creeping away from the baseline
694
695	return (1) if $goodness > .50;
696	return (1) if abs($goodness - $baseline) > $goodness_mindelta;
697	return (0);
698}
699
700# do_reconfig(), do_reconfig_cpu(), and do_reconfig_cpu2cpu(), are the
701# decision-making functions responsible for generating a new interrupt
702# distribution. They are designed with the definition of goodness() in
703# mind, i.e. they use the same definition of "good distribution" as does
704# goodness().
705#
706# do_reconfig() is responsible for deciding whether a redistribution is
707# actually warranted. If the goodness is already pretty good, it doesn't
708# waste the CPU time to generate a new distribution. If it
709# calculates a new distribution and finds that it is not sufficiently
710# improved from the prior distirbution, it will not do the redistribution,
711# mainly to avoid the disruption to system performance caused by
712# rejuggling interrupts.
713#
714# Its main loop works by going through a list of cpus sorted from
715# highest to lowest interrupt load. It removes the highest-load cpus
716# one at a time and hands them off to do_reconfig_cpu(). This function
717# then re-sorts the remaining CPUs from lowest to highest interrupt load,
718# and one at a time attempts to rejuggle interrupts between the original
719# high-load CPU and the low-load CPU. Rejuggling on a high-load CPU is
720# considered finished as soon as its interrupt load is within
721# $goodness_mindelta of the average interrupt load. Such a CPU will have
722# a goodness of below the $goodness_mindelta threshold.
723
724#
725# move_intr(\%delta, $inum, $oldcpu, $newcpu)
726# used by reconfiguration code to move an interrupt between cpus within
727# a delta. This manipulates data structures, and does not actually move
728# the interrupt on the running system.
729#
730sub move_intr($$$$)		# private function
731{
732	my ($delta, $inum, $oldcpuid, $newcpuid) = @_;
733
734	my $ivec = $delta->{$oldcpuid}{ivecs}{$inum};
735
736	# Remove ivec from old cpu
737
738	my $oldcpu = $delta->{$oldcpuid};
739	$oldcpu->{intrs} -= $ivec->{time};
740	$oldcpu->{intrload} = $oldcpu->{intrs} / $oldcpu->{tot};
741	delete($oldcpu->{ivecs}{$inum});
742
743	VERIFY($oldcpu->{intrs} >= 0, "move_intr: intr's time > total time?");
744	VERIFY($ivec->{time} <= $oldcpu->{bigintr},
745	       "move_intr: intr's time > bigintr?");
746
747	if ($ivec->{time} >= $oldcpu->{bigintr}) {
748		my $bigtime = 0;
749
750		foreach my $ivec (values(%{$oldcpu->{ivecs}})) {
751			$bigtime = $ivec->{time} if $ivec->{time} > $bigtime;
752		}
753		$oldcpu->{bigintr} = $bigtime;
754	}
755
756	# Add ivec onto new cpu
757
758	my $newcpu = $delta->{$newcpuid};
759
760	$ivec->{nowcpu} = $newcpuid;
761	$newcpu->{intrs} += $ivec->{time};
762	$newcpu->{intrload} = $newcpu->{intrs} / $newcpu->{tot};
763	$newcpu->{ivecs}{$inum} = $ivec;
764
765	$newcpu->{bigintr} = $ivec->{time}
766		if $ivec->{time} > $newcpu->{bigintr};
767}
768
769sub move_intr_check($$$)	# private function
770{
771	my ($delta, $oldcpuid, $newcpuid) = @_;
772
773	VERIFY($delta->{$oldcpuid}{tot} >= $delta->{$oldcpuid}{intrs},
774	       "Moved interrupts left 100+%% load on src cpu");
775	VERIFY($delta->{$newcpuid}{tot} >= $delta->{$newcpuid}{intrs},
776	       "Moved interrupts left 100+%% load on tgt cpu");
777}
778
779sub ivecs_to_string(@)		# private function
780{
781	my $str = "";
782	foreach my $ivec (@_) {
783		$str = "$str $ivec->{inum}";
784	}
785	return ($str);
786}
787
788
789sub do_reconfig($)
790{
791	my ($delta) = @_;
792
793	my $goodness = $delta->{goodness};
794
795	# We can't improve goodness to better than 0. We should stop here
796	# if, even if we achieve a goodness of 0, the improvement is still
797	# too small to merit the action.
798
799	if ($goodness - 0 < $goodness_mindelta) {
800		syslog('debug', "goodness good enough, don't reconfig");
801		return (0);
802	}
803
804	syslog('notice', "Optimizing interrupt assignments");
805
806	if (VERIFY ($delta->{missing} == 0, "RECONFIG Aborted: should not ".
807	    "have a delta with missing")) {
808		return (-1);
809	}
810
811	# Make a list of all cpuids, and also add some extra information
812	# to the ivec structures.
813
814	my @cpusortlist = ();
815
816	while (my ($cpuid, $cpu) = each %$delta) {
817		next if !ref($cpu);	# skip non-cpu entries
818
819		push(@cpusortlist, $cpuid);
820		while (my ($inum, $ivec) = each %{$cpu->{ivecs}}) {
821			$ivec->{origcpu} = $cpuid;
822			$ivec->{nowcpu} = $cpuid;
823			$ivec->{inum} = $inum;
824		}
825	}
826
827	# Sort the list of CPUs from highest to lowest interrupt load.
828	# Remove the top CPU from that list and attempt to redistribute
829	# its interrupts. If the CPU has a goodness below a threshold,
830	# just ignore the CPU and move to the next one. If the CPU's
831	# load falls below the average load plus that same threshold,
832	# then there are no CPUs left worth reconfiguring, and we're done.
833
834	while (@cpusortlist) {
835		# Re-sort cpusortlist each time, since do_reconfig_cpu can
836		# move interrupts around.
837
838		@cpusortlist =
839		    sort({$delta->{$b}{intrload} <=> $delta->{$a}{intrload}}
840		    @cpusortlist);
841
842		my $cpu = shift(@cpusortlist);
843		if (($delta->{$cpu}{intrload} <= $goodness_unsafe_load) &&
844		    ($delta->{$cpu}{intrload} <=
845		    $delta->{avgintrload} + $goodness_mindelta)) {
846			syslog('debug', "finished reconfig: cpu $cpu load ".
847			    "$delta->{$cpu}{intrload} avgload ".
848			    "$delta->{avgintrload}");
849			last;
850		}
851		if (goodness_cpu($delta->{$cpu}, $delta->{avgintrload}) <
852		    $goodness_mindelta) {
853			next;
854		}
855		do_reconfig_cpu($delta, \@cpusortlist, $cpu);
856	}
857
858	# How good a job did we do? If the improvement was minimal, and
859	# our goodness wasn't pathological (and thus needing any help it
860	# can get), then don't bother moving the interrupts.
861
862	my $newgoodness = goodness($delta);
863	VERIFY($newgoodness <= $goodness,
864	       "reconfig: result has worse goodness?");
865
866	if (($goodness != 1 || $newgoodness == 1) &&
867	    $goodness - $newgoodness < $goodness_mindelta) {
868		syslog('debug', "goodness already near optimum, ".
869		       "don't reconfig");
870		return (0);
871	}
872	syslog('debug', "goodness %5.2f%% --> %5.2f%%", $goodness*100,
873	       $newgoodness*100);
874
875	# Time to move those interrupts!
876
877	my $ret = 1;
878	my $warned = 0;
879	while (my ($cpuid, $cpu) = each %$delta) {
880		next if $cpuid =~ /\D/;
881		while (my ($inum, $ivec) = each %{$cpu->{ivecs}}) {
882			next if ($ivec->{origcpu} == $cpuid);
883
884			if (!intrmove($ivec->{buspath}, $ivec->{ino},
885			    $cpuid)) {
886				syslog('warning', "Unable to move interrupts")
887				    if $warned++ == 0;
888				syslog('debug', "Unable to move buspath ".
889				    "$ivec->{buspath} ino $ivec->{ino} to ".
890				    "cpu $cpuid");
891				$ret = -1;
892			}
893		}
894	}
895
896	syslog('notice', "Interrupt assignments optimized");
897	return ($ret);
898}
899
900sub do_reconfig_cpu($$$)	# private function
901{
902	my ($delta, $cpusortlist, $oldcpuid) = @_;
903
904	# We have been asked to rejuggle interrupts between $oldcpuid and
905	# other CPUs found on $cpusortlist so as to improve the load on
906	# $oldcpuid. We reverse $cpusortlist to get our own copy of the
907	# list, sorted from lowest to highest interrupt load. One at a
908	# time, shift a CPU off of this list of CPUs, and attempt to
909	# rejuggle interrupts between the two CPUs. Don't do this if the
910	# other CPU has a higher load than oldcpuid. We're done rejuggling
911	# once $oldcpuid's goodness falls below a threshold.
912
913	syslog('debug', "reconfiguring $oldcpuid");
914
915	my $cpu = $delta->{$oldcpuid};
916	my $avgintrload = $delta->{avgintrload};
917
918	my @cputargetlist = reverse(@$cpusortlist); # make a copy of the list
919	while ($#cputargetlist != -1) {
920 		last if goodness_cpu($cpu, $avgintrload) < $goodness_mindelta;
921
922		my $tgtcpuid = shift(@cputargetlist);
923		my $tgt = $delta->{$tgtcpuid};
924		my $load = $cpu->{intrload};
925		my $tgtload = $tgt->{intrload};
926		last if $tgtload > $load;
927		do_reconfig_cpu2cpu($delta, $oldcpuid, $tgtcpuid, $load);
928	}
929}
930
931sub do_reconfig_cpu2cpu($$$$)	# private function
932{
933	my ($delta, $srccpuid, $tgtcpuid, $srcload) = @_;
934
935	# We've been asked to consider interrupt juggling between srccpuid
936	# (with a high interrupt load) and tgtcpuid (with a lower interrupt
937	# load). First, make a single list with all of the ivecs from both
938	# CPUs, and sort the list from highest to lowest load.
939
940	syslog('debug', "exchanging intrs between $srccpuid and $tgtcpuid");
941
942	# Gather together all the ivecs and sort by load
943
944	my @ivecs = (values(%{$delta->{$srccpuid}{ivecs}}),
945	    values(%{$delta->{$tgtcpuid}{ivecs}}));
946	return if $#ivecs == -1;
947
948	@ivecs = sort({$b->{time} <=> $a->{time}} @ivecs);
949
950	# Our "goal" load for srccpuid is the average load across all CPUs.
951	# find_goal() will find determine the optimum selection of the
952	# available interrupts which comes closest to this goal without
953	# falling below the goal.
954
955	my $goal = $delta->{avgintrnsec};
956
957	# We know that the interrupt load on tgtcpuid is less than that on
958	# srccpuid, but its load could still be above avgintrnsec. Don't
959	# choose a goal which would bring srccpuid below the load on tgtcpuid.
960
961	my $avgnsec =
962	    ($delta->{$srccpuid}{intrs} + $delta->{$tgtcpuid}{intrs}) / 2;
963	if ($goal < $avgnsec) {
964		$goal = $avgnsec;
965	}
966
967	# If the largest of the interrupts is on srccpuid, leave it there.
968	# This can help minimize the disruption caused by moving interrupts.
969
970	if ($ivecs[0]->{origcpu} == $srccpuid) {
971		syslog('debug', "Keeping $ivecs[0]->{inum} on $srccpuid");
972		$goal -= $ivecs[0]->{time};
973		shift(@ivecs);
974	}
975
976	syslog('debug', "GOAL: inums should total $goal");
977	find_goal(\@ivecs, $goal);
978
979	# find_goal() returned its results to us by setting $ivec->{goal} if
980	# the ivec should be on srccpuid, or clearing it for tgtcpuid.
981	# Call move_intr() to update our $delta with the new results.
982
983	foreach my $ivec (@ivecs) {
984		syslog('debug', "ivec $ivec->{inum} goal $ivec->{goal}");
985		VERIFY($ivec->{nowcpu} == $srccpuid ||
986		    $ivec->{nowcpu} == $tgtcpuid, "cpu2cpu found an ".
987		    "interrupt not currently on src or tgt cpu");
988
989		if ($ivec->{goal} && $ivec->{nowcpu} != $srccpuid) {
990			move_intr($delta, $ivec->{inum}, $ivec->{nowcpu},
991			    $srccpuid);
992		} elsif ($ivec->{goal} == 0 && $ivec->{nowcpu} != $tgtcpuid) {
993			move_intr($delta, $ivec->{inum}, $ivec->{nowcpu},
994			    $tgtcpuid);
995		}
996	}
997	move_intr_check($delta, $srccpuid, $tgtcpuid); # asserts
998
999	my $newload = $delta->{$srccpuid}{intrs} / $delta->{$srccpuid}{tot};
1000	VERIFY($newload <= $srcload && $newload > $delta->{avgintrload},
1001	    "cpu2cpu: new load didn't end up in expected range");
1002}
1003
1004
1005# find_goal() and its helper do_find_goal() are used to find the best
1006# combination of interrupts in order to generate a load that is as close
1007# as possible to a goal load without falling below that goal. Before returning
1008# to its caller, find_goal() sets a new value in the hash of each interrupt,
1009# {goal}, which if set signifies that this interrupt is one of the interrupts
1010# identified as part of the set of interrupts which best meet the goal.
1011#
1012# The arguments to find_goal are a list of ivecs (hash references), sorted
1013# by descending {time}, and the goal load. The goal is relative to {time}.
1014# The best fit is determined by performing a depth-first search. do_find_goal
1015# is the recursive subroutine which carries out the search.
1016#
1017# It is passed an index as an argument, originally 0. On a given invocation,
1018# it is only to consider interrupts in the ivecs array starting at that index.
1019# It then considers two possibilities:
1020#   1) What is the best goal-fit if I include ivecs[index]?
1021#   2) What is the best goal-fit if I exclude ivecs[index]?
1022# To determine case 1, it subtracts the load of ivecs[index] from the goal,
1023# and calls itself recursively with that new goal and index++.
1024# To determine case 2, it calls itself recursively with the same goal and
1025# index++.
1026#
1027# It then compares the two results, decide which one best meets the goals,
1028# and returns the result. The return value is the best-fit's interrupt load,
1029# followed by a list of all the interrupts which make up that best-fit.
1030#
1031# As an optimization, a second array loads[] is created which mirrors ivecs[].
1032# loads[i] will equal the total loads of all ivecs[i..$#ivecs]. This is used
1033# by do_find_goal to avoid recursing all the way to the end of the ivecs
1034# array if including all remaining interrupts will still leave the best-fit
1035# at below goal load. If so, it then includes all remaining interrupts on
1036# the goal list and returns.
1037#
1038sub find_goal($$)		# private function
1039{
1040	my ($ivecs, $goal) = @_;
1041
1042	my @goals;
1043	my $load;
1044	my $ivec;
1045
1046	if ($goal <= 0) {
1047		@goals = ();	# the empty set will best meet the goal
1048	} else {
1049		syslog('debug', "finding goal from intrs %s",
1050		    ivecs_to_string(@$ivecs));
1051
1052		# Generate @loads array
1053
1054		my $tot = 0;
1055		foreach $ivec (@$ivecs) {
1056			$tot += $ivec->{time};
1057		}
1058		my @loads = ();
1059		foreach $ivec (@$ivecs) {
1060			push(@loads, $tot);
1061			$tot -= $ivec->{time};
1062		}
1063		($load, @goals) = do_find_goal($ivecs, \@loads, $goal, 0);
1064		VERIFY($load >= $goal, "find_goal didn't meet goals");
1065	}
1066	syslog('debug', "goals found: %s", ivecs_to_string(@goals));
1067
1068	# Set or clear $ivec->{goal} for each ivec, based on returned @goals
1069
1070	foreach $ivec (@$ivecs) {
1071		if ($#goals > -1 && $ivec == $goals[0]) {
1072			syslog('debug', "inum $ivec->{inum} on source cpu");
1073			$ivec->{goal} = 1;
1074			shift(@goals);
1075		} else {
1076			syslog('debug', "inum $ivec->{inum} on target cpu");
1077			$ivec->{goal} = 0;
1078		}
1079	}
1080}
1081
1082
1083sub do_find_goal($$$$)		# private function
1084{
1085	my ($ivecs, $loads, $goal, $idx) = @_;
1086
1087	if ($idx > $#{$ivecs}) {
1088		return (0);
1089	}
1090	syslog('debug', "$idx: finding goal $goal inum $ivecs->[$idx]{inum}");
1091
1092	my $load = $ivecs->[$idx]{time};
1093	my @goals_with = ();
1094	my @goals_without = ();
1095	my ($with, $without);
1096
1097	# If we include all remaining items and we're still below goal,
1098	# stop here. We can just return a result that includes $idx and all
1099	# subsequent ivecs. Since this will still be below goal, there's
1100	# nothing better to be done.
1101
1102	if ($loads->[$idx] <= $goal) {
1103		syslog('debug',
1104		    "$idx: including all remaining intrs %s with load %d",
1105		    ivecs_to_string(@$ivecs[$idx .. $#{$ivecs}]),
1106		    $loads->[$idx]);
1107		return ($loads->[$idx], @$ivecs[$idx .. $#{$ivecs}]);
1108	}
1109
1110	# Evaluate the "with" option, i.e. the best matching goal which
1111	# includes $ivecs->[$idx]. If idx's load is more than our goal load,
1112	# stop here. Once we're above the goal, there is no need to consider
1113	# further interrupts since they'll only take us further from the goal.
1114
1115	if ($goal <= $load) {
1116		$with = $load;	# stop here
1117	} else {
1118		($with, @goals_with) =
1119		    do_find_goal($ivecs, $loads, $goal - $load, $idx + 1);
1120		$with += $load;
1121	}
1122	syslog('debug', "$idx: with-load $with intrs %s",
1123	       ivecs_to_string($ivecs->[$idx], @goals_with));
1124
1125	# Evaluate the "without" option, i.e. the best matching goal which
1126	# excludes $ivecs->[$idx].
1127
1128	($without, @goals_without) =
1129	    &do_find_goal($ivecs, $loads, $goal, $idx + 1);
1130	syslog('debug', "$idx: without-load $without intrs %s",
1131	       ivecs_to_string(@goals_without));
1132
1133	# We now have our "with" and "without" options, and we choose which
1134	# best fits the goal. If one is greater than goal and the other is
1135	# below goal, we choose the one that is greater. If they are both
1136	# below goal, then we choose the one that is greater. If they are
1137	# both above goal, then we choose the smaller.
1138
1139	my $which;		# 0 == with, 1 == without
1140	if ($with >= $goal && $without < $goal) {
1141		$which = 0;
1142	} elsif ($with < $goal && $without >= $goal) {
1143		$which = 1;
1144	} elsif ($with >= $goal && $without >= $goal) {
1145		$which = ($without < $with);
1146	} else {
1147		$which = ($without > $with);
1148	}
1149
1150	# Return the load of our best case scenario, followed by all the ivecs
1151	# which compose that goal.
1152
1153	if ($which == 1) {	# without
1154		syslog('debug', "$idx: going without");
1155		return ($without, @goals_without);
1156	} else {
1157		syslog('debug', "$idx: going with");
1158		return ($with, $ivecs->[$idx], @goals_with);
1159	}
1160	# Not reached
1161}
1162
1163
1164
1165
1166syslog('debug', "intrd is starting".($debug ? " (debug)" : ""));
1167
1168my @deltas = ();
1169my $deltas_tottime = 0;		# sum of maxsnap-minsnap across @deltas
1170my $avggoodness;
1171my $baseline_goodness = 0;
1172my $compdelta;
1173
1174my $do_reconfig;
1175
1176# temp variables
1177my $goodness;
1178my $deltatime;
1179my $olddelta;
1180my $olddeltatime;
1181my $delta;
1182my $newstat;
1183my $below_statslen;
1184my $newtime;
1185my $ret;
1186
1187
1188my $gotsig = 0;
1189$SIG{INT} = sub { $gotsig = 1; };     # don't die in the middle of retargeting
1190$SIG{HUP} = $SIG{INT};
1191$SIG{TERM} = $SIG{INT};
1192
1193my $ks;
1194if ($using_scengen == 0) {
1195	$ks = Sun::Solaris::Kstat->new();
1196} else {
1197	$ks = myks_update();	# supplied by the simulator
1198}
1199
1200# If no pci_intrs kstats were found, we need to exit, but we can't because
1201# SMF will restart us and/or report an error to the administrator. But
1202# there's nothing an administrator can do. So print out a message for SMF
1203# logs and silently pause forever.
1204
1205if (!exists($ks->{pci_intrs})) {
1206	print STDERR "$cmdname: no interrupts were found; ".
1207	    "your PCI bus may not yet be supported\n";
1208	pause() while $gotsig == 0;
1209	exit 0;
1210}
1211
1212my $stat = getstat($ks);
1213
1214
1215
1216for (;;) {
1217	sub clear_deltas {
1218		@deltas = ();
1219		$deltas_tottime = 0;
1220		$stat = 0;   # prevent next gen_delta() from setting {missing}
1221	}
1222
1223	# 1. Sleep, update the kstats, and save the new stats in $newstat.
1224
1225	exit 0 if $gotsig;		# if we got ^C / SIGTERM, exit
1226	if ($using_scengen == 0) {
1227		sleep($sleeptime);
1228		exit 0 if $gotsig;	# if we got ^C / SIGTERM, exit
1229		$ks->update();
1230	} else {
1231		$ks = myks_update();
1232	}
1233	$newstat = getstat($ks);
1234
1235	# $stat or $newstat could be zero if they're uninitialized, or if
1236	# getstat() failed. If $stat is zero, move $newstat to $stat, sleep
1237	# and try again. If $newstat is zero, then we also sleep and try
1238	# again, hoping the problem will clear up.
1239
1240	next if (!ref $newstat);
1241	if (!ref $stat) {
1242		$stat = $newstat;
1243		next;
1244	}
1245
1246
1247	# 2. Compare $newstat with the prior set of values, result in %$delta.
1248
1249	$delta = generate_delta($stat, $newstat);
1250	dumpdelta($delta) if $debug;	# Dump most recent stats to stdout.
1251	$stat = $newstat;	# The new stats now become the old stats.
1252
1253
1254	# 3. If $delta->{missing}, then there has been a reconfiguration of
1255	# either cpus or interrupts (probably both). We need to toss out our
1256	# old set of statistics and start from scratch.
1257	#
1258	# Also, if the delta covers a very long range of time, then we've
1259	# been experiencing a system overload that has resulted in intrd
1260	# not being allowed to run effectively for a while now. As above,
1261	# toss our old statistics and start from scratch.
1262
1263	$deltatime = $delta->{maxsnap} - $delta->{minsnap};
1264	if ($delta->{missing} > 0 || $deltatime > $statslen) {
1265		clear_deltas();
1266		syslog('debug', "evaluating interrupt assignments");
1267		next;
1268	}
1269
1270
1271	# 4. Incorporate new delta into the list of deltas, and associated
1272	# statistics. If we've just now received $statslen deltas, then it's
1273	# time to evaluate a reconfiguration.
1274
1275	$below_statslen = ($deltas_tottime < $statslen);
1276	$deltas_tottime += $deltatime;
1277	$do_reconfig = ($below_statslen && $deltas_tottime >= $statslen);
1278	push(@deltas, $delta);
1279
1280	# 5. Remove old deltas if total time is more than $statslen. We use
1281	# @deltas as a moving average of the last $statslen seconds. Shift
1282	# off the olders deltas, but only if that doesn't cause us to fall
1283	# below $statslen seconds.
1284
1285	while (@deltas > 1) {
1286		$olddelta = $deltas[0];
1287		$olddeltatime = $olddelta->{maxsnap} - $olddelta->{minsnap};
1288		$newtime = $deltas_tottime - $olddeltatime;
1289		last if ($newtime < $statslen);
1290
1291		shift(@deltas);
1292		$deltas_tottime = $newtime;
1293	}
1294
1295	# 6. The brains of the operation are here. First, check if we're
1296	# imbalanced, and if so set $do_reconfig. If $do_reconfig is set,
1297	# either because of imbalance or above in step 4, we evaluate a
1298	# new configuration.
1299	#
1300	# First, take @deltas and generate a single "compressed" delta
1301	# which summarizes them all. Pass that to do_reconfig and see
1302	# what it does with it:
1303	#
1304	# $ret == -1 : failure
1305	# $ret ==  0 : current config is optimal (or close enough)
1306	# $ret ==  1 : reconfiguration has occurred
1307	#
1308	# If $ret is -1 or 1, dump all our deltas and start from scratch.
1309	# Step 4 above will set do_reconfig soon thereafter.
1310	#
1311	# If $ret is 0, then nothing has happened because we're already
1312	# good enough. Set baseline_goodness to current goodness.
1313
1314	$compdelta = compress_deltas(\@deltas);
1315	if (VERIFY(ref($compdelta) eq "HASH", "couldn't compress deltas")) {
1316		clear_deltas();
1317		next;
1318	}
1319	$compdelta->{goodness} = goodness($compdelta);
1320	dumpdelta($compdelta) if $debug;
1321
1322	$goodness = $compdelta->{goodness};
1323	syslog('debug', "GOODNESS: %5.2f%%", $goodness * 100);
1324
1325	if ($deltas_tottime >= $statslen &&
1326	    imbalanced($goodness, $baseline_goodness)) {
1327		$do_reconfig = 1;
1328	}
1329
1330	if ($do_reconfig) {
1331		$ret = do_reconfig($compdelta);
1332
1333		if ($ret != 0) {
1334			clear_deltas();
1335			syslog('debug', "do_reconfig FAILED!") if $ret == -1;
1336		} else {
1337			syslog('debug', "setting new baseline of $goodness");
1338			$baseline_goodness = $goodness;
1339		}
1340	}
1341	syslog('debug', "---------------------------------------");
1342}
1343