xref: /illumos-gate/usr/src/uts/common/os/softint.c (revision d96925c4)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/param.h>
28 #include <sys/t_lock.h>
29 #include <sys/systm.h>
30 #include <sys/spl.h>
31 #include <sys/cmn_err.h>
32 #include <sys/debug.h>
33 #include <sys/kdi_impl.h>
34 #include <sys/cpuvar.h>
35 #include <sys/cpuvar.h>
36 #include <sys/archsystm.h>
37 
38 /*
39  * Handle software interrupts through 'softcall' mechanism
40  *
41  * At present softcall mechanism uses a global list headed by softhead.
42  * Entries are added to tail and removed from head so as to preserve FIFO
43  * nature of entries in the softcall list. softcall() takes care of adding
44  * entries to the softtail.
45  *
46  * softint must take care of executing the entries in the FIFO
47  * order. It could be called simultaneously from multiple cpus, however only
48  * one instance of softint should process the softcall list with the exception
49  * when CPU is stuck due to high interrupt load and can't execute callbacks.
50  * State diagram is as follows :-
51  *
52  *	- Upper half which is same as old state machine
53  *	  (IDLE->PEND->DRAIN->IDLE)
54  *
55  *	- Lower half which steals the entries from softcall queue and execute
56  *        in the context of softint interrupt handler. The interrupt handler
57  *        is fired on a different CPU by sending a cross-call.
58  *
59  * Starting state is IDLE.
60  *
61  *				softint()
62  *
63  *
64  *				(c)
65  *	____________________________________________________
66  *	|			   ^			     ^
67  *	v	     (a)	   |	       (b)	     |
68  *	IDLE--------------------->PEND--------------------->DRAIN
69  *	^			  |			    |
70  *	|			  |			    |
71  *	|			  |			    |
72  *	|			  |			    |
73  *	|			  |			    |
74  *	|			  d			    d
75  *	|			  |			    |
76  *	|			  v			    v
77  *	|			  PEND			    DRAIN
78  *	|	     (e)	   &			      &
79  *	|<-----------------------STEAL			    STEAL
80  *	^						     |
81  *	|						     |
82  *	|			  (e)			     v
83  *	|_________________________<__________________________|
84  *
85  *
86  *
87  * Edge (a)->(b)->(c) are same as old state machine and these
88  * are mutually exclusive state.
89  *
90  * a - When an entry is being enqueued to softcall queue then the state
91  *     moves from IDLE to PEND.
92  *
93  * b - When interrupt handler has started processing softcall queue.
94  *
95  * c - When interrupt handler finished processing softcall queue, the
96  *     state of machines goes back to IDLE.
97  *
98  * d - softcall() generates another softlevel1 iff interrupt handler
99  *     hasn't run recently.
100  *
101  * e - Either PEND|STEAL or DRAIN|STEAL is set. We let softlevel1
102  *     handler exit because we have processed all the entries.
103  *
104  * When CPU is being pinned by higher level interrupts for more than
105  * softcall_delay clock ticks, SOFT_STEAL is OR'ed so that softlevel1
106  * handler on the other CPU can drain the queue.
107  *
108  * These states are needed for softcall mechanism since Solaris has only
109  * one interface (ie. siron ) as of now for :
110  *
111  * - raising a soft interrupt architecture independently (ie not through
112  *   setsoftint(..) )
113  * - to process the softcall queue.
114  */
115 
116 #define	NSOFTCALLS	200
117 
118 /*
119  * Defined states for softcall processing.
120  */
121 #define	SOFT_IDLE		0x01	/* no processing is needed */
122 #define	SOFT_PEND		0x02	/* softcall list needs processing */
123 #define	SOFT_DRAIN		0x04	/* list is being processed */
124 #define	SOFT_STEAL		0x08	/* list is being stolen for draining */
125 
126 typedef struct softcall {
127 	void (*sc_func)(void *);	/* function to call */
128 	void *sc_arg;			/* arg to pass to func */
129 	struct softcall *sc_next;	/* next in list */
130 } softcall_t;
131 
132 /*
133  * softcall list and state variables.
134  */
135 static softcall_t *softcalls;
136 static softcall_t *softhead, *softtail, *softfree;
137 static uint_t	softcall_state;
138 static clock_t softcall_tick;
139 static clock_t softcall_countstart, softcall_lastpoke;
140 static uint_t softcall_pokecount;
141 
142 /*
143  * Max number of pokes per second before increasing softcall_delay
144  */
145 uint_t softcall_pokemax = 10;
146 
147 /*
148  * This ensures that softcall entries don't get stuck for long. It's expressed
149  * in 10 milliseconds as 1 unit. Regardless of the value of hires_tick or
150  * clock frequency, softcall_init() ensures that it's still expressed as 1 =
151  * 10 milliseconds.
152  */
153 unsigned int softcall_delay = 1;
154 
155 /*
156  * The last CPU which will drain softcall queue.
157  */
158 static int softcall_latest_cpuid = -1;
159 
160 /*
161  * CPUSET to hold the CPU which is processing softcall queue
162  * currently. There can be more than one CPU having bit set
163  * but it will happen only when they are stuck.
164  */
165 static cpuset_t *softcall_cpuset = NULL;
166 
167 /*
168  * protects softcall lists and control variable softcall_state.
169  */
170 static kmutex_t	softcall_lock;
171 
172 static void (*kdi_softcall_func)(void);
173 extern void siron_poke_cpu(cpuset_t);
174 
175 extern void siron(void);
176 extern void kdi_siron(void);
177 
178 
179 void
softcall_init(void)180 softcall_init(void)
181 {
182 	softcall_t *sc;
183 
184 	softcalls = kmem_zalloc(sizeof (softcall_t) * NSOFTCALLS, KM_SLEEP);
185 	softcall_cpuset = kmem_zalloc(sizeof (cpuset_t), KM_SLEEP);
186 	for (sc = softcalls; sc < &softcalls[NSOFTCALLS]; sc++) {
187 		sc->sc_next = softfree;
188 		softfree = sc;
189 	}
190 	mutex_init(&softcall_lock, NULL, MUTEX_SPIN,
191 	    (void *)ipltospl(SPL8));
192 	softcall_state = SOFT_IDLE;
193 	softcall_tick = ddi_get_lbolt();
194 
195 	/*
196 	 * Since softcall_delay is expressed as 1 = 10 milliseconds.
197 	 */
198 	softcall_delay = softcall_delay * (hz/100);
199 	CPUSET_ZERO(*softcall_cpuset);
200 }
201 
202 /*
203  * Gets called when softcall queue is not moving forward. We choose
204  * a CPU and poke except the ones which are already poked.
205  */
206 static int
softcall_choose_cpu()207 softcall_choose_cpu()
208 {
209 	cpu_t *cplist = CPU;
210 	cpu_t *cp;
211 	int intr_load = INT_MAX;
212 	int cpuid = -1;
213 	cpuset_t poke;
214 	int s;
215 
216 	ASSERT(getpil() >= DISP_LEVEL);
217 	ASSERT(ncpus > 1);
218 	ASSERT(MUTEX_HELD(&softcall_lock));
219 
220 	CPUSET_ZERO(poke);
221 
222 	/*
223 	 * The hint is to start from current CPU.
224 	 */
225 	cp = cplist;
226 	do {
227 		/*
228 		 * Don't select this CPU if :
229 		 *   - in cpuset already
230 		 *   - CPU is not accepting interrupts
231 		 *   - CPU is being offlined
232 		 */
233 		if (CPU_IN_SET(*softcall_cpuset, cp->cpu_id) ||
234 		    (cp->cpu_flags & CPU_ENABLE) == 0 ||
235 		    (cp == cpu_inmotion))
236 			continue;
237 #if defined(__x86)
238 		/*
239 		 * Don't select this CPU if a hypervisor indicates it
240 		 * isn't currently scheduled onto a physical cpu.  We are
241 		 * looking for a cpu that can respond quickly and the time
242 		 * to get the virtual cpu scheduled and switched to running
243 		 * state is likely to be relatively lengthy.
244 		 */
245 		if (vcpu_on_pcpu(cp->cpu_id) == VCPU_NOT_ON_PCPU)
246 			continue;
247 #endif	/* __x86 */
248 
249 		/* if CPU is not busy */
250 		if (cp->cpu_intrload == 0) {
251 			cpuid = cp->cpu_id;
252 			break;
253 		}
254 
255 		if (cp->cpu_intrload < intr_load) {
256 			cpuid = cp->cpu_id;
257 			intr_load = cp->cpu_intrload;
258 		} else if (cp->cpu_intrload == intr_load) {
259 			/*
260 			 * We want to poke CPUs having similar
261 			 * load because we don't know which CPU is
262 			 * can acknowledge level1 interrupt. The
263 			 * list of such CPUs should not be large.
264 			 */
265 			if (cpuid != -1) {
266 				/*
267 				 * Put the last CPU chosen because
268 				 * it also has same interrupt load.
269 				 */
270 				CPUSET_ADD(poke, cpuid);
271 				cpuid = -1;
272 			}
273 
274 			CPUSET_ADD(poke, cp->cpu_id);
275 		}
276 	} while ((cp = cp->cpu_next_onln) != cplist);
277 
278 	/* if we found a CPU which suits best to poke */
279 	if (cpuid != -1) {
280 		CPUSET_ZERO(poke);
281 		CPUSET_ADD(poke, cpuid);
282 	}
283 
284 	if (CPUSET_ISNULL(poke)) {
285 		mutex_exit(&softcall_lock);
286 		return (0);
287 	}
288 
289 	/*
290 	 * We first set the bit in cpuset and then poke.
291 	 */
292 	CPUSET_XOR(*softcall_cpuset, poke);
293 	mutex_exit(&softcall_lock);
294 
295 	/*
296 	 * If softcall() was called at low pil then we may
297 	 * get preempted before we raise PIL. It should be okay
298 	 * because we are just going to poke CPUs now or at most
299 	 * another thread may start choosing CPUs in this routine.
300 	 */
301 	s = splhigh();
302 	siron_poke_cpu(poke);
303 	splx(s);
304 	return (1);
305 }
306 
307 
308 /*
309  * Call function func with argument arg
310  * at some later time at software interrupt priority
311  */
312 void
softcall(void (* func)(void *),void * arg)313 softcall(void (*func)(void *), void *arg)
314 {
315 	softcall_t *sc;
316 	clock_t w, now;
317 
318 	/*
319 	 * protect against cross-calls
320 	 */
321 	mutex_enter(&softcall_lock);
322 	/* coalesce identical softcalls */
323 	for (sc = softhead; sc != 0; sc = sc->sc_next) {
324 		if (sc->sc_func == func && sc->sc_arg == arg) {
325 			goto intr;
326 		}
327 	}
328 
329 	if ((sc = softfree) == 0)
330 		panic("too many softcalls");
331 
332 	softfree = sc->sc_next;
333 	sc->sc_func = func;
334 	sc->sc_arg = arg;
335 	sc->sc_next = 0;
336 
337 	if (softhead) {
338 		softtail->sc_next = sc;
339 		softtail = sc;
340 	} else
341 		softhead = softtail = sc;
342 
343 intr:
344 	if (softcall_state & SOFT_IDLE) {
345 		softcall_state = SOFT_PEND;
346 		softcall_tick = ddi_get_lbolt();
347 		mutex_exit(&softcall_lock);
348 		siron();
349 	} else if (softcall_state & (SOFT_DRAIN|SOFT_PEND)) {
350 		now = ddi_get_lbolt();
351 		w = now - softcall_tick;
352 		if (w <= softcall_delay || ncpus == 1) {
353 			mutex_exit(&softcall_lock);
354 			return;
355 		}
356 		/*
357 		 * Did we poke less than a second ago?
358 		 */
359 		if (now - softcall_lastpoke < hz) {
360 			/*
361 			 * We did, increment the poke count and
362 			 * see if we are poking too often
363 			 */
364 			if (softcall_pokecount++ == 0)
365 				softcall_countstart = now;
366 			if (softcall_pokecount > softcall_pokemax) {
367 				/*
368 				 * If poking too much increase the delay
369 				 */
370 				if (now - softcall_countstart <= hz)
371 					softcall_delay++;
372 				softcall_pokecount = 0;
373 			}
374 		} else {
375 			/*
376 			 * poke rate has dropped off, reset the poke monitor
377 			 */
378 			softcall_pokecount = 0;
379 		}
380 		softcall_lastpoke = now;
381 		if (!(softcall_state & SOFT_STEAL)) {
382 			softcall_state |= SOFT_STEAL;
383 
384 			/*
385 			 * We want to give some more chance before
386 			 * fishing around again.
387 			 */
388 			softcall_tick = now;
389 		}
390 
391 		/* softcall_lock will be released by this routine */
392 		(void) softcall_choose_cpu();
393 	}
394 }
395 
396 void
kdi_softcall(void (* func)(void))397 kdi_softcall(void (*func)(void))
398 {
399 	kdi_softcall_func = func;
400 
401 	if (softhead == NULL)
402 		kdi_siron();
403 }
404 
405 /*
406  * Called to process software interrupts take one off queue, call it,
407  * repeat.
408  *
409  * Note queue may change during call; softcall_lock, state variables
410  * softcall_state and softcall_latest_cpuid ensures that -
411  * - we don't have multiple cpus pulling from the list (thus causing
412  *   a violation of FIFO order with an exception when we are stuck).
413  * - we don't miss a new entry having been added to the head.
414  * - we don't miss a wakeup.
415  */
416 
417 void
softint(void)418 softint(void)
419 {
420 	softcall_t *sc = NULL;
421 	void (*func)();
422 	caddr_t arg;
423 	int cpu_id = CPU->cpu_id;
424 
425 	/*
426 	 * Don't process softcall queue if current CPU is quiesced or
427 	 * offlined. This can happen when a CPU is running pause
428 	 * thread but softcall already sent a xcall.
429 	 */
430 	if (CPU->cpu_flags & (CPU_QUIESCED|CPU_OFFLINE)) {
431 		if (softcall_cpuset != NULL &&
432 		    CPU_IN_SET(*softcall_cpuset, cpu_id)) {
433 			CPUSET_DEL(*softcall_cpuset, cpu_id);
434 			goto out;
435 		}
436 	}
437 
438 	mutex_enter(&softcall_lock);
439 
440 	if (softcall_state & (SOFT_STEAL|SOFT_PEND)) {
441 		softcall_state = SOFT_DRAIN;
442 	} else  {
443 		/*
444 		 * The check for softcall_cpuset being
445 		 * NULL is required because it may get
446 		 * called very early during boot.
447 		 */
448 		if (softcall_cpuset != NULL &&
449 		    CPU_IN_SET(*softcall_cpuset, cpu_id))
450 			CPUSET_DEL(*softcall_cpuset, cpu_id);
451 		mutex_exit(&softcall_lock);
452 		goto out;
453 	}
454 
455 	/*
456 	 * Setting softcall_latest_cpuid to current CPU ensures
457 	 * that there is only one active softlevel1 handler to
458 	 * process softcall queues.
459 	 *
460 	 * Since softcall_lock lock is dropped before calling
461 	 * func (callback), we need softcall_latest_cpuid
462 	 * to prevent two softlevel1 hanlders working on the
463 	 * queue when the first softlevel1 handler gets
464 	 * stuck due to high interrupt load.
465 	 */
466 	softcall_latest_cpuid = cpu_id;
467 
468 	/* add ourself to the cpuset */
469 	if (!CPU_IN_SET(*softcall_cpuset, cpu_id))
470 		CPUSET_ADD(*softcall_cpuset, cpu_id);
471 
472 	for (;;) {
473 		softcall_tick = ddi_get_lbolt();
474 		if ((sc = softhead) != NULL) {
475 			func = sc->sc_func;
476 			arg = sc->sc_arg;
477 			softhead = sc->sc_next;
478 			sc->sc_next = softfree;
479 			softfree = sc;
480 		}
481 
482 		if (sc == NULL) {
483 			if (CPU_IN_SET(*softcall_cpuset, cpu_id))
484 				CPUSET_DEL(*softcall_cpuset, cpu_id);
485 
486 			softcall_state = SOFT_IDLE;
487 			ASSERT(softcall_latest_cpuid == cpu_id);
488 			softcall_latest_cpuid = -1;
489 
490 			mutex_exit(&softcall_lock);
491 			break;
492 		}
493 
494 		mutex_exit(&softcall_lock);
495 		func(arg);
496 		mutex_enter(&softcall_lock);
497 
498 		/*
499 		 * No longer need softcall processing from current
500 		 * interrupt handler because either
501 		 *  (a) softcall is in SOFT_IDLE state or
502 		 *  (b) There is a CPU already draining softcall
503 		 *	queue and the current softlevel1 is no
504 		 *	longer required.
505 		 */
506 		if (softcall_latest_cpuid != cpu_id) {
507 			if (CPU_IN_SET(*softcall_cpuset, cpu_id))
508 				CPUSET_DEL(*softcall_cpuset, cpu_id);
509 
510 			mutex_exit(&softcall_lock);
511 			break;
512 		}
513 	}
514 
515 out:
516 	if ((func = kdi_softcall_func) != NULL) {
517 		kdi_softcall_func = NULL;
518 		func();
519 	}
520 }
521