1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #ifndef _SYS_CYCLIC_IMPL_H
27 #define	_SYS_CYCLIC_IMPL_H
28 
29 #ifdef	__cplusplus
30 extern "C" {
31 #endif
32 
33 #include <sys/cyclic.h>
34 #include <sys/rwlock.h>
35 
36 /*
37  *  Cyclic Subsystem Backend-supplied Interfaces
38  *  --------------------------------------------
39  *
40  *  0  Background
41  *
42  *    The design, implementation and interfaces of the cyclic subsystem are
43  *    covered in detail in block comments in the implementation.  This
44  *    comment covers the interface from the cyclic subsystem into the cyclic
45  *    backend.  The backend is specified by a structure of function pointers
46  *    defined below.
47  *
48  *  1  Overview
49  *
50  *      cyb_configure()      <-- Configures the backend on the specified CPU
51  *      cyb_unconfigure()    <-- Unconfigures the backend
52  *      cyb_enable()         <-- Enables the CY_HIGH_LEVEL interrupt source
53  *      cyb_disable()        <-- Disables the CY_HIGH_LEVEL interrupt source
54  *      cyb_reprogram()      <-- Reprograms the CY_HIGH_LEVEL interrupt source
55  *      cyb_softint()        <-- Generates a soft interrupt
56  *      cyb_set_level()      <-- Sets the programmable interrupt level
57  *      cyb_restore_level()  <-- Restores the programmable interrupt level
58  *      cyb_xcall()          <-- Cross calls to the specified CPU
59  *      cyb_suspend()        <-- Suspends the backend
60  *      cyb_resume()         <-- Resumes the backend
61  *
62  *  2  cyb_arg_t cyb_configure(cpu_t *)
63  *
64  *  2.1  Overview
65  *
66  *    cyb_configure() should configure the specified CPU for cyclic operation.
67  *
68  *  2.2  Arguments and notes
69  *
70  *    cyb_configure() should initialize any backend-specific per-CPU
71  *    structures for the specified CPU.  cyb_configure() will be called for
72  *    each CPU (including the boot CPU) during boot.  If the platform
73  *    supports dynamic reconfiguration, cyb_configure() will be called for
74  *    new CPUs as they are configured into the system.
75  *
76  *  2.3  Return value
77  *
78  *    cyb_configure() is expected to return a cookie (a cyb_arg_t, which is
79  *    of type void *) which will be used as the first argument for all future
80  *    cyclic calls into the backend on the specified CPU.
81  *
82  *  2.4  Caller's context
83  *
84  *    cpu_lock will be held.  The caller's CPU is unspecified, and may or
85  *    may not be the CPU specified to cyb_configure().
86  *
87  *  3  void cyb_unconfigure(cyb_arg_t arg)
88  *
89  *  3.1  Overview
90  *
91  *    cyb_unconfigure() should unconfigure the specified backend.
92  *
93  *  3.2  Arguments and notes
94  *
95  *    The only argument to cyb_unconfigure() is a cookie as returned from
96  *    cyb_configure().
97  *
98  *    cyb_unconfigure() should free any backend-specific per-CPU structures
99  *    for the specified backend.  cyb_unconfigure() will _only_ be called on
100  *    platforms which support dynamic reconfiguration.  If the platform does
101  *    not support dynamic reconfiguration, cyb_unconfigure() may panic.
102  *
103  *    After cyb_unconfigure() returns, the backend must not call cyclic_fire()
104  *    on the corresponding CPU; doing so will result in a bad trap.
105  *
106  *  3.3  Return value
107  *
108  *    None.
109  *
110  *  3.4  Caller's context
111  *
112  *    cpu_lock will be held.  The caller's CPU is unspecified, and may or
113  *    may not be the CPU specified to cyb_unconfigure().  The specified
114  *    CPU is guaranteed to exist at the time cyb_unconfigure() is called.
115  *    The cyclic subsystem is guaranteed to be suspended when cyb_unconfigure()
116  *    is called, and interrupts are guaranteed to be disabled.
117  *
118  *  4  void cyb_enable(cyb_arg_t arg)
119  *
120  *  4.1  Overview
121  *
122  *    cyb_enable() should enable the CY_HIGH_LEVEL interrupt source on
123  *    the specified backend.
124  *
125  *  4.2  Arguments and notes
126  *
127  *    The only argument to cyb_enable() is a backend cookie as returned from
128  *    cyb_configure().
129  *
130  *    cyb_enable() will only be called if a) the specified backend has never
131  *    been enabled or b) the specified backend has been explicitly disabled with
132  *    cyb_disable().  In either case, cyb_enable() will only be called if
133  *    the cyclic subsystem wishes to add a cyclic to the CPU corresponding
134  *    to the specified backend.  cyb_enable() will be called before
135  *    cyb_reprogram() for a given backend.
136  *
137  *    cyclic_fire() should not be called on a CPU which has not had its backend
138  *    explicitly cyb_enable()'d, but to do so does not constitute fatal error.
139  *
140  *  4.3  Return value
141  *
142  *    None.
143  *
144  *  4.4  Caller's context
145  *
146  *    cyb_enable() will only be called from CY_HIGH_LEVEL context on the CPU
147  *    corresponding to the specified backend.
148  *
149  *  5  void cyb_disable(cyb_arg_t arg)
150  *
151  *  5.1  Overview
152  *
153  *    cyb_disable() should disable the CY_HIGH_LEVEL interrupt source on
154  *    the specified backend.
155  *
156  *  5.2  Arguments and notes
157  *
158  *    The only argument to cyb_disable() is a backend cookie as returned from
159  *    cyb_configure().
160  *
161  *    cyb_disable() will only be called on backends which have been previously
162  *    been cyb_enable()'d.  cyb_disable() will be called when all cyclics have
163  *    been juggled away or removed from a cyb_enable()'d CPU.
164  *
165  *    cyclic_fire() should not be called on a CPU which has had its backend
166  *    explicitly cyb_disable()'d, but to do so does not constitute fatal
167  *    error.  cyb_disable() is thus not required to check for a pending
168  *    CY_HIGH_LEVEL interrupt.
169  *
170  *  5.3  Return value
171  *
172  *    None.
173  *
174  *  5.4  Caller's context
175  *
176  *    cyb_disable() will only be called from CY_HIGH_LEVEL context on the CPU
177  *    corresponding to the specified backend.
178  *
179  *  6  void cyb_reprogram(cyb_arg_t arg, hrtime_t time)
180  *
181  *  6.1  Overview
182  *
183  *    cyb_reprogram() should reprogram the CY_HIGH_LEVEL interrupt source
184  *    to fire at the absolute time specified.
185  *
186  *  6.2  Arguments and notes
187  *
188  *    The first argument to cyb_reprogram() is a backend cookie as returned from
189  *    cyb_configure().
190  *
191  *    The second argument is an absolute time at which the CY_HIGH_LEVEL
192  *    interrupt should fire.  The specified time _may_ be in the past (albeit
193  *    the very recent past).  If this is the case, the backend should generate
194  *    a CY_HIGH_LEVEL interrupt as soon as possible.
195  *
196  *    The platform should not assume that cyb_reprogram() will be called with
197  *    monotonically increasing values.
198  *
199  *    If the platform does not allow for interrupts at arbitrary times in the
200  *    future, cyb_reprogram() may do nothing -- as long as cyclic_fire() is
201  *    called periodically at CY_HIGH_LEVEL.  While this is clearly suboptimal
202  *    (cyclic granularity will be bounded by the length of the period between
203  *    cyclic_fire()'s), it allows the cyclic subsystem to be implemented on
204  *    inferior hardware.
205  *
206  *  6.3  Return value
207  *
208  *     None.
209  *
210  *  6.4  Caller's context
211  *
212  *    cyb_reprogram() will only be called from CY_HIGH_LEVEL context on the CPU
213  *    corresponding to the specified backend.
214  *
215  *  7  void cyb_softint(cyb_arg_t arg, cyc_level_t level)
216  *
217  *  7.1  Overview
218  *
219  *    cyb_softint() should generate a software interrupt on the specified
220  *    backend at the specified level.
221  *
222  *  7.2  Arguments and notes
223  *
224  *    The first argument to cyb_softint() is a backend cookie as returned from
225  *    cyb_configure().  The second argument is the interrupt level at which
226  *    the software interrupt should be generated; it will be either
227  *    CY_LOCK_LEVEL or CY_LOW_LEVEL.
228  *
229  *    The software interrupt _must_ be generated on the CPU corresponding
230  *    to the specified backend; platforms are _required_ to have a per-CPU
231  *    notion of a software interrupt.
232  *
233  *    Unless a software interrupt is already pending at the specified level,
234  *    the software interrupt _must_ be generated.  Once cyclic_softint()
235  *    has been called at a given level, the software interrupt at that level
236  *    should no longer be considered pending; an intervening CY_HIGH_LEVEL
237  *    interrupt and subsequent cyb_softint() must generate another software
238  *    interrupt.
239  *
240  *  7.3  Return value
241  *
242  *    None.
243  *
244  *  7.4  Caller's context
245  *
246  *    cyb_softint() will only be called at a level higher than the one
247  *    specified:  if CY_LOCK_LEVEL is specified, the caller will be at
248  *    CY_HIGH_LEVEL; if CY_LOW_LEVEL is specified, the caller will be at
249  *    either CY_HIGH_LEVEL or CY_LOCK_LEVEL.  cyb_softint() will only be
250  *    called on the CPU corresponding to the specified backend.
251  *
252  *  8  cyb_set_level(cyb_arg_t arg, cyc_level_t level)
253  *
254  *  8.1  Overview
255  *
256  *    cyb_set_level() should set the programmable interrupt level to the
257  *    level specified.
258  *
259  *  8.2  Arguments and notes
260  *
261  *    The first argument to cyb_set_level() is a backend cookie as returned
262  *    from cyb_configure().  The second argument is the level to which
263  *    the programmable interrupt level should be set; it will be one of
264  *    CY_HIGH_LEVEL, CY_LOCK_LEVEL or CY_LOW_LEVEL.
265  *
266  *    After cyb_set_level() returns, the CPU associated with the specified
267  *    backend should accept no interrupt at a level greater than or equal to
268  *    the specified level.  This will generally be a wrapper around splx().
269  *
270  *    The cyclic subsystem will never call cyb_set_level() twice consecutively
271  *    on the same backend; there will always be an intervening
272  *    cyb_restore_level();
273  *
274  *  8.3  Return value
275  *
276  *    cyb_set_level() should return a cookie to be passed back to
277  *    cyb_restore_level().  On most implementations, this cookie will be
278  *    the spl at the time of cyb_set_level().
279  *
280  *  8.4  Caller's context
281  *
282  *    cyb_set_level() is unique in that it is the only backend-provided
283  *    interface which may be called in cross call context (see cyb_xcall(),
284  *    below).  cyb_set_level() may also be called from any of the cyclic
285  *
286  *  9  cyb_restore_level(cyb_arg_t arg, cyc_cookie_t cookie)
287  *
288  *  9.1  Overview
289  *
290  *    cyb_restore_level() should restore the programmable interrupt level
291  *    based upon the specified cookie.
292  *
293  *  9.2  Arguments and notes
294  *
295  *    The first argument to cyb_restore_level() is a backend cookie as returned
296  *    from cyb_configure().  The second argument is a cookie as returned from
297  *    cyb_set_level().
298  *
299  *    cyb_restore_level() should restore the programmable interrupt level
300  *    to its value when cyb_set_level() was called; the cookie is used
301  *    to provide a hint to the backend.  cyb_restore_level() will not be
302  *    called without a proceeding call to cyb_set_level(), and
303  *    cyb_restore_level() will never be called twice consecutively on the
304  *    same backend.
305  *
306  *  9.3  Return value
307  *
308  *    None.
309  *
310  *  9.4  Caller's context
311  *
312  *    The constraints outlined in 5.9.2 imply that cyb_restore_level() can
313  *    only be called from CY_HIGH_LEVEL, CY_LOCK_LEVEL or CY_LOW_LEVEL context.
314  *    cyb_restore_level() is always called on the CPU associated with the
315  *    specified backend.
316  *
317  *  10  cyb_xcall(cyb_arg_t arg, cpu_t *, void(*func)(void *), void *farg)
318  *
319  *  10.1  Overview
320  *
321  *    cyb_xcall() should execute the specified function on the specified CPU.
322  *
323  *  10.2  Arguments and notes
324  *
325  *    The first argument to cyb_restore_level() is a backend cookie as returned
326  *    from cyb_configure().  The second argument is a CPU on which the third
327  *    argument, a function pointer, should be executed.  The fourth argument,
328  *    a void *, should be passed as the argument to the specified function.
329  *
330  *    cyb_xcall() must provide exactly-once semantics.  If the specified
331  *    function is called more than once, or not at all, the cyclic subsystem
332  *    will become internally inconsistent.  The specified function must be
333  *    be executed on the specified CPU, but may be executed in any context
334  *    (any interrupt context or kernel context).
335  *
336  *    cyb_xcall() cannot block.  Any resources which cyb_xcall() needs to
337  *    acquire must thus be protected by synchronization primitives which
338  *    never require the caller to block.
339  *
340  *  10.3  Return value
341  *
342  *    None.
343  *
344  *  10.4  Caller's context
345  *
346  *    cpu_lock will be held and kernel preemption may be disabled.  The caller
347  *    may be unable to block, giving rise to the constraint outlined in
348  *    10.2, above.
349  *
350  *  11  cyb_suspend(cyb_arg_t arg)
351  *
352  *  11.1  Overview
353  *
354  *    cyb_suspend() should suspend the specified backend.
355  *
356  *  11.2  Arguments and notes
357  *
358  *    The only argument to cyb_suspend() is a backend cookie as returned from
359  *    cyb_configure().
360  *
361  *    cyb_suspend() will never be called on enabled backends.  The backend
362  *    should assume that the machine may be subsequently powered off; any
363  *    volatile hardware state should be preserved and restored in cyb_resume().
364  *    However, the backend should not _assume_ that the machine will be
365  *    powered off; cyb_suspend() may also be called as part of dynamic
366  *    reconfiguration.
367  *
368  *    cyb_suspend() will be called on the corresponding backend of each
369  *    CPU in the system in succession, regardless of CPU state (P_ONLINE,
370  *    P_OFFLINE, P_NOINTR).  The cyclic subsystem will not suspend only a
371  *    fraction of the CPUs.
372  *
373  *  11.3  Return value
374  *
375  *    None.
376  *
377  *  11.4  Caller's context
378  *
379  *    cyb_suspend() will be called in cross call context on the CPU associated
380  *    with the specified backend.
381  *
382  *  12  cyb_resume(cyb_arg_t arg)
383  *
384  *  12.1  Overview
385  *
386  *    cyb_resume() should resume the specified backend.
387  *
388  *  12.2  Arguments and notes
389  *
390  *    The only argument to cyb_resume() is a backend cookie as returned from
391  *    cyb_resume().
392  *
393  *    Calls to cyb_resume() will always have been proceeded by corresponding
394  *    calls to cyb_suspend().  The machine may have been powered off between
395  *    cyb_suspend() and the call to cyb_resume().  cyb_resume() may decide
396  *    to restore hardware to its state at the time cyb_suspend() was called.
397  *
398  *    The cyclic subsystem will make no calls into the backend between
399  *    cyb_suspend() and cyb_resume().
400  *
401  *  12.3  Return value
402  *
403  *    None.
404  *
405  *  12.4  Caller's context
406  *
407  *    cyb_resume() will be called in cross call context on the CPU associated
408  *    with the specified backend.
409  */
410 typedef struct cyc_backend {
411 	cyb_arg_t (*cyb_configure)(cpu_t *);
412 	void (*cyb_unconfigure)(cyb_arg_t);
413 	void (*cyb_enable)(cyb_arg_t);
414 	void (*cyb_disable)(cyb_arg_t);
415 	void (*cyb_reprogram)(cyb_arg_t, hrtime_t);
416 	void (*cyb_softint)(cyb_arg_t, cyc_level_t);
417 	cyc_cookie_t (*cyb_set_level)(cyb_arg_t, cyc_level_t);
418 	void (*cyb_restore_level)(cyb_arg_t, cyc_cookie_t);
419 	void (*cyb_xcall)(cyb_arg_t, cpu_t *, cyc_func_t, void *);
420 	void (*cyb_suspend)(cyb_arg_t);
421 	void (*cyb_resume)(cyb_arg_t);
422 	cyb_arg_t cyb_arg;
423 } cyc_backend_t;
424 
425 extern void cyclic_init(cyc_backend_t *be, hrtime_t resolution);
426 extern void cyclic_mp_init();
427 
428 #ifdef DEBUG
429 #define	CYCLIC_TRACE
430 #endif
431 
432 typedef enum {
433 	CYS_ONLINE,
434 	CYS_OFFLINE,
435 	CYS_EXPANDING,
436 	CYS_REMOVING,
437 	CYS_SUSPENDED
438 } cyc_state_t;
439 
440 #define	CYF_FREE		0x0001
441 #define	CYF_CPU_BOUND		0x0002
442 #define	CYF_PART_BOUND		0x0004
443 
444 typedef struct cyclic {
445 	hrtime_t cy_expire;
446 	hrtime_t cy_interval;
447 	void (*cy_handler)(void *);
448 	void *cy_arg;
449 	uint32_t cy_pend;
450 	uint16_t cy_flags;
451 	cyc_level_t cy_level;
452 } cyclic_t;
453 
454 typedef struct cyc_pcbuffer {
455 	cyc_index_t *cypc_buf;
456 	int cypc_prodndx;
457 	int cypc_consndx;
458 	int cypc_sizemask;
459 } cyc_pcbuffer_t;
460 
461 typedef struct cyc_softbuf {
462 	uchar_t cys_hard;		/* Can only be zero or one */
463 	uchar_t cys_soft;		/* Can only be zero or one */
464 	cyc_pcbuffer_t cys_buf[2];
465 } cyc_softbuf_t;
466 
467 #define	CY_NTRACEREC		512
468 
469 typedef struct cyc_tracerec {
470 	hrtime_t cyt_tstamp;
471 	char *cyt_why;
472 	uint64_t cyt_arg0;
473 	uint64_t cyt_arg1;
474 } cyc_tracerec_t;
475 
476 typedef struct cyc_tracebuf {
477 	int cyt_ndx;
478 	cyc_tracerec_t cyt_buf[CY_NTRACEREC];
479 } cyc_tracebuf_t;
480 
481 #define	CY_NCOVERAGE	127
482 
483 typedef struct cyc_coverage {
484 	char *cyv_why;
485 	int cyv_passive_count;
486 	int cyv_count[CY_LEVELS];
487 	uint64_t cyv_arg0;
488 	uint64_t cyv_arg1;
489 } cyc_coverage_t;
490 
491 typedef struct cyc_cpu {
492 	cpu_t *cyp_cpu;
493 	cyc_index_t *cyp_heap;
494 	cyclic_t *cyp_cyclics;
495 	cyc_index_t cyp_nelems;
496 	cyc_index_t cyp_size;
497 	cyc_state_t cyp_state;
498 	cyc_softbuf_t cyp_softbuf[CY_SOFT_LEVELS];
499 	cyc_backend_t *cyp_backend;
500 	ksema_t cyp_modify_wait;
501 	uint32_t cyp_modify_levels;
502 	uint32_t cyp_rpend;
503 #ifdef CYCLIC_TRACE
504 	cyc_tracebuf_t cyp_trace[CY_LEVELS];
505 #endif
506 } cyc_cpu_t;
507 
508 typedef struct cyc_omni_cpu {
509 	cyc_cpu_t *cyo_cpu;
510 	cyc_index_t cyo_ndx;
511 	void *cyo_arg;
512 	struct cyc_omni_cpu *cyo_next;
513 } cyc_omni_cpu_t;
514 
515 typedef struct cyc_id {
516 	krwlock_t cyi_lock;
517 	cyc_cpu_t *cyi_cpu;
518 	cyc_index_t cyi_ndx;
519 	struct cyc_id *cyi_prev;
520 	struct cyc_id *cyi_next;
521 	cyc_omni_handler_t cyi_omni_hdlr;
522 	cyc_omni_cpu_t *cyi_omni_list;
523 } cyc_id_t;
524 
525 typedef struct cyc_xcallarg {
526 	cyc_cpu_t *cyx_cpu;
527 	cyc_handler_t *cyx_hdlr;
528 	cyc_time_t *cyx_when;
529 	cyc_index_t cyx_ndx;
530 	cyc_index_t *cyx_heap;
531 	cyclic_t *cyx_cyclics;
532 	cyc_index_t cyx_size;
533 	uint16_t cyx_flags;
534 	int cyx_wait;
535 } cyc_xcallarg_t;
536 
537 #define	CY_DEFAULT_PERCPU	1
538 #define	CY_PASSIVE_LEVEL	-1
539 
540 #define	CY_WAIT			0
541 #define	CY_NOWAIT		1
542 
543 #define	CYC_HEAP_PARENT(ndx)		(((ndx) - 1) >> 1)
544 #define	CYC_HEAP_RIGHT(ndx)		(((ndx) + 1) << 1)
545 #define	CYC_HEAP_LEFT(ndx)		((((ndx) + 1) << 1) - 1)
546 
547 #ifdef	__cplusplus
548 }
549 #endif
550 
551 #endif /* _SYS_CYCLIC_IMPL_H */
552