17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5f635d46aSqiao * Common Development and Distribution License (the "License"). 6f635d46aSqiao * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 217c478bd9Sstevel@tonic-gate /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 22*b4203d75SMarcel Telka /* All Rights Reserved */ 237c478bd9Sstevel@tonic-gate 247c478bd9Sstevel@tonic-gate 257c478bd9Sstevel@tonic-gate /* 26060cedfbSMadhavan Venkataraman * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 27f635d46aSqiao * Use is subject to license terms. 287c478bd9Sstevel@tonic-gate */ 297c478bd9Sstevel@tonic-gate 307c478bd9Sstevel@tonic-gate #ifndef _SYS_CALLO_H 317c478bd9Sstevel@tonic-gate #define _SYS_CALLO_H 327c478bd9Sstevel@tonic-gate 337c478bd9Sstevel@tonic-gate #include <sys/t_lock.h> 347c478bd9Sstevel@tonic-gate #include <sys/taskq.h> 3587a18d3fSMadhavan Venkataraman #include <sys/lgrp.h> 3687a18d3fSMadhavan Venkataraman #include <sys/processor.h> 3787a18d3fSMadhavan Venkataraman #include <sys/cyclic.h> 3887a18d3fSMadhavan Venkataraman #include <sys/kstat.h> 3987a18d3fSMadhavan Venkataraman #include <sys/systm.h> 407c478bd9Sstevel@tonic-gate 417c478bd9Sstevel@tonic-gate #ifdef __cplusplus 427c478bd9Sstevel@tonic-gate extern "C" { 437c478bd9Sstevel@tonic-gate #endif 447c478bd9Sstevel@tonic-gate 4587a18d3fSMadhavan Venkataraman #ifdef _KERNEL 4687a18d3fSMadhavan Venkataraman 4787a18d3fSMadhavan Venkataraman typedef struct callout_list callout_list_t; 487c478bd9Sstevel@tonic-gate 497c478bd9Sstevel@tonic-gate /* 507c478bd9Sstevel@tonic-gate * The callout mechanism provides general-purpose event scheduling: 517c478bd9Sstevel@tonic-gate * an arbitrary function is called in a specified amount of time. 5287a18d3fSMadhavan Venkataraman * The expiration time for a callout is kept in its callout list 5387a18d3fSMadhavan Venkataraman * structure. 547c478bd9Sstevel@tonic-gate */ 557c478bd9Sstevel@tonic-gate typedef struct callout { 567c478bd9Sstevel@tonic-gate struct callout *c_idnext; /* next in ID hash, or on freelist */ 577c478bd9Sstevel@tonic-gate struct callout *c_idprev; /* prev in ID hash */ 5887a18d3fSMadhavan Venkataraman struct callout *c_clnext; /* next in callout list */ 5987a18d3fSMadhavan Venkataraman struct callout *c_clprev; /* prev in callout list */ 607c478bd9Sstevel@tonic-gate callout_id_t c_xid; /* extended callout ID; see below */ 6187a18d3fSMadhavan Venkataraman callout_list_t *c_list; /* callout list */ 627c478bd9Sstevel@tonic-gate void (*c_func)(void *); /* function to call */ 637c478bd9Sstevel@tonic-gate void *c_arg; /* argument to function */ 6407247649SMadhavan Venkataraman kthread_t *c_executor; /* executing thread */ 6507247649SMadhavan Venkataraman kcondvar_t c_done; /* signal callout completion */ 6607247649SMadhavan Venkataraman ushort_t c_waiting; /* untimeout waiting flag */ 677c478bd9Sstevel@tonic-gate } callout_t; 687c478bd9Sstevel@tonic-gate 697c478bd9Sstevel@tonic-gate /* 7087a18d3fSMadhavan Venkataraman * The callout ID (callout_id_t) uniquely identifies a callout. The callout 7187a18d3fSMadhavan Venkataraman * ID is always 64 bits internally. The lower 32 bits contain an ID value. 7287a18d3fSMadhavan Venkataraman * The upper 32 bits contain a generation number and flags. When the ID value 7387a18d3fSMadhavan Venkataraman * wraps the generation number is incremented during ID generation. This 7487a18d3fSMadhavan Venkataraman * protects callers from ID collisions that can happen as a result of the wrap. 7587a18d3fSMadhavan Venkataraman * 7687a18d3fSMadhavan Venkataraman * The kernel internal interface, timeout_generic(), always returns a 7787a18d3fSMadhavan Venkataraman * callout_id_t. But the legacy interfaces, timeout() and realtime_timeout() 7887a18d3fSMadhavan Venkataraman * return a timeout_id_t. On a 64-bit system, timeout_id_t is also 64 bits. 7987a18d3fSMadhavan Venkataraman * So, the full 64-bit ID (sans the flags) can be returned. However, on 32-bit 8087a18d3fSMadhavan Venkataraman * systems, timeout_id_t is 32 bits. So, only the lower 32 bits can be 8187a18d3fSMadhavan Venkataraman * returned. In such cases, a default generation number of 0 is assigned to 8287a18d3fSMadhavan Venkataraman * the legacy IDs. 8387a18d3fSMadhavan Venkataraman * 8451b32bddSMadhavan Venkataraman * The lower 32-bit ID space is partitioned into two spaces - one for 8551b32bddSMadhavan Venkataraman * short-term callouts and one for long-term. 8687a18d3fSMadhavan Venkataraman * 8787a18d3fSMadhavan Venkataraman * Here is the bit layout for the callout ID: 8887a18d3fSMadhavan Venkataraman * 8951b32bddSMadhavan Venkataraman * 63 62 61 ... 32 31 30 29 .. X+1 X ... 1 0 9051b32bddSMadhavan Venkataraman * ----------------------------------------------------------------------- 9151b32bddSMadhavan Venkataraman * | Free | Exec | Generation | Long | Counter | ID bits | Table | Type | 9251b32bddSMadhavan Venkataraman * | | | number | term | High | | number | | 9351b32bddSMadhavan Venkataraman * ----------------------------------------------------------------------- 9451b32bddSMadhavan Venkataraman * 9551b32bddSMadhavan Venkataraman * Free: 9651b32bddSMadhavan Venkataraman * This bit indicates that this callout has been freed. This is for 9751b32bddSMadhavan Venkataraman * debugging purposes. 9887a18d3fSMadhavan Venkataraman * 9987a18d3fSMadhavan Venkataraman * Exec(uting): 10087a18d3fSMadhavan Venkataraman * This is the executing bit which is only set in the extended callout 10187a18d3fSMadhavan Venkataraman * ID. This bit indicates that the callout handler is currently being 10287a18d3fSMadhavan Venkataraman * executed. 10387a18d3fSMadhavan Venkataraman * 10487a18d3fSMadhavan Venkataraman * Generation number: 10587a18d3fSMadhavan Venkataraman * This is the generation part of the ID. 10687a18d3fSMadhavan Venkataraman * 10787a18d3fSMadhavan Venkataraman * Long term: 10887a18d3fSMadhavan Venkataraman * This bit indicates whether this is a short-term or a long-term callout. 10987a18d3fSMadhavan Venkataraman * The long-term bit exists to address the problem of callout ID collision 11087a18d3fSMadhavan Venkataraman * on 32-bit systems. This is an issue because the system typically 11187a18d3fSMadhavan Venkataraman * generates a large number of timeout() requests, which means that callout 11287a18d3fSMadhavan Venkataraman * IDs eventually get recycled. Most timeouts are very short-lived, so that 11387a18d3fSMadhavan Venkataraman * ID recycling isn't a problem; but there are a handful of timeouts which 11487a18d3fSMadhavan Venkataraman * are sufficiently long-lived to see their own IDs reused. We use the 11587a18d3fSMadhavan Venkataraman * long-term bit to partition the ID namespace into pieces; the short-term 11687a18d3fSMadhavan Venkataraman * space gets all the heavy traffic and can wrap frequently (i.e., on the 11787a18d3fSMadhavan Venkataraman * order of a day) with no ill effects; the long-term space gets very little 11887a18d3fSMadhavan Venkataraman * traffic and thus never wraps. That said, we need to future proof callouts 11987a18d3fSMadhavan Venkataraman * in case 32-bit systems grow in size and are able to consume callout IDs 12087a18d3fSMadhavan Venkataraman * at faster rates. So, we should make all the kernel clients that use 12187a18d3fSMadhavan Venkataraman * callouts to use the internal interface so that they can use IDs outside 12287a18d3fSMadhavan Venkataraman * of the legacy space with a proper generation number. 12387a18d3fSMadhavan Venkataraman * 12487a18d3fSMadhavan Venkataraman * Counter High + ID counter bits: 12587a18d3fSMadhavan Venkataraman * These bits represent the actual ID bits in the callout ID. 12687a18d3fSMadhavan Venkataraman * The highest bit of the running counter is always set; this ensures that 12787a18d3fSMadhavan Venkataraman * the callout ID is always non-zero, thus eliminating the need for an 12887a18d3fSMadhavan Venkataraman * explicit wrap-around test during ID generation. 12987a18d3fSMadhavan Venkataraman * 13087a18d3fSMadhavan Venkataraman * Table number: 13187a18d3fSMadhavan Venkataraman * These bits carry the table number for the callout table where the callout 13287a18d3fSMadhavan Venkataraman * is queued. Each CPU has its own callout table. So, the callout tables are 13387a18d3fSMadhavan Venkataraman * numbered from 0 - (max_ncpus - 1). Because max_ncpus is different on 13487a18d3fSMadhavan Venkataraman * different systems, the actual number of table number bits will vary 13587a18d3fSMadhavan Venkataraman * accordingly. And so will the ID counter bits. 13687a18d3fSMadhavan Venkataraman * 13787a18d3fSMadhavan Venkataraman * Type: 13887a18d3fSMadhavan Venkataraman * This bit represents the callout (table) type. Each CPU has one realtime 13987a18d3fSMadhavan Venkataraman * and one normal callout table. 1407c478bd9Sstevel@tonic-gate */ 141060cedfbSMadhavan Venkataraman #define CALLOUT_ID_FREE 0x8000000000000000ULL 14251b32bddSMadhavan Venkataraman #define CALLOUT_EXECUTING 0x4000000000000000ULL 143060cedfbSMadhavan Venkataraman #define CALLOUT_ID_FLAGS (CALLOUT_ID_FREE | CALLOUT_EXECUTING) 14451b32bddSMadhavan Venkataraman #define CALLOUT_ID_MASK ~CALLOUT_ID_FLAGS 14587a18d3fSMadhavan Venkataraman #define CALLOUT_GENERATION_LOW 0x100000000ULL 14687a18d3fSMadhavan Venkataraman #define CALLOUT_LONGTERM 0x80000000 14787a18d3fSMadhavan Venkataraman #define CALLOUT_COUNTER_HIGH 0x40000000 1487c478bd9Sstevel@tonic-gate #define CALLOUT_TYPE_BITS 1 1497c478bd9Sstevel@tonic-gate #define CALLOUT_NTYPES (1 << CALLOUT_TYPE_BITS) 15087a18d3fSMadhavan Venkataraman #define CALLOUT_TYPE_MASK (CALLOUT_NTYPES - 1) 15187a18d3fSMadhavan Venkataraman #define CALLOUT_COUNTER_SHIFT callout_table_bits 15287a18d3fSMadhavan Venkataraman #define CALLOUT_TABLE(t, f) (((f) << CALLOUT_TYPE_BITS) | (t)) 15387a18d3fSMadhavan Venkataraman #define CALLOUT_TABLE_NUM(ct) ((ct) - callout_table) 15487a18d3fSMadhavan Venkataraman #define CALLOUT_TABLE_SEQID(ct) (CALLOUT_TABLE_NUM(ct) >> CALLOUT_TYPE_BITS) 1557c478bd9Sstevel@tonic-gate 1567c478bd9Sstevel@tonic-gate /* 1577c478bd9Sstevel@tonic-gate * We assume that during any period of CALLOUT_LONGTERM_TICKS ticks, at most 15887a18d3fSMadhavan Venkataraman * (CALLOUT_COUNTER_HIGH / callout_counter_low) callouts will be generated. 1597c478bd9Sstevel@tonic-gate */ 16087a18d3fSMadhavan Venkataraman #define CALLOUT_LONGTERM_TICKS 0x4000UL 16187a18d3fSMadhavan Venkataraman #define CALLOUT_BUCKET_SHIFT 9 16287a18d3fSMadhavan Venkataraman #define CALLOUT_BUCKETS (1 << CALLOUT_BUCKET_SHIFT) 1637c478bd9Sstevel@tonic-gate #define CALLOUT_BUCKET_MASK (CALLOUT_BUCKETS - 1) 1647c478bd9Sstevel@tonic-gate #define CALLOUT_HASH(x) ((x) & CALLOUT_BUCKET_MASK) 1657c478bd9Sstevel@tonic-gate #define CALLOUT_IDHASH(x) CALLOUT_HASH((x) >> CALLOUT_COUNTER_SHIFT) 16687a18d3fSMadhavan Venkataraman /* 16787a18d3fSMadhavan Venkataraman * The multiply by 0 and 1 below are cosmetic. Just to align things better 16887a18d3fSMadhavan Venkataraman * and make it more readable. The multiplications will be done at compile 16987a18d3fSMadhavan Venkataraman * time. 17087a18d3fSMadhavan Venkataraman */ 17187a18d3fSMadhavan Venkataraman #define CALLOUT_CLHASH(x) \ 17287a18d3fSMadhavan Venkataraman CALLOUT_HASH( \ 17387a18d3fSMadhavan Venkataraman ((x)>>(CALLOUT_BUCKET_SHIFT*0)) ^ \ 17487a18d3fSMadhavan Venkataraman ((x)>>(CALLOUT_BUCKET_SHIFT*1)) ^ \ 17587a18d3fSMadhavan Venkataraman ((x)>>(CALLOUT_BUCKET_SHIFT*2)) ^ \ 17687a18d3fSMadhavan Venkataraman ((x)>>(CALLOUT_BUCKET_SHIFT*3))) 17787a18d3fSMadhavan Venkataraman 17887a18d3fSMadhavan Venkataraman #define CALLOUT_ID_TO_TABLE(id) ((id) & callout_table_mask) 17987a18d3fSMadhavan Venkataraman 18087a18d3fSMadhavan Venkataraman #define CALLOUT_SHORT_ID(table) \ 18187a18d3fSMadhavan Venkataraman ((callout_id_t)(table) | CALLOUT_COUNTER_HIGH) 18287a18d3fSMadhavan Venkataraman #define CALLOUT_LONG_ID(table) \ 18387a18d3fSMadhavan Venkataraman (CALLOUT_SHORT_ID(table) | CALLOUT_LONGTERM) 1847c478bd9Sstevel@tonic-gate 18551b32bddSMadhavan Venkataraman #define CALLOUT_THREADS 2 1867c478bd9Sstevel@tonic-gate 1877c478bd9Sstevel@tonic-gate #define CALLOUT_REALTIME 0 /* realtime callout type */ 1887c478bd9Sstevel@tonic-gate #define CALLOUT_NORMAL 1 /* normal callout type */ 1897c478bd9Sstevel@tonic-gate 19087a18d3fSMadhavan Venkataraman /* 19187a18d3fSMadhavan Venkataraman * callout_t's are cache-aligned structures allocated from kmem caches. One kmem 19287a18d3fSMadhavan Venkataraman * cache is created per lgrp and is shared by all CPUs in that lgrp. Benefits: 19387a18d3fSMadhavan Venkataraman * - cache pages are mapped only in the TLBs of the CPUs of the lgrp 19487a18d3fSMadhavan Venkataraman * - data in cache pages is present only in those CPU caches 19587a18d3fSMadhavan Venkataraman * - memory access performance improves with locality-awareness in kmem 19687a18d3fSMadhavan Venkataraman * 19787a18d3fSMadhavan Venkataraman * The following structure is used to manage per-lgroup kmem caches. 19887a18d3fSMadhavan Venkataraman * 19987a18d3fSMadhavan Venkataraman * NOTE: Free callout_t's go to a callout table's freelist. CPUs map to callout 20087a18d3fSMadhavan Venkataraman * tables via their sequence IDs, not CPU IDs. DR operations can cause a 20187a18d3fSMadhavan Venkataraman * free list to have callouts from multiple lgrp caches. This takes away some 20287a18d3fSMadhavan Venkataraman * performance, but is no worse than if we did not use lgrp caches at all. 20387a18d3fSMadhavan Venkataraman */ 20487a18d3fSMadhavan Venkataraman typedef struct callout_cache { 20587a18d3fSMadhavan Venkataraman struct callout_cache *cc_next; /* link in the global list */ 20687a18d3fSMadhavan Venkataraman lgrp_handle_t cc_hand; /* lgroup handle */ 20787a18d3fSMadhavan Venkataraman kmem_cache_t *cc_cache; /* kmem cache pointer */ 20887a18d3fSMadhavan Venkataraman kmem_cache_t *cc_lcache; /* kmem cache pointer */ 20987a18d3fSMadhavan Venkataraman } callout_cache_t; 21087a18d3fSMadhavan Venkataraman 21187a18d3fSMadhavan Venkataraman /* 21287a18d3fSMadhavan Venkataraman * The callout hash structure is used for queueing both callouts and 21387a18d3fSMadhavan Venkataraman * callout lists. That is why the fields are declared as void *. 21487a18d3fSMadhavan Venkataraman */ 21587a18d3fSMadhavan Venkataraman typedef struct callout_hash { 21687a18d3fSMadhavan Venkataraman void *ch_head; 21787a18d3fSMadhavan Venkataraman void *ch_tail; 21887a18d3fSMadhavan Venkataraman } callout_hash_t; 21987a18d3fSMadhavan Venkataraman 22051b32bddSMadhavan Venkataraman /* 22151b32bddSMadhavan Venkataraman * CALLOUT_LIST_FLAG_FREE 22251b32bddSMadhavan Venkataraman * Callout list is free. 22351b32bddSMadhavan Venkataraman * CALLOUT_LIST_FLAG_ABSOLUTE 22451b32bddSMadhavan Venkataraman * Callout list contains absolute timers. 22551b32bddSMadhavan Venkataraman * CALLOUT_LIST_FLAG_HRESTIME 22651b32bddSMadhavan Venkataraman * Callout list contains hrestime timers. 22751b32bddSMadhavan Venkataraman * CALLOUT_LIST_FLAG_NANO 22851b32bddSMadhavan Venkataraman * Callout list contains 1-nanosecond resolution callouts. 229060cedfbSMadhavan Venkataraman * CALLOUT_LIST_FLAG_HEAPED 230060cedfbSMadhavan Venkataraman * Callout list is present in the callout heap. 231060cedfbSMadhavan Venkataraman * CALLOUT_LIST_FLAG_QUEUED 232060cedfbSMadhavan Venkataraman * Callout list is present in the callout queue. 23351b32bddSMadhavan Venkataraman */ 23451b32bddSMadhavan Venkataraman #define CALLOUT_LIST_FLAG_FREE 0x1 23551b32bddSMadhavan Venkataraman #define CALLOUT_LIST_FLAG_ABSOLUTE 0x2 23651b32bddSMadhavan Venkataraman #define CALLOUT_LIST_FLAG_HRESTIME 0x4 23751b32bddSMadhavan Venkataraman #define CALLOUT_LIST_FLAG_NANO 0x8 238060cedfbSMadhavan Venkataraman #define CALLOUT_LIST_FLAG_HEAPED 0x10 239060cedfbSMadhavan Venkataraman #define CALLOUT_LIST_FLAG_QUEUED 0x20 24051b32bddSMadhavan Venkataraman 24187a18d3fSMadhavan Venkataraman struct callout_list { 24287a18d3fSMadhavan Venkataraman callout_list_t *cl_next; /* next in clhash */ 24387a18d3fSMadhavan Venkataraman callout_list_t *cl_prev; /* prev in clhash */ 24487a18d3fSMadhavan Venkataraman hrtime_t cl_expiration; /* expiration for callouts in list */ 24587a18d3fSMadhavan Venkataraman callout_hash_t cl_callouts; /* list of callouts */ 24607247649SMadhavan Venkataraman int cl_flags; /* callout flags */ 24787a18d3fSMadhavan Venkataraman }; 24887a18d3fSMadhavan Venkataraman 24951b32bddSMadhavan Venkataraman /* 25051b32bddSMadhavan Venkataraman * Callout heap element. Each element in the heap stores the expiration 25151b32bddSMadhavan Venkataraman * as well as the corresponding callout list. This is to avoid a lookup 25251b32bddSMadhavan Venkataraman * of the callout list when the heap is processed. Because we store the 25351b32bddSMadhavan Venkataraman * callout list pointer in the heap element, we have to always remove 25451b32bddSMadhavan Venkataraman * a heap element and its callout list together. We cannot remove one 25551b32bddSMadhavan Venkataraman * without the other. 256060cedfbSMadhavan Venkataraman * 257060cedfbSMadhavan Venkataraman * This structure's size must be a power of two because we want an 258060cedfbSMadhavan Venkataraman * integral number of these to fit into a page. 25951b32bddSMadhavan Venkataraman */ 26051b32bddSMadhavan Venkataraman typedef struct callout_heap { 26151b32bddSMadhavan Venkataraman hrtime_t ch_expiration; 26251b32bddSMadhavan Venkataraman callout_list_t *ch_list; 2631671524dSMadhavan Venkataraman #ifndef _LP64 2641671524dSMadhavan Venkataraman char ch_pad[4]; /* pad to power of 2 */ 2651671524dSMadhavan Venkataraman #endif 26651b32bddSMadhavan Venkataraman } callout_heap_t; 26751b32bddSMadhavan Venkataraman 26851b32bddSMadhavan Venkataraman /* 26951b32bddSMadhavan Venkataraman * When the heap contains too many empty callout lists, it needs to be 27051b32bddSMadhavan Venkataraman * cleaned up. The decision to clean up the heap is a function of the 27151b32bddSMadhavan Venkataraman * number of empty entries and the heap size. Also, we don't want to 27251b32bddSMadhavan Venkataraman * clean up small heaps. 27351b32bddSMadhavan Venkataraman */ 27451b32bddSMadhavan Venkataraman #define CALLOUT_MIN_REAP (CALLOUT_BUCKETS >> 3) 27551b32bddSMadhavan Venkataraman #define CALLOUT_CLEANUP(ct) ((ct->ct_nreap >= callout_min_reap) && \ 27651b32bddSMadhavan Venkataraman (ct->ct_nreap >= (ct->ct_heap_num >> 1))) 27751b32bddSMadhavan Venkataraman 27887a18d3fSMadhavan Venkataraman /* 27987a18d3fSMadhavan Venkataraman * Per-callout table kstats. 28087a18d3fSMadhavan Venkataraman * 28187a18d3fSMadhavan Venkataraman * CALLOUT_TIMEOUTS 28287a18d3fSMadhavan Venkataraman * Callouts created since boot. 28387a18d3fSMadhavan Venkataraman * CALLOUT_TIMEOUTS_PENDING 28487a18d3fSMadhavan Venkataraman * Number of outstanding callouts. 28587a18d3fSMadhavan Venkataraman * CALLOUT_UNTIMEOUTS_UNEXPIRED 28687a18d3fSMadhavan Venkataraman * Number of cancelled callouts that have not expired. 28787a18d3fSMadhavan Venkataraman * CALLOUT_UNTIMEOUTS_EXECUTING 28887a18d3fSMadhavan Venkataraman * Number of cancelled callouts that were executing at the time of 28987a18d3fSMadhavan Venkataraman * cancellation. 29087a18d3fSMadhavan Venkataraman * CALLOUT_UNTIMEOUTS_EXPIRED 29187a18d3fSMadhavan Venkataraman * Number of cancelled callouts that had already expired at the time 29287a18d3fSMadhavan Venkataraman * of cancellations. 29387a18d3fSMadhavan Venkataraman * CALLOUT_EXPIRATIONS 29487a18d3fSMadhavan Venkataraman * Number of callouts that expired. 29587a18d3fSMadhavan Venkataraman * CALLOUT_ALLOCATIONS 29687a18d3fSMadhavan Venkataraman * Number of callout structures allocated. 29751b32bddSMadhavan Venkataraman * CALLOUT_CLEANUPS 29851b32bddSMadhavan Venkataraman * Number of times a callout table is cleaned up. 29987a18d3fSMadhavan Venkataraman */ 30087a18d3fSMadhavan Venkataraman typedef enum callout_stat_type { 30187a18d3fSMadhavan Venkataraman CALLOUT_TIMEOUTS, 30287a18d3fSMadhavan Venkataraman CALLOUT_TIMEOUTS_PENDING, 30387a18d3fSMadhavan Venkataraman CALLOUT_UNTIMEOUTS_UNEXPIRED, 30487a18d3fSMadhavan Venkataraman CALLOUT_UNTIMEOUTS_EXECUTING, 30587a18d3fSMadhavan Venkataraman CALLOUT_UNTIMEOUTS_EXPIRED, 30687a18d3fSMadhavan Venkataraman CALLOUT_EXPIRATIONS, 30787a18d3fSMadhavan Venkataraman CALLOUT_ALLOCATIONS, 30851b32bddSMadhavan Venkataraman CALLOUT_CLEANUPS, 30987a18d3fSMadhavan Venkataraman CALLOUT_NUM_STATS 31087a18d3fSMadhavan Venkataraman } callout_stat_type_t; 31187a18d3fSMadhavan Venkataraman 31287a18d3fSMadhavan Venkataraman /* 31387a18d3fSMadhavan Venkataraman * Callout flags: 31487a18d3fSMadhavan Venkataraman * 31587a18d3fSMadhavan Venkataraman * CALLOUT_FLAG_ROUNDUP 31607247649SMadhavan Venkataraman * Roundup the expiration time to the next resolution boundary. 31787a18d3fSMadhavan Venkataraman * If this flag is not specified, the expiration time is rounded down. 31887a18d3fSMadhavan Venkataraman * CALLOUT_FLAG_ABSOLUTE 31987a18d3fSMadhavan Venkataraman * Normally, the expiration passed to the timeout API functions is an 32087a18d3fSMadhavan Venkataraman * expiration interval. If this flag is specified, then it is 32187a18d3fSMadhavan Venkataraman * interpreted as the expiration time itself. 32287a18d3fSMadhavan Venkataraman * CALLOUT_FLAG_HRESTIME 32387a18d3fSMadhavan Venkataraman * Normally, callouts are not affected by changes to system time 32487a18d3fSMadhavan Venkataraman * (hrestime). This flag is used to create a callout that is affected 32507247649SMadhavan Venkataraman * by system time. If system time changes, these timers must be 32607247649SMadhavan Venkataraman * handled in a special way (see callout.c). These are used by condition 32707247649SMadhavan Venkataraman * variables and LWP timers that need this behavior. 32887a18d3fSMadhavan Venkataraman * CALLOUT_FLAG_32BIT 32987a18d3fSMadhavan Venkataraman * Legacy interfaces timeout() and realtime_timeout() pass this flag 33087a18d3fSMadhavan Venkataraman * to timeout_generic() to indicate that a 32-bit ID should be allocated. 33187a18d3fSMadhavan Venkataraman */ 33287a18d3fSMadhavan Venkataraman #define CALLOUT_FLAG_ROUNDUP 0x1 33387a18d3fSMadhavan Venkataraman #define CALLOUT_FLAG_ABSOLUTE 0x2 33487a18d3fSMadhavan Venkataraman #define CALLOUT_FLAG_HRESTIME 0x4 33587a18d3fSMadhavan Venkataraman #define CALLOUT_FLAG_32BIT 0x8 33687a18d3fSMadhavan Venkataraman 33787a18d3fSMadhavan Venkataraman /* 33887a18d3fSMadhavan Venkataraman * On 32-bit systems, the legacy interfaces, timeout() and realtime_timeout(), 33987a18d3fSMadhavan Venkataraman * must pass CALLOUT_FLAG_32BIT to timeout_generic() so that a 32-bit ID 34087a18d3fSMadhavan Venkataraman * can be generated. 34187a18d3fSMadhavan Venkataraman */ 34287a18d3fSMadhavan Venkataraman #ifdef _LP64 34387a18d3fSMadhavan Venkataraman #define CALLOUT_LEGACY 0 34487a18d3fSMadhavan Venkataraman #else 34587a18d3fSMadhavan Venkataraman #define CALLOUT_LEGACY CALLOUT_FLAG_32BIT 34687a18d3fSMadhavan Venkataraman #endif 34787a18d3fSMadhavan Venkataraman 3487c478bd9Sstevel@tonic-gate /* 3497c478bd9Sstevel@tonic-gate * All of the state information associated with a callout table. 3507c478bd9Sstevel@tonic-gate * The fields are ordered with cache performance in mind. 3517c478bd9Sstevel@tonic-gate */ 3527c478bd9Sstevel@tonic-gate typedef struct callout_table { 35387a18d3fSMadhavan Venkataraman kmutex_t ct_mutex; /* protects all callout state */ 35487a18d3fSMadhavan Venkataraman callout_t *ct_free; /* free callout structures */ 35587a18d3fSMadhavan Venkataraman callout_list_t *ct_lfree; /* free callout list structures */ 3567c478bd9Sstevel@tonic-gate callout_id_t ct_short_id; /* most recently issued short-term ID */ 3577c478bd9Sstevel@tonic-gate callout_id_t ct_long_id; /* most recently issued long-term ID */ 35887a18d3fSMadhavan Venkataraman callout_hash_t *ct_idhash; /* ID hash chains */ 35987a18d3fSMadhavan Venkataraman callout_hash_t *ct_clhash; /* callout list hash */ 36087a18d3fSMadhavan Venkataraman kstat_named_t *ct_kstat_data; /* callout kstat data */ 36187a18d3fSMadhavan Venkataraman 362454ab202SMadhavan Venkataraman uint_t ct_type; /* callout table type */ 363454ab202SMadhavan Venkataraman uint_t ct_suspend; /* suspend count */ 36487a18d3fSMadhavan Venkataraman cyclic_id_t ct_cyclic; /* cyclic for this table */ 36551b32bddSMadhavan Venkataraman callout_heap_t *ct_heap; /* callout expiration heap */ 36687a18d3fSMadhavan Venkataraman ulong_t ct_heap_num; /* occupied slots in the heap */ 36787a18d3fSMadhavan Venkataraman ulong_t ct_heap_max; /* end of the heap */ 36887a18d3fSMadhavan Venkataraman kmem_cache_t *ct_cache; /* callout kmem cache */ 36987a18d3fSMadhavan Venkataraman kmem_cache_t *ct_lcache; /* callout list kmem cache */ 37087a18d3fSMadhavan Venkataraman callout_id_t ct_gen_id; /* generation based ID */ 37187a18d3fSMadhavan Venkataraman 37287a18d3fSMadhavan Venkataraman callout_hash_t ct_expired; /* list of expired callout lists */ 37387a18d3fSMadhavan Venkataraman taskq_t *ct_taskq; /* taskq to execute normal callouts */ 37487a18d3fSMadhavan Venkataraman kstat_t *ct_kstats; /* callout kstats */ 37551b32bddSMadhavan Venkataraman int ct_nreap; /* # heap entries that need reaping */ 376060cedfbSMadhavan Venkataraman cyclic_id_t ct_qcyclic; /* cyclic for the callout queue */ 377060cedfbSMadhavan Venkataraman callout_hash_t ct_queue; /* overflow queue of callouts */ 378060cedfbSMadhavan Venkataraman #ifndef _LP64 379060cedfbSMadhavan Venkataraman char ct_pad[12]; /* cache alignment */ 38087a18d3fSMadhavan Venkataraman #endif 381060cedfbSMadhavan Venkataraman /* 382060cedfbSMadhavan Venkataraman * This structure should be aligned to a 64-byte (cache-line) 383060cedfbSMadhavan Venkataraman * boundary. Make sure the padding is right for 32-bit as well 384060cedfbSMadhavan Venkataraman * as 64-bit kernels. 385060cedfbSMadhavan Venkataraman */ 3867c478bd9Sstevel@tonic-gate } callout_table_t; 3877c478bd9Sstevel@tonic-gate 38887a18d3fSMadhavan Venkataraman /* 38987a18d3fSMadhavan Venkataraman * Short hand definitions for the callout kstats. 39087a18d3fSMadhavan Venkataraman */ 39187a18d3fSMadhavan Venkataraman #define ct_timeouts \ 39287a18d3fSMadhavan Venkataraman ct_kstat_data[CALLOUT_TIMEOUTS].value.ui64 39387a18d3fSMadhavan Venkataraman #define ct_timeouts_pending \ 39487a18d3fSMadhavan Venkataraman ct_kstat_data[CALLOUT_TIMEOUTS_PENDING].value.ui64 39587a18d3fSMadhavan Venkataraman #define ct_untimeouts_unexpired \ 39687a18d3fSMadhavan Venkataraman ct_kstat_data[CALLOUT_UNTIMEOUTS_UNEXPIRED].value.ui64 39787a18d3fSMadhavan Venkataraman #define ct_untimeouts_executing \ 39887a18d3fSMadhavan Venkataraman ct_kstat_data[CALLOUT_UNTIMEOUTS_EXECUTING].value.ui64 39987a18d3fSMadhavan Venkataraman #define ct_untimeouts_expired \ 40087a18d3fSMadhavan Venkataraman ct_kstat_data[CALLOUT_UNTIMEOUTS_EXPIRED].value.ui64 40187a18d3fSMadhavan Venkataraman #define ct_expirations \ 40287a18d3fSMadhavan Venkataraman ct_kstat_data[CALLOUT_EXPIRATIONS].value.ui64 40387a18d3fSMadhavan Venkataraman #define ct_allocations \ 40487a18d3fSMadhavan Venkataraman ct_kstat_data[CALLOUT_ALLOCATIONS].value.ui64 40551b32bddSMadhavan Venkataraman #define ct_cleanups \ 40651b32bddSMadhavan Venkataraman ct_kstat_data[CALLOUT_CLEANUPS].value.ui64 40787a18d3fSMadhavan Venkataraman 408060cedfbSMadhavan Venkataraman /* 409060cedfbSMadhavan Venkataraman * CALLOUT_CHUNK is the minimum initial size of each heap, and the amount 410060cedfbSMadhavan Venkataraman * by which a full heap is expanded to make room for new entries. 411060cedfbSMadhavan Venkataraman */ 412060cedfbSMadhavan Venkataraman #define CALLOUT_CHUNK (PAGESIZE / sizeof (callout_heap_t)) 413060cedfbSMadhavan Venkataraman 414060cedfbSMadhavan Venkataraman /* 415060cedfbSMadhavan Venkataraman * CALLOUT_MIN_HEAP_SIZE defines the minimum size for the callout heap for 416060cedfbSMadhavan Venkataraman * the whole system. 417060cedfbSMadhavan Venkataraman */ 418060cedfbSMadhavan Venkataraman #define CALLOUT_MIN_HEAP_SIZE (64 * 1024 * sizeof (callout_heap_t)) 419060cedfbSMadhavan Venkataraman 420060cedfbSMadhavan Venkataraman /* 421060cedfbSMadhavan Venkataraman * CALLOUT_MEM_FRACTION defines the fraction of available physical memory that 422060cedfbSMadhavan Venkataraman * can be allocated towards the callout heap for the whole system. 423060cedfbSMadhavan Venkataraman */ 424060cedfbSMadhavan Venkataraman #define CALLOUT_MEM_FRACTION 4096 42587a18d3fSMadhavan Venkataraman 42687a18d3fSMadhavan Venkataraman #define CALLOUT_HEAP_PARENT(index) (((index) - 1) >> 1) 42787a18d3fSMadhavan Venkataraman #define CALLOUT_HEAP_RIGHT(index) (((index) + 1) << 1) 42887a18d3fSMadhavan Venkataraman #define CALLOUT_HEAP_LEFT(index) ((((index) + 1) << 1) - 1) 42987a18d3fSMadhavan Venkataraman 43087a18d3fSMadhavan Venkataraman #define CALLOUT_TCP_RESOLUTION 10000000ULL 43187a18d3fSMadhavan Venkataraman 43287a18d3fSMadhavan Venkataraman #define CALLOUT_ALIGN 64 /* cache line size */ 43387a18d3fSMadhavan Venkataraman 434454ab202SMadhavan Venkataraman #ifdef _LP64 435454ab202SMadhavan Venkataraman #define CALLOUT_MAX_TICKS NSEC_TO_TICK(CY_INFINITY); 436454ab202SMadhavan Venkataraman #else 437454ab202SMadhavan Venkataraman #define CALLOUT_MAX_TICKS LONG_MAX 438454ab202SMadhavan Venkataraman #endif 439454ab202SMadhavan Venkataraman 44051b32bddSMadhavan Venkataraman #define CALLOUT_TOLERANCE 200000 /* nanoseconds */ 44151b32bddSMadhavan Venkataraman 44287a18d3fSMadhavan Venkataraman extern void callout_init(void); 44387a18d3fSMadhavan Venkataraman extern void membar_sync(void); 44487a18d3fSMadhavan Venkataraman extern void callout_cpu_online(cpu_t *); 445454ab202SMadhavan Venkataraman extern void callout_cpu_offline(cpu_t *); 44687a18d3fSMadhavan Venkataraman extern void callout_hrestime(void); 447454ab202SMadhavan Venkataraman 4487c478bd9Sstevel@tonic-gate #endif 4497c478bd9Sstevel@tonic-gate 4507c478bd9Sstevel@tonic-gate #ifdef __cplusplus 4517c478bd9Sstevel@tonic-gate } 4527c478bd9Sstevel@tonic-gate #endif 4537c478bd9Sstevel@tonic-gate 4547c478bd9Sstevel@tonic-gate #endif /* _SYS_CALLO_H */ 455