17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5ae115bc7Smrj * Common Development and Distribution License (the "License"). 6ae115bc7Smrj * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21843e1988Sjohnlev 227c478bd9Sstevel@tonic-gate /* 23*7997e108SSurya Prakki * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #include <sys/types.h> 287c478bd9Sstevel@tonic-gate #include <sys/param.h> 297c478bd9Sstevel@tonic-gate #include <sys/systm.h> 307c478bd9Sstevel@tonic-gate #include <sys/disp.h> 317c478bd9Sstevel@tonic-gate #include <sys/var.h> 327c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 337c478bd9Sstevel@tonic-gate #include <sys/debug.h> 347c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h> 357c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 367c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 377c478bd9Sstevel@tonic-gate #include <sys/psm_defs.h> 387c478bd9Sstevel@tonic-gate #include <sys/clock.h> 397c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 407c478bd9Sstevel@tonic-gate #include <sys/lockstat.h> 417c478bd9Sstevel@tonic-gate #include <sys/smp_impldefs.h> 427c478bd9Sstevel@tonic-gate #include <sys/dtrace.h> 437c478bd9Sstevel@tonic-gate #include <sys/time.h> 44843e1988Sjohnlev #include <sys/panic.h> 45b3c18020SSudheer A #include <sys/cpu.h> 467c478bd9Sstevel@tonic-gate 477c478bd9Sstevel@tonic-gate /* 487c478bd9Sstevel@tonic-gate * Using the Pentium's TSC register for gethrtime() 497c478bd9Sstevel@tonic-gate * ------------------------------------------------ 507c478bd9Sstevel@tonic-gate * 517c478bd9Sstevel@tonic-gate * The Pentium family, like many chip architectures, has a high-resolution 527c478bd9Sstevel@tonic-gate * timestamp counter ("TSC") which increments once per CPU cycle. The contents 537c478bd9Sstevel@tonic-gate * of the timestamp counter are read with the RDTSC instruction. 547c478bd9Sstevel@tonic-gate * 557c478bd9Sstevel@tonic-gate * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count 567c478bd9Sstevel@tonic-gate * must be translated into nanoseconds in order to implement gethrtime(). 577c478bd9Sstevel@tonic-gate * We avoid inducing floating point operations in this conversion by 587c478bd9Sstevel@tonic-gate * implementing the same nsec_scale algorithm as that found in the sun4u 597c478bd9Sstevel@tonic-gate * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains 607c478bd9Sstevel@tonic-gate * a detailed description of the algorithm; the comment is not reproduced 617c478bd9Sstevel@tonic-gate * here. This implementation differs only in its value for NSEC_SHIFT: 627c478bd9Sstevel@tonic-gate * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for 637c478bd9Sstevel@tonic-gate * 60 MHz Pentiums. 647c478bd9Sstevel@tonic-gate * 657c478bd9Sstevel@tonic-gate * While TSC and %tick are both cycle counting registers, TSC's functionality 667c478bd9Sstevel@tonic-gate * falls short in several critical ways: 677c478bd9Sstevel@tonic-gate * 687c478bd9Sstevel@tonic-gate * (a) TSCs on different CPUs are not guaranteed to be in sync. While in 697c478bd9Sstevel@tonic-gate * practice they often _are_ in sync, this isn't guaranteed by the 707c478bd9Sstevel@tonic-gate * architecture. 717c478bd9Sstevel@tonic-gate * 727c478bd9Sstevel@tonic-gate * (b) The TSC cannot be reliably set to an arbitrary value. The architecture 737c478bd9Sstevel@tonic-gate * only supports writing the low 32-bits of TSC, making it impractical 747c478bd9Sstevel@tonic-gate * to rewrite. 757c478bd9Sstevel@tonic-gate * 767c478bd9Sstevel@tonic-gate * (c) The architecture doesn't have the capacity to interrupt based on 777c478bd9Sstevel@tonic-gate * arbitrary values of TSC; there is no TICK_CMPR equivalent. 787c478bd9Sstevel@tonic-gate * 797c478bd9Sstevel@tonic-gate * Together, (a) and (b) imply that software must track the skew between 807c478bd9Sstevel@tonic-gate * TSCs and account for it (it is assumed that while there may exist skew, 817c478bd9Sstevel@tonic-gate * there does not exist drift). To determine the skew between CPUs, we 827c478bd9Sstevel@tonic-gate * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing 83b3c18020SSudheer A * the online operation calls tsc_sync_master(). 847c478bd9Sstevel@tonic-gate * 857c478bd9Sstevel@tonic-gate * In the absence of time-of-day clock adjustments, gethrtime() must stay in 867c478bd9Sstevel@tonic-gate * sync with gettimeofday(). This is problematic; given (c), the software 877c478bd9Sstevel@tonic-gate * cannot drive its time-of-day source from TSC, and yet they must somehow be 887c478bd9Sstevel@tonic-gate * kept in sync. We implement this by having a routine, tsc_tick(), which 897c478bd9Sstevel@tonic-gate * is called once per second from the interrupt which drives time-of-day. 907c478bd9Sstevel@tonic-gate * 917c478bd9Sstevel@tonic-gate * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified 927c478bd9Sstevel@tonic-gate * atomically with nsec_scale under CLOCK_LOCK. This assures that time 937c478bd9Sstevel@tonic-gate * monotonically increases. 947c478bd9Sstevel@tonic-gate */ 957c478bd9Sstevel@tonic-gate 967c478bd9Sstevel@tonic-gate #define NSEC_SHIFT 5 977c478bd9Sstevel@tonic-gate 987c478bd9Sstevel@tonic-gate static uint_t nsec_scale; 997c478bd9Sstevel@tonic-gate 1007c478bd9Sstevel@tonic-gate /* 1017c478bd9Sstevel@tonic-gate * These two variables used to be grouped together inside of a structure that 1027c478bd9Sstevel@tonic-gate * lived on a single cache line. A regression (bug ID 4623398) caused the 1037c478bd9Sstevel@tonic-gate * compiler to emit code that "optimized" away the while-loops below. The 1047c478bd9Sstevel@tonic-gate * result was that no synchronization between the onlining and onlined CPUs 1057c478bd9Sstevel@tonic-gate * took place. 1067c478bd9Sstevel@tonic-gate */ 1077c478bd9Sstevel@tonic-gate static volatile int tsc_ready; 1087c478bd9Sstevel@tonic-gate static volatile int tsc_sync_go; 1097c478bd9Sstevel@tonic-gate 1107c478bd9Sstevel@tonic-gate /* 1117c478bd9Sstevel@tonic-gate * Used as indices into the tsc_sync_snaps[] array. 1127c478bd9Sstevel@tonic-gate */ 1137c478bd9Sstevel@tonic-gate #define TSC_MASTER 0 1147c478bd9Sstevel@tonic-gate #define TSC_SLAVE 1 1157c478bd9Sstevel@tonic-gate 1167c478bd9Sstevel@tonic-gate /* 1177c478bd9Sstevel@tonic-gate * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous. 1187c478bd9Sstevel@tonic-gate */ 1197c478bd9Sstevel@tonic-gate #define TSC_SYNC_STOP 1 1207c478bd9Sstevel@tonic-gate #define TSC_SYNC_GO 2 121b3c18020SSudheer A #define TSC_SYNC_DONE 3 122b3c18020SSudheer A #define SYNC_ITERATIONS 10 1237c478bd9Sstevel@tonic-gate 124843e1988Sjohnlev #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \ 125ae115bc7Smrj unsigned int *_l = (unsigned int *)&(tsc); \ 126ae115bc7Smrj (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \ 1277c478bd9Sstevel@tonic-gate (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 1287c478bd9Sstevel@tonic-gate } 1297c478bd9Sstevel@tonic-gate 130ae115bc7Smrj #define TSC_CONVERT(tsc, hrt, scale) { \ 131ae115bc7Smrj unsigned int *_l = (unsigned int *)&(tsc); \ 132ae115bc7Smrj (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \ 1337c478bd9Sstevel@tonic-gate (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 1347c478bd9Sstevel@tonic-gate } 1357c478bd9Sstevel@tonic-gate 136ae115bc7Smrj int tsc_master_slave_sync_needed = 1; 137*7997e108SSurya Prakki extern int platform_is_virt; 1387c478bd9Sstevel@tonic-gate 1397c478bd9Sstevel@tonic-gate static int tsc_max_delta; 1407c478bd9Sstevel@tonic-gate static hrtime_t tsc_sync_tick_delta[NCPU]; 141b3c18020SSudheer A typedef struct tsc_sync { 142b3c18020SSudheer A volatile hrtime_t master_tsc, slave_tsc; 143b3c18020SSudheer A } tsc_sync_t; 144b3c18020SSudheer A static tsc_sync_t *tscp; 145b3c18020SSudheer A static hrtime_t largest_tsc_delta = 0; 146b3c18020SSudheer A static ulong_t shortest_write_time = ~0UL; 147b3c18020SSudheer A 1487c478bd9Sstevel@tonic-gate static hrtime_t tsc_last = 0; 1497c478bd9Sstevel@tonic-gate static hrtime_t tsc_last_jumped = 0; 1507c478bd9Sstevel@tonic-gate static hrtime_t tsc_hrtime_base = 0; 1517c478bd9Sstevel@tonic-gate static int tsc_jumped = 0; 1527c478bd9Sstevel@tonic-gate 1537c478bd9Sstevel@tonic-gate static hrtime_t shadow_tsc_hrtime_base; 1547c478bd9Sstevel@tonic-gate static hrtime_t shadow_tsc_last; 1557c478bd9Sstevel@tonic-gate static uint_t shadow_nsec_scale; 1567c478bd9Sstevel@tonic-gate static uint32_t shadow_hres_lock; 1572df1fe9cSrandyf int get_tsc_ready(); 1587c478bd9Sstevel@tonic-gate 159843e1988Sjohnlev hrtime_t 160843e1988Sjohnlev tsc_gethrtime(void) 161843e1988Sjohnlev { 162843e1988Sjohnlev uint32_t old_hres_lock; 163843e1988Sjohnlev hrtime_t tsc, hrt; 164843e1988Sjohnlev 165843e1988Sjohnlev do { 166843e1988Sjohnlev old_hres_lock = hres_lock; 167843e1988Sjohnlev 168843e1988Sjohnlev if ((tsc = tsc_read()) >= tsc_last) { 169843e1988Sjohnlev /* 170843e1988Sjohnlev * It would seem to be obvious that this is true 171843e1988Sjohnlev * (that is, the past is less than the present), 172843e1988Sjohnlev * but it isn't true in the presence of suspend/resume 173843e1988Sjohnlev * cycles. If we manage to call gethrtime() 174843e1988Sjohnlev * after a resume, but before the first call to 175843e1988Sjohnlev * tsc_tick(), we will see the jump. In this case, 176843e1988Sjohnlev * we will simply use the value in TSC as the delta. 177843e1988Sjohnlev */ 178843e1988Sjohnlev tsc -= tsc_last; 179843e1988Sjohnlev } else if (tsc >= tsc_last - 2*tsc_max_delta) { 180843e1988Sjohnlev /* 181843e1988Sjohnlev * There is a chance that tsc_tick() has just run on 182843e1988Sjohnlev * another CPU, and we have drifted just enough so that 183843e1988Sjohnlev * we appear behind tsc_last. In this case, force the 184843e1988Sjohnlev * delta to be zero. 185843e1988Sjohnlev */ 186843e1988Sjohnlev tsc = 0; 187843e1988Sjohnlev } 188843e1988Sjohnlev 189843e1988Sjohnlev hrt = tsc_hrtime_base; 190843e1988Sjohnlev 191843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 192843e1988Sjohnlev } while ((old_hres_lock & ~1) != hres_lock); 193843e1988Sjohnlev 194843e1988Sjohnlev return (hrt); 195843e1988Sjohnlev } 196843e1988Sjohnlev 197843e1988Sjohnlev hrtime_t 198843e1988Sjohnlev tsc_gethrtime_delta(void) 199843e1988Sjohnlev { 200843e1988Sjohnlev uint32_t old_hres_lock; 201843e1988Sjohnlev hrtime_t tsc, hrt; 202a563a037Sbholler ulong_t flags; 203843e1988Sjohnlev 204843e1988Sjohnlev do { 205843e1988Sjohnlev old_hres_lock = hres_lock; 206843e1988Sjohnlev 207843e1988Sjohnlev /* 208843e1988Sjohnlev * We need to disable interrupts here to assure that we 209843e1988Sjohnlev * don't migrate between the call to tsc_read() and 210843e1988Sjohnlev * adding the CPU's TSC tick delta. Note that disabling 211843e1988Sjohnlev * and reenabling preemption is forbidden here because 212843e1988Sjohnlev * we may be in the middle of a fast trap. In the amd64 213843e1988Sjohnlev * kernel we cannot tolerate preemption during a fast 214843e1988Sjohnlev * trap. See _update_sregs(). 215843e1988Sjohnlev */ 216843e1988Sjohnlev 217843e1988Sjohnlev flags = clear_int_flag(); 218843e1988Sjohnlev tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; 219843e1988Sjohnlev restore_int_flag(flags); 220843e1988Sjohnlev 221843e1988Sjohnlev /* See comments in tsc_gethrtime() above */ 222843e1988Sjohnlev 223843e1988Sjohnlev if (tsc >= tsc_last) { 224843e1988Sjohnlev tsc -= tsc_last; 225843e1988Sjohnlev } else if (tsc >= tsc_last - 2 * tsc_max_delta) { 226843e1988Sjohnlev tsc = 0; 227843e1988Sjohnlev } 228843e1988Sjohnlev 229843e1988Sjohnlev hrt = tsc_hrtime_base; 230843e1988Sjohnlev 231843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 232843e1988Sjohnlev } while ((old_hres_lock & ~1) != hres_lock); 233843e1988Sjohnlev 234843e1988Sjohnlev return (hrt); 235843e1988Sjohnlev } 236843e1988Sjohnlev 237843e1988Sjohnlev /* 238843e1988Sjohnlev * This is similar to the above, but it cannot actually spin on hres_lock. 239843e1988Sjohnlev * As a result, it caches all of the variables it needs; if the variables 240843e1988Sjohnlev * don't change, it's done. 241843e1988Sjohnlev */ 242843e1988Sjohnlev hrtime_t 243843e1988Sjohnlev dtrace_gethrtime(void) 244843e1988Sjohnlev { 245843e1988Sjohnlev uint32_t old_hres_lock; 246843e1988Sjohnlev hrtime_t tsc, hrt; 247a563a037Sbholler ulong_t flags; 248843e1988Sjohnlev 249843e1988Sjohnlev do { 250843e1988Sjohnlev old_hres_lock = hres_lock; 251843e1988Sjohnlev 252843e1988Sjohnlev /* 253843e1988Sjohnlev * Interrupts are disabled to ensure that the thread isn't 254843e1988Sjohnlev * migrated between the tsc_read() and adding the CPU's 255843e1988Sjohnlev * TSC tick delta. 256843e1988Sjohnlev */ 257843e1988Sjohnlev flags = clear_int_flag(); 258843e1988Sjohnlev 259843e1988Sjohnlev tsc = tsc_read(); 260843e1988Sjohnlev 261843e1988Sjohnlev if (gethrtimef == tsc_gethrtime_delta) 262843e1988Sjohnlev tsc += tsc_sync_tick_delta[CPU->cpu_id]; 263843e1988Sjohnlev 264843e1988Sjohnlev restore_int_flag(flags); 265843e1988Sjohnlev 266843e1988Sjohnlev /* 267843e1988Sjohnlev * See the comments in tsc_gethrtime(), above. 268843e1988Sjohnlev */ 269843e1988Sjohnlev if (tsc >= tsc_last) 270843e1988Sjohnlev tsc -= tsc_last; 271843e1988Sjohnlev else if (tsc >= tsc_last - 2*tsc_max_delta) 272843e1988Sjohnlev tsc = 0; 273843e1988Sjohnlev 274843e1988Sjohnlev hrt = tsc_hrtime_base; 275843e1988Sjohnlev 276843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 277843e1988Sjohnlev 278843e1988Sjohnlev if ((old_hres_lock & ~1) == hres_lock) 279843e1988Sjohnlev break; 280843e1988Sjohnlev 281843e1988Sjohnlev /* 282843e1988Sjohnlev * If we're here, the clock lock is locked -- or it has been 283843e1988Sjohnlev * unlocked and locked since we looked. This may be due to 284843e1988Sjohnlev * tsc_tick() running on another CPU -- or it may be because 285843e1988Sjohnlev * some code path has ended up in dtrace_probe() with 286843e1988Sjohnlev * CLOCK_LOCK held. We'll try to determine that we're in 287843e1988Sjohnlev * the former case by taking another lap if the lock has 288843e1988Sjohnlev * changed since when we first looked at it. 289843e1988Sjohnlev */ 290843e1988Sjohnlev if (old_hres_lock != hres_lock) 291843e1988Sjohnlev continue; 292843e1988Sjohnlev 293843e1988Sjohnlev /* 294843e1988Sjohnlev * So the lock was and is locked. We'll use the old data 295843e1988Sjohnlev * instead. 296843e1988Sjohnlev */ 297843e1988Sjohnlev old_hres_lock = shadow_hres_lock; 298843e1988Sjohnlev 299843e1988Sjohnlev /* 300843e1988Sjohnlev * Again, disable interrupts to ensure that the thread 301843e1988Sjohnlev * isn't migrated between the tsc_read() and adding 302843e1988Sjohnlev * the CPU's TSC tick delta. 303843e1988Sjohnlev */ 304843e1988Sjohnlev flags = clear_int_flag(); 305843e1988Sjohnlev 306843e1988Sjohnlev tsc = tsc_read(); 307843e1988Sjohnlev 308843e1988Sjohnlev if (gethrtimef == tsc_gethrtime_delta) 309843e1988Sjohnlev tsc += tsc_sync_tick_delta[CPU->cpu_id]; 310843e1988Sjohnlev 311843e1988Sjohnlev restore_int_flag(flags); 312843e1988Sjohnlev 313843e1988Sjohnlev /* 314843e1988Sjohnlev * See the comments in tsc_gethrtime(), above. 315843e1988Sjohnlev */ 316843e1988Sjohnlev if (tsc >= shadow_tsc_last) 317843e1988Sjohnlev tsc -= shadow_tsc_last; 318843e1988Sjohnlev else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta) 319843e1988Sjohnlev tsc = 0; 320843e1988Sjohnlev 321843e1988Sjohnlev hrt = shadow_tsc_hrtime_base; 322843e1988Sjohnlev 323843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale); 324843e1988Sjohnlev } while ((old_hres_lock & ~1) != shadow_hres_lock); 325843e1988Sjohnlev 326843e1988Sjohnlev return (hrt); 327843e1988Sjohnlev } 328843e1988Sjohnlev 329843e1988Sjohnlev hrtime_t 330843e1988Sjohnlev tsc_gethrtimeunscaled(void) 331843e1988Sjohnlev { 332843e1988Sjohnlev uint32_t old_hres_lock; 333843e1988Sjohnlev hrtime_t tsc; 334843e1988Sjohnlev 335843e1988Sjohnlev do { 336843e1988Sjohnlev old_hres_lock = hres_lock; 337843e1988Sjohnlev 338843e1988Sjohnlev /* See tsc_tick(). */ 339843e1988Sjohnlev tsc = tsc_read() + tsc_last_jumped; 340843e1988Sjohnlev } while ((old_hres_lock & ~1) != hres_lock); 341843e1988Sjohnlev 342843e1988Sjohnlev return (tsc); 343843e1988Sjohnlev } 344843e1988Sjohnlev 345843e1988Sjohnlev 346843e1988Sjohnlev /* Convert a tsc timestamp to nanoseconds */ 347843e1988Sjohnlev void 348843e1988Sjohnlev tsc_scalehrtime(hrtime_t *tsc) 349843e1988Sjohnlev { 350843e1988Sjohnlev hrtime_t hrt; 351843e1988Sjohnlev hrtime_t mytsc; 352843e1988Sjohnlev 353843e1988Sjohnlev if (tsc == NULL) 354843e1988Sjohnlev return; 355843e1988Sjohnlev mytsc = *tsc; 356843e1988Sjohnlev 357843e1988Sjohnlev TSC_CONVERT(mytsc, hrt, nsec_scale); 358843e1988Sjohnlev *tsc = hrt; 359843e1988Sjohnlev } 360843e1988Sjohnlev 361843e1988Sjohnlev hrtime_t 362843e1988Sjohnlev tsc_gethrtimeunscaled_delta(void) 363843e1988Sjohnlev { 364843e1988Sjohnlev hrtime_t hrt; 365a563a037Sbholler ulong_t flags; 366843e1988Sjohnlev 367843e1988Sjohnlev /* 368843e1988Sjohnlev * Similarly to tsc_gethrtime_delta, we need to disable preemption 369843e1988Sjohnlev * to prevent migration between the call to tsc_gethrtimeunscaled 370843e1988Sjohnlev * and adding the CPU's hrtime delta. Note that disabling and 371843e1988Sjohnlev * reenabling preemption is forbidden here because we may be in the 372843e1988Sjohnlev * middle of a fast trap. In the amd64 kernel we cannot tolerate 373843e1988Sjohnlev * preemption during a fast trap. See _update_sregs(). 374843e1988Sjohnlev */ 375843e1988Sjohnlev 376843e1988Sjohnlev flags = clear_int_flag(); 377843e1988Sjohnlev hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id]; 378843e1988Sjohnlev restore_int_flag(flags); 379843e1988Sjohnlev 380843e1988Sjohnlev return (hrt); 381843e1988Sjohnlev } 382843e1988Sjohnlev 3837c478bd9Sstevel@tonic-gate /* 384b3c18020SSudheer A * Called by the master in the TSC sync operation (usually the boot CPU). 385b3c18020SSudheer A * If the slave is discovered to have a skew, gethrtimef will be changed to 386b3c18020SSudheer A * point to tsc_gethrtime_delta(). Calculating skews is precise only when 387b3c18020SSudheer A * the master and slave TSCs are read simultaneously; however, there is no 388b3c18020SSudheer A * algorithm that can read both CPUs in perfect simultaneity. The proposed 389b3c18020SSudheer A * algorithm is an approximate method based on the behaviour of cache 390b3c18020SSudheer A * management. The slave CPU continuously reads TSC and then reads a global 391b3c18020SSudheer A * variable which the master CPU updates. The moment the master's update reaches 392b3c18020SSudheer A * the slave's visibility (being forced by an mfence operation) we use the TSC 393b3c18020SSudheer A * reading taken on the slave. A corresponding TSC read will be taken on the 394b3c18020SSudheer A * master as soon as possible after finishing the mfence operation. But the 395b3c18020SSudheer A * delay between causing the slave to notice the invalid cache line and the 396b3c18020SSudheer A * competion of mfence is not repeatable. This error is heuristically assumed 397b3c18020SSudheer A * to be 1/4th of the total write time as being measured by the two TSC reads 398b3c18020SSudheer A * on the master sandwiching the mfence. Furthermore, due to the nature of 399b3c18020SSudheer A * bus arbitration, contention on memory bus, etc., the time taken for the write 400b3c18020SSudheer A * to reflect globally can vary a lot. So instead of taking a single reading, 401b3c18020SSudheer A * a set of readings are taken and the one with least write time is chosen 402b3c18020SSudheer A * to calculate the final skew. 4034af20bbdSSudheer A * 4044af20bbdSSudheer A * TSC sync is disabled in the context of virtualization because the CPUs 4054af20bbdSSudheer A * assigned to the guest are virtual CPUs which means the real CPUs on which 4064af20bbdSSudheer A * guest runs keep changing during life time of guest OS. So we would end up 4074af20bbdSSudheer A * calculating TSC skews for a set of CPUs during boot whereas the guest 4084af20bbdSSudheer A * might migrate to a different set of physical CPUs at a later point of 4094af20bbdSSudheer A * time. 4107c478bd9Sstevel@tonic-gate */ 4117c478bd9Sstevel@tonic-gate void 4127c478bd9Sstevel@tonic-gate tsc_sync_master(processorid_t slave) 4137c478bd9Sstevel@tonic-gate { 414b3c18020SSudheer A ulong_t flags, source, min_write_time = ~0UL; 415b3c18020SSudheer A hrtime_t write_time, x, mtsc_after, tdelta; 416b3c18020SSudheer A tsc_sync_t *tsc = tscp; 417b3c18020SSudheer A int cnt; 4187c478bd9Sstevel@tonic-gate 419*7997e108SSurya Prakki if (!tsc_master_slave_sync_needed || platform_is_virt) 420ae115bc7Smrj return; 421ae115bc7Smrj 4227c478bd9Sstevel@tonic-gate flags = clear_int_flag(); 423b3c18020SSudheer A source = CPU->cpu_id; 424b3c18020SSudheer A 425b3c18020SSudheer A for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { 426b3c18020SSudheer A while (tsc_sync_go != TSC_SYNC_GO) 427b3c18020SSudheer A SMT_PAUSE(); 428b3c18020SSudheer A 429b3c18020SSudheer A tsc->master_tsc = tsc_read(); 430b3c18020SSudheer A membar_enter(); 431b3c18020SSudheer A mtsc_after = tsc_read(); 432b3c18020SSudheer A while (tsc_sync_go != TSC_SYNC_DONE) 433b3c18020SSudheer A SMT_PAUSE(); 434b3c18020SSudheer A write_time = mtsc_after - tsc->master_tsc; 435b3c18020SSudheer A if (write_time <= min_write_time) { 436b3c18020SSudheer A min_write_time = write_time; 437b3c18020SSudheer A /* 438b3c18020SSudheer A * Apply heuristic adjustment only if the calculated 439b3c18020SSudheer A * delta is > 1/4th of the write time. 440b3c18020SSudheer A */ 441b3c18020SSudheer A x = tsc->slave_tsc - mtsc_after; 442b3c18020SSudheer A if (x < 0) 443b3c18020SSudheer A x = -x; 444b3c18020SSudheer A if (x > (min_write_time/4)) 445b3c18020SSudheer A /* 446b3c18020SSudheer A * Subtract 1/4th of the measured write time 447b3c18020SSudheer A * from the master's TSC value, as an estimate 448b3c18020SSudheer A * of how late the mfence completion came 449b3c18020SSudheer A * after the slave noticed the cache line 450b3c18020SSudheer A * change. 451b3c18020SSudheer A */ 452b3c18020SSudheer A tdelta = tsc->slave_tsc - 453b3c18020SSudheer A (mtsc_after - (min_write_time/4)); 454b3c18020SSudheer A else 455b3c18020SSudheer A tdelta = tsc->slave_tsc - mtsc_after; 456b3c18020SSudheer A tsc_sync_tick_delta[slave] = 457b3c18020SSudheer A tsc_sync_tick_delta[source] - tdelta; 458b3c18020SSudheer A } 4597c478bd9Sstevel@tonic-gate 460b3c18020SSudheer A tsc->master_tsc = tsc->slave_tsc = write_time = 0; 461b3c18020SSudheer A membar_enter(); 462b3c18020SSudheer A tsc_sync_go = TSC_SYNC_STOP; 463b3c18020SSudheer A } 464b3c18020SSudheer A if (tdelta < 0) 465b3c18020SSudheer A tdelta = -tdelta; 466b3c18020SSudheer A if (tdelta > largest_tsc_delta) 467b3c18020SSudheer A largest_tsc_delta = tdelta; 468b3c18020SSudheer A if (min_write_time < shortest_write_time) 469b3c18020SSudheer A shortest_write_time = min_write_time; 4707c478bd9Sstevel@tonic-gate /* 471b3c18020SSudheer A * Enable delta variants of tsc functions if the largest of all chosen 472b3c18020SSudheer A * deltas is > smallest of the write time. 4737c478bd9Sstevel@tonic-gate */ 474b3c18020SSudheer A if (largest_tsc_delta > shortest_write_time) { 475b3c18020SSudheer A gethrtimef = tsc_gethrtime_delta; 476b3c18020SSudheer A gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 477b3c18020SSudheer A } 4787c478bd9Sstevel@tonic-gate restore_int_flag(flags); 4797c478bd9Sstevel@tonic-gate } 4807c478bd9Sstevel@tonic-gate 4814af20bbdSSudheer A /* 4824af20bbdSSudheer A * Called by a CPU which has just been onlined. It is expected that the CPU 4834af20bbdSSudheer A * performing the online operation will call tsc_sync_master(). 4844af20bbdSSudheer A * 4854af20bbdSSudheer A * TSC sync is disabled in the context of virtualization. See comments 4864af20bbdSSudheer A * above tsc_sync_master. 4874af20bbdSSudheer A */ 4887c478bd9Sstevel@tonic-gate void 4897c478bd9Sstevel@tonic-gate tsc_sync_slave(void) 4907c478bd9Sstevel@tonic-gate { 491ae115bc7Smrj ulong_t flags; 492b3c18020SSudheer A hrtime_t s1; 493b3c18020SSudheer A tsc_sync_t *tsc = tscp; 494b3c18020SSudheer A int cnt; 4957c478bd9Sstevel@tonic-gate 496*7997e108SSurya Prakki if (!tsc_master_slave_sync_needed || platform_is_virt) 497ae115bc7Smrj return; 498ae115bc7Smrj 4997c478bd9Sstevel@tonic-gate flags = clear_int_flag(); 5007c478bd9Sstevel@tonic-gate 501b3c18020SSudheer A for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { 502b3c18020SSudheer A /* Re-fill the cache line */ 503b3c18020SSudheer A s1 = tsc->master_tsc; 504b3c18020SSudheer A membar_enter(); 505b3c18020SSudheer A tsc_sync_go = TSC_SYNC_GO; 506b3c18020SSudheer A do { 507b3c18020SSudheer A /* 508b3c18020SSudheer A * Do not put an SMT_PAUSE here. For instance, 509b3c18020SSudheer A * if the master and slave are really the same 510b3c18020SSudheer A * hyper-threaded CPU, then you want the master 511b3c18020SSudheer A * to yield to the slave as quickly as possible here, 512b3c18020SSudheer A * but not the other way. 513b3c18020SSudheer A */ 514b3c18020SSudheer A s1 = tsc_read(); 515b3c18020SSudheer A } while (tsc->master_tsc == 0); 516b3c18020SSudheer A tsc->slave_tsc = s1; 517b3c18020SSudheer A membar_enter(); 518b3c18020SSudheer A tsc_sync_go = TSC_SYNC_DONE; 519b3c18020SSudheer A 520b3c18020SSudheer A while (tsc_sync_go != TSC_SYNC_STOP) 521b3c18020SSudheer A SMT_PAUSE(); 522b3c18020SSudheer A } 5237c478bd9Sstevel@tonic-gate 5247c478bd9Sstevel@tonic-gate restore_int_flag(flags); 5257c478bd9Sstevel@tonic-gate } 5267c478bd9Sstevel@tonic-gate 5277c478bd9Sstevel@tonic-gate /* 528ae115bc7Smrj * Called once per second on a CPU from the cyclic subsystem's 529ae115bc7Smrj * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only) 5307c478bd9Sstevel@tonic-gate */ 5317c478bd9Sstevel@tonic-gate void 5327c478bd9Sstevel@tonic-gate tsc_tick(void) 5337c478bd9Sstevel@tonic-gate { 5347c478bd9Sstevel@tonic-gate hrtime_t now, delta; 5357c478bd9Sstevel@tonic-gate ushort_t spl; 5367c478bd9Sstevel@tonic-gate 5377c478bd9Sstevel@tonic-gate /* 5387c478bd9Sstevel@tonic-gate * Before we set the new variables, we set the shadow values. This 5397c478bd9Sstevel@tonic-gate * allows for lock free operation in dtrace_gethrtime(). 5407c478bd9Sstevel@tonic-gate */ 5417c478bd9Sstevel@tonic-gate lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET, 5427c478bd9Sstevel@tonic-gate ipltospl(CBE_HIGH_PIL), &spl); 5437c478bd9Sstevel@tonic-gate 5447c478bd9Sstevel@tonic-gate shadow_tsc_hrtime_base = tsc_hrtime_base; 5457c478bd9Sstevel@tonic-gate shadow_tsc_last = tsc_last; 5467c478bd9Sstevel@tonic-gate shadow_nsec_scale = nsec_scale; 5477c478bd9Sstevel@tonic-gate 5487c478bd9Sstevel@tonic-gate shadow_hres_lock++; 5497c478bd9Sstevel@tonic-gate splx(spl); 5507c478bd9Sstevel@tonic-gate 5517c478bd9Sstevel@tonic-gate CLOCK_LOCK(&spl); 5527c478bd9Sstevel@tonic-gate 5537c478bd9Sstevel@tonic-gate now = tsc_read(); 5547c478bd9Sstevel@tonic-gate 555d90554ebSdmick if (gethrtimef == tsc_gethrtime_delta) 556d90554ebSdmick now += tsc_sync_tick_delta[CPU->cpu_id]; 557d90554ebSdmick 5587c478bd9Sstevel@tonic-gate if (now < tsc_last) { 5597c478bd9Sstevel@tonic-gate /* 5607c478bd9Sstevel@tonic-gate * The TSC has just jumped into the past. We assume that 5617c478bd9Sstevel@tonic-gate * this is due to a suspend/resume cycle, and we're going 5627c478bd9Sstevel@tonic-gate * to use the _current_ value of TSC as the delta. This 5637c478bd9Sstevel@tonic-gate * will keep tsc_hrtime_base correct. We're also going to 5647c478bd9Sstevel@tonic-gate * assume that rate of tsc does not change after a suspend 5657c478bd9Sstevel@tonic-gate * resume (i.e nsec_scale remains the same). 5667c478bd9Sstevel@tonic-gate */ 5677c478bd9Sstevel@tonic-gate delta = now; 5687c478bd9Sstevel@tonic-gate tsc_last_jumped += tsc_last; 5697c478bd9Sstevel@tonic-gate tsc_jumped = 1; 5707c478bd9Sstevel@tonic-gate } else { 5717c478bd9Sstevel@tonic-gate /* 5727c478bd9Sstevel@tonic-gate * Determine the number of TSC ticks since the last clock 5737c478bd9Sstevel@tonic-gate * tick, and add that to the hrtime base. 5747c478bd9Sstevel@tonic-gate */ 5757c478bd9Sstevel@tonic-gate delta = now - tsc_last; 5767c478bd9Sstevel@tonic-gate } 5777c478bd9Sstevel@tonic-gate 5787c478bd9Sstevel@tonic-gate TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale); 5797c478bd9Sstevel@tonic-gate tsc_last = now; 5807c478bd9Sstevel@tonic-gate 5817c478bd9Sstevel@tonic-gate CLOCK_UNLOCK(spl); 5827c478bd9Sstevel@tonic-gate } 5837c478bd9Sstevel@tonic-gate 5847c478bd9Sstevel@tonic-gate void 585843e1988Sjohnlev tsc_hrtimeinit(uint64_t cpu_freq_hz) 5867c478bd9Sstevel@tonic-gate { 587843e1988Sjohnlev extern int gethrtime_hires; 588843e1988Sjohnlev longlong_t tsc; 589843e1988Sjohnlev ulong_t flags; 5907c478bd9Sstevel@tonic-gate 591843e1988Sjohnlev /* 592843e1988Sjohnlev * cpu_freq_hz is the measured cpu frequency in hertz 593843e1988Sjohnlev */ 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate /* 596843e1988Sjohnlev * We can't accommodate CPUs slower than 31.25 MHz. 5977c478bd9Sstevel@tonic-gate */ 598843e1988Sjohnlev ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT)); 599843e1988Sjohnlev nsec_scale = 600843e1988Sjohnlev (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz); 6017c478bd9Sstevel@tonic-gate 6027c478bd9Sstevel@tonic-gate flags = clear_int_flag(); 603843e1988Sjohnlev tsc = tsc_read(); 604843e1988Sjohnlev (void) tsc_gethrtime(); 605843e1988Sjohnlev tsc_max_delta = tsc_read() - tsc; 6067c478bd9Sstevel@tonic-gate restore_int_flag(flags); 607843e1988Sjohnlev gethrtimef = tsc_gethrtime; 608843e1988Sjohnlev gethrtimeunscaledf = tsc_gethrtimeunscaled; 609843e1988Sjohnlev scalehrtimef = tsc_scalehrtime; 610843e1988Sjohnlev hrtime_tick = tsc_tick; 611843e1988Sjohnlev gethrtime_hires = 1; 612b3c18020SSudheer A /* 613b3c18020SSudheer A * Allocate memory for the structure used in the tsc sync logic. 614b3c18020SSudheer A * This structure should be aligned on a multiple of cache line size. 615b3c18020SSudheer A */ 616b3c18020SSudheer A tscp = kmem_zalloc(PAGESIZE, KM_SLEEP); 6177c478bd9Sstevel@tonic-gate } 6182df1fe9cSrandyf 6192df1fe9cSrandyf int 6202df1fe9cSrandyf get_tsc_ready() 6212df1fe9cSrandyf { 6222df1fe9cSrandyf return (tsc_ready); 6232df1fe9cSrandyf } 6242df1fe9cSrandyf 6252df1fe9cSrandyf /* 6262df1fe9cSrandyf * Adjust all the deltas by adding the passed value to the array. 6272df1fe9cSrandyf * Then use the "delt" versions of the the gethrtime functions. 6282df1fe9cSrandyf * Note that 'tdelta' _could_ be a negative number, which should 6292df1fe9cSrandyf * reduce the values in the array (used, for example, if the Solaris 6302df1fe9cSrandyf * instance was moved by a virtual manager to a machine with a higher 6312df1fe9cSrandyf * value of tsc). 6322df1fe9cSrandyf */ 6332df1fe9cSrandyf void 6342df1fe9cSrandyf tsc_adjust_delta(hrtime_t tdelta) 6352df1fe9cSrandyf { 6362df1fe9cSrandyf int i; 6372df1fe9cSrandyf 6382df1fe9cSrandyf for (i = 0; i < NCPU; i++) { 6392df1fe9cSrandyf tsc_sync_tick_delta[i] += tdelta; 6402df1fe9cSrandyf } 6412df1fe9cSrandyf 6422df1fe9cSrandyf gethrtimef = tsc_gethrtime_delta; 6432df1fe9cSrandyf gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 6442df1fe9cSrandyf } 6452df1fe9cSrandyf 6462df1fe9cSrandyf /* 6472df1fe9cSrandyf * Functions to manage TSC and high-res time on suspend and resume. 6482df1fe9cSrandyf */ 6492df1fe9cSrandyf 6502df1fe9cSrandyf /* 6512df1fe9cSrandyf * declarations needed for time adjustment 6522df1fe9cSrandyf */ 6532df1fe9cSrandyf extern void rtcsync(void); 6542df1fe9cSrandyf extern tod_ops_t *tod_ops; 6552df1fe9cSrandyf /* There must be a better way than exposing nsec_scale! */ 6562df1fe9cSrandyf extern uint_t nsec_scale; 6572df1fe9cSrandyf static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */ 6582df1fe9cSrandyf static timestruc_t tsc_saved_ts; 6592df1fe9cSrandyf static int tsc_needs_resume = 0; /* We only want to do this once. */ 6602df1fe9cSrandyf int tsc_delta_onsuspend = 0; 6612df1fe9cSrandyf int tsc_adjust_seconds = 1; 6622df1fe9cSrandyf int tsc_suspend_count = 0; 6632df1fe9cSrandyf int tsc_resume_in_cyclic = 0; 6642df1fe9cSrandyf 6652df1fe9cSrandyf /* 6662df1fe9cSrandyf * Let timestamp.c know that we are suspending. It needs to take 6672df1fe9cSrandyf * snapshots of the current time, and do any pre-suspend work. 6682df1fe9cSrandyf */ 6692df1fe9cSrandyf void 6702df1fe9cSrandyf tsc_suspend(void) 6712df1fe9cSrandyf { 6722df1fe9cSrandyf /* 6732df1fe9cSrandyf * What we need to do here, is to get the time we suspended, so that we 6742df1fe9cSrandyf * know how much we should add to the resume. 6752df1fe9cSrandyf * This routine is called by each CPU, so we need to handle reentry. 6762df1fe9cSrandyf */ 6772df1fe9cSrandyf if (tsc_gethrtime_enable) { 6782df1fe9cSrandyf /* 6792df1fe9cSrandyf * We put the tsc_read() inside the lock as it 6802df1fe9cSrandyf * as no locking constraints, and it puts the 6812df1fe9cSrandyf * aquired value closer to the time stamp (in 6822df1fe9cSrandyf * case we delay getting the lock). 6832df1fe9cSrandyf */ 6842df1fe9cSrandyf mutex_enter(&tod_lock); 6852df1fe9cSrandyf tsc_saved_tsc = tsc_read(); 6862df1fe9cSrandyf tsc_saved_ts = TODOP_GET(tod_ops); 6872df1fe9cSrandyf mutex_exit(&tod_lock); 6882df1fe9cSrandyf /* We only want to do this once. */ 6892df1fe9cSrandyf if (tsc_needs_resume == 0) { 6902df1fe9cSrandyf if (tsc_delta_onsuspend) { 6912df1fe9cSrandyf tsc_adjust_delta(tsc_saved_tsc); 6922df1fe9cSrandyf } else { 6932df1fe9cSrandyf tsc_adjust_delta(nsec_scale); 6942df1fe9cSrandyf } 6952df1fe9cSrandyf tsc_suspend_count++; 6962df1fe9cSrandyf } 6972df1fe9cSrandyf } 6982df1fe9cSrandyf 6992df1fe9cSrandyf invalidate_cache(); 7002df1fe9cSrandyf tsc_needs_resume = 1; 7012df1fe9cSrandyf } 7022df1fe9cSrandyf 7032df1fe9cSrandyf /* 7042df1fe9cSrandyf * Restore all timestamp state based on the snapshots taken at 7052df1fe9cSrandyf * suspend time. 7062df1fe9cSrandyf */ 7072df1fe9cSrandyf void 7082df1fe9cSrandyf tsc_resume(void) 7092df1fe9cSrandyf { 7102df1fe9cSrandyf /* 7112df1fe9cSrandyf * We only need to (and want to) do this once. So let the first 7122df1fe9cSrandyf * caller handle this (we are locked by the cpu lock), as it 7132df1fe9cSrandyf * is preferential that we get the earliest sync. 7142df1fe9cSrandyf */ 7152df1fe9cSrandyf if (tsc_needs_resume) { 7162df1fe9cSrandyf /* 7172df1fe9cSrandyf * If using the TSC, adjust the delta based on how long 7182df1fe9cSrandyf * we were sleeping (or away). We also adjust for 7192df1fe9cSrandyf * migration and a grown TSC. 7202df1fe9cSrandyf */ 7212df1fe9cSrandyf if (tsc_saved_tsc != 0) { 7222df1fe9cSrandyf timestruc_t ts; 7232df1fe9cSrandyf hrtime_t now, sleep_tsc = 0; 7242df1fe9cSrandyf int sleep_sec; 7252df1fe9cSrandyf extern void tsc_tick(void); 7262df1fe9cSrandyf extern uint64_t cpu_freq_hz; 7272df1fe9cSrandyf 7282df1fe9cSrandyf /* tsc_read() MUST be before TODOP_GET() */ 7292df1fe9cSrandyf mutex_enter(&tod_lock); 7302df1fe9cSrandyf now = tsc_read(); 7312df1fe9cSrandyf ts = TODOP_GET(tod_ops); 7322df1fe9cSrandyf mutex_exit(&tod_lock); 7332df1fe9cSrandyf 7342df1fe9cSrandyf /* Compute seconds of sleep time */ 7352df1fe9cSrandyf sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec; 7362df1fe9cSrandyf 7372df1fe9cSrandyf /* 7382df1fe9cSrandyf * If the saved sec is less that or equal to 7392df1fe9cSrandyf * the current ts, then there is likely a 7402df1fe9cSrandyf * problem with the clock. Assume at least 7412df1fe9cSrandyf * one second has passed, so that time goes forward. 7422df1fe9cSrandyf */ 7432df1fe9cSrandyf if (sleep_sec <= 0) { 7442df1fe9cSrandyf sleep_sec = 1; 7452df1fe9cSrandyf } 7462df1fe9cSrandyf 7472df1fe9cSrandyf /* How many TSC's should have occured while sleeping */ 7482df1fe9cSrandyf if (tsc_adjust_seconds) 7492df1fe9cSrandyf sleep_tsc = sleep_sec * cpu_freq_hz; 7502df1fe9cSrandyf 7512df1fe9cSrandyf /* 7522df1fe9cSrandyf * We also want to subtract from the "sleep_tsc" 7532df1fe9cSrandyf * the current value of tsc_read(), so that our 7542df1fe9cSrandyf * adjustment accounts for the amount of time we 7552df1fe9cSrandyf * have been resumed _or_ an adjustment based on 7562df1fe9cSrandyf * the fact that we didn't actually power off the 7572df1fe9cSrandyf * CPU (migration is another issue, but _should_ 7582df1fe9cSrandyf * also comply with this calculation). If the CPU 7592df1fe9cSrandyf * never powered off, then: 7602df1fe9cSrandyf * 'now == sleep_tsc + saved_tsc' 7612df1fe9cSrandyf * and the delta will effectively be "0". 7622df1fe9cSrandyf */ 7632df1fe9cSrandyf sleep_tsc -= now; 7642df1fe9cSrandyf if (tsc_delta_onsuspend) { 7652df1fe9cSrandyf tsc_adjust_delta(sleep_tsc); 7662df1fe9cSrandyf } else { 7672df1fe9cSrandyf tsc_adjust_delta(tsc_saved_tsc + sleep_tsc); 7682df1fe9cSrandyf } 7692df1fe9cSrandyf tsc_saved_tsc = 0; 7702df1fe9cSrandyf 7712df1fe9cSrandyf tsc_tick(); 7722df1fe9cSrandyf } 7732df1fe9cSrandyf tsc_needs_resume = 0; 7742df1fe9cSrandyf } 7752df1fe9cSrandyf 7762df1fe9cSrandyf } 777