17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5ae115bc7Smrj * Common Development and Distribution License (the "License"). 6ae115bc7Smrj * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21843e1988Sjohnlev 227c478bd9Sstevel@tonic-gate /* 237997e108SSurya Prakki * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 2579ec9da8SYuri Pankov * 2679ec9da8SYuri Pankov * Copyright 2012 Nexenta Systems, Inc. All rights reserved. 277c478bd9Sstevel@tonic-gate */ 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/types.h> 307c478bd9Sstevel@tonic-gate #include <sys/param.h> 317c478bd9Sstevel@tonic-gate #include <sys/systm.h> 327c478bd9Sstevel@tonic-gate #include <sys/disp.h> 337c478bd9Sstevel@tonic-gate #include <sys/var.h> 347c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 357c478bd9Sstevel@tonic-gate #include <sys/debug.h> 367c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h> 377c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 387c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 397c478bd9Sstevel@tonic-gate #include <sys/psm_defs.h> 407c478bd9Sstevel@tonic-gate #include <sys/clock.h> 417c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 427c478bd9Sstevel@tonic-gate #include <sys/lockstat.h> 437c478bd9Sstevel@tonic-gate #include <sys/smp_impldefs.h> 447c478bd9Sstevel@tonic-gate #include <sys/dtrace.h> 457c478bd9Sstevel@tonic-gate #include <sys/time.h> 46843e1988Sjohnlev #include <sys/panic.h> 47b3c18020SSudheer A #include <sys/cpu.h> 487c478bd9Sstevel@tonic-gate 497c478bd9Sstevel@tonic-gate /* 507c478bd9Sstevel@tonic-gate * Using the Pentium's TSC register for gethrtime() 517c478bd9Sstevel@tonic-gate * ------------------------------------------------ 527c478bd9Sstevel@tonic-gate * 537c478bd9Sstevel@tonic-gate * The Pentium family, like many chip architectures, has a high-resolution 547c478bd9Sstevel@tonic-gate * timestamp counter ("TSC") which increments once per CPU cycle. The contents 557c478bd9Sstevel@tonic-gate * of the timestamp counter are read with the RDTSC instruction. 567c478bd9Sstevel@tonic-gate * 577c478bd9Sstevel@tonic-gate * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count 587c478bd9Sstevel@tonic-gate * must be translated into nanoseconds in order to implement gethrtime(). 597c478bd9Sstevel@tonic-gate * We avoid inducing floating point operations in this conversion by 607c478bd9Sstevel@tonic-gate * implementing the same nsec_scale algorithm as that found in the sun4u 617c478bd9Sstevel@tonic-gate * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains 627c478bd9Sstevel@tonic-gate * a detailed description of the algorithm; the comment is not reproduced 637c478bd9Sstevel@tonic-gate * here. This implementation differs only in its value for NSEC_SHIFT: 647c478bd9Sstevel@tonic-gate * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for 657c478bd9Sstevel@tonic-gate * 60 MHz Pentiums. 667c478bd9Sstevel@tonic-gate * 677c478bd9Sstevel@tonic-gate * While TSC and %tick are both cycle counting registers, TSC's functionality 687c478bd9Sstevel@tonic-gate * falls short in several critical ways: 697c478bd9Sstevel@tonic-gate * 707c478bd9Sstevel@tonic-gate * (a) TSCs on different CPUs are not guaranteed to be in sync. While in 717c478bd9Sstevel@tonic-gate * practice they often _are_ in sync, this isn't guaranteed by the 727c478bd9Sstevel@tonic-gate * architecture. 737c478bd9Sstevel@tonic-gate * 747c478bd9Sstevel@tonic-gate * (b) The TSC cannot be reliably set to an arbitrary value. The architecture 757c478bd9Sstevel@tonic-gate * only supports writing the low 32-bits of TSC, making it impractical 767c478bd9Sstevel@tonic-gate * to rewrite. 777c478bd9Sstevel@tonic-gate * 787c478bd9Sstevel@tonic-gate * (c) The architecture doesn't have the capacity to interrupt based on 797c478bd9Sstevel@tonic-gate * arbitrary values of TSC; there is no TICK_CMPR equivalent. 807c478bd9Sstevel@tonic-gate * 817c478bd9Sstevel@tonic-gate * Together, (a) and (b) imply that software must track the skew between 827c478bd9Sstevel@tonic-gate * TSCs and account for it (it is assumed that while there may exist skew, 837c478bd9Sstevel@tonic-gate * there does not exist drift). To determine the skew between CPUs, we 847c478bd9Sstevel@tonic-gate * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing 85b3c18020SSudheer A * the online operation calls tsc_sync_master(). 867c478bd9Sstevel@tonic-gate * 877c478bd9Sstevel@tonic-gate * In the absence of time-of-day clock adjustments, gethrtime() must stay in 887c478bd9Sstevel@tonic-gate * sync with gettimeofday(). This is problematic; given (c), the software 897c478bd9Sstevel@tonic-gate * cannot drive its time-of-day source from TSC, and yet they must somehow be 907c478bd9Sstevel@tonic-gate * kept in sync. We implement this by having a routine, tsc_tick(), which 917c478bd9Sstevel@tonic-gate * is called once per second from the interrupt which drives time-of-day. 927c478bd9Sstevel@tonic-gate * 937c478bd9Sstevel@tonic-gate * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified 947c478bd9Sstevel@tonic-gate * atomically with nsec_scale under CLOCK_LOCK. This assures that time 957c478bd9Sstevel@tonic-gate * monotonically increases. 967c478bd9Sstevel@tonic-gate */ 977c478bd9Sstevel@tonic-gate 987c478bd9Sstevel@tonic-gate #define NSEC_SHIFT 5 997c478bd9Sstevel@tonic-gate 1007c478bd9Sstevel@tonic-gate static uint_t nsec_scale; 101113b131bSEric Saxe static uint_t nsec_unscale; 1027c478bd9Sstevel@tonic-gate 1037c478bd9Sstevel@tonic-gate /* 1047c478bd9Sstevel@tonic-gate * These two variables used to be grouped together inside of a structure that 1057c478bd9Sstevel@tonic-gate * lived on a single cache line. A regression (bug ID 4623398) caused the 1067c478bd9Sstevel@tonic-gate * compiler to emit code that "optimized" away the while-loops below. The 1077c478bd9Sstevel@tonic-gate * result was that no synchronization between the onlining and onlined CPUs 1087c478bd9Sstevel@tonic-gate * took place. 1097c478bd9Sstevel@tonic-gate */ 1107c478bd9Sstevel@tonic-gate static volatile int tsc_ready; 1117c478bd9Sstevel@tonic-gate static volatile int tsc_sync_go; 1127c478bd9Sstevel@tonic-gate 1137c478bd9Sstevel@tonic-gate /* 1147c478bd9Sstevel@tonic-gate * Used as indices into the tsc_sync_snaps[] array. 1157c478bd9Sstevel@tonic-gate */ 1167c478bd9Sstevel@tonic-gate #define TSC_MASTER 0 1177c478bd9Sstevel@tonic-gate #define TSC_SLAVE 1 1187c478bd9Sstevel@tonic-gate 1197c478bd9Sstevel@tonic-gate /* 1207c478bd9Sstevel@tonic-gate * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous. 1217c478bd9Sstevel@tonic-gate */ 1227c478bd9Sstevel@tonic-gate #define TSC_SYNC_STOP 1 1237c478bd9Sstevel@tonic-gate #define TSC_SYNC_GO 2 124b3c18020SSudheer A #define TSC_SYNC_DONE 3 125b3c18020SSudheer A #define SYNC_ITERATIONS 10 1267c478bd9Sstevel@tonic-gate 127843e1988Sjohnlev #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \ 128ae115bc7Smrj unsigned int *_l = (unsigned int *)&(tsc); \ 129ae115bc7Smrj (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \ 1307c478bd9Sstevel@tonic-gate (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 1317c478bd9Sstevel@tonic-gate } 1327c478bd9Sstevel@tonic-gate 133ae115bc7Smrj #define TSC_CONVERT(tsc, hrt, scale) { \ 134ae115bc7Smrj unsigned int *_l = (unsigned int *)&(tsc); \ 135ae115bc7Smrj (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \ 1367c478bd9Sstevel@tonic-gate (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 1377c478bd9Sstevel@tonic-gate } 1387c478bd9Sstevel@tonic-gate 139ae115bc7Smrj int tsc_master_slave_sync_needed = 1; 1407c478bd9Sstevel@tonic-gate 1417c478bd9Sstevel@tonic-gate static int tsc_max_delta; 1427c478bd9Sstevel@tonic-gate static hrtime_t tsc_sync_tick_delta[NCPU]; 143b3c18020SSudheer A typedef struct tsc_sync { 144b3c18020SSudheer A volatile hrtime_t master_tsc, slave_tsc; 145b3c18020SSudheer A } tsc_sync_t; 146b3c18020SSudheer A static tsc_sync_t *tscp; 147b3c18020SSudheer A static hrtime_t largest_tsc_delta = 0; 148b3c18020SSudheer A static ulong_t shortest_write_time = ~0UL; 149b3c18020SSudheer A 1507c478bd9Sstevel@tonic-gate static hrtime_t tsc_last = 0; 1517c478bd9Sstevel@tonic-gate static hrtime_t tsc_last_jumped = 0; 1527c478bd9Sstevel@tonic-gate static hrtime_t tsc_hrtime_base = 0; 1537c478bd9Sstevel@tonic-gate static int tsc_jumped = 0; 1547c478bd9Sstevel@tonic-gate 1557c478bd9Sstevel@tonic-gate static hrtime_t shadow_tsc_hrtime_base; 1567c478bd9Sstevel@tonic-gate static hrtime_t shadow_tsc_last; 1577c478bd9Sstevel@tonic-gate static uint_t shadow_nsec_scale; 1587c478bd9Sstevel@tonic-gate static uint32_t shadow_hres_lock; 1592df1fe9cSrandyf int get_tsc_ready(); 1607c478bd9Sstevel@tonic-gate 161843e1988Sjohnlev hrtime_t 162843e1988Sjohnlev tsc_gethrtime(void) 163843e1988Sjohnlev { 164843e1988Sjohnlev uint32_t old_hres_lock; 165843e1988Sjohnlev hrtime_t tsc, hrt; 166843e1988Sjohnlev 167843e1988Sjohnlev do { 168843e1988Sjohnlev old_hres_lock = hres_lock; 169843e1988Sjohnlev 170843e1988Sjohnlev if ((tsc = tsc_read()) >= tsc_last) { 171843e1988Sjohnlev /* 172843e1988Sjohnlev * It would seem to be obvious that this is true 173843e1988Sjohnlev * (that is, the past is less than the present), 174843e1988Sjohnlev * but it isn't true in the presence of suspend/resume 175843e1988Sjohnlev * cycles. If we manage to call gethrtime() 176843e1988Sjohnlev * after a resume, but before the first call to 177843e1988Sjohnlev * tsc_tick(), we will see the jump. In this case, 178843e1988Sjohnlev * we will simply use the value in TSC as the delta. 179843e1988Sjohnlev */ 180843e1988Sjohnlev tsc -= tsc_last; 181843e1988Sjohnlev } else if (tsc >= tsc_last - 2*tsc_max_delta) { 182843e1988Sjohnlev /* 183843e1988Sjohnlev * There is a chance that tsc_tick() has just run on 184843e1988Sjohnlev * another CPU, and we have drifted just enough so that 185843e1988Sjohnlev * we appear behind tsc_last. In this case, force the 186843e1988Sjohnlev * delta to be zero. 187843e1988Sjohnlev */ 188843e1988Sjohnlev tsc = 0; 189843e1988Sjohnlev } 190843e1988Sjohnlev 191843e1988Sjohnlev hrt = tsc_hrtime_base; 192843e1988Sjohnlev 193843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 194843e1988Sjohnlev } while ((old_hres_lock & ~1) != hres_lock); 195843e1988Sjohnlev 196843e1988Sjohnlev return (hrt); 197843e1988Sjohnlev } 198843e1988Sjohnlev 199843e1988Sjohnlev hrtime_t 200843e1988Sjohnlev tsc_gethrtime_delta(void) 201843e1988Sjohnlev { 202843e1988Sjohnlev uint32_t old_hres_lock; 203843e1988Sjohnlev hrtime_t tsc, hrt; 204a563a037Sbholler ulong_t flags; 205843e1988Sjohnlev 206843e1988Sjohnlev do { 207843e1988Sjohnlev old_hres_lock = hres_lock; 208843e1988Sjohnlev 209843e1988Sjohnlev /* 210843e1988Sjohnlev * We need to disable interrupts here to assure that we 211843e1988Sjohnlev * don't migrate between the call to tsc_read() and 212843e1988Sjohnlev * adding the CPU's TSC tick delta. Note that disabling 213843e1988Sjohnlev * and reenabling preemption is forbidden here because 214843e1988Sjohnlev * we may be in the middle of a fast trap. In the amd64 215843e1988Sjohnlev * kernel we cannot tolerate preemption during a fast 216843e1988Sjohnlev * trap. See _update_sregs(). 217843e1988Sjohnlev */ 218843e1988Sjohnlev 219843e1988Sjohnlev flags = clear_int_flag(); 220843e1988Sjohnlev tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; 221843e1988Sjohnlev restore_int_flag(flags); 222843e1988Sjohnlev 223843e1988Sjohnlev /* See comments in tsc_gethrtime() above */ 224843e1988Sjohnlev 225843e1988Sjohnlev if (tsc >= tsc_last) { 226843e1988Sjohnlev tsc -= tsc_last; 227843e1988Sjohnlev } else if (tsc >= tsc_last - 2 * tsc_max_delta) { 228843e1988Sjohnlev tsc = 0; 229843e1988Sjohnlev } 230843e1988Sjohnlev 231843e1988Sjohnlev hrt = tsc_hrtime_base; 232843e1988Sjohnlev 233843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 234843e1988Sjohnlev } while ((old_hres_lock & ~1) != hres_lock); 235843e1988Sjohnlev 236843e1988Sjohnlev return (hrt); 237843e1988Sjohnlev } 238843e1988Sjohnlev 239*9278ddffSRobert Mustacchi hrtime_t 240*9278ddffSRobert Mustacchi tsc_gethrtime_tick_delta(void) 241*9278ddffSRobert Mustacchi { 242*9278ddffSRobert Mustacchi hrtime_t hrt; 243*9278ddffSRobert Mustacchi ulong_t flags; 244*9278ddffSRobert Mustacchi 245*9278ddffSRobert Mustacchi flags = clear_int_flag(); 246*9278ddffSRobert Mustacchi hrt = tsc_sync_tick_delta[CPU->cpu_id]; 247*9278ddffSRobert Mustacchi restore_int_flag(flags); 248*9278ddffSRobert Mustacchi 249*9278ddffSRobert Mustacchi return (hrt); 250*9278ddffSRobert Mustacchi } 251*9278ddffSRobert Mustacchi 252843e1988Sjohnlev /* 253843e1988Sjohnlev * This is similar to the above, but it cannot actually spin on hres_lock. 254843e1988Sjohnlev * As a result, it caches all of the variables it needs; if the variables 255843e1988Sjohnlev * don't change, it's done. 256843e1988Sjohnlev */ 257843e1988Sjohnlev hrtime_t 258843e1988Sjohnlev dtrace_gethrtime(void) 259843e1988Sjohnlev { 260843e1988Sjohnlev uint32_t old_hres_lock; 261843e1988Sjohnlev hrtime_t tsc, hrt; 262a563a037Sbholler ulong_t flags; 263843e1988Sjohnlev 264843e1988Sjohnlev do { 265843e1988Sjohnlev old_hres_lock = hres_lock; 266843e1988Sjohnlev 267843e1988Sjohnlev /* 268843e1988Sjohnlev * Interrupts are disabled to ensure that the thread isn't 269843e1988Sjohnlev * migrated between the tsc_read() and adding the CPU's 270843e1988Sjohnlev * TSC tick delta. 271843e1988Sjohnlev */ 272843e1988Sjohnlev flags = clear_int_flag(); 273843e1988Sjohnlev 274843e1988Sjohnlev tsc = tsc_read(); 275843e1988Sjohnlev 276843e1988Sjohnlev if (gethrtimef == tsc_gethrtime_delta) 277843e1988Sjohnlev tsc += tsc_sync_tick_delta[CPU->cpu_id]; 278843e1988Sjohnlev 279843e1988Sjohnlev restore_int_flag(flags); 280843e1988Sjohnlev 281843e1988Sjohnlev /* 282843e1988Sjohnlev * See the comments in tsc_gethrtime(), above. 283843e1988Sjohnlev */ 284843e1988Sjohnlev if (tsc >= tsc_last) 285843e1988Sjohnlev tsc -= tsc_last; 286843e1988Sjohnlev else if (tsc >= tsc_last - 2*tsc_max_delta) 287843e1988Sjohnlev tsc = 0; 288843e1988Sjohnlev 289843e1988Sjohnlev hrt = tsc_hrtime_base; 290843e1988Sjohnlev 291843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 292843e1988Sjohnlev 293843e1988Sjohnlev if ((old_hres_lock & ~1) == hres_lock) 294843e1988Sjohnlev break; 295843e1988Sjohnlev 296843e1988Sjohnlev /* 297843e1988Sjohnlev * If we're here, the clock lock is locked -- or it has been 298843e1988Sjohnlev * unlocked and locked since we looked. This may be due to 299843e1988Sjohnlev * tsc_tick() running on another CPU -- or it may be because 300843e1988Sjohnlev * some code path has ended up in dtrace_probe() with 301843e1988Sjohnlev * CLOCK_LOCK held. We'll try to determine that we're in 302843e1988Sjohnlev * the former case by taking another lap if the lock has 303843e1988Sjohnlev * changed since when we first looked at it. 304843e1988Sjohnlev */ 305843e1988Sjohnlev if (old_hres_lock != hres_lock) 306843e1988Sjohnlev continue; 307843e1988Sjohnlev 308843e1988Sjohnlev /* 309843e1988Sjohnlev * So the lock was and is locked. We'll use the old data 310843e1988Sjohnlev * instead. 311843e1988Sjohnlev */ 312843e1988Sjohnlev old_hres_lock = shadow_hres_lock; 313843e1988Sjohnlev 314843e1988Sjohnlev /* 315843e1988Sjohnlev * Again, disable interrupts to ensure that the thread 316843e1988Sjohnlev * isn't migrated between the tsc_read() and adding 317843e1988Sjohnlev * the CPU's TSC tick delta. 318843e1988Sjohnlev */ 319843e1988Sjohnlev flags = clear_int_flag(); 320843e1988Sjohnlev 321843e1988Sjohnlev tsc = tsc_read(); 322843e1988Sjohnlev 323843e1988Sjohnlev if (gethrtimef == tsc_gethrtime_delta) 324843e1988Sjohnlev tsc += tsc_sync_tick_delta[CPU->cpu_id]; 325843e1988Sjohnlev 326843e1988Sjohnlev restore_int_flag(flags); 327843e1988Sjohnlev 328843e1988Sjohnlev /* 329843e1988Sjohnlev * See the comments in tsc_gethrtime(), above. 330843e1988Sjohnlev */ 331843e1988Sjohnlev if (tsc >= shadow_tsc_last) 332843e1988Sjohnlev tsc -= shadow_tsc_last; 333843e1988Sjohnlev else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta) 334843e1988Sjohnlev tsc = 0; 335843e1988Sjohnlev 336843e1988Sjohnlev hrt = shadow_tsc_hrtime_base; 337843e1988Sjohnlev 338843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale); 339843e1988Sjohnlev } while ((old_hres_lock & ~1) != shadow_hres_lock); 340843e1988Sjohnlev 341843e1988Sjohnlev return (hrt); 342843e1988Sjohnlev } 343843e1988Sjohnlev 344843e1988Sjohnlev hrtime_t 345843e1988Sjohnlev tsc_gethrtimeunscaled(void) 346843e1988Sjohnlev { 347843e1988Sjohnlev uint32_t old_hres_lock; 348843e1988Sjohnlev hrtime_t tsc; 349843e1988Sjohnlev 350843e1988Sjohnlev do { 351843e1988Sjohnlev old_hres_lock = hres_lock; 352843e1988Sjohnlev 353843e1988Sjohnlev /* See tsc_tick(). */ 354843e1988Sjohnlev tsc = tsc_read() + tsc_last_jumped; 355843e1988Sjohnlev } while ((old_hres_lock & ~1) != hres_lock); 356843e1988Sjohnlev 357843e1988Sjohnlev return (tsc); 358843e1988Sjohnlev } 359843e1988Sjohnlev 360113b131bSEric Saxe /* 361113b131bSEric Saxe * Convert a nanosecond based timestamp to tsc 362113b131bSEric Saxe */ 363113b131bSEric Saxe uint64_t 364113b131bSEric Saxe tsc_unscalehrtime(hrtime_t nsec) 365113b131bSEric Saxe { 366113b131bSEric Saxe hrtime_t tsc; 367113b131bSEric Saxe 368113b131bSEric Saxe if (tsc_gethrtime_enable) { 369113b131bSEric Saxe TSC_CONVERT(nsec, tsc, nsec_unscale); 370113b131bSEric Saxe return (tsc); 371113b131bSEric Saxe } 372113b131bSEric Saxe return ((uint64_t)nsec); 373113b131bSEric Saxe } 374843e1988Sjohnlev 375843e1988Sjohnlev /* Convert a tsc timestamp to nanoseconds */ 376843e1988Sjohnlev void 377843e1988Sjohnlev tsc_scalehrtime(hrtime_t *tsc) 378843e1988Sjohnlev { 379843e1988Sjohnlev hrtime_t hrt; 380843e1988Sjohnlev hrtime_t mytsc; 381843e1988Sjohnlev 382843e1988Sjohnlev if (tsc == NULL) 383843e1988Sjohnlev return; 384843e1988Sjohnlev mytsc = *tsc; 385843e1988Sjohnlev 386843e1988Sjohnlev TSC_CONVERT(mytsc, hrt, nsec_scale); 387843e1988Sjohnlev *tsc = hrt; 388843e1988Sjohnlev } 389843e1988Sjohnlev 390843e1988Sjohnlev hrtime_t 391843e1988Sjohnlev tsc_gethrtimeunscaled_delta(void) 392843e1988Sjohnlev { 393843e1988Sjohnlev hrtime_t hrt; 394a563a037Sbholler ulong_t flags; 395843e1988Sjohnlev 396843e1988Sjohnlev /* 397843e1988Sjohnlev * Similarly to tsc_gethrtime_delta, we need to disable preemption 398843e1988Sjohnlev * to prevent migration between the call to tsc_gethrtimeunscaled 399843e1988Sjohnlev * and adding the CPU's hrtime delta. Note that disabling and 400843e1988Sjohnlev * reenabling preemption is forbidden here because we may be in the 401843e1988Sjohnlev * middle of a fast trap. In the amd64 kernel we cannot tolerate 402843e1988Sjohnlev * preemption during a fast trap. See _update_sregs(). 403843e1988Sjohnlev */ 404843e1988Sjohnlev 405843e1988Sjohnlev flags = clear_int_flag(); 406843e1988Sjohnlev hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id]; 407843e1988Sjohnlev restore_int_flag(flags); 408843e1988Sjohnlev 409843e1988Sjohnlev return (hrt); 410843e1988Sjohnlev } 411843e1988Sjohnlev 4127c478bd9Sstevel@tonic-gate /* 413b3c18020SSudheer A * Called by the master in the TSC sync operation (usually the boot CPU). 414b3c18020SSudheer A * If the slave is discovered to have a skew, gethrtimef will be changed to 415b3c18020SSudheer A * point to tsc_gethrtime_delta(). Calculating skews is precise only when 416b3c18020SSudheer A * the master and slave TSCs are read simultaneously; however, there is no 417b3c18020SSudheer A * algorithm that can read both CPUs in perfect simultaneity. The proposed 418b3c18020SSudheer A * algorithm is an approximate method based on the behaviour of cache 419b3c18020SSudheer A * management. The slave CPU continuously reads TSC and then reads a global 420b3c18020SSudheer A * variable which the master CPU updates. The moment the master's update reaches 421b3c18020SSudheer A * the slave's visibility (being forced by an mfence operation) we use the TSC 422b3c18020SSudheer A * reading taken on the slave. A corresponding TSC read will be taken on the 423b3c18020SSudheer A * master as soon as possible after finishing the mfence operation. But the 424b3c18020SSudheer A * delay between causing the slave to notice the invalid cache line and the 425b3c18020SSudheer A * competion of mfence is not repeatable. This error is heuristically assumed 426b3c18020SSudheer A * to be 1/4th of the total write time as being measured by the two TSC reads 427b3c18020SSudheer A * on the master sandwiching the mfence. Furthermore, due to the nature of 428b3c18020SSudheer A * bus arbitration, contention on memory bus, etc., the time taken for the write 429b3c18020SSudheer A * to reflect globally can vary a lot. So instead of taking a single reading, 430b3c18020SSudheer A * a set of readings are taken and the one with least write time is chosen 431b3c18020SSudheer A * to calculate the final skew. 4324af20bbdSSudheer A * 4334af20bbdSSudheer A * TSC sync is disabled in the context of virtualization because the CPUs 4344af20bbdSSudheer A * assigned to the guest are virtual CPUs which means the real CPUs on which 4354af20bbdSSudheer A * guest runs keep changing during life time of guest OS. So we would end up 4364af20bbdSSudheer A * calculating TSC skews for a set of CPUs during boot whereas the guest 4374af20bbdSSudheer A * might migrate to a different set of physical CPUs at a later point of 4384af20bbdSSudheer A * time. 4397c478bd9Sstevel@tonic-gate */ 4407c478bd9Sstevel@tonic-gate void 4417c478bd9Sstevel@tonic-gate tsc_sync_master(processorid_t slave) 4427c478bd9Sstevel@tonic-gate { 443b3c18020SSudheer A ulong_t flags, source, min_write_time = ~0UL; 444b3c18020SSudheer A hrtime_t write_time, x, mtsc_after, tdelta; 445b3c18020SSudheer A tsc_sync_t *tsc = tscp; 446b3c18020SSudheer A int cnt; 447b9bfdccdSStuart Maybee int hwtype; 4487c478bd9Sstevel@tonic-gate 449b9bfdccdSStuart Maybee hwtype = get_hwenv(); 45079ec9da8SYuri Pankov if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0) 451ae115bc7Smrj return; 452ae115bc7Smrj 4537c478bd9Sstevel@tonic-gate flags = clear_int_flag(); 454b3c18020SSudheer A source = CPU->cpu_id; 455b3c18020SSudheer A 456b3c18020SSudheer A for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { 457b3c18020SSudheer A while (tsc_sync_go != TSC_SYNC_GO) 458b3c18020SSudheer A SMT_PAUSE(); 459b3c18020SSudheer A 460b3c18020SSudheer A tsc->master_tsc = tsc_read(); 461b3c18020SSudheer A membar_enter(); 462b3c18020SSudheer A mtsc_after = tsc_read(); 463b3c18020SSudheer A while (tsc_sync_go != TSC_SYNC_DONE) 464b3c18020SSudheer A SMT_PAUSE(); 465b3c18020SSudheer A write_time = mtsc_after - tsc->master_tsc; 466b3c18020SSudheer A if (write_time <= min_write_time) { 467b3c18020SSudheer A min_write_time = write_time; 468b3c18020SSudheer A /* 469b3c18020SSudheer A * Apply heuristic adjustment only if the calculated 470b3c18020SSudheer A * delta is > 1/4th of the write time. 471b3c18020SSudheer A */ 472b3c18020SSudheer A x = tsc->slave_tsc - mtsc_after; 473b3c18020SSudheer A if (x < 0) 474b3c18020SSudheer A x = -x; 475b3c18020SSudheer A if (x > (min_write_time/4)) 476b3c18020SSudheer A /* 477b3c18020SSudheer A * Subtract 1/4th of the measured write time 478b3c18020SSudheer A * from the master's TSC value, as an estimate 479b3c18020SSudheer A * of how late the mfence completion came 480b3c18020SSudheer A * after the slave noticed the cache line 481b3c18020SSudheer A * change. 482b3c18020SSudheer A */ 483b3c18020SSudheer A tdelta = tsc->slave_tsc - 484b3c18020SSudheer A (mtsc_after - (min_write_time/4)); 485b3c18020SSudheer A else 486b3c18020SSudheer A tdelta = tsc->slave_tsc - mtsc_after; 487b3c18020SSudheer A tsc_sync_tick_delta[slave] = 488b3c18020SSudheer A tsc_sync_tick_delta[source] - tdelta; 489b3c18020SSudheer A } 4907c478bd9Sstevel@tonic-gate 491b3c18020SSudheer A tsc->master_tsc = tsc->slave_tsc = write_time = 0; 492b3c18020SSudheer A membar_enter(); 493b3c18020SSudheer A tsc_sync_go = TSC_SYNC_STOP; 494b3c18020SSudheer A } 495b3c18020SSudheer A if (tdelta < 0) 496b3c18020SSudheer A tdelta = -tdelta; 497b3c18020SSudheer A if (tdelta > largest_tsc_delta) 498b3c18020SSudheer A largest_tsc_delta = tdelta; 499b3c18020SSudheer A if (min_write_time < shortest_write_time) 500b3c18020SSudheer A shortest_write_time = min_write_time; 5017c478bd9Sstevel@tonic-gate /* 502b3c18020SSudheer A * Enable delta variants of tsc functions if the largest of all chosen 503b3c18020SSudheer A * deltas is > smallest of the write time. 5047c478bd9Sstevel@tonic-gate */ 505b3c18020SSudheer A if (largest_tsc_delta > shortest_write_time) { 506b3c18020SSudheer A gethrtimef = tsc_gethrtime_delta; 507b3c18020SSudheer A gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 508b3c18020SSudheer A } 5097c478bd9Sstevel@tonic-gate restore_int_flag(flags); 5107c478bd9Sstevel@tonic-gate } 5117c478bd9Sstevel@tonic-gate 5124af20bbdSSudheer A /* 5134af20bbdSSudheer A * Called by a CPU which has just been onlined. It is expected that the CPU 5144af20bbdSSudheer A * performing the online operation will call tsc_sync_master(). 5154af20bbdSSudheer A * 5164af20bbdSSudheer A * TSC sync is disabled in the context of virtualization. See comments 5174af20bbdSSudheer A * above tsc_sync_master. 5184af20bbdSSudheer A */ 5197c478bd9Sstevel@tonic-gate void 5207c478bd9Sstevel@tonic-gate tsc_sync_slave(void) 5217c478bd9Sstevel@tonic-gate { 522ae115bc7Smrj ulong_t flags; 523b3c18020SSudheer A hrtime_t s1; 524b3c18020SSudheer A tsc_sync_t *tsc = tscp; 525b3c18020SSudheer A int cnt; 526b9bfdccdSStuart Maybee int hwtype; 5277c478bd9Sstevel@tonic-gate 528b9bfdccdSStuart Maybee hwtype = get_hwenv(); 52979ec9da8SYuri Pankov if (!tsc_master_slave_sync_needed || (hwtype & HW_VIRTUAL) != 0) 530ae115bc7Smrj return; 531ae115bc7Smrj 5327c478bd9Sstevel@tonic-gate flags = clear_int_flag(); 5337c478bd9Sstevel@tonic-gate 534b3c18020SSudheer A for (cnt = 0; cnt < SYNC_ITERATIONS; cnt++) { 535b3c18020SSudheer A /* Re-fill the cache line */ 536b3c18020SSudheer A s1 = tsc->master_tsc; 537b3c18020SSudheer A membar_enter(); 538b3c18020SSudheer A tsc_sync_go = TSC_SYNC_GO; 539b3c18020SSudheer A do { 540b3c18020SSudheer A /* 541b3c18020SSudheer A * Do not put an SMT_PAUSE here. For instance, 542b3c18020SSudheer A * if the master and slave are really the same 543b3c18020SSudheer A * hyper-threaded CPU, then you want the master 544b3c18020SSudheer A * to yield to the slave as quickly as possible here, 545b3c18020SSudheer A * but not the other way. 546b3c18020SSudheer A */ 547b3c18020SSudheer A s1 = tsc_read(); 548b3c18020SSudheer A } while (tsc->master_tsc == 0); 549b3c18020SSudheer A tsc->slave_tsc = s1; 550b3c18020SSudheer A membar_enter(); 551b3c18020SSudheer A tsc_sync_go = TSC_SYNC_DONE; 552b3c18020SSudheer A 553b3c18020SSudheer A while (tsc_sync_go != TSC_SYNC_STOP) 554b3c18020SSudheer A SMT_PAUSE(); 555b3c18020SSudheer A } 5567c478bd9Sstevel@tonic-gate 5577c478bd9Sstevel@tonic-gate restore_int_flag(flags); 5587c478bd9Sstevel@tonic-gate } 5597c478bd9Sstevel@tonic-gate 5607c478bd9Sstevel@tonic-gate /* 561ae115bc7Smrj * Called once per second on a CPU from the cyclic subsystem's 562ae115bc7Smrj * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only) 5637c478bd9Sstevel@tonic-gate */ 5647c478bd9Sstevel@tonic-gate void 5657c478bd9Sstevel@tonic-gate tsc_tick(void) 5667c478bd9Sstevel@tonic-gate { 5677c478bd9Sstevel@tonic-gate hrtime_t now, delta; 5687c478bd9Sstevel@tonic-gate ushort_t spl; 5697c478bd9Sstevel@tonic-gate 5707c478bd9Sstevel@tonic-gate /* 5717c478bd9Sstevel@tonic-gate * Before we set the new variables, we set the shadow values. This 5727c478bd9Sstevel@tonic-gate * allows for lock free operation in dtrace_gethrtime(). 5737c478bd9Sstevel@tonic-gate */ 5747c478bd9Sstevel@tonic-gate lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET, 5757c478bd9Sstevel@tonic-gate ipltospl(CBE_HIGH_PIL), &spl); 5767c478bd9Sstevel@tonic-gate 5777c478bd9Sstevel@tonic-gate shadow_tsc_hrtime_base = tsc_hrtime_base; 5787c478bd9Sstevel@tonic-gate shadow_tsc_last = tsc_last; 5797c478bd9Sstevel@tonic-gate shadow_nsec_scale = nsec_scale; 5807c478bd9Sstevel@tonic-gate 5817c478bd9Sstevel@tonic-gate shadow_hres_lock++; 5827c478bd9Sstevel@tonic-gate splx(spl); 5837c478bd9Sstevel@tonic-gate 5847c478bd9Sstevel@tonic-gate CLOCK_LOCK(&spl); 5857c478bd9Sstevel@tonic-gate 5867c478bd9Sstevel@tonic-gate now = tsc_read(); 5877c478bd9Sstevel@tonic-gate 588d90554ebSdmick if (gethrtimef == tsc_gethrtime_delta) 589d90554ebSdmick now += tsc_sync_tick_delta[CPU->cpu_id]; 590d90554ebSdmick 5917c478bd9Sstevel@tonic-gate if (now < tsc_last) { 5927c478bd9Sstevel@tonic-gate /* 5937c478bd9Sstevel@tonic-gate * The TSC has just jumped into the past. We assume that 5947c478bd9Sstevel@tonic-gate * this is due to a suspend/resume cycle, and we're going 5957c478bd9Sstevel@tonic-gate * to use the _current_ value of TSC as the delta. This 5967c478bd9Sstevel@tonic-gate * will keep tsc_hrtime_base correct. We're also going to 5977c478bd9Sstevel@tonic-gate * assume that rate of tsc does not change after a suspend 5987c478bd9Sstevel@tonic-gate * resume (i.e nsec_scale remains the same). 5997c478bd9Sstevel@tonic-gate */ 6007c478bd9Sstevel@tonic-gate delta = now; 6017c478bd9Sstevel@tonic-gate tsc_last_jumped += tsc_last; 6027c478bd9Sstevel@tonic-gate tsc_jumped = 1; 6037c478bd9Sstevel@tonic-gate } else { 6047c478bd9Sstevel@tonic-gate /* 6057c478bd9Sstevel@tonic-gate * Determine the number of TSC ticks since the last clock 6067c478bd9Sstevel@tonic-gate * tick, and add that to the hrtime base. 6077c478bd9Sstevel@tonic-gate */ 6087c478bd9Sstevel@tonic-gate delta = now - tsc_last; 6097c478bd9Sstevel@tonic-gate } 6107c478bd9Sstevel@tonic-gate 6117c478bd9Sstevel@tonic-gate TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale); 6127c478bd9Sstevel@tonic-gate tsc_last = now; 6137c478bd9Sstevel@tonic-gate 6147c478bd9Sstevel@tonic-gate CLOCK_UNLOCK(spl); 6157c478bd9Sstevel@tonic-gate } 6167c478bd9Sstevel@tonic-gate 6177c478bd9Sstevel@tonic-gate void 618843e1988Sjohnlev tsc_hrtimeinit(uint64_t cpu_freq_hz) 6197c478bd9Sstevel@tonic-gate { 620843e1988Sjohnlev extern int gethrtime_hires; 621843e1988Sjohnlev longlong_t tsc; 622843e1988Sjohnlev ulong_t flags; 6237c478bd9Sstevel@tonic-gate 624843e1988Sjohnlev /* 625843e1988Sjohnlev * cpu_freq_hz is the measured cpu frequency in hertz 626843e1988Sjohnlev */ 6277c478bd9Sstevel@tonic-gate 6287c478bd9Sstevel@tonic-gate /* 629843e1988Sjohnlev * We can't accommodate CPUs slower than 31.25 MHz. 6307c478bd9Sstevel@tonic-gate */ 631843e1988Sjohnlev ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT)); 632843e1988Sjohnlev nsec_scale = 633843e1988Sjohnlev (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz); 634113b131bSEric Saxe nsec_unscale = 635113b131bSEric Saxe (uint_t)(((uint64_t)cpu_freq_hz << (32 - NSEC_SHIFT)) / NANOSEC); 6367c478bd9Sstevel@tonic-gate 6377c478bd9Sstevel@tonic-gate flags = clear_int_flag(); 638843e1988Sjohnlev tsc = tsc_read(); 639843e1988Sjohnlev (void) tsc_gethrtime(); 640843e1988Sjohnlev tsc_max_delta = tsc_read() - tsc; 6417c478bd9Sstevel@tonic-gate restore_int_flag(flags); 642843e1988Sjohnlev gethrtimef = tsc_gethrtime; 643843e1988Sjohnlev gethrtimeunscaledf = tsc_gethrtimeunscaled; 644843e1988Sjohnlev scalehrtimef = tsc_scalehrtime; 645113b131bSEric Saxe unscalehrtimef = tsc_unscalehrtime; 646843e1988Sjohnlev hrtime_tick = tsc_tick; 647843e1988Sjohnlev gethrtime_hires = 1; 648b3c18020SSudheer A /* 649b3c18020SSudheer A * Allocate memory for the structure used in the tsc sync logic. 650b3c18020SSudheer A * This structure should be aligned on a multiple of cache line size. 651b3c18020SSudheer A */ 652b3c18020SSudheer A tscp = kmem_zalloc(PAGESIZE, KM_SLEEP); 6537c478bd9Sstevel@tonic-gate } 6542df1fe9cSrandyf 6552df1fe9cSrandyf int 6562df1fe9cSrandyf get_tsc_ready() 6572df1fe9cSrandyf { 6582df1fe9cSrandyf return (tsc_ready); 6592df1fe9cSrandyf } 6602df1fe9cSrandyf 6612df1fe9cSrandyf /* 6622df1fe9cSrandyf * Adjust all the deltas by adding the passed value to the array. 6632df1fe9cSrandyf * Then use the "delt" versions of the the gethrtime functions. 6642df1fe9cSrandyf * Note that 'tdelta' _could_ be a negative number, which should 6652df1fe9cSrandyf * reduce the values in the array (used, for example, if the Solaris 6662df1fe9cSrandyf * instance was moved by a virtual manager to a machine with a higher 6672df1fe9cSrandyf * value of tsc). 6682df1fe9cSrandyf */ 6692df1fe9cSrandyf void 6702df1fe9cSrandyf tsc_adjust_delta(hrtime_t tdelta) 6712df1fe9cSrandyf { 6722df1fe9cSrandyf int i; 6732df1fe9cSrandyf 6742df1fe9cSrandyf for (i = 0; i < NCPU; i++) { 6752df1fe9cSrandyf tsc_sync_tick_delta[i] += tdelta; 6762df1fe9cSrandyf } 6772df1fe9cSrandyf 6782df1fe9cSrandyf gethrtimef = tsc_gethrtime_delta; 6792df1fe9cSrandyf gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 6802df1fe9cSrandyf } 6812df1fe9cSrandyf 6822df1fe9cSrandyf /* 6832df1fe9cSrandyf * Functions to manage TSC and high-res time on suspend and resume. 6842df1fe9cSrandyf */ 6852df1fe9cSrandyf 6862df1fe9cSrandyf /* 6872df1fe9cSrandyf * declarations needed for time adjustment 6882df1fe9cSrandyf */ 6892df1fe9cSrandyf extern void rtcsync(void); 6902df1fe9cSrandyf extern tod_ops_t *tod_ops; 6912df1fe9cSrandyf /* There must be a better way than exposing nsec_scale! */ 6922df1fe9cSrandyf extern uint_t nsec_scale; 6932df1fe9cSrandyf static uint64_t tsc_saved_tsc = 0; /* 1 in 2^64 chance this'll screw up! */ 6942df1fe9cSrandyf static timestruc_t tsc_saved_ts; 6952df1fe9cSrandyf static int tsc_needs_resume = 0; /* We only want to do this once. */ 6962df1fe9cSrandyf int tsc_delta_onsuspend = 0; 6972df1fe9cSrandyf int tsc_adjust_seconds = 1; 6982df1fe9cSrandyf int tsc_suspend_count = 0; 6992df1fe9cSrandyf int tsc_resume_in_cyclic = 0; 7002df1fe9cSrandyf 7012df1fe9cSrandyf /* 7022df1fe9cSrandyf * Let timestamp.c know that we are suspending. It needs to take 7032df1fe9cSrandyf * snapshots of the current time, and do any pre-suspend work. 7042df1fe9cSrandyf */ 7052df1fe9cSrandyf void 7062df1fe9cSrandyf tsc_suspend(void) 7072df1fe9cSrandyf { 7082df1fe9cSrandyf /* 7092df1fe9cSrandyf * What we need to do here, is to get the time we suspended, so that we 7102df1fe9cSrandyf * know how much we should add to the resume. 7112df1fe9cSrandyf * This routine is called by each CPU, so we need to handle reentry. 7122df1fe9cSrandyf */ 7132df1fe9cSrandyf if (tsc_gethrtime_enable) { 7142df1fe9cSrandyf /* 7152df1fe9cSrandyf * We put the tsc_read() inside the lock as it 7162df1fe9cSrandyf * as no locking constraints, and it puts the 7172df1fe9cSrandyf * aquired value closer to the time stamp (in 7182df1fe9cSrandyf * case we delay getting the lock). 7192df1fe9cSrandyf */ 7202df1fe9cSrandyf mutex_enter(&tod_lock); 7212df1fe9cSrandyf tsc_saved_tsc = tsc_read(); 7222df1fe9cSrandyf tsc_saved_ts = TODOP_GET(tod_ops); 7232df1fe9cSrandyf mutex_exit(&tod_lock); 7242df1fe9cSrandyf /* We only want to do this once. */ 7252df1fe9cSrandyf if (tsc_needs_resume == 0) { 7262df1fe9cSrandyf if (tsc_delta_onsuspend) { 7272df1fe9cSrandyf tsc_adjust_delta(tsc_saved_tsc); 7282df1fe9cSrandyf } else { 7292df1fe9cSrandyf tsc_adjust_delta(nsec_scale); 7302df1fe9cSrandyf } 7312df1fe9cSrandyf tsc_suspend_count++; 7322df1fe9cSrandyf } 7332df1fe9cSrandyf } 7342df1fe9cSrandyf 7352df1fe9cSrandyf invalidate_cache(); 7362df1fe9cSrandyf tsc_needs_resume = 1; 7372df1fe9cSrandyf } 7382df1fe9cSrandyf 7392df1fe9cSrandyf /* 7402df1fe9cSrandyf * Restore all timestamp state based on the snapshots taken at 7412df1fe9cSrandyf * suspend time. 7422df1fe9cSrandyf */ 7432df1fe9cSrandyf void 7442df1fe9cSrandyf tsc_resume(void) 7452df1fe9cSrandyf { 7462df1fe9cSrandyf /* 7472df1fe9cSrandyf * We only need to (and want to) do this once. So let the first 7482df1fe9cSrandyf * caller handle this (we are locked by the cpu lock), as it 7492df1fe9cSrandyf * is preferential that we get the earliest sync. 7502df1fe9cSrandyf */ 7512df1fe9cSrandyf if (tsc_needs_resume) { 7522df1fe9cSrandyf /* 7532df1fe9cSrandyf * If using the TSC, adjust the delta based on how long 7542df1fe9cSrandyf * we were sleeping (or away). We also adjust for 7552df1fe9cSrandyf * migration and a grown TSC. 7562df1fe9cSrandyf */ 7572df1fe9cSrandyf if (tsc_saved_tsc != 0) { 7582df1fe9cSrandyf timestruc_t ts; 7592df1fe9cSrandyf hrtime_t now, sleep_tsc = 0; 7602df1fe9cSrandyf int sleep_sec; 7612df1fe9cSrandyf extern void tsc_tick(void); 7622df1fe9cSrandyf extern uint64_t cpu_freq_hz; 7632df1fe9cSrandyf 7642df1fe9cSrandyf /* tsc_read() MUST be before TODOP_GET() */ 7652df1fe9cSrandyf mutex_enter(&tod_lock); 7662df1fe9cSrandyf now = tsc_read(); 7672df1fe9cSrandyf ts = TODOP_GET(tod_ops); 7682df1fe9cSrandyf mutex_exit(&tod_lock); 7692df1fe9cSrandyf 7702df1fe9cSrandyf /* Compute seconds of sleep time */ 7712df1fe9cSrandyf sleep_sec = ts.tv_sec - tsc_saved_ts.tv_sec; 7722df1fe9cSrandyf 7732df1fe9cSrandyf /* 7742df1fe9cSrandyf * If the saved sec is less that or equal to 7752df1fe9cSrandyf * the current ts, then there is likely a 7762df1fe9cSrandyf * problem with the clock. Assume at least 7772df1fe9cSrandyf * one second has passed, so that time goes forward. 7782df1fe9cSrandyf */ 7792df1fe9cSrandyf if (sleep_sec <= 0) { 7802df1fe9cSrandyf sleep_sec = 1; 7812df1fe9cSrandyf } 7822df1fe9cSrandyf 7832df1fe9cSrandyf /* How many TSC's should have occured while sleeping */ 7842df1fe9cSrandyf if (tsc_adjust_seconds) 7852df1fe9cSrandyf sleep_tsc = sleep_sec * cpu_freq_hz; 7862df1fe9cSrandyf 7872df1fe9cSrandyf /* 7882df1fe9cSrandyf * We also want to subtract from the "sleep_tsc" 7892df1fe9cSrandyf * the current value of tsc_read(), so that our 7902df1fe9cSrandyf * adjustment accounts for the amount of time we 7912df1fe9cSrandyf * have been resumed _or_ an adjustment based on 7922df1fe9cSrandyf * the fact that we didn't actually power off the 7932df1fe9cSrandyf * CPU (migration is another issue, but _should_ 7942df1fe9cSrandyf * also comply with this calculation). If the CPU 7952df1fe9cSrandyf * never powered off, then: 7962df1fe9cSrandyf * 'now == sleep_tsc + saved_tsc' 7972df1fe9cSrandyf * and the delta will effectively be "0". 7982df1fe9cSrandyf */ 7992df1fe9cSrandyf sleep_tsc -= now; 8002df1fe9cSrandyf if (tsc_delta_onsuspend) { 8012df1fe9cSrandyf tsc_adjust_delta(sleep_tsc); 8022df1fe9cSrandyf } else { 8032df1fe9cSrandyf tsc_adjust_delta(tsc_saved_tsc + sleep_tsc); 8042df1fe9cSrandyf } 8052df1fe9cSrandyf tsc_saved_tsc = 0; 8062df1fe9cSrandyf 8072df1fe9cSrandyf tsc_tick(); 8082df1fe9cSrandyf } 8092df1fe9cSrandyf tsc_needs_resume = 0; 8102df1fe9cSrandyf } 8112df1fe9cSrandyf 8122df1fe9cSrandyf } 813