17c478bd9Sstevel@tonic-gate /* 27c478bd9Sstevel@tonic-gate * CDDL HEADER START 37c478bd9Sstevel@tonic-gate * 47c478bd9Sstevel@tonic-gate * The contents of this file are subject to the terms of the 5ae115bc7Smrj * Common Development and Distribution License (the "License"). 6ae115bc7Smrj * You may not use this file except in compliance with the License. 77c478bd9Sstevel@tonic-gate * 87c478bd9Sstevel@tonic-gate * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 97c478bd9Sstevel@tonic-gate * or http://www.opensolaris.org/os/licensing. 107c478bd9Sstevel@tonic-gate * See the License for the specific language governing permissions 117c478bd9Sstevel@tonic-gate * and limitations under the License. 127c478bd9Sstevel@tonic-gate * 137c478bd9Sstevel@tonic-gate * When distributing Covered Code, include this CDDL HEADER in each 147c478bd9Sstevel@tonic-gate * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 157c478bd9Sstevel@tonic-gate * If applicable, add the following below this CDDL HEADER, with the 167c478bd9Sstevel@tonic-gate * fields enclosed by brackets "[]" replaced with your own identifying 177c478bd9Sstevel@tonic-gate * information: Portions Copyright [yyyy] [name of copyright owner] 187c478bd9Sstevel@tonic-gate * 197c478bd9Sstevel@tonic-gate * CDDL HEADER END 207c478bd9Sstevel@tonic-gate */ 21*843e1988Sjohnlev 227c478bd9Sstevel@tonic-gate /* 23ae115bc7Smrj * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 247c478bd9Sstevel@tonic-gate * Use is subject to license terms. 257c478bd9Sstevel@tonic-gate */ 267c478bd9Sstevel@tonic-gate 277c478bd9Sstevel@tonic-gate #pragma ident "%Z%%M% %I% %E% SMI" 287c478bd9Sstevel@tonic-gate 297c478bd9Sstevel@tonic-gate #include <sys/types.h> 307c478bd9Sstevel@tonic-gate #include <sys/param.h> 317c478bd9Sstevel@tonic-gate #include <sys/systm.h> 327c478bd9Sstevel@tonic-gate #include <sys/disp.h> 337c478bd9Sstevel@tonic-gate #include <sys/var.h> 347c478bd9Sstevel@tonic-gate #include <sys/cmn_err.h> 357c478bd9Sstevel@tonic-gate #include <sys/debug.h> 367c478bd9Sstevel@tonic-gate #include <sys/x86_archext.h> 377c478bd9Sstevel@tonic-gate #include <sys/archsystm.h> 387c478bd9Sstevel@tonic-gate #include <sys/cpuvar.h> 397c478bd9Sstevel@tonic-gate #include <sys/psm_defs.h> 407c478bd9Sstevel@tonic-gate #include <sys/clock.h> 417c478bd9Sstevel@tonic-gate #include <sys/atomic.h> 427c478bd9Sstevel@tonic-gate #include <sys/lockstat.h> 437c478bd9Sstevel@tonic-gate #include <sys/smp_impldefs.h> 447c478bd9Sstevel@tonic-gate #include <sys/dtrace.h> 457c478bd9Sstevel@tonic-gate #include <sys/time.h> 46*843e1988Sjohnlev #include <sys/panic.h> 477c478bd9Sstevel@tonic-gate 487c478bd9Sstevel@tonic-gate /* 497c478bd9Sstevel@tonic-gate * Using the Pentium's TSC register for gethrtime() 507c478bd9Sstevel@tonic-gate * ------------------------------------------------ 517c478bd9Sstevel@tonic-gate * 527c478bd9Sstevel@tonic-gate * The Pentium family, like many chip architectures, has a high-resolution 537c478bd9Sstevel@tonic-gate * timestamp counter ("TSC") which increments once per CPU cycle. The contents 547c478bd9Sstevel@tonic-gate * of the timestamp counter are read with the RDTSC instruction. 557c478bd9Sstevel@tonic-gate * 567c478bd9Sstevel@tonic-gate * As with its UltraSPARC equivalent (the %tick register), TSC's cycle count 577c478bd9Sstevel@tonic-gate * must be translated into nanoseconds in order to implement gethrtime(). 587c478bd9Sstevel@tonic-gate * We avoid inducing floating point operations in this conversion by 597c478bd9Sstevel@tonic-gate * implementing the same nsec_scale algorithm as that found in the sun4u 607c478bd9Sstevel@tonic-gate * platform code. The sun4u NATIVE_TIME_TO_NSEC_SCALE block comment contains 617c478bd9Sstevel@tonic-gate * a detailed description of the algorithm; the comment is not reproduced 627c478bd9Sstevel@tonic-gate * here. This implementation differs only in its value for NSEC_SHIFT: 637c478bd9Sstevel@tonic-gate * we implement an NSEC_SHIFT of 5 (instead of sun4u's 4) to allow for 647c478bd9Sstevel@tonic-gate * 60 MHz Pentiums. 657c478bd9Sstevel@tonic-gate * 667c478bd9Sstevel@tonic-gate * While TSC and %tick are both cycle counting registers, TSC's functionality 677c478bd9Sstevel@tonic-gate * falls short in several critical ways: 687c478bd9Sstevel@tonic-gate * 697c478bd9Sstevel@tonic-gate * (a) TSCs on different CPUs are not guaranteed to be in sync. While in 707c478bd9Sstevel@tonic-gate * practice they often _are_ in sync, this isn't guaranteed by the 717c478bd9Sstevel@tonic-gate * architecture. 727c478bd9Sstevel@tonic-gate * 737c478bd9Sstevel@tonic-gate * (b) The TSC cannot be reliably set to an arbitrary value. The architecture 747c478bd9Sstevel@tonic-gate * only supports writing the low 32-bits of TSC, making it impractical 757c478bd9Sstevel@tonic-gate * to rewrite. 767c478bd9Sstevel@tonic-gate * 777c478bd9Sstevel@tonic-gate * (c) The architecture doesn't have the capacity to interrupt based on 787c478bd9Sstevel@tonic-gate * arbitrary values of TSC; there is no TICK_CMPR equivalent. 797c478bd9Sstevel@tonic-gate * 807c478bd9Sstevel@tonic-gate * Together, (a) and (b) imply that software must track the skew between 817c478bd9Sstevel@tonic-gate * TSCs and account for it (it is assumed that while there may exist skew, 827c478bd9Sstevel@tonic-gate * there does not exist drift). To determine the skew between CPUs, we 837c478bd9Sstevel@tonic-gate * have newly onlined CPUs call tsc_sync_slave(), while the CPU performing 847c478bd9Sstevel@tonic-gate * the online operation calls tsc_sync_master(). Once both CPUs are ready, 857c478bd9Sstevel@tonic-gate * the master sets a shared flag, and each reads its TSC register. To reduce 867c478bd9Sstevel@tonic-gate * bias, we then wait until both CPUs are ready again, but this time the 877c478bd9Sstevel@tonic-gate * slave sets the shared flag, and each reads its TSC register again. The 887c478bd9Sstevel@tonic-gate * master compares the average of the two sample values, and, if observable 897c478bd9Sstevel@tonic-gate * skew is found, changes the gethrtimef function pointer to point to a 907c478bd9Sstevel@tonic-gate * gethrtime() implementation which will take the discovered skew into 917c478bd9Sstevel@tonic-gate * consideration. 927c478bd9Sstevel@tonic-gate * 937c478bd9Sstevel@tonic-gate * In the absence of time-of-day clock adjustments, gethrtime() must stay in 947c478bd9Sstevel@tonic-gate * sync with gettimeofday(). This is problematic; given (c), the software 957c478bd9Sstevel@tonic-gate * cannot drive its time-of-day source from TSC, and yet they must somehow be 967c478bd9Sstevel@tonic-gate * kept in sync. We implement this by having a routine, tsc_tick(), which 977c478bd9Sstevel@tonic-gate * is called once per second from the interrupt which drives time-of-day. 987c478bd9Sstevel@tonic-gate * tsc_tick() recalculates nsec_scale based on the number of the CPU cycles 997c478bd9Sstevel@tonic-gate * since boot versus the number of seconds since boot. This algorithm 1007c478bd9Sstevel@tonic-gate * becomes more accurate over time and converges quickly; the error in 1017c478bd9Sstevel@tonic-gate * nsec_scale is typically under 1 ppm less than 10 seconds after boot, and 1027c478bd9Sstevel@tonic-gate * is less than 100 ppb 1 minute after boot. 1037c478bd9Sstevel@tonic-gate * 1047c478bd9Sstevel@tonic-gate * Note that the hrtime base for gethrtime, tsc_hrtime_base, is modified 1057c478bd9Sstevel@tonic-gate * atomically with nsec_scale under CLOCK_LOCK. This assures that time 1067c478bd9Sstevel@tonic-gate * monotonically increases. 1077c478bd9Sstevel@tonic-gate */ 1087c478bd9Sstevel@tonic-gate 1097c478bd9Sstevel@tonic-gate #define NSEC_SHIFT 5 1107c478bd9Sstevel@tonic-gate 1117c478bd9Sstevel@tonic-gate static uint_t nsec_scale; 1127c478bd9Sstevel@tonic-gate 1137c478bd9Sstevel@tonic-gate /* 1147c478bd9Sstevel@tonic-gate * These two variables used to be grouped together inside of a structure that 1157c478bd9Sstevel@tonic-gate * lived on a single cache line. A regression (bug ID 4623398) caused the 1167c478bd9Sstevel@tonic-gate * compiler to emit code that "optimized" away the while-loops below. The 1177c478bd9Sstevel@tonic-gate * result was that no synchronization between the onlining and onlined CPUs 1187c478bd9Sstevel@tonic-gate * took place. 1197c478bd9Sstevel@tonic-gate */ 1207c478bd9Sstevel@tonic-gate static volatile int tsc_ready; 1217c478bd9Sstevel@tonic-gate static volatile int tsc_sync_go; 1227c478bd9Sstevel@tonic-gate 1237c478bd9Sstevel@tonic-gate /* 1247c478bd9Sstevel@tonic-gate * Used as indices into the tsc_sync_snaps[] array. 1257c478bd9Sstevel@tonic-gate */ 1267c478bd9Sstevel@tonic-gate #define TSC_MASTER 0 1277c478bd9Sstevel@tonic-gate #define TSC_SLAVE 1 1287c478bd9Sstevel@tonic-gate 1297c478bd9Sstevel@tonic-gate /* 1307c478bd9Sstevel@tonic-gate * Used in the tsc_master_sync()/tsc_slave_sync() rendezvous. 1317c478bd9Sstevel@tonic-gate */ 1327c478bd9Sstevel@tonic-gate #define TSC_SYNC_STOP 1 1337c478bd9Sstevel@tonic-gate #define TSC_SYNC_GO 2 1347c478bd9Sstevel@tonic-gate #define TSC_SYNC_AGAIN 3 1357c478bd9Sstevel@tonic-gate 136*843e1988Sjohnlev #define TSC_CONVERT_AND_ADD(tsc, hrt, scale) { \ 137ae115bc7Smrj unsigned int *_l = (unsigned int *)&(tsc); \ 138ae115bc7Smrj (hrt) += mul32(_l[1], scale) << NSEC_SHIFT; \ 1397c478bd9Sstevel@tonic-gate (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 1407c478bd9Sstevel@tonic-gate } 1417c478bd9Sstevel@tonic-gate 142ae115bc7Smrj #define TSC_CONVERT(tsc, hrt, scale) { \ 143ae115bc7Smrj unsigned int *_l = (unsigned int *)&(tsc); \ 144ae115bc7Smrj (hrt) = mul32(_l[1], scale) << NSEC_SHIFT; \ 1457c478bd9Sstevel@tonic-gate (hrt) += mul32(_l[0], scale) >> (32 - NSEC_SHIFT); \ 1467c478bd9Sstevel@tonic-gate } 1477c478bd9Sstevel@tonic-gate 148ae115bc7Smrj int tsc_master_slave_sync_needed = 1; 1497c478bd9Sstevel@tonic-gate 1507c478bd9Sstevel@tonic-gate static int tsc_max_delta; 1517c478bd9Sstevel@tonic-gate static hrtime_t tsc_sync_snaps[2]; 1527c478bd9Sstevel@tonic-gate static hrtime_t tsc_sync_delta[NCPU]; 1537c478bd9Sstevel@tonic-gate static hrtime_t tsc_sync_tick_delta[NCPU]; 1547c478bd9Sstevel@tonic-gate static hrtime_t tsc_last = 0; 1557c478bd9Sstevel@tonic-gate static hrtime_t tsc_last_jumped = 0; 1567c478bd9Sstevel@tonic-gate static hrtime_t tsc_hrtime_base = 0; 1577c478bd9Sstevel@tonic-gate static int tsc_jumped = 0; 1587c478bd9Sstevel@tonic-gate 1597c478bd9Sstevel@tonic-gate static hrtime_t shadow_tsc_hrtime_base; 1607c478bd9Sstevel@tonic-gate static hrtime_t shadow_tsc_last; 1617c478bd9Sstevel@tonic-gate static uint_t shadow_nsec_scale; 1627c478bd9Sstevel@tonic-gate static uint32_t shadow_hres_lock; 1637c478bd9Sstevel@tonic-gate 164*843e1988Sjohnlev hrtime_t 165*843e1988Sjohnlev tsc_gethrtime(void) 166*843e1988Sjohnlev { 167*843e1988Sjohnlev uint32_t old_hres_lock; 168*843e1988Sjohnlev hrtime_t tsc, hrt; 169*843e1988Sjohnlev 170*843e1988Sjohnlev do { 171*843e1988Sjohnlev old_hres_lock = hres_lock; 172*843e1988Sjohnlev 173*843e1988Sjohnlev if ((tsc = tsc_read()) >= tsc_last) { 174*843e1988Sjohnlev /* 175*843e1988Sjohnlev * It would seem to be obvious that this is true 176*843e1988Sjohnlev * (that is, the past is less than the present), 177*843e1988Sjohnlev * but it isn't true in the presence of suspend/resume 178*843e1988Sjohnlev * cycles. If we manage to call gethrtime() 179*843e1988Sjohnlev * after a resume, but before the first call to 180*843e1988Sjohnlev * tsc_tick(), we will see the jump. In this case, 181*843e1988Sjohnlev * we will simply use the value in TSC as the delta. 182*843e1988Sjohnlev */ 183*843e1988Sjohnlev tsc -= tsc_last; 184*843e1988Sjohnlev } else if (tsc >= tsc_last - 2*tsc_max_delta) { 185*843e1988Sjohnlev /* 186*843e1988Sjohnlev * There is a chance that tsc_tick() has just run on 187*843e1988Sjohnlev * another CPU, and we have drifted just enough so that 188*843e1988Sjohnlev * we appear behind tsc_last. In this case, force the 189*843e1988Sjohnlev * delta to be zero. 190*843e1988Sjohnlev */ 191*843e1988Sjohnlev tsc = 0; 192*843e1988Sjohnlev } 193*843e1988Sjohnlev 194*843e1988Sjohnlev hrt = tsc_hrtime_base; 195*843e1988Sjohnlev 196*843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 197*843e1988Sjohnlev } while ((old_hres_lock & ~1) != hres_lock); 198*843e1988Sjohnlev 199*843e1988Sjohnlev return (hrt); 200*843e1988Sjohnlev } 201*843e1988Sjohnlev 202*843e1988Sjohnlev hrtime_t 203*843e1988Sjohnlev tsc_gethrtime_delta(void) 204*843e1988Sjohnlev { 205*843e1988Sjohnlev uint32_t old_hres_lock; 206*843e1988Sjohnlev hrtime_t tsc, hrt; 207*843e1988Sjohnlev int flags; 208*843e1988Sjohnlev 209*843e1988Sjohnlev do { 210*843e1988Sjohnlev old_hres_lock = hres_lock; 211*843e1988Sjohnlev 212*843e1988Sjohnlev /* 213*843e1988Sjohnlev * We need to disable interrupts here to assure that we 214*843e1988Sjohnlev * don't migrate between the call to tsc_read() and 215*843e1988Sjohnlev * adding the CPU's TSC tick delta. Note that disabling 216*843e1988Sjohnlev * and reenabling preemption is forbidden here because 217*843e1988Sjohnlev * we may be in the middle of a fast trap. In the amd64 218*843e1988Sjohnlev * kernel we cannot tolerate preemption during a fast 219*843e1988Sjohnlev * trap. See _update_sregs(). 220*843e1988Sjohnlev */ 221*843e1988Sjohnlev 222*843e1988Sjohnlev flags = clear_int_flag(); 223*843e1988Sjohnlev tsc = tsc_read() + tsc_sync_tick_delta[CPU->cpu_id]; 224*843e1988Sjohnlev restore_int_flag(flags); 225*843e1988Sjohnlev 226*843e1988Sjohnlev /* See comments in tsc_gethrtime() above */ 227*843e1988Sjohnlev 228*843e1988Sjohnlev if (tsc >= tsc_last) { 229*843e1988Sjohnlev tsc -= tsc_last; 230*843e1988Sjohnlev } else if (tsc >= tsc_last - 2 * tsc_max_delta) { 231*843e1988Sjohnlev tsc = 0; 232*843e1988Sjohnlev } 233*843e1988Sjohnlev 234*843e1988Sjohnlev hrt = tsc_hrtime_base; 235*843e1988Sjohnlev 236*843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 237*843e1988Sjohnlev } while ((old_hres_lock & ~1) != hres_lock); 238*843e1988Sjohnlev 239*843e1988Sjohnlev return (hrt); 240*843e1988Sjohnlev } 241*843e1988Sjohnlev 242*843e1988Sjohnlev /* 243*843e1988Sjohnlev * This is similar to the above, but it cannot actually spin on hres_lock. 244*843e1988Sjohnlev * As a result, it caches all of the variables it needs; if the variables 245*843e1988Sjohnlev * don't change, it's done. 246*843e1988Sjohnlev */ 247*843e1988Sjohnlev hrtime_t 248*843e1988Sjohnlev dtrace_gethrtime(void) 249*843e1988Sjohnlev { 250*843e1988Sjohnlev uint32_t old_hres_lock; 251*843e1988Sjohnlev hrtime_t tsc, hrt; 252*843e1988Sjohnlev int flags; 253*843e1988Sjohnlev 254*843e1988Sjohnlev do { 255*843e1988Sjohnlev old_hres_lock = hres_lock; 256*843e1988Sjohnlev 257*843e1988Sjohnlev /* 258*843e1988Sjohnlev * Interrupts are disabled to ensure that the thread isn't 259*843e1988Sjohnlev * migrated between the tsc_read() and adding the CPU's 260*843e1988Sjohnlev * TSC tick delta. 261*843e1988Sjohnlev */ 262*843e1988Sjohnlev flags = clear_int_flag(); 263*843e1988Sjohnlev 264*843e1988Sjohnlev tsc = tsc_read(); 265*843e1988Sjohnlev 266*843e1988Sjohnlev if (gethrtimef == tsc_gethrtime_delta) 267*843e1988Sjohnlev tsc += tsc_sync_tick_delta[CPU->cpu_id]; 268*843e1988Sjohnlev 269*843e1988Sjohnlev restore_int_flag(flags); 270*843e1988Sjohnlev 271*843e1988Sjohnlev /* 272*843e1988Sjohnlev * See the comments in tsc_gethrtime(), above. 273*843e1988Sjohnlev */ 274*843e1988Sjohnlev if (tsc >= tsc_last) 275*843e1988Sjohnlev tsc -= tsc_last; 276*843e1988Sjohnlev else if (tsc >= tsc_last - 2*tsc_max_delta) 277*843e1988Sjohnlev tsc = 0; 278*843e1988Sjohnlev 279*843e1988Sjohnlev hrt = tsc_hrtime_base; 280*843e1988Sjohnlev 281*843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, nsec_scale); 282*843e1988Sjohnlev 283*843e1988Sjohnlev if ((old_hres_lock & ~1) == hres_lock) 284*843e1988Sjohnlev break; 285*843e1988Sjohnlev 286*843e1988Sjohnlev /* 287*843e1988Sjohnlev * If we're here, the clock lock is locked -- or it has been 288*843e1988Sjohnlev * unlocked and locked since we looked. This may be due to 289*843e1988Sjohnlev * tsc_tick() running on another CPU -- or it may be because 290*843e1988Sjohnlev * some code path has ended up in dtrace_probe() with 291*843e1988Sjohnlev * CLOCK_LOCK held. We'll try to determine that we're in 292*843e1988Sjohnlev * the former case by taking another lap if the lock has 293*843e1988Sjohnlev * changed since when we first looked at it. 294*843e1988Sjohnlev */ 295*843e1988Sjohnlev if (old_hres_lock != hres_lock) 296*843e1988Sjohnlev continue; 297*843e1988Sjohnlev 298*843e1988Sjohnlev /* 299*843e1988Sjohnlev * So the lock was and is locked. We'll use the old data 300*843e1988Sjohnlev * instead. 301*843e1988Sjohnlev */ 302*843e1988Sjohnlev old_hres_lock = shadow_hres_lock; 303*843e1988Sjohnlev 304*843e1988Sjohnlev /* 305*843e1988Sjohnlev * Again, disable interrupts to ensure that the thread 306*843e1988Sjohnlev * isn't migrated between the tsc_read() and adding 307*843e1988Sjohnlev * the CPU's TSC tick delta. 308*843e1988Sjohnlev */ 309*843e1988Sjohnlev flags = clear_int_flag(); 310*843e1988Sjohnlev 311*843e1988Sjohnlev tsc = tsc_read(); 312*843e1988Sjohnlev 313*843e1988Sjohnlev if (gethrtimef == tsc_gethrtime_delta) 314*843e1988Sjohnlev tsc += tsc_sync_tick_delta[CPU->cpu_id]; 315*843e1988Sjohnlev 316*843e1988Sjohnlev restore_int_flag(flags); 317*843e1988Sjohnlev 318*843e1988Sjohnlev /* 319*843e1988Sjohnlev * See the comments in tsc_gethrtime(), above. 320*843e1988Sjohnlev */ 321*843e1988Sjohnlev if (tsc >= shadow_tsc_last) 322*843e1988Sjohnlev tsc -= shadow_tsc_last; 323*843e1988Sjohnlev else if (tsc >= shadow_tsc_last - 2 * tsc_max_delta) 324*843e1988Sjohnlev tsc = 0; 325*843e1988Sjohnlev 326*843e1988Sjohnlev hrt = shadow_tsc_hrtime_base; 327*843e1988Sjohnlev 328*843e1988Sjohnlev TSC_CONVERT_AND_ADD(tsc, hrt, shadow_nsec_scale); 329*843e1988Sjohnlev } while ((old_hres_lock & ~1) != shadow_hres_lock); 330*843e1988Sjohnlev 331*843e1988Sjohnlev return (hrt); 332*843e1988Sjohnlev } 333*843e1988Sjohnlev 334*843e1988Sjohnlev hrtime_t 335*843e1988Sjohnlev tsc_gethrtimeunscaled(void) 336*843e1988Sjohnlev { 337*843e1988Sjohnlev uint32_t old_hres_lock; 338*843e1988Sjohnlev hrtime_t tsc; 339*843e1988Sjohnlev 340*843e1988Sjohnlev do { 341*843e1988Sjohnlev old_hres_lock = hres_lock; 342*843e1988Sjohnlev 343*843e1988Sjohnlev /* See tsc_tick(). */ 344*843e1988Sjohnlev tsc = tsc_read() + tsc_last_jumped; 345*843e1988Sjohnlev } while ((old_hres_lock & ~1) != hres_lock); 346*843e1988Sjohnlev 347*843e1988Sjohnlev return (tsc); 348*843e1988Sjohnlev } 349*843e1988Sjohnlev 350*843e1988Sjohnlev 351*843e1988Sjohnlev /* Convert a tsc timestamp to nanoseconds */ 352*843e1988Sjohnlev void 353*843e1988Sjohnlev tsc_scalehrtime(hrtime_t *tsc) 354*843e1988Sjohnlev { 355*843e1988Sjohnlev hrtime_t hrt; 356*843e1988Sjohnlev hrtime_t mytsc; 357*843e1988Sjohnlev 358*843e1988Sjohnlev if (tsc == NULL) 359*843e1988Sjohnlev return; 360*843e1988Sjohnlev mytsc = *tsc; 361*843e1988Sjohnlev 362*843e1988Sjohnlev TSC_CONVERT(mytsc, hrt, nsec_scale); 363*843e1988Sjohnlev *tsc = hrt; 364*843e1988Sjohnlev } 365*843e1988Sjohnlev 366*843e1988Sjohnlev hrtime_t 367*843e1988Sjohnlev tsc_gethrtimeunscaled_delta(void) 368*843e1988Sjohnlev { 369*843e1988Sjohnlev hrtime_t hrt; 370*843e1988Sjohnlev int flags; 371*843e1988Sjohnlev 372*843e1988Sjohnlev /* 373*843e1988Sjohnlev * Similarly to tsc_gethrtime_delta, we need to disable preemption 374*843e1988Sjohnlev * to prevent migration between the call to tsc_gethrtimeunscaled 375*843e1988Sjohnlev * and adding the CPU's hrtime delta. Note that disabling and 376*843e1988Sjohnlev * reenabling preemption is forbidden here because we may be in the 377*843e1988Sjohnlev * middle of a fast trap. In the amd64 kernel we cannot tolerate 378*843e1988Sjohnlev * preemption during a fast trap. See _update_sregs(). 379*843e1988Sjohnlev */ 380*843e1988Sjohnlev 381*843e1988Sjohnlev flags = clear_int_flag(); 382*843e1988Sjohnlev hrt = tsc_gethrtimeunscaled() + tsc_sync_tick_delta[CPU->cpu_id]; 383*843e1988Sjohnlev restore_int_flag(flags); 384*843e1988Sjohnlev 385*843e1988Sjohnlev return (hrt); 386*843e1988Sjohnlev } 387*843e1988Sjohnlev 3887c478bd9Sstevel@tonic-gate /* 3897c478bd9Sstevel@tonic-gate * Called by the master after the sync operation is complete. If the 3907c478bd9Sstevel@tonic-gate * slave is discovered to lag, gethrtimef will be changed to point to 3917c478bd9Sstevel@tonic-gate * tsc_gethrtime_delta(). 3927c478bd9Sstevel@tonic-gate */ 3937c478bd9Sstevel@tonic-gate static void 3947c478bd9Sstevel@tonic-gate tsc_digest(processorid_t target) 3957c478bd9Sstevel@tonic-gate { 3967c478bd9Sstevel@tonic-gate hrtime_t tdelta, hdelta = 0; 3977c478bd9Sstevel@tonic-gate int max = tsc_max_delta; 3987c478bd9Sstevel@tonic-gate processorid_t source = CPU->cpu_id; 3997c478bd9Sstevel@tonic-gate int update; 4007c478bd9Sstevel@tonic-gate 4017c478bd9Sstevel@tonic-gate update = tsc_sync_delta[source] != 0 || 4027c478bd9Sstevel@tonic-gate gethrtimef == tsc_gethrtime_delta; 4037c478bd9Sstevel@tonic-gate 4047c478bd9Sstevel@tonic-gate /* 4057c478bd9Sstevel@tonic-gate * We divide by 2 since each of the data points is the sum of two TSC 4067c478bd9Sstevel@tonic-gate * reads; this takes the average of the two. 4077c478bd9Sstevel@tonic-gate */ 4087c478bd9Sstevel@tonic-gate tdelta = (tsc_sync_snaps[TSC_SLAVE] - tsc_sync_snaps[TSC_MASTER]) / 2; 4097c478bd9Sstevel@tonic-gate if ((tdelta > max) || ((tdelta >= 0) && update)) { 4107c478bd9Sstevel@tonic-gate TSC_CONVERT_AND_ADD(tdelta, hdelta, nsec_scale); 4117c478bd9Sstevel@tonic-gate tsc_sync_delta[target] = tsc_sync_delta[source] - hdelta; 4127c478bd9Sstevel@tonic-gate tsc_sync_tick_delta[target] = -tdelta; 4137c478bd9Sstevel@tonic-gate gethrtimef = tsc_gethrtime_delta; 4147c478bd9Sstevel@tonic-gate gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 4157c478bd9Sstevel@tonic-gate return; 4167c478bd9Sstevel@tonic-gate } 4177c478bd9Sstevel@tonic-gate 4187c478bd9Sstevel@tonic-gate tdelta = -tdelta; 4197c478bd9Sstevel@tonic-gate if ((tdelta > max) || update) { 4207c478bd9Sstevel@tonic-gate TSC_CONVERT_AND_ADD(tdelta, hdelta, nsec_scale); 4217c478bd9Sstevel@tonic-gate tsc_sync_delta[target] = tsc_sync_delta[source] + hdelta; 4227c478bd9Sstevel@tonic-gate tsc_sync_tick_delta[target] = tdelta; 4237c478bd9Sstevel@tonic-gate gethrtimef = tsc_gethrtime_delta; 4247c478bd9Sstevel@tonic-gate gethrtimeunscaledf = tsc_gethrtimeunscaled_delta; 4257c478bd9Sstevel@tonic-gate } 4267c478bd9Sstevel@tonic-gate 4277c478bd9Sstevel@tonic-gate } 4287c478bd9Sstevel@tonic-gate 4297c478bd9Sstevel@tonic-gate /* 4307c478bd9Sstevel@tonic-gate * Called by a CPU which has just performed an online operation on another 4317c478bd9Sstevel@tonic-gate * CPU. It is expected that the newly onlined CPU will call tsc_sync_slave(). 4327c478bd9Sstevel@tonic-gate */ 4337c478bd9Sstevel@tonic-gate void 4347c478bd9Sstevel@tonic-gate tsc_sync_master(processorid_t slave) 4357c478bd9Sstevel@tonic-gate { 436ae115bc7Smrj ulong_t flags; 4377c478bd9Sstevel@tonic-gate hrtime_t hrt; 4387c478bd9Sstevel@tonic-gate 439ae115bc7Smrj if (!tsc_master_slave_sync_needed) 440ae115bc7Smrj return; 441ae115bc7Smrj 4427c478bd9Sstevel@tonic-gate ASSERT(tsc_sync_go != TSC_SYNC_GO); 4437c478bd9Sstevel@tonic-gate 4447c478bd9Sstevel@tonic-gate flags = clear_int_flag(); 4457c478bd9Sstevel@tonic-gate 4467c478bd9Sstevel@tonic-gate /* 4477c478bd9Sstevel@tonic-gate * Wait for the slave CPU to arrive. 4487c478bd9Sstevel@tonic-gate */ 4497c478bd9Sstevel@tonic-gate while (tsc_ready != TSC_SYNC_GO) 4507c478bd9Sstevel@tonic-gate continue; 4517c478bd9Sstevel@tonic-gate 4527c478bd9Sstevel@tonic-gate /* 4537c478bd9Sstevel@tonic-gate * Tell the slave CPU to begin reading its TSC; read our own. 4547c478bd9Sstevel@tonic-gate */ 4557c478bd9Sstevel@tonic-gate tsc_sync_go = TSC_SYNC_GO; 4567c478bd9Sstevel@tonic-gate hrt = tsc_read(); 4577c478bd9Sstevel@tonic-gate 4587c478bd9Sstevel@tonic-gate /* 4597c478bd9Sstevel@tonic-gate * Tell the slave that we're ready, and wait for the slave to tell us 4607c478bd9Sstevel@tonic-gate * to read our TSC again. 4617c478bd9Sstevel@tonic-gate */ 4627c478bd9Sstevel@tonic-gate tsc_ready = TSC_SYNC_AGAIN; 4637c478bd9Sstevel@tonic-gate while (tsc_sync_go != TSC_SYNC_AGAIN) 4647c478bd9Sstevel@tonic-gate continue; 4657c478bd9Sstevel@tonic-gate 4667c478bd9Sstevel@tonic-gate hrt += tsc_read(); 4677c478bd9Sstevel@tonic-gate tsc_sync_snaps[TSC_MASTER] = hrt; 4687c478bd9Sstevel@tonic-gate 4697c478bd9Sstevel@tonic-gate /* 4707c478bd9Sstevel@tonic-gate * Wait for the slave to finish reading its TSC. 4717c478bd9Sstevel@tonic-gate */ 4727c478bd9Sstevel@tonic-gate while (tsc_ready != TSC_SYNC_STOP) 4737c478bd9Sstevel@tonic-gate continue; 4747c478bd9Sstevel@tonic-gate 4757c478bd9Sstevel@tonic-gate /* 4767c478bd9Sstevel@tonic-gate * At this point, both CPUs have performed their tsc_read() calls. 4777c478bd9Sstevel@tonic-gate * We'll digest it now before letting the slave CPU return. 4787c478bd9Sstevel@tonic-gate */ 4797c478bd9Sstevel@tonic-gate tsc_digest(slave); 4807c478bd9Sstevel@tonic-gate tsc_sync_go = TSC_SYNC_STOP; 4817c478bd9Sstevel@tonic-gate 4827c478bd9Sstevel@tonic-gate restore_int_flag(flags); 4837c478bd9Sstevel@tonic-gate } 4847c478bd9Sstevel@tonic-gate 4857c478bd9Sstevel@tonic-gate /* 4867c478bd9Sstevel@tonic-gate * Called by a CPU which has just been onlined. It is expected that the CPU 4877c478bd9Sstevel@tonic-gate * performing the online operation will call tsc_sync_master(). 4887c478bd9Sstevel@tonic-gate */ 4897c478bd9Sstevel@tonic-gate void 4907c478bd9Sstevel@tonic-gate tsc_sync_slave(void) 4917c478bd9Sstevel@tonic-gate { 492ae115bc7Smrj ulong_t flags; 4937c478bd9Sstevel@tonic-gate hrtime_t hrt; 4947c478bd9Sstevel@tonic-gate 495ae115bc7Smrj if (!tsc_master_slave_sync_needed) 496ae115bc7Smrj return; 497ae115bc7Smrj 4987c478bd9Sstevel@tonic-gate ASSERT(tsc_sync_go != TSC_SYNC_GO); 4997c478bd9Sstevel@tonic-gate 5007c478bd9Sstevel@tonic-gate flags = clear_int_flag(); 5017c478bd9Sstevel@tonic-gate 502d90554ebSdmick /* to test tsc_gethrtime_delta, add wrmsr(REG_TSC, 0) here */ 503d90554ebSdmick 5047c478bd9Sstevel@tonic-gate /* 5057c478bd9Sstevel@tonic-gate * Tell the master CPU that we're ready, and wait for the master to 5067c478bd9Sstevel@tonic-gate * tell us to begin reading our TSC. 5077c478bd9Sstevel@tonic-gate */ 5087c478bd9Sstevel@tonic-gate tsc_ready = TSC_SYNC_GO; 5097c478bd9Sstevel@tonic-gate while (tsc_sync_go != TSC_SYNC_GO) 5107c478bd9Sstevel@tonic-gate continue; 5117c478bd9Sstevel@tonic-gate 5127c478bd9Sstevel@tonic-gate hrt = tsc_read(); 5137c478bd9Sstevel@tonic-gate 5147c478bd9Sstevel@tonic-gate /* 5157c478bd9Sstevel@tonic-gate * Wait for the master CPU to be ready to read its TSC again. 5167c478bd9Sstevel@tonic-gate */ 5177c478bd9Sstevel@tonic-gate while (tsc_ready != TSC_SYNC_AGAIN) 5187c478bd9Sstevel@tonic-gate continue; 5197c478bd9Sstevel@tonic-gate 5207c478bd9Sstevel@tonic-gate /* 5217c478bd9Sstevel@tonic-gate * Tell the master CPU to read its TSC again; read ours again. 5227c478bd9Sstevel@tonic-gate */ 5237c478bd9Sstevel@tonic-gate tsc_sync_go = TSC_SYNC_AGAIN; 5247c478bd9Sstevel@tonic-gate 5257c478bd9Sstevel@tonic-gate hrt += tsc_read(); 5267c478bd9Sstevel@tonic-gate tsc_sync_snaps[TSC_SLAVE] = hrt; 5277c478bd9Sstevel@tonic-gate 5287c478bd9Sstevel@tonic-gate /* 5297c478bd9Sstevel@tonic-gate * Tell the master that we're done, and wait to be dismissed. 5307c478bd9Sstevel@tonic-gate */ 5317c478bd9Sstevel@tonic-gate tsc_ready = TSC_SYNC_STOP; 5327c478bd9Sstevel@tonic-gate while (tsc_sync_go != TSC_SYNC_STOP) 5337c478bd9Sstevel@tonic-gate continue; 5347c478bd9Sstevel@tonic-gate 5357c478bd9Sstevel@tonic-gate restore_int_flag(flags); 5367c478bd9Sstevel@tonic-gate } 5377c478bd9Sstevel@tonic-gate 5387c478bd9Sstevel@tonic-gate /* 539ae115bc7Smrj * Called once per second on a CPU from the cyclic subsystem's 540ae115bc7Smrj * CY_HIGH_LEVEL interrupt. (No longer just cpu0-only) 5417c478bd9Sstevel@tonic-gate */ 5427c478bd9Sstevel@tonic-gate void 5437c478bd9Sstevel@tonic-gate tsc_tick(void) 5447c478bd9Sstevel@tonic-gate { 5457c478bd9Sstevel@tonic-gate hrtime_t now, delta; 5467c478bd9Sstevel@tonic-gate ushort_t spl; 5477c478bd9Sstevel@tonic-gate 5487c478bd9Sstevel@tonic-gate /* 5497c478bd9Sstevel@tonic-gate * Before we set the new variables, we set the shadow values. This 5507c478bd9Sstevel@tonic-gate * allows for lock free operation in dtrace_gethrtime(). 5517c478bd9Sstevel@tonic-gate */ 5527c478bd9Sstevel@tonic-gate lock_set_spl((lock_t *)&shadow_hres_lock + HRES_LOCK_OFFSET, 5537c478bd9Sstevel@tonic-gate ipltospl(CBE_HIGH_PIL), &spl); 5547c478bd9Sstevel@tonic-gate 5557c478bd9Sstevel@tonic-gate shadow_tsc_hrtime_base = tsc_hrtime_base; 5567c478bd9Sstevel@tonic-gate shadow_tsc_last = tsc_last; 5577c478bd9Sstevel@tonic-gate shadow_nsec_scale = nsec_scale; 5587c478bd9Sstevel@tonic-gate 5597c478bd9Sstevel@tonic-gate shadow_hres_lock++; 5607c478bd9Sstevel@tonic-gate splx(spl); 5617c478bd9Sstevel@tonic-gate 5627c478bd9Sstevel@tonic-gate CLOCK_LOCK(&spl); 5637c478bd9Sstevel@tonic-gate 5647c478bd9Sstevel@tonic-gate now = tsc_read(); 5657c478bd9Sstevel@tonic-gate 566d90554ebSdmick if (gethrtimef == tsc_gethrtime_delta) 567d90554ebSdmick now += tsc_sync_tick_delta[CPU->cpu_id]; 568d90554ebSdmick 5697c478bd9Sstevel@tonic-gate if (now < tsc_last) { 5707c478bd9Sstevel@tonic-gate /* 5717c478bd9Sstevel@tonic-gate * The TSC has just jumped into the past. We assume that 5727c478bd9Sstevel@tonic-gate * this is due to a suspend/resume cycle, and we're going 5737c478bd9Sstevel@tonic-gate * to use the _current_ value of TSC as the delta. This 5747c478bd9Sstevel@tonic-gate * will keep tsc_hrtime_base correct. We're also going to 5757c478bd9Sstevel@tonic-gate * assume that rate of tsc does not change after a suspend 5767c478bd9Sstevel@tonic-gate * resume (i.e nsec_scale remains the same). 5777c478bd9Sstevel@tonic-gate */ 5787c478bd9Sstevel@tonic-gate delta = now; 5797c478bd9Sstevel@tonic-gate tsc_last_jumped += tsc_last; 5807c478bd9Sstevel@tonic-gate tsc_jumped = 1; 5817c478bd9Sstevel@tonic-gate } else { 5827c478bd9Sstevel@tonic-gate /* 5837c478bd9Sstevel@tonic-gate * Determine the number of TSC ticks since the last clock 5847c478bd9Sstevel@tonic-gate * tick, and add that to the hrtime base. 5857c478bd9Sstevel@tonic-gate */ 5867c478bd9Sstevel@tonic-gate delta = now - tsc_last; 5877c478bd9Sstevel@tonic-gate } 5887c478bd9Sstevel@tonic-gate 5897c478bd9Sstevel@tonic-gate TSC_CONVERT_AND_ADD(delta, tsc_hrtime_base, nsec_scale); 5907c478bd9Sstevel@tonic-gate tsc_last = now; 5917c478bd9Sstevel@tonic-gate 5927c478bd9Sstevel@tonic-gate CLOCK_UNLOCK(spl); 5937c478bd9Sstevel@tonic-gate } 5947c478bd9Sstevel@tonic-gate 5957c478bd9Sstevel@tonic-gate void 596*843e1988Sjohnlev tsc_hrtimeinit(uint64_t cpu_freq_hz) 5977c478bd9Sstevel@tonic-gate { 598*843e1988Sjohnlev extern int gethrtime_hires; 599*843e1988Sjohnlev longlong_t tsc; 600*843e1988Sjohnlev ulong_t flags; 6017c478bd9Sstevel@tonic-gate 602*843e1988Sjohnlev /* 603*843e1988Sjohnlev * cpu_freq_hz is the measured cpu frequency in hertz 604*843e1988Sjohnlev */ 6057c478bd9Sstevel@tonic-gate 6067c478bd9Sstevel@tonic-gate /* 607*843e1988Sjohnlev * We can't accommodate CPUs slower than 31.25 MHz. 6087c478bd9Sstevel@tonic-gate */ 609*843e1988Sjohnlev ASSERT(cpu_freq_hz > NANOSEC / (1 << NSEC_SHIFT)); 610*843e1988Sjohnlev nsec_scale = 611*843e1988Sjohnlev (uint_t)(((uint64_t)NANOSEC << (32 - NSEC_SHIFT)) / cpu_freq_hz); 6127c478bd9Sstevel@tonic-gate 6137c478bd9Sstevel@tonic-gate flags = clear_int_flag(); 614*843e1988Sjohnlev tsc = tsc_read(); 615*843e1988Sjohnlev (void) tsc_gethrtime(); 616*843e1988Sjohnlev tsc_max_delta = tsc_read() - tsc; 6177c478bd9Sstevel@tonic-gate restore_int_flag(flags); 618*843e1988Sjohnlev gethrtimef = tsc_gethrtime; 619*843e1988Sjohnlev gethrtimeunscaledf = tsc_gethrtimeunscaled; 620*843e1988Sjohnlev scalehrtimef = tsc_scalehrtime; 621*843e1988Sjohnlev hrtime_tick = tsc_tick; 622*843e1988Sjohnlev gethrtime_hires = 1; 6237c478bd9Sstevel@tonic-gate } 624