1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright (c) 2017 by Delphix. All rights reserved.
24  */
25 /*
26  * Copyright (c) 2010, Intel Corporation.
27  * All rights reserved.
28  */
29 /*
30  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
31  */
32 
33 #include <sys/time.h>
34 #include <sys/psm.h>
35 #include <sys/psm_common.h>
36 #include <sys/apic.h>
37 #include <sys/pit.h>
38 #include <sys/x86_archext.h>
39 #include <sys/archsystm.h>
40 #include <sys/machsystm.h>
41 #include <sys/cpuvar.h>
42 #include <sys/clock.h>
43 #include <sys/apic_timer.h>
44 
45 /*
46  * preferred apic timer mode, allow tuning from the /etc/system file.
47  */
48 int		apic_timer_preferred_mode = APIC_TIMER_MODE_DEADLINE;
49 
50 int		apic_oneshot = 0;
51 uint_t		apic_hertz_count;
52 uint_t		apic_nsec_per_intr = 0;
53 uint64_t	apic_ticks_per_SFnsecs;		/* # of ticks in SF nsecs */
54 
55 static int		apic_min_timer_ticks = 1; /* minimum timer tick */
56 static hrtime_t		apic_nsec_max;
57 
58 static void	periodic_timer_enable(void);
59 static void	periodic_timer_disable(void);
60 static void	periodic_timer_reprogram(hrtime_t);
61 static void	oneshot_timer_enable(void);
62 static void	oneshot_timer_disable(void);
63 static void	oneshot_timer_reprogram(hrtime_t);
64 static void	deadline_timer_enable(void);
65 static void	deadline_timer_disable(void);
66 static void	deadline_timer_reprogram(hrtime_t);
67 
68 extern int	apic_clkvect;
69 extern uint32_t	apic_divide_reg_init;
70 
71 /*
72  * apic timer data structure
73  */
74 typedef struct apic_timer {
75 	int	mode;
76 	void	(*apic_timer_enable_ops)(void);
77 	void	(*apic_timer_disable_ops)(void);
78 	void	(*apic_timer_reprogram_ops)(hrtime_t);
79 } apic_timer_t;
80 
81 static apic_timer_t	apic_timer;
82 
83 /*
84  * apic timer initialization
85  *
86  * For the one-shot mode request case, the function returns the
87  * resolution (in nanoseconds) for the hardware timer interrupt.
88  * If one-shot mode capability is not available, the return value
89  * will be 0.
90  */
91 int
apic_timer_init(int hertz)92 apic_timer_init(int hertz)
93 {
94 	int		ret, timer_mode;
95 	static int	firsttime = 1;
96 
97 	if (firsttime) {
98 		/* first time calibrate on CPU0 only */
99 		apic_ticks_per_SFnsecs = apic_calibrate();
100 
101 		/* the interval timer initial count is 32 bit max */
102 		apic_nsec_max = APIC_TICKS_TO_NSECS(APIC_MAXVAL);
103 		firsttime = 0;
104 	}
105 
106 	if (hertz == 0) {
107 		/* requested one_shot */
108 
109 		/*
110 		 * return 0 if TSC is not supported.
111 		 */
112 		if (!tsc_gethrtime_enable)
113 			return (0);
114 		/*
115 		 * return 0 if one_shot is not preferred.
116 		 * here, APIC_TIMER_DEADLINE is also an one_shot mode.
117 		 */
118 		if ((apic_timer_preferred_mode != APIC_TIMER_MODE_ONESHOT) &&
119 		    (apic_timer_preferred_mode != APIC_TIMER_MODE_DEADLINE))
120 			return (0);
121 
122 		apic_oneshot = 1;
123 		ret = (int)APIC_TICKS_TO_NSECS(1);
124 		if ((apic_timer_preferred_mode == APIC_TIMER_MODE_DEADLINE) &&
125 		    cpuid_deadline_tsc_supported()) {
126 			timer_mode = APIC_TIMER_MODE_DEADLINE;
127 		} else {
128 			timer_mode = APIC_TIMER_MODE_ONESHOT;
129 		}
130 	} else {
131 		/* periodic */
132 		apic_nsec_per_intr = NANOSEC / hertz;
133 		apic_hertz_count = APIC_NSECS_TO_TICKS(apic_nsec_per_intr);
134 
135 		/* program the local APIC to interrupt at the given frequency */
136 		apic_reg_ops->apic_write(APIC_INIT_COUNT, apic_hertz_count);
137 		apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
138 		    (apic_clkvect + APIC_BASE_VECT) | AV_PERIODIC);
139 		apic_oneshot = 0;
140 		timer_mode = APIC_TIMER_MODE_PERIODIC;
141 		ret = NANOSEC / hertz;
142 	}
143 
144 	/*
145 	 * initialize apic_timer data structure, install the timer ops
146 	 */
147 	apic_timer.mode = timer_mode;
148 	switch (timer_mode) {
149 	default:
150 		/* FALLTHROUGH */
151 	case APIC_TIMER_MODE_ONESHOT:
152 		apic_timer.apic_timer_enable_ops = oneshot_timer_enable;
153 		apic_timer.apic_timer_disable_ops = oneshot_timer_disable;
154 		apic_timer.apic_timer_reprogram_ops = oneshot_timer_reprogram;
155 		break;
156 
157 	case APIC_TIMER_MODE_PERIODIC:
158 		apic_timer.apic_timer_enable_ops = periodic_timer_enable;
159 		apic_timer.apic_timer_disable_ops = periodic_timer_disable;
160 		apic_timer.apic_timer_reprogram_ops = periodic_timer_reprogram;
161 		break;
162 
163 	case APIC_TIMER_MODE_DEADLINE:
164 		apic_timer.apic_timer_enable_ops = deadline_timer_enable;
165 		apic_timer.apic_timer_disable_ops = deadline_timer_disable;
166 		apic_timer.apic_timer_reprogram_ops = deadline_timer_reprogram;
167 		break;
168 	}
169 
170 	return (ret);
171 }
172 
173 /*
174  * periodic timer mode ops
175  */
176 /* periodic timer enable */
177 static void
periodic_timer_enable(void)178 periodic_timer_enable(void)
179 {
180 	apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
181 	    (apic_clkvect + APIC_BASE_VECT) | AV_PERIODIC);
182 }
183 
184 /* periodic timer disable */
185 static void
periodic_timer_disable(void)186 periodic_timer_disable(void)
187 {
188 	apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
189 	    (apic_clkvect + APIC_BASE_VECT) | AV_MASK);
190 }
191 
192 /* periodic timer reprogram */
193 static void
periodic_timer_reprogram(hrtime_t time)194 periodic_timer_reprogram(hrtime_t time)
195 {
196 	uint_t	ticks;
197 	/* time is the interval for periodic mode */
198 	ticks = APIC_NSECS_TO_TICKS(time);
199 
200 	if (ticks < apic_min_timer_ticks)
201 		ticks = apic_min_timer_ticks;
202 
203 	apic_reg_ops->apic_write(APIC_INIT_COUNT, ticks);
204 }
205 
206 /*
207  * oneshot timer mode ops
208  */
209 /* oneshot timer enable */
210 static void
oneshot_timer_enable(void)211 oneshot_timer_enable(void)
212 {
213 	apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
214 	    (apic_clkvect + APIC_BASE_VECT));
215 }
216 
217 /* oneshot timer disable */
218 static void
oneshot_timer_disable(void)219 oneshot_timer_disable(void)
220 {
221 	apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
222 	    (apic_clkvect + APIC_BASE_VECT) | AV_MASK);
223 }
224 
225 /* oneshot timer reprogram */
226 static void
oneshot_timer_reprogram(hrtime_t time)227 oneshot_timer_reprogram(hrtime_t time)
228 {
229 	hrtime_t	now;
230 	int64_t		delta;
231 	uint_t		ticks;
232 
233 	now = gethrtime();
234 	delta = time - now;
235 
236 	if (delta <= 0) {
237 		/*
238 		 * requested to generate an interrupt in the past
239 		 * generate an interrupt as soon as possible
240 		 */
241 		ticks = apic_min_timer_ticks;
242 	} else if (delta > apic_nsec_max) {
243 		/*
244 		 * requested to generate an interrupt at a time
245 		 * further than what we are capable of. Set to max
246 		 * the hardware can handle
247 		 */
248 		ticks = APIC_MAXVAL;
249 #ifdef DEBUG
250 		cmn_err(CE_CONT, "apic_timer_reprogram, request at"
251 		    "  %lld  too far in future, current time"
252 		    "  %lld \n", time, now);
253 #endif
254 	} else {
255 		ticks = APIC_NSECS_TO_TICKS(delta);
256 	}
257 
258 	if (ticks < apic_min_timer_ticks)
259 		ticks = apic_min_timer_ticks;
260 
261 	apic_reg_ops->apic_write(APIC_INIT_COUNT, ticks);
262 }
263 
264 /*
265  * deadline timer mode ops
266  */
267 /* deadline timer enable */
268 static void
deadline_timer_enable(void)269 deadline_timer_enable(void)
270 {
271 	uint64_t ticks;
272 
273 	apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
274 	    (apic_clkvect + APIC_BASE_VECT) | AV_DEADLINE);
275 	/*
276 	 * Now we have to serialize this per the SDM.  That is to
277 	 * say, the above enabling can race in the pipeline with
278 	 * changes to the MSR.  We need to make sure the above
279 	 * operation is complete before we proceed to reprogram
280 	 * the deadline value in reprogram().  The algorithm
281 	 * recommended by the Intel SDM 3A in 10.5.1.4 is:
282 	 *
283 	 * a) write a big value to the deadline register
284 	 * b) read the register back
285 	 * c) if it reads zero, go back to a and try again
286 	 */
287 
288 	do {
289 		/* write a really big value */
290 		wrmsr(IA32_DEADLINE_TSC_MSR, 1ULL << 63);
291 		ticks = rdmsr(IA32_DEADLINE_TSC_MSR);
292 	} while (ticks == 0);
293 }
294 
295 /* deadline timer disable */
296 static void
deadline_timer_disable(void)297 deadline_timer_disable(void)
298 {
299 	apic_reg_ops->apic_write(APIC_LOCAL_TIMER,
300 	    (apic_clkvect + APIC_BASE_VECT) | AV_MASK);
301 }
302 
303 /* deadline timer reprogram */
304 static void
deadline_timer_reprogram(hrtime_t time)305 deadline_timer_reprogram(hrtime_t time)
306 {
307 	int64_t		delta;
308 	uint64_t	ticks;
309 
310 	/*
311 	 * Note that this entire routine is called with
312 	 * CBE_HIGH_PIL, so we needn't worry about preemption.
313 	 */
314 	delta = time - gethrtime();
315 
316 	/* The unscalehrtime wants unsigned values. */
317 	delta = max(delta, 0);
318 
319 	/* Now we shouldn't be interrupted, we can set the deadline */
320 	ticks = (uint64_t)tsc_read() + unscalehrtime(delta);
321 	wrmsr(IA32_DEADLINE_TSC_MSR, ticks);
322 }
323 
324 /*
325  * This function will reprogram the timer.
326  *
327  * When in oneshot mode the argument is the absolute time in future to
328  * generate the interrupt at.
329  *
330  * When in periodic mode, the argument is the interval at which the
331  * interrupts should be generated. There is no need to support the periodic
332  * mode timer change at this time.
333  */
334 void
apic_timer_reprogram(hrtime_t time)335 apic_timer_reprogram(hrtime_t time)
336 {
337 	/*
338 	 * we should be Called from high PIL context (CBE_HIGH_PIL),
339 	 * so kpreempt is disabled.
340 	 */
341 	apic_timer.apic_timer_reprogram_ops(time);
342 }
343 
344 /*
345  * This function will enable timer interrupts.
346  */
347 void
apic_timer_enable(void)348 apic_timer_enable(void)
349 {
350 	/*
351 	 * we should be Called from high PIL context (CBE_HIGH_PIL),
352 	 * so kpreempt is disabled.
353 	 */
354 	apic_timer.apic_timer_enable_ops();
355 }
356 
357 /*
358  * This function will disable timer interrupts.
359  */
360 void
apic_timer_disable(void)361 apic_timer_disable(void)
362 {
363 	/*
364 	 * we should be Called from high PIL context (CBE_HIGH_PIL),
365 	 * so kpreempt is disabled.
366 	 */
367 	apic_timer.apic_timer_disable_ops();
368 }
369 
370 /*
371  * Set timer far into the future and return timer
372  * current count in nanoseconds.
373  */
374 hrtime_t
apic_timer_stop_count(void)375 apic_timer_stop_count(void)
376 {
377 	hrtime_t	ns_val;
378 	int		enable_val, count_val;
379 
380 	/*
381 	 * Should be called with interrupts disabled.
382 	 */
383 	ASSERT(!interrupts_enabled());
384 
385 	enable_val = apic_reg_ops->apic_read(APIC_LOCAL_TIMER);
386 	if ((enable_val & AV_MASK) == AV_MASK)
387 		return ((hrtime_t)-1);	/* timer is disabled */
388 
389 	count_val = apic_reg_ops->apic_read(APIC_CURR_COUNT);
390 	ns_val = APIC_TICKS_TO_NSECS(count_val);
391 
392 	apic_reg_ops->apic_write(APIC_INIT_COUNT, APIC_MAXVAL);
393 
394 	return (ns_val);
395 }
396 
397 /*
398  * Reprogram timer after Deep C-State.
399  */
400 void
apic_timer_restart(hrtime_t time)401 apic_timer_restart(hrtime_t time)
402 {
403 	apic_timer_reprogram(time);
404 }
405