1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*  paravirtual clock -- common code used by kvm/xen
3 
4 */
5 
6 #include <linux/clocksource.h>
7 #include <linux/kernel.h>
8 #include <linux/percpu.h>
9 #include <linux/notifier.h>
10 #include <linux/sched.h>
11 #include <linux/gfp.h>
12 #include <linux/memblock.h>
13 #include <linux/nmi.h>
14 
15 #include <asm/fixmap.h>
16 #include <asm/pvclock.h>
17 #include <asm/vgtod.h>
18 
19 static u8 valid_flags __read_mostly = 0;
20 static struct pvclock_vsyscall_time_info *pvti_cpu0_va __read_mostly;
21 
pvclock_set_flags(u8 flags)22 void pvclock_set_flags(u8 flags)
23 {
24 	valid_flags = flags;
25 }
26 
pvclock_tsc_khz(struct pvclock_vcpu_time_info * src)27 unsigned long pvclock_tsc_khz(struct pvclock_vcpu_time_info *src)
28 {
29 	u64 pv_tsc_khz = 1000000ULL << 32;
30 
31 	do_div(pv_tsc_khz, src->tsc_to_system_mul);
32 	if (src->tsc_shift < 0)
33 		pv_tsc_khz <<= -src->tsc_shift;
34 	else
35 		pv_tsc_khz >>= src->tsc_shift;
36 	return pv_tsc_khz;
37 }
38 
pvclock_touch_watchdogs(void)39 void pvclock_touch_watchdogs(void)
40 {
41 	touch_softlockup_watchdog_sync();
42 	clocksource_touch_watchdog();
43 	rcu_cpu_stall_reset();
44 	reset_hung_task_detector();
45 }
46 
47 static atomic64_t last_value = ATOMIC64_INIT(0);
48 
pvclock_resume(void)49 void pvclock_resume(void)
50 {
51 	atomic64_set(&last_value, 0);
52 }
53 
pvclock_read_flags(struct pvclock_vcpu_time_info * src)54 u8 pvclock_read_flags(struct pvclock_vcpu_time_info *src)
55 {
56 	unsigned version;
57 	u8 flags;
58 
59 	do {
60 		version = pvclock_read_begin(src);
61 		flags = src->flags;
62 	} while (pvclock_read_retry(src, version));
63 
64 	return flags & valid_flags;
65 }
66 
pvclock_clocksource_read(struct pvclock_vcpu_time_info * src)67 u64 pvclock_clocksource_read(struct pvclock_vcpu_time_info *src)
68 {
69 	unsigned version;
70 	u64 ret;
71 	u64 last;
72 	u8 flags;
73 
74 	do {
75 		version = pvclock_read_begin(src);
76 		ret = __pvclock_read_cycles(src, rdtsc_ordered());
77 		flags = src->flags;
78 	} while (pvclock_read_retry(src, version));
79 
80 	if (unlikely((flags & PVCLOCK_GUEST_STOPPED) != 0)) {
81 		src->flags &= ~PVCLOCK_GUEST_STOPPED;
82 		pvclock_touch_watchdogs();
83 	}
84 
85 	if ((valid_flags & PVCLOCK_TSC_STABLE_BIT) &&
86 		(flags & PVCLOCK_TSC_STABLE_BIT))
87 		return ret;
88 
89 	/*
90 	 * Assumption here is that last_value, a global accumulator, always goes
91 	 * forward. If we are less than that, we should not be much smaller.
92 	 * We assume there is an error margin we're inside, and then the correction
93 	 * does not sacrifice accuracy.
94 	 *
95 	 * For reads: global may have changed between test and return,
96 	 * but this means someone else updated poked the clock at a later time.
97 	 * We just need to make sure we are not seeing a backwards event.
98 	 *
99 	 * For updates: last_value = ret is not enough, since two vcpus could be
100 	 * updating at the same time, and one of them could be slightly behind,
101 	 * making the assumption that last_value always go forward fail to hold.
102 	 */
103 	last = atomic64_read(&last_value);
104 	do {
105 		if (ret < last)
106 			return last;
107 		last = atomic64_cmpxchg(&last_value, last, ret);
108 	} while (unlikely(last != ret));
109 
110 	return ret;
111 }
112 
pvclock_read_wallclock(struct pvclock_wall_clock * wall_clock,struct pvclock_vcpu_time_info * vcpu_time,struct timespec64 * ts)113 void pvclock_read_wallclock(struct pvclock_wall_clock *wall_clock,
114 			    struct pvclock_vcpu_time_info *vcpu_time,
115 			    struct timespec64 *ts)
116 {
117 	u32 version;
118 	u64 delta;
119 	struct timespec64 now;
120 
121 	/* get wallclock at system boot */
122 	do {
123 		version = wall_clock->version;
124 		rmb();		/* fetch version before time */
125 		/*
126 		 * Note: wall_clock->sec is a u32 value, so it can
127 		 * only store dates between 1970 and 2106. To allow
128 		 * times beyond that, we need to create a new hypercall
129 		 * interface with an extended pvclock_wall_clock structure
130 		 * like ARM has.
131 		 */
132 		now.tv_sec  = wall_clock->sec;
133 		now.tv_nsec = wall_clock->nsec;
134 		rmb();		/* fetch time before checking version */
135 	} while ((wall_clock->version & 1) || (version != wall_clock->version));
136 
137 	delta = pvclock_clocksource_read(vcpu_time);	/* time since system boot */
138 	delta += now.tv_sec * NSEC_PER_SEC + now.tv_nsec;
139 
140 	now.tv_nsec = do_div(delta, NSEC_PER_SEC);
141 	now.tv_sec = delta;
142 
143 	set_normalized_timespec64(ts, now.tv_sec, now.tv_nsec);
144 }
145 
pvclock_set_pvti_cpu0_va(struct pvclock_vsyscall_time_info * pvti)146 void pvclock_set_pvti_cpu0_va(struct pvclock_vsyscall_time_info *pvti)
147 {
148 	WARN_ON(vclock_was_used(VDSO_CLOCKMODE_PVCLOCK));
149 	pvti_cpu0_va = pvti;
150 }
151 
pvclock_get_pvti_cpu0_va(void)152 struct pvclock_vsyscall_time_info *pvclock_get_pvti_cpu0_va(void)
153 {
154 	return pvti_cpu0_va;
155 }
156 EXPORT_SYMBOL_GPL(pvclock_get_pvti_cpu0_va);
157