1 /*
2 * Intel SMP support routines.
3 *
4 * (c) 1995 Alan Cox, Building #3 <alan@redhat.com>
5 * (c) 1998-99, 2000 Ingo Molnar <mingo@redhat.com>
6 *
7 * This code is released under the GNU General Public License version 2 or
8 * later.
9 */
10
11 #include <xen/cpu.h>
12 #include <xen/irq.h>
13 #include <xen/sched.h>
14 #include <xen/delay.h>
15 #include <xen/perfc.h>
16 #include <xen/spinlock.h>
17 #include <asm/current.h>
18 #include <asm/guest.h>
19 #include <asm/smp.h>
20 #include <asm/mc146818rtc.h>
21 #include <asm/flushtlb.h>
22 #include <asm/hardirq.h>
23 #include <asm/hpet.h>
24 #include <asm/hvm/support.h>
25 #include <irq_vectors.h>
26 #include <mach_apic.h>
27
28 /* Helper functions to prepare APIC register values. */
prepare_ICR(unsigned int shortcut,int vector)29 static unsigned int prepare_ICR(unsigned int shortcut, int vector)
30 {
31 return APIC_DM_FIXED | shortcut | vector;
32 }
33
prepare_ICR2(unsigned int mask)34 static unsigned int prepare_ICR2(unsigned int mask)
35 {
36 return SET_xAPIC_DEST_FIELD(mask);
37 }
38
apic_wait_icr_idle(void)39 void apic_wait_icr_idle(void)
40 {
41 if ( x2apic_enabled )
42 return;
43
44 while ( apic_read(APIC_ICR) & APIC_ICR_BUSY )
45 cpu_relax();
46 }
47
48 /* Helper for sending APIC IPIs using a shorthand. */
send_IPI_shortcut(unsigned int shortcut,int vector,unsigned int dest)49 static void send_IPI_shortcut(unsigned int shortcut, int vector,
50 unsigned int dest)
51 {
52 unsigned int cfg;
53
54 /* Wait for idle. */
55 apic_wait_icr_idle();
56 /* Prepare target chip field. */
57 cfg = prepare_ICR(shortcut, vector) | dest;
58 /* Send the IPI. The write to APIC_ICR fires this off. */
59 apic_write(APIC_ICR, cfg);
60 }
61
62 /*
63 * send_IPI_mask(cpumask, vector): sends @vector IPI to CPUs in @cpumask,
64 * excluding the local CPU. @cpumask may be empty.
65 */
66
send_IPI_mask(const cpumask_t * mask,int vector)67 void send_IPI_mask(const cpumask_t *mask, int vector)
68 {
69 bool cpus_locked = false;
70 cpumask_t *scratch = this_cpu(send_ipi_cpumask);
71
72 if ( in_irq() || in_mce_handler() || in_nmi_handler() )
73 {
74 /*
75 * When in IRQ, NMI or #MC context fallback to the old (and simpler)
76 * IPI sending routine, and avoid doing any performance optimizations
77 * (like using a shorthand) in order to avoid using the scratch
78 * cpumask which cannot be used in interrupt context.
79 */
80 alternative_vcall(genapic.send_IPI_mask, mask, vector);
81 return;
82 }
83
84 /*
85 * This can only be safely used when no CPU hotplug or unplug operations
86 * are taking place, there are no offline CPUs (unless those have been
87 * onlined and parked), there are no disabled CPUs and all possible CPUs in
88 * the system have been accounted for.
89 */
90 if ( system_state > SYS_STATE_smp_boot &&
91 !unaccounted_cpus && !disabled_cpus &&
92 /* NB: get_cpu_maps lock requires enabled interrupts. */
93 local_irq_is_enabled() && (cpus_locked = get_cpu_maps()) &&
94 (park_offline_cpus ||
95 cpumask_equal(&cpu_online_map, &cpu_present_map)) )
96 cpumask_or(scratch, mask, cpumask_of(smp_processor_id()));
97 else
98 {
99 if ( cpus_locked )
100 {
101 put_cpu_maps();
102 cpus_locked = false;
103 }
104 cpumask_clear(scratch);
105 }
106
107 if ( cpumask_equal(scratch, &cpu_online_map) )
108 send_IPI_shortcut(APIC_DEST_ALLBUT, vector, APIC_DEST_PHYSICAL);
109 else
110 alternative_vcall(genapic.send_IPI_mask, mask, vector);
111
112 if ( cpus_locked )
113 put_cpu_maps();
114 }
115
send_IPI_self(int vector)116 void send_IPI_self(int vector)
117 {
118 alternative_vcall(genapic.send_IPI_self, vector);
119 }
120
121 /*
122 * Some notes on x86 processor bugs affecting SMP operation:
123 *
124 * Pentium, Pentium Pro, II, III (and all CPUs) have bugs.
125 * The Linux implications for SMP are handled as follows:
126 *
127 * Pentium III / [Xeon]
128 * None of the E1AP-E3AP errata are visible to the user.
129 *
130 * E1AP. see PII A1AP
131 * E2AP. see PII A2AP
132 * E3AP. see PII A3AP
133 *
134 * Pentium II / [Xeon]
135 * None of the A1AP-A3AP errata are visible to the user.
136 *
137 * A1AP. see PPro 1AP
138 * A2AP. see PPro 2AP
139 * A3AP. see PPro 7AP
140 *
141 * Pentium Pro
142 * None of 1AP-9AP errata are visible to the normal user,
143 * except occasional delivery of 'spurious interrupt' as trap #15.
144 * This is very rare and a non-problem.
145 *
146 * 1AP. Linux maps APIC as non-cacheable
147 * 2AP. worked around in hardware
148 * 3AP. fixed in C0 and above steppings microcode update.
149 * Linux does not use excessive STARTUP_IPIs.
150 * 4AP. worked around in hardware
151 * 5AP. symmetric IO mode (normal Linux operation) not affected.
152 * 'noapic' mode has vector 0xf filled out properly.
153 * 6AP. 'noapic' mode might be affected - fixed in later steppings
154 * 7AP. We do not assume writes to the LVT deassering IRQs
155 * 8AP. We do not enable low power mode (deep sleep) during MP bootup
156 * 9AP. We do not use mixed mode
157 */
158
159 /*
160 * The following functions deal with sending IPIs between CPUs.
161 */
162
send_IPI_self_legacy(uint8_t vector)163 void send_IPI_self_legacy(uint8_t vector)
164 {
165 send_IPI_shortcut(APIC_DEST_SELF, vector, APIC_DEST_PHYSICAL);
166 }
167
send_IPI_mask_flat(const cpumask_t * cpumask,int vector)168 void send_IPI_mask_flat(const cpumask_t *cpumask, int vector)
169 {
170 unsigned long mask = cpumask_bits(cpumask)[0];
171 unsigned long cfg;
172 unsigned long flags;
173
174 mask &= cpumask_bits(&cpu_online_map)[0];
175 mask &= ~(1UL << smp_processor_id());
176 if ( mask == 0 )
177 return;
178
179 local_irq_save(flags);
180
181 /*
182 * Wait for idle.
183 */
184 apic_wait_icr_idle();
185
186 /*
187 * prepare target chip field
188 */
189 cfg = prepare_ICR2(mask);
190 apic_write(APIC_ICR2, cfg);
191
192 /*
193 * program the ICR
194 */
195 cfg = prepare_ICR(0, vector) | APIC_DEST_LOGICAL;
196
197 /*
198 * Send the IPI. The write to APIC_ICR fires this off.
199 */
200 apic_write(APIC_ICR, cfg);
201
202 local_irq_restore(flags);
203 }
204
send_IPI_mask_phys(const cpumask_t * mask,int vector)205 void send_IPI_mask_phys(const cpumask_t *mask, int vector)
206 {
207 unsigned long cfg, flags;
208 unsigned int query_cpu;
209
210 local_irq_save(flags);
211
212 for_each_cpu ( query_cpu, mask )
213 {
214 if ( !cpu_online(query_cpu) || (query_cpu == smp_processor_id()) )
215 continue;
216
217 /*
218 * Wait for idle.
219 */
220 apic_wait_icr_idle();
221
222 /*
223 * prepare target chip field
224 */
225 cfg = prepare_ICR2(cpu_physical_id(query_cpu));
226 apic_write(APIC_ICR2, cfg);
227
228 /*
229 * program the ICR
230 */
231 cfg = prepare_ICR(0, vector) | APIC_DEST_PHYSICAL;
232
233 /*
234 * Send the IPI. The write to APIC_ICR fires this off.
235 */
236 apic_write(APIC_ICR, cfg);
237 }
238
239 local_irq_restore(flags);
240 }
241
242 static DEFINE_SPINLOCK(flush_lock);
243 static cpumask_t flush_cpumask;
244 static const void *flush_va;
245 static unsigned int flush_flags;
246
invalidate_interrupt(struct cpu_user_regs * regs)247 void invalidate_interrupt(struct cpu_user_regs *regs)
248 {
249 unsigned int flags = flush_flags;
250 ack_APIC_irq();
251 perfc_incr(ipis);
252 if ( (flags & FLUSH_VCPU_STATE) && __sync_local_execstate() )
253 flags &= ~(FLUSH_TLB | FLUSH_TLB_GLOBAL | FLUSH_ROOT_PGTBL);
254 if ( flags & ~(FLUSH_VCPU_STATE | FLUSH_ORDER_MASK) )
255 flush_area_local(flush_va, flags);
256 cpumask_clear_cpu(smp_processor_id(), &flush_cpumask);
257 }
258
flush_area_mask(const cpumask_t * mask,const void * va,unsigned int flags)259 void flush_area_mask(const cpumask_t *mask, const void *va, unsigned int flags)
260 {
261 unsigned int cpu = smp_processor_id();
262
263 ASSERT(local_irq_is_enabled());
264
265 if ( (flags & ~(FLUSH_VCPU_STATE | FLUSH_ORDER_MASK)) &&
266 cpumask_test_cpu(cpu, mask) )
267 flags = flush_area_local(va, flags);
268
269 if ( (flags & ~FLUSH_ORDER_MASK) &&
270 !cpumask_subset(mask, cpumask_of(cpu)) )
271 {
272 if ( cpu_has_hypervisor &&
273 !(flags & ~(FLUSH_TLB | FLUSH_TLB_GLOBAL | FLUSH_VA_VALID |
274 FLUSH_ORDER_MASK)) &&
275 !hypervisor_flush_tlb(mask, va, flags) )
276 return;
277
278 spin_lock(&flush_lock);
279 cpumask_and(&flush_cpumask, mask, &cpu_online_map);
280 cpumask_clear_cpu(cpu, &flush_cpumask);
281 flush_va = va;
282 flush_flags = flags;
283 send_IPI_mask(&flush_cpumask, INVALIDATE_TLB_VECTOR);
284 while ( !cpumask_empty(&flush_cpumask) )
285 cpu_relax();
286 spin_unlock(&flush_lock);
287 }
288 }
289
290 /* Call with no locks held and interrupts enabled (e.g., softirq context). */
new_tlbflush_clock_period(void)291 void new_tlbflush_clock_period(void)
292 {
293 cpumask_t allbutself;
294
295 /* Flush everyone else. We definitely flushed just before entry. */
296 cpumask_andnot(&allbutself, &cpu_online_map,
297 cpumask_of(smp_processor_id()));
298 flush_mask(&allbutself, FLUSH_TLB);
299
300 /* No need for atomicity: we are the only possible updater. */
301 ASSERT(tlbflush_clock == 0);
302 tlbflush_clock++;
303 }
304
smp_send_event_check_mask(const cpumask_t * mask)305 void smp_send_event_check_mask(const cpumask_t *mask)
306 {
307 send_IPI_mask(mask, EVENT_CHECK_VECTOR);
308 }
309
smp_send_call_function_mask(const cpumask_t * mask)310 void smp_send_call_function_mask(const cpumask_t *mask)
311 {
312 send_IPI_mask(mask, CALL_FUNCTION_VECTOR);
313
314 if ( cpumask_test_cpu(smp_processor_id(), mask) )
315 {
316 local_irq_disable();
317 smp_call_function_interrupt();
318 local_irq_enable();
319 }
320 }
321
__stop_this_cpu(void)322 void __stop_this_cpu(void)
323 {
324 ASSERT(!local_irq_is_enabled());
325
326 disable_local_APIC();
327
328 hvm_cpu_down();
329
330 /*
331 * Clear FPU, zapping any pending exceptions. Needed for warm reset with
332 * some BIOSes.
333 */
334 clts();
335 asm volatile ( "fninit" );
336
337 cpumask_clear_cpu(smp_processor_id(), &cpu_online_map);
338 }
339
stop_this_cpu(void * dummy)340 static void stop_this_cpu(void *dummy)
341 {
342 __stop_this_cpu();
343 for ( ; ; )
344 halt();
345 }
346
347 /*
348 * Stop all CPUs and turn off local APICs and the IO-APIC, so other OSs see a
349 * clean IRQ state.
350 */
smp_send_stop(void)351 void smp_send_stop(void)
352 {
353 unsigned int cpu = smp_processor_id();
354
355 if ( num_online_cpus() > 1 )
356 {
357 int timeout = 10;
358
359 local_irq_disable();
360 fixup_irqs(cpumask_of(cpu), 0);
361 local_irq_enable();
362
363 smp_call_function(stop_this_cpu, NULL, 0);
364
365 /* Wait 10ms for all other CPUs to go offline. */
366 while ( (num_online_cpus() > 1) && (timeout-- > 0) )
367 mdelay(1);
368 }
369
370 if ( cpu_online(cpu) )
371 {
372 local_irq_disable();
373 disable_IO_APIC();
374 hpet_disable();
375 __stop_this_cpu();
376 x2apic_enabled = (current_local_apic_mode() == APIC_MODE_X2APIC);
377 local_irq_enable();
378 }
379 }
380
smp_send_nmi_allbutself(void)381 void smp_send_nmi_allbutself(void)
382 {
383 send_IPI_mask(&cpu_online_map, APIC_DM_NMI);
384 }
385
event_check_interrupt(struct cpu_user_regs * regs)386 void event_check_interrupt(struct cpu_user_regs *regs)
387 {
388 ack_APIC_irq();
389 perfc_incr(ipis);
390 this_cpu(irq_count)++;
391 }
392
call_function_interrupt(struct cpu_user_regs * regs)393 void call_function_interrupt(struct cpu_user_regs *regs)
394 {
395 ack_APIC_irq();
396 perfc_incr(ipis);
397 smp_call_function_interrupt();
398 }
399