Lines Matching refs:cpu_base
139 .cpu_base = &migration_cpu_base,
173 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); in lock_hrtimer_base()
177 raw_spin_unlock_irqrestore(&base->cpu_base->lock, *flags); in lock_hrtimer_base()
198 return expires < new_base->cpu_base->expires_next; in hrtimer_check_target()
252 raw_spin_unlock(&base->cpu_base->lock); in switch_hrtimer_base()
253 raw_spin_lock(&new_base->cpu_base->lock); in switch_hrtimer_base()
257 raw_spin_unlock(&new_base->cpu_base->lock); in switch_hrtimer_base()
258 raw_spin_lock(&base->cpu_base->lock); in switch_hrtimer_base()
286 raw_spin_lock_irqsave(&base->cpu_base->lock, *flags); in lock_hrtimer_base()
486 __next_base(struct hrtimer_cpu_base *cpu_base, unsigned int *active) in __next_base() argument
496 return &cpu_base->clock_base[idx]; in __next_base()
499 #define for_each_active_base(base, cpu_base, active) \ argument
500 while ((base = __next_base((cpu_base), &(active))))
502 static ktime_t __hrtimer_next_event_base(struct hrtimer_cpu_base *cpu_base, in __hrtimer_next_event_base() argument
510 for_each_active_base(base, cpu_base, active) { in __hrtimer_next_event_base()
533 cpu_base->softirq_next_timer = timer; in __hrtimer_next_event_base()
535 cpu_base->next_timer = timer; in __hrtimer_next_event_base()
569 __hrtimer_get_next_event(struct hrtimer_cpu_base *cpu_base, unsigned int active_mask) in __hrtimer_get_next_event() argument
575 if (!cpu_base->softirq_activated && (active_mask & HRTIMER_ACTIVE_SOFT)) { in __hrtimer_get_next_event()
576 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; in __hrtimer_get_next_event()
577 cpu_base->softirq_next_timer = NULL; in __hrtimer_get_next_event()
578 expires_next = __hrtimer_next_event_base(cpu_base, NULL, in __hrtimer_get_next_event()
581 next_timer = cpu_base->softirq_next_timer; in __hrtimer_get_next_event()
585 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; in __hrtimer_get_next_event()
586 cpu_base->next_timer = next_timer; in __hrtimer_get_next_event()
587 expires_next = __hrtimer_next_event_base(cpu_base, NULL, active, in __hrtimer_get_next_event()
594 static ktime_t hrtimer_update_next_event(struct hrtimer_cpu_base *cpu_base) in hrtimer_update_next_event() argument
603 if (!cpu_base->softirq_activated) { in hrtimer_update_next_event()
604 soft = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT); in hrtimer_update_next_event()
609 cpu_base->softirq_expires_next = soft; in hrtimer_update_next_event()
612 expires_next = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_HARD); in hrtimer_update_next_event()
618 cpu_base->next_timer = cpu_base->softirq_next_timer; in hrtimer_update_next_event()
644 static inline int __hrtimer_hres_active(struct hrtimer_cpu_base *cpu_base) in __hrtimer_hres_active() argument
647 cpu_base->hres_active : 0; in __hrtimer_hres_active()
655 static void __hrtimer_reprogram(struct hrtimer_cpu_base *cpu_base, in __hrtimer_reprogram() argument
659 cpu_base->expires_next = expires_next; in __hrtimer_reprogram()
678 if (!__hrtimer_hres_active(cpu_base) || cpu_base->hang_detected) in __hrtimer_reprogram()
690 hrtimer_force_reprogram(struct hrtimer_cpu_base *cpu_base, int skip_equal) in hrtimer_force_reprogram() argument
694 expires_next = hrtimer_update_next_event(cpu_base); in hrtimer_force_reprogram()
696 if (skip_equal && expires_next == cpu_base->expires_next) in hrtimer_force_reprogram()
699 __hrtimer_reprogram(cpu_base, cpu_base->next_timer, expires_next); in hrtimer_force_reprogram()
810 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); in hrtimer_reprogram() local
831 struct hrtimer_cpu_base *timer_cpu_base = base->cpu_base; in hrtimer_reprogram()
851 if (base->cpu_base != cpu_base) in hrtimer_reprogram()
854 if (expires >= cpu_base->expires_next) in hrtimer_reprogram()
861 if (cpu_base->in_hrtirq) in hrtimer_reprogram()
864 cpu_base->next_timer = timer; in hrtimer_reprogram()
866 __hrtimer_reprogram(cpu_base, timer, expires); in hrtimer_reprogram()
869 static bool update_needs_ipi(struct hrtimer_cpu_base *cpu_base, in update_needs_ipi() argument
886 seq = cpu_base->clock_was_set_seq; in update_needs_ipi()
887 hrtimer_update_base(cpu_base); in update_needs_ipi()
893 if (seq == cpu_base->clock_was_set_seq) in update_needs_ipi()
901 if (cpu_base->in_hrtirq) in update_needs_ipi()
910 active &= cpu_base->active_bases; in update_needs_ipi()
912 for_each_active_base(base, cpu_base, active) { in update_needs_ipi()
917 if (expires < cpu_base->expires_next) in update_needs_ipi()
923 if (cpu_base->softirq_activated) in update_needs_ipi()
925 if (expires < cpu_base->softirq_expires_next) in update_needs_ipi()
947 struct hrtimer_cpu_base *cpu_base = raw_cpu_ptr(&hrtimer_bases); in clock_was_set() local
951 if (!__hrtimer_hres_active(cpu_base) && !tick_nohz_active) in clock_was_set()
964 cpu_base = &per_cpu(hrtimer_bases, cpu); in clock_was_set()
965 raw_spin_lock_irqsave(&cpu_base->lock, flags); in clock_was_set()
967 if (update_needs_ipi(cpu_base, bases)) in clock_was_set()
970 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in clock_was_set()
1017 raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); in unlock_hrtimer_base()
1086 base->cpu_base->active_bases |= 1 << base->index; in enqueue_hrtimer()
1108 struct hrtimer_cpu_base *cpu_base = base->cpu_base; in __remove_hrtimer() local
1117 cpu_base->active_bases &= ~(1 << base->index); in __remove_hrtimer()
1127 if (reprogram && timer == cpu_base->next_timer) in __remove_hrtimer()
1128 hrtimer_force_reprogram(cpu_base, 1); in __remove_hrtimer()
1152 reprogram = base->cpu_base == this_cpu_ptr(&hrtimer_bases); in remove_hrtimer()
1188 hrtimer_update_softirq_timer(struct hrtimer_cpu_base *cpu_base, bool reprogram) in hrtimer_update_softirq_timer() argument
1195 expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_SOFT); in hrtimer_update_softirq_timer()
1209 hrtimer_reprogram(cpu_base->softirq_next_timer, reprogram); in hrtimer_update_softirq_timer()
1227 force_local = base->cpu_base == this_cpu_ptr(&hrtimer_bases); in __hrtimer_start_range_ns()
1228 force_local &= base->cpu_base->next_timer == timer; in __hrtimer_start_range_ns()
1267 hrtimer_force_reprogram(new_base->cpu_base, 1); in __hrtimer_start_range_ns()
1366 static void hrtimer_sync_wait_running(struct hrtimer_cpu_base *cpu_base, in hrtimer_sync_wait_running() argument
1369 if (atomic_read(&cpu_base->timer_waiters)) { in hrtimer_sync_wait_running()
1370 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in hrtimer_sync_wait_running()
1371 spin_unlock(&cpu_base->softirq_expiry_lock); in hrtimer_sync_wait_running()
1372 spin_lock(&cpu_base->softirq_expiry_lock); in hrtimer_sync_wait_running()
1373 raw_spin_lock_irq(&cpu_base->lock); in hrtimer_sync_wait_running()
1414 atomic_inc(&base->cpu_base->timer_waiters); in hrtimer_cancel_wait_running()
1415 spin_lock_bh(&base->cpu_base->softirq_expiry_lock); in hrtimer_cancel_wait_running()
1416 atomic_dec(&base->cpu_base->timer_waiters); in hrtimer_cancel_wait_running()
1417 spin_unlock_bh(&base->cpu_base->softirq_expiry_lock); in hrtimer_cancel_wait_running()
1481 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); in hrtimer_get_next_event() local
1485 raw_spin_lock_irqsave(&cpu_base->lock, flags); in hrtimer_get_next_event()
1487 if (!__hrtimer_hres_active(cpu_base)) in hrtimer_get_next_event()
1488 expires = __hrtimer_get_next_event(cpu_base, HRTIMER_ACTIVE_ALL); in hrtimer_get_next_event()
1490 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in hrtimer_get_next_event()
1504 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); in hrtimer_next_event_without() local
1508 raw_spin_lock_irqsave(&cpu_base->lock, flags); in hrtimer_next_event_without()
1510 if (__hrtimer_hres_active(cpu_base)) { in hrtimer_next_event_without()
1513 if (!cpu_base->softirq_activated) { in hrtimer_next_event_without()
1514 active = cpu_base->active_bases & HRTIMER_ACTIVE_SOFT; in hrtimer_next_event_without()
1515 expires = __hrtimer_next_event_base(cpu_base, exclude, in hrtimer_next_event_without()
1518 active = cpu_base->active_bases & HRTIMER_ACTIVE_HARD; in hrtimer_next_event_without()
1519 expires = __hrtimer_next_event_base(cpu_base, exclude, active, in hrtimer_next_event_without()
1523 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in hrtimer_next_event_without()
1545 struct hrtimer_cpu_base *cpu_base; in __hrtimer_init() local
1559 cpu_base = raw_cpu_ptr(&hrtimer_bases); in __hrtimer_init()
1573 timer->base = &cpu_base->clock_base[base]; in __hrtimer_init()
1642 static void __run_hrtimer(struct hrtimer_cpu_base *cpu_base, in __run_hrtimer() argument
1645 unsigned long flags) __must_hold(&cpu_base->lock) in __run_hrtimer()
1651 lockdep_assert_held(&cpu_base->lock); in __run_hrtimer()
1681 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in __run_hrtimer()
1689 raw_spin_lock_irq(&cpu_base->lock); in __run_hrtimer()
1717 static void __hrtimer_run_queues(struct hrtimer_cpu_base *cpu_base, ktime_t now, in __hrtimer_run_queues() argument
1721 unsigned int active = cpu_base->active_bases & active_mask; in __hrtimer_run_queues()
1723 for_each_active_base(base, cpu_base, active) { in __hrtimer_run_queues()
1749 __run_hrtimer(cpu_base, base, timer, &basenow, flags); in __hrtimer_run_queues()
1751 hrtimer_sync_wait_running(cpu_base, flags); in __hrtimer_run_queues()
1758 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); in hrtimer_run_softirq() local
1762 hrtimer_cpu_base_lock_expiry(cpu_base); in hrtimer_run_softirq()
1763 raw_spin_lock_irqsave(&cpu_base->lock, flags); in hrtimer_run_softirq()
1765 now = hrtimer_update_base(cpu_base); in hrtimer_run_softirq()
1766 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_SOFT); in hrtimer_run_softirq()
1768 cpu_base->softirq_activated = 0; in hrtimer_run_softirq()
1769 hrtimer_update_softirq_timer(cpu_base, true); in hrtimer_run_softirq()
1771 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in hrtimer_run_softirq()
1772 hrtimer_cpu_base_unlock_expiry(cpu_base); in hrtimer_run_softirq()
1783 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); in hrtimer_interrupt() local
1788 BUG_ON(!cpu_base->hres_active); in hrtimer_interrupt()
1789 cpu_base->nr_events++; in hrtimer_interrupt()
1792 raw_spin_lock_irqsave(&cpu_base->lock, flags); in hrtimer_interrupt()
1793 entry_time = now = hrtimer_update_base(cpu_base); in hrtimer_interrupt()
1795 cpu_base->in_hrtirq = 1; in hrtimer_interrupt()
1803 cpu_base->expires_next = KTIME_MAX; in hrtimer_interrupt()
1805 if (!ktime_before(now, cpu_base->softirq_expires_next)) { in hrtimer_interrupt()
1806 cpu_base->softirq_expires_next = KTIME_MAX; in hrtimer_interrupt()
1807 cpu_base->softirq_activated = 1; in hrtimer_interrupt()
1811 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD); in hrtimer_interrupt()
1814 expires_next = hrtimer_update_next_event(cpu_base); in hrtimer_interrupt()
1819 cpu_base->expires_next = expires_next; in hrtimer_interrupt()
1820 cpu_base->in_hrtirq = 0; in hrtimer_interrupt()
1821 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in hrtimer_interrupt()
1825 cpu_base->hang_detected = 0; in hrtimer_interrupt()
1842 raw_spin_lock_irqsave(&cpu_base->lock, flags); in hrtimer_interrupt()
1843 now = hrtimer_update_base(cpu_base); in hrtimer_interrupt()
1844 cpu_base->nr_retries++; in hrtimer_interrupt()
1853 cpu_base->nr_hangs++; in hrtimer_interrupt()
1854 cpu_base->hang_detected = 1; in hrtimer_interrupt()
1855 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in hrtimer_interrupt()
1858 if ((unsigned int)delta > cpu_base->max_hang_time) in hrtimer_interrupt()
1859 cpu_base->max_hang_time = (unsigned int) delta; in hrtimer_interrupt()
1896 struct hrtimer_cpu_base *cpu_base = this_cpu_ptr(&hrtimer_bases); in hrtimer_run_queues() local
1900 if (__hrtimer_hres_active(cpu_base)) in hrtimer_run_queues()
1915 raw_spin_lock_irqsave(&cpu_base->lock, flags); in hrtimer_run_queues()
1916 now = hrtimer_update_base(cpu_base); in hrtimer_run_queues()
1918 if (!ktime_before(now, cpu_base->softirq_expires_next)) { in hrtimer_run_queues()
1919 cpu_base->softirq_expires_next = KTIME_MAX; in hrtimer_run_queues()
1920 cpu_base->softirq_activated = 1; in hrtimer_run_queues()
1924 __hrtimer_run_queues(cpu_base, now, flags, HRTIMER_ACTIVE_HARD); in hrtimer_run_queues()
1925 raw_spin_unlock_irqrestore(&cpu_base->lock, flags); in hrtimer_run_queues()
2162 struct hrtimer_cpu_base *cpu_base = &per_cpu(hrtimer_bases, cpu); in hrtimers_prepare_cpu() local
2166 struct hrtimer_clock_base *clock_b = &cpu_base->clock_base[i]; in hrtimers_prepare_cpu()
2168 clock_b->cpu_base = cpu_base; in hrtimers_prepare_cpu()
2169 seqcount_raw_spinlock_init(&clock_b->seq, &cpu_base->lock); in hrtimers_prepare_cpu()
2173 cpu_base->cpu = cpu; in hrtimers_prepare_cpu()
2174 cpu_base->active_bases = 0; in hrtimers_prepare_cpu()
2175 cpu_base->hres_active = 0; in hrtimers_prepare_cpu()
2176 cpu_base->hang_detected = 0; in hrtimers_prepare_cpu()
2177 cpu_base->next_timer = NULL; in hrtimers_prepare_cpu()
2178 cpu_base->softirq_next_timer = NULL; in hrtimers_prepare_cpu()
2179 cpu_base->expires_next = KTIME_MAX; in hrtimers_prepare_cpu()
2180 cpu_base->softirq_expires_next = KTIME_MAX; in hrtimers_prepare_cpu()
2181 hrtimer_cpu_base_init_expiry_lock(cpu_base); in hrtimers_prepare_cpu()