Lines Matching refs:timer
165 struct hrtimer_clock_base *lock_hrtimer_base(const struct hrtimer *timer, in lock_hrtimer_base() argument
171 base = READ_ONCE(timer->base); in lock_hrtimer_base()
174 if (likely(base == timer->base)) in lock_hrtimer_base()
193 hrtimer_check_target(struct hrtimer *timer, struct hrtimer_clock_base *new_base) in hrtimer_check_target() argument
197 expires = ktime_sub(hrtimer_get_expires(timer), new_base->offset); in hrtimer_check_target()
225 switch_hrtimer_base(struct hrtimer *timer, struct hrtimer_clock_base *base, in switch_hrtimer_base() argument
247 if (unlikely(hrtimer_callback_running(timer))) in switch_hrtimer_base()
251 WRITE_ONCE(timer->base, &migration_base); in switch_hrtimer_base()
256 hrtimer_check_target(timer, new_base)) { in switch_hrtimer_base()
260 WRITE_ONCE(timer->base, base); in switch_hrtimer_base()
263 WRITE_ONCE(timer->base, new_base); in switch_hrtimer_base()
266 hrtimer_check_target(timer, new_base)) { in switch_hrtimer_base()
282 lock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) in lock_hrtimer_base() argument
284 struct hrtimer_clock_base *base = timer->base; in lock_hrtimer_base()
358 struct hrtimer *timer = addr; in hrtimer_fixup_init() local
362 hrtimer_cancel(timer); in hrtimer_fixup_init()
363 debug_object_init(timer, &hrtimer_debug_descr); in hrtimer_fixup_init()
392 struct hrtimer *timer = addr; in hrtimer_fixup_free() local
396 hrtimer_cancel(timer); in hrtimer_fixup_free()
397 debug_object_free(timer, &hrtimer_debug_descr); in hrtimer_fixup_free()
412 static inline void debug_hrtimer_init(struct hrtimer *timer) in debug_hrtimer_init() argument
414 debug_object_init(timer, &hrtimer_debug_descr); in debug_hrtimer_init()
417 static inline void debug_hrtimer_activate(struct hrtimer *timer, in debug_hrtimer_activate() argument
420 debug_object_activate(timer, &hrtimer_debug_descr); in debug_hrtimer_activate()
423 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) in debug_hrtimer_deactivate() argument
425 debug_object_deactivate(timer, &hrtimer_debug_descr); in debug_hrtimer_deactivate()
428 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id,
431 void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t clock_id, in hrtimer_init_on_stack() argument
434 debug_object_init_on_stack(timer, &hrtimer_debug_descr); in hrtimer_init_on_stack()
435 __hrtimer_init(timer, clock_id, mode); in hrtimer_init_on_stack()
445 debug_object_init_on_stack(&sl->timer, &hrtimer_debug_descr); in hrtimer_init_sleeper_on_stack()
450 void destroy_hrtimer_on_stack(struct hrtimer *timer) in destroy_hrtimer_on_stack() argument
452 debug_object_free(timer, &hrtimer_debug_descr); in destroy_hrtimer_on_stack()
458 static inline void debug_hrtimer_init(struct hrtimer *timer) { } in debug_hrtimer_init() argument
459 static inline void debug_hrtimer_activate(struct hrtimer *timer, in debug_hrtimer_activate() argument
461 static inline void debug_hrtimer_deactivate(struct hrtimer *timer) { } in debug_hrtimer_deactivate() argument
465 debug_init(struct hrtimer *timer, clockid_t clockid, in debug_init() argument
468 debug_hrtimer_init(timer); in debug_init()
469 trace_hrtimer_init(timer, clockid, mode); in debug_init()
472 static inline void debug_activate(struct hrtimer *timer, in debug_activate() argument
475 debug_hrtimer_activate(timer, mode); in debug_activate()
476 trace_hrtimer_start(timer, mode); in debug_activate()
479 static inline void debug_deactivate(struct hrtimer *timer) in debug_deactivate() argument
481 debug_hrtimer_deactivate(timer); in debug_deactivate()
482 trace_hrtimer_cancel(timer); in debug_deactivate()
512 struct hrtimer *timer; in __hrtimer_next_event_base() local
515 timer = container_of(next, struct hrtimer, node); in __hrtimer_next_event_base()
516 if (timer == exclude) { in __hrtimer_next_event_base()
522 timer = container_of(next, struct hrtimer, node); in __hrtimer_next_event_base()
524 expires = ktime_sub(hrtimer_get_expires(timer), base->offset); in __hrtimer_next_event_base()
532 if (timer->is_soft) in __hrtimer_next_event_base()
533 cpu_base->softirq_next_timer = timer; in __hrtimer_next_event_base()
535 cpu_base->next_timer = timer; in __hrtimer_next_event_base()
808 static void hrtimer_reprogram(struct hrtimer *timer, bool reprogram) in hrtimer_reprogram() argument
811 struct hrtimer_clock_base *base = timer->base; in hrtimer_reprogram()
812 ktime_t expires = ktime_sub(hrtimer_get_expires(timer), base->offset); in hrtimer_reprogram()
814 WARN_ON_ONCE(hrtimer_get_expires_tv64(timer) < 0); in hrtimer_reprogram()
823 if (timer->is_soft) { in hrtimer_reprogram()
839 timer_cpu_base->softirq_next_timer = timer; in hrtimer_reprogram()
864 cpu_base->next_timer = timer; in hrtimer_reprogram()
866 __hrtimer_reprogram(cpu_base, timer, expires); in hrtimer_reprogram()
1015 void unlock_hrtimer_base(const struct hrtimer *timer, unsigned long *flags) in unlock_hrtimer_base() argument
1017 raw_spin_unlock_irqrestore(&timer->base->cpu_base->lock, *flags); in unlock_hrtimer_base()
1037 u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval) in hrtimer_forward() argument
1042 delta = ktime_sub(now, hrtimer_get_expires(timer)); in hrtimer_forward()
1047 if (WARN_ON(timer->state & HRTIMER_STATE_ENQUEUED)) in hrtimer_forward()
1057 hrtimer_add_expires_ns(timer, incr * orun); in hrtimer_forward()
1058 if (hrtimer_get_expires_tv64(timer) > now) in hrtimer_forward()
1066 hrtimer_add_expires(timer, interval); in hrtimer_forward()
1080 static int enqueue_hrtimer(struct hrtimer *timer, in enqueue_hrtimer() argument
1084 debug_activate(timer, mode); in enqueue_hrtimer()
1089 WRITE_ONCE(timer->state, HRTIMER_STATE_ENQUEUED); in enqueue_hrtimer()
1091 return timerqueue_add(&base->active, &timer->node); in enqueue_hrtimer()
1104 static void __remove_hrtimer(struct hrtimer *timer, in __remove_hrtimer() argument
1109 u8 state = timer->state; in __remove_hrtimer()
1112 WRITE_ONCE(timer->state, newstate); in __remove_hrtimer()
1116 if (!timerqueue_del(&base->active, &timer->node)) in __remove_hrtimer()
1127 if (reprogram && timer == cpu_base->next_timer) in __remove_hrtimer()
1135 remove_hrtimer(struct hrtimer *timer, struct hrtimer_clock_base *base, in remove_hrtimer() argument
1138 u8 state = timer->state; in remove_hrtimer()
1151 debug_deactivate(timer); in remove_hrtimer()
1165 __remove_hrtimer(timer, base, state, reprogram); in remove_hrtimer()
1171 static inline ktime_t hrtimer_update_lowres(struct hrtimer *timer, ktime_t tim, in hrtimer_update_lowres() argument
1180 timer->is_rel = mode & HRTIMER_MODE_REL; in hrtimer_update_lowres()
1181 if (timer->is_rel) in hrtimer_update_lowres()
1212 static int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, in __hrtimer_start_range_ns() argument
1228 force_local &= base->cpu_base->next_timer == timer; in __hrtimer_start_range_ns()
1241 remove_hrtimer(timer, base, true, force_local); in __hrtimer_start_range_ns()
1246 tim = hrtimer_update_lowres(timer, tim, mode); in __hrtimer_start_range_ns()
1248 hrtimer_set_expires_range_ns(timer, tim, delta_ns); in __hrtimer_start_range_ns()
1252 new_base = switch_hrtimer_base(timer, base, in __hrtimer_start_range_ns()
1258 first = enqueue_hrtimer(timer, new_base, mode); in __hrtimer_start_range_ns()
1280 void hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, in hrtimer_start_range_ns() argument
1292 WARN_ON_ONCE(!(mode & HRTIMER_MODE_SOFT) ^ !timer->is_soft); in hrtimer_start_range_ns()
1294 WARN_ON_ONCE(!(mode & HRTIMER_MODE_HARD) ^ !timer->is_hard); in hrtimer_start_range_ns()
1296 base = lock_hrtimer_base(timer, &flags); in hrtimer_start_range_ns()
1298 if (__hrtimer_start_range_ns(timer, tim, delta_ns, mode, base)) in hrtimer_start_range_ns()
1299 hrtimer_reprogram(timer, true); in hrtimer_start_range_ns()
1301 unlock_hrtimer_base(timer, &flags); in hrtimer_start_range_ns()
1316 int hrtimer_try_to_cancel(struct hrtimer *timer) in hrtimer_try_to_cancel() argument
1328 if (!hrtimer_active(timer)) in hrtimer_try_to_cancel()
1331 base = lock_hrtimer_base(timer, &flags); in hrtimer_try_to_cancel()
1333 if (!hrtimer_callback_running(timer)) in hrtimer_try_to_cancel()
1334 ret = remove_hrtimer(timer, base, false, false); in hrtimer_try_to_cancel()
1336 unlock_hrtimer_base(timer, &flags); in hrtimer_try_to_cancel()
1393 void hrtimer_cancel_wait_running(const struct hrtimer *timer) in hrtimer_cancel_wait_running() argument
1396 struct hrtimer_clock_base *base = READ_ONCE(timer->base); in hrtimer_cancel_wait_running()
1402 if (!timer->is_soft || is_migration_base(base)) { in hrtimer_cancel_wait_running()
1438 int hrtimer_cancel(struct hrtimer *timer) in hrtimer_cancel() argument
1443 ret = hrtimer_try_to_cancel(timer); in hrtimer_cancel()
1446 hrtimer_cancel_wait_running(timer); in hrtimer_cancel()
1457 ktime_t __hrtimer_get_remaining(const struct hrtimer *timer, bool adjust) in __hrtimer_get_remaining() argument
1462 lock_hrtimer_base(timer, &flags); in __hrtimer_get_remaining()
1464 rem = hrtimer_expires_remaining_adjusted(timer); in __hrtimer_get_remaining()
1466 rem = hrtimer_expires_remaining(timer); in __hrtimer_get_remaining()
1467 unlock_hrtimer_base(timer, &flags); in __hrtimer_get_remaining()
1541 static void __hrtimer_init(struct hrtimer *timer, clockid_t clock_id, in __hrtimer_init() argument
1557 memset(timer, 0, sizeof(struct hrtimer)); in __hrtimer_init()
1571 timer->is_soft = softtimer; in __hrtimer_init()
1572 timer->is_hard = !!(mode & HRTIMER_MODE_HARD); in __hrtimer_init()
1573 timer->base = &cpu_base->clock_base[base]; in __hrtimer_init()
1574 timerqueue_init(&timer->node); in __hrtimer_init()
1589 void hrtimer_init(struct hrtimer *timer, clockid_t clock_id, in hrtimer_init() argument
1592 debug_init(timer, clock_id, mode); in hrtimer_init()
1593 __hrtimer_init(timer, clock_id, mode); in hrtimer_init()
1604 bool hrtimer_active(const struct hrtimer *timer) in hrtimer_active() argument
1610 base = READ_ONCE(timer->base); in hrtimer_active()
1613 if (timer->state != HRTIMER_STATE_INACTIVE || in hrtimer_active()
1614 base->running == timer) in hrtimer_active()
1618 base != READ_ONCE(timer->base)); in hrtimer_active()
1644 struct hrtimer *timer, ktime_t *now, in __run_hrtimer() argument
1653 debug_deactivate(timer); in __run_hrtimer()
1654 base->running = timer; in __run_hrtimer()
1665 __remove_hrtimer(timer, base, HRTIMER_STATE_INACTIVE, 0); in __run_hrtimer()
1666 fn = timer->function; in __run_hrtimer()
1674 timer->is_rel = false; in __run_hrtimer()
1682 trace_hrtimer_expire_entry(timer, now); in __run_hrtimer()
1683 expires_in_hardirq = lockdep_hrtimer_enter(timer); in __run_hrtimer()
1685 restart = fn(timer); in __run_hrtimer()
1688 trace_hrtimer_expire_exit(timer); in __run_hrtimer()
1701 !(timer->state & HRTIMER_STATE_ENQUEUED)) in __run_hrtimer()
1702 enqueue_hrtimer(timer, base, HRTIMER_MODE_ABS); in __run_hrtimer()
1713 WARN_ON_ONCE(base->running != timer); in __run_hrtimer()
1730 struct hrtimer *timer; in __hrtimer_run_queues() local
1732 timer = container_of(node, struct hrtimer, node); in __hrtimer_run_queues()
1746 if (basenow < hrtimer_get_softexpires_tv64(timer)) in __hrtimer_run_queues()
1749 __run_hrtimer(cpu_base, base, timer, &basenow, flags); in __hrtimer_run_queues()
1931 static enum hrtimer_restart hrtimer_wakeup(struct hrtimer *timer) in hrtimer_wakeup() argument
1934 container_of(timer, struct hrtimer_sleeper, timer); in hrtimer_wakeup()
1962 if (IS_ENABLED(CONFIG_PREEMPT_RT) && sl->timer.is_hard) in hrtimer_sleeper_start_expires()
1965 hrtimer_start_expires(&sl->timer, mode); in hrtimer_sleeper_start_expires()
1996 __hrtimer_init(&sl->timer, clock_id, mode); in __hrtimer_init_sleeper()
1997 sl->timer.function = hrtimer_wakeup; in __hrtimer_init_sleeper()
2010 debug_init(&sl->timer, clock_id, mode); in hrtimer_init_sleeper()
2046 hrtimer_cancel(&t->timer); in do_nanosleep()
2058 ktime_t rem = hrtimer_expires_remaining(&t->timer); in do_nanosleep()
2077 hrtimer_set_expires_tv64(&t.timer, restart->nanosleep.expires); in hrtimer_nanosleep_restart()
2079 destroy_hrtimer_on_stack(&t.timer); in hrtimer_nanosleep_restart()
2096 hrtimer_set_expires_range_ns(&t.timer, rqtp, slack); in hrtimer_nanosleep()
2108 restart->nanosleep.clockid = t.timer.base->clockid; in hrtimer_nanosleep()
2109 restart->nanosleep.expires = hrtimer_get_expires_tv64(&t.timer); in hrtimer_nanosleep()
2112 destroy_hrtimer_on_stack(&t.timer); in hrtimer_nanosleep()
2190 struct hrtimer *timer; in migrate_hrtimer_list() local
2194 timer = container_of(node, struct hrtimer, node); in migrate_hrtimer_list()
2195 BUG_ON(hrtimer_callback_running(timer)); in migrate_hrtimer_list()
2196 debug_deactivate(timer); in migrate_hrtimer_list()
2203 __remove_hrtimer(timer, old_base, HRTIMER_STATE_ENQUEUED, 0); in migrate_hrtimer_list()
2204 timer->base = new_base; in migrate_hrtimer_list()
2213 enqueue_hrtimer(timer, new_base, HRTIMER_MODE_ABS); in migrate_hrtimer_list()
2301 hrtimer_set_expires_range_ns(&t.timer, *expires, delta); in schedule_hrtimeout_range_clock()
2307 hrtimer_cancel(&t.timer); in schedule_hrtimeout_range_clock()
2308 destroy_hrtimer_on_stack(&t.timer); in schedule_hrtimeout_range_clock()