Lines Matching refs:worker
170 struct worker *manager; /* L: purely informational */
269 struct worker *rescuer; /* MD: rescue worker */
426 #define for_each_pool_worker(worker, pool) \ argument
427 list_for_each_entry((worker), &(pool)->workers, node) \
834 static struct worker *first_idle_worker(struct worker_pool *pool) in first_idle_worker()
839 return list_first_entry(&pool->idle_list, struct worker, entry); in first_idle_worker()
853 struct worker *worker = first_idle_worker(pool); in wake_up_worker() local
855 if (likely(worker)) in wake_up_worker()
856 wake_up_process(worker->task); in wake_up_worker()
867 struct worker *worker = kthread_data(task); in wq_worker_running() local
869 if (!worker->sleeping) in wq_worker_running()
871 if (!(worker->flags & WORKER_NOT_RUNNING)) in wq_worker_running()
872 atomic_inc(&worker->pool->nr_running); in wq_worker_running()
873 worker->sleeping = 0; in wq_worker_running()
886 struct worker *next, *worker = kthread_data(task); in wq_worker_sleeping() local
894 if (worker->flags & WORKER_NOT_RUNNING) in wq_worker_sleeping()
897 pool = worker->pool; in wq_worker_sleeping()
900 if (worker->sleeping) in wq_worker_sleeping()
903 worker->sleeping = 1; in wq_worker_sleeping()
952 struct worker *worker = kthread_data(task); in wq_worker_last_func() local
954 return worker->last_func; in wq_worker_last_func()
967 static inline void worker_set_flags(struct worker *worker, unsigned int flags) in worker_set_flags() argument
969 struct worker_pool *pool = worker->pool; in worker_set_flags()
971 WARN_ON_ONCE(worker->task != current); in worker_set_flags()
975 !(worker->flags & WORKER_NOT_RUNNING)) { in worker_set_flags()
979 worker->flags |= flags; in worker_set_flags()
992 static inline void worker_clr_flags(struct worker *worker, unsigned int flags) in worker_clr_flags() argument
994 struct worker_pool *pool = worker->pool; in worker_clr_flags()
995 unsigned int oflags = worker->flags; in worker_clr_flags()
997 WARN_ON_ONCE(worker->task != current); in worker_clr_flags()
999 worker->flags &= ~flags; in worker_clr_flags()
1007 if (!(worker->flags & WORKER_NOT_RUNNING)) in worker_clr_flags()
1044 static struct worker *find_worker_executing_work(struct worker_pool *pool, in find_worker_executing_work()
1047 struct worker *worker; in find_worker_executing_work() local
1049 hash_for_each_possible(pool->busy_hash, worker, hentry, in find_worker_executing_work()
1051 if (worker->current_work == work && in find_worker_executing_work()
1052 worker->current_func == work->func) in find_worker_executing_work()
1053 return worker; in find_worker_executing_work()
1378 struct worker *worker; in is_chained_work() local
1380 worker = current_wq_worker(); in is_chained_work()
1385 return worker && worker->current_pwq->wq == wq; in is_chained_work()
1463 struct worker *worker; in __queue_work() local
1467 worker = find_worker_executing_work(last_pool, work); in __queue_work()
1469 if (worker && worker->current_pwq->wq == wq) { in __queue_work()
1470 pwq = worker->current_pwq; in __queue_work()
1794 static void worker_enter_idle(struct worker *worker) in worker_enter_idle() argument
1796 struct worker_pool *pool = worker->pool; in worker_enter_idle()
1798 if (WARN_ON_ONCE(worker->flags & WORKER_IDLE) || in worker_enter_idle()
1799 WARN_ON_ONCE(!list_empty(&worker->entry) && in worker_enter_idle()
1800 (worker->hentry.next || worker->hentry.pprev))) in worker_enter_idle()
1804 worker->flags |= WORKER_IDLE; in worker_enter_idle()
1806 worker->last_active = jiffies; in worker_enter_idle()
1809 list_add(&worker->entry, &pool->idle_list); in worker_enter_idle()
1834 static void worker_leave_idle(struct worker *worker) in worker_leave_idle() argument
1836 struct worker_pool *pool = worker->pool; in worker_leave_idle()
1838 if (WARN_ON_ONCE(!(worker->flags & WORKER_IDLE))) in worker_leave_idle()
1840 worker_clr_flags(worker, WORKER_IDLE); in worker_leave_idle()
1842 list_del_init(&worker->entry); in worker_leave_idle()
1845 static struct worker *alloc_worker(int node) in alloc_worker()
1847 struct worker *worker; in alloc_worker() local
1849 worker = kzalloc_node(sizeof(*worker), GFP_KERNEL, node); in alloc_worker()
1850 if (worker) { in alloc_worker()
1851 INIT_LIST_HEAD(&worker->entry); in alloc_worker()
1852 INIT_LIST_HEAD(&worker->scheduled); in alloc_worker()
1853 INIT_LIST_HEAD(&worker->node); in alloc_worker()
1855 worker->flags = WORKER_PREP; in alloc_worker()
1857 return worker; in alloc_worker()
1869 static void worker_attach_to_pool(struct worker *worker, in worker_attach_to_pool() argument
1880 worker->flags |= WORKER_UNBOUND; in worker_attach_to_pool()
1882 kthread_set_per_cpu(worker->task, pool->cpu); in worker_attach_to_pool()
1884 if (worker->rescue_wq) in worker_attach_to_pool()
1885 set_cpus_allowed_ptr(worker->task, pool->attrs->cpumask); in worker_attach_to_pool()
1887 list_add_tail(&worker->node, &pool->workers); in worker_attach_to_pool()
1888 worker->pool = pool; in worker_attach_to_pool()
1901 static void worker_detach_from_pool(struct worker *worker) in worker_detach_from_pool() argument
1903 struct worker_pool *pool = worker->pool; in worker_detach_from_pool()
1908 kthread_set_per_cpu(worker->task, -1); in worker_detach_from_pool()
1909 list_del(&worker->node); in worker_detach_from_pool()
1910 worker->pool = NULL; in worker_detach_from_pool()
1917 worker->flags &= ~(WORKER_UNBOUND | WORKER_REBOUND); in worker_detach_from_pool()
1935 static struct worker *create_worker(struct worker_pool *pool) in create_worker()
1937 struct worker *worker; in create_worker() local
1946 worker = alloc_worker(pool->node); in create_worker()
1947 if (!worker) in create_worker()
1950 worker->id = id; in create_worker()
1958 worker->task = kthread_create_on_node(worker_thread, worker, pool->node, in create_worker()
1960 if (IS_ERR(worker->task)) in create_worker()
1963 set_user_nice(worker->task, pool->attrs->nice); in create_worker()
1964 kthread_bind_mask(worker->task, pool->attrs->cpumask); in create_worker()
1967 worker_attach_to_pool(worker, pool); in create_worker()
1971 worker->pool->nr_workers++; in create_worker()
1972 worker_enter_idle(worker); in create_worker()
1973 wake_up_process(worker->task); in create_worker()
1976 return worker; in create_worker()
1980 kfree(worker); in create_worker()
1994 static void destroy_worker(struct worker *worker) in destroy_worker() argument
1996 struct worker_pool *pool = worker->pool; in destroy_worker()
2001 if (WARN_ON(worker->current_work) || in destroy_worker()
2002 WARN_ON(!list_empty(&worker->scheduled)) || in destroy_worker()
2003 WARN_ON(!(worker->flags & WORKER_IDLE))) in destroy_worker()
2009 list_del_init(&worker->entry); in destroy_worker()
2010 worker->flags |= WORKER_DIE; in destroy_worker()
2011 wake_up_process(worker->task); in destroy_worker()
2021 struct worker *worker; in idle_worker_timeout() local
2025 worker = list_entry(pool->idle_list.prev, struct worker, entry); in idle_worker_timeout()
2026 expires = worker->last_active + IDLE_WORKER_TIMEOUT; in idle_worker_timeout()
2033 destroy_worker(worker); in idle_worker_timeout()
2158 static bool manage_workers(struct worker *worker) in manage_workers() argument
2160 struct worker_pool *pool = worker->pool; in manage_workers()
2166 pool->manager = worker; in manage_workers()
2190 static void process_one_work(struct worker *worker, struct work_struct *work) in process_one_work() argument
2195 struct worker_pool *pool = worker->pool; in process_one_work()
2198 struct worker *collision; in process_one_work()
2229 hash_add(pool->busy_hash, &worker->hentry, (unsigned long)work); in process_one_work()
2230 worker->current_work = work; in process_one_work()
2231 worker->current_func = work->func; in process_one_work()
2232 worker->current_pwq = pwq; in process_one_work()
2234 worker->current_color = get_work_color(work_data); in process_one_work()
2240 strscpy(worker->desc, pwq->wq->name, WORKER_DESC_LEN); in process_one_work()
2251 worker_set_flags(worker, WORKER_CPU_INTENSIVE); in process_one_work()
2298 worker->current_func(work); in process_one_work()
2303 trace_workqueue_execute_end(work, worker->current_func); in process_one_work()
2311 worker->current_func); in process_one_work()
2330 worker_clr_flags(worker, WORKER_CPU_INTENSIVE); in process_one_work()
2333 worker->last_func = worker->current_func; in process_one_work()
2336 hash_del(&worker->hentry); in process_one_work()
2337 worker->current_work = NULL; in process_one_work()
2338 worker->current_func = NULL; in process_one_work()
2339 worker->current_pwq = NULL; in process_one_work()
2340 worker->current_color = INT_MAX; in process_one_work()
2356 static void process_scheduled_works(struct worker *worker) in process_scheduled_works() argument
2358 while (!list_empty(&worker->scheduled)) { in process_scheduled_works()
2359 struct work_struct *work = list_first_entry(&worker->scheduled, in process_scheduled_works()
2361 process_one_work(worker, work); in process_scheduled_works()
2389 struct worker *worker = __worker; in worker_thread() local
2390 struct worker_pool *pool = worker->pool; in worker_thread()
2398 if (unlikely(worker->flags & WORKER_DIE)) { in worker_thread()
2400 WARN_ON_ONCE(!list_empty(&worker->entry)); in worker_thread()
2403 set_task_comm(worker->task, "kworker/dying"); in worker_thread()
2404 ida_free(&pool->worker_ida, worker->id); in worker_thread()
2405 worker_detach_from_pool(worker); in worker_thread()
2406 kfree(worker); in worker_thread()
2410 worker_leave_idle(worker); in worker_thread()
2417 if (unlikely(!may_start_working(pool)) && manage_workers(worker)) in worker_thread()
2425 WARN_ON_ONCE(!list_empty(&worker->scheduled)); in worker_thread()
2434 worker_clr_flags(worker, WORKER_PREP | WORKER_REBOUND); in worker_thread()
2445 process_one_work(worker, work); in worker_thread()
2446 if (unlikely(!list_empty(&worker->scheduled))) in worker_thread()
2447 process_scheduled_works(worker); in worker_thread()
2449 move_linked_works(work, &worker->scheduled, NULL); in worker_thread()
2450 process_scheduled_works(worker); in worker_thread()
2454 worker_set_flags(worker, WORKER_PREP); in worker_thread()
2463 worker_enter_idle(worker); in worker_thread()
2493 struct worker *rescuer = __rescuer; in rescuer_thread()
2627 struct worker *worker; in check_flush_dependency() local
2632 worker = current_wq_worker(); in check_flush_dependency()
2637 WARN_ONCE(worker && ((worker->current_pwq->wq->flags & in check_flush_dependency()
2640 worker->current_pwq->wq->name, worker->current_func, in check_flush_dependency()
2682 struct work_struct *target, struct worker *worker) in insert_wq_barrier() argument
2708 if (worker) { in insert_wq_barrier()
2709 head = worker->scheduled.next; in insert_wq_barrier()
2710 work_color = worker->current_color; in insert_wq_barrier()
3015 struct worker *worker = NULL; in start_flush_work() local
3035 worker = find_worker_executing_work(pool, work); in start_flush_work()
3036 if (!worker) in start_flush_work()
3038 pwq = worker->current_pwq; in start_flush_work()
3043 insert_wq_barrier(pwq, barr, work, worker); in start_flush_work()
3571 struct worker *worker; in put_unbound_pool() local
3599 while ((worker = first_idle_worker(pool))) in put_unbound_pool()
3600 destroy_worker(worker); in put_unbound_pool()
4263 struct worker *rescuer; in init_rescuer()
4424 struct worker *rescuer = wq->rescuer; in destroy_workqueue()
4537 struct worker *worker = current_wq_worker(); in current_work() local
4539 return worker ? worker->current_work : NULL; in current_work()
4553 struct worker *worker = current_wq_worker(); in current_is_workqueue_rescuer() local
4555 return worker && worker->rescue_wq; in current_is_workqueue_rescuer()
4646 struct worker *worker = current_wq_worker(); in set_worker_desc() local
4649 if (worker) { in set_worker_desc()
4651 vsnprintf(worker->desc, sizeof(worker->desc), fmt, args); in set_worker_desc()
4677 struct worker *worker; in print_worker_info() local
4686 worker = kthread_probe_data(task); in print_worker_info()
4692 copy_from_kernel_nofault(&fn, &worker->current_func, sizeof(fn)); in print_worker_info()
4693 copy_from_kernel_nofault(&pwq, &worker->current_pwq, sizeof(pwq)); in print_worker_info()
4696 copy_from_kernel_nofault(desc, worker->desc, sizeof(desc) - 1); in print_worker_info()
4732 struct worker *worker; in show_pwq() local
4743 hash_for_each(pool->busy_hash, bkt, worker, hentry) { in show_pwq()
4744 if (worker->current_pwq == pwq) { in show_pwq()
4753 hash_for_each(pool->busy_hash, bkt, worker, hentry) { in show_pwq()
4754 if (worker->current_pwq != pwq) in show_pwq()
4758 task_pid_nr(worker->task), in show_pwq()
4759 worker->rescue_wq ? "(RESCUER)" : "", in show_pwq()
4760 worker->current_func); in show_pwq()
4761 list_for_each_entry(work, &worker->scheduled, entry) in show_pwq()
4850 struct worker *worker; in show_one_worker_pool() local
4871 list_for_each_entry(worker, &pool->idle_list, entry) { in show_one_worker_pool()
4873 task_pid_nr(worker->task)); in show_one_worker_pool()
4928 struct worker *worker = kthread_data(task); in wq_worker_comm() local
4929 struct worker_pool *pool = worker->pool; in wq_worker_comm()
4938 if (worker->desc[0] != '\0') { in wq_worker_comm()
4939 if (worker->current_work) in wq_worker_comm()
4941 worker->desc); in wq_worker_comm()
4944 worker->desc); in wq_worker_comm()
4973 struct worker *worker; in unbind_workers() local
4986 for_each_pool_worker(worker, pool) in unbind_workers()
4987 worker->flags |= WORKER_UNBOUND; in unbind_workers()
4993 for_each_pool_worker(worker, pool) { in unbind_workers()
4994 kthread_set_per_cpu(worker->task, -1); in unbind_workers()
4995 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, cpu_possible_mask) < 0); in unbind_workers()
5037 struct worker *worker; in rebind_workers() local
5048 for_each_pool_worker(worker, pool) { in rebind_workers()
5049 kthread_set_per_cpu(worker->task, pool->cpu); in rebind_workers()
5050 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, in rebind_workers()
5058 for_each_pool_worker(worker, pool) { in rebind_workers()
5059 unsigned int worker_flags = worker->flags; in rebind_workers()
5070 wake_up_process(worker->task); in rebind_workers()
5090 WRITE_ONCE(worker->flags, worker_flags); in rebind_workers()
5109 struct worker *worker; in restore_unbound_workers_cpumask() local
5120 for_each_pool_worker(worker, pool) in restore_unbound_workers_cpumask()
5121 WARN_ON_ONCE(set_cpus_allowed_ptr(worker->task, &cpumask) < 0); in restore_unbound_workers_cpumask()