1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  *  Housekeeping management. Manage the targets for routine code that can run on
4  *  any CPU: unbound workqueues, timers, kthreads and any offloadable work.
5  *
6  * Copyright (C) 2017 Red Hat, Inc., Frederic Weisbecker
7  * Copyright (C) 2017-2018 SUSE, Frederic Weisbecker
8  *
9  */
10 #include "sched.h"
11 
12 DEFINE_STATIC_KEY_FALSE(housekeeping_overridden);
13 EXPORT_SYMBOL_GPL(housekeeping_overridden);
14 static cpumask_var_t housekeeping_mask;
15 static unsigned int housekeeping_flags;
16 
housekeeping_enabled(enum hk_flags flags)17 bool housekeeping_enabled(enum hk_flags flags)
18 {
19 	return !!(housekeeping_flags & flags);
20 }
21 EXPORT_SYMBOL_GPL(housekeeping_enabled);
22 
housekeeping_any_cpu(enum hk_flags flags)23 int housekeeping_any_cpu(enum hk_flags flags)
24 {
25 	int cpu;
26 
27 	if (static_branch_unlikely(&housekeeping_overridden)) {
28 		if (housekeeping_flags & flags) {
29 			cpu = sched_numa_find_closest(housekeeping_mask, smp_processor_id());
30 			if (cpu < nr_cpu_ids)
31 				return cpu;
32 
33 			return cpumask_any_and(housekeeping_mask, cpu_online_mask);
34 		}
35 	}
36 	return smp_processor_id();
37 }
38 EXPORT_SYMBOL_GPL(housekeeping_any_cpu);
39 
housekeeping_cpumask(enum hk_flags flags)40 const struct cpumask *housekeeping_cpumask(enum hk_flags flags)
41 {
42 	if (static_branch_unlikely(&housekeeping_overridden))
43 		if (housekeeping_flags & flags)
44 			return housekeeping_mask;
45 	return cpu_possible_mask;
46 }
47 EXPORT_SYMBOL_GPL(housekeeping_cpumask);
48 
housekeeping_affine(struct task_struct * t,enum hk_flags flags)49 void housekeeping_affine(struct task_struct *t, enum hk_flags flags)
50 {
51 	if (static_branch_unlikely(&housekeeping_overridden))
52 		if (housekeeping_flags & flags)
53 			set_cpus_allowed_ptr(t, housekeeping_mask);
54 }
55 EXPORT_SYMBOL_GPL(housekeeping_affine);
56 
housekeeping_test_cpu(int cpu,enum hk_flags flags)57 bool housekeeping_test_cpu(int cpu, enum hk_flags flags)
58 {
59 	if (static_branch_unlikely(&housekeeping_overridden))
60 		if (housekeeping_flags & flags)
61 			return cpumask_test_cpu(cpu, housekeeping_mask);
62 	return true;
63 }
64 EXPORT_SYMBOL_GPL(housekeeping_test_cpu);
65 
housekeeping_init(void)66 void __init housekeeping_init(void)
67 {
68 	if (!housekeeping_flags)
69 		return;
70 
71 	static_branch_enable(&housekeeping_overridden);
72 
73 	if (housekeeping_flags & HK_FLAG_TICK)
74 		sched_tick_offload_init();
75 
76 	/* We need at least one CPU to handle housekeeping work */
77 	WARN_ON_ONCE(cpumask_empty(housekeeping_mask));
78 }
79 
housekeeping_setup(char * str,enum hk_flags flags)80 static int __init housekeeping_setup(char *str, enum hk_flags flags)
81 {
82 	cpumask_var_t non_housekeeping_mask;
83 	cpumask_var_t tmp;
84 
85 	alloc_bootmem_cpumask_var(&non_housekeeping_mask);
86 	if (cpulist_parse(str, non_housekeeping_mask) < 0) {
87 		pr_warn("Housekeeping: nohz_full= or isolcpus= incorrect CPU range\n");
88 		free_bootmem_cpumask_var(non_housekeeping_mask);
89 		return 0;
90 	}
91 
92 	alloc_bootmem_cpumask_var(&tmp);
93 	if (!housekeeping_flags) {
94 		alloc_bootmem_cpumask_var(&housekeeping_mask);
95 		cpumask_andnot(housekeeping_mask,
96 			       cpu_possible_mask, non_housekeeping_mask);
97 
98 		cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
99 		if (cpumask_empty(tmp)) {
100 			pr_warn("Housekeeping: must include one present CPU, "
101 				"using boot CPU:%d\n", smp_processor_id());
102 			__cpumask_set_cpu(smp_processor_id(), housekeeping_mask);
103 			__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
104 		}
105 	} else {
106 		cpumask_andnot(tmp, cpu_present_mask, non_housekeeping_mask);
107 		if (cpumask_empty(tmp))
108 			__cpumask_clear_cpu(smp_processor_id(), non_housekeeping_mask);
109 		cpumask_andnot(tmp, cpu_possible_mask, non_housekeeping_mask);
110 		if (!cpumask_equal(tmp, housekeeping_mask)) {
111 			pr_warn("Housekeeping: nohz_full= must match isolcpus=\n");
112 			free_bootmem_cpumask_var(tmp);
113 			free_bootmem_cpumask_var(non_housekeeping_mask);
114 			return 0;
115 		}
116 	}
117 	free_bootmem_cpumask_var(tmp);
118 
119 	if ((flags & HK_FLAG_TICK) && !(housekeeping_flags & HK_FLAG_TICK)) {
120 		if (IS_ENABLED(CONFIG_NO_HZ_FULL)) {
121 			tick_nohz_full_setup(non_housekeeping_mask);
122 		} else {
123 			pr_warn("Housekeeping: nohz unsupported."
124 				" Build with CONFIG_NO_HZ_FULL\n");
125 			free_bootmem_cpumask_var(non_housekeeping_mask);
126 			return 0;
127 		}
128 	}
129 
130 	housekeeping_flags |= flags;
131 
132 	free_bootmem_cpumask_var(non_housekeeping_mask);
133 
134 	return 1;
135 }
136 
housekeeping_nohz_full_setup(char * str)137 static int __init housekeeping_nohz_full_setup(char *str)
138 {
139 	unsigned int flags;
140 
141 	flags = HK_FLAG_TICK | HK_FLAG_WQ | HK_FLAG_TIMER | HK_FLAG_RCU |
142 		HK_FLAG_MISC | HK_FLAG_KTHREAD;
143 
144 	return housekeeping_setup(str, flags);
145 }
146 __setup("nohz_full=", housekeeping_nohz_full_setup);
147 
housekeeping_isolcpus_setup(char * str)148 static int __init housekeeping_isolcpus_setup(char *str)
149 {
150 	unsigned int flags = 0;
151 	bool illegal = false;
152 	char *par;
153 	int len;
154 
155 	while (isalpha(*str)) {
156 		if (!strncmp(str, "nohz,", 5)) {
157 			str += 5;
158 			flags |= HK_FLAG_TICK;
159 			continue;
160 		}
161 
162 		if (!strncmp(str, "domain,", 7)) {
163 			str += 7;
164 			flags |= HK_FLAG_DOMAIN;
165 			continue;
166 		}
167 
168 		if (!strncmp(str, "managed_irq,", 12)) {
169 			str += 12;
170 			flags |= HK_FLAG_MANAGED_IRQ;
171 			continue;
172 		}
173 
174 		/*
175 		 * Skip unknown sub-parameter and validate that it is not
176 		 * containing an invalid character.
177 		 */
178 		for (par = str, len = 0; *str && *str != ','; str++, len++) {
179 			if (!isalpha(*str) && *str != '_')
180 				illegal = true;
181 		}
182 
183 		if (illegal) {
184 			pr_warn("isolcpus: Invalid flag %.*s\n", len, par);
185 			return 0;
186 		}
187 
188 		pr_info("isolcpus: Skipped unknown flag %.*s\n", len, par);
189 		str++;
190 	}
191 
192 	/* Default behaviour for isolcpus without flags */
193 	if (!flags)
194 		flags |= HK_FLAG_DOMAIN;
195 
196 	return housekeeping_setup(str, flags);
197 }
198 __setup("isolcpus=", housekeeping_isolcpus_setup);
199