1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kdebug.h>
3 #include <linux/kprobes.h>
4 #include <linux/export.h>
5 #include <linux/notifier.h>
6 #include <linux/rcupdate.h>
7 #include <linux/vmalloc.h>
8 #include <linux/reboot.h>
9
10 /*
11 * Notifier list for kernel code which wants to be called
12 * at shutdown. This is used to stop any idling DMA operations
13 * and the like.
14 */
15 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
16
17 /*
18 * Notifier chain core routines. The exported routines below
19 * are layered on top of these, with appropriate locking added.
20 */
21
notifier_chain_register(struct notifier_block ** nl,struct notifier_block * n)22 static int notifier_chain_register(struct notifier_block **nl,
23 struct notifier_block *n)
24 {
25 while ((*nl) != NULL) {
26 if (unlikely((*nl) == n)) {
27 WARN(1, "double register detected");
28 return 0;
29 }
30 if (n->priority > (*nl)->priority)
31 break;
32 nl = &((*nl)->next);
33 }
34 n->next = *nl;
35 rcu_assign_pointer(*nl, n);
36 return 0;
37 }
38
notifier_chain_unregister(struct notifier_block ** nl,struct notifier_block * n)39 static int notifier_chain_unregister(struct notifier_block **nl,
40 struct notifier_block *n)
41 {
42 while ((*nl) != NULL) {
43 if ((*nl) == n) {
44 rcu_assign_pointer(*nl, n->next);
45 return 0;
46 }
47 nl = &((*nl)->next);
48 }
49 return -ENOENT;
50 }
51
52 /**
53 * notifier_call_chain - Informs the registered notifiers about an event.
54 * @nl: Pointer to head of the blocking notifier chain
55 * @val: Value passed unmodified to notifier function
56 * @v: Pointer passed unmodified to notifier function
57 * @nr_to_call: Number of notifier functions to be called. Don't care
58 * value of this parameter is -1.
59 * @nr_calls: Records the number of notifications sent. Don't care
60 * value of this field is NULL.
61 * @returns: notifier_call_chain returns the value returned by the
62 * last notifier function called.
63 */
notifier_call_chain(struct notifier_block ** nl,unsigned long val,void * v,int nr_to_call,int * nr_calls)64 static int notifier_call_chain(struct notifier_block **nl,
65 unsigned long val, void *v,
66 int nr_to_call, int *nr_calls)
67 {
68 int ret = NOTIFY_DONE;
69 struct notifier_block *nb, *next_nb;
70
71 nb = rcu_dereference_raw(*nl);
72
73 while (nb && nr_to_call) {
74 next_nb = rcu_dereference_raw(nb->next);
75
76 #ifdef CONFIG_DEBUG_NOTIFIERS
77 if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
78 WARN(1, "Invalid notifier called!");
79 nb = next_nb;
80 continue;
81 }
82 #endif
83 ret = nb->notifier_call(nb, val, v);
84
85 if (nr_calls)
86 (*nr_calls)++;
87
88 if (ret & NOTIFY_STOP_MASK)
89 break;
90 nb = next_nb;
91 nr_to_call--;
92 }
93 return ret;
94 }
95 NOKPROBE_SYMBOL(notifier_call_chain);
96
97 /**
98 * notifier_call_chain_robust - Inform the registered notifiers about an event
99 * and rollback on error.
100 * @nl: Pointer to head of the blocking notifier chain
101 * @val_up: Value passed unmodified to the notifier function
102 * @val_down: Value passed unmodified to the notifier function when recovering
103 * from an error on @val_up
104 * @v Pointer passed unmodified to the notifier function
105 *
106 * NOTE: It is important the @nl chain doesn't change between the two
107 * invocations of notifier_call_chain() such that we visit the
108 * exact same notifier callbacks; this rules out any RCU usage.
109 *
110 * Returns: the return value of the @val_up call.
111 */
notifier_call_chain_robust(struct notifier_block ** nl,unsigned long val_up,unsigned long val_down,void * v)112 static int notifier_call_chain_robust(struct notifier_block **nl,
113 unsigned long val_up, unsigned long val_down,
114 void *v)
115 {
116 int ret, nr = 0;
117
118 ret = notifier_call_chain(nl, val_up, v, -1, &nr);
119 if (ret & NOTIFY_STOP_MASK)
120 notifier_call_chain(nl, val_down, v, nr-1, NULL);
121
122 return ret;
123 }
124
125 /*
126 * Atomic notifier chain routines. Registration and unregistration
127 * use a spinlock, and call_chain is synchronized by RCU (no locks).
128 */
129
130 /**
131 * atomic_notifier_chain_register - Add notifier to an atomic notifier chain
132 * @nh: Pointer to head of the atomic notifier chain
133 * @n: New entry in notifier chain
134 *
135 * Adds a notifier to an atomic notifier chain.
136 *
137 * Currently always returns zero.
138 */
atomic_notifier_chain_register(struct atomic_notifier_head * nh,struct notifier_block * n)139 int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
140 struct notifier_block *n)
141 {
142 unsigned long flags;
143 int ret;
144
145 spin_lock_irqsave(&nh->lock, flags);
146 ret = notifier_chain_register(&nh->head, n);
147 spin_unlock_irqrestore(&nh->lock, flags);
148 return ret;
149 }
150 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
151
152 /**
153 * atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
154 * @nh: Pointer to head of the atomic notifier chain
155 * @n: Entry to remove from notifier chain
156 *
157 * Removes a notifier from an atomic notifier chain.
158 *
159 * Returns zero on success or %-ENOENT on failure.
160 */
atomic_notifier_chain_unregister(struct atomic_notifier_head * nh,struct notifier_block * n)161 int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
162 struct notifier_block *n)
163 {
164 unsigned long flags;
165 int ret;
166
167 spin_lock_irqsave(&nh->lock, flags);
168 ret = notifier_chain_unregister(&nh->head, n);
169 spin_unlock_irqrestore(&nh->lock, flags);
170 synchronize_rcu();
171 return ret;
172 }
173 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
174
175 /**
176 * atomic_notifier_call_chain - Call functions in an atomic notifier chain
177 * @nh: Pointer to head of the atomic notifier chain
178 * @val: Value passed unmodified to notifier function
179 * @v: Pointer passed unmodified to notifier function
180 *
181 * Calls each function in a notifier chain in turn. The functions
182 * run in an atomic context, so they must not block.
183 * This routine uses RCU to synchronize with changes to the chain.
184 *
185 * If the return value of the notifier can be and'ed
186 * with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
187 * will return immediately, with the return value of
188 * the notifier function which halted execution.
189 * Otherwise the return value is the return value
190 * of the last notifier function called.
191 */
atomic_notifier_call_chain(struct atomic_notifier_head * nh,unsigned long val,void * v)192 int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
193 unsigned long val, void *v)
194 {
195 int ret;
196
197 rcu_read_lock();
198 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
199 rcu_read_unlock();
200
201 return ret;
202 }
203 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
204 NOKPROBE_SYMBOL(atomic_notifier_call_chain);
205
206 /*
207 * Blocking notifier chain routines. All access to the chain is
208 * synchronized by an rwsem.
209 */
210
211 /**
212 * blocking_notifier_chain_register - Add notifier to a blocking notifier chain
213 * @nh: Pointer to head of the blocking notifier chain
214 * @n: New entry in notifier chain
215 *
216 * Adds a notifier to a blocking notifier chain.
217 * Must be called in process context.
218 *
219 * Currently always returns zero.
220 */
blocking_notifier_chain_register(struct blocking_notifier_head * nh,struct notifier_block * n)221 int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
222 struct notifier_block *n)
223 {
224 int ret;
225
226 /*
227 * This code gets used during boot-up, when task switching is
228 * not yet working and interrupts must remain disabled. At
229 * such times we must not call down_write().
230 */
231 if (unlikely(system_state == SYSTEM_BOOTING))
232 return notifier_chain_register(&nh->head, n);
233
234 down_write(&nh->rwsem);
235 ret = notifier_chain_register(&nh->head, n);
236 up_write(&nh->rwsem);
237 return ret;
238 }
239 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
240
241 /**
242 * blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
243 * @nh: Pointer to head of the blocking notifier chain
244 * @n: Entry to remove from notifier chain
245 *
246 * Removes a notifier from a blocking notifier chain.
247 * Must be called from process context.
248 *
249 * Returns zero on success or %-ENOENT on failure.
250 */
blocking_notifier_chain_unregister(struct blocking_notifier_head * nh,struct notifier_block * n)251 int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
252 struct notifier_block *n)
253 {
254 int ret;
255
256 /*
257 * This code gets used during boot-up, when task switching is
258 * not yet working and interrupts must remain disabled. At
259 * such times we must not call down_write().
260 */
261 if (unlikely(system_state == SYSTEM_BOOTING))
262 return notifier_chain_unregister(&nh->head, n);
263
264 down_write(&nh->rwsem);
265 ret = notifier_chain_unregister(&nh->head, n);
266 up_write(&nh->rwsem);
267 return ret;
268 }
269 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
270
blocking_notifier_call_chain_robust(struct blocking_notifier_head * nh,unsigned long val_up,unsigned long val_down,void * v)271 int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
272 unsigned long val_up, unsigned long val_down, void *v)
273 {
274 int ret = NOTIFY_DONE;
275
276 /*
277 * We check the head outside the lock, but if this access is
278 * racy then it does not matter what the result of the test
279 * is, we re-check the list after having taken the lock anyway:
280 */
281 if (rcu_access_pointer(nh->head)) {
282 down_read(&nh->rwsem);
283 ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
284 up_read(&nh->rwsem);
285 }
286 return ret;
287 }
288 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
289
290 /**
291 * blocking_notifier_call_chain - Call functions in a blocking notifier chain
292 * @nh: Pointer to head of the blocking notifier chain
293 * @val: Value passed unmodified to notifier function
294 * @v: Pointer passed unmodified to notifier function
295 *
296 * Calls each function in a notifier chain in turn. The functions
297 * run in a process context, so they are allowed to block.
298 *
299 * If the return value of the notifier can be and'ed
300 * with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
301 * will return immediately, with the return value of
302 * the notifier function which halted execution.
303 * Otherwise the return value is the return value
304 * of the last notifier function called.
305 */
blocking_notifier_call_chain(struct blocking_notifier_head * nh,unsigned long val,void * v)306 int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
307 unsigned long val, void *v)
308 {
309 int ret = NOTIFY_DONE;
310
311 /*
312 * We check the head outside the lock, but if this access is
313 * racy then it does not matter what the result of the test
314 * is, we re-check the list after having taken the lock anyway:
315 */
316 if (rcu_access_pointer(nh->head)) {
317 down_read(&nh->rwsem);
318 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
319 up_read(&nh->rwsem);
320 }
321 return ret;
322 }
323 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
324
325 /*
326 * Raw notifier chain routines. There is no protection;
327 * the caller must provide it. Use at your own risk!
328 */
329
330 /**
331 * raw_notifier_chain_register - Add notifier to a raw notifier chain
332 * @nh: Pointer to head of the raw notifier chain
333 * @n: New entry in notifier chain
334 *
335 * Adds a notifier to a raw notifier chain.
336 * All locking must be provided by the caller.
337 *
338 * Currently always returns zero.
339 */
raw_notifier_chain_register(struct raw_notifier_head * nh,struct notifier_block * n)340 int raw_notifier_chain_register(struct raw_notifier_head *nh,
341 struct notifier_block *n)
342 {
343 return notifier_chain_register(&nh->head, n);
344 }
345 EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
346
347 /**
348 * raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
349 * @nh: Pointer to head of the raw notifier chain
350 * @n: Entry to remove from notifier chain
351 *
352 * Removes a notifier from a raw notifier chain.
353 * All locking must be provided by the caller.
354 *
355 * Returns zero on success or %-ENOENT on failure.
356 */
raw_notifier_chain_unregister(struct raw_notifier_head * nh,struct notifier_block * n)357 int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
358 struct notifier_block *n)
359 {
360 return notifier_chain_unregister(&nh->head, n);
361 }
362 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
363
raw_notifier_call_chain_robust(struct raw_notifier_head * nh,unsigned long val_up,unsigned long val_down,void * v)364 int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
365 unsigned long val_up, unsigned long val_down, void *v)
366 {
367 return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
368 }
369 EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
370
371 /**
372 * raw_notifier_call_chain - Call functions in a raw notifier chain
373 * @nh: Pointer to head of the raw notifier chain
374 * @val: Value passed unmodified to notifier function
375 * @v: Pointer passed unmodified to notifier function
376 *
377 * Calls each function in a notifier chain in turn. The functions
378 * run in an undefined context.
379 * All locking must be provided by the caller.
380 *
381 * If the return value of the notifier can be and'ed
382 * with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
383 * will return immediately, with the return value of
384 * the notifier function which halted execution.
385 * Otherwise the return value is the return value
386 * of the last notifier function called.
387 */
raw_notifier_call_chain(struct raw_notifier_head * nh,unsigned long val,void * v)388 int raw_notifier_call_chain(struct raw_notifier_head *nh,
389 unsigned long val, void *v)
390 {
391 return notifier_call_chain(&nh->head, val, v, -1, NULL);
392 }
393 EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
394
395 #ifdef CONFIG_SRCU
396 /*
397 * SRCU notifier chain routines. Registration and unregistration
398 * use a mutex, and call_chain is synchronized by SRCU (no locks).
399 */
400
401 /**
402 * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
403 * @nh: Pointer to head of the SRCU notifier chain
404 * @n: New entry in notifier chain
405 *
406 * Adds a notifier to an SRCU notifier chain.
407 * Must be called in process context.
408 *
409 * Currently always returns zero.
410 */
srcu_notifier_chain_register(struct srcu_notifier_head * nh,struct notifier_block * n)411 int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
412 struct notifier_block *n)
413 {
414 int ret;
415
416 /*
417 * This code gets used during boot-up, when task switching is
418 * not yet working and interrupts must remain disabled. At
419 * such times we must not call mutex_lock().
420 */
421 if (unlikely(system_state == SYSTEM_BOOTING))
422 return notifier_chain_register(&nh->head, n);
423
424 mutex_lock(&nh->mutex);
425 ret = notifier_chain_register(&nh->head, n);
426 mutex_unlock(&nh->mutex);
427 return ret;
428 }
429 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
430
431 /**
432 * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
433 * @nh: Pointer to head of the SRCU notifier chain
434 * @n: Entry to remove from notifier chain
435 *
436 * Removes a notifier from an SRCU notifier chain.
437 * Must be called from process context.
438 *
439 * Returns zero on success or %-ENOENT on failure.
440 */
srcu_notifier_chain_unregister(struct srcu_notifier_head * nh,struct notifier_block * n)441 int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
442 struct notifier_block *n)
443 {
444 int ret;
445
446 /*
447 * This code gets used during boot-up, when task switching is
448 * not yet working and interrupts must remain disabled. At
449 * such times we must not call mutex_lock().
450 */
451 if (unlikely(system_state == SYSTEM_BOOTING))
452 return notifier_chain_unregister(&nh->head, n);
453
454 mutex_lock(&nh->mutex);
455 ret = notifier_chain_unregister(&nh->head, n);
456 mutex_unlock(&nh->mutex);
457 synchronize_srcu(&nh->srcu);
458 return ret;
459 }
460 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
461
462 /**
463 * srcu_notifier_call_chain - Call functions in an SRCU notifier chain
464 * @nh: Pointer to head of the SRCU notifier chain
465 * @val: Value passed unmodified to notifier function
466 * @v: Pointer passed unmodified to notifier function
467 *
468 * Calls each function in a notifier chain in turn. The functions
469 * run in a process context, so they are allowed to block.
470 *
471 * If the return value of the notifier can be and'ed
472 * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
473 * will return immediately, with the return value of
474 * the notifier function which halted execution.
475 * Otherwise the return value is the return value
476 * of the last notifier function called.
477 */
srcu_notifier_call_chain(struct srcu_notifier_head * nh,unsigned long val,void * v)478 int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
479 unsigned long val, void *v)
480 {
481 int ret;
482 int idx;
483
484 idx = srcu_read_lock(&nh->srcu);
485 ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
486 srcu_read_unlock(&nh->srcu, idx);
487 return ret;
488 }
489 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
490
491 /**
492 * srcu_init_notifier_head - Initialize an SRCU notifier head
493 * @nh: Pointer to head of the srcu notifier chain
494 *
495 * Unlike other sorts of notifier heads, SRCU notifier heads require
496 * dynamic initialization. Be sure to call this routine before
497 * calling any of the other SRCU notifier routines for this head.
498 *
499 * If an SRCU notifier head is deallocated, it must first be cleaned
500 * up by calling srcu_cleanup_notifier_head(). Otherwise the head's
501 * per-cpu data (used by the SRCU mechanism) will leak.
502 */
srcu_init_notifier_head(struct srcu_notifier_head * nh)503 void srcu_init_notifier_head(struct srcu_notifier_head *nh)
504 {
505 mutex_init(&nh->mutex);
506 if (init_srcu_struct(&nh->srcu) < 0)
507 BUG();
508 nh->head = NULL;
509 }
510 EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
511
512 #endif /* CONFIG_SRCU */
513
514 static ATOMIC_NOTIFIER_HEAD(die_chain);
515
notify_die(enum die_val val,const char * str,struct pt_regs * regs,long err,int trap,int sig)516 int notrace notify_die(enum die_val val, const char *str,
517 struct pt_regs *regs, long err, int trap, int sig)
518 {
519 struct die_args args = {
520 .regs = regs,
521 .str = str,
522 .err = err,
523 .trapnr = trap,
524 .signr = sig,
525
526 };
527 RCU_LOCKDEP_WARN(!rcu_is_watching(),
528 "notify_die called but RCU thinks we're quiescent");
529 return atomic_notifier_call_chain(&die_chain, val, &args);
530 }
531 NOKPROBE_SYMBOL(notify_die);
532
register_die_notifier(struct notifier_block * nb)533 int register_die_notifier(struct notifier_block *nb)
534 {
535 return atomic_notifier_chain_register(&die_chain, nb);
536 }
537 EXPORT_SYMBOL_GPL(register_die_notifier);
538
unregister_die_notifier(struct notifier_block * nb)539 int unregister_die_notifier(struct notifier_block *nb)
540 {
541 return atomic_notifier_chain_unregister(&die_chain, nb);
542 }
543 EXPORT_SYMBOL_GPL(unregister_die_notifier);
544