1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/kdebug.h>
3 #include <linux/kprobes.h>
4 #include <linux/export.h>
5 #include <linux/notifier.h>
6 #include <linux/rcupdate.h>
7 #include <linux/vmalloc.h>
8 #include <linux/reboot.h>
9 
10 /*
11  *	Notifier list for kernel code which wants to be called
12  *	at shutdown. This is used to stop any idling DMA operations
13  *	and the like.
14  */
15 BLOCKING_NOTIFIER_HEAD(reboot_notifier_list);
16 
17 /*
18  *	Notifier chain core routines.  The exported routines below
19  *	are layered on top of these, with appropriate locking added.
20  */
21 
notifier_chain_register(struct notifier_block ** nl,struct notifier_block * n)22 static int notifier_chain_register(struct notifier_block **nl,
23 		struct notifier_block *n)
24 {
25 	while ((*nl) != NULL) {
26 		if (unlikely((*nl) == n)) {
27 			WARN(1, "double register detected");
28 			return 0;
29 		}
30 		if (n->priority > (*nl)->priority)
31 			break;
32 		nl = &((*nl)->next);
33 	}
34 	n->next = *nl;
35 	rcu_assign_pointer(*nl, n);
36 	return 0;
37 }
38 
notifier_chain_unregister(struct notifier_block ** nl,struct notifier_block * n)39 static int notifier_chain_unregister(struct notifier_block **nl,
40 		struct notifier_block *n)
41 {
42 	while ((*nl) != NULL) {
43 		if ((*nl) == n) {
44 			rcu_assign_pointer(*nl, n->next);
45 			return 0;
46 		}
47 		nl = &((*nl)->next);
48 	}
49 	return -ENOENT;
50 }
51 
52 /**
53  * notifier_call_chain - Informs the registered notifiers about an event.
54  *	@nl:		Pointer to head of the blocking notifier chain
55  *	@val:		Value passed unmodified to notifier function
56  *	@v:		Pointer passed unmodified to notifier function
57  *	@nr_to_call:	Number of notifier functions to be called. Don't care
58  *			value of this parameter is -1.
59  *	@nr_calls:	Records the number of notifications sent. Don't care
60  *			value of this field is NULL.
61  *	@returns:	notifier_call_chain returns the value returned by the
62  *			last notifier function called.
63  */
notifier_call_chain(struct notifier_block ** nl,unsigned long val,void * v,int nr_to_call,int * nr_calls)64 static int notifier_call_chain(struct notifier_block **nl,
65 			       unsigned long val, void *v,
66 			       int nr_to_call, int *nr_calls)
67 {
68 	int ret = NOTIFY_DONE;
69 	struct notifier_block *nb, *next_nb;
70 
71 	nb = rcu_dereference_raw(*nl);
72 
73 	while (nb && nr_to_call) {
74 		next_nb = rcu_dereference_raw(nb->next);
75 
76 #ifdef CONFIG_DEBUG_NOTIFIERS
77 		if (unlikely(!func_ptr_is_kernel_text(nb->notifier_call))) {
78 			WARN(1, "Invalid notifier called!");
79 			nb = next_nb;
80 			continue;
81 		}
82 #endif
83 		ret = nb->notifier_call(nb, val, v);
84 
85 		if (nr_calls)
86 			(*nr_calls)++;
87 
88 		if (ret & NOTIFY_STOP_MASK)
89 			break;
90 		nb = next_nb;
91 		nr_to_call--;
92 	}
93 	return ret;
94 }
95 NOKPROBE_SYMBOL(notifier_call_chain);
96 
97 /**
98  * notifier_call_chain_robust - Inform the registered notifiers about an event
99  *                              and rollback on error.
100  * @nl:		Pointer to head of the blocking notifier chain
101  * @val_up:	Value passed unmodified to the notifier function
102  * @val_down:	Value passed unmodified to the notifier function when recovering
103  *              from an error on @val_up
104  * @v		Pointer passed unmodified to the notifier function
105  *
106  * NOTE:	It is important the @nl chain doesn't change between the two
107  *		invocations of notifier_call_chain() such that we visit the
108  *		exact same notifier callbacks; this rules out any RCU usage.
109  *
110  * Returns:	the return value of the @val_up call.
111  */
notifier_call_chain_robust(struct notifier_block ** nl,unsigned long val_up,unsigned long val_down,void * v)112 static int notifier_call_chain_robust(struct notifier_block **nl,
113 				     unsigned long val_up, unsigned long val_down,
114 				     void *v)
115 {
116 	int ret, nr = 0;
117 
118 	ret = notifier_call_chain(nl, val_up, v, -1, &nr);
119 	if (ret & NOTIFY_STOP_MASK)
120 		notifier_call_chain(nl, val_down, v, nr-1, NULL);
121 
122 	return ret;
123 }
124 
125 /*
126  *	Atomic notifier chain routines.  Registration and unregistration
127  *	use a spinlock, and call_chain is synchronized by RCU (no locks).
128  */
129 
130 /**
131  *	atomic_notifier_chain_register - Add notifier to an atomic notifier chain
132  *	@nh: Pointer to head of the atomic notifier chain
133  *	@n: New entry in notifier chain
134  *
135  *	Adds a notifier to an atomic notifier chain.
136  *
137  *	Currently always returns zero.
138  */
atomic_notifier_chain_register(struct atomic_notifier_head * nh,struct notifier_block * n)139 int atomic_notifier_chain_register(struct atomic_notifier_head *nh,
140 		struct notifier_block *n)
141 {
142 	unsigned long flags;
143 	int ret;
144 
145 	spin_lock_irqsave(&nh->lock, flags);
146 	ret = notifier_chain_register(&nh->head, n);
147 	spin_unlock_irqrestore(&nh->lock, flags);
148 	return ret;
149 }
150 EXPORT_SYMBOL_GPL(atomic_notifier_chain_register);
151 
152 /**
153  *	atomic_notifier_chain_unregister - Remove notifier from an atomic notifier chain
154  *	@nh: Pointer to head of the atomic notifier chain
155  *	@n: Entry to remove from notifier chain
156  *
157  *	Removes a notifier from an atomic notifier chain.
158  *
159  *	Returns zero on success or %-ENOENT on failure.
160  */
atomic_notifier_chain_unregister(struct atomic_notifier_head * nh,struct notifier_block * n)161 int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh,
162 		struct notifier_block *n)
163 {
164 	unsigned long flags;
165 	int ret;
166 
167 	spin_lock_irqsave(&nh->lock, flags);
168 	ret = notifier_chain_unregister(&nh->head, n);
169 	spin_unlock_irqrestore(&nh->lock, flags);
170 	synchronize_rcu();
171 	return ret;
172 }
173 EXPORT_SYMBOL_GPL(atomic_notifier_chain_unregister);
174 
175 /**
176  *	atomic_notifier_call_chain - Call functions in an atomic notifier chain
177  *	@nh: Pointer to head of the atomic notifier chain
178  *	@val: Value passed unmodified to notifier function
179  *	@v: Pointer passed unmodified to notifier function
180  *
181  *	Calls each function in a notifier chain in turn.  The functions
182  *	run in an atomic context, so they must not block.
183  *	This routine uses RCU to synchronize with changes to the chain.
184  *
185  *	If the return value of the notifier can be and'ed
186  *	with %NOTIFY_STOP_MASK then atomic_notifier_call_chain()
187  *	will return immediately, with the return value of
188  *	the notifier function which halted execution.
189  *	Otherwise the return value is the return value
190  *	of the last notifier function called.
191  */
atomic_notifier_call_chain(struct atomic_notifier_head * nh,unsigned long val,void * v)192 int atomic_notifier_call_chain(struct atomic_notifier_head *nh,
193 			       unsigned long val, void *v)
194 {
195 	int ret;
196 
197 	rcu_read_lock();
198 	ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
199 	rcu_read_unlock();
200 
201 	return ret;
202 }
203 EXPORT_SYMBOL_GPL(atomic_notifier_call_chain);
204 NOKPROBE_SYMBOL(atomic_notifier_call_chain);
205 
206 /*
207  *	Blocking notifier chain routines.  All access to the chain is
208  *	synchronized by an rwsem.
209  */
210 
211 /**
212  *	blocking_notifier_chain_register - Add notifier to a blocking notifier chain
213  *	@nh: Pointer to head of the blocking notifier chain
214  *	@n: New entry in notifier chain
215  *
216  *	Adds a notifier to a blocking notifier chain.
217  *	Must be called in process context.
218  *
219  *	Currently always returns zero.
220  */
blocking_notifier_chain_register(struct blocking_notifier_head * nh,struct notifier_block * n)221 int blocking_notifier_chain_register(struct blocking_notifier_head *nh,
222 		struct notifier_block *n)
223 {
224 	int ret;
225 
226 	/*
227 	 * This code gets used during boot-up, when task switching is
228 	 * not yet working and interrupts must remain disabled.  At
229 	 * such times we must not call down_write().
230 	 */
231 	if (unlikely(system_state == SYSTEM_BOOTING))
232 		return notifier_chain_register(&nh->head, n);
233 
234 	down_write(&nh->rwsem);
235 	ret = notifier_chain_register(&nh->head, n);
236 	up_write(&nh->rwsem);
237 	return ret;
238 }
239 EXPORT_SYMBOL_GPL(blocking_notifier_chain_register);
240 
241 /**
242  *	blocking_notifier_chain_unregister - Remove notifier from a blocking notifier chain
243  *	@nh: Pointer to head of the blocking notifier chain
244  *	@n: Entry to remove from notifier chain
245  *
246  *	Removes a notifier from a blocking notifier chain.
247  *	Must be called from process context.
248  *
249  *	Returns zero on success or %-ENOENT on failure.
250  */
blocking_notifier_chain_unregister(struct blocking_notifier_head * nh,struct notifier_block * n)251 int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh,
252 		struct notifier_block *n)
253 {
254 	int ret;
255 
256 	/*
257 	 * This code gets used during boot-up, when task switching is
258 	 * not yet working and interrupts must remain disabled.  At
259 	 * such times we must not call down_write().
260 	 */
261 	if (unlikely(system_state == SYSTEM_BOOTING))
262 		return notifier_chain_unregister(&nh->head, n);
263 
264 	down_write(&nh->rwsem);
265 	ret = notifier_chain_unregister(&nh->head, n);
266 	up_write(&nh->rwsem);
267 	return ret;
268 }
269 EXPORT_SYMBOL_GPL(blocking_notifier_chain_unregister);
270 
blocking_notifier_call_chain_robust(struct blocking_notifier_head * nh,unsigned long val_up,unsigned long val_down,void * v)271 int blocking_notifier_call_chain_robust(struct blocking_notifier_head *nh,
272 		unsigned long val_up, unsigned long val_down, void *v)
273 {
274 	int ret = NOTIFY_DONE;
275 
276 	/*
277 	 * We check the head outside the lock, but if this access is
278 	 * racy then it does not matter what the result of the test
279 	 * is, we re-check the list after having taken the lock anyway:
280 	 */
281 	if (rcu_access_pointer(nh->head)) {
282 		down_read(&nh->rwsem);
283 		ret = notifier_call_chain_robust(&nh->head, val_up, val_down, v);
284 		up_read(&nh->rwsem);
285 	}
286 	return ret;
287 }
288 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain_robust);
289 
290 /**
291  *	blocking_notifier_call_chain - Call functions in a blocking notifier chain
292  *	@nh: Pointer to head of the blocking notifier chain
293  *	@val: Value passed unmodified to notifier function
294  *	@v: Pointer passed unmodified to notifier function
295  *
296  *	Calls each function in a notifier chain in turn.  The functions
297  *	run in a process context, so they are allowed to block.
298  *
299  *	If the return value of the notifier can be and'ed
300  *	with %NOTIFY_STOP_MASK then blocking_notifier_call_chain()
301  *	will return immediately, with the return value of
302  *	the notifier function which halted execution.
303  *	Otherwise the return value is the return value
304  *	of the last notifier function called.
305  */
blocking_notifier_call_chain(struct blocking_notifier_head * nh,unsigned long val,void * v)306 int blocking_notifier_call_chain(struct blocking_notifier_head *nh,
307 		unsigned long val, void *v)
308 {
309 	int ret = NOTIFY_DONE;
310 
311 	/*
312 	 * We check the head outside the lock, but if this access is
313 	 * racy then it does not matter what the result of the test
314 	 * is, we re-check the list after having taken the lock anyway:
315 	 */
316 	if (rcu_access_pointer(nh->head)) {
317 		down_read(&nh->rwsem);
318 		ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
319 		up_read(&nh->rwsem);
320 	}
321 	return ret;
322 }
323 EXPORT_SYMBOL_GPL(blocking_notifier_call_chain);
324 
325 /*
326  *	Raw notifier chain routines.  There is no protection;
327  *	the caller must provide it.  Use at your own risk!
328  */
329 
330 /**
331  *	raw_notifier_chain_register - Add notifier to a raw notifier chain
332  *	@nh: Pointer to head of the raw notifier chain
333  *	@n: New entry in notifier chain
334  *
335  *	Adds a notifier to a raw notifier chain.
336  *	All locking must be provided by the caller.
337  *
338  *	Currently always returns zero.
339  */
raw_notifier_chain_register(struct raw_notifier_head * nh,struct notifier_block * n)340 int raw_notifier_chain_register(struct raw_notifier_head *nh,
341 		struct notifier_block *n)
342 {
343 	return notifier_chain_register(&nh->head, n);
344 }
345 EXPORT_SYMBOL_GPL(raw_notifier_chain_register);
346 
347 /**
348  *	raw_notifier_chain_unregister - Remove notifier from a raw notifier chain
349  *	@nh: Pointer to head of the raw notifier chain
350  *	@n: Entry to remove from notifier chain
351  *
352  *	Removes a notifier from a raw notifier chain.
353  *	All locking must be provided by the caller.
354  *
355  *	Returns zero on success or %-ENOENT on failure.
356  */
raw_notifier_chain_unregister(struct raw_notifier_head * nh,struct notifier_block * n)357 int raw_notifier_chain_unregister(struct raw_notifier_head *nh,
358 		struct notifier_block *n)
359 {
360 	return notifier_chain_unregister(&nh->head, n);
361 }
362 EXPORT_SYMBOL_GPL(raw_notifier_chain_unregister);
363 
raw_notifier_call_chain_robust(struct raw_notifier_head * nh,unsigned long val_up,unsigned long val_down,void * v)364 int raw_notifier_call_chain_robust(struct raw_notifier_head *nh,
365 		unsigned long val_up, unsigned long val_down, void *v)
366 {
367 	return notifier_call_chain_robust(&nh->head, val_up, val_down, v);
368 }
369 EXPORT_SYMBOL_GPL(raw_notifier_call_chain_robust);
370 
371 /**
372  *	raw_notifier_call_chain - Call functions in a raw notifier chain
373  *	@nh: Pointer to head of the raw notifier chain
374  *	@val: Value passed unmodified to notifier function
375  *	@v: Pointer passed unmodified to notifier function
376  *
377  *	Calls each function in a notifier chain in turn.  The functions
378  *	run in an undefined context.
379  *	All locking must be provided by the caller.
380  *
381  *	If the return value of the notifier can be and'ed
382  *	with %NOTIFY_STOP_MASK then raw_notifier_call_chain()
383  *	will return immediately, with the return value of
384  *	the notifier function which halted execution.
385  *	Otherwise the return value is the return value
386  *	of the last notifier function called.
387  */
raw_notifier_call_chain(struct raw_notifier_head * nh,unsigned long val,void * v)388 int raw_notifier_call_chain(struct raw_notifier_head *nh,
389 		unsigned long val, void *v)
390 {
391 	return notifier_call_chain(&nh->head, val, v, -1, NULL);
392 }
393 EXPORT_SYMBOL_GPL(raw_notifier_call_chain);
394 
395 #ifdef CONFIG_SRCU
396 /*
397  *	SRCU notifier chain routines.    Registration and unregistration
398  *	use a mutex, and call_chain is synchronized by SRCU (no locks).
399  */
400 
401 /**
402  *	srcu_notifier_chain_register - Add notifier to an SRCU notifier chain
403  *	@nh: Pointer to head of the SRCU notifier chain
404  *	@n: New entry in notifier chain
405  *
406  *	Adds a notifier to an SRCU notifier chain.
407  *	Must be called in process context.
408  *
409  *	Currently always returns zero.
410  */
srcu_notifier_chain_register(struct srcu_notifier_head * nh,struct notifier_block * n)411 int srcu_notifier_chain_register(struct srcu_notifier_head *nh,
412 		struct notifier_block *n)
413 {
414 	int ret;
415 
416 	/*
417 	 * This code gets used during boot-up, when task switching is
418 	 * not yet working and interrupts must remain disabled.  At
419 	 * such times we must not call mutex_lock().
420 	 */
421 	if (unlikely(system_state == SYSTEM_BOOTING))
422 		return notifier_chain_register(&nh->head, n);
423 
424 	mutex_lock(&nh->mutex);
425 	ret = notifier_chain_register(&nh->head, n);
426 	mutex_unlock(&nh->mutex);
427 	return ret;
428 }
429 EXPORT_SYMBOL_GPL(srcu_notifier_chain_register);
430 
431 /**
432  *	srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain
433  *	@nh: Pointer to head of the SRCU notifier chain
434  *	@n: Entry to remove from notifier chain
435  *
436  *	Removes a notifier from an SRCU notifier chain.
437  *	Must be called from process context.
438  *
439  *	Returns zero on success or %-ENOENT on failure.
440  */
srcu_notifier_chain_unregister(struct srcu_notifier_head * nh,struct notifier_block * n)441 int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh,
442 		struct notifier_block *n)
443 {
444 	int ret;
445 
446 	/*
447 	 * This code gets used during boot-up, when task switching is
448 	 * not yet working and interrupts must remain disabled.  At
449 	 * such times we must not call mutex_lock().
450 	 */
451 	if (unlikely(system_state == SYSTEM_BOOTING))
452 		return notifier_chain_unregister(&nh->head, n);
453 
454 	mutex_lock(&nh->mutex);
455 	ret = notifier_chain_unregister(&nh->head, n);
456 	mutex_unlock(&nh->mutex);
457 	synchronize_srcu(&nh->srcu);
458 	return ret;
459 }
460 EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister);
461 
462 /**
463  *	srcu_notifier_call_chain - Call functions in an SRCU notifier chain
464  *	@nh: Pointer to head of the SRCU notifier chain
465  *	@val: Value passed unmodified to notifier function
466  *	@v: Pointer passed unmodified to notifier function
467  *
468  *	Calls each function in a notifier chain in turn.  The functions
469  *	run in a process context, so they are allowed to block.
470  *
471  *	If the return value of the notifier can be and'ed
472  *	with %NOTIFY_STOP_MASK then srcu_notifier_call_chain()
473  *	will return immediately, with the return value of
474  *	the notifier function which halted execution.
475  *	Otherwise the return value is the return value
476  *	of the last notifier function called.
477  */
srcu_notifier_call_chain(struct srcu_notifier_head * nh,unsigned long val,void * v)478 int srcu_notifier_call_chain(struct srcu_notifier_head *nh,
479 		unsigned long val, void *v)
480 {
481 	int ret;
482 	int idx;
483 
484 	idx = srcu_read_lock(&nh->srcu);
485 	ret = notifier_call_chain(&nh->head, val, v, -1, NULL);
486 	srcu_read_unlock(&nh->srcu, idx);
487 	return ret;
488 }
489 EXPORT_SYMBOL_GPL(srcu_notifier_call_chain);
490 
491 /**
492  *	srcu_init_notifier_head - Initialize an SRCU notifier head
493  *	@nh: Pointer to head of the srcu notifier chain
494  *
495  *	Unlike other sorts of notifier heads, SRCU notifier heads require
496  *	dynamic initialization.  Be sure to call this routine before
497  *	calling any of the other SRCU notifier routines for this head.
498  *
499  *	If an SRCU notifier head is deallocated, it must first be cleaned
500  *	up by calling srcu_cleanup_notifier_head().  Otherwise the head's
501  *	per-cpu data (used by the SRCU mechanism) will leak.
502  */
srcu_init_notifier_head(struct srcu_notifier_head * nh)503 void srcu_init_notifier_head(struct srcu_notifier_head *nh)
504 {
505 	mutex_init(&nh->mutex);
506 	if (init_srcu_struct(&nh->srcu) < 0)
507 		BUG();
508 	nh->head = NULL;
509 }
510 EXPORT_SYMBOL_GPL(srcu_init_notifier_head);
511 
512 #endif /* CONFIG_SRCU */
513 
514 static ATOMIC_NOTIFIER_HEAD(die_chain);
515 
notify_die(enum die_val val,const char * str,struct pt_regs * regs,long err,int trap,int sig)516 int notrace notify_die(enum die_val val, const char *str,
517 	       struct pt_regs *regs, long err, int trap, int sig)
518 {
519 	struct die_args args = {
520 		.regs	= regs,
521 		.str	= str,
522 		.err	= err,
523 		.trapnr	= trap,
524 		.signr	= sig,
525 
526 	};
527 	RCU_LOCKDEP_WARN(!rcu_is_watching(),
528 			   "notify_die called but RCU thinks we're quiescent");
529 	return atomic_notifier_call_chain(&die_chain, val, &args);
530 }
531 NOKPROBE_SYMBOL(notify_die);
532 
register_die_notifier(struct notifier_block * nb)533 int register_die_notifier(struct notifier_block *nb)
534 {
535 	return atomic_notifier_chain_register(&die_chain, nb);
536 }
537 EXPORT_SYMBOL_GPL(register_die_notifier);
538 
unregister_die_notifier(struct notifier_block * nb)539 int unregister_die_notifier(struct notifier_block *nb)
540 {
541 	return atomic_notifier_chain_unregister(&die_chain, nb);
542 }
543 EXPORT_SYMBOL_GPL(unregister_die_notifier);
544