1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * arch/arm64/kernel/entry-ftrace.S
4 *
5 * Copyright (C) 2013 Linaro Limited
6 * Author: AKASHI Takahiro <takahiro.akashi@linaro.org>
7 */
8
9#include <linux/linkage.h>
10#include <asm/asm-offsets.h>
11#include <asm/assembler.h>
12#include <asm/ftrace.h>
13#include <asm/insn.h>
14
15#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
16/*
17 * Due to -fpatchable-function-entry=2, the compiler has placed two NOPs before
18 * the regular function prologue. For an enabled callsite, ftrace_init_nop() and
19 * ftrace_make_call() have patched those NOPs to:
20 *
21 * 	MOV	X9, LR
22 * 	BL	<entry>
23 *
24 * ... where <entry> is either ftrace_caller or ftrace_regs_caller.
25 *
26 * Each instrumented function follows the AAPCS, so here x0-x8 and x18-x30 are
27 * live (x18 holds the Shadow Call Stack pointer), and x9-x17 are safe to
28 * clobber.
29 *
30 * We save the callsite's context into a pt_regs before invoking any ftrace
31 * callbacks. So that we can get a sensible backtrace, we create a stack record
32 * for the callsite and the ftrace entry assembly. This is not sufficient for
33 * reliable stacktrace: until we create the callsite stack record, its caller
34 * is missing from the LR and existing chain of frame records.
35 */
36	.macro  ftrace_regs_entry, allregs=0
37	/* Make room for pt_regs, plus a callee frame */
38	sub	sp, sp, #(PT_REGS_SIZE + 16)
39
40	/* Save function arguments (and x9 for simplicity) */
41	stp	x0, x1, [sp, #S_X0]
42	stp	x2, x3, [sp, #S_X2]
43	stp	x4, x5, [sp, #S_X4]
44	stp	x6, x7, [sp, #S_X6]
45	stp	x8, x9, [sp, #S_X8]
46
47	/* Optionally save the callee-saved registers, always save the FP */
48	.if \allregs == 1
49	stp	x10, x11, [sp, #S_X10]
50	stp	x12, x13, [sp, #S_X12]
51	stp	x14, x15, [sp, #S_X14]
52	stp	x16, x17, [sp, #S_X16]
53	stp	x18, x19, [sp, #S_X18]
54	stp	x20, x21, [sp, #S_X20]
55	stp	x22, x23, [sp, #S_X22]
56	stp	x24, x25, [sp, #S_X24]
57	stp	x26, x27, [sp, #S_X26]
58	stp	x28, x29, [sp, #S_X28]
59	.else
60	str	x29, [sp, #S_FP]
61	.endif
62
63	/* Save the callsite's SP and LR */
64	add	x10, sp, #(PT_REGS_SIZE + 16)
65	stp	x9, x10, [sp, #S_LR]
66
67	/* Save the PC after the ftrace callsite */
68	str	x30, [sp, #S_PC]
69
70	/* Create a frame record for the callsite above pt_regs */
71	stp	x29, x9, [sp, #PT_REGS_SIZE]
72	add	x29, sp, #PT_REGS_SIZE
73
74	/* Create our frame record within pt_regs. */
75	stp	x29, x30, [sp, #S_STACKFRAME]
76	add	x29, sp, #S_STACKFRAME
77	.endm
78
79SYM_CODE_START(ftrace_regs_caller)
80#ifdef BTI_C
81	BTI_C
82#endif
83	ftrace_regs_entry	1
84	b	ftrace_common
85SYM_CODE_END(ftrace_regs_caller)
86
87SYM_CODE_START(ftrace_caller)
88#ifdef BTI_C
89	BTI_C
90#endif
91	ftrace_regs_entry	0
92	b	ftrace_common
93SYM_CODE_END(ftrace_caller)
94
95SYM_CODE_START(ftrace_common)
96	sub	x0, x30, #AARCH64_INSN_SIZE	// ip (callsite's BL insn)
97	mov	x1, x9				// parent_ip (callsite's LR)
98	ldr_l	x2, function_trace_op		// op
99	mov	x3, sp				// regs
100
101SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
102	bl	ftrace_stub
103
104#ifdef CONFIG_FUNCTION_GRAPH_TRACER
105SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
106	nop				// If enabled, this will be replaced
107					// "b ftrace_graph_caller"
108#endif
109
110/*
111 * At the callsite x0-x8 and x19-x30 were live. Any C code will have preserved
112 * x19-x29 per the AAPCS, and we created frame records upon entry, so we need
113 * to restore x0-x8, x29, and x30.
114 */
115ftrace_common_return:
116	/* Restore function arguments */
117	ldp	x0, x1, [sp]
118	ldp	x2, x3, [sp, #S_X2]
119	ldp	x4, x5, [sp, #S_X4]
120	ldp	x6, x7, [sp, #S_X6]
121	ldr	x8, [sp, #S_X8]
122
123	/* Restore the callsite's FP, LR, PC */
124	ldr	x29, [sp, #S_FP]
125	ldr	x30, [sp, #S_LR]
126	ldr	x9, [sp, #S_PC]
127
128	/* Restore the callsite's SP */
129	add	sp, sp, #PT_REGS_SIZE + 16
130
131	ret	x9
132SYM_CODE_END(ftrace_common)
133
134#ifdef CONFIG_FUNCTION_GRAPH_TRACER
135SYM_CODE_START(ftrace_graph_caller)
136	ldr	x0, [sp, #S_PC]
137	sub	x0, x0, #AARCH64_INSN_SIZE	// ip (callsite's BL insn)
138	add	x1, sp, #S_LR			// parent_ip (callsite's LR)
139	ldr	x2, [sp, #PT_REGS_SIZE]	   	// parent fp (callsite's FP)
140	bl	prepare_ftrace_return
141	b	ftrace_common_return
142SYM_CODE_END(ftrace_graph_caller)
143#endif
144
145#else /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
146
147/*
148 * Gcc with -pg will put the following code in the beginning of each function:
149 *      mov x0, x30
150 *      bl _mcount
151 *	[function's body ...]
152 * "bl _mcount" may be replaced to "bl ftrace_caller" or NOP if dynamic
153 * ftrace is enabled.
154 *
155 * Please note that x0 as an argument will not be used here because we can
156 * get lr(x30) of instrumented function at any time by winding up call stack
157 * as long as the kernel is compiled without -fomit-frame-pointer.
158 * (or CONFIG_FRAME_POINTER, this is forced on arm64)
159 *
160 * stack layout after mcount_enter in _mcount():
161 *
162 * current sp/fp =>  0:+-----+
163 * in _mcount()        | x29 | -> instrumented function's fp
164 *                     +-----+
165 *                     | x30 | -> _mcount()'s lr (= instrumented function's pc)
166 * old sp       => +16:+-----+
167 * when instrumented   |     |
168 * function calls      | ... |
169 * _mcount()           |     |
170 *                     |     |
171 * instrumented => +xx:+-----+
172 * function's fp       | x29 | -> parent's fp
173 *                     +-----+
174 *                     | x30 | -> instrumented function's lr (= parent's pc)
175 *                     +-----+
176 *                     | ... |
177 */
178
179	.macro mcount_enter
180	stp	x29, x30, [sp, #-16]!
181	mov	x29, sp
182	.endm
183
184	.macro mcount_exit
185	ldp	x29, x30, [sp], #16
186	ret
187	.endm
188
189	.macro mcount_adjust_addr rd, rn
190	sub	\rd, \rn, #AARCH64_INSN_SIZE
191	.endm
192
193	/* for instrumented function's parent */
194	.macro mcount_get_parent_fp reg
195	ldr	\reg, [x29]
196	ldr	\reg, [\reg]
197	.endm
198
199	/* for instrumented function */
200	.macro mcount_get_pc0 reg
201	mcount_adjust_addr	\reg, x30
202	.endm
203
204	.macro mcount_get_pc reg
205	ldr	\reg, [x29, #8]
206	mcount_adjust_addr	\reg, \reg
207	.endm
208
209	.macro mcount_get_lr reg
210	ldr	\reg, [x29]
211	ldr	\reg, [\reg, #8]
212	.endm
213
214	.macro mcount_get_lr_addr reg
215	ldr	\reg, [x29]
216	add	\reg, \reg, #8
217	.endm
218
219#ifndef CONFIG_DYNAMIC_FTRACE
220/*
221 * void _mcount(unsigned long return_address)
222 * @return_address: return address to instrumented function
223 *
224 * This function makes calls, if enabled, to:
225 *     - tracer function to probe instrumented function's entry,
226 *     - ftrace_graph_caller to set up an exit hook
227 */
228SYM_FUNC_START(_mcount)
229	mcount_enter
230
231	ldr_l	x2, ftrace_trace_function
232	adr	x0, ftrace_stub
233	cmp	x0, x2			// if (ftrace_trace_function
234	b.eq	skip_ftrace_call	//     != ftrace_stub) {
235
236	mcount_get_pc	x0		//       function's pc
237	mcount_get_lr	x1		//       function's lr (= parent's pc)
238	blr	x2			//   (*ftrace_trace_function)(pc, lr);
239
240skip_ftrace_call:			// }
241#ifdef CONFIG_FUNCTION_GRAPH_TRACER
242	ldr_l	x2, ftrace_graph_return
243	cmp	x0, x2			//   if ((ftrace_graph_return
244	b.ne	ftrace_graph_caller	//        != ftrace_stub)
245
246	ldr_l	x2, ftrace_graph_entry	//     || (ftrace_graph_entry
247	adr_l	x0, ftrace_graph_entry_stub //     != ftrace_graph_entry_stub))
248	cmp	x0, x2
249	b.ne	ftrace_graph_caller	//     ftrace_graph_caller();
250#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
251	mcount_exit
252SYM_FUNC_END(_mcount)
253EXPORT_SYMBOL(_mcount)
254NOKPROBE(_mcount)
255
256#else /* CONFIG_DYNAMIC_FTRACE */
257/*
258 * _mcount() is used to build the kernel with -pg option, but all the branch
259 * instructions to _mcount() are replaced to NOP initially at kernel start up,
260 * and later on, NOP to branch to ftrace_caller() when enabled or branch to
261 * NOP when disabled per-function base.
262 */
263SYM_FUNC_START(_mcount)
264	ret
265SYM_FUNC_END(_mcount)
266EXPORT_SYMBOL(_mcount)
267NOKPROBE(_mcount)
268
269/*
270 * void ftrace_caller(unsigned long return_address)
271 * @return_address: return address to instrumented function
272 *
273 * This function is a counterpart of _mcount() in 'static' ftrace, and
274 * makes calls to:
275 *     - tracer function to probe instrumented function's entry,
276 *     - ftrace_graph_caller to set up an exit hook
277 */
278SYM_FUNC_START(ftrace_caller)
279	mcount_enter
280
281	mcount_get_pc0	x0		//     function's pc
282	mcount_get_lr	x1		//     function's lr
283
284SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)	// tracer(pc, lr);
285	nop				// This will be replaced with "bl xxx"
286					// where xxx can be any kind of tracer.
287
288#ifdef CONFIG_FUNCTION_GRAPH_TRACER
289SYM_INNER_LABEL(ftrace_graph_call, SYM_L_GLOBAL) // ftrace_graph_caller();
290	nop				// If enabled, this will be replaced
291					// "b ftrace_graph_caller"
292#endif
293
294	mcount_exit
295SYM_FUNC_END(ftrace_caller)
296#endif /* CONFIG_DYNAMIC_FTRACE */
297
298#ifdef CONFIG_FUNCTION_GRAPH_TRACER
299/*
300 * void ftrace_graph_caller(void)
301 *
302 * Called from _mcount() or ftrace_caller() when function_graph tracer is
303 * selected.
304 * This function w/ prepare_ftrace_return() fakes link register's value on
305 * the call stack in order to intercept instrumented function's return path
306 * and run return_to_handler() later on its exit.
307 */
308SYM_FUNC_START(ftrace_graph_caller)
309	mcount_get_pc		  x0	//     function's pc
310	mcount_get_lr_addr	  x1	//     pointer to function's saved lr
311	mcount_get_parent_fp	  x2	//     parent's fp
312	bl	prepare_ftrace_return	// prepare_ftrace_return(pc, &lr, fp)
313
314	mcount_exit
315SYM_FUNC_END(ftrace_graph_caller)
316#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
317#endif /* CONFIG_DYNAMIC_FTRACE_WITH_REGS */
318
319SYM_FUNC_START(ftrace_stub)
320	ret
321SYM_FUNC_END(ftrace_stub)
322
323#ifdef CONFIG_FUNCTION_GRAPH_TRACER
324/*
325 * void return_to_handler(void)
326 *
327 * Run ftrace_return_to_handler() before going back to parent.
328 * @fp is checked against the value passed by ftrace_graph_caller().
329 */
330SYM_CODE_START(return_to_handler)
331	/* save return value regs */
332	sub sp, sp, #64
333	stp x0, x1, [sp]
334	stp x2, x3, [sp, #16]
335	stp x4, x5, [sp, #32]
336	stp x6, x7, [sp, #48]
337
338	mov	x0, x29			//     parent's fp
339	bl	ftrace_return_to_handler// addr = ftrace_return_to_hander(fp);
340	mov	x30, x0			// restore the original return address
341
342	/* restore return value regs */
343	ldp x0, x1, [sp]
344	ldp x2, x3, [sp, #16]
345	ldp x4, x5, [sp, #32]
346	ldp x6, x7, [sp, #48]
347	add sp, sp, #64
348
349	ret
350SYM_CODE_END(return_to_handler)
351#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
352