Skip to content

Commit b21cdb9

Browse files
AndybnACTpalmer-dabbelt
AndybnACT
authored and
palmer-dabbelt
committed
riscv: ftrace: support direct call using call_ops
jump to FTRACE_ADDR if distance is out of reach Co-developed-by: Björn Töpel <bjorn@rivosinc.com> Signed-off-by: Björn Töpel <bjorn@rivosinc.com> Signed-off-by: Andy Chiu <andybnac@gmail.com> Link: https://lore.kernel.org/r/20250407180838.42877-11-andybnac@gmail.com Signed-off-by: Alexandre Ghiti <alexghiti@rivosinc.com> Signed-off-by: Palmer Dabbelt <palmer@dabbelt.com>
1 parent c217157 commit b21cdb9

5 files changed

Lines changed: 48 additions & 27 deletions

File tree

‎arch/riscv/Kconfig‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ config RISCV
152152
select HAVE_DMA_CONTIGUOUS if MMU
153153
select HAVE_DYNAMIC_FTRACE if !XIP_KERNEL && MMU && (CLANG_SUPPORTS_DYNAMIC_FTRACE || GCC_SUPPORTS_DYNAMIC_FTRACE)
154154
select FUNCTION_ALIGNMENT_4B if HAVE_DYNAMIC_FTRACE && RISCV_ISA_C
155-
select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
155+
select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS if HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS
156156
select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS if (DYNAMIC_FTRACE_WITH_ARGS && !CFI_CLANG)
157157
select HAVE_DYNAMIC_FTRACE_WITH_ARGS if HAVE_DYNAMIC_FTRACE
158158
select HAVE_FTRACE_GRAPH_FUNC

‎arch/riscv/include/asm/ftrace.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,9 @@ struct __arch_ftrace_regs {
130130
unsigned long sp;
131131
unsigned long s0;
132132
unsigned long t1;
133+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
134+
unsigned long direct_tramp;
135+
#endif
133136
union {
134137
unsigned long args[8];
135138
struct {
@@ -223,10 +226,13 @@ void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
223226
struct ftrace_ops *op, struct ftrace_regs *fregs);
224227
#define ftrace_graph_func ftrace_graph_func
225228

229+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
226230
static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs, unsigned long addr)
227231
{
228232
arch_ftrace_regs(fregs)->t1 = addr;
229233
}
234+
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
235+
230236
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_ARGS */
231237

232238
#endif /* __ASSEMBLY__ */

‎arch/riscv/kernel/asm-offsets.c‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -495,6 +495,9 @@ void asm_offsets(void)
495495
OFFSET(STACKFRAME_RA, stackframe, ra);
496496
#ifdef CONFIG_FUNCTION_TRACER
497497
DEFINE(FTRACE_OPS_FUNC, offsetof(struct ftrace_ops, func));
498+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
499+
DEFINE(FTRACE_OPS_DIRECT_CALL, offsetof(struct ftrace_ops, direct_call));
500+
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
498501
#endif
499502

500503
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS

‎arch/riscv/kernel/ftrace.c‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
unsigned long ftrace_call_adjust(unsigned long addr)
1818
{
1919
if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS))
20-
return addr + 8;
20+
return addr + 8 + MCOUNT_AUIPC_SIZE;
2121

2222
return addr + MCOUNT_AUIPC_SIZE;
2323
}
@@ -83,10 +83,9 @@ static const struct ftrace_ops *riscv64_rec_get_ops(struct dyn_ftrace *rec)
8383
return ops;
8484
}
8585

86-
static int ftrace_rec_set_ops(const struct dyn_ftrace *rec,
87-
const struct ftrace_ops *ops)
86+
static int ftrace_rec_set_ops(const struct dyn_ftrace *rec, const struct ftrace_ops *ops)
8887
{
89-
unsigned long literal = rec->ip - 8;
88+
unsigned long literal = ALIGN_DOWN(rec->ip - 12, 8);
9089

9190
return patch_text_nosync((void *)literal, &ops, sizeof(ops));
9291
}
@@ -117,7 +116,7 @@ int ftrace_make_call(struct dyn_ftrace *rec, unsigned long addr)
117116
orig_addr = (unsigned long)&ftrace_caller;
118117
distance = addr > orig_addr ? addr - orig_addr : orig_addr - addr;
119118
if (distance > JALR_RANGE)
120-
return -EINVAL;
119+
addr = FTRACE_ADDR;
121120

122121
return __ftrace_modify_call(pc, addr, false);
123122
}
@@ -204,15 +203,13 @@ int ftrace_modify_call(struct dyn_ftrace *rec, unsigned long old_addr,
204203
unsigned long addr)
205204
{
206205
unsigned long caller = rec->ip - MCOUNT_AUIPC_SIZE;
207-
unsigned int call[2];
208206
int ret;
209207

210-
make_call_t0(caller, old_addr, call);
211208
ret = ftrace_rec_update_ops(rec);
212209
if (ret)
213210
return ret;
214211

215-
return __ftrace_modify_call(caller, addr, true);
212+
return __ftrace_modify_call(caller, FTRACE_ADDR, true);
216213
}
217214
#endif
218215

‎arch/riscv/kernel/mcount-dyn.S‎

Lines changed: 33 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -82,12 +82,9 @@
8282
* +++++++++
8383
**/
8484
.macro SAVE_ABI_REGS
85-
mv t4, sp // Save original SP in T4
8685
addi sp, sp, -FREGS_SIZE_ON_STACK
87-
8886
REG_S t0, FREGS_EPC(sp)
8987
REG_S x1, FREGS_RA(sp)
90-
REG_S t4, FREGS_SP(sp) // Put original SP on stack
9188
#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
9289
REG_S x8, FREGS_S0(sp)
9390
#endif
@@ -108,9 +105,12 @@
108105
REG_S x15, FREGS_A5(sp)
109106
REG_S x16, FREGS_A6(sp)
110107
REG_S x17, FREGS_A7(sp)
108+
mv a0, sp
109+
addi a0, a0, FREGS_SIZE_ON_STACK
110+
REG_S a0, FREGS_SP(sp) // Put original SP on stack
111111
.endm
112112

113-
.macro RESTORE_ABI_REGS, all=0
113+
.macro RESTORE_ABI_REGS
114114
REG_L t0, FREGS_EPC(sp)
115115
REG_L x1, FREGS_RA(sp)
116116
#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
@@ -139,6 +139,19 @@
139139

140140
.macro PREPARE_ARGS
141141
addi a0, t0, -MCOUNT_JALR_SIZE // ip (callsite's jalr insn)
142+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
143+
mv a1, ra // parent_ip
144+
REG_L a2, -16(t0) // op
145+
REG_L ra, FTRACE_OPS_FUNC(a2) // op->func
146+
#else
147+
la a1, function_trace_op
148+
REG_L a2, 0(a1) // op
149+
mv a1, ra // parent_ip
150+
#endif
151+
mv a3, sp // regs
152+
.endm
153+
154+
SYM_FUNC_START(ftrace_caller)
142155
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS
143156
/*
144157
* When CALL_OPS is enabled (2 or 4) nops [8B] are placed before the
@@ -158,19 +171,17 @@
158171
* t0 is set to ip+8 after the jalr is executed at the callsite,
159172
* so we find the associated op at t0-16.
160173
*/
161-
mv a1, ra // parent_ip
162-
REG_L a2, -16(t0) // op
163-
REG_L ra, FTRACE_OPS_FUNC(a2) // op->func
164-
#else
165-
la a1, function_trace_op
166-
REG_L a2, 0(a1) // op
167-
mv a1, ra // parent_ip
168-
#endif
169-
mv a3, sp // regs
170-
.endm
174+
REG_L t1, -16(t0) // op Should be SZ_REG instead of 16
171175

172-
SYM_FUNC_START(ftrace_caller)
173-
mv t1, zero
176+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
177+
/*
178+
* If the op has a direct call, handle it immediately without
179+
* saving/restoring registers.
180+
*/
181+
REG_L t1, FTRACE_OPS_DIRECT_CALL(t1)
182+
bnez t1, ftrace_caller_direct
183+
#endif
184+
#endif
174185
SAVE_ABI_REGS
175186
PREPARE_ARGS
176187

@@ -182,10 +193,14 @@ SYM_INNER_LABEL(ftrace_call, SYM_L_GLOBAL)
182193
jalr ra, 0(ra)
183194
#endif
184195
RESTORE_ABI_REGS
185-
bnez t1, .Ldirect
196+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
197+
bnez t1, ftrace_caller_direct
198+
#endif
186199
jr t0
187-
.Ldirect:
200+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
201+
SYM_INNER_LABEL(ftrace_caller_direct, SYM_L_LOCAL)
188202
jr t1
203+
#endif
189204
SYM_FUNC_END(ftrace_caller)
190205

191206
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS

0 commit comments

Comments
 (0)