bpf: Compute parameters liveness in bpf_compute_live_registers()

This computes exact liveness for the arguments of a call to a
subprogram instead of assuming that r1..r5 are read:
- For static subprograms compute use registers at the
  bpf_compute_live_registers() fixpoint phase.
- For global subprograms use BTF information.

Add a separate pass doing try_btf_prepare_func_args() in order for
bpf_subprog_info->arg_slot_cnt to be available during
bpf_compute_live_registers().

The conservative liveness marks hindered SCEV widening logic on a set
of production programs.

Signed-off-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://patch.msgid.link/20261009-scev-minimal-rebase-v3-11-bcba9bc53ac9@gmail.com

kernel/bpf/liveness.c

@@ -2472,6 +2472,12 @@ static inline u32 mask_widen(u32 m) { return m | (m << 16); }
 static inline u16 mask_lo(u32 m) { return (u16)m; }
 static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); }
 
+/* Index of the subprogram called by pseudo call @insn */
+static int callee_subprog(struct bpf_verifier_env *env, const struct bpf_insn *insn)
+{
+	return bpf_find_subprog(env, insn - env->prog->insnsi + insn->imm + 1);
+}
+
 /* Compute info->{use,def} fields for the instruction */
 static void compute_insn_live_regs(struct bpf_verifier_env *env,
 				   struct bpf_insn *insn,
@@ -2489,6 +2495,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env,
 	const u32 dst32 = mask_lo(dst);
 	const u32 r0  = reg64_mask(0);
 	const u32 r2  = reg64_mask(BPF_REG_2);
+	int subprog, reg_args;
 	u32 def = 0;
 	u32 use = U32_MAX;
 
@@ -2613,8 +2620,18 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env,
 		case BPF_CALL:
 			def = ALL_CALLER_SAVED_REGS;
 			use = def & ~BIT(BPF_REG_0);
-			if (bpf_get_call_summary(env, insn, &cs))
+			if (bpf_get_call_summary(env, insn, &cs)) {
 				use = GENMASK(min_t(u8, cs.arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS), 1);
+			} else if (bpf_pseudo_call(insn)) {
+				/* a static callee's reads are added in bpf_compute_live_registers() */
+				subprog = callee_subprog(env, insn);
+				if (bpf_subprog_is_global(env, subprog)) {
+					reg_args = min_t(u8, env->subprog_info[subprog].arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS);
+					use = GENMASK(reg_args, 1);
+				} else {
+					use = 0;
+				}
+			}
 			def = mask_widen(def);
 			use = mask_widen(use);
 			/* callx reads the address of the callee from dst_reg */
@@ -2699,11 +2716,21 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env)
 			struct bpf_iarray *succ;
 			u32 new_out = 0;
 			u32 new_in = 0;
+			u32 use;
 
 			succ = bpf_insn_successors(env, insn_idx);
 			for (int s = 0; s < succ->cnt; ++s)
 				new_out |= state[succ->items[s]].in;
-			new_in = (new_out & ~live->def) | live->use;
+			use = live->use;
+			/*
+			 * For a call to a static subprogram include the callee's
+			 * entry 'in' as computed so far.
+			 */
+			if (bpf_pseudo_call(&insns[insn_idx]) &&
+			    !bpf_subprog_is_global(env, callee_subprog(env, &insns[insn_idx])))
+				use |= state[insn_idx + insns[insn_idx].imm + 1].in &
+				       mask_widen(GENMASK(BPF_REG_5, BPF_REG_1));
+			new_in = (new_out & ~live->def) | use;
 			if (new_out != live->out || new_in != live->in) {
 				live->in = new_in;
 				live->out = new_out;

kernel/bpf/verifier.c

@@ -10913,6 +10913,7 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, s
 	struct bpf_call_arg_meta meta;
 	struct bpf_func_proto *fn;
 	int ret, err;
+	u32 i;
 
 	memset(&meta, 0, sizeof(meta));
 	meta.btf = btf;
@@ -10939,6 +10940,15 @@ static int btf_check_func_arg_match(struct bpf_verifier_env *env, int subprog, s
 	meta.fn = fn;
 	meta.func_proto = func_proto;
 
+	/*
+	 * Check if all register arguments declared in BTF are initialized.
+	 * If they aren't the BTF is unreliable.
+	 */
+	if (!bpf_subprog_is_global(env, subprog)) {
+		for (i = 0; i < min_t(u32, sub->arg_slot_cnt, MAX_BPF_FUNC_REG_ARGS); i++)
+			if (caller->regs[BPF_REG_1 + i].type == NOT_INIT)
+				return -EINVAL;
+	}
 	return check_func_args(env, &meta, env->insn_idx);
 }
 
@@ -14591,6 +14601,18 @@ static bool try_btf_prepare_func_args(struct bpf_verifier_env *env, int subprog)
 	return !err;
 }
 
+/* Prepare argument metadata for liveness; the main program is handled separately. */
+static void prepare_subprog_args(struct bpf_verifier_env *env)
+{
+	int i;
+
+	for (i = 1; i < env->subprog_cnt; i++) {
+		if (!try_btf_prepare_func_args(env, i) && bpf_subprog_is_global(env, i))
+			/* Keep all argument registers live if the prototype is unavailable. */
+			env->subprog_info[i].arg_slot_cnt = MAX_BPF_FUNC_REG_ARGS;
+	}
+}
+
 /*
  * Derive stack effects for a global subprog's argument slot @arg (0-based)
  * from its declared type, matching the main pass's argument checks.
@@ -22836,6 +22858,8 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr,
 	if (ret < 0)
 		goto skip_full_check;
 
+	prepare_subprog_args(env);
+
 	ret = bpf_compute_live_registers(env);
 	if (ret < 0)
 		goto skip_full_check;

tools/testing/selftests/bpf/prog_tests/exceptions.c

@@ -322,7 +322,7 @@ static void test_exceptions_success(void)
 				       bpf_program__fd(skel->progs.exception_throw_subprog),
 				       "subprog"), "set_attach_target"))
 				goto done;
-		}), "subprog() is not a global function", 0);
+		}), "Cannot replace static functions", 0);
 
 	if (test__start_subtest("non-throwing extension -> throwing subprog"))
 		/* non-throwing extension -> throwing subprog : BAD (!global) */

tools/testing/selftests/bpf/progs/compute_live_registers.c

@@ -416,11 +416,11 @@ static __used __naked int aux1(void)
 
 SEC("socket")
 __log_level(2)
-__msg("0: ....45.... (b7) r1 = 1")
-__msg("1: .1..45.... (b7) r2 = 2")
-__msg("2: .12.45.... (b7) r3 = 3")
-/* Conservative liveness for subprog parameters. */
-__msg("3: .12345.... (85) call pc+2")
+__msg("0: .......... (b7) r1 = 1")
+__msg("1: .1........ (b7) r2 = 2")
+__msg("2: .12....... (b7) r3 = 3")
+/* A static callee reads what its code reads: aux1 reads r1 and r2. */
+__msg("3: .12....... (85) call pc+2")
 __msg("4: .......... (b7) r0 = 0")
 __msg("5: 0......... (95) exit")
 __msg("6: .12....... (bf) r0 = r1")
@@ -452,7 +452,7 @@ SEC("socket")
 /* A program observing the pair needs the JIT; see bpf_compute_subprog_ret_regs(). */
 __load_if_JITed()
 __log_level(2)
-__msg("0: .12345.... (85) call pc+2")
+__msg("0: .......... (85) call pc+2")
 __msg("1: ..2....... (bf) r0 = r2")
 __msg("2: 0......... (95) exit")
 __msg("3: .......... (b7) r0 = 1")

tools/testing/selftests/bpf/progs/verifier_live_stack.c

@@ -1470,7 +1470,7 @@ static __used __naked void inner_reader(void)
 SEC("socket")
 __log_level(2)
 /* fp-8 NOT live at call: callee writes before reading (parent_def kills it) */
-__msg("2: .12345.... (85) call pc+")
+__msg("2: .1........ (85) call pc+")
 __naked void callee_must_write_before_read(void)
 {
 	asm volatile (