driver: remove implicit test binaries after execution (#31236)

TESTS.md

@@ -43,6 +43,11 @@ and will exit with a 0 exit code, if they all had 0 failing assertions.
 `v -stats file_test.v` - this will run the test_ functions, and show a
 report about how much time it took to run each of them too.
 
+Implicit test executables and their macOS debug symbols are removed after the test process exits,
+including failed assertions, panics, and nonzero exits. This also applies with `-stats`, `-keepc`,
+and `v run file_test.v`; `-keepc` still retains generated C. An explicit `-o` keeps the requested
+output for later execution or debugging.
+
 NB 2: You can also run many test files at once (in parallel, depending on
 how many cores you have), with:
 `v test folder` - this will run *all* `_test.v` files in `folder`,

vlib/v/compiler_tests/test_binary_cleanup_test.v

@@ -0,0 +1,128 @@
+import os
+import v.cmdexec
+
+fn binary_cleanup_root(name string) !string {
+	root := os.join_path(os.vtmp_dir(), 'test binary cleanup ${name} ${os.getpid()}')
+	os.mkdir_all(root)!
+	return root
+}
+
+fn binary_cleanup_source(body string) string {
+	return 'import os
+fn test_cleanup() {
+	os.write_file("binary_path", os.executable())!
+	assert os.is_file(os.executable())
+	${body}
+}
+'
+}
+
+fn binary_cleanup_compile(root string, args []string) os.Result {
+	old_vtmp := os.getenv_opt('VTMP')
+	os.setenv('VTMP', root, true)
+	defer {
+		if value := old_vtmp {
+			os.setenv('VTMP', value, true)
+		} else {
+			os.unsetenv('VTMP')
+		}
+	}
+	vexe := if os.base(@VEXE) in ['v1_fallback', 'v1_fallback.exe'] {
+		os.join_path(os.dir(@VEXE), 'v' + $if windows { '.exe' } $else { '' })
+	} else {
+		@VEXE
+	}
+	cc := $if windows { 'gcc' } $else { 'cc' }
+	mut flags := ['-new-compiler', '-no-retry-compilation', '-memory-limit', '10G', '-cc', cc]
+	flags << args
+	return cmdexec.run_in(vexe, flags, root)
+}
+
+fn test_implicit_test_binaries_are_removed_after_success_and_failure() {
+	root := binary_cleanup_root('exit')!
+	defer {
+		os.rmdir_all(root) or {}
+	}
+	source := os.join_path(root, 'sample_test.v')
+	marker := os.join_path(root, 'binary_path')
+	for body in ['assert true', 'assert false', 'panic("cleanup panic")', 'exit(23)'] {
+		text := binary_cleanup_source(body)
+		os.write_file(source, text)!
+		result := binary_cleanup_compile(root, ['sample_test.v'])
+		if body == 'assert true' {
+			assert result.exit_code == 0, result.output
+		} else if body == 'exit(23)' {
+			assert result.exit_code == 23, result.output
+		} else {
+			assert result.exit_code != 0, result.output
+		}
+		binary := os.read_file(marker)!.trim_space()
+		assert !os.exists(binary), 'test binary remains after ${body}: ${binary}'
+		assert !os.exists(binary + '.dSYM')
+		assert os.read_file(source)! == text
+		os.rm(marker)!
+	}
+}
+
+fn test_implicit_test_binary_cleanup_with_run_stats_keepc_and_debug() {
+	root := binary_cleanup_root('flags')!
+	defer {
+		os.rmdir_all(root) or {}
+	}
+	source := os.join_path(root, 'sample_test.v')
+	marker := os.join_path(root, 'binary_path')
+	os.write_file(source, binary_cleanup_source('assert false'))!
+	for flags in [['-stats'], ['-keepc'], ['-g'], ['run'], ['-keepc', 'run'], ['crun']] {
+		c_files_before := os.ls(root)!.filter(it.ends_with('.tmp.c')).len
+		// A previous implicit test executable is still a compiler-owned artifact.
+		os.write_file(os.join_path(root, 'sample_test' + $if windows { '.exe' } $else { '' }),
+			'old test artifact')!
+		mut args := flags.clone()
+		args << 'sample_test.v'
+		result := binary_cleanup_compile(root, args)
+		assert result.exit_code != 0, result.output
+		binary := os.read_file(marker)!.trim_space()
+		assert !os.exists(binary), 'test binary remains with ${flags}: ${binary}'
+		assert !os.exists(binary + '.dSYM')
+		if '-keepc' in flags {
+			assert os.ls(root)!.filter(it.ends_with('.tmp.c')).len > c_files_before
+		}
+		os.rm(marker)!
+	}
+}
+
+fn test_explicit_test_outputs_are_preserved() {
+	root := binary_cleanup_root('explicit')!
+	defer {
+		os.rmdir_all(root) or {}
+	}
+	source := os.join_path(root, 'sample_test.v')
+	marker := os.join_path(root, 'binary_path')
+	binary := os.join_path(root, 'chosen output' + $if windows { '.exe' } $else { '' })
+	os.write_file(source, binary_cleanup_source('assert true'))!
+	build := binary_cleanup_compile(root, ['-skip-running', '-o', binary, source])
+	assert build.exit_code == 0, build.output
+	assert !os.exists(marker)
+	assert os.is_file(binary)
+	run := cmdexec.run_in(binary, [], root)
+	assert run.exit_code == 0, run.output
+	assert os.read_file(marker)!.trim_space() == os.real_path(binary)
+	os.rm(marker)!
+	os.write_file(source, binary_cleanup_source('assert false'))!
+	run_with_output := binary_cleanup_compile(root, ['-stats', '-o', binary, source])
+	assert run_with_output.exit_code != 0, run_with_output.output
+	assert os.read_file(marker)!.trim_space() == os.real_path(binary)
+	assert os.is_file(binary)
+}
+
+fn test_directory_test_command_removes_implicit_binary() {
+	root := binary_cleanup_root('directory')!
+	defer {
+		os.rmdir_all(root) or {}
+	}
+	os.write_file(os.join_path(root, 'sample_test.v'), binary_cleanup_source('assert true'))!
+	result := binary_cleanup_compile(root, ['test', root])
+	assert result.exit_code == 0, result.output
+	binary := os.read_file(os.join_path(root, 'binary_path'))!.trim_space()
+	assert !os.exists(binary), 'directory test binary remains: ${binary}'
+}

vlib/v/driver/driver.v

@@ -2220,14 +2220,20 @@ fn cleanup_c_build_dir(dir string) {
 	}
 }
 
-fn run_test_binary(bin_file string) int {
-	return run_binary(bin_file, []string{})
+fn run_test_binary(bin_file string, keep_binary bool) int {
+	return run_binary(bin_file, []string{}, !keep_binary)
 }
 
 fn ignore_run_signal(_ os.Signal) {
 }
 
-fn run_binary(bin_file string, args []string) int {
+fn run_binary(bin_file string, args []string, remove_after_run bool) int {
+	defer {
+		if remove_after_run {
+			os.rm(bin_file) or {}
+			v3_remove_macos_debug_symbols(bin_file)
+		}
+	}
 	// The build is published: the builds that wait to populate the cache need
 	// not wait for the program as well.
 	release_v3_cache_population_lock()
@@ -2316,7 +2322,7 @@ fn run_v_inline_source(args []string, option_index int, consumed int, source str
 	mut child_args := args[..option_index].clone()
 	child_args << ['run', temporary_source]
 	child_args << args[option_index + consumed..]
-	exit_code := run_binary(os.executable(), child_args)
+	exit_code := run_binary(os.executable(), child_args, false)
 	os.rm(temporary_source) or {
 		eprintln('cannot remove temporary inline source file: ${err.msg()}')
 		if exit_code == 0 {
@@ -12358,8 +12364,8 @@ pub fn run(args []string) {
 		exit(1)
 	}
 	binary_existed_before := os.exists(bin_file)
-	remove_binary_after_run := should_run && !is_crun && !is_direct_vsh && !explicit_output && !keep_c
-		&& !binary_existed_before
+	mut remove_binary_after_run := should_run && !explicit_output
+		&& (fastc_test_file || (!is_crun && !is_direct_vsh && !keep_c && !binary_existed_before))
 
 	// Decide which backend modules to compile into the output. Standalone compilers
 	// default to C; optional backends and their SSA dependencies can be skipped. This
@@ -12932,10 +12938,7 @@ pub fn run(args []string) {
 				}
 			}
 			if should_run {
-				run_result := run_binary(bin_file, run_args)
-				if remove_binary_after_run {
-					os.rm(bin_file) or {}
-				}
+				run_result := run_binary(bin_file, run_args, remove_binary_after_run)
 				if run_result != 0 {
 					exit(run_result)
 				}
@@ -13410,6 +13413,9 @@ pub fn run(args []string) {
 		exit(1)
 	}
 	test_files := test_input_files(user_files, backend, prefs.target)
+	if should_run && test_files.len > 0 && !explicit_output {
+		remove_binary_after_run = true
+	}
 	// A test binary keeps the harness counters (`__test_failures` and the
 	// setjmp buffer that `assert` longjmps out of) as `static` definitions inside
 	// the program body, so they live in whichever split unit the body lands in
@@ -13804,7 +13810,7 @@ pub fn run(args []string) {
 		}
 		if os.is_file(bin_file) && v3_crun_cache_matches(bin_file, crun_build_identity, input_file) {
 			clear_macos_v3_compiler_error_fallback(macos_v3_fallback_file)
-			run_result := run_binary(bin_file, run_args)
+			run_result := run_binary(bin_file, run_args, remove_binary_after_run)
 			if run_result != 0 {
 				exit(run_result)
 			}
@@ -13927,10 +13933,7 @@ pub fn run(args []string) {
 					b.step(if cache_with_tcc { 'tcc (cached)' } else { 'cc (cached)' })
 					clear_macos_v3_compiler_error_fallback(macos_v3_fallback_file)
 					if should_run {
-						run_result := run_binary(bin_file, run_args)
-						if remove_binary_after_run {
-							os.rm(bin_file) or {}
-						}
+						run_result := run_binary(bin_file, run_args, remove_binary_after_run)
 						if run_result != 0 {
 							exit(run_result)
 						}
@@ -15744,19 +15747,13 @@ pub fn run(args []string) {
 			b.step('link')
 			clear_macos_v3_compiler_error_fallback(macos_v3_fallback_file)
 			if should_run {
-				run_result := run_binary(bin_file, run_args)
-				if remove_binary_after_run {
-					os.rm(bin_file) or {}
-					if (is_debug || race) && target.os == 'macos' {
-						v3_remove_macos_debug_symbols(bin_file)
-					}
-				}
+				run_result := run_binary(bin_file, run_args, remove_binary_after_run)
 				if run_result != 0 {
 					exit(run_result)
 				}
 				b.step('run')
 			} else if test_files.len > 0 && (!explicit_output || is_checker_fixture || show_test_stats) {
-				test_result := run_test_binary(bin_file)
+				test_result := run_test_binary(bin_file, explicit_output)
 				if test_result != 0 {
 					exit(test_result)
 				}
@@ -17591,22 +17588,16 @@ Please install the corresponding development package/libraries and make sure the
 		// The C compiler is done. What was set up for `cl` must not reach the program that runs next.
 		msvc_restore_environment(msvc_saved_environment)
 		if should_run {
-			if (is_crun || is_direct_vsh) && !explicit_output {
+			if (is_crun || is_direct_vsh) && !explicit_output && !remove_binary_after_run {
 				write_v3_crun_cache_marker(bin_file, crun_build_identity) or {}
 			}
-			run_result := run_binary(bin_file, run_args)
-			if remove_binary_after_run {
-				os.rm(bin_file) or {}
-				if (is_debug || race) && target.os == 'macos' {
-					v3_remove_macos_debug_symbols(bin_file)
-				}
-			}
+			run_result := run_binary(bin_file, run_args, remove_binary_after_run)
 			if run_result != 0 {
 				exit(run_result)
 			}
 			b.step('run')
 		} else if test_files.len > 0 && (!explicit_output || is_checker_fixture || show_test_stats) {
-			test_result := run_test_binary(bin_file)
+			test_result := run_test_binary(bin_file, explicit_output)
 			if test_result != 0 {
 				exit(test_result)
 			}

vlib/v/driver/race.v

@@ -135,7 +135,7 @@ fn v3_keep_macos_debug_symbols(staged_binary string, bin_file string) {
 }
 
 // v3_remove_macos_debug_symbols removes the debug symbols of a macOS debug binary that
-// `v run` deletes after running it.
+// a run or test deletes after running it.
 fn v3_remove_macos_debug_symbols(bin_file string) {
 	os.rmdir_all(bin_file + '.dSYM') or {}
 }