mirror of
https://github.com/vercel-labs/fx.git
synced 2026-10-28 10:15:17 +08:00
Add a fuzz build step, nightly fuzzing, and shell parity tests
- `zig build fuzz` builds only the fuzz tests listed in `tests/fuzz/targets.json`. `-Dfuzz-target=<name>` picks one target, and `--fuzz=N` searches for new failing inputs. - `zig build test` replays each target's saved inputs from `tests/fuzz/corpus/<target>/` on every pull request. - A nightly Fuzz workflow fuzzes every target for about 20 minutes, uploads any failing input, and opens or updates an issue for that target. A pull request that changes the fuzzing setup fuzzes each target for two minutes. - The shell parity tests run every command form fx runs without review through sh, dash, bash, and zsh, with and without `extendedglob`, and check that each one runs exactly as fx parsed it. The `shell_parity` target checks generated commands the same way.
This commit is contained in:
27 files changed
+1682
-43
No files matched your search
@@ -56,7 +56,7 @@ jobs:
|
||||
run: ./scripts/check-indexer-boundary.sh
|
||||
|
||||
- name: Test CI and release scripts
|
||||
run: python3 -m unittest scripts.tests.test_macos_signing scripts.tests.test_signed_macos scripts.tests.test_detect_macos_need -q
|
||||
run: python3 -m unittest scripts.tests.test_macos_signing scripts.tests.test_signed_macos scripts.tests.test_detect_macos_need scripts.tests.test_fuzz_nightly -q
|
||||
|
||||
- name: Build
|
||||
run: zig build -Doptimize=${{ matrix.optimize }}
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
name: Fuzz
|
||||
|
||||
# Nightly coverage-guided fuzzing. Each target in tests/fuzz/targets.json gets
|
||||
# its own job and about 20 minutes of fuzzing. A failure uploads the recovered
|
||||
# input and opens or updates the target's issue, titled
|
||||
# "Nightly fuzz failure: <target>". scripts/fuzz_nightly.py does the work.
|
||||
# A pull request that changes the fuzzing setup fuzzes every target for two
|
||||
# minutes, so a change that breaks the nightly run fails before it merges.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "23 7 * * *"
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/fuzz.yml
|
||||
- scripts/fuzz_nightly.py
|
||||
- src/fuzz_targets.zig
|
||||
- tests/fuzz/**
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
target:
|
||||
description: Fuzz only this target (leave empty for every target)
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: fuzz-${{ github.ref }}
|
||||
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
|
||||
|
||||
jobs:
|
||||
targets:
|
||||
name: List fuzz targets
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
targets: ${{ steps.list.outputs.targets }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: List targets
|
||||
id: list
|
||||
env:
|
||||
ONLY: ${{ inputs.target }}
|
||||
run: python3 -m scripts.fuzz_nightly matrix --only "$ONLY" >> "$GITHUB_OUTPUT"
|
||||
|
||||
fuzz:
|
||||
name: Fuzz ${{ matrix.target }}
|
||||
needs: targets
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
target: ${{ fromJSON(needs.targets.outputs.targets) }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
# Restores main's unit-test cache for the dependencies, compiler_rt, and
|
||||
# libc it keeps. This job never saves a cache.
|
||||
- name: Setup Zig
|
||||
uses: ./.github/actions/setup-zig
|
||||
with:
|
||||
version: "0.17.0"
|
||||
cache-key: ci-unit-ReleaseSafe
|
||||
|
||||
# shell_parity compares fx's parse with sh, dash, bash, and zsh.
|
||||
- name: Install zsh
|
||||
if: matrix.target == 'shell_parity'
|
||||
run: command -v zsh >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq zsh; }
|
||||
|
||||
- name: Fuzz
|
||||
env:
|
||||
TARGET: ${{ matrix.target }}
|
||||
BUDGET_SECONDS: ${{ github.event_name == 'pull_request' && '120' || '1200' }}
|
||||
run: >-
|
||||
python3 -m scripts.fuzz_nightly run
|
||||
--target "$TARGET"
|
||||
--out fuzz-output
|
||||
--budget-seconds "$BUDGET_SECONDS"
|
||||
|
||||
- name: Upload failure evidence
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: fuzz-${{ matrix.target }}
|
||||
path: fuzz-output/
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
# Only main reports, so a dispatch on a branch cannot open issues.
|
||||
- name: Report failure
|
||||
if: failure() && github.ref == 'refs/heads/main'
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
TARGET: ${{ matrix.target }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: >-
|
||||
python3 -m scripts.fuzz_nightly report
|
||||
--target "$TARGET"
|
||||
--out fuzz-output
|
||||
--repo "$GITHUB_REPOSITORY"
|
||||
--run-url "$RUN_URL"
|
||||
--commit "$GITHUB_SHA"
|
||||
@@ -166,6 +166,8 @@ Security is permission-first. All sensitive tool behavior must integrate with `s
|
||||
|
||||
* Host-generated review holds retain their advice for the agent and transcript, but carry a saved `review_feedback` marker that excludes them from later security evidence, including after recovery. Old unmarked results remain untrusted evidence; never infer the marker from output text. Quoted review accusations are not proof of an attack, and handling plans or test instructions as data is not itself prompt injection
|
||||
|
||||
* A command form that fx runs without review, through `command_effect.plan` or `command_effect.knownReversibleAutoCommand`, must be listed in `parity_review_free_commands` in `src/core/shell_command/command_effect.zig`. The shell parity tests check that sh, dash, bash, and zsh, with and without `extendedglob`, run each one exactly as fx parsed it, and the nightly `shell_parity` fuzz target checks generated commands the same way
|
||||
|
||||
Do not bypass the permission system for new tools.
|
||||
|
||||
## Zig-Specific Patterns
|
||||
@@ -244,6 +246,8 @@ fx does not use declarations that Zig 0.17 deprecates. They still compile, so ch
|
||||
|
||||
* In test blocks, use `std.testing.io` for the `Io` parameter. `io_mod.getIo()` automatically returns `std.testing.io` in test builds.
|
||||
|
||||
* A new `std.testing.fuzz` test needs an entry in `tests/fuzz/targets.json` and its saved inputs passed as its corpus, as **Fuzz a parser** in `CONTRIBUTING.md` describes. The nightly Fuzz workflow fuzzes every entry.
|
||||
|
||||
* Use `io_mod.dirRealpathAlloc(alloc, dir, sub_path)` to resolve paths within `std.testing.tmpDir()`.
|
||||
|
||||
* Back `std.testing.checkAllAllocationFailures`, hand-written `FailingAllocator` sweeps, and allocation-count comparisons with `testing_allocator.no_resize` from `src/core/shared/testing_allocator.zig`. `std.testing.allocator` may grow an allocation in place on one run and not the next, which changes the allocation sequence and makes those tests flaky.
|
||||
|
||||
@@ -98,6 +98,23 @@ fx prints into your terminal's scrollback instead of taking over the screen, so
|
||||
|
||||
When a tmux test or a tape exposes a bug, reproduce it as a Zig unit test before you fix it, so the fix lands with its regression test.
|
||||
|
||||
### Fuzz a parser
|
||||
|
||||
A fuzz test is a `test` block that calls `std.testing.fuzz`. `tests/fuzz/targets.json` lists each one as a target, with its exact test name and the module that holds it. `zig build test` runs every fuzz test once on its inline seeds and on each file in `tests/fuzz/corpus/<target>/`. `zig build fuzz` builds only the fuzz tests, and with `--fuzz` Zig searches for new inputs that fail:
|
||||
|
||||
```bash
|
||||
zig build fuzz -Dfuzz-target=sse_framing # replay the saved inputs
|
||||
zig build fuzz -Dfuzz-target=sse_framing --fuzz=200K # try 200,000 new inputs
|
||||
```
|
||||
|
||||
When the search finds a failure, Zig 0.17 prints `error: test '...'` but still exits with status 0, so read the output. Without a limit, `--fuzz` runs until you stop it and opens Zig's web interface.
|
||||
|
||||
The Fuzz workflow fuzzes each target on `main` every night for about 20 minutes, and for two minutes on a pull request that changes the fuzzing setup. When the nightly run finds a failure, it uploads the failing input as an artifact and opens an issue titled `Nightly fuzz failure: <target>`, or comments on that issue if it is already open. To fix the failure, copy the input into `tests/fuzz/corpus/<target>/` and run `zig build fuzz -Dfuzz-target=<target>` to reproduce it. That build uses Debug by default, which adds allocation stack traces to leak reports. Keep the input file in the fix's pull request, so `zig build test` replays it on every pull request after that.
|
||||
|
||||
A saved input holds the values the test reads from its `std.testing.Smith`, in order. For example, a test that calls `smith.slice` reads a 4-byte little-endian length and then that many bytes. Inputs that the fuzzer saves are already in this form.
|
||||
|
||||
To add a target, write the fuzz test in the file it covers and add an entry to `tests/fuzz/targets.json`. For a target outside the session manager, also import its file in `src/fuzz_targets.zig`. Pass the saved inputs to the test with `.corpus = &(.{ <seeds> } ++ @import("fuzz_corpus").<target>.*)`, or with `@import("build_options").fuzz_corpus_<target>` in the session manager. Set `iterations` so that one round of `--fuzz` takes about 30 seconds on a developer machine. `scripts/tests/test_fuzz_nightly.py` fails when an entry names a test that doesn't exist or that `zig build fuzz` can't reach.
|
||||
|
||||
### Check startup performance
|
||||
|
||||
The `Startup Latency` check fails a pull request when any benchmarked command takes longer than 2 ms on the Linux CI runner. To measure before you push, run:
|
||||
@@ -197,6 +214,7 @@ Other runs start only on request:
|
||||
| Live ACP suite | `gh workflow run ci.yml --ref your-branch -f acp_live=true` |
|
||||
| Long-turn memory benchmark | `gh workflow run bench.yml --ref your-branch -f long_turn_memory=true` |
|
||||
| libfx runtime benchmark | `gh workflow run bench.yml --ref your-branch -f libfx_runtime=true` |
|
||||
| Fuzzing of one target | `gh workflow run fuzz.yml --ref your-branch -f target=<target>` |
|
||||
|
||||
Only maintainers can start these runs or add the `ci:macos` label. If you're an outside contributor and your change needs one of them, ask in the pull request and a maintainer will start it. If CI doesn't start at all on your first pull request, a maintainer may need to approve the workflow run.
|
||||
|
||||
|
||||
@@ -52,6 +52,9 @@ pub fn build(b: *std.Build) void {
|
||||
// binary uses, such as its E2E test controls.
|
||||
build_options.addOption(bool, "libfx", false);
|
||||
|
||||
const fuzz_corpora = readFuzzCorpora(b);
|
||||
const fuzz_corpus = addFuzzCorpusModule(b, fuzz_corpora);
|
||||
|
||||
const exe = b.addExecutable(.{
|
||||
.name = "fx",
|
||||
.root_module = b.createModule(.{
|
||||
@@ -68,7 +71,8 @@ pub fn build(b: *std.Build) void {
|
||||
}),
|
||||
});
|
||||
exe.root_module.addImport("build_options", build_options.createModule());
|
||||
const session_manager = addSessionManager(b, exe.root_module, target, optimize, null);
|
||||
exe.root_module.addImport("fuzz_corpus", fuzz_corpus);
|
||||
const session_manager = addSessionManager(b, exe.root_module, target, optimize, null, fuzz_corpora);
|
||||
|
||||
b.installArtifact(exe);
|
||||
|
||||
@@ -89,7 +93,8 @@ pub fn build(b: *std.Build) void {
|
||||
const test_paths = b.addOptions();
|
||||
test_paths.addOptionPathUntracked("product_exe", b.graph.path(.install_bin, "fx"));
|
||||
test_paths.addOptionPathUntracked("source_root", b.path("src"));
|
||||
exe.root_module.addImport("test_paths", test_paths.createModule());
|
||||
const test_paths_module = test_paths.createModule();
|
||||
exe.root_module.addImport("test_paths", test_paths_module);
|
||||
|
||||
// The session manager's own tests, as fx compiles it (no hooks).
|
||||
const session_manager_tests = b.addTest(.{ .root_module = session_manager });
|
||||
@@ -97,7 +102,7 @@ pub fn build(b: *std.Build) void {
|
||||
const run_session_manager_tests = b.addRunArtifact(session_manager_tests);
|
||||
// And with its fault and trace hooks, whose tests fx's build leaves out.
|
||||
const session_manager_hook_tests = b.addTest(.{
|
||||
.root_module = sessionManagerModule(b, target, optimize, null, true),
|
||||
.root_module = sessionManagerModule(b, target, optimize, null, true, fuzz_corpora),
|
||||
});
|
||||
const run_session_manager_hook_tests = b.addRunArtifact(session_manager_hook_tests);
|
||||
const session_manager_test_step = b.step("test-session-manager", "Run the session manager's tests");
|
||||
@@ -109,6 +114,30 @@ pub fn build(b: *std.Build) void {
|
||||
test_step.dependOn(&run_session_manager_tests.step);
|
||||
test_step.dependOn(&run_session_manager_hook_tests.step);
|
||||
|
||||
const fuzz_step = b.step(
|
||||
"fuzz",
|
||||
"Replay the fuzz targets' saved inputs; add --fuzz=N to search N new inputs",
|
||||
);
|
||||
const fuzz_target = b.option(
|
||||
[]const u8,
|
||||
"fuzz-target",
|
||||
"Run only this fuzz target from " ++ fuzz_registry_path,
|
||||
);
|
||||
const fuzz_root = b.createModule(.{
|
||||
.root_source_file = b.path("src/fuzz_targets.zig"),
|
||||
.target = target,
|
||||
.optimize = optimize,
|
||||
.link_libc = true,
|
||||
});
|
||||
fuzz_root.addImport("build_options", build_options.createModule());
|
||||
fuzz_root.addImport("fuzz_corpus", fuzz_corpus);
|
||||
fuzz_root.addImport("session_manager", session_manager);
|
||||
fuzz_root.addImport("test_paths", test_paths_module);
|
||||
addFuzzStep(b, fuzz_step, fuzz_corpora, fuzz_target, .{
|
||||
.fx = fuzz_root,
|
||||
.session_manager = session_manager,
|
||||
});
|
||||
|
||||
if (wasm_surface != .none) {
|
||||
addWasmArtifact(b, wasm_surface, git_commit, app_version, update_channel);
|
||||
}
|
||||
@@ -329,7 +358,7 @@ fn addWasmArtifact(
|
||||
});
|
||||
if (surface == .core) wasm_exe.stack_size = 1024 * 1024;
|
||||
wasm_exe.root_module.addImport("build_options", wasm_options.createModule());
|
||||
_ = addSessionManager(b, wasm_exe.root_module, wasm_target, .small, true);
|
||||
_ = addSessionManager(b, wasm_exe.root_module, wasm_target, .small, true, &.{});
|
||||
|
||||
const install_wasm = b.addInstallArtifact(wasm_exe, .{});
|
||||
const wasm_step = b.step(name ++ "-wasm", description);
|
||||
@@ -368,7 +397,7 @@ fn addNapiArtifact(
|
||||
}),
|
||||
});
|
||||
lib.root_module.addImport("build_options", napi_options.createModule());
|
||||
_ = addSessionManager(b, lib.root_module, target, .safe, null);
|
||||
_ = addSessionManager(b, lib.root_module, target, .safe, null, &.{});
|
||||
const node_include = b.option(
|
||||
[]const u8,
|
||||
"node-include-dir",
|
||||
@@ -418,15 +447,17 @@ fn readGitCommit(b: *std.Build) []const u8 {
|
||||
|
||||
/// The session manager as its own module, rooted at its API. It imports
|
||||
/// nothing but its own options, and a file cannot belong to two modules, so
|
||||
/// fx reaches it only through `@import("session_manager")`.
|
||||
/// fx reaches it only through `@import("session_manager")`. Its fuzz targets'
|
||||
/// saved inputs travel in those options as `fuzz_corpus_<target>`.
|
||||
fn addSessionManager(
|
||||
b: *std.Build,
|
||||
importer: *std.Build.Module,
|
||||
target: std.Build.ResolvedTarget,
|
||||
optimize: std.lang.Optimize,
|
||||
single_threaded: ?bool,
|
||||
fuzz_corpora: []const FuzzCorpus,
|
||||
) *std.Build.Module {
|
||||
const module = sessionManagerModule(b, target, optimize, single_threaded, false);
|
||||
const module = sessionManagerModule(b, target, optimize, single_threaded, false, fuzz_corpora);
|
||||
importer.addImport("session_manager", module);
|
||||
return module;
|
||||
}
|
||||
@@ -438,11 +469,20 @@ fn sessionManagerModule(
|
||||
optimize: std.lang.Optimize,
|
||||
single_threaded: ?bool,
|
||||
hooks: bool,
|
||||
fuzz_corpora: []const FuzzCorpus,
|
||||
) *std.Build.Module {
|
||||
const options = b.addOptions();
|
||||
options.addOption(bool, "hooks", hooks);
|
||||
options.addOptionPathUntracked("trace_dir", b.graph.path(.install_prefix, "session-manager-traces"));
|
||||
options.addOptionPathUntracked("src_dir", b.path("src/core/session_manager"));
|
||||
for (fuzz_corpora) |corpus| {
|
||||
if (corpus.target.module != .session_manager) continue;
|
||||
options.addOption(
|
||||
[]const []const u8,
|
||||
b.fmt("fuzz_corpus_{s}", .{corpus.target.name}),
|
||||
corpus.inputs,
|
||||
);
|
||||
}
|
||||
const module = b.createModule(.{
|
||||
.root_source_file = b.path("src/core/session_manager/api.zig"),
|
||||
.target = target,
|
||||
@@ -469,3 +509,205 @@ fn readAppVersion(b: *std.Build) []const u8 {
|
||||
return b.allocator.dupe(u8, bytes[start .. start + end_rel]) catch
|
||||
@panic("could not allocate app version");
|
||||
}
|
||||
|
||||
const fuzz_registry_path = "tests/fuzz/targets.json";
|
||||
const fuzz_corpus_dir = "tests/fuzz/corpus";
|
||||
/// A saved input this large is almost certainly a mistake, such as a build
|
||||
/// artifact copied into a corpus folder.
|
||||
const max_fuzz_input_bytes = 1024 * 1024;
|
||||
|
||||
const FuzzModule = enum { fx, session_manager };
|
||||
|
||||
/// One entry in `tests/fuzz/targets.json`.
|
||||
const FuzzTarget = struct {
|
||||
/// Names the target's corpus folder and its `fuzz_corpus` declaration.
|
||||
name: []const u8,
|
||||
/// The module whose test binary holds the fuzz test.
|
||||
module: FuzzModule,
|
||||
/// The fuzz test's exact name, used as the test filter.
|
||||
@"test": []const u8,
|
||||
/// Iterations for one nightly run. Only the nightly workflow reads it.
|
||||
iterations: u64,
|
||||
};
|
||||
|
||||
const FuzzRegistry = struct {
|
||||
targets: []const FuzzTarget,
|
||||
};
|
||||
|
||||
/// A registered fuzz target and its saved inputs, sorted by file name.
|
||||
const FuzzCorpus = struct {
|
||||
target: FuzzTarget,
|
||||
inputs: []const []const u8,
|
||||
};
|
||||
|
||||
const FuzzRoots = struct {
|
||||
fx: *std.Build.Module,
|
||||
session_manager: *std.Build.Module,
|
||||
};
|
||||
|
||||
/// Reads the fuzz registry and each target's saved inputs. Configuration
|
||||
/// reruns on every build (see `readGitCommit`), so a newly added input is
|
||||
/// picked up without listing it anywhere.
|
||||
fn readFuzzCorpora(b: *std.Build) []const FuzzCorpus {
|
||||
b.dependOnFileContents(b.path(fuzz_registry_path));
|
||||
const registry_bytes = b.root.root_dir.handle.readFileAlloc(
|
||||
b.graph.io,
|
||||
b.pathJoin(&.{ b.root.sub_path, fuzz_registry_path }),
|
||||
b.allocator,
|
||||
.limited(max_fuzz_input_bytes),
|
||||
) catch |err| std.process.fatal("could not read {s}: {t}", .{ fuzz_registry_path, err });
|
||||
const registry = std.json.parseFromSliceLeaky(FuzzRegistry, b.allocator, registry_bytes, .{}) catch |err|
|
||||
std.process.fatal("{s} is not a valid fuzz registry: {t}", .{ fuzz_registry_path, err });
|
||||
|
||||
const corpora = b.allocator.alloc(FuzzCorpus, registry.targets.len) catch @panic("OOM");
|
||||
for (registry.targets, corpora, 0..) |fuzz_target, *corpus, index| {
|
||||
if (!isFuzzTargetName(fuzz_target.name)) std.process.fatal(
|
||||
"{s}: fuzz target name '{s}' must start with a lowercase letter and use only lowercase letters, digits, and underscores",
|
||||
.{ fuzz_registry_path, fuzz_target.name },
|
||||
);
|
||||
for (registry.targets[0..index]) |earlier| {
|
||||
if (std.mem.eql(u8, earlier.name, fuzz_target.name)) std.process.fatal(
|
||||
"{s}: fuzz target '{s}' is listed twice",
|
||||
.{ fuzz_registry_path, fuzz_target.name },
|
||||
);
|
||||
}
|
||||
corpus.* = .{ .target = fuzz_target, .inputs = readFuzzInputs(b, fuzz_target.name) };
|
||||
}
|
||||
rejectUnregisteredCorpusFolders(b, registry.targets);
|
||||
return corpora;
|
||||
}
|
||||
|
||||
fn isFuzzTargetName(name: []const u8) bool {
|
||||
if (name.len == 0 or !std.ascii.isLower(name[0])) return false;
|
||||
for (name) |byte| {
|
||||
if (!std.ascii.isLower(byte) and !std.ascii.isDigit(byte) and byte != '_') return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Returns the files in `tests/fuzz/corpus/<name>/`, sorted by file name so
|
||||
/// the generated module does not depend on directory order. A missing folder
|
||||
/// means the target has no saved inputs yet. Dotfiles are skipped.
|
||||
fn readFuzzInputs(b: *std.Build, name: []const u8) []const []const u8 {
|
||||
const io = b.graph.io;
|
||||
const folder = b.pathJoin(&.{ fuzz_corpus_dir, name });
|
||||
var dir = b.root.root_dir.handle.openDir(io, b.pathJoin(&.{ b.root.sub_path, folder }), .{
|
||||
.iterate = true,
|
||||
}) catch |err| switch (err) {
|
||||
error.FileNotFound => return &.{},
|
||||
else => std.process.fatal("could not open {s}: {t}", .{ folder, err }),
|
||||
};
|
||||
defer dir.close(io);
|
||||
|
||||
var file_names: std.ArrayList([]const u8) = .empty;
|
||||
var entries = dir.iterate();
|
||||
while (entries.next(io) catch |err| std.process.fatal("could not list {s}: {t}", .{ folder, err })) |entry| {
|
||||
if (entry.name[0] == '.') continue;
|
||||
if (entry.kind != .file) std.process.fatal(
|
||||
"{s}/{s}: saved fuzz inputs must be regular files",
|
||||
.{ folder, entry.name },
|
||||
);
|
||||
file_names.append(b.allocator, b.dupe(entry.name)) catch @panic("OOM");
|
||||
}
|
||||
std.mem.sort([]const u8, file_names.items, {}, lessThanBytes);
|
||||
|
||||
const inputs = b.allocator.alloc([]const u8, file_names.items.len) catch @panic("OOM");
|
||||
for (file_names.items, inputs) |file_name, *input| {
|
||||
b.dependOnFileContents(b.path(b.pathJoin(&.{ folder, file_name })));
|
||||
input.* = dir.readFileAlloc(io, file_name, b.allocator, .limited(max_fuzz_input_bytes + 1)) catch |err| switch (err) {
|
||||
error.StreamTooLong => std.process.fatal("{s}/{s} is larger than 1 MiB", .{ folder, file_name }),
|
||||
else => std.process.fatal("could not read {s}/{s}: {t}", .{ folder, file_name, err }),
|
||||
};
|
||||
}
|
||||
return inputs;
|
||||
}
|
||||
|
||||
/// Fails the build when a corpus folder has no registry entry, so a typo in
|
||||
/// a folder name cannot leave saved inputs that never run.
|
||||
fn rejectUnregisteredCorpusFolders(b: *std.Build, targets: []const FuzzTarget) void {
|
||||
const io = b.graph.io;
|
||||
var dir = b.root.root_dir.handle.openDir(io, b.pathJoin(&.{ b.root.sub_path, fuzz_corpus_dir }), .{
|
||||
.iterate = true,
|
||||
}) catch |err| switch (err) {
|
||||
error.FileNotFound => return,
|
||||
else => std.process.fatal("could not open {s}: {t}", .{ fuzz_corpus_dir, err }),
|
||||
};
|
||||
defer dir.close(io);
|
||||
|
||||
var entries = dir.iterate();
|
||||
entries: while (entries.next(io) catch |err| std.process.fatal("could not list {s}: {t}", .{ fuzz_corpus_dir, err })) |entry| {
|
||||
if (entry.name[0] == '.') continue;
|
||||
for (targets) |fuzz_target| {
|
||||
if (entry.kind == .directory and std.mem.eql(u8, entry.name, fuzz_target.name)) continue :entries;
|
||||
}
|
||||
std.process.fatal(
|
||||
"{s}/{s} does not match a fuzz target in {s}",
|
||||
.{ fuzz_corpus_dir, entry.name, fuzz_registry_path },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn lessThanBytes(_: void, lhs: []const u8, rhs: []const u8) bool {
|
||||
return std.mem.lessThan(u8, lhs, rhs);
|
||||
}
|
||||
|
||||
/// The `fuzz_corpus` module: one `[]const []const u8` of saved inputs per fx
|
||||
/// fuzz target, named after the target.
|
||||
fn addFuzzCorpusModule(b: *std.Build, corpora: []const FuzzCorpus) *std.Build.Module {
|
||||
const options = b.addOptions();
|
||||
for (corpora) |corpus| {
|
||||
if (corpus.target.module != .fx) continue;
|
||||
options.addOption([]const []const u8, corpus.target.name, corpus.inputs);
|
||||
}
|
||||
return options.createModule();
|
||||
}
|
||||
|
||||
/// Runs the selected fuzz targets, or all of them. Each module's test binary
|
||||
/// is filtered to its targets' fuzz tests, so `--fuzz` spends every iteration
|
||||
/// on them and ordinary unit tests never run here.
|
||||
fn addFuzzStep(
|
||||
b: *std.Build,
|
||||
step: *std.Build.Step,
|
||||
corpora: []const FuzzCorpus,
|
||||
selected: ?[]const u8,
|
||||
roots: FuzzRoots,
|
||||
) void {
|
||||
var fx_filters: std.ArrayList([]const u8) = .empty;
|
||||
var session_manager_filters: std.ArrayList([]const u8) = .empty;
|
||||
for (corpora) |corpus| {
|
||||
if (selected) |name| {
|
||||
if (!std.mem.eql(u8, name, corpus.target.name)) continue;
|
||||
}
|
||||
const filters = switch (corpus.target.module) {
|
||||
.fx => &fx_filters,
|
||||
.session_manager => &session_manager_filters,
|
||||
};
|
||||
filters.append(b.allocator, corpus.target.@"test") catch @panic("OOM");
|
||||
}
|
||||
if (fx_filters.items.len == 0 and session_manager_filters.items.len == 0) {
|
||||
const message = if (selected) |name|
|
||||
b.fmt("unknown fuzz target '{s}'; the targets are listed in {s}", .{ name, fuzz_registry_path })
|
||||
else
|
||||
"no fuzz targets are listed in " ++ fuzz_registry_path;
|
||||
step.dependOn(&b.addFail(message).step);
|
||||
return;
|
||||
}
|
||||
|
||||
const binaries = [_]struct {
|
||||
name: []const u8,
|
||||
root: *std.Build.Module,
|
||||
filters: []const []const u8,
|
||||
}{
|
||||
.{ .name = "fuzz", .root = roots.fx, .filters = fx_filters.items },
|
||||
.{ .name = "fuzz-session-manager", .root = roots.session_manager, .filters = session_manager_filters.items },
|
||||
};
|
||||
for (binaries) |binary| {
|
||||
if (binary.filters.len == 0) continue;
|
||||
const tests = b.addTest(.{
|
||||
.name = binary.name,
|
||||
.root_module = binary.root,
|
||||
.filters = binary.filters,
|
||||
});
|
||||
step.dependOn(&b.addRunArtifact(tests).step);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,420 @@
|
||||
"""Nightly fuzzing driver for `.github/workflows/fuzz.yml`.
|
||||
|
||||
`matrix` prints the job matrix from `tests/fuzz/targets.json`.
|
||||
|
||||
`run` fuzzes one target in rounds of `zig build fuzz --fuzz=<iterations>`
|
||||
until its time budget is spent. Zig keeps the fuzzer's corpus in
|
||||
`.zig-cache/f/` between rounds, so each round continues where the last one
|
||||
stopped, and a round that outlives its timeout is a hang rather than an
|
||||
exhausted budget. When a round fails, `run` recovers the failing input,
|
||||
confirms it by replaying it as a saved input of the target, and writes it to
|
||||
the output folder with the log and `result.json`.
|
||||
|
||||
`report` opens the target's GitHub issue, or comments on it when one is
|
||||
already open.
|
||||
|
||||
Two Zig 0.17 behaviors shape this script. `zig build --fuzz` exits 0 even
|
||||
when it finds a failure, so a round is judged by its `error: test` line. The
|
||||
input it copies to `.zig-cache/f/crash` can be empty, so the inputs are also
|
||||
read from the fuzzer's `in<N>` files.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import dataclasses
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import signal
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
|
||||
REPO_ROOT = pathlib.Path(__file__).resolve().parents[1]
|
||||
REGISTRY_PATH = pathlib.PurePosixPath("tests/fuzz/targets.json")
|
||||
CORPUS_PATH = pathlib.PurePosixPath("tests/fuzz/corpus")
|
||||
|
||||
# Zig 0.17 `std.Build.abi.fuzz.MmapInputHeader`, little-endian: pc_digest
|
||||
# (u64), instance_id (u32), test_i (u32), len (u32). The input follows it.
|
||||
INPUT_HEADER = struct.Struct("<QIII")
|
||||
INPUT_FILE_NAME = re.compile(r"in[0-9a-f]+")
|
||||
|
||||
DEFAULT_BUDGET_SECONDS = 20 * 60
|
||||
# The first round also compiles the instrumented test binary.
|
||||
DEFAULT_ROUND_TIMEOUT_SECONDS = 10 * 60
|
||||
DEFAULT_REPLAY_TIMEOUT_SECONDS = 10 * 60
|
||||
|
||||
STATUS_OK = "ok"
|
||||
STATUS_FAILURE = "failure"
|
||||
STATUS_HANG = "hang"
|
||||
STATUS_BUILD_ERROR = "build-error"
|
||||
STATUS_MISSING_TEST = "missing-test"
|
||||
|
||||
|
||||
class FuzzNightlyError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Target:
|
||||
name: str
|
||||
module: str
|
||||
test: str
|
||||
iterations: int
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class Outcome:
|
||||
"""What one `zig build` invocation printed. `returncode` is None when the
|
||||
invocation ran past its timeout and was killed."""
|
||||
|
||||
output: str
|
||||
returncode: int | None
|
||||
|
||||
|
||||
Runner = Callable[[Sequence[str], pathlib.Path, float], Outcome]
|
||||
|
||||
|
||||
def load_targets(root: pathlib.Path = REPO_ROOT) -> list[Target]:
|
||||
data = json.loads((root / REGISTRY_PATH).read_text())
|
||||
targets = [
|
||||
Target(
|
||||
name=entry["name"],
|
||||
module=entry["module"],
|
||||
test=entry["test"],
|
||||
iterations=int(entry["iterations"]),
|
||||
)
|
||||
for entry in data["targets"]
|
||||
]
|
||||
names = [target.name for target in targets]
|
||||
if len(set(names)) != len(names):
|
||||
raise FuzzNightlyError(f"{REGISTRY_PATH} lists a target twice")
|
||||
for target in targets:
|
||||
if target.iterations <= 0:
|
||||
raise FuzzNightlyError(f"{target.name}: iterations must be positive")
|
||||
return targets
|
||||
|
||||
|
||||
def find_target(targets: Sequence[Target], name: str) -> Target:
|
||||
for target in targets:
|
||||
if target.name == name:
|
||||
return target
|
||||
raise FuzzNightlyError(f"unknown fuzz target {name!r}; see {REGISTRY_PATH}")
|
||||
|
||||
|
||||
def matrix_names(targets: Sequence[Target], only: str = "") -> list[str]:
|
||||
if only:
|
||||
return [find_target(targets, only).name]
|
||||
return [target.name for target in targets]
|
||||
|
||||
|
||||
def fuzz_command(name: str, iterations: int | None) -> list[str]:
|
||||
"""`zig build fuzz` for one target. Without iterations it only replays the
|
||||
target's saved inputs."""
|
||||
command = ["zig", "build", "fuzz", "-Doptimize=ReleaseSafe", f"-Dfuzz-target={name}"]
|
||||
if iterations is not None:
|
||||
command.append(f"--fuzz={iterations}")
|
||||
return command
|
||||
|
||||
|
||||
def run_process(command: Sequence[str], cwd: pathlib.Path, timeout: float) -> Outcome:
|
||||
"""Runs `command` in its own process group, so a timeout also stops the
|
||||
fuzzer processes the build runner started."""
|
||||
process = subprocess.Popen(
|
||||
list(command),
|
||||
cwd=cwd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
errors="replace",
|
||||
start_new_session=True,
|
||||
)
|
||||
try:
|
||||
output, _ = process.communicate(timeout=timeout)
|
||||
return Outcome(output, process.returncode)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
output, _ = process.communicate()
|
||||
return Outcome(output, None)
|
||||
|
||||
|
||||
def classify(outcome: Outcome, test: str) -> str:
|
||||
if outcome.returncode is None:
|
||||
return STATUS_HANG
|
||||
lines = outcome.output.splitlines()
|
||||
if any(line.startswith("error: test '") for line in lines):
|
||||
return STATUS_FAILURE
|
||||
if outcome.returncode != 0:
|
||||
return STATUS_BUILD_ERROR
|
||||
if not any(line.startswith('Fuzz test: "') and test in line for line in lines):
|
||||
return STATUS_MISSING_TEST
|
||||
return STATUS_OK
|
||||
|
||||
|
||||
def clear_fuzzer_inputs(cache: pathlib.Path) -> None:
|
||||
"""Removes inputs left by earlier runs, keeping the fuzzer's corpus."""
|
||||
inputs = cache / "f"
|
||||
if not inputs.is_dir():
|
||||
return
|
||||
for path in inputs.iterdir():
|
||||
if path.is_file() and (path.name == "crash" or INPUT_FILE_NAME.fullmatch(path.name)):
|
||||
path.unlink()
|
||||
|
||||
|
||||
def recover_inputs(cache: pathlib.Path) -> list[bytes]:
|
||||
"""Every input the fuzzer was running when it stopped, without duplicates,
|
||||
starting with the one the build runner saved when it is not empty."""
|
||||
inputs_dir = cache / "f"
|
||||
found: list[bytes] = []
|
||||
crash = inputs_dir / "crash"
|
||||
if crash.is_file() and crash.stat().st_size > 0:
|
||||
found.append(crash.read_bytes())
|
||||
if inputs_dir.is_dir():
|
||||
for path in sorted(inputs_dir.iterdir()):
|
||||
if not path.is_file() or not INPUT_FILE_NAME.fullmatch(path.name):
|
||||
continue
|
||||
data = path.read_bytes()
|
||||
if len(data) < INPUT_HEADER.size:
|
||||
continue
|
||||
length = INPUT_HEADER.unpack_from(data)[3]
|
||||
payload = data[INPUT_HEADER.size : INPUT_HEADER.size + length]
|
||||
if len(payload) == length:
|
||||
found.append(payload)
|
||||
unique: list[bytes] = []
|
||||
for data in found:
|
||||
if data not in unique:
|
||||
unique.append(data)
|
||||
return unique
|
||||
|
||||
|
||||
def input_digest(data: bytes) -> str:
|
||||
return hashlib.sha256(data).hexdigest()[:12]
|
||||
|
||||
|
||||
def confirm_reproducer(
|
||||
target: Target,
|
||||
root: pathlib.Path,
|
||||
candidates: Sequence[bytes],
|
||||
timeout: float,
|
||||
runner: Runner,
|
||||
) -> bytes | None:
|
||||
"""Replays each candidate as a saved input of the target and returns the
|
||||
first one that fails or hangs. Returns None when the target's replay
|
||||
already fails without a candidate, since no candidate can then be blamed."""
|
||||
if not candidates:
|
||||
return None
|
||||
if runner(fuzz_command(target.name, None), root, timeout).returncode != 0:
|
||||
return None
|
||||
folder = root / CORPUS_PATH / target.name
|
||||
# Deepest first, so the cleanup below can remove each one once it is empty.
|
||||
created = [path for path in (folder, *folder.parents) if path.is_relative_to(root) and not path.exists()]
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
for data in candidates:
|
||||
path = folder / f"nightly-{input_digest(data)}"
|
||||
path.write_bytes(data)
|
||||
try:
|
||||
outcome = runner(fuzz_command(target.name, None), root, timeout)
|
||||
finally:
|
||||
path.unlink()
|
||||
if outcome.returncode != 0:
|
||||
return data
|
||||
finally:
|
||||
for path in created:
|
||||
if path.is_dir() and not any(path.iterdir()):
|
||||
path.rmdir()
|
||||
return None
|
||||
|
||||
|
||||
def fuzz_target(
|
||||
target: Target,
|
||||
root: pathlib.Path,
|
||||
out_dir: pathlib.Path,
|
||||
*,
|
||||
budget: float = DEFAULT_BUDGET_SECONDS,
|
||||
round_timeout: float = DEFAULT_ROUND_TIMEOUT_SECONDS,
|
||||
replay_timeout: float = DEFAULT_REPLAY_TIMEOUT_SECONDS,
|
||||
runner: Runner = run_process,
|
||||
clock: Callable[[], float] = time.monotonic,
|
||||
) -> dict:
|
||||
"""Fuzzes `target` until the budget is spent or a round does not end
|
||||
cleanly, then writes `fuzz.log`, `result.json`, and any confirmed
|
||||
reproducer to `out_dir`. Returns the result."""
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
cache = root / ".zig-cache"
|
||||
clear_fuzzer_inputs(cache)
|
||||
deadline = clock() + budget
|
||||
log: list[str] = []
|
||||
rounds = 0
|
||||
status = STATUS_OK
|
||||
while True:
|
||||
outcome = runner(fuzz_command(target.name, target.iterations), root, round_timeout)
|
||||
rounds += 1
|
||||
status = classify(outcome, target.test)
|
||||
log.append(f"=== round {rounds}: {status} ===\n{outcome.output}")
|
||||
print(f"{target.name}: round {rounds}: {status}", flush=True)
|
||||
if status != STATUS_OK or clock() >= deadline:
|
||||
break
|
||||
(out_dir / "fuzz.log").write_text("\n".join(log))
|
||||
|
||||
result: dict = {
|
||||
"target": target.name,
|
||||
"test": target.test,
|
||||
"status": status,
|
||||
"rounds": rounds,
|
||||
"round_timeout_seconds": round_timeout,
|
||||
"reproducer": None,
|
||||
}
|
||||
if status in (STATUS_FAILURE, STATUS_HANG):
|
||||
reproducer = confirm_reproducer(
|
||||
target, root, recover_inputs(cache), replay_timeout, runner
|
||||
)
|
||||
if reproducer is not None:
|
||||
name = f"{target.name}-{input_digest(reproducer)}"
|
||||
(out_dir / name).write_bytes(reproducer)
|
||||
result["reproducer"] = name
|
||||
(out_dir / "result.json").write_text(json.dumps(result, indent=2) + "\n")
|
||||
return result
|
||||
|
||||
|
||||
def issue_title(name: str) -> str:
|
||||
return f"Nightly fuzz failure: {name}"
|
||||
|
||||
|
||||
def issue_body(result: dict, run_url: str, commit: str) -> str:
|
||||
name = result["target"]
|
||||
status = result["status"]
|
||||
what = {
|
||||
STATUS_FAILURE: f"found an input that fails `{result['test']}`",
|
||||
STATUS_HANG: (
|
||||
f"found an input that hangs `{result['test']}`: a round ran past "
|
||||
f"{int(result['round_timeout_seconds'])} seconds"
|
||||
),
|
||||
STATUS_BUILD_ERROR: "could not build or start the fuzz target",
|
||||
STATUS_MISSING_TEST: (
|
||||
f"ran no fuzz test named `{result['test']}`, so the entry in "
|
||||
f"`{REGISTRY_PATH}` no longer matches a test"
|
||||
),
|
||||
}.get(status, "failed before it started fuzzing")
|
||||
lines = [
|
||||
f"Nightly fuzzing of `{name}` {what}.",
|
||||
"",
|
||||
f"- Run: {run_url}",
|
||||
f"- Commit: `{commit}`",
|
||||
f"- Rounds: {result['rounds']}",
|
||||
]
|
||||
reproducer = result.get("reproducer")
|
||||
if reproducer:
|
||||
lines += [
|
||||
f"- Reproducer: `{reproducer}` in the `fuzz-{name}` artifact",
|
||||
"",
|
||||
"To reproduce:",
|
||||
"",
|
||||
f"1. Download the `fuzz-{name}` artifact from the run.",
|
||||
f"2. Copy `{reproducer}` into `{CORPUS_PATH}/{name}/`.",
|
||||
f"3. Run `zig build fuzz -Dfuzz-target={name}`. It builds in Debug, which adds allocation stack traces to leak reports.",
|
||||
"4. Fix the bug and keep the file in the fix PR, so `zig build test` replays it on every PR.",
|
||||
]
|
||||
elif status in (STATUS_FAILURE, STATUS_HANG):
|
||||
lines += [
|
||||
"- Reproducer: none of the recovered inputs reproduced it on replay. `fuzz.log` in the artifact has the failure.",
|
||||
]
|
||||
else:
|
||||
lines += ["", "`fuzz.log` in the artifact has the output."]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def read_result(out_dir: pathlib.Path, target: str) -> dict:
|
||||
path = out_dir / "result.json"
|
||||
if path.is_file():
|
||||
return json.loads(path.read_text())
|
||||
return {"target": target, "test": "", "status": "setup", "rounds": 0, "reproducer": None}
|
||||
|
||||
|
||||
GhRunner = Callable[[Sequence[str], str | None], str]
|
||||
|
||||
|
||||
def run_gh(args: Sequence[str], stdin: str | None) -> str:
|
||||
completed = subprocess.run(
|
||||
["gh", *args], input=stdin, text=True, capture_output=True, check=True
|
||||
)
|
||||
return completed.stdout
|
||||
|
||||
|
||||
def report(
|
||||
result: dict,
|
||||
*,
|
||||
repo: str,
|
||||
run_url: str,
|
||||
commit: str,
|
||||
gh: GhRunner = run_gh,
|
||||
) -> str:
|
||||
"""Comments on the target's open issue, or opens one. Returns the action."""
|
||||
title = issue_title(result["target"])
|
||||
body = issue_body(result, run_url, commit)
|
||||
listed = gh(
|
||||
[
|
||||
"issue", "list", "--repo", repo, "--state", "open",
|
||||
"--search", f'"{title}" in:title', "--json", "number,title", "--limit", "100",
|
||||
],
|
||||
None,
|
||||
)
|
||||
for issue in json.loads(listed or "[]"):
|
||||
if issue["title"] == title:
|
||||
gh(["issue", "comment", str(issue["number"]), "--repo", repo, "--body-file", "-"], body)
|
||||
return f"commented on #{issue['number']}"
|
||||
gh(["issue", "create", "--repo", repo, "--title", title, "--body-file", "-"], body)
|
||||
return "opened an issue"
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(prog="python3 -m scripts.fuzz_nightly")
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
matrix = commands.add_parser("matrix", help="print the job matrix as a GitHub output line")
|
||||
matrix.add_argument("--only", default="", help="limit the matrix to one target")
|
||||
|
||||
run = commands.add_parser("run", help="fuzz one target")
|
||||
run.add_argument("--target", required=True)
|
||||
run.add_argument("--out", required=True, type=pathlib.Path)
|
||||
run.add_argument("--budget-seconds", type=float, default=DEFAULT_BUDGET_SECONDS)
|
||||
run.add_argument("--round-timeout-seconds", type=float, default=DEFAULT_ROUND_TIMEOUT_SECONDS)
|
||||
|
||||
report_parser = commands.add_parser("report", help="open or update the target's issue")
|
||||
report_parser.add_argument("--target", required=True)
|
||||
report_parser.add_argument("--out", required=True, type=pathlib.Path)
|
||||
report_parser.add_argument("--repo", required=True)
|
||||
report_parser.add_argument("--run-url", required=True)
|
||||
report_parser.add_argument("--commit", required=True)
|
||||
|
||||
args = parser.parse_args(argv)
|
||||
try:
|
||||
if args.command == "matrix":
|
||||
names = matrix_names(load_targets(), args.only)
|
||||
print(f"targets={json.dumps(names)}")
|
||||
return 0
|
||||
if args.command == "run":
|
||||
target = find_target(load_targets(), args.target)
|
||||
result = fuzz_target(
|
||||
target,
|
||||
REPO_ROOT,
|
||||
args.out,
|
||||
budget=args.budget_seconds,
|
||||
round_timeout=args.round_timeout_seconds,
|
||||
)
|
||||
return 0 if result["status"] == STATUS_OK else 1
|
||||
result = read_result(args.out, args.target)
|
||||
print(report(result, repo=args.repo, run_url=args.run_url, commit=args.commit))
|
||||
return 0
|
||||
except FuzzNightlyError as error:
|
||||
print(f"error: {error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,270 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import pathlib
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from scripts import fuzz_nightly
|
||||
from scripts.fuzz_nightly import (
|
||||
INPUT_HEADER,
|
||||
Outcome,
|
||||
Target,
|
||||
classify,
|
||||
confirm_reproducer,
|
||||
fuzz_target,
|
||||
issue_body,
|
||||
load_targets,
|
||||
matrix_names,
|
||||
recover_inputs,
|
||||
report,
|
||||
)
|
||||
|
||||
TARGET = Target(
|
||||
name="sse_framing",
|
||||
module="fx",
|
||||
test="provider framing fuzzes chunk-invariant event data",
|
||||
iterations=1000,
|
||||
)
|
||||
|
||||
CLEAN_ROUND = """\
|
||||
======= FUZZING REPORT =======
|
||||
Step: run test fuzz
|
||||
Fuzz test: "gateway.sse.test.provider framing fuzzes chunk-invariant event data" (153f9ee4d2a0f02a)
|
||||
Runs: 0 -> 4113
|
||||
==============================
|
||||
"""
|
||||
|
||||
# What Zig 0.17 prints, with exit status 0, when fuzzing finds a failure.
|
||||
FAILED_ROUND = (
|
||||
"error: test 'gateway.sse.test.provider framing fuzzes chunk-invariant event data' "
|
||||
"exited with code 1; input saved to '.zig-cache/f/crash'\n" + CLEAN_ROUND
|
||||
)
|
||||
|
||||
|
||||
def write_input_file(path: pathlib.Path, payload: bytes, padding: int = 64) -> None:
|
||||
header = INPUT_HEADER.pack(0x153F9EE4D2A0F02A, 0, 0, len(payload))
|
||||
path.write_bytes(header + payload + b"\x00" * padding)
|
||||
|
||||
|
||||
class FakeRunner:
|
||||
"""Answers `zig build fuzz` calls from a script of outcomes. Replays fail
|
||||
when the target's corpus folder holds the bad input."""
|
||||
|
||||
def __init__(self, rounds: list[Outcome], bad_input: bytes | None = None) -> None:
|
||||
self.rounds = rounds
|
||||
self.bad_input = bad_input
|
||||
self.calls: list[list[str]] = []
|
||||
|
||||
def __call__(self, command, cwd: pathlib.Path, timeout: float) -> Outcome:
|
||||
self.calls.append(list(command))
|
||||
if any(arg.startswith("--fuzz=") for arg in command):
|
||||
return self.rounds.pop(0)
|
||||
folder = cwd / "tests" / "fuzz" / "corpus" / TARGET.name
|
||||
saved = [path.read_bytes() for path in folder.iterdir()] if folder.is_dir() else []
|
||||
if self.bad_input is not None and self.bad_input in saved:
|
||||
return Outcome("error: replay failed\n", 1)
|
||||
return Outcome("", 0)
|
||||
|
||||
|
||||
class RegistryTests(unittest.TestCase):
|
||||
def test_registry_names_are_unique_and_matrix_lists_them(self) -> None:
|
||||
targets = load_targets()
|
||||
|
||||
names = matrix_names(targets)
|
||||
|
||||
self.assertEqual(len(names), len(set(names)))
|
||||
self.assertIn("sse_framing", names)
|
||||
self.assertEqual(["sse_framing"], matrix_names(targets, "sse_framing"))
|
||||
|
||||
def test_unknown_target_is_an_error(self) -> None:
|
||||
with self.assertRaises(fuzz_nightly.FuzzNightlyError):
|
||||
matrix_names(load_targets(), "no_such_target")
|
||||
|
||||
def test_every_target_names_one_fuzz_test_its_binary_reaches(self) -> None:
|
||||
src = fuzz_nightly.REPO_ROOT / "src"
|
||||
fuzz_root = (src / "fuzz_targets.zig").read_text()
|
||||
session_manager = src / "core" / "session_manager"
|
||||
sources = {path: path.read_text() for path in src.rglob("*.zig")}
|
||||
for target in load_targets():
|
||||
with self.subTest(target=target.name):
|
||||
declaration = f'test "{target.test}" {{'
|
||||
files = [path for path, text in sources.items() if declaration in text]
|
||||
self.assertEqual(1, len(files), f"{target.name}: one file must declare {declaration!r}")
|
||||
body = sources[files[0]].split(declaration, 1)[1].split('\ntest "', 1)[0]
|
||||
self.assertIn("testing.fuzz(", body)
|
||||
in_session_manager = files[0].is_relative_to(session_manager)
|
||||
self.assertEqual(target.module == "session_manager", in_session_manager)
|
||||
if target.module == "fx":
|
||||
self.assertIn(f'@import("{files[0].relative_to(src).as_posix()}")', fuzz_root)
|
||||
|
||||
|
||||
class ClassifyTests(unittest.TestCase):
|
||||
def test_clean_round_is_ok(self) -> None:
|
||||
self.assertEqual("ok", classify(Outcome(CLEAN_ROUND, 0), TARGET.test))
|
||||
|
||||
def test_failure_is_found_even_with_exit_status_zero(self) -> None:
|
||||
self.assertEqual("failure", classify(Outcome(FAILED_ROUND, 0), TARGET.test))
|
||||
|
||||
def test_timeout_is_a_hang(self) -> None:
|
||||
self.assertEqual("hang", classify(Outcome(CLEAN_ROUND, None), TARGET.test))
|
||||
|
||||
def test_nonzero_exit_without_a_test_failure_is_a_build_error(self) -> None:
|
||||
self.assertEqual("build-error", classify(Outcome("error: compile failed\n", 1), TARGET.test))
|
||||
|
||||
def test_round_without_the_registered_test_is_reported(self) -> None:
|
||||
self.assertEqual("missing-test", classify(Outcome("Build Summary: 1/1\n", 0), TARGET.test))
|
||||
|
||||
|
||||
class RecoverInputsTests(unittest.TestCase):
|
||||
def test_reads_inputs_from_fuzzer_files_when_the_saved_crash_is_empty(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
inputs = pathlib.Path(tmp) / "f"
|
||||
inputs.mkdir()
|
||||
(inputs / "crash").write_bytes(b"")
|
||||
write_input_file(inputs / "in0", b"\x4d\x00\x00\x00bad")
|
||||
write_input_file(inputs / "in1", b"\x4d\x00\x00\x00bad")
|
||||
write_input_file(inputs / "in2", b"other")
|
||||
(inputs / "d1e77c9a9250cf56").mkdir()
|
||||
|
||||
self.assertEqual([b"\x4d\x00\x00\x00bad", b"other"], recover_inputs(pathlib.Path(tmp)))
|
||||
|
||||
def test_a_non_empty_saved_crash_comes_first(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
inputs = pathlib.Path(tmp) / "f"
|
||||
inputs.mkdir()
|
||||
(inputs / "crash").write_bytes(b"saved")
|
||||
write_input_file(inputs / "in0", b"other")
|
||||
|
||||
self.assertEqual([b"saved", b"other"], recover_inputs(pathlib.Path(tmp)))
|
||||
|
||||
|
||||
class ConfirmReproducerTests(unittest.TestCase):
|
||||
def test_returns_the_candidate_that_fails_on_replay_and_cleans_up(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = pathlib.Path(tmp)
|
||||
runner = FakeRunner([], bad_input=b"bad")
|
||||
|
||||
found = confirm_reproducer(TARGET, root, [b"good", b"bad"], 60, runner)
|
||||
|
||||
self.assertEqual(b"bad", found)
|
||||
self.assertEqual([], list(root.iterdir()))
|
||||
|
||||
def test_keeps_corpus_folders_that_already_existed(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = pathlib.Path(tmp)
|
||||
existing = root / "tests" / "fuzz" / "corpus"
|
||||
existing.mkdir(parents=True)
|
||||
|
||||
confirm_reproducer(TARGET, root, [b"bad"], 60, FakeRunner([], bad_input=b"bad"))
|
||||
|
||||
self.assertEqual([], list(existing.iterdir()))
|
||||
|
||||
def test_no_candidate_is_blamed_when_the_replay_already_fails(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
def always_fails(command, cwd, timeout):
|
||||
return Outcome("", 1)
|
||||
|
||||
self.assertIsNone(confirm_reproducer(TARGET, pathlib.Path(tmp), [b"bad"], 60, always_fails))
|
||||
|
||||
|
||||
class FuzzTargetTests(unittest.TestCase):
|
||||
def test_runs_rounds_until_the_budget_is_spent(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = pathlib.Path(tmp)
|
||||
ticks = iter([0, 10, 20, 30])
|
||||
runner = FakeRunner([Outcome(CLEAN_ROUND, 0)] * 3)
|
||||
|
||||
result = fuzz_target(
|
||||
TARGET, root, root / "out", budget=25, runner=runner, clock=lambda: next(ticks)
|
||||
)
|
||||
|
||||
self.assertEqual("ok", result["status"])
|
||||
self.assertEqual(3, result["rounds"])
|
||||
self.assertEqual("ok", json.loads((root / "out" / "result.json").read_text())["status"])
|
||||
|
||||
def test_failure_writes_the_confirmed_reproducer(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = pathlib.Path(tmp)
|
||||
inputs = root / ".zig-cache" / "f"
|
||||
inputs.mkdir(parents=True)
|
||||
bad = b"\x4d\x00\x00\x00bad"
|
||||
replay = FakeRunner([], bad_input=bad)
|
||||
|
||||
def runner(command, cwd, timeout):
|
||||
if any(arg.startswith("--fuzz=") for arg in command):
|
||||
write_input_file(inputs / "in0", bad)
|
||||
return Outcome(FAILED_ROUND, 0)
|
||||
return replay(command, cwd, timeout)
|
||||
|
||||
result = fuzz_target(TARGET, root, root / "out", runner=runner, clock=lambda: 0)
|
||||
|
||||
self.assertEqual("failure", result["status"])
|
||||
self.assertEqual(bad, (root / "out" / result["reproducer"]).read_bytes())
|
||||
|
||||
def test_stale_inputs_from_earlier_runs_are_removed_first(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = pathlib.Path(tmp)
|
||||
inputs = root / ".zig-cache" / "f"
|
||||
inputs.mkdir(parents=True)
|
||||
write_input_file(inputs / "in0", b"stale")
|
||||
(inputs / "crash").write_bytes(b"stale")
|
||||
corpus = inputs / "153f9ee4d2a0f02a"
|
||||
corpus.mkdir()
|
||||
|
||||
fuzz_target(
|
||||
TARGET, root, root / "out", budget=0, runner=FakeRunner([Outcome(CLEAN_ROUND, 0)]), clock=lambda: 0
|
||||
)
|
||||
|
||||
self.assertEqual([corpus], list(inputs.iterdir()))
|
||||
|
||||
|
||||
class ReportTests(unittest.TestCase):
|
||||
RESULT = {
|
||||
"target": "sse_framing",
|
||||
"test": TARGET.test,
|
||||
"status": "failure",
|
||||
"rounds": 2,
|
||||
"round_timeout_seconds": 600,
|
||||
"reproducer": "sse_framing-0123456789ab",
|
||||
}
|
||||
|
||||
def test_body_explains_how_to_reproduce(self) -> None:
|
||||
body = issue_body(self.RESULT, "https://example.invalid/run/1", "abc123")
|
||||
|
||||
self.assertIn("fails `provider framing", body)
|
||||
self.assertIn("tests/fuzz/corpus/sse_framing/", body)
|
||||
self.assertIn("zig build fuzz -Dfuzz-target=sse_framing", body)
|
||||
|
||||
def test_comments_on_the_open_issue_with_the_same_title(self) -> None:
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def gh(args, stdin):
|
||||
calls.append(list(args))
|
||||
if args[:2] == ["issue", "list"]:
|
||||
return json.dumps([
|
||||
{"number": 7, "title": "Nightly fuzz failure: sse_framing_v2"},
|
||||
{"number": 9, "title": "Nightly fuzz failure: sse_framing"},
|
||||
])
|
||||
return ""
|
||||
|
||||
action = report(self.RESULT, repo="vercel-labs/fx", run_url="u", commit="c", gh=gh)
|
||||
|
||||
self.assertEqual("commented on #9", action)
|
||||
self.assertEqual(["issue", "comment", "9"], calls[-1][:3])
|
||||
|
||||
def test_opens_an_issue_when_none_is_open(self) -> None:
|
||||
calls: list[list[str]] = []
|
||||
|
||||
def gh(args, stdin):
|
||||
calls.append(list(args))
|
||||
return "[]" if args[:2] == ["issue", "list"] else ""
|
||||
|
||||
action = report(self.RESULT, repo="vercel-labs/fx", run_url="u", commit="c", gh=gh)
|
||||
|
||||
self.assertEqual("opened an issue", action)
|
||||
self.assertIn("Nightly fuzz failure: sse_framing", calls[-1])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -560,12 +560,12 @@ fn fuzzParse(_: void, smith: *std.testing.Smith) !void {
|
||||
|
||||
test "ACP MCP parser handles fuzzed request bytes" {
|
||||
try std.testing.fuzz({}, fuzzParse, .{
|
||||
.corpus = &.{
|
||||
.corpus = &(.{
|
||||
"",
|
||||
"{}",
|
||||
"{\"mcpServers\":[]}",
|
||||
"{\"mcpServers\":[{\"name\":\"fixture\",\"command\":\"/bin/echo\",\"args\":[],\"env\":[]}]}",
|
||||
"{\"mcpServers\":[{\"type\":\"http\",\"name\":\"fixture\",\"url\":\"http://localhost:4321/mcp\",\"headers\":[]}]}",
|
||||
},
|
||||
} ++ @import("fuzz_corpus").acp_mcp_servers.*),
|
||||
});
|
||||
}
|
||||
@@ -1017,11 +1017,11 @@ fn fuzz_vision_json(_: void, smith: *std.testing.Smith) !void {
|
||||
|
||||
test "Vision JSON parsers tolerate arbitrary bounded bytes" {
|
||||
try std.testing.fuzz({}, fuzz_vision_json, .{
|
||||
.corpus = &.{
|
||||
.corpus = &(.{
|
||||
"{\"image_ids\":[1],\"focus\":\"inspect\"}",
|
||||
"{\"images\":[{\"image_id\":1,\"status\":\"failed\",\"error\":\"vision_unavailable\"}]}",
|
||||
"{",
|
||||
},
|
||||
} ++ @import("fuzz_corpus").vision_json.*),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -544,7 +544,7 @@ test "encoded image dimensions agree with raw headers at every truncation" {
|
||||
|
||||
test "image dimension parsing stays bounded on arbitrary bytes" {
|
||||
try std.testing.fuzz({}, fuzzImageDimensions, .{
|
||||
.corpus = &.{
|
||||
.corpus = &(.{
|
||||
&testPngHeader(3420, 2224),
|
||||
&testJpeg(2001, 1),
|
||||
&testGif(1, 1),
|
||||
@@ -556,7 +556,7 @@ test "image dimension parsing stays bounded on arbitrary bytes" {
|
||||
"\x89PNG\r\n\x1a\n\x00\x00\x00\x00IHDR\xff\xff\xff\xff\xff\xff\xff\xff",
|
||||
// RIFF header naming WebP with no chunk.
|
||||
"RIFF\x00\x00\x00\x00WEBP",
|
||||
},
|
||||
} ++ @import("fuzz_corpus").image_dimensions.*),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -2143,12 +2143,12 @@ fn fuzzElicitationJson(_: void, smith: *std.testing.Smith) !void {
|
||||
|
||||
test "elicitation request and response parsers stay bounded under fuzzed bytes" {
|
||||
try std.testing.fuzz({}, fuzzElicitationJson, .{
|
||||
.corpus = &.{
|
||||
.corpus = &(.{
|
||||
"{}",
|
||||
"{\"mode\":\"form\",\"requestedSchema\":{}}",
|
||||
"{\"action\":\"accept\",\"content\":{}}",
|
||||
"{\"mode\":\"url\",\"url\":\"https://example.test\"}",
|
||||
"{\"jsonrpc\":\"2.0\",\"method\":\"notifications/elicitation/complete\",\"params\":{\"elicitationId\":\"id\"}}",
|
||||
},
|
||||
} ++ @import("fuzz_corpus").mcp_elicitation.*),
|
||||
});
|
||||
}
|
||||
@@ -5951,7 +5951,7 @@ test "an old session's usage field is skipped in its place, and the rest loads"
|
||||
}
|
||||
|
||||
test "durable session optional fields handle fuzzed ownership paths" {
|
||||
try std.testing.fuzz({}, fuzzDurableSessionOptionalFields, .{ .corpus = &.{
|
||||
try std.testing.fuzz({}, fuzzDurableSessionOptionalFields, .{ .corpus = &(.{
|
||||
"{\"id\":\"session\",\"origin_workspace_root\":\"/tmp/origin\",\"workspace_root\":\"/tmp/current\",\"created_at_ms\":1,\"updated_at_ms\":1,\"conversation_language\":\"en\",\"preferences\":{\"model\":\"test/model\",\"effort\":\"auto\",\"fast_mode\":false},\"history\":[],\"total_input_tokens\":0,\"total_output_tokens\":0}",
|
||||
"{\"id\":\"session\",\"origin_workspace_root\":\"/tmp/origin\",\"workspace_root\":\"/tmp/current\",\"created_at_ms\":1,\"updated_at_ms\":1,\"conversation_language\":\"en\",\"preferences\":{\"model\":\"test/model\",\"effort\":\"auto\",\"fast_mode\":false},\"history\":[],\"total_input_tokens\":0,\"total_output_tokens\":0,\"context_history_start\":0,\"last_subagent_work_id\":\"work\"}",
|
||||
"{\"id\":\"session\",\"origin_workspace_root\":\"/tmp/origin\",\"workspace_root\":\"/tmp/current\",\"created_at_ms\":1,\"updated_at_ms\":1,\"conversation_language\":\"en\",\"preferences\":{\"model\":\"test/model\",\"effort\":\"auto\",\"fast_mode\":false},\"history\":[],\"total_input_tokens\":0,\"total_output_tokens\":0,\"last_subagent_work_id\":\"work\",\"context_history_start\":0}",
|
||||
@@ -5960,17 +5960,17 @@ test "durable session optional fields handle fuzzed ownership paths" {
|
||||
"{\"last_subagent_work_id\":null}",
|
||||
"{\"usage\":null,\"last_subagent_work_id\":\"work\"}",
|
||||
"{\"last_subagent_work_id\":\"work\",\"usage\":null}",
|
||||
} });
|
||||
} ++ @import("fuzz_corpus").session_optional_fields.*) });
|
||||
}
|
||||
|
||||
test "per-turn work provenance parser handles fuzzed ownership paths" {
|
||||
try std.testing.fuzz({}, fuzzWorkProvenanceHistoryTurn, .{ .corpus = &.{
|
||||
try std.testing.fuzz({}, fuzzWorkProvenanceHistoryTurn, .{ .corpus = &(.{
|
||||
"{\"kind\":\"assistant\",\"user\":{\"text\":\"prompt\",\"images\":[],\"work_id\":\"work-1\"},\"assistant\":\"reply\",\"execution\":{\"schema_version\":3,\"tool_steps\":[],\"files\":[]}}",
|
||||
"{\"kind\":\"assistant\",\"user\":{\"text\":\"prompt\",\"images\":[]},\"assistant\":\"reply\",\"execution\":{\"schema_version\":3,\"tool_steps\":[],\"files\":[]}}",
|
||||
"{\"kind\":\"assistant\",\"user\":{\"text\":\"prompt\",\"images\":[],\"work_id\":\"\"}}",
|
||||
"",
|
||||
"{}",
|
||||
} });
|
||||
} ++ @import("fuzz_corpus").session_work_provenance.*) });
|
||||
}
|
||||
|
||||
fn fuzzWorkProvenanceHistoryTurn(_: void, smith: *std.testing.Smith) !void {
|
||||
|
||||
@@ -3171,12 +3171,12 @@ test "conversation frame rejects an oversized diff content handle" {
|
||||
|
||||
test "conversation frame decoder stays bounded under fuzzed bytes" {
|
||||
try std.testing.fuzz({}, fuzzConversationFrame, .{
|
||||
.corpus = &.{
|
||||
.corpus = &(.{
|
||||
"",
|
||||
"{}\n",
|
||||
"{\"schema_version\":1}\n",
|
||||
"{\"schema_version\":1,\"seq\":1,\"timestamp_ms\":0,\"event\":{\"assistant\":{\"text\":\"ok\"}}}\n",
|
||||
},
|
||||
} ++ @import("fuzz_corpus").session_conversation_frame.*),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -773,10 +773,10 @@ test "any prefix of a valid log scans to its last complete line" {
|
||||
}
|
||||
|
||||
test "scanning never panics on arbitrary bytes" {
|
||||
try testing.fuzz({}, fuzzScan, .{ .corpus = &.{
|
||||
try testing.fuzz({}, fuzzScan, .{ .corpus = &(.{
|
||||
"{\"v\":1,\"seq\":1,\"ts\":0,\"kind\":\"item\",\"crc\":\"00000000\"}\n",
|
||||
"\n\n\n",
|
||||
} });
|
||||
} ++ @import("build_options").fuzz_corpus_session_log_scan.*) });
|
||||
}
|
||||
|
||||
fn fuzzScan(_: void, smith: *testing.Smith) anyerror!void {
|
||||
|
||||
@@ -806,10 +806,10 @@ test "ids follow v1's alphabet and avoid the reserved names" {
|
||||
}
|
||||
|
||||
test "header parsing never panics on arbitrary bytes" {
|
||||
try testing.fuzz({}, fuzzHeader, .{ .corpus = &.{
|
||||
try testing.fuzz({}, fuzzHeader, .{ .corpus = &(.{
|
||||
"{\"v\":1,\"seq\":1,\"ts\":0,\"kind\":\"item\"",
|
||||
"{\"v\":99999999999,\"seq\":1",
|
||||
} });
|
||||
} ++ @import("build_options").fuzz_corpus_session_schema_header.*) });
|
||||
}
|
||||
|
||||
fn fuzzHeader(_: void, smith: *testing.Smith) anyerror!void {
|
||||
|
||||
@@ -828,14 +828,14 @@ test "display unit clipping preserves sequence boundaries" {
|
||||
|
||||
test "fuzz display unit boundaries" {
|
||||
try std.testing.fuzz({}, fuzzDisplayUnitBoundaries, .{
|
||||
.corpus = &.{
|
||||
.corpus = &(.{
|
||||
"",
|
||||
"plain text",
|
||||
"\xff\xf0\x28\x8c\x28",
|
||||
"\u{2600}\u{FE0E}\u{2600}\u{FE0F}",
|
||||
"\u{1F44D}\u{1F3FD}\u{1F1FA}\u{1F1F8}",
|
||||
"#\u{FE0F}\u{20E3}\u{1F469}\u{200D}\u{1F4BB}",
|
||||
},
|
||||
} ++ @import("fuzz_corpus").display_width.*),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -3262,13 +3262,13 @@ fn fuzzToolArgumentDiagnostic(_: void, smith: *std.testing.Smith) anyerror!void
|
||||
|
||||
test "fuzz tool argument diagnostic" {
|
||||
try std.testing.fuzz({}, fuzzToolArgumentDiagnostic, .{
|
||||
.corpus = &.{
|
||||
.corpus = &(.{
|
||||
"{\"request\":{\"task\":\"cut",
|
||||
"{\"a\":1,}",
|
||||
"{\"a\":1,\"a\":2}",
|
||||
"{\"t\":\"\xc3\xa9\xe2\x82",
|
||||
"\xff\x00{",
|
||||
},
|
||||
} ++ @import("fuzz_corpus").tool_argument_diagnostic.*),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -79,8 +79,9 @@ fn isUnsupportedAutoControlOperator(token: command_lex.ArgvToken) bool {
|
||||
std.mem.eql(u8, token.value, "}");
|
||||
}
|
||||
|
||||
fn reversibleAutoStage(raw_tokens: []const command_lex.ArgvToken) bool {
|
||||
var tokens = raw_tokens;
|
||||
/// `tokens` without a trailing unquoted `2>&1`, the one redirection a
|
||||
/// review-free stage may end with.
|
||||
fn withoutStderrToStdout(tokens: []const command_lex.ArgvToken) []const command_lex.ArgvToken {
|
||||
if (tokens.len >= 3 and
|
||||
!tokens[tokens.len - 3].quoted and
|
||||
!tokens[tokens.len - 2].quoted and
|
||||
@@ -89,8 +90,13 @@ fn reversibleAutoStage(raw_tokens: []const command_lex.ArgvToken) bool {
|
||||
std.mem.eql(u8, tokens[tokens.len - 2].value, ">&") and
|
||||
std.mem.eql(u8, tokens[tokens.len - 1].value, "1"))
|
||||
{
|
||||
tokens = tokens[0 .. tokens.len - 3];
|
||||
return tokens[0 .. tokens.len - 3];
|
||||
}
|
||||
return tokens;
|
||||
}
|
||||
|
||||
fn reversibleAutoStage(raw_tokens: []const command_lex.ArgvToken) bool {
|
||||
const tokens = withoutStderrToStdout(raw_tokens);
|
||||
if (tokens.len == 0) return false;
|
||||
for (tokens) |token| {
|
||||
if (isUnsupportedAutoOperator(token) or
|
||||
@@ -1916,3 +1922,525 @@ test "planner bounds ls operands at sixty four" {
|
||||
try accepted.appendSlice(std.testing.allocator, " overflow");
|
||||
try expectApproval(accepted.items, false, .linux, .unsupported_argument);
|
||||
}
|
||||
|
||||
const parity_programs = [_][]const u8{
|
||||
"printf", "pwd", "ls", "wc", "cat", "head", "tail", "grep",
|
||||
"git", "node", "which", "npm", "bun", "pnpm", "yarn", "zig",
|
||||
};
|
||||
|
||||
/// Builtins that a function can replace in every shell here, including bash
|
||||
/// in POSIX mode, which is `/bin/sh` on macOS.
|
||||
const parity_guarded_builtins = [_][]const u8{
|
||||
"kill", "ulimit", "umask", "cd", "command", "builtin", "hash", "alias",
|
||||
"unalias", "wait", "fg", "bg", "jobs", "fc", "read",
|
||||
};
|
||||
|
||||
/// Special builtins that bash and zsh, outside POSIX mode, also let a
|
||||
/// function replace.
|
||||
const parity_guarded_special_builtins = [_][]const u8{
|
||||
"enable", "disable", "disown", "suspend", "history", "logout",
|
||||
"pushd", "popd", "source", "exec", "eval", "trap",
|
||||
};
|
||||
|
||||
const ParityShell = struct {
|
||||
name: []const u8,
|
||||
paths: []const []const u8,
|
||||
flags: []const []const u8,
|
||||
guards_special_builtins: bool,
|
||||
/// Runs before anything else in the script. fx turns off zsh's `equals`
|
||||
/// option before every command (`zsh_command_prelude` in
|
||||
/// `src/core/terminal/shell_resolver.zig`), so zsh here does too.
|
||||
prelude: []const u8 = "",
|
||||
/// Whether this shell stands in for options a user's startup files can
|
||||
/// turn on. fx runs direct read-only commands without a shell, so only
|
||||
/// the commands it hands to a shell are compared with this one.
|
||||
user_options: bool = false,
|
||||
};
|
||||
|
||||
/// `/bin/sh` is dash on Debian and Ubuntu and bash in POSIX mode on macOS,
|
||||
/// so dash is listed on its own as well. fx replays a user's zsh glob
|
||||
/// options, and `extendedglob` makes `^` and `#` glob operators, so zsh also
|
||||
/// runs with it on.
|
||||
const parity_shells = [_]ParityShell{
|
||||
.{ .name = "sh", .paths = &.{"/bin/sh"}, .flags = &.{"-c"}, .guards_special_builtins = false },
|
||||
.{ .name = "dash", .paths = &.{ "/bin/dash", "/usr/bin/dash" }, .flags = &.{"-c"}, .guards_special_builtins = false },
|
||||
.{
|
||||
.name = "bash",
|
||||
.paths = &.{ "/bin/bash", "/usr/bin/bash" },
|
||||
.flags = &.{ "--norc", "--noprofile", "-c" },
|
||||
.guards_special_builtins = true,
|
||||
},
|
||||
.{
|
||||
.name = "zsh",
|
||||
.paths = &.{ "/bin/zsh", "/usr/bin/zsh" },
|
||||
.flags = &.{ "-f", "-c" },
|
||||
.guards_special_builtins = true,
|
||||
.prelude = "\\builtin unsetopt equals\n",
|
||||
},
|
||||
.{
|
||||
.name = "zsh with extendedglob",
|
||||
.paths = &.{ "/bin/zsh", "/usr/bin/zsh" },
|
||||
.flags = &.{ "-f", "-c" },
|
||||
.guards_special_builtins = true,
|
||||
.prelude = "\\builtin unsetopt equals\n\\builtin setopt extendedglob\n",
|
||||
.user_options = true,
|
||||
},
|
||||
};
|
||||
|
||||
const parity_recorder = "#!/bin/sh\nprintf '%s\\0' \"$#\" \"$@\" >> \"$FX_PARITY_LOG\"\n";
|
||||
|
||||
const ParityOrder = enum {
|
||||
/// `&&` stages run one after another.
|
||||
sequence,
|
||||
/// Pipeline stages run at the same time, so their records can interleave.
|
||||
any_order,
|
||||
};
|
||||
|
||||
const ParityExpectation = struct {
|
||||
/// Each stage's words, program first.
|
||||
stages: []const []const []const u8,
|
||||
order: ParityOrder,
|
||||
/// Whether fx hands the command to a shell, rather than running it
|
||||
/// itself.
|
||||
runs_in_shell: bool,
|
||||
};
|
||||
|
||||
const ParityRun = struct {
|
||||
term: std.process.Child.Term,
|
||||
/// One record per program call: its name, then its arguments.
|
||||
records: []const []const []const u8,
|
||||
stderr: []const u8,
|
||||
/// Files the command left in its empty working directory.
|
||||
created: []const []const u8,
|
||||
};
|
||||
|
||||
const ParityMode = enum { must_run_without_review, if_run_without_review };
|
||||
|
||||
/// What fx parsed for each review-free path that accepts `command`. Empty
|
||||
/// when both send it to review. Allocates from `arena`.
|
||||
fn parityExpectations(arena: std.mem.Allocator, command: []const u8) ![]const ParityExpectation {
|
||||
var expectations: std.ArrayList(ParityExpectation) = .empty;
|
||||
if (try knownReversibleAutoCommand(arena, command, false)) {
|
||||
// Splits the way `knownReversibleAutoCommand` does.
|
||||
const argv = try command_lex.tokenize_argv(arena, std.mem.trim(u8, command, " \t"));
|
||||
var stages: std.ArrayList([]const []const u8) = .empty;
|
||||
var start: usize = 0;
|
||||
for (0..argv.tokens.len + 1) |index| {
|
||||
if (index < argv.tokens.len and !isStageSeparator(argv.tokens[index])) continue;
|
||||
try stages.append(arena, try parityWords(arena, withoutStderrToStdout(argv.tokens[start..index])));
|
||||
start = index + 1;
|
||||
}
|
||||
try expectations.append(arena, .{ .stages = stages.items, .order = .sequence, .runs_in_shell = true });
|
||||
}
|
||||
const admission = try planWithGit(arena, command, "/workspace", false, @import("builtin").target.os.tag, test_git);
|
||||
if (admission == .direct_read_only) {
|
||||
// Splits the way `planWithGit` does: one stage per pipeline segment.
|
||||
const segments = try command_lex.pipe_segments(arena, std.mem.trim(u8, command, " \t\n"));
|
||||
var stages: std.ArrayList([]const []const u8) = .empty;
|
||||
for (segments) |segment| {
|
||||
const argv = try command_lex.tokenize_argv(arena, segment.text);
|
||||
try stages.append(arena, try parityWords(arena, argv.tokens));
|
||||
}
|
||||
try expectations.append(arena, .{ .stages = stages.items, .order = .any_order, .runs_in_shell = false });
|
||||
}
|
||||
return expectations.items;
|
||||
}
|
||||
|
||||
fn parityWords(arena: std.mem.Allocator, tokens: []const command_lex.ArgvToken) ![]const []const u8 {
|
||||
const words = try arena.alloc([]const u8, tokens.len);
|
||||
for (tokens, words) |token, *word| word.* = token.value;
|
||||
return words;
|
||||
}
|
||||
|
||||
fn checkShellParity(command: []const u8, mode: ParityMode) !void {
|
||||
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
|
||||
defer arena_state.deinit();
|
||||
const arena = arena_state.allocator();
|
||||
|
||||
const expectations = try parityExpectations(arena, command);
|
||||
if (expectations.len == 0) {
|
||||
if (mode == .if_run_without_review) return;
|
||||
std.debug.print("expected fx to run this command without review: {s}\n", .{command});
|
||||
return error.TestExpectedReviewFreeCommand;
|
||||
}
|
||||
for (parity_shells) |shell| {
|
||||
const path = try parityShellPath(shell) orelse continue;
|
||||
const run = try runParityShell(arena, path, shell, command);
|
||||
for (expectations) |expectation| {
|
||||
if (shell.user_options and !expectation.runs_in_shell) continue;
|
||||
try expectParity(arena, expectation, run, shell.name, command);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Finds the shell, or returns null to skip it. CI installs every one, so a
|
||||
/// missing shell there fails instead of quietly narrowing the check.
|
||||
fn parityShellPath(shell: ParityShell) !?[]const u8 {
|
||||
for (shell.paths) |path| {
|
||||
std.Io.Dir.accessAbsolute(std.testing.io, path, .{}) catch continue;
|
||||
return path;
|
||||
}
|
||||
if (@import("../shared/io.zig").getenv("CI") != null) {
|
||||
std.debug.print("the shell parity tests need {s}\n", .{shell.name});
|
||||
return error.TestShellParityShellMissing;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
fn runParityShell(
|
||||
arena: std.mem.Allocator,
|
||||
shell_path: []const u8,
|
||||
shell: ParityShell,
|
||||
command: []const u8,
|
||||
) !ParityRun {
|
||||
const io = std.testing.io;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
for ([_][]const u8{ "cwd", "empty", "home" }) |name| try tmp.dir.createDirPath(io, name);
|
||||
const root = try @import("../shared/io.zig").dirRealpathAlloc(arena, tmp.dir, ".");
|
||||
const recorder = try std.Io.Dir.path.join(arena, &.{ root, "record" });
|
||||
{
|
||||
var file = try tmp.dir.createFile(io, "record", .{ .truncate = true });
|
||||
defer file.close(io);
|
||||
try file.writeStreamingAll(io, parity_recorder);
|
||||
try file.setPermissions(io, std.Io.File.Permissions.fromMode(0o700));
|
||||
}
|
||||
|
||||
var script: std.ArrayList(u8) = .empty;
|
||||
// Before the functions, since one of them replaces `builtin`.
|
||||
try script.appendSlice(arena, shell.prelude);
|
||||
try appendParityFunctions(&script, arena, recorder, &parity_programs);
|
||||
try appendParityFunctions(&script, arena, recorder, &parity_guarded_builtins);
|
||||
if (shell.guards_special_builtins) {
|
||||
try appendParityFunctions(&script, arena, recorder, &parity_guarded_special_builtins);
|
||||
}
|
||||
try script.appendSlice(arena, command);
|
||||
try script.append(arena, '\n');
|
||||
|
||||
var environ: std.process.Environ.Map = .init(arena);
|
||||
try environ.put("PATH", try std.Io.Dir.path.join(arena, &.{ root, "empty" }));
|
||||
try environ.put("HOME", try std.Io.Dir.path.join(arena, &.{ root, "home" }));
|
||||
try environ.put("FX_PARITY_LOG", try std.Io.Dir.path.join(arena, &.{ root, "log" }));
|
||||
|
||||
var argv: std.ArrayList([]const u8) = .empty;
|
||||
try argv.append(arena, shell_path);
|
||||
try argv.appendSlice(arena, shell.flags);
|
||||
try argv.append(arena, script.items);
|
||||
const result = try std.process.run(arena, io, .{
|
||||
.argv = argv.items,
|
||||
.cwd = .{ .path = try std.Io.Dir.path.join(arena, &.{ root, "cwd" }) },
|
||||
.environ_map = &environ,
|
||||
.timeout = .{ .duration = .{ .raw = .fromSeconds(10), .clock = .awake } },
|
||||
});
|
||||
|
||||
const log = tmp.dir.readFileAlloc(io, "log", arena, .limited(1024 * 1024)) catch |err| switch (err) {
|
||||
error.FileNotFound => "",
|
||||
else => return err,
|
||||
};
|
||||
var created: std.ArrayList([]const u8) = .empty;
|
||||
var cwd = try tmp.dir.openDir(io, "cwd", .{ .iterate = true });
|
||||
defer cwd.close(io);
|
||||
var entries = cwd.iterate();
|
||||
while (try entries.next(io)) |entry| try created.append(arena, try arena.dupe(u8, entry.name));
|
||||
|
||||
return .{
|
||||
.term = result.term,
|
||||
.records = try parseParityLog(arena, log),
|
||||
.stderr = result.stderr,
|
||||
.created = created.items,
|
||||
};
|
||||
}
|
||||
|
||||
fn appendParityFunctions(
|
||||
script: *std.ArrayList(u8),
|
||||
arena: std.mem.Allocator,
|
||||
recorder: []const u8,
|
||||
names: []const []const u8,
|
||||
) !void {
|
||||
for (names) |name| {
|
||||
try script.appendSlice(arena, name);
|
||||
try script.appendSlice(arena, "() { ");
|
||||
try appendShellSingleQuoted(script, arena, recorder);
|
||||
try script.append(arena, ' ');
|
||||
try script.appendSlice(arena, name);
|
||||
try script.appendSlice(arena, " \"$@\"; }\n");
|
||||
}
|
||||
}
|
||||
|
||||
/// Each record is a count, then that many fields: the program's name and
|
||||
/// its arguments. Every field ends with a NUL.
|
||||
fn parseParityLog(arena: std.mem.Allocator, log: []const u8) ![]const []const []const u8 {
|
||||
var records: std.ArrayList([]const []const u8) = .empty;
|
||||
if (log.len == 0) return records.items;
|
||||
if (log[log.len - 1] != 0) return error.TestTruncatedParityLog;
|
||||
var fields = std.mem.splitScalar(u8, log[0 .. log.len - 1], 0);
|
||||
while (fields.next()) |count_text| {
|
||||
const record = try arena.alloc([]const u8, try std.fmt.parseInt(usize, count_text, 10));
|
||||
for (record) |*field| field.* = fields.next() orelse return error.TestTruncatedParityLog;
|
||||
try records.append(arena, record);
|
||||
}
|
||||
return records.items;
|
||||
}
|
||||
|
||||
fn expectParity(
|
||||
arena: std.mem.Allocator,
|
||||
expectation: ParityExpectation,
|
||||
run: ParityRun,
|
||||
shell_name: []const u8,
|
||||
command: []const u8,
|
||||
) !void {
|
||||
var expected = expectation.stages;
|
||||
var actual = run.records;
|
||||
if (expectation.order == .any_order) {
|
||||
expected = try sortedParityRecords(arena, expected);
|
||||
actual = try sortedParityRecords(arena, actual);
|
||||
}
|
||||
const exited_cleanly = switch (run.term) {
|
||||
.exited => |code| code == 0,
|
||||
else => false,
|
||||
};
|
||||
if (exited_cleanly and run.created.len == 0 and sameParityRecords(expected, actual)) return;
|
||||
|
||||
std.debug.print("shell parity mismatch in {s} for: {s}\n", .{ shell_name, command });
|
||||
printParityRecords("fx parsed", expectation.stages);
|
||||
printParityRecords("shell ran", run.records);
|
||||
std.debug.print(" term: {any}\n stderr: {s}\n", .{ run.term, run.stderr });
|
||||
for (run.created) |name| std.debug.print(" created: {s}\n", .{name});
|
||||
return error.TestShellParityMismatch;
|
||||
}
|
||||
|
||||
fn sortedParityRecords(
|
||||
arena: std.mem.Allocator,
|
||||
records: []const []const []const u8,
|
||||
) ![]const []const []const u8 {
|
||||
const sorted = try arena.dupe([]const []const u8, records);
|
||||
std.mem.sort([]const []const u8, sorted, {}, parityRecordLessThan);
|
||||
return sorted;
|
||||
}
|
||||
|
||||
fn parityRecordLessThan(_: void, lhs: []const []const u8, rhs: []const []const u8) bool {
|
||||
for (lhs[0..@min(lhs.len, rhs.len)], rhs[0..@min(lhs.len, rhs.len)]) |left, right| {
|
||||
switch (std.mem.order(u8, left, right)) {
|
||||
.lt => return true,
|
||||
.gt => return false,
|
||||
.eq => {},
|
||||
}
|
||||
}
|
||||
return lhs.len < rhs.len;
|
||||
}
|
||||
|
||||
fn sameParityRecords(expected: []const []const []const u8, actual: []const []const []const u8) bool {
|
||||
if (expected.len != actual.len) return false;
|
||||
for (expected, actual) |expected_record, actual_record| {
|
||||
if (expected_record.len != actual_record.len) return false;
|
||||
for (expected_record, actual_record) |expected_field, actual_field| {
|
||||
if (!std.mem.eql(u8, expected_field, actual_field)) return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
fn printParityRecords(label: []const u8, records: []const []const []const u8) void {
|
||||
std.debug.print(" {s}:", .{label});
|
||||
for (records) |record| {
|
||||
std.debug.print(" [", .{});
|
||||
for (record, 0..) |field, index| {
|
||||
std.debug.print("{s}\"{f}\"", .{ if (index == 0) "" else " ", std.zig.fmtString(field) });
|
||||
}
|
||||
std.debug.print("]", .{});
|
||||
}
|
||||
std.debug.print("\n", .{});
|
||||
}
|
||||
|
||||
/// Every form of command that fx runs without review. Each must still be
|
||||
/// review-free and must run in every shell exactly as fx parsed it.
|
||||
const parity_review_free_commands = [_][]const u8{
|
||||
"node -v",
|
||||
"node --version",
|
||||
"npm -v",
|
||||
"which node",
|
||||
"which node npm bun",
|
||||
"git status",
|
||||
"git status --short --branch",
|
||||
"git remote -v",
|
||||
"git worktree list",
|
||||
"git worktree list --porcelain",
|
||||
"git fetch",
|
||||
"git fetch origin main",
|
||||
"git fetch --all --tags",
|
||||
"git fetch --depth=1 origin main",
|
||||
"git fetch -q origin -- main",
|
||||
"npm install",
|
||||
"npm i",
|
||||
"npm ci",
|
||||
"npm test",
|
||||
"npm install --save-dev typescript",
|
||||
"npm run build",
|
||||
"npm run lint -- --fix",
|
||||
"bun -v",
|
||||
"bun install",
|
||||
"bun test",
|
||||
"bun run dev",
|
||||
"pnpm install",
|
||||
"pnpm test",
|
||||
"pnpm run build",
|
||||
"yarn install",
|
||||
"yarn test",
|
||||
"yarn run lint",
|
||||
"zig build",
|
||||
"zig build test",
|
||||
"zig build -Doptimize=ReleaseSafe",
|
||||
"node -v && npm -v",
|
||||
"git status && npm test",
|
||||
"npm test 2>&1",
|
||||
"git fetch 2>&1 && git status",
|
||||
"npm run 'build:prod'",
|
||||
"npm run \"build:prod\"",
|
||||
"npm run 'with space'",
|
||||
"git status",
|
||||
"npm\ttest",
|
||||
"pwd",
|
||||
"pwd -P",
|
||||
"ls",
|
||||
"ls -la",
|
||||
"ls -1 src",
|
||||
"ls -- -dashed",
|
||||
"ls 'with space'",
|
||||
"wc -l",
|
||||
"cat",
|
||||
"head",
|
||||
"head -n 5",
|
||||
"tail -n 20",
|
||||
"git diff",
|
||||
"git log -n 5",
|
||||
"printf 'hello\\n'",
|
||||
"ls | wc -l",
|
||||
"printf x | cat",
|
||||
"git log -n 5 | head -n 3",
|
||||
};
|
||||
|
||||
/// Near misses that fx may or may not run without review. When it does,
|
||||
/// they must still run as fx parsed them.
|
||||
const parity_probe_commands = [_][]const u8{
|
||||
"git fetch '&&' git status",
|
||||
"git fetch \\&\\& git status --prune",
|
||||
"npm test \\&\\& npm run build",
|
||||
"npm test &\\&",
|
||||
"npm test \\&&",
|
||||
"npm run =ls",
|
||||
"which =node",
|
||||
"git status #note",
|
||||
"npm run a#b",
|
||||
"git fetch ^x .",
|
||||
"git fetch --prun#e",
|
||||
"npm install react@^18",
|
||||
"npm test x##",
|
||||
"ls ^x",
|
||||
"git status&&npm test",
|
||||
"npm run build 2> &1",
|
||||
"npm run x\\ y",
|
||||
"npm run \"a\\\"b\"",
|
||||
"zig build -Dname=value",
|
||||
"npm run !x",
|
||||
"git fetch origin +main:main",
|
||||
"npm run x,y",
|
||||
"npm run %x",
|
||||
"npm run ^x",
|
||||
"ls =x",
|
||||
"ls '#x'",
|
||||
"printf '%s' '&&'",
|
||||
"npm run 'a''b'",
|
||||
"npm run \"\"",
|
||||
"npm run ''",
|
||||
"git status ''",
|
||||
"npm run é",
|
||||
"grep -n needle file.txt",
|
||||
"grep 'two words' file.txt",
|
||||
"git diff --stat",
|
||||
"printf '%s\\n' a b",
|
||||
};
|
||||
|
||||
test "shell parity: fx runs every review-free command form as sh, dash, bash, and zsh parse it" {
|
||||
const os = @import("builtin").target.os.tag;
|
||||
if (os != .macos and os != .linux) return error.SkipZigTest;
|
||||
var failures: usize = 0;
|
||||
for (parity_review_free_commands) |command| {
|
||||
checkShellParity(command, .must_run_without_review) catch |err| switch (err) {
|
||||
error.TestExpectedReviewFreeCommand, error.TestShellParityMismatch => failures += 1,
|
||||
else => return err,
|
||||
};
|
||||
}
|
||||
for (parity_probe_commands) |command| {
|
||||
checkShellParity(command, .if_run_without_review) catch |err| switch (err) {
|
||||
error.TestShellParityMismatch => failures += 1,
|
||||
else => return err,
|
||||
};
|
||||
}
|
||||
try std.testing.expectEqual(@as(usize, 0), failures);
|
||||
}
|
||||
|
||||
/// Program words that start a generated stage, weighted toward commands fx
|
||||
/// runs without review.
|
||||
const parity_stage_starts = [_][]const u8{
|
||||
"node -v", "which", "git status", "git remote -v", "git worktree list", "git fetch",
|
||||
"git diff", "git log", "npm install", "npm test", "npm run", "npm ci",
|
||||
"bun test", "bun run", "pnpm install", "pnpm run", "yarn run", "zig build",
|
||||
"pwd", "ls", "wc -l", "cat", "head -n 3", "tail",
|
||||
"grep", "printf",
|
||||
};
|
||||
|
||||
/// Pieces of generated words. None contains `/` or a builtin's name, so a
|
||||
/// stage the shell reads differently cannot reach a real program.
|
||||
const parity_fragments = [_][]const u8{
|
||||
"a", "build", "src", "x.txt", "-n", "-l", "--", "--flag",
|
||||
"--o=v", "-Dx=1", "=x", "x=", "#", "#x", "a#b", "'q s'",
|
||||
"\"d q\"", "''", "\"\"", "'", "\"", "\\", "\\ ", "\\#",
|
||||
"&&", "'&&'", "\"&&\"", "&", "|", "||", "2>&1", ">",
|
||||
"<", ">>", "2>", ";", "$x", "${x}", "`x`", "$(x)",
|
||||
"*", "?", "[a]", "~", "{a,b}", "(x)", "!", "!x",
|
||||
"%", "^", ",", ":", "@", "+", "\t", "\u{e9}",
|
||||
"\u{a0}", "\x1b", "\\&\\&", "\\&", "^x", "x#", "x##",
|
||||
};
|
||||
|
||||
fn writeParityCommand(smith: *std.testing.Smith, out: *std.Io.Writer) std.Io.Writer.Error!void {
|
||||
const joiner: []const u8 = if (smith.boolWeighted(1, 1)) " && " else " | ";
|
||||
const stage_count = smith.valueRangeAtMost(u8, 1, 4);
|
||||
for (0..stage_count) |stage| {
|
||||
if (stage > 0) try out.writeAll(joiner);
|
||||
try out.writeAll(parity_stage_starts[smith.index(parity_stage_starts.len)]);
|
||||
for (0..smith.valueRangeAtMost(u8, 0, 4)) |_| {
|
||||
try out.writeByte(if (smith.boolWeighted(7, 1)) ' ' else '\t');
|
||||
for (0..smith.valueRangeAtMost(u8, 1, 3)) |_| {
|
||||
if (smith.boolWeighted(1, 7)) {
|
||||
try out.writeAll(parity_fragments[smith.index(parity_fragments.len)]);
|
||||
continue;
|
||||
}
|
||||
var raw: [8]u8 = undefined;
|
||||
for (raw[0..smith.slice(&raw)]) |byte| {
|
||||
// No letters, so raw bytes cannot spell a program or
|
||||
// builtin; no `/`, so they cannot name a path to one.
|
||||
const safe = !std.ascii.isAlphabetic(byte) and byte != '/' and byte != 0;
|
||||
try out.writeByte(if (safe) byte else '_');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn fuzzShellParity(_: void, smith: *std.testing.Smith) !void {
|
||||
const os = @import("builtin").target.os.tag;
|
||||
if (os != .macos and os != .linux) return error.SkipZigTest;
|
||||
var buffer: [512]u8 = undefined;
|
||||
var command: std.Io.Writer = .fixed(&buffer);
|
||||
writeParityCommand(smith, &command) catch |err| switch (err) {
|
||||
// A full buffer ends the command early, which is still a command.
|
||||
error.WriteFailed => {},
|
||||
};
|
||||
try checkShellParity(command.buffered(), .if_run_without_review);
|
||||
}
|
||||
|
||||
test "shell parity holds for generated commands" {
|
||||
try std.testing.fuzz({}, fuzzShellParity, .{
|
||||
.corpus = &(.{""} ++ @import("fuzz_corpus").shell_parity.*),
|
||||
});
|
||||
}
|
||||
@@ -1061,14 +1061,14 @@ test "parseSkillFile rejects indented continuations for recognized metadata" {
|
||||
|
||||
test "parseSkillFile keeps fuzzed borrowed slices inside the input" {
|
||||
try std.testing.fuzz({}, fuzzParseSkillFile, .{
|
||||
.corpus = &.{
|
||||
.corpus = &(.{
|
||||
"",
|
||||
"plain body",
|
||||
"---\nname: valid\n---\nbody",
|
||||
"---\r\nname: \"valid\"\r\n---\r\nbody",
|
||||
"---\nname: quoted\ndescription: 'single line'\n---\nbody",
|
||||
"---\nname: block\ndescription: >-\n first line\n second line\n---\nbody",
|
||||
},
|
||||
} ++ @import("fuzz_corpus").skill_file.*),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -113,11 +113,11 @@ test "skill display name parsing stays bounded" {
|
||||
try std.testing.expect(name.len <= buffer.len);
|
||||
try std.testing.expect(skill_contract.invalidSkillNameCause(name) == null);
|
||||
}
|
||||
}.check, .{ .corpus = &.{
|
||||
}.check, .{ .corpus = &(.{
|
||||
content_name_prefix ++ "workflow\" location=\"/skills/workflow\">",
|
||||
discovery_model_notice ++ content_name_prefix ++ "name&value\" resource=\"SKILL.md\">",
|
||||
content_name_prefix ++ "bad&unknown;\" location=\"/skills/workflow\">",
|
||||
} });
|
||||
} ++ @import("fuzz_corpus").skill_display_name.*) });
|
||||
}
|
||||
|
||||
/// Borrows a previously discovered skill inventory for one invocation.
|
||||
|
||||
@@ -707,11 +707,11 @@ fn fuzzCommandOutputContent(_: void, smith: *std.testing.Smith) anyerror!void {
|
||||
|
||||
test "fuzz command output content" {
|
||||
try std.testing.fuzz({}, fuzzCommandOutputContent, .{
|
||||
.corpus = &.{
|
||||
.corpus = &(.{
|
||||
"progress\roverwrite\n",
|
||||
"\x1b[31mred\x1b[0m\n",
|
||||
"exit_code=0\n<stdout>\none\n</stdout>\n",
|
||||
"\xff\x00\xe2\x82",
|
||||
},
|
||||
} ++ @import("fuzz_corpus").command_output_content.*),
|
||||
});
|
||||
}
|
||||
@@ -4238,7 +4238,7 @@ test "deinit cancels and joins one loader while suppressing queued refresh" {
|
||||
|
||||
test "file index raw-list and Unicode query bytes remain bounded" {
|
||||
try std.testing.fuzz({}, fuzzRawListAndQuery, .{
|
||||
.corpus = &.{
|
||||
.corpus = &(.{
|
||||
"safe.txt\x00ärger",
|
||||
"escape-\x1b[2J.txt\x00\xff",
|
||||
"Ärger-file.txt\x00ärger",
|
||||
@@ -4247,7 +4247,7 @@ test "file index raw-list and Unicode query bytes remain bounded" {
|
||||
"docs/Cafe\u{0301}.txt\x00e\u{0301}",
|
||||
"src/axaya.zig\x00aaa",
|
||||
"truncated.txt\x00\xc3",
|
||||
},
|
||||
} ++ @import("fuzz_corpus").file_index.*),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
//! Root of the `zig build fuzz` test binary. Each import below brings in a
|
||||
//! file that holds an fx fuzz target listed in `tests/fuzz/targets.json`.
|
||||
//! The build filters this binary to the registered fuzz tests, so ordinary
|
||||
//! unit tests in these files never run here. A new fx fuzz target needs its
|
||||
//! file imported here and an entry in the registry.
|
||||
|
||||
test {
|
||||
_ = @import("acp/mcp_servers.zig");
|
||||
_ = @import("core/agent/runtime/vision_contracts.zig");
|
||||
_ = @import("core/images/image_data.zig");
|
||||
_ = @import("core/mcp/elicitation.zig");
|
||||
_ = @import("core/session/session_codec.zig");
|
||||
_ = @import("core/session/session_event.zig");
|
||||
_ = @import("core/shared/display_width.zig");
|
||||
_ = @import("core/shared/types.zig");
|
||||
_ = @import("core/shell_command/command_effect.zig");
|
||||
_ = @import("core/skills/skill_contract.zig");
|
||||
_ = @import("core/skills/skill_invocation.zig");
|
||||
_ = @import("core/tooling/command_output_content.zig");
|
||||
_ = @import("core/workspace/file_index.zig");
|
||||
_ = @import("gateway/responses_protocol.zig");
|
||||
_ = @import("gateway/sse.zig");
|
||||
}
|
||||
@@ -2167,7 +2167,7 @@ test "Responses text finalization fuzzes chunking and capture boundaries" {
|
||||
try stream.finish(buffer[0 .. len + separator.len + 4]);
|
||||
}
|
||||
};
|
||||
try std.testing.fuzz({}, Probe.run, .{ .corpus = &.{ "", "a", "two chunks", "capture limit is not receipt progress" } });
|
||||
try std.testing.fuzz({}, Probe.run, .{ .corpus = &(.{ "", "a", "two chunks", "capture limit is not receipt progress" } ++ @import("fuzz_corpus").responses_text.*) });
|
||||
}
|
||||
|
||||
test "Responses captures assistant commentary phase" {
|
||||
|
||||
+1
-1
@@ -263,7 +263,7 @@ test "provider framing fuzzes chunk-invariant event data" {
|
||||
try std.testing.expectEqual(null, try reader.next(alloc, &source.interface, &cancelled));
|
||||
}
|
||||
};
|
||||
try std.testing.fuzz({}, Probe.run, .{ .corpus = &.{ "", "one", "split every field" } });
|
||||
try std.testing.fuzz({}, Probe.run, .{ .corpus = &(.{ "", "one", "split every field" } ++ @import("fuzz_corpus").sse_framing.*) });
|
||||
}
|
||||
|
||||
fn check_allocations(alloc: std.mem.Allocator) !void {
|
||||
|
||||
@@ -16,6 +16,7 @@ Pick the lowest layer that proves the behavior, and extend the closest existing
|
||||
| fx against the official MCP client conformance suite | `tests/e2e/conformance/`, an isolated npm project |
|
||||
| Evidence tooling for terminal rendering bugs | `tests/e2e/render-lab/` |
|
||||
| Task quality or tool choice with real models | `tests/evals/` |
|
||||
| Saved inputs for a fuzz test | `tests/fuzz/corpus/<target>/`, with the target listed in `tests/fuzz/targets.json` |
|
||||
| A protocol or crash-safety design, checked with TLC | A folder under `tests/tla/` named after the module it models |
|
||||
| libfx SDK behavior | `sdk/tests/`, which `sdk/AGENTS.md` owns |
|
||||
| Startup latency or memory | `benchmarks/` |
|
||||
@@ -34,6 +35,9 @@ tests/
|
||||
├── fixtures/
|
||||
│ └── v1-sessions/ sessions the v0.0.14 release saved, which the converter's tests copy;
|
||||
│ generate.ts rebuilds them and manifest.json lists them
|
||||
├── fuzz/
|
||||
│ ├── targets.json every fuzz test, for zig build fuzz and the nightly Fuzz workflow
|
||||
│ └── corpus/<target>/ saved inputs that zig build test replays; CONTRIBUTING.md explains them
|
||||
└── tla/ TLA+ models, their checkers, and their Bun drivers
|
||||
├── session_manager/ one folder per model area, named after the module it describes,
|
||||
├── mcp/ each with a check.sh; tla/README.md lists them all
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"targets": [
|
||||
{"name": "acp_mcp_servers", "module": "fx", "test": "ACP MCP parser handles fuzzed request bytes", "iterations": 1500000},
|
||||
{"name": "vision_json", "module": "fx", "test": "Vision JSON parsers tolerate arbitrary bounded bytes", "iterations": 1200000},
|
||||
{"name": "image_dimensions", "module": "fx", "test": "image dimension parsing stays bounded on arbitrary bytes", "iterations": 5000000},
|
||||
{"name": "mcp_elicitation", "module": "fx", "test": "elicitation request and response parsers stay bounded under fuzzed bytes", "iterations": 1500000},
|
||||
{"name": "session_optional_fields", "module": "fx", "test": "durable session optional fields handle fuzzed ownership paths", "iterations": 3000000},
|
||||
{"name": "session_work_provenance", "module": "fx", "test": "per-turn work provenance parser handles fuzzed ownership paths", "iterations": 2100000},
|
||||
{"name": "session_conversation_frame", "module": "fx", "test": "conversation frame decoder stays bounded under fuzzed bytes", "iterations": 3000000},
|
||||
{"name": "session_log_scan", "module": "session_manager", "test": "scanning never panics on arbitrary bytes", "iterations": 5000000},
|
||||
{"name": "session_schema_header", "module": "session_manager", "test": "header parsing never panics on arbitrary bytes", "iterations": 5000000},
|
||||
{"name": "display_width", "module": "fx", "test": "fuzz display unit boundaries", "iterations": 620000},
|
||||
{"name": "tool_argument_diagnostic", "module": "fx", "test": "fuzz tool argument diagnostic", "iterations": 3000000},
|
||||
{"name": "skill_file", "module": "fx", "test": "parseSkillFile keeps fuzzed borrowed slices inside the input", "iterations": 5000000},
|
||||
{"name": "skill_display_name", "module": "fx", "test": "skill display name parsing stays bounded", "iterations": 5000000},
|
||||
{"name": "command_output_content", "module": "fx", "test": "fuzz command output content", "iterations": 160000},
|
||||
{"name": "file_index", "module": "fx", "test": "file index raw-list and Unicode query bytes remain bounded", "iterations": 620000},
|
||||
{"name": "responses_text", "module": "fx", "test": "Responses text finalization fuzzes chunking and capture boundaries", "iterations": 430000},
|
||||
{"name": "sse_framing", "module": "fx", "test": "provider framing fuzzes chunk-invariant event data", "iterations": 1300000},
|
||||
{"name": "shell_parity", "module": "fx", "test": "shell parity holds for generated commands", "iterations": 600}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user