diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae7eccec1..b8f9abc4b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,7 @@ jobs: run: ./scripts/check-indexer-boundary.sh - name: Test CI and release scripts - run: python3 -m unittest scripts.tests.test_macos_signing scripts.tests.test_signed_macos scripts.tests.test_detect_macos_need -q + run: python3 -m unittest scripts.tests.test_macos_signing scripts.tests.test_signed_macos scripts.tests.test_detect_macos_need scripts.tests.test_fuzz_nightly -q - name: Build run: zig build -Doptimize=${{ matrix.optimize }} diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml new file mode 100644 index 000000000..0307f65ac --- /dev/null +++ b/.github/workflows/fuzz.yml @@ -0,0 +1,108 @@ +name: Fuzz + +# Nightly coverage-guided fuzzing. Each target in tests/fuzz/targets.json gets +# its own job and about 20 minutes of fuzzing. A failure uploads the recovered +# input and opens or updates the target's issue, titled +# "Nightly fuzz failure: ". scripts/fuzz_nightly.py does the work. +# A pull request that changes the fuzzing setup fuzzes every target for two +# minutes, so a change that breaks the nightly run fails before it merges. + +on: + schedule: + - cron: "23 7 * * *" + pull_request: + paths: + - .github/workflows/fuzz.yml + - scripts/fuzz_nightly.py + - src/fuzz_targets.zig + - tests/fuzz/** + workflow_dispatch: + inputs: + target: + description: Fuzz only this target (leave empty for every target) + type: string + default: "" + +permissions: + contents: read + +concurrency: + group: fuzz-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + targets: + name: List fuzz targets + runs-on: ubuntu-latest + outputs: + targets: ${{ steps.list.outputs.targets }} + steps: + - uses: actions/checkout@v4 + + - name: List targets + id: list + env: + ONLY: ${{ inputs.target }} + run: python3 -m scripts.fuzz_nightly matrix --only "$ONLY" >> "$GITHUB_OUTPUT" + + fuzz: + name: Fuzz ${{ matrix.target }} + needs: targets + runs-on: ubuntu-latest + timeout-minutes: 45 + permissions: + contents: read + issues: write + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(needs.targets.outputs.targets) }} + steps: + - uses: actions/checkout@v4 + + # Restores main's unit-test cache for the dependencies, compiler_rt, and + # libc it keeps. This job never saves a cache. + - name: Setup Zig + uses: ./.github/actions/setup-zig + with: + version: "0.17.0" + cache-key: ci-unit-ReleaseSafe + + # shell_parity compares fx's parse with sh, dash, bash, and zsh. + - name: Install zsh + if: matrix.target == 'shell_parity' + run: command -v zsh >/dev/null || { sudo apt-get update -qq && sudo apt-get install -y -qq zsh; } + + - name: Fuzz + env: + TARGET: ${{ matrix.target }} + BUDGET_SECONDS: ${{ github.event_name == 'pull_request' && '120' || '1200' }} + run: >- + python3 -m scripts.fuzz_nightly run + --target "$TARGET" + --out fuzz-output + --budget-seconds "$BUDGET_SECONDS" + + - name: Upload failure evidence + if: failure() + uses: actions/upload-artifact@v4 + with: + name: fuzz-${{ matrix.target }} + path: fuzz-output/ + if-no-files-found: warn + retention-days: 30 + + # Only main reports, so a dispatch on a branch cannot open issues. + - name: Report failure + if: failure() && github.ref == 'refs/heads/main' + env: + GH_TOKEN: ${{ github.token }} + TARGET: ${{ matrix.target }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: >- + python3 -m scripts.fuzz_nightly report + --target "$TARGET" + --out fuzz-output + --repo "$GITHUB_REPOSITORY" + --run-url "$RUN_URL" + --commit "$GITHUB_SHA" diff --git a/AGENTS.md b/AGENTS.md index 91b9e0b36..487159bf9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -166,6 +166,8 @@ Security is permission-first. All sensitive tool behavior must integrate with `s * Host-generated review holds retain their advice for the agent and transcript, but carry a saved `review_feedback` marker that excludes them from later security evidence, including after recovery. Old unmarked results remain untrusted evidence; never infer the marker from output text. Quoted review accusations are not proof of an attack, and handling plans or test instructions as data is not itself prompt injection +* A command form that fx runs without review, through `command_effect.plan` or `command_effect.knownReversibleAutoCommand`, must be listed in `parity_review_free_commands` in `src/core/shell_command/command_effect.zig`. The shell parity tests check that sh, dash, bash, and zsh, with and without `extendedglob`, run each one exactly as fx parsed it, and the nightly `shell_parity` fuzz target checks generated commands the same way + Do not bypass the permission system for new tools. ## Zig-Specific Patterns @@ -244,6 +246,8 @@ fx does not use declarations that Zig 0.17 deprecates. They still compile, so ch * In test blocks, use `std.testing.io` for the `Io` parameter. `io_mod.getIo()` automatically returns `std.testing.io` in test builds. +* A new `std.testing.fuzz` test needs an entry in `tests/fuzz/targets.json` and its saved inputs passed as its corpus, as **Fuzz a parser** in `CONTRIBUTING.md` describes. The nightly Fuzz workflow fuzzes every entry. + * Use `io_mod.dirRealpathAlloc(alloc, dir, sub_path)` to resolve paths within `std.testing.tmpDir()`. * Back `std.testing.checkAllAllocationFailures`, hand-written `FailingAllocator` sweeps, and allocation-count comparisons with `testing_allocator.no_resize` from `src/core/shared/testing_allocator.zig`. `std.testing.allocator` may grow an allocation in place on one run and not the next, which changes the allocation sequence and makes those tests flaky. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 57aa51434..4d68c3521 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -98,6 +98,23 @@ fx prints into your terminal's scrollback instead of taking over the screen, so When a tmux test or a tape exposes a bug, reproduce it as a Zig unit test before you fix it, so the fix lands with its regression test. +### Fuzz a parser + +A fuzz test is a `test` block that calls `std.testing.fuzz`. `tests/fuzz/targets.json` lists each one as a target, with its exact test name and the module that holds it. `zig build test` runs every fuzz test once on its inline seeds and on each file in `tests/fuzz/corpus//`. `zig build fuzz` builds only the fuzz tests, and with `--fuzz` Zig searches for new inputs that fail: + +```bash +zig build fuzz -Dfuzz-target=sse_framing # replay the saved inputs +zig build fuzz -Dfuzz-target=sse_framing --fuzz=200K # try 200,000 new inputs +``` + +When the search finds a failure, Zig 0.17 prints `error: test '...'` but still exits with status 0, so read the output. Without a limit, `--fuzz` runs until you stop it and opens Zig's web interface. + +The Fuzz workflow fuzzes each target on `main` every night for about 20 minutes, and for two minutes on a pull request that changes the fuzzing setup. When the nightly run finds a failure, it uploads the failing input as an artifact and opens an issue titled `Nightly fuzz failure: `, or comments on that issue if it is already open. To fix the failure, copy the input into `tests/fuzz/corpus//` and run `zig build fuzz -Dfuzz-target=` to reproduce it. That build uses Debug by default, which adds allocation stack traces to leak reports. Keep the input file in the fix's pull request, so `zig build test` replays it on every pull request after that. + +A saved input holds the values the test reads from its `std.testing.Smith`, in order. For example, a test that calls `smith.slice` reads a 4-byte little-endian length and then that many bytes. Inputs that the fuzzer saves are already in this form. + +To add a target, write the fuzz test in the file it covers and add an entry to `tests/fuzz/targets.json`. For a target outside the session manager, also import its file in `src/fuzz_targets.zig`. Pass the saved inputs to the test with `.corpus = &(.{ } ++ @import("fuzz_corpus")..*)`, or with `@import("build_options").fuzz_corpus_` in the session manager. Set `iterations` so that one round of `--fuzz` takes about 30 seconds on a developer machine. `scripts/tests/test_fuzz_nightly.py` fails when an entry names a test that doesn't exist or that `zig build fuzz` can't reach. + ### Check startup performance The `Startup Latency` check fails a pull request when any benchmarked command takes longer than 2 ms on the Linux CI runner. To measure before you push, run: @@ -197,6 +214,7 @@ Other runs start only on request: | Live ACP suite | `gh workflow run ci.yml --ref your-branch -f acp_live=true` | | Long-turn memory benchmark | `gh workflow run bench.yml --ref your-branch -f long_turn_memory=true` | | libfx runtime benchmark | `gh workflow run bench.yml --ref your-branch -f libfx_runtime=true` | +| Fuzzing of one target | `gh workflow run fuzz.yml --ref your-branch -f target=` | Only maintainers can start these runs or add the `ci:macos` label. If you're an outside contributor and your change needs one of them, ask in the pull request and a maintainer will start it. If CI doesn't start at all on your first pull request, a maintainer may need to approve the workflow run. diff --git a/build.zig b/build.zig index 49f421020..1d2756097 100644 --- a/build.zig +++ b/build.zig @@ -52,6 +52,9 @@ pub fn build(b: *std.Build) void { // binary uses, such as its E2E test controls. build_options.addOption(bool, "libfx", false); + const fuzz_corpora = readFuzzCorpora(b); + const fuzz_corpus = addFuzzCorpusModule(b, fuzz_corpora); + const exe = b.addExecutable(.{ .name = "fx", .root_module = b.createModule(.{ @@ -68,7 +71,8 @@ pub fn build(b: *std.Build) void { }), }); exe.root_module.addImport("build_options", build_options.createModule()); - const session_manager = addSessionManager(b, exe.root_module, target, optimize, null); + exe.root_module.addImport("fuzz_corpus", fuzz_corpus); + const session_manager = addSessionManager(b, exe.root_module, target, optimize, null, fuzz_corpora); b.installArtifact(exe); @@ -89,7 +93,8 @@ pub fn build(b: *std.Build) void { const test_paths = b.addOptions(); test_paths.addOptionPathUntracked("product_exe", b.graph.path(.install_bin, "fx")); test_paths.addOptionPathUntracked("source_root", b.path("src")); - exe.root_module.addImport("test_paths", test_paths.createModule()); + const test_paths_module = test_paths.createModule(); + exe.root_module.addImport("test_paths", test_paths_module); // The session manager's own tests, as fx compiles it (no hooks). const session_manager_tests = b.addTest(.{ .root_module = session_manager }); @@ -97,7 +102,7 @@ pub fn build(b: *std.Build) void { const run_session_manager_tests = b.addRunArtifact(session_manager_tests); // And with its fault and trace hooks, whose tests fx's build leaves out. const session_manager_hook_tests = b.addTest(.{ - .root_module = sessionManagerModule(b, target, optimize, null, true), + .root_module = sessionManagerModule(b, target, optimize, null, true, fuzz_corpora), }); const run_session_manager_hook_tests = b.addRunArtifact(session_manager_hook_tests); const session_manager_test_step = b.step("test-session-manager", "Run the session manager's tests"); @@ -109,6 +114,30 @@ pub fn build(b: *std.Build) void { test_step.dependOn(&run_session_manager_tests.step); test_step.dependOn(&run_session_manager_hook_tests.step); + const fuzz_step = b.step( + "fuzz", + "Replay the fuzz targets' saved inputs; add --fuzz=N to search N new inputs", + ); + const fuzz_target = b.option( + []const u8, + "fuzz-target", + "Run only this fuzz target from " ++ fuzz_registry_path, + ); + const fuzz_root = b.createModule(.{ + .root_source_file = b.path("src/fuzz_targets.zig"), + .target = target, + .optimize = optimize, + .link_libc = true, + }); + fuzz_root.addImport("build_options", build_options.createModule()); + fuzz_root.addImport("fuzz_corpus", fuzz_corpus); + fuzz_root.addImport("session_manager", session_manager); + fuzz_root.addImport("test_paths", test_paths_module); + addFuzzStep(b, fuzz_step, fuzz_corpora, fuzz_target, .{ + .fx = fuzz_root, + .session_manager = session_manager, + }); + if (wasm_surface != .none) { addWasmArtifact(b, wasm_surface, git_commit, app_version, update_channel); } @@ -329,7 +358,7 @@ fn addWasmArtifact( }); if (surface == .core) wasm_exe.stack_size = 1024 * 1024; wasm_exe.root_module.addImport("build_options", wasm_options.createModule()); - _ = addSessionManager(b, wasm_exe.root_module, wasm_target, .small, true); + _ = addSessionManager(b, wasm_exe.root_module, wasm_target, .small, true, &.{}); const install_wasm = b.addInstallArtifact(wasm_exe, .{}); const wasm_step = b.step(name ++ "-wasm", description); @@ -368,7 +397,7 @@ fn addNapiArtifact( }), }); lib.root_module.addImport("build_options", napi_options.createModule()); - _ = addSessionManager(b, lib.root_module, target, .safe, null); + _ = addSessionManager(b, lib.root_module, target, .safe, null, &.{}); const node_include = b.option( []const u8, "node-include-dir", @@ -418,15 +447,17 @@ fn readGitCommit(b: *std.Build) []const u8 { /// The session manager as its own module, rooted at its API. It imports /// nothing but its own options, and a file cannot belong to two modules, so -/// fx reaches it only through `@import("session_manager")`. +/// fx reaches it only through `@import("session_manager")`. Its fuzz targets' +/// saved inputs travel in those options as `fuzz_corpus_`. fn addSessionManager( b: *std.Build, importer: *std.Build.Module, target: std.Build.ResolvedTarget, optimize: std.lang.Optimize, single_threaded: ?bool, + fuzz_corpora: []const FuzzCorpus, ) *std.Build.Module { - const module = sessionManagerModule(b, target, optimize, single_threaded, false); + const module = sessionManagerModule(b, target, optimize, single_threaded, false, fuzz_corpora); importer.addImport("session_manager", module); return module; } @@ -438,11 +469,20 @@ fn sessionManagerModule( optimize: std.lang.Optimize, single_threaded: ?bool, hooks: bool, + fuzz_corpora: []const FuzzCorpus, ) *std.Build.Module { const options = b.addOptions(); options.addOption(bool, "hooks", hooks); options.addOptionPathUntracked("trace_dir", b.graph.path(.install_prefix, "session-manager-traces")); options.addOptionPathUntracked("src_dir", b.path("src/core/session_manager")); + for (fuzz_corpora) |corpus| { + if (corpus.target.module != .session_manager) continue; + options.addOption( + []const []const u8, + b.fmt("fuzz_corpus_{s}", .{corpus.target.name}), + corpus.inputs, + ); + } const module = b.createModule(.{ .root_source_file = b.path("src/core/session_manager/api.zig"), .target = target, @@ -469,3 +509,205 @@ fn readAppVersion(b: *std.Build) []const u8 { return b.allocator.dupe(u8, bytes[start .. start + end_rel]) catch @panic("could not allocate app version"); } + +const fuzz_registry_path = "tests/fuzz/targets.json"; +const fuzz_corpus_dir = "tests/fuzz/corpus"; +/// A saved input this large is almost certainly a mistake, such as a build +/// artifact copied into a corpus folder. +const max_fuzz_input_bytes = 1024 * 1024; + +const FuzzModule = enum { fx, session_manager }; + +/// One entry in `tests/fuzz/targets.json`. +const FuzzTarget = struct { + /// Names the target's corpus folder and its `fuzz_corpus` declaration. + name: []const u8, + /// The module whose test binary holds the fuzz test. + module: FuzzModule, + /// The fuzz test's exact name, used as the test filter. + @"test": []const u8, + /// Iterations for one nightly run. Only the nightly workflow reads it. + iterations: u64, +}; + +const FuzzRegistry = struct { + targets: []const FuzzTarget, +}; + +/// A registered fuzz target and its saved inputs, sorted by file name. +const FuzzCorpus = struct { + target: FuzzTarget, + inputs: []const []const u8, +}; + +const FuzzRoots = struct { + fx: *std.Build.Module, + session_manager: *std.Build.Module, +}; + +/// Reads the fuzz registry and each target's saved inputs. Configuration +/// reruns on every build (see `readGitCommit`), so a newly added input is +/// picked up without listing it anywhere. +fn readFuzzCorpora(b: *std.Build) []const FuzzCorpus { + b.dependOnFileContents(b.path(fuzz_registry_path)); + const registry_bytes = b.root.root_dir.handle.readFileAlloc( + b.graph.io, + b.pathJoin(&.{ b.root.sub_path, fuzz_registry_path }), + b.allocator, + .limited(max_fuzz_input_bytes), + ) catch |err| std.process.fatal("could not read {s}: {t}", .{ fuzz_registry_path, err }); + const registry = std.json.parseFromSliceLeaky(FuzzRegistry, b.allocator, registry_bytes, .{}) catch |err| + std.process.fatal("{s} is not a valid fuzz registry: {t}", .{ fuzz_registry_path, err }); + + const corpora = b.allocator.alloc(FuzzCorpus, registry.targets.len) catch @panic("OOM"); + for (registry.targets, corpora, 0..) |fuzz_target, *corpus, index| { + if (!isFuzzTargetName(fuzz_target.name)) std.process.fatal( + "{s}: fuzz target name '{s}' must start with a lowercase letter and use only lowercase letters, digits, and underscores", + .{ fuzz_registry_path, fuzz_target.name }, + ); + for (registry.targets[0..index]) |earlier| { + if (std.mem.eql(u8, earlier.name, fuzz_target.name)) std.process.fatal( + "{s}: fuzz target '{s}' is listed twice", + .{ fuzz_registry_path, fuzz_target.name }, + ); + } + corpus.* = .{ .target = fuzz_target, .inputs = readFuzzInputs(b, fuzz_target.name) }; + } + rejectUnregisteredCorpusFolders(b, registry.targets); + return corpora; +} + +fn isFuzzTargetName(name: []const u8) bool { + if (name.len == 0 or !std.ascii.isLower(name[0])) return false; + for (name) |byte| { + if (!std.ascii.isLower(byte) and !std.ascii.isDigit(byte) and byte != '_') return false; + } + return true; +} + +/// Returns the files in `tests/fuzz/corpus//`, sorted by file name so +/// the generated module does not depend on directory order. A missing folder +/// means the target has no saved inputs yet. Dotfiles are skipped. +fn readFuzzInputs(b: *std.Build, name: []const u8) []const []const u8 { + const io = b.graph.io; + const folder = b.pathJoin(&.{ fuzz_corpus_dir, name }); + var dir = b.root.root_dir.handle.openDir(io, b.pathJoin(&.{ b.root.sub_path, folder }), .{ + .iterate = true, + }) catch |err| switch (err) { + error.FileNotFound => return &.{}, + else => std.process.fatal("could not open {s}: {t}", .{ folder, err }), + }; + defer dir.close(io); + + var file_names: std.ArrayList([]const u8) = .empty; + var entries = dir.iterate(); + while (entries.next(io) catch |err| std.process.fatal("could not list {s}: {t}", .{ folder, err })) |entry| { + if (entry.name[0] == '.') continue; + if (entry.kind != .file) std.process.fatal( + "{s}/{s}: saved fuzz inputs must be regular files", + .{ folder, entry.name }, + ); + file_names.append(b.allocator, b.dupe(entry.name)) catch @panic("OOM"); + } + std.mem.sort([]const u8, file_names.items, {}, lessThanBytes); + + const inputs = b.allocator.alloc([]const u8, file_names.items.len) catch @panic("OOM"); + for (file_names.items, inputs) |file_name, *input| { + b.dependOnFileContents(b.path(b.pathJoin(&.{ folder, file_name }))); + input.* = dir.readFileAlloc(io, file_name, b.allocator, .limited(max_fuzz_input_bytes + 1)) catch |err| switch (err) { + error.StreamTooLong => std.process.fatal("{s}/{s} is larger than 1 MiB", .{ folder, file_name }), + else => std.process.fatal("could not read {s}/{s}: {t}", .{ folder, file_name, err }), + }; + } + return inputs; +} + +/// Fails the build when a corpus folder has no registry entry, so a typo in +/// a folder name cannot leave saved inputs that never run. +fn rejectUnregisteredCorpusFolders(b: *std.Build, targets: []const FuzzTarget) void { + const io = b.graph.io; + var dir = b.root.root_dir.handle.openDir(io, b.pathJoin(&.{ b.root.sub_path, fuzz_corpus_dir }), .{ + .iterate = true, + }) catch |err| switch (err) { + error.FileNotFound => return, + else => std.process.fatal("could not open {s}: {t}", .{ fuzz_corpus_dir, err }), + }; + defer dir.close(io); + + var entries = dir.iterate(); + entries: while (entries.next(io) catch |err| std.process.fatal("could not list {s}: {t}", .{ fuzz_corpus_dir, err })) |entry| { + if (entry.name[0] == '.') continue; + for (targets) |fuzz_target| { + if (entry.kind == .directory and std.mem.eql(u8, entry.name, fuzz_target.name)) continue :entries; + } + std.process.fatal( + "{s}/{s} does not match a fuzz target in {s}", + .{ fuzz_corpus_dir, entry.name, fuzz_registry_path }, + ); + } +} + +fn lessThanBytes(_: void, lhs: []const u8, rhs: []const u8) bool { + return std.mem.lessThan(u8, lhs, rhs); +} + +/// The `fuzz_corpus` module: one `[]const []const u8` of saved inputs per fx +/// fuzz target, named after the target. +fn addFuzzCorpusModule(b: *std.Build, corpora: []const FuzzCorpus) *std.Build.Module { + const options = b.addOptions(); + for (corpora) |corpus| { + if (corpus.target.module != .fx) continue; + options.addOption([]const []const u8, corpus.target.name, corpus.inputs); + } + return options.createModule(); +} + +/// Runs the selected fuzz targets, or all of them. Each module's test binary +/// is filtered to its targets' fuzz tests, so `--fuzz` spends every iteration +/// on them and ordinary unit tests never run here. +fn addFuzzStep( + b: *std.Build, + step: *std.Build.Step, + corpora: []const FuzzCorpus, + selected: ?[]const u8, + roots: FuzzRoots, +) void { + var fx_filters: std.ArrayList([]const u8) = .empty; + var session_manager_filters: std.ArrayList([]const u8) = .empty; + for (corpora) |corpus| { + if (selected) |name| { + if (!std.mem.eql(u8, name, corpus.target.name)) continue; + } + const filters = switch (corpus.target.module) { + .fx => &fx_filters, + .session_manager => &session_manager_filters, + }; + filters.append(b.allocator, corpus.target.@"test") catch @panic("OOM"); + } + if (fx_filters.items.len == 0 and session_manager_filters.items.len == 0) { + const message = if (selected) |name| + b.fmt("unknown fuzz target '{s}'; the targets are listed in {s}", .{ name, fuzz_registry_path }) + else + "no fuzz targets are listed in " ++ fuzz_registry_path; + step.dependOn(&b.addFail(message).step); + return; + } + + const binaries = [_]struct { + name: []const u8, + root: *std.Build.Module, + filters: []const []const u8, + }{ + .{ .name = "fuzz", .root = roots.fx, .filters = fx_filters.items }, + .{ .name = "fuzz-session-manager", .root = roots.session_manager, .filters = session_manager_filters.items }, + }; + for (binaries) |binary| { + if (binary.filters.len == 0) continue; + const tests = b.addTest(.{ + .name = binary.name, + .root_module = binary.root, + .filters = binary.filters, + }); + step.dependOn(&b.addRunArtifact(tests).step); + } +} diff --git a/scripts/fuzz_nightly.py b/scripts/fuzz_nightly.py new file mode 100644 index 000000000..7e9798efb --- /dev/null +++ b/scripts/fuzz_nightly.py @@ -0,0 +1,420 @@ +"""Nightly fuzzing driver for `.github/workflows/fuzz.yml`. + +`matrix` prints the job matrix from `tests/fuzz/targets.json`. + +`run` fuzzes one target in rounds of `zig build fuzz --fuzz=` +until its time budget is spent. Zig keeps the fuzzer's corpus in +`.zig-cache/f/` between rounds, so each round continues where the last one +stopped, and a round that outlives its timeout is a hang rather than an +exhausted budget. When a round fails, `run` recovers the failing input, +confirms it by replaying it as a saved input of the target, and writes it to +the output folder with the log and `result.json`. + +`report` opens the target's GitHub issue, or comments on it when one is +already open. + +Two Zig 0.17 behaviors shape this script. `zig build --fuzz` exits 0 even +when it finds a failure, so a round is judged by its `error: test` line. The +input it copies to `.zig-cache/f/crash` can be empty, so the inputs are also +read from the fuzzer's `in` files. +""" + +from __future__ import annotations + +import argparse +import dataclasses +import hashlib +import json +import os +import pathlib +import re +import signal +import struct +import subprocess +import sys +import time +from collections.abc import Callable, Sequence + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +REGISTRY_PATH = pathlib.PurePosixPath("tests/fuzz/targets.json") +CORPUS_PATH = pathlib.PurePosixPath("tests/fuzz/corpus") + +# Zig 0.17 `std.Build.abi.fuzz.MmapInputHeader`, little-endian: pc_digest +# (u64), instance_id (u32), test_i (u32), len (u32). The input follows it. +INPUT_HEADER = struct.Struct(" list[Target]: + data = json.loads((root / REGISTRY_PATH).read_text()) + targets = [ + Target( + name=entry["name"], + module=entry["module"], + test=entry["test"], + iterations=int(entry["iterations"]), + ) + for entry in data["targets"] + ] + names = [target.name for target in targets] + if len(set(names)) != len(names): + raise FuzzNightlyError(f"{REGISTRY_PATH} lists a target twice") + for target in targets: + if target.iterations <= 0: + raise FuzzNightlyError(f"{target.name}: iterations must be positive") + return targets + + +def find_target(targets: Sequence[Target], name: str) -> Target: + for target in targets: + if target.name == name: + return target + raise FuzzNightlyError(f"unknown fuzz target {name!r}; see {REGISTRY_PATH}") + + +def matrix_names(targets: Sequence[Target], only: str = "") -> list[str]: + if only: + return [find_target(targets, only).name] + return [target.name for target in targets] + + +def fuzz_command(name: str, iterations: int | None) -> list[str]: + """`zig build fuzz` for one target. Without iterations it only replays the + target's saved inputs.""" + command = ["zig", "build", "fuzz", "-Doptimize=ReleaseSafe", f"-Dfuzz-target={name}"] + if iterations is not None: + command.append(f"--fuzz={iterations}") + return command + + +def run_process(command: Sequence[str], cwd: pathlib.Path, timeout: float) -> Outcome: + """Runs `command` in its own process group, so a timeout also stops the + fuzzer processes the build runner started.""" + process = subprocess.Popen( + list(command), + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", + start_new_session=True, + ) + try: + output, _ = process.communicate(timeout=timeout) + return Outcome(output, process.returncode) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + output, _ = process.communicate() + return Outcome(output, None) + + +def classify(outcome: Outcome, test: str) -> str: + if outcome.returncode is None: + return STATUS_HANG + lines = outcome.output.splitlines() + if any(line.startswith("error: test '") for line in lines): + return STATUS_FAILURE + if outcome.returncode != 0: + return STATUS_BUILD_ERROR + if not any(line.startswith('Fuzz test: "') and test in line for line in lines): + return STATUS_MISSING_TEST + return STATUS_OK + + +def clear_fuzzer_inputs(cache: pathlib.Path) -> None: + """Removes inputs left by earlier runs, keeping the fuzzer's corpus.""" + inputs = cache / "f" + if not inputs.is_dir(): + return + for path in inputs.iterdir(): + if path.is_file() and (path.name == "crash" or INPUT_FILE_NAME.fullmatch(path.name)): + path.unlink() + + +def recover_inputs(cache: pathlib.Path) -> list[bytes]: + """Every input the fuzzer was running when it stopped, without duplicates, + starting with the one the build runner saved when it is not empty.""" + inputs_dir = cache / "f" + found: list[bytes] = [] + crash = inputs_dir / "crash" + if crash.is_file() and crash.stat().st_size > 0: + found.append(crash.read_bytes()) + if inputs_dir.is_dir(): + for path in sorted(inputs_dir.iterdir()): + if not path.is_file() or not INPUT_FILE_NAME.fullmatch(path.name): + continue + data = path.read_bytes() + if len(data) < INPUT_HEADER.size: + continue + length = INPUT_HEADER.unpack_from(data)[3] + payload = data[INPUT_HEADER.size : INPUT_HEADER.size + length] + if len(payload) == length: + found.append(payload) + unique: list[bytes] = [] + for data in found: + if data not in unique: + unique.append(data) + return unique + + +def input_digest(data: bytes) -> str: + return hashlib.sha256(data).hexdigest()[:12] + + +def confirm_reproducer( + target: Target, + root: pathlib.Path, + candidates: Sequence[bytes], + timeout: float, + runner: Runner, +) -> bytes | None: + """Replays each candidate as a saved input of the target and returns the + first one that fails or hangs. Returns None when the target's replay + already fails without a candidate, since no candidate can then be blamed.""" + if not candidates: + return None + if runner(fuzz_command(target.name, None), root, timeout).returncode != 0: + return None + folder = root / CORPUS_PATH / target.name + # Deepest first, so the cleanup below can remove each one once it is empty. + created = [path for path in (folder, *folder.parents) if path.is_relative_to(root) and not path.exists()] + folder.mkdir(parents=True, exist_ok=True) + try: + for data in candidates: + path = folder / f"nightly-{input_digest(data)}" + path.write_bytes(data) + try: + outcome = runner(fuzz_command(target.name, None), root, timeout) + finally: + path.unlink() + if outcome.returncode != 0: + return data + finally: + for path in created: + if path.is_dir() and not any(path.iterdir()): + path.rmdir() + return None + + +def fuzz_target( + target: Target, + root: pathlib.Path, + out_dir: pathlib.Path, + *, + budget: float = DEFAULT_BUDGET_SECONDS, + round_timeout: float = DEFAULT_ROUND_TIMEOUT_SECONDS, + replay_timeout: float = DEFAULT_REPLAY_TIMEOUT_SECONDS, + runner: Runner = run_process, + clock: Callable[[], float] = time.monotonic, +) -> dict: + """Fuzzes `target` until the budget is spent or a round does not end + cleanly, then writes `fuzz.log`, `result.json`, and any confirmed + reproducer to `out_dir`. Returns the result.""" + out_dir.mkdir(parents=True, exist_ok=True) + cache = root / ".zig-cache" + clear_fuzzer_inputs(cache) + deadline = clock() + budget + log: list[str] = [] + rounds = 0 + status = STATUS_OK + while True: + outcome = runner(fuzz_command(target.name, target.iterations), root, round_timeout) + rounds += 1 + status = classify(outcome, target.test) + log.append(f"=== round {rounds}: {status} ===\n{outcome.output}") + print(f"{target.name}: round {rounds}: {status}", flush=True) + if status != STATUS_OK or clock() >= deadline: + break + (out_dir / "fuzz.log").write_text("\n".join(log)) + + result: dict = { + "target": target.name, + "test": target.test, + "status": status, + "rounds": rounds, + "round_timeout_seconds": round_timeout, + "reproducer": None, + } + if status in (STATUS_FAILURE, STATUS_HANG): + reproducer = confirm_reproducer( + target, root, recover_inputs(cache), replay_timeout, runner + ) + if reproducer is not None: + name = f"{target.name}-{input_digest(reproducer)}" + (out_dir / name).write_bytes(reproducer) + result["reproducer"] = name + (out_dir / "result.json").write_text(json.dumps(result, indent=2) + "\n") + return result + + +def issue_title(name: str) -> str: + return f"Nightly fuzz failure: {name}" + + +def issue_body(result: dict, run_url: str, commit: str) -> str: + name = result["target"] + status = result["status"] + what = { + STATUS_FAILURE: f"found an input that fails `{result['test']}`", + STATUS_HANG: ( + f"found an input that hangs `{result['test']}`: a round ran past " + f"{int(result['round_timeout_seconds'])} seconds" + ), + STATUS_BUILD_ERROR: "could not build or start the fuzz target", + STATUS_MISSING_TEST: ( + f"ran no fuzz test named `{result['test']}`, so the entry in " + f"`{REGISTRY_PATH}` no longer matches a test" + ), + }.get(status, "failed before it started fuzzing") + lines = [ + f"Nightly fuzzing of `{name}` {what}.", + "", + f"- Run: {run_url}", + f"- Commit: `{commit}`", + f"- Rounds: {result['rounds']}", + ] + reproducer = result.get("reproducer") + if reproducer: + lines += [ + f"- Reproducer: `{reproducer}` in the `fuzz-{name}` artifact", + "", + "To reproduce:", + "", + f"1. Download the `fuzz-{name}` artifact from the run.", + f"2. Copy `{reproducer}` into `{CORPUS_PATH}/{name}/`.", + f"3. Run `zig build fuzz -Dfuzz-target={name}`. It builds in Debug, which adds allocation stack traces to leak reports.", + "4. Fix the bug and keep the file in the fix PR, so `zig build test` replays it on every PR.", + ] + elif status in (STATUS_FAILURE, STATUS_HANG): + lines += [ + "- Reproducer: none of the recovered inputs reproduced it on replay. `fuzz.log` in the artifact has the failure.", + ] + else: + lines += ["", "`fuzz.log` in the artifact has the output."] + return "\n".join(lines) + "\n" + + +def read_result(out_dir: pathlib.Path, target: str) -> dict: + path = out_dir / "result.json" + if path.is_file(): + return json.loads(path.read_text()) + return {"target": target, "test": "", "status": "setup", "rounds": 0, "reproducer": None} + + +GhRunner = Callable[[Sequence[str], str | None], str] + + +def run_gh(args: Sequence[str], stdin: str | None) -> str: + completed = subprocess.run( + ["gh", *args], input=stdin, text=True, capture_output=True, check=True + ) + return completed.stdout + + +def report( + result: dict, + *, + repo: str, + run_url: str, + commit: str, + gh: GhRunner = run_gh, +) -> str: + """Comments on the target's open issue, or opens one. Returns the action.""" + title = issue_title(result["target"]) + body = issue_body(result, run_url, commit) + listed = gh( + [ + "issue", "list", "--repo", repo, "--state", "open", + "--search", f'"{title}" in:title', "--json", "number,title", "--limit", "100", + ], + None, + ) + for issue in json.loads(listed or "[]"): + if issue["title"] == title: + gh(["issue", "comment", str(issue["number"]), "--repo", repo, "--body-file", "-"], body) + return f"commented on #{issue['number']}" + gh(["issue", "create", "--repo", repo, "--title", title, "--body-file", "-"], body) + return "opened an issue" + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python3 -m scripts.fuzz_nightly") + commands = parser.add_subparsers(dest="command", required=True) + + matrix = commands.add_parser("matrix", help="print the job matrix as a GitHub output line") + matrix.add_argument("--only", default="", help="limit the matrix to one target") + + run = commands.add_parser("run", help="fuzz one target") + run.add_argument("--target", required=True) + run.add_argument("--out", required=True, type=pathlib.Path) + run.add_argument("--budget-seconds", type=float, default=DEFAULT_BUDGET_SECONDS) + run.add_argument("--round-timeout-seconds", type=float, default=DEFAULT_ROUND_TIMEOUT_SECONDS) + + report_parser = commands.add_parser("report", help="open or update the target's issue") + report_parser.add_argument("--target", required=True) + report_parser.add_argument("--out", required=True, type=pathlib.Path) + report_parser.add_argument("--repo", required=True) + report_parser.add_argument("--run-url", required=True) + report_parser.add_argument("--commit", required=True) + + args = parser.parse_args(argv) + try: + if args.command == "matrix": + names = matrix_names(load_targets(), args.only) + print(f"targets={json.dumps(names)}") + return 0 + if args.command == "run": + target = find_target(load_targets(), args.target) + result = fuzz_target( + target, + REPO_ROOT, + args.out, + budget=args.budget_seconds, + round_timeout=args.round_timeout_seconds, + ) + return 0 if result["status"] == STATUS_OK else 1 + result = read_result(args.out, args.target) + print(report(result, repo=args.repo, run_url=args.run_url, commit=args.commit)) + return 0 + except FuzzNightlyError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_fuzz_nightly.py b/scripts/tests/test_fuzz_nightly.py new file mode 100644 index 000000000..74487e357 --- /dev/null +++ b/scripts/tests/test_fuzz_nightly.py @@ -0,0 +1,270 @@ +from __future__ import annotations + +import json +import pathlib +import tempfile +import unittest + +from scripts import fuzz_nightly +from scripts.fuzz_nightly import ( + INPUT_HEADER, + Outcome, + Target, + classify, + confirm_reproducer, + fuzz_target, + issue_body, + load_targets, + matrix_names, + recover_inputs, + report, +) + +TARGET = Target( + name="sse_framing", + module="fx", + test="provider framing fuzzes chunk-invariant event data", + iterations=1000, +) + +CLEAN_ROUND = """\ +======= FUZZING REPORT ======= +Step: run test fuzz +Fuzz test: "gateway.sse.test.provider framing fuzzes chunk-invariant event data" (153f9ee4d2a0f02a) +Runs: 0 -> 4113 +============================== +""" + +# What Zig 0.17 prints, with exit status 0, when fuzzing finds a failure. +FAILED_ROUND = ( + "error: test 'gateway.sse.test.provider framing fuzzes chunk-invariant event data' " + "exited with code 1; input saved to '.zig-cache/f/crash'\n" + CLEAN_ROUND +) + + +def write_input_file(path: pathlib.Path, payload: bytes, padding: int = 64) -> None: + header = INPUT_HEADER.pack(0x153F9EE4D2A0F02A, 0, 0, len(payload)) + path.write_bytes(header + payload + b"\x00" * padding) + + +class FakeRunner: + """Answers `zig build fuzz` calls from a script of outcomes. Replays fail + when the target's corpus folder holds the bad input.""" + + def __init__(self, rounds: list[Outcome], bad_input: bytes | None = None) -> None: + self.rounds = rounds + self.bad_input = bad_input + self.calls: list[list[str]] = [] + + def __call__(self, command, cwd: pathlib.Path, timeout: float) -> Outcome: + self.calls.append(list(command)) + if any(arg.startswith("--fuzz=") for arg in command): + return self.rounds.pop(0) + folder = cwd / "tests" / "fuzz" / "corpus" / TARGET.name + saved = [path.read_bytes() for path in folder.iterdir()] if folder.is_dir() else [] + if self.bad_input is not None and self.bad_input in saved: + return Outcome("error: replay failed\n", 1) + return Outcome("", 0) + + +class RegistryTests(unittest.TestCase): + def test_registry_names_are_unique_and_matrix_lists_them(self) -> None: + targets = load_targets() + + names = matrix_names(targets) + + self.assertEqual(len(names), len(set(names))) + self.assertIn("sse_framing", names) + self.assertEqual(["sse_framing"], matrix_names(targets, "sse_framing")) + + def test_unknown_target_is_an_error(self) -> None: + with self.assertRaises(fuzz_nightly.FuzzNightlyError): + matrix_names(load_targets(), "no_such_target") + + def test_every_target_names_one_fuzz_test_its_binary_reaches(self) -> None: + src = fuzz_nightly.REPO_ROOT / "src" + fuzz_root = (src / "fuzz_targets.zig").read_text() + session_manager = src / "core" / "session_manager" + sources = {path: path.read_text() for path in src.rglob("*.zig")} + for target in load_targets(): + with self.subTest(target=target.name): + declaration = f'test "{target.test}" {{' + files = [path for path, text in sources.items() if declaration in text] + self.assertEqual(1, len(files), f"{target.name}: one file must declare {declaration!r}") + body = sources[files[0]].split(declaration, 1)[1].split('\ntest "', 1)[0] + self.assertIn("testing.fuzz(", body) + in_session_manager = files[0].is_relative_to(session_manager) + self.assertEqual(target.module == "session_manager", in_session_manager) + if target.module == "fx": + self.assertIn(f'@import("{files[0].relative_to(src).as_posix()}")', fuzz_root) + + +class ClassifyTests(unittest.TestCase): + def test_clean_round_is_ok(self) -> None: + self.assertEqual("ok", classify(Outcome(CLEAN_ROUND, 0), TARGET.test)) + + def test_failure_is_found_even_with_exit_status_zero(self) -> None: + self.assertEqual("failure", classify(Outcome(FAILED_ROUND, 0), TARGET.test)) + + def test_timeout_is_a_hang(self) -> None: + self.assertEqual("hang", classify(Outcome(CLEAN_ROUND, None), TARGET.test)) + + def test_nonzero_exit_without_a_test_failure_is_a_build_error(self) -> None: + self.assertEqual("build-error", classify(Outcome("error: compile failed\n", 1), TARGET.test)) + + def test_round_without_the_registered_test_is_reported(self) -> None: + self.assertEqual("missing-test", classify(Outcome("Build Summary: 1/1\n", 0), TARGET.test)) + + +class RecoverInputsTests(unittest.TestCase): + def test_reads_inputs_from_fuzzer_files_when_the_saved_crash_is_empty(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + inputs = pathlib.Path(tmp) / "f" + inputs.mkdir() + (inputs / "crash").write_bytes(b"") + write_input_file(inputs / "in0", b"\x4d\x00\x00\x00bad") + write_input_file(inputs / "in1", b"\x4d\x00\x00\x00bad") + write_input_file(inputs / "in2", b"other") + (inputs / "d1e77c9a9250cf56").mkdir() + + self.assertEqual([b"\x4d\x00\x00\x00bad", b"other"], recover_inputs(pathlib.Path(tmp))) + + def test_a_non_empty_saved_crash_comes_first(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + inputs = pathlib.Path(tmp) / "f" + inputs.mkdir() + (inputs / "crash").write_bytes(b"saved") + write_input_file(inputs / "in0", b"other") + + self.assertEqual([b"saved", b"other"], recover_inputs(pathlib.Path(tmp))) + + +class ConfirmReproducerTests(unittest.TestCase): + def test_returns_the_candidate_that_fails_on_replay_and_cleans_up(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + runner = FakeRunner([], bad_input=b"bad") + + found = confirm_reproducer(TARGET, root, [b"good", b"bad"], 60, runner) + + self.assertEqual(b"bad", found) + self.assertEqual([], list(root.iterdir())) + + def test_keeps_corpus_folders_that_already_existed(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + existing = root / "tests" / "fuzz" / "corpus" + existing.mkdir(parents=True) + + confirm_reproducer(TARGET, root, [b"bad"], 60, FakeRunner([], bad_input=b"bad")) + + self.assertEqual([], list(existing.iterdir())) + + def test_no_candidate_is_blamed_when_the_replay_already_fails(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + def always_fails(command, cwd, timeout): + return Outcome("", 1) + + self.assertIsNone(confirm_reproducer(TARGET, pathlib.Path(tmp), [b"bad"], 60, always_fails)) + + +class FuzzTargetTests(unittest.TestCase): + def test_runs_rounds_until_the_budget_is_spent(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + ticks = iter([0, 10, 20, 30]) + runner = FakeRunner([Outcome(CLEAN_ROUND, 0)] * 3) + + result = fuzz_target( + TARGET, root, root / "out", budget=25, runner=runner, clock=lambda: next(ticks) + ) + + self.assertEqual("ok", result["status"]) + self.assertEqual(3, result["rounds"]) + self.assertEqual("ok", json.loads((root / "out" / "result.json").read_text())["status"]) + + def test_failure_writes_the_confirmed_reproducer(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + inputs = root / ".zig-cache" / "f" + inputs.mkdir(parents=True) + bad = b"\x4d\x00\x00\x00bad" + replay = FakeRunner([], bad_input=bad) + + def runner(command, cwd, timeout): + if any(arg.startswith("--fuzz=") for arg in command): + write_input_file(inputs / "in0", bad) + return Outcome(FAILED_ROUND, 0) + return replay(command, cwd, timeout) + + result = fuzz_target(TARGET, root, root / "out", runner=runner, clock=lambda: 0) + + self.assertEqual("failure", result["status"]) + self.assertEqual(bad, (root / "out" / result["reproducer"]).read_bytes()) + + def test_stale_inputs_from_earlier_runs_are_removed_first(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + inputs = root / ".zig-cache" / "f" + inputs.mkdir(parents=True) + write_input_file(inputs / "in0", b"stale") + (inputs / "crash").write_bytes(b"stale") + corpus = inputs / "153f9ee4d2a0f02a" + corpus.mkdir() + + fuzz_target( + TARGET, root, root / "out", budget=0, runner=FakeRunner([Outcome(CLEAN_ROUND, 0)]), clock=lambda: 0 + ) + + self.assertEqual([corpus], list(inputs.iterdir())) + + +class ReportTests(unittest.TestCase): + RESULT = { + "target": "sse_framing", + "test": TARGET.test, + "status": "failure", + "rounds": 2, + "round_timeout_seconds": 600, + "reproducer": "sse_framing-0123456789ab", + } + + def test_body_explains_how_to_reproduce(self) -> None: + body = issue_body(self.RESULT, "https://example.invalid/run/1", "abc123") + + self.assertIn("fails `provider framing", body) + self.assertIn("tests/fuzz/corpus/sse_framing/", body) + self.assertIn("zig build fuzz -Dfuzz-target=sse_framing", body) + + def test_comments_on_the_open_issue_with_the_same_title(self) -> None: + calls: list[list[str]] = [] + + def gh(args, stdin): + calls.append(list(args)) + if args[:2] == ["issue", "list"]: + return json.dumps([ + {"number": 7, "title": "Nightly fuzz failure: sse_framing_v2"}, + {"number": 9, "title": "Nightly fuzz failure: sse_framing"}, + ]) + return "" + + action = report(self.RESULT, repo="vercel-labs/fx", run_url="u", commit="c", gh=gh) + + self.assertEqual("commented on #9", action) + self.assertEqual(["issue", "comment", "9"], calls[-1][:3]) + + def test_opens_an_issue_when_none_is_open(self) -> None: + calls: list[list[str]] = [] + + def gh(args, stdin): + calls.append(list(args)) + return "[]" if args[:2] == ["issue", "list"] else "" + + action = report(self.RESULT, repo="vercel-labs/fx", run_url="u", commit="c", gh=gh) + + self.assertEqual("opened an issue", action) + self.assertIn("Nightly fuzz failure: sse_framing", calls[-1]) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/acp/mcp_servers.zig b/src/acp/mcp_servers.zig index 097ec96b9..355186544 100644 --- a/src/acp/mcp_servers.zig +++ b/src/acp/mcp_servers.zig @@ -560,12 +560,12 @@ fn fuzzParse(_: void, smith: *std.testing.Smith) !void { test "ACP MCP parser handles fuzzed request bytes" { try std.testing.fuzz({}, fuzzParse, .{ - .corpus = &.{ + .corpus = &(.{ "", "{}", "{\"mcpServers\":[]}", "{\"mcpServers\":[{\"name\":\"fixture\",\"command\":\"/bin/echo\",\"args\":[],\"env\":[]}]}", "{\"mcpServers\":[{\"type\":\"http\",\"name\":\"fixture\",\"url\":\"http://localhost:4321/mcp\",\"headers\":[]}]}", - }, + } ++ @import("fuzz_corpus").acp_mcp_servers.*), }); } diff --git a/src/core/agent/runtime/vision_contracts.zig b/src/core/agent/runtime/vision_contracts.zig index 86a8855ab..d0f4654c0 100644 --- a/src/core/agent/runtime/vision_contracts.zig +++ b/src/core/agent/runtime/vision_contracts.zig @@ -1017,11 +1017,11 @@ fn fuzz_vision_json(_: void, smith: *std.testing.Smith) !void { test "Vision JSON parsers tolerate arbitrary bounded bytes" { try std.testing.fuzz({}, fuzz_vision_json, .{ - .corpus = &.{ + .corpus = &(.{ "{\"image_ids\":[1],\"focus\":\"inspect\"}", "{\"images\":[{\"image_id\":1,\"status\":\"failed\",\"error\":\"vision_unavailable\"}]}", "{", - }, + } ++ @import("fuzz_corpus").vision_json.*), }); } diff --git a/src/core/images/image_data.zig b/src/core/images/image_data.zig index 17c61ede9..04f3e949f 100644 --- a/src/core/images/image_data.zig +++ b/src/core/images/image_data.zig @@ -544,7 +544,7 @@ test "encoded image dimensions agree with raw headers at every truncation" { test "image dimension parsing stays bounded on arbitrary bytes" { try std.testing.fuzz({}, fuzzImageDimensions, .{ - .corpus = &.{ + .corpus = &(.{ &testPngHeader(3420, 2224), &testJpeg(2001, 1), &testGif(1, 1), @@ -556,7 +556,7 @@ test "image dimension parsing stays bounded on arbitrary bytes" { "\x89PNG\r\n\x1a\n\x00\x00\x00\x00IHDR\xff\xff\xff\xff\xff\xff\xff\xff", // RIFF header naming WebP with no chunk. "RIFF\x00\x00\x00\x00WEBP", - }, + } ++ @import("fuzz_corpus").image_dimensions.*), }); } diff --git a/src/core/mcp/elicitation.zig b/src/core/mcp/elicitation.zig index 73f5e0107..d93e80c98 100644 --- a/src/core/mcp/elicitation.zig +++ b/src/core/mcp/elicitation.zig @@ -2143,12 +2143,12 @@ fn fuzzElicitationJson(_: void, smith: *std.testing.Smith) !void { test "elicitation request and response parsers stay bounded under fuzzed bytes" { try std.testing.fuzz({}, fuzzElicitationJson, .{ - .corpus = &.{ + .corpus = &(.{ "{}", "{\"mode\":\"form\",\"requestedSchema\":{}}", "{\"action\":\"accept\",\"content\":{}}", "{\"mode\":\"url\",\"url\":\"https://example.test\"}", "{\"jsonrpc\":\"2.0\",\"method\":\"notifications/elicitation/complete\",\"params\":{\"elicitationId\":\"id\"}}", - }, + } ++ @import("fuzz_corpus").mcp_elicitation.*), }); } diff --git a/src/core/session/session_codec.zig b/src/core/session/session_codec.zig index 0288c17e5..46ccce98a 100644 --- a/src/core/session/session_codec.zig +++ b/src/core/session/session_codec.zig @@ -5951,7 +5951,7 @@ test "an old session's usage field is skipped in its place, and the rest loads" } test "durable session optional fields handle fuzzed ownership paths" { - try std.testing.fuzz({}, fuzzDurableSessionOptionalFields, .{ .corpus = &.{ + try std.testing.fuzz({}, fuzzDurableSessionOptionalFields, .{ .corpus = &(.{ "{\"id\":\"session\",\"origin_workspace_root\":\"/tmp/origin\",\"workspace_root\":\"/tmp/current\",\"created_at_ms\":1,\"updated_at_ms\":1,\"conversation_language\":\"en\",\"preferences\":{\"model\":\"test/model\",\"effort\":\"auto\",\"fast_mode\":false},\"history\":[],\"total_input_tokens\":0,\"total_output_tokens\":0}", "{\"id\":\"session\",\"origin_workspace_root\":\"/tmp/origin\",\"workspace_root\":\"/tmp/current\",\"created_at_ms\":1,\"updated_at_ms\":1,\"conversation_language\":\"en\",\"preferences\":{\"model\":\"test/model\",\"effort\":\"auto\",\"fast_mode\":false},\"history\":[],\"total_input_tokens\":0,\"total_output_tokens\":0,\"context_history_start\":0,\"last_subagent_work_id\":\"work\"}", "{\"id\":\"session\",\"origin_workspace_root\":\"/tmp/origin\",\"workspace_root\":\"/tmp/current\",\"created_at_ms\":1,\"updated_at_ms\":1,\"conversation_language\":\"en\",\"preferences\":{\"model\":\"test/model\",\"effort\":\"auto\",\"fast_mode\":false},\"history\":[],\"total_input_tokens\":0,\"total_output_tokens\":0,\"last_subagent_work_id\":\"work\",\"context_history_start\":0}", @@ -5960,17 +5960,17 @@ test "durable session optional fields handle fuzzed ownership paths" { "{\"last_subagent_work_id\":null}", "{\"usage\":null,\"last_subagent_work_id\":\"work\"}", "{\"last_subagent_work_id\":\"work\",\"usage\":null}", - } }); + } ++ @import("fuzz_corpus").session_optional_fields.*) }); } test "per-turn work provenance parser handles fuzzed ownership paths" { - try std.testing.fuzz({}, fuzzWorkProvenanceHistoryTurn, .{ .corpus = &.{ + try std.testing.fuzz({}, fuzzWorkProvenanceHistoryTurn, .{ .corpus = &(.{ "{\"kind\":\"assistant\",\"user\":{\"text\":\"prompt\",\"images\":[],\"work_id\":\"work-1\"},\"assistant\":\"reply\",\"execution\":{\"schema_version\":3,\"tool_steps\":[],\"files\":[]}}", "{\"kind\":\"assistant\",\"user\":{\"text\":\"prompt\",\"images\":[]},\"assistant\":\"reply\",\"execution\":{\"schema_version\":3,\"tool_steps\":[],\"files\":[]}}", "{\"kind\":\"assistant\",\"user\":{\"text\":\"prompt\",\"images\":[],\"work_id\":\"\"}}", "", "{}", - } }); + } ++ @import("fuzz_corpus").session_work_provenance.*) }); } fn fuzzWorkProvenanceHistoryTurn(_: void, smith: *std.testing.Smith) !void { diff --git a/src/core/session/session_event.zig b/src/core/session/session_event.zig index 293f43aad..644a40874 100644 --- a/src/core/session/session_event.zig +++ b/src/core/session/session_event.zig @@ -3171,12 +3171,12 @@ test "conversation frame rejects an oversized diff content handle" { test "conversation frame decoder stays bounded under fuzzed bytes" { try std.testing.fuzz({}, fuzzConversationFrame, .{ - .corpus = &.{ + .corpus = &(.{ "", "{}\n", "{\"schema_version\":1}\n", "{\"schema_version\":1,\"seq\":1,\"timestamp_ms\":0,\"event\":{\"assistant\":{\"text\":\"ok\"}}}\n", - }, + } ++ @import("fuzz_corpus").session_conversation_frame.*), }); } diff --git a/src/core/session_manager/log.zig b/src/core/session_manager/log.zig index e1137235a..c8cc3464f 100644 --- a/src/core/session_manager/log.zig +++ b/src/core/session_manager/log.zig @@ -773,10 +773,10 @@ test "any prefix of a valid log scans to its last complete line" { } test "scanning never panics on arbitrary bytes" { - try testing.fuzz({}, fuzzScan, .{ .corpus = &.{ + try testing.fuzz({}, fuzzScan, .{ .corpus = &(.{ "{\"v\":1,\"seq\":1,\"ts\":0,\"kind\":\"item\",\"crc\":\"00000000\"}\n", "\n\n\n", - } }); + } ++ @import("build_options").fuzz_corpus_session_log_scan.*) }); } fn fuzzScan(_: void, smith: *testing.Smith) anyerror!void { diff --git a/src/core/session_manager/schema.zig b/src/core/session_manager/schema.zig index 08142a955..19685b544 100644 --- a/src/core/session_manager/schema.zig +++ b/src/core/session_manager/schema.zig @@ -806,10 +806,10 @@ test "ids follow v1's alphabet and avoid the reserved names" { } test "header parsing never panics on arbitrary bytes" { - try testing.fuzz({}, fuzzHeader, .{ .corpus = &.{ + try testing.fuzz({}, fuzzHeader, .{ .corpus = &(.{ "{\"v\":1,\"seq\":1,\"ts\":0,\"kind\":\"item\"", "{\"v\":99999999999,\"seq\":1", - } }); + } ++ @import("build_options").fuzz_corpus_session_schema_header.*) }); } fn fuzzHeader(_: void, smith: *testing.Smith) anyerror!void { diff --git a/src/core/shared/display_width.zig b/src/core/shared/display_width.zig index 63683f326..565229b43 100644 --- a/src/core/shared/display_width.zig +++ b/src/core/shared/display_width.zig @@ -828,14 +828,14 @@ test "display unit clipping preserves sequence boundaries" { test "fuzz display unit boundaries" { try std.testing.fuzz({}, fuzzDisplayUnitBoundaries, .{ - .corpus = &.{ + .corpus = &(.{ "", "plain text", "\xff\xf0\x28\x8c\x28", "\u{2600}\u{FE0E}\u{2600}\u{FE0F}", "\u{1F44D}\u{1F3FD}\u{1F1FA}\u{1F1F8}", "#\u{FE0F}\u{20E3}\u{1F469}\u{200D}\u{1F4BB}", - }, + } ++ @import("fuzz_corpus").display_width.*), }); } diff --git a/src/core/shared/types.zig b/src/core/shared/types.zig index d443ac00f..9b9da5d3a 100644 --- a/src/core/shared/types.zig +++ b/src/core/shared/types.zig @@ -3262,13 +3262,13 @@ fn fuzzToolArgumentDiagnostic(_: void, smith: *std.testing.Smith) anyerror!void test "fuzz tool argument diagnostic" { try std.testing.fuzz({}, fuzzToolArgumentDiagnostic, .{ - .corpus = &.{ + .corpus = &(.{ "{\"request\":{\"task\":\"cut", "{\"a\":1,}", "{\"a\":1,\"a\":2}", "{\"t\":\"\xc3\xa9\xe2\x82", "\xff\x00{", - }, + } ++ @import("fuzz_corpus").tool_argument_diagnostic.*), }); } diff --git a/src/core/shell_command/command_effect.zig b/src/core/shell_command/command_effect.zig index 9d5fb5525..f313fe190 100644 --- a/src/core/shell_command/command_effect.zig +++ b/src/core/shell_command/command_effect.zig @@ -79,8 +79,9 @@ fn isUnsupportedAutoControlOperator(token: command_lex.ArgvToken) bool { std.mem.eql(u8, token.value, "}"); } -fn reversibleAutoStage(raw_tokens: []const command_lex.ArgvToken) bool { - var tokens = raw_tokens; +/// `tokens` without a trailing unquoted `2>&1`, the one redirection a +/// review-free stage may end with. +fn withoutStderrToStdout(tokens: []const command_lex.ArgvToken) []const command_lex.ArgvToken { if (tokens.len >= 3 and !tokens[tokens.len - 3].quoted and !tokens[tokens.len - 2].quoted and @@ -89,8 +90,13 @@ fn reversibleAutoStage(raw_tokens: []const command_lex.ArgvToken) bool { std.mem.eql(u8, tokens[tokens.len - 2].value, ">&") and std.mem.eql(u8, tokens[tokens.len - 1].value, "1")) { - tokens = tokens[0 .. tokens.len - 3]; + return tokens[0 .. tokens.len - 3]; } + return tokens; +} + +fn reversibleAutoStage(raw_tokens: []const command_lex.ArgvToken) bool { + const tokens = withoutStderrToStdout(raw_tokens); if (tokens.len == 0) return false; for (tokens) |token| { if (isUnsupportedAutoOperator(token) or @@ -1916,3 +1922,525 @@ test "planner bounds ls operands at sixty four" { try accepted.appendSlice(std.testing.allocator, " overflow"); try expectApproval(accepted.items, false, .linux, .unsupported_argument); } + +const parity_programs = [_][]const u8{ + "printf", "pwd", "ls", "wc", "cat", "head", "tail", "grep", + "git", "node", "which", "npm", "bun", "pnpm", "yarn", "zig", +}; + +/// Builtins that a function can replace in every shell here, including bash +/// in POSIX mode, which is `/bin/sh` on macOS. +const parity_guarded_builtins = [_][]const u8{ + "kill", "ulimit", "umask", "cd", "command", "builtin", "hash", "alias", + "unalias", "wait", "fg", "bg", "jobs", "fc", "read", +}; + +/// Special builtins that bash and zsh, outside POSIX mode, also let a +/// function replace. +const parity_guarded_special_builtins = [_][]const u8{ + "enable", "disable", "disown", "suspend", "history", "logout", + "pushd", "popd", "source", "exec", "eval", "trap", +}; + +const ParityShell = struct { + name: []const u8, + paths: []const []const u8, + flags: []const []const u8, + guards_special_builtins: bool, + /// Runs before anything else in the script. fx turns off zsh's `equals` + /// option before every command (`zsh_command_prelude` in + /// `src/core/terminal/shell_resolver.zig`), so zsh here does too. + prelude: []const u8 = "", + /// Whether this shell stands in for options a user's startup files can + /// turn on. fx runs direct read-only commands without a shell, so only + /// the commands it hands to a shell are compared with this one. + user_options: bool = false, +}; + +/// `/bin/sh` is dash on Debian and Ubuntu and bash in POSIX mode on macOS, +/// so dash is listed on its own as well. fx replays a user's zsh glob +/// options, and `extendedglob` makes `^` and `#` glob operators, so zsh also +/// runs with it on. +const parity_shells = [_]ParityShell{ + .{ .name = "sh", .paths = &.{"/bin/sh"}, .flags = &.{"-c"}, .guards_special_builtins = false }, + .{ .name = "dash", .paths = &.{ "/bin/dash", "/usr/bin/dash" }, .flags = &.{"-c"}, .guards_special_builtins = false }, + .{ + .name = "bash", + .paths = &.{ "/bin/bash", "/usr/bin/bash" }, + .flags = &.{ "--norc", "--noprofile", "-c" }, + .guards_special_builtins = true, + }, + .{ + .name = "zsh", + .paths = &.{ "/bin/zsh", "/usr/bin/zsh" }, + .flags = &.{ "-f", "-c" }, + .guards_special_builtins = true, + .prelude = "\\builtin unsetopt equals\n", + }, + .{ + .name = "zsh with extendedglob", + .paths = &.{ "/bin/zsh", "/usr/bin/zsh" }, + .flags = &.{ "-f", "-c" }, + .guards_special_builtins = true, + .prelude = "\\builtin unsetopt equals\n\\builtin setopt extendedglob\n", + .user_options = true, + }, +}; + +const parity_recorder = "#!/bin/sh\nprintf '%s\\0' \"$#\" \"$@\" >> \"$FX_PARITY_LOG\"\n"; + +const ParityOrder = enum { + /// `&&` stages run one after another. + sequence, + /// Pipeline stages run at the same time, so their records can interleave. + any_order, +}; + +const ParityExpectation = struct { + /// Each stage's words, program first. + stages: []const []const []const u8, + order: ParityOrder, + /// Whether fx hands the command to a shell, rather than running it + /// itself. + runs_in_shell: bool, +}; + +const ParityRun = struct { + term: std.process.Child.Term, + /// One record per program call: its name, then its arguments. + records: []const []const []const u8, + stderr: []const u8, + /// Files the command left in its empty working directory. + created: []const []const u8, +}; + +const ParityMode = enum { must_run_without_review, if_run_without_review }; + +/// What fx parsed for each review-free path that accepts `command`. Empty +/// when both send it to review. Allocates from `arena`. +fn parityExpectations(arena: std.mem.Allocator, command: []const u8) ![]const ParityExpectation { + var expectations: std.ArrayList(ParityExpectation) = .empty; + if (try knownReversibleAutoCommand(arena, command, false)) { + // Splits the way `knownReversibleAutoCommand` does. + const argv = try command_lex.tokenize_argv(arena, std.mem.trim(u8, command, " \t")); + var stages: std.ArrayList([]const []const u8) = .empty; + var start: usize = 0; + for (0..argv.tokens.len + 1) |index| { + if (index < argv.tokens.len and !isStageSeparator(argv.tokens[index])) continue; + try stages.append(arena, try parityWords(arena, withoutStderrToStdout(argv.tokens[start..index]))); + start = index + 1; + } + try expectations.append(arena, .{ .stages = stages.items, .order = .sequence, .runs_in_shell = true }); + } + const admission = try planWithGit(arena, command, "/workspace", false, @import("builtin").target.os.tag, test_git); + if (admission == .direct_read_only) { + // Splits the way `planWithGit` does: one stage per pipeline segment. + const segments = try command_lex.pipe_segments(arena, std.mem.trim(u8, command, " \t\n")); + var stages: std.ArrayList([]const []const u8) = .empty; + for (segments) |segment| { + const argv = try command_lex.tokenize_argv(arena, segment.text); + try stages.append(arena, try parityWords(arena, argv.tokens)); + } + try expectations.append(arena, .{ .stages = stages.items, .order = .any_order, .runs_in_shell = false }); + } + return expectations.items; +} + +fn parityWords(arena: std.mem.Allocator, tokens: []const command_lex.ArgvToken) ![]const []const u8 { + const words = try arena.alloc([]const u8, tokens.len); + for (tokens, words) |token, *word| word.* = token.value; + return words; +} + +fn checkShellParity(command: []const u8, mode: ParityMode) !void { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const expectations = try parityExpectations(arena, command); + if (expectations.len == 0) { + if (mode == .if_run_without_review) return; + std.debug.print("expected fx to run this command without review: {s}\n", .{command}); + return error.TestExpectedReviewFreeCommand; + } + for (parity_shells) |shell| { + const path = try parityShellPath(shell) orelse continue; + const run = try runParityShell(arena, path, shell, command); + for (expectations) |expectation| { + if (shell.user_options and !expectation.runs_in_shell) continue; + try expectParity(arena, expectation, run, shell.name, command); + } + } +} + +/// Finds the shell, or returns null to skip it. CI installs every one, so a +/// missing shell there fails instead of quietly narrowing the check. +fn parityShellPath(shell: ParityShell) !?[]const u8 { + for (shell.paths) |path| { + std.Io.Dir.accessAbsolute(std.testing.io, path, .{}) catch continue; + return path; + } + if (@import("../shared/io.zig").getenv("CI") != null) { + std.debug.print("the shell parity tests need {s}\n", .{shell.name}); + return error.TestShellParityShellMissing; + } + return null; +} + +fn runParityShell( + arena: std.mem.Allocator, + shell_path: []const u8, + shell: ParityShell, + command: []const u8, +) !ParityRun { + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + for ([_][]const u8{ "cwd", "empty", "home" }) |name| try tmp.dir.createDirPath(io, name); + const root = try @import("../shared/io.zig").dirRealpathAlloc(arena, tmp.dir, "."); + const recorder = try std.Io.Dir.path.join(arena, &.{ root, "record" }); + { + var file = try tmp.dir.createFile(io, "record", .{ .truncate = true }); + defer file.close(io); + try file.writeStreamingAll(io, parity_recorder); + try file.setPermissions(io, std.Io.File.Permissions.fromMode(0o700)); + } + + var script: std.ArrayList(u8) = .empty; + // Before the functions, since one of them replaces `builtin`. + try script.appendSlice(arena, shell.prelude); + try appendParityFunctions(&script, arena, recorder, &parity_programs); + try appendParityFunctions(&script, arena, recorder, &parity_guarded_builtins); + if (shell.guards_special_builtins) { + try appendParityFunctions(&script, arena, recorder, &parity_guarded_special_builtins); + } + try script.appendSlice(arena, command); + try script.append(arena, '\n'); + + var environ: std.process.Environ.Map = .init(arena); + try environ.put("PATH", try std.Io.Dir.path.join(arena, &.{ root, "empty" })); + try environ.put("HOME", try std.Io.Dir.path.join(arena, &.{ root, "home" })); + try environ.put("FX_PARITY_LOG", try std.Io.Dir.path.join(arena, &.{ root, "log" })); + + var argv: std.ArrayList([]const u8) = .empty; + try argv.append(arena, shell_path); + try argv.appendSlice(arena, shell.flags); + try argv.append(arena, script.items); + const result = try std.process.run(arena, io, .{ + .argv = argv.items, + .cwd = .{ .path = try std.Io.Dir.path.join(arena, &.{ root, "cwd" }) }, + .environ_map = &environ, + .timeout = .{ .duration = .{ .raw = .fromSeconds(10), .clock = .awake } }, + }); + + const log = tmp.dir.readFileAlloc(io, "log", arena, .limited(1024 * 1024)) catch |err| switch (err) { + error.FileNotFound => "", + else => return err, + }; + var created: std.ArrayList([]const u8) = .empty; + var cwd = try tmp.dir.openDir(io, "cwd", .{ .iterate = true }); + defer cwd.close(io); + var entries = cwd.iterate(); + while (try entries.next(io)) |entry| try created.append(arena, try arena.dupe(u8, entry.name)); + + return .{ + .term = result.term, + .records = try parseParityLog(arena, log), + .stderr = result.stderr, + .created = created.items, + }; +} + +fn appendParityFunctions( + script: *std.ArrayList(u8), + arena: std.mem.Allocator, + recorder: []const u8, + names: []const []const u8, +) !void { + for (names) |name| { + try script.appendSlice(arena, name); + try script.appendSlice(arena, "() { "); + try appendShellSingleQuoted(script, arena, recorder); + try script.append(arena, ' '); + try script.appendSlice(arena, name); + try script.appendSlice(arena, " \"$@\"; }\n"); + } +} + +/// Each record is a count, then that many fields: the program's name and +/// its arguments. Every field ends with a NUL. +fn parseParityLog(arena: std.mem.Allocator, log: []const u8) ![]const []const []const u8 { + var records: std.ArrayList([]const []const u8) = .empty; + if (log.len == 0) return records.items; + if (log[log.len - 1] != 0) return error.TestTruncatedParityLog; + var fields = std.mem.splitScalar(u8, log[0 .. log.len - 1], 0); + while (fields.next()) |count_text| { + const record = try arena.alloc([]const u8, try std.fmt.parseInt(usize, count_text, 10)); + for (record) |*field| field.* = fields.next() orelse return error.TestTruncatedParityLog; + try records.append(arena, record); + } + return records.items; +} + +fn expectParity( + arena: std.mem.Allocator, + expectation: ParityExpectation, + run: ParityRun, + shell_name: []const u8, + command: []const u8, +) !void { + var expected = expectation.stages; + var actual = run.records; + if (expectation.order == .any_order) { + expected = try sortedParityRecords(arena, expected); + actual = try sortedParityRecords(arena, actual); + } + const exited_cleanly = switch (run.term) { + .exited => |code| code == 0, + else => false, + }; + if (exited_cleanly and run.created.len == 0 and sameParityRecords(expected, actual)) return; + + std.debug.print("shell parity mismatch in {s} for: {s}\n", .{ shell_name, command }); + printParityRecords("fx parsed", expectation.stages); + printParityRecords("shell ran", run.records); + std.debug.print(" term: {any}\n stderr: {s}\n", .{ run.term, run.stderr }); + for (run.created) |name| std.debug.print(" created: {s}\n", .{name}); + return error.TestShellParityMismatch; +} + +fn sortedParityRecords( + arena: std.mem.Allocator, + records: []const []const []const u8, +) ![]const []const []const u8 { + const sorted = try arena.dupe([]const []const u8, records); + std.mem.sort([]const []const u8, sorted, {}, parityRecordLessThan); + return sorted; +} + +fn parityRecordLessThan(_: void, lhs: []const []const u8, rhs: []const []const u8) bool { + for (lhs[0..@min(lhs.len, rhs.len)], rhs[0..@min(lhs.len, rhs.len)]) |left, right| { + switch (std.mem.order(u8, left, right)) { + .lt => return true, + .gt => return false, + .eq => {}, + } + } + return lhs.len < rhs.len; +} + +fn sameParityRecords(expected: []const []const []const u8, actual: []const []const []const u8) bool { + if (expected.len != actual.len) return false; + for (expected, actual) |expected_record, actual_record| { + if (expected_record.len != actual_record.len) return false; + for (expected_record, actual_record) |expected_field, actual_field| { + if (!std.mem.eql(u8, expected_field, actual_field)) return false; + } + } + return true; +} + +fn printParityRecords(label: []const u8, records: []const []const []const u8) void { + std.debug.print(" {s}:", .{label}); + for (records) |record| { + std.debug.print(" [", .{}); + for (record, 0..) |field, index| { + std.debug.print("{s}\"{f}\"", .{ if (index == 0) "" else " ", std.zig.fmtString(field) }); + } + std.debug.print("]", .{}); + } + std.debug.print("\n", .{}); +} + +/// Every form of command that fx runs without review. Each must still be +/// review-free and must run in every shell exactly as fx parsed it. +const parity_review_free_commands = [_][]const u8{ + "node -v", + "node --version", + "npm -v", + "which node", + "which node npm bun", + "git status", + "git status --short --branch", + "git remote -v", + "git worktree list", + "git worktree list --porcelain", + "git fetch", + "git fetch origin main", + "git fetch --all --tags", + "git fetch --depth=1 origin main", + "git fetch -q origin -- main", + "npm install", + "npm i", + "npm ci", + "npm test", + "npm install --save-dev typescript", + "npm run build", + "npm run lint -- --fix", + "bun -v", + "bun install", + "bun test", + "bun run dev", + "pnpm install", + "pnpm test", + "pnpm run build", + "yarn install", + "yarn test", + "yarn run lint", + "zig build", + "zig build test", + "zig build -Doptimize=ReleaseSafe", + "node -v && npm -v", + "git status && npm test", + "npm test 2>&1", + "git fetch 2>&1 && git status", + "npm run 'build:prod'", + "npm run \"build:prod\"", + "npm run 'with space'", + "git status", + "npm\ttest", + "pwd", + "pwd -P", + "ls", + "ls -la", + "ls -1 src", + "ls -- -dashed", + "ls 'with space'", + "wc -l", + "cat", + "head", + "head -n 5", + "tail -n 20", + "git diff", + "git log -n 5", + "printf 'hello\\n'", + "ls | wc -l", + "printf x | cat", + "git log -n 5 | head -n 3", +}; + +/// Near misses that fx may or may not run without review. When it does, +/// they must still run as fx parsed them. +const parity_probe_commands = [_][]const u8{ + "git fetch '&&' git status", + "git fetch \\&\\& git status --prune", + "npm test \\&\\& npm run build", + "npm test &\\&", + "npm test \\&&", + "npm run =ls", + "which =node", + "git status #note", + "npm run a#b", + "git fetch ^x .", + "git fetch --prun#e", + "npm install react@^18", + "npm test x##", + "ls ^x", + "git status&&npm test", + "npm run build 2> &1", + "npm run x\\ y", + "npm run \"a\\\"b\"", + "zig build -Dname=value", + "npm run !x", + "git fetch origin +main:main", + "npm run x,y", + "npm run %x", + "npm run ^x", + "ls =x", + "ls '#x'", + "printf '%s' '&&'", + "npm run 'a''b'", + "npm run \"\"", + "npm run ''", + "git status ''", + "npm run é", + "grep -n needle file.txt", + "grep 'two words' file.txt", + "git diff --stat", + "printf '%s\\n' a b", +}; + +test "shell parity: fx runs every review-free command form as sh, dash, bash, and zsh parse it" { + const os = @import("builtin").target.os.tag; + if (os != .macos and os != .linux) return error.SkipZigTest; + var failures: usize = 0; + for (parity_review_free_commands) |command| { + checkShellParity(command, .must_run_without_review) catch |err| switch (err) { + error.TestExpectedReviewFreeCommand, error.TestShellParityMismatch => failures += 1, + else => return err, + }; + } + for (parity_probe_commands) |command| { + checkShellParity(command, .if_run_without_review) catch |err| switch (err) { + error.TestShellParityMismatch => failures += 1, + else => return err, + }; + } + try std.testing.expectEqual(@as(usize, 0), failures); +} + +/// Program words that start a generated stage, weighted toward commands fx +/// runs without review. +const parity_stage_starts = [_][]const u8{ + "node -v", "which", "git status", "git remote -v", "git worktree list", "git fetch", + "git diff", "git log", "npm install", "npm test", "npm run", "npm ci", + "bun test", "bun run", "pnpm install", "pnpm run", "yarn run", "zig build", + "pwd", "ls", "wc -l", "cat", "head -n 3", "tail", + "grep", "printf", +}; + +/// Pieces of generated words. None contains `/` or a builtin's name, so a +/// stage the shell reads differently cannot reach a real program. +const parity_fragments = [_][]const u8{ + "a", "build", "src", "x.txt", "-n", "-l", "--", "--flag", + "--o=v", "-Dx=1", "=x", "x=", "#", "#x", "a#b", "'q s'", + "\"d q\"", "''", "\"\"", "'", "\"", "\\", "\\ ", "\\#", + "&&", "'&&'", "\"&&\"", "&", "|", "||", "2>&1", ">", + "<", ">>", "2>", ";", "$x", "${x}", "`x`", "$(x)", + "*", "?", "[a]", "~", "{a,b}", "(x)", "!", "!x", + "%", "^", ",", ":", "@", "+", "\t", "\u{e9}", + "\u{a0}", "\x1b", "\\&\\&", "\\&", "^x", "x#", "x##", +}; + +fn writeParityCommand(smith: *std.testing.Smith, out: *std.Io.Writer) std.Io.Writer.Error!void { + const joiner: []const u8 = if (smith.boolWeighted(1, 1)) " && " else " | "; + const stage_count = smith.valueRangeAtMost(u8, 1, 4); + for (0..stage_count) |stage| { + if (stage > 0) try out.writeAll(joiner); + try out.writeAll(parity_stage_starts[smith.index(parity_stage_starts.len)]); + for (0..smith.valueRangeAtMost(u8, 0, 4)) |_| { + try out.writeByte(if (smith.boolWeighted(7, 1)) ' ' else '\t'); + for (0..smith.valueRangeAtMost(u8, 1, 3)) |_| { + if (smith.boolWeighted(1, 7)) { + try out.writeAll(parity_fragments[smith.index(parity_fragments.len)]); + continue; + } + var raw: [8]u8 = undefined; + for (raw[0..smith.slice(&raw)]) |byte| { + // No letters, so raw bytes cannot spell a program or + // builtin; no `/`, so they cannot name a path to one. + const safe = !std.ascii.isAlphabetic(byte) and byte != '/' and byte != 0; + try out.writeByte(if (safe) byte else '_'); + } + } + } + } +} + +fn fuzzShellParity(_: void, smith: *std.testing.Smith) !void { + const os = @import("builtin").target.os.tag; + if (os != .macos and os != .linux) return error.SkipZigTest; + var buffer: [512]u8 = undefined; + var command: std.Io.Writer = .fixed(&buffer); + writeParityCommand(smith, &command) catch |err| switch (err) { + // A full buffer ends the command early, which is still a command. + error.WriteFailed => {}, + }; + try checkShellParity(command.buffered(), .if_run_without_review); +} + +test "shell parity holds for generated commands" { + try std.testing.fuzz({}, fuzzShellParity, .{ + .corpus = &(.{""} ++ @import("fuzz_corpus").shell_parity.*), + }); +} diff --git a/src/core/skills/skill_contract.zig b/src/core/skills/skill_contract.zig index d9f46a89c..3e2666de3 100644 --- a/src/core/skills/skill_contract.zig +++ b/src/core/skills/skill_contract.zig @@ -1061,14 +1061,14 @@ test "parseSkillFile rejects indented continuations for recognized metadata" { test "parseSkillFile keeps fuzzed borrowed slices inside the input" { try std.testing.fuzz({}, fuzzParseSkillFile, .{ - .corpus = &.{ + .corpus = &(.{ "", "plain body", "---\nname: valid\n---\nbody", "---\r\nname: \"valid\"\r\n---\r\nbody", "---\nname: quoted\ndescription: 'single line'\n---\nbody", "---\nname: block\ndescription: >-\n first line\n second line\n---\nbody", - }, + } ++ @import("fuzz_corpus").skill_file.*), }); } diff --git a/src/core/skills/skill_invocation.zig b/src/core/skills/skill_invocation.zig index 6ba742663..e76b41b18 100644 --- a/src/core/skills/skill_invocation.zig +++ b/src/core/skills/skill_invocation.zig @@ -113,11 +113,11 @@ test "skill display name parsing stays bounded" { try std.testing.expect(name.len <= buffer.len); try std.testing.expect(skill_contract.invalidSkillNameCause(name) == null); } - }.check, .{ .corpus = &.{ + }.check, .{ .corpus = &(.{ content_name_prefix ++ "workflow\" location=\"/skills/workflow\">", discovery_model_notice ++ content_name_prefix ++ "name&value\" resource=\"SKILL.md\">", content_name_prefix ++ "bad&unknown;\" location=\"/skills/workflow\">", - } }); + } ++ @import("fuzz_corpus").skill_display_name.*) }); } /// Borrows a previously discovered skill inventory for one invocation. diff --git a/src/core/tooling/command_output_content.zig b/src/core/tooling/command_output_content.zig index 4f7ba45c3..9b77f0774 100644 --- a/src/core/tooling/command_output_content.zig +++ b/src/core/tooling/command_output_content.zig @@ -707,11 +707,11 @@ fn fuzzCommandOutputContent(_: void, smith: *std.testing.Smith) anyerror!void { test "fuzz command output content" { try std.testing.fuzz({}, fuzzCommandOutputContent, .{ - .corpus = &.{ + .corpus = &(.{ "progress\roverwrite\n", "\x1b[31mred\x1b[0m\n", "exit_code=0\n\none\n\n", "\xff\x00\xe2\x82", - }, + } ++ @import("fuzz_corpus").command_output_content.*), }); } diff --git a/src/core/workspace/file_index.zig b/src/core/workspace/file_index.zig index a5e666321..03e9d0c96 100644 --- a/src/core/workspace/file_index.zig +++ b/src/core/workspace/file_index.zig @@ -4238,7 +4238,7 @@ test "deinit cancels and joins one loader while suppressing queued refresh" { test "file index raw-list and Unicode query bytes remain bounded" { try std.testing.fuzz({}, fuzzRawListAndQuery, .{ - .corpus = &.{ + .corpus = &(.{ "safe.txt\x00ärger", "escape-\x1b[2J.txt\x00\xff", "Ärger-file.txt\x00ärger", @@ -4247,7 +4247,7 @@ test "file index raw-list and Unicode query bytes remain bounded" { "docs/Cafe\u{0301}.txt\x00e\u{0301}", "src/axaya.zig\x00aaa", "truncated.txt\x00\xc3", - }, + } ++ @import("fuzz_corpus").file_index.*), }); } diff --git a/src/fuzz_targets.zig b/src/fuzz_targets.zig new file mode 100644 index 000000000..0e840c208 --- /dev/null +++ b/src/fuzz_targets.zig @@ -0,0 +1,23 @@ +//! Root of the `zig build fuzz` test binary. Each import below brings in a +//! file that holds an fx fuzz target listed in `tests/fuzz/targets.json`. +//! The build filters this binary to the registered fuzz tests, so ordinary +//! unit tests in these files never run here. A new fx fuzz target needs its +//! file imported here and an entry in the registry. + +test { + _ = @import("acp/mcp_servers.zig"); + _ = @import("core/agent/runtime/vision_contracts.zig"); + _ = @import("core/images/image_data.zig"); + _ = @import("core/mcp/elicitation.zig"); + _ = @import("core/session/session_codec.zig"); + _ = @import("core/session/session_event.zig"); + _ = @import("core/shared/display_width.zig"); + _ = @import("core/shared/types.zig"); + _ = @import("core/shell_command/command_effect.zig"); + _ = @import("core/skills/skill_contract.zig"); + _ = @import("core/skills/skill_invocation.zig"); + _ = @import("core/tooling/command_output_content.zig"); + _ = @import("core/workspace/file_index.zig"); + _ = @import("gateway/responses_protocol.zig"); + _ = @import("gateway/sse.zig"); +} diff --git a/src/gateway/responses_protocol.zig b/src/gateway/responses_protocol.zig index 3df2e1673..75475c06f 100644 --- a/src/gateway/responses_protocol.zig +++ b/src/gateway/responses_protocol.zig @@ -2167,7 +2167,7 @@ test "Responses text finalization fuzzes chunking and capture boundaries" { try stream.finish(buffer[0 .. len + separator.len + 4]); } }; - try std.testing.fuzz({}, Probe.run, .{ .corpus = &.{ "", "a", "two chunks", "capture limit is not receipt progress" } }); + try std.testing.fuzz({}, Probe.run, .{ .corpus = &(.{ "", "a", "two chunks", "capture limit is not receipt progress" } ++ @import("fuzz_corpus").responses_text.*) }); } test "Responses captures assistant commentary phase" { diff --git a/src/gateway/sse.zig b/src/gateway/sse.zig index f4d432158..16515947b 100644 --- a/src/gateway/sse.zig +++ b/src/gateway/sse.zig @@ -263,7 +263,7 @@ test "provider framing fuzzes chunk-invariant event data" { try std.testing.expectEqual(null, try reader.next(alloc, &source.interface, &cancelled)); } }; - try std.testing.fuzz({}, Probe.run, .{ .corpus = &.{ "", "one", "split every field" } }); + try std.testing.fuzz({}, Probe.run, .{ .corpus = &(.{ "", "one", "split every field" } ++ @import("fuzz_corpus").sse_framing.*) }); } fn check_allocations(alloc: std.mem.Allocator) !void { diff --git a/tests/AGENTS.md b/tests/AGENTS.md index ba48f77c0..41d0d9714 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -16,6 +16,7 @@ Pick the lowest layer that proves the behavior, and extend the closest existing | fx against the official MCP client conformance suite | `tests/e2e/conformance/`, an isolated npm project | | Evidence tooling for terminal rendering bugs | `tests/e2e/render-lab/` | | Task quality or tool choice with real models | `tests/evals/` | +| Saved inputs for a fuzz test | `tests/fuzz/corpus//`, with the target listed in `tests/fuzz/targets.json` | | A protocol or crash-safety design, checked with TLC | A folder under `tests/tla/` named after the module it models | | libfx SDK behavior | `sdk/tests/`, which `sdk/AGENTS.md` owns | | Startup latency or memory | `benchmarks/` | @@ -34,6 +35,9 @@ tests/ ├── fixtures/ │ └── v1-sessions/ sessions the v0.0.14 release saved, which the converter's tests copy; │ generate.ts rebuilds them and manifest.json lists them +├── fuzz/ +│ ├── targets.json every fuzz test, for zig build fuzz and the nightly Fuzz workflow +│ └── corpus// saved inputs that zig build test replays; CONTRIBUTING.md explains them └── tla/ TLA+ models, their checkers, and their Bun drivers ├── session_manager/ one folder per model area, named after the module it describes, ├── mcp/ each with a check.sh; tla/README.md lists them all diff --git a/tests/fuzz/targets.json b/tests/fuzz/targets.json new file mode 100644 index 000000000..edfadc10a --- /dev/null +++ b/tests/fuzz/targets.json @@ -0,0 +1,22 @@ +{ + "targets": [ + {"name": "acp_mcp_servers", "module": "fx", "test": "ACP MCP parser handles fuzzed request bytes", "iterations": 1500000}, + {"name": "vision_json", "module": "fx", "test": "Vision JSON parsers tolerate arbitrary bounded bytes", "iterations": 1200000}, + {"name": "image_dimensions", "module": "fx", "test": "image dimension parsing stays bounded on arbitrary bytes", "iterations": 5000000}, + {"name": "mcp_elicitation", "module": "fx", "test": "elicitation request and response parsers stay bounded under fuzzed bytes", "iterations": 1500000}, + {"name": "session_optional_fields", "module": "fx", "test": "durable session optional fields handle fuzzed ownership paths", "iterations": 3000000}, + {"name": "session_work_provenance", "module": "fx", "test": "per-turn work provenance parser handles fuzzed ownership paths", "iterations": 2100000}, + {"name": "session_conversation_frame", "module": "fx", "test": "conversation frame decoder stays bounded under fuzzed bytes", "iterations": 3000000}, + {"name": "session_log_scan", "module": "session_manager", "test": "scanning never panics on arbitrary bytes", "iterations": 5000000}, + {"name": "session_schema_header", "module": "session_manager", "test": "header parsing never panics on arbitrary bytes", "iterations": 5000000}, + {"name": "display_width", "module": "fx", "test": "fuzz display unit boundaries", "iterations": 620000}, + {"name": "tool_argument_diagnostic", "module": "fx", "test": "fuzz tool argument diagnostic", "iterations": 3000000}, + {"name": "skill_file", "module": "fx", "test": "parseSkillFile keeps fuzzed borrowed slices inside the input", "iterations": 5000000}, + {"name": "skill_display_name", "module": "fx", "test": "skill display name parsing stays bounded", "iterations": 5000000}, + {"name": "command_output_content", "module": "fx", "test": "fuzz command output content", "iterations": 160000}, + {"name": "file_index", "module": "fx", "test": "file index raw-list and Unicode query bytes remain bounded", "iterations": 620000}, + {"name": "responses_text", "module": "fx", "test": "Responses text finalization fuzzes chunking and capture boundaries", "iterations": 430000}, + {"name": "sse_framing", "module": "fx", "test": "provider framing fuzzes chunk-invariant event data", "iterations": 1300000}, + {"name": "shell_parity", "module": "fx", "test": "shell parity holds for generated commands", "iterations": 600} + ] +}