From 8925f00b74c21736e2505d548f194f56d6926b6f Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 8 Oct 2026 13:15:40 -0400 Subject: [PATCH 01/23] Add the native workspace indexer foundation - Add src/core/indexer/, which reads gitignore rules, repository layout, and git config as files and never starts a process. - Match git's ignore decisions for nested .gitignore files, info/exclude, and the global excludes file, including include and includeIf config. - Report the filter drivers a repository's own config defines, and fail closed when that config cannot be read. - Check in CI that outside code imports only indexer.zig and that the indexer imports nothing that can start a process. --- .github/workflows/ci.yml | 4 + AGENTS.md | 4 +- CONTRIBUTING.md | 2 +- scripts/check-indexer-boundary.sh | 41 ++ src/core/indexer/bounded_read.zig | 59 +++ src/core/indexer/git_config.zig | 758 ++++++++++++++++++++++++++++++ src/core/indexer/ignore.zig | 297 ++++++++++++ src/core/indexer/indexer.zig | 110 +++++ src/core/indexer/layout.zig | 203 ++++++++ src/core/indexer/sources.zig | 133 ++++++ src/core/indexer/wildmatch.zig | 304 ++++++++++++ src/main.zig | 1 + 12 files changed, 1914 insertions(+), 2 deletions(-) create mode 100755 scripts/check-indexer-boundary.sh create mode 100644 src/core/indexer/bounded_read.zig create mode 100644 src/core/indexer/git_config.zig create mode 100644 src/core/indexer/ignore.zig create mode 100644 src/core/indexer/indexer.zig create mode 100644 src/core/indexer/layout.zig create mode 100644 src/core/indexer/sources.zig create mode 100644 src/core/indexer/wildmatch.zig diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9bd394825..85c40c2a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,6 +52,9 @@ jobs: - name: Check compactor boundary run: ./scripts/check-compactor-boundary.sh + - name: Check indexer boundary + run: ./scripts/check-indexer-boundary.sh + - name: Test CI and release scripts run: python3 -m unittest scripts.tests.test_macos_signing scripts.tests.test_signed_macos scripts.tests.test_detect_macos_need -q @@ -625,6 +628,7 @@ jobs: scripts=( benchmarks/startup.sh scripts/check-compactor-boundary.sh + scripts/check-indexer-boundary.sh scripts/check-public-surface.sh scripts/detect-macos-need.sh scripts/smoke-binary.sh diff --git a/AGENTS.md b/AGENTS.md index 5b375f310..21ccd3473 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,6 +69,8 @@ Key rules: * `src/core/compactor/` owns context compaction. Outside code imports only its front door, `compactor.zig`, and the compactor imports only shared basics and model configuration; its caller hands in the model caller, the record store, and the session's projection of saved turns into chat messages. `scripts/check-compactor-boundary.sh` enforces this in CI. +* `src/core/indexer/` is the native workspace indexer. It reads git's ignore rules, repository layout, and git config as data and never starts a process, so a repository's configuration cannot make it run a program. Outside code imports only its front door, `indexer.zig`, and the indexer imports only the standard library, its own files, `io.zig`, and `debug_trace.zig`. `scripts/check-indexer-boundary.sh` enforces this in CI. + * `src/tools/` owns built-in tool implementations. Generic tool contracts and dispatch live in `src/core/tooling/`. Default tool specs are centralized in `src/core/tooling/tool_specs.zig` or `src/builtins/tools.zig`, not in individual tool files. * `src/ui/` owns terminal rendering, event loop, input, transcript. It must not own product state. @@ -278,7 +280,7 @@ Do not run the complete deterministic test suite locally as the default developm After the focused checks pass, create a clean checkpoint commit, push the non-`main` feature branch, and open a draft PR immediately. Linux is the gate for every change because most fx behavior is identical on every platform: -* `.github/workflows/ci.yml` runs on Linux x86_64. It checks formatting, the PGSO corpus, the public surface, and the compactor boundary, then runs the ReleaseSafe unit tests. It builds fx once, smoke-tests it, and shares that binary with four duration-balanced E2E shards and the MCP conformance baseline. Checked-in weights assign every E2E file to exactly one shard, and files inside each shard run sequentially in separate Bun processes so terminal fixtures and process state cannot leak between files. A failed file receives one retry after its tmux server is reset. A file that passes only on retry gets a warning annotation; investigate it as a possible race. The SDK jobs build WASM and the Node-API addons once and run the complete package qualification. `Build & Test` passes only when the unit tests and every SDK job pass, because Publish libfx relies on that result instead of qualifying the package again. +* `.github/workflows/ci.yml` runs on Linux x86_64. It checks formatting, the PGSO corpus, the public surface, and the compactor and indexer boundaries, then runs the ReleaseSafe unit tests. It builds fx once, smoke-tests it, and shares that binary with four duration-balanced E2E shards and the MCP conformance baseline. Checked-in weights assign every E2E file to exactly one shard, and files inside each shard run sequentially in separate Bun processes so terminal fixtures and process state cannot leak between files. A failed file receives one retry after its tmux server is reset. A file that passes only on retry gets a warning annotation; investigate it as a possible race. The SDK jobs build WASM and the Node-API addons once and run the complete package qualification. `Build & Test` passes only when the unit tests and every SDK job pass, because Publish libfx relies on that result instead of qualifying the package again. * `.github/workflows/binary-size.yml` builds each release target on its native runner, reports the size change, and smoke-tests every target's binary. See **Binary Size Observability**. * `.github/workflows/bench.yml` enforces the startup latency budget. See **Benchmarks**. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0411d5eca..871eeca45 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,7 +39,7 @@ zig build run Keep the local development loop focused: run the narrowest test that covers the changed path, build fx, and exercise the change using `./zig-out/bin/fx`. The installed `fx` on `PATH` is not valid development evidence. -Once the focused checks pass, create a clean checkpoint commit, push the non-`main` feature branch, and open a draft PR immediately. **CI** runs the complete deterministic suite on Linux x86_64, which is the gate for every change. It checks formatting, the PGSO corpus, the public surface, and the compactor boundary, runs the ReleaseSafe unit tests, and builds fx once for the E2E jobs. Four duration-balanced E2E shards use checked-in weights to assign every Bun test file once, and files inside each shard run sequentially in separate Bun processes so terminal fixtures and process state cannot leak between files. A failed file receives one retry after tmux is reset, and a file that passes only on retry gets a warning annotation. +Once the focused checks pass, create a clean checkpoint commit, push the non-`main` feature branch, and open a draft PR immediately. **CI** runs the complete deterministic suite on Linux x86_64, which is the gate for every change. It checks formatting, the PGSO corpus, the public surface, and the compactor and indexer boundaries, runs the ReleaseSafe unit tests, and builds fx once for the E2E jobs. Four duration-balanced E2E shards use checked-in weights to assign every Bun test file once, and files inside each shard run sequentially in separate Bun processes so terminal fixtures and process state cannot leak between files. A failed file receives one retry after tmux is reset, and a file that passes only on retry gets a warning annotation. macOS checks run only when a change can behave differently on macOS: a Zig file with a macOS, BSD, or Linux code path, `build.zig`, the macOS signing script, the native SDK addon, an E2E file listed in `tests/e2e/macos-platform-tests.json`, or a shared E2E helper that reads the platform. Add the `ci:macos` label or run `gh workflow run macos.yml --ref ` to request them for any other change. The `macOS arm64` check passes without a macOS runner when the change does not need one. diff --git a/scripts/check-indexer-boundary.sh b/scripts/check-indexer-boundary.sh new file mode 100755 index 000000000..5d3fcf4b5 --- /dev/null +++ b/scripts/check-indexer-boundary.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# fx-indexer keeps native workspace discovery in src/core/indexer behind one +# front door, indexer.zig. Code outside imports only that file. The indexer +# imports only the standard library, its own files, and fx's shared io.zig and +# debug_trace.zig, and it never starts a process: it reads repository metadata +# as data, so a repository's configuration can never make it run a program. + +set -euo pipefail + +repo_root="$(git rev-parse --show-toplevel)" +cd "$repo_root" + +folder="src/core/indexer" + +reaching_in="$({ + git grep --untracked -n -E '@import\("[^"]*indexer/[a-z_]+\.zig"\)' -- src ":(exclude)$folder" || true +} | grep -v 'indexer/indexer\.zig")' || true)" +if [[ -n "$reaching_in" ]]; then + printf 'Only %s/indexer.zig may be imported from outside the indexer:\n%s\n' "$folder" "$reaching_in" >&2 + exit 1 +fi + +allowed='^(std|builtin|[a-z_]+\.zig|\.\./shared/(io|debug_trace)\.zig)$' +reaching_out="" +while IFS= read -r line; do + target="${line#*@import(\"}" + target="${target%\")}" + if [[ ! "$target" =~ $allowed ]]; then + reaching_out+="$line"$'\n' + fi +done < <(git grep --untracked -n -o -E '@import\("[^"]+"\)' -- "$folder" || true) +if [[ -n "$reaching_out" ]]; then + printf 'The indexer may import only std, its own files, io.zig and debug_trace.zig:\n%s' "$reaching_out" >&2 + exit 1 +fi + +spawning="$(git grep --untracked -n -E 'std\.process|posix_spawn|execv[pe]*\(|fork\(|popen\(' -- "$folder" || true)" +if [[ -n "$spawning" ]]; then + printf 'The indexer must never start a process:\n%s\n' "$spawning" >&2 + exit 1 +fi diff --git a/src/core/indexer/bounded_read.zig b/src/core/indexer/bounded_read.zig new file mode 100644 index 000000000..2f736868c --- /dev/null +++ b/src/core/indexer/bounded_read.zig @@ -0,0 +1,59 @@ +//! Bounded reads of untrusted metadata files. Only existing regular files are +//! read: a final symlink is not followed, FIFOs and devices are refused +//! without blocking, and content larger than the cap is refused rather than +//! truncated, so a caller never acts on a partial file. + +const std = @import("std"); +const io_mod = @import("../shared/io.zig"); + +const Allocator = std.mem.Allocator; + +pub const Outcome = union(enum) { + /// Content owned by the allocator passed to the read. + content: []u8, + missing, + /// Present but unsafe, unreadable, or over the cap. Static string. + unavailable: []const u8, +}; + +/// Reads `sub_path` relative to `dir`, accepting at most `max_bytes`. +pub fn readAt(alloc: Allocator, dir: std.Io.Dir, sub_path: []const u8, max_bytes: usize) Allocator.Error!Outcome { + var file = io_mod.openExistingReadOnlyRegularFile(dir, sub_path, .no_follow) catch |err| return switch (err) { + error.FileNotFound, error.NotDir => .missing, + else => .{ .unavailable = @errorName(err) }, + }; + defer file.close(io_mod.getIo()); + // The reader reports a stream that fills its limit as too long, so allow + // one extra byte to accept a file of exactly `max_bytes`. + const data = io_mod.readFileToEnd(alloc, &file, max_bytes +| 1) catch |err| return switch (err) { + error.OutOfMemory => error.OutOfMemory, + error.StreamTooLong => .{ .unavailable = "too_large" }, + else => .{ .unavailable = @errorName(err) }, + }; + return .{ .content = data }; +} + +/// Reads an absolute `path`. +pub fn readAbsolute(alloc: Allocator, path: []const u8, max_bytes: usize) Allocator.Error!Outcome { + if (!std.fs.path.isAbsolute(path)) return .{ .unavailable = "relative_path" }; + return readAt(alloc, std.Io.Dir.cwd(), path, max_bytes); +} + +test "bounded reads return content, missing, oversize and non-regular outcomes" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "small", .data = "abc" }); + try tmp.dir.createDirPath(std.testing.io, "dir"); + + const small = try readAt(alloc, tmp.dir, "small", 3); + defer alloc.free(small.content); + try std.testing.expectEqualStrings("abc", small.content); + try std.testing.expect(try readAt(alloc, tmp.dir, "absent", 16) == .missing); + try std.testing.expectEqualStrings("too_large", (try readAt(alloc, tmp.dir, "small", 2)).unavailable); + try std.testing.expect(try readAt(alloc, tmp.dir, "dir", 16) == .unavailable); + + tmp.dir.symLink(std.testing.io, "small", "link", .{}) catch return error.SkipZigTest; + try std.testing.expect(try readAt(alloc, tmp.dir, "link", 16) == .unavailable); + try std.testing.expect(try readAbsolute(alloc, "relative", 16) == .unavailable); +} diff --git a/src/core/indexer/git_config.zig b/src/core/indexer/git_config.zig new file mode 100644 index 000000000..6cbd936d6 --- /dev/null +++ b/src/core/indexer/git_config.zig @@ -0,0 +1,758 @@ +//! Git config files read as text, never through git. The parser follows git's +//! config.c syntax: sections, quoted subsections, value quoting and escapes, +//! line continuations, comments and implicit booleans. The loader follows +//! `include.path` and `includeIf` like git, in one of two modes: +//! +//! - `values` evaluates `gitdir:`, `gitdir/i:` and `onbranch:` conditions to +//! read settings, and treats `hasconfig:` as unmatched with a trace, an +//! accepted parity limitation; +//! - `every_include` counts every `includeIf` as included, for callers that +//! must over-approximate, such as finding filter drivers to disable. + +const std = @import("std"); +const io_mod = @import("../shared/io.zig"); +const debug_trace = @import("../shared/debug_trace.zig"); +const bounded_read = @import("bounded_read.zig"); +const layout_mod = @import("layout.zig"); +const wildmatch = @import("wildmatch.zig"); + +const Allocator = std.mem.Allocator; + +pub const max_config_file_bytes: usize = 1024 * 1024; +pub const max_include_depth: usize = 10; + +pub const Entry = struct { + /// Lowercase. + section: []const u8, + /// Exact for `[section "sub"]`; lowercase for legacy `[section.sub]`. + subsection: ?[]const u8, + /// Lowercase. + name: []const u8, + /// Null when the line has no `=`, which git reads as boolean true. + value: ?[]const u8, +}; + +pub const ParseError = error{ OutOfMemory, InvalidConfig }; + +const Reader = struct { + text: []const u8, + pos: usize = 0, + eof: bool = false, + + /// git's get_next_char: folds CRLF to LF and reports EOF as LF. + fn next(self: *Reader) u8 { + if (self.pos >= self.text.len) { + self.eof = true; + return '\n'; + } + const c = self.text[self.pos]; + self.pos += 1; + if (c == '\r' and self.pos < self.text.len and self.text[self.pos] == '\n') { + self.pos += 1; + return '\n'; + } + return c; + } +}; + +fn isSpace(c: u8) bool { + return c == ' ' or c == '\t' or c == '\n' or c == '\r'; +} + +fn isKeyChar(c: u8) bool { + return std.ascii.isAlphanumeric(c) or c == '-'; +} + +/// Parses config `text`. Memory belongs to `arena`. Malformed input is +/// `error.InvalidConfig`, as git refuses the whole file. +pub fn parse(arena: Allocator, text: []const u8) ParseError![]Entry { + var reader: Reader = .{ .text = text }; + if (std.mem.startsWith(u8, text, "\xef\xbb\xbf")) reader.pos = 3; + var entries: std.ArrayList(Entry) = .empty; + var section: ?[]const u8 = null; + var subsection: ?[]const u8 = null; + var comment = false; + while (true) { + const c = reader.next(); + if (c == '\n') { + if (reader.eof) break; + comment = false; + continue; + } + if (comment or isSpace(c)) continue; + if (c == '#' or c == ';') { + comment = true; + continue; + } + if (c == '[') { + const header = try parseHeader(arena, &reader); + section = header.section; + subsection = header.subsection; + continue; + } + if (!std.ascii.isAlphabetic(c)) return error.InvalidConfig; + const current_section = section orelse return error.InvalidConfig; + + var name: std.ArrayList(u8) = .empty; + try name.append(arena, std.ascii.toLower(c)); + var ch: u8 = undefined; + while (true) { + ch = reader.next(); + if (reader.eof or !isKeyChar(ch)) break; + try name.append(arena, std.ascii.toLower(ch)); + } + while (ch == ' ' or ch == '\t') ch = reader.next(); + var value: ?[]const u8 = null; + if (ch != '\n') { + if (ch != '=') return error.InvalidConfig; + value = try parseValue(arena, &reader); + } + try entries.append(arena, .{ + .section = current_section, + .subsection = subsection, + .name = try name.toOwnedSlice(arena), + .value = value, + }); + } + return entries.toOwnedSlice(arena); +} + +const Header = struct { section: []const u8, subsection: ?[]const u8 }; + +/// git's get_base_var and get_extended_base_var. +fn parseHeader(arena: Allocator, reader: *Reader) ParseError!Header { + var base: std.ArrayList(u8) = .empty; + var quoted: ?[]const u8 = null; + while (true) { + const c = reader.next(); + if (reader.eof) return error.InvalidConfig; + if (c == ']') break; + if (isSpace(c)) { + var ch = c; + while (true) { + if (ch == '\n') return error.InvalidConfig; + ch = reader.next(); + if (!isSpace(ch)) break; + } + if (ch != '"') return error.InvalidConfig; + var sub: std.ArrayList(u8) = .empty; + while (true) { + var q = reader.next(); + if (q == '\n') return error.InvalidConfig; + if (q == '"') break; + if (q == '\\') { + q = reader.next(); + if (q == '\n') return error.InvalidConfig; + } + try sub.append(arena, q); + } + if (reader.next() != ']') return error.InvalidConfig; + quoted = try sub.toOwnedSlice(arena); + break; + } + if (!isKeyChar(c) and c != '.') return error.InvalidConfig; + try base.append(arena, std.ascii.toLower(c)); + } + if (base.items.len == 0) return error.InvalidConfig; + const dot = std.mem.findScalar(u8, base.items, '.'); + const section = if (dot) |i| base.items[0..i] else base.items; + if (section.len == 0) return error.InvalidConfig; + const dotted_rest: ?[]const u8 = if (dot) |i| base.items[i + 1 ..] else null; + const subsection: ?[]const u8 = if (quoted) |q| + (if (dotted_rest) |rest| try std.mem.concat(arena, u8, &.{ rest, ".", q }) else q) + else + dotted_rest; + return .{ .section = section, .subsection = subsection }; +} + +/// git's parse_value: quotes toggle, `\t \b \n \\ \"` escape, a backslash at +/// line end continues the value, unquoted `#` or `;` starts a comment, and +/// unquoted trailing whitespace is dropped. +fn parseValue(arena: Allocator, reader: *Reader) ParseError![]const u8 { + var value: std.ArrayList(u8) = .empty; + var quote = false; + var comment = false; + var trim_len: usize = 0; + while (true) { + var c = reader.next(); + if (c == '\n') { + if (quote) return error.InvalidConfig; + if (trim_len != 0) value.shrinkRetainingCapacity(trim_len); + return value.toOwnedSlice(arena); + } + if (comment) continue; + if (isSpace(c) and !quote) { + if (trim_len == 0) trim_len = value.items.len; + if (value.items.len != 0) try value.append(arena, c); + continue; + } + if (!quote and (c == ';' or c == '#')) { + comment = true; + continue; + } + trim_len = 0; + if (c == '\\') { + c = reader.next(); + switch (c) { + '\n' => continue, + 't' => c = '\t', + 'b' => c = 0x08, + 'n' => c = '\n', + '\\', '"' => {}, + else => return error.InvalidConfig, + } + try value.append(arena, c); + continue; + } + if (c == '"') { + quote = !quote; + continue; + } + try value.append(arena, c); + } +} + +/// git's git_config_bool. Null for a value git rejects. +pub fn parseBool(value: ?[]const u8) ?bool { + const v = value orelse return true; + if (v.len == 0) return false; + for ([_][]const u8{ "true", "yes", "on" }) |word| { + if (std.ascii.eqlIgnoreCase(v, word)) return true; + } + for ([_][]const u8{ "false", "no", "off" }) |word| { + if (std.ascii.eqlIgnoreCase(v, word)) return false; + } + const number = std.fmt.parseInt(i64, v, 10) catch return null; + return number != 0; +} + +pub const Scope = enum { system, global, local, worktree }; + +pub const Mode = enum { values, every_include }; + +pub const LoadedEntry = struct { + entry: Entry, + scope: Scope, +}; + +pub const Unavailable = struct { + path: []const u8, + /// Static string. + reason: []const u8, +}; + +pub const Loaded = struct { + /// In file order, with included files spliced in at their directive. + entries: []const LoadedEntry, + unavailable: []const Unavailable, + + /// The last entry for the key, or null when unset. + pub fn last(self: Loaded, section: []const u8, subsection: ?[]const u8, name: []const u8) ?*const Entry { + var i = self.entries.len; + while (i > 0) { + i -= 1; + const entry = &self.entries[i].entry; + if (keyMatches(entry, section, subsection, name)) return entry; + } + return null; + } +}; + +fn keyMatches(entry: *const Entry, section: []const u8, subsection: ?[]const u8, name: []const u8) bool { + if (!std.mem.eql(u8, entry.section, section) or !std.mem.eql(u8, entry.name, name)) return false; + if (subsection) |want| { + const have = entry.subsection orelse return false; + return std.mem.eql(u8, have, want); + } + return entry.subsection == null; +} + +/// Where config comes from and what conditions are evaluated against. +pub const Context = struct { + home: ?[]const u8 = null, + /// Never empty: git treats an empty `XDG_CONFIG_HOME` as unset. + xdg_config_home: ?[]const u8 = null, + /// `GIT_CONFIG_GLOBAL`: replaces the user files when set. + global_override: ?[]const u8 = null, + /// `GIT_CONFIG_SYSTEM`, or the conventional `/etc/gitconfig`. Git's + /// compiled-in system path can differ; that is a parity limitation. + system_path: []const u8 = "/etc/gitconfig", + no_system: bool = false, + layout: ?layout_mod.Layout = null, + /// `realpath` of `layout.git_dir`, tried first by `gitdir:` like git. + git_dir_real: ?[]const u8 = null, + branch: ?[]const u8 = null, + + /// Reads git's environment variables and the repository's HEAD. Memory + /// belongs to `arena`. + pub fn fromEnvironment(arena: Allocator, layout: ?layout_mod.Layout) Allocator.Error!Context { + var context: Context = .{ + .home = io_mod.getenv("HOME"), + .xdg_config_home = if (io_mod.getenv("XDG_CONFIG_HOME")) |xdg| if (xdg.len > 0) xdg else null else null, + .global_override = io_mod.getenv("GIT_CONFIG_GLOBAL"), + .no_system = if (io_mod.getenv("GIT_CONFIG_NOSYSTEM")) |v| parseBool(v) orelse false else false, + .layout = layout, + }; + if (io_mod.getenv("GIT_CONFIG_SYSTEM")) |path| context.system_path = path; + if (layout) |found| { + context.git_dir_real = io_mod.realpathAlloc(arena, found.git_dir) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => null, + }; + context.branch = try layout_mod.currentBranch(arena, found); + } + return context; + } +}; + +const Loader = struct { + arena: Allocator, + context: *const Context, + mode: Mode, + entries: std.ArrayList(LoadedEntry) = .empty, + unavailable: std.ArrayList(Unavailable) = .empty, + + fn readFile(self: *Loader, path: []const u8, scope: Scope, depth: usize) Allocator.Error!void { + if (depth > max_include_depth) return self.markUnavailable(path, "include_depth"); + // git reads /dev/null as an empty file; it is commonly used to + // disable a config level. + if (std.mem.eql(u8, path, "/dev/null")) return; + const content = switch (try bounded_read.readAbsolute(self.arena, path, max_config_file_bytes)) { + .content => |bytes| bytes, + .missing => return, + .unavailable => |reason| return self.markUnavailable(path, reason), + }; + const parsed = parse(self.arena, content) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.InvalidConfig => return self.markUnavailable(path, "invalid_config"), + }; + for (parsed) |entry| { + try self.entries.append(self.arena, .{ .entry = entry, .scope = scope }); + if (!std.mem.eql(u8, entry.name, "path")) continue; + const include_value = entry.value orelse continue; + const included = if (std.mem.eql(u8, entry.section, "include") and entry.subsection == null) + true + else if (std.mem.eql(u8, entry.section, "includeif") and entry.subsection != null) + self.mode == .every_include or try self.conditionMatches(entry.subsection.?, path) + else + false; + if (!included) continue; + const target = try self.includePath(include_value, path) orelse continue; + try self.readFile(target, scope, depth + 1); + } + } + + fn markUnavailable(self: *Loader, path: []const u8, reason: []const u8) Allocator.Error!void { + try self.unavailable.append(self.arena, .{ .path = path, .reason = reason }); + } + + /// git's include path handling: `~/` from HOME, relative to the + /// including file's directory otherwise. Null when it cannot expand. + fn includePath(self: *Loader, raw: []const u8, config_path: []const u8) Allocator.Error!?[]const u8 { + const expanded = try self.expandHome(raw) orelse return null; + if (std.fs.path.isAbsolute(expanded)) return expanded; + const dir = std.fs.path.dirname(config_path) orelse return null; + return try std.fs.path.join(self.arena, &.{ dir, expanded }); + } + + fn expandHome(self: *Loader, raw: []const u8) Allocator.Error!?[]const u8 { + if (raw.len == 0) return null; + if (raw[0] != '~') return raw; + if (!std.mem.startsWith(u8, raw, "~/")) return null; + const home = self.context.home orelse return null; + return try std.fs.path.join(self.arena, &.{ home, raw[2..] }); + } + + fn conditionMatches(self: *Loader, condition: []const u8, config_path: []const u8) Allocator.Error!bool { + if (std.mem.startsWith(u8, condition, "gitdir:")) return self.gitdirMatches(condition["gitdir:".len..], false, config_path); + if (std.mem.startsWith(u8, condition, "gitdir/i:")) return self.gitdirMatches(condition["gitdir/i:".len..], true, config_path); + if (std.mem.startsWith(u8, condition, "onbranch:")) return self.branchMatches(condition["onbranch:".len..]); + if (std.mem.startsWith(u8, condition, "hasconfig:")) { + debug_trace.logf("indexer", "config includeIf hasconfig treated as unmatched path={s}", .{config_path}); + } + return false; + } + + /// git's include_by_gitdir and prepare_include_condition_pattern. + fn gitdirMatches(self: *Loader, raw: []const u8, fold_case: bool, config_path: []const u8) Allocator.Error!bool { + const layout = self.context.layout orelse return false; + var pattern = try self.expandHome(raw) orelse return false; + var literal_len: usize = 0; + if (std.mem.startsWith(u8, pattern, "./")) { + const real_config = io_mod.realpathAlloc(self.arena, config_path) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return false, + }; + const dir = std.fs.path.dirname(real_config) orelse return false; + pattern = try std.mem.concat(self.arena, u8, &.{ dir, pattern[1..] }); + literal_len = dir.len + 1; + } else if (!std.fs.path.isAbsolute(pattern)) { + pattern = try std.mem.concat(self.arena, u8, &.{ "**/", pattern }); + } + if (std.mem.endsWith(u8, pattern, "/")) pattern = try std.mem.concat(self.arena, u8, &.{ pattern, "**" }); + + const candidates = [_]?[]const u8{ self.context.git_dir_real, layout.git_dir }; + for (candidates) |maybe_text| { + const text = maybe_text orelse continue; + if (literal_len > 0) { + if (text.len < literal_len) continue; + const same = if (fold_case) + std.ascii.eqlIgnoreCase(pattern[0..literal_len], text[0..literal_len]) + else + std.mem.eql(u8, pattern[0..literal_len], text[0..literal_len]); + if (!same) continue; + } + if (wildmatch.match(pattern[literal_len..], text[literal_len..], .{ .pathname = true, .casefold = fold_case })) return true; + } + return false; + } + + /// git's include_by_branch. + fn branchMatches(self: *Loader, raw: []const u8) Allocator.Error!bool { + const branch = self.context.branch orelse return false; + const pattern = if (std.mem.endsWith(u8, raw, "/")) try std.mem.concat(self.arena, u8, &.{ raw, "**" }) else raw; + return wildmatch.match(pattern, branch, .{ .pathname = true }); + } + + fn result(self: *Loader) Allocator.Error!Loaded { + return .{ + .entries = try self.entries.toOwnedSlice(self.arena), + .unavailable = try self.unavailable.toOwnedSlice(self.arena), + }; + } +}; + +/// Loads system, user and repository config in git's precedence order, in +/// `values` mode. Memory belongs to `arena`. +pub fn loadAll(arena: Allocator, context: *const Context) Allocator.Error!Loaded { + var loader: Loader = .{ .arena = arena, .context = context, .mode = .values }; + if (!context.no_system) try loader.readFile(context.system_path, .system, 0); + if (context.global_override) |path| { + if (path.len > 0) try loader.readFile(path, .global, 0); + } else { + if (context.xdg_config_home) |xdg| { + try loader.readFile(try std.fs.path.join(arena, &.{ xdg, "git", "config" }), .global, 0); + } else if (context.home) |home| { + try loader.readFile(try std.fs.path.join(arena, &.{ home, ".config", "git", "config" }), .global, 0); + } + if (context.home) |home| try loader.readFile(try std.fs.path.join(arena, &.{ home, ".gitconfig" }), .global, 0); + } + if (context.layout) |layout| { + const local_start = loader.entries.items.len; + try loader.readFile(try std.fs.path.join(arena, &.{ layout.common_dir, "config" }), .local, 0); + var worktree_config = false; + for (loader.entries.items[local_start..]) |loaded| { + const entry = loaded.entry; + if (std.mem.eql(u8, entry.section, "extensions") and entry.subsection == null and std.mem.eql(u8, entry.name, "worktreeconfig")) { + worktree_config = parseBool(entry.value) orelse false; + } + } + if (worktree_config) try loader.readFile(try std.fs.path.join(arena, &.{ layout.git_dir, "config.worktree" }), .worktree, 0); + } + return loader.result(); +} + +/// Loads only the repository's own config (`config`, `config.worktree` and +/// everything they include) with every `includeIf` counted as included. +/// Memory belongs to `arena`. +pub fn loadRepositoryEveryInclude(arena: Allocator, context: *const Context) Allocator.Error!Loaded { + var loader: Loader = .{ .arena = arena, .context = context, .mode = .every_include }; + if (context.layout) |layout| { + try loader.readFile(try std.fs.path.join(arena, &.{ layout.common_dir, "config" }), .local, 0); + try loader.readFile(try std.fs.path.join(arena, &.{ layout.git_dir, "config.worktree" }), .worktree, 0); + } + return loader.result(); +} + +/// Distinct filter driver names with a `clean`, `smudge` or `process` +/// command. Memory belongs to `arena`. +pub fn filterDriverNames(arena: Allocator, loaded: Loaded) Allocator.Error![]const []const u8 { + var names: std.ArrayList([]const u8) = .empty; + for (loaded.entries) |loaded_entry| { + const entry = loaded_entry.entry; + if (!std.mem.eql(u8, entry.section, "filter")) continue; + const name = entry.subsection orelse continue; + const program = std.mem.eql(u8, entry.name, "clean") or std.mem.eql(u8, entry.name, "smudge") or std.mem.eql(u8, entry.name, "process"); + if (!program) continue; + for (names.items) |known| { + if (std.mem.eql(u8, known, name)) break; + } else try names.append(arena, name); + } + return names.toOwnedSlice(arena); +} + +pub const ObjectFormat = enum { sha1, sha256 }; + +pub const IgnoreSettings = struct { + /// Absolute path of the excludes file to read, or null for none. + excludes_file: ?[]const u8, + ignore_case: bool, + object_format: ObjectFormat, + /// Settings git would refuse, by key. Static strings. + invalid: []const []const u8, +}; + +/// Settings that shape ignore decisions and index parsing, from config +/// loaded in `values` mode. Memory belongs to `arena`. +pub fn ignoreSettings(arena: Allocator, loaded: Loaded, context: *const Context) Allocator.Error!IgnoreSettings { + var invalid: std.ArrayList([]const u8) = .empty; + const ignore_case = if (loaded.last("core", null, "ignorecase")) |entry| + parseBool(entry.value) orelse blk: { + try invalid.append(arena, "core.ignorecase"); + break :blk false; + } + else + false; + + var object_format: ObjectFormat = .sha1; + var i = loaded.entries.len; + while (i > 0) { + i -= 1; + const loaded_entry = loaded.entries[i]; + if (loaded_entry.scope != .local) continue; + if (!keyMatches(&loaded_entry.entry, "extensions", null, "objectformat")) continue; + const value = loaded_entry.entry.value orelse ""; + if (std.ascii.eqlIgnoreCase(value, "sha256")) { + object_format = .sha256; + } else if (!std.ascii.eqlIgnoreCase(value, "sha1")) { + try invalid.append(arena, "extensions.objectformat"); + } + break; + } + + const excludes_file: ?[]const u8 = if (loaded.last("core", null, "excludesfile")) |entry| blk: { + const raw = entry.value orelse break :blk null; + if (raw.len == 0) break :blk null; + if (std.mem.startsWith(u8, raw, "~/")) { + const home = context.home orelse break :blk null; + break :blk try std.fs.path.join(arena, &.{ home, raw[2..] }); + } + if (std.fs.path.isAbsolute(raw)) break :blk raw; + // git resolves a relative path from the worktree root, its cwd. + const layout = context.layout orelse break :blk null; + break :blk try std.fs.path.join(arena, &.{ layout.worktree_root, raw }); + } else if (context.xdg_config_home) |xdg| + try std.fs.path.join(arena, &.{ xdg, "git", "ignore" }) + else if (context.home) |home| + try std.fs.path.join(arena, &.{ home, ".config", "git", "ignore" }) + else + null; + + return .{ + .excludes_file = excludes_file, + .ignore_case = ignore_case, + .object_format = object_format, + .invalid = try invalid.toOwnedSlice(arena), + }; +} + +fn expectParsed(text: []const u8, expected: []const Entry) !void { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const entries = try parse(arena_state.allocator(), text); + try std.testing.expectEqual(expected.len, entries.len); + for (expected, entries) |want, have| { + try std.testing.expectEqualStrings(want.section, have.section); + try std.testing.expectEqualStrings(want.name, have.name); + if (want.subsection) |sub| try std.testing.expectEqualStrings(sub, have.subsection.?) else try std.testing.expect(have.subsection == null); + if (want.value) |value| try std.testing.expectEqualStrings(value, have.value.?) else try std.testing.expect(have.value == null); + } +} + +test "config parser follows git's section, key and value syntax" { + try expectParsed( + "\xef\xbb\xbf# comment\r\n[Core]\r\n\tExcludesFile = ~/ignore ; trailing comment\r\n" ++ + "\tbare\n[filter \"My \\\"LFS\\\"\"]\n clean = run \"a b\" # c\n" ++ + "[Section.Sub]\nkey=one\\\n two\nesc = \"tab\\there\\n\"\nspaced = inner value \n", + &.{ + .{ .section = "core", .subsection = null, .name = "excludesfile", .value = "~/ignore" }, + .{ .section = "core", .subsection = null, .name = "bare", .value = null }, + .{ .section = "filter", .subsection = "My \"LFS\"", .name = "clean", .value = "run a b" }, + .{ .section = "section", .subsection = "sub", .name = "key", .value = "one two" }, + .{ .section = "section", .subsection = "sub", .name = "esc", .value = "tab\there\n" }, + .{ .section = "section", .subsection = "sub", .name = "spaced", .value = "inner value" }, + }, + ); +} + +test "config parser rejects input git refuses" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const bad = [_][]const u8{ + "key = value\n", + "[core]\nname # no equals\n", + "[core]\nname = \"unterminated\n", + "[core]\nname = bad\\q\n", + "[core\n", + "[]\n", + "[core \"sub\" ]\n", + "[core]\n9name = x\n", + }; + for (bad) |text| { + if (parse(arena, text)) |_| { + std.debug.print("accepted: {s}\n", .{text}); + return error.TestUnexpectedResult; + } else |err| try std.testing.expectEqual(error.InvalidConfig, err); + } +} + +test "config booleans follow git" { + try std.testing.expectEqual(@as(?bool, true), parseBool(null)); + try std.testing.expectEqual(@as(?bool, false), parseBool("")); + try std.testing.expectEqual(@as(?bool, true), parseBool("Yes")); + try std.testing.expectEqual(@as(?bool, false), parseBool("off")); + try std.testing.expectEqual(@as(?bool, true), parseBool("2")); + try std.testing.expectEqual(@as(?bool, false), parseBool("0")); + try std.testing.expectEqual(@as(?bool, null), parseBool("maybe")); +} + +fn writeTestFile(dir: std.Io.Dir, path: []const u8, content: []const u8) !void { + if (std.fs.path.dirname(path)) |parent| try dir.createDirPath(std.testing.io, parent); + try dir.writeFile(std.testing.io, .{ .sub_path = path, .data = content }); +} + +const TestRepo = struct { + root: []const u8, + context: Context, +}; + +/// Builds `home/` and a repository at `repo_path` inside `tmp`, and a context +/// that reads only those files. +fn testRepo(arena: Allocator, tmp: *std.testing.TmpDir, repo_path: []const u8, branch: []const u8) !TestRepo { + const head = try std.fmt.allocPrint(arena, "{s}/.git/HEAD", .{repo_path}); + try writeTestFile(tmp.dir, head, try std.fmt.allocPrint(arena, "ref: refs/heads/{s}\n", .{branch})); + try tmp.dir.createDirPath(std.testing.io, try std.fmt.allocPrint(arena, "{s}/.git/objects", .{repo_path})); + try tmp.dir.createDirPath(std.testing.io, "home"); + const root = try io_mod.dirRealpathAlloc(arena, tmp.dir, "."); + const layout = (try layout_mod.discover(arena, try std.fs.path.join(arena, &.{ root, repo_path }))).?; + var context = try Context.fromEnvironment(arena, layout); + context.home = try std.fs.path.join(arena, &.{ root, "home" }); + context.xdg_config_home = null; + context.global_override = null; + context.no_system = true; + return .{ .root = root, .context = context }; +} + +test "config includes follow relative and home paths, skip missing files and stop at depth" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try testRepo(arena, &tmp, "repo", "main"); + try writeTestFile(tmp.dir, "home/.gitconfig", "[include]\npath = conf/a\npath = ~/conf/missing\n[core]\nignorecase = false\n"); + try writeTestFile(tmp.dir, "home/conf/a", "[include]\npath = b\n[core]\nignorecase = true\n"); + try writeTestFile(tmp.dir, "home/conf/b", "[core]\nexcludesfile = ~/from-b\n"); + try writeTestFile(tmp.dir, "repo/.git/config", "[include]\npath = loop\n"); + try writeTestFile(tmp.dir, "repo/.git/loop", "[include]\npath = loop\n"); + + const loaded = try loadAll(arena, &repo.context); + const settings = try ignoreSettings(arena, loaded, &repo.context); + // The last value wins in file order, after spliced-in includes. + try std.testing.expect(!settings.ignore_case); + try std.testing.expectEqualStrings(try std.fs.path.join(arena, &.{ repo.root, "home/from-b" }), settings.excludes_file.?); + try std.testing.expectEqual(@as(usize, 1), loaded.unavailable.len); + try std.testing.expectEqualStrings("include_depth", loaded.unavailable[0].reason); +} + +test "config includeIf evaluates gitdir, gitdir/i and onbranch, and never hasconfig" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const matching = try testRepo(arena, &tmp, "work/one", "feat/x"); + const other = try testRepo(arena, &tmp, "play/two", "main"); + const root = matching.root; + const gitconfig = try std.fmt.allocPrint(arena, + \\[includeIf "gitdir:{s}/work/"] + \\path = work.inc + \\[includeIf "gitdir/i:{s}/PLAY/"] + \\path = play.inc + \\[includeIf "onbranch:feat/"] + \\path = feat.inc + \\[includeIf "hasconfig:remote.*.url:https://example.com/**"] + \\path = hasconfig.inc + \\ + , .{ root, root }); + try writeTestFile(tmp.dir, "home/.gitconfig", gitconfig); + try writeTestFile(tmp.dir, "home/work.inc", "[core]\nexcludesfile = /work-excludes\n"); + try writeTestFile(tmp.dir, "home/play.inc", "[core]\nexcludesfile = /play-excludes\n"); + try writeTestFile(tmp.dir, "home/feat.inc", "[core]\nignorecase = true\n"); + try writeTestFile(tmp.dir, "home/hasconfig.inc", "[core]\nexcludesfile = /never\n"); + + const work_loaded = try loadAll(arena, &matching.context); + const work = try ignoreSettings(arena, work_loaded, &matching.context); + try std.testing.expectEqualStrings("/work-excludes", work.excludes_file.?); + try std.testing.expect(work.ignore_case); + + const play_loaded = try loadAll(arena, &other.context); + const play = try ignoreSettings(arena, play_loaded, &other.context); + try std.testing.expectEqualStrings("/play-excludes", play.excludes_file.?); + try std.testing.expect(!play.ignore_case); + + // Outside any repository no condition matches. + var no_repo = matching.context; + no_repo.layout = null; + no_repo.git_dir_real = null; + no_repo.branch = null; + const none = try ignoreSettings(arena, try loadAll(arena, &no_repo), &no_repo); + try std.testing.expectEqualStrings(try std.fs.path.join(arena, &.{ root, "home/.config/git/ignore" }), none.excludes_file.?); +} + +test "config gitdir patterns relative to the including file" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try testRepo(arena, &tmp, "home/projects/app", "main"); + try writeTestFile(tmp.dir, "home/.gitconfig", "[includeIf \"gitdir:./projects/\"]\npath = projects.inc\n"); + try writeTestFile(tmp.dir, "home/projects.inc", "[core]\nexcludesfile = relative/ignore\n"); + const settings = try ignoreSettings(arena, try loadAll(arena, &repo.context), &repo.context); + const expected = try std.fs.path.join(arena, &.{ repo.context.layout.?.worktree_root, "relative/ignore" }); + try std.testing.expectEqualStrings(expected, settings.excludes_file.?); +} + +test "repository filter names come only from repository config and count every includeIf" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try testRepo(arena, &tmp, "repo", "main"); + try writeTestFile(tmp.dir, "home/.gitconfig", "[filter \"lfs\"]\nclean = git-lfs clean -- %f\n"); + try writeTestFile(tmp.dir, "repo/.git/config", "[filter \"evil\"]\nclean = /tmp/x\n[includeIf \"onbranch:never/\"]\npath = hidden\n[filter \"evil\"]\nsmudge = /tmp/y\n[filter \"meta\"]\nrequired = true\n"); + try writeTestFile(tmp.dir, "repo/.git/hidden", "[filter \"Hidden\"]\nprocess = /tmp/z\n"); + try writeTestFile(tmp.dir, "repo/.git/config.worktree", "[filter \"wt\"]\nclean = /tmp/w\n"); + + const every = try loadRepositoryEveryInclude(arena, &repo.context); + const names = try filterDriverNames(arena, every); + try std.testing.expectEqual(@as(usize, 3), names.len); + try std.testing.expectEqualStrings("evil", names[0]); + try std.testing.expectEqualStrings("Hidden", names[1]); + try std.testing.expectEqualStrings("wt", names[2]); + + // In values mode the onbranch include does not match and config.worktree + // is read only when extensions.worktreeConfig is set. + const values = try filterDriverNames(arena, try loadAll(arena, &repo.context)); + try std.testing.expectEqual(@as(usize, 2), values.len); + try std.testing.expectEqualStrings("lfs", values[0]); + try std.testing.expectEqualStrings("evil", values[1]); +} + +test "config sources that are not bounded regular files are unavailable" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try testRepo(arena, &tmp, "repo", "main"); + try tmp.dir.createDirPath(std.testing.io, "repo/.git/config"); + try writeTestFile(tmp.dir, "home/.gitconfig", "[core]\nbroken = \"x\n"); + const loaded = try loadAll(arena, &repo.context); + try std.testing.expectEqual(@as(usize, 2), loaded.unavailable.len); + try std.testing.expectEqualStrings("invalid_config", loaded.unavailable[0].reason); + try std.testing.expect(loaded.entries.len == 0); +} diff --git a/src/core/indexer/ignore.zig b/src/core/indexer/ignore.zig new file mode 100644 index 000000000..d49ebc4e8 --- /dev/null +++ b/src/core/indexer/ignore.zig @@ -0,0 +1,297 @@ +//! Gitignore pattern lists and exclusion decisions, ported from git's dir.c: +//! line parsing (BOM, CRLF, comments, escaped trailing spaces), negation, +//! directory-only patterns, basename versus anchored matching (including +//! git's literal-prefix split), and last-match precedence across sources. + +const std = @import("std"); +const wildmatch = @import("wildmatch.zig"); + +const Allocator = std.mem.Allocator; + +/// Largest pattern file read, per source (D7). +pub const max_pattern_file_bytes: usize = 1024 * 1024; + +const Pattern = struct { + /// Pattern bytes without a leading `!` or a trailing `/`; escapes kept. + text: []const u8, + /// Length of the leading part free of glob characters (git's + /// `nowildcardlen`), never more than `text.len`. + literal_len: usize, + negative: bool, + must_be_dir: bool, + /// No slash in `text`: the pattern matches a basename at any depth. + basename_only: bool, +}; + +pub const PatternList = struct { + /// Directory the list belongs to, relative to the worktree root, ending + /// in `/`; empty for root-level sources such as `info/exclude`. + base: []const u8, + patterns: []const Pattern, + + /// Parses gitignore `contents` for the directory `base` (relative, empty + /// or ending in `/`). All memory belongs to `arena`. + pub fn parse(arena: Allocator, base: []const u8, contents: []const u8) Allocator.Error!PatternList { + std.debug.assert(base.len == 0 or base[base.len - 1] == '/'); + var patterns: std.ArrayList(Pattern) = .empty; + var rest = contents; + if (std.mem.startsWith(u8, rest, "\xef\xbb\xbf")) rest = rest[3..]; + var lines = std.mem.splitScalar(u8, rest, '\n'); + while (lines.next()) |raw_line| { + var line = raw_line; + if (line.len > 0 and line[line.len - 1] == '\r') line = line[0 .. line.len - 1]; + if (std.mem.findScalar(u8, line, 0)) |nul| line = line[0..nul]; + if (line.len == 0 or line[0] == '#') continue; + line = trimTrailingSpaces(line); + if (parsePattern(line)) |pattern| { + try patterns.append(arena, .{ + .text = try arena.dupe(u8, pattern.text), + .literal_len = pattern.literal_len, + .negative = pattern.negative, + .must_be_dir = pattern.must_be_dir, + .basename_only = pattern.basename_only, + }); + } + } + return .{ + .base = try arena.dupe(u8, base), + .patterns = try patterns.toOwnedSlice(arena), + }; + } +}; + +/// git's trim_trailing_spaces: drops unescaped trailing spaces (not tabs). +fn trimTrailingSpaces(line: []const u8) []const u8 { + var last_space: ?usize = null; + var i: usize = 0; + while (i < line.len) : (i += 1) { + switch (line[i]) { + ' ' => { + if (last_space == null) last_space = i; + }, + '\\' => { + i += 1; + if (i >= line.len) return line; + last_space = null; + }, + else => last_space = null, + } + } + return if (last_space) |cut| line[0..cut] else line; +} + +/// git's parse_path_pattern. Returns null for a pattern that can never match. +fn parsePattern(line: []const u8) ?Pattern { + var text = line; + var negative = false; + if (text.len > 0 and text[0] == '!') { + negative = true; + text = text[1..]; + } + var must_be_dir = false; + if (text.len > 0 and text[text.len - 1] == '/') { + must_be_dir = true; + text = text[0 .. text.len - 1]; + } + if (text.len == 0) return null; + return .{ + .text = text, + .literal_len = @min(literalLength(text), text.len), + .negative = negative, + .must_be_dir = must_be_dir, + .basename_only = std.mem.findScalar(u8, text, '/') == null, + }; +} + +fn literalLength(text: []const u8) usize { + for (text, 0..) |c, i| { + if (c == '*' or c == '?' or c == '[' or c == '\\') return i; + } + return text.len; +} + +pub const Decision = enum { + /// No pattern matched. + none, + excluded, + /// The last matching pattern was a negation. + included, +}; + +/// Returns the decision of the last pattern that matches `path` (relative to +/// the worktree root, no leading or trailing slash). `lists` run from lowest +/// to highest precedence: the global excludes file, then `info/exclude`, +/// then `.gitignore` files from the root down. Lists whose `base` does not +/// contain `path` are skipped. This decides `path` alone; callers walking a +/// tree stop at excluded directories, as git does. +pub fn decide(lists: []const *const PatternList, path: []const u8, is_dir: bool, fold_case: bool) Decision { + const basename = if (std.mem.findScalarLast(u8, path, '/')) |slash| path[slash + 1 ..] else path; + var result: Decision = .none; + for (lists) |list| { + if (list.base.len > 0) { + if (path.len <= list.base.len or !eqlFold(path[0..list.base.len], list.base, fold_case)) continue; + } + for (list.patterns) |*pattern| { + if (pattern.must_be_dir and !is_dir) continue; + const matched = if (pattern.basename_only) + matchBasename(pattern, basename, fold_case) + else + matchPathname(pattern, path, list.base, fold_case); + if (matched) result = if (pattern.negative) .included else .excluded; + } + } + return result; +} + +/// Reports whether `path` is excluded, including through an excluded +/// ancestor directory, which git never looks inside. +pub fn isPathExcluded(lists: []const *const PatternList, path: []const u8, is_dir: bool, fold_case: bool) bool { + var i: usize = 0; + while (std.mem.findScalarPos(u8, path, i, '/')) |slash| : (i = slash + 1) { + if (decide(lists, path[0..slash], true, fold_case) == .excluded) return true; + } + return decide(lists, path, is_dir, fold_case) == .excluded; +} + +/// git's match_basename. Basenames contain no slash, so WM_PATHNAME is moot. +fn matchBasename(pattern: *const Pattern, basename: []const u8, fold_case: bool) bool { + if (pattern.literal_len == pattern.text.len) return eqlFold(pattern.text, basename, fold_case); + return wildmatch.match(pattern.text, basename, .{ .casefold = fold_case }); +} + +/// git's match_pathname, keeping its literal-prefix split: the literal head +/// is compared directly and only the remainder goes through wildmatch, so a +/// remainder that starts with `**` is treated as a leading `**`. +fn matchPathname(pattern: *const Pattern, path: []const u8, base: []const u8, fold_case: bool) bool { + var text = pattern.text; + var literal_len = pattern.literal_len; + if (text.len > 0 and text[0] == '/') { + text = text[1..]; + literal_len -|= 1; + } + // `base` ends in `/`, or is empty; `decide` already checked the prefix. + var name = path[base.len..]; + if (literal_len > 0) { + if (literal_len > name.len) return false; + if (!eqlFold(text[0..literal_len], name[0..literal_len], fold_case)) return false; + text = text[literal_len..]; + name = name[literal_len..]; + if (text.len == 0 and name.len == 0) return true; + } + return wildmatch.match(text, name, .{ .pathname = true, .casefold = fold_case }); +} + +fn eqlFold(a: []const u8, b: []const u8, fold_case: bool) bool { + return if (fold_case) std.ascii.eqlIgnoreCase(a, b) else std.mem.eql(u8, a, b); +} + +fn testList(arena: Allocator, base: []const u8, contents: []const u8) !*PatternList { + const list = try arena.create(PatternList); + list.* = try PatternList.parse(arena, base, contents); + return list; +} + +test "gitignore parsing handles comments, escapes, trailing spaces, BOM and CRLF" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const list = try testList(arena, "", "# comment\n\n\\#hash\n\\!bang\ntrail \nkeep\\ \n!neg\ndir/\n/anchored\n \n"); + const texts = [_][]const u8{ "\\#hash", "\\!bang", "trail", "keep\\ ", "neg", "dir", "/anchored" }; + try std.testing.expectEqual(texts.len, list.patterns.len); + for (texts, list.patterns) |expected, pattern| try std.testing.expectEqualStrings(expected, pattern.text); + try std.testing.expect(list.patterns[4].negative); + try std.testing.expect(list.patterns[5].must_be_dir); + try std.testing.expect(!list.patterns[6].basename_only); + + const crlf = try testList(arena, "", "\xef\xbb\xbfa.log\r\nb\r\n"); + try std.testing.expectEqual(@as(usize, 2), crlf.patterns.len); + try std.testing.expectEqualStrings("a.log", crlf.patterns[0].text); + try std.testing.expectEqualStrings("b", crlf.patterns[1].text); +} + +test "gitignore decisions follow git's basename, anchoring and directory rules" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const root = try testList(arena, "", "*.log\n/top.txt\nbuild/\ndocs/*.md\na/**/z\n\\#literal\nkeep\\ \n"); + const lists = [_]*const PatternList{root}; + const cases = [_]struct { path: []const u8, is_dir: bool, expected: Decision }{ + .{ .path = "x.log", .is_dir = false, .expected = .excluded }, + .{ .path = "deep/er/x.log", .is_dir = false, .expected = .excluded }, + .{ .path = "top.txt", .is_dir = false, .expected = .excluded }, + .{ .path = "sub/top.txt", .is_dir = false, .expected = .none }, + .{ .path = "build", .is_dir = true, .expected = .excluded }, + .{ .path = "build", .is_dir = false, .expected = .none }, + .{ .path = "sub/build", .is_dir = true, .expected = .excluded }, + .{ .path = "docs/a.md", .is_dir = false, .expected = .excluded }, + .{ .path = "docs/sub/a.md", .is_dir = false, .expected = .none }, + .{ .path = "a/z", .is_dir = false, .expected = .excluded }, + .{ .path = "a/b/c/z", .is_dir = false, .expected = .excluded }, + .{ .path = "#literal", .is_dir = false, .expected = .excluded }, + .{ .path = "keep ", .is_dir = false, .expected = .excluded }, + .{ .path = "keep", .is_dir = false, .expected = .none }, + }; + for (cases) |case| { + const actual = decide(&lists, case.path, case.is_dir, false); + if (actual != case.expected) { + std.debug.print("path={s} dir={} expected={s} actual={s}\n", .{ case.path, case.is_dir, @tagName(case.expected), @tagName(actual) }); + return error.TestExpectedEqual; + } + } +} + +test "gitignore precedence: deeper lists win, negation re-includes, excluded parents stay excluded" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const global = try testList(arena, "", "*.tmp\nsecret\n"); + const info = try testList(arena, "", "!keep.tmp\n"); + const root = try testList(arena, "", "logs/\n*.out\n!important.out\n"); + const sub = try testList(arena, "sub/", "!x.tmp\n/only-here\n"); + const lists = [_]*const PatternList{ global, info, root, sub }; + + try std.testing.expectEqual(Decision.excluded, decide(&lists, "a.tmp", false, false)); + try std.testing.expectEqual(Decision.included, decide(&lists, "keep.tmp", false, false)); + try std.testing.expectEqual(Decision.included, decide(&lists, "sub/x.tmp", false, false)); + try std.testing.expectEqual(Decision.excluded, decide(&lists, "x.tmp", false, false)); + try std.testing.expectEqual(Decision.included, decide(&lists, "important.out", false, false)); + try std.testing.expectEqual(Decision.excluded, decide(&lists, "sub/only-here", false, false)); + // A sub list never applies to the directory that holds it. + try std.testing.expectEqual(Decision.none, decide(&lists, "only-here", false, false)); + // Re-including a file inside an excluded directory does not work in git. + const reinclude = try testList(arena, "", "logs/\n!logs/keep.txt\n"); + const reinclude_lists = [_]*const PatternList{reinclude}; + try std.testing.expect(isPathExcluded(&reinclude_lists, "logs/keep.txt", false, false)); + try std.testing.expect(isPathExcluded(&lists, "logs/a/b.txt", false, false)); + try std.testing.expect(!isPathExcluded(&lists, "src/a.txt", false, false)); +} + +test "gitignore case folding applies to patterns and list bases" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const root = try testList(arena, "", "*.LOG\n/Build/\n"); + const sub = try testList(arena, "Sub/", "x\n"); + const lists = [_]*const PatternList{ root, sub }; + try std.testing.expectEqual(Decision.excluded, decide(&lists, "a.log", false, true)); + try std.testing.expectEqual(Decision.none, decide(&lists, "a.log", false, false)); + try std.testing.expectEqual(Decision.excluded, decide(&lists, "build", true, true)); + try std.testing.expectEqual(Decision.excluded, decide(&lists, "sub/x", false, true)); + try std.testing.expectEqual(Decision.none, decide(&lists, "sub/x", false, false)); +} + +test "gitignore keeps git's literal-prefix split before double asterisks" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + // Literal head `a`, remainder `**/b` matched as a leading `**/`, so the + // pattern crosses a slash that a plain wildmatch of `a**/b` would not. + // Verified against git 2.50: both paths below are ignored. + const root = try testList(arena, "", "a**/b\nfoo/**\n"); + const lists = [_]*const PatternList{root}; + try std.testing.expectEqual(Decision.excluded, decide(&lists, "a/x/b", false, false)); + try std.testing.expectEqual(Decision.excluded, decide(&lists, "ax/b", false, false)); + try std.testing.expectEqual(Decision.excluded, decide(&lists, "foo/x/y", false, false)); + try std.testing.expectEqual(Decision.none, decide(&lists, "foo", true, false)); + try std.testing.expect(!wildmatch.match("a**/b", "a/x/b", .{ .pathname = true })); +} diff --git a/src/core/indexer/indexer.zig b/src/core/indexer/indexer.zig new file mode 100644 index 000000000..ab8e34598 --- /dev/null +++ b/src/core/indexer/indexer.zig @@ -0,0 +1,110 @@ +//! Native workspace indexer: which files and folders exist under a set of +//! roots, decided by git's ignore rules read as data. It never starts a +//! process, so a repository's configuration cannot make it run a program. +//! Outside code imports only this file; `scripts/check-indexer-boundary.sh` +//! enforces that, the import allowlist and the no-process rule in CI. + +const std = @import("std"); +const io_mod = @import("../shared/io.zig"); +const debug_trace = @import("../shared/debug_trace.zig"); +const git_config = @import("git_config.zig"); +const layout = @import("layout.zig"); + +const Allocator = std.mem.Allocator; + +pub const FilterNames = struct { + /// The slice and every name are owned by the allocator passed to + /// `repoFilterNames`. + names: []const []const u8, + + pub fn deinit(self: FilterNames, alloc: Allocator) void { + for (self.names) |name| alloc.free(name); + alloc.free(self.names); + } +}; + +pub const RepoFilterNamesError = error{ OutOfMemory, RepoConfigUnavailable }; + +/// Returns every filter driver name with a `clean`, `smudge` or `process` +/// command in the repository's own config: `config`, `config.worktree` and +/// the files they include, counting every `includeIf` as included so no +/// condition can hide a driver. Empty outside a git repository. A repository +/// config file that cannot be read within bounds, a malformed one, or an +/// invalid `.git` file fails closed with `error.RepoConfigUnavailable`. +pub fn repoFilterNames(alloc: Allocator, workspace_root: []const u8) RepoFilterNamesError!FilterNames { + if (!std.fs.path.isAbsolute(workspace_root)) return error.RepoConfigUnavailable; + var arena_state = std.heap.ArenaAllocator.init(alloc); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + const found = layout.discover(arena, workspace_root) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.InvalidGitFile => { + debug_trace.logf("indexer", "filter names unavailable reason=invalid_git_file root={s}", .{workspace_root}); + return error.RepoConfigUnavailable; + }, + }; + const context = try git_config.Context.fromEnvironment(arena, found); + const loaded = try git_config.loadRepositoryEveryInclude(arena, &context); + if (loaded.unavailable.len > 0) { + for (loaded.unavailable) |source| { + debug_trace.logf("indexer", "filter names unavailable reason={s} path={s}", .{ source.reason, source.path }); + } + return error.RepoConfigUnavailable; + } + + const names = try git_config.filterDriverNames(arena, loaded); + const owned = try alloc.alloc([]const u8, names.len); + var copied: usize = 0; + errdefer { + for (owned[0..copied]) |name| alloc.free(name); + alloc.free(owned); + } + for (names) |name| { + owned[copied] = try alloc.dupe(u8, name); + copied += 1; + } + return .{ .names = owned }; +} + +test { + _ = @import("bounded_read.zig"); + _ = @import("wildmatch.zig"); + _ = @import("ignore.zig"); + _ = @import("sources.zig"); + _ = layout; + _ = git_config; +} + +fn writeTestFile(dir: std.Io.Dir, path: []const u8, content: []const u8) !void { + if (std.fs.path.dirname(path)) |parent| try dir.createDirPath(std.testing.io, parent); + try dir.writeFile(std.testing.io, .{ .sub_path = path, .data = content }); +} + +test "repoFilterNames returns repository drivers and fails closed on unreadable config" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try writeTestFile(tmp.dir, "repo/.git/HEAD", "ref: refs/heads/main\n"); + try tmp.dir.createDirPath(std.testing.io, "repo/.git/objects"); + try writeTestFile(tmp.dir, "repo/.git/config", "[includeIf \"gitdir:/nowhere/\"]\npath = extra\n"); + try writeTestFile(tmp.dir, "repo/.git/extra", "[filter \"x\"]\nclean = /tmp/run-me\n"); + try tmp.dir.createDirPath(std.testing.io, "repo/src"); + try writeTestFile(tmp.dir, "bad/.git/HEAD", "ref: refs/heads/main\n"); + try tmp.dir.createDirPath(std.testing.io, "bad/.git/objects"); + try tmp.dir.createDirPath(std.testing.io, "bad/.git/config"); + const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); + defer alloc.free(root); + + const repo_src = try std.fs.path.join(alloc, &.{ root, "repo/src" }); + defer alloc.free(repo_src); + const names = try repoFilterNames(alloc, repo_src); + defer names.deinit(alloc); + try std.testing.expectEqual(@as(usize, 1), names.names.len); + try std.testing.expectEqualStrings("x", names.names[0]); + + const bad = try std.fs.path.join(alloc, &.{ root, "bad" }); + defer alloc.free(bad); + try std.testing.expectError(error.RepoConfigUnavailable, repoFilterNames(alloc, bad)); + try std.testing.expectError(error.RepoConfigUnavailable, repoFilterNames(alloc, "relative")); +} diff --git a/src/core/indexer/layout.zig b/src/core/indexer/layout.zig new file mode 100644 index 000000000..9b0ed61cb --- /dev/null +++ b/src/core/indexer/layout.zig @@ -0,0 +1,203 @@ +//! Locates the git repository that contains a workspace root the way git's +//! discovery does, reading only data: `.git` directories, `.git` files with a +//! `gitdir:` pointer (linked worktrees and submodules), and `commondir`. + +const std = @import("std"); +const io_mod = @import("../shared/io.zig"); +const bounded_read = @import("bounded_read.zig"); + +const Allocator = std.mem.Allocator; + +const max_metadata_bytes: usize = 4096; + +pub const Layout = struct { + /// Absolute directory holding `.git`; repository paths are relative to it. + worktree_root: []const u8, + /// Absolute per-worktree git directory, holding `HEAD` and `index`. + git_dir: []const u8, + /// Absolute shared git directory, holding `config` and `info/exclude`. + common_dir: []const u8, + /// Workspace root relative to `worktree_root`: empty, or `a/b` without + /// a leading or trailing slash. + prefix: []const u8, +}; + +pub const DiscoverError = error{ OutOfMemory, InvalidGitFile }; + +/// Returns the innermost repository containing the absolute directory +/// `start`, or null when there is none. All memory belongs to `arena`. As in +/// git, a `.git` file that is not a valid `gitdir:` pointer is an error, and a +/// `.git` directory that is not a git directory is skipped. +pub fn discover(arena: Allocator, start: []const u8) DiscoverError!?Layout { + std.debug.assert(std.fs.path.isAbsolute(start)); + const normalized = try std.fs.path.resolve(arena, &.{start}); + var candidate: []const u8 = normalized; + while (true) { + if (try layoutAt(arena, candidate)) |found| { + const prefix = if (found.worktree_root.len == normalized.len) + "" + else if (std.mem.eql(u8, found.worktree_root, "/")) + normalized[1..] + else + normalized[found.worktree_root.len + 1 ..]; + return .{ + .worktree_root = found.worktree_root, + .git_dir = found.git_dir, + .common_dir = found.common_dir, + .prefix = prefix, + }; + } + const parent = std.fs.path.dirname(candidate) orelse return null; + if (std.mem.eql(u8, parent, candidate)) return null; + candidate = parent; + } +} + +fn layoutAt(arena: Allocator, dir: []const u8) DiscoverError!?Layout { + const dot_git = try std.fs.path.join(arena, &.{ dir, ".git" }); + const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), dot_git, .{ .follow_symlinks = true }) catch return null; + switch (stat.kind) { + .directory => { + const dirs = try gitDirs(arena, dot_git) orelse return null; + return .{ .worktree_root = dir, .git_dir = dirs.git_dir, .common_dir = dirs.common_dir, .prefix = "" }; + }, + .file => { + const target = try readGitFile(arena, dir, dot_git) orelse return error.InvalidGitFile; + const dirs = try gitDirs(arena, target) orelse return error.InvalidGitFile; + return .{ .worktree_root = dir, .git_dir = dirs.git_dir, .common_dir = dirs.common_dir, .prefix = "" }; + }, + else => return null, + } +} + +/// git's read_gitfile: `gitdir: `, relative to the `.git` file's +/// directory. +fn readGitFile(arena: Allocator, dir: []const u8, dot_git: []const u8) Allocator.Error!?[]const u8 { + const content = switch (try bounded_read.readAbsolute(arena, dot_git, max_metadata_bytes)) { + .content => |bytes| bytes, + .missing, .unavailable => return null, + }; + const prefix = "gitdir: "; + if (!std.mem.startsWith(u8, content, prefix)) return null; + const raw = std.mem.trimEnd(u8, content[prefix.len..], " \t\r\n"); + if (raw.len == 0) return null; + return try std.fs.path.resolve(arena, &.{ dir, raw }); +} + +const GitDirs = struct { git_dir: []const u8, common_dir: []const u8 }; + +/// A light form of git's is_git_directory: `HEAD` in the git directory and +/// `objects` in the common directory, which `commondir` may relocate. +fn gitDirs(arena: Allocator, git_dir: []const u8) Allocator.Error!?GitDirs { + var dir = std.Io.Dir.openDirAbsolute(io_mod.getIo(), git_dir, .{}) catch return null; + defer dir.close(io_mod.getIo()); + const head = dir.statFile(io_mod.getIo(), "HEAD", .{ .follow_symlinks = false }) catch return null; + if (head.kind != .file) return null; + const common_dir = switch (try bounded_read.readAt(arena, dir, "commondir", max_metadata_bytes)) { + .content => |bytes| blk: { + const raw = std.mem.trimEnd(u8, bytes, " \t\r\n"); + if (raw.len == 0) return null; + break :blk try std.fs.path.resolve(arena, &.{ git_dir, raw }); + }, + .missing => git_dir, + .unavailable => return null, + }; + const objects = try std.fs.path.join(arena, &.{ common_dir, "objects" }); + const objects_stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), objects, .{ .follow_symlinks = true }) catch return null; + if (objects_stat.kind != .directory) return null; + return .{ .git_dir = git_dir, .common_dir = common_dir }; +} + +/// Returns the branch `HEAD` points at (`refs/heads/`), or null when +/// detached or unreadable. Memory belongs to `arena`. +pub fn currentBranch(arena: Allocator, layout: Layout) Allocator.Error!?[]const u8 { + const head_path = try std.fs.path.join(arena, &.{ layout.git_dir, "HEAD" }); + const content = switch (try bounded_read.readAbsolute(arena, head_path, max_metadata_bytes)) { + .content => |bytes| bytes, + .missing, .unavailable => return null, + }; + const prefix = "ref: refs/heads/"; + const trimmed = std.mem.trim(u8, content, " \t\r\n"); + if (!std.mem.startsWith(u8, trimmed, prefix) or trimmed.len == prefix.len) return null; + return trimmed[prefix.len..]; +} + +fn writeTestFile(dir: std.Io.Dir, path: []const u8, content: []const u8) !void { + if (std.fs.path.dirname(path)) |parent| try dir.createDirPath(std.testing.io, parent); + try dir.writeFile(std.testing.io, .{ .sub_path = path, .data = content }); +} + +fn makeTestRepo(dir: std.Io.Dir, git_dir: []const u8) !void { + const head = try std.fs.path.join(std.testing.allocator, &.{ git_dir, "HEAD" }); + defer std.testing.allocator.free(head); + try writeTestFile(dir, head, "ref: refs/heads/main\n"); + const objects = try std.fs.path.join(std.testing.allocator, &.{ git_dir, "objects" }); + defer std.testing.allocator.free(objects); + try dir.createDirPath(std.testing.io, objects); +} + +test "layout discovery finds the innermost repository and the workspace prefix" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try makeTestRepo(tmp.dir, "repo/.git"); + try tmp.dir.createDirPath(std.testing.io, "repo/a/b"); + const root = try io_mod.dirRealpathAlloc(arena, tmp.dir, "."); + + const start = try std.fs.path.join(arena, &.{ root, "repo/a/b" }); + const found = (try discover(arena, start)).?; + try std.testing.expectEqualStrings(try std.fs.path.join(arena, &.{ root, "repo" }), found.worktree_root); + try std.testing.expectEqualStrings(try std.fs.path.join(arena, &.{ root, "repo/.git" }), found.git_dir); + try std.testing.expectEqualStrings(found.git_dir, found.common_dir); + try std.testing.expectEqualStrings("a/b", found.prefix); + try std.testing.expectEqualStrings("main", (try currentBranch(arena, found)).?); + + const at_root = (try discover(arena, try std.fs.path.join(arena, &.{ root, "repo/" }))).?; + try std.testing.expectEqualStrings("", at_root.prefix); +} + +test "layout discovery follows gitdir files and commondir for linked worktrees" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try makeTestRepo(tmp.dir, "main/.git"); + try writeTestFile(tmp.dir, "main/.git/worktrees/wt/HEAD", "ref: refs/heads/feature/x\n"); + try writeTestFile(tmp.dir, "main/.git/worktrees/wt/commondir", "../..\n"); + try writeTestFile(tmp.dir, "wt/.git", "gitdir: ../main/.git/worktrees/wt\n"); + const root = try io_mod.dirRealpathAlloc(arena, tmp.dir, "."); + + const found = (try discover(arena, try std.fs.path.join(arena, &.{ root, "wt" }))).?; + try std.testing.expectEqualStrings(try std.fs.path.join(arena, &.{ root, "wt" }), found.worktree_root); + try std.testing.expectEqualStrings(try std.fs.path.join(arena, &.{ root, "main/.git/worktrees/wt" }), found.git_dir); + try std.testing.expectEqualStrings(try std.fs.path.join(arena, &.{ root, "main/.git" }), found.common_dir); + try std.testing.expectEqualStrings("feature/x", (try currentBranch(arena, found)).?); +} + +test "layout discovery rejects bad gitdir files and skips non-git .git directories" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try writeTestFile(tmp.dir, "bad/.git", "not a pointer\n"); + try writeTestFile(tmp.dir, "dangling/.git", "gitdir: ./missing\n"); + try makeTestRepo(tmp.dir, "outer/.git"); + try tmp.dir.createDirPath(std.testing.io, "outer/inner/.git"); + try tmp.dir.createDirPath(std.testing.io, "plain/sub"); + const root = try io_mod.dirRealpathAlloc(arena, tmp.dir, "."); + + try std.testing.expectError(error.InvalidGitFile, discover(arena, try std.fs.path.join(arena, &.{ root, "bad" }))); + try std.testing.expectError(error.InvalidGitFile, discover(arena, try std.fs.path.join(arena, &.{ root, "dangling" }))); + const outer = (try discover(arena, try std.fs.path.join(arena, &.{ root, "outer/inner" }))).?; + try std.testing.expectEqualStrings(try std.fs.path.join(arena, &.{ root, "outer" }), outer.worktree_root); + try std.testing.expectEqualStrings("inner", outer.prefix); + // The test directory may itself sit inside a checkout, so only require + // that discovery does not stop inside it. + if (try discover(arena, try std.fs.path.join(arena, &.{ root, "plain/sub" }))) |enclosing| { + try std.testing.expect(!std.mem.startsWith(u8, enclosing.worktree_root, root)); + } +} diff --git a/src/core/indexer/sources.zig b/src/core/indexer/sources.zig new file mode 100644 index 000000000..5a5f21f56 --- /dev/null +++ b/src/core/indexer/sources.zig @@ -0,0 +1,133 @@ +//! Ignore sources in git's precedence, lowest first: the user's excludes file, +//! then the repository's `info/exclude`, then `.gitignore` files from the root +//! down, which a walker loads as it enters each directory. Every source is a +//! bounded regular-file read; one that cannot be read is reported so the +//! caller can mark its result incomplete, never replaced by running git. + +const std = @import("std"); +const io_mod = @import("../shared/io.zig"); +const bounded_read = @import("bounded_read.zig"); +const git_config = @import("git_config.zig"); +const ignore = @import("ignore.zig"); +const layout_mod = @import("layout.zig"); + +const Allocator = std.mem.Allocator; + +pub const Unavailable = struct { + path: []const u8, + /// Static string. + reason: []const u8, +}; + +pub const RepositoryWide = struct { + /// Lowest precedence first. + lists: []const *const ignore.PatternList, + unavailable: []const Unavailable, +}; + +/// Loads the excludes file named by `settings` and `info/exclude` from the +/// common git directory. Memory belongs to `arena`. +pub fn loadRepositoryWide(arena: Allocator, layout: layout_mod.Layout, settings: git_config.IgnoreSettings) Allocator.Error!RepositoryWide { + var lists: std.ArrayList(*const ignore.PatternList) = .empty; + var unavailable: std.ArrayList(Unavailable) = .empty; + const info_exclude = try std.fs.path.join(arena, &.{ layout.common_dir, "info", "exclude" }); + const paths = [_]?[]const u8{ settings.excludes_file, info_exclude }; + for (paths) |maybe_path| { + const path = maybe_path orelse continue; + switch (try bounded_read.readAbsolute(arena, path, ignore.max_pattern_file_bytes)) { + .content => |content| { + const list = try arena.create(ignore.PatternList); + list.* = try ignore.PatternList.parse(arena, "", content); + try lists.append(arena, list); + }, + .missing => {}, + .unavailable => |reason| try unavailable.append(arena, .{ .path = path, .reason = reason }), + } + } + return .{ + .lists = try lists.toOwnedSlice(arena), + .unavailable = try unavailable.toOwnedSlice(arena), + }; +} + +pub const DirectoryList = union(enum) { + absent, + list: *const ignore.PatternList, + /// Static string. + unavailable: []const u8, +}; + +/// Loads `.gitignore` from the open directory `dir`, whose path relative to +/// the worktree root is `base` (empty, or ending in `/`). A symlinked +/// `.gitignore` is not followed, as in git. Memory belongs to `arena`. +pub fn loadDirectory(arena: Allocator, dir: std.Io.Dir, base: []const u8) Allocator.Error!DirectoryList { + return switch (try bounded_read.readAt(arena, dir, ".gitignore", ignore.max_pattern_file_bytes)) { + .content => |content| blk: { + const list = try arena.create(ignore.PatternList); + list.* = try ignore.PatternList.parse(arena, base, content); + break :blk .{ .list = list }; + }, + .missing => .absent, + .unavailable => |reason| .{ .unavailable = reason }, + }; +} + +fn writeTestFile(dir: std.Io.Dir, path: []const u8, content: []const u8) !void { + if (std.fs.path.dirname(path)) |parent| try dir.createDirPath(std.testing.io, parent); + try dir.writeFile(std.testing.io, .{ .sub_path = path, .data = content }); +} + +test "repository-wide ignore sources load in git's precedence and report unreadable ones" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try writeTestFile(tmp.dir, "repo/.git/HEAD", "ref: refs/heads/main\n"); + try tmp.dir.createDirPath(std.testing.io, "repo/.git/objects"); + try writeTestFile(tmp.dir, "repo/.git/info/exclude", "!keep.tmp\n"); + try writeTestFile(tmp.dir, "global-ignore", "*.tmp\n"); + const root = try io_mod.dirRealpathAlloc(arena, tmp.dir, "."); + const layout = (try layout_mod.discover(arena, try std.fs.path.join(arena, &.{ root, "repo" }))).?; + + const settings: git_config.IgnoreSettings = .{ + .excludes_file = try std.fs.path.join(arena, &.{ root, "global-ignore" }), + .ignore_case = false, + .object_format = .sha1, + .invalid = &.{}, + }; + const wide = try loadRepositoryWide(arena, layout, settings); + try std.testing.expectEqual(@as(usize, 2), wide.lists.len); + try std.testing.expectEqual(@as(usize, 0), wide.unavailable.len); + try std.testing.expectEqual(ignore.Decision.excluded, ignore.decide(wide.lists, "a.tmp", false, false)); + try std.testing.expectEqual(ignore.Decision.included, ignore.decide(wide.lists, "keep.tmp", false, false)); + + var unreadable = settings; + unreadable.excludes_file = root; + const partial = try loadRepositoryWide(arena, layout, unreadable); + try std.testing.expectEqual(@as(usize, 1), partial.lists.len); + try std.testing.expectEqual(@as(usize, 1), partial.unavailable.len); +} + +test "directory .gitignore loads with its base and refuses symlinks" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try writeTestFile(tmp.dir, "sub/.gitignore", "/local\n"); + try writeTestFile(tmp.dir, "target", "x\n"); + try tmp.dir.createDirPath(std.testing.io, "linked"); + var sub = try tmp.dir.openDir(std.testing.io, "sub", .{}); + defer sub.close(std.testing.io); + const loaded = try loadDirectory(arena, sub, "sub/"); + const lists = [_]*const ignore.PatternList{loaded.list}; + try std.testing.expectEqual(ignore.Decision.excluded, ignore.decide(&lists, "sub/local", false, false)); + try std.testing.expectEqual(ignore.Decision.none, ignore.decide(&lists, "local", false, false)); + try std.testing.expect(try loadDirectory(arena, tmp.dir, "") == .absent); + + var linked = try tmp.dir.openDir(std.testing.io, "linked", .{}); + defer linked.close(std.testing.io); + linked.symLink(std.testing.io, "../target", ".gitignore", .{}) catch return error.SkipZigTest; + try std.testing.expect(try loadDirectory(arena, linked, "linked/") == .unavailable); +} diff --git a/src/core/indexer/wildmatch.zig b/src/core/indexer/wildmatch.zig new file mode 100644 index 000000000..003d4e258 --- /dev/null +++ b/src/core/indexer/wildmatch.zig @@ -0,0 +1,304 @@ +//! Port of git's wildmatch (wildmatch.c), used for ignore patterns and config +//! include conditions. Matching is byte-wise and case folding is ASCII-only, +//! as in git. Quirks are preserved on purpose: parity with git matters more +//! than tidiness here. + +const std = @import("std"); + +pub const Flags = struct { + /// `WM_PATHNAME`: `*`, `?` and bracket expressions never match `/`, and + /// `**` between slashes matches any number of directories. + pathname: bool = false, + /// `WM_CASEFOLD`: ASCII case-insensitive matching. + casefold: bool = false, +}; + +const Result = enum { match, no_match, abort_all, abort_to_starstar }; + +/// Reports whether `text` matches `pattern`. Like git, a NUL byte ends either +/// string. +pub fn match(pattern: []const u8, text: []const u8, flags: Flags) bool { + return doWild(pattern, 0, text, 0, flags) == .match; +} + +fn at(s: []const u8, i: usize) u8 { + return if (i < s.len) s[i] else 0; +} + +fn lower(c: u8) u8 { + return std.ascii.toLower(c); +} + +fn isGlobSpecial(c: u8) bool { + return c == '*' or c == '?' or c == '[' or c == '\\'; +} + +fn hasSlashFrom(s: []const u8, start: usize) ?usize { + var i = start; + while (i < s.len and s[i] != 0) : (i += 1) { + if (s[i] == '/') return i; + } + return null; +} + +fn doWild(pattern: []const u8, p_start: usize, text: []const u8, t_start: usize, flags: Flags) Result { + var p = p_start; + var t = t_start; + outer: while (at(pattern, p) != 0) : ({ + t += 1; + p += 1; + }) { + var p_ch = at(pattern, p); + var t_ch = at(text, t); + if (t_ch == 0 and p_ch != '*') return .abort_all; + if (flags.casefold) { + t_ch = lower(t_ch); + p_ch = lower(p_ch); + } + switch (p_ch) { + '\\' => { + // Literal match with the following character, which git does + // not case-fold. + p += 1; + if (t_ch != at(pattern, p)) return .no_match; + }, + '?' => { + if (flags.pathname and t_ch == '/') return .no_match; + }, + '*' => { + p += 1; + var match_slash: bool = undefined; + if (at(pattern, p) == '*') { + const prev_is_boundary = p < p_start + 2 or pattern[p - 2] == '/'; + while (at(pattern, p) == '*') p += 1; + const next = at(pattern, p); + if (prev_is_boundary and (next == 0 or next == '/' or + (next == '\\' and at(pattern, p + 1) == '/'))) + { + // `**/` may match no directories at all. + if (next == '/' and doWild(pattern, p + 1, text, t, flags) == .match) return .match; + match_slash = true; + } else { + match_slash = !flags.pathname; + } + } else { + match_slash = !flags.pathname; + } + + const next = at(pattern, p); + if (next == 0) { + // A trailing `**` matches everything; a trailing `*` only + // when no slash remains. + if (!match_slash and hasSlashFrom(text, t) != null) return .no_match; + return .match; + } else if (!match_slash and next == '/') { + // One `*` followed by a slash matches the next directory; + // the loop's increment consumes both slashes. + t = hasSlashFrom(text, t) orelse return .no_match; + continue :outer; + } + + while (true) { + if (t_ch == 0) break; + // Skip ahead to the next occurrence of a literal that + // follows the asterisk. + if (!isGlobSpecial(at(pattern, p))) { + var literal = at(pattern, p); + if (flags.casefold) literal = lower(literal); + while (true) { + t_ch = at(text, t); + if (t_ch == 0) break; + if (!match_slash and t_ch == '/') break; + if (flags.casefold) t_ch = lower(t_ch); + if (t_ch == literal) break; + t += 1; + } + if (t_ch != literal) return if (match_slash) .abort_all else .abort_to_starstar; + } + const matched = doWild(pattern, p, text, t, flags); + if (matched != .no_match) { + if (!match_slash or matched != .abort_to_starstar) return matched; + } else if (!match_slash and t_ch == '/') { + return .abort_to_starstar; + } + t += 1; + t_ch = at(text, t); + } + return .abort_all; + }, + '[' => { + p += 1; + p_ch = at(pattern, p); + if (p_ch == '^') p_ch = '!'; + const negated = p_ch == '!'; + if (negated) { + p += 1; + p_ch = at(pattern, p); + } + var prev_ch: u8 = 0; + var matched = false; + // Mirrors git's do-while: the first member may be `]`. + while (true) { + if (p_ch == 0) return .abort_all; + if (p_ch == '\\') { + p += 1; + p_ch = at(pattern, p); + if (p_ch == 0) return .abort_all; + if (t_ch == p_ch) matched = true; + } else if (p_ch == '-' and prev_ch != 0 and at(pattern, p + 1) != 0 and at(pattern, p + 1) != ']') { + p += 1; + p_ch = at(pattern, p); + if (p_ch == '\\') { + p += 1; + p_ch = at(pattern, p); + if (p_ch == 0) return .abort_all; + } + if (t_ch <= p_ch and t_ch >= prev_ch) { + matched = true; + } else if (flags.casefold and std.ascii.isLower(t_ch)) { + const upper = std.ascii.toUpper(t_ch); + if (upper <= p_ch and upper >= prev_ch) matched = true; + } + p_ch = 0; + } else if (p_ch == '[' and at(pattern, p + 1) == ':') { + p += 2; + const class_start = p; + while (at(pattern, p) != 0 and at(pattern, p) != ']') p += 1; + p_ch = at(pattern, p); + if (p_ch == 0) return .abort_all; + if (p < class_start + 1 or pattern[p - 1] != ':') { + // No closing `:]`: treat `[` as a plain member. + p = class_start - 2; + p_ch = '['; + if (t_ch == p_ch) matched = true; + } else { + const class = pattern[class_start .. p - 1]; + const in_class = classMatches(class, t_ch, flags.casefold) orelse return .abort_all; + if (in_class) matched = true; + p_ch = 0; + } + } else if (t_ch == p_ch) { + matched = true; + } + prev_ch = p_ch; + p += 1; + p_ch = at(pattern, p); + if (p_ch == ']') break; + } + if (matched == negated or (flags.pathname and t_ch == '/')) return .no_match; + }, + else => { + if (t_ch != p_ch) return .no_match; + }, + } + } + return if (at(text, t) != 0) .no_match else .match; +} + +/// Returns whether `c` belongs to the POSIX class, or null for an unknown +/// class name, which git treats as a malformed pattern. +fn classMatches(class: []const u8, c: u8, casefold: bool) ?bool { + const eql = std.mem.eql; + if (eql(u8, class, "alnum")) return std.ascii.isAlphanumeric(c); + if (eql(u8, class, "alpha")) return std.ascii.isAlphabetic(c); + if (eql(u8, class, "blank")) return c == ' ' or c == '\t'; + if (eql(u8, class, "cntrl")) return c < 0x20 or c == 0x7f; + if (eql(u8, class, "digit")) return std.ascii.isDigit(c); + if (eql(u8, class, "graph")) return c > 0x20 and c < 0x7f; + if (eql(u8, class, "lower")) return std.ascii.isLower(c); + if (eql(u8, class, "print")) return c >= 0x20 and c < 0x7f; + if (eql(u8, class, "punct")) return c > 0x20 and c < 0x7f and !std.ascii.isAlphanumeric(c); + if (eql(u8, class, "space")) return c == ' ' or c == '\t' or c == '\n' or c == '\r'; + if (eql(u8, class, "upper")) return std.ascii.isUpper(c) or (casefold and std.ascii.isLower(c)); + if (eql(u8, class, "xdigit")) return std.ascii.isHex(c); + return null; +} + +const Case = struct { pattern: []const u8, text: []const u8, expected: bool }; + +fn expectCases(cases: []const Case, flags: Flags) !void { + for (cases) |case| { + if (match(case.pattern, case.text, flags) != case.expected) { + std.debug.print("pattern={s} text={s} expected={}\n", .{ case.pattern, case.text, case.expected }); + return error.TestExpectedEqual; + } + } +} + +test "wildmatch literal, single-character and bracket rules" { + try expectCases(&.{ + .{ .pattern = "foo", .text = "foo", .expected = true }, + .{ .pattern = "foo", .text = "bar", .expected = false }, + .{ .pattern = "", .text = "", .expected = true }, + .{ .pattern = "???", .text = "foo", .expected = true }, + .{ .pattern = "??", .text = "foo", .expected = false }, + .{ .pattern = "*", .text = "foo", .expected = true }, + .{ .pattern = "f*", .text = "foo", .expected = true }, + .{ .pattern = "*f", .text = "foo", .expected = false }, + .{ .pattern = "*foo*", .text = "foo", .expected = true }, + .{ .pattern = "*ob*a*r*", .text = "foobar", .expected = true }, + .{ .pattern = "*ab", .text = "aaaaaaabababab", .expected = true }, + .{ .pattern = "foo\\*", .text = "foo*", .expected = true }, + .{ .pattern = "foo\\*bar", .text = "foobar", .expected = false }, + .{ .pattern = "f\\\\oo", .text = "f\\oo", .expected = true }, + .{ .pattern = "*[al]?", .text = "ball", .expected = true }, + .{ .pattern = "[ten]", .text = "ten", .expected = false }, + .{ .pattern = "t[a-g]n", .text = "ten", .expected = true }, + .{ .pattern = "t[!a-g]n", .text = "ten", .expected = false }, + .{ .pattern = "t[!a-g]n", .text = "ton", .expected = true }, + .{ .pattern = "t[^a-g]n", .text = "ton", .expected = true }, + .{ .pattern = "a[]]b", .text = "a]b", .expected = true }, + .{ .pattern = "a[]-]b", .text = "a-b", .expected = true }, + .{ .pattern = "a[]-]b", .text = "aab", .expected = false }, + .{ .pattern = "a[]a-]b", .text = "aab", .expected = true }, + .{ .pattern = "]", .text = "]", .expected = true }, + .{ .pattern = "[[:digit:]]x", .text = "7x", .expected = true }, + .{ .pattern = "[[:digit:]]x", .text = "ax", .expected = false }, + .{ .pattern = "[[:bogus:]]", .text = "a", .expected = false }, + .{ .pattern = "[abc", .text = "a", .expected = false }, + }, .{ .pathname = true }); +} + +test "wildmatch pathname rules for slashes and double asterisks" { + try expectCases(&.{ + .{ .pattern = "foo*bar", .text = "foo/baz/bar", .expected = false }, + .{ .pattern = "foo**bar", .text = "foo/baz/bar", .expected = false }, + .{ .pattern = "foo**bar", .text = "foobazbar", .expected = true }, + .{ .pattern = "foo/**/bar", .text = "foo/baz/bar", .expected = true }, + .{ .pattern = "foo/**/bar", .text = "foo/b/a/z/bar", .expected = true }, + .{ .pattern = "foo/**/bar", .text = "foo/bar", .expected = true }, + .{ .pattern = "foo/**/**/bar", .text = "foo/bar", .expected = true }, + .{ .pattern = "foo?bar", .text = "foo/bar", .expected = false }, + .{ .pattern = "foo[/]bar", .text = "foo/bar", .expected = false }, + .{ .pattern = "foo[^a-z]bar", .text = "foo/bar", .expected = false }, + .{ .pattern = "**/foo", .text = "foo", .expected = true }, + .{ .pattern = "**/foo", .text = "bar/baz/foo", .expected = true }, + .{ .pattern = "*/foo", .text = "bar/baz/foo", .expected = false }, + .{ .pattern = "**/bar*", .text = "foo/bar/baz", .expected = false }, + .{ .pattern = "**/bar/*", .text = "deep/foo/bar/baz", .expected = true }, + .{ .pattern = "**/bar/*", .text = "deep/foo/bar/baz/", .expected = false }, + .{ .pattern = "**/bar/**", .text = "deep/foo/bar/baz/", .expected = true }, + .{ .pattern = "**/bar/*", .text = "deep/foo/bar", .expected = false }, + .{ .pattern = "**/bar**", .text = "foo/bar/baz", .expected = false }, + .{ .pattern = "*/bar/**", .text = "foo/bar/baz/x", .expected = true }, + .{ .pattern = "*/bar/**", .text = "deep/foo/bar/baz/x", .expected = false }, + .{ .pattern = "**", .text = "a/b/c", .expected = true }, + .{ .pattern = "a/*", .text = "a/b/c", .expected = false }, + }, .{ .pathname = true }); +} + +test "wildmatch case folding is ASCII-only and keeps git's escape quirk" { + try expectCases(&.{ + .{ .pattern = "FOO", .text = "foo", .expected = true }, + .{ .pattern = "f*O", .text = "Foo", .expected = true }, + .{ .pattern = "[a-c]x", .text = "Bx", .expected = true }, + .{ .pattern = "[[:upper:]]x", .text = "bx", .expected = true }, + // git compares an escaped character without folding it. + .{ .pattern = "\\Ax", .text = "ax", .expected = false }, + .{ .pattern = "\xc3\x89", .text = "\xc3\xa9", .expected = false }, + }, .{ .pathname = true, .casefold = true }); +} + +test "wildmatch stops at a NUL byte like git's C strings" { + try std.testing.expect(match("ab\x00zz", "ab", .{ .pathname = true })); +} diff --git a/src/main.zig b/src/main.zig index 50035e996..8a176f6a7 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4844,6 +4844,7 @@ test { _ = @import("core/workspace/file_index.zig"); _ = @import("core/workspace/path_completion.zig"); _ = @import("core/workspace/directory_completion_job.zig"); + _ = @import("core/indexer/indexer.zig"); _ = @import("core/input/file_completion_state.zig"); _ = @import("gateway/vercel_protocol.zig"); _ = @import("core/gateway/provider_set.zig"); From 9931ab7fefd520db92dff81f1fc0521a80f0e1a0 Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 8 Oct 2026 13:56:29 -0400 Subject: [PATCH 02/23] Run fx's own git commands through one hardened helper - Build the agent's auto-run git status, diff, and log and the fx pr and fx issue snapshot through one helper that turns off programs named by repository config: fsmonitor, hooks, the pager, diff drivers, filters, and signature verification. - Refuse to run git when it would use a different repository than the one fx read, when a filter driver cannot be turned off, or when git is older than 2.36. The agent's command then goes to normal review, and the snapshot reports git as unavailable. - Keep the user's global and system git config in those commands. - Recognize a git directory the way git does, so fx and git pick the same repository. --- src/core/github/git_context.zig | 107 ++-- src/core/indexer/git_config.zig | 1 + src/core/indexer/indexer.zig | 62 ++- src/core/indexer/layout.zig | 81 ++- src/core/indexer/sources.zig | 1 + src/core/permissions/direct_command.zig | 34 +- src/core/shell_command/command_effect.zig | 157 ++++-- src/core/shell_command/safe_git.zig | 614 ++++++++++++++++++++++ src/core/workspace/workspace_files.zig | 23 +- src/main.zig | 1 + 10 files changed, 942 insertions(+), 139 deletions(-) create mode 100644 src/core/shell_command/safe_git.zig diff --git a/src/core/github/git_context.zig b/src/core/github/git_context.zig index 96bfdb68f..a8e2bd1b7 100644 --- a/src/core/github/git_context.zig +++ b/src/core/github/git_context.zig @@ -1,5 +1,7 @@ const std = @import("std"); const io_mod = @import("../shared/io.zig"); +const debug_trace = @import("../shared/debug_trace.zig"); +const safe_git = @import("../shell_command/safe_git.zig"); const Allocator = std.mem.Allocator; @@ -12,46 +14,86 @@ pub const Snapshot = struct { } }; +const Query = struct { + subcommand: safe_git.Subcommand, + args: []const []const u8, +}; + +const branch_query: Query = .{ .subcommand = .branch, .args = &.{"--show-current"} }; +const status_query: Query = .{ .subcommand = .status, .args = &.{ "--short", "--branch" } }; +const log_query: Query = .{ .subcommand = .log, .args = &.{ "--oneline", "-5" } }; +const staged_query: Query = .{ .subcommand = .diff, .args = &.{ "--stat", "--cached" } }; +const unstaged_query: Query = .{ .subcommand = .diff, .args = &.{"--stat"} }; +const inside_query: Query = .{ .subcommand = .rev_parse, .args = &.{"--is-inside-work-tree"} }; +/// Every git command the snapshot runs. +const queries = [_]Query{ branch_query, status_query, log_query, staged_query, unstaged_query, inside_query }; + +/// Captures the working directory's git state through safe git. When fx +/// cannot run git safely there, every section reads as unavailable and the +/// snapshot does not claim the directory is outside a repository, so `fx pr` +/// still starts and its agent can inspect the files directly. pub fn snapshot(alloc: Allocator) !Snapshot { - const in_git_repo = isGitRepository(alloc); - const branch = try runGit(alloc, &.{ "branch", "--show-current" }); + var arena_state = std.heap.ArenaAllocator.init(alloc); + defer arena_state.deinit(); + const ready = switch (try prepareHere(arena_state.allocator())) { + .ready => |ready| ready, + .refused => return .{ + .in_git_repo = true, + .text = try formatSnapshot(alloc, null, null, null, null, null), + }, + }; + + const branch = try runGit(alloc, ready, branch_query); defer if (branch) |text| alloc.free(text); - const status = try runGit(alloc, &.{ "status", "--short", "--branch" }); + const status = try runGit(alloc, ready, status_query); defer if (status) |text| alloc.free(text); - const log = try runGit(alloc, &.{ "log", "--oneline", "-5" }); + const log = try runGit(alloc, ready, log_query); defer if (log) |text| alloc.free(text); - const staged = try runGit(alloc, &.{ "diff", "--stat", "--cached" }); + const staged = try runGit(alloc, ready, staged_query); defer if (staged) |text| alloc.free(text); - const unstaged = try runGit(alloc, &.{ "diff", "--stat" }); + const unstaged = try runGit(alloc, ready, unstaged_query); defer if (unstaged) |text| alloc.free(text); return .{ - .in_git_repo = in_git_repo, + .in_git_repo = isGitRepository(alloc, ready), .text = try formatSnapshot(alloc, branch, status, log, staged, unstaged), }; } -fn buildGitArgv(alloc: Allocator, args: []const []const u8) !std.ArrayList([]const u8) { - var argv = std.ArrayList([]const u8).empty; - errdefer argv.deinit(alloc); - try argv.append(alloc, "git"); - try argv.append(alloc, "--no-optional-locks"); - try argv.appendSlice(alloc, args); - return argv; +fn prepareHere(arena: Allocator) Allocator.Error!safe_git.Prepared { + const cwd = io_mod.realpathAlloc(arena, ".") catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => { + debug_trace.logf("core", "git snapshot unavailable reason=cwd_unresolved error={s}", .{@errorName(err)}); + return .{ .refused = .repo_config_unavailable }; + }, + }; + return safe_git.prepare(arena, cwd); } -pub fn isGitRepository(alloc: Allocator) bool { - const result = runGit(alloc, &.{ "rev-parse", "--is-inside-work-tree" }) catch return false; +/// Memory belongs to `arena`. +fn buildGitArgv(arena: Allocator, ready: safe_git.Ready, query: Query) Allocator.Error![]const []const u8 { + var argv: std.ArrayList([]const u8) = .empty; + try safe_git.appendCommand(arena, &argv, ready, query.subcommand); + try argv.appendSlice(arena, query.args); + return argv.items; +} + +fn isGitRepository(alloc: Allocator, ready: safe_git.Ready) bool { + const result = runGit(alloc, ready, inside_query) catch return false; defer if (result) |text| alloc.free(text); return if (result) |text| std.mem.eql(u8, text, "true") else false; } -fn runGit(alloc: Allocator, args: []const []const u8) !?[]u8 { - var argv = try buildGitArgv(alloc, args); - defer argv.deinit(alloc); +fn runGit(alloc: Allocator, ready: safe_git.Ready, query: Query) !?[]u8 { + var arena_state = std.heap.ArenaAllocator.init(alloc); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var environment = try safe_git.environment(arena); const result = std.process.run(alloc, io_mod.getIo(), .{ - .argv = argv.items, + .argv = try buildGitArgv(arena, ready, query), + .environ_map = &environment, }) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, else => return null, @@ -161,21 +203,26 @@ test "snapshot formatting: populated sections preserve layout exactly" { , text); } -test "git argv: read-only commands disable optional locks" { - var argv = try buildGitArgv(std.testing.allocator, &.{ "status", "--short", "--branch" }); - defer argv.deinit(std.testing.allocator); +test "git argv: every snapshot command is built by the safe-git helper" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const ready: safe_git.Ready = .{ .executable = "/usr/bin/git", .filter_names = &.{"x"} }; - try std.testing.expectEqual(@as(usize, 5), argv.items.len); - try std.testing.expectEqualStrings("git", argv.items[0]); - try std.testing.expectEqualStrings("--no-optional-locks", argv.items[1]); - try std.testing.expectEqualStrings("status", argv.items[2]); - try std.testing.expectEqualStrings("--short", argv.items[3]); - try std.testing.expectEqualStrings("--branch", argv.items[4]); + for (queries) |query| { + var prefix: std.ArrayList([]const u8) = .empty; + try safe_git.appendCommand(arena, &prefix, ready, query.subcommand); + const argv = try buildGitArgv(arena, ready, query); + try std.testing.expectEqual(prefix.items.len + query.args.len, argv.len); + for (prefix.items, argv[0..prefix.items.len]) |want, have| try std.testing.expectEqualStrings(want, have); + for (query.args, argv[prefix.items.len..]) |want, have| try std.testing.expectEqualStrings(want, have); + } } test "git repository detection treats allocation failure as unavailable" { var failing = std.testing.FailingAllocator.init(std.testing.allocator, .{ .fail_index = 0 }); - try std.testing.expect(!isGitRepository(failing.allocator())); + const ready: safe_git.Ready = .{ .executable = "/usr/bin/git", .filter_names = &.{} }; + try std.testing.expect(!isGitRepository(failing.allocator(), ready)); } test "snapshot returns owned Git status text" { diff --git a/src/core/indexer/git_config.zig b/src/core/indexer/git_config.zig index 6cbd936d6..df3844a12 100644 --- a/src/core/indexer/git_config.zig +++ b/src/core/indexer/git_config.zig @@ -623,6 +623,7 @@ fn testRepo(arena: Allocator, tmp: *std.testing.TmpDir, repo_path: []const u8, b const head = try std.fmt.allocPrint(arena, "{s}/.git/HEAD", .{repo_path}); try writeTestFile(tmp.dir, head, try std.fmt.allocPrint(arena, "ref: refs/heads/{s}\n", .{branch})); try tmp.dir.createDirPath(std.testing.io, try std.fmt.allocPrint(arena, "{s}/.git/objects", .{repo_path})); + try tmp.dir.createDirPath(std.testing.io, try std.fmt.allocPrint(arena, "{s}/.git/refs", .{repo_path})); try tmp.dir.createDirPath(std.testing.io, "home"); const root = try io_mod.dirRealpathAlloc(arena, tmp.dir, "."); const layout = (try layout_mod.discover(arena, try std.fs.path.join(arena, &.{ root, repo_path }))).?; diff --git a/src/core/indexer/indexer.zig b/src/core/indexer/indexer.zig index ab8e34598..b64d2b7fb 100644 --- a/src/core/indexer/indexer.zig +++ b/src/core/indexer/indexer.zig @@ -12,26 +12,30 @@ const layout = @import("layout.zig"); const Allocator = std.mem.Allocator; -pub const FilterNames = struct { - /// The slice and every name are owned by the allocator passed to - /// `repoFilterNames`. +pub const RepoFilters = struct { + /// Absolute git directory of the repository containing the root, found + /// the way git discovers one, or null outside a repository. + git_dir: ?[]const u8, + /// Filter driver names with a `clean`, `smudge` or `process` command. names: []const []const u8, - pub fn deinit(self: FilterNames, alloc: Allocator) void { + /// Frees memory owned by the allocator passed to `repoFilters`. + pub fn deinit(self: RepoFilters, alloc: Allocator) void { + if (self.git_dir) |git_dir| alloc.free(git_dir); for (self.names) |name| alloc.free(name); alloc.free(self.names); } }; -pub const RepoFilterNamesError = error{ OutOfMemory, RepoConfigUnavailable }; +pub const RepoFiltersError = error{ OutOfMemory, RepoConfigUnavailable }; -/// Returns every filter driver name with a `clean`, `smudge` or `process` -/// command in the repository's own config: `config`, `config.worktree` and -/// the files they include, counting every `includeIf` as included so no -/// condition can hide a driver. Empty outside a git repository. A repository -/// config file that cannot be read within bounds, a malformed one, or an -/// invalid `.git` file fails closed with `error.RepoConfigUnavailable`. -pub fn repoFilterNames(alloc: Allocator, workspace_root: []const u8) RepoFilterNamesError!FilterNames { +/// Finds the repository containing the absolute `workspace_root` and every +/// filter driver its own config defines: `config`, `config.worktree` and the +/// files they include, counting every `includeIf` as included so no condition +/// can hide a driver. A repository config file that cannot be read within +/// bounds, a malformed one, or an invalid `.git` file fails closed with +/// `error.RepoConfigUnavailable`. +pub fn repoFilters(alloc: Allocator, workspace_root: []const u8) RepoFiltersError!RepoFilters { if (!std.fs.path.isAbsolute(workspace_root)) return error.RepoConfigUnavailable; var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); @@ -54,6 +58,8 @@ pub fn repoFilterNames(alloc: Allocator, workspace_root: []const u8) RepoFilterN } const names = try git_config.filterDriverNames(arena, loaded); + const git_dir: ?[]const u8 = if (found) |repository| try alloc.dupe(u8, repository.git_dir) else null; + errdefer if (git_dir) |path| alloc.free(path); const owned = try alloc.alloc([]const u8, names.len); var copied: usize = 0; errdefer { @@ -64,7 +70,7 @@ pub fn repoFilterNames(alloc: Allocator, workspace_root: []const u8) RepoFilterN owned[copied] = try alloc.dupe(u8, name); copied += 1; } - return .{ .names = owned }; + return .{ .git_dir = git_dir, .names = owned }; } test { @@ -81,30 +87,38 @@ fn writeTestFile(dir: std.Io.Dir, path: []const u8, content: []const u8) !void { try dir.writeFile(std.testing.io, .{ .sub_path = path, .data = content }); } -test "repoFilterNames returns repository drivers and fails closed on unreadable config" { +fn makeTestRepo(dir: std.Io.Dir, name: []const u8) !void { + var buffer: [64]u8 = undefined; + try writeTestFile(dir, try std.fmt.bufPrint(&buffer, "{s}/.git/HEAD", .{name}), "ref: refs/heads/main\n"); + try dir.createDirPath(std.testing.io, try std.fmt.bufPrint(&buffer, "{s}/.git/objects", .{name})); + try dir.createDirPath(std.testing.io, try std.fmt.bufPrint(&buffer, "{s}/.git/refs", .{name})); +} + +test "repoFilters returns the repository and its drivers and fails closed on unreadable config" { const alloc = std.testing.allocator; var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); - try writeTestFile(tmp.dir, "repo/.git/HEAD", "ref: refs/heads/main\n"); - try tmp.dir.createDirPath(std.testing.io, "repo/.git/objects"); + try makeTestRepo(tmp.dir, "repo"); try writeTestFile(tmp.dir, "repo/.git/config", "[includeIf \"gitdir:/nowhere/\"]\npath = extra\n"); try writeTestFile(tmp.dir, "repo/.git/extra", "[filter \"x\"]\nclean = /tmp/run-me\n"); try tmp.dir.createDirPath(std.testing.io, "repo/src"); - try writeTestFile(tmp.dir, "bad/.git/HEAD", "ref: refs/heads/main\n"); - try tmp.dir.createDirPath(std.testing.io, "bad/.git/objects"); + try makeTestRepo(tmp.dir, "bad"); try tmp.dir.createDirPath(std.testing.io, "bad/.git/config"); const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); defer alloc.free(root); const repo_src = try std.fs.path.join(alloc, &.{ root, "repo/src" }); defer alloc.free(repo_src); - const names = try repoFilterNames(alloc, repo_src); - defer names.deinit(alloc); - try std.testing.expectEqual(@as(usize, 1), names.names.len); - try std.testing.expectEqualStrings("x", names.names[0]); + const filters = try repoFilters(alloc, repo_src); + defer filters.deinit(alloc); + const expected_git_dir = try std.fs.path.join(alloc, &.{ root, "repo/.git" }); + defer alloc.free(expected_git_dir); + try std.testing.expectEqualStrings(expected_git_dir, filters.git_dir.?); + try std.testing.expectEqual(@as(usize, 1), filters.names.len); + try std.testing.expectEqualStrings("x", filters.names[0]); const bad = try std.fs.path.join(alloc, &.{ root, "bad" }); defer alloc.free(bad); - try std.testing.expectError(error.RepoConfigUnavailable, repoFilterNames(alloc, bad)); - try std.testing.expectError(error.RepoConfigUnavailable, repoFilterNames(alloc, "relative")); + try std.testing.expectError(error.RepoConfigUnavailable, repoFilters(alloc, bad)); + try std.testing.expectError(error.RepoConfigUnavailable, repoFilters(alloc, "relative")); } diff --git a/src/core/indexer/layout.zig b/src/core/indexer/layout.zig index 9b0ed61cb..540662f08 100644 --- a/src/core/indexer/layout.zig +++ b/src/core/indexer/layout.zig @@ -86,13 +86,20 @@ fn readGitFile(arena: Allocator, dir: []const u8, dot_git: []const u8) Allocator const GitDirs = struct { git_dir: []const u8, common_dir: []const u8 }; -/// A light form of git's is_git_directory: `HEAD` in the git directory and -/// `objects` in the common directory, which `commondir` may relocate. +/// git's is_git_directory: a valid `HEAD` in the git directory, and `objects` +/// and `refs` directories in the common directory, which `commondir` may +/// relocate. Discovery must accept exactly the directories git accepts, or git +/// would walk past one fx stopped at and use another repository's config. A +/// symlinked `HEAD`, which git accepts when it points into `refs/`, is +/// rejected here; safe git then refuses because git and fx disagree. fn gitDirs(arena: Allocator, git_dir: []const u8) Allocator.Error!?GitDirs { var dir = std.Io.Dir.openDirAbsolute(io_mod.getIo(), git_dir, .{}) catch return null; defer dir.close(io_mod.getIo()); - const head = dir.statFile(io_mod.getIo(), "HEAD", .{ .follow_symlinks = false }) catch return null; - if (head.kind != .file) return null; + const head = switch (try bounded_read.readAt(arena, dir, "HEAD", max_metadata_bytes)) { + .content => |bytes| bytes, + .missing, .unavailable => return null, + }; + if (!validHead(head[0..@min(head.len, max_head_bytes)])) return null; const common_dir = switch (try bounded_read.readAt(arena, dir, "commondir", max_metadata_bytes)) { .content => |bytes| blk: { const raw = std.mem.trimEnd(u8, bytes, " \t\r\n"); @@ -102,12 +109,34 @@ fn gitDirs(arena: Allocator, git_dir: []const u8) Allocator.Error!?GitDirs { .missing => git_dir, .unavailable => return null, }; - const objects = try std.fs.path.join(arena, &.{ common_dir, "objects" }); - const objects_stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), objects, .{ .follow_symlinks = true }) catch return null; - if (objects_stat.kind != .directory) return null; + for ([_][]const u8{ "objects", "refs" }) |name| { + const path = try std.fs.path.join(arena, &.{ common_dir, name }); + const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), path, .{ .follow_symlinks = true }) catch return null; + if (stat.kind != .directory) return null; + } return .{ .git_dir = git_dir, .common_dir = common_dir }; } +/// git reads at most 255 bytes of `HEAD` when validating it. +const max_head_bytes: usize = 255; + +/// git's validate_headref for a regular file: a symbolic ref into `refs/`, +/// or a detached object id in either hash format. +fn validHead(content: []const u8) bool { + if (content.len < 4) return false; + if (std.mem.startsWith(u8, content, "ref:")) { + const target = std.mem.trimStart(u8, content["ref:".len..], " \t\r\n\x0b\x0c"); + if (std.mem.startsWith(u8, target, "refs/")) return true; + } + return hexPrefix(content, 40) or hexPrefix(content, 64); +} + +fn hexPrefix(content: []const u8, len: usize) bool { + if (content.len < len) return false; + for (content[0..len]) |byte| if (!std.ascii.isHex(byte)) return false; + return true; +} + /// Returns the branch `HEAD` points at (`refs/heads/`), or null when /// detached or unreadable. Memory belongs to `arena`. pub fn currentBranch(arena: Allocator, layout: Layout) Allocator.Error!?[]const u8 { @@ -131,9 +160,11 @@ fn makeTestRepo(dir: std.Io.Dir, git_dir: []const u8) !void { const head = try std.fs.path.join(std.testing.allocator, &.{ git_dir, "HEAD" }); defer std.testing.allocator.free(head); try writeTestFile(dir, head, "ref: refs/heads/main\n"); - const objects = try std.fs.path.join(std.testing.allocator, &.{ git_dir, "objects" }); - defer std.testing.allocator.free(objects); - try dir.createDirPath(std.testing.io, objects); + for ([_][]const u8{ "objects", "refs" }) |name| { + const path = try std.fs.path.join(std.testing.allocator, &.{ git_dir, name }); + defer std.testing.allocator.free(path); + try dir.createDirPath(std.testing.io, path); + } } test "layout discovery finds the innermost repository and the workspace prefix" { @@ -201,3 +232,33 @@ test "layout discovery rejects bad gitdir files and skips non-git .git directori try std.testing.expect(!std.mem.startsWith(u8, enclosing.worktree_root, root)); } } + +test "layout discovery accepts exactly the git directories git accepts" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try makeTestRepo(tmp.dir, "outer/.git"); + // Without `refs`, git skips the inner directory and uses the outer one. + try writeTestFile(tmp.dir, "outer/no-refs/.git/HEAD", "ref: refs/heads/main\n"); + try tmp.dir.createDirPath(std.testing.io, "outer/no-refs/.git/objects"); + try makeTestRepo(tmp.dir, "outer/bad-head/.git"); + try writeTestFile(tmp.dir, "outer/bad-head/.git/HEAD", "ref: heads/main\n"); + try makeTestRepo(tmp.dir, "outer/detached/.git"); + try writeTestFile(tmp.dir, "outer/detached/.git/HEAD", "0123456789abcdef0123456789abcdef01234567\n"); + try makeTestRepo(tmp.dir, "outer/spaced/.git"); + try writeTestFile(tmp.dir, "outer/spaced/.git/HEAD", "ref:\t refs/heads/main\n"); + const root = try io_mod.dirRealpathAlloc(arena, tmp.dir, "."); + + const cases = [_]struct { start: []const u8, worktree: []const u8 }{ + .{ .start = "outer/no-refs", .worktree = "outer" }, + .{ .start = "outer/bad-head", .worktree = "outer" }, + .{ .start = "outer/detached", .worktree = "outer/detached" }, + .{ .start = "outer/spaced", .worktree = "outer/spaced" }, + }; + for (cases) |case| { + const found = (try discover(arena, try std.fs.path.join(arena, &.{ root, case.start }))).?; + try std.testing.expectEqualStrings(try std.fs.path.join(arena, &.{ root, case.worktree }), found.worktree_root); + } +} diff --git a/src/core/indexer/sources.zig b/src/core/indexer/sources.zig index 5a5f21f56..c0716456d 100644 --- a/src/core/indexer/sources.zig +++ b/src/core/indexer/sources.zig @@ -85,6 +85,7 @@ test "repository-wide ignore sources load in git's precedence and report unreada defer tmp.cleanup(); try writeTestFile(tmp.dir, "repo/.git/HEAD", "ref: refs/heads/main\n"); try tmp.dir.createDirPath(std.testing.io, "repo/.git/objects"); + try tmp.dir.createDirPath(std.testing.io, "repo/.git/refs"); try writeTestFile(tmp.dir, "repo/.git/info/exclude", "!keep.tmp\n"); try writeTestFile(tmp.dir, "global-ignore", "*.tmp\n"); const root = try io_mod.dirRealpathAlloc(arena, tmp.dir, "."); diff --git a/src/core/permissions/direct_command.zig b/src/core/permissions/direct_command.zig index 8b2fc9a3e..2ec06b658 100644 --- a/src/core/permissions/direct_command.zig +++ b/src/core/permissions/direct_command.zig @@ -1,6 +1,7 @@ const std = @import("std"); const builtin = @import("builtin"); const command_effect = @import("../shell_command/command_effect.zig"); +const safe_git = @import("../shell_command/safe_git.zig"); const command_contract = @import("../execution/command_contract.zig"); const command_runner = @import("../execution/command_runner.zig"); const debug_trace = @import("../shared/debug_trace.zig"); @@ -410,24 +411,17 @@ fn environmentForProfile( alloc: std.mem.Allocator, profile: command_effect.EnvironmentProfile, ) !std.process.Environ.Map { - var environment = std.process.Environ.Map.init(alloc); - errdefer environment.deinit(); switch (profile) { - .basic_read_only, .git_read_only => { + .basic_read_only => { + var environment = std.process.Environ.Map.init(alloc); + errdefer environment.deinit(); try environment.put("PATH", "/usr/bin:/bin"); try environment.put("LC_ALL", "C"); try environment.put("LANG", "C"); + return environment; }, + .git_read_only => return safe_git.environment(alloc), } - if (profile == .git_read_only) { - try environment.put("GIT_CONFIG_NOSYSTEM", "1"); - try environment.put("GIT_CONFIG_GLOBAL", "/dev/null"); - try environment.put("GIT_OPTIONAL_LOCKS", "0"); - try environment.put("GIT_TERMINAL_PROMPT", "0"); - try environment.put("GIT_PAGER", "cat"); - try environment.put("PAGER", "cat"); - } - return environment; } const DirectTerminationCause = enum { @@ -956,17 +950,19 @@ test "direct executor runs fixed argv with sanitized environment and no artifact try std.testing.expectEqual(@as(?[]const u8, null), result.command_result.?.output_file); } -test "git direct profile removes ambient authority and disables optional mutation" { +test "git direct profile uses the safe-git environment" { var environment = try environmentForProfile(std.testing.allocator, .git_read_only); defer environment.deinit(); + var expected = try safe_git.environment(std.testing.allocator); + defer expected.deinit(); - try std.testing.expectEqualStrings("/usr/bin:/bin", environment.get("PATH").?); - try std.testing.expectEqualStrings("1", environment.get("GIT_CONFIG_NOSYSTEM").?); - try std.testing.expectEqualStrings("/dev/null", environment.get("GIT_CONFIG_GLOBAL").?); + try std.testing.expectEqual(expected.count(), environment.count()); + var entries = expected.iterator(); + while (entries.next()) |entry| { + try std.testing.expectEqualStrings(entry.value_ptr.*, environment.get(entry.key_ptr.*).?); + } try std.testing.expectEqualStrings("0", environment.get("GIT_OPTIONAL_LOCKS").?); - try std.testing.expectEqualStrings("0", environment.get("GIT_TERMINAL_PROMPT").?); - try std.testing.expectEqualStrings("cat", environment.get("GIT_PAGER").?); - try std.testing.expect(environment.get("HOME") == null); + try std.testing.expect(environment.get("GIT_CONFIG_PARAMETERS") == null); } test "direct executor runs a supported pipeline and reports final output" { diff --git a/src/core/shell_command/command_effect.zig b/src/core/shell_command/command_effect.zig index 9d00f7949..dc7c18615 100644 --- a/src/core/shell_command/command_effect.zig +++ b/src/core/shell_command/command_effect.zig @@ -1,5 +1,6 @@ const std = @import("std"); const command_lex = @import("command_lex.zig"); +const safe_git = @import("safe_git.zig"); const max_command_bytes = 8 * 1024; const max_ls_operands = 64; @@ -258,6 +259,8 @@ pub const ApprovalReason = enum { command_owned_input, unknown_command, planning_failure, + /// fx could not apply its git safety settings in this directory. + unsafe_git, }; pub const EnvironmentProfile = enum { @@ -307,6 +310,41 @@ pub fn plan( resolved_cwd: []const u8, background: bool, target_os: std.Target.Os.Tag, +) std.mem.Allocator.Error!Admission { + return planWithGit(alloc, command, resolved_cwd, background, target_os, .prepare); +} + +/// How git stages learn whether safe git can run: prepared in the command's +/// directory, or fixed by a test. +const GitSource = union(enum) { + prepare, + fixed: safe_git.Prepared, +}; + +const GitResolver = struct { + source: GitSource, + cwd: []const u8, + prepared: ?safe_git.Prepared = null, + + /// Prepares at most once per plan, and only for a supported git stage. + fn get(self: *GitResolver, arena: std.mem.Allocator) std.mem.Allocator.Error!safe_git.Prepared { + if (self.prepared) |prepared| return prepared; + const prepared = switch (self.source) { + .prepare => try safe_git.prepare(arena, self.cwd), + .fixed => |fixed| fixed, + }; + self.prepared = prepared; + return prepared; + } +}; + +fn planWithGit( + alloc: std.mem.Allocator, + command: []const u8, + resolved_cwd: []const u8, + background: bool, + target_os: std.Target.Os.Tag, + git_source: GitSource, ) std.mem.Allocator.Error!Admission { if (background) return .{ .approval_required = .background_process }; if (target_os != .macos and target_os != .linux) { @@ -333,6 +371,7 @@ pub fn plan( return .{ .approval_required = .process_or_system }; } + var git: GitResolver = .{ .source = git_source, .cwd = resolved_cwd }; var planned: std.ArrayList(TemporaryStage) = .empty; for (segments) |segment| { if (segment.text.len == 0) return .{ .approval_required = .unsupported_shell }; @@ -341,7 +380,7 @@ pub fn plan( error.OutOfMemory => return error.OutOfMemory, else => return .{ .approval_required = .unsupported_shell }, }; - const parsed = try parseStage(scratch, argv.tokens, target_os); + const parsed = try parseStage(scratch, argv.tokens, target_os, &git); switch (parsed) { .approval_required => |reason| return .{ .approval_required = reason }, .direct => |stage| try planned.append(scratch, stage), @@ -422,6 +461,7 @@ fn parseStage( alloc: std.mem.Allocator, tokens: []const command_lex.ArgvToken, target_os: std.Target.Os.Tag, + git: *GitResolver, ) std.mem.Allocator.Error!StageAdmission { var words: std.ArrayList([]const u8) = .empty; var index: usize = 0; @@ -440,7 +480,7 @@ fn parseStage( return .{ .approval_required = .dynamic_shell }; } - const family = try planFamily(alloc, words.items, target_os); + const family = try planFamily(alloc, words.items, target_os, git); return switch (family) { .approval_required => |reason| .{ .approval_required = reason }, .direct => |stage| .{ .direct = .{ @@ -465,6 +505,7 @@ fn planFamily( alloc: std.mem.Allocator, words: []const []const u8, target_os: std.Target.Os.Tag, + git: *GitResolver, ) std.mem.Allocator.Error!StageAdmission { const command = words[0]; if (isFilesystemMutation(command)) return .{ .approval_required = .filesystem_write }; @@ -478,7 +519,7 @@ fn planFamily( if (std.mem.eql(u8, command, "head")) return planHeadOrTail(alloc, words, "/usr/bin/head"); if (std.mem.eql(u8, command, "tail")) return planHeadOrTail(alloc, words, "/usr/bin/tail"); if (std.mem.eql(u8, command, "grep")) return planGrep(alloc, words); - if (std.mem.eql(u8, command, "git")) return planGit(alloc, words); + if (std.mem.eql(u8, command, "git")) return planGit(alloc, words, git); return .{ .approval_required = .unknown_command }; } @@ -723,31 +764,32 @@ fn planGrep( fn planGit( alloc: std.mem.Allocator, words: []const []const u8, + git: *GitResolver, ) std.mem.Allocator.Error!StageAdmission { if (words.len < 2) return .{ .approval_required = .command_owned_input }; - if (std.mem.eql(u8, words[1], "status")) return planGitStatus(alloc, words[2..]); - if (std.mem.eql(u8, words[1], "diff")) return planGitDiff(alloc, words[2..]); - if (std.mem.eql(u8, words[1], "log")) return planGitLog(alloc, words[2..]); - return .{ .approval_required = .command_owned_input }; + const subcommand: safe_git.Subcommand = if (std.mem.eql(u8, words[1], "status")) + .status + else if (std.mem.eql(u8, words[1], "diff")) + .diff + else if (std.mem.eql(u8, words[1], "log")) + .log + else + return .{ .approval_required = .command_owned_input }; + const ready = switch (try git.get(alloc)) { + .ready => |ready| ready, + .refused => return .{ .approval_required = .unsafe_git }, + }; + return switch (subcommand) { + .status => planGitStatus(alloc, ready, words[2..]), + .diff => planGitDiff(alloc, ready, words[2..]), + .log => planGitLog(alloc, ready, words[2..]), + .branch, .rev_parse => unreachable, + }; } -fn appendGitPrelude( - alloc: std.mem.Allocator, - argv: *std.ArrayList([]const u8), -) std.mem.Allocator.Error!void { - try argv.appendSlice(alloc, &.{ - "/usr/bin/git", - "--no-pager", - "-c", - "core.hooksPath=/dev/null", - "-c", - "core.fsmonitor=false", - }); -} - -fn directGit(argv: []const []const u8) StageAdmission { +fn directGit(ready: safe_git.Ready, argv: []const []const u8) StageAdmission { return .{ .direct = .{ - .executable = "/usr/bin/git", + .executable = ready.executable, .argv = argv, .environment_profile = .git_read_only, } }; @@ -755,11 +797,11 @@ fn directGit(argv: []const []const u8) StageAdmission { fn planGitStatus( alloc: std.mem.Allocator, + ready: safe_git.Ready, arguments: []const []const u8, ) std.mem.Allocator.Error!StageAdmission { var argv: std.ArrayList([]const u8) = .empty; - try appendGitPrelude(alloc, &argv); - try argv.appendSlice(alloc, &.{ "status", "--ignore-submodules=all" }); + try safe_git.appendCommand(alloc, &argv, ready, .status); for (arguments) |argument| { const canonical = if (std.mem.eql(u8, argument, "-s")) "--short" @@ -777,16 +819,17 @@ fn planGitStatus( return .{ .approval_required = .unsupported_argument }; try argv.append(alloc, canonical); } - return directGit(try argv.toOwnedSlice(alloc)); + return directGit(ready, try argv.toOwnedSlice(alloc)); } fn planGitDiff( alloc: std.mem.Allocator, + ready: safe_git.Ready, arguments: []const []const u8, ) std.mem.Allocator.Error!StageAdmission { var argv: std.ArrayList([]const u8) = .empty; - try appendGitPrelude(alloc, &argv); - try argv.appendSlice(alloc, &.{ "diff", "--no-ext-diff", "--no-textconv", "--color=never" }); + try safe_git.appendCommand(alloc, &argv, ready, .diff); + try argv.append(alloc, "--color=never"); var index: usize = 0; while (index < arguments.len and !std.mem.eql(u8, arguments[index], "--")) : (index += 1) { @@ -811,16 +854,17 @@ fn planGitDiff( } try argv.append(alloc, "--"); try argv.appendSlice(alloc, operands); - return directGit(try argv.toOwnedSlice(alloc)); + return directGit(ready, try argv.toOwnedSlice(alloc)); } fn planGitLog( alloc: std.mem.Allocator, + ready: safe_git.Ready, arguments: []const []const u8, ) std.mem.Allocator.Error!StageAdmission { var argv: std.ArrayList([]const u8) = .empty; - try appendGitPrelude(alloc, &argv); - try argv.appendSlice(alloc, &.{ "log", "--no-ext-diff", "--no-textconv", "--color=never", "--max-count=100" }); + try safe_git.appendCommand(alloc, &argv, ready, .log); + try argv.appendSlice(alloc, &.{ "--color=never", "--max-count=100" }); var index: usize = 0; while (index < arguments.len and !std.mem.eql(u8, arguments[index], "--")) : (index += 1) { @@ -860,7 +904,7 @@ fn planGitLog( } try argv.append(alloc, "--"); try argv.appendSlice(alloc, operands); - return directGit(try argv.toOwnedSlice(alloc)); + return directGit(ready, try argv.toOwnedSlice(alloc)); } fn boundedDecimal(value: []const u8, maximum: usize) bool { @@ -978,13 +1022,18 @@ test "planner canonicalizes pwd to a fixed physical path" { } } +/// Planner tests use a fixed safe-git result, so they do not depend on the +/// host's git. `safe_git.zig` tests the real preparation. +const test_git: GitSource = .{ .fixed = .{ .ready = .{ .executable = "/usr/bin/git", .filter_names = &.{} } } }; + fn expectDirect(command: []const u8, target_os: std.Target.Os.Tag) !Admission { - const admission = try plan( + const admission = try planWithGit( std.testing.allocator, command, "/workspace", false, target_os, + test_git, ); if (admission == .approval_required) { std.debug.print( @@ -1004,12 +1053,23 @@ fn expectApproval( target_os: std.Target.Os.Tag, expected: ApprovalReason, ) !void { - var admission = try plan( + return expectApprovalWithGit(command, background, target_os, expected, test_git); +} + +fn expectApprovalWithGit( + command: []const u8, + background: bool, + target_os: std.Target.Os.Tag, + expected: ApprovalReason, + git_source: GitSource, +) !void { + var admission = try planWithGit( std.testing.allocator, command, "/workspace", background, target_os, + git_source, ); defer admission.deinit(std.testing.allocator); @@ -1304,13 +1364,17 @@ test "planner pins read-only git inspection to hardened argv and environment" { const expected = [_][]const u8{ "/usr/bin/git", "--no-pager", + "--no-optional-locks", + "-c", + "core.fsmonitor=false", "-c", "core.hooksPath=/dev/null", "-c", - "core.fsmonitor=false", + "log.showSignature=false", "diff", "--no-ext-diff", "--no-textconv", + "--ignore-submodules=all", "--color=never", "--name-only", "--", @@ -1319,6 +1383,29 @@ test "planner pins read-only git inspection to hardened argv and environment" { try expectArgv(&expected, stage.argv); } +test "planner blanks repository filters and falls back to review when safe git refuses" { + const with_filter: GitSource = .{ .fixed = .{ .ready = .{ .executable = "/usr/bin/git", .filter_names = &.{"evil"} } } }; + var admission = try planWithGit(std.testing.allocator, "git status --short", "/workspace", false, .linux, with_filter); + defer admission.deinit(std.testing.allocator); + const argv = admission.direct_read_only.stages[0].argv; + for ([_][]const u8{ "filter.evil.clean=", "filter.evil.smudge=", "filter.evil.process=" }) |override| { + for (argv, 0..) |arg, index| { + if (std.mem.eql(u8, arg, override)) { + try std.testing.expectEqualStrings("-c", argv[index - 1]); + break; + } + } else return error.TestExpectedFilterOverride; + } + + const refused: GitSource = .{ .fixed = .{ .refused = .repository_mismatch } }; + for ([_][]const u8{ "git status", "git diff --stat", "git log --oneline -n 5", "git status --short | wc -l" }) |command| { + try expectApprovalWithGit(command, false, .linux, .unsafe_git, refused); + } + // Subcommands that are never auto-run keep their own reason. + try expectApprovalWithGit("git push origin HEAD", false, .linux, .command_owned_input, refused); + try expectApprovalWithGit("git status --ignored", false, .linux, .unsupported_argument, test_git); +} + test "planner limits bounded readers to pipeline input" { const cases = [_]struct { command: []const u8, diff --git a/src/core/shell_command/safe_git.zig b/src/core/shell_command/safe_git.zig new file mode 100644 index 000000000..9a007f8cf --- /dev/null +++ b/src/core/shell_command/safe_git.zig @@ -0,0 +1,614 @@ +//! The one way fx runs git itself: the `fx pr` and `fx issue` snapshot and the +//! agent's auto-run `git status`, `git diff` and `git log`. A repository's own +//! config can name programs: fsmonitor, hooks, a pager, diff drivers, filter +//! drivers and a signature verifier. Every invocation turns those off, while +//! the user's global and system config stay trusted, so a global Git LFS still +//! works. When fx cannot establish that, `prepare` refuses and git does not run. + +const std = @import("std"); +const builtin = @import("builtin"); +const io_mod = @import("../shared/io.zig"); +const debug_trace = @import("../shared/debug_trace.zig"); +const indexer = @import("../indexer/indexer.zig"); + +const Allocator = std.mem.Allocator; + +/// `core.fsmonitor` became a boolean in git 2.36. Older git reads the value +/// as a hook program. +const minimum_major: u32 = 2; +const minimum_minor: u32 = 36; + +const max_probe_output: usize = 4096; + +pub const Refusal = enum { + git_missing, + git_version_unknown, + git_too_old, + repo_config_unavailable, + /// `-c` splits at the first `=`, so such a driver cannot be blanked. + unsupported_filter_name, + /// git would use a different repository than the one whose config fx read. + repository_mismatch, +}; + +pub const Ready = struct { + /// Static path from `trustedExecutable`. + executable: []const u8, + /// Filter drivers defined by repository-scope config. + filter_names: []const []const u8, +}; + +pub const Prepared = union(enum) { + ready: Ready, + refused: Refusal, +}; + +pub const Subcommand = enum { status, diff, log, branch, rev_parse }; + +/// Decides whether git may run in the absolute directory `cwd`: reads the +/// repository's config as data, checks the trusted git's version once per +/// process, and confirms git resolves the same repository fx read. All memory +/// belongs to `arena`. +pub fn prepare(arena: Allocator, cwd: []const u8) Allocator.Error!Prepared { + const filters = indexer.repoFilters(arena, cwd) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.RepoConfigUnavailable => return refuse(.repo_config_unavailable, cwd), + }; + for (filters.names) |name| { + if (name.len == 0 or std.mem.findScalar(u8, name, '=') != null) return refuse(.unsupported_filter_name, cwd); + } + const executable = trustedExecutable() orelse return refuse(.git_missing, cwd); + switch (try versionCheck(arena, executable)) { + .supported => {}, + .too_old => return refuse(.git_too_old, cwd), + .unknown => return refuse(.git_version_unknown, cwd), + } + const ready: Ready = .{ .executable = executable, .filter_names = filters.names }; + if (!try sameRepository(arena, ready, cwd, filters.git_dir)) return refuse(.repository_mismatch, cwd); + return .{ .ready = ready }; +} + +fn refuse(reason: Refusal, cwd: []const u8) Prepared { + debug_trace.logf("core", "safe git refused reason={s} cwd={s}", .{ @tagName(reason), cwd }); + return .{ .refused = reason }; +} + +/// Appends the trusted executable, the global options and `-c` overrides that +/// turn off repository programs, then `subcommand` with its own safety flags. +/// Callers append the remaining arguments. Formatted overrides are allocated +/// with `arena`; the other strings are static or borrowed from `ready`. +pub fn appendCommand( + arena: Allocator, + argv: *std.ArrayList([]const u8), + ready: Ready, + subcommand: Subcommand, +) Allocator.Error!void { + try appendPrelude(arena, argv, ready); + try argv.appendSlice(arena, switch (subcommand) { + .status => &.{ "status", "--ignore-submodules=all" }, + .diff => &.{ "diff", "--no-ext-diff", "--no-textconv", "--ignore-submodules=all" }, + .log => &.{ "log", "--no-ext-diff", "--no-textconv", "--ignore-submodules=all" }, + .branch => &.{"branch"}, + .rev_parse => &.{"rev-parse"}, + }); +} + +fn appendPrelude(arena: Allocator, argv: *std.ArrayList([]const u8), ready: Ready) Allocator.Error!void { + try argv.appendSlice(arena, &.{ + ready.executable, + "--no-pager", + "--no-optional-locks", + "-c", + "core.fsmonitor=false", + "-c", + "core.hooksPath=/dev/null", + "-c", + "log.showSignature=false", + }); + for (ready.filter_names) |name| { + for ([_][]const u8{ "clean", "smudge", "process" }) |program| { + try argv.append(arena, "-c"); + try argv.append(arena, try std.fmt.allocPrint(arena, "filter.{s}.{s}=", .{ name, program })); + } + } +} + +/// The child environment for every safe git call: a fixed PATH and locale, no +/// prompts, pagers or optional locks, and only the variables that locate the +/// user's own git config. Nothing else from fx's environment, such as +/// `GIT_DIR` or `GIT_CONFIG_PARAMETERS`, reaches git. +pub fn environment(alloc: Allocator) Allocator.Error!std.process.Environ.Map { + var map = std.process.Environ.Map.init(alloc); + errdefer map.deinit(); + try map.put("PATH", "/usr/bin:/bin"); + try map.put("LC_ALL", "C"); + try map.put("LANG", "C"); + try map.put("GIT_OPTIONAL_LOCKS", "0"); + try map.put("GIT_TERMINAL_PROMPT", "0"); + try map.put("GIT_PAGER", "cat"); + try map.put("PAGER", "cat"); + for ([_][]const u8{ "HOME", "XDG_CONFIG_HOME", "GIT_CONFIG_GLOBAL", "GIT_CONFIG_SYSTEM", "GIT_CONFIG_NOSYSTEM" }) |key| { + if (io_mod.getenv(key)) |value| try map.put(key, value); + } + return map; +} + +/// The first git found at a fixed system location. git from PATH is never +/// used, because PATH can include directories a repository controls. +pub fn trustedExecutable() ?[]const u8 { + const candidates = switch (builtin.os.tag) { + .windows => &[_][]const u8{ + "C:\\Program Files\\Git\\cmd\\git.exe", + "C:\\Program Files\\Git\\bin\\git.exe", + }, + else => &[_][]const u8{ + "/usr/bin/git", + "/bin/git", + "/usr/local/bin/git", + "/opt/homebrew/bin/git", + "/opt/local/bin/git", + "/run/current-system/sw/bin/git", + }, + }; + for (candidates) |candidate| { + const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), candidate, .{ .follow_symlinks = true }) catch continue; + if (stat.kind == .file) return candidate; + } + return null; +} + +const VersionCheck = enum { supported, too_old, unknown }; + +var version_lock: std.Io.Mutex = .init; +var cached_version: ?struct { executable: []const u8, check: VersionCheck } = null; + +/// Runs `git --version` once per executable and process. A failed probe is not +/// cached, so a transient spawn failure does not disable git for the session. +fn versionCheck(arena: Allocator, executable: []const u8) Allocator.Error!VersionCheck { + const io = io_mod.getIo(); + version_lock.lockUncancelable(io); + defer version_lock.unlock(io); + if (cached_version) |cached| { + if (std.mem.eql(u8, cached.executable, executable)) return cached.check; + } + const output = try runCaptured(arena, &.{ executable, "--version" }, "/") orelse return .unknown; + const check = classifyVersion(output); + cached_version = .{ .executable = executable, .check = check }; + return check; +} + +fn classifyVersion(output: []const u8) VersionCheck { + const prefix = "git version "; + if (!std.mem.startsWith(u8, output, prefix)) return .unknown; + var parts = std.mem.splitScalar(u8, output[prefix.len..], '.'); + const major = leadingNumber(parts.next() orelse "") orelse return .unknown; + const minor = leadingNumber(parts.next() orelse "") orelse return .unknown; + if (major != minimum_major) return if (major > minimum_major) .supported else .too_old; + return if (minor >= minimum_minor) .supported else .too_old; +} + +fn leadingNumber(text: []const u8) ?u32 { + var end: usize = 0; + while (end < text.len and std.ascii.isDigit(text[end])) end += 1; + if (end == 0) return null; + return std.fmt.parseUnsigned(u32, text[0..end], 10) catch null; +} + +/// Confirms git resolves the repository whose config fx read. git accepts a +/// git directory used as the workspace, and fx's discovery does not, so +/// without this check git could run with config fx never read. `rev-parse` +/// reads config but runs none of the programs above. +fn sameRepository(arena: Allocator, ready: Ready, cwd: []const u8, expected: ?[]const u8) Allocator.Error!bool { + var argv: std.ArrayList([]const u8) = .empty; + try appendCommand(arena, &argv, ready, .rev_parse); + try argv.append(arena, "--absolute-git-dir"); + const output = try runCaptured(arena, argv.items, cwd); + const expected_dir = expected orelse return output == null; + const reported = std.mem.trimEnd(u8, output orelse return false, "\r\n"); + const expected_real = io_mod.realpathAlloc(arena, expected_dir) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return false, + }; + const reported_real = io_mod.realpathAlloc(arena, reported) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return false, + }; + return std.mem.eql(u8, expected_real, reported_real); +} + +/// Standard output of a successful run, or null when git could not start or +/// failed. Memory belongs to `arena`. +fn runCaptured(arena: Allocator, argv: []const []const u8, cwd: []const u8) Allocator.Error!?[]const u8 { + var env = try environment(arena); + const result = std.process.run(arena, io_mod.getIo(), .{ + .argv = argv, + .cwd = .{ .path = cwd }, + .environ_map = &env, + .stdout_limit = .limited(max_probe_output), + .stderr_limit = .limited(max_probe_output), + }) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + else => return null, + }; + return switch (result.term) { + .exited => |code| if (code == 0) result.stdout else null, + else => null, + }; +} + +test "git version check requires 2.36 and reads vendor suffixes" { + const cases = [_]struct { output: []const u8, expected: VersionCheck }{ + .{ .output = "git version 2.50.1 (Apple Git-155)\n", .expected = .supported }, + .{ .output = "git version 2.36.0\n", .expected = .supported }, + .{ .output = "git version 2.39.3.windows.1\n", .expected = .supported }, + .{ .output = "git version 3.0.0\n", .expected = .supported }, + .{ .output = "git version 2.35.8\n", .expected = .too_old }, + .{ .output = "git version 1.9.5\n", .expected = .too_old }, + .{ .output = "hub version 2.14\n", .expected = .unknown }, + .{ .output = "git version x.y\n", .expected = .unknown }, + }; + for (cases) |case| try std.testing.expectEqual(case.expected, classifyVersion(case.output)); +} + +test "safe git argv turns off repository programs for every subcommand" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const ready: Ready = .{ .executable = "/usr/bin/git", .filter_names = &.{ "lfs", "evil.x" } }; + var argv: std.ArrayList([]const u8) = .empty; + try appendCommand(arena, &argv, ready, .diff); + const expected = [_][]const u8{ + "/usr/bin/git", "--no-pager", "--no-optional-locks", + "-c", "core.fsmonitor=false", "-c", + "core.hooksPath=/dev/null", "-c", "log.showSignature=false", + "-c", "filter.lfs.clean=", "-c", + "filter.lfs.smudge=", "-c", "filter.lfs.process=", + "-c", "filter.evil.x.clean=", "-c", + "filter.evil.x.smudge=", "-c", "filter.evil.x.process=", + "diff", "--no-ext-diff", "--no-textconv", + "--ignore-submodules=all", + }; + try std.testing.expectEqual(expected.len, argv.items.len); + for (expected, argv.items) |want, have| try std.testing.expectEqualStrings(want, have); +} + +test "safe git environment keeps only fixed values and user config locations" { + var env = try environment(std.testing.allocator); + defer env.deinit(); + try std.testing.expectEqualStrings("/usr/bin:/bin", env.get("PATH").?); + try std.testing.expectEqualStrings("0", env.get("GIT_OPTIONAL_LOCKS").?); + try std.testing.expectEqualStrings("0", env.get("GIT_TERMINAL_PROMPT").?); + try std.testing.expectEqualStrings("cat", env.get("GIT_PAGER").?); + for ([_][]const u8{ "GIT_DIR", "GIT_WORK_TREE", "GIT_CONFIG_PARAMETERS", "GIT_CONFIG_COUNT", "GIT_EXEC_PATH" }) |key| { + try std.testing.expect(env.get(key) == null); + } +} + +/// A real repository with a sentinel program that appends to a log whenever +/// git runs it. Setup and control runs use plain git; helper runs use the +/// helper's environment. Both are isolated from the developer's git config. +/// There is no pager route here: git never pages when its output is not a +/// terminal, and fx always pipes it, so `--no-pager` and `GIT_PAGER=cat` are +/// covered by the argv and environment tests above. +const SentinelRepo = struct { + arena: Allocator, + git: []const u8, + root: []const u8, + repo: []const u8, + home: []const u8, + log: []const u8, + sentinel: []const u8, + + const Mode = union(enum) { + plain, + /// The helper's environment, optionally reading this system config. + safe: ?[]const u8, + }; + + fn init(arena: Allocator, dir: std.Io.Dir) !SentinelRepo { + const git = trustedExecutable() orelse return error.SkipZigTest; + if (try versionCheck(arena, git) != .supported) return error.SkipZigTest; + const root = try io_mod.dirRealpathAlloc(arena, dir, "."); + const self: SentinelRepo = .{ + .arena = arena, + .git = git, + .root = root, + .repo = try std.fs.path.join(arena, &.{ root, "repo" }), + .home = try std.fs.path.join(arena, &.{ root, "home" }), + .log = try std.fs.path.join(arena, &.{ root, "sentinel.log" }), + .sentinel = try std.fs.path.join(arena, &.{ root, "sentinel.sh" }), + }; + try dir.createDirPath(std.testing.io, "home"); + try dir.createDirPath(std.testing.io, "repo"); + try self.writeExecutable(self.sentinel); + try self.plain(self.repo, &.{ "init", "-q", "--template=", "-b", "main" }); + try self.write("repo/a.txt", "one\n"); + try self.plain(self.repo, &.{ "add", "a.txt" }); + try self.commit(self.repo); + return self; + } + + fn path(self: SentinelRepo, sub_path: []const u8) ![]const u8 { + return std.fs.path.join(self.arena, &.{ self.root, sub_path }); + } + + fn writeExecutable(self: SentinelRepo, absolute: []const u8) !void { + const script = try std.fmt.allocPrint(self.arena, "#!/bin/sh\necho ran >> '{s}'\ncat\n", .{self.log}); + try std.Io.Dir.cwd().writeFile(std.testing.io, .{ .sub_path = absolute, .data = script, .flags = .{ .permissions = .fromMode(0o755) } }); + } + + fn write(self: SentinelRepo, sub_path: []const u8, data: []const u8) !void { + const absolute = try self.path(sub_path); + if (std.fs.path.dirname(absolute)) |parent| try std.Io.Dir.cwd().createDirPath(std.testing.io, parent); + try std.Io.Dir.cwd().writeFile(std.testing.io, .{ .sub_path = absolute, .data = data }); + } + + fn commit(self: SentinelRepo, cwd: []const u8) !void { + try self.plain(cwd, &.{ "-c", "user.name=fixture", "-c", "user.email=fixture@example.invalid", "commit", "-q", "-m", "fixture" }); + } + + fn childEnvironment(self: SentinelRepo, mode: Mode) !std.process.Environ.Map { + var env = switch (mode) { + .plain => blk: { + var plain_env = std.process.Environ.Map.init(self.arena); + try plain_env.put("PATH", "/usr/bin:/bin"); + try plain_env.put("LC_ALL", "C"); + break :blk plain_env; + }, + .safe => try environment(self.arena), + }; + try env.put("HOME", self.home); + try env.put("XDG_CONFIG_HOME", try std.fs.path.join(self.arena, &.{ self.home, ".config" })); + try env.put("GIT_CONFIG_GLOBAL", try std.fs.path.join(self.arena, &.{ self.home, ".gitconfig" })); + const system: ?[]const u8 = switch (mode) { + .plain => null, + .safe => |value| value, + }; + if (system) |config_path| { + try env.put("GIT_CONFIG_SYSTEM", config_path); + try env.put("GIT_CONFIG_NOSYSTEM", "0"); + } else try env.put("GIT_CONFIG_NOSYSTEM", "1"); + return env; + } + + fn run(self: SentinelRepo, cwd: []const u8, argv: []const []const u8, mode: Mode) !std.process.RunResult { + var env = try self.childEnvironment(mode); + return std.process.run(self.arena, std.testing.io, .{ + .argv = argv, + .cwd = .{ .path = cwd }, + .environ_map = &env, + .stdout_limit = .limited(1 << 20), + .stderr_limit = .limited(1 << 20), + }); + } + + fn plainArgv(self: SentinelRepo, args: []const []const u8) ![]const []const u8 { + var argv: std.ArrayList([]const u8) = .empty; + try argv.append(self.arena, self.git); + try argv.appendSlice(self.arena, args); + return argv.items; + } + + fn plainOutput(self: SentinelRepo, cwd: []const u8, args: []const []const u8) ![]const u8 { + const result = try self.run(cwd, try self.plainArgv(args), .plain); + switch (result.term) { + .exited => |code| if (code == 0) return std.mem.trimEnd(u8, result.stdout, "\n"), + else => {}, + } + std.debug.print("fixture git {s} failed: {s}\n", .{ args[0], result.stderr }); + return error.FixtureGitFailed; + } + + fn plain(self: SentinelRepo, cwd: []const u8, args: []const []const u8) !void { + _ = try self.plainOutput(cwd, args); + } + + fn runs(self: SentinelRepo) !usize { + var file = std.Io.Dir.openFileAbsolute(std.testing.io, self.log, .{}) catch |err| switch (err) { + error.FileNotFound => return 0, + else => return err, + }; + const data = try io_mod.readFileToEnd(self.arena, &file, 1 << 16); + file.close(std.testing.io); + try std.Io.Dir.deleteFileAbsolute(std.testing.io, self.log); + return std.mem.count(u8, data, "ran\n"); + } + + fn ready(self: SentinelRepo, cwd: []const u8) !Ready { + return switch (try prepare(self.arena, cwd)) { + .ready => |value| value, + .refused => |reason| { + std.debug.print("safe git refused: {s}\n", .{@tagName(reason)}); + return error.TestUnexpectedRefusal; + }, + }; + } + + fn helperArgv(self: SentinelRepo, cwd: []const u8, subcommand: Subcommand, args: []const []const u8) ![]const []const u8 { + var argv: std.ArrayList([]const u8) = .empty; + try appendCommand(self.arena, &argv, try self.ready(cwd), subcommand); + try argv.appendSlice(self.arena, args); + return argv.items; + } + + /// Runs `subcommand` through the helper, then `control` through plain + /// git. The helper must never run the sentinel; the control must, or the + /// fixture does not exercise the route. The helper runs first because it + /// never refreshes the index, which would hide the change from the control. + fn expectBlocked(self: SentinelRepo, cwd: []const u8, subcommand: Subcommand, args: []const []const u8, control: []const []const u8) !void { + _ = try self.run(cwd, try self.helperArgv(cwd, subcommand, args), .{ .safe = null }); + try std.testing.expectEqual(@as(usize, 0), try self.runs()); + _ = try self.run(cwd, try self.plainArgv(control), .plain); + try std.testing.expect(try self.runs() > 0); + } +}; + +test "safe git blocks a repository fsmonitor" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try SentinelRepo.init(arena_state.allocator(), tmp.dir); + try repo.plain(repo.repo, &.{ "config", "core.fsmonitor", repo.sentinel }); + try repo.write("repo/a.txt", "one\n"); + try repo.expectBlocked(repo.repo, .status, &.{"--short"}, &.{ "status", "--short" }); +} + +test "safe git blocks repository hooks" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try SentinelRepo.init(arena_state.allocator(), tmp.dir); + const hooks = try repo.path("hooks"); + try std.Io.Dir.cwd().createDirPath(std.testing.io, hooks); + try repo.writeExecutable(try repo.path("hooks/post-index-change")); + try repo.plain(repo.repo, &.{ "config", "core.hooksPath", hooks }); + try repo.write("repo/a.txt", "one\n"); + try repo.expectBlocked(repo.repo, .status, &.{"--short"}, &.{ "status", "--short" }); +} + +test "safe git blocks a repository external diff and textconv" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try SentinelRepo.init(arena_state.allocator(), tmp.dir); + try repo.plain(repo.repo, &.{ "config", "diff.external", repo.sentinel }); + try repo.write("repo/a.txt", "two\n"); + try repo.expectBlocked(repo.repo, .diff, &.{"--stat"}, &.{"diff"}); + + try repo.plain(repo.repo, &.{ "config", "--unset", "diff.external" }); + try repo.write("repo/.git/info/attributes", "a.txt diff=tc\n"); + try repo.plain(repo.repo, &.{ "config", "diff.tc.textconv", repo.sentinel }); + try repo.expectBlocked(repo.repo, .diff, &.{"--stat"}, &.{"diff"}); +} + +test "safe git blocks repository clean, process and smudge filters" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try SentinelRepo.init(arena_state.allocator(), tmp.dir); + try repo.write("repo/.git/info/attributes", "a.txt filter=evil\n"); + + try repo.plain(repo.repo, &.{ "config", "filter.evil.clean", repo.sentinel }); + try repo.write("repo/a.txt", "two\n"); + try repo.expectBlocked(repo.repo, .status, &.{"--short"}, &.{ "status", "--short" }); + try repo.write("repo/a.txt", "three\n"); + try repo.expectBlocked(repo.repo, .diff, &.{"--stat"}, &.{ "diff", "--stat" }); + try repo.plain(repo.repo, &.{ "config", "--unset", "filter.evil.clean" }); + + try repo.plain(repo.repo, &.{ "config", "filter.evil.process", repo.sentinel }); + try repo.write("repo/a.txt", "four\n"); + try repo.expectBlocked(repo.repo, .status, &.{"--short"}, &.{ "status", "--short" }); + try repo.plain(repo.repo, &.{ "config", "--unset", "filter.evil.process" }); + + // No read-only command checks files out, so smudge is shown through the + // shared prelude on `checkout`, where git would run it. + try repo.plain(repo.repo, &.{ "config", "filter.evil.smudge", repo.sentinel }); + const file = try repo.path("repo/a.txt"); + var argv: std.ArrayList([]const u8) = .empty; + try appendPrelude(repo.arena, &argv, try repo.ready(repo.repo)); + try argv.appendSlice(repo.arena, &.{ "checkout", "--", "a.txt" }); + try std.Io.Dir.deleteFileAbsolute(std.testing.io, file); + _ = try repo.run(repo.repo, argv.items, .{ .safe = null }); + try std.testing.expectEqual(@as(usize, 0), try repo.runs()); + try std.Io.Dir.deleteFileAbsolute(std.testing.io, file); + _ = try repo.run(repo.repo, try repo.plainArgv(&.{ "checkout", "--", "a.txt" }), .plain); + try std.testing.expect(try repo.runs() > 0); +} + +test "safe git blocks a repository signature program" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try SentinelRepo.init(arena_state.allocator(), tmp.dir); + const object = try std.fmt.allocPrint(repo.arena, + \\tree {s} + \\parent {s} + \\author fixture 1 +0000 + \\committer fixture 1 +0000 + \\gpgsig -----BEGIN PGP SIGNATURE----- + \\ + \\ AAAA + \\ -----END PGP SIGNATURE----- + \\ + \\signed + \\ + , .{ try repo.plainOutput(repo.repo, &.{ "rev-parse", "HEAD^{tree}" }), try repo.plainOutput(repo.repo, &.{ "rev-parse", "HEAD" }) }); + const object_path = try repo.path("signed-commit"); + try std.Io.Dir.cwd().writeFile(std.testing.io, .{ .sub_path = object_path, .data = object }); + const signed = try repo.plainOutput(repo.repo, &.{ "hash-object", "-t", "commit", "-w", object_path }); + try repo.plain(repo.repo, &.{ "update-ref", "refs/heads/main", signed }); + try repo.plain(repo.repo, &.{ "config", "log.showSignature", "true" }); + try repo.plain(repo.repo, &.{ "config", "gpg.program", repo.sentinel }); + try repo.expectBlocked(repo.repo, .log, &.{ "--oneline", "-1" }, &.{ "log", "-1" }); +} + +test "safe git does not enter submodules, whose config it never read" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try SentinelRepo.init(arena_state.allocator(), tmp.dir); + const sub = try repo.path("repo/sub"); + try std.Io.Dir.cwd().createDirPath(std.testing.io, sub); + try repo.plain(sub, &.{ "init", "-q", "--template=", "-b", "main" }); + try repo.write("repo/sub/b.txt", "one\n"); + try repo.plain(sub, &.{ "add", "b.txt" }); + try repo.commit(sub); + try repo.plain(repo.repo, &.{ "add", "sub" }); + try repo.commit(repo.repo); + try repo.write("repo/sub/.git/info/attributes", "b.txt filter=evil\n"); + try repo.plain(sub, &.{ "config", "filter.evil.clean", repo.sentinel }); + try repo.write("repo/sub/b.txt", "two\n"); + try repo.expectBlocked(repo.repo, .status, &.{"--short"}, &.{ "status", "--short" }); +} + +test "safe git keeps trusted system config filters" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try SentinelRepo.init(arena_state.allocator(), tmp.dir); + const system = try repo.path("system.gitconfig"); + try std.Io.Dir.cwd().writeFile(std.testing.io, .{ + .sub_path = system, + .data = try std.fmt.allocPrint(repo.arena, "[filter \"trusted\"]\n\tclean = {s}\n", .{repo.sentinel}), + }); + try repo.write("repo/.git/info/attributes", "a.txt filter=trusted\n"); + try repo.write("repo/a.txt", "two\n"); + _ = try repo.run(repo.repo, try repo.helperArgv(repo.repo, .status, &.{"--short"}), .{ .safe = system }); + try std.testing.expect(try repo.runs() > 0); +} + +test "safe git follows git's repository choice and refuses what it cannot blank" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try SentinelRepo.init(arena_state.allocator(), tmp.dir); + + // git skips an inner `.git` without `refs` and uses the outer repository. + // fx makes the same choice, so the outer repository's filter is blanked. + try repo.write("repo/.git/info/attributes", "a.txt filter=evil\n"); + try repo.plain(repo.repo, &.{ "config", "filter.evil.clean", repo.sentinel }); + try repo.write("repo/inner/.git/HEAD", "ref: refs/heads/main\n"); + try std.Io.Dir.cwd().createDirPath(std.testing.io, try repo.path("repo/inner/.git/objects")); + const inner = try repo.path("repo/inner"); + try repo.write("repo/a.txt", "two\n"); + try repo.expectBlocked(inner, .status, &.{"--short"}, &.{ "status", "--short" }); + + // A git directory used as the workspace, with a separate work tree. git + // uses it; fx's discovery does not, so safe git refuses. + const gitdir = try repo.path("gitdir"); + try repo.plain(repo.root, &.{ "init", "-q", "--bare", "--template=", gitdir }); + try repo.plain(gitdir, &.{ "config", "core.bare", "false" }); + try repo.plain(gitdir, &.{ "config", "core.worktree", repo.repo }); + try std.testing.expectEqual(Refusal.repository_mismatch, (try prepare(repo.arena, gitdir)).refused); + + try repo.plain(repo.repo, &.{ "config", "filter.a=b.clean", repo.sentinel }); + try std.testing.expectEqual(Refusal.unsupported_filter_name, (try prepare(repo.arena, repo.repo)).refused); +} diff --git a/src/core/workspace/workspace_files.zig b/src/core/workspace/workspace_files.zig index f6cd4c39b..87d97eedb 100644 --- a/src/core/workspace/workspace_files.zig +++ b/src/core/workspace/workspace_files.zig @@ -3,6 +3,7 @@ const builtin = @import("builtin"); const debug_trace = @import("../shared/debug_trace.zig"); const ignored_dirs = @import("ignored_dirs.zig"); const io_mod = @import("../shared/io.zig"); +const safe_git = @import("../shell_command/safe_git.zig"); const sort_utils = @import("../shared/sort_utils.zig"); const Allocator = std.mem.Allocator; @@ -257,27 +258,7 @@ fn runGitRawList( } } -fn trustedGitExecutable() ?[]const u8 { - const candidates = switch (builtin.os.tag) { - .windows => &[_][]const u8{ - "C:\\Program Files\\Git\\cmd\\git.exe", - "C:\\Program Files\\Git\\bin\\git.exe", - }, - else => &[_][]const u8{ - "/usr/bin/git", - "/bin/git", - "/usr/local/bin/git", - "/opt/homebrew/bin/git", - "/opt/local/bin/git", - "/run/current-system/sw/bin/git", - }, - }; - for (candidates) |candidate| { - const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), candidate, .{ .follow_symlinks = true }) catch continue; - if (stat.kind == .file) return candidate; - } - return null; -} +const trustedGitExecutable = safe_git.trustedExecutable; fn runCancellable( alloc: Allocator, diff --git a/src/main.zig b/src/main.zig index 8a176f6a7..85cdaa557 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4864,6 +4864,7 @@ test { _ = @import("core/shared/message.zig"); _ = @import("core/shared/token_estimate.zig"); _ = @import("core/shell_command/command_effect.zig"); + _ = @import("core/shell_command/safe_git.zig"); _ = @import("core/execution/router.zig"); _ = @import("core/execution/command_runner.zig"); _ = @import("core/permissions/direct_command.zig"); From d5cd8a2044b3a19abf81671119cc7457f9ec5715 Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 8 Oct 2026 14:20:09 -0400 Subject: [PATCH 03/23] List workspace files natively with git's rules - Read git's index file as data, versions 2 to 4 with SHA-1 or SHA-256 ids, and treat an index with a split or sparse extension as unavailable. - List a root's files and folders the way git shows them: tracked files, including committed files an ignore rule matches, untracked files no rule excludes, and git's folder rules. - Show submodules and nested repositories as folders without listing what is inside them. - Apply .gitignore files and the skipped folder names outside repositories and below a root that is itself ignored. - Read folders with a bounded pool of worker threads and return the same sorted list regardless of scheduling. - Record stamps for every folder and rule file the listing depended on. --- src/core/indexer/git_config.zig | 6 + src/core/indexer/git_index.zig | 284 +++++++++++ src/core/indexer/indexer.zig | 17 + src/core/indexer/layout.zig | 11 + src/core/indexer/scan.zig | 808 ++++++++++++++++++++++++++++++++ src/core/indexer/tree.zig | 164 +++++++ 6 files changed, 1290 insertions(+) create mode 100644 src/core/indexer/git_index.zig create mode 100644 src/core/indexer/scan.zig create mode 100644 src/core/indexer/tree.zig diff --git a/src/core/indexer/git_config.zig b/src/core/indexer/git_config.zig index df3844a12..764b9340e 100644 --- a/src/core/indexer/git_config.zig +++ b/src/core/indexer/git_config.zig @@ -245,6 +245,9 @@ pub const Loaded = struct { /// In file order, with included files spliced in at their directive. entries: []const LoadedEntry, unavailable: []const Unavailable, + /// Every config file read or looked for, missing ones included, so a + /// later change to any of them can be detected. + paths: []const []const u8 = &.{}, /// The last entry for the key, or null when unset. pub fn last(self: Loaded, section: []const u8, subsection: ?[]const u8, name: []const u8) ?*const Entry { @@ -311,12 +314,14 @@ const Loader = struct { mode: Mode, entries: std.ArrayList(LoadedEntry) = .empty, unavailable: std.ArrayList(Unavailable) = .empty, + paths: std.ArrayList([]const u8) = .empty, fn readFile(self: *Loader, path: []const u8, scope: Scope, depth: usize) Allocator.Error!void { if (depth > max_include_depth) return self.markUnavailable(path, "include_depth"); // git reads /dev/null as an empty file; it is commonly used to // disable a config level. if (std.mem.eql(u8, path, "/dev/null")) return; + try self.paths.append(self.arena, path); const content = switch (try bounded_read.readAbsolute(self.arena, path, max_config_file_bytes)) { .content => |bytes| bytes, .missing => return, @@ -418,6 +423,7 @@ const Loader = struct { return .{ .entries = try self.entries.toOwnedSlice(self.arena), .unavailable = try self.unavailable.toOwnedSlice(self.arena), + .paths = try self.paths.toOwnedSlice(self.arena), }; } }; diff --git a/src/core/indexer/git_index.zig b/src/core/indexer/git_index.zig new file mode 100644 index 000000000..acff32971 --- /dev/null +++ b/src/core/indexer/git_index.zig @@ -0,0 +1,284 @@ +//! git's index file (`.git/index`) read as data: which paths are tracked and +//! which are gitlinks (submodules). Versions 2 to 4 and SHA-1 or SHA-256 +//! object ids are supported. Every read is bounds-checked, so a truncated or +//! hostile file is reported as unreadable instead of being trusted. + +const std = @import("std"); +const bounded_read = @import("bounded_read.zig"); + +const Allocator = std.mem.Allocator; + +/// D7: the index is never read beyond this size. +pub const max_index_bytes: usize = 128 * 1024 * 1024; + +const mode_type_mask: u32 = 0o170000; +const mode_gitlink: u32 = 0o160000; + +pub const Entry = struct { + /// Relative to the worktree root, as stored in the index. + path: []const u8, + gitlink: bool, +}; + +pub const Index = struct { + /// In index order (sorted by path). Unmerged paths appear once. + entries: []const Entry, +}; + +pub const Outcome = union(enum) { + /// No index file: a repository with nothing staged yet. + missing, + index: Index, + /// Static string naming why the index cannot be used. + unavailable: []const u8, +}; + +pub const HashFormat = enum { sha1, sha256 }; + +/// Reads the index at the absolute `path`. Memory belongs to `arena`. +pub fn read(arena: Allocator, path: []const u8, hash: HashFormat) Allocator.Error!Outcome { + const data = switch (try bounded_read.readAbsolute(arena, path, max_index_bytes)) { + .content => |bytes| bytes, + .missing => return .missing, + .unavailable => |reason| return .{ .unavailable = reason }, + }; + return parse(arena, data, hash); +} + +const Reader = struct { + data: []const u8, + pos: usize = 0, + + fn take(self: *Reader, len: usize) ?[]const u8 { + if (len > self.data.len - self.pos) return null; + const bytes = self.data[self.pos .. self.pos + len]; + self.pos += len; + return bytes; + } + + fn int(self: *Reader, comptime T: type) ?T { + const bytes = self.take(@sizeOf(T)) orelse return null; + return std.mem.readInt(T, bytes[0..@sizeOf(T)], .big); + } + + /// git's decode_varint, used by v4 path compression. + fn varint(self: *Reader) ?usize { + var byte = (self.take(1) orelse return null)[0]; + var value: usize = byte & 0x7f; + while (byte & 0x80 != 0) { + value = std.math.add(usize, value, 1) catch return null; + if (value > std.math.maxInt(usize) >> 7) return null; + byte = (self.take(1) orelse return null)[0]; + value = (value << 7) | (byte & 0x7f); + } + return value; + } +}; + +pub fn parse(arena: Allocator, data: []const u8, hash: HashFormat) Allocator.Error!Outcome { + const hash_len: usize = switch (hash) { + .sha1 => 20, + .sha256 => 32, + }; + if (data.len < 12 + hash_len) return .{ .unavailable = "index_truncated" }; + // The trailing checksum is not verified; every field below is bounds-checked. + var reader: Reader = .{ .data = data[0 .. data.len - hash_len] }; + const signature = reader.take(4).?; + if (!std.mem.eql(u8, signature, "DIRC")) return .{ .unavailable = "index_signature" }; + const version = reader.int(u32).?; + if (version < 2 or version > 4) return .{ .unavailable = "index_version" }; + const count = reader.int(u32).?; + // The smallest v4 entry is 40 + hash + 2 + 1 + 1 bytes. + if (count > (reader.data.len - reader.pos) / (44 + hash_len)) return .{ .unavailable = "index_truncated" }; + + const entries = try arena.alloc(Entry, count); + var kept: usize = 0; + var previous: []const u8 = ""; + var path_buffer: std.ArrayList(u8) = .empty; + for (0..count) |_| { + const start = reader.pos; + _ = reader.take(24) orelse return .{ .unavailable = "index_truncated" }; + const mode = reader.int(u32) orelse return .{ .unavailable = "index_truncated" }; + _ = reader.take(12 + hash_len) orelse return .{ .unavailable = "index_truncated" }; + const flags = reader.int(u16) orelse return .{ .unavailable = "index_truncated" }; + if (flags & 0x4000 != 0) { + if (version < 3) return .{ .unavailable = "index_extended_flags" }; + _ = reader.int(u16) orelse return .{ .unavailable = "index_truncated" }; + } + const stage = (flags >> 12) & 0x3; + + const path: []const u8 = if (version == 4) blk: { + const strip = reader.varint() orelse return .{ .unavailable = "index_prefix" }; + if (strip > previous.len) return .{ .unavailable = "index_prefix" }; + const rest = reader.data[reader.pos..]; + const end = std.mem.findScalar(u8, rest, 0) orelse return .{ .unavailable = "index_truncated" }; + path_buffer.clearRetainingCapacity(); + try path_buffer.appendSlice(arena, previous[0 .. previous.len - strip]); + try path_buffer.appendSlice(arena, rest[0..end]); + reader.pos += end + 1; + break :blk try arena.dupe(u8, path_buffer.items); + } else blk: { + const rest = reader.data[reader.pos..]; + const end = std.mem.findScalar(u8, rest, 0) orelse return .{ .unavailable = "index_truncated" }; + const name = rest[0..end]; + // Entries are padded with 1 to 8 NUL bytes to a multiple of 8. + const fixed = reader.pos - start; + const size = (fixed + name.len + 8) & ~@as(usize, 7); + if (size > reader.data.len - start) return .{ .unavailable = "index_truncated" }; + reader.pos = start + size; + break :blk name; + }; + previous = path; + if (path.len == 0) return .{ .unavailable = "index_empty_path" }; + // Unmerged paths carry stages 1 to 3 in sequence; keep the first. + if (stage != 0 and kept > 0 and std.mem.eql(u8, entries[kept - 1].path, path)) continue; + entries[kept] = .{ .path = path, .gitlink = mode & mode_type_mask == mode_gitlink }; + kept += 1; + } + + // Extensions follow the entries. One starting with an upper-case letter is + // optional; any other is required and changes how entries must be read. + while (reader.pos < reader.data.len) { + const name = reader.take(4) orelse return .{ .unavailable = "index_truncated" }; + const size = reader.int(u32) orelse return .{ .unavailable = "index_truncated" }; + if (!std.ascii.isUpper(name[0])) { + if (std.mem.eql(u8, name, "link")) return .{ .unavailable = "index_split" }; + if (std.mem.eql(u8, name, "sdir")) return .{ .unavailable = "index_sparse" }; + return .{ .unavailable = "index_required_extension" }; + } + _ = reader.take(size) orelse return .{ .unavailable = "index_truncated" }; + } + return .{ .index = .{ .entries = entries[0..kept] } }; +} + +pub const TestEntry = struct { path: []const u8, mode: u32 = 0o100644, stage: u2 = 0 }; + +/// Builds an index the way git lays it out. Test support only. +pub fn buildForTest(alloc: Allocator, version: u32, hash: HashFormat, entries: []const TestEntry, extensions: []const u8) ![]u8 { + const hash_len: usize = if (hash == .sha1) 20 else 32; + var out: std.ArrayList(u8) = .empty; + try out.appendSlice(alloc, "DIRC"); + try appendInt(alloc, &out, u32, version); + try appendInt(alloc, &out, u32, @intCast(entries.len)); + var previous: []const u8 = ""; + for (entries) |entry| { + const start = out.items.len; + try out.appendNTimes(alloc, 0, 24); + try appendInt(alloc, &out, u32, entry.mode); + try out.appendNTimes(alloc, 0, 12 + hash_len); + const name_len: u16 = @intCast(@min(entry.path.len, 0xfff)); + try appendInt(alloc, &out, u16, (@as(u16, entry.stage) << 12) | name_len); + if (version == 4) { + var common: usize = 0; + while (common < previous.len and common < entry.path.len and previous[common] == entry.path[common]) common += 1; + try appendVarint(alloc, &out, previous.len - common); + try out.appendSlice(alloc, entry.path[common..]); + try out.append(alloc, 0); + } else { + try out.appendSlice(alloc, entry.path); + const fixed = out.items.len - start - entry.path.len; + const size = (fixed + entry.path.len + 8) & ~@as(usize, 7); + try out.appendNTimes(alloc, 0, size - (out.items.len - start)); + } + previous = entry.path; + } + try out.appendSlice(alloc, extensions); + try out.appendNTimes(alloc, 0xab, hash_len); + return out.toOwnedSlice(alloc); +} + +fn appendInt(alloc: Allocator, out: *std.ArrayList(u8), comptime T: type, value: T) !void { + var bytes: [@sizeOf(T)]u8 = undefined; + std.mem.writeInt(T, &bytes, value, .big); + try out.appendSlice(alloc, &bytes); +} + +/// git's encode_varint. +fn appendVarint(alloc: Allocator, out: *std.ArrayList(u8), value: usize) !void { + var buffer: [16]u8 = undefined; + var pos: usize = buffer.len - 1; + var remaining = value; + buffer[pos] = @intCast(remaining & 0x7f); + while (true) { + remaining >>= 7; + if (remaining == 0) break; + remaining -= 1; + pos -= 1; + buffer[pos] = @intCast(0x80 | (remaining & 0x7f)); + } + try out.appendSlice(alloc, buffer[pos..]); +} + +fn expectPaths(outcome: Outcome, expected: []const []const u8, gitlinks: []const []const u8) !void { + const index = switch (outcome) { + .index => |value| value, + else => return error.TestExpectedIndex, + }; + try std.testing.expectEqual(expected.len, index.entries.len); + for (expected, index.entries) |want, entry| { + try std.testing.expectEqualStrings(want, entry.path); + var is_link = false; + for (gitlinks) |link| is_link = is_link or std.mem.eql(u8, link, want); + try std.testing.expectEqual(is_link, entry.gitlink); + } +} + +test "index reader handles versions 2 to 4, both hash formats, gitlinks and unmerged paths" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const entries = [_]TestEntry{ + .{ .path = "a.txt" }, + .{ .path = "conflict", .stage = 1 }, + .{ .path = "conflict", .stage = 2 }, + .{ .path = "conflict", .stage = 3 }, + .{ .path = "dir/deep/file.zig", .mode = 0o100755 }, + .{ .path = "dir/deep/link", .mode = 0o120000 }, + .{ .path = "mod", .mode = 0o160000 }, + }; + const expected = [_][]const u8{ "a.txt", "conflict", "dir/deep/file.zig", "dir/deep/link", "mod" }; + for ([_]u32{ 2, 3, 4 }) |version| { + for ([_]HashFormat{ .sha1, .sha256 }) |hash| { + const data = try buildForTest(arena, version, hash, &entries, "TREE\x00\x00\x00\x02ab"); + try expectPaths(try parse(arena, data, hash), &expected, &.{"mod"}); + } + } +} + +test "index reader rejects truncated, hostile and unsupported indexes" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const entries = [_]TestEntry{ .{ .path = "a.txt" }, .{ .path = "b/c.txt" } }; + const good = try buildForTest(arena, 2, .sha1, &entries, ""); + try expectPaths(try parse(arena, good, .sha1), &.{ "a.txt", "b/c.txt" }, &.{}); + + // Every truncation point is reported, never read past. + for (0..good.len - 1) |len| { + try std.testing.expect(try parse(arena, good[0..len], .sha1) != .index); + } + var bad_signature = try arena.dupe(u8, good); + bad_signature[0] = 'X'; + try std.testing.expectEqualStrings("index_signature", (try parse(arena, bad_signature, .sha1)).unavailable); + var bad_version = try arena.dupe(u8, good); + std.mem.writeInt(u32, bad_version[4..8], 9, .big); + try std.testing.expectEqualStrings("index_version", (try parse(arena, bad_version, .sha1)).unavailable); + var huge_count = try arena.dupe(u8, good); + std.mem.writeInt(u32, huge_count[8..12], std.math.maxInt(u32), .big); + try std.testing.expectEqualStrings("index_truncated", (try parse(arena, huge_count, .sha1)).unavailable); + + const split = try buildForTest(arena, 2, .sha1, &entries, "link\x00\x00\x00\x00"); + try std.testing.expectEqualStrings("index_split", (try parse(arena, split, .sha1)).unavailable); + const sparse = try buildForTest(arena, 2, .sha1, &entries, "sdir\x00\x00\x00\x00"); + try std.testing.expectEqualStrings("index_sparse", (try parse(arena, sparse, .sha1)).unavailable); + const oversized_extension = try buildForTest(arena, 2, .sha1, &entries, "TREE\x7f\xff\xff\xff"); + try std.testing.expectEqualStrings("index_truncated", (try parse(arena, oversized_extension, .sha1)).unavailable); + + const v4 = try buildForTest(arena, 4, .sha1, &entries, ""); + var bad_prefix = try arena.dupe(u8, v4); + // The second entry's strip count, just before its stored suffix, would + // remove more than the previous path holds. + const suffix = std.mem.find(u8, bad_prefix, "b/c.txt").?; + bad_prefix[suffix - 1] = 0x7f; + try std.testing.expectEqualStrings("index_prefix", (try parse(arena, bad_prefix, .sha1)).unavailable); +} diff --git a/src/core/indexer/indexer.zig b/src/core/indexer/indexer.zig index b64d2b7fb..719ff3a52 100644 --- a/src/core/indexer/indexer.zig +++ b/src/core/indexer/indexer.zig @@ -9,9 +9,23 @@ const io_mod = @import("../shared/io.zig"); const debug_trace = @import("../shared/debug_trace.zig"); const git_config = @import("git_config.zig"); const layout = @import("layout.zig"); +const scan_mod = @import("scan.zig"); const Allocator = std.mem.Allocator; +pub const Kind = scan_mod.Kind; +pub const Entry = scan_mod.Entry; +pub const ScanOptions = scan_mod.Options; +pub const ScanResult = scan_mod.Result; +pub const ScanError = scan_mod.Error; +pub const default_skipped_names = scan_mod.default_skipped_names; + +/// Lists the files and folders under the absolute `root`: inside a git +/// repository exactly what git shows, elsewhere everything except `.gitignore` +/// matches and `options.skipped_names`. See `scan.zig` for the rules. `alloc` +/// must be safe to use from several threads. +pub const scan = scan_mod.scan; + pub const RepoFilters = struct { /// Absolute git directory of the repository containing the root, found /// the way git discovers one, or null outside a repository. @@ -78,6 +92,9 @@ test { _ = @import("wildmatch.zig"); _ = @import("ignore.zig"); _ = @import("sources.zig"); + _ = @import("git_index.zig"); + _ = @import("tree.zig"); + _ = scan_mod; _ = layout; _ = git_config; } diff --git a/src/core/indexer/layout.zig b/src/core/indexer/layout.zig index 540662f08..449af8af1 100644 --- a/src/core/indexer/layout.zig +++ b/src/core/indexer/layout.zig @@ -53,6 +53,17 @@ pub fn discover(arena: Allocator, start: []const u8) DiscoverError!?Layout { } } +/// Whether the absolute directory `dir` holds a repository of its own, as +/// git's is_nonbare_repository_dir decides: a `.git` git directory or a valid +/// `gitdir:` file. An invalid `.git` file does not count. +pub fn hasOwnRepository(arena: Allocator, dir: []const u8) Allocator.Error!bool { + const found = layoutAt(arena, dir) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.InvalidGitFile => return false, + }; + return found != null; +} + fn layoutAt(arena: Allocator, dir: []const u8) DiscoverError!?Layout { const dot_git = try std.fs.path.join(arena, &.{ dir, ".git" }); const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), dot_git, .{ .follow_symlinks = true }) catch return null; diff --git a/src/core/indexer/scan.zig b/src/core/indexer/scan.zig new file mode 100644 index 000000000..b31cb43a1 --- /dev/null +++ b/src/core/indexer/scan.zig @@ -0,0 +1,808 @@ +//! Lists the files and folders under a root. Inside a git repository the list +//! is what git shows: files from the index, untracked files no ignore rule +//! excludes, and git's folder rules (a folder with no tracked files whose +//! contents are all ignored is hidden; an ignored folder with tracked files is +//! shown). Outside a repository, and below a root that is itself ignored, +//! `.gitignore` files and a fixed set of folder names decide. Symlinks are +//! listed and never followed; submodules and nested repositories are listed +//! and never entered. Folders are read by a bounded pool of worker threads, +//! all joined before `scan` returns, and the output is sorted, so it does not +//! depend on scheduling. + +const std = @import("std"); +const io_mod = @import("../shared/io.zig"); +const debug_trace = @import("../shared/debug_trace.zig"); +const git_config = @import("git_config.zig"); +const git_index = @import("git_index.zig"); +const ignore = @import("ignore.zig"); +const layout_mod = @import("layout.zig"); +const sources = @import("sources.zig"); +const tree_mod = @import("tree.zig"); + +const Allocator = std.mem.Allocator; + +pub const default_skipped_names = [_][]const u8{ ".git", ".zig-cache", "zig-out", "node_modules", ".next", "dist", "build", "coverage" }; + +pub const Kind = tree_mod.Kind; +pub const Entry = tree_mod.Entry; + +pub const Options = struct { + /// D7: at most this many entries are returned. + candidate_cap: usize = 100_000, + max_path_bytes: usize = 2048, + /// Folder and file names skipped outside repositories and below an + /// ignored root (D4, U3). Never applied inside a repository. + skipped_names: []const []const u8 = &default_skipped_names, + max_workers: usize = 8, +}; + +pub const Result = struct { + arena: std.heap.ArenaAllocator, + tree: tree_mod.Tree, + + pub fn deinit(self: *Result) void { + self.arena.deinit(); + self.* = undefined; + } +}; + +pub const Error = error{ OutOfMemory, Canceled, RootUnavailable }; + +/// Workers stop reading folders past this many collected entries, so a huge +/// tree cannot exhaust memory. Below it, the cap keeps a sorted prefix. +const collect_factor: usize = 4; +/// Helper threads start only once this many folders are waiting. +const parallel_threshold: usize = 16; + +const Rules = struct { + repository: bool, + fold_case: bool, + /// Worktree-relative path of the scan root: empty, or `a/b`. + prefix: []const u8, + /// Worktree-relative paths of tracked files, of every folder above a + /// tracked path, and of gitlinks. + tracked_files: std.StringHashMapUnmanaged(void) = .empty, + tracked_dirs: std.StringHashMapUnmanaged(void) = .empty, + gitlinks: std.StringHashMapUnmanaged(void) = .empty, + skipped_names: []const []const u8, + max_path_bytes: usize, + /// Files the rules read or looked for, stamped once the walk ends. + sources: std.ArrayList(SourceRequest) = .empty, + + fn skipped(self: *const Rules, name: []const u8) bool { + for (self.skipped_names) |skip| if (std.mem.eql(u8, skip, name)) return true; + return false; + } +}; + +const SourceRequest = struct { path: []const u8, identity_only: bool }; + +const Job = struct { + /// Relative to the scan root. + rel: []const u8, + /// Path the ignore rules see: worktree-relative in a repository, + /// root-relative otherwise. + rule_path: []const u8, + lists: []const *const ignore.PatternList, + excluded: bool, + tracked: bool, +}; + +const ChildState = enum { boundary, entered }; + +const Child = struct { rel: []const u8, state: ChildState }; + +const Listing = struct { + rel: []const u8, + tracked: bool, + excluded: bool, + /// An untracked folder that turned out to be a nested repository. + boundary: bool = false, + has_entries: bool = false, + /// Holds content that is not ignored but was not listed (over the path + /// limit), which keeps the folder visible. + unlisted: bool = false, + stamp: ?tree_mod.FolderStamp = null, + /// Absolute path of the folder's `.gitignore`, when it has one. + ignore_file: ?[]const u8 = null, + files: []const []const u8 = &.{}, + children: []const Child = &.{}, +}; + +const Worker = struct { + arena: std.heap.ArenaAllocator, + listings: std.ArrayList(Listing) = .empty, + thread: ?std.Thread = null, +}; + +const Shared = struct { + alloc: Allocator, + root_dir: std.Io.Dir, + root_abs: []const u8, + rules: *const Rules, + stop: ?*std.atomic.Value(bool), + collect_limit: usize, + scan_started_ns: i64, + + lock: std.Io.Mutex = .init, + wake: std.Io.Condition = .init, + queue: std.ArrayList(Job) = .empty, + /// Queued plus running jobs. + outstanding: usize = 0, + failure: ?Error = null, + + collected: std.atomic.Value(usize) = .init(0), + incomplete: std.atomic.Value(bool) = .init(false), + limit_reached: std.atomic.Value(bool) = .init(false), + overlong: std.atomic.Value(usize) = .init(0), + + fn canceled(self: *Shared) bool { + return if (self.stop) |flag| flag.load(.acquire) else false; + } + + fn fail(self: *Shared, err: Error) void { + const io = io_mod.getIo(); + self.lock.lockUncancelable(io); + defer self.lock.unlock(io); + if (self.failure == null) self.failure = err; + self.wake.broadcast(io); + } + + /// Takes the next job, or null when the walk is finished or failed. + fn take(self: *Shared) ?Job { + const io = io_mod.getIo(); + self.lock.lockUncancelable(io); + defer self.lock.unlock(io); + while (true) { + if (self.failure != null) return null; + if (self.queue.pop()) |job| return job; + if (self.outstanding == 0) return null; + self.wake.waitUncancelable(io, &self.lock); + } + } + + fn finish(self: *Shared, children: []const Job) Allocator.Error!void { + const io = io_mod.getIo(); + self.lock.lockUncancelable(io); + defer self.lock.unlock(io); + try self.queue.appendSlice(self.alloc, children); + self.outstanding = self.outstanding + children.len - 1; + self.wake.broadcast(io); + } + + fn queued(self: *Shared) usize { + const io = io_mod.getIo(); + self.lock.lockUncancelable(io); + defer self.lock.unlock(io); + return self.queue.items.len; + } +}; + +/// Lists `root` (absolute). `alloc` must be safe to use from several threads; +/// the result owns its memory. +pub fn scan(alloc: Allocator, root: []const u8, options: Options, stop: ?*std.atomic.Value(bool)) Error!Result { + // Taken before anything is read: a stamp is reusable only if its times + // are older than this by the granularity margin. + const scan_started_ns = tree_mod.nowNs(); + var result_arena = std.heap.ArenaAllocator.init(alloc); + errdefer result_arena.deinit(); + const arena = result_arena.allocator(); + if (!std.fs.path.isAbsolute(root)) return error.RootUnavailable; + const io = io_mod.getIo(); + var root_dir = std.Io.Dir.openDirAbsolute(io, root, .{ .iterate = true }) catch return error.RootUnavailable; + defer root_dir.close(io); + const root_stat = root_dir.stat(io) catch return error.RootUnavailable; + + var rules: Rules = .{ + .repository = false, + .fold_case = false, + .prefix = "", + .skipped_names = options.skipped_names, + .max_path_bytes = options.max_path_bytes, + }; + var incomplete = false; + var root_lists: []const *const ignore.PatternList = &.{}; + if (try repositoryRules(arena, root, &rules, &incomplete)) |lists| root_lists = lists; + + var shared: Shared = .{ + .alloc = alloc, + .root_dir = root_dir, + .root_abs = root, + .rules = &rules, + .stop = stop, + .collect_limit = std.math.mul(usize, options.candidate_cap, collect_factor) catch std.math.maxInt(usize), + .scan_started_ns = scan_started_ns, + }; + defer shared.queue.deinit(alloc); + try shared.queue.append(alloc, .{ + .rel = "", + .rule_path = rules.prefix, + .lists = root_lists, + .excluded = false, + .tracked = true, + }); + shared.outstanding = 1; + + const worker_count = @max(1, @min(options.max_workers, std.Thread.getCpuCount() catch 1)); + const workers = try alloc.alloc(Worker, worker_count); + defer alloc.free(workers); + for (workers) |*worker| worker.* = .{ .arena = std.heap.ArenaAllocator.init(alloc) }; + defer for (workers) |*worker| worker.arena.deinit(); + + runMain(&shared, workers); + if (shared.failure) |err| return err; + if (shared.canceled()) return error.Canceled; + + var listing_count: usize = 0; + for (workers) |*worker| listing_count += worker.listings.items.len; + const listings = try arena.alloc(*const Listing, listing_count); + var cursor: usize = 0; + for (workers) |*worker| { + for (worker.listings.items) |*listing| { + listings[cursor] = listing; + cursor += 1; + } + } + + var entries: std.ArrayList(Entry) = .empty; + try assemble(arena, listings, rules.repository, &entries); + std.mem.sortUnstable(Entry, entries.items, {}, entryLessThan); + var cap_reached = false; + if (entries.items.len > options.candidate_cap) { + entries.shrinkRetainingCapacity(options.candidate_cap); + cap_reached = true; + } + if (shared.limit_reached.load(.acquire)) cap_reached = true; + incomplete = incomplete or cap_reached or shared.incomplete.load(.acquire); + + var folders: std.ArrayList(tree_mod.FolderStamp) = .empty; + for (listings) |listing| { + const stamp = listing.stamp orelse continue; + try folders.append(arena, .{ .rel = try arena.dupe(u8, stamp.rel), .inode = stamp.inode, .mtime_ns = stamp.mtime_ns, .ctime_ns = stamp.ctime_ns, .reusable = stamp.reusable }); + if (listing.ignore_file) |path| try rules.sources.append(arena, .{ .path = try arena.dupe(u8, path), .identity_only = false }); + } + const stamps = try arena.alloc(tree_mod.SourceStamp, rules.sources.items.len); + for (rules.sources.items, stamps) |request, *stamp| stamp.* = tree_mod.sourceStamp(request.path, request.identity_only, scan_started_ns); + const skipped_names = try arena.alloc([]const u8, options.skipped_names.len); + for (options.skipped_names, skipped_names) |name, *copy| copy.* = try arena.dupe(u8, name); + + return .{ + .arena = result_arena, + .tree = .{ + .root = try arena.dupe(u8, root), + .root_inode = @intCast(root_stat.inode), + .scan_started_ns = scan_started_ns, + .repository = rules.repository, + .incomplete = incomplete, + .cap_reached = cap_reached, + .skipped_overlong = shared.overlong.load(.acquire), + .skipped_names = skipped_names, + .entries = entries.items, + .folders = folders.items, + .sources = stamps, + }, + }; +} + +/// Fills in repository rules when `root` is inside a repository and is not +/// itself ignored, and returns the ignore lists that apply above the root. +fn repositoryRules(arena: Allocator, root: []const u8, rules: *Rules, incomplete: *bool) Allocator.Error!?[]const *const ignore.PatternList { + const maybe_found = layout_mod.discover(arena, root) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.InvalidGitFile => blk: { + debug_trace.logf("indexer", "scan uses plain rules reason=invalid_git_file root={s}", .{root}); + break :blk null; + }, + }; + // A `.git` appearing above the root changes which rules apply. Inside a + // repository only the folders up to its worktree root matter. + const top = if (maybe_found) |found| found.worktree_root else "/"; + var ancestor = root; + while (true) { + try rules.sources.append(arena, .{ .path = try std.fs.path.join(arena, &.{ ancestor, ".git" }), .identity_only = true }); + if (std.mem.eql(u8, ancestor, top)) break; + ancestor = std.fs.path.dirname(ancestor) orelse break; + } + const found = maybe_found orelse return null; + const context = try git_config.Context.fromEnvironment(arena, found); + const loaded = try git_config.loadAll(arena, &context); + const settings = try git_config.ignoreSettings(arena, loaded, &context); + if (loaded.unavailable.len > 0 or settings.invalid.len > 0) incomplete.* = true; + const wide = try sources.loadRepositoryWide(arena, found, settings); + if (wide.unavailable.len > 0) incomplete.* = true; + for (loaded.paths) |path| try rules.sources.append(arena, .{ .path = path, .identity_only = false }); + if (settings.excludes_file) |path| try rules.sources.append(arena, .{ .path = path, .identity_only = false }); + for ([_][]const u8{"info/exclude"}) |name| try rules.sources.append(arena, .{ .path = try std.fs.path.join(arena, &.{ found.common_dir, name }), .identity_only = false }); + for ([_][]const u8{ "index", "HEAD", "commondir" }) |name| try rules.sources.append(arena, .{ .path = try std.fs.path.join(arena, &.{ found.git_dir, name }), .identity_only = false }); + + var lists: std.ArrayList(*const ignore.PatternList) = .empty; + try lists.appendSlice(arena, wide.lists); + // `.gitignore` files above the root, from the worktree root down. The + // root's own file is read by its job. + if (found.prefix.len > 0) { + var ancestors: std.ArrayList([]const u8) = .empty; + try ancestors.append(arena, ""); + var start: usize = 0; + while (std.mem.findScalarPos(u8, found.prefix, start, '/')) |slash| : (start = slash + 1) { + try ancestors.append(arena, found.prefix[0..slash]); + } + for (ancestors.items) |rule_dir| { + const absolute = if (rule_dir.len == 0) found.worktree_root else try std.fs.path.join(arena, &.{ found.worktree_root, rule_dir }); + var dir = std.Io.Dir.openDirAbsolute(io_mod.getIo(), absolute, .{}) catch { + incomplete.* = true; + continue; + }; + defer dir.close(io_mod.getIo()); + const base = if (rule_dir.len == 0) "" else try std.fmt.allocPrint(arena, "{s}/", .{rule_dir}); + try rules.sources.append(arena, .{ .path = try std.fs.path.join(arena, &.{ absolute, ".gitignore" }), .identity_only = false }); + switch (try sources.loadDirectory(arena, dir, base)) { + .list => |list| try lists.append(arena, list), + .absent => {}, + .unavailable => incomplete.* = true, + } + } + // U3: below an ignored root, plain rules apply. + if (ignore.isPathExcluded(lists.items, found.prefix, true, settings.ignore_case)) { + debug_trace.logf("indexer", "scan uses plain rules reason=ignored_root root={s}", .{root}); + return null; + } + } + + rules.repository = true; + rules.fold_case = settings.ignore_case; + rules.prefix = found.prefix; + const index_path = try std.fs.path.join(arena, &.{ found.git_dir, "index" }); + const hash: git_index.HashFormat = if (settings.object_format == .sha256) .sha256 else .sha1; + switch (try git_index.read(arena, index_path, hash)) { + .missing => {}, + .unavailable => |reason| { + debug_trace.logf("indexer", "scan index unavailable reason={s} root={s}", .{ reason, root }); + incomplete.* = true; + }, + .index => |index| for (index.entries) |entry| { + if (entry.gitlink) { + try rules.gitlinks.put(arena, entry.path, {}); + } else { + try rules.tracked_files.put(arena, entry.path, {}); + } + var slash = std.mem.findScalarLast(u8, entry.path, '/'); + while (slash) |end| { + const parent = entry.path[0..end]; + const slot = try rules.tracked_dirs.getOrPut(arena, parent); + if (slot.found_existing) break; + slash = std.mem.findScalarLast(u8, parent, '/'); + } + }, + } + return lists.items; +} + +fn runMain(shared: *Shared, workers: []Worker) void { + var started = false; + while (shared.take()) |job| { + process(shared, &workers[0], job); + if (!started and workers.len > 1 and shared.queued() >= parallel_threshold) { + started = true; + for (workers[1..]) |*worker| { + worker.thread = std.Thread.spawn(.{}, runHelper, .{ shared, worker }) catch |err| blk: { + debug_trace.logf("indexer", "scan worker not started err={s}", .{@errorName(err)}); + break :blk null; + }; + } + } + } + for (workers[1..]) |*worker| if (worker.thread) |thread| thread.join(); +} + +fn runHelper(shared: *Shared, worker: *Worker) void { + while (shared.take()) |job| process(shared, worker, job); +} + +fn process(shared: *Shared, worker: *Worker, job: Job) void { + var children: std.ArrayList(Job) = .empty; + listDirectory(shared, worker, job, &children) catch |err| switch (err) { + error.OutOfMemory => return shared.fail(error.OutOfMemory), + error.Canceled => return shared.fail(error.Canceled), + }; + shared.finish(children.items) catch shared.fail(error.OutOfMemory); +} + +fn listDirectory(shared: *Shared, worker: *Worker, job: Job, children: *std.ArrayList(Job)) (Allocator.Error || error{Canceled})!void { + const arena = worker.arena.allocator(); + const io = io_mod.getIo(); + const rules = shared.rules; + var listing: Listing = .{ .rel = job.rel, .tracked = job.tracked, .excluded = job.excluded }; + defer worker.listings.append(arena, listing) catch shared.fail(error.OutOfMemory); + if (shared.canceled()) return error.Canceled; + if (shared.limit_reached.load(.acquire)) return; + + var dir = shared.root_dir.openDir(io, if (job.rel.len == 0) "." else job.rel, .{ .iterate = true, .follow_symlinks = false }) catch |err| { + debug_trace.logf("indexer", "scan folder unreadable err={s} bytes={d}", .{ @errorName(err), job.rel.len }); + shared.incomplete.store(true, .release); + return; + }; + defer dir.close(io); + if (dir.stat(io)) |stat| { + listing.stamp = tree_mod.folderStamp(job.rel, stat, shared.scan_started_ns); + } else |_| shared.incomplete.store(true, .release); + + var lists = job.lists; + // Below an excluded folder everything is excluded, as in git, which stops + // reading `.gitignore` files there. + if (!job.excluded) { + const base = if (job.rule_path.len == 0) "" else try std.fmt.allocPrint(arena, "{s}/", .{job.rule_path}); + const loaded = try sources.loadDirectory(arena, dir, base); + if (loaded != .absent) listing.ignore_file = try std.fs.path.join(arena, &.{ shared.root_abs, job.rel, ".gitignore" }); + switch (loaded) { + .list => |list| { + const extended = try arena.alloc(*const ignore.PatternList, lists.len + 1); + @memcpy(extended[0..lists.len], lists); + extended[lists.len] = list; + lists = extended; + }, + .absent => {}, + .unavailable => |reason| { + debug_trace.logf("indexer", "scan ignore file unavailable reason={s} bytes={d}", .{ reason, job.rel.len }); + shared.incomplete.store(true, .release); + }, + } + } + + var files: std.ArrayList([]const u8) = .empty; + var child_entries: std.ArrayList(Child) = .empty; + var saw_dot_git = false; + var iterator = dir.iterate(); + while (true) { + if (shared.canceled()) return error.Canceled; + const entry = iterator.next(io) catch |err| { + debug_trace.logf("indexer", "scan folder read stopped err={s} bytes={d}", .{ @errorName(err), job.rel.len }); + shared.incomplete.store(true, .release); + break; + } orelse break; + if (std.mem.eql(u8, entry.name, ".git")) { + saw_dot_git = true; + continue; + } + listing.has_entries = true; + if (!rules.repository and rules.skipped(entry.name)) continue; + + const rel = try joinPath(arena, job.rel, entry.name); + const rule_path = if (rules.repository) try joinPath(arena, job.rule_path, entry.name) else rel; + // A path over the limit is not listed, but it still exists and is not + // ignored, so its folder stays visible. + const overlong = rel.len > rules.max_path_bytes; + var kind = entry.kind; + if (kind == .unknown) { + kind = (dir.statFile(io, entry.name, .{ .follow_symlinks = false }) catch continue).kind; + } + switch (kind) { + .file, .sym_link => { + const tracked = rules.repository and rules.tracked_files.contains(rule_path); + if (!tracked and (job.excluded or excludedPath(lists, rule_path, false, rules.fold_case))) continue; + if (overlong) { + _ = shared.overlong.fetchAdd(1, .acq_rel); + listing.unlisted = true; + continue; + } + try files.append(arena, rel); + }, + .directory => { + const gitlink = rules.repository and rules.gitlinks.contains(rule_path); + const tracked = rules.repository and rules.tracked_dirs.contains(rule_path); + const excluded = !gitlink and (job.excluded or excludedPath(lists, rule_path, true, rules.fold_case)); + if (excluded and !tracked) continue; + if (overlong) { + _ = shared.overlong.fetchAdd(1, .acq_rel); + listing.unlisted = true; + continue; + } + if (gitlink) { + try child_entries.append(arena, .{ .rel = rel, .state = .boundary }); + continue; + } + try children.append(arena, .{ .rel = rel, .rule_path = rule_path, .lists = lists, .excluded = excluded, .tracked = tracked }); + try child_entries.append(arena, .{ .rel = rel, .state = .entered }); + }, + // FIFOs, sockets and devices are never listed or opened. + else => {}, + } + } + + // An untracked folder with its own repository is a boundary, as in git. + if (rules.repository and saw_dot_git and !job.tracked and job.rel.len > 0) { + const absolute = try std.fs.path.join(arena, &.{ shared.root_abs, job.rel }); + if (try layout_mod.hasOwnRepository(arena, absolute)) { + listing.boundary = true; + children.clearRetainingCapacity(); + return; + } + } + listing.files = files.items; + listing.children = child_entries.items; + const total = shared.collected.fetchAdd(files.items.len + child_entries.items.len, .acq_rel) + files.items.len + child_entries.items.len; + if (total > shared.collect_limit) { + shared.limit_reached.store(true, .release); + children.clearRetainingCapacity(); + } +} + +fn excludedPath(lists: []const *const ignore.PatternList, path: []const u8, is_dir: bool, fold_case: bool) bool { + return ignore.decide(lists, path, is_dir, fold_case) == .excluded; +} + +fn joinPath(arena: Allocator, parent: []const u8, name: []const u8) Allocator.Error![]const u8 { + if (parent.len == 0) return arena.dupe(u8, name); + return std.fmt.allocPrint(arena, "{s}/{s}", .{ parent, name }); +} + +/// Applies git's folder rules bottom-up and gathers the visible entries. +fn assemble(arena: Allocator, listings: []const *const Listing, repository: bool, out: *std.ArrayList(Entry)) Allocator.Error!void { + var by_path: std.StringHashMapUnmanaged(usize) = .empty; + try by_path.ensureTotalCapacity(arena, @intCast(listings.len)); + for (listings, 0..) |listing, index| by_path.putAssumeCapacity(listing.rel, index); + + // Deeper folders first, so each folder sees its children's visibility. + const order = try arena.alloc(usize, listings.len); + for (order, 0..) |*slot, index| slot.* = index; + std.mem.sortUnstable(usize, order, listings, struct { + fn deeper(items: []const *const Listing, a: usize, b: usize) bool { + return std.mem.count(u8, items[a].rel, "/") > std.mem.count(u8, items[b].rel, "/"); + } + }.deeper); + const visible = try arena.alloc(bool, listings.len); + for (order) |index| { + const listing = listings[index]; + visible[index] = if (!repository or listing.tracked or listing.boundary or !listing.has_entries) true else blk: { + if (listing.excluded) break :blk false; + if (listing.files.len > 0 or listing.unlisted) break :blk true; + for (listing.children) |child| { + if (child.state == .boundary) break :blk true; + // A folder that could not be read counts as visible, as in git. + const child_index = by_path.get(child.rel) orelse break :blk true; + if (visible[child_index]) break :blk true; + } + break :blk false; + }; + } + + for (listings, 0..) |listing, index| { + if (!visible[index]) continue; + for (listing.files) |path| try out.append(arena, .{ .path = try arena.dupe(u8, path), .kind = .file }); + for (listing.children) |child| { + const shown = switch (child.state) { + .boundary => true, + .entered => if (by_path.get(child.rel)) |child_index| visible[child_index] else true, + }; + if (shown) try out.append(arena, .{ .path = try arena.dupe(u8, child.rel), .kind = .directory }); + } + } +} + +fn entryLessThan(_: void, a: Entry, b: Entry) bool { + return switch (std.mem.order(u8, a.path, b.path)) { + .lt => true, + .gt => false, + .eq => @intFromEnum(a.kind) < @intFromEnum(b.kind), + }; +} + +const TestTree = struct { + tmp: std.testing.TmpDir, + root: []const u8, + + fn init(arena: Allocator) !TestTree { + var tmp = std.testing.tmpDir(.{}); + errdefer tmp.cleanup(); + return .{ .tmp = tmp, .root = try io_mod.dirRealpathAlloc(arena, tmp.dir, ".") }; + } + + fn file(self: *TestTree, path: []const u8, data: []const u8) !void { + if (std.fs.path.dirname(path)) |parent| try self.tmp.dir.createDirPath(std.testing.io, parent); + try self.tmp.dir.writeFile(std.testing.io, .{ .sub_path = path, .data = data }); + } + + fn dir(self: *TestTree, path: []const u8) !void { + try self.tmp.dir.createDirPath(std.testing.io, path); + } + + /// A repository whose index tracks `tracked`. + fn repo(self: *TestTree, arena: Allocator, name: []const u8, tracked: []const git_index.TestEntry) !void { + try self.file(try std.fmt.allocPrint(arena, "{s}/.git/HEAD", .{name}), "ref: refs/heads/main\n"); + try self.dir(try std.fmt.allocPrint(arena, "{s}/.git/objects", .{name})); + try self.dir(try std.fmt.allocPrint(arena, "{s}/.git/refs", .{name})); + try self.file(try std.fmt.allocPrint(arena, "{s}/.git/index", .{name}), try git_index.buildForTest(arena, 2, .sha1, tracked, "")); + } +}; + +fn expectEntries(result: Result, expected: []const Entry) !void { + errdefer for (result.tree.entries) |entry| std.debug.print(" got {s} {s}\n", .{ @tagName(entry.kind), entry.path }); + try std.testing.expectEqual(expected.len, result.tree.entries.len); + for (expected, result.tree.entries) |want, have| { + try std.testing.expectEqualStrings(want.path, have.path); + try std.testing.expectEqual(want.kind, have.kind); + } +} + +test "repository scan follows git: index, ignore rules, folder rules and boundaries" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tree = try TestTree.init(arena); + defer tree.tmp.cleanup(); + try tree.repo(arena, "repo", &.{ + .{ .path = "a.txt" }, + .{ .path = "c.log" }, + .{ .path = "deleted.txt" }, + .{ .path = "dist/keep.txt" }, + .{ .path = "mod", .mode = 0o160000 }, + }); + try tree.file("repo/.gitignore", "*.log\ndist/\n*.tmp\n"); + try tree.file("repo/a.txt", "a"); + try tree.file("repo/b.log", "b"); + try tree.file("repo/c.log", "c"); + try tree.file("repo/dist/keep.txt", "k"); + try tree.file("repo/dist/new.txt", "n"); + try tree.file("repo/dist/sub/deep.txt", "d"); + try tree.file("repo/mod/inner.txt", "i"); + try tree.file("repo/nested/.git/HEAD", "ref: refs/heads/main\n"); + try tree.dir("repo/nested/.git/objects"); + try tree.dir("repo/nested/.git/refs"); + try tree.file("repo/nested/file.txt", "f"); + try tree.file("repo/onlyignored/x.tmp", "x"); + try tree.dir("repo/onlyignored/deeper"); + try tree.file("repo/onlyignored/deeper/y.tmp", "y"); + try tree.dir("repo/empty"); + try tree.file("repo/.hidden/h.txt", "h"); + try tree.tmp.dir.symLink(std.testing.io, "a.txt", "repo/link", .{}); + try tree.tmp.dir.symLink(std.testing.io, "dist", "repo/dirlink", .{}); + + var result = try scan(std.testing.allocator, try std.fs.path.join(arena, &.{ tree.root, "repo" }), .{}, null); + defer result.deinit(); + try std.testing.expect(result.tree.repository); + try std.testing.expect(!result.tree.incomplete); + try expectEntries(result, &.{ + .{ .path = ".gitignore", .kind = .file }, + .{ .path = ".hidden", .kind = .directory }, + .{ .path = ".hidden/h.txt", .kind = .file }, + .{ .path = "a.txt", .kind = .file }, + .{ .path = "c.log", .kind = .file }, + .{ .path = "dirlink", .kind = .file }, + .{ .path = "dist", .kind = .directory }, + .{ .path = "dist/keep.txt", .kind = .file }, + .{ .path = "empty", .kind = .directory }, + .{ .path = "link", .kind = .file }, + .{ .path = "mod", .kind = .directory }, + .{ .path = "nested", .kind = .directory }, + }); + + // Scanning a subfolder applies the rules above it, with paths relative to it. + try tree.file("repo/.hidden/skip.log", "s"); + var sub = try scan(std.testing.allocator, try std.fs.path.join(arena, &.{ tree.root, "repo/.hidden" }), .{}, null); + defer sub.deinit(); + try std.testing.expect(sub.tree.repository); + try expectEntries(sub, &.{.{ .path = "h.txt", .kind = .file }}); + + // U3: a root that is itself ignored is listed with plain rules. + var ignored_root = try scan(std.testing.allocator, try std.fs.path.join(arena, &.{ tree.root, "repo/dist" }), .{}, null); + defer ignored_root.deinit(); + try std.testing.expect(!ignored_root.tree.repository); + try expectEntries(ignored_root, &.{ + .{ .path = "keep.txt", .kind = .file }, + .{ .path = "new.txt", .kind = .file }, + .{ .path = "sub", .kind = .directory }, + .{ .path = "sub/deep.txt", .kind = .file }, + }); +} + +test "plain scan applies .gitignore files and the skipped names, also below an ignored root" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tree = try TestTree.init(arena); + defer tree.tmp.cleanup(); + try tree.repo(arena, "repo", &.{.{ .path = "plain/tracked.txt" }}); + try tree.file("repo/.gitignore", "plain/\n"); + try tree.file("repo/plain/.gitignore", "*.log\n"); + try tree.file("repo/plain/tracked.txt", "t"); + try tree.file("repo/plain/a.log", "a"); + try tree.file("repo/plain/src/main.zig", "m"); + try tree.file("repo/plain/node_modules/pkg/index.js", "p"); + try tree.file("repo/plain/build/out.o", "o"); + try tree.file("repo/plain/sub/dist", "named like a skipped folder"); + + var result = try scan(std.testing.allocator, try std.fs.path.join(arena, &.{ tree.root, "repo/plain" }), .{}, null); + defer result.deinit(); + try std.testing.expect(!result.tree.repository); + try expectEntries(result, &.{ + .{ .path = ".gitignore", .kind = .file }, + .{ .path = "src", .kind = .directory }, + .{ .path = "src/main.zig", .kind = .file }, + .{ .path = "sub", .kind = .directory }, + .{ .path = "tracked.txt", .kind = .file }, + }); + + var custom = try scan(std.testing.allocator, try std.fs.path.join(arena, &.{ tree.root, "repo/plain" }), .{ .skipped_names = &.{"src"} }, null); + defer custom.deinit(); + try expectEntries(custom, &.{ + .{ .path = ".gitignore", .kind = .file }, + .{ .path = "build", .kind = .directory }, + .{ .path = "build/out.o", .kind = .file }, + .{ .path = "node_modules", .kind = .directory }, + .{ .path = "node_modules/pkg", .kind = .directory }, + .{ .path = "node_modules/pkg/index.js", .kind = .file }, + .{ .path = "sub", .kind = .directory }, + .{ .path = "sub/dist", .kind = .file }, + .{ .path = "tracked.txt", .kind = .file }, + }); +} + +test "scan caps, skips overlong paths, cancels and is the same with one worker or many" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tree = try TestTree.init(arena); + defer tree.tmp.cleanup(); + try tree.repo(arena, "repo", &.{}); + for (0..60) |i| { + for (0..3) |j| try tree.file(try std.fmt.allocPrint(arena, "repo/d{d:0>2}/s{d}/f.txt", .{ i, j }), "x"); + } + const root = try std.fs.path.join(arena, &.{ tree.root, "repo" }); + + var serial = try scan(std.testing.allocator, root, .{ .max_workers = 1 }, null); + defer serial.deinit(); + var parallel = try scan(std.testing.allocator, root, .{ .max_workers = 8 }, null); + defer parallel.deinit(); + try std.testing.expectEqual(@as(usize, 60 * 7), serial.tree.entries.len); + try std.testing.expectEqual(serial.tree.entries.len, parallel.tree.entries.len); + for (serial.tree.entries, parallel.tree.entries) |a, b| { + try std.testing.expectEqualStrings(a.path, b.path); + try std.testing.expectEqual(a.kind, b.kind); + } + for (serial.tree.entries[1..], serial.tree.entries[0 .. serial.tree.entries.len - 1]) |later, earlier| { + try std.testing.expect(entryLessThan({}, earlier, later)); + } + + // Within the collection limit the cap keeps the sorted prefix. + var capped = try scan(std.testing.allocator, root, .{ .candidate_cap = 200 }, null); + defer capped.deinit(); + try std.testing.expect(capped.tree.incomplete and capped.tree.cap_reached); + try std.testing.expectEqual(@as(usize, 200), capped.tree.entries.len); + for (capped.tree.entries, serial.tree.entries[0..200]) |a, b| try std.testing.expectEqualStrings(b.path, a.path); + // Past it, reading stops early: still capped, sorted and incomplete. + var tiny = try scan(std.testing.allocator, root, .{ .candidate_cap = 10 }, null); + defer tiny.deinit(); + try std.testing.expect(tiny.tree.incomplete and tiny.tree.cap_reached); + try std.testing.expectEqual(@as(usize, 10), tiny.tree.entries.len); + for (tiny.tree.entries[1..], tiny.tree.entries[0 .. tiny.tree.entries.len - 1]) |later, earlier| try std.testing.expect(entryLessThan({}, earlier, later)); + + var short = try scan(std.testing.allocator, root, .{ .max_path_bytes = 6 }, null); + defer short.deinit(); + // Folders whose only files are over the limit stay listed. + try std.testing.expectEqual(@as(usize, 60 * 4), short.tree.entries.len); + for (short.tree.entries) |entry| try std.testing.expectEqual(Kind.directory, entry.kind); + try std.testing.expectEqual(@as(usize, 60 * 3), short.tree.skipped_overlong); + + var stop: std.atomic.Value(bool) = .init(true); + try std.testing.expectError(error.Canceled, scan(std.testing.allocator, root, .{}, &stop)); + try std.testing.expectError(error.RootUnavailable, scan(std.testing.allocator, try std.fs.path.join(arena, &.{ root, "missing" }), .{}, null)); +} + +test "repository scan reports an unusable index as incomplete and keeps untracked files" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tree = try TestTree.init(arena); + defer tree.tmp.cleanup(); + try tree.repo(arena, "repo", &.{}); + try tree.file("repo/.git/index", "DIRC garbage"); + try tree.file("repo/.gitignore", "*.log\n"); + try tree.file("repo/kept.txt", "k"); + try tree.file("repo/committed.log", "c"); + var result = try scan(std.testing.allocator, try std.fs.path.join(arena, &.{ tree.root, "repo" }), .{}, null); + defer result.deinit(); + try std.testing.expect(result.tree.incomplete); + try expectEntries(result, &.{ + .{ .path = ".gitignore", .kind = .file }, + .{ .path = "kept.txt", .kind = .file }, + }); +} diff --git a/src/core/indexer/tree.zig b/src/core/indexer/tree.zig new file mode 100644 index 000000000..4f0d36c79 --- /dev/null +++ b/src/core/indexer/tree.zig @@ -0,0 +1,164 @@ +//! The listing of one root together with the stamps that prove it is still +//! current: one per folder that was read, and one per ignore, config, index or +//! repository-marker file the rules depended on. A stamp is reusable only when +//! its times are older than the scan's start by the granularity margin, so a +//! change in the same clock tick as the scan, which a later stamp comparison +//! could not see, always forces a new scan. This is the reuse certificate the +//! refresh model (Refresh.tla) requires. + +const std = @import("std"); +const io_mod = @import("../shared/io.zig"); + +const Allocator = std.mem.Allocator; + +/// Covers coarse filesystem timestamps (FAT and some network filesystems use +/// one or two seconds) and small clock differences. +pub const granularity_margin_ns: i64 = 2 * std.time.ns_per_s; + +pub const Kind = enum(u8) { file, directory }; + +pub const Entry = struct { + /// Relative to the root, `/`-separated. + path: []const u8, + kind: Kind, +}; + +pub const FolderStamp = struct { + /// Relative to the root; empty for the root itself. + rel: []const u8, + inode: u64, + mtime_ns: i64, + ctime_ns: i64, + reusable: bool, +}; + +pub const SourceStamp = struct { + /// Absolute path of a file the rules read or looked for. + path: []const u8, + exists: bool, + /// Only existence and identity matter, as for a `.git` directory whose + /// times change with every git command. + identity_only: bool, + inode: u64 = 0, + size: u64 = 0, + mtime_ns: i64 = 0, + ctime_ns: i64 = 0, + reusable: bool, +}; + +pub const Tree = struct { + root: []const u8, + root_inode: u64, + scan_started_ns: i64, + repository: bool, + /// The cap, an unreadable source or an unreadable folder left something + /// out. An incomplete tree is never reused. + incomplete: bool, + cap_reached: bool, + skipped_overlong: usize, + skipped_names: []const []const u8, + /// Sorted by path, files before folders with the same path. + entries: []const Entry, + folders: []const FolderStamp, + sources: []const SourceStamp, +}; + +pub fn nowNs() i64 { + return clampNs(io_mod.nanoTimestamp()); +} + +fn clampNs(value: anytype) i64 { + return std.math.cast(i64, value) orelse if (value < 0) std.math.minInt(i64) else std.math.maxInt(i64); +} + +pub fn reusableTimes(mtime_ns: i64, ctime_ns: i64, scan_started_ns: i64) bool { + const limit = scan_started_ns -| granularity_margin_ns; + return mtime_ns < limit and ctime_ns < limit; +} + +pub fn folderStamp(rel: []const u8, stat: std.Io.File.Stat, scan_started_ns: i64) FolderStamp { + const mtime = clampNs(stat.mtime.nanoseconds); + const ctime = clampNs(stat.ctime.nanoseconds); + return .{ + .rel = rel, + .inode = @intCast(stat.inode), + .mtime_ns = mtime, + .ctime_ns = ctime, + .reusable = reusableTimes(mtime, ctime, scan_started_ns), + }; +} + +/// Stamps the absolute `path` without following a final symlink. +pub fn sourceStamp(path: []const u8, identity_only: bool, scan_started_ns: i64) SourceStamp { + const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), path, .{ .follow_symlinks = false }) catch { + return .{ .path = path, .exists = false, .identity_only = identity_only, .reusable = true }; + }; + const mtime = clampNs(stat.mtime.nanoseconds); + const ctime = clampNs(stat.ctime.nanoseconds); + // A `.git` file's contents choose the repository, so only a directory is + // stamped by identity alone. + const identity = identity_only and stat.kind == .directory; + return .{ + .path = path, + .exists = true, + .identity_only = identity, + .inode = @intCast(stat.inode), + .size = if (identity) 0 else stat.size, + .mtime_ns = if (identity) 0 else mtime, + .ctime_ns = if (identity) 0 else ctime, + .reusable = identity or reusableTimes(mtime, ctime, scan_started_ns), + }; +} + +fn sameSource(recorded: SourceStamp, now: SourceStamp) bool { + if (recorded.exists != now.exists) return false; + if (!recorded.exists) return true; + return recorded.inode == now.inode and recorded.size == now.size and + recorded.mtime_ns == now.mtime_ns and recorded.ctime_ns == now.ctime_ns; +} + +/// Whether `tree` still describes `root` under the same options, so it can be +/// answered from without a new scan. With `subtree` (relative to the root), +/// only the folders at or below it are checked; every source always is. +pub fn isCurrent(tree: *const Tree, root: []const u8, skipped_names: []const []const u8, subtree: ?[]const u8) bool { + if (tree.incomplete or !std.mem.eql(u8, tree.root, root)) return false; + if (tree.skipped_names.len != skipped_names.len) return false; + for (tree.skipped_names, skipped_names) |a, b| if (!std.mem.eql(u8, a, b)) return false; + const io = io_mod.getIo(); + const now = nowNs(); + var root_dir = std.Io.Dir.openDirAbsolute(io, root, .{}) catch return false; + defer root_dir.close(io); + const root_stat = root_dir.stat(io) catch return false; + if (@as(u64, @intCast(root_stat.inode)) != tree.root_inode) return false; + + for (tree.sources) |source| { + if (!source.reusable) return false; + const current = sourceStamp(source.path, source.identity_only, tree.scan_started_ns); + if (!sameSource(source, current)) return false; + if (current.mtime_ns > now or current.ctime_ns > now) return false; + } + for (tree.folders) |folder| { + if (subtree) |sub| { + if (sub.len > 0 and !(std.mem.eql(u8, folder.rel, sub) or + (folder.rel.len > sub.len and std.mem.startsWith(u8, folder.rel, sub) and folder.rel[sub.len] == '/'))) continue; + } + if (!folder.reusable) return false; + const stat = root_dir.statFile(io, if (folder.rel.len == 0) "." else folder.rel, .{ .follow_symlinks = false }) catch return false; + if (stat.kind != .directory) return false; + const current = folderStamp(folder.rel, stat, tree.scan_started_ns); + if (current.inode != folder.inode or current.mtime_ns != folder.mtime_ns or current.ctime_ns != folder.ctime_ns) return false; + if (current.mtime_ns > now or current.ctime_ns > now) return false; + } + return true; +} + +test "stamps reuse only times older than the scan start by the margin" { + const start: i64 = 100 * std.time.ns_per_s; + try std.testing.expect(reusableTimes(start - 3 * std.time.ns_per_s, start - 3 * std.time.ns_per_s, start)); + try std.testing.expect(!reusableTimes(start - std.time.ns_per_s, start - 3 * std.time.ns_per_s, start)); + try std.testing.expect(!reusableTimes(start - 3 * std.time.ns_per_s, start, start)); + try std.testing.expect(!reusableTimes(start + 1, start - 3 * std.time.ns_per_s, start)); + // A missing source stays missing until it appears. + const missing = sourceStamp("/nonexistent/fx-indexer-test", false, start); + try std.testing.expect(!missing.exists and missing.reusable); +} From 921bdfef68bd3799cbf7294c96ea8a2a4fb097b4 Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 8 Oct 2026 14:38:33 -0400 Subject: [PATCH 04/23] Save workspace listings and reuse them only while current - Save the listings for a workspace scope in one private file with a checksum, and load it only when it is a regular file owned by the user, matches the scope's roots in order, and passes every bounds check. - Treat a saved listing as current only when every folder and rule file it depended on is unchanged and was last changed at least two seconds before the scan began, so an edit in the same clock tick always forces a new scan. - Check only the folders under a subfolder when a caller asks about that subfolder. --- src/core/indexer/indexer.zig | 21 +- src/core/indexer/scan.zig | 53 +++++ src/core/indexer/snapshot.zig | 366 ++++++++++++++++++++++++++++++++++ 3 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 src/core/indexer/snapshot.zig diff --git a/src/core/indexer/indexer.zig b/src/core/indexer/indexer.zig index 719ff3a52..7e78439c7 100644 --- a/src/core/indexer/indexer.zig +++ b/src/core/indexer/indexer.zig @@ -10,6 +10,8 @@ const debug_trace = @import("../shared/debug_trace.zig"); const git_config = @import("git_config.zig"); const layout = @import("layout.zig"); const scan_mod = @import("scan.zig"); +const snapshot_mod = @import("snapshot.zig"); +const tree_mod = @import("tree.zig"); const Allocator = std.mem.Allocator; @@ -26,6 +28,22 @@ pub const default_skipped_names = scan_mod.default_skipped_names; /// must be safe to use from several threads. pub const scan = scan_mod.scan; +pub const Tree = tree_mod.Tree; +/// Whether a tree from `scan` or a snapshot still describes its root under +/// the same skipped names, checked through its folder and source stamps +/// without reading any folder. With a subtree, only folders at or below it +/// are checked. A stamp recorded within the granularity margin of its scan is +/// never trusted, so a change in the same clock tick always forces a scan. +pub const isCurrent = tree_mod.isCurrent; + +pub const Snapshot = snapshot_mod.Snapshot; +pub const snapshot_max_bytes = snapshot_mod.max_bytes; +/// Loads the saved trees for exactly `roots` from the absolute path the +/// caller chose, or null when missing or not admissible. +pub const loadSnapshot = snapshot_mod.load; +/// Saves trees atomically to the absolute path; an all-empty save deletes it. +pub const saveSnapshot = snapshot_mod.save; + pub const RepoFilters = struct { /// Absolute git directory of the repository containing the root, found /// the way git discovers one, or null outside a repository. @@ -93,7 +111,8 @@ test { _ = @import("ignore.zig"); _ = @import("sources.zig"); _ = @import("git_index.zig"); - _ = @import("tree.zig"); + _ = tree_mod; + _ = snapshot_mod; _ = scan_mod; _ = layout; _ = git_config; diff --git a/src/core/indexer/scan.zig b/src/core/indexer/scan.zig index b31cb43a1..96ffc81d3 100644 --- a/src/core/indexer/scan.zig +++ b/src/core/indexer/scan.zig @@ -806,3 +806,56 @@ test "repository scan reports an unusable index as incomplete and keeps untracke .{ .path = "kept.txt", .kind = .file }, }); } + +test "refresh trusts an old unchanged tree and rejects every kind of change" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tree = try TestTree.init(arena); + defer tree.tmp.cleanup(); + const cases = [_][]const u8{ "unchanged", "new_file", "ignore_edit", "index", "config", "replaced", "future" }; + for (cases) |name| { + try tree.repo(arena, name, &.{.{ .path = "src/a.txt" }}); + try tree.file(try std.fmt.allocPrint(arena, "{s}/.gitignore", .{name}), "*.log\n"); + try tree.file(try std.fmt.allocPrint(arena, "{s}/src/a.txt", .{name}), "a"); + try tree.file(try std.fmt.allocPrint(arena, "{s}/docs/b.md", .{name}), "b"); + } + const future: std.Io.Timestamp = .{ .nanoseconds = @as(i96, tree_mod.nowNs()) + 3600 * std.time.ns_per_s }; + try tree.tmp.dir.setTimestamps(std.testing.io, "future/docs", .{ .modify_timestamp = .{ .new = future } }); + + // A tree scanned right after its files changed is never reused. + { + const root = try std.fs.path.join(arena, &.{ tree.root, "unchanged" }); + var racy = try scan(std.testing.allocator, root, .{}, null); + defer racy.deinit(); + try std.testing.expect(!tree_mod.isCurrent(&racy.tree, root, &default_skipped_names, null)); + } + + io_mod.sleep(@intCast(tree_mod.granularity_margin_ns + std.time.ns_per_s / 2)); + var results: [cases.len]Result = undefined; + var roots: [cases.len][]const u8 = undefined; + for (cases, 0..) |name, i| { + roots[i] = try std.fs.path.join(arena, &.{ tree.root, name }); + results[i] = try scan(std.testing.allocator, roots[i], .{}, null); + } + defer for (&results) |*result| result.deinit(); + + try std.testing.expect(tree_mod.isCurrent(&results[0].tree, roots[0], &default_skipped_names, null)); + try std.testing.expect(!tree_mod.isCurrent(&results[0].tree, roots[0], &.{"node_modules"}, null)); + + try tree.file("new_file/src/c.txt", "c"); + try tree.file("ignore_edit/.gitignore", "*.md\n"); + try tree.file("index/.git/index", try git_index.buildForTest(arena, 2, .sha1, &.{ .{ .path = "src/a.txt" }, .{ .path = "docs/b.md" } }, "")); + try tree.file("config/.git/config", "[core]\n\tignorecase = true\n"); + try tree.tmp.dir.rename("replaced", tree.tmp.dir, "replaced-old", std.testing.io); + try tree.repo(arena, "replaced", &.{.{ .path = "src/a.txt" }}); + + // A change in `src` leaves a check of `docs` alone. + try std.testing.expect(tree_mod.isCurrent(&results[1].tree, roots[1], &default_skipped_names, "docs")); + for (results[1..], roots[1..], cases[1..]) |*result, root, name| { + if (tree_mod.isCurrent(&result.tree, root, &default_skipped_names, null)) { + std.debug.print("stale tree reported current: {s}\n", .{name}); + return error.TestExpectedStale; + } + } +} diff --git a/src/core/indexer/snapshot.zig b/src/core/indexer/snapshot.zig new file mode 100644 index 000000000..b9c9e98c7 --- /dev/null +++ b/src/core/indexer/snapshot.zig @@ -0,0 +1,366 @@ +//! Saved trees for one scope (format v2), so `@` can paint at launch and a +//! refresh can confirm nothing changed without reading folders again. The +//! caller chooses the file path. A file is used only when it is a private +//! regular file with one link, at most `max_bytes`, matching digest, exactly +//! the requested roots and well-formed contents; anything else is ignored +//! with a trace and a new scan follows. Older formats fail the magic check +//! and are overwritten by the next save. + +const std = @import("std"); +const io_mod = @import("../shared/io.zig"); +const debug_trace = @import("../shared/debug_trace.zig"); +const tree_mod = @import("tree.zig"); + +const Allocator = std.mem.Allocator; +const Sha256 = std.crypto.hash.sha2.Sha256; +const Tree = tree_mod.Tree; + +const magic = "fx-file-index-v2\n"; +pub const max_bytes: usize = 64 * 1024 * 1024; +const max_path_bytes: usize = 4096; + +pub const Snapshot = struct { + arena: std.heap.ArenaAllocator, + /// One per requested root, in the requested order. + trees: []const Tree, + + pub fn deinit(self: *Snapshot) void { + self.arena.deinit(); + self.* = undefined; + } +}; + +/// Loads the snapshot at the absolute `path` for exactly `roots`, or null when +/// it is missing or not admissible. +pub fn load(alloc: Allocator, path: []const u8, roots: []const []const u8) Allocator.Error!?Snapshot { + var arena_state = std.heap.ArenaAllocator.init(alloc); + errdefer arena_state.deinit(); + const arena = arena_state.allocator(); + const bytes = readAdmitted(arena, path) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.FileNotFound => { + arena_state.deinit(); + return null; + }, + else => { + debug_trace.logf("indexer", "snapshot ignored reason={s}", .{@errorName(err)}); + arena_state.deinit(); + return null; + }, + }; + const trees = decode(arena, bytes, roots) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.InvalidSnapshot => { + debug_trace.logf("indexer", "snapshot ignored reason=invalid_contents", .{}); + arena_state.deinit(); + return null; + }, + }; + return .{ .arena = arena_state, .trees = trees }; +} + +fn readAdmitted(arena: Allocator, path: []const u8) ![]const u8 { + const io = io_mod.getIo(); + var file = try std.Io.Dir.openFileAbsolute(io, path, .{ .follow_symlinks = false, .allow_directory = false }); + defer file.close(io); + const stat = try file.stat(io); + if (stat.kind != .file or stat.nlink != 1 or (stat.permissions.toMode() & 0o077) != 0 or stat.size > max_bytes) return error.InvalidSnapshot; + const bytes = try arena.alloc(u8, @intCast(stat.size)); + var offset: usize = 0; + while (offset < bytes.len) { + const end = @min(bytes.len, offset + 64 * 1024); + const read = try file.readPositionalAll(io, bytes[offset..end], offset); + if (read != end - offset) return error.InvalidSnapshot; + offset = end; + } + if (bytes.len < magic.len + Sha256.digest_length or !std.mem.startsWith(u8, bytes, magic)) return error.InvalidSnapshot; + const payload = bytes[magic.len + Sha256.digest_length ..]; + var digest: [Sha256.digest_length]u8 = undefined; + Sha256.hash(payload, &digest, .{}); + if (!std.mem.eql(u8, &digest, bytes[magic.len..][0..Sha256.digest_length])) return error.InvalidSnapshot; + return payload; +} + +/// Saves `trees` to the absolute `path`, replacing it atomically. When every +/// tree is empty the file is deleted instead, so stale entries are never +/// painted. +pub fn save(alloc: Allocator, path: []const u8, trees: []const Tree) !void { + const io = io_mod.getIo(); + var empty = true; + for (trees) |tree| empty = empty and tree.entries.len == 0; + if (empty) { + std.Io.Dir.deleteFileAbsolute(io, path) catch |err| switch (err) { + error.FileNotFound => {}, + else => return err, + }; + return; + } + var payload: std.Io.Writer.Allocating = .init(alloc); + defer payload.deinit(); + try encode(&payload.writer, trees); + if (payload.written().len > max_bytes - magic.len - Sha256.digest_length) return error.SnapshotTooLarge; + + var out: std.Io.Writer.Allocating = .init(alloc); + defer out.deinit(); + try out.writer.writeAll(magic); + var digest: [Sha256.digest_length]u8 = undefined; + Sha256.hash(payload.written(), &digest, .{}); + try out.writer.writeAll(&digest); + try out.writer.writeAll(payload.written()); + + const dir_path = std.fs.path.dirname(path) orelse return error.InvalidSnapshotPath; + try io_mod.makeDirRecursive(dir_path); + // `.iterate` matters on Linux: without it the descriptor is O_PATH and + // the directory fsync inside durableReplaceVerified fails with EBADF. + var dir = try std.Io.Dir.openDirAbsolute(io, dir_path, .{ .follow_symlinks = false, .iterate = true }); + defer dir.close(io); + var verified: io_mod.VerifiedDir = .{ .dir = dir }; + try io_mod.durableReplaceVerified(alloc, &verified, std.fs.path.basename(path), out.written()); + try dir.setPermissions(io, .fromMode(0o700)); +} + +fn writeInt(writer: *std.Io.Writer, comptime T: type, value: T) !void { + var bytes: [@sizeOf(T)]u8 = undefined; + std.mem.writeInt(T, &bytes, value, .little); + try writer.writeAll(&bytes); +} + +fn writeString(writer: *std.Io.Writer, value: []const u8) !void { + try writeInt(writer, u32, @intCast(value.len)); + try writer.writeAll(value); +} + +fn encode(writer: *std.Io.Writer, trees: []const Tree) !void { + try writeInt(writer, u32, @intCast(trees.len)); + for (trees) |tree| { + try writeString(writer, tree.root); + try writeInt(writer, u64, tree.root_inode); + try writeInt(writer, i64, tree.scan_started_ns); + const flags: u8 = @as(u8, @intFromBool(tree.repository)) | @as(u8, @intFromBool(tree.incomplete)) << 1 | @as(u8, @intFromBool(tree.cap_reached)) << 2; + try writer.writeByte(flags); + try writeInt(writer, u64, tree.skipped_overlong); + try writeInt(writer, u32, @intCast(tree.skipped_names.len)); + for (tree.skipped_names) |name| try writeString(writer, name); + try writeInt(writer, u32, @intCast(tree.entries.len)); + for (tree.entries) |entry| { + try writer.writeByte(@intFromEnum(entry.kind)); + try writeString(writer, entry.path); + } + try writeInt(writer, u32, @intCast(tree.folders.len)); + for (tree.folders) |folder| { + try writeString(writer, folder.rel); + try writeInt(writer, u64, folder.inode); + try writeInt(writer, i64, folder.mtime_ns); + try writeInt(writer, i64, folder.ctime_ns); + try writer.writeByte(@intFromBool(folder.reusable)); + } + try writeInt(writer, u32, @intCast(tree.sources.len)); + for (tree.sources) |source| { + try writeString(writer, source.path); + const source_flags: u8 = @as(u8, @intFromBool(source.exists)) | @as(u8, @intFromBool(source.identity_only)) << 1 | @as(u8, @intFromBool(source.reusable)) << 2; + try writer.writeByte(source_flags); + try writeInt(writer, u64, source.inode); + try writeInt(writer, u64, source.size); + try writeInt(writer, i64, source.mtime_ns); + try writeInt(writer, i64, source.ctime_ns); + } + } +} + +const Decoder = struct { + bytes: []const u8, + pos: usize = 0, + + fn take(self: *Decoder, len: usize) error{InvalidSnapshot}![]const u8 { + if (len > self.bytes.len - self.pos) return error.InvalidSnapshot; + defer self.pos += len; + return self.bytes[self.pos .. self.pos + len]; + } + + fn int(self: *Decoder, comptime T: type) error{InvalidSnapshot}!T { + return std.mem.readInt(T, (try self.take(@sizeOf(T)))[0..@sizeOf(T)], .little); + } + + /// A count of items each at least `min_item_bytes` long, bounded by what is left. + fn count(self: *Decoder, min_item_bytes: usize) error{InvalidSnapshot}!usize { + const value = try self.int(u32); + if (value > (self.bytes.len - self.pos) / min_item_bytes) return error.InvalidSnapshot; + return value; + } + + fn string(self: *Decoder, arena: Allocator, max: usize, allow_empty: bool) (Allocator.Error || error{InvalidSnapshot})![]const u8 { + const len = try self.int(u32); + if (len > max or (len == 0 and !allow_empty)) return error.InvalidSnapshot; + const value = try self.take(len); + if (std.mem.findScalar(u8, value, 0) != null) return error.InvalidSnapshot; + return arena.dupe(u8, value); + } +}; + +fn decode(arena: Allocator, bytes: []const u8, roots: []const []const u8) (Allocator.Error || error{InvalidSnapshot})![]const Tree { + var decoder: Decoder = .{ .bytes = bytes }; + const tree_count = try decoder.count(1); + if (tree_count != roots.len) return error.InvalidSnapshot; + const trees = try arena.alloc(Tree, tree_count); + for (trees, roots) |*tree, expected_root| { + const root = try decoder.string(arena, max_path_bytes, false); + if (!std.mem.eql(u8, root, expected_root)) return error.InvalidSnapshot; + const root_inode = try decoder.int(u64); + const scan_started_ns = try decoder.int(i64); + const flags = (try decoder.take(1))[0]; + if (flags & ~@as(u8, 0b111) != 0) return error.InvalidSnapshot; + const skipped_overlong = try decoder.int(u64); + const names = try arena.alloc([]const u8, try decoder.count(4)); + for (names) |*name| name.* = try decoder.string(arena, 255, false); + const entries = try arena.alloc(tree_mod.Entry, try decoder.count(5)); + for (entries) |*entry| { + const kind = (try decoder.take(1))[0]; + if (kind > 1) return error.InvalidSnapshot; + entry.* = .{ .kind = @enumFromInt(kind), .path = try decoder.string(arena, max_path_bytes, false) }; + } + const folders = try arena.alloc(tree_mod.FolderStamp, try decoder.count(29)); + for (folders) |*folder| { + const rel = try decoder.string(arena, max_path_bytes, true); + folder.* = .{ + .rel = rel, + .inode = try decoder.int(u64), + .mtime_ns = try decoder.int(i64), + .ctime_ns = try decoder.int(i64), + .reusable = switch ((try decoder.take(1))[0]) { + 0 => false, + 1 => true, + else => return error.InvalidSnapshot, + }, + }; + } + const stamps = try arena.alloc(tree_mod.SourceStamp, try decoder.count(37)); + for (stamps) |*source| { + const path = try decoder.string(arena, max_path_bytes, false); + const source_flags = (try decoder.take(1))[0]; + if (source_flags & ~@as(u8, 0b111) != 0) return error.InvalidSnapshot; + source.* = .{ + .path = path, + .exists = source_flags & 1 != 0, + .identity_only = source_flags & 2 != 0, + .reusable = source_flags & 4 != 0, + .inode = try decoder.int(u64), + .size = try decoder.int(u64), + .mtime_ns = try decoder.int(i64), + .ctime_ns = try decoder.int(i64), + }; + } + tree.* = .{ + .root = root, + .root_inode = root_inode, + .scan_started_ns = scan_started_ns, + .repository = flags & 1 != 0, + .incomplete = flags & 2 != 0, + .cap_reached = flags & 4 != 0, + .skipped_overlong = std.math.cast(usize, skipped_overlong) orelse return error.InvalidSnapshot, + .skipped_names = names, + .entries = entries, + .folders = folders, + .sources = stamps, + }; + } + if (decoder.pos != bytes.len) return error.InvalidSnapshot; + return trees; +} + +fn testTree(root: []const u8, entries: []const tree_mod.Entry) Tree { + return .{ + .root = root, + .root_inode = 7, + .scan_started_ns = 1_000, + .repository = true, + .incomplete = false, + .cap_reached = false, + .skipped_overlong = 2, + .skipped_names = &.{ "node_modules", "dist" }, + .entries = entries, + .folders = &.{ + .{ .rel = "", .inode = 7, .mtime_ns = 5, .ctime_ns = 6, .reusable = true }, + .{ .rel = "src", .inode = 9, .mtime_ns = 5, .ctime_ns = 900, .reusable = false }, + }, + .sources = &.{ + .{ .path = "/w/.gitignore", .exists = true, .identity_only = false, .inode = 3, .size = 10, .mtime_ns = 4, .ctime_ns = 4, .reusable = true }, + .{ .path = "/w/.git", .exists = true, .identity_only = true, .inode = 11, .reusable = true }, + .{ .path = "/home/.gitconfig", .exists = false, .identity_only = false, .reusable = true }, + }, + }; +} + +test "snapshot round trips, rejects tampering, other scopes and v1 files, and an empty save deletes it" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const home = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); + defer alloc.free(home); + const path = try std.fs.path.join(alloc, &.{ home, "file-index", "scope.idx" }); + defer alloc.free(path); + const roots = [_][]const u8{ "/w", "/extra" }; + const trees = [_]Tree{ + testTree("/w", &.{ .{ .path = "a.txt", .kind = .file }, .{ .path = "src", .kind = .directory } }), + testTree("/extra", &.{.{ .path = "b.txt", .kind = .file }}), + }; + + try std.testing.expect((try load(alloc, path, &roots)) == null); + try save(alloc, path, &trees); + { + var loaded = (try load(alloc, path, &roots)).?; + defer loaded.deinit(); + try std.testing.expectEqual(@as(usize, 2), loaded.trees.len); + const first = loaded.trees[0]; + try std.testing.expectEqualStrings("/w", first.root); + try std.testing.expectEqual(@as(u64, 7), first.root_inode); + try std.testing.expect(first.repository and !first.incomplete and !first.cap_reached); + try std.testing.expectEqual(@as(usize, 2), first.skipped_overlong); + try std.testing.expectEqualStrings("dist", first.skipped_names[1]); + try std.testing.expectEqualStrings("src", first.entries[1].path); + try std.testing.expectEqual(tree_mod.Kind.directory, first.entries[1].kind); + try std.testing.expectEqualStrings("", first.folders[0].rel); + try std.testing.expect(!first.folders[1].reusable); + try std.testing.expect(first.sources[1].identity_only and !first.sources[2].exists); + try std.testing.expectEqualStrings("b.txt", loaded.trees[1].entries[0].path); + } + + // A different scope, or the same roots in another order, never reuses it. + try std.testing.expect((try load(alloc, path, &.{"/w"})) == null); + try std.testing.expect((try load(alloc, path, &.{ "/extra", "/w" })) == null); + + // Every single-byte change breaks the digest. + var file = try std.Io.Dir.openFileAbsolute(std.testing.io, path, .{ .mode = .read_write }); + try file.writePositionalAll(std.testing.io, "X", magic.len + Sha256.digest_length + 4); + file.close(std.testing.io); + try std.testing.expect((try load(alloc, path, &roots)) == null); + + // A v1 file fails the magic check. + try std.Io.Dir.cwd().writeFile(std.testing.io, .{ .sub_path = path, .data = "fx-file-index-v1\n{}", .flags = .{ .permissions = .fromMode(0o600) } }); + try std.testing.expect((try load(alloc, path, &roots)) == null); + + // A file others can read is not trusted. + try save(alloc, path, &trees); + try std.Io.Dir.cwd().setFilePermissions(std.testing.io, path, .fromMode(0o644), .{}); + try std.testing.expect((try load(alloc, path, &roots)) == null); + + try save(alloc, path, &.{ testTree("/w", &.{}), testTree("/extra", &.{}) }); + try std.testing.expectError(error.FileNotFound, std.Io.Dir.cwd().statFile(std.testing.io, path, .{})); +} + +test "snapshot decoding rejects every truncation and malformed fields" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const trees = [_]Tree{testTree("/w", &.{.{ .path = "a.txt", .kind = .file }})}; + var payload: std.Io.Writer.Allocating = .init(arena); + try encode(&payload.writer, &trees); + const bytes = payload.written(); + _ = try decode(arena, bytes, &.{"/w"}); + for (0..bytes.len) |len| try std.testing.expectError(error.InvalidSnapshot, decode(arena, bytes[0..len], &.{"/w"})); + const extra = try std.mem.concat(arena, u8, &.{ bytes, "x" }); + try std.testing.expectError(error.InvalidSnapshot, decode(arena, extra, &.{"/w"})); + const bad_kind = try arena.dupe(u8, bytes); + const kind_at = std.mem.find(u8, bad_kind, "a.txt").? - 5; + bad_kind[kind_at] = 9; + try std.testing.expectError(error.InvalidSnapshot, decode(arena, bad_kind, &.{"/w"})); +} From f2e80bf5ac0fb2f0a2149cad61942cd6d7cb6ba7 Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 8 Oct 2026 16:11:13 -0400 Subject: [PATCH 05/23] Load @-completion through the native indexer - List @-completion candidates with the native indexer instead of git, so opening a repository no longer starts git or any program its config names. - Paint the picker from the saved listing, then reuse every root that is still current and scan only the others, saving again only when something was rescanned. - Take candidates straight from the walk instead of checking each file again. - Leave a scope with an unavailable root unsaved, so the next load scans it again. --- src/core/workspace/file_index.zig | 375 +++++++++++++----------- src/core/workspace/file_index_cache.zig | 246 ---------------- tests/e2e/tui-file-picker.test.ts | 19 +- 3 files changed, 220 insertions(+), 420 deletions(-) delete mode 100644 src/core/workspace/file_index_cache.zig diff --git a/src/core/workspace/file_index.zig b/src/core/workspace/file_index.zig index d45656c8a..de1854955 100644 --- a/src/core/workspace/file_index.zig +++ b/src/core/workspace/file_index.zig @@ -15,8 +15,7 @@ const text_utils = @import("../shared/text_utils.zig"); const unicode_simple_fold = @import("unicode_simple_fold.zig"); const pathing = @import("pathing.zig"); const workspace_access = @import("workspace_access.zig"); -const workspace_files = @import("workspace_files.zig"); -const file_index_cache = @import("file_index_cache.zig"); +const indexer = @import("../indexer/indexer.zig"); const Allocator = std.mem.Allocator; @@ -800,16 +799,22 @@ fn loadGeneration( defer arena_state.deinit(); const arena = arena_state.allocator(); + const snapshot_path = snapshotPath(arena, roots) catch |err| return .{ .failed = .{ .stage = .discovery, .err = err } }; + var previous = if (snapshot_path) |path| + indexer.loadSnapshot(alloc, path, roots) catch |err| return .{ .failed = .{ .stage = .discovery, .err = err } } + else + null; + defer if (previous) |*snapshot| snapshot.deinit(); + if (allow_cache) { - if (file_index_cache.load(arena, roots) catch |err| blk: { - if (err == error.OutOfMemory) return .{ .failed = .{ .stage = .discovery, .err = err } }; - break :blk null; - }) |loaded_value| { - var loaded = loaded_value; - defer loaded.deinit(arena); + if (previous) |*snapshot| { + const candidates = candidatesFromTrees(arena, roots, snapshot.trees, stop_requested) catch |err| { + if (err == error.Canceled or isStopRequested(stop_requested)) return .canceled; + return .{ .failed = .{ .stage = .discovery, .err = err } }; + }; generation.from_cache = true; generation.fillProgressive(alloc, .{ .typed = .{ - .candidates = loaded.candidates, + .candidates = candidates, .indices = null, } }, stop_requested) catch |err| { if (err == error.Canceled or isStopRequested(stop_requested)) return .canceled; @@ -821,16 +826,27 @@ fn loadGeneration( } } - const candidates = discoverScopeCandidates(arena, roots, stop_requested) catch |err| { + var refreshed = refreshTrees(alloc, roots, if (previous) |*snapshot| snapshot.trees else &.{}, stop_requested) catch |err| { if (err == error.Canceled or isStopRequested(stop_requested)) return .canceled; return .{ .failed = .{ .stage = .discovery, .err = err } }; }; + defer refreshed.deinit(alloc); - // Persist before the fill so even a canceled publish leaves a warm cache. - file_index_cache.save(arena, roots, candidates) catch |err| { - debug_trace.logf("core", "file index cache not saved generation={d} err={s}", .{ generation.id, @errorName(err) }); + // Persist before the fill so even a canceled publish leaves a warm + // snapshot. A scope with an unavailable root is not saved, so its next + // load scans again. + if (snapshot_path) |path| { + if (refreshed.scans.len > 0 and refreshed.trees.len == roots.len) { + indexer.saveSnapshot(arena, path, refreshed.trees) catch |err| { + debug_trace.logf("core", "file index cache not saved generation={d} err={s}", .{ generation.id, @errorName(err) }); + }; + } + } + + const candidates = candidatesFromTrees(arena, roots, refreshed.trees, stop_requested) catch |err| { + if (err == error.Canceled or isStopRequested(stop_requested)) return .canceled; + return .{ .failed = .{ .stage = .discovery, .err = err } }; }; - generation.fillProgressive(alloc, .{ .typed = .{ .candidates = candidates, .indices = null, @@ -842,6 +858,117 @@ fn loadGeneration( return .ready; } +/// `/.fx/file-index/.idx`, one file per scope, or +/// null without an absolute home directory. +pub fn snapshotPath(alloc: Allocator, roots: []const []const u8) Allocator.Error!?[]const u8 { + const home = io_mod.getenv("HOME") orelse return null; + if (!std.fs.path.isAbsolute(home)) return null; + return try snapshotPathIn(alloc, home, roots); +} + +fn snapshotPathIn(alloc: Allocator, home: []const u8, roots: []const []const u8) Allocator.Error![]const u8 { + const hex = std.fmt.bytesToHex(rootsDigest(roots), .lower); + return std.fmt.allocPrint(alloc, "{s}/.fx/file-index/{s}.idx", .{ home, &hex }); +} + +/// The trees for the available roots of a scope, in root order. Trees reused +/// from a snapshot borrow its memory; `scans` owns the rest. +const RefreshedTrees = struct { + trees: []indexer.Tree, + scans: []indexer.ScanResult, + + fn deinit(self: *RefreshedTrees, alloc: Allocator) void { + for (self.scans) |*result| result.deinit(); + alloc.free(self.scans); + alloc.free(self.trees); + self.* = undefined; + } +}; + +/// Reuses each saved tree that is still current and scans every other root. +/// `previous` is empty or holds one tree per root in order. An unavailable +/// root is left out; a scope with no available root fails. +fn refreshTrees( + alloc: Allocator, + roots: []const []const u8, + previous: []const indexer.Tree, + stop_requested: *std.atomic.Value(bool), +) (indexer.ScanError || error{FileNotFound})!RefreshedTrees { + std.debug.assert(previous.len == 0 or previous.len == roots.len); + var trees: std.ArrayList(indexer.Tree) = .empty; + defer trees.deinit(alloc); + var scans: std.ArrayList(indexer.ScanResult) = .empty; + defer { + for (scans.items) |*result| result.deinit(); + scans.deinit(alloc); + } + try trees.ensureTotalCapacity(alloc, roots.len); + try scans.ensureTotalCapacity(alloc, roots.len); + + for (roots, 0..) |root, root_index| { + if (isStopRequested(stop_requested)) return error.Canceled; + if (previous.len > 0 and indexer.isCurrent(&previous[root_index], root, &indexer.default_skipped_names, null)) { + debug_trace.logf("core", "file index root reused root={d} entries={d} thread={d}", .{ root_index, previous[root_index].entries.len, std.Thread.getCurrentId() }); + trees.appendAssumeCapacity(previous[root_index]); + continue; + } + const result = indexer.scan(alloc, root, .{ .candidate_cap = max_indexed_files }, stop_requested) catch |err| switch (err) { + error.RootUnavailable => { + debug_trace.logf("core", "file index root discovery omitted root={d} err={s}", .{ root_index, @errorName(err) }); + continue; + }, + else => |e| return e, + }; + scans.appendAssumeCapacity(result); + trees.appendAssumeCapacity(result.tree); + debug_trace.logf("core", "file index root scanned root={d} entries={d} incomplete={} thread={d}", .{ root_index, result.tree.entries.len, result.tree.incomplete, std.Thread.getCurrentId() }); + } + if (trees.items.len == 0 and roots.len > 0) return error.FileNotFound; + + const owned_trees = try trees.toOwnedSlice(alloc); + errdefer alloc.free(owned_trees); + return .{ .trees = owned_trees, .scans = try scans.toOwnedSlice(alloc) }; +} + +/// Converts trees into candidates in root order: primary-root paths stay +/// relative, added-root paths become absolute, overlapping roots appear once, +/// and at most `max_indexed_files` are kept. Paths may borrow tree memory. +fn candidatesFromTrees( + arena: Allocator, + roots: []const []const u8, + trees: []const indexer.Tree, + stop_requested: *std.atomic.Value(bool), +) (Allocator.Error || error{Canceled})![]const Candidate { + var candidates: std.ArrayList(Candidate) = .empty; + var seen: std.StringHashMapUnmanaged(void) = .empty; + for (trees) |*tree| { + const root_index: usize = if (roots.len > 0 and std.mem.eql(u8, tree.root, roots[0])) 0 else 1; + for (tree.entries) |entry| { + if (isStopRequested(stop_requested)) return error.Canceled; + if (candidates.items.len >= max_indexed_files) return candidates.toOwnedSlice(arena); + if (hasGitComponent(entry.path)) continue; + if (!text_utils.isTerminalSafe(entry.path)) { + debug_trace.logf("core", "file index omitted unsafe candidate bytes={d} kind={s}", .{ entry.path.len, @tagName(entry.kind) }); + continue; + } + const kind: CandidateKind = switch (entry.kind) { + .file => .file, + .directory => .directory, + }; + _ = try appendDiscoveredCandidate(arena, &candidates, &seen, tree.root, root_index, entry.path, kind); + } + } + return candidates.toOwnedSlice(arena); +} + +fn hasGitComponent(path: []const u8) bool { + var parts = std.mem.splitScalar(u8, path, '/'); + while (parts.next()) |part| { + if (std.mem.eql(u8, part, ".git")) return true; + } + return false; +} + /// Publishes the loader's terminal outcome only after its caller has released /// every worker-owned temporary. The release store is deliberately the final /// action so an acquire observer may join without waiting for worker cleanup. @@ -899,136 +1026,6 @@ fn rootsEqual(a: []const []const u8, b: []const []const u8) bool { return true; } -fn discoverScopeCandidates( - alloc: Allocator, - roots: []const []const u8, - stop_requested: *std.atomic.Value(bool), -) ![]const Candidate { - return discoverScopeCandidatesWithFallback(alloc, roots, stop_requested, false); -} - -fn discoverScopeCandidatesWithFallback( - alloc: Allocator, - roots: []const []const u8, - stop_requested: *std.atomic.Value(bool), - force_fallback: bool, -) ![]const Candidate { - if (isStopRequested(stop_requested)) return error.Canceled; - - var candidates: std.ArrayList(Candidate) = .empty; - errdefer candidates.deinit(alloc); - var seen: std.StringHashMapUnmanaged(void) = .empty; - defer seen.deinit(alloc); - var kept: usize = 0; - var succeeded: usize = 0; - - for (roots, 0..) |root_path, root_index| { - if (isStopRequested(stop_requested)) return error.Canceled; - if (kept >= max_indexed_files) break; - const remaining = max_indexed_files - kept; - - const discovered = workspace_files.discoverCancellable( - alloc, - root_path, - .{ - .candidate_cap = remaining, - .force_fallback = force_fallback, - .include_hidden = true, - .include_untracked = true, - .sort_paths = true, - .git_worktree_is_authoritative = true, - }, - stop_requested, - ) catch |err| { - if (err == error.Canceled or isStopRequested(stop_requested)) return error.Canceled; - debug_trace.logf("core", "file index root discovery omitted root={d} err={s}", .{ root_index, @errorName(err) }); - continue; - }; - succeeded += 1; - - const discovered_directories: []const []const u8 = directories: { - const result = workspace_files.discoverDirectoriesCancellable( - alloc, - root_path, - .{ - .candidate_cap = remaining, - .force_fallback = force_fallback, - .include_hidden = true, - .sort_paths = true, - .git_worktree_is_authoritative = true, - }, - stop_requested, - ) catch |err| { - if (err == error.Canceled or isStopRequested(stop_requested)) return error.Canceled; - debug_trace.logf("core", "file index root directory discovery omitted root={d} err={s}", .{ root_index, @errorName(err) }); - break :directories &.{}; - }; - break :directories result.directories; - }; - - var root = std.Io.Dir.openDirAbsolute(io_mod.getIo(), root_path, .{}) catch |err| { - debug_trace.logf("core", "file index root admission omitted root={d} err={s}", .{ root_index, @errorName(err) }); - succeeded -= 1; - continue; - }; - defer root.close(io_mod.getIo()); - - var file_cursor: usize = 0; - var directory_cursor: usize = 0; - while (nextSortedDiscoveredCandidate( - discovered.files, - discovered_directories, - &file_cursor, - &directory_cursor, - )) |candidate| { - if (isStopRequested(stop_requested)) return error.Canceled; - if (kept >= max_indexed_files) break; - if (workspace_files.pathContainsIgnoredDir(&.{".git"}, candidate.path)) continue; - if (!text_utils.isTerminalSafe(candidate.path)) { - debug_trace.logf("core", "file index omitted unsafe candidate bytes={d} kind={s}", .{ candidate.path.len, @tagName(candidate.kind) }); - continue; - } - if (candidate.kind == .file) { - const stat = root.statFile(io_mod.getIo(), candidate.path, .{ .follow_symlinks = false }) catch |err| { - debug_trace.logf("core", "file index omitted missing candidate bytes={d} err={s}", .{ candidate.path.len, @errorName(err) }); - continue; - }; - if (stat.kind != .file and stat.kind != .sym_link) continue; - } - - if (try appendDiscoveredCandidate( - alloc, - &candidates, - &seen, - root_path, - root_index, - candidate.path, - candidate.kind, - )) kept += 1; - } - } - - if (succeeded == 0 and roots.len > 0) return error.FileNotFound; - return candidates.toOwnedSlice(alloc); -} - -fn nextSortedDiscoveredCandidate( - files: []const []const u8, - directories: []const []const u8, - file_cursor: *usize, - directory_cursor: *usize, -) ?Candidate { - if (file_cursor.* >= files.len and directory_cursor.* >= directories.len) return null; - if (directory_cursor.* >= directories.len or - (file_cursor.* < files.len and std.mem.order(u8, files[file_cursor.*], directories[directory_cursor.*]) != .gt)) - { - defer file_cursor.* += 1; - return .{ .path = files[file_cursor.*], .kind = .file }; - } - defer directory_cursor.* += 1; - return .{ .path = directories[directory_cursor.*], .kind = .directory }; -} - fn appendDiscoveredCandidate( alloc: Allocator, candidates: *std.ArrayList(Candidate), @@ -1840,25 +1837,52 @@ test "typed candidate cap is shared across files and directories" { try std.testing.expectEqual(CandidateKind.directory, file_index.kindAt(max_indexed_files - 1)); } -test "sorted discovery merge retains directories at the exact shared cap" { - const files = try std.testing.allocator.alloc([]const u8, max_indexed_files); - defer std.testing.allocator.free(files); - @memset(files, "z-file"); - const directories = [_][]const u8{"a-empty-directory"}; - var file_cursor: usize = 0; - var directory_cursor: usize = 0; - var count: usize = 0; - var saw_directory = false; +test "tree candidates keep a sorted directory at the exact shared cap" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const entries = try arena.alloc(indexer.Entry, max_indexed_files + 1); + entries[0] = .{ .path = "a-empty-directory", .kind = .directory }; + for (entries[1..], 0..) |*entry, index| { + entry.* = .{ .path = try std.fmt.allocPrint(arena, "z-file-{d:0>6}", .{index}), .kind = .file }; + } + const tree: indexer.Tree = .{ + .root = "/workspace", + .root_inode = 1, + .scan_started_ns = 0, + .repository = false, + .incomplete = false, + .cap_reached = false, + .skipped_overlong = 0, + .skipped_names = &.{}, + .entries = entries, + .folders = &.{}, + .sources = &.{}, + }; + var stop_requested = std.atomic.Value(bool).init(false); + const candidates = try candidatesFromTrees(arena, &.{"/workspace"}, &.{tree}, &stop_requested); + try std.testing.expectEqual(max_indexed_files, candidates.len); + try std.testing.expectEqual(CandidateKind.directory, candidates[0].kind); + try std.testing.expectEqualStrings("z-file-099998", candidates[max_indexed_files - 1].path); +} - while (count < max_indexed_files) : (count += 1) { - const candidate = nextSortedDiscoveredCandidate(files, &directories, &file_cursor, &directory_cursor).?; - saw_directory = saw_directory or candidate.kind == .directory; +/// Scans `roots` without a snapshot and keeps the trees alive while the +/// caller inspects candidates that borrow their memory. +const TestScope = struct { + refreshed: RefreshedTrees, + candidates: []const Candidate, + + fn init(arena: Allocator, roots: []const []const u8) !TestScope { + var stop_requested = std.atomic.Value(bool).init(false); + var refreshed = try refreshTrees(std.testing.allocator, roots, &.{}, &stop_requested); + errdefer refreshed.deinit(std.testing.allocator); + return .{ .refreshed = refreshed, .candidates = try candidatesFromTrees(arena, roots, refreshed.trees, &stop_requested) }; } - try std.testing.expect(saw_directory); - try std.testing.expectEqual(max_indexed_files - 1, file_cursor); - try std.testing.expectEqual(@as(usize, 1), directory_cursor); -} + fn deinit(self: *TestScope) void { + self.refreshed.deinit(std.testing.allocator); + } +}; test "subsequence scoring prefers basename prefix over mid-path match" { const path_a = "src/main.zig"; @@ -2581,9 +2605,11 @@ test "persisted file index paints a stale preview and the real scan replaces it" const scanned_count = first.count(); try std.testing.expectEqual(@as(usize, 1), adoptions); try std.testing.expect(scanned_count >= 2); - var cached = (try file_index_cache.loadFrom(alloc, home, &roots)).?; - defer cached.deinit(alloc); - try std.testing.expect(cached.candidates.len > 0); + const snapshot_file = try snapshotPathIn(alloc, home, &roots); + defer alloc.free(snapshot_file); + var cached = (try indexer.loadSnapshot(alloc, snapshot_file, &roots)).?; + defer cached.deinit(); + try std.testing.expect(cached.trees[0].entries.len > 0); // The tree changes between launches. { @@ -2653,10 +2679,11 @@ test "scope discovery emits primary-relative and added-absolute paths in root or const shared_directory = try std.fs.path.join(alloc, &.{ shared, "nested" }); defer alloc.free(shared_directory); const roots = [_][]const u8{ primary, shared }; - var stop_requested = std.atomic.Value(bool).init(false); var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - const candidates = try discoverScopeCandidatesWithFallback(arena_state.allocator(), &roots, &stop_requested, true); + var scope = try TestScope.init(arena_state.allocator(), &roots); + defer scope.deinit(); + const candidates = scope.candidates; var index = FileIndex{}; defer index.deinit(alloc); @@ -2715,8 +2742,9 @@ test "production scope admits tracked untracked hidden and direct directory cand var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - var stop_requested = std.atomic.Value(bool).init(false); - const candidates = try discoverScopeCandidates(arena_state.allocator(), &.{root}, &stop_requested); + var scope = try TestScope.init(arena_state.allocator(), &.{root}); + defer scope.deinit(); + const candidates = scope.candidates; try std.testing.expect(containsCandidate(candidates, "tracked.txt", .file)); try std.testing.expect(containsCandidate(candidates, "untracked.txt", .file)); @@ -2732,7 +2760,7 @@ test "production scope admits tracked untracked hidden and direct directory cand try std.testing.expect(!containsCandidate(candidates, "ignored-dir", .directory)); try std.testing.expect(containsCandidate(candidates, "empty-dir", .directory)); for (candidates) |candidate| { - try std.testing.expect(!workspace_files.pathContainsIgnoredDir(&.{".git"}, candidate.path)); + try std.testing.expect(!hasGitComponent(candidate.path)); } var index = FileIndex{}; @@ -2761,10 +2789,11 @@ test "scope discovery deduplicates overlapping roots and tolerates one failed ro const missing = try std.fs.path.join(alloc, &.{ primary, "missing" }); defer alloc.free(missing); const roots = [_][]const u8{ primary, nested, missing }; - var stop_requested = std.atomic.Value(bool).init(false); var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - const candidates = try discoverScopeCandidatesWithFallback(arena_state.allocator(), &roots, &stop_requested, true); + var scope = try TestScope.init(arena_state.allocator(), &roots); + defer scope.deinit(); + const candidates = scope.candidates; var index = FileIndex{}; defer index.deinit(alloc); @@ -3314,7 +3343,8 @@ test "initial allocation failure reports failure and a later load retries" { try std.testing.expectEqual(GenerationState.ready, loading.currentState()); try std.testing.expect(index.joinThreadIfDone(alloc)); try std.testing.expectEqual(State.ready, index.currentState()); - try std.testing.expectEqual(@as(usize, 0), index.count()); + try std.testing.expectEqual(@as(usize, 1), index.count()); + try std.testing.expectEqualStrings("retry.txt", index.pathAt(0)); } test "latest and identical refresh roots coalesce without a second loader" { @@ -3405,7 +3435,8 @@ test "failed generation starts one queued refresh after reap" { try std.testing.expectEqual(GenerationState.ready, queued.currentState()); try std.testing.expect(index.joinThreadIfDone(alloc)); try std.testing.expect(index.active_generation.? != previous_active); - try std.testing.expectEqual(@as(usize, 0), index.count()); + try std.testing.expectEqual(@as(usize, 1), index.count()); + try std.testing.expectEqualStrings("queued.txt", index.pathAt(0)); } test "stop before and after completion suppresses queued work" { diff --git a/src/core/workspace/file_index_cache.zig b/src/core/workspace/file_index_cache.zig deleted file mode 100644 index 23807d2ca..000000000 --- a/src/core/workspace/file_index_cache.zig +++ /dev/null @@ -1,246 +0,0 @@ -//! Persisted workspace file index for instant @-completion on launch. -//! -//! One file per workspace scope under `/.fx/file-index/.idx`: -//! magic + SHA-256 of the payload + JSON payload listing the scope roots and -//! every indexed path with its kind. Freshness is advisory only: a background -//! rescan always follows a cache load and replaces it, and the index is a -//! completion aid, never a correctness boundary. - -const std = @import("std"); -const io_mod = @import("../shared/io.zig"); -const debug_trace = @import("../shared/debug_trace.zig"); -const text_utils = @import("../shared/text_utils.zig"); -const file_index = @import("file_index.zig"); - -const Allocator = std.mem.Allocator; -const Sha256 = std.crypto.hash.sha2.Sha256; - -const magic = "fx-file-index-v1\n"; -pub const max_bytes = 64 * 1024 * 1024; - -const Candidate = file_index.Candidate; - -const Payload = struct { - written_at_ms: i64, - roots: []const []const u8, - entries: []const Entry, - - const Entry = struct { - path: []const u8, - kind: u8, - }; -}; - -pub const Loaded = struct { - written_at_ms: i64, - /// Owned candidate slice; each path is owned by the same allocator. - candidates: []Candidate, - - pub fn deinit(self: *Loaded, alloc: Allocator) void { - for (self.candidates) |candidate| alloc.free(candidate.path); - alloc.free(self.candidates); - self.* = undefined; - } -}; - -fn cacheKeyHex(alloc: Allocator, roots: []const []const u8) ![]u8 { - var digest = Sha256.init(.{}); - for (roots) |root| { - digest.update(root); - digest.update(&.{0}); - } - var sum: [Sha256.digest_length]u8 = undefined; - digest.final(&sum); - const hex = std.fmt.bytesToHex(sum, .lower); - return try alloc.dupe(u8, &hex); -} - -fn cachePath(alloc: Allocator, home: []const u8, roots: []const []const u8) ![]u8 { - const key = try cacheKeyHex(alloc, roots); - defer alloc.free(key); - return try std.fmt.allocPrint(alloc, "{s}/.fx/file-index/{s}.idx", .{ home, key }); -} - -/// Loads the persisted index for `roots` under `$HOME`, or null when absent, -/// unreadable, or invalid. All validation failures degrade to a rescan. -pub fn load(alloc: Allocator, roots: []const []const u8) !?Loaded { - const home = io_mod.getenv("HOME") orelse return null; - return loadFrom(alloc, home, roots); -} - -/// Loads from an explicit home directory (tests and non-HOME callers). -pub fn loadFrom(alloc: Allocator, home: []const u8, roots: []const []const u8) !?Loaded { - const path = try cachePath(alloc, home, roots); - defer alloc.free(path); - return loadPath(alloc, path, roots) catch |err| switch (err) { - error.OutOfMemory => return err, - error.FileNotFound => null, - else => blk: { - debug_trace.logf("core", "file index cache ignored err={s}", .{@errorName(err)}); - break :blk null; - }, - }; -} - -fn loadPath(alloc: Allocator, path: []const u8, roots: []const []const u8) !?Loaded { - const zio = io_mod.getIo(); - var file = try std.Io.Dir.openFileAbsolute(zio, path, .{ .follow_symlinks = false, .allow_directory = false }); - defer file.close(zio); - const stat = try file.stat(zio); - if (stat.kind != .file or stat.nlink != 1 or (stat.permissions.toMode() & 0o077) != 0 or stat.size > max_bytes) return error.InvalidIndexCache; - const bytes = try alloc.alloc(u8, @intCast(stat.size)); - defer alloc.free(bytes); - var offset: usize = 0; - while (offset < bytes.len) { - const end = @min(bytes.len, offset + 64 * 1024); - const read = try file.readPositionalAll(zio, bytes[offset..end], offset); - if (read != end - offset) return error.InvalidIndexCache; - offset = end; - } - if (bytes.len < magic.len + Sha256.digest_length or !std.mem.startsWith(u8, bytes, magic)) return error.InvalidIndexCache; - const payload = bytes[magic.len + Sha256.digest_length ..]; - var digest: [Sha256.digest_length]u8 = undefined; - Sha256.hash(payload, &digest, .{}); - if (!std.mem.eql(u8, &digest, bytes[magic.len..][0..Sha256.digest_length])) return error.InvalidIndexCache; - var parsed = try std.json.parseFromSlice(Payload, alloc, payload, .{ .allocate = .alloc_if_needed, .ignore_unknown_fields = false, .max_value_len = max_bytes }); - defer parsed.deinit(); - const value = parsed.value; - if (value.written_at_ms < 0 or value.roots.len == 0) return error.InvalidIndexCache; - if (value.entries.len > file_index.max_indexed_files) return error.InvalidIndexCache; - if (value.roots.len != roots.len) return error.InvalidIndexCache; - for (value.roots, roots) |cached, expected| { - if (!std.mem.eql(u8, cached, expected)) return error.InvalidIndexCache; - } - var candidates = try std.ArrayList(Candidate).initCapacity(alloc, value.entries.len); - errdefer { - for (candidates.items) |candidate| alloc.free(candidate.path); - candidates.deinit(alloc); - } - for (value.entries) |entry| { - if (entry.path.len == 0 or entry.path.len > file_index.max_path_len) return error.InvalidIndexCache; - if (!text_utils.isTerminalSafe(entry.path)) return error.InvalidIndexCache; - const kind: file_index.CandidateKind = switch (entry.kind) { - 0 => .file, - 1 => .directory, - else => return error.InvalidIndexCache, - }; - candidates.appendAssumeCapacity(.{ - .path = try alloc.dupe(u8, entry.path), - .kind = kind, - }); - } - return .{ - .written_at_ms = value.written_at_ms, - .candidates = try candidates.toOwnedSlice(alloc), - }; -} - -/// Persists the index for `roots` under `$HOME`. Best-effort by contract: -/// callers log and continue on failure because the in-memory index is -/// already complete. -pub fn save(alloc: Allocator, roots: []const []const u8, candidates: []const Candidate) !void { - const home = io_mod.getenv("HOME") orelse return; - return saveTo(alloc, home, roots, candidates); -} - -/// Saves under an explicit home directory (tests and non-HOME callers). -pub fn saveTo(alloc: Allocator, home: []const u8, roots: []const []const u8, candidates: []const Candidate) !void { - if (roots.len == 0) return; - const path = try cachePath(alloc, home, roots); - defer alloc.free(path); - if (candidates.len == 0) { - // An empty index paints nothing, so never create one. If a prior scan - // persisted entries and the workspace is now empty, drop the stale - // file instead of serving ghosts. - std.Io.Dir.deleteFileAbsolute(io_mod.getIo(), path) catch |err| switch (err) { - error.FileNotFound => {}, - else => return err, - }; - return; - } - const zio = io_mod.getIo(); - const dir_path = std.fs.path.dirname(path) orelse return; - try io_mod.makeDirRecursive(dir_path); - - var payload: std.Io.Writer.Allocating = .init(alloc); - defer payload.deinit(); - const writer = &payload.writer; - try writer.writeAll("{\"written_at_ms\":"); - try writer.print("{d}", .{io_mod.milliTimestamp()}); - try writer.writeAll(",\"roots\":"); - try std.json.Stringify.value(roots, .{}, writer); - try writer.writeAll(",\"entries\":["); - var written: usize = 0; - for (candidates) |candidate| { - if (candidate.path.len == 0 or candidate.path.len > file_index.max_path_len) continue; - if (!text_utils.isTerminalSafe(candidate.path)) continue; - if (written > 0) try writer.writeByte(','); - try writer.writeAll("{\"path\":"); - try std.json.Stringify.value(candidate.path, .{}, writer); - try writer.print(",\"kind\":{d}}}", .{@intFromEnum(candidate.kind)}); - written += 1; - if (payload.written().len > max_bytes) return error.IndexCacheTooLarge; - } - try writer.writeAll("]}"); - - var out: std.Io.Writer.Allocating = .init(alloc); - defer out.deinit(); - try out.writer.writeAll(magic); - var digest: [Sha256.digest_length]u8 = undefined; - Sha256.hash(payload.written(), &digest, .{}); - try out.writer.writeAll(&digest); - try out.writer.writeAll(payload.written()); - - // `.iterate` matters on Linux: without it the descriptor is O_PATH and - // the directory fsync inside durableReplaceVerified fails with EBADF. - var dir = try std.Io.Dir.openDirAbsolute(zio, dir_path, .{ .follow_symlinks = false, .iterate = true }); - defer dir.close(zio); - var verified: io_mod.VerifiedDir = .{ .dir = dir }; - try io_mod.durableReplaceVerified(alloc, &verified, std.fs.path.basename(path), out.written()); - try dir.setPermissions(zio, .fromMode(0o700)); -} - -test "file index cache round trips and rejects tampering" { - const alloc = std.testing.allocator; - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - const home = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); - defer alloc.free(home); - const roots = [_][]const u8{"/workspace"}; - const candidates = [_]Candidate{ - .{ .path = "src/main.zig", .kind = .file }, - .{ .path = "docs", .kind = .directory }, - }; - - try std.testing.expect((try loadFrom(alloc, home, &roots)) == null); - try saveTo(alloc, home, &roots, &candidates); - var loaded = (try loadFrom(alloc, home, &roots)).?; - defer loaded.deinit(alloc); - try std.testing.expectEqual(@as(usize, 2), loaded.candidates.len); - try std.testing.expectEqualStrings("src/main.zig", loaded.candidates[0].path); - try std.testing.expectEqual(.directory, loaded.candidates[1].kind); - - // A different scope must never reuse this file. - const other_roots = [_][]const u8{"/elsewhere"}; - try std.testing.expect((try loadFrom(alloc, home, &other_roots)) == null); - - // Bit flips break the digest and degrade to a rescan. - const path = try cachePath(alloc, home, &roots); - defer alloc.free(path); - var file = try std.Io.Dir.openFileAbsolute(std.testing.io, path, .{ .mode = .read_write }); - try file.writePositionalAll(std.testing.io, "X", magic.len + Sha256.digest_length + 4); - file.close(std.testing.io); - try std.testing.expect((try loadFrom(alloc, home, &roots)) == null); - - // Unsafe entries never reach disk: a save of only-unsafe paths produces a - // valid but empty index, and the next load filters nothing further. - const evil = [_]Candidate{.{ .path = "bad\x1bpath", .kind = .file }}; - try saveTo(alloc, home, &roots, &evil); - var empty = (try loadFrom(alloc, home, &roots)).?; - defer empty.deinit(alloc); - try std.testing.expectEqual(@as(usize, 0), empty.candidates.len); - - // An empty scan deletes the persisted index rather than serving ghosts. - try saveTo(alloc, home, &roots, &.{}); - try std.testing.expect((try loadFrom(alloc, home, &roots)) == null); -} diff --git a/tests/e2e/tui-file-picker.test.ts b/tests/e2e/tui-file-picker.test.ts index 4fc01995b..1a3bf6158 100644 --- a/tests/e2e/tui-file-picker.test.ts +++ b/tests/e2e/tui-file-picker.test.ts @@ -9,6 +9,7 @@ import { realpathSync, rmSync, unlinkSync, + utimesSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -359,7 +360,7 @@ function expectFileDiscoveryOffMainThread(trace: string): void { ); const discoveryThreads = traceThreadIds( trace, - /workspace file discovery process spawning thread=(\d+)/g, + /file index root (?:scanned|reused) root=\d+ .*thread=(\d+)/g, ); const mainThreads = new Set(callerThreads); @@ -679,7 +680,14 @@ describe("@ file picker", () => { } })(); + // A refresh of an unchanged tree reuses the saved listing in a few + // milliseconds. Touching the git index, as `git add` would, makes each + // refresh rescan every file without changing the result, so frames can + // observe a replacement in flight. + const gitIndexPath = join(current.workspace, ".git", "index"); const injectCycle = () => { + const now = new Date(); + utimesSync(gitIndexPath, now, now); active.sendKeysImmediate(["Escape", "C-u"]); active.sendLiteralImmediate(`${LIFECYCLE_QUERY.slice(0, -1)}`); active.sendLiteralImmediate(LIFECYCLE_QUERY.slice(-1)); @@ -838,8 +846,15 @@ describe("@ file picker", () => { for (const snapshot of activeQueryFrames) { expect(snapshot.pane).not.toContain("indexing files..."); } + // A replacement finishes in tens of milliseconds, close to one frame + // capture, so a frame inside it is luck. What must hold is that + // replacements began while the query was on screen and every frame + // around them kept the completed result. expect( - activeQueryFrames.some((snapshot) => replacementIncomplete(snapshot.trace)), + activeQueryFrames.some((snapshot) => + traceGenerationNumbers(snapshot.trace, "file index generation started") + .some((generation) => generation >= firstReplacementGeneration) + ), ).toBe(true); let acceptedFilePrompt = ""; From becf3256df533ed1573819aff2c501aae4bcc05f Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 8 Oct 2026 16:31:53 -0400 Subject: [PATCH 06/23] Search workspace files without git - Run glob_files and grep_files on the native indexer's listing, so a search starts no git process and sees files created a moment earlier, including untracked files in subfolders. - Reuse the saved @-completion listing for a search of the workspace or one of its ordinary folders while it is current, and scan the search root otherwise. - Search file contents on several threads, with the same results and the same cut at the match cap as a single-threaded search. - Keep searching hidden folders inside a repository, and skip them elsewhere unless the search path or pattern names one. - Reuse a listing cut at the entry cap while nothing it read has changed. - Check in CI that @-completion, glob_files and grep_files never start a process. --- AGENTS.md | 4 +- scripts/check-indexer-boundary.sh | 22 + src/core/indexer/indexer.zig | 1 + src/core/indexer/scan.zig | 7 +- src/core/indexer/snapshot.zig | 26 +- src/core/indexer/tree.zig | 10 +- src/core/workspace/grep_search.zig | 740 +++++++---------- src/core/workspace/ignored_dirs.zig | 51 -- src/core/workspace/tool_files.zig | 185 +++++ src/core/workspace/workspace_files.zig | 1060 ------------------------ src/tools/filesystem/glob_files.zig | 181 +--- src/tools/filesystem/grep_files.zig | 7 +- 12 files changed, 575 insertions(+), 1719 deletions(-) delete mode 100644 src/core/workspace/ignored_dirs.zig create mode 100644 src/core/workspace/tool_files.zig delete mode 100644 src/core/workspace/workspace_files.zig diff --git a/AGENTS.md b/AGENTS.md index 21ccd3473..5f90915bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -69,7 +69,9 @@ Key rules: * `src/core/compactor/` owns context compaction. Outside code imports only its front door, `compactor.zig`, and the compactor imports only shared basics and model configuration; its caller hands in the model caller, the record store, and the session's projection of saved turns into chat messages. `scripts/check-compactor-boundary.sh` enforces this in CI. -* `src/core/indexer/` is the native workspace indexer. It reads git's ignore rules, repository layout, and git config as data and never starts a process, so a repository's configuration cannot make it run a program. Outside code imports only its front door, `indexer.zig`, and the indexer imports only the standard library, its own files, `io.zig`, and `debug_trace.zig`. `scripts/check-indexer-boundary.sh` enforces this in CI. +* `src/core/indexer/` is the native workspace indexer. It reads git's ignore rules, repository layout, and git config as data and never starts a process, so a repository's configuration cannot make it run a program. Outside code imports only its front door, `indexer.zig`, and the indexer imports only the standard library, its own files, `io.zig`, and `debug_trace.zig`. `@`-completion, `glob_files`, and `grep_files` list files only through it and never start a process. `scripts/check-indexer-boundary.sh` enforces this in CI. + +* `src/core/shell_command/safe_git.zig` is the only way fx runs git itself, for the agent's auto-run read-only git commands and the `fx pr` and `fx issue` snapshot. It turns off every program a repository's own config can name and refuses to run when git would use a different repository than the one fx read. * `src/tools/` owns built-in tool implementations. Generic tool contracts and dispatch live in `src/core/tooling/`. Default tool specs are centralized in `src/core/tooling/tool_specs.zig` or `src/builtins/tools.zig`, not in individual tool files. diff --git a/scripts/check-indexer-boundary.sh b/scripts/check-indexer-boundary.sh index 5d3fcf4b5..c4c763f82 100755 --- a/scripts/check-indexer-boundary.sh +++ b/scripts/check-indexer-boundary.sh @@ -4,6 +4,9 @@ # imports only the standard library, its own files, and fx's shared io.zig and # debug_trace.zig, and it never starts a process: it reads repository metadata # as data, so a repository's configuration can never make it run a program. +# @-completion and the glob and grep tools list files only through it, so +# their production code, everything above each file's first test, never +# starts a process either. set -euo pipefail @@ -39,3 +42,22 @@ if [[ -n "$spawning" ]]; then printf 'The indexer must never start a process:\n%s\n' "$spawning" >&2 exit 1 fi + +listing_features=( + src/core/workspace/file_index.zig + src/core/workspace/tool_files.zig + src/core/workspace/grep_search.zig + src/tools/filesystem/glob_files.zig + src/tools/filesystem/grep_files.zig +) +feature_spawning="" +for file in "${listing_features[@]}"; do + found="$(awk '/^test "/ { exit } /std\.process\.(run|spawn|Child)|safe_git/ { print FILENAME ":" FNR ": " $0 }' "$file")" + if [[ -n "$found" ]]; then + feature_spawning+="$found"$'\n' + fi +done +if [[ -n "$feature_spawning" ]]; then + printf '@-completion, glob_files and grep_files must list files through the indexer and never start a process:\n%s' "$feature_spawning" >&2 + exit 1 +fi diff --git a/src/core/indexer/indexer.zig b/src/core/indexer/indexer.zig index 7e78439c7..0f4f039ee 100644 --- a/src/core/indexer/indexer.zig +++ b/src/core/indexer/indexer.zig @@ -29,6 +29,7 @@ pub const default_skipped_names = scan_mod.default_skipped_names; pub const scan = scan_mod.scan; pub const Tree = tree_mod.Tree; +pub const FolderStamp = tree_mod.FolderStamp; /// Whether a tree from `scan` or a snapshot still describes its root under /// the same skipped names, checked through its folder and source stamps /// without reading any folder. With a subtree, only folders at or below it diff --git a/src/core/indexer/scan.zig b/src/core/indexer/scan.zig index 96ffc81d3..6b6492d79 100644 --- a/src/core/indexer/scan.zig +++ b/src/core/indexer/scan.zig @@ -253,12 +253,12 @@ pub fn scan(alloc: Allocator, root: []const u8, options: Options, stop: ?*std.at cap_reached = true; } if (shared.limit_reached.load(.acquire)) cap_reached = true; - incomplete = incomplete or cap_reached or shared.incomplete.load(.acquire); + incomplete = incomplete or shared.limit_reached.load(.acquire) or shared.incomplete.load(.acquire); var folders: std.ArrayList(tree_mod.FolderStamp) = .empty; for (listings) |listing| { const stamp = listing.stamp orelse continue; - try folders.append(arena, .{ .rel = try arena.dupe(u8, stamp.rel), .inode = stamp.inode, .mtime_ns = stamp.mtime_ns, .ctime_ns = stamp.ctime_ns, .reusable = stamp.reusable }); + try folders.append(arena, .{ .rel = try arena.dupe(u8, stamp.rel), .inode = stamp.inode, .mtime_ns = stamp.mtime_ns, .ctime_ns = stamp.ctime_ns, .reusable = stamp.reusable, .boundary = listing.boundary, .excluded = listing.excluded }); if (listing.ignore_file) |path| try rules.sources.append(arena, .{ .path = try arena.dupe(u8, path), .identity_only = false }); } const stamps = try arena.alloc(tree_mod.SourceStamp, rules.sources.items.len); @@ -765,7 +765,8 @@ test "scan caps, skips overlong paths, cancels and is the same with one worker o // Within the collection limit the cap keeps the sorted prefix. var capped = try scan(std.testing.allocator, root, .{ .candidate_cap = 200 }, null); defer capped.deinit(); - try std.testing.expect(capped.tree.incomplete and capped.tree.cap_reached); + // Below it the walk read everything, so only the cap was reached. + try std.testing.expect(!capped.tree.incomplete and capped.tree.cap_reached); try std.testing.expectEqual(@as(usize, 200), capped.tree.entries.len); for (capped.tree.entries, serial.tree.entries[0..200]) |a, b| try std.testing.expectEqualStrings(b.path, a.path); // Past it, reading stops early: still capped, sorted and incomplete. diff --git a/src/core/indexer/snapshot.zig b/src/core/indexer/snapshot.zig index b9c9e98c7..cbdd58298 100644 --- a/src/core/indexer/snapshot.zig +++ b/src/core/indexer/snapshot.zig @@ -152,7 +152,7 @@ fn encode(writer: *std.Io.Writer, trees: []const Tree) !void { try writeInt(writer, u64, folder.inode); try writeInt(writer, i64, folder.mtime_ns); try writeInt(writer, i64, folder.ctime_ns); - try writer.writeByte(@intFromBool(folder.reusable)); + try writer.writeByte(@as(u8, @intFromBool(folder.reusable)) | @as(u8, @intFromBool(folder.boundary)) << 1 | @as(u8, @intFromBool(folder.excluded)) << 2); } try writeInt(writer, u32, @intCast(tree.sources.len)); for (tree.sources) |source| { @@ -221,16 +221,19 @@ fn decode(arena: Allocator, bytes: []const u8, roots: []const []const u8) (Alloc const folders = try arena.alloc(tree_mod.FolderStamp, try decoder.count(29)); for (folders) |*folder| { const rel = try decoder.string(arena, max_path_bytes, true); + const inode = try decoder.int(u64); + const mtime_ns = try decoder.int(i64); + const ctime_ns = try decoder.int(i64); + const folder_flags = (try decoder.take(1))[0]; + if (folder_flags & ~@as(u8, 7) != 0) return error.InvalidSnapshot; folder.* = .{ .rel = rel, - .inode = try decoder.int(u64), - .mtime_ns = try decoder.int(i64), - .ctime_ns = try decoder.int(i64), - .reusable = switch ((try decoder.take(1))[0]) { - 0 => false, - 1 => true, - else => return error.InvalidSnapshot, - }, + .inode = inode, + .mtime_ns = mtime_ns, + .ctime_ns = ctime_ns, + .reusable = folder_flags & 1 != 0, + .boundary = folder_flags & 2 != 0, + .excluded = folder_flags & 4 != 0, }; } const stamps = try arena.alloc(tree_mod.SourceStamp, try decoder.count(37)); @@ -280,7 +283,7 @@ fn testTree(root: []const u8, entries: []const tree_mod.Entry) Tree { .entries = entries, .folders = &.{ .{ .rel = "", .inode = 7, .mtime_ns = 5, .ctime_ns = 6, .reusable = true }, - .{ .rel = "src", .inode = 9, .mtime_ns = 5, .ctime_ns = 900, .reusable = false }, + .{ .rel = "src", .inode = 9, .mtime_ns = 5, .ctime_ns = 900, .reusable = false, .boundary = true, .excluded = true }, }, .sources = &.{ .{ .path = "/w/.gitignore", .exists = true, .identity_only = false, .inode = 3, .size = 10, .mtime_ns = 4, .ctime_ns = 4, .reusable = true }, @@ -319,7 +322,8 @@ test "snapshot round trips, rejects tampering, other scopes and v1 files, and an try std.testing.expectEqualStrings("src", first.entries[1].path); try std.testing.expectEqual(tree_mod.Kind.directory, first.entries[1].kind); try std.testing.expectEqualStrings("", first.folders[0].rel); - try std.testing.expect(!first.folders[1].reusable); + try std.testing.expect(!first.folders[1].reusable and first.folders[1].boundary and first.folders[1].excluded); + try std.testing.expect(!first.folders[0].boundary and !first.folders[0].excluded); try std.testing.expect(first.sources[1].identity_only and !first.sources[2].exists); try std.testing.expectEqualStrings("b.txt", loaded.trees[1].entries[0].path); } diff --git a/src/core/indexer/tree.zig b/src/core/indexer/tree.zig index 4f0d36c79..c520ebba0 100644 --- a/src/core/indexer/tree.zig +++ b/src/core/indexer/tree.zig @@ -30,6 +30,10 @@ pub const FolderStamp = struct { mtime_ns: i64, ctime_ns: i64, reusable: bool, + /// A folder with its own repository, listed but not entered. + boundary: bool = false, + /// Ignored, so only its tracked files are listed. + excluded: bool = false, }; pub const SourceStamp = struct { @@ -51,9 +55,11 @@ pub const Tree = struct { root_inode: u64, scan_started_ns: i64, repository: bool, - /// The cap, an unreadable source or an unreadable folder left something - /// out. An incomplete tree is never reused. + /// An unreadable source or folder, or a walk stopped at its reading limit, + /// left something out. An incomplete tree is never reused. incomplete: bool, + /// The entries were cut to the cap. The walk still read everything, so for + /// unchanged stamps the cut is the same and the tree can be reused. cap_reached: bool, skipped_overlong: usize, skipped_names: []const []const u8, diff --git a/src/core/workspace/grep_search.zig b/src/core/workspace/grep_search.zig index 38334371f..2263e66c2 100644 --- a/src/core/workspace/grep_search.zig +++ b/src/core/workspace/grep_search.zig @@ -2,11 +2,10 @@ const std = @import("std"); const builtin = @import("builtin"); const debug_trace = @import("../shared/debug_trace.zig"); const glob_pattern = @import("glob_pattern.zig"); -const ignored_dirs = @import("ignored_dirs.zig"); const io_mod = @import("../shared/io.zig"); const pathing = @import("pathing.zig"); const text_utils = @import("../shared/text_utils.zig"); -const workspace_files = @import("workspace_files.zig"); +const tool_files = @import("tool_files.zig"); const Allocator = std.mem.Allocator; @@ -17,7 +16,6 @@ pub const output_cap: usize = 200; pub const collection_cap: usize = 2000; const file_byte_cap: usize = 50 * 1024 * 4; -const git_grep_stdout_limit: usize = 8 * 1024 * 1024; /// Filesystem match collected by grep search. pub const Match = struct { @@ -36,7 +34,7 @@ pub const Result = struct { matches: []const Match, truncated_reason: ?TruncatedReason = null, candidate_count: usize = 0, - candidate_cap: usize = workspace_files.default_candidate_cap, + candidate_cap: usize = tool_files.default_candidate_cap, candidate_incomplete: bool = false, skipped_overlong: usize = 0, }; @@ -46,13 +44,17 @@ pub const CountResult = struct { matching_lines: usize = 0, matching_files: usize = 0, candidate_count: usize = 0, - candidate_cap: usize = workspace_files.default_candidate_cap, + candidate_cap: usize = tool_files.default_candidate_cap, candidate_incomplete: bool = false, skipped_overlong: usize = 0, }; -/// Collects literal line matches below a directory root. Returned slices are owned by arena. +/// Collects literal line matches below a directory root. Returned slices are +/// owned by `arena`, which only the calling thread uses. `scratch` must be +/// thread-safe; it backs the listing and the content scan and is released +/// before return, so retained memory grows with matches, not scanned bytes. pub fn collectDirectoryMatches( + scratch: Allocator, arena: Allocator, workspace_root: []const u8, absolute_root: []const u8, @@ -60,10 +62,11 @@ pub fn collectDirectoryMatches( case_insensitive: bool, include: ?glob_pattern.Pattern, ) !Result { - return collectDirectoryMatchesWithIgnored(arena, workspace_root, absolute_root, pattern, case_insensitive, ignored_dirs.ignored_directory_names, include); + return collectDirectoryMatchesWithIgnored(scratch, arena, workspace_root, absolute_root, pattern, case_insensitive, tool_files.default_skipped_names, include); } pub fn collectDirectoryMatchesWithIgnored( + scratch: Allocator, arena: Allocator, workspace_root: []const u8, absolute_root: []const u8, @@ -72,10 +75,11 @@ pub fn collectDirectoryMatchesWithIgnored( ignored_names: []const []const u8, include: ?glob_pattern.Pattern, ) !Result { - return collectDirectoryMatchesWithOptions(arena, workspace_root, absolute_root, pattern, case_insensitive, ignored_names, include, .{}); + return collectDirectoryMatchesWithOptions(scratch, arena, workspace_root, absolute_root, pattern, case_insensitive, ignored_names, include, .{}); } pub fn countDirectoryMatchesWithIgnored( + scratch: Allocator, arena: Allocator, workspace_root: []const u8, absolute_root: []const u8, @@ -84,10 +88,11 @@ pub fn countDirectoryMatchesWithIgnored( ignored_names: []const []const u8, include: ?glob_pattern.Pattern, ) !CountResult { - return countDirectoryMatchesWithOptions(arena, workspace_root, absolute_root, pattern, case_insensitive, ignored_names, include, .{}); + return countDirectoryMatchesWithOptions(scratch, arena, workspace_root, absolute_root, pattern, case_insensitive, ignored_names, include, .{}); } fn collectDirectoryMatchesWithOptions( + scratch: Allocator, arena: Allocator, workspace_root: []const u8, absolute_root: []const u8, @@ -95,50 +100,17 @@ fn collectDirectoryMatchesWithOptions( case_insensitive: bool, ignored_names: []const []const u8, include: ?glob_pattern.Pattern, - workspace_options: workspace_files.Options, + list_options: tool_files.Options, ) !Result { + const listing = try listCandidates(scratch, arena, workspace_root, absolute_root, ignored_names, list_options); var matches: std.ArrayList(Match) = .empty; errdefer matches.deinit(arena); - - var truncated_reason: ?TruncatedReason = null; - var stats: CandidateStats = .{ .cap = workspace_options.candidate_cap }; - - if (!workspace_options.force_fallback and !gitIgnoresRoot(arena, absolute_root)) git: { - const tracked = gitGrepTrackedMatches(arena, workspace_root, absolute_root, pattern, case_insensitive, include, &matches) catch |err| { - if (err == error.OutOfMemory) return error.OutOfMemory; - debug_trace.logf("core", "grep_files git grep fallback root={s} err={s}", .{ absolute_root, @errorName(err) }); - break :git; - }; - stats.count += tracked.candidate_count; - truncated_reason = tracked.truncated_reason; - - if (truncated_reason == null) { - const untracked = workspaceFileCandidates(arena, absolute_root, ignored_names, .{ - .candidate_cap = workspace_options.candidate_cap, - .git_stdout_limit = workspace_options.git_stdout_limit, - .only_untracked = true, - }) catch |err| { - if (err == error.OutOfMemory) return error.OutOfMemory; - debug_trace.logf("core", "grep_files untracked fallback skipped root={s} err={s}", .{ absolute_root, @errorName(err) }); - return finishMatches(arena, &matches, truncated_reason, stats); - }; - stats.add(untracked.result); - truncated_reason = try scanCandidateList(arena, workspace_root, untracked.provider_root, untracked.result.files, pattern, case_insensitive, include, &matches); - } - - return finishMatches(arena, &matches, truncated_reason, stats); - } - - var discovery_options = workspace_options; - discovery_options.include_untracked = true; - const candidates = try workspaceFileCandidates(arena, absolute_root, ignored_names, discovery_options); - stats.add(candidates.result); - truncated_reason = try scanCandidateList(arena, workspace_root, candidates.provider_root, candidates.result.files, pattern, case_insensitive, include, &matches); - - return finishMatches(arena, &matches, truncated_reason, stats); + const truncated_reason = try scanCandidateList(scratch, arena, workspace_root, absolute_root, listing.files, pattern, case_insensitive, include, &matches); + return finishMatches(arena, &matches, truncated_reason, .fromListing(listing)); } fn countDirectoryMatchesWithOptions( + scratch: Allocator, arena: Allocator, workspace_root: []const u8, absolute_root: []const u8, @@ -146,60 +118,26 @@ fn countDirectoryMatchesWithOptions( case_insensitive: bool, ignored_names: []const []const u8, include: ?glob_pattern.Pattern, - workspace_options: workspace_files.Options, + list_options: tool_files.Options, ) !CountResult { - var count_result: CountResult = .{ .candidate_cap = workspace_options.candidate_cap }; - var stats: CandidateStats = .{ .cap = workspace_options.candidate_cap }; - - if (!workspace_options.force_fallback and !gitIgnoresRoot(arena, absolute_root)) git: { - const tracked = gitGrepTrackedCounts(arena, workspace_root, absolute_root, pattern, case_insensitive, include) catch |err| { - if (err == error.OutOfMemory) return error.OutOfMemory; - debug_trace.logf("core", "grep_files git grep count fallback root={s} err={s}", .{ absolute_root, @errorName(err) }); - break :git; - }; - count_result.matching_lines += tracked.matching_lines; - count_result.matching_files += tracked.matching_files; - stats.count += tracked.candidate_count; - - const untracked = workspaceFileCandidates(arena, absolute_root, ignored_names, .{ - .candidate_cap = workspace_options.candidate_cap, - .git_stdout_limit = workspace_options.git_stdout_limit, - .only_untracked = true, - }) catch |err| { - if (err == error.OutOfMemory) return error.OutOfMemory; - debug_trace.logf("core", "grep_files untracked count fallback skipped root={s} err={s}", .{ absolute_root, @errorName(err) }); - return finishCount(count_result, stats); - }; - stats.add(untracked.result); - try countCandidateList(arena, workspace_root, untracked.provider_root, untracked.result.files, pattern, case_insensitive, include, &count_result); - - return finishCount(count_result, stats); - } - - var discovery_options = workspace_options; - discovery_options.include_untracked = true; - const candidates = try workspaceFileCandidates(arena, absolute_root, ignored_names, discovery_options); - stats.add(candidates.result); - try countCandidateList(arena, workspace_root, candidates.provider_root, candidates.result.files, pattern, case_insensitive, include, &count_result); - - return finishCount(count_result, stats); + const listing = try listCandidates(scratch, arena, workspace_root, absolute_root, ignored_names, list_options); + var count_result: CountResult = .{ .candidate_cap = listing.candidate_cap }; + try countCandidateList(scratch, workspace_root, absolute_root, listing.files, pattern, case_insensitive, include, &count_result); + return finishCount(count_result, .fromListing(listing)); } -fn gitIgnoresRoot(arena: Allocator, absolute_root: []const u8) bool { - const argv = [_][]const u8{ "git", "--no-optional-locks", "check-ignore", "-q", "--", "." }; - const result = std.process.run(arena, io_mod.getIo(), .{ - .argv = &argv, - .cwd = .{ .path = absolute_root }, - .stdout_limit = std.Io.Limit.limited(1024), - .stderr_limit = std.Io.Limit.limited(1024), - }) catch return false; - defer arena.free(result.stdout); - defer arena.free(result.stderr); - - return switch (result.term) { - .exited => |code| code == 0, - else => false, - }; +fn listCandidates( + scratch: Allocator, + arena: Allocator, + workspace_root: []const u8, + absolute_root: []const u8, + ignored_names: []const []const u8, + list_options: tool_files.Options, +) !tool_files.Listing { + var options = list_options; + options.skipped_names = ignored_names; + options.include_hidden_in_repository = true; + return tool_files.list(scratch, arena, workspace_root, absolute_root, options); } /// Collects literal line matches from a single regular-file root. @@ -218,7 +156,7 @@ pub fn collectRegularFileRoot( if (include) |include_pattern| { if (!include_pattern.matchesBasename(std.fs.path.basename(absolute_path))) { return finishMatches(arena, &matches, null, .{ - .cap = workspace_files.default_candidate_cap, + .cap = tool_files.default_candidate_cap, }); } } @@ -227,7 +165,7 @@ pub fn collectRegularFileRoot( return finishMatches(arena, &matches, truncated_reason, .{ .count = 1, - .cap = workspace_files.default_candidate_cap, + .cap = tool_files.default_candidate_cap, }); } @@ -243,7 +181,7 @@ pub fn countRegularFileRoot( _ = workspace_root; if (include) |include_pattern| { if (!include_pattern.matchesBasename(std.fs.path.basename(absolute_path))) { - return finishCount(.{}, .{ .cap = workspace_files.default_candidate_cap }); + return finishCount(.{}, .{ .cap = tool_files.default_candidate_cap }); } } @@ -253,21 +191,23 @@ pub fn countRegularFileRoot( .matching_files = if (file_count.matching_lines > 0) 1 else 0, }, .{ .count = 1, - .cap = workspace_files.default_candidate_cap, + .cap = tool_files.default_candidate_cap, }); } const CandidateStats = struct { count: usize = 0, - cap: usize = workspace_files.default_candidate_cap, + cap: usize = tool_files.default_candidate_cap, incomplete: bool = false, skipped_overlong: usize = 0, - fn add(self: *CandidateStats, result: workspace_files.Result) void { - self.count += result.files.len; - self.cap = result.candidate_cap; - self.incomplete = self.incomplete or result.incomplete; - self.skipped_overlong += result.skipped_overlong; + fn fromListing(listing: tool_files.Listing) CandidateStats { + return .{ + .count = listing.files.len, + .cap = listing.candidate_cap, + .incomplete = listing.incomplete, + .skipped_overlong = listing.skipped_overlong, + }; } }; @@ -291,26 +231,11 @@ fn finishCount(result: CountResult, stats: CandidateStats) CountResult { return out; } -const WorkspaceCandidates = struct { - result: workspace_files.Result, - provider_root: []const u8, -}; - const CandidateFile = struct { display_path: []const u8, read_path: []const u8, }; -fn workspaceFileCandidates(arena: Allocator, absolute_root: []const u8, ignored_names: []const []const u8, options: workspace_files.Options) !WorkspaceCandidates { - var discover_options = options; - discover_options.ignored_names = ignored_names; - - return .{ - .result = try workspace_files.discover(arena, absolute_root, discover_options), - .provider_root = absolute_root, - }; -} - fn candidateKind(absolute_path: []const u8) !std.Io.File.Kind { const stat = try std.Io.Dir.cwd().statFile(io_mod.getIo(), absolute_path, .{ .follow_symlinks = false }); return stat.kind; @@ -345,7 +270,159 @@ fn resolveCandidateFile( }; } +/// Files per unit of parallel work. Workers take batches in candidate order. +const batch_files: usize = 16; +/// Fewer selected files than this are scanned on the calling thread. +const parallel_min_files: usize = 64; +const max_scan_workers: usize = 8; + +const ScanMode = enum { matches, count }; + +/// One selected candidate's outcome, written only by the worker that owns its +/// batch and read only after every worker has joined. +const FileOutcome = struct { + matches: []const Match = &.{}, + matching_lines: usize = 0, +}; + +/// Shared state of one parallel content scan. Merging in candidate order makes +/// the output identical to a single-threaded scan, including where the +/// collection cap truncates it. +const ParallelScan = struct { + /// Thread-safe and temporary: candidate resolution and per-file matches. + work: Allocator, + workspace_root: []const u8, + provider_root: []const u8, + files: []const []const u8, + pattern: []const u8, + case_insensitive: bool, + mode: ScanMode, + outcomes: []FileOutcome, + next_batch: std.atomic.Value(usize) = .init(0), + stop: std.atomic.Value(bool) = .init(false), + mutex: std.Io.Mutex = .init, + /// Guarded by `mutex`: which batches are done, how many leading batches + /// are done, and how many matches those leading batches hold. + batch_done: []bool, + done_prefix: usize = 0, + prefix_matches: usize = 0, + out_of_memory: bool = false, + + fn run(self: *ParallelScan) void { + while (!self.stop.load(.acquire)) { + const batch = self.next_batch.fetchAdd(1, .monotonic); + const first = batch * batch_files; + if (first >= self.files.len) return; + const last = @min(first + batch_files, self.files.len); + for (first..last) |index| { + _ = self.scanOne(index) catch { + self.mutex.lockUncancelable(io_mod.getIo()); + self.out_of_memory = true; + self.mutex.unlock(io_mod.getIo()); + self.stop.store(true, .release); + return; + }; + } + self.finishBatch(batch); + } + } + + fn scanOne(self: *ParallelScan, index: usize) Allocator.Error!void { + var absolute_match_buf: [std.fs.max_path_bytes]u8 = undefined; + const candidate_file = try resolveCandidateFile(self.work, self.workspace_root, self.provider_root, self.files[index], absolute_match_buf[0..]) orelse return; + switch (self.mode) { + .matches => { + var local: std.ArrayList(Match) = .empty; + _ = scanFileAt(self.work, candidate_file.display_path, candidate_file.read_path, self.pattern, self.case_insensitive, &local) catch |err| { + if (err == error.OutOfMemory) return error.OutOfMemory; + debug_trace.logf("core", "grep_files scan skipped path={s} err={s}", .{ candidate_file.display_path, @errorName(err) }); + return; + }; + self.outcomes[index] = .{ .matches = local.items }; + }, + .count => { + const file_count = countFileAt(candidate_file.display_path, candidate_file.read_path, self.pattern, self.case_insensitive) catch |err| { + debug_trace.logf("core", "grep_files scan skipped path={s} err={s}", .{ candidate_file.display_path, @errorName(err) }); + return; + }; + self.outcomes[index] = .{ .matching_lines = file_count.matching_lines }; + }, + } + } + + /// Once the leading done batches hold the collection cap, no later file + /// can change the merged result, so the remaining work stops. + fn finishBatch(self: *ParallelScan, batch: usize) void { + self.mutex.lockUncancelable(io_mod.getIo()); + defer self.mutex.unlock(io_mod.getIo()); + self.batch_done[batch] = true; + while (self.done_prefix < self.batch_done.len and self.batch_done[self.done_prefix]) : (self.done_prefix += 1) { + const first = self.done_prefix * batch_files; + for (self.outcomes[first..@min(first + batch_files, self.outcomes.len)]) |outcome| self.prefix_matches += outcome.matches.len; + } + if (self.mode == .matches and self.prefix_matches >= collection_cap) self.stop.store(true, .release); + } +}; + +fn workerCount(files: usize) usize { + if (builtin.single_threaded or files < parallel_min_files) return 1; + const cpus = std.Thread.getCpuCount() catch 1; + return @max(1, @min(@min(max_scan_workers, cpus), std.math.divCeil(usize, files, batch_files) catch 1)); +} + +/// Scans `files` with up to `max_scan_workers` threads, the calling thread +/// included. A helper that cannot start leaves its share to the others. +fn runParallelScan(scan: *ParallelScan) Allocator.Error!void { + const helpers = workerCount(scan.files.len) - 1; + var threads: [max_scan_workers]std.Thread = undefined; + var started: usize = 0; + if (!builtin.single_threaded) { + while (started < helpers) : (started += 1) { + threads[started] = std.Thread.spawn(.{}, ParallelScan.run, .{scan}) catch break; + } + } + scan.run(); + for (threads[0..started]) |thread| thread.join(); + if (scan.out_of_memory) return error.OutOfMemory; +} + +fn selectCandidates(work: Allocator, candidates: []const []const u8, include: ?glob_pattern.Pattern) ![]const []const u8 { + const include_pattern = include orelse return candidates; + var selected: std.ArrayList([]const u8) = .empty; + for (candidates) |candidate| { + if (include_pattern.matchesPath(candidate)) try selected.append(work, candidate); + } + return selected.toOwnedSlice(work); +} + +fn initParallelScan( + work: Allocator, + workspace_root: []const u8, + provider_root: []const u8, + files: []const []const u8, + pattern: []const u8, + case_insensitive: bool, + mode: ScanMode, +) !ParallelScan { + const outcomes = try work.alloc(FileOutcome, files.len); + @memset(outcomes, .{}); + const batch_done = try work.alloc(bool, std.math.divCeil(usize, files.len, batch_files) catch 0); + @memset(batch_done, false); + return .{ + .work = work, + .workspace_root = workspace_root, + .provider_root = provider_root, + .files = files, + .pattern = pattern, + .case_insensitive = case_insensitive, + .mode = mode, + .outcomes = outcomes, + .batch_done = batch_done, + }; +} + fn scanCandidateList( + scratch: Allocator, arena: Allocator, workspace_root: []const u8, provider_root: []const u8, @@ -355,25 +432,25 @@ fn scanCandidateList( include: ?glob_pattern.Pattern, matches: *std.ArrayList(Match), ) !?TruncatedReason { - for (candidates) |candidate| { - if (include) |include_pattern| { - if (!include_pattern.matchesPath(candidate)) continue; + var work_state = std.heap.ArenaAllocator.init(scratch); + defer work_state.deinit(); + const work = work_state.allocator(); + const files = try selectCandidates(work, candidates, include); + var scan = try initParallelScan(work, workspace_root, provider_root, files, pattern, case_insensitive, .matches); + try runParallelScan(&scan); + for (scan.outcomes) |outcome| { + var retained_path: ?[]const u8 = null; + for (outcome.matches) |match| { + if (matches.items.len >= collection_cap) return .collection_cap; + try appendMatch(arena, matches, &retained_path, match.absolute_path, match.line_number, match.line); + if (matches.items.len >= collection_cap) return .collection_cap; } - - var absolute_match_buf: [std.fs.max_path_bytes]u8 = undefined; - const candidate_file = try resolveCandidateFile(arena, workspace_root, provider_root, candidate, absolute_match_buf[0..]) orelse continue; - const scan_result = scanFileAt(arena, candidate_file.display_path, candidate_file.read_path, pattern, case_insensitive, matches) catch |err| { - if (err == error.OutOfMemory) return error.OutOfMemory; - debug_trace.logf("core", "grep_files scan skipped path={s} err={s}", .{ candidate_file.display_path, @errorName(err) }); - continue; - }; - if (scan_result) |reason| return reason; } return null; } fn countCandidateList( - arena: Allocator, + scratch: Allocator, workspace_root: []const u8, provider_root: []const u8, candidates: []const []const u8, @@ -382,275 +459,23 @@ fn countCandidateList( include: ?glob_pattern.Pattern, count_result: *CountResult, ) !void { - for (candidates) |candidate| { - if (include) |include_pattern| { - if (!include_pattern.matchesPath(candidate)) continue; - } - - var absolute_match_buf: [std.fs.max_path_bytes]u8 = undefined; - const candidate_file = try resolveCandidateFile(arena, workspace_root, provider_root, candidate, absolute_match_buf[0..]) orelse continue; - const file_count = countFileAt(candidate_file.display_path, candidate_file.read_path, pattern, case_insensitive) catch |err| { - debug_trace.logf("core", "grep_files scan skipped path={s} err={s}", .{ candidate_file.display_path, @errorName(err) }); - continue; - }; - if (file_count.matching_lines == 0) continue; + var work_state = std.heap.ArenaAllocator.init(scratch); + defer work_state.deinit(); + const work = work_state.allocator(); + const files = try selectCandidates(work, candidates, include); + var scan = try initParallelScan(work, workspace_root, provider_root, files, pattern, case_insensitive, .count); + try runParallelScan(&scan); + for (scan.outcomes) |outcome| { + if (outcome.matching_lines == 0) continue; count_result.matching_files += 1; - count_result.matching_lines += file_count.matching_lines; + count_result.matching_lines += outcome.matching_lines; } } -pub fn gitGrepArgvForTest() [9][]const u8 { - return .{ "git", "--no-optional-locks", "grep", "-n", "-I", "-F", "-z", "-e", "needle" }; -} - -const GitGrepMatchesResult = struct { - candidate_count: usize = 0, - truncated_reason: ?TruncatedReason = null, -}; - const FileCount = struct { matching_lines: usize = 0, }; -fn gitGrepTrackedMatches( - arena: Allocator, - workspace_root: []const u8, - absolute_root: []const u8, - pattern: []const u8, - case_insensitive: bool, - include: ?glob_pattern.Pattern, - matches: *std.ArrayList(Match), -) !GitGrepMatchesResult { - if (pattern.len == 0) return error.GitGrepUnsupported; - - var argv: std.ArrayList([]const u8) = .empty; - defer argv.deinit(arena); - try argv.appendSlice(arena, &.{ "git", "--no-optional-locks", "grep", "-n", "-I", "-F", "-z" }); - if (case_insensitive) try argv.append(arena, "-i"); - try argv.appendSlice(arena, &.{ "-e", pattern, "--" }); - if (safeGitIncludePathspec(include)) |pathspec| { - try argv.append(arena, pathspec); - } else { - try argv.append(arena, "."); - } - - const result = std.process.run(arena, io_mod.getIo(), .{ - .argv = argv.items, - .cwd = .{ .path = absolute_root }, - .stdout_limit = std.Io.Limit.limited(git_grep_stdout_limit), - .stderr_limit = std.Io.Limit.limited(1024), - }) catch return error.GitGrepFailed; - defer arena.free(result.stderr); - - switch (result.term) { - .exited => |code| { - if (code == 0) return parseGitGrepMatches(arena, workspace_root, absolute_root, result.stdout, include, matches); - arena.free(result.stdout); - if (code == 1) return .{}; - return error.GitGrepFailed; - }, - else => { - arena.free(result.stdout); - return error.GitGrepFailed; - }, - } -} - -fn gitGrepTrackedCounts( - arena: Allocator, - workspace_root: []const u8, - absolute_root: []const u8, - pattern: []const u8, - case_insensitive: bool, - include: ?glob_pattern.Pattern, -) !CountResult { - if (pattern.len == 0) return error.GitGrepUnsupported; - - var argv: std.ArrayList([]const u8) = .empty; - defer argv.deinit(arena); - try argv.appendSlice(arena, &.{ "git", "--no-optional-locks", "grep", "--count", "-I", "-F", "-z" }); - if (case_insensitive) try argv.append(arena, "-i"); - try argv.appendSlice(arena, &.{ "-e", pattern, "--" }); - if (safeGitIncludePathspec(include)) |pathspec| { - try argv.append(arena, pathspec); - } else { - try argv.append(arena, "."); - } - - const result = std.process.run(arena, io_mod.getIo(), .{ - .argv = argv.items, - .cwd = .{ .path = absolute_root }, - .stdout_limit = std.Io.Limit.limited(git_grep_stdout_limit), - .stderr_limit = std.Io.Limit.limited(1024), - }) catch return error.GitGrepFailed; - defer arena.free(result.stderr); - - switch (result.term) { - .exited => |code| { - if (code == 0) return parseGitGrepCounts(arena, workspace_root, absolute_root, result.stdout, include); - arena.free(result.stdout); - if (code == 1) return .{}; - return error.GitGrepFailed; - }, - else => { - arena.free(result.stdout); - return error.GitGrepFailed; - }, - } -} - -fn parseGitGrepMatches( - arena: Allocator, - workspace_root: []const u8, - absolute_root: []const u8, - raw: []const u8, - include: ?glob_pattern.Pattern, - matches: *std.ArrayList(Match), -) !GitGrepMatchesResult { - var result: GitGrepMatchesResult = .{}; - var display_paths = std.StringHashMap([]const u8).init(arena); - defer display_paths.deinit(); - var skipped_paths = std.StringHashMap(void).init(arena); - defer skipped_paths.deinit(); - - var index: usize = 0; - while (index < raw.len) { - const path_end = findScalarFrom(raw, index, 0) orelse break; - const path = raw[index..path_end]; - const line_start = path_end + 1; - const line_end = findScalarFrom(raw, line_start, 0) orelse break; - const content_start = line_end + 1; - const content_end = findScalarFrom(raw, content_start, '\n') orelse raw.len; - index = if (content_end < raw.len) content_end + 1 else raw.len; - - if (path.len == 0 or path.len > workspace_files.max_relative_path_bytes) continue; - if (include) |include_pattern| { - if (!include_pattern.matchesPath(path)) continue; - } - if (skipped_paths.contains(path)) continue; - - const line_number = std.fmt.parseInt(usize, raw[line_start..line_end], 10) catch continue; - const line = raw[content_start..content_end]; - if (!text_utils.isModelSafeText(line)) { - debug_trace.logf("core", "grep_files skipped non-text git grep line root={s} path={s}", .{ absolute_root, path }); - try skipped_paths.put(path, {}); - continue; - } - - const display_path = display_paths.get(path) orelse blk: { - const absolute_match = try validateMatchedGitFile(arena, workspace_root, absolute_root, path) orelse { - try skipped_paths.put(path, {}); - continue; - }; - result.candidate_count += 1; - try display_paths.put(path, absolute_match); - break :blk absolute_match; - }; - - if (matches.items.len >= collection_cap) { - result.truncated_reason = .collection_cap; - break; - } - var retained_path: ?[]const u8 = null; - try appendMatch(arena, matches, &retained_path, display_path, line_number, line); - if (matches.items.len >= collection_cap) { - result.truncated_reason = .collection_cap; - break; - } - } - - return result; -} - -fn parseGitGrepCounts( - arena: Allocator, - workspace_root: []const u8, - absolute_root: []const u8, - raw: []const u8, - include: ?glob_pattern.Pattern, -) !CountResult { - var result: CountResult = .{}; - var skipped_paths = std.StringHashMap(void).init(arena); - defer skipped_paths.deinit(); - - var index: usize = 0; - while (index < raw.len) { - const path_end = findScalarFrom(raw, index, 0) orelse break; - const path = raw[index..path_end]; - const count_start = path_end + 1; - const count_end = findScalarFrom(raw, count_start, '\n') orelse raw.len; - index = if (count_end < raw.len) count_end + 1 else raw.len; - - if (path.len == 0 or path.len > workspace_files.max_relative_path_bytes) continue; - if (include) |include_pattern| { - if (!include_pattern.matchesPath(path)) continue; - } - if (skipped_paths.contains(path)) continue; - - const raw_count = std.mem.trimEnd(u8, raw[count_start..count_end], "\r"); - const line_count = std.fmt.parseInt(usize, raw_count, 10) catch continue; - if (line_count == 0) continue; - if ((try validateMatchedGitFile(arena, workspace_root, absolute_root, path)) == null) { - try skipped_paths.put(path, {}); - continue; - } - result.candidate_count += 1; - result.matching_files += 1; - result.matching_lines += line_count; - } - - return result; -} - -fn validateMatchedGitFile( - arena: Allocator, - workspace_root: []const u8, - absolute_root: []const u8, - path: []const u8, -) !?[]const u8 { - const absolute_match = std.fs.path.join(arena, &.{ absolute_root, path }) catch return error.OutOfMemory; - const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), absolute_match, .{ .follow_symlinks = false }) catch |err| { - debug_trace.logf("core", "grep_files scan skipped path={s} err={s}", .{ absolute_match, @errorName(err) }); - return null; - }; - if (stat.kind != .file and stat.kind != .sym_link) return null; - if (stat.size > file_byte_cap) { - logOversizedFile(absolute_match); - return null; - } - const resolved_match = if (stat.kind == .sym_link) - resolveDirectoryEntryTarget(arena, workspace_root, absolute_match, stat.kind) catch |err| { - if (err == error.OutOfMemory) return error.OutOfMemory; - debug_trace.logf("core", "grep_files scan skipped path={s} err={s}", .{ absolute_match, @errorName(err) }); - return null; - } orelse return null - else - absolute_match; - var content_buf: [file_byte_cap + 1]u8 = undefined; - if ((try readModelSafeContent(absolute_match, resolved_match, &content_buf)) == null) return null; - return absolute_match; -} - -fn safeGitIncludePathspec(include: ?glob_pattern.Pattern) ?[]const u8 { - const raw = if (include) |pattern| pattern.raw else return null; - if (raw.len == 0 or std.mem.findScalar(u8, raw, '/') != null) return null; - for (raw) |ch| { - switch (ch) { - '\\', '{', '}' => return null, - else => {}, - } - } - return raw; -} - -fn findScalarFrom(haystack: []const u8, start: usize, needle: u8) ?usize { - var index = start; - while (index < haystack.len) : (index += 1) { - if (haystack[index] == needle) return index; - } - return null; -} - fn joinAbsolutePathScratch(buffer: []u8, absolute_root: []const u8, entry_path: []const u8) ![]const u8 { var fba = std.heap.FixedBufferAllocator.init(buffer); return std.fs.path.join(fba.allocator(), &.{ absolute_root, entry_path }) catch |err| switch (err) { @@ -845,40 +670,6 @@ fn readTrace(alloc: Allocator, trace_path: []const u8) ![]u8 { return reader.interface.allocRemaining(alloc, std.Io.Limit.limited(4096)); } -fn runGitForTest(alloc: Allocator, cwd: []const u8, args: []const []const u8) !void { - var argv: std.ArrayList([]const u8) = .empty; - defer argv.deinit(alloc); - try argv.append(alloc, "git"); - try argv.appendSlice(alloc, args); - - const result = std.process.run(alloc, std.testing.io, .{ - .argv = argv.items, - .cwd = .{ .path = cwd }, - .stdout_limit = .limited(4096), - .stderr_limit = .limited(4096), - }) catch return error.SkipZigTest; - defer alloc.free(result.stdout); - defer alloc.free(result.stderr); - - switch (result.term) { - .exited => |code| if (code != 0) return error.SkipZigTest, - else => return error.SkipZigTest, - } -} - -test "grep search git grep argv uses literal fixed-string flags" { - const argv = gitGrepArgvForTest(); - - try std.testing.expectEqualStrings("git", argv[0]); - try std.testing.expectEqualStrings("--no-optional-locks", argv[1]); - try std.testing.expectEqualStrings("grep", argv[2]); - try std.testing.expectEqualStrings("-n", argv[3]); - try std.testing.expectEqualStrings("-I", argv[4]); - try std.testing.expectEqualStrings("-F", argv[5]); - try std.testing.expectEqualStrings("-z", argv[6]); - try std.testing.expectEqualStrings("-e", argv[7]); -} - test "grep search preserves explicitly requested ignored directory roots" { const alloc = std.testing.allocator; var tmp = std.testing.tmpDir(.{}); @@ -890,7 +681,7 @@ test "grep search preserves explicitly requested ignored directory roots" { var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - const result = try collectDirectoryMatches(arena_state.allocator(), workspace, try io_mod.dirRealpathAlloc(arena_state.allocator(), tmp.dir, "node_modules/pkg"), "needle", false, null); + const result = try collectDirectoryMatches(std.testing.allocator, arena_state.allocator(), workspace, try io_mod.dirRealpathAlloc(arena_state.allocator(), tmp.dir, "node_modules/pkg"), "needle", false, null); try std.testing.expectEqual(@as(usize, 1), result.matches.len); try std.testing.expect(result.truncated_reason == null); @@ -928,7 +719,7 @@ test "grep search does not ignore workspace because ignored name is outside work var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - const result = try collectDirectoryMatches(arena_state.allocator(), workspace, workspace, "needle", false, null); + const result = try collectDirectoryMatches(std.testing.allocator, arena_state.allocator(), workspace, workspace, "needle", false, null); try std.testing.expectEqual(@as(usize, 1), result.matches.len); try std.testing.expect(result.truncated_reason == null); @@ -946,7 +737,7 @@ test "grep search falls back to Zig scanner outside git repositories" { var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - const result = try collectDirectoryMatches(arena_state.allocator(), workspace, workspace, "needle", false, null); + const result = try collectDirectoryMatches(std.testing.allocator, arena_state.allocator(), workspace, workspace, "needle", false, null); try std.testing.expectEqual(@as(usize, 1), result.matches.len); try std.testing.expect(std.mem.endsWith(u8, result.matches[0].absolute_path, "src/main.zig")); @@ -969,7 +760,7 @@ test "grep search scans untracked files after git grep tracked backend" { var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - const result = try collectDirectoryMatches(arena_state.allocator(), workspace, workspace, "needle", false, null); + const result = try collectDirectoryMatches(std.testing.allocator, arena_state.allocator(), workspace, workspace, "needle", false, null); try std.testing.expectEqual(@as(usize, 1), result.matches.len); try std.testing.expect(std.mem.endsWith(u8, result.matches[0].absolute_path, "untracked.txt")); @@ -990,7 +781,7 @@ test "grep search git grep backend skips files with unsafe bytes outside matched var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - const result = try collectDirectoryMatches(arena_state.allocator(), workspace, workspace, "needle", false, null); + const result = try collectDirectoryMatches(std.testing.allocator, arena_state.allocator(), workspace, workspace, "needle", false, null); try std.testing.expectEqual(@as(usize, 0), result.matches.len); } @@ -1043,7 +834,7 @@ test "grep search logs skipped per-file scan errors during directory traversal" defer pattern.deinit(alloc); var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - const result = try collectDirectoryMatches(arena_state.allocator(), workspace, workspace, "needle", false, pattern); + const result = try collectDirectoryMatches(std.testing.allocator, arena_state.allocator(), workspace, workspace, "needle", false, pattern); try std.testing.expectEqual(@as(usize, 1), result.matches.len); try std.testing.expect(result.truncated_reason == null); @@ -1079,7 +870,7 @@ test "grep search directory traversal skips external symlink targets with trace" var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); const links_root = try io_mod.dirRealpathAlloc(arena_state.allocator(), workspace_tmp.dir, "links"); - const result = try collectDirectoryMatches(arena_state.allocator(), workspace, links_root, "needle", false, null); + const result = try collectDirectoryMatches(std.testing.allocator, arena_state.allocator(), workspace, links_root, "needle", false, null); try std.testing.expectEqual(@as(usize, 1), result.matches.len); try std.testing.expect(std.mem.endsWith(u8, result.matches[0].absolute_path, "links/internal.txt")); @@ -1170,7 +961,7 @@ test "grep search retained allocations scale with matches not scanned bytes" { var retained_buf: [8 * 1024]u8 = undefined; var retained_fba = std.heap.FixedBufferAllocator.init(&retained_buf); - const result = try collectDirectoryMatches(retained_fba.allocator(), workspace, workspace, "needle", false, null); + const result = try collectDirectoryMatches(std.testing.allocator, retained_fba.allocator(), workspace, workspace, "needle", false, null); try std.testing.expectEqual(@as(usize, 1), result.matches.len); try std.testing.expect(result.truncated_reason == null); @@ -1199,7 +990,7 @@ test "grep search logs oversized files and continues directory traversal" { var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - const result = try collectDirectoryMatches(arena_state.allocator(), workspace, workspace, "needle", false, null); + const result = try collectDirectoryMatches(std.testing.allocator, arena_state.allocator(), workspace, workspace, "needle", false, null); try std.testing.expectEqual(@as(usize, 1), result.matches.len); try std.testing.expect(result.truncated_reason == null); try std.testing.expect(std.mem.endsWith(u8, result.matches[0].absolute_path, "good.txt")); @@ -1257,7 +1048,7 @@ test "grep search finds match beyond former traversal cap" { var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - const result = try collectDirectoryMatches(arena_state.allocator(), workspace, workspace, "needle", false, null); + const result = try collectDirectoryMatches(std.testing.allocator, arena_state.allocator(), workspace, workspace, "needle", false, null); try std.testing.expectEqual(@as(usize, 1), result.matches.len); try std.testing.expect(result.truncated_reason == null); @@ -1281,17 +1072,15 @@ test "grep_files path narrowing applies before candidate cap" { var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); const result = try collectDirectoryMatchesWithOptions( + std.testing.allocator, arena_state.allocator(), workspace, narrowed_root, "needle", false, - ignored_dirs.ignored_directory_names, + tool_files.default_skipped_names, null, - .{ - .candidate_cap = 1, - .force_fallback = true, - }, + .{ .candidate_cap = 1 }, ); try std.testing.expectEqual(@as(usize, 1), result.candidate_count); @@ -1351,7 +1140,7 @@ test "grep search collection cap takes precedence at traversal boundary" { var arena_state = std.heap.ArenaAllocator.init(alloc); defer arena_state.deinit(); - const result = try collectDirectoryMatches(arena_state.allocator(), workspace, workspace, "needle", false, null); + const result = try collectDirectoryMatches(std.testing.allocator, arena_state.allocator(), workspace, workspace, "needle", false, null); try std.testing.expectEqual(collection_cap, result.matches.len); try std.testing.expectEqual(TruncatedReason.collection_cap, result.truncated_reason.?); @@ -1367,3 +1156,60 @@ test "grep search does not import tool dispatch layer" { const forbidden = "tool_" ++ "dispatch.zig"; try std.testing.expect(std.mem.find(u8, source, forbidden) == null); } + +fn runGitForTest(alloc: Allocator, cwd: []const u8, args: []const []const u8) !void { + var argv: std.ArrayList([]const u8) = .empty; + defer argv.deinit(alloc); + try argv.append(alloc, "git"); + try argv.appendSlice(alloc, args); + + const result = std.process.run(alloc, std.testing.io, .{ + .argv = argv.items, + .cwd = .{ .path = cwd }, + .stdout_limit = .limited(4096), + .stderr_limit = .limited(4096), + }) catch return error.SkipZigTest; + defer alloc.free(result.stdout); + defer alloc.free(result.stderr); + + switch (result.term) { + .exited => |code| if (code != 0) return error.SkipZigTest, + else => return error.SkipZigTest, + } +} + +test "grep search parallel scan matches single-threaded order and cap truncation" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const workspace = try workspaceRoot(alloc, tmp); + defer alloc.free(workspace); + + // 1500 files of 2 matches each exceed the collection cap, so the first + // 1000 files in sorted order must fill it exactly. Many more batches than + // workers means a premature stop would leave files unscanned. + const content = "needle one\nother\nneedle two\n"; + for (0..1500) |index| { + var name_buf: [64]u8 = undefined; + const name = try std.fmt.bufPrint(&name_buf, "many/file-{d:0>4}.txt", .{index}); + alloc.free(try writeTempFile(alloc, &tmp, name, content)); + } + + for (0..5) |_| { + var arena_state = std.heap.ArenaAllocator.init(alloc); + defer arena_state.deinit(); + const result = try collectDirectoryMatches(std.testing.allocator, arena_state.allocator(), workspace, workspace, "needle", false, null); + try std.testing.expectEqual(TruncatedReason.collection_cap, result.truncated_reason.?); + try std.testing.expectEqual(collection_cap, result.matches.len); + for (result.matches, 0..) |match, position| { + var want_buf: [64]u8 = undefined; + const want = try std.fmt.bufPrint(&want_buf, "many/file-{d:0>4}.txt", .{position / 2}); + try std.testing.expect(std.mem.endsWith(u8, match.absolute_path, want)); + try std.testing.expectEqual(2 * (position % 2) + 1, match.line_number); + } + + const counted = try countDirectoryMatchesWithIgnored(std.testing.allocator, arena_state.allocator(), workspace, workspace, "needle", false, tool_files.default_skipped_names, null); + try std.testing.expectEqual(@as(usize, 3000), counted.matching_lines); + try std.testing.expectEqual(@as(usize, 1500), counted.matching_files); + } +} diff --git a/src/core/workspace/ignored_dirs.zig b/src/core/workspace/ignored_dirs.zig deleted file mode 100644 index 9798abf6c..000000000 --- a/src/core/workspace/ignored_dirs.zig +++ /dev/null @@ -1,51 +0,0 @@ -const std = @import("std"); - -pub const ignored_directory_names: []const []const u8 = &.{ - ".git", - ".zig-cache", - "zig-out", - "node_modules", - ".next", - "dist", - "build", - "coverage", -}; - -pub fn pathContainsIgnoredDirectory(path: []const u8) bool { - var it = std.fs.path.componentIterator(path); - while (it.next()) |component| { - if (isIgnoredDirectoryName(component.name)) return true; - } - return false; -} - -pub fn isIgnoredDirectoryName(name: []const u8) bool { - for (ignored_directory_names) |ignored| { - if (std.mem.eql(u8, name, ignored)) return true; - } - return false; -} - -test "pathContainsIgnoredDirectory detects ignored path components" { - try std.testing.expect(pathContainsIgnoredDirectory("node_modules/pkg/index.zig")); - try std.testing.expect(pathContainsIgnoredDirectory("src/.git/config")); - try std.testing.expect(pathContainsIgnoredDirectory(".zig-cache/o/file.o")); - try std.testing.expect(pathContainsIgnoredDirectory("zig-out/bin/fx")); - try std.testing.expect(pathContainsIgnoredDirectory(".next/server/app.js")); - try std.testing.expect(pathContainsIgnoredDirectory("dist/app.js")); - try std.testing.expect(pathContainsIgnoredDirectory("build/out.o")); - try std.testing.expect(pathContainsIgnoredDirectory("coverage/lcov.info")); -} - -test "pathContainsIgnoredDirectory ignores non-matching path components" { - try std.testing.expect(!pathContainsIgnoredDirectory("src/module/index.zig")); - try std.testing.expect(!pathContainsIgnoredDirectory("src/node_modules_backup/index.zig")); - try std.testing.expect(!pathContainsIgnoredDirectory("src/builders/output.zig")); -} - -test "isIgnoredDirectoryName detects exact ignored basenames" { - try std.testing.expect(isIgnoredDirectoryName(".git")); - try std.testing.expect(isIgnoredDirectoryName("node_modules")); - try std.testing.expect(!isIgnoredDirectoryName("src/.git")); - try std.testing.expect(!isIgnoredDirectoryName("node_modules_backup")); -} diff --git a/src/core/workspace/tool_files.zig b/src/core/workspace/tool_files.zig new file mode 100644 index 000000000..50c1dbb40 --- /dev/null +++ b/src/core/workspace/tool_files.zig @@ -0,0 +1,185 @@ +//! Files under a search root for `glob_files` and `grep_files`, listed by the +//! native indexer and current at the time of the call. A search of the whole +//! workspace reuses the saved `@`-completion snapshot when every folder and +//! rule file it depended on is unchanged; any other search scans its root. +//! Nothing here saves a snapshot, so searches outside the workspace leave no +//! trace under `~/.fx/file-index/`. + +const std = @import("std"); +const debug_trace = @import("../shared/debug_trace.zig"); +const file_index = @import("file_index.zig"); +const indexer = @import("../indexer/indexer.zig"); + +const Allocator = std.mem.Allocator; + +pub const default_candidate_cap: usize = 100_000; +pub const max_relative_path_bytes: usize = 2048; +pub const default_skipped_names: []const []const u8 = &indexer.default_skipped_names; + +pub const Options = struct { + candidate_cap: usize = default_candidate_cap, + /// Folder and file names skipped outside repositories and below an + /// ignored root. Never applied inside a repository. + skipped_names: []const []const u8 = default_skipped_names, + include_hidden: bool = false, + /// Keep hidden folders inside a repository, where git's rules alone + /// decide, as `git grep` did. Outside one, `include_hidden` decides. + include_hidden_in_repository: bool = false, +}; + +pub const Listing = struct { + /// Relative to the search root, sorted bytewise. Owned by the arena. + files: []const []const u8, + candidate_cap: usize, + /// The cap or an unreadable folder or rule file left files out. + incomplete: bool, + skipped_overlong: usize, +}; + +pub const Error = Allocator.Error || error{FileNotFound}; + +/// Lists the files under `absolute_root`. `alloc` must be thread-safe; the +/// result is owned by `arena`. +pub fn list(alloc: Allocator, arena: Allocator, workspace_root: []const u8, absolute_root: []const u8, options: Options) Error!Listing { + if (relativeInside(workspace_root, absolute_root)) |subtree| { + if (try listFromSnapshot(alloc, arena, workspace_root, subtree, options)) |listing| return listing; + } + var result = indexer.scan(alloc, absolute_root, .{ + .candidate_cap = options.candidate_cap, + .max_path_bytes = max_relative_path_bytes, + .skipped_names = options.skipped_names, + }, null) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.RootUnavailable => return error.FileNotFound, + error.Canceled => unreachable, + }; + defer result.deinit(); + debug_trace.logf("core", "tool files scanned entries={d} incomplete={} cap_reached={}", .{ result.tree.entries.len, result.tree.incomplete, result.tree.cap_reached }); + return collect(arena, &result.tree, "", options); +} + +/// `absolute_root` relative to `workspace_root`, "" for the root itself, or +/// null when it lies outside. +fn relativeInside(workspace_root: []const u8, absolute_root: []const u8) ?[]const u8 { + if (std.mem.eql(u8, absolute_root, workspace_root)) return ""; + if (absolute_root.len <= workspace_root.len + 1 or !std.mem.startsWith(u8, absolute_root, workspace_root)) return null; + if (absolute_root[workspace_root.len] != '/') return null; + return absolute_root[workspace_root.len + 1 ..]; +} + +/// Whether the workspace tree's entries below `subtree` are exactly what a +/// scan of that folder would list: the walk entered it as an ordinary folder +/// of the workspace, not as a nested repository's boundary and not below an +/// ignored folder, where a direct scan applies the plain rules instead (U3). +fn subtreeFromSnapshot(tree: *const indexer.Tree, subtree: []const u8) bool { + if (subtree.len == 0) return true; + for (tree.folders) |folder| { + if (std.mem.eql(u8, folder.rel, subtree)) return !folder.boundary and !folder.excluded; + } + return false; +} + +fn listFromSnapshot(alloc: Allocator, arena: Allocator, workspace_root: []const u8, subtree: []const u8, options: Options) Allocator.Error!?Listing { + const roots = [_][]const u8{workspace_root}; + const path = (try file_index.snapshotPath(arena, &roots)) orelse return null; + var snapshot = (try indexer.loadSnapshot(alloc, path, &roots)) orelse return null; + defer snapshot.deinit(); + const tree = &snapshot.trees[0]; + if (tree.entries.len > options.candidate_cap) return null; + if (!subtreeFromSnapshot(tree, subtree)) return null; + if (!indexer.isCurrent(tree, workspace_root, options.skipped_names, if (subtree.len == 0) null else subtree)) return null; + debug_trace.logf("core", "tool files reused snapshot entries={d} subtree_bytes={d}", .{ tree.entries.len, subtree.len }); + return try collect(arena, tree, subtree, options); +} + +fn collect(arena: Allocator, tree: *const indexer.Tree, subtree: []const u8, options: Options) Allocator.Error!Listing { + const keep_hidden = options.include_hidden or (options.include_hidden_in_repository and tree.repository); + var files: std.ArrayList([]const u8) = .empty; + for (tree.entries) |entry| { + if (entry.kind != .file) continue; + const path = if (subtree.len == 0) entry.path else blk: { + if (entry.path.len <= subtree.len + 1 or !std.mem.startsWith(u8, entry.path, subtree) or entry.path[subtree.len] != '/') continue; + break :blk entry.path[subtree.len + 1 ..]; + }; + if (!keep_hidden and pathContainsHiddenDirectoryComponent(path)) continue; + try files.append(arena, try arena.dupe(u8, path)); + } + return .{ + .files = try files.toOwnedSlice(arena), + .candidate_cap = options.candidate_cap, + .incomplete = tree.incomplete or tree.cap_reached, + .skipped_overlong = tree.skipped_overlong, + }; +} + +/// Whether a folder in `path` (not its last component) is hidden. +pub fn pathContainsHiddenDirectoryComponent(path: []const u8) bool { + var rest = path; + while (std.mem.findScalar(u8, rest, '/')) |slash| { + const component = rest[0..slash]; + if (component.len > 1 and component[0] == '.') return true; + rest = rest[slash + 1 ..]; + } + return false; +} + +test "hidden directory components exclude descendants but not hidden files" { + try std.testing.expect(pathContainsHiddenDirectoryComponent(".github/workflows/ci.yml")); + try std.testing.expect(pathContainsHiddenDirectoryComponent("src/.cache/item")); + try std.testing.expect(!pathContainsHiddenDirectoryComponent(".gitignore")); + try std.testing.expect(!pathContainsHiddenDirectoryComponent("src/.env")); + try std.testing.expect(!pathContainsHiddenDirectoryComponent("./src/main.zig")); +} + +test "listing sees a file created a moment earlier and honors hidden and custom names" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const io = std.testing.io; + for ([_][]const u8{ "root/a.txt", "root/.hidden/h.txt", "root/build/out.o", "root/vendor/v.txt", "root/sub/s.txt" }) |sub_path| { + try tmp.dir.createDirPath(io, std.fs.path.dirname(sub_path).?); + try tmp.dir.writeFile(io, .{ .sub_path = sub_path, .data = "x" }); + } + const root = try @import("../shared/io.zig").dirRealpathAlloc(arena, tmp.dir, "root"); + + // The temporary folder sits below an ignored path, so the skipped names + // apply (U3); `build` is skipped by default. + const first = try list(std.testing.allocator, arena, root, root, .{}); + try std.testing.expectEqual(@as(usize, 3), first.files.len); + try std.testing.expectEqualStrings("a.txt", first.files[0]); + try std.testing.expectEqualStrings("sub/s.txt", first.files[1]); + try std.testing.expectEqualStrings("vendor/v.txt", first.files[2]); + + try tmp.dir.writeFile(io, .{ .sub_path = "root/sub/new.txt", .data = "x" }); + const custom = try list(std.testing.allocator, arena, root, root, .{ .skipped_names = &.{"vendor"}, .include_hidden = true }); + const want = [_][]const u8{ ".hidden/h.txt", "a.txt", "build/out.o", "sub/new.txt", "sub/s.txt" }; + try std.testing.expectEqual(want.len, custom.files.len); + for (want, custom.files) |expected, actual| try std.testing.expectEqualStrings(expected, actual); + + const sub = try list(std.testing.allocator, arena, root, try std.fs.path.join(arena, &.{ root, "sub" }), .{}); + try std.testing.expectEqual(@as(usize, 2), sub.files.len); + try std.testing.expectEqualStrings("new.txt", sub.files[0]); + + try std.testing.expectError(error.FileNotFound, list(std.testing.allocator, arena, root, try std.fs.path.join(arena, &.{ root, "missing" }), .{})); +} + +test "subfolder searches reuse the workspace tree only for ordinary folders" { + const folders = [_]indexer.FolderStamp{ + .{ .rel = "", .inode = 1, .mtime_ns = 0, .ctime_ns = 0, .reusable = true }, + .{ .rel = "src", .inode = 2, .mtime_ns = 0, .ctime_ns = 0, .reusable = true }, + .{ .rel = "inner", .inode = 3, .mtime_ns = 0, .ctime_ns = 0, .reusable = true, .boundary = true }, + .{ .rel = "build", .inode = 4, .mtime_ns = 0, .ctime_ns = 0, .reusable = true, .excluded = true }, + }; + const tree: indexer.Tree = .{ .root = "/w", .root_inode = 1, .scan_started_ns = 0, .repository = true, .incomplete = false, .cap_reached = false, .skipped_overlong = 0, .skipped_names = &.{}, .entries = &.{}, .folders = &folders, .sources = &.{} }; + try std.testing.expect(subtreeFromSnapshot(&tree, "")); + try std.testing.expect(subtreeFromSnapshot(&tree, "src")); + try std.testing.expect(!subtreeFromSnapshot(&tree, "inner")); + try std.testing.expect(!subtreeFromSnapshot(&tree, "build")); + try std.testing.expect(!subtreeFromSnapshot(&tree, "vendor")); + try std.testing.expectEqualStrings("src/core", relativeInside("/w", "/w/src/core").?); + try std.testing.expectEqualStrings("", relativeInside("/w", "/w").?); + try std.testing.expect(relativeInside("/w", "/workspace") == null); + try std.testing.expect(relativeInside("/w", "/other") == null); +} diff --git a/src/core/workspace/workspace_files.zig b/src/core/workspace/workspace_files.zig deleted file mode 100644 index 87d97eedb..000000000 --- a/src/core/workspace/workspace_files.zig +++ /dev/null @@ -1,1060 +0,0 @@ -const std = @import("std"); -const builtin = @import("builtin"); -const debug_trace = @import("../shared/debug_trace.zig"); -const ignored_dirs = @import("ignored_dirs.zig"); -const io_mod = @import("../shared/io.zig"); -const safe_git = @import("../shell_command/safe_git.zig"); -const sort_utils = @import("../shared/sort_utils.zig"); - -const Allocator = std.mem.Allocator; - -pub const default_candidate_cap: usize = 100_000; -pub const default_git_stdout_limit: usize = 32 * 1024 * 1024; -pub const max_relative_path_bytes: usize = 2048; - -pub const Source = enum { - git, - recursive, -}; - -pub const CapReason = enum { - candidate_cap, -}; - -pub const Options = struct { - candidate_cap: usize = default_candidate_cap, - git_stdout_limit: usize = default_git_stdout_limit, - ignored_names: []const []const u8 = ignored_dirs.ignored_directory_names, - force_fallback: bool = false, - include_untracked: bool = false, - only_untracked: bool = false, - include_hidden: bool = false, - sort_paths: bool = false, - git_worktree_is_authoritative: bool = false, -}; - -pub const Result = struct { - files: []const []const u8, - source: Source, - candidate_cap: usize, - incomplete: bool = false, - cap_reason: ?CapReason = null, - skipped_overlong: usize = 0, -}; - -pub const DirectoryResult = struct { - directories: []const []const u8, - source: Source, - candidate_cap: usize, - incomplete: bool = false, - cap_reason: ?CapReason = null, - skipped_overlong: usize = 0, -}; - -pub fn discover(arena: Allocator, workspace_root: []const u8, options: Options) !Result { - return discoverWithStop(arena, workspace_root, options, null, trustedGitExecutable()); -} - -pub fn discoverCancellable( - arena: Allocator, - workspace_root: []const u8, - options: Options, - stop_requested: *std.atomic.Value(bool), -) !Result { - return discoverWithStop(arena, workspace_root, options, stop_requested, trustedGitExecutable()); -} - -pub fn discoverDirectoriesCancellable( - arena: Allocator, - workspace_root: []const u8, - options: Options, - stop_requested: *std.atomic.Value(bool), -) !DirectoryResult { - return discoverDirectoriesWithStop(arena, workspace_root, options, stop_requested, trustedGitExecutable()); -} - -fn discoverWithStop( - arena: Allocator, - workspace_root: []const u8, - options: Options, - stop_requested: ?*std.atomic.Value(bool), - git_executable: ?[]const u8, -) !Result { - try checkCanceled(stop_requested); - if (workspace_root.len == 0) { - return .{ - .files = &.{}, - .source = .recursive, - .candidate_cap = options.candidate_cap, - }; - } - - if (!options.force_fallback) git: { - const executable = git_executable orelse { - debug_trace.logf("core", "workspace file discovery trusted Git unavailable; falling back root={s}", .{workspace_root}); - break :git; - }; - if (gitRawList(arena, workspace_root, options, stop_requested, executable)) |raw| { - const parsed = try parseRawList(arena, raw, .git, options); - if (options.only_untracked or - options.git_worktree_is_authoritative or - parsed.files.len > 0 or - parsed.incomplete or - parsed.skipped_overlong > 0) - { - return parsed; - } - debug_trace.logf("core", "workspace file discovery git returned no candidates; falling back root={s}", .{workspace_root}); - } else |err| { - if (err == error.Canceled) return error.Canceled; - if (options.git_worktree_is_authoritative and try hasGitMetadata(arena, workspace_root)) { - return error.GitFileListFailed; - } - debug_trace.logf("core", "workspace file discovery falling back root={s} err={s}", .{ workspace_root, @errorName(err) }); - } - } - - return walkWorkspace(arena, workspace_root, options, stop_requested); -} - -fn discoverDirectoriesWithStop( - arena: Allocator, - workspace_root: []const u8, - options: Options, - stop_requested: ?*std.atomic.Value(bool), - git_executable: ?[]const u8, -) !DirectoryResult { - try checkCanceled(stop_requested); - if (workspace_root.len == 0) { - return .{ - .directories = &.{}, - .source = .recursive, - .candidate_cap = options.candidate_cap, - }; - } - - if (!options.force_fallback) git: { - const executable = git_executable orelse { - debug_trace.logf("core", "workspace directory discovery trusted Git unavailable; falling back root={s}", .{workspace_root}); - break :git; - }; - if (gitRawIgnoredDirectoryList(arena, workspace_root, options, stop_requested, executable)) |raw| { - var ignored_paths = try parseRawIgnoredDirectories(arena, raw); - defer ignored_paths.deinit(arena); - var git_options = options; - git_options.ignored_names = &.{".git"}; - return walkWorkspaceDirectories(arena, workspace_root, git_options, stop_requested, &ignored_paths, .git); - } else |err| { - if (err == error.Canceled) return error.Canceled; - if (options.git_worktree_is_authoritative and try hasGitMetadata(arena, workspace_root)) { - return error.GitFileListFailed; - } - debug_trace.logf("core", "workspace directory discovery falling back root={s} err={s}", .{ workspace_root, @errorName(err) }); - } - } - - return walkWorkspaceDirectories(arena, workspace_root, options, stop_requested, null, .recursive); -} - -fn hasGitMetadata(alloc: Allocator, workspace_root: []const u8) !bool { - var current = workspace_root; - while (current.len > 0) { - const found = found: { - const marker = try std.fs.path.join(alloc, &.{ current, ".git" }); - defer alloc.free(marker); - _ = std.Io.Dir.cwd().statFile(io_mod.getIo(), marker, .{ .follow_symlinks = false }) catch |err| switch (err) { - error.FileNotFound => break :found false, - else => |e| return e, - }; - break :found true; - }; - if (found) return true; - - const parent = std.fs.path.dirname(current) orelse return false; - if (std.mem.eql(u8, parent, current)) return false; - current = parent; - } - return false; -} - -fn gitTrackedFilesArgv(git_executable: []const u8) [5][]const u8 { - return .{ git_executable, "--no-optional-locks", "ls-files", "-z", "--cached" }; -} - -fn gitTrackedAndOtherFilesArgv(git_executable: []const u8) [7][]const u8 { - return .{ git_executable, "--no-optional-locks", "ls-files", "-z", "--cached", "--others", "--exclude-standard" }; -} - -fn gitOtherFilesArgv(git_executable: []const u8) [6][]const u8 { - return .{ git_executable, "--no-optional-locks", "ls-files", "-z", "--others", "--exclude-standard" }; -} - -fn gitIgnoredDirectoriesArgv(git_executable: []const u8) [8][]const u8 { - return .{ git_executable, "--no-optional-locks", "ls-files", "-z", "--others", "--ignored", "--directory", "--exclude-standard" }; -} - -fn gitRawList( - arena: Allocator, - workspace_root: []const u8, - options: Options, - stop_requested: ?*std.atomic.Value(bool), - git_executable: []const u8, -) ![]u8 { - const tracked_argv = gitTrackedFilesArgv(git_executable); - const tracked_and_other_argv = gitTrackedAndOtherFilesArgv(git_executable); - const other_argv = gitOtherFilesArgv(git_executable); - const argv: []const []const u8 = if (options.only_untracked) - &other_argv - else if (options.include_untracked) - &tracked_and_other_argv - else - &tracked_argv; - return runGitRawList(arena, workspace_root, argv, options.git_stdout_limit, stop_requested); -} - -fn gitRawIgnoredDirectoryList( - arena: Allocator, - workspace_root: []const u8, - options: Options, - stop_requested: ?*std.atomic.Value(bool), - git_executable: []const u8, -) ![]u8 { - const argv = gitIgnoredDirectoriesArgv(git_executable); - return runGitRawList(arena, workspace_root, &argv, options.git_stdout_limit, stop_requested); -} - -fn runGitRawList( - arena: Allocator, - workspace_root: []const u8, - argv: []const []const u8, - stdout_limit: usize, - stop_requested: ?*std.atomic.Value(bool), -) ![]u8 { - const run_options: std.process.RunOptions = .{ - .argv = argv, - .cwd = .{ .path = workspace_root }, - .stdout_limit = std.Io.Limit.limited(stdout_limit), - .stderr_limit = std.Io.Limit.limited(1024), - }; - const result = if (stop_requested) |stop| - runCancellable(arena, run_options, stop) catch |err| switch (err) { - error.Canceled => return error.Canceled, - else => return error.GitFileListFailed, - } - else - std.process.run(arena, io_mod.getIo(), run_options) catch return error.GitFileListFailed; - defer arena.free(result.stderr); - - switch (result.term) { - .exited => |code| { - if (code == 0) return result.stdout; - arena.free(result.stdout); - return error.GitFileListFailed; - }, - else => { - arena.free(result.stdout); - return error.GitFileListFailed; - }, - } -} - -const trustedGitExecutable = safe_git.trustedExecutable; - -fn runCancellable( - alloc: Allocator, - options: std.process.RunOptions, - stop_requested: *std.atomic.Value(bool), -) !std.process.RunResult { - try checkCanceled(stop_requested); - - const zio = io_mod.getIo(); - debug_trace.logf("core", "workspace file discovery process spawning thread={d}", .{std.Thread.getCurrentId()}); - var child = try std.process.spawn(zio, .{ - .argv = options.argv, - .cwd = options.cwd, - .environ_map = options.environ_map, - .expand_arg0 = options.expand_arg0, - .progress_node = options.progress_node, - .create_no_window = options.create_no_window, - .disable_aslr = options.disable_aslr, - .stdin = .ignore, - .stdout = .pipe, - .stderr = .pipe, - }); - defer child.kill(zio); - - var multi_reader_buffer: std.Io.File.MultiReader.Buffer(2) = undefined; - var multi_reader: std.Io.File.MultiReader = undefined; - multi_reader.init(alloc, zio, multi_reader_buffer.toStreams(), &.{ child.stdout.?, child.stderr.? }); - defer multi_reader.deinit(); - - const stdout_reader = multi_reader.reader(0); - const stderr_reader = multi_reader.reader(1); - const poll_timeout: std.Io.Timeout = .{ - .duration = .{ .raw = .{ .nanoseconds = 8_000_000 }, .clock = .awake }, - }; - while (true) { - try checkCanceled(stop_requested); - const keep_reading = if (multi_reader.fill(options.reserve_amount, poll_timeout)) - true - else |err| switch (err) { - error.EndOfStream => false, - error.Timeout => true, - else => |e| return e, - }; - if (options.stdout_limit.toInt()) |limit| { - if (stdout_reader.buffered().len > limit) return error.StreamTooLong; - } - if (options.stderr_limit.toInt()) |limit| { - if (stderr_reader.buffered().len > limit) return error.StreamTooLong; - } - if (!keep_reading) break; - } - - try multi_reader.checkAnyError(); - const term = try child.wait(zio); - const stdout = try multi_reader.toOwnedSlice(0); - errdefer alloc.free(stdout); - const stderr = try multi_reader.toOwnedSlice(1); - errdefer alloc.free(stderr); - return .{ .term = term, .stdout = stdout, .stderr = stderr }; -} - -fn checkCanceled(stop_requested: ?*std.atomic.Value(bool)) !void { - if (stop_requested) |stop| { - if (stop.load(.seq_cst)) return error.Canceled; - } -} - -fn parseRawList(arena: Allocator, raw: []const u8, source: Source, options: Options) !Result { - var files: std.ArrayList([]const u8) = .empty; - errdefer files.deinit(arena); - - var incomplete = false; - var skipped_overlong: usize = 0; - const sep: u8 = if (std.mem.findScalar(u8, raw, 0) != null) 0 else '\n'; - var iter = std.mem.splitScalar(u8, raw, sep); - while (iter.next()) |entry| { - const trimmed = if (sep == '\n') std.mem.trimEnd(u8, entry, "\r") else entry; - if (trimmed.len == 0) continue; - if (!options.include_hidden and pathContainsHiddenDirectoryComponent(trimmed)) continue; - if (source != .git and pathContainsIgnoredDir(options.ignored_names, trimmed)) continue; - if (trimmed.len > max_relative_path_bytes) { - skipped_overlong += 1; - continue; - } - if (files.items.len >= options.candidate_cap) { - incomplete = true; - break; - } - - try files.append(arena, try arena.dupe(u8, trimmed)); - } - if (options.sort_paths) sortPathList(files.items); - - return .{ - .files = try files.toOwnedSlice(arena), - .source = source, - .candidate_cap = options.candidate_cap, - .incomplete = incomplete, - .cap_reason = if (incomplete) .candidate_cap else null, - .skipped_overlong = skipped_overlong, - }; -} - -const PathSet = std.StringHashMapUnmanaged(void); - -fn parseRawIgnoredDirectories(arena: Allocator, raw: []const u8) !PathSet { - var ignored: PathSet = .empty; - errdefer ignored.deinit(arena); - const separator: u8 = if (std.mem.findScalar(u8, raw, 0) != null) 0 else '\n'; - var entries = std.mem.splitScalar(u8, raw, separator); - while (entries.next()) |entry| { - const trimmed = if (separator == '\n') std.mem.trimEnd(u8, entry, "\r") else entry; - if (trimmed.len == 0 or !std.fs.path.isSep(trimmed[trimmed.len - 1])) continue; - - var path_end = trimmed.len; - while (path_end > 0 and std.fs.path.isSep(trimmed[path_end - 1])) : (path_end -= 1) {} - if (path_end == 0) continue; - const path = trimmed[0..path_end]; - if (ignored.contains(path)) continue; - const owned = try arena.dupe(u8, path); - errdefer arena.free(owned); - try ignored.put(arena, owned, {}); - } - return ignored; -} - -fn walkWorkspace( - arena: Allocator, - workspace_root: []const u8, - options: Options, - stop_requested: ?*std.atomic.Value(bool), -) !Result { - const walked = try walkWorkspacePaths(arena, workspace_root, options, stop_requested, .files, null); - return .{ - .files = walked.paths, - .source = .recursive, - .candidate_cap = options.candidate_cap, - .incomplete = walked.incomplete, - .cap_reason = if (walked.incomplete) .candidate_cap else null, - .skipped_overlong = walked.skipped_overlong, - }; -} - -fn walkWorkspaceDirectories( - arena: Allocator, - workspace_root: []const u8, - options: Options, - stop_requested: ?*std.atomic.Value(bool), - ignored_paths: ?*const PathSet, - source: Source, -) !DirectoryResult { - const walked = try walkWorkspacePaths(arena, workspace_root, options, stop_requested, .directories, ignored_paths); - return .{ - .directories = walked.paths, - .source = source, - .candidate_cap = options.candidate_cap, - .incomplete = walked.incomplete, - .cap_reason = if (walked.incomplete) .candidate_cap else null, - .skipped_overlong = walked.skipped_overlong, - }; -} - -const WalkTarget = enum { - files, - directories, -}; - -const WalkResult = struct { - paths: []const []const u8, - incomplete: bool, - skipped_overlong: usize, -}; - -fn walkWorkspacePaths( - arena: Allocator, - workspace_root: []const u8, - options: Options, - stop_requested: ?*std.atomic.Value(bool), - target: WalkTarget, - ignored_paths: ?*const PathSet, -) !WalkResult { - try checkCanceled(stop_requested); - var paths: std.ArrayList([]const u8) = .empty; - errdefer paths.deinit(arena); - - var root = std.Io.Dir.openDirAbsolute(io_mod.getIo(), workspace_root, .{ .iterate = true }) catch { - return .{ - .paths = try paths.toOwnedSlice(arena), - .incomplete = false, - .skipped_overlong = 0, - }; - }; - - var stack: std.ArrayList(FrameEntry) = .empty; - defer { - for (stack.items) |*entry| { - entry.dir.close(io_mod.getIo()); - if (!entry.retain_prefix) arena.free(entry.prefix); - } - stack.deinit(arena); - } - - { - errdefer root.close(io_mod.getIo()); - const initial_prefix = try arena.dupe(u8, ""); - errdefer arena.free(initial_prefix); - try stack.append(arena, .{ .dir = root, .prefix = initial_prefix, .iter = root.iterate() }); - } - - var incomplete = false; - var skipped_overlong: usize = 0; - while (stack.items.len > 0) { - try checkCanceled(stop_requested); - var top = &stack.items[stack.items.len - 1]; - const maybe_entry = top.iter.next(io_mod.getIo()) catch { - var popped = stack.pop().?; - popped.dir.close(io_mod.getIo()); - if (!popped.retain_prefix) arena.free(popped.prefix); - continue; - }; - - if (maybe_entry) |entry| { - try checkCanceled(stop_requested); - switch (entry.kind) { - .file, .sym_link => { - if (target != .files) continue; - if (isIgnoredName(options.ignored_names, entry.name)) continue; - if (paths.items.len >= options.candidate_cap) { - incomplete = true; - break; - } - const rel = try joinRelative(arena, top.prefix, entry.name); - if (rel.len > max_relative_path_bytes) { - arena.free(rel); - skipped_overlong += 1; - continue; - } - try paths.append(arena, rel); - }, - .directory => { - if (!options.include_hidden and isHiddenName(entry.name)) continue; - if (isIgnoredName(options.ignored_names, entry.name)) continue; - const new_prefix = try joinRelative(arena, top.prefix, entry.name); - errdefer arena.free(new_prefix); - if (ignored_paths) |ignored| { - if (ignored.contains(new_prefix)) { - arena.free(new_prefix); - continue; - } - } - if (target == .directories and paths.items.len >= options.candidate_cap) { - arena.free(new_prefix); - incomplete = true; - break; - } - if (target == .directories and new_prefix.len > max_relative_path_bytes) { - arena.free(new_prefix); - skipped_overlong += 1; - continue; - } - const retain_prefix = target == .directories; - if (retain_prefix) try paths.append(arena, new_prefix); - var sub = top.dir.openDir(io_mod.getIo(), entry.name, .{ .iterate = true }) catch { - if (!retain_prefix) arena.free(new_prefix); - continue; - }; - errdefer sub.close(io_mod.getIo()); - try stack.append(arena, .{ - .dir = sub, - .prefix = new_prefix, - .iter = sub.iterate(), - .retain_prefix = retain_prefix, - }); - }, - else => {}, - } - } else { - var popped = stack.pop().?; - popped.dir.close(io_mod.getIo()); - if (!popped.retain_prefix) arena.free(popped.prefix); - } - } - - if (options.sort_paths) sortPathList(paths.items); - - return .{ - .paths = try paths.toOwnedSlice(arena), - .incomplete = incomplete, - .skipped_overlong = skipped_overlong, - }; -} - -fn sortPathList(paths: [][]const u8) void { - sort_utils.sort([]const u8, paths, {}, struct { - fn lessThan(_: void, a: []const u8, b: []const u8) bool { - return std.mem.lessThan(u8, a, b); - } - }.lessThan); -} - -const FrameEntry = struct { - dir: std.Io.Dir, - prefix: []u8, - iter: std.Io.Dir.Iterator, - retain_prefix: bool = false, -}; - -fn joinRelative(arena: Allocator, prefix: []const u8, name: []const u8) ![]u8 { - if (prefix.len == 0) return arena.dupe(u8, name); - return std.fmt.allocPrint(arena, "{s}/{s}", .{ prefix, name }); -} - -pub fn pathContainsIgnoredDir(ignored: []const []const u8, path: []const u8) bool { - var it = std.fs.path.componentIterator(path); - while (it.next()) |component| { - if (isIgnoredName(ignored, component.name)) return true; - } - return false; -} - -pub fn pathContainsHiddenDirectoryComponent(path: []const u8) bool { - var rest = path; - while (std.mem.findScalar(u8, rest, '/')) |slash| { - const component = rest[0..slash]; - if (isHiddenName(component)) return true; - rest = rest[slash + 1 ..]; - } - return false; -} - -pub fn isHiddenName(name: []const u8) bool { - return name.len > 1 and name[0] == '.'; -} - -fn isIgnoredName(ignored: []const []const u8, name: []const u8) bool { - for (ignored) |entry| { - if (std.mem.eql(u8, name, entry)) return true; - } - return false; -} - -fn writeTestFile(dir: std.Io.Dir, path: []const u8, content: []const u8) !void { - if (std.fs.path.dirname(path)) |parent| { - try dir.createDirPath(io_mod.getIo(), parent); - } - var file = try dir.createFile(std.testing.io, path, .{}); - defer file.close(io_mod.getIo()); - try file.writeStreamingAll(io_mod.getIo(), content); -} - -fn containsPath(files: []const []const u8, needle: []const u8) bool { - for (files) |path| { - if (std.mem.eql(u8, path, needle)) return true; - } - return false; -} - -test "workspace file provider tracked git argv uses nul-separated cached files" { - const argv = gitTrackedFilesArgv("git"); - - try std.testing.expectEqualStrings("git", argv[0]); - try std.testing.expectEqualStrings("--no-optional-locks", argv[1]); - try std.testing.expectEqualStrings("ls-files", argv[2]); - try std.testing.expectEqualStrings("-z", argv[3]); - try std.testing.expectEqualStrings("--cached", argv[4]); -} - -test "workspace file provider optional untracked git argv includes exclude-standard" { - const argv = gitTrackedAndOtherFilesArgv("git"); - - try std.testing.expectEqualStrings("git", argv[0]); - try std.testing.expectEqualStrings("--no-optional-locks", argv[1]); - try std.testing.expectEqualStrings("ls-files", argv[2]); - try std.testing.expectEqualStrings("-z", argv[3]); - try std.testing.expectEqualStrings("--cached", argv[4]); - try std.testing.expectEqualStrings("--others", argv[5]); - try std.testing.expectEqualStrings("--exclude-standard", argv[6]); -} - -test "workspace file provider untracked git argv uses others exclude-standard" { - const argv = gitOtherFilesArgv("git"); - - try std.testing.expectEqualStrings("git", argv[0]); - try std.testing.expectEqualStrings("--no-optional-locks", argv[1]); - try std.testing.expectEqualStrings("ls-files", argv[2]); - try std.testing.expectEqualStrings("-z", argv[3]); - try std.testing.expectEqualStrings("--others", argv[4]); - try std.testing.expectEqualStrings("--exclude-standard", argv[5]); -} - -test "workspace directory provider asks Git only for ignored directory roots" { - const argv = gitIgnoredDirectoriesArgv("git"); - - try std.testing.expectEqualStrings("git", argv[0]); - try std.testing.expectEqualStrings("--no-optional-locks", argv[1]); - try std.testing.expectEqualStrings("ls-files", argv[2]); - try std.testing.expectEqualStrings("-z", argv[3]); - try std.testing.expectEqualStrings("--others", argv[4]); - try std.testing.expectEqualStrings("--ignored", argv[5]); - try std.testing.expectEqualStrings("--directory", argv[6]); - try std.testing.expectEqualStrings("--exclude-standard", argv[7]); -} - -test "workspace directory provider parses only ignored directory entries" { - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - - var ignored = try parseRawIgnoredDirectories( - arena_state.allocator(), - "ignored-dir/\x00ignored.txt\x00nested/ignored/\x00", - ); - defer ignored.deinit(arena_state.allocator()); - - try std.testing.expect(ignored.contains("ignored-dir")); - try std.testing.expect(ignored.contains("nested/ignored")); - try std.testing.expect(!ignored.contains("ignored.txt")); -} - -test "workspace file provider caps candidates and reports incomplete status" { - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - - const raw = "a.zig\x00b.zig\x00c.zig\x00"; - const result = try parseRawList(arena_state.allocator(), raw, .git, .{ .candidate_cap = 2 }); - - try std.testing.expectEqual(@as(usize, 2), result.files.len); - try std.testing.expect(result.incomplete); - try std.testing.expectEqual(CapReason.candidate_cap, result.cap_reason.?); - try std.testing.expectEqualStrings("a.zig", result.files[0]); - try std.testing.expectEqualStrings("b.zig", result.files[1]); -} - -test "workspace file provider applies bytewise ordering to Git results when requested" { - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - - const result = try parseRawList(arena_state.allocator(), "z.zig\x00a.zig\x00m.zig\x00", .git, .{ .sort_paths = true }); - - try std.testing.expectEqualStrings("a.zig", result.files[0]); - try std.testing.expectEqualStrings("m.zig", result.files[1]); - try std.testing.expectEqualStrings("z.zig", result.files[2]); -} - -test "workspace file provider git discovery includes tracked build files and skips hidden directories" { - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - - const raw = "build/script.ts\x00.eslint-plugin-local/rule.ts\x00src/.esbuild.ts\x00src/main.ts\x00"; - const result = try parseRawList(arena_state.allocator(), raw, .git, .{}); - - try std.testing.expectEqual(@as(usize, 3), result.files.len); - try std.testing.expect(containsPath(result.files, "build/script.ts")); - try std.testing.expect(containsPath(result.files, "src/.esbuild.ts")); - try std.testing.expect(containsPath(result.files, "src/main.ts")); - try std.testing.expect(!containsPath(result.files, ".eslint-plugin-local/rule.ts")); -} - -test "workspace file provider recursive fallback skips ignored directories and includes nested files" { - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - try writeTestFile(tmp.dir, "src/main.zig", "main\n"); - try writeTestFile(tmp.dir, "src/nested/lib.zig", "lib\n"); - try writeTestFile(tmp.dir, "node_modules/pkg/ignored.zig", "ignored\n"); - try writeTestFile(tmp.dir, ".zig-cache/o/ignored.o", "ignored\n"); - - const alloc = std.testing.allocator; - const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); - defer alloc.free(root); - - var arena_state = std.heap.ArenaAllocator.init(alloc); - defer arena_state.deinit(); - const result = try discover(arena_state.allocator(), root, .{ .force_fallback = true }); - - try std.testing.expectEqual(Source.recursive, result.source); - try std.testing.expect(containsPath(result.files, "src/main.zig")); - try std.testing.expect(containsPath(result.files, "src/nested/lib.zig")); - try std.testing.expect(!containsPath(result.files, "node_modules/pkg/ignored.zig")); - try std.testing.expect(!containsPath(result.files, ".zig-cache/o/ignored.o")); -} - -test "workspace directory provider fallback includes nested empty directories and skips ignored roots" { - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try tmp.dir.createDirPath(std.testing.io, "empty/deep"); - try tmp.dir.createDirPath(std.testing.io, ".hidden/empty"); - try tmp.dir.createDirPath(std.testing.io, "ignored-dir/deep"); - - const alloc = std.testing.allocator; - const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); - defer alloc.free(root); - var arena_state = std.heap.ArenaAllocator.init(alloc); - defer arena_state.deinit(); - const result = try discoverDirectoriesWithStop( - arena_state.allocator(), - root, - .{ - .force_fallback = true, - .include_hidden = true, - .ignored_names = &.{"ignored-dir"}, - }, - null, - null, - ); - - try std.testing.expectEqual(Source.recursive, result.source); - try std.testing.expect(containsPath(result.directories, ".hidden")); - try std.testing.expect(containsPath(result.directories, ".hidden/empty")); - try std.testing.expect(containsPath(result.directories, "empty")); - try std.testing.expect(containsPath(result.directories, "empty/deep")); - try std.testing.expect(!containsPath(result.directories, "ignored-dir")); -} - -test "workspace directory provider honors its cap and cancellation" { - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try tmp.dir.createDir(std.testing.io, "first", .default_dir); - try tmp.dir.createDir(std.testing.io, "second", .default_dir); - - const alloc = std.testing.allocator; - const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); - defer alloc.free(root); - var arena_state = std.heap.ArenaAllocator.init(alloc); - defer arena_state.deinit(); - const result = try discoverDirectoriesWithStop( - arena_state.allocator(), - root, - .{ .candidate_cap = 1, .force_fallback = true }, - null, - null, - ); - try std.testing.expectEqual(@as(usize, 1), result.directories.len); - try std.testing.expect(result.incomplete); - try std.testing.expectEqual(CapReason.candidate_cap, result.cap_reason.?); - - var stop_requested = std.atomic.Value(bool).init(true); - try std.testing.expectError( - error.Canceled, - discoverDirectoriesWithStop(arena_state.allocator(), root, .{ .force_fallback = true }, &stop_requested, null), - ); -} - -test "workspace file provider recursive fallback does not recurse symlink directories" { - if (comptime @import("builtin").os.tag == .windows) return error.SkipZigTest; - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - try writeTestFile(tmp.dir, "real/inside.txt", "inside\n"); - try tmp.dir.symLink(std.testing.io, "real", "linked", .{ .is_directory = true }); - - const alloc = std.testing.allocator; - const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); - defer alloc.free(root); - - var arena_state = std.heap.ArenaAllocator.init(alloc); - defer arena_state.deinit(); - const result = try discover(arena_state.allocator(), root, .{ .force_fallback = true }); - - try std.testing.expect(containsPath(result.files, "linked")); - try std.testing.expect(!containsPath(result.files, "linked/inside.txt")); -} - -test "workspace file provider overlong paths are skipped without aborting" { - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - - const long_path = try std.testing.allocator.alloc(u8, max_relative_path_bytes + 1); - defer std.testing.allocator.free(long_path); - @memset(long_path, 'a'); - const raw = try std.mem.concat(std.testing.allocator, u8, &.{ "kept.zig\x00", long_path, "\x00also-kept.zig\x00" }); - defer std.testing.allocator.free(raw); - - const result = try parseRawList(arena_state.allocator(), raw, .git, .{}); - - try std.testing.expectEqual(@as(usize, 2), result.files.len); - try std.testing.expectEqual(@as(usize, 1), result.skipped_overlong); - try std.testing.expect(containsPath(result.files, "kept.zig")); - try std.testing.expect(containsPath(result.files, "also-kept.zig")); -} - -test "workspace file provider falls back when git is skipped" { - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - - try writeTestFile(tmp.dir, "nested/file.txt", "file\n"); - - const alloc = std.testing.allocator; - const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); - defer alloc.free(root); - - var arena_state = std.heap.ArenaAllocator.init(alloc); - defer arena_state.deinit(); - const result = try discover(arena_state.allocator(), root, .{ .force_fallback = true }); - - try std.testing.expectEqual(Source.recursive, result.source); - try std.testing.expect(containsPath(result.files, "nested/file.txt")); -} - -test "workspace file provider treats empty successful git result as authoritative" { - if (comptime @import("builtin").os.tag == .windows or @import("builtin").os.tag == .wasi) return error.SkipZigTest; - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try writeTestFile(tmp.dir, "untracked.txt", "not tracked\n"); - - const alloc = std.testing.allocator; - const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); - defer alloc.free(root); - try runGitForTest(alloc, root, &.{ "git", "init", "--quiet" }); - - var arena_state = std.heap.ArenaAllocator.init(alloc); - defer arena_state.deinit(); - const result = try discoverWithStop( - arena_state.allocator(), - root, - .{ .git_worktree_is_authoritative = true }, - null, - "git", - ); - - try std.testing.expectEqual(Source.git, result.source); - try std.testing.expectEqual(@as(usize, 0), result.files.len); -} - -test "workspace file provider walks a Git worktree when no trusted Git executable is available" { - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try tmp.dir.createDir(std.testing.io, ".git", .default_dir); - try writeTestFile(tmp.dir, "fallback.txt", "fallback\n"); - - const alloc = std.testing.allocator; - const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); - defer alloc.free(root); - - var arena_state = std.heap.ArenaAllocator.init(alloc); - defer arena_state.deinit(); - const result = try discoverWithStop( - arena_state.allocator(), - root, - .{ .git_worktree_is_authoritative = true }, - null, - null, - ); - - try std.testing.expectEqual(Source.recursive, result.source); - try std.testing.expect(containsPath(result.files, "fallback.txt")); -} - -test "workspace file provider does not recurse after a selected Git executable fails" { - if (comptime builtin.os.tag == .wasi) return error.SkipZigTest; - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try writeTestFile(tmp.dir, ".git", "gitdir: /missing\n"); - try writeTestFile(tmp.dir, "untracked.txt", "not tracked\n"); - - const alloc = std.testing.allocator; - const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); - defer alloc.free(root); - - var arena_state = std.heap.ArenaAllocator.init(alloc); - defer arena_state.deinit(); - try std.testing.expectError( - error.GitFileListFailed, - discoverWithStop( - arena_state.allocator(), - root, - .{ .git_worktree_is_authoritative = true }, - null, - "/definitely/missing/fx-git", - ), - ); -} - -test "workspace file provider git result contains tracked files only" { - if (comptime @import("builtin").os.tag == .windows or @import("builtin").os.tag == .wasi) return error.SkipZigTest; - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try writeTestFile(tmp.dir, ".gitignore", "ignored.txt\n"); - try writeTestFile(tmp.dir, "tracked.txt", "tracked\n"); - try writeTestFile(tmp.dir, ".hidden/tracked.txt", "hidden\n"); - try writeTestFile(tmp.dir, "ignored.txt", "ignored\n"); - try writeTestFile(tmp.dir, "untracked.txt", "untracked\n"); - - const alloc = std.testing.allocator; - const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); - defer alloc.free(root); - try runGitForTest(alloc, root, &.{ "git", "init", "--quiet" }); - try runGitForTest(alloc, root, &.{ "git", "add", ".gitignore", "tracked.txt", ".hidden/tracked.txt" }); - - var arena_state = std.heap.ArenaAllocator.init(alloc); - defer arena_state.deinit(); - const result = try discoverWithStop(arena_state.allocator(), root, .{ .include_hidden = true }, null, "git"); - - try std.testing.expectEqual(Source.git, result.source); - try std.testing.expect(containsPath(result.files, ".gitignore")); - try std.testing.expect(containsPath(result.files, "tracked.txt")); - try std.testing.expect(containsPath(result.files, ".hidden/tracked.txt")); - try std.testing.expect(!containsPath(result.files, "ignored.txt")); - try std.testing.expect(!containsPath(result.files, "untracked.txt")); -} - -test "workspace directory provider uses Git ignores without collapsing nested empty directories" { - if (comptime builtin.os.tag == .windows or builtin.os.tag == .wasi) return error.SkipZigTest; - - var tmp = std.testing.tmpDir(.{}); - defer tmp.cleanup(); - try writeTestFile(tmp.dir, ".gitignore", "ignored-dir/\n"); - try tmp.dir.createDirPath(std.testing.io, "empty-dir"); - try tmp.dir.createDirPath(std.testing.io, "nested-empty/deep"); - try tmp.dir.createDirPath(std.testing.io, ".hidden/empty"); - try tmp.dir.createDirPath(std.testing.io, "ignored-dir/deep"); - try writeTestFile(tmp.dir, "build/tracked.txt", "tracked\n"); - - const alloc = std.testing.allocator; - const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); - defer alloc.free(root); - try runGitForTest(alloc, root, &.{ "git", "init", "--quiet" }); - try runGitForTest(alloc, root, &.{ "git", "add", "build/tracked.txt" }); - - var arena_state = std.heap.ArenaAllocator.init(alloc); - defer arena_state.deinit(); - const result = try discoverDirectoriesWithStop( - arena_state.allocator(), - root, - .{ .include_hidden = true }, - null, - "git", - ); - - try std.testing.expectEqual(Source.git, result.source); - try std.testing.expect(containsPath(result.directories, ".hidden")); - try std.testing.expect(containsPath(result.directories, ".hidden/empty")); - try std.testing.expect(containsPath(result.directories, "empty-dir")); - try std.testing.expect(containsPath(result.directories, "nested-empty")); - try std.testing.expect(containsPath(result.directories, "nested-empty/deep")); - try std.testing.expect(containsPath(result.directories, "build")); - try std.testing.expect(!containsPath(result.directories, "ignored-dir")); -} - -test "workspace file provider cancellable fallback stops before traversal" { - var stop_requested = std.atomic.Value(bool).init(true); - try std.testing.expectError( - error.Canceled, - discoverCancellable(std.testing.allocator, "/unused", .{ .force_fallback = true }, &stop_requested), - ); -} - -test "workspace file provider cancellation terminates an active child" { - if (comptime builtin.os.tag == .windows or builtin.os.tag == .wasi) return error.SkipZigTest; - - var stop_requested = std.atomic.Value(bool).init(false); - const RequestStop = struct { - fn run(stop: *std.atomic.Value(bool)) void { - sleepBlocking(20); - stop.store(true, .seq_cst); - } - }; - const stop_thread = try std.Thread.spawn(.{}, RequestStop.run, .{&stop_requested}); - defer stop_thread.join(); - - const started = std.Io.Clock.Timestamp.now(io_mod.getIo(), .awake); - try std.testing.expectError( - error.Canceled, - runCancellable( - std.testing.allocator, - .{ - .argv = &.{ "/bin/sleep", "60" }, - .stdout_limit = .limited(1024), - .stderr_limit = .limited(1024), - }, - &stop_requested, - ), - ); - const elapsed_ms = started.durationTo(std.Io.Clock.Timestamp.now(io_mod.getIo(), .awake)).raw.toMilliseconds(); - try std.testing.expect(elapsed_ms < 1000); -} - -fn sleepBlocking(milliseconds: u64) void { - var sleep_io_backend: std.Io.Threaded = .init_single_threaded; - sleep_io_backend.io().sleep(.fromMilliseconds(@intCast(milliseconds)), .real) catch {}; -} - -fn runGitForTest(alloc: Allocator, cwd: []const u8, argv: []const []const u8) !void { - const result = try std.process.run(alloc, io_mod.getIo(), .{ - .argv = argv, - .cwd = .{ .path = cwd }, - .stdout_limit = .limited(1024 * 1024), - .stderr_limit = .limited(1024 * 1024), - }); - defer alloc.free(result.stdout); - defer alloc.free(result.stderr); - switch (result.term) { - .exited => |code| try std.testing.expectEqual(@as(u8, 0), code), - else => return error.TestUnexpectedResult, - } -} diff --git a/src/tools/filesystem/glob_files.zig b/src/tools/filesystem/glob_files.zig index 7a6617e74..22351967e 100644 --- a/src/tools/filesystem/glob_files.zig +++ b/src/tools/filesystem/glob_files.zig @@ -6,7 +6,7 @@ const mem_utils = @import("../../core/shared/mem_utils.zig"); const pathing = @import("../../core/workspace/pathing.zig"); const tool_dispatch = @import("../../core/tooling/tool_dispatch.zig"); const tool_result_errors = @import("../../core/tooling/tool_result_errors.zig"); -const workspace_files = @import("../../core/workspace/workspace_files.zig"); +const tool_files = @import("../../core/workspace/tool_files.zig"); const sort_utils = @import("../../core/shared/sort_utils.zig"); const Allocator = std.mem.Allocator; @@ -94,7 +94,7 @@ pub fn call(ctx: tool_dispatch.DispatchContext, erased: tool_dispatch.ToolInput) return callWithWorkspaceOptions(ctx, erased, .{}); } -fn callWithWorkspaceOptions(ctx: tool_dispatch.DispatchContext, erased: tool_dispatch.ToolInput, workspace_options: workspace_files.Options) tool_dispatch.DispatchError!tool_dispatch.ToolResult { +fn callWithWorkspaceOptions(ctx: tool_dispatch.DispatchContext, erased: tool_dispatch.ToolInput, list_options: tool_files.Options) tool_dispatch.DispatchError!tool_dispatch.ToolResult { const input = erased.as(Input); var arena_state = std.heap.ArenaAllocator.init(ctx.allocator); @@ -124,9 +124,9 @@ fn callWithWorkspaceOptions(ctx: tool_dispatch.DispatchContext, erased: tool_dis requested_root; const root = maybe_root orelse { if (input.mode == .count) { - return .{ .success = try formatCount(ctx.allocator, input.pattern, 0, false, workspace_files.default_candidate_cap, 0) }; + return .{ .success = try formatCount(ctx.allocator, input.pattern, 0, false, tool_files.default_candidate_cap, 0) }; } - return .{ .success = try formatMatches(ctx.allocator, input.pattern, &.{}, false, false, workspace_files.default_candidate_cap, 0, ctx.max_list_entries) }; + return .{ .success = try formatMatches(ctx.allocator, input.pattern, &.{}, false, false, tool_files.default_candidate_cap, 0, ctx.max_list_entries) }; }; const root_relative = try pathing.workspaceRelativePath(arena, ctx.workspace_root, root.absolute); var compiled_pattern = glob_pattern.Pattern.compile(arena, static_base.pattern) catch |err| switch (err) { @@ -140,39 +140,21 @@ fn callWithWorkspaceOptions(ctx: tool_dispatch.DispatchContext, erased: tool_dis var match_count: usize = 0; var output_truncated = false; var candidate_incomplete = false; - var candidate_cap: usize = workspace_files.default_candidate_cap; + var candidate_cap: usize = tool_files.default_candidate_cap; var skipped_overlong: usize = 0; - const is_scoped_search_root = root_relative.len > 0 and !std.mem.eql(u8, root_relative, "."); - var discovery_options = workspace_options; - discovery_options.include_untracked = is_scoped_search_root; - discovery_options.force_fallback = discovery_options.force_fallback or is_scoped_search_root; - discovery_options.include_hidden = discovery_options.include_hidden or shouldIncludeHidden(root_relative, input.pattern); - discovery_options.sort_paths = true; + var options = list_options; + options.skipped_names = ctx.ignored_list_entries; + options.include_hidden = options.include_hidden or shouldIncludeHidden(root_relative, input.pattern); if (root.is_directory) { - const discovered = discoverWorkspaceFiles(arena, root.absolute, ctx.ignored_list_entries, discovery_options) catch |err| { - if (err == error.OutOfMemory) return error.OutOfMemory; - if (tool_result_errors.isFilesystemAccessDenied(err)) { - return .{ .failure = try tool_result_errors.filesystemAccessDeniedJson(ctx.allocator, "glob_files", root.absolute, err) }; - } - return .{ .failure = try std.fmt.allocPrint(ctx.allocator, "Unable to discover glob candidates: {s} ({s})", .{ root.absolute, @errorName(err) }) }; + const listing = tool_files.list(ctx.allocator, arena, ctx.workspace_root, root.absolute, options) catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.FileNotFound => return .{ .failure = try std.fmt.allocPrint(ctx.allocator, "Unable to discover glob candidates: {s} ({s})", .{ root.absolute, @errorName(err) }) }, }; - var candidate_files = discovered.files; - candidate_incomplete = discovered.incomplete; - candidate_cap = discovered.candidate_cap; - skipped_overlong = discovered.skipped_overlong; - if (shouldMergeRootUntracked(root_relative, discovered, discovery_options)) { - const merged = mergeRootUntrackedCandidates(arena, root.absolute, ctx.ignored_list_entries, discovered) catch |err| { - if (err == error.OutOfMemory) return error.OutOfMemory; - if (tool_result_errors.isFilesystemAccessDenied(err)) { - return .{ .failure = try tool_result_errors.filesystemAccessDeniedJson(ctx.allocator, "glob_files", root.absolute, err) }; - } - return .{ .failure = try std.fmt.allocPrint(ctx.allocator, "Unable to discover glob candidates: {s} ({s})", .{ root.absolute, @errorName(err) }) }; - }; - candidate_files = merged.files; - candidate_incomplete = candidate_incomplete or merged.incomplete; - skipped_overlong += merged.skipped_overlong; - } + const candidate_files = listing.files; + candidate_incomplete = listing.incomplete; + candidate_cap = listing.candidate_cap; + skipped_overlong = listing.skipped_overlong; for (candidate_files) |candidate| { if (!compiled_pattern.matchesPath(candidate)) continue; @@ -339,94 +321,11 @@ fn joinRelativeSearchPath(arena: Allocator, root_relative: []const u8, child_rel } fn shouldIncludeHidden(root_relative: []const u8, pattern: []const u8) bool { - return workspace_files.pathContainsHiddenDirectoryComponent(root_relative) or patternContainsHiddenDirectoryComponent(pattern); -} - -const MergedRootCandidates = struct { - files: []const []const u8, - incomplete: bool = false, - skipped_overlong: usize = 0, -}; - -fn discoverWorkspaceFiles(arena: Allocator, absolute_root: []const u8, ignored: []const []const u8, options: workspace_files.Options) !workspace_files.Result { - var discover_options = options; - discover_options.ignored_names = ignored; - return workspace_files.discover(arena, absolute_root, discover_options); -} - -fn shouldMergeRootUntracked(root_relative: []const u8, result: workspace_files.Result, options: workspace_files.Options) bool { - return result.source == .git and !options.include_untracked and !options.force_fallback and (root_relative.len == 0 or std.mem.eql(u8, root_relative, ".")); -} - -fn mergeRootUntrackedCandidates( - arena: Allocator, - absolute_root: []const u8, - ignored: []const []const u8, - discovered: workspace_files.Result, -) !MergedRootCandidates { - var files: std.ArrayList([]const u8) = .empty; - errdefer files.deinit(arena); - try files.appendSlice(arena, discovered.files); - - var seen = std.StringHashMap(void).init(arena); - defer seen.deinit(); - for (discovered.files) |path| { - try seen.put(path, {}); - } - - var dir = std.Io.Dir.openDirAbsolute(io_mod.getIo(), absolute_root, .{ .iterate = true }) catch { - return .{ .files = try files.toOwnedSlice(arena) }; - }; - defer dir.close(io_mod.getIo()); - - var incomplete = false; - var skipped_overlong: usize = 0; - var iter = dir.iterate(); - while (try iter.next(io_mod.getIo())) |entry| { - switch (entry.kind) { - .file, .sym_link => {}, - else => continue, - } - if (rootNameIgnored(ignored, entry.name)) continue; - if (seen.contains(entry.name)) continue; - if (entry.name.len > workspace_files.max_relative_path_bytes) { - skipped_overlong += 1; - continue; - } - if (files.items.len >= discovered.candidate_cap) { - incomplete = true; - break; - } - const owned = try arena.dupe(u8, entry.name); - try seen.put(owned, {}); - try files.append(arena, owned); - } - - sortCandidatePaths(files.items); - return .{ - .files = try files.toOwnedSlice(arena), - .incomplete = incomplete, - .skipped_overlong = skipped_overlong, - }; -} - -fn rootNameIgnored(ignored: []const []const u8, name: []const u8) bool { - for (ignored) |entry| { - if (std.mem.eql(u8, name, entry)) return true; - } - return false; -} - -fn sortCandidatePaths(paths: [][]const u8) void { - sort_utils.sort([]const u8, paths, {}, struct { - fn lessThan(_: void, a: []const u8, b: []const u8) bool { - return std.mem.lessThan(u8, a, b); - } - }.lessThan); + return tool_files.pathContainsHiddenDirectoryComponent(root_relative) or patternContainsHiddenDirectoryComponent(pattern); } fn patternContainsHiddenDirectoryComponent(pattern: []const u8) bool { - return workspace_files.pathContainsHiddenDirectoryComponent(pattern); + return tool_files.pathContainsHiddenDirectoryComponent(pattern); } /// Reports that glob_files only observes filesystem names and metadata. @@ -528,27 +427,6 @@ fn dispatchGlobFilesRaw(alloc: Allocator, workspace_root: []const u8, args_json: }); } -fn runGitForTest(alloc: Allocator, cwd: []const u8, args: []const []const u8) !void { - var argv: std.ArrayList([]const u8) = .empty; - defer argv.deinit(alloc); - try argv.append(alloc, "git"); - try argv.appendSlice(alloc, args); - - const result = std.process.run(alloc, std.testing.io, .{ - .argv = argv.items, - .cwd = .{ .path = cwd }, - .stdout_limit = .limited(4096), - .stderr_limit = .limited(4096), - }) catch return error.SkipZigTest; - defer alloc.free(result.stdout); - defer alloc.free(result.stderr); - - switch (result.term) { - .exited => |code| if (code != 0) return error.SkipZigTest, - else => return error.SkipZigTest, - } -} - fn writeTempFile(alloc: Allocator, tmp: *std.testing.TmpDir, sub_path: []const u8, content: []const u8) ![]u8 { if (std.fs.path.dirname(sub_path)) |parent| { try tmp.dir.createDirPath(io_mod.getIo(), parent); @@ -847,7 +725,7 @@ test "glob_files count mode reports exact matches without rendering entries" { test "glob_files formatter uses active truncation wording" { const alloc = std.testing.allocator; const matches = [_][]const u8{ "a.txt", "b.txt" }; - const body = try formatMatches(alloc, "*.txt", &matches, true, false, workspace_files.default_candidate_cap, 0, 2); + const body = try formatMatches(alloc, "*.txt", &matches, true, false, tool_files.default_candidate_cap, 0, 2); defer alloc.free(body); try std.testing.expectEqualStrings("[glob] 2 matches for *.txt\n - a.txt\n - b.txt\n... truncated to first 2 matches\n", body); @@ -879,7 +757,6 @@ test "glob_files path narrowing applies before candidate cap" { .max_list_entries = 10, }, erased, .{ .candidate_cap = 1, - .force_fallback = true, }); defer result.deinit(alloc); @@ -916,7 +793,6 @@ test "glob_files extracts static base before candidate cap" { .max_list_entries = 10, }, erased, .{ .candidate_cap = 1, - .force_fallback = true, }); defer result.deinit(alloc); @@ -1067,3 +943,24 @@ test "glob_files classifiers are read-only and reversible" { try std.testing.expect(readsOnly(erased)); try std.testing.expect(!isIrreversible(erased)); } + +fn runGitForTest(alloc: Allocator, cwd: []const u8, args: []const []const u8) !void { + var argv: std.ArrayList([]const u8) = .empty; + defer argv.deinit(alloc); + try argv.append(alloc, "git"); + try argv.appendSlice(alloc, args); + + const result = std.process.run(alloc, std.testing.io, .{ + .argv = argv.items, + .cwd = .{ .path = cwd }, + .stdout_limit = .limited(4096), + .stderr_limit = .limited(4096), + }) catch return error.SkipZigTest; + defer alloc.free(result.stdout); + defer alloc.free(result.stderr); + + switch (result.term) { + .exited => |code| if (code != 0) return error.SkipZigTest, + else => return error.SkipZigTest, + } +} diff --git a/src/tools/filesystem/grep_files.zig b/src/tools/filesystem/grep_files.zig index 7f731200c..8c0aaf38c 100644 --- a/src/tools/filesystem/grep_files.zig +++ b/src/tools/filesystem/grep_files.zig @@ -169,6 +169,7 @@ pub fn call(ctx: tool_dispatch.DispatchContext, erased: tool_dispatch.ToolInput) const StatRootFn = *const fn ([]const u8) anyerror!std.Io.File.Stat; const CollectDirectoryFn = *const fn ( + Allocator, Allocator, []const u8, []const u8, @@ -186,6 +187,7 @@ const CollectFileFn = *const fn ( ?glob_pattern.Pattern, ) anyerror!grep_search.Result; const CountDirectoryFn = *const fn ( + Allocator, Allocator, []const u8, []const u8, @@ -269,10 +271,10 @@ fn searchGrepRoot( ) tool_dispatch.DispatchError!SearchOutcome { return switch (root_stat.kind) { .directory => switch (input.mode) { - .count => .{ .count = ops.count_directory(arena, ctx.workspace_root, absolute_root, input.pattern, input.case_insensitive, ctx.ignored_list_entries, include_pattern) catch |err| { + .count => .{ .count = ops.count_directory(ctx.allocator, arena, ctx.workspace_root, absolute_root, input.pattern, input.case_insensitive, ctx.ignored_list_entries, include_pattern) catch |err| { return searchFailure(ctx.allocator, absolute_root, "walk grep search root", err); } }, - .matches, .files_with_matches => .{ .matches = ops.collect_directory(arena, ctx.workspace_root, absolute_root, input.pattern, input.case_insensitive, ctx.ignored_list_entries, include_pattern) catch |err| { + .matches, .files_with_matches => .{ .matches = ops.collect_directory(ctx.allocator, arena, ctx.workspace_root, absolute_root, input.pattern, input.case_insensitive, ctx.ignored_list_entries, include_pattern) catch |err| { return searchFailure(ctx.allocator, absolute_root, "walk grep search root", err); } }, }, @@ -785,6 +787,7 @@ fn namedPipeStatRoot(_: []const u8) anyerror!std.Io.File.Stat { } fn failingCollectDirectory( + _: Allocator, _: Allocator, _: []const u8, _: []const u8, From 23930e634a1b78bab678b3b12d7520736cbe1279 Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 8 Oct 2026 16:32:02 -0400 Subject: [PATCH 07/23] Test that a poisoned repository runs none of its programs - Add an end-to-end test with a repository whose own config names fsmonitor, hooks, filters, diff drivers and a pager, and check that interactive startup, @-completion, glob_files, grep_files, auto-run git status and fx pr run none of them while still answering from the repository. - Classify it as verification-only for macOS arm64 PGSO and give it a CI shard weight. --- scripts/pgso/corpus.json | 1 + scripts/pgso/tests/test_corpus.py | 3 +- tests/e2e/ci-shard-weights.json | 1 + tests/e2e/poisoned-repository.test.ts | 247 ++++++++++++++++++++++++++ 4 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 tests/e2e/poisoned-repository.test.ts diff --git a/scripts/pgso/corpus.json b/scripts/pgso/corpus.json index 15dbed9a6..89af8e535 100644 --- a/scripts/pgso/corpus.json +++ b/scripts/pgso/corpus.json @@ -117,6 +117,7 @@ {"name": "verify-tui-composer-edit-contracts", "argv": ["bun", "test", "--max-concurrency", "1", "./tui-composer-edit-contracts.test.ts"], "test_file": "tui-composer-edit-contracts.test.ts"}, {"name": "verify-tui-cost", "argv": ["bun", "test", "--max-concurrency", "1", "./tui-cost.test.ts"], "test_file": "tui-cost.test.ts"}, {"name": "verify-tui-decision-prompts", "argv": ["bun", "test", "--max-concurrency", "1", "./tui-decision-prompts.test.ts"], "test_file": "tui-decision-prompts.test.ts"}, + {"name": "verify-poisoned-repository", "argv": ["bun", "test", "--max-concurrency", "1", "./poisoned-repository.test.ts"], "test_file": "poisoned-repository.test.ts"}, {"name": "verify-tui-file-picker", "argv": ["bun", "test", "--max-concurrency", "1", "./tui-file-picker.test.ts"], "test_file": "tui-file-picker.test.ts"}, {"name": "verify-tui-input-line-delete", "argv": ["bun", "test", "--max-concurrency", "1", "./tui-input-line-delete.test.ts"], "test_file": "tui-input-line-delete.test.ts"}, {"name": "verify-tui-input-navigation", "argv": ["bun", "test", "--max-concurrency", "1", "./tui-input-navigation.test.ts"], "test_file": "tui-input-navigation.test.ts"}, diff --git a/scripts/pgso/tests/test_corpus.py b/scripts/pgso/tests/test_corpus.py index e0c2336ea..57846b57f 100644 --- a/scripts/pgso/tests/test_corpus.py +++ b/scripts/pgso/tests/test_corpus.py @@ -65,6 +65,7 @@ VERIFICATION_E2E_TESTS = ( "tui-composer-edit-contracts.test.ts", "tui-cost.test.ts", "tui-decision-prompts.test.ts", + "poisoned-repository.test.ts", "tui-file-picker.test.ts", "tui-input-line-delete.test.ts", "tui-input-navigation.test.ts", @@ -372,7 +373,7 @@ class PgsoCorpusTests(unittest.TestCase): tuple(test_file for test_file, _ in corpus.intentional_exclusions), ) self.assertEqual(35, len(corpus.scenarios)) - self.assertEqual(59, len(corpus.candidate_scenarios)) + self.assertEqual(60, len(corpus.candidate_scenarios)) self.assertEqual( { "direct-help": 100, diff --git a/tests/e2e/ci-shard-weights.json b/tests/e2e/ci-shard-weights.json index d3b3bda15..b8f49b173 100644 --- a/tests/e2e/ci-shard-weights.json +++ b/tests/e2e/ci-shard-weights.json @@ -12,6 +12,7 @@ { "file": "configured-providers.test.ts", "weight": 3 }, { "file": "context-limits-live.test.ts", "weight": 1 }, { "file": "file-tool-paths.test.ts", "weight": 6 }, + { "file": "poisoned-repository.test.ts", "weight": 5 }, { "file": "file-tool-permissions.test.ts", "weight": 2 }, { "file": "gateway-stream-lifecycle.test.ts", "weight": 98 }, { "file": "host-managed-auth.test.ts", "weight": 2 }, diff --git a/tests/e2e/poisoned-repository.test.ts b/tests/e2e/poisoned-repository.test.ts new file mode 100644 index 000000000..d27fa1901 --- /dev/null +++ b/tests/e2e/poisoned-repository.test.ts @@ -0,0 +1,247 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { + chmodSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runFx } from "../evals/eval-helpers"; +import { + fakeGatewayFinalText, + fakeGatewayToolCall, + startFakeGateway, + TmuxSession, + tmuxAvailable, +} from "./tmux-helpers"; + +// A repository's own git config can name programs: fsmonitor, hooks, clean +// and smudge filters, diff drivers and the pager. Opening a repository in fx +// must never run them. A control first proves the fixture's programs run +// under plain git; every fx path below must then leave the sentinel log +// empty while still answering from the repository's real contents. + +const TIMEOUT = 60_000; +const MODEL = "openai/gpt-5"; +const tmuxTest = test.skipIf(!tmuxAvailable()); + +type Fixture = { + base: string; + home: string; + workspace: string; + log: string; +}; + +type Gateway = ReturnType; + +const fixtures: Fixture[] = []; +const gateways: Gateway[] = []; +let session: TmuxSession | null = null; + +afterEach(async () => { + await session?.kill(); + session = null; + for (const gateway of gateways.splice(0)) gateway.stop(); + for (const fixture of fixtures.splice(0)) { + rmSync(fixture.base, { recursive: true, force: true }); + } +}); + +function git(fixture: Fixture, args: string[]): string { + const result = Bun.spawnSync(["git", ...args], { + cwd: fixture.workspace, + env: { + PATH: process.env.PATH ?? "/usr/bin:/bin", + HOME: fixture.home, + GIT_CONFIG_NOSYSTEM: "1", + LC_ALL: "C", + }, + }); + if (result.exitCode !== 0) { + throw new Error(`git ${args.join(" ")} failed: ${result.stderr.toString()}`); + } + return result.stdout.toString(); +} + +function sentinelRuns(fixture: Fixture): string[] { + if (!existsSync(fixture.log)) return []; + return readFileSync(fixture.log, "utf8").split("\n").filter(Boolean); +} + +function poisonedRepository(): Fixture { + const base = realpathSync(mkdtempSync(join(tmpdir(), "fx-poisoned-repo-"))); + const fixture: Fixture = { + base, + home: join(base, "home"), + workspace: join(base, "workspace"), + log: join(base, "sentinel.log"), + }; + fixtures.push(fixture); + mkdirSync(join(fixture.home, ".fx"), { recursive: true }); + mkdirSync(join(fixture.workspace, "notes"), { recursive: true }); + writeFileSync( + join(fixture.home, ".fx", "settings.json"), + JSON.stringify({ auto_upgrade: false }), + ); + const sentinel = join(base, "sentinel.sh"); + writeFileSync(sentinel, `#!/bin/sh\necho "ran $0 $*" >> '${fixture.log}'\ncat\n`); + chmodSync(sentinel, 0o755); + const hooks = join(base, "hooks"); + mkdirSync(hooks); + for (const hook of ["post-index-change", "pre-commit", "post-checkout"]) { + writeFileSync(join(hooks, hook), readFileSync(sentinel)); + chmodSync(join(hooks, hook), 0o755); + } + + git(fixture, ["init", "-q", "--template=", "-b", "main"]); + writeFileSync(join(fixture.workspace, "alpha-tracked.txt"), "needle tracked one\n"); + writeFileSync(join(fixture.workspace, ".gitattributes"), "*.txt filter=evil diff=evil\n"); + git(fixture, ["add", "-A"]); + git(fixture, ["-c", "user.name=fixture", "-c", "user.email=fixture@example.invalid", "commit", "-q", "-m", "fixture commit"]); + for (const [key, value] of [ + ["core.fsmonitor", sentinel], + ["core.hooksPath", hooks], + ["core.pager", sentinel], + ["filter.evil.clean", sentinel], + ["filter.evil.smudge", sentinel], + ["diff.external", sentinel], + ["diff.evil.textconv", sentinel], + ]) git(fixture, ["config", key!, value!]); + writeFileSync(join(fixture.workspace, "alpha-tracked.txt"), "needle tracked two\n"); + writeFileSync(join(fixture.workspace, "notes", "alpha-untracked.txt"), "needle untracked\n"); + writeFileSync(fixture.log, ""); + return fixture; +} + +function fxEnv(fixture: Fixture, gateway: Gateway): Record { + return { + HOME: fixture.home, + AI_GATEWAY_API_KEY: "fake-poisoned-repo-key", + FX_GATEWAY_BASE_URL: gateway.baseUrl, + FX_GATEWAY_CHAT_URL: gateway.chatUrl, + FX_MODEL: MODEL, + FX_PERMISSION_MODE: "auto", + FX_SOUND: "0", + NO_COLOR: "1", + }; +} + +function toolResult(body: string, toolCallId: string): string { + const request = JSON.parse(body) as { + prompt?: Array<{ content?: Array> }>; + }; + const part = (request.prompt ?? []) + .flatMap((message) => message.content ?? []) + .find((item) => item.type === "tool-result" && item.toolCallId === toolCallId); + if (!part) throw new Error(`No tool result for ${toolCallId}`); + const output = part.output as { value?: unknown }; + return typeof output.value === "string" ? output.value : JSON.stringify(output.value); +} + +describe("poisoned repository", () => { + test("plain git runs the fixture's programs", () => { + const fixture = poisonedRepository(); + git(fixture, ["status", "--short"]); + expect(sentinelRuns(fixture).length).toBeGreaterThan(0); + }); + + tmuxTest( + "interactive startup and @ completion run none of the repository's programs", + async () => { + const fixture = poisonedRepository(); + const gateway = startFakeGateway([], { classifierResponses: [] }); + gateways.push(gateway); + const stderrPath = join(fixture.base, "stderr.log"); + session = await TmuxSession.create({ + cwd: fixture.workspace, + env: fxEnv(fixture, gateway), + stderrPath, + }); + await session.waitForComposer(TIMEOUT); + await session.sendLiteral("@alpha"); + await session.waitForPane( + (pane) => pane.includes("alpha-tracked.txt") && pane.includes("alpha-untracked.txt"), + TIMEOUT, + ); + await session.sendKeys("Escape"); + await session.sendKeys("C-u"); + await session.sendText("/quit"); + expect(await session.waitForSessionEnd(TIMEOUT)).toBe(true); + session = null; + + expect(readFileSync(stderrPath, "utf8")).toBe(""); + expect(sentinelRuns(fixture)).toEqual([]); + expect(gateway.requests).toHaveLength(0); + }, + TIMEOUT, + ); + + test( + "glob, grep and auto-run git status answer without running the repository's programs", + async () => { + const fixture = poisonedRepository(); + const gateway = startFakeGateway([ + fakeGatewayToolCall("glob", "glob_files", { pattern: "**/alpha-*.txt" }), + fakeGatewayToolCall("grep", "grep_files", { pattern: "needle" }), + fakeGatewayToolCall("status", "shell", { + request: { action: "run", command: "git status --short", profile: "clean", yield_time_ms: 30_000 }, + }), + fakeGatewayFinalText("inspection complete"), + ], { classifierResponses: [] }); + gateways.push(gateway); + + const result = await runFx( + ["ask", "--quiet", "--json", "--no-save", "Inspect the repository."], + { cwd: fixture.workspace, env: fxEnv(fixture, gateway), timeoutMs: TIMEOUT }, + ); + expect(result.code, result.stderr).toBe(0); + // `fx ask` reports each tool's activity on stderr. + expect(result.stderr).toContain("Running git status --short"); + expect(result.stderr).not.toMatch(/panic|error/i); + expect(sentinelRuns(fixture)).toEqual([]); + expect(gateway.classifierRequests).toHaveLength(0); + + const bodies = gateway.requests.map((request: { body: string }) => request.body); + const glob = toolResult(bodies[1]!, "glob"); + expect(glob).toContain("alpha-tracked.txt"); + expect(glob).toContain("notes/alpha-untracked.txt"); + const grep = toolResult(bodies[2]!, "grep"); + expect(grep).toContain("needle tracked two"); + expect(grep).toContain("needle untracked"); + const status = toolResult(bodies[3]!, "status"); + expect(status).toContain("M alpha-tracked.txt"); + expect(status).toContain("?? notes/"); + }, + TIMEOUT, + ); + + test( + "fx pr snapshots the repository without running its programs", + async () => { + const fixture = poisonedRepository(); + const gateway = startFakeGateway( + [fakeGatewayFinalText("Draft title\n\n## Summary\n\n- Draft")], + { classifierResponses: [] }, + ); + gateways.push(gateway); + + const result = await runFx(["pr"], { + cwd: fixture.workspace, + env: fxEnv(fixture, gateway), + timeoutMs: TIMEOUT, + }); + expect(result.code, result.stderr).toBe(0); + expect(sentinelRuns(fixture)).toEqual([]); + const prompt = gateway.requests[0]?.body ?? ""; + for (const expected of ["Branch: main", "M alpha-tracked.txt", "fixture commit", "alpha-tracked.txt |"]) { + expect(prompt).toContain(expected); + } + }, + TIMEOUT, + ); +}); From a23a49d91d16635cf7b8ef4afe0980f9199cc44a Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 8 Oct 2026 17:08:03 -0400 Subject: [PATCH 08/23] Classify the indexer test diagnostics in the direct-write audit - Allow the failure output of the new indexer and safe-git test helpers as test diagnostics, like the existing command_effect helpers. --- tests/e2e/render-lab/audit-direct-writes.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/e2e/render-lab/audit-direct-writes.ts b/tests/e2e/render-lab/audit-direct-writes.ts index 487951a76..b5cd44211 100644 --- a/tests/e2e/render-lab/audit-direct-writes.ts +++ b/tests/e2e/render-lab/audit-direct-writes.ts @@ -83,6 +83,9 @@ const allowlist: AllowRule[] = [ rule("src/core/shell_command/command_effect.zig", "(?:expectDirect|expectNativePrintfEquivalent)", /debug_print/, "tests", "test diagnostics"), rule("src/core/app/app_render_runtime.zig", "checkRewritePublicationRetry", /debug_print/, "tests", "test diagnostics"), rule("src/core/app/app_worker_runtime.zig", "(?:printModelTrace|printLifecycleDrainTrace)", /debug_print/, "tests", "test diagnostics"), + rule("src/core/indexer/scan.zig", "expectEntries", /debug_print/, "tests", "test diagnostics"), + rule("src/core/indexer/wildmatch.zig", "expectCases", /debug_print/, "tests", "test diagnostics"), + rule("src/core/shell_command/safe_git.zig", "(?:plainOutput|ready)", /debug_print/, "tests", "test diagnostics"), ]; function rule( From 95df7a43b1cb04c517a7e9f80f5792f893a74a27 Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 8 Oct 2026 17:15:03 -0400 Subject: [PATCH 09/23] Trim the indexer's public surface and steady a safe-git test - Make declarations used only in their own file private, and drop two front-door exports nothing uses. - Check search-root containment with pathing.pathInside. - Change a.txt without changing its size in the safe-git process-filter test, so plain git always reads the file and runs the filter there. --- src/core/indexer/git_config.zig | 6 +++--- src/core/indexer/git_index.zig | 2 +- src/core/indexer/indexer.zig | 2 -- src/core/indexer/snapshot.zig | 2 +- src/core/indexer/tree.zig | 2 +- src/core/shell_command/safe_git.zig | 6 ++++-- src/core/workspace/tool_files.zig | 9 ++++----- 7 files changed, 14 insertions(+), 15 deletions(-) diff --git a/src/core/indexer/git_config.zig b/src/core/indexer/git_config.zig index 764b9340e..958264621 100644 --- a/src/core/indexer/git_config.zig +++ b/src/core/indexer/git_config.zig @@ -18,8 +18,8 @@ const wildmatch = @import("wildmatch.zig"); const Allocator = std.mem.Allocator; -pub const max_config_file_bytes: usize = 1024 * 1024; -pub const max_include_depth: usize = 10; +const max_config_file_bytes: usize = 1024 * 1024; +const max_include_depth: usize = 10; pub const Entry = struct { /// Lowercase. @@ -213,7 +213,7 @@ fn parseValue(arena: Allocator, reader: *Reader) ParseError![]const u8 { } /// git's git_config_bool. Null for a value git rejects. -pub fn parseBool(value: ?[]const u8) ?bool { +fn parseBool(value: ?[]const u8) ?bool { const v = value orelse return true; if (v.len == 0) return false; for ([_][]const u8{ "true", "yes", "on" }) |word| { diff --git a/src/core/indexer/git_index.zig b/src/core/indexer/git_index.zig index acff32971..aa5341cf8 100644 --- a/src/core/indexer/git_index.zig +++ b/src/core/indexer/git_index.zig @@ -9,7 +9,7 @@ const bounded_read = @import("bounded_read.zig"); const Allocator = std.mem.Allocator; /// D7: the index is never read beyond this size. -pub const max_index_bytes: usize = 128 * 1024 * 1024; +const max_index_bytes: usize = 128 * 1024 * 1024; const mode_type_mask: u32 = 0o170000; const mode_gitlink: u32 = 0o160000; diff --git a/src/core/indexer/indexer.zig b/src/core/indexer/indexer.zig index 0f4f039ee..a674b27b2 100644 --- a/src/core/indexer/indexer.zig +++ b/src/core/indexer/indexer.zig @@ -17,7 +17,6 @@ const Allocator = std.mem.Allocator; pub const Kind = scan_mod.Kind; pub const Entry = scan_mod.Entry; -pub const ScanOptions = scan_mod.Options; pub const ScanResult = scan_mod.Result; pub const ScanError = scan_mod.Error; pub const default_skipped_names = scan_mod.default_skipped_names; @@ -38,7 +37,6 @@ pub const FolderStamp = tree_mod.FolderStamp; pub const isCurrent = tree_mod.isCurrent; pub const Snapshot = snapshot_mod.Snapshot; -pub const snapshot_max_bytes = snapshot_mod.max_bytes; /// Loads the saved trees for exactly `roots` from the absolute path the /// caller chose, or null when missing or not admissible. pub const loadSnapshot = snapshot_mod.load; diff --git a/src/core/indexer/snapshot.zig b/src/core/indexer/snapshot.zig index cbdd58298..50ab6537e 100644 --- a/src/core/indexer/snapshot.zig +++ b/src/core/indexer/snapshot.zig @@ -16,7 +16,7 @@ const Sha256 = std.crypto.hash.sha2.Sha256; const Tree = tree_mod.Tree; const magic = "fx-file-index-v2\n"; -pub const max_bytes: usize = 64 * 1024 * 1024; +const max_bytes: usize = 64 * 1024 * 1024; const max_path_bytes: usize = 4096; pub const Snapshot = struct { diff --git a/src/core/indexer/tree.zig b/src/core/indexer/tree.zig index c520ebba0..7d25de498 100644 --- a/src/core/indexer/tree.zig +++ b/src/core/indexer/tree.zig @@ -77,7 +77,7 @@ fn clampNs(value: anytype) i64 { return std.math.cast(i64, value) orelse if (value < 0) std.math.minInt(i64) else std.math.maxInt(i64); } -pub fn reusableTimes(mtime_ns: i64, ctime_ns: i64, scan_started_ns: i64) bool { +fn reusableTimes(mtime_ns: i64, ctime_ns: i64, scan_started_ns: i64) bool { const limit = scan_started_ns -| granularity_margin_ns; return mtime_ns < limit and ctime_ns < limit; } diff --git a/src/core/shell_command/safe_git.zig b/src/core/shell_command/safe_git.zig index 9a007f8cf..df8f0df14 100644 --- a/src/core/shell_command/safe_git.zig +++ b/src/core/shell_command/safe_git.zig @@ -135,7 +135,7 @@ pub fn environment(alloc: Allocator) Allocator.Error!std.process.Environ.Map { /// The first git found at a fixed system location. git from PATH is never /// used, because PATH can include directories a repository controls. -pub fn trustedExecutable() ?[]const u8 { +fn trustedExecutable() ?[]const u8 { const candidates = switch (builtin.os.tag) { .windows => &[_][]const u8{ "C:\\Program Files\\Git\\cmd\\git.exe", @@ -500,7 +500,9 @@ test "safe git blocks repository clean, process and smudge filters" { try repo.plain(repo.repo, &.{ "config", "--unset", "filter.evil.clean" }); try repo.plain(repo.repo, &.{ "config", "filter.evil.process", repo.sentinel }); - try repo.write("repo/a.txt", "four\n"); + // Same size as the committed "one\n": with a different size, git status + // can report the change from the index stat alone and skip the filter. + try repo.write("repo/a.txt", "ten\n"); try repo.expectBlocked(repo.repo, .status, &.{"--short"}, &.{ "status", "--short" }); try repo.plain(repo.repo, &.{ "config", "--unset", "filter.evil.process" }); diff --git a/src/core/workspace/tool_files.zig b/src/core/workspace/tool_files.zig index 50c1dbb40..e465e1d65 100644 --- a/src/core/workspace/tool_files.zig +++ b/src/core/workspace/tool_files.zig @@ -9,11 +9,12 @@ const std = @import("std"); const debug_trace = @import("../shared/debug_trace.zig"); const file_index = @import("file_index.zig"); const indexer = @import("../indexer/indexer.zig"); +const pathing = @import("pathing.zig"); const Allocator = std.mem.Allocator; pub const default_candidate_cap: usize = 100_000; -pub const max_relative_path_bytes: usize = 2048; +const max_relative_path_bytes: usize = 2048; pub const default_skipped_names: []const []const u8 = &indexer.default_skipped_names; pub const Options = struct { @@ -61,10 +62,8 @@ pub fn list(alloc: Allocator, arena: Allocator, workspace_root: []const u8, abso /// `absolute_root` relative to `workspace_root`, "" for the root itself, or /// null when it lies outside. fn relativeInside(workspace_root: []const u8, absolute_root: []const u8) ?[]const u8 { - if (std.mem.eql(u8, absolute_root, workspace_root)) return ""; - if (absolute_root.len <= workspace_root.len + 1 or !std.mem.startsWith(u8, absolute_root, workspace_root)) return null; - if (absolute_root[workspace_root.len] != '/') return null; - return absolute_root[workspace_root.len + 1 ..]; + if (workspace_root.len == 0 or !pathing.pathInside(workspace_root, absolute_root)) return null; + return std.mem.trimStart(u8, absolute_root[workspace_root.len..], "/"); } /// Whether the workspace tree's entries below `subtree` are exactly what a From 7529a42cc3834d8f07719feb128027a21da8f977 Mon Sep 17 00:00:00 2001 From: Pranit Date: Thu, 8 Oct 2026 17:29:59 -0400 Subject: [PATCH 10/23] Bound ignore-pattern matching per path - Stop matching one ignore pattern against one path after a fixed number of steps and treat it as no match, so a repository's .gitignore cannot make fx backtrack for minutes or hang on patterns such as many chained **/. --- src/core/indexer/wildmatch.zig | 35 ++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/core/indexer/wildmatch.zig b/src/core/indexer/wildmatch.zig index 003d4e258..71563f4cb 100644 --- a/src/core/indexer/wildmatch.zig +++ b/src/core/indexer/wildmatch.zig @@ -4,6 +4,7 @@ //! than tidiness here. const std = @import("std"); +const debug_trace = @import("../shared/debug_trace.zig"); pub const Flags = struct { /// `WM_PATHNAME`: `*`, `?` and bracket expressions never match `/`, and @@ -15,10 +16,22 @@ pub const Flags = struct { const Result = enum { match, no_match, abort_all, abort_to_starstar }; +/// Recursive steps one match may take. Git's algorithm backtracks +/// exponentially on patterns such as many chained `**/`, and a repository +/// chooses its own ignore patterns, so a match that runs out counts as no +/// match: the path is listed rather than hidden, and the cost per path stays +/// bounded. Ordinary patterns need a small fraction of this. +const max_steps: u32 = 20_000; + /// Reports whether `text` matches `pattern`. Like git, a NUL byte ends either /// string. pub fn match(pattern: []const u8, text: []const u8, flags: Flags) bool { - return doWild(pattern, 0, text, 0, flags) == .match; + var budget: u32 = max_steps; + const result = doWild(pattern, 0, text, 0, flags, &budget); + if (budget == 0 and result != .match) { + debug_trace.logf("indexer", "wildmatch step budget exhausted pattern_bytes={d} text_bytes={d}", .{ pattern.len, text.len }); + } + return result == .match; } fn at(s: []const u8, i: usize) u8 { @@ -41,7 +54,9 @@ fn hasSlashFrom(s: []const u8, start: usize) ?usize { return null; } -fn doWild(pattern: []const u8, p_start: usize, text: []const u8, t_start: usize, flags: Flags) Result { +fn doWild(pattern: []const u8, p_start: usize, text: []const u8, t_start: usize, flags: Flags, budget: *u32) Result { + if (budget.* == 0) return .abort_all; + budget.* -= 1; var p = p_start; var t = t_start; outer: while (at(pattern, p) != 0) : ({ @@ -76,7 +91,7 @@ fn doWild(pattern: []const u8, p_start: usize, text: []const u8, t_start: usize, (next == '\\' and at(pattern, p + 1) == '/'))) { // `**/` may match no directories at all. - if (next == '/' and doWild(pattern, p + 1, text, t, flags) == .match) return .match; + if (next == '/' and doWild(pattern, p + 1, text, t, flags, budget) == .match) return .match; match_slash = true; } else { match_slash = !flags.pathname; @@ -115,7 +130,7 @@ fn doWild(pattern: []const u8, p_start: usize, text: []const u8, t_start: usize, } if (t_ch != literal) return if (match_slash) .abort_all else .abort_to_starstar; } - const matched = doWild(pattern, p, text, t, flags); + const matched = doWild(pattern, p, text, t, flags, budget); if (matched != .no_match) { if (!match_slash or matched != .abort_to_starstar) return matched; } else if (!match_slash and t_ch == '/') { @@ -302,3 +317,15 @@ test "wildmatch case folding is ASCII-only and keeps git's escape quirk" { test "wildmatch stops at a NUL byte like git's C strings" { try std.testing.expect(match("ab\x00zz", "ab", .{ .pathname = true })); } + +test "hostile patterns stop at the step budget instead of backtracking forever" { + const long_text = "a/" ** 200 ++ "b"; + const hostile = [_][]const u8{ "*/" ** 40 ++ "c", "**/" ** 30 ++ "c", "*a*a*a*a*a*a*a*a*a*a*a*a*c", "a/**/a/**/a/**/a/**/a/**/a/**/c" }; + for (hostile) |pattern| { + try std.testing.expect(!match(pattern, long_text, .{ .pathname = true })); + try std.testing.expect(!match(pattern, long_text, .{})); + } + // Ordinary patterns on a long path still match. + try std.testing.expect(match("**/b", long_text, .{ .pathname = true })); + try std.testing.expect(match("a/**/a/b", long_text, .{ .pathname = true })); +} From 067414e06a4226a38dffb23cb783b3c653fc0b29 Mon Sep 17 00:00:00 2001 From: Pranit Date: Fri, 9 Oct 2026 01:26:46 -0400 Subject: [PATCH 11/23] Paint the @ list from a saved snapshot without per-entry path work - Turn a single primary root's saved entries into @ candidates with one check each, instead of resolving an absolute path and tracking overlaps for every entry. On large repositories the warm paint had become about 20% slower than before the indexer; it is now faster. - Keep the multi-root path, which still resolves paths and drops overlapping entries. --- src/core/workspace/file_index.zig | 85 ++++++++++++++++++++++++++++++- 1 file changed, 84 insertions(+), 1 deletion(-) diff --git a/src/core/workspace/file_index.zig b/src/core/workspace/file_index.zig index de1854955..4041a36a5 100644 --- a/src/core/workspace/file_index.zig +++ b/src/core/workspace/file_index.zig @@ -939,6 +939,9 @@ fn candidatesFromTrees( trees: []const indexer.Tree, stop_requested: *std.atomic.Value(bool), ) (Allocator.Error || error{Canceled})![]const Candidate { + if (trees.len == 1 and roots.len > 0 and std.mem.eql(u8, trees[0].root, roots[0])) { + return primaryTreeCandidates(arena, &trees[0], stop_requested); + } var candidates: std.ArrayList(Candidate) = .empty; var seen: std.StringHashMapUnmanaged(void) = .empty; for (trees) |*tree| { @@ -961,6 +964,44 @@ fn candidatesFromTrees( return candidates.toOwnedSlice(arena); } +/// The primary root alone: its entries are unique and already relative, so +/// each one is checked once and kept as is, with no absolute path or +/// overlap check per entry. This runs on every `@` paint. +fn primaryTreeCandidates( + arena: Allocator, + tree: *const indexer.Tree, + stop_requested: *std.atomic.Value(bool), +) (Allocator.Error || error{Canceled})![]const Candidate { + var candidates: std.ArrayList(Candidate) = .empty; + try candidates.ensureTotalCapacity(arena, @min(tree.entries.len, max_indexed_files)); + for (tree.entries) |entry| { + if (isStopRequested(stop_requested)) return error.Canceled; + if (candidates.items.len >= max_indexed_files) break; + if (!isPlainRelativePath(entry.path)) continue; + if (!text_utils.isTerminalSafe(entry.path)) { + debug_trace.logf("core", "file index omitted unsafe candidate bytes={d} kind={s}", .{ entry.path.len, @tagName(entry.kind) }); + continue; + } + const kind: CandidateKind = switch (entry.kind) { + .file => .file, + .directory => .directory, + }; + const accepted = acceptedSafeCandidate(.{ .path = entry.path, .kind = kind }) orelse continue; + candidates.appendAssumeCapacity(accepted); + } + return candidates.toOwnedSlice(arena); +} + +/// A relative `/`-separated path that stays inside its root and avoids +/// `.git`: no empty, `.`, `..` or `.git` component. +fn isPlainRelativePath(path: []const u8) bool { + var parts = std.mem.splitScalar(u8, path, '/'); + while (parts.next()) |part| { + if (part.len == 0 or std.mem.eql(u8, part, ".") or std.mem.eql(u8, part, "..") or std.mem.eql(u8, part, ".git")) return false; + } + return true; +} + fn hasGitComponent(path: []const u8) bool { var parts = std.mem.splitScalar(u8, path, '/'); while (parts.next()) |part| { @@ -1073,9 +1114,14 @@ fn acceptedRawPath(sep: u8, raw: []const u8) ?[]const u8 { } fn acceptedCandidate(candidate: Candidate) ?Candidate { + if (!text_utils.isTerminalSafe(candidate.path)) return null; + return acceptedSafeCandidate(candidate); +} + +/// `acceptedCandidate` for a path already known to be terminal-safe. +fn acceptedSafeCandidate(candidate: Candidate) ?Candidate { if (candidate.path.len == 0) return null; if (candidate.path.len > max_path_len) return null; - if (!text_utils.isTerminalSafe(candidate.path)) return null; if (!file_picker_path.isRepresentable(candidate.path)) return null; if (candidate.kind == .directory and std.fs.path.isSep(candidate.path[candidate.path.len - 1])) return null; return candidate; @@ -1866,6 +1912,43 @@ test "tree candidates keep a sorted directory at the exact shared cap" { try std.testing.expectEqualStrings("z-file-099998", candidates[max_indexed_files - 1].path); } +test "primary tree candidates keep plain paths and drop git, escaping and unsafe ones" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const entries = [_]indexer.Entry{ + .{ .path = ".git/config", .kind = .file }, + .{ .path = "a/../../outside", .kind = .file }, + .{ .path = "/absolute", .kind = .file }, + .{ .path = "a//b", .kind = .file }, + .{ .path = "./a", .kind = .file }, + .{ .path = "bad-\x1b[2J", .kind = .file }, + .{ .path = "src", .kind = .directory }, + .{ .path = "src/.gitignore", .kind = .file }, + .{ .path = "src/main.zig", .kind = .file }, + }; + const tree: indexer.Tree = .{ + .root = "/workspace", + .root_inode = 1, + .scan_started_ns = 0, + .repository = true, + .incomplete = false, + .cap_reached = false, + .skipped_overlong = 0, + .skipped_names = &.{}, + .entries = &entries, + .folders = &.{}, + .sources = &.{}, + }; + var stop_requested = std.atomic.Value(bool).init(false); + const candidates = try candidatesFromTrees(arena, &.{"/workspace"}, &.{tree}, &stop_requested); + try std.testing.expectEqual(@as(usize, 3), candidates.len); + try std.testing.expectEqualStrings("src", candidates[0].path); + try std.testing.expectEqual(CandidateKind.directory, candidates[0].kind); + try std.testing.expectEqualStrings("src/.gitignore", candidates[1].path); + try std.testing.expectEqualStrings("src/main.zig", candidates[2].path); +} + /// Scans `roots` without a snapshot and keeps the trees alive while the /// caller inspects candidates that borrow their memory. const TestScope = struct { From 7c8ef4a334a85be6efa4c79ffe8732dd3bccfeca Mon Sep 17 00:00:00 2001 From: Pranit Date: Fri, 9 Oct 2026 04:14:58 -0400 Subject: [PATCH 12/23] Select folders with Enter in the @ menu - Press Enter on a folder row to mention that folder and close the menu; Tab still opens the folder to browse inside. - Show an opened folder as the first row, above its contents, so it can be selected from inside, including an empty folder. --- src/core/app/app_input_runtime.zig | 27 ++++-- src/core/app/input_completion_runtime.zig | 65 +++++++++++-- src/core/workspace/path_completion.zig | 108 +++++++++++++++++----- tests/e2e/tui-file-picker.test.ts | 65 ++++++++++++- 4 files changed, 222 insertions(+), 43 deletions(-) diff --git a/src/core/app/app_input_runtime.zig b/src/core/app/app_input_runtime.zig index 3ec571527..4133ff94c 100644 --- a/src/core/app/app_input_runtime.zig +++ b/src/core/app/app_input_runtime.zig @@ -7163,30 +7163,37 @@ test "app_input_runtime file picker reuses one existing terminator and preserves } } -test "app_input_runtime accepts typed directories with one synthetic slash" { +test "app_input_runtime opens typed directories with Tab and selects them with Enter" { const alloc = std.testing.allocator; const completions = [_]file_index.Candidate{.{ .path = "src/nested", .kind = .directory }}; const cases = [_]struct { input: []const u8, cursor: usize, - expected: []const u8, - expected_cursor: usize, + opened: []const u8, + opened_cursor: usize, + selected: []const u8, + selected_cursor: usize, }{ .{ .input = "@srcn", .cursor = "@srcn".len, - .expected = "@src/nested/", - .expected_cursor = "@src/nested/".len, + .opened = "@src/nested/", + .opened_cursor = "@src/nested/".len, + .selected = "@src/nested/ ", + .selected_cursor = "@src/nested/ ".len, }, .{ .input = "before @srcn tail", .cursor = "before @srcn".len, - .expected = "before @src/nested/ tail", - .expected_cursor = "before @src/nested/".len, + .opened = "before @src/nested/ tail", + .opened_cursor = "before @src/nested/".len, + .selected = "before @src/nested/ tail", + .selected_cursor = "before @src/nested/ ".len, }, }; for ([_]u8{ '\t', '\r' }) |byte| { + const opens = byte == '\t'; for (cases) |case| { var app = try RoutingFakeApp.init(alloc); defer app.deinit(); @@ -7198,10 +7205,10 @@ test "app_input_runtime accepts typed directories with one synthetic slash" { presentRoutingFilePicker(&app); try Runtime(RoutingFakeApp).handleByte(&app, byte, 4096, 100); - try std.testing.expectEqualStrings(case.expected, app.input_runtime.edit_state.input.items); - try std.testing.expectEqual(case.expected_cursor, app.input_runtime.edit_state.cursor); + try std.testing.expectEqualStrings(if (opens) case.opened else case.selected, app.input_runtime.edit_state.input.items); + try std.testing.expectEqual(if (opens) case.opened_cursor else case.selected_cursor, app.input_runtime.edit_state.cursor); try std.testing.expectEqual(file_index.CandidateKind.directory, app.validated_file_completion_kind.?); - try std.testing.expect(app.input_runtime.picker.activeFilePickerQuery(&app.input_runtime.edit_state) != null); + try std.testing.expectEqual(opens, app.input_runtime.picker.activeFilePickerQuery(&app.input_runtime.edit_state) != null); try std.testing.expectEqual(@as(usize, 0), app.submitted_prompt_count); } } diff --git a/src/core/app/input_completion_runtime.zig b/src/core/app/input_completion_runtime.zig index a0e8f2e0f..d1d306aed 100644 --- a/src/core/app/input_completion_runtime.zig +++ b/src/core/app/input_completion_runtime.zig @@ -73,6 +73,10 @@ pub fn CompletionRuntime(comptime App: type) type { choice: file_index.SearchResult, }; + /// What accepting a folder row does: Tab opens it to list its + /// contents, Enter selects the folder itself. + const FolderAction = enum { open, select }; + pub fn slashCompletionCandidateCount(app: *App) usize { const prefix = command_specs.slashCompletionPrefix( app.slashRegistry(), @@ -885,7 +889,7 @@ pub fn CompletionRuntime(comptime App: type) type { app.shell.render_requests.request(.footer); return .inactive; }; - return applyFilePickerSelection(app, selection, max_input_len); + return applyFilePickerSelection(app, selection, max_input_len, .open); } fn resolveFilePickerSelection(app: *App) ?FilePickerSelection { @@ -899,6 +903,7 @@ pub fn CompletionRuntime(comptime App: type) type { app: *App, selection: FilePickerSelection, max_input_len: usize, + folder_action: FolderAction, ) !edit_contract.InsertResult { if (!file_picker_path.isRepresentable(selection.choice.path)) { debug_trace.logf("input", "file picker rejected unrepresentable selection bytes={d}", .{selection.choice.path.len}); @@ -920,17 +925,26 @@ pub fn CompletionRuntime(comptime App: type) type { const tail = items[replace_end..]; const reuses_terminator = tail.len > 0 and picker_state.isFilePickerTerminator(tail[0]); const is_directory = selection.choice.kind == .directory; - const adds_terminator = !is_directory and !reuses_terminator; - const encoded = try file_picker_path.encode(app.alloc, selection.choice.path, .{ + const opens_directory = is_directory and folder_action == .open; + const adds_terminator = !opens_directory and !reuses_terminator; + // An opened folder stays an active query so its contents list next. + // A selected folder is finished like a file: its path with the + // trailing slash, any quote closed, and a separating space. + const selected_folder = if (is_directory and !opens_directory) + try std.mem.concat(app.alloc, u8, &.{ selection.choice.path, "/" }) + else + null; + defer if (selected_folder) |path| app.alloc.free(path); + const encoded = try file_picker_path.encode(app.alloc, selected_folder orelse selection.choice.path, .{ .quoted = selection.query.quoted, - .directory = is_directory, + .directory = opens_directory, .workspace_relative = indexed, }); var replacement = std.ArrayList(u8).fromOwnedSlice(encoded); defer replacement.deinit(app.alloc); const separator_offset = replace_start + replacement.items.len; if (adds_terminator) try replacement.append(app.alloc, ' '); - const cursor_after_replacement = replace_start + replacement.items.len + @intFromBool(!is_directory and reuses_terminator); + const cursor_after_replacement = replace_start + replacement.items.len + @intFromBool(!opens_directory and reuses_terminator); const result = try app.input_runtime.replacementState(&app.pending_images).replaceRangeBounded( app.alloc, @@ -947,7 +961,7 @@ pub fn CompletionRuntime(comptime App: type) type { separator_offset, ); } - if (is_directory) app.input_runtime.picker.resetFilePickerIndex(); + if (opens_directory) app.input_runtime.picker.resetFilePickerIndex(); } return result; } @@ -960,7 +974,7 @@ pub fn CompletionRuntime(comptime App: type) type { if (app.input_runtime.picker.file_completion.acceptedEmpty()) return null; return .inactive; }; - const result = try applyFilePickerSelection(app, selection, max_input_len); + const result = try applyFilePickerSelection(app, selection, max_input_len, .select); debug_trace.logf( "input", "file picker Enter consumed selected={s} limit_exceeded={s}", @@ -2063,6 +2077,43 @@ test "filesystem file picker pipeline drills into a directory and selects a file try std.testing.expect(app.input_runtime.picker.activeFilePickerQuery(&app.input_runtime.edit_state) == null); } +test "filesystem file picker lists an opened folder first and Enter selects it" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.createDirPath(std.testing.io, "workspace/space dir"); + try tmp.dir.writeFile(std.testing.io, .{ + .sub_path = "workspace/space dir/item.txt", + .data = "inside", + }); + + const io_mod = @import("../shared/io.zig"); + const workspace_root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "workspace"); + defer alloc.free(workspace_root); + + const rt = CompletionRuntime(FilesystemFilePickerTestApp); + var app = FilesystemFilePickerTestApp{ + .alloc = alloc, + .workspace_root = workspace_root, + }; + defer app.deinit(); + + try app.input_runtime.textReplacementState().replace(alloc, "@./space"); + presentFilePickerForTest(&app); + try std.testing.expectEqual(edit_contract.InsertResult.inserted, try rt.autocompleteFilePickerSelection(&app, 4096)); + try std.testing.expectEqualStrings("@\"./space dir/", app.input_runtime.edit_state.input.items); + + presentFilePickerForTest(&app); + const folder = app.input_runtime.picker.file_completion.selected(0).?; + try std.testing.expectEqualStrings("./space dir", folder.path); + try std.testing.expectEqual(file_index.CandidateKind.directory, folder.kind); + try std.testing.expectEqualStrings("./space dir/item.txt", app.input_runtime.picker.file_completion.selected(1).?.path); + + try std.testing.expectEqual(edit_contract.InsertResult.inserted, (try rt.submitFilePickerOnEnter(&app, 4096)).?); + try std.testing.expectEqualStrings("@\"./space dir/\" ", app.input_runtime.edit_state.input.items); + try std.testing.expect(app.input_runtime.picker.activeFilePickerQuery(&app.input_runtime.edit_state) == null); +} + test "file picker loading Tab preserves pending request and Enter never queues acceptance" { const alloc = std.testing.allocator; const rt = CompletionRuntime(FilePickerTestApp); diff --git a/src/core/workspace/path_completion.zig b/src/core/workspace/path_completion.zig index 1785fa813..921d58530 100644 --- a/src/core/workspace/path_completion.zig +++ b/src/core/workspace/path_completion.zig @@ -76,6 +76,13 @@ pub fn completeCancellable( var dir = std.Io.Dir.openDirAbsolute(io, resolved, .{ .iterate = true }) catch return error.PathUnavailable; defer dir.close(io); + // Listing a folder's contents: the folder itself is the first row, so it + // can be selected and not only opened. + const folder = listedFolder(parsed); + const reserved: usize = @intFromBool(folder != null); + const entries = out[reserved..]; + const entry_storage = path_storage[reserved * slot_len ..]; + var count: usize = 0; var iterator = dir.iterate(); while (true) { @@ -96,11 +103,11 @@ pub fn completeCancellable( const candidate = candidate_storage[0..candidate_len]; if (!text_utils.isTerminalSafe(candidate)) continue; if (!file_picker_path.isRepresentable(candidate)) continue; - count = insertCandidate(out, path_storage, &scores, &matcher, count, candidate, kind, score); + count = insertCandidate(entries, entry_storage, &scores, &matcher, count, candidate, kind, score); } var spans_used: usize = 0; - for (out[0..count]) |*result| { + for (entries[0..count]) |*result| { try checkCancellation(cancel); const span_count = matcher.match_spans(result.path[parsed.display_prefix.len..], match_spans[spans_used..]) catch |err| switch (err) { error.NoSpaceLeft => return error.NoSpaceLeft, @@ -114,8 +121,25 @@ pub fn completeCancellable( result.matched_spans = spans; spans_used += span_count; } + if (folder) |path| { + const slot = pathSlot(path_storage, 0); + @memcpy(slot[0..path.len], path); + out[0] = .{ .path = slot[0..path.len], .kind = .directory, .matched_spans = &.{} }; + } try checkCancellation(cancel); - return count; + return reserved + count; +} + +/// The folder whose contents a query lists (`src/` lists `src`, `~` lists +/// `~`), or null when the query names a partial entry or the root. +fn listedFolder(parsed: ParsedQuery) ?[]const u8 { + if (parsed.basename_query.len != 0) return null; + const prefix = parsed.display_prefix; + if (prefix.len < 2 or !std.fs.path.isSep(prefix[prefix.len - 1])) return null; + const folder = prefix[0 .. prefix.len - 1]; + if (std.fs.path.isSep(folder[folder.len - 1])) return null; + if (!text_utils.isTerminalSafe(folder) or !file_picker_path.isRepresentable(folder)) return null; + return folder; } fn checkCancellation(cancel: ?*const std.atomic.Value(bool)) error{Cancelled}!void { @@ -292,15 +316,18 @@ test "path completion browses bare current and parent directories" { var spans: [4]file_index.MatchSpan = undefined; var paths: [4 * file_index.max_path_len]u8 = undefined; - try std.testing.expectEqual(@as(usize, 1), try complete(root, ".", &results, &spans, &paths)); - try std.testing.expectEqualStrings("./local.txt", results[0].path); - try std.testing.expectEqual(@as(usize, 0), results[0].matched_spans.len); - try std.testing.expect(isCurrentCandidateKind(root, results[0].path, .file)); + try std.testing.expectEqual(@as(usize, 2), try complete(root, ".", &results, &spans, &paths)); + try std.testing.expectEqualStrings(".", results[0].path); + try std.testing.expectEqual(file_index.CandidateKind.directory, results[0].kind); + try std.testing.expectEqualStrings("./local.txt", results[1].path); + try std.testing.expectEqual(@as(usize, 0), results[1].matched_spans.len); + try std.testing.expect(isCurrentCandidateKind(root, results[1].path, .file)); - try std.testing.expectEqual(@as(usize, 2), try complete(root, "..", &results, &spans, &paths)); - try std.testing.expectEqualStrings("../parent.txt", results[0].path); - try std.testing.expectEqualStrings("../workspace", results[1].path); - try std.testing.expect(isCurrentCandidateKind(root, results[1].path, .directory)); + try std.testing.expectEqual(@as(usize, 3), try complete(root, "..", &results, &spans, &paths)); + try std.testing.expectEqualStrings("..", results[0].path); + try std.testing.expectEqualStrings("../parent.txt", results[1].path); + try std.testing.expectEqualStrings("../workspace", results[2].path); + try std.testing.expect(isCurrentCandidateKind(root, results[2].path, .directory)); } test "path completion enumerates immediate entries with deterministic bounded order" { @@ -317,15 +344,16 @@ test "path completion enumerates immediate entries with deterministic bounded or const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "workspace"); defer alloc.free(root); - var results: [3]file_index.SearchResult = undefined; - var spans: [3]file_index.MatchSpan = undefined; - var paths: [3 * file_index.max_path_len]u8 = undefined; + var results: [4]file_index.SearchResult = undefined; + var spans: [4]file_index.MatchSpan = undefined; + var paths: [4 * file_index.max_path_len]u8 = undefined; const count = try complete(root, "./src/", &results, &spans, &paths); - try std.testing.expectEqual(@as(usize, 3), count); - try std.testing.expectEqualStrings("./src/.hidden.txt", results[0].path); - try std.testing.expectEqualStrings("./src/Alpha.txt", results[1].path); - try std.testing.expectEqualStrings("./src/beta.txt", results[2].path); + try std.testing.expectEqual(@as(usize, 4), count); + try std.testing.expectEqualStrings("./src", results[0].path); + try std.testing.expectEqualStrings("./src/.hidden.txt", results[1].path); + try std.testing.expectEqualStrings("./src/Alpha.txt", results[2].path); + try std.testing.expectEqualStrings("./src/beta.txt", results[3].path); const filtered_count = try complete(root, "src/al", &results, &spans, &paths); try std.testing.expectEqual(@as(usize, 2), filtered_count); @@ -355,10 +383,11 @@ test "path completion resolves parent and absolute forms without recursive trave var spans: [8]file_index.MatchSpan = undefined; var paths: [8 * file_index.max_path_len]u8 = undefined; const parent_count = try complete(root, "../outside/", &results, &spans, &paths); - try std.testing.expectEqual(@as(usize, 2), parent_count); - try std.testing.expectEqualStrings("../outside/empty", results[0].path); - try std.testing.expectEqual(file_index.CandidateKind.directory, results[0].kind); - try std.testing.expectEqualStrings("../outside/external.txt", results[1].path); + try std.testing.expectEqual(@as(usize, 3), parent_count); + try std.testing.expectEqualStrings("../outside", results[0].path); + try std.testing.expectEqualStrings("../outside/empty", results[1].path); + try std.testing.expectEqual(file_index.CandidateKind.directory, results[1].kind); + try std.testing.expectEqualStrings("../outside/external.txt", results[2].path); var absolute_query_storage: [file_index.max_path_len]u8 = undefined; const absolute_query = try std.fmt.bufPrint(&absolute_query_storage, "{s}/ex", .{outside}); @@ -369,6 +398,41 @@ test "path completion resolves parent and absolute forms without recursive trave try std.testing.expectEqualStrings(expected, results[0].path); } +test "path completion lists a folder first so it can be selected" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try writeTestFile(tmp.dir, "workspace/src/main.zig"); + try writeTestFile(tmp.dir, "home/notes.txt"); + const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "workspace"); + defer alloc.free(root); + const home = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "home"); + defer alloc.free(home); + var results: [4]file_index.SearchResult = undefined; + var spans: [4]file_index.MatchSpan = undefined; + var paths: [4 * file_index.max_path_len]u8 = undefined; + + try std.testing.expectEqual(@as(usize, 2), try complete(root, "src/", &results, &spans, &paths)); + try std.testing.expectEqualStrings("src", results[0].path); + try std.testing.expectEqual(file_index.CandidateKind.directory, results[0].kind); + try std.testing.expectEqual(@as(usize, 0), results[0].matched_spans.len); + try std.testing.expectEqualStrings("src/main.zig", results[1].path); + try std.testing.expect(isCurrentCandidateKind(root, results[0].path, .directory)); + + // A partial name lists matching entries only. + try std.testing.expectEqual(@as(usize, 1), try complete(root, "src/ma", &results, &spans, &paths)); + try std.testing.expectEqualStrings("src/main.zig", results[0].path); + + var cancel: std.atomic.Value(bool) = .init(false); + try std.testing.expectEqual(@as(usize, 2), try completeCancellable(root, home, "~", &cancel, &results, &spans, &paths)); + try std.testing.expectEqualStrings("~", results[0].path); + try std.testing.expectEqualStrings("~/notes.txt", results[1].path); + + // With room for one row, the folder itself is kept. + try std.testing.expectEqual(@as(usize, 1), try complete(root, "src/", results[0..1], &spans, paths[0..file_index.max_path_len])); + try std.testing.expectEqualStrings("src", results[0].path); +} + test "path completion follows listed symlinks and filters unsafe names" { if (comptime builtin.os.tag == .windows or builtin.os.tag == .wasi) return error.SkipZigTest; diff --git a/tests/e2e/tui-file-picker.test.ts b/tests/e2e/tui-file-picker.test.ts index 1a3bf6158..26955b4dd 100644 --- a/tests/e2e/tui-file-picker.test.ts +++ b/tests/e2e/tui-file-picker.test.ts @@ -129,6 +129,16 @@ function processRssKb(pid: number): number { return rss; } +// The @ menu rows: the lines between the two rules below the composer. +function pickerRows(grid: string[]): string[] { + const composer = grid.findIndex(isComposerLine); + const rules = grid.flatMap((line, index) => + index > composer && /^─+$/.test(line.trim()) ? [index] : [] + ); + if (composer < 0 || rules.length < 2) return []; + return grid.slice(rules[0]! + 1, rules[1]!).map((line) => line.trim()).filter(Boolean); +} + function composerTextFromPane(pane: string): string { return pane.split("\n") .filter(isComposerLine) @@ -428,6 +438,46 @@ async function replayTape(current: Fixture): Promise { } describe("@ file picker", () => { + tmuxTest( + "Tab opens a folder that lists itself first, and Enter selects a folder", + async () => { + const current = createFixture("fx-file-picker-folder-"); + mkdirSync(join(current.workspace, "notes", "drafts"), { recursive: true }); + writeFileSync(join(current.workspace, "notes", "todo.md"), "todo"); + writeFileSync(join(current.workspace, "notes", "drafts", "idea.md"), "idea"); + const active = await startMockFx(current, [], 1000); + + await active.sendLiteral("Review @notes"); + await active.waitForText("notes/", TIMEOUT); + await active.sendKeys("Tab"); + await active.waitForText("notes/todo.md", TIMEOUT); + expect(await composerPlainText(active)).toBe("Review @notes/"); + expect(pickerRows(await active.capturePaneGrid())).toEqual(["notes/", "notes/drafts/", "notes/todo.md"]); + + // Enter on the first row selects the opened folder itself. + await active.sendKeys("Enter"); + await active.sendLiteral("please"); + await active.waitForPane((pane) => composerTextFromPane(pane) === "Review @notes/ please", TIMEOUT); + expect(pickerRows(await active.capturePaneGrid())).toEqual([]); + + // Enter on a listed child folder selects that folder. + await clearComposer(active); + await active.sendLiteral("@notes/"); + await active.waitForText("notes/drafts/", TIMEOUT); + await active.sendKeys("Down"); + await active.sendKeys("Enter"); + await active.sendLiteral("x"); + await active.waitForPane((pane) => composerTextFromPane(pane) === "@notes/drafts/ x", TIMEOUT); + expect(gateway?.requests ?? []).toHaveLength(0); + expectCleanRuntime(current, active); + await clearComposer(active); + await active.sendText("/quit"); + expect(await active.waitForSessionEnd(TIMEOUT)).toBe(true); + session = null; + }, + TIMEOUT, + ); + tmuxTest( "opens and selects a file while a turn is in flight", async () => { @@ -1651,7 +1701,8 @@ describe("@ file picker", () => { expect(await composerPlainText(active)).toBe("@./second/"); expect(starts()).toBe(2); expect(gateway.requests).toHaveLength(0); - await active.sendKeys("Tab"); + // The first row is the opened folder itself; its file follows. + await active.sendKeys("Down Tab"); expect(await composerPlainText(active)).toBe("@./second/beta.txt"); await clearComposer(active); const scrollback = await active.captureFullScrollbackEscapes(); @@ -1722,7 +1773,8 @@ describe("@ file picker", () => { await active.waitForText("Selection unavailable", TIMEOUT); expect(await composerPlainText(active)).toBe("@./retry/"); expect(gateway?.requests).toHaveLength(0); - await active.sendKeys("Down Enter"); + // The rows are now the folder and b.txt; move from the stale slot to b.txt. + await active.sendKeys("Down Down Enter"); expect(await composerPlainText(active)).toBe("@./retry/b.txt"); await clearComposer(active); @@ -1736,7 +1788,8 @@ describe("@ file picker", () => { await active.sendKeys("Tab"); await active.waitForText("./missing/recovered.txt", TIMEOUT); expect(await composerPlainText(active)).toBe("@./missing/"); - await active.sendKeys("Tab"); + // The recovered folder lists itself first; its file follows. + await active.sendKeys("Down Tab"); expect(await composerPlainText(active)).toBe("@./missing/recovered.txt"); await active.sendLiteral(" Reply with the words PRESENTED, PICKER, and OK joined by underscores."); await active.sendKeys("Enter"); @@ -1863,7 +1916,8 @@ describe("@ file picker", () => { await clearComposer(active); await active.sendLiteral("@~"); await active.waitForText("~/home-target.txt", TIMEOUT); - await active.sendKeys("Down Enter"); + // Rows: the home folder itself, ~/.fx/, then ~/home-target.txt. + await active.sendKeys("Down Down Enter"); expect(await composerPlainText(active)).toBe("@~/home-target.txt"); expect(gateway?.requests).toHaveLength(0); await active.sendLiteral(" Reply with the words SHORTCUT, FLOW, and OK joined by underscores."); @@ -1934,6 +1988,9 @@ describe("@ file picker", () => { await active.waitForText("empty-workspace/", TIMEOUT); await active.sendKeys("Tab"); expect(await composerPlainText(active)).toBe("@empty-workspace/"); + // An opened empty folder still lists itself, so it can be selected. + await active.waitForPane((pane) => pickerRows(pane.split("\n")).join("|") === "empty-workspace/", TIMEOUT); + await active.sendLiteral("zzz"); await active.waitForText("no matching files", TIMEOUT); await active.sendKeys("Escape"); await active.waitForPane((pane) => !pane.includes("no matching files"), TIMEOUT); From 09d2e9340e30b9c409d11a095c84029637c01c0f Mon Sep 17 00:00:00 2001 From: Pranit Date: Fri, 9 Oct 2026 06:27:03 -0400 Subject: [PATCH 13/23] Paint @ folder rows with the keystroke that lists them - Hold the frame for a keystroke that starts a folder listing until its rows arrive, up to one 8 ms event-loop tick, so the menu no longer flashes an empty "indexing files..." list first. - Check back every millisecond while that frame waits, so folder rows appear about 2 ms after the keystroke instead of 9. - Skip the wait once a listing outlasts it, so typing in a slow folder paints each keystroke at once. --- src/core/app/app_input_runtime.zig | 2 + src/core/app/input_completion_runtime.zig | 62 +++++++++++++++++++++ src/core/input/picker_state.zig | 60 +++++++++++++++++++++ src/main.zig | 20 ++++--- tests/e2e/tui-file-picker.test.ts | 65 +++++++++++++++++++++++ 5 files changed, 201 insertions(+), 8 deletions(-) diff --git a/src/core/app/app_input_runtime.zig b/src/core/app/app_input_runtime.zig index 4133ff94c..e796cac19 100644 --- a/src/core/app/app_input_runtime.zig +++ b/src/core/app/app_input_runtime.zig @@ -837,6 +837,8 @@ pub fn Runtime(comptime App: type) type { pub const prepareFilePicker = completion_rt.prepareFilePicker; pub const collectFilePickerFacts = completion_rt.collectFilePickerFacts; + pub const holdFrameForFileListing = completion_rt.holdFrameForFileListing; + pub const fileListingHoldPending = completion_rt.fileListingHoldPending; pub fn handleByte(app: *App, byte: u8, max_input_len: usize, max_prompt_history: usize) !void { return handleTerminalByteWithLimits( diff --git a/src/core/app/input_completion_runtime.zig b/src/core/app/input_completion_runtime.zig index d1d306aed..34426f17d 100644 --- a/src/core/app/input_completion_runtime.zig +++ b/src/core/app/input_completion_runtime.zig @@ -778,6 +778,33 @@ pub fn CompletionRuntime(comptime App: type) type { app.harvestDirectoryCompletion(filePickerEligible(app)); } + /// Returns true while the frame for a keystroke that started a folder + /// listing should wait for its rows, so they paint together. + pub fn holdFrameForFileListing(app: *App, now_ms: i64) bool { + if (comptime !runtime_profile.allows(App, .file_index)) return false; + app.harvestDirectoryCompletion(filePickerEligible(app)); + const picker = &app.input_runtime.picker; + const state = &picker.file_completion; + if (!state.active) { + picker.file_listing_wait = .{}; + return false; + } + const in_flight = !state.indexed and state.directory_request != null; + const was_holding = picker.file_listing_wait.active(now_ms); + const hold = picker.file_listing_wait.hold(in_flight, now_ms); + if (was_holding and !hold and in_flight) { + debug_trace.logf("input", "file picker painted before folder listing request={d}", .{state.directory_request.?}); + } + return hold; + } + + /// While a frame waits for a folder listing, the event loop checks back + /// every millisecond instead of every tick. + pub fn fileListingHoldPending(app: *const App, now_ms: i64) bool { + if (comptime !runtime_profile.allows(App, .file_index)) return false; + return app.input_runtime.picker.file_listing_wait.active(now_ms); + } + /// The sole acquisition boundary. Called after input batching, never by /// painters, navigation or acceptance. Directory work is scheduled, not run. pub fn prepareFilePicker(app: *App) void { @@ -2114,6 +2141,41 @@ test "filesystem file picker lists an opened folder first and Enter selects it" try std.testing.expect(app.input_runtime.picker.activeFilePickerQuery(&app.input_runtime.edit_state) == null); } +test "filesystem file picker releases a held folder frame only with its rows" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.createDirPath(std.testing.io, "workspace/tree"); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "workspace/tree/leaf.txt", .data = "leaf" }); + + const io_mod = @import("../shared/io.zig"); + const workspace_root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "workspace"); + defer alloc.free(workspace_root); + + const rt = CompletionRuntime(FilesystemFilePickerTestApp); + var app = FilesystemFilePickerTestApp{ + .alloc = alloc, + .workspace_root = workspace_root, + }; + defer app.deinit(); + + try app.input_runtime.textReplacementState().replace(alloc, "@./tree/"); + rt.prepareFilePicker(&app); + // The clock stands still, so the wait can end only because the rows arrived. + const now_ms: i64 = 1_000; + var polls: usize = 0; + while (rt.holdFrameForFileListing(&app, now_ms)) : (polls += 1) { + if (polls == 5_000) return error.TestUnexpectedResult; + std.Io.sleep(std.testing.io, .fromMilliseconds(1), .awake) catch unreachable; + } + const view = rt.filePickerView(&app); + try std.testing.expectEqual(file_completion_state.Status.ready, view.status); + try std.testing.expectEqual(@as(usize, 2), view.items.len); + try std.testing.expectEqualStrings("./tree", view.items[0].path); + try std.testing.expectEqualStrings("./tree/leaf.txt", view.items[1].path); + try std.testing.expect(!rt.fileListingHoldPending(&app, now_ms)); +} + test "file picker loading Tab preserves pending request and Enter never queues acceptance" { const alloc = std.testing.allocator; const rt = CompletionRuntime(FilePickerTestApp); diff --git a/src/core/input/picker_state.zig b/src/core/input/picker_state.zig index 1cdcd3590..9e68340b1 100644 --- a/src/core/input/picker_state.zig +++ b/src/core/input/picker_state.zig @@ -87,6 +87,41 @@ pub const InlineSlashQuery = struct { prefix: []const u8, }; +/// Paces the frame for a keystroke that started a folder listing. Most folders +/// list well within one event-loop tick, so that frame waits for the rows and +/// the keystroke paints together with them instead of over an empty +/// "indexing files..." list. Times are monotonic milliseconds. +const FileListingWait = struct { + const hold_ms: i64 = 8; + + started_ms: ?i64 = null, + // The previous listing outlasted the wait, so the next keystroke paints at + // once instead of paying the wait again on a slow folder. + last_outlasted_hold: bool = false, + + /// Returns true while the next frame should wait for an in-flight listing. + /// Keystrokes during one listing share its wait. + pub fn hold(self: *FileListingWait, listing_in_flight: bool, now_ms: i64) bool { + if (!listing_in_flight) { + if (self.started_ms) |started| self.last_outlasted_hold = !withinHold(started, now_ms); + self.started_ms = null; + return false; + } + if (self.started_ms == null) self.started_ms = now_ms; + return self.active(now_ms); + } + + pub fn active(self: FileListingWait, now_ms: i64) bool { + const started = self.started_ms orelse return false; + return !self.last_outlasted_hold and withinHold(started, now_ms); + } + + fn withinHold(started_ms: i64, now_ms: i64) bool { + // A clock that steps backwards ends the wait instead of extending it. + return now_ms >= started_ms and now_ms - started_ms < hold_ms; + } +}; + /// Owns layout-independent composer picker state. Call `deinit` with the /// allocator used by `beginModelPickerFlow`. pub const State = struct { @@ -118,6 +153,7 @@ pub const State = struct { file_completion_index: usize = 0, file_completion_window_start: usize = 0, file_picker_episode_seen: bool = false, + file_listing_wait: FileListingWait = .{}, pub fn initInto(self: *State) void { inline for (std.meta.fields(State)) |field| { @@ -1011,3 +1047,27 @@ test "committed tokens only match at word boundaries" { state.reconcileInlinePickerAfterEdit(&editor); try std.testing.expectEqual(ProviderPickerStage.provider, state.provider_picker_stage); } + +test "folder listing wait spans one tick and is skipped after a slow listing" { + var wait: FileListingWait = .{}; + try std.testing.expect(!wait.hold(false, 100)); + // Keystrokes during one listing share the first keystroke's wait. + try std.testing.expect(wait.hold(true, 100)); + try std.testing.expect(wait.hold(true, 100 + FileListingWait.hold_ms - 1)); + try std.testing.expect(wait.active(100 + FileListingWait.hold_ms - 1)); + // A listing that outlasts the wait paints at once from then on. + try std.testing.expect(!wait.hold(true, 100 + FileListingWait.hold_ms)); + try std.testing.expect(!wait.hold(true, 140)); + try std.testing.expect(!wait.active(140)); + try std.testing.expect(!wait.hold(false, 141)); + // The folder was slow, so the next keystroke does not wait for it. + try std.testing.expect(!wait.hold(true, 200)); + try std.testing.expect(!wait.active(200)); + // A listing that finishes within the wait turns waiting back on. + try std.testing.expect(!wait.hold(false, 202)); + try std.testing.expect(wait.hold(true, 300)); + try std.testing.expect(!wait.hold(false, 301)); + try std.testing.expect(wait.hold(true, 400)); + // A clock that steps backwards ends the wait. + try std.testing.expect(!wait.hold(true, 350)); +} diff --git a/src/main.zig b/src/main.zig index 85cdaa557..557c8a8b9 100644 --- a/src/main.zig +++ b/src/main.zig @@ -196,11 +196,13 @@ fn nativeLoopPollTimeoutMs( auth_refresh_active: bool, skills_refresh_active: bool, transcript_page_work_active: bool, + file_listing_hold_pending: bool, ) i32 { return if (first_frame_pending or auth_refresh_active or skills_refresh_active or - transcript_page_work_active) + transcript_page_work_active or + file_listing_hold_pending) @min(default_timeout_ms, focused_ui_worker_poll_timeout_ms) else default_timeout_ms; @@ -1153,6 +1155,7 @@ const App = struct { self.auth.sourceInventoryRefreshActive(), self.skills.refreshActive(), self.fullTranscriptFocusedWorkActive(), + InputAppRuntime.fileListingHoldPending(self, app_permission_runtime.monotonicMillis()), ); } return if (self.pacer.hasPending()) default_timeout_ms else idle_wasm_poll_timeout_ms; @@ -3226,7 +3229,7 @@ const App = struct { } _ = try InputAppRuntime.flushDeferredSessionInput(self, input_limits, max_prompt_history); if (self.should_exit) return; - try self.flushRequestedFrame(); + if (!InputAppRuntime.holdFrameForFileListing(self, app_permission_runtime.monotonicMillis())) try self.flushRequestedFrame(); if (comptime !host_target.is_wasm) { try SessionAppRuntime.settlePendingLiveSessionTransition(self); if (try InputAppRuntime.flushDeferredSessionInput(self, input_limits, max_prompt_history)) try self.flushRequestedFrame(); @@ -3850,12 +3853,13 @@ test "lightweight local commands do not request early threaded io" { } test "focused UI workers retain a bounded native poll timeout" { - try std.testing.expectEqual(@as(i32, 8), nativeLoopPollTimeoutMs(8, false, false, false, false)); - try std.testing.expectEqual(@as(i32, 1), nativeLoopPollTimeoutMs(8, true, false, false, false)); - try std.testing.expectEqual(@as(i32, 1), nativeLoopPollTimeoutMs(8, false, true, false, false)); - try std.testing.expectEqual(@as(i32, 1), nativeLoopPollTimeoutMs(8, false, false, true, false)); - try std.testing.expectEqual(@as(i32, 1), nativeLoopPollTimeoutMs(8, false, false, false, true)); - try std.testing.expectEqual(@as(i32, 1), nativeLoopPollTimeoutMs(8, true, true, true, true)); + try std.testing.expectEqual(@as(i32, 8), nativeLoopPollTimeoutMs(8, false, false, false, false, false)); + try std.testing.expectEqual(@as(i32, 1), nativeLoopPollTimeoutMs(8, true, false, false, false, false)); + try std.testing.expectEqual(@as(i32, 1), nativeLoopPollTimeoutMs(8, false, true, false, false, false)); + try std.testing.expectEqual(@as(i32, 1), nativeLoopPollTimeoutMs(8, false, false, true, false, false)); + try std.testing.expectEqual(@as(i32, 1), nativeLoopPollTimeoutMs(8, false, false, false, true, false)); + try std.testing.expectEqual(@as(i32, 1), nativeLoopPollTimeoutMs(8, false, false, false, false, true)); + try std.testing.expectEqual(@as(i32, 1), nativeLoopPollTimeoutMs(8, true, true, true, true, true)); } test "footer runtime compatibility facade exports composeFooterFrame" { diff --git a/tests/e2e/tui-file-picker.test.ts b/tests/e2e/tui-file-picker.test.ts index 26955b4dd..676254df4 100644 --- a/tests/e2e/tui-file-picker.test.ts +++ b/tests/e2e/tui-file-picker.test.ts @@ -5,6 +5,7 @@ import { existsSync, mkdirSync, mkdtempSync, + readdirSync, readFileSync, realpathSync, rmSync, @@ -478,6 +479,70 @@ describe("@ file picker", () => { TIMEOUT, ); + tmuxTest( + "paints a folder's rows with the keystroke instead of a loading frame", + async () => { + const current = createFixture("fx-file-picker-folder-frame-"); + mkdirSync(join(current.workspace, "tree", "inner"), { recursive: true }); + writeFileSync(join(current.workspace, "tree", "top.txt"), "top"); + writeFileSync(join(current.workspace, "tree", "inner", "leaf-a.txt"), "a"); + writeFileSync(join(current.workspace, "tree", "inner", "leaf-b.txt"), "b"); + const active = await startMockFx(current, [], 1000); + + await active.sendLiteral("@./tree/"); + await active.waitForText("./tree/top.txt", TIMEOUT); + await active.sendKeys("Down"); + await active.sendKeys("Tab"); + await active.waitForText("./tree/inner/leaf-a.txt", TIMEOUT); + await active.sendLiteral("leaf-b"); + await active.waitForPane((pane) => pickerRows(pane.split("\n")).join("|") === "./tree/inner/leaf-b.txt", TIMEOUT); + expectCleanRuntime(current, active); + await clearComposer(active); + await active.sendText("/quit"); + expect(await active.waitForSessionEnd(TIMEOUT)).toBe(true); + session = null; + + // Every frame fx wrote, interleaved with the keystrokes that caused them. + const framesDir = join(current.root, "frames"); + const replay = await runFx(["replay", current.tapePath, "--frames-dir", framesDir], { + cwd: current.workspace, env: { HOME: current.home }, timeoutMs: TIMEOUT, + }); + expect(replay.code).toBe(0); + const frames = readdirSync(join(framesDir, "frames")) + .filter((name) => name.endsWith(".json")) + .map((name) => JSON.parse(readFileSync(join(framesDir, "frames", name), "utf8")) as { + index: number; + elapsed_ms: number; + kind: string; + }) + .sort((a, b) => a.index - b.index); + let lastInputMs = Number.NEGATIVE_INFINITY; + let folderRowsPainted = false; + for (const frame of frames) { + if (frame.kind === "stdin") { + lastInputMs = frame.elapsed_ms; + continue; + } + if (frame.kind !== "stdout") continue; + // Grid files frame each terminal row as |row|. + const grid = readFileSync( + join(framesDir, "frames", `${String(frame.index).padStart(4, "0")}.grid.txt`), + "utf8", + ).split("\n").map((line) => line.replace(/^\|(.*)\|$/, "$1")); + if (!composerTextFromPane(grid.join("\n")).startsWith("@./tree/")) continue; + if (pickerRows(grid).includes("./tree/inner/leaf-a.txt")) folderRowsPainted = true; + // A folder that lists within one 8 ms tick paints together with its + // keystroke. A loading frame is allowed only once that wait has run + // out; 7 allows for millisecond rounding between the tape and the wait. + if (grid.some((line) => line.includes("indexing files..."))) { + expect(frame.elapsed_ms - lastInputMs).toBeGreaterThanOrEqual(7); + } + } + expect(folderRowsPainted).toBe(true); + }, + TIMEOUT, + ); + tmuxTest( "opens and selects a file while a turn is in flight", async () => { From a77d458a421f9de0b3e3555824df4c5f89fe045a Mon Sep 17 00:00:00 2001 From: Pranit Date: Sun, 11 Oct 2026 05:36:25 -0400 Subject: [PATCH 14/23] Hold auto-run git when safe git refuses - In auto mode, a git command that safe git refuses (repository config fx cannot read, git choosing a different repository, or git older than 2.25) now returns an explanation to the agent instead of running plain git through the routine-command rule. Ask mode still prompts. - Allow git 2.25 and later in safe git. On 2.25 and 2.34 its flags turn off fsmonitor, filters, diff drivers, textconv, signature programs and hooks, as on current git. - Test the hold end to end with a repository config included through a symlink, and test that safe git refuses an inner repository whose HEAD is a symlink. - State in AGENTS.md which git runs go through safe git, and that git run as a shell command uses the repository's full config. --- AGENTS.md | 2 +- src/core/shell_command/safe_git.zig | 24 +++++++--- src/core/tooling/tool_admission.zig | 63 ++++++++++++++++++++++++++- tests/e2e/poisoned-repository.test.ts | 32 ++++++++++++++ 4 files changed, 113 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5f90915bf..887aaa16f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,7 +71,7 @@ Key rules: * `src/core/indexer/` is the native workspace indexer. It reads git's ignore rules, repository layout, and git config as data and never starts a process, so a repository's configuration cannot make it run a program. Outside code imports only its front door, `indexer.zig`, and the indexer imports only the standard library, its own files, `io.zig`, and `debug_trace.zig`. `@`-completion, `glob_files`, and `grep_files` list files only through it and never start a process. `scripts/check-indexer-boundary.sh` enforces this in CI. -* `src/core/shell_command/safe_git.zig` is the only way fx runs git itself, for the agent's auto-run read-only git commands and the `fx pr` and `fx issue` snapshot. It turns off every program a repository's own config can name and refuses to run when git would use a different repository than the one fx read. +* `src/core/shell_command/safe_git.zig` runs git for the `fx pr` and `fx issue` snapshot and for the agent's direct read-only `git status`, `git diff`, and `git log` when the command does not use the user's own shell environment. It turns off every program a repository's own config can name and refuses to run when git would use a different repository than the one fx read. In `auto` mode a git command it refuses is returned to the agent without running. Commands that run in a shell use git with the repository's full config: git in the shell tool's default `user` environment, and the routine commands `auto` mode runs without review, such as `git status` with other flags, `git fetch`, or `npm test`. * `src/tools/` owns built-in tool implementations. Generic tool contracts and dispatch live in `src/core/tooling/`. Default tool specs are centralized in `src/core/tooling/tool_specs.zig` or `src/builtins/tools.zig`, not in individual tool files. diff --git a/src/core/shell_command/safe_git.zig b/src/core/shell_command/safe_git.zig index df8f0df14..d3b4a46a2 100644 --- a/src/core/shell_command/safe_git.zig +++ b/src/core/shell_command/safe_git.zig @@ -13,10 +13,12 @@ const indexer = @import("../indexer/indexer.zig"); const Allocator = std.mem.Allocator; -/// `core.fsmonitor` became a boolean in git 2.36. Older git reads the value -/// as a hook program. +/// The oldest git these settings were checked against with planted programs +/// for every route: 2.25.1, 2.34.1 and 2.54.0. `core.fsmonitor` became a +/// boolean in git 2.36; older git runs the value as a hook command, and the +/// fixed PATH resolves `false` to the system `false`, which does nothing. const minimum_major: u32 = 2; -const minimum_minor: u32 = 36; +const minimum_minor: u32 = 25; const max_probe_output: usize = 4096; @@ -236,13 +238,15 @@ fn runCaptured(arena: Allocator, argv: []const []const u8, cwd: []const u8) Allo }; } -test "git version check requires 2.36 and reads vendor suffixes" { +test "git version check requires 2.25 and reads vendor suffixes" { const cases = [_]struct { output: []const u8, expected: VersionCheck }{ .{ .output = "git version 2.50.1 (Apple Git-155)\n", .expected = .supported }, .{ .output = "git version 2.36.0\n", .expected = .supported }, + .{ .output = "git version 2.34.1\n", .expected = .supported }, + .{ .output = "git version 2.25.0\n", .expected = .supported }, .{ .output = "git version 2.39.3.windows.1\n", .expected = .supported }, .{ .output = "git version 3.0.0\n", .expected = .supported }, - .{ .output = "git version 2.35.8\n", .expected = .too_old }, + .{ .output = "git version 2.24.4\n", .expected = .too_old }, .{ .output = "git version 1.9.5\n", .expected = .too_old }, .{ .output = "hub version 2.14\n", .expected = .unknown }, .{ .output = "git version x.y\n", .expected = .unknown }, @@ -611,6 +615,16 @@ test "safe git follows git's repository choice and refuses what it cannot blank" try repo.plain(gitdir, &.{ "config", "core.worktree", repo.repo }); try std.testing.expectEqual(Refusal.repository_mismatch, (try prepare(repo.arena, gitdir)).refused); + // An inner repository whose `HEAD` is a symlink into `refs/`. git uses + // it; fx's discovery rejects it and reads the outer repository, so the two + // disagree and safe git refuses instead of running with the wrong config. + const linked = try repo.path("repo/linked"); + try repo.plain(repo.repo, &.{ "init", "-q", "--template=", linked }); + const linked_head = try repo.path("repo/linked/.git/HEAD"); + try std.Io.Dir.cwd().deleteFile(std.testing.io, linked_head); + try std.Io.Dir.cwd().symLink(std.testing.io, "refs/heads/main", linked_head, .{ .is_directory = false }); + try std.testing.expectEqual(Refusal.repository_mismatch, (try prepare(repo.arena, linked)).refused); + try repo.plain(repo.repo, &.{ "config", "filter.a=b.clean", repo.sentinel }); try std.testing.expectEqual(Refusal.unsupported_filter_name, (try prepare(repo.arena, repo.repo)).refused); } diff --git a/src/core/tooling/tool_admission.zig b/src/core/tooling/tool_admission.zig index 1fb8cc919..8c5d3964b 100644 --- a/src/core/tooling/tool_admission.zig +++ b/src/core/tooling/tool_admission.zig @@ -1105,7 +1105,7 @@ fn resolveOrdinaryPermissionOutcome( call, permission_mode, ); - if (default_outcome.execution_authority != null) return default_outcome; + if (default_outcome.execution_authority != null or default_outcome.tool_failure != null) return default_outcome; } if (toolApprovalPolicy(input, call.name) == .ask_only) { return if (permission_mode == .auto) @@ -2107,10 +2107,21 @@ fn defaultRunCommandPermissionOutcome( .decision = .once, .execution_authority = .{ .run_command = .{ .direct_only = fingerprint } }, }, - .approval_required => .{ .decision = .permission_required }, + // Review cannot see a repository's git config, and the routine-command + // rule would run plain git, so an automatic git command that fx could + // not make safe does not run at all. + .approval_required => |reason| if (reason == .unsafe_git and permission_mode == .auto) + .{ .tool_failure = unsafe_git_hold_output } + else + .{ .decision = .permission_required }, }; } +const unsafe_git_hold_output = + "fx did not run this git command. It could not confirm that this repository's git config is safe, " ++ + "and git could start programs that config names. Inspect files with read_file, glob_files or grep_files, " ++ + "or ask the user to run git themselves."; + pub fn runCommandContext( input: Input, arena: Allocator, @@ -5787,6 +5798,54 @@ test "known reversible auto commands bypass the reviewer" { ); } +test "automatic git that safe git refuses is held instead of running plain git" { + shell_snapshot.resetProcessOwnerForTest(shell_snapshot.real_capture_fn); + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + // The repository config includes a file through a symlink, which fx's + // config reader refuses, so safe git cannot prepare and no git starts. + try tmp.dir.createDirPath(std.testing.io, "workspace/.git/objects"); + try tmp.dir.createDirPath(std.testing.io, "workspace/.git/refs"); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "workspace/.git/HEAD", .data = "ref: refs/heads/main\n" }); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "fsmonitor.inc", .data = "[core]\n\tfsmonitor = /bin/sh\n" }); + const include_target = try io_mod.dirRealpathAlloc(arena, tmp.dir, "fsmonitor.inc"); + try tmp.dir.symLink(std.testing.io, include_target, "workspace/.git/fsmonitor.inc", .{ .is_directory = false }); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "workspace/.git/config", .data = "[include]\n\tpath = fsmonitor.inc\n" }); + + var worker: WorkerRuntime = .{}; + defer worker.deinit(std.testing.allocator); + var fake = FakeAutoClassifier{}; + var input = testInputWithClassifier( + &worker, + permission_auto_classifier.Classifier.withOverride( + @ptrCast(&fake), + FakeAutoClassifier.classify, + ), + ); + input.workspace_root = try io_mod.dirRealpathAlloc(arena, tmp.dir, "workspace"); + + for ([_][]const u8{ "git status", "git status --short --branch", "git diff" }) |command| { + const arguments = try std.fmt.allocPrint( + arena, + "{{\"action\":\"run\",\"command\":{f},\"profile\":\"clean\"}}", + .{std.json.fmt(command, .{})}, + ); + const call: ToolCall = .{ .id = "refused-git", .name = "shell", .arguments_json = arguments }; + const held = try requestPermissionOutcome(input, arena, call, .auto, &.{}); + try std.testing.expect(held.execution_authority == null); + try std.testing.expectEqualStrings(unsafe_git_hold_output, held.tool_failure.?); + + // Ask mode keeps the normal approval prompt for the exact command. + const asked = try requestPermissionOutcome(input, arena, call, .ask, &.{}); + try std.testing.expect(asked.tool_failure == null); + try std.testing.expectEqual(ToolPermissionDecision.permission_required, asked.decision); + } + try std.testing.expectEqual(@as(usize, 0), fake.calls); +} + test "session deny narrows configured command allow" { const alloc = std.testing.allocator; var arena_state = std.heap.ArenaAllocator.init(alloc); diff --git a/tests/e2e/poisoned-repository.test.ts b/tests/e2e/poisoned-repository.test.ts index d27fa1901..ba3d7ce03 100644 --- a/tests/e2e/poisoned-repository.test.ts +++ b/tests/e2e/poisoned-repository.test.ts @@ -7,6 +7,7 @@ import { readFileSync, realpathSync, rmSync, + symlinkSync, writeFileSync, } from "node:fs"; import { tmpdir } from "node:os"; @@ -220,6 +221,37 @@ describe("poisoned repository", () => { TIMEOUT, ); + test( + "auto mode holds a git command whose repository config fx cannot read", + async () => { + const fixture = poisonedRepository(); + // fx reads repository config only through regular files, so an include + // that is a symlink leaves the config unread and safe git refuses. + writeFileSync(join(fixture.base, "real.inc"), `[core]\n\tfsmonitor = ${join(fixture.base, "sentinel.sh")}\n`); + symlinkSync(join(fixture.base, "real.inc"), join(fixture.workspace, ".git", "linked.inc")); + git(fixture, ["config", "include.path", "linked.inc"]); + const gateway = startFakeGateway([ + fakeGatewayToolCall("status", "shell", { + request: { action: "run", command: "git status --short", profile: "clean", yield_time_ms: 30_000 }, + }), + fakeGatewayFinalText("inspection complete"), + ], { classifierResponses: [] }); + gateways.push(gateway); + + const result = await runFx( + ["ask", "--quiet", "--json", "--no-save", "Inspect the repository."], + { cwd: fixture.workspace, env: fxEnv(fixture, gateway), timeoutMs: TIMEOUT }, + ); + expect(result.code, result.stderr).toBe(0); + expect(result.stderr).not.toMatch(/panic/i); + expect(sentinelRuns(fixture)).toEqual([]); + expect(gateway.classifierRequests).toHaveLength(0); + const bodies = gateway.requests.map((request: { body: string }) => request.body); + expect(toolResult(bodies[1]!, "status")).toContain("fx did not run this git command"); + }, + TIMEOUT, + ); + test( "fx pr snapshots the repository without running its programs", async () => { From 7478628d86938305a8d7cf65598a9c1013b072ad Mon Sep 17 00:00:00 2001 From: Pranit Date: Sun, 11 Oct 2026 05:36:38 -0400 Subject: [PATCH 15/23] Read the user's linked git config and key saved listings on its location - Follow a final symlink for the system and global git config, their includes and an excludes file they name, as git does. Paths a repository names still refuse symlinks. A `~/.gitconfig` linked by a dotfile manager no longer drops `core.excludesFile`, so globally ignored files stay out of glob and grep. - Stamp followed files through the link and move saved listings to format v3, so older listings are rebuilt. - Key a saved listing on HOME, XDG_CONFIG_HOME and the GIT_CONFIG_* variables as well as its roots, so a changed config location never reuses an old listing. - Say in the scan header that a walk stopped at the collection limit can hold different folders between runs, and trace when that happens. --- src/core/indexer/bounded_read.zig | 70 +++++++++++++++++++--- src/core/indexer/git_config.zig | 83 ++++++++++++++++++++++++--- src/core/indexer/indexer.zig | 3 + src/core/indexer/scan.zig | 26 ++++++--- src/core/indexer/snapshot.zig | 22 ++++--- src/core/indexer/sources.zig | 31 ++++++++-- src/core/indexer/tree.zig | 34 +++++++++-- src/core/workspace/file_index.zig | 51 ++++++++++++++-- tests/e2e/poisoned-repository.test.ts | 44 ++++++++++++++ 9 files changed, 318 insertions(+), 46 deletions(-) diff --git a/src/core/indexer/bounded_read.zig b/src/core/indexer/bounded_read.zig index 2f736868c..f24110f36 100644 --- a/src/core/indexer/bounded_read.zig +++ b/src/core/indexer/bounded_read.zig @@ -18,7 +18,32 @@ pub const Outcome = union(enum) { /// Reads `sub_path` relative to `dir`, accepting at most `max_bytes`. pub fn readAt(alloc: Allocator, dir: std.Io.Dir, sub_path: []const u8, max_bytes: usize) Allocator.Error!Outcome { - var file = io_mod.openExistingReadOnlyRegularFile(dir, sub_path, .no_follow) catch |err| return switch (err) { + return readWith(alloc, dir, sub_path, max_bytes, .no_follow); +} + +/// Reads an absolute `path`. +pub fn readAbsolute(alloc: Allocator, path: []const u8, max_bytes: usize) Allocator.Error!Outcome { + if (!std.fs.path.isAbsolute(path)) return .{ .unavailable = "relative_path" }; + return readAt(alloc, std.Io.Dir.cwd(), path, max_bytes); +} + +/// Reads an absolute `path` that the user's own git config or environment +/// names, following a final symlink as git does, since dotfile managers often +/// link these files. The target must still be a regular file within the cap. +/// Paths a repository names use `readAbsolute`. +pub fn readUserAbsolute(alloc: Allocator, path: []const u8, max_bytes: usize) Allocator.Error!Outcome { + if (!std.fs.path.isAbsolute(path)) return .{ .unavailable = "relative_path" }; + return readWith(alloc, std.Io.Dir.cwd(), path, max_bytes, .follow); +} + +fn readWith( + alloc: Allocator, + dir: std.Io.Dir, + sub_path: []const u8, + max_bytes: usize, + final_symlink: io_mod.FinalSymlinkPolicy, +) Allocator.Error!Outcome { + var file = io_mod.openExistingReadOnlyRegularFile(dir, sub_path, final_symlink) catch |err| return switch (err) { error.FileNotFound, error.NotDir => .missing, else => .{ .unavailable = @errorName(err) }, }; @@ -33,12 +58,6 @@ pub fn readAt(alloc: Allocator, dir: std.Io.Dir, sub_path: []const u8, max_bytes return .{ .content = data }; } -/// Reads an absolute `path`. -pub fn readAbsolute(alloc: Allocator, path: []const u8, max_bytes: usize) Allocator.Error!Outcome { - if (!std.fs.path.isAbsolute(path)) return .{ .unavailable = "relative_path" }; - return readAt(alloc, std.Io.Dir.cwd(), path, max_bytes); -} - test "bounded reads return content, missing, oversize and non-regular outcomes" { const alloc = std.testing.allocator; var tmp = std.testing.tmpDir(.{}); @@ -57,3 +76,40 @@ test "bounded reads return content, missing, oversize and non-regular outcomes" try std.testing.expect(try readAt(alloc, tmp.dir, "link", 16) == .unavailable); try std.testing.expect(try readAbsolute(alloc, "relative", 16) == .unavailable); } + +test "user reads follow a final symlink to a regular file and still refuse the rest" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "gitconfig", .data = "[core]\n" }); + try tmp.dir.createDirPath(std.testing.io, "dir"); + tmp.dir.symLink(std.testing.io, "gitconfig", "linked", .{}) catch return error.SkipZigTest; + try tmp.dir.symLink(std.testing.io, "dir", "linked-dir", .{}); + const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); + defer alloc.free(root); + + const linked = try std.fs.path.join(alloc, &.{ root, "linked" }); + defer alloc.free(linked); + const followed = try readUserAbsolute(alloc, linked, 16); + defer alloc.free(followed.content); + try std.testing.expectEqualStrings("[core]\n", followed.content); + // A repository-named path still refuses the same link. + try std.testing.expect(try readAbsolute(alloc, linked, 16) == .unavailable); + + const linked_dir = try std.fs.path.join(alloc, &.{ root, "linked-dir" }); + defer alloc.free(linked_dir); + try std.testing.expect(try readUserAbsolute(alloc, linked_dir, 16) == .unavailable); + try std.testing.expectEqualStrings("too_large", (try readUserAbsolute(alloc, linked, 2)).unavailable); + try std.testing.expect(try readUserAbsolute(alloc, "relative", 16) == .unavailable); + + // A link to a FIFO is refused without waiting for a writer. + const fifo = try std.fs.path.joinZ(alloc, &.{ root, "fifo" }); + defer alloc.free(fifo); + if (mkfifo(fifo, 0o600) != 0) return error.SkipZigTest; + try tmp.dir.symLink(std.testing.io, "fifo", "linked-fifo", .{}); + const linked_fifo = try std.fs.path.join(alloc, &.{ root, "linked-fifo" }); + defer alloc.free(linked_fifo); + try std.testing.expect(try readUserAbsolute(alloc, linked_fifo, 16) == .unavailable); +} + +extern "c" fn mkfifo(path: [*:0]const u8, mode: std.c.mode_t) c_int; diff --git a/src/core/indexer/git_config.zig b/src/core/indexer/git_config.zig index 958264621..20461ca01 100644 --- a/src/core/indexer/git_config.zig +++ b/src/core/indexer/git_config.zig @@ -241,21 +241,34 @@ pub const Unavailable = struct { reason: []const u8, }; +/// A config file read or looked for. System and user config follow a final +/// symlink, as git does; repository config never does. +pub const ReadPath = struct { + path: []const u8, + follow_symlinks: bool, +}; + pub const Loaded = struct { /// In file order, with included files spliced in at their directive. entries: []const LoadedEntry, unavailable: []const Unavailable, /// Every config file read or looked for, missing ones included, so a /// later change to any of them can be detected. - paths: []const []const u8 = &.{}, + paths: []const ReadPath = &.{}, /// The last entry for the key, or null when unset. pub fn last(self: Loaded, section: []const u8, subsection: ?[]const u8, name: []const u8) ?*const Entry { + const loaded = self.lastLoaded(section, subsection, name) orelse return null; + return &loaded.entry; + } + + /// Like `last`, with the scope the entry came from. + pub fn lastLoaded(self: Loaded, section: []const u8, subsection: ?[]const u8, name: []const u8) ?*const LoadedEntry { var i = self.entries.len; while (i > 0) { i -= 1; - const entry = &self.entries[i].entry; - if (keyMatches(entry, section, subsection, name)) return entry; + const loaded = &self.entries[i]; + if (keyMatches(&loaded.entry, section, subsection, name)) return loaded; } return null; } @@ -270,6 +283,10 @@ fn keyMatches(entry: *const Entry, section: []const u8, subsection: ?[]const u8, return entry.subsection == null; } +/// Every environment variable `Context.fromEnvironment` reads to locate +/// config, so a saved listing is reused only under the same values. +pub const environment_names = [_][]const u8{ "HOME", "XDG_CONFIG_HOME", "GIT_CONFIG_GLOBAL", "GIT_CONFIG_SYSTEM", "GIT_CONFIG_NOSYSTEM" }; + /// Where config comes from and what conditions are evaluated against. pub const Context = struct { home: ?[]const u8 = null, @@ -314,15 +331,22 @@ const Loader = struct { mode: Mode, entries: std.ArrayList(LoadedEntry) = .empty, unavailable: std.ArrayList(Unavailable) = .empty, - paths: std.ArrayList([]const u8) = .empty, + paths: std.ArrayList(ReadPath) = .empty, fn readFile(self: *Loader, path: []const u8, scope: Scope, depth: usize) Allocator.Error!void { if (depth > max_include_depth) return self.markUnavailable(path, "include_depth"); // git reads /dev/null as an empty file; it is commonly used to // disable a config level. if (std.mem.eql(u8, path, "/dev/null")) return; - try self.paths.append(self.arena, path); - const content = switch (try bounded_read.readAbsolute(self.arena, path, max_config_file_bytes)) { + // System and user config, and what they include, are the user's own + // files, often links from a dotfile manager. + const follow = scope == .system or scope == .global; + try self.paths.append(self.arena, .{ .path = path, .follow_symlinks = follow }); + const outcome = if (follow) + try bounded_read.readUserAbsolute(self.arena, path, max_config_file_bytes) + else + try bounded_read.readAbsolute(self.arena, path, max_config_file_bytes); + const content = switch (outcome) { .content => |bytes| bytes, .missing => return, .unavailable => |reason| return self.markUnavailable(path, reason), @@ -492,6 +516,9 @@ pub const ObjectFormat = enum { sha1, sha256 }; pub const IgnoreSettings = struct { /// Absolute path of the excludes file to read, or null for none. excludes_file: ?[]const u8, + /// Whether system or user config named the excludes file, or it is git's + /// default location, so it is the user's own and a symlink is followed. + excludes_file_follows_symlinks: bool, ignore_case: bool, object_format: ObjectFormat, /// Settings git would refuse, by key. Static strings. @@ -526,8 +553,10 @@ pub fn ignoreSettings(arena: Allocator, loaded: Loaded, context: *const Context) break; } - const excludes_file: ?[]const u8 = if (loaded.last("core", null, "excludesfile")) |entry| blk: { - const raw = entry.value orelse break :blk null; + var excludes_file_follows_symlinks = true; + const excludes_file: ?[]const u8 = if (loaded.lastLoaded("core", null, "excludesfile")) |entry| blk: { + excludes_file_follows_symlinks = entry.scope == .system or entry.scope == .global; + const raw = entry.entry.value orelse break :blk null; if (raw.len == 0) break :blk null; if (std.mem.startsWith(u8, raw, "~/")) { const home = context.home orelse break :blk null; @@ -546,6 +575,7 @@ pub fn ignoreSettings(arena: Allocator, loaded: Loaded, context: *const Context) return .{ .excludes_file = excludes_file, + .excludes_file_follows_symlinks = excludes_file_follows_symlinks, .ignore_case = ignore_case, .object_format = object_format, .invalid = try invalid.toOwnedSlice(arena), @@ -722,6 +752,43 @@ test "config gitdir patterns relative to the including file" { try std.testing.expectEqualStrings(expected, settings.excludes_file.?); } +test "user config links are followed and repository config links are not" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const repo = try testRepo(arena, &tmp, "repo", "main"); + // A dotfile manager links the user's files into the home folder. + try writeTestFile(tmp.dir, "dotfiles/gitconfig", "[include]\npath = ~/.gitconfig.local\n[core]\nexcludesfile = ~/.gitignore_global\n"); + try writeTestFile(tmp.dir, "dotfiles/local", "[core]\nignorecase = true\n"); + try writeTestFile(tmp.dir, "dotfiles/ignore", "*.env\n"); + tmp.dir.symLink(std.testing.io, "../dotfiles/gitconfig", "home/.gitconfig", .{}) catch return error.SkipZigTest; + try tmp.dir.symLink(std.testing.io, "../dotfiles/local", "home/.gitconfig.local", .{}); + try tmp.dir.symLink(std.testing.io, "../dotfiles/ignore", "home/.gitignore_global", .{}); + + const loaded = try loadAll(arena, &repo.context); + try std.testing.expectEqual(@as(usize, 0), loaded.unavailable.len); + const settings = try ignoreSettings(arena, loaded, &repo.context); + try std.testing.expect(settings.ignore_case); + try std.testing.expectEqualStrings(try std.fs.path.join(arena, &.{ repo.root, "home/.gitignore_global" }), settings.excludes_file.?); + try std.testing.expect(settings.excludes_file_follows_symlinks); + const home_prefix = try std.fs.path.join(arena, &.{ repo.root, "home/" }); + for (loaded.paths) |read| { + try std.testing.expectEqual(std.mem.startsWith(u8, read.path, home_prefix), read.follow_symlinks); + } + + // The repository's own config never follows a link, and an excludes file + // it names is not the user's. + try writeTestFile(tmp.dir, "repo/.git/config", "[include]\npath = linked.inc\n[core]\nexcludesfile = /repo-excludes\n"); + try tmp.dir.symLink(std.testing.io, "../../dotfiles/local", "repo/.git/linked.inc", .{}); + const repo_loaded = try loadAll(arena, &repo.context); + try std.testing.expectEqual(@as(usize, 1), repo_loaded.unavailable.len); + const repo_settings = try ignoreSettings(arena, repo_loaded, &repo.context); + try std.testing.expectEqualStrings("/repo-excludes", repo_settings.excludes_file.?); + try std.testing.expect(!repo_settings.excludes_file_follows_symlinks); +} + test "repository filter names come only from repository config and count every includeIf" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); diff --git a/src/core/indexer/indexer.zig b/src/core/indexer/indexer.zig index a674b27b2..a53ef9090 100644 --- a/src/core/indexer/indexer.zig +++ b/src/core/indexer/indexer.zig @@ -20,6 +20,9 @@ pub const Entry = scan_mod.Entry; pub const ScanResult = scan_mod.Result; pub const ScanError = scan_mod.Error; pub const default_skipped_names = scan_mod.default_skipped_names; +/// The environment variables that locate git config. A saved listing is valid +/// only under the values it was built with. +pub const config_environment_names = git_config.environment_names; /// Lists the files and folders under the absolute `root`: inside a git /// repository exactly what git shows, elsewhere everything except `.gitignore` diff --git a/src/core/indexer/scan.zig b/src/core/indexer/scan.zig index 6b6492d79..a53fd63ee 100644 --- a/src/core/indexer/scan.zig +++ b/src/core/indexer/scan.zig @@ -7,7 +7,9 @@ //! listed and never followed; submodules and nested repositories are listed //! and never entered. Folders are read by a bounded pool of worker threads, //! all joined before `scan` returns, and the output is sorted, so it does not -//! depend on scheduling. +//! depend on scheduling unless the walk stops at the collection limit (see +//! `collect_factor`). A listing cut there is marked incomplete, and which +//! folders it holds can vary between runs. const std = @import("std"); const io_mod = @import("../shared/io.zig"); @@ -49,7 +51,9 @@ pub const Result = struct { pub const Error = error{ OutOfMemory, Canceled, RootUnavailable }; /// Workers stop reading folders past this many collected entries, so a huge -/// tree cannot exhaust memory. Below it, the cap keeps a sorted prefix. +/// tree cannot exhaust memory. Below it, the cap keeps a sorted prefix. Past +/// it, which folders were read depends on thread timing, so the result is +/// marked incomplete with the cap reached. const collect_factor: usize = 4; /// Helper threads start only once this many folders are waiting. const parallel_threshold: usize = 16; @@ -75,7 +79,13 @@ const Rules = struct { } }; -const SourceRequest = struct { path: []const u8, identity_only: bool }; +const SourceRequest = struct { + path: []const u8, + identity_only: bool, + /// Stamp the file a final symlink points at: the user's own config may be + /// a link, and an edit to its target must invalidate the listing. + follow_symlinks: bool = false, +}; const Job = struct { /// Relative to the scan root. @@ -262,7 +272,7 @@ pub fn scan(alloc: Allocator, root: []const u8, options: Options, stop: ?*std.at if (listing.ignore_file) |path| try rules.sources.append(arena, .{ .path = try arena.dupe(u8, path), .identity_only = false }); } const stamps = try arena.alloc(tree_mod.SourceStamp, rules.sources.items.len); - for (rules.sources.items, stamps) |request, *stamp| stamp.* = tree_mod.sourceStamp(request.path, request.identity_only, scan_started_ns); + for (rules.sources.items, stamps) |request, *stamp| stamp.* = tree_mod.sourceStamp(request.path, request.identity_only, request.follow_symlinks, scan_started_ns); const skipped_names = try arena.alloc([]const u8, options.skipped_names.len); for (options.skipped_names, skipped_names) |name, *copy| copy.* = try arena.dupe(u8, name); @@ -310,8 +320,8 @@ fn repositoryRules(arena: Allocator, root: []const u8, rules: *Rules, incomplete if (loaded.unavailable.len > 0 or settings.invalid.len > 0) incomplete.* = true; const wide = try sources.loadRepositoryWide(arena, found, settings); if (wide.unavailable.len > 0) incomplete.* = true; - for (loaded.paths) |path| try rules.sources.append(arena, .{ .path = path, .identity_only = false }); - if (settings.excludes_file) |path| try rules.sources.append(arena, .{ .path = path, .identity_only = false }); + for (loaded.paths) |read| try rules.sources.append(arena, .{ .path = read.path, .identity_only = false, .follow_symlinks = read.follow_symlinks }); + if (settings.excludes_file) |path| try rules.sources.append(arena, .{ .path = path, .identity_only = false, .follow_symlinks = settings.excludes_file_follows_symlinks }); for ([_][]const u8{"info/exclude"}) |name| try rules.sources.append(arena, .{ .path = try std.fs.path.join(arena, &.{ found.common_dir, name }), .identity_only = false }); for ([_][]const u8{ "index", "HEAD", "commondir" }) |name| try rules.sources.append(arena, .{ .path = try std.fs.path.join(arena, &.{ found.git_dir, name }), .identity_only = false }); @@ -521,7 +531,9 @@ fn listDirectory(shared: *Shared, worker: *Worker, job: Job, children: *std.Arra listing.children = child_entries.items; const total = shared.collected.fetchAdd(files.items.len + child_entries.items.len, .acq_rel) + files.items.len + child_entries.items.len; if (total > shared.collect_limit) { - shared.limit_reached.store(true, .release); + if (!shared.limit_reached.swap(true, .acq_rel)) { + debug_trace.logf("indexer", "scan collection limit reached limit={d} dropped_folders={d}", .{ shared.collect_limit, children.items.len }); + } children.clearRetainingCapacity(); } } diff --git a/src/core/indexer/snapshot.zig b/src/core/indexer/snapshot.zig index 50ab6537e..d723223fa 100644 --- a/src/core/indexer/snapshot.zig +++ b/src/core/indexer/snapshot.zig @@ -1,4 +1,4 @@ -//! Saved trees for one scope (format v2), so `@` can paint at launch and a +//! Saved trees for one scope (format v3), so `@` can paint at launch and a //! refresh can confirm nothing changed without reading folders again. The //! caller chooses the file path. A file is used only when it is a private //! regular file with one link, at most `max_bytes`, matching digest, exactly @@ -15,7 +15,7 @@ const Allocator = std.mem.Allocator; const Sha256 = std.crypto.hash.sha2.Sha256; const Tree = tree_mod.Tree; -const magic = "fx-file-index-v2\n"; +const magic = "fx-file-index-v3\n"; const max_bytes: usize = 64 * 1024 * 1024; const max_path_bytes: usize = 4096; @@ -157,7 +157,8 @@ fn encode(writer: *std.Io.Writer, trees: []const Tree) !void { try writeInt(writer, u32, @intCast(tree.sources.len)); for (tree.sources) |source| { try writeString(writer, source.path); - const source_flags: u8 = @as(u8, @intFromBool(source.exists)) | @as(u8, @intFromBool(source.identity_only)) << 1 | @as(u8, @intFromBool(source.reusable)) << 2; + const source_flags: u8 = @as(u8, @intFromBool(source.exists)) | @as(u8, @intFromBool(source.identity_only)) << 1 | + @as(u8, @intFromBool(source.reusable)) << 2 | @as(u8, @intFromBool(source.follow_symlinks)) << 3; try writer.writeByte(source_flags); try writeInt(writer, u64, source.inode); try writeInt(writer, u64, source.size); @@ -240,12 +241,13 @@ fn decode(arena: Allocator, bytes: []const u8, roots: []const []const u8) (Alloc for (stamps) |*source| { const path = try decoder.string(arena, max_path_bytes, false); const source_flags = (try decoder.take(1))[0]; - if (source_flags & ~@as(u8, 0b111) != 0) return error.InvalidSnapshot; + if (source_flags & ~@as(u8, 0b1111) != 0) return error.InvalidSnapshot; source.* = .{ .path = path, .exists = source_flags & 1 != 0, .identity_only = source_flags & 2 != 0, .reusable = source_flags & 4 != 0, + .follow_symlinks = source_flags & 8 != 0, .inode = try decoder.int(u64), .size = try decoder.int(u64), .mtime_ns = try decoder.int(i64), @@ -288,7 +290,7 @@ fn testTree(root: []const u8, entries: []const tree_mod.Entry) Tree { .sources = &.{ .{ .path = "/w/.gitignore", .exists = true, .identity_only = false, .inode = 3, .size = 10, .mtime_ns = 4, .ctime_ns = 4, .reusable = true }, .{ .path = "/w/.git", .exists = true, .identity_only = true, .inode = 11, .reusable = true }, - .{ .path = "/home/.gitconfig", .exists = false, .identity_only = false, .reusable = true }, + .{ .path = "/home/.gitconfig", .exists = false, .identity_only = false, .reusable = true, .follow_symlinks = true }, }, }; } @@ -325,6 +327,7 @@ test "snapshot round trips, rejects tampering, other scopes and v1 files, and an try std.testing.expect(!first.folders[1].reusable and first.folders[1].boundary and first.folders[1].excluded); try std.testing.expect(!first.folders[0].boundary and !first.folders[0].excluded); try std.testing.expect(first.sources[1].identity_only and !first.sources[2].exists); + try std.testing.expect(first.sources[2].follow_symlinks and !first.sources[0].follow_symlinks); try std.testing.expectEqualStrings("b.txt", loaded.trees[1].entries[0].path); } @@ -338,9 +341,12 @@ test "snapshot round trips, rejects tampering, other scopes and v1 files, and an file.close(std.testing.io); try std.testing.expect((try load(alloc, path, &roots)) == null); - // A v1 file fails the magic check. - try std.Io.Dir.cwd().writeFile(std.testing.io, .{ .sub_path = path, .data = "fx-file-index-v1\n{}", .flags = .{ .permissions = .fromMode(0o600) } }); - try std.testing.expect((try load(alloc, path, &roots)) == null); + // Older formats fail the magic check. A v2 file stamped every source + // without following symlinks. + for ([_][]const u8{ "fx-file-index-v1\n{}", "fx-file-index-v2\n" }) |old| { + try std.Io.Dir.cwd().writeFile(std.testing.io, .{ .sub_path = path, .data = old, .flags = .{ .permissions = .fromMode(0o600) } }); + try std.testing.expect((try load(alloc, path, &roots)) == null); + } // A file others can read is not trusted. try save(alloc, path, &trees); diff --git a/src/core/indexer/sources.zig b/src/core/indexer/sources.zig index c0716456d..3335c8796 100644 --- a/src/core/indexer/sources.zig +++ b/src/core/indexer/sources.zig @@ -26,15 +26,23 @@ pub const RepositoryWide = struct { }; /// Loads the excludes file named by `settings` and `info/exclude` from the -/// common git directory. Memory belongs to `arena`. +/// common git directory. Only an excludes file the user's own config names +/// follows a symlink. Memory belongs to `arena`. pub fn loadRepositoryWide(arena: Allocator, layout: layout_mod.Layout, settings: git_config.IgnoreSettings) Allocator.Error!RepositoryWide { var lists: std.ArrayList(*const ignore.PatternList) = .empty; var unavailable: std.ArrayList(Unavailable) = .empty; const info_exclude = try std.fs.path.join(arena, &.{ layout.common_dir, "info", "exclude" }); - const paths = [_]?[]const u8{ settings.excludes_file, info_exclude }; - for (paths) |maybe_path| { - const path = maybe_path orelse continue; - switch (try bounded_read.readAbsolute(arena, path, ignore.max_pattern_file_bytes)) { + const files = [_]struct { path: ?[]const u8, follow_symlinks: bool }{ + .{ .path = settings.excludes_file, .follow_symlinks = settings.excludes_file_follows_symlinks }, + .{ .path = info_exclude, .follow_symlinks = false }, + }; + for (files) |file| { + const path = file.path orelse continue; + const outcome = if (file.follow_symlinks) + try bounded_read.readUserAbsolute(arena, path, ignore.max_pattern_file_bytes) + else + try bounded_read.readAbsolute(arena, path, ignore.max_pattern_file_bytes); + switch (outcome) { .content => |content| { const list = try arena.create(ignore.PatternList); list.* = try ignore.PatternList.parse(arena, "", content); @@ -93,6 +101,7 @@ test "repository-wide ignore sources load in git's precedence and report unreada const settings: git_config.IgnoreSettings = .{ .excludes_file = try std.fs.path.join(arena, &.{ root, "global-ignore" }), + .excludes_file_follows_symlinks = true, .ignore_case = false, .object_format = .sha1, .invalid = &.{}, @@ -108,6 +117,18 @@ test "repository-wide ignore sources load in git's precedence and report unreada const partial = try loadRepositoryWide(arena, layout, unreadable); try std.testing.expectEqual(@as(usize, 1), partial.lists.len); try std.testing.expectEqual(@as(usize, 1), partial.unavailable.len); + + // A linked excludes file is followed only when the user's config named it. + tmp.dir.symLink(std.testing.io, "global-ignore", "linked-ignore", .{}) catch return error.SkipZigTest; + var linked = settings; + linked.excludes_file = try std.fs.path.join(arena, &.{ root, "linked-ignore" }); + const followed = try loadRepositoryWide(arena, layout, linked); + try std.testing.expectEqual(@as(usize, 2), followed.lists.len); + try std.testing.expectEqual(ignore.Decision.excluded, ignore.decide(followed.lists, "a.tmp", false, false)); + linked.excludes_file_follows_symlinks = false; + const refused = try loadRepositoryWide(arena, layout, linked); + try std.testing.expectEqual(@as(usize, 1), refused.lists.len); + try std.testing.expectEqual(@as(usize, 1), refused.unavailable.len); } test "directory .gitignore loads with its base and refuses symlinks" { diff --git a/src/core/indexer/tree.zig b/src/core/indexer/tree.zig index 7d25de498..9f3b7d1af 100644 --- a/src/core/indexer/tree.zig +++ b/src/core/indexer/tree.zig @@ -48,6 +48,8 @@ pub const SourceStamp = struct { mtime_ns: i64 = 0, ctime_ns: i64 = 0, reusable: bool, + /// Stamped through a final symlink, as the file was read. + follow_symlinks: bool = false, }; pub const Tree = struct { @@ -95,9 +97,9 @@ pub fn folderStamp(rel: []const u8, stat: std.Io.File.Stat, scan_started_ns: i64 } /// Stamps the absolute `path` without following a final symlink. -pub fn sourceStamp(path: []const u8, identity_only: bool, scan_started_ns: i64) SourceStamp { - const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), path, .{ .follow_symlinks = false }) catch { - return .{ .path = path, .exists = false, .identity_only = identity_only, .reusable = true }; +pub fn sourceStamp(path: []const u8, identity_only: bool, follow_symlinks: bool, scan_started_ns: i64) SourceStamp { + const stat = std.Io.Dir.cwd().statFile(io_mod.getIo(), path, .{ .follow_symlinks = follow_symlinks }) catch { + return .{ .path = path, .exists = false, .identity_only = identity_only, .reusable = true, .follow_symlinks = follow_symlinks }; }; const mtime = clampNs(stat.mtime.nanoseconds); const ctime = clampNs(stat.ctime.nanoseconds); @@ -113,6 +115,7 @@ pub fn sourceStamp(path: []const u8, identity_only: bool, scan_started_ns: i64) .mtime_ns = if (identity) 0 else mtime, .ctime_ns = if (identity) 0 else ctime, .reusable = identity or reusableTimes(mtime, ctime, scan_started_ns), + .follow_symlinks = follow_symlinks, }; } @@ -139,7 +142,7 @@ pub fn isCurrent(tree: *const Tree, root: []const u8, skipped_names: []const []c for (tree.sources) |source| { if (!source.reusable) return false; - const current = sourceStamp(source.path, source.identity_only, tree.scan_started_ns); + const current = sourceStamp(source.path, source.identity_only, source.follow_symlinks, tree.scan_started_ns); if (!sameSource(source, current)) return false; if (current.mtime_ns > now or current.ctime_ns > now) return false; } @@ -165,6 +168,27 @@ test "stamps reuse only times older than the scan start by the margin" { try std.testing.expect(!reusableTimes(start - 3 * std.time.ns_per_s, start, start)); try std.testing.expect(!reusableTimes(start + 1, start - 3 * std.time.ns_per_s, start)); // A missing source stays missing until it appears. - const missing = sourceStamp("/nonexistent/fx-indexer-test", false, start); + const missing = sourceStamp("/nonexistent/fx-indexer-test", false, false, start); try std.testing.expect(!missing.exists and missing.reusable); } + +test "a followed source stamps the file its link points at" { + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "target", .data = "[core]\n" }); + tmp.dir.symLink(std.testing.io, "target", "link", .{}) catch return error.SkipZigTest; + const root = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); + defer alloc.free(root); + const link = try std.fs.path.join(alloc, &.{ root, "link" }); + defer alloc.free(link); + + const start = nowNs(); + const followed = sourceStamp(link, false, true, start); + const unfollowed = sourceStamp(link, false, false, start); + try std.testing.expect(followed.follow_symlinks and !unfollowed.follow_symlinks); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "target", .data = "[core]\n\tignorecase = true\n" }); + // An edit to the target is seen through the link, never by the link alone. + try std.testing.expect(!sameSource(followed, sourceStamp(link, false, true, start))); + try std.testing.expect(sameSource(unfollowed, sourceStamp(link, false, false, start))); +} diff --git a/src/core/workspace/file_index.zig b/src/core/workspace/file_index.zig index 4041a36a5..5d65d82dd 100644 --- a/src/core/workspace/file_index.zig +++ b/src/core/workspace/file_index.zig @@ -858,16 +858,35 @@ fn loadGeneration( return .ready; } -/// `/.fx/file-index/.idx`, one file per scope, or -/// null without an absolute home directory. +/// `/.fx/file-index/.idx`, or null without an absolute home +/// directory. The digest covers the roots and every variable that locates git +/// config, so a listing built under one config is never reused under another. pub fn snapshotPath(alloc: Allocator, roots: []const []const u8) Allocator.Error!?[]const u8 { const home = io_mod.getenv("HOME") orelse return null; if (!std.fs.path.isAbsolute(home)) return null; - return try snapshotPathIn(alloc, home, roots); + var config_environment: [indexer.config_environment_names.len]?[]const u8 = undefined; + for (indexer.config_environment_names, &config_environment) |name, *value| value.* = io_mod.getenv(name); + return try snapshotPathIn(alloc, home, roots, &config_environment); } -fn snapshotPathIn(alloc: Allocator, home: []const u8, roots: []const []const u8) Allocator.Error![]const u8 { - const hex = std.fmt.bytesToHex(rootsDigest(roots), .lower); +fn snapshotPathIn( + alloc: Allocator, + home: []const u8, + roots: []const []const u8, + config_environment: []const ?[]const u8, +) Allocator.Error![]const u8 { + var digest = std.crypto.hash.sha2.Sha256.init(.{}); + digest.update(&rootsDigest(roots)); + for (config_environment) |value| { + if (value) |bytes| { + digest.update(&.{1}); + digest.update(bytes); + digest.update(&.{0}); + } else digest.update(&.{0}); + } + var sum: [sha256_digest_len]u8 = undefined; + digest.final(&sum); + const hex = std.fmt.bytesToHex(sum, .lower); return std.fmt.allocPrint(alloc, "{s}/.fx/file-index/{s}.idx", .{ home, &hex }); } @@ -2688,7 +2707,7 @@ test "persisted file index paints a stale preview and the real scan replaces it" const scanned_count = first.count(); try std.testing.expectEqual(@as(usize, 1), adoptions); try std.testing.expect(scanned_count >= 2); - const snapshot_file = try snapshotPathIn(alloc, home, &roots); + const snapshot_file = (try snapshotPath(alloc, &roots)).?; defer alloc.free(snapshot_file); var cached = (try indexer.loadSnapshot(alloc, snapshot_file, &roots)).?; defer cached.deinit(); @@ -3675,3 +3694,23 @@ fn sleepBlocking(milliseconds: u64) void { var sleep_io_backend: std.Io.Threaded = .init_single_threaded; sleep_io_backend.io().sleep(.fromMilliseconds(@intCast(milliseconds)), .real) catch {}; } + +test "snapshot path changes with where git config is located" { + const alloc = std.testing.allocator; + const roots = [_][]const u8{"/work"}; + const cases = [_][indexer.config_environment_names.len]?[]const u8{ + .{ "/home/u", null, null, null, null }, + .{ "/home/u", "/home/u/.xdg", null, null, null }, + .{ "/home/u", "/home/u/.other", null, null, null }, + .{ "/home/u", null, "/home/u/gitconfig", null, null }, + }; + var paths: [cases.len][]const u8 = undefined; + for (cases, &paths) |case, *path| path.* = try snapshotPathIn(alloc, "/home/u", &roots, &case); + defer for (paths) |path| alloc.free(path); + for (paths, 0..) |a, i| for (paths[i + 1 ..]) |b| try std.testing.expect(!std.mem.eql(u8, a, b)); + + const again = try snapshotPathIn(alloc, "/home/u", &roots, &cases[1]); + defer alloc.free(again); + try std.testing.expectEqualStrings(paths[1], again); + try std.testing.expect(std.mem.startsWith(u8, again, "/home/u/.fx/file-index/")); +} diff --git a/tests/e2e/poisoned-repository.test.ts b/tests/e2e/poisoned-repository.test.ts index ba3d7ce03..78b27fa76 100644 --- a/tests/e2e/poisoned-repository.test.ts +++ b/tests/e2e/poisoned-repository.test.ts @@ -277,3 +277,47 @@ describe("poisoned repository", () => { TIMEOUT, ); }); + +// Dotfile managers often make `~/.gitconfig` a symlink. Files the user's own +// excludes file ignores must stay out of glob and grep through it, as in git. +describe("user git config through a symlink", () => { + test( + "glob and grep honor an excludes file named by a symlinked ~/.gitconfig", + async () => { + const base = realpathSync(mkdtempSync(join(tmpdir(), "fx-linked-gitconfig-"))); + const fixture: Fixture = { base, home: join(base, "home"), workspace: join(base, "workspace"), log: join(base, "unused.log") }; + fixtures.push(fixture); + mkdirSync(join(fixture.home, ".fx"), { recursive: true }); + mkdirSync(join(fixture.home, "dotfiles")); + mkdirSync(fixture.workspace); + writeFileSync(join(fixture.home, ".fx", "settings.json"), JSON.stringify({ auto_upgrade: false })); + writeFileSync(join(fixture.home, "dotfiles", "ignore"), "*.secret\n"); + writeFileSync(join(fixture.home, "dotfiles", "gitconfig"), "[core]\n\texcludesFile = ~/dotfiles/ignore\n"); + symlinkSync(join(fixture.home, "dotfiles", "gitconfig"), join(fixture.home, ".gitconfig")); + git(fixture, ["init", "-q", "--template=", "-b", "main"]); + writeFileSync(join(fixture.workspace, "public.txt"), "needle public\n"); + writeFileSync(join(fixture.workspace, "api.secret"), "needle hidden\n"); + expect(git(fixture, ["ls-files", "-co", "--exclude-standard"]).trim()).toBe("public.txt"); + + const gateway = startFakeGateway([ + fakeGatewayToolCall("glob", "glob_files", { pattern: "**/*" }), + fakeGatewayToolCall("grep", "grep_files", { pattern: "needle" }), + fakeGatewayFinalText("inspection complete"), + ], { classifierResponses: [] }); + gateways.push(gateway); + const result = await runFx( + ["ask", "--quiet", "--json", "--no-save", "Inspect the repository."], + { cwd: fixture.workspace, env: { ...fxEnv(fixture, gateway), GIT_CONFIG_NOSYSTEM: "1" }, timeoutMs: TIMEOUT }, + ); + expect(result.code, result.stderr).toBe(0); + const bodies = gateway.requests.map((request: { body: string }) => request.body); + const glob = toolResult(bodies[1]!, "glob"); + expect(glob).toContain("public.txt"); + expect(glob).not.toContain("api.secret"); + const grep = toolResult(bodies[2]!, "grep"); + expect(grep).toContain("needle public"); + expect(grep).not.toContain("needle hidden"); + }, + TIMEOUT, + ); +}); From c5df4efe51a7720bf1016f115e3241a513ea1836 Mon Sep 17 00:00:00 2001 From: Pranit Date: Sun, 11 Oct 2026 05:36:38 -0400 Subject: [PATCH 16/23] Name why a file listing is incomplete and bound grep's look-ahead - glob_files and grep_files report the candidate cap only when it was reached. A listing cut short by unreadable folders or git files says so instead. One note writer replaces the two copies. - Stop parallel grep once finished batches hold the 2,000-match cut, even while an earlier batch is still running, so later files are no longer buffered. - Running out of memory fails grep only when it happens before the cut. Counts still need every file. --- src/core/workspace/grep_search.zig | 114 +++++++++++++++++++++------- src/core/workspace/tool_files.zig | 35 +++++++++ src/tools/filesystem/glob_files.zig | 43 ++++------- src/tools/filesystem/grep_files.zig | 46 ++++++----- 4 files changed, 157 insertions(+), 81 deletions(-) diff --git a/src/core/workspace/grep_search.zig b/src/core/workspace/grep_search.zig index 2263e66c2..edafd785d 100644 --- a/src/core/workspace/grep_search.zig +++ b/src/core/workspace/grep_search.zig @@ -36,6 +36,7 @@ pub const Result = struct { candidate_count: usize = 0, candidate_cap: usize = tool_files.default_candidate_cap, candidate_incomplete: bool = false, + candidate_cap_reached: bool = false, skipped_overlong: usize = 0, }; @@ -46,6 +47,7 @@ pub const CountResult = struct { candidate_count: usize = 0, candidate_cap: usize = tool_files.default_candidate_cap, candidate_incomplete: bool = false, + candidate_cap_reached: bool = false, skipped_overlong: usize = 0, }; @@ -199,6 +201,7 @@ const CandidateStats = struct { count: usize = 0, cap: usize = tool_files.default_candidate_cap, incomplete: bool = false, + cap_reached: bool = false, skipped_overlong: usize = 0, fn fromListing(listing: tool_files.Listing) CandidateStats { @@ -206,6 +209,7 @@ const CandidateStats = struct { .count = listing.files.len, .cap = listing.candidate_cap, .incomplete = listing.incomplete, + .cap_reached = listing.cap_reached, .skipped_overlong = listing.skipped_overlong, }; } @@ -218,6 +222,7 @@ fn finishMatches(arena: Allocator, matches: *std.ArrayList(Match), truncated_rea .candidate_count = stats.count, .candidate_cap = stats.cap, .candidate_incomplete = stats.incomplete, + .candidate_cap_reached = stats.cap_reached, .skipped_overlong = stats.skipped_overlong, }; } @@ -227,6 +232,7 @@ fn finishCount(result: CountResult, stats: CandidateStats) CountResult { out.candidate_count = stats.count; out.candidate_cap = stats.cap; out.candidate_incomplete = stats.incomplete; + out.candidate_cap_reached = stats.cap_reached; out.skipped_overlong += stats.skipped_overlong; return out; } @@ -287,7 +293,9 @@ const FileOutcome = struct { /// Shared state of one parallel content scan. Merging in candidate order makes /// the output identical to a single-threaded scan, including where the -/// collection cap truncates it. +/// collection cap truncates it. Batches are handed out in candidate order, so +/// once finished batches hold the cap, no batch that has not started can +/// reach the merged result. const ParallelScan = struct { /// Thread-safe and temporary: candidate resolution and per-file matches. work: Allocator, @@ -301,12 +309,10 @@ const ParallelScan = struct { next_batch: std.atomic.Value(usize) = .init(0), stop: std.atomic.Value(bool) = .init(false), mutex: std.Io.Mutex = .init, - /// Guarded by `mutex`: which batches are done, how many leading batches - /// are done, and how many matches those leading batches hold. - batch_done: []bool, - done_prefix: usize = 0, - prefix_matches: usize = 0, - out_of_memory: bool = false, + /// Guarded by `mutex`: the matches every finished batch holds, and the + /// first file of the earliest batch that ran out of memory. + done_matches: usize = 0, + out_of_memory_from: ?usize = null, fn run(self: *ParallelScan) void { while (!self.stop.load(.acquire)) { @@ -317,13 +323,13 @@ const ParallelScan = struct { for (first..last) |index| { _ = self.scanOne(index) catch { self.mutex.lockUncancelable(io_mod.getIo()); - self.out_of_memory = true; + self.out_of_memory_from = @min(self.out_of_memory_from orelse first, first); self.mutex.unlock(io_mod.getIo()); self.stop.store(true, .release); return; }; } - self.finishBatch(batch); + self.finishBatch(first, last); } } @@ -350,17 +356,16 @@ const ParallelScan = struct { } } - /// Once the leading done batches hold the collection cap, no later file - /// can change the merged result, so the remaining work stops. - fn finishBatch(self: *ParallelScan, batch: usize) void { + /// Every finished batch came before any batch not yet started, so once + /// they hold the collection cap the remaining work stops. Matches past the + /// cut are never buffered beyond the batches already running. + fn finishBatch(self: *ParallelScan, first: usize, last: usize) void { + var batch_matches: usize = 0; + for (self.outcomes[first..last]) |outcome| batch_matches += outcome.matches.len; self.mutex.lockUncancelable(io_mod.getIo()); defer self.mutex.unlock(io_mod.getIo()); - self.batch_done[batch] = true; - while (self.done_prefix < self.batch_done.len and self.batch_done[self.done_prefix]) : (self.done_prefix += 1) { - const first = self.done_prefix * batch_files; - for (self.outcomes[first..@min(first + batch_files, self.outcomes.len)]) |outcome| self.prefix_matches += outcome.matches.len; - } - if (self.mode == .matches and self.prefix_matches >= collection_cap) self.stop.store(true, .release); + self.done_matches += batch_matches; + if (self.mode == .matches and self.done_matches >= collection_cap) self.stop.store(true, .release); } }; @@ -371,8 +376,9 @@ fn workerCount(files: usize) usize { } /// Scans `files` with up to `max_scan_workers` threads, the calling thread -/// included. A helper that cannot start leaves its share to the others. -fn runParallelScan(scan: *ParallelScan) Allocator.Error!void { +/// included. A helper that cannot start leaves its share to the others. Running +/// out of memory is recorded in `out_of_memory_from` for the caller to judge. +fn runParallelScan(scan: *ParallelScan) void { const helpers = workerCount(scan.files.len) - 1; var threads: [max_scan_workers]std.Thread = undefined; var started: usize = 0; @@ -383,7 +389,6 @@ fn runParallelScan(scan: *ParallelScan) Allocator.Error!void { } scan.run(); for (threads[0..started]) |thread| thread.join(); - if (scan.out_of_memory) return error.OutOfMemory; } fn selectCandidates(work: Allocator, candidates: []const []const u8, include: ?glob_pattern.Pattern) ![]const []const u8 { @@ -406,8 +411,6 @@ fn initParallelScan( ) !ParallelScan { const outcomes = try work.alloc(FileOutcome, files.len); @memset(outcomes, .{}); - const batch_done = try work.alloc(bool, std.math.divCeil(usize, files.len, batch_files) catch 0); - @memset(batch_done, false); return .{ .work = work, .workspace_root = workspace_root, @@ -417,7 +420,6 @@ fn initParallelScan( .case_insensitive = case_insensitive, .mode = mode, .outcomes = outcomes, - .batch_done = batch_done, }; } @@ -437,8 +439,16 @@ fn scanCandidateList( const work = work_state.allocator(); const files = try selectCandidates(work, candidates, include); var scan = try initParallelScan(work, workspace_root, provider_root, files, pattern, case_insensitive, .matches); - try runParallelScan(&scan); - for (scan.outcomes) |outcome| { + runParallelScan(&scan); + return mergeMatches(arena, &scan, matches); +} + +/// Appends the scanned matches in candidate order up to the collection cap. +/// Running out of memory fails the search only when the cut is not reached +/// before the file where it happened. +fn mergeMatches(arena: Allocator, scan: *const ParallelScan, matches: *std.ArrayList(Match)) Allocator.Error!?TruncatedReason { + for (scan.outcomes, 0..) |outcome, index| { + if (scan.out_of_memory_from) |from| if (index >= from) return error.OutOfMemory; var retained_path: ?[]const u8 = null; for (outcome.matches) |match| { if (matches.items.len >= collection_cap) return .collection_cap; @@ -464,7 +474,9 @@ fn countCandidateList( const work = work_state.allocator(); const files = try selectCandidates(work, candidates, include); var scan = try initParallelScan(work, workspace_root, provider_root, files, pattern, case_insensitive, .count); - try runParallelScan(&scan); + runParallelScan(&scan); + // A count needs every file. + if (scan.out_of_memory_from != null) return error.OutOfMemory; for (scan.outcomes) |outcome| { if (outcome.matching_lines == 0) continue; count_result.matching_files += 1; @@ -1213,3 +1225,51 @@ test "grep search parallel scan matches single-threaded order and cap truncation try std.testing.expectEqual(@as(usize, 1500), counted.matching_files); } } + +test "parallel grep stops once finished batches hold the cap, even with an earlier batch still running" { + var work_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer work_state.deinit(); + const work = work_state.allocator(); + const files = try work.alloc([]const u8, batch_files * 4); + for (files) |*file| file.* = "f"; + var scan = try initParallelScan(work, "/w", "/w", files, "x", false, .matches); + const many = try work.alloc(Match, collection_cap); + for (many) |*match| match.* = .{ .absolute_path = "/w/f", .line_number = 1, .line = "x" }; + + // Batch 0 is still running; batch 1 finishes holding the cap. + scan.outcomes[batch_files] = .{ .matches = many }; + scan.finishBatch(batch_files, batch_files * 2); + try std.testing.expect(scan.stop.load(.acquire)); + + // Below the cap the scan keeps going. + var short = try initParallelScan(work, "/w", "/w", files, "x", false, .matches); + short.outcomes[batch_files] = .{ .matches = many[0 .. collection_cap - 1] }; + short.finishBatch(batch_files, batch_files * 2); + try std.testing.expect(!short.stop.load(.acquire)); +} + +test "grep fails for running out of memory only before the cut" { + var work_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer work_state.deinit(); + const work = work_state.allocator(); + const files = try work.alloc([]const u8, batch_files * 3); + for (files) |*file| file.* = "f"; + const many = try work.alloc(Match, collection_cap); + for (many) |*match| match.* = .{ .absolute_path = "/w/f", .line_number = 1, .line = "x" }; + + // The cut lands in the first file, so a later batch running out of + // memory changes nothing. + var past = try initParallelScan(work, "/w", "/w", files, "x", false, .matches); + past.outcomes[0] = .{ .matches = many }; + past.out_of_memory_from = batch_files * 2; + var kept: std.ArrayList(Match) = .empty; + try std.testing.expectEqual(@as(?TruncatedReason, .collection_cap), try mergeMatches(work, &past, &kept)); + try std.testing.expectEqual(collection_cap, kept.items.len); + + // Before the cut, the missing matches would change the result. + var before = try initParallelScan(work, "/w", "/w", files, "x", false, .matches); + before.outcomes[0] = .{ .matches = many[0..10] }; + before.out_of_memory_from = batch_files; + var partial: std.ArrayList(Match) = .empty; + try std.testing.expectError(error.OutOfMemory, mergeMatches(work, &before, &partial)); +} diff --git a/src/core/workspace/tool_files.zig b/src/core/workspace/tool_files.zig index e465e1d65..b128adee2 100644 --- a/src/core/workspace/tool_files.zig +++ b/src/core/workspace/tool_files.zig @@ -34,6 +34,8 @@ pub const Listing = struct { candidate_cap: usize, /// The cap or an unreadable folder or rule file left files out. incomplete: bool, + /// The candidate cap stopped the listing. + cap_reached: bool, skipped_overlong: usize, }; @@ -107,10 +109,43 @@ fn collect(arena: Allocator, tree: *const indexer.Tree, subtree: []const u8, opt .files = try files.toOwnedSlice(arena), .candidate_cap = options.candidate_cap, .incomplete = tree.incomplete or tree.cap_reached, + .cap_reached = tree.cap_reached, .skipped_overlong = tree.skipped_overlong, }; } +/// What a candidate list may have left out, for the notes a file tool appends. +pub const Completeness = struct { + /// The cap or an unreadable folder or git file left files out. + incomplete: bool = false, + /// The candidate cap stopped the listing. + cap_reached: bool = false, + candidate_cap: usize = default_candidate_cap, + skipped_overlong: usize = 0, + + pub fn of(listing: Listing) Completeness { + return .{ + .incomplete = listing.incomplete, + .cap_reached = listing.cap_reached, + .candidate_cap = listing.candidate_cap, + .skipped_overlong = listing.skipped_overlong, + }; + } + + /// Names the reason files may be missing: the cap, or folders and git + /// files fx could not read. + pub fn writeNotes(self: Completeness, writer: *std.Io.Writer) error{OutOfMemory}!void { + if (self.cap_reached) { + writer.print("... candidate list may be incomplete; candidate cap {d} reached before all files were discovered\n", .{self.candidate_cap}) catch return error.OutOfMemory; + } else if (self.incomplete) { + writer.writeAll("... candidate list may be incomplete; some folders or git files could not be read\n") catch return error.OutOfMemory; + } + if (self.skipped_overlong > 0) { + writer.print("... skipped {d} overlong candidate path{s}\n", .{ self.skipped_overlong, if (self.skipped_overlong == 1) "" else "s" }) catch return error.OutOfMemory; + } + } +}; + /// Whether a folder in `path` (not its last component) is hidden. pub fn pathContainsHiddenDirectoryComponent(path: []const u8) bool { var rest = path; diff --git a/src/tools/filesystem/glob_files.zig b/src/tools/filesystem/glob_files.zig index 22351967e..9da3b19e6 100644 --- a/src/tools/filesystem/glob_files.zig +++ b/src/tools/filesystem/glob_files.zig @@ -124,9 +124,9 @@ fn callWithWorkspaceOptions(ctx: tool_dispatch.DispatchContext, erased: tool_dis requested_root; const root = maybe_root orelse { if (input.mode == .count) { - return .{ .success = try formatCount(ctx.allocator, input.pattern, 0, false, tool_files.default_candidate_cap, 0) }; + return .{ .success = try formatCount(ctx.allocator, input.pattern, 0, .{}) }; } - return .{ .success = try formatMatches(ctx.allocator, input.pattern, &.{}, false, false, tool_files.default_candidate_cap, 0, ctx.max_list_entries) }; + return .{ .success = try formatMatches(ctx.allocator, input.pattern, &.{}, false, .{}, ctx.max_list_entries) }; }; const root_relative = try pathing.workspaceRelativePath(arena, ctx.workspace_root, root.absolute); var compiled_pattern = glob_pattern.Pattern.compile(arena, static_base.pattern) catch |err| switch (err) { @@ -139,9 +139,7 @@ fn callWithWorkspaceOptions(ctx: tool_dispatch.DispatchContext, erased: tool_dis defer matches.deinit(arena); var match_count: usize = 0; var output_truncated = false; - var candidate_incomplete = false; - var candidate_cap: usize = tool_files.default_candidate_cap; - var skipped_overlong: usize = 0; + var completeness: tool_files.Completeness = .{}; var options = list_options; options.skipped_names = ctx.ignored_list_entries; options.include_hidden = options.include_hidden or shouldIncludeHidden(root_relative, input.pattern); @@ -152,9 +150,7 @@ fn callWithWorkspaceOptions(ctx: tool_dispatch.DispatchContext, erased: tool_dis error.FileNotFound => return .{ .failure = try std.fmt.allocPrint(ctx.allocator, "Unable to discover glob candidates: {s} ({s})", .{ root.absolute, @errorName(err) }) }, }; const candidate_files = listing.files; - candidate_incomplete = listing.incomplete; - candidate_cap = listing.candidate_cap; - skipped_overlong = listing.skipped_overlong; + completeness = .of(listing); for (candidate_files) |candidate| { if (!compiled_pattern.matchesPath(candidate)) continue; @@ -180,24 +176,22 @@ fn callWithWorkspaceOptions(ctx: tool_dispatch.DispatchContext, erased: tool_dis } if (input.mode == .count) { - return .{ .success = try formatCount(ctx.allocator, input.pattern, match_count, candidate_incomplete, candidate_cap, skipped_overlong) }; + return .{ .success = try formatCount(ctx.allocator, input.pattern, match_count, completeness) }; } - return .{ .success = try formatMatches(ctx.allocator, input.pattern, matches.items, output_truncated, candidate_incomplete, candidate_cap, skipped_overlong, ctx.max_list_entries) }; + return .{ .success = try formatMatches(ctx.allocator, input.pattern, matches.items, output_truncated, completeness, ctx.max_list_entries) }; } fn formatCount( alloc: Allocator, pattern: []const u8, match_count: usize, - candidate_incomplete: bool, - candidate_cap: usize, - skipped_overlong: usize, + completeness: tool_files.Completeness, ) tool_dispatch.DispatchError![]u8 { var out: std.Io.Writer.Allocating = .init(alloc); defer out.deinit(); out.writer.print("[glob] count {d} matches for {s}\n", .{ match_count, pattern }) catch return error.OutOfMemory; - try writeCandidateNotes(&out.writer, candidate_incomplete, candidate_cap, skipped_overlong); + try completeness.writeNotes(&out.writer); return try out.toOwnedSlice(); } @@ -206,9 +200,7 @@ fn formatMatches( pattern: []const u8, matches: []const []const u8, output_truncated: bool, - candidate_incomplete: bool, - candidate_cap: usize, - skipped_overlong: usize, + completeness: tool_files.Completeness, max_list_entries: usize, ) tool_dispatch.DispatchError![]u8 { var out: std.Io.Writer.Allocating = .init(alloc); @@ -216,7 +208,7 @@ fn formatMatches( if (matches.len == 0) { out.writer.print("[glob] no matches for {s}\n", .{pattern}) catch return error.OutOfMemory; - try writeCandidateNotes(&out.writer, candidate_incomplete, candidate_cap, skipped_overlong); + try completeness.writeNotes(&out.writer); return try out.toOwnedSlice(); } @@ -227,20 +219,11 @@ fn formatMatches( if (output_truncated) { out.writer.print("... truncated to first {d} matches\n", .{max_list_entries}) catch return error.OutOfMemory; } - try writeCandidateNotes(&out.writer, candidate_incomplete, candidate_cap, skipped_overlong); + try completeness.writeNotes(&out.writer); return try out.toOwnedSlice(); } -fn writeCandidateNotes(writer: anytype, candidate_incomplete: bool, candidate_cap: usize, skipped_overlong: usize) tool_dispatch.DispatchError!void { - if (candidate_incomplete) { - writer.print("... candidate list may be incomplete; candidate cap {d} reached before all files were discovered\n", .{candidate_cap}) catch return error.OutOfMemory; - } - if (skipped_overlong > 0) { - writer.print("... skipped {d} overlong candidate path{s}\n", .{ skipped_overlong, if (skipped_overlong == 1) "" else "s" }) catch return error.OutOfMemory; - } -} - const SearchRoot = struct { absolute: []const u8, is_directory: bool, @@ -725,7 +708,7 @@ test "glob_files count mode reports exact matches without rendering entries" { test "glob_files formatter uses active truncation wording" { const alloc = std.testing.allocator; const matches = [_][]const u8{ "a.txt", "b.txt" }; - const body = try formatMatches(alloc, "*.txt", &matches, true, false, tool_files.default_candidate_cap, 0, 2); + const body = try formatMatches(alloc, "*.txt", &matches, true, .{}, 2); defer alloc.free(body); try std.testing.expectEqualStrings("[glob] 2 matches for *.txt\n - a.txt\n - b.txt\n... truncated to first 2 matches\n", body); @@ -860,7 +843,7 @@ test "glob_files finds match beyond former traversal cap" { test "glob_files distinguishes candidate cap from output cap" { const alloc = std.testing.allocator; const matches = [_][]const u8{ "a.txt", "b.txt" }; - const body = try formatMatches(alloc, "*.txt", &matches, true, true, 2, 0, 2); + const body = try formatMatches(alloc, "*.txt", &matches, true, .{ .incomplete = true, .cap_reached = true, .candidate_cap = 2 }, 2); defer alloc.free(body); try std.testing.expect(std.mem.find(u8, body, "... truncated to first 2 matches\n") != null); diff --git a/src/tools/filesystem/grep_files.zig b/src/tools/filesystem/grep_files.zig index 8c0aaf38c..c64175e9b 100644 --- a/src/tools/filesystem/grep_files.zig +++ b/src/tools/filesystem/grep_files.zig @@ -1,6 +1,7 @@ const std = @import("std"); const glob_pattern = @import("../../core/workspace/glob_pattern.zig"); const grep_search = @import("../../core/workspace/grep_search.zig"); +const tool_files = @import("../../core/workspace/tool_files.zig"); const io_mod = @import("../../core/shared/io.zig"); const mem_utils = @import("../../core/shared/mem_utils.zig"); const pathing = @import("../../core/workspace/pathing.zig"); @@ -498,26 +499,17 @@ fn writeGrepResultNotes( .collection_cap => writer.print("... match collection cap reached at {d} matches before all candidate files were scanned\n", .{grep_search.collection_cap}) catch return error.OutOfMemory, } } - try writeCandidateNotes( - writer, - result.candidate_incomplete, - result.candidate_cap, - result.skipped_overlong, - ); + try candidateCompleteness(result).writeNotes(writer); } -fn writeCandidateNotes( - writer: *std.Io.Writer, - candidate_incomplete: bool, - candidate_cap: usize, - skipped_overlong: usize, -) error{OutOfMemory}!void { - if (candidate_incomplete) { - writer.print("... candidate list may be incomplete; candidate cap {d} reached before all files were discovered\n", .{candidate_cap}) catch return error.OutOfMemory; - } - if (skipped_overlong > 0) { - writer.print("... skipped {d} overlong candidate path{s}\n", .{ skipped_overlong, if (skipped_overlong == 1) "" else "s" }) catch return error.OutOfMemory; - } +/// The candidate fields of a `grep_search.Result` or `CountResult`. +fn candidateCompleteness(result: anytype) tool_files.Completeness { + return .{ + .incomplete = result.candidate_incomplete, + .cap_reached = result.candidate_cap_reached, + .candidate_cap = result.candidate_cap, + .skipped_overlong = result.skipped_overlong, + }; } fn formatCount( @@ -529,12 +521,7 @@ fn formatCount( defer out.deinit(); out.writer.print("[grep] count {d} matching lines in {d} files for {s}\n", .{ result.matching_lines, result.matching_files, pattern }) catch return error.OutOfMemory; - try writeCandidateNotes( - &out.writer, - result.candidate_incomplete, - result.candidate_cap, - result.skipped_overlong, - ); + try candidateCompleteness(result).writeNotes(&out.writer); return try out.toOwnedSlice(); } @@ -1424,11 +1411,22 @@ test "grep_files formatter reports candidate cap without output truncation wordi const body = try formatMatches(alloc, alloc, "", "needle", .{ .matches = &.{}, .candidate_incomplete = true, + .candidate_cap_reached = true, .candidate_cap = 2, }, output_cap, 0, 0, tool_dispatch.default_max_list_entries, 2000); defer alloc.free(body); try std.testing.expectEqualStrings("[grep] no matches for needle\n... candidate list may be incomplete; candidate cap 2 reached before all files were discovered\n", body); + + // A listing left incomplete by unreadable folders or git files says so, + // not that the cap was reached. + const unreadable = try formatMatches(alloc, alloc, "", "needle", .{ + .matches = &.{}, + .candidate_incomplete = true, + .candidate_cap = 100000, + }, output_cap, 0, 0, tool_dispatch.default_max_list_entries, 2000); + defer alloc.free(unreadable); + try std.testing.expectEqualStrings("[grep] no matches for needle\n... candidate list may be incomplete; some folders or git files could not be read\n", unreadable); } test "grep_files classifiers are read-only and reversible" { From 6aa678d9d6cb7504674128f64d68008461e5fb98 Mon Sep 17 00:00:00 2001 From: Pranit Date: Sun, 11 Oct 2026 05:36:38 -0400 Subject: [PATCH 17/23] Keep the @ folder wait after a slow folder - Check back every millisecond for the first tick of every folder listing, including ones that skip the wait, so a fast listing after a slow folder is seen in time and the wait turns back on. Before, one slow folder brought back the "indexing files..." flash for the rest of that menu session. --- src/core/app/input_completion_runtime.zig | 20 +++++++++++++++++--- src/core/input/picker_state.zig | 13 +++++++++++-- 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/core/app/input_completion_runtime.zig b/src/core/app/input_completion_runtime.zig index 34426f17d..e8776a163 100644 --- a/src/core/app/input_completion_runtime.zig +++ b/src/core/app/input_completion_runtime.zig @@ -798,11 +798,13 @@ pub fn CompletionRuntime(comptime App: type) type { return hold; } - /// While a frame waits for a folder listing, the event loop checks back - /// every millisecond instead of every tick. + /// For the first tick of every folder listing, the event loop checks + /// back every millisecond instead of every tick. A listing that + /// finishes in that time is seen at once, which also turns waiting + /// back on after a slow folder. pub fn fileListingHoldPending(app: *const App, now_ms: i64) bool { if (comptime !runtime_profile.allows(App, .file_index)) return false; - return app.input_runtime.picker.file_listing_wait.active(now_ms); + return app.input_runtime.picker.file_listing_wait.measuring(now_ms); } /// The sole acquisition boundary. Called after input batching, never by @@ -2176,6 +2178,18 @@ test "filesystem file picker releases a held folder frame only with its rows" { try std.testing.expect(!rt.fileListingHoldPending(&app, now_ms)); } +test "file picker keeps watching a listing after a slow folder so waiting can turn back on" { + const rt = CompletionRuntime(FilePickerTestApp); + var app: FilePickerTestApp = .{ .alloc = std.testing.allocator }; + defer app.deinit(); + + // The previous folder outlasted the wait, so this listing's frame paints + // at once, but the loop still checks back every millisecond. + app.input_runtime.picker.file_listing_wait = .{ .started_ms = 1_000, .last_outlasted_hold = true }; + try std.testing.expect(!app.input_runtime.picker.file_listing_wait.active(1_001)); + try std.testing.expect(rt.fileListingHoldPending(&app, 1_001)); +} + test "file picker loading Tab preserves pending request and Enter never queues acceptance" { const alloc = std.testing.allocator; const rt = CompletionRuntime(FilePickerTestApp); diff --git a/src/core/input/picker_state.zig b/src/core/input/picker_state.zig index 9e68340b1..a52adefbb 100644 --- a/src/core/input/picker_state.zig +++ b/src/core/input/picker_state.zig @@ -112,8 +112,14 @@ const FileListingWait = struct { } pub fn active(self: FileListingWait, now_ms: i64) bool { + return !self.last_outlasted_hold and self.measuring(now_ms); + } + + /// True during the first tick of every listing, waited for or not, so a + /// listing that finishes in time is seen at once and waiting turns back on. + pub fn measuring(self: FileListingWait, now_ms: i64) bool { const started = self.started_ms orelse return false; - return !self.last_outlasted_hold and withinHold(started, now_ms); + return withinHold(started, now_ms); } fn withinHold(started_ms: i64, now_ms: i64) bool { @@ -1060,9 +1066,12 @@ test "folder listing wait spans one tick and is skipped after a slow listing" { try std.testing.expect(!wait.hold(true, 140)); try std.testing.expect(!wait.active(140)); try std.testing.expect(!wait.hold(false, 141)); - // The folder was slow, so the next keystroke does not wait for it. + // The folder was slow, so the next keystroke does not wait for it, but + // the listing is still watched for its first tick. try std.testing.expect(!wait.hold(true, 200)); try std.testing.expect(!wait.active(200)); + try std.testing.expect(wait.measuring(200 + FileListingWait.hold_ms - 1)); + try std.testing.expect(!wait.measuring(200 + FileListingWait.hold_ms)); // A listing that finishes within the wait turns waiting back on. try std.testing.expect(!wait.hold(false, 202)); try std.testing.expect(wait.hold(true, 300)); From 32a9a2c52a6d07b1a8ed573ecd11f68777c8425a Mon Sep 17 00:00:00 2001 From: Pranit Date: Sun, 11 Oct 2026 05:40:18 -0400 Subject: [PATCH 18/23] Word the git hold message for every refusal reason - Say that fx could not run git with the repository's configured programs turned off, which is also true when git is missing from the fixed path or too old, instead of blaming the repository's config. --- src/core/tooling/tool_admission.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/core/tooling/tool_admission.zig b/src/core/tooling/tool_admission.zig index 8c5d3964b..abc705208 100644 --- a/src/core/tooling/tool_admission.zig +++ b/src/core/tooling/tool_admission.zig @@ -2118,8 +2118,8 @@ fn defaultRunCommandPermissionOutcome( } const unsafe_git_hold_output = - "fx did not run this git command. It could not confirm that this repository's git config is safe, " ++ - "and git could start programs that config names. Inspect files with read_file, glob_files or grep_files, " ++ + "fx did not run this git command. It could not run git here with every program this repository's config " ++ + "names turned off, and plain git could start them. Inspect files with read_file, glob_files or grep_files, " ++ "or ask the user to run git themselves."; pub fn runCommandContext( From 31f018c407d3ee071182827fed1c3dd58f494655 Mon Sep 17 00:00:00 2001 From: Pranit Date: Sun, 11 Oct 2026 06:10:28 -0400 Subject: [PATCH 19/23] Count both new E2E files in the PGSO corpus test - This branch and main each added one verification E2E file and raised the expected candidate count from 59 to 60. With both merged there are 61. --- scripts/pgso/tests/test_corpus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/pgso/tests/test_corpus.py b/scripts/pgso/tests/test_corpus.py index 207acac62..383ccc40f 100644 --- a/scripts/pgso/tests/test_corpus.py +++ b/scripts/pgso/tests/test_corpus.py @@ -374,7 +374,7 @@ class PgsoCorpusTests(unittest.TestCase): tuple(test_file for test_file, _ in corpus.intentional_exclusions), ) self.assertEqual(35, len(corpus.scenarios)) - self.assertEqual(60, len(corpus.candidate_scenarios)) + self.assertEqual(61, len(corpus.candidate_scenarios)) self.assertEqual( { "direct-help": 100, From ddf8e11ec1b5e277c542b4b0b4e327b8177b640b Mon Sep 17 00:00:00 2001 From: Pranit Date: Sun, 11 Oct 2026 22:45:43 -0400 Subject: [PATCH 20/23] Count only files toward the candidate cap The scan cut its sorted list at 100,000 entries, files and folder rows together. On the Linux kernel (96,055 files, 6,281 folders) glob_files and grep_files never saw 2,101 files and reported no matches for code that exists. The cap now counts files, as the 100,000-file limit says. The collection limit still bounds folder rows, and the @ menu keeps its own cap on rows. A folder search also reused a saved listing cut at the cap, so after an interactive session a search of a folder past the cut found nothing. A cut listing now answers only a search of the whole workspace with the same cap; any other search scans its root. --- src/core/indexer/scan.zig | 59 ++++++++++++++++++++++++++----- src/core/workspace/tool_files.zig | 41 ++++++++++++++++++--- 2 files changed, 88 insertions(+), 12 deletions(-) diff --git a/src/core/indexer/scan.zig b/src/core/indexer/scan.zig index a53fd63ee..e2aa0f4fa 100644 --- a/src/core/indexer/scan.zig +++ b/src/core/indexer/scan.zig @@ -29,7 +29,8 @@ pub const Kind = tree_mod.Kind; pub const Entry = tree_mod.Entry; pub const Options = struct { - /// D7: at most this many entries are returned. + /// D7: at most this many files are returned. Folder rows do not count; + /// the collection limit (`collect_factor`) bounds them. candidate_cap: usize = 100_000, max_path_bytes: usize = 2048, /// Folder and file names skipped outside repositories and below an @@ -258,8 +259,8 @@ pub fn scan(alloc: Allocator, root: []const u8, options: Options, stop: ?*std.at try assemble(arena, listings, rules.repository, &entries); std.mem.sortUnstable(Entry, entries.items, {}, entryLessThan); var cap_reached = false; - if (entries.items.len > options.candidate_cap) { - entries.shrinkRetainingCapacity(options.candidate_cap); + if (capCut(entries.items, options.candidate_cap)) |len| { + entries.shrinkRetainingCapacity(len); cap_reached = true; } if (shared.limit_reached.load(.acquire)) cap_reached = true; @@ -598,6 +599,39 @@ fn entryLessThan(_: void, a: Entry, b: Entry) bool { }; } +/// Length of the sorted prefix that holds the first `cap` files, or null when +/// `entries` has no more than `cap` files. Folder rows do not count (D7). +fn capCut(entries: []const Entry, cap: usize) ?usize { + var files: usize = 0; + for (entries, 0..) |entry, index| { + if (entry.kind != .file) continue; + if (files == cap) return index; + files += 1; + } + return null; +} + +fn testFileCount(entries: []const Entry) usize { + var files: usize = 0; + for (entries) |entry| files += @intFromBool(entry.kind == .file); + return files; +} + +test "the cap counts files and keeps the folder rows before the next file" { + const entries = [_]Entry{ + .{ .path = "a", .kind = .directory }, + .{ .path = "a/1", .kind = .file }, + .{ .path = "b", .kind = .directory }, + .{ .path = "b/2", .kind = .file }, + .{ .path = "c", .kind = .directory }, + }; + try std.testing.expectEqual(@as(?usize, 1), capCut(&entries, 0)); + try std.testing.expectEqual(@as(?usize, 3), capCut(&entries, 1)); + try std.testing.expectEqual(@as(?usize, null), capCut(&entries, 2)); + try std.testing.expectEqual(@as(?usize, null), capCut(&entries, 3)); + try std.testing.expectEqual(@as(?usize, null), capCut(entries[4..], 0)); +} + const TestTree = struct { tmp: std.testing.TmpDir, root: []const u8, @@ -774,18 +808,27 @@ test "scan caps, skips overlong paths, cancels and is the same with one worker o try std.testing.expect(entryLessThan({}, earlier, later)); } - // Within the collection limit the cap keeps the sorted prefix. - var capped = try scan(std.testing.allocator, root, .{ .candidate_cap = 200 }, null); + // The cap counts files only: 180 files and 240 folder rows fit a cap of + // 180, though the entries outnumber it. + var exact = try scan(std.testing.allocator, root, .{ .candidate_cap = 180 }, null); + defer exact.deinit(); + try std.testing.expect(!exact.tree.incomplete and !exact.tree.cap_reached); + try std.testing.expectEqual(serial.tree.entries.len, exact.tree.entries.len); + // Within the collection limit the cap keeps the sorted prefix that holds + // the first files, ending just before the next file. + var capped = try scan(std.testing.allocator, root, .{ .candidate_cap = 120 }, null); defer capped.deinit(); // Below it the walk read everything, so only the cap was reached. try std.testing.expect(!capped.tree.incomplete and capped.tree.cap_reached); - try std.testing.expectEqual(@as(usize, 200), capped.tree.entries.len); - for (capped.tree.entries, serial.tree.entries[0..200]) |a, b| try std.testing.expectEqualStrings(b.path, a.path); + try std.testing.expectEqual(@as(usize, 120), testFileCount(capped.tree.entries)); + const kept = capped.tree.entries.len; + for (capped.tree.entries, serial.tree.entries[0..kept]) |a, b| try std.testing.expectEqualStrings(b.path, a.path); + try std.testing.expectEqual(Kind.file, serial.tree.entries[kept].kind); // Past it, reading stops early: still capped, sorted and incomplete. var tiny = try scan(std.testing.allocator, root, .{ .candidate_cap = 10 }, null); defer tiny.deinit(); try std.testing.expect(tiny.tree.incomplete and tiny.tree.cap_reached); - try std.testing.expectEqual(@as(usize, 10), tiny.tree.entries.len); + try std.testing.expect(testFileCount(tiny.tree.entries) <= 10); for (tiny.tree.entries[1..], tiny.tree.entries[0 .. tiny.tree.entries.len - 1]) |later, earlier| try std.testing.expect(entryLessThan({}, earlier, later)); var short = try scan(std.testing.allocator, root, .{ .max_path_bytes = 6 }, null); diff --git a/src/core/workspace/tool_files.zig b/src/core/workspace/tool_files.zig index b128adee2..f344865b5 100644 --- a/src/core/workspace/tool_files.zig +++ b/src/core/workspace/tool_files.zig @@ -1,7 +1,9 @@ //! Files under a search root for `glob_files` and `grep_files`, listed by the -//! native indexer and current at the time of the call. A search of the whole -//! workspace reuses the saved `@`-completion snapshot when every folder and -//! rule file it depended on is unchanged; any other search scans its root. +//! native indexer and current at the time of the call. A search of the +//! workspace or of an ordinary folder in it reuses the saved `@`-completion +//! snapshot when the folders and rule files it depended on are unchanged; a +//! folder search never reuses a snapshot cut at the candidate cap. Any other +//! search scans its root. //! Nothing here saves a snapshot, so searches outside the workspace leave no //! trace under `~/.fx/file-index/`. @@ -80,13 +82,24 @@ fn subtreeFromSnapshot(tree: *const indexer.Tree, subtree: []const u8) bool { return false; } +/// Whether the cap lets `tree` answer a search of `subtree`. A tree cut at +/// the cap holds only the sorted prefix with its first files: a folder past +/// the cut lost its files, and another cap would cut elsewhere. Only a search +/// of the whole workspace with the same cap gets that list from a new scan. +fn capAllowsReuse(tree: *const indexer.Tree, subtree: []const u8, candidate_cap: usize) bool { + var files: usize = 0; + for (tree.entries) |entry| files += @intFromBool(entry.kind == .file); + if (files > candidate_cap) return false; + return !tree.cap_reached or (subtree.len == 0 and files == candidate_cap); +} + fn listFromSnapshot(alloc: Allocator, arena: Allocator, workspace_root: []const u8, subtree: []const u8, options: Options) Allocator.Error!?Listing { const roots = [_][]const u8{workspace_root}; const path = (try file_index.snapshotPath(arena, &roots)) orelse return null; var snapshot = (try indexer.loadSnapshot(alloc, path, &roots)) orelse return null; defer snapshot.deinit(); const tree = &snapshot.trees[0]; - if (tree.entries.len > options.candidate_cap) return null; + if (!capAllowsReuse(tree, subtree, options.candidate_cap)) return null; if (!subtreeFromSnapshot(tree, subtree)) return null; if (!indexer.isCurrent(tree, workspace_root, options.skipped_names, if (subtree.len == 0) null else subtree)) return null; debug_trace.logf("core", "tool files reused snapshot entries={d} subtree_bytes={d}", .{ tree.entries.len, subtree.len }); @@ -217,3 +230,23 @@ test "subfolder searches reuse the workspace tree only for ordinary folders" { try std.testing.expect(relativeInside("/w", "/workspace") == null); try std.testing.expect(relativeInside("/w", "/other") == null); } + +test "a tree cut at the cap answers only a whole-workspace search with the same cap" { + const entries = [_]indexer.Entry{ + .{ .path = "src", .kind = .directory }, + .{ .path = "src/a.zig", .kind = .file }, + .{ .path = "src/b.zig", .kind = .file }, + .{ .path = "tools", .kind = .directory }, + }; + var tree: indexer.Tree = .{ .root = "/w", .root_inode = 1, .scan_started_ns = 0, .repository = true, .incomplete = false, .cap_reached = false, .skipped_overlong = 0, .skipped_names = &.{}, .entries = &entries, .folders = &.{}, .sources = &.{} }; + // Folder rows do not count: two files fit a cap of two. + try std.testing.expect(capAllowsReuse(&tree, "", 2)); + try std.testing.expect(capAllowsReuse(&tree, "src", 2)); + try std.testing.expect(!capAllowsReuse(&tree, "", 1)); + + tree.cap_reached = true; + try std.testing.expect(capAllowsReuse(&tree, "", 2)); + try std.testing.expect(!capAllowsReuse(&tree, "", 3)); + try std.testing.expect(!capAllowsReuse(&tree, "src", 2)); + try std.testing.expect(!capAllowsReuse(&tree, "tools", 2)); +} From f62e85ce49d3463bc5ddfdcf83a145dd44817de0 Mon Sep 17 00:00:00 2001 From: Pranit Date: Mon, 12 Oct 2026 01:23:44 -0400 Subject: [PATCH 21/23] Check the folders above a searched folder before reusing the listing A folder search reused the saved @ listing when that folder and the ones below it were unchanged. A .gitignore created above it, at the root or in a parent folder that had none, changes only that folder's stamp, so glob_files and grep_files kept returning files git now ignores. The check now covers every folder above the searched one too. A new entry in one of them makes the search scan fresh; changes elsewhere in the workspace still leave the saved listing in use. --- src/core/indexer/scan.zig | 24 ++++++++++++++++++++++++ src/core/indexer/tree.zig | 25 ++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/core/indexer/scan.zig b/src/core/indexer/scan.zig index e2aa0f4fa..2e3aadaa9 100644 --- a/src/core/indexer/scan.zig +++ b/src/core/indexer/scan.zig @@ -878,6 +878,13 @@ test "refresh trusts an old unchanged tree and rejects every kind of change" { } const future: std.Io.Timestamp = .{ .nanoseconds = @as(i96, tree_mod.nowNs()) + 3600 * std.time.ns_per_s }; try tree.tmp.dir.setTimestamps(std.testing.io, "future/docs", .{ .modify_timestamp = .{ .new = future } }); + // Folders above a searched one that have no `.gitignore` yet. + const first_rules = [_][]const u8{ "root_rules", "parent_rules" }; + for (first_rules) |name| { + try tree.repo(arena, name, &.{.{ .path = "a/b/x.txt" }}); + try tree.file(try std.fmt.allocPrint(arena, "{s}/a/b/x.txt", .{name}), "x"); + try tree.file(try std.fmt.allocPrint(arena, "{s}/a/b/y.secret", .{name}), "y"); + } // A tree scanned right after its files changed is never reused. { @@ -895,6 +902,14 @@ test "refresh trusts an old unchanged tree and rejects every kind of change" { results[i] = try scan(std.testing.allocator, roots[i], .{}, null); } defer for (&results) |*result| result.deinit(); + var rule_results: [first_rules.len]Result = undefined; + var rule_roots: [first_rules.len][]const u8 = undefined; + for (first_rules, 0..) |name, i| { + rule_roots[i] = try std.fs.path.join(arena, &.{ tree.root, name }); + rule_results[i] = try scan(std.testing.allocator, rule_roots[i], .{}, null); + } + defer for (&rule_results) |*result| result.deinit(); + for (&rule_results, rule_roots) |*result, root| try std.testing.expect(tree_mod.isCurrent(&result.tree, root, &default_skipped_names, "a/b")); try std.testing.expect(tree_mod.isCurrent(&results[0].tree, roots[0], &default_skipped_names, null)); try std.testing.expect(!tree_mod.isCurrent(&results[0].tree, roots[0], &.{"node_modules"}, null)); @@ -908,6 +923,15 @@ test "refresh trusts an old unchanged tree and rejects every kind of change" { // A change in `src` leaves a check of `docs` alone. try std.testing.expect(tree_mod.isCurrent(&results[1].tree, roots[1], &default_skipped_names, "docs")); + // A first `.gitignore` above a searched folder makes it stale. + try tree.file("root_rules/.gitignore", "*.secret\n"); + try tree.file("parent_rules/a/.gitignore", "*.secret\n"); + for (&rule_results, rule_roots, first_rules) |*result, root, name| { + if (tree_mod.isCurrent(&result.tree, root, &default_skipped_names, "a/b")) { + std.debug.print("subtree reported current after a new .gitignore above it: {s}\n", .{name}); + return error.TestExpectedStale; + } + } for (results[1..], roots[1..], cases[1..]) |*result, root, name| { if (tree_mod.isCurrent(&result.tree, root, &default_skipped_names, null)) { std.debug.print("stale tree reported current: {s}\n", .{name}); diff --git a/src/core/indexer/tree.zig b/src/core/indexer/tree.zig index 9f3b7d1af..ca8dd8b45 100644 --- a/src/core/indexer/tree.zig +++ b/src/core/indexer/tree.zig @@ -128,7 +128,9 @@ fn sameSource(recorded: SourceStamp, now: SourceStamp) bool { /// Whether `tree` still describes `root` under the same options, so it can be /// answered from without a new scan. With `subtree` (relative to the root), -/// only the folders at or below it are checked; every source always is. +/// only the folders at, below or above it are checked; every source always +/// is. A folder above it counts because a `.gitignore` it gains changes what +/// the subtree lists, and only that folder's stamp shows the new file. pub fn isCurrent(tree: *const Tree, root: []const u8, skipped_names: []const []const u8, subtree: ?[]const u8) bool { if (tree.incomplete or !std.mem.eql(u8, tree.root, root)) return false; if (tree.skipped_names.len != skipped_names.len) return false; @@ -148,8 +150,7 @@ pub fn isCurrent(tree: *const Tree, root: []const u8, skipped_names: []const []c } for (tree.folders) |folder| { if (subtree) |sub| { - if (sub.len > 0 and !(std.mem.eql(u8, folder.rel, sub) or - (folder.rel.len > sub.len and std.mem.startsWith(u8, folder.rel, sub) and folder.rel[sub.len] == '/'))) continue; + if (sub.len > 0 and !onSubtreePath(folder.rel, sub)) continue; } if (!folder.reusable) return false; const stat = root_dir.statFile(io, if (folder.rel.len == 0) "." else folder.rel, .{ .follow_symlinks = false }) catch return false; @@ -161,6 +162,24 @@ pub fn isCurrent(tree: *const Tree, root: []const u8, skipped_names: []const []c return true; } +/// Whether folder `rel` is `sub`, below it, or above it (the root included). +fn onSubtreePath(rel: []const u8, sub: []const u8) bool { + if (rel.len == 0 or std.mem.eql(u8, rel, sub)) return true; + if (rel.len > sub.len) return std.mem.startsWith(u8, rel, sub) and rel[sub.len] == '/'; + return std.mem.startsWith(u8, sub, rel) and sub[rel.len] == '/'; +} + +test "a subtree check covers the folder, the folders below it and the folders above it" { + try std.testing.expect(onSubtreePath("", "a/b")); + try std.testing.expect(onSubtreePath("a", "a/b")); + try std.testing.expect(onSubtreePath("a/b", "a/b")); + try std.testing.expect(onSubtreePath("a/b/c", "a/b")); + try std.testing.expect(!onSubtreePath("a/bc", "a/b")); + try std.testing.expect(!onSubtreePath("ab", "a/b")); + try std.testing.expect(!onSubtreePath("a/c", "a/b")); + try std.testing.expect(!onSubtreePath("b", "a/b")); +} + test "stamps reuse only times older than the scan start by the margin" { const start: i64 = 100 * std.time.ns_per_s; try std.testing.expect(reusableTimes(start - 3 * std.time.ns_per_s, start - 3 * std.time.ns_per_s, start)); From 541a13469ed04d40ab58c596832cf2b7cc2c774b Mon Sep 17 00:00:00 2001 From: Pranit Date: Mon, 12 Oct 2026 06:25:12 -0400 Subject: [PATCH 22/23] Free each file search's scan memory when the search ends glob_files and grep_files ran the workspace scan on the dispatch allocator, which fx sets to the turn's arena. Every search kept its whole scan until the turn ended, so one turn with 16 root globs on this repository peaked at 707 MiB. The tools now scan with DispatchContext.scratch_allocator, whose frees return memory, and only the result body comes from the turn's allocator. The same turn now peaks at 50 MiB. The scan also holds less while it runs. Paths are joined with one exact allocation, and inside the repository root a path doubles as its rule path. A folder's rule prefix is built only when it has a .gitignore, small files are read into a buffer sized from their stat, and lists are reserved at their final size. Rules, listings and their bookkeeping live in an arena freed when the scan returns, and the index bytes come from the scan's allocator so they do not inflate later arena blocks. On the Linux kernel a scan peaks at 40 MiB instead of 79 MiB, and the tree the @ menu keeps drops from 58 MiB to 9 MiB. scan() copied its result arena before allocating the root path, so a block added by that allocation could leak. The copy now comes last. Saved listings for added roots now drop the same malformed paths as the workspace root's. --- src/core/indexer/bounded_read.zig | 52 +++++++++++++ src/core/indexer/git_index.zig | 50 ++++++++++++- src/core/indexer/scan.zig | 112 ++++++++++++++++++---------- src/core/indexer/sources.zig | 13 ++-- src/core/tooling/tool_dispatch.zig | 4 + src/core/workspace/file_index.zig | 48 +++++++++--- src/core/workspace/tool_files.zig | 28 +++++-- src/tools/filesystem/glob_files.zig | 42 ++++++++++- src/tools/filesystem/grep_files.zig | 47 +++++++++++- 9 files changed, 323 insertions(+), 73 deletions(-) diff --git a/src/core/indexer/bounded_read.zig b/src/core/indexer/bounded_read.zig index 548232d8e..b391bf26d 100644 --- a/src/core/indexer/bounded_read.zig +++ b/src/core/indexer/bounded_read.zig @@ -4,6 +4,7 @@ //! truncated, so a caller never acts on a partial file. const std = @import("std"); +const builtin = @import("builtin"); const io_mod = @import("../shared/io.zig"); const Allocator = std.mem.Allocator; @@ -48,6 +49,8 @@ fn readWith( else => .{ .unavailable = @errorName(err) }, }; defer file.close(io_mod.getIo()); + if (try readSized(alloc, file, max_bytes)) |outcome| return outcome; + // The file grew after the stat, or reports no size: read it as a stream. // The reader reports a stream that fills its limit as too long, so allow // one extra byte to accept a file of exactly `max_bytes`. const data = io_mod.readFileToEnd(alloc, &file, max_bytes +| 1) catch |err| return switch (err) { @@ -58,6 +61,28 @@ fn readWith( return .{ .content = data }; } +/// Reads the whole file into a buffer sized from its stat, so a small file +/// does not claim a stream reader's growth buffer. Returns null when the file +/// holds more than its stat said, which the caller reads as a stream. +fn readSized(alloc: Allocator, file: std.Io.File, max_bytes: usize) Allocator.Error!?Outcome { + const io = io_mod.getIo(); + const stat = file.stat(io) catch |err| return .{ .unavailable = @errorName(err) }; + if (stat.size > max_bytes) return .{ .unavailable = "too_large" }; + // `stat.size <= max_bytes`, so it fits in a usize. + const size: usize = @intCast(stat.size); + // One byte past the size shows whether the file grew since the stat. + const buffer = try alloc.alloc(u8, size + 1); + const read = file.readPositionalAll(io, buffer, 0) catch |err| { + alloc.free(buffer); + return .{ .unavailable = @errorName(err) }; + }; + if (read > size) { + alloc.free(buffer); + return null; + } + return .{ .content = try alloc.realloc(buffer, read) }; +} + test "bounded reads return content, missing, oversize and non-regular outcomes" { const alloc = std.testing.allocator; var tmp = std.testing.tmpDir(.{}); @@ -77,6 +102,33 @@ test "bounded reads return content, missing, oversize and non-regular outcomes" try std.testing.expect(try readAbsolute(alloc, "relative", 16) == .unavailable); } +test "bounded reads allocate the file's size, not a stream buffer" { + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "small", .data = "0123456789" }); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "empty", .data = "" }); + var counting = std.testing.FailingAllocator.init(std.testing.allocator, .{}); + const alloc = counting.allocator(); + + const small = try readAt(alloc, tmp.dir, "small", 1024 * 1024); + defer alloc.free(small.content); + try std.testing.expectEqualStrings("0123456789", small.content); + try std.testing.expect(counting.allocated_bytes <= 64); + + const empty = try readAt(alloc, tmp.dir, "empty", 1024 * 1024); + defer alloc.free(empty.content); + try std.testing.expectEqual(@as(usize, 0), empty.content.len); +} + +test "bounded reads stream a regular file whose stat reports no size" { + // Linux reports 0 bytes for the regular files under /proc. + if (builtin.target.os.tag != .linux) return error.SkipZigTest; + const alloc = std.testing.allocator; + const status = try readUserAbsolute(alloc, "/proc/self/status", 1024 * 1024); + defer alloc.free(status.content); + try std.testing.expect(std.mem.startsWith(u8, status.content, "Name:")); +} + test "user reads follow a final symlink to a regular file and still refuse the rest" { const alloc = std.testing.allocator; var tmp = std.testing.tmpDir(.{}); diff --git a/src/core/indexer/git_index.zig b/src/core/indexer/git_index.zig index aa5341cf8..122a2d127 100644 --- a/src/core/indexer/git_index.zig +++ b/src/core/indexer/git_index.zig @@ -5,6 +5,7 @@ const std = @import("std"); const bounded_read = @import("bounded_read.zig"); +const io_mod = @import("../shared/io.zig"); const Allocator = std.mem.Allocator; @@ -23,6 +24,8 @@ pub const Entry = struct { pub const Index = struct { /// In index order (sorted by path). Unmerged paths appear once. entries: []const Entry, + /// The file's bytes when `read` loaded them, which entry paths may borrow. + data: []const u8 = &.{}, }; pub const Outcome = union(enum) { @@ -35,14 +38,27 @@ pub const Outcome = union(enum) { pub const HashFormat = enum { sha1, sha256 }; -/// Reads the index at the absolute `path`. Memory belongs to `arena`. -pub fn read(arena: Allocator, path: []const u8, hash: HashFormat) Allocator.Error!Outcome { - const data = switch (try bounded_read.readAbsolute(arena, path, max_index_bytes)) { +/// Reads the index at the absolute `path`. The file's bytes come from +/// `data_alloc` and are returned as `Index.data`, which the entries may +/// borrow; the caller frees it once it is done with the entries. Everything +/// else belongs to `arena`. +pub fn read(arena: Allocator, data_alloc: Allocator, path: []const u8, hash: HashFormat) Allocator.Error!Outcome { + const data = switch (try bounded_read.readAbsolute(data_alloc, path, max_index_bytes)) { .content => |bytes| bytes, .missing => return .missing, .unavailable => |reason| return .{ .unavailable = reason }, }; - return parse(arena, data, hash); + const outcome = parse(arena, data, hash) catch |err| { + data_alloc.free(data); + return err; + }; + switch (outcome) { + .index => |index| return .{ .index = .{ .entries = index.entries, .data = data } }, + .missing, .unavailable => { + data_alloc.free(data); + return outcome; + }, + } } const Reader = struct { @@ -245,6 +261,32 @@ test "index reader handles versions 2 to 4, both hash formats, gitlinks and unme } } +test "read leaves the index bytes to the caller and frees them when the index is unusable" { + const alloc = std.testing.allocator; + var arena_state = std.heap.ArenaAllocator.init(alloc); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const good = try buildForTest(arena, 2, .sha1, &.{.{ .path = "a.txt" }}, ""); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "good", .data = good }); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = "bad", .data = "not an index" }); + const root = try io_mod.dirRealpathAlloc(arena, tmp.dir, "."); + + // The entries borrow `data`, which stays the caller's to free. + const outcome = try read(arena, alloc, try std.Io.Dir.path.join(arena, &.{ root, "good" }), .sha1); + const index = switch (outcome) { + .index => |value| value, + else => return error.TestExpectedIndex, + }; + defer alloc.free(index.data); + try expectPaths(outcome, &.{"a.txt"}, &.{}); + try std.testing.expectEqual(good.len, index.data.len); + + // An unusable index frees its bytes before returning. + try std.testing.expect(try read(arena, alloc, try std.Io.Dir.path.join(arena, &.{ root, "bad" }), .sha1) == .unavailable); +} + test "index reader rejects truncated, hostile and unsupported indexes" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); diff --git a/src/core/indexer/scan.zig b/src/core/indexer/scan.zig index 84b65625d..a8b7569b1 100644 --- a/src/core/indexer/scan.zig +++ b/src/core/indexer/scan.zig @@ -69,6 +69,10 @@ const Rules = struct { tracked_files: std.StringHashMapUnmanaged(void) = .empty, tracked_dirs: std.StringHashMapUnmanaged(void) = .empty, gitlinks: std.StringHashMapUnmanaged(void) = .empty, + /// The index file's bytes, which the three sets above borrow. Owned by + /// the scan's allocator rather than an arena, since a large index would + /// otherwise grow every later arena block. + index_bytes: []const u8 = &.{}, skipped_names: []const []const u8, max_path_bytes: usize, /// Files the rules read or looked for, stamped once the walk ends. @@ -198,6 +202,11 @@ pub fn scan(alloc: Allocator, root: []const u8, options: Options, stop: ?*std.at var result_arena = std.heap.ArenaAllocator.init(alloc); errdefer result_arena.deinit(); const arena = result_arena.allocator(); + // The rules, the listings and their bookkeeping matter only while + // scanning; the tree keeps just what it returns. + var scratch_arena = std.heap.ArenaAllocator.init(alloc); + defer scratch_arena.deinit(); + const scratch = scratch_arena.allocator(); if (!std.Io.Dir.path.isAbsolute(root)) return error.RootUnavailable; const io = io_mod.getIo(); var root_dir = std.Io.Dir.openDirAbsolute(io, root, .{ .iterate = true }) catch return error.RootUnavailable; @@ -211,9 +220,10 @@ pub fn scan(alloc: Allocator, root: []const u8, options: Options, stop: ?*std.at .skipped_names = options.skipped_names, .max_path_bytes = options.max_path_bytes, }; + defer alloc.free(rules.index_bytes); var incomplete = false; var root_lists: []const *const ignore.PatternList = &.{}; - if (try repositoryRules(arena, root, &rules, &incomplete)) |lists| root_lists = lists; + if (try repositoryRules(scratch, alloc, root, &rules, &incomplete)) |lists| root_lists = lists; var shared: Shared = .{ .alloc = alloc, @@ -238,7 +248,10 @@ pub fn scan(alloc: Allocator, root: []const u8, options: Options, stop: ?*std.at const workers = try alloc.alloc(Worker, worker_count); defer alloc.free(workers); for (workers) |*worker| worker.* = .{ .arena = std.heap.ArenaAllocator.init(alloc) }; - defer for (workers) |*worker| worker.arena.deinit(); + defer for (workers) |*worker| { + worker.listings.deinit(alloc); + worker.arena.deinit(); + }; runMain(&shared, workers); if (shared.failure) |err| return err; @@ -246,7 +259,7 @@ pub fn scan(alloc: Allocator, root: []const u8, options: Options, stop: ?*std.at var listing_count: usize = 0; for (workers) |*worker| listing_count += worker.listings.items.len; - const listings = try arena.alloc(*const Listing, listing_count); + const listings = try scratch.alloc(*const Listing, listing_count); var cursor: usize = 0; for (workers) |*worker| { for (worker.listings.items) |*listing| { @@ -256,7 +269,7 @@ pub fn scan(alloc: Allocator, root: []const u8, options: Options, stop: ?*std.at } var entries: std.ArrayList(Entry) = .empty; - try assemble(arena, listings, rules.repository, &entries); + try assemble(arena, scratch, listings, rules.repository, &entries); std.mem.sortUnstable(Entry, entries.items, {}, entryLessThan); var cap_reached = false; if (capCut(entries.items, options.candidate_cap)) |len| { @@ -267,20 +280,24 @@ pub fn scan(alloc: Allocator, root: []const u8, options: Options, stop: ?*std.at incomplete = incomplete or shared.limit_reached.load(.acquire) or shared.incomplete.load(.acquire); var folders: std.ArrayList(tree_mod.FolderStamp) = .empty; + try folders.ensureTotalCapacityPrecise(arena, listings.len); for (listings) |listing| { const stamp = listing.stamp orelse continue; - try folders.append(arena, .{ .rel = try arena.dupe(u8, stamp.rel), .inode = stamp.inode, .mtime_ns = stamp.mtime_ns, .ctime_ns = stamp.ctime_ns, .reusable = stamp.reusable, .boundary = listing.boundary, .excluded = listing.excluded }); - if (listing.ignore_file) |path| try rules.sources.append(arena, .{ .path = try arena.dupe(u8, path), .identity_only = false }); + folders.appendAssumeCapacity(.{ .rel = try arena.dupe(u8, stamp.rel), .inode = stamp.inode, .mtime_ns = stamp.mtime_ns, .ctime_ns = stamp.ctime_ns, .reusable = stamp.reusable, .boundary = listing.boundary, .excluded = listing.excluded }); + if (listing.ignore_file) |path| try rules.sources.append(scratch, .{ .path = path, .identity_only = false }); } const stamps = try arena.alloc(tree_mod.SourceStamp, rules.sources.items.len); - for (rules.sources.items, stamps) |request, *stamp| stamp.* = tree_mod.sourceStamp(request.path, request.identity_only, request.follow_symlinks, scan_started_ns); + for (rules.sources.items, stamps) |request, *stamp| stamp.* = tree_mod.sourceStamp(try arena.dupe(u8, request.path), request.identity_only, request.follow_symlinks, scan_started_ns); const skipped_names = try arena.alloc([]const u8, options.skipped_names.len); for (options.skipped_names, skipped_names) |name, *copy| copy.* = try arena.dupe(u8, name); + // Every allocation comes before `result_arena` is copied into the result: + // a block the arena adds after the copy would be missing from it and leak. + const root_copy = try arena.dupe(u8, root); return .{ .arena = result_arena, .tree = .{ - .root = try arena.dupe(u8, root), + .root = root_copy, .root_inode = @intCast(root_stat.inode), .scan_started_ns = scan_started_ns, .repository = rules.repository, @@ -297,7 +314,8 @@ pub fn scan(alloc: Allocator, root: []const u8, options: Options, stop: ?*std.at /// Fills in repository rules when `root` is inside a repository and is not /// itself ignored, and returns the ignore lists that apply above the root. -fn repositoryRules(arena: Allocator, root: []const u8, rules: *Rules, incomplete: *bool) Allocator.Error!?[]const *const ignore.PatternList { +/// The index's bytes come from `index_alloc` and are left in `rules`. +fn repositoryRules(arena: Allocator, index_alloc: Allocator, root: []const u8, rules: *Rules, incomplete: *bool) Allocator.Error!?[]const *const ignore.PatternList { const maybe_found = layout_mod.discover(arena, root) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.InvalidGitFile => blk: { @@ -344,9 +362,8 @@ fn repositoryRules(arena: Allocator, root: []const u8, rules: *Rules, incomplete continue; }; defer dir.close(io_mod.getIo()); - const base = if (rule_dir.len == 0) "" else try arena.print("{s}/", .{rule_dir}); try rules.sources.append(arena, .{ .path = try std.Io.Dir.path.join(arena, &.{ absolute, ".gitignore" }), .identity_only = false }); - switch (try sources.loadDirectory(arena, dir, base)) { + switch (try sources.loadDirectory(arena, dir, rule_dir)) { .list => |list| try lists.append(arena, list), .absent => {}, .unavailable => incomplete.* = true, @@ -364,24 +381,27 @@ fn repositoryRules(arena: Allocator, root: []const u8, rules: *Rules, incomplete rules.prefix = found.prefix; const index_path = try std.Io.Dir.path.join(arena, &.{ found.git_dir, "index" }); const hash: git_index.HashFormat = if (settings.object_format == .sha256) .sha256 else .sha1; - switch (try git_index.read(arena, index_path, hash)) { + switch (try git_index.read(arena, index_alloc, index_path, hash)) { .missing => {}, .unavailable => |reason| { debug_trace.logf("indexer", "scan index unavailable reason={s} root={s}", .{ reason, root }); incomplete.* = true; }, - .index => |index| for (index.entries) |entry| { - if (entry.gitlink) { - try rules.gitlinks.put(arena, entry.path, {}); - } else { - try rules.tracked_files.put(arena, entry.path, {}); - } - var slash = std.mem.findScalarLast(u8, entry.path, '/'); - while (slash) |end| { - const parent = entry.path[0..end]; - const slot = try rules.tracked_dirs.getOrPut(arena, parent); - if (slot.found_existing) break; - slash = std.mem.findScalarLast(u8, parent, '/'); + .index => |index| { + rules.index_bytes = index.data; + for (index.entries) |entry| { + if (entry.gitlink) { + try rules.gitlinks.put(arena, entry.path, {}); + } else { + try rules.tracked_files.put(arena, entry.path, {}); + } + var slash = std.mem.findScalarLast(u8, entry.path, '/'); + while (slash) |end| { + const parent = entry.path[0..end]; + const slot = try rules.tracked_dirs.getOrPut(arena, parent); + if (slot.found_existing) break; + slash = std.mem.findScalarLast(u8, parent, '/'); + } } }, } @@ -423,7 +443,7 @@ fn listDirectory(shared: *Shared, worker: *Worker, job: Job, children: *std.Arra const io = io_mod.getIo(); const rules = shared.rules; var listing: Listing = .{ .rel = job.rel, .tracked = job.tracked, .excluded = job.excluded }; - defer worker.listings.append(arena, listing) catch shared.fail(error.OutOfMemory); + defer worker.listings.append(shared.alloc, listing) catch shared.fail(error.OutOfMemory); if (shared.canceled()) return error.Canceled; if (shared.limit_reached.load(.acquire)) return; @@ -441,8 +461,7 @@ fn listDirectory(shared: *Shared, worker: *Worker, job: Job, children: *std.Arra // Below an excluded folder everything is excluded, as in git, which stops // reading `.gitignore` files there. if (!job.excluded) { - const base = if (job.rule_path.len == 0) "" else try arena.print("{s}/", .{job.rule_path}); - const loaded = try sources.loadDirectory(arena, dir, base); + const loaded = try sources.loadDirectory(arena, dir, job.rule_path); if (loaded != .absent) listing.ignore_file = try std.Io.Dir.path.join(arena, &.{ shared.root_abs, job.rel, ".gitignore" }); switch (loaded) { .list => |list| { @@ -459,6 +478,8 @@ fn listDirectory(shared: *Shared, worker: *Worker, job: Job, children: *std.Arra } } + // At and below the repository root the rule path is the same string. + const shares_rule_path = !rules.repository or std.mem.eql(u8, job.rule_path, job.rel); var files: std.ArrayList([]const u8) = .empty; var child_entries: std.ArrayList(Child) = .empty; var saw_dot_git = false; @@ -478,7 +499,7 @@ fn listDirectory(shared: *Shared, worker: *Worker, job: Job, children: *std.Arra if (!rules.repository and rules.skipped(entry.name)) continue; const rel = try joinPath(arena, job.rel, entry.name); - const rule_path = if (rules.repository) try joinPath(arena, job.rule_path, entry.name) else rel; + const rule_path = if (shares_rule_path) rel else try joinPath(arena, job.rule_path, entry.name); // A path over the limit is not listed, but it still exists and is not // ignored, so its folder stays visible. const overlong = rel.len > rules.max_path_bytes; @@ -545,24 +566,25 @@ fn excludedPath(lists: []const *const ignore.PatternList, path: []const u8, is_d fn joinPath(arena: Allocator, parent: []const u8, name: []const u8) Allocator.Error![]const u8 { if (parent.len == 0) return arena.dupe(u8, name); - return arena.print("{s}/{s}", .{ parent, name }); + return std.mem.concat(arena, u8, &.{ parent, "/", name }); } /// Applies git's folder rules bottom-up and gathers the visible entries. -fn assemble(arena: Allocator, listings: []const *const Listing, repository: bool, out: *std.ArrayList(Entry)) Allocator.Error!void { +/// `out` and its paths use `arena`; the bookkeeping uses `scratch`. +fn assemble(arena: Allocator, scratch: Allocator, listings: []const *const Listing, repository: bool, out: *std.ArrayList(Entry)) Allocator.Error!void { var by_path: std.StringHashMapUnmanaged(usize) = .empty; - try by_path.ensureTotalCapacity(arena, @intCast(listings.len)); + try by_path.ensureTotalCapacity(scratch, @intCast(listings.len)); for (listings, 0..) |listing, index| by_path.putAssumeCapacity(listing.rel, index); // Deeper folders first, so each folder sees its children's visibility. - const order = try arena.alloc(usize, listings.len); + const order = try scratch.alloc(usize, listings.len); for (order, 0..) |*slot, index| slot.* = index; std.mem.sortUnstable(usize, order, listings, struct { fn deeper(items: []const *const Listing, a: usize, b: usize) bool { return std.mem.count(u8, items[a].rel, "/") > std.mem.count(u8, items[b].rel, "/"); } }.deeper); - const visible = try arena.alloc(bool, listings.len); + const visible = try scratch.alloc(bool, listings.len); for (order) |index| { const listing = listings[index]; visible[index] = if (!repository or listing.tracked or listing.boundary or !listing.has_entries) true else blk: { @@ -578,19 +600,31 @@ fn assemble(arena: Allocator, listings: []const *const Listing, repository: bool }; } + // Reserve exactly once: growing the list inside the arena would leave every + // smaller copy behind. + var count: usize = 0; for (listings, 0..) |listing, index| { if (!visible[index]) continue; - for (listing.files) |path| try out.append(arena, .{ .path = try arena.dupe(u8, path), .kind = .file }); + count += listing.files.len; + for (listing.children) |child| count += @intFromBool(childShown(child, &by_path, visible)); + } + try out.ensureTotalCapacityPrecise(arena, out.items.len + count); + for (listings, 0..) |listing, index| { + if (!visible[index]) continue; + for (listing.files) |path| out.appendAssumeCapacity(.{ .path = try arena.dupe(u8, path), .kind = .file }); for (listing.children) |child| { - const shown = switch (child.state) { - .boundary => true, - .entered => if (by_path.get(child.rel)) |child_index| visible[child_index] else true, - }; - if (shown) try out.append(arena, .{ .path = try arena.dupe(u8, child.rel), .kind = .directory }); + if (childShown(child, &by_path, visible)) out.appendAssumeCapacity(.{ .path = try arena.dupe(u8, child.rel), .kind = .directory }); } } } +fn childShown(child: Child, by_path: *const std.StringHashMapUnmanaged(usize), visible: []const bool) bool { + return switch (child.state) { + .boundary => true, + .entered => if (by_path.get(child.rel)) |child_index| visible[child_index] else true, + }; +} + fn entryLessThan(_: void, a: Entry, b: Entry) bool { return switch (std.mem.order(u8, a.path, b.path)) { .lt => true, diff --git a/src/core/indexer/sources.zig b/src/core/indexer/sources.zig index 65208978f..1353892f8 100644 --- a/src/core/indexer/sources.zig +++ b/src/core/indexer/sources.zig @@ -66,11 +66,14 @@ pub const DirectoryList = union(enum) { }; /// Loads `.gitignore` from the open directory `dir`, whose path relative to -/// the worktree root is `base` (empty, or ending in `/`). A symlinked -/// `.gitignore` is not followed, as in git. Memory belongs to `arena`. -pub fn loadDirectory(arena: Allocator, dir: std.Io.Dir, base: []const u8) Allocator.Error!DirectoryList { +/// the worktree root is `rule_dir` (empty at the root, no trailing `/`). A +/// symlinked `.gitignore` is not followed, as in git. Memory belongs to +/// `arena`. +pub fn loadDirectory(arena: Allocator, dir: std.Io.Dir, rule_dir: []const u8) Allocator.Error!DirectoryList { return switch (try bounded_read.readAt(arena, dir, ".gitignore", ignore.max_pattern_file_bytes)) { .content => |content| blk: { + // Built only here, since most folders have no `.gitignore`. + const base = if (rule_dir.len == 0) "" else try std.mem.concat(arena, u8, &.{ rule_dir, "/" }); const list = try arena.create(ignore.PatternList); list.* = try ignore.PatternList.parse(arena, base, content); break :blk .{ .list = list }; @@ -142,7 +145,7 @@ test "directory .gitignore loads with its base and refuses symlinks" { try tmp.dir.createDirPath(std.testing.io, "linked"); var sub = try tmp.dir.openDir(std.testing.io, "sub", .{}); defer sub.close(std.testing.io); - const loaded = try loadDirectory(arena, sub, "sub/"); + const loaded = try loadDirectory(arena, sub, "sub"); const lists = [_]*const ignore.PatternList{loaded.list}; try std.testing.expectEqual(ignore.Decision.excluded, ignore.decide(&lists, "sub/local", false, false)); try std.testing.expectEqual(ignore.Decision.none, ignore.decide(&lists, "local", false, false)); @@ -151,5 +154,5 @@ test "directory .gitignore loads with its base and refuses symlinks" { var linked = try tmp.dir.openDir(std.testing.io, "linked", .{}); defer linked.close(std.testing.io); linked.symLink(std.testing.io, "../target", ".gitignore", .{}) catch return error.SkipZigTest; - try std.testing.expect(try loadDirectory(arena, linked, "linked/") == .unavailable); + try std.testing.expect(try loadDirectory(arena, linked, "linked") == .unavailable); } diff --git a/src/core/tooling/tool_dispatch.zig b/src/core/tooling/tool_dispatch.zig index dac7e371e..d507dc799 100644 --- a/src/core/tooling/tool_dispatch.zig +++ b/src/core/tooling/tool_dispatch.zig @@ -283,6 +283,10 @@ pub const DispatchContext = struct { terminal_owner_session_id: ?[]const u8 = null, terminal_transport_role: terminal_contracts.TransportRole = .interactive, lifecycle_allocator: Allocator = std.heap.c_allocator, + /// Thread-safe allocator whose frees return memory, for work that must not + /// outlive one call, such as a workspace scan. `allocator` is often the + /// turn's arena, which keeps everything until the turn ends. + scratch_allocator: Allocator = std.heap.c_allocator, command_timeout_ms: ?usize = null, captured_command_host: command_environment.Host = .native, run_command_backend: ?RunCommandBackend = null, diff --git a/src/core/workspace/file_index.zig b/src/core/workspace/file_index.zig index b80fff576..09a6dc407 100644 --- a/src/core/workspace/file_index.zig +++ b/src/core/workspace/file_index.zig @@ -969,7 +969,7 @@ fn candidatesFromTrees( for (tree.entries) |entry| { if (isStopRequested(stop_requested)) return error.Canceled; if (candidates.items.len >= max_indexed_files) return candidates.toOwnedSlice(arena); - if (hasGitComponent(entry.path)) continue; + if (!isPlainRelativePath(entry.path)) continue; if (!text_utils.isTerminalSafe(entry.path)) { debug_trace.logf("core", "file index omitted unsafe candidate bytes={d} kind={s}", .{ entry.path.len, @tagName(entry.kind) }); continue; @@ -1022,14 +1022,6 @@ fn isPlainRelativePath(path: []const u8) bool { return true; } -fn hasGitComponent(path: []const u8) bool { - var parts = std.mem.splitScalar(u8, path, '/'); - while (parts.next()) |part| { - if (std.mem.eql(u8, part, ".git")) return true; - } - return false; -} - /// Publishes the loader's terminal outcome only after its caller has released /// every worker-owned temporary. The release store is deliberately the final /// action so an acquire observer may join without waiting for worker cleanup. @@ -1932,6 +1924,42 @@ test "tree candidates keep a sorted directory at the exact shared cap" { try std.testing.expectEqualStrings("z-file-099998", candidates[max_indexed_files - 1].path); } +test "added-root candidates drop the same paths as primary-root ones" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + const primary = [_]indexer.Entry{.{ .path = "main.zig", .kind = .file }}; + // Only a damaged snapshot holds such paths; a scan never produces them. + const added = [_]indexer.Entry{ + .{ .path = ".git/config", .kind = .file }, + .{ .path = "a/../../outside", .kind = .file }, + .{ .path = "a//b", .kind = .file }, + .{ .path = "./a", .kind = .file }, + .{ .path = "ok.txt", .kind = .file }, + }; + var trees: [2]indexer.Tree = undefined; + for (&trees, [_][]const u8{ "/workspace", "/extra" }, [_][]const indexer.Entry{ &primary, &added }) |*tree, root, entries| { + tree.* = .{ + .root = root, + .root_inode = 1, + .scan_started_ns = 0, + .repository = true, + .incomplete = false, + .cap_reached = false, + .skipped_overlong = 0, + .skipped_names = &.{}, + .entries = entries, + .folders = &.{}, + .sources = &.{}, + }; + } + var stop_requested = std.atomic.Value(bool).init(false); + const candidates = try candidatesFromTrees(arena, &.{ "/workspace", "/extra" }, &trees, &stop_requested); + try std.testing.expectEqual(@as(usize, 2), candidates.len); + try std.testing.expectEqualStrings("main.zig", candidates[0].path); + try std.testing.expectEqualStrings("/extra/ok.txt", candidates[1].path); +} + test "primary tree candidates keep plain paths and drop git, escaping and unsafe ones" { var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_state.deinit(); @@ -2863,7 +2891,7 @@ test "production scope admits tracked untracked hidden and direct directory cand try std.testing.expect(!containsCandidate(candidates, "ignored-dir", .directory)); try std.testing.expect(containsCandidate(candidates, "empty-dir", .directory)); for (candidates) |candidate| { - try std.testing.expect(!hasGitComponent(candidate.path)); + try std.testing.expect(isPlainRelativePath(candidate.path)); } var index = FileIndex{}; diff --git a/src/core/workspace/tool_files.zig b/src/core/workspace/tool_files.zig index 5521468f4..89833c363 100644 --- a/src/core/workspace/tool_files.zig +++ b/src/core/workspace/tool_files.zig @@ -106,20 +106,32 @@ fn listFromSnapshot(alloc: Allocator, arena: Allocator, workspace_root: []const return try collect(arena, tree, subtree, options); } +/// The path of a file entry relative to `subtree`, or null when the entry is +/// not a file or is outside the subtree. +fn subtreePath(entry: indexer.Entry, subtree: []const u8) ?[]const u8 { + if (entry.kind != .file) return null; + if (subtree.len == 0) return entry.path; + if (entry.path.len <= subtree.len + 1 or !std.mem.startsWith(u8, entry.path, subtree) or entry.path[subtree.len] != '/') return null; + return entry.path[subtree.len + 1 ..]; +} + fn collect(arena: Allocator, tree: *const indexer.Tree, subtree: []const u8, options: Options) Allocator.Error!Listing { const keep_hidden = options.include_hidden or (options.include_hidden_in_repository and tree.repository); - var files: std.ArrayList([]const u8) = .empty; + // Reserve once for every file in the subtree, so the list never grows. + // Only the cheap checks run here; hidden paths are left out while filling. + var upper: usize = 0; + for (tree.entries) |entry| upper += @intFromBool(subtreePath(entry, subtree) != null); + const files = try arena.alloc([]const u8, upper); + var count: usize = 0; for (tree.entries) |entry| { - if (entry.kind != .file) continue; - const path = if (subtree.len == 0) entry.path else blk: { - if (entry.path.len <= subtree.len + 1 or !std.mem.startsWith(u8, entry.path, subtree) or entry.path[subtree.len] != '/') continue; - break :blk entry.path[subtree.len + 1 ..]; - }; + const path = subtreePath(entry, subtree) orelse continue; if (!keep_hidden and pathContainsHiddenDirectoryComponent(path)) continue; - try files.append(arena, try arena.dupe(u8, path)); + files[count] = try arena.dupe(u8, path); + count += 1; } return .{ - .files = try files.toOwnedSlice(arena), + // The unused tail belongs to the arena like the rest of the listing. + .files = files[0..count], .candidate_cap = options.candidate_cap, .incomplete = tree.incomplete or tree.cap_reached, .cap_reached = tree.cap_reached, diff --git a/src/tools/filesystem/glob_files.zig b/src/tools/filesystem/glob_files.zig index 7e011ec83..50c5a3273 100644 --- a/src/tools/filesystem/glob_files.zig +++ b/src/tools/filesystem/glob_files.zig @@ -97,7 +97,10 @@ pub fn call(ctx: tool_dispatch.DispatchContext, erased: tool_dispatch.ToolInput) fn callWithWorkspaceOptions(ctx: tool_dispatch.DispatchContext, erased: tool_dispatch.ToolInput, list_options: tool_files.Options) tool_dispatch.DispatchError!tool_dispatch.ToolResult { const input = erased.as(Input); - var arena_state = std.heap.ArenaAllocator.init(ctx.allocator); + // The scan and the per-call work use the scratch allocator, so each search + // returns its memory when it ends; only the result body comes from + // `ctx.allocator`. + var arena_state = std.heap.ArenaAllocator.init(ctx.scratch_allocator); defer mem_utils.deinit_arena(arena_state); const arena = arena_state.allocator(); @@ -145,7 +148,7 @@ fn callWithWorkspaceOptions(ctx: tool_dispatch.DispatchContext, erased: tool_dis options.include_hidden = options.include_hidden or shouldIncludeHidden(root_relative, input.pattern); if (root.is_directory) { - const listing = tool_files.list(ctx.allocator, arena, ctx.workspace_root, root.absolute, options) catch |err| switch (err) { + const listing = tool_files.list(ctx.scratch_allocator, arena, ctx.workspace_root, root.absolute, options) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.FileNotFound => return .{ .failure = try ctx.allocator.print("Unable to discover glob candidates: {s} ({s})", .{ root.absolute, @errorName(err) }) }, }; @@ -401,6 +404,7 @@ fn dispatchGlobFilesRaw(alloc: Allocator, workspace_root: []const u8, args_json: const registry = tool_dispatch.Registry{ .tools = &.{glob_files_dispatch_tool} }; return tool_dispatch.dispatchToolCall(.{ .allocator = alloc, + .scratch_allocator = alloc, .permission_mode = .auto, .workspace_root = workspace_root, }, registry, .{ @@ -516,7 +520,7 @@ test "glob_files reports overlong patterns during matching" { }; defer input.deinit(alloc); - const result = try call(.{ .allocator = alloc, .workspace_root = workspace }, .{ .ptr = &input, .deinit_fn = noopInputDeinit }); + const result = try call(.{ .allocator = alloc, .scratch_allocator = alloc, .workspace_root = workspace }, .{ .ptr = &input, .deinit_fn = noopInputDeinit }); defer result.deinit(alloc); const expected = try alloc.print("glob_files field \"pattern\" must be at most {d} bytes", .{glob_pattern.max_pattern_bytes}); defer alloc.free(expected); @@ -734,6 +738,7 @@ test "glob_files path narrowing applies before candidate cap" { const erased = tool_dispatch.ToolInput{ .ptr = &input, .deinit_fn = noopInputDeinit }; var result = try callWithWorkspaceOptions(.{ .allocator = alloc, + .scratch_allocator = alloc, .permission_mode = .auto, .workspace_root = workspace, .max_list_entries = 10, @@ -770,6 +775,7 @@ test "glob_files extracts static base before candidate cap" { const erased = tool_dispatch.ToolInput{ .ptr = &input, .deinit_fn = noopInputDeinit }; var result = try callWithWorkspaceOptions(.{ .allocator = alloc, + .scratch_allocator = alloc, .permission_mode = .auto, .workspace_root = workspace, .max_list_entries = 10, @@ -839,6 +845,36 @@ test "glob_files finds match beyond former traversal cap" { try std.testing.expect(std.mem.find(u8, result.body, "traversal cap") == null); } +test "glob_files keeps only its result in the turn's allocator" { + // fx passes the turn's arena as `allocator`, so a scan left there stays + // until the turn ends and every search adds another. + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const workspace = try io_mod.dirRealpathAlloc(alloc, tmp.dir, "."); + defer alloc.free(workspace); + try createNumberedFiles(&tmp, 2050); + const args = try globArgsJson(alloc, "**/file-0001.txt", null); + defer alloc.free(args); + + var turn_state = std.heap.ArenaAllocator.init(alloc); + defer turn_state.deinit(); + const registry = tool_dispatch.Registry{ .tools = &.{glob_files_dispatch_tool} }; + for (0..2) |_| { + const before = turn_state.queryCapacity(); + const result = try tool_dispatch.dispatchToolCall(.{ + .allocator = turn_state.allocator(), + .scratch_allocator = alloc, + .permission_mode = .auto, + .workspace_root = workspace, + }, registry, .{ .id = "call_1", .name = "glob_files", .arguments_json = args }); + try std.testing.expectEqual(.success, result.status); + try std.testing.expect(std.mem.find(u8, result.body, "many/file-0001.txt") != null); + // Listing 2,050 files takes far more than this; the result does not. + try std.testing.expect(turn_state.queryCapacity() - before < 64 * 1024); + } +} + test "glob_files distinguishes candidate cap from output cap" { const alloc = std.testing.allocator; const matches = [_][]const u8{ "a.txt", "b.txt" }; diff --git a/src/tools/filesystem/grep_files.zig b/src/tools/filesystem/grep_files.zig index f8a25d61b..c0a661e02 100644 --- a/src/tools/filesystem/grep_files.zig +++ b/src/tools/filesystem/grep_files.zig @@ -223,7 +223,10 @@ const SearchOutcome = union(enum) { fn callWithOps(ctx: tool_dispatch.DispatchContext, erased: tool_dispatch.ToolInput, ops: CallOps) tool_dispatch.DispatchError!tool_dispatch.ToolResult { const input = erased.as(Input); - var arena_state = std.heap.ArenaAllocator.init(ctx.allocator); + // The scan and the per-call work use the scratch allocator, so each search + // returns its memory when it ends; only the result body comes from + // `ctx.allocator`. + var arena_state = std.heap.ArenaAllocator.init(ctx.scratch_allocator); defer mem_utils.deinit_arena(arena_state); const arena = arena_state.allocator(); @@ -272,10 +275,10 @@ fn searchGrepRoot( ) tool_dispatch.DispatchError!SearchOutcome { return switch (root_stat.kind) { .directory => switch (input.mode) { - .count => .{ .count = ops.count_directory(ctx.allocator, arena, ctx.workspace_root, absolute_root, input.pattern, input.case_insensitive, ctx.ignored_list_entries, include_pattern) catch |err| { + .count => .{ .count = ops.count_directory(ctx.scratch_allocator, arena, ctx.workspace_root, absolute_root, input.pattern, input.case_insensitive, ctx.ignored_list_entries, include_pattern) catch |err| { return searchFailure(ctx.allocator, absolute_root, "walk grep search root", err); } }, - .matches, .files_with_matches => .{ .matches = ops.collect_directory(ctx.allocator, arena, ctx.workspace_root, absolute_root, input.pattern, input.case_insensitive, ctx.ignored_list_entries, include_pattern) catch |err| { + .matches, .files_with_matches => .{ .matches = ops.collect_directory(ctx.scratch_allocator, arena, ctx.workspace_root, absolute_root, input.pattern, input.case_insensitive, ctx.ignored_list_entries, include_pattern) catch |err| { return searchFailure(ctx.allocator, absolute_root, "walk grep search root", err); } }, }, @@ -667,6 +670,7 @@ fn dispatchGrepFilesRaw(alloc: Allocator, workspace_root: []const u8, args_json: const registry = tool_dispatch.Registry{ .tools = &.{grep_files_dispatch_tool} }; return tool_dispatch.dispatchToolCall(.{ .allocator = alloc, + .scratch_allocator = alloc, .permission_mode = .auto, .workspace_root = workspace_root, }, registry, .{ @@ -676,6 +680,41 @@ fn dispatchGrepFilesRaw(alloc: Allocator, workspace_root: []const u8, args_json: }); } +test "grep_files keeps only its result in the turn's allocator" { + // fx passes the turn's arena as `allocator`, so a scan left there stays + // until the turn ends and every search adds another. + const alloc = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const workspace = try workspaceRoot(alloc, tmp); + defer alloc.free(workspace); + try tmp.dir.createDirPath(io_mod.getIo(), "many"); + for (0..2050) |i| { + var name_buf: [64]u8 = undefined; + const name = try std.mem.print(&name_buf, "many/file-{d:0>4}.txt", .{i}); + try tmp.dir.writeFile(std.testing.io, .{ .sub_path = name, .data = if (i == 1) "needle\n" else "x\n" }); + } + const args = try grepArgsJson(alloc, "needle", null, null, null, null, null); + defer alloc.free(args); + + var turn_state = std.heap.ArenaAllocator.init(alloc); + defer turn_state.deinit(); + const registry = tool_dispatch.Registry{ .tools = &.{grep_files_dispatch_tool} }; + for (0..2) |_| { + const before = turn_state.queryCapacity(); + const result = try tool_dispatch.dispatchToolCall(.{ + .allocator = turn_state.allocator(), + .scratch_allocator = alloc, + .permission_mode = .auto, + .workspace_root = workspace, + }, registry, .{ .id = "call_1", .name = "grep_files", .arguments_json = args }); + try std.testing.expectEqual(.success, result.status); + try std.testing.expect(std.mem.find(u8, result.body, "many/file-0001.txt") != null); + // Searching 2,050 files takes far more than this; the result does not. + try std.testing.expect(turn_state.queryCapacity() - before < 64 * 1024); + } +} + fn writeTempFile(alloc: Allocator, tmp: *std.testing.TmpDir, sub_path: []const u8, content: []const u8) ![]u8 { if (std.Io.Dir.path.dirname(sub_path)) |parent| { try tmp.dir.createDirPath(io_mod.getIo(), parent); @@ -728,7 +767,7 @@ fn grepFilesFailureWithOps( }; defer input.deinit(alloc); - const result = try callWithOps(.{ .allocator = alloc, .workspace_root = workspace_root }, .{ .ptr = &input, .deinit_fn = noopInputDeinit }, ops); + const result = try callWithOps(.{ .allocator = alloc, .scratch_allocator = alloc, .workspace_root = workspace_root }, .{ .ptr = &input, .deinit_fn = noopInputDeinit }, ops); switch (result) { .rich => return error.TestUnexpectedRichResult, .failure => |body| return body, From 5493dd5d5e603d4d883ab4e234fbe2e4b9983f55 Mon Sep 17 00:00:00 2001 From: Pranit Date: Mon, 12 Oct 2026 06:52:12 -0400 Subject: [PATCH 23/23] Read a file with no stat size as a stream without trusting a size When a file holds more than its stat said, because it grew or because its file system reports no size as Linux does for /proc, the bounded reader fell back to a positional reader. That reader stops at the stat size, so such a file came back empty. The fallback now reads the file as a stream, which never trusts a size. --- src/core/indexer/bounded_read.zig | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/core/indexer/bounded_read.zig b/src/core/indexer/bounded_read.zig index b391bf26d..76c787f09 100644 --- a/src/core/indexer/bounded_read.zig +++ b/src/core/indexer/bounded_read.zig @@ -44,21 +44,27 @@ fn readWith( max_bytes: usize, final_symlink: io_mod.FinalSymlinkPolicy, ) Allocator.Error!Outcome { - var file = io_mod.openExistingReadOnlyRegularFile(dir, sub_path, final_symlink) catch |err| return switch (err) { + const file = io_mod.openExistingReadOnlyRegularFile(dir, sub_path, final_symlink) catch |err| return switch (err) { error.FileNotFound, error.NotDir => .missing, else => .{ .unavailable = @errorName(err) }, }; defer file.close(io_mod.getIo()); if (try readSized(alloc, file, max_bytes)) |outcome| return outcome; - // The file grew after the stat, or reports no size: read it as a stream. - // The reader reports a stream that fills its limit as too long, so allow - // one extra byte to accept a file of exactly `max_bytes`. - const data = io_mod.readFileToEnd(alloc, &file, max_bytes +| 1) catch |err| return switch (err) { + // The file holds more than its stat said: it grew, or its file system + // reports no size, as Linux does for /proc. A positional reader would stop + // at the stat size, so read it as a stream, which never trusts one. + var read_buffer: [8192]u8 = undefined; + var reader = file.readerStreaming(io_mod.getIo(), &read_buffer); + var content: std.ArrayList(u8) = .empty; + defer content.deinit(alloc); + // A stream that fills its limit reads as too long, so allow one extra byte + // to accept a file of exactly `max_bytes`. + reader.interface.appendRemaining(alloc, &content, .limited(max_bytes +| 1)) catch |err| return switch (err) { error.OutOfMemory => error.OutOfMemory, error.StreamTooLong => .{ .unavailable = "too_large" }, - else => .{ .unavailable = @errorName(err) }, + error.ReadFailed => .{ .unavailable = if (reader.err) |read_err| @errorName(read_err) else @errorName(err) }, }; - return .{ .content = data }; + return .{ .content = try content.toOwnedSlice(alloc) }; } /// Reads the whole file into a buffer sized from its stat, so a small file